File size: 3,406 Bytes
59839e3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
#!/usr/bin/env python3
"""Select v802 component weights + decoder on dev seeds; confirm on check seeds.

Components are OOF files saved by the v802 experiments as
artifacts/models/v802_trials/oof/<prefix>_seed<seed>.npz (key "probs"),
plus the special component "v801" (re-run v801 legacy+kNN mix).
"""

from __future__ import annotations

import argparse
import itertools
import json
from pathlib import Path

import numpy as np

from v802_common import (
    CHECK_SEEDS, DEV_SEEDS, ROOT, dump, evaluate, load_data, make_folds, summarize, v801_oof,
)

TRIALS = ROOT / "artifacts" / "models" / "v802_trials"


def component(data, prefix: str, seed: int) -> np.ndarray:
    if prefix == "v801":
        base = v801_oof(data, seed, TRIALS / "cache")
        return 0.8 * base["legacy"] + 0.2 * base["knn"]
    stored = np.load(TRIALS / "oof" / f"{prefix}_seed{seed}.npz")
    assert (stored["names"] == data.names[data.target_idx]).all()
    return stored["probs"]


def weight_grid(count: int, step: float) -> list[tuple[float, ...]]:
    ticks = np.round(np.arange(0, 1 + 1e-9, step), 3)
    return [w for w in itertools.product(ticks, repeat=count) if abs(sum(w) - 1) < 1e-6]


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--components", nargs="+", required=True)
    parser.add_argument("--step", type=float, default=0.1)
    parser.add_argument("--priors", type=float, nargs="+", default=[0.0, 0.25, 0.5, 1.0])
    parser.add_argument("--cards", type=float, nargs="+", default=[0.0, 0.25, 0.5, 1.0])
    parser.add_argument("--top", type=int, default=5)
    parser.add_argument("--name", required=True)
    parser.add_argument("--feature-meta", type=Path,
                        default=ROOT / "artifacts/features/v802/dinov3-raw-640x352/meta.json")
    args = parser.parse_args()
    data = load_data(json.loads(args.feature_meta.read_text())["file_names"])

    def run(seeds, weights, prior, card):
        rows = []
        for seed in seeds:
            folds = make_folds(data, seed)
            probs = sum(w * cache[(c, seed)] for w, c in zip(weights, args.components) if w)
            rows.append(evaluate(data, probs, folds, prior, card))
        return summarize(rows)

    cache = {(c, s): component(data, c, s) for c in args.components for s in DEV_SEEDS}
    results = []
    for weights in weight_grid(len(args.components), args.step):
        for prior in args.priors:
            for card in args.cards:
                summary = run(DEV_SEEDS, weights, prior, card)
                results.append({"weights": weights, "prior": prior, "card": card, "dev": summary})
    results.sort(key=lambda r: (r["dev"]["mean_exact"], r["dev"]["min_exact"]), reverse=True)
    top = results[: args.top]
    have_check = all((TRIALS / "oof" / f"{c}_seed{s}.npz").exists() or c == "v801"
                     for c in args.components for s in CHECK_SEEDS)
    if have_check:
        cache.update({(c, s): component(data, c, s) for c in args.components for s in CHECK_SEEDS})
        for row in top:
            row["check"] = run(CHECK_SEEDS, row["weights"], row["prior"], row["card"])
    for row in top:
        print(json.dumps(row))
    dump(TRIALS / f"{args.name}.json", {"components": args.components, "top": top,
                                        "evaluated": len(results), "check_available": have_check})


if __name__ == "__main__":
    main()