Add learned dominance selector diagnostic
Browse files
workspace/tests/test_dominance_selector.py
CHANGED
|
@@ -1,6 +1,9 @@
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
|
|
|
|
|
|
| 3 |
from scripts.eval_dominance_selector import _choose_tau, _conformal_quantile, _evaluate_case
|
|
|
|
| 4 |
|
| 5 |
|
| 6 |
def test_conformal_quantile_and_fallback_selection() -> None:
|
|
@@ -64,3 +67,45 @@ def test_choose_tau_prefers_success_then_utility_then_coverage() -> None:
|
|
| 64 |
tau = _choose_tau(cases, residual_quantile=0.0)
|
| 65 |
evaluated = [_evaluate_case(case, residual_quantile=0.0, tau=tau) for case in cases]
|
| 66 |
assert sum(row["selected_success"] for row in evaluated) == 2.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
+
import numpy as np
|
| 4 |
+
|
| 5 |
from scripts.eval_dominance_selector import _choose_tau, _conformal_quantile, _evaluate_case
|
| 6 |
+
from scripts.eval_learned_dominance_selector import _evaluate_dataset, _fit_select_ridge
|
| 7 |
|
| 8 |
|
| 9 |
def test_conformal_quantile_and_fallback_selection() -> None:
|
|
|
|
| 67 |
tau = _choose_tau(cases, residual_quantile=0.0)
|
| 68 |
evaluated = [_evaluate_case(case, residual_quantile=0.0, tau=tau) for case in cases]
|
| 69 |
assert sum(row["selected_success"] for row in evaluated) == 2.0
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def test_learned_dominance_ridge_prefers_positive_candidate() -> None:
|
| 73 |
+
def sample(row: int, candidate: int, score: float, margin: float, success: float) -> dict:
|
| 74 |
+
feature = np.zeros(10, dtype=float)
|
| 75 |
+
feature[0] = 1.0
|
| 76 |
+
feature[1] = score
|
| 77 |
+
feature[2] = margin
|
| 78 |
+
feature[4] = float(candidate)
|
| 79 |
+
return {
|
| 80 |
+
"row_index": row,
|
| 81 |
+
"candidate_index": candidate,
|
| 82 |
+
"feature": feature,
|
| 83 |
+
"target_margin": success - 0.2,
|
| 84 |
+
"candidate_utility": success,
|
| 85 |
+
"candidate_success": success,
|
| 86 |
+
"base_utility": 0.2,
|
| 87 |
+
"base_success": 0.0,
|
| 88 |
+
"proposal_oracle_utility": success,
|
| 89 |
+
"proposal_oracle_success": success,
|
| 90 |
+
"hidden_chart_oracle_utility": 1.0,
|
| 91 |
+
"hidden_chart_oracle_success": 1.0,
|
| 92 |
+
"outcome_ptr": success,
|
| 93 |
+
"chart_id": f"c{row}",
|
| 94 |
+
"task_id": "pick",
|
| 95 |
+
"seed": "0",
|
| 96 |
+
"train_seed": "0",
|
| 97 |
+
}
|
| 98 |
+
|
| 99 |
+
dataset = {
|
| 100 |
+
"samples": [
|
| 101 |
+
sample(0, 0, 2.0, 2.0, 1.0),
|
| 102 |
+
sample(0, 1, -1.0, -1.0, 0.0),
|
| 103 |
+
sample(1, 0, 2.0, 2.0, 1.0),
|
| 104 |
+
sample(1, 1, -1.0, -1.0, 0.0),
|
| 105 |
+
],
|
| 106 |
+
"by_row": {0: [0, 1], 1: [2, 3]},
|
| 107 |
+
"num_rows": 2,
|
| 108 |
+
}
|
| 109 |
+
fit = _fit_select_ridge(dataset, lambdas=[0.1])
|
| 110 |
+
rows = _evaluate_dataset(dataset, fit["weights"], fit["mean"], fit["std"], tau=fit["tau"])
|
| 111 |
+
assert [row["selected_success"] for row in rows] == [1.0, 1.0]
|