Test nonlinear chart-compat provenance
Browse files
workspace/tests/test_dominance_selector.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
|
|
|
| 3 |
from types import SimpleNamespace
|
| 4 |
|
| 5 |
import numpy as np
|
|
@@ -18,6 +19,7 @@ from scripts.eval_learned_dominance_selector import (
|
|
| 18 |
_target_value,
|
| 19 |
)
|
| 20 |
from scripts.eval_nonlinear_dominance_selector import _split_rows, _subset_dataset
|
|
|
|
| 21 |
|
| 22 |
|
| 23 |
def test_conformal_quantile_and_fallback_selection() -> None:
|
|
@@ -412,3 +414,44 @@ def test_nonlinear_dominance_split_is_row_disjoint() -> None:
|
|
| 412 |
subset = _subset_dataset(dataset, select_rows)
|
| 413 |
assert subset["num_rows"] == len(select_rows)
|
| 414 |
assert len(subset["samples"]) == 2 * len(select_rows)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
+
import json
|
| 4 |
from types import SimpleNamespace
|
| 5 |
|
| 6 |
import numpy as np
|
|
|
|
| 19 |
_target_value,
|
| 20 |
)
|
| 21 |
from scripts.eval_nonlinear_dominance_selector import _split_rows, _subset_dataset
|
| 22 |
+
from scripts.eval_nonlinear_dominance_selector import _write_provenance as _write_nonlinear_provenance
|
| 23 |
|
| 24 |
|
| 25 |
def test_conformal_quantile_and_fallback_selection() -> None:
|
|
|
|
| 414 |
subset = _subset_dataset(dataset, select_rows)
|
| 415 |
assert subset["num_rows"] == len(select_rows)
|
| 416 |
assert len(subset["samples"]) == 2 * len(select_rows)
|
| 417 |
+
|
| 418 |
+
|
| 419 |
+
def test_nonlinear_chart_compat_provenance_records_default_source_index(tmp_path) -> None:
|
| 420 |
+
calibration_input = tmp_path / "calibration.json"
|
| 421 |
+
eval_input = tmp_path / "eval.json"
|
| 422 |
+
train_index = tmp_path / "train_index.json"
|
| 423 |
+
test_index = tmp_path / "test_index.json"
|
| 424 |
+
calibration_input.write_text("{}\n")
|
| 425 |
+
eval_input.write_text("{}\n")
|
| 426 |
+
train_index.write_text(
|
| 427 |
+
'{"split":"train","content_hash":"train-content","split_hash":"train-split","retrieval_index_allowed":true}\n'
|
| 428 |
+
)
|
| 429 |
+
test_index.write_text(
|
| 430 |
+
'{"split":"test","content_hash":"test-content","split_hash":"test-split","retrieval_index_allowed":false}\n'
|
| 431 |
+
)
|
| 432 |
+
args = SimpleNamespace(
|
| 433 |
+
calibration_input=calibration_input,
|
| 434 |
+
calibration_target_index=train_index,
|
| 435 |
+
eval_input=eval_input,
|
| 436 |
+
eval_target_index=test_index,
|
| 437 |
+
source_index=None,
|
| 438 |
+
checkpoint_template="runs/ctt_residual_base_context_obs_seed{seed}/model.pt",
|
| 439 |
+
out_dir=tmp_path / "out",
|
| 440 |
+
k=16,
|
| 441 |
+
feature_set="chart_compat",
|
| 442 |
+
selector_chart_feature_mode="base_context_obs",
|
| 443 |
+
target="utility_margin",
|
| 444 |
+
model_types="hgb_regressor",
|
| 445 |
+
selection_frac=0.35,
|
| 446 |
+
seed=0,
|
| 447 |
+
bootstrap_samples=10,
|
| 448 |
+
)
|
| 449 |
+
|
| 450 |
+
args.out_dir.mkdir()
|
| 451 |
+
_write_nonlinear_provenance(args.out_dir, args)
|
| 452 |
+
data_hash = json.loads((args.out_dir / "data_hash.txt").read_text())
|
| 453 |
+
split_hash = json.loads((args.out_dir / "split_hash.txt").read_text())
|
| 454 |
+
|
| 455 |
+
assert "source_index" in data_hash
|
| 456 |
+
assert split_hash["source_index"]["split"] == "train"
|
| 457 |
+
assert split_hash["source_index"]["retrieval_index_allowed"] is True
|