Spaces:
Configuration error
Configuration error
File size: 5,307 Bytes
0125724 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 | """
TradeSense AI — Multiple-Testing & Experiment Registry Unit Tests
Validates Benjamini-Hochberg FDR, Holm-Bonferroni correction,
pre-registered hypothesis tracking, and promotion status adjustments.
"""
from datetime import datetime, timezone
import pytest
import numpy as np
from quant.evaluation.experiment_registry import (
ExperimentRegistry,
MultipleTestingCorrector,
MultipleTestingMethod,
HypothesisType,
ExperimentHypothesis,
ExperimentRecord,
)
class TestMultipleTestingCorrections:
def test_benjamini_hochberg_ordering_and_adjustment(self):
"""
BH adjustment must produce p_adj >= p_raw and preserve ranking.
"""
raw_p = [0.001, 0.01, 0.04, 0.10, 0.50]
adj_p, sigs = MultipleTestingCorrector.benjamini_hochberg(raw_p, q_threshold=0.05)
for p_r, p_a in zip(raw_p, adj_p):
assert p_a >= p_r
assert adj_p[0] < adj_p[1] < adj_p[2] <= adj_p[3] <= adj_p[4]
# First 3 should remain significant at FDR 0.05
assert sigs[0] is True
assert sigs[1] is True
assert sigs[4] is False
def test_holm_bonferroni_step_down(self):
"""
Holm-Bonferroni step down tests: p_adj_1 = 5 * 0.005 = 0.025.
"""
raw_p = [0.005, 0.02, 0.03, 0.08, 0.20]
adj_p, sigs = MultipleTestingCorrector.holm_bonferroni(raw_p, alpha=0.05)
assert adj_p[0] == pytest.approx(0.025, abs=1e-4)
assert sigs[0] is True
assert all(adj_p[i] >= raw_p[i] for i in range(len(raw_p)))
class TestExperimentRegistry:
def test_pre_registered_hypothesis_locking(self):
registry = ExperimentRegistry()
hypo = registry.register_hypothesis(
trial_family_id="fam_test_001",
hypothesis_statement="Candidate model outperforms Naive persistence on 21D horizon.",
hypothesis_type=HypothesisType.CONFIRMATORY,
)
assert hypo.is_locked is True
assert hypo.hypothesis_id.startswith("hypo_fam_test_001_")
assert hypo.hypothesis_type == HypothesisType.CONFIRMATORY
def test_multiple_testing_downgrades_fragile_promotion(self):
"""
If a model is promoted on raw p=0.04, but after testing 10 models
its adjusted p becomes 0.20, the registry must downgrade it to
NO_ACTIONABLE_EDGE with explicit rejection reason.
"""
registry = ExperimentRegistry()
hypo = registry.register_hypothesis(
trial_family_id="fam_downgrade_test",
hypothesis_statement="Confirmatory multi-model test.",
)
# Record 1: Candidate with borderline raw p=0.04
registry.record_experiment_result(
hypothesis=hypo,
ticker="AAPL",
horizon_sessions=21,
dataset_lineage_hash="hash_aapl",
feature_version="fts_v1",
model_version="xgboost_v1",
hyperparameter_version="hp_v1",
promotion_policy_version="prom_v1",
random_seed=42,
raw_dm_p_value=0.04, # Borderline significant before correction
oos_rmse_improvement_pct=15.0,
directional_accuracy_pct=58.0,
mean_calibration_error_pct=2.0,
gross_return_pct=18.0,
net_return_pct=14.0,
net_sharpe=1.5,
max_drawdown_pct=10.0,
is_robust=True,
initial_promotion_status="PROMOTED",
initial_signal_decision="BULLISH",
initial_rejection_reasons=[],
)
# Add 9 other non-significant trials to inflate the family size
for i in range(2, 11):
registry.record_experiment_result(
hypothesis=hypo,
ticker=f"STOCK_{i}",
horizon_sessions=21,
dataset_lineage_hash=f"hash_{i}",
feature_version="fts_v1",
model_version="xgboost_v1",
hyperparameter_version="hp_v1",
promotion_policy_version="prom_v1",
random_seed=42,
raw_dm_p_value=0.30 + (i * 0.05),
oos_rmse_improvement_pct=2.0,
directional_accuracy_pct=50.0,
mean_calibration_error_pct=4.0,
gross_return_pct=2.0,
net_return_pct=-1.0,
net_sharpe=0.1,
max_drawdown_pct=18.0,
is_robust=True,
initial_promotion_status="REJECTED",
initial_signal_decision="NO_ACTIONABLE_EDGE",
initial_rejection_reasons=["INSUFFICIENT_RMSE_EDGE"],
)
# Apply Benjamini-Hochberg
corrected = registry.apply_multiple_testing_correction(
trial_family_id="fam_downgrade_test",
method=MultipleTestingMethod.BENJAMINI_HOCHBERG,
alpha_fdr=0.05,
)
aapl_rec = [r for r in corrected if r.ticker == "AAPL"][0]
# In family of 10, raw p=0.04 adjusted is min(1.0, 10/1 * 0.04) = 0.40 > 0.05
assert aapl_rec.adjusted_dm_p_value > 0.05
assert aapl_rec.promotion_status == "REJECTED"
assert aapl_rec.signal_decision == "NO_ACTIONABLE_EDGE"
assert any("MULTIPLE_TESTING_FAIL" in r for r in aapl_rec.rejection_reasons)
|