Spaces:
Configuration error
Configuration error
Niranjan-Murugarasu024
feat: TradeSense AI Chairman Edition production release ready for Hugging Face Spaces
0125724 | """ | |
| TradeSense AI — Multiple-Testing & Experiment Registry Unit Tests | |
| Validates Benjamini-Hochberg FDR, Holm-Bonferroni correction, | |
| pre-registered hypothesis tracking, and promotion status adjustments. | |
| """ | |
| from datetime import datetime, timezone | |
| import pytest | |
| import numpy as np | |
| from quant.evaluation.experiment_registry import ( | |
| ExperimentRegistry, | |
| MultipleTestingCorrector, | |
| MultipleTestingMethod, | |
| HypothesisType, | |
| ExperimentHypothesis, | |
| ExperimentRecord, | |
| ) | |
| class TestMultipleTestingCorrections: | |
| def test_benjamini_hochberg_ordering_and_adjustment(self): | |
| """ | |
| BH adjustment must produce p_adj >= p_raw and preserve ranking. | |
| """ | |
| raw_p = [0.001, 0.01, 0.04, 0.10, 0.50] | |
| adj_p, sigs = MultipleTestingCorrector.benjamini_hochberg(raw_p, q_threshold=0.05) | |
| for p_r, p_a in zip(raw_p, adj_p): | |
| assert p_a >= p_r | |
| assert adj_p[0] < adj_p[1] < adj_p[2] <= adj_p[3] <= adj_p[4] | |
| # First 3 should remain significant at FDR 0.05 | |
| assert sigs[0] is True | |
| assert sigs[1] is True | |
| assert sigs[4] is False | |
| def test_holm_bonferroni_step_down(self): | |
| """ | |
| Holm-Bonferroni step down tests: p_adj_1 = 5 * 0.005 = 0.025. | |
| """ | |
| raw_p = [0.005, 0.02, 0.03, 0.08, 0.20] | |
| adj_p, sigs = MultipleTestingCorrector.holm_bonferroni(raw_p, alpha=0.05) | |
| assert adj_p[0] == pytest.approx(0.025, abs=1e-4) | |
| assert sigs[0] is True | |
| assert all(adj_p[i] >= raw_p[i] for i in range(len(raw_p))) | |
| class TestExperimentRegistry: | |
| def test_pre_registered_hypothesis_locking(self): | |
| registry = ExperimentRegistry() | |
| hypo = registry.register_hypothesis( | |
| trial_family_id="fam_test_001", | |
| hypothesis_statement="Candidate model outperforms Naive persistence on 21D horizon.", | |
| hypothesis_type=HypothesisType.CONFIRMATORY, | |
| ) | |
| assert hypo.is_locked is True | |
| assert hypo.hypothesis_id.startswith("hypo_fam_test_001_") | |
| assert hypo.hypothesis_type == HypothesisType.CONFIRMATORY | |
| def test_multiple_testing_downgrades_fragile_promotion(self): | |
| """ | |
| If a model is promoted on raw p=0.04, but after testing 10 models | |
| its adjusted p becomes 0.20, the registry must downgrade it to | |
| NO_ACTIONABLE_EDGE with explicit rejection reason. | |
| """ | |
| registry = ExperimentRegistry() | |
| hypo = registry.register_hypothesis( | |
| trial_family_id="fam_downgrade_test", | |
| hypothesis_statement="Confirmatory multi-model test.", | |
| ) | |
| # Record 1: Candidate with borderline raw p=0.04 | |
| registry.record_experiment_result( | |
| hypothesis=hypo, | |
| ticker="AAPL", | |
| horizon_sessions=21, | |
| dataset_lineage_hash="hash_aapl", | |
| feature_version="fts_v1", | |
| model_version="xgboost_v1", | |
| hyperparameter_version="hp_v1", | |
| promotion_policy_version="prom_v1", | |
| random_seed=42, | |
| raw_dm_p_value=0.04, # Borderline significant before correction | |
| oos_rmse_improvement_pct=15.0, | |
| directional_accuracy_pct=58.0, | |
| mean_calibration_error_pct=2.0, | |
| gross_return_pct=18.0, | |
| net_return_pct=14.0, | |
| net_sharpe=1.5, | |
| max_drawdown_pct=10.0, | |
| is_robust=True, | |
| initial_promotion_status="PROMOTED", | |
| initial_signal_decision="BULLISH", | |
| initial_rejection_reasons=[], | |
| ) | |
| # Add 9 other non-significant trials to inflate the family size | |
| for i in range(2, 11): | |
| registry.record_experiment_result( | |
| hypothesis=hypo, | |
| ticker=f"STOCK_{i}", | |
| horizon_sessions=21, | |
| dataset_lineage_hash=f"hash_{i}", | |
| feature_version="fts_v1", | |
| model_version="xgboost_v1", | |
| hyperparameter_version="hp_v1", | |
| promotion_policy_version="prom_v1", | |
| random_seed=42, | |
| raw_dm_p_value=0.30 + (i * 0.05), | |
| oos_rmse_improvement_pct=2.0, | |
| directional_accuracy_pct=50.0, | |
| mean_calibration_error_pct=4.0, | |
| gross_return_pct=2.0, | |
| net_return_pct=-1.0, | |
| net_sharpe=0.1, | |
| max_drawdown_pct=18.0, | |
| is_robust=True, | |
| initial_promotion_status="REJECTED", | |
| initial_signal_decision="NO_ACTIONABLE_EDGE", | |
| initial_rejection_reasons=["INSUFFICIENT_RMSE_EDGE"], | |
| ) | |
| # Apply Benjamini-Hochberg | |
| corrected = registry.apply_multiple_testing_correction( | |
| trial_family_id="fam_downgrade_test", | |
| method=MultipleTestingMethod.BENJAMINI_HOCHBERG, | |
| alpha_fdr=0.05, | |
| ) | |
| aapl_rec = [r for r in corrected if r.ticker == "AAPL"][0] | |
| # In family of 10, raw p=0.04 adjusted is min(1.0, 10/1 * 0.04) = 0.40 > 0.05 | |
| assert aapl_rec.adjusted_dm_p_value > 0.05 | |
| assert aapl_rec.promotion_status == "REJECTED" | |
| assert aapl_rec.signal_decision == "NO_ACTIONABLE_EDGE" | |
| assert any("MULTIPLE_TESTING_FAIL" in r for r in aapl_rec.rejection_reasons) | |