""" TradeSense AI — Quantitative Statistical Testing Engine Implements rigorous statistical validation methods: 1. Multi-step Diebold-Mariano Test with Newey-West HAC long-run covariance and Harvey-Leybourne-Newbold (1997) small-sample correction. 2. Directional Accuracy Hypothesis Testing with Wilson Score Confidence Intervals and Binomial Exact / Normal p-values vs baseline drift rate. 3. Genuinely Out-of-Sample Prediction Interval Calibration (80% and 95%). """ import math from typing import Tuple, Dict, Any, Optional import numpy as np from scipy import stats from pydantic import BaseModel, Field class DieboldMarianoResult(BaseModel): """Structured result of Diebold-Mariano forecast comparison test.""" stat: float = Field(..., description="Harvey-Leybourne-Newbold adjusted DM statistic") p_value: float = Field(..., description="Two-tailed p-value (Student-t distribution)") is_statistically_significant: bool = Field(..., description="Whether p_value < alpha (default 0.05)") candidate_loss_mean: float = Field(..., description="Mean squared loss of candidate model") baseline_loss_mean: float = Field(..., description="Mean squared loss of baseline model") loss_differential_mean: float = Field(..., description="Mean loss difference (baseline - candidate)") hac_standard_error: float = Field(..., description="Newey-West HAC standard error") sample_size: int = Field(..., description="Number of out-of-sample forecast observations") forecast_horizon: int = Field(..., description="Forecast horizon in sessions (h)") class DirectionalTestResult(BaseModel): """Structured result of directional accuracy statistical test.""" directional_accuracy_pct: float = Field(..., description="Observed hit rate (%)") sample_size: int = Field(..., description="Number of evaluated directional trades") correct_count: int = Field(..., description="Number of correct directional predictions") baseline_rate_pct: float = Field(..., description="Base rate / drift rate (%) for comparison") directional_edge_pct: float = Field(..., description="Observed accuracy minus baseline rate (%)") ci_95_lower: float = Field(..., description="Wilson score 95% CI lower bound (%)") ci_95_upper: float = Field(..., description="Wilson score 95% CI upper bound (%)") p_value: float = Field(..., description="One-tailed p-value testing H1: Accuracy > Base Rate") is_significant: bool = Field(..., description="Whether directional edge is statistically significant") class IntervalCalibrationResult(BaseModel): """Structured result of empirical prediction interval coverage evaluation.""" nominal_80_coverage_pct: float = Field(..., description="Observed coverage of nominal 80% prediction interval") nominal_95_coverage_pct: float = Field(..., description="Observed coverage of nominal 95% prediction interval") calibration_error_80_pct: float = Field(..., description="|Observed - 80%| (%)") calibration_error_95_pct: float = Field(..., description="|Observed - 95%| (%)") mean_calibration_error_pct: float = Field(..., description="Average calibration error across intervals (%)") is_well_calibrated: bool = Field(..., description="Whether mean calibration error <= threshold (default 5.0%)") class StatisticalValidationEngine: """ Executes formal statistical hypothesis testing and out-of-sample calibration. """ @staticmethod def diebold_mariano_test( y_true: np.ndarray, y_pred_candidate: np.ndarray, y_pred_baseline: np.ndarray, horizon_sessions: int = 1, alpha: float = 0.05, loss_type: str = "squared", ) -> DieboldMarianoResult: """ Computes the Diebold-Mariano (1995) test statistic for equal predictive accuracy, incorporating Newey-West HAC covariance for h-step autocorrelation and the Harvey-Leybourne-Newbold (1997) finite sample correction. H0: Candidate and Baseline have equal forecast accuracy (mean loss differential = 0). H1: Candidate and Baseline have different forecast accuracy. """ y_true = np.asarray(y_true, dtype=float).ravel() y_cand = np.asarray(y_pred_candidate, dtype=float).ravel() y_base = np.asarray(y_pred_baseline, dtype=float).ravel() n = len(y_true) if n < 10: return DieboldMarianoResult( stat=0.0, p_value=1.0, is_statistically_significant=False, candidate_loss_mean=0.0, baseline_loss_mean=0.0, loss_differential_mean=0.0, hac_standard_error=1.0, sample_size=n, forecast_horizon=horizon_sessions, ) # 1. Compute loss differential: d_t = Loss(baseline) - Loss(candidate) # Positive d_t means candidate has LOWER loss (better performance) if loss_type == "squared": loss_cand = (y_true - y_cand) ** 2 loss_base = (y_true - y_base) ** 2 else: loss_cand = np.abs(y_true - y_cand) loss_base = np.abs(y_true - y_base) d = loss_base - loss_cand d_bar = float(np.mean(d)) cand_mean_loss = float(np.mean(loss_cand)) base_mean_loss = float(np.mean(loss_base)) # 2. Newey-West HAC Covariance Estimator for lag truncation k = h - 1 h = max(1, int(horizon_sessions)) max_lag = max(1, h - 1) # Sample autocovariances d_centered = d - d_bar gamma_0 = float(np.var(d_centered, ddof=0)) gamma_sum = 0.0 for lag in range(1, min(max_lag + 1, n)): gamma_lag = float(np.mean(d_centered[lag:] * d_centered[:-lag])) # Bartlett kernel weight: 1 - lag / (max_lag + 1) weight = 1.0 - (lag / (max_lag + 1.0)) gamma_sum += 2.0 * weight * gamma_lag long_run_var = gamma_0 + gamma_sum if long_run_var <= 1e-12: long_run_var = max(1e-6, gamma_0) hac_se = math.sqrt(long_run_var / n) # 3. Standard DM statistic dm_stat = d_bar / hac_se if hac_se > 0 else 0.0 # 4. Harvey-Leybourne-Newbold (1997) small-sample correction factor # HLN = DM * sqrt([n + 1 - 2h + h(h - 1)/n] / n) correction_num = n + 1 - 2 * h + (h * (h - 1)) / n if correction_num > 0 and n > 0: hln_correction = math.sqrt(correction_num / n) adjusted_stat = float(dm_stat * hln_correction) else: adjusted_stat = float(dm_stat) # 5. Two-tailed p-value using Student-t distribution with (n - 1) degrees of freedom df_deg = max(1, n - 1) p_val = float(2.0 * (1.0 - stats.t.cdf(abs(adjusted_stat), df=df_deg))) return DieboldMarianoResult( stat=round(adjusted_stat, 4), p_value=round(p_val, 5), is_statistically_significant=(p_val < alpha and d_bar > 0), candidate_loss_mean=round(cand_mean_loss, 4), baseline_loss_mean=round(base_mean_loss, 4), loss_differential_mean=round(d_bar, 4), hac_standard_error=round(hac_se, 5), sample_size=n, forecast_horizon=h, ) @staticmethod def directional_accuracy_test( y_true_returns: np.ndarray, y_pred_returns: np.ndarray, base_rate: float = 0.50, alpha: float = 0.05, ) -> DirectionalTestResult: """ Evaluates directional forecast hit rate against base rate with Wilson Score CI and binomial significance testing. """ y_t = np.asarray(y_true_returns, dtype=float).ravel() y_p = np.asarray(y_pred_returns, dtype=float).ravel() n = len(y_t) if n == 0: return DirectionalTestResult( directional_accuracy_pct=0.0, sample_size=0, correct_count=0, baseline_rate_pct=base_rate * 100.0, directional_edge_pct=0.0, ci_95_lower=0.0, ci_95_upper=0.0, p_value=1.0, is_significant=False, ) # Direction match: sign(actual) == sign(pred) # Exclude flat 0 cases from incorrect penalty actual_sign = np.sign(y_t) pred_sign = np.sign(y_p) correct = np.sum((actual_sign * pred_sign) > 0) hit_rate = float(correct / n) # Wilson score interval for binomial proportion (95% CI) z = 1.95996 denom = 1.0 + (z**2) / n center = (hit_rate + (z**2) / (2.0 * n)) / denom margin = (z * math.sqrt((hit_rate * (1.0 - hit_rate) / n) + (z**2) / (4.0 * (n**2)))) / denom ci_lower = max(0.0, (center - margin) * 100.0) ci_upper = min(100.0, (center + margin) * 100.0) # Binomial exact test p-value for H1: hit_rate > base_rate (one-tailed) try: p_val = float(stats.binomtest(k=int(correct), n=n, p=base_rate, alternative="greater").pvalue) except Exception: # Normal approximation fallback se = math.sqrt(base_rate * (1.0 - base_rate) / n) z_score = (hit_rate - base_rate) / se if se > 0 else 0.0 p_val = float(1.0 - stats.norm.cdf(z_score)) dir_acc_pct = round(hit_rate * 100.0, 2) base_rate_pct = round(base_rate * 100.0, 2) edge_pct = round(dir_acc_pct - base_rate_pct, 2) return DirectionalTestResult( directional_accuracy_pct=dir_acc_pct, sample_size=n, correct_count=int(correct), baseline_rate_pct=base_rate_pct, directional_edge_pct=edge_pct, ci_95_lower=round(ci_lower, 2), ci_95_upper=round(ci_upper, 2), p_value=round(p_val, 5), is_significant=(p_val < alpha and edge_pct > 0), ) @staticmethod def evaluate_interval_calibration( y_true: np.ndarray, lower_80: np.ndarray, upper_80: np.ndarray, lower_95: np.ndarray, upper_95: np.ndarray, tolerance_pct: float = 5.0, ) -> IntervalCalibrationResult: """ Evaluates empirical coverage of 80% and 95% prediction intervals. Well-calibrated models have |Observed_80 - 80%| <= tolerance_pct and |Observed_95 - 95%| <= tolerance_pct. """ y_t = np.asarray(y_true, dtype=float).ravel() l80 = np.asarray(lower_80, dtype=float).ravel() u80 = np.asarray(upper_80, dtype=float).ravel() l95 = np.asarray(lower_95, dtype=float).ravel() u95 = np.asarray(upper_95, dtype=float).ravel() n = len(y_t) if n == 0: return IntervalCalibrationResult( nominal_80_coverage_pct=0.0, nominal_95_coverage_pct=0.0, calibration_error_80_pct=80.0, calibration_error_95_pct=95.0, mean_calibration_error_pct=87.5, is_well_calibrated=False, ) cov_80 = float(np.mean((y_t >= l80) & (y_t <= u80)) * 100.0) cov_95 = float(np.mean((y_t >= l95) & (y_t <= u95)) * 100.0) err_80 = abs(cov_80 - 80.0) err_95 = abs(cov_95 - 95.0) mean_err = (err_80 + err_95) / 2.0 return IntervalCalibrationResult( nominal_80_coverage_pct=round(cov_80, 2), nominal_95_coverage_pct=round(cov_95, 2), calibration_error_80_pct=round(err_80, 2), calibration_error_95_pct=round(err_95, 2), mean_calibration_error_pct=round(mean_err, 2), is_well_calibrated=(mean_err <= tolerance_pct), )