tradesenseAi / quant /evaluation /statistical_tests.py
Niranjan-Murugarasu024
feat: TradeSense AI Chairman Edition production release ready for Hugging Face Spaces
0125724
Raw History Blame Contribute Delete
11.8 kB
"""
TradeSense AI — Quantitative Statistical Testing Engine
Implements rigorous statistical validation methods:
1. Multi-step Diebold-Mariano Test with Newey-West HAC long-run covariance and
Harvey-Leybourne-Newbold (1997) small-sample correction.
2. Directional Accuracy Hypothesis Testing with Wilson Score Confidence Intervals
and Binomial Exact / Normal p-values vs baseline drift rate.
3. Genuinely Out-of-Sample Prediction Interval Calibration (80% and 95%).
"""
import math
from typing import Tuple, Dict, Any, Optional
import numpy as np
from scipy import stats
from pydantic import BaseModel, Field
class DieboldMarianoResult(BaseModel):
"""Structured result of Diebold-Mariano forecast comparison test."""
stat: float = Field(..., description="Harvey-Leybourne-Newbold adjusted DM statistic")
p_value: float = Field(..., description="Two-tailed p-value (Student-t distribution)")
is_statistically_significant: bool = Field(..., description="Whether p_value < alpha (default 0.05)")
candidate_loss_mean: float = Field(..., description="Mean squared loss of candidate model")
baseline_loss_mean: float = Field(..., description="Mean squared loss of baseline model")
loss_differential_mean: float = Field(..., description="Mean loss difference (baseline - candidate)")
hac_standard_error: float = Field(..., description="Newey-West HAC standard error")
sample_size: int = Field(..., description="Number of out-of-sample forecast observations")
forecast_horizon: int = Field(..., description="Forecast horizon in sessions (h)")
class DirectionalTestResult(BaseModel):
"""Structured result of directional accuracy statistical test."""
directional_accuracy_pct: float = Field(..., description="Observed hit rate (%)")
sample_size: int = Field(..., description="Number of evaluated directional trades")
correct_count: int = Field(..., description="Number of correct directional predictions")
baseline_rate_pct: float = Field(..., description="Base rate / drift rate (%) for comparison")
directional_edge_pct: float = Field(..., description="Observed accuracy minus baseline rate (%)")
ci_95_lower: float = Field(..., description="Wilson score 95% CI lower bound (%)")
ci_95_upper: float = Field(..., description="Wilson score 95% CI upper bound (%)")
p_value: float = Field(..., description="One-tailed p-value testing H1: Accuracy > Base Rate")
is_significant: bool = Field(..., description="Whether directional edge is statistically significant")
class IntervalCalibrationResult(BaseModel):
"""Structured result of empirical prediction interval coverage evaluation."""
nominal_80_coverage_pct: float = Field(..., description="Observed coverage of nominal 80% prediction interval")
nominal_95_coverage_pct: float = Field(..., description="Observed coverage of nominal 95% prediction interval")
calibration_error_80_pct: float = Field(..., description="|Observed - 80%| (%)")
calibration_error_95_pct: float = Field(..., description="|Observed - 95%| (%)")
mean_calibration_error_pct: float = Field(..., description="Average calibration error across intervals (%)")
is_well_calibrated: bool = Field(..., description="Whether mean calibration error <= threshold (default 5.0%)")
class StatisticalValidationEngine:
"""
Executes formal statistical hypothesis testing and out-of-sample calibration.
"""
@staticmethod
def diebold_mariano_test(
y_true: np.ndarray,
y_pred_candidate: np.ndarray,
y_pred_baseline: np.ndarray,
horizon_sessions: int = 1,
alpha: float = 0.05,
loss_type: str = "squared",
) -> DieboldMarianoResult:
"""
Computes the Diebold-Mariano (1995) test statistic for equal predictive accuracy,
incorporating Newey-West HAC covariance for h-step autocorrelation and the
Harvey-Leybourne-Newbold (1997) finite sample correction.
H0: Candidate and Baseline have equal forecast accuracy (mean loss differential = 0).
H1: Candidate and Baseline have different forecast accuracy.
"""
y_true = np.asarray(y_true, dtype=float).ravel()
y_cand = np.asarray(y_pred_candidate, dtype=float).ravel()
y_base = np.asarray(y_pred_baseline, dtype=float).ravel()
n = len(y_true)
if n < 10:
return DieboldMarianoResult(
stat=0.0,
p_value=1.0,
is_statistically_significant=False,
candidate_loss_mean=0.0,
baseline_loss_mean=0.0,
loss_differential_mean=0.0,
hac_standard_error=1.0,
sample_size=n,
forecast_horizon=horizon_sessions,
)
# 1. Compute loss differential: d_t = Loss(baseline) - Loss(candidate)
# Positive d_t means candidate has LOWER loss (better performance)
if loss_type == "squared":
loss_cand = (y_true - y_cand) ** 2
loss_base = (y_true - y_base) ** 2
else:
loss_cand = np.abs(y_true - y_cand)
loss_base = np.abs(y_true - y_base)
d = loss_base - loss_cand
d_bar = float(np.mean(d))
cand_mean_loss = float(np.mean(loss_cand))
base_mean_loss = float(np.mean(loss_base))
# 2. Newey-West HAC Covariance Estimator for lag truncation k = h - 1
h = max(1, int(horizon_sessions))
max_lag = max(1, h - 1)
# Sample autocovariances
d_centered = d - d_bar
gamma_0 = float(np.var(d_centered, ddof=0))
gamma_sum = 0.0
for lag in range(1, min(max_lag + 1, n)):
gamma_lag = float(np.mean(d_centered[lag:] * d_centered[:-lag]))
# Bartlett kernel weight: 1 - lag / (max_lag + 1)
weight = 1.0 - (lag / (max_lag + 1.0))
gamma_sum += 2.0 * weight * gamma_lag
long_run_var = gamma_0 + gamma_sum
if long_run_var <= 1e-12:
long_run_var = max(1e-6, gamma_0)
hac_se = math.sqrt(long_run_var / n)
# 3. Standard DM statistic
dm_stat = d_bar / hac_se if hac_se > 0 else 0.0
# 4. Harvey-Leybourne-Newbold (1997) small-sample correction factor
# HLN = DM * sqrt([n + 1 - 2h + h(h - 1)/n] / n)
correction_num = n + 1 - 2 * h + (h * (h - 1)) / n
if correction_num > 0 and n > 0:
hln_correction = math.sqrt(correction_num / n)
adjusted_stat = float(dm_stat * hln_correction)
else:
adjusted_stat = float(dm_stat)
# 5. Two-tailed p-value using Student-t distribution with (n - 1) degrees of freedom
df_deg = max(1, n - 1)
p_val = float(2.0 * (1.0 - stats.t.cdf(abs(adjusted_stat), df=df_deg)))
return DieboldMarianoResult(
stat=round(adjusted_stat, 4),
p_value=round(p_val, 5),
is_statistically_significant=(p_val < alpha and d_bar > 0),
candidate_loss_mean=round(cand_mean_loss, 4),
baseline_loss_mean=round(base_mean_loss, 4),
loss_differential_mean=round(d_bar, 4),
hac_standard_error=round(hac_se, 5),
sample_size=n,
forecast_horizon=h,
)
@staticmethod
def directional_accuracy_test(
y_true_returns: np.ndarray,
y_pred_returns: np.ndarray,
base_rate: float = 0.50,
alpha: float = 0.05,
) -> DirectionalTestResult:
"""
Evaluates directional forecast hit rate against base rate with Wilson Score CI
and binomial significance testing.
"""
y_t = np.asarray(y_true_returns, dtype=float).ravel()
y_p = np.asarray(y_pred_returns, dtype=float).ravel()
n = len(y_t)
if n == 0:
return DirectionalTestResult(
directional_accuracy_pct=0.0,
sample_size=0,
correct_count=0,
baseline_rate_pct=base_rate * 100.0,
directional_edge_pct=0.0,
ci_95_lower=0.0,
ci_95_upper=0.0,
p_value=1.0,
is_significant=False,
)
# Direction match: sign(actual) == sign(pred)
# Exclude flat 0 cases from incorrect penalty
actual_sign = np.sign(y_t)
pred_sign = np.sign(y_p)
correct = np.sum((actual_sign * pred_sign) > 0)
hit_rate = float(correct / n)
# Wilson score interval for binomial proportion (95% CI)
z = 1.95996
denom = 1.0 + (z**2) / n
center = (hit_rate + (z**2) / (2.0 * n)) / denom
margin = (z * math.sqrt((hit_rate * (1.0 - hit_rate) / n) + (z**2) / (4.0 * (n**2)))) / denom
ci_lower = max(0.0, (center - margin) * 100.0)
ci_upper = min(100.0, (center + margin) * 100.0)
# Binomial exact test p-value for H1: hit_rate > base_rate (one-tailed)
try:
p_val = float(stats.binomtest(k=int(correct), n=n, p=base_rate, alternative="greater").pvalue)
except Exception:
# Normal approximation fallback
se = math.sqrt(base_rate * (1.0 - base_rate) / n)
z_score = (hit_rate - base_rate) / se if se > 0 else 0.0
p_val = float(1.0 - stats.norm.cdf(z_score))
dir_acc_pct = round(hit_rate * 100.0, 2)
base_rate_pct = round(base_rate * 100.0, 2)
edge_pct = round(dir_acc_pct - base_rate_pct, 2)
return DirectionalTestResult(
directional_accuracy_pct=dir_acc_pct,
sample_size=n,
correct_count=int(correct),
baseline_rate_pct=base_rate_pct,
directional_edge_pct=edge_pct,
ci_95_lower=round(ci_lower, 2),
ci_95_upper=round(ci_upper, 2),
p_value=round(p_val, 5),
is_significant=(p_val < alpha and edge_pct > 0),
)
@staticmethod
def evaluate_interval_calibration(
y_true: np.ndarray,
lower_80: np.ndarray,
upper_80: np.ndarray,
lower_95: np.ndarray,
upper_95: np.ndarray,
tolerance_pct: float = 5.0,
) -> IntervalCalibrationResult:
"""
Evaluates empirical coverage of 80% and 95% prediction intervals.
Well-calibrated models have |Observed_80 - 80%| <= tolerance_pct and
|Observed_95 - 95%| <= tolerance_pct.
"""
y_t = np.asarray(y_true, dtype=float).ravel()
l80 = np.asarray(lower_80, dtype=float).ravel()
u80 = np.asarray(upper_80, dtype=float).ravel()
l95 = np.asarray(lower_95, dtype=float).ravel()
u95 = np.asarray(upper_95, dtype=float).ravel()
n = len(y_t)
if n == 0:
return IntervalCalibrationResult(
nominal_80_coverage_pct=0.0,
nominal_95_coverage_pct=0.0,
calibration_error_80_pct=80.0,
calibration_error_95_pct=95.0,
mean_calibration_error_pct=87.5,
is_well_calibrated=False,
)
cov_80 = float(np.mean((y_t >= l80) & (y_t <= u80)) * 100.0)
cov_95 = float(np.mean((y_t >= l95) & (y_t <= u95)) * 100.0)
err_80 = abs(cov_80 - 80.0)
err_95 = abs(cov_95 - 95.0)
mean_err = (err_80 + err_95) / 2.0
return IntervalCalibrationResult(
nominal_80_coverage_pct=round(cov_80, 2),
nominal_95_coverage_pct=round(cov_95, 2),
calibration_error_80_pct=round(err_80, 2),
calibration_error_95_pct=round(err_95, 2),
mean_calibration_error_pct=round(mean_err, 2),
is_well_calibrated=(mean_err <= tolerance_pct),
)