Spaces:
Configuration error
Configuration error
Niranjan-Murugarasu024
feat: TradeSense AI Chairman Edition production release ready for Hugging Face Spaces
0125724 Download quant/evaluation/statistical_tests.py from BlueMen/tradesenseAi: direct link, hf CLI and curl.
- Browser
- Download file 11.8 kB
-
https://huggingface.co/spaces/BlueMen/tradesenseAi/resolve/main/quant/evaluation/statistical_tests.py
- Command line
-
hf download hf://spaces/BlueMen/tradesenseAi/quant/evaluation/statistical_tests.py
-
curl -L -o statistical_tests.py https://huggingface.co/spaces/BlueMen/tradesenseAi/resolve/main/quant/evaluation/statistical_tests.py
11.8 kB
| """ | |
| TradeSense AI — Quantitative Statistical Testing Engine | |
| Implements rigorous statistical validation methods: | |
| 1. Multi-step Diebold-Mariano Test with Newey-West HAC long-run covariance and | |
| Harvey-Leybourne-Newbold (1997) small-sample correction. | |
| 2. Directional Accuracy Hypothesis Testing with Wilson Score Confidence Intervals | |
| and Binomial Exact / Normal p-values vs baseline drift rate. | |
| 3. Genuinely Out-of-Sample Prediction Interval Calibration (80% and 95%). | |
| """ | |
| import math | |
| from typing import Tuple, Dict, Any, Optional | |
| import numpy as np | |
| from scipy import stats | |
| from pydantic import BaseModel, Field | |
| class DieboldMarianoResult(BaseModel): | |
| """Structured result of Diebold-Mariano forecast comparison test.""" | |
| stat: float = Field(..., description="Harvey-Leybourne-Newbold adjusted DM statistic") | |
| p_value: float = Field(..., description="Two-tailed p-value (Student-t distribution)") | |
| is_statistically_significant: bool = Field(..., description="Whether p_value < alpha (default 0.05)") | |
| candidate_loss_mean: float = Field(..., description="Mean squared loss of candidate model") | |
| baseline_loss_mean: float = Field(..., description="Mean squared loss of baseline model") | |
| loss_differential_mean: float = Field(..., description="Mean loss difference (baseline - candidate)") | |
| hac_standard_error: float = Field(..., description="Newey-West HAC standard error") | |
| sample_size: int = Field(..., description="Number of out-of-sample forecast observations") | |
| forecast_horizon: int = Field(..., description="Forecast horizon in sessions (h)") | |
| class DirectionalTestResult(BaseModel): | |
| """Structured result of directional accuracy statistical test.""" | |
| directional_accuracy_pct: float = Field(..., description="Observed hit rate (%)") | |
| sample_size: int = Field(..., description="Number of evaluated directional trades") | |
| correct_count: int = Field(..., description="Number of correct directional predictions") | |
| baseline_rate_pct: float = Field(..., description="Base rate / drift rate (%) for comparison") | |
| directional_edge_pct: float = Field(..., description="Observed accuracy minus baseline rate (%)") | |
| ci_95_lower: float = Field(..., description="Wilson score 95% CI lower bound (%)") | |
| ci_95_upper: float = Field(..., description="Wilson score 95% CI upper bound (%)") | |
| p_value: float = Field(..., description="One-tailed p-value testing H1: Accuracy > Base Rate") | |
| is_significant: bool = Field(..., description="Whether directional edge is statistically significant") | |
| class IntervalCalibrationResult(BaseModel): | |
| """Structured result of empirical prediction interval coverage evaluation.""" | |
| nominal_80_coverage_pct: float = Field(..., description="Observed coverage of nominal 80% prediction interval") | |
| nominal_95_coverage_pct: float = Field(..., description="Observed coverage of nominal 95% prediction interval") | |
| calibration_error_80_pct: float = Field(..., description="|Observed - 80%| (%)") | |
| calibration_error_95_pct: float = Field(..., description="|Observed - 95%| (%)") | |
| mean_calibration_error_pct: float = Field(..., description="Average calibration error across intervals (%)") | |
| is_well_calibrated: bool = Field(..., description="Whether mean calibration error <= threshold (default 5.0%)") | |
| class StatisticalValidationEngine: | |
| """ | |
| Executes formal statistical hypothesis testing and out-of-sample calibration. | |
| """ | |
| def diebold_mariano_test( | |
| y_true: np.ndarray, | |
| y_pred_candidate: np.ndarray, | |
| y_pred_baseline: np.ndarray, | |
| horizon_sessions: int = 1, | |
| alpha: float = 0.05, | |
| loss_type: str = "squared", | |
| ) -> DieboldMarianoResult: | |
| """ | |
| Computes the Diebold-Mariano (1995) test statistic for equal predictive accuracy, | |
| incorporating Newey-West HAC covariance for h-step autocorrelation and the | |
| Harvey-Leybourne-Newbold (1997) finite sample correction. | |
| H0: Candidate and Baseline have equal forecast accuracy (mean loss differential = 0). | |
| H1: Candidate and Baseline have different forecast accuracy. | |
| """ | |
| y_true = np.asarray(y_true, dtype=float).ravel() | |
| y_cand = np.asarray(y_pred_candidate, dtype=float).ravel() | |
| y_base = np.asarray(y_pred_baseline, dtype=float).ravel() | |
| n = len(y_true) | |
| if n < 10: | |
| return DieboldMarianoResult( | |
| stat=0.0, | |
| p_value=1.0, | |
| is_statistically_significant=False, | |
| candidate_loss_mean=0.0, | |
| baseline_loss_mean=0.0, | |
| loss_differential_mean=0.0, | |
| hac_standard_error=1.0, | |
| sample_size=n, | |
| forecast_horizon=horizon_sessions, | |
| ) | |
| # 1. Compute loss differential: d_t = Loss(baseline) - Loss(candidate) | |
| # Positive d_t means candidate has LOWER loss (better performance) | |
| if loss_type == "squared": | |
| loss_cand = (y_true - y_cand) ** 2 | |
| loss_base = (y_true - y_base) ** 2 | |
| else: | |
| loss_cand = np.abs(y_true - y_cand) | |
| loss_base = np.abs(y_true - y_base) | |
| d = loss_base - loss_cand | |
| d_bar = float(np.mean(d)) | |
| cand_mean_loss = float(np.mean(loss_cand)) | |
| base_mean_loss = float(np.mean(loss_base)) | |
| # 2. Newey-West HAC Covariance Estimator for lag truncation k = h - 1 | |
| h = max(1, int(horizon_sessions)) | |
| max_lag = max(1, h - 1) | |
| # Sample autocovariances | |
| d_centered = d - d_bar | |
| gamma_0 = float(np.var(d_centered, ddof=0)) | |
| gamma_sum = 0.0 | |
| for lag in range(1, min(max_lag + 1, n)): | |
| gamma_lag = float(np.mean(d_centered[lag:] * d_centered[:-lag])) | |
| # Bartlett kernel weight: 1 - lag / (max_lag + 1) | |
| weight = 1.0 - (lag / (max_lag + 1.0)) | |
| gamma_sum += 2.0 * weight * gamma_lag | |
| long_run_var = gamma_0 + gamma_sum | |
| if long_run_var <= 1e-12: | |
| long_run_var = max(1e-6, gamma_0) | |
| hac_se = math.sqrt(long_run_var / n) | |
| # 3. Standard DM statistic | |
| dm_stat = d_bar / hac_se if hac_se > 0 else 0.0 | |
| # 4. Harvey-Leybourne-Newbold (1997) small-sample correction factor | |
| # HLN = DM * sqrt([n + 1 - 2h + h(h - 1)/n] / n) | |
| correction_num = n + 1 - 2 * h + (h * (h - 1)) / n | |
| if correction_num > 0 and n > 0: | |
| hln_correction = math.sqrt(correction_num / n) | |
| adjusted_stat = float(dm_stat * hln_correction) | |
| else: | |
| adjusted_stat = float(dm_stat) | |
| # 5. Two-tailed p-value using Student-t distribution with (n - 1) degrees of freedom | |
| df_deg = max(1, n - 1) | |
| p_val = float(2.0 * (1.0 - stats.t.cdf(abs(adjusted_stat), df=df_deg))) | |
| return DieboldMarianoResult( | |
| stat=round(adjusted_stat, 4), | |
| p_value=round(p_val, 5), | |
| is_statistically_significant=(p_val < alpha and d_bar > 0), | |
| candidate_loss_mean=round(cand_mean_loss, 4), | |
| baseline_loss_mean=round(base_mean_loss, 4), | |
| loss_differential_mean=round(d_bar, 4), | |
| hac_standard_error=round(hac_se, 5), | |
| sample_size=n, | |
| forecast_horizon=h, | |
| ) | |
| def directional_accuracy_test( | |
| y_true_returns: np.ndarray, | |
| y_pred_returns: np.ndarray, | |
| base_rate: float = 0.50, | |
| alpha: float = 0.05, | |
| ) -> DirectionalTestResult: | |
| """ | |
| Evaluates directional forecast hit rate against base rate with Wilson Score CI | |
| and binomial significance testing. | |
| """ | |
| y_t = np.asarray(y_true_returns, dtype=float).ravel() | |
| y_p = np.asarray(y_pred_returns, dtype=float).ravel() | |
| n = len(y_t) | |
| if n == 0: | |
| return DirectionalTestResult( | |
| directional_accuracy_pct=0.0, | |
| sample_size=0, | |
| correct_count=0, | |
| baseline_rate_pct=base_rate * 100.0, | |
| directional_edge_pct=0.0, | |
| ci_95_lower=0.0, | |
| ci_95_upper=0.0, | |
| p_value=1.0, | |
| is_significant=False, | |
| ) | |
| # Direction match: sign(actual) == sign(pred) | |
| # Exclude flat 0 cases from incorrect penalty | |
| actual_sign = np.sign(y_t) | |
| pred_sign = np.sign(y_p) | |
| correct = np.sum((actual_sign * pred_sign) > 0) | |
| hit_rate = float(correct / n) | |
| # Wilson score interval for binomial proportion (95% CI) | |
| z = 1.95996 | |
| denom = 1.0 + (z**2) / n | |
| center = (hit_rate + (z**2) / (2.0 * n)) / denom | |
| margin = (z * math.sqrt((hit_rate * (1.0 - hit_rate) / n) + (z**2) / (4.0 * (n**2)))) / denom | |
| ci_lower = max(0.0, (center - margin) * 100.0) | |
| ci_upper = min(100.0, (center + margin) * 100.0) | |
| # Binomial exact test p-value for H1: hit_rate > base_rate (one-tailed) | |
| try: | |
| p_val = float(stats.binomtest(k=int(correct), n=n, p=base_rate, alternative="greater").pvalue) | |
| except Exception: | |
| # Normal approximation fallback | |
| se = math.sqrt(base_rate * (1.0 - base_rate) / n) | |
| z_score = (hit_rate - base_rate) / se if se > 0 else 0.0 | |
| p_val = float(1.0 - stats.norm.cdf(z_score)) | |
| dir_acc_pct = round(hit_rate * 100.0, 2) | |
| base_rate_pct = round(base_rate * 100.0, 2) | |
| edge_pct = round(dir_acc_pct - base_rate_pct, 2) | |
| return DirectionalTestResult( | |
| directional_accuracy_pct=dir_acc_pct, | |
| sample_size=n, | |
| correct_count=int(correct), | |
| baseline_rate_pct=base_rate_pct, | |
| directional_edge_pct=edge_pct, | |
| ci_95_lower=round(ci_lower, 2), | |
| ci_95_upper=round(ci_upper, 2), | |
| p_value=round(p_val, 5), | |
| is_significant=(p_val < alpha and edge_pct > 0), | |
| ) | |
| def evaluate_interval_calibration( | |
| y_true: np.ndarray, | |
| lower_80: np.ndarray, | |
| upper_80: np.ndarray, | |
| lower_95: np.ndarray, | |
| upper_95: np.ndarray, | |
| tolerance_pct: float = 5.0, | |
| ) -> IntervalCalibrationResult: | |
| """ | |
| Evaluates empirical coverage of 80% and 95% prediction intervals. | |
| Well-calibrated models have |Observed_80 - 80%| <= tolerance_pct and | |
| |Observed_95 - 95%| <= tolerance_pct. | |
| """ | |
| y_t = np.asarray(y_true, dtype=float).ravel() | |
| l80 = np.asarray(lower_80, dtype=float).ravel() | |
| u80 = np.asarray(upper_80, dtype=float).ravel() | |
| l95 = np.asarray(lower_95, dtype=float).ravel() | |
| u95 = np.asarray(upper_95, dtype=float).ravel() | |
| n = len(y_t) | |
| if n == 0: | |
| return IntervalCalibrationResult( | |
| nominal_80_coverage_pct=0.0, | |
| nominal_95_coverage_pct=0.0, | |
| calibration_error_80_pct=80.0, | |
| calibration_error_95_pct=95.0, | |
| mean_calibration_error_pct=87.5, | |
| is_well_calibrated=False, | |
| ) | |
| cov_80 = float(np.mean((y_t >= l80) & (y_t <= u80)) * 100.0) | |
| cov_95 = float(np.mean((y_t >= l95) & (y_t <= u95)) * 100.0) | |
| err_80 = abs(cov_80 - 80.0) | |
| err_95 = abs(cov_95 - 95.0) | |
| mean_err = (err_80 + err_95) / 2.0 | |
| return IntervalCalibrationResult( | |
| nominal_80_coverage_pct=round(cov_80, 2), | |
| nominal_95_coverage_pct=round(cov_95, 2), | |
| calibration_error_80_pct=round(err_80, 2), | |
| calibration_error_95_pct=round(err_95, 2), | |
| mean_calibration_error_pct=round(mean_err, 2), | |
| is_well_calibrated=(mean_err <= tolerance_pct), | |
| ) | |