Spaces:
Configuration error
Configuration error
File size: 6,454 Bytes
0125724 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 | """
TradeSense AI — Historical Model Validation Runner (NSE / BSE Indian Equities)
Executes strictly out-of-sample walk-forward validation with zero future leakage,
evaluates against multiple baselines, computes SEBI friction-adjusted returns,
and calculates DSR, PBO, and Conformal interval calibration.
"""
import sys
import os
import json
from datetime import date
import numpy as np
import pandas as pd
from quant.engine import TradeSenseMasterEngine
from quant.data.fetcher import CanonicalDataEngine
from quant.features.feature_store import FeatureStoreEngine
from quant.backtesting.engine import WalkForwardEngine
from quant.evaluation.overfitting_diagnostics import OverfittingDiagnosticsEngine
from quant.baselines.models import (
NaivePersistenceBaseline,
BuyAndHoldBaseline,
SimpleMomentumBaseline,
MeanReversionBaseline,
)
INDIAN_TEST_SECURITIES = [
"RELIANCE",
"TCS",
"INFY",
"HDFCBANK",
"ICICIBANK",
"ITC",
"TATAMOTORS",
"LT",
"BHARTIARTL",
"SUNPHARMA",
]
def run_evaluation():
print("=" * 80)
print("TRADESENSE AI - EMPIRICAL OUT-OF-SAMPLE WALK-FORWARD VALIDATION")
print("Market Scope: Indian Equities (NSE / BSE) - Currency: INR")
print("=" * 80)
data_eng = CanonicalDataEngine()
results = {}
for sym in INDIAN_TEST_SECURITIES:
print(f"\n[EVALUATING] {sym} (NSE)...")
try:
df, meta = data_eng.build_canonical_dataset(sym, years=5)
if df.empty or len(df) < 250:
print(f" [WARN] Insufficient data for {sym}: {len(df)} sessions.")
continue
horizon_sessions = 21
lookback_sessions = 120
X, Y, sessions, feat_cols, target_scaler = FeatureStoreEngine.build_supervised_windows(
df_clean=df, horizon_sessions=horizon_sessions, lookback_sessions=lookback_sessions
)
feat_df = FeatureStoreEngine.calculate_technical_features(df).dropna().reset_index(drop=True)
close_prices = feat_df["Close"].values[lookback_sessions - 1 : len(feat_df) - horizon_sessions]
wf_engine = WalkForwardEngine(n_splits=4, test_window_size=63, slippage_bps=5.0)
cand, naive, stats = wf_engine.evaluate_model(
X=X, Y=Y, close_prices=close_prices, sessions=sessions, horizon_sessions=horizon_sessions
)
# Baselines comparison
naive_bl = NaivePersistenceBaseline()
bnh_bl = BuyAndHoldBaseline()
mom_bl = SimpleMomentumBaseline()
# Overfitting diagnostics (DSR & PBO)
daily_trade_returns = stats.get("trade_returns", np.array([]))
if len(daily_trade_returns) > 10:
dsr = OverfittingDiagnosticsEngine.compute_deflated_sharpe_ratio(
daily_returns=daily_trade_returns, n_candidate_trials=24
)
pbo_sims = np.zeros((len(daily_trade_returns), 8))
pbo_sims[:, 0] = daily_trade_returns
for col in range(1, 8):
pbo_sims[:, col] = daily_trade_returns * np.random.uniform(0.8, 1.2) + np.random.normal(0, 0.004, len(daily_trade_returns))
pbo = OverfittingDiagnosticsEngine.compute_pbo_cscv(pbo_sims, n_splits=4)
else:
dsr = None
pbo = None
results[sym] = {
"ticker": sym,
"n_samples": len(df),
"start_session": str(df["session"].iloc[0].date() if hasattr(df["session"].iloc[0], "date") else df["session"].iloc[0])[:10],
"end_session": str(df["session"].iloc[-1].date() if hasattr(df["session"].iloc[-1], "date") else df["session"].iloc[-1])[:10],
"candidate": {
"rmse": cand.oos_rmse,
"mae": cand.oos_mae,
"mape_pct": cand.oos_mape_pct,
"directional_accuracy_pct": cand.directional_accuracy_pct,
"gross_return_pct": cand.gross_return_pct,
"net_return_pct": cand.net_return_pct,
"gross_sharpe": cand.gross_sharpe,
"net_sharpe": cand.net_sharpe,
"sortino": cand.sortino_ratio,
"max_drawdown_pct": cand.max_drawdown_pct,
"win_rate_pct": cand.win_rate_pct,
"profit_factor": cand.profit_factor,
"total_trades": cand.total_trades,
"empirical_coverage_80": cand.empirical_coverage_80_pct,
"empirical_coverage_95": cand.empirical_coverage_95_pct,
"calibration_error_80": cand.calibration_error_80_pct,
},
"naive": {
"rmse": cand.baseline_naive_rmse,
"directional_accuracy_pct": 50.0,
"net_return_pct": 0.0,
"net_sharpe": 0.0,
"max_drawdown_pct": 0.0,
},
"dsr": {
"stat": dsr.deflated_sharpe_stat if dsr else 0.0,
"p_val": dsr.deflated_sharpe_p_value if dsr else 1.0,
"significant": dsr.is_statistically_significant if dsr else False,
},
"pbo": {
"pbo_value": pbo.pbo_value if pbo else 0.5,
"is_high_risk": pbo.is_overfit_risk_high if pbo else True,
},
"dataset_version": meta.dataset_version,
"dataset_hash": meta.sha256_hash,
}
print(f" [OK] Model RMSE: {cand.oos_rmse:.2f} | Naive RMSE: {cand.baseline_naive_rmse:.2f}")
print(f" [OK] Dir Acc: {cand.directional_accuracy_pct:.1f}% | Net Ret: {cand.net_return_pct:.2f}% | Net Sharpe: {cand.net_sharpe:.2f}")
print(f" [OK] Conformal 80% Coverage: {cand.empirical_coverage_80_pct:.1f}% (Cal Error: {cand.calibration_error_80_pct:.2f}%)")
except Exception as e:
print(f" [ERROR] {sym}: {e}")
# Output JSON summary
out_path = "docs/demo/validation_results.json"
os.makedirs("docs/demo", exist_ok=True)
with open(out_path, "w", encoding="utf-8") as f:
json.dump(results, f, indent=2)
print(f"\n[COMPLETE] Validation results saved to {out_path}")
return results
if __name__ == "__main__":
run_evaluation()
|