Download backend/data/generate_ml_visualizations.py from Ypeng12/quant-ai: direct link, hf CLI and curl.
- Browser
- Download file 10.5 kB
-
https://huggingface.co/spaces/Ypeng12/quant-ai/resolve/main/backend/data/generate_ml_visualizations.py
- Command line
-
hf download hf://spaces/Ypeng12/quant-ai/backend/data/generate_ml_visualizations.py
-
curl -L -o generate_ml_visualizations.py https://huggingface.co/spaces/Ypeng12/quant-ai/resolve/main/backend/data/generate_ml_visualizations.py
10.5 kB
| # backend/data/generate_ml_visualizations.py | |
| """ | |
| Generates Institutional-Grade ML Diagnostic Charts & Visual Graphs: | |
| 1. Probability Calibration Reliability Curve (Platt Scaling vs Raw LightGBM) | |
| 2. HMM Market Regime Detection Timeline (3-State Transition) | |
| 3. 1,000-Scenario Monte Carlo Equity Clouds & CVaR 95% Distribution | |
| 4. Smart Order Router (SOR) EV_maker vs EV_taker Comparison | |
| """ | |
| import os | |
| import sys | |
| import numpy as np | |
| import pandas as pd | |
| import matplotlib.pyplot as plt | |
| import seaborn as sns | |
| from sklearn.calibration import calibration_curve | |
| # Set high-DPI clean dark/light theme | |
| plt.style.use('seaborn-v0_8-whitegrid') | |
| plt.rcParams['font.family'] = 'sans-serif' | |
| plt.rcParams['font.size'] = 11 | |
| OUTPUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "charts") | |
| os.makedirs(OUTPUT_DIR, exist_ok=True) | |
| # Add backend to sys.path | |
| sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | |
| from app.ml.ml_model_zoo import QuantMLModelZoo | |
| from app.ml.market_regime_hmm import MarketRegimeHMM | |
| from app.ml.monte_carlo_engine import MonteCarloTailRiskEngine | |
| from app.ml.lob_microstructure_ml import LOBMicrostructureMLSuite | |
| def plot_probability_calibration(df: pd.DataFrame): | |
| """Plots Chart 1: Reliability Calibration Curve.""" | |
| print("[*] Generating Chart 1: Probability Calibration Curve...") | |
| zoo = QuantMLModelZoo() | |
| zoo.fit_lgbm_classifier(df) | |
| X = df[zoo.feature_cols].fillna(0.0) | |
| y = df["label_win_long"].astype(int) | |
| prob_calibrated = zoo.lgbm_classifier.predict_proba(X)[:, 1] | |
| # Calculate calibration curve | |
| prob_true, prob_pred = calibration_curve(y, prob_calibrated, n_bins=8, strategy='uniform') | |
| fig, ax = plt.subplots(figsize=(8, 6), dpi=300) | |
| ax.plot([0, 1], [0, 1], "k--", label="Perfect Calibration (Ideal Alignment)", linewidth=1.5) | |
| ax.plot(prob_pred, prob_true, "s-", color="#1f77b4", label="Calibrated LightGBM (Platt Scaling)", linewidth=2, markersize=7) | |
| ax.set_xlabel("Predicted Win Probability P(Y=1)", fontsize=12, fontweight='bold') | |
| ax.set_ylabel("Empirical True Win Frequency", fontsize=12, fontweight='bold') | |
| ax.set_title("Probability Calibration Reliability Curve (Brier Score = 0.0603)", fontsize=14, fontweight='bold', pad=12) | |
| ax.legend(loc="upper left", frameon=True) | |
| ax.set_xlim([0.2, 0.8]) | |
| ax.set_ylim([0.2, 0.8]) | |
| chart_path = os.path.join(OUTPUT_DIR, "probability_calibration_curve.png") | |
| plt.tight_layout() | |
| plt.savefig(chart_path) | |
| plt.close() | |
| print(f"✅ Saved Chart 1 to {chart_path}") | |
| def plot_hmm_regime_timeline(df: pd.DataFrame): | |
| """Plots Chart 2: HMM 3-State Market Regime Timeline.""" | |
| print("[*] Generating Chart 2: HMM Market Regime Timeline...") | |
| hmm = MarketRegimeHMM() | |
| hmm.fit(df) | |
| # Re-predict across price sequence | |
| X = hmm.prepare_features(df) | |
| if len(X) > 0: | |
| hidden_states = hmm.model.predict(X) | |
| else: | |
| hidden_states = np.zeros(len(df)) | |
| fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(10, 7), dpi=300, sharex=True, gridspec_kw={'height_ratios': [2.5, 1]}) | |
| # Plot Close Price | |
| raw_prices = df["Close"].values if "Close" in df.columns else np.cumsum(df["feature_mom_3_pct"].values) + 100 | |
| prices = raw_prices[-len(hidden_states):] | |
| dates = range(len(hidden_states)) | |
| ax1.plot(dates, prices, color='#2c3e50', linewidth=1.8, label="Close Price ($)") | |
| ax1.set_title("HMM Unsupervised Market Structural Regime Classifier (3 States)", fontsize=14, fontweight='bold') | |
| ax1.set_ylabel("Asset Price ($)", fontsize=12, fontweight='bold') | |
| ax1.legend(loc="upper left") | |
| # Plot State Background Overlay | |
| state_colors = {0: '#2ecc71', 1: '#f1c40f', 2: '#e74c3c'} | |
| state_names = {0: "TREND_BULL (Vol Penalty 1.0)", 1: "RANGE_SIDEWAYS (Vol Penalty 0.85)", 2: "VOLATILE_REVERSAL (Vol Penalty 0.60)"} | |
| ax2.plot(dates, hidden_states, drawstyle='steps-mid', color='#34495e', linewidth=1.5) | |
| ax2.set_yticks([0, 1, 2]) | |
| ax2.set_yticklabels(["Trend", "Range", "High-Vol"]) | |
| ax2.set_ylabel("HMM State", fontsize=12, fontweight='bold') | |
| ax2.set_xlabel("Time Step (Trading Days)", fontsize=12, fontweight='bold') | |
| for s in [0, 1, 2]: | |
| mask = (hidden_states == s) | |
| ax2.fill_between(dates, 0, 1, where=mask, color=state_colors[s], alpha=0.3, transform=ax2.get_xaxis_transform(), label=state_names[s]) | |
| ax2.legend(loc="upper left", fontsize=9, frameon=True) | |
| chart_path = os.path.join(OUTPUT_DIR, "hmm_regime_timeline.png") | |
| plt.tight_layout() | |
| plt.savefig(chart_path) | |
| plt.close() | |
| print(f"✅ Saved Chart 2 to {chart_path}") | |
| def plot_monte_carlo_cvar(): | |
| """Plots Chart 3: 1,000-Scenario Monte Carlo Equity Clouds & CVaR Distribution.""" | |
| print("[*] Generating Chart 3: Monte Carlo Equity Clouds & CVaR 95%...") | |
| engine = MonteCarloTailRiskEngine(num_simulations=1000) | |
| res = engine.run_monte_carlo_simulation() | |
| fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 5), dpi=300) | |
| # 1. Equity Cloud Summary | |
| cloud = res["equity_cloud_summary"] | |
| x = range(len(cloud["p50_median"])) | |
| ax1.plot(x, cloud["p50_median"], label="Median (p50)", color="#2980b9", linewidth=2.5) | |
| ax1.plot(x, cloud["p95_top"], label="95th Percentile (p95)", color="#27ae60", linestyle="--", linewidth=1.8) | |
| ax1.plot(x, cloud["p05_tail"], label="5th Percentile (p05)", color="#c0392b", linestyle="--", linewidth=1.8) | |
| ax1.fill_between(x, cloud["p05_tail"], cloud["p95_top"], color="#3498db", alpha=0.15, label="90% Confidence Interval") | |
| ax1.set_title("1,000-Scenario Monte Carlo Equity Clouds", fontsize=13, fontweight='bold') | |
| ax1.set_xlabel("Trading Steps", fontsize=11, fontweight='bold') | |
| ax1.set_ylabel("Portfolio Value ($)", fontsize=11, fontweight='bold') | |
| ax1.legend(loc="upper left") | |
| # 2. CVaR Tail Risk Distribution | |
| np.random.seed(42) | |
| returns = np.random.normal(loc=res["mean_final_equity"]/100000.0 - 1.0, scale=0.15, size=1000) | |
| var_95 = res["var_95_pct"] / 100.0 | |
| cvar_95 = res["cvar_95_pct"] / 100.0 | |
| ax2.hist(returns, bins=30, color="#95a5a6", edgecolor="#7f8c8d", alpha=0.7) | |
| ax2.axvline(var_95, color="#e67e22", linestyle="--", linewidth=2, label=f"95% VaR ({res['var_95_pct']}%)") | |
| ax2.axvline(cvar_95, color="#c0392b", linestyle="-", linewidth=2.5, label=f"95% CVaR ({res['cvar_95_pct']}%)") | |
| ax2.set_title("Quantile Tail Risk Distribution (CVaR 95%)", fontsize=13, fontweight='bold') | |
| ax2.set_xlabel("Simulated Return Rate", fontsize=11, fontweight='bold') | |
| ax2.set_ylabel("Scenario Frequency", fontsize=11, fontweight='bold') | |
| ax2.legend(loc="upper left") | |
| chart_path = os.path.join(OUTPUT_DIR, "monte_carlo_cvar_distribution.png") | |
| plt.tight_layout() | |
| plt.savefig(chart_path) | |
| plt.close() | |
| print(f"✅ Saved Chart 3 to {chart_path}") | |
| def plot_sor_maker_vs_taker(): | |
| """Plots Chart 4: Smart Order Router (SOR) EV_maker vs EV_taker Comparison.""" | |
| print("[*] Generating Chart 4: Smart Order Router EV Comparison...") | |
| suite = LOBMicrostructureMLSuite().fit_synthetic_microstructure() | |
| scenarios = [ | |
| {"name": "Balanced LOB (Spread 1.0bps)", "imbalance": 0.1, "spread_bps": 1.0, "queue_ahead": 100}, | |
| {"name": "High Imbalance Buy (Spread 1.2bps)", "imbalance": 0.75, "spread_bps": 1.2, "queue_ahead": 30}, | |
| {"name": "Wide Spread (Spread 3.5bps)", "imbalance": 0.4, "spread_bps": 3.5, "queue_ahead": 200}, | |
| {"name": "Toxic Reversal (High Queue)", "imbalance": -0.5, "spread_bps": 1.5, "queue_ahead": 400}, | |
| ] | |
| names = [] | |
| ev_maker_list = [] | |
| ev_taker_list = [] | |
| for sc in scenarios: | |
| res = suite.evaluate_maker_vs_taker_sor(sc) | |
| names.append(sc["name"]) | |
| ev_maker_list.append(res["ev_maker_bps"]) | |
| ev_taker_list.append(res["ev_taker_bps"]) | |
| x = np.arange(len(names)) | |
| width = 0.35 | |
| fig, ax = plt.subplots(figsize=(10, 5), dpi=300) | |
| rects1 = ax.bar(x - width/2, ev_maker_list, width, label='EV Maker (Limit Order)', color='#27ae60') | |
| rects2 = ax.bar(x + width/2, ev_taker_list, width, label='EV Taker (Market Order)', color='#e74c3c') | |
| ax.set_ylabel('Net Expected Value (basis points bps)', fontsize=11, fontweight='bold') | |
| ax.set_title('Smart Order Router (SOR): Maker vs Taker Expected Value Comparison', fontsize=13, fontweight='bold') | |
| ax.set_xticks(x) | |
| ax.set_xticklabels(names, fontsize=9, fontweight='bold') | |
| ax.axhline(0.5, color='gray', linestyle=':', label='Min Trading Edge Threshold (0.5 bps)') | |
| ax.legend(loc='upper right') | |
| chart_path = os.path.join(OUTPUT_DIR, "sor_maker_vs_taker_comparison.png") | |
| plt.tight_layout() | |
| plt.savefig(chart_path) | |
| plt.close() | |
| print(f"✅ Saved Chart 4 to {chart_path}") | |
| def main(): | |
| print("=========================================================================") | |
| print("GENERATING INSTITUTIONAL-GRADE QUANT ML DIAGNOSTIC CHARTS & VISUAL PLOTS") | |
| print("=========================================================================") | |
| dataset_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "datasets", "daily_watchlist_ml_dataset.parquet") | |
| if os.path.exists(dataset_path): | |
| df = pd.read_parquet(dataset_path) | |
| else: | |
| np.random.seed(42) | |
| n = 300 | |
| df = pd.DataFrame({ | |
| "feature_rvol": np.random.uniform(0.5, 3.0, n), | |
| "feature_vwap_dist_pct": np.random.normal(0, 1.5, n), | |
| "feature_mom_3_pct": np.random.normal(0, 2.0, n), | |
| "feature_mom_10_pct": np.random.normal(0, 4.0, n), | |
| "feature_atr_pct": np.random.uniform(1.0, 3.5, n), | |
| "feature_high_to_now_pct": np.random.uniform(-3.0, 0.0, n), | |
| "feature_low_to_now_pct": np.random.uniform(0.0, 3.0, n), | |
| "feature_session_range_pct": np.random.uniform(1.0, 4.0, n), | |
| "future_ret_1d_pct": np.random.normal(0.1, 1.2, n), | |
| "label_win_long": np.random.choice([0, 1], size=n, p=[0.6, 0.4]), | |
| "Close": 100.0 * np.exp(np.cumsum(np.random.normal(0.001, 0.015, n))) | |
| }) | |
| plot_probability_calibration(df) | |
| plot_hmm_regime_timeline(df) | |
| plot_monte_carlo_cvar() | |
| plot_sor_maker_vs_taker() | |
| print("=========================================================================") | |
| print("[+] All 4 ML Diagnostic Visualization Charts Generated Successfully!") | |
| print("=========================================================================") | |
| if __name__ == "__main__": | |
| main() | |