PROSTLE commited on
Commit
bea598f
Β·
unverified Β·
1 Parent(s): 697957e

Create backtest_lstm

Browse files
Files changed (1) hide show
  1. backend/trained_models/backtest_lstm +368 -0
backend/trained_models/backtest_lstm ADDED
@@ -0,0 +1,368 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ StockSense AI β€” Backtest Engine
3
+ ================================
4
+ Validates the 5-factor prediction system against real historical data.
5
+
6
+ Usage:
7
+ python backtest_lstm.py # Interactive mode
8
+ python backtest_lstm.py RELIANCE.NS # Single stock
9
+ python backtest_lstm.py RELIANCE.NS TCS.NS INFY.NS # Multiple stocks
10
+
11
+ What it does:
12
+ 1. Fetches 2 years of real market data
13
+ 2. Picks evaluation windows every 5 trading days (non-overlapping)
14
+ 3. Runs the FULL prediction engine (LSTM + Enterprise + Technical) on each window
15
+ 4. Compares 5-day predicted prices vs actual prices
16
+ 5. Reports: price accuracy %, direction accuracy %, per-day breakdown
17
+ """
18
+
19
+ import os
20
+ import sys
21
+ import time
22
+ import numpy as np
23
+ import pandas as pd
24
+ from datetime import datetime
25
+
26
+ # Add backend to path
27
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
28
+
29
+ from stock_data import fetch_stock_data, add_technical_indicators
30
+ from model import train_and_predict, _prediction_cache
31
+
32
+
33
+ # ── Configuration ─────────────────────────────────────────────────────────────
34
+ LOOKBACK = 60 # Days of history needed before prediction window
35
+ PREDICT_DAYS = 5 # Must match model.py PREDICT_DAYS
36
+ STEP_SIZE = 5 # Evaluate every N trading days (non-overlapping)
37
+ MIN_DATA_ROWS = 200 # Minimum rows after indicators (NaN drop)
38
+ MAX_WINDOWS = 30 # Cap evaluation windows to keep runtime reasonable
39
+
40
+
41
+ def backtest_stock(ticker: str, verbose: bool = True) -> dict:
42
+ """
43
+ Run full backtest on a single stock.
44
+ Returns a dict with all accuracy metrics.
45
+ """
46
+ if verbose:
47
+ print(f"\n{'='*70}")
48
+ print(f" BACKTESTING: {ticker}")
49
+ print(f"{'='*70}")
50
+
51
+ # ── 1. Fetch full historical data ─────────────────────────────────────
52
+ if verbose:
53
+ print(f" Fetching 2 years of data...")
54
+
55
+ try:
56
+ df_full = fetch_stock_data(ticker, period="2y", interval="1d")
57
+ df_full = add_technical_indicators(df_full)
58
+ except Exception as e:
59
+ print(f" ERROR: Could not fetch data for {ticker}: {e}")
60
+ return {"ticker": ticker, "error": str(e)}
61
+
62
+ df_full = df_full.dropna().reset_index(drop=True)
63
+
64
+ if len(df_full) < MIN_DATA_ROWS:
65
+ print(f" ERROR: Only {len(df_full)} rows after cleanup (need {MIN_DATA_ROWS}+)")
66
+ return {"ticker": ticker, "error": "Insufficient data"}
67
+
68
+ if verbose:
69
+ print(f" Data: {len(df_full)} trading days")
70
+ first_date = str(df_full.iloc[0]["Date"])[:10] if "Date" in df_full.columns else "N/A"
71
+ last_date = str(df_full.iloc[-1]["Date"])[:10] if "Date" in df_full.columns else "N/A"
72
+ print(f" Range: {first_date} β†’ {last_date}")
73
+
74
+ # ── 2. Build evaluation windows ──────────────────────────────────────
75
+ # Each window: use df[0 : cutoff] for training, compare prediction vs df[cutoff : cutoff+5]
76
+ windows = []
77
+ # Start from LOOKBACK+50 (need enough data for LSTM + enterprise training)
78
+ start_from = max(LOOKBACK + 50, len(df_full) - MAX_WINDOWS * STEP_SIZE - PREDICT_DAYS)
79
+ start_from = max(LOOKBACK + 50, start_from)
80
+
81
+ for cutoff in range(start_from, len(df_full) - PREDICT_DAYS, STEP_SIZE):
82
+ hist_df = df_full.iloc[:cutoff].copy()
83
+ future_df = df_full.iloc[cutoff:cutoff + PREDICT_DAYS]
84
+
85
+ if len(hist_df) < LOOKBACK + 50 or len(future_df) < PREDICT_DAYS:
86
+ continue
87
+
88
+ actual_prices = future_df["Close"].values.tolist()
89
+ current_price = float(hist_df["Close"].iloc[-1])
90
+
91
+ if "Date" in hist_df.columns:
92
+ pred_date = str(hist_df["Date"].iloc[-1])[:10]
93
+ else:
94
+ pred_date = f"window_{cutoff}"
95
+
96
+ windows.append({
97
+ "cutoff": cutoff,
98
+ "hist_df": hist_df,
99
+ "actual_prices": actual_prices,
100
+ "current_price": current_price,
101
+ "date": pred_date,
102
+ })
103
+
104
+ if not windows:
105
+ print(f" ERROR: No valid evaluation windows found")
106
+ return {"ticker": ticker, "error": "No valid windows"}
107
+
108
+ # Cap at MAX_WINDOWS (take the most recent ones)
109
+ if len(windows) > MAX_WINDOWS:
110
+ windows = windows[-MAX_WINDOWS:]
111
+
112
+ if verbose:
113
+ print(f" Evaluation windows: {len(windows)} (every {STEP_SIZE} days)")
114
+ print(f" Windows span: {windows[0]['date']} β†’ {windows[-1]['date']}")
115
+ print()
116
+
117
+ # ── 3. Run predictions on each window ────────────────────────────────
118
+ results = []
119
+ total_windows = len(windows)
120
+
121
+ for idx, w in enumerate(windows):
122
+ # Clear prediction cache so each window trains fresh
123
+ _prediction_cache.clear()
124
+
125
+ if verbose:
126
+ pct_done = int((idx / total_windows) * 100)
127
+ print(f" [{pct_done:3d}%] Window {idx+1}/{total_windows}: {w['date']} ", end="", flush=True)
128
+
129
+ t0 = time.time()
130
+ try:
131
+ prediction = train_and_predict(w["hist_df"])
132
+ elapsed = time.time() - t0
133
+ pred_prices = prediction["predicted_prices"]
134
+ actual = w["actual_prices"]
135
+ current = w["current_price"]
136
+
137
+ # ── Per-day metrics ──
138
+ day_errors = []
139
+ day_pct_errors = []
140
+ direction_hits = []
141
+
142
+ for day in range(PREDICT_DAYS):
143
+ p = pred_prices[day]
144
+ a = actual[day]
145
+
146
+ # Absolute error
147
+ err = abs(p - a)
148
+ day_errors.append(err)
149
+
150
+ # Percentage error
151
+ pct_err = abs(p - a) / a * 100 if a != 0 else 0
152
+ day_pct_errors.append(pct_err)
153
+
154
+ # Direction accuracy (did we predict up/down correctly?)
155
+ pred_dir = "up" if p > current else "down"
156
+ actual_dir = "up" if a > current else "down"
157
+ direction_hits.append(1 if pred_dir == actual_dir else 0)
158
+
159
+ result = {
160
+ "date": w["date"],
161
+ "current_price": round(current, 2),
162
+ "predicted": [round(p, 2) for p in pred_prices],
163
+ "actual": [round(a, 2) for a in actual],
164
+ "mae": round(np.mean(day_errors), 2),
165
+ "mape": round(np.mean(day_pct_errors), 2),
166
+ "price_accuracy": round(100 - np.mean(day_pct_errors), 2),
167
+ "direction_accuracy": round(np.mean(direction_hits) * 100, 1),
168
+ "day5_pred": round(pred_prices[-1], 2),
169
+ "day5_actual": round(actual[-1], 2),
170
+ "day5_pct_error": round(abs(pred_prices[-1] - actual[-1]) / actual[-1] * 100, 2),
171
+ "confidence": prediction.get("confidence", 0),
172
+ "risk": prediction.get("risk", "?"),
173
+ "mode": prediction.get("factor_breakdown", {}).get("mode", "Normal"),
174
+ "time_sec": round(elapsed, 1),
175
+ }
176
+ results.append(result)
177
+
178
+ if verbose:
179
+ acc = result["price_accuracy"]
180
+ dir_acc = result["direction_accuracy"]
181
+ emoji = "βœ…" if acc >= 95 else "⚠️" if acc >= 90 else "❌"
182
+ print(f"Price: {acc:.1f}% | Direction: {dir_acc:.0f}% | {result['mode']} {emoji} ({elapsed:.1f}s)")
183
+
184
+ except Exception as e:
185
+ elapsed = time.time() - t0
186
+ if verbose:
187
+ print(f"FAILED: {str(e)[:60]} ({elapsed:.1f}s)")
188
+ continue
189
+
190
+ if not results:
191
+ print(f" ERROR: All prediction windows failed")
192
+ return {"ticker": ticker, "error": "All windows failed"}
193
+
194
+ # ── 4. Aggregate metrics ─────────────────────────────────────────────
195
+ df_results = pd.DataFrame(results)
196
+
197
+ avg_price_accuracy = round(df_results["price_accuracy"].mean(), 2)
198
+ avg_direction_accuracy = round(df_results["direction_accuracy"].mean(), 1)
199
+ avg_mape = round(df_results["mape"].mean(), 2)
200
+ avg_mae = round(df_results["mae"].mean(), 2)
201
+ avg_confidence = round(df_results["confidence"].mean(), 1)
202
+ best_accuracy = round(df_results["price_accuracy"].max(), 2)
203
+ worst_accuracy = round(df_results["price_accuracy"].min(), 2)
204
+ crisis_windows = int((df_results["mode"] == "Crisis").sum())
205
+
206
+ # Per-day breakdown
207
+ day_accuracies = []
208
+ for day in range(PREDICT_DAYS):
209
+ day_errs = []
210
+ for r in results:
211
+ p = r["predicted"][day]
212
+ a = r["actual"][day]
213
+ pct = abs(p - a) / a * 100 if a else 0
214
+ day_errs.append(pct)
215
+ day_acc = round(100 - np.mean(day_errs), 2)
216
+ day_accuracies.append(day_acc)
217
+
218
+ summary = {
219
+ "ticker": ticker,
220
+ "windows_tested": len(results),
221
+ "date_range": f"{results[0]['date']} β†’ {results[-1]['date']}",
222
+ "avg_price_accuracy_pct": avg_price_accuracy,
223
+ "avg_direction_accuracy_pct": avg_direction_accuracy,
224
+ "avg_mape_pct": avg_mape,
225
+ "avg_mae_rupees": avg_mae,
226
+ "avg_confidence": avg_confidence,
227
+ "best_window_accuracy": best_accuracy,
228
+ "worst_window_accuracy": worst_accuracy,
229
+ "crisis_mode_windows": crisis_windows,
230
+ "per_day_accuracy": day_accuracies,
231
+ "results": results,
232
+ }
233
+
234
+ # ── 5. Print report ──────────────────────────────────────────────────
235
+ if verbose:
236
+ print(f"\n{'─'*70}")
237
+ print(f" πŸ“Š BACKTEST RESULTS: {ticker}")
238
+ print(f"{'─'*70}")
239
+ print(f" Windows tested : {len(results)}")
240
+ print(f" Date range : {results[0]['date']} β†’ {results[-1]['date']}")
241
+ print()
242
+ print(f" β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”")
243
+ print(f" β”‚ PRICE ACCURACY : {avg_price_accuracy:6.2f}% β”‚")
244
+ print(f" β”‚ DIRECTION ACCURACY : {avg_direction_accuracy:6.1f}% β”‚")
245
+ print(f" β”‚ AVG ERROR (MAPE) : {avg_mape:6.2f}% β”‚")
246
+ print(f" β”‚ AVG ERROR (MAE) : β‚Ή{avg_mae:<8.2f} β”‚")
247
+ print(f" β”‚ AVG CONFIDENCE : {avg_confidence:6.1f}% β”‚")
248
+ print(f" β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜")
249
+ print()
250
+ print(f" Per-Day Accuracy:")
251
+ for d, acc in enumerate(day_accuracies):
252
+ bar = "β–ˆ" * int(acc / 2) + "β–‘" * (50 - int(acc / 2))
253
+ print(f" Day {d+1}: {bar} {acc:.1f}%")
254
+ print()
255
+ print(f" Best window : {best_accuracy}%")
256
+ print(f" Worst window : {worst_accuracy}%")
257
+ print(f" Crisis mode : {crisis_windows}/{len(results)} windows")
258
+
259
+ # ── 6. Save CSV ──────────────────────────────────────────────────────
260
+ csv_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), f"{ticker.replace('.', '_')}_backtest.csv")
261
+ df_results.to_csv(csv_path, index=False)
262
+ if verbose:
263
+ print(f"\n πŸ“ Detailed results saved to: {csv_path}")
264
+
265
+ return summary
266
+
267
+
268
+ def backtest_multiple(tickers: list[str]) -> dict:
269
+ """Run backtest on multiple stocks and produce a combined report."""
270
+ all_summaries = []
271
+
272
+ for ticker in tickers:
273
+ summary = backtest_stock(ticker, verbose=True)
274
+ if "error" not in summary:
275
+ all_summaries.append(summary)
276
+
277
+ if not all_summaries:
278
+ print("\n❌ No successful backtests.")
279
+ return {}
280
+
281
+ # ── Combined report ──
282
+ print(f"\n\n{'='*70}")
283
+ print(f" πŸ“Š COMBINED BACKTEST REPORT β€” {len(all_summaries)} STOCKS")
284
+ print(f"{'='*70}")
285
+ print()
286
+ print(f" {'Stock':<18} {'Price Acc':>10} {'Dir Acc':>10} {'MAPE':>8} {'Confidence':>12} {'Windows':>8}")
287
+ print(f" {'─'*18} {'─'*10} {'─'*10} {'─'*8} {'─'*12} {'─'*8}")
288
+
289
+ total_acc = []
290
+ total_dir = []
291
+
292
+ for s in all_summaries:
293
+ acc = s["avg_price_accuracy_pct"]
294
+ dir_acc = s["avg_direction_accuracy_pct"]
295
+ total_acc.append(acc)
296
+ total_dir.append(dir_acc)
297
+ emoji = "βœ…" if acc >= 95 else "⚠️" if acc >= 90 else "❌"
298
+ print(f" {s['ticker']:<18} {acc:>9.2f}% {dir_acc:>9.1f}% {s['avg_mape_pct']:>7.2f}% {s['avg_confidence']:>11.1f}% {s['windows_tested']:>7} {emoji}")
299
+
300
+ print(f" {'─'*18} {'─'*10} {'─'*10} {'─'*8} {'─'*12} {'─'*8}")
301
+ avg_price = np.mean(total_acc)
302
+ avg_dir = np.mean(total_dir)
303
+ print(f" {'OVERALL':<18} {avg_price:>9.2f}% {avg_dir:>9.1f}%")
304
+ print()
305
+
306
+ grade = "A+" if avg_price >= 97 else "A" if avg_price >= 95 else "B+" if avg_price >= 93 else "B" if avg_price >= 90 else "C" if avg_price >= 85 else "D"
307
+ print(f" πŸ† Overall Grade: {grade}")
308
+ print(f" Average Price Accuracy: {avg_price:.2f}%")
309
+ print(f" Average Direction Accuracy: {avg_dir:.1f}%")
310
+ print()
311
+
312
+ return {
313
+ "stocks_tested": len(all_summaries),
314
+ "overall_price_accuracy": round(avg_price, 2),
315
+ "overall_direction_accuracy": round(avg_dir, 1),
316
+ "grade": grade,
317
+ "summaries": all_summaries,
318
+ }
319
+
320
+
321
+ # ── Default tickers for multi-stock backtest ──────────────────────────────────
322
+ DEFAULT_TICKERS = [
323
+ "RELIANCE.NS",
324
+ "TCS.NS",
325
+ "INFY.NS",
326
+ "HDFCBANK.NS",
327
+ "TATAMOTORS.NS",
328
+ ]
329
+
330
+
331
+ if __name__ == "__main__":
332
+ print()
333
+ print("╔══════════════════════════════════════════════════════════╗")
334
+ print("β•‘ StockSense AI β€” Prediction Backtest Engine β•‘")
335
+ print("β•‘ 5-Factor Model: LSTM + Enterprise + Tech + News β•‘")
336
+ print("β•‘ + Gemini LLM Analysis β•‘")
337
+ print("β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•")
338
+ print()
339
+
340
+ if len(sys.argv) > 1:
341
+ tickers = sys.argv[1:]
342
+ else:
343
+ print("Options:")
344
+ print(" 1. Single stock backtest")
345
+ print(" 2. Multi-stock backtest (5 Nifty stocks)")
346
+ print(" 3. Custom stocks")
347
+ print()
348
+ choice = input("Choose (1/2/3): ").strip()
349
+
350
+ if choice == "1":
351
+ ticker = input("Enter stock ticker (e.g. RELIANCE.NS): ").strip()
352
+ if not ticker:
353
+ ticker = "RELIANCE.NS"
354
+ tickers = [ticker]
355
+ elif choice == "2":
356
+ tickers = DEFAULT_TICKERS
357
+ elif choice == "3":
358
+ raw = input("Enter tickers separated by commas: ").strip()
359
+ tickers = [t.strip() for t in raw.split(",") if t.strip()]
360
+ else:
361
+ tickers = DEFAULT_TICKERS
362
+
363
+ if len(tickers) == 1:
364
+ backtest_stock(tickers[0], verbose=True)
365
+ else:
366
+ backtest_multiple(tickers)
367
+
368
+ print("\nβœ… Backtest complete.")