Upload 6 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+lstm_model.keras filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,393 @@
+# app.py — DataSynthis_ML_JobTask (ARIMA + optional LSTM, simple UI)
+
+import os, json, math, pickle, traceback
+from datetime import datetime, timedelta
+
+import numpy as np
+import pandas as pd
+import yfinance as yf
+from pandas.tseries.offsets import BDay
+import matplotlib.pyplot as plt
+import gradio as gr
+
+from statsmodels.tsa.arima.model import ARIMA
+from sklearn.metrics import mean_squared_error, mean_absolute_error, mean_absolute_percentage_error
+
+# -------------------------
+# Artifact Paths
+# -------------------------
+ARIMA_ORDER_JSON = "arima_order.json"     # preferred way to get (p,d,q)
+ARIMA_PICKLE = "arima_model.pkl"          # optional local archive (not used for inference)
+LSTM_KERAS = "lstm_model.keras"           # optional: pre-trained LSTM (Keras 3)
+LSTM_INFO = "lstm_info.json"              # optional: {"lookback": int, "train_min": float, "train_max": float}
+
+# -------------------------
+# Globals (lazy-load LSTM)
+# -------------------------
+_LSTM_MODEL = None
+_LSTM_META = None
+
+# -------------------------
+# Data utilities
+# -------------------------
+def fetch_data(ticker: str, start: str, end: str) -> pd.DataFrame:
+    """
+    Download daily data, convert to business-day frequency, forward-fill close.
+    """
+    df = yf.download(ticker, start=start, end=end, interval="1d",
+                     auto_adjust=True, progress=False)
+    if df is None or df.empty:
+        raise ValueError(f"No data returned for '{ticker}'. Check ticker or dates.")
+    df = df[['Close']].copy()
+    if df.index.tz is not None:
+        df.index = df.index.tz_convert(None)
+    df = df.asfreq('B')
+    df['Close'] = df['Close'].ffill()
+    return df
+
+def load_arima_order(default=(2, 1, 2)):
+    """
+    Prefer reading (p,d,q) from arima_order.json; fall back to pickle; else default.
+    """
+    # JSON first (version-proof)
+    if os.path.isfile(ARIMA_ORDER_JSON):
+        try:
+            with open(ARIMA_ORDER_JSON, "r") as f:
+                obj = json.load(f)
+            order = tuple(int(x) for x in obj.get("order", list(default)))
+            return order
+        except Exception as e:
+            print(f"[WARN] Failed to read {ARIMA_ORDER_JSON}: {e}")
+
+    # Fallback: try to extract order from pickle (not used for inference otherwise)
+    if os.path.isfile(ARIMA_PICKLE):
+        try:
+            with open(ARIMA_PICKLE, "rb") as f:
+                res = pickle.load(f)
+            if hasattr(res, "model") and hasattr(res.model, "order"):
+                return res.model.order
+        except Exception as e:
+            print(f"[WARN] Failed to read order from {ARIMA_PICKLE}: {e}")
+
+    # Final fallback
+    return default
+
+# -------------------------
+# ARIMA evaluation / forecast
+# -------------------------
+def arima_roll_single_fit(train_s: pd.Series, test_s: pd.Series, order=(2, 1, 2)):
+    """
+    Fair evaluation: fit once on TRAIN, then roll 1-step ahead through TEST.
+    Uses ARIMAResults.append() to update state; refits if append not available.
+    """
+    res = ARIMA(train_s, order=order,
+                enforce_stationarity=False, enforce_invertibility=False).fit()
+    preds, trues = [], []
+    for i in range(len(test_s)):
+        preds.append(res.forecast(1)[0])
+        y_true = test_s.iloc[i]
+        trues.append(y_true)
+        try:
+            # Fast state update
+            res = res.append(test_s.iloc[i:i+1])
+        except Exception:
+            # Rare fallback: refit to train + observed test slice
+            res = ARIMA(pd.concat([train_s, test_s.iloc[:i+1]]), order=order,
+                        enforce_stationarity=False, enforce_invertibility=False).fit()
+    return np.array(trues), np.array(preds)
+
+def arima_future_forecast(all_close: pd.Series, order, steps: int) -> np.ndarray:
+    """
+    Fit ARIMA on all data and forecast next 'steps' business days.
+    """
+    fitted = ARIMA(all_close, order=order,
+                   enforce_stationarity=False, enforce_invertibility=False).fit()
+    fc = fitted.get_forecast(steps=steps)
+    return np.asarray(fc.predicted_mean).reshape(-1,)
+
+# -------------------------
+# LSTM helpers (Keras .keras + simple min-max scaling)
+# -------------------------
+def _maybe_load_lstm() -> bool:
+    """
+    Load LSTM and metadata once, if present. Returns True if available.
+    """
+    global _LSTM_MODEL, _LSTM_META
+    if _LSTM_MODEL is not None and _LSTM_META is not None:
+        return True
+    if not (os.path.isfile(LSTM_KERAS) and os.path.isfile(LSTM_INFO)):
+        return False
+    try:
+        import tensorflow as tf  # import lazily
+        _LSTM_MODEL = tf.keras.models.load_model(LSTM_KERAS, compile=False)
+        with open(LSTM_INFO, "r") as f:
+            _LSTM_META = json.load(f)
+        assert all(k in _LSTM_META for k in ("lookback", "train_min", "train_max"))
+        return True
+    except Exception as e:
+        print(f"[WARN] LSTM load failed: {e}")
+        _LSTM_MODEL, _LSTM_META = None, None
+        return False
+
+def _scale(x, xmin, xmax):
+    denom = (xmax - xmin) if (xmax - xmin) != 0 else 1.0
+    return (x - xmin) / denom
+
+def _unscale(x, xmin, xmax):
+    return x * (xmax - xmin) + xmin
+
+def _seq(data_1d: np.ndarray, L: int):
+    X, y = [], []
+    for i in range(L, len(data_1d)):
+        X.append(data_1d[i-L:i])
+        y.append(data_1d[i])
+    X = np.asarray(X)[..., None]   # (N, L, 1)
+    y = np.asarray(y)
+    return X, y
+
+def lstm_roll_eval_and_forecast(close_train: pd.Series, close_test: pd.Series, horizon: int):
+    """
+    Use the fixed pre-trained LSTM:
+      (1) fair 1-step-ahead evaluation across TEST
+      (2) recursive forecast for 'horizon' steps
+    """
+    if not _maybe_load_lstm():
+        return None, None, None
+
+    import tensorflow as tf  # assured by requirements
+    model = _LSTM_MODEL
+    L      = int(_LSTM_META["lookback"])
+    xmin   = float(_LSTM_META["train_min"])
+    xmax   = float(_LSTM_META["train_max"])
+
+    # Scale with training min/max
+    train_vals = close_train.values.astype("float32")
+    test_vals  = close_test.values.astype("float32")
+
+    train_s = _scale(train_vals, xmin, xmax)
+    test_s  = _scale(test_vals,  xmin, xmax)
+
+    # Build windows over train+test to start test windows with exactly L prior points
+    all_scaled = np.concatenate([train_s, test_s], axis=0)
+    X_all, y_all = _seq(all_scaled, L)
+
+    test_start_idx = len(train_s) - L
+    if test_start_idx < 0:
+        return None, None, None  # not enough history for the lookback
+
+    X_test = X_all[test_start_idx : test_start_idx + len(test_s)]
+    y_test = y_all[test_start_idx : test_start_idx + len(test_s)]
+
+    # Predict test
+    y_pred_scaled = model.predict(X_test, verbose=0).reshape(-1)
+    y_true = _unscale(y_test, xmin, xmax)
+    y_pred = _unscale(y_pred_scaled, xmin, xmax)
+
+    # Recursive future forecast using last L points from all history
+    hist_vals = np.concatenate([train_vals, test_vals], axis=0)
+    hist_scaled = _scale(hist_vals, xmin, xmax)
+    window = hist_scaled[-L:].copy()
+
+    fut = []
+    for _ in range(horizon):
+        x = window.reshape(1, L, 1)
+        yhat_s = model.predict(x, verbose=0).reshape(-1)[0]
+        fut.append(_unscale(yhat_s, xmin, xmax))
+        # roll the window (note: keep in scaled space)
+        window = np.concatenate([window[1:], [yhat_s]], axis=0)
+
+    return y_true, y_pred, np.asarray(fut)
+
+# -------------------------
+# Metrics / formatting / plotting
+# -------------------------
+def metrics_df(y_true, y_pred) -> pd.DataFrame:
+    rmse = math.sqrt(mean_squared_error(y_true, y_pred))
+    mae  = mean_absolute_error(y_true, y_pred)
+    mape = mean_absolute_percentage_error(y_true, y_pred) * 100
+    return pd.DataFrame({"RMSE": [rmse], "MAE": [mae], "MAPE (%)": [mape]})
+
+def format_prices_df(df: pd.DataFrame, date_col: str, value_col: str, value_label: str):
+    dates_str = pd.Index(df[date_col]).astype(str)
+    vals = np.asarray(df[value_col]).reshape(-1,)
+    return pd.DataFrame({"Date": dates_str, value_label: np.round(vals, 2)})
+
+def make_chart(hist_df: pd.DataFrame,
+               arima_fut: pd.DataFrame | None,
+               lstm_fut: pd.DataFrame | None):
+    try:
+        h = hist_df.copy()
+        h['Date'] = pd.to_datetime(h['Date'])
+        h_last = h.iloc[-252:] if len(h) > 252 else h
+
+        fig, ax = plt.subplots(figsize=(9, 4))
+        ax.plot(h_last['Date'], h_last['Close ($)'], label="Historical Close", linewidth=2)
+
+        if arima_fut is not None and not arima_fut.empty:
+            f = arima_fut.copy()
+            f['Date'] = pd.to_datetime(f['Date'])
+            ax.plot(f['Date'], f['Forecast ($)'], label="ARIMA Forecast", linewidth=2)
+
+        if lstm_fut is not None and not lstm_fut.empty:
+            lf = lstm_fut.copy()
+            lf['Date'] = pd.to_datetime(lf['Date'])
+            ax.plot(lf['Date'], lf['Forecast ($)'], label="LSTM Forecast", linewidth=2, linestyle="--")
+
+        ax.set_title("Price (last ~1 year) + next forecasts")
+        ax.set_xlabel("Date"); ax.set_ylabel("USD ($)")
+        ax.grid(True, alpha=0.3); ax.legend()
+        fig.tight_layout()
+        return fig
+    except Exception as e:
+        print(f"[WARN] chart error: {e}")
+        return None
+
+# -------------------------
+# Gradio callback
+# -------------------------
+def do_forecast(ticker_choice: str, years_back: int, horizon_days: int):
+    """
+    Returns:
+      - Historical table
+      - Future ARIMA forecast table
+      - KPI markdown (ARIMA + LSTM if available)
+      - Chart (ARIMA + optional LSTM)
+      - Comparison table (ARIMA vs LSTM) if LSTM present
+    """
+    try:
+        ticker = (ticker_choice or "").strip().upper()
+        end = datetime.utcnow().date()
+        start = end - timedelta(days=int(years_back * 365.25))
+
+        df = fetch_data(ticker, start.isoformat(), end.isoformat())
+        if len(df) < 80:
+            msg = "Not enough data to evaluate. Increase 'How much history?'."
+            empty = pd.DataFrame()
+            return empty, empty, msg, None, pd.DataFrame()
+
+        split_idx = int(len(df) * 0.8)
+        order = load_arima_order(default=(2, 1, 2))
+
+        # ---------- ARIMA ----------
+        train_s, test_s = df['Close'][:split_idx], df['Close'][split_idx:]
+        y_true_a, y_pred_a = arima_roll_single_fit(train_s, test_s, order=order)
+        ar_tbl = metrics_df(y_true_a, y_pred_a)
+
+        fut_ar = arima_future_forecast(df['Close'], order, horizon_days)
+
+        # Tables for UI (ARIMA)
+        hist = pd.DataFrame({"date": pd.Index(df.index).astype(str),
+                             "close": np.asarray(df['Close']).reshape(-1,)})
+        future_dates = pd.bdate_range(df.index[-1] + BDay(1), periods=horizon_days)
+        fut_ar_df = pd.DataFrame({"date": pd.Index(future_dates).astype(str),
+                                  "forecast": fut_ar})
+        hist_out = format_prices_df(hist, "date", "close", "Close ($)")
+        fut_ar_out = format_prices_df(fut_ar_df, "date", "forecast", "Forecast ($)")
+
+        # ---------- LSTM (optional) ----------
+        lstm_available = _maybe_load_lstm()
+        fut_lstm_out = None
+        comp_tbl = None
+        kpi_extra = ""
+
+        if lstm_available:
+            lstm_true, lstm_pred, fut_lstm = lstm_roll_eval_and_forecast(train_s, test_s, horizon_days)
+            if lstm_true is not None:
+                lstm_tbl = metrics_df(lstm_true, lstm_pred)
+                fut_lstm_df = pd.DataFrame({"date": pd.Index(future_dates).astype(str),
+                                            "forecast": fut_lstm.reshape(-1,)})
+                fut_lstm_out = format_prices_df(fut_lstm_df, "date", "forecast", "Forecast ($)")
+
+                comp_tbl = pd.DataFrame({
+                    "Model": ["ARIMA", "LSTM"],
+                    "RMSE": [ar_tbl["RMSE"][0], lstm_tbl["RMSE"][0]],
+                    "MAE": [ar_tbl["MAE"][0], lstm_tbl["MAE"][0]],
+                    "MAPE (%)": [ar_tbl["MAPE (%)"][0], lstm_tbl["MAPE (%)"][0]],
+                }).round(3)
+
+                kpi_extra = (f"\n**LSTM** | Test RMSE ≈ {lstm_tbl['RMSE'][0]:.2f}, "
+                             f"MAE ≈ {lstm_tbl['MAE'][0]:.2f}, "
+                             f"MAPE ≈ {lstm_tbl['MAPE (%)'][0]:.2f}%")
+
+        if comp_tbl is None:
+            comp_tbl = pd.DataFrame(columns=["Model", "RMSE", "MAE", "MAPE (%)"])
+
+        # KPI text (always show ARIMA; add LSTM if available)
+        kpi = (
+            f"**ARIMA {order}** | Test RMSE ≈ {ar_tbl['RMSE'][0]:.2f}, "
+            f"MAE ≈ {ar_tbl['MAE'][0]:.2f}, MAPE ≈ {ar_tbl['MAPE (%)'][0]:.2f}%  "
+            f"| **Ticker:** {ticker}  |  **Forecast days:** {horizon_days}"
+        ) + kpi_extra
+
+        # Chart overlay
+        fig = make_chart(hist_out, fut_ar_out, fut_lstm_out)
+
+        return hist_out, fut_ar_out, kpi, fig, comp_tbl
+
+    except Exception as e:
+        traceback.print_exc()
+        err = f"❌ {str(e)}"
+        empty = pd.DataFrame()
+        return empty, empty, err, None, pd.DataFrame()
+
+# -------------------------
+# UI (simple & clear)
+# -------------------------
+POPULAR = ["AAPL", "MSFT", "GOOGL", "AMZN", "TSLA", "NVDA", "META"]
+
+with gr.Blocks(title="DataSynthis_ML_JobTask", theme=gr.themes.Soft()) as demo:
+    gr.Markdown(
+        """
+        # DataSynthis_ML_JobTask — Stock Forecasts (Simple & Clear)
+        **What it does:** Downloads recent daily prices, evaluates an ARIMA model fairly,
+        predicts the next trading days, and (if provided) compares with a pre-trained LSTM.  
+        **How to use:** Pick a stock, choose how many years of history and how far ahead to forecast, then press **Run Forecast**.
+        """
+    )
+
+    with gr.Tab("Forecast"):
+        with gr.Row():
+            ticker_choice = gr.Dropdown(
+                choices=POPULAR, value="AAPL",
+                label="Choose a popular stock",
+                info="Pick one from the list."
+            )
+
+        with gr.Row():
+            years = gr.Slider(
+                1, 15, value=5, step=1, label="How much history?",
+                info="We pull this many *years* of past daily prices."
+            )
+            horizon = gr.Slider(
+                1, 30, value=5, step=1, label="How many days to predict?",
+                info="Business days only (markets closed on weekends/holidays)."
+            )
+
+        run_btn = gr.Button("Run Forecast", variant="primary")
+
+        gr.Markdown("### Results")
+        kpi = gr.Markdown(label="Model summary (lower is better)", value="")
+        chart = gr.Plot(label="Chart (last year + forecast)")
+        with gr.Accordion("Tables (click to expand)", open=False):
+            hist_df = gr.Dataframe(label="Historical Prices", wrap=True, interactive=False)
+            fut_df  = gr.Dataframe(label="Future Forecasts (ARIMA)", wrap=True, interactive=False)
+            comp_df = gr.Dataframe(label="Score Comparison (ARIMA vs LSTM)", wrap=True, interactive=False)
+
+        run_btn.click(
+            do_forecast,
+            inputs=[ticker_choice, years, horizon],
+            outputs=[hist_df, fut_df, kpi, chart, comp_df]
+        )
+
+    with gr.Tab("Help & Glossary"):
+        gr.Markdown(
+            """
+            ### Notes
+            - **ARIMA (p,d,q)** is read from `arima_order.json` and fit on your *selected history* for future forecasts.
+            - **Fair evaluation:** For both ARIMA and (if present) LSTM, we train on the first 80% of your selected history,
+              then predict **one day ahead** across the remaining 20% (no peeking), and report RMSE/MAE/MAPE.
+            - **LSTM (optional):** If `lstm_model.keras` and `lstm_info.json` are present, we score and overlay its forecast.
+            """
+        )
+
+demo.launch()

arima_model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1c2a5cba0ba9ead4441778d5f14a9877cb982d9decc16b0aeddcffe458cf350
+size 6492730

arima_order.json

@@ -0,0 +1 @@
+{"order": [0, 1, 3]}

lstm_info.json

@@ -0,0 +1 @@
+{"lookback": 40, "train_min": 105.80250549316406, "train_max": 233.4619903564453, "feature_range": [0, 1]}

lstm_model.keras

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea052f11008a93ab9dcd5dccb3b5552efd503d4bd0c5057176042ccbda04f461
+size 1446141

requirements.txt

@@ -0,0 +1,7 @@
+gradio>=4.0.0
+yfinance>=0.2.36
+pandas>=2.0.0
+numpy>=1.24.0
+statsmodels>=0.13.5
+scikit-learn>=1.2.0
+tensorflow-cpu==2.12.0