"""Expiry-aware derivative premium features and validated model inference.""" from datetime import datetime, timedelta import hashlib import json import logging import math import os import threading import time import joblib import mibian import numpy as np import pandas as pd from fo import FoUnavailable, IST, number from market_calendar import next_trading_day logger = logging.getLogger(__name__) FEATURE_COLUMNS = ["is_put", "moneyness", "premium_to_spot", "minutes_to_expiry", "session_minute", "iv", "delta", "gamma", "theta", "vega", "rho", "oi_log", "premium_return_15m", "premium_return_30m", "premium_volatility_15m"] HISTORY_COLUMNS = ["premium_return_15m", "premium_return_30m", "premium_volatility_15m"] AI_MAX_DATA_AGE_SECS = 300 SESSION_OPEN_MINUTE, SESSION_CLOSE_MINUTE = 9 * 60 + 15, 15 * 60 + 30 OUTLOOK_MAX_AGE_SECS = 5 * 86400 OUTLOOK_MAX_CONFIDENCE = "MEDIUM" def market_closed(current): from market_calendar import is_trading_day minute = current.hour * 60 + current.minute return not is_trading_day(current.date()) or not SESSION_OPEN_MINUTE <= minute < SESSION_CLOSE_MINUTE def next_session_open(current): from market_calendar import is_trading_day day = current.date() if not is_trading_day(day) or current.hour * 60 + current.minute >= SESSION_OPEN_MINUTE: day = next_trading_day(day + timedelta(days=1)) return datetime(day.year, day.month, day.day, 9, 15, tzinfo=IST) def outlook_observation(contract, quote, service, current): """Anchor a forecast to the last observed session when the market is not trading.""" price = float(number(quote.get("price"), "contract premium")) float(number(quote.get("underlying_price"), "underlying price")) quote_time = float(number(quote.get("quote_time"), "quote time")) if not 0 <= current.timestamp() - quote_time <= OUTLOOK_MAX_AGE_SECS: raise ValueError("Last-session snapshot is unavailable or too old") anchor, history = quote_time, [{"time": quote_time, "close": price}] try: candles = service.chart(contract)["candles"] if service is not None else [] chart_anchor = float(number(candles[-1]["time"], "chart time")) close = float(number(candles[-1]["close"], "chart premium")) if 0 <= current.timestamp() - chart_anchor <= OUTLOOK_MAX_AGE_SECS and abs(quote_time - chart_anchor) <= 1800: anchor = chart_anchor price = price if abs(price / close - 1) <= 0.03 else close history = [point for point in candles if point["time"] < anchor] + [{"time": anchor, "close": price}] except (FoUnavailable, ValueError, TypeError, KeyError, IndexError): pass # no usable chart history yet: fall back to the last-session snapshot alone premium_history_context(history, as_of=datetime.fromtimestamp(anchor, IST), min_span=0) return {**quote, "price": price, "quote_time": anchor, "context_quote_time": anchor, "premium_history": history, "ai_snapshot_usable": True, "stale": True, "outlook": True} class ForecastInputError(ValueError): def __init__(self, reason, message): super().__init__(message) self.reason = reason def _premium_history_prices(points, *, as_of): end = as_of.timestamp() prices = {} for point in points: try: timestamp = float(number(point.get("time"), "history time")) price = float(number(point.get("close"), "history price")) except (ValueError, TypeError, AttributeError): continue if end - 32 * 60 <= timestamp <= end: if timestamp in prices and prices[timestamp] != price: raise ForecastInputError("invalid_history", "Conflicting premium history observations") prices[timestamp] = price times = sorted(prices) if not times or end - times[-1] > 120: raise ForecastInputError("history_unavailable", "Recent premium history is required") return prices def premium_history_features(points, *, as_of): end = as_of.timestamp() prices = _premium_history_prices(points, as_of=as_of) times = sorted(prices) latest = prices[times[-1]] result = {} for minutes in (15, 30): lookback = end - minutes * 60 candidates = [timestamp for timestamp in times if abs(timestamp - lookback) <= 120] if not candidates: raise ForecastInputError("history_unavailable", "Premium history does not cover the required lookback") nearest = min(candidates, key=lambda timestamp: (abs(timestamp - lookback), timestamp)) result[f"premium_return_{minutes}m"] = latest / prices[nearest] - 1 recent = [prices[timestamp] for timestamp in times if timestamp >= end - 15 * 60] if len(recent) < 12: raise ForecastInputError("history_unavailable", "Premium history is too sparse for volatility features") result["premium_volatility_15m"] = float(np.std(np.diff(np.log(recent))) * math.sqrt(15)) return result def premium_history_context(points, *, as_of, min_span=60): prices = _premium_history_prices(points, as_of=as_of) times = sorted(prices) latest = times[-1] candidates = [timestamp for timestamp in times if 60 <= latest - timestamp <= 120] anchor = max(candidates) if candidates else None try: features = premium_history_features(points, as_of=as_of) limited = False except ForecastInputError as error: if error.reason != "history_unavailable" or latest - times[0] < min_span: raise features = dict.fromkeys(HISTORY_COLUMNS) limited = True return {"features": features, "limited_history": limited, "history_minutes": round((latest - times[0]) / 60, 2), "history_samples": len(times), "premium_return_1m": prices[latest] / prices[anchor] - 1 if anchor is not None else None} def build_features(contract, observation, *, as_of=None, allow_short_history=False): current = (as_of or datetime.now(IST)).astimezone(IST) expiry = datetime.fromisoformat(contract["expiry"] + "T15:30:00+05:30") minutes = (expiry - current).total_seconds() / 60 if minutes <= 0: raise ValueError("The derivative contract has expired") if contract["instrument_type"] != "OPTION": raise ValueError("A separate futures training universe is required") price = float(number(observation.get("price"), "premium")) spot = float(number(observation.get("underlying_price"), "underlying price")) strike = float(number(contract["strike"], "strike")) quoted = float(number(observation.get("quote_time"), "quote time")) if not -5 <= current.timestamp() - quoted <= 120: raise ValueError("The derivative observation is stale or future-dated") put = contract["option_type"] == "PE" iv_value = observation.get("iv") if iv_value is None: intrinsic = max(strike - spot if put else spot - strike, 0) if not intrinsic < price < (strike if put else spot): raise ForecastInputError("invalid_iv_inputs", "Observed premium is outside valid implied-volatility bounds") inputs = [spot, strike, 0, minutes / 1440] solved = mibian.BS(inputs, putPrice=price) if put else mibian.BS(inputs, callPrice=price) iv_value = solved.impliedVolatility iv = float(number(iv_value, "implied volatility")) if iv > 500: raise ValueError("Implied volatility is outside the supported domain") greeks = mibian.BS([spot, strike, 0, minutes / 1440], volatility=iv) values = [float(put), math.log(spot / strike), price / spot, minutes, current.hour * 60 + current.minute - 555, iv, greeks.putDelta if put else greeks.callDelta, greeks.gamma, greeks.putTheta if put else greeks.callTheta, greeks.vega, greeks.putRho if put else greeks.callRho, math.log1p(float(number(observation.get("open_interest"), "open interest", allow_zero=True)))] if observation.get("premium_history") is not None: if allow_short_history: history = premium_history_context(observation["premium_history"], as_of=current)["features"] history = {column: float("nan") if value is None else value for column, value in history.items()} else: history = premium_history_features(observation["premium_history"], as_of=current) else: history = {column: float(observation[column]) for column in HISTORY_COLUMNS} values.extend(history[column] for column in HISTORY_COLUMNS) if not np.isfinite(values[:-len(HISTORY_COLUMNS)]).all() or any( not math.isfinite(value) and not (allow_short_history and math.isnan(value)) for value in values[-len(HISTORY_COLUMNS):]): raise ValueError("Derivative features are not finite") return dict(zip(FEATURE_COLUMNS, values)) def attach_targets(frame, timeframe): required = {"contract_id", "timestamp", "price"} if not required.issubset(frame.columns): raise ValueError("Training data requires exact contract_id, timestamp and price") if timeframe not in {"INTRADAY", "1D"}: raise ValueError("Unsupported derivative timeframe") data = frame.copy() data["timestamp"] = pd.to_datetime(data["timestamp"], utc=True) if data.duplicated(["contract_id", "timestamp"]).any(): raise ValueError("Duplicate exact-contract observations are not allowed") local = data["timestamp"].dt.tz_convert(IST) if timeframe == "INTRADAY": data["target_at"] = local.dt.normalize() + pd.Timedelta(hours=15) else: sessions = {day: next_trading_day(day + timedelta(days=1)) for day in local.dt.date.unique()} data["target_at"] = pd.to_datetime(local.dt.date.map(sessions).astype(str)).dt.tz_localize(IST) + pd.Timedelta(hours=15, minutes=30) expiry = data["expiry"] if "expiry" in data else data["contract_id"].str.extract(r"(\d{4}-\d{2}-\d{2})", expand=False) if expiry.isna().any(): raise ValueError("Exact contract expiry is required") expires_at = pd.to_datetime(expiry.astype(str)).dt.tz_localize(IST) + pd.Timedelta(hours=15, minutes=30) targets = data[["contract_id", "timestamp", "price"]].rename(columns={"timestamp": "target_at", "price": "target_price"}) data = data[(data["target_at"] <= expires_at) & (data["timestamp"] < data["target_at"])].copy() data["target_at"] = data["target_at"].dt.tz_convert("UTC") data = data.merge(targets, on=["contract_id", "target_at"], how="inner", validate="many_to_one") data = data[(data["price"] > 0) & (data["target_price"] >= 0)].copy() data["target_log_return"] = np.log1p(data["target_price"]) - np.log1p(data["price"]) return data def attach_excursion_targets(frame, timeframe): if not {"high", "low"}.issubset(frame.columns): raise ValueError("Observed premium high/low bars are required for reachable targets") labels = attach_targets(frame, timeframe) if labels.empty: return labels.assign(future_high=pd.Series(dtype=float), future_low=pd.Series(dtype=float), up_log_move=pd.Series(dtype=float), down_log_move=pd.Series(dtype=float)) sampled_columns = {"intraday_high", "intraday_low", "remaining_high", "remaining_low", "session_high", "session_low"} if sampled_columns.issubset(frame.columns): if timeframe == "INTRADAY": labels["future_high"], labels["future_low"] = labels["intraday_high"], labels["intraday_low"] if "intraday_complete" in labels: labels = labels[labels["intraday_complete"].eq(True)].copy() else: sessions = frame[["contract_id", "timestamp", "session_high", "session_low"]].copy() sessions["next_session_complete"] = frame["session_complete"].to_numpy() if "session_complete" in frame else True sessions["timestamp"] = pd.to_datetime(sessions["timestamp"], utc=True) sessions = sessions.rename(columns={"timestamp": "target_at", "session_high": "next_session_high", "session_low": "next_session_low"}) labels = labels.merge(sessions, on=["contract_id", "target_at"], how="inner", validate="many_to_one") labels["future_high"] = labels[["remaining_high", "next_session_high"]].max(axis=1) labels["future_low"] = labels[["remaining_low", "next_session_low"]].min(axis=1) labels = labels[labels["next_session_complete"].eq(True)].copy() if "remaining_complete" in labels: labels = labels[labels["remaining_complete"].eq(True)].copy() else: history = frame.copy() history["timestamp"] = pd.to_datetime(history["timestamp"], utc=True) history = history[(history["low"] >= 0) & (history["high"] >= history["price"]) & (history["low"] <= history["price"])] groups = {identity: group.sort_values("timestamp") for identity, group in history.groupby("contract_id")} highs, lows = [], [] for row in labels.itertuples(index=False): bars = groups.get(row.contract_id) window = bars[(bars["timestamp"] > row.timestamp) & (bars["timestamp"] <= row.target_at)] if bars is not None else None highs.append(window["high"].max() if window is not None and not window.empty else float("nan")) lows.append(window["low"].min() if window is not None and not window.empty else float("nan")) labels["future_high"], labels["future_low"] = highs, lows labels = labels.dropna(subset=["future_high", "future_low"]).copy() labels["up_log_move"] = np.maximum(0, np.log1p(labels["future_high"]) - np.log1p(labels["price"])) labels["down_log_move"] = np.maximum(0, np.log1p(labels["price"]) - np.log1p(labels["future_low"])) return labels def promotion_passes(metrics): keys = ("selected_samples", "test_expiries", "forecast_coverage", "direction_accuracy", "range_coverage", "majority_baseline") if not isinstance(metrics, dict) or any(not isinstance(metrics.get(key), (int, float)) or isinstance(metrics[key], bool) or not math.isfinite(metrics[key]) for key in keys): return False if any(not 0 <= metrics[key] <= 1 for key in ("forecast_coverage", "direction_accuracy", "range_coverage", "majority_baseline")): return False return bool(metrics["selected_samples"] >= 200 and metrics["test_expiries"] >= 5 and metrics["forecast_coverage"] >= 0.10 and metrics["direction_accuracy"] > 0.75 and metrics["range_coverage"] > 0.75 and metrics["direction_accuracy"] >= metrics["majority_baseline"] + 0.01) def reachable_quality_passes(metrics): keys = ("selected_samples", "test_expiries", "forecast_coverage", "target_reach_rate", "far_bound_coverage", "minimum_target_move_pct") if not isinstance(metrics, dict) or any(not isinstance(metrics.get(key), (int, float)) or isinstance(metrics[key], bool) or not math.isfinite(metrics[key]) for key in keys): return False if any(not 0 <= metrics[key] <= 1 for key in ("forecast_coverage", "target_reach_rate", "far_bound_coverage")): return False return bool(metrics["selected_samples"] >= 200 and metrics["test_expiries"] >= 5 and metrics["forecast_coverage"] >= 0.10 and metrics["target_reach_rate"] > 0.75 and metrics["far_bound_coverage"] > 0.75 and metrics["minimum_target_move_pct"] >= 5) def model_quality_passes(metadata): return reachable_quality_passes(metadata.get("metrics")) if metadata.get("target_kind") == "reachable" else promotion_passes(metadata.get("metrics")) def reachable_levels(models, frame, adjustments): result = {} for side in ("up", "down"): near = np.maximum(0, models[side + "_near"].predict(frame)) target = np.maximum(0, models[side + "_target"].predict(frame) - adjustments[side + "_target"]) far = np.maximum(target, np.maximum(near, models[side + "_far"].predict(frame) + adjustments[side + "_far"])) result[side + "_near"] = np.minimum(near, target) result[side + "_target"], result[side + "_far"] = target, far if not all(np.isfinite(value).all() for value in result.values()): raise ValueError("Non-finite reachable premium levels") result["is_up"] = result["up_target"] >= result["down_target"] return result def reachable_prediction(models, vector, metadata, observation, *, allow_non_actionable=False): levels = reachable_levels(models, vector, metadata["reach_adjustments"]) upward = bool(levels["is_up"][0]) side = "up" if upward else "down" price = float(observation["price"]) anchor = math.log1p(price) sign = 1 if upward else -1 prices = [max(0, math.expm1(anchor + sign * levels[side + role][0])) for role in ("_near", "_target", "_far")] actionable = abs(prices[1] / price - 1) * 100 >= metadata["minimum_target_move_pct"] if not actionable and not allow_non_actionable: return {"available": False, "no_trade_reason": "insufficient_move"} metrics = metadata["metrics"] return {"available": True, "no_trade_reason": None, "direction": "BULLISH" if upward else "BEARISH", "actionable": actionable, "minimum_target_move_pct": metadata["minimum_target_move_pct"], "confidence": "HIGH" if metrics["target_reach_rate"] >= 0.90 else "MEDIUM", "confidence_prob": None, "confidence_basis": "backtested_target_reach_rate", "target_price_lo": round(min(prices[0], prices[2]), 2), "expected_target_price": round(prices[1], 2), "target_price_hi": round(max(prices[0], prices[2]), 2), "current_price": price, "estimated_high": round(math.expm1(anchor + levels["up_far"][0]), 2), "estimated_low": round(max(0, math.expm1(anchor - levels["down_far"][0])), 2), "range_semantics": "reachable_premium_excursions", "target_quantile": 0.2, "target_reach_rate": metrics["target_reach_rate"], "far_bound_coverage": metrics["far_bound_coverage"], "closing_direction_accuracy": metrics.get("direction_accuracy_all"), "as_of": datetime.fromtimestamp(observation["quote_time"], IST).isoformat()} def model_data_fresh(manifest, *, now=None): try: last_observed = datetime.fromisoformat(manifest["data_end"]).date() age = ((now or datetime.now(IST)).astimezone(IST).date() - last_observed).days maximum = int(os.getenv("FO_MODEL_MAX_AGE_DAYS", "180")) return 0 <= age <= maximum except (KeyError, ValueError, TypeError): return False class FoPremiumPredictor: def __init__(self, model_dir=None): self.model_dir = model_dir or os.getenv("FO_MODEL_DIR", os.path.join(os.path.dirname(__file__), "fo_models")) self._manifest = None self._manifest_key = None self._models = {} self._lock = threading.Lock() def manifest(self): path = os.path.join(self.model_dir, "manifest.json") with self._lock: try: details = os.stat(path) key = (details.st_mtime_ns, details.st_size) if self._manifest_key != key: if details.st_size > 1_000_000: raise ValueError("Invalid derivative model metadata") with open(path, encoding="utf-8") as stream: manifest = json.load(stream) if not isinstance(manifest, dict) or manifest.get("schema_version") != 1 or manifest.get("feature_columns") != FEATURE_COLUMNS: raise ValueError("Incompatible derivative model metadata") self._manifest, self._manifest_key = manifest, key self._models.clear() return self._manifest except (OSError, ValueError, TypeError): self._manifest, self._manifest_key = None, None self._models.clear() return None def has_approved_models(self, contract): manifest = self.manifest() return bool(manifest and model_data_fresh(manifest) and contract["underlying"] in manifest.get("supported_underlyings", []) and contract["instrument_type"] == manifest.get("instrument_type") and any(metadata.get("approved") is True and model_quality_passes(metadata) for metadata in manifest.get("timeframes", {}).values())) def _load_models(self, timeframe, metadata): artifact = metadata.get("artifact") if not artifact or os.path.basename(artifact) != artifact or not artifact.endswith(".joblib"): raise ValueError("Invalid derivative model artifact") key = (timeframe, metadata.get("artifact_sha256")) with self._lock: if key not in self._models: path = os.path.join(self.model_dir, artifact) if os.path.getsize(path) > 50_000_000: raise ValueError("Derivative artifact exceeds its size limit") with open(path, "rb") as stream: if hashlib.file_digest(stream, "sha256").hexdigest() != key[1]: raise ValueError("Derivative artifact checksum mismatch") self._models[key] = joblib.load(path) return self._models[key] def predict(self, contract, observation, windows, *, now=None): manifest = self.manifest() result = {"forecast_status": "not_trained", "timeframes": {}, "forecast_metrics": {}} if manifest: result["forecast_model"] = {key: manifest.get(key) for key in ("source", "source_license", "data_end", "range_semantics", "serving_mode")} for timeframe, window in windows.items(): status = "not_trained" metadata = (manifest or {}).get("timeframes", {}).get(timeframe, {}) metrics = metadata.get("metrics", {}) result["forecast_metrics"][timeframe] = metrics if not window.get("eligible"): status = window.get("status", "after_expiry") elif manifest: if contract["underlying"] not in manifest.get("supported_underlyings", []) or contract["instrument_type"] != manifest.get("instrument_type"): status = "unsupported_contract" elif metadata: status = "ready" if metadata.get("approved") is True and model_quality_passes(metadata) else "quality_gate_failed" if status == "ready" and not model_data_fresh(manifest, now=now): status = "model_stale" if status in {"quality_gate_failed", "model_stale"} and manifest.get("serving_mode") == "experimental" and metadata.get("artifact"): status = "experimental" prediction = {"available": False, "no_trade_reason": status, "end_at": window.get("end_at"), "source": "fo_ml"} if status == "experimental": prediction.update({"experimental": True, "accuracy_validated": False, "validation_passed": model_quality_passes(metadata), "model_data_stale": not model_data_fresh(manifest, now=now)}) if status in {"ready", "experimental"}: input_status = "invalid_features" try: input_status = "stale_input" current = (now or datetime.now(IST)).astimezone(IST) quoted = float(number(observation.get("quote_time"), "quote time")) if observation.get("stale") is True: raise ValueError("Stale derivative observation") if not -5 <= current.timestamp() - quoted <= 120: raise ValueError("The derivative observation is stale or future-dated") input_status = "invalid_features" quoted_at = datetime.fromtimestamp(quoted, IST) if status == "experimental" and observation.get("premium_history") is not None: history_context = premium_history_context(observation["premium_history"], as_of=quoted_at) prediction.update({key: history_context[key] for key in ("limited_history", "history_minutes", "history_samples")}) features = build_features(contract, {**observation, "iv": None}, as_of=quoted_at, allow_short_history=status == "experimental") input_status = "outside_training_domain" for column in ("minutes_to_expiry", "moneyness", "iv", "premium_to_spot", "session_minute"): lower, upper = metadata["feature_bounds"][column] if not lower <= features[column] <= upper: raise ValueError("Observation is outside the derivative training domain") input_status = "artifact_unavailable" models = self._load_models(timeframe, metadata) input_status = "invalid_model_output" vector = pd.DataFrame([features], columns=FEATURE_COLUMNS) if metadata.get("target_kind") == "reachable": prediction.update(reachable_prediction(models, vector, metadata, observation, allow_non_actionable=status == "experimental")) result["timeframes"][timeframe] = prediction continue classifier = models["direction"] probabilities = classifier.predict_proba(vector)[0] if not np.isfinite(probabilities).all() or np.any(probabilities < 0) or np.any(probabilities > 1): raise ValueError("Invalid derivative model probabilities") index = int(np.argmax(probabilities)) if probabilities[index] < metadata["min_probability"]: prediction["no_trade_reason"] = "low_confidence" else: quantiles = sorted(float(models[name].predict(vector)[0]) for name in ("lo", "mid", "hi")) quantiles[0] -= metadata["conformal_adjustment"] quantiles[2] += metadata["conformal_adjustment"] prices = [max(0.0, math.expm1(math.log1p(float(observation["price"])) + value)) for value in quantiles] if not np.isfinite(prices).all(): raise ValueError("Invalid derivative forecast") prediction.update({"available": True, "no_trade_reason": None, "direction": str(classifier.classes_[index]), "confidence": "MEDIUM", "confidence_prob": float(probabilities[index]), "current_price": observation["price"], "target_price_lo": round(prices[0], 2), "expected_target_price": round(prices[1], 2), "target_price_hi": round(prices[2], 2), "range_semantics": "terminal_premium_q10_q90", "as_of": datetime.fromtimestamp(observation["quote_time"], IST).isoformat(), "holdout_direction_accuracy": metrics["direction_accuracy"], "holdout_range_coverage": metrics["range_coverage"]}) except (OSError, ValueError, TypeError, KeyError, IndexError, AttributeError, OverflowError) as error: prediction["no_trade_reason"] = "model_input_unavailable" prediction["input_status"] = getattr(error, "reason", input_status) result["timeframes"][timeframe] = prediction for prediction in result["timeframes"].values(): if prediction.get("experimental"): prediction.update({"confidence": "LOW", "confidence_prob": None, "confidence_basis": "experimental_unvalidated"}) reasons = {value["no_trade_reason"] for value in result["timeframes"].values()} if any(value["available"] for value in result["timeframes"].values()): result["forecast_status"] = "experimental" if any(value.get("experimental") and value["available"] for value in result["timeframes"].values()) else "ready" elif "quality_gate_failed" in reasons: result["forecast_status"] = "quality_gate_failed" elif "unsupported_contract" in reasons: result["forecast_status"] = "unsupported_contract" elif "model_input_unavailable" in reasons: result["forecast_status"] = "model_input_unavailable" elif "model_stale" in reasons: result["forecast_status"] = "model_stale" elif "low_confidence" in reasons: result["forecast_status"] = "low_confidence" elif "insufficient_move" in reasons: result["forecast_status"] = "insufficient_move" return result _predictor = None _predictor_lock = threading.Lock() _reviews = {} _review_lock = threading.Lock() _review_slots = threading.BoundedSemaphore(3) AI_SLOT_WAIT_SECS = 8 AI_PROVIDER_ATTEMPTS = 2 AI_RETRY_PAUSE_SECS = 1.5 # Total time one forecast may spend waiting for a rate-limited provider's short cooldown to end, # instead of failing at once with "provider unavailable" while one frees up in a few seconds. AI_COOLDOWN_WAIT_SECS = 10 def get_predictor(): global _predictor with _predictor_lock: if _predictor is None: _predictor = FoPremiumPredictor() return _predictor def review_forecast(contract, observation, prediction): if not prediction.get("available") or os.getenv("FO_AI_ENABLED", "1") != "1": return {"available": False, "source": "fo_ai_review"} key = (contract["contract_id"], prediction.get("end_at"), prediction.get("direction"), prediction["target_price_lo"], prediction["expected_target_price"], prediction["target_price_hi"]) with _review_lock: cached = _reviews.get(key) if cached and datetime.now(IST).timestamp() - cached[0] < 300: return dict(cached[1]) if not _review_slots.acquire(blocking=False): return {"available": False, "source": "fo_ai_review", "reason": "busy"} try: from llm_client import make_chat_call context = {"underlying": contract["underlying"], "expiry": contract["expiry"], "strike": contract["strike"], "option_type": contract["option_type"], "premium": observation.get("price"), "underlying_price": observation.get("underlying_price"), "quote_time": observation.get("quote_time"), "validated_model": prediction} content, provider, model = make_chat_call(messages=[ {"role": "system", "content": "Review a trained NSE option-premium forecast for paper-trading research. " "Use only the supplied contract, expiry, observation and model output. Never invent price levels, IV, " "Greeks, news, accuracy or returns. You are commentary, not an independent calibrated price model. " "Explain premium direction versus underlying direction and expiry/time-decay risk. " "Return JSON with exactly one summary string, at most 400 characters."}, {"role": "user", "content": json.dumps(context, allow_nan=False)}], max_tokens=220, temperature=0, fast_fail_on_rate_limit=True, max_retries=0, call_timeout=6) parsed = json.loads(content) summary = parsed.get("summary") if not isinstance(summary, str) or not summary.strip() or len(summary) > 600: raise ValueError("Invalid derivative AI review") result = {"available": True, "source": provider + ":" + model, "summary": summary.strip(), "independent_range": False, "basis": "validated_model_review", "direction": prediction["direction"], "target_price_lo": prediction["target_price_lo"], "expected_target_price": prediction["expected_target_price"], "target_price_hi": prediction["target_price_hi"]} with _review_lock: if len(_reviews) >= 256: _reviews.clear() _reviews[key] = (datetime.now(IST).timestamp(), result) return result except (RuntimeError, OSError, ValueError, TypeError, KeyError, AttributeError): return {"available": False, "source": "fo_ai_review", "reason": "unavailable"} finally: _review_slots.release() def ai_forecasts(contract, observation, windows, *, refresh=False): return _ai_forecasts(contract, observation, windows, refresh=refresh) def _wait_for_provider(budget): """Sleep until a cooling-down provider is usable, if that happens within `budget` seconds. Returns the seconds slept. A cooldown longer than the budget (or none that will recover) is not waited on: the attempt then fails fast and the frontend's own retry schedule takes over. """ if budget <= 0: return 0.0 from llm_client import seconds_until_provider_available wait = seconds_until_provider_available() if not wait or wait > budget: return 0.0 time.sleep(wait + 0.2) return wait + 0.2 def _chat_with_retry(make_chat_call, messages, call_timeout): """Retry transient provider failures with a rotated lead provider and a fresh time budget.""" last_error = None cooldown_budget = AI_COOLDOWN_WAIT_SECS for attempt in range(max(1, AI_PROVIDER_ATTEMPTS)): cooldown_budget -= _wait_for_provider(cooldown_budget) try: return make_chat_call(messages=messages, max_tokens=550, temperature=0, fast_fail_on_rate_limit=True, max_retries=0, call_timeout=call_timeout, json_mode=True, task_offset=attempt) except (RuntimeError, OSError) as error: last_error = error logger.warning("F&O AI provider attempt %d/%d failed: %s", attempt + 1, AI_PROVIDER_ATTEMPTS, error) if attempt + 1 < AI_PROVIDER_ATTEMPTS: time.sleep(AI_RETRY_PAUSE_SECS) raise last_error def _ai_forecasts(contract, observation, windows, *, refresh=False, call_timeout=20, repair=True, repair_errors=None): current = datetime.now(IST) results = {timeframe: {"available": False, "source": "fo_ai", "accuracy_validated": False, "independent_range": True, "basis": "independent_llm_contract_forecast", "no_trade_reason": window.get("status") if not window.get("eligible") else "ai_unavailable"} for timeframe, window in windows.items()} if os.getenv("FO_AI_ENABLED", "1") != "1": for timeframe, window in windows.items(): if window.get("eligible"): results[timeframe]["no_trade_reason"] = "ai_disabled" return results eligible = {timeframe: window for timeframe, window in windows.items() if window.get("eligible")} if not eligible: return results try: price = float(number(observation.get("price"), "contract premium")) spot = float(number(observation.get("underlying_price"), "underlying price")) quoted = float(number(observation.get("quote_time"), "quote time")) context_quoted = float(number(observation.get("context_quote_time", quoted), "context quote time")) outlook = observation.get("outlook") is True max_age = OUTLOOK_MAX_AGE_SECS if outlook else AI_MAX_DATA_AGE_SECS if observation.get("stale") is True and not observation.get("ai_snapshot_usable"): raise ValueError("Stale derivative observation") if any(not -5 <= current.timestamp() - timestamp <= max_age for timestamp in (quoted, context_quoted)): raise ValueError("Derivative snapshot is too old") start = next_session_open(current) if outlook else current expiry = datetime.fromisoformat(contract["expiry"] + "T15:30:00+05:30") for window in eligible.values(): end = datetime.fromisoformat(window["end_at"]) if not current < end <= expiry: raise ValueError("Forecast window exceeds contract expiry") as_of = datetime.fromtimestamp(quoted, IST) history_context = premium_history_context(observation.get("premium_history") or [], as_of=as_of, min_span=0) history = history_context["features"] except (ValueError, TypeError, KeyError): for timeframe in eligible: results[timeframe]["no_trade_reason"] = "ai_input_unavailable" return results key = ("independent_ai", contract["contract_id"], history_context["limited_history"], outlook, tuple((timeframe, window["end_at"]) for timeframe, window in eligible.items())) with _review_lock: if refresh: _reviews.pop(key, None) cached = _reviews.get(key) if cached and 0 <= current.timestamp() - cached[0] < 120: copies = {timeframe: dict(prediction) for timeframe, prediction in cached[1].items()} cache_valid = all(prediction.get("available") for prediction in copies.values()) for prediction in copies.values(): if not prediction.get("available"): continue target = prediction.get("expected_target_price") direction = prediction.get("direction") if target is not None and ((direction == "BULLISH" and price >= target) or (direction == "BEARISH" and price <= target) or (direction == "NEUTRAL" and not prediction["target_price_lo"] <= price <= prediction["target_price_hi"])): cache_valid = False break age = current.timestamp() - datetime.fromisoformat(prediction["as_of"]).timestamp() context_age = current.timestamp() - datetime.fromisoformat(prediction["context_as_of"]).timestamp() if any(not -5 <= value <= max_age for value in (age, context_age)): cache_valid = False break prediction.update({"data_age_seconds": round(age), "context_age_seconds": round(context_age), "input_delayed": max(age, context_age) > 120, "cached": True}) if cache_valid: return {**results, **copies} if not _review_slots.acquire(timeout=AI_SLOT_WAIT_SECS): for timeframe in eligible: results[timeframe]["no_trade_reason"] = "ai_busy" return results try: context = {"contract_id": contract["contract_id"], "underlying": contract["underlying"], "instrument_type": contract["instrument_type"], "expiry": contract["expiry"], "strike": contract.get("strike"), "option_type": contract.get("option_type"), "current_price": price, "underlying_price": spot, "as_of": datetime.fromtimestamp(quoted, IST).isoformat(), "forecast_at": current.isoformat(), "horizons": {timeframe: {**window, "start_at": start.isoformat(), "calendar_minutes": round((datetime.fromisoformat(window["end_at"]) - start).total_seconds() / 60)} for timeframe, window in eligible.items()}, "data_age_seconds": round(current.timestamp() - quoted), "context_as_of": datetime.fromtimestamp(context_quoted, IST).isoformat(), "history_context": {key: value for key, value in history_context.items() if key != "features"}, "premium_history_features": history, "open_interest": observation.get("open_interest"), "volume": observation.get("volume"), "provider_iv": observation.get("provider_iv")} if outlook: context["market_state"] = {"mode": "previous_session_outlook", "market_open_now": False, "next_session_open": start.isoformat(), "note": "current_price and underlying_price are the last observed values of the previous session"} if repair_errors: context["validation_errors"] = repair_errors if contract["instrument_type"] == "OPTION": strike = float(number(contract["strike"], "strike")) intrinsic = max(strike - spot if contract["option_type"] == "PE" else spot - strike, 0) context["option_context"] = {"moneyness": "ITM" if intrinsic > 0 else "ATM" if strike == spot else "OTM", "intrinsic_value": intrinsic, "observed_time_value": price - intrinsic, "iv_inputs_valid": intrinsic < price < (strike if contract["option_type"] == "PE" else spot)} try: context["derived_features"] = build_features(contract, {**observation, "iv": None}, as_of=as_of) context["greeks_basis"] = "Mibian zero-rate Black-Scholes approximation from observed prices" except (ValueError, TypeError, KeyError, OverflowError): context["derived_features"] = None outlook_note = ("The market is closed or not yet open, so this is a previous-session outlook: horizons start at " "market_state.next_session_open from the last observed premium. The opening gap is unknown; widen the range " "for it and keep confidence modest. " if outlook else "") from llm_client import make_chat_call call_started = time.monotonic() messages = [ {"role": "system", "content": outlook_note + "Estimate the change in the exact NSE derivative contract price for paper-trading research. " "Predict reachable premium levels during the supplied window, not just its closing price. " "INTRADAY ends today at 15:00 IST; 1D includes the remaining current session and ends at the " "next trading session's 15:30 IST close. Assess each horizon separately using its end_at, " "time to expiry and additional overnight/time-decay risk. Never copy a horizon's estimate " "to another merely because both use the same current quote; equal estimates require your own assessment. " "The expected change is your actionable reachable target; lo/hi describe nearer/farther reachable levels. " "Make your own directional assessment from the supplied observed contract history and expiry context; " "do not copy an equity forecast or assume a failed ML model is valid. BULLISH/BEARISH describe the " "contract premium, not its underlying: a put may lose value when the index rises. Consider time decay, " "Use the supplied option_context moneyness as a fact; an ITM put has strike above spot. " "If IV inputs are invalid or derived features are absent, do not invent Greeks or implied volatility. " "moneyness, liquidity and volatility. Use only supplied data; do not invent news, metrics or accuracy. " "When limited_history is true, only a short observed window is available. Treat missing long-lookback " "features as unknown, never zero or extrapolated 15/30-minute returns. Keep confidence LOW and " "acknowledge the limited history in each summary. " "Your forecast has no validated accuracy. Return JSON {\"forecasts\":{TIMEFRAME:{\"direction\": " "\"BULLISH|BEARISH|NEUTRAL\",\"expected_change_pct\":number,\"predicted_return_lo\":number," "\"predicted_return_hi\":number,\"confidence\":\"LOW|MEDIUM|HIGH\",\"summary\":string}}}. " "Include each supplied horizon only. Percent changes must be relative to current_price, ordered " "-100 <= lo <= expected <= hi <= 500. BULLISH requires expected > 1; BEARISH expected < -1; " "NEUTRAL requires -1 <= expected <= 1. Each summary must be at most 300 characters. " "Do not output target prices, confidence probabilities or claims of backtested reliability."}, {"role": "user", "content": json.dumps(context, allow_nan=False)}] content, provider, model = _chat_with_retry(make_chat_call, messages, call_timeout) parsed = json.JSONDecoder().raw_decode(content[content.find("{"):])[0] forecasts = parsed.get("forecasts") if isinstance(parsed, dict) else None if not isinstance(forecasts, dict): raise ValueError("Invalid derivative AI forecast") completed = datetime.now(IST) response_errors = {} for timeframe, window in eligible.items(): if completed >= datetime.fromisoformat(window["end_at"]): results[timeframe]["no_trade_reason"] = "expired" if completed >= expiry else "session_closed" continue if any(completed.timestamp() - timestamp > max_age for timestamp in (quoted, context_quoted)): results[timeframe]["no_trade_reason"] = "ai_input_unavailable" continue raw = forecasts.get(timeframe) if not isinstance(raw, dict): results[timeframe]["no_trade_reason"] = "ai_invalid_response" response_errors[timeframe] = "A forecast object is required for this horizon" continue try: values = [raw[field] for field in ("predicted_return_lo", "expected_change_pct", "predicted_return_hi")] if any(isinstance(value, bool) or not isinstance(value, (float, int)) or not math.isfinite(value) for value in values): raise ValueError("Invalid derivative change estimate") lower, change, upper = values direction, confidence, summary = raw.get("direction"), raw.get("confidence"), raw.get("summary") if not -100 <= lower <= change <= upper <= 500 or confidence not in {"LOW", "MEDIUM", "HIGH"}: raise ValueError("Invalid derivative forecast range") expected_direction = "BULLISH" if change > 1 else "BEARISH" if change < -1 else "NEUTRAL" if direction != expected_direction: raise ValueError(f"Direction must be {expected_direction} for expected_change_pct={change}") if not isinstance(summary, str) or not summary.strip() or len(summary) > 600: raise ValueError("A nonempty summary of at most 600 characters is required") results[timeframe] = {"available": True, "source": provider + ":" + model, "basis": "independent_llm_contract_forecast", "accuracy_validated": False, "range_semantics": "ai_reachable_premium_targets", "independent_range": True, "direction": direction, "confidence": "LOW" if history_context["limited_history"] else "MEDIUM" if outlook and confidence == "HIGH" else confidence, "premarket_outlook": outlook, **{key: history_context[key] for key in ("limited_history", "history_minutes", "history_samples")}, "expected_change_pct": change, "predicted_return_lo": lower, "predicted_return_hi": upper, "current_price": price, "target_price_lo": round(price * (1 + lower / 100), 2), "expected_target_price": round(price * (1 + change / 100), 2), "target_price_hi": round(price * (1 + upper / 100), 2), "summary": summary.strip(), "as_of": datetime.fromtimestamp(quoted, IST).isoformat(), "end_at": window["end_at"], "generated_at": completed.isoformat(), "cached": False, "context_as_of": datetime.fromtimestamp(context_quoted, IST).isoformat(), "data_age_seconds": round(completed.timestamp() - quoted), "context_age_seconds": round(completed.timestamp() - context_quoted), "input_delayed": max(completed.timestamp() - quoted, completed.timestamp() - context_quoted) > 120, "no_trade_reason": None} except (ValueError, TypeError, KeyError) as error: results[timeframe]["no_trade_reason"] = "ai_invalid_response" response_errors[timeframe] = str(error) continue except (RuntimeError, OSError): for timeframe in eligible: results[timeframe]["no_trade_reason"] = "ai_provider_unavailable" return results except (ValueError, TypeError, KeyError, AttributeError): for timeframe in eligible: results[timeframe]["no_trade_reason"] = "ai_invalid_response" return results finally: _review_slots.release() failed = {timeframe: window for timeframe, window in eligible.items() if results[timeframe].get("no_trade_reason") == "ai_invalid_response"} remaining = call_timeout - (time.monotonic() - call_started) if repair and failed and remaining > 0: results.update(_ai_forecasts(contract, observation, failed, refresh=True, call_timeout=remaining, repair=False, repair_errors={timeframe: response_errors[timeframe] for timeframe in failed})) with _review_lock: if all(results[timeframe].get("available") for timeframe in eligible): if len(_reviews) >= 256: _reviews.clear() _reviews[key] = (datetime.now(IST).timestamp(), {timeframe: results[timeframe] for timeframe in eligible}) return results def contract_forecasts(contract, quote, windows, *, service=None, refresh=False): predictor = get_predictor() observation = quote or {} input_reason = None needs_context = any(window.get("eligible") for window in windows.values()) and ( predictor.has_approved_models(contract) or (predictor.manifest() or {}).get("serving_mode") == "experimental" or os.getenv("FO_AI_ENABLED", "1") == "1") outlook = False if needs_context and quote: current = datetime.now(IST) if market_closed(current): try: observation = outlook_observation(contract, quote, service, current) outlook = True except (FoUnavailable, ValueError, TypeError, KeyError, IndexError): observation = quote if needs_context and not outlook: try: input_reason = "ai_quote_unavailable" number(observation.get("price"), "contract premium") number(observation.get("underlying_price"), "underlying price") context_time = float(number(observation.get("quote_time"), "context quote time")) current = datetime.now(IST) limit = AI_MAX_DATA_AGE_SECS if os.getenv("FO_AI_ENABLED", "1") == "1" else 120 input_reason = "ai_stale_data" if not -5 <= current.timestamp() - context_time <= limit: raise ValueError("Contract context is stale") # Chart history only enriches the forecast. A newly added contract often has none yet (or NSE's # chart lags the quote), and a fresh quote alone is a valid single-observation, LOW-confidence # history, so a missing/lagging chart must not block the forecast. input_reason = "ai_history_unavailable" quote_point = {"time": context_time, "close": observation["price"]} chart_points = [] try: chart_points = [point for point in service.chart(contract)["candles"] if isinstance(point, dict) and point["time"] < context_time] if service is not None else [] except (FoUnavailable, ValueError, TypeError, KeyError, IndexError): chart_points = [] context_at = datetime.fromtimestamp(context_time, IST) try: premium_history_context(chart_points + [quote_point], as_of=context_at, min_span=0) history = chart_points + [quote_point] except ValueError: history = [quote_point] premium_history_context(history, as_of=context_at, min_span=0) current = datetime.now(IST) input_reason = "ai_stale_data" if not -5 <= current.timestamp() - context_time <= limit: raise ValueError("Contract context is stale") observation = {**observation, "premium_history": history, "context_quote_time": context_time, "ai_snapshot_usable": True, "stale": observation.get("stale") is True or current.timestamp() - context_time > 120} input_reason = None except (FoUnavailable, ValueError, TypeError, KeyError, IndexError): observation = {"stale": True} result = predictor.predict(contract, {"stale": True} if outlook else observation, windows) if outlook or input_reason: for prediction in result["timeframes"].values(): if prediction.get("no_trade_reason") == "model_input_unavailable": prediction["input_status"] = "market_not_open" if outlook else input_reason.removeprefix("ai_") forecasts = ai_forecasts(contract, observation, windows, refresh=refresh) for forecast in forecasts.values(): if input_reason and forecast.get("no_trade_reason") == "ai_input_unavailable": forecast["no_trade_reason"] = input_reason for timeframe, prediction in result["timeframes"].items(): prediction["ai_forecast"] = forecasts.get(timeframe, {"available": False, "source": "fo_ai"}) result["ai_forecast_status"] = "ready" if any(prediction.get("available") for prediction in forecasts.values()) else "unavailable" return result