PaperTrade / fo_predictor.py
Khanna, Videh Rakesh Rakesh
Claude Sonnet 5.5
F&O AI: wait out a provider's short cooldown instead of failing at once
dbf6890
Raw History Blame Contribute Delete
53.9 kB
"""Expiry-aware derivative premium features and validated model inference."""
from datetime import datetime, timedelta
import hashlib
import json
import logging
import math
import os
import threading
import time
import joblib
import mibian
import numpy as np
import pandas as pd
from fo import FoUnavailable, IST, number
from market_calendar import next_trading_day
logger = logging.getLogger(__name__)
FEATURE_COLUMNS = ["is_put", "moneyness", "premium_to_spot", "minutes_to_expiry",
"session_minute", "iv", "delta", "gamma", "theta", "vega", "rho",
"oi_log", "premium_return_15m", "premium_return_30m", "premium_volatility_15m"]
HISTORY_COLUMNS = ["premium_return_15m", "premium_return_30m", "premium_volatility_15m"]
AI_MAX_DATA_AGE_SECS = 300
SESSION_OPEN_MINUTE, SESSION_CLOSE_MINUTE = 9 * 60 + 15, 15 * 60 + 30
OUTLOOK_MAX_AGE_SECS = 5 * 86400
OUTLOOK_MAX_CONFIDENCE = "MEDIUM"
def market_closed(current):
from market_calendar import is_trading_day
minute = current.hour * 60 + current.minute
return not is_trading_day(current.date()) or not SESSION_OPEN_MINUTE <= minute < SESSION_CLOSE_MINUTE
def next_session_open(current):
from market_calendar import is_trading_day
day = current.date()
if not is_trading_day(day) or current.hour * 60 + current.minute >= SESSION_OPEN_MINUTE:
day = next_trading_day(day + timedelta(days=1))
return datetime(day.year, day.month, day.day, 9, 15, tzinfo=IST)
def outlook_observation(contract, quote, service, current):
"""Anchor a forecast to the last observed session when the market is not trading."""
price = float(number(quote.get("price"), "contract premium"))
float(number(quote.get("underlying_price"), "underlying price"))
quote_time = float(number(quote.get("quote_time"), "quote time"))
if not 0 <= current.timestamp() - quote_time <= OUTLOOK_MAX_AGE_SECS:
raise ValueError("Last-session snapshot is unavailable or too old")
anchor, history = quote_time, [{"time": quote_time, "close": price}]
try:
candles = service.chart(contract)["candles"] if service is not None else []
chart_anchor = float(number(candles[-1]["time"], "chart time"))
close = float(number(candles[-1]["close"], "chart premium"))
if 0 <= current.timestamp() - chart_anchor <= OUTLOOK_MAX_AGE_SECS and abs(quote_time - chart_anchor) <= 1800:
anchor = chart_anchor
price = price if abs(price / close - 1) <= 0.03 else close
history = [point for point in candles if point["time"] < anchor] + [{"time": anchor, "close": price}]
except (FoUnavailable, ValueError, TypeError, KeyError, IndexError):
pass # no usable chart history yet: fall back to the last-session snapshot alone
premium_history_context(history, as_of=datetime.fromtimestamp(anchor, IST), min_span=0)
return {**quote, "price": price, "quote_time": anchor, "context_quote_time": anchor,
"premium_history": history, "ai_snapshot_usable": True, "stale": True, "outlook": True}
class ForecastInputError(ValueError):
def __init__(self, reason, message):
super().__init__(message)
self.reason = reason
def _premium_history_prices(points, *, as_of):
end = as_of.timestamp()
prices = {}
for point in points:
try:
timestamp = float(number(point.get("time"), "history time"))
price = float(number(point.get("close"), "history price"))
except (ValueError, TypeError, AttributeError):
continue
if end - 32 * 60 <= timestamp <= end:
if timestamp in prices and prices[timestamp] != price:
raise ForecastInputError("invalid_history", "Conflicting premium history observations")
prices[timestamp] = price
times = sorted(prices)
if not times or end - times[-1] > 120:
raise ForecastInputError("history_unavailable", "Recent premium history is required")
return prices
def premium_history_features(points, *, as_of):
end = as_of.timestamp()
prices = _premium_history_prices(points, as_of=as_of)
times = sorted(prices)
latest = prices[times[-1]]
result = {}
for minutes in (15, 30):
lookback = end - minutes * 60
candidates = [timestamp for timestamp in times if abs(timestamp - lookback) <= 120]
if not candidates:
raise ForecastInputError("history_unavailable", "Premium history does not cover the required lookback")
nearest = min(candidates, key=lambda timestamp: (abs(timestamp - lookback), timestamp))
result[f"premium_return_{minutes}m"] = latest / prices[nearest] - 1
recent = [prices[timestamp] for timestamp in times if timestamp >= end - 15 * 60]
if len(recent) < 12:
raise ForecastInputError("history_unavailable", "Premium history is too sparse for volatility features")
result["premium_volatility_15m"] = float(np.std(np.diff(np.log(recent))) * math.sqrt(15))
return result
def premium_history_context(points, *, as_of, min_span=60):
prices = _premium_history_prices(points, as_of=as_of)
times = sorted(prices)
latest = times[-1]
candidates = [timestamp for timestamp in times if 60 <= latest - timestamp <= 120]
anchor = max(candidates) if candidates else None
try:
features = premium_history_features(points, as_of=as_of)
limited = False
except ForecastInputError as error:
if error.reason != "history_unavailable" or latest - times[0] < min_span:
raise
features = dict.fromkeys(HISTORY_COLUMNS)
limited = True
return {"features": features, "limited_history": limited,
"history_minutes": round((latest - times[0]) / 60, 2), "history_samples": len(times),
"premium_return_1m": prices[latest] / prices[anchor] - 1 if anchor is not None else None}
def build_features(contract, observation, *, as_of=None, allow_short_history=False):
current = (as_of or datetime.now(IST)).astimezone(IST)
expiry = datetime.fromisoformat(contract["expiry"] + "T15:30:00+05:30")
minutes = (expiry - current).total_seconds() / 60
if minutes <= 0:
raise ValueError("The derivative contract has expired")
if contract["instrument_type"] != "OPTION":
raise ValueError("A separate futures training universe is required")
price = float(number(observation.get("price"), "premium"))
spot = float(number(observation.get("underlying_price"), "underlying price"))
strike = float(number(contract["strike"], "strike"))
quoted = float(number(observation.get("quote_time"), "quote time"))
if not -5 <= current.timestamp() - quoted <= 120:
raise ValueError("The derivative observation is stale or future-dated")
put = contract["option_type"] == "PE"
iv_value = observation.get("iv")
if iv_value is None:
intrinsic = max(strike - spot if put else spot - strike, 0)
if not intrinsic < price < (strike if put else spot):
raise ForecastInputError("invalid_iv_inputs", "Observed premium is outside valid implied-volatility bounds")
inputs = [spot, strike, 0, minutes / 1440]
solved = mibian.BS(inputs, putPrice=price) if put else mibian.BS(inputs, callPrice=price)
iv_value = solved.impliedVolatility
iv = float(number(iv_value, "implied volatility"))
if iv > 500:
raise ValueError("Implied volatility is outside the supported domain")
greeks = mibian.BS([spot, strike, 0, minutes / 1440], volatility=iv)
values = [float(put), math.log(spot / strike), price / spot, minutes,
current.hour * 60 + current.minute - 555, iv,
greeks.putDelta if put else greeks.callDelta, greeks.gamma,
greeks.putTheta if put else greeks.callTheta, greeks.vega,
greeks.putRho if put else greeks.callRho,
math.log1p(float(number(observation.get("open_interest"), "open interest", allow_zero=True)))]
if observation.get("premium_history") is not None:
if allow_short_history:
history = premium_history_context(observation["premium_history"], as_of=current)["features"]
history = {column: float("nan") if value is None else value for column, value in history.items()}
else:
history = premium_history_features(observation["premium_history"], as_of=current)
else:
history = {column: float(observation[column]) for column in HISTORY_COLUMNS}
values.extend(history[column] for column in HISTORY_COLUMNS)
if not np.isfinite(values[:-len(HISTORY_COLUMNS)]).all() or any(
not math.isfinite(value) and not (allow_short_history and math.isnan(value))
for value in values[-len(HISTORY_COLUMNS):]):
raise ValueError("Derivative features are not finite")
return dict(zip(FEATURE_COLUMNS, values))
def attach_targets(frame, timeframe):
required = {"contract_id", "timestamp", "price"}
if not required.issubset(frame.columns):
raise ValueError("Training data requires exact contract_id, timestamp and price")
if timeframe not in {"INTRADAY", "1D"}:
raise ValueError("Unsupported derivative timeframe")
data = frame.copy()
data["timestamp"] = pd.to_datetime(data["timestamp"], utc=True)
if data.duplicated(["contract_id", "timestamp"]).any():
raise ValueError("Duplicate exact-contract observations are not allowed")
local = data["timestamp"].dt.tz_convert(IST)
if timeframe == "INTRADAY":
data["target_at"] = local.dt.normalize() + pd.Timedelta(hours=15)
else:
sessions = {day: next_trading_day(day + timedelta(days=1)) for day in local.dt.date.unique()}
data["target_at"] = pd.to_datetime(local.dt.date.map(sessions).astype(str)).dt.tz_localize(IST) + pd.Timedelta(hours=15, minutes=30)
expiry = data["expiry"] if "expiry" in data else data["contract_id"].str.extract(r"(\d{4}-\d{2}-\d{2})", expand=False)
if expiry.isna().any():
raise ValueError("Exact contract expiry is required")
expires_at = pd.to_datetime(expiry.astype(str)).dt.tz_localize(IST) + pd.Timedelta(hours=15, minutes=30)
targets = data[["contract_id", "timestamp", "price"]].rename(columns={"timestamp": "target_at", "price": "target_price"})
data = data[(data["target_at"] <= expires_at) & (data["timestamp"] < data["target_at"])].copy()
data["target_at"] = data["target_at"].dt.tz_convert("UTC")
data = data.merge(targets, on=["contract_id", "target_at"], how="inner", validate="many_to_one")
data = data[(data["price"] > 0) & (data["target_price"] >= 0)].copy()
data["target_log_return"] = np.log1p(data["target_price"]) - np.log1p(data["price"])
return data
def attach_excursion_targets(frame, timeframe):
if not {"high", "low"}.issubset(frame.columns):
raise ValueError("Observed premium high/low bars are required for reachable targets")
labels = attach_targets(frame, timeframe)
if labels.empty:
return labels.assign(future_high=pd.Series(dtype=float), future_low=pd.Series(dtype=float),
up_log_move=pd.Series(dtype=float), down_log_move=pd.Series(dtype=float))
sampled_columns = {"intraday_high", "intraday_low", "remaining_high", "remaining_low", "session_high", "session_low"}
if sampled_columns.issubset(frame.columns):
if timeframe == "INTRADAY":
labels["future_high"], labels["future_low"] = labels["intraday_high"], labels["intraday_low"]
if "intraday_complete" in labels:
labels = labels[labels["intraday_complete"].eq(True)].copy()
else:
sessions = frame[["contract_id", "timestamp", "session_high", "session_low"]].copy()
sessions["next_session_complete"] = frame["session_complete"].to_numpy() if "session_complete" in frame else True
sessions["timestamp"] = pd.to_datetime(sessions["timestamp"], utc=True)
sessions = sessions.rename(columns={"timestamp": "target_at", "session_high": "next_session_high", "session_low": "next_session_low"})
labels = labels.merge(sessions, on=["contract_id", "target_at"], how="inner", validate="many_to_one")
labels["future_high"] = labels[["remaining_high", "next_session_high"]].max(axis=1)
labels["future_low"] = labels[["remaining_low", "next_session_low"]].min(axis=1)
labels = labels[labels["next_session_complete"].eq(True)].copy()
if "remaining_complete" in labels:
labels = labels[labels["remaining_complete"].eq(True)].copy()
else:
history = frame.copy()
history["timestamp"] = pd.to_datetime(history["timestamp"], utc=True)
history = history[(history["low"] >= 0) & (history["high"] >= history["price"]) & (history["low"] <= history["price"])]
groups = {identity: group.sort_values("timestamp") for identity, group in history.groupby("contract_id")}
highs, lows = [], []
for row in labels.itertuples(index=False):
bars = groups.get(row.contract_id)
window = bars[(bars["timestamp"] > row.timestamp) & (bars["timestamp"] <= row.target_at)] if bars is not None else None
highs.append(window["high"].max() if window is not None and not window.empty else float("nan"))
lows.append(window["low"].min() if window is not None and not window.empty else float("nan"))
labels["future_high"], labels["future_low"] = highs, lows
labels = labels.dropna(subset=["future_high", "future_low"]).copy()
labels["up_log_move"] = np.maximum(0, np.log1p(labels["future_high"]) - np.log1p(labels["price"]))
labels["down_log_move"] = np.maximum(0, np.log1p(labels["price"]) - np.log1p(labels["future_low"]))
return labels
def promotion_passes(metrics):
keys = ("selected_samples", "test_expiries", "forecast_coverage", "direction_accuracy", "range_coverage", "majority_baseline")
if not isinstance(metrics, dict) or any(not isinstance(metrics.get(key), (int, float))
or isinstance(metrics[key], bool) or not math.isfinite(metrics[key]) for key in keys):
return False
if any(not 0 <= metrics[key] <= 1 for key in ("forecast_coverage", "direction_accuracy", "range_coverage", "majority_baseline")):
return False
return bool(metrics["selected_samples"] >= 200 and metrics["test_expiries"] >= 5
and metrics["forecast_coverage"] >= 0.10
and metrics["direction_accuracy"] > 0.75 and metrics["range_coverage"] > 0.75
and metrics["direction_accuracy"] >= metrics["majority_baseline"] + 0.01)
def reachable_quality_passes(metrics):
keys = ("selected_samples", "test_expiries", "forecast_coverage", "target_reach_rate", "far_bound_coverage", "minimum_target_move_pct")
if not isinstance(metrics, dict) or any(not isinstance(metrics.get(key), (int, float))
or isinstance(metrics[key], bool) or not math.isfinite(metrics[key]) for key in keys):
return False
if any(not 0 <= metrics[key] <= 1 for key in ("forecast_coverage", "target_reach_rate", "far_bound_coverage")):
return False
return bool(metrics["selected_samples"] >= 200 and metrics["test_expiries"] >= 5
and metrics["forecast_coverage"] >= 0.10 and metrics["target_reach_rate"] > 0.75
and metrics["far_bound_coverage"] > 0.75 and metrics["minimum_target_move_pct"] >= 5)
def model_quality_passes(metadata):
return reachable_quality_passes(metadata.get("metrics")) if metadata.get("target_kind") == "reachable" else promotion_passes(metadata.get("metrics"))
def reachable_levels(models, frame, adjustments):
result = {}
for side in ("up", "down"):
near = np.maximum(0, models[side + "_near"].predict(frame))
target = np.maximum(0, models[side + "_target"].predict(frame) - adjustments[side + "_target"])
far = np.maximum(target, np.maximum(near, models[side + "_far"].predict(frame) + adjustments[side + "_far"]))
result[side + "_near"] = np.minimum(near, target)
result[side + "_target"], result[side + "_far"] = target, far
if not all(np.isfinite(value).all() for value in result.values()):
raise ValueError("Non-finite reachable premium levels")
result["is_up"] = result["up_target"] >= result["down_target"]
return result
def reachable_prediction(models, vector, metadata, observation, *, allow_non_actionable=False):
levels = reachable_levels(models, vector, metadata["reach_adjustments"])
upward = bool(levels["is_up"][0])
side = "up" if upward else "down"
price = float(observation["price"])
anchor = math.log1p(price)
sign = 1 if upward else -1
prices = [max(0, math.expm1(anchor + sign * levels[side + role][0])) for role in ("_near", "_target", "_far")]
actionable = abs(prices[1] / price - 1) * 100 >= metadata["minimum_target_move_pct"]
if not actionable and not allow_non_actionable:
return {"available": False, "no_trade_reason": "insufficient_move"}
metrics = metadata["metrics"]
return {"available": True, "no_trade_reason": None, "direction": "BULLISH" if upward else "BEARISH",
"actionable": actionable, "minimum_target_move_pct": metadata["minimum_target_move_pct"],
"confidence": "HIGH" if metrics["target_reach_rate"] >= 0.90 else "MEDIUM",
"confidence_prob": None, "confidence_basis": "backtested_target_reach_rate",
"target_price_lo": round(min(prices[0], prices[2]), 2), "expected_target_price": round(prices[1], 2),
"target_price_hi": round(max(prices[0], prices[2]), 2), "current_price": price,
"estimated_high": round(math.expm1(anchor + levels["up_far"][0]), 2),
"estimated_low": round(max(0, math.expm1(anchor - levels["down_far"][0])), 2),
"range_semantics": "reachable_premium_excursions", "target_quantile": 0.2,
"target_reach_rate": metrics["target_reach_rate"], "far_bound_coverage": metrics["far_bound_coverage"],
"closing_direction_accuracy": metrics.get("direction_accuracy_all"),
"as_of": datetime.fromtimestamp(observation["quote_time"], IST).isoformat()}
def model_data_fresh(manifest, *, now=None):
try:
last_observed = datetime.fromisoformat(manifest["data_end"]).date()
age = ((now or datetime.now(IST)).astimezone(IST).date() - last_observed).days
maximum = int(os.getenv("FO_MODEL_MAX_AGE_DAYS", "180"))
return 0 <= age <= maximum
except (KeyError, ValueError, TypeError):
return False
class FoPremiumPredictor:
def __init__(self, model_dir=None):
self.model_dir = model_dir or os.getenv("FO_MODEL_DIR", os.path.join(os.path.dirname(__file__), "fo_models"))
self._manifest = None
self._manifest_key = None
self._models = {}
self._lock = threading.Lock()
def manifest(self):
path = os.path.join(self.model_dir, "manifest.json")
with self._lock:
try:
details = os.stat(path)
key = (details.st_mtime_ns, details.st_size)
if self._manifest_key != key:
if details.st_size > 1_000_000:
raise ValueError("Invalid derivative model metadata")
with open(path, encoding="utf-8") as stream:
manifest = json.load(stream)
if not isinstance(manifest, dict) or manifest.get("schema_version") != 1 or manifest.get("feature_columns") != FEATURE_COLUMNS:
raise ValueError("Incompatible derivative model metadata")
self._manifest, self._manifest_key = manifest, key
self._models.clear()
return self._manifest
except (OSError, ValueError, TypeError):
self._manifest, self._manifest_key = None, None
self._models.clear()
return None
def has_approved_models(self, contract):
manifest = self.manifest()
return bool(manifest and model_data_fresh(manifest)
and contract["underlying"] in manifest.get("supported_underlyings", [])
and contract["instrument_type"] == manifest.get("instrument_type")
and any(metadata.get("approved") is True and model_quality_passes(metadata)
for metadata in manifest.get("timeframes", {}).values()))
def _load_models(self, timeframe, metadata):
artifact = metadata.get("artifact")
if not artifact or os.path.basename(artifact) != artifact or not artifact.endswith(".joblib"):
raise ValueError("Invalid derivative model artifact")
key = (timeframe, metadata.get("artifact_sha256"))
with self._lock:
if key not in self._models:
path = os.path.join(self.model_dir, artifact)
if os.path.getsize(path) > 50_000_000:
raise ValueError("Derivative artifact exceeds its size limit")
with open(path, "rb") as stream:
if hashlib.file_digest(stream, "sha256").hexdigest() != key[1]:
raise ValueError("Derivative artifact checksum mismatch")
self._models[key] = joblib.load(path)
return self._models[key]
def predict(self, contract, observation, windows, *, now=None):
manifest = self.manifest()
result = {"forecast_status": "not_trained", "timeframes": {}, "forecast_metrics": {}}
if manifest:
result["forecast_model"] = {key: manifest.get(key) for key in ("source", "source_license", "data_end", "range_semantics", "serving_mode")}
for timeframe, window in windows.items():
status = "not_trained"
metadata = (manifest or {}).get("timeframes", {}).get(timeframe, {})
metrics = metadata.get("metrics", {})
result["forecast_metrics"][timeframe] = metrics
if not window.get("eligible"):
status = window.get("status", "after_expiry")
elif manifest:
if contract["underlying"] not in manifest.get("supported_underlyings", []) or contract["instrument_type"] != manifest.get("instrument_type"):
status = "unsupported_contract"
elif metadata:
status = "ready" if metadata.get("approved") is True and model_quality_passes(metadata) else "quality_gate_failed"
if status == "ready" and not model_data_fresh(manifest, now=now):
status = "model_stale"
if status in {"quality_gate_failed", "model_stale"} and manifest.get("serving_mode") == "experimental" and metadata.get("artifact"):
status = "experimental"
prediction = {"available": False, "no_trade_reason": status, "end_at": window.get("end_at"), "source": "fo_ml"}
if status == "experimental":
prediction.update({"experimental": True, "accuracy_validated": False,
"validation_passed": model_quality_passes(metadata),
"model_data_stale": not model_data_fresh(manifest, now=now)})
if status in {"ready", "experimental"}:
input_status = "invalid_features"
try:
input_status = "stale_input"
current = (now or datetime.now(IST)).astimezone(IST)
quoted = float(number(observation.get("quote_time"), "quote time"))
if observation.get("stale") is True:
raise ValueError("Stale derivative observation")
if not -5 <= current.timestamp() - quoted <= 120:
raise ValueError("The derivative observation is stale or future-dated")
input_status = "invalid_features"
quoted_at = datetime.fromtimestamp(quoted, IST)
if status == "experimental" and observation.get("premium_history") is not None:
history_context = premium_history_context(observation["premium_history"], as_of=quoted_at)
prediction.update({key: history_context[key] for key in
("limited_history", "history_minutes", "history_samples")})
features = build_features(contract, {**observation, "iv": None},
as_of=quoted_at, allow_short_history=status == "experimental")
input_status = "outside_training_domain"
for column in ("minutes_to_expiry", "moneyness", "iv", "premium_to_spot", "session_minute"):
lower, upper = metadata["feature_bounds"][column]
if not lower <= features[column] <= upper:
raise ValueError("Observation is outside the derivative training domain")
input_status = "artifact_unavailable"
models = self._load_models(timeframe, metadata)
input_status = "invalid_model_output"
vector = pd.DataFrame([features], columns=FEATURE_COLUMNS)
if metadata.get("target_kind") == "reachable":
prediction.update(reachable_prediction(models, vector, metadata, observation,
allow_non_actionable=status == "experimental"))
result["timeframes"][timeframe] = prediction
continue
classifier = models["direction"]
probabilities = classifier.predict_proba(vector)[0]
if not np.isfinite(probabilities).all() or np.any(probabilities < 0) or np.any(probabilities > 1):
raise ValueError("Invalid derivative model probabilities")
index = int(np.argmax(probabilities))
if probabilities[index] < metadata["min_probability"]:
prediction["no_trade_reason"] = "low_confidence"
else:
quantiles = sorted(float(models[name].predict(vector)[0]) for name in ("lo", "mid", "hi"))
quantiles[0] -= metadata["conformal_adjustment"]
quantiles[2] += metadata["conformal_adjustment"]
prices = [max(0.0, math.expm1(math.log1p(float(observation["price"])) + value)) for value in quantiles]
if not np.isfinite(prices).all():
raise ValueError("Invalid derivative forecast")
prediction.update({"available": True, "no_trade_reason": None,
"direction": str(classifier.classes_[index]), "confidence": "MEDIUM",
"confidence_prob": float(probabilities[index]), "current_price": observation["price"],
"target_price_lo": round(prices[0], 2), "expected_target_price": round(prices[1], 2),
"target_price_hi": round(prices[2], 2), "range_semantics": "terminal_premium_q10_q90",
"as_of": datetime.fromtimestamp(observation["quote_time"], IST).isoformat(),
"holdout_direction_accuracy": metrics["direction_accuracy"],
"holdout_range_coverage": metrics["range_coverage"]})
except (OSError, ValueError, TypeError, KeyError, IndexError, AttributeError, OverflowError) as error:
prediction["no_trade_reason"] = "model_input_unavailable"
prediction["input_status"] = getattr(error, "reason", input_status)
result["timeframes"][timeframe] = prediction
for prediction in result["timeframes"].values():
if prediction.get("experimental"):
prediction.update({"confidence": "LOW", "confidence_prob": None,
"confidence_basis": "experimental_unvalidated"})
reasons = {value["no_trade_reason"] for value in result["timeframes"].values()}
if any(value["available"] for value in result["timeframes"].values()):
result["forecast_status"] = "experimental" if any(value.get("experimental") and value["available"] for value in result["timeframes"].values()) else "ready"
elif "quality_gate_failed" in reasons:
result["forecast_status"] = "quality_gate_failed"
elif "unsupported_contract" in reasons:
result["forecast_status"] = "unsupported_contract"
elif "model_input_unavailable" in reasons:
result["forecast_status"] = "model_input_unavailable"
elif "model_stale" in reasons:
result["forecast_status"] = "model_stale"
elif "low_confidence" in reasons:
result["forecast_status"] = "low_confidence"
elif "insufficient_move" in reasons:
result["forecast_status"] = "insufficient_move"
return result
_predictor = None
_predictor_lock = threading.Lock()
_reviews = {}
_review_lock = threading.Lock()
_review_slots = threading.BoundedSemaphore(3)
AI_SLOT_WAIT_SECS = 8
AI_PROVIDER_ATTEMPTS = 2
AI_RETRY_PAUSE_SECS = 1.5
# Total time one forecast may spend waiting for a rate-limited provider's short cooldown to end,
# instead of failing at once with "provider unavailable" while one frees up in a few seconds.
AI_COOLDOWN_WAIT_SECS = 10
def get_predictor():
global _predictor
with _predictor_lock:
if _predictor is None:
_predictor = FoPremiumPredictor()
return _predictor
def review_forecast(contract, observation, prediction):
if not prediction.get("available") or os.getenv("FO_AI_ENABLED", "1") != "1":
return {"available": False, "source": "fo_ai_review"}
key = (contract["contract_id"], prediction.get("end_at"), prediction.get("direction"),
prediction["target_price_lo"], prediction["expected_target_price"], prediction["target_price_hi"])
with _review_lock:
cached = _reviews.get(key)
if cached and datetime.now(IST).timestamp() - cached[0] < 300:
return dict(cached[1])
if not _review_slots.acquire(blocking=False):
return {"available": False, "source": "fo_ai_review", "reason": "busy"}
try:
from llm_client import make_chat_call
context = {"underlying": contract["underlying"], "expiry": contract["expiry"],
"strike": contract["strike"], "option_type": contract["option_type"],
"premium": observation.get("price"), "underlying_price": observation.get("underlying_price"),
"quote_time": observation.get("quote_time"), "validated_model": prediction}
content, provider, model = make_chat_call(messages=[
{"role": "system", "content": "Review a trained NSE option-premium forecast for paper-trading research. "
"Use only the supplied contract, expiry, observation and model output. Never invent price levels, IV, "
"Greeks, news, accuracy or returns. You are commentary, not an independent calibrated price model. "
"Explain premium direction versus underlying direction and expiry/time-decay risk. "
"Return JSON with exactly one summary string, at most 400 characters."},
{"role": "user", "content": json.dumps(context, allow_nan=False)}],
max_tokens=220, temperature=0, fast_fail_on_rate_limit=True, max_retries=0, call_timeout=6)
parsed = json.loads(content)
summary = parsed.get("summary")
if not isinstance(summary, str) or not summary.strip() or len(summary) > 600:
raise ValueError("Invalid derivative AI review")
result = {"available": True, "source": provider + ":" + model, "summary": summary.strip(),
"independent_range": False, "basis": "validated_model_review",
"direction": prediction["direction"], "target_price_lo": prediction["target_price_lo"],
"expected_target_price": prediction["expected_target_price"], "target_price_hi": prediction["target_price_hi"]}
with _review_lock:
if len(_reviews) >= 256:
_reviews.clear()
_reviews[key] = (datetime.now(IST).timestamp(), result)
return result
except (RuntimeError, OSError, ValueError, TypeError, KeyError, AttributeError):
return {"available": False, "source": "fo_ai_review", "reason": "unavailable"}
finally:
_review_slots.release()
def ai_forecasts(contract, observation, windows, *, refresh=False):
return _ai_forecasts(contract, observation, windows, refresh=refresh)
def _wait_for_provider(budget):
"""Sleep until a cooling-down provider is usable, if that happens within `budget` seconds.
Returns the seconds slept. A cooldown longer than the budget (or none that will recover) is not
waited on: the attempt then fails fast and the frontend's own retry schedule takes over.
"""
if budget <= 0:
return 0.0
from llm_client import seconds_until_provider_available
wait = seconds_until_provider_available()
if not wait or wait > budget:
return 0.0
time.sleep(wait + 0.2)
return wait + 0.2
def _chat_with_retry(make_chat_call, messages, call_timeout):
"""Retry transient provider failures with a rotated lead provider and a fresh time budget."""
last_error = None
cooldown_budget = AI_COOLDOWN_WAIT_SECS
for attempt in range(max(1, AI_PROVIDER_ATTEMPTS)):
cooldown_budget -= _wait_for_provider(cooldown_budget)
try:
return make_chat_call(messages=messages, max_tokens=550, temperature=0, fast_fail_on_rate_limit=True,
max_retries=0, call_timeout=call_timeout, json_mode=True, task_offset=attempt)
except (RuntimeError, OSError) as error:
last_error = error
logger.warning("F&O AI provider attempt %d/%d failed: %s", attempt + 1, AI_PROVIDER_ATTEMPTS, error)
if attempt + 1 < AI_PROVIDER_ATTEMPTS:
time.sleep(AI_RETRY_PAUSE_SECS)
raise last_error
def _ai_forecasts(contract, observation, windows, *, refresh=False, call_timeout=20, repair=True, repair_errors=None):
current = datetime.now(IST)
results = {timeframe: {"available": False, "source": "fo_ai", "accuracy_validated": False,
"independent_range": True, "basis": "independent_llm_contract_forecast",
"no_trade_reason": window.get("status") if not window.get("eligible") else "ai_unavailable"}
for timeframe, window in windows.items()}
if os.getenv("FO_AI_ENABLED", "1") != "1":
for timeframe, window in windows.items():
if window.get("eligible"):
results[timeframe]["no_trade_reason"] = "ai_disabled"
return results
eligible = {timeframe: window for timeframe, window in windows.items() if window.get("eligible")}
if not eligible:
return results
try:
price = float(number(observation.get("price"), "contract premium"))
spot = float(number(observation.get("underlying_price"), "underlying price"))
quoted = float(number(observation.get("quote_time"), "quote time"))
context_quoted = float(number(observation.get("context_quote_time", quoted), "context quote time"))
outlook = observation.get("outlook") is True
max_age = OUTLOOK_MAX_AGE_SECS if outlook else AI_MAX_DATA_AGE_SECS
if observation.get("stale") is True and not observation.get("ai_snapshot_usable"):
raise ValueError("Stale derivative observation")
if any(not -5 <= current.timestamp() - timestamp <= max_age for timestamp in (quoted, context_quoted)):
raise ValueError("Derivative snapshot is too old")
start = next_session_open(current) if outlook else current
expiry = datetime.fromisoformat(contract["expiry"] + "T15:30:00+05:30")
for window in eligible.values():
end = datetime.fromisoformat(window["end_at"])
if not current < end <= expiry:
raise ValueError("Forecast window exceeds contract expiry")
as_of = datetime.fromtimestamp(quoted, IST)
history_context = premium_history_context(observation.get("premium_history") or [], as_of=as_of, min_span=0)
history = history_context["features"]
except (ValueError, TypeError, KeyError):
for timeframe in eligible:
results[timeframe]["no_trade_reason"] = "ai_input_unavailable"
return results
key = ("independent_ai", contract["contract_id"], history_context["limited_history"], outlook,
tuple((timeframe, window["end_at"]) for timeframe, window in eligible.items()))
with _review_lock:
if refresh:
_reviews.pop(key, None)
cached = _reviews.get(key)
if cached and 0 <= current.timestamp() - cached[0] < 120:
copies = {timeframe: dict(prediction) for timeframe, prediction in cached[1].items()}
cache_valid = all(prediction.get("available") for prediction in copies.values())
for prediction in copies.values():
if not prediction.get("available"):
continue
target = prediction.get("expected_target_price")
direction = prediction.get("direction")
if target is not None and ((direction == "BULLISH" and price >= target)
or (direction == "BEARISH" and price <= target)
or (direction == "NEUTRAL" and not prediction["target_price_lo"] <= price <= prediction["target_price_hi"])):
cache_valid = False
break
age = current.timestamp() - datetime.fromisoformat(prediction["as_of"]).timestamp()
context_age = current.timestamp() - datetime.fromisoformat(prediction["context_as_of"]).timestamp()
if any(not -5 <= value <= max_age for value in (age, context_age)):
cache_valid = False
break
prediction.update({"data_age_seconds": round(age), "context_age_seconds": round(context_age),
"input_delayed": max(age, context_age) > 120, "cached": True})
if cache_valid:
return {**results, **copies}
if not _review_slots.acquire(timeout=AI_SLOT_WAIT_SECS):
for timeframe in eligible:
results[timeframe]["no_trade_reason"] = "ai_busy"
return results
try:
context = {"contract_id": contract["contract_id"], "underlying": contract["underlying"],
"instrument_type": contract["instrument_type"], "expiry": contract["expiry"],
"strike": contract.get("strike"), "option_type": contract.get("option_type"),
"current_price": price, "underlying_price": spot,
"as_of": datetime.fromtimestamp(quoted, IST).isoformat(), "forecast_at": current.isoformat(),
"horizons": {timeframe: {**window, "start_at": start.isoformat(),
"calendar_minutes": round((datetime.fromisoformat(window["end_at"]) - start).total_seconds() / 60)}
for timeframe, window in eligible.items()},
"data_age_seconds": round(current.timestamp() - quoted),
"context_as_of": datetime.fromtimestamp(context_quoted, IST).isoformat(),
"history_context": {key: value for key, value in history_context.items() if key != "features"},
"premium_history_features": history, "open_interest": observation.get("open_interest"),
"volume": observation.get("volume"), "provider_iv": observation.get("provider_iv")}
if outlook:
context["market_state"] = {"mode": "previous_session_outlook", "market_open_now": False,
"next_session_open": start.isoformat(),
"note": "current_price and underlying_price are the last observed values of the previous session"}
if repair_errors:
context["validation_errors"] = repair_errors
if contract["instrument_type"] == "OPTION":
strike = float(number(contract["strike"], "strike"))
intrinsic = max(strike - spot if contract["option_type"] == "PE" else spot - strike, 0)
context["option_context"] = {"moneyness": "ITM" if intrinsic > 0 else "ATM" if strike == spot else "OTM",
"intrinsic_value": intrinsic, "observed_time_value": price - intrinsic,
"iv_inputs_valid": intrinsic < price < (strike if contract["option_type"] == "PE" else spot)}
try:
context["derived_features"] = build_features(contract, {**observation, "iv": None}, as_of=as_of)
context["greeks_basis"] = "Mibian zero-rate Black-Scholes approximation from observed prices"
except (ValueError, TypeError, KeyError, OverflowError):
context["derived_features"] = None
outlook_note = ("The market is closed or not yet open, so this is a previous-session outlook: horizons start at "
"market_state.next_session_open from the last observed premium. The opening gap is unknown; widen the range "
"for it and keep confidence modest. " if outlook else "")
from llm_client import make_chat_call
call_started = time.monotonic()
messages = [
{"role": "system", "content": outlook_note + "Estimate the change in the exact NSE derivative contract price for paper-trading research. "
"Predict reachable premium levels during the supplied window, not just its closing price. "
"INTRADAY ends today at 15:00 IST; 1D includes the remaining current session and ends at the "
"next trading session's 15:30 IST close. Assess each horizon separately using its end_at, "
"time to expiry and additional overnight/time-decay risk. Never copy a horizon's estimate "
"to another merely because both use the same current quote; equal estimates require your own assessment. "
"The expected change is your actionable reachable target; lo/hi describe nearer/farther reachable levels. "
"Make your own directional assessment from the supplied observed contract history and expiry context; "
"do not copy an equity forecast or assume a failed ML model is valid. BULLISH/BEARISH describe the "
"contract premium, not its underlying: a put may lose value when the index rises. Consider time decay, "
"Use the supplied option_context moneyness as a fact; an ITM put has strike above spot. "
"If IV inputs are invalid or derived features are absent, do not invent Greeks or implied volatility. "
"moneyness, liquidity and volatility. Use only supplied data; do not invent news, metrics or accuracy. "
"When limited_history is true, only a short observed window is available. Treat missing long-lookback "
"features as unknown, never zero or extrapolated 15/30-minute returns. Keep confidence LOW and "
"acknowledge the limited history in each summary. "
"Your forecast has no validated accuracy. Return JSON {\"forecasts\":{TIMEFRAME:{\"direction\": "
"\"BULLISH|BEARISH|NEUTRAL\",\"expected_change_pct\":number,\"predicted_return_lo\":number,"
"\"predicted_return_hi\":number,\"confidence\":\"LOW|MEDIUM|HIGH\",\"summary\":string}}}. "
"Include each supplied horizon only. Percent changes must be relative to current_price, ordered "
"-100 <= lo <= expected <= hi <= 500. BULLISH requires expected > 1; BEARISH expected < -1; "
"NEUTRAL requires -1 <= expected <= 1. Each summary must be at most 300 characters. "
"Do not output target prices, confidence probabilities or claims of backtested reliability."},
{"role": "user", "content": json.dumps(context, allow_nan=False)}]
content, provider, model = _chat_with_retry(make_chat_call, messages, call_timeout)
parsed = json.JSONDecoder().raw_decode(content[content.find("{"):])[0]
forecasts = parsed.get("forecasts") if isinstance(parsed, dict) else None
if not isinstance(forecasts, dict):
raise ValueError("Invalid derivative AI forecast")
completed = datetime.now(IST)
response_errors = {}
for timeframe, window in eligible.items():
if completed >= datetime.fromisoformat(window["end_at"]):
results[timeframe]["no_trade_reason"] = "expired" if completed >= expiry else "session_closed"
continue
if any(completed.timestamp() - timestamp > max_age for timestamp in (quoted, context_quoted)):
results[timeframe]["no_trade_reason"] = "ai_input_unavailable"
continue
raw = forecasts.get(timeframe)
if not isinstance(raw, dict):
results[timeframe]["no_trade_reason"] = "ai_invalid_response"
response_errors[timeframe] = "A forecast object is required for this horizon"
continue
try:
values = [raw[field] for field in ("predicted_return_lo", "expected_change_pct", "predicted_return_hi")]
if any(isinstance(value, bool) or not isinstance(value, (float, int)) or not math.isfinite(value) for value in values):
raise ValueError("Invalid derivative change estimate")
lower, change, upper = values
direction, confidence, summary = raw.get("direction"), raw.get("confidence"), raw.get("summary")
if not -100 <= lower <= change <= upper <= 500 or confidence not in {"LOW", "MEDIUM", "HIGH"}:
raise ValueError("Invalid derivative forecast range")
expected_direction = "BULLISH" if change > 1 else "BEARISH" if change < -1 else "NEUTRAL"
if direction != expected_direction:
raise ValueError(f"Direction must be {expected_direction} for expected_change_pct={change}")
if not isinstance(summary, str) or not summary.strip() or len(summary) > 600:
raise ValueError("A nonempty summary of at most 600 characters is required")
results[timeframe] = {"available": True, "source": provider + ":" + model,
"basis": "independent_llm_contract_forecast", "accuracy_validated": False,
"range_semantics": "ai_reachable_premium_targets",
"independent_range": True, "direction": direction,
"confidence": "LOW" if history_context["limited_history"] else
"MEDIUM" if outlook and confidence == "HIGH" else confidence,
"premarket_outlook": outlook,
**{key: history_context[key] for key in ("limited_history", "history_minutes", "history_samples")},
"expected_change_pct": change, "predicted_return_lo": lower, "predicted_return_hi": upper,
"current_price": price, "target_price_lo": round(price * (1 + lower / 100), 2),
"expected_target_price": round(price * (1 + change / 100), 2),
"target_price_hi": round(price * (1 + upper / 100), 2), "summary": summary.strip(),
"as_of": datetime.fromtimestamp(quoted, IST).isoformat(), "end_at": window["end_at"],
"generated_at": completed.isoformat(), "cached": False,
"context_as_of": datetime.fromtimestamp(context_quoted, IST).isoformat(),
"data_age_seconds": round(completed.timestamp() - quoted),
"context_age_seconds": round(completed.timestamp() - context_quoted),
"input_delayed": max(completed.timestamp() - quoted, completed.timestamp() - context_quoted) > 120,
"no_trade_reason": None}
except (ValueError, TypeError, KeyError) as error:
results[timeframe]["no_trade_reason"] = "ai_invalid_response"
response_errors[timeframe] = str(error)
continue
except (RuntimeError, OSError):
for timeframe in eligible:
results[timeframe]["no_trade_reason"] = "ai_provider_unavailable"
return results
except (ValueError, TypeError, KeyError, AttributeError):
for timeframe in eligible:
results[timeframe]["no_trade_reason"] = "ai_invalid_response"
return results
finally:
_review_slots.release()
failed = {timeframe: window for timeframe, window in eligible.items()
if results[timeframe].get("no_trade_reason") == "ai_invalid_response"}
remaining = call_timeout - (time.monotonic() - call_started)
if repair and failed and remaining > 0:
results.update(_ai_forecasts(contract, observation, failed, refresh=True,
call_timeout=remaining, repair=False, repair_errors={timeframe: response_errors[timeframe] for timeframe in failed}))
with _review_lock:
if all(results[timeframe].get("available") for timeframe in eligible):
if len(_reviews) >= 256:
_reviews.clear()
_reviews[key] = (datetime.now(IST).timestamp(), {timeframe: results[timeframe] for timeframe in eligible})
return results
def contract_forecasts(contract, quote, windows, *, service=None, refresh=False):
predictor = get_predictor()
observation = quote or {}
input_reason = None
needs_context = any(window.get("eligible") for window in windows.values()) and (
predictor.has_approved_models(contract) or (predictor.manifest() or {}).get("serving_mode") == "experimental"
or os.getenv("FO_AI_ENABLED", "1") == "1")
outlook = False
if needs_context and quote:
current = datetime.now(IST)
if market_closed(current):
try:
observation = outlook_observation(contract, quote, service, current)
outlook = True
except (FoUnavailable, ValueError, TypeError, KeyError, IndexError):
observation = quote
if needs_context and not outlook:
try:
input_reason = "ai_quote_unavailable"
number(observation.get("price"), "contract premium")
number(observation.get("underlying_price"), "underlying price")
context_time = float(number(observation.get("quote_time"), "context quote time"))
current = datetime.now(IST)
limit = AI_MAX_DATA_AGE_SECS if os.getenv("FO_AI_ENABLED", "1") == "1" else 120
input_reason = "ai_stale_data"
if not -5 <= current.timestamp() - context_time <= limit:
raise ValueError("Contract context is stale")
# Chart history only enriches the forecast. A newly added contract often has none yet (or NSE's
# chart lags the quote), and a fresh quote alone is a valid single-observation, LOW-confidence
# history, so a missing/lagging chart must not block the forecast.
input_reason = "ai_history_unavailable"
quote_point = {"time": context_time, "close": observation["price"]}
chart_points = []
try:
chart_points = [point for point in service.chart(contract)["candles"]
if isinstance(point, dict) and point["time"] < context_time] if service is not None else []
except (FoUnavailable, ValueError, TypeError, KeyError, IndexError):
chart_points = []
context_at = datetime.fromtimestamp(context_time, IST)
try:
premium_history_context(chart_points + [quote_point], as_of=context_at, min_span=0)
history = chart_points + [quote_point]
except ValueError:
history = [quote_point]
premium_history_context(history, as_of=context_at, min_span=0)
current = datetime.now(IST)
input_reason = "ai_stale_data"
if not -5 <= current.timestamp() - context_time <= limit:
raise ValueError("Contract context is stale")
observation = {**observation, "premium_history": history, "context_quote_time": context_time,
"ai_snapshot_usable": True,
"stale": observation.get("stale") is True or current.timestamp() - context_time > 120}
input_reason = None
except (FoUnavailable, ValueError, TypeError, KeyError, IndexError):
observation = {"stale": True}
result = predictor.predict(contract, {"stale": True} if outlook else observation, windows)
if outlook or input_reason:
for prediction in result["timeframes"].values():
if prediction.get("no_trade_reason") == "model_input_unavailable":
prediction["input_status"] = "market_not_open" if outlook else input_reason.removeprefix("ai_")
forecasts = ai_forecasts(contract, observation, windows, refresh=refresh)
for forecast in forecasts.values():
if input_reason and forecast.get("no_trade_reason") == "ai_input_unavailable":
forecast["no_trade_reason"] = input_reason
for timeframe, prediction in result["timeframes"].items():
prediction["ai_forecast"] = forecasts.get(timeframe, {"available": False, "source": "fo_ai"})
result["ai_forecast_status"] = "ready" if any(prediction.get("available") for prediction in forecasts.values()) else "unavailable"
return result