"""Verified, explicitly licensed minute history for derivative reach research.""" from concurrent.futures import ThreadPoolExecutor from datetime import date import json import os import re import tempfile from urllib.parse import quote import numpy as np import pandas as pd import requests from fo import IST from fo_predictor import HISTORY_COLUMNS, premium_history_features from research.train_fo_predictor import ANCHORS, CACHE_ROOT, archive_digest, write_json_atomic OPTION_SOURCE = "samardubey/niftybanknifty-options-data" OPTION_PREFIX = "filtered_feather_folder/filtered_feather_folder/" SPOT_SOURCE = "debashis74017/nifty-50-minute-data" RECENT_CACHE = os.path.join(CACHE_ROOT, "recent_options") def download_file(source, name, destination, maximum): if os.path.isfile(destination) and os.path.getsize(destination) > 0: return destination directory = os.path.dirname(os.path.abspath(destination)) os.makedirs(directory, exist_ok=True) temporary = None try: url = "https://www.kaggle.com/api/v1/datasets/download/" + source + "/" + quote(name, safe="") with requests.get(url, stream=True, timeout=(5, 30)) as response: response.raise_for_status() if int(response.headers.get("Content-Length", "0")) > maximum: raise ValueError("Source file exceeds its download budget") size = 0 with tempfile.NamedTemporaryFile(dir=directory, delete=False) as output: temporary = output.name for chunk in response.iter_content(1024 * 1024): size += len(chunk) if size > maximum: raise ValueError("Source file exceeded its download budget") output.write(chunk) output.flush() os.fsync(output.fileno()) os.replace(temporary, destination) temporary = None return destination finally: if temporary and os.path.exists(temporary): os.unlink(temporary) def list_option_files(): files = [] parameters = {"pageSize": 200} for page in range(20): response = requests.get("https://www.kaggle.com/api/v1/datasets/list/" + OPTION_SOURCE, params=parameters, timeout=(5, 20)) response.raise_for_status() body = response.json() for row in body.get("datasetFiles", []): name = row.get("name", "") if re.fullmatch(re.escape(OPTION_PREFIX) + r"\d{4}-\d{2}-\d{2}-index-nfo-data\.feather", name): files.append(name) token = body.get("nextPageToken") if not token: return sorted(set(files)) parameters["pageToken"] = token raise ValueError("Option catalog exceeded its page budget") def load_spots(directory): path = download_file(SPOT_SOURCE, "NIFTY 50_minute.csv", os.path.join(directory, "nifty_spot_minute.csv"), 80_000_000) raw = pd.read_csv(path, usecols=["date", "close"]) timestamps = pd.to_datetime(raw["date"]) timestamps = timestamps.dt.tz_localize(IST) if timestamps.dt.tz is None else timestamps.dt.tz_convert(IST) result = pd.DataFrame({"timestamp": timestamps + pd.Timedelta(minutes=1), "underlying_price": pd.to_numeric(raw["close"], errors="raise")}).drop_duplicates() if result.duplicated("timestamp").any(): raise ValueError("Conflicting underlying minute observations") return result, {"source": SPOT_SOURCE, "license": "CC0-1.0", "sha256": archive_digest(path), "data_end": result["timestamp"].max().date().isoformat()} def normalize_options(raw, spots): required = {"date", "high", "low", "close", "volume", "oi", "name", "expiry", "strike", "instrument_type"} if not required.issubset(raw.columns): raise ValueError("Recent option data does not match the verified schema") frame = raw[raw["name"].eq("NIFTY") & raw["instrument_type"].isin(["CE", "PE"])].copy() frame["timestamp"] = pd.to_datetime(frame["date"], utc=True).dt.tz_convert(IST) + pd.Timedelta(minutes=1) frame["expiry"] = pd.to_datetime(frame["expiry"]).dt.strftime("%Y-%m-%d") remaining = (pd.to_datetime(frame["expiry"]) - pd.to_datetime(frame["timestamp"].dt.date)).dt.days frame = frame[remaining.between(0, 7) & (frame["strike"] > 0) & (frame["low"] > 0) & (frame["close"] >= frame["low"]) & (frame["close"] <= frame["high"])].copy() minute = frame["timestamp"].dt.hour * 60 + frame["timestamp"].dt.minute frame = frame[minute.between(556, 930)].copy() frame = frame.rename(columns={"close": "price", "oi": "open_interest", "instrument_type": "option_type"}) frame["contract_id"] = "NSE:OPTION:INDEX:NIFTY:" + frame["expiry"] + ":" + frame["strike"].map(lambda value: format(float(value), "g")) + ":" + frame["option_type"] columns = ["timestamp", "expiry", "contract_id", "price", "high", "low", "strike", "option_type", "volume", "open_interest"] frame = frame[columns].drop_duplicates() conflicting = frame.duplicated(["contract_id", "timestamp"], keep=False) excluded = int(conflicting.sum()) frame = frame[~conflicting].merge(spots, on="timestamp", how="inner", validate="many_to_one") frame.attrs["excluded_conflicting_rows"] = excluded return frame.sort_values(["contract_id", "timestamp"]).reset_index(drop=True) def sample_day(frame): samples = [] for identity, group in frame.groupby("contract_id", sort=False): group = group.sort_values("timestamp") times = np.array([timestamp.value for timestamp in group["timestamp"]], dtype="int64") highs, lows = group["high"].to_numpy(), group["low"].to_numpy() points = [{"time": timestamp.timestamp(), "close": price} for timestamp, price in zip(group["timestamp"], group["price"])] for row in group[group["timestamp"].dt.strftime("%H:%M").isin(ANCHORS | {"15:00", "15:30"})].to_dict("records"): timestamp = row["timestamp"] index = int(np.searchsorted(times, timestamp.value, side="right")) cutoff = timestamp.normalize() + pd.Timedelta(hours=15) end = int(np.searchsorted(times, cutoff.value, side="right")) row["intraday_high"] = float(np.max(highs[index:end])) if end > index else float("nan") row["intraday_low"] = float(np.min(lows[index:end])) if end > index else float("nan") row["remaining_high"] = float(np.max(highs[index:])) if index < len(highs) else float("nan") row["remaining_low"] = float(np.min(lows[index:])) if index < len(lows) else float("nan") row["session_high"], row["session_low"] = float(np.max(highs)), float(np.min(lows)) close = timestamp.normalize() + pd.Timedelta(hours=15, minutes=30) row["intraday_complete"] = end > index and group["timestamp"].iloc[end - 1] == cutoff and end - index >= max(1, (cutoff - timestamp).total_seconds() / 60 * 0.8) row["remaining_complete"] = index < len(group) and group["timestamp"].iloc[-1] == close and len(group) - index >= max(1, (close - timestamp).total_seconds() / 60 * 0.8) row["session_complete"] = group["timestamp"].iloc[0] <= timestamp.normalize() + pd.Timedelta(hours=9, minutes=20) and group["timestamp"].iloc[-1] == close and len(group) >= 300 try: if timestamp.strftime("%H:%M") not in ANCHORS or row["price"] < 5 or row["volume"] <= 0 or row["open_interest"] <= 0: raise ValueError("Not a liquid training anchor") if abs(np.log(row["underlying_price"] / row["strike"])) > 0.05: raise ValueError("Outside the supported moneyness domain") start = int(np.searchsorted(times, timestamp.value - 32 * 60 * 1_000_000_000, side="left")) row.update(premium_history_features(points[start:index], as_of=timestamp)) except ValueError: row.update(dict.fromkeys(HISTORY_COLUMNS, float("nan"))) samples.append(row) return pd.DataFrame(samples) def prepare_recent(start="2025-10-01", end="2026-04-13", directory=RECENT_CACHE): start, end = date.fromisoformat(start).isoformat(), date.fromisoformat(end).isoformat() os.makedirs(directory, exist_ok=True) prepared = os.path.join(directory, "sampled_" + start + "_" + end + ".parquet") metadata_path = prepared + ".json" identity = {"schema_version": 1, "start": start, "end": end, "history_columns": HISTORY_COLUMNS, "anchors": sorted(ANCHORS), "option_source": OPTION_SOURCE, "spot_source": SPOT_SOURCE} try: with open(metadata_path, encoding="utf-8") as source: metadata = json.load(source) if metadata.get("identity") == identity and metadata.get("sha256") == archive_digest(prepared): return pd.read_parquet(prepared) except (OSError, ValueError, TypeError, AttributeError): pass files = [name for name in list_option_files() if start <= os.path.basename(name)[:10] <= end] if not files: raise ValueError("No exact-date option files in the requested research window") spots, spot_metadata = load_spots(directory) spot_dates = spots["timestamp"].dt.strftime("%Y-%m-%d") spots = spots[spot_dates.between(start, end)].copy() def load(name): path = download_file(OPTION_SOURCE, name, os.path.join(directory, os.path.basename(name)), 50_000_000) frame = normalize_options(pd.read_feather(path), spots) excluded = frame.attrs.get("excluded_conflicting_rows", 0) selected = sample_day(frame) return selected, {"file": name, "sha256": archive_digest(path), "excluded_conflicting_rows": excluded} frames, provenance = [], [] with ThreadPoolExecutor(max_workers=4) as workers: for index, (frame, metadata) in enumerate(workers.map(load, files)): if not frame.empty: frames.append(frame) provenance.append(metadata) if index % 10 == 0: print("Recent option files:", index + 1, "/", len(files), flush=True) if not frames: raise ValueError("No usable exact-contract observations matched the underlying history") history = pd.concat(frames, ignore_index=True).drop_duplicates() history = history[~history.duplicated(["contract_id", "timestamp"], keep=False)].copy() history.attrs["source"] = OPTION_SOURCE history.attrs["source_license"] = "MIT" history.attrs["data_quality"] = {"source_files": len(files), "excluded_conflicting_rows": sum(row["excluded_conflicting_rows"] for row in provenance)} history.attrs["source_provenance"] = {"options": provenance, "spot": spot_metadata} write_json_atomic(os.path.join(directory, "source_provenance.json"), history.attrs["source_provenance"]) with tempfile.NamedTemporaryFile(dir=directory, suffix=".parquet", delete=False) as output: temporary = output.name try: history.to_parquet(temporary, index=False) os.replace(temporary, prepared) finally: if os.path.exists(temporary): os.unlink(temporary) write_json_atomic(metadata_path, {"identity": identity, "sha256": archive_digest(prepared)}) return history