PaperTrade / research /fo_recent_data.py
Khanna, Videh Rakesh Rakesh
Fix portfolio valuation and wire independent F&O targets
8b054f7
Raw History Blame Contribute Delete
11.4 kB
"""Verified, explicitly licensed minute history for derivative reach research."""
from concurrent.futures import ThreadPoolExecutor
from datetime import date
import json
import os
import re
import tempfile
from urllib.parse import quote
import numpy as np
import pandas as pd
import requests
from fo import IST
from fo_predictor import HISTORY_COLUMNS, premium_history_features
from research.train_fo_predictor import ANCHORS, CACHE_ROOT, archive_digest, write_json_atomic
OPTION_SOURCE = "samardubey/niftybanknifty-options-data"
OPTION_PREFIX = "filtered_feather_folder/filtered_feather_folder/"
SPOT_SOURCE = "debashis74017/nifty-50-minute-data"
RECENT_CACHE = os.path.join(CACHE_ROOT, "recent_options")
def download_file(source, name, destination, maximum):
if os.path.isfile(destination) and os.path.getsize(destination) > 0:
return destination
directory = os.path.dirname(os.path.abspath(destination))
os.makedirs(directory, exist_ok=True)
temporary = None
try:
url = "https://www.kaggle.com/api/v1/datasets/download/" + source + "/" + quote(name, safe="")
with requests.get(url, stream=True, timeout=(5, 30)) as response:
response.raise_for_status()
if int(response.headers.get("Content-Length", "0")) > maximum:
raise ValueError("Source file exceeds its download budget")
size = 0
with tempfile.NamedTemporaryFile(dir=directory, delete=False) as output:
temporary = output.name
for chunk in response.iter_content(1024 * 1024):
size += len(chunk)
if size > maximum:
raise ValueError("Source file exceeded its download budget")
output.write(chunk)
output.flush()
os.fsync(output.fileno())
os.replace(temporary, destination)
temporary = None
return destination
finally:
if temporary and os.path.exists(temporary):
os.unlink(temporary)
def list_option_files():
files = []
parameters = {"pageSize": 200}
for page in range(20):
response = requests.get("https://www.kaggle.com/api/v1/datasets/list/" + OPTION_SOURCE,
params=parameters, timeout=(5, 20))
response.raise_for_status()
body = response.json()
for row in body.get("datasetFiles", []):
name = row.get("name", "")
if re.fullmatch(re.escape(OPTION_PREFIX) + r"\d{4}-\d{2}-\d{2}-index-nfo-data\.feather", name):
files.append(name)
token = body.get("nextPageToken")
if not token:
return sorted(set(files))
parameters["pageToken"] = token
raise ValueError("Option catalog exceeded its page budget")
def load_spots(directory):
path = download_file(SPOT_SOURCE, "NIFTY 50_minute.csv", os.path.join(directory, "nifty_spot_minute.csv"), 80_000_000)
raw = pd.read_csv(path, usecols=["date", "close"])
timestamps = pd.to_datetime(raw["date"])
timestamps = timestamps.dt.tz_localize(IST) if timestamps.dt.tz is None else timestamps.dt.tz_convert(IST)
result = pd.DataFrame({"timestamp": timestamps + pd.Timedelta(minutes=1),
"underlying_price": pd.to_numeric(raw["close"], errors="raise")}).drop_duplicates()
if result.duplicated("timestamp").any():
raise ValueError("Conflicting underlying minute observations")
return result, {"source": SPOT_SOURCE, "license": "CC0-1.0", "sha256": archive_digest(path),
"data_end": result["timestamp"].max().date().isoformat()}
def normalize_options(raw, spots):
required = {"date", "high", "low", "close", "volume", "oi", "name", "expiry", "strike", "instrument_type"}
if not required.issubset(raw.columns):
raise ValueError("Recent option data does not match the verified schema")
frame = raw[raw["name"].eq("NIFTY") & raw["instrument_type"].isin(["CE", "PE"])].copy()
frame["timestamp"] = pd.to_datetime(frame["date"], utc=True).dt.tz_convert(IST) + pd.Timedelta(minutes=1)
frame["expiry"] = pd.to_datetime(frame["expiry"]).dt.strftime("%Y-%m-%d")
remaining = (pd.to_datetime(frame["expiry"]) - pd.to_datetime(frame["timestamp"].dt.date)).dt.days
frame = frame[remaining.between(0, 7) & (frame["strike"] > 0) & (frame["low"] > 0)
& (frame["close"] >= frame["low"]) & (frame["close"] <= frame["high"])].copy()
minute = frame["timestamp"].dt.hour * 60 + frame["timestamp"].dt.minute
frame = frame[minute.between(556, 930)].copy()
frame = frame.rename(columns={"close": "price", "oi": "open_interest", "instrument_type": "option_type"})
frame["contract_id"] = "NSE:OPTION:INDEX:NIFTY:" + frame["expiry"] + ":" + frame["strike"].map(lambda value: format(float(value), "g")) + ":" + frame["option_type"]
columns = ["timestamp", "expiry", "contract_id", "price", "high", "low", "strike", "option_type", "volume", "open_interest"]
frame = frame[columns].drop_duplicates()
conflicting = frame.duplicated(["contract_id", "timestamp"], keep=False)
excluded = int(conflicting.sum())
frame = frame[~conflicting].merge(spots, on="timestamp", how="inner", validate="many_to_one")
frame.attrs["excluded_conflicting_rows"] = excluded
return frame.sort_values(["contract_id", "timestamp"]).reset_index(drop=True)
def sample_day(frame):
samples = []
for identity, group in frame.groupby("contract_id", sort=False):
group = group.sort_values("timestamp")
times = np.array([timestamp.value for timestamp in group["timestamp"]], dtype="int64")
highs, lows = group["high"].to_numpy(), group["low"].to_numpy()
points = [{"time": timestamp.timestamp(), "close": price} for timestamp, price in zip(group["timestamp"], group["price"])]
for row in group[group["timestamp"].dt.strftime("%H:%M").isin(ANCHORS | {"15:00", "15:30"})].to_dict("records"):
timestamp = row["timestamp"]
index = int(np.searchsorted(times, timestamp.value, side="right"))
cutoff = timestamp.normalize() + pd.Timedelta(hours=15)
end = int(np.searchsorted(times, cutoff.value, side="right"))
row["intraday_high"] = float(np.max(highs[index:end])) if end > index else float("nan")
row["intraday_low"] = float(np.min(lows[index:end])) if end > index else float("nan")
row["remaining_high"] = float(np.max(highs[index:])) if index < len(highs) else float("nan")
row["remaining_low"] = float(np.min(lows[index:])) if index < len(lows) else float("nan")
row["session_high"], row["session_low"] = float(np.max(highs)), float(np.min(lows))
close = timestamp.normalize() + pd.Timedelta(hours=15, minutes=30)
row["intraday_complete"] = end > index and group["timestamp"].iloc[end - 1] == cutoff and end - index >= max(1, (cutoff - timestamp).total_seconds() / 60 * 0.8)
row["remaining_complete"] = index < len(group) and group["timestamp"].iloc[-1] == close and len(group) - index >= max(1, (close - timestamp).total_seconds() / 60 * 0.8)
row["session_complete"] = group["timestamp"].iloc[0] <= timestamp.normalize() + pd.Timedelta(hours=9, minutes=20) and group["timestamp"].iloc[-1] == close and len(group) >= 300
try:
if timestamp.strftime("%H:%M") not in ANCHORS or row["price"] < 5 or row["volume"] <= 0 or row["open_interest"] <= 0:
raise ValueError("Not a liquid training anchor")
if abs(np.log(row["underlying_price"] / row["strike"])) > 0.05:
raise ValueError("Outside the supported moneyness domain")
start = int(np.searchsorted(times, timestamp.value - 32 * 60 * 1_000_000_000, side="left"))
row.update(premium_history_features(points[start:index], as_of=timestamp))
except ValueError:
row.update(dict.fromkeys(HISTORY_COLUMNS, float("nan")))
samples.append(row)
return pd.DataFrame(samples)
def prepare_recent(start="2025-10-01", end="2026-04-13", directory=RECENT_CACHE):
start, end = date.fromisoformat(start).isoformat(), date.fromisoformat(end).isoformat()
os.makedirs(directory, exist_ok=True)
prepared = os.path.join(directory, "sampled_" + start + "_" + end + ".parquet")
metadata_path = prepared + ".json"
identity = {"schema_version": 1, "start": start, "end": end,
"history_columns": HISTORY_COLUMNS, "anchors": sorted(ANCHORS),
"option_source": OPTION_SOURCE, "spot_source": SPOT_SOURCE}
try:
with open(metadata_path, encoding="utf-8") as source:
metadata = json.load(source)
if metadata.get("identity") == identity and metadata.get("sha256") == archive_digest(prepared):
return pd.read_parquet(prepared)
except (OSError, ValueError, TypeError, AttributeError):
pass
files = [name for name in list_option_files() if start <= os.path.basename(name)[:10] <= end]
if not files:
raise ValueError("No exact-date option files in the requested research window")
spots, spot_metadata = load_spots(directory)
spot_dates = spots["timestamp"].dt.strftime("%Y-%m-%d")
spots = spots[spot_dates.between(start, end)].copy()
def load(name):
path = download_file(OPTION_SOURCE, name, os.path.join(directory, os.path.basename(name)), 50_000_000)
frame = normalize_options(pd.read_feather(path), spots)
excluded = frame.attrs.get("excluded_conflicting_rows", 0)
selected = sample_day(frame)
return selected, {"file": name, "sha256": archive_digest(path), "excluded_conflicting_rows": excluded}
frames, provenance = [], []
with ThreadPoolExecutor(max_workers=4) as workers:
for index, (frame, metadata) in enumerate(workers.map(load, files)):
if not frame.empty:
frames.append(frame)
provenance.append(metadata)
if index % 10 == 0:
print("Recent option files:", index + 1, "/", len(files), flush=True)
if not frames:
raise ValueError("No usable exact-contract observations matched the underlying history")
history = pd.concat(frames, ignore_index=True).drop_duplicates()
history = history[~history.duplicated(["contract_id", "timestamp"], keep=False)].copy()
history.attrs["source"] = OPTION_SOURCE
history.attrs["source_license"] = "MIT"
history.attrs["data_quality"] = {"source_files": len(files),
"excluded_conflicting_rows": sum(row["excluded_conflicting_rows"] for row in provenance)}
history.attrs["source_provenance"] = {"options": provenance, "spot": spot_metadata}
write_json_atomic(os.path.join(directory, "source_provenance.json"), history.attrs["source_provenance"])
with tempfile.NamedTemporaryFile(dir=directory, suffix=".parquet", delete=False) as output:
temporary = output.name
try:
history.to_parquet(temporary, index=False)
os.replace(temporary, prepared)
finally:
if os.path.exists(temporary):
os.unlink(temporary)
write_json_atomic(metadata_path, {"identity": identity, "sha256": archive_digest(prepared)})
return history