Spaces:
Sleeping
Sleeping
Download research/fo_recent_data.py from V1deh/PaperTrade: direct link, hf CLI and curl.
- Browser
- Download file 11.4 kB
-
https://huggingface.co/spaces/V1deh/PaperTrade/resolve/main/research/fo_recent_data.py
- Command line
-
hf download hf://spaces/V1deh/PaperTrade/research/fo_recent_data.py
-
curl -L -o fo_recent_data.py https://huggingface.co/spaces/V1deh/PaperTrade/resolve/main/research/fo_recent_data.py
11.4 kB
| """Verified, explicitly licensed minute history for derivative reach research.""" | |
| from concurrent.futures import ThreadPoolExecutor | |
| from datetime import date | |
| import json | |
| import os | |
| import re | |
| import tempfile | |
| from urllib.parse import quote | |
| import numpy as np | |
| import pandas as pd | |
| import requests | |
| from fo import IST | |
| from fo_predictor import HISTORY_COLUMNS, premium_history_features | |
| from research.train_fo_predictor import ANCHORS, CACHE_ROOT, archive_digest, write_json_atomic | |
| OPTION_SOURCE = "samardubey/niftybanknifty-options-data" | |
| OPTION_PREFIX = "filtered_feather_folder/filtered_feather_folder/" | |
| SPOT_SOURCE = "debashis74017/nifty-50-minute-data" | |
| RECENT_CACHE = os.path.join(CACHE_ROOT, "recent_options") | |
| def download_file(source, name, destination, maximum): | |
| if os.path.isfile(destination) and os.path.getsize(destination) > 0: | |
| return destination | |
| directory = os.path.dirname(os.path.abspath(destination)) | |
| os.makedirs(directory, exist_ok=True) | |
| temporary = None | |
| try: | |
| url = "https://www.kaggle.com/api/v1/datasets/download/" + source + "/" + quote(name, safe="") | |
| with requests.get(url, stream=True, timeout=(5, 30)) as response: | |
| response.raise_for_status() | |
| if int(response.headers.get("Content-Length", "0")) > maximum: | |
| raise ValueError("Source file exceeds its download budget") | |
| size = 0 | |
| with tempfile.NamedTemporaryFile(dir=directory, delete=False) as output: | |
| temporary = output.name | |
| for chunk in response.iter_content(1024 * 1024): | |
| size += len(chunk) | |
| if size > maximum: | |
| raise ValueError("Source file exceeded its download budget") | |
| output.write(chunk) | |
| output.flush() | |
| os.fsync(output.fileno()) | |
| os.replace(temporary, destination) | |
| temporary = None | |
| return destination | |
| finally: | |
| if temporary and os.path.exists(temporary): | |
| os.unlink(temporary) | |
| def list_option_files(): | |
| files = [] | |
| parameters = {"pageSize": 200} | |
| for page in range(20): | |
| response = requests.get("https://www.kaggle.com/api/v1/datasets/list/" + OPTION_SOURCE, | |
| params=parameters, timeout=(5, 20)) | |
| response.raise_for_status() | |
| body = response.json() | |
| for row in body.get("datasetFiles", []): | |
| name = row.get("name", "") | |
| if re.fullmatch(re.escape(OPTION_PREFIX) + r"\d{4}-\d{2}-\d{2}-index-nfo-data\.feather", name): | |
| files.append(name) | |
| token = body.get("nextPageToken") | |
| if not token: | |
| return sorted(set(files)) | |
| parameters["pageToken"] = token | |
| raise ValueError("Option catalog exceeded its page budget") | |
| def load_spots(directory): | |
| path = download_file(SPOT_SOURCE, "NIFTY 50_minute.csv", os.path.join(directory, "nifty_spot_minute.csv"), 80_000_000) | |
| raw = pd.read_csv(path, usecols=["date", "close"]) | |
| timestamps = pd.to_datetime(raw["date"]) | |
| timestamps = timestamps.dt.tz_localize(IST) if timestamps.dt.tz is None else timestamps.dt.tz_convert(IST) | |
| result = pd.DataFrame({"timestamp": timestamps + pd.Timedelta(minutes=1), | |
| "underlying_price": pd.to_numeric(raw["close"], errors="raise")}).drop_duplicates() | |
| if result.duplicated("timestamp").any(): | |
| raise ValueError("Conflicting underlying minute observations") | |
| return result, {"source": SPOT_SOURCE, "license": "CC0-1.0", "sha256": archive_digest(path), | |
| "data_end": result["timestamp"].max().date().isoformat()} | |
| def normalize_options(raw, spots): | |
| required = {"date", "high", "low", "close", "volume", "oi", "name", "expiry", "strike", "instrument_type"} | |
| if not required.issubset(raw.columns): | |
| raise ValueError("Recent option data does not match the verified schema") | |
| frame = raw[raw["name"].eq("NIFTY") & raw["instrument_type"].isin(["CE", "PE"])].copy() | |
| frame["timestamp"] = pd.to_datetime(frame["date"], utc=True).dt.tz_convert(IST) + pd.Timedelta(minutes=1) | |
| frame["expiry"] = pd.to_datetime(frame["expiry"]).dt.strftime("%Y-%m-%d") | |
| remaining = (pd.to_datetime(frame["expiry"]) - pd.to_datetime(frame["timestamp"].dt.date)).dt.days | |
| frame = frame[remaining.between(0, 7) & (frame["strike"] > 0) & (frame["low"] > 0) | |
| & (frame["close"] >= frame["low"]) & (frame["close"] <= frame["high"])].copy() | |
| minute = frame["timestamp"].dt.hour * 60 + frame["timestamp"].dt.minute | |
| frame = frame[minute.between(556, 930)].copy() | |
| frame = frame.rename(columns={"close": "price", "oi": "open_interest", "instrument_type": "option_type"}) | |
| frame["contract_id"] = "NSE:OPTION:INDEX:NIFTY:" + frame["expiry"] + ":" + frame["strike"].map(lambda value: format(float(value), "g")) + ":" + frame["option_type"] | |
| columns = ["timestamp", "expiry", "contract_id", "price", "high", "low", "strike", "option_type", "volume", "open_interest"] | |
| frame = frame[columns].drop_duplicates() | |
| conflicting = frame.duplicated(["contract_id", "timestamp"], keep=False) | |
| excluded = int(conflicting.sum()) | |
| frame = frame[~conflicting].merge(spots, on="timestamp", how="inner", validate="many_to_one") | |
| frame.attrs["excluded_conflicting_rows"] = excluded | |
| return frame.sort_values(["contract_id", "timestamp"]).reset_index(drop=True) | |
| def sample_day(frame): | |
| samples = [] | |
| for identity, group in frame.groupby("contract_id", sort=False): | |
| group = group.sort_values("timestamp") | |
| times = np.array([timestamp.value for timestamp in group["timestamp"]], dtype="int64") | |
| highs, lows = group["high"].to_numpy(), group["low"].to_numpy() | |
| points = [{"time": timestamp.timestamp(), "close": price} for timestamp, price in zip(group["timestamp"], group["price"])] | |
| for row in group[group["timestamp"].dt.strftime("%H:%M").isin(ANCHORS | {"15:00", "15:30"})].to_dict("records"): | |
| timestamp = row["timestamp"] | |
| index = int(np.searchsorted(times, timestamp.value, side="right")) | |
| cutoff = timestamp.normalize() + pd.Timedelta(hours=15) | |
| end = int(np.searchsorted(times, cutoff.value, side="right")) | |
| row["intraday_high"] = float(np.max(highs[index:end])) if end > index else float("nan") | |
| row["intraday_low"] = float(np.min(lows[index:end])) if end > index else float("nan") | |
| row["remaining_high"] = float(np.max(highs[index:])) if index < len(highs) else float("nan") | |
| row["remaining_low"] = float(np.min(lows[index:])) if index < len(lows) else float("nan") | |
| row["session_high"], row["session_low"] = float(np.max(highs)), float(np.min(lows)) | |
| close = timestamp.normalize() + pd.Timedelta(hours=15, minutes=30) | |
| row["intraday_complete"] = end > index and group["timestamp"].iloc[end - 1] == cutoff and end - index >= max(1, (cutoff - timestamp).total_seconds() / 60 * 0.8) | |
| row["remaining_complete"] = index < len(group) and group["timestamp"].iloc[-1] == close and len(group) - index >= max(1, (close - timestamp).total_seconds() / 60 * 0.8) | |
| row["session_complete"] = group["timestamp"].iloc[0] <= timestamp.normalize() + pd.Timedelta(hours=9, minutes=20) and group["timestamp"].iloc[-1] == close and len(group) >= 300 | |
| try: | |
| if timestamp.strftime("%H:%M") not in ANCHORS or row["price"] < 5 or row["volume"] <= 0 or row["open_interest"] <= 0: | |
| raise ValueError("Not a liquid training anchor") | |
| if abs(np.log(row["underlying_price"] / row["strike"])) > 0.05: | |
| raise ValueError("Outside the supported moneyness domain") | |
| start = int(np.searchsorted(times, timestamp.value - 32 * 60 * 1_000_000_000, side="left")) | |
| row.update(premium_history_features(points[start:index], as_of=timestamp)) | |
| except ValueError: | |
| row.update(dict.fromkeys(HISTORY_COLUMNS, float("nan"))) | |
| samples.append(row) | |
| return pd.DataFrame(samples) | |
| def prepare_recent(start="2025-10-01", end="2026-04-13", directory=RECENT_CACHE): | |
| start, end = date.fromisoformat(start).isoformat(), date.fromisoformat(end).isoformat() | |
| os.makedirs(directory, exist_ok=True) | |
| prepared = os.path.join(directory, "sampled_" + start + "_" + end + ".parquet") | |
| metadata_path = prepared + ".json" | |
| identity = {"schema_version": 1, "start": start, "end": end, | |
| "history_columns": HISTORY_COLUMNS, "anchors": sorted(ANCHORS), | |
| "option_source": OPTION_SOURCE, "spot_source": SPOT_SOURCE} | |
| try: | |
| with open(metadata_path, encoding="utf-8") as source: | |
| metadata = json.load(source) | |
| if metadata.get("identity") == identity and metadata.get("sha256") == archive_digest(prepared): | |
| return pd.read_parquet(prepared) | |
| except (OSError, ValueError, TypeError, AttributeError): | |
| pass | |
| files = [name for name in list_option_files() if start <= os.path.basename(name)[:10] <= end] | |
| if not files: | |
| raise ValueError("No exact-date option files in the requested research window") | |
| spots, spot_metadata = load_spots(directory) | |
| spot_dates = spots["timestamp"].dt.strftime("%Y-%m-%d") | |
| spots = spots[spot_dates.between(start, end)].copy() | |
| def load(name): | |
| path = download_file(OPTION_SOURCE, name, os.path.join(directory, os.path.basename(name)), 50_000_000) | |
| frame = normalize_options(pd.read_feather(path), spots) | |
| excluded = frame.attrs.get("excluded_conflicting_rows", 0) | |
| selected = sample_day(frame) | |
| return selected, {"file": name, "sha256": archive_digest(path), "excluded_conflicting_rows": excluded} | |
| frames, provenance = [], [] | |
| with ThreadPoolExecutor(max_workers=4) as workers: | |
| for index, (frame, metadata) in enumerate(workers.map(load, files)): | |
| if not frame.empty: | |
| frames.append(frame) | |
| provenance.append(metadata) | |
| if index % 10 == 0: | |
| print("Recent option files:", index + 1, "/", len(files), flush=True) | |
| if not frames: | |
| raise ValueError("No usable exact-contract observations matched the underlying history") | |
| history = pd.concat(frames, ignore_index=True).drop_duplicates() | |
| history = history[~history.duplicated(["contract_id", "timestamp"], keep=False)].copy() | |
| history.attrs["source"] = OPTION_SOURCE | |
| history.attrs["source_license"] = "MIT" | |
| history.attrs["data_quality"] = {"source_files": len(files), | |
| "excluded_conflicting_rows": sum(row["excluded_conflicting_rows"] for row in provenance)} | |
| history.attrs["source_provenance"] = {"options": provenance, "spot": spot_metadata} | |
| write_json_atomic(os.path.join(directory, "source_provenance.json"), history.attrs["source_provenance"]) | |
| with tempfile.NamedTemporaryFile(dir=directory, suffix=".parquet", delete=False) as output: | |
| temporary = output.name | |
| try: | |
| history.to_parquet(temporary, index=False) | |
| os.replace(temporary, prepared) | |
| finally: | |
| if os.path.exists(temporary): | |
| os.unlink(temporary) | |
| write_json_atomic(metadata_path, {"identity": identity, "sha256": archive_digest(prepared)}) | |
| return history |