Spaces:
Running on Zero
Running on Zero
deveshc23
feat: Add Free Gradio Space support, Git LFS tracking for data artifacts, and Quant Allocator
3e33e2e | """Extract historical bars for key liquid symbols into SQLite and JSON cache.""" | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sqlite3 | |
| import time | |
| from pathlib import Path | |
| from drift_quant.nse_data.loader import ( | |
| load_selected_symbols, | |
| save_selected_cache, | |
| ) | |
| CORE_UNIVERSE = ( | |
| # Banking | |
| "HDFCBANK", | |
| "ICICIBANK", | |
| "SBIN", | |
| "AXISBANK", | |
| "KOTAKBANK", | |
| "INDUSINDBK", | |
| "BANKBARODA", | |
| # Financial Services | |
| "BAJFINANCE", | |
| "BAJAJFINSV", | |
| "HDFCLIFE", | |
| "SBILIFE", | |
| # Information Technology | |
| "TCS", | |
| "INFY", | |
| "WIPRO", | |
| "HCLTECH", | |
| "TECHM", | |
| "LTIM", | |
| # Energy, Oil & Gas | |
| "RELIANCE", | |
| "ONGC", | |
| "BPCL", | |
| "POWERGRID", | |
| "NTPC", | |
| "TATAPOWER", | |
| "COALINDIA", | |
| # FMCG & Consumer | |
| "ITC", | |
| "HINDUNILVR", | |
| "NESTLEIND", | |
| "BRITANNIA", | |
| "DABUR", | |
| "TATACONSUM", | |
| # Automobile | |
| "TATAMOTORS", | |
| "MARUTI", | |
| "M&M", | |
| "BAJAJ-AUTO", | |
| "EICHERMOT", | |
| "HEROMOTOCO", | |
| # Metals & Mining | |
| "TATASTEEL", | |
| "JSWSTEEL", | |
| "HINDALCO", | |
| "VEDL", | |
| "JINDALSTEL", | |
| # Pharma & Healthcare | |
| "SUNPHARMA", | |
| "DRREDDY", | |
| "CIPLA", | |
| "DIVISLAB", | |
| "APOLLOHOSP", | |
| # Infrastructure, Industrials & Conglomerates | |
| "LT", | |
| "BHARTIARTL", | |
| "ULTRACEMCO", | |
| "GRASIM", | |
| "ADANIENT", | |
| "ADANIPORTS", | |
| "TITAN", | |
| "ASIANPAINT", | |
| ) | |
| def populate_sqlite(dataset, db_path: Path) -> None: | |
| db_path.parent.mkdir(parents=True, exist_ok=True) | |
| if db_path.is_file(): | |
| db_path.unlink() | |
| conn = sqlite3.connect(db_path) | |
| cur = conn.cursor() | |
| cur.execute( | |
| """ | |
| CREATE TABLE bars ( | |
| symbol TEXT NOT NULL, | |
| timestamp TEXT NOT NULL, | |
| open REAL NOT NULL, | |
| high REAL NOT NULL, | |
| low REAL NOT NULL, | |
| close REAL NOT NULL, | |
| volume REAL NOT NULL, | |
| source_row INTEGER NOT NULL | |
| ) | |
| """ | |
| ) | |
| records = [] | |
| for _symbol, bars in dataset.bars_by_symbol.items(): | |
| for bar in bars: | |
| records.append( | |
| ( | |
| bar.symbol, | |
| bar.timestamp.isoformat(sep=" "), | |
| bar.open, | |
| bar.high, | |
| bar.low, | |
| bar.close, | |
| bar.volume, | |
| bar.source_row, | |
| ) | |
| ) | |
| cur.executemany( | |
| """ | |
| INSERT INTO bars (symbol, timestamp, open, high, low, close, volume, source_row) | |
| VALUES (?, ?, ?, ?, ?, ?, ?, ?) | |
| """, | |
| records, | |
| ) | |
| cur.execute("CREATE INDEX idx_bars_symbol_ts ON bars (symbol, timestamp)") | |
| conn.commit() | |
| conn.close() | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument( | |
| "archive", | |
| type=Path, | |
| nargs="?", | |
| default=Path("all_NSE500_15min_data_2019_onwards.zip"), | |
| ) | |
| parser.add_argument("--audit", type=Path, default=Path("artifacts/dataset_audit.json")) | |
| parser.add_argument( | |
| "--json-cache", type=Path, default=Path("artifacts/cache/nse_universe_20.json") | |
| ) | |
| parser.add_argument( | |
| "--sqlite-cache", type=Path, default=Path("artifacts/cache/market_data.sqlite") | |
| ) | |
| args = parser.parse_args() | |
| audit = json.loads(args.audit.read_text(encoding="utf-8")) | |
| sha = audit["archive_sha256"] | |
| print(f"Scanning {args.archive} for {len(CORE_UNIVERSE)} symbols across 2019-2025...") | |
| t0 = time.time() | |
| dataset = load_selected_symbols( | |
| args.archive, | |
| sha, | |
| CORE_UNIVERSE, | |
| progress=print, | |
| ) | |
| elapsed = time.time() - t0 | |
| print( | |
| f"Extracted {dataset.rows_selected:,} bars across {len(dataset.symbols)} symbols " | |
| f"in {elapsed:.1f}s." | |
| ) | |
| print(f"Writing JSON cache to {args.json_cache}...") | |
| save_selected_cache(dataset, args.json_cache) | |
| print(f"Writing SQLite database to {args.sqlite_cache}...") | |
| populate_sqlite(dataset, args.sqlite_cache) | |
| print("Market data caching complete.") | |
| if __name__ == "__main__": | |
| main() | |