Spaces:
Sleeping
Sleeping
Download update_data.py from Serena2026/qlib-quant-platform: direct link, hf CLI and curl.
- Browser
- Download file 8.74 kB
-
https://huggingface.co/spaces/Serena2026/qlib-quant-platform/resolve/main/update_data.py
- Command line
-
hf download hf://spaces/Serena2026/qlib-quant-platform/update_data.py
-
curl -L -o update_data.py https://huggingface.co/spaces/Serena2026/qlib-quant-platform/resolve/main/update_data.py
8.74 kB
| """ | |
| Qlib A股数据自动更新脚本 | |
| 基于 Yahoo Finance 获取最新数据并更新 Qlib 数据库 | |
| """ | |
| import os | |
| import sys | |
| import logging | |
| from datetime import datetime, timedelta | |
| from pathlib import Path | |
| logging.basicConfig( | |
| level=logging.INFO, | |
| format="%(asctime)s [%(levelname)s] %(message)s", | |
| handlers=[logging.StreamHandler()] | |
| ) | |
| logger = logging.getLogger(__name__) | |
| # 自动检测数据目录(兼容 Docker 容器和本地环境) | |
| _CANDIDATES = [ | |
| "/root/.qlib/qlib_data/cn_data", | |
| os.path.expanduser("~/.qlib/qlib_data/cn_data"), | |
| ] | |
| QLIB_DATA_DIR = next((p for p in _CANDIDATES if os.path.exists(p)), _CANDIDATES[0]) | |
| def get_last_date_from_calendar(): | |
| """从日历文件直接读取最新日期""" | |
| calendar_file = Path(QLIB_DATA_DIR) / "calendars" / "day.txt" | |
| if calendar_file.exists(): | |
| lines = [l.strip() for l in calendar_file.read_text().splitlines() if l.strip()] | |
| if lines: | |
| last = lines[-1] | |
| logger.info(f"日历文件最新日期: {last}") | |
| return last | |
| return "2020-09-25" | |
| def update_calendar_file(new_dates): | |
| """向日历文件追加新的交易日""" | |
| calendar_file = Path(QLIB_DATA_DIR) / "calendars" / "day.txt" | |
| if not calendar_file.exists(): | |
| logger.warning("日历文件不存在,跳过更新") | |
| return | |
| existing = set(l.strip() for l in calendar_file.read_text().splitlines() if l.strip()) | |
| to_add = [d for d in new_dates if d not in existing] | |
| if to_add: | |
| with open(calendar_file, 'a') as f: | |
| for d in sorted(to_add): | |
| f.write(d + '\n') | |
| logger.info(f"日历文件新增 {len(to_add)} 个交易日(最新: {sorted(to_add)[-1]})") | |
| else: | |
| logger.info("日历文件无需更新") | |
| def update_instruments_end_date(end_date): | |
| """更新 instruments 文件中每只股票的结束日期""" | |
| instruments_dir = Path(QLIB_DATA_DIR) / "instruments" | |
| for fname in ["all.txt", "csi300.txt", "csi500.txt"]: | |
| fpath = instruments_dir / fname | |
| if not fpath.exists(): | |
| continue | |
| lines = fpath.read_text().splitlines() | |
| new_lines = [] | |
| for line in lines: | |
| parts = line.split("\t") | |
| if len(parts) >= 3: | |
| parts[2] = end_date | |
| new_lines.append("\t".join(parts)) | |
| else: | |
| new_lines.append(line) | |
| fpath.write_text("\n".join(new_lines) + "\n") | |
| logger.info(f"instruments 文件结束日期已更新为 {end_date}") | |
| def get_stock_list(): | |
| """读取全部股票代码""" | |
| all_file = Path(QLIB_DATA_DIR) / "instruments" / "all.txt" | |
| stocks = [] | |
| if all_file.exists(): | |
| for line in all_file.read_text().splitlines(): | |
| parts = line.strip().split("\t") | |
| if parts and parts[0]: | |
| stocks.append(parts[0]) | |
| if not stocks: | |
| stocks = [ | |
| "SH600519", "SH600036", "SH601318", "SH600276", "SH600000", | |
| "SH600887", "SH600030", "SH601012", "SH600031", "SH600900", | |
| "SZ000001", "SZ000858", "SZ000333", "SZ300750", "SZ002415", | |
| "SZ000002", "SZ000651", "SZ002594", "SZ300059", "SZ000568", | |
| ] | |
| return stocks | |
| def to_yahoo_code(qlib_code): | |
| if qlib_code.startswith("SH"): | |
| return qlib_code[2:] + ".SS" | |
| elif qlib_code.startswith("SZ"): | |
| return qlib_code[2:] + ".SZ" | |
| return qlib_code | |
| def download_and_write(stocks, start_date, end_date): | |
| """下载 Yahoo Finance 数据并写入 Qlib 二进制格式""" | |
| try: | |
| import yfinance as yf | |
| import pandas as pd | |
| import numpy as np | |
| except ImportError: | |
| os.system("pip install yfinance pandas numpy -q") | |
| import yfinance as yf | |
| import pandas as pd | |
| import numpy as np | |
| features_dir = Path(QLIB_DATA_DIR) / "features" | |
| features_dir.mkdir(parents=True, exist_ok=True) | |
| all_trading_dates = set() | |
| success_count = 0 | |
| batch_size = 30 | |
| # 只更新已有数据的股票 | |
| existing_stocks = [s for s in stocks if (features_dir / s.lower()).exists()] | |
| if not existing_stocks: | |
| existing_stocks = stocks[:100] | |
| target_stocks = existing_stocks[:300] | |
| logger.info(f"准备更新 {len(target_stocks)} 只股票的数据...") | |
| for i in range(0, len(target_stocks), batch_size): | |
| batch = target_stocks[i:i + batch_size] | |
| yahoo_batch = [to_yahoo_code(c) for c in batch] | |
| try: | |
| if len(yahoo_batch) == 1: | |
| raw = yf.download(yahoo_batch[0], start=start_date, end=end_date, | |
| auto_adjust=True, progress=False) | |
| if not raw.empty: | |
| raw.columns = [c.lower() for c in raw.columns] | |
| data_map = {batch[0]: raw} | |
| else: | |
| data_map = {} | |
| else: | |
| raw = yf.download(yahoo_batch, start=start_date, end=end_date, | |
| group_by='ticker', auto_adjust=True, | |
| progress=False, threads=True) | |
| data_map = {} | |
| if not raw.empty: | |
| for qlib_code, ycode in zip(batch, yahoo_batch): | |
| try: | |
| if ycode in raw.columns.get_level_values(0): | |
| df = raw[ycode].copy() | |
| df.columns = [c.lower() for c in df.columns] | |
| df = df.dropna(how='all') | |
| if not df.empty: | |
| data_map[qlib_code] = df | |
| except Exception: | |
| pass | |
| for qlib_code, df in data_map.items(): | |
| if df.empty: | |
| continue | |
| df.index = pd.to_datetime(df.index).tz_localize(None) | |
| trading_dates = [d.strftime('%Y-%m-%d') for d in df.index] | |
| all_trading_dates.update(trading_dates) | |
| stock_dir = features_dir / qlib_code.lower() | |
| stock_dir.mkdir(parents=True, exist_ok=True) | |
| field_map = {'close': 'close', 'open': 'open', | |
| 'high': 'high', 'low': 'low', 'volume': 'volume'} | |
| for field, col in field_map.items(): | |
| if col not in df.columns: | |
| continue | |
| series = df[col].dropna().astype(np.float32) | |
| if series.empty: | |
| continue | |
| bin_file = stock_dir / f"{field}.day.bin" | |
| if bin_file.exists(): | |
| existing_data = np.fromfile(str(bin_file), dtype=np.float32) | |
| combined = np.concatenate([existing_data, series.values]) | |
| combined.tofile(str(bin_file)) | |
| else: | |
| series.values.tofile(str(bin_file)) | |
| success_count += 1 | |
| except Exception as e: | |
| logger.debug(f"批次 {i//batch_size+1} 下载失败: {e}") | |
| continue | |
| if (i // batch_size + 1) % 5 == 0: | |
| logger.info(f"进度: {min(i+batch_size, len(target_stocks))}/{len(target_stocks)} 只") | |
| logger.info(f"数据写入完成: 成功 {success_count}/{len(target_stocks)} 只") | |
| return success_count, sorted(all_trading_dates) | |
| def main(): | |
| import pandas as pd | |
| today = datetime.today().strftime("%Y-%m-%d") | |
| logger.info("=" * 50) | |
| logger.info("Qlib 数据增量更新开始") | |
| logger.info("=" * 50) | |
| if not Path(QLIB_DATA_DIR).exists(): | |
| logger.error(f"Qlib 数据目录不存在: {QLIB_DATA_DIR}") | |
| return | |
| last_date = get_last_date_from_calendar() | |
| start_date = (datetime.strptime(last_date, "%Y-%m-%d") + timedelta(days=1)).strftime("%Y-%m-%d") | |
| if start_date >= today: | |
| logger.info("✅ 数据已是最新,无需更新") | |
| return | |
| logger.info(f"增量更新: {start_date} → {today}") | |
| stocks = get_stock_list() | |
| logger.info(f"股票总数: {len(stocks)}") | |
| count, trading_dates = download_and_write(stocks, start_date, today) | |
| if count > 0: | |
| if trading_dates: | |
| update_calendar_file(trading_dates) | |
| else: | |
| bdays = pd.bdate_range(start=start_date, end=today) | |
| update_calendar_file([d.strftime('%Y-%m-%d') for d in bdays]) | |
| update_instruments_end_date(today) | |
| logger.info(f"🎉 数据更新完成!已更新 {count} 只股票,数据覆盖至 {today}") | |
| else: | |
| logger.warning("⚠️ 未能更新任何股票数据,将使用现有数据继续运行") | |
| if __name__ == "__main__": | |
| main() | |