File size: 8,738 Bytes
d50c18a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
566df98
 
 
 
 
 
d50c18a
 
cf92bc7
566df98
cf92bc7
 
 
 
 
 
 
d50c18a
 
 
cf92bc7
 
 
 
 
 
d50c18a
cf92bc7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d50c18a
cf92bc7
d50c18a
 
cf92bc7
 
 
 
 
 
d50c18a
cf92bc7
 
 
d50c18a
 
 
 
 
cf92bc7
 
 
 
d50c18a
cf92bc7
 
d50c18a
cf92bc7
d50c18a
cf92bc7
d50c18a
566df98
cf92bc7
 
 
 
d50c18a
cf92bc7
d50c18a
cf92bc7
 
 
d50c18a
cf92bc7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d50c18a
cf92bc7
 
 
 
 
 
 
 
 
 
d50c18a
cf92bc7
d50c18a
cf92bc7
 
 
d50c18a
cf92bc7
 
d50c18a
cf92bc7
 
d50c18a
 
cf92bc7
 
d50c18a
cf92bc7
d50c18a
cf92bc7
 
 
d50c18a
cf92bc7
 
 
d50c18a
cf92bc7
 
d50c18a
cf92bc7
 
 
d50c18a
cf92bc7
d50c18a
cf92bc7
 
d50c18a
cf92bc7
d50c18a
cf92bc7
 
 
 
 
 
d50c18a
cf92bc7
 
d50c18a
cf92bc7
d50c18a
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
"""
Qlib A股数据自动更新脚本
基于 Yahoo Finance 获取最新数据并更新 Qlib 数据库
"""

import os
import sys
import logging
from datetime import datetime, timedelta
from pathlib import Path

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(message)s",
    handlers=[logging.StreamHandler()]
)
logger = logging.getLogger(__name__)

# 自动检测数据目录(兼容 Docker 容器和本地环境)
_CANDIDATES = [
    "/root/.qlib/qlib_data/cn_data",
    os.path.expanduser("~/.qlib/qlib_data/cn_data"),
]
QLIB_DATA_DIR = next((p for p in _CANDIDATES if os.path.exists(p)), _CANDIDATES[0])


def get_last_date_from_calendar():
    """从日历文件直接读取最新日期"""
    calendar_file = Path(QLIB_DATA_DIR) / "calendars" / "day.txt"
    if calendar_file.exists():
        lines = [l.strip() for l in calendar_file.read_text().splitlines() if l.strip()]
        if lines:
            last = lines[-1]
            logger.info(f"日历文件最新日期: {last}")
            return last
    return "2020-09-25"


def update_calendar_file(new_dates):
    """向日历文件追加新的交易日"""
    calendar_file = Path(QLIB_DATA_DIR) / "calendars" / "day.txt"
    if not calendar_file.exists():
        logger.warning("日历文件不存在,跳过更新")
        return

    existing = set(l.strip() for l in calendar_file.read_text().splitlines() if l.strip())
    to_add = [d for d in new_dates if d not in existing]

    if to_add:
        with open(calendar_file, 'a') as f:
            for d in sorted(to_add):
                f.write(d + '\n')
        logger.info(f"日历文件新增 {len(to_add)} 个交易日(最新: {sorted(to_add)[-1]})")
    else:
        logger.info("日历文件无需更新")


def update_instruments_end_date(end_date):
    """更新 instruments 文件中每只股票的结束日期"""
    instruments_dir = Path(QLIB_DATA_DIR) / "instruments"
    for fname in ["all.txt", "csi300.txt", "csi500.txt"]:
        fpath = instruments_dir / fname
        if not fpath.exists():
            continue
        lines = fpath.read_text().splitlines()
        new_lines = []
        for line in lines:
            parts = line.split("\t")
            if len(parts) >= 3:
                parts[2] = end_date
                new_lines.append("\t".join(parts))
            else:
                new_lines.append(line)
        fpath.write_text("\n".join(new_lines) + "\n")
    logger.info(f"instruments 文件结束日期已更新为 {end_date}")


def get_stock_list():
    """读取全部股票代码"""
    all_file = Path(QLIB_DATA_DIR) / "instruments" / "all.txt"
    stocks = []
    if all_file.exists():
        for line in all_file.read_text().splitlines():
            parts = line.strip().split("\t")
            if parts and parts[0]:
                stocks.append(parts[0])
    if not stocks:
        stocks = [
            "SH600519", "SH600036", "SH601318", "SH600276", "SH600000",
            "SH600887", "SH600030", "SH601012", "SH600031", "SH600900",
            "SZ000001", "SZ000858", "SZ000333", "SZ300750", "SZ002415",
            "SZ000002", "SZ000651", "SZ002594", "SZ300059", "SZ000568",
        ]
    return stocks


def to_yahoo_code(qlib_code):
    if qlib_code.startswith("SH"):
        return qlib_code[2:] + ".SS"
    elif qlib_code.startswith("SZ"):
        return qlib_code[2:] + ".SZ"
    return qlib_code


def download_and_write(stocks, start_date, end_date):
    """下载 Yahoo Finance 数据并写入 Qlib 二进制格式"""
    try:
        import yfinance as yf
        import pandas as pd
        import numpy as np
    except ImportError:
        os.system("pip install yfinance pandas numpy -q")
        import yfinance as yf
        import pandas as pd
        import numpy as np

    features_dir = Path(QLIB_DATA_DIR) / "features"
    features_dir.mkdir(parents=True, exist_ok=True)

    all_trading_dates = set()
    success_count = 0
    batch_size = 30

    # 只更新已有数据的股票
    existing_stocks = [s for s in stocks if (features_dir / s.lower()).exists()]
    if not existing_stocks:
        existing_stocks = stocks[:100]
    target_stocks = existing_stocks[:300]

    logger.info(f"准备更新 {len(target_stocks)} 只股票的数据...")

    for i in range(0, len(target_stocks), batch_size):
        batch = target_stocks[i:i + batch_size]
        yahoo_batch = [to_yahoo_code(c) for c in batch]

        try:
            if len(yahoo_batch) == 1:
                raw = yf.download(yahoo_batch[0], start=start_date, end=end_date,
                                  auto_adjust=True, progress=False)
                if not raw.empty:
                    raw.columns = [c.lower() for c in raw.columns]
                    data_map = {batch[0]: raw}
                else:
                    data_map = {}
            else:
                raw = yf.download(yahoo_batch, start=start_date, end=end_date,
                                  group_by='ticker', auto_adjust=True,
                                  progress=False, threads=True)
                data_map = {}
                if not raw.empty:
                    for qlib_code, ycode in zip(batch, yahoo_batch):
                        try:
                            if ycode in raw.columns.get_level_values(0):
                                df = raw[ycode].copy()
                                df.columns = [c.lower() for c in df.columns]
                                df = df.dropna(how='all')
                                if not df.empty:
                                    data_map[qlib_code] = df
                        except Exception:
                            pass

            for qlib_code, df in data_map.items():
                if df.empty:
                    continue
                df.index = pd.to_datetime(df.index).tz_localize(None)
                trading_dates = [d.strftime('%Y-%m-%d') for d in df.index]
                all_trading_dates.update(trading_dates)

                stock_dir = features_dir / qlib_code.lower()
                stock_dir.mkdir(parents=True, exist_ok=True)

                field_map = {'close': 'close', 'open': 'open',
                             'high': 'high', 'low': 'low', 'volume': 'volume'}
                for field, col in field_map.items():
                    if col not in df.columns:
                        continue
                    series = df[col].dropna().astype(np.float32)
                    if series.empty:
                        continue
                    bin_file = stock_dir / f"{field}.day.bin"
                    if bin_file.exists():
                        existing_data = np.fromfile(str(bin_file), dtype=np.float32)
                        combined = np.concatenate([existing_data, series.values])
                        combined.tofile(str(bin_file))
                    else:
                        series.values.tofile(str(bin_file))

                success_count += 1

        except Exception as e:
            logger.debug(f"批次 {i//batch_size+1} 下载失败: {e}")
            continue

        if (i // batch_size + 1) % 5 == 0:
            logger.info(f"进度: {min(i+batch_size, len(target_stocks))}/{len(target_stocks)} 只")

    logger.info(f"数据写入完成: 成功 {success_count}/{len(target_stocks)} 只")
    return success_count, sorted(all_trading_dates)


def main():
    import pandas as pd

    today = datetime.today().strftime("%Y-%m-%d")

    logger.info("=" * 50)
    logger.info("Qlib 数据增量更新开始")
    logger.info("=" * 50)

    if not Path(QLIB_DATA_DIR).exists():
        logger.error(f"Qlib 数据目录不存在: {QLIB_DATA_DIR}")
        return

    last_date = get_last_date_from_calendar()
    start_date = (datetime.strptime(last_date, "%Y-%m-%d") + timedelta(days=1)).strftime("%Y-%m-%d")

    if start_date >= today:
        logger.info("✅ 数据已是最新,无需更新")
        return

    logger.info(f"增量更新: {start_date} → {today}")

    stocks = get_stock_list()
    logger.info(f"股票总数: {len(stocks)}")

    count, trading_dates = download_and_write(stocks, start_date, today)

    if count > 0:
        if trading_dates:
            update_calendar_file(trading_dates)
        else:
            bdays = pd.bdate_range(start=start_date, end=today)
            update_calendar_file([d.strftime('%Y-%m-%d') for d in bdays])

        update_instruments_end_date(today)
        logger.info(f"🎉 数据更新完成!已更新 {count} 只股票,数据覆盖至 {today}")
    else:
        logger.warning("⚠️ 未能更新任何股票数据,将使用现有数据继续运行")


if __name__ == "__main__":
    main()