""" 网格交易和定投交易回测工具 - 后端主入口 """ import warnings warnings.filterwarnings("ignore", category=DeprecationWarning) from fastapi import FastAPI, File, UploadFile, Form, HTTPException, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse from contextlib import asynccontextmanager from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor import threading import json import pandas as pd import io from datetime import datetime, timedelta, timezone import traceback import os import sys import asyncio import time import difflib import socket import hashlib from typing import Any try: import httpx except ImportError: httpx = None from v2.api import router as v2_router from v2.db import ( init_db as init_db_v2, DatabaseUnavailableError, test_postgres_socket, manual_bootstrap_database, ) from v2.time_sync import now_utc if hasattr(sys.stdout, "reconfigure"): sys.stdout.reconfigure(encoding="utf-8", errors="replace") if hasattr(sys.stderr, "reconfigure"): sys.stderr.reconfigure(encoding="utf-8", errors="replace") # 添加app目录到路径 _app_dir = os.path.join(os.path.dirname(__file__), 'app') if os.path.isdir(_app_dir): sys.path.insert(0, _app_dir) # 完整的SQLite数据库实现 def get_db_path(): """获取数据库绝对路径""" backend_dir = os.path.dirname(os.path.abspath(__file__)) data_dir = os.path.join(backend_dir, 'data') os.makedirs(data_dir, exist_ok=True) return os.path.join(data_dir, 'backtest.db') def init_database(): """初始化SQLite数据库""" return False def save_backtest_to_database(strategy_type: str, symbol: str, params: dict, results: dict, task_id: str): """保存回测结果到数据库""" return False def load_backtest_history(): """从数据库加载历史记录""" return [] def delete_backtest_from_database(record_id: str): """从数据库删除记录""" return False # 初始化数据库 HAS_DATABASE = False def _maybe_interactive_db_fix(exc: Exception) -> None: """ 当 v2 数据库初始化失败时,在本地开发环境下提供一个简单的交互式修复菜单。 仅在标准输入为 TTY 且 `WGJY_DB_INTERACTIVE_FIX` 未显式设置为 `0` 时生效, 避免在 Docker/守护进程等非交互环境中卡死。 """ enable = (os.getenv('WGJY_DB_INTERACTIVE_FIX') or '1').strip().lower() if enable in ('0', 'false', 'no'): return # 非交互环境直接跳过 if not sys.stdin or not sys.stdin.isatty(): return print('\n检测到 v2 数据库初始化失败,可以尝试以下快速诊断/修复操作:') print(' 1) 显示当前数据库连接配置(隐藏密码)') print(' 2) 测试数据库端口连通性') print(' 3) 如果已配置 WGJY_PG_ADMIN_URL,尝试自动初始化数据库/用户') print(' 4) 如果已配置 WGJY_PG_ADMIN_URL,尝试先删除同名数据库再重建(危险,仅开发/测试环境)') print(' 0) 跳过,不做任何操作') try: choice = input('请输入选项编号并回车(直接回车等同于 0):').strip() except Exception: return if not choice or choice == '0': return from sqlalchemy.engine.url import make_url from v2.config import get_database_url if choice == '1': try: raw = get_database_url() u = make_url(raw) # SQLAlchemy URL 对象在打印时会自动隐藏密码,这里直接打印即可 print(f'当前数据库连接:{u}') except Exception as e: print(f'读取或解析数据库连接串失败:{type(e).__name__}: {e}') elif choice == '2': try: raw = get_database_url() test_postgres_socket(raw) print('✅ PostgreSQL 端口连通性正常(TCP 探测通过)。') except Exception as e: print(f'❌ PostgreSQL 端口或主机不可达:{type(e).__name__}: {e}') elif choice == '3': msg = manual_bootstrap_database() print(msg) elif choice == '4': # 临时开启"删除同名数据库再重建"的行为,只在本次交互中生效 old = os.getenv('WGJY_PG_BOOTSTRAP_DROP_DB') os.environ['WGJY_PG_BOOTSTRAP_DROP_DB'] = '1' try: msg = manual_bootstrap_database() print(msg) finally: if old is None: os.environ.pop('WGJY_PG_BOOTSTRAP_DROP_DB', None) else: os.environ['WGJY_PG_BOOTSTRAP_DROP_DB'] = old else: print('未识别的选项,跳过交互式修复。') @asynccontextmanager async def lifespan(app: FastAPI): # 启动时的操作 print("启动回测系统后端服务...") db_retry_task = None try: init_db_v2() except Exception as e: print(f"❌ v2 数据库初始化失败: {e}") _maybe_interactive_db_fix(e) async def _db_retry_loop(): while True: try: await asyncio.sleep(3) init_db_v2() except asyncio.CancelledError: raise except Exception: continue else: print('✅ v2 数据库初始化成功(已自动恢复)') return db_retry_task = asyncio.create_task(_db_retry_loop()) startup_refresh_task = asyncio.create_task(asyncio.to_thread(_refresh_akshare_spot_cache, True)) spot_task = asyncio.create_task(_spot_auto_refresh_loop()) async def _auto_update_daily_close_at_15_10(): """每个交易日下午 15:10(北京时间)自动执行持仓日线增量更新。""" while True: try: await asyncio.sleep(60) bj = now_utc().astimezone(timezone(timedelta(hours=8))) if bj.weekday() >= 5: continue if bj.hour != 15 or bj.minute != 10: continue if httpx is None: continue base = os.environ.get('WGJY_SERVER_URL', 'http://127.0.0.1:8000').rstrip('/') secret = os.environ.get('WGJY_AUTO_UPDATE_SECRET', '') try: async with httpx.AsyncClient(timeout=120.0) as client: await client.post( f'{base}/api/v2/trades/prices/close/auto-update', headers={'X-Auto-Update-Secret': secret} if secret else {}, ) except Exception: pass await asyncio.sleep(60) except asyncio.CancelledError: raise except Exception: continue auto_update_task = asyncio.create_task(_auto_update_daily_close_at_15_10()) yield # 关闭时的操作 try: if db_retry_task is not None: db_retry_task.cancel() await db_retry_task except Exception: pass try: startup_refresh_task.cancel() await startup_refresh_task except Exception: pass try: spot_task.cancel() await spot_task except Exception: pass try: auto_update_task.cancel() await auto_update_task except Exception: pass print("关闭回测系统后端服务...") # 创建FastAPI应用 app = FastAPI( title="网格交易和定投交易回测工具", description="专业的量化交易策略回测平台", version="1.0.0", lifespan=lifespan, ) @app.exception_handler(DatabaseUnavailableError) async def _db_unavailable_handler(request: Request, exc: DatabaseUnavailableError): return JSONResponse( status_code=503, content={ 'detail': str(exc) or '数据库不可用,请稍后重试', 'code': 'DATABASE_UNAVAILABLE', }, headers={'Retry-After': '3'}, ) @app.middleware('http') async def _block_v1_api(request: Request, call_next): path = request.url.path or '' if path == '/api/v1' or path.startswith('/api/v1/'): return JSONResponse(status_code=410, content={'detail': 'api v1 已下线,请使用 /api/v2'}) return await call_next(request) app.include_router(v2_router) # ==================== 并发执行器(避免长任务阻塞接口) ==================== _backtest_history_lock = threading.RLock() _download_tasks_lock = threading.RLock() _baostock_lock = threading.RLock() _baostock_login_state = {'pid': None, 'logged_in': False} def _ensure_baostock_login(bs): pid = os.getpid() if _baostock_login_state.get('pid') != pid: _baostock_login_state['pid'] = pid _baostock_login_state['logged_in'] = False if not _baostock_login_state.get('logged_in'): lg = bs.login() if getattr(lg, 'error_code', None) != '0': raise RuntimeError(getattr(lg, 'error_msg', 'login failed')) _baostock_login_state['logged_in'] = True def _make_backtest_executor(): """Windows 下进程池要求函数可 picklable(不能是 __main__)。 - 以 `python -m uvicorn server:app` 启动时:模块名是 `server`,可用进程池获得真正并行。 - 以 `python server.py` 启动时:模块名是 `__main__`,进程池可能异常;退化为线程池。 """ cpu = os.cpu_count() or 2 workers = max(2, min(4, cpu)) if __name__ == "__main__": return ThreadPoolExecutor(max_workers=workers) return ProcessPoolExecutor(max_workers=workers) BACKTEST_EXECUTOR = _make_backtest_executor() DOWNLOAD_EXECUTOR = ThreadPoolExecutor(max_workers=8) PARSE_EXECUTOR = ThreadPoolExecutor(max_workers=4) # 上传文件解析缓存(避免重复读取/解析 Excel 导致每次回测都很慢) _PARSED_UPLOAD_CACHE_TTL_SECONDS = 7 * 86400 _PARSED_UPLOAD_PARSER_VERSION = 'v2' _parsed_upload_cache_lock = threading.Lock() _parsed_upload_cache: dict[str, dict[str, Any]] = {} def _get_parsed_upload_cache_dir() -> str: backend_dir = os.path.dirname(__file__) cache_dir = os.path.join(backend_dir, 'cache', 'parsed_uploads') os.makedirs(cache_dir, exist_ok=True) return cache_dir def _get_parsed_upload_cache_path(key: str) -> str: return os.path.join(_get_parsed_upload_cache_dir(), f"{key}.json") BAOSTOCK_SOCKET_TIMEOUT_SECONDS = 25 BAOSTOCK_MAX_RETRIES = 2 BAOSTOCK_SEGMENT_PARALLELISM = 4 BAOSTOCK_SEGMENT_PARALLEL_THRESHOLD = 4 def _make_baostock_segment_executor(): workers = max(1, int(BAOSTOCK_SEGMENT_PARALLELISM or 1)) if __name__ == "__main__": return ThreadPoolExecutor(max_workers=1) return ProcessPoolExecutor(max_workers=workers) BAOSTOCK_SEGMENT_EXECUTOR = _make_baostock_segment_executor() def _iter_date_ranges(start_date: str, end_date: str, days_per_segment: int): try: start_dt = datetime.strptime(start_date, '%Y-%m-%d') end_dt = datetime.strptime(end_date, '%Y-%m-%d') except Exception: yield start_date, end_date return if end_dt < start_dt: yield start_date, end_date return cur = start_dt step = max(1, int(days_per_segment or 1)) while cur <= end_dt: seg_end = min(end_dt, cur + timedelta(days=step - 1)) yield cur.strftime('%Y-%m-%d'), seg_end.strftime('%Y-%m-%d') cur = seg_end + timedelta(days=1) def _baostock_query_history_segment(bs, symbol: str, fields: str, start_date: str, end_date: str, frequency: str, adjustflag: str): last_timeout = socket.getdefaulttimeout() socket.setdefaulttimeout(BAOSTOCK_SOCKET_TIMEOUT_SECONDS) try: with _baostock_lock: _ensure_baostock_login(bs) rs = bs.query_history_k_data_plus( symbol, fields, start_date=start_date, end_date=end_date, frequency=frequency, adjustflag=adjustflag, ) if rs is None: raise RuntimeError('Baostock返回None') if getattr(rs, 'error_code', None) != '0': raise RuntimeError(getattr(rs, 'error_msg', 'unknown')) data_list = [] while rs.next(): data_list.append(rs.get_row_data()) return data_list, getattr(rs, 'fields', None) finally: socket.setdefaulttimeout(last_timeout) def _baostock_segment_worker(payload: tuple[str, str, str, str, str, str]): symbol, fields, start_date, end_date, frequency, adjustflag = payload import baostock as bs # type: ignore last_timeout = socket.getdefaulttimeout() socket.setdefaulttimeout(BAOSTOCK_SOCKET_TIMEOUT_SECONDS) try: with _baostock_lock: _ensure_baostock_login(bs) rs = bs.query_history_k_data_plus( symbol, fields, start_date=start_date, end_date=end_date, frequency=frequency, adjustflag=adjustflag, ) if rs is None: raise RuntimeError('Baostock返回None') if getattr(rs, 'error_code', None) != '0': raise RuntimeError(getattr(rs, 'error_msg', 'unknown')) rows = [] while rs.next(): rows.append(rs.get_row_data()) return rows, list(getattr(rs, 'fields', None) or []) finally: socket.setdefaulttimeout(last_timeout) def _grid_backtest_job(params: dict, market_data: list): engine = GridBacktestEngine(params) return engine.run(market_data) def _dca_backtest_job(params: dict, market_data: list): engine = DCABacktestEngine(params) return engine.run(market_data) def _run_download_coroutine(task_id: int) -> None: """Run the async download coroutine in a dedicated thread.""" try: asyncio.run(download_stock_data(task_id)) except Exception as e: try: with _download_tasks_lock: task = download_tasks.get(task_id) if task: task['status'] = 'failed' task['stage'] = 'failed' task['progress'] = 100 task['error'] = str(e) _save_download_history() except Exception: pass # 全局计数器,用于生成唯一ID backtest_counter = 0 # 启动时从数据库加载历史记录 backtest_history = load_backtest_history() if HAS_DATABASE else [] # 简化的数据库存储 import os def init_simple_database(): """初始化简化数据库""" db_path = "data/backtest_simple.db" # 确保数据目录存在 os.makedirs("data", exist_ok=True) return False def save_to_simple_database(record): """保存记录到简化数据库""" return False def load_from_simple_database(): """从简化数据库加载所有记录""" return [] def delete_from_simple_database(record_id): """从简化数据库删除记录""" return False # 注意:backtest_history 已在上面从主数据库加载,不需要重复加载 # backtest_history = load_from_simple_database() # 已注释掉,避免覆盖主数据库的数据 def generate_backtest_id(): """生成基于时间和次数的回测任务ID""" global backtest_counter backtest_counter += 1 now = datetime.now() # 格式:YYYYMMDD_HHMMSS_次数 return f"{now.strftime('%Y%m%d_%H%M%S')}_{backtest_counter:03d}" def save_to_memory_history(strategy_type: str, symbol: str, params: dict, results: dict, task_id: str, market_data: list): """保存回测结果到内存历史记录和数据库""" global backtest_history # 创建历史记录 history_record = { "id": task_id, "strategy_type": strategy_type, "symbol": symbol, "start_date": market_data[0]['date'] if market_data else datetime.now().isoformat(), "end_date": market_data[-1]['date'] if market_data else datetime.now().isoformat(), "initial_capital": params.get('initial_capital', 100000), "parameters": params, "results": results, "status": "completed", "created_at": datetime.now().isoformat(), "completed_at": datetime.now().isoformat() } # 保存到数据库(持久化) if HAS_DATABASE: save_success = save_backtest_to_database( strategy_type, symbol, params, results, task_id ) if save_success: print(f"✅ 回测记录 {task_id} 已持久化到数据库") else: print(f"⚠️ 回测记录 {task_id} 数据库保存失败,仅保存在内存") with _backtest_history_lock: backtest_history.insert(0, history_record) # 插入到列表开头,最新的在前面 if len(backtest_history) > 100: backtest_history = backtest_history[:100] print(f"✅ 回测记录 {task_id} 已保存,当前内存中有 {len(backtest_history)} 条记录") # 这个函数已经在上面定义了,删除重复的版本 # 配置CORS app.add_middleware( CORSMiddleware, allow_origins=["http://localhost:3000", "http://127.0.0.1:3000", "http://localhost:3001", "http://127.0.0.1:3001"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"], ) @app.get("/") async def root(): return { "message": "网格交易和定投交易回测工具 API", "version": "1.0.0", "docs": "/docs" } @app.get("/health") async def health_check(): return {"status": "healthy"} class GridBacktestEngine: """真实网格交易回测引擎""" def __init__(self, params): self.params = params # 基本参数 self.initial_capital = float(params.get('initial_capital', 100000)) self.initial_shares = int(params.get('initial_shares', 0)) # 日期范围 self.start_date = params.get('start_date', None) self.end_date = params.get('end_date', None) # 网格参数 self.trigger_price_min = float(params.get('trigger_price_min', 8.5)) self.trigger_price_max = float(params.get('trigger_price_max', 76.5)) self.rise_trigger = float(params.get('rise_trigger', 1)) / 100 # 转换为小数 self.fall_trigger = float(params.get('fall_trigger', 1)) / 100 # 转换为小数 # 回落反弹参数 self.rise_pullback_enabled = params.get('rise_pullback_enabled', False) self.rise_pullback = float(params.get('rise_pullback', 0)) / 100 if params.get('rise_pullback_enabled') else 0 self.fall_rebound_enabled = params.get('fall_rebound_enabled', False) self.fall_rebound = float(params.get('fall_rebound', 0)) / 100 if params.get('fall_rebound_enabled') else 0 # 委托设置 self.order_type = params.get('order_type', 'shares') # shares 或 amount self.buy_amount = float(params.get('buy_amount', 100)) self.sell_amount = float(params.get('sell_amount', 100)) # 手续费 - 兼容前端字段名 self.commission_rate = float(params.get('fee_rate', params.get('commission_rate', 0.0005))) # 默认万分之50 = 0.5% self.min_commission = float(params.get('min_fee', params.get('min_commission', 0.01))) # 默认最低0.01元 self.fee_bilateral = params.get('fee_bilateral', True) # 初始化基准价 self.base_price = float(params.get('custom_base_price', 25.0)) # 基准价更新方式 self.base_price_update_mode = params.get('base_price_update_mode', 'trigger_price') # trigger_price 或 trade_price # 倍数委托:仅在"成交价更新基准价"模式下生效 # 当实际涨跌幅远大于设定触发比例时,按「实际涨跌幅 / 触发比例」放大单次委托数量/金额。 # 例如:触发比例 1%,实际上涨 3.2%,则按约 3.2 倍基础委托量下单(仍会按 100 股整数倍取整)。 self.multiple_order_enabled = bool(params.get('multiple_order_enabled', False)) # 持仓限制 self.position_limit_enabled = bool(params.get('position_limit_enabled', True)) self.position_limit_type = params.get('position_limit_type', 'shares') if self.position_limit_enabled: self.max_position = float(params.get('max_position', 0)) if params.get('max_position') else None self.min_position = float(params.get('min_position', 0)) if params.get('min_position') else None else: self.max_position = None self.min_position = None # 回落反弹状态跟踪 self.pending_sell = False # 是否有待执行的卖出(等待回落) self.pending_buy = False # 是否有待执行的买入(等待反弹) self.peak_price = self.base_price # 上涨过程中的峰值价格,初始化为基准价 self.valley_price = self.base_price # 下跌过程中的谷值价格,初始化为基准价 print(f"回测引擎初始化完成") print(f"初始资金: {self.initial_capital}, 初始股数: {self.initial_shares}") print(f"网格区间: {self.trigger_price_min} - {self.trigger_price_max}") print(f"触发幅度: 上涨{self.rise_trigger*100:.2f}%, 下跌{self.fall_trigger*100:.2f}%") print(f"基准价更新方式: {self.base_price_update_mode}") if self.rise_pullback_enabled: print(f"上涨回落: {self.rise_pullback*100:.2f}%") if self.fall_rebound_enabled: print(f"下跌反弹: {self.fall_rebound*100:.2f}%") print(f"手续费参数: 费率={self.commission_rate*10000}万分之一, 最低手续费={self.min_commission}元, 双边收费={self.fee_bilateral}") print(f"委托参数: 类型={self.order_type}, 买入量={self.buy_amount}, 卖出量={self.sell_amount}") print(f"初始资金: {self.initial_capital}元, 初始股数: {self.initial_shares}股") print(f"回测日期: {self.start_date} 至 {self.end_date}") def run(self, market_data): """执行真实网格回测""" if not market_data or len(market_data) == 0: raise ValueError("市场数据为空") # 根据日期范围过滤数据 if self.start_date or self.end_date: original_count = len(market_data) filtered_data = [] # 调试:显示前几条和最后几条数据的日期格式 print(f"调试信息 - 前3条数据的日期:") for i, row in enumerate(market_data[:3]): print(f" 第{i+1}条: {row.get('date', 'NO_DATE')} (原始格式)") print(f"调试信息 - 最后3条数据的日期:") for i, row in enumerate(market_data[-3:]): print(f" 倒数第{3-i}条: {row.get('date', 'NO_DATE')} (原始格式)") for row in market_data: row_date = str(row['date']).split(' ')[0] # 只取日期部分 # 检查是否在日期范围内 include_row = True try: # 使用字符串比较进行日期过滤 if self.start_date and row_date < self.start_date: include_row = False if self.end_date and row_date > self.end_date: include_row = False except Exception as e: print(f"日期比较错误: {row_date}, {e}") include_row = True # 如果日期比较失败,保留数据 if include_row: filtered_data.append(row) market_data = filtered_data print(f"日期过滤: 原始数据{original_count}条 -> 过滤后{len(market_data)}条") print(f"查找日期范围: {self.start_date} 至 {self.end_date}") if not market_data: raise ValueError(f"指定日期范围 {self.start_date} 至 {self.end_date} 内没有数据") print(f"开始回测,数据条数: {len(market_data)}") print(f"价格范围: {market_data[0]['close']} -> {market_data[-1]['close']}") print(f"日期范围: {market_data[0]['date']} -> {market_data[-1]['date']}") # 初始化账户 cash = self.initial_capital shares = self.initial_shares total_fee = 0 trades = [] equity_curve = [] # 新增:按"交易日"统计的权益序列,用于计算盈利/亏损天数与连续天数 _daily_equity: list[float] = [] _daily_position: list[float] = [] _prev_day: str | None = None _last_day_equity: float | None = None _last_day_position: float | None = None # 设置基准价为第一天收盘价 first_price = float(market_data[0]['close']) self.base_price = first_price initial_position_value = shares * first_price print(f"初始状态: 现金={cash}, 股数={shares}, 基准价={self.base_price}") # 统计变量 max_position_value = initial_position_value min_position_value = initial_position_value max_position_shares = shares min_position_shares = shares peak_equity = self.initial_capital + initial_position_value max_drawdown = 0 # 遍历每个交易日 for i, row in enumerate(market_data): current_price = float(row['close']) current_date = str(row['date']) # 检查价格是否在网格区间内 if current_price < self.trigger_price_min or current_price > self.trigger_price_max: # 价格超出网格区间,不交易 pass else: # 计算相对于基准价的涨跌幅 if self.base_price > 0: price_change = (current_price - self.base_price) / self.base_price # 回落反弹交易逻辑 trigger_reason = "" should_buy = False should_sell = False # 保存当前的待执行状态(在状态重置前) was_pending_buy = self.pending_buy was_pending_sell = self.pending_sell current_valley_price = self.valley_price current_peak_price = self.peak_price # 检查是否有待执行的买入(下跌反弹) if self.pending_buy: # 更新谷值价格 self.valley_price = min(self.valley_price, current_price) # 检查是否有足够的反弹 rebound_change = (current_price - self.valley_price) / self.valley_price if rebound_change >= self.fall_rebound: should_buy = True original_change = (self.valley_price - self.base_price) / self.base_price trigger_reason = f"下跌{abs(original_change)*100:.2f}%触发,反弹{rebound_change*100:.2f}%执行" # 注意:不在这里重置pending_buy,在交易执行后再重置 print(f"执行买入: {current_date} 价格={current_price:.2f} {trigger_reason}") # 检查是否有待执行的卖出(上涨回落) elif self.pending_sell: # 更新峰值价格 self.peak_price = max(self.peak_price, current_price) # 检查是否有足够的回落 pullback_change = (self.peak_price - current_price) / self.peak_price if pullback_change >= self.rise_pullback: should_sell = True original_change = (self.peak_price - self.base_price) / self.base_price trigger_reason = f"上涨{original_change*100:.2f}%触发,回落{pullback_change*100:.2f}%执行" # 注意:不在这里重置pending_sell,在交易执行后再重置 print(f"执行卖出: {current_date} 价格={current_price:.2f} {trigger_reason}") # 检查新的触发条件 else: # 检查下跌触发买入条件 if price_change <= -self.fall_trigger: # 下跌触发买入 if self.fall_rebound_enabled: # 启用下跌反弹:设置待执行状态 self.pending_buy = True self.valley_price = current_price print(f"下跌触发买入信号,等待反弹: {current_date} 价格={current_price:.2f} 跌幅={abs(price_change)*100:.2f}%") else: # 未启用下跌反弹:直接买入 should_buy = True trigger_reason = f"下跌{abs(price_change)*100:.2f}%触发" # 检查上涨触发卖出条件 elif price_change >= self.rise_trigger: # 上涨触发卖出 if self.rise_pullback_enabled: # 启用上涨回落:设置待执行状态 self.pending_sell = True self.peak_price = current_price print(f"上涨触发卖出信号,等待回落: {current_date} 价格={current_price:.2f} 涨幅={price_change*100:.2f}%") else: # 未启用上涨回落:直接卖出 should_sell = True trigger_reason = f"上涨{price_change*100:.2f}%触发" # 执行买入交易 if should_buy: # 每笔基础委托股数(不因跨多格而放大) base_buy_shares = int(self.buy_amount) if self.order_type == 'shares' else int(self.buy_amount / current_price) # 触发价更新模式下:如果一次性跌破多个网格,就拆成多笔委托 trigger_steps_buy = 1 if ( self.base_price_update_mode == 'trigger_price' and not was_pending_buy and self.fall_trigger > 0 ): try: total_change_abs = abs(price_change) trigger_steps_buy = max(1, int(total_change_abs / self.fall_trigger)) except Exception: trigger_steps_buy = 1 executions = ( trigger_steps_buy if (self.base_price_update_mode == 'trigger_price' and not was_pending_buy) else 1 ) for _ in range(executions): buy_shares = base_buy_shares # 倍数委托:仅在成交价更新基准价模式下生效 if self.multiple_order_enabled and self.base_price_update_mode == 'trade_price': try: # 计算本次相对基准价的总涨跌幅(绝对值) if was_pending_buy: # 下跌+反弹模式:参考触发时的谷值 total_change = abs((current_valley_price - old_base_price) / old_base_price) else: # 普通模式:直接使用当前价相对基准价的跌幅 total_change = abs(price_change) unit_change = self.fall_trigger # 每"1 格"对应的跌幅(小数) if unit_change > 0: multiple = max(1.0, total_change / unit_change) buy_shares = int(buy_shares * multiple) except Exception as e: print(f"倍数委托(买入)计算失败,按基础委托量执行: {e}") # 股数必须为100的倍数 buy_shares = (buy_shares // 100) * 100 # 检查持仓上限 if self.max_position is not None and buy_shares > 0: if self.position_limit_type == 'shares': if shares + buy_shares > self.max_position: # 调整买入数量以满足上限 original_buy_shares = buy_shares buy_shares = int(self.max_position - shares) buy_shares = (buy_shares // 100) * 100 # 再次确保是100的倍数 print(f"持仓上限警告: 计划买入 {original_buy_shares} 股将超过最大持股数 {self.max_position}。调整为买入 {buy_shares} 股。") elif self.position_limit_type == 'amount': if (shares + buy_shares) * current_price > self.max_position: # 调整买入数量以满足上限 original_buy_shares = buy_shares allowed_value = self.max_position - (shares * current_price) buy_shares = int(allowed_value / current_price) buy_shares = (buy_shares // 100) * 100 print(f"持仓上限警告: 计划买入 {original_buy_shares} 股将超过最大持仓金额 {self.max_position}。调整为买入 {buy_shares} 股。") cost = buy_shares * current_price fee = max(cost * self.commission_rate, self.min_commission) if not (cash >= cost + fee and buy_shares >= 100): # 至少100股才交易 break cash -= (cost + fee) shares += buy_shares total_fee += fee # 根据基准价更新方式更新基准价 old_base_price = self.base_price # 保存旧基准价用于计算触发价 print(f"DEBUG买入: base_price_update_mode = '{self.base_price_update_mode}', was_pending_buy = {was_pending_buy}") if self.base_price_update_mode == 'trigger_price': # 触发价更新:使用触发条件对应的价格 if was_pending_buy: # 回落反弹模式:使用谷值价格(触发点),只更新一次 self.base_price = current_valley_price print(f"基准价更新为触发价: {current_valley_price:.2f}元(谷值价格)") break else: # 普通模式:每次按一个网格级差更新一次基准价 trigger_price = old_base_price * (1 - self.fall_trigger) self.base_price = trigger_price print(f"基准价更新为触发价: {trigger_price:.4f}元(单格理论触发价)") else: # 成交价更新:使用成交价格 self.base_price = current_price print(f"基准价更新为成交价: {current_price:.2f}元") trade = { "date": current_date, "type": "买入", "price": round(current_price, 2), "shares": buy_shares, "amount": round(cost, 2), "fee": round(fee, 2), "cash_balance": round(cash, 2), "position_value": round(shares * current_price, 2), "current_shares": shares, # 当前持仓股数 "total_value": round(cash + shares * current_price, 2), "base_price": round(self.base_price, 4), "trigger_reason": trigger_reason } trades.append(trade) print(f"买入: {current_date} 价格={current_price:.2f} 数量={buy_shares} 触发条件={trigger_reason}") # 重置待执行状态(如果是回落反弹交易) if was_pending_buy: self.pending_buy = False # 执行卖出交易 elif should_sell and shares > 0: # 每笔基础委托股数(不因跨多格而放大) base_sell_shares = min(int(self.sell_amount), shares) if self.order_type == 'shares' else min(int(self.sell_amount / current_price), shares) # 触发价更新模式下:如果一次性上涨超过多个网格,就拆成多笔委托 trigger_steps_sell = 1 if ( self.base_price_update_mode == 'trigger_price' and not was_pending_sell and self.rise_trigger > 0 ): try: total_change_up = max(0.0, price_change) trigger_steps_sell = max(1, int(total_change_up / self.rise_trigger)) except Exception: trigger_steps_sell = 1 executions_sell = ( trigger_steps_sell if (self.base_price_update_mode == 'trigger_price' and not was_pending_sell) else 1 ) for _ in range(executions_sell): if shares <= 0: break sell_shares = min(base_sell_shares, shares) # 倍数委托:仅在成交价更新基准价模式下生效 if self.multiple_order_enabled and self.base_price_update_mode == 'trade_price': try: if was_pending_sell: # 上涨+回落模式:参考触发时的峰值 total_change = (current_peak_price - old_base_price) / old_base_price else: total_change = price_change # 上涨为正 unit_change = self.rise_trigger if unit_change > 0: multiple = max(1.0, total_change / unit_change) sell_shares = int(sell_shares * multiple) except Exception as e: print(f"倍数委托(卖出)计算失败,按基础委托量执行: {e}") # 股数必须为100的倍数 sell_shares = (sell_shares // 100) * 100 # 检查持仓下限 if self.min_position is not None and sell_shares > 0: if self.position_limit_type == 'shares': if shares - sell_shares < self.min_position: # 调整卖出数量以满足下限 original_sell_shares = sell_shares sell_shares = int(shares - self.min_position) sell_shares = (sell_shares // 100) * 100 print(f"持仓下限警告: 计划卖出 {original_sell_shares} 股将低于最小持股数 {self.min_position}。调整为卖出 {sell_shares} 股。") elif self.position_limit_type == 'amount': if (shares - sell_shares) * current_price < self.min_position: # 调整卖出数量以满足下限 original_sell_shares = sell_shares allowed_sell_value = (shares * current_price) - self.min_position sell_shares = int(allowed_sell_value / current_price) sell_shares = (sell_shares // 100) * 100 print(f"持仓下限警告: 计划卖出 {original_sell_shares} 股将低于最小持仓金额 {self.min_position}。调整为卖出 {sell_shares} 股。") # 再次确保本次卖出数量不会超过当前持仓(防止出现负持仓) if sell_shares > shares: sell_shares = (shares // 100) * 100 if sell_shares < 100: # 至少100股才交易 break revenue = sell_shares * current_price fee = max(revenue * self.commission_rate, self.min_commission) if self.fee_bilateral else 0 cash += (revenue - fee) shares -= sell_shares total_fee += fee # 根据基准价更新方式更新基准价 old_base_price = self.base_price # 保存旧基准价用于计算触发价 print(f"DEBUG卖出: base_price_update_mode = '{self.base_price_update_mode}', was_pending_sell = {was_pending_sell}") if self.base_price_update_mode == 'trigger_price': # 触发价更新:使用触发条件对应的价格 if was_pending_sell: # 回落反弹模式:使用峰值价格(触发点),只更新一次 self.base_price = current_peak_price print(f"基准价更新为触发价: {current_peak_price:.2f}元(峰值价格)") break else: # 普通模式:每次按一个网格级差更新一次基准价 trigger_price = old_base_price * (1 + self.rise_trigger) self.base_price = trigger_price print(f"基准价更新为触发价: {trigger_price:.4f}元(单格理论触发价)") else: # 成交价更新:使用成交价格 self.base_price = current_price print(f"基准价更新为成交价: {current_price:.2f}元") trade = { "date": current_date, "type": "卖出", "price": round(current_price, 2), "shares": sell_shares, "amount": round(revenue, 2), "fee": round(fee, 2), "cash_balance": round(cash, 2), "position_value": round(shares * current_price, 2), "current_shares": shares, # 当前持仓股数 "total_value": round(cash + shares * current_price, 2), "base_price": round(self.base_price, 4), "trigger_reason": trigger_reason } trades.append(trade) print(f"卖出: {current_date} 价格={current_price:.2f} 数量={sell_shares} 触发条件={trigger_reason}") # 重置待执行状态(如果是回落反弹交易) if was_pending_sell: self.pending_sell = False # 更新持仓统计 current_position_value = shares * current_price max_position_value = max(max_position_value, current_position_value) min_position_value = min(min_position_value, current_position_value) max_position_shares = max(max_position_shares, shares) min_position_shares = min(min_position_shares, shares) # 计算权益曲线和回撤 current_equity = cash + current_position_value if current_equity > peak_equity: peak_equity = current_equity drawdown = (peak_equity - current_equity) / peak_equity if peak_equity > 0 else 0 max_drawdown = max(max_drawdown, drawdown) # 新增:按日聚合权益(用于盈利/亏损天数、连续天数、资金利用率) day_key = str(current_date).split(' ')[0].split('T')[0] if _prev_day is None: _prev_day = day_key if day_key != _prev_day: if _last_day_equity is not None and _last_day_position is not None: _daily_equity.append(float(_last_day_equity)) _daily_position.append(float(_last_day_position)) _prev_day = day_key _last_day_equity = float(current_equity) _last_day_position = float(current_position_value) # 记录权益曲线(每10天记录一次以减少数据量) if i % 10 == 0 or i == len(market_data) - 1: equity_curve.append({ "date": current_date, "value": round(current_equity, 2), "cash": round(cash, 2), "position": round(current_position_value, 2), "return_rate": round((current_equity - self.initial_capital - initial_position_value) / self.initial_capital * 100, 2) }) # 计算最终指标 last_price = float(market_data[-1]['close']) final_equity = cash + shares * last_price initial_equity = self.initial_capital + initial_position_value total_profit = final_equity - initial_equity total_return = (total_profit / initial_equity) * 100 if initial_equity > 0 else 0 # 新增:补齐最后一个交易日 if _last_day_equity is not None and _last_day_position is not None: _daily_equity.append(float(_last_day_equity)) _daily_position.append(float(_last_day_position)) profit_days = 0 loss_days = 0 max_profit_streak = 0 max_loss_streak = 0 _cur_profit = 0 _cur_loss = 0 if len(_daily_equity) >= 2: for i in range(1, len(_daily_equity)): diff = _daily_equity[i] - _daily_equity[i - 1] if diff > 0: profit_days += 1 _cur_profit += 1 _cur_loss = 0 elif diff < 0: loss_days += 1 _cur_loss += 1 _cur_profit = 0 else: _cur_profit = 0 _cur_loss = 0 max_profit_streak = max(max_profit_streak, _cur_profit) max_loss_streak = max(max_loss_streak, _cur_loss) capital_utilization = 0.0 try: if initial_equity > 0 and len(_daily_position) > 0: capital_utilization = (sum(_daily_position) / len(_daily_position)) / initial_equity * 100 except Exception: capital_utilization = 0.0 # 计算年化收益率 start_dt = None end_dt = None try: start_dt = pd.to_datetime(market_data[0].get('date')).to_pydatetime() end_dt = pd.to_datetime(market_data[-1].get('date')).to_pydatetime() except Exception: start_dt = None end_dt = None if start_dt and end_dt: days = max((end_dt - start_dt).days, 1) else: days = max(len(market_data), 1) years = max(days / 365.25, 1 / 365.25) # 至少1天 annual_return = ((final_equity / initial_equity) ** (1 / years) - 1) * 100 if initial_equity > 0 else 0 # 计算胜率 buy_trades = [t for t in trades if t['type'] == '买入'] sell_trades = [t for t in trades if t['type'] == '卖出'] winning_trades = 0 # 简单计算:卖出价格高于对应买入价格 for i, sell_trade in enumerate(sell_trades): if i < len(buy_trades) and sell_trade['price'] > buy_trades[i]['price']: winning_trades += 1 win_rate = (winning_trades / len(sell_trades) * 100) if len(sell_trades) > 0 else 0 # 计算最大收益率 max_return_rate = 0 if len(equity_curve) > 0: initial_equity = self.initial_capital + initial_position_value max_equity_in_curve = max([point['value'] for point in equity_curve]) max_return_rate = ((max_equity_in_curve - initial_equity) / initial_equity) * 100 print(f"回测完成: 交易{len(trades)}次, 最终权益={final_equity:.2f}, 收益率={total_return:.2f}%") return { "final_equity": round(final_equity, 2), "total_profit": round(total_profit, 2), "total_return": round(total_return, 2), "annual_return": round(annual_return, 2), "total_trades": len(trades), "buy_trades": len(buy_trades), # 买入次数 "sell_trades": len(sell_trades), # 卖出次数 "win_rate": round(win_rate, 2), "max_invested_return": round(max_return_rate, 2), # 最大投入收益率 "max_drawdown": round(max_drawdown * 100, 2), "sharpe_ratio": round(annual_return / max(max_drawdown * 100, 1), 2), "total_fee": round(total_fee, 2), "max_position_value": round(max_position_value, 2), "min_position_value": round(min_position_value, 2), "max_position_shares": max_position_shares, "min_position_shares": min_position_shares, "profit_days": int(profit_days), "loss_days": int(loss_days), "max_profit_streak": int(max_profit_streak), "max_loss_streak": int(max_loss_streak), "capital_utilization": round(float(capital_utilization), 2), "trade_records": trades, # 返回所有交易记录 "equity_curve": equity_curve, "summary": { "data_points": len(market_data), "trading_days": days, "price_range": f"{market_data[0]['close']:.2f} - {market_data[-1]['close']:.2f}", "buy_trades": len(buy_trades), "sell_trades": len(sell_trades), "final_cash": round(cash, 2), "final_shares": shares, "final_position_value": round(shares * last_price, 2) } } def parse_uploaded_file(file_content: bytes, filename: str, keep_intraday: bool = False): """解析上传的文件""" try: t0 = time.perf_counter() # 解析缓存:同一个文件重复回测时直接复用(尤其 xlsx 读取很慢) base_hash = hashlib.sha1(file_content).hexdigest() cache_key = f"{base_hash}_{'intraday' if keep_intraday else 'daily'}_{_PARSED_UPLOAD_PARSER_VERSION}" now_ts = time.time() try: with _parsed_upload_cache_lock: hit = _parsed_upload_cache.get(cache_key) if hit and (now_ts - float(hit.get('ts', 0.0) or 0.0)) < _PARSED_UPLOAD_CACHE_TTL_SECONDS: data = hit.get('data') if isinstance(data, list) and data: print(f"✅ 命中解析缓存: {cache_key}({len(data)}条)") return data except Exception: pass # 先尝试从磁盘缓存加载(支持重启后复用) try: disk_path = _get_parsed_upload_cache_path(cache_key) if os.path.exists(disk_path): mtime = os.path.getmtime(disk_path) if (now_ts - mtime) < _PARSED_UPLOAD_CACHE_TTL_SECONDS: with open(disk_path, 'r', encoding='utf-8') as f: data = json.load(f) if isinstance(data, list) and data: with _parsed_upload_cache_lock: _parsed_upload_cache[cache_key] = {'ts': now_ts, 'data': data} print(f"✅ 命中磁盘解析缓存: {cache_key}({len(data)}条)") return data except Exception: pass if filename.endswith('.csv'): # 直接从 bytes 读取,避免大文件 decode 成巨大的字符串 try: df = pd.read_csv(io.BytesIO(file_content), encoding='utf-8', engine='c', low_memory=False) except Exception: df = pd.read_csv(io.BytesIO(file_content), encoding='gbk', engine='c', low_memory=False) elif filename.endswith('.day'): # 通达信日线 .day(二进制) from v2.tdx_day import parse_tdx_day_bytes # 这里不依赖 code6:回测只需要价格序列 rows = parse_tdx_day_bytes(buf=file_content, code6='000000') if not isinstance(rows, list) or not rows: raise ValueError('文件中未解析出有效的 .day 数据') market_data = [] for r in rows: try: market_data.append( { 'date': str(r.get('trade_date') or r.get('date') or '').strip(), 'open': float(r.get('open') or r.get('close') or 0), 'high': float(r.get('high') or r.get('close') or 0), 'low': float(r.get('low') or r.get('close') or 0), 'close': float(r.get('close') or 0), 'volume': int(float(r.get('volume') or 0)), } ) except Exception: continue # 写入缓存(内存+磁盘)并直接返回 try: if isinstance(market_data, list) and market_data: with _parsed_upload_cache_lock: _parsed_upload_cache[cache_key] = {'ts': now_ts, 'data': market_data} disk_path = _get_parsed_upload_cache_path(cache_key) with open(disk_path, 'w', encoding='utf-8') as f: json.dump(market_data, f, ensure_ascii=False) except Exception: pass print(f"成功解析 {len(market_data)} 条数据(耗时 {(time.perf_counter() - t0):.3f}s)") if market_data: print(f"样例数据: {market_data[0]}") return market_data elif filename.endswith(('.xlsx', '.xls')): try: wanted = { '日期', '时间', 'Date', 'Time', '开盘价', '开盘', 'Open', '最高价', '最高', 'High', '最低价', '最低', 'Low', '收盘价', '收盘', 'Close', '成交量', 'Volume', } wanted_lower = {str(x).strip().lower() for x in wanted} | { 'date', 'time', 'open', 'high', 'low', 'close', 'volume', } df = pd.read_excel( io.BytesIO(file_content), engine='openpyxl', usecols=lambda c: str(c).strip().lower() in wanted_lower, ) except Exception: df = pd.read_excel(io.BytesIO(file_content)) elif filename.endswith('.json'): data = json.loads(file_content.decode('utf-8')) df = pd.DataFrame(data) else: raise ValueError("不支持的文件格式") df.columns = [str(c).strip() for c in df.columns] print(f"原始列名: {df.columns.tolist()}") # 标准化列名 # 注意:分钟线常见同时存在"日期"和"时间",不能都映射为 date,否则会出现重复列 column_mapping = { '日期': 'date', 'Date': 'date', 'DATE': 'date', '时间': 'time', 'Time': 'time', 'TIME': 'time', '开盘价': 'open', '开盘': 'open', 'Open': 'open', 'OPEN': 'open', '最高价': 'high', '最高': 'high', 'High': 'high', 'HIGH': 'high', '最低价': 'low', '最低': 'low', 'Low': 'low', 'LOW': 'low', '收盘价': 'close', '收盘': 'close', 'Close': 'close', 'CLOSE': 'close', '成交量': 'volume', 'Volume': 'volume', 'VOLUME': 'volume' } df = df.rename(columns=column_mapping) print(f"标准化后列名: {df.columns.tolist()}") # 确保必需的列存在 # date/time 二选一至少要有一个;close 必须有 missing_columns = [] if 'close' not in df.columns: missing_columns.append('close') if 'date' not in df.columns and 'time' not in df.columns: missing_columns.append('date') if missing_columns: # 尝试自动检测列 if len(df.columns) >= 5: df.columns = ['date', 'open', 'high', 'low', 'close', 'volume'][:len(df.columns)] else: raise ValueError(f"缺少必需的列: {missing_columns}") # 统一补齐/清洗字段(向量化,避免 iterrows 带来的 O(N) Python 循环开销) if 'open' not in df.columns: df['open'] = df.get('close') if 'high' not in df.columns: df['high'] = df.get('close') if 'low' not in df.columns: df['low'] = df.get('close') if 'volume' not in df.columns: df['volume'] = 0 # 数值列转为 numeric for c in ['open', 'high', 'low', 'close', 'volume']: if c in df.columns: df[c] = pd.to_numeric(df[c], errors='coerce') # 日期列:尽量保留时间信息(分钟线/日线都兼容) if 'date' in df.columns and 'time' in df.columns: date_dt = pd.to_datetime(df['date'], errors='coerce') date_base = date_dt.dt.floor('D') time_raw = df['time'] # 1) 优先识别"完整时间戳"格式:YYYYMMDDHHMMSS(fff) # 常见于分钟线导出:时间列可能是 20190102093500000 这种数字/字符串。 dt_full = None try: time_s = time_raw.astype(str).str.strip() # 去掉 .0 / 科学计数等,保留数字 digits = time_s.str.replace(r"\D", "", regex=True) # 取前14位作为 YYYYMMDDHHMMSS d14 = digits.str.slice(0, 14) dt1 = pd.to_datetime(d14, format='%Y%m%d%H%M%S', errors='coerce') # 部分数据可能只有到分钟:YYYYMMDDHHMM d12 = digits.str.slice(0, 12) dt2 = pd.to_datetime(d12, format='%Y%m%d%H%M', errors='coerce') # 选择解析成功率更高的 r1 = float(dt1.notna().mean()) if len(dt1) else 0.0 r2 = float(dt2.notna().mean()) if len(dt2) else 0.0 if max(r1, r2) >= 0.8: dt_full = dt1 if r1 >= r2 else dt2 except Exception: dt_full = None if dt_full is not None: dt = dt_full else: # 2) 退化:把 time 当作"当天时间"来解析(Excel 小数 / 0 days 09:35:00 / 09:35:00) td = None try: if pd.api.types.is_numeric_dtype(time_raw): time_num = pd.to_numeric(time_raw, errors='coerce') # Excel 时间通常是 [0,1) 的小数(天的小数部分) if float(time_num.max(skipna=True) or 0) <= 1.5: td = pd.to_timedelta(time_num.fillna(0) * 86400.0, unit='s') except Exception: td = None if td is None: time_s2 = time_raw.astype(str).str.strip() time_s2 = time_s2.replace({'nan': '', 'None': '', 'NaT': ''}) td = pd.to_timedelta(time_s2.replace('', pd.NA), errors='coerce') td = td.fillna(pd.Timedelta(0)) dt = date_base + td elif 'date' in df.columns: dt = pd.to_datetime(df['date'], errors='coerce') elif 'time' in df.columns: dt = pd.to_datetime(df['time'], errors='coerce') else: dt = pd.Series([pd.NaT] * len(df)) tmp = pd.DataFrame({ 'dt': dt, 'open': df['open'].fillna(df['close']).fillna(0.0).astype(float), 'high': df['high'].fillna(df['close']).fillna(0.0).astype(float), 'low': df['low'].fillna(df['close']).fillna(0.0).astype(float), 'close': df['close'].fillna(0.0).astype(float), 'volume': df['volume'].fillna(0).astype(int), }) tmp = tmp[tmp['dt'].notna()].copy() # 如果是分钟/多条/天数据: # - keep_intraday=True(网格需要真实时间)则保留原始时间戳 # - 否则聚合成日线(显著减少数据量,提升速度) # 聚合逻辑:open=首条、high=max、low=min、close=末条、volume=sum tmp['date_only'] = tmp['dt'].dt.floor('D') try: is_intraday = bool(tmp['date_only'].duplicated().any()) except Exception: is_intraday = False if is_intraday and keep_intraday: out_df = tmp.sort_values('dt').copy() # 保留分钟线真实时间(前端展示时以空格切分出 HH:MM:SS) out_df['date'] = out_df['dt'].dt.strftime('%Y-%m-%d %H:%M:%S') out_df = out_df[['date', 'open', 'high', 'low', 'close', 'volume']] elif is_intraday: tmp = tmp.sort_values('dt') g = tmp.groupby('date_only', sort=True) out_df = g.agg( open=('open', 'first'), high=('high', 'max'), low=('low', 'min'), close=('close', 'last'), volume=('volume', 'sum'), ).reset_index() out_df['date'] = out_df['date_only'].dt.strftime('%Y-%m-%d') + ' 15:00:00' out_df = out_df[['date', 'open', 'high', 'low', 'close', 'volume']] else: out_df = tmp.copy() # 日线统一使用收盘时间 15:00:00,避免前端默认显示 09:30 out_df['date'] = out_df['date_only'].dt.strftime('%Y-%m-%d') + ' 15:00:00' out_df = out_df[['date', 'open', 'high', 'low', 'close', 'volume']] market_data = out_df.to_dict('records') # 写入缓存(内存+磁盘) try: if isinstance(market_data, list) and market_data: with _parsed_upload_cache_lock: _parsed_upload_cache[cache_key] = {'ts': now_ts, 'data': market_data} disk_path = _get_parsed_upload_cache_path(cache_key) with open(disk_path, 'w', encoding='utf-8') as f: json.dump(market_data, f, ensure_ascii=False) except Exception: pass print(f"成功解析 {len(market_data)} 条数据(耗时 {(time.perf_counter() - t0):.3f}s)") if market_data: print(f"样例数据: {market_data[0]}") return market_data except Exception as e: print(f"文件解析错误: {str(e)}") print(f"详细错误: {traceback.format_exc()}") raise ValueError(f"文件解析失败: {str(e)}") async def grid_backtest_with_file( file: UploadFile = File(...), params: str = Form(...) ): print("--- 收到网格回测请求 ---") print(f"原始参数 (JSON): {params}") """使用上传文件进行网格交易回测""" try: print(f"\n=== 收到回测请求 ===") print(f"文件名: {file.filename}") print(f"文件类型: {file.content_type}") # 解析参数 params_dict = json.loads(params) print(f"参数: {json.dumps(params_dict, indent=2, ensure_ascii=False)}") # 读取文件 file_content = await file.read() print(f"文件大小: {len(file_content)} 字节") # --- 新增:保存上传的文件 --- try: # 1. 定义 uploads 目录路径 backend_dir = os.path.dirname(__file__) uploads_dir = os.path.join(backend_dir, 'uploads') # 2. 按当前日期创建子目录 today_str = datetime.now().strftime('%Y-%m-%d') date_specific_dir = os.path.join(uploads_dir, today_str) os.makedirs(date_specific_dir, exist_ok=True) # 3. 构造安全的文件名并保存 timestamp_str = datetime.now().strftime('%H%M%S') safe_filename = os.path.basename(file.filename) # 移除路径部分,防止路径遍历攻击 file_path = os.path.join(date_specific_dir, f"{timestamp_str}_{safe_filename}") with open(file_path, 'wb') as f: f.write(file_content) print(f"✅ 文件已保存至: {file_path}") except Exception as e: print(f"⚠️ 警告:保存上传文件失败: {e}") # --- 文件保存逻辑结束 --- loop = asyncio.get_running_loop() t_parse0 = time.perf_counter() market_data = await loop.run_in_executor( PARSE_EXECUTOR, parse_uploaded_file, file_content, file.filename, True, ) print(f"解析阶段总耗时: {(time.perf_counter() - t_parse0):.3f}s") if not market_data: raise ValueError("解析后的市场数据为空") print(f"市场数据条数: {len(market_data)}") t_bt0 = time.perf_counter() results = await loop.run_in_executor( BACKTEST_EXECUTOR, _grid_backtest_job, params_dict, market_data, ) print(f"回测阶段总耗时: {(time.perf_counter() - t_bt0):.3f}s") print(f"回测完成,交易次数: {results['total_trades']}") print(f"最终权益: {results['final_equity']}") # 生成动态任务ID with _backtest_history_lock: task_id = generate_backtest_id() # 保存到内存历史记录 save_to_memory_history("grid", file.filename or 'uploaded_data', params_dict, results, task_id, market_data) # 数据已经通过 save_to_memory_history 保存到数据库了 return { "id": task_id, "status": "completed", "created_at": str(datetime.now()), "completed_at": str(datetime.now()), "results": results } except Exception as e: print(f"\n=== 回测错误 ===") print(f"错误: {str(e)}") print(f"详细错误:\n{traceback.format_exc()}") # 提供更详细的错误信息 error_detail = f"回测失败: {str(e)}" if "parse" in str(e).lower(): error_detail = f"文件解析失败: {str(e)}" elif "grid" in str(e).lower(): error_detail = f"网格引擎错误: {str(e)}" elif "param" in str(e).lower(): error_detail = f"参数错误: {str(e)}" raise HTTPException(status_code=500, detail=error_detail) async def grid_backtest_simple(request: dict): """网格交易回测(需要提供真实数据)""" try: print(f"\n=== 网格回测请求 ===") print(f"参数: {json.dumps(request, indent=2, ensure_ascii=False)}") # 检查是否提供了市场数据 if 'market_data' not in request or not request['market_data']: raise HTTPException( status_code=400, detail="缺少市场数据。请使用文件上传接口 /api/v1/grid/backtest/file 上传真实数据文件。" ) market_data = request['market_data'] print(f"收到市场数据: {len(market_data)} 条记录") loop = asyncio.get_running_loop() results = await loop.run_in_executor( BACKTEST_EXECUTOR, _grid_backtest_job, request, market_data, ) print(f"回测完成,交易次数: {results['total_trades']}") # 生成动态任务ID with _backtest_history_lock: task_id = generate_backtest_id() # 保存到内存历史记录 save_to_memory_history("grid", request.get('symbol', 'virtual_data'), request, results, task_id, market_data) # 数据已经通过 save_to_memory_history 保存到数据库了 return { "id": task_id, "status": "completed", "created_at": str(datetime.now()), "completed_at": str(datetime.now()), "results": results } except Exception as e: print(f"\n=== 回测错误 ===") print(f"错误: {str(e)}") print(f"详细错误:\n{traceback.format_exc()}") raise HTTPException(status_code=500, detail=f"回测失败: {str(e)}") async def dca_backtest_with_file( file: UploadFile = File(...), params: str = Form(...) ): """使用上传文件进行定投交易回测""" try: print(f"\n=== 定投回测请求 ===") print(f"文件名: {file.filename}") print(f"文件类型: {file.content_type}") # 解析参数 params_dict = json.loads(params) print(f"参数: {json.dumps(params_dict, indent=2, ensure_ascii=False)}") # 读取文件 file_content = await file.read() print(f"文件大小: {len(file_content)} 字节") # --- 新增:保存上传的文件 --- try: # 1. 定义 uploads 目录路径 backend_dir = os.path.dirname(__file__) uploads_dir = os.path.join(backend_dir, 'uploads') # 2. 按当前日期创建子目录 today_str = datetime.now().strftime('%Y-%m-%d') date_specific_dir = os.path.join(uploads_dir, today_str) os.makedirs(date_specific_dir, exist_ok=True) # 3. 构造安全的文件名并保存 timestamp_str = datetime.now().strftime('%H%M%S') safe_filename = os.path.basename(file.filename) # 移除路径部分,防止路径遍历攻击 file_path = os.path.join(date_specific_dir, f"{timestamp_str}_{safe_filename}") with open(file_path, 'wb') as f: f.write(file_content) print(f"✅ 文件已保存至: {file_path}") except Exception as e: print(f"⚠️ 警告:保存上传文件失败: {e}") # --- 文件保存逻辑结束 --- loop = asyncio.get_running_loop() t_parse0 = time.perf_counter() market_data = await loop.run_in_executor( PARSE_EXECUTOR, parse_uploaded_file, file_content, file.filename, False, ) print(f"解析阶段总耗时: {(time.perf_counter() - t_parse0):.3f}s") if not market_data: raise ValueError("解析后的市场数据为空") print(f"市场数据条数: {len(market_data)}") t_bt0 = time.perf_counter() results = await loop.run_in_executor( BACKTEST_EXECUTOR, _dca_backtest_job, params_dict, market_data, ) print(f"回测阶段总耗时: {(time.perf_counter() - t_bt0):.3f}s") print(f"定投回测完成") # 生成动态任务ID with _backtest_history_lock: task_id = generate_backtest_id() # 保存到内存历史记录 save_to_memory_history("dca", file.filename or 'uploaded_dca_data', params_dict, results, task_id, market_data) # 数据已经通过 save_to_memory_history 保存到数据库了 return { "id": task_id, "status": "completed", "created_at": str(datetime.now()), "completed_at": str(datetime.now()), "results": results } except Exception as e: print(f"\n=== 定投回测错误 ===") print(f"错误: {str(e)}") print(f"详细错误:\n{traceback.format_exc()}") raise HTTPException(status_code=500, detail=f"定投回测失败: {str(e)}") class DCABacktestEngine: """定投交易回测引擎""" def __init__(self, params): self.params = params # 处理初始资金:None或0表示无上限,使用足够大的默认值 initial_capital_input = params.get('initial_capital', None) if initial_capital_input is None or initial_capital_input == 0 or initial_capital_input == '': # 无上限:使用足够大的值(1亿) self.initial_capital = 100000000.0 self.is_unlimited_capital = True else: self.initial_capital = float(initial_capital_input) self.is_unlimited_capital = False self.investment_type = params.get('investment_type', 'shares') # 'shares' or 'amount' self.investment_value = float(params.get('investment_value', 100)) # 金额或股数 self.dca_price_type = params.get('dca_price_type', 'close') # 'open' or 'close' self.investment_frequency = params.get('investment_frequency', 'monthly') # daily, weekly, monthly self.day_of_week = int(params.get('day_of_week', 1)) if params.get('day_of_week') is not None else 1 # 默认为周一 self.day_of_month = int(params.get('day_of_month', 1)) if params.get('day_of_month') is not None else 1 # 默认为每月1号 # 日期范围 self.start_date = params.get('start_date', None) self.end_date = params.get('end_date', None) # 手续费 - 兼容前端字段名 self.commission_rate = float(params.get('fee_rate', params.get('commission_rate', 0.0005))) # 默认万分之50 = 0.5% self.min_commission = float(params.get('min_fee', params.get('min_commission', 0.01))) # 默认最低0.01元 capital_desc = "无上限" if self.is_unlimited_capital else f"{self.initial_capital}" print(f"定投参数: 初始资金={capital_desc}, 投资类型={self.investment_type}, 投资数值={self.investment_value}, 频率={self.investment_frequency}") print(f"手续费参数: 费率={self.commission_rate*10000}万分之一, 最低手续费={self.min_commission}元") print(f"回测日期: {self.start_date} 至 {self.end_date}") def run(self, market_data): """执行定投回测""" if not market_data or len(market_data) == 0: raise ValueError("市场数据为空") # 根据日期范围过滤数据 if self.start_date or self.end_date: original_count = len(market_data) filtered_data = [] for row in market_data: row_date = str(row['date']).split(' ')[0] # 只取日期部分 # 检查是否在日期范围内 include_row = True if self.start_date and row_date < self.start_date: include_row = False if self.end_date and row_date > self.end_date: include_row = False if include_row: filtered_data.append(row) market_data = filtered_data print(f"日期过滤: 原始数据{original_count}条 -> 过滤后{len(market_data)}条") if not market_data: raise ValueError(f"指定日期范围 {self.start_date} 至 {self.end_date} 内没有数据") print(f"开始定投回测,数据条数: {len(market_data)}") print(f"价格范围: {market_data[0]['close']} -> {market_data[-1]['close']}") print(f"日期范围: {market_data[0]['date']} -> {market_data[-1]['date']}") # 资金初始化:无上限模式不受现金约束,仅记录投入成本 cash = 0 if self.is_unlimited_capital else self.initial_capital shares = 0 total_fee = 0 total_invested = 0 trades = [] # --- 重构定投策略以支持精确时间点 --- # 1) 转为 DataFrame 并按"交易日"分组,避免每个日历日都做一次全表过滤(性能瓶颈) df = pd.DataFrame(market_data) df['dt'] = pd.to_datetime(df['date'], errors='coerce') df = df[df['dt'].notna()].copy() df = df.sort_values('dt') df['date_only'] = df['dt'].dt.floor('D') grouped = {d: g for d, g in df.groupby('date_only', sort=True)} trading_days = list(grouped.keys()) if not trading_days: raise ValueError('市场数据为空') # 如果每个交易日只有一条记录,则视为日线;否则视为分钟线 is_minute_data = any(len(grouped[d]) > 1 for d in trading_days) last_investment_date = None # 只遍历实际存在的交易日(而不是遍历所有日历日) for current_date in trading_days: should_invest_today = False # 检查今天是否是投资日 if last_investment_date is None: if self.investment_frequency == 'daily': should_invest_today = True elif self.investment_frequency == 'weekly' and current_date.isoweekday() == self.day_of_week: should_invest_today = True elif self.investment_frequency == 'monthly' and current_date.day == self.day_of_month: should_invest_today = True else: if current_date.date() > last_investment_date.date(): if self.investment_frequency == 'daily': should_invest_today = True elif self.investment_frequency == 'weekly' and current_date.isoweekday() == self.day_of_week and (current_date.date() - last_investment_date.date()).days >= 6: should_invest_today = True elif self.investment_frequency == 'monthly' and current_date.day == self.day_of_month and (current_date.month != last_investment_date.month or current_date.year > last_investment_date.year): should_invest_today = True if should_invest_today: day_data = grouped.get(current_date) if not day_data.empty: trade_row = None if is_minute_data: # 分钟数据:查找特定时间点 if self.dca_price_type == 'open': # 开盘价:查找最接近 09:30 的数据 target_dt = pd.Timestamp(current_date.date()).replace(hour=9, minute=30) else: # 'close' # 收盘价:查找最接近 15:00 的数据 target_dt = pd.Timestamp(current_date.date()).replace(hour=15, minute=0) # 找到离目标时间点最近的行(向量化) idx = (day_data['dt'] - target_dt).abs().idxmin() trade_row = day_data.loc[idx] trade_timestamp = trade_row['dt'].to_pydatetime() else: # 日数据:直接使用当天的记录 trade_row = day_data.iloc[0] trade_timestamp = None # 日数据时间在后面设置 if trade_row is not None: if is_minute_data: # 分钟数据:已经定位到精确时间点,使用该行的收盘价 current_price = float(trade_row['close']) # trade_timestamp 已在上面设置 else: # 日数据:根据选择的价格类型获取价格和设置时间 current_price = float(trade_row[self.dca_price_type]) if self.dca_price_type == 'open': # 开盘价:时间为 09:30:00 from datetime import datetime as dt trade_timestamp = dt(current_date.year, current_date.month, current_date.day, 9, 30, 0) else: # 'close' # 收盘价:时间为 15:00:00 from datetime import datetime as dt trade_timestamp = dt(current_date.year, current_date.month, current_date.day, 15, 0, 0) # --- 执行交易逻辑 --- buy_shares = 0 if self.investment_type == 'amount': potential_shares = int(self.investment_value / current_price) buy_shares = (potential_shares // 100) * 100 else: # 'shares' buy_shares = int(self.investment_value) if buy_shares >= 100: cost = buy_shares * current_price fee = max(cost * self.commission_rate, self.min_commission) # 现金约束:仅在有限资金模式下检查 if self.is_unlimited_capital or cash >= cost + fee: if not self.is_unlimited_capital: cash -= (cost + fee) shares += buy_shares total_fee += fee total_invested += cost # 格式化时间戳为 ISO 格式字符串,确保包含完整的时间信息 date_str = trade_timestamp.strftime('%Y-%m-%dT%H:%M:%S') trade = { "date": date_str, # ISO 格式:2023-01-15T09:30:00 "type": "定投买入", "price": round(current_price, 2), "shares": buy_shares, "amount": round(cost, 2), "fee": round(fee, 2), "cash_balance": round(cash, 2) if not self.is_unlimited_capital else None, "position_value": round(shares * current_price, 2), "current_shares": shares, "total_value": round((cash if not self.is_unlimited_capital else 0) + shares * current_price, 2), "base_price": round(current_price, 4), "cumulative_invested": round(total_invested, 2) } trades.append(trade) if len(trades) <= 5: print(f"定投: {date_str} 价格={current_price:.2f} 数量={buy_shares}") last_investment_date = trade_timestamp # --- 指标计算 --- last_price = float(market_data[-1]['close']) if self.is_unlimited_capital: # 无上限模式:现金不纳入权益,但手续费应从最终金额中扣除 final_value = shares * last_price - total_fee else: final_value = cash + shares * last_price # 投入资本基准:无上限模式使用累计投入,有限模式使用初始资金 capital_base = total_invested if self.is_unlimited_capital else self.initial_capital total_profit = final_value - capital_base total_return = (total_profit / capital_base) * 100 if capital_base > 0 else 0 # --- 详细指标计算:最大回撤、盈亏平衡点时长等 --- equity_curve = [] peak_equity = capital_base if capital_base > 0 else 0 max_drawdown = 0 breakeven_duration = 0 max_breakeven_duration = 0 underwater_start_date = None # 为了计算每日权益,我们需要一个每日投资额/持仓股数的映射 investments_by_date = {t['date'].split('T')[0]: t.get('cumulative_invested', 0) for t in trades} shares_by_date = {t['date'].split('T')[0]: int(t.get('current_shares', 0) or 0) for t in trades} daily_total_invested = 0 last_known_investment = 0 daily_shares = 0 last_known_shares = 0 # 新增:用于盈利/亏损天数、连续天数、资金利用率 _daily_equity_dca: list[float] = [] _daily_invested_dca: list[float] = [] for row in market_data: current_date_str = str(row['date']).split(' ')[0] current_date = datetime.fromisoformat(current_date_str) price = float(row['close']) # 更新当日的总投资额 if current_date_str in investments_by_date: daily_total_invested = investments_by_date[current_date_str] last_known_investment = daily_total_invested else: daily_total_invested = last_known_investment if current_date_str in shares_by_date: daily_shares = shares_by_date[current_date_str] last_known_shares = daily_shares else: daily_shares = last_known_shares if self.is_unlimited_capital: # 权益=持仓市值-累计投入(现金无限制,不计入权益) current_equity = daily_shares * price - daily_total_invested else: current_equity = self.initial_capital - daily_total_invested + daily_shares * price equity_curve.append(current_equity) _daily_equity_dca.append(float(current_equity)) _daily_invested_dca.append(float(daily_total_invested)) # 计算最大回撤 if current_equity > peak_equity: peak_equity = current_equity drawdown = (peak_equity - current_equity) / peak_equity if peak_equity > 0 else 0 if drawdown > max_drawdown: max_drawdown = drawdown # 计算盈亏平衡点时长 if self.is_unlimited_capital: is_underwater = current_equity < 0 else: is_underwater = current_equity < daily_total_invested if is_underwater and underwater_start_date is None: # 刚进入水下 underwater_start_date = current_date elif not is_underwater and underwater_start_date is not None: # 刚浮出水面 duration = (current_date - underwater_start_date).days if duration > max_breakeven_duration: max_breakeven_duration = duration underwater_start_date = None # 如果结束时仍在水下,计算最后一段水下时长 if underwater_start_date is not None: end_date_dt = datetime.fromisoformat(str(market_data[-1]['date']).split(' ')[0]) duration = (end_date_dt - underwater_start_date).days if duration > max_breakeven_duration: max_breakeven_duration = duration import math def _finite(v: float, default: float = 0.0) -> float: try: fv = float(v) return fv if math.isfinite(fv) else float(default) except Exception: return float(default) # 年化收益率 days = (datetime.fromisoformat(self.end_date) - datetime.fromisoformat(self.start_date)).days if self.start_date and self.end_date else len(market_data) years = max(days / 365.25, 1/365.25) annual_return = ((1 + total_return / 100) ** (1 / years) - 1) * 100 if total_return is not None else 0 annual_return = _finite(annual_return, 0.0) # 波动率 (年化标准差) returns = pd.Series(equity_curve).pct_change().dropna() try: volatility = float(returns.std()) * (252**0.5) if not returns.empty else 0.0 # 假设每年252个交易日 except Exception: volatility = 0.0 volatility = _finite(volatility, 0.0) # 夏普比率 (假设无风险利率为0) try: sharpe_ratio = float((annual_return / 100) / volatility) if volatility > 0 else 0.0 except Exception: sharpe_ratio = 0.0 sharpe_ratio = _finite(sharpe_ratio, 0.0) # 胜率 (定投场景下,胜率定义为最终价格高于平均成本) win_rate = 100 if shares > 0 and last_price > (total_invested / shares) else 0 avg_cost = total_invested / shares if shares > 0 else 0 profit_days = 0 loss_days = 0 max_profit_streak = 0 max_loss_streak = 0 _cur_profit = 0 _cur_loss = 0 if len(_daily_equity_dca) >= 2: for i in range(1, len(_daily_equity_dca)): diff = _daily_equity_dca[i] - _daily_equity_dca[i - 1] if diff > 0: profit_days += 1 _cur_profit += 1 _cur_loss = 0 elif diff < 0: loss_days += 1 _cur_loss += 1 _cur_profit = 0 else: _cur_profit = 0 _cur_loss = 0 max_profit_streak = max(max_profit_streak, _cur_profit) max_loss_streak = max(max_loss_streak, _cur_loss) capital_utilization = 0.0 try: if self.is_unlimited_capital: denom = float(total_invested) if float(total_invested) > 0 else 0.0 if denom > 0 and len(_daily_invested_dca) > 0: capital_utilization = (sum(_daily_invested_dca) / len(_daily_invested_dca)) / denom * 100 else: denom = float(self.initial_capital) if float(self.initial_capital) > 0 else 0.0 if denom > 0 and len(_daily_invested_dca) > 0: capital_utilization = (sum(_daily_invested_dca) / len(_daily_invested_dca)) / denom * 100 except Exception: capital_utilization = 0.0 # --- 超额收益率计算 --- # 基准:定期定额等权基准(每期投入相同金额,最后一期以末期价格计算) # 假设每期投入金额为平均投入额 if len(trades) > 0: # 计算每期平均投入金额 avg_investment_per_period = total_invested / len(trades) # 基准策略:每期投入相同金额,购买相同数量的股票 # 基准最终权益 = 每期投入金额 * 期数 + (每期投入金额 / 首期价格) * 期数 * (末期价格 - 首期价格) first_trade_price = float(trades[0]['price']) if trades else last_price # 简化计算:基准权益 = 总投入 + 总投入 * (末期价格 / 首期价格 - 1) benchmark_return = ((last_price / first_trade_price) - 1) * 100 if first_trade_price > 0 else 0 benchmark_return = _finite(benchmark_return, 0.0) # 超额收益率 = 策略收益率 - 基准收益率 excess_return = _finite(total_return - benchmark_return, 0.0) else: excess_return = 0 benchmark_return = 0 print(f"定投回测完成: 投资{len(trades)}次, 最终权益={final_value:.2f}, 收益率={total_return:.2f}%, 超额收益率={excess_return:.2f}%") total_return = _finite(total_return, 0.0) max_drawdown = _finite(max_drawdown, 0.0) return { "final_value": round(_finite(final_value, 0.0), 2), # 统一为final_value "total_profit": round(_finite(total_profit, 0.0), 2), "total_return": round(_finite(total_return, 0.0), 2), "annual_return": round(_finite(annual_return, 0.0), 2), "max_drawdown": round(_finite(max_drawdown * 100, 0.0), 2), "volatility": round(_finite(volatility, 0.0), 4), "sharpe_ratio": round(_finite(sharpe_ratio, 0.0), 4), "win_rate": round(_finite(win_rate, 0.0), 2), "profit_days": int(profit_days), "loss_days": int(loss_days), "max_profit_streak": int(max_profit_streak), "max_loss_streak": int(max_loss_streak), "capital_utilization": round(float(capital_utilization), 2), "total_periods": len(trades), # 对应总投入期数 "total_invested": round(total_invested, 2), "average_cost": round(avg_cost, 2), "final_shares": shares, "total_fee": round(total_fee, 2), "investment_records": trades, # 统一为investment_records # --- 其他前端可能需要的字段 --- "excess_return": round(_finite(excess_return, 0.0), 2), # 超额收益率 = 策略收益率 - 基准收益率 "benchmark_return": round(_finite(benchmark_return, 0.0), 2), # 基准收益率(首期价格到末期价格) "breakeven_duration": max_breakeven_duration, # 盈亏平衡点时长 "return_drawdown_ratio": round(total_return / (max_drawdown * 100), 4) if max_drawdown > 0 else 0, "summary": { "data_points": len(market_data), "trading_days": days } } async def get_backtest_history( page: int = 1, page_size: int = 20, strategy_types: str = None ): """获取回测历史列表""" try: with _backtest_history_lock: filtered_history = list(backtest_history) if strategy_types: # 支持多个策略类型,用逗号分隔 strategy_list = [s.strip() for s in strategy_types.split(',')] filtered_history = [record for record in filtered_history if record['strategy_type'] in strategy_list] # 分页 start_idx = (page - 1) * page_size end_idx = start_idx + page_size paginated_records = filtered_history[start_idx:end_idx] print(f"历史记录API被调用,返回 {len(paginated_records)} 条记录") return paginated_records except Exception as e: print(f"获取历史记录失败: {e}") return [] async def get_statistics_summary(): """获取统计摘要""" try: with _backtest_history_lock: snapshot = list(backtest_history) total_count = len(snapshot) grid_count = len([r for r in snapshot if r['strategy_type'] == 'grid']) dca_count = len([r for r in snapshot if r['strategy_type'] == 'dca']) completed_count = len([r for r in snapshot if r['status'] == 'completed']) # 计算平均收益率 total_return_sum = 0 count = 0 for record in snapshot: if record['results'] and 'total_return' in record['results']: total_return_sum += record['results']['total_return'] count += 1 avg_return = total_return_sum / count if count > 0 else 0 print(f"统计API被调用,当前历史记录数: {total_count}") return { "total_backtests": total_count, "grid_backtests": grid_count, "dca_backtests": dca_count, "completed_backtests": completed_count, "average_return": f"{avg_return:.2f}%", "success_rate": f"{(completed_count/total_count*100 if total_count > 0 else 0):.2f}%" } except Exception as e: print(f"获取统计数据失败: {e}") return { "total_backtests": 0, "grid_backtests": 0, "dca_backtests": 0, "completed_backtests": 0, "average_return": "0.00%", "success_rate": "0.00%" } # 辅助函数:翻译和格式化 def _extract_stock_info_from_data(record): """从记录中提取股票信息(名称和代码)""" # 先尝试从市场数据中读取 results = record.get('results', {}) trade_records = results.get('trade_records', []) stock_name = None stock_code = None # 检查交易记录中是否有标的信息 if trade_records: first_trade = trade_records[0] stock_name = first_trade.get('stock_name') or first_trade.get('symbol_name') stock_code = first_trade.get('stock_code') or first_trade.get('symbol_code') # 如果交易记录中没有,尝试从参数中读取 if not stock_name or not stock_code: params = record.get('parameters', {}) market_data = params.get('market_data', []) if market_data and len(market_data) > 0: first_data = market_data[0] stock_name = stock_name or first_data.get('stock_name') or first_data.get('name') or first_data.get('symbol_name') stock_code = stock_code or first_data.get('stock_code') or first_data.get('code') or first_data.get('symbol_code') or first_data.get('symbol') # 如果仍然没有,尝试从文件名解析 if not stock_name or not stock_code: symbol_str = record.get('symbol', '') if '_' in symbol_str and '.' in symbol_str: # 文件名格式:sz_000001_5min.json parts = symbol_str.split('_') if len(parts) >= 2: parsed_code = parts[1] # 000001 stock_code = stock_code or parsed_code stock_name = stock_name or _get_stock_name_by_code(parsed_code) return stock_name or '未知', stock_code or '未知' def _extract_stock_name(record): """提取股票名称""" name, code = _extract_stock_info_from_data(record) return name def _extract_stock_code(record): """提取股票代码""" name, code = _extract_stock_info_from_data(record) return code def _get_stock_name_by_code(code): """根据股票代码获取名称(简化版)""" stock_names = { '000001': '平安银行', '000002': '万科A', '000858': '五粮液', '600036': '招商银行', '600519': '贵州茅台', '000858': '五粮液' } return stock_names.get(code, f'股票{code}') def _translate_base_price_type(price_type): """翻译基准价类型""" translations = { 'open': '开盘价', 'close': '收盘价', 'high': '最高价', 'low': '最低价', 'custom': '自定义价格' } return translations.get(price_type, price_type) def _translate_change_type(change_type): """翻译变动类型""" translations = { 'percentage': '百分比', 'absolute': '绝对值' } return translations.get(change_type, change_type) def _translate_trigger_mode(trigger_mode): """翻译触发模式""" translations = { 'direct': '直接触发', 'pullback': '回落触发', 'rebound': '反弹触发' } return translations.get(trigger_mode, trigger_mode) def _translate_order_type(order_type): """翻译委托类型""" translations = { 'shares': '按股数', 'amount': '按金额' } return translations.get(order_type, order_type) def _translate_price_type(price_type): """翻译价格类型""" translations = { 'market': '市价单', 'limit': '限价单' } return translations.get(price_type, price_type) def _translate_position_limit_type(limit_type): """翻译持仓限制类型""" translations = { 'shares': '按股数', 'amount': '按金额' } return translations.get(limit_type, limit_type) def _translate_fee_type(fee_type): """翻译手续费类型""" translations = { 'percentage': '按比例', 'fixed': '固定金额' } return translations.get(fee_type, fee_type) def _translate_investment_frequency(frequency): """翻译定投频率""" translations = { 'daily': '日度', 'weekly': '周度', 'monthly': '月度' } return translations.get(frequency, frequency) def _format_fee_rate(fee_rate): """格式化手续费率为万分比符号""" rate_in_bp = fee_rate * 10000 # 转为万分比 if rate_in_bp == int(rate_in_bp): return f"{int(rate_in_bp)}‱" # 万分比符号 else: return f"{rate_in_bp:.1f}‱" async def export_backtest_record( record_id: str, format: str = "excel" ): """导出回测结果 - 支持Excel、PDF、Word格式""" try: # 查找记录 record = None for r in backtest_history: if r['id'] == record_id: record = r break if not record: raise HTTPException(status_code=404, detail="记录不存在") if not record.get('results'): raise HTTPException(status_code=400, detail="无回测结果可导出") # 获取交易记录 results = record['results'] trade_records = results.get('trade_records', []) if not trade_records: raise HTTPException(status_code=400, detail="无交易记录可导出") # 导出数据处理 import io # 根据格式选择导出方式 if format == "pdf": # PDF导出功能 - 横向布局 from reportlab.lib.pagesizes import letter, A4, landscape from reportlab.platypus import SimpleDocTemplate, Table, TableStyle, Paragraph, Spacer, PageBreak from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle from reportlab.lib import colors from reportlab.lib.units import inch from reportlab.pdfbase.ttfonts import TTFont from reportlab.pdfbase import pdfmetrics # 注册中文字体(使用系统字体) try: pdfmetrics.registerFont(TTFont('SimSun', 'C:/Windows/Fonts/simsun.ttc')) font_name = 'SimSun' except: # 如果找不到中文字体,使用默认字体 font_name = 'Helvetica' pdf_buffer = io.BytesIO() # 使用横向布局 doc = SimpleDocTemplate(pdf_buffer, pagesize=landscape(A4), leftMargin=0.5*inch, rightMargin=0.5*inch, topMargin=0.5*inch, bottomMargin=0.5*inch) story = [] # 样式设置 styles = getSampleStyleSheet() title_style = ParagraphStyle('CustomTitle', parent=styles['Heading1'], fontSize=18, fontName=font_name, alignment=1) section_style = ParagraphStyle('SectionTitle', parent=styles['Heading2'], fontSize=14, fontName=font_name) normal_style = ParagraphStyle('CustomNormal', parent=styles['Normal'], fontSize=9, fontName=font_name) # 添加标题 story.append(Paragraph(f'回测报告 - {record_id}', title_style)) story.append(Spacer(1, 16)) # 1. 统计信息部分 story.append(Paragraph('统计信息', section_style)) story.append(Spacer(1, 8)) if record['strategy_type'] == 'grid': stats_data = [ ['最终权益', f"¥{results.get('final_equity', 0):,.2f}"], ['总收益', f"¥{results.get('total_profit', 0):,.2f}"], ['总收益率', f"{results.get('total_return', 0):.2f}%"], ['年化收益率', f"{results.get('annual_return', 0):.2f}%"], ['总交易次数', str(results.get('total_trades', 0))], ['胜率', f"{results.get('win_rate', 0):.2f}%"], ['最大回撤', f"{results.get('max_drawdown', 0):.2f}%"], ['总手续费', f"¥{results.get('total_fee', 0):,.2f}"] ] else: # dca stats_data = [ ['最终权益', f"¥{results.get('final_equity', 0):,.2f}"], ['总收益', f"¥{results.get('total_profit', 0):,.2f}"], ['总收益率', f"{results.get('total_return', 0):.2f}%"], ['年化收益率', f"{results.get('annual_return', 0):.2f}%"], ['投资次数', str(results.get('total_investments', 0))], ['总投入', f"¥{results.get('total_invested', 0):,.2f}"], ['平均成本', f"¥{results.get('average_cost', 0):.2f}"], ['总手续费', f"¥{results.get('total_fee', 0):,.2f}"] ] stats_table = Table(stats_data, colWidths=[2*inch, 2*inch]) stats_table.setStyle(TableStyle([ ('BACKGROUND', (0, 0), (0, -1), colors.lightblue), ('TEXTCOLOR', (0, 0), (-1, -1), colors.black), ('ALIGN', (0, 0), (-1, -1), 'LEFT'), ('FONTNAME', (0, 0), (-1, -1), font_name), ('FONTSIZE', (0, 0), (-1, -1), 9), ('BOTTOMPADDING', (0, 0), (-1, -1), 4), ('GRID', (0, 0), (-1, -1), 1, colors.black) ])) story.append(stats_table) story.append(Spacer(1, 16)) # 2. 回测参数部分 story.append(Paragraph('回测参数', section_style)) story.append(Spacer(1, 8)) params = record['parameters'] if record['strategy_type'] == 'grid': param_data = [ ['标的名称', _extract_stock_name(record)], ['标的代码', _extract_stock_code(record)], ['初始资金', f"¥{params.get('initial_capital', 0):,.2f}"], ['初始股数', f"{params.get('initial_shares', 0)}股"], ['上涨触发', f"{params.get('rise_trigger', 0):.2f}%"], ['下跌触发', f"{params.get('fall_trigger', 0):.2f}%"], ['买入金额', f"{params.get('buy_amount', 0)}股"], ['手续费率', _format_fee_rate(float(params.get('fee_rate', 0)))] ] else: # dca param_data = [ ['标的名称', _extract_stock_name(record)], ['标的代码', _extract_stock_code(record)], ['初始资金', f"¥{params.get('initial_capital', 0):,.2f}"], ['定投金额', f"¥{params.get('investment_amount', 0):,.2f}"], ['定投频率', _translate_investment_frequency(params.get('investment_frequency', 'monthly'))], ['手续费率', _format_fee_rate(float(params.get('fee_rate', 0)))] ] param_table = Table(param_data, colWidths=[2*inch, 2*inch]) param_table.setStyle(TableStyle([ ('BACKGROUND', (0, 0), (0, -1), colors.lightgreen), ('TEXTCOLOR', (0, 0), (-1, -1), colors.black), ('ALIGN', (0, 0), (-1, -1), 'LEFT'), ('FONTNAME', (0, 0), (-1, -1), font_name), ('FONTSIZE', (0, 0), (-1, -1), 9), ('BOTTOMPADDING', (0, 0), (-1, -1), 4), ('GRID', (0, 0), (-1, -1), 1, colors.black) ])) story.append(param_table) story.append(PageBreak()) # 3. 交易记录部分 - 新页面,横向布局优势 story.append(Paragraph('交易记录明细', section_style)) story.append(Spacer(1, 8)) # 完整的交易记录表头(横向布局可容纳更多列) if record['strategy_type'] == 'grid': trade_data = [['序号', '日期', '时间', '委托', '价格', '数量', '手续费', '现金余额', '持仓市值', '持仓股数', '总资产', '基准价', '触发条件']] for index, trade in enumerate(trade_records, 1): date_str = trade.get('date', '') date_part = date_str.split(' ')[0] if ' ' in date_str else date_str time_part = date_str.split(' ')[1] if ' ' in date_str else '15:00:00' trade_data.append([ str(index), date_part, time_part, trade.get('type', ''), f"¥{trade.get('price', 0):.2f}", str(trade.get('shares', 0)), f"¥{trade.get('fee', 0):.2f}", f"¥{trade.get('cash_balance', 0):,.0f}", f"¥{trade.get('position_value', 0):,.0f}", str(trade.get('current_shares', 0)), f"¥{trade.get('total_value', 0):,.0f}", f"¥{trade.get('base_price', 0):.2f}", trade.get('trigger_reason', '')[:10] # 截断长文本 ]) col_widths = [0.4*inch, 0.8*inch, 0.6*inch, 0.5*inch, 0.6*inch, 0.5*inch, 0.6*inch, 0.8*inch, 0.8*inch, 0.6*inch, 0.8*inch, 0.6*inch, 1.2*inch] else: # dca trade_data = [['序号', '日期', '时间', '委托', '价格', '数量', '手续费', '现金余额', '持仓市值', '持仓股数', '总资产', '累计投入']] for index, trade in enumerate(trade_records, 1): date_str = trade.get('date', '') date_part = date_str.split(' ')[0] if ' ' in date_str else date_str time_part = date_str.split(' ')[1] if ' ' in date_str else '15:00:00' trade_data.append([ str(index), date_part, time_part, trade.get('type', ''), f"¥{trade.get('price', 0):.2f}", str(trade.get('shares', 0)), f"¥{trade.get('fee', 0):.2f}", f"¥{trade.get('cash_balance', 0):,.0f}", f"¥{trade.get('position_value', 0):,.0f}", str(trade.get('current_shares', 0)), f"¥{trade.get('total_value', 0):,.0f}", f"¥{trade.get('cumulative_invested', 0):,.0f}" ]) col_widths = [0.4*inch, 0.8*inch, 0.6*inch, 0.5*inch, 0.6*inch, 0.5*inch, 0.6*inch, 0.8*inch, 0.8*inch, 0.6*inch, 0.8*inch, 0.8*inch] trade_table = Table(trade_data, colWidths=col_widths) trade_table.setStyle(TableStyle([ ('BACKGROUND', (0, 0), (-1, 0), colors.grey), ('TEXTCOLOR', (0, 0), (-1, 0), colors.whitesmoke), ('ALIGN', (0, 0), (-1, -1), 'CENTER'), ('FONTNAME', (0, 0), (-1, -1), font_name), ('FONTSIZE', (0, 0), (-1, -1), 9), ('BOTTOMPADDING', (0, 0), (-1, -1), 6), ('GRID', (0, 0), (-1, -1), 1, colors.black) ])) story.append(trade_table) doc.build(story) pdf_buffer.seek(0) from fastapi.responses import Response return Response( content=pdf_buffer.getvalue(), media_type='application/pdf', headers={ "Content-Disposition": f"attachment; filename=backtest_{record_id}.pdf" } ) elif format == "word": # Word导出功能 from docx import Document from docx.shared import Inches from docx.enum.text import WD_ALIGN_PARAGRAPH doc = Document() # 添加标题 title = doc.add_heading(f'回测报告 - {record_id}', 0) title.alignment = WD_ALIGN_PARAGRAPH.CENTER # 添加基本信息 doc.add_heading('基本信息', level=1) basic_table = doc.add_table(rows=1, cols=2) basic_table.style = 'Table Grid' basic_info = [ ['标的名称', _extract_stock_name(record)], ['标的代码', _extract_stock_code(record)], ['策略类型', '网格交易' if record['strategy_type'] == 'grid' else '定投交易'], ['初始资金', f"¥{record['initial_capital']:,.2f}"], ['回测日期', f"{record.get('start_date', 'N/A')} 至 {record.get('end_date', 'N/A')}"], ] for info in basic_info: row_cells = basic_table.add_row().cells row_cells[0].text = info[0] row_cells[1].text = str(info[1]) # 添加交易记录 doc.add_heading('交易记录', level=1) trade_table = doc.add_table(rows=1, cols=7) trade_table.style = 'Table Grid' # 表头 hdr_cells = trade_table.rows[0].cells hdr_cells[0].text = '序号' hdr_cells[1].text = '日期' hdr_cells[2].text = '委托' hdr_cells[3].text = '价格' hdr_cells[4].text = '数量' hdr_cells[5].text = '手续费' hdr_cells[6].text = '资产' # 数据行 for index, trade in enumerate(trade_records[:50], 1): # 限制显示50条 row_cells = trade_table.add_row().cells date_str = trade.get('date', '') date_part = date_str.split(' ')[0] if ' ' in date_str else date_str time_part = date_str.split(' ')[1] if ' ' in date_str else '15:00:00' row_cells[0].text = str(index) row_cells[1].text = date_part row_cells[2].text = trade.get('type', '') row_cells[3].text = f"¥{trade.get('price', 0):.2f}" row_cells[4].text = str(trade.get('shares', 0)) row_cells[5].text = f"¥{trade.get('fee', 0):.2f}" row_cells[6].text = f"¥{trade.get('total_value', 0):,.2f}" # 保存到内存 word_buffer = io.BytesIO() doc.save(word_buffer) word_buffer.seek(0) from fastapi.responses import Response return Response( content=word_buffer.getvalue(), media_type='application/vnd.openxmlformats-officedocument.wordprocessingml.document', headers={ "Content-Disposition": f"attachment; filename=backtest_{record_id}.docx" } ) elif format == "excel": # 使用pandas创建Excel import pandas as pd import io # 创建 DataFrame if record['strategy_type'] == 'grid': data_list = [] for index, trade in enumerate(trade_records, 1): date_str = trade.get('date', '') date_part = date_str.split(' ')[0] if ' ' in date_str else date_str time_part = date_str.split(' ')[1] if ' ' in date_str else '15:00:00' data_list.append({ '序号': index, '日期': date_part, '时间': time_part, '委托': trade.get('type', ''), '价格': trade.get('price', 0), '数量': trade.get('shares', 0), '手续费': trade.get('fee', 0), '现金余额': trade.get('cash_balance', 0), '当前持仓市值': trade.get('position_value', 0), '当前持仓股数': trade.get('current_shares', 0), '资产': trade.get('total_value', 0), '最新基准价': trade.get('base_price', 0), '触发条件': trade.get('trigger_reason', trade.get('trigger_condition', '')) }) df = pd.DataFrame(data_list) else: # dca data_list = [] for index, trade in enumerate(trade_records, 1): date_str = trade.get('date', '') date_part = date_str.split(' ')[0] if ' ' in date_str else date_str time_part = date_str.split(' ')[1] if ' ' in date_str else '09:30:00' data_list.append({ '序号': index, '日期': date_part, '时间': time_part, '委托': trade.get('type', ''), '价格': trade.get('price', 0), '数量': trade.get('shares', 0), '手续费': trade.get('fee', 0), '现金余额': trade.get('cash_balance', 0), '当前持仓市值': trade.get('position_value', 0), '当前持仓股数': trade.get('current_shares', 0), '资产': trade.get('total_value', 0), '累计投入': trade.get('cumulative_invested', 0) }) df = pd.DataFrame(data_list) # 创建Excel文件 excel_buffer = io.BytesIO() with pd.ExcelWriter(excel_buffer, engine='openpyxl') as writer: df.to_excel(writer, sheet_name='交\u6613\u8bb0\u5f55', index=False) # 添加统计信息工作表 if record['strategy_type'] == 'grid': stats_data = { '指标': [ '最终权益', '总收益', '总收益率', '年化收益率', '总交易次数', '买入次数', '卖出次数', '胜率', '最大回撤', '总手续费', '最大投入收益率', '最大持仓市值', '最小持仓市值', '最大持仓股数', '最小持仓股数' ], '数值': [ results.get('final_equity', 0), results.get('total_profit', 0), f"{results.get('total_return', 0):.2f}%", f"{results.get('annual_return', 0):.2f}%", results.get('total_trades', 0), results.get('buy_trades', 0), results.get('sell_trades', 0), f"{results.get('win_rate', 0):.2f}%", f"{results.get('max_drawdown', 0):.2f}%", results.get('total_fee', 0), f"{results.get('max_invested_return', 0):.2f}%", results.get('max_position_value', 0), results.get('min_position_value', 0), results.get('max_position_shares', 0), results.get('min_position_shares', 0) ] } else: # dca stats_data = { '指标': [ '最终权益', '总收益', '总收益率', '年化收益率', '投资次数', '总投入金额', '平均成本', '最终股数', '总手续费' ], '数值': [ results.get('final_equity', 0), results.get('total_profit', 0), f"{results.get('total_return', 0):.2f}%", f"{results.get('annual_return', 0):.2f}%", results.get('total_investments', 0), results.get('total_invested', 0), results.get('average_cost', 0), results.get('final_shares', 0), results.get('total_fee', 0) ] } stats_df = pd.DataFrame(stats_data) stats_df.to_excel(writer, sheet_name='统计信息', index=False) # 添加参数信息工作表 params = record['parameters'] if record['strategy_type'] == 'grid': param_data = { '参数名称': [ # 基本信息 '标的名称', '标的代码', '回测开始日期', '回测结束日期', # 资金参数 '初始资金', '初始股数', # 触发参数 '触发价格最小值', '触发价格最大值', '上涨触发', '下跌触发', # 基准价参数 '基准价类型', '初始基准价', '基准价更新方式', # 交易参数 '变动类型', '触发模式', '委托类型', '买入金额', '卖出金额', '委托价格类型', # 持仓限制 '持仓限制类型', '最大持仓', '最小持仓', # 手续费参数 '手续费类型', '手续费率', '最低手续费(元)', '双边手续费' ], '参数值': [ # 基本信息 - 从上传数据中读取标的名称和代码 _extract_stock_name(record), _extract_stock_code(record), record.get('start_date', '未设置'), record.get('end_date', '未设置'), # 资金参数 f"¥{params.get('initial_capital', 0):,.2f}", f"{params.get('initial_shares', 0)}股", # 触发参数 f"¥{params.get('trigger_price_min', 0):.2f}", f"¥{params.get('trigger_price_max', 0):.2f}", f"{params.get('rise_trigger', 0):.2f}%", f"{params.get('fall_trigger', 0):.2f}%", # 基准价参数 _translate_base_price_type(params.get('base_price_type', 'open')), f"¥{params.get('custom_base_price', 0):.2f}" if params.get('custom_base_price') else '自动', '成交价更新', # 默认方式 # 交易参数 _translate_change_type(params.get('change_type', 'percentage')), _translate_trigger_mode(params.get('trigger_mode', 'direct')), _translate_order_type(params.get('order_type', 'shares')), f"{params.get('buy_amount', 0)}股" if params.get('order_type') == 'shares' else f"¥{params.get('buy_amount', 0)}", f"{params.get('sell_amount', 0)}股" if params.get('order_type') == 'shares' else f"¥{params.get('sell_amount', 0)}", _translate_price_type(params.get('order_price_type', 'market')), # 持仓限制 _translate_position_limit_type(params.get('position_limit_type', 'shares')) if params.get('position_limit_enabled', True) else '不限制', f"{params.get('max_position', 0)}" if params.get('position_limit_enabled', True) and params.get('max_position') else '无限制', f"{params.get('min_position', 0)}" if params.get('position_limit_enabled', True) and params.get('min_position') else '无限制', # 手续费参数 _translate_fee_type(params.get('fee_type', 'percentage')), _format_fee_rate(float(params.get('fee_rate', 0))), f"¥{params.get('min_fee', 5):.2f}", '是' if params.get('fee_bilateral', True) else '否' ] } else: # dca param_data = { '参数名称': [ # 基本信息 '标的名称', '标的代码', '回测开始日期', '回测结束日期', # 资金参数 '初始资金', # 定投参数 '定投金额', '定投频率', '定投日(周)', '定投日(月)', # 手续费参数 '手续费类型', '手续费率', '最低手续费(元)' ], '参数值': [ # 基本信息 - 从上传数据中读取标的名称和代码 _extract_stock_name(record), _extract_stock_code(record), record.get('start_date', '未设置'), record.get('end_date', '未设置'), # 资金参数 f"¥{params.get('initial_capital', 0):,.2f}", # 定投参数 f"¥{params.get('investment_amount', 0):,.2f}", _translate_investment_frequency(params.get('investment_frequency', 'monthly')), f"周{params.get('day_of_week', '')}" if params.get('day_of_week') else '未设置', f"{params.get('day_of_month', '')}日" if params.get('day_of_month') else '未设置', # 手续费参数 '按比例', _format_fee_rate(float(params.get('fee_rate', 0))), f"¥{params.get('min_fee', 5):.2f}" ] } param_df = pd.DataFrame(param_data) param_df.to_excel(writer, sheet_name='回测参数', index=False) excel_buffer.seek(0) from fastapi.responses import Response return Response( content=excel_buffer.getvalue(), media_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', headers={ "Content-Disposition": f"attachment; filename=backtest_{record_id}.xlsx" } ) else: raise HTTPException(status_code=400, detail="不支持的导出格式") except HTTPException: raise except Exception as e: print(f"导出错误: {e}") raise HTTPException(status_code=500, detail=f"导出失败: {str(e)}") async def delete_backtest_record(record_id: str): """删除历史记录(从数据库和内存中)""" global backtest_history try: # 从数据库删除(持久化) db_deleted = False if HAS_DATABASE: db_deleted = delete_backtest_from_database(record_id) if db_deleted: print(f"✅ 记录 {record_id} 已从数据库删除") else: print(f"⚠️ 记录 {record_id} 在数据库中不存在或删除失败") # 从内存删除 original_count = len(backtest_history) backtest_history = [record for record in backtest_history if record['id'] != record_id] memory_deleted = len(backtest_history) < original_count if memory_deleted: print(f"✅ 记录 {record_id} 已从内存删除") if db_deleted or memory_deleted: return {"message": "删除成功"} else: return {"error": "记录不存在"} except Exception as e: print(f"❌ 删除记录失败: {e}") return {"error": f"删除失败: {str(e)}"} async def batch_delete_backtest_records(request: dict): """批量删除历史记录""" global backtest_history try: record_ids = request.get('record_ids', []) if not record_ids: return {"error": "请提供要删除的记录ID列表"} deleted_count = 0 failed_ids = [] for record_id in record_ids: try: # 从数据库删除 db_deleted = False if HAS_DATABASE: db_deleted = delete_backtest_from_database(record_id) # 从内存删除 original_count = len(backtest_history) backtest_history = [record for record in backtest_history if record['id'] != record_id] memory_deleted = len(backtest_history) < original_count if db_deleted or memory_deleted: deleted_count += 1 print(f"✅ 批量删除: 记录 {record_id} 已删除") else: failed_ids.append(record_id) print(f"⚠️ 批量删除: 记录 {record_id} 删除失败或不存在") except Exception as e: failed_ids.append(record_id) print(f"❌ 批量删除: 记录 {record_id} 删除失败: {e}") result = { "message": f"批量删除完成", "deleted_count": deleted_count, "total_count": len(record_ids), "success_rate": f"{(deleted_count / len(record_ids) * 100):.1f}%" } if failed_ids: result["failed_ids"] = failed_ids result["warning"] = f"{len(failed_ids)} 条记录删除失败" return result except Exception as e: print(f"❌ 批量删除失败: {e}") return {"error": f"批量删除失败: {str(e)}"} # ==================== 数据下载API ==================== download_tasks = {} download_task_counter = 0 def _get_download_history_path() -> str: backend_dir = os.path.dirname(__file__) cache_dir = os.path.join(backend_dir, 'cache') os.makedirs(cache_dir, exist_ok=True) return os.path.join(cache_dir, 'download_tasks.json') def _serialize_download_task(task: dict) -> dict: if task is None: return {} allowed_keys = [ 'id', 'symbol', 'stock_name', 'data_type', 'data_provider', 'security_type', 'start_date', 'end_date', 'effective_start_date', 'effective_end_date', 'adjust_flag', 'save_format', 'limit', 'order', 'columns', 'exchange_code', 'ticker', 'data_range_start', 'data_range_end', 'file_size_bytes', 'status', 'stage', 'progress', 'message', 'created_at', 'started_at', 'completed_at', 'data_count', 'file_path', 'cache_key', 'cached', 'adjustments', 'error', ] out = {k: task.get(k) for k in allowed_keys} return out def _infer_market_label_from_code(code: str) -> str: c = str(code or '').strip().lower() if c.startswith('sh.'): return '沪' if c.startswith('sz.'): return '深' c6 = ''.join(ch for ch in c if ch.isdigit()) if c6.startswith(('5', '6', '9')): return '沪' if c6: return '深' return '' def _persist_spot_to_market_lists_xlsx() -> None: try: path = _get_market_list_xlsx_path() if not path or not os.path.exists(path): return with _akshare_spot_lock: stocks_spot = dict(_akshare_spot_cache.get('stocks') or {}) etf_spot = dict(_akshare_spot_cache.get('etf') or {}) lof_spot = dict(_akshare_spot_cache.get('lof') or {}) with _market_list_file_lock: sheets = pd.read_excel(path, sheet_name=None) def _update_sheet(sheet_name: str, spot_map: dict[str, Any], label: str) -> None: df = sheets.get(sheet_name) if df is None or not isinstance(df, pd.DataFrame) or df.empty: return # 确保 code6 if 'code6' not in df.columns: if 'code' in df.columns: df['code6'] = df['code'].map(lambda x: _normalize_code6(_safe_str(x).split('.')[-1])) else: return df['code6'] = df['code6'].map(_normalize_code6) spot_rows = [] for k, v in (spot_map or {}).items(): if not isinstance(v, dict): continue code6 = _normalize_code6(k) if not code6: continue amount_wan = None try: if v.get('amount') is not None: amount_wan = float(v.get('amount')) / 10000.0 except Exception: amount_wan = None spot_rows.append({ 'code6': code6, '最新价': v.get('latest_price'), '当日涨跌幅': v.get('pct_change'), '开盘价': v.get('open'), '最高价': v.get('high'), '最低价': v.get('low'), '成交额(万)': amount_wan, '成交量(手)': v.get('volume'), '换手率(%)': v.get('turnover_rate'), '市盈率': v.get('pe'), }) spot_df = pd.DataFrame(spot_rows) if spot_df is None or spot_df.empty: # 仍然写入"市场类型/市"列,保持结构一致 if '市场类型' not in df.columns: df['市场类型'] = label else: df['市场类型'] = label if '市' not in df.columns: code_col = 'code' if 'code' in df.columns else None if code_col: df['市'] = df[code_col].map(_infer_market_label_from_code) else: df['市'] = '' sheets[sheet_name] = df return merged = df.merge(spot_df, on='code6', how='left', suffixes=('', '_spot')) for col in ['最新价', '当日涨跌幅', '开盘价', '最高价', '最低价', '成交额(万)', '成交量(手)', '换手率(%)', '市盈率']: if col not in merged.columns: merged[col] = None # 只在 spot 有值时更新,否则保留旧值 if f"{col}_spot" in merged.columns: try: mask = merged[f"{col}_spot"].notna() merged.loc[mask, col] = merged.loc[mask, f"{col}_spot"] except Exception: pass merged['市场类型'] = label if '市' not in merged.columns: merged['市'] = '' try: if 'code' in merged.columns: merged['市'] = merged['code'].map(_infer_market_label_from_code) except Exception: pass drop_cols = [c for c in merged.columns if str(c).endswith('_spot')] if drop_cols: merged = merged.drop(columns=drop_cols) sheets[sheet_name] = merged _update_sheet('stocks', stocks_spot, '股票') _update_sheet('etf', etf_spot, 'ETF') _update_sheet('lof', lof_spot, 'LOF') # 记录当日是否已更新过行情(用于跨重启的"每天只自动更新一次") try: meta_df = sheets.get('meta') if meta_df is None or not isinstance(meta_df, pd.DataFrame) or meta_df.empty: meta_df = pd.DataFrame([{}]) if 'spot_updated_at' not in meta_df.columns: meta_df['spot_updated_at'] = '' meta_df.loc[0, 'spot_updated_at'] = _now_bj_iso() sheets['meta'] = meta_df market_list_state['spot_updated_at'] = str(meta_df.loc[0, 'spot_updated_at']) except Exception: pass with pd.ExcelWriter(path, engine='openpyxl') as writer: for sn, sdf in sheets.items(): try: if isinstance(sdf, pd.DataFrame): sdf.to_excel(writer, sheet_name=str(sn), index=False) except Exception: continue market_list_cache['loaded'] = False except Exception: return def _save_download_history() -> None: try: with _download_tasks_lock: path = _get_download_history_path() tasks = list(download_tasks.values()) tasks.sort(key=lambda t: str(t.get('created_at') or ''), reverse=True) tasks = tasks[:200] payload = { 'saved_at': _now_bj_iso(), 'max_id': max([int(t.get('id', 0)) for t in tasks], default=0), 'tasks': [_serialize_download_task(t) for t in tasks], } tmp_path = f"{path}.tmp" with open(tmp_path, 'w', encoding='utf-8') as f: json.dump(payload, f, ensure_ascii=False, indent=2) os.replace(tmp_path, path) except Exception: pass def _load_download_history() -> None: global download_task_counter path = _get_download_history_path() if not os.path.exists(path): return try: with open(path, 'r', encoding='utf-8') as f: payload = json.load(f) or {} tasks = payload.get('tasks') or [] restored = {} max_id = 0 for t in tasks: if not isinstance(t, dict): continue tid = int(t.get('id', 0) or 0) if tid <= 0: continue restored[tid] = { 'id': tid, 'symbol': t.get('symbol'), 'stock_name': t.get('stock_name'), 'data_type': t.get('data_type'), 'data_provider': t.get('data_provider') or 'baostock', 'security_type': t.get('security_type'), 'start_date': t.get('start_date'), 'end_date': t.get('end_date'), 'effective_start_date': t.get('effective_start_date'), 'effective_end_date': t.get('effective_end_date'), 'adjust_flag': t.get('adjust_flag'), 'save_format': t.get('save_format'), 'limit': t.get('limit'), 'order': t.get('order'), 'columns': t.get('columns'), 'exchange_code': t.get('exchange_code'), 'ticker': t.get('ticker'), 'data_range_start': t.get('data_range_start'), 'data_range_end': t.get('data_range_end'), 'file_size_bytes': t.get('file_size_bytes'), 'status': t.get('status'), 'stage': t.get('stage'), 'progress': t.get('progress', 0), 'message': t.get('message'), 'created_at': t.get('created_at'), 'started_at': t.get('started_at'), 'completed_at': t.get('completed_at'), 'data': None, 'data_count': t.get('data_count', 0), 'file_path': t.get('file_path'), 'cache_key': t.get('cache_key'), 'cached': t.get('cached', False), 'adjustments': t.get('adjustments', []), 'error': t.get('error'), } max_id = max(max_id, tid) download_tasks.clear() download_tasks.update(restored) download_task_counter = max(download_task_counter, max_id) except Exception: return _load_download_history() market_list_state: dict[str, Any] = { 'status': 'idle', 'progress': 0, 'message': '未更新', 'started_at': None, 'completed_at': None, 'updated_at': None, 'spot_updated_at': None, 'counts': {'stocks': 0, 'etf': 0, 'lof': 0}, 'error': None, } market_list_cache: dict[str, Any] = { 'loaded': False, 'path': None, 'items': [], } def _now_bj_iso() -> str: return now_utc().astimezone(timezone(timedelta(hours=8))).isoformat() # AkShare 行情缓存(避免频繁拉取导致接口变慢) _akshare_spot_cache: dict[str, Any] = { 'ts': 0.0, 'ttl': 1800.0, 'stocks': {}, 'etf': {}, 'lof': {}, } _akshare_spot_lock = threading.Lock() _market_list_file_lock = threading.Lock() # 连续 akshare 失败次数,>=2 时自动尝试 efinance 兜底 _akshare_consecutive_failures: int = 0 # 行情请求重试次数与间隔(秒) _SPOT_FETCH_MAX_RETRIES = 3 _SPOT_FETCH_RETRY_DELAY = 2.0 def _is_trading_time_bj() -> bool: try: bj = now_utc().astimezone(timezone(timedelta(hours=8))) if bj.weekday() >= 5: return False m = bj.hour * 60 + bj.minute if (9 * 60 + 30) <= m < (11 * 60 + 30): return True if (13 * 60) <= m < (15 * 60): return True return False except Exception: return False def _apply_proxy_env() -> None: """从环境变量应用 HTTP/HTTPS 代理,供 akshare/efinance 等请求使用。支持 MARKET_HTTP_PROXY / HTTP_PROXY 等。""" for key_in, key_out in [ ('MARKET_HTTP_PROXY', 'HTTP_PROXY'), ('MARKET_HTTPS_PROXY', 'HTTPS_PROXY'), ('HTTP_PROXY', 'HTTP_PROXY'), ('HTTPS_PROXY', 'HTTPS_PROXY'), ]: val = os.environ.get(key_in, '').strip() if val and not os.environ.get(key_out): os.environ[key_out] = val def _retry_spot_fetch(func, max_retries: int = None) -> Any: """对行情拉取函数进行重试;失败时等待 _SPOT_FETCH_RETRY_DELAY 秒后重试。""" n = max_retries if max_retries is not None else _SPOT_FETCH_MAX_RETRIES last_exc = None for attempt in range(n): try: return func() except Exception as e: last_exc = e if attempt < n - 1: time.sleep(_SPOT_FETCH_RETRY_DELAY) if last_exc is not None: raise last_exc return None async def _spot_auto_refresh_loop(): while True: try: await asyncio.sleep(60) if not _is_trading_time_bj(): continue await asyncio.to_thread(_refresh_akshare_spot_cache, False) except asyncio.CancelledError: raise except Exception: continue def _to_float(x: Any) -> float | None: try: if x is None: return None s = str(x).strip().replace('%', '') if s == '' or s.lower() in ('nan', 'none'): return None return float(s) except Exception: return None def _normalize_code6(x: Any) -> str: try: if x is None: return '' s = str(x).strip() if s == '' or s.lower() in ('nan', 'none'): return '' # Excel 里可能把 code6 读成 float(例如 1.0),这里先取小数点前部分 if '.' in s: s = s.split('.')[0] digits = ''.join(ch for ch in s if ch.isdigit()) if not digits: return '' if len(digits) < 6: return digits.zfill(6) if len(digits) > 6: return digits[-6:] return digits except Exception: return '' def _first_existing_col(df: 'pd.DataFrame', candidates: list[str]) -> str | None: try: cols = set([str(c) for c in df.columns]) for c in candidates: if c in cols: return c except Exception: pass return None def _build_spot_map( df: 'pd.DataFrame', code_cols: list[str], name_cols: list[str], price_cols: list[str], pct_cols: list[str], open_cols: list[str], high_cols: list[str], low_cols: list[str], amount_cols: list[str], volume_cols: list[str], turnover_cols: list[str], pe_cols: list[str], ) -> dict[str, dict[str, Any]]: if df is None or not isinstance(df, pd.DataFrame) or df.empty: return {} code_col = _first_existing_col(df, code_cols) name_col = _first_existing_col(df, name_cols) price_col = _first_existing_col(df, price_cols) pct_col = _first_existing_col(df, pct_cols) open_col = _first_existing_col(df, open_cols) high_col = _first_existing_col(df, high_cols) low_col = _first_existing_col(df, low_cols) amount_col = _first_existing_col(df, amount_cols) volume_col = _first_existing_col(df, volume_cols) turnover_col = _first_existing_col(df, turnover_cols) pe_col = _first_existing_col(df, pe_cols) out: dict[str, dict[str, Any]] = {} for _, row in df.iterrows(): code_raw = _safe_str(row.get(code_col)) if code_col else '' code6 = _normalize_code6(code_raw) if not code6: continue out[code6] = { 'code6': code6, 'name': _safe_str(row.get(name_col)) if name_col else '', 'latest_price': _to_float(row.get(price_col)) if price_col else None, 'pct_change': _to_float(row.get(pct_col)) if pct_col else None, 'open': _to_float(row.get(open_col)) if open_col else None, 'high': _to_float(row.get(high_col)) if high_col else None, 'low': _to_float(row.get(low_col)) if low_col else None, 'amount': _to_float(row.get(amount_col)) if amount_col else None, 'volume': _to_float(row.get(volume_col)) if volume_col else None, 'turnover_rate': _to_float(row.get(turnover_col)) if turnover_col else None, 'pe': _to_float(row.get(pe_col)) if pe_col else None, } return out def _refresh_akshare_spot_cache(force: bool = False) -> None: global _akshare_consecutive_failures now = time.time() if not force and not _is_trading_time_bj(): return with _akshare_spot_lock: ttl = float(_akshare_spot_cache.get('ttl', 60.0) or 60.0) if not force and (now - float(_akshare_spot_cache.get('ts', 0.0) or 0.0)) < ttl: return _apply_proxy_env() # 不在 lock 内执行网络/第三方调用,避免阻塞其它请求 stocks_map: dict[str, Any] = {} etf_map: dict[str, Any] = {} lof_map: dict[str, Any] = {} akshare_ok = False try: import akshare as ak def _ak_stocks(): df = ak.stock_zh_a_spot_em() return _build_spot_map( df, code_cols=['代码', '股票代码', 'symbol', 'code'], name_cols=['名称', '股票简称', 'name'], price_cols=['最新价', '最新', '现价', 'price'], pct_cols=['涨跌幅', '涨跌幅(%)', 'pct', '涨跌幅%', '涨跌幅(%)'], open_cols=['今开', '开盘', '开盘价', 'open'], high_cols=['最高', '最高价', 'high'], low_cols=['最低', '最低价', 'low'], amount_cols=['成交额', '成交额(元)', 'amount'], volume_cols=['成交量', '成交量(手)', 'volume'], turnover_cols=['换手率', '换手率(%)', 'turnover_rate'], pe_cols=['市盈率-动态', '市盈率', 'pe', '市盈率(动)'], ) def _ak_etf(): df = ak.fund_etf_spot_em() return _build_spot_map( df, code_cols=['基金代码', '代码', 'symbol', 'code'], name_cols=['基金简称', '名称', 'name'], price_cols=['最新价', '最新', '现价', 'price'], pct_cols=['涨跌幅', '涨跌幅(%)', 'pct'], open_cols=['开盘', '开盘价', '今开', 'open'], high_cols=['最高', '最高价', 'high'], low_cols=['最低', '最低价', 'low'], amount_cols=['成交额', '成交额(元)', 'amount'], volume_cols=['成交量', '成交量(手)', 'volume'], turnover_cols=['换手率', '换手率(%)', 'turnover_rate'], pe_cols=['市盈率', 'pe'], ) def _ak_lof(): df = ak.fund_lof_spot_em() return _build_spot_map( df, code_cols=['基金代码', '代码', 'symbol', 'code'], name_cols=['基金简称', '名称', 'name'], price_cols=['最新价', '最新', '现价', 'price'], pct_cols=['涨跌幅', '涨跌幅(%)', 'pct'], open_cols=['开盘', '开盘价', '今开', 'open'], high_cols=['最高', '最高价', 'high'], low_cols=['最低', '最低价', 'low'], amount_cols=['成交额', '成交额(元)', 'amount'], volume_cols=['成交量', '成交量(手)', 'volume'], turnover_cols=['换手率', '换手率(%)', 'turnover_rate'], pe_cols=['市盈率', 'pe'], ) try: stocks_map = _retry_spot_fetch(_ak_stocks) akshare_ok = bool(stocks_map) except Exception: stocks_map = {} try: etf_map = _retry_spot_fetch(_ak_etf) akshare_ok = akshare_ok or bool(etf_map) except Exception: etf_map = {} try: lof_map = _retry_spot_fetch(_ak_lof) akshare_ok = akshare_ok or bool(lof_map) except Exception: lof_map = {} if akshare_ok: _akshare_consecutive_failures = 0 else: _akshare_consecutive_failures += 1 except Exception: stocks_map = {} etf_map = {} lof_map = {} _akshare_consecutive_failures += 1 # 连续失败 >=2 时用 efinance 兜底 fail_count = _akshare_consecutive_failures if fail_count >= 2: try: import efinance as ef ef_cols = dict( code_cols=['股票代码', '代码', 'symbol', 'code'], name_cols=['股票名称', '名称', 'name'], price_cols=['最新价', 'price'], pct_cols=['涨跌幅', 'pct'], open_cols=['今开', 'open'], high_cols=['最高', 'high'], low_cols=['最低', 'low'], amount_cols=['成交额', 'amount'], volume_cols=['成交量', 'volume'], turnover_cols=['换手率', 'turnover_rate'], pe_cols=['动态市盈率', 'pe'], ) if not stocks_map: try: df_ef_stock = _retry_spot_fetch(lambda: ef.stock.get_realtime_quotes()) if df_ef_stock is not None and not df_ef_stock.empty: stocks_map = _build_spot_map(df_ef_stock, **ef_cols) except Exception: pass if not etf_map: try: df_ef_etf = _retry_spot_fetch(lambda: ef.stock.get_realtime_quotes(['ETF'])) if df_ef_etf is not None and not df_ef_etf.empty: etf_map = _build_spot_map(df_ef_etf, **ef_cols) except Exception: pass if not lof_map: try: df_ef_lof = _retry_spot_fetch(lambda: ef.stock.get_realtime_quotes(['LOF'])) if df_ef_lof is not None and not df_ef_lof.empty: lof_map = _build_spot_map(df_ef_lof, **ef_cols) except Exception: pass except Exception: pass with _akshare_spot_lock: _akshare_spot_cache['stocks'] = stocks_map _akshare_spot_cache['etf'] = etf_map _akshare_spot_cache['lof'] = lof_map _akshare_spot_cache['ts'] = time.time() _persist_spot_to_market_lists_xlsx() def _get_market_list_xlsx_path() -> str: backend_dir = os.path.dirname(__file__) cache_dir = os.path.join(backend_dir, 'cache') os.makedirs(cache_dir, exist_ok=True) return os.path.join(cache_dir, 'market_lists.xlsx') def _guess_market_prefix(code6: str) -> str: c = (code6 or '').strip() if not c: return '' if c.lower().startswith('sh.') or c.lower().startswith('sz.'): return c[:3].lower() if c[0] in ('6', '9', '5'): return 'sh.' return 'sz.' def _to_prefixed_code(code: str) -> str: c = (code or '').strip() if not c: return '' if c.lower().startswith('sh.') or c.lower().startswith('sz.'): return c.lower() if c.isdigit(): return f"{_guess_market_prefix(c)}{c}" return c.lower() def _safe_str(x: Any) -> str: if x is None: return '' return str(x).strip() def _normalize_q(s: str) -> str: s = (s or '').strip().lower() return ''.join(ch for ch in s if ch not in (' ', '\t', '\n', '\r')) def _get_pinyin_initials(text: str) -> str: try: from pypinyin import pinyin, Style parts = pinyin(text, style=Style.FIRST_LETTER, strict=False) return ''.join(p[0] for p in parts if p and p[0]).lower() except Exception: return '' def _get_pinyin_full(text: str) -> str: try: from pypinyin import pinyin, Style parts = pinyin(text, style=Style.NORMAL, strict=False) return ''.join(p[0] for p in parts if p and p[0]).lower() except Exception: return '' def _expand_fuzzy_pinyin(q: str) -> list[str]: """对拼音输入做少量模糊音扩展,避免爆炸式组合。""" base = _normalize_q(q) if not base: return [] variants = {base} replacements = [ ('zh', 'z'), ('ch', 'c'), ('sh', 's'), ('z', 'zh'), ('c', 'ch'), ('s', 'sh'), ('l', 'n'), ('n', 'l'), ('ang', 'an'), ('an', 'ang'), ('eng', 'en'), ('en', 'eng'), ('ing', 'in'), ('in', 'ing'), ] for a, b in replacements: if a in base: variants.add(base.replace(a, b)) return list(variants)[:10] def _score_item(item: dict[str, Any], q_variants: list[str]) -> float: code = _normalize_q(item.get('code', '')) code6 = _normalize_q(item.get('code6', '')) name = _normalize_q(item.get('name', '')) py_i = _normalize_q(item.get('pinyin_initials', '')) py_f = _normalize_q(item.get('pinyin_full', '')) best = 0.0 for q in q_variants: if not q: continue if q == code or q == code6: best = max(best, 100.0) continue if code.startswith(q) or code6.startswith(q): best = max(best, 92.0) if q in code or q in code6: best = max(best, 88.0) if name and q in name: best = max(best, 84.0) if py_i and (py_i.startswith(q) or q in py_i): best = max(best, 78.0) if py_f and (py_f.startswith(q) or q in py_f): best = max(best, 74.0) if name: sim = difflib.SequenceMatcher(None, q, name).ratio() if sim >= 0.60: best = max(best, 50.0 + sim * 30.0) if py_f: sim2 = difflib.SequenceMatcher(None, q, py_f).ratio() if sim2 >= 0.70: best = max(best, 45.0 + sim2 * 25.0) return best def _load_market_lists_from_disk(force: bool = False) -> bool: if market_list_cache.get('loaded') and not force: return True path = _get_market_list_xlsx_path() market_list_cache['path'] = path if not os.path.exists(path): market_list_cache['loaded'] = False market_list_cache['items'] = [] return False try: sheets = pd.read_excel(path, sheet_name=None) items: list[dict[str, Any]] = [] counts = {'stocks': 0, 'etf': 0, 'lof': 0} for sheet_name, df in sheets.items(): if sheet_name not in ('stocks', 'etf', 'lof'): continue if df is None or df.empty: continue for _, row in df.iterrows(): code = _safe_str(row.get('code', row.get('代码', ''))) name = _safe_str(row.get('name', row.get('名称', ''))) code6 = _normalize_code6(row.get('code6', '')) if not code: if code6: code = _to_prefixed_code(code6) if not code6 and code: code6 = _normalize_code6(code.split('.')[-1]) if not code or not name: continue it = { 'code': _to_prefixed_code(code), 'code6': code6, 'name': name, 'type': sheet_name, 'ipo_date': _safe_str(row.get('ipo_date', row.get('ipoDate', row.get('上市日期', '')))), 'industry': _safe_str(row.get('industry', row.get('行业', ''))), # 以下为行情缓存列(可被 AkShare 刷新写入,但不影响行业/上市日期) 'market_type': _safe_str(row.get('市场类型', '')), 'market': _safe_str(row.get('市', '')), 'latest_price': _to_float(row.get('最新价')), 'pct_change': _to_float(row.get('当日涨跌幅')), 'open': _to_float(row.get('开盘价')), 'high': _to_float(row.get('最高价')), 'low': _to_float(row.get('最低价')), 'amount': _to_float(row.get('成交额(万)')), 'volume': _to_float(row.get('成交量(手)')), 'turnover_rate': _to_float(row.get('换手率(%)')), 'pe': _to_float(row.get('市盈率')), 'pinyin_initials': _safe_str(row.get('pinyin_initials', '')), 'pinyin_full': _safe_str(row.get('pinyin_full', '')), } items.append(it) counts[sheet_name] = len(df) market_list_cache['items'] = items market_list_cache['loaded'] = True market_list_state['counts'] = counts try: meta_df = sheets.get('meta') if meta_df is not None and not meta_df.empty: updated_at = _safe_str(meta_df.iloc[0].get('updated_at', '')) if updated_at: market_list_state['updated_at'] = updated_at spot_updated_at = _safe_str(meta_df.iloc[0].get('spot_updated_at', '')) if spot_updated_at: market_list_state['spot_updated_at'] = spot_updated_at except Exception: pass return True except Exception as e: market_list_cache['loaded'] = False market_list_cache['items'] = [] msg = str(e) if 'openpyxl' in msg and 'Missing optional dependency' in msg: msg = f"读取本地列表缓存失败: {e}(请安装 openpyxl:pip install openpyxl)" market_list_state['error'] = f"读取本地列表缓存失败: {msg}" return False async def _update_market_lists_job(): market_list_state['status'] = 'updating' market_list_state['progress'] = 0 market_list_state['message'] = '开始更新列表...' market_list_state['started_at'] = _now_bj_iso() market_list_state['completed_at'] = None market_list_state['error'] = None path = _get_market_list_xlsx_path() market_list_cache['path'] = path try: market_list_state['message'] = '拉取股票列表...' market_list_state['progress'] = 10 df_stocks = None try: import akshare as ak df_tmp = ak.stock_info_a_code_name() cols = {c: str(c) for c in df_tmp.columns} code_col = None name_col = None for c in df_tmp.columns: if str(c).lower() in ('code', 'symbol', '股票代码') or '代码' in str(c): code_col = c break for c in df_tmp.columns: if str(c).lower() in ('name', '股票简称', '股票名称') or '名称' in str(c) or '简称' in str(c): name_col = c break if code_col and name_col: df_stocks = df_tmp[[code_col, name_col]].rename(columns={code_col: 'code6', name_col: 'name'}) except Exception: df_stocks = None if df_stocks is None: try: import baostock as bs with _baostock_lock: lg = bs.login() if lg.error_code != '0': raise RuntimeError(lg.error_msg) rs = bs.query_stock_basic() data_list = [] while (rs.error_code == '0') and rs.next(): data_list.append(rs.get_row_data()) bs.logout() df_tmp = pd.DataFrame(data_list, columns=rs.fields) if 'code' in df_tmp.columns and 'code_name' in df_tmp.columns: df_stocks = df_tmp[['code', 'code_name']].rename(columns={'code': 'code', 'code_name': 'name'}) df_stocks['code6'] = df_stocks['code'].map(lambda x: _safe_str(x).split('.')[-1]) df_stocks['code'] = df_stocks['code'].map(_to_prefixed_code) except Exception: df_stocks = None if df_stocks is None or df_stocks.empty: raise RuntimeError('股票列表获取失败(AkShare/Baostock 均失败)') if 'code' not in df_stocks.columns: df_stocks['code'] = df_stocks['code6'].map(lambda x: _to_prefixed_code(_safe_str(x))) market_list_state['message'] = '拉取 ETF 列表...' market_list_state['progress'] = 35 df_etf = pd.DataFrame(columns=['code6', 'name', 'code']) try: import akshare as ak df_tmp = ak.fund_etf_spot_em() code_col = None name_col = None for c in df_tmp.columns: if str(c).lower() in ('code', 'symbol') or '代码' in str(c): code_col = c break for c in df_tmp.columns: if str(c).lower() in ('name',) or '名称' in str(c) or '简称' in str(c): name_col = c break if code_col and name_col: df_etf = df_tmp[[code_col, name_col]].rename(columns={code_col: 'code6', name_col: 'name'}) except Exception: df_etf = pd.DataFrame(columns=['code6', 'name']) if 'code' not in df_etf.columns: df_etf['code'] = df_etf['code6'].map(lambda x: _to_prefixed_code(_safe_str(x))) market_list_state['message'] = '拉取 LOF 列表...' market_list_state['progress'] = 55 df_lof = pd.DataFrame(columns=['code6', 'name', 'code']) try: import akshare as ak df_tmp = ak.fund_lof_spot_em() code_col = None name_col = None for c in df_tmp.columns: if str(c).lower() in ('code', 'symbol') or '代码' in str(c): code_col = c break for c in df_tmp.columns: if str(c).lower() in ('name',) or '名称' in str(c) or '简称' in str(c): name_col = c break if code_col and name_col: df_lof = df_tmp[[code_col, name_col]].rename(columns={code_col: 'code6', name_col: 'name'}) except Exception: df_lof = pd.DataFrame(columns=['code6', 'name']) if 'code' not in df_lof.columns: df_lof['code'] = df_lof['code6'].map(lambda x: _to_prefixed_code(_safe_str(x))) market_list_state['message'] = '补充上市时间和行业分类...' market_list_state['progress'] = 65 try: import baostock as bs with _baostock_lock: lg = bs.login() if lg.error_code != '0': raise RuntimeError(lg.error_msg) rs_basic = bs.query_stock_basic() basic_list = [] while (rs_basic.error_code == '0') and rs_basic.next(): basic_list.append(rs_basic.get_row_data()) df_basic = pd.DataFrame(basic_list, columns=rs_basic.fields) rs_ind = bs.query_stock_industry() ind_list = [] while (rs_ind.error_code == '0') and rs_ind.next(): ind_list.append(rs_ind.get_row_data()) df_ind = pd.DataFrame(ind_list, columns=rs_ind.fields) bs.logout() if isinstance(df_basic, pd.DataFrame) and not df_basic.empty: if 'code' in df_basic.columns: df_basic['code'] = df_basic['code'].map(_to_prefixed_code) if 'code6' not in df_basic.columns and 'code' in df_basic.columns: df_basic['code6'] = df_basic['code'].map(lambda x: _safe_str(x).split('.')[-1]) if 'ipoDate' in df_basic.columns and 'ipo_date' not in df_basic.columns: df_basic['ipo_date'] = df_basic['ipoDate'].map(_safe_str) keep_cols = [c for c in ['code', 'code6', 'ipo_date'] if c in df_basic.columns] df_basic2 = df_basic[keep_cols].dropna(subset=['code6']) if keep_cols else None else: df_basic2 = None if isinstance(df_ind, pd.DataFrame) and not df_ind.empty: if 'code' in df_ind.columns: df_ind['code'] = df_ind['code'].map(_to_prefixed_code) if 'code6' not in df_ind.columns and 'code' in df_ind.columns: df_ind['code6'] = df_ind['code'].map(lambda x: _safe_str(x).split('.')[-1]) industry_col = None for c in df_ind.columns: lc = str(c).lower() if lc in ('industry', 'industryname', 'industry_name'): industry_col = c break if industry_col is None: for c in df_ind.columns: if '行业' in str(c): industry_col = c break if industry_col is None: for c in df_ind.columns: lc = str(c).lower() if 'industry' in lc and 'class' not in lc: industry_col = c break if industry_col is not None: df_ind2 = df_ind[['code6', industry_col]].rename(columns={industry_col: 'industry'}) df_ind2['industry'] = df_ind2['industry'].map(_safe_str) else: df_ind2 = None else: df_ind2 = None def _merge_enrichment(df: pd.DataFrame) -> pd.DataFrame: if df is None or df.empty: return df out = df.copy() if 'code6' not in out.columns: out['code6'] = out['code'].map(lambda x: _safe_str(x).split('.')[-1]) out['code6'] = out['code6'].map(_safe_str) if df_basic2 is not None and not df_basic2.empty: out = out.merge(df_basic2[['code6', 'ipo_date']], on='code6', how='left') if df_ind2 is not None and not df_ind2.empty: out = out.merge(df_ind2[['code6', 'industry']], on='code6', how='left') if 'ipo_date' not in out.columns: out['ipo_date'] = '' else: out['ipo_date'] = out['ipo_date'].map(_safe_str) if 'industry' not in out.columns: out['industry'] = '' else: out['industry'] = out['industry'].map(_safe_str) return out df_stocks = _merge_enrichment(df_stocks) df_etf = _merge_enrichment(df_etf) df_lof = _merge_enrichment(df_lof) except Exception: # 补充信息失败不影响主流程 pass market_list_state['message'] = '生成拼音索引...' market_list_state['progress'] = 70 def _append_pinyin(df: pd.DataFrame) -> pd.DataFrame: if df is None or df.empty: return df df = df.copy() df['name'] = df['name'].map(_safe_str) df['code'] = df['code'].map(_to_prefixed_code) if 'code6' in df.columns: df['code6'] = df['code6'].map(_safe_str) else: df['code6'] = df['code'].map(lambda x: _safe_str(x).split('.')[-1]) if 'ipo_date' not in df.columns: df['ipo_date'] = '' else: df['ipo_date'] = df['ipo_date'].map(_safe_str) if 'industry' not in df.columns: df['industry'] = '' else: df['industry'] = df['industry'].map(_safe_str) df['pinyin_initials'] = df['name'].map(_get_pinyin_initials) df['pinyin_full'] = df['name'].map(_get_pinyin_full) return df[['code', 'code6', 'name', 'ipo_date', 'industry', 'pinyin_initials', 'pinyin_full']] df_stocks2 = _append_pinyin(df_stocks) df_etf2 = _append_pinyin(df_etf) df_lof2 = _append_pinyin(df_lof) market_list_state['message'] = '写入本地缓存文件...' market_list_state['progress'] = 88 meta_df = pd.DataFrame([ { 'updated_at': _now_bj_iso(), 'source': 'akshare/baostock', 'stocks_count': int(len(df_stocks2)) if df_stocks2 is not None else 0, 'etf_count': int(len(df_etf2)) if df_etf2 is not None else 0, 'lof_count': int(len(df_lof2)) if df_lof2 is not None else 0, } ]) with pd.ExcelWriter(path, engine='openpyxl') as writer: df_stocks2.to_excel(writer, sheet_name='stocks', index=False) df_etf2.to_excel(writer, sheet_name='etf', index=False) df_lof2.to_excel(writer, sheet_name='lof', index=False) meta_df.to_excel(writer, sheet_name='meta', index=False) market_list_state['message'] = '加载到内存...' market_list_state['progress'] = 95 _load_market_lists_from_disk(force=True) market_list_state['status'] = 'completed' market_list_state['progress'] = 100 market_list_state['message'] = '更新完成' market_list_state['completed_at'] = _now_bj_iso() market_list_state['updated_at'] = market_list_state['completed_at'] market_list_state['counts'] = { 'stocks': int(len(df_stocks2)) if df_stocks2 is not None else 0, 'etf': int(len(df_etf2)) if df_etf2 is not None else 0, 'lof': int(len(df_lof2)) if df_lof2 is not None else 0, } except Exception as e: market_list_state['status'] = 'failed' market_list_state['progress'] = 100 market_list_state['message'] = '更新失败' market_list_state['error'] = str(e) market_list_state['completed_at'] = _now_bj_iso() async def get_market_lists_status(): path = _get_market_list_xlsx_path() market_list_state['file_exists'] = os.path.exists(path) market_list_state['file_path'] = path if market_list_state.get('updated_at') is None and os.path.exists(path): _load_market_lists_from_disk(force=False) return market_list_state async def update_market_lists(request: Request): if market_list_state.get('status') == 'updating': return market_list_state asyncio.create_task(_update_market_lists_job()) return market_list_state async def get_market_lists_items( q: str = '', market_type: str = 'all', market: str = 'all', industry: str = 'all', sort_by: str = '', sort_dir: str = 'asc', page: int = 1, page_size: int = 50, ): if not _load_market_lists_from_disk(force=False): raise HTTPException(status_code=400, detail='本地数据列表尚未缓存,请先点击u201c一键更新列表”') try: page = int(page or 1) except Exception: page = 1 page = max(1, page) try: page_size = int(page_size or 50) except Exception: page_size = 50 page_size = max(1, min(200, page_size)) all_items = list(market_list_cache.get('items', []) or []) industries = sorted({str(it.get('industry') or '').strip() for it in all_items if str(it.get('industry') or '').strip()}) mt = str(market_type or 'all').strip().lower() mk_raw = str(market or 'all').strip().lower() ind = str(industry or 'all').strip() mk = 'all' if mk_raw in ('sh', '沪'): mk = '沪' elif mk_raw in ('sz', '深'): mk = '深' items = all_items if mt in ('stocks', 'etf', 'lof'): items = [it for it in items if str(it.get('type') or '').lower() == mt] if mk in ('沪', '深'): items = [ it for it in items if _infer_market_label_from_code(it.get('code') or '') == mk ] if ind and ind.lower() != 'all': items = [it for it in items if str(it.get('industry') or '').strip() == ind] q_raw = str(q or '').strip() sort_key = str(sort_by or '').strip().lower() sort_direction = str(sort_dir or 'asc').strip().lower() reverse = sort_direction == 'desc' if q_raw and not sort_key: q_norm = _normalize_q(q_raw) q_variants = {q_norm} if q_norm.isdigit(): q_variants.add(q_norm) q_variants.add(_normalize_q(_to_prefixed_code(q_norm))) if q_norm.startswith('sh.') or q_norm.startswith('sz.'): q_variants.add(q_norm.split('.')[-1]) q_variants.update(_expand_fuzzy_pinyin(q_norm)) q_list = [qq for qq in q_variants if qq] scored = [] for it in items: s = _score_item(it, q_list) if s > 0: scored.append((s, it)) scored.sort(key=lambda x: x[0], reverse=True) items = [it for _, it in scored] else: def _to_num(v: Any) -> float | None: try: if v is None: return None if isinstance(v, (int, float)): if not (v == v): return None return float(v) s = str(v).strip().replace('%', '') if s == '' or s.lower() in ('nan', 'none'): return None return float(s) except Exception: return None def _sort_val(it: dict[str, Any]) -> tuple[int, Any]: # None/空值统一排到最后 if sort_key in ('code', '代码'): v = _normalize_code6(it.get('code6') or it.get('code') or '') return (0 if v else 1, v) if sort_key in ('name', '名称'): v = _safe_str(it.get('name', '')) return (0 if v else 1, v) if sort_key in ('type', 'market_type', '市场类型'): v = _safe_str(it.get('market_type') or it.get('type') or '') return (0 if v else 1, v) if sort_key in ('market', '市'): v = _safe_str(it.get('market') or _infer_market_label_from_code(it.get('code') or '')) return (0 if v else 1, v) num_fields = { 'latest_price': 'latest_price', 'pct_change': 'pct_change', 'open': 'open', 'high': 'high', 'low': 'low', 'amount': 'amount', 'volume': 'volume', 'turnover_rate': 'turnover_rate', 'pe': 'pe', } if sort_key in num_fields: n = _to_num(it.get(num_fields[sort_key])) return (0 if n is not None else 1, n if n is not None else 0.0) # 默认排序 v = _safe_str(it.get('code', '')) return (0 if v else 1, v) if sort_key: items = sorted(items, key=_sort_val, reverse=reverse) else: items = sorted(items, key=lambda it: (str(it.get('type') or ''), str(it.get('code') or ''))) total = len(items) start = (page - 1) * page_size end = start + page_size paged = items[start:end] def _infer_market_label(code: str) -> str: c = str(code or '').strip().lower() if c.startswith('sh.'): return '沪' if c.startswith('sz.'): return '深' return '' out_items = [] # 仅在 TTL 到期时刷新 AkShare 行情并写回 market_lists.xlsx;平时优先使用 xlsx 中已缓存的行情列 _refresh_akshare_spot_cache(force=False) for it in paged: code = it.get('code') latest_price = it.get('latest_price') pct_change = it.get('pct_change') open_p = it.get('open') high_p = it.get('high') low_p = it.get('low') amount_wan = it.get('amount') volume = it.get('volume') turnover_rate = it.get('turnover_rate') pe = it.get('pe') out_items.append({ 'code': code, 'name': it.get('name'), 'type': it.get('type'), 'ipo_date': it.get('ipo_date', ''), 'industry': it.get('industry', ''), 'market': _infer_market_label(code), 'latest_price': latest_price, 'pct_change': pct_change, 'open': open_p, 'high': high_p, 'low': low_p, 'amount': amount_wan, 'volume': volume, 'turnover_rate': turnover_rate, 'pe': pe, }) stats = { 'total': int(market_list_state.get('counts', {}).get('stocks', 0) or 0) + int(market_list_state.get('counts', {}).get('etf', 0) or 0) + int(market_list_state.get('counts', {}).get('lof', 0) or 0), 'stocks': int(market_list_state.get('counts', {}).get('stocks', 0) or 0), 'etf': int(market_list_state.get('counts', {}).get('etf', 0) or 0), 'lof': int(market_list_state.get('counts', {}).get('lof', 0) or 0), 'updated_at': market_list_state.get('updated_at'), 'spot_updated_at': market_list_state.get('spot_updated_at'), } return { 'items': out_items, 'total': total, 'page': page, 'page_size': page_size, 'stats': stats, 'industries': industries, } async def refresh_market_lists_spot(): try: await asyncio.to_thread(_refresh_akshare_spot_cache, True) except Exception as e: msg = str(e).strip() or '行情刷新异常' if 'No module named' in msg and 'akshare' in msg: raise HTTPException( status_code=503, detail='行情拉取失败:未安装 akshare,请安装 pip install akshare;若已安装仍失败可尝试安装备用源 pip install efinance', ) if 'No module named' in msg and 'efinance' in msg: raise HTTPException( status_code=503, detail='行情拉取失败:AKShare 连续失败且未安装 efinance 兜底,请安装 pip install efinance 或检查网络/代理(可设置 HTTP_PROXY、HTTPS_PROXY)', ) raise HTTPException( status_code=503, detail=f'行情刷新失败:{msg}。可检查网络或设置环境变量 HTTP_PROXY/HTTPS_PROXY 后重试。', ) with _akshare_spot_lock: ts = float(_akshare_spot_cache.get('ts', 0.0) or 0.0) stocks = _akshare_spot_cache.get('stocks') or {} etf = _akshare_spot_cache.get('etf') or {} lof = _akshare_spot_cache.get('lof') or {} if not stocks and not etf and not lof: raise HTTPException( status_code=503, detail='行情拉取失败:AKShare 与 efinance 均未获取到数据,请检查网络或配置代理(环境变量 HTTP_PROXY/HTTPS_PROXY)后重试。', ) return { 'status': 'ok', 'ts': ts, 'refreshed_at': datetime.now().isoformat(), } async def search_stocks(keyword: str = ""): """搜索股票""" try: print(f"🔍 搜索股票: {keyword}") if not keyword.strip(): print("⚠️ 关键词为空") return [] if not _load_market_lists_from_disk(force=False): raise HTTPException(status_code=400, detail='本地股票/ETF/LOF 列表尚未缓存,请先点击“一键更新列表”') q_raw = keyword.strip() q_norm = _normalize_q(q_raw) q_variants = {q_norm} if q_norm.isdigit(): q_variants.add(q_norm) q_variants.add(_normalize_q(_to_prefixed_code(q_norm))) if q_norm.startswith('sh.') or q_norm.startswith('sz.'): q_variants.add(q_norm.split('.')[-1]) q_variants.update(_expand_fuzzy_pinyin(q_norm)) q_list = [q for q in q_variants if q] scored = [] for item in market_list_cache.get('items', []): s = _score_item(item, q_list) if s > 0: scored.append((s, item)) scored.sort(key=lambda x: x[0], reverse=True) top = scored[:20] results = [] for _, it in top: results.append({ 'code': it.get('code'), 'name': it.get('name'), 'type': it.get('type'), 'ipo_date': it.get('ipo_date', ''), 'industry': it.get('industry', ''), }) print(f"✅ 搜索完成: 找到{len(results)}只匹配") return results except Exception as e: print(f"❌ 搜索股票失败: {e}") import traceback traceback.print_exc() return [] def _normalize_symbol_for_baostock(symbol: str) -> str: symbol = (symbol or "").strip() if not symbol: return symbol if symbol.lower().startswith('sh.') or symbol.lower().startswith('sz.'): return symbol.lower() if symbol.isdigit(): # Baostock 使用前缀:sh./sz. # 常见规则: # - 上交所:6/9 开头股票;50/51/56/58 等 ETF # - 深交所:0/1/2/3 开头股票;15/16 等 ETF/LOF if symbol.startswith(('6', '9')): return f"sh.{symbol}" if symbol.startswith(('50', '51', '56', '58')): return f"sh.{symbol}" if symbol.startswith(('0', '1', '2', '3', '15', '16')): return f"sz.{symbol}" return f"sz.{symbol}" return symbol def _symbol_to_code6(symbol: str) -> str: s = (symbol or '').strip().lower() if s.startswith(('sh.', 'sz.')): s = s.split('.', 1)[1] s = ''.join(ch for ch in s if ch.isdigit()) return s def _map_adjust_flag_to_akshare(adjust_flag: str) -> str: # Baostock: 1=前复权, 2=后复权, 3=不复权 m = { '1': 'qfq', '2': 'hfq', '3': '', 1: 'qfq', 2: 'hfq', 3: '', } return m.get(adjust_flag, '') def _map_adjust_flag_to_efinance(adjust_flag: Any) -> int: # Best-effort mapping (efinance may ignore/handle differently). # 0: 不复权, 1: 前复权, 2: 后复权 m = { '1': 1, '2': 2, '3': 0, 1: 1, 2: 2, 3: 0, } return int(m.get(adjust_flag, 0)) def _normalize_symbol_for_efinance(symbol: str) -> str: s = (symbol or '').strip() if not s: return s low = s.lower() if low.startswith(('sh.', 'sz.')): return s.split('.', 1)[1] if low.startswith('sh') and low[2:].isdigit(): return low[2:] if low.startswith('sz') and low[2:].isdigit(): return low[2:] return s def _efinance_kline(task: dict) -> 'pd.DataFrame': import efinance as ef raw_symbol = task.get('symbol') symbol = _normalize_symbol_for_efinance(raw_symbol) code6 = _symbol_to_code6(str(raw_symbol or '')) data_type = str(task.get('data_type') or 'daily').strip().lower() klt: int | None if data_type == 'daily': klt = None elif data_type.endswith('min'): klt = int(data_type.replace('min', '')) else: raise RuntimeError('efinance 不支持的数据类型') fqt = _map_adjust_flag_to_efinance(task.get('adjust_flag', '3')) def _infer_exchange_prefix(sym: str) -> str | None: s = (sym or '').strip().lower() if s.startswith('sh.') or s.startswith('sh'): return 'sh' if s.startswith('sz.') or s.startswith('sz'): return 'sz' c = _symbol_to_code6(s) if c.startswith(('5', '6', '9')): return 'sh' return 'sz' def _build_symbol_candidates() -> list[str]: candidates: list[str] = [] if code6: candidates.append(code6) exch = _infer_exchange_prefix(str(raw_symbol or '')) if exch and code6: candidates.append(f"{exch}{code6}") # ETF 兜底:同时尝试 sh/sz 前缀(不同数据源/代码可能不一致) sec_type = _infer_security_type(str(raw_symbol or ''), task.get('security_type')) if sec_type == 'etf' and code6: candidates.append(f"sh{code6}") candidates.append(f"sz{code6}") # 最后保留一次原 normalize 结果(可能是非 6 位代码) if symbol and symbol not in candidates: candidates.append(symbol) # 去重并保持顺序 seen: set[str] = set() out: list[str] = [] for c in candidates: cc = (c or '').strip() if not cc or cc in seen: continue seen.add(cc) out.append(cc) return out def _call_quote_history(sym: str) -> 'pd.DataFrame | None': try: if klt is None: return ef.stock.get_quote_history(sym, fqt=fqt) return ef.stock.get_quote_history(sym, klt=klt, fqt=fqt) except TypeError: # Some versions may not accept fqt if klt is None: return ef.stock.get_quote_history(sym) return ef.stock.get_quote_history(sym, klt=klt) df = None last_err: Exception | None = None for sym in _build_symbol_candidates(): try: df = _call_quote_history(sym) if isinstance(df, pd.DataFrame) and not df.empty: break except Exception as e: last_err = e df = None continue if df is None and last_err is not None: # 统一抛出,便于上层记录错误 raise last_err if df is None or not isinstance(df, pd.DataFrame) or df.empty: return pd.DataFrame() start_date = task.get('effective_start_date') or task.get('start_date') end_date = task.get('effective_end_date') or task.get('end_date') if (start_date or end_date) and '日期' in df.columns: try: date_only = df['日期'].astype(str).str.slice(0, 10) mask = pd.Series([True] * len(df)) if start_date: mask &= date_only >= str(start_date) if end_date: mask &= date_only <= str(end_date) df = df.loc[mask].copy() except Exception: pass return df def _infer_security_type(symbol: str, declared_type: Any = None) -> str: t = (declared_type or '').strip().lower() if isinstance(declared_type, str) else '' if t in ('stocks', 'stock', 'a', 'a股'): return 'stocks' if t in ('etf',): return 'etf' if t in ('lof',): return 'lof' # 经验规则兜底 code6 = _symbol_to_code6(symbol) if code6.startswith(('15', '51', '56', '58')): # 常见 ETF: 15/51/56/58 return 'etf' if code6.startswith('16'): return 'lof' # 使用本地市场列表缓存做最终判断 try: if _load_market_lists_from_disk(force=False): code_prefixed = _to_prefixed_code(code6) for it in market_list_cache.get('items', []): if it.get('code') == code_prefixed: it_type = (it.get('type') or '').lower() if it_type in ('stocks', 'etf', 'lof'): return it_type except Exception: pass return 'stocks' def _akshare_kline_daily(task: dict) -> 'pd.DataFrame': import akshare as ak symbol = task.get('symbol') code6 = _symbol_to_code6(symbol) sec_type = _infer_security_type(symbol, task.get('security_type')) start_date = (task.get('effective_start_date') or task.get('start_date') or '').replace('-', '') end_date = (task.get('effective_end_date') or task.get('end_date') or '').replace('-', '') adjust = _map_adjust_flag_to_akshare(task.get('adjust_flag', '3')) if sec_type == 'etf': try: df = ak.fund_etf_hist_em(symbol=code6, period='daily', start_date=start_date, end_date=end_date, adjust=adjust) except Exception: # Fallback: 腾讯行情接口 (web.ifzq.gtimg.cn) import requests prefix = 'sh' if code6.startswith(('5', '6', '9', '8')) else 'sz' url = (f'https://web.ifzq.gtimg.cn/appstock/app/fqkline/get' f'?_var=kline_dayqfq¶m={prefix}{code6},day,{start_date[:4]}-{start_date[4:6]}-{start_date[6:8]},' f'{end_date[:4]}-{end_date[4:6]}-{end_date[6:8]},2000,qfq&r=0.1') resp = requests.get(url, timeout=10) raw = resp.text import json raw = raw[raw.index('{'):] data = json.loads(raw) klines = data['data'][f'{prefix}{code6}'].get('qfqday') or data['data'][f'{prefix}{code6}'].get('day') or [] rows = [] for k in klines: rows.append({'日期': k[0], '开盘': k[1], '收盘': k[2], '最高': k[3], '最低': k[4], '成交量': k[5]}) df = pd.DataFrame(rows) elif sec_type == 'lof': df = ak.fund_lof_hist_em(symbol=code6, period='daily', start_date=start_date, end_date=end_date, adjust=adjust) else: df = ak.stock_zh_a_hist(symbol=code6, period='daily', start_date=start_date, end_date=end_date, adjust=adjust) # 确保有代码列,便于后续插入"股票名称"列 if isinstance(df, pd.DataFrame) and not df.empty: if '股票代码' not in df.columns and 'code' not in df.columns: df.insert(0, '股票代码', code6) return df def _derive_canghai_exchange_code(symbol: str) -> str: s = (symbol or '').strip().lower() if s.startswith('sh.'): return 'XSHG' if s.startswith('sz.'): return 'XSHE' code6 = _symbol_to_code6(symbol) if code6.startswith(('5', '6', '9')): return 'XSHG' return 'XSHE' def _mask_token(token: Any) -> str: t = '' if token is None else str(token) if not t: return '' if len(t) <= 8: return '***' return f"{t[:4]}***{t[-4:]}" def _canghai_etf_5min(task: dict) -> 'pd.DataFrame': import requests token = task.get('canghai_token') or task.get('token') if not token: raise RuntimeError('缺少沧海数据 token') exchange_code = (task.get('exchange_code') or '').strip() or _derive_canghai_exchange_code(task.get('symbol', '')) ticker = (task.get('ticker') or '').strip() or _symbol_to_code6(task.get('symbol', '')) if not exchange_code: raise RuntimeError('缺少沧海数据 exchange_code') if not ticker: raise RuntimeError('缺少沧海数据 ticker') start_date = task.get('effective_start_date') or task.get('start_date') end_date = task.get('effective_end_date') or task.get('end_date') limit = task.get('limit') order = task.get('order') columns = task.get('columns') url = f"https://www.tsanghi.com/api/fin/etf/{exchange_code}/5min" params: dict[str, Any] = { 'token': token, 'ticker': ticker, } if start_date: params['start_date'] = start_date if end_date: params['end_date'] = end_date if limit is not None and str(limit).strip() != '': params['limit'] = int(limit) if order is not None and str(order).strip() != '': params['order'] = int(order) params['fmt'] = 'json' resp = requests.get(url, params=params, timeout=30) resp.raise_for_status() payload = resp.json() if resp.content else {} if not isinstance(payload, dict): raise RuntimeError('沧海数据返回格式异常') code = payload.get('code') if code not in (200, '200', 0, '0', None): raise RuntimeError(payload.get('msg') or f"沧海数据返回错误 code={code}") rows = payload.get('data') if rows is None: rows = payload.get('rows') if not isinstance(rows, list) or not rows: raise RuntimeError('沧海数据未返回数据') df = pd.DataFrame(rows) if 'ticker' in df.columns and 'code' not in df.columns: df.rename(columns={'ticker': 'code'}, inplace=True) if 'date' in df.columns: df['date'] = df['date'].astype(str) if 'time' not in df.columns and 'date' in df.columns: try: df['time'] = df['date'].str.split(' ').str[-1] except Exception: pass return df def _canghai_minute_bars(task: dict) -> 'pd.DataFrame': import requests token = task.get('canghai_token') or task.get('token') if not token: raise RuntimeError('缺少沧海数据 token') sec_type = _infer_security_type(task.get('symbol', ''), task.get('security_type')) if sec_type not in ('stocks', 'etf'): raise RuntimeError('沧海数据仅支持 股票/ETF') data_type = str(task.get('data_type') or '').strip().lower() if data_type not in ('5min', '15min', '30min', '60min'): raise RuntimeError('沧海数据仅支持 5min/15min/30min/60min') minutes = int(data_type.replace('min', '')) symbol = task.get('symbol', '') exchange_code = (task.get('exchange_code') or '').strip() or _derive_canghai_exchange_code(symbol) ticker = (task.get('ticker') or '').strip() or _symbol_to_code6(symbol) if not exchange_code: raise RuntimeError('缺少沧海数据 exchange_code') if not ticker: raise RuntimeError('缺少沧海数据 ticker') start_date = task.get('effective_start_date') or task.get('start_date') end_date = task.get('effective_end_date') or task.get('end_date') limit = task.get('limit') order = task.get('order') columns = task.get('columns') product = 'etf' if sec_type == 'etf' else 'stock' url = f"https://www.tsanghi.com/api/fin/{product}/{exchange_code}/{minutes}min" params: dict[str, Any] = { 'token': token, 'ticker': ticker, 'fmt': 'json', } if start_date: params['start_date'] = start_date if end_date: params['end_date'] = end_date if limit is not None and str(limit).strip() != '': params['limit'] = int(limit) if order is not None and str(order).strip() != '': params['order'] = int(order) if columns is None or str(columns).strip() == '': params['columns'] = 'ticker,date,open,high,low,close,volume,amount,pre_close' else: params['columns'] = str(columns) masked = dict(params) masked['token'] = _mask_token(masked.get('token')) started = time.perf_counter() print(f"🌊 沧海数据请求: GET {url} params={masked}") resp = requests.get(url, params=params, timeout=30) elapsed_ms = int((time.perf_counter() - started) * 1000) print(f"🌊 沧海数据响应: status={resp.status_code} elapsed_ms={elapsed_ms} bytes={len(resp.content or b'')}") resp.raise_for_status() payload = resp.json() if resp.content else {} if not isinstance(payload, dict): raise RuntimeError('沧海数据返回格式异常') code = payload.get('code') if code not in (200, '200', 0, '0', None): raise RuntimeError(payload.get('msg') or f"沧海数据返回错误 code={code}") rows = payload.get('data') if rows is None: rows = payload.get('rows') if not isinstance(rows, list) or not rows: raise RuntimeError('沧海数据未返回数据') df = pd.DataFrame(rows) if 'ticker' in df.columns and 'code' not in df.columns: df.rename(columns={'ticker': 'code'}, inplace=True) for col in ('open', 'high', 'low', 'close', 'volume', 'amount', 'pre_close'): if col in df.columns: df[col] = pd.to_numeric(df[col], errors='coerce') if 'date' in df.columns: df['date'] = df['date'].astype(str) try: dt = pd.to_datetime(df['date'], errors='coerce') if dt.notna().any(): df = df.assign(_dt=dt).sort_values('_dt').drop(columns=['_dt']) except Exception: pass return df def _canghai_daily_bars(task: dict) -> 'pd.DataFrame': import requests token = task.get('canghai_token') or task.get('token') if not token: raise RuntimeError('缺少沧海数据 token') sec_type = _infer_security_type(task.get('symbol', ''), task.get('security_type')) if sec_type not in ('stocks', 'etf'): raise RuntimeError('沧海数据仅支持 股票/ETF') symbol = task.get('symbol', '') exchange_code = (task.get('exchange_code') or '').strip() or _derive_canghai_exchange_code(symbol) ticker = (task.get('ticker') or '').strip() or _symbol_to_code6(symbol) if not exchange_code: raise RuntimeError('缺少沧海数据 exchange_code') if not ticker: raise RuntimeError('缺少沧海数据 ticker') start_date = task.get('effective_start_date') or task.get('start_date') end_date = task.get('effective_end_date') or task.get('end_date') limit = task.get('limit') order = task.get('order') columns = task.get('columns') product = 'etf' if sec_type == 'etf' else 'stock' url = f"https://www.tsanghi.com/api/fin/{product}/{exchange_code}/daily" params: dict[str, Any] = { 'token': token, 'ticker': ticker, 'fmt': 'json', } if start_date: params['start_date'] = start_date if end_date: params['end_date'] = end_date if limit is not None and str(limit).strip() != '': params['limit'] = int(limit) if order is not None and str(order).strip() != '': params['order'] = int(order) if columns is None or str(columns).strip() == '': params['columns'] = 'ticker,date,open,high,low,close,volume,amount,pre_close' else: params['columns'] = str(columns) masked = dict(params) masked['token'] = _mask_token(masked.get('token')) started = time.perf_counter() print(f"🌊 沧海数据请求: GET {url} params={masked}") resp = requests.get(url, params=params, timeout=30) elapsed_ms = int((time.perf_counter() - started) * 1000) print(f"🌊 沧海数据响应: status={resp.status_code} elapsed_ms={elapsed_ms} bytes={len(resp.content or b'')}") resp.raise_for_status() payload = resp.json() if resp.content else {} if not isinstance(payload, dict): raise RuntimeError('沧海数据返回格式异常') code = payload.get('code') if code not in (200, '200', 0, '0', None): raise RuntimeError(payload.get('msg') or f"沧海数据返回错误 code={code}") rows = payload.get('data') if rows is None: rows = payload.get('rows') if not isinstance(rows, list) or not rows: raise RuntimeError('沧海数据未返回数据') df = pd.DataFrame(rows) if 'ticker' in df.columns and 'code' not in df.columns: df.rename(columns={'ticker': 'code'}, inplace=True) for col in ('open', 'high', 'low', 'close', 'volume', 'amount', 'pre_close'): if col in df.columns: df[col] = pd.to_numeric(df[col], errors='coerce') if 'date' in df.columns: df['date'] = df['date'].astype(str) try: dt = pd.to_datetime(df['date'], errors='coerce') if dt.notna().any(): df = df.assign(_dt=dt).sort_values('_dt').drop(columns=['_dt']) except Exception: pass return df def _akshare_kline_min(task: dict) -> 'pd.DataFrame': """AKShare 分时接口限制较多:通常只能获取近期分时数据。""" import akshare as ak symbol = task.get('symbol') code6 = _symbol_to_code6(symbol) sec_type = _infer_security_type(symbol, task.get('security_type')) adjust = _map_adjust_flag_to_akshare(task.get('adjust_flag', '3')) start_date = task.get('effective_start_date') or task.get('start_date') end_date = task.get('effective_end_date') or task.get('end_date') start_dt = f"{start_date} 09:30:00" end_dt = f"{end_date} 15:00:00" data_type = task.get('data_type') period = '60' if data_type == '60min' else ('30' if data_type == '30min' else '5') if sec_type == 'etf': df = ak.fund_etf_hist_min_em(symbol=code6, period=period, adjust=adjust, start_date=start_dt, end_date=end_dt) elif sec_type == 'lof': df = ak.fund_lof_hist_min_em(symbol=code6, period=period, adjust=adjust, start_date=start_dt, end_date=end_dt) else: df = ak.stock_zh_a_hist_min_em(symbol=code6, start_date=start_dt, end_date=end_dt, period=period, adjust=adjust) if isinstance(df, pd.DataFrame) and not df.empty: if '股票代码' not in df.columns and 'code' not in df.columns: df.insert(0, '股票代码', code6) return df def _clamp_date_str(date_str: str, min_date: str | None = None, max_date: str | None = None) -> tuple[str, bool]: """基于 YYYY-MM-DD 的字符串比较进行范围裁剪,返回 (裁剪后日期, 是否发生裁剪)。""" changed = False if not date_str: return date_str, False new_val = date_str if min_date and new_val < min_date: new_val = min_date changed = True if max_date and new_val > max_date: new_val = max_date changed = True return new_val, changed def _build_download_cache_key(task: dict) -> str: return "|".join([ str(task.get('data_provider', 'baostock')), str(task.get('symbol', '')), str(task.get('data_type', '')), str(task.get('effective_start_date', task.get('start_date', ''))), str(task.get('effective_end_date', task.get('end_date', ''))), str(task.get('adjust_flag', '')), str(task.get('save_format', '')), str(task.get('limit', '')), str(task.get('order', '')), str(task.get('columns', '')), str(task.get('exchange_code', '')), str(task.get('ticker', '')), ]) def _safe_filename(text: str) -> str: # 简单文件名清洗,避免非法字符 invalid = '<>:/\\|?*\"' out = ''.join('_' if ch in invalid else ch for ch in (text or "")) return out.strip() or 'data' async def create_download_task(data: dict): """创建下载任务""" global download_task_counter try: safe_data = dict(data or {}) if 'canghai_token' in safe_data: safe_data['canghai_token'] = _mask_token(safe_data.get('canghai_token')) if 'token' in safe_data: safe_data['token'] = _mask_token(safe_data.get('token')) print(f"📥 收到下载请求: {safe_data}") with _download_tasks_lock: download_task_counter += 1 task_id = download_task_counter # 解析请求参数 symbol = data.get('symbol', '') stock_name = data.get('stock_name', '') data_type = data.get('data_type', 'daily') # daily, 60min, 30min, 5min, basic_info start_date = data.get('start_date', '') end_date = data.get('end_date', '') adjust_flag = data.get('adjust_flag', '3') # 1=前复权, 2=后复权, 3=不复权 save_format = data.get('save_format', 'excel') # csv, excel, json security_type = data.get('security_type') or data.get('type') data_provider = (data.get('data_provider') or 'baostock') exchange_code = data.get('exchange_code') ticker = data.get('ticker') canghai_token = data.get('canghai_token') or data.get('token') limit = data.get('limit') order = data.get('order') columns = data.get('columns') if not symbol: raise HTTPException(status_code=400, detail="请选择股票") if not start_date or not end_date: raise HTTPException(status_code=400, detail="请选择日期范围") # 后端统一处理:结束日期默认不超过昨天,开始日期不早于 1990-01-01 yesterday = (datetime.now() - timedelta(days=1)).strftime('%Y-%m-%d') effective_start, start_changed = _clamp_date_str(start_date, min_date='1990-01-01') effective_end, end_changed = _clamp_date_str(end_date, max_date=yesterday) adjustments = [] if start_changed: adjustments.append(f"开始日期已调整为 {effective_start}") if end_changed: adjustments.append(f"结束日期已调整为 {effective_end}") if effective_start and effective_end and effective_start > effective_end: raise HTTPException(status_code=400, detail="日期范围不合法:开始日期不能晚于结束日期") provider_norm = str(data_provider or 'baostock').strip().lower() inferred_sec_type = _infer_security_type(symbol, security_type) if provider_norm == 'canghai': if not str(canghai_token or '').strip(): raise HTTPException(status_code=400, detail='沧海数据:缺少 token') if inferred_sec_type not in ('stocks', 'etf'): raise HTTPException(status_code=400, detail='沧海数据仅支持 股票/ETF') if data_type not in ('daily', '5min', '15min', '30min', '60min'): raise HTTPException(status_code=400, detail='沧海数据仅支持 daily/5min/15min/30min/60min') exchange_code = (exchange_code or '').strip() or _derive_canghai_exchange_code(symbol) ticker = (ticker or '').strip() or _symbol_to_code6(symbol) try: start_dt = datetime.strptime(effective_start, '%Y-%m-%d') end_dt = datetime.strptime(effective_end, '%Y-%m-%d') if abs((end_dt - start_dt).days) > 366: raise HTTPException(status_code=400, detail='沧海数据:start_date 和 end_date 最大跨度为 1 年') except HTTPException: raise except Exception: raise HTTPException(status_code=400, detail='沧海数据:日期格式错误,应为 YYYY-MM-DD') if provider_norm == 'efinance': if data_type not in ('daily', '5min', '15min', '30min', '60min'): raise HTTPException(status_code=400, detail='efinance 仅支持 daily/5min/15min/30min/60min') # 创建任务 task = { 'id': task_id, 'data_provider': data_provider, 'symbol': symbol, 'stock_name': stock_name, 'data_type': data_type, 'security_type': inferred_sec_type, 'start_date': start_date, 'end_date': end_date, 'effective_start_date': effective_start, 'effective_end_date': effective_end, 'adjust_flag': adjust_flag, 'save_format': save_format, 'limit': limit, 'order': order, 'columns': columns, 'exchange_code': exchange_code, 'ticker': ticker, 'canghai_token': canghai_token, 'status': 'downloading', 'stage': 'queued', 'progress': 0, 'message': '任务已创建,等待执行', 'created_at': datetime.now().isoformat(), 'started_at': None, 'completed_at': None, 'data': None, 'data_count': 0, 'file_path': None, 'cache_key': None, 'cached': False, 'adjustments': adjustments, 'error': None } task['cache_key'] = _build_download_cache_key(task) # 缓存命中:直接返回已完成任务(无需再次请求数据源) backend_dir = os.path.dirname(__file__) cache_dir = os.path.join(backend_dir, 'cache', 'downloads') os.makedirs(cache_dir, exist_ok=True) cache_index_path = os.path.join(cache_dir, 'index.json') cache_index = {} try: if os.path.exists(cache_index_path): with open(cache_index_path, 'r', encoding='utf-8') as f: cache_index = json.load(f) or {} except Exception: cache_index = {} existing_path = cache_index.get(task['cache_key']) if existing_path and os.path.exists(existing_path): task['status'] = 'completed' task['stage'] = 'completed' task['progress'] = 100 task['message'] = '缓存命中:已直接返回本地缓存文件' task['cached'] = True task['file_path'] = existing_path task['started_at'] = task.get('created_at') task['completed_at'] = datetime.now().isoformat() with _download_tasks_lock: download_tasks[task_id] = task _save_download_history() print(f"✅ 任务 {task_id} 缓存命中: {existing_path}") return task with _download_tasks_lock: download_tasks[task_id] = task _save_download_history() print(f"✅ 创建下载任务 {task_id}: {symbol} ({data_type})") DOWNLOAD_EXECUTOR.submit(_run_download_coroutine, task_id) return task except HTTPException: raise except Exception as e: print(f"❌ 创建下载任务失败: {e}") import traceback traceback.print_exc() raise HTTPException(status_code=500, detail=str(e)) async def download_stock_data(task_id: int): """异步下载股票数据""" try: with _download_tasks_lock: task = download_tasks.get(task_id) if not task: return print(f"📥 开始下载任务 {task_id}: {task['symbol']} ({task['data_type']})") if not task.get('started_at'): task['started_at'] = datetime.now().isoformat() task['stage'] = 'initializing' task['progress'] = 5 task['message'] = '初始化下载任务' data_type = task['data_type'] provider = (task.get('data_provider') or 'baostock').strip().lower() if provider == 'canghai': sec_type = _infer_security_type(task.get('symbol', ''), task.get('security_type')) task['security_type'] = sec_type if sec_type not in ('stocks', 'etf'): task['status'] = 'failed' task['stage'] = 'failed' task['progress'] = 100 task['error'] = '沧海数据仅支持 股票/ETF' _save_download_history() return if data_type not in ('daily', '5min', '15min', '30min', '60min'): task['status'] = 'failed' task['stage'] = 'failed' task['progress'] = 100 task['error'] = '沧海数据仅支持 daily/5min/15min/30min/60min' _save_download_history() return task['stage'] = 'query' task['progress'] = 35 task['message'] = f"沧海数据 获取 {('ETF' if sec_type == 'etf' else '股票')} {data_type} 数据" if data_type == 'daily': df = _canghai_daily_bars(task) else: df = _canghai_minute_bars(task) task['stage'] = 'saving' task['progress'] = 85 task['message'] = '正在保存文件' await _persist_download_result(task_id, df) return if provider == 'efinance': if data_type not in ('daily', '5min', '15min', '30min', '60min'): task['status'] = 'failed' task['stage'] = 'failed' task['progress'] = 100 task['error'] = 'efinance 仅支持 daily/5min/15min/30min/60min' _save_download_history() return task['stage'] = 'query' task['progress'] = 35 task['message'] = f"efinance 获取 {data_type} 数据" try: df = _efinance_kline(task) if df is None or not isinstance(df, pd.DataFrame) or df.empty: raise RuntimeError('efinance 未获取到数据') except Exception as e: task['status'] = 'failed' task['stage'] = 'failed' task['progress'] = 100 msg = str(e) if 'No module named' in msg and 'efinance' in msg: msg = '缺少 efinance 模块,请安装:pip install efinance' task['error'] = f"efinance 获取数据失败: {msg}" _save_download_history() return task['stage'] = 'saving' task['progress'] = 85 task['message'] = '正在保存文件' await _persist_download_result(task_id, df) return # basic_info 优先使用 AKShare if data_type == 'basic_info': task['stage'] = 'fetching_basic_info' task['progress'] = 30 task['message'] = '正在获取基本信息' symbol_raw = task.get('symbol', '') symbol_6 = symbol_raw if symbol_6.lower().startswith(('sh.', 'sz.')): symbol_6 = symbol_6.split('.', 1)[1] symbol_6 = ''.join(ch for ch in symbol_6 if ch.isdigit()) result = None if provider == 'baostock': try: import baostock as bs with _baostock_lock: lg = bs.login() if lg.error_code != '0': raise Exception(lg.error_msg) bs_symbol = _normalize_symbol_for_baostock(symbol_raw) rs = bs.query_stock_basic(code=bs_symbol) data_list = [] while (rs.error_code == '0') & rs.next(): data_list.append(rs.get_row_data()) result = pd.DataFrame(data_list, columns=rs.fields) bs.logout() task['message'] = 'Baostock 获取基本信息成功' except Exception as e: try: with _baostock_lock: bs.logout() except Exception: pass task['status'] = 'failed' task['stage'] = 'failed' task['progress'] = 100 task['error'] = f"Baostock 获取基本信息失败: {e}" _save_download_history() return else: try: import akshare as ak df_info = ak.stock_individual_info_em(symbol=symbol_6) result = df_info task['message'] = 'AKShare 获取基本信息成功' except Exception as e: if provider == 'akshare': task['status'] = 'failed' task['stage'] = 'failed' task['progress'] = 100 task['error'] = f"AKShare 获取基本信息失败: {e}" _save_download_history() return print(f"⚠️ AKShare 获取基本信息失败,尝试 Baostock 兜底: {e}") try: import baostock as bs with _baostock_lock: lg = bs.login() if lg.error_code != '0': raise Exception(lg.error_msg) bs_symbol = _normalize_symbol_for_baostock(symbol_raw) rs = bs.query_stock_basic(code=bs_symbol) data_list = [] while (rs.error_code == '0') & rs.next(): data_list.append(rs.get_row_data()) result = pd.DataFrame(data_list, columns=rs.fields) bs.logout() task['message'] = 'Baostock 获取基本信息成功(兜底)' except Exception as e2: task['status'] = 'failed' task['stage'] = 'failed' task['progress'] = 100 task['error'] = f"获取基本信息失败: {e2}" _save_download_history() return # 统一落盘 task['progress'] = 80 task['stage'] = 'saving' task['message'] = '正在保存文件' await _persist_download_result(task_id, result) return # K线类:优先 Baostock,失败/无数据才回退 AKShare sec_type = _infer_security_type(task.get('symbol', ''), task.get('security_type')) task['security_type'] = sec_type data_type = task.get('data_type') # Baostock 优先,失败再用 AKShare(auto 模式) bs = None try: import baostock as bs # type: ignore except Exception: bs = None if provider == 'baostock' and bs is None: task['status'] = 'failed' task['stage'] = 'failed' task['progress'] = 100 task['error'] = '缺少 baostock 模块,请安装:pip install baostock' _save_download_history() return result = None used_baostock = False if provider != 'akshare' and bs is not None and data_type in ('daily', '60min', '30min', '5min'): try: symbol_bs = _normalize_symbol_for_baostock(task.get('symbol', '')) data_type = task.get('data_type', 'daily') if data_type == 'daily': frequency = 'd' fields = 'date,code,open,high,low,close,preclose,volume,amount,adjustflag,turn,tradestatus,pctChg,isST' elif data_type == '60min': frequency = '60' fields = 'date,time,code,open,high,low,close,volume,amount,adjustflag' elif data_type == '30min': frequency = '30' fields = 'date,time,code,open,high,low,close,volume,amount,adjustflag' elif data_type == '5min': frequency = '5' fields = 'date,time,code,open,high,low,close,volume,amount,adjustflag' else: frequency = 'd' fields = 'date,code,open,high,low,close,volume,amount,adjustflag' start_date_str = task.get('effective_start_date') or task.get('start_date') end_date_str = task.get('effective_end_date') or task.get('end_date') # Baostock 不支持 ETF/LOF 分钟线 if frequency in ('5', '30', '60') and sec_type in ('etf', 'lof'): if provider == 'baostock': raise RuntimeError('Baostock 不支持 ETF/LOF 分钟线,请改用其它数据源(如 AKShare)') raise RuntimeError('skip_baostock_etf_lof_min') # 分钟线起始时间不早于 2019-01-01 if frequency in ('5', '30', '60'): try: _min_floor = datetime(2019, 1, 1) _sdt = datetime.strptime(str(start_date_str), '%Y-%m-%d') if _sdt < _min_floor: start_date_str = '2019-01-01' except Exception: # 解析失败则用字符串兜底 if str(start_date_str) < '2019-01-01': start_date_str = '2019-01-01' print(f"📅 查询日期范围: {start_date_str} 至 {end_date_str}") print(f"🔄 查询参数: symbol={symbol_bs}, frequency={frequency}, adjustflag={task.get('adjust_flag')}") task['stage'] = 'query' task['progress'] = 35 task['message'] = f"Baostock 查询: {start_date_str} 至 {end_date_str}" loop = asyncio.get_running_loop() last_err = None rows = None flds = None payload = ( str(symbol_bs), str(fields), str(start_date_str), str(end_date_str), str(frequency), str(task.get('adjust_flag', '3')), ) for attempt in range(BAOSTOCK_MAX_RETRIES + 1): try: rows, flds = await loop.run_in_executor( BAOSTOCK_SEGMENT_EXECUTOR, _baostock_segment_worker, payload, ) break except Exception as e: last_err = e if attempt >= BAOSTOCK_MAX_RETRIES: raise await asyncio.sleep(0.5) if not rows: raise RuntimeError('未获取到数据') result = pd.DataFrame(rows, columns=(flds or fields.split(','))) try: if 'date' in result.columns and 'time' in result.columns: result = result.sort_values(['date', 'time']).reset_index(drop=True) elif 'date' in result.columns: result = result.sort_values(['date']).reset_index(drop=True) except Exception: pass used_baostock = True except Exception as e: if str(e) == 'skip_baostock_etf_lof_min': used_baostock = False result = None task['message'] = 'Baostock 不支持 ETF/LOF 分钟线,尝试 AKShare 兜底' else: used_baostock = False result = None if provider == 'baostock': task['status'] = 'failed' task['stage'] = 'failed' task['progress'] = 100 task['error'] = f"Baostock 获取数据失败: {e}" _save_download_history() return task['message'] = f"Baostock 失败,尝试 AKShare 兜底: {e}" if result is None: try: task['stage'] = 'query' task['progress'] = 35 if data_type == 'daily': task['message'] = 'AKShare 获取日线数据(兜底)' result = _akshare_kline_daily(task) elif data_type in ('60min', '30min', '5min'): task['message'] = 'AKShare 获取分时数据(兜底,仅支持近期)' result = _akshare_kline_min(task) else: raise RuntimeError('不支持的数据类型') if result is None or result.empty: raise RuntimeError('AKShare 未获取到数据') except Exception as e: task['status'] = 'failed' task['stage'] = 'failed' task['progress'] = 100 msg = str(e) if 'No module named' in msg and 'akshare' in msg: if bs is None: msg = '未安装 baostock 与 akshare,无法下载数据(请优先安装:pip install baostock;可选安装:pip install akshare)' else: msg = '未安装 akshare,无法使用 AKShare 兜底(可选安装:pip install akshare)' task['error'] = f"数据获取失败(Baostock/AKShare): {msg}" _save_download_history() return task['stage'] = 'saving' task['progress'] = 85 task['message'] = '正在保存文件' await _persist_download_result(task_id, result) except Exception as e: print(f"❌ 下载任务 {task_id} 失败: {e}") import traceback traceback.print_exc() task = download_tasks.get(task_id) if task: task['status'] = 'failed' task['stage'] = 'failed' task['progress'] = 100 task['error'] = str(e) _save_download_history() async def _persist_download_result(task_id: int, df: 'pd.DataFrame'): task = download_tasks.get(task_id) if task is None: return try: def _infer_data_range(dataframe: 'pd.DataFrame') -> tuple[str | None, str | None]: if dataframe is None or not isinstance(dataframe, pd.DataFrame) or dataframe.empty: return None, None date_col = None for c in ('日期', 'date', 'Date', 'datetime', '时间'): if c in dataframe.columns: date_col = c break if date_col is None: return None, None time_col = None for c in ('时间', 'time'): if c in dataframe.columns: time_col = c break def _normalize_time_str(val: str) -> str: t = (val or '').strip() if not t: return t if t.isdigit(): if len(t) == 6: return f"{t[0:2]}:{t[2:4]}:{t[4:6]}" if len(t) == 4: return f"{t[0:2]}:{t[2:4]}:00" if len(t) == 2: return f"{t}:00:00" return t try: if time_col and date_col != time_col: time_series = dataframe[time_col].astype(str).map(_normalize_time_str) series = pd.to_datetime( dataframe[date_col].astype(str).str.strip() + ' ' + time_series.astype(str).str.strip(), errors='coerce' ) else: series = pd.to_datetime(dataframe[date_col].astype(str).str.strip(), errors='coerce') except Exception: return None, None series = series.dropna() if series.empty: # 若时间列导致解析失败,回退到仅用日期列推断范围 try: series = pd.to_datetime(dataframe[date_col].astype(str).str.strip(), errors='coerce').dropna() except Exception: return None, None if series.empty: return None, None dt_min = series.min() dt_max = series.max() if time_col and date_col != time_col: return dt_min.strftime('%Y-%m-%d %H:%M:%S'), dt_max.strftime('%Y-%m-%d %H:%M:%S') return dt_min.strftime('%Y-%m-%d'), dt_max.strftime('%Y-%m-%d') def _standardize_download_dataframe(df: 'pd.DataFrame') -> 'pd.DataFrame': """标准化数据下载输出列: 1) 若日期列包含时间信息,拆分为"日期""时间" 2) 列顺序固定为:股票代码、股票名称、日期、时间(如存在)、开盘价、最高价、最低价、收盘价 3) 其它列统一放到收盘价之后 """ if df is None or df.empty: return df df = df.copy() # 兼容中英文列名(在COLUMN_MAP应用前后都可工作) rename_candidates = { 'code': '股票代码', 'symbol': '股票代码', '股票代码': '股票代码', 'code_name': '股票名称', 'name': '股票名称', '股票名称': '股票名称', 'date': '日期', '日期': '日期', 'time': '时间', '时间': '时间', 'open': '开盘价', '开盘': '开盘价', '开盘价': '开盘价', 'high': '最高价', '最高': '最高价', '最高价': '最高价', 'low': '最低价', '最低': '最低价', '最低价': '最低价', 'close': '收盘价', '收盘': '收盘价', '收盘价': '收盘价', } df.rename(columns={k: v for k, v in rename_candidates.items() if k in df.columns}, inplace=True) # 拆分"日期/时间" if '日期' in df.columns: date_series = df['日期'] # 若已有"时间"列,优先保留;否则判断"日期"是否包含时间信息 if '时间' not in df.columns: try: as_str = date_series.astype(str) has_time = as_str.str.contains(r'\d{1,2}:\d{2}', regex=True, na=False).any() except Exception: has_time = False if has_time: dt = pd.to_datetime(date_series.astype(str), errors='coerce') if dt.notna().any(): df['日期'] = dt.dt.strftime('%Y-%m-%d') df.insert(df.columns.get_loc('日期') + 1, '时间', dt.dt.strftime('%H:%M:%S')) else: # 同时存在"日期""时间"时,确保"日期"仅保留日期部分 try: dt = pd.to_datetime(date_series.astype(str), errors='coerce') if dt.notna().any(): df['日期'] = dt.dt.strftime('%Y-%m-%d') except Exception: pass base_order = ['股票代码', '股票名称', '日期', '时间', '开盘价', '最高价', '最低价', '收盘价'] if '时间' not in df.columns: base_order = [c for c in base_order if c != '时间'] ordered = [c for c in base_order if c in df.columns] extras = [c for c in df.columns if c not in ordered] return df[ordered + extras] # 定义列名中英文字典 COLUMN_MAP = { 'date': '日期', 'time': '时间', 'code': '股票代码', 'open': '开盘价', 'high': '最高价', 'low': '最低价', 'close': '收盘价', 'volume': '成交量', 'amount': '成交额', 'pre_close': '昨收价', 'adjustflag': '复权状态', 'turn': '换手率', 'tradestatus': '交易状态', 'pctChg': '涨跌幅', 'peTTM': '市盈率TTM', 'pbMRQ': '市净率', 'psTTM': '市销率TTM', 'pcfNcfTTM': '市现率TTM', 'isST': '是否ST', 'code_name': '股票名称', 'ipoDate': '上市日期', 'outDate': '退市日期', 'type': '股票类型', 'status': '上市状态', # AKShare a stock_individual_info_em 'item': '项目', 'value': '值', } # 应用列名翻译 df.rename(columns=COLUMN_MAP, inplace=True) data_range_start, data_range_end = _infer_data_range(df) task['data_range_start'] = data_range_start task['data_range_end'] = data_range_end # 插入股票名称列 if '股票代码' in df.columns: # 获取股票代码列的位置 try: code_col_index = df.columns.get_loc('股票代码') stock_name_val = task.get('stock_name', '') # 若已存在"股票名称"(例如 efinance 返回自带该列),则不要重复 insert if '股票名称' not in df.columns: # 插入股票名称 df.insert(code_col_index + 1, '股票名称', stock_name_val) else: # 仅在原值为空时用任务里的 stock_name 补齐,避免覆盖数据源自带的名称 if stock_name_val: try: s = df['股票名称'] empty_mask = s.isna() | (s.astype(str).str.strip() == '') if empty_mask.any(): df.loc[empty_mask, '股票名称'] = stock_name_val except Exception: pass except KeyError: pass # 如果找不到列,则不插入 # 翻译复权状态的值 if '复权状态' in df.columns: adjust_map = { '1': '前复权', '2': '后复权', '3': '不复权', } # 使用 .astype(str) 确保即使是数字也能匹配 df['复权状态'] = df['复权状态'].astype(str).map(adjust_map).fillna(df['复权状态']) # 统一标准化列顺序(CSV/Excel/JSON) df = _standardize_download_dataframe(df) backend_dir = os.path.dirname(__file__) cache_dir = os.path.join(backend_dir, 'cache', 'downloads') os.makedirs(cache_dir, exist_ok=True) save_format = task.get('save_format', 'csv') stock_name = task.get('stock_name') or task.get('symbol') or 'data' data_type = task.get('data_type', 'daily') start_date = task.get('effective_start_date') or task.get('start_date') end_date = task.get('effective_end_date') or task.get('end_date') extension = 'xlsx' if save_format == 'excel' else save_format filename = f"{_safe_filename(stock_name)}_{data_type}_{start_date}_{end_date}.{extension}" file_path = os.path.join(cache_dir, filename) if save_format == 'csv': df.to_csv(file_path, index=False, encoding='utf-8-sig') elif save_format == 'excel': df.to_excel(file_path, index=False, engine='openpyxl') elif save_format == 'json': with open(file_path, 'w', encoding='utf-8') as f: json.dump(df.to_dict('records'), f, ensure_ascii=False, indent=2) else: raise ValueError('不支持的格式') try: task['file_size_bytes'] = int(os.path.getsize(file_path)) if os.path.exists(file_path) else None except Exception: task['file_size_bytes'] = None task['file_path'] = file_path task['data'] = None task['data_count'] = int(len(df)) task['status'] = 'completed' task['stage'] = 'completed' task['progress'] = 100 task['message'] = '下载完成' task['completed_at'] = datetime.now().isoformat() _save_download_history() cache_index_path = os.path.join(cache_dir, 'index.json') cache_index = {} try: if os.path.exists(cache_index_path): with open(cache_index_path, 'r', encoding='utf-8') as f: cache_index = json.load(f) or {} except Exception: cache_index = {} cache_key = task.get('cache_key') if cache_key: cache_index[cache_key] = file_path try: with open(cache_index_path, 'w', encoding='utf-8') as f: json.dump(cache_index, f, ensure_ascii=False, indent=2) except Exception: pass print(f"✅ 任务 {task_id} 文件已保存: {file_path}") except Exception as e: task['status'] = 'failed' task['stage'] = 'failed' task['progress'] = 100 task['error'] = f"保存文件失败: {e}" print(f"❌ 任务 {task_id} 保存失败: {e}") _save_download_history() def _infer_data_range_from_file(file_path: str) -> tuple[str | None, str | None]: try: if not file_path or not os.path.exists(file_path): return None, None ext = os.path.splitext(file_path)[1].lower().lstrip('.') def _normalize_time_str(val: str) -> str: t = (val or '').strip() if not t: return t if t.isdigit(): if len(t) == 6: return f"{t[0:2]}:{t[2:4]}:{t[4:6]}" if len(t) == 4: return f"{t[0:2]}:{t[2:4]}:00" if len(t) == 2: return f"{t}:00:00" return t def _infer_from_df(df: 'pd.DataFrame') -> tuple[str | None, str | None]: if df is None or not isinstance(df, pd.DataFrame) or df.empty: return None, None date_col = None for c in ('日期', 'date', 'Date', 'datetime', '时间'): if c in df.columns: date_col = c break if date_col is None: return None, None time_col = None for c in ('时间', 'time'): if c in df.columns: time_col = c break try: if time_col and date_col != time_col: time_series = df[time_col].astype(str).map(_normalize_time_str) series = pd.to_datetime( df[date_col].astype(str).str.strip() + ' ' + time_series.astype(str).str.strip(), errors='coerce' ) else: series = pd.to_datetime(df[date_col].astype(str).str.strip(), errors='coerce') except Exception: return None, None series = series.dropna() if series.empty: try: series = pd.to_datetime(df[date_col].astype(str).str.strip(), errors='coerce').dropna() except Exception: return None, None if series.empty: return None, None dt_min = series.min() dt_max = series.max() if time_col and date_col != time_col: return dt_min.strftime('%Y-%m-%d %H:%M:%S'), dt_max.strftime('%Y-%m-%d %H:%M:%S') return dt_min.strftime('%Y-%m-%d'), dt_max.strftime('%Y-%m-%d') if ext == 'csv': head = pd.read_csv(file_path, nrows=0, encoding='utf-8-sig') cols = list(head.columns) date_col = next((c for c in ('日期', 'date', 'Date', 'datetime') if c in cols), None) time_col = next((c for c in ('时间', 'time') if c in cols), None) usecols = [c for c in (date_col, time_col) if c] if not usecols: return None, None df = pd.read_csv(file_path, usecols=usecols, dtype=str, encoding='utf-8-sig') return _infer_from_df(df) if ext in ('xlsx', 'xls'): head = pd.read_excel(file_path, nrows=0, engine='openpyxl') cols = list(head.columns) date_col = next((c for c in ('日期', 'date', 'Date', 'datetime') if c in cols), None) time_col = next((c for c in ('时间', 'time') if c in cols), None) usecols = [c for c in (date_col, time_col) if c] if not usecols: return None, None df = pd.read_excel(file_path, usecols=usecols, dtype=str, engine='openpyxl') return _infer_from_df(df) if ext == 'json': with open(file_path, 'r', encoding='utf-8') as f: data = json.load(f) if not isinstance(data, list) or not data: return None, None df = pd.DataFrame(data) return _infer_from_df(df) return None, None except Exception: return None, None async def list_download_history(): with _download_tasks_lock: tasks = list(download_tasks.values()) tasks.sort(key=lambda t: str(t.get('created_at') or ''), reverse=True) changed = False for t in tasks: try: if (not t.get('data_range_start') or not t.get('data_range_end')) and t.get('file_path') and os.path.exists(t.get('file_path')): s, e = _infer_data_range_from_file(t.get('file_path')) if s and e: with _download_tasks_lock: # t 是引用,直接更新即可;加锁保证并发安全 t['data_range_start'] = s t['data_range_end'] = e changed = True except Exception: pass if changed: _save_download_history() def _iso_to_dt(v: str | None) -> datetime | None: if not v: return None try: return datetime.fromisoformat(str(v)) except Exception: return None return [ { 'id': t.get('id'), 'data_provider': t.get('data_provider') or 'baostock', 'symbol': t.get('symbol'), 'stock_name': t.get('stock_name'), 'data_type': t.get('data_type'), 'security_type': t.get('security_type'), 'start_date': t.get('start_date'), 'end_date': t.get('end_date'), 'effective_start_date': t.get('effective_start_date'), 'effective_end_date': t.get('effective_end_date'), 'data_range_start': t.get('data_range_start'), 'data_range_end': t.get('data_range_end'), 'adjust_flag': t.get('adjust_flag'), 'save_format': t.get('save_format'), 'limit': t.get('limit'), 'order': t.get('order'), 'columns': t.get('columns'), 'exchange_code': t.get('exchange_code'), 'ticker': t.get('ticker'), 'file_size_bytes': t.get('file_size_bytes'), 'status': t.get('status'), 'stage': t.get('stage'), 'progress': t.get('progress', 0), 'message': t.get('message'), 'created_at': t.get('created_at'), 'started_at': t.get('started_at'), 'completed_at': t.get('completed_at'), 'download_seconds': ( (lambda s, e: int((e - s).total_seconds()) if s and e else None)( _iso_to_dt(t.get('started_at')), _iso_to_dt(t.get('completed_at')), ) ), 'error': t.get('error'), 'data_count': t.get('data_count', 0), 'cached': t.get('cached', False), 'file_path': t.get('file_path'), 'adjustments': t.get('adjustments', []), } for t in tasks ] async def delete_download_history(task_id: int, request: Request, delete_file: bool = False): with _download_tasks_lock: if task_id not in download_tasks: raise HTTPException(status_code=404, detail='任务不存在') task = download_tasks.pop(task_id) if delete_file: try: file_path = task.get('file_path') if file_path and os.path.exists(file_path): os.remove(file_path) cache_key = task.get('cache_key') if cache_key: backend_dir = os.path.dirname(__file__) cache_dir = os.path.join(backend_dir, 'cache', 'downloads') cache_index_path = os.path.join(cache_dir, 'index.json') if os.path.exists(cache_index_path): with open(cache_index_path, 'r', encoding='utf-8') as f: cache_index = json.load(f) or {} if cache_index.get(cache_key) == file_path: cache_index.pop(cache_key, None) with open(cache_index_path, 'w', encoding='utf-8') as f: json.dump(cache_index, f, ensure_ascii=False, indent=2) except Exception: pass _save_download_history() return {'deleted': task_id} async def batch_delete_download_history(payload: dict): ids = payload.get('ids') or [] delete_file = bool(payload.get('delete_file', False)) deleted = [] not_found = [] for raw_id in ids: try: tid = int(raw_id) except Exception: continue with _download_tasks_lock: if tid not in download_tasks: not_found.append(tid) continue task = download_tasks.pop(tid) deleted.append(tid) if delete_file: try: file_path = task.get('file_path') if file_path and os.path.exists(file_path): os.remove(file_path) cache_key = task.get('cache_key') if cache_key: backend_dir = os.path.dirname(__file__) cache_dir = os.path.join(backend_dir, 'cache', 'downloads') cache_index_path = os.path.join(cache_dir, 'index.json') if os.path.exists(cache_index_path): with open(cache_index_path, 'r', encoding='utf-8') as f: cache_index = json.load(f) or {} if cache_index.get(cache_key) == file_path: cache_index.pop(cache_key, None) with open(cache_index_path, 'w', encoding='utf-8') as f: json.dump(cache_index, f, ensure_ascii=False, indent=2) except Exception: pass _save_download_history() return {'deleted': deleted, 'not_found': not_found} async def get_download_status(task_id: int): """获取下载任务状态""" try: with _download_tasks_lock: if task_id not in download_tasks: raise HTTPException(status_code=404, detail="任务不存在") task = download_tasks[task_id] if (not task.get('data_range_start') or not task.get('data_range_end')) and task.get('file_path') and os.path.exists(task.get('file_path')): s, e = _infer_data_range_from_file(task.get('file_path')) if s and e: with _download_tasks_lock: task['data_range_start'] = s task['data_range_end'] = e _save_download_history() # 返回任务信息(不包含完整数据) return { 'id': task['id'], 'data_provider': task.get('data_provider') or 'baostock', 'symbol': task['symbol'], 'stock_name': task['stock_name'], 'data_type': task['data_type'], 'start_date': task['start_date'], 'end_date': task['end_date'], 'effective_start_date': task.get('effective_start_date'), 'effective_end_date': task.get('effective_end_date'), 'data_range_start': task.get('data_range_start'), 'data_range_end': task.get('data_range_end'), 'adjust_flag': task['adjust_flag'], 'save_format': task['save_format'], 'limit': task.get('limit'), 'order': task.get('order'), 'columns': task.get('columns'), 'exchange_code': task.get('exchange_code'), 'ticker': task.get('ticker'), 'file_size_bytes': task.get('file_size_bytes'), 'status': task['status'], 'stage': task.get('stage'), 'progress': task.get('progress', 0), 'message': task.get('message'), 'created_at': task['created_at'], 'completed_at': task.get('completed_at'), 'error': task.get('error'), 'data_count': task.get('data_count', len(task['data']) if task.get('data') else 0), 'cached': task.get('cached', False), 'file_path': task.get('file_path'), 'adjustments': task.get('adjustments', []) } except Exception as e: print(f"获取下载状态失败: {e}") raise HTTPException(status_code=500, detail=str(e)) async def download_file(task_id: int, format: str = None): """下载数据文件""" try: if task_id not in download_tasks: raise HTTPException(status_code=404, detail="任务不存在") task = download_tasks[task_id] if task['status'] != 'completed': raise HTTPException(status_code=400, detail="任务未完成或已失败") # 优先返回已落盘文件 file_path = task.get('file_path') if file_path and os.path.exists(file_path): from fastapi.responses import FileResponse return FileResponse( file_path, media_type='application/octet-stream', filename=os.path.basename(file_path) ) if not task.get('data'): raise HTTPException(status_code=400, detail="没有可下载的数据") # 使用指定格式或任务中的格式 save_format = format or task['save_format'] # 转换数据为DataFrame df = pd.DataFrame(task['data']) # 统一标准化列顺序(与落盘保存一致) try: df = _standardize_download_dataframe(df) except Exception: pass # 根据格式导出 if save_format == 'csv': output = io.StringIO() df.to_csv(output, index=False, encoding='utf-8') content = output.getvalue().encode('utf-8') media_type = 'text/csv' filename = f"{task['stock_name']}_{task['data_type']}.csv" elif save_format == 'excel': output = io.BytesIO() df.to_excel(output, index=False, engine='openpyxl') content = output.getvalue() media_type = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' filename = f"{task['stock_name']}_{task['data_type']}.xlsx" elif save_format == 'json': content = json.dumps(task['data'], ensure_ascii=False, indent=2).encode('utf-8') media_type = 'application/json' filename = f"{task['stock_name']}_{task['data_type']}.json" else: raise HTTPException(status_code=400, detail="不支持的格式") from fastapi.responses import StreamingResponse return StreamingResponse( iter([content]), media_type=media_type, headers={"Content-Disposition": f"attachment; filename={filename}"} ) except Exception as e: print(f"下载文件失败: {e}") raise HTTPException(status_code=500, detail=str(e)) if __name__ == "__main__": import uvicorn print("=" * 50) print("启动回测系统后端服务...") print("服务地址: http://localhost:8000") print("API文档: http://localhost:8000/docs") print("=" * 50) uvicorn.run(app, host="0.0.0.0", port=8000, reload=False)