ds2api / agy2api /src /storage /database.py
luckfun233
feat(agy): 内嵌 agy2api,接入 Gemini 模型与独立 AGY 面板
b0074f5
Raw History Blame Contribute Delete
47.7 kB
""" 数据库层(SQLAlchemy 2.0 异步 ORM) 统一支持 SQLite(默认)与 PostgreSQL。 凭证以 email 为唯一键;schema 变更不做旧数据兼容(新表直接重建)。 """ import asyncio import json import os import random import time from contextlib import asynccontextmanager from typing import Any, Dict, List, Optional, Set, Tuple from sqlalchemy import ( Boolean, Float, Index, Integer, String, Text, func, inspect, select, text, ) from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column from log import log # ====================== ORM 模型 ====================== class Base(DeclarativeBase): pass class AntigravityCredential(Base): """Antigravity 凭证表(email 唯一键)""" __tablename__ = "antigravity_credentials" id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) email: Mapped[str] = mapped_column(String(320), unique=True, nullable=False) # 凭证结构化字段 access_token: Mapped[str] = mapped_column(Text, default="") refresh_token: Mapped[str] = mapped_column(Text, default="") client_id: Mapped[str] = mapped_column(Text, default="") client_secret: Mapped[str] = mapped_column(Text, default="") project_id: Mapped[str] = mapped_column(Text, default="") expiry: Mapped[str] = mapped_column(Text, default="") # 状态字段 disabled: Mapped[bool] = mapped_column(Boolean, default=False) error_codes: Mapped[str] = mapped_column(Text, default="[]") error_messages: Mapped[str] = mapped_column(Text, default="[]") last_success: Mapped[Optional[float]] = mapped_column(Float, nullable=True) # 模型级冷却 (JSON: {model_name: cooldown_timestamp}) model_cooldowns: Mapped[str] = mapped_column(Text, default="{}") # 凭证支持的模型列表 (JSON: [model_id, ...];空数组 [] 表示尚未探测) supported_models: Mapped[str] = mapped_column(Text, default="[]") # tier 状态 (free/pro/ultra) tier: Mapped[Optional[str]] = mapped_column(String(32), default="pro") # 时间戳 created_at: Mapped[float] = mapped_column(Float, default=time.time) updated_at: Mapped[float] = mapped_column(Float, default=time.time, onupdate=time.time) __table_args__ = ( Index("idx_ag_disabled", "disabled"), ) class ConfigItem(Base): """配置表""" __tablename__ = "config" key: Mapped[str] = mapped_column(String(255), primary_key=True) value: Mapped[str] = mapped_column(Text, nullable=False) updated_at: Mapped[float] = mapped_column(Float, default=time.time) class UsageLog(Base): """请求日志表:每次 API 调用记录一条(成功与失败均记录)""" __tablename__ = "usage_log" id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) created_at: Mapped[float] = mapped_column(Float, default=time.time, index=True) # 请求来源 api_format: Mapped[str] = mapped_column(String(16), default="") # openai / gemini / anthropic client_ip: Mapped[str] = mapped_column(String(64), default="") request_id: Mapped[str] = mapped_column(String(64), default="") # 调用信息 model_name: Mapped[str] = mapped_column(String(128), default="") # 客户端请求的模型名(原始,可能带功能前缀) upstream_model_name: Mapped[str] = mapped_column(String(128), default="") # 后端实际使用的真实模型名(映射后) credential_email: Mapped[str] = mapped_column(String(320), default="") is_stream: Mapped[bool] = mapped_column(Boolean, default=False) # 结果 status_code: Mapped[int] = mapped_column(Integer, default=0) success: Mapped[bool] = mapped_column(Boolean, default=False) use_time: Mapped[float] = mapped_column(Float, default=0.0) # 耗时(秒) # Token 用量 prompt_tokens: Mapped[int] = mapped_column(Integer, default=0) completion_tokens: Mapped[int] = mapped_column(Integer, default=0) total_tokens: Mapped[int] = mapped_column(Integer, default=0) cached_tokens: Mapped[int] = mapped_column(Integer, default=0) # prompt 中命中缓存的 token(含在 prompt_tokens 内) # 失败时的错误摘要 error_message: Mapped[str] = mapped_column(Text, default="") # 请求体详细日志文件索引(data/request_logs 相对路径;空=未记录) request_log_file: Mapped[str] = mapped_column(String(512), default="") __table_args__ = ( Index("idx_ul_created_model", "created_at", "model_name"), Index("idx_ul_created_email", "created_at", "credential_email"), ) # ====================== JSON 序列化辅助 ====================== def _dump_json(value: Any) -> str: return json.dumps(value, ensure_ascii=False) def _load_json(raw: Optional[str], default: Any) -> Any: if not raw: return default try: return json.loads(raw) except (json.JSONDecodeError, TypeError): return default def _row_to_credential_dict(row: AntigravityCredential) -> Dict[str, Any]: """ORM 行 -> 凭证字典""" return { "access_token": row.access_token or "", "refresh_token": row.refresh_token or "", "client_id": row.client_id or "", "client_secret": row.client_secret or "", "project_id": row.project_id or "", "expiry": row.expiry or "", } def get_cooldown_group(model_name: str) -> str: """ 模型 CD 分组:同家族模型共享一个冷却键 - 名字带 image -> "image"(图片生成独立配额,需在 gemini 之前判断) - 名字带 argon -> "argon" - 名字带 gemini -> "gemini" - 名字带 claude 或 gpt -> "claude_gpt" - 其他 -> 原始模型名 """ lowered = (model_name or "").lower() if "image" in lowered: return "image" if "argon" in lowered: return "argon" if "gemini" in lowered: return "gemini" if "claude" in lowered or "gpt" in lowered: return "claude_gpt" return model_name def _credential_supports_model( supported_models: List[str], model_name: Optional[str] ) -> Optional[bool]: """ 检查凭证的 supported_models 是否支持指定 model_name。 Returns: True: 明确支持(或无需检查 model_name) None: 未知(supported_models 为空,尚未探测) False: 已探测过模型列表,但明确不包含该模型 """ if not model_name: return True if not supported_models: return None model_set = {str(m).strip().lower() for m in supported_models if m} if not model_set: return None target = model_name.strip().lower() if target in model_set: return True # 兼容衍生模型别名映射 if target == "claude-sonnet-4-6-thinking" and "claude-sonnet-4-6" in model_set: return True if target == "claude-sonnet-4-6" and "claude-sonnet-4-6-thinking" in model_set: return True if target == "claude-opus-4-6" and "claude-opus-4-6-thinking" in model_set: return True if target == "claude-opus-4-6-thinking" and "claude-opus-4-6" in model_set: return True return False # ====================== 数据库管理器 ====================== class Database: """统一数据库管理器(SQLite / PostgreSQL)""" def __init__(self): self._engine: Optional[AsyncEngine] = None self._session_factory: Optional[async_sessionmaker[AsyncSession]] = None self._initialized = False self._lock = asyncio.Lock() self._backend_type = "sqlite" # 内存配置缓存 self._config_cache: Dict[str, Any] = {} self._config_loaded = False # ---------- 初始化 ---------- async def initialize(self) -> None: if self._initialized: return async with self._lock: if self._initialized: return postgres_uri = os.getenv("POSTGRESQL_URI", "") credentials_dir = os.getenv("CREDENTIALS_DIR", "./creds") os.makedirs(credentials_dir, exist_ok=True) if postgres_uri: # PostgreSQL(postgres:// 或 postgresql:// -> postgresql+asyncpg://) dsn = postgres_uri if dsn.startswith("postgresql://"): dsn = "postgresql+asyncpg://" + dsn[len("postgresql://"):] elif dsn.startswith("postgres://"): dsn = "postgresql+asyncpg://" + dsn[len("postgres://"):] elif not dsn.startswith("postgresql+"): dsn = "postgresql+asyncpg://" + dsn self._engine = create_async_engine(dsn, pool_size=5, max_overflow=5, pool_pre_ping=True) self._backend_type = "postgresql" else: # SQLite 默认 db_path = os.path.join(credentials_dir, "credentials.db") self._engine = create_async_engine( f"sqlite+aiosqlite:///{db_path}", # 连接池:多个长连接复用(默认 NullPool 每会话新建连接开销大); # WAL 模式允许读写并发,单写者由 busy_timeout 兜底 pool_size=5, max_overflow=10, pool_pre_ping=True, connect_args={"timeout": 30}, ) # WAL + NORMAL:读写并发 + 减少每次提交的 fsync(PRAGMA 需在连接建立后执行) from sqlalchemy import event @event.listens_for(self._engine.sync_engine, "connect") def _set_sqlite_pragma(dbapi_conn, conn_record): cursor = dbapi_conn.cursor() cursor.execute("PRAGMA journal_mode=WAL") cursor.execute("PRAGMA synchronous=NORMAL") cursor.execute("PRAGMA busy_timeout=15000") cursor.execute("PRAGMA cache_size=-8000") cursor.close() self._backend_type = "sqlite" self._session_factory = async_sessionmaker( self._engine, expire_on_commit=False, class_=AsyncSession ) # 建表(不兼容旧结构:旧的 filename 键结构表会被重建为 email 键结构) async with self._engine.begin() as conn: await conn.run_sync(self._create_tables) # 标记初始化完成(供 session() 使用),再加载配置缓存 self._initialized = True # 加载配置缓存 await self._load_config_cache() log.info(f"Database initialized (backend={self._backend_type})") def _create_tables(self, sync_conn) -> None: """建表。 - 凭证表:结构性变更(键从 filename 变为 email)时删除重建 - 其余表:缺列时用 ALTER TABLE ADD COLUMN 补齐,保留原有数据 """ inspector = inspect(sync_conn) table_name = AntigravityCredential.__tablename__ if inspector.has_table(table_name): existing_columns = {c["name"] for c in inspector.get_columns(table_name)} # 旧结构以 filename 为键、无 email 列 —— 删除重建(数据无法迁移) if "filename" in existing_columns or "email" not in existing_columns: log.info("检测到旧版凭证表结构(filename 键),重建为 email 键结构") sync_conn.execute(text(f"DROP TABLE {table_name}")) # 已存在的表补齐缺失列(ALTER TABLE ADD COLUMN,保留数据) for orm_model in (AntigravityCredential, UsageLog): table = orm_model.__table__ if not inspector.has_table(table.name): continue existing_columns = {c["name"] for c in inspector.get_columns(table.name)} dialect = inspector.bind.dialect for column in table.columns: if column.name in existing_columns: continue column_type = column.type.compile(dialect) # 补列:有 server_default 用之;有 Python default 用字面量;否则 DEFAULT NULL if column.server_default is not None: default_clause = f" DEFAULT {column.server_default.arg}" elif column.default is not None and column.default.is_scalar: default_clause = f" DEFAULT {column.default.arg!r}" else: default_clause = " DEFAULT NULL" sync_conn.execute( text(f"ALTER TABLE {table.name} ADD COLUMN {column.name} {column_type}{default_clause}") ) log.info(f"表 {table.name} 补充列 {column.name} ({column_type})") Base.metadata.create_all(sync_conn) async def close(self) -> None: if self._engine: await self._engine.dispose() self._engine = None self._session_factory = None self._initialized = False log.debug("Database closed") @property def backend_type(self) -> str: return self._backend_type def _ensure_initialized(self): if not self._initialized or not self._session_factory: raise RuntimeError("Database not initialized") @asynccontextmanager async def session(self): """获取数据库会话(自动提交/回滚)""" self._ensure_initialized() async with self._session_factory() as session: try: yield session await session.commit() except Exception: await session.rollback() raise # ---------- 凭证管理 ---------- async def store_credential( self, email: str, credential_data: Dict[str, Any] ) -> bool: """存储或更新凭证(以 email 为唯一键,保留状态字段)""" async with self.session() as s: row = await s.scalar( select(AntigravityCredential).where(AntigravityCredential.email == email) ) access_token = credential_data.get("access_token") or credential_data.get("token") or "" refresh_token = credential_data.get("refresh_token") or "" client_id = credential_data.get("client_id") or "" client_secret = credential_data.get("client_secret") or "" project_id = credential_data.get("project_id") or "" expiry_val = credential_data.get("expiry") or "" expiry = expiry_val.isoformat() if hasattr(expiry_val, "isoformat") else str(expiry_val) tier = credential_data.get("tier") supported_models = credential_data.get("supported_models") if row: row.access_token = access_token row.refresh_token = refresh_token row.client_id = client_id row.client_secret = client_secret row.project_id = project_id row.expiry = expiry if tier: row.tier = tier if isinstance(supported_models, list): row.supported_models = _dump_json(supported_models) row.updated_at = time.time() else: s.add(AntigravityCredential( email=email, access_token=access_token, refresh_token=refresh_token, client_id=client_id, client_secret=client_secret, project_id=project_id, expiry=expiry, tier=tier or "pro", supported_models=_dump_json(supported_models) if isinstance(supported_models, list) else "[]", last_success=time.time(), )) log.debug(f"Stored credential: {email}") return True async def get_credential( self, email: str ) -> Optional[Dict[str, Any]]: """获取凭证数据""" async with self.session() as s: row = await s.scalar( select(AntigravityCredential).where(AntigravityCredential.email == email) ) return _row_to_credential_dict(row) if row else None async def list_credentials(self) -> List[str]: """列出所有凭证 email(创建顺序)""" async with self.session() as s: rows = await s.scalars( select(AntigravityCredential.email).order_by(AntigravityCredential.id) ) return list(rows) async def delete_credential(self, email: str) -> bool: """删除凭证""" async with self.session() as s: row = await s.scalar( select(AntigravityCredential).where(AntigravityCredential.email == email) ) if row: await s.delete(row) return True return False async def get_next_available_credential( self, model_name: Optional[str] = None, exclude_emails: Optional[Set[str]] = None, ) -> Optional[Tuple[str, Dict[str, Any]]]: """随机获取一个可用凭证(负载均衡 + 模型家族级冷却过滤 + 模型能力匹配),返回 (email, credential)""" current_time = time.time() cd_group = get_cooldown_group(model_name) if model_name else None excluded = {e.strip().lower() for e in exclude_emails if e} if exclude_emails else set() async with self.session() as s: # 轻量查询:只取必要的小字段(email/冷却/支持模型/禁用),避免全表加载 # access_token/refresh_token 等大字段(凭证明文可达数 KB) rows = ( await s.execute( select( AntigravityCredential.email, AntigravityCredential.model_cooldowns, AntigravityCredential.supported_models, ) .where(AntigravityCredential.disabled == False) # noqa: E712 .order_by(AntigravityCredential.id) ) ).all() if not rows: return None # Python 侧过滤:排除已失败凭证 + 模型家族冷却 + 模型支持检查 matched_candidates: List[str] = [] unknown_candidates: List[str] = [] for email, model_cooldowns, supported_models_raw in rows: if excluded and email.strip().lower() in excluded: continue if cd_group: cooldowns = _load_json(model_cooldowns, {}) cd = cooldowns.get(cd_group) if cd is not None and current_time < cd: continue supported_models = _load_json(supported_models_raw, []) support_state = _credential_supports_model( supported_models if isinstance(supported_models, list) else [], model_name, ) if support_state is True: matched_candidates.append(email) elif support_state is None: unknown_candidates.append(email) # 优先从明确支持该模型的凭证中随机选;若无,则从尚未探测过模型列表的凭证中选 pool = matched_candidates if matched_candidates else unknown_candidates if not pool: return None picked_email = random.choice(pool) # 只取被选中凭证的完整数据 row = await s.scalar( select(AntigravityCredential).where( AntigravityCredential.email == picked_email ) ) if not row or row.disabled: return None return row.email, _row_to_credential_dict(row) async def get_available_credential_by_email( self, email: str, model_name: Optional[str] = None, exclude_emails: Optional[Set[str]] = None, ) -> Optional[Tuple[str, Dict[str, Any]]]: """ 按指定 email 获取可用凭证(凭证亲和/粘性模式使用)。 凭证不存在/被禁用/该模型家族冷却中/明确不支持该模型 -> None。 """ if not email: return None if exclude_emails and email.strip().lower() in {e.strip().lower() for e in exclude_emails if e}: return None current_time = time.time() cd_group = get_cooldown_group(model_name) if model_name else None async with self.session() as s: row = await s.scalar( select(AntigravityCredential).where(AntigravityCredential.email == email) ) if not row or row.disabled: return None if cd_group: cooldowns = _load_json(row.model_cooldowns, {}) cd = cooldowns.get(cd_group) if cd is not None and current_time < cd: return None supported_models = _load_json(row.supported_models, []) support_state = _credential_supports_model( supported_models if isinstance(supported_models, list) else [], model_name, ) if support_state is False: return None return row.email, _row_to_credential_dict(row) # ---------- 状态管理 ---------- async def update_credential_state( self, email: str, state_updates: Dict[str, Any] ) -> bool: """更新凭证状态字段""" json_fields = {"error_codes", "error_messages", "model_cooldowns", "supported_models"} float_fields = {"last_success"} str_fields = {"user_email", "tier", "project_id"} bool_fields = {"disabled"} async with self.session() as s: values: Dict[str, Any] = {} for key, value in state_updates.items(): if key in json_fields: values[key] = _dump_json(value) elif key in float_fields: values[key] = float(value) if value is not None else None elif key in str_fields: values[key] = value elif key in bool_fields: values[key] = bool(value) if not values: return True values["updated_at"] = time.time() result = await s.execute( AntigravityCredential.__table__.update() .where(AntigravityCredential.email == email) .values(**values) ) return result.rowcount > 0 async def get_credential_state( self, email: str ) -> Dict[str, Any]: """获取凭证状态(不含 error_messages)""" async with self.session() as s: row = await s.scalar( select(AntigravityCredential).where(AntigravityCredential.email == email) ) if not row: return {} return { "disabled": bool(row.disabled), "error_codes": _load_json(row.error_codes, []), "last_success": row.last_success or time.time(), "email": row.email, "model_cooldowns": _load_json(row.model_cooldowns, {}), "supported_models": _load_json(row.supported_models, []), "tier": row.tier or "pro", "project_id": row.project_id or "", } async def get_all_credential_states(self) -> Dict[str, Dict[str, Any]]: """获取所有凭证状态(自动过滤过期CD)""" current_time = time.time() async with self.session() as s: rows = (await s.scalars(select(AntigravityCredential))).all() states = {} for row in rows: cooldowns = _load_json(row.model_cooldowns, {}) cooldowns = {k: v for k, v in cooldowns.items() if v > current_time} states[row.email] = { "disabled": bool(row.disabled), "error_codes": _load_json(row.error_codes, []), "last_success": row.last_success or time.time(), "email": row.email, "model_cooldowns": cooldowns, "supported_models": _load_json(row.supported_models, []), "tier": row.tier or "pro", "project_id": row.project_id or "", } return states async def get_enabled_credentials_models_info(self) -> Tuple[List[str], List[str]]: """ 获取所有启用凭证的模型并集与尚未探测模型列表的启用凭证 email 列表。 Returns: (union_model_ids, unprobed_emails) """ async with self.session() as s: rows = ( await s.execute( select( AntigravityCredential.email, AntigravityCredential.supported_models, ) .where(AntigravityCredential.disabled == False) # noqa: E712 .order_by(AntigravityCredential.id) ) ).all() seen: Set[str] = set() union_models: List[str] = [] unprobed_emails: List[str] = [] for email, supported_models_raw in rows: models = _load_json(supported_models_raw, []) if not isinstance(models, list) or not models: unprobed_emails.append(email) continue for m in models: if isinstance(m, str) and m and m not in seen: seen.add(m) union_models.append(m) return union_models, unprobed_emails async def remove_model_from_credential(self, email: str, model_name: str) -> bool: """从指定凭证的 supported_models 中移除某个不支持的模型""" if not email or not model_name: return False async with self.session() as s: row = await s.scalar( select(AntigravityCredential).where(AntigravityCredential.email == email) ) if not row: return False models = _load_json(row.supported_models, []) if not isinstance(models, list) or not models: return False target = model_name.strip().lower() # 同时移除衍生映射模型名 remove_targets = {target} if target == "claude-sonnet-4-6": remove_targets.add("claude-sonnet-4-6-thinking") elif target == "claude-sonnet-4-6-thinking": remove_targets.add("claude-sonnet-4-6") elif target == "claude-opus-4-6-thinking": remove_targets.add("claude-opus-4-6") elif target == "claude-opus-4-6": remove_targets.add("claude-opus-4-6-thinking") new_models = [ m for m in models if isinstance(m, str) and m.strip().lower() not in remove_targets ] if len(new_models) != len(models): row.supported_models = _dump_json(new_models) row.updated_at = time.time() return True return False async def get_credentials_summary( self, offset: int = 0, limit: Optional[int] = None, status_filter: str = "all", error_code_filter: Optional[str] = None, cooldown_filter: Optional[str] = None, tier_filter: Optional[str] = None, ) -> Dict[str, Any]: """凭证摘要(分页 + 状态筛选,内存过滤)""" current_time = time.time() async with self.session() as s: total = await s.scalar(select(func.count()).select_from(AntigravityCredential)) or 0 disabled_count = await s.scalar( select(func.count()).select_from(AntigravityCredential) .where(AntigravityCredential.disabled == True) # noqa: E712 ) or 0 stmt = select(AntigravityCredential).order_by(AntigravityCredential.id) if status_filter == "enabled": stmt = stmt.where(AntigravityCredential.disabled == False) # noqa: E712 elif status_filter == "disabled": stmt = stmt.where(AntigravityCredential.disabled == True) # noqa: E712 rows = (await s.scalars(stmt)).all() filter_value = None filter_int = None filter_none = False if error_code_filter and error_code_filter.strip().lower() != "all": if error_code_filter.strip().lower() == "none": filter_none = True else: filter_value = error_code_filter.strip() try: filter_int = int(filter_value) except ValueError: filter_int = None summaries = [] for row in rows: error_codes = _load_json(row.error_codes, []) cooldowns = _load_json(row.model_cooldowns, {}) active_cooldowns = {k: v for k, v in cooldowns.items() if v > current_time} if filter_none and error_codes: continue if filter_value is not None: match = False for code in error_codes: if code == filter_value or code == filter_int: match = True break if isinstance(code, str) and filter_int is not None: try: if int(code) == filter_int: match = True break except ValueError: pass if not match: continue supported_models = _load_json(row.supported_models, []) if not isinstance(supported_models, list): supported_models = [] summary = { "email": row.email, "disabled": bool(row.disabled), "error_codes": error_codes, "last_success": row.last_success or current_time, "model_cooldowns": active_cooldowns, "supported_models": supported_models, "tier": row.tier or "pro", "project_id": row.project_id or "", } if tier_filter and tier_filter in ("free", "pro", "ultra"): if summary["tier"] != tier_filter: continue if cooldown_filter == "in_cooldown" and not active_cooldowns: continue if cooldown_filter == "no_cooldown" and active_cooldowns: continue summaries.append(summary) total_count = len(summaries) page_items = summaries[offset:offset + limit] if limit is not None else summaries[offset:] return { "items": page_items, "total": total_count, "offset": offset, "limit": limit, "stats": { "total": total, "normal": total - disabled_count, "disabled": disabled_count, }, } async def get_duplicate_credentials_by_email(self) -> Dict[str, Any]: """按邮箱分组的重复凭证(email 即唯一键,此结构保留用于展示统计)""" async with self.session() as s: rows = (await s.scalars(select(AntigravityCredential).order_by(AntigravityCredential.id))).all() emails = [row.email for row in rows] return { "email_groups": {}, "duplicate_groups": [], "duplicate_count": 0, "no_email_files": [], "no_email_count": 0, "unique_email_count": len(emails), "total_count": len(rows), } async def get_credential_errors(self, email: str) -> Dict[str, Any]: """获取凭证的错误信息""" async with self.session() as s: row = await s.scalar( select(AntigravityCredential).where(AntigravityCredential.email == email) ) if row: return { "email": email, "error_codes": _load_json(row.error_codes, []), "error_messages": _load_json(row.error_messages, []), } return {"email": email, "error_codes": [], "error_messages": []} # ---------- 模型级冷却 ---------- async def set_model_cooldown( self, email: str, model_name: str, cooldown_until: Optional[float], ) -> bool: """设置/清除模型的冷却时间(按家族分组:image / argon / gemini / claude_gpt / 其他原名)""" cd_group = get_cooldown_group(model_name) async with self.session() as s: row = await s.scalar( select(AntigravityCredential).where(AntigravityCredential.email == email) ) if not row: return False cooldowns = _load_json(row.model_cooldowns, {}) if cooldown_until is None: cooldowns.pop(cd_group, None) else: cooldowns[cd_group] = cooldown_until row.model_cooldowns = _dump_json(cooldowns) row.updated_at = time.time() return True async def clear_all_model_cooldowns(self, email: str) -> bool: """清除某凭证的所有模型冷却""" async with self.session() as s: row = await s.scalar( select(AntigravityCredential).where(AntigravityCredential.email == email) ) if not row: return False row.model_cooldowns = "{}" row.updated_at = time.time() return True async def record_success( self, email: str, model_name: Optional[str] = None ) -> None: """成功调用后的条件写入(清除错误 + 该模型家族的CD)""" async with self.session() as s: row = await s.scalar( select(AntigravityCredential).where(AntigravityCredential.email == email) ) if not row: return changed = False if _load_json(row.error_codes, []): row.error_codes = "[]" row.error_messages = "{}" row.last_success = time.time() row.updated_at = time.time() changed = True if model_name: cd_group = get_cooldown_group(model_name) cooldowns = _load_json(row.model_cooldowns, {}) if cd_group in cooldowns: cooldowns.pop(cd_group) row.model_cooldowns = _dump_json(cooldowns) row.updated_at = time.time() changed = True if changed: log.debug(f"Recorded success for {email} (model={model_name})") # ---------- 配置管理(内存缓存) ---------- async def _load_config_cache(self) -> None: async with self.session() as s: rows = (await s.scalars(select(ConfigItem))).all() for item in rows: try: self._config_cache[item.key] = json.loads(item.value) except (json.JSONDecodeError, TypeError): self._config_cache[item.key] = item.value self._config_loaded = True log.debug(f"Loaded {len(self._config_cache)} config items") async def reload_config_cache(self) -> None: self._ensure_initialized() self._config_cache.clear() await self._load_config_cache() log.info("Config cache reloaded from database") async def set_config(self, key: str, value: Any) -> bool: """设置配置(写库 + 更新缓存)""" async with self.session() as s: item = await s.get(ConfigItem, key) serialized = _dump_json(value) if item: item.value = serialized item.updated_at = time.time() else: s.add(ConfigItem(key=key, value=serialized)) self._config_cache[key] = value return True async def get_config(self, key: str, default: Any = None) -> Any: self._ensure_initialized() return self._config_cache.get(key, default) async def get_all_config(self) -> Dict[str, Any]: self._ensure_initialized() return self._config_cache.copy() async def delete_config(self, key: str) -> bool: async with self.session() as s: item = await s.get(ConfigItem, key) if item: await s.delete(item) self._config_cache.pop(key, None) return True return False # ---------- 请求日志(usage_log) ---------- async def insert_usage_log(self, entry: Dict[str, Any]) -> bool: """写入一条请求日志(失败仅告警,不影响主流程)""" try: async with self.session() as s: s.add(UsageLog( created_at=entry.get("created_at") or time.time(), api_format=str(entry.get("api_format", "")), client_ip=str(entry.get("client_ip", "")), request_id=str(entry.get("request_id", "")), model_name=str(entry.get("model_name", "")), upstream_model_name=str(entry.get("upstream_model_name", "")), credential_email=str(entry.get("credential_email", "")), is_stream=bool(entry.get("is_stream", False)), status_code=int(entry.get("status_code", 0) or 0), success=bool(entry.get("success", False)), use_time=float(entry.get("use_time", 0.0) or 0.0), prompt_tokens=int(entry.get("prompt_tokens", 0) or 0), completion_tokens=int(entry.get("completion_tokens", 0) or 0), total_tokens=int(entry.get("total_tokens", 0) or 0), cached_tokens=int(entry.get("cached_tokens", 0) or 0), error_message=str(entry.get("error_message", ""))[:2000], request_log_file=str(entry.get("request_log_file", ""))[:512], )) return True except SQLAlchemyError as e: log.warning(f"写入请求日志失败: {e}") return False async def query_usage_logs( self, offset: int = 0, limit: int = 50, start_timestamp: Optional[float] = None, end_timestamp: Optional[float] = None, model_name: str = "", credential_email: str = "", api_format: str = "", status: str = "all", # all / success / error is_stream: Optional[bool] = None, status_code: Optional[int] = None, ) -> Tuple[List[Dict[str, Any]], int]: """分页查询请求日志,返回 (日志列表, 总数),新 -> 旧""" stmt = select(UsageLog) count_stmt = select(func.count()).select_from(UsageLog) if start_timestamp: stmt = stmt.where(UsageLog.created_at >= start_timestamp) count_stmt = count_stmt.where(UsageLog.created_at >= start_timestamp) if end_timestamp: stmt = stmt.where(UsageLog.created_at <= end_timestamp) count_stmt = count_stmt.where(UsageLog.created_at <= end_timestamp) if model_name: stmt = stmt.where(UsageLog.model_name == model_name) count_stmt = count_stmt.where(UsageLog.model_name == model_name) if credential_email: stmt = stmt.where(UsageLog.credential_email == credential_email) count_stmt = count_stmt.where(UsageLog.credential_email == credential_email) if api_format: stmt = stmt.where(UsageLog.api_format == api_format) count_stmt = count_stmt.where(UsageLog.api_format == api_format) if status_code is not None: stmt = stmt.where(UsageLog.status_code == status_code) count_stmt = count_stmt.where(UsageLog.status_code == status_code) if status == "success": stmt = stmt.where(UsageLog.success.is_(True)) count_stmt = count_stmt.where(UsageLog.success.is_(True)) elif status == "error": stmt = stmt.where(UsageLog.success.is_(False)) count_stmt = count_stmt.where(UsageLog.success.is_(False)) if is_stream is not None: stmt = stmt.where(UsageLog.is_stream.is_(is_stream)) count_stmt = count_stmt.where(UsageLog.is_stream.is_(is_stream)) async with self.session() as s: total = (await s.execute(count_stmt)).scalar() or 0 rows = ( ( await s.execute( stmt.order_by(UsageLog.created_at.desc(), UsageLog.id.desc()) .offset(offset) .limit(limit) ) ) .scalars() .all() ) items = [ { "id": r.id, "created_at": r.created_at, "api_format": r.api_format or "", "client_ip": r.client_ip or "", "request_id": r.request_id or "", "model_name": r.model_name or "", "upstream_model_name": r.upstream_model_name or "", "credential_email": r.credential_email or "", "is_stream": r.is_stream, "status_code": r.status_code, "success": r.success, "use_time": r.use_time, "prompt_tokens": r.prompt_tokens, "completion_tokens": r.completion_tokens, "total_tokens": r.total_tokens, "cached_tokens": r.cached_tokens or 0, "error_message": r.error_message or "", "request_log_file": r.request_log_file or "", } for r in rows ] return items, total async def usage_log_stats( self, start_timestamp: Optional[float] = None, end_timestamp: Optional[float] = None, model_name: str = "", credential_email: str = "", api_format: str = "", ) -> Dict[str, Any]: """聚合统计:总请求数 / 成功数 / 失败数 / token 汇总 / 平均耗时 / 每模型聚合""" conditions = [] if start_timestamp: conditions.append(UsageLog.created_at >= start_timestamp) if end_timestamp: conditions.append(UsageLog.created_at <= end_timestamp) if model_name: conditions.append(UsageLog.model_name == model_name) if credential_email: conditions.append(UsageLog.credential_email == credential_email) if api_format: conditions.append(UsageLog.api_format == api_format) success_cond = func.cast(UsageLog.success, Integer).label("success_int") agg = select( func.count().label("total_requests"), func.coalesce(func.sum(success_cond), 0).label("success_count"), func.coalesce(func.sum(UsageLog.prompt_tokens), 0).label("prompt_tokens"), func.coalesce(func.sum(UsageLog.completion_tokens), 0).label("completion_tokens"), func.coalesce(func.sum(UsageLog.total_tokens), 0).label("total_tokens"), func.coalesce(func.sum(UsageLog.cached_tokens), 0).label("cached_tokens"), func.coalesce(func.avg(UsageLog.use_time), 0).label("avg_use_time"), ).where(*conditions) by_model = ( select( UsageLog.model_name.label("model_name"), func.count().label("count"), func.coalesce(func.sum(UsageLog.total_tokens), 0).label("total_tokens"), ) .where(*conditions) .group_by(UsageLog.model_name) .order_by(func.count().desc()) .limit(50) ) async with self.session() as s: row = (await s.execute(agg)).one() model_rows = (await s.execute(by_model)).all() return { "total_requests": row.total_requests or 0, "success_count": row.success_count or 0, "error_count": (row.total_requests or 0) - (row.success_count or 0), "prompt_tokens": row.prompt_tokens or 0, "completion_tokens": row.completion_tokens or 0, "total_tokens": row.total_tokens or 0, "cached_tokens": row.cached_tokens or 0, "avg_use_time": round(float(row.avg_use_time or 0), 2), "by_model": [ {"model_name": m.model_name or "(unknown)", "count": m.count, "total_tokens": m.total_tokens} for m in model_rows ], } async def delete_usage_logs_before(self, before_timestamp: float) -> int: """删除指定时间之前的日志,返回删除条数""" async with self.session() as s: result = await s.execute( text("DELETE FROM usage_log WHERE created_at < :ts"), {"ts": before_timestamp}, ) return result.rowcount or 0 # ---------- 后端信息 ---------- def get_backend_info(self) -> Dict[str, Any]: return { "backend_type": self._backend_type, "initialized": self._initialized, "database_path": ( os.path.join(os.getenv("CREDENTIALS_DIR", "./creds"), "credentials.db") if self._backend_type == "sqlite" else None ), "credentials_dir": os.getenv("CREDENTIALS_DIR", "./creds"), } # ====================== 全局单例 ====================== _database: Optional[Database] = None _database_lock = asyncio.Lock() async def get_database() -> Database: """获取全局数据库实例""" global _database if _database is None: async with _database_lock: if _database is None: db = Database() await db.initialize() _database = db return _database async def close_database() -> None: """关闭全局数据库""" global _database if _database: await _database.close() _database = None
"""
数据库层(SQLAlchemy 2.0 异步 ORM)
统一支持 SQLite(默认)与 PostgreSQL。
凭证以 email 为唯一键;schema 变更不做旧数据兼容(新表直接重建)。
"""
import asyncio
import json
import os
import random
import time
from contextlib import asynccontextmanager
from typing import Any, Dict, List, Optional, Set, Tuple
from sqlalchemy import (
Boolean,
Float,
Index,
Integer,
String,
Text,
func,
inspect,
select,
text,
)
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
from log import log
# ====================== ORM 模型 ======================
class Base(DeclarativeBase):
pass
class AntigravityCredential(Base):
"""Antigravity 凭证表(email 唯一键)"""
__tablename__ = "antigravity_credentials"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
email: Mapped[str] = mapped_column(String(320), unique=True, nullable=False)
# 凭证结构化字段
access_token: Mapped[str] = mapped_column(Text, default="")
refresh_token: Mapped[str] = mapped_column(Text, default="")
client_id: Mapped[str] = mapped_column(Text, default="")
client_secret: Mapped[str] = mapped_column(Text, default="")
project_id: Mapped[str] = mapped_column(Text, default="")
expiry: Mapped[str] = mapped_column(Text, default="")
# 状态字段
disabled: Mapped[bool] = mapped_column(Boolean, default=False)
error_codes: Mapped[str] = mapped_column(Text, default="[]")
error_messages: Mapped[str] = mapped_column(Text, default="[]")
last_success: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
# 模型级冷却 (JSON: {model_name: cooldown_timestamp})
model_cooldowns: Mapped[str] = mapped_column(Text, default="{}")
# 凭证支持的模型列表 (JSON: [model_id, ...];空数组 [] 表示尚未探测)
supported_models: Mapped[str] = mapped_column(Text, default="[]")
# tier 状态 (free/pro/ultra)
tier: Mapped[Optional[str]] = mapped_column(String(32), default="pro")
# 时间戳
created_at: Mapped[float] = mapped_column(Float, default=time.time)
updated_at: Mapped[float] = mapped_column(Float, default=time.time, onupdate=time.time)
__table_args__ = (
Index("idx_ag_disabled", "disabled"),
)
class ConfigItem(Base):
"""配置表"""
__tablename__ = "config"
key: Mapped[str] = mapped_column(String(255), primary_key=True)
value: Mapped[str] = mapped_column(Text, nullable=False)
updated_at: Mapped[float] = mapped_column(Float, default=time.time)
class UsageLog(Base):
"""请求日志表:每次 API 调用记录一条(成功与失败均记录)"""
__tablename__ = "usage_log"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
created_at: Mapped[float] = mapped_column(Float, default=time.time, index=True)
# 请求来源
api_format: Mapped[str] = mapped_column(String(16), default="") # openai / gemini / anthropic
client_ip: Mapped[str] = mapped_column(String(64), default="")
request_id: Mapped[str] = mapped_column(String(64), default="")
# 调用信息