File size: 12,309 Bytes
b0074f5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
"""
凭证管理(无状态函数封装)
以 email 为唯一键管理 Antigravity 凭证:负载均衡选取、自动 token 刷新、错误记录与模型级冷却。

全部函数直接使用全局数据库单例(get_database),无实例状态;
唯一的状态是 sticky 模式的当前粘性凭证指针(进程内模块级变量)。
"""

import asyncio
from datetime import datetime, timezone
from typing import Any, Dict, Optional, Set, Tuple

from log import log

from config import get_credential_select_mode
from src import session_cache
from src.google_oauth_api import Credentials
from src.storage.database import get_database

# sticky 模式的当前粘性凭证(进程内唯一状态;重置函数供测试使用)
_sticky_email: Optional[str] = None


def _reset_sticky() -> None:
    """清除粘性凭证指针(测试用)"""
    global _sticky_email
    _sticky_email = None


# ====================== 凭证选取 ======================

async def get_valid_credential(
    model_name: Optional[str] = None,
    session_key: Optional[str] = None,
    exclude_emails: Optional[Set[str]] = None,
) -> Optional[Tuple[str, Dict[str, Any]]]:
    """
    获取可用凭证:按凭证选择模式选取,自动跳过禁用/冷却/不支持目标模型的凭证。
    Token 失效时自动刷新;刷新失败(凭证永久失效)自动禁用并尝试下一个。

    凭证选择模式(config: credential_select_mode):
    - random: 完全随机负载均衡(忽略 session_key)
    - cache:  会话缓存匹配——session_key 命中绑定时复用该凭证(prompt cache 亲和)
    - sticky: 粘性——尽量保持同一凭证直到其进入 CD 或被禁用

    Returns:
        (email, credential_data) 或 None
    """
    global _sticky_email
    database = await get_database()
    mode = await get_credential_select_mode()
    tried_emails: Set[str] = set(exclude_emails) if exclude_emails else set()

    # cache 模式:先查会话绑定
    if mode == "cache" and session_key:
        bound_email = await session_cache.get_session_binding(session_key)
        if bound_email:
            result = await database.get_available_credential_by_email(
                bound_email,
                model_name=model_name,
                exclude_emails=tried_emails,
            )
            if result:
                log.debug(f"会话缓存命中: {bound_email}")
                final = await _validate_or_refresh(result[0], result[1])
                if final is None:
                    # 绑定凭证 token 永久失效等,解绑走正常选取
                    tried_emails.add(bound_email)
                    await session_cache.delete_session_binding(session_key)
                else:
                    return final
            else:
                # 绑定凭证不可用(禁用/CD/删除/不支持当前模型),解绑
                await session_cache.delete_session_binding(session_key)

    # sticky 模式:优先当前粘性凭证
    if mode == "sticky" and _sticky_email:
        result = await database.get_available_credential_by_email(
            _sticky_email,
            model_name=model_name,
            exclude_emails=tried_emails,
        )
        if result:
            final = await _validate_or_refresh(result[0], result[1])
            if final is not None:
                return final
            tried_emails.add(_sticky_email)
        # 粘性凭证 CD/禁用/失效/不支持当前模型,清除指针换新
        _sticky_email = None

    # 通用随机选取(random 模式 / cache 未命中 / sticky 无可用粘性凭证)
    max_retries = 3
    for attempt in range(max_retries):
        result = await database.get_next_available_credential(
            model_name=model_name,
            exclude_emails=tried_emails,
        )

        if not result:
            if attempt == 0:
                log.warning(f"没有可用凭证 (model_name={model_name})")
            return None

        email, credential_data = result
        tried_emails.add(email)

        if await should_refresh_token(credential_data):
            log.debug(f"Token需要刷新: {email}")
            refreshed = await refresh_token(credential_data, email)
            if refreshed:
                await _after_picked(mode, email, session_key)
                return email, refreshed
            log.warning(
                f"Token刷新失败,尝试下一个凭证: {email} (attempt={attempt + 1}/{max_retries})"
            )
            continue

        await _after_picked(mode, email, session_key)
        return email, credential_data

    log.error(f"重试{max_retries}次后仍无可用凭证 (model_name={model_name})")
    return None


async def _validate_or_refresh(
    email: str, credential_data: Dict[str, Any]
) -> Optional[Tuple[str, Dict[str, Any]]]:
    """对已选中的凭证做 token 校验;需要刷新则刷新并返回,永久失效返回 None"""
    if await should_refresh_token(credential_data):
        refreshed = await refresh_token(credential_data, email)
        if refreshed is None:
            return None
        credential_data = refreshed
    return email, credential_data


async def _after_picked(mode: str, email: str, session_key: Optional[str]) -> None:
    """凭证选取成功后的模式相关处理"""
    global _sticky_email
    try:
        if mode == "cache" and session_key:
            await session_cache.set_session_binding(session_key, email)
        elif mode == "sticky":
            _sticky_email = email
    except Exception as e:
        log.debug(f"记录凭证亲和失败(不影响请求): {e}")


# ====================== 凭证增删改 ======================

async def add_credential(email: str, credential_data: Dict[str, Any]):
    """新增或更新凭证(同 email 覆盖,自动去重)"""
    database = await get_database()
    await database.store_credential(email, credential_data)
    log.info(f"Credential added/updated: {email}")


async def remove_credential(email: str) -> bool:
    """删除凭证"""
    database = await get_database()
    try:
        await database.delete_credential(email)
        log.info(f"Credential removed: {email}")
        return True
    except Exception as e:
        log.error(f"Error removing credential {email}: {e}")
        return False


async def update_credential_state(email: str, state_updates: Dict[str, Any]) -> bool:
    """更新凭证状态字段(disabled/error_codes/tier/project_id 等)"""
    database = await get_database()
    try:
        success = await database.update_credential_state(email, state_updates)
        if not success:
            log.warning(f"Failed to update credential state: {email}")
        return success
    except Exception as e:
        log.error(f"Error updating credential state {email}: {e}")
        return False


async def set_cred_disabled(email: str, disabled: bool):
    """设置凭证的启用/禁用状态"""
    success = await update_credential_state(email, {"disabled": disabled})
    if success:
        log.info(f"Credential {'disabled' if disabled else 'enabled'}: {email}")
    return success


# ====================== API 调用结果记录 ======================

async def record_api_call_result(
    email: str,
    success: bool,
    error_code: Optional[int] = None,
    cooldown_until: Optional[float] = None,
    model_name: Optional[str] = None,
    error_message: Optional[str] = None,
):
    """
    记录 API 调用结果:
    - 成功:清除错误状态与该模型的冷却(fire-and-forget)
    - 失败:记录错误码/错误信息,必要时设置模型级冷却
    """
    database = await get_database()
    try:
        if success:
            asyncio.create_task(database.record_success(email, model_name=model_name))
        elif error_code:
            error_messages = {}
            if error_message:
                error_messages[str(error_code)] = error_message

            await update_credential_state(email, {
                "error_codes": [error_code],
                "error_messages": error_messages,
            })

            if cooldown_until is not None and model_name:
                await database.set_model_cooldown(email, model_name, cooldown_until)
                log.info(
                    f"设置模型级冷却: {email}, model={model_name}, "
                    f"冷却至: {datetime.fromtimestamp(cooldown_until, timezone.utc).isoformat()}"
                )
    except Exception as e:
        log.error(f"Error recording API call result for {email}: {e}")


# ====================== Token 刷新 ======================

async def should_refresh_token(credential_data: Dict[str, Any]) -> bool:
    """检查 token 是否需要刷新(剩余有效期不足 5 分钟视为需要刷新)"""
    access_token = credential_data.get("access_token") or credential_data.get("token")
    if not access_token:
        return True

    expiry_str = credential_data.get("expiry")
    if not expiry_str:
        return True

    try:
        if not isinstance(expiry_str, str):
            return True
        if "+" in expiry_str:
            expiry = datetime.fromisoformat(expiry_str)
        elif expiry_str.endswith("Z"):
            expiry = datetime.fromisoformat(expiry_str.replace("Z", "+00:00"))
        else:
            expiry = datetime.fromisoformat(expiry_str)

        if expiry.tzinfo is None:
            expiry = expiry.replace(tzinfo=timezone.utc)

        time_left = (expiry - datetime.now(timezone.utc)).total_seconds()
        return time_left <= 300  # 5 分钟缓冲

    except Exception as e:
        log.warning(f"解析过期时间失败: {e},需要刷新")
        return True


async def refresh_token(
    credential_data: Dict[str, Any], email: str
) -> Optional[Dict[str, Any]]:
    """刷新 token 并保存;凭证永久失效时自动禁用,返回 None"""
    database = await get_database()
    try:
        creds = Credentials.from_dict(credential_data)

        if not creds.refresh_token:
            log.error(f"没有refresh_token,无法刷新: {email}")
            await update_credential_state(email, {"disabled": True})
            log.warning(f"凭证已自动禁用(缺少refresh_token): {email}")
            return None

        log.debug(f"正在刷新token: {email}")
        await creds.refresh()

        if creds.access_token:
            credential_data["access_token"] = creds.access_token
            credential_data["token"] = creds.access_token
        if creds.expires_at:
            credential_data["expiry"] = creds.expires_at.isoformat() if hasattr(creds.expires_at, "isoformat") else str(creds.expires_at)

        await database.store_credential(email, credential_data)
        log.info(f"Token刷新成功并已保存: {email}")
        return credential_data

    except Exception as e:
        error_msg = str(e)
        log.error(f"Token刷新失败 {email}: {error_msg}")

        status_code = getattr(e, "status_code", None)

        if _is_permanent_refresh_failure(error_msg, status_code):
            log.warning(f"检测到凭证永久失效 (HTTP {status_code}): {email}")
            await record_api_call_result(email, False, error_code=status_code or 400)
            if await update_credential_state(email, {"disabled": True}):
                log.warning(f"永久失效凭证已禁用: {email}")
        else:
            log.warning(f"Token刷新失败但非永久性错误 (HTTP {status_code}),不封禁凭证: {email}")

        return None


def _is_permanent_refresh_failure(error_msg: str, status_code: Optional[int] = None) -> bool:
    """判断是否是凭证永久失效(客户端错误/明确失效错误)"""
    if status_code is not None:
        if status_code in (400, 401, 403):
            return True
        if status_code in (500, 502, 503, 504):
            return False
        if status_code == 429:
            return False

    permanent_patterns = (
        "invalid_grant",
        "refresh_token_expired",
        "invalid_refresh_token",
        "unauthorized_client",
        "access_denied",
    )
    error_msg_lower = error_msg.lower()
    return any(p in error_msg_lower for p in permanent_patterns)