xiufu jiandeng yu shangdian huxiang dengdai de sizi suo; tanzhen buzai zhudong tuichu

api/app.py

@@ -213,14 +213,15 @@ def create_app() -> FastAPI:
                     "strikes": strikes,
                     "threads": active_count(),
                 })
-                if strikes >= 3:
-                    _logger.error({"event": "event_loop_wedged", "action": "dump_stacks_and_exit"})
+                # ⚠️ 只抓堆栈、**不主动退出**：os._exit 会让 HF 把 Space 置为
+                # RUNTIME_ERROR（不会自动重建），反而要人工点重启；只留证据更好。
+                if strikes == 3 or strikes % 12 == 0:
+                    _logger.error({"event": "event_loop_wedged", "action": "dump_stacks"})
                     try:
                         faulthandler.dump_traceback(file=sys.stderr, all_threads=True)
                     except Exception:
                         pass
                     sys.stderr.flush()
-                    os._exit(70)
 
         Thread(target=run, daemon=True, name="loop-wedge-guard").start()
 

services/auth_service.py

@@ -7,7 +7,7 @@ import time
 import uuid
 from datetime import datetime, timedelta, timezone
 from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
-from threading import Lock
+from threading import RLock
 from typing import Callable, Literal, TypeVar
 
 try:
@@ -462,7 +462,7 @@ class AuthService:
 
     def __init__(self, storage: StorageBackend):
         self.storage = storage
-        self._lock = Lock()
+        self._lock = RLock()
         self._items = self._load()
         # key_id -> 上次写 last_used_at 的时间戳；只在本进程内存里，重启即清空（无害）。
         self._last_touch_ts: dict[str, float] = {}
@@ -592,19 +592,26 @@ class AuthService:
         update_auth_keys_locked = getattr(self.storage, "update_auth_keys_locked", None)
         if callable(update_auth_keys_locked):
             def wrapped(raw_items: list[dict[str, object]]) -> tuple[list[dict[str, object]], ResultT]:
-                current = self._normalize_items(raw_items, fallback=self._items)
-                self._items = current
+                # ⚠️ 这里已经持有**存储**锁（update_auth_keys_locked 内部），再去拿鉴权锁时
+                # 必须保证没有「持鉴权锁等存储锁」的线程 —— 这就是 ABBA 死锁的另一半。
+                # 现在 authenticate 已改为「先放鉴权锁、再写存储」，加上 list_keys 不再回读
+                # 存储，环被打断；锁顺序统一为 存储 → 鉴权。
+                with self._lock:
+                    current = self._normalize_items(raw_items, fallback=self._items)
+                    self._items = current
                 next_items, result = mutator(current)
-                normalized_next = self._normalize_items(next_items, fallback=current)
-                self._items = normalized_next
+                with self._lock:
+                    normalized_next = self._normalize_items(next_items, fallback=current)
+                    self._items = normalized_next
                 return normalized_next, result
 
             return update_auth_keys_locked(wrapped)
 
-        self._reload_locked()
-        next_items, result = mutator(self._items)
-        self._items = self._normalize_items(next_items, fallback=self._items)
-        self._save()
+        with self._lock:
+            self._reload_locked()
+            next_items, result = mutator(self._items)
+            self._items = self._normalize_items(next_items, fallback=self._items)
+            self._save()
         return result
 
     @staticmethod
@@ -782,8 +789,12 @@ class AuthService:
         }
 
     def list_keys(self, role: AuthRole | None = None) -> list[dict[str, object]]:
+        # ⚠️⚠️ 这里**绝对不能**再调 _reload_locked()（那会去拿存储锁）。
+        # 调用方常常是「已经持有存储事务锁」的商店逻辑（兑换卡密 / 每日礼包在算可用模型），
+        # 一旦这里回头去要存储锁，就和 authenticate（持鉴权锁去写存储）形成 ABBA 死锁，
+        # 而 authenticate 是跑在**事件循环线程**上的 ⇒ 整站彻底无响应、连接被关闭。
+        # 2026-10-19 事故根因。单进程内所有写入都会同步更新 self._items，缓存即权威。
         with self._lock:
-            self._reload_locked()
             items = [item for item in self._items if role is None or item.get("role") == role]
             return [self._public_item(item) for item in items]
 
@@ -2641,40 +2652,40 @@ class AuthService:
                 return next_items, self._public_item(next_item)
             return items, None
 
+        # ⚠️⚠️ 锁顺序铁律：**存储 → 鉴权**。鉴权锁内绝不等待存储锁。
+        # 之前 authenticate 在 `with self._lock:` 里调 _update_items_locked（=去拿存储锁），
+        # 而商店侧（兑换卡密）持存储锁反过来调 list_keys（=去拿鉴权锁），形成 ABBA 死锁。
+        # authenticate 跑在事件循环线程上 ⇒ 整站 HTTP 全部无响应（2026-10-19 事故）。
         with self._lock:
-            for attempt in range(2):
+            matched_item = find_match(self._items)
+            if matched_item is None:
+                # 缓存未命中才回读一次；这是冷路径（未知/刚创建的密钥），不在热路径上。
+                self._reload_locked()
                 matched_item = find_match(self._items)
-                if matched_item is not None:
-                    if not touch:
-                        return self._public_item(matched_item)
-                    # ⚠️⚠️ last_used_at 必须节流，不能每个请求都写。
-                    # `_update_items_locked` 是「读出整张 auth_keys 表 → 改一条 → 写回整表」，
-                    # 而且是在**事件循环线程**上同步执行的；前端轮询（0.8s/次 × 几十人）
-                    # 会把整表重写几百次/分钟，SQLite 事务锁排队直接卡死整个 HTTP 服务
-                    # （2026-10-19：发了 30 个卡密、��间集中兑换后网站彻底无响应）。
-                    # last_used_at 只用于展示「最近使用」，60 秒粒度完全够用。
-                    throttle_key = self._clean(matched_item.get("id")) or candidate_hash
-                    now_ts = time.time()
-                    if now_ts - self._last_touch_ts.get(throttle_key, 0.0) < self._TOUCH_INTERVAL_SECONDS:
-                        return self._public_item(matched_item)
-                    try:
-                        updated = self._update_items_locked(mutate)
-                    except Exception as exc:
-                        print(f"[auth-service] update last_used_at failed: {exc}")
-                        fallback_item = dict(matched_item)
-                        fallback_item["last_used_at"] = datetime.now(timezone.utc).isoformat()
-                        return self._public_item(fallback_item)
-                    self._last_touch_ts[throttle_key] = now_ts
-                    if len(self._last_touch_ts) > 4096:
-                        # 防止长期运行后字典无限增长
-                        cutoff = now_ts - self._TOUCH_INTERVAL_SECONDS
-                        self._last_touch_ts = {
-                            key: ts for key, ts in self._last_touch_ts.items() if ts >= cutoff
-                        }
-                    return updated
-                if attempt == 0:
-                    self._reload_locked()
-        return None
+            if matched_item is None:
+                return None
+            if not touch:
+                return self._public_item(matched_item)
+            # last_used_at 节流：同一密钥 60 秒内只回写一次（见下）。
+            throttle_key = self._clean(matched_item.get("id")) or candidate_hash
+            now_ts = time.time()
+            if now_ts - self._last_touch_ts.get(throttle_key, 0.0) < self._TOUCH_INTERVAL_SECONDS:
+                return self._public_item(matched_item)
+            self._last_touch_ts[throttle_key] = now_ts
+            if len(self._last_touch_ts) > 4096:
+                cutoff = now_ts - self._TOUCH_INTERVAL_SECONDS
+                self._last_touch_ts = {
+                    key: ts for key, ts in self._last_touch_ts.items() if ts >= cutoff
+                }
+
+        # ---- 已放掉鉴权锁，再去写存储 ----
+        try:
+            return self._update_items_locked(mutate)
+        except Exception as exc:
+            print(f"[auth-service] update last_used_at failed: {exc}")
+            fallback_item = dict(matched_item)
+            fallback_item["last_used_at"] = datetime.now(timezone.utc).isoformat()
+            return self._public_item(fallback_item)
 
 
 auth_service = AuthService(config.get_user_storage_backend())