Use persistent SQLite storage for user data

.env.example

@@ -19,8 +19,9 @@ CHATGPT2API_AUTH_KEY=your_secret_key_here
 # ============================================
 
 # 存储后端类型（可选值: json, sqlite, postgres, git）
-# 默认: json
-STORAGE_BACKEND=json
+# 推荐 Hugging Face Storage Bucket 挂载到 /app/persist 后使用 sqlite。
+# 如果没有设置 STORAGE_BACKEND 且 /app/persist 存在，程序会自动使用 /app/persist/accounts.db。
+# STORAGE_BACKEND=sqlite
 
 # ============================================
 # 数据库配置（当 STORAGE_BACKEND=sqlite/postgres 时使用）
@@ -32,8 +33,8 @@ STORAGE_BACKEND=json
 # Supabase 示例
 # DATABASE_URL=postgresql://postgres.xxx:password@aws-1-us-west-1.pooler.supabase.com:5432/postgres
 
-# SQLite 示例（不指定时自动使用 data/accounts.db）
-# DATABASE_URL=sqlite:///app/data/accounts.db
+# SQLite 示例（不指定时自动使用持久化目录 accounts.db）
+# DATABASE_URL=sqlite:////app/persist/accounts.db
 
 # ============================================
 # Git 仓库配置（当 STORAGE_BACKEND=git 时使用）

api/system.py

@@ -90,11 +90,15 @@ def create_router(app_version: str) -> APIRouter:
         if not openid:
             raise HTTPException(status_code=502, detail={"error": "微信登录服务未返回 openid"})
 
-        item, raw_key, created = auth_service.get_or_create_wechat_normal_user(
-            openid=openid,
-            nickname=str(user.get("nickname") or "").strip(),
-            avatar=str(user.get("avatar") or "").strip(),
-        )
+        try:
+            item, raw_key, created = auth_service.get_or_create_wechat_normal_user(
+                openid=openid,
+                nickname=str(user.get("nickname") or "").strip(),
+                avatar=str(user.get("avatar") or "").strip(),
+            )
+        except Exception as exc:
+            print(f"[wechat-login] failed to save user: {exc}")
+            raise HTTPException(status_code=503, detail={"error": f"微信登录已验证，但保存用户失败：{exc}"}) from exc
         return {
             "ok": True,
             "version": app_version,

services/auth_service.py

@@ -80,20 +80,21 @@ class AuthService:
             "wechat_bound_at": self._clean(raw.get("wechat_bound_at")) or None,
         }
 
-    def _load(self) -> list[dict[str, object]]:
+    def _load(self, *, fallback: list[dict[str, object]] | None = None) -> list[dict[str, object]]:
         try:
             items = self.storage.load_auth_keys()
-        except Exception:
-            return []
+        except Exception as exc:
+            print(f"[auth-service] load auth keys failed: {exc}")
+            return [dict(item) for item in (fallback or [])]
         if not isinstance(items, list):
-            return []
+            return [dict(item) for item in (fallback or [])]
         return [normalized for item in items if (normalized := self._normalize_item(item)) is not None]
 
     def _save(self) -> None:
         self.storage.save_auth_keys(self._items)
 
     def _reload_locked(self) -> None:
-        self._items = self._load()
+        self._items = self._load(fallback=self._items)
 
     @staticmethod
     def _public_item(item: dict[str, object]) -> dict[str, object]:

services/config.py

@@ -10,7 +10,6 @@ import time
 from services.storage.base import StorageBackend
 
 BASE_DIR = Path(__file__).resolve().parents[1]
-DATA_DIR = BASE_DIR / "data"
 PERSIST_DIR = Path(os.getenv("CHATGPT2API_PERSIST_DIR", "").strip() or "/app/persist")
 
 
@@ -19,6 +18,7 @@ def _env_path(name: str, fallback: Path) -> Path:
     return Path(value) if value else fallback
 
 
+DATA_DIR = _env_path("CHATGPT2API_DATA_DIR", PERSIST_DIR if PERSIST_DIR.exists() else BASE_DIR / "data")
 CONFIG_FILE = _env_path("CHATGPT2API_CONFIG_FILE", PERSIST_DIR / "config" / "config.json" if PERSIST_DIR.exists() else BASE_DIR / "config.json")
 VERSION_FILE = BASE_DIR / "VERSION"
 BACKUP_STATE_FILE = DATA_DIR / "backup_state.json"

services/storage/database_storage.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
 import json
+from pathlib import Path
+from threading import Lock
 from typing import Any
 
 from sqlalchemy import Column, String, Text, create_engine, Integer, text
@@ -42,13 +44,21 @@ class DatabaseStorageBackend(StorageBackend):
 
     def __init__(self, database_url: str):
         self.database_url = database_url
+        connect_args: dict[str, Any] = {}
+        if database_url.startswith("sqlite:///"):
+            db_path = database_url.removeprefix("sqlite:///")
+            if db_path and db_path != ":memory:":
+                Path(db_path).parent.mkdir(parents=True, exist_ok=True)
+            connect_args["check_same_thread"] = False
         self.engine = create_engine(
             database_url,
             pool_pre_ping=True,  # 自动检测连接是否有效
             pool_recycle=3600,   # 1小时回收连接
+            connect_args=connect_args,
         )
         Base.metadata.create_all(self.engine)
         self.Session = sessionmaker(bind=self.engine)
+        self._lock = Lock()
 
     def load_accounts(self) -> list[dict[str, Any]]:
         """从数据库加载账号数据"""
@@ -68,7 +78,8 @@ class DatabaseStorageBackend(StorageBackend):
 
     def save_accounts(self, accounts: list[dict[str, Any]]) -> None:
         """保存账号数据到数据库"""
-        self._save_rows(AccountModel, accounts, "access_token")
+        with self._lock:
+            self._save_rows(AccountModel, accounts, "access_token")
 
     def load_auth_keys(self) -> list[dict[str, Any]]:
         """从数据库加载鉴权密钥数据"""
@@ -76,7 +87,8 @@ class DatabaseStorageBackend(StorageBackend):
 
     def save_auth_keys(self, auth_keys: list[dict[str, Any]]) -> None:
         """保存鉴权密钥数据到数据库"""
-        self._save_rows(AuthKeyModel, auth_keys, "id", "key_id")
+        with self._lock:
+            self._save_rows(AuthKeyModel, auth_keys, "id", "key_id")
 
     def load_shop_state(self) -> dict[str, Any]:
         session = self.Session()
@@ -92,20 +104,21 @@ class DatabaseStorageBackend(StorageBackend):
             session.close()
 
     def save_shop_state(self, state: dict[str, Any]) -> None:
-        session = self.Session()
-        try:
-            row = session.query(ShopStateModel).filter_by(key="default").first()
-            payload = json.dumps(state, ensure_ascii=False)
-            if row is None:
-                session.add(ShopStateModel(key="default", data=payload))
-            else:
-                row.data = payload
-            session.commit()
-        except Exception as e:
-            session.rollback()
-            raise e
-        finally:
-            session.close()
+        with self._lock:
+            session = self.Session()
+            try:
+                row = session.query(ShopStateModel).filter_by(key="default").first()
+                payload = json.dumps(state, ensure_ascii=False)
+                if row is None:
+                    session.add(ShopStateModel(key="default", data=payload))
+                else:
+                    row.data = payload
+                session.commit()
+            except Exception as e:
+                session.rollback()
+                raise e
+            finally:
+                session.close()
 
     def _load_rows(self, model: type[AccountModel] | type[AuthKeyModel]) -> list[dict[str, Any]]:
         session = self.Session()

services/storage/factory.py

@@ -10,6 +10,81 @@ from services.storage.git_storage import GitStorageBackend
 from services.storage.json_storage import JSONStorageBackend
 
 
+def _default_backend_type(data_dir: Path) -> str:
+    if os.getenv("STORAGE_BACKEND") is not None:
+        return os.getenv("STORAGE_BACKEND", "json").lower().strip()
+    persist_dir = Path(os.getenv("CHATGPT2API_PERSIST_DIR", "").strip() or "/app/persist")
+    if persist_dir.exists() or str(data_dir).startswith(str(persist_dir)):
+        return "sqlite"
+    return "json"
+
+
+def _sqlite_database_url(data_dir: Path) -> str:
+    return f"sqlite:///{data_dir / 'accounts.db'}"
+
+
+def _is_database_empty(backend: DatabaseStorageBackend) -> bool:
+    return not backend.load_accounts() and not backend.load_auth_keys() and not backend.load_shop_state()
+
+
+def _build_git_backend_from_env() -> GitStorageBackend | None:
+    repo_url = os.getenv("GIT_REPO_URL", "").strip()
+    token = os.getenv("GIT_TOKEN", "").strip()
+    if not repo_url:
+        return None
+    branch = os.getenv("GIT_BRANCH", "main").strip()
+    file_path = os.getenv("GIT_FILE_PATH", "accounts.json").strip()
+    auth_keys_file_path = os.getenv("GIT_AUTH_KEYS_FILE_PATH", "auth_keys.json").strip()
+    shop_state_file_path = os.getenv("GIT_SHOP_STATE_FILE_PATH", "shop_state.json").strip()
+    cache_dir = Path(os.getenv("GIT_CACHE_DIR", "").strip() or (Path(tempfile.gettempdir()) / "chatgpt2api_git_cache"))
+    return GitStorageBackend(
+        repo_url=repo_url,
+        token=token,
+        branch=branch,
+        file_path=file_path,
+        auth_keys_file_path=auth_keys_file_path,
+        shop_state_file_path=shop_state_file_path,
+        local_cache_dir=cache_dir,
+    )
+
+
+def _migrate_git_to_database_if_empty(backend: DatabaseStorageBackend) -> None:
+    if os.getenv("CHATGPT2API_SKIP_STORAGE_MIGRATION", "").strip().lower() in {"1", "true", "yes", "on"}:
+        return
+    try:
+        if not _is_database_empty(backend):
+            return
+    except Exception as exc:
+        print(f"[storage] skip migration, database check failed: {exc}")
+        return
+
+    git_backend = _build_git_backend_from_env()
+    if git_backend is None:
+        return
+
+    try:
+        accounts = git_backend.load_accounts()
+        auth_keys = git_backend.load_auth_keys()
+        shop_state = git_backend.load_shop_state()
+    except Exception as exc:
+        print(f"[storage] Git to SQLite migration skipped: {exc}")
+        return
+
+    if not accounts and not auth_keys and not shop_state:
+        return
+
+    print(
+        "[storage] Migrating Git data to SQLite: "
+        f"{len(accounts)} accounts, {len(auth_keys)} auth keys, shop_state={bool(shop_state)}"
+    )
+    if accounts:
+        backend.save_accounts(accounts)
+    if auth_keys:
+        backend.save_auth_keys(auth_keys)
+    if shop_state:
+        backend.save_shop_state(shop_state)
+
+
 def create_storage_backend(data_dir: Path) -> StorageBackend:
     """
     根据环境变量创建存储后端
@@ -23,7 +98,7 @@ def create_storage_backend(data_dir: Path) -> StorageBackend:
     - GIT_FILE_PATH: Git 仓库中的文件路径 (默认 accounts.json)
     - GIT_CACHE_DIR: Git 本地缓存目录 (默认系统临时目录)
     """
-    backend_type = os.getenv("STORAGE_BACKEND", "json").lower().strip()
+    backend_type = _default_backend_type(data_dir)
     
     print(f"[storage] Initializing storage backend: {backend_type}")
     
@@ -41,12 +116,15 @@ def create_storage_backend(data_dir: Path) -> StorageBackend:
         
         if not database_url:
             # 如果没有指定 DATABASE_URL，使用本地 SQLite
-            database_url = f"sqlite:///{data_dir / 'accounts.db'}"
+            database_url = _sqlite_database_url(data_dir)
             print(f"[storage] No DATABASE_URL provided, using local SQLite: {database_url}")
         else:
             print(f"[storage] Using database storage: {_mask_password(database_url)}")
-        
-        return DatabaseStorageBackend(database_url)
+
+        backend = DatabaseStorageBackend(database_url)
+        if backend_type in {"sqlite", "database"}:
+            _migrate_git_to_database_if_empty(backend)
+        return backend
     
     elif backend_type == "git":
         # Git 仓库存储

services/storage/git_storage.py

@@ -4,6 +4,7 @@ import json
 import shutil
 import tempfile
 from pathlib import Path
+from threading import Lock
 from typing import Any
 
 from git import Repo
@@ -31,6 +32,7 @@ class GitStorageBackend(StorageBackend):
         self.file_path = file_path
         self.auth_keys_file_path = auth_keys_file_path
         self.shop_state_file_path = shop_state_file_path
+        self._lock = Lock()
         
         # 本地缓存目录
         if local_cache_dir is None:
@@ -61,6 +63,28 @@ class GitStorageBackend(StorageBackend):
         
         return repo_url
 
+    def _safe_remove_repo(self, repo_path: Path) -> None:
+        try:
+            if repo_path.exists():
+                shutil.rmtree(repo_path)
+        except Exception as exc:
+            print(f"[git-storage] failed to remove broken cache {repo_path}: {exc}")
+
+    def _clone_repo(self, repo_path: Path) -> Repo:
+        for attempt in range(2):
+            try:
+                return Repo.clone_from(
+                    self.auth_repo_url,
+                    repo_path,
+                    branch=self.branch,
+                )
+            except Exception:
+                self._safe_remove_repo(repo_path)
+                if attempt == 0:
+                    continue
+                raise
+        raise RuntimeError("git clone failed")
+
     def _clone_or_pull(self) -> Repo:
         """克隆或拉取仓库"""
         repo_path = self.local_cache_dir / "repo"
@@ -74,15 +98,14 @@ class GitStorageBackend(StorageBackend):
                 return repo
             except GitCommandError:
                 # 拉取失败，删除重新克隆
-                shutil.rmtree(repo_path)
-        
+                self._safe_remove_repo(repo_path)
+            except Exception:
+                self._safe_remove_repo(repo_path)
+        elif repo_path.exists():
+            self._safe_remove_repo(repo_path)
+
         # 克隆仓库
-        repo = Repo.clone_from(
-            self.auth_repo_url,
-            repo_path,
-            branch=self.branch,
-        )
-        return repo
+        return self._clone_repo(repo_path)
 
     def load_accounts(self) -> list[dict[str, Any]]:
         """从 Git 仓库加载账号数据"""
@@ -139,31 +162,40 @@ class GitStorageBackend(StorageBackend):
         return data if isinstance(data, list) else []
 
     def _load_json_value(self, file_path: str) -> Any:
-        repo = self._clone_or_pull()
-        file_full_path = Path(repo.working_dir) / file_path
-        if not file_full_path.exists():
-            return None
-        try:
-            content = file_full_path.read_text(encoding="utf-8")
-            return json.loads(content)
-        except (json.JSONDecodeError, UnicodeDecodeError) as e:
-            print(f"[git-storage] Failed to load {file_path}: {e}, returning empty data")
-            return None
+        with self._lock:
+            repo = self._clone_or_pull()
+            file_full_path = Path(repo.working_dir) / file_path
+            if not file_full_path.exists():
+                return None
+            try:
+                content = file_full_path.read_text(encoding="utf-8")
+                return json.loads(content)
+            except (json.JSONDecodeError, UnicodeDecodeError) as e:
+                print(f"[git-storage] Failed to load {file_path}: {e}, returning empty data")
+                return None
 
     def _save_json_file(self, file_path: str, items: Any, message: str) -> None:
-        repo = self._clone_or_pull()
-        file_full_path = Path(repo.working_dir) / file_path
-        file_full_path.parent.mkdir(parents=True, exist_ok=True)
-        file_full_path.write_text(
-            json.dumps(items, ensure_ascii=False, indent=2) + "\n",
-            encoding="utf-8",
-        )
-        repo.index.add([file_path])
-        if repo.is_dirty():
-            # The data repository only stores JSON state; deployment volumes may
-            # preserve non-executable sample hooks, so skip hooks for reliability.
-            repo.index.commit(message, skip_hooks=True)
-            repo.remote("origin").push(self.branch)
+        with self._lock:
+            payload = json.dumps(items, ensure_ascii=False, indent=2) + "\n"
+            for attempt in range(2):
+                repo = self._clone_or_pull()
+                file_full_path = Path(repo.working_dir) / file_path
+                file_full_path.parent.mkdir(parents=True, exist_ok=True)
+                file_full_path.write_text(payload, encoding="utf-8")
+                repo.index.add([file_path])
+                if not repo.is_dirty():
+                    return
+                try:
+                    # The data repository only stores JSON state; deployment volumes may
+                    # preserve non-executable sample hooks, so skip hooks for reliability.
+                    repo.index.commit(message, skip_hooks=True)
+                    repo.remote("origin").push(self.branch)
+                    return
+                except GitCommandError:
+                    if attempt == 0:
+                        self._safe_remove_repo(self.local_cache_dir / "repo")
+                        continue
+                    raise
 
     def health_check(self) -> dict[str, Any]:
         """健康检查"""