"""一次性维护脚本:把 Hub 上「已有备份」里的长期密钥就地脱敏。 为什么需要单独一步 ------------------ 代码侧(V3)只拦住了「新上传」的备份。修复前已经躺在 Hub 上的备份里, `config.accessKey` 仍是明文 —— 只要 V1(namespace 复用)和 V2(恢复不校验归属) 还没上线,或者旧备份还在,任何人都能通过同一条通道把它读出来。 所以轮换 access key 之前,必须先把这批存量备份洗干净,否则就是「换一次泄一次」。 用法(在 Space 容器内或本地装好依赖后执行) ------------------------------------------ # 1) 先看清楚有什么(不写任何东西) python -m scripts.scrub_hub_backups --dry-run # 2) 就地脱敏(默认会把脱敏后的字节重新推回 Hub) python -m scripts.scrub_hub_backups --apply # 3) 更彻底:直接删掉含密钥的备份(用户需要重新导出一次) python -m scripts.scrub_hub_backups --apply --delete # 4) 只处理指定 namespace(排查问题时用) python -m scripts.scrub_hub_backups --apply --ids 1,7 产物 ---- `scrub_report_<时间戳>.json`:每个 namespace / key 的处理结果(scrubbed / clean / deleted / failed),事后可核对,也可作为「存量已清理」的凭证。 注意 ---- - 脚本只碰 `user_backups//`,不动 `user_files/`、不动 meta。 - 每个被改写的 key 会先把原始字节备份到本地 `scrub_backup_<时间戳>/` 目录, 出问题可以手工回滚。 - 需要 HF_TOKEN + HF_CONFIG_REPO 已配置(`is_hub_enabled()` 为 True),否则什么都不做。 """ from __future__ import annotations import argparse import asyncio import json import logging import sys import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from app.hf_storage import ( # noqa: E402 download_bytes, is_hub_enabled, list_hub_keys, push_to_hub, upload_bytes, delete_from_hub, ) from app.services import backup_sanitizer, user_files_store # noqa: E402 logger = logging.getLogger("scrub") def _content_path(nsid: str, key: str) -> str: return f"{user_files_store._PREFIX_BACKUPS}/{nsid}" def _backup_original(nsid: str, key: str, raw: bytes, out_dir: Path) -> None: d = out_dir / nsid d.mkdir(parents=True, exist_ok=True) (d / f"{key}.orig").write_bytes(raw) async def _scrub_one(nsid: str, key: str, *, apply: bool, delete: bool, out_dir: Path) -> dict: ns = _content_path(nsid, key) raw = download_bytes(ns, key) if raw is None: return {"nsid": nsid, "key": key, "result": "missing"} if not raw: return {"nsid": nsid, "key": key, "result": "empty"} hits = backup_sanitizer.detect_sensitive_keys(raw) if not hits: return {"nsid": nsid, "key": key, "result": "clean"} rec = {"nsid": nsid, "key": key, "keys": hits, "size_before": len(raw)} if not apply: rec["result"] = "would_scrub" if not delete else "would_delete" return rec _backup_original(nsid, key, raw, out_dir) if delete: ok = delete_from_hub(ns, key) rec.update(result="deleted" if ok else "failed", action="delete") return rec new, redacted = backup_sanitizer.scrub_backup_bytes(raw) if not redacted: # 缩水守卫 / 结构异常导致保守回退:不动数据,留给人工处理 rec.update(result="skipped_guard", note="scrub returned raw (guard/fallback)") return rec try: upload_bytes(ns, key, new) # 写本地缓存 pushed = await push_to_hub(ns, key) # 推远端 except Exception as e: rec.update(result="failed", error=str(e)[:200]) return rec rec.update( result="scrubbed" if pushed else "scrubbed_local_only", size_after=len(new), redacted=redacted, ) return rec async def main_async(args) -> int: if not is_hub_enabled(): print("HF_TOKEN / HF_CONFIG_REPO 未配置,Hub 未启用,无事可做。") return 2 file_ids, backup_ids = user_files_store.list_all_hub_user_ids_with_files() print(f"Hub 上有文件的 namespace: {file_ids}") print(f"Hub 上有备份的 namespace: {backup_ids}") targets = [str(x) for x in backup_ids] if args.ids: wanted = [s.strip() for s in args.ids.split(",") if s.strip()] targets = [t for t in targets if t in wanted] if not targets: print("没有需要处理的备份 namespace。") return 0 ts = time.strftime("%Y%m%d_%H%M%S") out_dir = Path(f"scrub_backup_{ts}") report: list[dict] = [] for nsid in targets: keys = list_hub_keys(f"{user_files_store._PREFIX_BACKUPS}/{nsid}") print(f"\n[{nsid}] {len(keys)} 个备份") for key in keys: rec = await _scrub_one( nsid, key, apply=args.apply, delete=args.delete, out_dir=out_dir ) report.append(rec) print(f" {key[:16]}… {rec['result']:<18} {rec.get('keys') or ''}") report_path = Path(f"scrub_report_{ts}.json") report_path.write_text( json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8" ) counts: dict[str, int] = {} for r in report: counts[r["result"]] = counts.get(r["result"], 0) + 1 print(f"\n汇总: {counts}") print(f"报告: {report_path}") if args.apply and any(r["result"].startswith("scrubbed") for r in report): print("提示:脱敏已生效,但下载校验请以备份 meta 的 sha256 为准" "(旧 meta 的 sha256 仍是明文版本的哈希,属于已知脏数据)。") return 0 def main() -> int: ap = argparse.ArgumentParser(description="就地脱敏 Hub 上已有备份中的长期密钥") ap.add_argument("--dry-run", action="store_true", help="只扫描报告,不改动(默认行为)") ap.add_argument("--apply", action="store_true", help="真正改写 Hub 上的备份内容") ap.add_argument("--delete", action="store_true", help="直接删除含密钥的备份(配合 --apply)") ap.add_argument("--ids", default="", help="只处理这些 namespace,逗号分隔,如 1,7") ap.add_argument("--verbose", action="store_true") args = ap.parse_args() logging.basicConfig( level=logging.DEBUG if args.verbose else logging.WARNING, format="%(levelname)s %(name)s %(message)s", ) if args.delete and not args.apply: print("--delete 必须配合 --apply 使用(避免误删)。") return 2 return asyncio.run(main_async(args)) if __name__ == "__main__": raise SystemExit(main())