xtc-backend / scripts /scrub_hub_backups.py
a3216's picture
fix(security): 修复用户侧水平越权与备份明文密钥外流(V1/V2/V3/V5/V6/V8/V9)
e06d90f
Raw History Blame Contribute Delete
6.78 kB
"""一次性维护脚本:把 Hub 上「已有备份」里的长期密钥就地脱敏。
为什么需要单独一步
------------------
代码侧(V3)只拦住了「新上传」的备份。修复前已经躺在 Hub 上的备份里,
`config.accessKey` 仍是明文 —— 只要 V1(namespace 复用)和 V2(恢复不校验归属)
还没上线,或者旧备份还在,任何人都能通过同一条通道把它读出来。
所以轮换 access key 之前,必须先把这批存量备份洗干净,否则就是「换一次泄一次」。
用法(在 Space 容器内或本地装好依赖后执行)
------------------------------------------
# 1) 先看清楚有什么(不写任何东西)
python -m scripts.scrub_hub_backups --dry-run
# 2) 就地脱敏(默认会把脱敏后的字节重新推回 Hub)
python -m scripts.scrub_hub_backups --apply
# 3) 更彻底:直接删掉含密钥的备份(用户需要重新导出一次)
python -m scripts.scrub_hub_backups --apply --delete
# 4) 只处理指定 namespace(排查问题时用)
python -m scripts.scrub_hub_backups --apply --ids 1,7
产物
----
`scrub_report_<时间戳>.json`:每个 namespace / key 的处理结果(scrubbed / clean /
deleted / failed),事后可核对,也可作为「存量已清理」的凭证。
注意
----
- 脚本只碰 `user_backups/<nsid>/<key>`,不动 `user_files/`、不动 meta。
- 每个被改写的 key 会先把原始字节备份到本地 `scrub_backup_<时间戳>/` 目录,
出问题可以手工回滚。
- 需要 HF_TOKEN + HF_CONFIG_REPO 已配置(`is_hub_enabled()` 为 True),否则什么都不做。
"""
from __future__ import annotations
import argparse
import asyncio
import json
import logging
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from app.hf_storage import ( # noqa: E402
download_bytes,
is_hub_enabled,
list_hub_keys,
push_to_hub,
upload_bytes,
delete_from_hub,
)
from app.services import backup_sanitizer, user_files_store # noqa: E402
logger = logging.getLogger("scrub")
def _content_path(nsid: str, key: str) -> str:
return f"{user_files_store._PREFIX_BACKUPS}/{nsid}"
def _backup_original(nsid: str, key: str, raw: bytes, out_dir: Path) -> None:
d = out_dir / nsid
d.mkdir(parents=True, exist_ok=True)
(d / f"{key}.orig").write_bytes(raw)
async def _scrub_one(nsid: str, key: str, *, apply: bool, delete: bool,
out_dir: Path) -> dict:
ns = _content_path(nsid, key)
raw = download_bytes(ns, key)
if raw is None:
return {"nsid": nsid, "key": key, "result": "missing"}
if not raw:
return {"nsid": nsid, "key": key, "result": "empty"}
hits = backup_sanitizer.detect_sensitive_keys(raw)
if not hits:
return {"nsid": nsid, "key": key, "result": "clean"}
rec = {"nsid": nsid, "key": key, "keys": hits, "size_before": len(raw)}
if not apply:
rec["result"] = "would_scrub" if not delete else "would_delete"
return rec
_backup_original(nsid, key, raw, out_dir)
if delete:
ok = delete_from_hub(ns, key)
rec.update(result="deleted" if ok else "failed", action="delete")
return rec
new, redacted = backup_sanitizer.scrub_backup_bytes(raw)
if not redacted:
# 缩水守卫 / 结构异常导致保守回退:不动数据,留给人工处理
rec.update(result="skipped_guard", note="scrub returned raw (guard/fallback)")
return rec
try:
upload_bytes(ns, key, new) # 写本地缓存
pushed = await push_to_hub(ns, key) # 推远端
except Exception as e:
rec.update(result="failed", error=str(e)[:200])
return rec
rec.update(
result="scrubbed" if pushed else "scrubbed_local_only",
size_after=len(new),
redacted=redacted,
)
return rec
async def main_async(args) -> int:
if not is_hub_enabled():
print("HF_TOKEN / HF_CONFIG_REPO 未配置,Hub 未启用,无事可做。")
return 2
file_ids, backup_ids = user_files_store.list_all_hub_user_ids_with_files()
print(f"Hub 上有文件的 namespace: {file_ids}")
print(f"Hub 上有备份的 namespace: {backup_ids}")
targets = [str(x) for x in backup_ids]
if args.ids:
wanted = [s.strip() for s in args.ids.split(",") if s.strip()]
targets = [t for t in targets if t in wanted]
if not targets:
print("没有需要处理的备份 namespace。")
return 0
ts = time.strftime("%Y%m%d_%H%M%S")
out_dir = Path(f"scrub_backup_{ts}")
report: list[dict] = []
for nsid in targets:
keys = list_hub_keys(f"{user_files_store._PREFIX_BACKUPS}/{nsid}")
print(f"\n[{nsid}] {len(keys)} 个备份")
for key in keys:
rec = await _scrub_one(
nsid, key, apply=args.apply, delete=args.delete, out_dir=out_dir
)
report.append(rec)
print(f" {key[:16]}… {rec['result']:<18} {rec.get('keys') or ''}")
report_path = Path(f"scrub_report_{ts}.json")
report_path.write_text(
json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8"
)
counts: dict[str, int] = {}
for r in report:
counts[r["result"]] = counts.get(r["result"], 0) + 1
print(f"\n汇总: {counts}")
print(f"报告: {report_path}")
if args.apply and any(r["result"].startswith("scrubbed") for r in report):
print("提示:脱敏已生效,但下载校验请以备份 meta 的 sha256 为准"
"(旧 meta 的 sha256 仍是明文版本的哈希,属于已知脏数据)。")
return 0
def main() -> int:
ap = argparse.ArgumentParser(description="就地脱敏 Hub 上已有备份中的长期密钥")
ap.add_argument("--dry-run", action="store_true", help="只扫描报告,不改动(默认行为)")
ap.add_argument("--apply", action="store_true", help="真正改写 Hub 上的备份内容")
ap.add_argument("--delete", action="store_true", help="直接删除含密钥的备份(配合 --apply)")
ap.add_argument("--ids", default="", help="只处理这些 namespace,逗号分隔,如 1,7")
ap.add_argument("--verbose", action="store_true")
args = ap.parse_args()
logging.basicConfig(
level=logging.DEBUG if args.verbose else logging.WARNING,
format="%(levelname)s %(name)s %(message)s",
)
if args.delete and not args.apply:
print("--delete 必须配合 --apply 使用(避免误删)。")
return 2
return asyncio.run(main_async(args))
if __name__ == "__main__":
raise SystemExit(main())