Spaces:
Running
Running
File size: 6,778 Bytes
e06d90f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 | """一次性维护脚本:把 Hub 上「已有备份」里的长期密钥就地脱敏。
为什么需要单独一步
------------------
代码侧(V3)只拦住了「新上传」的备份。修复前已经躺在 Hub 上的备份里,
`config.accessKey` 仍是明文 —— 只要 V1(namespace 复用)和 V2(恢复不校验归属)
还没上线,或者旧备份还在,任何人都能通过同一条通道把它读出来。
所以轮换 access key 之前,必须先把这批存量备份洗干净,否则就是「换一次泄一次」。
用法(在 Space 容器内或本地装好依赖后执行)
------------------------------------------
# 1) 先看清楚有什么(不写任何东西)
python -m scripts.scrub_hub_backups --dry-run
# 2) 就地脱敏(默认会把脱敏后的字节重新推回 Hub)
python -m scripts.scrub_hub_backups --apply
# 3) 更彻底:直接删掉含密钥的备份(用户需要重新导出一次)
python -m scripts.scrub_hub_backups --apply --delete
# 4) 只处理指定 namespace(排查问题时用)
python -m scripts.scrub_hub_backups --apply --ids 1,7
产物
----
`scrub_report_<时间戳>.json`:每个 namespace / key 的处理结果(scrubbed / clean /
deleted / failed),事后可核对,也可作为「存量已清理」的凭证。
注意
----
- 脚本只碰 `user_backups/<nsid>/<key>`,不动 `user_files/`、不动 meta。
- 每个被改写的 key 会先把原始字节备份到本地 `scrub_backup_<时间戳>/` 目录,
出问题可以手工回滚。
- 需要 HF_TOKEN + HF_CONFIG_REPO 已配置(`is_hub_enabled()` 为 True),否则什么都不做。
"""
from __future__ import annotations
import argparse
import asyncio
import json
import logging
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from app.hf_storage import ( # noqa: E402
download_bytes,
is_hub_enabled,
list_hub_keys,
push_to_hub,
upload_bytes,
delete_from_hub,
)
from app.services import backup_sanitizer, user_files_store # noqa: E402
logger = logging.getLogger("scrub")
def _content_path(nsid: str, key: str) -> str:
return f"{user_files_store._PREFIX_BACKUPS}/{nsid}"
def _backup_original(nsid: str, key: str, raw: bytes, out_dir: Path) -> None:
d = out_dir / nsid
d.mkdir(parents=True, exist_ok=True)
(d / f"{key}.orig").write_bytes(raw)
async def _scrub_one(nsid: str, key: str, *, apply: bool, delete: bool,
out_dir: Path) -> dict:
ns = _content_path(nsid, key)
raw = download_bytes(ns, key)
if raw is None:
return {"nsid": nsid, "key": key, "result": "missing"}
if not raw:
return {"nsid": nsid, "key": key, "result": "empty"}
hits = backup_sanitizer.detect_sensitive_keys(raw)
if not hits:
return {"nsid": nsid, "key": key, "result": "clean"}
rec = {"nsid": nsid, "key": key, "keys": hits, "size_before": len(raw)}
if not apply:
rec["result"] = "would_scrub" if not delete else "would_delete"
return rec
_backup_original(nsid, key, raw, out_dir)
if delete:
ok = delete_from_hub(ns, key)
rec.update(result="deleted" if ok else "failed", action="delete")
return rec
new, redacted = backup_sanitizer.scrub_backup_bytes(raw)
if not redacted:
# 缩水守卫 / 结构异常导致保守回退:不动数据,留给人工处理
rec.update(result="skipped_guard", note="scrub returned raw (guard/fallback)")
return rec
try:
upload_bytes(ns, key, new) # 写本地缓存
pushed = await push_to_hub(ns, key) # 推远端
except Exception as e:
rec.update(result="failed", error=str(e)[:200])
return rec
rec.update(
result="scrubbed" if pushed else "scrubbed_local_only",
size_after=len(new),
redacted=redacted,
)
return rec
async def main_async(args) -> int:
if not is_hub_enabled():
print("HF_TOKEN / HF_CONFIG_REPO 未配置,Hub 未启用,无事可做。")
return 2
file_ids, backup_ids = user_files_store.list_all_hub_user_ids_with_files()
print(f"Hub 上有文件的 namespace: {file_ids}")
print(f"Hub 上有备份的 namespace: {backup_ids}")
targets = [str(x) for x in backup_ids]
if args.ids:
wanted = [s.strip() for s in args.ids.split(",") if s.strip()]
targets = [t for t in targets if t in wanted]
if not targets:
print("没有需要处理的备份 namespace。")
return 0
ts = time.strftime("%Y%m%d_%H%M%S")
out_dir = Path(f"scrub_backup_{ts}")
report: list[dict] = []
for nsid in targets:
keys = list_hub_keys(f"{user_files_store._PREFIX_BACKUPS}/{nsid}")
print(f"\n[{nsid}] {len(keys)} 个备份")
for key in keys:
rec = await _scrub_one(
nsid, key, apply=args.apply, delete=args.delete, out_dir=out_dir
)
report.append(rec)
print(f" {key[:16]}… {rec['result']:<18} {rec.get('keys') or ''}")
report_path = Path(f"scrub_report_{ts}.json")
report_path.write_text(
json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8"
)
counts: dict[str, int] = {}
for r in report:
counts[r["result"]] = counts.get(r["result"], 0) + 1
print(f"\n汇总: {counts}")
print(f"报告: {report_path}")
if args.apply and any(r["result"].startswith("scrubbed") for r in report):
print("提示:脱敏已生效,但下载校验请以备份 meta 的 sha256 为准"
"(旧 meta 的 sha256 仍是明文版本的哈希,属于已知脏数据)。")
return 0
def main() -> int:
ap = argparse.ArgumentParser(description="就地脱敏 Hub 上已有备份中的长期密钥")
ap.add_argument("--dry-run", action="store_true", help="只扫描报告,不改动(默认行为)")
ap.add_argument("--apply", action="store_true", help="真正改写 Hub 上的备份内容")
ap.add_argument("--delete", action="store_true", help="直接删除含密钥的备份(配合 --apply)")
ap.add_argument("--ids", default="", help="只处理这些 namespace,逗号分隔,如 1,7")
ap.add_argument("--verbose", action="store_true")
args = ap.parse_args()
logging.basicConfig(
level=logging.DEBUG if args.verbose else logging.WARNING,
format="%(levelname)s %(name)s %(message)s",
)
if args.delete and not args.apply:
print("--delete 必须配合 --apply 使用(避免误删)。")
return 2
return asyncio.run(main_async(args))
if __name__ == "__main__":
raise SystemExit(main())
|