Spaces:
Running
Running
Download scripts/scrub_hub_backups.py from a3216/xtc-backend: direct link, hf CLI and curl.
- Browser
- Download file 6.78 kB
-
https://huggingface.co/spaces/a3216/xtc-backend/resolve/main/scripts/scrub_hub_backups.py
- Command line
-
hf download hf://spaces/a3216/xtc-backend/scripts/scrub_hub_backups.py
-
curl -L -o scrub_hub_backups.py https://huggingface.co/spaces/a3216/xtc-backend/resolve/main/scripts/scrub_hub_backups.py
6.78 kB
| """一次性维护脚本:把 Hub 上「已有备份」里的长期密钥就地脱敏。 | |
| 为什么需要单独一步 | |
| ------------------ | |
| 代码侧(V3)只拦住了「新上传」的备份。修复前已经躺在 Hub 上的备份里, | |
| `config.accessKey` 仍是明文 —— 只要 V1(namespace 复用)和 V2(恢复不校验归属) | |
| 还没上线,或者旧备份还在,任何人都能通过同一条通道把它读出来。 | |
| 所以轮换 access key 之前,必须先把这批存量备份洗干净,否则就是「换一次泄一次」。 | |
| 用法(在 Space 容器内或本地装好依赖后执行) | |
| ------------------------------------------ | |
| # 1) 先看清楚有什么(不写任何东西) | |
| python -m scripts.scrub_hub_backups --dry-run | |
| # 2) 就地脱敏(默认会把脱敏后的字节重新推回 Hub) | |
| python -m scripts.scrub_hub_backups --apply | |
| # 3) 更彻底:直接删掉含密钥的备份(用户需要重新导出一次) | |
| python -m scripts.scrub_hub_backups --apply --delete | |
| # 4) 只处理指定 namespace(排查问题时用) | |
| python -m scripts.scrub_hub_backups --apply --ids 1,7 | |
| 产物 | |
| ---- | |
| `scrub_report_<时间戳>.json`:每个 namespace / key 的处理结果(scrubbed / clean / | |
| deleted / failed),事后可核对,也可作为「存量已清理」的凭证。 | |
| 注意 | |
| ---- | |
| - 脚本只碰 `user_backups/<nsid>/<key>`,不动 `user_files/`、不动 meta。 | |
| - 每个被改写的 key 会先把原始字节备份到本地 `scrub_backup_<时间戳>/` 目录, | |
| 出问题可以手工回滚。 | |
| - 需要 HF_TOKEN + HF_CONFIG_REPO 已配置(`is_hub_enabled()` 为 True),否则什么都不做。 | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import asyncio | |
| import json | |
| import logging | |
| import sys | |
| import time | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) | |
| from app.hf_storage import ( # noqa: E402 | |
| download_bytes, | |
| is_hub_enabled, | |
| list_hub_keys, | |
| push_to_hub, | |
| upload_bytes, | |
| delete_from_hub, | |
| ) | |
| from app.services import backup_sanitizer, user_files_store # noqa: E402 | |
| logger = logging.getLogger("scrub") | |
| def _content_path(nsid: str, key: str) -> str: | |
| return f"{user_files_store._PREFIX_BACKUPS}/{nsid}" | |
| def _backup_original(nsid: str, key: str, raw: bytes, out_dir: Path) -> None: | |
| d = out_dir / nsid | |
| d.mkdir(parents=True, exist_ok=True) | |
| (d / f"{key}.orig").write_bytes(raw) | |
| async def _scrub_one(nsid: str, key: str, *, apply: bool, delete: bool, | |
| out_dir: Path) -> dict: | |
| ns = _content_path(nsid, key) | |
| raw = download_bytes(ns, key) | |
| if raw is None: | |
| return {"nsid": nsid, "key": key, "result": "missing"} | |
| if not raw: | |
| return {"nsid": nsid, "key": key, "result": "empty"} | |
| hits = backup_sanitizer.detect_sensitive_keys(raw) | |
| if not hits: | |
| return {"nsid": nsid, "key": key, "result": "clean"} | |
| rec = {"nsid": nsid, "key": key, "keys": hits, "size_before": len(raw)} | |
| if not apply: | |
| rec["result"] = "would_scrub" if not delete else "would_delete" | |
| return rec | |
| _backup_original(nsid, key, raw, out_dir) | |
| if delete: | |
| ok = delete_from_hub(ns, key) | |
| rec.update(result="deleted" if ok else "failed", action="delete") | |
| return rec | |
| new, redacted = backup_sanitizer.scrub_backup_bytes(raw) | |
| if not redacted: | |
| # 缩水守卫 / 结构异常导致保守回退:不动数据,留给人工处理 | |
| rec.update(result="skipped_guard", note="scrub returned raw (guard/fallback)") | |
| return rec | |
| try: | |
| upload_bytes(ns, key, new) # 写本地缓存 | |
| pushed = await push_to_hub(ns, key) # 推远端 | |
| except Exception as e: | |
| rec.update(result="failed", error=str(e)[:200]) | |
| return rec | |
| rec.update( | |
| result="scrubbed" if pushed else "scrubbed_local_only", | |
| size_after=len(new), | |
| redacted=redacted, | |
| ) | |
| return rec | |
| async def main_async(args) -> int: | |
| if not is_hub_enabled(): | |
| print("HF_TOKEN / HF_CONFIG_REPO 未配置,Hub 未启用,无事可做。") | |
| return 2 | |
| file_ids, backup_ids = user_files_store.list_all_hub_user_ids_with_files() | |
| print(f"Hub 上有文件的 namespace: {file_ids}") | |
| print(f"Hub 上有备份的 namespace: {backup_ids}") | |
| targets = [str(x) for x in backup_ids] | |
| if args.ids: | |
| wanted = [s.strip() for s in args.ids.split(",") if s.strip()] | |
| targets = [t for t in targets if t in wanted] | |
| if not targets: | |
| print("没有需要处理的备份 namespace。") | |
| return 0 | |
| ts = time.strftime("%Y%m%d_%H%M%S") | |
| out_dir = Path(f"scrub_backup_{ts}") | |
| report: list[dict] = [] | |
| for nsid in targets: | |
| keys = list_hub_keys(f"{user_files_store._PREFIX_BACKUPS}/{nsid}") | |
| print(f"\n[{nsid}] {len(keys)} 个备份") | |
| for key in keys: | |
| rec = await _scrub_one( | |
| nsid, key, apply=args.apply, delete=args.delete, out_dir=out_dir | |
| ) | |
| report.append(rec) | |
| print(f" {key[:16]}… {rec['result']:<18} {rec.get('keys') or ''}") | |
| report_path = Path(f"scrub_report_{ts}.json") | |
| report_path.write_text( | |
| json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8" | |
| ) | |
| counts: dict[str, int] = {} | |
| for r in report: | |
| counts[r["result"]] = counts.get(r["result"], 0) + 1 | |
| print(f"\n汇总: {counts}") | |
| print(f"报告: {report_path}") | |
| if args.apply and any(r["result"].startswith("scrubbed") for r in report): | |
| print("提示:脱敏已生效,但下载校验请以备份 meta 的 sha256 为准" | |
| "(旧 meta 的 sha256 仍是明文版本的哈希,属于已知脏数据)。") | |
| return 0 | |
| def main() -> int: | |
| ap = argparse.ArgumentParser(description="就地脱敏 Hub 上已有备份中的长期密钥") | |
| ap.add_argument("--dry-run", action="store_true", help="只扫描报告,不改动(默认行为)") | |
| ap.add_argument("--apply", action="store_true", help="真正改写 Hub 上的备份内容") | |
| ap.add_argument("--delete", action="store_true", help="直接删除含密钥的备份(配合 --apply)") | |
| ap.add_argument("--ids", default="", help="只处理这些 namespace,逗号分隔,如 1,7") | |
| ap.add_argument("--verbose", action="store_true") | |
| args = ap.parse_args() | |
| logging.basicConfig( | |
| level=logging.DEBUG if args.verbose else logging.WARNING, | |
| format="%(levelname)s %(name)s %(message)s", | |
| ) | |
| if args.delete and not args.apply: | |
| print("--delete 必须配合 --apply 使用(避免误删)。") | |
| return 2 | |
| return asyncio.run(main_async(args)) | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |