import os import sys import tarfile import shutil import subprocess import datetime from huggingface_hub import HfApi, hf_hub_download, CommitOperationAdd, CommitOperationDelete api = HfApi() repo_id = os.getenv("HF_DATASET") token = os.getenv("H_TOKEN") FILENAME = "latest_backup.tar.gz" # rclone 目标配置 RCLONE_DEST = os.getenv("RCLONE_BACKUP_DEST", "nvsu:latest_backup.tar.gz") # ==================== 配置 ==================== LFS_KEEP_COUNT = int(os.getenv("LFS_KEEP_COUNT", "5")) # 保留最近 N 个 LFS 对象(存储层面) LFS_DRY_RUN = os.getenv("LFS_DRY_RUN", "false").lower() in ("true", "1", "yes") # ================================================ def upload_to_rclone(local_path, remote_path): """尝试用 rclone 上传文件到远程存储,失败时仅打印警告""" if not shutil.which("rclone"): print("rclone 命令不存在,跳过 rclone 上传") return False try: cmd = ["rclone", "copy", local_path, remote_path, "--progress"] print(f"正在使用 rclone 上传到 {remote_path} ...") result = subprocess.run(cmd, capture_output=True, text=True, timeout=300) if result.returncode == 0: print(f"rclone 上传成功: {remote_path}") return True else: print(f"rclone 上传失败 (返回码 {result.returncode}): {result.stderr}") return False except Exception as e: print(f"rclone 上传异常: {str(e)}") return False def clean_lfs_objects(): """永久删除旧版本的 LFS 对象以释放存储配额(针对 latest_backup.tar.gz)""" try: print("\n🧹 开始清理冗余 LFS 存储对象...") lfs_files = list(api.list_lfs_files(repo_id=repo_id, repo_type="dataset", token=token)) candidates = [f for f in lfs_files if f.filename == FILENAME] print(f" 匹配到 {len(candidates)} 个 '{FILENAME}' 的 LFS 版本") if not candidates: print(" 没有发现可清理的 LFS 对象。") return True candidates.sort(key=lambda x: x.pushed_at, reverse=True) to_delete = candidates[LFS_KEEP_COUNT:] if not to_delete: print(f" 备份数量 ({len(candidates)}) 未超过保留阈值 ({LFS_KEEP_COUNT}),无需清理。") return True print(f" 计划删除 {len(to_delete)} 个旧 LFS 对象:") for f in to_delete: print(f" • OID: {f.oid} | Pushed: {f.pushed_at} | Size: {f.size / 1024 / 1024:.2f} MB") if LFS_DRY_RUN: print(" [DRY-RUN] 未执行实际删除操作。") else: api.permanently_delete_lfs_files( repo_id=repo_id, lfs_files=to_delete, repo_type="dataset", token=token ) print(f"✅ 已永久删除 {len(to_delete)} 个旧 LFS 对象。") return True except Exception as e: print(f"LFS 清理异常: {str(e)}") return False def safe_merge_files(src_root, dst_root): """ 递归地将 src_root 的内容合并到 dst_root。 - 目录:使用 dirs_exist_ok=True 进行合并。 - 文件:如果目标文件已存在,先将其备份为 .bak,再覆盖。 """ for item in os.listdir(src_root): src_path = os.path.join(src_root, item) dst_path = os.path.join(dst_root, item) if os.path.isdir(src_path): os.makedirs(dst_path, exist_ok=True) # 递归处理子目录,以便对子目录中的文件也能执行 .bak 备份 safe_merge_files(src_path, dst_path) elif os.path.isfile(src_path): if os.path.exists(dst_path): bak_path = dst_path + ".bak" shutil.copy2(dst_path, bak_path) print(f" [保护] 已将现有文件备份至: {os.path.basename(bak_path)}") shutil.copy2(src_path, dst_path) print(f" [恢复] {item}") def restore(): try: if not repo_id or not token: print("Skip Restore: HF_DATASET or H_TOKEN not set") return False print(f"正在从 Hugging Face 下载备份文件 {FILENAME} (repo: {repo_id}) ...") downloaded_path = hf_hub_download( repo_id=repo_id, filename=FILENAME, repo_type="dataset", token=token ) temp_dir = "/root/.openclaw/restore_temp" if os.path.exists(temp_dir): shutil.rmtree(temp_dir) os.makedirs(temp_dir, exist_ok=True) print(f"解压备份到临时目录: {temp_dir}") with tarfile.open(downloaded_path, "r:gz") as tar: tar.extractall(path=temp_dir) # 1. 处理 /app/data (合并) temp_data = os.path.join(temp_dir, "data") target_data = "/app/data" if os.path.exists(temp_data): os.makedirs(target_data, exist_ok=True) shutil.copytree(temp_data, target_data, dirs_exist_ok=True) print(f"✅ 已合并备份中的 data 目录到 {target_data}") else: print("备份包中未找到 data 目录,跳过 /app/data 恢复") # 2. 处理 /root/.openclaw 下的其他配置 (安全合并) print("\n🛡️ 开始安全合并配置到 /root/.openclaw ...") # 排除 data 目录,因为它已经处理过了 for item in os.listdir(temp_dir): if item == "data": continue src_path = os.path.join(temp_dir, item) dst_path = os.path.join("/root/.openclaw", item) if os.path.isdir(src_path): os.makedirs(dst_path, exist_ok=True) safe_merge_files(src_path, dst_path) elif os.path.isfile(src_path): if os.path.exists(dst_path): shutil.copy2(dst_path, dst_path + ".bak") print(f" [保护] 已将现有文件备份至: {item}.bak") shutil.copy2(src_path, dst_path) print(f" [恢复] {item}") shutil.rmtree(temp_dir, ignore_errors=True) print(f"\n✅ 恢复完成!所有冲突文件已备份为 .bak,请检查必要配置。") return True except Exception as e: print(f"恢复失败: {str(e)}") return False def backup(): try: if not repo_id or not token: print("Skip Backup: HF_DATASET or H_TOKEN not set") return False print("开始创建备份文件...") timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 1. 创建压缩包 with tarfile.open(FILENAME, "w:gz") as tar: paths_to_backup = [ "/app/data", "/root/.openclaw/agents", "/root/.openclaw/canvas", "/root/.openclaw/credentials", "/root/.openclaw/cron", "/root/.openclaw/identity", "/root/.openclaw/openclaw.json", "/root/.openclaw/sessions", "/root/.openclaw/skills", "/root/.openclaw/update-check.json", "/root/.openclaw/workspace", "/root/.openclaw/hexo-blog", # 新增 ] for path in paths_to_backup: if os.path.exists(path): # 针对不同路径生成合适的 arcname(压缩包内路径) if path.startswith("/root/.openclaw/"): arcname = path.replace("/root/.openclaw/", "") elif path.startswith("/app/"): arcname = path.replace("/app/", "") elif path == "/ql/data": arcname = "ql/data" else: arcname = path.lstrip('/') tar.add(path, arcname=arcname) print(f"已加入: {arcname}") print(f"本地备份文件创建完成: {FILENAME}") # 2. 原子 commit (覆盖 latest_backup.tar.gz) operations = [] repo_files = api.list_repo_files(repo_id=repo_id, repo_type="dataset", token=token) if FILENAME in repo_files: operations.append(CommitOperationDelete(path_in_repo=FILENAME)) operations.append(CommitOperationAdd(path_in_repo=FILENAME, path_or_fileobj=FILENAME)) api.create_commit( repo_id=repo_id, repo_type="dataset", operations=operations, commit_message=f"Update latest backup {timestamp}", token=token ) print(f"✅ HF 备份成功!已更新 {FILENAME}") # 3. 执行 LFS 存储清理(彻底释放空间) clean_lfs_objects() # 4. rclone 次要备份 upload_to_rclone(FILENAME, RCLONE_DEST) # 5. 清理本地临时文件 if os.path.exists(FILENAME): os.remove(FILENAME) return True except Exception as e: print(f"备份失败: {str(e)}") if os.path.exists(FILENAME) and os.path.isfile(FILENAME): os.remove(FILENAME) return False if __name__ == "__main__": if len(sys.argv) > 1 and sys.argv[1] == "backup": success = backup() sys.exit(0 if success else 1) else: success = restore() sys.exit(0 if success else 1)