Spaces:
Sleeping
Sleeping
Download sync.py from Alexiabertie/openclaw: direct link, hf CLI and curl.
- Browser
- Download file 9.41 kB
-
https://huggingface.co/spaces/Alexiabertie/openclaw/resolve/main/sync.py
- Command line
-
hf download hf://spaces/Alexiabertie/openclaw/sync.py
-
curl -L -o sync.py https://huggingface.co/spaces/Alexiabertie/openclaw/resolve/main/sync.py
9.41 kB
| import os | |
| import sys | |
| import tarfile | |
| import shutil | |
| import subprocess | |
| import datetime | |
| from huggingface_hub import HfApi, hf_hub_download, CommitOperationAdd, CommitOperationDelete | |
| api = HfApi() | |
| repo_id = os.getenv("HF_DATASET") | |
| token = os.getenv("H_TOKEN") | |
| FILENAME = "latest_backup.tar.gz" | |
| # rclone 目标配置 | |
| RCLONE_DEST = os.getenv("RCLONE_BACKUP_DEST", "nvsu:latest_backup.tar.gz") | |
| # ==================== 配置 ==================== | |
| LFS_KEEP_COUNT = int(os.getenv("LFS_KEEP_COUNT", "5")) # 保留最近 N 个 LFS 对象(存储层面) | |
| LFS_DRY_RUN = os.getenv("LFS_DRY_RUN", "false").lower() in ("true", "1", "yes") | |
| # ================================================ | |
| def upload_to_rclone(local_path, remote_path): | |
| """尝试用 rclone 上传文件到远程存储,失败时仅打印警告""" | |
| if not shutil.which("rclone"): | |
| print("rclone 命令不存在,跳过 rclone 上传") | |
| return False | |
| try: | |
| cmd = ["rclone", "copy", local_path, remote_path, "--progress"] | |
| print(f"正在使用 rclone 上传到 {remote_path} ...") | |
| result = subprocess.run(cmd, capture_output=True, text=True, timeout=300) | |
| if result.returncode == 0: | |
| print(f"rclone 上传成功: {remote_path}") | |
| return True | |
| else: | |
| print(f"rclone 上传失败 (返回码 {result.returncode}): {result.stderr}") | |
| return False | |
| except Exception as e: | |
| print(f"rclone 上传异常: {str(e)}") | |
| return False | |
| def clean_lfs_objects(): | |
| """永久删除旧版本的 LFS 对象以释放存储配额(针对 latest_backup.tar.gz)""" | |
| try: | |
| print("\n🧹 开始清理冗余 LFS 存储对象...") | |
| lfs_files = list(api.list_lfs_files(repo_id=repo_id, repo_type="dataset", token=token)) | |
| candidates = [f for f in lfs_files if f.filename == FILENAME] | |
| print(f" 匹配到 {len(candidates)} 个 '{FILENAME}' 的 LFS 版本") | |
| if not candidates: | |
| print(" 没有发现可清理的 LFS 对象。") | |
| return True | |
| candidates.sort(key=lambda x: x.pushed_at, reverse=True) | |
| to_delete = candidates[LFS_KEEP_COUNT:] | |
| if not to_delete: | |
| print(f" 备份数量 ({len(candidates)}) 未超过保留阈值 ({LFS_KEEP_COUNT}),无需清理。") | |
| return True | |
| print(f" 计划删除 {len(to_delete)} 个旧 LFS 对象:") | |
| for f in to_delete: | |
| print(f" • OID: {f.oid} | Pushed: {f.pushed_at} | Size: {f.size / 1024 / 1024:.2f} MB") | |
| if LFS_DRY_RUN: | |
| print(" [DRY-RUN] 未执行实际删除操作。") | |
| else: | |
| api.permanently_delete_lfs_files( | |
| repo_id=repo_id, | |
| lfs_files=to_delete, | |
| repo_type="dataset", | |
| token=token | |
| ) | |
| print(f"✅ 已永久删除 {len(to_delete)} 个旧 LFS 对象。") | |
| return True | |
| except Exception as e: | |
| print(f"LFS 清理异常: {str(e)}") | |
| return False | |
| def safe_merge_files(src_root, dst_root): | |
| """ | |
| 递归地将 src_root 的内容合并到 dst_root。 | |
| - 目录:使用 dirs_exist_ok=True 进行合并。 | |
| - 文件:如果目标文件已存在,先将其备份为 .bak,再覆盖。 | |
| """ | |
| for item in os.listdir(src_root): | |
| src_path = os.path.join(src_root, item) | |
| dst_path = os.path.join(dst_root, item) | |
| if os.path.isdir(src_path): | |
| os.makedirs(dst_path, exist_ok=True) | |
| # 递归处理子目录,以便对子目录中的文件也能执行 .bak 备份 | |
| safe_merge_files(src_path, dst_path) | |
| elif os.path.isfile(src_path): | |
| if os.path.exists(dst_path): | |
| bak_path = dst_path + ".bak" | |
| shutil.copy2(dst_path, bak_path) | |
| print(f" [保护] 已将现有文件备份至: {os.path.basename(bak_path)}") | |
| shutil.copy2(src_path, dst_path) | |
| print(f" [恢复] {item}") | |
| def restore(): | |
| try: | |
| if not repo_id or not token: | |
| print("Skip Restore: HF_DATASET or H_TOKEN not set") | |
| return False | |
| print(f"正在从 Hugging Face 下载备份文件 {FILENAME} (repo: {repo_id}) ...") | |
| downloaded_path = hf_hub_download( | |
| repo_id=repo_id, | |
| filename=FILENAME, | |
| repo_type="dataset", | |
| token=token | |
| ) | |
| temp_dir = "/root/.openclaw/restore_temp" | |
| if os.path.exists(temp_dir): | |
| shutil.rmtree(temp_dir) | |
| os.makedirs(temp_dir, exist_ok=True) | |
| print(f"解压备份到临时目录: {temp_dir}") | |
| with tarfile.open(downloaded_path, "r:gz") as tar: | |
| tar.extractall(path=temp_dir) | |
| # 1. 处理 /app/data (合并) | |
| temp_data = os.path.join(temp_dir, "data") | |
| target_data = "/app/data" | |
| if os.path.exists(temp_data): | |
| os.makedirs(target_data, exist_ok=True) | |
| shutil.copytree(temp_data, target_data, dirs_exist_ok=True) | |
| print(f"✅ 已合并备份中的 data 目录到 {target_data}") | |
| else: | |
| print("备份包中未找到 data 目录,跳过 /app/data 恢复") | |
| # 2. 处理 /root/.openclaw 下的其他配置 (安全合并) | |
| print("\n🛡️ 开始安全合并配置到 /root/.openclaw ...") | |
| # 排除 data 目录,因为它已经处理过了 | |
| for item in os.listdir(temp_dir): | |
| if item == "data": | |
| continue | |
| src_path = os.path.join(temp_dir, item) | |
| dst_path = os.path.join("/root/.openclaw", item) | |
| if os.path.isdir(src_path): | |
| os.makedirs(dst_path, exist_ok=True) | |
| safe_merge_files(src_path, dst_path) | |
| elif os.path.isfile(src_path): | |
| if os.path.exists(dst_path): | |
| shutil.copy2(dst_path, dst_path + ".bak") | |
| print(f" [保护] 已将现有文件备份至: {item}.bak") | |
| shutil.copy2(src_path, dst_path) | |
| print(f" [恢复] {item}") | |
| shutil.rmtree(temp_dir, ignore_errors=True) | |
| print(f"\n✅ 恢复完成!所有冲突文件已备份为 .bak,请检查必要配置。") | |
| return True | |
| except Exception as e: | |
| print(f"恢复失败: {str(e)}") | |
| return False | |
| def backup(): | |
| try: | |
| if not repo_id or not token: | |
| print("Skip Backup: HF_DATASET or H_TOKEN not set") | |
| return False | |
| print("开始创建备份文件...") | |
| timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") | |
| # 1. 创建压缩包 | |
| with tarfile.open(FILENAME, "w:gz") as tar: | |
| paths_to_backup = [ | |
| "/app/data", | |
| "/root/.openclaw/agents", | |
| "/root/.openclaw/canvas", | |
| "/root/.openclaw/credentials", | |
| "/root/.openclaw/cron", | |
| "/root/.openclaw/identity", | |
| "/root/.openclaw/openclaw.json", | |
| "/root/.openclaw/sessions", | |
| "/root/.openclaw/skills", | |
| "/root/.openclaw/update-check.json", | |
| "/root/.openclaw/workspace", | |
| "/root/.openclaw/hexo-blog", # 新增 | |
| ] | |
| for path in paths_to_backup: | |
| if os.path.exists(path): | |
| # 针对不同路径生成合适的 arcname(压缩包内路径) | |
| if path.startswith("/root/.openclaw/"): | |
| arcname = path.replace("/root/.openclaw/", "") | |
| elif path.startswith("/app/"): | |
| arcname = path.replace("/app/", "") | |
| elif path == "/ql/data": | |
| arcname = "ql/data" | |
| else: | |
| arcname = path.lstrip('/') | |
| tar.add(path, arcname=arcname) | |
| print(f"已加入: {arcname}") | |
| print(f"本地备份文件创建完成: {FILENAME}") | |
| # 2. 原子 commit (覆盖 latest_backup.tar.gz) | |
| operations = [] | |
| repo_files = api.list_repo_files(repo_id=repo_id, repo_type="dataset", token=token) | |
| if FILENAME in repo_files: | |
| operations.append(CommitOperationDelete(path_in_repo=FILENAME)) | |
| operations.append(CommitOperationAdd(path_in_repo=FILENAME, path_or_fileobj=FILENAME)) | |
| api.create_commit( | |
| repo_id=repo_id, | |
| repo_type="dataset", | |
| operations=operations, | |
| commit_message=f"Update latest backup {timestamp}", | |
| token=token | |
| ) | |
| print(f"✅ HF 备份成功!已更新 {FILENAME}") | |
| # 3. 执行 LFS 存储清理(彻底释放空间) | |
| clean_lfs_objects() | |
| # 4. rclone 次要备份 | |
| upload_to_rclone(FILENAME, RCLONE_DEST) | |
| # 5. 清理本地临时文件 | |
| if os.path.exists(FILENAME): | |
| os.remove(FILENAME) | |
| return True | |
| except Exception as e: | |
| print(f"备份失败: {str(e)}") | |
| if os.path.exists(FILENAME) and os.path.isfile(FILENAME): | |
| os.remove(FILENAME) | |
| return False | |
| if __name__ == "__main__": | |
| if len(sys.argv) > 1 and sys.argv[1] == "backup": | |
| success = backup() | |
| sys.exit(0 if success else 1) | |
| else: | |
| success = restore() | |
| sys.exit(0 if success else 1) |