ACCC1380/molab-moss / backup.sh
ACCC1380's picture
download
raw
9.08 kB
#!/usr/bin/env bash
set -euo pipefail
script_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
if [[ "${1:-}" == --help || "${1:-}" == -h ]]; then
echo "用法:bash backup.sh [目录] [--dry-run]"
echo "固定专用桶:ACCC1380/molab-moss;禁止访问原桶;不删除远端文件。"
echo "包含代码、工作流和公共目录中的模型;不依赖生图输出。"
exit 0
fi
if [[ -n "${HF_BUCKET:-}" && "${HF_BUCKET#hf://buckets/}" != ACCC1380/molab-moss ]]; then
echo "拒绝 HF_BUCKET=$HF_BUCKET;MOSS 脚本固定使用 ACCC1380/molab-moss。" >&2
exit 2
fi
transfer_python="$script_root/.backup-venv/bin/python"
if [[ ! -x "$transfer_python" ]]; then
/usr/local/bin/python -m venv "$script_root/.backup-venv"
fi
if ! "$transfer_python" -I -c 'from huggingface_hub import sync_bucket' >/dev/null 2>&1; then
"$transfer_python" -I -m pip install 'huggingface_hub>=1.9,<2'
fi
"$transfer_python" -I - backup "$script_root" "$@" <<'BUCKET_PY'
import argparse
import json
import os
import re
import stat
import subprocess
import sys
from pathlib import Path
from huggingface_hub import HfApi, get_token, sync_bucket
from huggingface_hub.errors import BucketNotFoundError
mode, default_root = sys.argv[1:3]
parser = argparse.ArgumentParser()
parser.add_argument("directory", nargs="?", default=default_root)
parser.add_argument("--dry-run", action="store_true")
args = parser.parse_args(sys.argv[3:])
root = Path(args.directory).expanduser().resolve()
# This standalone copy is permanently isolated from the original backup bucket.
bucket = "ACCC1380/molab-moss"
requested = os.environ.get("HF_BUCKET", bucket).removeprefix("hf://buckets/").rstrip("/")
if requested != bucket:
parser.error(f"MOSS 脚本仅允许 {bucket},拒绝其他 HF_BUCKET(包括原桶)。")
if root == Path('/marimo') or Path('/marimo') in root.parents:
parser.error("MOSS 脚本禁止读写原 /marimo 目录;请使用 /marimo_moss。")
if mode == 'backup' and not root.is_dir():
parser.error(f"备份目录不存在:{root}")
uri = "hf://buckets/" + bucket
print(f"MOSS 专用桶:{uri};delete=False;原桶不会被写入。", flush=True)
token = get_token()
if not token and (root / '.env').is_file():
for line in (root / '.env').read_text().splitlines():
match = re.match(r"(?:export\s+)?(?:HF_TOKEN|HUGGING_FACE_HUB_TOKEN)\s*=\s*['\"]?(hf_[A-Za-z0-9]+)", line.strip())
if match:
token = match.group(1)
break
if not token:
raise SystemExit("需要 HF 认证:设置 HF_TOKEN 或先执行 hf auth login。")
api = HfApi(token=token)
api.whoami()
if mode == 'backup':
manifest = json.loads((root / 'moss-model-manifest.json').read_text())
subprocess.run(['bash', str(root / 'restore-model-links.sh'), '--check'], check=True)
verified = json.loads((root / 'moss-model-verification.json').read_text())
if len(verified) != len(manifest) or any(not f.get('verified') or f['path'] != m['path'] or f['sha256'] != m['sha256'] for f, m in zip(verified, manifest)):
raise SystemExit('模型校验未完成;禁止上传。')
for item in manifest:
target = root / item['path']
if target.stat().st_size != item['bytes'] or target.with_name(target.name + '.aria2').exists():
raise SystemExit(f'模型不完整:{target}')
try:
api.bucket_info(bucket)
except BucketNotFoundError:
if mode != 'backup' or args.dry_run:
raise SystemExit(f'专用桶尚不存在:{bucket};首次实际备份会创建私有桶。')
api.create_bucket(bucket, private=True, exist_ok=False)
exclude = [
"codex-oneclick-kaggle.sh", "**/codex-oneclick-kaggle.sh",
# Authentication state is recreated on the restored machine.
"ComfyUI/login", "ComfyUI/login/**",
"PASSWORD", "**/PASSWORD", "PASSWORD.*", "**/PASSWORD.*",
".secret-key.txt", "**/.secret-key.txt", ".secret-key.txt.*", "**/.secret-key.txt.*",
"login.txt", "**/login.txt", "credentials.txt", "**/credentials.txt",
"credentials.json", "**/credentials.json",
"comfyui-login", "comfyui-login/**", "**/comfyui-login", "**/comfyui-login/**",
"comfyui-admin", "comfyui-admin/**", "**/comfyui-admin", "**/comfyui-admin/**",
".ssh", ".ssh/**", "**/.ssh", "**/.ssh/**",
".git", ".git/**", "**/.git", "**/.git/**",
"ComfyUI/input/**", "ComfyUI/output/**", "comfyui-output/**",
"ComfyUI/models/SEEDVR2", "ComfyUI/models/SEEDVR2/**",
"ComfyUI/models/text_encoders/text_encoder", "ComfyUI/models/text_encoders/text_encoder/**",
"ComfyUI/models/vae/VAE", "ComfyUI/models/vae/VAE/**",
# Exclude inherited shared-model symlink paths.
"ComfyUI/models/loras/Lora", "ComfyUI/models/loras/Lora/**",
"ComfyUI/models/checkpoints/Stable-diffusion", "ComfyUI/models/checkpoints/Stable-diffusion/**",
"ComfyUI/models/diffusion_models/Stable-diffusion", "ComfyUI/models/diffusion_models/Stable-diffusion/**",
# Krea weights live in /tmp; do not upload through model symlinks.
"Stable-diffusion/krea2_turbo_bf16.safetensors",
"sd-webui-forge-neo/models/Stable-diffusion/krea2_turbo_bf16.safetensors",
"sd-webui-forge-neo/models/text_encoder/qwen3vl_4b_bf16.safetensors",
"sd-webui-forge-neo/models/VAE/krea2_vae_fp32.safetensors",
"ComfyUI/models/vae/qwen_image_vae.safetensors",
"ComfyUI/models/diffusion_models/krea2_turbo_bf16.safetensors",
"ComfyUI/models/text_encoders/qwen3vl_4b_bf16.safetensors",
"ComfyUI/models/vae/krea2_vae_fp32.safetensors",
"comfyui.log", "comfyui.pid", "comfyui-install.log", "qwen-download.log",
"ComfyUI/temp/**", "ComfyUI/user/*.db*", "ComfyUI/user/*.log",
"notebook.py", "**/notebook.py",
".env", ".env.*", "**/.env", "**/.env.*",
"**/__pycache__/**", "__pycache__/**", "__marimo__/**",
"**/.ipynb_checkpoints/**", ".ipynb_checkpoints/**",
".backup-venv/**", "moss-downloads.txt", "*.log", "*.pid", "backup-preview.jsonl",
".venv/**", "venv/**", "**/.venv/**", "**/venv/**",
".cache/**", "**/.cache/**", "*.part", "**/*.part",
"**/.uv-cache/**",
"sd-webui-forge-neo/cache/**",
"sd-webui-forge-neo/extensions/sd-webui-infinite-image-browsing/iib.db*",
"sd-webui-forge-neo/tmp/**", "sd-webui-forge-neo/output/**",
"sd-webui-forge-neo/outputs/**",
]
# Model directories in applications are links, recreated after restore.
model_aliases = ['ComfyUI/models/checkpoints', 'ComfyUI/models/diffusion_models', 'ComfyUI/models/loras', 'ComfyUI/models/text_encoders', 'ComfyUI/models/vae', 'sd-webui-forge-neo/models/Stable-diffusion', 'sd-webui-forge-neo/models/Lora', 'sd-webui-forge-neo/models/text_encoder', 'sd-webui-forge-neo/models/VAE']
for alias in model_aliases:
exclude.extend([alias, alias + "/**"])
# Generated pictures and inherited image assets are not part of the remote backup.
image_extensions = "png apng jpg jpeg webp gif bmp tif tiff avif heic svg ico exr psd".split()
exclude.extend("*." + "".join(f"[{c}{c.upper()}]" for c in ext) for ext in image_extensions)
exclude.extend(["ComfyUI/user/__manager/cache/**", "**/*.log", "**/*.pid"])
# Back up weights from shared physical directories only.
if mode == 'backup':
allowed = {item['shared'] for item in json.loads((root / 'model-links.json').read_text())}
weight_extensions = {'.safetensors', '.ckpt', '.pt', '.pth', '.bin', '.gguf', '.onnx', '.h5', '.npz', '.pkl', '.tflite', '.pb', '.task'}
for directory, dirs, files in os.walk(root, followlinks=False):
dirs[:] = [d for d in dirs if d not in {'.venv', '.backup-venv', '.git', '.cache', '__pycache__'}]
for name in files:
path = Path(directory) / name
relative = path.relative_to(root).as_posix()
if path.suffix.lower() in weight_extensions and relative.split('/', 1)[0] not in allowed:
exclude.append(relative)
exclude.extend(['*.aria2', '*.incomplete', '*.part', '**/blobs/**', '**/xet/**'])
source, destination = (str(root), uri) if mode == 'backup' else (uri, str(root))
print(f'{source} -> {destination}', flush=True)
result = sync_bucket(source, destination, token=token, exclude=exclude, delete=False, dry_run=args.dry_run)
print(result.summary(), flush=True)
if args.dry_run:
print('预览完成;未传输文件。')
raise SystemExit(0)
if mode == 'restore':
for name in ('backup.sh', 'restore.sh', 'start.sh', 'comfyui.sh', 'download_moss.py', 'restore-model-links.sh'):
file = root / name
if file.is_file():
file.chmod(file.stat().st_mode | stat.S_IXUSR)
subprocess.run(['bash', str(root / 'restore-model-links.sh'), str(root)], check=True)
print(f'恢复完成,模型链接已自动重建。运行 cd {root} && bash comfyui.sh;首次启动自动安装已锁定的依赖并初始化登录。')
print('默认端口 8189;登录信息:/var/lib/comfyui-moss-admin/login.txt。')
else:
print('MOSS 备份完成:代码、工作流和公共目录中的模型已上传;原桶未写入;远端文件未删除。')
BUCKET_PY

Xet Storage Details

Size:
9.08 kB
·
Xet hash:
35daf3bb47c3dd9132453eec62ba75cca9b313560a70112b1315ac9481f68fb1

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.