SoundStage / setup_weights.py
mamathew's picture
Initial Soundstage ZeroGPU Space
7d72aa0 verified
Raw History Blame Contribute Delete
6 kB
"""
Prepare inference code + checkpoints for the Soundstage Space.
Modes:
1) Monorepo: if parent folder already has inference_v2st.py (this project), use it.
2) Standalone Space: clone Foley-Omni from GitHub, then download Hub weights.
"""
from __future__ import annotations
import os
import subprocess
from pathlib import Path
HERE = Path(__file__).resolve().parent
PARENT = HERE.parent
REPO_URL = os.getenv("FOLEY_REPO_URL", "https://github.com/SAIL-PRLab/Foley-Omni.git").strip()
REPO_COMMIT = os.getenv("FOLEY_COMMIT", "").strip()
WEIGHTS_REPO = os.getenv("FOLEY_WEIGHTS_REPO", "CocoBro/Foley-Omni").strip()
# When Space repo is only hf_space/*, code lives under vendor/
VENDOR_DIR = HERE / "vendor" / "Foley-Omni"
CKPT_FILES = [
"ckpts/Foley-Omni/v2st.pth",
"ckpts/Wan2.2-TI2V-5B/models_t5_umt5-xxl-enc-bf16.pth",
"ckpts/Wan2.2-TI2V-5B/google/umt5-xxl/special_tokens_map.json",
"ckpts/Wan2.2-TI2V-5B/google/umt5-xxl/spiece.model",
"ckpts/Wan2.2-TI2V-5B/google/umt5-xxl/tokenizer.json",
"ckpts/Wan2.2-TI2V-5B/google/umt5-xxl/tokenizer_config.json",
"ckpts/mmaudio/ext_weights/v1-16.pth",
"ckpts/mmaudio/ext_weights/best_netG.pt",
"ckpts/mmaudio/ext_weights/synchformer_state_dict.pth",
]
REPO_DIR: Path | None = None
def resolve_repo_dir() -> Path:
global REPO_DIR
if REPO_DIR is not None:
return REPO_DIR
if (PARENT / "inference_v2st.py").is_file() and (PARENT / "foley_omni").is_dir():
REPO_DIR = PARENT
print(f"[setup] using monorepo at {REPO_DIR}")
return REPO_DIR
REPO_DIR = VENDOR_DIR
print(f"[setup] using vendor dir {REPO_DIR}")
return REPO_DIR
def _apply_patch(file_path: Path, old: str, new: str, label: str) -> None:
content = file_path.read_text(encoding="utf-8")
if new in content:
print(f"[patch] {label}: already patched")
return
if old not in content:
print(f"[patch] {label}: pattern not found — skipping (upstream may differ)")
return
file_path.write_text(content.replace(old, new), encoding="utf-8")
print(f"[patch] {label}: patched")
def apply_zerogpu_patches(repo: Path) -> None:
"""Best-effort RAM / attention patches used by the working FOG Space."""
m = repo / "foley_omni"
if not m.is_dir():
return
_apply_patch(
m / "modules" / "t5.py",
old=""" # init model
model = umt5_xxl(
encoder_only=True,
return_tokenizer=False,
dtype=dtype,
device=device if not cpu_offload else "cpu").eval().requires_grad_(False)
logging.info(f'loading {checkpoint_path}')
model.load_state_dict(torch.load(checkpoint_path, map_location='cpu'))
self.model = model""",
new=""" # init model on the meta device (allocates no real memory yet)
model = umt5_xxl(
encoder_only=True,
return_tokenizer=False,
dtype=dtype,
device="meta").eval().requires_grad_(False)
logging.info(f'loading {checkpoint_path}')
try:
state_dict = torch.load(checkpoint_path, map_location='cpu', mmap=True, weights_only=True)
except Exception:
state_dict = torch.load(checkpoint_path, map_location='cpu')
model.load_state_dict(state_dict, assign=True)
del state_dict
target_device = device if not cpu_offload else "cpu"
model = model.to(target_device)
self.model = model""",
label="t5.py",
)
_apply_patch(
m / "fusion_engine.py",
old=""" logging.info(f"Loading model checkpoint from {checkpoint_path}")
checkpoint = torch.load(checkpoint_path, map_location="cpu")
if isinstance(checkpoint, dict) and "model" in checkpoint:""",
new=""" logging.info(f"Loading model checkpoint from {checkpoint_path}")
try:
checkpoint = torch.load(checkpoint_path, map_location="cpu", mmap=True, weights_only=True)
except Exception:
checkpoint = torch.load(checkpoint_path, map_location="cpu")
if isinstance(checkpoint, dict) and "model" in checkpoint:""",
label="fusion_engine.py",
)
imp_old = "from .attention import flash_attention"
imp_new = "from .attention import attention as flash_attention"
for rel in ("modules/model.py", "modules/clip.py", "modules/__init__.py"):
p = m / rel
if p.is_file():
_apply_patch(p, imp_old, imp_new, f"{rel} attention import")
def clone_repo_if_needed(repo: Path) -> None:
if (repo / "inference_v2st.py").is_file():
print("[setup] inference code already present")
return
repo.parent.mkdir(parents=True, exist_ok=True)
if REPO_COMMIT:
subprocess.run(["git", "clone", REPO_URL, str(repo)], check=True)
subprocess.run(["git", "-C", str(repo), "checkout", REPO_COMMIT], check=True)
else:
subprocess.run(["git", "clone", "--depth", "1", REPO_URL, str(repo)], check=True)
def download_checkpoints(repo: Path) -> None:
missing = [f for f in CKPT_FILES if not (repo / f).is_file()]
if not missing:
print("[setup] checkpoints already present")
return
from huggingface_hub import snapshot_download
print(f"[setup] downloading {len(missing)} files from {WEIGHTS_REPO} ...")
token = os.getenv("HF_TOKEN") or os.getenv("HUGGING_FACE_HUB_TOKEN")
snapshot_download(
WEIGHTS_REPO,
local_dir=str(repo),
allow_patterns=CKPT_FILES,
token=token,
)
still = [f for f in CKPT_FILES if not (repo / f).is_file()]
if still:
raise RuntimeError(f"Checkpoint download incomplete: {still}")
def ensure_ready() -> Path:
repo = resolve_repo_dir()
clone_repo_if_needed(repo)
apply_zerogpu_patches(repo)
download_checkpoints(repo)
print("[setup] ready")
return repo
if __name__ == "__main__":
ensure_ready()