File size: 4,730 Bytes
c0aa76e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 | #!/usr/bin/env python3
"""
rec7_render.py — "cover" a real recording through M3.
Reads the recording with rec7, builds the renderer condition, and runs
the M3 pipeline with that condition injected in place of the language
model's states. The LM is skipped entirely; timbre, guitar and
performance come from the recording.
python rec7_render.py song.mp3 --m3 /path/to/minimax_music3 \
--lyrics lyrics.txt --out cover.flac
Caption and lyrics are still fed to the pipeline (it expects them), but
the states drive the audio, so they matter little. 30 steps, guidance
as the pipeline defaults.
"""
import argparse
import sys
from pathlib import Path
import numpy as np
import torch
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
H_DIM = 4096
def render_with_condition(pipe, cond, caption, lyrics, steps=30, seed=7):
"""Run the M3 pipeline with cond [T, 32768] injected as the
semantic step's frame hiddens. Flat [1, T, 8*4096] is what the
condition encoder expects."""
from diffusers.modular_pipelines.minimax_music3.encoders import (
MiniMaxMusic3SemanticGenerationStep as SemStep)
orig = SemStep.__call__
T = cond.shape[0]
inject = cond.reshape(1, T, 8 * H_DIM).to("cuda", torch.bfloat16)
def patched(self, components, state):
bs = self.get_block_state(state)
bs.frame_hiddens = inject
self.set_block_state(state, bs)
return components, state
SemStep.__call__ = patched
try:
audio = pipe(prompt=caption, lyrics=lyrics, audio_duration=T / 25.0,
num_inference_steps=steps,
generator=torch.Generator("cuda").manual_seed(seed),
output="audios")[0]
finally:
SemStep.__call__ = orig
arr = np.asarray(audio, dtype=np.float32)
return arr[None, :] if arr.ndim == 1 else arr
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("audio", type=Path)
ap.add_argument("--m3", type=Path, required=True)
ap.add_argument("--weights", type=Path, default=HERE / "weights")
ap.add_argument("--lyrics", type=Path, default=None)
ap.add_argument("--caption", type=str, default="acoustic song")
ap.add_argument("--seconds", type=float, default=60.0,
help="length to cover (0 = whole file)")
ap.add_argument("--steps", type=int, default=30)
ap.add_argument("--seed", type=int, default=7)
ap.add_argument("--out", type=Path, default=None)
a = ap.parse_args()
for k in ("audio", "m3", "weights", "out", "lyrics"):
v = getattr(a, k, None)
if isinstance(v, Path):
setattr(a, k, v.expanduser().resolve())
if not a.audio.exists():
print(f"audio not found: {a.audio}")
return 1
import soundfile as sf
import torchaudio
from rec7_model import (load_rec7, load_dav, lm_semantic_tables,
encode_audio, read_states, states_to_streams)
from diffusers import ModularPipeline
dev = "cuda"
wav, sr = torchaudio.load(str(a.audio))
if sr != 44100:
wav = torchaudio.transforms.Resample(sr, 44100)(wav)
if wav.shape[0] == 1:
wav = wav.repeat(2, 1)
wav = wav[:2].float()
if a.seconds > 0:
wav = wav[:, : int(a.seconds * 44100)]
dav = load_dav(a.m3, dev)
z = encode_audio(dav, wav, dev)
del dav
trunk, head = load_rec7(a.weights, dev)
h = read_states(trunk, head, z, dev)
del trunk, head
torch.cuda.empty_cache()
pipe = ModularPipeline.from_pretrained(str(a.m3))
try:
pipe.load_components(
names=["tokenizer", "rvq_depth_decoder", "condition_encoder",
"transformer", "scheduler", "vocoder", "guider"],
dtype=torch.bfloat16)
except TypeError:
pipe.load_components(dtype=torch.bfloat16)
pipe.to(dev)
W_head, W_embed = lm_semantic_tables(a.m3, dev)
_codes, cond = states_to_streams(h, pipe.rvq_depth_decoder,
W_head, W_embed, dev)
# the pipeline's text encoder refuses empty lyrics; the injected
# states drive the audio, so a placeholder is fine when none are given
lyr = a.lyrics.read_text(encoding="utf-8").strip() if a.lyrics else ""
if not lyr:
lyr = "[verse]\nla la la la la\nla la la la la\n"
arr = render_with_condition(pipe, cond, a.caption, lyr, a.steps, a.seed)
if arr.shape[0] == 1:
arr = np.repeat(arr, 2, axis=0)
p = a.out or a.audio.with_suffix(".cover.flac")
sf.write(str(p), arr.T, 44100, subtype="PCM_24")
print(f"{p}: {arr.shape[-1] / 44100:.1f}s")
return 0
if __name__ == "__main__":
raise SystemExit(main())
|