Music_Generator / inference.py
hephaestusonfire's picture
Upload 11 files
9c7e38e verified
Raw History Blame Contribute Delete
14.6 kB
"""
Inference utilities: load trained model and generate music from seed grids.
"""
from typing import Dict, Tuple
import math
import os
import numpy as np
import torch
import config
from model import build_model
from utils import sampling_utils, midi_utils
# =========================
# Model loading
# =========================
_model: torch.nn.Module = None
def load_model(
weights_path: str = "models/transformer_music_best.pt",
device: str = "cpu",
) -> torch.nn.Module:
"""
Load the trained MusicTransformer model for inference.
Args:
weights_path: Path to the .pt weights file.
device: Device to map the model to ("cpu" for HF Spaces).
Returns:
model: Loaded MusicTransformer in eval mode.
"""
global _model
if _model is not None:
return _model
model = build_model(config)
state_dict = torch.load(weights_path, map_location=device)
model.load_state_dict(state_dict)
model.to(device)
model.eval()
_model = model
return _model
# =========================
# Seed conversion
# =========================
def seed_grids_to_sequence(
strings_seed: np.ndarray,
woodwinds_seed: np.ndarray,
brass_seed: np.ndarray,
perc_seed: np.ndarray,
) -> np.ndarray:
"""
Convert 4 seed grids into a [T_seed, 39] feature sequence.
Seed shapes:
strings_seed: [INPUT_BARS, STEPS_PER_BAR, 12]
woodwinds_seed: [INPUT_BARS, STEPS_PER_BAR, 12]
brass_seed: [INPUT_BARS, STEPS_PER_BAR, 12]
perc_seed: [INPUT_BARS, STEPS_PER_BAR, 3]
Returns:
seq: [T_seed, 39] float32
"""
# Basic checks
bars, steps, p_s = strings_seed.shape
assert bars == config.INPUT_BARS
assert steps == config.STEPS_PER_BAR
assert p_s == config.PITCHES_MELODIC
# Flatten bars × steps → T
T_seed = bars * steps
strings_flat = strings_seed.reshape(T_seed, config.PITCHES_MELODIC)
woodwinds_flat = woodwinds_seed.reshape(T_seed, config.PITCHES_MELODIC)
brass_flat = brass_seed.reshape(T_seed, config.PITCHES_MELODIC)
perc_flat = perc_seed.reshape(T_seed, config.PITCHES_PERC)
seq = np.concatenate(
[strings_flat, woodwinds_flat, brass_flat, perc_flat],
axis=-1,
).astype(np.float32) # [T_seed, 39]
return seq
def sequence_to_grids(
seq: np.ndarray,
) -> Dict[str, np.ndarray]:
"""
Convert a [T, 39] sequence back into per-family grids.
Returns:
{
"strings": [bars, STEPS_PER_BAR, 12],
"woodwinds": [bars, STEPS_PER_BAR, 12],
"brass": [bars, STEPS_PER_BAR, 12],
"perc": [bars, STEPS_PER_BAR, 3],
}
"""
T = seq.shape[0]
bars = T // config.STEPS_PER_BAR
seq = seq[: bars * config.STEPS_PER_BAR] # trim to full bars if needed
strings = seq[:, 0:12]
woodwinds = seq[:, 12:24]
brass = seq[:, 24:36]
perc = seq[:, 36:39]
strings_grid = strings.reshape(bars, config.STEPS_PER_BAR, config.PITCHES_MELODIC)
woodwinds_grid = woodwinds.reshape(bars, config.STEPS_PER_BAR, config.PITCHES_MELODIC)
brass_grid = brass.reshape(bars, config.STEPS_PER_BAR, config.PITCHES_MELODIC)
perc_grid = perc.reshape(bars, config.STEPS_PER_BAR, config.PITCHES_PERC)
return {
"strings": strings_grid,
"woodwinds": woodwinds_grid,
"brass": brass_grid,
"perc": perc_grid,
}
# =========================
# Generation
# =========================
def compute_target_bars(target_minutes: float) -> int:
"""
Compute number of bars needed to approximate target_minutes,
given fixed tempo and meter.
Returns:
target_bars: int
"""
total_seconds = target_minutes * 60.0
seconds_per_bar = config.SECONDS_PER_BAR
target_bars = math.ceil(total_seconds / seconds_per_bar)
return target_bars
def generate_from_seed(
strings_seed: np.ndarray,
woodwinds_seed: np.ndarray,
brass_seed: np.ndarray,
perc_seed: np.ndarray,
target_minutes: float,
temperature: float = 1.0,
top_k: int = 0,
device: str = "cpu",
) -> Dict[str, np.ndarray]:
"""
Generate full grids from seed grids using the trained model.
Args:
strings_seed, woodwinds_seed, brass_seed, perc_seed:
Seed grids from UI, shapes:
[INPUT_BARS, STEPS_PER_BAR, 12] for melodic families,
[INPUT_BARS, STEPS_PER_BAR, 3] for perc.
target_minutes: Desired approximate length in minutes.
temperature: Sampling temperature.
top_k: top-k filter.
device: "cpu" or "cuda".
Returns:
full_grids dict:
{
"strings": [total_bars, STEPS_PER_BAR, 12],
"woodwinds": [total_bars, STEPS_PER_BAR, 12],
"brass": [total_bars, STEPS_PER_BAR, 12],
"perc": [total_bars, STEPS_PER_BAR, 3],
}
This includes seed bars + generated bars.
"""
model = load_model(device=device) # global cached
# Convert seeds to [T_seed, 39]
seed_seq = seed_grids_to_sequence(
strings_seed=strings_seed,
woodwinds_seed=woodwinds_seed,
brass_seed=brass_seed,
perc_seed=perc_seed,
) # [T_seed, 39]
T_seed = seed_seq.shape[0]
seed_bars = T_seed // config.STEPS_PER_BAR
target_bars = compute_target_bars(target_minutes)
# Ensure target_bars >= seed_bars
if target_bars <= seed_bars:
target_bars = seed_bars + 1
total_bars_to_generate = target_bars - seed_bars
# Start with seed sequence as context
seq = seed_seq.copy() # [T_cur, 39]
# Keep track of bar dicts (including seed bars)
all_bars: Dict[str, list] = {
"strings": [],
"woodwinds": [],
"brass": [],
"perc": [],
}
# First, add the seed bars to all_bars
seed_grids = sequence_to_grids(seed_seq)
for key in all_bars.keys():
for b in range(seed_grids[key].shape[0]):
all_bars[key].append(seed_grids[key][b])
# Previous generated bar for similarity check (full dict)
prev_bar_dict = {
"strings": all_bars["strings"][-1],
"woodwinds": all_bars["woodwinds"][-1],
"brass": all_bars["brass"][-1],
"perc": all_bars["perc"][-1],
}
for bar_idx in range(total_bars_to_generate):
# Collect logits for one bar
bar_step_logits = {
"strings": [],
"woodwinds": [],
"brass": [],
"perc": [],
}
for step_in_bar in range(config.STEPS_PER_BAR):
seq_tensor = torch.from_numpy(seq).unsqueeze(0).to(device) # [1, T_cur, 39]
with torch.no_grad():
outputs = model(seq_tensor) # dict of [1, T_cur, pitches]
# Take logits for the last time step
strings_logits = outputs["strings"][:, -1, :] # [1, 12]
woodwinds_logits = outputs["woodwinds"][:, -1, :]
brass_logits = outputs["brass"][:, -1, :]
perc_logits = outputs["perc"][:, -1, :]
bar_step_logits["strings"].append(strings_logits)
bar_step_logits["woodwinds"].append(woodwinds_logits)
bar_step_logits["brass"].append(brass_logits)
bar_step_logits["perc"].append(perc_logits)
# Append placeholder next-step feature
next_step_placeholder = np.zeros((1, config.INPUT_FEATURES), dtype=np.float32)
seq = np.concatenate([seq, next_step_placeholder], axis=0) # [T_cur+1, 39]
# Stack logits to [1, T, P] per family
logits_dict = {
"strings": torch.stack(bar_step_logits["strings"], dim=1), # [1, T, 12]
"woodwinds": torch.stack(bar_step_logits["woodwinds"], dim=1),
"brass": torch.stack(bar_step_logits["brass"], dim=1),
"perc": torch.stack(bar_step_logits["perc"], dim=1), # [1, T, 3]
}
# Sample a bar with constraints
bar_dict = sampling_utils.resample_bar_if_needed(
prev_bar=prev_bar_dict,
logits_dict=logits_dict,
temperature=temperature,
top_k=top_k,
# using defaults inside resample for min_notes_per_bar, max_notes_per_step
)
# Append this bar
for key in all_bars.keys():
all_bars[key].append(bar_dict[key])
# Update prev_bar_dict
prev_bar_dict = bar_dict
# Update the last STEPS_PER_BAR placeholders in seq with actual features
T_cur = seq.shape[0]
start_idx = T_cur - config.STEPS_PER_BAR
end_idx = T_cur
strings_bar = bar_dict["strings"] # [T, 12]
woodwinds_bar = bar_dict["woodwinds"]
brass_bar = bar_dict["brass"]
perc_bar = bar_dict["perc"]
bar_features = np.concatenate(
[strings_bar, woodwinds_bar, brass_bar, perc_bar],
axis=-1,
).astype(np.float32) # [T, 39]
seq[start_idx:end_idx, :] = bar_features
# Convert all_bars lists to numpy grids
strings_grid = np.stack(all_bars["strings"], axis=0) # [bars, STEPS_PER_BAR, 12]
woodwinds_grid = np.stack(all_bars["woodwinds"], axis=0)
brass_grid = np.stack(all_bars["brass"], axis=0)
perc_grid = np.stack(all_bars["perc"], axis=0)
return {
"strings": strings_grid,
"woodwinds": woodwinds_grid,
"brass": brass_grid,
"perc": perc_grid,
}
# =========================
# Orchestration from primary family
# =========================
def orchestrate_from_primary(
full_grids: Dict[str, np.ndarray],
primary_family: str,
) -> Dict[str, np.ndarray]:
"""
Build other families from a chosen primary melodic family using simple rules.
Args:
full_grids: raw model output:
"strings": [B, T, 12]
"woodwinds": [B, T, 12]
"brass": [B, T, 12]
"perc": [B, T, 3]
primary_family: one of "strings", "woodwinds", "brass".
Returns:
new_grids: dict with same keys, but non-primary families derived from primary.
"""
grids = {k: v.copy() for k, v in full_grids.items()}
if primary_family not in ["strings", "woodwinds", "brass"]:
return grids
primary = grids[primary_family] # [B, T, 12]
B, T, P = primary.shape
# Reset melodic families to zero
for fam in ["strings", "woodwinds", "brass"]:
grids[fam] = np.zeros_like(primary)
strong_steps = [0, 2, 4, 6]
def thinned_copy(src: np.ndarray) -> np.ndarray:
dst = np.zeros_like(src)
dst[:, strong_steps, :] = src[:, strong_steps, :]
return dst
def root_pattern(src: np.ndarray) -> np.ndarray:
dst = np.zeros_like(src)
keep_steps = [0, 4]
dst[:, keep_steps, :] = src[:, keep_steps, :]
return dst
if primary_family == "strings":
grids["strings"] = primary
grids["woodwinds"] = thinned_copy(primary)
grids["brass"] = root_pattern(primary)
elif primary_family == "woodwinds":
grids["woodwinds"] = primary
grids["strings"] = thinned_copy(primary)
grids["brass"] = root_pattern(primary)
elif primary_family == "brass":
grids["brass"] = primary
grids["strings"] = thinned_copy(primary)
grids["woodwinds"] = root_pattern(primary)
# Percussion: simple fixed pattern
perc = np.zeros_like(grids["perc"])
for b in range(B):
for t in range(T):
if t in [0, 4]:
perc[b, t, 0] = 1.0 # kick-like
if t in [2, 6]:
perc[b, t, 1] = 1.0 # snare-like
grids["perc"] = perc
return grids
# =========================
# MIDI / audio helpers
# =========================
def generate_midi_file(
strings_seed: np.ndarray,
woodwinds_seed: np.ndarray,
brass_seed: np.ndarray,
perc_seed: np.ndarray,
target_minutes: float,
temperature: float,
top_k: int,
midi_out_path: str = "output.mid",
device: str = "cpu",
primary_family: str = "strings",
) -> str:
"""
High-level helper: generate music from seed grids and write to a MIDI file.
Returns:
Path to the written .mid file.
"""
full_grids = generate_from_seed(
strings_seed=strings_seed,
woodwinds_seed=woodwinds_seed,
brass_seed=brass_seed,
perc_seed=perc_seed,
target_minutes=target_minutes,
temperature=temperature,
top_k=top_k,
device=device,
)
# Rebuild orchestration from chosen primary family
orchestrated = orchestrate_from_primary(full_grids, primary_family=primary_family)
pm = midi_utils.create_pretty_midi_from_grids(
strings_grid=orchestrated["strings"],
woodwinds_grid=orchestrated["woodwinds"],
brass_grid=orchestrated["brass"],
perc_grid=orchestrated["perc"],
)
os.makedirs(os.path.dirname(midi_out_path) or ".", exist_ok=True)
return midi_utils.save_midi(pm, midi_out_path)
def generate_midi_and_audio(
strings_seed: np.ndarray,
woodwinds_seed: np.ndarray,
brass_seed: np.ndarray,
perc_seed: np.ndarray,
target_minutes: float,
temperature: float,
top_k: int,
midi_out_path: str = "output.mid",
audio_out_path: str = "output.wav",
device: str = "cpu",
primary_family: str = "strings",
) -> Tuple[str, str]:
"""
Generate MIDI and render audio via Fluidsynth.
Returns:
(midi_path, audio_path)
"""
midi_path = generate_midi_file(
strings_seed=strings_seed,
woodwinds_seed=woodwinds_seed,
brass_seed=brass_seed,
perc_seed=perc_seed,
target_minutes=target_minutes,
temperature=temperature,
top_k=top_k,
midi_out_path=midi_out_path,
device=device,
primary_family=primary_family,
)
# Render audio (requires fluidsynth and soundfont)
try:
import fluidsynth
import pretty_midi
from scipy.io import wavfile
fs = fluidsynth.Synth()
sfid = fs.sfload("soundfont.sf2")
fs.program_select(0, sfid, 0, 0)
pm = pretty_midi.PrettyMIDI(midi_path)
audio = pm.fluidsynth(fs=16000)
wavfile.write(audio_out_path, 16000, audio.astype(np.float32))
fs.delete()
except Exception as e:
print(f"Audio rendering failed: {e}")
audio_out_path = ""
return midi_path, audio_out_path