Spaces:
Sleeping
Sleeping
Download data_preprocessing.py from hephaestusonfire/Music_Generator: direct link, hf CLI and curl.
- Browser
- Download file 11.3 kB
-
https://huggingface.co/spaces/hephaestusonfire/Music_Generator/resolve/main/data_preprocessing.py
- Command line
-
hf download hf://spaces/hephaestusonfire/Music_Generator/data_preprocessing.py
-
curl -L -o data_preprocessing.py https://huggingface.co/spaces/hephaestusonfire/Music_Generator/resolve/main/data_preprocessing.py
11.3 kB
| """ | |
| Data preprocessing for grid-based multi-instrument music generation. | |
| - Loads MIDI files from a directory (e.g., MusicNet). | |
| - Maps raw MIDI instruments to 4 families: Strings, Woodwinds, Brass, Percussion. | |
| - Quantizes notes to a fixed grid at 120 BPM, with 8 steps per bar. | |
| - Limits melodic pitches to one octave (C4–B4) and maps to indices 0–11. | |
| - Maps percussion to Kick/Snare/Hi-hat rows. | |
| - Builds sequences of length SEQ_LEN (in steps), with each time step a 39-dim vector: | |
| [12 strings, 12 woodwinds, 12 brass, 3 perc] | |
| - Saves X (inputs) and Y (targets) arrays to disk. | |
| """ | |
| from typing import List, Tuple, Dict | |
| import glob | |
| import os | |
| import numpy as np | |
| import pretty_midi | |
| import config | |
| # One octave: C4 (60) to B4 (71) | |
| MELODIC_BASE_PITCH: int = 60 | |
| MELODIC_MAX_PITCH: int = MELODIC_BASE_PITCH + config.PITCHES_MELODIC - 1 # 71 | |
| # Percussion pitches | |
| PERC_KICK_PITCH: int = 36 # Bass Drum 1 | |
| PERC_SNARE_PITCH: int = 38 # Acoustic Snare | |
| PERC_HIHAT_PITCH: int = 42 # Closed Hi-Hat | |
| def load_midi_paths_from_dir(midi_root: str, pattern: str = "**/*.mid*") -> List[str]: | |
| """ | |
| Recursively collect MIDI file paths from a root directory. | |
| Args: | |
| midi_root: Root directory containing MIDI files. | |
| pattern: Glob pattern relative to root. | |
| Returns: | |
| List of file paths. | |
| """ | |
| search_path = os.path.join(midi_root, pattern) | |
| filepaths = glob.glob(search_path, recursive=True) | |
| return filepaths | |
| # ============ | |
| # Instrument family mapping | |
| # ============ | |
| def get_family_for_program(program: int, is_drum: bool) -> str: | |
| """ | |
| Map a MIDI program number to one of the four families. | |
| Args: | |
| program: MIDI program number (0–127). | |
| is_drum: Whether this is a drum channel. | |
| Returns: | |
| One of: "strings", "woodwinds", "brass", "perc". | |
| """ | |
| if is_drum: | |
| return "perc" | |
| # Very simple heuristic based on General MIDI program ranges. | |
| # You can refine this mapping later as needed. | |
| # Strings: Violins, Violas, Celli, etc. (40–47) | |
| if 40 <= program <= 47: | |
| return "strings" | |
| # Woodwinds/Flutes/Clarinets: Flute (73), Clarinet (71), etc. | |
| if 64 <= program <= 79: | |
| return "woodwinds" | |
| # Brass: Trumpet, Trombone, etc. (56–63) | |
| if 56 <= program <= 63: | |
| return "brass" | |
| # Fallback: treat others as strings | |
| return "strings" | |
| # ============ | |
| # Quantization and grid building | |
| # ============ | |
| def time_to_step_index(t: float) -> int: | |
| """ | |
| Convert time (seconds) to global step index on the quantized grid. | |
| Using config.SECONDS_PER_STEP. | |
| Args: | |
| t: Time in seconds. | |
| Returns: | |
| Global step index (int). | |
| """ | |
| return int(np.floor(t / config.SECONDS_PER_STEP)) | |
| def clip_pitch_to_melodic_range(pitch: int) -> int: | |
| """ | |
| Map MIDI pitch to [MELODIC_BASE_PITCH, MELODIC_MAX_PITCH] if possible. | |
| Returns: | |
| -1 if outside range (should be ignored), or | |
| pitch_idx in [0, PITCHES_MELODIC-1] otherwise. | |
| """ | |
| if pitch < MELODIC_BASE_PITCH or pitch > MELODIC_MAX_PITCH: | |
| return -1 | |
| return pitch - MELODIC_BASE_PITCH | |
| def instrument_to_family_grids( | |
| pm: pretty_midi.PrettyMIDI, | |
| max_bars: int = None, | |
| ) -> Dict[str, np.ndarray]: | |
| """ | |
| Convert a PrettyMIDI object into per-family grids. | |
| Grids have shape: | |
| strings: [bars, STEPS_PER_BAR, 12] | |
| woodwinds: [bars, STEPS_PER_BAR, 12] | |
| brass: [bars, STEPS_PER_BAR, 12] | |
| perc: [bars, STEPS_PER_BAR, 3] | |
| Args: | |
| pm: PrettyMIDI object. | |
| max_bars: Optional cap on number of bars. | |
| Returns: | |
| Dict of family → grid. | |
| """ | |
| total_seconds = pm.get_end_time() | |
| bar_duration = config.SECONDS_PER_BAR | |
| total_bars = int(np.ceil(total_seconds / bar_duration)) | |
| if max_bars is not None: | |
| total_bars = min(total_bars, max_bars) | |
| T = total_bars * config.STEPS_PER_BAR | |
| strings = np.zeros((total_bars, config.STEPS_PER_BAR, config.PITCHES_MELODIC), dtype=np.float32) | |
| woodwinds = np.zeros_like(strings) | |
| brass = np.zeros_like(strings) | |
| perc = np.zeros((total_bars, config.STEPS_PER_BAR, config.PITCHES_PERC), dtype=np.float32) | |
| # Helper to convert time to (bar_idx, step_idx) | |
| def t_to_bar_step(t: float) -> Tuple[int, int]: | |
| step_idx_global = time_to_step_index(t) | |
| bar_idx = step_idx_global // config.STEPS_PER_BAR | |
| step_idx = step_idx_global % config.STEPS_PER_BAR | |
| if bar_idx >= total_bars: | |
| bar_idx = total_bars - 1 | |
| return bar_idx, step_idx | |
| for inst in pm.instruments: | |
| is_drum = inst.is_drum | |
| family = get_family_for_program(inst.program, is_drum=is_drum) | |
| if family == "perc": | |
| for note in inst.notes: | |
| b, s = t_to_bar_step(note.start) | |
| if note.pitch == PERC_KICK_PITCH: | |
| k = 0 | |
| elif note.pitch == PERC_SNARE_PITCH: | |
| k = 1 | |
| elif note.pitch == PERC_HIHAT_PITCH: | |
| k = 2 | |
| else: | |
| continue | |
| perc[b, s, k] = 1.0 | |
| else: | |
| # Select grid | |
| if family == "strings": | |
| grid = strings | |
| elif family == "woodwinds": | |
| grid = woodwinds | |
| elif family == "brass": | |
| grid = brass | |
| else: | |
| grid = strings # fallback | |
| for note in inst.notes: | |
| b, s = t_to_bar_step(note.start) | |
| pitch_idx = clip_pitch_to_melodic_range(note.pitch) | |
| if pitch_idx == -1: | |
| continue | |
| grid[b, s, pitch_idx] = 1.0 | |
| return { | |
| "strings": strings, | |
| "woodwinds": woodwinds, | |
| "brass": brass, | |
| "perc": perc, | |
| } | |
| def grids_to_feature_sequences( | |
| family_grids: Dict[str, np.ndarray], | |
| ) -> np.ndarray: | |
| """ | |
| Convert per-family grids into a [T, 39] feature sequence. | |
| Families: | |
| strings: [bars, STEPS_PER_BAR, 12] | |
| woodwinds: [bars, STEPS_PER_BAR, 12] | |
| brass: [bars, STEPS_PER_BAR, 12] | |
| perc: [bars, STEPS_PER_BAR, 3] | |
| Returns: | |
| seq: [T, 39], where T = bars * STEPS_PER_BAR. | |
| """ | |
| strings = family_grids["strings"] | |
| woodwinds = family_grids["woodwinds"] | |
| brass = family_grids["brass"] | |
| perc = family_grids["perc"] | |
| bars, steps_per_bar, n_strings = strings.shape | |
| T = bars * steps_per_bar | |
| # Reshape to [T, pitches] | |
| strings_flat = strings.reshape(T, n_strings) | |
| woodwinds_flat = woodwinds.reshape(T, woodwinds.shape[-1]) | |
| brass_flat = brass.reshape(T, brass.shape[-1]) | |
| perc_flat = perc.reshape(T, perc.shape[-1]) | |
| # Concatenate along feature dimension: [T, 12+12+12+3] = [T, 39] | |
| seq = np.concatenate( | |
| [strings_flat, woodwinds_flat, brass_flat, perc_flat], | |
| axis=-1, | |
| ) | |
| return seq | |
| def build_sequences_for_file( | |
| midi_path: str, | |
| seq_len_steps: int, | |
| max_bars: int = None, | |
| ) -> Tuple[np.ndarray, np.ndarray]: | |
| """ | |
| Build (X, Y) sequences for a single MIDI file. | |
| - Convert MIDI → family grids → [T, 39] sequence. | |
| - Build sliding window sequences of length seq_len_steps: | |
| X: first seq_len_steps-1 steps | |
| Y: next step (for next-step prediction) | |
| For convenience, we pack them as: | |
| X: [N, seq_len_steps, 39] | |
| Y: [N, seq_len_steps, 39] (shifted by 1). | |
| Args: | |
| midi_path: Path to MIDI file. | |
| seq_len_steps: Sequence length in grid steps (e.g., 64, 128). | |
| max_bars: Optional max bars to consider from this file. | |
| Returns: | |
| X: [N, seq_len_steps, 39] | |
| Y: [N, seq_len_steps, 39] | |
| """ | |
| try: | |
| pm = pretty_midi.PrettyMIDI(midi_path) | |
| except Exception: | |
| # Skip unreadable files | |
| return np.empty((0, seq_len_steps, config.INPUT_FEATURES), dtype=np.float32), \ | |
| np.empty((0, seq_len_steps, config.INPUT_FEATURES), dtype=np.float32) | |
| family_grids = instrument_to_family_grids(pm, max_bars=max_bars) | |
| seq = grids_to_feature_sequences(family_grids) # [T, 39] | |
| T = seq.shape[0] | |
| if T <= seq_len_steps: | |
| return np.empty((0, seq_len_steps, config.INPUT_FEATURES), dtype=np.float32), \ | |
| np.empty((0, seq_len_steps, config.INPUT_FEATURES), dtype=np.float32) | |
| # Sliding window | |
| X_list = [] | |
| Y_list = [] | |
| # We'll create sequences of length seq_len_steps, | |
| # with Y being the same sequence shifted by 1 step. | |
| for start in range(0, T - seq_len_steps): | |
| end = start + seq_len_steps | |
| seq_window = seq[start:end] # [seq_len_steps, 39] | |
| seq_window_next = seq[start + 1:end + 1] # shifted by 1 step | |
| X_list.append(seq_window) | |
| Y_list.append(seq_window_next) | |
| X = np.stack(X_list, axis=0) # [N, seq_len_steps, 39] | |
| Y = np.stack(Y_list, axis=0) # [N, seq_len_steps, 39] | |
| return X.astype(np.float32), Y.astype(np.float32) | |
| def build_dataset_from_dir( | |
| midi_root: str, | |
| seq_len_bars: int, | |
| max_files: int = None, | |
| max_bars_per_file: int = None, | |
| pattern: str = "**/*.mid*", | |
| ) -> Tuple[np.ndarray, np.ndarray]: | |
| """ | |
| Build a full dataset (X, Y) from all MIDI files in a directory. | |
| Args: | |
| midi_root: Root directory containing MIDI files (e.g., MusicNet midis). | |
| seq_len_bars: Sequence length in bars. | |
| max_files: Optional max number of files to use. | |
| max_bars_per_file: Optional cap on bars per file for preprocessing. | |
| pattern: Glob pattern for finding midis under midi_root. | |
| Returns: | |
| X_all: [N_total, seq_len_steps, 39] | |
| Y_all: [N_total, seq_len_steps, 39] | |
| """ | |
| midi_paths = load_midi_paths_from_dir(midi_root, pattern=pattern) | |
| if max_files is not None: | |
| midi_paths = midi_paths[:max_files] | |
| seq_len_steps = seq_len_bars * config.STEPS_PER_BAR | |
| X_all_list = [] | |
| Y_all_list = [] | |
| for path in midi_paths: | |
| X, Y = build_sequences_for_file( | |
| midi_path=path, | |
| seq_len_steps=seq_len_steps, | |
| max_bars=max_bars_per_file, | |
| ) | |
| if X.shape[0] == 0: | |
| continue | |
| X_all_list.append(X) | |
| Y_all_list.append(Y) | |
| if not X_all_list: | |
| # No data | |
| return np.empty((0, seq_len_steps, config.INPUT_FEATURES), dtype=np.float32), \ | |
| np.empty((0, seq_len_steps, config.INPUT_FEATURES), dtype=np.float32) | |
| X_all = np.concatenate(X_all_list, axis=0) | |
| Y_all = np.concatenate(Y_all_list, axis=0) | |
| return X_all, Y_all | |
| def save_dataset_npz( | |
| X: np.ndarray, | |
| Y: np.ndarray, | |
| out_path: str, | |
| ) -> None: | |
| """ | |
| Save (X, Y) arrays to a .npz file. | |
| Args: | |
| X: [N, seq_len_steps, 39] | |
| Y: [N, seq_len_steps, 39] | |
| out_path: path to .npz file. | |
| """ | |
| np.savez_compressed(out_path, X=X, Y=Y) | |
| def save_dataset_pt( | |
| X: np.ndarray, | |
| Y: np.ndarray, | |
| out_path: str, | |
| ) -> None: | |
| """ | |
| Save (X, Y) arrays to a .pt file for PyTorch. | |
| Args: | |
| X: [N, seq_len_steps, 39] | |
| Y: [N, seq_len_steps, 39] | |
| out_path: path to .pt file. | |
| """ | |
| import torch | |
| data = {"X": torch.from_numpy(X), "Y": torch.from_numpy(Y)} | |
| torch.save(data, out_path) |