| import sys |
| sys.path.append('code') |
|
|
|
|
| import numpy as np |
| import os |
| import pretty_midi as pyd |
| import torch |
| import sys |
| from model import VAE |
| from util_tools.format_converter import melody_data2matrix, melody_matrix2data, chord_data2matrix, chord_matrix2data |
| from torch.distributions import kl_divergence, Normal |
| from nottingham_dataset import Nottingham |
| import copy |
|
|
| def chord_grid2data(est_pitch, bpm=60., start=0., max_simu_note=6, pitch_eos=129, num_step=32, min_pitch=0): |
| est_pitch = est_pitch[:, :, 0] |
| if est_pitch.shape[1] == max_simu_note: |
| est_pitch = est_pitch[:, 1:] |
| |
| |
| |
| harmonic_rhythm = 1. - (est_pitch[:, 0]==pitch_eos) * 1. |
| |
|
|
| pr = np.zeros((32, 128), dtype=int) |
| alpha = 0.25 * 60 / bpm |
| notes = [] |
| for t in range(num_step): |
| for n in range(max_simu_note-1): |
| note = est_pitch[t, n] |
| if note == pitch_eos: |
| break |
| pitch = note + 12*4 |
| duration = 1 |
| for j in range(t+1, num_step): |
| if harmonic_rhythm[j] == 1: |
| break |
| duration +=1 |
| pr[t, pitch] = min(duration, 32 - t) |
| notes.append( |
| pyd.Note(100, int(pitch), start + t * alpha, |
| start + (t + duration) * alpha)) |
| chord = pyd.Instrument(program=pyd.instrument_name_to_program('Acoustic Grand Piano')) |
| chord.notes = notes |
| return chord |
|
|
| def melody_matrix2data(melody_matrix, tempo=120, start_time=0.0, get_list=False): |
| HOLD_PITCH = 12 |
| REST_PITCH = 13 |
| |
| chroma = np.concatenate((melody_matrix[:, :12], melody_matrix[:, 15: 17]), axis=-1) |
| register = melody_matrix[:, -10:] |
| |
| melodySequence = np.argmax(chroma, axis=-1) |
| |
| |
| melody_notes = [] |
| minStep = 60 / tempo / 4 |
| onset_or_rest = [i for i in range(len(melodySequence)) if not melodySequence[i]==HOLD_PITCH] |
| onset_or_rest.append(len(melodySequence)) |
|
|
| for idx, onset in enumerate(onset_or_rest[:-1]): |
| if melodySequence[onset] == REST_PITCH: |
| continue |
| else: |
| pitch = melodySequence[onset] + 12 * np.argmax(register[onset]) |
| |
| start = onset * minStep |
| end = onset_or_rest[idx+1] * minStep |
| noteRecon = pyd.Note(velocity=100, pitch=pitch, start=start_time+start, end=start_time+end) |
| melody_notes.append(noteRecon) |
| if get_list: |
| return melody_notes |
| else: |
| melody = pyd.Instrument(program=pyd.instrument_name_to_program('Acoustic Grand Piano')) |
| melody.notes = melody_notes |
| return melody |
|
|
|
|
| def get_gt(chord, melody): |
| |
| |
| chord_recon = chord_grid2data(chord, 30, pitch_eos=13) |
| melody_recon = melody_matrix2data(melody, 120) |
| music = pyd.PrettyMIDI(initial_tempo=120) |
| music.instruments.append(melody_recon) |
| music.instruments.append(chord_recon) |
| return music |
|
|
| def shift(original_melody, p_shift): |
| melody = copy.deepcopy(original_melody).cpu().detach().numpy()[0] |
| onsets, pitch = np.nonzero(melody[:, :12]) |
| onsets, register = np.nonzero(melody[:, -10:]) |
| onset130 = pitch + register*12 |
| onset130 += p_shift |
| onset12 = onset130 % 12 |
| register = onset130 // 12 |
| melody[onsets, :] = 0 |
| melody[onsets, onset12] = 1. |
| melody[onsets, register+17] = 1. |
| return torch.from_numpy(melody).float().unsqueeze(0) |
|
|
|
|
| def reconstruct(chord, melody): |
| |
| |
| lengths = model.get_len_index_tensor(chord) |
| chord = model.index_tensor_to_multihot_tensor(chord) |
| chord = model.enc_note_embedding(chord) |
| mel_ebd = model.enc_note_embedding(melody) |
| melody_beat_summary = mel_ebd[:, ::4, :] + mel_ebd[:, 1::4, :] + mel_ebd[:, 2::4, :] + mel_ebd[:, 3::4, :] |
| dist, mu, = model.encoder(chord, lengths, melody_beat_summary) |
| z = dist.mean |
| pitch_outs = model.decoder(z, melody_beat_summary, |
| inference=True, x=None, lengths=None, |
| teacher_forcing_ratio1=0., teacher_forcing_ratio2=0.) |
| pitch_outs = pitch_outs.max(-1, keepdim=True)[1] |
| pitch_outs = pitch_outs.cpu().detach().numpy() |
| chord_track = chord_grid2data(pitch_outs[0], bpm=120//4, start=0, pitch_eos=13) |
| |
| melody = melody.cpu().detach().numpy()[0] |
| melody_track = melody_matrix2data(melody, tempo=120) |
|
|
| music = pyd.PrettyMIDI() |
| music.instruments.append(melody_track) |
| music.instruments.append(chord_track) |
| return music |
|
|
| def melody_control(chord, melody, new_melody): |
| |
| |
| |
|
|
| lengths = model.get_len_index_tensor(chord) |
| chord = model.index_tensor_to_multihot_tensor(chord) |
| chord = model.enc_note_embedding(chord) |
| mel_ebd = model.enc_note_embedding(melody) |
| melody_beat_summary = mel_ebd[:, ::4, :] + mel_ebd[:, 1::4, :] + mel_ebd[:, 2::4, :] + mel_ebd[:, 3::4, :] |
| new_mel_ebd = model.enc_note_embedding(new_melody) |
| new_melody_beat_summary = new_mel_ebd[:, ::4, :] + new_mel_ebd[:, 1::4, :] + new_mel_ebd[:, 2::4, :] + new_mel_ebd[:, 3::4, :] |
| dist, mu, = model.encoder(chord, lengths, melody_beat_summary) |
| z = dist.mean |
| pitch_outs = model.decoder(z, new_melody_beat_summary, |
| inference=True, x=None, lengths=None, |
| teacher_forcing_ratio1=0., teacher_forcing_ratio2=0.) |
| pitch_outs = pitch_outs.max(-1, keepdim=True)[1] |
| pitch_outs = pitch_outs.cpu().detach().numpy() |
| chord_track = chord_grid2data(pitch_outs[0], bpm=120//4, start=0, pitch_eos=13) |
| |
| new_melody = new_melody.cpu().detach().numpy()[0] |
| melody_track = melody_matrix2data(new_melody, tempo=120) |
|
|
| music = pyd.PrettyMIDI() |
| music.instruments.append(melody_track) |
| music.instruments.append(chord_track) |
| return music |
|
|
| def melody_prior_control(new_melody): |
| |
| new_mel_ebd = model.enc_note_embedding(new_melody) |
| new_melody_beat_summary = new_mel_ebd[:, ::4, :] + new_mel_ebd[:, 1::4, :] + new_mel_ebd[:, 2::4, :] + new_mel_ebd[:, 3::4, :] |
| z = Normal(torch.zeros(128), torch.ones(128)).rsample().unsqueeze(0) |
| pitch_outs = model.decoder(z, new_melody_beat_summary, |
| inference=True, x=None, lengths=None, |
| teacher_forcing_ratio1=0., teacher_forcing_ratio2=0.) |
| pitch_outs = pitch_outs.max(-1, keepdim=True)[1] |
| pitch_outs = pitch_outs.cpu().detach().numpy() |
| chord_track = chord_grid2data(pitch_outs[0], bpm=120//4, start=0, pitch_eos=13) |
| |
| new_melody = new_melody.cpu().detach().numpy()[0] |
| melody_track = melody_matrix2data(new_melody, tempo=120) |
|
|
| music = pyd.PrettyMIDI() |
| music.instruments.append(melody_track) |
| music.instruments.append(chord_track) |
| return music |
|
|
|
|
| import utils |
| config_fn = './code/model_config.json' |
| |
| model_params = utils.load_params_dict('model_params', config_fn) |
| data_repr_params = utils.load_params_dict('data_repr', config_fn) |
| |
| |
|
|
| device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') |
|
|
| model = VAE(max_simu_note=data_repr_params['max_simu_note'], |
| max_pitch=data_repr_params['max_pitch'], |
| min_pitch=data_repr_params['min_pitch'], |
| pitch_sos=data_repr_params['pitch_sos'], |
| pitch_eos=data_repr_params['pitch_eos'], |
| pitch_pad=data_repr_params['pitch_pad'], |
| num_step=data_repr_params['num_time_step'], |
|
|
| note_emb_size=model_params['note_emb_size'], |
| enc_notes_hid_size=model_params['enc_notes_hid_size'], |
| enc_time_hid_size=model_params['enc_time_hid_size'], |
| z_size=model_params['z_size'], |
| dec_emb_hid_size=model_params['dec_emb_hid_size'], |
| dec_time_hid_size=model_params['dec_time_hid_size'], |
| dec_notes_hid_size=model_params['dec_notes_hid_size'], |
| discr_nhead = model_params["discr_nhead"], |
| discr_hid_size = model_params["discr_hid_size"], |
| discr_dropout = model_params["discr_dropout"], |
| discr_nlayer = model_params["discr_nlayer"], |
|
|
| device=device |
| ) |
|
|
|
|
| weight_path = './code/ad-ptvae_param.pt' |
| params = torch.load(weight_path,map_location=torch.device(device)) |
| if 'model_state_dict' in params: |
| params = params['model_state_dict'] |
| model.load_state_dict(params) |
| if torch.cuda.is_available(): |
| model.cuda() |
| else: |
| model.cpu() |
| |
| model.eval() |
| print('-'*100) |
| print(f'Loaded {weight_path}') |
| print('-'*100) |
|
|
|
|
| dataset = np.load('./code/data.npy', allow_pickle=True).T |
| print('-'*100) |
| print(f'Loaded ./code/data.npy') |
| print('-'*100) |
| np.random.seed(0) |
| np.random.shuffle(dataset) |
| anchor = int(dataset.shape[0] * 0.95) |
| val_data = dataset[anchor:, :] |
| val_set = Nottingham(dataset=val_data.T, |
| length=128, |
| step_size=16, |
| chord_fomat='pr', shift_high=0, shift_low=0) |
| print(len(val_set)) |
| WRITE_PATH = './code/demo_generate' |
| if not os.path.exists(WRITE_PATH): |
| os.makedirs(WRITE_PATH) |
|
|
|
|
| chord_1, _, melody_1, _ = val_set.__getitem__(338) |
| music = get_gt(chord_1, melody_1) |
| music.write(os.path.join(WRITE_PATH, 'gt_1.mid')) |
| chord_1 = torch.from_numpy(chord_1).long().unsqueeze(0) |
| melody_1 = torch.from_numpy(melody_1).float().unsqueeze(0) |
| music = reconstruct(chord_1, melody_1) |
| music.write(os.path.join(WRITE_PATH, 'recon_1.mid')) |
| print(f'Saved to {WRITE_PATH}/recon_1.mid') |
|
|
| chord_2, _, melody_2, _ = val_set.__getitem__(2749) |
| music = get_gt(chord_2, melody_2) |
| music.write(os.path.join(WRITE_PATH, 'gt_2.mid')) |
| chord_2 = torch.from_numpy(chord_2).long().unsqueeze(0) |
| melody_2 = torch.from_numpy(melody_2).float().unsqueeze(0) |
| music = reconstruct(chord_2, melody_2) |
| music.write(os.path.join(WRITE_PATH, 'recon_2.mid')) |
| print(f'Saved to {WRITE_PATH}/recon_2.mid') |
|
|
| chord_3, _, melody_3, _ = val_set.__getitem__(3413) |
| music = get_gt(chord_3, melody_3) |
| music.write(os.path.join(WRITE_PATH, 'gt_3.mid')) |
| chord_3 = torch.from_numpy(chord_3).long().unsqueeze(0) |
| melody_3 = torch.from_numpy(melody_3).float().unsqueeze(0) |
| music = reconstruct(chord_3, melody_3) |
| music.write(os.path.join(WRITE_PATH, 'recon_3.mid')) |
| print(f'Saved to {WRITE_PATH}/recon_3.mid') |
|
|
| chord_4, _, melody_4, _ = val_set.__getitem__(5126) |
| music = get_gt(chord_4, melody_4) |
| music.write(os.path.join(WRITE_PATH, 'gt_4.mid')) |
| chord_4 = torch.from_numpy(chord_4).long().unsqueeze(0) |
| melody_4 = torch.from_numpy(melody_4).float().unsqueeze(0) |
| music = reconstruct(chord_4, melody_4) |
| music.write(os.path.join(WRITE_PATH, 'recon_4.mid')) |
|
|
|
|
| midi = pyd.PrettyMIDI(f'{WRITE_PATH}/recon_1.mid') |
| melody = melody_data2matrix(midi.instruments[0], midi.get_downbeats()) |
| melody = val_set.truncate_melody(melody) |
| melody_1_modal_change = torch.from_numpy(melody).float().unsqueeze(0) |
|
|
| midi = pyd.PrettyMIDI(f'{WRITE_PATH}/recon_2.mid') |
| melody = melody_data2matrix(midi.instruments[0], midi.get_downbeats()) |
| melody = val_set.truncate_melody(melody) |
| melody_2_modal_change = torch.from_numpy(melody).float().unsqueeze(0) |
|
|
| midi = pyd.PrettyMIDI(f'{WRITE_PATH}/recon_3.mid') |
| melody = melody_data2matrix(midi.instruments[0], midi.get_downbeats()) |
| melody = val_set.truncate_melody(melody) |
| melody_3_modal_change = torch.from_numpy(melody).float().unsqueeze(0) |
|
|
| midi = pyd.PrettyMIDI(f'{WRITE_PATH}/recon_4.mid') |
| melody = melody_data2matrix(midi.instruments[0], midi.get_downbeats()) |
| melody = val_set.truncate_melody(melody) |
| melody_4_modal_change = torch.from_numpy(melody).float().unsqueeze(0) |
|
|
|
|
|
|
|
|
| music = melody_control(chord_1, melody_1, shift(melody_1, 6)) |
| music.write(os.path.join(WRITE_PATH, 'control_1_transpose.mid')) |
| print(f'Saved to {WRITE_PATH}/control_1_transpose.mid') |
|
|
| music = melody_control(chord_2, melody_2, shift(melody_2, 6)) |
| music.write(os.path.join(WRITE_PATH, 'control_2_transpose.mid')) |
| print(f'Saved to {WRITE_PATH}/control_2_transpose.mid') |
|
|
| music = melody_control(chord_3, melody_3, shift(melody_3, 6)) |
| music.write(os.path.join(WRITE_PATH, 'control_3_transpose.mid')) |
| print(f'Saved to {WRITE_PATH}/control_3_transpose.mid') |
|
|
| music = melody_control(chord_4, melody_4, shift(melody_4, 6)) |
| music.write(os.path.join(WRITE_PATH, 'control_4_transpose1.mid')) |
| print(f'Saved to {WRITE_PATH}/control_4_transpose.mid') |
|
|
|
|
|
|
| music = melody_control(chord_1, melody_1, melody_1_modal_change) |
| music.write(os.path.join(WRITE_PATH, 'control_1_modal_change.mid')) |
| print(f'Saved to {WRITE_PATH}/control_1_modal_change.mid') |
|
|
| music = melody_control(chord_2, melody_2, melody_2_modal_change) |
| music.write(os.path.join(WRITE_PATH, 'control_2_modal_change.mid')) |
| print(f'Saved to {WRITE_PATH}/control_2_modal_change.mid') |
|
|
| music = melody_control(chord_3, melody_3, melody_3_modal_change) |
| music.write(os.path.join(WRITE_PATH, 'control_3_modal_change.mid')) |
| print(f'Saved to {WRITE_PATH}/control_3_modal_change.mid') |
|
|
| music = melody_control(chord_4, melody_4, melody_4_modal_change) |
| music.write(os.path.join(WRITE_PATH, 'control_4_modal_change.mid')) |
| print(f'Saved to {WRITE_PATH}/control_4_modal_change.mid') |
|
|
|
|
|
|
|
|
| music = melody_prior_control(melody_1) |
| music.write(os.path.join(WRITE_PATH, 'control_1_prior.mid')) |
| print(f'Saved to {WRITE_PATH}/control_1_prior.mid') |
|
|
| music = melody_prior_control(melody_2) |
| music.write(os.path.join(WRITE_PATH, 'control_2_prior.mid')) |
| print(f'Saved to {WRITE_PATH}/control_2_prior.mid') |
|
|
| music = melody_prior_control(melody_3) |
| music.write(os.path.join(WRITE_PATH, 'control_3_prior.mid')) |
| print(f'Saved to {WRITE_PATH}/control_3_prior.mid') |
|
|
| music = melody_prior_control(melody_4) |
| music.write(os.path.join(WRITE_PATH, 'control_4_prior.mid')) |
| print(f'Saved to {WRITE_PATH}/control_4_prior.mid') |
|
|
|
|
|
|
|
|
| music = melody_control(chord_1, melody_1, melody_2) |
| music.write(os.path.join(WRITE_PATH, 'control_1c+2m.mid')) |
|
|
| music = melody_control(chord_2, melody_2, melody_1) |
| music.write(os.path.join(WRITE_PATH, 'control_2c+1m.mid')) |
|
|
| music = melody_control(chord_1, melody_1, melody_3) |
| music.write(os.path.join(WRITE_PATH, 'control_1c+3m.mid')) |
|
|
| music = melody_control(chord_3, melody_3, melody_1) |
| music.write(os.path.join(WRITE_PATH, 'control_3c+1m.mid')) |
|
|
| music = melody_control(chord_1, melody_1, melody_4) |
| music.write(os.path.join(WRITE_PATH, 'control_1c+4m.mid')) |
|
|
| music = melody_control(chord_4, melody_4, melody_1) |
| music.write(os.path.join(WRITE_PATH, 'control_4c+1m.mid')) |
|
|
| music = melody_control(chord_2, melody_2, melody_3) |
| music.write(os.path.join(WRITE_PATH, 'control_2c+3m.mid')) |
|
|
| music = melody_control(chord_3, melody_3, melody_2) |
| music.write(os.path.join(WRITE_PATH, 'control_3c+2m.mid')) |
|
|
| music = melody_control(chord_2, melody_2, melody_4) |
| music.write(os.path.join(WRITE_PATH, 'control_2c+4m.mid')) |
|
|
| music = melody_control(chord_4, melody_4, melody_2) |
| music.write(os.path.join(WRITE_PATH, 'control_4c+2m.mid')) |
|
|
| music = melody_control(chord_3, melody_3, melody_4) |
| music.write(os.path.join(WRITE_PATH, 'control_3c+4m.mid')) |
|
|
| music = melody_control(chord_4, melody_4, melody_3) |
| music.write(os.path.join(WRITE_PATH, 'control_4c+3m.mid')) |
|
|
|
|
