File size: 5,872 Bytes
b213d4c
 
4504581
 
 
 
 
 
 
07e756d
 
4504581
b213d4c
4504581
b938262
4504581
b213d4c
 
 
 
 
 
 
 
b105d68
b213d4c
 
4504581
 
b213d4c
 
4504581
07e756d
 
 
 
 
4504581
eefbd1f
 
 
 
 
 
 
 
 
4504581
 
b213d4c
4504581
 
 
 
 
 
 
 
 
 
 
 
eefbd1f
4504581
b213d4c
 
4504581
eefbd1f
4504581
3b93001
eefbd1f
 
 
 
 
 
 
 
 
 
 
4504581
 
eefbd1f
 
4504581
 
 
 
 
 
 
07e756d
4504581
 
 
eefbd1f
b9dda3d
 
eefbd1f
 
4504581
6765837
 
 
 
 
 
 
 
 
 
 
 
 
eefbd1f
 
 
 
 
 
 
 
 
4504581
6765837
 
 
4504581
 
38d0bd2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b213d4c
e171372
 
 
38d0bd2
 
 
 
 
6765837
38d0bd2
 
 
 
 
557bf2d
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
import spaces  # imported first, as ZeroGPU requires

import os
import pickle
import torch
import torch.nn as nn
import gradio as gr
from transformers import T5Tokenizer
from huggingface_hub import hf_hub_download
from time import time_ns
from uuid import uuid4

from transformer_model import Transformer
from pyharp.core import ModelCard, build_endpoint
from pyharp.tags import Subcategory

# HARP Model Card
model_card = ModelCard(
    name="text2midi",
    description=(
        "Generates a MIDI sequence from a text description of the music. Repository: "
        "https://github.com/amaai-lab/text2midi Paper: https://arxiv.org/abs/2412.16526"
    ),
    author="Keshav Bhandari, Abhinaba Roy, Kyra Wang, Geeta Puri, Simon Colton, Dorien Herremans",
    tags=[Subcategory.HOLISTIC, 'domain:music']
)

# Model/artifacts from HF Hub
REPO_ID = "amaai-lab/text2midi"
REVISION = "9bc91479c4a9b08a9a28a7dcfb3f1a56ee2ded36"


# Helpers
def _unique_path(ext: str) -> str:
    """Create a unique file path in /tmp to avoid naming collisions."""
    return os.path.join("/tmp", f"t2m_{time_ns()}_{uuid4().hex[:8]}{ext}")


DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
model = None
r_tokenizer = None
tokenizer = None


def load_model():
    """Builds the model on the CPU, then places it on DEVICE."""
    global model, r_tokenizer, tokenizer

    # Load REMI vocab/tokenizer (pickle dict used by the provided model)
    with open(hf_hub_download(repo_id=REPO_ID, filename="vocab_remi.pkl", revision=REVISION), "rb") as f:
        r_tokenizer = pickle.load(f)

    vocab_size = len(r_tokenizer)
    model = Transformer(
        vocab_size,    # vocab size
        768,           # d_model
        8,             # nhead
        2048,          # dim_feedforward
        18,            # nlayers
        1024,          # max_seq_len
        False,         # use_rotary
        8,             # rotary_dim
        device="cpu"   # device
    )
    model_path = hf_hub_download(repo_id=REPO_ID, filename="pytorch_model.bin", revision=REVISION)
    model.load_state_dict(torch.load(model_path, map_location="cpu"))
    model.eval()
    model.to(DEVICE)

    tokenizer = T5Tokenizer.from_pretrained("google/flan-t5-base", revision="7bcac572ce56db69c1ea7c8af255c5d7c9672fc2")


# Built in the worker, which runs process_fn, on its main thread, where ZeroGPU's emulation records
# the CUDA placement, so that no request spends the caller's allowance on building it. The server
# never runs process_fn.
if __name__ != "__main__":
    load_model()


# Core Text -> MIDI
def generate_midi(prompt: str, temperature: float = 0.9, max_len: int = 500) -> str:
    inputs = tokenizer(prompt, return_tensors="pt", padding=True, truncation=True)

    input_ids = nn.utils.rnn.pad_sequence(inputs.input_ids, batch_first=True, padding_value=0).to(DEVICE)
    attention_mask = nn.utils.rnn.pad_sequence(inputs.attention_mask, batch_first=True, padding_value=0).to(DEVICE)

    with torch.no_grad():
        output = model.generate(input_ids, attention_mask, max_len=max_len, temperature=temperature)

    output_list = output[0].tolist()
    generated_midi = r_tokenizer.decode(output_list)

    midi_path = _unique_path(".mid")
    generated_midi.dump_midi(midi_path)
    return midi_path

def gpu_seconds(prompt: str, temperature: float, max_length: int) -> float:
    # Measured at about 42 s for the longest, 1500 tokens, padded
    return 10 + 0.05 * int(max_length)


def process_fn(prompt: str, temperature: float, max_length: int):
    """
    Generate a MIDI from a text description.

    Args:
        prompt (str): What the music should be.
        temperature (float): How freely the model departs from its likeliest
            continuation.
        max_length (int): Ceiling on the number of tokens generated.

    Returns:
        midi_path (str): The generated MIDI.
    """

    # Checked before the GPU is reserved, so a request the model cannot take spends none of the
    # caller's ZeroGPU allowance
    if not (prompt or "").strip():
        raise gr.Error("Describe the music to generate.")
    return compose(prompt, temperature, max_length)


@spaces.GPU(duration=gpu_seconds)
def compose(prompt: str, temperature: float, max_length: int) -> str:
    try:
        return generate_midi(prompt, float(temperature), int(max_length))
    except Exception as error:
        raise gr.Error(str(error))

# Gradio + HARP UI
if __name__ == "__main__":
    with gr.Blocks() as demo:
        gr.Markdown("## 🎶 text2midi")

        # Inputs
        prompt_in = gr.Textbox(
            label="Describe Your Music",
            info="Type a short phrase like 'calm piano with flowing arpeggios' ",
            ).harp_required(True)
        temperature_in = gr.Slider(minimum=0.8, maximum=1.1, value=0.9, step=0.1, label="Creativity", info=(
            "Adjusts how much freedom the model takes while composing.\n"
            "Lower = safer and more predictable (structured), "
            "Higher = more varied and expressive."
        ), interactive=True)

        maxlen_in = gr.Slider(minimum=500, maximum=1500, step=100, value=500, label="Composition Length", info=(
            "Determines how long the generated piece is in musical tokens.\n"
            "Higher values produce longer phrases (roughly more measures of music).")
        )

        # Outputs
        midi_out = gr.File(label="Generated MIDI", file_types=[".mid", ".midi"], type="filepath").set_info((
            "The generated notes, as a MIDI file."
        ))

        # Build HARP endpoint
        _ = build_endpoint(
            model_card=model_card,
            input_components=[prompt_in, temperature_in, maxlen_in],
            output_components=[midi_out],
            process_fn=process_fn
        )


    # Launch App
    demo.queue().launch(pwa=True, show_error=True, ssr_mode=False)