Spaces:
Running on Zero
Running on Zero
Build the model once in the worker, and declare the format
Browse filesBuilds and places the model in the worker rather than inside every GPU call, declares 44.1 kHz mono, removes harp_required from the output, and pins numpy and librosa.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
- README.md +2 -0
- app.py +19 -17
- requirements.txt +2 -2
README.md
CHANGED
|
@@ -14,6 +14,8 @@ tags:
|
|
| 14 |
- subcategory:music-analysis
|
| 15 |
- input:audio
|
| 16 |
- output:midi
|
|
|
|
|
|
|
| 17 |
---
|
| 18 |
|
| 19 |
Transcribes solo piano performance to MIDI.
|
|
|
|
| 14 |
- subcategory:music-analysis
|
| 15 |
- input:audio
|
| 16 |
- output:midi
|
| 17 |
+
- sample-rate:44100
|
| 18 |
+
- channels:mono
|
| 19 |
---
|
| 20 |
|
| 21 |
Transcribes solo piano performance to MIDI.
|
app.py
CHANGED
|
@@ -2,7 +2,6 @@ import spaces
|
|
| 2 |
from transkun.ModelTransformer import TransKun, writeMidi
|
| 3 |
import numpy as np
|
| 4 |
import torch
|
| 5 |
-
from functools import lru_cache
|
| 6 |
import librosa
|
| 7 |
|
| 8 |
import tempfile
|
|
@@ -18,21 +17,24 @@ import moduleconf
|
|
| 18 |
|
| 19 |
from pyharp.core import ModelCard, build_endpoint
|
| 20 |
from pyharp.media.audio import load_audio
|
| 21 |
-
from pyharp.tags import Subcategory
|
| 22 |
import gradio as gr
|
| 23 |
|
| 24 |
model_card = ModelCard(
|
| 25 |
name="Transkun",
|
| 26 |
description=(
|
| 27 |
-
"Transcribes solo piano performance into MIDI notation
|
| 28 |
-
"
|
|
|
|
| 29 |
"https://arxiv.org/abs/2404.09466"
|
| 30 |
),
|
| 31 |
author="Yujia Yan, Zhiyao Duan",
|
| 32 |
-
tags=[Subcategory.MUSIC_ANALYSIS]
|
| 33 |
)
|
| 34 |
|
| 35 |
-
|
|
|
|
|
|
|
| 36 |
def load_model(device: str):
|
| 37 |
defaultWeight = str(files("transkun") / "pretrained" / "2.0.pt")
|
| 38 |
defaultConf = str(files("transkun") / "pretrained" / "2.0.conf")
|
|
@@ -40,8 +42,8 @@ def load_model(device: str):
|
|
| 40 |
confManager = moduleconf.parseFromFile(defaultConf)
|
| 41 |
conf = confManager["Model"].config
|
| 42 |
|
| 43 |
-
checkpoint = torch.load(defaultWeight, map_location=
|
| 44 |
-
model = TransKun(conf=conf)
|
| 45 |
|
| 46 |
# Checkpoints keep their weights under either key
|
| 47 |
if "best_state_dict" in checkpoint:
|
|
@@ -50,7 +52,12 @@ def load_model(device: str):
|
|
| 50 |
model.load_state_dict(checkpoint["state_dict"], strict=False)
|
| 51 |
|
| 52 |
model.eval()
|
| 53 |
-
return model
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
|
| 55 |
# The recording is transcribed in overlapping segments, so the cost grows with its length
|
| 56 |
# PyHARP stops a job that runs past its timeout_s, 15 minutes by default, so a request whose
|
|
@@ -66,9 +73,6 @@ def gpu_seconds(input_file):
|
|
| 66 |
|
| 67 |
@spaces.GPU(duration=gpu_seconds)
|
| 68 |
def transcribe(input_file):
|
| 69 |
-
device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 70 |
-
model = load_model(device)
|
| 71 |
-
|
| 72 |
signal = load_audio(input_file)
|
| 73 |
waveform = np.asarray(signal.audio_data)
|
| 74 |
sr = int(signal.sample_rate)
|
|
@@ -86,7 +90,7 @@ def transcribe(input_file):
|
|
| 86 |
waveform = librosa.resample(waveform.astype(np.float32), orig_sr=sr, target_sr=model.fs)
|
| 87 |
sr = model.fs
|
| 88 |
|
| 89 |
-
x = torch.from_numpy(waveform.reshape(-1, 1)).to(
|
| 90 |
|
| 91 |
notesEst = model.transcribe(x, discardSecondHalf=False)
|
| 92 |
|
|
@@ -120,14 +124,12 @@ if __name__ == "__main__":
|
|
| 120 |
"A solo piano recording. Trained on piano alone, so other "
|
| 121 |
"instruments in the recording will confuse it."
|
| 122 |
))
|
| 123 |
-
#output_midi = gr.File(label="Output MIDI File", file_types=[".mid"]).harp_required(True)
|
| 124 |
|
| 125 |
-
# ensure this is serialized as midi_track
|
| 126 |
output_midi = gr.File(
|
| 127 |
label="Transcription",
|
| 128 |
-
file_types=[".mid", ".midi"],
|
| 129 |
type="filepath"
|
| 130 |
-
).
|
| 131 |
|
| 132 |
|
| 133 |
app = build_endpoint(
|
|
|
|
| 2 |
from transkun.ModelTransformer import TransKun, writeMidi
|
| 3 |
import numpy as np
|
| 4 |
import torch
|
|
|
|
| 5 |
import librosa
|
| 6 |
|
| 7 |
import tempfile
|
|
|
|
| 17 |
|
| 18 |
from pyharp.core import ModelCard, build_endpoint
|
| 19 |
from pyharp.media.audio import load_audio
|
| 20 |
+
from pyharp.tags import Channels, SampleRate, Subcategory
|
| 21 |
import gradio as gr
|
| 22 |
|
| 23 |
model_card = ModelCard(
|
| 24 |
name="Transkun",
|
| 25 |
description=(
|
| 26 |
+
"Transcribes solo piano performance into MIDI notation, with onsets, offsets, and "
|
| 27 |
+
"velocities. The input is mixed to mono and resampled to 44.1 kHz internally. "
|
| 28 |
+
"Repository: https://github.com/Yujia-Yan/Transkun Paper: "
|
| 29 |
"https://arxiv.org/abs/2404.09466"
|
| 30 |
),
|
| 31 |
author="Yujia Yan, Zhiyao Duan",
|
| 32 |
+
tags=[Subcategory.MUSIC_ANALYSIS, SampleRate(44100), Channels(1)]
|
| 33 |
)
|
| 34 |
|
| 35 |
+
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
|
| 36 |
+
|
| 37 |
+
|
| 38 |
def load_model(device: str):
|
| 39 |
defaultWeight = str(files("transkun") / "pretrained" / "2.0.pt")
|
| 40 |
defaultConf = str(files("transkun") / "pretrained" / "2.0.conf")
|
|
|
|
| 42 |
confManager = moduleconf.parseFromFile(defaultConf)
|
| 43 |
conf = confManager["Model"].config
|
| 44 |
|
| 45 |
+
checkpoint = torch.load(defaultWeight, map_location="cpu")
|
| 46 |
+
model = TransKun(conf=conf)
|
| 47 |
|
| 48 |
# Checkpoints keep their weights under either key
|
| 49 |
if "best_state_dict" in checkpoint:
|
|
|
|
| 52 |
model.load_state_dict(checkpoint["state_dict"], strict=False)
|
| 53 |
|
| 54 |
model.eval()
|
| 55 |
+
return model.to(device)
|
| 56 |
+
|
| 57 |
+
# pyharp runs process_fn in a worker, which imports this file as __mp_main__ and starts with the
|
| 58 |
+
# server, so the model is built and placed there alone, rather than inside every GPU call
|
| 59 |
+
if __name__ != "__main__":
|
| 60 |
+
model = load_model(DEVICE)
|
| 61 |
|
| 62 |
# The recording is transcribed in overlapping segments, so the cost grows with its length
|
| 63 |
# PyHARP stops a job that runs past its timeout_s, 15 minutes by default, so a request whose
|
|
|
|
| 73 |
|
| 74 |
@spaces.GPU(duration=gpu_seconds)
|
| 75 |
def transcribe(input_file):
|
|
|
|
|
|
|
|
|
|
| 76 |
signal = load_audio(input_file)
|
| 77 |
waveform = np.asarray(signal.audio_data)
|
| 78 |
sr = int(signal.sample_rate)
|
|
|
|
| 90 |
waveform = librosa.resample(waveform.astype(np.float32), orig_sr=sr, target_sr=model.fs)
|
| 91 |
sr = model.fs
|
| 92 |
|
| 93 |
+
x = torch.from_numpy(waveform.reshape(-1, 1)).to(DEVICE)
|
| 94 |
|
| 95 |
notesEst = model.transcribe(x, discardSecondHalf=False)
|
| 96 |
|
|
|
|
| 124 |
"A solo piano recording. Trained on piano alone, so other "
|
| 125 |
"instruments in the recording will confuse it."
|
| 126 |
))
|
|
|
|
| 127 |
|
|
|
|
| 128 |
output_midi = gr.File(
|
| 129 |
label="Transcription",
|
| 130 |
+
file_types=[".mid", ".midi"],
|
| 131 |
type="filepath"
|
| 132 |
+
).set_info("The transcribed notes, with onsets, offsets and velocities.")
|
| 133 |
|
| 134 |
|
| 135 |
app = build_endpoint(
|
requirements.txt
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
torch==2.8.0
|
| 2 |
torchaudio==2.8.0
|
| 3 |
-
numpy
|
| 4 |
git+https://github.com/TEAMuP-dev/pyharp.git@v0.4.2
|
| 5 |
transkun==2.0.1
|
| 6 |
-
librosa
|
| 7 |
spaces>=0.51.1
|
|
|
|
| 1 |
torch==2.8.0
|
| 2 |
torchaudio==2.8.0
|
| 3 |
+
numpy==2.2.6
|
| 4 |
git+https://github.com/TEAMuP-dev/pyharp.git@v0.4.2
|
| 5 |
transkun==2.0.1
|
| 6 |
+
librosa==0.11.0
|
| 7 |
spaces>=0.51.1
|