Spaces:
Sleeping
Sleeping
| """Analyse-Pipeline: RMVPE-F0-Kurve, MIDI aus F0, Whisper-Transkript mit | |
| Wort-Timestamps, und ein Zeit-Mapping Wort <-> Note <-> F0-Segment. | |
| Alle Outputs liegen auf derselben Zeitachse (RMVPE: 100 fps / 10 ms Grid, | |
| identisch zur pretty_midi-Piano-Roll mit fs=100, die Diff-Pitcher liest). | |
| """ | |
| import json | |
| import os | |
| import tempfile | |
| import numpy as np | |
| import pretty_midi | |
| F0_FPS = 100 # RMVPE native hop: 10 ms | |
| CONF_THRESHOLD = 0.03 # RMVPE-Voicing-Schwelle | |
| MIN_NOTE_FRAMES = 3 # <30 ms -> kein eigenstaendiger Note-Wert | |
| MERGE_GAP_FRAMES = 3 # gleichhohe Segmente mit <=30 ms Luecke verschmelzen | |
| NOTE_NAMES = ["C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B"] | |
| def note_name(midi_note): | |
| m = int(round(midi_note)) | |
| return f"{NOTE_NAMES[m % 12]}{m // 12 - 1}" | |
| # ---------------------------------------------------------------- RMVPE | |
| _RMVPE = None | |
| def get_rmvpe(): | |
| global _RMVPE | |
| if _RMVPE is None: | |
| from rmvpe_onnx import RMVPE | |
| _RMVPE = RMVPE() # laedt die Gewichte beim ersten Aufruf automatisch | |
| return _RMVPE | |
| def rmvpe_f0(audio, sr): | |
| """F0-Kurve (Hz) @100 fps. Unvoiced/low-confidence -> 0.0.""" | |
| t, freq, conf, _act = get_rmvpe().predict(audio=audio, sr=sr) | |
| freq = np.asarray(freq, dtype=float).copy() | |
| conf = np.asarray(conf, dtype=float) | |
| freq[conf < CONF_THRESHOLD] = 0.0 | |
| return t, freq | |
| # ---------------------------------------------------------------- MIDI aus F0 | |
| def f0_to_notes(freq, shift=0): | |
| """Segmentiert die F0-Kurve zu [pitch, start_s, end_s]-Noten. | |
| Saemtliche Zeiten liegen auf dem 10-ms-Raster, damit die Piano-Roll | |
| (fs=100) sauber ist — genau den Onset-Contract, den Diff-Pitchers | |
| prepare_midi_wav erwartet (roll>0 -> 100, Onset-Frame genullt). | |
| """ | |
| voiced = freq > 0 | |
| semis = np.zeros(len(freq)) | |
| semis[voiced] = np.round(69 + 12 * np.log2(freq[voiced] / 440.0)) | |
| notes = [] | |
| n = len(freq) | |
| i = 0 | |
| while i < n: | |
| if not voiced[i]: | |
| i += 1 | |
| continue | |
| j = i | |
| while j < n and voiced[j]: | |
| j += 1 | |
| if j - i >= MIN_NOTE_FRAMES: | |
| pitch = int(round(float(np.median(semis[i:j])))) + shift | |
| notes.append([pitch, round(i / 100.0, 2), round(j / 100.0, 2)]) | |
| i = j | |
| merged = [] | |
| for pitch, start, end in notes: | |
| if merged and merged[-1][0] == pitch and start - merged[-1][2] <= 0.03: | |
| merged[-1][2] = end | |
| else: | |
| merged.append([pitch, start, end]) | |
| return merged | |
| def notes_to_midi(notes, path, tempo=120): | |
| os.makedirs(os.path.dirname(path), exist_ok=True) | |
| pm = pretty_midi.PrettyMIDI(initial_tempo=tempo) | |
| inst = pretty_midi.Instrument(program=0, name="rmvpe-derived") | |
| for pitch, start, end in notes: | |
| inst.notes.append(pretty_midi.Note( | |
| velocity=100, pitch=int(pitch), start=float(start), end=float(end))) | |
| pm.instruments.append(inst) | |
| pm.write(path) | |
| return path | |
| # ---------------------------------------------------------------- Whisper | |
| _WHISPER = {} | |
| def get_whisper(model_size="small", device="cpu", compute_type="int8"): | |
| from faster_whisper import WhisperModel | |
| key = (model_size, device, compute_type) | |
| if key not in _WHISPER: | |
| _WHISPER[key] = WhisperModel(model_size, device=device, compute_type=compute_type) | |
| return _WHISPER[key] | |
| def whisper_words(audio_path, model_size="small", language=None, device="cpu"): | |
| """Transkript + Wort-Timestamps via faster-whisper.""" | |
| compute_type = "float16" if device == "cuda" else "int8" | |
| model = get_whisper(model_size, device, compute_type) | |
| segments, info = model.transcribe( | |
| os.fspath(audio_path), | |
| language=language, | |
| word_timestamps=True, | |
| vad_filter=True, | |
| ) | |
| words = [] | |
| for seg in segments: | |
| for w in (seg.words or []): | |
| words.append({ | |
| "word": w.word.strip(), | |
| "start": round(float(w.start), 3), | |
| "end": round(float(w.end), 3), | |
| "probability": round(float(w.probability), 3), | |
| }) | |
| return words, {"language": info.language, | |
| "language_probability": round(float(info.language_probability), 3)} | |
| # ---------------------------------------------------------------- Mapping | |
| def build_mapping(freq, notes, words): | |
| """Wort <-> Note <-> F0-Segment auf der 100-fps-Zeitachse.""" | |
| def f0_stats(a, b): | |
| i0 = max(0, int(a * F0_FPS)) | |
| i1 = min(len(freq), int(np.ceil(b * F0_FPS))) | |
| v = freq[i0:i1] | |
| v = v[v > 0] | |
| if len(v) == 0: | |
| return {"voiced_frames": 0, "mean_hz": None, "stdev_semitones": None} | |
| return { | |
| "voiced_frames": int(len(v)), | |
| "mean_hz": round(float(np.mean(v)), 2), | |
| "stdev_semitones": round(float(np.std(12 * np.log2(v / v.mean()))), 3), | |
| } | |
| words_out = [] | |
| for w in words: | |
| overlaps = [ | |
| {"pitch": int(p), "name": note_name(p), "start": s, "end": e, | |
| "overlap_s": round(max(0.0, min(w["end"], e) - max(w["start"], s)), 3)} | |
| for p, s, e in notes | |
| if min(w["end"], e) - max(w["start"], s) > 0 | |
| ] | |
| words_out.append({**w, "notes": overlaps, "f0": f0_stats(w["start"], w["end"])}) | |
| notes_out = [ | |
| {"pitch": int(p), "name": note_name(p), "start": s, "end": e, "f0": f0_stats(s, e)} | |
| for p, s, e in notes | |
| ] | |
| return {"fps": F0_FPS, "words": words_out, "notes": notes_out} | |
| # ---------------------------------------------------------------- Plot | |
| def render_plot(freq, notes, words, path): | |
| import matplotlib | |
| matplotlib.use("Agg") | |
| import matplotlib.pyplot as plt | |
| fig, ax = plt.subplots(figsize=(14, 4)) | |
| for p, s, e in notes: | |
| ax.add_patch(plt.Rectangle((s, p - 0.4), e - s, 0.8, | |
| facecolor="tab:blue", alpha=0.5, edgecolor="none")) | |
| t = np.arange(len(freq)) / F0_FPS | |
| voiced = freq > 0 | |
| ax.plot(t[voiced], 69 + 12 * np.log2(freq[voiced] / 440.0), | |
| ".", ms=1.5, color="k", label="RMVPE F0") | |
| for w in words: | |
| ax.axvline(w["start"], color="red", lw=0.6, alpha=0.5) | |
| if w["word"]: | |
| ax.text(w["start"], 84, w["word"], rotation=90, fontsize=6, va="bottom") | |
| ax.set_xlabel("Zeit (s)") | |
| ax.set_ylabel("MIDI-Note") | |
| ax.set_title("Piano-Roll (MIDI aus F0) + RMVPE-F0 + Wortgrenzen") | |
| ax.legend(loc="lower right") | |
| fig.tight_layout() | |
| fig.savefig(path, dpi=110) | |
| plt.close(fig) | |
| return path | |
| # ---------------------------------------------------------------- Top-Level | |
| def analyze(audio_path, sr=24000, max_seconds=20, whisper_model="small", | |
| language=None, out_dir=None, device="cpu"): | |
| """Komplette Analyse eines Recordings. | |
| Liefert dict mit summary, transcript, mapping und Datei-Pfaden | |
| (midi/json/png/npy). Alle Outputs teilen sich die 10-ms-Zeitachse. | |
| """ | |
| import librosa | |
| out_dir = out_dir or tempfile.mkdtemp(prefix="analysis_") | |
| os.makedirs(out_dir, exist_ok=True) | |
| wav, _ = librosa.load(audio_path, sr=sr, mono=True, duration=max_seconds) | |
| t, freq = rmvpe_f0(wav, sr) | |
| notes = f0_to_notes(freq) | |
| midi_path = notes_to_midi(notes, os.path.join(out_dir, "derived_from_f0.mid")) | |
| words, lang_info = whisper_words(audio_path, model_size=whisper_model, | |
| language=language, device=device) | |
| mapping = build_mapping(freq, notes, words) | |
| mapping["audio"] = {"sr": sr, "duration_s": round(len(wav) / sr, 3), **lang_info} | |
| mapping["transcript"] = " ".join(w["word"] for w in words) | |
| f0_path = os.path.join(out_dir, "f0_rmvpe_100fps.npy") | |
| np.save(f0_path, np.stack([t, freq])) | |
| json_path = os.path.join(out_dir, "mapping.json") | |
| with open(json_path, "w", encoding="utf-8") as fh: | |
| json.dump(mapping, fh, ensure_ascii=False, indent=2) | |
| plot_path = render_plot(freq, notes, words, os.path.join(out_dir, "overview.png")) | |
| summary = { | |
| "n_words": len(words), | |
| "n_notes": len(notes), | |
| "voiced_ratio": round(float((freq > 0).mean()), 3), | |
| "language": lang_info.get("language"), | |
| "whisper_model": whisper_model, | |
| "alignment": "faster-whisper word_timestamps (MFA ist conda-only und im Space nicht installierbar)", | |
| } | |
| return { | |
| "summary": summary, | |
| "transcript": " ".join(w["word"] for w in words), | |
| "mapping": mapping, | |
| "midi_path": midi_path, | |
| "json_path": json_path, | |
| "plot_path": plot_path, | |
| "f0_path": f0_path, | |
| "midi_notes": notes, | |
| } |