Improve short-hum note articulation fallback

README.md

@@ -30,9 +30,12 @@ guitar and fingered bass SoundFonts, with FFmpeg compression, filtering,
 oversampled soft clipping, and a short guitar room echo in processed versions.
 
 If SheetSage2 returns an empty vocal melody, a conservative pYIN fallback
-extracts source-timed notes from sustained voiced pitch and supplies a simple
-instrumental ABC for YuE2. It rejects unpitched noise. Fallback note boundaries
-are approximate, and the run details identify which transcription was used.
+extracts source-timed notes from sustained voiced pitch. Spectral attacks can
+separate repeated syllables at the same pitch; stable abrupt pitch changes can
+separate legato notes without turning a smooth glide into repeated keypresses.
+It supplies a simple instrumental ABC for YuE2 and rejects unpitched noise.
+Fallback note boundaries remain approximate, and the run details identify
+which transcription was used.
 SheetSage2's beat grid remains the source for sampled drums when available.
 
 YuE2 is on by default, so the same click also starts three experiments from

f0_fallback.py

@@ -11,6 +11,7 @@ def transcribe_f0(source: Path, target: Path) -> int:
     import numpy as np
     import pretty_midi
     from scipy.ndimage import binary_closing, median_filter
+    from scipy.signal import find_peaks
     import soundfile as sf
 
     audio, rate = sf.read(source, dtype="float32")
@@ -19,6 +20,11 @@ def transcribe_f0(source: Path, target: Path) -> int:
     reduced = librosa.resample(audio, orig_sr=rate, target_sr=16000)
     f0, voiced, _ = librosa.pyin(reduced, sr=16000, fmin=55., fmax=1000.,
                                  frame_length=1024, hop_length=160, center=True)
+    flux = librosa.onset.onset_strength(y=reduced, sr=16000, hop_length=160,
+                                       n_fft=512, n_mels=64)[:len(f0)]
+    attack_threshold = max(2.5, float(np.percentile(flux, 95)) * .8)
+    attacks, _ = find_peaks(flux, height=attack_threshold,
+                            prominence=attack_threshold * .8, distance=12)
     rms = librosa.feature.rms(y=reduced, frame_length=1024,
                               hop_length=160, center=True)[0][:len(f0)]
     threshold = max(.001, float(np.percentile(rms, 95)) * 10 ** (-35 / 20))
@@ -31,23 +37,28 @@ def transcribe_f0(source: Path, target: Path) -> int:
     for first, last in zip(boundaries[::2], boundaries[1::2]):
         if last - first < 18:  # Reject short accidental pitched patches in noise.
             continue
-        pitch = median_filter(69 + 12 * np.log2(np.maximum(f0[first:last], 55.) / 440),
-                              size=9, mode="nearest")
+        pitch = 69 + 12 * np.log2(np.maximum(f0[first:last], 55.) / 440)
         # pYIN may be unvoiced for a few interpolated frames.
         good = np.isfinite(pitch)
         if not good.any():
             continue
         pitch = np.interp(np.arange(len(pitch)), np.flatnonzero(good), pitch[good])
-        rounded = median_filter(np.rint(pitch), size=11, mode="nearest").astype(int)
-        changes = np.r_[0, np.flatnonzero(np.diff(rounded)) + 1, len(rounded)]
+        pitch = median_filter(pitch, size=5, mode="nearest")
         cuts = [0]
-        for index in range(1, len(changes) - 1):
-            point, next_point = int(changes[index]), int(changes[index + 1])
-            if (point - cuts[-1] >= 12 and next_point - point >= 12
-                    and abs(np.median(pitch[cuts[-1]:point])
-                            - np.median(pitch[point:next_point])) >= 1.5):
+        for attack in attacks:
+            point = int(attack - first)
+            # The first/last spectral burst is the span edge, not another keypress.
+            if 12 <= point <= len(pitch) - 12 and point - cuts[-1] >= 12:
                 cuts.append(point)
-        cuts.append(len(rounded))
+        # An abrupt stable F0 step can be a legato note even without a strong
+        # energy attack. A continuous glide never has stable plateaus on both sides.
+        for point in range(12, len(pitch) - 12):
+            before, after = pitch[point - 10:point - 2], pitch[point + 2:point + 10]
+            if (abs(np.median(after) - np.median(before)) >= 1.5
+                    and np.ptp(before) < .65 and np.ptp(after) < .65
+                    and all(abs(point - cut) >= 12 for cut in cuts)):
+                cuts.append(point)
+        cuts = sorted(cuts) + [len(pitch)]
         for start, end in zip(cuts, cuts[1:]):
             if end - start < 8:
                 continue

test_f0_fallback.py

@@ -38,3 +38,39 @@ class F0FallbackTest(unittest.TestCase):
             with self.assertRaisesRegex(ValueError, "устойчивую высоту"):
                 transcribe_f0(source, target)
             self.assertFalse(target.exists())
+
+    def test_glide_stays_one_note_but_connected_syllables_retrigger(self):
+        import numpy as np
+        import pretty_midi
+        import soundfile as sf
+        from f0_fallback import transcribe_f0
+
+        rate = 24000
+        times = np.arange(round(2.5 * rate)) / rate
+        active = (times >= .2) & (times < 2.2)
+        for name in ("glide", "repeats"):
+            pitch = np.linspace(57, 62, active.sum()) if name == "glide" else np.full(active.sum(), 57)
+            frequency = 440 * 2 ** ((pitch - 69) / 12)
+            phase = 2 * np.pi * np.cumsum(frequency) / rate
+            signal = np.zeros(len(times), dtype=np.float32)
+            signal[active] = .13 * (np.sin(phase) + .28 * np.sin(2 * phase))
+            if name == "repeats":
+                rng = np.random.default_rng(818)
+                for boundary in (.8, 1.5):
+                    center = round(boundary * rate)
+                    width = round(.035 * rate)
+                    indices = np.arange(-width, width)
+                    dip = .82 * np.exp(-.5 * (indices / (width / 2.5)) ** 2)
+                    signal[center - width:center + width] *= 1 - dip
+                    signal[center:center + round(.018 * rate)] += rng.normal(
+                        0, .018, round(.018 * rate)).astype(np.float32)
+            with self.subTest(name=name), tempfile.TemporaryDirectory() as directory:
+                source, target = (Path(directory) / filename for filename in ("hum.wav", "notes.mid"))
+                sf.write(source, signal, rate)
+                expected = 1 if name == "glide" else 3
+                self.assertEqual(transcribe_f0(source, target), expected)
+                notes = pretty_midi.PrettyMIDI(str(target)).instruments[0].notes
+                if name == "repeats":
+                    for note, start in zip(notes, (.2, .8, 1.5)):
+                        self.assertEqual(note.pitch, 57)
+                        self.assertAlmostEqual(note.start, start, delta=.06)