Hugging Face's logo

Spaces:
abalanescu
/
flow2
Running on Zero

App Files Community
1
flow2
File size: 4,366 Bytes
90907c1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5fbb200
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
90907c1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
#!/usr/bin/env python3
"""@file compare_music_tests.py
@description A/B compare Moldovan test-song WAVs across the 3 music models.
              Reads /tmp/moldovan_tests/*.wav (MiniMax baseline + YuE2 + ACE-Step),
              reports ffprobe audio metrics per file, verifies all three exist,
              and prints the verdict template for the lyric-quality A/B.

Usage:
  python3 scripts/compare_music_tests.py [--dir /tmp/moldovan_tests]
"""
from __future__ import annotations

import argparse
import json
import subprocess
import sys
from pathlib import Path

# Keys match run_music_tests.sh slugify() output exactly (dashes, cut at 30 chars).
MODELS = {
    "minimax-music-3--full-song---v": "MiniMax Music 3 (baseline)",
    "yue2-3b--editable-scores--48kh": "YuE2-3B",
    "ace-step-v1-5-xl-turbo--commer": "ACE-Step v1.5 XL Turbo",
}


def probe(path: Path) -> dict | None:
    try:
        out = subprocess.run(
            [
                "ffprobe", "-v", "quiet", "-print_format", "json",
                "-show_format", "-show_streams", str(path),
            ],
            capture_output=True, text=True, timeout=30,
        ).stdout
        d = json.loads(out)
        st = d.get("streams", [{}])[0]
        fmt = d.get("format", {})
        return {
            "file": path.name,
            "seconds": round(float(fmt.get("duration", 0)), 1),
            "sample_rate": st.get("sample_rate"),
            "channels": st.get("channels"),
            "codec": st.get("codec_name"),
            "bytes": path.stat().st_size,
        }
    except Exception as exc:
        return {"file": path.name, "error": str(exc)}


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--dir", default="/tmp/moldovan_tests")
    args = ap.parse_args()
    base = Path(args.dir)

    rows = []
    missing = []
    for slug, label in MODELS.items():
        p = base / f"{slug}_test.wav"
        if p.exists():
            rows.append((label, probe(p)))
        else:
            missing.append((label, slug))

    print(f"=== Moldovan test-song A/B ({base}) ===")
    for label, m in rows:
        if "error" in m:
            print(f"{label:28} ERROR: {m['error']}")
            continue
        print(
            f"{label:28} {m['seconds']:>6}s  {m['sample_rate']}Hz  "
            f"{m['channels']}ch  {m['codec']}  {m['bytes']//1024}KB"
        )
    for label, slug in missing:
        print(f"{label:28} MISSING ({slug}_test.wav) - waiting on quota ladder")

    if len(rows) < len(MODELS):
        print(f"\n[{len(rows)}/{len(MODELS)} models present] Verdict blocked until the quota ladder renders the rest.")
        return 2

    # Objective lyric-alignment probe: force Romanian and count expected words.
    # Uses faster-whisper when importable (scratch venv), else skips cleanly.
    try:
        from faster_whisper import WhisperModel  # type: ignore

        wm = WhisperModel("small", device="cpu", compute_type="int8")
        expected = ["liniste", "ploua", "paine", "poiana"]
        print("\n=== Vocal lyric alignment (Whisper small, language=ro) ===")
        for slug, label in MODELS.items():
            p = base / f"{slug}_test.wav"
            if not p.exists():
                continue
            segs, _ = wm.transcribe(str(p), language="ro", beam_size=5)
            txt = " ".join(s.text.strip() for s in segs).lower()
            hits = sum(1 for w in expected if w in txt)
            print(f"{label:28} ro-match {hits}/4 | {txt[:150]}")
    except ImportError:
        print("\n(faster-whisper not importable; skipping vocal alignment probe)")

    # Lyric-quality judgement is human/LLM-eval; print the checklist template.
    print("\n=== Lyric-quality A/B checklist (Moldovan lines: 'Liniste, liniste, afara ploua...') ===")
    for line in [
        "1. Pronunciation of Romanian diacritics (ă/â/ș/ț) by the vocal model",
        "2. Lyric-to-vocal alignment (are the Moldovan lines actually sung?)",
        "3. Instrumental authenticity (manele/balkan vibe vs generic EDM)",
        "4. Intelligibility at 15s length (hook formation)",
        "5. Artifacts/clipping (48kHz sources: YuE2, ACE-Step)",
    ]:
        print(f"  {line}")
    print("\nPlay the three WAVs and score 1-5 per row; overall verdict = best score sum.")
    return 0


if __name__ == "__main__":
    sys.exit(main())