Spaces:
Running
Running
File size: 4,292 Bytes
4cccf8f 11ca35b 4cccf8f 7a48bd2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 | """End-to-end checks with the real models on six Nigerian English recordings (tests/fixtures/slr70).
They download about 3 GB of models, so they run only when PG_MODEL_TESTS=1 (the CI "models" job, which guards
every library upgrade before it is deployed). expected.json holds what the tested environment heard on these
clips; a new environment must agree with it.
PG_MODEL_TESTS=1 python -m pytest -q tests/test_models.py
PG_MODEL_TESTS=1 PG_WRITE_EXPECTED=1 python -m pytest -q tests/test_models.py # record a new reference
"""
import json
import os
from pathlib import Path
import numpy as np
import pytest
pytestmark = pytest.mark.skipif(os.environ.get("PG_MODEL_TESTS") != "1", reason="set PG_MODEL_TESTS=1 to run")
FIXTURES = Path(__file__).parent / "fixtures" / "slr70"
EXPECTED = FIXTURES / "expected.json"
WORDS = ["think", "three", "north", "south", "something", "thousand"]
def _tsv(name):
return dict(line.rstrip("\n").split("\t", 1) for line in (FIXTURES / name).read_text().splitlines(True))
@pytest.fixture(scope="module")
def clips():
from pg.audio import load
transcripts, speakers = _tsv("transcripts.tsv"), _tsv("speakers.tsv")
return [(n, load(FIXTURES / n), transcripts[n], speakers[n]) for n in sorted(transcripts)]
@pytest.fixture(scope="module")
def conversation(clips):
"""The six clips joined into one recording, as in a short panel conversation."""
gap = np.zeros(8000, dtype=np.float32)
pieces, truth, t = [], [], 0.0
for name, audio, _, speaker in clips:
pieces += [audio, gap]
truth.append((t, t + len(audio) / 16000, speaker))
t += (len(audio) + len(gap)) / 16000
return np.concatenate(pieces), truth
@pytest.fixture(scope="module")
def analyzer():
from pg.pipeline import Analyzer
return Analyzer()
def test_variants_and_preferred_form_match_the_reference(analyzer, clips):
from pg.pipeline import Recording
recs = [Recording(n, a, speaker=s, transcript=t) for n, a, t, s in clips]
result = analyzer.analyse(recs, WORDS, "TH-voiceless")
got = {f"{t.recording}:{t.word}": t.variant for t in result.tokens}
if os.environ.get("PG_WRITE_EXPECTED") == "1":
EXPECTED.write_text(json.dumps({"tokens": got, "preferred": result.preferred}, indent=1, ensure_ascii=False))
expected = json.loads(EXPECTED.read_text())
assert set(got) == set(expected["tokens"]), "the target words were not all found"
same = sum(got[k] == v for k, v in expected["tokens"].items())
assert same >= len(expected["tokens"]) - 1, f"variants changed: {got} vs {expected['tokens']}"
assert result.preferred == expected["preferred"]
def test_speech_recognition_finds_the_target_words(analyzer, conversation):
audio, _ = conversation
heard = {w.word for w in analyzer.transcriber._transcribe(audio)}
assert len(heard & set(WORDS)) >= 4, heard
# and they survive the loop filter that every transcript goes through (whatever transformers' timestamps)
from pg.asr import drop_loops
kept = {w.word for w in drop_loops(analyzer.transcriber._transcribe(audio))}
assert len(kept & set(WORDS)) >= 4, kept
def test_speaker_separation_finds_the_three_speakers(analyzer, conversation):
audio, truth = conversation
for dia in (analyzer.diarizer(audio), analyzer.diarizer(audio, num_speakers=3)):
assert dia.speakers == 3
# every clip is credited to one label, and different speakers to different labels
label = {}
for start, end, speaker in truth:
label.setdefault(speaker, set()).add(dia.speaker_at(start, end))
assert all(len(v) == 1 for v in label.values()), label
assert len({next(iter(v)) for v in label.values()}) == 3, label
def test_unscripted_audio_finds_and_scores_the_words(analyzer, conversation):
"""The podcast path end to end: speech recognition, re-alignment of each hit, scoring, speakers."""
from pg.pipeline import Recording
audio, _ = conversation
result = analyzer.analyse([Recording("conversation.wav", audio, transcript="auto", num_speakers=3)],
WORDS, "TH-voiceless")
assert len(result.tokens) >= 4, result.warnings
assert len({t.speaker for t in result.tokens}) >= 2, result.warnings
|