File size: 4,292 Bytes
4cccf8f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11ca35b
 
 
 
 
4cccf8f
 
 
 
 
 
 
 
 
 
 
 
7a48bd2
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
"""End-to-end checks with the real models on six Nigerian English recordings (tests/fixtures/slr70).

They download about 3 GB of models, so they run only when PG_MODEL_TESTS=1 (the CI "models" job, which guards
every library upgrade before it is deployed). expected.json holds what the tested environment heard on these
clips; a new environment must agree with it.

    PG_MODEL_TESTS=1 python -m pytest -q tests/test_models.py
    PG_MODEL_TESTS=1 PG_WRITE_EXPECTED=1 python -m pytest -q tests/test_models.py   # record a new reference
"""
import json
import os
from pathlib import Path

import numpy as np
import pytest

pytestmark = pytest.mark.skipif(os.environ.get("PG_MODEL_TESTS") != "1", reason="set PG_MODEL_TESTS=1 to run")

FIXTURES = Path(__file__).parent / "fixtures" / "slr70"
EXPECTED = FIXTURES / "expected.json"
WORDS = ["think", "three", "north", "south", "something", "thousand"]


def _tsv(name):
    return dict(line.rstrip("\n").split("\t", 1) for line in (FIXTURES / name).read_text().splitlines(True))


@pytest.fixture(scope="module")
def clips():
    from pg.audio import load

    transcripts, speakers = _tsv("transcripts.tsv"), _tsv("speakers.tsv")
    return [(n, load(FIXTURES / n), transcripts[n], speakers[n]) for n in sorted(transcripts)]


@pytest.fixture(scope="module")
def conversation(clips):
    """The six clips joined into one recording, as in a short panel conversation."""
    gap = np.zeros(8000, dtype=np.float32)
    pieces, truth, t = [], [], 0.0
    for name, audio, _, speaker in clips:
        pieces += [audio, gap]
        truth.append((t, t + len(audio) / 16000, speaker))
        t += (len(audio) + len(gap)) / 16000
    return np.concatenate(pieces), truth


@pytest.fixture(scope="module")
def analyzer():
    from pg.pipeline import Analyzer

    return Analyzer()


def test_variants_and_preferred_form_match_the_reference(analyzer, clips):
    from pg.pipeline import Recording

    recs = [Recording(n, a, speaker=s, transcript=t) for n, a, t, s in clips]
    result = analyzer.analyse(recs, WORDS, "TH-voiceless")
    got = {f"{t.recording}:{t.word}": t.variant for t in result.tokens}
    if os.environ.get("PG_WRITE_EXPECTED") == "1":
        EXPECTED.write_text(json.dumps({"tokens": got, "preferred": result.preferred}, indent=1, ensure_ascii=False))
    expected = json.loads(EXPECTED.read_text())
    assert set(got) == set(expected["tokens"]), "the target words were not all found"
    same = sum(got[k] == v for k, v in expected["tokens"].items())
    assert same >= len(expected["tokens"]) - 1, f"variants changed: {got} vs {expected['tokens']}"
    assert result.preferred == expected["preferred"]


def test_speech_recognition_finds_the_target_words(analyzer, conversation):
    audio, _ = conversation
    heard = {w.word for w in analyzer.transcriber._transcribe(audio)}
    assert len(heard & set(WORDS)) >= 4, heard
    # and they survive the loop filter that every transcript goes through (whatever transformers' timestamps)
    from pg.asr import drop_loops

    kept = {w.word for w in drop_loops(analyzer.transcriber._transcribe(audio))}
    assert len(kept & set(WORDS)) >= 4, kept


def test_speaker_separation_finds_the_three_speakers(analyzer, conversation):
    audio, truth = conversation
    for dia in (analyzer.diarizer(audio), analyzer.diarizer(audio, num_speakers=3)):
        assert dia.speakers == 3
        # every clip is credited to one label, and different speakers to different labels
        label = {}
        for start, end, speaker in truth:
            label.setdefault(speaker, set()).add(dia.speaker_at(start, end))
        assert all(len(v) == 1 for v in label.values()), label
        assert len({next(iter(v)) for v in label.values()}) == 3, label


def test_unscripted_audio_finds_and_scores_the_words(analyzer, conversation):
    """The podcast path end to end: speech recognition, re-alignment of each hit, scoring, speakers."""
    from pg.pipeline import Recording

    audio, _ = conversation
    result = analyzer.analyse([Recording("conversation.wav", audio, transcript="auto", num_speakers=3)],
                              WORDS, "TH-voiceless")
    assert len(result.tokens) >= 4, result.warnings
    assert len({t.speaker for t in result.tokens}) >= 2, result.warnings