SCQC / tests /test_defects.py
raianand's picture
Upload folder using huggingface_hub
e0d876f verified
Raw History Blame Contribute Delete
9.43 kB
import json
from pathlib import Path
import numpy as np
from scqc import defects as D
from scqc import mfa as M
from scqc import synth
from scqc.asr import FixedASR
SENTS = ["आज मौसम बहुत अच्छा है", "मुझे एक गिलास पानी चाहिए", "बच्चे पार्क में खेल रहे हैं",
"कृपया दरवाज़ा बंद कर दीजिए", "रेलगाड़ी प्लेटफॉर्म तीन पर आएगी", "किताब मेज़ पर रखी है"]
LONG = "रेलगाड़ी प्लेटफॉर्म नंबर तीन पर आज शाम सात बजे आएगी और यात्री कृपया ध्यान दें"
def build(tmp: Path):
"""60 clips with planted defects. Returns clips, hyps for two detectors, fake MFA, and truth."""
clips, h1, h2, mfa, truth = [], {}, {}, {}, {}
rng = np.random.default_rng(0)
for i in range(60):
g = "a" if i % 2 == 0 else "b"
ref = SENTS[i % len(SENTS)]
x = synth.utterance(2.0 + 0.2 * rng.standard_normal(), seed=i)
kind = "valid"
hyp1 = hyp2 = ref
if i in (0, 1, 2, 3): # speaker said something else: both ASRs agree on it
hyp1 = hyp2 = SENTS[(i + 3) % len(SENTS)]
kind = "speaker_deviation"
elif i in (4, 5): # truncated: long text, short audio running into the end
ref = LONG
x = np.concatenate([synth.silence(0.3), synth.speechlike(1.4)])
hyp1 = hyp2 = "रेलगाड़ी प्लेटफॉर्म"
kind = "truncated"
elif i == 6:
x = synth.silence(2.0, noise_db=-60)
kind = "no_speech"
elif i in (7, 9, 11, 13, 15): # hard audio in group b: ASRs fail differently, audio is fine
hyp1, hyp2 = "कुछ अलग शब्द यहाँ", "पूरी तरह दूसरा वाक्य"
p = tmp / f"c{i}.wav"
p.write_bytes(synth.wav_bytes(x))
cid = f"c{i}"
clips.append({"clip_id": cid, "audio_path": p.name, "reference": ref, "language": "hi", "gender": g})
h1[cid], h2[cid] = hyp1, hyp2
bad = kind in ("speaker_deviation", "truncated")
mfa[cid] = {"mfa_aligned": not (i in (0, 1)), "oov_rate": 0.0,
"mfa_speech_log_likelihood": float(-50 + rng.normal(0, 1) - (12 if bad else 0)),
"mfa_phone_duration_deviation": float(3 + rng.normal(0, 0.3) + (4 if bad else 0)),
"mfa_short_word_frac": 0.5 if kind == "truncated" else 0.0}
truth[cid] = kind
return clips, {"sravaani": h1, "indicconformer": h2}, mfa, truth
def test_evidence_levels(tmp_path):
clips, hyps, mfa, truth = build(tmp_path)
ev = {r["clip_id"]: r for r in D.score(clips, hyps, ["sravaani", "indicconformer"], mfa, str(tmp_path), progress=False)}
for i in (0, 1, 2, 3):
assert ev[f"c{i}"]["evidence_level"] == "strong", ev[f"c{i}"]
assert "asr_consensus_mismatch" in ev[f"c{i}"]["evidence_kinds"]
for i in (4, 5):
assert ev[f"c{i}"]["evidence_level"] == "strong"
assert "truncated_or_missing_audio" in ev[f"c{i}"]["evidence_kinds"]
assert ev["c6"]["evidence_kinds"] == "no_speech"
# Divergent ASR on clean audio with a clean MFA fit cancels out: not reported as a defect.
for i in (7, 9, 11):
assert "asr_divergent" in ev[f"c{i}"]["evidence_kinds"]
assert ev[f"c{i}"]["evidence_level"] == "none"
clean = [r for k, r in ev.items() if truth[k] == "valid" and int(k[1:]) > 15]
assert all(r["evidence_level"] == "none" for r in clean)
def test_without_mfa_divergence_is_weak_only(tmp_path):
clips, hyps, _, _ = build(tmp_path)
ev = {r["clip_id"]: r for r in D.score(clips, hyps, ["sravaani", "indicconformer"], None, str(tmp_path), progress=False)}
assert ev["c7"]["evidence_level"] == "weak"
assert ev["c0"]["evidence_level"] == "moderate"
def test_sample_report_rescore(tmp_path):
clips, hyps, mfa, truth = build(tmp_path)
ev = D.score(clips, hyps, ["sravaani", "indicconformer"], mfa, str(tmp_path), progress=False)
blind, key = D.sample(ev, ["gender"], per_stratum=100)
assert "evidence_level" not in blind[0] and "hyp_sravaani" not in blind[0]
adj = [{"clip_id": b["clip_id"], "human_issue": "valid" if truth[b["clip_id"]] == "valid" else truth[b["clip_id"]],
"annotator": "x"} for b in blind]
rep = D.report(ev, key, adj, by="gender", flag_at="moderate", n_boot=100)
assert rep["overall"]["est_defect_rate"] == round(7 / 60, 4)
assert rep["overall"]["flag_precision"] == 1.0
assert rep["overall"]["defects_caught"] == 1.0
assert rep["precision_by_evidence_level"]["strong"]["defect_precision"] == 1.0
assert rep["groups"]["b"]["flags_on_valid_clips"] == 0.0
whisper = {c["clip_id"]: c["reference"] for c in clips} # an independent model, perfect on valid clips
whisper["c0"] = "बिल्कुल गलत"
rs = D.rescore(ev, {**hyps, "whisper": whisper}, by="gender", key=key, adjudicated=adj)
assert rs["models"]["sravaani"]["used_for_detection"] and not rs["models"]["whisper"]["used_for_detection"]
w = rs["models"]["sravaani"]["wer"]["_all"]
assert w["sample_human_valid"] < w["sample_all"] and w["delta_human"] < 0
def test_transcribe_resumes(tmp_path):
clips, hyps, _, _ = build(tmp_path)
out = tmp_path / "h.jsonl"
D.transcribe(clips[:5], FixedASR(hyps["sravaani"], "sv"), str(tmp_path), "sravaani", str(out), progress=False)
D.transcribe(clips[:8], FixedASR(hyps["sravaani"], "sv"), str(tmp_path), "sravaani", str(out), progress=False)
lines = out.read_text(encoding="utf-8").splitlines()
assert len(lines) == 8 and D.model_name_of(str(out)) == "sravaani"
assert D.read_hyps(str(out))["c0"] == hyps["sravaani"]["c0"]
def test_mfa_prepare_and_parse(tmp_path):
clips, _, _, _ = build(tmp_path)
clips = clips[:3]
clips[0]["reference"] = "आज, मौसम बहुत अच्छा है।"
dic = tmp_path / "hi.dict"
dic.write_text("आज\ta a dʒ\nमौसम\tm ɔ s ə m\nबहुत\tb ə h u t\n", encoding="utf-8")
corpus, out = tmp_path / "corpus", tmp_path / "mfa_out"
r = M.prepare_corpus(clips, str(corpus), str(tmp_path), dictionary_file=str(dic))
assert r["n_clips"] == 3
assert (corpus / "c0" / "c0.lab").read_text(encoding="utf-8").strip() == "आज मौसम बहुत अच्छा है"
# Fake MFA output: c0 as JSON, c1 as TextGrid, c2 unaligned.
(out / "c0").mkdir(parents=True)
(out / "c1").mkdir()
(out / "c0" / "c0.json").write_text(json.dumps({"start": 0, "end": 2.4, "tiers": {
"words": {"type": "interval", "entries": [[0.3, 0.6, "आज"], [0.6, 1.0, "मौसम"], [1.0, 1.02, "बहुत"],
[1.1, 1.5, "<unk>"], [1.5, 1.9, "है"]]},
"phones": {"type": "interval", "entries": []}}}, ensure_ascii=False), encoding="utf-8")
(out / "c1" / "c1.TextGrid").write_text('''File type = "ooTextFile"
Object class = "TextGrid"
xmin = 0
xmax = 2
tiers? <exists>
size = 1
item []:
item [1]:
class = "IntervalTier"
name = "words"
xmin = 0
xmax = 2
intervals: size = 2
intervals [1]:
xmin = 0.2
xmax = 0.9
text = "मुझे"
intervals [2]:
xmin = 0.9
xmax = 1.6
text = "पानी"
''', encoding="utf-8")
(out / "alignment_analysis.csv").write_text(
"file,begin,end,speaker,overall_log_likelihood,speech_log_likelihood,phone_duration_deviation,snr\n"
"c0,0.0,2.4,c0,-45.1,-49.0,3.2,20\nc1,0.0,2.0,c1,-44.0,-47.5,2.1,25\n", encoding="utf-8")
feats = M.parse_results(str(corpus), str(out))
assert feats["c0"]["mfa_aligned"] and feats["c0"]["mfa_unk_words"] == 1 and feats["c0"]["mfa_n_words"] == 5
assert feats["c0"]["mfa_short_word_frac"] == 0.2 and feats["c0"]["oov_rate"] == 0.4
assert feats["c0"]["mfa_speech_log_likelihood"] == -49.0
assert feats["c1"]["mfa_n_words"] == 2 and feats["c1"]["mfa_aligned"]
assert feats["c2"]["mfa_aligned"] is False
cmd = M.align_command("corpus", "hindi_cv", "hindi_cv", "out")
assert cmd[:6] == ["mfa", "align", "corpus", "hindi_cv", "hindi_cv", "out"] and "json" in cmd
def test_calibrate_cli(tmp_path):
import csv
from scqc import cli
clips, hyps, mfa, truth = build(tmp_path)
ev = D.score(clips, hyps, ["sravaani", "indicconformer"], mfa, str(tmp_path), progress=False)
D.write_csv(str(tmp_path / "ev.csv"), ev)
rows = [{"clip_id": c["clip_id"], "human_issue": "valid" if truth[c["clip_id"]] == "valid" else truth[c["clip_id"]],
"annotator": a} for c in clips for a in ("p", "q", "r")]
D.write_csv(str(tmp_path / "adj.csv"), rows)
cli.main(["calibrate", "--evidence", str(tmp_path / "ev.csv"), "--adjudicated", str(tmp_path / "adj.csv"),
"--out", str(tmp_path / "m.json")])
m = json.loads((tmp_path / "m.json").read_text())
assert m["calibrated"] and m["annotator_agreement"]["n_annotators"] == 3 and m["n_train"] == 60