import json from pathlib import Path import numpy as np from scqc import defects as D from scqc import mfa as M from scqc import synth from scqc.asr import FixedASR SENTS = ["आज मौसम बहुत अच्छा है", "मुझे एक गिलास पानी चाहिए", "बच्चे पार्क में खेल रहे हैं", "कृपया दरवाज़ा बंद कर दीजिए", "रेलगाड़ी प्लेटफॉर्म तीन पर आएगी", "किताब मेज़ पर रखी है"] LONG = "रेलगाड़ी प्लेटफॉर्म नंबर तीन पर आज शाम सात बजे आएगी और यात्री कृपया ध्यान दें" def build(tmp: Path): """60 clips with planted defects. Returns clips, hyps for two detectors, fake MFA, and truth.""" clips, h1, h2, mfa, truth = [], {}, {}, {}, {} rng = np.random.default_rng(0) for i in range(60): g = "a" if i % 2 == 0 else "b" ref = SENTS[i % len(SENTS)] x = synth.utterance(2.0 + 0.2 * rng.standard_normal(), seed=i) kind = "valid" hyp1 = hyp2 = ref if i in (0, 1, 2, 3): # speaker said something else: both ASRs agree on it hyp1 = hyp2 = SENTS[(i + 3) % len(SENTS)] kind = "speaker_deviation" elif i in (4, 5): # truncated: long text, short audio running into the end ref = LONG x = np.concatenate([synth.silence(0.3), synth.speechlike(1.4)]) hyp1 = hyp2 = "रेलगाड़ी प्लेटफॉर्म" kind = "truncated" elif i == 6: x = synth.silence(2.0, noise_db=-60) kind = "no_speech" elif i in (7, 9, 11, 13, 15): # hard audio in group b: ASRs fail differently, audio is fine hyp1, hyp2 = "कुछ अलग शब्द यहाँ", "पूरी तरह दूसरा वाक्य" p = tmp / f"c{i}.wav" p.write_bytes(synth.wav_bytes(x)) cid = f"c{i}" clips.append({"clip_id": cid, "audio_path": p.name, "reference": ref, "language": "hi", "gender": g}) h1[cid], h2[cid] = hyp1, hyp2 bad = kind in ("speaker_deviation", "truncated") mfa[cid] = {"mfa_aligned": not (i in (0, 1)), "oov_rate": 0.0, "mfa_speech_log_likelihood": float(-50 + rng.normal(0, 1) - (12 if bad else 0)), "mfa_phone_duration_deviation": float(3 + rng.normal(0, 0.3) + (4 if bad else 0)), "mfa_short_word_frac": 0.5 if kind == "truncated" else 0.0} truth[cid] = kind return clips, {"sravaani": h1, "indicconformer": h2}, mfa, truth def test_evidence_levels(tmp_path): clips, hyps, mfa, truth = build(tmp_path) ev = {r["clip_id"]: r for r in D.score(clips, hyps, ["sravaani", "indicconformer"], mfa, str(tmp_path), progress=False)} for i in (0, 1, 2, 3): assert ev[f"c{i}"]["evidence_level"] == "strong", ev[f"c{i}"] assert "asr_consensus_mismatch" in ev[f"c{i}"]["evidence_kinds"] for i in (4, 5): assert ev[f"c{i}"]["evidence_level"] == "strong" assert "truncated_or_missing_audio" in ev[f"c{i}"]["evidence_kinds"] assert ev["c6"]["evidence_kinds"] == "no_speech" # Divergent ASR on clean audio with a clean MFA fit cancels out: not reported as a defect. for i in (7, 9, 11): assert "asr_divergent" in ev[f"c{i}"]["evidence_kinds"] assert ev[f"c{i}"]["evidence_level"] == "none" clean = [r for k, r in ev.items() if truth[k] == "valid" and int(k[1:]) > 15] assert all(r["evidence_level"] == "none" for r in clean) def test_without_mfa_divergence_is_weak_only(tmp_path): clips, hyps, _, _ = build(tmp_path) ev = {r["clip_id"]: r for r in D.score(clips, hyps, ["sravaani", "indicconformer"], None, str(tmp_path), progress=False)} assert ev["c7"]["evidence_level"] == "weak" assert ev["c0"]["evidence_level"] == "moderate" def test_sample_report_rescore(tmp_path): clips, hyps, mfa, truth = build(tmp_path) ev = D.score(clips, hyps, ["sravaani", "indicconformer"], mfa, str(tmp_path), progress=False) blind, key = D.sample(ev, ["gender"], per_stratum=100) assert "evidence_level" not in blind[0] and "hyp_sravaani" not in blind[0] adj = [{"clip_id": b["clip_id"], "human_issue": "valid" if truth[b["clip_id"]] == "valid" else truth[b["clip_id"]], "annotator": "x"} for b in blind] rep = D.report(ev, key, adj, by="gender", flag_at="moderate", n_boot=100) assert rep["overall"]["est_defect_rate"] == round(7 / 60, 4) assert rep["overall"]["flag_precision"] == 1.0 assert rep["overall"]["defects_caught"] == 1.0 assert rep["precision_by_evidence_level"]["strong"]["defect_precision"] == 1.0 assert rep["groups"]["b"]["flags_on_valid_clips"] == 0.0 whisper = {c["clip_id"]: c["reference"] for c in clips} # an independent model, perfect on valid clips whisper["c0"] = "बिल्कुल गलत" rs = D.rescore(ev, {**hyps, "whisper": whisper}, by="gender", key=key, adjudicated=adj) assert rs["models"]["sravaani"]["used_for_detection"] and not rs["models"]["whisper"]["used_for_detection"] w = rs["models"]["sravaani"]["wer"]["_all"] assert w["sample_human_valid"] < w["sample_all"] and w["delta_human"] < 0 def test_transcribe_resumes(tmp_path): clips, hyps, _, _ = build(tmp_path) out = tmp_path / "h.jsonl" D.transcribe(clips[:5], FixedASR(hyps["sravaani"], "sv"), str(tmp_path), "sravaani", str(out), progress=False) D.transcribe(clips[:8], FixedASR(hyps["sravaani"], "sv"), str(tmp_path), "sravaani", str(out), progress=False) lines = out.read_text(encoding="utf-8").splitlines() assert len(lines) == 8 and D.model_name_of(str(out)) == "sravaani" assert D.read_hyps(str(out))["c0"] == hyps["sravaani"]["c0"] def test_mfa_prepare_and_parse(tmp_path): clips, _, _, _ = build(tmp_path) clips = clips[:3] clips[0]["reference"] = "आज, मौसम बहुत अच्छा है।" dic = tmp_path / "hi.dict" dic.write_text("आज\ta a dʒ\nमौसम\tm ɔ s ə m\nबहुत\tb ə h u t\n", encoding="utf-8") corpus, out = tmp_path / "corpus", tmp_path / "mfa_out" r = M.prepare_corpus(clips, str(corpus), str(tmp_path), dictionary_file=str(dic)) assert r["n_clips"] == 3 assert (corpus / "c0" / "c0.lab").read_text(encoding="utf-8").strip() == "आज मौसम बहुत अच्छा है" # Fake MFA output: c0 as JSON, c1 as TextGrid, c2 unaligned. (out / "c0").mkdir(parents=True) (out / "c1").mkdir() (out / "c0" / "c0.json").write_text(json.dumps({"start": 0, "end": 2.4, "tiers": { "words": {"type": "interval", "entries": [[0.3, 0.6, "आज"], [0.6, 1.0, "मौसम"], [1.0, 1.02, "बहुत"], [1.1, 1.5, ""], [1.5, 1.9, "है"]]}, "phones": {"type": "interval", "entries": []}}}, ensure_ascii=False), encoding="utf-8") (out / "c1" / "c1.TextGrid").write_text('''File type = "ooTextFile" Object class = "TextGrid" xmin = 0 xmax = 2 tiers? size = 1 item []: item [1]: class = "IntervalTier" name = "words" xmin = 0 xmax = 2 intervals: size = 2 intervals [1]: xmin = 0.2 xmax = 0.9 text = "मुझे" intervals [2]: xmin = 0.9 xmax = 1.6 text = "पानी" ''', encoding="utf-8") (out / "alignment_analysis.csv").write_text( "file,begin,end,speaker,overall_log_likelihood,speech_log_likelihood,phone_duration_deviation,snr\n" "c0,0.0,2.4,c0,-45.1,-49.0,3.2,20\nc1,0.0,2.0,c1,-44.0,-47.5,2.1,25\n", encoding="utf-8") feats = M.parse_results(str(corpus), str(out)) assert feats["c0"]["mfa_aligned"] and feats["c0"]["mfa_unk_words"] == 1 and feats["c0"]["mfa_n_words"] == 5 assert feats["c0"]["mfa_short_word_frac"] == 0.2 and feats["c0"]["oov_rate"] == 0.4 assert feats["c0"]["mfa_speech_log_likelihood"] == -49.0 assert feats["c1"]["mfa_n_words"] == 2 and feats["c1"]["mfa_aligned"] assert feats["c2"]["mfa_aligned"] is False cmd = M.align_command("corpus", "hindi_cv", "hindi_cv", "out") assert cmd[:6] == ["mfa", "align", "corpus", "hindi_cv", "hindi_cv", "out"] and "json" in cmd def test_calibrate_cli(tmp_path): import csv from scqc import cli clips, hyps, mfa, truth = build(tmp_path) ev = D.score(clips, hyps, ["sravaani", "indicconformer"], mfa, str(tmp_path), progress=False) D.write_csv(str(tmp_path / "ev.csv"), ev) rows = [{"clip_id": c["clip_id"], "human_issue": "valid" if truth[c["clip_id"]] == "valid" else truth[c["clip_id"]], "annotator": a} for c in clips for a in ("p", "q", "r")] D.write_csv(str(tmp_path / "adj.csv"), rows) cli.main(["calibrate", "--evidence", str(tmp_path / "ev.csv"), "--adjudicated", str(tmp_path / "adj.csv"), "--out", str(tmp_path / "m.json")]) m = json.loads((tmp_path / "m.json").read_text()) assert m["calibrated"] and m["annotator_agreement"]["n_annotators"] == 3 and m["n_train"] == 60