Download tests/test_defects.py from raianand/SCQC: direct link, hf CLI and curl.
- Browser
- Download file 9.43 kB
-
https://huggingface.co/spaces/raianand/SCQC/resolve/main/tests/test_defects.py
- Command line
-
hf download hf://spaces/raianand/SCQC/tests/test_defects.py
-
curl -L -o test_defects.py https://huggingface.co/spaces/raianand/SCQC/resolve/main/tests/test_defects.py
9.43 kB
| import json | |
| from pathlib import Path | |
| import numpy as np | |
| from scqc import defects as D | |
| from scqc import mfa as M | |
| from scqc import synth | |
| from scqc.asr import FixedASR | |
| SENTS = ["आज मौसम बहुत अच्छा है", "मुझे एक गिलास पानी चाहिए", "बच्चे पार्क में खेल रहे हैं", | |
| "कृपया दरवाज़ा बंद कर दीजिए", "रेलगाड़ी प्लेटफॉर्म तीन पर आएगी", "किताब मेज़ पर रखी है"] | |
| LONG = "रेलगाड़ी प्लेटफॉर्म नंबर तीन पर आज शाम सात बजे आएगी और यात्री कृपया ध्यान दें" | |
| def build(tmp: Path): | |
| """60 clips with planted defects. Returns clips, hyps for two detectors, fake MFA, and truth.""" | |
| clips, h1, h2, mfa, truth = [], {}, {}, {}, {} | |
| rng = np.random.default_rng(0) | |
| for i in range(60): | |
| g = "a" if i % 2 == 0 else "b" | |
| ref = SENTS[i % len(SENTS)] | |
| x = synth.utterance(2.0 + 0.2 * rng.standard_normal(), seed=i) | |
| kind = "valid" | |
| hyp1 = hyp2 = ref | |
| if i in (0, 1, 2, 3): # speaker said something else: both ASRs agree on it | |
| hyp1 = hyp2 = SENTS[(i + 3) % len(SENTS)] | |
| kind = "speaker_deviation" | |
| elif i in (4, 5): # truncated: long text, short audio running into the end | |
| ref = LONG | |
| x = np.concatenate([synth.silence(0.3), synth.speechlike(1.4)]) | |
| hyp1 = hyp2 = "रेलगाड़ी प्लेटफॉर्म" | |
| kind = "truncated" | |
| elif i == 6: | |
| x = synth.silence(2.0, noise_db=-60) | |
| kind = "no_speech" | |
| elif i in (7, 9, 11, 13, 15): # hard audio in group b: ASRs fail differently, audio is fine | |
| hyp1, hyp2 = "कुछ अलग शब्द यहाँ", "पूरी तरह दूसरा वाक्य" | |
| p = tmp / f"c{i}.wav" | |
| p.write_bytes(synth.wav_bytes(x)) | |
| cid = f"c{i}" | |
| clips.append({"clip_id": cid, "audio_path": p.name, "reference": ref, "language": "hi", "gender": g}) | |
| h1[cid], h2[cid] = hyp1, hyp2 | |
| bad = kind in ("speaker_deviation", "truncated") | |
| mfa[cid] = {"mfa_aligned": not (i in (0, 1)), "oov_rate": 0.0, | |
| "mfa_speech_log_likelihood": float(-50 + rng.normal(0, 1) - (12 if bad else 0)), | |
| "mfa_phone_duration_deviation": float(3 + rng.normal(0, 0.3) + (4 if bad else 0)), | |
| "mfa_short_word_frac": 0.5 if kind == "truncated" else 0.0} | |
| truth[cid] = kind | |
| return clips, {"sravaani": h1, "indicconformer": h2}, mfa, truth | |
| def test_evidence_levels(tmp_path): | |
| clips, hyps, mfa, truth = build(tmp_path) | |
| ev = {r["clip_id"]: r for r in D.score(clips, hyps, ["sravaani", "indicconformer"], mfa, str(tmp_path), progress=False)} | |
| for i in (0, 1, 2, 3): | |
| assert ev[f"c{i}"]["evidence_level"] == "strong", ev[f"c{i}"] | |
| assert "asr_consensus_mismatch" in ev[f"c{i}"]["evidence_kinds"] | |
| for i in (4, 5): | |
| assert ev[f"c{i}"]["evidence_level"] == "strong" | |
| assert "truncated_or_missing_audio" in ev[f"c{i}"]["evidence_kinds"] | |
| assert ev["c6"]["evidence_kinds"] == "no_speech" | |
| # Divergent ASR on clean audio with a clean MFA fit cancels out: not reported as a defect. | |
| for i in (7, 9, 11): | |
| assert "asr_divergent" in ev[f"c{i}"]["evidence_kinds"] | |
| assert ev[f"c{i}"]["evidence_level"] == "none" | |
| clean = [r for k, r in ev.items() if truth[k] == "valid" and int(k[1:]) > 15] | |
| assert all(r["evidence_level"] == "none" for r in clean) | |
| def test_without_mfa_divergence_is_weak_only(tmp_path): | |
| clips, hyps, _, _ = build(tmp_path) | |
| ev = {r["clip_id"]: r for r in D.score(clips, hyps, ["sravaani", "indicconformer"], None, str(tmp_path), progress=False)} | |
| assert ev["c7"]["evidence_level"] == "weak" | |
| assert ev["c0"]["evidence_level"] == "moderate" | |
| def test_sample_report_rescore(tmp_path): | |
| clips, hyps, mfa, truth = build(tmp_path) | |
| ev = D.score(clips, hyps, ["sravaani", "indicconformer"], mfa, str(tmp_path), progress=False) | |
| blind, key = D.sample(ev, ["gender"], per_stratum=100) | |
| assert "evidence_level" not in blind[0] and "hyp_sravaani" not in blind[0] | |
| adj = [{"clip_id": b["clip_id"], "human_issue": "valid" if truth[b["clip_id"]] == "valid" else truth[b["clip_id"]], | |
| "annotator": "x"} for b in blind] | |
| rep = D.report(ev, key, adj, by="gender", flag_at="moderate", n_boot=100) | |
| assert rep["overall"]["est_defect_rate"] == round(7 / 60, 4) | |
| assert rep["overall"]["flag_precision"] == 1.0 | |
| assert rep["overall"]["defects_caught"] == 1.0 | |
| assert rep["precision_by_evidence_level"]["strong"]["defect_precision"] == 1.0 | |
| assert rep["groups"]["b"]["flags_on_valid_clips"] == 0.0 | |
| whisper = {c["clip_id"]: c["reference"] for c in clips} # an independent model, perfect on valid clips | |
| whisper["c0"] = "बिल्कुल गलत" | |
| rs = D.rescore(ev, {**hyps, "whisper": whisper}, by="gender", key=key, adjudicated=adj) | |
| assert rs["models"]["sravaani"]["used_for_detection"] and not rs["models"]["whisper"]["used_for_detection"] | |
| w = rs["models"]["sravaani"]["wer"]["_all"] | |
| assert w["sample_human_valid"] < w["sample_all"] and w["delta_human"] < 0 | |
| def test_transcribe_resumes(tmp_path): | |
| clips, hyps, _, _ = build(tmp_path) | |
| out = tmp_path / "h.jsonl" | |
| D.transcribe(clips[:5], FixedASR(hyps["sravaani"], "sv"), str(tmp_path), "sravaani", str(out), progress=False) | |
| D.transcribe(clips[:8], FixedASR(hyps["sravaani"], "sv"), str(tmp_path), "sravaani", str(out), progress=False) | |
| lines = out.read_text(encoding="utf-8").splitlines() | |
| assert len(lines) == 8 and D.model_name_of(str(out)) == "sravaani" | |
| assert D.read_hyps(str(out))["c0"] == hyps["sravaani"]["c0"] | |
| def test_mfa_prepare_and_parse(tmp_path): | |
| clips, _, _, _ = build(tmp_path) | |
| clips = clips[:3] | |
| clips[0]["reference"] = "आज, मौसम बहुत अच्छा है।" | |
| dic = tmp_path / "hi.dict" | |
| dic.write_text("आज\ta a dʒ\nमौसम\tm ɔ s ə m\nबहुत\tb ə h u t\n", encoding="utf-8") | |
| corpus, out = tmp_path / "corpus", tmp_path / "mfa_out" | |
| r = M.prepare_corpus(clips, str(corpus), str(tmp_path), dictionary_file=str(dic)) | |
| assert r["n_clips"] == 3 | |
| assert (corpus / "c0" / "c0.lab").read_text(encoding="utf-8").strip() == "आज मौसम बहुत अच्छा है" | |
| # Fake MFA output: c0 as JSON, c1 as TextGrid, c2 unaligned. | |
| (out / "c0").mkdir(parents=True) | |
| (out / "c1").mkdir() | |
| (out / "c0" / "c0.json").write_text(json.dumps({"start": 0, "end": 2.4, "tiers": { | |
| "words": {"type": "interval", "entries": [[0.3, 0.6, "आज"], [0.6, 1.0, "मौसम"], [1.0, 1.02, "बहुत"], | |
| [1.1, 1.5, "<unk>"], [1.5, 1.9, "है"]]}, | |
| "phones": {"type": "interval", "entries": []}}}, ensure_ascii=False), encoding="utf-8") | |
| (out / "c1" / "c1.TextGrid").write_text('''File type = "ooTextFile" | |
| Object class = "TextGrid" | |
| xmin = 0 | |
| xmax = 2 | |
| tiers? <exists> | |
| size = 1 | |
| item []: | |
| item [1]: | |
| class = "IntervalTier" | |
| name = "words" | |
| xmin = 0 | |
| xmax = 2 | |
| intervals: size = 2 | |
| intervals [1]: | |
| xmin = 0.2 | |
| xmax = 0.9 | |
| text = "मुझे" | |
| intervals [2]: | |
| xmin = 0.9 | |
| xmax = 1.6 | |
| text = "पानी" | |
| ''', encoding="utf-8") | |
| (out / "alignment_analysis.csv").write_text( | |
| "file,begin,end,speaker,overall_log_likelihood,speech_log_likelihood,phone_duration_deviation,snr\n" | |
| "c0,0.0,2.4,c0,-45.1,-49.0,3.2,20\nc1,0.0,2.0,c1,-44.0,-47.5,2.1,25\n", encoding="utf-8") | |
| feats = M.parse_results(str(corpus), str(out)) | |
| assert feats["c0"]["mfa_aligned"] and feats["c0"]["mfa_unk_words"] == 1 and feats["c0"]["mfa_n_words"] == 5 | |
| assert feats["c0"]["mfa_short_word_frac"] == 0.2 and feats["c0"]["oov_rate"] == 0.4 | |
| assert feats["c0"]["mfa_speech_log_likelihood"] == -49.0 | |
| assert feats["c1"]["mfa_n_words"] == 2 and feats["c1"]["mfa_aligned"] | |
| assert feats["c2"]["mfa_aligned"] is False | |
| cmd = M.align_command("corpus", "hindi_cv", "hindi_cv", "out") | |
| assert cmd[:6] == ["mfa", "align", "corpus", "hindi_cv", "hindi_cv", "out"] and "json" in cmd | |
| def test_calibrate_cli(tmp_path): | |
| import csv | |
| from scqc import cli | |
| clips, hyps, mfa, truth = build(tmp_path) | |
| ev = D.score(clips, hyps, ["sravaani", "indicconformer"], mfa, str(tmp_path), progress=False) | |
| D.write_csv(str(tmp_path / "ev.csv"), ev) | |
| rows = [{"clip_id": c["clip_id"], "human_issue": "valid" if truth[c["clip_id"]] == "valid" else truth[c["clip_id"]], | |
| "annotator": a} for c in clips for a in ("p", "q", "r")] | |
| D.write_csv(str(tmp_path / "adj.csv"), rows) | |
| cli.main(["calibrate", "--evidence", str(tmp_path / "ev.csv"), "--adjudicated", str(tmp_path / "adj.csv"), | |
| "--out", str(tmp_path / "m.json")]) | |
| m = json.loads((tmp_path / "m.json").read_text()) | |
| assert m["calibrated"] and m["annotator_agreement"]["n_annotators"] == 3 and m["n_train"] == 60 | |