SCQC / server /mfa_worker.py
raianand's picture
Upload folder using huggingface_hub
e0d876f verified
Raw History Blame Contribute Delete
8.38 kB
"""Forced alignment in the background while contributors keep recording.
Enabled with SCQC_MFA_MODELS, a JSON map from language to MFA models, e.g.
SCQC_MFA_MODELS='{"hi": {"dictionary": "hindi_cv", "acoustic": "hindi_cv"}}'
and `mfa` on PATH. Recordings that passed every other check wait in status
"checking" (a few seconds to a minute); the worker then:
* aligns the prompt to the audio and stores word timings and fit statistics,
* re-runs the prosody check with real word timings (comma pauses, emphasis),
* recomputes the confidence score,
* accepts the recording (and credits payment), or sends it to review if the
alignment failed or fits poorly. It never rejects a recording.
If MFA errors, the recording falls back to its earlier automatic decision.
"""
from __future__ import annotations
import json
import os
import queue
import shutil
import statistics
import tempfile
import threading
import time
from pathlib import Path
from typing import Callable, Optional
from scqc import confidence as C
from scqc import mfa as M
from scqc import prosody as P
from scqc.audio import load_audio
from scqc.thresholds import resolve
ELIGIBLE = {"read_speech", "tts_prosody", "prosody_pair", "command"}
class MFAWorker:
def __init__(self):
try:
self.models = json.loads(os.environ.get("SCQC_MFA_MODELS", "") or "{}")
except json.JSONDecodeError:
self.models = {}
self.runner: Callable[[list[str]], int] = M.run_mfa
self.available = bool(self.models) and shutil.which("mfa") is not None
self.q: "queue.Queue[str]" = queue.Queue()
self.ll_by_lang: dict[str, list[float]] = {}
self.thread: Optional[threading.Thread] = None
self.hooks = None # set by the app: (db, manifest, pay, log)
self.last_error: Optional[str] = None
def eligible(self, task) -> bool:
return self.available and task.task_type in ELIGIBLE and task.language in self.models and bool(task.prompt.text)
def enqueue(self, rid: str) -> None:
if not self.available:
return
self.q.put(rid)
if self.thread is None or not self.thread.is_alive():
self.thread = threading.Thread(target=self._loop, daemon=True)
self.thread.start()
def _loop(self):
while True:
try:
rid = self.q.get(timeout=30)
except queue.Empty:
return
try:
self.process(rid)
except Exception as e: # never let one recording stop the worker
self.last_error = f"{type(e).__name__}: {e}"[:300]
self._finish(rid, None)
# -------------------------------------------------------------- one recording
def process(self, rid: str) -> dict:
db, manifest, _, _ = self.hooks
rec = db.one("SELECT * FROM recordings WHERE recording_id=?", (rid,))
if not rec or not rec["audio_path"]:
return {}
m = manifest(rec["project_id"])
task = m.task(rec["task_id"])
cfg = self.models.get(task.language) or {}
work = Path(tempfile.mkdtemp(prefix="scqc_mfa_"))
try:
corpus, out = work / "corpus", work / "out"
M.prepare_corpus([{"clip_id": rid, "audio_path": rec["audio_path"], "reference": task.prompt.text,
"speaker_id": rec["speaker_id"]}], str(corpus), "")
cmd = M.align_command(str(corpus), cfg.get("dictionary", ""), cfg.get("acoustic", ""), str(out), jobs=1)
code = self.runner(cmd)
feats = M.parse_results(str(corpus), str(out)).get(rid, {"mfa_aligned": False}) if code == 0 else None
finally:
shutil.rmtree(work, ignore_errors=True)
return self._finish(rid, feats)
def _ll_z(self, lang: str, ll: Optional[float]) -> Optional[float]:
if ll is None:
return None
hist = self.ll_by_lang.setdefault(lang, [])
z = None
if len(hist) >= 20:
med = statistics.median(hist)
mad = statistics.median(abs(v - med) for v in hist) * 1.4826
z = round((ll - med) / mad, 3) if mad > 1e-9 else None
hist.append(ll)
del hist[:-2000]
return z
def _finish(self, rid: str, feats: Optional[dict]) -> dict:
db, manifest, pay, log = self.hooks
rec = db.one("SELECT * FROM recordings WHERE recording_id=?", (rid,))
if not rec:
return {}
m = manifest(rec["project_id"])
task = m.task(rec["task_id"])
qc = json.loads(rec["qc"])
new_status = rec["status"]
if feats is None: # MFA failed: keep the earlier automatic decision
qc["validators"].append({"name": "mfa_alignment", "level": "L3", "available": False, "findings": [],
"scores": {}, "data": {"error": self.last_error}})
if rec["status"] == "checking":
new_status = "accepted"
else:
spans = feats.get("mfa_word_spans") or []
aligned = bool(feats.get("mfa_aligned"))
data = {"aligned": aligned, "speech_log_likelihood": feats.get("mfa_speech_log_likelihood"),
"ll_z": self._ll_z(task.language, feats.get("mfa_speech_log_likelihood")),
"short_word_frac": feats.get("mfa_short_word_frac"), "word_spans": spans}
poor = (not aligned) or (data["ll_z"] is not None and data["ll_z"] < -2.5) or \
((data["short_word_frac"] or 0) > 0.3)
finding = []
if poor:
finding = [{"code": "mfa_poor_fit", "action": "review", "validator": "mfa_alignment",
"message": "The prompt could not be fitted cleanly to the audio; a reviewer will listen.",
"detail": {k: v for k, v in data.items() if k != "word_spans"},
"model_based": True, "contributor_visible": False}]
qc["validators"] = [v for v in qc["validators"] if v["name"] != "mfa_alignment"]
qc["validators"].append({"name": "mfa_alignment", "level": "L3", "available": True,
"findings": finding, "scores": {}, "data": data})
qc["findings"].extend(finding)
# Re-run prosody with real word timings.
if task.task_type in ("tts_prosody", "prosody_pair") and spans and rec["audio_path"]:
th = resolve(task.task_type, m.validation_for(task).overrides)
a = load_audio(rec["audio_path"])
crit = "punctuation_prosody" if task.task_type == "tts_prosody" else "prosodic_realization"
pr = P.check(task.prompt.text or "", a.samples, a.sr, th, spans, task.prompt.emphasis, crit)
qc["validators"] = [v for v in qc["validators"] if v["name"] != "prosody"] + [pr.as_dict()]
if pr.scores.get(crit) is not None:
qc["rubric"]["criteria"].setdefault(crit, {})["score"] = pr.scores[crit]
qc["confidence"] = C.score(C.signals_from_report(qc, task.language, task.prompt.text))
v = m.validation_for(task)
low_conf = v.accept_confidence is not None and qc["confidence"]["score"] < v.accept_confidence
if rec["status"] == "checking":
new_status = "review" if (poor or low_conf) else "accepted"
if new_status == "review":
qc["route_notes"] = qc.get("route_notes", []) + (["mfa_poor_fit"] if poor else ["below_confidence_threshold"])
with db.tx() as c:
c.execute("UPDATE recordings SET qc=?, status=?, required_reviewers=MAX(required_reviewers, ?),"
" final_at=CASE WHEN ?='accepted' THEN ? ELSE final_at END WHERE recording_id=?",
(json.dumps(qc, ensure_ascii=False), new_status, 1 if new_status == "review" else 0,
new_status, time.time(), rid))
if rec["status"] == "checking" and new_status == "accepted":
pay(c, m, c.execute("SELECT * FROM recordings WHERE recording_id=?", (rid,)).fetchone())
log(c, "mfa", "mfa.done", rid, {"status": new_status, "ok": feats is not None})
return {"status": new_status, "feats": feats}
MFA = MFAWorker()