"""SCQC collection server. Roles are token-based for the demo: admin (manifests, export, payouts, reports) and reviewer (queue, audio, verdicts). Participants are identified only by a pseudonymous speaker_id issued at consent time. Environment SCQC_DATA data directory (default ./scqc_data) SCQC_ASR ASR backend spec, see scqc/asr.py (default: none) SCQC_ADMIN_TOKEN admin token (default "admin-dev"; change it) SCQC_REVIEW_TOKEN reviewer token (default "review-dev"; change it) """ from __future__ import annotations import hashlib import io import random import json import os import time import urllib.request import uuid import zipfile from pathlib import Path from typing import Optional from fastapi import Depends, FastAPI, File, Form, Header, HTTPException, Query, UploadFile from fastapi.responses import FileResponse, JSONResponse, PlainTextResponse, RedirectResponse, StreamingResponse from fastapi.staticfiles import StaticFiles from pydantic import BaseModel, Field from scqc import __version__ from datetime import date from scqc import ethics as E from scqc.asr import load_backend from scqc.audio import AudioDecodeError, ensure_wav from scqc.checks.universal import RecordingContext from scqc.manifest import Manifest, ManifestError, drive_direct_url, parse_manifest from scqc.pipeline import QCInput, run_qc from . import auth as AUTH from .db import DB from .mfa_worker import MFA DATA = Path(os.environ.get("SCQC_DATA", "./scqc_data")).resolve() AUDIO_DIR = DATA / "audio" ASSET_DIR = DATA / "assets" for d in (DATA, AUDIO_DIR, ASSET_DIR): d.mkdir(parents=True, exist_ok=True) ADMIN_TOKEN = os.environ.get("SCQC_ADMIN_TOKEN", "admin-dev") REVIEW_TOKEN = os.environ.get("SCQC_REVIEW_TOKEN", "review-dev") MAX_UPLOAD_BYTES = 25 * 1024 * 1024 db = DB(str(DATA / "scqc.sqlite")) AUTH.init(db) ASR_SPEC = os.environ.get("SCQC_ASR", "none") ASR_ERROR: Optional[str] = None try: ASR = load_backend(ASR_SPEC) except Exception as e: # keep the demo up: fall back to acoustic checks + human review ASR_ERROR = f"{type(e).__name__}: {e}"[:400] print(f"[scqc] ASR backend {ASR_SPEC!r} failed to load, running without ASR: {ASR_ERROR}", flush=True) ASR = load_backend("none") _manifests: dict[str, Manifest] = {} def _seed_examples() -> None: """SCQC_SEED_EXAMPLES=1 loads examples/manifest_*.json on start (for a hosted demo).""" ex = Path(__file__).resolve().parents[1] / "examples" for path in sorted(ex.glob("manifest_*.json")): m = parse_manifest(path.read_text(encoding="utf-8")) if db.one("SELECT 1 FROM projects WHERE project_id=?", (m.project_id,)): continue with db.tx() as c: key = E.approval_key(m) c.execute("INSERT INTO projects(project_id, manifest, created_at, ethics_key, ethics_scope)" " VALUES (?,?,?,?,?)", (m.project_id, m.model_dump_json(), time.time(), key, json.dumps(E.scope(m)) if key else None)) c.executemany("INSERT INTO tasks(project_id, task_id, ord, task_type, language) VALUES (?,?,?,?,?)", [(m.project_id, t.task_id, i, t.task_type, t.language) for i, t in enumerate(m.tasks)]) for aid, asset in m.assets.items(): local = ex / asset.url if local.is_file(): (ASSET_DIR / m.project_id).mkdir(parents=True, exist_ok=True) (ASSET_DIR / m.project_id / aid).write_bytes(local.read_bytes()) if os.environ.get("SCQC_SEED_EXAMPLES") == "1": _seed_examples() app = FastAPI(title="SCQC", version=__version__) STATIC = Path(__file__).parent / "static" app.mount("/static", StaticFiles(directory=STATIC), name="static") # ------------------------------------------------------------------ helpers def manifest(pid: str) -> Manifest: if pid not in _manifests: row = db.one("SELECT manifest FROM projects WHERE project_id=?", (pid,)) if not row: raise HTTPException(404, f"project '{pid}' not found") _manifests[pid] = parse_manifest(row["manifest"]) return _manifests[pid] def _identity(token: Optional[str]) -> Optional[dict]: """A logged-in account, or one of the shared deployment tokens (for scripts).""" if not token: return None if token == ADMIN_TOKEN: return {"username": None, "role": "admin", "shared": True} if token == REVIEW_TOKEN: return {"username": None, "role": "reviewer", "shared": True} u = AUTH.session_user(db, token) return {**u, "shared": False} if u else None def require_admin(x_scqc_token: Optional[str] = Header(default=None)) -> str: who = _identity(x_scqc_token) if not who or who["role"] != "admin": raise HTTPException(401, "admin login required") return who["username"] or "admin" def require_reviewer(x_scqc_token: Optional[str] = Header(default=None)) -> str: who = _identity(x_scqc_token) if not who: raise HTTPException(401, "reviewer login required") return who["username"] or "reviewer" def reviewer_identity(x_scqc_token: Optional[str] = Header(default=None)) -> dict: who = _identity(x_scqc_token) if not who: raise HTTPException(401, "reviewer login required") return who def _speaker(sid: str): sp = db.one("SELECT * FROM speakers WHERE speaker_id=?", (sid,)) if not sp: raise HTTPException(404, "unknown speaker; give consent first") if sp["withdrawn_at"]: raise HTTPException(410, "this speaker has withdrawn from the project") return sp def _qc_summary(qc: dict) -> dict: return {k: qc[k] for k in ("decision", "reasons", "route_notes", "feedback", "rubric", "required_reviewers")} _ACOUSTIC_VALIDATORS = {"integrity", "acoustic", "metadata"} _MATCH_VALIDATORS = ("read_speech_match", "command_match") def contributor_view(qc: dict, condition: Optional[str]) -> dict: """What the participant app is allowed to show, given the experiment condition. Reviewer-only findings (model uncertainty, human-only criteria) are never included. Word-level alignment is a *tentative* hint for read speech and commands, shown only after the utterance and only in the task-aware arm. """ if condition == "no_qc": return {"show": False, "retry_suggested": False, "messages": [], "words": None} visible = [f for f in qc["findings"] if f["contributor_visible"] and f["action"] != "info"] info = [f for f in qc["findings"] if f["contributor_visible"] and f["action"] == "info"] if condition == "acoustic_only": visible = [f for f in visible if f["validator"] in _ACOUSTIC_VALIDATORS] info = [f for f in info if f["validator"] in _ACOUSTIC_VALIDATORS] words = None if condition in (None, "task_aware"): m = next((v["data"] for v in qc["validators"] if v["name"] in _MATCH_VALIDATORS and v["data"]), None) if m and "alignment" in m: words = [{"op": o["op"], "ref": o["ref"], "hyp": o["hyp"]} for o in m["alignment"]] model_hint = any(f["model_based"] for f in visible) conf = qc.get("confidence") or {} return { "show": True, "confidence": {"score": conf.get("score"), "label": conf.get("label"), "calibrated": conf.get("calibrated")} if conf and condition != "acoustic_only" else None, "retry_suggested": qc["decision"] == "retry" or model_hint, "messages": [{"code": f["code"], "action": f["action"], "text": f["message"]} for f in visible], "notes": [{"code": f["code"], "text": f["message"]} for f in info], "words": words, } def _assign_condition(m: Manifest) -> Optional[str]: if not m.experiment: return None conds = m.experiment.conditions if m.experiment.assignment == "random": return random.Random().choice(conds) n = db.one("SELECT COUNT(*) AS n FROM speakers WHERE project_id=?", (m.project_id,))["n"] return conds[n % len(conds)] def _speaker_baseline(speaker_id: str, project_id: str, task_type: str) -> Optional[dict]: """Median loudness and pace of this speaker's earlier TTS takes (for the consistency check).""" if task_type != "tts_prosody": return None rows = db.q("SELECT r.qc FROM recordings r JOIN tasks t ON t.project_id=r.project_id AND t.task_id=r.task_id" " WHERE r.speaker_id=? AND r.project_id=? AND t.task_type='tts_prosody'" " AND r.status NOT IN ('superseded','withdrawn','blocked') ORDER BY r.created_at DESC LIMIT 30", (speaker_id, project_id)) levels, rates = [], [] for r in rows: qc = json.loads(r["qc"]) lv = (qc.get("signal") or {}).get("speech_level_db") if lv is not None: levels.append(lv) for v in qc.get("validators", []): if v["name"] == "consistency" and (v.get("data") or {}).get("rate_cps"): rates.append(v["data"]["rate_cps"]) if not levels: return None import statistics return {"n": len(levels), "level_db": statistics.median(levels), "rate_cps": statistics.median(rates) if rates else None} def _pay(conn, m: Manifest, rec) -> None: task = m.task(rec["task_id"]) p = m.payment_for(task) if p.mode == "none" or p.amount <= 0: return amount = p.amount if p.mode == "per_approved_minute": qc = json.loads(rec["qc"]) amount = round(p.amount * (qc.get("signal") or {}).get("speech_s", 0) / 60.0, 2) conn.execute("INSERT OR IGNORE INTO ledger(speaker_id, project_id, recording_id, amount, currency, created_at)" " VALUES (?,?,?,?,?,?)", (rec["speaker_id"], rec["project_id"], rec["recording_id"], amount, p.currency, time.time())) # ------------------------------------------------------------------ pages @app.get("/", include_in_schema=False) def participant_page(): return FileResponse(STATIC / "login.html") @app.get("/demo", include_in_schema=False) def public_demo_page(): """Public, read-only demo; no production API calls or research records.""" return FileResponse(STATIC / "demo.html") @app.get("/participant", include_in_schema=False) def participant_workspace(): return FileResponse(STATIC / "participant.html") @app.get("/review", include_in_schema=False) def review_page(): return FileResponse(STATIC / "admin.html") # role-scoped review workspace @app.get("/admin", include_in_schema=False) def admin_page(): return FileResponse(STATIC / "admin.html") @app.get("/sw.js", include_in_schema=False) def service_worker(): # Served from the root so its scope covers the whole app. return FileResponse(STATIC / "sw.js", media_type="application/javascript", headers={"Service-Worker-Allowed": "/"}) # ------------------------------------------------------------------ accounts class LoginIn(BaseModel): username: str = Field(min_length=1, max_length=40) password: str = Field(min_length=1, max_length=200) class UserIn(BaseModel): username: str = Field(min_length=1, max_length=40) password: str = Field(min_length=10, max_length=200) role: str = Field(pattern="^(admin|reviewer|participant)$") display_name: Optional[str] = Field(default=None, max_length=80) class UserPatch(BaseModel): password: Optional[str] = Field(default=None, min_length=10, max_length=200) role: Optional[str] = Field(default=None, pattern="^(admin|reviewer|participant)$") disabled: Optional[bool] = None display_name: Optional[str] = Field(default=None, max_length=80) @app.get("/api/auth/status") def auth_status(): return {"has_users": AUTH.has_users(db)} @app.post("/api/auth/login") def auth_login(body: LoginIn): try: r = AUTH.login(db, body.username, body.password) except PermissionError as e: raise HTTPException(429, str(e)) if not r: raise HTTPException(401, "wrong user name or password") return r @app.post("/api/auth/logout") def auth_logout(x_scqc_token: Optional[str] = Header(default=None)): if x_scqc_token: AUTH.logout(db, x_scqc_token) return {"ok": True} @app.get("/api/auth/me") def auth_me(who: dict = Depends(reviewer_identity)): return who @app.post("/api/auth/setup", status_code=201) def auth_setup(body: UserIn, x_scqc_token: Optional[str] = Header(default=None)): """Create the first admin account. Needs the deployment's SCQC_ADMIN_TOKEN.""" if AUTH.has_users(db): raise HTTPException(409, "accounts already exist; log in instead") if x_scqc_token != ADMIN_TOKEN: raise HTTPException(401, "enter the server's admin token (SCQC_ADMIN_TOKEN) to create the first account") try: AUTH.create_user(db, body.username, body.password, "admin", body.display_name) except ValueError as e: raise HTTPException(422, str(e)) return AUTH.login(db, body.username, body.password) @app.get("/api/users") def list_users(_: str = Depends(require_admin)): return [dict(r) for r in db.q("SELECT username, display_name, role, disabled, created_at FROM users ORDER BY created_at")] @app.post("/api/users", status_code=201) def add_user(body: UserIn, actor: str = Depends(require_admin)): if db.one("SELECT 1 FROM users WHERE username=?", (body.username,)): raise HTTPException(409, "that user name is taken") try: return AUTH.create_user(db, body.username, body.password, body.role, body.display_name) except ValueError as e: raise HTTPException(422, str(e)) @app.patch("/api/users/{username}") def update_user(username: str, body: UserPatch, actor: str = Depends(require_admin)): u = db.one("SELECT * FROM users WHERE username=?", (username,)) if not u: raise HTTPException(404) if username == actor and (body.disabled or body.role == "reviewer"): raise HTTPException(409, "you can't disable or demote your own account") with db.tx() as c: if body.password: h, salt = AUTH.hash_password(body.password) c.execute("UPDATE users SET pw_hash=?, salt=? WHERE username=?", (h, salt, username)) c.execute("DELETE FROM sessions WHERE username=?", (username,)) if body.role: c.execute("UPDATE users SET role=? WHERE username=?", (body.role, username)) if body.disabled is not None: c.execute("UPDATE users SET disabled=? WHERE username=?", (int(body.disabled), username)) if body.disabled: c.execute("DELETE FROM sessions WHERE username=?", (username,)) if body.display_name: c.execute("UPDATE users SET display_name=? WHERE username=?", (body.display_name, username)) db.log(c, actor, "user.updated", username, body.model_dump(exclude={"password"}, exclude_none=True)) return {"ok": True} @app.get("/api/schema") def manifest_schema(): return Manifest.model_json_schema() @app.get("/api/health") def health(): return {"ok": True, "version": __version__, "asr": {"backend": ASR.name, "model_id": ASR.model_id, "requested": ASR_SPEC, "error": ASR_ERROR, "load_errors": getattr(ASR, "load_errors", None) or None}, "mfa": {"enabled": MFA.available, "languages": sorted(MFA.models), "queued": MFA.q.qsize(), "last_error": MFA.last_error}, "default_tokens": ADMIN_TOKEN == "admin-dev" or REVIEW_TOKEN == "review-dev"} # ------------------------------------------------------------------ projects @app.post("/api/projects/validate") def validate_project(body: dict, _: str = Depends(require_admin)): """Dry run: check a manifest without saving it.""" try: m = parse_manifest(body) except ManifestError as e: return JSONResponse(status_code=422, content={"errors": e.errors}) counts: dict[str, int] = {} for t in m.tasks: counts[t.task_type] = counts.get(t.task_type, 0) + 1 exists = db.one("SELECT 1 FROM projects WHERE project_id=?", (m.project_id,)) is not None return {"project_id": m.project_id, "n_tasks": len(m.tasks), "task_types": counts, "languages": m.languages, "exists": exists, "experiment": bool(m.experiment)} @app.post("/api/projects", status_code=201) async def create_project(body: dict, fetch_assets: bool = False, replace: bool = False, actor: str = Depends(require_admin)): try: m = parse_manifest(body) except ManifestError as e: return JSONResponse(status_code=422, content={"errors": e.errors}) exists = db.one("SELECT ethics_key, ethics_scope FROM projects WHERE project_id=?", (m.project_id,)) if exists and not replace: raise HTTPException(409, "project exists; pass ?replace=true to update its manifest") key, new_scope = E.approval_key(m), E.scope(m) if exists and key and exists["ethics_key"] == key and exists["ethics_scope"]: changes = E.scope_changes(json.loads(exists["ethics_scope"]), new_scope) if changes: return JSONResponse(status_code=409, content={ "errors": [{"loc": ["ethics"], "msg": "These changes are outside the recorded ethics approval. " "Get an amendment approved and record it (ethics.amendment_id, and the new " "approval_date if it changed), or undo the changes: " + "; ".join(changes)}]}) with db.tx() as c: c.execute("INSERT OR REPLACE INTO projects(project_id, manifest, created_at, ethics_key, ethics_scope)" " VALUES (?,?,?,?,?)", (m.project_id, m.model_dump_json(), time.time(), key, json.dumps(new_scope) if key else None)) c.execute("DELETE FROM tasks WHERE project_id=?", (m.project_id,)) c.executemany("INSERT INTO tasks(project_id, task_id, ord, task_type, language) VALUES (?,?,?,?,?)", [(m.project_id, t.task_id, i, t.task_type, t.language) for i, t in enumerate(m.tasks)]) db.log(c, actor, "project.upsert", m.project_id, {"n_tasks": len(m.tasks), "ethics": E.export_view(m)}) _manifests[m.project_id] = m fetched = _fetch_assets(m) if fetch_assets else {} return {"project_id": m.project_id, "n_tasks": len(m.tasks), "assets": fetched} def _fetch_assets(m: Manifest) -> dict: """Resolve (Drive) asset links and cache them with checksum verification.""" out = {} d = ASSET_DIR / m.project_id d.mkdir(parents=True, exist_ok=True) for aid, a in m.assets.items(): try: with urllib.request.urlopen(drive_direct_url(a.url), timeout=30) as r: data = r.read(20 * 1024 * 1024) digest = hashlib.sha256(data).hexdigest() if a.sha256 and digest != a.sha256: out[aid] = "checksum_mismatch" continue (d / aid).write_bytes(data) out[aid] = "cached" except Exception as e: out[aid] = f"failed: {type(e).__name__}" return out @app.put("/api/projects/{pid}/assets/{aid}") async def upload_asset(pid: str, aid: str, file: UploadFile = File(...), actor: str = Depends(require_admin)): m = manifest(pid) if aid not in m.assets: raise HTTPException(404, "asset id not in manifest") data = await file.read() want = m.assets[aid].sha256 if want and hashlib.sha256(data).hexdigest() != want: raise HTTPException(422, "checksum does not match manifest") (ASSET_DIR / pid).mkdir(parents=True, exist_ok=True) (ASSET_DIR / pid / aid).write_bytes(data) return {"asset": aid, "bytes": len(data)} def recruitment_state(m: Manifest) -> tuple[bool, Optional[str]]: """Whether new participants may join, and if not, why (shown to them).""" e = m.ethics if not e.required or e.status == "internal_test": return True, None if e.status == "pending": return False, "This project is waiting for ethics committee approval and is not open yet." today = date.today() if today < e.approval_date: return False, f"This project opens on {e.approval_date.isoformat()}." if today > e.expiry_date: return False, "This project's ethics approval has expired, so it is closed." if e.max_participants: n = db.one("SELECT COUNT(*) AS n FROM speakers WHERE project_id=? AND COALESCE(is_test,0)=0", (m.project_id,))["n"] if n >= e.max_participants: return False, "This project has all the participants it needs. Thank you for your interest." return True, None def recording_allowed(m: Manifest) -> Optional[str]: """Existing participants may keep recording until approval expires.""" e = m.ethics if e.required and e.status == "pending": return "This project is waiting for ethics committee approval." if e.required and e.status == "approved" and date.today() > e.expiry_date: return "This project's ethics approval has expired, so it is closed." return None _retention_checked = 0.0 def enforce_retention(force: bool = False) -> int: """Delete audio older than each project's approved retention period (hourly at most).""" global _retention_checked if not force and time.time() - _retention_checked < 3600: return 0 _retention_checked = time.time() deleted = 0 for p in db.q("SELECT project_id FROM projects"): m = manifest(p["project_id"]) days = m.ethics.retention_days if not days: continue cutoff = time.time() - days * 86400 old = db.q("SELECT recording_id, audio_path FROM recordings WHERE project_id=? AND created_at float: """System Usability Scale: odd items positive, even items negative, 1–5 each, scaled to 0–100.""" return 2.5 * sum((x - 1) if i % 2 == 0 else (5 - x) for i, x in enumerate(a)) @app.post("/api/speakers/{sid}/survey", status_code=201) def submit_survey(sid: str, body: SurveyIn): sp = _speaker(sid) if any(x < 1 or x > 5 for x in body.answers): raise HTTPException(422, "answers are 1 to 5") score = sus_score(body.answers) with db.tx() as c: c.execute("INSERT OR REPLACE INTO surveys(speaker_id, project_id, instrument, answers, score, created_at)" " VALUES (?,?,?,?,?,?)", (sid, sp["project_id"], body.instrument, json.dumps(body.answers), score, time.time())) return {"score": score} # ------------------------------------------------------------------ simulated payouts @app.post("/api/projects/{pid}/payouts/batches", status_code=201) def create_payout_batch(pid: str, actor: str = Depends(require_admin)): """Collect every earned, unpaid entry into a payout batch (simulated; no money moves).""" manifest(pid) rows = db.q("SELECT entry_id, amount, currency FROM ledger WHERE project_id=? AND status='earned'", (pid,)) if not rows: raise HTTPException(409, "nothing to pay out") currencies = {r["currency"] for r in rows} bid = "PB-" + time.strftime("%Y%m%d") + "-" + uuid.uuid4().hex[:6].upper() with db.tx() as c: c.execute("INSERT INTO payout_batches(batch_id, project_id, created_at, created_by, status, n, total, currency)" " VALUES (?,?,?,?,?,?,?,?)", (bid, pid, time.time(), actor, "pending", len(rows), round(sum(r["amount"] for r in rows), 2), "/".join(sorted(currencies)))) c.executemany("UPDATE ledger SET status='pending', batch_id=? WHERE entry_id=?", [(bid, r["entry_id"]) for r in rows]) db.log(c, actor, "payout.batch_created", bid, {"n": len(rows)}) return {"batch_id": bid, "n": len(rows), "simulated": True} @app.post("/api/payouts/batches/{bid}/settle") def settle_payout_batch(bid: str, actor: str = Depends(require_admin)): """Mark a batch paid with simulated transaction references.""" b = db.one("SELECT * FROM payout_batches WHERE batch_id=?", (bid,)) if not b: raise HTTPException(404) if b["status"] == "paid": raise HTTPException(409, "batch already settled") now = time.time() rows = db.q("SELECT entry_id FROM ledger WHERE batch_id=?", (bid,)) with db.tx() as c: c.executemany("UPDATE ledger SET status='paid', paid_at=?, txn_id=? WHERE entry_id=?", [(now, "SIM-UPI-" + uuid.uuid4().hex[:10].upper(), r["entry_id"]) for r in rows]) c.execute("UPDATE payout_batches SET status='paid', settled_at=? WHERE batch_id=?", (now, bid)) db.log(c, actor, "payout.batch_settled", bid, {"n": len(rows), "simulated": True}) return {"batch_id": bid, "status": "paid", "n": len(rows), "simulated": True} @app.get("/api/projects/{pid}/payouts/batches") def list_payout_batches(pid: str, _: str = Depends(require_admin)): earned = db.one("SELECT COUNT(*) AS n, COALESCE(SUM(amount),0) AS total FROM ledger WHERE project_id=?" " AND status='earned'", (pid,)) return {"unbatched": dict(earned), "simulated": True, "batches": [dict(r) for r in db.q("SELECT * FROM payout_batches WHERE project_id=? ORDER BY created_at DESC", (pid,))]} # ------------------------------------------------------------------ dashboard @app.get("/api/projects/{pid}/dashboard") def dashboard(pid: str, days: int = Query(14, ge=1, le=120), _: str = Depends(require_reviewer)): m = manifest(pid) tasks = {t.task_id: t for t in m.tasks} # In an internal-test project every participant is a test participant, so count them. test_ok = int(m.ethics.required and m.ethics.status == "internal_test") contributors = db.one("SELECT COUNT(*) AS n FROM speakers WHERE project_id=? AND COALESCE(is_test,0)<=?" " AND withdrawn_at IS NULL", (pid, test_ok))["n"] recs = db.q("SELECT task_id, status, auto_decision, created_at, final_at FROM recordings WHERE project_id=?" " AND status NOT IN ('superseded','withdrawn')", (pid,)) paid = db.one("SELECT COALESCE(SUM(amount),0) AS total FROM ledger WHERE project_id=?", (pid,))["total"] now = time.time() day0 = time.mktime(time.localtime(now)[:3] + (0, 0, 0, 0, 0, -1)) - (days - 1) * 86400 daily = [] for i in range(days): a, b = day0 + i * 86400, day0 + (i + 1) * 86400 on_day = [r for r in recs if a <= r["created_at"] < b] decided = [r for r in on_day if r["status"] in ("accepted", "rejected")] daily.append({"date": time.strftime("%Y-%m-%d", time.localtime(a)), "recordings": len(on_day), "approved": sum(r["status"] == "accepted" for r in on_day), "approval_rate": round(sum(r["status"] == "accepted" for r in decided) / len(decided), 4) if decided else None}) def breakdown(key): out: dict[str, dict] = {} for r in recs: t = tasks.get(r["task_id"]) if not t: continue k = getattr(t, key) b = out.setdefault(k, {"recordings": 0, "approved": 0, "rejected": 0, "review": 0}) b["recordings"] += 1 b["approved"] += r["status"] == "accepted" b["rejected"] += r["status"] in ("rejected", "blocked") b["review"] += r["status"] in ("review", "checking") for b in out.values(): done = b["approved"] + b["rejected"] b["approval_rate"] = round(b["approved"] / done, 4) if done else None return out decided = [r for r in recs if r["status"] in ("accepted", "rejected", "blocked")] return { "project_id": pid, "title": m.title, "contributors": contributors, "recordings": len(recs), "approved": sum(r["status"] == "accepted" for r in recs), "pending_review": sum(r["status"] == "review" for r in recs), "checking": sum(r["status"] == "checking" for r in recs), "approval_rate": round(sum(r["status"] == "accepted" for r in decided) / len(decided), 4) if decided else None, "total_payout": round(paid, 2), "currency": m.defaults.payment.currency, "daily": daily, "by_language": breakdown("language"), "by_task_type": breakdown("task_type"), } # ------------------------------------------------------------------ user study @app.get("/api/projects/{pid}/study") def study_report(pid: str, _: str = Depends(require_admin)): """User-study results by condition and language: usable yield, attempts, whether blind listeners heard the intended meaning, agreement between listeners, agreement between the automatic prosody check and listeners, and System Usability Scale scores.""" import statistics from scqc.audit import fleiss_kappa m = manifest(pid) tasks = {t.task_id: t for t in m.tasks} sp = {r["speaker_id"]: r for r in db.q("SELECT * FROM speakers WHERE project_id=? AND COALESCE(is_test,0)=0" " AND withdrawn_at IS NULL", (pid,))} recs = db.q("SELECT * FROM recordings WHERE project_id=?", (pid,)) revs = db.q("SELECT v.* FROM reviews v JOIN recordings r USING (recording_id) WHERE r.project_id=?", (pid,)) surveys = {r["speaker_id"]: r["score"] for r in db.q("SELECT * FROM surveys WHERE project_id=?", (pid,))} perceived: dict[str, list[str]] = {} for v in revs: lab = json.loads(v["scores"] or "{}").get("_perceived") if lab: perceived.setdefault(v["recording_id"], []).append(lab) cells: dict[tuple, dict] = {} for sid, s_ in sp.items(): cell = cells.setdefault((s_["condition"] or "none", s_["language"] or "?"), { "participants": 0, "completed": 0, "attempts": 0, "kept": 0, "accepted": 0, "judgements": 0, "judged_correct": 0, "recordings_judged": 0, "majority_correct": 0, "auto_vs_listeners": [0, 0], "sus": []}) cell["participants"] += 1 mine = [r for r in recs if r["speaker_id"] == sid] kept = [r for r in mine if r["status"] not in ("superseded", "withdrawn")] lang_tasks = [t for t in m.tasks if t.language == s_["language"]] cell["completed"] += len({r["task_id"] for r in kept}) >= len(lang_tasks) > 0 cell["attempts"] += len(mine) cell["kept"] += len(kept) cell["accepted"] += sum(r["status"] == "accepted" for r in kept) for r in kept: t = tasks.get(r["task_id"]) labs = perceived.get(r["recording_id"], []) if not t or not labs or not t.prompt.interpretation: continue correct = [l == t.prompt.interpretation for l in labs] cell["judgements"] += len(labs) cell["judged_correct"] += sum(correct) cell["recordings_judged"] += 1 maj = sum(correct) > len(correct) / 2 cell["majority_correct"] += maj qc = json.loads(r["qc"]) auto = next((v["scores"].get("prosodic_realization") for v in qc.get("validators", []) if v["name"] == "prosody"), None) if auto is not None: cell["auto_vs_listeners"][0] += 1 cell["auto_vs_listeners"][1] += (auto >= 0.75) == maj if sid in surveys: cell["sus"].append(surveys[sid]) out = [] for (cond, lang), c in sorted(cells.items()): sus = c.pop("sus") n_auto, agree = c.pop("auto_vs_listeners") row = {"condition": cond, "language": lang, **c, "usable_yield": round(c["accepted"] / c["kept"], 4) if c["kept"] else None, "attempts_per_kept": round(c["attempts"] / c["kept"], 3) if c["kept"] else None, "listener_accuracy": round(c["judged_correct"] / c["judgements"], 4) if c["judgements"] else None, "majority_accuracy": round(c["majority_correct"] / c["recordings_judged"], 4) if c["recordings_judged"] else None, "auto_prosody_agreement": round(agree / n_auto, 4) if n_auto else None, "auto_prosody_n": n_auto, "sus_n": len(sus), "sus_mean": round(statistics.mean(sus), 2) if sus else None} if len(sus) >= 2: sd = statistics.stdev(sus) row["sus_ci95"] = [round(row["sus_mean"] - 1.96 * sd / len(sus) ** 0.5, 2), round(row["sus_mean"] + 1.96 * sd / len(sus) ** 0.5, 2)] out.append(row) return {"project_id": pid, "cells": out, "listener_fleiss_kappa": fleiss_kappa(perceived), "notes": "listener_accuracy: share of blind judgements naming the intended meaning; majority_accuracy: " "share of recordings whose majority judgement was right; auto_prosody_agreement: how often the " "automatic prosody check (pass at >= 0.75) agrees with the listener majority."} # ------------------------------------------------------------------ tasks @app.get("/api/projects/{pid}/next-task") def next_task(pid: str, speaker_id: str, language: Optional[str] = None, limit: int = Query(10, ge=1, le=100)): """Tasks this speaker can still record, least-collected first (for prefetching offline).""" m = manifest(pid) _speaker(speaker_id) mine = {r["task_id"]: r["n"] for r in db.q( "SELECT task_id, COUNT(*) AS n FROM recordings WHERE speaker_id=? AND project_id=?" " AND status IN ('review','accepted','checking') GROUP BY task_id", (speaker_id, pid))} totals = {r["task_id"]: r["n"] for r in db.q( "SELECT task_id, COUNT(*) AS n FROM recordings WHERE project_id=? AND status IN ('review','accepted')" " GROUP BY task_id", (pid,))} out = [] for t in m.tasks: if language and t.language != language: continue q = m.quota_for(t) if mine.get(t.task_id, 0) >= q.max_per_speaker: continue if q.max_recordings and totals.get(t.task_id, 0) >= q.max_recordings: continue out.append((totals.get(t.task_id, 0), len(out), t.task_id)) out.sort() ids = [tid for _, _, tid in out[:limit]] proj = get_project(pid) by_id = {t["task_id"]: t for t in proj["tasks"]} return {"tasks": [by_id[i] for i in ids], "remaining": len(out)} # ------------------------------------------------------------------ recordings @app.post("/api/recordings") async def upload_recording( project_id: str = Form(...), task_id: str = Form(...), speaker_id: str = Form(...), client_upload_id: str = Form(...), client_meta: str = Form("{}"), file: UploadFile = File(...), ): prev = db.one("SELECT recording_id, qc, status, condition FROM recordings WHERE client_upload_id=?", (client_upload_id,)) if prev: # idempotent retries from the offline sync queue return {"recording_id": prev["recording_id"], "status": prev["status"], "duplicate": True, "feedback": contributor_view(json.loads(prev["qc"]), prev["condition"])} m = manifest(project_id) blocked = recording_allowed(m) if blocked: raise HTTPException(403, blocked) enforce_retention() try: task = m.task(task_id) except KeyError: raise HTTPException(404, "unknown task") sp = _speaker(speaker_id) if sp["project_id"] != project_id: raise HTTPException(403, "speaker is registered for a different project") data = await file.read(MAX_UPLOAD_BYTES + 1) if len(data) > MAX_UPLOAD_BYTES: raise HTTPException(413, "recording too large") try: meta = json.loads(client_meta or "{}") except json.JSONDecodeError: meta = {"unparsed": client_meta[:500]} try: data = ensure_wav(data) except AudioDecodeError: pass # run_qc reports decode_failed and blocks the recording rid = "rec_" + uuid.uuid4().hex[:16] condition = sp["condition"] ctx = RecordingContext(task_id, task.task_type, task.language, speaker_id=speaker_id, consent_version=sp["consent_version"], project_consent_version=m.consent_version, declared_language=meta.get("language")) qc = run_qc(data, QCInput(task, m.validation_for(task), ctx, {k: v.model_dump() for k, v in m.intents.items()}, key=rid, speaker_baseline=_speaker_baseline(speaker_id, project_id, task.task_type)), ASR) decision = qc["decision"] status = decision if condition == "no_qc" and decision == "retry": status = "review" # the contributor was never offered a retry, so the recording is kept if status == "accepted" and MFA.eligible(task): status = "checking" # forced alignment runs in the background before the final decision path = None if "decode_failed" not in qc["reasons"]: path = AUDIO_DIR / project_id / f"{rid}.wav" path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(data) with db.tx() as c: attempt = 1 + c.execute("SELECT COUNT(*) FROM recordings WHERE speaker_id=? AND task_id=? AND project_id=?", (speaker_id, task_id, project_id)).fetchone()[0] c.execute("UPDATE recordings SET status='superseded', superseded_by=? WHERE speaker_id=? AND task_id=?" " AND project_id=? AND status='retry'", (rid, speaker_id, task_id, project_id)) c.execute("INSERT INTO recordings(recording_id, client_upload_id, project_id, task_id, speaker_id, attempt," " status, auto_decision, required_reviewers, audio_path, qc, client_meta, created_at, final_at," " condition) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", (rid, client_upload_id, project_id, task_id, speaker_id, attempt, status, decision, max(1, qc["required_reviewers"]) if status == "review" else qc["required_reviewers"], str(path) if path else None, json.dumps(qc, ensure_ascii=False), json.dumps(meta, ensure_ascii=False), time.time(), time.time() if status in ("accepted", "blocked") else None, condition)) if status == "accepted": _pay(c, m, c.execute("SELECT * FROM recordings WHERE recording_id=?", (rid,)).fetchone()) if status == "checking" or (path and MFA.eligible(task) and status == "review"): MFA.enqueue(rid) return {"recording_id": rid, "status": status, "attempt": attempt, "condition": condition, "feedback": contributor_view(qc, condition)} class KeepIn(BaseModel): speaker_id: str @app.post("/api/recordings/{rid}/keep") def keep_recording(rid: str, body: KeepIn): """Contributor chose to keep a recording that had retry guidance: send it to review.""" rec = db.one("SELECT * FROM recordings WHERE recording_id=?", (rid,)) if not rec or rec["speaker_id"] != body.speaker_id: raise HTTPException(404) if rec["status"] != "retry": return {"recording_id": rid, "status": rec["status"]} with db.tx() as c: c.execute("UPDATE recordings SET status='review', required_reviewers=MAX(required_reviewers,1)" " WHERE recording_id=?", (rid,)) return {"recording_id": rid, "status": "review"} # ------------------------------------------------------------------ review @app.get("/api/review/queue") def review_queue(project_id: str, reviewer_id: Optional[str] = None, limit: int = Query(20, ge=1, le=500), filter: str = Query("all", pattern="^(all|flagged|uncertain|high)$"), who: dict = Depends(reviewer_identity)): reviewer_id = who["username"] or reviewer_id if not reviewer_id: raise HTTPException(422, "reviewer_id is required with a shared token") rows = db.q( "SELECT r.* FROM recordings r WHERE r.project_id=? AND r.status='review' AND r.audio_path IS NOT NULL" " AND NOT EXISTS (SELECT 1 FROM reviews v WHERE v.recording_id=r.recording_id AND v.reviewer_id=?)" " ORDER BY r.created_at LIMIT 2000", (project_id, reviewer_id)) m = manifest(project_id) out = [] for r in rows: qc = json.loads(r["qc"]) t = m.task(r["task_id"]) match = next((v["data"] for v in qc["validators"] if v["name"] in ("read_speech_match", "command_match", "prosody_pair_lexical", "emotion_lexical")), None) n_reviews = db.one("SELECT COUNT(*) AS n FROM reviews WHERE recording_id=?", (r["recording_id"],))["n"] out.append({ "recording_id": r["recording_id"], "task_id": r["task_id"], "task_type": t.task_type, "language": t.language, "prompt": t.prompt.model_dump(exclude_none=True), "asset_url": f"/api/projects/{project_id}/assets/{t.prompt.image_asset_id}" if t.prompt.image_asset_id else None, "audio_url": f"/api/recordings/{r['recording_id']}/audio", "reasons": qc["reasons"], "route_notes": qc["route_notes"], "findings": [f for f in qc["findings"] if f["action"] != "info"], "rubric": qc["rubric"], "match": match, "signal": qc["signal"], "reviews_so_far": n_reviews, "required_reviewers": r["required_reviewers"], # Blind review for perceptual tasks: hide ASR text and auto reasons. "blind": t.task_type in ("prosody_pair", "emotion"), "confidence": qc.get("confidence"), "auto_decision": r["auto_decision"], "created_at": r["created_at"], "validators": [ {"name": v["name"], "data": {k: v["data"][k] for k in ("pauses", "pitch_range_st", "final_slope_st_per_s", "pauses_found", "targets", "suggested_coverage", "per_model", "cer_between", "aligned", "ll_z") if k in (v.get("data") or {})}} for v in qc["validators"] if v["name"] in ("prosody", "image_relevance", "mfa_alignment")], **_priority(qc, r, m.validation_for(t)), }) if filter != "all": out = [i for i in out if (filter == "high" and i["priority"] == "high") or (filter in ("flagged", "uncertain") and filter in i["categories"])] rank = {"high": 0, "medium": 1, "low": 2} out.sort(key=lambda i: (rank[i["priority"]], i["created_at"])) out = out[:limit] for item in out: if item["blind"]: item["match"], item["reasons"], item["findings"] = None, [], [] item["prompt"].pop("emotion", None) item["prompt"].pop("interpretation", None) item["perceived_options"] = _perceived_options(m, m.task(item["task_id"]), item["recording_id"]) return out HIGH_REASONS = {"possible_mismatch", "command_mismatch", "confusable_intent", "mfa_poor_fit", "clipping", "cut_off_end", "cut_off_start", "no_speech", "too_short"} def _priority(qc: dict, rec, v) -> dict: """Review priority: likely defects first, model uncertainty next, routine checks last.""" conf = (qc.get("confidence") or {}).get("score") reasons = set(qc.get("reasons") or []) flagged = bool(reasons - {"needs_human_relevance", "blind_listening_required", "asr_unavailable"}) codes = {f["code"] for f in qc.get("findings", [])} uncertain = (conf is not None and conf < (v.accept_confidence or 0.8)) or "asr_models_disagree" in codes \ or "asr_unavailable" in reasons or "low_asr_confidence" in reasons if (reasons & HIGH_REASONS) or rec["auto_decision"] == "retry" or (conf is not None and conf < 0.5): pr = "high" elif flagged or uncertain: pr = "medium" else: pr = "low" return {"priority": pr, "categories": [c for c, ok in (("flagged", flagged), ("uncertain", uncertain)) if ok]} def _perceived_options(m: Manifest, t, rid: str) -> list[str]: """Choices for a blind listener, in an order that does not reveal the target.""" if t.task_type == "prosody_pair": opts = [x.prompt.interpretation for x in m.tasks if x.task_type == "prosody_pair" and x.prompt.pair_id == t.prompt.pair_id] else: opts = sorted({x.prompt.emotion for x in m.tasks if x.task_type == "emotion" and x.prompt.emotion}) opts = [o for o in opts if o] random.Random(rid).shuffle(opts) # stable per recording, unrelated to the target return opts + ["Unclear"] @app.get("/api/recordings/{rid}/audio") def recording_audio(rid: str, token: Optional[str] = None, x_scqc_token: Optional[str] = Header(default=None)): #