SCQC / server /app.py
raianand's picture
Upload folder using huggingface_hub
ea934f7 verified
Raw History Blame Contribute Delete
65.6 kB
"""SCQC collection server. Roles are token-based for the demo: admin (manifests, export, payouts, reports) and reviewer (queue, audio, verdicts). Participants are identified only by a pseudonymous speaker_id issued at consent time. Environment SCQC_DATA data directory (default ./scqc_data) SCQC_ASR ASR backend spec, see scqc/asr.py (default: none) SCQC_ADMIN_TOKEN admin token (default "admin-dev"; change it) SCQC_REVIEW_TOKEN reviewer token (default "review-dev"; change it) """ from __future__ import annotations import hashlib import io import random import json import os import time import urllib.request import uuid import zipfile from pathlib import Path from typing import Optional from fastapi import Depends, FastAPI, File, Form, Header, HTTPException, Query, UploadFile from fastapi.responses import FileResponse, JSONResponse, PlainTextResponse, RedirectResponse, StreamingResponse from fastapi.staticfiles import StaticFiles from pydantic import BaseModel, Field from scqc import __version__ from datetime import date from scqc import ethics as E from scqc.asr import load_backend from scqc.audio import AudioDecodeError, ensure_wav from scqc.checks.universal import RecordingContext from scqc.manifest import Manifest, ManifestError, drive_direct_url, parse_manifest from scqc.pipeline import QCInput, run_qc from . import auth as AUTH from .db import DB from .mfa_worker import MFA DATA = Path(os.environ.get("SCQC_DATA", "./scqc_data")).resolve() AUDIO_DIR = DATA / "audio" ASSET_DIR = DATA / "assets" for d in (DATA, AUDIO_DIR, ASSET_DIR): d.mkdir(parents=True, exist_ok=True) ADMIN_TOKEN = os.environ.get("SCQC_ADMIN_TOKEN", "admin-dev") REVIEW_TOKEN = os.environ.get("SCQC_REVIEW_TOKEN", "review-dev") MAX_UPLOAD_BYTES = 25 * 1024 * 1024 db = DB(str(DATA / "scqc.sqlite")) AUTH.init(db) ASR_SPEC = os.environ.get("SCQC_ASR", "none") ASR_ERROR: Optional[str] = None try: ASR = load_backend(ASR_SPEC) except Exception as e: # keep the demo up: fall back to acoustic checks + human review ASR_ERROR = f"{type(e).__name__}: {e}"[:400] print(f"[scqc] ASR backend {ASR_SPEC!r} failed to load, running without ASR: {ASR_ERROR}", flush=True) ASR = load_backend("none") _manifests: dict[str, Manifest] = {} def _seed_examples() -> None: """SCQC_SEED_EXAMPLES=1 loads examples/manifest_*.json on start (for a hosted demo).""" ex = Path(__file__).resolve().parents[1] / "examples" for path in sorted(ex.glob("manifest_*.json")): m = parse_manifest(path.read_text(encoding="utf-8")) if db.one("SELECT 1 FROM projects WHERE project_id=?", (m.project_id,)): continue with db.tx() as c: key = E.approval_key(m) c.execute("INSERT INTO projects(project_id, manifest, created_at, ethics_key, ethics_scope)" " VALUES (?,?,?,?,?)", (m.project_id, m.model_dump_json(), time.time(), key, json.dumps(E.scope(m)) if key else None)) c.executemany("INSERT INTO tasks(project_id, task_id, ord, task_type, language) VALUES (?,?,?,?,?)", [(m.project_id, t.task_id, i, t.task_type, t.language) for i, t in enumerate(m.tasks)]) for aid, asset in m.assets.items(): local = ex / asset.url if local.is_file(): (ASSET_DIR / m.project_id).mkdir(parents=True, exist_ok=True) (ASSET_DIR / m.project_id / aid).write_bytes(local.read_bytes()) if os.environ.get("SCQC_SEED_EXAMPLES") == "1": _seed_examples() app = FastAPI(title="SCQC", version=__version__) STATIC = Path(__file__).parent / "static" app.mount("/static", StaticFiles(directory=STATIC), name="static") # ------------------------------------------------------------------ helpers def manifest(pid: str) -> Manifest: if pid not in _manifests: row = db.one("SELECT manifest FROM projects WHERE project_id=?", (pid,)) if not row: raise HTTPException(404, f"project '{pid}' not found") _manifests[pid] = parse_manifest(row["manifest"]) return _manifests[pid] def _identity(token: Optional[str]) -> Optional[dict]: """A logged-in account, or one of the shared deployment tokens (for scripts).""" if not token: return None if token == ADMIN_TOKEN: return {"username": None, "role": "admin", "shared": True} if token == REVIEW_TOKEN: return {"username": None, "role": "reviewer", "shared": True} u = AUTH.session_user(db, token) return {**u, "shared": False} if u else None def require_admin(x_scqc_token: Optional[str] = Header(default=None)) -> str: who = _identity(x_scqc_token) if not who or who["role"] != "admin": raise HTTPException(401, "admin login required") return who["username"] or "admin" def require_reviewer(x_scqc_token: Optional[str] = Header(default=None)) -> str: who = _identity(x_scqc_token) if not who: raise HTTPException(401, "reviewer login required") return who["username"] or "reviewer" def reviewer_identity(x_scqc_token: Optional[str] = Header(default=None)) -> dict: who = _identity(x_scqc_token) if not who: raise HTTPException(401, "reviewer login required") return who def _speaker(sid: str): sp = db.one("SELECT * FROM speakers WHERE speaker_id=?", (sid,)) if not sp: raise HTTPException(404, "unknown speaker; give consent first") if sp["withdrawn_at"]: raise HTTPException(410, "this speaker has withdrawn from the project") return sp def _qc_summary(qc: dict) -> dict: return {k: qc[k] for k in ("decision", "reasons", "route_notes", "feedback", "rubric", "required_reviewers")} _ACOUSTIC_VALIDATORS = {"integrity", "acoustic", "metadata"} _MATCH_VALIDATORS = ("read_speech_match", "command_match") def contributor_view(qc: dict, condition: Optional[str]) -> dict: """What the participant app is allowed to show, given the experiment condition. Reviewer-only findings (model uncertainty, human-only criteria) are never included. Word-level alignment is a *tentative* hint for read speech and commands, shown only after the utterance and only in the task-aware arm. """ if condition == "no_qc": return {"show": False, "retry_suggested": False, "messages": [], "words": None} visible = [f for f in qc["findings"] if f["contributor_visible"] and f["action"] != "info"] info = [f for f in qc["findings"] if f["contributor_visible"] and f["action"] == "info"] if condition == "acoustic_only": visible = [f for f in visible if f["validator"] in _ACOUSTIC_VALIDATORS] info = [f for f in info if f["validator"] in _ACOUSTIC_VALIDATORS] words = None if condition in (None, "task_aware"): m = next((v["data"] for v in qc["validators"] if v["name"] in _MATCH_VALIDATORS and v["data"]), None) if m and "alignment" in m: words = [{"op": o["op"], "ref": o["ref"], "hyp": o["hyp"]} for o in m["alignment"]] model_hint = any(f["model_based"] for f in visible) conf = qc.get("confidence") or {} return { "show": True, "confidence": {"score": conf.get("score"), "label": conf.get("label"), "calibrated": conf.get("calibrated")} if conf and condition != "acoustic_only" else None, "retry_suggested": qc["decision"] == "retry" or model_hint, "messages": [{"code": f["code"], "action": f["action"], "text": f["message"]} for f in visible], "notes": [{"code": f["code"], "text": f["message"]} for f in info], "words": words, } def _assign_condition(m: Manifest) -> Optional[str]: if not m.experiment: return None conds = m.experiment.conditions if m.experiment.assignment == "random": return random.Random().choice(conds) n = db.one("SELECT COUNT(*) AS n FROM speakers WHERE project_id=?", (m.project_id,))["n"] return conds[n % len(conds)] def _speaker_baseline(speaker_id: str, project_id: str, task_type: str) -> Optional[dict]: """Median loudness and pace of this speaker's earlier TTS takes (for the consistency check).""" if task_type != "tts_prosody": return None rows = db.q("SELECT r.qc FROM recordings r JOIN tasks t ON t.project_id=r.project_id AND t.task_id=r.task_id" " WHERE r.speaker_id=? AND r.project_id=? AND t.task_type='tts_prosody'" " AND r.status NOT IN ('superseded','withdrawn','blocked') ORDER BY r.created_at DESC LIMIT 30", (speaker_id, project_id)) levels, rates = [], [] for r in rows: qc = json.loads(r["qc"]) lv = (qc.get("signal") or {}).get("speech_level_db") if lv is not None: levels.append(lv) for v in qc.get("validators", []): if v["name"] == "consistency" and (v.get("data") or {}).get("rate_cps"): rates.append(v["data"]["rate_cps"]) if not levels: return None import statistics return {"n": len(levels), "level_db": statistics.median(levels), "rate_cps": statistics.median(rates) if rates else None} def _pay(conn, m: Manifest, rec) -> None: task = m.task(rec["task_id"]) p = m.payment_for(task) if p.mode == "none" or p.amount <= 0: return amount = p.amount if p.mode == "per_approved_minute": qc = json.loads(rec["qc"]) amount = round(p.amount * (qc.get("signal") or {}).get("speech_s", 0) / 60.0, 2) conn.execute("INSERT OR IGNORE INTO ledger(speaker_id, project_id, recording_id, amount, currency, created_at)" " VALUES (?,?,?,?,?,?)", (rec["speaker_id"], rec["project_id"], rec["recording_id"], amount, p.currency, time.time())) # ------------------------------------------------------------------ pages @app.get("/", include_in_schema=False) def participant_page(): return FileResponse(STATIC / "login.html") @app.get("/demo", include_in_schema=False) def public_demo_page(): """Public, read-only demo; no production API calls or research records.""" return FileResponse(STATIC / "demo.html") @app.get("/participant", include_in_schema=False) def participant_workspace(): return FileResponse(STATIC / "participant.html") @app.get("/review", include_in_schema=False) def review_page(): return FileResponse(STATIC / "admin.html") # role-scoped review workspace @app.get("/admin", include_in_schema=False) def admin_page(): return FileResponse(STATIC / "admin.html") @app.get("/sw.js", include_in_schema=False) def service_worker(): # Served from the root so its scope covers the whole app. return FileResponse(STATIC / "sw.js", media_type="application/javascript", headers={"Service-Worker-Allowed": "/"}) # ------------------------------------------------------------------ accounts class LoginIn(BaseModel): username: str = Field(min_length=1, max_length=40) password: str = Field(min_length=1, max_length=200) class UserIn(BaseModel): username: str = Field(min_length=1, max_length=40) password: str = Field(min_length=10, max_length=200) role: str = Field(pattern="^(admin|reviewer|participant)$") display_name: Optional[str] = Field(default=None, max_length=80) class UserPatch(BaseModel): password: Optional[str] = Field(default=None, min_length=10, max_length=200) role: Optional[str] = Field(default=None, pattern="^(admin|reviewer|participant)$") disabled: Optional[bool] = None display_name: Optional[str] = Field(default=None, max_length=80) @app.get("/api/auth/status") def auth_status(): return {"has_users": AUTH.has_users(db)} @app.post("/api/auth/login") def auth_login(body: LoginIn): try: r = AUTH.login(db, body.username, body.password) except PermissionError as e: raise HTTPException(429, str(e)) if not r: raise HTTPException(401, "wrong user name or password") return r @app.post("/api/auth/logout") def auth_logout(x_scqc_token: Optional[str] = Header(default=None)): if x_scqc_token: AUTH.logout(db, x_scqc_token) return {"ok": True} @app.get("/api/auth/me") def auth_me(who: dict = Depends(reviewer_identity)): return who @app.post("/api/auth/setup", status_code=201) def auth_setup(body: UserIn, x_scqc_token: Optional[str] = Header(default=None)): """Create the first admin account. Needs the deployment's SCQC_ADMIN_TOKEN.""" if AUTH.has_users(db): raise HTTPException(409, "accounts already exist; log in instead") if x_scqc_token != ADMIN_TOKEN: raise HTTPException(401, "enter the server's admin token (SCQC_ADMIN_TOKEN) to create the first account") try: AUTH.create_user(db, body.username, body.password, "admin", body.display_name) except ValueError as e: raise HTTPException(422, str(e)) return AUTH.login(db, body.username, body.password) @app.get("/api/users") def list_users(_: str = Depends(require_admin)): return [dict(r) for r in db.q("SELECT username, display_name, role, disabled, created_at FROM users ORDER BY created_at")] @app.post("/api/users", status_code=201) def add_user(body: UserIn, actor: str = Depends(require_admin)): if db.one("SELECT 1 FROM users WHERE username=?", (body.username,)): raise HTTPException(409, "that user name is taken") try: return AUTH.create_user(db, body.username, body.password, body.role, body.display_name) except ValueError as e: raise HTTPException(422, str(e)) @app.patch("/api/users/{username}") def update_user(username: str, body: UserPatch, actor: str = Depends(require_admin)): u = db.one("SELECT * FROM users WHERE username=?", (username,)) if not u: raise HTTPException(404) if username == actor and (body.disabled or body.role == "reviewer"): raise HTTPException(409, "you can't disable or demote your own account") with db.tx() as c: if body.password: h, salt = AUTH.hash_password(body.password) c.execute("UPDATE users SET pw_hash=?, salt=? WHERE username=?", (h, salt, username)) c.execute("DELETE FROM sessions WHERE username=?", (username,)) if body.role: c.execute("UPDATE users SET role=? WHERE username=?", (body.role, username)) if body.disabled is not None: c.execute("UPDATE users SET disabled=? WHERE username=?", (int(body.disabled), username)) if body.disabled: c.execute("DELETE FROM sessions WHERE username=?", (username,)) if body.display_name: c.execute("UPDATE users SET display_name=? WHERE username=?", (body.display_name, username)) db.log(c, actor, "user.updated", username, body.model_dump(exclude={"password"}, exclude_none=True)) return {"ok": True} @app.get("/api/schema") def manifest_schema(): return Manifest.model_json_schema() @app.get("/api/health") def health(): return {"ok": True, "version": __version__, "asr": {"backend": ASR.name, "model_id": ASR.model_id, "requested": ASR_SPEC, "error": ASR_ERROR, "load_errors": getattr(ASR, "load_errors", None) or None}, "mfa": {"enabled": MFA.available, "languages": sorted(MFA.models), "queued": MFA.q.qsize(), "last_error": MFA.last_error}, "default_tokens": ADMIN_TOKEN == "admin-dev" or REVIEW_TOKEN == "review-dev"} # ------------------------------------------------------------------ projects @app.post("/api/projects/validate") def validate_project(body: dict, _: str = Depends(require_admin)): """Dry run: check a manifest without saving it.""" try: m = parse_manifest(body) except ManifestError as e: return JSONResponse(status_code=422, content={"errors": e.errors}) counts: dict[str, int] = {} for t in m.tasks: counts[t.task_type] = counts.get(t.task_type, 0) + 1 exists = db.one("SELECT 1 FROM projects WHERE project_id=?", (m.project_id,)) is not None return {"project_id": m.project_id, "n_tasks": len(m.tasks), "task_types": counts, "languages": m.languages, "exists": exists, "experiment": bool(m.experiment)} @app.post("/api/projects", status_code=201) async def create_project(body: dict, fetch_assets: bool = False, replace: bool = False, actor: str = Depends(require_admin)): try: m = parse_manifest(body) except ManifestError as e: return JSONResponse(status_code=422, content={"errors": e.errors}) exists = db.one("SELECT ethics_key, ethics_scope FROM projects WHERE project_id=?", (m.project_id,)) if exists and not replace: raise HTTPException(409, "project exists; pass ?replace=true to update its manifest") key, new_scope = E.approval_key(m), E.scope(m) if exists and key and exists["ethics_key"] == key and exists["ethics_scope"]: changes = E.scope_changes(json.loads(exists["ethics_scope"]), new_scope) if changes: return JSONResponse(status_code=409, content={ "errors": [{"loc": ["ethics"], "msg": "These changes are outside the recorded ethics approval. " "Get an amendment approved and record it (ethics.amendment_id, and the new " "approval_date if it changed), or undo the changes: " + "; ".join(changes)}]}) with db.tx() as c: c.execute("INSERT OR REPLACE INTO projects(project_id, manifest, created_at, ethics_key, ethics_scope)" " VALUES (?,?,?,?,?)", (m.project_id, m.model_dump_json(), time.time(), key, json.dumps(new_scope) if key else None)) c.execute("DELETE FROM tasks WHERE project_id=?", (m.project_id,)) c.executemany("INSERT INTO tasks(project_id, task_id, ord, task_type, language) VALUES (?,?,?,?,?)", [(m.project_id, t.task_id, i, t.task_type, t.language) for i, t in enumerate(m.tasks)]) db.log(c, actor, "project.upsert", m.project_id, {"n_tasks": len(m.tasks), "ethics": E.export_view(m)}) _manifests[m.project_id] = m fetched = _fetch_assets(m) if fetch_assets else {} return {"project_id": m.project_id, "n_tasks": len(m.tasks), "assets": fetched} def _fetch_assets(m: Manifest) -> dict: """Resolve (Drive) asset links and cache them with checksum verification.""" out = {} d = ASSET_DIR / m.project_id d.mkdir(parents=True, exist_ok=True) for aid, a in m.assets.items(): try: with urllib.request.urlopen(drive_direct_url(a.url), timeout=30) as r: data = r.read(20 * 1024 * 1024) digest = hashlib.sha256(data).hexdigest() if a.sha256 and digest != a.sha256: out[aid] = "checksum_mismatch" continue (d / aid).write_bytes(data) out[aid] = "cached" except Exception as e: out[aid] = f"failed: {type(e).__name__}" return out @app.put("/api/projects/{pid}/assets/{aid}") async def upload_asset(pid: str, aid: str, file: UploadFile = File(...), actor: str = Depends(require_admin)): m = manifest(pid) if aid not in m.assets: raise HTTPException(404, "asset id not in manifest") data = await file.read() want = m.assets[aid].sha256 if want and hashlib.sha256(data).hexdigest() != want: raise HTTPException(422, "checksum does not match manifest") (ASSET_DIR / pid).mkdir(parents=True, exist_ok=True) (ASSET_DIR / pid / aid).write_bytes(data) return {"asset": aid, "bytes": len(data)} def recruitment_state(m: Manifest) -> tuple[bool, Optional[str]]: """Whether new participants may join, and if not, why (shown to them).""" e = m.ethics if not e.required or e.status == "internal_test": return True, None if e.status == "pending": return False, "This project is waiting for ethics committee approval and is not open yet." today = date.today() if today < e.approval_date: return False, f"This project opens on {e.approval_date.isoformat()}." if today > e.expiry_date: return False, "This project's ethics approval has expired, so it is closed." if e.max_participants: n = db.one("SELECT COUNT(*) AS n FROM speakers WHERE project_id=? AND COALESCE(is_test,0)=0", (m.project_id,))["n"] if n >= e.max_participants: return False, "This project has all the participants it needs. Thank you for your interest." return True, None def recording_allowed(m: Manifest) -> Optional[str]: """Existing participants may keep recording until approval expires.""" e = m.ethics if e.required and e.status == "pending": return "This project is waiting for ethics committee approval." if e.required and e.status == "approved" and date.today() > e.expiry_date: return "This project's ethics approval has expired, so it is closed." return None _retention_checked = 0.0 def enforce_retention(force: bool = False) -> int: """Delete audio older than each project's approved retention period (hourly at most).""" global _retention_checked if not force and time.time() - _retention_checked < 3600: return 0 _retention_checked = time.time() deleted = 0 for p in db.q("SELECT project_id FROM projects"): m = manifest(p["project_id"]) days = m.ethics.retention_days if not days: continue cutoff = time.time() - days * 86400 old = db.q("SELECT recording_id, audio_path FROM recordings WHERE project_id=? AND created_at<?" " AND audio_path IS NOT NULL", (m.project_id, cutoff)) for r in old: if Path(r["audio_path"]).exists(): Path(r["audio_path"]).unlink() if old: with db.tx() as c: c.executemany("UPDATE recordings SET audio_path=NULL WHERE recording_id=?", [(r["recording_id"],) for r in old]) db.log(c, "system", "retention.deleted", m.project_id, {"n": len(old), "days": days}) deleted += len(old) return deleted @app.get("/api/projects") def list_projects(): rows = db.q("SELECT project_id, manifest, created_at FROM projects ORDER BY created_at DESC") out = [] for r in rows: m = json.loads(r["manifest"]) mm = manifest(r["project_id"]) is_open, why = recruitment_state(mm) out.append({"project_id": r["project_id"], "title": m.get("title"), "languages": m["languages"], "n_tasks": len(m["tasks"]), "consent_version": m["consent_version"], "ethics_status": E.public_view(mm)["status"], "open": is_open, "closed_reason": why}) return out @app.get("/api/projects/{pid}") def get_project(pid: str): """Participant-safe view: prompts and assets, no hidden targets (e.g. emotion).""" m = manifest(pid) tasks = [] for t in m.tasks: p = t.prompt.model_dump(exclude_none=True) p.pop("emotion", None) # never steer contributors toward a predicted label tasks.append({"task_id": t.task_id, "task_type": t.task_type, "language": t.language, "prompt": p, "level": m.validation_for(t).preset}) is_open, why = recruitment_state(m) return {"project_id": m.project_id, "title": m.title, "languages": m.languages, "consent_version": m.consent_version, "ethics": E.public_view(m), "open": is_open, "closed_reason": why, "assets": {k: {"alt": a.alt, "url": f"/api/projects/{pid}/assets/{k}"} for k, a in m.assets.items() if k != m.ethics.approval_document}, "tasks": tasks} @app.get("/api/projects/{pid}/manifest") def get_manifest(pid: str, _: str = Depends(require_admin)): """Full manifest, including hidden targets; admin only.""" return json.loads(manifest(pid).model_dump_json(exclude_none=True)) @app.get("/api/projects/{pid}/assets/{aid}") def get_asset(pid: str, aid: str, token: Optional[str] = None, x_scqc_token: Optional[str] = Header(default=None)): m = manifest(pid) if aid not in m.assets: raise HTTPException(404) if aid == m.ethics.approval_document and (_identity(x_scqc_token or token) or {}).get("role") != "admin": raise HTTPException(401, "the ethics approval document is visible to admins only") local = ASSET_DIR / pid / aid if local.exists(): return FileResponse(local, media_type=m.assets[aid].mime or None, headers={"Cache-Control": "public, max-age=604800"}) return RedirectResponse(drive_direct_url(m.assets[aid].url)) # ------------------------------------------------------------------ speakers class ConsentIn(BaseModel): project_id: str consent_version: str consent_sha: Optional[str] = None # fingerprint of the consent text shown (from the project view) age_confirmed: bool = False language: Optional[str] = None gender: Optional[str] = Field(default=None, max_length=32) age_group: Optional[str] = Field(default=None, max_length=32) region: Optional[str] = Field(default=None, max_length=64) @app.post("/api/speakers", status_code=201) def register_speaker(body: ConsentIn): m = manifest(body.project_id) if body.consent_version != m.consent_version: raise HTTPException(409, f"current consent version is {m.consent_version}") if body.language and body.language not in m.languages: raise HTTPException(422, f"language must be one of {m.languages}") is_open, why = recruitment_state(m) if not is_open: raise HTTPException(403, why) if m.ethics.minimum_age and not body.age_confirmed: raise HTTPException(422, f"participants must confirm they are {m.ethics.minimum_age} or older") want = E.consent_sha(m, body.language or m.languages[0]) if body.consent_sha != want: raise HTTPException(409, "the consent text has changed; reload the page and read it again") is_test = int(m.ethics.required and m.ethics.status == "internal_test") sid = "spk_" + uuid.uuid4().hex[:12] condition = _assign_condition(m) with db.tx() as c: c.execute("INSERT INTO speakers(speaker_id, project_id, consent_version, consented_at, language, gender," " age_group, region, condition, consent_sha, is_test) VALUES (?,?,?,?,?,?,?,?,?,?,?)", (sid, body.project_id, body.consent_version, time.time(), body.language, body.gender, body.age_group, body.region, condition, want, is_test)) db.log(c, sid, "consent.given", body.project_id, {"version": body.consent_version, "consent_sha": want, "condition": condition, "test": bool(is_test)}) return {"speaker_id": sid, "condition": condition, "test": bool(is_test)} @app.delete("/api/speakers/{sid}") def withdraw_speaker(sid: str): """Deletion request: removes audio, keeps earned ledger entries so contributors still get paid.""" _speaker(sid) recs = db.q("SELECT recording_id, audio_path FROM recordings WHERE speaker_id=?", (sid,)) for r in recs: if r["audio_path"] and Path(r["audio_path"]).exists(): Path(r["audio_path"]).unlink() with db.tx() as c: c.execute("UPDATE recordings SET status='withdrawn', audio_path=NULL WHERE speaker_id=?", (sid,)) c.execute("UPDATE speakers SET withdrawn_at=? WHERE speaker_id=?", (time.time(), sid)) db.log(c, sid, "consent.withdrawn", sid, {"recordings_deleted": len(recs)}) return {"withdrawn": sid, "recordings_deleted": len(recs)} @app.get("/api/speakers/{sid}/history") def speaker_history(sid: str): _speaker(sid) rows = db.q("SELECT recording_id, task_id, attempt, status, created_at FROM recordings WHERE speaker_id=?" " ORDER BY created_at DESC", (sid,)) return [dict(r) for r in rows] @app.get("/api/speakers/{sid}/wallet") def wallet(sid: str): _speaker(sid) rows = db.q("SELECT currency, status, SUM(amount) AS total, COUNT(*) AS n FROM ledger WHERE speaker_id=?" " GROUP BY currency, status", (sid,)) tx = db.q("SELECT l.recording_id, r.task_id, l.amount, l.currency, l.status, l.txn_id, l.batch_id," " l.created_at, l.paid_at FROM ledger l LEFT JOIN recordings r USING (recording_id)" " WHERE l.speaker_id=? ORDER BY l.created_at DESC LIMIT 200", (sid,)) return {"speaker_id": sid, "balances": [dict(r) for r in rows], "transactions": [dict(r) for r in tx], "simulated_payments": True} @app.get("/api/speakers/{sid}/profile") def speaker_profile(sid: str): sp = _speaker(sid) m = manifest(sp["project_id"]) counts = {r["status"]: r["n"] for r in db.q("SELECT status, COUNT(*) AS n FROM recordings WHERE speaker_id=?" " AND status NOT IN ('superseded') GROUP BY status", (sid,))} survey = db.one("SELECT instrument, score FROM surveys WHERE speaker_id=?", (sid,)) return {"speaker_id": sid, "project_id": sp["project_id"], "project_title": m.title, "language": sp["language"], "joined_at": sp["consented_at"], "consent_version": sp["consent_version"], "condition": sp["condition"], "test": bool(sp["is_test"]) if "is_test" in sp.keys() else False, "tasks_total": sum(1 for t in m.tasks if not sp["language"] or t.language == sp["language"]), "recordings": counts, "survey_done": bool(survey), "survey": m.experiment.survey if m.experiment else None} @app.get("/api/speakers/{sid}/recordings/{rid}/audio") def speaker_audio(sid: str, rid: str): """A contributor can play back their own recordings (the speaker id is their key).""" _speaker(sid) rec = db.one("SELECT audio_path FROM recordings WHERE recording_id=? AND speaker_id=?", (rid, sid)) if not rec or not rec["audio_path"] or not Path(rec["audio_path"]).exists(): raise HTTPException(404) return FileResponse(rec["audio_path"], media_type="audio/wav") SUS_ITEMS = 10 class SurveyIn(BaseModel): instrument: str = Field(pattern="^sus$") answers: list[int] = Field(min_length=SUS_ITEMS, max_length=SUS_ITEMS) def sus_score(a: list[int]) -> float: """System Usability Scale: odd items positive, even items negative, 1–5 each, scaled to 0–100.""" return 2.5 * sum((x - 1) if i % 2 == 0 else (5 - x) for i, x in enumerate(a)) @app.post("/api/speakers/{sid}/survey", status_code=201) def submit_survey(sid: str, body: SurveyIn): sp = _speaker(sid) if any(x < 1 or x > 5 for x in body.answers): raise HTTPException(422, "answers are 1 to 5") score = sus_score(body.answers) with db.tx() as c: c.execute("INSERT OR REPLACE INTO surveys(speaker_id, project_id, instrument, answers, score, created_at)" " VALUES (?,?,?,?,?,?)", (sid, sp["project_id"], body.instrument, json.dumps(body.answers), score, time.time())) return {"score": score} # ------------------------------------------------------------------ simulated payouts @app.post("/api/projects/{pid}/payouts/batches", status_code=201) def create_payout_batch(pid: str, actor: str = Depends(require_admin)): """Collect every earned, unpaid entry into a payout batch (simulated; no money moves).""" manifest(pid) rows = db.q("SELECT entry_id, amount, currency FROM ledger WHERE project_id=? AND status='earned'", (pid,)) if not rows: raise HTTPException(409, "nothing to pay out") currencies = {r["currency"] for r in rows} bid = "PB-" + time.strftime("%Y%m%d") + "-" + uuid.uuid4().hex[:6].upper() with db.tx() as c: c.execute("INSERT INTO payout_batches(batch_id, project_id, created_at, created_by, status, n, total, currency)" " VALUES (?,?,?,?,?,?,?,?)", (bid, pid, time.time(), actor, "pending", len(rows), round(sum(r["amount"] for r in rows), 2), "/".join(sorted(currencies)))) c.executemany("UPDATE ledger SET status='pending', batch_id=? WHERE entry_id=?", [(bid, r["entry_id"]) for r in rows]) db.log(c, actor, "payout.batch_created", bid, {"n": len(rows)}) return {"batch_id": bid, "n": len(rows), "simulated": True} @app.post("/api/payouts/batches/{bid}/settle") def settle_payout_batch(bid: str, actor: str = Depends(require_admin)): """Mark a batch paid with simulated transaction references.""" b = db.one("SELECT * FROM payout_batches WHERE batch_id=?", (bid,)) if not b: raise HTTPException(404) if b["status"] == "paid": raise HTTPException(409, "batch already settled") now = time.time() rows = db.q("SELECT entry_id FROM ledger WHERE batch_id=?", (bid,)) with db.tx() as c: c.executemany("UPDATE ledger SET status='paid', paid_at=?, txn_id=? WHERE entry_id=?", [(now, "SIM-UPI-" + uuid.uuid4().hex[:10].upper(), r["entry_id"]) for r in rows]) c.execute("UPDATE payout_batches SET status='paid', settled_at=? WHERE batch_id=?", (now, bid)) db.log(c, actor, "payout.batch_settled", bid, {"n": len(rows), "simulated": True}) return {"batch_id": bid, "status": "paid", "n": len(rows), "simulated": True} @app.get("/api/projects/{pid}/payouts/batches") def list_payout_batches(pid: str, _: str = Depends(require_admin)): earned = db.one("SELECT COUNT(*) AS n, COALESCE(SUM(amount),0) AS total FROM ledger WHERE project_id=?" " AND status='earned'", (pid,)) return {"unbatched": dict(earned), "simulated": True, "batches": [dict(r) for r in db.q("SELECT * FROM payout_batches WHERE project_id=? ORDER BY created_at DESC", (pid,))]} # ------------------------------------------------------------------ dashboard @app.get("/api/projects/{pid}/dashboard") def dashboard(pid: str, days: int = Query(14, ge=1, le=120), _: str = Depends(require_reviewer)): m = manifest(pid) tasks = {t.task_id: t for t in m.tasks} # In an internal-test project every participant is a test participant, so count them. test_ok = int(m.ethics.required and m.ethics.status == "internal_test") contributors = db.one("SELECT COUNT(*) AS n FROM speakers WHERE project_id=? AND COALESCE(is_test,0)<=?" " AND withdrawn_at IS NULL", (pid, test_ok))["n"] recs = db.q("SELECT task_id, status, auto_decision, created_at, final_at FROM recordings WHERE project_id=?" " AND status NOT IN ('superseded','withdrawn')", (pid,)) paid = db.one("SELECT COALESCE(SUM(amount),0) AS total FROM ledger WHERE project_id=?", (pid,))["total"] now = time.time() day0 = time.mktime(time.localtime(now)[:3] + (0, 0, 0, 0, 0, -1)) - (days - 1) * 86400 daily = [] for i in range(days): a, b = day0 + i * 86400, day0 + (i + 1) * 86400 on_day = [r for r in recs if a <= r["created_at"] < b] decided = [r for r in on_day if r["status"] in ("accepted", "rejected")] daily.append({"date": time.strftime("%Y-%m-%d", time.localtime(a)), "recordings": len(on_day), "approved": sum(r["status"] == "accepted" for r in on_day), "approval_rate": round(sum(r["status"] == "accepted" for r in decided) / len(decided), 4) if decided else None}) def breakdown(key): out: dict[str, dict] = {} for r in recs: t = tasks.get(r["task_id"]) if not t: continue k = getattr(t, key) b = out.setdefault(k, {"recordings": 0, "approved": 0, "rejected": 0, "review": 0}) b["recordings"] += 1 b["approved"] += r["status"] == "accepted" b["rejected"] += r["status"] in ("rejected", "blocked") b["review"] += r["status"] in ("review", "checking") for b in out.values(): done = b["approved"] + b["rejected"] b["approval_rate"] = round(b["approved"] / done, 4) if done else None return out decided = [r for r in recs if r["status"] in ("accepted", "rejected", "blocked")] return { "project_id": pid, "title": m.title, "contributors": contributors, "recordings": len(recs), "approved": sum(r["status"] == "accepted" for r in recs), "pending_review": sum(r["status"] == "review" for r in recs), "checking": sum(r["status"] == "checking" for r in recs), "approval_rate": round(sum(r["status"] == "accepted" for r in decided) / len(decided), 4) if decided else None, "total_payout": round(paid, 2), "currency": m.defaults.payment.currency, "daily": daily, "by_language": breakdown("language"), "by_task_type": breakdown("task_type"), } # ------------------------------------------------------------------ user study @app.get("/api/projects/{pid}/study") def study_report(pid: str, _: str = Depends(require_admin)): """User-study results by condition and language: usable yield, attempts, whether blind listeners heard the intended meaning, agreement between listeners, agreement between the automatic prosody check and listeners, and System Usability Scale scores.""" import statistics from scqc.audit import fleiss_kappa m = manifest(pid) tasks = {t.task_id: t for t in m.tasks} sp = {r["speaker_id"]: r for r in db.q("SELECT * FROM speakers WHERE project_id=? AND COALESCE(is_test,0)=0" " AND withdrawn_at IS NULL", (pid,))} recs = db.q("SELECT * FROM recordings WHERE project_id=?", (pid,)) revs = db.q("SELECT v.* FROM reviews v JOIN recordings r USING (recording_id) WHERE r.project_id=?", (pid,)) surveys = {r["speaker_id"]: r["score"] for r in db.q("SELECT * FROM surveys WHERE project_id=?", (pid,))} perceived: dict[str, list[str]] = {} for v in revs: lab = json.loads(v["scores"] or "{}").get("_perceived") if lab: perceived.setdefault(v["recording_id"], []).append(lab) cells: dict[tuple, dict] = {} for sid, s_ in sp.items(): cell = cells.setdefault((s_["condition"] or "none", s_["language"] or "?"), { "participants": 0, "completed": 0, "attempts": 0, "kept": 0, "accepted": 0, "judgements": 0, "judged_correct": 0, "recordings_judged": 0, "majority_correct": 0, "auto_vs_listeners": [0, 0], "sus": []}) cell["participants"] += 1 mine = [r for r in recs if r["speaker_id"] == sid] kept = [r for r in mine if r["status"] not in ("superseded", "withdrawn")] lang_tasks = [t for t in m.tasks if t.language == s_["language"]] cell["completed"] += len({r["task_id"] for r in kept}) >= len(lang_tasks) > 0 cell["attempts"] += len(mine) cell["kept"] += len(kept) cell["accepted"] += sum(r["status"] == "accepted" for r in kept) for r in kept: t = tasks.get(r["task_id"]) labs = perceived.get(r["recording_id"], []) if not t or not labs or not t.prompt.interpretation: continue correct = [l == t.prompt.interpretation for l in labs] cell["judgements"] += len(labs) cell["judged_correct"] += sum(correct) cell["recordings_judged"] += 1 maj = sum(correct) > len(correct) / 2 cell["majority_correct"] += maj qc = json.loads(r["qc"]) auto = next((v["scores"].get("prosodic_realization") for v in qc.get("validators", []) if v["name"] == "prosody"), None) if auto is not None: cell["auto_vs_listeners"][0] += 1 cell["auto_vs_listeners"][1] += (auto >= 0.75) == maj if sid in surveys: cell["sus"].append(surveys[sid]) out = [] for (cond, lang), c in sorted(cells.items()): sus = c.pop("sus") n_auto, agree = c.pop("auto_vs_listeners") row = {"condition": cond, "language": lang, **c, "usable_yield": round(c["accepted"] / c["kept"], 4) if c["kept"] else None, "attempts_per_kept": round(c["attempts"] / c["kept"], 3) if c["kept"] else None, "listener_accuracy": round(c["judged_correct"] / c["judgements"], 4) if c["judgements"] else None, "majority_accuracy": round(c["majority_correct"] / c["recordings_judged"], 4) if c["recordings_judged"] else None, "auto_prosody_agreement": round(agree / n_auto, 4) if n_auto else None, "auto_prosody_n": n_auto, "sus_n": len(sus), "sus_mean": round(statistics.mean(sus), 2) if sus else None} if len(sus) >= 2: sd = statistics.stdev(sus) row["sus_ci95"] = [round(row["sus_mean"] - 1.96 * sd / len(sus) ** 0.5, 2), round(row["sus_mean"] + 1.96 * sd / len(sus) ** 0.5, 2)] out.append(row) return {"project_id": pid, "cells": out, "listener_fleiss_kappa": fleiss_kappa(perceived), "notes": "listener_accuracy: share of blind judgements naming the intended meaning; majority_accuracy: " "share of recordings whose majority judgement was right; auto_prosody_agreement: how often the " "automatic prosody check (pass at >= 0.75) agrees with the listener majority."} # ------------------------------------------------------------------ tasks @app.get("/api/projects/{pid}/next-task") def next_task(pid: str, speaker_id: str, language: Optional[str] = None, limit: int = Query(10, ge=1, le=100)): """Tasks this speaker can still record, least-collected first (for prefetching offline).""" m = manifest(pid) _speaker(speaker_id) mine = {r["task_id"]: r["n"] for r in db.q( "SELECT task_id, COUNT(*) AS n FROM recordings WHERE speaker_id=? AND project_id=?" " AND status IN ('review','accepted','checking') GROUP BY task_id", (speaker_id, pid))} totals = {r["task_id"]: r["n"] for r in db.q( "SELECT task_id, COUNT(*) AS n FROM recordings WHERE project_id=? AND status IN ('review','accepted')" " GROUP BY task_id", (pid,))} out = [] for t in m.tasks: if language and t.language != language: continue q = m.quota_for(t) if mine.get(t.task_id, 0) >= q.max_per_speaker: continue if q.max_recordings and totals.get(t.task_id, 0) >= q.max_recordings: continue out.append((totals.get(t.task_id, 0), len(out), t.task_id)) out.sort() ids = [tid for _, _, tid in out[:limit]] proj = get_project(pid) by_id = {t["task_id"]: t for t in proj["tasks"]} return {"tasks": [by_id[i] for i in ids], "remaining": len(out)} # ------------------------------------------------------------------ recordings @app.post("/api/recordings") async def upload_recording( project_id: str = Form(...), task_id: str = Form(...), speaker_id: str = Form(...), client_upload_id: str = Form(...), client_meta: str = Form("{}"), file: UploadFile = File(...), ): prev = db.one("SELECT recording_id, qc, status, condition FROM recordings WHERE client_upload_id=?", (client_upload_id,)) if prev: # idempotent retries from the offline sync queue return {"recording_id": prev["recording_id"], "status": prev["status"], "duplicate": True, "feedback": contributor_view(json.loads(prev["qc"]), prev["condition"])} m = manifest(project_id) blocked = recording_allowed(m) if blocked: raise HTTPException(403, blocked) enforce_retention() try: task = m.task(task_id) except KeyError: raise HTTPException(404, "unknown task") sp = _speaker(speaker_id) if sp["project_id"] != project_id: raise HTTPException(403, "speaker is registered for a different project") data = await file.read(MAX_UPLOAD_BYTES + 1) if len(data) > MAX_UPLOAD_BYTES: raise HTTPException(413, "recording too large") try: meta = json.loads(client_meta or "{}") except json.JSONDecodeError: meta = {"unparsed": client_meta[:500]} try: data = ensure_wav(data) except AudioDecodeError: pass # run_qc reports decode_failed and blocks the recording rid = "rec_" + uuid.uuid4().hex[:16] condition = sp["condition"] ctx = RecordingContext(task_id, task.task_type, task.language, speaker_id=speaker_id, consent_version=sp["consent_version"], project_consent_version=m.consent_version, declared_language=meta.get("language")) qc = run_qc(data, QCInput(task, m.validation_for(task), ctx, {k: v.model_dump() for k, v in m.intents.items()}, key=rid, speaker_baseline=_speaker_baseline(speaker_id, project_id, task.task_type)), ASR) decision = qc["decision"] status = decision if condition == "no_qc" and decision == "retry": status = "review" # the contributor was never offered a retry, so the recording is kept if status == "accepted" and MFA.eligible(task): status = "checking" # forced alignment runs in the background before the final decision path = None if "decode_failed" not in qc["reasons"]: path = AUDIO_DIR / project_id / f"{rid}.wav" path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(data) with db.tx() as c: attempt = 1 + c.execute("SELECT COUNT(*) FROM recordings WHERE speaker_id=? AND task_id=? AND project_id=?", (speaker_id, task_id, project_id)).fetchone()[0] c.execute("UPDATE recordings SET status='superseded', superseded_by=? WHERE speaker_id=? AND task_id=?" " AND project_id=? AND status='retry'", (rid, speaker_id, task_id, project_id)) c.execute("INSERT INTO recordings(recording_id, client_upload_id, project_id, task_id, speaker_id, attempt," " status, auto_decision, required_reviewers, audio_path, qc, client_meta, created_at, final_at," " condition) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", (rid, client_upload_id, project_id, task_id, speaker_id, attempt, status, decision, max(1, qc["required_reviewers"]) if status == "review" else qc["required_reviewers"], str(path) if path else None, json.dumps(qc, ensure_ascii=False), json.dumps(meta, ensure_ascii=False), time.time(), time.time() if status in ("accepted", "blocked") else None, condition)) if status == "accepted": _pay(c, m, c.execute("SELECT * FROM recordings WHERE recording_id=?", (rid,)).fetchone()) if status == "checking" or (path and MFA.eligible(task) and status == "review"): MFA.enqueue(rid) return {"recording_id": rid, "status": status, "attempt": attempt, "condition": condition, "feedback": contributor_view(qc, condition)} class KeepIn(BaseModel): speaker_id: str @app.post("/api/recordings/{rid}/keep") def keep_recording(rid: str, body: KeepIn): """Contributor chose to keep a recording that had retry guidance: send it to review.""" rec = db.one("SELECT * FROM recordings WHERE recording_id=?", (rid,)) if not rec or rec["speaker_id"] != body.speaker_id: raise HTTPException(404) if rec["status"] != "retry": return {"recording_id": rid, "status": rec["status"]} with db.tx() as c: c.execute("UPDATE recordings SET status='review', required_reviewers=MAX(required_reviewers,1)" " WHERE recording_id=?", (rid,)) return {"recording_id": rid, "status": "review"} # ------------------------------------------------------------------ review @app.get("/api/review/queue") def review_queue(project_id: str, reviewer_id: Optional[str] = None, limit: int = Query(20, ge=1, le=500), filter: str = Query("all", pattern="^(all|flagged|uncertain|high)$"), who: dict = Depends(reviewer_identity)): reviewer_id = who["username"] or reviewer_id if not reviewer_id: raise HTTPException(422, "reviewer_id is required with a shared token") rows = db.q( "SELECT r.* FROM recordings r WHERE r.project_id=? AND r.status='review' AND r.audio_path IS NOT NULL" " AND NOT EXISTS (SELECT 1 FROM reviews v WHERE v.recording_id=r.recording_id AND v.reviewer_id=?)" " ORDER BY r.created_at LIMIT 2000", (project_id, reviewer_id)) m = manifest(project_id) out = [] for r in rows: qc = json.loads(r["qc"]) t = m.task(r["task_id"]) match = next((v["data"] for v in qc["validators"] if v["name"] in ("read_speech_match", "command_match", "prosody_pair_lexical", "emotion_lexical")), None) n_reviews = db.one("SELECT COUNT(*) AS n FROM reviews WHERE recording_id=?", (r["recording_id"],))["n"] out.append({ "recording_id": r["recording_id"], "task_id": r["task_id"], "task_type": t.task_type, "language": t.language, "prompt": t.prompt.model_dump(exclude_none=True), "asset_url": f"/api/projects/{project_id}/assets/{t.prompt.image_asset_id}" if t.prompt.image_asset_id else None, "audio_url": f"/api/recordings/{r['recording_id']}/audio", "reasons": qc["reasons"], "route_notes": qc["route_notes"], "findings": [f for f in qc["findings"] if f["action"] != "info"], "rubric": qc["rubric"], "match": match, "signal": qc["signal"], "reviews_so_far": n_reviews, "required_reviewers": r["required_reviewers"], # Blind review for perceptual tasks: hide ASR text and auto reasons. "blind": t.task_type in ("prosody_pair", "emotion"), "confidence": qc.get("confidence"), "auto_decision": r["auto_decision"], "created_at": r["created_at"], "validators": [ {"name": v["name"], "data": {k: v["data"][k] for k in ("pauses", "pitch_range_st", "final_slope_st_per_s", "pauses_found", "targets", "suggested_coverage", "per_model", "cer_between", "aligned", "ll_z") if k in (v.get("data") or {})}} for v in qc["validators"] if v["name"] in ("prosody", "image_relevance", "mfa_alignment")], **_priority(qc, r, m.validation_for(t)), }) if filter != "all": out = [i for i in out if (filter == "high" and i["priority"] == "high") or (filter in ("flagged", "uncertain") and filter in i["categories"])] rank = {"high": 0, "medium": 1, "low": 2} out.sort(key=lambda i: (rank[i["priority"]], i["created_at"])) out = out[:limit] for item in out: if item["blind"]: item["match"], item["reasons"], item["findings"] = None, [], [] item["prompt"].pop("emotion", None) item["prompt"].pop("interpretation", None) item["perceived_options"] = _perceived_options(m, m.task(item["task_id"]), item["recording_id"]) return out HIGH_REASONS = {"possible_mismatch", "command_mismatch", "confusable_intent", "mfa_poor_fit", "clipping", "cut_off_end", "cut_off_start", "no_speech", "too_short"} def _priority(qc: dict, rec, v) -> dict: """Review priority: likely defects first, model uncertainty next, routine checks last.""" conf = (qc.get("confidence") or {}).get("score") reasons = set(qc.get("reasons") or []) flagged = bool(reasons - {"needs_human_relevance", "blind_listening_required", "asr_unavailable"}) codes = {f["code"] for f in qc.get("findings", [])} uncertain = (conf is not None and conf < (v.accept_confidence or 0.8)) or "asr_models_disagree" in codes \ or "asr_unavailable" in reasons or "low_asr_confidence" in reasons if (reasons & HIGH_REASONS) or rec["auto_decision"] == "retry" or (conf is not None and conf < 0.5): pr = "high" elif flagged or uncertain: pr = "medium" else: pr = "low" return {"priority": pr, "categories": [c for c, ok in (("flagged", flagged), ("uncertain", uncertain)) if ok]} def _perceived_options(m: Manifest, t, rid: str) -> list[str]: """Choices for a blind listener, in an order that does not reveal the target.""" if t.task_type == "prosody_pair": opts = [x.prompt.interpretation for x in m.tasks if x.task_type == "prosody_pair" and x.prompt.pair_id == t.prompt.pair_id] else: opts = sorted({x.prompt.emotion for x in m.tasks if x.task_type == "emotion" and x.prompt.emotion}) opts = [o for o in opts if o] random.Random(rid).shuffle(opts) # stable per recording, unrelated to the target return opts + ["Unclear"] @app.get("/api/recordings/{rid}/audio") def recording_audio(rid: str, token: Optional[str] = None, x_scqc_token: Optional[str] = Header(default=None)): # <audio> elements cannot send headers, so the token may come as a query parameter. if not _identity(x_scqc_token or token): raise HTTPException(401, "reviewer login required") rec = db.one("SELECT audio_path FROM recordings WHERE recording_id=?", (rid,)) if not rec or not rec["audio_path"] or not Path(rec["audio_path"]).exists(): raise HTTPException(404) return FileResponse(rec["audio_path"], media_type="audio/wav") class ReviewIn(BaseModel): reviewer_id: Optional[str] = Field(default=None, min_length=1, max_length=64) verdict: str = Field(pattern="^(accept|reject)$") issues: list[str] = Field(default_factory=list) scores: dict[str, float] = Field(default_factory=dict) # human rubric criteria in [0,1] perceived: Optional[str] = None # blind label (prosody meaning / emotion) seconds_spent: Optional[float] = Field(default=None, ge=0, le=3600) @app.post("/api/review/{rid}") def submit_review(rid: str, body: ReviewIn, who: dict = Depends(reviewer_identity)): body.reviewer_id = who["username"] or body.reviewer_id # logged-in reviewers can't review as someone else if not body.reviewer_id: raise HTTPException(422, "reviewer_id is required with a shared token") rec = db.one("SELECT * FROM recordings WHERE recording_id=?", (rid,)) if not rec: raise HTTPException(404) if rec["status"] != "review": raise HTTPException(409, f"recording is {rec['status']}, not in review") scores = dict(body.scores) if body.perceived: scores["_perceived"] = body.perceived with db.tx() as c: try: c.execute("INSERT INTO reviews(recording_id, reviewer_id, verdict, issues, scores, seconds_spent, created_at)" " VALUES (?,?,?,?,?,?,?)", (rid, body.reviewer_id, body.verdict, json.dumps(body.issues), json.dumps(scores, ensure_ascii=False), body.seconds_spent, time.time())) except Exception: raise HTTPException(409, "this reviewer already reviewed this recording") votes = c.execute("SELECT verdict FROM reviews WHERE recording_id=?", (rid,)).fetchall() acc = sum(v["verdict"] == "accept" for v in votes) rej = len(votes) - acc need = max(1, rec["required_reviewers"]) final = None if acc >= need and rej == 0: final = "accepted" elif rej >= need and acc == 0: final = "rejected" elif len(votes) >= need + 1 and acc != rej: # disagreement resolved by an extra reviewer final = "accepted" if acc > rej else "rejected" if final: c.execute("UPDATE recordings SET status=?, final_at=? WHERE recording_id=?", (final, time.time(), rid)) if final == "accepted": _pay(c, manifest(rec["project_id"]), rec) db.log(c, body.reviewer_id, "review", rid, {"verdict": body.verdict, "final": final}) return {"recording_id": rid, "status": final or "review", "votes": {"accept": acc, "reject": rej}, "needs_more_reviews": final is None} # ------------------------------------------------------------------ admin: export, payouts, report def _export_rows(pid: str): m = manifest(pid) ethics = E.export_view(m) for r in db.q("SELECT r.*, s.consent_version AS sc, s.consent_sha AS csha FROM recordings r" " JOIN speakers s USING (speaker_id) WHERE r.project_id=? AND r.status='accepted'" " AND r.audio_path IS NOT NULL AND COALESCE(s.is_test,0)=0", (pid,)): qc = json.loads(r["qc"]) t = m.task(r["task_id"]) revs = [{"reviewer": hashlib.sha256(v["reviewer_id"].encode()).hexdigest()[:10], "verdict": v["verdict"], "issues": json.loads(v["issues"] or "[]"), "scores": json.loads(v["scores"] or "{}")} for v in db.q("SELECT * FROM reviews WHERE recording_id=?", (r["recording_id"],))] yield { "recording_id": r["recording_id"], "project_id": pid, "task_id": r["task_id"], "task_type": t.task_type, "language": t.language, "prompt": t.prompt.model_dump(exclude_none=True), "speaker_id": r["speaker_id"], "consent_version": r["sc"], "consent_sha": r["csha"], "ethics": ethics, "attempt": r["attempt"], "audio": f"audio/{r['recording_id']}.wav", "duration_s": (qc.get("signal") or {}).get("duration_s"), "auto_decision": r["auto_decision"], "rubric": qc["rubric"], "reasons": qc["reasons"], "asr_text": next((v["data"].get("asr_text") for v in qc["validators"] if v["data"].get("asr_text")), None), "reviews": revs, "provenance": qc["provenance"], } @app.get("/api/projects/{pid}/export.jsonl") def export_jsonl(pid: str, _: str = Depends(require_admin)): body = "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in _export_rows(pid)) return PlainTextResponse(body, media_type="application/x-ndjson") @app.get("/api/projects/{pid}/export.zip") def export_zip(pid: str, _: str = Depends(require_admin)): buf = io.BytesIO() with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as z: lines = [] for row in _export_rows(pid): lines.append(json.dumps(row, ensure_ascii=False)) z.write(AUDIO_DIR / pid / f"{row['recording_id']}.wav", row["audio"]) z.writestr("manifest.jsonl", "\n".join(lines) + ("\n" if lines else "")) z.writestr("project.json", manifest(pid).model_dump_json(indent=2)) buf.seek(0) return StreamingResponse(buf, media_type="application/zip", headers={"Content-Disposition": f'attachment; filename="{pid}_export.zip"'}) @app.get("/api/projects/{pid}/payouts.csv") def payouts(pid: str, _: str = Depends(require_admin)): """Pseudonymous totals; join to payee details inside the restricted vault, not here.""" rows = db.q("SELECT speaker_id, currency, SUM(amount) AS total, COUNT(*) AS n FROM ledger" " WHERE project_id=? AND status='earned' GROUP BY speaker_id, currency", (pid,)) lines = ["speaker_id,currency,approved_recordings,amount"] + \ [f"{r['speaker_id']},{r['currency']},{r['n']},{r['total']:.2f}" for r in rows] return PlainTextResponse("\n".join(lines) + "\n", media_type="text/csv") @app.get("/api/projects/{pid}/report") def project_report(pid: str, group_by: str = Query("gender", pattern="^(gender|age_group|region|language|condition)$"), _: str = Depends(require_admin)): """Experiment B metrics, live: yield, retries, reviewer effort, subgroup rejection.""" manifest(pid) recs = db.q(f"SELECT r.*, s.{group_by} AS grp FROM recordings r JOIN speakers s USING (speaker_id)" " WHERE r.project_id=?", (pid,)) by_condition: dict[str, dict] = {} for r in recs: if r["status"] in ("superseded", "withdrawn"): continue b = by_condition.setdefault(r["condition"] or "none", {"recordings": 0, "accepted": 0, "attempts": 0}) b["recordings"] += 1 b["accepted"] += r["status"] == "accepted" for r in recs: by_condition.setdefault(r["condition"] or "none", {"recordings": 0, "accepted": 0, "attempts": 0})["attempts"] += 1 for b in by_condition.values(): b["usable_yield"] = round(b["accepted"] / b["recordings"], 4) if b["recordings"] else None b["attempts_per_kept_recording"] = round(b["attempts"] / b["recordings"], 3) if b["recordings"] else None status = {} for r in recs: status[r["status"]] = status.get(r["status"], 0) + 1 live = [r for r in recs if r["status"] not in ("superseded", "withdrawn")] accepted = sum(r["status"] == "accepted" for r in live) chains: dict[tuple, list] = {} for r in sorted(recs, key=lambda x: x["attempt"]): chains.setdefault((r["speaker_id"], r["task_id"]), []).append(r) retried = [ch for ch in chains.values() if len(ch) > 1] retry_ok = sum(ch[-1]["status"] in ("accepted", "review") for ch in retried) rev = db.one("SELECT COUNT(*) AS n, SUM(seconds_spent) AS s FROM reviews v JOIN recordings r USING (recording_id)" " WHERE r.project_id=?", (pid,)) groups: dict[str, dict] = {} for r in live: g = groups.setdefault(r["grp"] or "unreported", {"n": 0, "auto_flagged": 0, "human_rejected": 0, "flagged_then_accepted": 0, "human_decided": 0}) g["n"] += 1 flagged = r["auto_decision"] != "accepted" g["auto_flagged"] += flagged if r["status"] in ("accepted", "rejected") and r["auto_decision"] != "accepted": g["human_decided"] += 1 g["human_rejected"] += r["status"] == "rejected" g["flagged_then_accepted"] += r["status"] == "accepted" for g in groups.values(): g["auto_flag_rate"] = round(g["auto_flagged"] / g["n"], 4) if g["n"] else None # Share of human-decided flagged clips that humans accepted: an online proxy for false rejection. g["flag_overturn_rate"] = round(g["flagged_then_accepted"] / g["human_decided"], 4) if g["human_decided"] else None m = manifest(pid) enforce_retention() is_open, why = recruitment_state(m) e = m.ethics n_part = db.one("SELECT COUNT(*) AS n FROM speakers WHERE project_id=? AND COALESCE(is_test,0)=0", (pid,))["n"] n_test = db.one("SELECT COUNT(*) AS n FROM speakers WHERE project_id=? AND COALESCE(is_test,0)=1", (pid,))["n"] ethics_summary = {**E.export_view(m), "open": is_open, "closed_reason": why, "participants": n_part, "test_participants": n_test, "max_participants": e.max_participants, "days_to_expiry": (e.expiry_date - date.today()).days if e.required and e.expiry_date else None, "approval_document": e.approval_document} return { "project_id": pid, "ethics": ethics_summary, "status_counts": status, "recordings": len(recs), "usable_yield": round(accepted / len(live), 4) if live else None, "retry_chains": len(retried), "retry_success_rate": round(retry_ok / len(retried), 4) if retried else None, "reviews": rev["n"], "reviewer_minutes": round((rev["s"] or 0) / 60, 2), "reviewer_minutes_per_accepted": round((rev["s"] or 0) / 60 / accepted, 3) if accepted else None, "group_by": group_by, "groups": groups, "by_condition": by_condition, } @app.get("/api/projects/{pid}/ethics-pack.zip") def ethics_pack(pid: str, _: str = Depends(require_admin)): """Draft application documents generated from the project, for the researcher's committee.""" return StreamingResponse(io.BytesIO(E.pack_zip(manifest(pid))), media_type="application/zip", headers={"Content-Disposition": f'attachment; filename="{pid}_ethics_drafts.zip"'}) @app.post("/api/ethics/draft") def ethics_draft(body: dict, _: str = Depends(require_admin)): """Same drafts for a manifest that has not been published yet (apply before you publish).""" try: m = parse_manifest(body) except ManifestError as e: return JSONResponse(status_code=422, content={"errors": e.errors}) return StreamingResponse(io.BytesIO(E.pack_zip(m)), media_type="application/zip", headers={"Content-Disposition": f'attachment; filename="{m.project_id}_ethics_drafts.zip"'}) @app.get("/api/projects/{pid}/recordings") def list_recordings(pid: str, status: Optional[str] = None, limit: int = Query(100, le=1000), _: str = Depends(require_admin)): sql = "SELECT recording_id, task_id, speaker_id, attempt, status, auto_decision, created_at FROM recordings WHERE project_id=?" params: list = [pid] if status: sql += " AND status=?" params.append(status) rows = db.q(sql + " ORDER BY created_at DESC LIMIT ?", (*params, limit)) return [dict(r) for r in rows] MFA.hooks = (db, manifest, _pay, db.log)
"""SCQC collection server.
Roles are token-based for the demo: admin (manifests, export, payouts,
reports) and reviewer (queue, audio, verdicts). Participants are identified
only by a pseudonymous speaker_id issued at consent time.
Environment
SCQC_DATA data directory (default ./scqc_data)
SCQC_ASR ASR backend spec, see scqc/asr.py (default: none)
SCQC_ADMIN_TOKEN admin token (default "admin-dev"; change it)
SCQC_REVIEW_TOKEN reviewer token (default "review-dev"; change it)
"""
from __future__ import annotations
import hashlib
import io
import random
import json
import os
import time
import urllib.request
import uuid
import zipfile
from pathlib import Path
from typing import Optional
from fastapi import Depends, FastAPI, File, Form, Header, HTTPException, Query, UploadFile
from fastapi.responses import FileResponse, JSONResponse, PlainTextResponse, RedirectResponse, StreamingResponse
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel, Field
from scqc import __version__
from datetime import date
from scqc import ethics as E
from scqc.asr import load_backend
from scqc.audio import AudioDecodeError, ensure_wav
from scqc.checks.universal import RecordingContext
from scqc.manifest import Manifest, ManifestError, drive_direct_url, parse_manifest
from scqc.pipeline import QCInput, run_qc
from . import auth as AUTH
from .db import DB
from .mfa_worker import MFA
DATA = Path(os.environ.get("SCQC_DATA", "./scqc_data")).resolve()
AUDIO_DIR = DATA / "audio"
ASSET_DIR = DATA / "assets"
for d in (DATA, AUDIO_DIR, ASSET_DIR):
d.mkdir(parents=True, exist_ok=True)
ADMIN_TOKEN = os.environ.get("SCQC_ADMIN_TOKEN", "admin-dev")
REVIEW_TOKEN = os.environ.get("SCQC_REVIEW_TOKEN", "review-dev")
MAX_UPLOAD_BYTES = 25 * 1024 * 1024
db = DB(str(DATA / "scqc.sqlite"))
AUTH.init(db)
ASR_SPEC = os.environ.get("SCQC_ASR", "none")
ASR_ERROR: Optional[str] = None
try:
ASR = load_backend(ASR_SPEC)
except Exception as e: # keep the demo up: fall back to acoustic checks + human review
ASR_ERROR = f"{type(e).__name__}: {e}"[:400]
print(f"[scqc] ASR backend {ASR_SPEC!r} failed to load, running without ASR: {ASR_ERROR}", flush=True)
ASR = load_backend("none")
_manifests: dict[str, Manifest] = {}
def _seed_examples() -> None:
"""SCQC_SEED_EXAMPLES=1 loads examples/manifest_*.json on start (for a hosted demo)."""
ex = Path(__file__).resolve().parents[1] / "examples"
for path in sorted(ex.glob("manifest_*.json")):
m = parse_manifest(path.read_text(encoding="utf-8"))
if db.one("SELECT 1 FROM projects WHERE project_id=?", (m.project_id,)):
continue
with db.tx() as c:
key = E.approval_key(m)
c.execute("INSERT INTO projects(project_id, manifest, created_at, ethics_key, ethics_scope)"
" VALUES (?,?,?,?,?)", (m.project_id, m.model_dump_json(), time.time(), key,
json.dumps(E.scope(m)) if key else None))
c.executemany("INSERT INTO tasks(project_id, task_id, ord, task_type, language) VALUES (?,?,?,?,?)",
[(m.project_id, t.task_id, i, t.task_type, t.language) for i, t in enumerate(m.tasks)])
for aid, asset in m.assets.items():
local = ex / asset.url
if local.is_file():
(ASSET_DIR / m.project_id).mkdir(parents=True, exist_ok=True)
(ASSET_DIR / m.project_id / aid).write_bytes(local.read_bytes())
if os.environ.get("SCQC_SEED_EXAMPLES") == "1":
_seed_examples()
app = FastAPI(title="SCQC", version=__version__)
STATIC = Path(__file__).parent / "static"
app.mount("/static", StaticFiles(directory=STATIC), name="static")
# ------------------------------------------------------------------ helpers
def manifest(pid: str) -> Manifest:
if pid not in _manifests:
row = db.one("SELECT manifest FROM projects WHERE project_id=?", (pid,))