SCQC / tests /test_api.py
raianand's picture
Upload folder using huggingface_hub
e0d876f verified
Raw History Blame Contribute Delete
10 kB
import json
from pathlib import Path
import pytest
from fastapi.testclient import TestClient
from conftest import CLEAN, CMD_FLIP, MISREAD
from scqc import synth
EX = Path(__file__).resolve().parents[1] / "examples"
ADM = {"X-SCQC-Token": "adm"}
REV = {"X-SCQC-Token": "rev"}
@pytest.fixture(scope="module")
def client():
from server.app import app
c = TestClient(app)
for name in ("manifest_demo.json", "manifest_pilot.json", "manifest_specialist.json"):
m = json.loads((EX / name).read_text(encoding="utf-8"))
r = c.post("/api/projects?replace=true", json=m, headers=ADM)
assert r.status_code == 201, r.text
svg = (EX / "assets" / "street_scene.svg").read_bytes()
r = c.put("/api/projects/SCQC_DEMO_001/assets/street_scene", files={"file": ("s.svg", svg, "image/svg+xml")}, headers=ADM)
assert r.status_code == 200
return c
def speaker(c, pid="SCQC_DEMO_001", consent="2026-11-v1", **kw):
sha = c.get(f"/api/projects/{pid}").json()["ethics"]["consent"]["hi"]["sha"]
r = c.post("/api/speakers", json={"project_id": pid, "consent_version": consent, "language": "hi",
"consent_sha": sha, "age_confirmed": True, **kw})
assert r.status_code == 201, r.text
return r.json()
def upload(c, sid, task, data, pid="SCQC_DEMO_001", uid=None):
import uuid
return c.post("/api/recordings", data={"project_id": pid, "task_id": task, "speaker_id": sid,
"client_upload_id": uid or uuid.uuid4().hex, "client_meta": "{}"},
files={"file": ("a.wav", data, "audio/wav")})
def test_admin_requires_token(client):
m = json.loads((EX / "manifest_demo.json").read_text(encoding="utf-8"))
assert client.post("/api/projects", json=m).status_code == 401
assert client.post("/api/projects/validate", json=m, headers=ADM).json()["n_tasks"] == 9
def test_invalid_manifest_lists_errors(client):
r = client.post("/api/projects/validate", json={"project_id": "x"}, headers=ADM)
assert r.status_code == 422 and len(r.json()["errors"]) >= 3
def test_participant_view_hides_emotion_target(client):
p = client.get("/api/projects/SCQC_PROSODY_EMOTION").json()
assert all("emotion" not in t["prompt"] for t in p["tasks"])
def test_stale_consent_rejected(client):
r = client.post("/api/speakers", json={"project_id": "SCQC_DEMO_001", "consent_version": "old"})
assert r.status_code == 409
def test_clean_read_is_accepted_and_paid(client):
sid = speaker(client)["speaker_id"]
r = upload(client, sid, "hi_read_001", CLEAN).json()
assert r["status"] in ("accepted", "review") # 5% of clean recordings are spot-checked by a reviewer
if r["status"] == "accepted":
w = client.get(f"/api/speakers/{sid}/wallet").json()
assert w["balances"][0]["total"] == 2.0
def test_misread_goes_to_review_with_word_hints(client):
sid = speaker(client)["speaker_id"]
r = upload(client, sid, "hi_read_001", MISREAD).json()
assert r["status"] == "review"
fb = r["feedback"]
assert fb["retry_suggested"] and fb["words"]
assert any(w["op"] == "del" and w["ref"] == "बहुत" for w in fb["words"])
# Reviewer sees the alignment; contributor-facing payload never includes reviewer-only data.
assert "findings" not in r and "rubric" not in r
def test_idempotent_upload(client):
sid = speaker(client)["speaker_id"]
a = upload(client, sid, "hi_read_002", CLEAN, uid="same-id").json()
b = upload(client, sid, "hi_read_002", CLEAN, uid="same-id").json()
assert a["recording_id"] == b["recording_id"] and b["duplicate"]
def test_retry_then_keep_and_supersede(client):
sid = speaker(client)["speaker_id"]
clipped = synth.wav_bytes(synth.utterance(2.0, level=1.8))
r1 = upload(client, sid, "hi_read_003", clipped).json()
assert r1["status"] == "retry"
assert any(m["code"] == "clipping" for m in r1["feedback"]["messages"])
# task is offered again (retry does not use up the quota)
nt = client.get(f"/api/projects/SCQC_DEMO_001/next-task?speaker_id={sid}&limit=100").json()
assert "hi_read_003" in [t["task_id"] for t in nt["tasks"]]
r2 = upload(client, sid, "hi_read_003", clipped).json()
hist = {h["recording_id"]: h["status"] for h in client.get(f"/api/speakers/{sid}/history").json()}
assert hist[r1["recording_id"]] == "superseded"
k = client.post(f"/api/recordings/{r2['recording_id']}/keep", json={"speaker_id": sid}).json()
assert k["status"] == "review"
def test_corrupt_upload_blocked_and_not_stored(client):
sid = speaker(client)["speaker_id"]
r = upload(client, sid, "hi_read_004", b"garbage").json()
assert r["status"] == "blocked"
def test_review_flow_and_export(client):
sid = speaker(client, gender="Woman")["speaker_id"]
rec = upload(client, sid, "bn_image_001", synth.wav_bytes(synth.utterance(6.0))).json()
assert rec["status"] == "review"
assert client.get("/api/review/queue?project_id=SCQC_DEMO_001&reviewer_id=r1").status_code == 401
q = client.get("/api/review/queue?project_id=SCQC_DEMO_001&reviewer_id=r1&limit=200", headers=REV).json()
item = next(i for i in q if i["recording_id"] == rec["recording_id"])
assert item["asset_url"] and item["rubric"]["criteria"]["image_relevance"]["score"] is None
audio = client.get(item["audio_url"] + "?token=rev")
assert audio.status_code == 200 and audio.content[:4] == b"RIFF"
v = client.post(f"/api/review/{rec['recording_id']}", headers=REV,
json={"reviewer_id": "r1", "verdict": "accept", "scores": {"image_relevance": 1}, "seconds_spent": 12}).json()
assert v["status"] == "accepted"
# The demo project is in internal-test mode: its recordings never reach exports.
rows = [json.loads(l) for l in client.get("/api/projects/SCQC_DEMO_001/export.jsonl", headers=ADM).text.splitlines()]
assert all(r["recording_id"] != rec["recording_id"] for r in rows)
# An approved project exports with its ethics reference and consent fingerprint.
sp = speaker(client, pid="SCQC_PILOT_HI", consent="2026-11-pilot-v1")["speaker_id"]
pr = upload(client, sp, "p_001", CLEAN, pid="SCQC_PILOT_HI").json()
client.post(f"/api/review/{pr['recording_id']}", headers=REV, json={"reviewer_id": "r1", "verdict": "accept"})
rows = [json.loads(l) for l in client.get("/api/projects/SCQC_PILOT_HI/export.jsonl", headers=ADM).text.splitlines()]
row = next(r for r in rows if r["recording_id"] == pr["recording_id"])
assert row["reviews"][0]["reviewer"] != "r1" # reviewer identity pseudonymised
assert row["provenance"]["thresholds_sha"]
assert row["ethics"]["protocol_id"] == "EXAMPLE-IEC-2026-000" and row["consent_sha"]
rep = client.get("/api/projects/SCQC_DEMO_001/report?group_by=gender", headers=ADM).json()
assert rep["groups"]["Woman"]["flag_overturn_rate"] == 1.0
assert client.get("/api/projects/SCQC_DEMO_001/export.zip", headers=ADM).content[:2] == b"PK"
def test_blind_review_for_prosody_and_two_reviewers(client):
sid = speaker(client, pid="SCQC_PROSODY_EMOTION", consent="2026-11-v1")["speaker_id"]
rec = upload(client, sid, "pp_ruko_a", CLEAN, pid="SCQC_PROSODY_EMOTION").json()
q = client.get("/api/review/queue?project_id=SCQC_PROSODY_EMOTION&reviewer_id=r1", headers=REV).json()
item = next(i for i in q if i["recording_id"] == rec["recording_id"])
assert item["blind"] and item["match"] is None and "interpretation" not in item["prompt"]
assert set(item["perceived_options"]) == {"Stop. Do not go.", "Do not stop. Go.", "Unclear"}
a = client.post(f"/api/review/{rec['recording_id']}", headers=REV, json={"reviewer_id": "r1", "verdict": "accept",
"perceived": "Stop. Do not go."}).json()
assert a["needs_more_reviews"]
b = client.post(f"/api/review/{rec['recording_id']}", headers=REV, json={"reviewer_id": "r2", "verdict": "accept"}).json()
assert b["status"] == "accepted"
def test_command_intent_flip_routes_to_review(client):
sid = speaker(client)["speaker_id"]
r = upload(client, sid, "hi_cmd_lights_on", CMD_FLIP).json()
assert r["status"] == "review"
assert any(m["code"] == "confusable_intent" for m in r["feedback"]["messages"])
def test_experiment_conditions(client):
conds = [speaker(client, pid="SCQC_PILOT_HI", consent="2026-11-pilot-v1")["condition"] for _ in range(3)]
assert sorted(conds) == ["acoustic_only", "no_qc", "task_aware"]
clipped = synth.wav_bytes(synth.utterance(2.0, level=1.8))
seen = {}
for _ in range(3):
sp = speaker(client, pid="SCQC_PILOT_HI", consent="2026-11-pilot-v1")
seen[sp["condition"]] = upload(client, sp["speaker_id"], "p_001", MISREAD, pid="SCQC_PILOT_HI").json()
if sp["condition"] == "no_qc":
nq = upload(client, sp["speaker_id"], "p_002", clipped, pid="SCQC_PILOT_HI").json()
assert nq["status"] == "review" and not nq["feedback"]["show"] # kept; no retry offered
assert seen["no_qc"]["feedback"]["show"] is False
assert seen["acoustic_only"]["feedback"]["words"] is None
assert not any(m["code"] == "possible_mismatch" for m in seen["acoustic_only"]["feedback"]["messages"])
assert seen["task_aware"]["feedback"]["words"]
rep = client.get("/api/projects/SCQC_PILOT_HI/report?group_by=condition", headers=ADM).json()
assert set(rep["by_condition"]) >= {"no_qc", "acoustic_only", "task_aware"}
def test_withdrawal_deletes_audio(client):
sid = speaker(client)["speaker_id"]
upload(client, sid, "hi_read_005", CLEAN)
r = client.delete(f"/api/speakers/{sid}").json()
assert r["recordings_deleted"] == 1
assert client.get(f"/api/speakers/{sid}/history").status_code == 410
def test_pages_served(client):
for path in ("/", "/review", "/admin", "/sw.js", "/static/participant.js", "/api/schema"):
assert client.get(path).status_code == 200, path