File size: 10,000 Bytes
912d2b5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e0d876f
 
 
912d2b5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e0d876f
912d2b5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e0d876f
912d2b5
e0d876f
 
 
 
 
 
 
912d2b5
 
e0d876f
912d2b5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
import json
from pathlib import Path

import pytest
from fastapi.testclient import TestClient

from conftest import CLEAN, CMD_FLIP, MISREAD
from scqc import synth

EX = Path(__file__).resolve().parents[1] / "examples"
ADM = {"X-SCQC-Token": "adm"}
REV = {"X-SCQC-Token": "rev"}


@pytest.fixture(scope="module")
def client():
    from server.app import app
    c = TestClient(app)
    for name in ("manifest_demo.json", "manifest_pilot.json", "manifest_specialist.json"):
        m = json.loads((EX / name).read_text(encoding="utf-8"))
        r = c.post("/api/projects?replace=true", json=m, headers=ADM)
        assert r.status_code == 201, r.text
    svg = (EX / "assets" / "street_scene.svg").read_bytes()
    r = c.put("/api/projects/SCQC_DEMO_001/assets/street_scene", files={"file": ("s.svg", svg, "image/svg+xml")}, headers=ADM)
    assert r.status_code == 200
    return c


def speaker(c, pid="SCQC_DEMO_001", consent="2026-11-v1", **kw):
    sha = c.get(f"/api/projects/{pid}").json()["ethics"]["consent"]["hi"]["sha"]
    r = c.post("/api/speakers", json={"project_id": pid, "consent_version": consent, "language": "hi",
                                      "consent_sha": sha, "age_confirmed": True, **kw})
    assert r.status_code == 201, r.text
    return r.json()


def upload(c, sid, task, data, pid="SCQC_DEMO_001", uid=None):
    import uuid
    return c.post("/api/recordings", data={"project_id": pid, "task_id": task, "speaker_id": sid,
                                           "client_upload_id": uid or uuid.uuid4().hex, "client_meta": "{}"},
                  files={"file": ("a.wav", data, "audio/wav")})


def test_admin_requires_token(client):
    m = json.loads((EX / "manifest_demo.json").read_text(encoding="utf-8"))
    assert client.post("/api/projects", json=m).status_code == 401
    assert client.post("/api/projects/validate", json=m, headers=ADM).json()["n_tasks"] == 9


def test_invalid_manifest_lists_errors(client):
    r = client.post("/api/projects/validate", json={"project_id": "x"}, headers=ADM)
    assert r.status_code == 422 and len(r.json()["errors"]) >= 3


def test_participant_view_hides_emotion_target(client):
    p = client.get("/api/projects/SCQC_PROSODY_EMOTION").json()
    assert all("emotion" not in t["prompt"] for t in p["tasks"])


def test_stale_consent_rejected(client):
    r = client.post("/api/speakers", json={"project_id": "SCQC_DEMO_001", "consent_version": "old"})
    assert r.status_code == 409


def test_clean_read_is_accepted_and_paid(client):
    sid = speaker(client)["speaker_id"]
    r = upload(client, sid, "hi_read_001", CLEAN).json()
    assert r["status"] in ("accepted", "review")  # 5% of clean recordings are spot-checked by a reviewer
    if r["status"] == "accepted":
        w = client.get(f"/api/speakers/{sid}/wallet").json()
        assert w["balances"][0]["total"] == 2.0


def test_misread_goes_to_review_with_word_hints(client):
    sid = speaker(client)["speaker_id"]
    r = upload(client, sid, "hi_read_001", MISREAD).json()
    assert r["status"] == "review"
    fb = r["feedback"]
    assert fb["retry_suggested"] and fb["words"]
    assert any(w["op"] == "del" and w["ref"] == "बहुत" for w in fb["words"])
    # Reviewer sees the alignment; contributor-facing payload never includes reviewer-only data.
    assert "findings" not in r and "rubric" not in r


def test_idempotent_upload(client):
    sid = speaker(client)["speaker_id"]
    a = upload(client, sid, "hi_read_002", CLEAN, uid="same-id").json()
    b = upload(client, sid, "hi_read_002", CLEAN, uid="same-id").json()
    assert a["recording_id"] == b["recording_id"] and b["duplicate"]


def test_retry_then_keep_and_supersede(client):
    sid = speaker(client)["speaker_id"]
    clipped = synth.wav_bytes(synth.utterance(2.0, level=1.8))
    r1 = upload(client, sid, "hi_read_003", clipped).json()
    assert r1["status"] == "retry"
    assert any(m["code"] == "clipping" for m in r1["feedback"]["messages"])
    # task is offered again (retry does not use up the quota)
    nt = client.get(f"/api/projects/SCQC_DEMO_001/next-task?speaker_id={sid}&limit=100").json()
    assert "hi_read_003" in [t["task_id"] for t in nt["tasks"]]
    r2 = upload(client, sid, "hi_read_003", clipped).json()
    hist = {h["recording_id"]: h["status"] for h in client.get(f"/api/speakers/{sid}/history").json()}
    assert hist[r1["recording_id"]] == "superseded"
    k = client.post(f"/api/recordings/{r2['recording_id']}/keep", json={"speaker_id": sid}).json()
    assert k["status"] == "review"


def test_corrupt_upload_blocked_and_not_stored(client):
    sid = speaker(client)["speaker_id"]
    r = upload(client, sid, "hi_read_004", b"garbage").json()
    assert r["status"] == "blocked"


def test_review_flow_and_export(client):
    sid = speaker(client, gender="Woman")["speaker_id"]
    rec = upload(client, sid, "bn_image_001", synth.wav_bytes(synth.utterance(6.0))).json()
    assert rec["status"] == "review"
    assert client.get("/api/review/queue?project_id=SCQC_DEMO_001&reviewer_id=r1").status_code == 401
    q = client.get("/api/review/queue?project_id=SCQC_DEMO_001&reviewer_id=r1&limit=200", headers=REV).json()
    item = next(i for i in q if i["recording_id"] == rec["recording_id"])
    assert item["asset_url"] and item["rubric"]["criteria"]["image_relevance"]["score"] is None
    audio = client.get(item["audio_url"] + "?token=rev")
    assert audio.status_code == 200 and audio.content[:4] == b"RIFF"
    v = client.post(f"/api/review/{rec['recording_id']}", headers=REV,
                    json={"reviewer_id": "r1", "verdict": "accept", "scores": {"image_relevance": 1}, "seconds_spent": 12}).json()
    assert v["status"] == "accepted"
    # The demo project is in internal-test mode: its recordings never reach exports.
    rows = [json.loads(l) for l in client.get("/api/projects/SCQC_DEMO_001/export.jsonl", headers=ADM).text.splitlines()]
    assert all(r["recording_id"] != rec["recording_id"] for r in rows)
    # An approved project exports with its ethics reference and consent fingerprint.
    sp = speaker(client, pid="SCQC_PILOT_HI", consent="2026-11-pilot-v1")["speaker_id"]
    pr = upload(client, sp, "p_001", CLEAN, pid="SCQC_PILOT_HI").json()
    client.post(f"/api/review/{pr['recording_id']}", headers=REV, json={"reviewer_id": "r1", "verdict": "accept"})
    rows = [json.loads(l) for l in client.get("/api/projects/SCQC_PILOT_HI/export.jsonl", headers=ADM).text.splitlines()]
    row = next(r for r in rows if r["recording_id"] == pr["recording_id"])
    assert row["reviews"][0]["reviewer"] != "r1"          # reviewer identity pseudonymised
    assert row["provenance"]["thresholds_sha"]
    assert row["ethics"]["protocol_id"] == "EXAMPLE-IEC-2026-000" and row["consent_sha"]
    rep = client.get("/api/projects/SCQC_DEMO_001/report?group_by=gender", headers=ADM).json()
    assert rep["groups"]["Woman"]["flag_overturn_rate"] == 1.0
    assert client.get("/api/projects/SCQC_DEMO_001/export.zip", headers=ADM).content[:2] == b"PK"


def test_blind_review_for_prosody_and_two_reviewers(client):
    sid = speaker(client, pid="SCQC_PROSODY_EMOTION", consent="2026-11-v1")["speaker_id"]
    rec = upload(client, sid, "pp_ruko_a", CLEAN, pid="SCQC_PROSODY_EMOTION").json()
    q = client.get("/api/review/queue?project_id=SCQC_PROSODY_EMOTION&reviewer_id=r1", headers=REV).json()
    item = next(i for i in q if i["recording_id"] == rec["recording_id"])
    assert item["blind"] and item["match"] is None and "interpretation" not in item["prompt"]
    assert set(item["perceived_options"]) == {"Stop. Do not go.", "Do not stop. Go.", "Unclear"}
    a = client.post(f"/api/review/{rec['recording_id']}", headers=REV, json={"reviewer_id": "r1", "verdict": "accept",
                    "perceived": "Stop. Do not go."}).json()
    assert a["needs_more_reviews"]
    b = client.post(f"/api/review/{rec['recording_id']}", headers=REV, json={"reviewer_id": "r2", "verdict": "accept"}).json()
    assert b["status"] == "accepted"


def test_command_intent_flip_routes_to_review(client):
    sid = speaker(client)["speaker_id"]
    r = upload(client, sid, "hi_cmd_lights_on", CMD_FLIP).json()
    assert r["status"] == "review"
    assert any(m["code"] == "confusable_intent" for m in r["feedback"]["messages"])


def test_experiment_conditions(client):
    conds = [speaker(client, pid="SCQC_PILOT_HI", consent="2026-11-pilot-v1")["condition"] for _ in range(3)]
    assert sorted(conds) == ["acoustic_only", "no_qc", "task_aware"]
    clipped = synth.wav_bytes(synth.utterance(2.0, level=1.8))
    seen = {}
    for _ in range(3):
        sp = speaker(client, pid="SCQC_PILOT_HI", consent="2026-11-pilot-v1")
        seen[sp["condition"]] = upload(client, sp["speaker_id"], "p_001", MISREAD, pid="SCQC_PILOT_HI").json()
        if sp["condition"] == "no_qc":
            nq = upload(client, sp["speaker_id"], "p_002", clipped, pid="SCQC_PILOT_HI").json()
            assert nq["status"] == "review" and not nq["feedback"]["show"]  # kept; no retry offered
    assert seen["no_qc"]["feedback"]["show"] is False
    assert seen["acoustic_only"]["feedback"]["words"] is None
    assert not any(m["code"] == "possible_mismatch" for m in seen["acoustic_only"]["feedback"]["messages"])
    assert seen["task_aware"]["feedback"]["words"]
    rep = client.get("/api/projects/SCQC_PILOT_HI/report?group_by=condition", headers=ADM).json()
    assert set(rep["by_condition"]) >= {"no_qc", "acoustic_only", "task_aware"}


def test_withdrawal_deletes_audio(client):
    sid = speaker(client)["speaker_id"]
    upload(client, sid, "hi_read_005", CLEAN)
    r = client.delete(f"/api/speakers/{sid}").json()
    assert r["recordings_deleted"] == 1
    assert client.get(f"/api/speakers/{sid}/history").status_code == 410


def test_pages_served(client):
    for path in ("/", "/review", "/admin", "/sw.js", "/static/participant.js", "/api/schema"):
        assert client.get(path).status_code == 200, path