"""Small, offline regression tests for the opt-in VQA admission pilot.""" from __future__ import annotations import importlib.util from pathlib import Path import pytest ROOT = Path(__file__).resolve().parents[1] SPEC = importlib.util.spec_from_file_location( "hfd_vqa_candidate_pilot", ROOT / "scripts/data/build_hfd_vqa_candidate_pilot.py" ) assert SPEC is not None and SPEC.loader is not None PILOT = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(PILOT) def _row(tmp_path: Path, *, split: str = "train", accepted: bool = True) -> dict: body = tmp_path / f"{split}.pt" body.write_bytes(b"body") voice = tmp_path / f"{split}.wav" voice.write_bytes(b"wav") return { "id": f"vqav2_{split}_1", "split": split, "path": str(body), "frontend_root": str(tmp_path / "frontend"), "cohort": "vqav2_static", "question": { "question": "Is this picture in color?", "answer": "yes", "human_votes": 10, }, "visual_review": {"accepted": accepted}, "timeline": {"question_voice": str(voice)}, } def test_vqav2_gate_uses_row_level_blind_review(tmp_path: Path) -> None: assert PILOT._quality_gate(_row(tmp_path))[0] accepted, reason = PILOT._quality_gate(_row(tmp_path, accepted=False)) assert not accepted assert reason == "vqav2_agreement_or_visual_review_failed" def test_pilot_selection_is_split_and_body_disjoint(tmp_path: Path) -> None: rows = [_row(tmp_path, split="train"), _row(tmp_path, split="val")] selected, audit = PILOT.select_hfd_rows(rows, train_limit=1, val_limit=1) assert len(selected) == 2 assert audit["formal_source_plan_changed"] is False assert {row["split"] for row in selected} == {"train", "val"} def test_gqa_gate_rejects_ambiguous_questions() -> None: base = { "isBalanced": True, "types": {"structural": "verify"}, "question": "Is the person holding a cup?", "answer": "yes", } assert PILOT.conservative_gqa_row(base)[0] ambiguous = {**base, "question": "Why might the person be there?"} accepted, reason = PILOT.conservative_gqa_row(ambiguous) assert not accepted and reason == "speculative_or_text_question" @pytest.mark.skipif(not __import__("shutil").which("ffmpeg"), reason="ffmpeg unavailable") def test_review_answer_waveform_is_separate_from_input(tmp_path: Path) -> None: spec = importlib.util.spec_from_file_location( "hfd_gt_render", ROOT / "scripts/data/render_hfd_unified_multimodal_gt.py" ) assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) output = tmp_path / "answer.wav" assert module._synthesize_review_answer("Camera lens", output) import wave with wave.open(str(output), "rb") as handle: assert handle.getframerate() == 16_000 assert handle.getnchannels() == 1 assert handle.getnframes() > 0