code-HFD / tests /test_hfd_vqa_candidate_pilot.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
aa02c28 verified
Raw History Blame Contribute Delete
3.04 kB
"""Small, offline regression tests for the opt-in VQA admission pilot."""
from __future__ import annotations
import importlib.util
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
SPEC = importlib.util.spec_from_file_location(
"hfd_vqa_candidate_pilot", ROOT / "scripts/data/build_hfd_vqa_candidate_pilot.py"
)
assert SPEC is not None and SPEC.loader is not None
PILOT = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(PILOT)
def _row(tmp_path: Path, *, split: str = "train", accepted: bool = True) -> dict:
body = tmp_path / f"{split}.pt"
body.write_bytes(b"body")
voice = tmp_path / f"{split}.wav"
voice.write_bytes(b"wav")
return {
"id": f"vqav2_{split}_1",
"split": split,
"path": str(body),
"frontend_root": str(tmp_path / "frontend"),
"cohort": "vqav2_static",
"question": {
"question": "Is this picture in color?",
"answer": "yes",
"human_votes": 10,
},
"visual_review": {"accepted": accepted},
"timeline": {"question_voice": str(voice)},
}
def test_vqav2_gate_uses_row_level_blind_review(tmp_path: Path) -> None:
assert PILOT._quality_gate(_row(tmp_path))[0]
accepted, reason = PILOT._quality_gate(_row(tmp_path, accepted=False))
assert not accepted
assert reason == "vqav2_agreement_or_visual_review_failed"
def test_pilot_selection_is_split_and_body_disjoint(tmp_path: Path) -> None:
rows = [_row(tmp_path, split="train"), _row(tmp_path, split="val")]
selected, audit = PILOT.select_hfd_rows(rows, train_limit=1, val_limit=1)
assert len(selected) == 2
assert audit["formal_source_plan_changed"] is False
assert {row["split"] for row in selected} == {"train", "val"}
def test_gqa_gate_rejects_ambiguous_questions() -> None:
base = {
"isBalanced": True,
"types": {"structural": "verify"},
"question": "Is the person holding a cup?",
"answer": "yes",
}
assert PILOT.conservative_gqa_row(base)[0]
ambiguous = {**base, "question": "Why might the person be there?"}
accepted, reason = PILOT.conservative_gqa_row(ambiguous)
assert not accepted and reason == "speculative_or_text_question"
@pytest.mark.skipif(not __import__("shutil").which("ffmpeg"), reason="ffmpeg unavailable")
def test_review_answer_waveform_is_separate_from_input(tmp_path: Path) -> None:
spec = importlib.util.spec_from_file_location(
"hfd_gt_render", ROOT / "scripts/data/render_hfd_unified_multimodal_gt.py"
)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
output = tmp_path / "answer.wav"
assert module._synthesize_review_answer("Camera lens", output)
import wave
with wave.open(str(output), "rb") as handle:
assert handle.getframerate() == 16_000
assert handle.getnchannels() == 1
assert handle.getnframes() > 0