code-HFD / tests /test_swa_scaleup.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
aa02c28 verified
Raw History Blame Contribute Delete
3.75 kB
import importlib.util
from pathlib import Path
import sys
import pytest
import torch
ROOT = Path(__file__).resolve().parents[1]
def load_script(name):
sys.path.insert(0, str(ROOT / "scripts/data"))
spec = importlib.util.spec_from_file_location(name, ROOT / "scripts/data" / f"{name}.py")
module = importlib.util.module_from_spec(spec);spec.loader.exec_module(module)
return module
def test_vqa_candidate_gate_requires_nine_human_votes_and_visual_question():
m = load_script("prepare_swa_vqav2")
questions = [{"question_id": i, "image_id": i, "question": q} for i, q in enumerate(
["What color is this shirt?", "What color is this shirt?", "Why is she wearing this shirt?"])]
annotations = [{"question_id": i, "image_id": i, "multiple_choice_answer": "red", "answer_type": "other",
"answers": [{"answer": "red"}] * n + [{"answer": "blue"}] * (10 - n)}
for i, n in enumerate([9, 8, 10])]
rows, rejected = m.select(questions, annotations, split="train", limit=10)
assert [r["question_id"] for r in rows] == [0]
assert rejected["human_agreement_below_threshold"] == 1
assert rejected["question_not_conservative_visible_state"] == 1
assert not m.conservative_question("Can this boy get injured easily?")
assert not m.conservative_question("Was this taken in the United States?")
assert not m.conservative_question("Is this scene in Paris?")
assert m.conservative_question("What is the horse doing?")
def test_speech_gate_does_not_hide_omitted_words():
m = load_script("encode_swa_vqav2")
assert m.canonical_speech("Are there two dogs?") == m.canonical_speech("Are there 2 dogs.")
assert m.canonical_speech("Is the dog red?") != m.canonical_speech("Is dog red?")
def test_camera_packet_grouping_causal_and_same_cadence():
m = load_script("encode_swa_vqav2")
available, groups = m.camera_groups(45)
assert groups[0] == [0, 1, 2, 3, 4]
assert set(map(len, groups)) == {3, 4, 5}
flat = [i for group in groups for i in group]
assert flat == list(range(len(flat)))
assert all(max(group) / 8 <= deadline + 1e-6 for group, deadline in zip(groups, available))
def test_factorized_packets_reconstruct_without_mutating_shared_bases(tmp_path):
from psi.data.seed.swa_frontend import load_swa_frontend
packet = {"embeds": torch.tensor([[0., 0.], [1., 1.]]), "visual_mask": torch.tensor([True, False]),
"audio_mask": torch.tensor([False, True]), "camera_indices": [0, 1, 2], "answer_target": ""}
base = tmp_path / "base.pt";visual = tmp_path / "visual.pt";front = tmp_path / "front.pt"
torch.save({"packets": [packet, packet], "system": "test"}, base)
torch.save({"3": torch.tensor([[3., 3.]])}, visual)
torch.save({"format": "static_factorized_v1", "base_frontend_path": str(base), "visual_features_path": str(visual),
"audio_patches": {1: torch.tensor([[4., 4.]])}, "reply_windows": [1], "answer_target": "yes", "timeline": {}}, front)
first = load_swa_frontend(front)["packets"]
assert first[0]["embeds"].tolist() == [[3., 3.], [1., 1.]]
assert first[1]["embeds"].tolist() == [[3., 3.], [4., 4.]]
assert first[0]["answer_target"] == "" and first[1]["answer_target"] == "yes"
first[0]["embeds"].zero_()
assert load_swa_frontend(front)["packets"][0]["embeds"].tolist() == [[3., 3.], [1., 1.]]
def test_swa_eval_indices_cover_entire_manifest_without_repeats():
from psi.trainers.swa_validation import validation_indices
assert validation_indices(0) == []
assert validation_indices(3) == [0, 1, 2]
indices = validation_indices(1000)
assert len(indices) == len(set(indices)) == 16
assert indices[-1] > 900