code-HFD / tests /test_swa_formal_integration.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
aa02c28 verified
Raw History Blame Contribute Delete
14.6 kB
from __future__ import annotations
import importlib.util
import json
import sys
from pathlib import Path
from types import SimpleNamespace
import pytest
import torch
from psi.data.seed.swa_visual_qa_dataset import SwaVisualQaDataset
from psi.data.seed.swa_visual_qa_dataset import bind_swa_batch, modality_batches
PROJECT_ROOT = Path(__file__).resolve().parents[1]
@pytest.mark.parametrize("auxiliary", [False, True])
def test_swa_losses_survive_accelerate_wrapper_and_backprop(monkeypatch, auxiliary):
from accelerate.utils import convert_outputs_to_fp32
from psi.models.full_duplex_psi0 import FullDuplexPsi0Model, FullDuplexPsi0Output
from psi.models.full_duplex_swa import FullDuplexSwaModel, swa_auxiliary_losses
model = FullDuplexSwaModel.__new__(FullDuplexSwaModel)
torch.nn.Module.__init__(model)
model.reply_head = torch.nn.Linear(4, 2)
model.qa_parameter = torch.nn.Parameter(torch.tensor(3.0))
model.swa_packets = [{}]
model.swa_compute_auxiliary = auxiliary
model.swa_reply_targets = torch.tensor([1])
model.swa_answer_targets = ["yes"]
model.reply_logits = model.reply_head(torch.ones(1, 4))
model.qa_loss = lambda cache, targets: (
model.qa_parameter.square(), torch.tensor(1.0), torch.tensor(2.0)
)
monkeypatch.setattr(FullDuplexPsi0Model, "forward", lambda *a, **kw:
FullDuplexPsi0Output(action=torch.zeros(1, dtype=torch.bfloat16)))
wrapped = convert_outputs_to_fp32(model.forward)
output = wrapped(source_ids=["source"])
values = swa_auxiliary_losses(output, output.action, required=True)
assert all(name in output for name in values)
assert output.action.dtype == torch.float32
objective = values["swa_reply_loss"].sum() + values["swa_qa_loss_sum"]
objective.backward()
if auxiliary:
assert values["swa_qa_token_count"].item() == 2
assert model.reply_head.weight.grad.abs().sum() > 0
assert model.qa_parameter.grad.abs() > 0
else:
assert values["swa_qa_token_count"].item() == 0
assert model.qa_parameter.grad is None
def test_swa_missing_loss_fails_closed_but_audio_only_is_compatible():
from psi.models.full_duplex_psi0 import FullDuplexPsi0Output
from psi.models.full_duplex_swa import swa_auxiliary_losses
output = FullDuplexPsi0Output(action=torch.zeros(2))
with pytest.raises(RuntimeError, match="was lost"):
swa_auxiliary_losses(output, output.action, required=True)
assert all(value.eq(0).all() for value in
swa_auxiliary_losses(output, output.action, required=False).values())
@pytest.mark.parametrize("swa", [False, True])
def test_training_attaches_loaded_processor_before_first_forward(monkeypatch, swa):
from psi.trainers import posttrain_full_duplex as trainer_module
from psi.models import full_duplex_swa as swa_module
processor = SimpleNamespace(tokenizer=object())
class InitializationReached(Exception):
pass
class Model:
def __init__(self, **kwargs):
pass
def enable_full_source_gradient_checkpointing(self):
assert self.vlm_processor is processor
raise InitializationReached
monkeypatch.setattr(trainer_module, "FullDuplexPsi0Model", Model)
monkeypatch.setattr(swa_module, "FullDuplexSwaModel", Model)
trainer = trainer_module.PosttrainFullDuplexTrainer.__new__(
trainer_module.PosttrainFullDuplexTrainer
)
trainer.cfg = SimpleNamespace(
train=SimpleNamespace(full_duplex_semantics_version=9),
data=SimpleNamespace(),
model=SimpleNamespace(swa_visual_qa_enabled=swa),
)
trainer.vlm_processor = processor
trainer.init_vlm_models = lambda: object()
with pytest.raises(InitializationReached):
trainer.init_models()
def test_swa_binding_masks_padding_and_clears_audio_transition():
model = SimpleNamespace()
source = {"source_modality": "swa_visual_qa", "swa_packets": [
{"reply_label": 1, "answer_target": "yes"},
{"reply_label": 0, "answer_target": ""},
]}
assert bind_swa_batch(model, [source, None], [0, None], device="cpu", auxiliary=True)
assert model.swa_reply_targets.tolist() == [1, 0]
assert model.swa_answer_targets == ["yes", ""]
assert model.swa_compute_auxiliary
assert bind_swa_batch(model, [source], [1], device="cpu", auxiliary=False)
assert model.swa_packets == [source["swa_packets"][1]]
assert not model.swa_compute_auxiliary
assert not bind_swa_batch(model, [{}], [0], device="cpu")
assert model.swa_packets is None and model.swa_answer_targets is None
with pytest.raises(RuntimeError, match="outside"):
bind_swa_batch(model, [source], [2], device="cpu")
def test_eval_modality_seams_preserve_every_source_and_order():
rows = [{"id": "a"}, {"id": "b"},
{"id": "c", "source_modality": "swa_visual_qa"},
{"id": "d", "source_modality": "swa_visual_qa"}]
batches = list(modality_batches([{"sources": rows}]))
assert [len(batch["sources"]) for batch in batches] == [2, 2]
assert [row for batch in batches for row in batch["sources"]] == rows
def test_all_padding_swa_rank_retains_modality_without_targets():
model = SimpleNamespace()
shape_source = {"source_modality": "swa_visual_qa", "swa_packets": [
{"reply_label": 1, "answer_target": "do not supervise padding"}]}
assert bind_swa_batch(model, [None, None], [0, 0], device="cpu", auxiliary=True,
padding_source=shape_source)
assert model.swa_answer_targets == ["", ""]
assert model.swa_reply_targets.tolist() == [0, 0]
assert len(model.swa_packets) == 2
assert not bind_swa_batch(model, [None], [0], device="cpu", padding_source={})
def _load_infer_suite():
path = PROJECT_ROOT / "scripts/infer/run_multiturn_infer_suite.py"
spec = importlib.util.spec_from_file_location("hfd_swa_infer_suite_test", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def _load_charades_builder():
path = PROJECT_ROOT / "scripts/data/build_swa_charades_ego.py"
spec = importlib.util.spec_from_file_location("hfd_charades_builder_test", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def _load_charades_filter():
path = PROJECT_ROOT / "scripts/data/filter_swa_charades_reviews.py"
spec = importlib.util.spec_from_file_location("hfd_charades_filter_test", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def _load_plan_merger():
path = PROJECT_ROOT / "scripts/data/merge_swa_source_plans.py"
spec = importlib.util.spec_from_file_location("hfd_swa_plan_merger_test", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_formal_defaults_use_unique_swa_sources_and_one_inference_queue():
from psi.hfd_training_launcher import resolve
recipe, _ = resolve(PROJECT_ROOT, {})
infer_script = (PROJECT_ROOT / "infer_render_auto.sh").read_text(
encoding="utf-8"
)
assert recipe["data.swa-train-repeat"] == 1
assert "run_multiturn_infer_suite.py" in infer_script
assert "watch_swa_visual_qa.sh" not in infer_script
assert "HFD_INCLUDE_AUDIO_SUITE" not in infer_script
assert recipe["data.swa-source-plan"].endswith("hfd_swa_vqav2_v1/source_plan_5x.json")
def test_swa_queue_keeps_video_and_adds_static_train_val(tmp_path):
module = _load_infer_suite()
plan = tmp_path / "plan.json"
plan.write_text(json.dumps([
{"split": "train", "cohort": "video"},
{"split": "val", "cohort": "video"},
{"split": "train", "cohort": "vqav2_static"},
{"split": "val", "cohort": "vqav2_static"},
]))
cases = module.swa_cases_for_run({"data": {"swa_source_plan": str(plan)}})
assert [(c.data_split, c.index) for c in cases] == [("train", 0), ("val", 0), ("train", 1), ("val", 1)]
def test_explicit_none_checkpoint_still_uses_fresh_run_warmup():
from psi.hfd_training_launcher import resolve
for value in ("", "None"):
config, _ = resolve(PROJECT_ROOT, {"HFD_RESUME_FROM_CHECKPOINT": value,
"HFD_MAX_TRAINING_STEPS": "10000"})
assert config["train.warmup-steps"] == 100
def test_swa_jobs_share_dynamic_gpu_queue_without_faking_video_completion(monkeypatch, tmp_path):
module = _load_infer_suite()
(tmp_path / "argv.txt").write_text("fixture arguments")
(tmp_path / "run_config.json").write_text("{}")
launches = []
class CompleteProcess:
returncode = 0
def __init__(self, command, *, cwd, env, start_new_session, pass_fds):
assert start_new_session
assert len(pass_fds) == 1
launches.append((command, cwd, env))
output = Path(command[command.index("--output") + 1])
output.mkdir(parents=True, exist_ok=True)
(output / "metrics.json").write_text(
json.dumps({"id": "fixture", "events": [{"seconds": .6}]}), encoding="utf-8"
)
for name in ("generated_abs.npy", "body_pos.csv", "body_quat.csv", "joint_pos.csv"):
(output / name).write_text("fixture")
def poll(self):
return self.returncode
monkeypatch.setattr(module.subprocess, "Popen", CompleteProcess)
checkpoint = tmp_path / "checkpoints" / "ckpt_5000"
checkpoint.mkdir(parents=True)
for name in ("model.safetensors", "hfd_fused_delta_manifest.json"):
(checkpoint / name).write_text("fixture")
(checkpoint / "full_duplex_source_boundary.json").write_text(
json.dumps({"source_group_complete": True})
)
case = module.SwaInferenceCase(data_split="val", index=3)
module.run_dynamic_queue(tmp_path, [case], step=5000, gpus=["2"])
result = module.case_result_dir(tmp_path, case, 5000)
assert (result / ".inferred").is_file()
assert not (result / ".complete").exists()
assert not (result / ".failed").exists()
command, cwd, environment = launches[0]
assert command[0].endswith("/.venv-psi/bin/python")
assert command[command.index("--split") + 1] == "val"
assert command[command.index("--index") + 1] == "3"
assert environment["CUDA_VISIBLE_DEVICES"] == "2"
assert cwd == PROJECT_ROOT
def test_unique_swa_source_keeps_qa_and_explicit_negative_removes_it(tmp_path):
from psi.runtime.swa_rollout import inplace_future_progress
body = {
"source": {
"source_id": "body",
"steps": [{"source_id": "body", "phase": "motion", "frame_id": 0,
"motion_duration_target_seconds": 2.,
"future_progress": inplace_future_progress(0, 100)}],
}
}
body_path = tmp_path / "body.pt"
torch.save(body, body_path)
frontend_root = tmp_path / "frontend-root"
frontend_dir = frontend_root / "data/frontend"
frontend_dir.mkdir(parents=True)
packet = {
"embeds": torch.tensor([[1.0], [2.0]]),
"ordinary_audio": torch.tensor([[9.0]]),
"audio_mask": torch.tensor([False, True]),
"reply_label": 1,
"answer_target": "visible answer",
}
rows = []
for source_id, has_question in (("qa", None), ("negative", False)):
row = {
"id": source_id,
"split": "train",
"path": str(body_path),
"frontend_root": str(frontend_root),
}
if has_question is not None:
row["swa_has_question"] = has_question
rows.append(row)
torch.save({"packets": [packet]}, frontend_dir / f"{source_id}.pt")
plan = tmp_path / "plan.json"
plan.write_text(json.dumps(rows), encoding="utf-8")
dataset = SwaVisualQaDataset(str(plan), "train", repeat=1)
qa = dataset[0]["swa_packets"][0]
negative = dataset[1]["swa_packets"][0]
assert qa["reply_label"] == 1
assert qa["answer_target"] == "visible answer"
assert negative["reply_label"] == 0
assert negative["answer_target"] == ""
assert negative["embeds"][1, 0].item() == 9.0
def test_charades_builder_parses_official_intervals_conservatively():
module = _load_charades_builder()
actions = module.parse_actions(
"c001 1.0 4.5;c002 4.0 3.0;c999 0.0 2.0",
{"c001": "Someone is opening a door.", "c002": "Someone sits."},
)
assert actions == [
{
"action_id": "c001",
"start": 1.0,
"end": 4.5,
"label": "Someone is opening a door.",
}
]
assert module.natural_action(actions[0]["label"]) == "opening a door"
assert module.interval_distance(2.0, actions[0]) == 0.0
assert module.interval_distance(7.0, actions[0]) == 2.5
assert module.words("Someone is opening a door") == {"opening", "door"}
def test_charades_review_filter_requires_support_for_qa_but_keeps_ordinary():
module = _load_charades_filter()
candidates = [
{"id": "yes", "swa_has_question": True},
{"id": "bad", "swa_has_question": True},
{"id": "ordinary", "swa_has_question": False},
]
reviews = {
"yes": {
"model": "qwen3",
"mode": "verify",
"decoded": {"supported": True, "answer": "Yes", "reason": "visible"},
},
"bad": {
"model": "qwen3",
"mode": "verify",
"decoded": {"supported": False, "answer": "No", "reason": "unclear"},
},
}
accepted, rejected = module.admit(candidates, reviews)
assert [row["id"] for row in accepted] == ["yes", "ordinary"]
assert rejected == {"reference_not_visually_supported": 1}
def test_swa_plan_merge_propagates_explicit_unique_negative_contract():
module = _load_plan_merger()
merged = module.merge(
[
[{"id": "legacy", "split": "train"}],
[
{
"id": "new-negative",
"split": "train",
"question": {"swa_has_question": False},
}
],
]
)
assert merged[0]["swa_has_question"] is True
assert merged[1]["swa_has_question"] is False