from __future__ import annotations import importlib.util import json import sys from pathlib import Path from types import SimpleNamespace import pytest import torch from psi.data.seed.swa_visual_qa_dataset import SwaVisualQaDataset from psi.data.seed.swa_visual_qa_dataset import bind_swa_batch, modality_batches PROJECT_ROOT = Path(__file__).resolve().parents[1] @pytest.mark.parametrize("auxiliary", [False, True]) def test_swa_losses_survive_accelerate_wrapper_and_backprop(monkeypatch, auxiliary): from accelerate.utils import convert_outputs_to_fp32 from psi.models.full_duplex_psi0 import FullDuplexPsi0Model, FullDuplexPsi0Output from psi.models.full_duplex_swa import FullDuplexSwaModel, swa_auxiliary_losses model = FullDuplexSwaModel.__new__(FullDuplexSwaModel) torch.nn.Module.__init__(model) model.reply_head = torch.nn.Linear(4, 2) model.qa_parameter = torch.nn.Parameter(torch.tensor(3.0)) model.swa_packets = [{}] model.swa_compute_auxiliary = auxiliary model.swa_reply_targets = torch.tensor([1]) model.swa_answer_targets = ["yes"] model.reply_logits = model.reply_head(torch.ones(1, 4)) model.qa_loss = lambda cache, targets: ( model.qa_parameter.square(), torch.tensor(1.0), torch.tensor(2.0) ) monkeypatch.setattr(FullDuplexPsi0Model, "forward", lambda *a, **kw: FullDuplexPsi0Output(action=torch.zeros(1, dtype=torch.bfloat16))) wrapped = convert_outputs_to_fp32(model.forward) output = wrapped(source_ids=["source"]) values = swa_auxiliary_losses(output, output.action, required=True) assert all(name in output for name in values) assert output.action.dtype == torch.float32 objective = values["swa_reply_loss"].sum() + values["swa_qa_loss_sum"] objective.backward() if auxiliary: assert values["swa_qa_token_count"].item() == 2 assert model.reply_head.weight.grad.abs().sum() > 0 assert model.qa_parameter.grad.abs() > 0 else: assert values["swa_qa_token_count"].item() == 0 assert model.qa_parameter.grad is None def test_swa_missing_loss_fails_closed_but_audio_only_is_compatible(): from psi.models.full_duplex_psi0 import FullDuplexPsi0Output from psi.models.full_duplex_swa import swa_auxiliary_losses output = FullDuplexPsi0Output(action=torch.zeros(2)) with pytest.raises(RuntimeError, match="was lost"): swa_auxiliary_losses(output, output.action, required=True) assert all(value.eq(0).all() for value in swa_auxiliary_losses(output, output.action, required=False).values()) @pytest.mark.parametrize("swa", [False, True]) def test_training_attaches_loaded_processor_before_first_forward(monkeypatch, swa): from psi.trainers import posttrain_full_duplex as trainer_module from psi.models import full_duplex_swa as swa_module processor = SimpleNamespace(tokenizer=object()) class InitializationReached(Exception): pass class Model: def __init__(self, **kwargs): pass def enable_full_source_gradient_checkpointing(self): assert self.vlm_processor is processor raise InitializationReached monkeypatch.setattr(trainer_module, "FullDuplexPsi0Model", Model) monkeypatch.setattr(swa_module, "FullDuplexSwaModel", Model) trainer = trainer_module.PosttrainFullDuplexTrainer.__new__( trainer_module.PosttrainFullDuplexTrainer ) trainer.cfg = SimpleNamespace( train=SimpleNamespace(full_duplex_semantics_version=9), data=SimpleNamespace(), model=SimpleNamespace(swa_visual_qa_enabled=swa), ) trainer.vlm_processor = processor trainer.init_vlm_models = lambda: object() with pytest.raises(InitializationReached): trainer.init_models() def test_swa_binding_masks_padding_and_clears_audio_transition(): model = SimpleNamespace() source = {"source_modality": "swa_visual_qa", "swa_packets": [ {"reply_label": 1, "answer_target": "yes"}, {"reply_label": 0, "answer_target": ""}, ]} assert bind_swa_batch(model, [source, None], [0, None], device="cpu", auxiliary=True) assert model.swa_reply_targets.tolist() == [1, 0] assert model.swa_answer_targets == ["yes", ""] assert model.swa_compute_auxiliary assert bind_swa_batch(model, [source], [1], device="cpu", auxiliary=False) assert model.swa_packets == [source["swa_packets"][1]] assert not model.swa_compute_auxiliary assert not bind_swa_batch(model, [{}], [0], device="cpu") assert model.swa_packets is None and model.swa_answer_targets is None with pytest.raises(RuntimeError, match="outside"): bind_swa_batch(model, [source], [2], device="cpu") def test_eval_modality_seams_preserve_every_source_and_order(): rows = [{"id": "a"}, {"id": "b"}, {"id": "c", "source_modality": "swa_visual_qa"}, {"id": "d", "source_modality": "swa_visual_qa"}] batches = list(modality_batches([{"sources": rows}])) assert [len(batch["sources"]) for batch in batches] == [2, 2] assert [row for batch in batches for row in batch["sources"]] == rows def test_all_padding_swa_rank_retains_modality_without_targets(): model = SimpleNamespace() shape_source = {"source_modality": "swa_visual_qa", "swa_packets": [ {"reply_label": 1, "answer_target": "do not supervise padding"}]} assert bind_swa_batch(model, [None, None], [0, 0], device="cpu", auxiliary=True, padding_source=shape_source) assert model.swa_answer_targets == ["", ""] assert model.swa_reply_targets.tolist() == [0, 0] assert len(model.swa_packets) == 2 assert not bind_swa_batch(model, [None], [0], device="cpu", padding_source={}) def _load_infer_suite(): path = PROJECT_ROOT / "scripts/infer/run_multiturn_infer_suite.py" spec = importlib.util.spec_from_file_location("hfd_swa_infer_suite_test", path) assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def _load_charades_builder(): path = PROJECT_ROOT / "scripts/data/build_swa_charades_ego.py" spec = importlib.util.spec_from_file_location("hfd_charades_builder_test", path) assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module def _load_charades_filter(): path = PROJECT_ROOT / "scripts/data/filter_swa_charades_reviews.py" spec = importlib.util.spec_from_file_location("hfd_charades_filter_test", path) assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module def _load_plan_merger(): path = PROJECT_ROOT / "scripts/data/merge_swa_source_plans.py" spec = importlib.util.spec_from_file_location("hfd_swa_plan_merger_test", path) assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module def test_formal_defaults_use_unique_swa_sources_and_one_inference_queue(): from psi.hfd_training_launcher import resolve recipe, _ = resolve(PROJECT_ROOT, {}) infer_script = (PROJECT_ROOT / "infer_render_auto.sh").read_text( encoding="utf-8" ) assert recipe["data.swa-train-repeat"] == 1 assert "run_multiturn_infer_suite.py" in infer_script assert "watch_swa_visual_qa.sh" not in infer_script assert "HFD_INCLUDE_AUDIO_SUITE" not in infer_script assert recipe["data.swa-source-plan"].endswith("hfd_swa_vqav2_v1/source_plan_5x.json") def test_swa_queue_keeps_video_and_adds_static_train_val(tmp_path): module = _load_infer_suite() plan = tmp_path / "plan.json" plan.write_text(json.dumps([ {"split": "train", "cohort": "video"}, {"split": "val", "cohort": "video"}, {"split": "train", "cohort": "vqav2_static"}, {"split": "val", "cohort": "vqav2_static"}, ])) cases = module.swa_cases_for_run({"data": {"swa_source_plan": str(plan)}}) assert [(c.data_split, c.index) for c in cases] == [("train", 0), ("val", 0), ("train", 1), ("val", 1)] def test_explicit_none_checkpoint_still_uses_fresh_run_warmup(): from psi.hfd_training_launcher import resolve for value in ("", "None"): config, _ = resolve(PROJECT_ROOT, {"HFD_RESUME_FROM_CHECKPOINT": value, "HFD_MAX_TRAINING_STEPS": "10000"}) assert config["train.warmup-steps"] == 100 def test_swa_jobs_share_dynamic_gpu_queue_without_faking_video_completion(monkeypatch, tmp_path): module = _load_infer_suite() (tmp_path / "argv.txt").write_text("fixture arguments") (tmp_path / "run_config.json").write_text("{}") launches = [] class CompleteProcess: returncode = 0 def __init__(self, command, *, cwd, env, start_new_session, pass_fds): assert start_new_session assert len(pass_fds) == 1 launches.append((command, cwd, env)) output = Path(command[command.index("--output") + 1]) output.mkdir(parents=True, exist_ok=True) (output / "metrics.json").write_text( json.dumps({"id": "fixture", "events": [{"seconds": .6}]}), encoding="utf-8" ) for name in ("generated_abs.npy", "body_pos.csv", "body_quat.csv", "joint_pos.csv"): (output / name).write_text("fixture") def poll(self): return self.returncode monkeypatch.setattr(module.subprocess, "Popen", CompleteProcess) checkpoint = tmp_path / "checkpoints" / "ckpt_5000" checkpoint.mkdir(parents=True) for name in ("model.safetensors", "hfd_fused_delta_manifest.json"): (checkpoint / name).write_text("fixture") (checkpoint / "full_duplex_source_boundary.json").write_text( json.dumps({"source_group_complete": True}) ) case = module.SwaInferenceCase(data_split="val", index=3) module.run_dynamic_queue(tmp_path, [case], step=5000, gpus=["2"]) result = module.case_result_dir(tmp_path, case, 5000) assert (result / ".inferred").is_file() assert not (result / ".complete").exists() assert not (result / ".failed").exists() command, cwd, environment = launches[0] assert command[0].endswith("/.venv-psi/bin/python") assert command[command.index("--split") + 1] == "val" assert command[command.index("--index") + 1] == "3" assert environment["CUDA_VISIBLE_DEVICES"] == "2" assert cwd == PROJECT_ROOT def test_unique_swa_source_keeps_qa_and_explicit_negative_removes_it(tmp_path): from psi.runtime.swa_rollout import inplace_future_progress body = { "source": { "source_id": "body", "steps": [{"source_id": "body", "phase": "motion", "frame_id": 0, "motion_duration_target_seconds": 2., "future_progress": inplace_future_progress(0, 100)}], } } body_path = tmp_path / "body.pt" torch.save(body, body_path) frontend_root = tmp_path / "frontend-root" frontend_dir = frontend_root / "data/frontend" frontend_dir.mkdir(parents=True) packet = { "embeds": torch.tensor([[1.0], [2.0]]), "ordinary_audio": torch.tensor([[9.0]]), "audio_mask": torch.tensor([False, True]), "reply_label": 1, "answer_target": "visible answer", } rows = [] for source_id, has_question in (("qa", None), ("negative", False)): row = { "id": source_id, "split": "train", "path": str(body_path), "frontend_root": str(frontend_root), } if has_question is not None: row["swa_has_question"] = has_question rows.append(row) torch.save({"packets": [packet]}, frontend_dir / f"{source_id}.pt") plan = tmp_path / "plan.json" plan.write_text(json.dumps(rows), encoding="utf-8") dataset = SwaVisualQaDataset(str(plan), "train", repeat=1) qa = dataset[0]["swa_packets"][0] negative = dataset[1]["swa_packets"][0] assert qa["reply_label"] == 1 assert qa["answer_target"] == "visible answer" assert negative["reply_label"] == 0 assert negative["answer_target"] == "" assert negative["embeds"][1, 0].item() == 9.0 def test_charades_builder_parses_official_intervals_conservatively(): module = _load_charades_builder() actions = module.parse_actions( "c001 1.0 4.5;c002 4.0 3.0;c999 0.0 2.0", {"c001": "Someone is opening a door.", "c002": "Someone sits."}, ) assert actions == [ { "action_id": "c001", "start": 1.0, "end": 4.5, "label": "Someone is opening a door.", } ] assert module.natural_action(actions[0]["label"]) == "opening a door" assert module.interval_distance(2.0, actions[0]) == 0.0 assert module.interval_distance(7.0, actions[0]) == 2.5 assert module.words("Someone is opening a door") == {"opening", "door"} def test_charades_review_filter_requires_support_for_qa_but_keeps_ordinary(): module = _load_charades_filter() candidates = [ {"id": "yes", "swa_has_question": True}, {"id": "bad", "swa_has_question": True}, {"id": "ordinary", "swa_has_question": False}, ] reviews = { "yes": { "model": "qwen3", "mode": "verify", "decoded": {"supported": True, "answer": "Yes", "reason": "visible"}, }, "bad": { "model": "qwen3", "mode": "verify", "decoded": {"supported": False, "answer": "No", "reason": "unclear"}, }, } accepted, rejected = module.admit(candidates, reviews) assert [row["id"] for row in accepted] == ["yes", "ordinary"] assert rejected == {"reference_not_visually_supported": 1} def test_swa_plan_merge_propagates_explicit_unique_negative_contract(): module = _load_plan_merger() merged = module.merge( [ [{"id": "legacy", "split": "train"}], [ { "id": "new-negative", "split": "train", "question": {"swa_has_question": False}, } ], ] ) assert merged[0]["swa_has_question"] is True assert merged[1]["swa_has_question"] is False