"""Weights-only joint-training initialization from a verified control adapter. Call prepare_joint_initialization before GPU handoff, then initialize_joint_weights immediately after the original trainer.init_models, BEFORE creating an optimizer. No accelerator.load_state/resume, old optimizer, scheduler, RNG or source cursor is read. The original trainer's Qwen-tail/action/control/duration trainability stays intact, so its normal compact checkpoint hook continues saving the Qwen delta. """ from __future__ import annotations import json from pathlib import Path import re from types import SimpleNamespace from scripts import train_third_party_control_repair as repair from scripts import verify_third_party_control_adapter as verify from scripts.data.hfd_third_party_common import canonical_json_sha256 PROJECT = Path(__file__).resolve().parents[1] POLICY = "joint_train_weights_only_from_verified_control_v1" # Original Psi0 intentionally stores both as Parameters, not buffers: fixed # positional encoding and _TimeNetwork's Fourier frequencies (learnable_w=False). FIXED_NON_QWEN_PARAMETERS = {"action_header.obs_proj.enc_pos.pe", "action_header.time_ins_embed.w"} def _dict(config): return config.model_dump(mode="json") if hasattr(config, "model_dump") else config def _fresh_config(config): train = _dict(config)["train"] if (train.get("resume_from_checkpoint") is not None or train.get("resume_load_full_state") is not False or train.get("resume_checkpoint_step", 0) != 0): raise ValueError("Joint warm start requires no resume path, full-state=False and resume step=0") def _contract(config): from psi.trainers.posttrain_third_party import PosttrainThirdPartyTrainer trainer = object.__new__(PosttrainThirdPartyTrainer) trainer.cfg = config return trainer._hfdv2_checkpoint_contract() def prepare_joint_initialization(config, *, base_checkpoint, adapter, verification_run, manifest, run_dir): """Read-only identity gate; does not require or create the new run directory.""" from safetensors import safe_open _fresh_config(config) checkpoint, adapter, verified, manifest, output = ( repair.local(path, output=output_only) for path, output_only in ((base_checkpoint, True), (adapter, True), (verification_run, True), (manifest, False), (run_dir, True))) if checkpoint.parent.name != "checkpoints" or not re.fullmatch(r"ckpt_[0-9]+", checkpoint.name): raise ValueError("Use an explicit complete base checkpoint directory") verify._completed(verified) record = json.loads((verified / "identity.json").read_text()) result = json.loads((verified / "result.json").read_text()) if (result.get("status") != "complete" or result.get("candidate") != "uniform_ce" or result.get("pilot_acceptance_passed") is not True or not result.get("acceptance_checks") or not all(v is True for v in result["acceptance_checks"].values()) or result.get("adapter") != str(adapter) or record.get("adapter") != str(adapter) or record.get("candidate") != "uniform_ce" or result.get("not_full_state_resume_checkpoint") is not True or result.get("old_weights_modified") is not False): raise ValueError("Only the completed, accepted uniform_ce adapter may initialize this joint phase") if record.get("verification_entry_sha256") != repair.digest(PROJECT / "scripts/verify_third_party_control_adapter.py"): raise ValueError("Verification implementation changed since the accepted adapter") for immutable in (verified, checkpoint.parent.parent, adapter.parent.parent): if output.is_relative_to(immutable) or immutable.is_relative_to(output): raise ValueError("New joint run must not overlap any immutable source run") prepared = verify.prepare_verification(SimpleNamespace(source_run=Path(record["source_run"]), candidate="uniform_ce", run_dir=output, manifest=manifest)) if (prepared["adapter"] != adapter or prepared["args"].base_run != checkpoint.parent.parent or prepared["args"].step != int(checkpoint.name.removeprefix("ckpt_")) or prepared["source_hashes"] != record.get("source_hashes")): raise ValueError("Verification sources no longer match base/adapter/manifest identity") active = _dict(config) saved = json.loads((checkpoint.parent.parent / "run_config.json").read_text()) # This field lives in model config but is solely the NEW optimizer's Qwen LR. comparable_model = lambda model: {k: v for k, v in model.items() if k != "lang_backbone_lr"} if comparable_model(active["model"]) != comparable_model(saved["model"]) or active["data"] != saved["data"]: raise ValueError("Joint initialization cannot change model architecture or frozen data") if Path(active["data"]["frozen_manifest"]).resolve() != manifest: raise ValueError("Active frozen manifest differs from warm-start selection") delta_manifest = json.loads((checkpoint / "hfd_fused_delta_manifest.json").read_text()) contract = _contract(config) if (delta_manifest.get("format") != "hfd_qwen_serial_delta_v1" or delta_manifest.get("contract") != contract or delta_manifest.get("qwen_trainable_last_layers") != 2 or delta_manifest.get("qwen_train_final_norm") is not True): raise ValueError("Base fused delta contract or trainable Qwen tail differs") boundary = json.loads((checkpoint / "full_duplex_source_boundary.json").read_text()) if (boundary.get("global_step") != prepared["args"].step or boundary.get("source_group_complete") is not True or boundary.get("train_infer_contract") != contract): raise ValueError("Base is not a complete contract-matching checkpoint") with safe_open(str(checkpoint / "model.safetensors"), framework="pt", device="cpu") as tensors: keys = sorted(tensors.keys()) shapes = {name: list(tensors.get_slice(name).get_shape()) for name in keys} if len(keys) != delta_manifest.get("tensor_count") or not any(k.startswith("vlm_model.") for k in keys): raise ValueError("Fused checkpoint tensor count mismatch or missing Qwen delta") seeds = {r["session_id"]: r["seed"] for r in prepared["reports"]["baseline_mixed"]["samples"]} selection = json.loads((verified / "selection.json").read_text()) if selection != {"fixed_sixty_ids": prepared["sixty"], "paired_twentyfour_ids": prepared["panel"]}: raise ValueError("Accepted verification panel changed") report_paths = [] for name, ids, view in (("candidate_mixed", prepared["sixty"], "mixed"), ("candidate_clean", prepared["panel"], "clean")): report, path = verify._report(verified, name, ids, seeds, view=view, adapter=str(adapter)) if report.get("aggregate") != result.get(name): raise ValueError("Accepted adapter metrics differ from its completed reports") report_paths.append(path) hashes = dict(prepared["source_hashes"]) for path in [checkpoint / "hfd_fused_delta_manifest.json", checkpoint / "full_duplex_source_boundary.json", verified / "identity.json", verified / "result.json", verified / "selection.json", verified / "run_status.json", *report_paths]: hashes[str(path)] = repair.digest(path) return {"schema": 1, "policy": POLICY, "base_checkpoint": str(checkpoint), "base_step": prepared["args"].step, "adapter": str(adapter), "verification_run": str(verified), "manifest": str(manifest), "manifest_sha256": prepared["identity"]["manifest_sha256"], "run_dir": str(output), "new_global_step": 0, "source_hashes": hashes, "delta_keys": keys, "delta_shapes": shapes, "train_infer_contract": contract, "model_data_sha256": canonical_json_sha256({k: active[k] for k in ("model", "data")}), "qwen_learning_rate_change": {"saved": saved["model"].get("lang_backbone_lr"), "active": active["model"].get("lang_backbone_lr")}, "optimizer_restored": False, "scheduler_restored": False, "rng_restored": False, "source_cursor_restored": False, "initialization_order": "original pretrained backbone -> complete fused delta -> verified uniform control head", "trainable_scope": "original Qwen last2+norm, action, control, duration and existing auxiliary parameters", "adapter_not_a_full_checkpoint": True} def _trainability(model, config): cfg = _dict(config)["model"] if cfg.get("streaming_qwen_trainable_last_layers") != 2 or cfg.get("streaming_qwen_train_final_norm") is not True: raise ValueError("Joint phase must retain the original Qwen last2+norm trainability") layers = model.vlm_model.model.layers if len(layers) < 2: raise ValueError("Missing original Qwen decoder layers") prefixes = tuple(f"vlm_model.model.layers.{i}." for i in range(len(layers) - 2, len(layers))) + ("vlm_model.model.norm.",) parameters = dict(model.named_parameters()) expected_qwen = {name for name in parameters if name.startswith(prefixes)} actual_qwen = {name for name, p in parameters.items() if name.startswith("vlm_model.") and p.requires_grad} if not expected_qwen or actual_qwen != expected_qwen: raise ValueError("Qwen delta would be lost or frozen prefix trained: trainable parameter set differs") frozen_nonqwen = {name for name, p in parameters.items() if not name.startswith("vlm_model.") and not p.requires_grad} if frozen_nonqwen - FIXED_NON_QWEN_PARAMETERS: raise ValueError(f"Unexpected frozen action/control/duration parameters: {sorted(frozen_nonqwen)}") for prefix in ("action_header.", "control_head.", "motion_duration_head."): if not any(name.startswith(prefix) and p.requires_grad for name, p in parameters.items()): raise ValueError(f"Required joint module is missing/frozen: {prefix}") expected_delta = {name for name in model.state_dict() if not name.startswith("vlm_model.") or name in expected_qwen} return parameters, expected_qwen, expected_delta def initialize_joint_weights(trainer, identity): """Load only tensors into the freshly initialized ORIGINAL trainer model. Refuses an existing optimizer/scheduler, nonzero step, wrapped model, changed input identities or any frozen Qwen-tail/action module. Parameter objects and requires_grad flags are preserved; register normal full-state hooks later via trainer.prepare(accelerator), with a newly created optimizer and scheduler. """ from safetensors.torch import load_file import torch if identity.get("policy") != POLICY: raise ValueError("Unknown joint initialization identity") _fresh_config(trainer.cfg) if (getattr(trainer, "optimizer", None) is not None or getattr(trainer, "lr_scheduler", None) is not None or getattr(trainer, "global_step", 0) != 0): raise ValueError("Initialize joint weights before a fresh optimizer/scheduler at global_step=0") cfg = _dict(trainer.cfg) if canonical_json_sha256({k: cfg[k] for k in ("model", "data")}) != identity["model_data_sha256"]: raise ValueError("Model/data config changed after warm-start preflight") if trainer._hfdv2_checkpoint_contract() != identity["train_infer_contract"]: raise ValueError("Runtime joint checkpoint contract changed") for path, expected in identity["source_hashes"].items(): if repair.digest(repair.local(path)) != expected: raise ValueError(f"Warm-start source changed after preflight: {path}") model = trainer.model parameters, qwen_keys, expected_delta = _trainability(model, trainer.cfg) if expected_delta != set(identity["delta_keys"]): raise ValueError("Active compact-save key set differs from the COMPLETE source delta") parameter_identity = {name: (id(p), p.requires_grad) for name, p in parameters.items()} checkpoint = repair.local(identity["base_checkpoint"], output=True) state = load_file(str(checkpoint / "model.safetensors"), device="cpu") adapter = load_file(str(repair.local(identity["adapter"], output=True)), device="cpu") expected_head = set(model.control_head.state_dict()) if set(state) != expected_delta or set(adapter) != expected_head: raise ValueError("Warm-start tensor keys do not match base/control head") active_state = model.state_dict() for name, tensor in state.items(): if (list(tensor.shape) != identity["delta_shapes"][name] or tensor.shape != active_state[name].shape or (tensor.is_floating_point() and not torch.isfinite(tensor).all())): raise ValueError(f"Invalid fused delta tensor: {name}") for name, tensor in adapter.items(): if tensor.shape != active_state[f"control_head.{name}"].shape or not torch.isfinite(tensor).all(): raise ValueError(f"Invalid verified control tensor: {name}") incompatible = model.load_state_dict(state, strict=False) if set(incompatible.missing_keys) != set(active_state) - expected_delta or incompatible.unexpected_keys: raise RuntimeError("Unexpected missing/unexpected keys while applying complete fused delta") model.control_head.load_state_dict(adapter, strict=True) after, after_qwen, after_delta = _trainability(model, trainer.cfg) if ({name: (id(p), p.requires_grad) for name, p in after.items()} != parameter_identity or after_qwen != qwen_keys or after_delta != expected_delta): raise RuntimeError("Warm initialization changed parameter identities/trainability/save scope") return {"policy": POLICY, "base_step": identity["base_step"], "global_step": 0, "loaded_delta_tensors": len(state), "loaded_qwen_delta_tensors": len(qwen_keys), "overlaid_control_tensors": len(adapter), "future_saved_delta_tensors": len(after_delta), "qwen_trainable_parameter_names": sorted(qwen_keys), "trainable_parameters": sum(p.numel() for p in after.values() if p.requires_grad), "trainable_parameter_tensors": sum(p.requires_grad for p in after.values()), "parameter_identities_preserved": True, "requires_grad_preserved": True, "optimizer_restored": False, "scheduler_restored": False, "rng_restored": False, "source_cursor_restored": False}