code-HFD_lzk / scripts /third_party_joint_init.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
55e0599 verified
Raw History Blame Contribute Delete
14.6 kB
"""Weights-only joint-training initialization from a verified control adapter.
Call prepare_joint_initialization before GPU handoff, then initialize_joint_weights
immediately after the original trainer.init_models, BEFORE creating an optimizer.
No accelerator.load_state/resume, old optimizer, scheduler, RNG or source cursor is
read. The original trainer's Qwen-tail/action/control/duration trainability stays
intact, so its normal compact checkpoint hook continues saving the Qwen delta.
"""
from __future__ import annotations
import json
from pathlib import Path
import re
from types import SimpleNamespace
from scripts import train_third_party_control_repair as repair
from scripts import verify_third_party_control_adapter as verify
from scripts.data.hfd_third_party_common import canonical_json_sha256
PROJECT = Path(__file__).resolve().parents[1]
POLICY = "joint_train_weights_only_from_verified_control_v1"
# Original Psi0 intentionally stores both as Parameters, not buffers: fixed
# positional encoding and _TimeNetwork's Fourier frequencies (learnable_w=False).
FIXED_NON_QWEN_PARAMETERS = {"action_header.obs_proj.enc_pos.pe", "action_header.time_ins_embed.w"}
def _dict(config):
return config.model_dump(mode="json") if hasattr(config, "model_dump") else config
def _fresh_config(config):
train = _dict(config)["train"]
if (train.get("resume_from_checkpoint") is not None or train.get("resume_load_full_state") is not False
or train.get("resume_checkpoint_step", 0) != 0):
raise ValueError("Joint warm start requires no resume path, full-state=False and resume step=0")
def _contract(config):
from psi.trainers.posttrain_third_party import PosttrainThirdPartyTrainer
trainer = object.__new__(PosttrainThirdPartyTrainer)
trainer.cfg = config
return trainer._hfdv2_checkpoint_contract()
def prepare_joint_initialization(config, *, base_checkpoint, adapter, verification_run,
manifest, run_dir):
"""Read-only identity gate; does not require or create the new run directory."""
from safetensors import safe_open
_fresh_config(config)
checkpoint, adapter, verified, manifest, output = (
repair.local(path, output=output_only) for path, output_only in
((base_checkpoint, True), (adapter, True), (verification_run, True), (manifest, False), (run_dir, True)))
if checkpoint.parent.name != "checkpoints" or not re.fullmatch(r"ckpt_[0-9]+", checkpoint.name):
raise ValueError("Use an explicit complete base checkpoint directory")
verify._completed(verified)
record = json.loads((verified / "identity.json").read_text())
result = json.loads((verified / "result.json").read_text())
if (result.get("status") != "complete" or result.get("candidate") != "uniform_ce"
or result.get("pilot_acceptance_passed") is not True
or not result.get("acceptance_checks") or not all(v is True for v in result["acceptance_checks"].values())
or result.get("adapter") != str(adapter) or record.get("adapter") != str(adapter)
or record.get("candidate") != "uniform_ce"
or result.get("not_full_state_resume_checkpoint") is not True
or result.get("old_weights_modified") is not False):
raise ValueError("Only the completed, accepted uniform_ce adapter may initialize this joint phase")
if record.get("verification_entry_sha256") != repair.digest(PROJECT / "scripts/verify_third_party_control_adapter.py"):
raise ValueError("Verification implementation changed since the accepted adapter")
for immutable in (verified, checkpoint.parent.parent, adapter.parent.parent):
if output.is_relative_to(immutable) or immutable.is_relative_to(output):
raise ValueError("New joint run must not overlap any immutable source run")
prepared = verify.prepare_verification(SimpleNamespace(source_run=Path(record["source_run"]),
candidate="uniform_ce", run_dir=output, manifest=manifest))
if (prepared["adapter"] != adapter or prepared["args"].base_run != checkpoint.parent.parent
or prepared["args"].step != int(checkpoint.name.removeprefix("ckpt_"))
or prepared["source_hashes"] != record.get("source_hashes")):
raise ValueError("Verification sources no longer match base/adapter/manifest identity")
active = _dict(config)
saved = json.loads((checkpoint.parent.parent / "run_config.json").read_text())
# This field lives in model config but is solely the NEW optimizer's Qwen LR.
comparable_model = lambda model: {k: v for k, v in model.items() if k != "lang_backbone_lr"}
if comparable_model(active["model"]) != comparable_model(saved["model"]) or active["data"] != saved["data"]:
raise ValueError("Joint initialization cannot change model architecture or frozen data")
if Path(active["data"]["frozen_manifest"]).resolve() != manifest:
raise ValueError("Active frozen manifest differs from warm-start selection")
delta_manifest = json.loads((checkpoint / "hfd_fused_delta_manifest.json").read_text())
contract = _contract(config)
if (delta_manifest.get("format") != "hfd_qwen_serial_delta_v1"
or delta_manifest.get("contract") != contract
or delta_manifest.get("qwen_trainable_last_layers") != 2
or delta_manifest.get("qwen_train_final_norm") is not True):
raise ValueError("Base fused delta contract or trainable Qwen tail differs")
boundary = json.loads((checkpoint / "full_duplex_source_boundary.json").read_text())
if (boundary.get("global_step") != prepared["args"].step
or boundary.get("source_group_complete") is not True
or boundary.get("train_infer_contract") != contract):
raise ValueError("Base is not a complete contract-matching checkpoint")
with safe_open(str(checkpoint / "model.safetensors"), framework="pt", device="cpu") as tensors:
keys = sorted(tensors.keys())
shapes = {name: list(tensors.get_slice(name).get_shape()) for name in keys}
if len(keys) != delta_manifest.get("tensor_count") or not any(k.startswith("vlm_model.") for k in keys):
raise ValueError("Fused checkpoint tensor count mismatch or missing Qwen delta")
seeds = {r["session_id"]: r["seed"] for r in prepared["reports"]["baseline_mixed"]["samples"]}
selection = json.loads((verified / "selection.json").read_text())
if selection != {"fixed_sixty_ids": prepared["sixty"], "paired_twentyfour_ids": prepared["panel"]}:
raise ValueError("Accepted verification panel changed")
report_paths = []
for name, ids, view in (("candidate_mixed", prepared["sixty"], "mixed"),
("candidate_clean", prepared["panel"], "clean")):
report, path = verify._report(verified, name, ids, seeds, view=view, adapter=str(adapter))
if report.get("aggregate") != result.get(name):
raise ValueError("Accepted adapter metrics differ from its completed reports")
report_paths.append(path)
hashes = dict(prepared["source_hashes"])
for path in [checkpoint / "hfd_fused_delta_manifest.json", checkpoint / "full_duplex_source_boundary.json",
verified / "identity.json", verified / "result.json", verified / "selection.json",
verified / "run_status.json", *report_paths]:
hashes[str(path)] = repair.digest(path)
return {"schema": 1, "policy": POLICY, "base_checkpoint": str(checkpoint),
"base_step": prepared["args"].step, "adapter": str(adapter), "verification_run": str(verified),
"manifest": str(manifest), "manifest_sha256": prepared["identity"]["manifest_sha256"],
"run_dir": str(output), "new_global_step": 0,
"source_hashes": hashes, "delta_keys": keys, "delta_shapes": shapes,
"train_infer_contract": contract,
"model_data_sha256": canonical_json_sha256({k: active[k] for k in ("model", "data")}),
"qwen_learning_rate_change": {"saved": saved["model"].get("lang_backbone_lr"),
"active": active["model"].get("lang_backbone_lr")},
"optimizer_restored": False, "scheduler_restored": False, "rng_restored": False,
"source_cursor_restored": False,
"initialization_order": "original pretrained backbone -> complete fused delta -> verified uniform control head",
"trainable_scope": "original Qwen last2+norm, action, control, duration and existing auxiliary parameters",
"adapter_not_a_full_checkpoint": True}
def _trainability(model, config):
cfg = _dict(config)["model"]
if cfg.get("streaming_qwen_trainable_last_layers") != 2 or cfg.get("streaming_qwen_train_final_norm") is not True:
raise ValueError("Joint phase must retain the original Qwen last2+norm trainability")
layers = model.vlm_model.model.layers
if len(layers) < 2:
raise ValueError("Missing original Qwen decoder layers")
prefixes = tuple(f"vlm_model.model.layers.{i}." for i in range(len(layers) - 2, len(layers))) + ("vlm_model.model.norm.",)
parameters = dict(model.named_parameters())
expected_qwen = {name for name in parameters if name.startswith(prefixes)}
actual_qwen = {name for name, p in parameters.items() if name.startswith("vlm_model.") and p.requires_grad}
if not expected_qwen or actual_qwen != expected_qwen:
raise ValueError("Qwen delta would be lost or frozen prefix trained: trainable parameter set differs")
frozen_nonqwen = {name for name, p in parameters.items() if not name.startswith("vlm_model.") and not p.requires_grad}
if frozen_nonqwen - FIXED_NON_QWEN_PARAMETERS:
raise ValueError(f"Unexpected frozen action/control/duration parameters: {sorted(frozen_nonqwen)}")
for prefix in ("action_header.", "control_head.", "motion_duration_head."):
if not any(name.startswith(prefix) and p.requires_grad for name, p in parameters.items()):
raise ValueError(f"Required joint module is missing/frozen: {prefix}")
expected_delta = {name for name in model.state_dict() if not name.startswith("vlm_model.") or name in expected_qwen}
return parameters, expected_qwen, expected_delta
def initialize_joint_weights(trainer, identity):
"""Load only tensors into the freshly initialized ORIGINAL trainer model.
Refuses an existing optimizer/scheduler, nonzero step, wrapped model, changed
input identities or any frozen Qwen-tail/action module. Parameter objects and
requires_grad flags are preserved; register normal full-state hooks later via
trainer.prepare(accelerator), with a newly created optimizer and scheduler.
"""
from safetensors.torch import load_file
import torch
if identity.get("policy") != POLICY:
raise ValueError("Unknown joint initialization identity")
_fresh_config(trainer.cfg)
if (getattr(trainer, "optimizer", None) is not None or getattr(trainer, "lr_scheduler", None) is not None
or getattr(trainer, "global_step", 0) != 0):
raise ValueError("Initialize joint weights before a fresh optimizer/scheduler at global_step=0")
cfg = _dict(trainer.cfg)
if canonical_json_sha256({k: cfg[k] for k in ("model", "data")}) != identity["model_data_sha256"]:
raise ValueError("Model/data config changed after warm-start preflight")
if trainer._hfdv2_checkpoint_contract() != identity["train_infer_contract"]:
raise ValueError("Runtime joint checkpoint contract changed")
for path, expected in identity["source_hashes"].items():
if repair.digest(repair.local(path)) != expected:
raise ValueError(f"Warm-start source changed after preflight: {path}")
model = trainer.model
parameters, qwen_keys, expected_delta = _trainability(model, trainer.cfg)
if expected_delta != set(identity["delta_keys"]):
raise ValueError("Active compact-save key set differs from the COMPLETE source delta")
parameter_identity = {name: (id(p), p.requires_grad) for name, p in parameters.items()}
checkpoint = repair.local(identity["base_checkpoint"], output=True)
state = load_file(str(checkpoint / "model.safetensors"), device="cpu")
adapter = load_file(str(repair.local(identity["adapter"], output=True)), device="cpu")
expected_head = set(model.control_head.state_dict())
if set(state) != expected_delta or set(adapter) != expected_head:
raise ValueError("Warm-start tensor keys do not match base/control head")
active_state = model.state_dict()
for name, tensor in state.items():
if (list(tensor.shape) != identity["delta_shapes"][name] or tensor.shape != active_state[name].shape
or (tensor.is_floating_point() and not torch.isfinite(tensor).all())):
raise ValueError(f"Invalid fused delta tensor: {name}")
for name, tensor in adapter.items():
if tensor.shape != active_state[f"control_head.{name}"].shape or not torch.isfinite(tensor).all():
raise ValueError(f"Invalid verified control tensor: {name}")
incompatible = model.load_state_dict(state, strict=False)
if set(incompatible.missing_keys) != set(active_state) - expected_delta or incompatible.unexpected_keys:
raise RuntimeError("Unexpected missing/unexpected keys while applying complete fused delta")
model.control_head.load_state_dict(adapter, strict=True)
after, after_qwen, after_delta = _trainability(model, trainer.cfg)
if ({name: (id(p), p.requires_grad) for name, p in after.items()} != parameter_identity
or after_qwen != qwen_keys or after_delta != expected_delta):
raise RuntimeError("Warm initialization changed parameter identities/trainability/save scope")
return {"policy": POLICY, "base_step": identity["base_step"], "global_step": 0,
"loaded_delta_tensors": len(state), "loaded_qwen_delta_tensors": len(qwen_keys),
"overlaid_control_tensors": len(adapter), "future_saved_delta_tensors": len(after_delta),
"qwen_trainable_parameter_names": sorted(qwen_keys),
"trainable_parameters": sum(p.numel() for p in after.values() if p.requires_grad),
"trainable_parameter_tensors": sum(p.requires_grad for p in after.values()),
"parameter_identities_preserved": True, "requires_grad_preserved": True,
"optimizer_restored": False, "scheduler_restored": False, "rng_restored": False,
"source_cursor_restored": False}