Download src/psi/models/interaction_arch/runtime.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 22.1 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/models/interaction_arch/runtime.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/src/psi/models/interaction_arch/runtime.py
-
curl -L -o runtime.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/models/interaction_arch/runtime.py
22.1 kB
| """Isolated full-Trainer A1/A2 runtime and fail-closed checkpoint contract. | |
| A1: genuinely aliased frozen prefix plus independent tail-eight, optionally | |
| penultimate causal cross. A2: one body AV pass; detached KV/text-base references | |
| feed private QA LoRA. No loss, data, optimizer or checkpoint cadence is changed. | |
| """ | |
| from __future__ import annotations | |
| from contextlib import contextmanager | |
| from dataclasses import asdict, dataclass | |
| from functools import wraps | |
| import hashlib | |
| from functools import lru_cache | |
| import json | |
| import os | |
| from pathlib import Path | |
| import re | |
| import sys | |
| from types import MethodType | |
| import torch | |
| from .shared_prefix import SharedPrefixReplay, install_on_swa | |
| from .shared_av import install_shared_av | |
| from .coupled_tail_cross import CoupledTailCrossRuntime, register_coupled_tail_cross | |
| ARCH_KEY = "isolated_architecture" | |
| FORMAT = "hfd_a1_c_production_v1" | |
| class ArchitectureSpec: | |
| arm: str | |
| interaction: str = "none" | |
| prefix_layers: int = 28 | |
| cross_layer: int = 34 | |
| bridge_dim: int = 256 | |
| heads: int = 8 | |
| max_cross_tokens: int = 32 | |
| bridge_seed: int = 941 | |
| control_token_arm: str = "C" | |
| def __post_init__(self): | |
| if self.arm not in {"a1", "a2"} or self.interaction not in {"none", "cross"}: | |
| raise ValueError("Only a1/a2 and none/cross are implemented") | |
| if self.arm != "a1" or self.interaction != "cross" or self.control_token_arm not in {"A", "B", "C"}: | |
| raise ValueError("Control A/B uses the identical A1 cross backbone") | |
| if self.arm == "a2" and self.interaction != "none": | |
| raise ValueError("Selected A2 is shared-AV stop-gradient without cross") | |
| if self.prefix_layers < 1: | |
| raise ValueError("A frozen nonempty prefix is required") | |
| if self.interaction == "cross" and not self.prefix_layers <= self.cross_layer < self.prefix_layers + 7: | |
| raise ValueError("Cross must leave a later AV KV layer in the private tail") | |
| def code_identity(): | |
| # Hash architecture implementation only: launcher/metric edits do not | |
| # change checkpoint semantics. No dependence on an experiment workspace. | |
| root = Path(__file__).parent | |
| names = ("runtime.py", "control_tokens.py", "shared_prefix.py", | |
| "shared_av.py", "coupled_tail_cross.py", "coupled_joint.py") | |
| return {"files": {name: hashlib.sha256((root / name).read_bytes()).hexdigest() | |
| for name in names}} | |
| def architecture_contract(spec): | |
| return dict(format=FORMAT, spec=asdict(spec), implementation_sha256=code_identity(), | |
| frozen_prefix_parameter_alias=spec.arm == "a1", | |
| private_mutable_qa_kv=True, answer_text_never_writes_av=True, | |
| qa_to_body_gradient=False, | |
| semantics=("shared_frozen_prefix_independent_tail8" if spec.arm == "a1" | |
| else "single_body_av_detached_snapshot_private_text_lora_live_detached_body_base")) | |
| class PrefixCrossRuntime(CoupledTailCrossRuntime): | |
| """Body-first rendezvous: record prefix BEFORE QA attempts prefix replay. | |
| The original paired executor is QA-first, which cannot compose with true | |
| prefix compute sharing. Here the body pauses at cross-layer entry; QA replays | |
| the already computed prefix and rendezvouses at the same layer. Both native | |
| self-attentions/crosses finish, then QA suffix, then body suffix. No foreign | |
| history is fabricated, and the formal later QA call consumes its real result. | |
| """ | |
| def enable_with_prefix(self, prefix, *, optimizer=None, prepared=False): | |
| model = self._model_ref() | |
| self.prefix = SharedPrefixReplay(model.vlm_model.model, model.qa_model.vlm_model.model, | |
| prefix_layers=prefix, history_mode="lineage", alias_prefix=True) | |
| try: | |
| super().enable(optimizer=optimizer, prepared=prepared) | |
| original_condition = model._encode_qwen_stream_input | |
| def paired(_model, **kwargs): | |
| with self.prefix.pair(): | |
| return original_condition(**kwargs) | |
| self._patch(model, "_encode_qwen_stream_input", MethodType(paired, model)) | |
| except Exception: | |
| if self.enabled: | |
| super().close() | |
| self.prefix.close() | |
| raise | |
| return self | |
| def stats(self): | |
| return {**super().stats, "frozen_prefix_replay": dict(self.prefix.stats) if hasattr(self, "prefix") else {}, | |
| "paired_execution_order": "body_prefix_then_qa_replay_then_paired_tail"} | |
| def _body_forward(self, *args, **kwargs): | |
| pair = self._pair | |
| if pair is None or pair.phase == "consumed": | |
| return self._original_body_forward(*args, **kwargs) | |
| if pair.phase != "body_entry" or args: | |
| raise RuntimeError("Unexpected shared-prefix body model entry") | |
| self._validate_model_args(kwargs) | |
| pair.body_args, pair.phase = kwargs, "body_running" | |
| with self.prefix._packet.body(): | |
| result = self._original_body_forward(**kwargs) | |
| if pair.phase != "qa_suffix" or pair.qa_encoded is None: | |
| raise RuntimeError("Body-first rendezvous did not finish the real QA branch") | |
| pair.phase = "body_return" | |
| return result | |
| def _body_layer(self, hidden, **kwargs): | |
| pair = self._pair | |
| if pair is None or pair.phase == "consumed": | |
| self._counts["text_layer_bypasses"] += 1 | |
| return self._original_body_layer(hidden, **kwargs) | |
| if pair.phase != "body_running": | |
| raise RuntimeError("Body-first cross layer is out of order") | |
| pair.body_layer_input, pair.body_layer_kwargs = hidden, kwargs | |
| packet = self.prefix._packet | |
| if packet.role != "body": | |
| raise RuntimeError("Shared prefix body scope was lost") | |
| # The prefix is complete although the body suffix has not returned. | |
| # This is needed for a sound replay lineage certificate, not just speed. | |
| packet.body_cache = kwargs["past_key_values"] | |
| pair.phase = "qa_prefix" | |
| model = self._model_ref() | |
| model.qa_model.vlm_processor = model.vlm_processor | |
| packet.role = None # Suspend the body scope only while QA executes. | |
| try: | |
| with packet.qa(), torch.set_grad_enabled(pair.qa_grad): | |
| pair.qa_encoded = self._original_encode( | |
| model.qa_model.vlm_model, pair.qa_cache, pair.packets, pair.active, | |
| memory_seconds=model.qa_model.memory_seconds, | |
| context_limit=model.qa_model.av_context_limit, prepared=pair.prepared) | |
| finally: | |
| packet.role = "body" | |
| if pair.phase != "qa_suffix" or not hasattr(pair, "body_layer_output"): | |
| raise RuntimeError("QA did not finish the shared-prefix cross rendezvous") | |
| self._counts["qa_av_forwards"] += 1 | |
| return pair.body_layer_output | |
| def _qa_layer(self, hidden, **kwargs): | |
| pair = self._pair | |
| if pair is None or pair.phase == "consumed": | |
| self._counts["text_layer_bypasses"] += 1 | |
| return self._original_qa_layer(hidden, **kwargs) | |
| if pair.phase != "qa_prefix" or not hasattr(pair, "body_layer_input"): | |
| raise RuntimeError("QA has no same-packet body cross input") | |
| pair.qa_layer_input, pair.qa_layer_kwargs = hidden, kwargs | |
| body, qa = self._joint_layers(pair.body_layer_input, pair.body_layer_kwargs, pair) | |
| pair.body_layer_output = body | |
| pair.phase = "qa_suffix" | |
| return qa | |
| def close(self): | |
| super().close() | |
| if hasattr(self, "prefix"): | |
| self.prefix.close() | |
| class ArchitectureRuntime: | |
| def __init__(self, model, spec): | |
| if hasattr(model, "_isolated_architecture_runtime"): | |
| raise RuntimeError("Architecture already registered") | |
| layers = model.vlm_model.model.layers | |
| if len(layers) != spec.prefix_layers + 8: | |
| raise ValueError("Architecture manifest must match frozen-prefix plus exactly eight private tail layers") | |
| self.model, self.spec = model, spec | |
| self.contract = architecture_contract(spec) | |
| self.operator = None | |
| self.installed = False | |
| self._inference_originals = [] | |
| if spec.arm == "a1" and spec.interaction == "cross": | |
| self.operator = register_coupled_tail_cross(model, layer_index=spec.cross_layer, | |
| bridge_dim=spec.bridge_dim, heads=spec.heads, max_cross_tokens=spec.max_cross_tokens, | |
| seed=spec.bridge_seed) | |
| self.operator.__class__ = PrefixCrossRuntime | |
| model._isolated_architecture_runtime = self | |
| def install(self, *, optimizer=None, prepared=False): | |
| if self.installed or not prepared: | |
| raise RuntimeError("Install once after final prepare/device/dtype placement") | |
| trainable_before = {id(p) for p in self.model.parameters() if p.requires_grad} | |
| optimizer_before = None if optimizer is None else [id(p) for g in optimizer.param_groups for p in g["params"]] | |
| if self.spec.arm == "a2": | |
| self.operator = install_shared_av(self.model, joint=False, optimizer=optimizer, prepared=True) | |
| elif self.spec.interaction == "none": | |
| self.operator = install_on_swa(self.model, prefix_layers=self.spec.prefix_layers, | |
| history_mode="lineage", alias_prefix=True) | |
| else: | |
| self.operator.enable_with_prefix(self.spec.prefix_layers, optimizer=optimizer, prepared=True) | |
| if {id(p) for p in self.model.parameters() if p.requires_grad} != trainable_before: | |
| raise RuntimeError("Architecture install changed trainable parameters after optimizer creation") | |
| if optimizer is not None: | |
| after = [id(p) for g in optimizer.param_groups for p in g["params"]] | |
| if after != optimizer_before or len(after) != len(set(after)) or set(after) != trainable_before: | |
| raise RuntimeError("Architecture optimizer ownership mismatch") | |
| self.installed = True | |
| return self | |
| def stats(self): | |
| return dict(arm=self.spec.arm, interaction=self.spec.interaction, | |
| installed=self.installed, runtime={} if self.operator is None else dict(self.operator.stats)) | |
| def close(self): | |
| for target, name, original in reversed(self._inference_originals): | |
| setattr(target, name, original) | |
| self._inference_originals.clear() | |
| if self.operator is not None: | |
| self.operator.close() | |
| def serialize_inference_dispatch(self): | |
| """Protect single-threaded AV hooks from the native answer worker. | |
| Serialize Python forward dispatch only, not the whole speech job. AV | |
| and answer KV remain independent; speech keeps its original CUDA stream | |
| and records actual queue/PCM availability, including lock contention. | |
| """ | |
| from threading import RLock | |
| if not self.installed or self._inference_originals: | |
| raise RuntimeError("Install inference dispatch guard exactly once after architecture install") | |
| lock = RLock() | |
| for target, name in ((self.model, "_encode_qwen_stream_input"), | |
| (self.model.qa_model.vlm_model.model, "forward")): | |
| original = getattr(target, name) | |
| def guarded(*args, _original=original, **kwargs): | |
| with lock: | |
| return _original(*args, **kwargs) | |
| self._inference_originals.append((target, name, original)) | |
| setattr(target, name, wraps(original)(guarded)) | |
| def resolve_checkpoint(path): | |
| """Match formal Trainer best-or-latest resolution, but never silently reset.""" | |
| root = Path(path).expanduser().resolve() | |
| if (root / "model.safetensors").is_file(): | |
| return root | |
| folder = root / "checkpoints" | |
| if (folder / "best_ckpt" / "model.safetensors").is_file(): | |
| return folder / "best_ckpt" | |
| choices = sorted((p for p in folder.glob("ckpt_*") if re.fullmatch(r"ckpt_\d+", p.name)), | |
| key=lambda p: int(p.name[5:])) | |
| if choices and (choices[-1] / "model.safetensors").is_file(): | |
| return choices[-1] | |
| raise FileNotFoundError(f"Explicit resume checkpoint not found: {root}") | |
| def validate_checkpoint(path, contract, expected_keys=None): | |
| from safetensors import safe_open | |
| path = Path(path) | |
| manifest = json.loads((path / "hfd_fused_delta_manifest.json").read_text()) | |
| if manifest.get("format") != "hfd_qwen_serial_delta_v1": | |
| raise RuntimeError("Unsupported formal checkpoint format") | |
| if manifest.get("contract", {}).get(ARCH_KEY) != contract: | |
| raise RuntimeError("Architecture/checkpoint mismatch or unmarked legacy checkpoint; cold migration is not resume") | |
| with safe_open(str(path / "model.safetensors"), framework="pt", device="cpu") as handle: | |
| keys = set(handle.keys()) | |
| if any(name.startswith("qa_model.") and ".base." in name for name in keys): | |
| raise RuntimeError("Checkpoint illegally contains an independent QA base") | |
| if expected_keys is not None and keys != set(expected_keys): | |
| raise RuntimeError("Architecture checkpoint tensor key set does not match the live model") | |
| return manifest | |
| def trainer_architecture(spec): | |
| """Patch only lifecycle boundaries; execute the entire unchanged Trainer.""" | |
| from psi.trainers.posttrain_full_duplex import PosttrainFullDuplexTrainer as Trainer | |
| originals = {name: getattr(Trainer, name) for name in ( | |
| "init_models", "prepare", "_build_hfdv2_checkpoint_contract", "resume_from_checkpoint", "log", "evaluate")} | |
| runtimes = [] | |
| def initialize(self): | |
| originals["init_models"](self) | |
| if int(getattr(self.train_cfg, "stage", 0)) != 0: | |
| raise ValueError("Architecture long training is stage 0 only") | |
| from .control_tokens import install_token_rows | |
| install_token_rows(self.model, arm=spec.control_token_arm) | |
| self._architecture = ArchitectureRuntime(self.model, spec) | |
| runtimes.append(self._architecture) | |
| def contract(self, *args, **kwargs): | |
| result = dict(originals["_build_hfdv2_checkpoint_contract"](self, *args, **kwargs)) | |
| result[ARCH_KEY] = self._architecture.contract | |
| return result | |
| def prepare(self, accelerator): | |
| if self.train_cfg.data_parallel in {"fsdp", "deepspeed"}: | |
| raise RuntimeError("A1/C requires DDP") | |
| result = originals["prepare"](self, accelerator) | |
| model = accelerator.unwrap_model(self.model) | |
| if model is not self._architecture.model: | |
| raise RuntimeError("Unexpected model replacement during prepare") | |
| self._architecture.install(optimizer=self.optimizer, prepared=True) | |
| from .control_tokens import enable_packet_write | |
| enable_packet_write(model) | |
| if accelerator.is_main_process: | |
| project = Path(self.project_dir) | |
| project.mkdir(parents=True, exist_ok=True) | |
| (project / "architecture_manifest.json").write_text(json.dumps(self._architecture.contract, indent=2)) | |
| (project / "control_token_contract.json").write_text(json.dumps(model.control_token_contract, indent=2)) | |
| model.vlm_processor.tokenizer.save_pretrained(project / "control_tokenizer") | |
| parameters = {id(p): p for p in model.parameters()} | |
| (project / "architecture_parameters.json").write_text(json.dumps(dict( | |
| unique_parameters=sum(p.numel() for p in parameters.values()), | |
| actual_trainable_parameters=sum(p.numel() for p in parameters.values() if p.requires_grad), | |
| note="Measured after bridge registration and alias; earlier formal init log predates bridge registration"), indent=2)) | |
| for tracker in accelerator.trackers: | |
| if tracker.name == "wandb": | |
| tracker.run.config.update({ARCH_KEY: self._architecture.contract}, allow_val_change=False) | |
| return result | |
| def resume(self): | |
| path = self.train_cfg.resume_from_checkpoint | |
| if path is not None: | |
| checkpoint = resolve_checkpoint(path) | |
| model = self.accelerator.unwrap_model(self.model) | |
| validate_checkpoint(checkpoint, self._architecture.contract, self._hfdv2_delta_keys(model)) | |
| return originals["resume_from_checkpoint"](self) | |
| def log(self, metrics, start_time=None): | |
| model = self.accelerator.unwrap_model(self.model) | |
| anchor = getattr(model, "control_token_semantic_anchor", None) | |
| if anchor is not None: | |
| weight = model.control_condition_embedding.weight.float() | |
| target = anchor.float().to(weight.device) | |
| metrics = dict(metrics) | |
| metrics["train/control_token_anchor_cosine"] = float( | |
| torch.nn.functional.cosine_similarity(weight, target, dim=-1).mean().detach().cpu()) | |
| metrics["train/control_token_anchor_rmse"] = float( | |
| (weight - target).square().mean().sqrt().detach().cpu()) | |
| result = originals["log"](self, metrics, start_time=start_time) | |
| from .control_metrics import flush | |
| extra = flush(self, "train") | |
| if extra: | |
| self.accelerator.log(extra, step=self.global_step) | |
| if self.accelerator.is_main_process: | |
| from psi.runtime.local_metrics import append_metrics | |
| append_metrics(self.project_dir, self.global_step, extra) | |
| return result | |
| def evaluate(self): | |
| from .control_metrics import flush | |
| getattr(self, "_token_control_meters", {}).pop("eval", None) | |
| was_training = self.model.training | |
| self.model.eval() | |
| try: | |
| result = originals["evaluate"](self) | |
| finally: | |
| self.model.train(was_training) | |
| extra = flush(self, "eval") | |
| if extra: | |
| self.accelerator.log(extra, step=self.global_step) | |
| if self.accelerator.is_main_process: | |
| from psi.runtime.local_metrics import append_metrics | |
| append_metrics(self.project_dir, self.global_step, extra) | |
| if result is not None: | |
| result.update({k.split("/",1)[1]: v for k,v in extra.items()}) | |
| return result | |
| for name, method in {"init_models": initialize, "prepare": prepare, | |
| "_build_hfdv2_checkpoint_contract": contract, "resume_from_checkpoint": resume, | |
| "log": log, "evaluate": evaluate}.items(): | |
| setattr(Trainer, name, method) | |
| try: | |
| yield | |
| finally: | |
| for runtime in reversed(runtimes): | |
| runtime.close() | |
| for name, method in originals.items(): | |
| setattr(Trainer, name, method) | |
| def load_inference_model(run_dir, checkpoint_step, launch_config, device, *, expected_spec=None): | |
| """Formal fused-delta loader with explicit architecture reconstruction. | |
| Standard unadapted inference fails its contract check, rather than silently | |
| loading A2 LoRA onto an old independent QA base. No KV is resumed across sources. | |
| """ | |
| if not str(device).startswith("cuda") or os.environ.get("PSI_INFER_FLOAT32", "0") == "1": | |
| raise NotImplementedError("Architecture inference currently targets CUDA BF16 only; CPU/FP32 QA backend parity is unvalidated") | |
| from psi.models.full_duplex_swa import FullDuplexSwaModel | |
| from psi.data.seed import qwen_streaming_audio as streaming | |
| run = Path(run_dir).resolve() | |
| name = str(checkpoint_step) | |
| name = "best_ckpt" if name in {"best", "best_ckpt"} else name if name.startswith("ckpt_") else "ckpt_" + name | |
| path = run / "checkpoints" / name | |
| if not path.is_dir(): | |
| path = run / name | |
| saved = json.loads((path / "hfd_fused_delta_manifest.json").read_text()).get("contract", {}).get(ARCH_KEY) | |
| if not isinstance(saved, dict) or saved.get("format") != FORMAT: | |
| raise RuntimeError("An architecture-marked checkpoint is required") | |
| spec = ArchitectureSpec(**saved["spec"]) | |
| if expected_spec is not None and spec != expected_spec: | |
| raise RuntimeError("Requested inference architecture does not match checkpoint") | |
| current = architecture_contract(spec) | |
| validate_checkpoint(path, current) | |
| class InferenceModel(FullDuplexSwaModel): | |
| def __init__(self, *args, **kwargs): | |
| super().__init__(*args, **kwargs) | |
| from transformers import Qwen2_5OmniProcessor | |
| from .control_tokens import install_token_rows | |
| self.vlm_processor = Qwen2_5OmniProcessor.from_pretrained(launch_config.model.model_name_or_path) | |
| install_token_rows(self, arm=spec.control_token_arm) | |
| ArchitectureRuntime(self, spec) # Before formal expected-delta check/load. | |
| original_contract = streaming.hfdv2_train_infer_contract | |
| def augmented(*args, **kwargs): | |
| return {**original_contract(*args, **kwargs), ARCH_KEY: current} | |
| streaming.hfdv2_train_infer_contract = augmented | |
| try: | |
| with torch.inference_mode(False): | |
| model = InferenceModel.from_streaming_checkpoint(run, checkpoint_step, launch_config, device) | |
| from transformers import AutoTokenizer | |
| from .control_tokens import enable_packet_write | |
| model.vlm_processor.tokenizer = AutoTokenizer.from_pretrained(run / "control_tokenizer") | |
| if tuple(model.vlm_processor.tokenizer.convert_tokens_to_ids(t) for t in ("<|listen|>", "<|motion|>", "<|idle|>")) != model.control_token_ids: | |
| raise RuntimeError("Saved inference state token IDs disagree with training") | |
| model._isolated_architecture_runtime.install(prepared=True) | |
| enable_packet_write(model) | |
| model._isolated_architecture_runtime.serialize_inference_dispatch() | |
| return model | |
| finally: | |
| streaming.hfdv2_train_infer_contract = original_contract | |