File size: 3,373 Bytes
3671737 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 | """Weight loaders for PI_BEHAVIOR model initialization from Pi05 checkpoints.
Reference: https://github.com/Physical-Intelligence
"""
import dataclasses
import logging
import re
import flax.traverse_util
import numpy as np
import orbax.checkpoint as ocp
import openpi.shared.array_typing as at
import openpi.shared.download as download
# Re-export base loaders from OpenPI
from openpi.training.weight_loaders import (
WeightLoader,
NoOpWeightLoader,
CheckpointWeightLoader,
_merge_params,
)
logger = logging.getLogger(__name__)
@dataclasses.dataclass(frozen=True)
class PiBehaviorWeightLoader(WeightLoader):
"""Loads checkpoints for PI_BEHAVIOR model.
Automatically detects:
- Pi05 checkpoint: Loads weights, preserves new PI_BEHAVIOR parameters
- PI_BEHAVIOR checkpoint: Loads all weights directly
"""
params_path: str
def load(self, params: at.Params) -> at.Params:
# Load checkpoint
params_path = download.maybe_download(self.params_path)
# Restore as host numpy arrays; newer orbax refuses a bare restore() without restore args.
import openpi.models.model as _openpi_model
restored = _openpi_model.restore_params(params_path, restore_type=np.ndarray)
# Handle nested 'params' key (from some checkpoint formats)
if isinstance(restored, dict) and "params" in restored:
loaded_params = restored["params"]
else:
loaded_params = restored
# Remove 'value' suffixes (from nnx.State format)
flat_params = flax.traverse_util.flatten_dict(loaded_params)
if all(kp[-1] == "value" for kp in flat_params if len(kp) > 0):
flat_params = {kp[:-1]: v for kp, v in flat_params.items()}
loaded_params = flax.traverse_util.unflatten_dict(flat_params)
# Detect checkpoint type
has_task_embeddings = 'task_embeddings' in loaded_params
if has_task_embeddings:
# Loading PI_BEHAVIOR checkpoint - load ALL weights from checkpoint
logging.info("Loading PI_BEHAVIOR checkpoint (all weights will be loaded)")
# LoRA adapters (lora_a/lora_b) are not in a full-weight checkpoint; they keep their init
# (lora_b starts at zero, so the model initially matches the checkpoint exactly).
return _merge_params(loaded_params, params, missing_regex=".*lora.*")
else:
# Loading Pi05 checkpoint - preserve new PI_BEHAVIOR-specific parameters
logging.info("Loading Pi05 checkpoint (new PI_BEHAVIOR parameters will use random init)")
# These parameters are NEW in PI_BEHAVIOR (not in Pi05), so keep them from params (random init)
missing_regex = (
".*task_embeddings.*|"
".*stage_pred_from_vlm.*|"
".*task_stage_embeddings.*|"
".*gate_sincos.*|"
".*gate_task_stage.*|"
".*gate_task.*|"
".*fusion_layer.*|"
".*stage_projection.*|"
".*task_subtask_fusion.*|"
".*fast_token_embedding.*|"
".*fast_token_proj.*|"
".*kv_transform.*"
)
return _merge_params(loaded_params, params, missing_regex=missing_regex)
|