File size: 8,251 Bytes
3671737 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 | """Checkpoint management with FAST tokenizer saving.
Reference: https://github.com/Physical-Intelligence/openpi
"""
from __future__ import annotations
import asyncio
import concurrent.futures as futures
import dataclasses
import logging
import shutil
from typing import Protocol
from etils import epath
import flax.traverse_util
import jax
import flax.nnx as nnx
import orbax.checkpoint as ocp
import orbax.checkpoint.future as future
from openpi.shared import array_typing as at
import openpi.training.data_loader as _data_loader
import openpi.training.utils as training_utils
# Use our custom normalize (has per_timestamp fields and can save/load them)
from b1k.shared import normalize as _normalize
def initialize_checkpoint_dir(
checkpoint_dir: epath.Path | str, *, keep_period: int | None, overwrite: bool, resume: bool
) -> tuple[ocp.CheckpointManager, bool]:
checkpoint_dir = epath.Path(checkpoint_dir).resolve()
resuming = False
if checkpoint_dir.exists():
if overwrite:
checkpoint_dir.rmtree()
checkpoint_dir.mkdir(parents=True, exist_ok=True)
logging.info(f"Wiped checkpoint directory {checkpoint_dir}")
elif resume:
resuming = True
else:
raise FileExistsError(
f"Checkpoint directory {checkpoint_dir} already exists. Use --overwrite or --resume "
"to indicate how to handle it."
)
checkpoint_dir.mkdir(parents=True, exist_ok=True)
mngr = ocp.CheckpointManager(
checkpoint_dir,
item_handlers={
"assets": CallbackHandler(),
"train_state": ocp.PyTreeCheckpointHandler(),
"params": ocp.PyTreeCheckpointHandler(),
},
options=ocp.CheckpointManagerOptions(
max_to_keep=1,
keep_period=keep_period,
create=False,
async_options=ocp.AsyncOptions(timeout_secs=7200),
),
)
# Special case: the checkpoint directory exists and the user requests to resume training, but the training run did
# not get to the first checkpoint saved. In this case, we don't actually want the train script to try and restore a
# checkpoint, since it will fail.
if resuming and tuple(mngr.all_steps()) in [(), (0,)]:
logging.info("Checkpoint directory exists, but does not contain any checkpoints. Aborting resume.")
resuming = False
return mngr, resuming
def save_state(
checkpoint_manager: ocp.CheckpointManager,
state: training_utils.TrainState,
data_loader: _data_loader.DataLoader,
step: int,
):
def save_assets(directory: epath.Path):
# Save the normalization stats.
data_config = data_loader.data_config()
norm_stats = data_config.norm_stats
if norm_stats is not None and data_config.asset_id is not None:
_normalize.save(directory / data_config.asset_id, norm_stats)
# Save FAST tokenizer if it exists
model = nnx.merge(state.model_def, state.params)
# Check if model has FAST enabled (indicated by having fast_token_embedding)
if hasattr(model, 'fast_token_embedding'):
# Get the checkpoint directory to determine assets base dir
# Directory structure: checkpoint_dir/step/assets/asset_id/fast_tokenizer
# We need to copy from: assets_base_dir/asset_id/fast_tokenizer
# The assets_base_dir is stored in the config, but we can infer it from checkpoint_dir
checkpoint_dir = checkpoint_manager.directory
# Infer assets_base_dir from checkpoint structure
# checkpoint_dir is like: ./outputs/checkpoints/config_name/exp_name
# assets_base_dir is like: ./outputs/assets/config_name
parts = checkpoint_dir.parts
if 'checkpoints' in parts:
idx = parts.index('checkpoints')
assets_base_parts = parts[:idx] + ('assets',) + (parts[idx + 1],) # Keep config_name
assets_base_dir = epath.Path(*assets_base_parts)
else:
# Fallback: try relative to checkpoint dir
assets_base_dir = checkpoint_dir.parent.parent / 'assets' / checkpoint_dir.parent.name
fast_tokenizer_source = assets_base_dir / data_config.asset_id / "fast_tokenizer"
fast_tokenizer_dest = directory / data_config.asset_id / "fast_tokenizer"
if fast_tokenizer_source.exists():
# Create parent directory
fast_tokenizer_dest.parent.mkdir(parents=True, exist_ok=True)
# Copy tokenizer directory
shutil.copytree(fast_tokenizer_source, fast_tokenizer_dest, dirs_exist_ok=True)
logging.info(f"Saved FAST tokenizer to checkpoint: {fast_tokenizer_dest}")
# Split params that can be used for inference into a separate item.
with at.disable_typechecking():
train_state, params = _split_params(state)
items = {
"assets": save_assets,
"train_state": train_state,
"params": {"params": params},
}
checkpoint_manager.save(step, items)
def restore_state(
checkpoint_manager: ocp.CheckpointManager,
state: training_utils.TrainState,
data_loader: _data_loader.DataLoader,
step: int | None = None,
) -> training_utils.TrainState:
del data_loader
with at.disable_typechecking():
# Split params that can be used for inference into a separate item.
train_state, params = _split_params(state)
restored = checkpoint_manager.restore(
step,
items={
"train_state": train_state,
"params": {"params": params},
},
)
return _merge_params(restored["train_state"], restored["params"])
def load_norm_stats(assets_dir: epath.Path | str, asset_id: str) -> dict[str, _normalize.NormStats] | None:
norm_stats_dir = epath.Path(assets_dir) / asset_id
norm_stats = _normalize.load(norm_stats_dir)
logging.info(f"Loaded norm stats from {norm_stats_dir}")
return norm_stats
class Callback(Protocol):
def __call__(self, directory: epath.Path) -> None: ...
class CallbackHandler(ocp.AsyncCheckpointHandler):
"""A CheckpointHandler for calling an arbitrary function asynchronously. Only for saving, not for restoring."""
def save(self, directory: epath.Path, args: CallbackSave):
if jax.process_index() == 0:
args.callback(directory)
async def async_save(self, directory: epath.Path, args: CallbackSave) -> list[futures.Future]:
return [future.CommitFutureAwaitingContractedSignals(asyncio.to_thread(self.save, directory, args))]
def restore(self, *args, **kwargs):
raise NotImplementedError("CallbackHandler does not support restore")
@ocp.args.register_with_handler(CallbackHandler, for_save=True)
@dataclasses.dataclass
class CallbackSave(ocp.args.CheckpointArgs):
callback: Callback
@ocp.args.register_with_handler(CallbackHandler, for_restore=True)
class CallbackRestore(ocp.args.CheckpointArgs): ...
def _split_params(state: training_utils.TrainState) -> tuple[training_utils.TrainState, at.Params]:
if state.ema_params is not None:
params = state.ema_params
train_state = dataclasses.replace(state, ema_params=None)
else:
params = state.params
train_state = dataclasses.replace(state, params={})
return train_state, params
def _merge_params(train_state: training_utils.TrainState, params: dict[str, at.Params]) -> training_utils.TrainState:
# Revert the logic inside `_split_params`.
# If the restored train_state has an empty `params` dict, it means we're in the non-EMA case
# and we should restore the main training params.
if not train_state.params:
# Non-EMA case: The saved 'params' are the training weights.
return dataclasses.replace(train_state, params=params["params"])
else:
# EMA case: The saved 'params' are the ema_params, and train_state already has the training params.
return dataclasses.replace(train_state, ema_params=params["params"])
|