task57b1k / task 57 /policy_code /src /b1k /training /checkpoints.py
Luke-Xing's picture
Upload 27 files
3671737 verified
Raw History Blame Contribute Delete
8.25 kB
"""Checkpoint management with FAST tokenizer saving.
Reference: https://github.com/Physical-Intelligence/openpi
"""
from __future__ import annotations
import asyncio
import concurrent.futures as futures
import dataclasses
import logging
import shutil
from typing import Protocol
from etils import epath
import flax.traverse_util
import jax
import flax.nnx as nnx
import orbax.checkpoint as ocp
import orbax.checkpoint.future as future
from openpi.shared import array_typing as at
import openpi.training.data_loader as _data_loader
import openpi.training.utils as training_utils
# Use our custom normalize (has per_timestamp fields and can save/load them)
from b1k.shared import normalize as _normalize
def initialize_checkpoint_dir(
checkpoint_dir: epath.Path | str, *, keep_period: int | None, overwrite: bool, resume: bool
) -> tuple[ocp.CheckpointManager, bool]:
checkpoint_dir = epath.Path(checkpoint_dir).resolve()
resuming = False
if checkpoint_dir.exists():
if overwrite:
checkpoint_dir.rmtree()
checkpoint_dir.mkdir(parents=True, exist_ok=True)
logging.info(f"Wiped checkpoint directory {checkpoint_dir}")
elif resume:
resuming = True
else:
raise FileExistsError(
f"Checkpoint directory {checkpoint_dir} already exists. Use --overwrite or --resume "
"to indicate how to handle it."
)
checkpoint_dir.mkdir(parents=True, exist_ok=True)
mngr = ocp.CheckpointManager(
checkpoint_dir,
item_handlers={
"assets": CallbackHandler(),
"train_state": ocp.PyTreeCheckpointHandler(),
"params": ocp.PyTreeCheckpointHandler(),
},
options=ocp.CheckpointManagerOptions(
max_to_keep=1,
keep_period=keep_period,
create=False,
async_options=ocp.AsyncOptions(timeout_secs=7200),
),
)
# Special case: the checkpoint directory exists and the user requests to resume training, but the training run did
# not get to the first checkpoint saved. In this case, we don't actually want the train script to try and restore a
# checkpoint, since it will fail.
if resuming and tuple(mngr.all_steps()) in [(), (0,)]:
logging.info("Checkpoint directory exists, but does not contain any checkpoints. Aborting resume.")
resuming = False
return mngr, resuming
def save_state(
checkpoint_manager: ocp.CheckpointManager,
state: training_utils.TrainState,
data_loader: _data_loader.DataLoader,
step: int,
):
def save_assets(directory: epath.Path):
# Save the normalization stats.
data_config = data_loader.data_config()
norm_stats = data_config.norm_stats
if norm_stats is not None and data_config.asset_id is not None:
_normalize.save(directory / data_config.asset_id, norm_stats)
# Save FAST tokenizer if it exists
model = nnx.merge(state.model_def, state.params)
# Check if model has FAST enabled (indicated by having fast_token_embedding)
if hasattr(model, 'fast_token_embedding'):
# Get the checkpoint directory to determine assets base dir
# Directory structure: checkpoint_dir/step/assets/asset_id/fast_tokenizer
# We need to copy from: assets_base_dir/asset_id/fast_tokenizer
# The assets_base_dir is stored in the config, but we can infer it from checkpoint_dir
checkpoint_dir = checkpoint_manager.directory
# Infer assets_base_dir from checkpoint structure
# checkpoint_dir is like: ./outputs/checkpoints/config_name/exp_name
# assets_base_dir is like: ./outputs/assets/config_name
parts = checkpoint_dir.parts
if 'checkpoints' in parts:
idx = parts.index('checkpoints')
assets_base_parts = parts[:idx] + ('assets',) + (parts[idx + 1],) # Keep config_name
assets_base_dir = epath.Path(*assets_base_parts)
else:
# Fallback: try relative to checkpoint dir
assets_base_dir = checkpoint_dir.parent.parent / 'assets' / checkpoint_dir.parent.name
fast_tokenizer_source = assets_base_dir / data_config.asset_id / "fast_tokenizer"
fast_tokenizer_dest = directory / data_config.asset_id / "fast_tokenizer"
if fast_tokenizer_source.exists():
# Create parent directory
fast_tokenizer_dest.parent.mkdir(parents=True, exist_ok=True)
# Copy tokenizer directory
shutil.copytree(fast_tokenizer_source, fast_tokenizer_dest, dirs_exist_ok=True)
logging.info(f"Saved FAST tokenizer to checkpoint: {fast_tokenizer_dest}")
# Split params that can be used for inference into a separate item.
with at.disable_typechecking():
train_state, params = _split_params(state)
items = {
"assets": save_assets,
"train_state": train_state,
"params": {"params": params},
}
checkpoint_manager.save(step, items)
def restore_state(
checkpoint_manager: ocp.CheckpointManager,
state: training_utils.TrainState,
data_loader: _data_loader.DataLoader,
step: int | None = None,
) -> training_utils.TrainState:
del data_loader
with at.disable_typechecking():
# Split params that can be used for inference into a separate item.
train_state, params = _split_params(state)
restored = checkpoint_manager.restore(
step,
items={
"train_state": train_state,
"params": {"params": params},
},
)
return _merge_params(restored["train_state"], restored["params"])
def load_norm_stats(assets_dir: epath.Path | str, asset_id: str) -> dict[str, _normalize.NormStats] | None:
norm_stats_dir = epath.Path(assets_dir) / asset_id
norm_stats = _normalize.load(norm_stats_dir)
logging.info(f"Loaded norm stats from {norm_stats_dir}")
return norm_stats
class Callback(Protocol):
def __call__(self, directory: epath.Path) -> None: ...
class CallbackHandler(ocp.AsyncCheckpointHandler):
"""A CheckpointHandler for calling an arbitrary function asynchronously. Only for saving, not for restoring."""
def save(self, directory: epath.Path, args: CallbackSave):
if jax.process_index() == 0:
args.callback(directory)
async def async_save(self, directory: epath.Path, args: CallbackSave) -> list[futures.Future]:
return [future.CommitFutureAwaitingContractedSignals(asyncio.to_thread(self.save, directory, args))]
def restore(self, *args, **kwargs):
raise NotImplementedError("CallbackHandler does not support restore")
@ocp.args.register_with_handler(CallbackHandler, for_save=True)
@dataclasses.dataclass
class CallbackSave(ocp.args.CheckpointArgs):
callback: Callback
@ocp.args.register_with_handler(CallbackHandler, for_restore=True)
class CallbackRestore(ocp.args.CheckpointArgs): ...
def _split_params(state: training_utils.TrainState) -> tuple[training_utils.TrainState, at.Params]:
if state.ema_params is not None:
params = state.ema_params
train_state = dataclasses.replace(state, ema_params=None)
else:
params = state.params
train_state = dataclasses.replace(state, params={})
return train_state, params
def _merge_params(train_state: training_utils.TrainState, params: dict[str, at.Params]) -> training_utils.TrainState:
# Revert the logic inside `_split_params`.
# If the restored train_state has an empty `params` dict, it means we're in the non-EMA case
# and we should restore the main training params.
if not train_state.params:
# Non-EMA case: The saved 'params' are the training weights.
return dataclasses.replace(train_state, params=params["params"])
else:
# EMA case: The saved 'params' are the ema_params, and train_state already has the training params.
return dataclasses.replace(train_state, ema_params=params["params"])