Joy-face_change_optimized / training /src /utils /activation_checkpointing.py
kylinfires's picture
Sync main: low-VRAM live test results (624x360)
b9fc164 verified
Raw History Blame Contribute Delete
902 Bytes
import torch
from torch.distributed.algorithms._checkpoint.checkpoint_wrapper import checkpoint_wrapper
TRANSFORMER_BLOCK_NAMES = ["double_blocks", "single_blocks"]
def apply_activation_checkpointing(
module: torch.nn.Module,
skip_interval: int = 1
) -> torch.nn.Module:
"""Apply checkpointing to transformer blocks with specified skip interval."""
if skip_interval < 1:
raise ValueError(
f"skip_interval must be >= 1, but got {skip_interval}")
for block_name in TRANSFORMER_BLOCK_NAMES:
blocks = getattr(module, block_name, None)
if blocks is None:
continue
for index, (layer_id, block) in enumerate(blocks.named_children()):
if index % skip_interval == 0:
checkpointed_block = checkpoint_wrapper(block)
blocks.register_module(layer_id, checkpointed_block)
return module