code-HFD / src /psi /checkpoint_lease.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
3.16 kB
"""POSIX leases for immutable checkpoints shared by training and inference."""
from contextlib import contextmanager
import errno
import fcntl
import os
from pathlib import Path
import re
import shutil
import tempfile
@contextmanager
def inference_checkpoint_snapshot(run_dir: Path, step: int):
"""Pin only inference inputs, including against older, non-locking trainers.
A hard link keeps the immutable weights alive after retention unlinks the
original. Small config/identity files are copied verbatim. No optimizer or
RNG state is retained; normal completion/cancellation removes this view.
Caller must reap all children BEFORE leaving this context.
"""
checkpoint = run_dir / "checkpoints" / f"ckpt_{int(step)}"
parent = run_dir / ".inference_snapshots"
parent.mkdir(exist_ok=True)
with tempfile.TemporaryDirectory(prefix=f"ckpt_{int(step)}-", dir=parent) as name:
snapshot = Path(name)
target = snapshot / "checkpoints" / checkpoint.name
target.mkdir(parents=True)
for filename in ("argv.txt", "run_config.json"):
shutil.copy2(run_dir / filename, snapshot / filename)
if (run_dir / "control_tokenizer").is_dir():
shutil.copytree(run_dir / "control_tokenizer", snapshot / "control_tokenizer")
for filename in ("hfd_fused_delta_manifest.json", "full_duplex_source_boundary.json"):
shutil.copy2(checkpoint / filename, target / filename)
source = checkpoint / "model.safetensors"
destination = target / source.name
try:
os.link(source, destination)
except OSError as error:
if error.errno not in (errno.EXDEV, errno.EPERM, errno.EOPNOTSUPP):
raise
shutil.copy2(source, destination)
yield snapshot
def _lock_path(checkpoint: Path) -> Path:
if not re.fullmatch(r"ckpt_[0-9]+", checkpoint.name):
raise ValueError(f"Expected an immutable ckpt_<step>: {checkpoint}")
directory = checkpoint.parent / ".inference_locks"
directory.mkdir(parents=True, exist_ok=True)
# Never unlink lock files: replacing the inode would split simultaneous
# readers/deleters onto different locks. These files contain no weights.
return directory / (checkpoint.name + ".lock")
@contextmanager
def inference_checkpoint_lease(checkpoint: Path):
"""Readers share a lease; pass its FD to children so a crashed watcher
cannot release protection while its inference children are still alive.
"""
with _lock_path(checkpoint).open("a+b") as handle:
fcntl.flock(handle, fcntl.LOCK_SH)
yield handle.fileno()
# Closing (not LOCK_UN) preserves the lease held by inherited child FDs.
def remove_checkpoint_unless_in_use(checkpoint: Path) -> bool:
"""Try existing retention policy without ever waiting for inference."""
with _lock_path(checkpoint).open("a+b") as handle:
try:
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
return False
if checkpoint.exists():
shutil.rmtree(checkpoint)
return True