"""POSIX leases for immutable checkpoints shared by training and inference.""" from contextlib import contextmanager import errno import fcntl import os from pathlib import Path import re import shutil import tempfile @contextmanager def inference_checkpoint_snapshot(run_dir: Path, step: int): """Pin only inference inputs, including against older, non-locking trainers. A hard link keeps the immutable weights alive after retention unlinks the original. Small config/identity files are copied verbatim. No optimizer or RNG state is retained; normal completion/cancellation removes this view. Caller must reap all children BEFORE leaving this context. """ checkpoint = run_dir / "checkpoints" / f"ckpt_{int(step)}" parent = run_dir / ".inference_snapshots" parent.mkdir(exist_ok=True) with tempfile.TemporaryDirectory(prefix=f"ckpt_{int(step)}-", dir=parent) as name: snapshot = Path(name) target = snapshot / "checkpoints" / checkpoint.name target.mkdir(parents=True) for filename in ("argv.txt", "run_config.json"): shutil.copy2(run_dir / filename, snapshot / filename) if (run_dir / "control_tokenizer").is_dir(): shutil.copytree(run_dir / "control_tokenizer", snapshot / "control_tokenizer") for filename in ("hfd_fused_delta_manifest.json", "full_duplex_source_boundary.json"): shutil.copy2(checkpoint / filename, target / filename) source = checkpoint / "model.safetensors" destination = target / source.name try: os.link(source, destination) except OSError as error: if error.errno not in (errno.EXDEV, errno.EPERM, errno.EOPNOTSUPP): raise shutil.copy2(source, destination) yield snapshot def _lock_path(checkpoint: Path) -> Path: if not re.fullmatch(r"ckpt_[0-9]+", checkpoint.name): raise ValueError(f"Expected an immutable ckpt_: {checkpoint}") directory = checkpoint.parent / ".inference_locks" directory.mkdir(parents=True, exist_ok=True) # Never unlink lock files: replacing the inode would split simultaneous # readers/deleters onto different locks. These files contain no weights. return directory / (checkpoint.name + ".lock") @contextmanager def inference_checkpoint_lease(checkpoint: Path): """Readers share a lease; pass its FD to children so a crashed watcher cannot release protection while its inference children are still alive. """ with _lock_path(checkpoint).open("a+b") as handle: fcntl.flock(handle, fcntl.LOCK_SH) yield handle.fileno() # Closing (not LOCK_UN) preserves the lease held by inherited child FDs. def remove_checkpoint_unless_in_use(checkpoint: Path) -> bool: """Try existing retention policy without ever waiting for inference.""" with _lock_path(checkpoint).open("a+b") as handle: try: fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError: return False if checkpoint.exists(): shutil.rmtree(checkpoint) return True