Download src/psi/checkpoint_lease.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 3.16 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/checkpoint_lease.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/src/psi/checkpoint_lease.py
-
curl -L -o checkpoint_lease.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/checkpoint_lease.py
3.16 kB
| """POSIX leases for immutable checkpoints shared by training and inference.""" | |
| from contextlib import contextmanager | |
| import errno | |
| import fcntl | |
| import os | |
| from pathlib import Path | |
| import re | |
| import shutil | |
| import tempfile | |
| def inference_checkpoint_snapshot(run_dir: Path, step: int): | |
| """Pin only inference inputs, including against older, non-locking trainers. | |
| A hard link keeps the immutable weights alive after retention unlinks the | |
| original. Small config/identity files are copied verbatim. No optimizer or | |
| RNG state is retained; normal completion/cancellation removes this view. | |
| Caller must reap all children BEFORE leaving this context. | |
| """ | |
| checkpoint = run_dir / "checkpoints" / f"ckpt_{int(step)}" | |
| parent = run_dir / ".inference_snapshots" | |
| parent.mkdir(exist_ok=True) | |
| with tempfile.TemporaryDirectory(prefix=f"ckpt_{int(step)}-", dir=parent) as name: | |
| snapshot = Path(name) | |
| target = snapshot / "checkpoints" / checkpoint.name | |
| target.mkdir(parents=True) | |
| for filename in ("argv.txt", "run_config.json"): | |
| shutil.copy2(run_dir / filename, snapshot / filename) | |
| if (run_dir / "control_tokenizer").is_dir(): | |
| shutil.copytree(run_dir / "control_tokenizer", snapshot / "control_tokenizer") | |
| for filename in ("hfd_fused_delta_manifest.json", "full_duplex_source_boundary.json"): | |
| shutil.copy2(checkpoint / filename, target / filename) | |
| source = checkpoint / "model.safetensors" | |
| destination = target / source.name | |
| try: | |
| os.link(source, destination) | |
| except OSError as error: | |
| if error.errno not in (errno.EXDEV, errno.EPERM, errno.EOPNOTSUPP): | |
| raise | |
| shutil.copy2(source, destination) | |
| yield snapshot | |
| def _lock_path(checkpoint: Path) -> Path: | |
| if not re.fullmatch(r"ckpt_[0-9]+", checkpoint.name): | |
| raise ValueError(f"Expected an immutable ckpt_<step>: {checkpoint}") | |
| directory = checkpoint.parent / ".inference_locks" | |
| directory.mkdir(parents=True, exist_ok=True) | |
| # Never unlink lock files: replacing the inode would split simultaneous | |
| # readers/deleters onto different locks. These files contain no weights. | |
| return directory / (checkpoint.name + ".lock") | |
| def inference_checkpoint_lease(checkpoint: Path): | |
| """Readers share a lease; pass its FD to children so a crashed watcher | |
| cannot release protection while its inference children are still alive. | |
| """ | |
| with _lock_path(checkpoint).open("a+b") as handle: | |
| fcntl.flock(handle, fcntl.LOCK_SH) | |
| yield handle.fileno() | |
| # Closing (not LOCK_UN) preserves the lease held by inherited child FDs. | |
| def remove_checkpoint_unless_in_use(checkpoint: Path) -> bool: | |
| """Try existing retention policy without ever waiting for inference.""" | |
| with _lock_path(checkpoint).open("a+b") as handle: | |
| try: | |
| fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) | |
| except BlockingIOError: | |
| return False | |
| if checkpoint.exists(): | |
| shutil.rmtree(checkpoint) | |
| return True | |