Download scripts/third_party_temp.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 2.95 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/third_party_temp.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/third_party_temp.py
-
curl -L -o third_party_temp.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/third_party_temp.py
2.95 kB
| """Keep regular scratch in the project and multiprocessing sockets local.""" | |
| from __future__ import annotations | |
| import hashlib | |
| import multiprocessing as mp | |
| from multiprocessing import util as mp_util | |
| import os | |
| from pathlib import Path | |
| import socket | |
| import stat | |
| import tempfile | |
| def project_temp_directory(project_root: Path) -> Path: | |
| directory = Path(project_root).resolve(strict=True) / "tmp" | |
| if directory.is_symlink(): | |
| raise ValueError("Project tmp must be a real directory, not a symlink") | |
| directory.mkdir(mode=0o700, exist_ok=True) | |
| return directory | |
| def _private_directory(directory: Path) -> None: | |
| directory.mkdir(mode=0o700, exist_ok=True) | |
| info = directory.lstat() | |
| if (not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() | |
| or stat.S_IMODE(info.st_mode) & 0o077): | |
| raise ValueError(f"IPC directory must be private and owned by this user: {directory}") | |
| def configure_project_temp(project_root: Path) -> dict[str, str]: | |
| """Call before constructing any DataLoader or starting model initialization. | |
| Python 3.10 caches the multiprocessing directory in the process config; | |
| fork/spawn/forkserver workers inherit it. Initialize that cache on the local | |
| filesystem, then direct ordinary tempfile calls to the project. The stdlib | |
| registers cleanup for its own per-run pymp directory. No stdlib monkeypatch, | |
| shared environment edit, or replacement of an already-active IPC path. | |
| """ | |
| ordinary = project_temp_directory(project_root) | |
| identity = hashlib.sha256(os.fsencode(ordinary.parent)).hexdigest()[:12] | |
| ipc_root = Path("/tmp") / f"hfd-lzk-ipc-{os.getuid()}-{identity}" | |
| # AF_UNIX's 108-byte limit includes the terminating NUL. | |
| if len(os.fsencode(ipc_root / "pymp-12345678/listener-12345678")) >= 108: | |
| raise ValueError("IPC directory would exceed the Unix-socket address limit") | |
| _private_directory(ipc_root) | |
| existing = mp.current_process()._config.get("tempdir") | |
| if existing is not None: | |
| cached = Path(existing) | |
| if (cached.parent != ipc_root or not cached.name.startswith("pymp-") | |
| or not cached.is_dir()): | |
| raise RuntimeError("Multiprocessing already has an unrelated temporary directory") | |
| _private_directory(cached) | |
| previous = tempfile.tempdir | |
| try: | |
| tempfile.tempdir = str(ipc_root) | |
| ipc = Path(mp_util.get_temp_dir()) | |
| finally: | |
| tempfile.tempdir = previous | |
| # Fail before loading a model if local sockets are unavailable. | |
| with tempfile.TemporaryDirectory(prefix="probe-", dir=ipc_root) as probe: | |
| with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener: | |
| listener.bind(str(Path(probe) / "listener")) | |
| for variable in ("TMPDIR", "TMP", "TEMP"): | |
| os.environ[variable] = str(ordinary) | |
| tempfile.tempdir = str(ordinary) | |
| return {"ordinary_directory": str(ordinary), "ipc_directory": str(ipc)} | |