"""Small shared IO, identity and precision helpers.""" import hashlib import json import os from contextlib import contextmanager, nullcontext from pathlib import Path import torch ROOT = Path(__file__).resolve().parent def digest(value): return hashlib.sha256( json.dumps( value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False, ).encode() ).hexdigest() def file_hash(path): result = hashlib.sha256() with Path(path).open("rb") as stream: for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): result.update(chunk) return result.hexdigest() def atomic_json(path, value): path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_suffix(path.suffix + ".tmp") with temporary.open("w") as stream: json.dump(value, stream, indent=2, allow_nan=False) stream.write("\n") stream.flush() os.fsync(stream.fileno()) temporary.replace(path) def atomic_save(value, path): path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_suffix(path.suffix + ".tmp") with temporary.open("wb") as stream: torch.save(value, stream) stream.flush() os.fsync(stream.fileno()) temporary.replace(path) def autocast(device, precision="bf16"): return ( torch.autocast("cuda", dtype=torch.bfloat16) if torch.device(device).type == "cuda" and precision == "bf16" else nullcontext() ) @contextmanager def job_lock(directory): import fcntl with (Path(directory) / ".job.lock").open("a") as stream: fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB) yield