File size: 1,799 Bytes
0f99db6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | """Small shared IO, identity and precision helpers."""
import hashlib
import json
import os
from contextlib import contextmanager, nullcontext
from pathlib import Path
import torch
ROOT = Path(__file__).resolve().parent
def digest(value):
return hashlib.sha256(
json.dumps(
value,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
).encode()
).hexdigest()
def file_hash(path):
result = hashlib.sha256()
with Path(path).open("rb") as stream:
for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""):
result.update(chunk)
return result.hexdigest()
def atomic_json(path, value):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_suffix(path.suffix + ".tmp")
with temporary.open("w") as stream:
json.dump(value, stream, indent=2, allow_nan=False)
stream.write("\n")
stream.flush()
os.fsync(stream.fileno())
temporary.replace(path)
def atomic_save(value, path):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_suffix(path.suffix + ".tmp")
with temporary.open("wb") as stream:
torch.save(value, stream)
stream.flush()
os.fsync(stream.fileno())
temporary.replace(path)
def autocast(device, precision="bf16"):
return (
torch.autocast("cuda", dtype=torch.bfloat16)
if torch.device(device).type == "cuda" and precision == "bf16"
else nullcontext()
)
@contextmanager
def job_lock(directory):
import fcntl
with (Path(directory) / ".job.lock").open("a") as stream:
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
yield
|