Bazza / utils.py
adrianjohnstonaus's picture
Release Bazza 0.5.0 decision head and standalone inference code
dd5c973 verified
Raw History Blame
1.8 kB
"""Small shared IO, identity and precision helpers."""
import hashlib
import json
import os
from contextlib import contextmanager, nullcontext
from pathlib import Path
import torch
ROOT = Path(__file__).resolve().parent
def digest(value):
return hashlib.sha256(
json.dumps(
value,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
).encode()
).hexdigest()
def file_hash(path):
result = hashlib.sha256()
with Path(path).open("rb") as stream:
for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""):
result.update(chunk)
return result.hexdigest()
def atomic_json(path, value):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_suffix(path.suffix + ".tmp")
with temporary.open("w") as stream:
json.dump(value, stream, indent=2, allow_nan=False)
stream.write("\n")
stream.flush()
os.fsync(stream.fileno())
temporary.replace(path)
def atomic_save(value, path):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_suffix(path.suffix + ".tmp")
with temporary.open("wb") as stream:
torch.save(value, stream)
stream.flush()
os.fsync(stream.fileno())
temporary.replace(path)
def autocast(device, precision="bf16"):
return (
torch.autocast("cuda", dtype=torch.bfloat16)
if torch.device(device).type == "cuda" and precision == "bf16"
else nullcontext()
)
@contextmanager
def job_lock(directory):
import fcntl
with (Path(directory) / ".job.lock").open("a") as stream:
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
yield