Download utils.py from 0xdeadbe3f/Bazza: direct link, hf CLI and curl.
- Browser
- Download file 1.8 kB
-
https://huggingface.co/0xdeadbe3f/Bazza/resolve/refs%2Fpr%2F1/utils.py
- Command line
-
hf download hf://0xdeadbe3f/Bazza@refs/pr/1/utils.py
-
curl -L -o utils.py https://huggingface.co/0xdeadbe3f/Bazza/resolve/refs%2Fpr%2F1/utils.py
1.8 kB
| """Small shared IO, identity and precision helpers.""" | |
| import hashlib | |
| import json | |
| import os | |
| from contextlib import contextmanager, nullcontext | |
| from pathlib import Path | |
| import torch | |
| ROOT = Path(__file__).resolve().parent | |
| def digest(value): | |
| return hashlib.sha256( | |
| json.dumps( | |
| value, | |
| ensure_ascii=False, | |
| sort_keys=True, | |
| separators=(",", ":"), | |
| allow_nan=False, | |
| ).encode() | |
| ).hexdigest() | |
| def file_hash(path): | |
| result = hashlib.sha256() | |
| with Path(path).open("rb") as stream: | |
| for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): | |
| result.update(chunk) | |
| return result.hexdigest() | |
| def atomic_json(path, value): | |
| path = Path(path) | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| temporary = path.with_suffix(path.suffix + ".tmp") | |
| with temporary.open("w") as stream: | |
| json.dump(value, stream, indent=2, allow_nan=False) | |
| stream.write("\n") | |
| stream.flush() | |
| os.fsync(stream.fileno()) | |
| temporary.replace(path) | |
| def atomic_save(value, path): | |
| path = Path(path) | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| temporary = path.with_suffix(path.suffix + ".tmp") | |
| with temporary.open("wb") as stream: | |
| torch.save(value, stream) | |
| stream.flush() | |
| os.fsync(stream.fileno()) | |
| temporary.replace(path) | |
| def autocast(device, precision="bf16"): | |
| return ( | |
| torch.autocast("cuda", dtype=torch.bfloat16) | |
| if torch.device(device).type == "cuda" and precision == "bf16" | |
| else nullcontext() | |
| ) | |
| def job_lock(directory): | |
| import fcntl | |
| with (Path(directory) / ".job.lock").open("a") as stream: | |
| fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB) | |
| yield | |