File size: 1,012 Bytes
357baae | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 | """Complete in-memory encoding followed by atomic artifact replacement."""
from io import BytesIO
import os
from pathlib import Path
import zipfile
import numpy as np
def atomic_write(path, payload):
path = Path(path)
temporary = path.with_name(path.name+f".{os.getpid()}.partial")
try:
with temporary.open("wb") as stream:
stream.write(payload)
stream.flush()
os.fsync(stream.fileno())
if temporary.stat().st_size != len(payload):
raise IOError("Incomplete temporary artifact write")
temporary.replace(path)
finally:
if temporary.exists():
temporary.unlink()
def save_arrays(path, **arrays):
buffer = BytesIO()
np.savez_compressed(buffer, **arrays)
payload = buffer.getvalue()
with zipfile.ZipFile(BytesIO(payload)) as archive:
if set(archive.namelist()) != {key+".npy" for key in arrays}:
raise IOError("Incomplete NPZ encoding")
atomic_write(path, payload)
|