File size: 1,012 Bytes
357baae
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
"""Complete in-memory encoding followed by atomic artifact replacement."""
from io import BytesIO
import os
from pathlib import Path
import zipfile
import numpy as np


def atomic_write(path, payload):
    path = Path(path)
    temporary = path.with_name(path.name+f".{os.getpid()}.partial")
    try:
        with temporary.open("wb") as stream:
            stream.write(payload)
            stream.flush()
            os.fsync(stream.fileno())
        if temporary.stat().st_size != len(payload):
            raise IOError("Incomplete temporary artifact write")
        temporary.replace(path)
    finally:
        if temporary.exists():
            temporary.unlink()


def save_arrays(path, **arrays):
    buffer = BytesIO()
    np.savez_compressed(buffer, **arrays)
    payload = buffer.getvalue()
    with zipfile.ZipFile(BytesIO(payload)) as archive:
        if set(archive.namelist()) != {key+".npy" for key in arrays}:
            raise IOError("Incomplete NPZ encoding")
    atomic_write(path, payload)