psiloceyeben's picture
Upload folder using huggingface_hub
3538060 verified
Raw History Blame Contribute Delete
1.22 kB
"""Minimal .npy reader (format v1/v2, little-endian float64/float32, C order). Stdlib only; refuses object arrays, so a
pickled payload can never be executed. Returns (shape, flat list)."""
import ast
import struct
def read(path):
with open(path, "rb") as f:
magic = f.read(6)
if magic != b"\x93NUMPY":
raise ValueError("not an npy file")
major = f.read(2)[0]
hlen = struct.unpack("<H" if major == 1 else "<I", f.read(2 if major == 1 else 4))[0]
header = ast.literal_eval(f.read(hlen).decode("latin1"))
descr, fortran, shape = header["descr"], header["fortran_order"], tuple(header["shape"])
if descr not in ("<f8", "<f4") or fortran:
raise ValueError("unsupported npy dtype/order: %s fortran=%s" % (descr, fortran))
n = 1
for d in shape:
n *= d
size = 8 if descr == "<f8" else 4
raw = f.read(n * size)
vals = struct.unpack("<%d%s" % (n, "d" if size == 8 else "f"), raw)
return shape, vals
def column(path, col=0):
shape, vals = read(path)
rows, cols = (shape[0], shape[1]) if len(shape) == 2 else (shape[0], 1)
return [vals[i * cols + col] for i in range(rows)]