File size: 841 Bytes
d5ef81d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 | import json, hashlib, sys
from pathlib import Path
from sklearn.datasets import fetch_openml
OUT = Path('/nvme0n1-disk/cache/foundry/tabular')
OUT.mkdir(parents=True, exist_ok=True)
SETS = {'adult': 1590, 'credit-g': 31, 'bank-marketing': 1461, 'phoneme': 1489, 'diabetes': 37}
meta = {}
for name, did in SETS.items():
b = fetch_openml(data_id=did, as_frame=True, parser='auto', data_home=str(OUT / 'openml'))
df = b.frame
p = OUT / f'{name}.parquet'
df.to_parquet(p)
meta[name] = dict(openml_id=did, rows=len(df), cols=df.shape[1], target=b.target.name,
sha256=hashlib.sha256(p.read_bytes()).hexdigest(), license=b.details.get('licence'),
version=b.details.get('version'))
print(name, meta[name], flush=True)
(OUT / 'manifest.json').write_text(json.dumps(meta, indent=2))
|