Download scripts/common.py from OneScience-Group/MoNo-light: direct link, hf CLI and curl.
- Browser
- Download file 6.51 kB
-
https://huggingface.co/OneScience-Group/MoNo-light/resolve/main/scripts/common.py
- Command line
-
hf download hf://OneScience-Group/MoNo-light/scripts/common.py
-
curl -L -o common.py https://huggingface.co/OneScience-Group/MoNo-light/resolve/main/scripts/common.py
6.51 kB
| """Data and evaluation contracts shared by the independently written runners.""" | |
| import argparse | |
| import hashlib | |
| import json | |
| import os | |
| from pathlib import Path | |
| import platform | |
| import random | |
| import sys | |
| import numpy as np | |
| from scipy.io import loadmat, whosmat | |
| import torch | |
| import yaml | |
| ROOT = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(ROOT)) | |
| from model.MoNo import MoNoLight | |
| def config_args(description): | |
| parser = argparse.ArgumentParser(description=description) | |
| parser.add_argument('--config', default=str(ROOT / 'config/config.yaml')) | |
| return parser | |
| def read_config(path): | |
| with open(path) as f: | |
| cfg = yaml.safe_load(f) | |
| assert (cfg['data']['train_samples'], cfg['data']['test_samples']) == (1000, 200) | |
| assert cfg['data']['stride'] == 5 and cfg['data']['grid_size'] == 85 | |
| t = cfg['training'] | |
| assert (t['epochs'], t['batch_size'], t['warmup_epochs'], t['learning_rate']) == (500, 4, 100, .001) | |
| return cfg | |
| def path(value): | |
| p = Path(value) | |
| return p if p.is_absolute() else ROOT / p | |
| def atomic_json(destination, obj): | |
| destination = Path(destination) | |
| destination.parent.mkdir(parents=True, exist_ok=True) | |
| tmp = destination.with_suffix(destination.suffix + '.tmp') | |
| tmp.write_text(json.dumps(obj, indent=2, ensure_ascii=False, allow_nan=False)) | |
| os.replace(tmp, destination) | |
| def atomic_save(destination, obj): | |
| destination = Path(destination) | |
| destination.parent.mkdir(parents=True, exist_ok=True) | |
| tmp = destination.with_suffix(destination.suffix + '.tmp') | |
| torch.save(obj, tmp) | |
| os.replace(tmp, destination) | |
| def seed_all(seed): | |
| random.seed(seed) | |
| np.random.seed(seed) | |
| torch.manual_seed(seed) | |
| torch.cuda.manual_seed_all(seed) | |
| def require_compute_node(): | |
| assert torch.cuda.is_available(), 'Assigned GPU/DCU unavailable; no CPU fallback' | |
| torch.set_num_threads(int(os.environ.get('OMP_NUM_THREADS', '8'))) | |
| return torch.device('cuda') | |
| def environment(): | |
| return dict(python=sys.version, torch=torch.__version__, hip=torch.version.hip, | |
| cuda=torch.version.cuda, device=torch.cuda.get_device_name(), | |
| device_count=torch.cuda.device_count(), platform=platform.platform(), | |
| hostname=platform.node(), slurm_job_id=os.environ.get('SLURM_JOB_ID')) | |
| def source_signature(cfg): | |
| d = cfg['data'] | |
| return [dict(file=str(Path(d['directory']) / d[k]), | |
| size=(Path(d['directory']) / d[k]).stat().st_size, | |
| mtime_ns=(Path(d['directory']) / d[k]).stat().st_mtime_ns) | |
| for k in ['train_file', 'test_file']] | |
| def load_data(cfg): | |
| d = cfg['data'] | |
| cache = path(d['cache']) | |
| signature = source_signature(cfg) | |
| if cache.exists(): | |
| with np.load(cache) as f: | |
| assert json.loads(str(f['signature'])) == signature, 'Data source changed' | |
| arrays = {k: f[k].copy() for k in ['train_coeff', 'train_sol', 'test_coeff', 'test_sol']} | |
| else: | |
| arrays = {} | |
| schema = {} | |
| for split, count in [('train', 1000), ('test', 200)]: | |
| src = Path(d['directory']) / d[split + '_file'] | |
| schema[split] = whosmat(src) | |
| print(f'DATA reading {src}, first {count} samples; stride=5', flush=True) | |
| mat = loadmat(src, variable_names=['coeff', 'sol']) | |
| for field in ['coeff', 'sol']: | |
| assert mat[field].shape == (1024, 421, 421), (src, field, mat[field].shape) | |
| a = np.ascontiguousarray(mat[field][:count, ::5, ::5], dtype=np.float32) | |
| assert a.shape == (count, 85, 85) and np.isfinite(a).all() | |
| arrays[split + '_' + field] = a | |
| del mat | |
| cache.parent.mkdir(parents=True, exist_ok=True) | |
| tmp = cache.with_suffix('.tmp.npz') | |
| np.savez(tmp, **arrays, signature=json.dumps(signature)) | |
| os.replace(tmp, cache) | |
| atomic_json(path(cfg['results']) / 'data_manifest.json', dict( | |
| sources=signature, schema=schema, train_samples=1000, test_samples=200, | |
| split='file1[0:1000], file2[0:200]', downsample='[:,::5,::5]', | |
| grid=[85,85], coeff='permeability', sol='pressure', units='stored numerical scale; unspecified')) | |
| lin = np.linspace(0, 1, 85, dtype=np.float32) | |
| grid = np.stack(np.meshgrid(lin, lin, indexing='xy'), -1).reshape(-1, 2) | |
| coordinates = torch.from_numpy(grid).unsqueeze(0) | |
| out = {'coordinates_raw': coordinates} | |
| for split in ['train', 'test']: | |
| coeff = torch.from_numpy(arrays[split + '_coeff']).reshape(-1, 7225, 1) | |
| sol = torch.from_numpy(arrays[split + '_sol']).reshape(-1, 7225, 1) | |
| out[split + '_coeff'] = coeff | |
| out[split + '_sol'] = sol | |
| out[split + '_features'] = torch.cat([coordinates.expand(coeff.shape[0], -1, -1), coeff], -1) | |
| return out | |
| def compute_stats(data): | |
| x, y = data['train_features'], data['train_sol'] | |
| return dict(x_mean=x.mean((0,1), keepdim=True), | |
| x_std=x.std((0,1), keepdim=True) + 1e-8, | |
| y_mean=y.mean((0,1), keepdim=True), | |
| y_std=y.std((0,1), keepdim=True) + 1e-8) | |
| def normalized(data, stats, device): | |
| s = {k: v.to(device) for k, v in stats.items()} | |
| out = {'coordinates': ((data['coordinates_raw'].to(device) - s['x_mean'][..., :2]) / | |
| s['x_std'][..., :2])} | |
| for split in ['train', 'test']: | |
| out[split + '_x'] = (data[split + '_features'].to(device) - s['x_mean']) / s['x_std'] | |
| out[split + '_y'] = data[split + '_sol'].to(device) | |
| return out, s | |
| def relative_l2(prediction, target): | |
| return (prediction - target).flatten(1).norm(dim=1) / target.flatten(1).norm(dim=1) | |
| def evaluate(model, data, stats, batch_size=4, return_predictions=False): | |
| model.eval() | |
| errors, predictions = [], [] | |
| for i in range(0, data['test_x'].shape[0], batch_size): | |
| y = model(data['coordinates'], data['test_x'][i:i+batch_size]) | |
| y = y * stats['y_std'] + stats['y_mean'] | |
| errors.append(relative_l2(y, data['test_y'][i:i+batch_size]).cpu()) | |
| if return_predictions: | |
| predictions.append(y.cpu()) | |
| return torch.cat(errors), (torch.cat(predictions) if return_predictions else None) | |
| def code_hashes(): | |
| return {str(f.relative_to(ROOT)): hashlib.sha256(f.read_bytes()).hexdigest() | |
| for f in [ROOT/'model/MoNo.py', ROOT/'config/config.yaml', | |
| ROOT/'scripts/train.py', ROOT/'scripts/common.py']} | |