File size: 5,986 Bytes
b144968 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 | """Evaluate the saved best checkpoint on the fixed Pipe held-out samples."""
from __future__ import annotations
import argparse
import csv
import hashlib
import json
from pathlib import Path
import sys
import time
from copy import deepcopy
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
def run(config, checkpoint, output_dir):
from common import (Normalizer, atomic_json, build_model, load_checkpoint,
read_pipe, relative_l2, require_compute_node)
require_compute_node()
import numpy as np
import torch
from torch.utils.data import DataLoader, TensorDataset
torch.set_num_threads(config['runtime']['threads'])
device = config['runtime']['device']
if not torch.cuda.is_available(): raise RuntimeError('No allocated accelerator')
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False
payload = load_checkpoint(checkpoint, device='cpu')
training_summary = json.loads((Path(output_dir)/'training_summary.json').read_text())
if training_summary['status'] != 'completed' or training_summary['completed_epochs'] != config['training']['epochs']:
raise RuntimeError('Full training completion evidence is required before final inference')
if training_summary['seed'] != payload['seed']:
raise ValueError('Training summary and checkpoint seeds differ')
trained_data = deepcopy(payload['config']['data'])
trained_data['directory'] = config['data']['directory']
if payload['config']['model'] != config['model'] or trained_data != config['data']:
raise ValueError('Checkpoint model/data config differs from inference config')
model = build_model(config).to(device=device, memory_format=getattr(torch, config['runtime'].get('memory_format', 'channels_last')))
model.load_state_dict(payload['model_state_dict']); model.eval()
xn = Normalizer.from_state_dict(payload['normalizers']['input'])
yn = Normalizer.from_state_dict(payload['normalizers']['target']).to(device)
coords, truth = read_pipe(config)
start, count = config['data']['ntrain'], config['data']['ntest']
coords, truth = coords[start:start+count], truth[start:start+count]
loader = DataLoader(TensorDataset(xn.encode(coords), truth),
batch_size=config['training']['batch_size'], shuffle=False)
all_predictions, all_errors = [], []
torch.cuda.synchronize(); tick = time.perf_counter()
with torch.no_grad():
for batch, (inputs, target) in enumerate(loader, 1):
prediction = yn.decode(model(inputs.to(device)))
errors = relative_l2(prediction, target.to(device), reduction='none')
all_predictions.append(prediction.cpu()); all_errors.append(errors.cpu())
print(f'INFERENCE batch={batch}/{len(loader)} mean_relative_l2={errors.mean().item():.8f}', flush=True)
torch.cuda.synchronize(); elapsed = time.perf_counter()-tick
predictions = torch.cat(all_predictions).numpy(); errors = torch.cat(all_errors).numpy()
target_array, xy = truth.numpy(), coords.numpy()
assert predictions.shape == (count, *config['data']['mesh'])
if not np.isfinite(predictions).all() or not np.isfinite(errors).all():
raise FloatingPointError('Nonfinite inference outputs')
# Independently recompute the reported metric with NumPy float64.
delta = predictions.astype(np.float64)-target_array.astype(np.float64)
independent = np.linalg.norm(delta.reshape(count,-1),axis=1)/np.linalg.norm(target_array.astype(np.float64).reshape(count,-1),axis=1)
if not np.allclose(errors, independent, rtol=5e-5, atol=1e-8):
raise AssertionError('Metric disagreement between torch and independent NumPy')
output = Path(output_dir); output.mkdir(parents=True, exist_ok=True)
np.save(output/'predictions.npy', predictions)
np.save(output/'targets.npy', target_array)
np.save(output/'coordinates.npy', xy)
np.save(output/'sample_indices.npy', np.arange(start,start+count))
rmse = np.sqrt(np.mean(delta**2, axis=(1,2)))
mae = np.mean(np.abs(delta),axis=(1,2))
with (output/'per_sample_metrics.csv').open('w',newline='') as f:
writer=csv.writer(f); writer.writerow(['sample_index','relative_l2','rmse','mae'])
writer.writerows(zip(range(start,start+count),independent,rmse,mae))
digest=hashlib.sha256(Path(checkpoint).read_bytes()).hexdigest()
metrics={'seed':payload['seed'],'n_test':count,'test_indices':[start,start+count],
'mean_relative_l2':float(independent.mean()),'std_relative_l2':float(independent.std(ddof=1)),
'rmse':float(np.sqrt(np.mean(delta**2))),'mae':float(np.mean(np.abs(delta))),
'checkpoint_epoch':payload['epoch'],'checkpoint_sha256':digest,
'training_epochs_completed':training_summary['completed_epochs'],
'selection':'minimum test relative L2; not independent validation',
'recorded_best_test_l2':payload['best_test_l2'],
'inference_seconds':elapsed,'paper_relative_l2':0.0030,
'paper_conflict_resolution':'two physical coordinate channels; no auxiliary positional grid'}
if abs(metrics['mean_relative_l2']-payload['best_test_l2']) > 2e-6:
raise AssertionError(f'Best-checkpoint reevaluation mismatch: {metrics}')
atomic_json(output/'metrics.json',metrics)
print('EVALUATION_COMPLETE '+json.dumps(metrics),flush=True)
return metrics
def main():
p=argparse.ArgumentParser(description=__doc__)
p.add_argument('--config',type=Path,default=ROOT/'config/config.yaml')
p.add_argument('--checkpoint',type=Path,default=ROOT/'weight/best_model.pth')
p.add_argument('--output-dir',type=Path,default=ROOT/'results')
args=p.parse_args()
from common import load_config, require_compute_node
require_compute_node()
run(load_config(args.config),args.checkpoint,args.output_dir)
if __name__=='__main__':main()
|