File size: 4,612 Bytes
db32cff
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
"""Independently reload epoch-500 weights; evaluate all 200 samples at raw scale."""
import hashlib
import importlib.metadata
from copy import deepcopy

import numpy as np
import torch

from common import (ROOT, MoNoLight, atomic_json, config_args, environment, evaluate,
                    load_data, normalized, path, read_config, require_compute_node)


def main():
    args = config_args(__doc__).parse_args()
    cfg = read_config(args.config)
    device = require_compute_node()
    checkpoint = path(cfg['training']['checkpoint'])
    state = torch.load(checkpoint, map_location='cpu', weights_only=False)
    assert state['epoch'] == 500 and state['global_step'] == 125000
    trained_config = deepcopy(state['config'])
    trained_config['data']['directory'] = cfg['data']['directory']
    assert trained_config == cfg
    model = MoNoLight(**cfg['model']).to(device)
    model.load_state_dict(state['model'], strict=True)
    raw = load_data(cfg)
    data, stats = normalized(raw, state['normalization'], device)
    errors, predictions = evaluate(model, data, stats, return_predictions=True)
    assert predictions.shape == (200, 7225, 1)
    # Independent double precision NumPy recomputation of the requested metric.
    pred = predictions.numpy().reshape(200,85,85)
    target = raw['test_sol'].numpy().reshape(200,85,85)
    coeff = raw['test_coeff'].numpy().reshape(200,85,85)
    per_sample = np.linalg.norm((pred.astype(np.float64)-target).reshape(200,-1),axis=1) / np.linalg.norm(target.astype(np.float64).reshape(200,-1),axis=1)
    np.testing.assert_allclose(errors.numpy(), per_sample, rtol=2e-5, atol=1e-8)
    result_dir = path(cfg['results'])
    np.savez(result_dir/'predictions.npz', prediction=pred, target=target,
             coeff=coeff, coordinates=raw['coordinates_raw'].numpy().reshape(85,85,2),
             relative_l2=per_sample, sample_id=np.arange(200))
    np.savetxt(result_dir/'per_sample_relative_l2.csv',
               np.column_stack((np.arange(200),per_sample)), delimiter=',',
               header='sample_id,relative_l2_raw', comments='', fmt=['%d','%.12g'])
    mean = float(per_sample.mean())
    reference = cfg['paper_relative_l2']
    metrics = dict(mean_relative_l2_raw=mean, test_samples=200,
                   actual_training_epochs=state['epoch'], optimizer_updates=state['global_step'],
                   paper_table3=reference, signed_gap=mean-reference,
                   absolute_gap=abs(mean-reference), relative_gap_percent=100*(mean/reference-1),
                   selection=state['selection'], checkpoint=str(checkpoint),
                   checkpoint_sha256=hashlib.sha256(checkpoint.read_bytes()).hexdigest(),
                   min=float(per_sample.min()), max=float(per_sample.max()),
                   median=float(np.median(per_sample)),
                   training_environment=state['environment'], inference_environment=environment(),
                   software_versions={name:importlib.metadata.version(name) for name in
                                      ['torch','triton','onescience','numpy','scipy','PyYAML','matplotlib']},
                   configured_cluster='SCNet compute node for the original training',
                   gpu_total_memory_bytes=torch.cuda.get_device_properties(device).total_memory,
                   limitations=[
                       'Single DCU/PyTorch 2.4.1 HIP backend versus paper RTX4090/PyTorch2.8 CUDA12.8.',
                       'Configured system RAM allocation is 30G, versus up to 600GB reported by the paper; all 1000/200 samples and model dimensions retained.',
                       'Installed Triton is 3.0.0 with DTK customizations, versus paper Triton3.4.0; author fused kernels are not used.',
                       'Independent implementation, packed QKV/SDPA attention, and RNG consumption differ from explicit author attention; not bitwise reproduction.',
                       'Portable PyTorch log-domain Sinkhorn instead of author fused Triton kernels; same finite-step equations and differentiation.',
                       'Identical Darcy geometry is computed once and broadcast over batch; arithmetic ordering differs.',
                       'Paper does not specify seed uncertainty or a tolerance for the Table3 result.',
                       'Final epoch checkpoint named best_model.pth; no test-set model selection.',
                       'Units unspecified by supplied data; metric uses stored pressure scale.'])
    atomic_json(result_dir/'metrics.json', metrics)
    print('INFERENCE', metrics, flush=True)


if __name__ == '__main__':
    main()