File size: 3,766 Bytes
db32cff
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
"""Isolated six-path check on real Darcy samples, never used as formal training."""
import math
import time

import numpy as np
import torch

from common import (MoNoLight, atomic_json, compute_stats, config_args, environment,
                    load_data, normalized, path, read_config, relative_l2,
                    require_compute_node, seed_all)
from model.MoNo import sinkhorn_transport


def main():
    cfg=read_config(config_args(__doc__).parse_args().config)
    device=require_compute_node()
    seed_all(0)
    raw=load_data(cfg)
    data,stats=normalized(raw,compute_stats(raw),device)
    model=MoNoLight(**cfg['model']).to(device)
    checks={}
    model.train()
    t=time.monotonic()
    y=model(data['coordinates'],data['train_x'][:4])
    assert y.shape==(4,7225,1) and torch.isfinite(y).all()
    checks['forward_pass']='passed'
    y=y*stats['y_std']+stats['y_mean']
    loss=relative_l2(y,data['train_y'][:4]).mean()
    loss.backward()
    assert all(p.grad is not None and torch.isfinite(p.grad).all() for p in model.parameters() if p.requires_grad)
    checks['backward_pass']='passed'
    torch.cuda.synchronize()
    print(f'SMOKE full-resolution forward/backward {time.monotonic()-t:.3f}s',flush=True)
    # One isolated epoch on 8 actual training samples, two complete optimizer updates.
    # Formal training is a separate fresh seed-0 model and still uses all 1000 samples.
    opt=torch.optim.AdamW(model.parameters(),lr=1e-5)
    losses=[]
    start=time.monotonic()
    for j in range(2):
        opt.zero_grad(set_to_none=True)
        y=model(data['coordinates'],data['train_x'][4*j:4*j+4])*stats['y_std']+stats['y_mean']
        loss=relative_l2(y,data['train_y'][4*j:4*j+4]).mean()
        loss.backward(); opt.step()
        losses.append(loss.item())
    assert np.isfinite(losses).all()
    checks['train_loop_dry_run']='passed: one isolated epoch on 8 real samples; formal budget unchanged'
    torch.cuda.synchronize()
    seconds=(time.monotonic()-start)/2
    model.eval()
    with torch.no_grad():
        y=model(data['coordinates'],data['test_x'][:4])*stats['y_std']+stats['y_mean']
        metric=relative_l2(y,data['test_y'][:4])
    assert torch.isfinite(metric).all()
    checks['validation_loop_dry_run']='passed'
    yn=y.cpu().numpy().astype(np.float64).reshape(4,-1)
    tn=data['test_y'][:4].cpu().numpy().astype(np.float64).reshape(4,-1)
    np.testing.assert_allclose(metric.cpu().numpy(),np.linalg.norm(yn-tn,axis=1)/np.linalg.norm(tn,axis=1),rtol=1e-5)
    # Independently verify finite Sinkhorn equations and derivatives in float64.
    z=torch.randn(1,17,7,device=device,dtype=torch.float64,requires_grad=True)
    p=sinkhorn_transport(z)
    torch.testing.assert_close(p.sum(-2),torch.full((1,7),1/7,device=device,dtype=torch.float64),atol=1e-10,rtol=1e-10)
    assert torch.autograd.gradcheck(sinkhorn_transport,(z,),fast_mode=True)
    checks['domain_metric_dry_run']='passed: raw relative L2 vs NumPy; OT marginal and finite-step gradcheck'
    assert cfg['model']['width']==96 and tuple(cfg['model']['latent_sizes'])==(512,256,128,64)
    assert tuple(cfg['model']['depths'])==(3,1,1,1)
    assert not any('weight_g' in k for k in model.state_dict())
    checks['config_consistency']='passed'
    result=dict(status='passed',attempt=1,checks=checks,environment=environment(),
                seconds_per_training_step=seconds,estimated_formal_hours=seconds*125000/3600,
                trainable_parameters=sum(p.numel() for p in model.parameters() if p.requires_grad),
                losses=losses,smoke_samples=8,formal_samples=1000,formal_epochs=500)
    atomic_json(path(cfg['results'])/'smoke_checks.json',result)
    print('SMOKE',result,flush=True)


if __name__=='__main__':
    main()