MoNo-light / scripts /smoke.py
yushuang88's picture
Upload folder using huggingface_hub
db32cff verified
Raw History Blame Contribute Delete
3.77 kB
"""Isolated six-path check on real Darcy samples, never used as formal training."""
import math
import time
import numpy as np
import torch
from common import (MoNoLight, atomic_json, compute_stats, config_args, environment,
load_data, normalized, path, read_config, relative_l2,
require_compute_node, seed_all)
from model.MoNo import sinkhorn_transport
def main():
cfg=read_config(config_args(__doc__).parse_args().config)
device=require_compute_node()
seed_all(0)
raw=load_data(cfg)
data,stats=normalized(raw,compute_stats(raw),device)
model=MoNoLight(**cfg['model']).to(device)
checks={}
model.train()
t=time.monotonic()
y=model(data['coordinates'],data['train_x'][:4])
assert y.shape==(4,7225,1) and torch.isfinite(y).all()
checks['forward_pass']='passed'
y=y*stats['y_std']+stats['y_mean']
loss=relative_l2(y,data['train_y'][:4]).mean()
loss.backward()
assert all(p.grad is not None and torch.isfinite(p.grad).all() for p in model.parameters() if p.requires_grad)
checks['backward_pass']='passed'
torch.cuda.synchronize()
print(f'SMOKE full-resolution forward/backward {time.monotonic()-t:.3f}s',flush=True)
# One isolated epoch on 8 actual training samples, two complete optimizer updates.
# Formal training is a separate fresh seed-0 model and still uses all 1000 samples.
opt=torch.optim.AdamW(model.parameters(),lr=1e-5)
losses=[]
start=time.monotonic()
for j in range(2):
opt.zero_grad(set_to_none=True)
y=model(data['coordinates'],data['train_x'][4*j:4*j+4])*stats['y_std']+stats['y_mean']
loss=relative_l2(y,data['train_y'][4*j:4*j+4]).mean()
loss.backward(); opt.step()
losses.append(loss.item())
assert np.isfinite(losses).all()
checks['train_loop_dry_run']='passed: one isolated epoch on 8 real samples; formal budget unchanged'
torch.cuda.synchronize()
seconds=(time.monotonic()-start)/2
model.eval()
with torch.no_grad():
y=model(data['coordinates'],data['test_x'][:4])*stats['y_std']+stats['y_mean']
metric=relative_l2(y,data['test_y'][:4])
assert torch.isfinite(metric).all()
checks['validation_loop_dry_run']='passed'
yn=y.cpu().numpy().astype(np.float64).reshape(4,-1)
tn=data['test_y'][:4].cpu().numpy().astype(np.float64).reshape(4,-1)
np.testing.assert_allclose(metric.cpu().numpy(),np.linalg.norm(yn-tn,axis=1)/np.linalg.norm(tn,axis=1),rtol=1e-5)
# Independently verify finite Sinkhorn equations and derivatives in float64.
z=torch.randn(1,17,7,device=device,dtype=torch.float64,requires_grad=True)
p=sinkhorn_transport(z)
torch.testing.assert_close(p.sum(-2),torch.full((1,7),1/7,device=device,dtype=torch.float64),atol=1e-10,rtol=1e-10)
assert torch.autograd.gradcheck(sinkhorn_transport,(z,),fast_mode=True)
checks['domain_metric_dry_run']='passed: raw relative L2 vs NumPy; OT marginal and finite-step gradcheck'
assert cfg['model']['width']==96 and tuple(cfg['model']['latent_sizes'])==(512,256,128,64)
assert tuple(cfg['model']['depths'])==(3,1,1,1)
assert not any('weight_g' in k for k in model.state_dict())
checks['config_consistency']='passed'
result=dict(status='passed',attempt=1,checks=checks,environment=environment(),
seconds_per_training_step=seconds,estimated_formal_hours=seconds*125000/3600,
trainable_parameters=sum(p.numel() for p in model.parameters() if p.requires_grad),
losses=losses,smoke_samples=8,formal_samples=1000,formal_epochs=500)
atomic_json(path(cfg['results'])/'smoke_checks.json',result)
print('SMOKE',result,flush=True)
if __name__=='__main__':
main()