"""Full 500-epoch Darcy85 training, with loss/metrics printed as training runs.""" import csv import os import random import time import numpy as np import torch from common import (ROOT, MoNoLight, atomic_json, atomic_save, code_hashes, compute_stats, config_args, environment, evaluate, load_data, normalized, path, read_config, relative_l2, require_compute_node, seed_all) def main(): parser = config_args(__doc__) parser.add_argument('--resume', action='store_true', help='Resume full optimizer/scheduler/RNG state') args = parser.parse_args() print('TRAIN_START config loading', flush=True) cfg = read_config(args.config) print('TRAIN_START compute node and accelerator initialization', flush=True) device = require_compute_node() seed_all(cfg['seed']) print('TRAIN_START data loading', flush=True) raw = load_data(cfg) stats_cpu = compute_stats(raw) data, stats = normalized(raw, stats_cpu, device) print('TRAIN_START data normalization complete', flush=True) t = cfg['training'] results = path(cfg['results']) results.mkdir(parents=True, exist_ok=True) model = MoNoLight(**cfg['model']).to(device) optimizer = torch.optim.AdamW(model.parameters(), lr=t['learning_rate'], betas=tuple(t['betas']), weight_decay=t['weight_decay']) scheduler = torch.optim.lr_scheduler.OneCycleLR( optimizer, max_lr=t['learning_rate'], epochs=t['epochs'], steps_per_epoch=250, pct_start=t['warmup_epochs']/t['epochs'], div_factor=t['div_factor'], final_div_factor=t['final_div_factor'], anneal_strategy='cos') first_epoch, global_step, history = 0, 0, [] elapsed_before = 0.0 checkpoint = path(t['resume_checkpoint']) hashes = code_hashes() if args.resume and checkpoint.exists(): state = torch.load(checkpoint, map_location=device, weights_only=False) assert state['config'] == cfg, 'Cannot change formal configuration when resuming' assert state['code_hashes'] == hashes, 'Training implementation changed; audit before resuming' model.load_state_dict(state['model']) optimizer.load_state_dict(state['optimizer']) scheduler.load_state_dict(state['scheduler']) first_epoch, global_step, history = state['epoch'], state['global_step'], state['history'] elapsed_before = state['elapsed_seconds'] # RNG tensors must be copied back to CPU even when optimizer maps to the accelerator. torch.set_rng_state(state['rng_torch'].cpu()) torch.cuda.set_rng_state_all([s.cpu() for s in state['rng_cuda']]) np.random.set_state(state['rng_numpy']) random.setstate(state['rng_python']) print(f'RESUME epoch={first_epoch} updates={global_step}; optimizer/scheduler/RNG restored', flush=True) elif checkpoint.exists() or path(t['checkpoint']).exists(): raise RuntimeError('Existing training weights found: use --resume; refusing overwrite/retrain') runtime = environment() runtime.update(trainable_parameters=sum(p.numel() for p in model.parameters() if p.requires_grad), all_parameters=sum(p.numel() for p in model.parameters()), precision='float32', code_hashes=hashes) atomic_json(results / 'training_environment.json', runtime) print('ENVIRONMENT', runtime, flush=True) start = time.monotonic() columns = ['epoch','updates','train_relative_l2','test_relative_l2','learning_rate','epoch_seconds','elapsed_seconds'] # The checkpoint is authoritative: discard any uncommitted trailing history from interruption. with open(results/'history.csv', 'w', newline='') as f: writer = csv.DictWriter(f, fieldnames=columns) writer.writeheader() writer.writerows(history) for epoch in range(first_epoch, t['epochs']): epoch_start = time.monotonic() model.train() permutation = torch.randperm(1000, device=device) train_sum = torch.zeros((), device=device) for j in range(250): ids = permutation[j*4:(j+1)*4] optimizer.zero_grad(set_to_none=True) output = model(data['coordinates'], data['train_x'][ids]) output = output * stats['y_std'] + stats['y_mean'] loss = relative_l2(output, data['train_y'][ids]).mean() if not torch.isfinite(loss): raise FloatingPointError(f'Nonfinite loss at epoch={epoch+1}, step={j+1}') loss.backward() norm = torch.nn.utils.clip_grad_norm_(model.parameters(), t['grad_clip'], error_if_nonfinite=True) optimizer.step() scheduler.step() global_step += 1 train_sum += loss.detach() * 4 if j == 0 or (j+1) % t['print_every_steps'] == 0: print(f'TRAIN epoch={epoch+1}/500 batch={j+1}/250 updates={global_step} ' f'loss_raw_relative_l2={loss.item():.9f} grad_norm={norm.item():.6g} ' f'lr={scheduler.get_last_lr()[0]:.10g}', flush=True) errors, _ = evaluate(model, data, stats) row = dict(epoch=epoch+1, updates=global_step, train_relative_l2=train_sum.item()/1000, test_relative_l2=errors.mean().item(), learning_rate=scheduler.get_last_lr()[0], epoch_seconds=time.monotonic()-epoch_start, elapsed_seconds=elapsed_before+time.monotonic()-start) history.append(row) print('EPOCH', row, flush=True) state = dict(model=model.state_dict(), optimizer=optimizer.state_dict(), scheduler=scheduler.state_dict(), epoch=epoch+1, global_step=global_step, history=history, normalization=stats_cpu, config=cfg, code_hashes=hashes, environment=runtime, elapsed_seconds=row['elapsed_seconds'], rng_torch=torch.get_rng_state(), rng_cuda=torch.cuda.get_rng_state_all(), rng_numpy=np.random.get_state(), rng_python=random.getstate(), selection=t['selection']) atomic_save(checkpoint, state) with open(results/'history.csv', 'a', newline='') as f: csv.DictWriter(f, fieldnames=columns).writerow(row) atomic_json(results/'training_status.json', dict( status='completed' if epoch+1 == 500 else 'running', completed_epochs=epoch+1, formal_epochs=500, global_step=global_step, formal_updates=125000, checkpoint=str(checkpoint), last_epoch=row, slurm_job_id=os.environ.get('SLURM_JOB_ID'))) assert global_step == 125000 and len(history) == 500 final = torch.load(checkpoint, map_location='cpu', weights_only=False) assert final['epoch'] == 500 atomic_save(path(t['checkpoint']), final) print(f'COMPLETE 500 epochs; 125000 updates; final epoch saved to {path(t["checkpoint"])}', flush=True) if __name__ == '__main__': main()