File size: 7,065 Bytes
db32cff | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 | """Full 500-epoch Darcy85 training, with loss/metrics printed as training runs."""
import csv
import os
import random
import time
import numpy as np
import torch
from common import (ROOT, MoNoLight, atomic_json, atomic_save, code_hashes,
compute_stats, config_args, environment, evaluate, load_data,
normalized, path, read_config, relative_l2,
require_compute_node, seed_all)
def main():
parser = config_args(__doc__)
parser.add_argument('--resume', action='store_true', help='Resume full optimizer/scheduler/RNG state')
args = parser.parse_args()
print('TRAIN_START config loading', flush=True)
cfg = read_config(args.config)
print('TRAIN_START compute node and accelerator initialization', flush=True)
device = require_compute_node()
seed_all(cfg['seed'])
print('TRAIN_START data loading', flush=True)
raw = load_data(cfg)
stats_cpu = compute_stats(raw)
data, stats = normalized(raw, stats_cpu, device)
print('TRAIN_START data normalization complete', flush=True)
t = cfg['training']
results = path(cfg['results'])
results.mkdir(parents=True, exist_ok=True)
model = MoNoLight(**cfg['model']).to(device)
optimizer = torch.optim.AdamW(model.parameters(), lr=t['learning_rate'],
betas=tuple(t['betas']), weight_decay=t['weight_decay'])
scheduler = torch.optim.lr_scheduler.OneCycleLR(
optimizer, max_lr=t['learning_rate'], epochs=t['epochs'], steps_per_epoch=250,
pct_start=t['warmup_epochs']/t['epochs'], div_factor=t['div_factor'],
final_div_factor=t['final_div_factor'], anneal_strategy='cos')
first_epoch, global_step, history = 0, 0, []
elapsed_before = 0.0
checkpoint = path(t['resume_checkpoint'])
hashes = code_hashes()
if args.resume and checkpoint.exists():
state = torch.load(checkpoint, map_location=device, weights_only=False)
assert state['config'] == cfg, 'Cannot change formal configuration when resuming'
assert state['code_hashes'] == hashes, 'Training implementation changed; audit before resuming'
model.load_state_dict(state['model'])
optimizer.load_state_dict(state['optimizer'])
scheduler.load_state_dict(state['scheduler'])
first_epoch, global_step, history = state['epoch'], state['global_step'], state['history']
elapsed_before = state['elapsed_seconds']
# RNG tensors must be copied back to CPU even when optimizer maps to the accelerator.
torch.set_rng_state(state['rng_torch'].cpu())
torch.cuda.set_rng_state_all([s.cpu() for s in state['rng_cuda']])
np.random.set_state(state['rng_numpy'])
random.setstate(state['rng_python'])
print(f'RESUME epoch={first_epoch} updates={global_step}; optimizer/scheduler/RNG restored', flush=True)
elif checkpoint.exists() or path(t['checkpoint']).exists():
raise RuntimeError('Existing training weights found: use --resume; refusing overwrite/retrain')
runtime = environment()
runtime.update(trainable_parameters=sum(p.numel() for p in model.parameters() if p.requires_grad),
all_parameters=sum(p.numel() for p in model.parameters()),
precision='float32', code_hashes=hashes)
atomic_json(results / 'training_environment.json', runtime)
print('ENVIRONMENT', runtime, flush=True)
start = time.monotonic()
columns = ['epoch','updates','train_relative_l2','test_relative_l2','learning_rate','epoch_seconds','elapsed_seconds']
# The checkpoint is authoritative: discard any uncommitted trailing history from interruption.
with open(results/'history.csv', 'w', newline='') as f:
writer = csv.DictWriter(f, fieldnames=columns)
writer.writeheader()
writer.writerows(history)
for epoch in range(first_epoch, t['epochs']):
epoch_start = time.monotonic()
model.train()
permutation = torch.randperm(1000, device=device)
train_sum = torch.zeros((), device=device)
for j in range(250):
ids = permutation[j*4:(j+1)*4]
optimizer.zero_grad(set_to_none=True)
output = model(data['coordinates'], data['train_x'][ids])
output = output * stats['y_std'] + stats['y_mean']
loss = relative_l2(output, data['train_y'][ids]).mean()
if not torch.isfinite(loss):
raise FloatingPointError(f'Nonfinite loss at epoch={epoch+1}, step={j+1}')
loss.backward()
norm = torch.nn.utils.clip_grad_norm_(model.parameters(), t['grad_clip'], error_if_nonfinite=True)
optimizer.step()
scheduler.step()
global_step += 1
train_sum += loss.detach() * 4
if j == 0 or (j+1) % t['print_every_steps'] == 0:
print(f'TRAIN epoch={epoch+1}/500 batch={j+1}/250 updates={global_step} '
f'loss_raw_relative_l2={loss.item():.9f} grad_norm={norm.item():.6g} '
f'lr={scheduler.get_last_lr()[0]:.10g}', flush=True)
errors, _ = evaluate(model, data, stats)
row = dict(epoch=epoch+1, updates=global_step,
train_relative_l2=train_sum.item()/1000,
test_relative_l2=errors.mean().item(),
learning_rate=scheduler.get_last_lr()[0],
epoch_seconds=time.monotonic()-epoch_start,
elapsed_seconds=elapsed_before+time.monotonic()-start)
history.append(row)
print('EPOCH', row, flush=True)
state = dict(model=model.state_dict(), optimizer=optimizer.state_dict(),
scheduler=scheduler.state_dict(), epoch=epoch+1, global_step=global_step,
history=history, normalization=stats_cpu, config=cfg,
code_hashes=hashes, environment=runtime,
elapsed_seconds=row['elapsed_seconds'], rng_torch=torch.get_rng_state(),
rng_cuda=torch.cuda.get_rng_state_all(), rng_numpy=np.random.get_state(),
rng_python=random.getstate(), selection=t['selection'])
atomic_save(checkpoint, state)
with open(results/'history.csv', 'a', newline='') as f:
csv.DictWriter(f, fieldnames=columns).writerow(row)
atomic_json(results/'training_status.json', dict(
status='completed' if epoch+1 == 500 else 'running',
completed_epochs=epoch+1, formal_epochs=500, global_step=global_step,
formal_updates=125000, checkpoint=str(checkpoint), last_epoch=row,
slurm_job_id=os.environ.get('SLURM_JOB_ID')))
assert global_step == 125000 and len(history) == 500
final = torch.load(checkpoint, map_location='cpu', weights_only=False)
assert final['epoch'] == 500
atomic_save(path(t['checkpoint']), final)
print(f'COMPLETE 500 epochs; 125000 updates; final epoch saved to {path(t["checkpoint"])}', flush=True)
if __name__ == '__main__':
main()
|