MoNo-light / scripts /train.py
yushuang88's picture
Upload folder using huggingface_hub
db32cff verified
Raw History Blame Contribute Delete
7.07 kB
"""Full 500-epoch Darcy85 training, with loss/metrics printed as training runs."""
import csv
import os
import random
import time
import numpy as np
import torch
from common import (ROOT, MoNoLight, atomic_json, atomic_save, code_hashes,
compute_stats, config_args, environment, evaluate, load_data,
normalized, path, read_config, relative_l2,
require_compute_node, seed_all)
def main():
parser = config_args(__doc__)
parser.add_argument('--resume', action='store_true', help='Resume full optimizer/scheduler/RNG state')
args = parser.parse_args()
print('TRAIN_START config loading', flush=True)
cfg = read_config(args.config)
print('TRAIN_START compute node and accelerator initialization', flush=True)
device = require_compute_node()
seed_all(cfg['seed'])
print('TRAIN_START data loading', flush=True)
raw = load_data(cfg)
stats_cpu = compute_stats(raw)
data, stats = normalized(raw, stats_cpu, device)
print('TRAIN_START data normalization complete', flush=True)
t = cfg['training']
results = path(cfg['results'])
results.mkdir(parents=True, exist_ok=True)
model = MoNoLight(**cfg['model']).to(device)
optimizer = torch.optim.AdamW(model.parameters(), lr=t['learning_rate'],
betas=tuple(t['betas']), weight_decay=t['weight_decay'])
scheduler = torch.optim.lr_scheduler.OneCycleLR(
optimizer, max_lr=t['learning_rate'], epochs=t['epochs'], steps_per_epoch=250,
pct_start=t['warmup_epochs']/t['epochs'], div_factor=t['div_factor'],
final_div_factor=t['final_div_factor'], anneal_strategy='cos')
first_epoch, global_step, history = 0, 0, []
elapsed_before = 0.0
checkpoint = path(t['resume_checkpoint'])
hashes = code_hashes()
if args.resume and checkpoint.exists():
state = torch.load(checkpoint, map_location=device, weights_only=False)
assert state['config'] == cfg, 'Cannot change formal configuration when resuming'
assert state['code_hashes'] == hashes, 'Training implementation changed; audit before resuming'
model.load_state_dict(state['model'])
optimizer.load_state_dict(state['optimizer'])
scheduler.load_state_dict(state['scheduler'])
first_epoch, global_step, history = state['epoch'], state['global_step'], state['history']
elapsed_before = state['elapsed_seconds']
# RNG tensors must be copied back to CPU even when optimizer maps to the accelerator.
torch.set_rng_state(state['rng_torch'].cpu())
torch.cuda.set_rng_state_all([s.cpu() for s in state['rng_cuda']])
np.random.set_state(state['rng_numpy'])
random.setstate(state['rng_python'])
print(f'RESUME epoch={first_epoch} updates={global_step}; optimizer/scheduler/RNG restored', flush=True)
elif checkpoint.exists() or path(t['checkpoint']).exists():
raise RuntimeError('Existing training weights found: use --resume; refusing overwrite/retrain')
runtime = environment()
runtime.update(trainable_parameters=sum(p.numel() for p in model.parameters() if p.requires_grad),
all_parameters=sum(p.numel() for p in model.parameters()),
precision='float32', code_hashes=hashes)
atomic_json(results / 'training_environment.json', runtime)
print('ENVIRONMENT', runtime, flush=True)
start = time.monotonic()
columns = ['epoch','updates','train_relative_l2','test_relative_l2','learning_rate','epoch_seconds','elapsed_seconds']
# The checkpoint is authoritative: discard any uncommitted trailing history from interruption.
with open(results/'history.csv', 'w', newline='') as f:
writer = csv.DictWriter(f, fieldnames=columns)
writer.writeheader()
writer.writerows(history)
for epoch in range(first_epoch, t['epochs']):
epoch_start = time.monotonic()
model.train()
permutation = torch.randperm(1000, device=device)
train_sum = torch.zeros((), device=device)
for j in range(250):
ids = permutation[j*4:(j+1)*4]
optimizer.zero_grad(set_to_none=True)
output = model(data['coordinates'], data['train_x'][ids])
output = output * stats['y_std'] + stats['y_mean']
loss = relative_l2(output, data['train_y'][ids]).mean()
if not torch.isfinite(loss):
raise FloatingPointError(f'Nonfinite loss at epoch={epoch+1}, step={j+1}')
loss.backward()
norm = torch.nn.utils.clip_grad_norm_(model.parameters(), t['grad_clip'], error_if_nonfinite=True)
optimizer.step()
scheduler.step()
global_step += 1
train_sum += loss.detach() * 4
if j == 0 or (j+1) % t['print_every_steps'] == 0:
print(f'TRAIN epoch={epoch+1}/500 batch={j+1}/250 updates={global_step} '
f'loss_raw_relative_l2={loss.item():.9f} grad_norm={norm.item():.6g} '
f'lr={scheduler.get_last_lr()[0]:.10g}', flush=True)
errors, _ = evaluate(model, data, stats)
row = dict(epoch=epoch+1, updates=global_step,
train_relative_l2=train_sum.item()/1000,
test_relative_l2=errors.mean().item(),
learning_rate=scheduler.get_last_lr()[0],
epoch_seconds=time.monotonic()-epoch_start,
elapsed_seconds=elapsed_before+time.monotonic()-start)
history.append(row)
print('EPOCH', row, flush=True)
state = dict(model=model.state_dict(), optimizer=optimizer.state_dict(),
scheduler=scheduler.state_dict(), epoch=epoch+1, global_step=global_step,
history=history, normalization=stats_cpu, config=cfg,
code_hashes=hashes, environment=runtime,
elapsed_seconds=row['elapsed_seconds'], rng_torch=torch.get_rng_state(),
rng_cuda=torch.cuda.get_rng_state_all(), rng_numpy=np.random.get_state(),
rng_python=random.getstate(), selection=t['selection'])
atomic_save(checkpoint, state)
with open(results/'history.csv', 'a', newline='') as f:
csv.DictWriter(f, fieldnames=columns).writerow(row)
atomic_json(results/'training_status.json', dict(
status='completed' if epoch+1 == 500 else 'running',
completed_epochs=epoch+1, formal_epochs=500, global_step=global_step,
formal_updates=125000, checkpoint=str(checkpoint), last_epoch=row,
slurm_job_id=os.environ.get('SLURM_JOB_ID')))
assert global_step == 125000 and len(history) == 500
final = torch.load(checkpoint, map_location='cpu', weights_only=False)
assert final['epoch'] == 500
atomic_save(path(t['checkpoint']), final)
print(f'COMPLETE 500 epochs; 125000 updates; final epoch saved to {path(t["checkpoint"])}', flush=True)
if __name__ == '__main__':
main()