Download scripts/train.py from OneScience-Group/MoNo-light: direct link, hf CLI and curl.
- Browser
- Download file 7.07 kB
-
https://huggingface.co/OneScience-Group/MoNo-light/resolve/main/scripts/train.py
- Command line
-
hf download hf://OneScience-Group/MoNo-light/scripts/train.py
-
curl -L -o train.py https://huggingface.co/OneScience-Group/MoNo-light/resolve/main/scripts/train.py
7.07 kB
| """Full 500-epoch Darcy85 training, with loss/metrics printed as training runs.""" | |
| import csv | |
| import os | |
| import random | |
| import time | |
| import numpy as np | |
| import torch | |
| from common import (ROOT, MoNoLight, atomic_json, atomic_save, code_hashes, | |
| compute_stats, config_args, environment, evaluate, load_data, | |
| normalized, path, read_config, relative_l2, | |
| require_compute_node, seed_all) | |
| def main(): | |
| parser = config_args(__doc__) | |
| parser.add_argument('--resume', action='store_true', help='Resume full optimizer/scheduler/RNG state') | |
| args = parser.parse_args() | |
| print('TRAIN_START config loading', flush=True) | |
| cfg = read_config(args.config) | |
| print('TRAIN_START compute node and accelerator initialization', flush=True) | |
| device = require_compute_node() | |
| seed_all(cfg['seed']) | |
| print('TRAIN_START data loading', flush=True) | |
| raw = load_data(cfg) | |
| stats_cpu = compute_stats(raw) | |
| data, stats = normalized(raw, stats_cpu, device) | |
| print('TRAIN_START data normalization complete', flush=True) | |
| t = cfg['training'] | |
| results = path(cfg['results']) | |
| results.mkdir(parents=True, exist_ok=True) | |
| model = MoNoLight(**cfg['model']).to(device) | |
| optimizer = torch.optim.AdamW(model.parameters(), lr=t['learning_rate'], | |
| betas=tuple(t['betas']), weight_decay=t['weight_decay']) | |
| scheduler = torch.optim.lr_scheduler.OneCycleLR( | |
| optimizer, max_lr=t['learning_rate'], epochs=t['epochs'], steps_per_epoch=250, | |
| pct_start=t['warmup_epochs']/t['epochs'], div_factor=t['div_factor'], | |
| final_div_factor=t['final_div_factor'], anneal_strategy='cos') | |
| first_epoch, global_step, history = 0, 0, [] | |
| elapsed_before = 0.0 | |
| checkpoint = path(t['resume_checkpoint']) | |
| hashes = code_hashes() | |
| if args.resume and checkpoint.exists(): | |
| state = torch.load(checkpoint, map_location=device, weights_only=False) | |
| assert state['config'] == cfg, 'Cannot change formal configuration when resuming' | |
| assert state['code_hashes'] == hashes, 'Training implementation changed; audit before resuming' | |
| model.load_state_dict(state['model']) | |
| optimizer.load_state_dict(state['optimizer']) | |
| scheduler.load_state_dict(state['scheduler']) | |
| first_epoch, global_step, history = state['epoch'], state['global_step'], state['history'] | |
| elapsed_before = state['elapsed_seconds'] | |
| # RNG tensors must be copied back to CPU even when optimizer maps to the accelerator. | |
| torch.set_rng_state(state['rng_torch'].cpu()) | |
| torch.cuda.set_rng_state_all([s.cpu() for s in state['rng_cuda']]) | |
| np.random.set_state(state['rng_numpy']) | |
| random.setstate(state['rng_python']) | |
| print(f'RESUME epoch={first_epoch} updates={global_step}; optimizer/scheduler/RNG restored', flush=True) | |
| elif checkpoint.exists() or path(t['checkpoint']).exists(): | |
| raise RuntimeError('Existing training weights found: use --resume; refusing overwrite/retrain') | |
| runtime = environment() | |
| runtime.update(trainable_parameters=sum(p.numel() for p in model.parameters() if p.requires_grad), | |
| all_parameters=sum(p.numel() for p in model.parameters()), | |
| precision='float32', code_hashes=hashes) | |
| atomic_json(results / 'training_environment.json', runtime) | |
| print('ENVIRONMENT', runtime, flush=True) | |
| start = time.monotonic() | |
| columns = ['epoch','updates','train_relative_l2','test_relative_l2','learning_rate','epoch_seconds','elapsed_seconds'] | |
| # The checkpoint is authoritative: discard any uncommitted trailing history from interruption. | |
| with open(results/'history.csv', 'w', newline='') as f: | |
| writer = csv.DictWriter(f, fieldnames=columns) | |
| writer.writeheader() | |
| writer.writerows(history) | |
| for epoch in range(first_epoch, t['epochs']): | |
| epoch_start = time.monotonic() | |
| model.train() | |
| permutation = torch.randperm(1000, device=device) | |
| train_sum = torch.zeros((), device=device) | |
| for j in range(250): | |
| ids = permutation[j*4:(j+1)*4] | |
| optimizer.zero_grad(set_to_none=True) | |
| output = model(data['coordinates'], data['train_x'][ids]) | |
| output = output * stats['y_std'] + stats['y_mean'] | |
| loss = relative_l2(output, data['train_y'][ids]).mean() | |
| if not torch.isfinite(loss): | |
| raise FloatingPointError(f'Nonfinite loss at epoch={epoch+1}, step={j+1}') | |
| loss.backward() | |
| norm = torch.nn.utils.clip_grad_norm_(model.parameters(), t['grad_clip'], error_if_nonfinite=True) | |
| optimizer.step() | |
| scheduler.step() | |
| global_step += 1 | |
| train_sum += loss.detach() * 4 | |
| if j == 0 or (j+1) % t['print_every_steps'] == 0: | |
| print(f'TRAIN epoch={epoch+1}/500 batch={j+1}/250 updates={global_step} ' | |
| f'loss_raw_relative_l2={loss.item():.9f} grad_norm={norm.item():.6g} ' | |
| f'lr={scheduler.get_last_lr()[0]:.10g}', flush=True) | |
| errors, _ = evaluate(model, data, stats) | |
| row = dict(epoch=epoch+1, updates=global_step, | |
| train_relative_l2=train_sum.item()/1000, | |
| test_relative_l2=errors.mean().item(), | |
| learning_rate=scheduler.get_last_lr()[0], | |
| epoch_seconds=time.monotonic()-epoch_start, | |
| elapsed_seconds=elapsed_before+time.monotonic()-start) | |
| history.append(row) | |
| print('EPOCH', row, flush=True) | |
| state = dict(model=model.state_dict(), optimizer=optimizer.state_dict(), | |
| scheduler=scheduler.state_dict(), epoch=epoch+1, global_step=global_step, | |
| history=history, normalization=stats_cpu, config=cfg, | |
| code_hashes=hashes, environment=runtime, | |
| elapsed_seconds=row['elapsed_seconds'], rng_torch=torch.get_rng_state(), | |
| rng_cuda=torch.cuda.get_rng_state_all(), rng_numpy=np.random.get_state(), | |
| rng_python=random.getstate(), selection=t['selection']) | |
| atomic_save(checkpoint, state) | |
| with open(results/'history.csv', 'a', newline='') as f: | |
| csv.DictWriter(f, fieldnames=columns).writerow(row) | |
| atomic_json(results/'training_status.json', dict( | |
| status='completed' if epoch+1 == 500 else 'running', | |
| completed_epochs=epoch+1, formal_epochs=500, global_step=global_step, | |
| formal_updates=125000, checkpoint=str(checkpoint), last_epoch=row, | |
| slurm_job_id=os.environ.get('SLURM_JOB_ID'))) | |
| assert global_step == 125000 and len(history) == 500 | |
| final = torch.load(checkpoint, map_location='cpu', weights_only=False) | |
| assert final['epoch'] == 500 | |
| atomic_save(path(t['checkpoint']), final) | |
| print(f'COMPLETE 500 epochs; 125000 updates; final epoch saved to {path(t["checkpoint"])}', flush=True) | |
| if __name__ == '__main__': | |
| main() | |