File size: 7,065 Bytes
db32cff
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
"""Full 500-epoch Darcy85 training, with loss/metrics printed as training runs."""
import csv
import os
import random
import time

import numpy as np
import torch

from common import (ROOT, MoNoLight, atomic_json, atomic_save, code_hashes,
                    compute_stats, config_args, environment, evaluate, load_data,
                    normalized, path, read_config, relative_l2,
                    require_compute_node, seed_all)


def main():
    parser = config_args(__doc__)
    parser.add_argument('--resume', action='store_true', help='Resume full optimizer/scheduler/RNG state')
    args = parser.parse_args()
    print('TRAIN_START config loading', flush=True)
    cfg = read_config(args.config)
    print('TRAIN_START compute node and accelerator initialization', flush=True)
    device = require_compute_node()
    seed_all(cfg['seed'])
    print('TRAIN_START data loading', flush=True)
    raw = load_data(cfg)
    stats_cpu = compute_stats(raw)
    data, stats = normalized(raw, stats_cpu, device)
    print('TRAIN_START data normalization complete', flush=True)
    t = cfg['training']
    results = path(cfg['results'])
    results.mkdir(parents=True, exist_ok=True)
    model = MoNoLight(**cfg['model']).to(device)
    optimizer = torch.optim.AdamW(model.parameters(), lr=t['learning_rate'],
                                 betas=tuple(t['betas']), weight_decay=t['weight_decay'])
    scheduler = torch.optim.lr_scheduler.OneCycleLR(
        optimizer, max_lr=t['learning_rate'], epochs=t['epochs'], steps_per_epoch=250,
        pct_start=t['warmup_epochs']/t['epochs'], div_factor=t['div_factor'],
        final_div_factor=t['final_div_factor'], anneal_strategy='cos')
    first_epoch, global_step, history = 0, 0, []
    elapsed_before = 0.0
    checkpoint = path(t['resume_checkpoint'])
    hashes = code_hashes()
    if args.resume and checkpoint.exists():
        state = torch.load(checkpoint, map_location=device, weights_only=False)
        assert state['config'] == cfg, 'Cannot change formal configuration when resuming'
        assert state['code_hashes'] == hashes, 'Training implementation changed; audit before resuming'
        model.load_state_dict(state['model'])
        optimizer.load_state_dict(state['optimizer'])
        scheduler.load_state_dict(state['scheduler'])
        first_epoch, global_step, history = state['epoch'], state['global_step'], state['history']
        elapsed_before = state['elapsed_seconds']
        # RNG tensors must be copied back to CPU even when optimizer maps to the accelerator.
        torch.set_rng_state(state['rng_torch'].cpu())
        torch.cuda.set_rng_state_all([s.cpu() for s in state['rng_cuda']])
        np.random.set_state(state['rng_numpy'])
        random.setstate(state['rng_python'])
        print(f'RESUME epoch={first_epoch} updates={global_step}; optimizer/scheduler/RNG restored', flush=True)
    elif checkpoint.exists() or path(t['checkpoint']).exists():
        raise RuntimeError('Existing training weights found: use --resume; refusing overwrite/retrain')
    runtime = environment()
    runtime.update(trainable_parameters=sum(p.numel() for p in model.parameters() if p.requires_grad),
                   all_parameters=sum(p.numel() for p in model.parameters()),
                   precision='float32', code_hashes=hashes)
    atomic_json(results / 'training_environment.json', runtime)
    print('ENVIRONMENT', runtime, flush=True)
    start = time.monotonic()
    columns = ['epoch','updates','train_relative_l2','test_relative_l2','learning_rate','epoch_seconds','elapsed_seconds']
    # The checkpoint is authoritative: discard any uncommitted trailing history from interruption.
    with open(results/'history.csv', 'w', newline='') as f:
        writer = csv.DictWriter(f, fieldnames=columns)
        writer.writeheader()
        writer.writerows(history)
    for epoch in range(first_epoch, t['epochs']):
        epoch_start = time.monotonic()
        model.train()
        permutation = torch.randperm(1000, device=device)
        train_sum = torch.zeros((), device=device)
        for j in range(250):
            ids = permutation[j*4:(j+1)*4]
            optimizer.zero_grad(set_to_none=True)
            output = model(data['coordinates'], data['train_x'][ids])
            output = output * stats['y_std'] + stats['y_mean']
            loss = relative_l2(output, data['train_y'][ids]).mean()
            if not torch.isfinite(loss):
                raise FloatingPointError(f'Nonfinite loss at epoch={epoch+1}, step={j+1}')
            loss.backward()
            norm = torch.nn.utils.clip_grad_norm_(model.parameters(), t['grad_clip'], error_if_nonfinite=True)
            optimizer.step()
            scheduler.step()
            global_step += 1
            train_sum += loss.detach() * 4
            if j == 0 or (j+1) % t['print_every_steps'] == 0:
                print(f'TRAIN epoch={epoch+1}/500 batch={j+1}/250 updates={global_step} '
                      f'loss_raw_relative_l2={loss.item():.9f} grad_norm={norm.item():.6g} '
                      f'lr={scheduler.get_last_lr()[0]:.10g}', flush=True)
        errors, _ = evaluate(model, data, stats)
        row = dict(epoch=epoch+1, updates=global_step,
                   train_relative_l2=train_sum.item()/1000,
                   test_relative_l2=errors.mean().item(),
                   learning_rate=scheduler.get_last_lr()[0],
                   epoch_seconds=time.monotonic()-epoch_start,
                   elapsed_seconds=elapsed_before+time.monotonic()-start)
        history.append(row)
        print('EPOCH', row, flush=True)
        state = dict(model=model.state_dict(), optimizer=optimizer.state_dict(),
                     scheduler=scheduler.state_dict(), epoch=epoch+1, global_step=global_step,
                     history=history, normalization=stats_cpu, config=cfg,
                     code_hashes=hashes, environment=runtime,
                     elapsed_seconds=row['elapsed_seconds'], rng_torch=torch.get_rng_state(),
                     rng_cuda=torch.cuda.get_rng_state_all(), rng_numpy=np.random.get_state(),
                     rng_python=random.getstate(), selection=t['selection'])
        atomic_save(checkpoint, state)
        with open(results/'history.csv', 'a', newline='') as f:
            csv.DictWriter(f, fieldnames=columns).writerow(row)
        atomic_json(results/'training_status.json', dict(
            status='completed' if epoch+1 == 500 else 'running',
            completed_epochs=epoch+1, formal_epochs=500, global_step=global_step,
            formal_updates=125000, checkpoint=str(checkpoint), last_epoch=row,
            slurm_job_id=os.environ.get('SLURM_JOB_ID')))
    assert global_step == 125000 and len(history) == 500
    final = torch.load(checkpoint, map_location='cpu', weights_only=False)
    assert final['epoch'] == 500
    atomic_save(path(t['checkpoint']), final)
    print(f'COMPLETE 500 epochs; 125000 updates; final epoch saved to {path(t["checkpoint"])}', flush=True)


if __name__ == '__main__':
    main()