File size: 1,975 Bytes
aa02c28
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
import importlib.util
import math
from pathlib import Path
import pytest

spec=importlib.util.spec_from_file_location('continuation',Path(__file__).parents[1]/'scripts/prepare_hfd_continuation.py')
module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module)


def test_lr_continuity_floor_and_clock():
    from transformers.optimization import get_scheduler
    import torch
    base=[8e-5,2e-6,8e-5,5e-5];step=1005983;old_total=1250000;warmup=12500
    f=.05+.95*.5*(1+math.cos(math.pi*(step-warmup)/(old_total-warmup)))
    saved=dict(base_lrs=base,_last_lr=[b*f for b in base],last_epoch=step*8,_step_count=step*8+1,
               lr_lambdas=[{} for _ in base])
    new,rate=module.extended_schedule(saved,step=step,old_world=8,new_world=4,
                                     warmup=warmup,total=1800000,floor=.05)
    assert saved['last_epoch']==step*8
    assert new['last_epoch']==step*4
    params=[torch.nn.Parameter(torch.ones(1)) for _ in base]
    opt=torch.optim.AdamW([dict(params=[p],lr=b) for p,b in zip(params,new['base_lrs'])])
    scheduler=get_scheduler('cosine_with_min_lr',opt,num_warmup_steps=warmup*4,
        num_training_steps=1800000*4,scheduler_specific_kwargs={'min_lr_rate':rate})
    scheduler.load_state_dict(new)
    for i,b in enumerate(new['base_lrs']):
        fn=scheduler.lr_lambdas[i]
        assert b*fn(step*4)==pytest.approx(saved['_last_lr'][i],rel=1e-12)
        assert b*fn(1800000*4)==pytest.approx(base[i]*.05,rel=1e-12)
        assert b*fn(step*4+4)<=saved['_last_lr'][i]


def test_reject_wrong_clock():
    with pytest.raises(ValueError,match='clock'):
        module.extended_schedule({'last_epoch':10},step=100,old_world=8,new_world=4,
                                 warmup=0,total=1000,floor=.05)


def test_reject_unsupported_world():
    with pytest.raises(ValueError,match='non-increasing-world'):
        module.extended_schedule({},step=100,old_world=4,new_world=8,warmup=0,total=1000,floor=.05)