code-HFD / tests /test_hfd_continuation.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
aa02c28 verified
Raw History Blame Contribute Delete
1.98 kB
import importlib.util
import math
from pathlib import Path
import pytest
spec=importlib.util.spec_from_file_location('continuation',Path(__file__).parents[1]/'scripts/prepare_hfd_continuation.py')
module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module)
def test_lr_continuity_floor_and_clock():
from transformers.optimization import get_scheduler
import torch
base=[8e-5,2e-6,8e-5,5e-5];step=1005983;old_total=1250000;warmup=12500
f=.05+.95*.5*(1+math.cos(math.pi*(step-warmup)/(old_total-warmup)))
saved=dict(base_lrs=base,_last_lr=[b*f for b in base],last_epoch=step*8,_step_count=step*8+1,
lr_lambdas=[{} for _ in base])
new,rate=module.extended_schedule(saved,step=step,old_world=8,new_world=4,
warmup=warmup,total=1800000,floor=.05)
assert saved['last_epoch']==step*8
assert new['last_epoch']==step*4
params=[torch.nn.Parameter(torch.ones(1)) for _ in base]
opt=torch.optim.AdamW([dict(params=[p],lr=b) for p,b in zip(params,new['base_lrs'])])
scheduler=get_scheduler('cosine_with_min_lr',opt,num_warmup_steps=warmup*4,
num_training_steps=1800000*4,scheduler_specific_kwargs={'min_lr_rate':rate})
scheduler.load_state_dict(new)
for i,b in enumerate(new['base_lrs']):
fn=scheduler.lr_lambdas[i]
assert b*fn(step*4)==pytest.approx(saved['_last_lr'][i],rel=1e-12)
assert b*fn(1800000*4)==pytest.approx(base[i]*.05,rel=1e-12)
assert b*fn(step*4+4)<=saved['_last_lr'][i]
def test_reject_wrong_clock():
with pytest.raises(ValueError,match='clock'):
module.extended_schedule({'last_epoch':10},step=100,old_world=8,new_world=4,
warmup=0,total=1000,floor=.05)
def test_reject_unsupported_world():
with pytest.raises(ValueError,match='non-increasing-world'):
module.extended_schedule({},step=100,old_world=4,new_world=8,warmup=0,total=1000,floor=.05)