import importlib.util import math from pathlib import Path import pytest spec=importlib.util.spec_from_file_location('continuation',Path(__file__).parents[1]/'scripts/prepare_hfd_continuation.py') module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module) def test_lr_continuity_floor_and_clock(): from transformers.optimization import get_scheduler import torch base=[8e-5,2e-6,8e-5,5e-5];step=1005983;old_total=1250000;warmup=12500 f=.05+.95*.5*(1+math.cos(math.pi*(step-warmup)/(old_total-warmup))) saved=dict(base_lrs=base,_last_lr=[b*f for b in base],last_epoch=step*8,_step_count=step*8+1, lr_lambdas=[{} for _ in base]) new,rate=module.extended_schedule(saved,step=step,old_world=8,new_world=4, warmup=warmup,total=1800000,floor=.05) assert saved['last_epoch']==step*8 assert new['last_epoch']==step*4 params=[torch.nn.Parameter(torch.ones(1)) for _ in base] opt=torch.optim.AdamW([dict(params=[p],lr=b) for p,b in zip(params,new['base_lrs'])]) scheduler=get_scheduler('cosine_with_min_lr',opt,num_warmup_steps=warmup*4, num_training_steps=1800000*4,scheduler_specific_kwargs={'min_lr_rate':rate}) scheduler.load_state_dict(new) for i,b in enumerate(new['base_lrs']): fn=scheduler.lr_lambdas[i] assert b*fn(step*4)==pytest.approx(saved['_last_lr'][i],rel=1e-12) assert b*fn(1800000*4)==pytest.approx(base[i]*.05,rel=1e-12) assert b*fn(step*4+4)<=saved['_last_lr'][i] def test_reject_wrong_clock(): with pytest.raises(ValueError,match='clock'): module.extended_schedule({'last_epoch':10},step=100,old_world=8,new_world=4, warmup=0,total=1000,floor=.05) def test_reject_unsupported_world(): with pytest.raises(ValueError,match='non-increasing-world'): module.extended_schedule({},step=100,old_world=4,new_world=8,warmup=0,total=1000,floor=.05)