Download tests/test_hfd_continuation.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 1.98 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/tests/test_hfd_continuation.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/tests/test_hfd_continuation.py
-
curl -L -o test_hfd_continuation.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/tests/test_hfd_continuation.py
1.98 kB
| import importlib.util | |
| import math | |
| from pathlib import Path | |
| import pytest | |
| spec=importlib.util.spec_from_file_location('continuation',Path(__file__).parents[1]/'scripts/prepare_hfd_continuation.py') | |
| module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module) | |
| def test_lr_continuity_floor_and_clock(): | |
| from transformers.optimization import get_scheduler | |
| import torch | |
| base=[8e-5,2e-6,8e-5,5e-5];step=1005983;old_total=1250000;warmup=12500 | |
| f=.05+.95*.5*(1+math.cos(math.pi*(step-warmup)/(old_total-warmup))) | |
| saved=dict(base_lrs=base,_last_lr=[b*f for b in base],last_epoch=step*8,_step_count=step*8+1, | |
| lr_lambdas=[{} for _ in base]) | |
| new,rate=module.extended_schedule(saved,step=step,old_world=8,new_world=4, | |
| warmup=warmup,total=1800000,floor=.05) | |
| assert saved['last_epoch']==step*8 | |
| assert new['last_epoch']==step*4 | |
| params=[torch.nn.Parameter(torch.ones(1)) for _ in base] | |
| opt=torch.optim.AdamW([dict(params=[p],lr=b) for p,b in zip(params,new['base_lrs'])]) | |
| scheduler=get_scheduler('cosine_with_min_lr',opt,num_warmup_steps=warmup*4, | |
| num_training_steps=1800000*4,scheduler_specific_kwargs={'min_lr_rate':rate}) | |
| scheduler.load_state_dict(new) | |
| for i,b in enumerate(new['base_lrs']): | |
| fn=scheduler.lr_lambdas[i] | |
| assert b*fn(step*4)==pytest.approx(saved['_last_lr'][i],rel=1e-12) | |
| assert b*fn(1800000*4)==pytest.approx(base[i]*.05,rel=1e-12) | |
| assert b*fn(step*4+4)<=saved['_last_lr'][i] | |
| def test_reject_wrong_clock(): | |
| with pytest.raises(ValueError,match='clock'): | |
| module.extended_schedule({'last_epoch':10},step=100,old_world=8,new_world=4, | |
| warmup=0,total=1000,floor=.05) | |
| def test_reject_unsupported_world(): | |
| with pytest.raises(ValueError,match='non-increasing-world'): | |
| module.extended_schedule({},step=100,old_world=4,new_world=8,warmup=0,total=1000,floor=.05) | |