File size: 788 Bytes
0f5d220
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
"""Explicit OpenTau comparison optimizer and stateless step-index LR schedule."""
import math
from .config import ADAMW_SETTINGS

OPENTAU_ADAMW={**ADAMW_SETTINGS,'lr':1e-4,'betas':[.9,.95],'weight_decay':1e-10}

def settings(profile):
    if profile=='legacy':return dict(ADAMW_SETTINGS),1.
    if profile=='opentau_3000':return dict(OPENTAU_ADAMW),10.
    raise ValueError('Unknown optimizer protocol')

def learning_rate(step,profile):
    if profile=='legacy':return ADAMW_SETTINGS['lr']
    if profile!='opentau_3000' or type(step) is not int or step<0:raise ValueError('Invalid schedule profile/step')
    if step<100:
        multiplier=1/101 if step==0 else (1/101-1)*(1-step/100)+1
    else:multiplier=.1+.9*.5*(1+math.cos(math.pi*min(step,3000)/3000))
    return 1e-4*multiplier