"""Explicit OpenTau comparison optimizer and stateless step-index LR schedule.""" import math from .config import ADAMW_SETTINGS OPENTAU_ADAMW={**ADAMW_SETTINGS,'lr':1e-4,'betas':[.9,.95],'weight_decay':1e-10} def settings(profile): if profile=='legacy':return dict(ADAMW_SETTINGS),1. if profile=='opentau_3000':return dict(OPENTAU_ADAMW),10. raise ValueError('Unknown optimizer protocol') def learning_rate(step,profile): if profile=='legacy':return ADAMW_SETTINGS['lr'] if profile!='opentau_3000' or type(step) is not int or step<0:raise ValueError('Invalid schedule profile/step') if step<100: multiplier=1/101 if step==0 else (1/101-1)*(1-step/100)+1 else:multiplier=.1+.9*.5*(1+math.cos(math.pi*min(step,3000)/3000)) return 1e-4*multiplier