jisue's picture
Experiment backup 2026-11-10T01:58:18.622752+00:00
0f5d220 verified
Raw History Blame Contribute Delete
788 Bytes
"""Explicit OpenTau comparison optimizer and stateless step-index LR schedule."""
import math
from .config import ADAMW_SETTINGS
OPENTAU_ADAMW={**ADAMW_SETTINGS,'lr':1e-4,'betas':[.9,.95],'weight_decay':1e-10}
def settings(profile):
if profile=='legacy':return dict(ADAMW_SETTINGS),1.
if profile=='opentau_3000':return dict(OPENTAU_ADAMW),10.
raise ValueError('Unknown optimizer protocol')
def learning_rate(step,profile):
if profile=='legacy':return ADAMW_SETTINGS['lr']
if profile!='opentau_3000' or type(step) is not int or step<0:raise ValueError('Invalid schedule profile/step')
if step<100:
multiplier=1/101 if step==0 else (1/101-1)*(1-step/100)+1
else:multiplier=.1+.9*.5*(1+math.cos(math.pi*min(step,3000)/3000))
return 1e-4*multiplier