code-HFD / src /psi /trainers /swa_optimizer.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
1.17 kB
"""QA has a separate update clock; silent windows must not move its Adam state."""
import torch
def prepare_qa_update(optimizer, *, has_targets):
"""After gradient averaging, before optimizer.step; state survives resume.
Clearing absent gradients also disables Adam momentum and weight decay on
the 19 action-only windows between semantic updates.
"""
for group in optimizer.param_groups:
if group.get("group_name") != "swa_qa":
continue
if not has_targets:
for param in group["params"]:
param.grad = None
continue
group["qa_updates"] += 1
group["lr"] = group["qa_base_lr"] * min(
1., group["qa_updates"] / max(1, group["qa_warmup_updates"]))
def split_gradient_parameters(optimizer):
body, qa = [], []
for group in optimizer.param_groups:
(qa if group.get("group_name") == "swa_qa" else body).extend(group["params"])
return body, qa
def clip_qa_gradients(params):
# A hard question must not scale down an otherwise valid action gradient.
return torch.nn.utils.clip_grad_norm_(params, 1., error_if_nonfinite=True)