Download src/psi/trainers/swa_optimizer.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 1.17 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/trainers/swa_optimizer.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/src/psi/trainers/swa_optimizer.py
-
curl -L -o swa_optimizer.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/trainers/swa_optimizer.py
1.17 kB
| """QA has a separate update clock; silent windows must not move its Adam state.""" | |
| import torch | |
| def prepare_qa_update(optimizer, *, has_targets): | |
| """After gradient averaging, before optimizer.step; state survives resume. | |
| Clearing absent gradients also disables Adam momentum and weight decay on | |
| the 19 action-only windows between semantic updates. | |
| """ | |
| for group in optimizer.param_groups: | |
| if group.get("group_name") != "swa_qa": | |
| continue | |
| if not has_targets: | |
| for param in group["params"]: | |
| param.grad = None | |
| continue | |
| group["qa_updates"] += 1 | |
| group["lr"] = group["qa_base_lr"] * min( | |
| 1., group["qa_updates"] / max(1, group["qa_warmup_updates"])) | |
| def split_gradient_parameters(optimizer): | |
| body, qa = [], [] | |
| for group in optimizer.param_groups: | |
| (qa if group.get("group_name") == "swa_qa" else body).extend(group["params"]) | |
| return body, qa | |
| def clip_qa_gradients(params): | |
| # A hard question must not scale down an otherwise valid action gradient. | |
| return torch.nn.utils.clip_grad_norm_(params, 1., error_if_nonfinite=True) | |