Download src/psi/trainers/trainer_performance_test.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 1.08 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/trainers/trainer_performance_test.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/src/psi/trainers/trainer_performance_test.py
-
curl -L -o trainer_performance_test.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/trainers/trainer_performance_test.py
1.08 kB
| from __future__ import annotations | |
| from types import SimpleNamespace | |
| import pytest | |
| import torch | |
| from psi.trainers.trainer import Trainer | |
| def test_foreach_grad_norm_matches_legacy_without_changing_gradients(monkeypatch): | |
| first = torch.nn.Parameter(torch.zeros(7, 5, dtype=torch.bfloat16)) | |
| second = torch.nn.Parameter(torch.zeros(3, dtype=torch.float32)) | |
| first.grad = torch.arange(35, dtype=torch.bfloat16).reshape(7, 5) | |
| second.grad = torch.tensor([1.5, -2.0, 0.25], dtype=torch.float32) | |
| parameters = [first, second] | |
| original_grads = [parameter.grad.clone() for parameter in parameters] | |
| holder = SimpleNamespace(model=torch.nn.Module()) | |
| monkeypatch.setenv("PSI_FOREACH_GRAD_NORM", "0") | |
| legacy = Trainer.get_total_grad_norm(holder, parameters) | |
| monkeypatch.setenv("PSI_FOREACH_GRAD_NORM", "1") | |
| foreach = Trainer.get_total_grad_norm(holder, parameters) | |
| assert foreach == pytest.approx(legacy, rel=1e-6, abs=1e-6) | |
| for parameter, original in zip(parameters, original_grads): | |
| assert torch.equal(parameter.grad, original) | |