code-HFD / src /psi /trainers /trainer_performance_test.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
1.08 kB
from __future__ import annotations
from types import SimpleNamespace
import pytest
import torch
from psi.trainers.trainer import Trainer
def test_foreach_grad_norm_matches_legacy_without_changing_gradients(monkeypatch):
first = torch.nn.Parameter(torch.zeros(7, 5, dtype=torch.bfloat16))
second = torch.nn.Parameter(torch.zeros(3, dtype=torch.float32))
first.grad = torch.arange(35, dtype=torch.bfloat16).reshape(7, 5)
second.grad = torch.tensor([1.5, -2.0, 0.25], dtype=torch.float32)
parameters = [first, second]
original_grads = [parameter.grad.clone() for parameter in parameters]
holder = SimpleNamespace(model=torch.nn.Module())
monkeypatch.setenv("PSI_FOREACH_GRAD_NORM", "0")
legacy = Trainer.get_total_grad_norm(holder, parameters)
monkeypatch.setenv("PSI_FOREACH_GRAD_NORM", "1")
foreach = Trainer.get_total_grad_norm(holder, parameters)
assert foreach == pytest.approx(legacy, rel=1e-6, abs=1e-6)
for parameter, original in zip(parameters, original_grads):
assert torch.equal(parameter.grad, original)