from __future__ import annotations from types import SimpleNamespace import pytest import torch from psi.trainers.trainer import Trainer def test_foreach_grad_norm_matches_legacy_without_changing_gradients(monkeypatch): first = torch.nn.Parameter(torch.zeros(7, 5, dtype=torch.bfloat16)) second = torch.nn.Parameter(torch.zeros(3, dtype=torch.float32)) first.grad = torch.arange(35, dtype=torch.bfloat16).reshape(7, 5) second.grad = torch.tensor([1.5, -2.0, 0.25], dtype=torch.float32) parameters = [first, second] original_grads = [parameter.grad.clone() for parameter in parameters] holder = SimpleNamespace(model=torch.nn.Module()) monkeypatch.setenv("PSI_FOREACH_GRAD_NORM", "0") legacy = Trainer.get_total_grad_norm(holder, parameters) monkeypatch.setenv("PSI_FOREACH_GRAD_NORM", "1") foreach = Trainer.get_total_grad_norm(holder, parameters) assert foreach == pytest.approx(legacy, rel=1e-6, abs=1e-6) for parameter, original in zip(parameters, original_grads): assert torch.equal(parameter.grad, original)