File size: 1,769 Bytes
e8b22d2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
"""Finite LoRA optimizer steps, including legitimate finite zero gradients."""

from __future__ import annotations

from typing import Any


class InvalidLoRAGradients(RuntimeError):
    def __init__(self, names: list[str]):
        self.names = names
        super().__init__("Missing/nonfinite gradients; optimizer step refused")


def checked_lora_step(
    parameters: dict[str, Any], optimizer: Any, scaler: Any
) -> dict[str, float | bool]:
    """Keep ordinary AdamW momentum semantics; zero signal is reported separately.

    The caller validates the LoRA parameter set and requires changed weights over
    the run. A zero gradient is distinct from a missing or nonfinite gradient.
    """
    import torch

    scaler.unscale_(optimizer)
    bad = [
        name
        for name, parameter in parameters.items()
        if parameter.grad is None or not torch.isfinite(parameter.grad).all().item()
    ]
    if bad:
        raise InvalidLoRAGradients(bad)
    norm = torch.nn.utils.clip_grad_norm_(
        parameters.values(), 1.0, error_if_nonfinite=True
    )
    # A float32 norm can underflow before its individual entries do.
    nonzero = norm.item() > 0 or any(p.grad.any().item() for p in parameters.values())
    # Even a zero gradient can produce a momentum update. Do not silently change
    # optimizer semantics or count this as a step with new gradient signal.
    scaler.step(optimizer)
    scaler.update()
    if any(not torch.isfinite(p).all().item() for p in parameters.values()):
        raise RuntimeError("Nonfinite adapter weights after update")
    return {
        "unclipped_gradient_norm": norm.item(),
        "nonzero_gradient": nonzero,
        "zero_gradient_norm": norm.item() == 0,
        "scale": scaler.get_scale(),
    }