File size: 3,426 Bytes
c8ba9c6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
"""The experimental loss retains the full vocabulary objective and model state."""

import copy

import pytest
import torch

from ops.loss_experiment import apply_loss_execution
from warpstate.config import WarpStateConfig
from warpstate.model import WarpStateLM


@pytest.mark.parametrize("mode", ["full", "compiled-full"])
@pytest.mark.parametrize("mask", ["none", "partial", "all"])
def test_full_loss_preserves_objective_gradients_and_parameters(mode, mask, monkeypatch):
    # Exercise full graph capture on CPU without hardware-specific Inductor code.
    if mode == "compiled-full":
        actual_compile = torch.compile
        monkeypatch.setattr(torch, "compile", lambda fn, **kw: actual_compile(fn, backend="eager", **kw))
    torch.manual_seed(572)
    cfg = WarpStateConfig(vocab_size=32, d_model=16, n_heads=2, n_cores=1,
                          logical_depth=2, ffn_hidden=24, chunk_size=4,
                          gradient_checkpointing=False, parallel_memory=True,
                          loss_chunk_size=3)
    reference = WarpStateLM(cfg).train()
    experiment = copy.deepcopy(reference)
    keys = list(experiment.state_dict())
    parameters = list(experiment.parameters())
    embedding = experiment.embed.weight
    apply_loss_execution(experiment, mode)
    assert list(experiment.state_dict()) == keys
    assert all(a is b for a, b in zip(parameters, experiment.parameters(), strict=True))
    assert experiment.embed.weight is embedding
    inputs = torch.randint(0, cfg.vocab_size, (2, 8))
    labels = torch.randint(0, cfg.vocab_size, (2, 8))
    if mask == "partial":
        labels.flatten()[3:11] = -100
    elif mask == "all":
        labels.fill_(-100)
    ref_logits, ref_loss = reference(inputs, labels, return_logits=False)
    logits, loss = experiment(inputs, labels, return_logits=False)
    assert ref_logits is None and logits is None
    assert torch.isfinite(loss)
    torch.testing.assert_close(loss, ref_loss, rtol=1e-6, atol=1e-6)
    ref_loss.backward()
    loss.backward()
    for (name, expected), (other_name, actual) in zip(reference.named_parameters(), experiment.named_parameters(), strict=True):
        assert name == other_name
        assert actual.grad is not None and torch.isfinite(actual.grad).all()
        torch.testing.assert_close(actual.grad, expected.grad, rtol=1e-6, atol=1e-6)
    if mask == "all":
        assert loss.item() == 0.0
        assert all(torch.count_nonzero(p.grad) == 0 for p in experiment.parameters())
    with torch.no_grad():
        reference.eval()
        experiment.eval()
        torch.testing.assert_close(experiment(inputs), reference(inputs), rtol=0, atol=0)
        torch.testing.assert_close(experiment(inputs, labels, return_logits=False)[1], ref_loss.detach(), rtol=1e-6, atol=1e-6)


def test_loss_mode_validation_and_checkpointed_noop():
    model = WarpStateLM(WarpStateConfig(vocab_size=16, d_model=8, n_heads=1,
                                      n_cores=1, logical_depth=1, ffn_hidden=8,
                                      chunk_size=4))
    forward = model.forward
    apply_loss_execution(model, "checkpointed")
    assert model.forward == forward
    with pytest.raises(ValueError, match="Unknown loss"):
        apply_loss_execution(model, "invalid")
    apply_loss_execution(model, "full")
    with pytest.raises(ValueError, match="already been patched"):
        apply_loss_execution(model, "full")