File size: 3,426 Bytes
c8ba9c6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | """The experimental loss retains the full vocabulary objective and model state."""
import copy
import pytest
import torch
from ops.loss_experiment import apply_loss_execution
from warpstate.config import WarpStateConfig
from warpstate.model import WarpStateLM
@pytest.mark.parametrize("mode", ["full", "compiled-full"])
@pytest.mark.parametrize("mask", ["none", "partial", "all"])
def test_full_loss_preserves_objective_gradients_and_parameters(mode, mask, monkeypatch):
# Exercise full graph capture on CPU without hardware-specific Inductor code.
if mode == "compiled-full":
actual_compile = torch.compile
monkeypatch.setattr(torch, "compile", lambda fn, **kw: actual_compile(fn, backend="eager", **kw))
torch.manual_seed(572)
cfg = WarpStateConfig(vocab_size=32, d_model=16, n_heads=2, n_cores=1,
logical_depth=2, ffn_hidden=24, chunk_size=4,
gradient_checkpointing=False, parallel_memory=True,
loss_chunk_size=3)
reference = WarpStateLM(cfg).train()
experiment = copy.deepcopy(reference)
keys = list(experiment.state_dict())
parameters = list(experiment.parameters())
embedding = experiment.embed.weight
apply_loss_execution(experiment, mode)
assert list(experiment.state_dict()) == keys
assert all(a is b for a, b in zip(parameters, experiment.parameters(), strict=True))
assert experiment.embed.weight is embedding
inputs = torch.randint(0, cfg.vocab_size, (2, 8))
labels = torch.randint(0, cfg.vocab_size, (2, 8))
if mask == "partial":
labels.flatten()[3:11] = -100
elif mask == "all":
labels.fill_(-100)
ref_logits, ref_loss = reference(inputs, labels, return_logits=False)
logits, loss = experiment(inputs, labels, return_logits=False)
assert ref_logits is None and logits is None
assert torch.isfinite(loss)
torch.testing.assert_close(loss, ref_loss, rtol=1e-6, atol=1e-6)
ref_loss.backward()
loss.backward()
for (name, expected), (other_name, actual) in zip(reference.named_parameters(), experiment.named_parameters(), strict=True):
assert name == other_name
assert actual.grad is not None and torch.isfinite(actual.grad).all()
torch.testing.assert_close(actual.grad, expected.grad, rtol=1e-6, atol=1e-6)
if mask == "all":
assert loss.item() == 0.0
assert all(torch.count_nonzero(p.grad) == 0 for p in experiment.parameters())
with torch.no_grad():
reference.eval()
experiment.eval()
torch.testing.assert_close(experiment(inputs), reference(inputs), rtol=0, atol=0)
torch.testing.assert_close(experiment(inputs, labels, return_logits=False)[1], ref_loss.detach(), rtol=1e-6, atol=1e-6)
def test_loss_mode_validation_and_checkpointed_noop():
model = WarpStateLM(WarpStateConfig(vocab_size=16, d_model=8, n_heads=1,
n_cores=1, logical_depth=1, ffn_hidden=8,
chunk_size=4))
forward = model.forward
apply_loss_execution(model, "checkpointed")
assert model.forward == forward
with pytest.raises(ValueError, match="Unknown loss"):
apply_loss_execution(model, "invalid")
apply_loss_execution(model, "full")
with pytest.raises(ValueError, match="already been patched"):
apply_loss_execution(model, "full")
|