File size: 7,699 Bytes
c8ba9c6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
"""CPU fixtures reject misleading comparisons and preserve per-rank evidence."""

import copy
import hashlib
import json

import pytest

from ops.compare_communication_benchmarks import compare_summaries, load_run


def fixture():
    identity = {"path": "/workspace/WarpState/fixture", "size": 100, "inode": 1, "mtime_ns": 2}
    config = {"checkpoint": identity, "checkpoint_step": 40, "checkpoint_tokens": 1280,
              "data": {**identity, "path": "/workspace/WarpState/data"},
              "world_size": 2, "grad_accum": 2, "parameters": 32,
              "parameter_dtype": "float32", "autocast_dtype": "bfloat16", "optimizer": "AdamW(fused=True)",
              "torch": "fixture", "cuda": "fixture", "gpu": "fixture",
              "update_timing_includes_gradient_zeroing": True, "checkpoint_writes": False,
              "source_sha256": {"kernel.py": "old"}, "allow_bf16_reduced_precision_reduction": False,
              "args": {"global_tokens_per_update": 32, "seq_len": 8, "micro_batch": 1,
                       "warmup_updates": 1, "measured_updates": 3, "seed": 1337,
                       "optimizer_state": "resume", "bf16_residuals": True, "profile": False}}
    ranks = []
    for rank in range(2):
        ranks.append({"configuration": {**copy.deepcopy(config), "rank": rank},
                      "checkpoint_path_identity_after": copy.deepcopy(identity), "updates": [
            {"rank": rank, "update": index + 1, "phase": "warmup" if index == 0 else "measured",
             "profiled": False, "global_tokens": 32, "seconds": 2.0 + rank,
             "tokens_per_second": 32 / (2.0 + rank), "loss": 2.5, "grad_norm": 0.25,
             "peak_allocated_gib": 60.0 + rank, "peak_reserved_gib": 70.0 + rank}
            for index in range(4)]})
    return {**copy.deepcopy(ranks[0]), "ranks": ranks}


def alter_config(run, key, value, *, argument=False):
    for summary in [run, *run["ranks"]]:
        config = summary["configuration"]
        (config["args"] if argument else config)[key] = value


def test_comparison_uses_slowest_rank_and_records_intended_source_changes():
    control, candidate = fixture(), fixture()
    alter_config(candidate, "source_sha256", {"kernel.py": "new", "scheduler.py": "added"})
    for summary in [candidate, *candidate["ranks"]]:
        for update in summary["updates"]:
            update["seconds"] /= 2
            update["tokens_per_second"] *= 2
            update["loss"] += 0.125
            update["grad_norm"] += 0.025
    result = compare_summaries(control, candidate)
    assert result["throughput_ratio"] == 2
    assert result["control_timing"]["mean_seconds"] == 3
    assert result["maximum_absolute_loss_difference"] == 0.125
    assert result["maximum_relative_grad_norm_difference"] == pytest.approx(0.1)
    assert result["rank_peak_vram"][1]["candidate"]["measured_updates"]["peak_reserved_gib"] == 71
    assert result["paired_updates"][0]["token_cursor"] == 1312
    assert result["paired_updates"][0]["ranks"][1]["microbatch_cursors"] == [1320, 1336]
    assert result["source_differences"]["kernel.py"]["candidate"] == "new"
    assert result["candidate_timing"]["drift_windows_overlap"]


@pytest.mark.parametrize(("key", "value", "argument"), [
    ("checkpoint_tokens", 1281, False), ("checkpoint_step", 41, False),
    ("parameter_dtype", "bfloat16", False), ("autocast_dtype", "float16", False),
    ("allow_bf16_reduced_precision_reduction", True, False),
    ("optimizer", "SGD", False), ("optimizer_state", "fresh", True),
    ("seed", 1338, True), ("bf16_residuals", False, True),
    ("warmup_updates", 0, True), ("micro_batch", 2, True),
])
def test_incompatible_inputs_and_numerical_controls_fail(key, value, argument):
    control, candidate = fixture(), fixture()
    alter_config(candidate, key, value, argument=argument)
    with pytest.raises(ValueError):
        compare_summaries(control, candidate)


@pytest.mark.parametrize("kind", ["profile", "missing_rank", "missing_update", "cursor", "phase", "identity", "rank_identity"])
def test_incomplete_or_inconsistent_recordings_fail(kind):
    candidate = fixture()
    if kind == "profile":
        candidate["ranks"][1]["updates"][2]["profiled"] = True
    elif kind == "missing_rank":
        candidate["ranks"].pop()
    elif kind == "missing_update":
        candidate["ranks"][1]["updates"].pop()
    elif kind == "cursor":
        candidate["ranks"][1]["updates"][2]["token_cursor"] = 0
    elif kind == "phase":
        candidate["ranks"][1]["updates"][2]["phase"] = "warmup"
    elif kind == "identity":
        candidate["ranks"][1]["checkpoint_path_identity_after"]["inode"] += 1
    elif kind == "rank_identity":
        candidate["ranks"][1]["configuration"]["rank"] = 0
    with pytest.raises(ValueError):
        compare_summaries(fixture(), candidate)


def test_missing_precision_controls_are_explicit_and_partial_recording_fails():
    control, candidate = fixture(), fixture()
    result = compare_summaries(control, candidate)
    assert "allow_tf32" in result["unrecorded_precision_optimizer_controls"]
    alter_config(candidate, "allow_tf32", True)
    with pytest.raises(ValueError, match="incompletely recorded"):
        compare_summaries(control, candidate)


def test_precision_mismatch_on_only_the_second_rank_is_rejected():
    candidate = fixture()
    candidate["ranks"][1]["configuration"]["allow_bf16_reduced_precision_reduction"] = True
    with pytest.raises(ValueError, match="Rank 1 precision"):
        compare_summaries(fixture(), candidate)


def test_data_identity_difference_is_rejected_even_with_equal_shapes():
    candidate = fixture()
    changed = {**candidate["configuration"]["data"], "mtime_ns": 3}
    alter_config(candidate, "data", changed)
    with pytest.raises(ValueError, match="configuration: data"):
        compare_summaries(fixture(), candidate)


@pytest.mark.parametrize(("argument", "value"), [
    ("overlap_swap_collectives", True), ("gradient_mode", "eager"), ("compile_mode", "default"),
])
def test_execution_argument_changes_on_only_one_rank_are_rejected(argument, value):
    candidate = fixture()
    candidate["ranks"][1]["configuration"]["args"][argument] = value
    with pytest.raises(ValueError, match="Rank 1 execution argument mismatch"):
        compare_summaries(fixture(), candidate)


def test_jsonl_fallback_hashes_every_consumed_input(tmp_path):
    run = fixture()
    for rank_summary in run["ranks"]:
        rank_summary["configuration"]["event"] = "configuration"
        for update in rank_summary["updates"]:
            update["event"] = "update"
    summary = copy.deepcopy(run["ranks"][0])
    (tmp_path / "summary.json").write_text(json.dumps(summary))
    for rank, rank_summary in enumerate(run["ranks"]):
        records = [rank_summary["configuration"], *rank_summary["updates"]]
        (tmp_path / f"rank-{rank}.jsonl").write_text("\n".join(json.dumps(row) for row in records) + "\n")
    loaded = load_run(tmp_path)
    paths = [tmp_path / name for name in ("summary.json", "rank-0.jsonl", "rank-1.jsonl")]
    assert loaded["input_files"] == [{"path": str(path), "sha256": hashlib.sha256(path.read_bytes()).hexdigest()}
                                     for path in paths]
    assert len(loaded["summary"]["ranks"]) == 2
    before = loaded["input_files"][2]["sha256"]
    # Semantically irrelevant trailing whitespace changes the exact consumed
    # input hash, while preserving successful reconstruction and validation.
    paths[2].write_text(paths[2].read_text().rstrip() + " \n")
    after = load_run(tmp_path)
    assert after["summary"] == loaded["summary"]
    assert after["input_files"][2]["sha256"] != before