Download test_compare_communication_benchmarks.py from zerdovzad/WarpStateAI: direct link, hf CLI and curl.
- Browser
- Download file 7.7 kB
-
https://huggingface.co/zerdovzad/WarpStateAI/resolve/main/test_compare_communication_benchmarks.py
- Command line
-
hf download hf://zerdovzad/WarpStateAI/test_compare_communication_benchmarks.py
-
curl -L -o test_compare_communication_benchmarks.py https://huggingface.co/zerdovzad/WarpStateAI/resolve/main/test_compare_communication_benchmarks.py
7.7 kB
| """CPU fixtures reject misleading comparisons and preserve per-rank evidence.""" | |
| import copy | |
| import hashlib | |
| import json | |
| import pytest | |
| from ops.compare_communication_benchmarks import compare_summaries, load_run | |
| def fixture(): | |
| identity = {"path": "/workspace/WarpState/fixture", "size": 100, "inode": 1, "mtime_ns": 2} | |
| config = {"checkpoint": identity, "checkpoint_step": 40, "checkpoint_tokens": 1280, | |
| "data": {**identity, "path": "/workspace/WarpState/data"}, | |
| "world_size": 2, "grad_accum": 2, "parameters": 32, | |
| "parameter_dtype": "float32", "autocast_dtype": "bfloat16", "optimizer": "AdamW(fused=True)", | |
| "torch": "fixture", "cuda": "fixture", "gpu": "fixture", | |
| "update_timing_includes_gradient_zeroing": True, "checkpoint_writes": False, | |
| "source_sha256": {"kernel.py": "old"}, "allow_bf16_reduced_precision_reduction": False, | |
| "args": {"global_tokens_per_update": 32, "seq_len": 8, "micro_batch": 1, | |
| "warmup_updates": 1, "measured_updates": 3, "seed": 1337, | |
| "optimizer_state": "resume", "bf16_residuals": True, "profile": False}} | |
| ranks = [] | |
| for rank in range(2): | |
| ranks.append({"configuration": {**copy.deepcopy(config), "rank": rank}, | |
| "checkpoint_path_identity_after": copy.deepcopy(identity), "updates": [ | |
| {"rank": rank, "update": index + 1, "phase": "warmup" if index == 0 else "measured", | |
| "profiled": False, "global_tokens": 32, "seconds": 2.0 + rank, | |
| "tokens_per_second": 32 / (2.0 + rank), "loss": 2.5, "grad_norm": 0.25, | |
| "peak_allocated_gib": 60.0 + rank, "peak_reserved_gib": 70.0 + rank} | |
| for index in range(4)]}) | |
| return {**copy.deepcopy(ranks[0]), "ranks": ranks} | |
| def alter_config(run, key, value, *, argument=False): | |
| for summary in [run, *run["ranks"]]: | |
| config = summary["configuration"] | |
| (config["args"] if argument else config)[key] = value | |
| def test_comparison_uses_slowest_rank_and_records_intended_source_changes(): | |
| control, candidate = fixture(), fixture() | |
| alter_config(candidate, "source_sha256", {"kernel.py": "new", "scheduler.py": "added"}) | |
| for summary in [candidate, *candidate["ranks"]]: | |
| for update in summary["updates"]: | |
| update["seconds"] /= 2 | |
| update["tokens_per_second"] *= 2 | |
| update["loss"] += 0.125 | |
| update["grad_norm"] += 0.025 | |
| result = compare_summaries(control, candidate) | |
| assert result["throughput_ratio"] == 2 | |
| assert result["control_timing"]["mean_seconds"] == 3 | |
| assert result["maximum_absolute_loss_difference"] == 0.125 | |
| assert result["maximum_relative_grad_norm_difference"] == pytest.approx(0.1) | |
| assert result["rank_peak_vram"][1]["candidate"]["measured_updates"]["peak_reserved_gib"] == 71 | |
| assert result["paired_updates"][0]["token_cursor"] == 1312 | |
| assert result["paired_updates"][0]["ranks"][1]["microbatch_cursors"] == [1320, 1336] | |
| assert result["source_differences"]["kernel.py"]["candidate"] == "new" | |
| assert result["candidate_timing"]["drift_windows_overlap"] | |
| def test_incompatible_inputs_and_numerical_controls_fail(key, value, argument): | |
| control, candidate = fixture(), fixture() | |
| alter_config(candidate, key, value, argument=argument) | |
| with pytest.raises(ValueError): | |
| compare_summaries(control, candidate) | |
| def test_incomplete_or_inconsistent_recordings_fail(kind): | |
| candidate = fixture() | |
| if kind == "profile": | |
| candidate["ranks"][1]["updates"][2]["profiled"] = True | |
| elif kind == "missing_rank": | |
| candidate["ranks"].pop() | |
| elif kind == "missing_update": | |
| candidate["ranks"][1]["updates"].pop() | |
| elif kind == "cursor": | |
| candidate["ranks"][1]["updates"][2]["token_cursor"] = 0 | |
| elif kind == "phase": | |
| candidate["ranks"][1]["updates"][2]["phase"] = "warmup" | |
| elif kind == "identity": | |
| candidate["ranks"][1]["checkpoint_path_identity_after"]["inode"] += 1 | |
| elif kind == "rank_identity": | |
| candidate["ranks"][1]["configuration"]["rank"] = 0 | |
| with pytest.raises(ValueError): | |
| compare_summaries(fixture(), candidate) | |
| def test_missing_precision_controls_are_explicit_and_partial_recording_fails(): | |
| control, candidate = fixture(), fixture() | |
| result = compare_summaries(control, candidate) | |
| assert "allow_tf32" in result["unrecorded_precision_optimizer_controls"] | |
| alter_config(candidate, "allow_tf32", True) | |
| with pytest.raises(ValueError, match="incompletely recorded"): | |
| compare_summaries(control, candidate) | |
| def test_precision_mismatch_on_only_the_second_rank_is_rejected(): | |
| candidate = fixture() | |
| candidate["ranks"][1]["configuration"]["allow_bf16_reduced_precision_reduction"] = True | |
| with pytest.raises(ValueError, match="Rank 1 precision"): | |
| compare_summaries(fixture(), candidate) | |
| def test_data_identity_difference_is_rejected_even_with_equal_shapes(): | |
| candidate = fixture() | |
| changed = {**candidate["configuration"]["data"], "mtime_ns": 3} | |
| alter_config(candidate, "data", changed) | |
| with pytest.raises(ValueError, match="configuration: data"): | |
| compare_summaries(fixture(), candidate) | |
| def test_execution_argument_changes_on_only_one_rank_are_rejected(argument, value): | |
| candidate = fixture() | |
| candidate["ranks"][1]["configuration"]["args"][argument] = value | |
| with pytest.raises(ValueError, match="Rank 1 execution argument mismatch"): | |
| compare_summaries(fixture(), candidate) | |
| def test_jsonl_fallback_hashes_every_consumed_input(tmp_path): | |
| run = fixture() | |
| for rank_summary in run["ranks"]: | |
| rank_summary["configuration"]["event"] = "configuration" | |
| for update in rank_summary["updates"]: | |
| update["event"] = "update" | |
| summary = copy.deepcopy(run["ranks"][0]) | |
| (tmp_path / "summary.json").write_text(json.dumps(summary)) | |
| for rank, rank_summary in enumerate(run["ranks"]): | |
| records = [rank_summary["configuration"], *rank_summary["updates"]] | |
| (tmp_path / f"rank-{rank}.jsonl").write_text("\n".join(json.dumps(row) for row in records) + "\n") | |
| loaded = load_run(tmp_path) | |
| paths = [tmp_path / name for name in ("summary.json", "rank-0.jsonl", "rank-1.jsonl")] | |
| assert loaded["input_files"] == [{"path": str(path), "sha256": hashlib.sha256(path.read_bytes()).hexdigest()} | |
| for path in paths] | |
| assert len(loaded["summary"]["ranks"]) == 2 | |
| before = loaded["input_files"][2]["sha256"] | |
| # Semantically irrelevant trailing whitespace changes the exact consumed | |
| # input hash, while preserving successful reconstruction and validation. | |
| paths[2].write_text(paths[2].read_text().rstrip() + " \n") | |
| after = load_run(tmp_path) | |
| assert after["summary"] == loaded["summary"] | |
| assert after["input_files"][2]["sha256"] != before | |