Download test_backward_layout_training.py from zerdovzad/WarpStateAI: direct link, hf CLI and curl.
- Browser
- Download file 5.59 kB
-
https://huggingface.co/zerdovzad/WarpStateAI/resolve/main/test_backward_layout_training.py
- Command line
-
hf download hf://zerdovzad/WarpStateAI/test_backward_layout_training.py
-
curl -L -o test_backward_layout_training.py https://huggingface.co/zerdovzad/WarpStateAI/resolve/main/test_backward_layout_training.py
5.59 kB
| """CPU checks of the full-training backward-layout validation oracle.""" | |
| import json | |
| import pytest | |
| import torch | |
| from ops.validate_backward_layout_training import ( | |
| ACCUMULATION, MICRO_TOKENS, TOLERANCES, arguments, compare_named_tensors, exact_tree_equal, | |
| fixture_batch_cpu, fixture_config, tensor_comparison, update_plan, within_tolerance, | |
| ) | |
| def test_fixture_retains_production_attention_and_ffn_geometry(): | |
| cfg = fixture_config() | |
| assert (cfg.d_model, cfg.n_heads, cfg.head_dim, cfg.chunk_size) == (2048, 32, 64, 128) | |
| assert cfg.ffn_hidden == 7168 and cfg.parallel_memory and cfg.dropout == 0 | |
| assert cfg.n_cores == cfg.logical_depth == 1 | |
| assert not cfg.gradient_checkpointing | |
| def test_multicore_fixture_has_distinct_parameters_at_each_logical_layer(): | |
| cfg = fixture_config(True, cores=2) | |
| assert cfg.n_cores == cfg.logical_depth == 2 | |
| assert cfg.d_model == 2048 and cfg.ffn_hidden == 7168 | |
| def test_fixture_rejects_invalid_core_counts(cores): | |
| with pytest.raises(ValueError, match="positive integer"): | |
| fixture_config(cores=cores) | |
| def test_unique_fixture_removes_repeated_embedding_indices_reproducibly(): | |
| assert fixture_config(True).vocab_size == MICRO_TOKENS | |
| first = fixture_batch_cpu(191, 0, 0, 0, True) | |
| again = fixture_batch_cpu(191, 0, 0, 0, True) | |
| next_micro = fixture_batch_cpu(191, 0, 0, 1, True) | |
| assert torch.equal(first[0], again[0]) and torch.equal(first[1], again[1]) | |
| assert not torch.equal(first[0], next_micro[0]) | |
| assert torch.equal(first[0].reshape(-1).sort().values, torch.arange(MICRO_TOKENS)) | |
| assert first[1].min() >= 0 and first[1].max() < MICRO_TOKENS | |
| repeated, _ = fixture_batch_cpu(191, 0, 0, 0, False) | |
| assert repeated.unique().numel() <= 512 < repeated.numel() | |
| def test_update_plan_has_accumulation_then_masked_final_rank(world_size): | |
| assert [update_plan(world_size, update) for update in range(3)] == [ | |
| MICRO_TOKENS * ACCUMULATION * world_size, MICRO_TOKENS * ACCUMULATION * world_size, 9] | |
| assert ACCUMULATION >= 3 | |
| final_valid = [max(0, min(MICRO_TOKENS, 9 - rank * MICRO_TOKENS)) for rank in range(world_size)] | |
| assert sum(final_valid) == 9 | |
| if world_size == 2: | |
| assert final_valid == [9, 0] | |
| def test_oracle_rejects_mismatched_tensor_contract(actual, expected): | |
| with pytest.raises(ValueError, match="shape/dtype"): | |
| tensor_comparison(actual, expected) | |
| def test_oracle_distinguishes_signed_zero_and_nonfinite_values(): | |
| expected = torch.zeros(3, dtype=torch.bfloat16) | |
| observed = expected.clone() | |
| observed[0] = -0.0 | |
| metrics = tensor_comparison(observed, expected) | |
| assert metrics["unequal"] == 0 and metrics["bitwise_unequal"] == 1 | |
| assert not within_tolerance(metrics, "exact") | |
| observed[1] = float("nan") | |
| metrics = tensor_comparison(observed, expected) | |
| assert not metrics["finite"] and not within_tolerance(metrics, "gradient") | |
| json.dumps(metrics, allow_nan=False) | |
| def test_oracle_checks_relative_norm_and_worst_element(): | |
| expected = torch.ones(1000) | |
| observed = expected + TOLERANCES["gradient_relative_l2"] / 2 | |
| assert within_tolerance(tensor_comparison(observed, expected), "gradient") | |
| observed = expected.clone() | |
| observed[0] += TOLERANCES["gradient_relative_max"] * 2 | |
| metrics = tensor_comparison(observed, expected) | |
| assert metrics["relative_l2"] < TOLERANCES["gradient_relative_l2"] | |
| assert not within_tolerance(metrics, "gradient") | |
| assert not within_tolerance(tensor_comparison(expected * 1.02, expected), "gradient") | |
| def test_oracle_uses_absolute_parameter_limit(): | |
| expected = torch.zeros(3) | |
| assert within_tolerance(tensor_comparison(expected + 1e-5, expected), "parameter") | |
| assert not within_tolerance(tensor_comparison(expected + 3e-5, expected), "parameter") | |
| def test_named_comparison_checks_every_parameter(): | |
| expected = {"first": torch.ones(4), "second": torch.ones(2)} | |
| result = compare_named_tensors(expected, expected, "gradient") | |
| assert result["accepted"] and result["bitwise_equal"] | |
| with pytest.raises(ValueError, match="names"): | |
| compare_named_tensors({"first": torch.ones(4)}, expected, "gradient") | |
| result = compare_named_tensors({**expected, "second": torch.zeros(2)}, expected, "gradient") | |
| assert not result["accepted"] and not result["bitwise_equal"] | |
| def test_optimizer_step_counter_must_match_exactly(): | |
| wanted = {"weight/step": torch.tensor(40081.0)} | |
| actual = {"weight/step": torch.tensor(40082.0)} | |
| assert not compare_named_tensors(actual, wanted, "moment")["accepted"] | |
| def test_resume_oracle_includes_dtype_signed_zero_and_metadata(): | |
| expected = {"model": {"weight": torch.tensor([0.0])}, "rng": [torch.tensor([1, 2])], | |
| "execution": {"identity": "version1"}} | |
| assert exact_tree_equal(expected, expected) | |
| assert not exact_tree_equal({**expected, "model": {"weight": torch.tensor([-0.0])}}, expected) | |
| assert not exact_tree_equal({**expected, "execution": {"identity": "version2"}}, expected) | |
| assert not exact_tree_equal({**expected, "rng": [torch.tensor([1.0, 2.0])]}, expected) | |
| def test_cli_rejects_outputs_outside_generated_validation_directory(): | |
| with pytest.raises(SystemExit): | |
| arguments(["--output-dir", "/workspace/WarpState/rtx6000_40b/ops/bad-output"]) | |