| """ | |
| Comprehensive Unit Tests for Agent 3 Refactoring: | |
| 1. FP16 Variance Underflow / FP32 Accumulators in RMSNorm and LayerNorm (layer_norm_fix.py). | |
| 2. Zipfian Power-Law Token Sampling and Perplexity Dataset (synthetic_generator.py). | |
| 3. Tikhonov Damped Matrix Square Root & 2-Wasserstein Hessian Drift (detectors.py). | |
| 4. Streaming Layerwise Analysis OOM Prevention (layerwise_analyzer.py). | |
| 5. Grassmann / Riemannian Manifold SVD Basis Federated Averaging (grassmann_fedavg.py). | |
| """ | |
| from __future__ import annotations | |
| import unittest | |
| import numpy as np | |
| import torch | |
| import torch.nn as nn | |
| # 1. LayerNorm / RMSNorm FP32 Accumulator | |
| from framework.utils.layer_norm_fix import ( | |
| RMSNorm, | |
| LayerNorm, | |
| SafeRMSNorm, | |
| SafeLayerNorm, | |
| fix_layernorm_naming, | |
| replace_layernorm_with_safe_norm, | |
| ) | |
| # 2. Synthetic Generator | |
| from framework.data.synthetic_generator import ( | |
| generate_zipfian_tokens, | |
| generate_zipfian_markov_tokens, | |
| compute_empirical_entropy, | |
| SyntheticPerplexityDataset, | |
| SyntheticClassificationDataset, | |
| ) | |
| # 3. Drift & Wasserstein-2 | |
| from framework.drift.detectors import ( | |
| matrix_square_root, | |
| bures_wasserstein_distance, | |
| hessian_wasserstein, | |
| wasserstein, | |
| DriftDetector, | |
| METHOD_REGISTRY, | |
| ) | |
| from framework.drift.config import DriftConfig | |
| # 4. Layerwise Analyzer | |
| from framework.analysis.layerwise_analyzer import ( | |
| LayerwiseAnalyzer, | |
| StreamingLayerAccumulator, | |
| estimate_intrinsic_dimension, | |
| ) | |
| # 5. Federated Learning Grassmann | |
| from framework.federated_learning.strategies.grassmann_fedavg import ( | |
| GrassmannFedAvgAggregator, | |
| grassmann_log_map, | |
| grassmann_exp_map, | |
| grassmann_karcher_mean, | |
| grassmann_frechet_mean_spectral, | |
| is_orthogonal_basis_tensor, | |
| ) | |
| from framework.federated_learning.config import FLConfig, AggregationStrategy | |
| from framework.federated_learning.strategies import AGGREGATOR_REGISTRY as FL_REGISTRY | |
| class TestAgent3Refactoring(unittest.TestCase): | |
| """Test suite for mathematical and statistical stability refactorings.""" | |
| # ------------------------------------------------------------------------- | |
| # 1. FP16 Accumulator & LayerNorm / RMSNorm Tests | |
| # ------------------------------------------------------------------------- | |
| def test_rmsnorm_fp16_large_activation_stability(self): | |
| """Test RMSNorm with extreme values that would overflow FP16 square if uncast.""" | |
| norm = RMSNorm(normalized_shape=128, eps=1e-6) | |
| # 300^2 = 90,000 > 65,504 (FP16 overflow boundary) | |
| x_fp16 = torch.full((4, 16, 128), 300.0, dtype=torch.float16) | |
| out = norm(x_fp16) | |
| self.assertEqual(out.dtype, torch.float16) | |
| self.assertFalse(torch.isnan(out).any()) | |
| self.assertFalse(torch.isinf(out).any()) | |
| # For constant tensor, rms = 300, x / rms = 1.0 | |
| self.assertTrue(torch.allclose(out, torch.ones_like(out), atol=1e-2)) | |
| def test_layernorm_fp16_stability(self): | |
| """Test LayerNorm with FP32 accumulator on mixed-precision tensors.""" | |
| norm = LayerNorm(normalized_shape=64, eps=1e-5) | |
| x = torch.randn(2, 8, 64, dtype=torch.float16) * 50.0 + 200.0 | |
| out = norm(x) | |
| self.assertEqual(out.dtype, torch.float16) | |
| self.assertFalse(torch.isnan(out).any()) | |
| self.assertFalse(torch.isinf(out).any()) | |
| def test_replace_layernorm_with_safe_norm(self): | |
| """Test recursive replacement of standard nn.LayerNorm.""" | |
| model = nn.Sequential( | |
| nn.Linear(32, 32), | |
| nn.LayerNorm(32), | |
| nn.Linear(32, 10), | |
| ) | |
| replace_layernorm_with_safe_norm(model) | |
| self.assertIsInstance(model[1], LayerNorm) | |
| # ------------------------------------------------------------------------- | |
| # 2. Zipfian Token Generator Tests | |
| # ------------------------------------------------------------------------- | |
| def test_zipfian_token_distribution(self): | |
| """Verify Zipfian power-law rank frequency decay.""" | |
| tokens = generate_zipfian_tokens( | |
| num_samples=100, seq_len=500, vocab_size=1000, zipf_param=1.2, seed=42 | |
| ) | |
| self.assertEqual(tokens.shape, (100, 500)) | |
| self.assertEqual(tokens.dtype, torch.long) | |
| self.assertTrue((tokens >= 1).all()) | |
| self.assertTrue((tokens <= 1000).all()) | |
| # Low-rank tokens should appear much more frequently than high-rank tokens | |
| flat = tokens.view(-1).numpy() | |
| counts = np.bincount(flat, minlength=1001) | |
| freq_top10 = np.sum(counts[1:11]) | |
| freq_tail = np.sum(counts[500:510]) | |
| self.assertGreater(freq_top10, freq_tail * 5) | |
| # Check empirical entropy | |
| entropy = compute_empirical_entropy(tokens, vocab_size=1000) | |
| self.assertGreater(entropy, 2.0) | |
| self.assertLess(entropy, 10.0) | |
| def test_synthetic_perplexity_dataset(self): | |
| """Verify SyntheticPerplexityDataset provides causal LM targets.""" | |
| ds = SyntheticPerplexityDataset(num_samples=16, seq_len=64, vocab_size=5000) | |
| self.assertEqual(len(ds), 16) | |
| item = ds[0] | |
| self.assertIn("input_ids", item) | |
| self.assertIn("attention_mask", item) | |
| self.assertIn("labels", item) | |
| self.assertEqual(item["input_ids"].shape, (64,)) | |
| self.assertTrue(torch.equal(item["input_ids"], item["labels"])) | |
| # ------------------------------------------------------------------------- | |
| # 3. Drift & Wasserstein-2 Hessian Stability Tests | |
| # ------------------------------------------------------------------------- | |
| def test_matrix_square_root_with_negative_eigenvalues(self): | |
| """Test matrix_square_root regularizes indefinite/ill-conditioned matrices.""" | |
| # Create an indefinite symmetric matrix with negative eigenvalues | |
| A = np.array([ | |
| [1.0, 2.0, 0.0], | |
| [2.0, 1.0, 0.0], | |
| [0.0, 0.0, -0.5], | |
| ]) | |
| sqrt_A = matrix_square_root(A, damping=1e-5, min_eig=1e-8) | |
| self.assertEqual(sqrt_A.shape, (3, 3)) | |
| self.assertFalse(np.isnan(sqrt_A).any()) | |
| self.assertFalse(np.isinf(sqrt_A).any()) | |
| # Square of square root should be positive semi-definite | |
| reconstructed = sqrt_A @ sqrt_A | |
| eigs = np.linalg.eigvalsh(reconstructed) | |
| self.assertTrue(np.all(eigs >= 0.0)) | |
| def test_bures_wasserstein_distance_properties(self): | |
| """Verify Bures 2-Wasserstein metric properties (identity, symmetry, positivity).""" | |
| cov1 = np.array([[2.0, 0.5], [0.5, 1.5]]) | |
| cov2 = np.array([[3.0, 0.2], [0.2, 2.0]]) | |
| mean1 = np.array([0.0, 1.0]) | |
| mean2 = np.array([1.0, 0.0]) | |
| # Distance to self must be 0 | |
| w_self = bures_wasserstein_distance(cov1, cov1, mean1, mean1) | |
| self.assertAlmostEqual(w_self, 0.0, places=4) | |
| # Symmetry: W(A, B) == W(B, A) | |
| w_12 = bures_wasserstein_distance(cov1, cov2, mean1, mean2) | |
| w_21 = bures_wasserstein_distance(cov2, cov1, mean2, mean1) | |
| self.assertAlmostEqual(w_12, w_21, places=5) | |
| self.assertGreater(w_12, 0.0) | |
| def test_drift_detector_hessian_wasserstein(self): | |
| """Verify DriftDetector integration with registered hessian_wasserstein method.""" | |
| ref_data = {"feat1": np.random.randn(100), "feat2": np.random.randn(100)} | |
| cur_data = {"feat1": np.random.randn(100) + 2.0, "feat2": np.random.randn(100) * 3.0} | |
| config = DriftConfig(methods=["ks", "hessian_wasserstein"], threshold=0.1) | |
| detector = DriftDetector(ref_data, config=config) | |
| report = detector.detect(cur_data) | |
| self.assertIsNotNone(report) | |
| self.assertTrue(len(report.features) == 2) | |
| self.assertIn("hessian_wasserstein", report.features[0].methods) | |
| # ------------------------------------------------------------------------- | |
| # 4. Streaming Layerwise Analysis OOM Prevention Tests | |
| # ------------------------------------------------------------------------- | |
| def test_streaming_layer_accumulator(self): | |
| """Test streaming accumulator updates moments on-the-fly.""" | |
| acc = StreamingLayerAccumulator() | |
| t1 = torch.randn(4, 16, 64) | |
| t2 = torch.randn(4, 16, 64) | |
| acc.update(t1) | |
| acc.update(t2) | |
| self.assertEqual(acc.total_tokens, 128) | |
| self.assertEqual(acc.running_mean_vector.shape, (64,)) | |
| self.assertGreater(acc.max_activation, 0.0) | |
| self.assertGreaterEqual(acc.compute_effective_rank(), 1.0) | |
| self.assertGreaterEqual(acc.compute_outlier_ratio(), 0.0) | |
| def test_layerwise_analyzer_streaming(self): | |
| """Test LayerwiseAnalyzer on mock transformer architecture.""" | |
| class MockLayer(nn.Module): | |
| def __init__(self): | |
| super().__init__() | |
| self.linear = nn.Linear(32, 32) | |
| def forward(self, x): | |
| return (self.linear(x),) | |
| class MockTransformer(nn.Module): | |
| def __init__(self): | |
| super().__init__() | |
| self.layers = nn.ModuleList([MockLayer() for _ in range(4)]) | |
| def forward(self, input_ids, **kwargs): | |
| x = torch.randn(input_ids.shape[0], input_ids.shape[1], 32) | |
| for layer in self.layers: | |
| x = layer(x)[0] | |
| return x | |
| class MockTokenizer: | |
| def __call__(self, texts, **kwargs): | |
| return { | |
| "input_ids": torch.randint(1, 100, (len(texts), 16)), | |
| "attention_mask": torch.ones((len(texts), 16), dtype=torch.long), | |
| } | |
| model = MockTransformer() | |
| tokenizer = MockTokenizer() | |
| analyzer = LayerwiseAnalyzer(model, tokenizer, device="cpu") | |
| results = analyzer.analyze(["Hello world", "Agent 3 refactoring test"], batch_size=2) | |
| self.assertEqual(results["num_layers"], 4) | |
| self.assertEqual(len(results["cosine_similarities"]), 3) | |
| self.assertEqual(len(results["effective_ranks"]), 4) | |
| self.assertEqual(len(results["max_outlier_magnitudes"]), 4) | |
| # ------------------------------------------------------------------------- | |
| # 5. Grassmann / Riemannian Federated Averaging Tests | |
| # ------------------------------------------------------------------------- | |
| def test_grassmann_log_exp_inversion(self): | |
| """Verify Grassmann exp(log(U_0, U_1)) recovers U_1 subspace.""" | |
| np.random.seed(42) | |
| # Generate two random orthonormal bases in St(8, 3) | |
| U0, _ = np.linalg.qr(np.random.randn(8, 3)) | |
| U1, _ = np.linalg.qr(np.random.randn(8, 3)) | |
| delta = grassmann_log_map(U0, U1) | |
| self.assertEqual(delta.shape, (8, 3)) | |
| # Tangent property: U0^T @ Delta should be approximately 0 | |
| self.assertTrue(np.allclose(U0.T @ delta, np.zeros((3, 3)), atol=1e-6)) | |
| U1_reconstructed = grassmann_exp_map(U0, delta) | |
| # Subspace distance ||P_reconstructed - P_target||_F should be negligible | |
| P_orig = U1 @ U1.T | |
| P_rec = U1_reconstructed @ U1_reconstructed.T | |
| self.assertTrue(np.allclose(P_orig, P_rec, atol=1e-5)) | |
| def test_grassmann_frechet_mean_orthogonality(self): | |
| """Verify Grassmann Fréchet mean produces strict orthonormal basis.""" | |
| np.random.seed(42) | |
| bases = [np.linalg.qr(np.random.randn(12, 4))[0] for _ in range(5)] | |
| weights = np.array([0.2, 0.2, 0.2, 0.2, 0.2]) | |
| # Spectral Fréchet Mean | |
| mean_spectral = grassmann_frechet_mean_spectral(bases, weights) | |
| self.assertEqual(mean_spectral.shape, (12, 4)) | |
| gram_spectral = mean_spectral.T @ mean_spectral | |
| self.assertTrue(np.allclose(gram_spectral, np.eye(4), atol=1e-6)) | |
| # Riemannian Karcher Mean | |
| mean_karcher = grassmann_karcher_mean(bases, weights, max_iter=10) | |
| self.assertEqual(mean_karcher.shape, (12, 4)) | |
| gram_karcher = mean_karcher.T @ mean_karcher | |
| self.assertTrue(np.allclose(gram_karcher, np.eye(4), atol=1e-6)) | |
| def test_grassmann_fedavg_aggregator_end_to_end(self): | |
| """Verify GrassmannFedAvgAggregator handles both SVD bases and standard weights.""" | |
| class MockSVDLinearAttention(nn.Module): | |
| def __init__(self): | |
| super().__init__() | |
| # Orthogonal basis U | |
| U, _ = torch.linalg.qr(torch.randn(16, 4)) | |
| self.u_proj = nn.Parameter(U) | |
| self.bias = nn.Parameter(torch.zeros(16)) | |
| model = MockSVDLinearAttention() | |
| config = FLConfig(aggregation=AggregationStrategy.GRASSMANN_FEDAVG) | |
| aggregator = FL_REGISTRY.resolve("grassmann_fedavg")(config) | |
| # Simulate 3 client updates | |
| client_updates = [] | |
| for _ in range(3): | |
| # Client delta for basis U | |
| U_client, _ = torch.linalg.qr(model.u_proj.data + 0.1 * torch.randn(16, 4)) | |
| delta_u = (U_client - model.u_proj.data).numpy() | |
| delta_b = np.random.randn(16) * 0.1 | |
| client_updates.append({"u_proj": delta_u, "bias": delta_b}) | |
| client_weights = np.array([100, 200, 100]) | |
| new_update = aggregator.aggregate(model, client_updates, client_weights, round_num=1) | |
| self.assertIn("u_proj", new_update) | |
| self.assertIn("bias", new_update) | |
| # Verify resulting aggregated basis is orthonormal | |
| new_global_u = model.u_proj.detach().numpy() + new_update["u_proj"] | |
| gram = new_global_u.T @ new_global_u | |
| self.assertTrue(np.allclose(gram, np.eye(4), atol=1e-5)) | |
| if __name__ == "__main__": | |
| unittest.main() | |
Xet Storage Details
- Size:
- 13.6 kB
- Xet hash:
- 887a268713a5f3cc45934dff418310486ebfc15b41e2deb898f4d19adf1a3949
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.