| """ |
| Verification Suite for VIU-1.5B-MoE Ultra-Deep 48-Layer Architecture. |
| Audits parameter counts, active compute, MLA latent dimensions, and expert distributions. |
| """ |
| import sys |
| from pathlib import Path |
| import torch |
|
|
| sys.path.append(str(Path(__file__).parent)) |
| from viu_moe import ViuArgs, Viu1MoE, load_args |
|
|
| def verify_architecture(config_path: str): |
| print("=" * 75) |
| print(f"VERIFYING FRONTIER ARCHITECTURE: {config_path}") |
| print("=" * 75) |
| |
| cfg = load_args(config_path) |
| print(f"[Config] dim={cfg.dim}, layers={cfg.n_layers}, heads={cfg.n_heads}") |
| print(f"[MLA] q_lora_rank={cfg.q_lora_rank}, kv_lora_rank={cfg.kv_lora_rank}, rope_dim={cfg.rope_dim}") |
| print(f"[Config] vocab={cfg.vocab_size}, max_seq={cfg.max_seq_len}, rope_theta={cfg.rope_theta}") |
| print(f"[MoE] routed_experts={cfg.moe_num_routed_experts}, shared_experts={cfg.moe_shared_experts}, top_k={cfg.moe_top_k}") |
| print(f"[MoE] moe_every={cfg.moe_every}, expert_div={cfg.moe_expert_div}, multi_token={cfg.multi_token}") |
| print(f"[Caps] attn_softcap={cfg.attn_logit_softcapping}, final_softcap={cfg.final_logit_softcapping}") |
| |
| |
| with torch.device('meta'): |
| model = Viu1MoE(cfg) |
| total_p, active_p = model.param_counts() |
| |
| print("-" * 75) |
| print(f"[Audit] Total Parameters: {total_p:,} ({total_p/1e6:.2f}M / {total_p/1e9:.3f}B)") |
| print(f"[Audit] Active Parameters: {active_p:,} ({active_p/1e6:.2f}M / {active_p/1e9:.3f}B)") |
| print(f"[Audit] Active Sparsity: {active_p / total_p * 100:.2f}% active per token") |
| |
| total_experts = cfg.n_layers * (cfg.moe_num_routed_experts + cfg.moe_shared_experts) |
| print(f"[Audit] Total Experts: {total_experts:,} experts ({cfg.moe_shared_experts} shared + {cfg.moe_num_routed_experts} routed per layer)") |
| print(f"[Audit] Attention Heads: {cfg.n_heads} Latent Query Heads (MLA with decoupled RoPE)") |
| print(f"[Audit] Head Dimension: {cfg.head_dim} (Standard LLM head size)") |
| |
| |
| n_moe = sum(1 for b in model.layers if b.is_moe) |
| n_dense = cfg.n_layers - n_moe |
| n_sliding = sum(1 for b in model.layers if b.sliding) |
| n_full = sum(1 for b in model.layers if not b.sliding) |
| print(f"[Layers] MoE Layers: {n_moe}/{cfg.n_layers}") |
| print(f"[Layers] Dense Layers: {n_dense}/{cfg.n_layers}") |
| print(f"[Layers] Sliding Window: {n_sliding}/{cfg.n_layers} (window={cfg.sliding_window})") |
| print(f"[Layers] Full Attention: {n_full}/{cfg.n_layers}") |
| print("-" * 75) |
| |
| |
| assert 1_800_000_000 <= total_p <= 1_920_000_000, f"Total params {total_p} outside expected range!" |
| assert 280_000_000 <= active_p <= 320_000_000, f"Active params {active_p} outside expected range!" |
| assert cfg.n_layers == 48, f"Expected 48 layers, got {cfg.n_layers}!" |
| assert total_experts == 1392, f"Expected 1,392 experts, got {total_experts}!" |
| assert cfg.moe_shared_experts == 1, f"Expected 1 shared expert, got {cfg.moe_shared_experts}!" |
| assert cfg.moe_num_routed_experts == 28, f"Expected 28 routed experts, got {cfg.moe_num_routed_experts}!" |
| assert cfg.q_lora_rank == 256 and cfg.kv_lora_rank == 256, "Expected MLA ranks 256!" |
| |
| print("SUCCESS: VIU-1.5B-MoE 48-Layer Frontier Architecture is 100% Validated!") |
| print("=" * 75) |
|
|
| if __name__ == "__main__": |
| default_cfg = Path(__file__).parent.parent / "configs" / "model_config.yaml" |
| verify_architecture(str(default_cfg)) |
|
|
