| { |
| "activation_recompute_policy": "layer", |
| "attention_backend": "varlen_fused_required", |
| "attention_indices": [ |
| 5, |
| 10, |
| 15 |
| ], |
| "attention_pass_lora_rank": 8, |
| "autotune": { |
| "compile_modes": [ |
| "none" |
| ], |
| "dispatch_overlap": [ |
| "on" |
| ], |
| "gates": { |
| "max_fp8_loss_relative_error": 0.02, |
| "max_grad_norm": 100.0, |
| "max_hbm_fraction": 0.9, |
| "max_ngram_layout_loss_relative_error": 0.001, |
| "max_update_to_weight_ratio": 0.01 |
| }, |
| "global_token_batch": { |
| "maximum": 12582912, |
| "minimum": 1572864, |
| "target": 6291456 |
| }, |
| "grad_accum_steps": [ |
| 1 |
| ], |
| "learning_rates": [ |
| 0.00012 |
| ], |
| "micro_batch_sizes": [ |
| 4, |
| 2 |
| ], |
| "ngram_table_modes": [ |
| "row_sharded" |
| ], |
| "precision_profiles": [ |
| "bf16" |
| ], |
| "preferred_learning_rate": 0.00012, |
| "status": "bounded_canary_candidates_not_validated_optima" |
| }, |
| "bucket_varlen_attention": false, |
| "budget_controller_leak": 0.0, |
| "budget_controller_rate": 0.0, |
| "budget_multiplier_limit": 1000.0, |
| "budgeted_depth_calibration_coefficient": 0.0, |
| "budgeted_depth_values": [], |
| "budgeted_k_calibration_coefficient": 0.0, |
| "causal_reference_ceiling": [], |
| "context_extension_train_length": 163840, |
| "context_parallel_size": 1, |
| "continuation_entropy_coefficient": 0.001, |
| "continuation_gate_init": 0.0, |
| "d_model": 2560, |
| "dense_ffn_intermediate_dim": 0, |
| "depth_budget_coefficient": 0.01, |
| "depth_budget_controller_rate": 0.0, |
| "expected_parameter_audit": { |
| "active_per_pass_max": 1378197900, |
| "active_per_pass_mean": 1189454220, |
| "active_per_pass_min": 1047896460, |
| "continuation": 2032129, |
| "depth_memory": 4211972, |
| "embedding": 167772160, |
| "expert_routers": 6107040, |
| "final_norm": 2560, |
| "latent_projections": 104857600, |
| "mhc": 176320, |
| "mixers": 833409535, |
| "ngram_fusion": 2726440, |
| "ngram_tables": 1771823360, |
| "routed_experts": 9059696640, |
| "shared_experts": 47185920, |
| "stored_total": 12000001676 |
| }, |
| "expert_balance_bias_update_rate": 0.001, |
| "expert_balance_coefficient": 0.0, |
| "expert_execution": "grouped_gemm", |
| "expert_gate_normalisation": "selected", |
| "expert_intermediate_dim": 768, |
| "expert_parallel_size": 1, |
| "expert_replicas": 1, |
| "expert_swiglu_fused": false, |
| "expert_weight_chunks": 1, |
| "family": "logos", |
| "ffn_mode": "moe", |
| "final_context_length": 131072, |
| "head_dim": 64, |
| "k_budget_coefficient": 0.01, |
| "latent_dim": 1024, |
| "lm_head_chunk_size": 1024, |
| "mamba_backend": "fused_required", |
| "mamba_chunk_size": 256, |
| "mamba_d_conv": 4, |
| "mamba_d_state": 128, |
| "mamba_expand": 2, |
| "mamba_head_dim": 64, |
| "mamba_ngroups": 10, |
| "max_passes": 5, |
| "max_routed_k": 8, |
| "memory_dim": 256, |
| "memory_gate_init": -6.0, |
| "memory_heads": 4, |
| "mhc_backend": "fused_required", |
| "mhc_pass_embedding_dim": 64, |
| "mhc_sinkhorn_iterations": 8, |
| "min_routed_k": 1, |
| "model_type": "metis16_more", |
| "moe_dispatch_capacity_factor": 0.0, |
| "moe_dispatch_chunks": 1, |
| "n_heads": 40, |
| "n_kv_heads": 10, |
| "n_layers": 20, |
| "n_routed_experts": 192, |
| "n_shared_experts": 1, |
| "n_streams": 4, |
| "name": "Metis-1.6-Logos", |
| "ngram_memory": { |
| "canonicalization": "tokenizer_sidecar_required", |
| "hash_seeds": [ |
| 2654435761, |
| 2246822519, |
| 3266489917, |
| 668265263, |
| 374761393, |
| 3550635117, |
| 4251991749, |
| 3043081225 |
| ], |
| "injection_layers": [ |
| 2, |
| 5 |
| ], |
| "learning_rate_scale": 2.0, |
| "optimizer": "sparse_adam", |
| "orders": [ |
| 2, |
| 3 |
| ], |
| "slots_by_order": { |
| "2": [ |
| 1723303, |
| 1724299, |
| 1725301, |
| 1726303, |
| 1727317, |
| 1728317, |
| 1729307, |
| 1730101 |
| ], |
| "3": [ |
| 1730507, |
| 1731311, |
| 1732301, |
| 1733297, |
| 1734311, |
| 1735301, |
| 1736303, |
| 1737161 |
| ] |
| }, |
| "sparse_gradients": false, |
| "table_mode": "replicated", |
| "tables_per_order": 8, |
| "value_dim": 64, |
| "weight_decay": 0.0 |
| }, |
| "no_position_embeddings": true, |
| "precision": { |
| "activation_dtype": "bfloat16", |
| "allow_bf16_fallback": true, |
| "backend": "auto", |
| "bf16_roles": [ |
| "embedding", |
| "ngram_table", |
| "residual_stream", |
| "mamba_state", |
| "normalization", |
| "gate", |
| "collective", |
| "attention_pass_lora_down", |
| "attention_pass_lora_up", |
| "memory_metadata_write_projection" |
| ], |
| "expert_collective_wire": "fp8", |
| "fp32_roles": [ |
| "master_weight", |
| "optimizer_state", |
| "router_logits", |
| "sinkhorn", |
| "memory_attention_logits", |
| "loss_accumulation", |
| "mamba_sensitive" |
| ], |
| "fp8_format": "hybrid_e4m3_e5m2", |
| "fp8_roles": [ |
| "mamba_in_projection", |
| "mamba_out_projection", |
| "attention_qkv_projection", |
| "attention_out_projection", |
| "expert_gate_up_projection", |
| "expert_down_projection", |
| "latent_down_projection", |
| "latent_up_projection", |
| "memory_state_write_projection", |
| "memory_query_projection", |
| "memory_key_projection", |
| "memory_value_projection", |
| "memory_output_projection", |
| "memory_route_projection", |
| "mhc_controller", |
| "ngram_projection", |
| "lm_head" |
| ], |
| "fp8_scaling": "delayed", |
| "master_weight_dtype": "float32", |
| "optimizer_state_dtype": "float32", |
| "parameter_dtype": "bfloat16", |
| "reduction_dtype": "float32", |
| "require_fp8_validation": true, |
| "router_dtype": "float32" |
| }, |
| "reference_operating_point_flops_guard": false, |
| "route_feature_dim": 256, |
| "router_z_loss_coefficient": 0.001, |
| "schema": "metis.model-family/v1", |
| "sequence_length": 4096, |
| "static_document_capacity": 0, |
| "target_mean_passes": 2.0, |
| "target_mean_routed_k": 4.0, |
| "tie_embeddings": true, |
| "vocab_size": 65536, |
| "world_size": 1 |
| } |
|
|
