{
  "activation_recompute_policy": "layer",
  "attention_backend": "varlen_fused_required",
  "attention_indices": [
    5,
    10,
    15
  ],
  "attention_pass_lora_rank": 8,
  "autotune": {
    "compile_modes": [
      "none"
    ],
    "dispatch_overlap": [
      "on"
    ],
    "gates": {
      "max_fp8_loss_relative_error": 0.02,
      "max_grad_norm": 100.0,
      "max_hbm_fraction": 0.9,
      "max_ngram_layout_loss_relative_error": 0.001,
      "max_update_to_weight_ratio": 0.01
    },
    "global_token_batch": {
      "maximum": 12582912,
      "minimum": 1572864,
      "target": 6291456
    },
    "grad_accum_steps": [
      1
    ],
    "learning_rates": [
      0.00012
    ],
    "micro_batch_sizes": [
      4,
      2
    ],
    "ngram_table_modes": [
      "row_sharded"
    ],
    "precision_profiles": [
      "bf16"
    ],
    "preferred_learning_rate": 0.00012,
    "status": "bounded_canary_candidates_not_validated_optima"
  },
  "bucket_varlen_attention": false,
  "budget_controller_leak": 0.0,
  "budget_controller_rate": 0.0,
  "budget_multiplier_limit": 1000.0,
  "budgeted_depth_calibration_coefficient": 0.0,
  "budgeted_depth_values": [],
  "budgeted_k_calibration_coefficient": 0.0,
  "causal_reference_ceiling": [],
  "context_extension_train_length": 163840,
  "context_parallel_size": 1,
  "continuation_entropy_coefficient": 0.001,
  "continuation_gate_init": 0.0,
  "d_model": 2560,
  "dense_ffn_intermediate_dim": 0,
  "depth_budget_coefficient": 0.01,
  "depth_budget_controller_rate": 0.0,
  "expected_parameter_audit": {
    "active_per_pass_max": 1378197900,
    "active_per_pass_mean": 1189454220,
    "active_per_pass_min": 1047896460,
    "continuation": 2032129,
    "depth_memory": 4211972,
    "embedding": 167772160,
    "expert_routers": 6107040,
    "final_norm": 2560,
    "latent_projections": 104857600,
    "mhc": 176320,
    "mixers": 833409535,
    "ngram_fusion": 2726440,
    "ngram_tables": 1771823360,
    "routed_experts": 9059696640,
    "shared_experts": 47185920,
    "stored_total": 12000001676
  },
  "expert_balance_bias_update_rate": 0.001,
  "expert_balance_coefficient": 0.0,
  "expert_execution": "grouped_gemm",
  "expert_gate_normalisation": "selected",
  "expert_intermediate_dim": 768,
  "expert_parallel_size": 1,
  "expert_replicas": 1,
  "expert_swiglu_fused": false,
  "expert_weight_chunks": 1,
  "family": "logos",
  "ffn_mode": "moe",
  "final_context_length": 131072,
  "head_dim": 64,
  "k_budget_coefficient": 0.01,
  "latent_dim": 1024,
  "lm_head_chunk_size": 1024,
  "mamba_backend": "fused_required",
  "mamba_chunk_size": 256,
  "mamba_d_conv": 4,
  "mamba_d_state": 128,
  "mamba_expand": 2,
  "mamba_head_dim": 64,
  "mamba_ngroups": 10,
  "max_passes": 5,
  "max_routed_k": 8,
  "memory_dim": 256,
  "memory_gate_init": -6.0,
  "memory_heads": 4,
  "mhc_backend": "fused_required",
  "mhc_pass_embedding_dim": 64,
  "mhc_sinkhorn_iterations": 8,
  "min_routed_k": 1,
  "model_type": "metis16_more",
  "moe_dispatch_capacity_factor": 0.0,
  "moe_dispatch_chunks": 1,
  "n_heads": 40,
  "n_kv_heads": 10,
  "n_layers": 20,
  "n_routed_experts": 192,
  "n_shared_experts": 1,
  "n_streams": 4,
  "name": "Metis-1.6-Logos",
  "ngram_memory": {
    "canonicalization": "tokenizer_sidecar_required",
    "hash_seeds": [
      2654435761,
      2246822519,
      3266489917,
      668265263,
      374761393,
      3550635117,
      4251991749,
      3043081225
    ],
    "injection_layers": [
      2,
      5
    ],
    "learning_rate_scale": 2.0,
    "optimizer": "sparse_adam",
    "orders": [
      2,
      3
    ],
    "slots_by_order": {
      "2": [
        1723303,
        1724299,
        1725301,
        1726303,
        1727317,
        1728317,
        1729307,
        1730101
      ],
      "3": [
        1730507,
        1731311,
        1732301,
        1733297,
        1734311,
        1735301,
        1736303,
        1737161
      ]
    },
    "sparse_gradients": false,
    "table_mode": "replicated",
    "tables_per_order": 8,
    "value_dim": 64,
    "weight_decay": 0.0
  },
  "no_position_embeddings": true,
  "precision": {
    "activation_dtype": "bfloat16",
    "allow_bf16_fallback": true,
    "backend": "auto",
    "bf16_roles": [
      "embedding",
      "ngram_table",
      "residual_stream",
      "mamba_state",
      "normalization",
      "gate",
      "collective",
      "attention_pass_lora_down",
      "attention_pass_lora_up",
      "memory_metadata_write_projection"
    ],
    "expert_collective_wire": "fp8",
    "fp32_roles": [
      "master_weight",
      "optimizer_state",
      "router_logits",
      "sinkhorn",
      "memory_attention_logits",
      "loss_accumulation",
      "mamba_sensitive"
    ],
    "fp8_format": "hybrid_e4m3_e5m2",
    "fp8_roles": [
      "mamba_in_projection",
      "mamba_out_projection",
      "attention_qkv_projection",
      "attention_out_projection",
      "expert_gate_up_projection",
      "expert_down_projection",
      "latent_down_projection",
      "latent_up_projection",
      "memory_state_write_projection",
      "memory_query_projection",
      "memory_key_projection",
      "memory_value_projection",
      "memory_output_projection",
      "memory_route_projection",
      "mhc_controller",
      "ngram_projection",
      "lm_head"
    ],
    "fp8_scaling": "delayed",
    "master_weight_dtype": "float32",
    "optimizer_state_dtype": "float32",
    "parameter_dtype": "bfloat16",
    "reduction_dtype": "float32",
    "require_fp8_validation": true,
    "router_dtype": "float32"
  },
  "reference_operating_point_flops_guard": false,
  "route_feature_dim": 256,
  "router_z_loss_coefficient": 0.001,
  "schema": "metis.model-family/v1",
  "sequence_length": 4096,
  "static_document_capacity": 0,
  "target_mean_passes": 2.0,
  "target_mean_routed_k": 4.0,
  "tie_embeddings": true,
  "vocab_size": 65536,
  "world_size": 1
}