config.json

{
  "architectures": [
    "RizeForCausalLM"
  ],
  "attention_arch": "kimi_linear",
  "attention_bias": false,
  "attention_dropout": 0.0,
  "auto_map": {
    "AutoConfig": "configuration_rize.RizeConfig",
    "AutoModel": "modeling_rize.RizeModel",
    "AutoModelForCausalLM": "modeling_rize.RizeForCausalLM"
  },
  "aux_loss_alpha": 0.0,
  "auxfree_bias_clip": 5.0,
  "auxfree_bias_lr": 0.0,
  "auxfree_update_interval": 8,
  "block_diag_causal_mask": true,
  "bos_token_id": null,
  "collect_moe_stats": true,
  "doc_sep_token": null,
  "doc_sep_token_id": null,
  "dtype": "bfloat16",
  "eos_token_id": 151645,
  "ep_size": 1,
  "first_k_dense_replace": 1,
  "force_chunk_attention": true,
  "force_chunk_mode": true,
  "force_training_chunk_mode": true,
  "freeze_router_on_sft": true,
  "global_lbl_buffer_across_ga": true,
  "global_lbl_enabled": true,
  "global_lbl_sync_across_ranks": true,
  "hidden_act": "silu",
  "hidden_size": 1536,
  "initializer_range": 0.02,
  "intermediate_size": 8192,
  "kv_lora_rank": 512,
  "linear_attn_config": {
    "full_attn_layers": [
      4,
      8,
      12,
      16,
      19
    ],
    "head_dim": 128,
    "kda_layers": [
      1,
      2,
      3,
      5,
      6,
      7,
      9,
      10,
      11,
      13,
      14,
      15,
      17,
      18
    ],
    "num_heads": 12,
    "short_conv_kernel_size": 4
  },
  "linear_ce_impl": "cce_exact",
  "max_position_embeddings": 32768,
  "model_type": "rize",
  "moe_intermediate_size": 704,
  "moe_layer_freq": 1,
  "moe_router_active_only": true,
  "n_group": 1,
  "n_routed_experts": 64,
  "n_shared_experts": 1,
  "norm_topk_prob": true,
  "num_attention_heads": 12,
  "num_experts_per_tok": 4,
  "num_hidden_layers": 19,
  "num_key_value_heads": 12,
  "num_nextn_predict_layers": 0,
  "pad_token_id": 151643,
  "pretraining_tp": 1,
  "prompt_loss_weight": 0.1,
  "q_lora_rank": null,
  "qk_nope_head_dim": 128,
  "qk_rope_head_dim": 64,
  "reset_position_ids_per_sample": true,
  "return_logits_in_train": false,
  "rms_norm_eps": 1e-05,
  "rope_beta_fast": 32.0,
  "rope_beta_slow": 1.0,
  "rope_mscale": 1.0,
  "rope_mscale_all_dim": 1.0,
  "rope_original_max_position_embeddings": 8192,
  "rope_scaling": {
    "beta_fast": 32.0,
    "beta_slow": 1.0,
    "factor": 4.0,
    "mscale": 1.0,
    "mscale_all_dim": 1.0,
    "original_max_position_embeddings": 8192,
    "rope_type": "yarn",
    "type": "yarn"
  },
  "rope_scaling_factor": 4.0,
  "rope_scaling_type": "yarn",
  "rope_theta": 50000.0,
  "routed_scaling_factor": 1.9984212625219382,
  "sample_sep_is_eos": true,
  "sample_sep_token": "<|im_end|>",
  "sample_sep_token_id": 151645,
  "scoring_func": "sigmoid",
  "seq_aux": false,
  "tie_word_embeddings": false,
  "topk_group": 1,
  "topk_method": "noaux_tc",
  "transformers_version": "4.57.3",
  "use_cache": false,
  "use_doc_sep": false,
  "use_linear_ce": true,
  "v_head_dim": 128,
  "vocab_size": 151669
}