| { |
| "allow_negative_eigenvalues": false, |
| "architecture": "SenryuForCausalLM", |
| "architectures": [ |
| "SenryuForCausalLM" |
| ], |
| "attention_dropout": 0.0, |
| "attention_head_dim": 128, |
| "attention_layer_type": "gated_mla", |
| "bos_token_id": 0, |
| "chat_template": "smollm3", |
| "checkpoint_step": 704, |
| "eos_token_id": 1, |
| "full_attention_interval": 4, |
| "hidden_size": 1024, |
| "initializer_range": 0.02, |
| "intermediate_size": 3392, |
| "kda_lower_bound": -5.0, |
| "kda_safe_gate": true, |
| "layer_types": [ |
| "gdn2", |
| "gdn2", |
| "gdn2", |
| "gated_mla", |
| "gdn2", |
| "gdn2", |
| "gdn2", |
| "gated_mla", |
| "gdn2", |
| "gdn2", |
| "gdn2", |
| "gated_mla", |
| "gdn2", |
| "gdn2", |
| "gdn2", |
| "gated_mla", |
| "gdn2", |
| "gdn2", |
| "gdn2", |
| "gated_mla", |
| "gdn2", |
| "gdn2", |
| "gdn2", |
| "gated_mla", |
| "gdn2", |
| "gdn2", |
| "gdn2", |
| "gated_mla", |
| "gdn2", |
| "gdn2", |
| "gdn2", |
| "gated_mla", |
| "gdn2", |
| "gdn2", |
| "gdn2", |
| "gated_mla", |
| "gdn2", |
| "gdn2", |
| "gdn2", |
| "gated_mla", |
| "gdn2", |
| "gdn2", |
| "gdn2", |
| "gated_mla" |
| ], |
| "linear_conv_kernel_dim": 4, |
| "linear_expand_v": 1.0, |
| "linear_head_dim": 128, |
| "linear_layer_type": "gdn2", |
| "linear_num_heads": 8, |
| "linear_num_value_heads": 8, |
| "max_position_embeddings": 32768, |
| "mla_kv_lora_rank": 512, |
| "mla_q_lora_rank": null, |
| "mla_qk_nope_head_dim": 128, |
| "mla_v_head_dim": 128, |
| "mlp_activation": "situ_glu", |
| "model_family": "Senryu", |
| "model_name": "Senryu", |
| "model_type": "senryu", |
| "mtp_adapter_rank": 128, |
| "mtp_loss_weight": 0.0, |
| "mtp_num_heads": 0, |
| "num_attention_heads": 8, |
| "num_hidden_layers": 44, |
| "num_key_value_heads": 8, |
| "pad_token_id": 2, |
| "partial_rotary_factor": 0.5, |
| "preference_beta": 0.05, |
| "preference_hours": 1.7632968792019852, |
| "preference_loss": "apo_zero", |
| "preference_reward_model": "Skywork/Skywork-Reward-V2-Llama-3.1-8B", |
| "preference_run": "apo-v2", |
| "preference_train_pairs": 22517, |
| "preference_val_loss_ema": 0.8922630808546166, |
| "preference_val_pairs": 903, |
| "preference_val_reward_accuracy_ema": 0.7895902547065338, |
| "preference_val_reward_margin_ema": 1.3302929308987403, |
| "pretrain_checkpoint_step": 2200, |
| "pretrain_hours": 10.380536459482197, |
| "pretrain_run": "pretrain-v1", |
| "pretrain_tokens": 1153433600, |
| "pretrain_val_loss_ema": 3.06275936961174, |
| "reasoning": true, |
| "rms_norm_eps": 1e-06, |
| "rope_theta": 1000000.0, |
| "sft_assistant_tokens": 5388440922, |
| "sft_checkpoint_step": 12800, |
| "sft_hours": 11.803919963316359, |
| "sft_run": "sft-v1", |
| "sft_val_loss_ema": 0.8266990829440273, |
| "sft_val_ppl_ema": 2.2857611656454533, |
| "shared_layer_indices": [], |
| "situ_gate_cap": 4.0, |
| "situ_up_cap": 25.0, |
| "stage": "preference", |
| "tie_word_embeddings": true, |
| "tool_calls": "hermes_json", |
| "torch_dtype": "bfloat16", |
| "training_sequence_length": 32768, |
| "transformers_version": "4.45.0", |
| "unk_token_id": 3, |
| "use_block_attn_res": true, |
| "vocab_size": 65536, |
| "weights": "ema" |
| } |
|
|
