attention_logit_softcapping: null attention_scores_scalar: null attn_bias: false bias: false block_size: 2048 final_logit_softcapping: null first_k_dense_replace: null gelu_approximate: none head_size: 64 hf_config: name: TinyLlama-1.1B-intermediate-step-1431k-3T org: TinyLlama intermediate_size: 5632 latent_attention: null lm_head_bias: false mlp_class_name: LLaMAMLP moe_intermediate_size: null n_embd: 2048 n_expert: 0 n_expert_groups: null n_expert_per_token: 0 n_head: 32 n_layer: 22 n_query_groups: 4 n_shared_expert: null n_topk_groups: null n_topk_scores_per_group: null name: tiny-llama-1.1b norm_1: true norm_2: true norm_class_name: RMSNorm norm_eps: 1.0e-05 norm_qk: false norm_qk_type: default norm_topk_prob: false padded_vocab_size: 32000 padding_multiple: 64 parallel_residual: false post_attention_norm: false post_mlp_norm: false rope_adjustments: null rope_base: 10000 rope_condense_ratio: 1 rope_indices: null rope_interleave: false rope_local_base_freq: null rotary_percentage: 1.0 routed_scaling_factor: 1.0 scale_embeddings: false shared_attention_norm: false sliding_window_indices: null sliding_window_size: null vocab_size: 32000