Prelude-1B-Base / model_config.yaml
Li
Upload exact final 20B-token base checkpoint
a0428b1 verified
Raw History Blame Contribute Delete
1.16 kB
attention_logit_softcapping: null
attention_scores_scalar: null
attn_bias: false
bias: false
block_size: 2048
final_logit_softcapping: null
first_k_dense_replace: null
gelu_approximate: none
head_size: 64
hf_config:
name: TinyLlama-1.1B-intermediate-step-1431k-3T
org: TinyLlama
intermediate_size: 5632
latent_attention: null
lm_head_bias: false
mlp_class_name: LLaMAMLP
moe_intermediate_size: null
n_embd: 2048
n_expert: 0
n_expert_groups: null
n_expert_per_token: 0
n_head: 32
n_layer: 22
n_query_groups: 4
n_shared_expert: null
n_topk_groups: null
n_topk_scores_per_group: null
name: tiny-llama-1.1b
norm_1: true
norm_2: true
norm_class_name: RMSNorm
norm_eps: 1.0e-05
norm_qk: false
norm_qk_type: default
norm_topk_prob: false
padded_vocab_size: 32000
padding_multiple: 64
parallel_residual: false
post_attention_norm: false
post_mlp_norm: false
rope_adjustments: null
rope_base: 10000
rope_condense_ratio: 1
rope_indices: null
rope_interleave: false
rope_local_base_freq: null
rotary_percentage: 1.0
routed_scaling_factor: 1.0
scale_embeddings: false
shared_attention_norm: false
sliding_window_indices: null
sliding_window_size: null
vocab_size: 32000