| from transformers import PretrainedConfig
|
| from transformers import AutoTokenizer
|
|
|
| class Config(PretrainedConfig):
|
| model_type = "tinyLLM"
|
|
|
| def __init__(self,
|
| eos_token_id: int | None = 73440,
|
| pad_token_id: int | None = 73440,
|
| use_tcn = False,
|
| tcn_layers = [],
|
| num_query_tokens=128,
|
| vocab_size=73448,
|
| hidden_size=1280,
|
| num_hidden_layers=24,
|
| num_attention_heads=20,
|
| num_key_value_heads=4,
|
| max_position_embeddings=8192,
|
| RoPE_base=1e4,
|
| rope_type="yarn",
|
| dropout=0.00,
|
| rms_norm_eps=1e-5,
|
| qkrms_norm_eps=1e-5,
|
| use_qk_norm=True,
|
| kv_cache_dtype="auto",
|
| use_moe=False,
|
| adapter_type="none",
|
| use_swiGLU=True,
|
| bos_token_id=None,
|
| use_ssm=False,
|
| ssm_layers=[],
|
| embeddingdropout=0.0,
|
| learnable_temp=True,
|
| use_affine=True,
|
| use_sampleattion=False,
|
| eso_loss_radio=1.0,
|
| train_maxlength=8192,
|
| drop_high_loss=0.0,
|
| mlp_ratio_front: float = 3.5,
|
| mlp_ratio_mid: float = 4.0,
|
| mlp_ratio_back: float = 4.5,
|
| mlp_mid_start: int | None = 8,
|
| mlp_back_start: int | None = 16,
|
| mlp_ratio_overrides: dict | None = {21: 5.0,22: 5.0, 23: 5.0},
|
| mlp_ratio=4.5,
|
| ignore_index=-100,
|
| loss_reduction: str = "token_mean",
|
| sample_mean_alpha: float = 0.75,
|
| drop_path=0.00,
|
| residual_dropout=0.00,
|
| moe_layers=[],
|
| num_expert=4,
|
| moe_use_detach=False,
|
| moe_cap_factor=1.5,
|
| moe_aux_weight=5.0,
|
| use_checkpoint=False,
|
| checkpoint_use_reentrant=False,
|
| use_adaptive_softmax: bool = True,
|
| adaptive_cutoffs = [20000, 60000],
|
| vision_dropout=0.0,
|
| adaptive_div: float = 4.0,
|
| adaptive_calibrate_every: int = 200,
|
| use_vision: bool = False,
|
| vision_feature_dim: int | None =1024,
|
| max_vision_tokens: int = 0,
|
| vision_mlp_ratio: float = 4.0,
|
| vision_use_swiglu: bool = True,
|
|
|
| **kwargs):
|
| super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id,pad_token_id=pad_token_id,**kwargs)
|
| self.vocab_size = vocab_size
|
| self.hidden_size = hidden_size
|
| self.num_hidden_layers = num_hidden_layers
|
| self.num_attention_heads = num_attention_heads
|
| self.num_key_value_heads = num_key_value_heads
|
| self.max_position_embeddings = max_position_embeddings
|
| self.RoPE_base=RoPE_base
|
| self.rope_type=rope_type
|
| self.dropout = dropout
|
| self.train_maxlength=train_maxlength
|
| self.rms_norm_eps = rms_norm_eps
|
| self.qkrms_norm_eps = qkrms_norm_eps
|
| self.use_qk_norm = use_qk_norm
|
| self.kv_cache_dtype = kv_cache_dtype
|
| self.use_moe = use_moe
|
| self.adapter_type = adapter_type
|
| self.bos_token_id = bos_token_id
|
| self.eos_token_id = eos_token_id
|
| self.use_ssm=use_ssm
|
| self.ssm_layers=ssm_layers
|
| self.embeddingdropout=embeddingdropout
|
| self.learnable_temp= learnable_temp
|
| self.use_affine=use_affine
|
| self.use_sampleatt=use_sampleattion
|
| self.use_swiGLU=use_swiGLU
|
| self.mlp_ratio=mlp_ratio
|
| self.ignore_index=ignore_index
|
| self.loss_reduction = str(loss_reduction)
|
| self.sample_mean_alpha = float(sample_mean_alpha)
|
| if self.loss_reduction not in {"token_mean", "sample_mean", "hybrid"}:
|
| raise ValueError(
|
| "loss_reduction must be one of: token_mean, sample_mean, hybrid"
|
| )
|
| if not 0.0 <= self.sample_mean_alpha <= 1.0:
|
| raise ValueError("sample_mean_alpha must be within [0, 1]")
|
| self.drop_path=drop_path
|
| self.residual_dropout=residual_dropout
|
| self.use_moe=use_moe
|
| self.moe_layers=moe_layers
|
| self.num_expert=num_expert
|
| self.moe_use_detach=moe_use_detach
|
| self.moe_cap_factor=moe_cap_factor
|
| self.moe_aux_weight=moe_aux_weight
|
| self.use_checkpoint = use_checkpoint
|
| self.checkpoint_use_reentrant = checkpoint_use_reentrant
|
| self.use_adaptive_softmax=use_adaptive_softmax
|
| self.adaptive_cutoffs= adaptive_cutoffs
|
| self.adaptive_div=adaptive_div
|
| self.adaptive_calibrate_every=adaptive_calibrate_every
|
| self.use_tcn=use_tcn
|
| self.tcn_layers=tcn_layers
|
| self.mlp_ratio = mlp_ratio
|
|
|
| self.drop_high_loss=drop_high_loss
|
| self.mlp_ratio_front = float(mlp_ratio_front)
|
| self.mlp_ratio_mid = float(mlp_ratio_mid)
|
| self.mlp_ratio_back = float(mlp_ratio_back)
|
| self.mlp_mid_start = mlp_mid_start
|
| self.mlp_back_start = mlp_back_start
|
| self.mlp_ratio_overrides = mlp_ratio_overrides or {}
|
| self.eso_loss_radio=eso_loss_radio
|
| self.use_vision = use_vision
|
| self.vision_feature_dim = vision_feature_dim
|
| self.max_vision_tokens = max_vision_tokens
|
| self.vision_mlp_ratio = vision_mlp_ratio
|
| self.vision_use_swiglu = vision_use_swiglu
|
| self.vision_dropout = vision_dropout if vision_dropout is not None else dropout
|
| self.num_query_tokens=num_query_tokens
|
|
|
