File size: 917 Bytes
3b2d368 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 | # === Model Identity ===
config_target: lmr.models.transformer_bert.TransformerConfig
model_target: lmr.models.transformer_bert.TransformerForMaskedLM
model_name: transformer
# === Model Architecture ===
vocab_size: 50257
hidden_size: 1024
embedding_size: null # if null, defaults to hidden_size in code
num_hidden_layers: 24
num_attention_heads: 16
intermediate_size: 4096
max_position_embeddings: 1024
tie_word_embeddings: true
use_causal_attention: false
# === Initialization ===
initializer_range: 0.02
# === Misc ===
use_cache: false
self.truncate_activation_size: false
# === Tokenizer ===
tokenizer_type: bert-base-uncased # Hugging Face tokenizer name or path
bos_token_id: null # if null, defaults to tokenizer bos_token_id
eos_token_id: null # if null, defaults to tokenizer eos_token_id
pad_token_id: null # if null, defaults to tokenizer pad_token_id |