config/model/_base_transformer_bert.yaml

# === Model Identity ===
config_target: lmr.models.transformer_bert.TransformerConfig
model_target: lmr.models.transformer_bert.TransformerForMaskedLM
model_name: transformer

# === Model Architecture ===
vocab_size: 50257
hidden_size: 1024
embedding_size: null          # if null, defaults to hidden_size in code
num_hidden_layers: 24
num_attention_heads: 16
intermediate_size: 4096
max_position_embeddings: 1024
tie_word_embeddings: true
use_causal_attention: false

# === Initialization ===
initializer_range: 0.02

# === Misc ===
use_cache: false
self.truncate_activation_size: false

# === Tokenizer ===
tokenizer_type: bert-base-uncased          # Hugging Face tokenizer name or path
bos_token_id: null            # if null, defaults to tokenizer bos_token_id
eos_token_id: null            # if null, defaults to tokenizer eos_token_id
pad_token_id: null            # if null, defaults to tokenizer pad_token_id