File size: 1,632 Bytes
3b2d368 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 | # === Model Identity ===
config_target: lmr.models.fst.FSTConfig
model_target: lmr.models.fst.FSTForSequenceClassification
model_name: fst
# === Model Architecture ===
vocab_size: 50257
hidden_size: 1024
embedding_size: null # if null, defaults to hidden_size
num_hidden_layers: 24
num_attention_heads: 16
intermediate_size: 4096
max_position_embeddings: 1024
tie_word_embeddings: true
use_causal_attention: true
share_f_and_phi_embedding: true
# === FST Specific Overrides (all default to non-fst values) ===
hidden_size_f: null
hidden_size_phi: null
embedding_size_f: null
embedding_size_phi: null
num_attention_heads_f: null
num_attention_heads_phi: null
intermediate_size_f: null
intermediate_size_phi: null
# === FST Experimental Settings (Not yet implemented) ===
# num_phi_updates: 1 # Number of phi updates in a single phi layer
# num_f_updates: 1 # Number of f updates in a single f layer
# duplicate_phi_updates: false # Whether the phi updates within a layer should share weights (only used when num_phi_updates > 1)
# duplicate_f_updates: false # Whether to duplicate f updates within a layer (only used when num_f_updates > 1)
# === Initialization ===
initializer_range: 0.02
# === Misc ===
use_cache: false
self.truncate_activation_size: false
# === Tokenizer ===
tokenizer_type: gpt2 # Hugging Face tokenizer name or path
bos_token_id: null # if null, defaults to tokenizer bos_token_id
eos_token_id: null # if null, defaults to tokenizer eos_token_id
pad_token_id: null # if null, defaults to tokenizer pad_token_id |