| # === Model Identity === | |
| config_target: lmr.models.fst.FSTConfig | |
| model_target: lmr.models.fst.FSTForCausalLM | |
| model_name: fst | |
| # === Model Architecture === | |
| vocab_size: 50257 | |
| hidden_size: 1024 | |
| embedding_size: null # if null, defaults to hidden_size | |
| num_hidden_layers: 24 | |
| num_attention_heads: 16 | |
| intermediate_size: 4096 | |
| max_position_embeddings: 1024 | |
| tie_word_embeddings: true | |
| use_causal_attention: true | |
| share_f_and_phi_embedding: true | |
| # === FST Specific Overrides (all default to non-fst values) === | |
| hidden_size_f: null | |
| hidden_size_phi: null | |
| embedding_size_f: null | |
| embedding_size_phi: null | |
| num_attention_heads_f: null | |
| num_attention_heads_phi: null | |
| intermediate_size_f: null | |
| intermediate_size_phi: null | |
| # === FST Experimental Settings (Not yet implemented) === | |
| # num_phi_updates: 1 # Number of phi updates in a single phi layer | |
| # num_f_updates: 1 # Number of f updates in a single f layer | |
| # duplicate_phi_updates: false # Whether the phi updates within a layer should share weights (only used when num_phi_updates > 1) | |
| # duplicate_f_updates: false # Whether to duplicate f updates within a layer (only used when num_f_updates > 1) | |
| # === Initialization === | |
| initializer_range: 0.02 | |
| # === Misc === | |
| use_cache: false | |
| self.truncate_activation_size: false | |
| # === Tokenizer === | |
| tokenizer_type: gpt2 # Hugging Face tokenizer name or path | |
| bos_token_id: null # if null, defaults to tokenizer bos_token_id | |
| eos_token_id: null # if null, defaults to tokenizer eos_token_id | |
| pad_token_id: null # if null, defaults to tokenizer pad_token_id |