| { | |
| "architectures": [ | |
| "PhaGenModel" | |
| ], | |
| "attn_dropout": 0.0, | |
| "attn_implementation": "flash_attention_2", | |
| "auto_map": { | |
| "AutoConfig": "configuration_phagen.PhaGenConfig", | |
| "AutoModel": "modeling_phagen.PhaGenModel", | |
| "AutoModelForCausalLM": "modeling_phagen.PhaGenModel" | |
| }, | |
| "depth": [ | |
| 8, | |
| 8, | |
| 8 | |
| ], | |
| "dim": [ | |
| 512, | |
| 256, | |
| 196 | |
| ], | |
| "dim_head": 64, | |
| "ff_dropout": 0.0, | |
| "ff_mult": 4, | |
| "heads": 8, | |
| "max_seq_len": [ | |
| 128, | |
| 64, | |
| 16 | |
| ], | |
| "model_type": "phagen", | |
| "num_tokens": 9, | |
| "pad_token_id": 7, | |
| "pos_emb": false, | |
| "rel_pos": true, | |
| "tie_word_embeddings": false, | |
| "transformers_version": "4.54.1", | |
| "vocab_size": 9 | |
| } | |