File size: 687 Bytes
264f262 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 | {
"architectures": [
"PhaGenModel"
],
"attn_dropout": 0.0,
"attn_implementation": "flash_attention_2",
"auto_map": {
"AutoConfig": "configuration_phagen.PhaGenConfig",
"AutoModel": "modeling_phagen.PhaGenModel",
"AutoModelForCausalLM": "modeling_phagen.PhaGenModel"
},
"depth": [
8,
8,
8
],
"dim": [
512,
256,
196
],
"dim_head": 64,
"ff_dropout": 0.0,
"ff_mult": 4,
"heads": 8,
"max_seq_len": [
128,
64,
16
],
"model_type": "phagen",
"num_tokens": 9,
"pad_token_id": 7,
"pos_emb": false,
"rel_pos": true,
"tie_word_embeddings": false,
"transformers_version": "4.54.1",
"vocab_size": 9
}
|