RetNet-100M / config.json
Ayaka157's picture
Upload RetNet-100M
8329698 verified
Raw History Blame Contribute Delete
682 Bytes
{
"architectures": [
"RetNetForCausalLM"
],
"checkpoint_layers": -1,
"d_model": 768,
"dropout": 0.1,
"dtype": "float32",
"embed_scale": null,
"ffn_dim": 1536,
"growth_old_dim": null,
"model_type": "retnet_hf",
"norm_eps": 1e-05,
"num_heads": 12,
"num_layers": 12,
"prompt_loss_weight": 0.1,
"tie_weights": true,
"transformers_version": "4.57.6",
"use_gla": false,
"use_growth_ln": false,
"use_swiglu": false,
"value_dim": 1536,
"vocab_size": 20000,
"auto_map": {
"AutoConfig": "modeling_retnet.RetNetHFConfig",
"AutoModelForCausalLM": "modeling_retnet.RetNetForCausalLM"
},
"mode": "parallel"
}