{ "architectures": [ "RetNetForCausalLM" ], "checkpoint_layers": -1, "d_model": 768, "dropout": 0.1, "dtype": "float32", "embed_scale": null, "ffn_dim": 1536, "growth_old_dim": null, "model_type": "retnet_hf", "norm_eps": 1e-05, "num_heads": 12, "num_layers": 12, "prompt_loss_weight": 0.1, "tie_weights": true, "transformers_version": "4.57.6", "use_gla": false, "use_growth_ln": false, "use_swiglu": false, "value_dim": 1536, "vocab_size": 20000, "auto_map": { "AutoConfig": "modeling_retnet.RetNetHFConfig", "AutoModelForCausalLM": "modeling_retnet.RetNetForCausalLM" }, "mode": "parallel" }