Download config.json from abe123/babylm-dat-strict-initial: direct link, hf CLI and curl.
- Browser
- Download file 1.96 kB
-
https://huggingface.co/abe123/babylm-dat-strict-initial/resolve/main/config.json
- Command line
-
hf download hf://abe123/babylm-dat-strict-initial/config.json
-
curl -L -o config.json https://huggingface.co/abe123/babylm-dat-strict-initial/resolve/main/config.json
1.96 kB
| { | |
| "architectures": [ | |
| "DatForCausalLM" | |
| ], | |
| "auto_map": { | |
| "AutoConfig": "configuration_dat.DatConfig", | |
| "AutoModel": "modeling_dat.DatModel", | |
| "AutoModelForCausalLM": "modeling_dat.DatForCausalLM", | |
| "AutoModelForMaskedLM": "modeling_dat.DatForMaskedLM", | |
| "AutoModelForSequenceClassification": "modeling_dat.DatForSequenceClassification" | |
| }, | |
| "bos_token_id": 1, | |
| "dff_factor": 4, | |
| "dropout": 0.1, | |
| "dtype": "float32", | |
| "eos_token_id": 2, | |
| "ffn_activation": "swiglu", | |
| "ffn_hidden_dim_mode": "dff_factor", | |
| "hidden_dim": 768, | |
| "init_range": 0.15, | |
| "init_scheme": "normal_0_02_scaled_projection", | |
| "max_position_embeddings": 513, | |
| "max_rel_pos": 512, | |
| "max_seq_len": 513, | |
| "mlm_head_enabled": false, | |
| "model_type": "dat", | |
| "n_heads_ra": 3, | |
| "n_heads_sa": 9, | |
| "n_layers": 12, | |
| "n_symbols": null, | |
| "norm_first": true, | |
| "norm_type": "layernorm", | |
| "pad_token_id": 3, | |
| "pe_type": "rope", | |
| "positional_symbols_sinusoidal": false, | |
| "ra_n_relations": null, | |
| "ra_rel_activation": "identity", | |
| "ra_symmetric_rels": false, | |
| "ra_type": "rca", | |
| "relative_symbols_rope": true, | |
| "relsymbolic_dropout": 0.0, | |
| "relsymbolic_include_self": false, | |
| "relsymbolic_neighborhood_size": 2, | |
| "relsymbolic_normalize_rels": true, | |
| "relsymbolic_rel_n_heads": 4, | |
| "relsymbolic_rel_scale": null, | |
| "relsymbolic_symbolic_attn_n_heads": 4, | |
| "relsymbolic_symbolic_attn_scale": null, | |
| "relsymbolic_trainable_symbols": true, | |
| "relsymbolic_use_bias": false, | |
| "rope_theta": 10000.0, | |
| "segment_boundary_token_id": null, | |
| "sequence_boundary_policy": "eos_document", | |
| "share_attn_params": false, | |
| "shared_symbol_retriever": true, | |
| "symbol_dim": null, | |
| "symbol_retrieval": "relative", | |
| "symbolic_attn_n_heads": null, | |
| "symbolic_use_bias": false, | |
| "tie_lm_head": true, | |
| "tie_word_embeddings": true, | |
| "transformers_version": "5.12.0", | |
| "use_bias_ffn": true, | |
| "use_bias_out": false, | |
| "use_bias_qkv": false, | |
| "vocab_size": 16384 | |
| } | |