| { |
| "add_sinusoidal_embeddings": true, |
| "architectures": [ |
| "TAMELM" |
| ], |
| "attention_backend": "flex", |
| "attention_top_k": 2, |
| "auto_map": { |
| "AutoConfig": "modeling_tamelm.TAMEConfig", |
| "AutoModelForCausalLM": "modeling_tamelm.TAMELM" |
| }, |
| "balance_coef": 0.002, |
| "bos_token_id": 1, |
| "checkpoint_backbone": true, |
| "depth_dim": 256, |
| "depth_gate_init": 0.01, |
| "depth_heads": 8, |
| "depth_mode": "attention", |
| "depth_slots": 4, |
| "dtype": "float32", |
| "embed_dim": 768, |
| "eos_token_id": 2, |
| "expert_backend": "grouped", |
| "expert_bank_top_k": 2, |
| "expert_banks": 2, |
| "expert_hidden_dim": 1536, |
| "expert_top_k": 2, |
| "graph_recent": 128, |
| "graph_top_r": 2, |
| "init_scheme": "v1", |
| "local_window": 256, |
| "loss_chunk_size": 128, |
| "max_seq_len": 8192, |
| "model_type": "tamelm_two_axis", |
| "n_kv_heads": 2, |
| "n_q_heads": 32, |
| "num_experts": 16, |
| "num_layers": 12, |
| "pad_token_id": 0, |
| "reasoning_chunk_size": 256, |
| "reasoning_direct_read": true, |
| "reasoning_gate_init": 0.1, |
| "reasoning_heads": 4, |
| "reasoning_initial_fold": true, |
| "reasoning_mode": "fold_unfold", |
| "reasoning_position_bias": "relative", |
| "reasoning_revisit_mode": "full", |
| "reasoning_slots": 2, |
| "reasoning_steps": 3, |
| "rope_type": "warped", |
| "router_init_std": 0.01, |
| "router_z_loss_coef": 0.001, |
| "strided_every": 4, |
| "strided_recent": 128, |
| "tie_word_embeddings": true, |
| "tile": 128, |
| "tokenizer_sha256": "0d93d8f943b1a6627d62395c2fd583894cfe577b8aca1ca946495f7abba50958", |
| "transformers_version": "5.5.0", |
| "use_cache": false, |
| "vocab_size": 49152 |
| } |
|
|
