| { |
| "architectures": [ |
| "OleanderForCausalLM" |
| ], |
| "model_type": "oleander", |
| "vocab_size": 32770, |
| "hidden_size": 576, |
| "num_hidden_layers": 28, |
| "oleander_layer_types": [ |
| "gdn2", |
| "gdn2", |
| "gdn2", |
| "gqa", |
| "gdn2", |
| "gdn2", |
| "gdn2", |
| "gqa", |
| "gdn2", |
| "gdn2", |
| "gdn2", |
| "gqa", |
| "gdn2", |
| "gdn2", |
| "gdn2", |
| "gqa", |
| "gdn2", |
| "gdn2", |
| "gdn2", |
| "gqa", |
| "gdn2", |
| "gdn2", |
| "gdn2", |
| "gqa", |
| "gdn2", |
| "gdn2", |
| "gdn2", |
| "gqa" |
| ], |
| "layer_types": [ |
| "linear_attention", |
| "linear_attention", |
| "linear_attention", |
| "full_attention", |
| "linear_attention", |
| "linear_attention", |
| "linear_attention", |
| "full_attention", |
| "linear_attention", |
| "linear_attention", |
| "linear_attention", |
| "full_attention", |
| "linear_attention", |
| "linear_attention", |
| "linear_attention", |
| "full_attention", |
| "linear_attention", |
| "linear_attention", |
| "linear_attention", |
| "full_attention", |
| "linear_attention", |
| "linear_attention", |
| "linear_attention", |
| "full_attention", |
| "linear_attention", |
| "linear_attention", |
| "linear_attention", |
| "full_attention" |
| ], |
| "intermediate_size": 1600, |
| "max_position_embeddings": 1024, |
| "rms_norm_eps": 1e-06, |
| "num_attention_heads": 9, |
| "num_key_value_heads": 3, |
| "head_dim": 64, |
| "tie_word_embeddings": true, |
| "use_cache": false, |
| "attn_implementation": "sdpa", |
| "oleander": { |
| "arch_name": "OleanderForCausalLM", |
| "vocab_size": 32770, |
| "hidden_size": 576, |
| "num_groups": 7, |
| "group_pattern": [ |
| "gdn2", |
| "gdn2", |
| "gdn2", |
| "gqa" |
| ], |
| "mlp_hidden": 1600, |
| "max_seq_len": 1024, |
| "norm_eps": 1e-06, |
| "init_std": 0.02, |
| "tie_embeddings": true, |
| "attn_res": true, |
| "residual_dtype": "bfloat16", |
| "gdn2": { |
| "heads": 9, |
| "head_dim": 64, |
| "conv_size": 4, |
| "norm_eps": 1e-05, |
| "allow_neg_eigval": false, |
| "gate_in_kernel": false |
| }, |
| "gqa": { |
| "q_heads": 9, |
| "kv_heads": 3, |
| "head_dim": 64, |
| "rotary_dim": 32, |
| "rope_theta": 100000.0, |
| "qk_norm": true, |
| "value_residual": true |
| } |
| }, |
| "model_name": "Oleander-150M", |
| "training": { |
| "step": 6500, |
| "kind": "export", |
| "max_steps": 28820, |
| "percent_complete": 22.554, |
| "tokens_seen": 20786970624, |
| "compute_tokens_seen": 3464495104 |
| }, |
| "transformers_version": "5.14.1", |
| "torch_dtype": "float32" |
| } |
