Hugging Face's logo

GTKING
/
ZFusionAI_Base

Safetensors
llama4_text
Model card Files
xet
 Community
ZFusionAI_Base
File size: 2,009 Bytes
de2a7bf
 
ea82f09
de2a7bf
ea82f09
 
 
 
 
 
efc8fe8
ea82f09
 
 
 
 
de2a7bf
ea82f09
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
67fb512
ea82f09
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
86c32d2
ea82f09
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
efc8fe8
de2a7bf
ea82f09
 
67fb512
ea82f09
67fb512
 
 
 
efc8fe8
ea82f09
 
efc8fe8
86c32d2
4ce7159
efc8fe8
ea82f09
de2a7bf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
{
  "architectures": [
    "Llama4ForCausalLM"
  ],
  "attention_bias": false,
  "attention_chunk_size": 8192,
  "attention_dropout": 0.0,
  "attn_scale": 0.1,
  "attn_temperature_tuning": true,
  "bos_token_id": 1,
  "dtype": "float32",
  "eos_token_id": 2,
  "floor_scale": 8192,
  "head_dim": 128,
  "hidden_act": "silu",
  "hidden_size": 384,
  "initializer_range": 0.02,
  "interleave_moe_layer_step": 1,
  "intermediate_size": 768,
  "intermediate_size_mlp": 1024,
  "layer_types": [
    "chunked_attention",
    "chunked_attention",
    "chunked_attention",
    "full_attention",
    "chunked_attention",
    "chunked_attention",
    "chunked_attention",
    "full_attention",
    "chunked_attention",
    "chunked_attention",
    "chunked_attention",
    "full_attention",
    "chunked_attention",
    "chunked_attention",
    "chunked_attention",
    "full_attention"
  ],
  "max_position_embeddings": 1024,
  "model_type": "llama4_text",
  "moe_layers": [
    0,
    1,
    2,
    3,
    4,
    5,
    6,
    7,
    8,
    9,
    10,
    11,
    12,
    13,
    14,
    15
  ],
  "no_rope_layer_interval": 4,
  "no_rope_layers": [
    1,
    1,
    1,
    0,
    1,
    1,
    1,
    0,
    1,
    1,
    1,
    0,
    1,
    1,
    1,
    0,
    1,
    1,
    1,
    0,
    1,
    1,
    1,
    0,
    1,
    1,
    1,
    0,
    1,
    1,
    1,
    0,
    1,
    1,
    1,
    0,
    1,
    1,
    1,
    0,
    1,
    1,
    1,
    0,
    1,
    1,
    1,
    0
  ],
  "num_attention_heads": 6,
  "num_experts_per_tok": 1,
  "num_hidden_layers": 16,
  "num_key_value_heads": 2,
  "num_local_experts": 4,
  "output_router_logits": false,
  "pad_token_id": null,
  "rms_norm_eps": 1e-05,
  "rope_parameters": {
    "rope_theta": 500000.0,
    "rope_type": "default"
  },
  "router_aux_loss_coef": 0.001,
  "router_jitter_noise": 0.0,
  "sliding_window": 512,
  "tie_word_embeddings": false,
  "transformers_version": "5.10.2",
  "use_cache": false,
  "use_qk_norm": true,
  "vocab_size": 202048
}