File size: 1,559 Bytes
634a34a
 
 
 
 
 
 
7ac4d9f
634a34a
0626191
634a34a
 
 
a4f15a9
634a34a
a4f15a9
634a34a
0626191
634a34a
 
0626191
634a34a
 
 
 
d55cabf
634a34a
 
 
0626191
634a34a
 
 
 
 
d572c50
 
 
 
 
a4f15a9
 
d572c50
 
634a34a
 
0626191
 
 
 
 
 
 
 
 
 
c40ec2c
634a34a
0626191
634a34a
 
0626191
d55cabf
634a34a
 
0626191
634a34a
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
{
  "architectures": [
    "DeepseekV3ForCausalLM"
  ],
  "attention_bias": false,
  "attention_dropout": 0.0,
  "bos_token_id": 0,
  "dtype": "bfloat16",
  "eos_token_id": 1,
  "ep_size": 1,
  "first_k_dense_replace": 3,
  "head_dim": 64,
  "hidden_act": "silu",
  "hidden_size": 32,
  "initializer_range": 0.02,
  "intermediate_size": 32,
  "kv_lora_rank": 512,
  "max_position_embeddings": 163840,
  "model_type": "deepseek_v3",
  "moe_intermediate_size": 2048,
  "moe_layer_freq": 1,
  "n_group": 8,
  "n_routed_experts": 256,
  "n_shared_experts": 1,
  "norm_topk_prob": true,
  "num_attention_heads": 2,
  "num_experts_per_tok": 8,
  "num_hidden_layers": 2,
  "num_key_value_heads": 2,
  "num_nextn_predict_layers": 1,
  "pretraining_tp": 1,
  "q_lora_rank": 1536,
  "qk_head_dim": 192,
  "qk_nope_head_dim": 128,
  "qk_rope_head_dim": 64,
  "quantization_config": {
    "activation_scheme": "dynamic",
    "fmt": "e4m3",
    "quant_method": "fp8",
    "weight_block_size": [
      32,
      32
    ]
  },
  "rms_norm_eps": 1e-06,
  "rope_interleave": true,
  "rope_scaling": {
    "beta_fast": 32.0,
    "beta_slow": 1.0,
    "factor": 40.0,
    "mscale": 1.0,
    "mscale_all_dim": 1.0,
    "original_max_position_embeddings": 4096,
    "rope_type": "yarn",
    "type": "yarn"
  },
  "rope_theta": 10000.0,
  "routed_scaling_factor": 2.5,
  "scoring_func": "sigmoid",
  "tie_word_embeddings": false,
  "topk_group": 4,
  "topk_method": "noaux_tc",
  "transformers_version": "4.56.2",
  "use_cache": true,
  "v_head_dim": 128,
  "vocab_size": 129280
}