Antonio0101 commited on
Commit
508c8aa
·
verified ·
1 Parent(s): 74801e4

Clean repository before upload

Browse files
Files changed (1) hide show
  1. config.json +0 -138
config.json DELETED
@@ -1,138 +0,0 @@
1
- {
2
- "allow_neg_eigval": false,
3
- "architectures": [
4
- "QuasarForCausalLM"
5
- ],
6
- "attn_mode": "chunk",
7
- "bigmac_r": 0.25,
8
- "bos_token_id": 2,
9
- "conv_bias": false,
10
- "conv_size": 4,
11
- "d_ff": 11008,
12
- "d_model": 4096,
13
- "dense_input_layers": 4,
14
- "dropout": 0.0,
15
- "dtype": "bfloat16",
16
- "eos_token_id": 1,
17
- "expand_k": 0.5,
18
- "expand_v": 1.0,
19
- "fuse_cross_entropy": true,
20
- "fuse_norm": true,
21
- "fuse_swiglu": true,
22
- "gated_layers": 2,
23
- "gla_mode": "chunk",
24
- "gradient_checkpointing": false,
25
- "head_dim": 128,
26
- "hidden_act": "silu",
27
- "hidden_ratio": 4,
28
- "hidden_size": 4096,
29
- "hybrid_layer_types": [
30
- "quasar",
31
- "quasar",
32
- "quasar",
33
- "quasar",
34
- "gla",
35
- "gla",
36
- "quasar",
37
- "quasar",
38
- "quasar",
39
- "quasar",
40
- "gla",
41
- "gla",
42
- "quasar",
43
- "quasar",
44
- "quasar",
45
- "quasar",
46
- "gla",
47
- "gla",
48
- "quasar",
49
- "quasar",
50
- "quasar",
51
- "quasar",
52
- "gla",
53
- "gla",
54
- "quasar",
55
- "quasar",
56
- "quasar",
57
- "quasar",
58
- "gla",
59
- "gla",
60
- "quasar",
61
- "quasar"
62
- ],
63
- "initializer_range": 0.02,
64
- "intermediate_size": 11008,
65
- "layer_types": [
66
- "linear_attention",
67
- "linear_attention",
68
- "linear_attention",
69
- "linear_attention",
70
- "linear_attention",
71
- "linear_attention",
72
- "linear_attention",
73
- "linear_attention",
74
- "linear_attention",
75
- "linear_attention",
76
- "linear_attention",
77
- "linear_attention",
78
- "linear_attention",
79
- "linear_attention",
80
- "linear_attention",
81
- "linear_attention",
82
- "linear_attention",
83
- "linear_attention",
84
- "linear_attention",
85
- "linear_attention",
86
- "linear_attention",
87
- "linear_attention",
88
- "linear_attention",
89
- "linear_attention",
90
- "linear_attention",
91
- "linear_attention",
92
- "linear_attention",
93
- "linear_attention",
94
- "linear_attention",
95
- "linear_attention",
96
- "linear_attention",
97
- "linear_attention"
98
- ],
99
- "looped_injection_init": 0.1,
100
- "max_position_embeddings": 16384,
101
- "max_seq_len": 16384,
102
- "memory_dim": 128,
103
- "memory_slots": 128,
104
- "model_type": "quasar",
105
- "moe_aux_loss_coeff": 0.0001,
106
- "moe_type": "bigmac",
107
- "moe_z_loss_coeff": 0.0001,
108
- "n_heads": 32,
109
- "n_layers": 32,
110
- "norm_eps": 1e-06,
111
- "num_attention_heads": 32,
112
- "num_heads": 32,
113
- "num_hidden_layers": 32,
114
- "num_key_value_heads": 32,
115
- "num_loops": 1,
116
- "num_routed_experts": 56,
117
- "num_shared_experts": 1,
118
- "num_v_heads": null,
119
- "pad_token_id": 0,
120
- "quasar_layers": 4,
121
- "residual_scale": 0.1,
122
- "rms_norm_eps": 1e-06,
123
- "rope_theta": 1000000.0,
124
- "routed_expert_size": 1024,
125
- "shared_expert_size": 2048,
126
- "smebu_beta": 0.5,
127
- "smebu_kappa": 2.0,
128
- "smebu_lambda": 0.002,
129
- "tie_word_embeddings": true,
130
- "top_k": 8,
131
- "transformers_version": "5.7.0",
132
- "use_cache": false,
133
- "use_gla_first": false,
134
- "use_l2warp": false,
135
- "use_looped_injection": false,
136
- "use_short_conv": true,
137
- "vocab_size": 262144
138
- }