Hugging Face's logo

redmingwei
/
HeteroMoEProjSpace-v2

Feature Extraction
Transformers
Safetensors
hetero_moe_projspace
mixture-of-experts
heterogeneous
dimension-projection-space
ffn-fusion
custom_code
Model card Files
xet
 Community
redmingwei commited on
Commit
22972b9
·
verified ·
1 Parent(s): 2038073

v67.0: report.json

Browse files
Files changed (1) hide show
  1. checkpoints/v67.0/report.json +226 -0
checkpoints/v67.0/report.json ADDED
@@ -0,0 +1,226 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "hist": [
3
+ {
4
+ "step": 0,
5
+ "ce": 1.1130203008651733,
6
+ "train_acc": 0.375,
7
+ "val_acc": 0.33266733266733267,
8
+ "bal": 8.399999618530273,
9
+ "load": [
10
+ 1.0,
11
+ 1.0,
12
+ 1.0,
13
+ 1.0
14
+ ],
15
+ "ent": 0.9433109093877855,
16
+ "router_wnorm": 0.7730511426925659,
17
+ "router_gnorm": 0.5350815057754517,
18
+ "slot_ent": 0.8563924431800842,
19
+ "slot_ema": {
20
+ "Q": 0.25212,
21
+ "K": 0.24952,
22
+ "V": 0.24881,
23
+ "O": 0.24955
24
+ },
25
+ "loss_ent": -1.1872119903564453,
26
+ "loss_ctr": 0.0
27
+ },
28
+ {
29
+ "step": 250,
30
+ "ce": 1.0140388011932373,
31
+ "train_acc": 0.46875,
32
+ "val_acc": 0.4095904095904096,
33
+ "bal": 8.399999618530273,
34
+ "load": [
35
+ 1.0,
36
+ 1.0,
37
+ 1.0,
38
+ 1.0
39
+ ],
40
+ "ent": 0.5006841913547126,
41
+ "router_wnorm": 0.7527693510055542,
42
+ "router_gnorm": 0.9435262680053711,
43
+ "slot_ent": 0.8829337358474731,
44
+ "slot_ema": {
45
+ "Q": 0.12123,
46
+ "K": 0.2471,
47
+ "V": 0.19158,
48
+ "O": 0.44008
49
+ },
50
+ "loss_ent": -1.2240060567855835,
51
+ "loss_ctr": 0.0
52
+ },
53
+ {
54
+ "step": 500,
55
+ "ce": 0.9942823052406311,
56
+ "train_acc": 0.6875,
57
+ "val_acc": 0.4745254745254745,
58
+ "bal": 8.399999618530273,
59
+ "load": [
60
+ 1.0,
61
+ 1.0,
62
+ 1.0,
63
+ 1.0
64
+ ],
65
+ "ent": 0.6621884122527403,
66
+ "router_wnorm": 0.7576786279678345,
67
+ "router_gnorm": 0.8200175166130066,
68
+ "slot_ent": 0.8271520137786865,
69
+ "slot_ema": {
70
+ "Q": 0.17791,
71
+ "K": 0.24659,
72
+ "V": 0.23269,
73
+ "O": 0.34281
74
+ },
75
+ "loss_ent": -1.1466761827468872,
76
+ "loss_ctr": 0.0
77
+ },
78
+ {
79
+ "step": 750,
80
+ "ce": 0.47897279262542725,
81
+ "train_acc": 0.8125,
82
+ "val_acc": 0.5414585414585414,
83
+ "bal": 8.399999618530273,
84
+ "load": [
85
+ 1.0,
86
+ 1.0,
87
+ 1.0,
88
+ 1.0
89
+ ],
90
+ "ent": 0.688850194685398,
91
+ "router_wnorm": 0.7590436935424805,
92
+ "router_gnorm": 0.9338673949241638,
93
+ "slot_ent": 0.8950353860855103,
94
+ "slot_ema": {
95
+ "Q": 0.20282,
96
+ "K": 0.24834,
97
+ "V": 0.24025,
98
+ "O": 0.30859
99
+ },
100
+ "loss_ent": -1.2407824993133545,
101
+ "loss_ctr": 0.0
102
+ },
103
+ {
104
+ "step": 1000,
105
+ "ce": 0.32081425189971924,
106
+ "train_acc": 0.875,
107
+ "val_acc": 0.5194805194805194,
108
+ "bal": 8.399999618530273,
109
+ "load": [
110
+ 1.0,
111
+ 1.0,
112
+ 1.0,
113
+ 1.0
114
+ ],
115
+ "ent": 0.737821308614003,
116
+ "router_wnorm": 0.7632025480270386,
117
+ "router_gnorm": 0.7711656093597412,
118
+ "slot_ent": 0.8849113583564758,
119
+ "slot_ema": {
120
+ "Q": 0.22373,
121
+ "K": 0.23974,
122
+ "V": 0.24578,
123
+ "O": 0.29075
124
+ },
125
+ "loss_ent": -1.2267476320266724,
126
+ "loss_ctr": 0.0
127
+ },
128
+ {
129
+ "step": 1250,
130
+ "ce": 0.23929917812347412,
131
+ "train_acc": 0.875,
132
+ "val_acc": 0.5474525474525475,
133
+ "bal": 8.399999618530273,
134
+ "load": [
135
+ 1.0,
136
+ 1.0,
137
+ 1.0,
138
+ 1.0
139
+ ],
140
+ "ent": 0.8096277629762192,
141
+ "router_wnorm": 0.7618330717086792,
142
+ "router_gnorm": 0.8540073037147522,
143
+ "slot_ent": 0.8828784227371216,
144
+ "slot_ema": {
145
+ "Q": 0.23265,
146
+ "K": 0.23657,
147
+ "V": 0.24934,
148
+ "O": 0.28144
149
+ },
150
+ "loss_ent": -1.2239294052124023,
151
+ "loss_ctr": 0.0
152
+ },
153
+ {
154
+ "step": 1500,
155
+ "ce": 0.06212315335869789,
156
+ "train_acc": 0.96875,
157
+ "val_acc": 0.5624375624375625,
158
+ "bal": 8.399999618530273,
159
+ "load": [
160
+ 1.0,
161
+ 1.0,
162
+ 1.0,
163
+ 1.0
164
+ ],
165
+ "ent": 0.7895812072176739,
166
+ "router_wnorm": 0.7516821622848511,
167
+ "router_gnorm": 0.694826066493988,
168
+ "slot_ent": 0.8929040431976318,
169
+ "slot_ema": {
170
+ "Q": 0.24367,
171
+ "K": 0.24157,
172
+ "V": 0.24913,
173
+ "O": 0.26563
174
+ },
175
+ "loss_ent": -1.2378278970718384,
176
+ "loss_ctr": 0.0
177
+ }
178
+ ],
179
+ "best_val_acc": 0.5624375624375625,
180
+ "split": "按类别分层 80/20",
181
+ "eval": {
182
+ "acc_heldout": 0.5624375624375625,
183
+ "chance": 0.25,
184
+ "load": [
185
+ 1.0,
186
+ 1.0,
187
+ 1.0,
188
+ 1.0
189
+ ],
190
+ "entropy": 0.793633978385256,
191
+ "balance": 0.9999999989999999,
192
+ "proj_space_slot_gate": {
193
+ "load": [
194
+ 1.0,
195
+ 1.0,
196
+ 1.0,
197
+ 1.0
198
+ ],
199
+ "entropy": 0.894803718659883,
200
+ "balance": 0.9999999989999999
201
+ },
202
+ "fusion_expert": {
203
+ "load": [
204
+ 1.0,
205
+ 1.0,
206
+ 1.0,
207
+ 1.0
208
+ ],
209
+ "entropy": 0.793633978385256,
210
+ "balance": 0.9999999989999999
211
+ }
212
+ },
213
+ "version": "67.0",
214
+ "fix": "proj_space.router 此前是死代码（输出未被使用），现作为 token 级槽门控乘进 canonical",
215
+ "hparams": {
216
+ "lr": 0.0002,
217
+ "steps": 1500,
218
+ "bs": 32,
219
+ "bal_w": 0.01
220
+ },
221
+ "params": {
222
+ "proj_space": 2762766,
223
+ "fusion": 16268800
224
+ },
225
+ "note": "融合层吃1024维，专家解冻（偏离项目原设定）"
226
+ }