granularity results tiera-rerun-fixed-moonep/ultraep/tierbprobe @ 2026-08-30T05:20:41Z from sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0
Browse files- tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/mem_samples.csv +72 -0
- tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/train.log +0 -0
- tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/mem_samples.csv +232 -0
- tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/run.env +70 -0
- tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/train.log +0 -0
- tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1/mem_samples.csv +224 -0
- tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1/run.env +70 -0
- tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1/train.log +0 -0
- tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/mem_samples.csv +232 -0
- tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/run.env +70 -0
- tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/train.log +0 -0
- tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1/mem_samples.csv +232 -0
- tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1/run.env +70 -0
- tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1/train.log +0 -0
- tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1/mem_samples.csv +224 -0
- tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1/run.env +70 -0
- tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1/train.log +0 -0
- tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/mem_samples.csv +232 -0
- tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/run.env +70 -0
- tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/train.log +0 -0
- tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/capacity_attempts.tsv +7 -0
- tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/capacity_selection.tsv +3 -0
- tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/probe_summary.json +849 -0
- tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/probe_summary.tsv +12 -0
- tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/results.tsv +7 -0
tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/mem_samples.csv
CHANGED
|
@@ -150,3 +150,75 @@
|
|
| 150 |
5, 85043
|
| 151 |
6, 85053
|
| 152 |
7, 85041
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 150 |
5, 85043
|
| 151 |
6, 85053
|
| 152 |
7, 85041
|
| 153 |
+
0, 86105
|
| 154 |
+
1, 86165
|
| 155 |
+
2, 86161
|
| 156 |
+
3, 86175
|
| 157 |
+
4, 86075
|
| 158 |
+
5, 86199
|
| 159 |
+
6, 86167
|
| 160 |
+
7, 86219
|
| 161 |
+
0, 87195
|
| 162 |
+
1, 87305
|
| 163 |
+
2, 87271
|
| 164 |
+
3, 87285
|
| 165 |
+
4, 87213
|
| 166 |
+
5, 87283
|
| 167 |
+
6, 87299
|
| 168 |
+
7, 87317
|
| 169 |
+
0, 110343
|
| 170 |
+
1, 110333
|
| 171 |
+
2, 110333
|
| 172 |
+
3, 110333
|
| 173 |
+
4, 110319
|
| 174 |
+
5, 110333
|
| 175 |
+
6, 110333
|
| 176 |
+
7, 110333
|
| 177 |
+
0, 110343
|
| 178 |
+
1, 110333
|
| 179 |
+
2, 110333
|
| 180 |
+
3, 110333
|
| 181 |
+
4, 110319
|
| 182 |
+
5, 110333
|
| 183 |
+
6, 110333
|
| 184 |
+
7, 110333
|
| 185 |
+
0, 110343
|
| 186 |
+
1, 110333
|
| 187 |
+
2, 110333
|
| 188 |
+
3, 110333
|
| 189 |
+
4, 110319
|
| 190 |
+
5, 110333
|
| 191 |
+
6, 110333
|
| 192 |
+
7, 110333
|
| 193 |
+
0, 105951
|
| 194 |
+
1, 106203
|
| 195 |
+
2, 106203
|
| 196 |
+
3, 105991
|
| 197 |
+
4, 105863
|
| 198 |
+
5, 105557
|
| 199 |
+
6, 105621
|
| 200 |
+
7, 106203
|
| 201 |
+
0, 104139
|
| 202 |
+
1, 103553
|
| 203 |
+
2, 104135
|
| 204 |
+
3, 103873
|
| 205 |
+
4, 104185
|
| 206 |
+
5, 104781
|
| 207 |
+
6, 104519
|
| 208 |
+
7, 104781
|
| 209 |
+
0, 4
|
| 210 |
+
1, 4
|
| 211 |
+
2, 4
|
| 212 |
+
3, 4
|
| 213 |
+
4, 4
|
| 214 |
+
5, 103101
|
| 215 |
+
6, 4
|
| 216 |
+
7, 103101
|
| 217 |
+
0, 0
|
| 218 |
+
1, 0
|
| 219 |
+
2, 0
|
| 220 |
+
3, 0
|
| 221 |
+
4, 0
|
| 222 |
+
5, 0
|
| 223 |
+
6, 0
|
| 224 |
+
7, 0
|
tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/train.log
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/mem_samples.csv
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
0, 0
|
| 2 |
+
1, 0
|
| 3 |
+
2, 0
|
| 4 |
+
3, 0
|
| 5 |
+
4, 0
|
| 6 |
+
5, 0
|
| 7 |
+
6, 0
|
| 8 |
+
7, 0
|
| 9 |
+
0, 0
|
| 10 |
+
1, 0
|
| 11 |
+
2, 0
|
| 12 |
+
3, 0
|
| 13 |
+
4, 0
|
| 14 |
+
5, 0
|
| 15 |
+
6, 0
|
| 16 |
+
7, 0
|
| 17 |
+
0, 1
|
| 18 |
+
1, 1
|
| 19 |
+
2, 1
|
| 20 |
+
3, 1
|
| 21 |
+
4, 1
|
| 22 |
+
5, 1
|
| 23 |
+
6, 1
|
| 24 |
+
7, 1
|
| 25 |
+
0, 4
|
| 26 |
+
1, 4
|
| 27 |
+
2, 4
|
| 28 |
+
3, 4
|
| 29 |
+
4, 1
|
| 30 |
+
5, 1
|
| 31 |
+
6, 1
|
| 32 |
+
7, 1
|
| 33 |
+
0, 4
|
| 34 |
+
1, 4
|
| 35 |
+
2, 4
|
| 36 |
+
3, 4
|
| 37 |
+
4, 4
|
| 38 |
+
5, 4
|
| 39 |
+
6, 4
|
| 40 |
+
7, 4
|
| 41 |
+
0, 4
|
| 42 |
+
1, 27
|
| 43 |
+
2, 27
|
| 44 |
+
3, 4
|
| 45 |
+
4, 4
|
| 46 |
+
5, 4
|
| 47 |
+
6, 4
|
| 48 |
+
7, 45
|
| 49 |
+
0, 887
|
| 50 |
+
1, 895
|
| 51 |
+
2, 879
|
| 52 |
+
3, 889
|
| 53 |
+
4, 911
|
| 54 |
+
5, 883
|
| 55 |
+
6, 887
|
| 56 |
+
7, 889
|
| 57 |
+
0, 1895
|
| 58 |
+
1, 1895
|
| 59 |
+
2, 1895
|
| 60 |
+
3, 1895
|
| 61 |
+
4, 1895
|
| 62 |
+
5, 1895
|
| 63 |
+
6, 1895
|
| 64 |
+
7, 1895
|
| 65 |
+
0, 2333
|
| 66 |
+
1, 2337
|
| 67 |
+
2, 2335
|
| 68 |
+
3, 2343
|
| 69 |
+
4, 2343
|
| 70 |
+
5, 2373
|
| 71 |
+
6, 2361
|
| 72 |
+
7, 2369
|
| 73 |
+
0, 12791
|
| 74 |
+
1, 14867
|
| 75 |
+
2, 12787
|
| 76 |
+
3, 12791
|
| 77 |
+
4, 12791
|
| 78 |
+
5, 12783
|
| 79 |
+
6, 35145
|
| 80 |
+
7, 19017
|
| 81 |
+
0, 42215
|
| 82 |
+
1, 42203
|
| 83 |
+
2, 42203
|
| 84 |
+
3, 42203
|
| 85 |
+
4, 42203
|
| 86 |
+
5, 42203
|
| 87 |
+
6, 42211
|
| 88 |
+
7, 42211
|
| 89 |
+
0, 42941
|
| 90 |
+
1, 42915
|
| 91 |
+
2, 42935
|
| 92 |
+
3, 42925
|
| 93 |
+
4, 42935
|
| 94 |
+
5, 42945
|
| 95 |
+
6, 42945
|
| 96 |
+
7, 42945
|
| 97 |
+
0, 43545
|
| 98 |
+
1, 43529
|
| 99 |
+
2, 43529
|
| 100 |
+
3, 43529
|
| 101 |
+
4, 43529
|
| 102 |
+
5, 43529
|
| 103 |
+
6, 43529
|
| 104 |
+
7, 43529
|
| 105 |
+
0, 44089
|
| 106 |
+
1, 44073
|
| 107 |
+
2, 44073
|
| 108 |
+
3, 44073
|
| 109 |
+
4, 44073
|
| 110 |
+
5, 44073
|
| 111 |
+
6, 44073
|
| 112 |
+
7, 44073
|
| 113 |
+
0, 45919
|
| 114 |
+
1, 45903
|
| 115 |
+
2, 45903
|
| 116 |
+
3, 45903
|
| 117 |
+
4, 45903
|
| 118 |
+
5, 45903
|
| 119 |
+
6, 45903
|
| 120 |
+
7, 45903
|
| 121 |
+
0, 45919
|
| 122 |
+
1, 45903
|
| 123 |
+
2, 45903
|
| 124 |
+
3, 45903
|
| 125 |
+
4, 45903
|
| 126 |
+
5, 45903
|
| 127 |
+
6, 45903
|
| 128 |
+
7, 45903
|
| 129 |
+
0, 73899
|
| 130 |
+
1, 73809
|
| 131 |
+
2, 73809
|
| 132 |
+
3, 73809
|
| 133 |
+
4, 73859
|
| 134 |
+
5, 73809
|
| 135 |
+
6, 73809
|
| 136 |
+
7, 73809
|
| 137 |
+
0, 73899
|
| 138 |
+
1, 73809
|
| 139 |
+
2, 73809
|
| 140 |
+
3, 73809
|
| 141 |
+
4, 73859
|
| 142 |
+
5, 73809
|
| 143 |
+
6, 73809
|
| 144 |
+
7, 73809
|
| 145 |
+
0, 73899
|
| 146 |
+
1, 73811
|
| 147 |
+
2, 73811
|
| 148 |
+
3, 73811
|
| 149 |
+
4, 73859
|
| 150 |
+
5, 73811
|
| 151 |
+
6, 73811
|
| 152 |
+
7, 73811
|
| 153 |
+
0, 73913
|
| 154 |
+
1, 73825
|
| 155 |
+
2, 73825
|
| 156 |
+
3, 73825
|
| 157 |
+
4, 73873
|
| 158 |
+
5, 73825
|
| 159 |
+
6, 73825
|
| 160 |
+
7, 73825
|
| 161 |
+
0, 87097
|
| 162 |
+
1, 87177
|
| 163 |
+
2, 87209
|
| 164 |
+
3, 87253
|
| 165 |
+
4, 87293
|
| 166 |
+
5, 87245
|
| 167 |
+
6, 87243
|
| 168 |
+
7, 87255
|
| 169 |
+
0, 88161
|
| 170 |
+
1, 88235
|
| 171 |
+
2, 88255
|
| 172 |
+
3, 88229
|
| 173 |
+
4, 88263
|
| 174 |
+
5, 88235
|
| 175 |
+
6, 88255
|
| 176 |
+
7, 88259
|
| 177 |
+
0, 106235
|
| 178 |
+
1, 106493
|
| 179 |
+
2, 106333
|
| 180 |
+
3, 106493
|
| 181 |
+
4, 105533
|
| 182 |
+
5, 106333
|
| 183 |
+
6, 106553
|
| 184 |
+
7, 107097
|
| 185 |
+
0, 111521
|
| 186 |
+
1, 111567
|
| 187 |
+
2, 111567
|
| 188 |
+
3, 111567
|
| 189 |
+
4, 111565
|
| 190 |
+
5, 111567
|
| 191 |
+
6, 111567
|
| 192 |
+
7, 111567
|
| 193 |
+
0, 111521
|
| 194 |
+
1, 111567
|
| 195 |
+
2, 111567
|
| 196 |
+
3, 111567
|
| 197 |
+
4, 111565
|
| 198 |
+
5, 111567
|
| 199 |
+
6, 111567
|
| 200 |
+
7, 111567
|
| 201 |
+
0, 111521
|
| 202 |
+
1, 111567
|
| 203 |
+
2, 111567
|
| 204 |
+
3, 111567
|
| 205 |
+
4, 111565
|
| 206 |
+
5, 111567
|
| 207 |
+
6, 111567
|
| 208 |
+
7, 111567
|
| 209 |
+
0, 107381
|
| 210 |
+
1, 107427
|
| 211 |
+
2, 107427
|
| 212 |
+
3, 107427
|
| 213 |
+
4, 107425
|
| 214 |
+
5, 107427
|
| 215 |
+
6, 107501
|
| 216 |
+
7, 107427
|
| 217 |
+
0, 105697
|
| 218 |
+
1, 106393
|
| 219 |
+
2, 105995
|
| 220 |
+
3, 106005
|
| 221 |
+
4, 105741
|
| 222 |
+
5, 105685
|
| 223 |
+
6, 106005
|
| 224 |
+
7, 106393
|
| 225 |
+
0, 1
|
| 226 |
+
1, 4
|
| 227 |
+
2, 132
|
| 228 |
+
3, 106005
|
| 229 |
+
4, 644
|
| 230 |
+
5, 4
|
| 231 |
+
6, 4
|
| 232 |
+
7, 4
|
tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/run.env
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=ultraep
|
| 2 |
+
config=G3
|
| 3 |
+
family=tierA
|
| 4 |
+
repo=/workspace/Megatron-UltraEP
|
| 5 |
+
branch=ultraep-token-drop-analysis
|
| 6 |
+
commit=eaf829e7
|
| 7 |
+
num_layers=16
|
| 8 |
+
hidden=4096
|
| 9 |
+
ffn_hidden=14336
|
| 10 |
+
moe_ffn_hidden=1792
|
| 11 |
+
num_experts=64
|
| 12 |
+
topk=16
|
| 13 |
+
experts_per_rank=8
|
| 14 |
+
expert_params=22020096
|
| 15 |
+
heads=32
|
| 16 |
+
kv_groups=8
|
| 17 |
+
kv_channels=
|
| 18 |
+
qk_layernorm=0
|
| 19 |
+
shared_ffn=0
|
| 20 |
+
shared_expert_overlap=0
|
| 21 |
+
moe_token_dispatcher=flex
|
| 22 |
+
mla=0
|
| 23 |
+
dense_first_layers=0
|
| 24 |
+
moe_layers=16
|
| 25 |
+
rotary_base=1000000
|
| 26 |
+
rotary_scaling=
|
| 27 |
+
qkv_bias=0
|
| 28 |
+
router_balancing=aux_loss
|
| 29 |
+
aux_loss_coeff=1e-2
|
| 30 |
+
router_pre_softmax=0
|
| 31 |
+
router_fusion=0
|
| 32 |
+
router_dtype=fp32
|
| 33 |
+
device_initiated_grouped_gemm=1
|
| 34 |
+
norm_epsilon=1e-5
|
| 35 |
+
seq_length=4096
|
| 36 |
+
micro_batch_size=1
|
| 37 |
+
global_batch_size=16
|
| 38 |
+
num_microbatches=2
|
| 39 |
+
assignments_per_rank=65536
|
| 40 |
+
mean_tokens_per_expert=8192
|
| 41 |
+
slot_policy=matched
|
| 42 |
+
slots_per_rank=8
|
| 43 |
+
capacity_mode=static
|
| 44 |
+
capacity=1.0625
|
| 45 |
+
cuda_graph=0
|
| 46 |
+
probe_mode=1
|
| 47 |
+
async_grad_reduce=0
|
| 48 |
+
branch_role=probe
|
| 49 |
+
memory_dump_mode=none
|
| 50 |
+
memory_snapshot_path=
|
| 51 |
+
cuda_graph_warmup_steps=3
|
| 52 |
+
recompute=none
|
| 53 |
+
force_balance=0
|
| 54 |
+
train_iters=12
|
| 55 |
+
ep_size=8
|
| 56 |
+
echo_dispatch_overlap=1
|
| 57 |
+
echo_dispatch_recompute=0
|
| 58 |
+
ultraep_quota_min_tokens=1024
|
| 59 |
+
ultraep_weight_sync_join=
|
| 60 |
+
profile_enable=0
|
| 61 |
+
kernel_trace=0
|
| 62 |
+
fp8_mode=mxfp8
|
| 63 |
+
fp8_recipe=mxfp8
|
| 64 |
+
training_seed=1234
|
| 65 |
+
learning_rate=1e-4
|
| 66 |
+
min_learning_rate=1.0e-5
|
| 67 |
+
lr_warmup_iters=5
|
| 68 |
+
state_dump_enable=0
|
| 69 |
+
state_dump_dir=
|
| 70 |
+
moonep_assert_no_pending_reduce=0
|
tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1/mem_samples.csv
ADDED
|
@@ -0,0 +1,224 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
0, 0
|
| 2 |
+
1, 0
|
| 3 |
+
2, 0
|
| 4 |
+
3, 0
|
| 5 |
+
4, 0
|
| 6 |
+
5, 0
|
| 7 |
+
6, 0
|
| 8 |
+
7, 0
|
| 9 |
+
0, 0
|
| 10 |
+
1, 0
|
| 11 |
+
2, 0
|
| 12 |
+
3, 0
|
| 13 |
+
4, 0
|
| 14 |
+
5, 0
|
| 15 |
+
6, 0
|
| 16 |
+
7, 0
|
| 17 |
+
0, 2
|
| 18 |
+
1, 1
|
| 19 |
+
2, 1
|
| 20 |
+
3, 1
|
| 21 |
+
4, 1
|
| 22 |
+
5, 1
|
| 23 |
+
6, 1
|
| 24 |
+
7, 1
|
| 25 |
+
0, 4
|
| 26 |
+
1, 4
|
| 27 |
+
2, 4
|
| 28 |
+
3, 4
|
| 29 |
+
4, 1
|
| 30 |
+
5, 1
|
| 31 |
+
6, 1
|
| 32 |
+
7, 1
|
| 33 |
+
0, 4
|
| 34 |
+
1, 4
|
| 35 |
+
2, 4
|
| 36 |
+
3, 4
|
| 37 |
+
4, 4
|
| 38 |
+
5, 4
|
| 39 |
+
6, 4
|
| 40 |
+
7, 1
|
| 41 |
+
0, 4
|
| 42 |
+
1, 4
|
| 43 |
+
2, 165
|
| 44 |
+
3, 4
|
| 45 |
+
4, 4
|
| 46 |
+
5, 4
|
| 47 |
+
6, 4
|
| 48 |
+
7, 4
|
| 49 |
+
0, 821
|
| 50 |
+
1, 821
|
| 51 |
+
2, 821
|
| 52 |
+
3, 821
|
| 53 |
+
4, 821
|
| 54 |
+
5, 821
|
| 55 |
+
6, 821
|
| 56 |
+
7, 821
|
| 57 |
+
0, 1895
|
| 58 |
+
1, 1895
|
| 59 |
+
2, 1895
|
| 60 |
+
3, 1895
|
| 61 |
+
4, 1895
|
| 62 |
+
5, 1895
|
| 63 |
+
6, 1895
|
| 64 |
+
7, 1895
|
| 65 |
+
0, 2301
|
| 66 |
+
1, 2307
|
| 67 |
+
2, 2307
|
| 68 |
+
3, 2301
|
| 69 |
+
4, 2315
|
| 70 |
+
5, 2309
|
| 71 |
+
6, 2313
|
| 72 |
+
7, 2323
|
| 73 |
+
0, 35145
|
| 74 |
+
1, 35145
|
| 75 |
+
2, 35145
|
| 76 |
+
3, 35145
|
| 77 |
+
4, 35145
|
| 78 |
+
5, 35145
|
| 79 |
+
6, 35145
|
| 80 |
+
7, 35145
|
| 81 |
+
0, 42227
|
| 82 |
+
1, 42213
|
| 83 |
+
2, 42213
|
| 84 |
+
3, 42213
|
| 85 |
+
4, 42213
|
| 86 |
+
5, 42213
|
| 87 |
+
6, 42213
|
| 88 |
+
7, 42213
|
| 89 |
+
0, 43545
|
| 90 |
+
1, 43529
|
| 91 |
+
2, 43529
|
| 92 |
+
3, 43529
|
| 93 |
+
4, 43529
|
| 94 |
+
5, 43529
|
| 95 |
+
6, 43529
|
| 96 |
+
7, 43529
|
| 97 |
+
0, 43545
|
| 98 |
+
1, 43529
|
| 99 |
+
2, 43529
|
| 100 |
+
3, 43529
|
| 101 |
+
4, 43529
|
| 102 |
+
5, 43529
|
| 103 |
+
6, 43529
|
| 104 |
+
7, 43529
|
| 105 |
+
0, 44185
|
| 106 |
+
1, 44169
|
| 107 |
+
2, 44169
|
| 108 |
+
3, 44169
|
| 109 |
+
4, 44169
|
| 110 |
+
5, 44169
|
| 111 |
+
6, 44169
|
| 112 |
+
7, 44169
|
| 113 |
+
0, 46451
|
| 114 |
+
1, 46435
|
| 115 |
+
2, 46435
|
| 116 |
+
3, 46435
|
| 117 |
+
4, 46435
|
| 118 |
+
5, 46435
|
| 119 |
+
6, 46435
|
| 120 |
+
7, 46435
|
| 121 |
+
0, 54689
|
| 122 |
+
1, 54671
|
| 123 |
+
2, 54643
|
| 124 |
+
3, 54673
|
| 125 |
+
4, 54757
|
| 126 |
+
5, 54673
|
| 127 |
+
6, 54701
|
| 128 |
+
7, 54687
|
| 129 |
+
0, 76579
|
| 130 |
+
1, 76563
|
| 131 |
+
2, 76563
|
| 132 |
+
3, 76563
|
| 133 |
+
4, 76633
|
| 134 |
+
5, 76563
|
| 135 |
+
6, 76563
|
| 136 |
+
7, 76563
|
| 137 |
+
0, 76579
|
| 138 |
+
1, 76563
|
| 139 |
+
2, 76563
|
| 140 |
+
3, 76563
|
| 141 |
+
4, 76633
|
| 142 |
+
5, 76563
|
| 143 |
+
6, 76563
|
| 144 |
+
7, 76563
|
| 145 |
+
0, 76579
|
| 146 |
+
1, 76563
|
| 147 |
+
2, 76563
|
| 148 |
+
3, 76563
|
| 149 |
+
4, 76633
|
| 150 |
+
5, 76563
|
| 151 |
+
6, 76563
|
| 152 |
+
7, 76563
|
| 153 |
+
0, 89049
|
| 154 |
+
1, 89033
|
| 155 |
+
2, 89033
|
| 156 |
+
3, 89033
|
| 157 |
+
4, 89083
|
| 158 |
+
5, 89033
|
| 159 |
+
6, 89033
|
| 160 |
+
7, 89033
|
| 161 |
+
0, 90195
|
| 162 |
+
1, 90201
|
| 163 |
+
2, 90219
|
| 164 |
+
3, 90229
|
| 165 |
+
4, 90267
|
| 166 |
+
5, 90191
|
| 167 |
+
6, 90229
|
| 168 |
+
7, 90197
|
| 169 |
+
0, 91359
|
| 170 |
+
1, 91327
|
| 171 |
+
2, 91325
|
| 172 |
+
3, 91321
|
| 173 |
+
4, 91417
|
| 174 |
+
5, 91347
|
| 175 |
+
6, 91331
|
| 176 |
+
7, 91357
|
| 177 |
+
0, 114205
|
| 178 |
+
1, 114249
|
| 179 |
+
2, 114249
|
| 180 |
+
3, 114249
|
| 181 |
+
4, 114319
|
| 182 |
+
5, 114249
|
| 183 |
+
6, 114249
|
| 184 |
+
7, 114249
|
| 185 |
+
0, 114205
|
| 186 |
+
1, 114249
|
| 187 |
+
2, 114249
|
| 188 |
+
3, 114249
|
| 189 |
+
4, 114319
|
| 190 |
+
5, 114249
|
| 191 |
+
6, 114249
|
| 192 |
+
7, 114249
|
| 193 |
+
0, 114205
|
| 194 |
+
1, 114249
|
| 195 |
+
2, 114249
|
| 196 |
+
3, 114249
|
| 197 |
+
4, 114319
|
| 198 |
+
5, 114249
|
| 199 |
+
6, 114249
|
| 200 |
+
7, 114249
|
| 201 |
+
0, 110711
|
| 202 |
+
1, 110755
|
| 203 |
+
2, 110755
|
| 204 |
+
3, 110755
|
| 205 |
+
4, 110825
|
| 206 |
+
5, 110755
|
| 207 |
+
6, 110755
|
| 208 |
+
7, 110755
|
| 209 |
+
0, 109031
|
| 210 |
+
1, 109075
|
| 211 |
+
2, 109075
|
| 212 |
+
3, 109075
|
| 213 |
+
4, 109145
|
| 214 |
+
5, 108751
|
| 215 |
+
6, 109721
|
| 216 |
+
7, 109395
|
| 217 |
+
0, 4
|
| 218 |
+
1, 107395
|
| 219 |
+
2, 107459
|
| 220 |
+
3, 108041
|
| 221 |
+
4, 324
|
| 222 |
+
5, 4
|
| 223 |
+
6, 4
|
| 224 |
+
7, 4
|
tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1/run.env
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=ultraep
|
| 2 |
+
config=G3
|
| 3 |
+
family=tierA
|
| 4 |
+
repo=/workspace/Megatron-UltraEP
|
| 5 |
+
branch=ultraep-token-drop-analysis
|
| 6 |
+
commit=eaf829e7
|
| 7 |
+
num_layers=16
|
| 8 |
+
hidden=4096
|
| 9 |
+
ffn_hidden=14336
|
| 10 |
+
moe_ffn_hidden=1792
|
| 11 |
+
num_experts=64
|
| 12 |
+
topk=16
|
| 13 |
+
experts_per_rank=8
|
| 14 |
+
expert_params=22020096
|
| 15 |
+
heads=32
|
| 16 |
+
kv_groups=8
|
| 17 |
+
kv_channels=
|
| 18 |
+
qk_layernorm=0
|
| 19 |
+
shared_ffn=0
|
| 20 |
+
shared_expert_overlap=0
|
| 21 |
+
moe_token_dispatcher=flex
|
| 22 |
+
mla=0
|
| 23 |
+
dense_first_layers=0
|
| 24 |
+
moe_layers=16
|
| 25 |
+
rotary_base=1000000
|
| 26 |
+
rotary_scaling=
|
| 27 |
+
qkv_bias=0
|
| 28 |
+
router_balancing=aux_loss
|
| 29 |
+
aux_loss_coeff=1e-2
|
| 30 |
+
router_pre_softmax=0
|
| 31 |
+
router_fusion=0
|
| 32 |
+
router_dtype=fp32
|
| 33 |
+
device_initiated_grouped_gemm=1
|
| 34 |
+
norm_epsilon=1e-5
|
| 35 |
+
seq_length=4096
|
| 36 |
+
micro_batch_size=1
|
| 37 |
+
global_batch_size=16
|
| 38 |
+
num_microbatches=2
|
| 39 |
+
assignments_per_rank=65536
|
| 40 |
+
mean_tokens_per_expert=8192
|
| 41 |
+
slot_policy=matched
|
| 42 |
+
slots_per_rank=8
|
| 43 |
+
capacity_mode=static
|
| 44 |
+
capacity=1.25
|
| 45 |
+
cuda_graph=0
|
| 46 |
+
probe_mode=1
|
| 47 |
+
async_grad_reduce=0
|
| 48 |
+
branch_role=probe
|
| 49 |
+
memory_dump_mode=none
|
| 50 |
+
memory_snapshot_path=
|
| 51 |
+
cuda_graph_warmup_steps=3
|
| 52 |
+
recompute=none
|
| 53 |
+
force_balance=0
|
| 54 |
+
train_iters=12
|
| 55 |
+
ep_size=8
|
| 56 |
+
echo_dispatch_overlap=1
|
| 57 |
+
echo_dispatch_recompute=0
|
| 58 |
+
ultraep_quota_min_tokens=1024
|
| 59 |
+
ultraep_weight_sync_join=
|
| 60 |
+
profile_enable=0
|
| 61 |
+
kernel_trace=0
|
| 62 |
+
fp8_mode=mxfp8
|
| 63 |
+
fp8_recipe=mxfp8
|
| 64 |
+
training_seed=1234
|
| 65 |
+
learning_rate=1e-4
|
| 66 |
+
min_learning_rate=1.0e-5
|
| 67 |
+
lr_warmup_iters=5
|
| 68 |
+
state_dump_enable=0
|
| 69 |
+
state_dump_dir=
|
| 70 |
+
moonep_assert_no_pending_reduce=0
|
tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1/train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/mem_samples.csv
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
0, 0
|
| 2 |
+
1, 0
|
| 3 |
+
2, 0
|
| 4 |
+
3, 0
|
| 5 |
+
4, 0
|
| 6 |
+
5, 0
|
| 7 |
+
6, 0
|
| 8 |
+
7, 0
|
| 9 |
+
0, 0
|
| 10 |
+
1, 0
|
| 11 |
+
2, 0
|
| 12 |
+
3, 0
|
| 13 |
+
4, 0
|
| 14 |
+
5, 0
|
| 15 |
+
6, 0
|
| 16 |
+
7, 0
|
| 17 |
+
0, 4
|
| 18 |
+
1, 1
|
| 19 |
+
2, 1
|
| 20 |
+
3, 1
|
| 21 |
+
4, 1
|
| 22 |
+
5, 1
|
| 23 |
+
6, 1
|
| 24 |
+
7, 1
|
| 25 |
+
0, 4
|
| 26 |
+
1, 4
|
| 27 |
+
2, 4
|
| 28 |
+
3, 4
|
| 29 |
+
4, 1
|
| 30 |
+
5, 1
|
| 31 |
+
6, 1
|
| 32 |
+
7, 1
|
| 33 |
+
0, 4
|
| 34 |
+
1, 4
|
| 35 |
+
2, 4
|
| 36 |
+
3, 4
|
| 37 |
+
4, 4
|
| 38 |
+
5, 4
|
| 39 |
+
6, 4
|
| 40 |
+
7, 4
|
| 41 |
+
0, 4
|
| 42 |
+
1, 4
|
| 43 |
+
2, 4
|
| 44 |
+
3, 4
|
| 45 |
+
4, 4
|
| 46 |
+
5, 4
|
| 47 |
+
6, 4
|
| 48 |
+
7, 4
|
| 49 |
+
0, 821
|
| 50 |
+
1, 821
|
| 51 |
+
2, 821
|
| 52 |
+
3, 821
|
| 53 |
+
4, 821
|
| 54 |
+
5, 821
|
| 55 |
+
6, 821
|
| 56 |
+
7, 821
|
| 57 |
+
0, 1895
|
| 58 |
+
1, 1895
|
| 59 |
+
2, 1895
|
| 60 |
+
3, 1895
|
| 61 |
+
4, 1895
|
| 62 |
+
5, 1895
|
| 63 |
+
6, 1895
|
| 64 |
+
7, 1895
|
| 65 |
+
0, 2337
|
| 66 |
+
1, 2345
|
| 67 |
+
2, 2333
|
| 68 |
+
3, 2347
|
| 69 |
+
4, 2333
|
| 70 |
+
5, 2337
|
| 71 |
+
6, 2345
|
| 72 |
+
7, 2353
|
| 73 |
+
0, 19017
|
| 74 |
+
1, 35145
|
| 75 |
+
2, 35145
|
| 76 |
+
3, 35145
|
| 77 |
+
4, 19017
|
| 78 |
+
5, 14867
|
| 79 |
+
6, 35145
|
| 80 |
+
7, 35145
|
| 81 |
+
0, 42227
|
| 82 |
+
1, 42213
|
| 83 |
+
2, 42213
|
| 84 |
+
3, 42213
|
| 85 |
+
4, 42213
|
| 86 |
+
5, 42213
|
| 87 |
+
6, 42213
|
| 88 |
+
7, 42213
|
| 89 |
+
0, 43545
|
| 90 |
+
1, 43529
|
| 91 |
+
2, 43529
|
| 92 |
+
3, 43529
|
| 93 |
+
4, 43529
|
| 94 |
+
5, 43529
|
| 95 |
+
6, 43529
|
| 96 |
+
7, 43529
|
| 97 |
+
0, 43545
|
| 98 |
+
1, 43529
|
| 99 |
+
2, 43529
|
| 100 |
+
3, 43529
|
| 101 |
+
4, 43529
|
| 102 |
+
5, 43529
|
| 103 |
+
6, 43529
|
| 104 |
+
7, 43529
|
| 105 |
+
0, 44185
|
| 106 |
+
1, 44169
|
| 107 |
+
2, 44169
|
| 108 |
+
3, 44169
|
| 109 |
+
4, 44169
|
| 110 |
+
5, 44169
|
| 111 |
+
6, 44169
|
| 112 |
+
7, 44169
|
| 113 |
+
0, 46323
|
| 114 |
+
1, 46307
|
| 115 |
+
2, 46307
|
| 116 |
+
3, 46307
|
| 117 |
+
4, 46307
|
| 118 |
+
5, 46307
|
| 119 |
+
6, 46307
|
| 120 |
+
7, 46307
|
| 121 |
+
0, 75741
|
| 122 |
+
1, 75725
|
| 123 |
+
2, 75725
|
| 124 |
+
3, 75725
|
| 125 |
+
4, 75785
|
| 126 |
+
5, 75725
|
| 127 |
+
6, 75725
|
| 128 |
+
7, 75725
|
| 129 |
+
0, 76591
|
| 130 |
+
1, 76575
|
| 131 |
+
2, 76575
|
| 132 |
+
3, 76575
|
| 133 |
+
4, 76635
|
| 134 |
+
5, 76575
|
| 135 |
+
6, 76575
|
| 136 |
+
7, 76575
|
| 137 |
+
0, 76591
|
| 138 |
+
1, 76575
|
| 139 |
+
2, 76575
|
| 140 |
+
3, 76575
|
| 141 |
+
4, 76635
|
| 142 |
+
5, 76575
|
| 143 |
+
6, 76575
|
| 144 |
+
7, 76575
|
| 145 |
+
0, 82235
|
| 146 |
+
1, 81325
|
| 147 |
+
2, 81467
|
| 148 |
+
3, 81487
|
| 149 |
+
4, 82357
|
| 150 |
+
5, 82251
|
| 151 |
+
6, 81427
|
| 152 |
+
7, 82337
|
| 153 |
+
0, 89973
|
| 154 |
+
1, 89957
|
| 155 |
+
2, 89937
|
| 156 |
+
3, 89947
|
| 157 |
+
4, 89979
|
| 158 |
+
5, 89967
|
| 159 |
+
6, 90009
|
| 160 |
+
7, 90037
|
| 161 |
+
0, 90997
|
| 162 |
+
1, 90991
|
| 163 |
+
2, 90947
|
| 164 |
+
3, 90989
|
| 165 |
+
4, 90959
|
| 166 |
+
5, 90989
|
| 167 |
+
6, 90977
|
| 168 |
+
7, 90991
|
| 169 |
+
0, 114175
|
| 170 |
+
1, 114267
|
| 171 |
+
2, 114267
|
| 172 |
+
3, 114265
|
| 173 |
+
4, 114259
|
| 174 |
+
5, 114267
|
| 175 |
+
6, 114265
|
| 176 |
+
7, 114267
|
| 177 |
+
0, 114175
|
| 178 |
+
1, 114267
|
| 179 |
+
2, 114267
|
| 180 |
+
3, 114265
|
| 181 |
+
4, 114261
|
| 182 |
+
5, 114267
|
| 183 |
+
6, 114265
|
| 184 |
+
7, 114267
|
| 185 |
+
0, 114175
|
| 186 |
+
1, 114267
|
| 187 |
+
2, 114267
|
| 188 |
+
3, 114265
|
| 189 |
+
4, 114261
|
| 190 |
+
5, 114267
|
| 191 |
+
6, 114265
|
| 192 |
+
7, 114267
|
| 193 |
+
0, 113895
|
| 194 |
+
1, 113987
|
| 195 |
+
2, 113987
|
| 196 |
+
3, 113985
|
| 197 |
+
4, 113981
|
| 198 |
+
5, 113987
|
| 199 |
+
6, 113985
|
| 200 |
+
7, 113987
|
| 201 |
+
0, 109389
|
| 202 |
+
1, 109481
|
| 203 |
+
2, 109481
|
| 204 |
+
3, 109479
|
| 205 |
+
4, 109539
|
| 206 |
+
5, 110127
|
| 207 |
+
6, 109737
|
| 208 |
+
7, 109801
|
| 209 |
+
0, 107705
|
| 210 |
+
1, 108059
|
| 211 |
+
2, 107797
|
| 212 |
+
3, 108057
|
| 213 |
+
4, 107791
|
| 214 |
+
5, 108059
|
| 215 |
+
6, 107795
|
| 216 |
+
7, 108059
|
| 217 |
+
0, 4
|
| 218 |
+
1, 4
|
| 219 |
+
2, 4
|
| 220 |
+
3, 107023
|
| 221 |
+
4, 4
|
| 222 |
+
5, 107025
|
| 223 |
+
6, 107023
|
| 224 |
+
7, 107025
|
| 225 |
+
0, 0
|
| 226 |
+
1, 0
|
| 227 |
+
2, 0
|
| 228 |
+
3, 0
|
| 229 |
+
4, 0
|
| 230 |
+
5, 0
|
| 231 |
+
6, 0
|
| 232 |
+
7, 0
|
tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/run.env
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=ultraep
|
| 2 |
+
config=G3
|
| 3 |
+
family=tierA
|
| 4 |
+
repo=/workspace/Megatron-UltraEP
|
| 5 |
+
branch=ultraep-token-drop-analysis
|
| 6 |
+
commit=eaf829e7
|
| 7 |
+
num_layers=16
|
| 8 |
+
hidden=4096
|
| 9 |
+
ffn_hidden=14336
|
| 10 |
+
moe_ffn_hidden=1792
|
| 11 |
+
num_experts=64
|
| 12 |
+
topk=16
|
| 13 |
+
experts_per_rank=8
|
| 14 |
+
expert_params=22020096
|
| 15 |
+
heads=32
|
| 16 |
+
kv_groups=8
|
| 17 |
+
kv_channels=
|
| 18 |
+
qk_layernorm=0
|
| 19 |
+
shared_ffn=0
|
| 20 |
+
shared_expert_overlap=0
|
| 21 |
+
moe_token_dispatcher=flex
|
| 22 |
+
mla=0
|
| 23 |
+
dense_first_layers=0
|
| 24 |
+
moe_layers=16
|
| 25 |
+
rotary_base=1000000
|
| 26 |
+
rotary_scaling=
|
| 27 |
+
qkv_bias=0
|
| 28 |
+
router_balancing=aux_loss
|
| 29 |
+
aux_loss_coeff=1e-2
|
| 30 |
+
router_pre_softmax=0
|
| 31 |
+
router_fusion=0
|
| 32 |
+
router_dtype=fp32
|
| 33 |
+
device_initiated_grouped_gemm=1
|
| 34 |
+
norm_epsilon=1e-5
|
| 35 |
+
seq_length=4096
|
| 36 |
+
micro_batch_size=1
|
| 37 |
+
global_batch_size=16
|
| 38 |
+
num_microbatches=2
|
| 39 |
+
assignments_per_rank=65536
|
| 40 |
+
mean_tokens_per_expert=8192
|
| 41 |
+
slot_policy=matched
|
| 42 |
+
slots_per_rank=8
|
| 43 |
+
capacity_mode=static
|
| 44 |
+
capacity=1.25
|
| 45 |
+
cuda_graph=0
|
| 46 |
+
probe_mode=1
|
| 47 |
+
async_grad_reduce=0
|
| 48 |
+
branch_role=probe
|
| 49 |
+
memory_dump_mode=none
|
| 50 |
+
memory_snapshot_path=
|
| 51 |
+
cuda_graph_warmup_steps=3
|
| 52 |
+
recompute=none
|
| 53 |
+
force_balance=1
|
| 54 |
+
train_iters=12
|
| 55 |
+
ep_size=8
|
| 56 |
+
echo_dispatch_overlap=1
|
| 57 |
+
echo_dispatch_recompute=0
|
| 58 |
+
ultraep_quota_min_tokens=1024
|
| 59 |
+
ultraep_weight_sync_join=
|
| 60 |
+
profile_enable=0
|
| 61 |
+
kernel_trace=0
|
| 62 |
+
fp8_mode=mxfp8
|
| 63 |
+
fp8_recipe=mxfp8
|
| 64 |
+
training_seed=1234
|
| 65 |
+
learning_rate=1e-4
|
| 66 |
+
min_learning_rate=1.0e-5
|
| 67 |
+
lr_warmup_iters=5
|
| 68 |
+
state_dump_enable=0
|
| 69 |
+
state_dump_dir=
|
| 70 |
+
moonep_assert_no_pending_reduce=0
|
tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1/mem_samples.csv
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
0, 0
|
| 2 |
+
1, 0
|
| 3 |
+
2, 0
|
| 4 |
+
3, 0
|
| 5 |
+
4, 0
|
| 6 |
+
5, 0
|
| 7 |
+
6, 0
|
| 8 |
+
7, 0
|
| 9 |
+
0, 0
|
| 10 |
+
1, 0
|
| 11 |
+
2, 0
|
| 12 |
+
3, 0
|
| 13 |
+
4, 0
|
| 14 |
+
5, 0
|
| 15 |
+
6, 0
|
| 16 |
+
7, 0
|
| 17 |
+
0, 2
|
| 18 |
+
1, 1
|
| 19 |
+
2, 1
|
| 20 |
+
3, 1
|
| 21 |
+
4, 1
|
| 22 |
+
5, 1
|
| 23 |
+
6, 1
|
| 24 |
+
7, 1
|
| 25 |
+
0, 4
|
| 26 |
+
1, 4
|
| 27 |
+
2, 4
|
| 28 |
+
3, 4
|
| 29 |
+
4, 1
|
| 30 |
+
5, 1
|
| 31 |
+
6, 1
|
| 32 |
+
7, 1
|
| 33 |
+
0, 4
|
| 34 |
+
1, 4
|
| 35 |
+
2, 4
|
| 36 |
+
3, 4
|
| 37 |
+
4, 4
|
| 38 |
+
5, 4
|
| 39 |
+
6, 4
|
| 40 |
+
7, 4
|
| 41 |
+
0, 4
|
| 42 |
+
1, 47
|
| 43 |
+
2, 629
|
| 44 |
+
3, 629
|
| 45 |
+
4, 629
|
| 46 |
+
5, 629
|
| 47 |
+
6, 47
|
| 48 |
+
7, 629
|
| 49 |
+
0, 893
|
| 50 |
+
1, 889
|
| 51 |
+
2, 901
|
| 52 |
+
3, 897
|
| 53 |
+
4, 899
|
| 54 |
+
5, 903
|
| 55 |
+
6, 931
|
| 56 |
+
7, 925
|
| 57 |
+
0, 1895
|
| 58 |
+
1, 1895
|
| 59 |
+
2, 1895
|
| 60 |
+
3, 1895
|
| 61 |
+
4, 1895
|
| 62 |
+
5, 1895
|
| 63 |
+
6, 1895
|
| 64 |
+
7, 1895
|
| 65 |
+
0, 2387
|
| 66 |
+
1, 2385
|
| 67 |
+
2, 2387
|
| 68 |
+
3, 2381
|
| 69 |
+
4, 2409
|
| 70 |
+
5, 2397
|
| 71 |
+
6, 2397
|
| 72 |
+
7, 2409
|
| 73 |
+
0, 12469
|
| 74 |
+
1, 12249
|
| 75 |
+
2, 12701
|
| 76 |
+
3, 12933
|
| 77 |
+
4, 12389
|
| 78 |
+
5, 12013
|
| 79 |
+
6, 12553
|
| 80 |
+
7, 12641
|
| 81 |
+
0, 42989
|
| 82 |
+
1, 43437
|
| 83 |
+
2, 43437
|
| 84 |
+
3, 43437
|
| 85 |
+
4, 42989
|
| 86 |
+
5, 43437
|
| 87 |
+
6, 43437
|
| 88 |
+
7, 43437
|
| 89 |
+
0, 43741
|
| 90 |
+
1, 44189
|
| 91 |
+
2, 44189
|
| 92 |
+
3, 44187
|
| 93 |
+
4, 43741
|
| 94 |
+
5, 44187
|
| 95 |
+
6, 44189
|
| 96 |
+
7, 44189
|
| 97 |
+
0, 44429
|
| 98 |
+
1, 44877
|
| 99 |
+
2, 44877
|
| 100 |
+
3, 44877
|
| 101 |
+
4, 44429
|
| 102 |
+
5, 44877
|
| 103 |
+
6, 44877
|
| 104 |
+
7, 44877
|
| 105 |
+
0, 45551
|
| 106 |
+
1, 45999
|
| 107 |
+
2, 45999
|
| 108 |
+
3, 45999
|
| 109 |
+
4, 45551
|
| 110 |
+
5, 45999
|
| 111 |
+
6, 45999
|
| 112 |
+
7, 45999
|
| 113 |
+
0, 47173
|
| 114 |
+
1, 47621
|
| 115 |
+
2, 47621
|
| 116 |
+
3, 47621
|
| 117 |
+
4, 47173
|
| 118 |
+
5, 47621
|
| 119 |
+
6, 47621
|
| 120 |
+
7, 48751
|
| 121 |
+
0, 48303
|
| 122 |
+
1, 48751
|
| 123 |
+
2, 48751
|
| 124 |
+
3, 48751
|
| 125 |
+
4, 48303
|
| 126 |
+
5, 48751
|
| 127 |
+
6, 48751
|
| 128 |
+
7, 48751
|
| 129 |
+
0, 79811
|
| 130 |
+
1, 79807
|
| 131 |
+
2, 79807
|
| 132 |
+
3, 79807
|
| 133 |
+
4, 79831
|
| 134 |
+
5, 79807
|
| 135 |
+
6, 79807
|
| 136 |
+
7, 79807
|
| 137 |
+
0, 79811
|
| 138 |
+
1, 79807
|
| 139 |
+
2, 79807
|
| 140 |
+
3, 79807
|
| 141 |
+
4, 79831
|
| 142 |
+
5, 79807
|
| 143 |
+
6, 79807
|
| 144 |
+
7, 79807
|
| 145 |
+
0, 80935
|
| 146 |
+
1, 80931
|
| 147 |
+
2, 80931
|
| 148 |
+
3, 80931
|
| 149 |
+
4, 80955
|
| 150 |
+
5, 80931
|
| 151 |
+
6, 80931
|
| 152 |
+
7, 80931
|
| 153 |
+
0, 82247
|
| 154 |
+
1, 82257
|
| 155 |
+
2, 82247
|
| 156 |
+
3, 82249
|
| 157 |
+
4, 82287
|
| 158 |
+
5, 82257
|
| 159 |
+
6, 82277
|
| 160 |
+
7, 82263
|
| 161 |
+
0, 92769
|
| 162 |
+
1, 92825
|
| 163 |
+
2, 92813
|
| 164 |
+
3, 92833
|
| 165 |
+
4, 92889
|
| 166 |
+
5, 92823
|
| 167 |
+
6, 92801
|
| 168 |
+
7, 92851
|
| 169 |
+
0, 93829
|
| 170 |
+
1, 93821
|
| 171 |
+
2, 93837
|
| 172 |
+
3, 93895
|
| 173 |
+
4, 93889
|
| 174 |
+
5, 93907
|
| 175 |
+
6, 93865
|
| 176 |
+
7, 93911
|
| 177 |
+
0, 109059
|
| 178 |
+
1, 109393
|
| 179 |
+
2, 109113
|
| 180 |
+
3, 109233
|
| 181 |
+
4, 109217
|
| 182 |
+
5, 109193
|
| 183 |
+
6, 109313
|
| 184 |
+
7, 109313
|
| 185 |
+
0, 117507
|
| 186 |
+
1, 117581
|
| 187 |
+
2, 117581
|
| 188 |
+
3, 117581
|
| 189 |
+
4, 118667
|
| 190 |
+
5, 117581
|
| 191 |
+
6, 117581
|
| 192 |
+
7, 117581
|
| 193 |
+
0, 117507
|
| 194 |
+
1, 117581
|
| 195 |
+
2, 117581
|
| 196 |
+
3, 117581
|
| 197 |
+
4, 118667
|
| 198 |
+
5, 117581
|
| 199 |
+
6, 117581
|
| 200 |
+
7, 117581
|
| 201 |
+
0, 117507
|
| 202 |
+
1, 117581
|
| 203 |
+
2, 117581
|
| 204 |
+
3, 117581
|
| 205 |
+
4, 118667
|
| 206 |
+
5, 117581
|
| 207 |
+
6, 117581
|
| 208 |
+
7, 117581
|
| 209 |
+
0, 113993
|
| 210 |
+
1, 114067
|
| 211 |
+
2, 114067
|
| 212 |
+
3, 114067
|
| 213 |
+
4, 115153
|
| 214 |
+
5, 114067
|
| 215 |
+
6, 114067
|
| 216 |
+
7, 114067
|
| 217 |
+
0, 112313
|
| 218 |
+
1, 112387
|
| 219 |
+
2, 112387
|
| 220 |
+
3, 112387
|
| 221 |
+
4, 113473
|
| 222 |
+
5, 112387
|
| 223 |
+
6, 112387
|
| 224 |
+
7, 112387
|
| 225 |
+
0, 1
|
| 226 |
+
1, 4
|
| 227 |
+
2, 4
|
| 228 |
+
3, 4
|
| 229 |
+
4, 132
|
| 230 |
+
5, 111999
|
| 231 |
+
6, 644
|
| 232 |
+
7, 4
|
tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1/run.env
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=ultraep
|
| 2 |
+
config=G4
|
| 3 |
+
family=tierA
|
| 4 |
+
repo=/workspace/Megatron-UltraEP
|
| 5 |
+
branch=ultraep-token-drop-analysis
|
| 6 |
+
commit=eaf829e7
|
| 7 |
+
num_layers=16
|
| 8 |
+
hidden=4096
|
| 9 |
+
ffn_hidden=14336
|
| 10 |
+
moe_ffn_hidden=896
|
| 11 |
+
num_experts=128
|
| 12 |
+
topk=32
|
| 13 |
+
experts_per_rank=16
|
| 14 |
+
expert_params=11010048
|
| 15 |
+
heads=32
|
| 16 |
+
kv_groups=8
|
| 17 |
+
kv_channels=
|
| 18 |
+
qk_layernorm=0
|
| 19 |
+
shared_ffn=0
|
| 20 |
+
shared_expert_overlap=0
|
| 21 |
+
moe_token_dispatcher=flex
|
| 22 |
+
mla=0
|
| 23 |
+
dense_first_layers=0
|
| 24 |
+
moe_layers=16
|
| 25 |
+
rotary_base=1000000
|
| 26 |
+
rotary_scaling=
|
| 27 |
+
qkv_bias=0
|
| 28 |
+
router_balancing=aux_loss
|
| 29 |
+
aux_loss_coeff=1e-2
|
| 30 |
+
router_pre_softmax=0
|
| 31 |
+
router_fusion=0
|
| 32 |
+
router_dtype=fp32
|
| 33 |
+
device_initiated_grouped_gemm=1
|
| 34 |
+
norm_epsilon=1e-5
|
| 35 |
+
seq_length=4096
|
| 36 |
+
micro_batch_size=1
|
| 37 |
+
global_batch_size=16
|
| 38 |
+
num_microbatches=2
|
| 39 |
+
assignments_per_rank=131072
|
| 40 |
+
mean_tokens_per_expert=8192
|
| 41 |
+
slot_policy=matched
|
| 42 |
+
slots_per_rank=16
|
| 43 |
+
capacity_mode=static
|
| 44 |
+
capacity=1.095703125
|
| 45 |
+
cuda_graph=0
|
| 46 |
+
probe_mode=1
|
| 47 |
+
async_grad_reduce=0
|
| 48 |
+
branch_role=probe
|
| 49 |
+
memory_dump_mode=none
|
| 50 |
+
memory_snapshot_path=
|
| 51 |
+
cuda_graph_warmup_steps=3
|
| 52 |
+
recompute=none
|
| 53 |
+
force_balance=0
|
| 54 |
+
train_iters=12
|
| 55 |
+
ep_size=8
|
| 56 |
+
echo_dispatch_overlap=1
|
| 57 |
+
echo_dispatch_recompute=0
|
| 58 |
+
ultraep_quota_min_tokens=1024
|
| 59 |
+
ultraep_weight_sync_join=
|
| 60 |
+
profile_enable=0
|
| 61 |
+
kernel_trace=0
|
| 62 |
+
fp8_mode=mxfp8
|
| 63 |
+
fp8_recipe=mxfp8
|
| 64 |
+
training_seed=1234
|
| 65 |
+
learning_rate=1e-4
|
| 66 |
+
min_learning_rate=1.0e-5
|
| 67 |
+
lr_warmup_iters=5
|
| 68 |
+
state_dump_enable=0
|
| 69 |
+
state_dump_dir=
|
| 70 |
+
moonep_assert_no_pending_reduce=0
|
tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1/train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1/mem_samples.csv
ADDED
|
@@ -0,0 +1,224 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
0, 0
|
| 2 |
+
1, 0
|
| 3 |
+
2, 0
|
| 4 |
+
3, 0
|
| 5 |
+
4, 0
|
| 6 |
+
5, 0
|
| 7 |
+
6, 0
|
| 8 |
+
7, 0
|
| 9 |
+
0, 0
|
| 10 |
+
1, 0
|
| 11 |
+
2, 0
|
| 12 |
+
3, 0
|
| 13 |
+
4, 0
|
| 14 |
+
5, 0
|
| 15 |
+
6, 0
|
| 16 |
+
7, 0
|
| 17 |
+
0, 2
|
| 18 |
+
1, 1
|
| 19 |
+
2, 1
|
| 20 |
+
3, 1
|
| 21 |
+
4, 1
|
| 22 |
+
5, 1
|
| 23 |
+
6, 1
|
| 24 |
+
7, 1
|
| 25 |
+
0, 4
|
| 26 |
+
1, 4
|
| 27 |
+
2, 4
|
| 28 |
+
3, 4
|
| 29 |
+
4, 1
|
| 30 |
+
5, 1
|
| 31 |
+
6, 1
|
| 32 |
+
7, 1
|
| 33 |
+
0, 4
|
| 34 |
+
1, 4
|
| 35 |
+
2, 4
|
| 36 |
+
3, 4
|
| 37 |
+
4, 4
|
| 38 |
+
5, 4
|
| 39 |
+
6, 4
|
| 40 |
+
7, 4
|
| 41 |
+
0, 821
|
| 42 |
+
1, 819
|
| 43 |
+
2, 819
|
| 44 |
+
3, 819
|
| 45 |
+
4, 819
|
| 46 |
+
5, 819
|
| 47 |
+
6, 819
|
| 48 |
+
7, 821
|
| 49 |
+
0, 1537
|
| 50 |
+
1, 1537
|
| 51 |
+
2, 1537
|
| 52 |
+
3, 1537
|
| 53 |
+
4, 1537
|
| 54 |
+
5, 1537
|
| 55 |
+
6, 1537
|
| 56 |
+
7, 1537
|
| 57 |
+
0, 2177
|
| 58 |
+
1, 2177
|
| 59 |
+
2, 2177
|
| 60 |
+
3, 2177
|
| 61 |
+
4, 2177
|
| 62 |
+
5, 2177
|
| 63 |
+
6, 2177
|
| 64 |
+
7, 2177
|
| 65 |
+
0, 2903
|
| 66 |
+
1, 2903
|
| 67 |
+
2, 2903
|
| 68 |
+
3, 2903
|
| 69 |
+
4, 2903
|
| 70 |
+
5, 2903
|
| 71 |
+
6, 2903
|
| 72 |
+
7, 2903
|
| 73 |
+
0, 42653
|
| 74 |
+
1, 43135
|
| 75 |
+
2, 43145
|
| 76 |
+
3, 43093
|
| 77 |
+
4, 42693
|
| 78 |
+
5, 43137
|
| 79 |
+
6, 43141
|
| 80 |
+
7, 43137
|
| 81 |
+
0, 43573
|
| 82 |
+
1, 44027
|
| 83 |
+
2, 44023
|
| 84 |
+
3, 44033
|
| 85 |
+
4, 43575
|
| 86 |
+
5, 44035
|
| 87 |
+
6, 44049
|
| 88 |
+
7, 44037
|
| 89 |
+
0, 44429
|
| 90 |
+
1, 44877
|
| 91 |
+
2, 44877
|
| 92 |
+
3, 44877
|
| 93 |
+
4, 44429
|
| 94 |
+
5, 44877
|
| 95 |
+
6, 44877
|
| 96 |
+
7, 44877
|
| 97 |
+
0, 45709
|
| 98 |
+
1, 46157
|
| 99 |
+
2, 46157
|
| 100 |
+
3, 46157
|
| 101 |
+
4, 45709
|
| 102 |
+
5, 46157
|
| 103 |
+
6, 46157
|
| 104 |
+
7, 46157
|
| 105 |
+
0, 47555
|
| 106 |
+
1, 48003
|
| 107 |
+
2, 48003
|
| 108 |
+
3, 48003
|
| 109 |
+
4, 47555
|
| 110 |
+
5, 48003
|
| 111 |
+
6, 48003
|
| 112 |
+
7, 48003
|
| 113 |
+
0, 48843
|
| 114 |
+
1, 49291
|
| 115 |
+
2, 49291
|
| 116 |
+
3, 49291
|
| 117 |
+
4, 48843
|
| 118 |
+
5, 49291
|
| 119 |
+
6, 49291
|
| 120 |
+
7, 49291
|
| 121 |
+
0, 82765
|
| 122 |
+
1, 82813
|
| 123 |
+
2, 82813
|
| 124 |
+
3, 82813
|
| 125 |
+
4, 82785
|
| 126 |
+
5, 82813
|
| 127 |
+
6, 82813
|
| 128 |
+
7, 82813
|
| 129 |
+
0, 82765
|
| 130 |
+
1, 82813
|
| 131 |
+
2, 82813
|
| 132 |
+
3, 82813
|
| 133 |
+
4, 82785
|
| 134 |
+
5, 82813
|
| 135 |
+
6, 82813
|
| 136 |
+
7, 82813
|
| 137 |
+
0, 84045
|
| 138 |
+
1, 84093
|
| 139 |
+
2, 84093
|
| 140 |
+
3, 84093
|
| 141 |
+
4, 84065
|
| 142 |
+
5, 84093
|
| 143 |
+
6, 84093
|
| 144 |
+
7, 84093
|
| 145 |
+
0, 85339
|
| 146 |
+
1, 85387
|
| 147 |
+
2, 85387
|
| 148 |
+
3, 84107
|
| 149 |
+
4, 85359
|
| 150 |
+
5, 85387
|
| 151 |
+
6, 85387
|
| 152 |
+
7, 85387
|
| 153 |
+
0, 95747
|
| 154 |
+
1, 95789
|
| 155 |
+
2, 95855
|
| 156 |
+
3, 95827
|
| 157 |
+
4, 95823
|
| 158 |
+
5, 95855
|
| 159 |
+
6, 95827
|
| 160 |
+
7, 95803
|
| 161 |
+
0, 96763
|
| 162 |
+
1, 96823
|
| 163 |
+
2, 96831
|
| 164 |
+
3, 96831
|
| 165 |
+
4, 96829
|
| 166 |
+
5, 96847
|
| 167 |
+
6, 96839
|
| 168 |
+
7, 96901
|
| 169 |
+
0, 113447
|
| 170 |
+
1, 113515
|
| 171 |
+
2, 112215
|
| 172 |
+
3, 113515
|
| 173 |
+
4, 111627
|
| 174 |
+
5, 113515
|
| 175 |
+
6, 112235
|
| 176 |
+
7, 112175
|
| 177 |
+
0, 120387
|
| 178 |
+
1, 120475
|
| 179 |
+
2, 120475
|
| 180 |
+
3, 120475
|
| 181 |
+
4, 120487
|
| 182 |
+
5, 120475
|
| 183 |
+
6, 120475
|
| 184 |
+
7, 120475
|
| 185 |
+
0, 120387
|
| 186 |
+
1, 120475
|
| 187 |
+
2, 120475
|
| 188 |
+
3, 120475
|
| 189 |
+
4, 120487
|
| 190 |
+
5, 120475
|
| 191 |
+
6, 120475
|
| 192 |
+
7, 120475
|
| 193 |
+
0, 120387
|
| 194 |
+
1, 120475
|
| 195 |
+
2, 120475
|
| 196 |
+
3, 120475
|
| 197 |
+
4, 120487
|
| 198 |
+
5, 120475
|
| 199 |
+
6, 120475
|
| 200 |
+
7, 120475
|
| 201 |
+
0, 119433
|
| 202 |
+
1, 119521
|
| 203 |
+
2, 119521
|
| 204 |
+
3, 119521
|
| 205 |
+
4, 119533
|
| 206 |
+
5, 119521
|
| 207 |
+
6, 119521
|
| 208 |
+
7, 119521
|
| 209 |
+
0, 115193
|
| 210 |
+
1, 114893
|
| 211 |
+
2, 115281
|
| 212 |
+
3, 115281
|
| 213 |
+
4, 115293
|
| 214 |
+
5, 115281
|
| 215 |
+
6, 115281
|
| 216 |
+
7, 115281
|
| 217 |
+
0, 3
|
| 218 |
+
1, 4
|
| 219 |
+
2, 68
|
| 220 |
+
3, 113863
|
| 221 |
+
4, 324
|
| 222 |
+
5, 4
|
| 223 |
+
6, 4
|
| 224 |
+
7, 4
|
tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1/run.env
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=ultraep
|
| 2 |
+
config=G4
|
| 3 |
+
family=tierA
|
| 4 |
+
repo=/workspace/Megatron-UltraEP
|
| 5 |
+
branch=ultraep-token-drop-analysis
|
| 6 |
+
commit=eaf829e7
|
| 7 |
+
num_layers=16
|
| 8 |
+
hidden=4096
|
| 9 |
+
ffn_hidden=14336
|
| 10 |
+
moe_ffn_hidden=896
|
| 11 |
+
num_experts=128
|
| 12 |
+
topk=32
|
| 13 |
+
experts_per_rank=16
|
| 14 |
+
expert_params=11010048
|
| 15 |
+
heads=32
|
| 16 |
+
kv_groups=8
|
| 17 |
+
kv_channels=
|
| 18 |
+
qk_layernorm=0
|
| 19 |
+
shared_ffn=0
|
| 20 |
+
shared_expert_overlap=0
|
| 21 |
+
moe_token_dispatcher=flex
|
| 22 |
+
mla=0
|
| 23 |
+
dense_first_layers=0
|
| 24 |
+
moe_layers=16
|
| 25 |
+
rotary_base=1000000
|
| 26 |
+
rotary_scaling=
|
| 27 |
+
qkv_bias=0
|
| 28 |
+
router_balancing=aux_loss
|
| 29 |
+
aux_loss_coeff=1e-2
|
| 30 |
+
router_pre_softmax=0
|
| 31 |
+
router_fusion=0
|
| 32 |
+
router_dtype=fp32
|
| 33 |
+
device_initiated_grouped_gemm=1
|
| 34 |
+
norm_epsilon=1e-5
|
| 35 |
+
seq_length=4096
|
| 36 |
+
micro_batch_size=1
|
| 37 |
+
global_batch_size=16
|
| 38 |
+
num_microbatches=2
|
| 39 |
+
assignments_per_rank=131072
|
| 40 |
+
mean_tokens_per_expert=8192
|
| 41 |
+
slot_policy=matched
|
| 42 |
+
slots_per_rank=16
|
| 43 |
+
capacity_mode=static
|
| 44 |
+
capacity=1.25
|
| 45 |
+
cuda_graph=0
|
| 46 |
+
probe_mode=1
|
| 47 |
+
async_grad_reduce=0
|
| 48 |
+
branch_role=probe
|
| 49 |
+
memory_dump_mode=none
|
| 50 |
+
memory_snapshot_path=
|
| 51 |
+
cuda_graph_warmup_steps=3
|
| 52 |
+
recompute=none
|
| 53 |
+
force_balance=0
|
| 54 |
+
train_iters=12
|
| 55 |
+
ep_size=8
|
| 56 |
+
echo_dispatch_overlap=1
|
| 57 |
+
echo_dispatch_recompute=0
|
| 58 |
+
ultraep_quota_min_tokens=1024
|
| 59 |
+
ultraep_weight_sync_join=
|
| 60 |
+
profile_enable=0
|
| 61 |
+
kernel_trace=0
|
| 62 |
+
fp8_mode=mxfp8
|
| 63 |
+
fp8_recipe=mxfp8
|
| 64 |
+
training_seed=1234
|
| 65 |
+
learning_rate=1e-4
|
| 66 |
+
min_learning_rate=1.0e-5
|
| 67 |
+
lr_warmup_iters=5
|
| 68 |
+
state_dump_enable=0
|
| 69 |
+
state_dump_dir=
|
| 70 |
+
moonep_assert_no_pending_reduce=0
|
tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1/train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/mem_samples.csv
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
0, 0
|
| 2 |
+
1, 0
|
| 3 |
+
2, 0
|
| 4 |
+
3, 0
|
| 5 |
+
4, 0
|
| 6 |
+
5, 0
|
| 7 |
+
6, 0
|
| 8 |
+
7, 0
|
| 9 |
+
0, 0
|
| 10 |
+
1, 0
|
| 11 |
+
2, 0
|
| 12 |
+
3, 0
|
| 13 |
+
4, 0
|
| 14 |
+
5, 0
|
| 15 |
+
6, 0
|
| 16 |
+
7, 0
|
| 17 |
+
0, 2
|
| 18 |
+
1, 1
|
| 19 |
+
2, 1
|
| 20 |
+
3, 1
|
| 21 |
+
4, 1
|
| 22 |
+
5, 1
|
| 23 |
+
6, 1
|
| 24 |
+
7, 1
|
| 25 |
+
0, 4
|
| 26 |
+
1, 4
|
| 27 |
+
2, 4
|
| 28 |
+
3, 4
|
| 29 |
+
4, 1
|
| 30 |
+
5, 1
|
| 31 |
+
6, 1
|
| 32 |
+
7, 1
|
| 33 |
+
0, 4
|
| 34 |
+
1, 4
|
| 35 |
+
2, 4
|
| 36 |
+
3, 4
|
| 37 |
+
4, 4
|
| 38 |
+
5, 4
|
| 39 |
+
6, 4
|
| 40 |
+
7, 4
|
| 41 |
+
0, 35
|
| 42 |
+
1, 725
|
| 43 |
+
2, 725
|
| 44 |
+
3, 725
|
| 45 |
+
4, 725
|
| 46 |
+
5, 725
|
| 47 |
+
6, 725
|
| 48 |
+
7, 725
|
| 49 |
+
0, 947
|
| 50 |
+
1, 955
|
| 51 |
+
2, 955
|
| 52 |
+
3, 969
|
| 53 |
+
4, 957
|
| 54 |
+
5, 973
|
| 55 |
+
6, 977
|
| 56 |
+
7, 969
|
| 57 |
+
0, 1895
|
| 58 |
+
1, 1895
|
| 59 |
+
2, 1895
|
| 60 |
+
3, 1895
|
| 61 |
+
4, 1895
|
| 62 |
+
5, 1895
|
| 63 |
+
6, 1895
|
| 64 |
+
7, 1895
|
| 65 |
+
0, 2411
|
| 66 |
+
1, 2441
|
| 67 |
+
2, 2413
|
| 68 |
+
3, 2429
|
| 69 |
+
4, 2445
|
| 70 |
+
5, 2421
|
| 71 |
+
6, 2443
|
| 72 |
+
7, 2427
|
| 73 |
+
0, 10421
|
| 74 |
+
1, 10853
|
| 75 |
+
2, 10933
|
| 76 |
+
3, 10933
|
| 77 |
+
4, 10653
|
| 78 |
+
5, 10421
|
| 79 |
+
6, 10853
|
| 80 |
+
7, 10613
|
| 81 |
+
0, 42851
|
| 82 |
+
1, 43299
|
| 83 |
+
2, 43299
|
| 84 |
+
3, 43299
|
| 85 |
+
4, 42851
|
| 86 |
+
5, 43299
|
| 87 |
+
6, 43299
|
| 88 |
+
7, 43299
|
| 89 |
+
0, 43745
|
| 90 |
+
1, 44193
|
| 91 |
+
2, 44193
|
| 92 |
+
3, 44193
|
| 93 |
+
4, 43745
|
| 94 |
+
5, 44193
|
| 95 |
+
6, 44193
|
| 96 |
+
7, 44193
|
| 97 |
+
0, 44429
|
| 98 |
+
1, 44877
|
| 99 |
+
2, 44877
|
| 100 |
+
3, 44877
|
| 101 |
+
4, 44429
|
| 102 |
+
5, 44877
|
| 103 |
+
6, 44877
|
| 104 |
+
7, 44877
|
| 105 |
+
0, 45709
|
| 106 |
+
1, 46157
|
| 107 |
+
2, 46157
|
| 108 |
+
3, 46157
|
| 109 |
+
4, 45709
|
| 110 |
+
5, 46157
|
| 111 |
+
6, 46157
|
| 112 |
+
7, 46157
|
| 113 |
+
0, 48843
|
| 114 |
+
1, 49291
|
| 115 |
+
2, 49291
|
| 116 |
+
3, 49291
|
| 117 |
+
4, 48843
|
| 118 |
+
5, 49291
|
| 119 |
+
6, 49291
|
| 120 |
+
7, 49291
|
| 121 |
+
0, 65585
|
| 122 |
+
1, 65619
|
| 123 |
+
2, 65619
|
| 124 |
+
3, 65617
|
| 125 |
+
4, 65611
|
| 126 |
+
5, 65613
|
| 127 |
+
6, 65615
|
| 128 |
+
7, 65613
|
| 129 |
+
0, 82765
|
| 130 |
+
1, 82813
|
| 131 |
+
2, 82813
|
| 132 |
+
3, 82813
|
| 133 |
+
4, 82785
|
| 134 |
+
5, 82813
|
| 135 |
+
6, 82813
|
| 136 |
+
7, 82813
|
| 137 |
+
0, 84045
|
| 138 |
+
1, 84093
|
| 139 |
+
2, 84093
|
| 140 |
+
3, 84093
|
| 141 |
+
4, 84065
|
| 142 |
+
5, 84093
|
| 143 |
+
6, 84093
|
| 144 |
+
7, 84093
|
| 145 |
+
0, 84045
|
| 146 |
+
1, 84093
|
| 147 |
+
2, 84093
|
| 148 |
+
3, 84093
|
| 149 |
+
4, 84065
|
| 150 |
+
5, 84093
|
| 151 |
+
6, 84093
|
| 152 |
+
7, 84093
|
| 153 |
+
0, 95279
|
| 154 |
+
1, 95407
|
| 155 |
+
2, 95377
|
| 156 |
+
3, 95417
|
| 157 |
+
4, 95399
|
| 158 |
+
5, 95497
|
| 159 |
+
6, 95457
|
| 160 |
+
7, 95477
|
| 161 |
+
0, 96129
|
| 162 |
+
1, 96143
|
| 163 |
+
2, 96171
|
| 164 |
+
3, 96169
|
| 165 |
+
4, 96139
|
| 166 |
+
5, 96165
|
| 167 |
+
6, 96173
|
| 168 |
+
7, 96157
|
| 169 |
+
0, 97173
|
| 170 |
+
1, 97195
|
| 171 |
+
2, 97211
|
| 172 |
+
3, 97187
|
| 173 |
+
4, 97233
|
| 174 |
+
5, 97223
|
| 175 |
+
6, 97179
|
| 176 |
+
7, 97205
|
| 177 |
+
0, 120387
|
| 178 |
+
1, 120475
|
| 179 |
+
2, 120475
|
| 180 |
+
3, 120475
|
| 181 |
+
4, 120487
|
| 182 |
+
5, 120475
|
| 183 |
+
6, 120475
|
| 184 |
+
7, 120475
|
| 185 |
+
0, 120387
|
| 186 |
+
1, 120475
|
| 187 |
+
2, 120475
|
| 188 |
+
3, 120475
|
| 189 |
+
4, 120487
|
| 190 |
+
5, 120475
|
| 191 |
+
6, 120475
|
| 192 |
+
7, 120475
|
| 193 |
+
0, 120387
|
| 194 |
+
1, 120475
|
| 195 |
+
2, 120475
|
| 196 |
+
3, 120475
|
| 197 |
+
4, 120487
|
| 198 |
+
5, 120475
|
| 199 |
+
6, 120475
|
| 200 |
+
7, 120475
|
| 201 |
+
0, 119441
|
| 202 |
+
1, 119529
|
| 203 |
+
2, 119529
|
| 204 |
+
3, 119529
|
| 205 |
+
4, 119541
|
| 206 |
+
5, 119529
|
| 207 |
+
6, 119529
|
| 208 |
+
7, 119529
|
| 209 |
+
0, 115193
|
| 210 |
+
1, 115281
|
| 211 |
+
2, 115281
|
| 212 |
+
3, 115281
|
| 213 |
+
4, 115293
|
| 214 |
+
5, 115281
|
| 215 |
+
6, 115669
|
| 216 |
+
7, 115281
|
| 217 |
+
0, 113125
|
| 218 |
+
1, 113213
|
| 219 |
+
2, 113601
|
| 220 |
+
3, 113213
|
| 221 |
+
4, 113613
|
| 222 |
+
5, 113601
|
| 223 |
+
6, 113601
|
| 224 |
+
7, 113601
|
| 225 |
+
0, 3
|
| 226 |
+
1, 4
|
| 227 |
+
2, 113601
|
| 228 |
+
3, 4
|
| 229 |
+
4, 4
|
| 230 |
+
5, 4
|
| 231 |
+
6, 4
|
| 232 |
+
7, 4
|
tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/run.env
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=ultraep
|
| 2 |
+
config=G4
|
| 3 |
+
family=tierA
|
| 4 |
+
repo=/workspace/Megatron-UltraEP
|
| 5 |
+
branch=ultraep-token-drop-analysis
|
| 6 |
+
commit=eaf829e7
|
| 7 |
+
num_layers=16
|
| 8 |
+
hidden=4096
|
| 9 |
+
ffn_hidden=14336
|
| 10 |
+
moe_ffn_hidden=896
|
| 11 |
+
num_experts=128
|
| 12 |
+
topk=32
|
| 13 |
+
experts_per_rank=16
|
| 14 |
+
expert_params=11010048
|
| 15 |
+
heads=32
|
| 16 |
+
kv_groups=8
|
| 17 |
+
kv_channels=
|
| 18 |
+
qk_layernorm=0
|
| 19 |
+
shared_ffn=0
|
| 20 |
+
shared_expert_overlap=0
|
| 21 |
+
moe_token_dispatcher=flex
|
| 22 |
+
mla=0
|
| 23 |
+
dense_first_layers=0
|
| 24 |
+
moe_layers=16
|
| 25 |
+
rotary_base=1000000
|
| 26 |
+
rotary_scaling=
|
| 27 |
+
qkv_bias=0
|
| 28 |
+
router_balancing=aux_loss
|
| 29 |
+
aux_loss_coeff=1e-2
|
| 30 |
+
router_pre_softmax=0
|
| 31 |
+
router_fusion=0
|
| 32 |
+
router_dtype=fp32
|
| 33 |
+
device_initiated_grouped_gemm=1
|
| 34 |
+
norm_epsilon=1e-5
|
| 35 |
+
seq_length=4096
|
| 36 |
+
micro_batch_size=1
|
| 37 |
+
global_batch_size=16
|
| 38 |
+
num_microbatches=2
|
| 39 |
+
assignments_per_rank=131072
|
| 40 |
+
mean_tokens_per_expert=8192
|
| 41 |
+
slot_policy=matched
|
| 42 |
+
slots_per_rank=16
|
| 43 |
+
capacity_mode=static
|
| 44 |
+
capacity=1.25
|
| 45 |
+
cuda_graph=0
|
| 46 |
+
probe_mode=1
|
| 47 |
+
async_grad_reduce=0
|
| 48 |
+
branch_role=probe
|
| 49 |
+
memory_dump_mode=none
|
| 50 |
+
memory_snapshot_path=
|
| 51 |
+
cuda_graph_warmup_steps=3
|
| 52 |
+
recompute=none
|
| 53 |
+
force_balance=1
|
| 54 |
+
train_iters=12
|
| 55 |
+
ep_size=8
|
| 56 |
+
echo_dispatch_overlap=1
|
| 57 |
+
echo_dispatch_recompute=0
|
| 58 |
+
ultraep_quota_min_tokens=1024
|
| 59 |
+
ultraep_weight_sync_join=
|
| 60 |
+
profile_enable=0
|
| 61 |
+
kernel_trace=0
|
| 62 |
+
fp8_mode=mxfp8
|
| 63 |
+
fp8_recipe=mxfp8
|
| 64 |
+
training_seed=1234
|
| 65 |
+
learning_rate=1e-4
|
| 66 |
+
min_learning_rate=1.0e-5
|
| 67 |
+
lr_warmup_iters=5
|
| 68 |
+
state_dump_enable=0
|
| 69 |
+
state_dump_dir=
|
| 70 |
+
moonep_assert_no_pending_reduce=0
|
tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/capacity_attempts.tsv
CHANGED
|
@@ -3,3 +3,10 @@ G1 ultraep measure 1.25 OK 108731 0 0.6209174791971842 1.724609 /workspace/tiera
|
|
| 3 |
G1 ultraep validate 1.8125 OK 113767 0 0.032329559326171875 2.309082 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1
|
| 4 |
G2 ultraep measure 1.25 OK 110343 1 0.0 1.010101 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1
|
| 5 |
G2 ultraep validate 1.0625 OK 107987 1 0.0 1.010101 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
G1 ultraep validate 1.8125 OK 113767 0 0.032329559326171875 2.309082 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1
|
| 4 |
G2 ultraep measure 1.25 OK 110343 1 0.0 1.010101 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1
|
| 5 |
G2 ultraep validate 1.0625 OK 107987 1 0.0 1.010101 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1
|
| 6 |
+
G2 ultraep forced 1.25 OK 110343 1 0.0 1.01001 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1
|
| 7 |
+
G3 ultraep measure 1.25 OK 114319 1 0.0 1.010056 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1
|
| 8 |
+
G3 ultraep validate 1.0625 OK 111567 1 0.0 1.010071 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1
|
| 9 |
+
G3 ultraep forced 1.25 OK 114267 1 0.0 1.01001 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1
|
| 10 |
+
G4 ultraep measure 1.25 OK 120487 1 0.0 1.042938 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1
|
| 11 |
+
G4 ultraep validate 1.095703125 OK 118667 1 0.0 1.042938 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1
|
| 12 |
+
G4 ultraep forced 1.25 OK 120487 1 0.0 1.007126 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1
|
tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/capacity_selection.tsv
CHANGED
|
@@ -1,2 +1,5 @@
|
|
| 1 |
model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir
|
| 2 |
G1 ultraep ROUNDED_CAPACITY_VALIDATION_DROPPED
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir
|
| 2 |
G1 ultraep ROUNDED_CAPACITY_VALIDATION_DROPPED
|
| 3 |
+
G2 ultraep SELECTED 1.0625 1.010101 0.0 107987 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1
|
| 4 |
+
G3 ultraep SELECTED 1.0625 1.010071 0.0 111567 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1
|
| 5 |
+
G4 ultraep SELECTED 1.095703125 1.042938 0.0 118667 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1
|
tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/probe_summary.json
ADDED
|
@@ -0,0 +1,849 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"arm": "natural",
|
| 4 |
+
"assignment_sum_mismatches": 0,
|
| 5 |
+
"aux_loss_coeff": "1e-2",
|
| 6 |
+
"backend": "ultraep",
|
| 7 |
+
"branch": "ultraep-token-drop-analysis",
|
| 8 |
+
"capacity": "1.25",
|
| 9 |
+
"capacity_mode": "static",
|
| 10 |
+
"commit": "eaf829e7",
|
| 11 |
+
"config": "G1",
|
| 12 |
+
"cuda_graph": "0",
|
| 13 |
+
"dense_first_layers": "0",
|
| 14 |
+
"drop_expected_assignments": 25165824.0,
|
| 15 |
+
"drop_rate_pct_max": 2.891776,
|
| 16 |
+
"drop_rate_pct_mean": 0.607152,
|
| 17 |
+
"drop_rate_pct_min": 0.0,
|
| 18 |
+
"drop_rate_pct_n": 96,
|
| 19 |
+
"drop_rate_pct_overall": 0.6209174791971842,
|
| 20 |
+
"dropped_tokens_sum_all_records": 156259.0,
|
| 21 |
+
"duplicate_vector_mismatches": 0,
|
| 22 |
+
"experts_per_rank": "1",
|
| 23 |
+
"force_balance": "0",
|
| 24 |
+
"global_batch_size": "16",
|
| 25 |
+
"global_snapshots": 192,
|
| 26 |
+
"incomplete_snapshots": 0,
|
| 27 |
+
"malformed_records": 0,
|
| 28 |
+
"moe_layers": "16",
|
| 29 |
+
"num_experts": "8",
|
| 30 |
+
"num_layers": "16",
|
| 31 |
+
"observed_dropped_tokens": 156259.0,
|
| 32 |
+
"peak_demand_max": 14128.0,
|
| 33 |
+
"peak_demand_mean": 10918.34375,
|
| 34 |
+
"peak_demand_min": 8275.0,
|
| 35 |
+
"peak_demand_n": 96,
|
| 36 |
+
"peak_rank_overflow_max": 3888.0,
|
| 37 |
+
"peak_rank_overflow_mean": 1076.4583333333333,
|
| 38 |
+
"peak_rank_overflow_min": 0.0,
|
| 39 |
+
"peak_rank_overflow_n": 96,
|
| 40 |
+
"post_physical_rank_cv_max": 0.6189049804826644,
|
| 41 |
+
"post_physical_rank_cv_mean": 0.08780981215252336,
|
| 42 |
+
"post_physical_rank_cv_min": 0.015802566559489504,
|
| 43 |
+
"post_physical_rank_cv_n": 192,
|
| 44 |
+
"post_physical_rank_imax_max": 1.724609375,
|
| 45 |
+
"post_physical_rank_imax_mean": 1.063955307006836,
|
| 46 |
+
"post_physical_rank_imax_min": 1.010009765625,
|
| 47 |
+
"post_physical_rank_imax_n": 192,
|
| 48 |
+
"pre_home_rank_cv_max": 1.7320508075688772,
|
| 49 |
+
"pre_home_rank_cv_mean": 1.4485647787782712,
|
| 50 |
+
"pre_home_rank_cv_min": 0.3594758110042677,
|
| 51 |
+
"pre_home_rank_cv_n": 192,
|
| 52 |
+
"pre_home_rank_imax_max": 4.0,
|
| 53 |
+
"pre_home_rank_imax_mean": 3.6084569295247397,
|
| 54 |
+
"pre_home_rank_imax_min": 1.6087646484375,
|
| 55 |
+
"pre_home_rank_imax_n": 192,
|
| 56 |
+
"records": {
|
| 57 |
+
"drop_dispatch": 1536,
|
| 58 |
+
"drop_experts": 1536,
|
| 59 |
+
"drop_setup": 1536,
|
| 60 |
+
"ultraep_drop": 96,
|
| 61 |
+
"ultraep_post": 1536,
|
| 62 |
+
"ultraep_pre": 1536
|
| 63 |
+
},
|
| 64 |
+
"required_capacity_max": 1.724609,
|
| 65 |
+
"required_capacity_mean": 1.33280565625,
|
| 66 |
+
"required_capacity_min": 1.010132,
|
| 67 |
+
"required_capacity_n": 96,
|
| 68 |
+
"router_balancing": "aux_loss",
|
| 69 |
+
"run_dir": "G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1",
|
| 70 |
+
"seq_length": "4096",
|
| 71 |
+
"shared_expert_overlap": "0",
|
| 72 |
+
"topk": "2",
|
| 73 |
+
"valid_max_rank_reduction_max": 0.7474516266861991,
|
| 74 |
+
"valid_max_rank_reduction_mean": 0.6845405986286021,
|
| 75 |
+
"valid_max_rank_reduction_min": 0.27725927612110174,
|
| 76 |
+
"valid_max_rank_reduction_n": 192,
|
| 77 |
+
"zero_drop": 0
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"arm": "natural",
|
| 81 |
+
"assignment_sum_mismatches": 0,
|
| 82 |
+
"aux_loss_coeff": "1e-2",
|
| 83 |
+
"backend": "ultraep",
|
| 84 |
+
"branch": "ultraep-token-drop-analysis",
|
| 85 |
+
"capacity": "1.8125",
|
| 86 |
+
"capacity_mode": "static",
|
| 87 |
+
"commit": "eaf829e7",
|
| 88 |
+
"config": "G1",
|
| 89 |
+
"cuda_graph": "0",
|
| 90 |
+
"dense_first_layers": "0",
|
| 91 |
+
"drop_expected_assignments": 25165824.0,
|
| 92 |
+
"drop_rate_pct_max": 1.545951,
|
| 93 |
+
"drop_rate_pct_mean": 0.03138885416666667,
|
| 94 |
+
"drop_rate_pct_min": 0.0,
|
| 95 |
+
"drop_rate_pct_n": 96,
|
| 96 |
+
"drop_rate_pct_overall": 0.032329559326171875,
|
| 97 |
+
"dropped_tokens_sum_all_records": 8136.0,
|
| 98 |
+
"duplicate_vector_mismatches": 0,
|
| 99 |
+
"experts_per_rank": "1",
|
| 100 |
+
"force_balance": "0",
|
| 101 |
+
"global_batch_size": "16",
|
| 102 |
+
"global_snapshots": 192,
|
| 103 |
+
"incomplete_snapshots": 0,
|
| 104 |
+
"malformed_records": 0,
|
| 105 |
+
"moe_layers": "16",
|
| 106 |
+
"num_experts": "8",
|
| 107 |
+
"num_layers": "16",
|
| 108 |
+
"observed_dropped_tokens": 8136.0,
|
| 109 |
+
"peak_demand_max": 18916.0,
|
| 110 |
+
"peak_demand_mean": 11069.770833333334,
|
| 111 |
+
"peak_demand_min": 8275.0,
|
| 112 |
+
"peak_demand_n": 96,
|
| 113 |
+
"peak_rank_overflow_max": 4068.0,
|
| 114 |
+
"peak_rank_overflow_mean": 84.75,
|
| 115 |
+
"peak_rank_overflow_min": 0.0,
|
| 116 |
+
"peak_rank_overflow_n": 96,
|
| 117 |
+
"post_physical_rank_cv_max": 1.0236043430060333,
|
| 118 |
+
"post_physical_rank_cv_mean": 0.09497773141059589,
|
| 119 |
+
"post_physical_rank_cv_min": 0.015224448662595366,
|
| 120 |
+
"post_physical_rank_cv_n": 192,
|
| 121 |
+
"post_physical_rank_imax_max": 2.30908203125,
|
| 122 |
+
"post_physical_rank_imax_mean": 1.0747025807698567,
|
| 123 |
+
"post_physical_rank_imax_min": 1.010009765625,
|
| 124 |
+
"post_physical_rank_imax_n": 192,
|
| 125 |
+
"pre_home_rank_cv_max": 1.7320508075688772,
|
| 126 |
+
"pre_home_rank_cv_mean": 1.444809403446171,
|
| 127 |
+
"pre_home_rank_cv_min": 0.3594758110042677,
|
| 128 |
+
"pre_home_rank_cv_n": 192,
|
| 129 |
+
"pre_home_rank_imax_max": 4.0,
|
| 130 |
+
"pre_home_rank_imax_mean": 3.6151854197184243,
|
| 131 |
+
"pre_home_rank_imax_min": 1.6087646484375,
|
| 132 |
+
"pre_home_rank_imax_n": 192,
|
| 133 |
+
"records": {
|
| 134 |
+
"drop_dispatch": 1536,
|
| 135 |
+
"drop_experts": 1536,
|
| 136 |
+
"drop_setup": 1536,
|
| 137 |
+
"ultraep_drop": 96,
|
| 138 |
+
"ultraep_post": 1536,
|
| 139 |
+
"ultraep_pre": 1536
|
| 140 |
+
},
|
| 141 |
+
"required_capacity_max": 2.309082,
|
| 142 |
+
"required_capacity_mean": 1.3512904166666668,
|
| 143 |
+
"required_capacity_min": 1.010132,
|
| 144 |
+
"required_capacity_n": 96,
|
| 145 |
+
"router_balancing": "aux_loss",
|
| 146 |
+
"run_dir": "G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1",
|
| 147 |
+
"seq_length": "4096",
|
| 148 |
+
"shared_expert_overlap": "0",
|
| 149 |
+
"topk": "2",
|
| 150 |
+
"valid_max_rank_reduction_max": 0.7474516266861991,
|
| 151 |
+
"valid_max_rank_reduction_mean": 0.6828229027074976,
|
| 152 |
+
"valid_max_rank_reduction_min": 0.27725927612110174,
|
| 153 |
+
"valid_max_rank_reduction_n": 192,
|
| 154 |
+
"zero_drop": 0
|
| 155 |
+
},
|
| 156 |
+
{
|
| 157 |
+
"arm": "natural",
|
| 158 |
+
"assignment_sum_mismatches": 0,
|
| 159 |
+
"aux_loss_coeff": "1e-2",
|
| 160 |
+
"backend": "ultraep",
|
| 161 |
+
"branch": "ultraep-token-drop-analysis",
|
| 162 |
+
"capacity": "1.0625",
|
| 163 |
+
"capacity_mode": "static",
|
| 164 |
+
"commit": "eaf829e7",
|
| 165 |
+
"config": "G2",
|
| 166 |
+
"cuda_graph": "0",
|
| 167 |
+
"dense_first_layers": "0",
|
| 168 |
+
"drop_expected_assignments": 100663296.0,
|
| 169 |
+
"drop_rate_pct_max": 0.0,
|
| 170 |
+
"drop_rate_pct_mean": 0.0,
|
| 171 |
+
"drop_rate_pct_min": 0.0,
|
| 172 |
+
"drop_rate_pct_n": 96,
|
| 173 |
+
"drop_rate_pct_overall": 0.0,
|
| 174 |
+
"dropped_tokens_sum_all_records": 0.0,
|
| 175 |
+
"duplicate_vector_mismatches": 0,
|
| 176 |
+
"experts_per_rank": "4",
|
| 177 |
+
"force_balance": "0",
|
| 178 |
+
"global_batch_size": "16",
|
| 179 |
+
"global_snapshots": 192,
|
| 180 |
+
"incomplete_snapshots": 0,
|
| 181 |
+
"malformed_records": 0,
|
| 182 |
+
"moe_layers": "16",
|
| 183 |
+
"num_experts": "32",
|
| 184 |
+
"num_layers": "16",
|
| 185 |
+
"observed_dropped_tokens": 0.0,
|
| 186 |
+
"peak_demand_max": 33099.0,
|
| 187 |
+
"peak_demand_mean": 33096.927083333336,
|
| 188 |
+
"peak_demand_min": 33095.0,
|
| 189 |
+
"peak_demand_n": 96,
|
| 190 |
+
"peak_rank_overflow_max": 0.0,
|
| 191 |
+
"peak_rank_overflow_mean": 0.0,
|
| 192 |
+
"peak_rank_overflow_min": 0.0,
|
| 193 |
+
"peak_rank_overflow_n": 96,
|
| 194 |
+
"post_physical_rank_cv_max": 0.026494891363095952,
|
| 195 |
+
"post_physical_rank_cv_mean": 0.013436746135527637,
|
| 196 |
+
"post_physical_rank_cv_min": 0.004813851711632,
|
| 197 |
+
"post_physical_rank_cv_n": 192,
|
| 198 |
+
"post_physical_rank_imax_max": 1.010101318359375,
|
| 199 |
+
"post_physical_rank_imax_mean": 1.0100231170654297,
|
| 200 |
+
"post_physical_rank_imax_min": 1.010009765625,
|
| 201 |
+
"post_physical_rank_imax_n": 192,
|
| 202 |
+
"pre_home_rank_cv_max": 1.3185407730096428,
|
| 203 |
+
"pre_home_rank_cv_mean": 0.7088423169085597,
|
| 204 |
+
"pre_home_rank_cv_min": 0.07964958686803787,
|
| 205 |
+
"pre_home_rank_cv_n": 192,
|
| 206 |
+
"pre_home_rank_imax_max": 3.99853515625,
|
| 207 |
+
"pre_home_rank_imax_mean": 2.2032313346862793,
|
| 208 |
+
"pre_home_rank_imax_min": 1.17706298828125,
|
| 209 |
+
"pre_home_rank_imax_n": 192,
|
| 210 |
+
"records": {
|
| 211 |
+
"drop_dispatch": 1536,
|
| 212 |
+
"drop_experts": 1536,
|
| 213 |
+
"drop_setup": 1536,
|
| 214 |
+
"ultraep_drop": 96,
|
| 215 |
+
"ultraep_post": 1536,
|
| 216 |
+
"ultraep_pre": 1536
|
| 217 |
+
},
|
| 218 |
+
"required_capacity_max": 1.010101,
|
| 219 |
+
"required_capacity_mean": 1.0100380104166666,
|
| 220 |
+
"required_capacity_min": 1.009979,
|
| 221 |
+
"required_capacity_n": 96,
|
| 222 |
+
"router_balancing": "aux_loss",
|
| 223 |
+
"run_dir": "G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1",
|
| 224 |
+
"seq_length": "4096",
|
| 225 |
+
"shared_expert_overlap": "0",
|
| 226 |
+
"topk": "8",
|
| 227 |
+
"valid_max_rank_reduction_max": 0.7474050555623397,
|
| 228 |
+
"valid_max_rank_reduction_mean": 0.5179521802733736,
|
| 229 |
+
"valid_max_rank_reduction_min": 0.14192377495462796,
|
| 230 |
+
"valid_max_rank_reduction_n": 192,
|
| 231 |
+
"zero_drop": 1
|
| 232 |
+
},
|
| 233 |
+
{
|
| 234 |
+
"arm": "natural",
|
| 235 |
+
"assignment_sum_mismatches": 0,
|
| 236 |
+
"aux_loss_coeff": "1e-2",
|
| 237 |
+
"backend": "ultraep",
|
| 238 |
+
"branch": "ultraep-token-drop-analysis",
|
| 239 |
+
"capacity": "1.25",
|
| 240 |
+
"capacity_mode": "static",
|
| 241 |
+
"commit": "eaf829e7",
|
| 242 |
+
"config": "G2",
|
| 243 |
+
"cuda_graph": "0",
|
| 244 |
+
"dense_first_layers": "0",
|
| 245 |
+
"drop_expected_assignments": 100663296.0,
|
| 246 |
+
"drop_rate_pct_max": 0.0,
|
| 247 |
+
"drop_rate_pct_mean": 0.0,
|
| 248 |
+
"drop_rate_pct_min": 0.0,
|
| 249 |
+
"drop_rate_pct_n": 96,
|
| 250 |
+
"drop_rate_pct_overall": 0.0,
|
| 251 |
+
"dropped_tokens_sum_all_records": 0.0,
|
| 252 |
+
"duplicate_vector_mismatches": 0,
|
| 253 |
+
"experts_per_rank": "4",
|
| 254 |
+
"force_balance": "0",
|
| 255 |
+
"global_batch_size": "16",
|
| 256 |
+
"global_snapshots": 192,
|
| 257 |
+
"incomplete_snapshots": 0,
|
| 258 |
+
"malformed_records": 0,
|
| 259 |
+
"moe_layers": "16",
|
| 260 |
+
"num_experts": "32",
|
| 261 |
+
"num_layers": "16",
|
| 262 |
+
"observed_dropped_tokens": 0.0,
|
| 263 |
+
"peak_demand_max": 33099.0,
|
| 264 |
+
"peak_demand_mean": 33096.875,
|
| 265 |
+
"peak_demand_min": 33096.0,
|
| 266 |
+
"peak_demand_n": 96,
|
| 267 |
+
"peak_rank_overflow_max": 0.0,
|
| 268 |
+
"peak_rank_overflow_mean": 0.0,
|
| 269 |
+
"peak_rank_overflow_min": 0.0,
|
| 270 |
+
"peak_rank_overflow_n": 96,
|
| 271 |
+
"post_physical_rank_cv_max": 0.026494891363095952,
|
| 272 |
+
"post_physical_rank_cv_mean": 0.013604201092033112,
|
| 273 |
+
"post_physical_rank_cv_min": 0.004841919368473151,
|
| 274 |
+
"post_physical_rank_cv_n": 192,
|
| 275 |
+
"post_physical_rank_imax_max": 1.01007080078125,
|
| 276 |
+
"post_physical_rank_imax_mean": 1.0100212097167969,
|
| 277 |
+
"post_physical_rank_imax_min": 1.010009765625,
|
| 278 |
+
"post_physical_rank_imax_n": 192,
|
| 279 |
+
"pre_home_rank_cv_max": 1.3181329746469168,
|
| 280 |
+
"pre_home_rank_cv_mean": 0.7143701949255296,
|
| 281 |
+
"pre_home_rank_cv_min": 0.22890444224298007,
|
| 282 |
+
"pre_home_rank_cv_n": 192,
|
| 283 |
+
"pre_home_rank_imax_max": 3.99853515625,
|
| 284 |
+
"pre_home_rank_imax_mean": 2.2135567665100098,
|
| 285 |
+
"pre_home_rank_imax_min": 1.33551025390625,
|
| 286 |
+
"pre_home_rank_imax_n": 192,
|
| 287 |
+
"records": {
|
| 288 |
+
"drop_dispatch": 1536,
|
| 289 |
+
"drop_experts": 1536,
|
| 290 |
+
"drop_setup": 1536,
|
| 291 |
+
"ultraep_drop": 96,
|
| 292 |
+
"ultraep_post": 1536,
|
| 293 |
+
"ultraep_pre": 1536
|
| 294 |
+
},
|
| 295 |
+
"required_capacity_max": 1.010101,
|
| 296 |
+
"required_capacity_mean": 1.0100364479166666,
|
| 297 |
+
"required_capacity_min": 1.01001,
|
| 298 |
+
"required_capacity_n": 96,
|
| 299 |
+
"router_balancing": "aux_loss",
|
| 300 |
+
"run_dir": "G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1",
|
| 301 |
+
"seq_length": "4096",
|
| 302 |
+
"shared_expert_overlap": "0",
|
| 303 |
+
"topk": "8",
|
| 304 |
+
"valid_max_rank_reduction_max": 0.7474050555623397,
|
| 305 |
+
"valid_max_rank_reduction_mean": 0.5202877868268106,
|
| 306 |
+
"valid_max_rank_reduction_min": 0.24370458388556282,
|
| 307 |
+
"valid_max_rank_reduction_n": 192,
|
| 308 |
+
"zero_drop": 1
|
| 309 |
+
},
|
| 310 |
+
{
|
| 311 |
+
"arm": "forced",
|
| 312 |
+
"assignment_sum_mismatches": 0,
|
| 313 |
+
"aux_loss_coeff": "1e-2",
|
| 314 |
+
"backend": "ultraep",
|
| 315 |
+
"branch": "ultraep-token-drop-analysis",
|
| 316 |
+
"capacity": "1.25",
|
| 317 |
+
"capacity_mode": "static",
|
| 318 |
+
"commit": "eaf829e7",
|
| 319 |
+
"config": "G2",
|
| 320 |
+
"cuda_graph": "0",
|
| 321 |
+
"dense_first_layers": "0",
|
| 322 |
+
"drop_expected_assignments": 100663296.0,
|
| 323 |
+
"drop_rate_pct_max": 0.0,
|
| 324 |
+
"drop_rate_pct_mean": 0.0,
|
| 325 |
+
"drop_rate_pct_min": 0.0,
|
| 326 |
+
"drop_rate_pct_n": 96,
|
| 327 |
+
"drop_rate_pct_overall": 0.0,
|
| 328 |
+
"dropped_tokens_sum_all_records": 0.0,
|
| 329 |
+
"duplicate_vector_mismatches": 0,
|
| 330 |
+
"experts_per_rank": "4",
|
| 331 |
+
"force_balance": "1",
|
| 332 |
+
"global_batch_size": "16",
|
| 333 |
+
"global_snapshots": 192,
|
| 334 |
+
"incomplete_snapshots": 0,
|
| 335 |
+
"malformed_records": 0,
|
| 336 |
+
"moe_layers": "16",
|
| 337 |
+
"num_experts": "32",
|
| 338 |
+
"num_layers": "16",
|
| 339 |
+
"observed_dropped_tokens": 0.0,
|
| 340 |
+
"peak_demand_max": 33096.0,
|
| 341 |
+
"peak_demand_mean": 33052.15625,
|
| 342 |
+
"peak_demand_min": 32901.0,
|
| 343 |
+
"peak_demand_n": 96,
|
| 344 |
+
"peak_rank_overflow_max": 0.0,
|
| 345 |
+
"peak_rank_overflow_mean": 0.0,
|
| 346 |
+
"peak_rank_overflow_min": 0.0,
|
| 347 |
+
"peak_rank_overflow_n": 96,
|
| 348 |
+
"post_physical_rank_cv_max": 0.00717872106815807,
|
| 349 |
+
"post_physical_rank_cv_mean": 0.004277032344446821,
|
| 350 |
+
"post_physical_rank_cv_min": 0.0016880800477701613,
|
| 351 |
+
"post_physical_rank_cv_n": 192,
|
| 352 |
+
"post_physical_rank_imax_max": 1.010009765625,
|
| 353 |
+
"post_physical_rank_imax_mean": 1.006682554880778,
|
| 354 |
+
"post_physical_rank_imax_min": 1.002349853515625,
|
| 355 |
+
"post_physical_rank_imax_n": 192,
|
| 356 |
+
"pre_home_rank_cv_max": 0.007852555942790041,
|
| 357 |
+
"pre_home_rank_cv_mean": 0.00436334506476056,
|
| 358 |
+
"pre_home_rank_cv_min": 0.0016880800477701613,
|
| 359 |
+
"pre_home_rank_cv_n": 192,
|
| 360 |
+
"pre_home_rank_imax_max": 1.01544189453125,
|
| 361 |
+
"pre_home_rank_imax_mean": 1.0068944295247395,
|
| 362 |
+
"pre_home_rank_imax_min": 1.002349853515625,
|
| 363 |
+
"pre_home_rank_imax_n": 192,
|
| 364 |
+
"records": {
|
| 365 |
+
"drop_dispatch": 1536,
|
| 366 |
+
"drop_experts": 1536,
|
| 367 |
+
"drop_setup": 1536,
|
| 368 |
+
"ultraep_drop": 96,
|
| 369 |
+
"ultraep_post": 1536,
|
| 370 |
+
"ultraep_pre": 1536
|
| 371 |
+
},
|
| 372 |
+
"required_capacity_max": 1.01001,
|
| 373 |
+
"required_capacity_mean": 1.0086718541666666,
|
| 374 |
+
"required_capacity_min": 1.004059,
|
| 375 |
+
"required_capacity_n": 96,
|
| 376 |
+
"router_balancing": "aux_loss",
|
| 377 |
+
"run_dir": "G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1",
|
| 378 |
+
"seq_length": "4096",
|
| 379 |
+
"shared_expert_overlap": "0",
|
| 380 |
+
"topk": "8",
|
| 381 |
+
"valid_max_rank_reduction_max": 0.005349522149425978,
|
| 382 |
+
"valid_max_rank_reduction_mean": 0.0002090865394183532,
|
| 383 |
+
"valid_max_rank_reduction_min": 0.0,
|
| 384 |
+
"valid_max_rank_reduction_n": 192,
|
| 385 |
+
"zero_drop": 1
|
| 386 |
+
},
|
| 387 |
+
{
|
| 388 |
+
"arm": "natural",
|
| 389 |
+
"assignment_sum_mismatches": 0,
|
| 390 |
+
"aux_loss_coeff": "1e-2",
|
| 391 |
+
"backend": "ultraep",
|
| 392 |
+
"branch": "ultraep-token-drop-analysis",
|
| 393 |
+
"capacity": "1.0625",
|
| 394 |
+
"capacity_mode": "static",
|
| 395 |
+
"commit": "eaf829e7",
|
| 396 |
+
"config": "G3",
|
| 397 |
+
"cuda_graph": "0",
|
| 398 |
+
"dense_first_layers": "0",
|
| 399 |
+
"drop_expected_assignments": 201326592.0,
|
| 400 |
+
"drop_rate_pct_max": 0.0,
|
| 401 |
+
"drop_rate_pct_mean": 0.0,
|
| 402 |
+
"drop_rate_pct_min": 0.0,
|
| 403 |
+
"drop_rate_pct_n": 96,
|
| 404 |
+
"drop_rate_pct_overall": 0.0,
|
| 405 |
+
"dropped_tokens_sum_all_records": 0.0,
|
| 406 |
+
"duplicate_vector_mismatches": 0,
|
| 407 |
+
"experts_per_rank": "8",
|
| 408 |
+
"force_balance": "0",
|
| 409 |
+
"global_batch_size": "16",
|
| 410 |
+
"global_snapshots": 192,
|
| 411 |
+
"incomplete_snapshots": 0,
|
| 412 |
+
"malformed_records": 0,
|
| 413 |
+
"moe_layers": "16",
|
| 414 |
+
"num_experts": "64",
|
| 415 |
+
"num_layers": "16",
|
| 416 |
+
"observed_dropped_tokens": 0.0,
|
| 417 |
+
"peak_demand_max": 66196.0,
|
| 418 |
+
"peak_demand_mean": 66193.28125,
|
| 419 |
+
"peak_demand_min": 66192.0,
|
| 420 |
+
"peak_demand_n": 96,
|
| 421 |
+
"peak_rank_overflow_max": 0.0,
|
| 422 |
+
"peak_rank_overflow_mean": 0.0,
|
| 423 |
+
"peak_rank_overflow_min": 0.0,
|
| 424 |
+
"peak_rank_overflow_n": 96,
|
| 425 |
+
"post_physical_rank_cv_max": 0.026489120474252784,
|
| 426 |
+
"post_physical_rank_cv_mean": 0.014233627693615905,
|
| 427 |
+
"post_physical_rank_cv_min": 0.004993448920418017,
|
| 428 |
+
"post_physical_rank_cv_n": 192,
|
| 429 |
+
"post_physical_rank_imax_max": 1.01007080078125,
|
| 430 |
+
"post_physical_rank_imax_mean": 1.0100196202596028,
|
| 431 |
+
"post_physical_rank_imax_min": 1.010009765625,
|
| 432 |
+
"post_physical_rank_imax_n": 192,
|
| 433 |
+
"pre_home_rank_cv_max": 0.8287584673770169,
|
| 434 |
+
"pre_home_rank_cv_mean": 0.47726214620952834,
|
| 435 |
+
"pre_home_rank_cv_min": 0.11019481148220499,
|
| 436 |
+
"pre_home_rank_cv_n": 192,
|
| 437 |
+
"pre_home_rank_imax_max": 2.6106719970703125,
|
| 438 |
+
"pre_home_rank_imax_mean": 1.7842089335123699,
|
| 439 |
+
"pre_home_rank_imax_min": 1.1877593994140625,
|
| 440 |
+
"pre_home_rank_imax_n": 192,
|
| 441 |
+
"records": {
|
| 442 |
+
"drop_dispatch": 1536,
|
| 443 |
+
"drop_experts": 1536,
|
| 444 |
+
"drop_setup": 1536,
|
| 445 |
+
"ultraep_drop": 96,
|
| 446 |
+
"ultraep_post": 1536,
|
| 447 |
+
"ultraep_pre": 1536
|
| 448 |
+
},
|
| 449 |
+
"required_capacity_max": 1.010071,
|
| 450 |
+
"required_capacity_mean": 1.0100292499999999,
|
| 451 |
+
"required_capacity_min": 1.01001,
|
| 452 |
+
"required_capacity_n": 96,
|
| 453 |
+
"router_balancing": "aux_loss",
|
| 454 |
+
"run_dir": "G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1",
|
| 455 |
+
"seq_length": "4096",
|
| 456 |
+
"shared_expert_overlap": "0",
|
| 457 |
+
"topk": "16",
|
| 458 |
+
"valid_max_rank_reduction_max": 0.6131226876610966,
|
| 459 |
+
"valid_max_rank_reduction_mean": 0.41320151666180793,
|
| 460 |
+
"valid_max_rank_reduction_min": 0.14965121208617566,
|
| 461 |
+
"valid_max_rank_reduction_n": 192,
|
| 462 |
+
"zero_drop": 1
|
| 463 |
+
},
|
| 464 |
+
{
|
| 465 |
+
"arm": "natural",
|
| 466 |
+
"assignment_sum_mismatches": 0,
|
| 467 |
+
"aux_loss_coeff": "1e-2",
|
| 468 |
+
"backend": "ultraep",
|
| 469 |
+
"branch": "ultraep-token-drop-analysis",
|
| 470 |
+
"capacity": "1.25",
|
| 471 |
+
"capacity_mode": "static",
|
| 472 |
+
"commit": "eaf829e7",
|
| 473 |
+
"config": "G3",
|
| 474 |
+
"cuda_graph": "0",
|
| 475 |
+
"dense_first_layers": "0",
|
| 476 |
+
"drop_expected_assignments": 201326592.0,
|
| 477 |
+
"drop_rate_pct_max": 0.0,
|
| 478 |
+
"drop_rate_pct_mean": 0.0,
|
| 479 |
+
"drop_rate_pct_min": 0.0,
|
| 480 |
+
"drop_rate_pct_n": 96,
|
| 481 |
+
"drop_rate_pct_overall": 0.0,
|
| 482 |
+
"dropped_tokens_sum_all_records": 0.0,
|
| 483 |
+
"duplicate_vector_mismatches": 0,
|
| 484 |
+
"experts_per_rank": "8",
|
| 485 |
+
"force_balance": "0",
|
| 486 |
+
"global_batch_size": "16",
|
| 487 |
+
"global_snapshots": 192,
|
| 488 |
+
"incomplete_snapshots": 0,
|
| 489 |
+
"malformed_records": 0,
|
| 490 |
+
"moe_layers": "16",
|
| 491 |
+
"num_experts": "64",
|
| 492 |
+
"num_layers": "16",
|
| 493 |
+
"observed_dropped_tokens": 0.0,
|
| 494 |
+
"peak_demand_max": 66195.0,
|
| 495 |
+
"peak_demand_mean": 66193.34375,
|
| 496 |
+
"peak_demand_min": 66192.0,
|
| 497 |
+
"peak_demand_n": 96,
|
| 498 |
+
"peak_rank_overflow_max": 0.0,
|
| 499 |
+
"peak_rank_overflow_mean": 0.0,
|
| 500 |
+
"peak_rank_overflow_min": 0.0,
|
| 501 |
+
"peak_rank_overflow_n": 96,
|
| 502 |
+
"post_physical_rank_cv_max": 0.026483352723689336,
|
| 503 |
+
"post_physical_rank_cv_mean": 0.014464507564652654,
|
| 504 |
+
"post_physical_rank_cv_min": 0.00571681991985643,
|
| 505 |
+
"post_physical_rank_cv_n": 192,
|
| 506 |
+
"post_physical_rank_imax_max": 1.0100555419921875,
|
| 507 |
+
"post_physical_rank_imax_mean": 1.0100204149882,
|
| 508 |
+
"post_physical_rank_imax_min": 1.010009765625,
|
| 509 |
+
"post_physical_rank_imax_n": 192,
|
| 510 |
+
"pre_home_rank_cv_max": 0.8316092132265209,
|
| 511 |
+
"pre_home_rank_cv_mean": 0.4753321564816673,
|
| 512 |
+
"pre_home_rank_cv_min": 0.11019481148220499,
|
| 513 |
+
"pre_home_rank_cv_n": 192,
|
| 514 |
+
"pre_home_rank_imax_max": 2.8477020263671875,
|
| 515 |
+
"pre_home_rank_imax_mean": 1.7751669883728027,
|
| 516 |
+
"pre_home_rank_imax_min": 1.1877593994140625,
|
| 517 |
+
"pre_home_rank_imax_n": 192,
|
| 518 |
+
"records": {
|
| 519 |
+
"drop_dispatch": 1536,
|
| 520 |
+
"drop_experts": 1536,
|
| 521 |
+
"drop_setup": 1536,
|
| 522 |
+
"ultraep_drop": 96,
|
| 523 |
+
"ultraep_post": 1536,
|
| 524 |
+
"ultraep_pre": 1536
|
| 525 |
+
},
|
| 526 |
+
"required_capacity_max": 1.010056,
|
| 527 |
+
"required_capacity_mean": 1.01003021875,
|
| 528 |
+
"required_capacity_min": 1.01001,
|
| 529 |
+
"required_capacity_n": 96,
|
| 530 |
+
"router_balancing": "aux_loss",
|
| 531 |
+
"run_dir": "G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1",
|
| 532 |
+
"seq_length": "4096",
|
| 533 |
+
"shared_expert_overlap": "0",
|
| 534 |
+
"topk": "16",
|
| 535 |
+
"valid_max_rank_reduction_max": 0.6453085566397145,
|
| 536 |
+
"valid_max_rank_reduction_mean": 0.41069380212188733,
|
| 537 |
+
"valid_max_rank_reduction_min": 0.14965121208617566,
|
| 538 |
+
"valid_max_rank_reduction_n": 192,
|
| 539 |
+
"zero_drop": 1
|
| 540 |
+
},
|
| 541 |
+
{
|
| 542 |
+
"arm": "forced",
|
| 543 |
+
"assignment_sum_mismatches": 0,
|
| 544 |
+
"aux_loss_coeff": "1e-2",
|
| 545 |
+
"backend": "ultraep",
|
| 546 |
+
"branch": "ultraep-token-drop-analysis",
|
| 547 |
+
"capacity": "1.25",
|
| 548 |
+
"capacity_mode": "static",
|
| 549 |
+
"commit": "eaf829e7",
|
| 550 |
+
"config": "G3",
|
| 551 |
+
"cuda_graph": "0",
|
| 552 |
+
"dense_first_layers": "0",
|
| 553 |
+
"drop_expected_assignments": 201326592.0,
|
| 554 |
+
"drop_rate_pct_max": 0.0,
|
| 555 |
+
"drop_rate_pct_mean": 0.0,
|
| 556 |
+
"drop_rate_pct_min": 0.0,
|
| 557 |
+
"drop_rate_pct_n": 96,
|
| 558 |
+
"drop_rate_pct_overall": 0.0,
|
| 559 |
+
"dropped_tokens_sum_all_records": 0.0,
|
| 560 |
+
"duplicate_vector_mismatches": 0,
|
| 561 |
+
"experts_per_rank": "8",
|
| 562 |
+
"force_balance": "1",
|
| 563 |
+
"global_batch_size": "16",
|
| 564 |
+
"global_snapshots": 192,
|
| 565 |
+
"incomplete_snapshots": 0,
|
| 566 |
+
"malformed_records": 0,
|
| 567 |
+
"moe_layers": "16",
|
| 568 |
+
"num_experts": "64",
|
| 569 |
+
"num_layers": "16",
|
| 570 |
+
"observed_dropped_tokens": 0.0,
|
| 571 |
+
"peak_demand_max": 66192.0,
|
| 572 |
+
"peak_demand_mean": 65961.36458333333,
|
| 573 |
+
"peak_demand_min": 65746.0,
|
| 574 |
+
"peak_demand_n": 96,
|
| 575 |
+
"peak_rank_overflow_max": 0.0,
|
| 576 |
+
"peak_rank_overflow_mean": 0.0,
|
| 577 |
+
"peak_rank_overflow_min": 0.0,
|
| 578 |
+
"peak_rank_overflow_n": 96,
|
| 579 |
+
"post_physical_rank_cv_max": 0.005936610286960278,
|
| 580 |
+
"post_physical_rank_cv_mean": 0.0030208295858681123,
|
| 581 |
+
"post_physical_rank_cv_min": 0.0014995787955522665,
|
| 582 |
+
"post_physical_rank_cv_n": 192,
|
| 583 |
+
"post_physical_rank_imax_max": 1.010009765625,
|
| 584 |
+
"post_physical_rank_imax_mean": 1.004675070444743,
|
| 585 |
+
"post_physical_rank_imax_min": 1.001800537109375,
|
| 586 |
+
"post_physical_rank_imax_n": 192,
|
| 587 |
+
"pre_home_rank_cv_max": 0.005936610286960278,
|
| 588 |
+
"pre_home_rank_cv_mean": 0.0030232844076121573,
|
| 589 |
+
"pre_home_rank_cv_min": 0.0014995787955522665,
|
| 590 |
+
"pre_home_rank_cv_n": 192,
|
| 591 |
+
"pre_home_rank_imax_max": 1.0113067626953125,
|
| 592 |
+
"pre_home_rank_imax_mean": 1.0046818256378174,
|
| 593 |
+
"pre_home_rank_imax_min": 1.001800537109375,
|
| 594 |
+
"pre_home_rank_imax_n": 192,
|
| 595 |
+
"records": {
|
| 596 |
+
"drop_dispatch": 1536,
|
| 597 |
+
"drop_experts": 1536,
|
| 598 |
+
"drop_setup": 1536,
|
| 599 |
+
"ultraep_drop": 96,
|
| 600 |
+
"ultraep_post": 1536,
|
| 601 |
+
"ultraep_pre": 1536
|
| 602 |
+
},
|
| 603 |
+
"required_capacity_max": 1.01001,
|
| 604 |
+
"required_capacity_mean": 1.0064905625,
|
| 605 |
+
"required_capacity_min": 1.003204,
|
| 606 |
+
"required_capacity_n": 96,
|
| 607 |
+
"router_balancing": "aux_loss",
|
| 608 |
+
"run_dir": "G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1",
|
| 609 |
+
"seq_length": "4096",
|
| 610 |
+
"shared_expert_overlap": "0",
|
| 611 |
+
"topk": "16",
|
| 612 |
+
"valid_max_rank_reduction_max": 0.0012824961902319055,
|
| 613 |
+
"valid_max_rank_reduction_mean": 6.679667657457841e-06,
|
| 614 |
+
"valid_max_rank_reduction_min": 0.0,
|
| 615 |
+
"valid_max_rank_reduction_n": 192,
|
| 616 |
+
"zero_drop": 1
|
| 617 |
+
},
|
| 618 |
+
{
|
| 619 |
+
"arm": "natural",
|
| 620 |
+
"assignment_sum_mismatches": 0,
|
| 621 |
+
"aux_loss_coeff": "1e-2",
|
| 622 |
+
"backend": "ultraep",
|
| 623 |
+
"branch": "ultraep-token-drop-analysis",
|
| 624 |
+
"capacity": "1.095703125",
|
| 625 |
+
"capacity_mode": "static",
|
| 626 |
+
"commit": "eaf829e7",
|
| 627 |
+
"config": "G4",
|
| 628 |
+
"cuda_graph": "0",
|
| 629 |
+
"dense_first_layers": "0",
|
| 630 |
+
"drop_expected_assignments": 402653184.0,
|
| 631 |
+
"drop_rate_pct_max": 0.0,
|
| 632 |
+
"drop_rate_pct_mean": 0.0,
|
| 633 |
+
"drop_rate_pct_min": 0.0,
|
| 634 |
+
"drop_rate_pct_n": 96,
|
| 635 |
+
"drop_rate_pct_overall": 0.0,
|
| 636 |
+
"dropped_tokens_sum_all_records": 0.0,
|
| 637 |
+
"duplicate_vector_mismatches": 0,
|
| 638 |
+
"experts_per_rank": "16",
|
| 639 |
+
"force_balance": "0",
|
| 640 |
+
"global_batch_size": "16",
|
| 641 |
+
"global_snapshots": 192,
|
| 642 |
+
"incomplete_snapshots": 0,
|
| 643 |
+
"malformed_records": 0,
|
| 644 |
+
"moe_layers": "16",
|
| 645 |
+
"num_experts": "128",
|
| 646 |
+
"num_layers": "16",
|
| 647 |
+
"observed_dropped_tokens": 0.0,
|
| 648 |
+
"peak_demand_max": 136700.0,
|
| 649 |
+
"peak_demand_mean": 132779.11458333334,
|
| 650 |
+
"peak_demand_min": 132383.0,
|
| 651 |
+
"peak_demand_n": 96,
|
| 652 |
+
"peak_rank_overflow_max": 0.0,
|
| 653 |
+
"peak_rank_overflow_mean": 0.0,
|
| 654 |
+
"peak_rank_overflow_min": 0.0,
|
| 655 |
+
"peak_rank_overflow_n": 96,
|
| 656 |
+
"post_physical_rank_cv_max": 0.026463165595102558,
|
| 657 |
+
"post_physical_rank_cv_mean": 0.014430625474517489,
|
| 658 |
+
"post_physical_rank_cv_min": 0.005656712434847103,
|
| 659 |
+
"post_physical_rank_cv_n": 192,
|
| 660 |
+
"post_physical_rank_imax_max": 1.0100250244140625,
|
| 661 |
+
"post_physical_rank_imax_mean": 1.0100077788035076,
|
| 662 |
+
"post_physical_rank_imax_min": 1.0100021362304688,
|
| 663 |
+
"post_physical_rank_imax_n": 192,
|
| 664 |
+
"pre_home_rank_cv_max": 0.607811574382448,
|
| 665 |
+
"pre_home_rank_cv_mean": 0.34190630383555115,
|
| 666 |
+
"pre_home_rank_cv_min": 0.08731759963810155,
|
| 667 |
+
"pre_home_rank_cv_n": 192,
|
| 668 |
+
"pre_home_rank_imax_max": 2.2518386840820312,
|
| 669 |
+
"pre_home_rank_imax_mean": 1.561224063237508,
|
| 670 |
+
"pre_home_rank_imax_min": 1.121063232421875,
|
| 671 |
+
"pre_home_rank_imax_n": 192,
|
| 672 |
+
"records": {
|
| 673 |
+
"drop_dispatch": 1536,
|
| 674 |
+
"drop_experts": 1536,
|
| 675 |
+
"drop_setup": 1536,
|
| 676 |
+
"ultraep_drop": 96,
|
| 677 |
+
"ultraep_post": 1536,
|
| 678 |
+
"ultraep_pre": 1536
|
| 679 |
+
},
|
| 680 |
+
"required_capacity_max": 1.042938,
|
| 681 |
+
"required_capacity_mean": 1.0130241979166665,
|
| 682 |
+
"required_capacity_min": 1.010002,
|
| 683 |
+
"required_capacity_n": 96,
|
| 684 |
+
"router_balancing": "aux_loss",
|
| 685 |
+
"run_dir": "G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1",
|
| 686 |
+
"seq_length": "4096",
|
| 687 |
+
"shared_expert_overlap": "0",
|
| 688 |
+
"topk": "32",
|
| 689 |
+
"valid_max_rank_reduction_max": 0.5514766917497027,
|
| 690 |
+
"valid_max_rank_reduction_mean": 0.3391114043117762,
|
| 691 |
+
"valid_max_rank_reduction_min": 0.099060841159657,
|
| 692 |
+
"valid_max_rank_reduction_n": 192,
|
| 693 |
+
"zero_drop": 1
|
| 694 |
+
},
|
| 695 |
+
{
|
| 696 |
+
"arm": "natural",
|
| 697 |
+
"assignment_sum_mismatches": 0,
|
| 698 |
+
"aux_loss_coeff": "1e-2",
|
| 699 |
+
"backend": "ultraep",
|
| 700 |
+
"branch": "ultraep-token-drop-analysis",
|
| 701 |
+
"capacity": "1.25",
|
| 702 |
+
"capacity_mode": "static",
|
| 703 |
+
"commit": "eaf829e7",
|
| 704 |
+
"config": "G4",
|
| 705 |
+
"cuda_graph": "0",
|
| 706 |
+
"dense_first_layers": "0",
|
| 707 |
+
"drop_expected_assignments": 402653184.0,
|
| 708 |
+
"drop_rate_pct_max": 0.0,
|
| 709 |
+
"drop_rate_pct_mean": 0.0,
|
| 710 |
+
"drop_rate_pct_min": 0.0,
|
| 711 |
+
"drop_rate_pct_n": 96,
|
| 712 |
+
"drop_rate_pct_overall": 0.0,
|
| 713 |
+
"dropped_tokens_sum_all_records": 0.0,
|
| 714 |
+
"duplicate_vector_mismatches": 0,
|
| 715 |
+
"experts_per_rank": "16",
|
| 716 |
+
"force_balance": "0",
|
| 717 |
+
"global_batch_size": "16",
|
| 718 |
+
"global_snapshots": 192,
|
| 719 |
+
"incomplete_snapshots": 0,
|
| 720 |
+
"malformed_records": 0,
|
| 721 |
+
"moe_layers": "16",
|
| 722 |
+
"num_experts": "128",
|
| 723 |
+
"num_layers": "16",
|
| 724 |
+
"observed_dropped_tokens": 0.0,
|
| 725 |
+
"peak_demand_max": 136700.0,
|
| 726 |
+
"peak_demand_mean": 132713.30208333334,
|
| 727 |
+
"peak_demand_min": 132383.0,
|
| 728 |
+
"peak_demand_n": 96,
|
| 729 |
+
"peak_rank_overflow_max": 0.0,
|
| 730 |
+
"peak_rank_overflow_mean": 0.0,
|
| 731 |
+
"peak_rank_overflow_min": 0.0,
|
| 732 |
+
"peak_rank_overflow_n": 96,
|
| 733 |
+
"post_physical_rank_cv_max": 0.02646316532015595,
|
| 734 |
+
"post_physical_rank_cv_mean": 0.01471717424328191,
|
| 735 |
+
"post_physical_rank_cv_min": 0.0057054839456758865,
|
| 736 |
+
"post_physical_rank_cv_n": 192,
|
| 737 |
+
"post_physical_rank_imax_max": 1.0100326538085938,
|
| 738 |
+
"post_physical_rank_imax_mean": 1.0100078185399373,
|
| 739 |
+
"post_physical_rank_imax_min": 1.0100021362304688,
|
| 740 |
+
"post_physical_rank_imax_n": 192,
|
| 741 |
+
"pre_home_rank_cv_max": 0.6043235604226133,
|
| 742 |
+
"pre_home_rank_cv_mean": 0.3401136548111448,
|
| 743 |
+
"pre_home_rank_cv_min": 0.08731759963810155,
|
| 744 |
+
"pre_home_rank_cv_n": 192,
|
| 745 |
+
"pre_home_rank_imax_max": 2.2515869140625,
|
| 746 |
+
"pre_home_rank_imax_mean": 1.5504355827967327,
|
| 747 |
+
"pre_home_rank_imax_min": 1.121063232421875,
|
| 748 |
+
"pre_home_rank_imax_n": 192,
|
| 749 |
+
"records": {
|
| 750 |
+
"drop_dispatch": 1536,
|
| 751 |
+
"drop_experts": 1536,
|
| 752 |
+
"drop_setup": 1536,
|
| 753 |
+
"ultraep_drop": 96,
|
| 754 |
+
"ultraep_post": 1536,
|
| 755 |
+
"ultraep_pre": 1536
|
| 756 |
+
},
|
| 757 |
+
"required_capacity_max": 1.042938,
|
| 758 |
+
"required_capacity_mean": 1.0125220937500001,
|
| 759 |
+
"required_capacity_min": 1.010002,
|
| 760 |
+
"required_capacity_n": 96,
|
| 761 |
+
"router_balancing": "aux_loss",
|
| 762 |
+
"run_dir": "G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1",
|
| 763 |
+
"seq_length": "4096",
|
| 764 |
+
"shared_expert_overlap": "0",
|
| 765 |
+
"topk": "32",
|
| 766 |
+
"valid_max_rank_reduction_max": 0.5514197614529683,
|
| 767 |
+
"valid_max_rank_reduction_mean": 0.3357737985870543,
|
| 768 |
+
"valid_max_rank_reduction_min": 0.099060841159657,
|
| 769 |
+
"valid_max_rank_reduction_n": 192,
|
| 770 |
+
"zero_drop": 1
|
| 771 |
+
},
|
| 772 |
+
{
|
| 773 |
+
"arm": "forced",
|
| 774 |
+
"assignment_sum_mismatches": 0,
|
| 775 |
+
"aux_loss_coeff": "1e-2",
|
| 776 |
+
"backend": "ultraep",
|
| 777 |
+
"branch": "ultraep-token-drop-analysis",
|
| 778 |
+
"capacity": "1.25",
|
| 779 |
+
"capacity_mode": "static",
|
| 780 |
+
"commit": "eaf829e7",
|
| 781 |
+
"config": "G4",
|
| 782 |
+
"cuda_graph": "0",
|
| 783 |
+
"dense_first_layers": "0",
|
| 784 |
+
"drop_expected_assignments": 402653184.0,
|
| 785 |
+
"drop_rate_pct_max": 0.0,
|
| 786 |
+
"drop_rate_pct_mean": 0.0,
|
| 787 |
+
"drop_rate_pct_min": 0.0,
|
| 788 |
+
"drop_rate_pct_n": 96,
|
| 789 |
+
"drop_rate_pct_overall": 0.0,
|
| 790 |
+
"dropped_tokens_sum_all_records": 0.0,
|
| 791 |
+
"duplicate_vector_mismatches": 0,
|
| 792 |
+
"experts_per_rank": "16",
|
| 793 |
+
"force_balance": "1",
|
| 794 |
+
"global_batch_size": "16",
|
| 795 |
+
"global_snapshots": 192,
|
| 796 |
+
"incomplete_snapshots": 0,
|
| 797 |
+
"malformed_records": 0,
|
| 798 |
+
"moe_layers": "16",
|
| 799 |
+
"num_experts": "128",
|
| 800 |
+
"num_layers": "16",
|
| 801 |
+
"observed_dropped_tokens": 0.0,
|
| 802 |
+
"peak_demand_max": 132006.0,
|
| 803 |
+
"peak_demand_mean": 131665.52083333334,
|
| 804 |
+
"peak_demand_min": 131398.0,
|
| 805 |
+
"peak_demand_n": 96,
|
| 806 |
+
"peak_rank_overflow_max": 0.0,
|
| 807 |
+
"peak_rank_overflow_mean": 0.0,
|
| 808 |
+
"peak_rank_overflow_min": 0.0,
|
| 809 |
+
"peak_rank_overflow_n": 96,
|
| 810 |
+
"post_physical_rank_cv_max": 0.00357507965332711,
|
| 811 |
+
"post_physical_rank_cv_mean": 0.0021567674165372404,
|
| 812 |
+
"post_physical_rank_cv_min": 0.0006899225660317828,
|
| 813 |
+
"post_physical_rank_cv_n": 192,
|
| 814 |
+
"post_physical_rank_imax_max": 1.0064849853515625,
|
| 815 |
+
"post_physical_rank_imax_mean": 1.0033422311147053,
|
| 816 |
+
"post_physical_rank_imax_min": 1.0009918212890625,
|
| 817 |
+
"post_physical_rank_imax_n": 192,
|
| 818 |
+
"pre_home_rank_cv_max": 0.00357507965332711,
|
| 819 |
+
"pre_home_rank_cv_mean": 0.0021567674165372404,
|
| 820 |
+
"pre_home_rank_cv_min": 0.0006899225660317828,
|
| 821 |
+
"pre_home_rank_cv_n": 192,
|
| 822 |
+
"pre_home_rank_imax_max": 1.0064849853515625,
|
| 823 |
+
"pre_home_rank_imax_mean": 1.0033422311147053,
|
| 824 |
+
"pre_home_rank_imax_min": 1.0009918212890625,
|
| 825 |
+
"pre_home_rank_imax_n": 192,
|
| 826 |
+
"records": {
|
| 827 |
+
"drop_dispatch": 1536,
|
| 828 |
+
"drop_experts": 1536,
|
| 829 |
+
"drop_setup": 1536,
|
| 830 |
+
"ultraep_drop": 96,
|
| 831 |
+
"ultraep_post": 1536,
|
| 832 |
+
"ultraep_pre": 1536
|
| 833 |
+
},
|
| 834 |
+
"required_capacity_max": 1.007126,
|
| 835 |
+
"required_capacity_mean": 1.0045282500000001,
|
| 836 |
+
"required_capacity_min": 1.002487,
|
| 837 |
+
"required_capacity_n": 96,
|
| 838 |
+
"router_balancing": "aux_loss",
|
| 839 |
+
"run_dir": "G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1",
|
| 840 |
+
"seq_length": "4096",
|
| 841 |
+
"shared_expert_overlap": "0",
|
| 842 |
+
"topk": "32",
|
| 843 |
+
"valid_max_rank_reduction_max": 0.0,
|
| 844 |
+
"valid_max_rank_reduction_mean": 0.0,
|
| 845 |
+
"valid_max_rank_reduction_min": 0.0,
|
| 846 |
+
"valid_max_rank_reduction_n": 192,
|
| 847 |
+
"zero_drop": 1
|
| 848 |
+
}
|
| 849 |
+
]
|
tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/probe_summary.tsv
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
run_dir arm config backend num_experts topk experts_per_rank num_layers moe_layers dense_first_layers seq_length global_batch_size capacity_mode capacity force_balance cuda_graph branch commit router_balancing aux_loss_coeff shared_expert_overlap assignment_sum_mismatches drop_expected_assignments drop_rate_pct_max drop_rate_pct_mean drop_rate_pct_min drop_rate_pct_n drop_rate_pct_overall dropped_tokens_sum_all_records duplicate_vector_mismatches global_snapshots incomplete_snapshots malformed_records observed_dropped_tokens peak_demand_max peak_demand_mean peak_demand_min peak_demand_n peak_rank_overflow_max peak_rank_overflow_mean peak_rank_overflow_min peak_rank_overflow_n post_physical_rank_cv_max post_physical_rank_cv_mean post_physical_rank_cv_min post_physical_rank_cv_n post_physical_rank_imax_max post_physical_rank_imax_mean post_physical_rank_imax_min post_physical_rank_imax_n pre_home_rank_cv_max pre_home_rank_cv_mean pre_home_rank_cv_min pre_home_rank_cv_n pre_home_rank_imax_max pre_home_rank_imax_mean pre_home_rank_imax_min pre_home_rank_imax_n required_capacity_max required_capacity_mean required_capacity_min required_capacity_n valid_max_rank_reduction_max valid_max_rank_reduction_mean valid_max_rank_reduction_min valid_max_rank_reduction_n zero_drop
|
| 2 |
+
G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1 natural G1 ultraep 8 2 1 16 16 0 4096 16 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 25165824.0 2.891776 0.607152 0.0 96 0.6209174791971842 156259.0 0 192 0 0 156259.0 14128.0 10918.34375 8275.0 96 3888.0 1076.4583333333333 0.0 96 0.6189049804826644 0.08780981215252336 0.015802566559489504 192 1.724609375 1.063955307006836 1.010009765625 192 1.7320508075688772 1.4485647787782712 0.3594758110042677 192 4.0 3.6084569295247397 1.6087646484375 192 1.724609 1.33280565625 1.010132 96 0.7474516266861991 0.6845405986286021 0.27725927612110174 192 0
|
| 3 |
+
G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1 natural G1 ultraep 8 2 1 16 16 0 4096 16 static 1.8125 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 25165824.0 1.545951 0.03138885416666667 0.0 96 0.032329559326171875 8136.0 0 192 0 0 8136.0 18916.0 11069.770833333334 8275.0 96 4068.0 84.75 0.0 96 1.0236043430060333 0.09497773141059589 0.015224448662595366 192 2.30908203125 1.0747025807698567 1.010009765625 192 1.7320508075688772 1.444809403446171 0.3594758110042677 192 4.0 3.6151854197184243 1.6087646484375 192 2.309082 1.3512904166666668 1.010132 96 0.7474516266861991 0.6828229027074976 0.27725927612110174 192 0
|
| 4 |
+
G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 natural G2 ultraep 32 8 4 16 16 0 4096 16 static 1.0625 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 100663296.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 33099.0 33096.927083333336 33095.0 96 0.0 0.0 0.0 96 0.026494891363095952 0.013436746135527637 0.004813851711632 192 1.010101318359375 1.0100231170654297 1.010009765625 192 1.3185407730096428 0.7088423169085597 0.07964958686803787 192 3.99853515625 2.2032313346862793 1.17706298828125 192 1.010101 1.0100380104166666 1.009979 96 0.7474050555623397 0.5179521802733736 0.14192377495462796 192 1
|
| 5 |
+
G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1 natural G2 ultraep 32 8 4 16 16 0 4096 16 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 100663296.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 33099.0 33096.875 33096.0 96 0.0 0.0 0.0 96 0.026494891363095952 0.013604201092033112 0.004841919368473151 192 1.01007080078125 1.0100212097167969 1.010009765625 192 1.3181329746469168 0.7143701949255296 0.22890444224298007 192 3.99853515625 2.2135567665100098 1.33551025390625 192 1.010101 1.0100364479166666 1.01001 96 0.7474050555623397 0.5202877868268106 0.24370458388556282 192 1
|
| 6 |
+
G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1 forced G2 ultraep 32 8 4 16 16 0 4096 16 static 1.25 1 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 100663296.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 33096.0 33052.15625 32901.0 96 0.0 0.0 0.0 96 0.00717872106815807 0.004277032344446821 0.0016880800477701613 192 1.010009765625 1.006682554880778 1.002349853515625 192 0.007852555942790041 0.00436334506476056 0.0016880800477701613 192 1.01544189453125 1.0068944295247395 1.002349853515625 192 1.01001 1.0086718541666666 1.004059 96 0.005349522149425978 0.0002090865394183532 0.0 192 1
|
| 7 |
+
G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 natural G3 ultraep 64 16 8 16 16 0 4096 16 static 1.0625 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 201326592.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 66196.0 66193.28125 66192.0 96 0.0 0.0 0.0 96 0.026489120474252784 0.014233627693615905 0.004993448920418017 192 1.01007080078125 1.0100196202596028 1.010009765625 192 0.8287584673770169 0.47726214620952834 0.11019481148220499 192 2.6106719970703125 1.7842089335123699 1.1877593994140625 192 1.010071 1.0100292499999999 1.01001 96 0.6131226876610966 0.41320151666180793 0.14965121208617566 192 1
|
| 8 |
+
G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1 natural G3 ultraep 64 16 8 16 16 0 4096 16 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 201326592.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 66195.0 66193.34375 66192.0 96 0.0 0.0 0.0 96 0.026483352723689336 0.014464507564652654 0.00571681991985643 192 1.0100555419921875 1.0100204149882 1.010009765625 192 0.8316092132265209 0.4753321564816673 0.11019481148220499 192 2.8477020263671875 1.7751669883728027 1.1877593994140625 192 1.010056 1.01003021875 1.01001 96 0.6453085566397145 0.41069380212188733 0.14965121208617566 192 1
|
| 9 |
+
G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1 forced G3 ultraep 64 16 8 16 16 0 4096 16 static 1.25 1 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 201326592.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 66192.0 65961.36458333333 65746.0 96 0.0 0.0 0.0 96 0.005936610286960278 0.0030208295858681123 0.0014995787955522665 192 1.010009765625 1.004675070444743 1.001800537109375 192 0.005936610286960278 0.0030232844076121573 0.0014995787955522665 192 1.0113067626953125 1.0046818256378174 1.001800537109375 192 1.01001 1.0064905625 1.003204 96 0.0012824961902319055 6.679667657457841e-06 0.0 192 1
|
| 10 |
+
G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1 natural G4 ultraep 128 32 16 16 16 0 4096 16 static 1.095703125 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 402653184.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 136700.0 132779.11458333334 132383.0 96 0.0 0.0 0.0 96 0.026463165595102558 0.014430625474517489 0.005656712434847103 192 1.0100250244140625 1.0100077788035076 1.0100021362304688 192 0.607811574382448 0.34190630383555115 0.08731759963810155 192 2.2518386840820312 1.561224063237508 1.121063232421875 192 1.042938 1.0130241979166665 1.010002 96 0.5514766917497027 0.3391114043117762 0.099060841159657 192 1
|
| 11 |
+
G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1 natural G4 ultraep 128 32 16 16 16 0 4096 16 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 402653184.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 136700.0 132713.30208333334 132383.0 96 0.0 0.0 0.0 96 0.02646316532015595 0.01471717424328191 0.0057054839456758865 192 1.0100326538085938 1.0100078185399373 1.0100021362304688 192 0.6043235604226133 0.3401136548111448 0.08731759963810155 192 2.2515869140625 1.5504355827967327 1.121063232421875 192 1.042938 1.0125220937500001 1.010002 96 0.5514197614529683 0.3357737985870543 0.099060841159657 192 1
|
| 12 |
+
G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1 forced G4 ultraep 128 32 16 16 16 0 4096 16 static 1.25 1 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 402653184.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 132006.0 131665.52083333334 131398.0 96 0.0 0.0 0.0 96 0.00357507965332711 0.0021567674165372404 0.0006899225660317828 192 1.0064849853515625 1.0033422311147053 1.0009918212890625 192 0.00357507965332711 0.0021567674165372404 0.0006899225660317828 192 1.0064849853515625 1.0033422311147053 1.0009918212890625 192 1.007126 1.0045282500000001 1.002487 96 0.0 0.0 0.0 192 1
|
tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/results.tsv
CHANGED
|
@@ -3,3 +3,10 @@ G1 ultraep matched 1 4096 16 1.25 none 1 OK 893.49 886.90 108731 /workspace/tier
|
|
| 3 |
G1 ultraep matched 1 4096 16 1.8125 none 1 OK 914.84 903.60 113767 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1
|
| 4 |
G2 ultraep matched 4 4096 16 1.25 none 1 OK 558.91 532.70 110343 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1
|
| 5 |
G2 ultraep matched 4 4096 16 1.0625 none 1 OK 551.94 527.00 107987 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
G1 ultraep matched 1 4096 16 1.8125 none 1 OK 914.84 903.60 113767 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1
|
| 4 |
G2 ultraep matched 4 4096 16 1.25 none 1 OK 558.91 532.70 110343 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1
|
| 5 |
G2 ultraep matched 4 4096 16 1.0625 none 1 OK 551.94 527.00 107987 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1
|
| 6 |
+
G2 ultraep matched 4 4096 16 1.25 none+forcebal 1 OK 426.26 422.10 110343 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1
|
| 7 |
+
G3 ultraep matched 8 4096 16 1.25 none 1 OK 536.43 529.00 114319 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1
|
| 8 |
+
G3 ultraep matched 8 4096 16 1.0625 none 1 OK 528.17 522.80 111567 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1
|
| 9 |
+
G3 ultraep matched 8 4096 16 1.25 none+forcebal 1 OK 472.34 470.60 114267 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1
|
| 10 |
+
G4 ultraep matched 16 4096 16 1.25 none 1 OK 610.06 573.90 120487 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1
|
| 11 |
+
G4 ultraep matched 16 4096 16 1.095703125 none 1 OK 594.23 563.90 118667 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1
|
| 12 |
+
G4 ultraep matched 16 4096 16 1.25 none+forcebal 1 OK 596.29 545.60 120487 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1
|