cleanup: remove misplaced root files and subfolder images
Browse files- merges.txt +0 -0
- net_capemb.ini +0 -23
- net_ctx.ini +0 -67
- net_final.ini +0 -35
- net_noise.ini +0 -106
- net_temb.ini +0 -28
- net_texenc.ini +0 -1141
- net_texenc.ini.bak +0 -27
- net_unified.ini +0 -1227
- net_vae.ini +0 -219
- net_xemb.ini +0 -20
- output.png +0 -3
- vocab.txt +0 -0
- z-image-turbo/output.png +0 -3
- z-image-turbo/seed_test_1.png +0 -3
- z-image-turbo/seed_test_2.png +0 -3
- z_image_turbo_text_encoder.cccc.bin +0 -3
- z_image_turbo_transformer_all_final_layer.cccc.bin +0 -3
- z_image_turbo_transformer_all_x_embedder.cccc.bin +0 -3
- z_image_turbo_transformer_cap_embedder.cccc.bin +0 -3
- z_image_turbo_transformer_context_refiner.cccc.bin +0 -3
- z_image_turbo_transformer_noise_refiner.cccc.bin +0 -3
- z_image_turbo_transformer_t_embedder.cccc.bin +0 -3
- z_image_turbo_transformer_unified.cccc.bin +0 -3
- z_image_turbo_vae.cccc.bin +0 -3
merges.txt
DELETED
|
The diff for this file is too large to render.
See raw diff
|
|
|
net_capemb.ini
DELETED
|
@@ -1,23 +0,0 @@
|
|
| 1 |
-
[train]
|
| 2 |
-
load_net = 1
|
| 3 |
-
load_file=z_image_turbo_transformer_cap_embedder.cccc.bin
|
| 4 |
-
train_epochs = 0
|
| 5 |
-
Batch = 1
|
| 6 |
-
gpu = 1
|
| 7 |
-
data_type = half
|
| 8 |
-
output_log = 0
|
| 9 |
-
need_free_mem = 0
|
| 10 |
-
|
| 11 |
-
[net]
|
| 12 |
-
structure='
|
| 13 |
-
B=1;
|
| 14 |
-
X = Matrix(2560, T, 1, B);
|
| 15 |
-
Ws = MatrixWithName("rmsn_1.weight", 2560, 1);
|
| 16 |
-
Xn = rmsNorm(X, Ws, 1e-5);
|
| 17 |
-
W = MatrixWithName("gemm_0.weight", 3840, 2560, 1, 1);
|
| 18 |
-
b = MatrixWithName("gemm_0.bias", 3840, 1, 1, 1);
|
| 19 |
-
Y = batchedMul(W, Xn) + b;
|
| 20 |
-
setXY(X, Y);
|
| 21 |
-
'
|
| 22 |
-
|
| 23 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
net_ctx.ini
DELETED
|
@@ -1,67 +0,0 @@
|
|
| 1 |
-
[train]
|
| 2 |
-
load_net = 1
|
| 3 |
-
load_file=z_image_turbo_transformer_context_refiner.cccc.bin
|
| 4 |
-
train_epochs = 0
|
| 5 |
-
Batch = 1
|
| 6 |
-
gpu = 1
|
| 7 |
-
data_type = half
|
| 8 |
-
output_log = 0
|
| 9 |
-
need_free_mem = 0
|
| 10 |
-
|
| 11 |
-
[net]
|
| 12 |
-
structure='
|
| 13 |
-
B=1; D=3840; DD=3968; I=10240; hd=128; HB=30;
|
| 14 |
-
Xw = Matrix(DD, T, 1, B);
|
| 15 |
-
X = sliceW(Xw, 0, D);
|
| 16 |
-
cos1 = sliceW(Xw, D, 64);
|
| 17 |
-
sin1 = sliceW(Xw, D+64, 64);
|
| 18 |
-
W_n16 = MatrixWithName("rmsn_16.weight", D, 1);
|
| 19 |
-
W_q1 = MatrixWithName("gemm_0.weight", D, D, 1, 1);
|
| 20 |
-
W_k1 = MatrixWithName("gemm_1.weight", D, D, 1, 1);
|
| 21 |
-
W_v1 = MatrixWithName("gemm_2.weight", D, D, 1, 1);
|
| 22 |
-
W_n17 = MatrixWithName("rmsn_17.weight", hd, 1);
|
| 23 |
-
W_n18 = MatrixWithName("rmsn_18.weight", hd, 1);
|
| 24 |
-
W_o1 = MatrixWithName("gemm_3.weight", D, D, 1, 1);
|
| 25 |
-
W_n19 = MatrixWithName("rmsn_19.weight", D, 1);
|
| 26 |
-
W_n20 = MatrixWithName("rmsn_20.weight", D, 1);
|
| 27 |
-
W_g1 = MatrixWithName("gemm_4.weight", I, D, 1, 1);
|
| 28 |
-
W_u1 = MatrixWithName("gemm_5.weight", I, D, 1, 1);
|
| 29 |
-
W_d1 = MatrixWithName("gemm_6.weight", D, I, 1, 1);
|
| 30 |
-
W_n21 = MatrixWithName("rmsn_21.weight", D, 1);
|
| 31 |
-
W_n22 = MatrixWithName("rmsn_22.weight", D, 1);
|
| 32 |
-
W_q2 = MatrixWithName("gemm_7.weight", D, D, 1, 1);
|
| 33 |
-
W_k2 = MatrixWithName("gemm_8.weight", D, D, 1, 1);
|
| 34 |
-
W_v2 = MatrixWithName("gemm_9.weight", D, D, 1, 1);
|
| 35 |
-
W_n23 = MatrixWithName("rmsn_23.weight", hd, 1);
|
| 36 |
-
W_n24 = MatrixWithName("rmsn_24.weight", hd, 1);
|
| 37 |
-
W_o2 = MatrixWithName("gemm_10.weight", D, D, 1, 1);
|
| 38 |
-
W_n25 = MatrixWithName("rmsn_25.weight", D, 1);
|
| 39 |
-
W_n26 = MatrixWithName("rmsn_26.weight", D, 1);
|
| 40 |
-
W_g2 = MatrixWithName("gemm_11.weight", I, D, 1, 1);
|
| 41 |
-
W_u2 = MatrixWithName("gemm_12.weight", I, D, 1, 1);
|
| 42 |
-
W_d2 = MatrixWithName("gemm_13.weight", D, I, 1, 1);
|
| 43 |
-
W_n27 = MatrixWithName("rmsn_27.weight", D, 1);
|
| 44 |
-
X1n = rmsNorm(X, W_n16, 1e-5);
|
| 45 |
-
Q1r = permute(reshape(batchedMul(W_q1, X1n), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 46 |
-
K1r = permute(reshape(batchedMul(W_k1, X1n), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 47 |
-
V1r = permute(reshape(batchedMul(W_v1, X1n), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 48 |
-
Q1n = rmsNorm(Q1r, W_n17, 1e-5);
|
| 49 |
-
K1n = rmsNorm(K1r, W_n18, 1e-5);
|
| 50 |
-
A1 = attention(rope2(Q1n, cos1, sin1), rope2(K1n, cos1, sin1), V1r, hd, 0);
|
| 51 |
-
R1 = X + rmsNorm(batchedMul(W_o1, reshape(permute(A1, {0, 3, 1, 2}), {D, T, 1, 1})), W_n19, 1e-5);
|
| 52 |
-
R1n = rmsNorm(R1, W_n20, 1e-5);
|
| 53 |
-
R1b = R1 + rmsNorm(batchedMul(W_d1, elementMul(silu(batchedMul(W_g1, R1n)), batchedMul(W_u1, R1n))), W_n21, 1e-5);
|
| 54 |
-
X2n = rmsNorm(R1b, W_n22, 1e-5);
|
| 55 |
-
Q2r = permute(reshape(batchedMul(W_q2, X2n), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 56 |
-
K2r = permute(reshape(batchedMul(W_k2, X2n), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 57 |
-
V2r = permute(reshape(batchedMul(W_v2, X2n), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 58 |
-
Q2n = rmsNorm(Q2r, W_n23, 1e-5);
|
| 59 |
-
K2n = rmsNorm(K2r, W_n24, 1e-5);
|
| 60 |
-
A2 = attention(rope2(Q2n, cos1, sin1), rope2(K2n, cos1, sin1), V2r, hd, 0);
|
| 61 |
-
R2 = R1b + rmsNorm(batchedMul(W_o2, reshape(permute(A2, {0, 3, 1, 2}), {D, T, 1, 1})), W_n25, 1e-5);
|
| 62 |
-
R2n = rmsNorm(R2, W_n26, 1e-5);
|
| 63 |
-
Y = R2 + rmsNorm(batchedMul(W_d2, elementMul(silu(batchedMul(W_g2, R2n)), batchedMul(W_u2, R2n))), W_n27, 1e-5);
|
| 64 |
-
setXY(Xw, Y);
|
| 65 |
-
'
|
| 66 |
-
|
| 67 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
net_final.ini
DELETED
|
@@ -1,35 +0,0 @@
|
|
| 1 |
-
[train]
|
| 2 |
-
load_net = 1
|
| 3 |
-
load_file=z_image_turbo_transformer_all_final_layer.cccc.bin
|
| 4 |
-
train_epochs = 0
|
| 5 |
-
Batch = 1
|
| 6 |
-
gpu = 1
|
| 7 |
-
data_type = float
|
| 8 |
-
output_log = 0
|
| 9 |
-
need_free_mem = 0
|
| 10 |
-
|
| 11 |
-
[net]
|
| 12 |
-
structure='
|
| 13 |
-
B=1; D=3840;
|
| 14 |
-
X = Matrix(D, SEQ+1, 1, B);
|
| 15 |
-
Xpm = permute(X, {1, 0, 2, 3});
|
| 16 |
-
t_slot = sliceW(Xpm, SEQ, 1);
|
| 17 |
-
t_full = permute(t_slot, {1, 0, 2, 3});
|
| 18 |
-
t256 = sliceW(t_full, 0, 256);
|
| 19 |
-
seq_pm = sliceW(Xpm, 0, P);
|
| 20 |
-
Xi = permute(seq_pm, {1, 0, 2, 3});
|
| 21 |
-
W0 = MatrixWithName("gemm_0.weight", D, 256, 1, 1);
|
| 22 |
-
b0 = MatrixWithName("gemm_0.bias", D, 1, 1, 1);
|
| 23 |
-
scale = batchedMul(W0, silu(t256)) + b0;
|
| 24 |
-
setNeedLoadState(0);
|
| 25 |
-
s_ln = Matrix(D, 1, 1, 1);
|
| 26 |
-
b_ln = Matrix(D, 1, 1, 1);
|
| 27 |
-
setNeedLoadState(1);
|
| 28 |
-
Xi_ln = layerNorm(Xi, s_ln, b_ln, 1e-6);
|
| 29 |
-
Xi_mod = Xi_ln + elementMul(Xi_ln, scale);
|
| 30 |
-
W1 = MatrixWithName("gemm_1.weight", 64, D, 1, 1);
|
| 31 |
-
b1 = MatrixWithName("gemm_1.bias", 64, 1, 1, 1);
|
| 32 |
-
Y = batchedMul(W1, Xi_mod) + b1;
|
| 33 |
-
setXY(X, Y);
|
| 34 |
-
'
|
| 35 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
net_noise.ini
DELETED
|
@@ -1,106 +0,0 @@
|
|
| 1 |
-
[train]
|
| 2 |
-
load_net = 1
|
| 3 |
-
load_file=z_image_turbo_transformer_noise_refiner.cccc.bin
|
| 4 |
-
train_epochs = 0
|
| 5 |
-
Batch = 1
|
| 6 |
-
gpu = 1
|
| 7 |
-
data_type = float
|
| 8 |
-
output_log = 0
|
| 9 |
-
need_free_mem = 0
|
| 10 |
-
|
| 11 |
-
[net]
|
| 12 |
-
structure='
|
| 13 |
-
B=1; D=3840; DD=3968; I=10240; hd=128; HB=30;
|
| 14 |
-
Xin_n = Matrix(DD, P+1, 1, B);
|
| 15 |
-
Xin_npm = permute(Xin_n, {1, 0, 2, 3});
|
| 16 |
-
t_sn = sliceW(Xin_npm, P, 1);
|
| 17 |
-
t_ful = permute(t_sn, {1, 0, 2, 3});
|
| 18 |
-
t_emb_n = sliceW(t_ful, 0, 256);
|
| 19 |
-
seq_npm = sliceW(Xin_npm, 0, P);
|
| 20 |
-
Xn0_wide = permute(seq_npm, {1, 0, 2, 3});
|
| 21 |
-
Xn0 = sliceW(Xn0_wide, 0, D);
|
| 22 |
-
cosn = sliceW(Xn0_wide, D, 64);
|
| 23 |
-
sinn = sliceW(Xn0_wide, D+64, 64);
|
| 24 |
-
Wmod_n0 = MatrixWithName("gemm_0.weight", 15360, 256, 1, 1);
|
| 25 |
-
bmod_n0 = MatrixWithName("gemm_0.bias", 15360, 1, 1, 1);
|
| 26 |
-
modn0 = batchedMul(Wmod_n0, t_emb_n) + bmod_n0;
|
| 27 |
-
sa_n0 = sliceW(modn0, 0, D);
|
| 28 |
-
ga_n0 = tanh(sliceW(modn0, D, D));
|
| 29 |
-
sf_n0 = sliceW(modn0, 2*D, D);
|
| 30 |
-
gf_n0 = tanh(sliceW(modn0, 3*D, D));
|
| 31 |
-
Wrnn22 = MatrixWithName("rmsn_22.weight", D, 1);
|
| 32 |
-
Xnn0 = rmsNorm(Xn0, Wrnn22, 1e-5);
|
| 33 |
-
Xmn0 = Xnn0 + elementMul(Xnn0, sa_n0);
|
| 34 |
-
Wqn0 = MatrixWithName("gemm_1.weight", D, D, 1, 1);
|
| 35 |
-
Wkn0 = MatrixWithName("gemm_2.weight", D, D, 1, 1);
|
| 36 |
-
Wvn0 = MatrixWithName("gemm_3.weight", D, D, 1, 1);
|
| 37 |
-
Wrnn23 = MatrixWithName("rmsn_23.weight", hd, 1);
|
| 38 |
-
Wrnn24 = MatrixWithName("rmsn_24.weight", hd, 1);
|
| 39 |
-
Wno0 = MatrixWithName("gemm_4.weight", D, D, 1, 1);
|
| 40 |
-
Wrnn25 = MatrixWithName("rmsn_25.weight", D, 1);
|
| 41 |
-
Qrn0 = permute(reshape(batchedMul(Wqn0, Xmn0), {hd, HB, P, 1}), {0, 2, 3, 1});
|
| 42 |
-
Krn0 = permute(reshape(batchedMul(Wkn0, Xmn0), {hd, HB, P, 1}), {0, 2, 3, 1});
|
| 43 |
-
Vrn0 = permute(reshape(batchedMul(Wvn0, Xmn0), {hd, HB, P, 1}), {0, 2, 3, 1});
|
| 44 |
-
Qnn0 = rmsNorm(Qrn0, Wrnn23, 1e-5);
|
| 45 |
-
Knn0 = rmsNorm(Krn0, Wrnn24, 1e-5);
|
| 46 |
-
Atn0 = attention(rope2(Qnn0, cosn, sinn), rope2(Knn0, cosn, sinn), Vrn0, hd, 0);
|
| 47 |
-
Aon0 = batchedMul(Wno0, reshape(permute(Atn0, {0, 3, 1, 2}), {D, P, 1, 1}));
|
| 48 |
-
Ann0 = rmsNorm(Aon0, Wrnn25, 1e-5);
|
| 49 |
-
Agn0 = elementMul(Ann0, ga_n0);
|
| 50 |
-
Rxn0 = Xn0 + Agn0;
|
| 51 |
-
Wrnn26 = MatrixWithName("rmsn_26.weight", D, 1);
|
| 52 |
-
Wgn0 = MatrixWithName("gemm_5.weight", I, D, 1, 1);
|
| 53 |
-
Wun0 = MatrixWithName("gemm_6.weight", I, D, 1, 1);
|
| 54 |
-
Wdn0 = MatrixWithName("gemm_7.weight", D, I, 1, 1);
|
| 55 |
-
Wrnn27 = MatrixWithName("rmsn_27.weight", D, 1);
|
| 56 |
-
Rfn0 = rmsNorm(Rxn0, Wrnn26, 1e-5);
|
| 57 |
-
Rfmn0 = Rfn0 + elementMul(Rfn0, sf_n0);
|
| 58 |
-
Fgn0 = batchedMul(Wgn0, Rfmn0);
|
| 59 |
-
Fun0 = batchedMul(Wun0, Rfmn0);
|
| 60 |
-
Ffn0 = batchedMul(Wdn0, elementMul(silu(Fgn0), Fun0));
|
| 61 |
-
Fnn0 = rmsNorm(Ffn0, Wrnn27, 1e-5);
|
| 62 |
-
Fggn0 = elementMul(Fnn0, gf_n0);
|
| 63 |
-
Xn1 = Rxn0 + Fggn0;
|
| 64 |
-
Wmod_n1 = MatrixWithName("gemm_8.weight", 15360, 256, 1, 1);
|
| 65 |
-
bmod_n1 = MatrixWithName("gemm_8.bias", 15360, 1, 1, 1);
|
| 66 |
-
modn1 = batchedMul(Wmod_n1, t_emb_n) + bmod_n1;
|
| 67 |
-
sa_n1 = sliceW(modn1, 0, D);
|
| 68 |
-
ga_n1 = tanh(sliceW(modn1, D, D));
|
| 69 |
-
sf_n1 = sliceW(modn1, 2*D, D);
|
| 70 |
-
gf_n1 = tanh(sliceW(modn1, 3*D, D));
|
| 71 |
-
Wrnn28 = MatrixWithName("rmsn_28.weight", D, 1);
|
| 72 |
-
Xnn1 = rmsNorm(Xn1, Wrnn28, 1e-5);
|
| 73 |
-
Xmn1 = Xnn1 + elementMul(Xnn1, sa_n1);
|
| 74 |
-
Wqn1 = MatrixWithName("gemm_9.weight", D, D, 1, 1);
|
| 75 |
-
Wkn1 = MatrixWithName("gemm_10.weight", D, D, 1, 1);
|
| 76 |
-
Wvn1 = MatrixWithName("gemm_11.weight", D, D, 1, 1);
|
| 77 |
-
Wrnn29 = MatrixWithName("rmsn_29.weight", hd, 1);
|
| 78 |
-
Wrnn30 = MatrixWithName("rmsn_30.weight", hd, 1);
|
| 79 |
-
Wno1 = MatrixWithName("gemm_12.weight", D, D, 1, 1);
|
| 80 |
-
Wrnn31 = MatrixWithName("rmsn_31.weight", D, 1);
|
| 81 |
-
Qrn1 = permute(reshape(batchedMul(Wqn1, Xmn1), {hd, HB, P, 1}), {0, 2, 3, 1});
|
| 82 |
-
Krn1 = permute(reshape(batchedMul(Wkn1, Xmn1), {hd, HB, P, 1}), {0, 2, 3, 1});
|
| 83 |
-
Vrn1 = permute(reshape(batchedMul(Wvn1, Xmn1), {hd, HB, P, 1}), {0, 2, 3, 1});
|
| 84 |
-
Qnn1 = rmsNorm(Qrn1, Wrnn29, 1e-5);
|
| 85 |
-
Knn1 = rmsNorm(Krn1, Wrnn30, 1e-5);
|
| 86 |
-
Atn1 = attention(rope2(Qnn1, cosn, sinn), rope2(Knn1, cosn, sinn), Vrn1, hd, 0);
|
| 87 |
-
Aon1 = batchedMul(Wno1, reshape(permute(Atn1, {0, 3, 1, 2}), {D, P, 1, 1}));
|
| 88 |
-
Ann1 = rmsNorm(Aon1, Wrnn31, 1e-5);
|
| 89 |
-
Agn1 = elementMul(Ann1, ga_n1);
|
| 90 |
-
Rxn1 = Xn1 + Agn1;
|
| 91 |
-
Wrnn32 = MatrixWithName("rmsn_32.weight", D, 1);
|
| 92 |
-
Wgn1 = MatrixWithName("gemm_13.weight", I, D, 1, 1);
|
| 93 |
-
Wun1 = MatrixWithName("gemm_14.weight", I, D, 1, 1);
|
| 94 |
-
Wdn1 = MatrixWithName("gemm_15.weight", D, I, 1, 1);
|
| 95 |
-
Wrnn33 = MatrixWithName("rmsn_33.weight", D, 1);
|
| 96 |
-
Rfn1 = rmsNorm(Rxn1, Wrnn32, 1e-5);
|
| 97 |
-
Rfmn1 = Rfn1 + elementMul(Rfn1, sf_n1);
|
| 98 |
-
Fgn1 = batchedMul(Wgn1, Rfmn1);
|
| 99 |
-
Fun1 = batchedMul(Wun1, Rfmn1);
|
| 100 |
-
Ffn1 = batchedMul(Wdn1, elementMul(silu(Fgn1), Fun1));
|
| 101 |
-
Fnn1 = rmsNorm(Ffn1, Wrnn33, 1e-5);
|
| 102 |
-
Fggn1 = elementMul(Fnn1, gf_n1);
|
| 103 |
-
Xn2 = Rxn1 + Fggn1;
|
| 104 |
-
setXY(Xin_n, Xn2);
|
| 105 |
-
'
|
| 106 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
net_temb.ini
DELETED
|
@@ -1,28 +0,0 @@
|
|
| 1 |
-
[train]
|
| 2 |
-
load_net = 1
|
| 3 |
-
load_file=z_image_turbo_transformer_t_embedder.cccc.bin
|
| 4 |
-
train_epochs = 0
|
| 5 |
-
Batch = 1
|
| 6 |
-
gpu = 1
|
| 7 |
-
data_type = half
|
| 8 |
-
output_log = 0
|
| 9 |
-
need_free_mem = 0
|
| 10 |
-
|
| 11 |
-
[net]
|
| 12 |
-
structure='
|
| 13 |
-
B=1;
|
| 14 |
-
freq = MatrixWithName("pnnx_fold_29_data.data", 128, 1);
|
| 15 |
-
t_in = Matrix(1, 1, 1, B);
|
| 16 |
-
t_sc = freq * t_in;
|
| 17 |
-
cos_p = cos(t_sc);
|
| 18 |
-
sin_p = sin(t_sc);
|
| 19 |
-
embed = reshape(concat(cos_p, sin_p), {256, 1, 1, 1});
|
| 20 |
-
W0 = MatrixWithName("gemm_0.weight", 1024, 256, 1, 1);
|
| 21 |
-
b0 = MatrixWithName("gemm_0.bias", 1024, 1, 1, 1);
|
| 22 |
-
h = silu(batchedMul(W0, embed) + b0);
|
| 23 |
-
W1 = MatrixWithName("gemm_1.weight", 256, 1024, 1, 1);
|
| 24 |
-
b1 = MatrixWithName("gemm_1.bias", 256, 1, 1, 1);
|
| 25 |
-
Y = batchedMul(W1, h) + b1;
|
| 26 |
-
setXY(t_in, Y);
|
| 27 |
-
'
|
| 28 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
net_texenc.ini
DELETED
|
@@ -1,1141 +0,0 @@
|
|
| 1 |
-
[train]
|
| 2 |
-
load_net = 1
|
| 3 |
-
load_file=z_image_turbo_text_encoder.cccc.bin
|
| 4 |
-
train_epochs = 0
|
| 5 |
-
Batch = 1
|
| 6 |
-
gpu = 1
|
| 7 |
-
data_type = half
|
| 8 |
-
output_log = 0
|
| 9 |
-
need_free_mem = 0
|
| 10 |
-
|
| 11 |
-
[net]
|
| 12 |
-
structure='
|
| 13 |
-
B=1; D=2560; Dq=4096; Dkv=1024; hd=128; HB=32; HkvB=8; HkvMul=4; I=9728; V=151936;
|
| 14 |
-
W_emb = MatrixWithName("embed_280.weight", D, 1, 1, V);
|
| 15 |
-
ids = MatrixF(T, 1, 1, B);
|
| 16 |
-
X0 = embed(ids, W_emb);
|
| 17 |
-
cos_te = ropeCosTbl(T, hd, 1000000.0);
|
| 18 |
-
sin_te = ropeSinTbl(T, hd, 1000000.0);
|
| 19 |
-
Wra0 = MatrixWithName("rmsn_281.weight", D, 1);
|
| 20 |
-
Xna0 = rmsNorm(X0, Wra0, 1e-5);
|
| 21 |
-
Wq0 = MatrixWithName("gemm_0.weight", Dq, D, 1, 1);
|
| 22 |
-
Qh0 = permute(reshape(batchedMul(Wq0, Xna0), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 23 |
-
Wqn0 = MatrixWithName("rmsn_282.weight", hd, 1);
|
| 24 |
-
Qn0 = rmsNorm(Qh0, Wqn0, 1e-5);
|
| 25 |
-
Qr0 = rope(Qn0, cos_te, sin_te);
|
| 26 |
-
Wk0 = MatrixWithName("gemm_1.weight", Dkv, D, 1, 1);
|
| 27 |
-
Kh0 = permute(reshape(batchedMul(Wk0, Xna0), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 28 |
-
Wkn0 = MatrixWithName("rmsn_283.weight", hd, 1);
|
| 29 |
-
Kn0 = rmsNorm(Kh0, Wkn0, 1e-5);
|
| 30 |
-
Kr0 = rope(Kn0, cos_te, sin_te);
|
| 31 |
-
Wv0 = MatrixWithName("gemm_2.weight", Dkv, D, 1, 1);
|
| 32 |
-
V0 = permute(reshape(batchedMul(Wv0, Xna0), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 33 |
-
Krc0 = reshapeBatch(Kr0, {hd, T, HkvB, 1});
|
| 34 |
-
Krt0 = tile(Krc0, {1, 1, 1, HkvMul});
|
| 35 |
-
Krp0 = permute(Krt0, {0, 1, 3, 2});
|
| 36 |
-
Kt0 = reshapeBatch(Krp0, {hd, T, 1, HB});
|
| 37 |
-
Vrc0 = reshapeBatch(V0, {hd, T, HkvB, 1});
|
| 38 |
-
Vrt0 = tile(Vrc0, {1, 1, 1, HkvMul});
|
| 39 |
-
Vrp0 = permute(Vrt0, {0, 1, 3, 2});
|
| 40 |
-
Vt0 = reshapeBatch(Vrp0, {hd, T, 1, HB});
|
| 41 |
-
A0 = attention(Qr0, Kt0, Vt0, hd, 1);
|
| 42 |
-
Wo0 = MatrixWithName("gemm_3.weight", D, Dq, 1, 1);
|
| 43 |
-
O0 = batchedMul(Wo0, reshape(permute(A0, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 44 |
-
R0 = X0 + O0;
|
| 45 |
-
Wrf0 = MatrixWithName("rmsn_284.weight", D, 1);
|
| 46 |
-
Rfn0 = rmsNorm(R0, Wrf0, 1e-5);
|
| 47 |
-
Wg0 = MatrixWithName("gemm_4.weight", I, D, 1, 1);
|
| 48 |
-
Wu0 = MatrixWithName("gemm_5.weight", I, D, 1, 1);
|
| 49 |
-
Wd0 = MatrixWithName("gemm_6.weight", D, I, 1, 1);
|
| 50 |
-
X1 = R0 + batchedMul(Wd0, elementMul(silu(batchedMul(Wg0, Rfn0)), batchedMul(Wu0, Rfn0)));
|
| 51 |
-
Wra1 = MatrixWithName("rmsn_285.weight", D, 1);
|
| 52 |
-
Xna1 = rmsNorm(X1, Wra1, 1e-5);
|
| 53 |
-
Wq1 = MatrixWithName("gemm_7.weight", Dq, D, 1, 1);
|
| 54 |
-
Qh1 = permute(reshape(batchedMul(Wq1, Xna1), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 55 |
-
Wqn1 = MatrixWithName("rmsn_286.weight", hd, 1);
|
| 56 |
-
Qn1 = rmsNorm(Qh1, Wqn1, 1e-5);
|
| 57 |
-
Qr1 = rope(Qn1, cos_te, sin_te);
|
| 58 |
-
Wk1 = MatrixWithName("gemm_8.weight", Dkv, D, 1, 1);
|
| 59 |
-
Kh1 = permute(reshape(batchedMul(Wk1, Xna1), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 60 |
-
Wkn1 = MatrixWithName("rmsn_287.weight", hd, 1);
|
| 61 |
-
Kn1 = rmsNorm(Kh1, Wkn1, 1e-5);
|
| 62 |
-
Kr1 = rope(Kn1, cos_te, sin_te);
|
| 63 |
-
Wv1 = MatrixWithName("gemm_9.weight", Dkv, D, 1, 1);
|
| 64 |
-
V1 = permute(reshape(batchedMul(Wv1, Xna1), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 65 |
-
Krc1 = reshapeBatch(Kr1, {hd, T, HkvB, 1});
|
| 66 |
-
Krt1 = tile(Krc1, {1, 1, 1, HkvMul});
|
| 67 |
-
Krp1 = permute(Krt1, {0, 1, 3, 2});
|
| 68 |
-
Kt1 = reshapeBatch(Krp1, {hd, T, 1, HB});
|
| 69 |
-
Vrc1 = reshapeBatch(V1, {hd, T, HkvB, 1});
|
| 70 |
-
Vrt1 = tile(Vrc1, {1, 1, 1, HkvMul});
|
| 71 |
-
Vrp1 = permute(Vrt1, {0, 1, 3, 2});
|
| 72 |
-
Vt1 = reshapeBatch(Vrp1, {hd, T, 1, HB});
|
| 73 |
-
A1 = attention(Qr1, Kt1, Vt1, hd, 1);
|
| 74 |
-
Wo1 = MatrixWithName("gemm_10.weight", D, Dq, 1, 1);
|
| 75 |
-
O1 = batchedMul(Wo1, reshape(permute(A1, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 76 |
-
R1 = X1 + O1;
|
| 77 |
-
Wrf1 = MatrixWithName("rmsn_288.weight", D, 1);
|
| 78 |
-
Rfn1 = rmsNorm(R1, Wrf1, 1e-5);
|
| 79 |
-
Wg1 = MatrixWithName("gemm_11.weight", I, D, 1, 1);
|
| 80 |
-
Wu1 = MatrixWithName("gemm_12.weight", I, D, 1, 1);
|
| 81 |
-
Wd1 = MatrixWithName("gemm_13.weight", D, I, 1, 1);
|
| 82 |
-
X2 = R1 + batchedMul(Wd1, elementMul(silu(batchedMul(Wg1, Rfn1)), batchedMul(Wu1, Rfn1)));
|
| 83 |
-
Wra2 = MatrixWithName("rmsn_289.weight", D, 1);
|
| 84 |
-
Xna2 = rmsNorm(X2, Wra2, 1e-5);
|
| 85 |
-
Wq2 = MatrixWithName("gemm_14.weight", Dq, D, 1, 1);
|
| 86 |
-
Qh2 = permute(reshape(batchedMul(Wq2, Xna2), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 87 |
-
Wqn2 = MatrixWithName("rmsn_290.weight", hd, 1);
|
| 88 |
-
Qn2 = rmsNorm(Qh2, Wqn2, 1e-5);
|
| 89 |
-
Qr2 = rope(Qn2, cos_te, sin_te);
|
| 90 |
-
Wk2 = MatrixWithName("gemm_15.weight", Dkv, D, 1, 1);
|
| 91 |
-
Kh2 = permute(reshape(batchedMul(Wk2, Xna2), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 92 |
-
Wkn2 = MatrixWithName("rmsn_291.weight", hd, 1);
|
| 93 |
-
Kn2 = rmsNorm(Kh2, Wkn2, 1e-5);
|
| 94 |
-
Kr2 = rope(Kn2, cos_te, sin_te);
|
| 95 |
-
Wv2 = MatrixWithName("gemm_16.weight", Dkv, D, 1, 1);
|
| 96 |
-
V2 = permute(reshape(batchedMul(Wv2, Xna2), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 97 |
-
Krc2 = reshapeBatch(Kr2, {hd, T, HkvB, 1});
|
| 98 |
-
Krt2 = tile(Krc2, {1, 1, 1, HkvMul});
|
| 99 |
-
Krp2 = permute(Krt2, {0, 1, 3, 2});
|
| 100 |
-
Kt2 = reshapeBatch(Krp2, {hd, T, 1, HB});
|
| 101 |
-
Vrc2 = reshapeBatch(V2, {hd, T, HkvB, 1});
|
| 102 |
-
Vrt2 = tile(Vrc2, {1, 1, 1, HkvMul});
|
| 103 |
-
Vrp2 = permute(Vrt2, {0, 1, 3, 2});
|
| 104 |
-
Vt2 = reshapeBatch(Vrp2, {hd, T, 1, HB});
|
| 105 |
-
A2 = attention(Qr2, Kt2, Vt2, hd, 1);
|
| 106 |
-
Wo2 = MatrixWithName("gemm_17.weight", D, Dq, 1, 1);
|
| 107 |
-
O2 = batchedMul(Wo2, reshape(permute(A2, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 108 |
-
R2 = X2 + O2;
|
| 109 |
-
Wrf2 = MatrixWithName("rmsn_292.weight", D, 1);
|
| 110 |
-
Rfn2 = rmsNorm(R2, Wrf2, 1e-5);
|
| 111 |
-
Wg2 = MatrixWithName("gemm_18.weight", I, D, 1, 1);
|
| 112 |
-
Wu2 = MatrixWithName("gemm_19.weight", I, D, 1, 1);
|
| 113 |
-
Wd2 = MatrixWithName("gemm_20.weight", D, I, 1, 1);
|
| 114 |
-
X3 = R2 + batchedMul(Wd2, elementMul(silu(batchedMul(Wg2, Rfn2)), batchedMul(Wu2, Rfn2)));
|
| 115 |
-
Wra3 = MatrixWithName("rmsn_293.weight", D, 1);
|
| 116 |
-
Xna3 = rmsNorm(X3, Wra3, 1e-5);
|
| 117 |
-
Wq3 = MatrixWithName("gemm_21.weight", Dq, D, 1, 1);
|
| 118 |
-
Qh3 = permute(reshape(batchedMul(Wq3, Xna3), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 119 |
-
Wqn3 = MatrixWithName("rmsn_294.weight", hd, 1);
|
| 120 |
-
Qn3 = rmsNorm(Qh3, Wqn3, 1e-5);
|
| 121 |
-
Qr3 = rope(Qn3, cos_te, sin_te);
|
| 122 |
-
Wk3 = MatrixWithName("gemm_22.weight", Dkv, D, 1, 1);
|
| 123 |
-
Kh3 = permute(reshape(batchedMul(Wk3, Xna3), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 124 |
-
Wkn3 = MatrixWithName("rmsn_295.weight", hd, 1);
|
| 125 |
-
Kn3 = rmsNorm(Kh3, Wkn3, 1e-5);
|
| 126 |
-
Kr3 = rope(Kn3, cos_te, sin_te);
|
| 127 |
-
Wv3 = MatrixWithName("gemm_23.weight", Dkv, D, 1, 1);
|
| 128 |
-
V3 = permute(reshape(batchedMul(Wv3, Xna3), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 129 |
-
Krc3 = reshapeBatch(Kr3, {hd, T, HkvB, 1});
|
| 130 |
-
Krt3 = tile(Krc3, {1, 1, 1, HkvMul});
|
| 131 |
-
Krp3 = permute(Krt3, {0, 1, 3, 2});
|
| 132 |
-
Kt3 = reshapeBatch(Krp3, {hd, T, 1, HB});
|
| 133 |
-
Vrc3 = reshapeBatch(V3, {hd, T, HkvB, 1});
|
| 134 |
-
Vrt3 = tile(Vrc3, {1, 1, 1, HkvMul});
|
| 135 |
-
Vrp3 = permute(Vrt3, {0, 1, 3, 2});
|
| 136 |
-
Vt3 = reshapeBatch(Vrp3, {hd, T, 1, HB});
|
| 137 |
-
A3 = attention(Qr3, Kt3, Vt3, hd, 1);
|
| 138 |
-
Wo3 = MatrixWithName("gemm_24.weight", D, Dq, 1, 1);
|
| 139 |
-
O3 = batchedMul(Wo3, reshape(permute(A3, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 140 |
-
R3 = X3 + O3;
|
| 141 |
-
Wrf3 = MatrixWithName("rmsn_296.weight", D, 1);
|
| 142 |
-
Rfn3 = rmsNorm(R3, Wrf3, 1e-5);
|
| 143 |
-
Wg3 = MatrixWithName("gemm_25.weight", I, D, 1, 1);
|
| 144 |
-
Wu3 = MatrixWithName("gemm_26.weight", I, D, 1, 1);
|
| 145 |
-
Wd3 = MatrixWithName("gemm_27.weight", D, I, 1, 1);
|
| 146 |
-
X4 = R3 + batchedMul(Wd3, elementMul(silu(batchedMul(Wg3, Rfn3)), batchedMul(Wu3, Rfn3)));
|
| 147 |
-
Wra4 = MatrixWithName("rmsn_297.weight", D, 1);
|
| 148 |
-
Xna4 = rmsNorm(X4, Wra4, 1e-5);
|
| 149 |
-
Wq4 = MatrixWithName("gemm_28.weight", Dq, D, 1, 1);
|
| 150 |
-
Qh4 = permute(reshape(batchedMul(Wq4, Xna4), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 151 |
-
Wqn4 = MatrixWithName("rmsn_298.weight", hd, 1);
|
| 152 |
-
Qn4 = rmsNorm(Qh4, Wqn4, 1e-5);
|
| 153 |
-
Qr4 = rope(Qn4, cos_te, sin_te);
|
| 154 |
-
Wk4 = MatrixWithName("gemm_29.weight", Dkv, D, 1, 1);
|
| 155 |
-
Kh4 = permute(reshape(batchedMul(Wk4, Xna4), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 156 |
-
Wkn4 = MatrixWithName("rmsn_299.weight", hd, 1);
|
| 157 |
-
Kn4 = rmsNorm(Kh4, Wkn4, 1e-5);
|
| 158 |
-
Kr4 = rope(Kn4, cos_te, sin_te);
|
| 159 |
-
Wv4 = MatrixWithName("gemm_30.weight", Dkv, D, 1, 1);
|
| 160 |
-
V4 = permute(reshape(batchedMul(Wv4, Xna4), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 161 |
-
Krc4 = reshapeBatch(Kr4, {hd, T, HkvB, 1});
|
| 162 |
-
Krt4 = tile(Krc4, {1, 1, 1, HkvMul});
|
| 163 |
-
Krp4 = permute(Krt4, {0, 1, 3, 2});
|
| 164 |
-
Kt4 = reshapeBatch(Krp4, {hd, T, 1, HB});
|
| 165 |
-
Vrc4 = reshapeBatch(V4, {hd, T, HkvB, 1});
|
| 166 |
-
Vrt4 = tile(Vrc4, {1, 1, 1, HkvMul});
|
| 167 |
-
Vrp4 = permute(Vrt4, {0, 1, 3, 2});
|
| 168 |
-
Vt4 = reshapeBatch(Vrp4, {hd, T, 1, HB});
|
| 169 |
-
A4 = attention(Qr4, Kt4, Vt4, hd, 1);
|
| 170 |
-
Wo4 = MatrixWithName("gemm_31.weight", D, Dq, 1, 1);
|
| 171 |
-
O4 = batchedMul(Wo4, reshape(permute(A4, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 172 |
-
R4 = X4 + O4;
|
| 173 |
-
Wrf4 = MatrixWithName("rmsn_300.weight", D, 1);
|
| 174 |
-
Rfn4 = rmsNorm(R4, Wrf4, 1e-5);
|
| 175 |
-
Wg4 = MatrixWithName("gemm_32.weight", I, D, 1, 1);
|
| 176 |
-
Wu4 = MatrixWithName("gemm_33.weight", I, D, 1, 1);
|
| 177 |
-
Wd4 = MatrixWithName("gemm_34.weight", D, I, 1, 1);
|
| 178 |
-
X5 = R4 + batchedMul(Wd4, elementMul(silu(batchedMul(Wg4, Rfn4)), batchedMul(Wu4, Rfn4)));
|
| 179 |
-
Wra5 = MatrixWithName("rmsn_301.weight", D, 1);
|
| 180 |
-
Xna5 = rmsNorm(X5, Wra5, 1e-5);
|
| 181 |
-
Wq5 = MatrixWithName("gemm_35.weight", Dq, D, 1, 1);
|
| 182 |
-
Qh5 = permute(reshape(batchedMul(Wq5, Xna5), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 183 |
-
Wqn5 = MatrixWithName("rmsn_302.weight", hd, 1);
|
| 184 |
-
Qn5 = rmsNorm(Qh5, Wqn5, 1e-5);
|
| 185 |
-
Qr5 = rope(Qn5, cos_te, sin_te);
|
| 186 |
-
Wk5 = MatrixWithName("gemm_36.weight", Dkv, D, 1, 1);
|
| 187 |
-
Kh5 = permute(reshape(batchedMul(Wk5, Xna5), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 188 |
-
Wkn5 = MatrixWithName("rmsn_303.weight", hd, 1);
|
| 189 |
-
Kn5 = rmsNorm(Kh5, Wkn5, 1e-5);
|
| 190 |
-
Kr5 = rope(Kn5, cos_te, sin_te);
|
| 191 |
-
Wv5 = MatrixWithName("gemm_37.weight", Dkv, D, 1, 1);
|
| 192 |
-
V5 = permute(reshape(batchedMul(Wv5, Xna5), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 193 |
-
Krc5 = reshapeBatch(Kr5, {hd, T, HkvB, 1});
|
| 194 |
-
Krt5 = tile(Krc5, {1, 1, 1, HkvMul});
|
| 195 |
-
Krp5 = permute(Krt5, {0, 1, 3, 2});
|
| 196 |
-
Kt5 = reshapeBatch(Krp5, {hd, T, 1, HB});
|
| 197 |
-
Vrc5 = reshapeBatch(V5, {hd, T, HkvB, 1});
|
| 198 |
-
Vrt5 = tile(Vrc5, {1, 1, 1, HkvMul});
|
| 199 |
-
Vrp5 = permute(Vrt5, {0, 1, 3, 2});
|
| 200 |
-
Vt5 = reshapeBatch(Vrp5, {hd, T, 1, HB});
|
| 201 |
-
A5 = attention(Qr5, Kt5, Vt5, hd, 1);
|
| 202 |
-
Wo5 = MatrixWithName("gemm_38.weight", D, Dq, 1, 1);
|
| 203 |
-
O5 = batchedMul(Wo5, reshape(permute(A5, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 204 |
-
R5 = X5 + O5;
|
| 205 |
-
Wrf5 = MatrixWithName("rmsn_304.weight", D, 1);
|
| 206 |
-
Rfn5 = rmsNorm(R5, Wrf5, 1e-5);
|
| 207 |
-
Wg5 = MatrixWithName("gemm_39.weight", I, D, 1, 1);
|
| 208 |
-
Wu5 = MatrixWithName("gemm_40.weight", I, D, 1, 1);
|
| 209 |
-
Wd5 = MatrixWithName("gemm_41.weight", D, I, 1, 1);
|
| 210 |
-
X6 = R5 + batchedMul(Wd5, elementMul(silu(batchedMul(Wg5, Rfn5)), batchedMul(Wu5, Rfn5)));
|
| 211 |
-
Wra6 = MatrixWithName("rmsn_305.weight", D, 1);
|
| 212 |
-
Xna6 = rmsNorm(X6, Wra6, 1e-5);
|
| 213 |
-
Wq6 = MatrixWithName("gemm_42.weight", Dq, D, 1, 1);
|
| 214 |
-
Qh6 = permute(reshape(batchedMul(Wq6, Xna6), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 215 |
-
Wqn6 = MatrixWithName("rmsn_306.weight", hd, 1);
|
| 216 |
-
Qn6 = rmsNorm(Qh6, Wqn6, 1e-5);
|
| 217 |
-
Qr6 = rope(Qn6, cos_te, sin_te);
|
| 218 |
-
Wk6 = MatrixWithName("gemm_43.weight", Dkv, D, 1, 1);
|
| 219 |
-
Kh6 = permute(reshape(batchedMul(Wk6, Xna6), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 220 |
-
Wkn6 = MatrixWithName("rmsn_307.weight", hd, 1);
|
| 221 |
-
Kn6 = rmsNorm(Kh6, Wkn6, 1e-5);
|
| 222 |
-
Kr6 = rope(Kn6, cos_te, sin_te);
|
| 223 |
-
Wv6 = MatrixWithName("gemm_44.weight", Dkv, D, 1, 1);
|
| 224 |
-
V6 = permute(reshape(batchedMul(Wv6, Xna6), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 225 |
-
Krc6 = reshapeBatch(Kr6, {hd, T, HkvB, 1});
|
| 226 |
-
Krt6 = tile(Krc6, {1, 1, 1, HkvMul});
|
| 227 |
-
Krp6 = permute(Krt6, {0, 1, 3, 2});
|
| 228 |
-
Kt6 = reshapeBatch(Krp6, {hd, T, 1, HB});
|
| 229 |
-
Vrc6 = reshapeBatch(V6, {hd, T, HkvB, 1});
|
| 230 |
-
Vrt6 = tile(Vrc6, {1, 1, 1, HkvMul});
|
| 231 |
-
Vrp6 = permute(Vrt6, {0, 1, 3, 2});
|
| 232 |
-
Vt6 = reshapeBatch(Vrp6, {hd, T, 1, HB});
|
| 233 |
-
A6 = attention(Qr6, Kt6, Vt6, hd, 1);
|
| 234 |
-
Wo6 = MatrixWithName("gemm_45.weight", D, Dq, 1, 1);
|
| 235 |
-
O6 = batchedMul(Wo6, reshape(permute(A6, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 236 |
-
R6 = X6 + O6;
|
| 237 |
-
Wrf6 = MatrixWithName("rmsn_308.weight", D, 1);
|
| 238 |
-
Rfn6 = rmsNorm(R6, Wrf6, 1e-5);
|
| 239 |
-
Wg6 = MatrixWithName("gemm_46.weight", I, D, 1, 1);
|
| 240 |
-
Wu6 = MatrixWithName("gemm_47.weight", I, D, 1, 1);
|
| 241 |
-
Wd6 = MatrixWithName("gemm_48.weight", D, I, 1, 1);
|
| 242 |
-
X7 = R6 + batchedMul(Wd6, elementMul(silu(batchedMul(Wg6, Rfn6)), batchedMul(Wu6, Rfn6)));
|
| 243 |
-
Wra7 = MatrixWithName("rmsn_309.weight", D, 1);
|
| 244 |
-
Xna7 = rmsNorm(X7, Wra7, 1e-5);
|
| 245 |
-
Wq7 = MatrixWithName("gemm_49.weight", Dq, D, 1, 1);
|
| 246 |
-
Qh7 = permute(reshape(batchedMul(Wq7, Xna7), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 247 |
-
Wqn7 = MatrixWithName("rmsn_310.weight", hd, 1);
|
| 248 |
-
Qn7 = rmsNorm(Qh7, Wqn7, 1e-5);
|
| 249 |
-
Qr7 = rope(Qn7, cos_te, sin_te);
|
| 250 |
-
Wk7 = MatrixWithName("gemm_50.weight", Dkv, D, 1, 1);
|
| 251 |
-
Kh7 = permute(reshape(batchedMul(Wk7, Xna7), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 252 |
-
Wkn7 = MatrixWithName("rmsn_311.weight", hd, 1);
|
| 253 |
-
Kn7 = rmsNorm(Kh7, Wkn7, 1e-5);
|
| 254 |
-
Kr7 = rope(Kn7, cos_te, sin_te);
|
| 255 |
-
Wv7 = MatrixWithName("gemm_51.weight", Dkv, D, 1, 1);
|
| 256 |
-
V7 = permute(reshape(batchedMul(Wv7, Xna7), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 257 |
-
Krc7 = reshapeBatch(Kr7, {hd, T, HkvB, 1});
|
| 258 |
-
Krt7 = tile(Krc7, {1, 1, 1, HkvMul});
|
| 259 |
-
Krp7 = permute(Krt7, {0, 1, 3, 2});
|
| 260 |
-
Kt7 = reshapeBatch(Krp7, {hd, T, 1, HB});
|
| 261 |
-
Vrc7 = reshapeBatch(V7, {hd, T, HkvB, 1});
|
| 262 |
-
Vrt7 = tile(Vrc7, {1, 1, 1, HkvMul});
|
| 263 |
-
Vrp7 = permute(Vrt7, {0, 1, 3, 2});
|
| 264 |
-
Vt7 = reshapeBatch(Vrp7, {hd, T, 1, HB});
|
| 265 |
-
A7 = attention(Qr7, Kt7, Vt7, hd, 1);
|
| 266 |
-
Wo7 = MatrixWithName("gemm_52.weight", D, Dq, 1, 1);
|
| 267 |
-
O7 = batchedMul(Wo7, reshape(permute(A7, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 268 |
-
R7 = X7 + O7;
|
| 269 |
-
Wrf7 = MatrixWithName("rmsn_312.weight", D, 1);
|
| 270 |
-
Rfn7 = rmsNorm(R7, Wrf7, 1e-5);
|
| 271 |
-
Wg7 = MatrixWithName("gemm_53.weight", I, D, 1, 1);
|
| 272 |
-
Wu7 = MatrixWithName("gemm_54.weight", I, D, 1, 1);
|
| 273 |
-
Wd7 = MatrixWithName("gemm_55.weight", D, I, 1, 1);
|
| 274 |
-
X8 = R7 + batchedMul(Wd7, elementMul(silu(batchedMul(Wg7, Rfn7)), batchedMul(Wu7, Rfn7)));
|
| 275 |
-
Wra8 = MatrixWithName("rmsn_313.weight", D, 1);
|
| 276 |
-
Xna8 = rmsNorm(X8, Wra8, 1e-5);
|
| 277 |
-
Wq8 = MatrixWithName("gemm_56.weight", Dq, D, 1, 1);
|
| 278 |
-
Qh8 = permute(reshape(batchedMul(Wq8, Xna8), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 279 |
-
Wqn8 = MatrixWithName("rmsn_314.weight", hd, 1);
|
| 280 |
-
Qn8 = rmsNorm(Qh8, Wqn8, 1e-5);
|
| 281 |
-
Qr8 = rope(Qn8, cos_te, sin_te);
|
| 282 |
-
Wk8 = MatrixWithName("gemm_57.weight", Dkv, D, 1, 1);
|
| 283 |
-
Kh8 = permute(reshape(batchedMul(Wk8, Xna8), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 284 |
-
Wkn8 = MatrixWithName("rmsn_315.weight", hd, 1);
|
| 285 |
-
Kn8 = rmsNorm(Kh8, Wkn8, 1e-5);
|
| 286 |
-
Kr8 = rope(Kn8, cos_te, sin_te);
|
| 287 |
-
Wv8 = MatrixWithName("gemm_58.weight", Dkv, D, 1, 1);
|
| 288 |
-
V8 = permute(reshape(batchedMul(Wv8, Xna8), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 289 |
-
Krc8 = reshapeBatch(Kr8, {hd, T, HkvB, 1});
|
| 290 |
-
Krt8 = tile(Krc8, {1, 1, 1, HkvMul});
|
| 291 |
-
Krp8 = permute(Krt8, {0, 1, 3, 2});
|
| 292 |
-
Kt8 = reshapeBatch(Krp8, {hd, T, 1, HB});
|
| 293 |
-
Vrc8 = reshapeBatch(V8, {hd, T, HkvB, 1});
|
| 294 |
-
Vrt8 = tile(Vrc8, {1, 1, 1, HkvMul});
|
| 295 |
-
Vrp8 = permute(Vrt8, {0, 1, 3, 2});
|
| 296 |
-
Vt8 = reshapeBatch(Vrp8, {hd, T, 1, HB});
|
| 297 |
-
A8 = attention(Qr8, Kt8, Vt8, hd, 1);
|
| 298 |
-
Wo8 = MatrixWithName("gemm_59.weight", D, Dq, 1, 1);
|
| 299 |
-
O8 = batchedMul(Wo8, reshape(permute(A8, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 300 |
-
R8 = X8 + O8;
|
| 301 |
-
Wrf8 = MatrixWithName("rmsn_316.weight", D, 1);
|
| 302 |
-
Rfn8 = rmsNorm(R8, Wrf8, 1e-5);
|
| 303 |
-
Wg8 = MatrixWithName("gemm_60.weight", I, D, 1, 1);
|
| 304 |
-
Wu8 = MatrixWithName("gemm_61.weight", I, D, 1, 1);
|
| 305 |
-
Wd8 = MatrixWithName("gemm_62.weight", D, I, 1, 1);
|
| 306 |
-
X9 = R8 + batchedMul(Wd8, elementMul(silu(batchedMul(Wg8, Rfn8)), batchedMul(Wu8, Rfn8)));
|
| 307 |
-
Wra9 = MatrixWithName("rmsn_317.weight", D, 1);
|
| 308 |
-
Xna9 = rmsNorm(X9, Wra9, 1e-5);
|
| 309 |
-
Wq9 = MatrixWithName("gemm_63.weight", Dq, D, 1, 1);
|
| 310 |
-
Qh9 = permute(reshape(batchedMul(Wq9, Xna9), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 311 |
-
Wqn9 = MatrixWithName("rmsn_318.weight", hd, 1);
|
| 312 |
-
Qn9 = rmsNorm(Qh9, Wqn9, 1e-5);
|
| 313 |
-
Qr9 = rope(Qn9, cos_te, sin_te);
|
| 314 |
-
Wk9 = MatrixWithName("gemm_64.weight", Dkv, D, 1, 1);
|
| 315 |
-
Kh9 = permute(reshape(batchedMul(Wk9, Xna9), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 316 |
-
Wkn9 = MatrixWithName("rmsn_319.weight", hd, 1);
|
| 317 |
-
Kn9 = rmsNorm(Kh9, Wkn9, 1e-5);
|
| 318 |
-
Kr9 = rope(Kn9, cos_te, sin_te);
|
| 319 |
-
Wv9 = MatrixWithName("gemm_65.weight", Dkv, D, 1, 1);
|
| 320 |
-
V9 = permute(reshape(batchedMul(Wv9, Xna9), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 321 |
-
Krc9 = reshapeBatch(Kr9, {hd, T, HkvB, 1});
|
| 322 |
-
Krt9 = tile(Krc9, {1, 1, 1, HkvMul});
|
| 323 |
-
Krp9 = permute(Krt9, {0, 1, 3, 2});
|
| 324 |
-
Kt9 = reshapeBatch(Krp9, {hd, T, 1, HB});
|
| 325 |
-
Vrc9 = reshapeBatch(V9, {hd, T, HkvB, 1});
|
| 326 |
-
Vrt9 = tile(Vrc9, {1, 1, 1, HkvMul});
|
| 327 |
-
Vrp9 = permute(Vrt9, {0, 1, 3, 2});
|
| 328 |
-
Vt9 = reshapeBatch(Vrp9, {hd, T, 1, HB});
|
| 329 |
-
A9 = attention(Qr9, Kt9, Vt9, hd, 1);
|
| 330 |
-
Wo9 = MatrixWithName("gemm_66.weight", D, Dq, 1, 1);
|
| 331 |
-
O9 = batchedMul(Wo9, reshape(permute(A9, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 332 |
-
R9 = X9 + O9;
|
| 333 |
-
Wrf9 = MatrixWithName("rmsn_320.weight", D, 1);
|
| 334 |
-
Rfn9 = rmsNorm(R9, Wrf9, 1e-5);
|
| 335 |
-
Wg9 = MatrixWithName("gemm_67.weight", I, D, 1, 1);
|
| 336 |
-
Wu9 = MatrixWithName("gemm_68.weight", I, D, 1, 1);
|
| 337 |
-
Wd9 = MatrixWithName("gemm_69.weight", D, I, 1, 1);
|
| 338 |
-
X10 = R9 + batchedMul(Wd9, elementMul(silu(batchedMul(Wg9, Rfn9)), batchedMul(Wu9, Rfn9)));
|
| 339 |
-
Wra10 = MatrixWithName("rmsn_321.weight", D, 1);
|
| 340 |
-
Xna10 = rmsNorm(X10, Wra10, 1e-5);
|
| 341 |
-
Wq10 = MatrixWithName("gemm_70.weight", Dq, D, 1, 1);
|
| 342 |
-
Qh10 = permute(reshape(batchedMul(Wq10, Xna10), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 343 |
-
Wqn10 = MatrixWithName("rmsn_322.weight", hd, 1);
|
| 344 |
-
Qn10 = rmsNorm(Qh10, Wqn10, 1e-5);
|
| 345 |
-
Qr10 = rope(Qn10, cos_te, sin_te);
|
| 346 |
-
Wk10 = MatrixWithName("gemm_71.weight", Dkv, D, 1, 1);
|
| 347 |
-
Kh10 = permute(reshape(batchedMul(Wk10, Xna10), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 348 |
-
Wkn10 = MatrixWithName("rmsn_323.weight", hd, 1);
|
| 349 |
-
Kn10 = rmsNorm(Kh10, Wkn10, 1e-5);
|
| 350 |
-
Kr10 = rope(Kn10, cos_te, sin_te);
|
| 351 |
-
Wv10 = MatrixWithName("gemm_72.weight", Dkv, D, 1, 1);
|
| 352 |
-
V10 = permute(reshape(batchedMul(Wv10, Xna10), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 353 |
-
Krc10 = reshapeBatch(Kr10, {hd, T, HkvB, 1});
|
| 354 |
-
Krt10 = tile(Krc10, {1, 1, 1, HkvMul});
|
| 355 |
-
Krp10 = permute(Krt10, {0, 1, 3, 2});
|
| 356 |
-
Kt10 = reshapeBatch(Krp10, {hd, T, 1, HB});
|
| 357 |
-
Vrc10 = reshapeBatch(V10, {hd, T, HkvB, 1});
|
| 358 |
-
Vrt10 = tile(Vrc10, {1, 1, 1, HkvMul});
|
| 359 |
-
Vrp10 = permute(Vrt10, {0, 1, 3, 2});
|
| 360 |
-
Vt10 = reshapeBatch(Vrp10, {hd, T, 1, HB});
|
| 361 |
-
A10 = attention(Qr10, Kt10, Vt10, hd, 1);
|
| 362 |
-
Wo10 = MatrixWithName("gemm_73.weight", D, Dq, 1, 1);
|
| 363 |
-
O10 = batchedMul(Wo10, reshape(permute(A10, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 364 |
-
R10 = X10 + O10;
|
| 365 |
-
Wrf10 = MatrixWithName("rmsn_324.weight", D, 1);
|
| 366 |
-
Rfn10 = rmsNorm(R10, Wrf10, 1e-5);
|
| 367 |
-
Wg10 = MatrixWithName("gemm_74.weight", I, D, 1, 1);
|
| 368 |
-
Wu10 = MatrixWithName("gemm_75.weight", I, D, 1, 1);
|
| 369 |
-
Wd10 = MatrixWithName("gemm_76.weight", D, I, 1, 1);
|
| 370 |
-
X11 = R10 + batchedMul(Wd10, elementMul(silu(batchedMul(Wg10, Rfn10)), batchedMul(Wu10, Rfn10)));
|
| 371 |
-
Wra11 = MatrixWithName("rmsn_325.weight", D, 1);
|
| 372 |
-
Xna11 = rmsNorm(X11, Wra11, 1e-5);
|
| 373 |
-
Wq11 = MatrixWithName("gemm_77.weight", Dq, D, 1, 1);
|
| 374 |
-
Qh11 = permute(reshape(batchedMul(Wq11, Xna11), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 375 |
-
Wqn11 = MatrixWithName("rmsn_326.weight", hd, 1);
|
| 376 |
-
Qn11 = rmsNorm(Qh11, Wqn11, 1e-5);
|
| 377 |
-
Qr11 = rope(Qn11, cos_te, sin_te);
|
| 378 |
-
Wk11 = MatrixWithName("gemm_78.weight", Dkv, D, 1, 1);
|
| 379 |
-
Kh11 = permute(reshape(batchedMul(Wk11, Xna11), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 380 |
-
Wkn11 = MatrixWithName("rmsn_327.weight", hd, 1);
|
| 381 |
-
Kn11 = rmsNorm(Kh11, Wkn11, 1e-5);
|
| 382 |
-
Kr11 = rope(Kn11, cos_te, sin_te);
|
| 383 |
-
Wv11 = MatrixWithName("gemm_79.weight", Dkv, D, 1, 1);
|
| 384 |
-
V11 = permute(reshape(batchedMul(Wv11, Xna11), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 385 |
-
Krc11 = reshapeBatch(Kr11, {hd, T, HkvB, 1});
|
| 386 |
-
Krt11 = tile(Krc11, {1, 1, 1, HkvMul});
|
| 387 |
-
Krp11 = permute(Krt11, {0, 1, 3, 2});
|
| 388 |
-
Kt11 = reshapeBatch(Krp11, {hd, T, 1, HB});
|
| 389 |
-
Vrc11 = reshapeBatch(V11, {hd, T, HkvB, 1});
|
| 390 |
-
Vrt11 = tile(Vrc11, {1, 1, 1, HkvMul});
|
| 391 |
-
Vrp11 = permute(Vrt11, {0, 1, 3, 2});
|
| 392 |
-
Vt11 = reshapeBatch(Vrp11, {hd, T, 1, HB});
|
| 393 |
-
A11 = attention(Qr11, Kt11, Vt11, hd, 1);
|
| 394 |
-
Wo11 = MatrixWithName("gemm_80.weight", D, Dq, 1, 1);
|
| 395 |
-
O11 = batchedMul(Wo11, reshape(permute(A11, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 396 |
-
R11 = X11 + O11;
|
| 397 |
-
Wrf11 = MatrixWithName("rmsn_328.weight", D, 1);
|
| 398 |
-
Rfn11 = rmsNorm(R11, Wrf11, 1e-5);
|
| 399 |
-
Wg11 = MatrixWithName("gemm_81.weight", I, D, 1, 1);
|
| 400 |
-
Wu11 = MatrixWithName("gemm_82.weight", I, D, 1, 1);
|
| 401 |
-
Wd11 = MatrixWithName("gemm_83.weight", D, I, 1, 1);
|
| 402 |
-
X12 = R11 + batchedMul(Wd11, elementMul(silu(batchedMul(Wg11, Rfn11)), batchedMul(Wu11, Rfn11)));
|
| 403 |
-
Wra12 = MatrixWithName("rmsn_329.weight", D, 1);
|
| 404 |
-
Xna12 = rmsNorm(X12, Wra12, 1e-5);
|
| 405 |
-
Wq12 = MatrixWithName("gemm_84.weight", Dq, D, 1, 1);
|
| 406 |
-
Qh12 = permute(reshape(batchedMul(Wq12, Xna12), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 407 |
-
Wqn12 = MatrixWithName("rmsn_330.weight", hd, 1);
|
| 408 |
-
Qn12 = rmsNorm(Qh12, Wqn12, 1e-5);
|
| 409 |
-
Qr12 = rope(Qn12, cos_te, sin_te);
|
| 410 |
-
Wk12 = MatrixWithName("gemm_85.weight", Dkv, D, 1, 1);
|
| 411 |
-
Kh12 = permute(reshape(batchedMul(Wk12, Xna12), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 412 |
-
Wkn12 = MatrixWithName("rmsn_331.weight", hd, 1);
|
| 413 |
-
Kn12 = rmsNorm(Kh12, Wkn12, 1e-5);
|
| 414 |
-
Kr12 = rope(Kn12, cos_te, sin_te);
|
| 415 |
-
Wv12 = MatrixWithName("gemm_86.weight", Dkv, D, 1, 1);
|
| 416 |
-
V12 = permute(reshape(batchedMul(Wv12, Xna12), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 417 |
-
Krc12 = reshapeBatch(Kr12, {hd, T, HkvB, 1});
|
| 418 |
-
Krt12 = tile(Krc12, {1, 1, 1, HkvMul});
|
| 419 |
-
Krp12 = permute(Krt12, {0, 1, 3, 2});
|
| 420 |
-
Kt12 = reshapeBatch(Krp12, {hd, T, 1, HB});
|
| 421 |
-
Vrc12 = reshapeBatch(V12, {hd, T, HkvB, 1});
|
| 422 |
-
Vrt12 = tile(Vrc12, {1, 1, 1, HkvMul});
|
| 423 |
-
Vrp12 = permute(Vrt12, {0, 1, 3, 2});
|
| 424 |
-
Vt12 = reshapeBatch(Vrp12, {hd, T, 1, HB});
|
| 425 |
-
A12 = attention(Qr12, Kt12, Vt12, hd, 1);
|
| 426 |
-
Wo12 = MatrixWithName("gemm_87.weight", D, Dq, 1, 1);
|
| 427 |
-
O12 = batchedMul(Wo12, reshape(permute(A12, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 428 |
-
R12 = X12 + O12;
|
| 429 |
-
Wrf12 = MatrixWithName("rmsn_332.weight", D, 1);
|
| 430 |
-
Rfn12 = rmsNorm(R12, Wrf12, 1e-5);
|
| 431 |
-
Wg12 = MatrixWithName("gemm_88.weight", I, D, 1, 1);
|
| 432 |
-
Wu12 = MatrixWithName("gemm_89.weight", I, D, 1, 1);
|
| 433 |
-
Wd12 = MatrixWithName("gemm_90.weight", D, I, 1, 1);
|
| 434 |
-
X13 = R12 + batchedMul(Wd12, elementMul(silu(batchedMul(Wg12, Rfn12)), batchedMul(Wu12, Rfn12)));
|
| 435 |
-
Wra13 = MatrixWithName("rmsn_333.weight", D, 1);
|
| 436 |
-
Xna13 = rmsNorm(X13, Wra13, 1e-5);
|
| 437 |
-
Wq13 = MatrixWithName("gemm_91.weight", Dq, D, 1, 1);
|
| 438 |
-
Qh13 = permute(reshape(batchedMul(Wq13, Xna13), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 439 |
-
Wqn13 = MatrixWithName("rmsn_334.weight", hd, 1);
|
| 440 |
-
Qn13 = rmsNorm(Qh13, Wqn13, 1e-5);
|
| 441 |
-
Qr13 = rope(Qn13, cos_te, sin_te);
|
| 442 |
-
Wk13 = MatrixWithName("gemm_92.weight", Dkv, D, 1, 1);
|
| 443 |
-
Kh13 = permute(reshape(batchedMul(Wk13, Xna13), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 444 |
-
Wkn13 = MatrixWithName("rmsn_335.weight", hd, 1);
|
| 445 |
-
Kn13 = rmsNorm(Kh13, Wkn13, 1e-5);
|
| 446 |
-
Kr13 = rope(Kn13, cos_te, sin_te);
|
| 447 |
-
Wv13 = MatrixWithName("gemm_93.weight", Dkv, D, 1, 1);
|
| 448 |
-
V13 = permute(reshape(batchedMul(Wv13, Xna13), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 449 |
-
Krc13 = reshapeBatch(Kr13, {hd, T, HkvB, 1});
|
| 450 |
-
Krt13 = tile(Krc13, {1, 1, 1, HkvMul});
|
| 451 |
-
Krp13 = permute(Krt13, {0, 1, 3, 2});
|
| 452 |
-
Kt13 = reshapeBatch(Krp13, {hd, T, 1, HB});
|
| 453 |
-
Vrc13 = reshapeBatch(V13, {hd, T, HkvB, 1});
|
| 454 |
-
Vrt13 = tile(Vrc13, {1, 1, 1, HkvMul});
|
| 455 |
-
Vrp13 = permute(Vrt13, {0, 1, 3, 2});
|
| 456 |
-
Vt13 = reshapeBatch(Vrp13, {hd, T, 1, HB});
|
| 457 |
-
A13 = attention(Qr13, Kt13, Vt13, hd, 1);
|
| 458 |
-
Wo13 = MatrixWithName("gemm_94.weight", D, Dq, 1, 1);
|
| 459 |
-
O13 = batchedMul(Wo13, reshape(permute(A13, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 460 |
-
R13 = X13 + O13;
|
| 461 |
-
Wrf13 = MatrixWithName("rmsn_336.weight", D, 1);
|
| 462 |
-
Rfn13 = rmsNorm(R13, Wrf13, 1e-5);
|
| 463 |
-
Wg13 = MatrixWithName("gemm_95.weight", I, D, 1, 1);
|
| 464 |
-
Wu13 = MatrixWithName("gemm_96.weight", I, D, 1, 1);
|
| 465 |
-
Wd13 = MatrixWithName("gemm_97.weight", D, I, 1, 1);
|
| 466 |
-
X14 = R13 + batchedMul(Wd13, elementMul(silu(batchedMul(Wg13, Rfn13)), batchedMul(Wu13, Rfn13)));
|
| 467 |
-
Wra14 = MatrixWithName("rmsn_337.weight", D, 1);
|
| 468 |
-
Xna14 = rmsNorm(X14, Wra14, 1e-5);
|
| 469 |
-
Wq14 = MatrixWithName("gemm_98.weight", Dq, D, 1, 1);
|
| 470 |
-
Qh14 = permute(reshape(batchedMul(Wq14, Xna14), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 471 |
-
Wqn14 = MatrixWithName("rmsn_338.weight", hd, 1);
|
| 472 |
-
Qn14 = rmsNorm(Qh14, Wqn14, 1e-5);
|
| 473 |
-
Qr14 = rope(Qn14, cos_te, sin_te);
|
| 474 |
-
Wk14 = MatrixWithName("gemm_99.weight", Dkv, D, 1, 1);
|
| 475 |
-
Kh14 = permute(reshape(batchedMul(Wk14, Xna14), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 476 |
-
Wkn14 = MatrixWithName("rmsn_339.weight", hd, 1);
|
| 477 |
-
Kn14 = rmsNorm(Kh14, Wkn14, 1e-5);
|
| 478 |
-
Kr14 = rope(Kn14, cos_te, sin_te);
|
| 479 |
-
Wv14 = MatrixWithName("gemm_100.weight", Dkv, D, 1, 1);
|
| 480 |
-
V14 = permute(reshape(batchedMul(Wv14, Xna14), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 481 |
-
Krc14 = reshapeBatch(Kr14, {hd, T, HkvB, 1});
|
| 482 |
-
Krt14 = tile(Krc14, {1, 1, 1, HkvMul});
|
| 483 |
-
Krp14 = permute(Krt14, {0, 1, 3, 2});
|
| 484 |
-
Kt14 = reshapeBatch(Krp14, {hd, T, 1, HB});
|
| 485 |
-
Vrc14 = reshapeBatch(V14, {hd, T, HkvB, 1});
|
| 486 |
-
Vrt14 = tile(Vrc14, {1, 1, 1, HkvMul});
|
| 487 |
-
Vrp14 = permute(Vrt14, {0, 1, 3, 2});
|
| 488 |
-
Vt14 = reshapeBatch(Vrp14, {hd, T, 1, HB});
|
| 489 |
-
A14 = attention(Qr14, Kt14, Vt14, hd, 1);
|
| 490 |
-
Wo14 = MatrixWithName("gemm_101.weight", D, Dq, 1, 1);
|
| 491 |
-
O14 = batchedMul(Wo14, reshape(permute(A14, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 492 |
-
R14 = X14 + O14;
|
| 493 |
-
Wrf14 = MatrixWithName("rmsn_340.weight", D, 1);
|
| 494 |
-
Rfn14 = rmsNorm(R14, Wrf14, 1e-5);
|
| 495 |
-
Wg14 = MatrixWithName("gemm_102.weight", I, D, 1, 1);
|
| 496 |
-
Wu14 = MatrixWithName("gemm_103.weight", I, D, 1, 1);
|
| 497 |
-
Wd14 = MatrixWithName("gemm_104.weight", D, I, 1, 1);
|
| 498 |
-
X15 = R14 + batchedMul(Wd14, elementMul(silu(batchedMul(Wg14, Rfn14)), batchedMul(Wu14, Rfn14)));
|
| 499 |
-
Wra15 = MatrixWithName("rmsn_341.weight", D, 1);
|
| 500 |
-
Xna15 = rmsNorm(X15, Wra15, 1e-5);
|
| 501 |
-
Wq15 = MatrixWithName("gemm_105.weight", Dq, D, 1, 1);
|
| 502 |
-
Qh15 = permute(reshape(batchedMul(Wq15, Xna15), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 503 |
-
Wqn15 = MatrixWithName("rmsn_342.weight", hd, 1);
|
| 504 |
-
Qn15 = rmsNorm(Qh15, Wqn15, 1e-5);
|
| 505 |
-
Qr15 = rope(Qn15, cos_te, sin_te);
|
| 506 |
-
Wk15 = MatrixWithName("gemm_106.weight", Dkv, D, 1, 1);
|
| 507 |
-
Kh15 = permute(reshape(batchedMul(Wk15, Xna15), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 508 |
-
Wkn15 = MatrixWithName("rmsn_343.weight", hd, 1);
|
| 509 |
-
Kn15 = rmsNorm(Kh15, Wkn15, 1e-5);
|
| 510 |
-
Kr15 = rope(Kn15, cos_te, sin_te);
|
| 511 |
-
Wv15 = MatrixWithName("gemm_107.weight", Dkv, D, 1, 1);
|
| 512 |
-
V15 = permute(reshape(batchedMul(Wv15, Xna15), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 513 |
-
Krc15 = reshapeBatch(Kr15, {hd, T, HkvB, 1});
|
| 514 |
-
Krt15 = tile(Krc15, {1, 1, 1, HkvMul});
|
| 515 |
-
Krp15 = permute(Krt15, {0, 1, 3, 2});
|
| 516 |
-
Kt15 = reshapeBatch(Krp15, {hd, T, 1, HB});
|
| 517 |
-
Vrc15 = reshapeBatch(V15, {hd, T, HkvB, 1});
|
| 518 |
-
Vrt15 = tile(Vrc15, {1, 1, 1, HkvMul});
|
| 519 |
-
Vrp15 = permute(Vrt15, {0, 1, 3, 2});
|
| 520 |
-
Vt15 = reshapeBatch(Vrp15, {hd, T, 1, HB});
|
| 521 |
-
A15 = attention(Qr15, Kt15, Vt15, hd, 1);
|
| 522 |
-
Wo15 = MatrixWithName("gemm_108.weight", D, Dq, 1, 1);
|
| 523 |
-
O15 = batchedMul(Wo15, reshape(permute(A15, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 524 |
-
R15 = X15 + O15;
|
| 525 |
-
Wrf15 = MatrixWithName("rmsn_344.weight", D, 1);
|
| 526 |
-
Rfn15 = rmsNorm(R15, Wrf15, 1e-5);
|
| 527 |
-
Wg15 = MatrixWithName("gemm_109.weight", I, D, 1, 1);
|
| 528 |
-
Wu15 = MatrixWithName("gemm_110.weight", I, D, 1, 1);
|
| 529 |
-
Wd15 = MatrixWithName("gemm_111.weight", D, I, 1, 1);
|
| 530 |
-
X16 = R15 + batchedMul(Wd15, elementMul(silu(batchedMul(Wg15, Rfn15)), batchedMul(Wu15, Rfn15)));
|
| 531 |
-
Wra16 = MatrixWithName("rmsn_345.weight", D, 1);
|
| 532 |
-
Xna16 = rmsNorm(X16, Wra16, 1e-5);
|
| 533 |
-
Wq16 = MatrixWithName("gemm_112.weight", Dq, D, 1, 1);
|
| 534 |
-
Qh16 = permute(reshape(batchedMul(Wq16, Xna16), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 535 |
-
Wqn16 = MatrixWithName("rmsn_346.weight", hd, 1);
|
| 536 |
-
Qn16 = rmsNorm(Qh16, Wqn16, 1e-5);
|
| 537 |
-
Qr16 = rope(Qn16, cos_te, sin_te);
|
| 538 |
-
Wk16 = MatrixWithName("gemm_113.weight", Dkv, D, 1, 1);
|
| 539 |
-
Kh16 = permute(reshape(batchedMul(Wk16, Xna16), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 540 |
-
Wkn16 = MatrixWithName("rmsn_347.weight", hd, 1);
|
| 541 |
-
Kn16 = rmsNorm(Kh16, Wkn16, 1e-5);
|
| 542 |
-
Kr16 = rope(Kn16, cos_te, sin_te);
|
| 543 |
-
Wv16 = MatrixWithName("gemm_114.weight", Dkv, D, 1, 1);
|
| 544 |
-
V16 = permute(reshape(batchedMul(Wv16, Xna16), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 545 |
-
Krc16 = reshapeBatch(Kr16, {hd, T, HkvB, 1});
|
| 546 |
-
Krt16 = tile(Krc16, {1, 1, 1, HkvMul});
|
| 547 |
-
Krp16 = permute(Krt16, {0, 1, 3, 2});
|
| 548 |
-
Kt16 = reshapeBatch(Krp16, {hd, T, 1, HB});
|
| 549 |
-
Vrc16 = reshapeBatch(V16, {hd, T, HkvB, 1});
|
| 550 |
-
Vrt16 = tile(Vrc16, {1, 1, 1, HkvMul});
|
| 551 |
-
Vrp16 = permute(Vrt16, {0, 1, 3, 2});
|
| 552 |
-
Vt16 = reshapeBatch(Vrp16, {hd, T, 1, HB});
|
| 553 |
-
A16 = attention(Qr16, Kt16, Vt16, hd, 1);
|
| 554 |
-
Wo16 = MatrixWithName("gemm_115.weight", D, Dq, 1, 1);
|
| 555 |
-
O16 = batchedMul(Wo16, reshape(permute(A16, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 556 |
-
R16 = X16 + O16;
|
| 557 |
-
Wrf16 = MatrixWithName("rmsn_348.weight", D, 1);
|
| 558 |
-
Rfn16 = rmsNorm(R16, Wrf16, 1e-5);
|
| 559 |
-
Wg16 = MatrixWithName("gemm_116.weight", I, D, 1, 1);
|
| 560 |
-
Wu16 = MatrixWithName("gemm_117.weight", I, D, 1, 1);
|
| 561 |
-
Wd16 = MatrixWithName("gemm_118.weight", D, I, 1, 1);
|
| 562 |
-
X17 = R16 + batchedMul(Wd16, elementMul(silu(batchedMul(Wg16, Rfn16)), batchedMul(Wu16, Rfn16)));
|
| 563 |
-
Wra17 = MatrixWithName("rmsn_349.weight", D, 1);
|
| 564 |
-
Xna17 = rmsNorm(X17, Wra17, 1e-5);
|
| 565 |
-
Wq17 = MatrixWithName("gemm_119.weight", Dq, D, 1, 1);
|
| 566 |
-
Qh17 = permute(reshape(batchedMul(Wq17, Xna17), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 567 |
-
Wqn17 = MatrixWithName("rmsn_350.weight", hd, 1);
|
| 568 |
-
Qn17 = rmsNorm(Qh17, Wqn17, 1e-5);
|
| 569 |
-
Qr17 = rope(Qn17, cos_te, sin_te);
|
| 570 |
-
Wk17 = MatrixWithName("gemm_120.weight", Dkv, D, 1, 1);
|
| 571 |
-
Kh17 = permute(reshape(batchedMul(Wk17, Xna17), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 572 |
-
Wkn17 = MatrixWithName("rmsn_351.weight", hd, 1);
|
| 573 |
-
Kn17 = rmsNorm(Kh17, Wkn17, 1e-5);
|
| 574 |
-
Kr17 = rope(Kn17, cos_te, sin_te);
|
| 575 |
-
Wv17 = MatrixWithName("gemm_121.weight", Dkv, D, 1, 1);
|
| 576 |
-
V17 = permute(reshape(batchedMul(Wv17, Xna17), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 577 |
-
Krc17 = reshapeBatch(Kr17, {hd, T, HkvB, 1});
|
| 578 |
-
Krt17 = tile(Krc17, {1, 1, 1, HkvMul});
|
| 579 |
-
Krp17 = permute(Krt17, {0, 1, 3, 2});
|
| 580 |
-
Kt17 = reshapeBatch(Krp17, {hd, T, 1, HB});
|
| 581 |
-
Vrc17 = reshapeBatch(V17, {hd, T, HkvB, 1});
|
| 582 |
-
Vrt17 = tile(Vrc17, {1, 1, 1, HkvMul});
|
| 583 |
-
Vrp17 = permute(Vrt17, {0, 1, 3, 2});
|
| 584 |
-
Vt17 = reshapeBatch(Vrp17, {hd, T, 1, HB});
|
| 585 |
-
A17 = attention(Qr17, Kt17, Vt17, hd, 1);
|
| 586 |
-
Wo17 = MatrixWithName("gemm_122.weight", D, Dq, 1, 1);
|
| 587 |
-
O17 = batchedMul(Wo17, reshape(permute(A17, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 588 |
-
R17 = X17 + O17;
|
| 589 |
-
Wrf17 = MatrixWithName("rmsn_352.weight", D, 1);
|
| 590 |
-
Rfn17 = rmsNorm(R17, Wrf17, 1e-5);
|
| 591 |
-
Wg17 = MatrixWithName("gemm_123.weight", I, D, 1, 1);
|
| 592 |
-
Wu17 = MatrixWithName("gemm_124.weight", I, D, 1, 1);
|
| 593 |
-
Wd17 = MatrixWithName("gemm_125.weight", D, I, 1, 1);
|
| 594 |
-
X18 = R17 + batchedMul(Wd17, elementMul(silu(batchedMul(Wg17, Rfn17)), batchedMul(Wu17, Rfn17)));
|
| 595 |
-
Wra18 = MatrixWithName("rmsn_353.weight", D, 1);
|
| 596 |
-
Xna18 = rmsNorm(X18, Wra18, 1e-5);
|
| 597 |
-
Wq18 = MatrixWithName("gemm_126.weight", Dq, D, 1, 1);
|
| 598 |
-
Qh18 = permute(reshape(batchedMul(Wq18, Xna18), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 599 |
-
Wqn18 = MatrixWithName("rmsn_354.weight", hd, 1);
|
| 600 |
-
Qn18 = rmsNorm(Qh18, Wqn18, 1e-5);
|
| 601 |
-
Qr18 = rope(Qn18, cos_te, sin_te);
|
| 602 |
-
Wk18 = MatrixWithName("gemm_127.weight", Dkv, D, 1, 1);
|
| 603 |
-
Kh18 = permute(reshape(batchedMul(Wk18, Xna18), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 604 |
-
Wkn18 = MatrixWithName("rmsn_355.weight", hd, 1);
|
| 605 |
-
Kn18 = rmsNorm(Kh18, Wkn18, 1e-5);
|
| 606 |
-
Kr18 = rope(Kn18, cos_te, sin_te);
|
| 607 |
-
Wv18 = MatrixWithName("gemm_128.weight", Dkv, D, 1, 1);
|
| 608 |
-
V18 = permute(reshape(batchedMul(Wv18, Xna18), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 609 |
-
Krc18 = reshapeBatch(Kr18, {hd, T, HkvB, 1});
|
| 610 |
-
Krt18 = tile(Krc18, {1, 1, 1, HkvMul});
|
| 611 |
-
Krp18 = permute(Krt18, {0, 1, 3, 2});
|
| 612 |
-
Kt18 = reshapeBatch(Krp18, {hd, T, 1, HB});
|
| 613 |
-
Vrc18 = reshapeBatch(V18, {hd, T, HkvB, 1});
|
| 614 |
-
Vrt18 = tile(Vrc18, {1, 1, 1, HkvMul});
|
| 615 |
-
Vrp18 = permute(Vrt18, {0, 1, 3, 2});
|
| 616 |
-
Vt18 = reshapeBatch(Vrp18, {hd, T, 1, HB});
|
| 617 |
-
A18 = attention(Qr18, Kt18, Vt18, hd, 1);
|
| 618 |
-
Wo18 = MatrixWithName("gemm_129.weight", D, Dq, 1, 1);
|
| 619 |
-
O18 = batchedMul(Wo18, reshape(permute(A18, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 620 |
-
R18 = X18 + O18;
|
| 621 |
-
Wrf18 = MatrixWithName("rmsn_356.weight", D, 1);
|
| 622 |
-
Rfn18 = rmsNorm(R18, Wrf18, 1e-5);
|
| 623 |
-
Wg18 = MatrixWithName("gemm_130.weight", I, D, 1, 1);
|
| 624 |
-
Wu18 = MatrixWithName("gemm_131.weight", I, D, 1, 1);
|
| 625 |
-
Wd18 = MatrixWithName("gemm_132.weight", D, I, 1, 1);
|
| 626 |
-
X19 = R18 + batchedMul(Wd18, elementMul(silu(batchedMul(Wg18, Rfn18)), batchedMul(Wu18, Rfn18)));
|
| 627 |
-
Wra19 = MatrixWithName("rmsn_357.weight", D, 1);
|
| 628 |
-
Xna19 = rmsNorm(X19, Wra19, 1e-5);
|
| 629 |
-
Wq19 = MatrixWithName("gemm_133.weight", Dq, D, 1, 1);
|
| 630 |
-
Qh19 = permute(reshape(batchedMul(Wq19, Xna19), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 631 |
-
Wqn19 = MatrixWithName("rmsn_358.weight", hd, 1);
|
| 632 |
-
Qn19 = rmsNorm(Qh19, Wqn19, 1e-5);
|
| 633 |
-
Qr19 = rope(Qn19, cos_te, sin_te);
|
| 634 |
-
Wk19 = MatrixWithName("gemm_134.weight", Dkv, D, 1, 1);
|
| 635 |
-
Kh19 = permute(reshape(batchedMul(Wk19, Xna19), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 636 |
-
Wkn19 = MatrixWithName("rmsn_359.weight", hd, 1);
|
| 637 |
-
Kn19 = rmsNorm(Kh19, Wkn19, 1e-5);
|
| 638 |
-
Kr19 = rope(Kn19, cos_te, sin_te);
|
| 639 |
-
Wv19 = MatrixWithName("gemm_135.weight", Dkv, D, 1, 1);
|
| 640 |
-
V19 = permute(reshape(batchedMul(Wv19, Xna19), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 641 |
-
Krc19 = reshapeBatch(Kr19, {hd, T, HkvB, 1});
|
| 642 |
-
Krt19 = tile(Krc19, {1, 1, 1, HkvMul});
|
| 643 |
-
Krp19 = permute(Krt19, {0, 1, 3, 2});
|
| 644 |
-
Kt19 = reshapeBatch(Krp19, {hd, T, 1, HB});
|
| 645 |
-
Vrc19 = reshapeBatch(V19, {hd, T, HkvB, 1});
|
| 646 |
-
Vrt19 = tile(Vrc19, {1, 1, 1, HkvMul});
|
| 647 |
-
Vrp19 = permute(Vrt19, {0, 1, 3, 2});
|
| 648 |
-
Vt19 = reshapeBatch(Vrp19, {hd, T, 1, HB});
|
| 649 |
-
A19 = attention(Qr19, Kt19, Vt19, hd, 1);
|
| 650 |
-
Wo19 = MatrixWithName("gemm_136.weight", D, Dq, 1, 1);
|
| 651 |
-
O19 = batchedMul(Wo19, reshape(permute(A19, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 652 |
-
R19 = X19 + O19;
|
| 653 |
-
Wrf19 = MatrixWithName("rmsn_360.weight", D, 1);
|
| 654 |
-
Rfn19 = rmsNorm(R19, Wrf19, 1e-5);
|
| 655 |
-
Wg19 = MatrixWithName("gemm_137.weight", I, D, 1, 1);
|
| 656 |
-
Wu19 = MatrixWithName("gemm_138.weight", I, D, 1, 1);
|
| 657 |
-
Wd19 = MatrixWithName("gemm_139.weight", D, I, 1, 1);
|
| 658 |
-
X20 = R19 + batchedMul(Wd19, elementMul(silu(batchedMul(Wg19, Rfn19)), batchedMul(Wu19, Rfn19)));
|
| 659 |
-
Wra20 = MatrixWithName("rmsn_361.weight", D, 1);
|
| 660 |
-
Xna20 = rmsNorm(X20, Wra20, 1e-5);
|
| 661 |
-
Wq20 = MatrixWithName("gemm_140.weight", Dq, D, 1, 1);
|
| 662 |
-
Qh20 = permute(reshape(batchedMul(Wq20, Xna20), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 663 |
-
Wqn20 = MatrixWithName("rmsn_362.weight", hd, 1);
|
| 664 |
-
Qn20 = rmsNorm(Qh20, Wqn20, 1e-5);
|
| 665 |
-
Qr20 = rope(Qn20, cos_te, sin_te);
|
| 666 |
-
Wk20 = MatrixWithName("gemm_141.weight", Dkv, D, 1, 1);
|
| 667 |
-
Kh20 = permute(reshape(batchedMul(Wk20, Xna20), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 668 |
-
Wkn20 = MatrixWithName("rmsn_363.weight", hd, 1);
|
| 669 |
-
Kn20 = rmsNorm(Kh20, Wkn20, 1e-5);
|
| 670 |
-
Kr20 = rope(Kn20, cos_te, sin_te);
|
| 671 |
-
Wv20 = MatrixWithName("gemm_142.weight", Dkv, D, 1, 1);
|
| 672 |
-
V20 = permute(reshape(batchedMul(Wv20, Xna20), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 673 |
-
Krc20 = reshapeBatch(Kr20, {hd, T, HkvB, 1});
|
| 674 |
-
Krt20 = tile(Krc20, {1, 1, 1, HkvMul});
|
| 675 |
-
Krp20 = permute(Krt20, {0, 1, 3, 2});
|
| 676 |
-
Kt20 = reshapeBatch(Krp20, {hd, T, 1, HB});
|
| 677 |
-
Vrc20 = reshapeBatch(V20, {hd, T, HkvB, 1});
|
| 678 |
-
Vrt20 = tile(Vrc20, {1, 1, 1, HkvMul});
|
| 679 |
-
Vrp20 = permute(Vrt20, {0, 1, 3, 2});
|
| 680 |
-
Vt20 = reshapeBatch(Vrp20, {hd, T, 1, HB});
|
| 681 |
-
A20 = attention(Qr20, Kt20, Vt20, hd, 1);
|
| 682 |
-
Wo20 = MatrixWithName("gemm_143.weight", D, Dq, 1, 1);
|
| 683 |
-
O20 = batchedMul(Wo20, reshape(permute(A20, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 684 |
-
R20 = X20 + O20;
|
| 685 |
-
Wrf20 = MatrixWithName("rmsn_364.weight", D, 1);
|
| 686 |
-
Rfn20 = rmsNorm(R20, Wrf20, 1e-5);
|
| 687 |
-
Wg20 = MatrixWithName("gemm_144.weight", I, D, 1, 1);
|
| 688 |
-
Wu20 = MatrixWithName("gemm_145.weight", I, D, 1, 1);
|
| 689 |
-
Wd20 = MatrixWithName("gemm_146.weight", D, I, 1, 1);
|
| 690 |
-
X21 = R20 + batchedMul(Wd20, elementMul(silu(batchedMul(Wg20, Rfn20)), batchedMul(Wu20, Rfn20)));
|
| 691 |
-
Wra21 = MatrixWithName("rmsn_365.weight", D, 1);
|
| 692 |
-
Xna21 = rmsNorm(X21, Wra21, 1e-5);
|
| 693 |
-
Wq21 = MatrixWithName("gemm_147.weight", Dq, D, 1, 1);
|
| 694 |
-
Qh21 = permute(reshape(batchedMul(Wq21, Xna21), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 695 |
-
Wqn21 = MatrixWithName("rmsn_366.weight", hd, 1);
|
| 696 |
-
Qn21 = rmsNorm(Qh21, Wqn21, 1e-5);
|
| 697 |
-
Qr21 = rope(Qn21, cos_te, sin_te);
|
| 698 |
-
Wk21 = MatrixWithName("gemm_148.weight", Dkv, D, 1, 1);
|
| 699 |
-
Kh21 = permute(reshape(batchedMul(Wk21, Xna21), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 700 |
-
Wkn21 = MatrixWithName("rmsn_367.weight", hd, 1);
|
| 701 |
-
Kn21 = rmsNorm(Kh21, Wkn21, 1e-5);
|
| 702 |
-
Kr21 = rope(Kn21, cos_te, sin_te);
|
| 703 |
-
Wv21 = MatrixWithName("gemm_149.weight", Dkv, D, 1, 1);
|
| 704 |
-
V21 = permute(reshape(batchedMul(Wv21, Xna21), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 705 |
-
Krc21 = reshapeBatch(Kr21, {hd, T, HkvB, 1});
|
| 706 |
-
Krt21 = tile(Krc21, {1, 1, 1, HkvMul});
|
| 707 |
-
Krp21 = permute(Krt21, {0, 1, 3, 2});
|
| 708 |
-
Kt21 = reshapeBatch(Krp21, {hd, T, 1, HB});
|
| 709 |
-
Vrc21 = reshapeBatch(V21, {hd, T, HkvB, 1});
|
| 710 |
-
Vrt21 = tile(Vrc21, {1, 1, 1, HkvMul});
|
| 711 |
-
Vrp21 = permute(Vrt21, {0, 1, 3, 2});
|
| 712 |
-
Vt21 = reshapeBatch(Vrp21, {hd, T, 1, HB});
|
| 713 |
-
A21 = attention(Qr21, Kt21, Vt21, hd, 1);
|
| 714 |
-
Wo21 = MatrixWithName("gemm_150.weight", D, Dq, 1, 1);
|
| 715 |
-
O21 = batchedMul(Wo21, reshape(permute(A21, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 716 |
-
R21 = X21 + O21;
|
| 717 |
-
Wrf21 = MatrixWithName("rmsn_368.weight", D, 1);
|
| 718 |
-
Rfn21 = rmsNorm(R21, Wrf21, 1e-5);
|
| 719 |
-
Wg21 = MatrixWithName("gemm_151.weight", I, D, 1, 1);
|
| 720 |
-
Wu21 = MatrixWithName("gemm_152.weight", I, D, 1, 1);
|
| 721 |
-
Wd21 = MatrixWithName("gemm_153.weight", D, I, 1, 1);
|
| 722 |
-
X22 = R21 + batchedMul(Wd21, elementMul(silu(batchedMul(Wg21, Rfn21)), batchedMul(Wu21, Rfn21)));
|
| 723 |
-
Wra22 = MatrixWithName("rmsn_369.weight", D, 1);
|
| 724 |
-
Xna22 = rmsNorm(X22, Wra22, 1e-5);
|
| 725 |
-
Wq22 = MatrixWithName("gemm_154.weight", Dq, D, 1, 1);
|
| 726 |
-
Qh22 = permute(reshape(batchedMul(Wq22, Xna22), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 727 |
-
Wqn22 = MatrixWithName("rmsn_370.weight", hd, 1);
|
| 728 |
-
Qn22 = rmsNorm(Qh22, Wqn22, 1e-5);
|
| 729 |
-
Qr22 = rope(Qn22, cos_te, sin_te);
|
| 730 |
-
Wk22 = MatrixWithName("gemm_155.weight", Dkv, D, 1, 1);
|
| 731 |
-
Kh22 = permute(reshape(batchedMul(Wk22, Xna22), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 732 |
-
Wkn22 = MatrixWithName("rmsn_371.weight", hd, 1);
|
| 733 |
-
Kn22 = rmsNorm(Kh22, Wkn22, 1e-5);
|
| 734 |
-
Kr22 = rope(Kn22, cos_te, sin_te);
|
| 735 |
-
Wv22 = MatrixWithName("gemm_156.weight", Dkv, D, 1, 1);
|
| 736 |
-
V22 = permute(reshape(batchedMul(Wv22, Xna22), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 737 |
-
Krc22 = reshapeBatch(Kr22, {hd, T, HkvB, 1});
|
| 738 |
-
Krt22 = tile(Krc22, {1, 1, 1, HkvMul});
|
| 739 |
-
Krp22 = permute(Krt22, {0, 1, 3, 2});
|
| 740 |
-
Kt22 = reshapeBatch(Krp22, {hd, T, 1, HB});
|
| 741 |
-
Vrc22 = reshapeBatch(V22, {hd, T, HkvB, 1});
|
| 742 |
-
Vrt22 = tile(Vrc22, {1, 1, 1, HkvMul});
|
| 743 |
-
Vrp22 = permute(Vrt22, {0, 1, 3, 2});
|
| 744 |
-
Vt22 = reshapeBatch(Vrp22, {hd, T, 1, HB});
|
| 745 |
-
A22 = attention(Qr22, Kt22, Vt22, hd, 1);
|
| 746 |
-
Wo22 = MatrixWithName("gemm_157.weight", D, Dq, 1, 1);
|
| 747 |
-
O22 = batchedMul(Wo22, reshape(permute(A22, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 748 |
-
R22 = X22 + O22;
|
| 749 |
-
Wrf22 = MatrixWithName("rmsn_372.weight", D, 1);
|
| 750 |
-
Rfn22 = rmsNorm(R22, Wrf22, 1e-5);
|
| 751 |
-
Wg22 = MatrixWithName("gemm_158.weight", I, D, 1, 1);
|
| 752 |
-
Wu22 = MatrixWithName("gemm_159.weight", I, D, 1, 1);
|
| 753 |
-
Wd22 = MatrixWithName("gemm_160.weight", D, I, 1, 1);
|
| 754 |
-
X23 = R22 + batchedMul(Wd22, elementMul(silu(batchedMul(Wg22, Rfn22)), batchedMul(Wu22, Rfn22)));
|
| 755 |
-
Wra23 = MatrixWithName("rmsn_373.weight", D, 1);
|
| 756 |
-
Xna23 = rmsNorm(X23, Wra23, 1e-5);
|
| 757 |
-
Wq23 = MatrixWithName("gemm_161.weight", Dq, D, 1, 1);
|
| 758 |
-
Qh23 = permute(reshape(batchedMul(Wq23, Xna23), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 759 |
-
Wqn23 = MatrixWithName("rmsn_374.weight", hd, 1);
|
| 760 |
-
Qn23 = rmsNorm(Qh23, Wqn23, 1e-5);
|
| 761 |
-
Qr23 = rope(Qn23, cos_te, sin_te);
|
| 762 |
-
Wk23 = MatrixWithName("gemm_162.weight", Dkv, D, 1, 1);
|
| 763 |
-
Kh23 = permute(reshape(batchedMul(Wk23, Xna23), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 764 |
-
Wkn23 = MatrixWithName("rmsn_375.weight", hd, 1);
|
| 765 |
-
Kn23 = rmsNorm(Kh23, Wkn23, 1e-5);
|
| 766 |
-
Kr23 = rope(Kn23, cos_te, sin_te);
|
| 767 |
-
Wv23 = MatrixWithName("gemm_163.weight", Dkv, D, 1, 1);
|
| 768 |
-
V23 = permute(reshape(batchedMul(Wv23, Xna23), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 769 |
-
Krc23 = reshapeBatch(Kr23, {hd, T, HkvB, 1});
|
| 770 |
-
Krt23 = tile(Krc23, {1, 1, 1, HkvMul});
|
| 771 |
-
Krp23 = permute(Krt23, {0, 1, 3, 2});
|
| 772 |
-
Kt23 = reshapeBatch(Krp23, {hd, T, 1, HB});
|
| 773 |
-
Vrc23 = reshapeBatch(V23, {hd, T, HkvB, 1});
|
| 774 |
-
Vrt23 = tile(Vrc23, {1, 1, 1, HkvMul});
|
| 775 |
-
Vrp23 = permute(Vrt23, {0, 1, 3, 2});
|
| 776 |
-
Vt23 = reshapeBatch(Vrp23, {hd, T, 1, HB});
|
| 777 |
-
A23 = attention(Qr23, Kt23, Vt23, hd, 1);
|
| 778 |
-
Wo23 = MatrixWithName("gemm_164.weight", D, Dq, 1, 1);
|
| 779 |
-
O23 = batchedMul(Wo23, reshape(permute(A23, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 780 |
-
R23 = X23 + O23;
|
| 781 |
-
Wrf23 = MatrixWithName("rmsn_376.weight", D, 1);
|
| 782 |
-
Rfn23 = rmsNorm(R23, Wrf23, 1e-5);
|
| 783 |
-
Wg23 = MatrixWithName("gemm_165.weight", I, D, 1, 1);
|
| 784 |
-
Wu23 = MatrixWithName("gemm_166.weight", I, D, 1, 1);
|
| 785 |
-
Wd23 = MatrixWithName("gemm_167.weight", D, I, 1, 1);
|
| 786 |
-
X24 = R23 + batchedMul(Wd23, elementMul(silu(batchedMul(Wg23, Rfn23)), batchedMul(Wu23, Rfn23)));
|
| 787 |
-
Wra24 = MatrixWithName("rmsn_377.weight", D, 1);
|
| 788 |
-
Xna24 = rmsNorm(X24, Wra24, 1e-5);
|
| 789 |
-
Wq24 = MatrixWithName("gemm_168.weight", Dq, D, 1, 1);
|
| 790 |
-
Qh24 = permute(reshape(batchedMul(Wq24, Xna24), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 791 |
-
Wqn24 = MatrixWithName("rmsn_378.weight", hd, 1);
|
| 792 |
-
Qn24 = rmsNorm(Qh24, Wqn24, 1e-5);
|
| 793 |
-
Qr24 = rope(Qn24, cos_te, sin_te);
|
| 794 |
-
Wk24 = MatrixWithName("gemm_169.weight", Dkv, D, 1, 1);
|
| 795 |
-
Kh24 = permute(reshape(batchedMul(Wk24, Xna24), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 796 |
-
Wkn24 = MatrixWithName("rmsn_379.weight", hd, 1);
|
| 797 |
-
Kn24 = rmsNorm(Kh24, Wkn24, 1e-5);
|
| 798 |
-
Kr24 = rope(Kn24, cos_te, sin_te);
|
| 799 |
-
Wv24 = MatrixWithName("gemm_170.weight", Dkv, D, 1, 1);
|
| 800 |
-
V24 = permute(reshape(batchedMul(Wv24, Xna24), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 801 |
-
Krc24 = reshapeBatch(Kr24, {hd, T, HkvB, 1});
|
| 802 |
-
Krt24 = tile(Krc24, {1, 1, 1, HkvMul});
|
| 803 |
-
Krp24 = permute(Krt24, {0, 1, 3, 2});
|
| 804 |
-
Kt24 = reshapeBatch(Krp24, {hd, T, 1, HB});
|
| 805 |
-
Vrc24 = reshapeBatch(V24, {hd, T, HkvB, 1});
|
| 806 |
-
Vrt24 = tile(Vrc24, {1, 1, 1, HkvMul});
|
| 807 |
-
Vrp24 = permute(Vrt24, {0, 1, 3, 2});
|
| 808 |
-
Vt24 = reshapeBatch(Vrp24, {hd, T, 1, HB});
|
| 809 |
-
A24 = attention(Qr24, Kt24, Vt24, hd, 1);
|
| 810 |
-
Wo24 = MatrixWithName("gemm_171.weight", D, Dq, 1, 1);
|
| 811 |
-
O24 = batchedMul(Wo24, reshape(permute(A24, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 812 |
-
R24 = X24 + O24;
|
| 813 |
-
Wrf24 = MatrixWithName("rmsn_380.weight", D, 1);
|
| 814 |
-
Rfn24 = rmsNorm(R24, Wrf24, 1e-5);
|
| 815 |
-
Wg24 = MatrixWithName("gemm_172.weight", I, D, 1, 1);
|
| 816 |
-
Wu24 = MatrixWithName("gemm_173.weight", I, D, 1, 1);
|
| 817 |
-
Wd24 = MatrixWithName("gemm_174.weight", D, I, 1, 1);
|
| 818 |
-
X25 = R24 + batchedMul(Wd24, elementMul(silu(batchedMul(Wg24, Rfn24)), batchedMul(Wu24, Rfn24)));
|
| 819 |
-
Wra25 = MatrixWithName("rmsn_381.weight", D, 1);
|
| 820 |
-
Xna25 = rmsNorm(X25, Wra25, 1e-5);
|
| 821 |
-
Wq25 = MatrixWithName("gemm_175.weight", Dq, D, 1, 1);
|
| 822 |
-
Qh25 = permute(reshape(batchedMul(Wq25, Xna25), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 823 |
-
Wqn25 = MatrixWithName("rmsn_382.weight", hd, 1);
|
| 824 |
-
Qn25 = rmsNorm(Qh25, Wqn25, 1e-5);
|
| 825 |
-
Qr25 = rope(Qn25, cos_te, sin_te);
|
| 826 |
-
Wk25 = MatrixWithName("gemm_176.weight", Dkv, D, 1, 1);
|
| 827 |
-
Kh25 = permute(reshape(batchedMul(Wk25, Xna25), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 828 |
-
Wkn25 = MatrixWithName("rmsn_383.weight", hd, 1);
|
| 829 |
-
Kn25 = rmsNorm(Kh25, Wkn25, 1e-5);
|
| 830 |
-
Kr25 = rope(Kn25, cos_te, sin_te);
|
| 831 |
-
Wv25 = MatrixWithName("gemm_177.weight", Dkv, D, 1, 1);
|
| 832 |
-
V25 = permute(reshape(batchedMul(Wv25, Xna25), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 833 |
-
Krc25 = reshapeBatch(Kr25, {hd, T, HkvB, 1});
|
| 834 |
-
Krt25 = tile(Krc25, {1, 1, 1, HkvMul});
|
| 835 |
-
Krp25 = permute(Krt25, {0, 1, 3, 2});
|
| 836 |
-
Kt25 = reshapeBatch(Krp25, {hd, T, 1, HB});
|
| 837 |
-
Vrc25 = reshapeBatch(V25, {hd, T, HkvB, 1});
|
| 838 |
-
Vrt25 = tile(Vrc25, {1, 1, 1, HkvMul});
|
| 839 |
-
Vrp25 = permute(Vrt25, {0, 1, 3, 2});
|
| 840 |
-
Vt25 = reshapeBatch(Vrp25, {hd, T, 1, HB});
|
| 841 |
-
A25 = attention(Qr25, Kt25, Vt25, hd, 1);
|
| 842 |
-
Wo25 = MatrixWithName("gemm_178.weight", D, Dq, 1, 1);
|
| 843 |
-
O25 = batchedMul(Wo25, reshape(permute(A25, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 844 |
-
R25 = X25 + O25;
|
| 845 |
-
Wrf25 = MatrixWithName("rmsn_384.weight", D, 1);
|
| 846 |
-
Rfn25 = rmsNorm(R25, Wrf25, 1e-5);
|
| 847 |
-
Wg25 = MatrixWithName("gemm_179.weight", I, D, 1, 1);
|
| 848 |
-
Wu25 = MatrixWithName("gemm_180.weight", I, D, 1, 1);
|
| 849 |
-
Wd25 = MatrixWithName("gemm_181.weight", D, I, 1, 1);
|
| 850 |
-
X26 = R25 + batchedMul(Wd25, elementMul(silu(batchedMul(Wg25, Rfn25)), batchedMul(Wu25, Rfn25)));
|
| 851 |
-
Wra26 = MatrixWithName("rmsn_385.weight", D, 1);
|
| 852 |
-
Xna26 = rmsNorm(X26, Wra26, 1e-5);
|
| 853 |
-
Wq26 = MatrixWithName("gemm_182.weight", Dq, D, 1, 1);
|
| 854 |
-
Qh26 = permute(reshape(batchedMul(Wq26, Xna26), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 855 |
-
Wqn26 = MatrixWithName("rmsn_386.weight", hd, 1);
|
| 856 |
-
Qn26 = rmsNorm(Qh26, Wqn26, 1e-5);
|
| 857 |
-
Qr26 = rope(Qn26, cos_te, sin_te);
|
| 858 |
-
Wk26 = MatrixWithName("gemm_183.weight", Dkv, D, 1, 1);
|
| 859 |
-
Kh26 = permute(reshape(batchedMul(Wk26, Xna26), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 860 |
-
Wkn26 = MatrixWithName("rmsn_387.weight", hd, 1);
|
| 861 |
-
Kn26 = rmsNorm(Kh26, Wkn26, 1e-5);
|
| 862 |
-
Kr26 = rope(Kn26, cos_te, sin_te);
|
| 863 |
-
Wv26 = MatrixWithName("gemm_184.weight", Dkv, D, 1, 1);
|
| 864 |
-
V26 = permute(reshape(batchedMul(Wv26, Xna26), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 865 |
-
Krc26 = reshapeBatch(Kr26, {hd, T, HkvB, 1});
|
| 866 |
-
Krt26 = tile(Krc26, {1, 1, 1, HkvMul});
|
| 867 |
-
Krp26 = permute(Krt26, {0, 1, 3, 2});
|
| 868 |
-
Kt26 = reshapeBatch(Krp26, {hd, T, 1, HB});
|
| 869 |
-
Vrc26 = reshapeBatch(V26, {hd, T, HkvB, 1});
|
| 870 |
-
Vrt26 = tile(Vrc26, {1, 1, 1, HkvMul});
|
| 871 |
-
Vrp26 = permute(Vrt26, {0, 1, 3, 2});
|
| 872 |
-
Vt26 = reshapeBatch(Vrp26, {hd, T, 1, HB});
|
| 873 |
-
A26 = attention(Qr26, Kt26, Vt26, hd, 1);
|
| 874 |
-
Wo26 = MatrixWithName("gemm_185.weight", D, Dq, 1, 1);
|
| 875 |
-
O26 = batchedMul(Wo26, reshape(permute(A26, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 876 |
-
R26 = X26 + O26;
|
| 877 |
-
Wrf26 = MatrixWithName("rmsn_388.weight", D, 1);
|
| 878 |
-
Rfn26 = rmsNorm(R26, Wrf26, 1e-5);
|
| 879 |
-
Wg26 = MatrixWithName("gemm_186.weight", I, D, 1, 1);
|
| 880 |
-
Wu26 = MatrixWithName("gemm_187.weight", I, D, 1, 1);
|
| 881 |
-
Wd26 = MatrixWithName("gemm_188.weight", D, I, 1, 1);
|
| 882 |
-
X27 = R26 + batchedMul(Wd26, elementMul(silu(batchedMul(Wg26, Rfn26)), batchedMul(Wu26, Rfn26)));
|
| 883 |
-
Wra27 = MatrixWithName("rmsn_389.weight", D, 1);
|
| 884 |
-
Xna27 = rmsNorm(X27, Wra27, 1e-5);
|
| 885 |
-
Wq27 = MatrixWithName("gemm_189.weight", Dq, D, 1, 1);
|
| 886 |
-
Qh27 = permute(reshape(batchedMul(Wq27, Xna27), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 887 |
-
Wqn27 = MatrixWithName("rmsn_390.weight", hd, 1);
|
| 888 |
-
Qn27 = rmsNorm(Qh27, Wqn27, 1e-5);
|
| 889 |
-
Qr27 = rope(Qn27, cos_te, sin_te);
|
| 890 |
-
Wk27 = MatrixWithName("gemm_190.weight", Dkv, D, 1, 1);
|
| 891 |
-
Kh27 = permute(reshape(batchedMul(Wk27, Xna27), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 892 |
-
Wkn27 = MatrixWithName("rmsn_391.weight", hd, 1);
|
| 893 |
-
Kn27 = rmsNorm(Kh27, Wkn27, 1e-5);
|
| 894 |
-
Kr27 = rope(Kn27, cos_te, sin_te);
|
| 895 |
-
Wv27 = MatrixWithName("gemm_191.weight", Dkv, D, 1, 1);
|
| 896 |
-
V27 = permute(reshape(batchedMul(Wv27, Xna27), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 897 |
-
Krc27 = reshapeBatch(Kr27, {hd, T, HkvB, 1});
|
| 898 |
-
Krt27 = tile(Krc27, {1, 1, 1, HkvMul});
|
| 899 |
-
Krp27 = permute(Krt27, {0, 1, 3, 2});
|
| 900 |
-
Kt27 = reshapeBatch(Krp27, {hd, T, 1, HB});
|
| 901 |
-
Vrc27 = reshapeBatch(V27, {hd, T, HkvB, 1});
|
| 902 |
-
Vrt27 = tile(Vrc27, {1, 1, 1, HkvMul});
|
| 903 |
-
Vrp27 = permute(Vrt27, {0, 1, 3, 2});
|
| 904 |
-
Vt27 = reshapeBatch(Vrp27, {hd, T, 1, HB});
|
| 905 |
-
A27 = attention(Qr27, Kt27, Vt27, hd, 1);
|
| 906 |
-
Wo27 = MatrixWithName("gemm_192.weight", D, Dq, 1, 1);
|
| 907 |
-
O27 = batchedMul(Wo27, reshape(permute(A27, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 908 |
-
R27 = X27 + O27;
|
| 909 |
-
Wrf27 = MatrixWithName("rmsn_392.weight", D, 1);
|
| 910 |
-
Rfn27 = rmsNorm(R27, Wrf27, 1e-5);
|
| 911 |
-
Wg27 = MatrixWithName("gemm_193.weight", I, D, 1, 1);
|
| 912 |
-
Wu27 = MatrixWithName("gemm_194.weight", I, D, 1, 1);
|
| 913 |
-
Wd27 = MatrixWithName("gemm_195.weight", D, I, 1, 1);
|
| 914 |
-
X28 = R27 + batchedMul(Wd27, elementMul(silu(batchedMul(Wg27, Rfn27)), batchedMul(Wu27, Rfn27)));
|
| 915 |
-
Wra28 = MatrixWithName("rmsn_393.weight", D, 1);
|
| 916 |
-
Xna28 = rmsNorm(X28, Wra28, 1e-5);
|
| 917 |
-
Wq28 = MatrixWithName("gemm_196.weight", Dq, D, 1, 1);
|
| 918 |
-
Qh28 = permute(reshape(batchedMul(Wq28, Xna28), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 919 |
-
Wqn28 = MatrixWithName("rmsn_394.weight", hd, 1);
|
| 920 |
-
Qn28 = rmsNorm(Qh28, Wqn28, 1e-5);
|
| 921 |
-
Qr28 = rope(Qn28, cos_te, sin_te);
|
| 922 |
-
Wk28 = MatrixWithName("gemm_197.weight", Dkv, D, 1, 1);
|
| 923 |
-
Kh28 = permute(reshape(batchedMul(Wk28, Xna28), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 924 |
-
Wkn28 = MatrixWithName("rmsn_395.weight", hd, 1);
|
| 925 |
-
Kn28 = rmsNorm(Kh28, Wkn28, 1e-5);
|
| 926 |
-
Kr28 = rope(Kn28, cos_te, sin_te);
|
| 927 |
-
Wv28 = MatrixWithName("gemm_198.weight", Dkv, D, 1, 1);
|
| 928 |
-
V28 = permute(reshape(batchedMul(Wv28, Xna28), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 929 |
-
Krc28 = reshapeBatch(Kr28, {hd, T, HkvB, 1});
|
| 930 |
-
Krt28 = tile(Krc28, {1, 1, 1, HkvMul});
|
| 931 |
-
Krp28 = permute(Krt28, {0, 1, 3, 2});
|
| 932 |
-
Kt28 = reshapeBatch(Krp28, {hd, T, 1, HB});
|
| 933 |
-
Vrc28 = reshapeBatch(V28, {hd, T, HkvB, 1});
|
| 934 |
-
Vrt28 = tile(Vrc28, {1, 1, 1, HkvMul});
|
| 935 |
-
Vrp28 = permute(Vrt28, {0, 1, 3, 2});
|
| 936 |
-
Vt28 = reshapeBatch(Vrp28, {hd, T, 1, HB});
|
| 937 |
-
A28 = attention(Qr28, Kt28, Vt28, hd, 1);
|
| 938 |
-
Wo28 = MatrixWithName("gemm_199.weight", D, Dq, 1, 1);
|
| 939 |
-
O28 = batchedMul(Wo28, reshape(permute(A28, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 940 |
-
R28 = X28 + O28;
|
| 941 |
-
Wrf28 = MatrixWithName("rmsn_396.weight", D, 1);
|
| 942 |
-
Rfn28 = rmsNorm(R28, Wrf28, 1e-5);
|
| 943 |
-
Wg28 = MatrixWithName("gemm_200.weight", I, D, 1, 1);
|
| 944 |
-
Wu28 = MatrixWithName("gemm_201.weight", I, D, 1, 1);
|
| 945 |
-
Wd28 = MatrixWithName("gemm_202.weight", D, I, 1, 1);
|
| 946 |
-
X29 = R28 + batchedMul(Wd28, elementMul(silu(batchedMul(Wg28, Rfn28)), batchedMul(Wu28, Rfn28)));
|
| 947 |
-
Wra29 = MatrixWithName("rmsn_397.weight", D, 1);
|
| 948 |
-
Xna29 = rmsNorm(X29, Wra29, 1e-5);
|
| 949 |
-
Wq29 = MatrixWithName("gemm_203.weight", Dq, D, 1, 1);
|
| 950 |
-
Qh29 = permute(reshape(batchedMul(Wq29, Xna29), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 951 |
-
Wqn29 = MatrixWithName("rmsn_398.weight", hd, 1);
|
| 952 |
-
Qn29 = rmsNorm(Qh29, Wqn29, 1e-5);
|
| 953 |
-
Qr29 = rope(Qn29, cos_te, sin_te);
|
| 954 |
-
Wk29 = MatrixWithName("gemm_204.weight", Dkv, D, 1, 1);
|
| 955 |
-
Kh29 = permute(reshape(batchedMul(Wk29, Xna29), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 956 |
-
Wkn29 = MatrixWithName("rmsn_399.weight", hd, 1);
|
| 957 |
-
Kn29 = rmsNorm(Kh29, Wkn29, 1e-5);
|
| 958 |
-
Kr29 = rope(Kn29, cos_te, sin_te);
|
| 959 |
-
Wv29 = MatrixWithName("gemm_205.weight", Dkv, D, 1, 1);
|
| 960 |
-
V29 = permute(reshape(batchedMul(Wv29, Xna29), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 961 |
-
Krc29 = reshapeBatch(Kr29, {hd, T, HkvB, 1});
|
| 962 |
-
Krt29 = tile(Krc29, {1, 1, 1, HkvMul});
|
| 963 |
-
Krp29 = permute(Krt29, {0, 1, 3, 2});
|
| 964 |
-
Kt29 = reshapeBatch(Krp29, {hd, T, 1, HB});
|
| 965 |
-
Vrc29 = reshapeBatch(V29, {hd, T, HkvB, 1});
|
| 966 |
-
Vrt29 = tile(Vrc29, {1, 1, 1, HkvMul});
|
| 967 |
-
Vrp29 = permute(Vrt29, {0, 1, 3, 2});
|
| 968 |
-
Vt29 = reshapeBatch(Vrp29, {hd, T, 1, HB});
|
| 969 |
-
A29 = attention(Qr29, Kt29, Vt29, hd, 1);
|
| 970 |
-
Wo29 = MatrixWithName("gemm_206.weight", D, Dq, 1, 1);
|
| 971 |
-
O29 = batchedMul(Wo29, reshape(permute(A29, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 972 |
-
R29 = X29 + O29;
|
| 973 |
-
Wrf29 = MatrixWithName("rmsn_400.weight", D, 1);
|
| 974 |
-
Rfn29 = rmsNorm(R29, Wrf29, 1e-5);
|
| 975 |
-
Wg29 = MatrixWithName("gemm_207.weight", I, D, 1, 1);
|
| 976 |
-
Wu29 = MatrixWithName("gemm_208.weight", I, D, 1, 1);
|
| 977 |
-
Wd29 = MatrixWithName("gemm_209.weight", D, I, 1, 1);
|
| 978 |
-
X30 = R29 + batchedMul(Wd29, elementMul(silu(batchedMul(Wg29, Rfn29)), batchedMul(Wu29, Rfn29)));
|
| 979 |
-
Wra30 = MatrixWithName("rmsn_401.weight", D, 1);
|
| 980 |
-
Xna30 = rmsNorm(X30, Wra30, 1e-5);
|
| 981 |
-
Wq30 = MatrixWithName("gemm_210.weight", Dq, D, 1, 1);
|
| 982 |
-
Qh30 = permute(reshape(batchedMul(Wq30, Xna30), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 983 |
-
Wqn30 = MatrixWithName("rmsn_402.weight", hd, 1);
|
| 984 |
-
Qn30 = rmsNorm(Qh30, Wqn30, 1e-5);
|
| 985 |
-
Qr30 = rope(Qn30, cos_te, sin_te);
|
| 986 |
-
Wk30 = MatrixWithName("gemm_211.weight", Dkv, D, 1, 1);
|
| 987 |
-
Kh30 = permute(reshape(batchedMul(Wk30, Xna30), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 988 |
-
Wkn30 = MatrixWithName("rmsn_403.weight", hd, 1);
|
| 989 |
-
Kn30 = rmsNorm(Kh30, Wkn30, 1e-5);
|
| 990 |
-
Kr30 = rope(Kn30, cos_te, sin_te);
|
| 991 |
-
Wv30 = MatrixWithName("gemm_212.weight", Dkv, D, 1, 1);
|
| 992 |
-
V30 = permute(reshape(batchedMul(Wv30, Xna30), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 993 |
-
Krc30 = reshapeBatch(Kr30, {hd, T, HkvB, 1});
|
| 994 |
-
Krt30 = tile(Krc30, {1, 1, 1, HkvMul});
|
| 995 |
-
Krp30 = permute(Krt30, {0, 1, 3, 2});
|
| 996 |
-
Kt30 = reshapeBatch(Krp30, {hd, T, 1, HB});
|
| 997 |
-
Vrc30 = reshapeBatch(V30, {hd, T, HkvB, 1});
|
| 998 |
-
Vrt30 = tile(Vrc30, {1, 1, 1, HkvMul});
|
| 999 |
-
Vrp30 = permute(Vrt30, {0, 1, 3, 2});
|
| 1000 |
-
Vt30 = reshapeBatch(Vrp30, {hd, T, 1, HB});
|
| 1001 |
-
A30 = attention(Qr30, Kt30, Vt30, hd, 1);
|
| 1002 |
-
Wo30 = MatrixWithName("gemm_213.weight", D, Dq, 1, 1);
|
| 1003 |
-
O30 = batchedMul(Wo30, reshape(permute(A30, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 1004 |
-
R30 = X30 + O30;
|
| 1005 |
-
Wrf30 = MatrixWithName("rmsn_404.weight", D, 1);
|
| 1006 |
-
Rfn30 = rmsNorm(R30, Wrf30, 1e-5);
|
| 1007 |
-
Wg30 = MatrixWithName("gemm_214.weight", I, D, 1, 1);
|
| 1008 |
-
Wu30 = MatrixWithName("gemm_215.weight", I, D, 1, 1);
|
| 1009 |
-
Wd30 = MatrixWithName("gemm_216.weight", D, I, 1, 1);
|
| 1010 |
-
X31 = R30 + batchedMul(Wd30, elementMul(silu(batchedMul(Wg30, Rfn30)), batchedMul(Wu30, Rfn30)));
|
| 1011 |
-
Wra31 = MatrixWithName("rmsn_405.weight", D, 1);
|
| 1012 |
-
Xna31 = rmsNorm(X31, Wra31, 1e-5);
|
| 1013 |
-
Wq31 = MatrixWithName("gemm_217.weight", Dq, D, 1, 1);
|
| 1014 |
-
Qh31 = permute(reshape(batchedMul(Wq31, Xna31), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 1015 |
-
Wqn31 = MatrixWithName("rmsn_406.weight", hd, 1);
|
| 1016 |
-
Qn31 = rmsNorm(Qh31, Wqn31, 1e-5);
|
| 1017 |
-
Qr31 = rope(Qn31, cos_te, sin_te);
|
| 1018 |
-
Wk31 = MatrixWithName("gemm_218.weight", Dkv, D, 1, 1);
|
| 1019 |
-
Kh31 = permute(reshape(batchedMul(Wk31, Xna31), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 1020 |
-
Wkn31 = MatrixWithName("rmsn_407.weight", hd, 1);
|
| 1021 |
-
Kn31 = rmsNorm(Kh31, Wkn31, 1e-5);
|
| 1022 |
-
Kr31 = rope(Kn31, cos_te, sin_te);
|
| 1023 |
-
Wv31 = MatrixWithName("gemm_219.weight", Dkv, D, 1, 1);
|
| 1024 |
-
V31 = permute(reshape(batchedMul(Wv31, Xna31), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 1025 |
-
Krc31 = reshapeBatch(Kr31, {hd, T, HkvB, 1});
|
| 1026 |
-
Krt31 = tile(Krc31, {1, 1, 1, HkvMul});
|
| 1027 |
-
Krp31 = permute(Krt31, {0, 1, 3, 2});
|
| 1028 |
-
Kt31 = reshapeBatch(Krp31, {hd, T, 1, HB});
|
| 1029 |
-
Vrc31 = reshapeBatch(V31, {hd, T, HkvB, 1});
|
| 1030 |
-
Vrt31 = tile(Vrc31, {1, 1, 1, HkvMul});
|
| 1031 |
-
Vrp31 = permute(Vrt31, {0, 1, 3, 2});
|
| 1032 |
-
Vt31 = reshapeBatch(Vrp31, {hd, T, 1, HB});
|
| 1033 |
-
A31 = attention(Qr31, Kt31, Vt31, hd, 1);
|
| 1034 |
-
Wo31 = MatrixWithName("gemm_220.weight", D, Dq, 1, 1);
|
| 1035 |
-
O31 = batchedMul(Wo31, reshape(permute(A31, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 1036 |
-
R31 = X31 + O31;
|
| 1037 |
-
Wrf31 = MatrixWithName("rmsn_408.weight", D, 1);
|
| 1038 |
-
Rfn31 = rmsNorm(R31, Wrf31, 1e-5);
|
| 1039 |
-
Wg31 = MatrixWithName("gemm_221.weight", I, D, 1, 1);
|
| 1040 |
-
Wu31 = MatrixWithName("gemm_222.weight", I, D, 1, 1);
|
| 1041 |
-
Wd31 = MatrixWithName("gemm_223.weight", D, I, 1, 1);
|
| 1042 |
-
X32 = R31 + batchedMul(Wd31, elementMul(silu(batchedMul(Wg31, Rfn31)), batchedMul(Wu31, Rfn31)));
|
| 1043 |
-
Wra32 = MatrixWithName("rmsn_409.weight", D, 1);
|
| 1044 |
-
Xna32 = rmsNorm(X32, Wra32, 1e-5);
|
| 1045 |
-
Wq32 = MatrixWithName("gemm_224.weight", Dq, D, 1, 1);
|
| 1046 |
-
Qh32 = permute(reshape(batchedMul(Wq32, Xna32), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 1047 |
-
Wqn32 = MatrixWithName("rmsn_410.weight", hd, 1);
|
| 1048 |
-
Qn32 = rmsNorm(Qh32, Wqn32, 1e-5);
|
| 1049 |
-
Qr32 = rope(Qn32, cos_te, sin_te);
|
| 1050 |
-
Wk32 = MatrixWithName("gemm_225.weight", Dkv, D, 1, 1);
|
| 1051 |
-
Kh32 = permute(reshape(batchedMul(Wk32, Xna32), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 1052 |
-
Wkn32 = MatrixWithName("rmsn_411.weight", hd, 1);
|
| 1053 |
-
Kn32 = rmsNorm(Kh32, Wkn32, 1e-5);
|
| 1054 |
-
Kr32 = rope(Kn32, cos_te, sin_te);
|
| 1055 |
-
Wv32 = MatrixWithName("gemm_226.weight", Dkv, D, 1, 1);
|
| 1056 |
-
V32 = permute(reshape(batchedMul(Wv32, Xna32), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 1057 |
-
Krc32 = reshapeBatch(Kr32, {hd, T, HkvB, 1});
|
| 1058 |
-
Krt32 = tile(Krc32, {1, 1, 1, HkvMul});
|
| 1059 |
-
Krp32 = permute(Krt32, {0, 1, 3, 2});
|
| 1060 |
-
Kt32 = reshapeBatch(Krp32, {hd, T, 1, HB});
|
| 1061 |
-
Vrc32 = reshapeBatch(V32, {hd, T, HkvB, 1});
|
| 1062 |
-
Vrt32 = tile(Vrc32, {1, 1, 1, HkvMul});
|
| 1063 |
-
Vrp32 = permute(Vrt32, {0, 1, 3, 2});
|
| 1064 |
-
Vt32 = reshapeBatch(Vrp32, {hd, T, 1, HB});
|
| 1065 |
-
A32 = attention(Qr32, Kt32, Vt32, hd, 1);
|
| 1066 |
-
Wo32 = MatrixWithName("gemm_227.weight", D, Dq, 1, 1);
|
| 1067 |
-
O32 = batchedMul(Wo32, reshape(permute(A32, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 1068 |
-
R32 = X32 + O32;
|
| 1069 |
-
Wrf32 = MatrixWithName("rmsn_412.weight", D, 1);
|
| 1070 |
-
Rfn32 = rmsNorm(R32, Wrf32, 1e-5);
|
| 1071 |
-
Wg32 = MatrixWithName("gemm_228.weight", I, D, 1, 1);
|
| 1072 |
-
Wu32 = MatrixWithName("gemm_229.weight", I, D, 1, 1);
|
| 1073 |
-
Wd32 = MatrixWithName("gemm_230.weight", D, I, 1, 1);
|
| 1074 |
-
X33 = R32 + batchedMul(Wd32, elementMul(silu(batchedMul(Wg32, Rfn32)), batchedMul(Wu32, Rfn32)));
|
| 1075 |
-
Wra33 = MatrixWithName("rmsn_413.weight", D, 1);
|
| 1076 |
-
Xna33 = rmsNorm(X33, Wra33, 1e-5);
|
| 1077 |
-
Wq33 = MatrixWithName("gemm_231.weight", Dq, D, 1, 1);
|
| 1078 |
-
Qh33 = permute(reshape(batchedMul(Wq33, Xna33), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 1079 |
-
Wqn33 = MatrixWithName("rmsn_414.weight", hd, 1);
|
| 1080 |
-
Qn33 = rmsNorm(Qh33, Wqn33, 1e-5);
|
| 1081 |
-
Qr33 = rope(Qn33, cos_te, sin_te);
|
| 1082 |
-
Wk33 = MatrixWithName("gemm_232.weight", Dkv, D, 1, 1);
|
| 1083 |
-
Kh33 = permute(reshape(batchedMul(Wk33, Xna33), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 1084 |
-
Wkn33 = MatrixWithName("rmsn_415.weight", hd, 1);
|
| 1085 |
-
Kn33 = rmsNorm(Kh33, Wkn33, 1e-5);
|
| 1086 |
-
Kr33 = rope(Kn33, cos_te, sin_te);
|
| 1087 |
-
Wv33 = MatrixWithName("gemm_233.weight", Dkv, D, 1, 1);
|
| 1088 |
-
V33 = permute(reshape(batchedMul(Wv33, Xna33), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 1089 |
-
Krc33 = reshapeBatch(Kr33, {hd, T, HkvB, 1});
|
| 1090 |
-
Krt33 = tile(Krc33, {1, 1, 1, HkvMul});
|
| 1091 |
-
Krp33 = permute(Krt33, {0, 1, 3, 2});
|
| 1092 |
-
Kt33 = reshapeBatch(Krp33, {hd, T, 1, HB});
|
| 1093 |
-
Vrc33 = reshapeBatch(V33, {hd, T, HkvB, 1});
|
| 1094 |
-
Vrt33 = tile(Vrc33, {1, 1, 1, HkvMul});
|
| 1095 |
-
Vrp33 = permute(Vrt33, {0, 1, 3, 2});
|
| 1096 |
-
Vt33 = reshapeBatch(Vrp33, {hd, T, 1, HB});
|
| 1097 |
-
A33 = attention(Qr33, Kt33, Vt33, hd, 1);
|
| 1098 |
-
Wo33 = MatrixWithName("gemm_234.weight", D, Dq, 1, 1);
|
| 1099 |
-
O33 = batchedMul(Wo33, reshape(permute(A33, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 1100 |
-
R33 = X33 + O33;
|
| 1101 |
-
Wrf33 = MatrixWithName("rmsn_416.weight", D, 1);
|
| 1102 |
-
Rfn33 = rmsNorm(R33, Wrf33, 1e-5);
|
| 1103 |
-
Wg33 = MatrixWithName("gemm_235.weight", I, D, 1, 1);
|
| 1104 |
-
Wu33 = MatrixWithName("gemm_236.weight", I, D, 1, 1);
|
| 1105 |
-
Wd33 = MatrixWithName("gemm_237.weight", D, I, 1, 1);
|
| 1106 |
-
X34 = R33 + batchedMul(Wd33, elementMul(silu(batchedMul(Wg33, Rfn33)), batchedMul(Wu33, Rfn33)));
|
| 1107 |
-
Wra34 = MatrixWithName("rmsn_417.weight", D, 1);
|
| 1108 |
-
Xna34 = rmsNorm(X34, Wra34, 1e-5);
|
| 1109 |
-
Wq34 = MatrixWithName("gemm_238.weight", Dq, D, 1, 1);
|
| 1110 |
-
Qh34 = permute(reshape(batchedMul(Wq34, Xna34), {hd, HB, T, 1}), {0, 2, 3, 1});
|
| 1111 |
-
Wqn34 = MatrixWithName("rmsn_418.weight", hd, 1);
|
| 1112 |
-
Qn34 = rmsNorm(Qh34, Wqn34, 1e-5);
|
| 1113 |
-
Qr34 = rope(Qn34, cos_te, sin_te);
|
| 1114 |
-
Wk34 = MatrixWithName("gemm_239.weight", Dkv, D, 1, 1);
|
| 1115 |
-
Kh34 = permute(reshape(batchedMul(Wk34, Xna34), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 1116 |
-
Wkn34 = MatrixWithName("rmsn_419.weight", hd, 1);
|
| 1117 |
-
Kn34 = rmsNorm(Kh34, Wkn34, 1e-5);
|
| 1118 |
-
Kr34 = rope(Kn34, cos_te, sin_te);
|
| 1119 |
-
Wv34 = MatrixWithName("gemm_240.weight", Dkv, D, 1, 1);
|
| 1120 |
-
V34 = permute(reshape(batchedMul(Wv34, Xna34), {hd, HkvB, T, 1}), {0, 2, 3, 1});
|
| 1121 |
-
Krc34 = reshapeBatch(Kr34, {hd, T, HkvB, 1});
|
| 1122 |
-
Krt34 = tile(Krc34, {1, 1, 1, HkvMul});
|
| 1123 |
-
Krp34 = permute(Krt34, {0, 1, 3, 2});
|
| 1124 |
-
Kt34 = reshapeBatch(Krp34, {hd, T, 1, HB});
|
| 1125 |
-
Vrc34 = reshapeBatch(V34, {hd, T, HkvB, 1});
|
| 1126 |
-
Vrt34 = tile(Vrc34, {1, 1, 1, HkvMul});
|
| 1127 |
-
Vrp34 = permute(Vrt34, {0, 1, 3, 2});
|
| 1128 |
-
Vt34 = reshapeBatch(Vrp34, {hd, T, 1, HB});
|
| 1129 |
-
A34 = attention(Qr34, Kt34, Vt34, hd, 1);
|
| 1130 |
-
Wo34 = MatrixWithName("gemm_241.weight", D, Dq, 1, 1);
|
| 1131 |
-
O34 = batchedMul(Wo34, reshape(permute(A34, {0, 3, 1, 2}), {Dq, T, 1, 1}));
|
| 1132 |
-
R34 = X34 + O34;
|
| 1133 |
-
Wrf34 = MatrixWithName("rmsn_420.weight", D, 1);
|
| 1134 |
-
Rfn34 = rmsNorm(R34, Wrf34, 1e-5);
|
| 1135 |
-
Wg34 = MatrixWithName("gemm_242.weight", I, D, 1, 1);
|
| 1136 |
-
Wu34 = MatrixWithName("gemm_243.weight", I, D, 1, 1);
|
| 1137 |
-
Wd34 = MatrixWithName("gemm_244.weight", D, I, 1, 1);
|
| 1138 |
-
X35 = R34 + batchedMul(Wd34, elementMul(silu(batchedMul(Wg34, Rfn34)), batchedMul(Wu34, Rfn34)));
|
| 1139 |
-
|
| 1140 |
-
setXY(ids, X35);
|
| 1141 |
-
'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
net_texenc.ini.bak
DELETED
|
@@ -1,27 +0,0 @@
|
|
| 1 |
-
[train]
|
| 2 |
-
load_net = 1
|
| 3 |
-
load_file=z_image_turbo_text_encoder.cccc.bin
|
| 4 |
-
train_epochs = 0
|
| 5 |
-
Batch = 1
|
| 6 |
-
gpu = 1
|
| 7 |
-
data_type = half
|
| 8 |
-
output_log = 0
|
| 9 |
-
need_free_mem = 0
|
| 10 |
-
|
| 11 |
-
[net]
|
| 12 |
-
structure='
|
| 13 |
-
B=1;
|
| 14 |
-
X = Matrix(768, T, 1, B);
|
| 15 |
-
Ws = MatrixWithName("rmsn_1.weight", 768, 1);
|
| 16 |
-
Xn = rmsNorm(X, Ws, 1e-5);
|
| 17 |
-
W = MatrixWithName("gemm_0.weight", 768, 768, 1, 1);
|
| 18 |
-
b = MatrixWithName("gemm_0.bias", 768, 1, 1, 1);
|
| 19 |
-
Y = batchedMul(W, Xn) + b;
|
| 20 |
-
setXA(X, Y);
|
| 21 |
-
'
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
net_unified.ini
DELETED
|
@@ -1,1227 +0,0 @@
|
|
| 1 |
-
[train]
|
| 2 |
-
load_net = 1
|
| 3 |
-
load_file=z_image_turbo_transformer_unified.cccc.bin
|
| 4 |
-
train_epochs = 0
|
| 5 |
-
Batch = 1
|
| 6 |
-
gpu = 1
|
| 7 |
-
data_type = float
|
| 8 |
-
output_log = 0
|
| 9 |
-
need_free_mem = 0
|
| 10 |
-
|
| 11 |
-
[net]
|
| 12 |
-
structure='
|
| 13 |
-
B=1; D=3840; DD=3968; I=10240; hd=128; HB=30;
|
| 14 |
-
Xin = Matrix(DD, SEQ+1, 1, B);
|
| 15 |
-
Xin_pm = permute(Xin, {1, 0, 2, 3});
|
| 16 |
-
t_slot = sliceW(Xin_pm, SEQ, 1);
|
| 17 |
-
t_full = permute(t_slot, {1, 0, 2, 3});
|
| 18 |
-
t_emb = sliceW(t_full, 0, 256);
|
| 19 |
-
seq_pm = sliceW(Xin_pm, 0, SEQ);
|
| 20 |
-
X0_wide = permute(seq_pm, {1, 0, 2, 3});
|
| 21 |
-
X0 = sliceW(X0_wide, 0, D);
|
| 22 |
-
cos1 = sliceW(X0_wide, D, 64);
|
| 23 |
-
sin1 = sliceW(X0_wide, D+64, 64);
|
| 24 |
-
Wmod0 = MatrixWithName("gemm_0.weight", 15360, 256, 1, 1);
|
| 25 |
-
bmod0 = MatrixWithName("gemm_0.bias", 15360, 1, 1, 1);
|
| 26 |
-
mod0 = batchedMul(Wmod0, t_emb) + bmod0;
|
| 27 |
-
sa0 = sliceW(mod0, 0, D);
|
| 28 |
-
ga0 = tanh(sliceW(mod0, D, D));
|
| 29 |
-
sf0 = sliceW(mod0, 2*D, D);
|
| 30 |
-
gf0 = tanh(sliceW(mod0, 3*D, D));
|
| 31 |
-
Wrn330 = MatrixWithName("rmsn_330.weight", D, 1);
|
| 32 |
-
Xn0 = rmsNorm(X0, Wrn330, 1e-5);
|
| 33 |
-
Xm0 = Xn0 + elementMul(Xn0, sa0);
|
| 34 |
-
Wq0 = MatrixWithName("gemm_1.weight", D, D, 1, 1);
|
| 35 |
-
Wk0 = MatrixWithName("gemm_2.weight", D, D, 1, 1);
|
| 36 |
-
Wv0 = MatrixWithName("gemm_3.weight", D, D, 1, 1);
|
| 37 |
-
Wrn331 = MatrixWithName("rmsn_331.weight", hd, 1);
|
| 38 |
-
Wrn332 = MatrixWithName("rmsn_332.weight", hd, 1);
|
| 39 |
-
Wo0 = MatrixWithName("gemm_4.weight", D, D, 1, 1);
|
| 40 |
-
Wrn333 = MatrixWithName("rmsn_333.weight", D, 1);
|
| 41 |
-
Qr0 = permute(reshape(batchedMul(Wq0, Xm0), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 42 |
-
Kr0 = permute(reshape(batchedMul(Wk0, Xm0), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 43 |
-
Vr0 = permute(reshape(batchedMul(Wv0, Xm0), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 44 |
-
Qn0 = rmsNorm(Qr0, Wrn331, 1e-5);
|
| 45 |
-
Kn0 = rmsNorm(Kr0, Wrn332, 1e-5);
|
| 46 |
-
At0 = attention(rope2(Qn0, cos1, sin1), rope2(Kn0, cos1, sin1), Vr0, hd, 0);
|
| 47 |
-
Ao0 = batchedMul(Wo0, reshape(permute(At0, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 48 |
-
An0 = rmsNorm(Ao0, Wrn333, 1e-5);
|
| 49 |
-
Ag0 = elementMul(An0, ga0);
|
| 50 |
-
Rx0 = X0 + Ag0;
|
| 51 |
-
Wrn334 = MatrixWithName("rmsn_334.weight", D, 1);
|
| 52 |
-
Wg0 = MatrixWithName("gemm_5.weight", I, D, 1, 1);
|
| 53 |
-
Wu0 = MatrixWithName("gemm_6.weight", I, D, 1, 1);
|
| 54 |
-
Wd0 = MatrixWithName("gemm_7.weight", D, I, 1, 1);
|
| 55 |
-
Wrn335 = MatrixWithName("rmsn_335.weight", D, 1);
|
| 56 |
-
Rf0 = rmsNorm(Rx0, Wrn334, 1e-5);
|
| 57 |
-
Rfm0 = Rf0 + elementMul(Rf0, sf0);
|
| 58 |
-
Fg0 = batchedMul(Wg0, Rfm0);
|
| 59 |
-
Fu0 = batchedMul(Wu0, Rfm0);
|
| 60 |
-
Ff0 = batchedMul(Wd0, elementMul(silu(Fg0), Fu0));
|
| 61 |
-
Fn0 = rmsNorm(Ff0, Wrn335, 1e-5);
|
| 62 |
-
Fgg0 = elementMul(Fn0, gf0);
|
| 63 |
-
X1 = Rx0 + Fgg0;
|
| 64 |
-
Wmod1 = MatrixWithName("gemm_8.weight", 15360, 256, 1, 1);
|
| 65 |
-
bmod1 = MatrixWithName("gemm_8.bias", 15360, 1, 1, 1);
|
| 66 |
-
mod1 = batchedMul(Wmod1, t_emb) + bmod1;
|
| 67 |
-
sa1 = sliceW(mod1, 0, D);
|
| 68 |
-
ga1 = tanh(sliceW(mod1, D, D));
|
| 69 |
-
sf1 = sliceW(mod1, 2*D, D);
|
| 70 |
-
gf1 = tanh(sliceW(mod1, 3*D, D));
|
| 71 |
-
Wrn336 = MatrixWithName("rmsn_336.weight", D, 1);
|
| 72 |
-
Xn1 = rmsNorm(X1, Wrn336, 1e-5);
|
| 73 |
-
Xm1 = Xn1 + elementMul(Xn1, sa1);
|
| 74 |
-
Wq1 = MatrixWithName("gemm_9.weight", D, D, 1, 1);
|
| 75 |
-
Wk1 = MatrixWithName("gemm_10.weight", D, D, 1, 1);
|
| 76 |
-
Wv1 = MatrixWithName("gemm_11.weight", D, D, 1, 1);
|
| 77 |
-
Wrn337 = MatrixWithName("rmsn_337.weight", hd, 1);
|
| 78 |
-
Wrn338 = MatrixWithName("rmsn_338.weight", hd, 1);
|
| 79 |
-
Wo1 = MatrixWithName("gemm_12.weight", D, D, 1, 1);
|
| 80 |
-
Wrn339 = MatrixWithName("rmsn_339.weight", D, 1);
|
| 81 |
-
Qr1 = permute(reshape(batchedMul(Wq1, Xm1), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 82 |
-
Kr1 = permute(reshape(batchedMul(Wk1, Xm1), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 83 |
-
Vr1 = permute(reshape(batchedMul(Wv1, Xm1), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 84 |
-
Qn1 = rmsNorm(Qr1, Wrn337, 1e-5);
|
| 85 |
-
Kn1 = rmsNorm(Kr1, Wrn338, 1e-5);
|
| 86 |
-
At1 = attention(rope2(Qn1, cos1, sin1), rope2(Kn1, cos1, sin1), Vr1, hd, 0);
|
| 87 |
-
Ao1 = batchedMul(Wo1, reshape(permute(At1, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 88 |
-
An1 = rmsNorm(Ao1, Wrn339, 1e-5);
|
| 89 |
-
Ag1 = elementMul(An1, ga1);
|
| 90 |
-
Rx1 = X1 + Ag1;
|
| 91 |
-
Wrn340 = MatrixWithName("rmsn_340.weight", D, 1);
|
| 92 |
-
Wg1 = MatrixWithName("gemm_13.weight", I, D, 1, 1);
|
| 93 |
-
Wu1 = MatrixWithName("gemm_14.weight", I, D, 1, 1);
|
| 94 |
-
Wd1 = MatrixWithName("gemm_15.weight", D, I, 1, 1);
|
| 95 |
-
Wrn341 = MatrixWithName("rmsn_341.weight", D, 1);
|
| 96 |
-
Rf1 = rmsNorm(Rx1, Wrn340, 1e-5);
|
| 97 |
-
Rfm1 = Rf1 + elementMul(Rf1, sf1);
|
| 98 |
-
Fg1 = batchedMul(Wg1, Rfm1);
|
| 99 |
-
Fu1 = batchedMul(Wu1, Rfm1);
|
| 100 |
-
Ff1 = batchedMul(Wd1, elementMul(silu(Fg1), Fu1));
|
| 101 |
-
Fn1 = rmsNorm(Ff1, Wrn341, 1e-5);
|
| 102 |
-
Fgg1 = elementMul(Fn1, gf1);
|
| 103 |
-
X2 = Rx1 + Fgg1;
|
| 104 |
-
Wmod2 = MatrixWithName("gemm_16.weight", 15360, 256, 1, 1);
|
| 105 |
-
bmod2 = MatrixWithName("gemm_16.bias", 15360, 1, 1, 1);
|
| 106 |
-
mod2 = batchedMul(Wmod2, t_emb) + bmod2;
|
| 107 |
-
sa2 = sliceW(mod2, 0, D);
|
| 108 |
-
ga2 = tanh(sliceW(mod2, D, D));
|
| 109 |
-
sf2 = sliceW(mod2, 2*D, D);
|
| 110 |
-
gf2 = tanh(sliceW(mod2, 3*D, D));
|
| 111 |
-
Wrn342 = MatrixWithName("rmsn_342.weight", D, 1);
|
| 112 |
-
Xn2 = rmsNorm(X2, Wrn342, 1e-5);
|
| 113 |
-
Xm2 = Xn2 + elementMul(Xn2, sa2);
|
| 114 |
-
Wq2 = MatrixWithName("gemm_17.weight", D, D, 1, 1);
|
| 115 |
-
Wk2 = MatrixWithName("gemm_18.weight", D, D, 1, 1);
|
| 116 |
-
Wv2 = MatrixWithName("gemm_19.weight", D, D, 1, 1);
|
| 117 |
-
Wrn343 = MatrixWithName("rmsn_343.weight", hd, 1);
|
| 118 |
-
Wrn344 = MatrixWithName("rmsn_344.weight", hd, 1);
|
| 119 |
-
Wo2 = MatrixWithName("gemm_20.weight", D, D, 1, 1);
|
| 120 |
-
Wrn345 = MatrixWithName("rmsn_345.weight", D, 1);
|
| 121 |
-
Qr2 = permute(reshape(batchedMul(Wq2, Xm2), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 122 |
-
Kr2 = permute(reshape(batchedMul(Wk2, Xm2), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 123 |
-
Vr2 = permute(reshape(batchedMul(Wv2, Xm2), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 124 |
-
Qn2 = rmsNorm(Qr2, Wrn343, 1e-5);
|
| 125 |
-
Kn2 = rmsNorm(Kr2, Wrn344, 1e-5);
|
| 126 |
-
At2 = attention(rope2(Qn2, cos1, sin1), rope2(Kn2, cos1, sin1), Vr2, hd, 0);
|
| 127 |
-
Ao2 = batchedMul(Wo2, reshape(permute(At2, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 128 |
-
An2 = rmsNorm(Ao2, Wrn345, 1e-5);
|
| 129 |
-
Ag2 = elementMul(An2, ga2);
|
| 130 |
-
Rx2 = X2 + Ag2;
|
| 131 |
-
Wrn346 = MatrixWithName("rmsn_346.weight", D, 1);
|
| 132 |
-
Wg2 = MatrixWithName("gemm_21.weight", I, D, 1, 1);
|
| 133 |
-
Wu2 = MatrixWithName("gemm_22.weight", I, D, 1, 1);
|
| 134 |
-
Wd2 = MatrixWithName("gemm_23.weight", D, I, 1, 1);
|
| 135 |
-
Wrn347 = MatrixWithName("rmsn_347.weight", D, 1);
|
| 136 |
-
Rf2 = rmsNorm(Rx2, Wrn346, 1e-5);
|
| 137 |
-
Rfm2 = Rf2 + elementMul(Rf2, sf2);
|
| 138 |
-
Fg2 = batchedMul(Wg2, Rfm2);
|
| 139 |
-
Fu2 = batchedMul(Wu2, Rfm2);
|
| 140 |
-
Ff2 = batchedMul(Wd2, elementMul(silu(Fg2), Fu2));
|
| 141 |
-
Fn2 = rmsNorm(Ff2, Wrn347, 1e-5);
|
| 142 |
-
Fgg2 = elementMul(Fn2, gf2);
|
| 143 |
-
X3 = Rx2 + Fgg2;
|
| 144 |
-
Wmod3 = MatrixWithName("gemm_24.weight", 15360, 256, 1, 1);
|
| 145 |
-
bmod3 = MatrixWithName("gemm_24.bias", 15360, 1, 1, 1);
|
| 146 |
-
mod3 = batchedMul(Wmod3, t_emb) + bmod3;
|
| 147 |
-
sa3 = sliceW(mod3, 0, D);
|
| 148 |
-
ga3 = tanh(sliceW(mod3, D, D));
|
| 149 |
-
sf3 = sliceW(mod3, 2*D, D);
|
| 150 |
-
gf3 = tanh(sliceW(mod3, 3*D, D));
|
| 151 |
-
Wrn348 = MatrixWithName("rmsn_348.weight", D, 1);
|
| 152 |
-
Xn3 = rmsNorm(X3, Wrn348, 1e-5);
|
| 153 |
-
Xm3 = Xn3 + elementMul(Xn3, sa3);
|
| 154 |
-
Wq3 = MatrixWithName("gemm_25.weight", D, D, 1, 1);
|
| 155 |
-
Wk3 = MatrixWithName("gemm_26.weight", D, D, 1, 1);
|
| 156 |
-
Wv3 = MatrixWithName("gemm_27.weight", D, D, 1, 1);
|
| 157 |
-
Wrn349 = MatrixWithName("rmsn_349.weight", hd, 1);
|
| 158 |
-
Wrn350 = MatrixWithName("rmsn_350.weight", hd, 1);
|
| 159 |
-
Wo3 = MatrixWithName("gemm_28.weight", D, D, 1, 1);
|
| 160 |
-
Wrn351 = MatrixWithName("rmsn_351.weight", D, 1);
|
| 161 |
-
Qr3 = permute(reshape(batchedMul(Wq3, Xm3), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 162 |
-
Kr3 = permute(reshape(batchedMul(Wk3, Xm3), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 163 |
-
Vr3 = permute(reshape(batchedMul(Wv3, Xm3), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 164 |
-
Qn3 = rmsNorm(Qr3, Wrn349, 1e-5);
|
| 165 |
-
Kn3 = rmsNorm(Kr3, Wrn350, 1e-5);
|
| 166 |
-
At3 = attention(rope2(Qn3, cos1, sin1), rope2(Kn3, cos1, sin1), Vr3, hd, 0);
|
| 167 |
-
Ao3 = batchedMul(Wo3, reshape(permute(At3, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 168 |
-
An3 = rmsNorm(Ao3, Wrn351, 1e-5);
|
| 169 |
-
Ag3 = elementMul(An3, ga3);
|
| 170 |
-
Rx3 = X3 + Ag3;
|
| 171 |
-
Wrn352 = MatrixWithName("rmsn_352.weight", D, 1);
|
| 172 |
-
Wg3 = MatrixWithName("gemm_29.weight", I, D, 1, 1);
|
| 173 |
-
Wu3 = MatrixWithName("gemm_30.weight", I, D, 1, 1);
|
| 174 |
-
Wd3 = MatrixWithName("gemm_31.weight", D, I, 1, 1);
|
| 175 |
-
Wrn353 = MatrixWithName("rmsn_353.weight", D, 1);
|
| 176 |
-
Rf3 = rmsNorm(Rx3, Wrn352, 1e-5);
|
| 177 |
-
Rfm3 = Rf3 + elementMul(Rf3, sf3);
|
| 178 |
-
Fg3 = batchedMul(Wg3, Rfm3);
|
| 179 |
-
Fu3 = batchedMul(Wu3, Rfm3);
|
| 180 |
-
Ff3 = batchedMul(Wd3, elementMul(silu(Fg3), Fu3));
|
| 181 |
-
Fn3 = rmsNorm(Ff3, Wrn353, 1e-5);
|
| 182 |
-
Fgg3 = elementMul(Fn3, gf3);
|
| 183 |
-
X4 = Rx3 + Fgg3;
|
| 184 |
-
Wmod4 = MatrixWithName("gemm_32.weight", 15360, 256, 1, 1);
|
| 185 |
-
bmod4 = MatrixWithName("gemm_32.bias", 15360, 1, 1, 1);
|
| 186 |
-
mod4 = batchedMul(Wmod4, t_emb) + bmod4;
|
| 187 |
-
sa4 = sliceW(mod4, 0, D);
|
| 188 |
-
ga4 = tanh(sliceW(mod4, D, D));
|
| 189 |
-
sf4 = sliceW(mod4, 2*D, D);
|
| 190 |
-
gf4 = tanh(sliceW(mod4, 3*D, D));
|
| 191 |
-
Wrn354 = MatrixWithName("rmsn_354.weight", D, 1);
|
| 192 |
-
Xn4 = rmsNorm(X4, Wrn354, 1e-5);
|
| 193 |
-
Xm4 = Xn4 + elementMul(Xn4, sa4);
|
| 194 |
-
Wq4 = MatrixWithName("gemm_33.weight", D, D, 1, 1);
|
| 195 |
-
Wk4 = MatrixWithName("gemm_34.weight", D, D, 1, 1);
|
| 196 |
-
Wv4 = MatrixWithName("gemm_35.weight", D, D, 1, 1);
|
| 197 |
-
Wrn355 = MatrixWithName("rmsn_355.weight", hd, 1);
|
| 198 |
-
Wrn356 = MatrixWithName("rmsn_356.weight", hd, 1);
|
| 199 |
-
Wo4 = MatrixWithName("gemm_36.weight", D, D, 1, 1);
|
| 200 |
-
Wrn357 = MatrixWithName("rmsn_357.weight", D, 1);
|
| 201 |
-
Qr4 = permute(reshape(batchedMul(Wq4, Xm4), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 202 |
-
Kr4 = permute(reshape(batchedMul(Wk4, Xm4), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 203 |
-
Vr4 = permute(reshape(batchedMul(Wv4, Xm4), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 204 |
-
Qn4 = rmsNorm(Qr4, Wrn355, 1e-5);
|
| 205 |
-
Kn4 = rmsNorm(Kr4, Wrn356, 1e-5);
|
| 206 |
-
At4 = attention(rope2(Qn4, cos1, sin1), rope2(Kn4, cos1, sin1), Vr4, hd, 0);
|
| 207 |
-
Ao4 = batchedMul(Wo4, reshape(permute(At4, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 208 |
-
An4 = rmsNorm(Ao4, Wrn357, 1e-5);
|
| 209 |
-
Ag4 = elementMul(An4, ga4);
|
| 210 |
-
Rx4 = X4 + Ag4;
|
| 211 |
-
Wrn358 = MatrixWithName("rmsn_358.weight", D, 1);
|
| 212 |
-
Wg4 = MatrixWithName("gemm_37.weight", I, D, 1, 1);
|
| 213 |
-
Wu4 = MatrixWithName("gemm_38.weight", I, D, 1, 1);
|
| 214 |
-
Wd4 = MatrixWithName("gemm_39.weight", D, I, 1, 1);
|
| 215 |
-
Wrn359 = MatrixWithName("rmsn_359.weight", D, 1);
|
| 216 |
-
Rf4 = rmsNorm(Rx4, Wrn358, 1e-5);
|
| 217 |
-
Rfm4 = Rf4 + elementMul(Rf4, sf4);
|
| 218 |
-
Fg4 = batchedMul(Wg4, Rfm4);
|
| 219 |
-
Fu4 = batchedMul(Wu4, Rfm4);
|
| 220 |
-
Ff4 = batchedMul(Wd4, elementMul(silu(Fg4), Fu4));
|
| 221 |
-
Fn4 = rmsNorm(Ff4, Wrn359, 1e-5);
|
| 222 |
-
Fgg4 = elementMul(Fn4, gf4);
|
| 223 |
-
X5 = Rx4 + Fgg4;
|
| 224 |
-
Wmod5 = MatrixWithName("gemm_40.weight", 15360, 256, 1, 1);
|
| 225 |
-
bmod5 = MatrixWithName("gemm_40.bias", 15360, 1, 1, 1);
|
| 226 |
-
mod5 = batchedMul(Wmod5, t_emb) + bmod5;
|
| 227 |
-
sa5 = sliceW(mod5, 0, D);
|
| 228 |
-
ga5 = tanh(sliceW(mod5, D, D));
|
| 229 |
-
sf5 = sliceW(mod5, 2*D, D);
|
| 230 |
-
gf5 = tanh(sliceW(mod5, 3*D, D));
|
| 231 |
-
Wrn360 = MatrixWithName("rmsn_360.weight", D, 1);
|
| 232 |
-
Xn5 = rmsNorm(X5, Wrn360, 1e-5);
|
| 233 |
-
Xm5 = Xn5 + elementMul(Xn5, sa5);
|
| 234 |
-
Wq5 = MatrixWithName("gemm_41.weight", D, D, 1, 1);
|
| 235 |
-
Wk5 = MatrixWithName("gemm_42.weight", D, D, 1, 1);
|
| 236 |
-
Wv5 = MatrixWithName("gemm_43.weight", D, D, 1, 1);
|
| 237 |
-
Wrn361 = MatrixWithName("rmsn_361.weight", hd, 1);
|
| 238 |
-
Wrn362 = MatrixWithName("rmsn_362.weight", hd, 1);
|
| 239 |
-
Wo5 = MatrixWithName("gemm_44.weight", D, D, 1, 1);
|
| 240 |
-
Wrn363 = MatrixWithName("rmsn_363.weight", D, 1);
|
| 241 |
-
Qr5 = permute(reshape(batchedMul(Wq5, Xm5), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 242 |
-
Kr5 = permute(reshape(batchedMul(Wk5, Xm5), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 243 |
-
Vr5 = permute(reshape(batchedMul(Wv5, Xm5), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 244 |
-
Qn5 = rmsNorm(Qr5, Wrn361, 1e-5);
|
| 245 |
-
Kn5 = rmsNorm(Kr5, Wrn362, 1e-5);
|
| 246 |
-
At5 = attention(rope2(Qn5, cos1, sin1), rope2(Kn5, cos1, sin1), Vr5, hd, 0);
|
| 247 |
-
Ao5 = batchedMul(Wo5, reshape(permute(At5, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 248 |
-
An5 = rmsNorm(Ao5, Wrn363, 1e-5);
|
| 249 |
-
Ag5 = elementMul(An5, ga5);
|
| 250 |
-
Rx5 = X5 + Ag5;
|
| 251 |
-
Wrn364 = MatrixWithName("rmsn_364.weight", D, 1);
|
| 252 |
-
Wg5 = MatrixWithName("gemm_45.weight", I, D, 1, 1);
|
| 253 |
-
Wu5 = MatrixWithName("gemm_46.weight", I, D, 1, 1);
|
| 254 |
-
Wd5 = MatrixWithName("gemm_47.weight", D, I, 1, 1);
|
| 255 |
-
Wrn365 = MatrixWithName("rmsn_365.weight", D, 1);
|
| 256 |
-
Rf5 = rmsNorm(Rx5, Wrn364, 1e-5);
|
| 257 |
-
Rfm5 = Rf5 + elementMul(Rf5, sf5);
|
| 258 |
-
Fg5 = batchedMul(Wg5, Rfm5);
|
| 259 |
-
Fu5 = batchedMul(Wu5, Rfm5);
|
| 260 |
-
Ff5 = batchedMul(Wd5, elementMul(silu(Fg5), Fu5));
|
| 261 |
-
Fn5 = rmsNorm(Ff5, Wrn365, 1e-5);
|
| 262 |
-
Fgg5 = elementMul(Fn5, gf5);
|
| 263 |
-
X6 = Rx5 + Fgg5;
|
| 264 |
-
Wmod6 = MatrixWithName("gemm_48.weight", 15360, 256, 1, 1);
|
| 265 |
-
bmod6 = MatrixWithName("gemm_48.bias", 15360, 1, 1, 1);
|
| 266 |
-
mod6 = batchedMul(Wmod6, t_emb) + bmod6;
|
| 267 |
-
sa6 = sliceW(mod6, 0, D);
|
| 268 |
-
ga6 = tanh(sliceW(mod6, D, D));
|
| 269 |
-
sf6 = sliceW(mod6, 2*D, D);
|
| 270 |
-
gf6 = tanh(sliceW(mod6, 3*D, D));
|
| 271 |
-
Wrn366 = MatrixWithName("rmsn_366.weight", D, 1);
|
| 272 |
-
Xn6 = rmsNorm(X6, Wrn366, 1e-5);
|
| 273 |
-
Xm6 = Xn6 + elementMul(Xn6, sa6);
|
| 274 |
-
Wq6 = MatrixWithName("gemm_49.weight", D, D, 1, 1);
|
| 275 |
-
Wk6 = MatrixWithName("gemm_50.weight", D, D, 1, 1);
|
| 276 |
-
Wv6 = MatrixWithName("gemm_51.weight", D, D, 1, 1);
|
| 277 |
-
Wrn367 = MatrixWithName("rmsn_367.weight", hd, 1);
|
| 278 |
-
Wrn368 = MatrixWithName("rmsn_368.weight", hd, 1);
|
| 279 |
-
Wo6 = MatrixWithName("gemm_52.weight", D, D, 1, 1);
|
| 280 |
-
Wrn369 = MatrixWithName("rmsn_369.weight", D, 1);
|
| 281 |
-
Qr6 = permute(reshape(batchedMul(Wq6, Xm6), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 282 |
-
Kr6 = permute(reshape(batchedMul(Wk6, Xm6), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 283 |
-
Vr6 = permute(reshape(batchedMul(Wv6, Xm6), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 284 |
-
Qn6 = rmsNorm(Qr6, Wrn367, 1e-5);
|
| 285 |
-
Kn6 = rmsNorm(Kr6, Wrn368, 1e-5);
|
| 286 |
-
At6 = attention(rope2(Qn6, cos1, sin1), rope2(Kn6, cos1, sin1), Vr6, hd, 0);
|
| 287 |
-
Ao6 = batchedMul(Wo6, reshape(permute(At6, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 288 |
-
An6 = rmsNorm(Ao6, Wrn369, 1e-5);
|
| 289 |
-
Ag6 = elementMul(An6, ga6);
|
| 290 |
-
Rx6 = X6 + Ag6;
|
| 291 |
-
Wrn370 = MatrixWithName("rmsn_370.weight", D, 1);
|
| 292 |
-
Wg6 = MatrixWithName("gemm_53.weight", I, D, 1, 1);
|
| 293 |
-
Wu6 = MatrixWithName("gemm_54.weight", I, D, 1, 1);
|
| 294 |
-
Wd6 = MatrixWithName("gemm_55.weight", D, I, 1, 1);
|
| 295 |
-
Wrn371 = MatrixWithName("rmsn_371.weight", D, 1);
|
| 296 |
-
Rf6 = rmsNorm(Rx6, Wrn370, 1e-5);
|
| 297 |
-
Rfm6 = Rf6 + elementMul(Rf6, sf6);
|
| 298 |
-
Fg6 = batchedMul(Wg6, Rfm6);
|
| 299 |
-
Fu6 = batchedMul(Wu6, Rfm6);
|
| 300 |
-
Ff6 = batchedMul(Wd6, elementMul(silu(Fg6), Fu6));
|
| 301 |
-
Fn6 = rmsNorm(Ff6, Wrn371, 1e-5);
|
| 302 |
-
Fgg6 = elementMul(Fn6, gf6);
|
| 303 |
-
X7 = Rx6 + Fgg6;
|
| 304 |
-
Wmod7 = MatrixWithName("gemm_56.weight", 15360, 256, 1, 1);
|
| 305 |
-
bmod7 = MatrixWithName("gemm_56.bias", 15360, 1, 1, 1);
|
| 306 |
-
mod7 = batchedMul(Wmod7, t_emb) + bmod7;
|
| 307 |
-
sa7 = sliceW(mod7, 0, D);
|
| 308 |
-
ga7 = tanh(sliceW(mod7, D, D));
|
| 309 |
-
sf7 = sliceW(mod7, 2*D, D);
|
| 310 |
-
gf7 = tanh(sliceW(mod7, 3*D, D));
|
| 311 |
-
Wrn372 = MatrixWithName("rmsn_372.weight", D, 1);
|
| 312 |
-
Xn7 = rmsNorm(X7, Wrn372, 1e-5);
|
| 313 |
-
Xm7 = Xn7 + elementMul(Xn7, sa7);
|
| 314 |
-
Wq7 = MatrixWithName("gemm_57.weight", D, D, 1, 1);
|
| 315 |
-
Wk7 = MatrixWithName("gemm_58.weight", D, D, 1, 1);
|
| 316 |
-
Wv7 = MatrixWithName("gemm_59.weight", D, D, 1, 1);
|
| 317 |
-
Wrn373 = MatrixWithName("rmsn_373.weight", hd, 1);
|
| 318 |
-
Wrn374 = MatrixWithName("rmsn_374.weight", hd, 1);
|
| 319 |
-
Wo7 = MatrixWithName("gemm_60.weight", D, D, 1, 1);
|
| 320 |
-
Wrn375 = MatrixWithName("rmsn_375.weight", D, 1);
|
| 321 |
-
Qr7 = permute(reshape(batchedMul(Wq7, Xm7), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 322 |
-
Kr7 = permute(reshape(batchedMul(Wk7, Xm7), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 323 |
-
Vr7 = permute(reshape(batchedMul(Wv7, Xm7), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 324 |
-
Qn7 = rmsNorm(Qr7, Wrn373, 1e-5);
|
| 325 |
-
Kn7 = rmsNorm(Kr7, Wrn374, 1e-5);
|
| 326 |
-
At7 = attention(rope2(Qn7, cos1, sin1), rope2(Kn7, cos1, sin1), Vr7, hd, 0);
|
| 327 |
-
Ao7 = batchedMul(Wo7, reshape(permute(At7, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 328 |
-
An7 = rmsNorm(Ao7, Wrn375, 1e-5);
|
| 329 |
-
Ag7 = elementMul(An7, ga7);
|
| 330 |
-
Rx7 = X7 + Ag7;
|
| 331 |
-
Wrn376 = MatrixWithName("rmsn_376.weight", D, 1);
|
| 332 |
-
Wg7 = MatrixWithName("gemm_61.weight", I, D, 1, 1);
|
| 333 |
-
Wu7 = MatrixWithName("gemm_62.weight", I, D, 1, 1);
|
| 334 |
-
Wd7 = MatrixWithName("gemm_63.weight", D, I, 1, 1);
|
| 335 |
-
Wrn377 = MatrixWithName("rmsn_377.weight", D, 1);
|
| 336 |
-
Rf7 = rmsNorm(Rx7, Wrn376, 1e-5);
|
| 337 |
-
Rfm7 = Rf7 + elementMul(Rf7, sf7);
|
| 338 |
-
Fg7 = batchedMul(Wg7, Rfm7);
|
| 339 |
-
Fu7 = batchedMul(Wu7, Rfm7);
|
| 340 |
-
Ff7 = batchedMul(Wd7, elementMul(silu(Fg7), Fu7));
|
| 341 |
-
Fn7 = rmsNorm(Ff7, Wrn377, 1e-5);
|
| 342 |
-
Fgg7 = elementMul(Fn7, gf7);
|
| 343 |
-
X8 = Rx7 + Fgg7;
|
| 344 |
-
Wmod8 = MatrixWithName("gemm_64.weight", 15360, 256, 1, 1);
|
| 345 |
-
bmod8 = MatrixWithName("gemm_64.bias", 15360, 1, 1, 1);
|
| 346 |
-
mod8 = batchedMul(Wmod8, t_emb) + bmod8;
|
| 347 |
-
sa8 = sliceW(mod8, 0, D);
|
| 348 |
-
ga8 = tanh(sliceW(mod8, D, D));
|
| 349 |
-
sf8 = sliceW(mod8, 2*D, D);
|
| 350 |
-
gf8 = tanh(sliceW(mod8, 3*D, D));
|
| 351 |
-
Wrn378 = MatrixWithName("rmsn_378.weight", D, 1);
|
| 352 |
-
Xn8 = rmsNorm(X8, Wrn378, 1e-5);
|
| 353 |
-
Xm8 = Xn8 + elementMul(Xn8, sa8);
|
| 354 |
-
Wq8 = MatrixWithName("gemm_65.weight", D, D, 1, 1);
|
| 355 |
-
Wk8 = MatrixWithName("gemm_66.weight", D, D, 1, 1);
|
| 356 |
-
Wv8 = MatrixWithName("gemm_67.weight", D, D, 1, 1);
|
| 357 |
-
Wrn379 = MatrixWithName("rmsn_379.weight", hd, 1);
|
| 358 |
-
Wrn380 = MatrixWithName("rmsn_380.weight", hd, 1);
|
| 359 |
-
Wo8 = MatrixWithName("gemm_68.weight", D, D, 1, 1);
|
| 360 |
-
Wrn381 = MatrixWithName("rmsn_381.weight", D, 1);
|
| 361 |
-
Qr8 = permute(reshape(batchedMul(Wq8, Xm8), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 362 |
-
Kr8 = permute(reshape(batchedMul(Wk8, Xm8), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 363 |
-
Vr8 = permute(reshape(batchedMul(Wv8, Xm8), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 364 |
-
Qn8 = rmsNorm(Qr8, Wrn379, 1e-5);
|
| 365 |
-
Kn8 = rmsNorm(Kr8, Wrn380, 1e-5);
|
| 366 |
-
At8 = attention(rope2(Qn8, cos1, sin1), rope2(Kn8, cos1, sin1), Vr8, hd, 0);
|
| 367 |
-
Ao8 = batchedMul(Wo8, reshape(permute(At8, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 368 |
-
An8 = rmsNorm(Ao8, Wrn381, 1e-5);
|
| 369 |
-
Ag8 = elementMul(An8, ga8);
|
| 370 |
-
Rx8 = X8 + Ag8;
|
| 371 |
-
Wrn382 = MatrixWithName("rmsn_382.weight", D, 1);
|
| 372 |
-
Wg8 = MatrixWithName("gemm_69.weight", I, D, 1, 1);
|
| 373 |
-
Wu8 = MatrixWithName("gemm_70.weight", I, D, 1, 1);
|
| 374 |
-
Wd8 = MatrixWithName("gemm_71.weight", D, I, 1, 1);
|
| 375 |
-
Wrn383 = MatrixWithName("rmsn_383.weight", D, 1);
|
| 376 |
-
Rf8 = rmsNorm(Rx8, Wrn382, 1e-5);
|
| 377 |
-
Rfm8 = Rf8 + elementMul(Rf8, sf8);
|
| 378 |
-
Fg8 = batchedMul(Wg8, Rfm8);
|
| 379 |
-
Fu8 = batchedMul(Wu8, Rfm8);
|
| 380 |
-
Ff8 = batchedMul(Wd8, elementMul(silu(Fg8), Fu8));
|
| 381 |
-
Fn8 = rmsNorm(Ff8, Wrn383, 1e-5);
|
| 382 |
-
Fgg8 = elementMul(Fn8, gf8);
|
| 383 |
-
X9 = Rx8 + Fgg8;
|
| 384 |
-
Wmod9 = MatrixWithName("gemm_72.weight", 15360, 256, 1, 1);
|
| 385 |
-
bmod9 = MatrixWithName("gemm_72.bias", 15360, 1, 1, 1);
|
| 386 |
-
mod9 = batchedMul(Wmod9, t_emb) + bmod9;
|
| 387 |
-
sa9 = sliceW(mod9, 0, D);
|
| 388 |
-
ga9 = tanh(sliceW(mod9, D, D));
|
| 389 |
-
sf9 = sliceW(mod9, 2*D, D);
|
| 390 |
-
gf9 = tanh(sliceW(mod9, 3*D, D));
|
| 391 |
-
Wrn384 = MatrixWithName("rmsn_384.weight", D, 1);
|
| 392 |
-
Xn9 = rmsNorm(X9, Wrn384, 1e-5);
|
| 393 |
-
Xm9 = Xn9 + elementMul(Xn9, sa9);
|
| 394 |
-
Wq9 = MatrixWithName("gemm_73.weight", D, D, 1, 1);
|
| 395 |
-
Wk9 = MatrixWithName("gemm_74.weight", D, D, 1, 1);
|
| 396 |
-
Wv9 = MatrixWithName("gemm_75.weight", D, D, 1, 1);
|
| 397 |
-
Wrn385 = MatrixWithName("rmsn_385.weight", hd, 1);
|
| 398 |
-
Wrn386 = MatrixWithName("rmsn_386.weight", hd, 1);
|
| 399 |
-
Wo9 = MatrixWithName("gemm_76.weight", D, D, 1, 1);
|
| 400 |
-
Wrn387 = MatrixWithName("rmsn_387.weight", D, 1);
|
| 401 |
-
Qr9 = permute(reshape(batchedMul(Wq9, Xm9), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 402 |
-
Kr9 = permute(reshape(batchedMul(Wk9, Xm9), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 403 |
-
Vr9 = permute(reshape(batchedMul(Wv9, Xm9), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 404 |
-
Qn9 = rmsNorm(Qr9, Wrn385, 1e-5);
|
| 405 |
-
Kn9 = rmsNorm(Kr9, Wrn386, 1e-5);
|
| 406 |
-
At9 = attention(rope2(Qn9, cos1, sin1), rope2(Kn9, cos1, sin1), Vr9, hd, 0);
|
| 407 |
-
Ao9 = batchedMul(Wo9, reshape(permute(At9, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 408 |
-
An9 = rmsNorm(Ao9, Wrn387, 1e-5);
|
| 409 |
-
Ag9 = elementMul(An9, ga9);
|
| 410 |
-
Rx9 = X9 + Ag9;
|
| 411 |
-
Wrn388 = MatrixWithName("rmsn_388.weight", D, 1);
|
| 412 |
-
Wg9 = MatrixWithName("gemm_77.weight", I, D, 1, 1);
|
| 413 |
-
Wu9 = MatrixWithName("gemm_78.weight", I, D, 1, 1);
|
| 414 |
-
Wd9 = MatrixWithName("gemm_79.weight", D, I, 1, 1);
|
| 415 |
-
Wrn389 = MatrixWithName("rmsn_389.weight", D, 1);
|
| 416 |
-
Rf9 = rmsNorm(Rx9, Wrn388, 1e-5);
|
| 417 |
-
Rfm9 = Rf9 + elementMul(Rf9, sf9);
|
| 418 |
-
Fg9 = batchedMul(Wg9, Rfm9);
|
| 419 |
-
Fu9 = batchedMul(Wu9, Rfm9);
|
| 420 |
-
Ff9 = batchedMul(Wd9, elementMul(silu(Fg9), Fu9));
|
| 421 |
-
Fn9 = rmsNorm(Ff9, Wrn389, 1e-5);
|
| 422 |
-
Fgg9 = elementMul(Fn9, gf9);
|
| 423 |
-
X10 = Rx9 + Fgg9;
|
| 424 |
-
Wmod10 = MatrixWithName("gemm_80.weight", 15360, 256, 1, 1);
|
| 425 |
-
bmod10 = MatrixWithName("gemm_80.bias", 15360, 1, 1, 1);
|
| 426 |
-
mod10 = batchedMul(Wmod10, t_emb) + bmod10;
|
| 427 |
-
sa10 = sliceW(mod10, 0, D);
|
| 428 |
-
ga10 = tanh(sliceW(mod10, D, D));
|
| 429 |
-
sf10 = sliceW(mod10, 2*D, D);
|
| 430 |
-
gf10 = tanh(sliceW(mod10, 3*D, D));
|
| 431 |
-
Wrn390 = MatrixWithName("rmsn_390.weight", D, 1);
|
| 432 |
-
Xn10 = rmsNorm(X10, Wrn390, 1e-5);
|
| 433 |
-
Xm10 = Xn10 + elementMul(Xn10, sa10);
|
| 434 |
-
Wq10 = MatrixWithName("gemm_81.weight", D, D, 1, 1);
|
| 435 |
-
Wk10 = MatrixWithName("gemm_82.weight", D, D, 1, 1);
|
| 436 |
-
Wv10 = MatrixWithName("gemm_83.weight", D, D, 1, 1);
|
| 437 |
-
Wrn391 = MatrixWithName("rmsn_391.weight", hd, 1);
|
| 438 |
-
Wrn392 = MatrixWithName("rmsn_392.weight", hd, 1);
|
| 439 |
-
Wo10 = MatrixWithName("gemm_84.weight", D, D, 1, 1);
|
| 440 |
-
Wrn393 = MatrixWithName("rmsn_393.weight", D, 1);
|
| 441 |
-
Qr10 = permute(reshape(batchedMul(Wq10, Xm10), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 442 |
-
Kr10 = permute(reshape(batchedMul(Wk10, Xm10), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 443 |
-
Vr10 = permute(reshape(batchedMul(Wv10, Xm10), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 444 |
-
Qn10 = rmsNorm(Qr10, Wrn391, 1e-5);
|
| 445 |
-
Kn10 = rmsNorm(Kr10, Wrn392, 1e-5);
|
| 446 |
-
At10 = attention(rope2(Qn10, cos1, sin1), rope2(Kn10, cos1, sin1), Vr10, hd, 0);
|
| 447 |
-
Ao10 = batchedMul(Wo10, reshape(permute(At10, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 448 |
-
An10 = rmsNorm(Ao10, Wrn393, 1e-5);
|
| 449 |
-
Ag10 = elementMul(An10, ga10);
|
| 450 |
-
Rx10 = X10 + Ag10;
|
| 451 |
-
Wrn394 = MatrixWithName("rmsn_394.weight", D, 1);
|
| 452 |
-
Wg10 = MatrixWithName("gemm_85.weight", I, D, 1, 1);
|
| 453 |
-
Wu10 = MatrixWithName("gemm_86.weight", I, D, 1, 1);
|
| 454 |
-
Wd10 = MatrixWithName("gemm_87.weight", D, I, 1, 1);
|
| 455 |
-
Wrn395 = MatrixWithName("rmsn_395.weight", D, 1);
|
| 456 |
-
Rf10 = rmsNorm(Rx10, Wrn394, 1e-5);
|
| 457 |
-
Rfm10 = Rf10 + elementMul(Rf10, sf10);
|
| 458 |
-
Fg10 = batchedMul(Wg10, Rfm10);
|
| 459 |
-
Fu10 = batchedMul(Wu10, Rfm10);
|
| 460 |
-
Ff10 = batchedMul(Wd10, elementMul(silu(Fg10), Fu10));
|
| 461 |
-
Fn10 = rmsNorm(Ff10, Wrn395, 1e-5);
|
| 462 |
-
Fgg10 = elementMul(Fn10, gf10);
|
| 463 |
-
X11 = Rx10 + Fgg10;
|
| 464 |
-
Wmod11 = MatrixWithName("gemm_88.weight", 15360, 256, 1, 1);
|
| 465 |
-
bmod11 = MatrixWithName("gemm_88.bias", 15360, 1, 1, 1);
|
| 466 |
-
mod11 = batchedMul(Wmod11, t_emb) + bmod11;
|
| 467 |
-
sa11 = sliceW(mod11, 0, D);
|
| 468 |
-
ga11 = tanh(sliceW(mod11, D, D));
|
| 469 |
-
sf11 = sliceW(mod11, 2*D, D);
|
| 470 |
-
gf11 = tanh(sliceW(mod11, 3*D, D));
|
| 471 |
-
Wrn396 = MatrixWithName("rmsn_396.weight", D, 1);
|
| 472 |
-
Xn11 = rmsNorm(X11, Wrn396, 1e-5);
|
| 473 |
-
Xm11 = Xn11 + elementMul(Xn11, sa11);
|
| 474 |
-
Wq11 = MatrixWithName("gemm_89.weight", D, D, 1, 1);
|
| 475 |
-
Wk11 = MatrixWithName("gemm_90.weight", D, D, 1, 1);
|
| 476 |
-
Wv11 = MatrixWithName("gemm_91.weight", D, D, 1, 1);
|
| 477 |
-
Wrn397 = MatrixWithName("rmsn_397.weight", hd, 1);
|
| 478 |
-
Wrn398 = MatrixWithName("rmsn_398.weight", hd, 1);
|
| 479 |
-
Wo11 = MatrixWithName("gemm_92.weight", D, D, 1, 1);
|
| 480 |
-
Wrn399 = MatrixWithName("rmsn_399.weight", D, 1);
|
| 481 |
-
Qr11 = permute(reshape(batchedMul(Wq11, Xm11), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 482 |
-
Kr11 = permute(reshape(batchedMul(Wk11, Xm11), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 483 |
-
Vr11 = permute(reshape(batchedMul(Wv11, Xm11), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 484 |
-
Qn11 = rmsNorm(Qr11, Wrn397, 1e-5);
|
| 485 |
-
Kn11 = rmsNorm(Kr11, Wrn398, 1e-5);
|
| 486 |
-
At11 = attention(rope2(Qn11, cos1, sin1), rope2(Kn11, cos1, sin1), Vr11, hd, 0);
|
| 487 |
-
Ao11 = batchedMul(Wo11, reshape(permute(At11, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 488 |
-
An11 = rmsNorm(Ao11, Wrn399, 1e-5);
|
| 489 |
-
Ag11 = elementMul(An11, ga11);
|
| 490 |
-
Rx11 = X11 + Ag11;
|
| 491 |
-
Wrn400 = MatrixWithName("rmsn_400.weight", D, 1);
|
| 492 |
-
Wg11 = MatrixWithName("gemm_93.weight", I, D, 1, 1);
|
| 493 |
-
Wu11 = MatrixWithName("gemm_94.weight", I, D, 1, 1);
|
| 494 |
-
Wd11 = MatrixWithName("gemm_95.weight", D, I, 1, 1);
|
| 495 |
-
Wrn401 = MatrixWithName("rmsn_401.weight", D, 1);
|
| 496 |
-
Rf11 = rmsNorm(Rx11, Wrn400, 1e-5);
|
| 497 |
-
Rfm11 = Rf11 + elementMul(Rf11, sf11);
|
| 498 |
-
Fg11 = batchedMul(Wg11, Rfm11);
|
| 499 |
-
Fu11 = batchedMul(Wu11, Rfm11);
|
| 500 |
-
Ff11 = batchedMul(Wd11, elementMul(silu(Fg11), Fu11));
|
| 501 |
-
Fn11 = rmsNorm(Ff11, Wrn401, 1e-5);
|
| 502 |
-
Fgg11 = elementMul(Fn11, gf11);
|
| 503 |
-
X12 = Rx11 + Fgg11;
|
| 504 |
-
Wmod12 = MatrixWithName("gemm_96.weight", 15360, 256, 1, 1);
|
| 505 |
-
bmod12 = MatrixWithName("gemm_96.bias", 15360, 1, 1, 1);
|
| 506 |
-
mod12 = batchedMul(Wmod12, t_emb) + bmod12;
|
| 507 |
-
sa12 = sliceW(mod12, 0, D);
|
| 508 |
-
ga12 = tanh(sliceW(mod12, D, D));
|
| 509 |
-
sf12 = sliceW(mod12, 2*D, D);
|
| 510 |
-
gf12 = tanh(sliceW(mod12, 3*D, D));
|
| 511 |
-
Wrn402 = MatrixWithName("rmsn_402.weight", D, 1);
|
| 512 |
-
Xn12 = rmsNorm(X12, Wrn402, 1e-5);
|
| 513 |
-
Xm12 = Xn12 + elementMul(Xn12, sa12);
|
| 514 |
-
Wq12 = MatrixWithName("gemm_97.weight", D, D, 1, 1);
|
| 515 |
-
Wk12 = MatrixWithName("gemm_98.weight", D, D, 1, 1);
|
| 516 |
-
Wv12 = MatrixWithName("gemm_99.weight", D, D, 1, 1);
|
| 517 |
-
Wrn403 = MatrixWithName("rmsn_403.weight", hd, 1);
|
| 518 |
-
Wrn404 = MatrixWithName("rmsn_404.weight", hd, 1);
|
| 519 |
-
Wo12 = MatrixWithName("gemm_100.weight", D, D, 1, 1);
|
| 520 |
-
Wrn405 = MatrixWithName("rmsn_405.weight", D, 1);
|
| 521 |
-
Qr12 = permute(reshape(batchedMul(Wq12, Xm12), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 522 |
-
Kr12 = permute(reshape(batchedMul(Wk12, Xm12), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 523 |
-
Vr12 = permute(reshape(batchedMul(Wv12, Xm12), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 524 |
-
Qn12 = rmsNorm(Qr12, Wrn403, 1e-5);
|
| 525 |
-
Kn12 = rmsNorm(Kr12, Wrn404, 1e-5);
|
| 526 |
-
At12 = attention(rope2(Qn12, cos1, sin1), rope2(Kn12, cos1, sin1), Vr12, hd, 0);
|
| 527 |
-
Ao12 = batchedMul(Wo12, reshape(permute(At12, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 528 |
-
An12 = rmsNorm(Ao12, Wrn405, 1e-5);
|
| 529 |
-
Ag12 = elementMul(An12, ga12);
|
| 530 |
-
Rx12 = X12 + Ag12;
|
| 531 |
-
Wrn406 = MatrixWithName("rmsn_406.weight", D, 1);
|
| 532 |
-
Wg12 = MatrixWithName("gemm_101.weight", I, D, 1, 1);
|
| 533 |
-
Wu12 = MatrixWithName("gemm_102.weight", I, D, 1, 1);
|
| 534 |
-
Wd12 = MatrixWithName("gemm_103.weight", D, I, 1, 1);
|
| 535 |
-
Wrn407 = MatrixWithName("rmsn_407.weight", D, 1);
|
| 536 |
-
Rf12 = rmsNorm(Rx12, Wrn406, 1e-5);
|
| 537 |
-
Rfm12 = Rf12 + elementMul(Rf12, sf12);
|
| 538 |
-
Fg12 = batchedMul(Wg12, Rfm12);
|
| 539 |
-
Fu12 = batchedMul(Wu12, Rfm12);
|
| 540 |
-
Ff12 = batchedMul(Wd12, elementMul(silu(Fg12), Fu12));
|
| 541 |
-
Fn12 = rmsNorm(Ff12, Wrn407, 1e-5);
|
| 542 |
-
Fgg12 = elementMul(Fn12, gf12);
|
| 543 |
-
X13 = Rx12 + Fgg12;
|
| 544 |
-
Wmod13 = MatrixWithName("gemm_104.weight", 15360, 256, 1, 1);
|
| 545 |
-
bmod13 = MatrixWithName("gemm_104.bias", 15360, 1, 1, 1);
|
| 546 |
-
mod13 = batchedMul(Wmod13, t_emb) + bmod13;
|
| 547 |
-
sa13 = sliceW(mod13, 0, D);
|
| 548 |
-
ga13 = tanh(sliceW(mod13, D, D));
|
| 549 |
-
sf13 = sliceW(mod13, 2*D, D);
|
| 550 |
-
gf13 = tanh(sliceW(mod13, 3*D, D));
|
| 551 |
-
Wrn408 = MatrixWithName("rmsn_408.weight", D, 1);
|
| 552 |
-
Xn13 = rmsNorm(X13, Wrn408, 1e-5);
|
| 553 |
-
Xm13 = Xn13 + elementMul(Xn13, sa13);
|
| 554 |
-
Wq13 = MatrixWithName("gemm_105.weight", D, D, 1, 1);
|
| 555 |
-
Wk13 = MatrixWithName("gemm_106.weight", D, D, 1, 1);
|
| 556 |
-
Wv13 = MatrixWithName("gemm_107.weight", D, D, 1, 1);
|
| 557 |
-
Wrn409 = MatrixWithName("rmsn_409.weight", hd, 1);
|
| 558 |
-
Wrn410 = MatrixWithName("rmsn_410.weight", hd, 1);
|
| 559 |
-
Wo13 = MatrixWithName("gemm_108.weight", D, D, 1, 1);
|
| 560 |
-
Wrn411 = MatrixWithName("rmsn_411.weight", D, 1);
|
| 561 |
-
Qr13 = permute(reshape(batchedMul(Wq13, Xm13), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 562 |
-
Kr13 = permute(reshape(batchedMul(Wk13, Xm13), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 563 |
-
Vr13 = permute(reshape(batchedMul(Wv13, Xm13), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 564 |
-
Qn13 = rmsNorm(Qr13, Wrn409, 1e-5);
|
| 565 |
-
Kn13 = rmsNorm(Kr13, Wrn410, 1e-5);
|
| 566 |
-
At13 = attention(rope2(Qn13, cos1, sin1), rope2(Kn13, cos1, sin1), Vr13, hd, 0);
|
| 567 |
-
Ao13 = batchedMul(Wo13, reshape(permute(At13, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 568 |
-
An13 = rmsNorm(Ao13, Wrn411, 1e-5);
|
| 569 |
-
Ag13 = elementMul(An13, ga13);
|
| 570 |
-
Rx13 = X13 + Ag13;
|
| 571 |
-
Wrn412 = MatrixWithName("rmsn_412.weight", D, 1);
|
| 572 |
-
Wg13 = MatrixWithName("gemm_109.weight", I, D, 1, 1);
|
| 573 |
-
Wu13 = MatrixWithName("gemm_110.weight", I, D, 1, 1);
|
| 574 |
-
Wd13 = MatrixWithName("gemm_111.weight", D, I, 1, 1);
|
| 575 |
-
Wrn413 = MatrixWithName("rmsn_413.weight", D, 1);
|
| 576 |
-
Rf13 = rmsNorm(Rx13, Wrn412, 1e-5);
|
| 577 |
-
Rfm13 = Rf13 + elementMul(Rf13, sf13);
|
| 578 |
-
Fg13 = batchedMul(Wg13, Rfm13);
|
| 579 |
-
Fu13 = batchedMul(Wu13, Rfm13);
|
| 580 |
-
Ff13 = batchedMul(Wd13, elementMul(silu(Fg13), Fu13));
|
| 581 |
-
Fn13 = rmsNorm(Ff13, Wrn413, 1e-5);
|
| 582 |
-
Fgg13 = elementMul(Fn13, gf13);
|
| 583 |
-
X14 = Rx13 + Fgg13;
|
| 584 |
-
Wmod14 = MatrixWithName("gemm_112.weight", 15360, 256, 1, 1);
|
| 585 |
-
bmod14 = MatrixWithName("gemm_112.bias", 15360, 1, 1, 1);
|
| 586 |
-
mod14 = batchedMul(Wmod14, t_emb) + bmod14;
|
| 587 |
-
sa14 = sliceW(mod14, 0, D);
|
| 588 |
-
ga14 = tanh(sliceW(mod14, D, D));
|
| 589 |
-
sf14 = sliceW(mod14, 2*D, D);
|
| 590 |
-
gf14 = tanh(sliceW(mod14, 3*D, D));
|
| 591 |
-
Wrn414 = MatrixWithName("rmsn_414.weight", D, 1);
|
| 592 |
-
Xn14 = rmsNorm(X14, Wrn414, 1e-5);
|
| 593 |
-
Xm14 = Xn14 + elementMul(Xn14, sa14);
|
| 594 |
-
Wq14 = MatrixWithName("gemm_113.weight", D, D, 1, 1);
|
| 595 |
-
Wk14 = MatrixWithName("gemm_114.weight", D, D, 1, 1);
|
| 596 |
-
Wv14 = MatrixWithName("gemm_115.weight", D, D, 1, 1);
|
| 597 |
-
Wrn415 = MatrixWithName("rmsn_415.weight", hd, 1);
|
| 598 |
-
Wrn416 = MatrixWithName("rmsn_416.weight", hd, 1);
|
| 599 |
-
Wo14 = MatrixWithName("gemm_116.weight", D, D, 1, 1);
|
| 600 |
-
Wrn417 = MatrixWithName("rmsn_417.weight", D, 1);
|
| 601 |
-
Qr14 = permute(reshape(batchedMul(Wq14, Xm14), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 602 |
-
Kr14 = permute(reshape(batchedMul(Wk14, Xm14), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 603 |
-
Vr14 = permute(reshape(batchedMul(Wv14, Xm14), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 604 |
-
Qn14 = rmsNorm(Qr14, Wrn415, 1e-5);
|
| 605 |
-
Kn14 = rmsNorm(Kr14, Wrn416, 1e-5);
|
| 606 |
-
At14 = attention(rope2(Qn14, cos1, sin1), rope2(Kn14, cos1, sin1), Vr14, hd, 0);
|
| 607 |
-
Ao14 = batchedMul(Wo14, reshape(permute(At14, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 608 |
-
An14 = rmsNorm(Ao14, Wrn417, 1e-5);
|
| 609 |
-
Ag14 = elementMul(An14, ga14);
|
| 610 |
-
Rx14 = X14 + Ag14;
|
| 611 |
-
Wrn418 = MatrixWithName("rmsn_418.weight", D, 1);
|
| 612 |
-
Wg14 = MatrixWithName("gemm_117.weight", I, D, 1, 1);
|
| 613 |
-
Wu14 = MatrixWithName("gemm_118.weight", I, D, 1, 1);
|
| 614 |
-
Wd14 = MatrixWithName("gemm_119.weight", D, I, 1, 1);
|
| 615 |
-
Wrn419 = MatrixWithName("rmsn_419.weight", D, 1);
|
| 616 |
-
Rf14 = rmsNorm(Rx14, Wrn418, 1e-5);
|
| 617 |
-
Rfm14 = Rf14 + elementMul(Rf14, sf14);
|
| 618 |
-
Fg14 = batchedMul(Wg14, Rfm14);
|
| 619 |
-
Fu14 = batchedMul(Wu14, Rfm14);
|
| 620 |
-
Ff14 = batchedMul(Wd14, elementMul(silu(Fg14), Fu14));
|
| 621 |
-
Fn14 = rmsNorm(Ff14, Wrn419, 1e-5);
|
| 622 |
-
Fgg14 = elementMul(Fn14, gf14);
|
| 623 |
-
X15 = Rx14 + Fgg14;
|
| 624 |
-
Wmod15 = MatrixWithName("gemm_120.weight", 15360, 256, 1, 1);
|
| 625 |
-
bmod15 = MatrixWithName("gemm_120.bias", 15360, 1, 1, 1);
|
| 626 |
-
mod15 = batchedMul(Wmod15, t_emb) + bmod15;
|
| 627 |
-
sa15 = sliceW(mod15, 0, D);
|
| 628 |
-
ga15 = tanh(sliceW(mod15, D, D));
|
| 629 |
-
sf15 = sliceW(mod15, 2*D, D);
|
| 630 |
-
gf15 = tanh(sliceW(mod15, 3*D, D));
|
| 631 |
-
Wrn420 = MatrixWithName("rmsn_420.weight", D, 1);
|
| 632 |
-
Xn15 = rmsNorm(X15, Wrn420, 1e-5);
|
| 633 |
-
Xm15 = Xn15 + elementMul(Xn15, sa15);
|
| 634 |
-
Wq15 = MatrixWithName("gemm_121.weight", D, D, 1, 1);
|
| 635 |
-
Wk15 = MatrixWithName("gemm_122.weight", D, D, 1, 1);
|
| 636 |
-
Wv15 = MatrixWithName("gemm_123.weight", D, D, 1, 1);
|
| 637 |
-
Wrn421 = MatrixWithName("rmsn_421.weight", hd, 1);
|
| 638 |
-
Wrn422 = MatrixWithName("rmsn_422.weight", hd, 1);
|
| 639 |
-
Wo15 = MatrixWithName("gemm_124.weight", D, D, 1, 1);
|
| 640 |
-
Wrn423 = MatrixWithName("rmsn_423.weight", D, 1);
|
| 641 |
-
Qr15 = permute(reshape(batchedMul(Wq15, Xm15), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 642 |
-
Kr15 = permute(reshape(batchedMul(Wk15, Xm15), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 643 |
-
Vr15 = permute(reshape(batchedMul(Wv15, Xm15), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 644 |
-
Qn15 = rmsNorm(Qr15, Wrn421, 1e-5);
|
| 645 |
-
Kn15 = rmsNorm(Kr15, Wrn422, 1e-5);
|
| 646 |
-
At15 = attention(rope2(Qn15, cos1, sin1), rope2(Kn15, cos1, sin1), Vr15, hd, 0);
|
| 647 |
-
Ao15 = batchedMul(Wo15, reshape(permute(At15, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 648 |
-
An15 = rmsNorm(Ao15, Wrn423, 1e-5);
|
| 649 |
-
Ag15 = elementMul(An15, ga15);
|
| 650 |
-
Rx15 = X15 + Ag15;
|
| 651 |
-
Wrn424 = MatrixWithName("rmsn_424.weight", D, 1);
|
| 652 |
-
Wg15 = MatrixWithName("gemm_125.weight", I, D, 1, 1);
|
| 653 |
-
Wu15 = MatrixWithName("gemm_126.weight", I, D, 1, 1);
|
| 654 |
-
Wd15 = MatrixWithName("gemm_127.weight", D, I, 1, 1);
|
| 655 |
-
Wrn425 = MatrixWithName("rmsn_425.weight", D, 1);
|
| 656 |
-
Rf15 = rmsNorm(Rx15, Wrn424, 1e-5);
|
| 657 |
-
Rfm15 = Rf15 + elementMul(Rf15, sf15);
|
| 658 |
-
Fg15 = batchedMul(Wg15, Rfm15);
|
| 659 |
-
Fu15 = batchedMul(Wu15, Rfm15);
|
| 660 |
-
Ff15 = batchedMul(Wd15, elementMul(silu(Fg15), Fu15));
|
| 661 |
-
Fn15 = rmsNorm(Ff15, Wrn425, 1e-5);
|
| 662 |
-
Fgg15 = elementMul(Fn15, gf15);
|
| 663 |
-
X16 = Rx15 + Fgg15;
|
| 664 |
-
Wmod16 = MatrixWithName("gemm_128.weight", 15360, 256, 1, 1);
|
| 665 |
-
bmod16 = MatrixWithName("gemm_128.bias", 15360, 1, 1, 1);
|
| 666 |
-
mod16 = batchedMul(Wmod16, t_emb) + bmod16;
|
| 667 |
-
sa16 = sliceW(mod16, 0, D);
|
| 668 |
-
ga16 = tanh(sliceW(mod16, D, D));
|
| 669 |
-
sf16 = sliceW(mod16, 2*D, D);
|
| 670 |
-
gf16 = tanh(sliceW(mod16, 3*D, D));
|
| 671 |
-
Wrn426 = MatrixWithName("rmsn_426.weight", D, 1);
|
| 672 |
-
Xn16 = rmsNorm(X16, Wrn426, 1e-5);
|
| 673 |
-
Xm16 = Xn16 + elementMul(Xn16, sa16);
|
| 674 |
-
Wq16 = MatrixWithName("gemm_129.weight", D, D, 1, 1);
|
| 675 |
-
Wk16 = MatrixWithName("gemm_130.weight", D, D, 1, 1);
|
| 676 |
-
Wv16 = MatrixWithName("gemm_131.weight", D, D, 1, 1);
|
| 677 |
-
Wrn427 = MatrixWithName("rmsn_427.weight", hd, 1);
|
| 678 |
-
Wrn428 = MatrixWithName("rmsn_428.weight", hd, 1);
|
| 679 |
-
Wo16 = MatrixWithName("gemm_132.weight", D, D, 1, 1);
|
| 680 |
-
Wrn429 = MatrixWithName("rmsn_429.weight", D, 1);
|
| 681 |
-
Qr16 = permute(reshape(batchedMul(Wq16, Xm16), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 682 |
-
Kr16 = permute(reshape(batchedMul(Wk16, Xm16), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 683 |
-
Vr16 = permute(reshape(batchedMul(Wv16, Xm16), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 684 |
-
Qn16 = rmsNorm(Qr16, Wrn427, 1e-5);
|
| 685 |
-
Kn16 = rmsNorm(Kr16, Wrn428, 1e-5);
|
| 686 |
-
At16 = attention(rope2(Qn16, cos1, sin1), rope2(Kn16, cos1, sin1), Vr16, hd, 0);
|
| 687 |
-
Ao16 = batchedMul(Wo16, reshape(permute(At16, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 688 |
-
An16 = rmsNorm(Ao16, Wrn429, 1e-5);
|
| 689 |
-
Ag16 = elementMul(An16, ga16);
|
| 690 |
-
Rx16 = X16 + Ag16;
|
| 691 |
-
Wrn430 = MatrixWithName("rmsn_430.weight", D, 1);
|
| 692 |
-
Wg16 = MatrixWithName("gemm_133.weight", I, D, 1, 1);
|
| 693 |
-
Wu16 = MatrixWithName("gemm_134.weight", I, D, 1, 1);
|
| 694 |
-
Wd16 = MatrixWithName("gemm_135.weight", D, I, 1, 1);
|
| 695 |
-
Wrn431 = MatrixWithName("rmsn_431.weight", D, 1);
|
| 696 |
-
Rf16 = rmsNorm(Rx16, Wrn430, 1e-5);
|
| 697 |
-
Rfm16 = Rf16 + elementMul(Rf16, sf16);
|
| 698 |
-
Fg16 = batchedMul(Wg16, Rfm16);
|
| 699 |
-
Fu16 = batchedMul(Wu16, Rfm16);
|
| 700 |
-
Ff16 = batchedMul(Wd16, elementMul(silu(Fg16), Fu16));
|
| 701 |
-
Fn16 = rmsNorm(Ff16, Wrn431, 1e-5);
|
| 702 |
-
Fgg16 = elementMul(Fn16, gf16);
|
| 703 |
-
X17 = Rx16 + Fgg16;
|
| 704 |
-
Wmod17 = MatrixWithName("gemm_136.weight", 15360, 256, 1, 1);
|
| 705 |
-
bmod17 = MatrixWithName("gemm_136.bias", 15360, 1, 1, 1);
|
| 706 |
-
mod17 = batchedMul(Wmod17, t_emb) + bmod17;
|
| 707 |
-
sa17 = sliceW(mod17, 0, D);
|
| 708 |
-
ga17 = tanh(sliceW(mod17, D, D));
|
| 709 |
-
sf17 = sliceW(mod17, 2*D, D);
|
| 710 |
-
gf17 = tanh(sliceW(mod17, 3*D, D));
|
| 711 |
-
Wrn432 = MatrixWithName("rmsn_432.weight", D, 1);
|
| 712 |
-
Xn17 = rmsNorm(X17, Wrn432, 1e-5);
|
| 713 |
-
Xm17 = Xn17 + elementMul(Xn17, sa17);
|
| 714 |
-
Wq17 = MatrixWithName("gemm_137.weight", D, D, 1, 1);
|
| 715 |
-
Wk17 = MatrixWithName("gemm_138.weight", D, D, 1, 1);
|
| 716 |
-
Wv17 = MatrixWithName("gemm_139.weight", D, D, 1, 1);
|
| 717 |
-
Wrn433 = MatrixWithName("rmsn_433.weight", hd, 1);
|
| 718 |
-
Wrn434 = MatrixWithName("rmsn_434.weight", hd, 1);
|
| 719 |
-
Wo17 = MatrixWithName("gemm_140.weight", D, D, 1, 1);
|
| 720 |
-
Wrn435 = MatrixWithName("rmsn_435.weight", D, 1);
|
| 721 |
-
Qr17 = permute(reshape(batchedMul(Wq17, Xm17), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 722 |
-
Kr17 = permute(reshape(batchedMul(Wk17, Xm17), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 723 |
-
Vr17 = permute(reshape(batchedMul(Wv17, Xm17), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 724 |
-
Qn17 = rmsNorm(Qr17, Wrn433, 1e-5);
|
| 725 |
-
Kn17 = rmsNorm(Kr17, Wrn434, 1e-5);
|
| 726 |
-
At17 = attention(rope2(Qn17, cos1, sin1), rope2(Kn17, cos1, sin1), Vr17, hd, 0);
|
| 727 |
-
Ao17 = batchedMul(Wo17, reshape(permute(At17, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 728 |
-
An17 = rmsNorm(Ao17, Wrn435, 1e-5);
|
| 729 |
-
Ag17 = elementMul(An17, ga17);
|
| 730 |
-
Rx17 = X17 + Ag17;
|
| 731 |
-
Wrn436 = MatrixWithName("rmsn_436.weight", D, 1);
|
| 732 |
-
Wg17 = MatrixWithName("gemm_141.weight", I, D, 1, 1);
|
| 733 |
-
Wu17 = MatrixWithName("gemm_142.weight", I, D, 1, 1);
|
| 734 |
-
Wd17 = MatrixWithName("gemm_143.weight", D, I, 1, 1);
|
| 735 |
-
Wrn437 = MatrixWithName("rmsn_437.weight", D, 1);
|
| 736 |
-
Rf17 = rmsNorm(Rx17, Wrn436, 1e-5);
|
| 737 |
-
Rfm17 = Rf17 + elementMul(Rf17, sf17);
|
| 738 |
-
Fg17 = batchedMul(Wg17, Rfm17);
|
| 739 |
-
Fu17 = batchedMul(Wu17, Rfm17);
|
| 740 |
-
Ff17 = batchedMul(Wd17, elementMul(silu(Fg17), Fu17));
|
| 741 |
-
Fn17 = rmsNorm(Ff17, Wrn437, 1e-5);
|
| 742 |
-
Fgg17 = elementMul(Fn17, gf17);
|
| 743 |
-
X18 = Rx17 + Fgg17;
|
| 744 |
-
Wmod18 = MatrixWithName("gemm_144.weight", 15360, 256, 1, 1);
|
| 745 |
-
bmod18 = MatrixWithName("gemm_144.bias", 15360, 1, 1, 1);
|
| 746 |
-
mod18 = batchedMul(Wmod18, t_emb) + bmod18;
|
| 747 |
-
sa18 = sliceW(mod18, 0, D);
|
| 748 |
-
ga18 = tanh(sliceW(mod18, D, D));
|
| 749 |
-
sf18 = sliceW(mod18, 2*D, D);
|
| 750 |
-
gf18 = tanh(sliceW(mod18, 3*D, D));
|
| 751 |
-
Wrn438 = MatrixWithName("rmsn_438.weight", D, 1);
|
| 752 |
-
Xn18 = rmsNorm(X18, Wrn438, 1e-5);
|
| 753 |
-
Xm18 = Xn18 + elementMul(Xn18, sa18);
|
| 754 |
-
Wq18 = MatrixWithName("gemm_145.weight", D, D, 1, 1);
|
| 755 |
-
Wk18 = MatrixWithName("gemm_146.weight", D, D, 1, 1);
|
| 756 |
-
Wv18 = MatrixWithName("gemm_147.weight", D, D, 1, 1);
|
| 757 |
-
Wrn439 = MatrixWithName("rmsn_439.weight", hd, 1);
|
| 758 |
-
Wrn440 = MatrixWithName("rmsn_440.weight", hd, 1);
|
| 759 |
-
Wo18 = MatrixWithName("gemm_148.weight", D, D, 1, 1);
|
| 760 |
-
Wrn441 = MatrixWithName("rmsn_441.weight", D, 1);
|
| 761 |
-
Qr18 = permute(reshape(batchedMul(Wq18, Xm18), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 762 |
-
Kr18 = permute(reshape(batchedMul(Wk18, Xm18), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 763 |
-
Vr18 = permute(reshape(batchedMul(Wv18, Xm18), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 764 |
-
Qn18 = rmsNorm(Qr18, Wrn439, 1e-5);
|
| 765 |
-
Kn18 = rmsNorm(Kr18, Wrn440, 1e-5);
|
| 766 |
-
At18 = attention(rope2(Qn18, cos1, sin1), rope2(Kn18, cos1, sin1), Vr18, hd, 0);
|
| 767 |
-
Ao18 = batchedMul(Wo18, reshape(permute(At18, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 768 |
-
An18 = rmsNorm(Ao18, Wrn441, 1e-5);
|
| 769 |
-
Ag18 = elementMul(An18, ga18);
|
| 770 |
-
Rx18 = X18 + Ag18;
|
| 771 |
-
Wrn442 = MatrixWithName("rmsn_442.weight", D, 1);
|
| 772 |
-
Wg18 = MatrixWithName("gemm_149.weight", I, D, 1, 1);
|
| 773 |
-
Wu18 = MatrixWithName("gemm_150.weight", I, D, 1, 1);
|
| 774 |
-
Wd18 = MatrixWithName("gemm_151.weight", D, I, 1, 1);
|
| 775 |
-
Wrn443 = MatrixWithName("rmsn_443.weight", D, 1);
|
| 776 |
-
Rf18 = rmsNorm(Rx18, Wrn442, 1e-5);
|
| 777 |
-
Rfm18 = Rf18 + elementMul(Rf18, sf18);
|
| 778 |
-
Fg18 = batchedMul(Wg18, Rfm18);
|
| 779 |
-
Fu18 = batchedMul(Wu18, Rfm18);
|
| 780 |
-
Ff18 = batchedMul(Wd18, elementMul(silu(Fg18), Fu18));
|
| 781 |
-
Fn18 = rmsNorm(Ff18, Wrn443, 1e-5);
|
| 782 |
-
Fgg18 = elementMul(Fn18, gf18);
|
| 783 |
-
X19 = Rx18 + Fgg18;
|
| 784 |
-
Wmod19 = MatrixWithName("gemm_152.weight", 15360, 256, 1, 1);
|
| 785 |
-
bmod19 = MatrixWithName("gemm_152.bias", 15360, 1, 1, 1);
|
| 786 |
-
mod19 = batchedMul(Wmod19, t_emb) + bmod19;
|
| 787 |
-
sa19 = sliceW(mod19, 0, D);
|
| 788 |
-
ga19 = tanh(sliceW(mod19, D, D));
|
| 789 |
-
sf19 = sliceW(mod19, 2*D, D);
|
| 790 |
-
gf19 = tanh(sliceW(mod19, 3*D, D));
|
| 791 |
-
Wrn444 = MatrixWithName("rmsn_444.weight", D, 1);
|
| 792 |
-
Xn19 = rmsNorm(X19, Wrn444, 1e-5);
|
| 793 |
-
Xm19 = Xn19 + elementMul(Xn19, sa19);
|
| 794 |
-
Wq19 = MatrixWithName("gemm_153.weight", D, D, 1, 1);
|
| 795 |
-
Wk19 = MatrixWithName("gemm_154.weight", D, D, 1, 1);
|
| 796 |
-
Wv19 = MatrixWithName("gemm_155.weight", D, D, 1, 1);
|
| 797 |
-
Wrn445 = MatrixWithName("rmsn_445.weight", hd, 1);
|
| 798 |
-
Wrn446 = MatrixWithName("rmsn_446.weight", hd, 1);
|
| 799 |
-
Wo19 = MatrixWithName("gemm_156.weight", D, D, 1, 1);
|
| 800 |
-
Wrn447 = MatrixWithName("rmsn_447.weight", D, 1);
|
| 801 |
-
Qr19 = permute(reshape(batchedMul(Wq19, Xm19), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 802 |
-
Kr19 = permute(reshape(batchedMul(Wk19, Xm19), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 803 |
-
Vr19 = permute(reshape(batchedMul(Wv19, Xm19), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 804 |
-
Qn19 = rmsNorm(Qr19, Wrn445, 1e-5);
|
| 805 |
-
Kn19 = rmsNorm(Kr19, Wrn446, 1e-5);
|
| 806 |
-
At19 = attention(rope2(Qn19, cos1, sin1), rope2(Kn19, cos1, sin1), Vr19, hd, 0);
|
| 807 |
-
Ao19 = batchedMul(Wo19, reshape(permute(At19, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 808 |
-
An19 = rmsNorm(Ao19, Wrn447, 1e-5);
|
| 809 |
-
Ag19 = elementMul(An19, ga19);
|
| 810 |
-
Rx19 = X19 + Ag19;
|
| 811 |
-
Wrn448 = MatrixWithName("rmsn_448.weight", D, 1);
|
| 812 |
-
Wg19 = MatrixWithName("gemm_157.weight", I, D, 1, 1);
|
| 813 |
-
Wu19 = MatrixWithName("gemm_158.weight", I, D, 1, 1);
|
| 814 |
-
Wd19 = MatrixWithName("gemm_159.weight", D, I, 1, 1);
|
| 815 |
-
Wrn449 = MatrixWithName("rmsn_449.weight", D, 1);
|
| 816 |
-
Rf19 = rmsNorm(Rx19, Wrn448, 1e-5);
|
| 817 |
-
Rfm19 = Rf19 + elementMul(Rf19, sf19);
|
| 818 |
-
Fg19 = batchedMul(Wg19, Rfm19);
|
| 819 |
-
Fu19 = batchedMul(Wu19, Rfm19);
|
| 820 |
-
Ff19 = batchedMul(Wd19, elementMul(silu(Fg19), Fu19));
|
| 821 |
-
Fn19 = rmsNorm(Ff19, Wrn449, 1e-5);
|
| 822 |
-
Fgg19 = elementMul(Fn19, gf19);
|
| 823 |
-
X20 = Rx19 + Fgg19;
|
| 824 |
-
Wmod20 = MatrixWithName("gemm_160.weight", 15360, 256, 1, 1);
|
| 825 |
-
bmod20 = MatrixWithName("gemm_160.bias", 15360, 1, 1, 1);
|
| 826 |
-
mod20 = batchedMul(Wmod20, t_emb) + bmod20;
|
| 827 |
-
sa20 = sliceW(mod20, 0, D);
|
| 828 |
-
ga20 = tanh(sliceW(mod20, D, D));
|
| 829 |
-
sf20 = sliceW(mod20, 2*D, D);
|
| 830 |
-
gf20 = tanh(sliceW(mod20, 3*D, D));
|
| 831 |
-
Wrn450 = MatrixWithName("rmsn_450.weight", D, 1);
|
| 832 |
-
Xn20 = rmsNorm(X20, Wrn450, 1e-5);
|
| 833 |
-
Xm20 = Xn20 + elementMul(Xn20, sa20);
|
| 834 |
-
Wq20 = MatrixWithName("gemm_161.weight", D, D, 1, 1);
|
| 835 |
-
Wk20 = MatrixWithName("gemm_162.weight", D, D, 1, 1);
|
| 836 |
-
Wv20 = MatrixWithName("gemm_163.weight", D, D, 1, 1);
|
| 837 |
-
Wrn451 = MatrixWithName("rmsn_451.weight", hd, 1);
|
| 838 |
-
Wrn452 = MatrixWithName("rmsn_452.weight", hd, 1);
|
| 839 |
-
Wo20 = MatrixWithName("gemm_164.weight", D, D, 1, 1);
|
| 840 |
-
Wrn453 = MatrixWithName("rmsn_453.weight", D, 1);
|
| 841 |
-
Qr20 = permute(reshape(batchedMul(Wq20, Xm20), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 842 |
-
Kr20 = permute(reshape(batchedMul(Wk20, Xm20), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 843 |
-
Vr20 = permute(reshape(batchedMul(Wv20, Xm20), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 844 |
-
Qn20 = rmsNorm(Qr20, Wrn451, 1e-5);
|
| 845 |
-
Kn20 = rmsNorm(Kr20, Wrn452, 1e-5);
|
| 846 |
-
At20 = attention(rope2(Qn20, cos1, sin1), rope2(Kn20, cos1, sin1), Vr20, hd, 0);
|
| 847 |
-
Ao20 = batchedMul(Wo20, reshape(permute(At20, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 848 |
-
An20 = rmsNorm(Ao20, Wrn453, 1e-5);
|
| 849 |
-
Ag20 = elementMul(An20, ga20);
|
| 850 |
-
Rx20 = X20 + Ag20;
|
| 851 |
-
Wrn454 = MatrixWithName("rmsn_454.weight", D, 1);
|
| 852 |
-
Wg20 = MatrixWithName("gemm_165.weight", I, D, 1, 1);
|
| 853 |
-
Wu20 = MatrixWithName("gemm_166.weight", I, D, 1, 1);
|
| 854 |
-
Wd20 = MatrixWithName("gemm_167.weight", D, I, 1, 1);
|
| 855 |
-
Wrn455 = MatrixWithName("rmsn_455.weight", D, 1);
|
| 856 |
-
Rf20 = rmsNorm(Rx20, Wrn454, 1e-5);
|
| 857 |
-
Rfm20 = Rf20 + elementMul(Rf20, sf20);
|
| 858 |
-
Fg20 = batchedMul(Wg20, Rfm20);
|
| 859 |
-
Fu20 = batchedMul(Wu20, Rfm20);
|
| 860 |
-
Ff20 = batchedMul(Wd20, elementMul(silu(Fg20), Fu20));
|
| 861 |
-
Fn20 = rmsNorm(Ff20, Wrn455, 1e-5);
|
| 862 |
-
Fgg20 = elementMul(Fn20, gf20);
|
| 863 |
-
X21 = Rx20 + Fgg20;
|
| 864 |
-
Wmod21 = MatrixWithName("gemm_168.weight", 15360, 256, 1, 1);
|
| 865 |
-
bmod21 = MatrixWithName("gemm_168.bias", 15360, 1, 1, 1);
|
| 866 |
-
mod21 = batchedMul(Wmod21, t_emb) + bmod21;
|
| 867 |
-
sa21 = sliceW(mod21, 0, D);
|
| 868 |
-
ga21 = tanh(sliceW(mod21, D, D));
|
| 869 |
-
sf21 = sliceW(mod21, 2*D, D);
|
| 870 |
-
gf21 = tanh(sliceW(mod21, 3*D, D));
|
| 871 |
-
Wrn456 = MatrixWithName("rmsn_456.weight", D, 1);
|
| 872 |
-
Xn21 = rmsNorm(X21, Wrn456, 1e-5);
|
| 873 |
-
Xm21 = Xn21 + elementMul(Xn21, sa21);
|
| 874 |
-
Wq21 = MatrixWithName("gemm_169.weight", D, D, 1, 1);
|
| 875 |
-
Wk21 = MatrixWithName("gemm_170.weight", D, D, 1, 1);
|
| 876 |
-
Wv21 = MatrixWithName("gemm_171.weight", D, D, 1, 1);
|
| 877 |
-
Wrn457 = MatrixWithName("rmsn_457.weight", hd, 1);
|
| 878 |
-
Wrn458 = MatrixWithName("rmsn_458.weight", hd, 1);
|
| 879 |
-
Wo21 = MatrixWithName("gemm_172.weight", D, D, 1, 1);
|
| 880 |
-
Wrn459 = MatrixWithName("rmsn_459.weight", D, 1);
|
| 881 |
-
Qr21 = permute(reshape(batchedMul(Wq21, Xm21), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 882 |
-
Kr21 = permute(reshape(batchedMul(Wk21, Xm21), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 883 |
-
Vr21 = permute(reshape(batchedMul(Wv21, Xm21), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 884 |
-
Qn21 = rmsNorm(Qr21, Wrn457, 1e-5);
|
| 885 |
-
Kn21 = rmsNorm(Kr21, Wrn458, 1e-5);
|
| 886 |
-
At21 = attention(rope2(Qn21, cos1, sin1), rope2(Kn21, cos1, sin1), Vr21, hd, 0);
|
| 887 |
-
Ao21 = batchedMul(Wo21, reshape(permute(At21, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 888 |
-
An21 = rmsNorm(Ao21, Wrn459, 1e-5);
|
| 889 |
-
Ag21 = elementMul(An21, ga21);
|
| 890 |
-
Rx21 = X21 + Ag21;
|
| 891 |
-
Wrn460 = MatrixWithName("rmsn_460.weight", D, 1);
|
| 892 |
-
Wg21 = MatrixWithName("gemm_173.weight", I, D, 1, 1);
|
| 893 |
-
Wu21 = MatrixWithName("gemm_174.weight", I, D, 1, 1);
|
| 894 |
-
Wd21 = MatrixWithName("gemm_175.weight", D, I, 1, 1);
|
| 895 |
-
Wrn461 = MatrixWithName("rmsn_461.weight", D, 1);
|
| 896 |
-
Rf21 = rmsNorm(Rx21, Wrn460, 1e-5);
|
| 897 |
-
Rfm21 = Rf21 + elementMul(Rf21, sf21);
|
| 898 |
-
Fg21 = batchedMul(Wg21, Rfm21);
|
| 899 |
-
Fu21 = batchedMul(Wu21, Rfm21);
|
| 900 |
-
Ff21 = batchedMul(Wd21, elementMul(silu(Fg21), Fu21));
|
| 901 |
-
Fn21 = rmsNorm(Ff21, Wrn461, 1e-5);
|
| 902 |
-
Fgg21 = elementMul(Fn21, gf21);
|
| 903 |
-
X22 = Rx21 + Fgg21;
|
| 904 |
-
Wmod22 = MatrixWithName("gemm_176.weight", 15360, 256, 1, 1);
|
| 905 |
-
bmod22 = MatrixWithName("gemm_176.bias", 15360, 1, 1, 1);
|
| 906 |
-
mod22 = batchedMul(Wmod22, t_emb) + bmod22;
|
| 907 |
-
sa22 = sliceW(mod22, 0, D);
|
| 908 |
-
ga22 = tanh(sliceW(mod22, D, D));
|
| 909 |
-
sf22 = sliceW(mod22, 2*D, D);
|
| 910 |
-
gf22 = tanh(sliceW(mod22, 3*D, D));
|
| 911 |
-
Wrn462 = MatrixWithName("rmsn_462.weight", D, 1);
|
| 912 |
-
Xn22 = rmsNorm(X22, Wrn462, 1e-5);
|
| 913 |
-
Xm22 = Xn22 + elementMul(Xn22, sa22);
|
| 914 |
-
Wq22 = MatrixWithName("gemm_177.weight", D, D, 1, 1);
|
| 915 |
-
Wk22 = MatrixWithName("gemm_178.weight", D, D, 1, 1);
|
| 916 |
-
Wv22 = MatrixWithName("gemm_179.weight", D, D, 1, 1);
|
| 917 |
-
Wrn463 = MatrixWithName("rmsn_463.weight", hd, 1);
|
| 918 |
-
Wrn464 = MatrixWithName("rmsn_464.weight", hd, 1);
|
| 919 |
-
Wo22 = MatrixWithName("gemm_180.weight", D, D, 1, 1);
|
| 920 |
-
Wrn465 = MatrixWithName("rmsn_465.weight", D, 1);
|
| 921 |
-
Qr22 = permute(reshape(batchedMul(Wq22, Xm22), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 922 |
-
Kr22 = permute(reshape(batchedMul(Wk22, Xm22), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 923 |
-
Vr22 = permute(reshape(batchedMul(Wv22, Xm22), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 924 |
-
Qn22 = rmsNorm(Qr22, Wrn463, 1e-5);
|
| 925 |
-
Kn22 = rmsNorm(Kr22, Wrn464, 1e-5);
|
| 926 |
-
At22 = attention(rope2(Qn22, cos1, sin1), rope2(Kn22, cos1, sin1), Vr22, hd, 0);
|
| 927 |
-
Ao22 = batchedMul(Wo22, reshape(permute(At22, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 928 |
-
An22 = rmsNorm(Ao22, Wrn465, 1e-5);
|
| 929 |
-
Ag22 = elementMul(An22, ga22);
|
| 930 |
-
Rx22 = X22 + Ag22;
|
| 931 |
-
Wrn466 = MatrixWithName("rmsn_466.weight", D, 1);
|
| 932 |
-
Wg22 = MatrixWithName("gemm_181.weight", I, D, 1, 1);
|
| 933 |
-
Wu22 = MatrixWithName("gemm_182.weight", I, D, 1, 1);
|
| 934 |
-
Wd22 = MatrixWithName("gemm_183.weight", D, I, 1, 1);
|
| 935 |
-
Wrn467 = MatrixWithName("rmsn_467.weight", D, 1);
|
| 936 |
-
Rf22 = rmsNorm(Rx22, Wrn466, 1e-5);
|
| 937 |
-
Rfm22 = Rf22 + elementMul(Rf22, sf22);
|
| 938 |
-
Fg22 = batchedMul(Wg22, Rfm22);
|
| 939 |
-
Fu22 = batchedMul(Wu22, Rfm22);
|
| 940 |
-
Ff22 = batchedMul(Wd22, elementMul(silu(Fg22), Fu22));
|
| 941 |
-
Fn22 = rmsNorm(Ff22, Wrn467, 1e-5);
|
| 942 |
-
Fgg22 = elementMul(Fn22, gf22);
|
| 943 |
-
X23 = Rx22 + Fgg22;
|
| 944 |
-
Wmod23 = MatrixWithName("gemm_184.weight", 15360, 256, 1, 1);
|
| 945 |
-
bmod23 = MatrixWithName("gemm_184.bias", 15360, 1, 1, 1);
|
| 946 |
-
mod23 = batchedMul(Wmod23, t_emb) + bmod23;
|
| 947 |
-
sa23 = sliceW(mod23, 0, D);
|
| 948 |
-
ga23 = tanh(sliceW(mod23, D, D));
|
| 949 |
-
sf23 = sliceW(mod23, 2*D, D);
|
| 950 |
-
gf23 = tanh(sliceW(mod23, 3*D, D));
|
| 951 |
-
Wrn468 = MatrixWithName("rmsn_468.weight", D, 1);
|
| 952 |
-
Xn23 = rmsNorm(X23, Wrn468, 1e-5);
|
| 953 |
-
Xm23 = Xn23 + elementMul(Xn23, sa23);
|
| 954 |
-
Wq23 = MatrixWithName("gemm_185.weight", D, D, 1, 1);
|
| 955 |
-
Wk23 = MatrixWithName("gemm_186.weight", D, D, 1, 1);
|
| 956 |
-
Wv23 = MatrixWithName("gemm_187.weight", D, D, 1, 1);
|
| 957 |
-
Wrn469 = MatrixWithName("rmsn_469.weight", hd, 1);
|
| 958 |
-
Wrn470 = MatrixWithName("rmsn_470.weight", hd, 1);
|
| 959 |
-
Wo23 = MatrixWithName("gemm_188.weight", D, D, 1, 1);
|
| 960 |
-
Wrn471 = MatrixWithName("rmsn_471.weight", D, 1);
|
| 961 |
-
Qr23 = permute(reshape(batchedMul(Wq23, Xm23), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 962 |
-
Kr23 = permute(reshape(batchedMul(Wk23, Xm23), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 963 |
-
Vr23 = permute(reshape(batchedMul(Wv23, Xm23), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 964 |
-
Qn23 = rmsNorm(Qr23, Wrn469, 1e-5);
|
| 965 |
-
Kn23 = rmsNorm(Kr23, Wrn470, 1e-5);
|
| 966 |
-
At23 = attention(rope2(Qn23, cos1, sin1), rope2(Kn23, cos1, sin1), Vr23, hd, 0);
|
| 967 |
-
Ao23 = batchedMul(Wo23, reshape(permute(At23, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 968 |
-
An23 = rmsNorm(Ao23, Wrn471, 1e-5);
|
| 969 |
-
Ag23 = elementMul(An23, ga23);
|
| 970 |
-
Rx23 = X23 + Ag23;
|
| 971 |
-
Wrn472 = MatrixWithName("rmsn_472.weight", D, 1);
|
| 972 |
-
Wg23 = MatrixWithName("gemm_189.weight", I, D, 1, 1);
|
| 973 |
-
Wu23 = MatrixWithName("gemm_190.weight", I, D, 1, 1);
|
| 974 |
-
Wd23 = MatrixWithName("gemm_191.weight", D, I, 1, 1);
|
| 975 |
-
Wrn473 = MatrixWithName("rmsn_473.weight", D, 1);
|
| 976 |
-
Rf23 = rmsNorm(Rx23, Wrn472, 1e-5);
|
| 977 |
-
Rfm23 = Rf23 + elementMul(Rf23, sf23);
|
| 978 |
-
Fg23 = batchedMul(Wg23, Rfm23);
|
| 979 |
-
Fu23 = batchedMul(Wu23, Rfm23);
|
| 980 |
-
Ff23 = batchedMul(Wd23, elementMul(silu(Fg23), Fu23));
|
| 981 |
-
Fn23 = rmsNorm(Ff23, Wrn473, 1e-5);
|
| 982 |
-
Fgg23 = elementMul(Fn23, gf23);
|
| 983 |
-
X24 = Rx23 + Fgg23;
|
| 984 |
-
Wmod24 = MatrixWithName("gemm_192.weight", 15360, 256, 1, 1);
|
| 985 |
-
bmod24 = MatrixWithName("gemm_192.bias", 15360, 1, 1, 1);
|
| 986 |
-
mod24 = batchedMul(Wmod24, t_emb) + bmod24;
|
| 987 |
-
sa24 = sliceW(mod24, 0, D);
|
| 988 |
-
ga24 = tanh(sliceW(mod24, D, D));
|
| 989 |
-
sf24 = sliceW(mod24, 2*D, D);
|
| 990 |
-
gf24 = tanh(sliceW(mod24, 3*D, D));
|
| 991 |
-
Wrn474 = MatrixWithName("rmsn_474.weight", D, 1);
|
| 992 |
-
Xn24 = rmsNorm(X24, Wrn474, 1e-5);
|
| 993 |
-
Xm24 = Xn24 + elementMul(Xn24, sa24);
|
| 994 |
-
Wq24 = MatrixWithName("gemm_193.weight", D, D, 1, 1);
|
| 995 |
-
Wk24 = MatrixWithName("gemm_194.weight", D, D, 1, 1);
|
| 996 |
-
Wv24 = MatrixWithName("gemm_195.weight", D, D, 1, 1);
|
| 997 |
-
Wrn475 = MatrixWithName("rmsn_475.weight", hd, 1);
|
| 998 |
-
Wrn476 = MatrixWithName("rmsn_476.weight", hd, 1);
|
| 999 |
-
Wo24 = MatrixWithName("gemm_196.weight", D, D, 1, 1);
|
| 1000 |
-
Wrn477 = MatrixWithName("rmsn_477.weight", D, 1);
|
| 1001 |
-
Qr24 = permute(reshape(batchedMul(Wq24, Xm24), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 1002 |
-
Kr24 = permute(reshape(batchedMul(Wk24, Xm24), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 1003 |
-
Vr24 = permute(reshape(batchedMul(Wv24, Xm24), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 1004 |
-
Qn24 = rmsNorm(Qr24, Wrn475, 1e-5);
|
| 1005 |
-
Kn24 = rmsNorm(Kr24, Wrn476, 1e-5);
|
| 1006 |
-
At24 = attention(rope2(Qn24, cos1, sin1), rope2(Kn24, cos1, sin1), Vr24, hd, 0);
|
| 1007 |
-
Ao24 = batchedMul(Wo24, reshape(permute(At24, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 1008 |
-
An24 = rmsNorm(Ao24, Wrn477, 1e-5);
|
| 1009 |
-
Ag24 = elementMul(An24, ga24);
|
| 1010 |
-
Rx24 = X24 + Ag24;
|
| 1011 |
-
Wrn478 = MatrixWithName("rmsn_478.weight", D, 1);
|
| 1012 |
-
Wg24 = MatrixWithName("gemm_197.weight", I, D, 1, 1);
|
| 1013 |
-
Wu24 = MatrixWithName("gemm_198.weight", I, D, 1, 1);
|
| 1014 |
-
Wd24 = MatrixWithName("gemm_199.weight", D, I, 1, 1);
|
| 1015 |
-
Wrn479 = MatrixWithName("rmsn_479.weight", D, 1);
|
| 1016 |
-
Rf24 = rmsNorm(Rx24, Wrn478, 1e-5);
|
| 1017 |
-
Rfm24 = Rf24 + elementMul(Rf24, sf24);
|
| 1018 |
-
Fg24 = batchedMul(Wg24, Rfm24);
|
| 1019 |
-
Fu24 = batchedMul(Wu24, Rfm24);
|
| 1020 |
-
Ff24 = batchedMul(Wd24, elementMul(silu(Fg24), Fu24));
|
| 1021 |
-
Fn24 = rmsNorm(Ff24, Wrn479, 1e-5);
|
| 1022 |
-
Fgg24 = elementMul(Fn24, gf24);
|
| 1023 |
-
X25 = Rx24 + Fgg24;
|
| 1024 |
-
Wmod25 = MatrixWithName("gemm_200.weight", 15360, 256, 1, 1);
|
| 1025 |
-
bmod25 = MatrixWithName("gemm_200.bias", 15360, 1, 1, 1);
|
| 1026 |
-
mod25 = batchedMul(Wmod25, t_emb) + bmod25;
|
| 1027 |
-
sa25 = sliceW(mod25, 0, D);
|
| 1028 |
-
ga25 = tanh(sliceW(mod25, D, D));
|
| 1029 |
-
sf25 = sliceW(mod25, 2*D, D);
|
| 1030 |
-
gf25 = tanh(sliceW(mod25, 3*D, D));
|
| 1031 |
-
Wrn480 = MatrixWithName("rmsn_480.weight", D, 1);
|
| 1032 |
-
Xn25 = rmsNorm(X25, Wrn480, 1e-5);
|
| 1033 |
-
Xm25 = Xn25 + elementMul(Xn25, sa25);
|
| 1034 |
-
Wq25 = MatrixWithName("gemm_201.weight", D, D, 1, 1);
|
| 1035 |
-
Wk25 = MatrixWithName("gemm_202.weight", D, D, 1, 1);
|
| 1036 |
-
Wv25 = MatrixWithName("gemm_203.weight", D, D, 1, 1);
|
| 1037 |
-
Wrn481 = MatrixWithName("rmsn_481.weight", hd, 1);
|
| 1038 |
-
Wrn482 = MatrixWithName("rmsn_482.weight", hd, 1);
|
| 1039 |
-
Wo25 = MatrixWithName("gemm_204.weight", D, D, 1, 1);
|
| 1040 |
-
Wrn483 = MatrixWithName("rmsn_483.weight", D, 1);
|
| 1041 |
-
Qr25 = permute(reshape(batchedMul(Wq25, Xm25), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 1042 |
-
Kr25 = permute(reshape(batchedMul(Wk25, Xm25), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 1043 |
-
Vr25 = permute(reshape(batchedMul(Wv25, Xm25), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 1044 |
-
Qn25 = rmsNorm(Qr25, Wrn481, 1e-5);
|
| 1045 |
-
Kn25 = rmsNorm(Kr25, Wrn482, 1e-5);
|
| 1046 |
-
At25 = attention(rope2(Qn25, cos1, sin1), rope2(Kn25, cos1, sin1), Vr25, hd, 0);
|
| 1047 |
-
Ao25 = batchedMul(Wo25, reshape(permute(At25, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 1048 |
-
An25 = rmsNorm(Ao25, Wrn483, 1e-5);
|
| 1049 |
-
Ag25 = elementMul(An25, ga25);
|
| 1050 |
-
Rx25 = X25 + Ag25;
|
| 1051 |
-
Wrn484 = MatrixWithName("rmsn_484.weight", D, 1);
|
| 1052 |
-
Wg25 = MatrixWithName("gemm_205.weight", I, D, 1, 1);
|
| 1053 |
-
Wu25 = MatrixWithName("gemm_206.weight", I, D, 1, 1);
|
| 1054 |
-
Wd25 = MatrixWithName("gemm_207.weight", D, I, 1, 1);
|
| 1055 |
-
Wrn485 = MatrixWithName("rmsn_485.weight", D, 1);
|
| 1056 |
-
Rf25 = rmsNorm(Rx25, Wrn484, 1e-5);
|
| 1057 |
-
Rfm25 = Rf25 + elementMul(Rf25, sf25);
|
| 1058 |
-
Fg25 = batchedMul(Wg25, Rfm25);
|
| 1059 |
-
Fu25 = batchedMul(Wu25, Rfm25);
|
| 1060 |
-
Ff25 = batchedMul(Wd25, elementMul(silu(Fg25), Fu25));
|
| 1061 |
-
Fn25 = rmsNorm(Ff25, Wrn485, 1e-5);
|
| 1062 |
-
Fgg25 = elementMul(Fn25, gf25);
|
| 1063 |
-
X26 = Rx25 + Fgg25;
|
| 1064 |
-
Wmod26 = MatrixWithName("gemm_208.weight", 15360, 256, 1, 1);
|
| 1065 |
-
bmod26 = MatrixWithName("gemm_208.bias", 15360, 1, 1, 1);
|
| 1066 |
-
mod26 = batchedMul(Wmod26, t_emb) + bmod26;
|
| 1067 |
-
sa26 = sliceW(mod26, 0, D);
|
| 1068 |
-
ga26 = tanh(sliceW(mod26, D, D));
|
| 1069 |
-
sf26 = sliceW(mod26, 2*D, D);
|
| 1070 |
-
gf26 = tanh(sliceW(mod26, 3*D, D));
|
| 1071 |
-
Wrn486 = MatrixWithName("rmsn_486.weight", D, 1);
|
| 1072 |
-
Xn26 = rmsNorm(X26, Wrn486, 1e-5);
|
| 1073 |
-
Xm26 = Xn26 + elementMul(Xn26, sa26);
|
| 1074 |
-
Wq26 = MatrixWithName("gemm_209.weight", D, D, 1, 1);
|
| 1075 |
-
Wk26 = MatrixWithName("gemm_210.weight", D, D, 1, 1);
|
| 1076 |
-
Wv26 = MatrixWithName("gemm_211.weight", D, D, 1, 1);
|
| 1077 |
-
Wrn487 = MatrixWithName("rmsn_487.weight", hd, 1);
|
| 1078 |
-
Wrn488 = MatrixWithName("rmsn_488.weight", hd, 1);
|
| 1079 |
-
Wo26 = MatrixWithName("gemm_212.weight", D, D, 1, 1);
|
| 1080 |
-
Wrn489 = MatrixWithName("rmsn_489.weight", D, 1);
|
| 1081 |
-
Qr26 = permute(reshape(batchedMul(Wq26, Xm26), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 1082 |
-
Kr26 = permute(reshape(batchedMul(Wk26, Xm26), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 1083 |
-
Vr26 = permute(reshape(batchedMul(Wv26, Xm26), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 1084 |
-
Qn26 = rmsNorm(Qr26, Wrn487, 1e-5);
|
| 1085 |
-
Kn26 = rmsNorm(Kr26, Wrn488, 1e-5);
|
| 1086 |
-
At26 = attention(rope2(Qn26, cos1, sin1), rope2(Kn26, cos1, sin1), Vr26, hd, 0);
|
| 1087 |
-
Ao26 = batchedMul(Wo26, reshape(permute(At26, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 1088 |
-
An26 = rmsNorm(Ao26, Wrn489, 1e-5);
|
| 1089 |
-
Ag26 = elementMul(An26, ga26);
|
| 1090 |
-
Rx26 = X26 + Ag26;
|
| 1091 |
-
Wrn490 = MatrixWithName("rmsn_490.weight", D, 1);
|
| 1092 |
-
Wg26 = MatrixWithName("gemm_213.weight", I, D, 1, 1);
|
| 1093 |
-
Wu26 = MatrixWithName("gemm_214.weight", I, D, 1, 1);
|
| 1094 |
-
Wd26 = MatrixWithName("gemm_215.weight", D, I, 1, 1);
|
| 1095 |
-
Wrn491 = MatrixWithName("rmsn_491.weight", D, 1);
|
| 1096 |
-
Rf26 = rmsNorm(Rx26, Wrn490, 1e-5);
|
| 1097 |
-
Rfm26 = Rf26 + elementMul(Rf26, sf26);
|
| 1098 |
-
Fg26 = batchedMul(Wg26, Rfm26);
|
| 1099 |
-
Fu26 = batchedMul(Wu26, Rfm26);
|
| 1100 |
-
Ff26 = batchedMul(Wd26, elementMul(silu(Fg26), Fu26));
|
| 1101 |
-
Fn26 = rmsNorm(Ff26, Wrn491, 1e-5);
|
| 1102 |
-
Fgg26 = elementMul(Fn26, gf26);
|
| 1103 |
-
X27 = Rx26 + Fgg26;
|
| 1104 |
-
Wmod27 = MatrixWithName("gemm_216.weight", 15360, 256, 1, 1);
|
| 1105 |
-
bmod27 = MatrixWithName("gemm_216.bias", 15360, 1, 1, 1);
|
| 1106 |
-
mod27 = batchedMul(Wmod27, t_emb) + bmod27;
|
| 1107 |
-
sa27 = sliceW(mod27, 0, D);
|
| 1108 |
-
ga27 = tanh(sliceW(mod27, D, D));
|
| 1109 |
-
sf27 = sliceW(mod27, 2*D, D);
|
| 1110 |
-
gf27 = tanh(sliceW(mod27, 3*D, D));
|
| 1111 |
-
Wrn492 = MatrixWithName("rmsn_492.weight", D, 1);
|
| 1112 |
-
Xn27 = rmsNorm(X27, Wrn492, 1e-5);
|
| 1113 |
-
Xm27 = Xn27 + elementMul(Xn27, sa27);
|
| 1114 |
-
Wq27 = MatrixWithName("gemm_217.weight", D, D, 1, 1);
|
| 1115 |
-
Wk27 = MatrixWithName("gemm_218.weight", D, D, 1, 1);
|
| 1116 |
-
Wv27 = MatrixWithName("gemm_219.weight", D, D, 1, 1);
|
| 1117 |
-
Wrn493 = MatrixWithName("rmsn_493.weight", hd, 1);
|
| 1118 |
-
Wrn494 = MatrixWithName("rmsn_494.weight", hd, 1);
|
| 1119 |
-
Wo27 = MatrixWithName("gemm_220.weight", D, D, 1, 1);
|
| 1120 |
-
Wrn495 = MatrixWithName("rmsn_495.weight", D, 1);
|
| 1121 |
-
Qr27 = permute(reshape(batchedMul(Wq27, Xm27), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 1122 |
-
Kr27 = permute(reshape(batchedMul(Wk27, Xm27), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 1123 |
-
Vr27 = permute(reshape(batchedMul(Wv27, Xm27), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 1124 |
-
Qn27 = rmsNorm(Qr27, Wrn493, 1e-5);
|
| 1125 |
-
Kn27 = rmsNorm(Kr27, Wrn494, 1e-5);
|
| 1126 |
-
At27 = attention(rope2(Qn27, cos1, sin1), rope2(Kn27, cos1, sin1), Vr27, hd, 0);
|
| 1127 |
-
Ao27 = batchedMul(Wo27, reshape(permute(At27, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 1128 |
-
An27 = rmsNorm(Ao27, Wrn495, 1e-5);
|
| 1129 |
-
Ag27 = elementMul(An27, ga27);
|
| 1130 |
-
Rx27 = X27 + Ag27;
|
| 1131 |
-
Wrn496 = MatrixWithName("rmsn_496.weight", D, 1);
|
| 1132 |
-
Wg27 = MatrixWithName("gemm_221.weight", I, D, 1, 1);
|
| 1133 |
-
Wu27 = MatrixWithName("gemm_222.weight", I, D, 1, 1);
|
| 1134 |
-
Wd27 = MatrixWithName("gemm_223.weight", D, I, 1, 1);
|
| 1135 |
-
Wrn497 = MatrixWithName("rmsn_497.weight", D, 1);
|
| 1136 |
-
Rf27 = rmsNorm(Rx27, Wrn496, 1e-5);
|
| 1137 |
-
Rfm27 = Rf27 + elementMul(Rf27, sf27);
|
| 1138 |
-
Fg27 = batchedMul(Wg27, Rfm27);
|
| 1139 |
-
Fu27 = batchedMul(Wu27, Rfm27);
|
| 1140 |
-
Ff27 = batchedMul(Wd27, elementMul(silu(Fg27), Fu27));
|
| 1141 |
-
Fn27 = rmsNorm(Ff27, Wrn497, 1e-5);
|
| 1142 |
-
Fgg27 = elementMul(Fn27, gf27);
|
| 1143 |
-
X28 = Rx27 + Fgg27;
|
| 1144 |
-
Wmod28 = MatrixWithName("gemm_224.weight", 15360, 256, 1, 1);
|
| 1145 |
-
bmod28 = MatrixWithName("gemm_224.bias", 15360, 1, 1, 1);
|
| 1146 |
-
mod28 = batchedMul(Wmod28, t_emb) + bmod28;
|
| 1147 |
-
sa28 = sliceW(mod28, 0, D);
|
| 1148 |
-
ga28 = tanh(sliceW(mod28, D, D));
|
| 1149 |
-
sf28 = sliceW(mod28, 2*D, D);
|
| 1150 |
-
gf28 = tanh(sliceW(mod28, 3*D, D));
|
| 1151 |
-
Wrn498 = MatrixWithName("rmsn_498.weight", D, 1);
|
| 1152 |
-
Xn28 = rmsNorm(X28, Wrn498, 1e-5);
|
| 1153 |
-
Xm28 = Xn28 + elementMul(Xn28, sa28);
|
| 1154 |
-
Wq28 = MatrixWithName("gemm_225.weight", D, D, 1, 1);
|
| 1155 |
-
Wk28 = MatrixWithName("gemm_226.weight", D, D, 1, 1);
|
| 1156 |
-
Wv28 = MatrixWithName("gemm_227.weight", D, D, 1, 1);
|
| 1157 |
-
Wrn499 = MatrixWithName("rmsn_499.weight", hd, 1);
|
| 1158 |
-
Wrn500 = MatrixWithName("rmsn_500.weight", hd, 1);
|
| 1159 |
-
Wo28 = MatrixWithName("gemm_228.weight", D, D, 1, 1);
|
| 1160 |
-
Wrn501 = MatrixWithName("rmsn_501.weight", D, 1);
|
| 1161 |
-
Qr28 = permute(reshape(batchedMul(Wq28, Xm28), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 1162 |
-
Kr28 = permute(reshape(batchedMul(Wk28, Xm28), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 1163 |
-
Vr28 = permute(reshape(batchedMul(Wv28, Xm28), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 1164 |
-
Qn28 = rmsNorm(Qr28, Wrn499, 1e-5);
|
| 1165 |
-
Kn28 = rmsNorm(Kr28, Wrn500, 1e-5);
|
| 1166 |
-
At28 = attention(rope2(Qn28, cos1, sin1), rope2(Kn28, cos1, sin1), Vr28, hd, 0);
|
| 1167 |
-
Ao28 = batchedMul(Wo28, reshape(permute(At28, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 1168 |
-
An28 = rmsNorm(Ao28, Wrn501, 1e-5);
|
| 1169 |
-
Ag28 = elementMul(An28, ga28);
|
| 1170 |
-
Rx28 = X28 + Ag28;
|
| 1171 |
-
Wrn502 = MatrixWithName("rmsn_502.weight", D, 1);
|
| 1172 |
-
Wg28 = MatrixWithName("gemm_229.weight", I, D, 1, 1);
|
| 1173 |
-
Wu28 = MatrixWithName("gemm_230.weight", I, D, 1, 1);
|
| 1174 |
-
Wd28 = MatrixWithName("gemm_231.weight", D, I, 1, 1);
|
| 1175 |
-
Wrn503 = MatrixWithName("rmsn_503.weight", D, 1);
|
| 1176 |
-
Rf28 = rmsNorm(Rx28, Wrn502, 1e-5);
|
| 1177 |
-
Rfm28 = Rf28 + elementMul(Rf28, sf28);
|
| 1178 |
-
Fg28 = batchedMul(Wg28, Rfm28);
|
| 1179 |
-
Fu28 = batchedMul(Wu28, Rfm28);
|
| 1180 |
-
Ff28 = batchedMul(Wd28, elementMul(silu(Fg28), Fu28));
|
| 1181 |
-
Fn28 = rmsNorm(Ff28, Wrn503, 1e-5);
|
| 1182 |
-
Fgg28 = elementMul(Fn28, gf28);
|
| 1183 |
-
X29 = Rx28 + Fgg28;
|
| 1184 |
-
Wmod29 = MatrixWithName("gemm_232.weight", 15360, 256, 1, 1);
|
| 1185 |
-
bmod29 = MatrixWithName("gemm_232.bias", 15360, 1, 1, 1);
|
| 1186 |
-
mod29 = batchedMul(Wmod29, t_emb) + bmod29;
|
| 1187 |
-
sa29 = sliceW(mod29, 0, D);
|
| 1188 |
-
ga29 = tanh(sliceW(mod29, D, D));
|
| 1189 |
-
sf29 = sliceW(mod29, 2*D, D);
|
| 1190 |
-
gf29 = tanh(sliceW(mod29, 3*D, D));
|
| 1191 |
-
Wrn504 = MatrixWithName("rmsn_504.weight", D, 1);
|
| 1192 |
-
Xn29 = rmsNorm(X29, Wrn504, 1e-5);
|
| 1193 |
-
Xm29 = Xn29 + elementMul(Xn29, sa29);
|
| 1194 |
-
Wq29 = MatrixWithName("gemm_233.weight", D, D, 1, 1);
|
| 1195 |
-
Wk29 = MatrixWithName("gemm_234.weight", D, D, 1, 1);
|
| 1196 |
-
Wv29 = MatrixWithName("gemm_235.weight", D, D, 1, 1);
|
| 1197 |
-
Wrn505 = MatrixWithName("rmsn_505.weight", hd, 1);
|
| 1198 |
-
Wrn506 = MatrixWithName("rmsn_506.weight", hd, 1);
|
| 1199 |
-
Wo29 = MatrixWithName("gemm_236.weight", D, D, 1, 1);
|
| 1200 |
-
Wrn507 = MatrixWithName("rmsn_507.weight", D, 1);
|
| 1201 |
-
Qr29 = permute(reshape(batchedMul(Wq29, Xm29), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 1202 |
-
Kr29 = permute(reshape(batchedMul(Wk29, Xm29), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 1203 |
-
Vr29 = permute(reshape(batchedMul(Wv29, Xm29), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
|
| 1204 |
-
Qn29 = rmsNorm(Qr29, Wrn505, 1e-5);
|
| 1205 |
-
Kn29 = rmsNorm(Kr29, Wrn506, 1e-5);
|
| 1206 |
-
At29 = attention(rope2(Qn29, cos1, sin1), rope2(Kn29, cos1, sin1), Vr29, hd, 0);
|
| 1207 |
-
Ao29 = batchedMul(Wo29, reshape(permute(At29, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
|
| 1208 |
-
An29 = rmsNorm(Ao29, Wrn507, 1e-5);
|
| 1209 |
-
Ag29 = elementMul(An29, ga29);
|
| 1210 |
-
Rx29 = X29 + Ag29;
|
| 1211 |
-
Wrn508 = MatrixWithName("rmsn_508.weight", D, 1);
|
| 1212 |
-
Wg29 = MatrixWithName("gemm_237.weight", I, D, 1, 1);
|
| 1213 |
-
Wu29 = MatrixWithName("gemm_238.weight", I, D, 1, 1);
|
| 1214 |
-
Wd29 = MatrixWithName("gemm_239.weight", D, I, 1, 1);
|
| 1215 |
-
Wrn509 = MatrixWithName("rmsn_509.weight", D, 1);
|
| 1216 |
-
Rf29 = rmsNorm(Rx29, Wrn508, 1e-5);
|
| 1217 |
-
Rfm29 = Rf29 + elementMul(Rf29, sf29);
|
| 1218 |
-
Fg29 = batchedMul(Wg29, Rfm29);
|
| 1219 |
-
Fu29 = batchedMul(Wu29, Rfm29);
|
| 1220 |
-
Ff29 = batchedMul(Wd29, elementMul(silu(Fg29), Fu29));
|
| 1221 |
-
Fn29 = rmsNorm(Ff29, Wrn509, 1e-5);
|
| 1222 |
-
Fgg29 = elementMul(Fn29, gf29);
|
| 1223 |
-
X30 = Rx29 + Fgg29;
|
| 1224 |
-
|
| 1225 |
-
setXY(Xin, X30);
|
| 1226 |
-
'
|
| 1227 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
net_vae.ini
DELETED
|
@@ -1,219 +0,0 @@
|
|
| 1 |
-
[train]
|
| 2 |
-
load_net = 1
|
| 3 |
-
load_file=z_image_turbo_vae.cccc.bin
|
| 4 |
-
train_epochs = 0
|
| 5 |
-
Batch = 1
|
| 6 |
-
gpu = 1
|
| 7 |
-
data_type = float
|
| 8 |
-
output_log = 0
|
| 9 |
-
need_free_mem = 0
|
| 10 |
-
disable_activation_memory_reuse = 1
|
| 11 |
-
|
| 12 |
-
[net]
|
| 13 |
-
structure='
|
| 14 |
-
B=1;
|
| 15 |
-
Xinp = Matrix(lH, lW, 16, B);
|
| 16 |
-
Wc32=MatrixWithName("conv_32.weight",3,3,16,512); bc32=MatrixWithName("conv_32.bias",1,1,512,1);
|
| 17 |
-
Z0=conv(Xinp,Wc32,{1,1},{1,1})+bc32;
|
| 18 |
-
print_message(Z0);
|
| 19 |
-
Gs67=MatrixWithName("gn_67.weight",512,1);
|
| 20 |
-
Gb67=MatrixWithName("gn_67.bias",512,1);
|
| 21 |
-
Wc33=MatrixWithName("conv_33.weight",3,3,512,512);
|
| 22 |
-
bc33=MatrixWithName("conv_33.bias",1,1,512,1);
|
| 23 |
-
Gs68=MatrixWithName("gn_68.weight",512,1);
|
| 24 |
-
Gb68=MatrixWithName("gn_68.bias",512,1);
|
| 25 |
-
Wc34=MatrixWithName("conv_34.weight",3,3,512,512);
|
| 26 |
-
bc34=MatrixWithName("conv_34.bias",1,1,512,1);
|
| 27 |
-
Zr33a=conv(silu(groupNorm(Z0,32,Gs67,Gb67,1e-6)),Wc33,{1,1},{1,1})+bc33;
|
| 28 |
-
Zr33b=conv(silu(groupNorm(Zr33a,32,Gs68,Gb68,1e-6)),Wc34,{1,1},{1,1})+bc34;
|
| 29 |
-
Z1=Z0+Zr33b;
|
| 30 |
-
print_message(Z1);
|
| 31 |
-
Gs69=MatrixWithName("gn_69.weight",512,1);
|
| 32 |
-
Gb69=MatrixWithName("gn_69.bias",512,1);
|
| 33 |
-
Za_n=groupNorm(Z1,32,Gs69,Gb69,1e-6);
|
| 34 |
-
Za_p=permute(Za_n,{2,0,1,3});
|
| 35 |
-
Za_r=reshapeBatch(Za_p,{512,lH*lW,1,1});
|
| 36 |
-
Wq97=MatrixWithName("attention_97.q_weight",512,512,1,1);
|
| 37 |
-
bq97=MatrixWithName("attention_97.q_bias",512,1,1,1);
|
| 38 |
-
Wk97=MatrixWithName("attention_97.k_weight",512,512,1,1);
|
| 39 |
-
bk97=MatrixWithName("attention_97.k_bias",512,1,1,1);
|
| 40 |
-
Wv97=MatrixWithName("attention_97.v_weight",512,512,1,1);
|
| 41 |
-
bv97=MatrixWithName("attention_97.v_bias",512,1,1,1);
|
| 42 |
-
Wo97=MatrixWithName("attention_97.out_weight",512,512,1,1);
|
| 43 |
-
bo97=MatrixWithName("attention_97.out_bias",512,1,1,1);
|
| 44 |
-
Q97=batchedMul(Wq97,Za_r,1,0)+bq97;
|
| 45 |
-
K97=batchedMul(Wk97,Za_r,1,0)+bk97;
|
| 46 |
-
V97=batchedMul(Wv97,Za_r,1,0)+bv97;
|
| 47 |
-
A97=attention(Q97,K97,V97,512,0);
|
| 48 |
-
Ao97=batchedMul(Wo97,A97,1,0)+bo97;
|
| 49 |
-
Ao97s=reshapeBatch(Ao97,{512,lH,lW,1});
|
| 50 |
-
Ao97b=permute(Ao97s,{1,2,0,3});
|
| 51 |
-
Z1a=Z1+Ao97b;
|
| 52 |
-
print_message(Z1a);
|
| 53 |
-
Gs70=MatrixWithName("gn_70.weight",512,1);
|
| 54 |
-
Gb70=MatrixWithName("gn_70.bias",512,1);
|
| 55 |
-
Wc35=MatrixWithName("conv_35.weight",3,3,512,512);
|
| 56 |
-
bc35=MatrixWithName("conv_35.bias",1,1,512,1);
|
| 57 |
-
Gs71=MatrixWithName("gn_71.weight",512,1);
|
| 58 |
-
Gb71=MatrixWithName("gn_71.bias",512,1);
|
| 59 |
-
Wc36=MatrixWithName("conv_36.weight",3,3,512,512);
|
| 60 |
-
bc36=MatrixWithName("conv_36.bias",1,1,512,1);
|
| 61 |
-
Zr35a=conv(silu(groupNorm(Z1a,32,Gs70,Gb70,1e-6)),Wc35,{1,1},{1,1})+bc35;
|
| 62 |
-
Zr35b=conv(silu(groupNorm(Zr35a,32,Gs71,Gb71,1e-6)),Wc36,{1,1},{1,1})+bc36;
|
| 63 |
-
Z2=Z1a+Zr35b;
|
| 64 |
-
Gs72=MatrixWithName("gn_72.weight",512,1);
|
| 65 |
-
Gb72=MatrixWithName("gn_72.bias",512,1);
|
| 66 |
-
Wc37=MatrixWithName("conv_37.weight",3,3,512,512);
|
| 67 |
-
bc37=MatrixWithName("conv_37.bias",1,1,512,1);
|
| 68 |
-
Gs73=MatrixWithName("gn_73.weight",512,1);
|
| 69 |
-
Gb73=MatrixWithName("gn_73.bias",512,1);
|
| 70 |
-
Wc38=MatrixWithName("conv_38.weight",3,3,512,512);
|
| 71 |
-
bc38=MatrixWithName("conv_38.bias",1,1,512,1);
|
| 72 |
-
Zr37a=conv(silu(groupNorm(Z2,32,Gs72,Gb72,1e-6)),Wc37,{1,1},{1,1})+bc37;
|
| 73 |
-
Zr37b=conv(silu(groupNorm(Zr37a,32,Gs73,Gb73,1e-6)),Wc38,{1,1},{1,1})+bc38;
|
| 74 |
-
Z3=Z2+Zr37b;
|
| 75 |
-
Gs74=MatrixWithName("gn_74.weight",512,1);
|
| 76 |
-
Gb74=MatrixWithName("gn_74.bias",512,1);
|
| 77 |
-
Wc39=MatrixWithName("conv_39.weight",3,3,512,512);
|
| 78 |
-
bc39=MatrixWithName("conv_39.bias",1,1,512,1);
|
| 79 |
-
Gs75=MatrixWithName("gn_75.weight",512,1);
|
| 80 |
-
Gb75=MatrixWithName("gn_75.bias",512,1);
|
| 81 |
-
Wc40=MatrixWithName("conv_40.weight",3,3,512,512);
|
| 82 |
-
bc40=MatrixWithName("conv_40.bias",1,1,512,1);
|
| 83 |
-
Zr39a=conv(silu(groupNorm(Z3,32,Gs74,Gb74,1e-6)),Wc39,{1,1},{1,1})+bc39;
|
| 84 |
-
Zr39b=conv(silu(groupNorm(Zr39a,32,Gs75,Gb75,1e-6)),Wc40,{1,1},{1,1})+bc40;
|
| 85 |
-
Z4=Z3+Zr39b;
|
| 86 |
-
Gs76=MatrixWithName("gn_76.weight",512,1);
|
| 87 |
-
Gb76=MatrixWithName("gn_76.bias",512,1);
|
| 88 |
-
Wc41=MatrixWithName("conv_41.weight",3,3,512,512);
|
| 89 |
-
bc41=MatrixWithName("conv_41.bias",1,1,512,1);
|
| 90 |
-
Gs77=MatrixWithName("gn_77.weight",512,1);
|
| 91 |
-
Gb77=MatrixWithName("gn_77.bias",512,1);
|
| 92 |
-
Wc42=MatrixWithName("conv_42.weight",3,3,512,512);
|
| 93 |
-
bc42=MatrixWithName("conv_42.bias",1,1,512,1);
|
| 94 |
-
Zr41a=conv(silu(groupNorm(Z4,32,Gs76,Gb76,1e-6)),Wc41,{1,1},{1,1})+bc41;
|
| 95 |
-
Zr41b=conv(silu(groupNorm(Zr41a,32,Gs77,Gb77,1e-6)),Wc42,{1,1},{1,1})+bc42;
|
| 96 |
-
Z5=Z4+Zr41b;
|
| 97 |
-
Z5u=upsample(Z5,2,2,0);
|
| 98 |
-
Wc43=MatrixWithName("conv_43.weight",3,3,512,512); bc43=MatrixWithName("conv_43.bias",1,1,512,1);
|
| 99 |
-
Z6=conv(Z5u,Wc43,{1,1},{1,1})+bc43;
|
| 100 |
-
Gs78=MatrixWithName("gn_78.weight",512,1);
|
| 101 |
-
Gb78=MatrixWithName("gn_78.bias",512,1);
|
| 102 |
-
Wc44=MatrixWithName("conv_44.weight",3,3,512,512);
|
| 103 |
-
bc44=MatrixWithName("conv_44.bias",1,1,512,1);
|
| 104 |
-
Gs79=MatrixWithName("gn_79.weight",512,1);
|
| 105 |
-
Gb79=MatrixWithName("gn_79.bias",512,1);
|
| 106 |
-
Wc45=MatrixWithName("conv_45.weight",3,3,512,512);
|
| 107 |
-
bc45=MatrixWithName("conv_45.bias",1,1,512,1);
|
| 108 |
-
Zr44a=conv(silu(groupNorm(Z6,32,Gs78,Gb78,1e-6)),Wc44,{1,1},{1,1})+bc44;
|
| 109 |
-
Zr44b=conv(silu(groupNorm(Zr44a,32,Gs79,Gb79,1e-6)),Wc45,{1,1},{1,1})+bc45;
|
| 110 |
-
Z7=Z6+Zr44b;
|
| 111 |
-
Gs80=MatrixWithName("gn_80.weight",512,1);
|
| 112 |
-
Gb80=MatrixWithName("gn_80.bias",512,1);
|
| 113 |
-
Wc46=MatrixWithName("conv_46.weight",3,3,512,512);
|
| 114 |
-
bc46=MatrixWithName("conv_46.bias",1,1,512,1);
|
| 115 |
-
Gs81=MatrixWithName("gn_81.weight",512,1);
|
| 116 |
-
Gb81=MatrixWithName("gn_81.bias",512,1);
|
| 117 |
-
Wc47=MatrixWithName("conv_47.weight",3,3,512,512);
|
| 118 |
-
bc47=MatrixWithName("conv_47.bias",1,1,512,1);
|
| 119 |
-
Zr46a=conv(silu(groupNorm(Z7,32,Gs80,Gb80,1e-6)),Wc46,{1,1},{1,1})+bc46;
|
| 120 |
-
Zr46b=conv(silu(groupNorm(Zr46a,32,Gs81,Gb81,1e-6)),Wc47,{1,1},{1,1})+bc47;
|
| 121 |
-
Z8=Z7+Zr46b;
|
| 122 |
-
Gs82=MatrixWithName("gn_82.weight",512,1);
|
| 123 |
-
Gb82=MatrixWithName("gn_82.bias",512,1);
|
| 124 |
-
Wc48=MatrixWithName("conv_48.weight",3,3,512,512);
|
| 125 |
-
bc48=MatrixWithName("conv_48.bias",1,1,512,1);
|
| 126 |
-
Gs83=MatrixWithName("gn_83.weight",512,1);
|
| 127 |
-
Gb83=MatrixWithName("gn_83.bias",512,1);
|
| 128 |
-
Wc49=MatrixWithName("conv_49.weight",3,3,512,512);
|
| 129 |
-
bc49=MatrixWithName("conv_49.bias",1,1,512,1);
|
| 130 |
-
Zr48a=conv(silu(groupNorm(Z8,32,Gs82,Gb82,1e-6)),Wc48,{1,1},{1,1})+bc48;
|
| 131 |
-
Zr48b=conv(silu(groupNorm(Zr48a,32,Gs83,Gb83,1e-6)),Wc49,{1,1},{1,1})+bc49;
|
| 132 |
-
Z9=Z8+Zr48b;
|
| 133 |
-
Z9u=upsample(Z9,2,2,0);
|
| 134 |
-
Wc50=MatrixWithName("conv_50.weight",3,3,512,512); bc50=MatrixWithName("conv_50.bias",1,1,512,1);
|
| 135 |
-
Z10=conv(Z9u,Wc50,{1,1},{1,1})+bc50;
|
| 136 |
-
Gs84=MatrixWithName("gn_84.weight",512,1);
|
| 137 |
-
Gb84=MatrixWithName("gn_84.bias",512,1);
|
| 138 |
-
Wc51=MatrixWithName("conv_51.weight",3,3,512,256);
|
| 139 |
-
bc51=MatrixWithName("conv_51.bias",1,1,256,1);
|
| 140 |
-
Gs85=MatrixWithName("gn_85.weight",256,1);
|
| 141 |
-
Gb85=MatrixWithName("gn_85.bias",256,1);
|
| 142 |
-
Wc52=MatrixWithName("conv_52.weight",3,3,256,256);
|
| 143 |
-
bc52=MatrixWithName("conv_52.bias",1,1,256,1);
|
| 144 |
-
Wc53=MatrixWithName("conv_53.weight",1,1,512,256);
|
| 145 |
-
bc53=MatrixWithName("conv_53.bias",1,1,256,1);
|
| 146 |
-
Zr51a=conv(silu(groupNorm(Z10,32,Gs84,Gb84,1e-6)),Wc51,{1,1},{1,1})+bc51;
|
| 147 |
-
Zr51b=conv(silu(groupNorm(Zr51a,32,Gs85,Gb85,1e-6)),Wc52,{1,1},{1,1})+bc52;
|
| 148 |
-
Zsk53=conv(Z10,Wc53,{1,1},{0,0})+bc53;
|
| 149 |
-
Z11=Zsk53+Zr51b;
|
| 150 |
-
Gs86=MatrixWithName("gn_86.weight",256,1);
|
| 151 |
-
Gb86=MatrixWithName("gn_86.bias",256,1);
|
| 152 |
-
Wc54=MatrixWithName("conv_54.weight",3,3,256,256);
|
| 153 |
-
bc54=MatrixWithName("conv_54.bias",1,1,256,1);
|
| 154 |
-
Gs87=MatrixWithName("gn_87.weight",256,1);
|
| 155 |
-
Gb87=MatrixWithName("gn_87.bias",256,1);
|
| 156 |
-
Wc55=MatrixWithName("conv_55.weight",3,3,256,256);
|
| 157 |
-
bc55=MatrixWithName("conv_55.bias",1,1,256,1);
|
| 158 |
-
Zr54a=conv(silu(groupNorm(Z11,32,Gs86,Gb86,1e-6)),Wc54,{1,1},{1,1})+bc54;
|
| 159 |
-
Zr54b=conv(silu(groupNorm(Zr54a,32,Gs87,Gb87,1e-6)),Wc55,{1,1},{1,1})+bc55;
|
| 160 |
-
Z12=Z11+Zr54b;
|
| 161 |
-
Gs88=MatrixWithName("gn_88.weight",256,1);
|
| 162 |
-
Gb88=MatrixWithName("gn_88.bias",256,1);
|
| 163 |
-
Wc56=MatrixWithName("conv_56.weight",3,3,256,256);
|
| 164 |
-
bc56=MatrixWithName("conv_56.bias",1,1,256,1);
|
| 165 |
-
Gs89=MatrixWithName("gn_89.weight",256,1);
|
| 166 |
-
Gb89=MatrixWithName("gn_89.bias",256,1);
|
| 167 |
-
Wc57=MatrixWithName("conv_57.weight",3,3,256,256);
|
| 168 |
-
bc57=MatrixWithName("conv_57.bias",1,1,256,1);
|
| 169 |
-
Zr56a=conv(silu(groupNorm(Z12,32,Gs88,Gb88,1e-6)),Wc56,{1,1},{1,1})+bc56;
|
| 170 |
-
Zr56b=conv(silu(groupNorm(Zr56a,32,Gs89,Gb89,1e-6)),Wc57,{1,1},{1,1})+bc57;
|
| 171 |
-
Z13=Z12+Zr56b;
|
| 172 |
-
Z13u=upsample(Z13,2,2,0);
|
| 173 |
-
Wc58=MatrixWithName("conv_58.weight",3,3,256,256); bc58=MatrixWithName("conv_58.bias",1,1,256,1);
|
| 174 |
-
Z14=conv(Z13u,Wc58,{1,1},{1,1})+bc58;
|
| 175 |
-
Gs90=MatrixWithName("gn_90.weight",256,1);
|
| 176 |
-
Gb90=MatrixWithName("gn_90.bias",256,1);
|
| 177 |
-
Wc59=MatrixWithName("conv_59.weight",3,3,256,128);
|
| 178 |
-
bc59=MatrixWithName("conv_59.bias",1,1,128,1);
|
| 179 |
-
Gs91=MatrixWithName("gn_91.weight",128,1);
|
| 180 |
-
Gb91=MatrixWithName("gn_91.bias",128,1);
|
| 181 |
-
Wc60=MatrixWithName("conv_60.weight",3,3,128,128);
|
| 182 |
-
bc60=MatrixWithName("conv_60.bias",1,1,128,1);
|
| 183 |
-
Wc61=MatrixWithName("conv_61.weight",1,1,256,128);
|
| 184 |
-
bc61=MatrixWithName("conv_61.bias",1,1,128,1);
|
| 185 |
-
Zr59a=conv(silu(groupNorm(Z14,32,Gs90,Gb90,1e-6)),Wc59,{1,1},{1,1})+bc59;
|
| 186 |
-
Zr59b=conv(silu(groupNorm(Zr59a,32,Gs91,Gb91,1e-6)),Wc60,{1,1},{1,1})+bc60;
|
| 187 |
-
Zsk61=conv(Z14,Wc61,{1,1},{0,0})+bc61;
|
| 188 |
-
Z15=Zsk61+Zr59b;
|
| 189 |
-
Gs92=MatrixWithName("gn_92.weight",128,1);
|
| 190 |
-
Gb92=MatrixWithName("gn_92.bias",128,1);
|
| 191 |
-
Wc62=MatrixWithName("conv_62.weight",3,3,128,128);
|
| 192 |
-
bc62=MatrixWithName("conv_62.bias",1,1,128,1);
|
| 193 |
-
Gs93=MatrixWithName("gn_93.weight",128,1);
|
| 194 |
-
Gb93=MatrixWithName("gn_93.bias",128,1);
|
| 195 |
-
Wc63=MatrixWithName("conv_63.weight",3,3,128,128);
|
| 196 |
-
bc63=MatrixWithName("conv_63.bias",1,1,128,1);
|
| 197 |
-
Zr62a=conv(silu(groupNorm(Z15,32,Gs92,Gb92,1e-6)),Wc62,{1,1},{1,1})+bc62;
|
| 198 |
-
Zr62b=conv(silu(groupNorm(Zr62a,32,Gs93,Gb93,1e-6)),Wc63,{1,1},{1,1})+bc63;
|
| 199 |
-
Z16=Z15+Zr62b;
|
| 200 |
-
Gs94=MatrixWithName("gn_94.weight",128,1);
|
| 201 |
-
Gb94=MatrixWithName("gn_94.bias",128,1);
|
| 202 |
-
Wc64=MatrixWithName("conv_64.weight",3,3,128,128);
|
| 203 |
-
bc64=MatrixWithName("conv_64.bias",1,1,128,1);
|
| 204 |
-
Gs95=MatrixWithName("gn_95.weight",128,1);
|
| 205 |
-
Gb95=MatrixWithName("gn_95.bias",128,1);
|
| 206 |
-
Wc65=MatrixWithName("conv_65.weight",3,3,128,128);
|
| 207 |
-
bc65=MatrixWithName("conv_65.bias",1,1,128,1);
|
| 208 |
-
Zr64a=conv(silu(groupNorm(Z16,32,Gs94,Gb94,1e-6)),Wc64,{1,1},{1,1})+bc64;
|
| 209 |
-
Zr64b=conv(silu(groupNorm(Zr64a,32,Gs95,Gb95,1e-6)),Wc65,{1,1},{1,1})+bc65;
|
| 210 |
-
Z17=Z16+Zr64b;
|
| 211 |
-
Gs96=MatrixWithName("gn_96.weight",128,1);
|
| 212 |
-
Gb96=MatrixWithName("gn_96.bias",128,1);
|
| 213 |
-
Wc66=MatrixWithName("conv_66.weight",3,3,128,3);
|
| 214 |
-
bc66=MatrixWithName("conv_66.bias",1,1,3,1);
|
| 215 |
-
Y=conv(silu(groupNorm(Z17,32,Gs96,Gb96,1e-6)),Wc66,{1,1},{1,1})+bc66;
|
| 216 |
-
print_message(Y);
|
| 217 |
-
setXY(Xinp, Y);
|
| 218 |
-
'
|
| 219 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
net_xemb.ini
DELETED
|
@@ -1,20 +0,0 @@
|
|
| 1 |
-
[train]
|
| 2 |
-
load_net = 1
|
| 3 |
-
load_file=z_image_turbo_transformer_all_x_embedder.cccc.bin
|
| 4 |
-
train_epochs = 0
|
| 5 |
-
Batch = 1
|
| 6 |
-
gpu = 1
|
| 7 |
-
data_type = half
|
| 8 |
-
output_log = 0
|
| 9 |
-
need_free_mem = 0
|
| 10 |
-
|
| 11 |
-
[net]
|
| 12 |
-
structure='
|
| 13 |
-
B=1;
|
| 14 |
-
W = MatrixWithName("gemm_0.weight", 3840, 64, 1, 1);
|
| 15 |
-
b = MatrixWithName("gemm_0.bias", 3840, 1, 1, 1);
|
| 16 |
-
X = Matrix(64, P, 1, B);
|
| 17 |
-
Y = batchedMul(W, X) + b;
|
| 18 |
-
setXY(X, Y);
|
| 19 |
-
'
|
| 20 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
output.png
DELETED
Git LFS Details
|
vocab.txt
DELETED
|
The diff for this file is too large to render.
See raw diff
|
|
|
z-image-turbo/output.png
DELETED
Git LFS Details
|
z-image-turbo/seed_test_1.png
DELETED
Git LFS Details
|
z-image-turbo/seed_test_2.png
DELETED
Git LFS Details
|
z_image_turbo_text_encoder.cccc.bin
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:c3db3dc860798d8e0978aa1a1a318acbffc1c0bc3a8e659b769186ca3bed8a53
|
| 3 |
-
size 7843085329
|
|
|
|
|
|
|
|
|
|
|
|
z_image_turbo_transformer_all_final_layer.cccc.bin
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:1d315e812a77b3523d9b9e6f5d4d09e233c49acf408884506b2d5558b095e326
|
| 3 |
-
size 2465674
|
|
|
|
|
|
|
|
|
|
|
|
z_image_turbo_transformer_all_x_embedder.cccc.bin
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:d1788f6bd48798a99a5c0b330b80c64903b62bd17ec31dbc20c58914bf2a6a8d
|
| 3 |
-
size 499396
|
|
|
|
|
|
|
|
|
|
|
|
z_image_turbo_transformer_cap_embedder.cccc.bin
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:e0c4d9ab33a4a11579356b7de35fc3e94394db349d26e95afaacf463402ecef6
|
| 3 |
-
size 19673829
|
|
|
|
|
|
|
|
|
|
|
|
z_image_turbo_transformer_context_refiner.cccc.bin
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:2f2df00443b15c29229a841c6bc394d8e054558163ed8bd99ae4715267366ab4
|
| 3 |
-
size 707852386
|
|
|
|
|
|
|
|
|
|
|
|
z_image_turbo_transformer_noise_refiner.cccc.bin
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:0f1e6b0a368167fc4c3001959b7ddc86b11d8f178f777d839bdd49b4a7541fb6
|
| 3 |
-
size 723642613
|
|
|
|
|
|
|
|
|
|
|
|
z_image_turbo_transformer_t_embedder.cccc.bin
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:06d39ed60ad841417a175a019f1a7f5e4e50723be36c3c34371dc94678d4879f
|
| 3 |
-
size 1051688
|
|
|
|
|
|
|
|
|
|
|
|
z_image_turbo_transformer_unified.cccc.bin
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:371b79b1491c8c2c019c9371432c29defe0e50c3240b134dde6ed7271a226770
|
| 3 |
-
size 10854638523
|
|
|
|
|
|
|
|
|
|
|
|
z_image_turbo_vae.cccc.bin
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:a3a36f49e125563517392e4ebc20a1309e818b8e69107097ed2bb57f77f179af
|
| 3 |
-
size 99095867
|
|
|
|
|
|
|
|
|
|
|
|