kernel.def export("v_row_scale_f16") @v_row_scale_f16(%rows: index) {
%c1 = index.constant 1 : index
%c256 = index.constant 256 : index
kernel.launch.config workgroups(%rows, %c1, %c1) workgroup_size(%c256, %c1, %c1) : index
} launch(%rows_arg: index, %k_arg: index, %x: buffer, %scales: buffer, %x16: buffer) {
%rows = index.assume %rows_arg [range(%rows_arg, 1, 2097152)] : index
%k = index.assume %k_arg [range(%k_arg, 8, 65536), multiple_of(%k_arg, 8)] : index
%row_id = kernel.workgroup.id<x> : index
%row = index.assume %row_id [lt(%row_id, %rows)] : index
%lane = kernel.workitem.id<x> : index
%zero = index.constant 0 : offset
%c0 = index.constant 0 : index
%c1 = index.constant 1 : index
%c256 = index.constant 256 : index
%zero_f = scalar.constant 0.0 : f32
%x_view = buffer.view %x[%zero] : buffer -> view<[%rows]x[%k]xf32>
%h_view = buffer.view %x16[%zero] : buffer -> view<[%rows]x[%k]xf16>
%local = scf.for %c = [%lane to %k step %c256](%m = %zero_f : f32) -> (f32) {
%bc = index.assume %c [lt(%c, %k)] : index
%v = view.load %x_view[%row, %bc] : view<[%rows]x[%k]xf32> -> f32
%a = scalar.absf %v : f32
%n = scalar.maxnumf %m, %a : f32
scf.yield %n : f32
}
%maximum = kernel.workgroup.reduce<maxnumf> %local : f32
%bits = scalar.bitcast %maximum : f32 to i32
%c23 = scalar.constant 23 : i32
%c255 = scalar.constant 255 : i32
%shifted = scalar.shrui %bits, %c23 : i32
%biased = scalar.andi %shifted, %c255 : i32
%c127 = scalar.constant 127 : i32
%c13 = scalar.constant 13 : i32
%c0_i32 = scalar.constant 0 : i32
%c100 = scalar.constant 100 : i32
%unbiased = scalar.subi %biased, %c127 : i32
%raw_e = scalar.subi %unbiased, %c13 : i32
%positive = scalar.maxsi %raw_e, %c0_i32 : i32
%e = scalar.minsi %positive, %c100 : i32
%scale_exp = scalar.subi %c127, %e : i32
%inverse_exp = scalar.addi %c127, %e : i32
%scale_bits = scalar.shli %scale_exp, %c23 : i32
%inverse_bits = scalar.shli %inverse_exp, %c23 : i32
%scale = scalar.bitcast %scale_bits : i32 to f32
%inverse = scalar.bitcast %inverse_bits : i32 to f32
scf.for %c = [%lane to %k step %c256] {
%bc = index.assume %c [lt(%c, %k)] : index
%v = view.load %x_view[%row, %bc] : view<[%rows]x[%k]xf32> -> f32
%sv = scalar.mulf %v, %scale : f32
%hv = scalar.fptrunc %sv : f32 to f16
view.store %hv, %h_view[%row, %bc] : f16, view<[%rows]x[%k]xf16>
}
%first = index.cmp eq, %lane, %c0 : index
scf.if %first {
%s_view = buffer.view %scales[%zero] : buffer -> view<[%rows]x2xf32>
view.store %scale, %s_view[%row, %c0] : f32, view<[%rows]x2xf32>
view.store %inverse, %s_view[%row, %c1] : f32, view<[%rows]x2xf32>
}
kernel.return
}
kernel.def export("v_pixels_0p5_0p5_0p5") @v_pixels_0p5_0p5_0p5(%n: index) {
%c1 = index.constant 1 : index
%c256 = index.constant 256 : index
%c255 = index.constant 255 : index
%n_up = index.add %n, %c255 : index
%groups = index.div %n_up, %c256 : index
kernel.launch.config workgroups(%groups, %c1, %c1) workgroup_size(%c256, %c1, %c1) : index
} launch(%n_arg: index, %src: buffer, %y: buffer) {
%n = index.assume %n_arg [range(%n_arg, 1, 268435456)] : index
%c256 = index.constant 256 : index
%c3 = index.constant 3 : index
%gid = kernel.workgroup.id<x> : index
%lid = kernel.workitem.id<x> : index
%i = index.madd %gid, %c256, %lid : index
%live = index.cmp ult, %i, %n : index
scf.if %live {
%bi = index.assume %i [lt(%i, %n)] : index
%zero = index.constant 0 : offset
%sv = buffer.view %src[%zero] : buffer -> view<[%n]xi8>
%raw = view.load %sv[%bi] : view<[%n]xi8> -> i8
%u = scalar.extui %raw : i8 to i32
%f = scalar.uitofp %u : i32 to f32
%ch = index.rem %bi, %c3 : index
%c0 = index.constant 0 : index
%c1 = index.constant 1 : index
%is0 = index.cmp eq, %ch, %c0 : index
%is1 = index.cmp eq, %ch, %c1 : index
%a0 = scalar.constant 0.00784313725490196 : f32
%a1 = scalar.constant 0.00784313725490196 : f32
%a2 = scalar.constant 0.00784313725490196 : f32
%b0 = scalar.constant -1.0 : f32
%b1 = scalar.constant -1.0 : f32
%b2 = scalar.constant -1.0 : f32
%a12 = scf.select %is1, %a1, %a2 : f32
%a = scf.select %is0, %a0, %a12 : f32
%b12 = scf.select %is1, %b1, %b2 : f32
%b = scf.select %is0, %b0, %b12 : f32
%r = scalar.fmaf %f, %a, %b : f32
%yv = buffer.view %y[%zero] : buffer -> view<[%n]xf32>
view.store %r, %yv[%bi] : f32, view<[%n]xf32>
}
kernel.return
}
kernel.def export("v_patch_16_1152_f16_block2_dual_bias") @v_patch_16_1152_f16_block2_dual_bias(%tokens: index) {
%c1 = index.constant 1 : index
%c256 = index.constant 256 : index
kernel.launch.config workgroups(%tokens, %c1, %c1) workgroup_size(%c256, %c1, %c1) : index
} launch(%tokens_arg: index, %gw_arg: index, %img_w_arg: index, %pix: buffer, %w: buffer, %w2: buffer, %b: buffer, %y: buffer) {
%tokens = index.assume %tokens_arg [range(%tokens_arg, 1, 65536)] : index
%gw = index.assume %gw_arg [range(%gw_arg, 1, 1024)] : index
%img_w = index.assume %img_w_arg [range(%img_w_arg, 1, 16384)] : index
%t_id = kernel.workgroup.id<x> : index
%t = index.assume %t_id [lt(%t_id, %tokens)] : index
%lid = kernel.workitem.id<x> : index
%lane = index.assume %lid [range(%lid, 0, 255)] : index
%zero = index.constant 0 : offset
%c0 = index.constant 0 : index
%c1 = index.constant 1 : index
%c3 = index.constant 3 : index
%c256 = index.constant 256 : index
%cP = index.constant 16 : index
%cPP = index.constant 256 : index
%cK = index.constant 768 : index
%cKP = index.constant 768 : index
%cE = index.constant 1152 : index
%zf = scalar.constant 0.0 : f32
%c2o = index.constant 2 : index
%c4o = index.constant 4 : index
%bw0 = index.div %gw, %c2o : index
%bw1 = index.max %bw0, %c1 : index
%bw = index.assume %bw1 [range(%bw1, 1, 4096)] : index
%blk = index.div %t, %c4o : index
%sub = index.rem %t, %c4o : index
%ty = index.div %blk, %bw : index
%tx = index.rem %blk, %bw : index
%dy = index.div %sub, %c2o : index
%dx = index.rem %sub, %c2o : index
%py = index.madd %ty, %c2o, %dy : index
%px = index.madd %tx, %c2o, %dx : index
%stage_bytes = index.constant 3072 : offset
%stage = buffer.alloca<workgroup> align(16) %stage_bytes : buffer
%sv = buffer.view %stage[%zero] : buffer -> view<768xf32>
%rows_px = index.mul %tokens, %c1 : index
%pix_n = index.constant 268435456 : index
%pv = buffer.view %pix[%zero] : buffer -> view<[%pix_n]xf32>
%y0 = index.mul %py, %cP : index
%x0 = index.mul %px, %cP : index
scf.for %i = [%lane to %cKP step %c256] {
%bi = index.assume %i [lt(%i, %cKP)] : index
%inside = index.cmp ult, %i, %cK : index
%v = scf.if %inside -> (f32) {
%ch = index.div %i, %cPP : index
%r = index.rem %i, %cPP : index
%ky = index.div %r, %cP : index
%kx = index.rem %r, %cP : index
%yy = index.add %y0, %ky : index
%xx = index.add %x0, %kx : index
%rowpix = index.mul %yy, %img_w : index
%pixel = index.add %rowpix, %xx : index
%el = index.madd %pixel, %c3, %ch : index
%bel = index.assume %el [lt(%el, %pix_n)] : index
%pval = view.load %pv[%bel] : view<[%pix_n]xf32> -> f32
scf.yield %pval : f32
} else {
scf.yield %zf : f32
}
view.store %v, %sv[%bi] : f32, view<768xf32>
}
kernel.barrier<workgroup> scope(workgroup) ordering(acq_rel)
%wn = index.constant 884736 : index
%wv = buffer.view %w[%zero] : buffer -> view<[%wn]xf16>
%wv2 = buffer.view %w2[%zero] : buffer -> view<[%wn]xf16>
%yv = buffer.view %y[%zero] : buffer -> view<[%tokens]x1152xf32>
scf.for %e = [%lane to %cE step %c256] {
%be = index.assume %e [lt(%e, %cE)] : index
%wrow = index.mul %be, %cK : index
%acc = scf.for %k = [%c0 to %cK step %c1](%a = %zf : f32) -> (f32) {
%bk = index.assume %k [lt(%k, %cKP)] : index
%xk = view.load %sv[%bk] : view<768xf32> -> f32
%wi = index.add %wrow, %k : index
%bwi = index.assume %wi [lt(%wi, %wn)] : index
%wk_h = view.load %wv[%bwi] : view<[%wn]xf16> -> f16
%wk = scalar.extf %wk_h : f16 to f32
%wk2_h = view.load %wv2[%bwi] : view<[%wn]xf16> -> f16
%wk2 = scalar.extf %wk2_h : f16 to f32
%wks = scalar.addf %wk, %wk2 : f32
%na = scalar.fmaf %wks, %xk, %a : f32
scf.yield %na : f32
}
%bview = buffer.view %b[%zero] : buffer -> view<1152xf32>
%bb = view.load %bview[%be] : view<1152xf32> -> f32
%out = scalar.addf %acc, %bb : f32
view.store %out, %yv[%t, %be] : f32, view<[%tokens]x1152xf32>
}
kernel.return
}
kernel.def export("v_pos_interp_1152_48_block2_ac") @v_pos_interp_1152_48_block2_ac(%tokens: index) {
%c1 = index.constant 1 : index
%c256 = index.constant 256 : index
kernel.launch.config workgroups(%tokens, %c1, %c1) workgroup_size(%c256, %c1, %c1) : index
} launch(%tokens_arg: index, %gw_arg: index, %gh_arg: index, %tab: buffer, %y: buffer) {
%tokens = index.assume %tokens_arg [range(%tokens_arg, 1, 65536)] : index
%gw = index.assume %gw_arg [range(%gw_arg, 1, 1024)] : index
%gh = index.assume %gh_arg [range(%gh_arg, 1, 1024)] : index
%t_id = kernel.workgroup.id<x> : index
%t = index.assume %t_id [lt(%t_id, %tokens)] : index
%lid = kernel.workitem.id<x> : index
%lane = index.assume %lid [range(%lid, 0, 255)] : index
%zero = index.constant 0 : offset
%c0 = index.constant 0 : index
%c1 = index.constant 1 : index
%c256 = index.constant 256 : index
%cE = index.constant 1152 : index
%nm1 = index.constant 47 : index
%c0i = scalar.constant 0 : i32
%nm1i = scalar.constant 47 : i32
%zf = scalar.constant 0.0 : f32
%onef = scalar.constant 1.0 : f32
%halff = scalar.constant 0.5 : f32
%nf = scalar.constant 48.0 : f32
%nm1f = scalar.constant 47.0 : f32
%c2o = index.constant 2 : index
%c4o = index.constant 4 : index
%bw0 = index.div %gw, %c2o : index
%bw1 = index.max %bw0, %c1 : index
%bw = index.assume %bw1 [range(%bw1, 1, 4096)] : index
%blk = index.div %t, %c4o : index
%sub = index.rem %t, %c4o : index
%ty = index.div %blk, %bw : index
%tx = index.rem %blk, %bw : index
%dy = index.div %sub, %c2o : index
%dx = index.rem %sub, %c2o : index
%py = index.madd %ty, %c2o, %dy : index
%px = index.madd %tx, %c2o, %dx : index
%py_of_i = index.cast %py : index to i32
%py_of = scalar.sitofp %py_of_i : i32 to f32
%gh_m1 = index.sub %gh, %c1 : index
%gh_m1f_i = index.cast %gh_m1 : index to i32
%gh_m1f = scalar.sitofp %gh_m1f_i : i32 to f32
%gh_den = scalar.maxnumf %gh_m1f, %onef : f32
%py_s0 = scalar.mulf %py_of, %nm1f : f32
%py_s = scalar.divf %py_s0, %gh_den : f32
%px_of_i = index.cast %px : index to i32
%px_of = scalar.sitofp %px_of_i : i32 to f32
%gw_m1 = index.sub %gw, %c1 : index
%gw_m1f_i = index.cast %gw_m1 : index to i32
%gw_m1f = scalar.sitofp %gw_m1f_i : i32 to f32
%gw_den = scalar.maxnumf %gw_m1f, %onef : f32
%px_s0 = scalar.mulf %px_of, %nm1f : f32
%px_s = scalar.divf %px_s0, %gw_den : f32
%py_fl = scalar.floorf %py_s : f32
%py_fr = scalar.subf %py_s, %py_fl : f32
%py_i0r = scalar.fptosi %py_fl : f32 to i32
%py_i0c = scalar.maxsi %py_i0r, %c0i : i32
%py_i0d = scalar.minsi %py_i0c, %nm1i : i32
%py_i0a = scalar.assume %py_i0d [range(%py_i0d, 0, 47)] : i32
%py_i0 = index.cast %py_i0a : i32 to index
%py_i1r = index.add %py_i0, %c1 : index
%py_i1m = index.min %py_i1r, %nm1 : index
%py_i1 = index.assume %py_i1m [range(%py_i1m, 0, 47)] : index
%px_fl = scalar.floorf %px_s : f32
%px_fr = scalar.subf %px_s, %px_fl : f32
%px_i0r = scalar.fptosi %px_fl : f32 to i32
%px_i0c = scalar.maxsi %px_i0r, %c0i : i32
%px_i0d = scalar.minsi %px_i0c, %nm1i : i32
%px_i0a = scalar.assume %px_i0d [range(%px_i0d, 0, 47)] : i32
%px_i0 = index.cast %px_i0a : i32 to index
%px_i1r = index.add %px_i0, %c1 : index
%px_i1m = index.min %px_i1r, %nm1 : index
%px_i1 = index.assume %px_i1m [range(%px_i1m, 0, 47)] : index
%fy1 = scalar.addf %py_fr, %zf : f32
%fy0 = scalar.subf %onef, %py_fr : f32
%fx1 = scalar.addf %px_fr, %zf : f32
%fx0 = scalar.subf %onef, %px_fr : f32
%tv = buffer.view %tab[%zero] : buffer -> view<48x48x1152xf32>
%yv = buffer.view %y[%zero] : buffer -> view<[%tokens]x1152xf32>
scf.for %e = [%lane to %cE step %c256] {
%be = index.assume %e [lt(%e, %cE)] : index
%v00 = view.load %tv[%py_i0, %px_i0, %be] : view<48x48x1152xf32> -> f32
%v01 = view.load %tv[%py_i0, %px_i1, %be] : view<48x48x1152xf32> -> f32
%v10 = view.load %tv[%py_i1, %px_i0, %be] : view<48x48x1152xf32> -> f32
%v11 = view.load %tv[%py_i1, %px_i1, %be] : view<48x48x1152xf32> -> f32
%r0a = scalar.mulf %v00, %fx0 : f32
%r0 = scalar.fmaf %v01, %fx1, %r0a : f32
%r1a = scalar.mulf %v10, %fx0 : f32
%r1 = scalar.fmaf %v11, %fx1, %r1a : f32
%ra = scalar.mulf %r0, %fy0 : f32
%r = scalar.fmaf %r1, %fy1, %ra : f32
%a = view.load %yv[%t, %be] : view<[%tokens]x1152xf32> -> f32
%s = scalar.addf %a, %r : f32
view.store %s, %yv[%t, %be] : f32, view<[%tokens]x1152xf32>
}
kernel.return
}
kernel.def export("v_norm_ln_1152") @v_norm_ln_1152(%rows: index) {
%c1 = index.constant 1 : index
%c256 = index.constant 256 : index
kernel.launch.config workgroups(%rows, %c1, %c1) workgroup_size(%c256, %c1, %c1) : index
} launch(%rows_arg: index, %x: buffer, %w: buffer, %b: buffer, %y: buffer) {
%rows = index.assume %rows_arg [range(%rows_arg, 1, 65536)] : index
%row_id = kernel.workgroup.id<x> : index
%row = index.assume %row_id [lt(%row_id, %rows)] : index
%lane_id = kernel.workitem.id<x> : index
%lane = index.assume %lane_id [range(%lane_id, 0, 255)] : index
%zero = index.constant 0 : offset
%c0 = index.constant 0 : index
%groups = index.constant 36 : index
%zero_f = scalar.constant 0.0 : f32
%zero32 = vector.constant 0.0 : vector<32xf32>
%count = scalar.constant 1152.0 : f32
%active = index.cmp ult, %lane, %groups : index
%x_view = buffer.view %x[%zero] : buffer -> view<[%rows]x36x32xf32>
%xv = scf.if %active -> (vector<32xf32>) {
%bl = index.assume %lane [range(%lane, 0, 35)] : index
%a = vector.load %x_view[%row, %bl, %c0] : view<[%rows]x36x32xf32> -> vector<32xf32>
scf.yield %a : vector<32xf32>
} else {
scf.yield %zero32 : vector<32xf32>
}
%xlocal = vector.reduce<addf> %xv, %zero_f : vector<32xf32>, f32
%xsum = kernel.workgroup.reduce<addf> %xlocal : f32
%xmean = scalar.divf %xsum, %count : f32
%xmeans = vector.splat %xmean : vector<32xf32>
%xd = vector.subf %xv, %xmeans : vector<32xf32>
%xc = scf.select %active, %xd, %zero32 : vector<32xf32>
%sq = vector.mulf %xc, %xc : vector<32xf32>
%local = vector.reduce<addf> %sq, %zero_f : vector<32xf32>, f32
%sum = kernel.workgroup.reduce<addf> %local : f32
%mean = scalar.divf %sum, %count : f32
%eps = scalar.constant 9.999999974752427e-07 : f32
%stable = scalar.addf %mean, %eps : f32
%inv = scalar.rsqrtf %stable : f32
%invs = vector.splat %inv : vector<32xf32>
scf.if %active {
%bl = index.assume %lane [range(%lane, 0, 35)] : index
%xn = vector.mulf %xc, %invs : vector<32xf32>
%w_view = buffer.view %w[%zero] : buffer -> view<36x32xf32>
%b_view = buffer.view %b[%zero] : buffer -> view<36x32xf32>
%wv = vector.load %w_view[%bl, %c0] : view<36x32xf32> -> vector<32xf32>
%bv = vector.load %b_view[%bl, %c0] : view<36x32xf32> -> vector<32xf32>
%xw = vector.mulf %xn, %wv : vector<32xf32>
%r = vector.addf %xw, %bv : vector<32xf32>
%y_view = buffer.view %y[%zero] : buffer -> view<[%rows]x36x32xf32>
vector.store %r, %y_view[%row, %bl, %c0] : vector<32xf32>, view<[%rows]x36x32xf32>
}
kernel.return
}
kernel.def export("v_gemm_f16_store_1152x3456") @v_gemm_f16_store_1152x3456(%rows: index, %n: index) {
%c1 = index.constant 1 : index
%c64 = index.constant 64 : index
%c256 = index.constant 256 : index
%row_tiles = index.div %rows, %c64 : index
%n_const = index.constant 3456 : index
%out_tiles = index.div %n_const, %c64 : index
kernel.launch.config workgroups(%row_tiles, %out_tiles, %c1) workgroup_size(%c256, %c1, %c1) : index
} launch(%rows_arg: index, %k_arg: index, %n_arg: index, %x: buffer, %scales: buffer, %w: buffer, %y: buffer) {
%rows = index.assume %rows_arg [range(%rows_arg, 64, 65536), multiple_of(%rows_arg, 64)] : index
%k = index.constant 1152 : index
%n = index.constant 3456 : index
%origin = index.constant 0 : offset
%two = index.constant 2 : offset
%sixteen = index.constant 16 : offset
%block_bytes = index.constant 64 : offset
%c0 = index.constant 0 : index
%c1 = index.constant 1 : index
%c2 = index.constant 2 : index
%c4 = index.constant 4 : index
%c8 = index.constant 8 : index
%c16 = index.constant 16 : index
%c32 = index.constant 32 : index
%c64 = index.constant 64 : index
%c256 = index.constant 256 : index
%nibble_mask = vector.constant 252645135 : vector<4xi32>
%stage_bytes = index.constant 9216 : offset
%row_tile_count = index.div %rows, %c64 : index
%out_tile_count = index.div %n, %c64 : index
%row_group_id = kernel.workgroup.id<x> : index
%out_group_id = kernel.workgroup.id<y> : index
%row_group = index.assume %row_group_id [lt(%row_group_id, %row_tile_count)] : index
%out_group = index.assume %out_group_id [lt(%out_group_id, %out_tile_count)] : index
%workitem_id = kernel.workitem.id<x> : index
%workitem = index.assume %workitem_id [range(%workitem_id, 0, 255)] : index
%subgroup = kernel.subgroup.id : index
%lane = kernel.subgroup.lane.id : index
%lane_row = index.rem %lane, %c16 : index
%bounded_subgroup = index.assume %subgroup [range(%subgroup, 0, 7)] : index
%wave_row_group = index.div %bounded_subgroup, %c2 : index
%wave_out_group = index.rem %bounded_subgroup, %c2 : index
%wave_row = index.mul %wave_row_group, %c16 : index
%wave_out = index.mul %wave_out_group, %c32 : index
%w_row0 = index.add %wave_out, %lane_row : index
%w_row1 = index.add %w_row0, %c16 : index
%row_origin = index.mul %row_group, %c64 : index
%out_origin = index.mul %out_group, %c64 : index
%result_row = index.add %row_origin, %wave_row : index
%result_col0 = index.add %out_origin, %wave_out : index
%result_col1 = index.add %result_col0, %c16 : index
%blocks = index.div %k, %c32 : index
%row_bytes = index.scale %blocks, %block_bytes : index, offset -> offset
%x_view = buffer.view %x[%origin] : buffer -> view<[%rows]x[%k]xf16>
// 64x64 output tile with the full row stride n.
%y_layout = encoding.layout.strided [%n, %c1] : encoding<layout>
%tile_first = index.madd %row_origin, %n, %out_origin : index
%four = index.constant 4 : offset
%tile_offset = index.scale %tile_first, %four : index, offset -> offset
%y_view = buffer.view %y[%tile_offset] : buffer -> view<64x64xf32, %y_layout>
%bounded_wave_row = index.assume %wave_row [range(%wave_row, 0, 48)] : index
%bounded_wave_out = index.assume %wave_out [range(%wave_out, 0, 32)] : index
%wave_out1 = index.add %bounded_wave_out, %c16 : index
%x_stage = buffer.alloca<workgroup> align(16) %stage_bytes : buffer
%x_stage_view = buffer.view %x_stage[%origin] : buffer -> view<64x72xf16>
%w_stage = buffer.alloca<workgroup> align(16) %stage_bytes : buffer
%w_stage_view = buffer.view %w_stage[%origin] : buffer -> view<64x72xf16>
// Thread t stages 16 values: row t/4, columns (t%4)*16.
%stage_row = index.div %workitem, %c4 : index
%stage_quarter = index.rem %workitem, %c4 : index
%stage_col = index.mul %stage_quarter, %c16 : index
%x_row = index.add %row_origin, %stage_row : index
%bounded_x_row = index.assume %x_row [lt(%x_row, %rows)] : index
%w_out_row = index.add %out_origin, %stage_row : index
%bounded_w_out_row = index.assume %w_out_row [lt(%w_out_row, %n)] : index
%w_row_origin = index.scale %bounded_w_out_row, %row_bytes : index, offset -> offset
%zero8 = vector.constant 0.0 : vector<8xf32>
%init = vector.fragment<init> %zero8 shape [%c16, %c16] : vector<8xf32>
%k_steps = index.div %k, %c64 : index
%s_view = buffer.view %scales[%origin] : buffer -> view<[%rows]x2xf32>
%x_scale = view.load %s_view[%bounded_x_row, %c0] : view<[%rows]x2xf32> -> f32
%x_scales = vector.splat %x_scale : vector<16xf32>
%a0_raw, %a1_raw = scf.for %step = [%c0 to %k_steps step %c1](%a0_acc = %init : vector<8xf32>, %a1_acc = %init : vector<8xf32>) -> (vector<8xf32>, vector<8xf32>) {
%k_origin = index.mul %step, %c64 : index
%x_col = index.add %k_origin, %stage_col : index
%x_half = vector.load %x_view[%bounded_x_row, %x_col] : view<[%rows]x[%k]xf16> -> vector<16xf16>
vector.store %x_half, %x_stage_view[%stage_row, %stage_col] : vector<16xf16>, view<64x72xf16>
%w_col = index.add %k_origin, %stage_col : index
%deq0_two_b = index.constant 2 : offset
%deq0_col_bytes = index.scale %w_col, %deq0_two_b : index, offset -> offset
%deq0_origin = index.add %w_row_origin, %deq0_col_bytes : offset
%deq0_view = buffer.view %w[%deq0_origin] : buffer -> view<16xf16>
%deq0 = vector.load %deq0_view[0] : view<16xf16> -> vector<16xf16>
vector.store %deq0, %w_stage_view[%stage_row, %stage_col] : vector<16xf16>, view<64x72xf16>
kernel.barrier<workgroup> scope(workgroup) ordering(acq_rel)
%fc0 = index.constant 0 : index
%lhs0 = vector.fragment.load<lhs> %x_stage_view[%wave_row, %fc0] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16>
%rv000 = vector.load %w_stage_view[%w_row0, %fc0] : view<64x72xf16> -> vector<16xf16>
%rhs000 = vector.fragment<rhs> %rv000 shape [%c16, %c16] : vector<16xf16>
%na0_0 = vector.mma %lhs0, %rhs000, %a0_acc : vector<16xf16>, vector<16xf16>, vector<8xf32>
%rv010 = vector.load %w_stage_view[%w_row1, %fc0] : view<64x72xf16> -> vector<16xf16>
%rhs010 = vector.fragment<rhs> %rv010 shape [%c16, %c16] : vector<16xf16>
%na1_0 = vector.mma %lhs0, %rhs010, %a1_acc : vector<16xf16>, vector<16xf16>, vector<8xf32>
%fc1 = index.constant 16 : index
%lhs1 = vector.fragment.load<lhs> %x_stage_view[%wave_row, %fc1] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16>
%rv001 = vector.load %w_stage_view[%w_row0, %fc1] : view<64x72xf16> -> vector<16xf16>
%rhs001 = vector.fragment<rhs> %rv001 shape [%c16, %c16] : vector<16xf16>
%na0_1 = vector.mma %lhs1, %rhs001, %na0_0 : vector<16xf16>, vector<16xf16>, vector<8xf32>
%rv011 = vector.load %w_stage_view[%w_row1, %fc1] : view<64x72xf16> -> vector<16xf16>
%rhs011 = vector.fragment<rhs> %rv011 shape [%c16, %c16] : vector<16xf16>
%na1_1 = vector.mma %lhs1, %rhs011, %na1_0 : vector<16xf16>, vector<16xf16>, vector<8xf32>
%fc2 = index.constant 32 : index
%lhs2 = vector.fragment.load<lhs> %x_stage_view[%wave_row, %fc2] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16>
%rv002 = vector.load %w_stage_view[%w_row0, %fc2] : view<64x72xf16> -> vector<16xf16>
%rhs002 = vector.fragment<rhs> %rv002 shape [%c16, %c16] : vector<16xf16>
%na0_2 = vector.mma %lhs2, %rhs002, %na0_1 : vector<16xf16>, vector<16xf16>, vector<8xf32>
%rv012 = vector.load %w_stage_view[%w_row1, %fc2] : view<64x72xf16> -> vector<16xf16>
%rhs012 = vector.fragment<rhs> %rv012 shape [%c16, %c16] : vector<16xf16>
%na1_2 = vector.mma %lhs2, %rhs012, %na1_1 : vector<16xf16>, vector<16xf16>, vector<8xf32>
%fc3 = index.constant 48 : index
%lhs3 = vector.fragment.load<lhs> %x_stage_view[%wave_row, %fc3] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16>
%rv003 = vector.load %w_stage_view[%w_row0, %fc3] : view<64x72xf16> -> vector<16xf16>
%rhs003 = vector.fragment<rhs> %rv003 shape [%c16, %c16] : vector<16xf16>
%na0_3 = vector.mma %lhs3, %rhs003, %na0_2 : vector<16xf16>, vector<16xf16>, vector<8xf32>
%rv013 = vector.load %w_stage_view[%w_row1, %fc3] : view<64x72xf16> -> vector<16xf16>
%rhs013 = vector.fragment<rhs> %rv013 shape [%c16, %c16] : vector<16xf16>
%na1_3 = vector.mma %lhs3, %rhs013, %na1_2 : vector<16xf16>, vector<16xf16>, vector<8xf32>
kernel.barrier<workgroup> scope(workgroup) ordering(acq_rel)
scf.yield %na0_3, %na1_3 : vector<8xf32>, vector<8xf32>
}
%inv_bytes = index.constant 4096 : offset
%inv_tile = buffer.alloca<workgroup> align(16) %inv_bytes : buffer
%inv_view = buffer.view %inv_tile[%origin] : buffer -> view<64x16xf32>
%writer = index.cmp ult, %workitem, %c64 : index
scf.if %writer {
%inv_row_value = index.add %row_origin, %workitem : index
%inv_row = index.assume %inv_row_value [lt(%inv_row_value, %rows)] : index
%inverse = view.load %s_view[%inv_row, %c1] : view<[%rows]x2xf32> -> f32
%inverse16 = vector.splat %inverse : vector<16xf32>
%bw = index.assume %workitem [range(%workitem, 0, 63)] : index
vector.store %inverse16, %inv_view[%bw, %c0] : vector<16xf32>, view<64x16xf32>
}
kernel.barrier<workgroup> scope(workgroup) ordering(acq_rel)
%inv_frag = vector.fragment.load<result> %inv_view[%wave_row, %c0] shape [%c16, %c16] : view<64x16xf32> -> vector<8xf32>
%a0 = vector.mulf %a0_raw, %inv_frag : vector<8xf32>
%a1 = vector.mulf %a1_raw, %inv_frag : vector<8xf32>
vector.fragment.store<result> %a0, %y_view[%bounded_wave_row, %bounded_wave_out] shape [%c16, %c16] : vector<8xf32>, view<64x64xf32, %y_layout>
vector.fragment.store<result> %a1, %y_view[%bounded_wave_row, %wave_out1] shape [%c16, %c16] : vector<8xf32>, view<64x64xf32, %y_layout>
kernel.return
}
kernel.def export("v_bias_none_3456") @v_bias_none_3456(%rows: index) {
%c1 = index.constant 1 : index
%c256 = index.constant 256 : index
%cols = index.constant 3456 : index
%cg = index.constant 14 : index
kernel.launch.config workgroups(%rows, %cg, %c1) workgroup_size(%c256, %c1, %c1) : index
} launch(%rows_arg: index, %x: buffer, %bias: buffer, %y: buffer) {
%rows = index.assume %rows_arg [range(%rows_arg, 1, 65536)] : index
%c256 = index.constant 256 : index
%cols = index.constant 3456 : index
%row_id = kernel.workgroup.id<x> : index
%row = index.assume %row_id [lt(%row_id, %rows)] : index
%cg = kernel.workgroup.id<y> : index
%lid = kernel.workitem.id<x> : index
%c = index.madd %cg, %c256, %lid : index
%live = index.cmp ult, %c, %cols : index
scf.if %live {
%bc = index.assume %c [lt(%c, %cols)] : index
%zero = index.constant 0 : offset
%zf = scalar.constant 0.0 : f32
%xv = buffer.view %x[%zero] : buffer -> view<[%rows]x3456xf32>
%bv = buffer.view %bias[%zero] : buffer -> view<3456xf32>
%x0 = view.load %xv[%row, %bc] : view<[%rows]x3456xf32> -> f32
%b0 = view.load %bv[%bc] : view<3456xf32> -> f32
%a = scalar.addf %x0, %b0 : f32
%r = scalar.addf %a, %zf : f32
%yv = buffer.view %y[%zero] : buffer -> view<[%rows]x3456xf32>
view.store %r, %yv[%row, %bc] : f32, view<[%rows]x3456xf32>
}
kernel.return
}
kernel.def export("v_rope_qwen_vision_72x16_s3456_o0_block2") @v_rope_qwen_vision_72x16_s3456_o0_block2(%tokens: index) {
%c1 = index.constant 1 : index
%c256 = index.constant 256 : index
kernel.launch.config workgroups(%tokens, %c1, %c1) workgroup_size(%c256, %c1, %c1) : index
} launch(%tokens_arg: index, %gw_arg: index, %x: buffer) {
%tokens = index.assume %tokens_arg [range(%tokens_arg, 1, 65536)] : index
%gw = index.assume %gw_arg [range(%gw_arg, 1, 4096)] : index
%t_id = kernel.workgroup.id<x> : index
%t = index.assume %t_id [lt(%t_id, %tokens)] : index
%lid = kernel.workitem.id<x> : index
%lane = index.assume %lid [range(%lid, 0, 255)] : index
%zero = index.constant 0 : offset
%c0 = index.constant 0 : index
%c1 = index.constant 1 : index
%c256 = index.constant 256 : index
%cNP = index.constant 36 : index
%cWORK = index.constant 576 : index
%cD = index.constant 72 : index
%cO = index.constant 0 : index
%c2o = index.constant 2 : index
%c4o = index.constant 4 : index
%bw0 = index.div %gw, %c2o : index
%bw1 = index.max %bw0, %c1 : index
%bw = index.assume %bw1 [range(%bw1, 1, 4096)] : index
%blk = index.div %t, %c4o : index
%sub = index.rem %t, %c4o : index
%ty = index.div %blk, %bw : index
%tx = index.rem %blk, %bw : index
%dy = index.div %sub, %c2o : index
%dx = index.rem %sub, %c2o : index
%py = index.madd %ty, %c2o, %dy : index
%px = index.madd %tx, %c2o, %dx : index
%pxi = index.cast %px : index to i32
%pyi = index.cast %py : index to i32
%offi = scalar.constant 0 : i32
%pxo = scalar.addi %pxi, %offi : i32
%pyo = scalar.addi %pyi, %offi : i32
%pxf = scalar.sitofp %pxo : i32 to f32
%pyf = scalar.sitofp %pyo : i32 to f32
%two_pi = scalar.constant 6.283185307179586 : f32
%inv_two_pi = scalar.constant 0.15915494309189535 : f32
%xv = buffer.view %x[%zero] : buffer -> view<[%tokens]x3456xf32>
%cq = index.constant 18 : index
%chalf = index.constant 36 : index
%lnth = scalar.constant 18.420680743952367 : f32
%halff = scalar.constant 36.0 : f32
scf.for %w = [%lane to %cWORK step %c256] {
%h = index.div %w, %cNP : index
%j0 = index.rem %w, %cNP : index
%j = index.assume %j0 [range(%j0, 0, 35)] : index
%isx = index.cmp uge, %j, %cq : index
%true = scalar.constant true : i1
%isy = scalar.xori %isx, %true : i1
%jj = index.rem %j, %cq : index
%lo_i = index.add %j, %c0 : index
%hi_i = index.add %j, %chalf : index
%ii = index.cast %jj : index to i32
%if0 = scalar.sitofp %ii : i32 to f32
%if1 = scalar.mulf %if0, %lnth : f32
%if2 = scalar.divf %if1, %halff : f32
%if3 = scalar.negf %if2 : f32
%invf = scalar.expf<afn> %if3 : f32
%pos = scf.select %isy, %pyf, %pxf : f32
%ang0 = scalar.mulf %pos, %invf : f32
%tr = scalar.mulf %ang0, %inv_two_pi : f32
%trf = scalar.floorf %tr : f32
%wrap = scalar.mulf %trf, %two_pi : f32
%ang = scalar.subf %ang0, %wrap : f32
%cs = scalar.cosf<afn> %ang : f32
%sn = scalar.sinf<afn> %ang : f32
%hcol = index.madd %h, %cD, %cO : index
%clo0 = index.add %hcol, %lo_i : index
%chi0 = index.add %hcol, %hi_i : index
%clo = index.assume %clo0 [lt(%clo0, 3456)] : index
%chi = index.assume %chi0 [lt(%chi0, 3456)] : index
%a = view.load %xv[%t, %clo] : view<[%tokens]x3456xf32> -> f32
%b = view.load %xv[%t, %chi] : view<[%tokens]x3456xf32> -> f32
%acs = scalar.mulf %a, %cs : f32
%bsn = scalar.mulf %b, %sn : f32
%ra = scalar.subf %acs, %bsn : f32
%asn = scalar.mulf %a, %sn : f32
%rb = scalar.fmaf %b, %cs, %asn : f32
view.store %ra, %xv[%t, %clo] : f32, view<[%tokens]x3456xf32>
view.store %rb, %xv[%t, %chi] : f32, view<[%tokens]x3456xf32>
}
kernel.return
}
kernel.def export("v_rope_qwen_vision_72x16_s3456_o1152_block2") @v_rope_qwen_vision_72x16_s3456_o1152_block2(%tokens: index) {
%c1 = index.constant 1 : index
%c256 = index.constant 256 : index
kernel.launch.config workgroups(%tokens, %c1, %c1) workgroup_size(%c256, %c1, %c1) : index
} launch(%tokens_arg: index, %gw_arg: index, %x: buffer) {
%tokens = index.assume %tokens_arg [range(%tokens_arg, 1, 65536)] : index
%gw = index.assume %gw_arg [range(%gw_arg, 1, 4096)] : index
%t_id = kernel.workgroup.id<x> : index
%t = index.assume %t_id [lt(%t_id, %tokens)] : index
%lid = kernel.workitem.id<x> : index
%lane = index.assume %lid [range(%lid, 0, 255)] : index
%zero = index.constant 0 : offset
%c0 = index.constant 0 : index
%c1 = index.constant 1 : index
%c256 = index.constant 256 : index
%cNP = index.constant 36 : index
%cWORK = index.constant 576 : index
%cD = index.constant 72 : index
%cO = index.constant 1152 : index
%c2o = index.constant 2 : index
%c4o = index.constant 4 : index
%bw0 = index.div %gw, %c2o : index
%bw1 = index.max %bw0, %c1 : index
%bw = index.assume %bw1 [range(%bw1, 1, 4096)] : index
%blk = index.div %t, %c4o : index
%sub = index.rem %t, %c4o : index
%ty = index.div %blk, %bw : index
%tx = index.rem %blk, %bw : index
%dy = index.div %sub, %c2o : index
%dx = index.rem %sub, %c2o : index
%py = index.madd %ty, %c2o, %dy : index
%px = index.madd %tx, %c2o, %dx : index
%pxi = index.cast %px : index to i32
%pyi = index.cast %py : index to i32
%offi = scalar.constant 0 : i32
%pxo = scalar.addi %pxi, %offi : i32
%pyo = scalar.addi %pyi, %offi : i32
%pxf = scalar.sitofp %pxo : i32 to f32
%pyf = scalar.sitofp %pyo : i32 to f32
%two_pi = scalar.constant 6.283185307179586 : f32
%inv_two_pi = scalar.constant 0.15915494309189535 : f32
%xv = buffer.view %x[%zero] : buffer -> view<[%tokens]x3456xf32>
%cq = index.constant 18 : index
%chalf = index.constant 36 : index
%lnth = scalar.constant 18.420680743952367 : f32
%halff = scalar.constant 36.0 : f32
scf.for %w = [%lane to %cWORK step %c256] {
%h = index.div %w, %cNP : index
%j0 = index.rem %w, %cNP : index
%j = index.assume %j0 [range(%j0, 0, 35)] : index
%isx = index.cmp uge, %j, %cq : index
%true = scalar.constant true : i1
%isy = scalar.xori %isx, %true : i1
%jj = index.rem %j, %cq : index
%lo_i = index.add %j, %c0 : index
%hi_i = index.add %j, %chalf : index
%ii = index.cast %jj : index to i32
%if0 = scalar.sitofp %ii : i32 to f32
%if1 = scalar.mulf %if0, %lnth : f32
%if2 = scalar.divf %if1, %halff : f32
%if3 = scalar.negf %if2 : f32
%invf = scalar.expf<afn> %if3 : f32
%pos = scf.select %isy, %pyf, %pxf : f32
%ang0 = scalar.mulf %pos, %invf : f32
%tr = scalar.mulf %ang0, %inv_two_pi : f32
%trf = scalar.floorf %tr : f32
%wrap = scalar.mulf %trf, %two_pi : f32
%ang = scalar.subf %ang0, %wrap : f32
%cs = scalar.cosf<afn> %ang : f32
%sn = scalar.sinf<afn> %ang : f32
%hcol = index.madd %h, %cD, %cO : index
%clo0 = index.add %hcol, %lo_i : index
%chi0 = index.add %hcol, %hi_i : index
%clo = index.assume %clo0 [lt(%clo0, 3456)] : index
%chi = index.assume %chi0 [lt(%chi0, 3456)] : index
%a = view.load %xv[%t, %clo] : view<[%tokens]x3456xf32> -> f32
%b = view.load %xv[%t, %chi] : view<[%tokens]x3456xf32> -> f32
%acs = scalar.mulf %a, %cs : f32
%bsn = scalar.mulf %b, %sn : f32
%ra = scalar.subf %acs, %bsn : f32
%asn = scalar.mulf %a, %sn : f32
%rb = scalar.fmaf %b, %cs, %asn : f32
view.store %ra, %xv[%t, %clo] : f32, view<[%tokens]x3456xf32>
view.store %rb, %xv[%t, %chi] : f32, view<[%tokens]x3456xf32>
}
kernel.return
}
kernel.def export("v_attn_72x16_s3456_0_1152_2304") @v_attn_72x16_s3456_0_1152_2304(%tokens: index) {
%c1 = index.constant 1 : index
%c32 = index.constant 32 : index
%heads = index.constant 16 : index
kernel.launch.config workgroups(%tokens, %heads, %c1) workgroup_size(%c32, %c1, %c1) : index
} launch(%tokens_arg: index, %gw_arg: index, %q: buffer, %k: buffer, %v: buffer, %o: buffer) {
%tokens = index.assume %tokens_arg [range(%tokens_arg, 1, 65536)] : index
%gw = index.assume %gw_arg [range(%gw_arg, 1, 4096)] : index
%t_id = kernel.workgroup.id<x> : index
%t = index.assume %t_id [lt(%t_id, %tokens)] : index
%h_id = kernel.workgroup.id<y> : index
%h = index.assume %h_id [range(%h_id, 0, 15)] : index
%lid = kernel.workitem.id<x> : index
%lane = index.assume %lid [range(%lid, 0, 31)] : index
%zero = index.constant 0 : offset
%c0 = index.constant 0 : index
%c1 = index.constant 1 : index
%c32 = index.constant 32 : index
%cD = index.constant 72 : index
%zf = scalar.constant 0.0 : f32
%neg_big = scalar.constant -3.0e38 : f32
%scale = scalar.constant 0.11785113019775792 : f32
%zero32 = vector.constant 0.0 : vector<8xf32>
%qv = buffer.view %q[%zero] : buffer -> view<[%tokens]x3456xf32>
%kv = buffer.view %k[%zero] : buffer -> view<[%tokens]x3456xf32>
%vv = buffer.view %v[%zero] : buffer -> view<[%tokens]x3456xf32>
%cOQ = index.constant 0 : index
%cOK = index.constant 1152 : index
%cOV = index.constant 2304 : index
%qcol = index.madd %h, %cD, %cOQ : index
%kcol = index.madd %h, %cD, %cOK : index
%vcol = index.madd %h, %cD, %cOV : index
%qc00 = index.constant 0 : index
%qc01 = index.add %qcol, %qc00 : index
%qc0 = index.assume %qc01 [range(%qc01, 0, 3448)] : index
%qr0 = vector.load %qv[%t, %qc0] : view<[%tokens]x3456xf32> -> vector<8xf32>
%scv0 = vector.splat %scale : vector<8xf32>
%q0 = vector.mulf %qr0, %scv0 : vector<8xf32>
%qc10 = index.constant 8 : index
%qc11 = index.add %qcol, %qc10 : index
%qc1 = index.assume %qc11 [range(%qc11, 0, 3448)] : index
%qr1 = vector.load %qv[%t, %qc1] : view<[%tokens]x3456xf32> -> vector<8xf32>
%scv1 = vector.splat %scale : vector<8xf32>
%q1 = vector.mulf %qr1, %scv1 : vector<8xf32>
%qc20 = index.constant 16 : index
%qc21 = index.add %qcol, %qc20 : index
%qc2 = index.assume %qc21 [range(%qc21, 0, 3448)] : index
%qr2 = vector.load %qv[%t, %qc2] : view<[%tokens]x3456xf32> -> vector<8xf32>
%scv2 = vector.splat %scale : vector<8xf32>
%q2 = vector.mulf %qr2, %scv2 : vector<8xf32>
%qc30 = index.constant 24 : index
%qc31 = index.add %qcol, %qc30 : index
%qc3 = index.assume %qc31 [range(%qc31, 0, 3448)] : index
%qr3 = vector.load %qv[%t, %qc3] : view<[%tokens]x3456xf32> -> vector<8xf32>
%scv3 = vector.splat %scale : vector<8xf32>
%q3 = vector.mulf %qr3, %scv3 : vector<8xf32>
%qc40 = index.constant 32 : index
%qc41 = index.add %qcol, %qc40 : index
%qc4 = index.assume %qc41 [range(%qc41, 0, 3448)] : index
%qr4 = vector.load %qv[%t, %qc4] : view<[%tokens]x3456xf32> -> vector<8xf32>
%scv4 = vector.splat %scale : vector<8xf32>
%q4 = vector.mulf %qr4, %scv4 : vector<8xf32>
%qc50 = index.constant 40 : index
%qc51 = index.add %qcol, %qc50 : index
%qc5 = index.assume %qc51 [range(%qc51, 0, 3448)] : index
%qr5 = vector.load %qv[%t, %qc5] : view<[%tokens]x3456xf32> -> vector<8xf32>
%scv5 = vector.splat %scale : vector<8xf32>
%q5 = vector.mulf %qr5, %scv5 : vector<8xf32>
%qc60 = index.constant 48 : index
%qc61 = index.add %qcol, %qc60 : index
%qc6 = index.assume %qc61 [range(%qc61, 0, 3448)] : index
%qr6 = vector.load %qv[%t, %qc6] : view<[%tokens]x3456xf32> -> vector<8xf32>
%scv6 = vector.splat %scale : vector<8xf32>
%q6 = vector.mulf %qr6, %scv6 : vector<8xf32>
%qc70 = index.constant 56 : index
%qc71 = index.add %qcol, %qc70 : index
%qc7 = index.assume %qc71 [range(%qc71, 0, 3448)] : index
%qr7 = vector.load %qv[%t, %qc7] : view<[%tokens]x3456xf32> -> vector<8xf32>
%scv7 = vector.splat %scale : vector<8xf32>
%q7 = vector.mulf %qr7, %scv7 : vector<8xf32>
%qc80 = index.constant 64 : index
%qc81 = index.add %qcol, %qc80 : index
%qc8 = index.assume %qc81 [range(%qc81, 0, 3448)] : index
%qr8 = vector.load %qv[%t, %qc8] : view<[%tokens]x3456xf32> -> vector<8xf32>
%scv8 = vector.splat %scale : vector<8xf32>
%q8 = vector.mulf %qr8, %scv8 : vector<8xf32>
%m_l, %l_l, %acc0_l, %acc1_l, %acc2_l, %acc3_l, %acc4_l, %acc5_l, %acc6_l, %acc7_l, %acc8_l = scf.for %j = [%lane to %tokens step %c32](%m_c = %neg_big : f32, %l_c = %zf : f32, %acc0_c = %zero32 : vector<8xf32>, %acc1_c = %zero32 : vector<8xf32>, %acc2_c = %zero32 : vector<8xf32>, %acc3_c = %zero32 : vector<8xf32>, %acc4_c = %zero32 : vector<8xf32>, %acc5_c = %zero32 : vector<8xf32>, %acc6_c = %zero32 : vector<8xf32>, %acc7_c = %zero32 : vector<8xf32>, %acc8_c = %zero32 : vector<8xf32>) -> (f32, f32, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>) {
%bj = index.assume %j [lt(%j, %tokens)] : index
%allowed = index.cmp ult, %bj, %tokens : index
%m_n, %l_n, %acc0_n, %acc1_n, %acc2_n, %acc3_n, %acc4_n, %acc5_n, %acc6_n, %acc7_n, %acc8_n = scf.if %allowed -> (f32, f32, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>) {
%kc00 = index.constant 0 : index
%kc01 = index.add %kcol, %kc00 : index
%kc0 = index.assume %kc01 [range(%kc01, 0, 3448)] : index
%kr0 = vector.load %kv[%bj, %kc0] : view<[%tokens]x3456xf32> -> vector<8xf32>
%vc01 = index.add %vcol, %kc00 : index
%vc0 = index.assume %vc01 [range(%vc01, 0, 3448)] : index
%vr0 = vector.load %vv[%bj, %vc0] : view<[%tokens]x3456xf32> -> vector<8xf32>
%pd0 = vector.mulf %q0, %kr0 : vector<8xf32>
%ps0 = vector.reduce<addf> %pd0, %zf : vector<8xf32>, f32
%kc10 = index.constant 8 : index
%kc11 = index.add %kcol, %kc10 : index
%kc1 = index.assume %kc11 [range(%kc11, 0, 3448)] : index
%kr1 = vector.load %kv[%bj, %kc1] : view<[%tokens]x3456xf32> -> vector<8xf32>
%vc11 = index.add %vcol, %kc10 : index
%vc1 = index.assume %vc11 [range(%vc11, 0, 3448)] : index
%vr1 = vector.load %vv[%bj, %vc1] : view<[%tokens]x3456xf32> -> vector<8xf32>
%pd1 = vector.mulf %q1, %kr1 : vector<8xf32>
%ps1 = vector.reduce<addf> %pd1, %zf : vector<8xf32>, f32
%kc20 = index.constant 16 : index
%kc21 = index.add %kcol, %kc20 : index
%kc2 = index.assume %kc21 [range(%kc21, 0, 3448)] : index
%kr2 = vector.load %kv[%bj, %kc2] : view<[%tokens]x3456xf32> -> vector<8xf32>
%vc21 = index.add %vcol, %kc20 : index
%vc2 = index.assume %vc21 [range(%vc21, 0, 3448)] : index
%vr2 = vector.load %vv[%bj, %vc2] : view<[%tokens]x3456xf32> -> vector<8xf32>
%pd2 = vector.mulf %q2, %kr2 : vector<8xf32>
%ps2 = vector.reduce<addf> %pd2, %zf : vector<8xf32>, f32
%kc30 = index.constant 24 : index
%kc31 = index.add %kcol, %kc30 : index
%kc3 = index.assume %kc31 [range(%kc31, 0, 3448)] : index
%kr3 = vector.load %kv[%bj, %kc3] : view<[%tokens]x3456xf32> -> vector<8xf32>
%vc31 = index.add %vcol, %kc30 : index
%vc3 = index.assume %vc31 [range(%vc31, 0, 3448)] : index
%vr3 = vector.load %vv[%bj, %vc3] : view<[%tokens]x3456xf32> -> vector<8xf32>
%pd3 = vector.mulf %q3, %kr3 : vector<8xf32>
%ps3 = vector.reduce<addf> %pd3, %zf : vector<8xf32>, f32
%kc40 = index.constant 32 : index
%kc41 = index.add %kcol, %kc40 : index
%kc4 = index.assume %kc41 [range(%kc41, 0, 3448)] : index
%kr4 = vector.load %kv[%bj, %kc4] : view<[%tokens]x3456xf32> -> vector<8xf32>
%vc41 = index.add %vcol, %kc40 : index
%vc4 = index.assume %vc41 [range(%vc41, 0, 3448)] : index
%vr4 = vector.load %vv[%bj, %vc4] : view<[%tokens]x3456xf32> -> vector<8xf32>
%pd4 = vector.mulf %q4, %kr4 : vector<8xf32>
%ps4 = vector.reduce<addf> %pd4, %zf : vector<8xf32>, f32
%kc50 = index.constant 40 : index
%kc51 = index.add %kcol, %kc50 : index
%kc5 = index.assume %kc51 [range(%kc51, 0, 3448)] : index
%kr5 = vector.load %kv[%bj, %kc5] : view<[%tokens]x3456xf32> -> vector<8xf32>
%vc51 = index.add %vcol, %kc50 : index
%vc5 = index.assume %vc51 [range(%vc51, 0, 3448)] : index
%vr5 = vector.load %vv[%bj, %vc5] : view<[%tokens]x3456xf32> -> vector<8xf32>
%pd5 = vector.mulf %q5, %kr5 : vector<8xf32>
%ps5 = vector.reduce<addf> %pd5, %zf : vector<8xf32>, f32
%kc60 = index.constant 48 : index
%kc61 = index.add %kcol, %kc60 : index
%kc6 = index.assume %kc61 [range(%kc61, 0, 3448)] : index
%kr6 = vector.load %kv[%bj, %kc6] : view<[%tokens]x3456xf32> -> vector<8xf32>
%vc61 = index.add %vcol, %kc60 : index
%vc6 = index.assume %vc61 [range(%vc61, 0, 3448)] : index
%vr6 = vector.load %vv[%bj, %vc6] : view<[%tokens]x3456xf32> -> vector<8xf32>
%pd6 = vector.mulf %q6, %kr6 : vector<8xf32>
%ps6 = vector.reduce<addf> %pd6, %zf : vector<8xf32>, f32
%kc70 = index.constant 56 : index
%kc71 = index.add %kcol, %kc70 : index
%kc7 = index.assume %kc71 [range(%kc71, 0, 3448)] : index
%kr7 = vector.load %kv[%bj, %kc7] : view<[%tokens]x3456xf32> -> vector<8xf32>
%vc71 = index.add %vcol, %kc70 : index
%vc7 = index.assume %vc71 [range(%vc71, 0, 3448)] : index
%vr7 = vector.load %vv[%bj, %vc7] : view<[%tokens]x3456xf32> -> vector<8xf32>
%pd7 = vector.mulf %q7, %kr7 : vector<8xf32>
%ps7 = vector.reduce<addf> %pd7, %zf : vector<8xf32>, f32
%kc80 = index.constant 64 : index
%kc81 = index.add %kcol, %kc80 : index
%kc8 = index.assume %kc81 [range(%kc81, 0, 3448)] : index
%kr8 = vector.load %kv[%bj, %kc8] : view<[%tokens]x3456xf32> -> vector<8xf32>
%vc81 = index.add %vcol, %kc80 : index
%vc8 = index.assume %vc81 [range(%vc81, 0, 3448)] : index
%vr8 = vector.load %vv[%bj, %vc8] : view<[%tokens]x3456xf32> -> vector<8xf32>
%pd8 = vector.mulf %q8, %kr8 : vector<8xf32>
%ps8 = vector.reduce<addf> %pd8, %zf : vector<8xf32>, f32
%ss1 = scalar.addf %ps0, %ps1 : f32
%ss2 = scalar.addf %ss1, %ps2 : f32
%ss3 = scalar.addf %ss2, %ps3 : f32
%ss4 = scalar.addf %ss3, %ps4 : f32
%ss5 = scalar.addf %ss4, %ps5 : f32
%ss6 = scalar.addf %ss5, %ps6 : f32
%ss7 = scalar.addf %ss6, %ps7 : f32
%ss8 = scalar.addf %ss7, %ps8 : f32
%s = scalar.addf %ss8, %zf : f32
%mx = scalar.maxnumf %m_c, %s : f32
%dm = scalar.subf %m_c, %mx : f32
%corr = scalar.expf<afn> %dm : f32
%ds = scalar.subf %s, %mx : f32
%p = scalar.expf<afn> %ds : f32
%lc = scalar.mulf %l_c, %corr : f32
%ln = scalar.addf %lc, %p : f32
%corrv = vector.splat %corr : vector<8xf32>
%pv = vector.splat %p : vector<8xf32>
%ac0 = vector.mulf %acc0_c, %corrv : vector<8xf32>
%an0 = vector.fmaf %vr0, %pv, %ac0 : vector<8xf32>
%ac1 = vector.mulf %acc1_c, %corrv : vector<8xf32>
%an1 = vector.fmaf %vr1, %pv, %ac1 : vector<8xf32>
%ac2 = vector.mulf %acc2_c, %corrv : vector<8xf32>
%an2 = vector.fmaf %vr2, %pv, %ac2 : vector<8xf32>
%ac3 = vector.mulf %acc3_c, %corrv : vector<8xf32>
%an3 = vector.fmaf %vr3, %pv, %ac3 : vector<8xf32>
%ac4 = vector.mulf %acc4_c, %corrv : vector<8xf32>
%an4 = vector.fmaf %vr4, %pv, %ac4 : vector<8xf32>
%ac5 = vector.mulf %acc5_c, %corrv : vector<8xf32>
%an5 = vector.fmaf %vr5, %pv, %ac5 : vector<8xf32>
%ac6 = vector.mulf %acc6_c, %corrv : vector<8xf32>
%an6 = vector.fmaf %vr6, %pv, %ac6 : vector<8xf32>
%ac7 = vector.mulf %acc7_c, %corrv : vector<8xf32>
%an7 = vector.fmaf %vr7, %pv, %ac7 : vector<8xf32>
%ac8 = vector.mulf %acc8_c, %corrv : vector<8xf32>
%an8 = vector.fmaf %vr8, %pv, %ac8 : vector<8xf32>
scf.yield %mx, %ln, %an0, %an1, %an2, %an3, %an4, %an5, %an6, %an7, %an8 : f32, f32, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>
} else {
scf.yield %m_c, %l_c, %acc0_c, %acc1_c, %acc2_c, %acc3_c, %acc4_c, %acc5_c, %acc6_c, %acc7_c, %acc8_c : f32, f32, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>
}
scf.yield %m_n, %l_n, %acc0_n, %acc1_n, %acc2_n, %acc3_n, %acc4_n, %acc5_n, %acc6_n, %acc7_n, %acc8_n : f32, f32, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>
}
%mg = kernel.subgroup.reduce<maxnumf> %m_l : f32
%dml = scalar.subf %m_l, %mg : f32
%cl = scalar.expf<afn> %dml : f32
%lw = scalar.mulf %l_l, %cl : f32
%lg = kernel.subgroup.reduce<addf> %lw : f32
%one = scalar.constant 1.0 : f32
%invl = scalar.divf %one, %lg : f32
%clv = vector.splat %cl : vector<8xf32>
%stage_bytes = index.constant 9216 : offset
%stage = buffer.alloca<workgroup> align(16) %stage_bytes : buffer
%st = buffer.view %stage[%zero] : buffer -> view<32x72xf32>
%aw0 = vector.mulf %acc0_l, %clv : vector<8xf32>
%sc0 = index.constant 0 : index
vector.store %aw0, %st[%lane, %sc0] : vector<8xf32>, view<32x72xf32>
%aw1 = vector.mulf %acc1_l, %clv : vector<8xf32>
%sc1 = index.constant 8 : index
vector.store %aw1, %st[%lane, %sc1] : vector<8xf32>, view<32x72xf32>
%aw2 = vector.mulf %acc2_l, %clv : vector<8xf32>
%sc2 = index.constant 16 : index
vector.store %aw2, %st[%lane, %sc2] : vector<8xf32>, view<32x72xf32>
%aw3 = vector.mulf %acc3_l, %clv : vector<8xf32>
%sc3 = index.constant 24 : index
vector.store %aw3, %st[%lane, %sc3] : vector<8xf32>, view<32x72xf32>
%aw4 = vector.mulf %acc4_l, %clv : vector<8xf32>
%sc4 = index.constant 32 : index
vector.store %aw4, %st[%lane, %sc4] : vector<8xf32>, view<32x72xf32>
%aw5 = vector.mulf %acc5_l, %clv : vector<8xf32>
%sc5 = index.constant 40 : index
vector.store %aw5, %st[%lane, %sc5] : vector<8xf32>, view<32x72xf32>
%aw6 = vector.mulf %acc6_l, %clv : vector<8xf32>
%sc6 = index.constant 48 : index
vector.store %aw6, %st[%lane, %sc6] : vector<8xf32>, view<32x72xf32>
%aw7 = vector.mulf %acc7_l, %clv : vector<8xf32>
%sc7 = index.constant 56 : index
vector.store %aw7, %st[%lane, %sc7] : vector<8xf32>, view<32x72xf32>
%aw8 = vector.mulf %acc8_l, %clv : vector<8xf32>
%sc8 = index.constant 64 : index
vector.store %aw8, %st[%lane, %sc8] : vector<8xf32>, view<32x72xf32>
kernel.barrier<workgroup> scope(workgroup) ordering(acq_rel)
%ov = buffer.view %o[%zero] : buffer -> view<[%tokens]x1152xf32>
%ocol0 = index.mul %h, %cD : index
scf.for %col = [%lane to %cD step %c32] {
%bcol = index.assume %col [lt(%col, %cD)] : index
%tot = scf.for %r = [%c0 to %c32 step %c1](%a = %zf : f32) -> (f32) {
%br = index.assume %r [range(%r, 0, 31)] : index
%x = view.load %st[%br, %bcol] : view<32x72xf32> -> f32
%na = scalar.addf %a, %x : f32
scf.yield %na : f32
}
%res = scalar.mulf %tot, %invl : f32
%oc0 = index.add %ocol0, %bcol : index
%oc = index.assume %oc0 [lt(%oc0, 1152)] : index
view.store %res, %ov[%t, %oc] : f32, view<[%tokens]x1152xf32>
}
kernel.return
}
kernel.def export("v_gemm_f16_store_1152x1152") @v_gemm_f16_store_1152x1152(%rows: index, %n: index) {
%c1 = index.constant 1 : index
%c64 = index.constant 64 : index
%c256 = index.constant 256 : index
%row_tiles = index.div %rows, %c64 : index
%n_const = index.constant 1152 : index
%out_tiles = index.div %n_const, %c64 : index
kernel.launch.config workgroups(%row_tiles, %out_tiles, %c1) workgroup_size(%c256, %c1, %c1) : index
} launch(%rows_arg: index, %k_arg: index, %n_arg: index, %x: buffer, %scales: buffer, %w: buffer, %y: buffer) {
%rows = index.assume %rows_arg [range(%rows_arg, 64, 65536), multiple_of(%rows_arg, 64)] : index
%k = index.constant 1152 : index
%n = index.constant 1152 : index
%origin = index.constant 0 : offset
%two = index.constant 2 : offset
%sixteen = index.constant 16 : offset
%block_bytes = index.constant 64 : offset
%c0 = index.constant 0 : index
%c1 = index.constant 1 : index
%c2 = index.constant 2 : index
%c4 = index.constant 4 : index
%c8 = index.constant 8 : index
%c16 = index.constant 16 : index
%c32 = index.constant 32 : index
%c64 = index.constant 64 : index
%c256 = index.constant 256 : index
%nibble_mask = vector.constant 252645135 : vector<4xi32>
%stage_bytes = index.constant 9216 : offset
%row_tile_count = index.div %rows, %c64 : index
%out_tile_count = index.div %n, %c64 : index
%row_group_id = kernel.workgroup.id<x> : index
%out_group_id = kernel.workgroup.id<y> : index
%row_group = index.assume %row_group_id [lt(%row_group_id, %row_tile_count)] : index
%out_group = index.assume %out_group_id [lt(%out_group_id, %out_tile_count)] : index
%workitem_id = kernel.workitem.id<x> : index
%workitem = index.assume %workitem_id [range(%workitem_id, 0, 255)] : index
%subgroup = kernel.subgroup.id : index
%lane = kernel.subgroup.lane.id : index
%lane_row = index.rem %lane, %c16 : index
%bounded_subgroup = index.assume %subgroup [range(%subgroup, 0, 7)] : index
%wave_row_group = index.div %bounded_subgroup, %c2 : index
%wave_out_group = index.rem %bounded_subgroup, %c2 : index
%wave_row = index.mul %wave_row_group, %c16 : index
%wave_out = index.mul %wave_out_group, %c32 : index
%w_row0 = index.add %wave_out, %lane_row : index
%w_row1 = index.add %w_row0, %c16 : index
%row_origin = index.mul %row_group, %c64 : index
%out_origin = index.mul %out_group, %c64 : index
%result_row = index.add %row_origin, %wave_row : index
%result_col0 = index.add %out_origin, %wave_out : index
%result_col1 = index.add %result_col0, %c16 : index
%blocks = index.div %k, %c32 : index
%row_bytes = index.scale %blocks, %block_bytes : index, offset -> offset
%x_view = buffer.view %x[%origin] : buffer -> view<[%rows]x[%k]xf16>
// 64x64 output tile with the full row stride n.
%y_layout = encoding.layout.strided [%n, %c1] : encoding<layout>
%tile_first = index.madd %row_origin, %n, %out_origin : index
%four = index.constant 4 : offset
%tile_offset = index.scale %tile_first, %four : index, offset -> offset
%y_view = buffer.view %y[%tile_offset] : buffer -> view<64x64xf32, %y_layout>
%bounded_wave_row = index.assume %wave_row [range(%wave_row, 0, 48)] : index
%bounded_wave_out = index.assume %wave_out [range(%wave_out, 0, 32)] : index
%wave_out1 = index.add %bounded_wave_out, %c16 : index
%x_stage = buffer.alloca<workgroup> align(16) %stage_bytes : buffer
%x_stage_view = buffer.view %x_stage[%origin] : buffer -> view<64x72xf16>
%w_stage = buffer.alloca<workgroup> align(16) %stage_bytes : buffer
%w_stage_view = buffer.view %w_stage[%origin] : buffer -> view<64x72xf16>
// Thread t stages 16 values: row t/4, columns (t%4)*16.
%stage_row = index.div %workitem, %c4 : index
%stage_quarter = index.rem %workitem, %c4 : index
%stage_col = index.mul %stage_quarter, %c16 : index
%x_row = index.add %row_origin, %stage_row : index
%bounded_x_row = index.assume %x_row [lt(%x_row, %rows)] : index
%w_out_row = index.add %out_origin, %stage_row : index
%bounded_w_out_row = index.assume %w_out_row [lt(%w_out_row, %n)] : index
%w_row_origin = index.scale %bounded_w_out_row, %row_bytes : index, offset -> offset
%zero8 = vector.constant 0.0 : vector<8xf32>
%init = vector.fragment<init> %zero8 shape [%c16, %c16] : vector<8xf32>
%k_steps = index.div %k, %c64 : index
%s_view = buffer.view %scales[%origin] : buffer -> view<[%rows]x2xf32>
%x_scale = view.load %s_view[%bounded_x_row, %c0] : view<[%rows]x2xf32> -> f32
%x_scales = vector.splat %x_scale : vector<16xf32>
%a0_raw, %a1_raw = scf.for %step = [%c0 to %k_steps step %c1](%a0_acc = %init : vector<8xf32>, %a1_acc = %init : vector<8xf32>) -> (vector<8xf32>, vector<8xf32>) {
%k_origin = index.mul %step, %c64 : index
%x_col = index.add %k_origin, %stage_col : index
%x_half = vector.load %x_view[%bounded_x_row, %x_col] : view<[%rows]x[%k]xf16> -> vector<16xf16>
vector.store %x_half, %x_stage_view[%stage_row, %stage_col] : vector<16xf16>, view<64x72xf16>
%w_col = index.add %k_origin, %stage_col : index
%deq0_two_b = index.constant 2 : offset
%deq0_col_bytes = index.scale %w_col, %deq0_two_b : index, offset -> offset
%deq0_origin = index.add %w_row_origin, %deq0_col_bytes : offset
%deq0_view = buffer.view %w[%deq0_origin] : buffer -> view<16xf16>
%deq0 = vector.load %deq0_view[0] : view<16xf16> -> vector<16xf16>
vector.store %deq0, %w_stage_view[%stage_row, %stage_col] : vector<16xf16>, view<64x72xf16>
kernel.barrier<workgroup> scope(workgroup) ordering(acq_rel)
%fc0 = index.constant 0 : index
%lhs0 = vector.fragment.load<lhs> %x_stage_view[%wave_row, %fc0] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16>
%rv000 = vector.load %w_stage_view[%w_row0, %fc0] : view<64x72xf16> -> vector<16xf16>
%rhs000 = vector.fragment<rhs> %rv000 shape [%c16, %c16] : vector<16xf16>
%na0_0 = vector.mma %lhs0, %rhs000, %a0_acc : vector<16xf16>, vector<16xf16>, vector<8xf32>
%rv010 = vector.load %w_stage_view[%w_row1, %fc0] : view<64x72xf16> -> vector<16xf16>
%rhs010 = vector.fragment<rhs> %rv010 shape [%c16, %c16] : vector<16xf16>
%na1_0 = vector.mma %lhs0, %rhs010, %a1_acc : vector<16xf16>, vector<16xf16>, vector<8xf32>
%fc1 = index.constant 16 : index
%lhs1 = vector.fragment.load<lhs> %x_stage_view[%wave_row, %fc1] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16>
%rv001 = vector.load %w_stage_view[%w_row0, %fc1] : view<64x72xf16> -> vector<16xf16>
%rhs001 = vector.fragment<rhs> %rv001 shape [%c16, %c16] : vector<16xf16>
%na0_1 = vector.mma %lhs1, %rhs001, %na0_0 : vector<16xf16>, vector<16xf16>, vector<8xf32>
%rv011 = vector.load %w_stage_view[%w_row1, %fc1] : view<64x72xf16> -> vector<16xf16>
%rhs011 = vector.fragment<rhs> %rv011 shape [%c16, %c16] : vector<16xf16>
%na1_1 = vector.mma %lhs1, %rhs011, %na1_0 : vector<16xf16>, vector<16xf16>, vector<8xf32>
%fc2 = index.constant 32 : index
%lhs2 = vector.fragment.load<lhs> %x_stage_view[%wave_row, %fc2] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16>
%rv002 = vector.load %w_stage_view[%w_row0, %fc2] : view<64x72xf16> -> vector<16xf16>
%rhs002 = vector.fragment<rhs> %rv002 shape [%c16, %c16] : vector<16xf16>
%na0_2 = vector.mma %lhs2, %rhs002, %na0_1 : vector<16xf16>, vector<16xf16>, vector<8xf32>
%rv012 = vector.load %w_stage_view[%w_row1, %fc2] : view<64x72xf16> -> vector<16xf16>
%rhs012 = vector.fragment<rhs> %rv012 shape [%c16, %c16] : vector<16xf16>
%na1_2 = vector.mma %lhs2, %rhs012, %na1_1 : vector<16xf16>, vector<16xf16>, vector<8xf32>
%fc3 = index.constant 48 : index
%lhs3 = vector.fragment.load<lhs> %x_stage_view[%wave_row, %fc3] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16>
%rv003 = vector.load %w_stage_view[%w_row0, %fc3] : view<64x72xf16> -> vector<16xf16>
%rhs003 = vector.fragment<rhs> %rv003 shape [%c16, %c16] : vector<16xf16>
%na0_3 = vector.mma %lhs3, %rhs003, %na0_2 : vector<16xf16>, vector<16xf16>, vector<8xf32>
%rv013 = vector.load %w_stage_view[%w_row1, %fc3] : view<64x72xf16> -> vector<16xf16>
%rhs013 = vector.fragment<rhs> %rv013 shape [%c16, %c16] : vector<16xf16>
%na1_3 = vector.mma %lhs3, %rhs013, %na1_2 : vector<16xf16>, vector<16xf16>, vector<8xf32>
kernel.barrier<workgroup> scope(workgroup) ordering(acq_rel)
scf.yield %na0_3, %na1_3 : vector<8xf32>, vector<8xf32>
}
%inv_bytes = index.constant 4096 : offset
%inv_tile = buffer.alloca<workgroup> align(16) %inv_bytes : buffer
%inv_view = buffer.view %inv_tile[%origin] : buffer -> view<64x16xf32>
%writer = index.cmp ult, %workitem, %c64 : index
scf.if %writer {
%inv_row_value = index.add %row_origin, %workitem : index
%inv_row = index.assume %inv_row_value [lt(%inv_row_value, %rows)] : index
%inverse = view.load %s_view[%inv_row, %c1] : view<[%rows]x2xf32> -> f32
%inverse16 = vector.splat %inverse : vector<16xf32>
%bw = index.assume %workitem [range(%workitem, 0, 63)] : index
vector.store %inverse16, %inv_view[%bw, %c0] : vector<16xf32>, view<64x16xf32>
}
kernel.barrier<workgroup> scope(workgroup) ordering(acq_rel)
%inv_frag = vector.fragment.load<result> %inv_view[%wave_row, %c0] shape [%c16, %c16] : view<64x16xf32> -> vector<8xf32>
%a0 = vector.mulf %a0_raw, %inv_frag : vector<8xf32>
%a1 = vector.mulf %a1_raw, %inv_frag : vector<8xf32>
vector.fragment.store<result> %a0, %y_view[%bounded_wave_row, %bounded_wave_out] shape [%c16, %c16] : vector<8xf32>, view<64x64xf32, %y_layout>
vector.fragment.store<result> %a1, %y_view[%bounded_wave_row, %wave_out1] shape [%c16, %c16] : vector<8xf32>, view<64x64xf32, %y_layout>
kernel.return
}
kernel.def export("v_bias_none_1152") @v_bias_none_1152(%rows: index) {
%c1 = index.constant 1 : index
%c256 = index.constant 256 : index
%cols = index.constant 1152 : index
%cg = index.constant 5 : index
kernel.launch.config workgroups(%rows, %cg, %c1) workgroup_size(%c256, %c1, %c1) : index
} launch(%rows_arg: index, %x: buffer, %bias: buffer, %y: buffer) {
%rows = index.assume %rows_arg [range(%rows_arg, 1, 65536)] : index
%c256 = index.constant 256 : index
%cols = index.constant 1152 : index
%row_id = kernel.workgroup.id<x> : index
%row = index.assume %row_id [lt(%row_id, %rows)] : index
%cg = kernel.workgroup.id<y> : index
%lid = kernel.workitem.id<x> : index
%c = index.madd %cg, %c256, %lid : index
%live = index.cmp ult, %c, %cols : index
scf.if %live {
%bc = index.assume %c [lt(%c, %cols)] : index
%zero = index.constant 0 : offset
%zf = scalar.constant 0.0 : f32
%xv = buffer.view %x[%zero] : buffer -> view<[%rows]x1152xf32>
%bv = buffer.view %bias[%zero] : buffer -> view<1152xf32>
%x0 = view.load %xv[%row, %bc] : view<[%rows]x1152xf32> -> f32
%b0 = view.load %bv[%bc] : view<1152xf32> -> f32
%a = scalar.addf %x0, %b0 : f32
%r = scalar.addf %a, %zf : f32
%yv = buffer.view %y[%zero] : buffer -> view<[%rows]x1152xf32>
view.store %r, %yv[%row, %bc] : f32, view<[%rows]x1152xf32>
}
kernel.return
}
kernel.def export("v_ew_add") @v_ew_add(%n: index) {
%c1 = index.constant 1 : index
%c256 = index.constant 256 : index
%c255 = index.constant 255 : index
%n_up = index.add %n, %c255 : index
%groups = index.div %n_up, %c256 : index
kernel.launch.config workgroups(%groups, %c1, %c1) workgroup_size(%c256, %c1, %c1) : index
} launch(%n_arg: index, %x: buffer, %b: buffer, %y: buffer) {
%n = index.assume %n_arg [range(%n_arg, 1, 268435456)] : index
%c256 = index.constant 256 : index
%gid = kernel.workgroup.id<x> : index
%lid = kernel.workitem.id<x> : index
%i = index.madd %gid, %c256, %lid : index
%live = index.cmp ult, %i, %n : index
scf.if %live {
%bi = index.assume %i [lt(%i, %n)] : index
%zero = index.constant 0 : offset
%xv = buffer.view %x[%zero] : buffer -> view<[%n]xf32>
%a = view.load %xv[%bi] : view<[%n]xf32> -> f32
%bview = buffer.view %b[%zero] : buffer -> view<[%n]xf32>
%bv = view.load %bview[%bi] : view<[%n]xf32> -> f32
%r = scalar.addf %a, %bv : f32
%yv = buffer.view %y[%zero] : buffer -> view<[%n]xf32>
view.store %r, %yv[%bi] : f32, view<[%n]xf32>
}
kernel.return
}
kernel.def export("v_gemm_any_f16_1152x4304") @v_gemm_any_f16_1152x4304(%rows: index, %n: index) {
%c1 = index.constant 1 : index
%c15 = index.constant 15 : index
%c16 = index.constant 16 : index
%c63 = index.constant 63 : index
%c64 = index.constant 64 : index
%rp = index.add %rows, %c15 : index
%row_tiles = index.div %rp, %c16 : index
%ncst = index.constant 4304 : index
%np = index.add %ncst, %c63 : index
%col_tiles = index.div %np, %c64 : index
kernel.launch.config workgroups(%row_tiles, %col_tiles, %c1) workgroup_size(%c64, %c1, %c1) : index
} launch(%rows_arg: index, %k_arg: index, %n_arg: index, %x: buffer, %scales: buffer, %w: buffer, %y: buffer) {
%rows = index.assume %rows_arg [range(%rows_arg, 1, 65536)] : index
%zero = index.constant 0 : offset
%c0 = index.constant 0 : index
%c1 = index.constant 1 : index
%c8 = index.constant 8 : index
%c16 = index.constant 16 : index
%c64 = index.constant 64 : index
%ck = index.constant 1152 : index
%cn = index.constant 4304 : index
%zf = scalar.constant 0.0 : f32
%rt_id = kernel.workgroup.id<x> : index
%ct_id = kernel.workgroup.id<y> : index
%lid = kernel.workitem.id<x> : index
%lane = index.assume %lid [range(%lid, 0, 63)] : index
%col0 = index.mul %ct_id, %c64 : index
%col = index.add %col0, %lane : index
%col_live = index.cmp ult, %col, %cn : index
%nlast = index.constant 4303 : index
%colc = index.min %col, %nlast : index
%wcol = index.assume %colc [range(%colc, 0, 4303)] : index
%row0 = index.mul %rt_id, %c16 : index
%x_view = buffer.view %x[%zero] : buffer -> view<[%rows]x1152xf16>
%w_view = buffer.view %w[%zero] : buffer -> view<4304x1152xf16>
%s_view = buffer.view %scales[%zero] : buffer -> view<[%rows]x2xf32>
%rlast = index.sub %rows, %c1 : index
%r0, %r1, %r2, %r3, %r4, %r5, %r6, %r7, %r8, %r9, %r10, %r11, %r12, %r13, %r14, %r15 = scf.for %kk = [%c0 to %ck step %c8](%a0 = %zf : f32, %a1 = %zf : f32, %a2 = %zf : f32, %a3 = %zf : f32, %a4 = %zf : f32, %a5 = %zf : f32, %a6 = %zf : f32, %a7 = %zf : f32, %a8 = %zf : f32, %a9 = %zf : f32, %a10 = %zf : f32, %a11 = %zf : f32, %a12 = %zf : f32, %a13 = %zf : f32, %a14 = %zf : f32, %a15 = %zf : f32) -> (f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32) {
%bk = index.assume %kk [range(%kk, 0, 1144), multiple_of(%kk, 8)] : index
%wh = vector.load %w_view[%wcol, %bk] : view<4304x1152xf16> -> vector<8xf16>
%wv = vector.extf %wh : vector<8xf16> to vector<8xf32>
%ri0 = index.constant 0 : index
%rr0 = index.add %row0, %ri0 : index
%rc0 = index.min %rr0, %rlast : index
%rb0 = index.assume %rc0 [lt(%rc0, %rows)] : index
%xh0 = vector.load %x_view[%rb0, %bk] : view<[%rows]x1152xf16> -> vector<8xf16>
%xv0 = vector.extf %xh0 : vector<8xf16> to vector<8xf32>
%d0 = vector.dotf<reassoc|contract> %wv, %xv0, %zf : vector<8xf32>, vector<8xf32>, f32
%n0 = scalar.addf %a0, %d0 : f32
%ri1 = index.constant 1 : index
%rr1 = index.add %row0, %ri1 : index
%rc1 = index.min %rr1, %rlast : index
%rb1 = index.assume %rc1 [lt(%rc1, %rows)] : index
%xh1 = vector.load %x_view[%rb1, %bk] : view<[%rows]x1152xf16> -> vector<8xf16>
%xv1 = vector.extf %xh1 : vector<8xf16> to vector<8xf32>
%d1 = vector.dotf<reassoc|contract> %wv, %xv1, %zf : vector<8xf32>, vector<8xf32>, f32
%n1 = scalar.addf %a1, %d1 : f32
%ri2 = index.constant 2 : index
%rr2 = index.add %row0, %ri2 : index
%rc2 = index.min %rr2, %rlast : index
%rb2 = index.assume %rc2 [lt(%rc2, %rows)] : index
%xh2 = vector.load %x_view[%rb2, %bk] : view<[%rows]x1152xf16> -> vector<8xf16>
%xv2 = vector.extf %xh2 : vector<8xf16> to vector<8xf32>
%d2 = vector.dotf<reassoc|contract> %wv, %xv2, %zf : vector<8xf32>, vector<8xf32>, f32
%n2 = scalar.addf %a2, %d2 : f32
%ri3 = index.constant 3 : index
%rr3 = index.add %row0, %ri3 : index
%rc3 = index.min %rr3, %rlast : index
%rb3 = index.assume %rc3 [lt(%rc3, %rows)] : index
%xh3 = vector.load %x_view[%rb3, %bk] : view<[%rows]x1152xf16> -> vector<8xf16>
%xv3 = vector.extf %xh3 : vector<8xf16> to vector<8xf32>
%d3 = vector.dotf<reassoc|contract> %wv, %xv3, %zf : vector<8xf32>, vector<8xf32>, f32
%n3 = scalar.addf %a3, %d3 : f32
%ri4 = index.constant 4 : index
%rr4 = index.add %row0, %ri4 : index
%rc4 = index.min %rr4, %rlast : index
%rb4 = index.assume %rc4 [lt(%rc4, %rows)] : index
%xh4 = vector.load %x_view[%rb4, %bk] : view<[%rows]x1152xf16> -> vector<8xf16>
%xv4 = vector.extf %xh4 : vector<8xf16> to vector<8xf32>
%d4 = vector.dotf<reassoc|contract> %wv, %xv4, %zf : vector<8xf32>, vector<8xf32>, f32
%n4 = scalar.addf %a4, %d4 : f32
%ri5 = index.constant 5 : index
%rr5 = index.add %row0, %ri5 : index
%rc5 = index.min %rr5, %rlast : index
%rb5 = index.assume %rc5 [lt(%rc5, %rows)] : index
%xh5 = vector.load %x_view[%rb5, %bk] : view<[%rows]x1152xf16> -> vector<8xf16>
%xv5 = vector.extf %xh5 : vector<8xf16> to vector<8xf32>
%d5 = vector.dotf<reassoc|contract> %wv, %xv5, %zf : vector<8xf32>, vector<8xf32>, f32
%n5 = scalar.addf %a5, %d5 : f32
%ri6 = index.constant 6 : index
%rr6 = index.add %row0, %ri6 : index
%rc6 = index.min %rr6, %rlast : index
%rb6 = index.assume %rc6 [lt(%rc6, %rows)] : index
%xh6 = vector.load %x_view[%rb6, %bk] : view<[%rows]x1152xf16> -> vector<8xf16>
%xv6 = vector.extf %xh6 : vector<8xf16> to vector<8xf32>
%d6 = vector.dotf<reassoc|contract> %wv, %xv6, %zf : vector<8xf32>, vector<8xf32>, f32
%n6 = scalar.addf %a6, %d6 : f32
%ri7 = index.constant 7 : index
%rr7 = index.add %row0, %ri7 : index
%rc7 = index.min %rr7, %rlast : index
%rb7 = index.assume %rc7 [lt(%rc7, %rows)] : index
%xh7 = vector.load %x_view[%rb7, %bk] : view<[%rows]x1152xf16> -> vector<8xf16>
%xv7 = vector.extf %xh7 : vector<8xf16> to vector<8xf32>
%d7 = vector.dotf<reassoc|contract> %wv, %xv7, %zf : vector<8xf32>, vector<8xf32>, f32
%n7 = scalar.addf %a7, %d7 : f32
%ri8 = index.constant 8 : index
%rr8 = index.add %row0, %ri8 : index
%rc8 = index.min %rr8, %rlast : index
%rb8 = index.assume %rc8 [lt(%rc8, %rows)] : index
%xh8 = vector.load %x_view[%rb8, %bk] : view<[%rows]x1152xf16> -> vector<8xf16>
%xv8 = vector.extf %xh8 : vector<8xf16> to vector<8xf32>
%d8 = vector.dotf<reassoc|contract> %wv, %xv8, %zf : vector<8xf32>, vector<8xf32>, f32
%n8 = scalar.addf %a8, %d8 : f32
%ri9 = index.constant 9 : index
%rr9 = index.add %row0, %ri9 : index
%rc9 = index.min %rr9, %rlast : index
%rb9 = index.assume %rc9 [lt(%rc9, %rows)] : index
%xh9 = vector.load %x_view[%rb9, %bk] : view<[%rows]x1152xf16> -> vector<8xf16>
%xv9 = vector.extf %xh9 : vector<8xf16> to vector<8xf32>
%d9 = vector.dotf<reassoc|contract> %wv, %xv9, %zf : vector<8xf32>, vector<8xf32>, f32
%n9 = scalar.addf %a9, %d9 : f32
%ri10 = index.constant 10 : index
%rr10 = index.add %row0, %ri10 : index
%rc10 = index.min %rr10, %rlast : index
%rb10 = index.assume %rc10 [lt(%rc10, %rows)] : index
%xh10 = vector.load %x_view[%rb10, %bk] : view<[%rows]x1152xf16> -> vector<8xf16>
%xv10 = vector.extf %xh10 : vector<8xf16> to vector<8xf32>
%d10 = vector.dotf<reassoc|contract> %wv, %xv10, %zf : vector<8xf32>, vector<8xf32>, f32
%n10 = scalar.addf %a10, %d10 : f32
%ri11 = index.constant 11 : index
%rr11 = index.add %row0, %ri11 : index
%rc11 = index.min %rr11, %rlast : index
%rb11 = index.assume %rc11 [lt(%rc11, %rows)] : index
%xh11 = vector.load %x_view[%rb11, %bk] : view<[%rows]x1152xf16> -> vector<8xf16>
%xv11 = vector.extf %xh11 : vector<8xf16> to vector<8xf32>
%d11 = vector.dotf<reassoc|contract> %wv, %xv11, %zf : vector<8xf32>, vector<8xf32>, f32
%n11 = scalar.addf %a11, %d11 : f32
%ri12 = index.constant 12 : index
%rr12 = index.add %row0, %ri12 : index
%rc12 = index.min %rr12, %rlast : index
%rb12 = index.assume %rc12 [lt(%rc12, %rows)] : index
%xh12 = vector.load %x_view[%rb12, %bk] : view<[%rows]x1152xf16> -> vector<8xf16>
%xv12 = vector.extf %xh12 : vector<8xf16> to vector<8xf32>
%d12 = vector.dotf<reassoc|contract> %wv, %xv12, %zf : vector<8xf32>, vector<8xf32>, f32
%n12 = scalar.addf %a12, %d12 : f32
%ri13 = index.constant 13 : index
%rr13 = index.add %row0, %ri13 : index
%rc13 = index.min %rr13, %rlast : index
%rb13 = index.assume %rc13 [lt(%rc13, %rows)] : index
%xh13 = vector.load %x_view[%rb13, %bk] : view<[%rows]x1152xf16> -> vector<8xf16>
%xv13 = vector.extf %xh13 : vector<8xf16> to vector<8xf32>
%d13 = vector.dotf<reassoc|contract> %wv, %xv13, %zf : vector<8xf32>, vector<8xf32>, f32
%n13 = scalar.addf %a13, %d13 : f32
%ri14 = index.constant 14 : index
%rr14 = index.add %row0, %ri14 : index
%rc14 = index.min %rr14, %rlast : index
%rb14 = index.assume %rc14 [lt(%rc14, %rows)] : index
%xh14 = vector.load %x_view[%rb14, %bk] : view<[%rows]x1152xf16> -> vector<8xf16>
%xv14 = vector.extf %xh14 : vector<8xf16> to vector<8xf32>
%d14 = vector.dotf<reassoc|contract> %wv, %xv14, %zf : vector<8xf32>, vector<8xf32>, f32
%n14 = scalar.addf %a14, %d14 : f32
%ri15 = index.constant 15 : index
%rr15 = index.add %row0, %ri15 : index
%rc15 = index.min %rr15, %rlast : index
%rb15 = index.assume %rc15 [lt(%rc15, %rows)] : index
%xh15 = vector.load %x_view[%rb15, %bk] : view<[%rows]x1152xf16> -> vector<8xf16>
%xv15 = vector.extf %xh15 : vector<8xf16> to vector<8xf32>
%d15 = vector.dotf<reassoc|contract> %wv, %xv15, %zf : vector<8xf32>, vector<8xf32>, f32
%n15 = scalar.addf %a15, %d15 : f32
scf.yield %n0, %n1, %n2, %n3, %n4, %n5, %n6, %n7, %n8, %n9, %n10, %n11, %n12, %n13, %n14, %n15 : f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32
}
%y_view = buffer.view %y[%zero] : buffer -> view<[%rows]x4304xf32>
%ro0 = index.constant 0 : index
%orow0 = index.add %row0, %ro0 : index
%olive0 = index.cmp ult, %orow0, %rows : index
%ok0 = scalar.andi %olive0, %col_live : i1
scf.if %ok0 {
%ob0 = index.assume %orow0 [lt(%orow0, %rows)] : index
%inv0 = view.load %s_view[%ob0, %c1] : view<[%rows]x2xf32> -> f32
%val0 = scalar.mulf %r0, %inv0 : f32
view.store %val0, %y_view[%ob0, %wcol] : f32, view<[%rows]x4304xf32>
}
%ro1 = index.constant 1 : index
%orow1 = index.add %row0, %ro1 : index
%olive1 = index.cmp ult, %orow1, %rows : index
%ok1 = scalar.andi %olive1, %col_live : i1
scf.if %ok1 {
%ob1 = index.assume %orow1 [lt(%orow1, %rows)] : index
%inv1 = view.load %s_view[%ob1, %c1] : view<[%rows]x2xf32> -> f32
%val1 = scalar.mulf %r1, %inv1 : f32
view.store %val1, %y_view[%ob1, %wcol] : f32, view<[%rows]x4304xf32>
}
%ro2 = index.constant 2 : index
%orow2 = index.add %row0, %ro2 : index
%olive2 = index.cmp ult, %orow2, %rows : index
%ok2 = scalar.andi %olive2, %col_live : i1
scf.if %ok2 {
%ob2 = index.assume %orow2 [lt(%orow2, %rows)] : index
%inv2 = view.load %s_view[%ob2, %c1] : view<[%rows]x2xf32> -> f32
%val2 = scalar.mulf %r2, %inv2 : f32
view.store %val2, %y_view[%ob2, %wcol] : f32, view<[%rows]x4304xf32>
}
%ro3 = index.constant 3 : index
%orow3 = index.add %row0, %ro3 : index
%olive3 = index.cmp ult, %orow3, %rows : index
%ok3 = scalar.andi %olive3, %col_live : i1
scf.if %ok3 {
%ob3 = index.assume %orow3 [lt(%orow3, %rows)] : index
%inv3 = view.load %s_view[%ob3, %c1] : view<[%rows]x2xf32> -> f32
%val3 = scalar.mulf %r3, %inv3 : f32
view.store %val3, %y_view[%ob3, %wcol] : f32, view<[%rows]x4304xf32>
}
%ro4 = index.constant 4 : index
%orow4 = index.add %row0, %ro4 : index
%olive4 = index.cmp ult, %orow4, %rows : index
%ok4 = scalar.andi %olive4, %col_live : i1
scf.if %ok4 {
%ob4 = index.assume %orow4 [lt(%orow4, %rows)] : index
%inv4 = view.load %s_view[%ob4, %c1] : view<[%rows]x2xf32> -> f32
%val4 = scalar.mulf %r4, %inv4 : f32
view.store %val4, %y_view[%ob4, %wcol] : f32, view<[%rows]x4304xf32>
}
%ro5 = index.constant 5 : index
%orow5 = index.add %row0, %ro5 : index
%olive5 = index.cmp ult, %orow5, %rows : index
%ok5 = scalar.andi %olive5, %col_live : i1
scf.if %ok5 {
%ob5 = index.assume %orow5 [lt(%orow5, %rows)] : index
%inv5 = view.load %s_view[%ob5, %c1] : view<[%rows]x2xf32> -> f32
%val5 = scalar.mulf %r5, %inv5 : f32
view.store %val5, %y_view[%ob5, %wcol] : f32, view<[%rows]x4304xf32>
}
%ro6 = index.constant 6 : index
%orow6 = index.add %row0, %ro6 : index
%olive6 = index.cmp ult, %orow6, %rows : index
%ok6 = scalar.andi %olive6, %col_live : i1
scf.if %ok6 {
%ob6 = index.assume %orow6 [lt(%orow6, %rows)] : index
%inv6 = view.load %s_view[%ob6, %c1] : view<[%rows]x2xf32> -> f32
%val6 = scalar.mulf %r6, %inv6 : f32
view.store %val6, %y_view[%ob6, %wcol] : f32, view<[%rows]x4304xf32>
}
%ro7 = index.constant 7 : index
%orow7 = index.add %row0, %ro7 : index
%olive7 = index.cmp ult, %orow7, %rows : index
%ok7 = scalar.andi %olive7, %col_live : i1
scf.if %ok7 {
%ob7 = index.assume %orow7 [lt(%orow7, %rows)] : index
%inv7 = view.load %s_view[%ob7, %c1] : view<[%rows]x2xf32> -> f32
%val7 = scalar.mulf %r7, %inv7 : f32
view.store %val7, %y_view[%ob7, %wcol] : f32, view<[%rows]x4304xf32>
}
%ro8 = index.constant 8 : index
%orow8 = index.add %row0, %ro8 : index
%olive8 = index.cmp ult, %orow8, %rows : index
%ok8 = scalar.andi %olive8, %col_live : i1
scf.if %ok8 {
%ob8 = index.assume %orow8 [lt(%orow8, %rows)] : index
%inv8 = view.load %s_view[%ob8, %c1] : view<[%rows]x2xf32> -> f32
%val8 = scalar.mulf %r8, %inv8 : f32
view.store %val8, %y_view[%ob8, %wcol] : f32, view<[%rows]x4304xf32>
}
%ro9 = index.constant 9 : index
%orow9 = index.add %row0, %ro9 : index
%olive9 = index.cmp ult, %orow9, %rows : index
%ok9 = scalar.andi %olive9, %col_live : i1
scf.if %ok9 {
%ob9 = index.assume %orow9 [lt(%orow9, %rows)] : index
%inv9 = view.load %s_view[%ob9, %c1] : view<[%rows]x2xf32> -> f32
%val9 = scalar.mulf %r9, %inv9 : f32
view.store %val9, %y_view[%ob9, %wcol] : f32, view<[%rows]x4304xf32>
}
%ro10 = index.constant 10 : index
%orow10 = index.add %row0, %ro10 : index
%olive10 = index.cmp ult, %orow10, %rows : index
%ok10 = scalar.andi %olive10, %col_live : i1
scf.if %ok10 {
%ob10 = index.assume %orow10 [lt(%orow10, %rows)] : index
%inv10 = view.load %s_view[%ob10, %c1] : view<[%rows]x2xf32> -> f32
%val10 = scalar.mulf %r10, %inv10 : f32
view.store %val10, %y_view[%ob10, %wcol] : f32, view<[%rows]x4304xf32>
}
%ro11 = index.constant 11 : index
%orow11 = index.add %row0, %ro11 : index
%olive11 = index.cmp ult, %orow11, %rows : index
%ok11 = scalar.andi %olive11, %col_live : i1
scf.if %ok11 {
%ob11 = index.assume %orow11 [lt(%orow11, %rows)] : index
%inv11 = view.load %s_view[%ob11, %c1] : view<[%rows]x2xf32> -> f32
%val11 = scalar.mulf %r11, %inv11 : f32
view.store %val11, %y_view[%ob11, %wcol] : f32, view<[%rows]x4304xf32>
}
%ro12 = index.constant 12 : index
%orow12 = index.add %row0, %ro12 : index
%olive12 = index.cmp ult, %orow12, %rows : index
%ok12 = scalar.andi %olive12, %col_live : i1
scf.if %ok12 {
%ob12 = index.assume %orow12 [lt(%orow12, %rows)] : index
%inv12 = view.load %s_view[%ob12, %c1] : view<[%rows]x2xf32> -> f32
%val12 = scalar.mulf %r12, %inv12 : f32
view.store %val12, %y_view[%ob12, %wcol] : f32, view<[%rows]x4304xf32>
}
%ro13 = index.constant 13 : index
%orow13 = index.add %row0, %ro13 : index
%olive13 = index.cmp ult, %orow13, %rows : index
%ok13 = scalar.andi %olive13, %col_live : i1
scf.if %ok13 {
%ob13 = index.assume %orow13 [lt(%orow13, %rows)] : index
%inv13 = view.load %s_view[%ob13, %c1] : view<[%rows]x2xf32> -> f32
%val13 = scalar.mulf %r13, %inv13 : f32
view.store %val13, %y_view[%ob13, %wcol] : f32, view<[%rows]x4304xf32>
}
%ro14 = index.constant 14 : index
%orow14 = index.add %row0, %ro14 : index
%olive14 = index.cmp ult, %orow14, %rows : index
%ok14 = scalar.andi %olive14, %col_live : i1
scf.if %ok14 {
%ob14 = index.assume %orow14 [lt(%orow14, %rows)] : index
%inv14 = view.load %s_view[%ob14, %c1] : view<[%rows]x2xf32> -> f32
%val14 = scalar.mulf %r14, %inv14 : f32
view.store %val14, %y_view[%ob14, %wcol] : f32, view<[%rows]x4304xf32>
}
%ro15 = index.constant 15 : index
%orow15 = index.add %row0, %ro15 : index
%olive15 = index.cmp ult, %orow15, %rows : index
%ok15 = scalar.andi %olive15, %col_live : i1
scf.if %ok15 {
%ob15 = index.assume %orow15 [lt(%orow15, %rows)] : index
%inv15 = view.load %s_view[%ob15, %c1] : view<[%rows]x2xf32> -> f32
%val15 = scalar.mulf %r15, %inv15 : f32
view.store %val15, %y_view[%ob15, %wcol] : f32, view<[%rows]x4304xf32>
}
kernel.return
}
kernel.def export("v_bias_gelu_tanh_4304") @v_bias_gelu_tanh_4304(%rows: index) {
%c1 = index.constant 1 : index
%c256 = index.constant 256 : index
%cols = index.constant 4304 : index
%cg = index.constant 17 : index
kernel.launch.config workgroups(%rows, %cg, %c1) workgroup_size(%c256, %c1, %c1) : index
} launch(%rows_arg: index, %x: buffer, %bias: buffer, %y: buffer) {
%rows = index.assume %rows_arg [range(%rows_arg, 1, 65536)] : index
%c256 = index.constant 256 : index
%cols = index.constant 4304 : index
%row_id = kernel.workgroup.id<x> : index
%row = index.assume %row_id [lt(%row_id, %rows)] : index
%cg = kernel.workgroup.id<y> : index
%lid = kernel.workitem.id<x> : index
%c = index.madd %cg, %c256, %lid : index
%live = index.cmp ult, %c, %cols : index
scf.if %live {
%bc = index.assume %c [lt(%c, %cols)] : index
%zero = index.constant 0 : offset
%zf = scalar.constant 0.0 : f32
%xv = buffer.view %x[%zero] : buffer -> view<[%rows]x4304xf32>
%bv = buffer.view %bias[%zero] : buffer -> view<4304xf32>
%x0 = view.load %xv[%row, %bc] : view<[%rows]x4304xf32> -> f32
%b0 = view.load %bv[%bc] : view<4304xf32> -> f32
%a = scalar.addf %x0, %b0 : f32
%x3 = scalar.mulf %a, %a : f32
%x3b = scalar.mulf %x3, %a : f32
%k1 = scalar.constant 0.044715 : f32
%inner0 = scalar.mulf %x3b, %k1 : f32
%inner1 = scalar.addf %inner0, %a : f32
%k2 = scalar.constant 1.5957691216057308 : f32
%z = scalar.mulf %inner1, %k2 : f32
%nz = scalar.negf %z : f32
%ez = scalar.expf<afn> %nz : f32
%one = scalar.constant 1.0 : f32
%den = scalar.addf %ez, %one : f32
%r = scalar.divf %a, %den : f32
%yv = buffer.view %y[%zero] : buffer -> view<[%rows]x4304xf32>
view.store %r, %yv[%row, %bc] : f32, view<[%rows]x4304xf32>
}
kernel.return
}
kernel.def export("v_gemm_any_f16_4304x1152") @v_gemm_any_f16_4304x1152(%rows: index, %n: index) {
%c1 = index.constant 1 : index
%c15 = index.constant 15 : index
%c16 = index.constant 16 : index
%c63 = index.constant 63 : index
%c64 = index.constant 64 : index
%rp = index.add %rows, %c15 : index
%row_tiles = index.div %rp, %c16 : index
%ncst = index.constant 1152 : index
%np = index.add %ncst, %c63 : index
%col_tiles = index.div %np, %c64 : index
kernel.launch.config workgroups(%row_tiles, %col_tiles, %c1) workgroup_size(%c64, %c1, %c1) : index
} launch(%rows_arg: index, %k_arg: index, %n_arg: index, %x: buffer, %scales: buffer, %w: buffer, %y: buffer) {
%rows = index.assume %rows_arg [range(%rows_arg, 1, 65536)] : index
%zero = index.constant 0 : offset
%c0 = index.constant 0 : index
%c1 = index.constant 1 : index
%c8 = index.constant 8 : index
%c16 = index.constant 16 : index
%c64 = index.constant 64 : index
%ck = index.constant 4304 : index
%cn = index.constant 1152 : index
%zf = scalar.constant 0.0 : f32
%rt_id = kernel.workgroup.id<x> : index
%ct_id = kernel.workgroup.id<y> : index
%lid = kernel.workitem.id<x> : index
%lane = index.assume %lid [range(%lid, 0, 63)] : index
%col0 = index.mul %ct_id, %c64 : index
%col = index.add %col0, %lane : index
%col_live = index.cmp ult, %col, %cn : index
%nlast = index.constant 1151 : index
%colc = index.min %col, %nlast : index
%wcol = index.assume %colc [range(%colc, 0, 1151)] : index
%row0 = index.mul %rt_id, %c16 : index
%x_view = buffer.view %x[%zero] : buffer -> view<[%rows]x4304xf16>
%w_view = buffer.view %w[%zero] : buffer -> view<1152x4304xf16>
%s_view = buffer.view %scales[%zero] : buffer -> view<[%rows]x2xf32>
%rlast = index.sub %rows, %c1 : index
%r0, %r1, %r2, %r3, %r4, %r5, %r6, %r7, %r8, %r9, %r10, %r11, %r12, %r13, %r14, %r15 = scf.for %kk = [%c0 to %ck step %c8](%a0 = %zf : f32, %a1 = %zf : f32, %a2 = %zf : f32, %a3 = %zf : f32, %a4 = %zf : f32, %a5 = %zf : f32, %a6 = %zf : f32, %a7 = %zf : f32, %a8 = %zf : f32, %a9 = %zf : f32, %a10 = %zf : f32, %a11 = %zf : f32, %a12 = %zf : f32, %a13 = %zf : f32, %a14 = %zf : f32, %a15 = %zf : f32) -> (f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32) {
%bk = index.assume %kk [range(%kk, 0, 4296), multiple_of(%kk, 8)] : index
%wh = vector.load %w_view[%wcol, %bk] : view<1152x4304xf16> -> vector<8xf16>
%wv = vector.extf %wh : vector<8xf16> to vector<8xf32>
%ri0 = index.constant 0 : index
%rr0 = index.add %row0, %ri0 : index
%rc0 = index.min %rr0, %rlast : index
%rb0 = index.assume %rc0 [lt(%rc0, %rows)] : index
%xh0 = vector.load %x_view[%rb0, %bk] : view<[%rows]x4304xf16> -> vector<8xf16>
%xv0 = vector.extf %xh0 : vector<8xf16> to vector<8xf32>
%d0 = vector.dotf<reassoc|contract> %wv, %xv0, %zf : vector<8xf32>, vector<8xf32>, f32
%n0 = scalar.addf %a0, %d0 : f32
%ri1 = index.constant 1 : index
%rr1 = index.add %row0, %ri1 : index
%rc1 = index.min %rr1, %rlast : index
%rb1 = index.assume %rc1 [lt(%rc1, %rows)] : index
%xh1 = vector.load %x_view[%rb1, %bk] : view<[%rows]x4304xf16> -> vector<8xf16>
%xv1 = vector.extf %xh1 : vector<8xf16> to vector<8xf32>
%d1 = vector.dotf<reassoc|contract> %wv, %xv1, %zf : vector<8xf32>, vector<8xf32>, f32
%n1 = scalar.addf %a1, %d1 : f32
%ri2 = index.constant 2 : index
%rr2 = index.add %row0, %ri2 : index
%rc2 = index.min %rr2, %rlast : index
%rb2 = index.assume %rc2 [lt(%rc2, %rows)] : index
%xh2 = vector.load %x_view[%rb2, %bk] : view<[%rows]x4304xf16> -> vector<8xf16>
%xv2 = vector.extf %xh2 : vector<8xf16> to vector<8xf32>
%d2 = vector.dotf<reassoc|contract> %wv, %xv2, %zf : vector<8xf32>, vector<8xf32>, f32
%n2 = scalar.addf %a2, %d2 : f32
%ri3 = index.constant 3 : index
%rr3 = index.add %row0, %ri3 : index
%rc3 = index.min %rr3, %rlast : index
%rb3 = index.assume %rc3 [lt(%rc3, %rows)] : index
%xh3 = vector.load %x_view[%rb3, %bk] : view<[%rows]x4304xf16> -> vector<8xf16>
%xv3 = vector.extf %xh3 : vector<8xf16> to vector<8xf32>
%d3 = vector.dotf<reassoc|contract> %wv, %xv3, %zf : vector<8xf32>, vector<8xf32>, f32
%n3 = scalar.addf %a3, %d3 : f32
%ri4 = index.constant 4 : index
%rr4 = index.add %row0, %ri4 : index
%rc4 = index.min %rr4, %rlast : index
%rb4 = index.assume %rc4 [lt(%rc4, %rows)] : index
%xh4 = vector.load %x_view[%rb4, %bk] : view<[%rows]x4304xf16> -> vector<8xf16>
%xv4 = vector.extf %xh4 : vector<8xf16> to vector<8xf32>
%d4 = vector.dotf<reassoc|contract> %wv, %xv4, %zf : vector<8xf32>, vector<8xf32>, f32
%n4 = scalar.addf %a4, %d4 : f32
%ri5 = index.constant 5 : index
%rr5 = index.add %row0, %ri5 : index
%rc5 = index.min %rr5, %rlast : index
%rb5 = index.assume %rc5 [lt(%rc5, %rows)] : index
%xh5 = vector.load %x_view[%rb5, %bk] : view<[%rows]x4304xf16> -> vector<8xf16>
%xv5 = vector.extf %xh5 : vector<8xf16> to vector<8xf32>
%d5 = vector.dotf<reassoc|contract> %wv, %xv5, %zf : vector<8xf32>, vector<8xf32>, f32
%n5 = scalar.addf %a5, %d5 : f32
%ri6 = index.constant 6 : index
%rr6 = index.add %row0, %ri6 : index
%rc6 = index.min %rr6, %rlast : index
%rb6 = index.assume %rc6 [lt(%rc6, %rows)] : index
%xh6 = vector.load %x_view[%rb6, %bk] : view<[%rows]x4304xf16> -> vector<8xf16>
%xv6 = vector.extf %xh6 : vector<8xf16> to vector<8xf32>
%d6 = vector.dotf<reassoc|contract> %wv, %xv6, %zf : vector<8xf32>, vector<8xf32>, f32
%n6 = scalar.addf %a6, %d6 : f32
%ri7 = index.constant 7 : index
%rr7 = index.add %row0, %ri7 : index
%rc7 = index.min %rr7, %rlast : index
%rb7 = index.assume %rc7 [lt(%rc7, %rows)] : index
%xh7 = vector.load %x_view[%rb7, %bk] : view<[%rows]x4304xf16> -> vector<8xf16>
%xv7 = vector.extf %xh7 : vector<8xf16> to vector<8xf32>
%d7 = vector.dotf<reassoc|contract> %wv, %xv7, %zf : vector<8xf32>, vector<8xf32>, f32
%n7 = scalar.addf %a7, %d7 : f32
%ri8 = index.constant 8 : index
%rr8 = index.add %row0, %ri8 : index
%rc8 = index.min %rr8, %rlast : index
%rb8 = index.assume %rc8 [lt(%rc8, %rows)] : index
%xh8 = vector.load %x_view[%rb8, %bk] : view<[%rows]x4304xf16> -> vector<8xf16>
%xv8 = vector.extf %xh8 : vector<8xf16> to vector<8xf32>
%d8 = vector.dotf<reassoc|contract> %wv, %xv8, %zf : vector<8xf32>, vector<8xf32>, f32
%n8 = scalar.addf %a8, %d8 : f32
%ri9 = index.constant 9 : index
%rr9 = index.add %row0, %ri9 : index
%rc9 = index.min %rr9, %rlast : index
%rb9 = index.assume %rc9 [lt(%rc9, %rows)] : index
%xh9 = vector.load %x_view[%rb9, %bk] : view<[%rows]x4304xf16> -> vector<8xf16>
%xv9 = vector.extf %xh9 : vector<8xf16> to vector<8xf32>
%d9 = vector.dotf<reassoc|contract> %wv, %xv9, %zf : vector<8xf32>, vector<8xf32>, f32
%n9 = scalar.addf %a9, %d9 : f32
%ri10 = index.constant 10 : index
%rr10 = index.add %row0, %ri10 : index
%rc10 = index.min %rr10, %rlast : index
%rb10 = index.assume %rc10 [lt(%rc10, %rows)] : index
%xh10 = vector.load %x_view[%rb10, %bk] : view<[%rows]x4304xf16> -> vector<8xf16>
%xv10 = vector.extf %xh10 : vector<8xf16> to vector<8xf32>
%d10 = vector.dotf<reassoc|contract> %wv, %xv10, %zf : vector<8xf32>, vector<8xf32>, f32
%n10 = scalar.addf %a10, %d10 : f32
%ri11 = index.constant 11 : index
%rr11 = index.add %row0, %ri11 : index
%rc11 = index.min %rr11, %rlast : index
%rb11 = index.assume %rc11 [lt(%rc11, %rows)] : index
%xh11 = vector.load %x_view[%rb11, %bk] : view<[%rows]x4304xf16> -> vector<8xf16>
%xv11 = vector.extf %xh11 : vector<8xf16> to vector<8xf32>
%d11 = vector.dotf<reassoc|contract> %wv, %xv11, %zf : vector<8xf32>, vector<8xf32>, f32
%n11 = scalar.addf %a11, %d11 : f32
%ri12 = index.constant 12 : index
%rr12 = index.add %row0, %ri12 : index
%rc12 = index.min %rr12, %rlast : index
%rb12 = index.assume %rc12 [lt(%rc12, %rows)] : index
%xh12 = vector.load %x_view[%rb12, %bk] : view<[%rows]x4304xf16> -> vector<8xf16>
%xv12 = vector.extf %xh12 : vector<8xf16> to vector<8xf32>
%d12 = vector.dotf<reassoc|contract> %wv, %xv12, %zf : vector<8xf32>, vector<8xf32>, f32
%n12 = scalar.addf %a12, %d12 : f32
%ri13 = index.constant 13 : index
%rr13 = index.add %row0, %ri13 : index
%rc13 = index.min %rr13, %rlast : index
%rb13 = index.assume %rc13 [lt(%rc13, %rows)] : index
%xh13 = vector.load %x_view[%rb13, %bk] : view<[%rows]x4304xf16> -> vector<8xf16>
%xv13 = vector.extf %xh13 : vector<8xf16> to vector<8xf32>
%d13 = vector.dotf<reassoc|contract> %wv, %xv13, %zf : vector<8xf32>, vector<8xf32>, f32
%n13 = scalar.addf %a13, %d13 : f32
%ri14 = index.constant 14 : index
%rr14 = index.add %row0, %ri14 : index
%rc14 = index.min %rr14, %rlast : index
%rb14 = index.assume %rc14 [lt(%rc14, %rows)] : index
%xh14 = vector.load %x_view[%rb14, %bk] : view<[%rows]x4304xf16> -> vector<8xf16>
%xv14 = vector.extf %xh14 : vector<8xf16> to vector<8xf32>
%d14 = vector.dotf<reassoc|contract> %wv, %xv14, %zf : vector<8xf32>, vector<8xf32>, f32
%n14 = scalar.addf %a14, %d14 : f32
%ri15 = index.constant 15 : index
%rr15 = index.add %row0, %ri15 : index
%rc15 = index.min %rr15, %rlast : index
%rb15 = index.assume %rc15 [lt(%rc15, %rows)] : index
%xh15 = vector.load %x_view[%rb15, %bk] : view<[%rows]x4304xf16> -> vector<8xf16>
%xv15 = vector.extf %xh15 : vector<8xf16> to vector<8xf32>
%d15 = vector.dotf<reassoc|contract> %wv, %xv15, %zf : vector<8xf32>, vector<8xf32>, f32
%n15 = scalar.addf %a15, %d15 : f32
scf.yield %n0, %n1, %n2, %n3, %n4, %n5, %n6, %n7, %n8, %n9, %n10, %n11, %n12, %n13, %n14, %n15 : f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32
}
%y_view = buffer.view %y[%zero] : buffer -> view<[%rows]x1152xf32>
%ro0 = index.constant 0 : index
%orow0 = index.add %row0, %ro0 : index
%olive0 = index.cmp ult, %orow0, %rows : index
%ok0 = scalar.andi %olive0, %col_live : i1
scf.if %ok0 {
%ob0 = index.assume %orow0 [lt(%orow0, %rows)] : index
%inv0 = view.load %s_view[%ob0, %c1] : view<[%rows]x2xf32> -> f32
%val0 = scalar.mulf %r0, %inv0 : f32
view.store %val0, %y_view[%ob0, %wcol] : f32, view<[%rows]x1152xf32>
}
%ro1 = index.constant 1 : index
%orow1 = index.add %row0, %ro1 : index
%olive1 = index.cmp ult, %orow1, %rows : index
%ok1 = scalar.andi %olive1, %col_live : i1
scf.if %ok1 {
%ob1 = index.assume %orow1 [lt(%orow1, %rows)] : index
%inv1 = view.load %s_view[%ob1, %c1] : view<[%rows]x2xf32> -> f32
%val1 = scalar.mulf %r1, %inv1 : f32
view.store %val1, %y_view[%ob1, %wcol] : f32, view<[%rows]x1152xf32>
}
%ro2 = index.constant 2 : index
%orow2 = index.add %row0, %ro2 : index
%olive2 = index.cmp ult, %orow2, %rows : index
%ok2 = scalar.andi %olive2, %col_live : i1
scf.if %ok2 {
%ob2 = index.assume %orow2 [lt(%orow2, %rows)] : index
%inv2 = view.load %s_view[%ob2, %c1] : view<[%rows]x2xf32> -> f32
%val2 = scalar.mulf %r2, %inv2 : f32
view.store %val2, %y_view[%ob2, %wcol] : f32, view<[%rows]x1152xf32>
}
%ro3 = index.constant 3 : index
%orow3 = index.add %row0, %ro3 : index
%olive3 = index.cmp ult, %orow3, %rows : index
%ok3 = scalar.andi %olive3, %col_live : i1
scf.if %ok3 {
%ob3 = index.assume %orow3 [lt(%orow3, %rows)] : index
%inv3 = view.load %s_view[%ob3, %c1] : view<[%rows]x2xf32> -> f32
%val3 = scalar.mulf %r3, %inv3 : f32
view.store %val3, %y_view[%ob3, %wcol] : f32, view<[%rows]x1152xf32>
}
%ro4 = index.constant 4 : index
%orow4 = index.add %row0, %ro4 : index
%olive4 = index.cmp ult, %orow4, %rows : index
%ok4 = scalar.andi %olive4, %col_live : i1
scf.if %ok4 {
%ob4 = index.assume %orow4 [lt(%orow4, %rows)] : index
%inv4 = view.load %s_view[%ob4, %c1] : view<[%rows]x2xf32> -> f32
%val4 = scalar.mulf %r4, %inv4 : f32
view.store %val4, %y_view[%ob4, %wcol] : f32, view<[%rows]x1152xf32>
}
%ro5 = index.constant 5 : index
%orow5 = index.add %row0, %ro5 : index
%olive5 = index.cmp ult, %orow5, %rows : index
%ok5 = scalar.andi %olive5, %col_live : i1
scf.if %ok5 {
%ob5 = index.assume %orow5 [lt(%orow5, %rows)] : index
%inv5 = view.load %s_view[%ob5, %c1] : view<[%rows]x2xf32> -> f32
%val5 = scalar.mulf %r5, %inv5 : f32
view.store %val5, %y_view[%ob5, %wcol] : f32, view<[%rows]x1152xf32>
}
%ro6 = index.constant 6 : index
%orow6 = index.add %row0, %ro6 : index
%olive6 = index.cmp ult, %orow6, %rows : index
%ok6 = scalar.andi %olive6, %col_live : i1
scf.if %ok6 {
%ob6 = index.assume %orow6 [lt(%orow6, %rows)] : index
%inv6 = view.load %s_view[%ob6, %c1] : view<[%rows]x2xf32> -> f32
%val6 = scalar.mulf %r6, %inv6 : f32
view.store %val6, %y_view[%ob6, %wcol] : f32, view<[%rows]x1152xf32>
}
%ro7 = index.constant 7 : index
%orow7 = index.add %row0, %ro7 : index
%olive7 = index.cmp ult, %orow7, %rows : index
%ok7 = scalar.andi %olive7, %col_live : i1
scf.if %ok7 {
%ob7 = index.assume %orow7 [lt(%orow7, %rows)] : index
%inv7 = view.load %s_view[%ob7, %c1] : view<[%rows]x2xf32> -> f32
%val7 = scalar.mulf %r7, %inv7 : f32
view.store %val7, %y_view[%ob7, %wcol] : f32, view<[%rows]x1152xf32>
}
%ro8 = index.constant 8 : index
%orow8 = index.add %row0, %ro8 : index
%olive8 = index.cmp ult, %orow8, %rows : index
%ok8 = scalar.andi %olive8, %col_live : i1
scf.if %ok8 {
%ob8 = index.assume %orow8 [lt(%orow8, %rows)] : index
%inv8 = view.load %s_view[%ob8, %c1] : view<[%rows]x2xf32> -> f32
%val8 = scalar.mulf %r8, %inv8 : f32
view.store %val8, %y_view[%ob8, %wcol] : f32, view<[%rows]x1152xf32>
}
%ro9 = index.constant 9 : index
%orow9 = index.add %row0, %ro9 : index
%olive9 = index.cmp ult, %orow9, %rows : index
%ok9 = scalar.andi %olive9, %col_live : i1
scf.if %ok9 {
%ob9 = index.assume %orow9 [lt(%orow9, %rows)] : index
%inv9 = view.load %s_view[%ob9, %c1] : view<[%rows]x2xf32> -> f32
%val9 = scalar.mulf %r9, %inv9 : f32
view.store %val9, %y_view[%ob9, %wcol] : f32, view<[%rows]x1152xf32>
}
%ro10 = index.constant 10 : index
%orow10 = index.add %row0, %ro10 : index
%olive10 = index.cmp ult, %orow10, %rows : index
%ok10 = scalar.andi %olive10, %col_live : i1
scf.if %ok10 {
%ob10 = index.assume %orow10 [lt(%orow10, %rows)] : index
%inv10 = view.load %s_view[%ob10, %c1] : view<[%rows]x2xf32> -> f32
%val10 = scalar.mulf %r10, %inv10 : f32
view.store %val10, %y_view[%ob10, %wcol] : f32, view<[%rows]x1152xf32>
}
%ro11 = index.constant 11 : index
%orow11 = index.add %row0, %ro11 : index
%olive11 = index.cmp ult, %orow11, %rows : index
%ok11 = scalar.andi %olive11, %col_live : i1
scf.if %ok11 {
%ob11 = index.assume %orow11 [lt(%orow11, %rows)] : index
%inv11 = view.load %s_view[%ob11, %c1] : view<[%rows]x2xf32> -> f32
%val11 = scalar.mulf %r11, %inv11 : f32
view.store %val11, %y_view[%ob11, %wcol] : f32, view<[%rows]x1152xf32>
}
%ro12 = index.constant 12 : index
%orow12 = index.add %row0, %ro12 : index
%olive12 = index.cmp ult, %orow12, %rows : index
%ok12 = scalar.andi %olive12, %col_live : i1
scf.if %ok12 {
%ob12 = index.assume %orow12 [lt(%orow12, %rows)] : index
%inv12 = view.load %s_view[%ob12, %c1] : view<[%rows]x2xf32> -> f32
%val12 = scalar.mulf %r12, %inv12 : f32
view.store %val12, %y_view[%ob12, %wcol] : f32, view<[%rows]x1152xf32>
}
%ro13 = index.constant 13 : index
%orow13 = index.add %row0, %ro13 : index
%olive13 = index.cmp ult, %orow13, %rows : index
%ok13 = scalar.andi %olive13, %col_live : i1
scf.if %ok13 {
%ob13 = index.assume %orow13 [lt(%orow13, %rows)] : index
%inv13 = view.load %s_view[%ob13, %c1] : view<[%rows]x2xf32> -> f32
%val13 = scalar.mulf %r13, %inv13 : f32
view.store %val13, %y_view[%ob13, %wcol] : f32, view<[%rows]x1152xf32>
}
%ro14 = index.constant 14 : index
%orow14 = index.add %row0, %ro14 : index
%olive14 = index.cmp ult, %orow14, %rows : index
%ok14 = scalar.andi %olive14, %col_live : i1
scf.if %ok14 {
%ob14 = index.assume %orow14 [lt(%orow14, %rows)] : index
%inv14 = view.load %s_view[%ob14, %c1] : view<[%rows]x2xf32> -> f32
%val14 = scalar.mulf %r14, %inv14 : f32
view.store %val14, %y_view[%ob14, %wcol] : f32, view<[%rows]x1152xf32>
}
%ro15 = index.constant 15 : index
%orow15 = index.add %row0, %ro15 : index
%olive15 = index.cmp ult, %orow15, %rows : index
%ok15 = scalar.andi %olive15, %col_live : i1
scf.if %ok15 {
%ob15 = index.assume %orow15 [lt(%orow15, %rows)] : index
%inv15 = view.load %s_view[%ob15, %c1] : view<[%rows]x2xf32> -> f32
%val15 = scalar.mulf %r15, %inv15 : f32
view.store %val15, %y_view[%ob15, %wcol] : f32, view<[%rows]x1152xf32>
}
kernel.return
}
kernel.def export("v_gemm_f16_store_4608x4608") @v_gemm_f16_store_4608x4608(%rows: index, %n: index) {
%c1 = index.constant 1 : index
%c64 = index.constant 64 : index
%c256 = index.constant 256 : index
%row_tiles = index.div %rows, %c64 : index
%n_const = index.constant 4608 : index
%out_tiles = index.div %n_const, %c64 : index
kernel.launch.config workgroups(%row_tiles, %out_tiles, %c1) workgroup_size(%c256, %c1, %c1) : index
} launch(%rows_arg: index, %k_arg: index, %n_arg: index, %x: buffer, %scales: buffer, %w: buffer, %y: buffer) {
%rows = index.assume %rows_arg [range(%rows_arg, 64, 65536), multiple_of(%rows_arg, 64)] : index
%k = index.constant 4608 : index
%n = index.constant 4608 : index
%origin = index.constant 0 : offset
%two = index.constant 2 : offset
%sixteen = index.constant 16 : offset
%block_bytes = index.constant 64 : offset
%c0 = index.constant 0 : index
%c1 = index.constant 1 : index
%c2 = index.constant 2 : index
%c4 = index.constant 4 : index
%c8 = index.constant 8 : index
%c16 = index.constant 16 : index
%c32 = index.constant 32 : index
%c64 = index.constant 64 : index
%c256 = index.constant 256 : index
%nibble_mask = vector.constant 252645135 : vector<4xi32>
%stage_bytes = index.constant 9216 : offset
%row_tile_count = index.div %rows, %c64 : index
%out_tile_count = index.div %n, %c64 : index
%row_group_id = kernel.workgroup.id<x> : index
%out_group_id = kernel.workgroup.id<y> : index
%row_group = index.assume %row_group_id [lt(%row_group_id, %row_tile_count)] : index
%out_group = index.assume %out_group_id [lt(%out_group_id, %out_tile_count)] : index
%workitem_id = kernel.workitem.id<x> : index
%workitem = index.assume %workitem_id [range(%workitem_id, 0, 255)] : index
%subgroup = kernel.subgroup.id : index
%lane = kernel.subgroup.lane.id : index
%lane_row = index.rem %lane, %c16 : index
%bounded_subgroup = index.assume %subgroup [range(%subgroup, 0, 7)] : index
%wave_row_group = index.div %bounded_subgroup, %c2 : index
%wave_out_group = index.rem %bounded_subgroup, %c2 : index
%wave_row = index.mul %wave_row_group, %c16 : index
%wave_out = index.mul %wave_out_group, %c32 : index
%w_row0 = index.add %wave_out, %lane_row : index
%w_row1 = index.add %w_row0, %c16 : index
%row_origin = index.mul %row_group, %c64 : index
%out_origin = index.mul %out_group, %c64 : index
%result_row = index.add %row_origin, %wave_row : index
%result_col0 = index.add %out_origin, %wave_out : index
%result_col1 = index.add %result_col0, %c16 : index
%blocks = index.div %k, %c32 : index
%row_bytes = index.scale %blocks, %block_bytes : index, offset -> offset
%x_view = buffer.view %x[%origin] : buffer -> view<[%rows]x[%k]xf16>
// 64x64 output tile with the full row stride n.
%y_layout = encoding.layout.strided [%n, %c1] : encoding<layout>
%tile_first = index.madd %row_origin, %n, %out_origin : index
%four = index.constant 4 : offset
%tile_offset = index.scale %tile_first, %four : index, offset -> offset
%y_view = buffer.view %y[%tile_offset] : buffer -> view<64x64xf32, %y_layout>
%bounded_wave_row = index.assume %wave_row [range(%wave_row, 0, 48)] : index
%bounded_wave_out = index.assume %wave_out [range(%wave_out, 0, 32)] : index
%wave_out1 = index.add %bounded_wave_out, %c16 : index
%x_stage = buffer.alloca<workgroup> align(16) %stage_bytes : buffer
%x_stage_view = buffer.view %x_stage[%origin] : buffer -> view<64x72xf16>
%w_stage = buffer.alloca<workgroup> align(16) %stage_bytes : buffer
%w_stage_view = buffer.view %w_stage[%origin] : buffer -> view<64x72xf16>
// Thread t stages 16 values: row t/4, columns (t%4)*16.
%stage_row = index.div %workitem, %c4 : index
%stage_quarter = index.rem %workitem, %c4 : index
%stage_col = index.mul %stage_quarter, %c16 : index
%x_row = index.add %row_origin, %stage_row : index
%bounded_x_row = index.assume %x_row [lt(%x_row, %rows)] : index
%w_out_row = index.add %out_origin, %stage_row : index
%bounded_w_out_row = index.assume %w_out_row [lt(%w_out_row, %n)] : index
%w_row_origin = index.scale %bounded_w_out_row, %row_bytes : index, offset -> offset
%zero8 = vector.constant 0.0 : vector<8xf32>
%init = vector.fragment<init> %zero8 shape [%c16, %c16] : vector<8xf32>
%k_steps = index.div %k, %c64 : index
%s_view = buffer.view %scales[%origin] : buffer -> view<[%rows]x2xf32>
%x_scale = view.load %s_view[%bounded_x_row, %c0] : view<[%rows]x2xf32> -> f32
%x_scales = vector.splat %x_scale : vector<16xf32>
%a0_raw, %a1_raw = scf.for %step = [%c0 to %k_steps step %c1](%a0_acc = %init : vector<8xf32>, %a1_acc = %init : vector<8xf32>) -> (vector<8xf32>, vector<8xf32>) {
%k_origin = index.mul %step, %c64 : index
%x_col = index.add %k_origin, %stage_col : index
%x_half = vector.load %x_view[%bounded_x_row, %x_col] : view<[%rows]x[%k]xf16> -> vector<16xf16>
vector.store %x_half, %x_stage_view[%stage_row, %stage_col] : vector<16xf16>, view<64x72xf16>
%w_col = index.add %k_origin, %stage_col : index
%deq0_two_b = index.constant 2 : offset
%deq0_col_bytes = index.scale %w_col, %deq0_two_b : index, offset -> offset
%deq0_origin = index.add %w_row_origin, %deq0_col_bytes : offset
%deq0_view = buffer.view %w[%deq0_origin] : buffer -> view<16xf16>
%deq0 = vector.load %deq0_view[0] : view<16xf16> -> vector<16xf16>
vector.store %deq0, %w_stage_view[%stage_row, %stage_col] : vector<16xf16>, view<64x72xf16>
kernel.barrier<workgroup> scope(workgroup) ordering(acq_rel)
%fc0 = index.constant 0 : index
%lhs0 = vector.fragment.load<lhs> %x_stage_view[%wave_row, %fc0] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16>
%rv000 = vector.load %w_stage_view[%w_row0, %fc0] : view<64x72xf16> -> vector<16xf16>
%rhs000 = vector.fragment<rhs> %rv000 shape [%c16, %c16] : vector<16xf16>
%na0_0 = vector.mma %lhs0, %rhs000, %a0_acc : vector<16xf16>, vector<16xf16>, vector<8xf32>
%rv010 = vector.load %w_stage_view[%w_row1, %fc0] : view<64x72xf16> -> vector<16xf16>
%rhs010 = vector.fragment<rhs> %rv010 shape [%c16, %c16] : vector<16xf16>
%na1_0 = vector.mma %lhs0, %rhs010, %a1_acc : vector<16xf16>, vector<16xf16>, vector<8xf32>
%fc1 = index.constant 16 : index
%lhs1 = vector.fragment.load<lhs> %x_stage_view[%wave_row, %fc1] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16>
%rv001 = vector.load %w_stage_view[%w_row0, %fc1] : view<64x72xf16> -> vector<16xf16>
%rhs001 = vector.fragment<rhs> %rv001 shape [%c16, %c16] : vector<16xf16>
%na0_1 = vector.mma %lhs1, %rhs001, %na0_0 : vector<16xf16>, vector<16xf16>, vector<8xf32>
%rv011 = vector.load %w_stage_view[%w_row1, %fc1] : view<64x72xf16> -> vector<16xf16>
%rhs011 = vector.fragment<rhs> %rv011 shape [%c16, %c16] : vector<16xf16>
%na1_1 = vector.mma %lhs1, %rhs011, %na1_0 : vector<16xf16>, vector<16xf16>, vector<8xf32>
%fc2 = index.constant 32 : index
%lhs2 = vector.fragment.load<lhs> %x_stage_view[%wave_row, %fc2] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16>
%rv002 = vector.load %w_stage_view[%w_row0, %fc2] : view<64x72xf16> -> vector<16xf16>
%rhs002 = vector.fragment<rhs> %rv002 shape [%c16, %c16] : vector<16xf16>
%na0_2 = vector.mma %lhs2, %rhs002, %na0_1 : vector<16xf16>, vector<16xf16>, vector<8xf32>
%rv012 = vector.load %w_stage_view[%w_row1, %fc2] : view<64x72xf16> -> vector<16xf16>
%rhs012 = vector.fragment<rhs> %rv012 shape [%c16, %c16] : vector<16xf16>
%na1_2 = vector.mma %lhs2, %rhs012, %na1_1 : vector<16xf16>, vector<16xf16>, vector<8xf32>
%fc3 = index.constant 48 : index
%lhs3 = vector.fragment.load<lhs> %x_stage_view[%wave_row, %fc3] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16>
%rv003 = vector.load %w_stage_view[%w_row0, %fc3] : view<64x72xf16> -> vector<16xf16>
%rhs003 = vector.fragment<rhs> %rv003 shape [%c16, %c16] : vector<16xf16>
%na0_3 = vector.mma %lhs3, %rhs003, %na0_2 : vector<16xf16>, vector<16xf16>, vector<8xf32>
%rv013 = vector.load %w_stage_view[%w_row1, %fc3] : view<64x72xf16> -> vector<16xf16>
%rhs013 = vector.fragment<rhs> %rv013 shape [%c16, %c16] : vector<16xf16>
%na1_3 = vector.mma %lhs3, %rhs013, %na1_2 : vector<16xf16>, vector<16xf16>, vector<8xf32>
kernel.barrier<workgroup> scope(workgroup) ordering(acq_rel)
scf.yield %na0_3, %na1_3 : vector<8xf32>, vector<8xf32>
}
%inv_bytes = index.constant 4096 : offset
%inv_tile = buffer.alloca<workgroup> align(16) %inv_bytes : buffer
%inv_view = buffer.view %inv_tile[%origin] : buffer -> view<64x16xf32>
%writer = index.cmp ult, %workitem, %c64 : index
scf.if %writer {
%inv_row_value = index.add %row_origin, %workitem : index
%inv_row = index.assume %inv_row_value [lt(%inv_row_value, %rows)] : index
%inverse = view.load %s_view[%inv_row, %c1] : view<[%rows]x2xf32> -> f32
%inverse16 = vector.splat %inverse : vector<16xf32>
%bw = index.assume %workitem [range(%workitem, 0, 63)] : index
vector.store %inverse16, %inv_view[%bw, %c0] : vector<16xf32>, view<64x16xf32>
}
kernel.barrier<workgroup> scope(workgroup) ordering(acq_rel)
%inv_frag = vector.fragment.load<result> %inv_view[%wave_row, %c0] shape [%c16, %c16] : view<64x16xf32> -> vector<8xf32>
%a0 = vector.mulf %a0_raw, %inv_frag : vector<8xf32>
%a1 = vector.mulf %a1_raw, %inv_frag : vector<8xf32>
vector.fragment.store<result> %a0, %y_view[%bounded_wave_row, %bounded_wave_out] shape [%c16, %c16] : vector<8xf32>, view<64x64xf32, %y_layout>
vector.fragment.store<result> %a1, %y_view[%bounded_wave_row, %wave_out1] shape [%c16, %c16] : vector<8xf32>, view<64x64xf32, %y_layout>
kernel.return
}
kernel.def export("v_bias_gelu_tanh_4608") @v_bias_gelu_tanh_4608(%rows: index) {
%c1 = index.constant 1 : index
%c256 = index.constant 256 : index
%cols = index.constant 4608 : index
%cg = index.constant 18 : index
kernel.launch.config workgroups(%rows, %cg, %c1) workgroup_size(%c256, %c1, %c1) : index
} launch(%rows_arg: index, %x: buffer, %bias: buffer, %y: buffer) {
%rows = index.assume %rows_arg [range(%rows_arg, 1, 65536)] : index
%c256 = index.constant 256 : index
%cols = index.constant 4608 : index
%row_id = kernel.workgroup.id<x> : index
%row = index.assume %row_id [lt(%row_id, %rows)] : index
%cg = kernel.workgroup.id<y> : index
%lid = kernel.workitem.id<x> : index
%c = index.madd %cg, %c256, %lid : index
%live = index.cmp ult, %c, %cols : index
scf.if %live {
%bc = index.assume %c [lt(%c, %cols)] : index
%zero = index.constant 0 : offset
%zf = scalar.constant 0.0 : f32
%xv = buffer.view %x[%zero] : buffer -> view<[%rows]x4608xf32>
%bv = buffer.view %bias[%zero] : buffer -> view<4608xf32>
%x0 = view.load %xv[%row, %bc] : view<[%rows]x4608xf32> -> f32
%b0 = view.load %bv[%bc] : view<4608xf32> -> f32
%a = scalar.addf %x0, %b0 : f32
%x3 = scalar.mulf %a, %a : f32
%x3b = scalar.mulf %x3, %a : f32
%k1 = scalar.constant 0.044715 : f32
%inner0 = scalar.mulf %x3b, %k1 : f32
%inner1 = scalar.addf %inner0, %a : f32
%k2 = scalar.constant 1.5957691216057308 : f32
%z = scalar.mulf %inner1, %k2 : f32
%nz = scalar.negf %z : f32
%ez = scalar.expf<afn> %nz : f32
%one = scalar.constant 1.0 : f32
%den = scalar.addf %ez, %one : f32
%r = scalar.divf %a, %den : f32
%yv = buffer.view %y[%zero] : buffer -> view<[%rows]x4608xf32>
view.store %r, %yv[%row, %bc] : f32, view<[%rows]x4608xf32>
}
kernel.return
}
kernel.def export("v_gemm_f16_store_4608x2048") @v_gemm_f16_store_4608x2048(%rows: index, %n: index) {
%c1 = index.constant 1 : index
%c64 = index.constant 64 : index
%c256 = index.constant 256 : index
%row_tiles = index.div %rows, %c64 : index
%n_const = index.constant 2048 : index
%out_tiles = index.div %n_const, %c64 : index
kernel.launch.config workgroups(%row_tiles, %out_tiles, %c1) workgroup_size(%c256, %c1, %c1) : index
} launch(%rows_arg: index, %k_arg: index, %n_arg: index, %x: buffer, %scales: buffer, %w: buffer, %y: buffer) {
%rows = index.assume %rows_arg [range(%rows_arg, 64, 65536), multiple_of(%rows_arg, 64)] : index
%k = index.constant 4608 : index
%n = index.constant 2048 : index
%origin = index.constant 0 : offset
%two = index.constant 2 : offset
%sixteen = index.constant 16 : offset
%block_bytes = index.constant 64 : offset
%c0 = index.constant 0 : index
%c1 = index.constant 1 : index
%c2 = index.constant 2 : index
%c4 = index.constant 4 : index
%c8 = index.constant 8 : index
%c16 = index.constant 16 : index
%c32 = index.constant 32 : index
%c64 = index.constant 64 : index
%c256 = index.constant 256 : index
%nibble_mask = vector.constant 252645135 : vector<4xi32>
%stage_bytes = index.constant 9216 : offset
%row_tile_count = index.div %rows, %c64 : index
%out_tile_count = index.div %n, %c64 : index
%row_group_id = kernel.workgroup.id<x> : index
%out_group_id = kernel.workgroup.id<y> : index
%row_group = index.assume %row_group_id [lt(%row_group_id, %row_tile_count)] : index
%out_group = index.assume %out_group_id [lt(%out_group_id, %out_tile_count)] : index
%workitem_id = kernel.workitem.id<x> : index
%workitem = index.assume %workitem_id [range(%workitem_id, 0, 255)] : index
%subgroup = kernel.subgroup.id : index
%lane = kernel.subgroup.lane.id : index
%lane_row = index.rem %lane, %c16 : index
%bounded_subgroup = index.assume %subgroup [range(%subgroup, 0, 7)] : index
%wave_row_group = index.div %bounded_subgroup, %c2 : index
%wave_out_group = index.rem %bounded_subgroup, %c2 : index
%wave_row = index.mul %wave_row_group, %c16 : index
%wave_out = index.mul %wave_out_group, %c32 : index
%w_row0 = index.add %wave_out, %lane_row : index
%w_row1 = index.add %w_row0, %c16 : index
%row_origin = index.mul %row_group, %c64 : index
%out_origin = index.mul %out_group, %c64 : index
%result_row = index.add %row_origin, %wave_row : index
%result_col0 = index.add %out_origin, %wave_out : index
%result_col1 = index.add %result_col0, %c16 : index
%blocks = index.div %k, %c32 : index
%row_bytes = index.scale %blocks, %block_bytes : index, offset -> offset
%x_view = buffer.view %x[%origin] : buffer -> view<[%rows]x[%k]xf16>
// 64x64 output tile with the full row stride n.
%y_layout = encoding.layout.strided [%n, %c1] : encoding<layout>
%tile_first = index.madd %row_origin, %n, %out_origin : index
%four = index.constant 4 : offset
%tile_offset = index.scale %tile_first, %four : index, offset -> offset
%y_view = buffer.view %y[%tile_offset] : buffer -> view<64x64xf32, %y_layout>
%bounded_wave_row = index.assume %wave_row [range(%wave_row, 0, 48)] : index
%bounded_wave_out = index.assume %wave_out [range(%wave_out, 0, 32)] : index
%wave_out1 = index.add %bounded_wave_out, %c16 : index
%x_stage = buffer.alloca<workgroup> align(16) %stage_bytes : buffer
%x_stage_view = buffer.view %x_stage[%origin] : buffer -> view<64x72xf16>
%w_stage = buffer.alloca<workgroup> align(16) %stage_bytes : buffer
%w_stage_view = buffer.view %w_stage[%origin] : buffer -> view<64x72xf16>
// Thread t stages 16 values: row t/4, columns (t%4)*16.
%stage_row = index.div %workitem, %c4 : index
%stage_quarter = index.rem %workitem, %c4 : index
%stage_col = index.mul %stage_quarter, %c16 : index
%x_row = index.add %row_origin, %stage_row : index
%bounded_x_row = index.assume %x_row [lt(%x_row, %rows)] : index
%w_out_row = index.add %out_origin, %stage_row : index
%bounded_w_out_row = index.assume %w_out_row [lt(%w_out_row, %n)] : index
%w_row_origin = index.scale %bounded_w_out_row, %row_bytes : index, offset -> offset
%zero8 = vector.constant 0.0 : vector<8xf32>
%init = vector.fragment<init> %zero8 shape [%c16, %c16] : vector<8xf32>
%k_steps = index.div %k, %c64 : index
%s_view = buffer.view %scales[%origin] : buffer -> view<[%rows]x2xf32>
%x_scale = view.load %s_view[%bounded_x_row, %c0] : view<[%rows]x2xf32> -> f32
%x_scales = vector.splat %x_scale : vector<16xf32>
%a0_raw, %a1_raw = scf.for %step = [%c0 to %k_steps step %c1](%a0_acc = %init : vector<8xf32>, %a1_acc = %init : vector<8xf32>) -> (vector<8xf32>, vector<8xf32>) {
%k_origin = index.mul %step, %c64 : index
%x_col = index.add %k_origin, %stage_col : index
%x_half = vector.load %x_view[%bounded_x_row, %x_col] : view<[%rows]x[%k]xf16> -> vector<16xf16>
vector.store %x_half, %x_stage_view[%stage_row, %stage_col] : vector<16xf16>, view<64x72xf16>
%w_col = index.add %k_origin, %stage_col : index
%deq0_two_b = index.constant 2 : offset
%deq0_col_bytes = index.scale %w_col, %deq0_two_b : index, offset -> offset
%deq0_origin = index.add %w_row_origin, %deq0_col_bytes : offset
%deq0_view = buffer.view %w[%deq0_origin] : buffer -> view<16xf16>
%deq0 = vector.load %deq0_view[0] : view<16xf16> -> vector<16xf16>
vector.store %deq0, %w_stage_view[%stage_row, %stage_col] : vector<16xf16>, view<64x72xf16>
kernel.barrier<workgroup> scope(workgroup) ordering(acq_rel)
%fc0 = index.constant 0 : index
%lhs0 = vector.fragment.load<lhs> %x_stage_view[%wave_row, %fc0] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16>
%rv000 = vector.load %w_stage_view[%w_row0, %fc0] : view<64x72xf16> -> vector<16xf16>
%rhs000 = vector.fragment<rhs> %rv000 shape [%c16, %c16] : vector<16xf16>
%na0_0 = vector.mma %lhs0, %rhs000, %a0_acc : vector<16xf16>, vector<16xf16>, vector<8xf32>
%rv010 = vector.load %w_stage_view[%w_row1, %fc0] : view<64x72xf16> -> vector<16xf16>
%rhs010 = vector.fragment<rhs> %rv010 shape [%c16, %c16] : vector<16xf16>
%na1_0 = vector.mma %lhs0, %rhs010, %a1_acc : vector<16xf16>, vector<16xf16>, vector<8xf32>
%fc1 = index.constant 16 : index
%lhs1 = vector.fragment.load<lhs> %x_stage_view[%wave_row, %fc1] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16>
%rv001 = vector.load %w_stage_view[%w_row0, %fc1] : view<64x72xf16> -> vector<16xf16>
%rhs001 = vector.fragment<rhs> %rv001 shape [%c16, %c16] : vector<16xf16>
%na0_1 = vector.mma %lhs1, %rhs001, %na0_0 : vector<16xf16>, vector<16xf16>, vector<8xf32>
%rv011 = vector.load %w_stage_view[%w_row1, %fc1] : view<64x72xf16> -> vector<16xf16>
%rhs011 = vector.fragment<rhs> %rv011 shape [%c16, %c16] : vector<16xf16>
%na1_1 = vector.mma %lhs1, %rhs011, %na1_0 : vector<16xf16>, vector<16xf16>, vector<8xf32>
%fc2 = index.constant 32 : index
%lhs2 = vector.fragment.load<lhs> %x_stage_view[%wave_row, %fc2] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16>
%rv002 = vector.load %w_stage_view[%w_row0, %fc2] : view<64x72xf16> -> vector<16xf16>
%rhs002 = vector.fragment<rhs> %rv002 shape [%c16, %c16] : vector<16xf16>
%na0_2 = vector.mma %lhs2, %rhs002, %na0_1 : vector<16xf16>, vector<16xf16>, vector<8xf32>
%rv012 = vector.load %w_stage_view[%w_row1, %fc2] : view<64x72xf16> -> vector<16xf16>
%rhs012 = vector.fragment<rhs> %rv012 shape [%c16, %c16] : vector<16xf16>
%na1_2 = vector.mma %lhs2, %rhs012, %na1_1 : vector<16xf16>, vector<16xf16>, vector<8xf32>
%fc3 = index.constant 48 : index
%lhs3 = vector.fragment.load<lhs> %x_stage_view[%wave_row, %fc3] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16>
%rv003 = vector.load %w_stage_view[%w_row0, %fc3] : view<64x72xf16> -> vector<16xf16>
%rhs003 = vector.fragment<rhs> %rv003 shape [%c16, %c16] : vector<16xf16>
%na0_3 = vector.mma %lhs3, %rhs003, %na0_2 : vector<16xf16>, vector<16xf16>, vector<8xf32>
%rv013 = vector.load %w_stage_view[%w_row1, %fc3] : view<64x72xf16> -> vector<16xf16>
%rhs013 = vector.fragment<rhs> %rv013 shape [%c16, %c16] : vector<16xf16>
%na1_3 = vector.mma %lhs3, %rhs013, %na1_2 : vector<16xf16>, vector<16xf16>, vector<8xf32>
kernel.barrier<workgroup> scope(workgroup) ordering(acq_rel)
scf.yield %na0_3, %na1_3 : vector<8xf32>, vector<8xf32>
}
%inv_bytes = index.constant 4096 : offset
%inv_tile = buffer.alloca<workgroup> align(16) %inv_bytes : buffer
%inv_view = buffer.view %inv_tile[%origin] : buffer -> view<64x16xf32>
%writer = index.cmp ult, %workitem, %c64 : index
scf.if %writer {
%inv_row_value = index.add %row_origin, %workitem : index
%inv_row = index.assume %inv_row_value [lt(%inv_row_value, %rows)] : index
%inverse = view.load %s_view[%inv_row, %c1] : view<[%rows]x2xf32> -> f32
%inverse16 = vector.splat %inverse : vector<16xf32>
%bw = index.assume %workitem [range(%workitem, 0, 63)] : index
vector.store %inverse16, %inv_view[%bw, %c0] : vector<16xf32>, view<64x16xf32>
}
kernel.barrier<workgroup> scope(workgroup) ordering(acq_rel)
%inv_frag = vector.fragment.load<result> %inv_view[%wave_row, %c0] shape [%c16, %c16] : view<64x16xf32> -> vector<8xf32>
%a0 = vector.mulf %a0_raw, %inv_frag : vector<8xf32>
%a1 = vector.mulf %a1_raw, %inv_frag : vector<8xf32>
vector.fragment.store<result> %a0, %y_view[%bounded_wave_row, %bounded_wave_out] shape [%c16, %c16] : vector<8xf32>, view<64x64xf32, %y_layout>
vector.fragment.store<result> %a1, %y_view[%bounded_wave_row, %wave_out1] shape [%c16, %c16] : vector<8xf32>, view<64x64xf32, %y_layout>
kernel.return
}
kernel.def export("v_bias_none_2048") @v_bias_none_2048(%rows: index) {
%c1 = index.constant 1 : index
%c256 = index.constant 256 : index
%cols = index.constant 2048 : index
%cg = index.constant 8 : index
kernel.launch.config workgroups(%rows, %cg, %c1) workgroup_size(%c256, %c1, %c1) : index
} launch(%rows_arg: index, %x: buffer, %bias: buffer, %y: buffer) {
%rows = index.assume %rows_arg [range(%rows_arg, 1, 65536)] : index
%c256 = index.constant 256 : index
%cols = index.constant 2048 : index
%row_id = kernel.workgroup.id<x> : index
%row = index.assume %row_id [lt(%row_id, %rows)] : index
%cg = kernel.workgroup.id<y> : index
%lid = kernel.workitem.id<x> : index
%c = index.madd %cg, %c256, %lid : index
%live = index.cmp ult, %c, %cols : index
scf.if %live {
%bc = index.assume %c [lt(%c, %cols)] : index
%zero = index.constant 0 : offset
%zf = scalar.constant 0.0 : f32
%xv = buffer.view %x[%zero] : buffer -> view<[%rows]x2048xf32>
%bv = buffer.view %bias[%zero] : buffer -> view<2048xf32>
%x0 = view.load %xv[%row, %bc] : view<[%rows]x2048xf32> -> f32
%b0 = view.load %bv[%bc] : view<2048xf32> -> f32
%a = scalar.addf %x0, %b0 : f32
%r = scalar.addf %a, %zf : f32
%yv = buffer.view %y[%zero] : buffer -> view<[%rows]x2048xf32>
view.store %r, %yv[%row, %bc] : f32, view<[%rows]x2048xf32>
}
kernel.return
}
kernel.def export("v_resize_bicubic_h_hdr") @v_resize_bicubic_h_hdr(%out_w: index, %max_rows: index) {
%count = index.mul %out_w, %max_rows : index
%c1 = index.constant 1 : index
%c256 = index.constant 256 : index
%c255 = index.constant 255 : index
%n_up = index.add %count, %c255 : index
%groups = index.div %n_up, %c256 : index
kernel.launch.config workgroups(%groups, %c1, %c1) workgroup_size(%c256, %c1, %c1) : index
} launch(%out_n_arg: index, %other_arg: index, %src: buffer, %dst: buffer) {
%zero_h = index.constant 0 : offset
%hv = buffer.view %src[%zero_h] : buffer -> view<16xi32>
%in_w32 = view.load %hv[0] : view<16xi32> -> i32
%rows32 = view.load %hv[1] : view<16xi32> -> i32
%cin32 = view.load %hv[2] : view<16xi32> -> i32
%pix32 = view.load %hv[3] : view<16xi32> -> i32
%in_b = scalar.assume %in_w32 [range(%in_w32, 1, 16384)] : i32
%in_n = index.cast %in_b : i32 to index
%rows_b = scalar.assume %rows32 [range(%rows32, 1, 16384)] : i32
%rows_i = index.cast %rows_b : i32 to index
%cin_b = scalar.assume %cin32 [range(%cin32, 3, 4)] : i32
%cin_i = index.cast %cin_b : i32 to index
%pix_b = scalar.assume %pix32 [range(%pix32, 0, 2147483647)] : i32
%pix_i = index.cast %pix_b : i32 to index
%out_n = index.assume %out_n_arg [range(%out_n_arg, 1, 65536)] : index
%other = index.assume %other_arg [range(%other_arg, 1, 65536)] : index
%count = index.mul %rows_i, %out_n : index
%c256 = index.constant 256 : index
%gid = kernel.workgroup.id<x> : index
%lid = kernel.workitem.id<x> : index
%i = index.madd %gid, %c256, %lid : index
%live = index.cmp ult, %i, %count : index
scf.if %live {
%zero = index.constant 0 : offset
%c0 = index.constant 0 : index
%c1 = index.constant 1 : index
%c3 = index.constant 3 : index
%ccin = index.add %cin_i, %c0 : index
%zf = scalar.constant 0.0 : f32
%onef = scalar.constant 1.0 : f32
%halff = scalar.constant 0.5 : f32
%twof = scalar.constant 2.0 : f32
%threef = scalar.constant 3.0 : f32
%m3f = scalar.constant -3.0 : f32
%fourf = scalar.constant 4.0 : f32
%fivef = scalar.constant 5.0 : f32
%eightf = scalar.constant 8.0 : f32
%c1p5 = scalar.constant 1.5 : f32
%c2p5 = scalar.constant 2.5 : f32
%mhalf = scalar.constant -0.5 : f32
%pif = scalar.constant 3.141592653589793 : f32
%tinyf = scalar.constant 1.0e-12 : f32
%fxp = scalar.constant 4194304.0 : f32
%o = index.rem %i, %out_n : index
%row = index.div %i, %out_n : index
%in_i = index.cast %in_n : index to i32
%out_i = index.cast %out_n : index to i32
%in_f = scalar.sitofp %in_i : i32 to f32
%out_f = scalar.sitofp %out_i : i32 to f32
%scale = scalar.divf %in_f, %out_f : f32
%fs = scalar.maxnumf %scale, %onef : f32
%supc = scalar.constant 2.0 : f32
%support = scalar.mulf %supc, %fs : f32
%ss = scalar.divf %onef, %fs : f32
%oi = index.cast %o : index to i32
%of = scalar.sitofp %oi : i32 to f32
%oh = scalar.addf %of, %halff : f32
%center = scalar.mulf %oh, %scale : f32
%lo0 = scalar.subf %center, %support : f32
%lo1 = scalar.addf %lo0, %halff : f32
%lo2 = scalar.fptosi %lo1 : f32 to i32
%zi = scalar.constant 0 : i32
%lo3 = scalar.maxsi %lo2, %zi : i32
%hi0 = scalar.addf %center, %support : f32
%hi1 = scalar.addf %hi0, %halff : f32
%hi2 = scalar.fptosi %hi1 : f32 to i32
%hi3 = scalar.minsi %hi2, %in_i : i32
%lo4 = scalar.assume %lo3 [range(%lo3, 0, 65536)] : i32
%hi4 = scalar.assume %hi3 [range(%hi3, 0, 65536)] : i32
%xlo = index.cast %lo4 : i32 to index
%xhi = index.cast %hi4 : i32 to index
%ww = scf.for %xi = [%xlo to %xhi step %c1](%acc = %zf : f32) -> (f32) {
%xii = index.cast %xi : index to i32
%xf = scalar.sitofp %xii : i32 to f32
%d0 = scalar.subf %xf, %center : f32
%d1 = scalar.addf %d0, %halff : f32
%d = scalar.mulf %d1, %ss : f32
%wv_x = scalar.absf %d : f32
%wv_lt1 = scalar.cmpf olt, %wv_x, %onef : f32
%wv_lt2 = scalar.cmpf olt, %wv_x, %twof : f32
%wv_p1a = scalar.mulf %wv_x, %c1p5 : f32
%wv_p1b = scalar.subf %wv_p1a, %c2p5 : f32
%wv_p1c = scalar.mulf %wv_p1b, %wv_x : f32
%wv_p1d = scalar.mulf %wv_p1c, %wv_x : f32
%wv_p1 = scalar.addf %wv_p1d, %onef : f32
%wv_p2a = scalar.subf %wv_x, %fivef : f32
%wv_p2b = scalar.mulf %wv_p2a, %wv_x : f32
%wv_p2c = scalar.addf %wv_p2b, %eightf : f32
%wv_p2d = scalar.mulf %wv_p2c, %wv_x : f32
%wv_p2e = scalar.subf %wv_p2d, %fourf : f32
%wv_p2 = scalar.mulf %wv_p2e, %mhalf : f32
%wv_q = scf.select %wv_lt2, %wv_p2, %zf : f32
%wv = scf.select %wv_lt1, %wv_p1, %wv_q : f32
%na = scalar.addf %acc, %wv : f32
scf.yield %na : f32
}
%wnz = scalar.cmpf one, %ww, %zf : f32
%wdiv = scf.select %wnz, %ww, %onef : f32
%srcn = index.constant 268435456 : index
%sv = buffer.view %src[%zero] : buffer -> view<[%srcn]xi8>
%bias = scalar.constant 2097152 : i32
%s0, %s1, %s2 = scf.for %xi = [%xlo to %xhi step %c1](%a0 = %bias : i32, %a1 = %bias : i32, %a2 = %bias : i32) -> (i32, i32, i32) {
%xii = index.cast %xi : index to i32
%xf = scalar.sitofp %xii : i32 to f32
%d0 = scalar.subf %xf, %center : f32
%d1 = scalar.addf %d0, %halff : f32
%d = scalar.mulf %d1, %ss : f32
%wv_x = scalar.absf %d : f32
%wv_lt1 = scalar.cmpf olt, %wv_x, %onef : f32
%wv_lt2 = scalar.cmpf olt, %wv_x, %twof : f32
%wv_p1a = scalar.mulf %wv_x, %c1p5 : f32
%wv_p1b = scalar.subf %wv_p1a, %c2p5 : f32
%wv_p1c = scalar.mulf %wv_p1b, %wv_x : f32
%wv_p1d = scalar.mulf %wv_p1c, %wv_x : f32
%wv_p1 = scalar.addf %wv_p1d, %onef : f32
%wv_p2a = scalar.subf %wv_x, %fivef : f32
%wv_p2b = scalar.mulf %wv_p2a, %wv_x : f32
%wv_p2c = scalar.addf %wv_p2b, %eightf : f32
%wv_p2d = scalar.mulf %wv_p2c, %wv_x : f32
%wv_p2e = scalar.subf %wv_p2d, %fourf : f32
%wv_p2 = scalar.mulf %wv_p2e, %mhalf : f32
%wv_q = scf.select %wv_lt2, %wv_p2, %zf : f32
%wv = scf.select %wv_lt1, %wv_p1, %wv_q : f32
%wn = scalar.divf %wv, %wdiv : f32
%wfx = scalar.mulf %wn, %fxp : f32
%neg = scalar.cmpf olt, %wn, %zf : f32
%rnd = scf.select %neg, %mhalf, %halff : f32
%wr = scalar.addf %wfx, %rnd : f32
%wi = scalar.fptosi %wr : f32 to i32
%src_idx = index.madd %row, %in_n, %xi : index
%el_rel = index.mul %src_idx, %ccin : index
%el0 = index.add %el_rel, %pix_i : index
%el = index.assume %el0 [lt(%el0, %srcn)] : index
%el1r = index.add %el0, %c1 : index
%el1 = index.assume %el1r [lt(%el1r, %srcn)] : index
%el2r = index.add %el1r, %c1 : index
%el2 = index.assume %el2r [lt(%el2r, %srcn)] : index
%p0r = view.load %sv[%el] : view<[%srcn]xi8> -> i8
%p1r = view.load %sv[%el1] : view<[%srcn]xi8> -> i8
%p2r = view.load %sv[%el2] : view<[%srcn]xi8> -> i8
%p0 = scalar.extui %p0r : i8 to i32
%p1 = scalar.extui %p1r : i8 to i32
%p2 = scalar.extui %p2r : i8 to i32
%m0 = scalar.muli %p0, %wi : i32
%m1 = scalar.muli %p1, %wi : i32
%m2 = scalar.muli %p2, %wi : i32
%n0 = scalar.addi %a0, %m0 : i32
%n1 = scalar.addi %a1, %m1 : i32
%n2 = scalar.addi %a2, %m2 : i32
scf.yield %n0, %n1, %n2 : i32, i32, i32
}
%sh = scalar.constant 22 : i32
%c255i = scalar.constant 255 : i32
%dst_idx = index.madd %row, %out_n, %o : index
%dstn = index.constant 268435456 : index
%dv = buffer.view %dst[%zero] : buffer -> view<[%dstn]xi8>
%de0 = index.mul %dst_idx, %c3 : index
%c2x = index.constant 2 : index
%q0a = scalar.shrsi %s0, %sh : i32
%q0b = scalar.maxsi %q0a, %zi : i32
%q0c = scalar.minsi %q0b, %c255i : i32
%q0 = scalar.trunci %q0c : i32 to i8
%dc0r = index.add %de0, %c0 : index
%dc0 = index.assume %dc0r [lt(%dc0r, %dstn)] : index
view.store %q0, %dv[%dc0] : i8, view<[%dstn]xi8>
%q1a = scalar.shrsi %s1, %sh : i32
%q1b = scalar.maxsi %q1a, %zi : i32
%q1c = scalar.minsi %q1b, %c255i : i32
%q1 = scalar.trunci %q1c : i32 to i8
%dc1r = index.add %de0, %c1 : index
%dc1 = index.assume %dc1r [lt(%dc1r, %dstn)] : index
view.store %q1, %dv[%dc1] : i8, view<[%dstn]xi8>
%q2a = scalar.shrsi %s2, %sh : i32
%q2b = scalar.maxsi %q2a, %zi : i32
%q2c = scalar.minsi %q2b, %c255i : i32
%q2 = scalar.trunci %q2c : i32 to i8
%dc2r = index.add %de0, %c2x : index
%dc2 = index.assume %dc2r [lt(%dc2r, %dstn)] : index
view.store %q2, %dv[%dc2] : i8, view<[%dstn]xi8>
}
kernel.return
}
kernel.def export("v_resize_bicubic_v_hdr") @v_resize_bicubic_v_hdr(%out_h: index, %cols: index) {
%count = index.mul %out_h, %cols : index
%c1 = index.constant 1 : index
%c256 = index.constant 256 : index
%c255 = index.constant 255 : index
%n_up = index.add %count, %c255 : index
%groups = index.div %n_up, %c256 : index
kernel.launch.config workgroups(%groups, %c1, %c1) workgroup_size(%c256, %c1, %c1) : index
} launch(%out_n_arg: index, %other_arg: index, %hdr: buffer, %src: buffer, %dst: buffer) {
%zero_h = index.constant 0 : offset
%hv = buffer.view %hdr[%zero_h] : buffer -> view<16xi32>
%in_w32 = view.load %hv[1] : view<16xi32> -> i32
%rows32 = scalar.constant 1 : i32
%cin32 = scalar.constant 3 : i32
%pix32 = scalar.constant 0 : i32
%in_b = scalar.assume %in_w32 [range(%in_w32, 1, 16384)] : i32
%in_n = index.cast %in_b : i32 to index
%rows_b = scalar.assume %rows32 [range(%rows32, 1, 16384)] : i32
%rows_i = index.cast %rows_b : i32 to index
%cin_b = scalar.assume %cin32 [range(%cin32, 3, 4)] : i32
%cin_i = index.cast %cin_b : i32 to index
%pix_b = scalar.assume %pix32 [range(%pix32, 0, 2147483647)] : i32
%pix_i = index.cast %pix_b : i32 to index
%out_n = index.assume %out_n_arg [range(%out_n_arg, 1, 65536)] : index
%other = index.assume %other_arg [range(%other_arg, 1, 65536)] : index
%count = index.mul %other, %out_n : index
%c256 = index.constant 256 : index
%gid = kernel.workgroup.id<x> : index
%lid = kernel.workitem.id<x> : index
%i = index.madd %gid, %c256, %lid : index
%live = index.cmp ult, %i, %count : index
scf.if %live {
%zero = index.constant 0 : offset
%c0 = index.constant 0 : index
%c1 = index.constant 1 : index
%c3 = index.constant 3 : index
%ccin = index.constant 3 : index
%zf = scalar.constant 0.0 : f32
%onef = scalar.constant 1.0 : f32
%halff = scalar.constant 0.5 : f32
%twof = scalar.constant 2.0 : f32
%threef = scalar.constant 3.0 : f32
%m3f = scalar.constant -3.0 : f32
%fourf = scalar.constant 4.0 : f32
%fivef = scalar.constant 5.0 : f32
%eightf = scalar.constant 8.0 : f32
%c1p5 = scalar.constant 1.5 : f32
%c2p5 = scalar.constant 2.5 : f32
%mhalf = scalar.constant -0.5 : f32
%pif = scalar.constant 3.141592653589793 : f32
%tinyf = scalar.constant 1.0e-12 : f32
%fxp = scalar.constant 4194304.0 : f32
%o_v = index.div %i, %other : index
%col = index.rem %i, %other : index
%in_i = index.cast %in_n : index to i32
%out_i = index.cast %out_n : index to i32
%in_f = scalar.sitofp %in_i : i32 to f32
%out_f = scalar.sitofp %out_i : i32 to f32
%scale = scalar.divf %in_f, %out_f : f32
%fs = scalar.maxnumf %scale, %onef : f32
%supc = scalar.constant 2.0 : f32
%support = scalar.mulf %supc, %fs : f32
%ss = scalar.divf %onef, %fs : f32
%oi = index.cast %o_v : index to i32
%of = scalar.sitofp %oi : i32 to f32
%oh = scalar.addf %of, %halff : f32
%center = scalar.mulf %oh, %scale : f32
%lo0 = scalar.subf %center, %support : f32
%lo1 = scalar.addf %lo0, %halff : f32
%lo2 = scalar.fptosi %lo1 : f32 to i32
%zi = scalar.constant 0 : i32
%lo3 = scalar.maxsi %lo2, %zi : i32
%hi0 = scalar.addf %center, %support : f32
%hi1 = scalar.addf %hi0, %halff : f32
%hi2 = scalar.fptosi %hi1 : f32 to i32
%hi3 = scalar.minsi %hi2, %in_i : i32
%lo4 = scalar.assume %lo3 [range(%lo3, 0, 65536)] : i32
%hi4 = scalar.assume %hi3 [range(%hi3, 0, 65536)] : i32
%xlo = index.cast %lo4 : i32 to index
%xhi = index.cast %hi4 : i32 to index
%ww = scf.for %xi = [%xlo to %xhi step %c1](%acc = %zf : f32) -> (f32) {
%xii = index.cast %xi : index to i32
%xf = scalar.sitofp %xii : i32 to f32
%d0 = scalar.subf %xf, %center : f32
%d1 = scalar.addf %d0, %halff : f32
%d = scalar.mulf %d1, %ss : f32
%wv_x = scalar.absf %d : f32
%wv_lt1 = scalar.cmpf olt, %wv_x, %onef : f32
%wv_lt2 = scalar.cmpf olt, %wv_x, %twof : f32
%wv_p1a = scalar.mulf %wv_x, %c1p5 : f32
%wv_p1b = scalar.subf %wv_p1a, %c2p5 : f32
%wv_p1c = scalar.mulf %wv_p1b, %wv_x : f32
%wv_p1d = scalar.mulf %wv_p1c, %wv_x : f32
%wv_p1 = scalar.addf %wv_p1d, %onef : f32
%wv_p2a = scalar.subf %wv_x, %fivef : f32
%wv_p2b = scalar.mulf %wv_p2a, %wv_x : f32
%wv_p2c = scalar.addf %wv_p2b, %eightf : f32
%wv_p2d = scalar.mulf %wv_p2c, %wv_x : f32
%wv_p2e = scalar.subf %wv_p2d, %fourf : f32
%wv_p2 = scalar.mulf %wv_p2e, %mhalf : f32
%wv_q = scf.select %wv_lt2, %wv_p2, %zf : f32
%wv = scf.select %wv_lt1, %wv_p1, %wv_q : f32
%na = scalar.addf %acc, %wv : f32
scf.yield %na : f32
}
%wnz = scalar.cmpf one, %ww, %zf : f32
%wdiv = scf.select %wnz, %ww, %onef : f32
%srcn = index.constant 268435456 : index
%sv = buffer.view %src[%zero] : buffer -> view<[%srcn]xi8>
%bias = scalar.constant 2097152 : i32
%s0, %s1, %s2 = scf.for %xi = [%xlo to %xhi step %c1](%a0 = %bias : i32, %a1 = %bias : i32, %a2 = %bias : i32) -> (i32, i32, i32) {
%xii = index.cast %xi : index to i32
%xf = scalar.sitofp %xii : i32 to f32
%d0 = scalar.subf %xf, %center : f32
%d1 = scalar.addf %d0, %halff : f32
%d = scalar.mulf %d1, %ss : f32
%wv_x = scalar.absf %d : f32
%wv_lt1 = scalar.cmpf olt, %wv_x, %onef : f32
%wv_lt2 = scalar.cmpf olt, %wv_x, %twof : f32
%wv_p1a = scalar.mulf %wv_x, %c1p5 : f32
%wv_p1b = scalar.subf %wv_p1a, %c2p5 : f32
%wv_p1c = scalar.mulf %wv_p1b, %wv_x : f32
%wv_p1d = scalar.mulf %wv_p1c, %wv_x : f32
%wv_p1 = scalar.addf %wv_p1d, %onef : f32
%wv_p2a = scalar.subf %wv_x, %fivef : f32
%wv_p2b = scalar.mulf %wv_p2a, %wv_x : f32
%wv_p2c = scalar.addf %wv_p2b, %eightf : f32
%wv_p2d = scalar.mulf %wv_p2c, %wv_x : f32
%wv_p2e = scalar.subf %wv_p2d, %fourf : f32
%wv_p2 = scalar.mulf %wv_p2e, %mhalf : f32
%wv_q = scf.select %wv_lt2, %wv_p2, %zf : f32
%wv = scf.select %wv_lt1, %wv_p1, %wv_q : f32
%wn = scalar.divf %wv, %wdiv : f32
%wfx = scalar.mulf %wn, %fxp : f32
%neg = scalar.cmpf olt, %wn, %zf : f32
%rnd = scf.select %neg, %mhalf, %halff : f32
%wr = scalar.addf %wfx, %rnd : f32
%wi = scalar.fptosi %wr : f32 to i32
%src_idx = index.madd %xi, %other, %col : index
%el0 = index.mul %src_idx, %ccin : index
%el = index.assume %el0 [lt(%el0, %srcn)] : index
%el1r = index.add %el0, %c1 : index
%el1 = index.assume %el1r [lt(%el1r, %srcn)] : index
%el2r = index.add %el1r, %c1 : index
%el2 = index.assume %el2r [lt(%el2r, %srcn)] : index
%p0r = view.load %sv[%el] : view<[%srcn]xi8> -> i8
%p1r = view.load %sv[%el1] : view<[%srcn]xi8> -> i8
%p2r = view.load %sv[%el2] : view<[%srcn]xi8> -> i8
%p0 = scalar.extui %p0r : i8 to i32
%p1 = scalar.extui %p1r : i8 to i32
%p2 = scalar.extui %p2r : i8 to i32
%m0 = scalar.muli %p0, %wi : i32
%m1 = scalar.muli %p1, %wi : i32
%m2 = scalar.muli %p2, %wi : i32
%n0 = scalar.addi %a0, %m0 : i32
%n1 = scalar.addi %a1, %m1 : i32
%n2 = scalar.addi %a2, %m2 : i32
scf.yield %n0, %n1, %n2 : i32, i32, i32
}
%sh = scalar.constant 22 : i32
%c255i = scalar.constant 255 : i32
%dst_idx = index.madd %o_v, %other, %col : index
%dstn = index.constant 268435456 : index
%dv = buffer.view %dst[%zero] : buffer -> view<[%dstn]xi8>
%de0 = index.mul %dst_idx, %c3 : index
%c2x = index.constant 2 : index
%q0a = scalar.shrsi %s0, %sh : i32
%q0b = scalar.maxsi %q0a, %zi : i32
%q0c = scalar.minsi %q0b, %c255i : i32
%q0 = scalar.trunci %q0c : i32 to i8
%dc0r = index.add %de0, %c0 : index
%dc0 = index.assume %dc0r [lt(%dc0r, %dstn)] : index
view.store %q0, %dv[%dc0] : i8, view<[%dstn]xi8>
%q1a = scalar.shrsi %s1, %sh : i32
%q1b = scalar.maxsi %q1a, %zi : i32
%q1c = scalar.minsi %q1b, %c255i : i32
%q1 = scalar.trunci %q1c : i32 to i8
%dc1r = index.add %de0, %c1 : index
%dc1 = index.assume %dc1r [lt(%dc1r, %dstn)] : index
view.store %q1, %dv[%dc1] : i8, view<[%dstn]xi8>
%q2a = scalar.shrsi %s2, %sh : i32
%q2b = scalar.maxsi %q2a, %zi : i32
%q2c = scalar.minsi %q2b, %c255i : i32
%q2 = scalar.trunci %q2c : i32 to i8
%dc2r = index.add %de0, %c2x : index
%dc2 = index.assume %dc2r [lt(%dc2r, %dstn)] : index
view.store %q2, %dv[%dc2] : i8, view<[%dstn]xi8>
}
kernel.return
}
kernel.def export("v_copy") @v_copy(%n: index) {
%c1 = index.constant 1 : index
%c256 = index.constant 256 : index
%c255 = index.constant 255 : index
%n_up = index.add %n, %c255 : index
%groups = index.div %n_up, %c256 : index
kernel.launch.config workgroups(%groups, %c1, %c1) workgroup_size(%c256, %c1, %c1) : index
} launch(%n_arg: index, %x: buffer, %y: buffer) {
%n = index.assume %n_arg [range(%n_arg, 1, 268435456)] : index
%c256 = index.constant 256 : index
%gid = kernel.workgroup.id<x> : index
%lid = kernel.workitem.id<x> : index
%i = index.madd %gid, %c256, %lid : index
%live = index.cmp ult, %i, %n : index
scf.if %live {
%bi = index.assume %i [lt(%i, %n)] : index
%zero = index.constant 0 : offset
%xv = buffer.view %x[%zero] : buffer -> view<[%n]xf32>
%a = view.load %xv[%bi] : view<[%n]xf32> -> f32
%yv = buffer.view %y[%zero] : buffer -> view<[%n]xf32>
view.store %a, %yv[%bi] : f32, view<[%n]xf32>
}
kernel.return
}
config.decl @vision.height : %value: index where [range(%value, 1, 16384)]
config.decl @vision.width : %value: index where [range(%value, 1, 16384)]
command.program.def public @vision_encode() launch(%weights: buffer, %input: buffer, %output: buffer) {
%img_h = config.get @vision.height : index
%img_w = config.get @vision.width : index
%cP = index.constant 16 : index
%cm = index.constant 2 : index
%c1 = index.constant 1 : index
%c3 = index.constant 3 : index
%c63 = index.constant 63 : index
%c64 = index.constant 64 : index
%c255 = index.constant 255 : index
%c256 = index.constant 256 : index
%gh = index.div %img_h, %cP : index
%gw = index.div %img_w, %cP : index
%T = index.mul %gh, %gw : index
%mh = index.div %gh, %cm : index
%mw = index.div %gw, %cm : index
%M = index.mul %mh, %mw : index
%T_up = index.add %T, %c255 : index
%T_t = index.div %T_up, %c256 : index
%Tp = index.mul %T_t, %c256 : index
%M_up = index.add %M, %c63 : index
%M_t = index.div %M_up, %c64 : index
%Mp = index.mul %M_t, %c64 : index
%NH = index.constant 16 : index
%img_px = index.mul %img_h, %img_w : index
%n_pix = index.mul %img_px, %c3 : index
%max_rows = index.constant 4096 : index
%hdr_bytes = index.constant 64 : offset
%zero_o = index.constant 0 : offset
%h_w = index.constant 1152 : index
%h_n = index.mul %Tp, %h_w : index
%h_bpe = index.constant 4 : offset
%h_size = index.scale %h_n, %h_bpe : index, offset -> offset
%h = buffer.alloca<global> align(256) %h_size : buffer
%x_w = index.constant 4608 : index
%x_n = index.mul %Tp, %x_w : index
%x_bpe = index.constant 4 : offset
%x_size = index.scale %x_n, %x_bpe : index, offset -> offset
%x = buffer.alloca<global> align(256) %x_size : buffer
%q_w = index.constant 3456 : index
%q_n = index.mul %Tp, %q_w : index
%q_bpe = index.constant 4 : offset
%q_size = index.scale %q_n, %q_bpe : index, offset -> offset
%q = buffer.alloca<global> align(256) %q_size : buffer
%k_w = index.constant 1152 : index
%k_n = index.mul %Tp, %k_w : index
%k_bpe = index.constant 4 : offset
%k_size = index.scale %k_n, %k_bpe : index, offset -> offset
%k = buffer.alloca<global> align(256) %k_size : buffer
%v_w = index.constant 1152 : index
%v_n = index.mul %Tp, %v_w : index
%v_bpe = index.constant 4 : offset
%v_size = index.scale %v_n, %v_bpe : index, offset -> offset
%v = buffer.alloca<global> align(256) %v_size : buffer
%o_w = index.constant 1152 : index
%o_n = index.mul %Tp, %o_w : index
%o_bpe = index.constant 4 : offset
%o_size = index.scale %o_n, %o_bpe : index, offset -> offset
%o = buffer.alloca<global> align(256) %o_size : buffer
%a_w = index.constant 1152 : index
%a_n = index.mul %Tp, %a_w : index
%a_bpe = index.constant 4 : offset
%a_size = index.scale %a_n, %a_bpe : index, offset -> offset
%a = buffer.alloca<global> align(256) %a_size : buffer
%f_w = index.constant 4304 : index
%f_n = index.mul %Tp, %f_w : index
%f_bpe = index.constant 4 : offset
%f_size = index.scale %f_n, %f_bpe : index, offset -> offset
%f = buffer.alloca<global> align(256) %f_size : buffer
%g_w = index.constant 4304 : index
%g_n = index.mul %Tp, %g_w : index
%g_bpe = index.constant 4 : offset
%g_size = index.scale %g_n, %g_bpe : index, offset -> offset
%g = buffer.alloca<global> align(256) %g_size : buffer
%xs_w = index.constant 4608 : index
%xs_n = index.mul %Tp, %xs_w : index
%xs_bpe = index.constant 4 : offset
%xs_size = index.scale %xs_n, %xs_bpe : index, offset -> offset
%xs = buffer.alloca<global> align(256) %xs_size : buffer
%x16_w = index.constant 4608 : index
%x16_n = index.mul %Tp, %x16_w : index
%x16_bpe = index.constant 2 : offset
%x16_size = index.scale %x16_n, %x16_bpe : index, offset -> offset
%x16 = buffer.alloca<global> align(256) %x16_size : buffer
%sc_w = index.constant 2 : index
%sc_n = index.mul %Tp, %sc_w : index
%sc_bpe = index.constant 4 : offset
%sc_size = index.scale %sc_n, %sc_bpe : index, offset -> offset
%sc = buffer.alloca<global> align(256) %sc_size : buffer
%m_in_w = index.constant 4608 : index
%m_in_n = index.mul %Mp, %m_in_w : index
%m_in_bpe = index.constant 4 : offset
%m_in_size = index.scale %m_in_n, %m_in_bpe : index, offset -> offset
%m_in = buffer.alloca<global> align(256) %m_in_size : buffer
%m1_w = index.constant 4608 : index
%m1_n = index.mul %Mp, %m1_w : index
%m1_bpe = index.constant 4 : offset
%m1_size = index.scale %m1_n, %m1_bpe : index, offset -> offset
%m1 = buffer.alloca<global> align(256) %m1_size : buffer
%m2_w = index.constant 4608 : index
%m2_n = index.mul %Mp, %m2_w : index
%m2_bpe = index.constant 4 : offset
%m2_size = index.scale %m2_n, %m2_bpe : index, offset -> offset
%m2 = buffer.alloca<global> align(256) %m2_size : buffer
%out_w = index.constant 2048 : index
%out_n = index.mul %Mp, %out_w : index
%out_bpe = index.constant 4 : offset
%out_size = index.scale %out_n, %out_bpe : index, offset -> offset
%out = buffer.alloca<global> align(256) %out_size : buffer
%wide_n = index.mul %max_rows, %img_w : index
%wide_n3 = index.mul %wide_n, %c3 : index
%wide_bpe = index.constant 1 : offset
%wide_size = index.scale %wide_n3, %wide_bpe : index, offset -> offset
%wide = buffer.alloca<global> align(256) %wide_size : buffer
%pix_size = index.scale %n_pix, %wide_bpe : index, offset -> offset
%pix = buffer.alloca<global> align(256) %pix_size : buffer
%pixf_bpe = index.constant 4 : offset
%pixf_size = index.scale %n_pix, %pixf_bpe : index, offset -> offset
%pixf = buffer.alloca<global> align(256) %pixf_size : buffer
kernel.launch @v_resize_bicubic_h_hdr[%img_w, %max_rows](%img_w, %max_rows, %input, %wide) : [index, index](index, index, buffer, buffer)
kernel.launch @v_resize_bicubic_v_hdr[%img_h, %img_w](%img_h, %img_w, %input, %wide, %pix) : [index, index](index, index, buffer, buffer, buffer)
%w_v_patch_embd_weight = command.parameter %weights, "v.patch_embd.weight" : view<1769472xi8>
%w_v_patch_embd_weight_1 = command.parameter %weights, "v.patch_embd.weight.1" : view<1769472xi8>
%w_v_patch_embd_bias = command.parameter %weights, "v.patch_embd.bias" : view<4608xi8>
%w_v_position_embd_weight = command.parameter %weights, "v.position_embd.weight" : view<10616832xi8>
%w_v_blk_0_ln1_weight = command.parameter %weights, "v.blk.0.ln1.weight" : view<4608xi8>
%w_v_blk_0_ln1_bias = command.parameter %weights, "v.blk.0.ln1.bias" : view<4608xi8>
%w_v_blk_0_attn_qkv_weight = command.parameter %weights, "v.blk.0.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_0_attn_qkv_bias = command.parameter %weights, "v.blk.0.attn_qkv.bias" : view<13824xi8>
%w_v_blk_0_attn_out_weight = command.parameter %weights, "v.blk.0.attn_out.weight" : view<2654208xi8>
%w_v_blk_0_attn_out_bias = command.parameter %weights, "v.blk.0.attn_out.bias" : view<4608xi8>
%w_v_blk_0_ln2_weight = command.parameter %weights, "v.blk.0.ln2.weight" : view<4608xi8>
%w_v_blk_0_ln2_bias = command.parameter %weights, "v.blk.0.ln2.bias" : view<4608xi8>
%w_v_blk_0_ffn_up_weight = command.parameter %weights, "v.blk.0.ffn_up.weight" : view<9916416xi8>
%w_v_blk_0_ffn_up_bias = command.parameter %weights, "v.blk.0.ffn_up.bias" : view<17216xi8>
%w_v_blk_0_ffn_down_weight = command.parameter %weights, "v.blk.0.ffn_down.weight" : view<9916416xi8>
%w_v_blk_0_ffn_down_bias = command.parameter %weights, "v.blk.0.ffn_down.bias" : view<4608xi8>
%w_v_blk_1_ln1_weight = command.parameter %weights, "v.blk.1.ln1.weight" : view<4608xi8>
%w_v_blk_1_ln1_bias = command.parameter %weights, "v.blk.1.ln1.bias" : view<4608xi8>
%w_v_blk_1_attn_qkv_weight = command.parameter %weights, "v.blk.1.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_1_attn_qkv_bias = command.parameter %weights, "v.blk.1.attn_qkv.bias" : view<13824xi8>
%w_v_blk_1_attn_out_weight = command.parameter %weights, "v.blk.1.attn_out.weight" : view<2654208xi8>
%w_v_blk_1_attn_out_bias = command.parameter %weights, "v.blk.1.attn_out.bias" : view<4608xi8>
%w_v_blk_1_ln2_weight = command.parameter %weights, "v.blk.1.ln2.weight" : view<4608xi8>
%w_v_blk_1_ln2_bias = command.parameter %weights, "v.blk.1.ln2.bias" : view<4608xi8>
%w_v_blk_1_ffn_up_weight = command.parameter %weights, "v.blk.1.ffn_up.weight" : view<9916416xi8>
%w_v_blk_1_ffn_up_bias = command.parameter %weights, "v.blk.1.ffn_up.bias" : view<17216xi8>
%w_v_blk_1_ffn_down_weight = command.parameter %weights, "v.blk.1.ffn_down.weight" : view<9916416xi8>
%w_v_blk_1_ffn_down_bias = command.parameter %weights, "v.blk.1.ffn_down.bias" : view<4608xi8>
%w_v_blk_2_ln1_weight = command.parameter %weights, "v.blk.2.ln1.weight" : view<4608xi8>
%w_v_blk_2_ln1_bias = command.parameter %weights, "v.blk.2.ln1.bias" : view<4608xi8>
%w_v_blk_2_attn_qkv_weight = command.parameter %weights, "v.blk.2.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_2_attn_qkv_bias = command.parameter %weights, "v.blk.2.attn_qkv.bias" : view<13824xi8>
%w_v_blk_2_attn_out_weight = command.parameter %weights, "v.blk.2.attn_out.weight" : view<2654208xi8>
%w_v_blk_2_attn_out_bias = command.parameter %weights, "v.blk.2.attn_out.bias" : view<4608xi8>
%w_v_blk_2_ln2_weight = command.parameter %weights, "v.blk.2.ln2.weight" : view<4608xi8>
%w_v_blk_2_ln2_bias = command.parameter %weights, "v.blk.2.ln2.bias" : view<4608xi8>
%w_v_blk_2_ffn_up_weight = command.parameter %weights, "v.blk.2.ffn_up.weight" : view<9916416xi8>
%w_v_blk_2_ffn_up_bias = command.parameter %weights, "v.blk.2.ffn_up.bias" : view<17216xi8>
%w_v_blk_2_ffn_down_weight = command.parameter %weights, "v.blk.2.ffn_down.weight" : view<9916416xi8>
%w_v_blk_2_ffn_down_bias = command.parameter %weights, "v.blk.2.ffn_down.bias" : view<4608xi8>
%w_v_blk_3_ln1_weight = command.parameter %weights, "v.blk.3.ln1.weight" : view<4608xi8>
%w_v_blk_3_ln1_bias = command.parameter %weights, "v.blk.3.ln1.bias" : view<4608xi8>
%w_v_blk_3_attn_qkv_weight = command.parameter %weights, "v.blk.3.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_3_attn_qkv_bias = command.parameter %weights, "v.blk.3.attn_qkv.bias" : view<13824xi8>
%w_v_blk_3_attn_out_weight = command.parameter %weights, "v.blk.3.attn_out.weight" : view<2654208xi8>
%w_v_blk_3_attn_out_bias = command.parameter %weights, "v.blk.3.attn_out.bias" : view<4608xi8>
%w_v_blk_3_ln2_weight = command.parameter %weights, "v.blk.3.ln2.weight" : view<4608xi8>
%w_v_blk_3_ln2_bias = command.parameter %weights, "v.blk.3.ln2.bias" : view<4608xi8>
%w_v_blk_3_ffn_up_weight = command.parameter %weights, "v.blk.3.ffn_up.weight" : view<9916416xi8>
%w_v_blk_3_ffn_up_bias = command.parameter %weights, "v.blk.3.ffn_up.bias" : view<17216xi8>
%w_v_blk_3_ffn_down_weight = command.parameter %weights, "v.blk.3.ffn_down.weight" : view<9916416xi8>
%w_v_blk_3_ffn_down_bias = command.parameter %weights, "v.blk.3.ffn_down.bias" : view<4608xi8>
%w_v_blk_4_ln1_weight = command.parameter %weights, "v.blk.4.ln1.weight" : view<4608xi8>
%w_v_blk_4_ln1_bias = command.parameter %weights, "v.blk.4.ln1.bias" : view<4608xi8>
%w_v_blk_4_attn_qkv_weight = command.parameter %weights, "v.blk.4.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_4_attn_qkv_bias = command.parameter %weights, "v.blk.4.attn_qkv.bias" : view<13824xi8>
%w_v_blk_4_attn_out_weight = command.parameter %weights, "v.blk.4.attn_out.weight" : view<2654208xi8>
%w_v_blk_4_attn_out_bias = command.parameter %weights, "v.blk.4.attn_out.bias" : view<4608xi8>
%w_v_blk_4_ln2_weight = command.parameter %weights, "v.blk.4.ln2.weight" : view<4608xi8>
%w_v_blk_4_ln2_bias = command.parameter %weights, "v.blk.4.ln2.bias" : view<4608xi8>
%w_v_blk_4_ffn_up_weight = command.parameter %weights, "v.blk.4.ffn_up.weight" : view<9916416xi8>
%w_v_blk_4_ffn_up_bias = command.parameter %weights, "v.blk.4.ffn_up.bias" : view<17216xi8>
%w_v_blk_4_ffn_down_weight = command.parameter %weights, "v.blk.4.ffn_down.weight" : view<9916416xi8>
%w_v_blk_4_ffn_down_bias = command.parameter %weights, "v.blk.4.ffn_down.bias" : view<4608xi8>
%w_v_blk_5_ln1_weight = command.parameter %weights, "v.blk.5.ln1.weight" : view<4608xi8>
%w_v_blk_5_ln1_bias = command.parameter %weights, "v.blk.5.ln1.bias" : view<4608xi8>
%w_v_blk_5_attn_qkv_weight = command.parameter %weights, "v.blk.5.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_5_attn_qkv_bias = command.parameter %weights, "v.blk.5.attn_qkv.bias" : view<13824xi8>
%w_v_blk_5_attn_out_weight = command.parameter %weights, "v.blk.5.attn_out.weight" : view<2654208xi8>
%w_v_blk_5_attn_out_bias = command.parameter %weights, "v.blk.5.attn_out.bias" : view<4608xi8>
%w_v_blk_5_ln2_weight = command.parameter %weights, "v.blk.5.ln2.weight" : view<4608xi8>
%w_v_blk_5_ln2_bias = command.parameter %weights, "v.blk.5.ln2.bias" : view<4608xi8>
%w_v_blk_5_ffn_up_weight = command.parameter %weights, "v.blk.5.ffn_up.weight" : view<9916416xi8>
%w_v_blk_5_ffn_up_bias = command.parameter %weights, "v.blk.5.ffn_up.bias" : view<17216xi8>
%w_v_blk_5_ffn_down_weight = command.parameter %weights, "v.blk.5.ffn_down.weight" : view<9916416xi8>
%w_v_blk_5_ffn_down_bias = command.parameter %weights, "v.blk.5.ffn_down.bias" : view<4608xi8>
%w_v_blk_6_ln1_weight = command.parameter %weights, "v.blk.6.ln1.weight" : view<4608xi8>
%w_v_blk_6_ln1_bias = command.parameter %weights, "v.blk.6.ln1.bias" : view<4608xi8>
%w_v_blk_6_attn_qkv_weight = command.parameter %weights, "v.blk.6.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_6_attn_qkv_bias = command.parameter %weights, "v.blk.6.attn_qkv.bias" : view<13824xi8>
%w_v_blk_6_attn_out_weight = command.parameter %weights, "v.blk.6.attn_out.weight" : view<2654208xi8>
%w_v_blk_6_attn_out_bias = command.parameter %weights, "v.blk.6.attn_out.bias" : view<4608xi8>
%w_v_blk_6_ln2_weight = command.parameter %weights, "v.blk.6.ln2.weight" : view<4608xi8>
%w_v_blk_6_ln2_bias = command.parameter %weights, "v.blk.6.ln2.bias" : view<4608xi8>
%w_v_blk_6_ffn_up_weight = command.parameter %weights, "v.blk.6.ffn_up.weight" : view<9916416xi8>
%w_v_blk_6_ffn_up_bias = command.parameter %weights, "v.blk.6.ffn_up.bias" : view<17216xi8>
%w_v_blk_6_ffn_down_weight = command.parameter %weights, "v.blk.6.ffn_down.weight" : view<9916416xi8>
%w_v_blk_6_ffn_down_bias = command.parameter %weights, "v.blk.6.ffn_down.bias" : view<4608xi8>
%w_v_blk_7_ln1_weight = command.parameter %weights, "v.blk.7.ln1.weight" : view<4608xi8>
%w_v_blk_7_ln1_bias = command.parameter %weights, "v.blk.7.ln1.bias" : view<4608xi8>
%w_v_blk_7_attn_qkv_weight = command.parameter %weights, "v.blk.7.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_7_attn_qkv_bias = command.parameter %weights, "v.blk.7.attn_qkv.bias" : view<13824xi8>
%w_v_blk_7_attn_out_weight = command.parameter %weights, "v.blk.7.attn_out.weight" : view<2654208xi8>
%w_v_blk_7_attn_out_bias = command.parameter %weights, "v.blk.7.attn_out.bias" : view<4608xi8>
%w_v_blk_7_ln2_weight = command.parameter %weights, "v.blk.7.ln2.weight" : view<4608xi8>
%w_v_blk_7_ln2_bias = command.parameter %weights, "v.blk.7.ln2.bias" : view<4608xi8>
%w_v_blk_7_ffn_up_weight = command.parameter %weights, "v.blk.7.ffn_up.weight" : view<9916416xi8>
%w_v_blk_7_ffn_up_bias = command.parameter %weights, "v.blk.7.ffn_up.bias" : view<17216xi8>
%w_v_blk_7_ffn_down_weight = command.parameter %weights, "v.blk.7.ffn_down.weight" : view<9916416xi8>
%w_v_blk_7_ffn_down_bias = command.parameter %weights, "v.blk.7.ffn_down.bias" : view<4608xi8>
%w_v_blk_8_ln1_weight = command.parameter %weights, "v.blk.8.ln1.weight" : view<4608xi8>
%w_v_blk_8_ln1_bias = command.parameter %weights, "v.blk.8.ln1.bias" : view<4608xi8>
%w_v_blk_8_attn_qkv_weight = command.parameter %weights, "v.blk.8.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_8_attn_qkv_bias = command.parameter %weights, "v.blk.8.attn_qkv.bias" : view<13824xi8>
%w_v_blk_8_attn_out_weight = command.parameter %weights, "v.blk.8.attn_out.weight" : view<2654208xi8>
%w_v_blk_8_attn_out_bias = command.parameter %weights, "v.blk.8.attn_out.bias" : view<4608xi8>
%w_v_blk_8_ln2_weight = command.parameter %weights, "v.blk.8.ln2.weight" : view<4608xi8>
%w_v_blk_8_ln2_bias = command.parameter %weights, "v.blk.8.ln2.bias" : view<4608xi8>
%w_v_blk_8_ffn_up_weight = command.parameter %weights, "v.blk.8.ffn_up.weight" : view<9916416xi8>
%w_v_blk_8_ffn_up_bias = command.parameter %weights, "v.blk.8.ffn_up.bias" : view<17216xi8>
%w_v_blk_8_ffn_down_weight = command.parameter %weights, "v.blk.8.ffn_down.weight" : view<9916416xi8>
%w_v_blk_8_ffn_down_bias = command.parameter %weights, "v.blk.8.ffn_down.bias" : view<4608xi8>
%w_v_blk_9_ln1_weight = command.parameter %weights, "v.blk.9.ln1.weight" : view<4608xi8>
%w_v_blk_9_ln1_bias = command.parameter %weights, "v.blk.9.ln1.bias" : view<4608xi8>
%w_v_blk_9_attn_qkv_weight = command.parameter %weights, "v.blk.9.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_9_attn_qkv_bias = command.parameter %weights, "v.blk.9.attn_qkv.bias" : view<13824xi8>
%w_v_blk_9_attn_out_weight = command.parameter %weights, "v.blk.9.attn_out.weight" : view<2654208xi8>
%w_v_blk_9_attn_out_bias = command.parameter %weights, "v.blk.9.attn_out.bias" : view<4608xi8>
%w_v_blk_9_ln2_weight = command.parameter %weights, "v.blk.9.ln2.weight" : view<4608xi8>
%w_v_blk_9_ln2_bias = command.parameter %weights, "v.blk.9.ln2.bias" : view<4608xi8>
%w_v_blk_9_ffn_up_weight = command.parameter %weights, "v.blk.9.ffn_up.weight" : view<9916416xi8>
%w_v_blk_9_ffn_up_bias = command.parameter %weights, "v.blk.9.ffn_up.bias" : view<17216xi8>
%w_v_blk_9_ffn_down_weight = command.parameter %weights, "v.blk.9.ffn_down.weight" : view<9916416xi8>
%w_v_blk_9_ffn_down_bias = command.parameter %weights, "v.blk.9.ffn_down.bias" : view<4608xi8>
%w_v_blk_10_ln1_weight = command.parameter %weights, "v.blk.10.ln1.weight" : view<4608xi8>
%w_v_blk_10_ln1_bias = command.parameter %weights, "v.blk.10.ln1.bias" : view<4608xi8>
%w_v_blk_10_attn_qkv_weight = command.parameter %weights, "v.blk.10.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_10_attn_qkv_bias = command.parameter %weights, "v.blk.10.attn_qkv.bias" : view<13824xi8>
%w_v_blk_10_attn_out_weight = command.parameter %weights, "v.blk.10.attn_out.weight" : view<2654208xi8>
%w_v_blk_10_attn_out_bias = command.parameter %weights, "v.blk.10.attn_out.bias" : view<4608xi8>
%w_v_blk_10_ln2_weight = command.parameter %weights, "v.blk.10.ln2.weight" : view<4608xi8>
%w_v_blk_10_ln2_bias = command.parameter %weights, "v.blk.10.ln2.bias" : view<4608xi8>
%w_v_blk_10_ffn_up_weight = command.parameter %weights, "v.blk.10.ffn_up.weight" : view<9916416xi8>
%w_v_blk_10_ffn_up_bias = command.parameter %weights, "v.blk.10.ffn_up.bias" : view<17216xi8>
%w_v_blk_10_ffn_down_weight = command.parameter %weights, "v.blk.10.ffn_down.weight" : view<9916416xi8>
%w_v_blk_10_ffn_down_bias = command.parameter %weights, "v.blk.10.ffn_down.bias" : view<4608xi8>
%w_v_blk_11_ln1_weight = command.parameter %weights, "v.blk.11.ln1.weight" : view<4608xi8>
%w_v_blk_11_ln1_bias = command.parameter %weights, "v.blk.11.ln1.bias" : view<4608xi8>
%w_v_blk_11_attn_qkv_weight = command.parameter %weights, "v.blk.11.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_11_attn_qkv_bias = command.parameter %weights, "v.blk.11.attn_qkv.bias" : view<13824xi8>
%w_v_blk_11_attn_out_weight = command.parameter %weights, "v.blk.11.attn_out.weight" : view<2654208xi8>
%w_v_blk_11_attn_out_bias = command.parameter %weights, "v.blk.11.attn_out.bias" : view<4608xi8>
%w_v_blk_11_ln2_weight = command.parameter %weights, "v.blk.11.ln2.weight" : view<4608xi8>
%w_v_blk_11_ln2_bias = command.parameter %weights, "v.blk.11.ln2.bias" : view<4608xi8>
%w_v_blk_11_ffn_up_weight = command.parameter %weights, "v.blk.11.ffn_up.weight" : view<9916416xi8>
%w_v_blk_11_ffn_up_bias = command.parameter %weights, "v.blk.11.ffn_up.bias" : view<17216xi8>
%w_v_blk_11_ffn_down_weight = command.parameter %weights, "v.blk.11.ffn_down.weight" : view<9916416xi8>
%w_v_blk_11_ffn_down_bias = command.parameter %weights, "v.blk.11.ffn_down.bias" : view<4608xi8>
%w_v_blk_12_ln1_weight = command.parameter %weights, "v.blk.12.ln1.weight" : view<4608xi8>
%w_v_blk_12_ln1_bias = command.parameter %weights, "v.blk.12.ln1.bias" : view<4608xi8>
%w_v_blk_12_attn_qkv_weight = command.parameter %weights, "v.blk.12.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_12_attn_qkv_bias = command.parameter %weights, "v.blk.12.attn_qkv.bias" : view<13824xi8>
%w_v_blk_12_attn_out_weight = command.parameter %weights, "v.blk.12.attn_out.weight" : view<2654208xi8>
%w_v_blk_12_attn_out_bias = command.parameter %weights, "v.blk.12.attn_out.bias" : view<4608xi8>
%w_v_blk_12_ln2_weight = command.parameter %weights, "v.blk.12.ln2.weight" : view<4608xi8>
%w_v_blk_12_ln2_bias = command.parameter %weights, "v.blk.12.ln2.bias" : view<4608xi8>
%w_v_blk_12_ffn_up_weight = command.parameter %weights, "v.blk.12.ffn_up.weight" : view<9916416xi8>
%w_v_blk_12_ffn_up_bias = command.parameter %weights, "v.blk.12.ffn_up.bias" : view<17216xi8>
%w_v_blk_12_ffn_down_weight = command.parameter %weights, "v.blk.12.ffn_down.weight" : view<9916416xi8>
%w_v_blk_12_ffn_down_bias = command.parameter %weights, "v.blk.12.ffn_down.bias" : view<4608xi8>
%w_v_blk_13_ln1_weight = command.parameter %weights, "v.blk.13.ln1.weight" : view<4608xi8>
%w_v_blk_13_ln1_bias = command.parameter %weights, "v.blk.13.ln1.bias" : view<4608xi8>
%w_v_blk_13_attn_qkv_weight = command.parameter %weights, "v.blk.13.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_13_attn_qkv_bias = command.parameter %weights, "v.blk.13.attn_qkv.bias" : view<13824xi8>
%w_v_blk_13_attn_out_weight = command.parameter %weights, "v.blk.13.attn_out.weight" : view<2654208xi8>
%w_v_blk_13_attn_out_bias = command.parameter %weights, "v.blk.13.attn_out.bias" : view<4608xi8>
%w_v_blk_13_ln2_weight = command.parameter %weights, "v.blk.13.ln2.weight" : view<4608xi8>
%w_v_blk_13_ln2_bias = command.parameter %weights, "v.blk.13.ln2.bias" : view<4608xi8>
%w_v_blk_13_ffn_up_weight = command.parameter %weights, "v.blk.13.ffn_up.weight" : view<9916416xi8>
%w_v_blk_13_ffn_up_bias = command.parameter %weights, "v.blk.13.ffn_up.bias" : view<17216xi8>
%w_v_blk_13_ffn_down_weight = command.parameter %weights, "v.blk.13.ffn_down.weight" : view<9916416xi8>
%w_v_blk_13_ffn_down_bias = command.parameter %weights, "v.blk.13.ffn_down.bias" : view<4608xi8>
%w_v_blk_14_ln1_weight = command.parameter %weights, "v.blk.14.ln1.weight" : view<4608xi8>
%w_v_blk_14_ln1_bias = command.parameter %weights, "v.blk.14.ln1.bias" : view<4608xi8>
%w_v_blk_14_attn_qkv_weight = command.parameter %weights, "v.blk.14.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_14_attn_qkv_bias = command.parameter %weights, "v.blk.14.attn_qkv.bias" : view<13824xi8>
%w_v_blk_14_attn_out_weight = command.parameter %weights, "v.blk.14.attn_out.weight" : view<2654208xi8>
%w_v_blk_14_attn_out_bias = command.parameter %weights, "v.blk.14.attn_out.bias" : view<4608xi8>
%w_v_blk_14_ln2_weight = command.parameter %weights, "v.blk.14.ln2.weight" : view<4608xi8>
%w_v_blk_14_ln2_bias = command.parameter %weights, "v.blk.14.ln2.bias" : view<4608xi8>
%w_v_blk_14_ffn_up_weight = command.parameter %weights, "v.blk.14.ffn_up.weight" : view<9916416xi8>
%w_v_blk_14_ffn_up_bias = command.parameter %weights, "v.blk.14.ffn_up.bias" : view<17216xi8>
%w_v_blk_14_ffn_down_weight = command.parameter %weights, "v.blk.14.ffn_down.weight" : view<9916416xi8>
%w_v_blk_14_ffn_down_bias = command.parameter %weights, "v.blk.14.ffn_down.bias" : view<4608xi8>
%w_v_blk_15_ln1_weight = command.parameter %weights, "v.blk.15.ln1.weight" : view<4608xi8>
%w_v_blk_15_ln1_bias = command.parameter %weights, "v.blk.15.ln1.bias" : view<4608xi8>
%w_v_blk_15_attn_qkv_weight = command.parameter %weights, "v.blk.15.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_15_attn_qkv_bias = command.parameter %weights, "v.blk.15.attn_qkv.bias" : view<13824xi8>
%w_v_blk_15_attn_out_weight = command.parameter %weights, "v.blk.15.attn_out.weight" : view<2654208xi8>
%w_v_blk_15_attn_out_bias = command.parameter %weights, "v.blk.15.attn_out.bias" : view<4608xi8>
%w_v_blk_15_ln2_weight = command.parameter %weights, "v.blk.15.ln2.weight" : view<4608xi8>
%w_v_blk_15_ln2_bias = command.parameter %weights, "v.blk.15.ln2.bias" : view<4608xi8>
%w_v_blk_15_ffn_up_weight = command.parameter %weights, "v.blk.15.ffn_up.weight" : view<9916416xi8>
%w_v_blk_15_ffn_up_bias = command.parameter %weights, "v.blk.15.ffn_up.bias" : view<17216xi8>
%w_v_blk_15_ffn_down_weight = command.parameter %weights, "v.blk.15.ffn_down.weight" : view<9916416xi8>
%w_v_blk_15_ffn_down_bias = command.parameter %weights, "v.blk.15.ffn_down.bias" : view<4608xi8>
%w_v_blk_16_ln1_weight = command.parameter %weights, "v.blk.16.ln1.weight" : view<4608xi8>
%w_v_blk_16_ln1_bias = command.parameter %weights, "v.blk.16.ln1.bias" : view<4608xi8>
%w_v_blk_16_attn_qkv_weight = command.parameter %weights, "v.blk.16.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_16_attn_qkv_bias = command.parameter %weights, "v.blk.16.attn_qkv.bias" : view<13824xi8>
%w_v_blk_16_attn_out_weight = command.parameter %weights, "v.blk.16.attn_out.weight" : view<2654208xi8>
%w_v_blk_16_attn_out_bias = command.parameter %weights, "v.blk.16.attn_out.bias" : view<4608xi8>
%w_v_blk_16_ln2_weight = command.parameter %weights, "v.blk.16.ln2.weight" : view<4608xi8>
%w_v_blk_16_ln2_bias = command.parameter %weights, "v.blk.16.ln2.bias" : view<4608xi8>
%w_v_blk_16_ffn_up_weight = command.parameter %weights, "v.blk.16.ffn_up.weight" : view<9916416xi8>
%w_v_blk_16_ffn_up_bias = command.parameter %weights, "v.blk.16.ffn_up.bias" : view<17216xi8>
%w_v_blk_16_ffn_down_weight = command.parameter %weights, "v.blk.16.ffn_down.weight" : view<9916416xi8>
%w_v_blk_16_ffn_down_bias = command.parameter %weights, "v.blk.16.ffn_down.bias" : view<4608xi8>
%w_v_blk_17_ln1_weight = command.parameter %weights, "v.blk.17.ln1.weight" : view<4608xi8>
%w_v_blk_17_ln1_bias = command.parameter %weights, "v.blk.17.ln1.bias" : view<4608xi8>
%w_v_blk_17_attn_qkv_weight = command.parameter %weights, "v.blk.17.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_17_attn_qkv_bias = command.parameter %weights, "v.blk.17.attn_qkv.bias" : view<13824xi8>
%w_v_blk_17_attn_out_weight = command.parameter %weights, "v.blk.17.attn_out.weight" : view<2654208xi8>
%w_v_blk_17_attn_out_bias = command.parameter %weights, "v.blk.17.attn_out.bias" : view<4608xi8>
%w_v_blk_17_ln2_weight = command.parameter %weights, "v.blk.17.ln2.weight" : view<4608xi8>
%w_v_blk_17_ln2_bias = command.parameter %weights, "v.blk.17.ln2.bias" : view<4608xi8>
%w_v_blk_17_ffn_up_weight = command.parameter %weights, "v.blk.17.ffn_up.weight" : view<9916416xi8>
%w_v_blk_17_ffn_up_bias = command.parameter %weights, "v.blk.17.ffn_up.bias" : view<17216xi8>
%w_v_blk_17_ffn_down_weight = command.parameter %weights, "v.blk.17.ffn_down.weight" : view<9916416xi8>
%w_v_blk_17_ffn_down_bias = command.parameter %weights, "v.blk.17.ffn_down.bias" : view<4608xi8>
%w_v_blk_18_ln1_weight = command.parameter %weights, "v.blk.18.ln1.weight" : view<4608xi8>
%w_v_blk_18_ln1_bias = command.parameter %weights, "v.blk.18.ln1.bias" : view<4608xi8>
%w_v_blk_18_attn_qkv_weight = command.parameter %weights, "v.blk.18.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_18_attn_qkv_bias = command.parameter %weights, "v.blk.18.attn_qkv.bias" : view<13824xi8>
%w_v_blk_18_attn_out_weight = command.parameter %weights, "v.blk.18.attn_out.weight" : view<2654208xi8>
%w_v_blk_18_attn_out_bias = command.parameter %weights, "v.blk.18.attn_out.bias" : view<4608xi8>
%w_v_blk_18_ln2_weight = command.parameter %weights, "v.blk.18.ln2.weight" : view<4608xi8>
%w_v_blk_18_ln2_bias = command.parameter %weights, "v.blk.18.ln2.bias" : view<4608xi8>
%w_v_blk_18_ffn_up_weight = command.parameter %weights, "v.blk.18.ffn_up.weight" : view<9916416xi8>
%w_v_blk_18_ffn_up_bias = command.parameter %weights, "v.blk.18.ffn_up.bias" : view<17216xi8>
%w_v_blk_18_ffn_down_weight = command.parameter %weights, "v.blk.18.ffn_down.weight" : view<9916416xi8>
%w_v_blk_18_ffn_down_bias = command.parameter %weights, "v.blk.18.ffn_down.bias" : view<4608xi8>
%w_v_blk_19_ln1_weight = command.parameter %weights, "v.blk.19.ln1.weight" : view<4608xi8>
%w_v_blk_19_ln1_bias = command.parameter %weights, "v.blk.19.ln1.bias" : view<4608xi8>
%w_v_blk_19_attn_qkv_weight = command.parameter %weights, "v.blk.19.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_19_attn_qkv_bias = command.parameter %weights, "v.blk.19.attn_qkv.bias" : view<13824xi8>
%w_v_blk_19_attn_out_weight = command.parameter %weights, "v.blk.19.attn_out.weight" : view<2654208xi8>
%w_v_blk_19_attn_out_bias = command.parameter %weights, "v.blk.19.attn_out.bias" : view<4608xi8>
%w_v_blk_19_ln2_weight = command.parameter %weights, "v.blk.19.ln2.weight" : view<4608xi8>
%w_v_blk_19_ln2_bias = command.parameter %weights, "v.blk.19.ln2.bias" : view<4608xi8>
%w_v_blk_19_ffn_up_weight = command.parameter %weights, "v.blk.19.ffn_up.weight" : view<9916416xi8>
%w_v_blk_19_ffn_up_bias = command.parameter %weights, "v.blk.19.ffn_up.bias" : view<17216xi8>
%w_v_blk_19_ffn_down_weight = command.parameter %weights, "v.blk.19.ffn_down.weight" : view<9916416xi8>
%w_v_blk_19_ffn_down_bias = command.parameter %weights, "v.blk.19.ffn_down.bias" : view<4608xi8>
%w_v_blk_20_ln1_weight = command.parameter %weights, "v.blk.20.ln1.weight" : view<4608xi8>
%w_v_blk_20_ln1_bias = command.parameter %weights, "v.blk.20.ln1.bias" : view<4608xi8>
%w_v_blk_20_attn_qkv_weight = command.parameter %weights, "v.blk.20.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_20_attn_qkv_bias = command.parameter %weights, "v.blk.20.attn_qkv.bias" : view<13824xi8>
%w_v_blk_20_attn_out_weight = command.parameter %weights, "v.blk.20.attn_out.weight" : view<2654208xi8>
%w_v_blk_20_attn_out_bias = command.parameter %weights, "v.blk.20.attn_out.bias" : view<4608xi8>
%w_v_blk_20_ln2_weight = command.parameter %weights, "v.blk.20.ln2.weight" : view<4608xi8>
%w_v_blk_20_ln2_bias = command.parameter %weights, "v.blk.20.ln2.bias" : view<4608xi8>
%w_v_blk_20_ffn_up_weight = command.parameter %weights, "v.blk.20.ffn_up.weight" : view<9916416xi8>
%w_v_blk_20_ffn_up_bias = command.parameter %weights, "v.blk.20.ffn_up.bias" : view<17216xi8>
%w_v_blk_20_ffn_down_weight = command.parameter %weights, "v.blk.20.ffn_down.weight" : view<9916416xi8>
%w_v_blk_20_ffn_down_bias = command.parameter %weights, "v.blk.20.ffn_down.bias" : view<4608xi8>
%w_v_blk_21_ln1_weight = command.parameter %weights, "v.blk.21.ln1.weight" : view<4608xi8>
%w_v_blk_21_ln1_bias = command.parameter %weights, "v.blk.21.ln1.bias" : view<4608xi8>
%w_v_blk_21_attn_qkv_weight = command.parameter %weights, "v.blk.21.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_21_attn_qkv_bias = command.parameter %weights, "v.blk.21.attn_qkv.bias" : view<13824xi8>
%w_v_blk_21_attn_out_weight = command.parameter %weights, "v.blk.21.attn_out.weight" : view<2654208xi8>
%w_v_blk_21_attn_out_bias = command.parameter %weights, "v.blk.21.attn_out.bias" : view<4608xi8>
%w_v_blk_21_ln2_weight = command.parameter %weights, "v.blk.21.ln2.weight" : view<4608xi8>
%w_v_blk_21_ln2_bias = command.parameter %weights, "v.blk.21.ln2.bias" : view<4608xi8>
%w_v_blk_21_ffn_up_weight = command.parameter %weights, "v.blk.21.ffn_up.weight" : view<9916416xi8>
%w_v_blk_21_ffn_up_bias = command.parameter %weights, "v.blk.21.ffn_up.bias" : view<17216xi8>
%w_v_blk_21_ffn_down_weight = command.parameter %weights, "v.blk.21.ffn_down.weight" : view<9916416xi8>
%w_v_blk_21_ffn_down_bias = command.parameter %weights, "v.blk.21.ffn_down.bias" : view<4608xi8>
%w_v_blk_22_ln1_weight = command.parameter %weights, "v.blk.22.ln1.weight" : view<4608xi8>
%w_v_blk_22_ln1_bias = command.parameter %weights, "v.blk.22.ln1.bias" : view<4608xi8>
%w_v_blk_22_attn_qkv_weight = command.parameter %weights, "v.blk.22.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_22_attn_qkv_bias = command.parameter %weights, "v.blk.22.attn_qkv.bias" : view<13824xi8>
%w_v_blk_22_attn_out_weight = command.parameter %weights, "v.blk.22.attn_out.weight" : view<2654208xi8>
%w_v_blk_22_attn_out_bias = command.parameter %weights, "v.blk.22.attn_out.bias" : view<4608xi8>
%w_v_blk_22_ln2_weight = command.parameter %weights, "v.blk.22.ln2.weight" : view<4608xi8>
%w_v_blk_22_ln2_bias = command.parameter %weights, "v.blk.22.ln2.bias" : view<4608xi8>
%w_v_blk_22_ffn_up_weight = command.parameter %weights, "v.blk.22.ffn_up.weight" : view<9916416xi8>
%w_v_blk_22_ffn_up_bias = command.parameter %weights, "v.blk.22.ffn_up.bias" : view<17216xi8>
%w_v_blk_22_ffn_down_weight = command.parameter %weights, "v.blk.22.ffn_down.weight" : view<9916416xi8>
%w_v_blk_22_ffn_down_bias = command.parameter %weights, "v.blk.22.ffn_down.bias" : view<4608xi8>
%w_v_blk_23_ln1_weight = command.parameter %weights, "v.blk.23.ln1.weight" : view<4608xi8>
%w_v_blk_23_ln1_bias = command.parameter %weights, "v.blk.23.ln1.bias" : view<4608xi8>
%w_v_blk_23_attn_qkv_weight = command.parameter %weights, "v.blk.23.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_23_attn_qkv_bias = command.parameter %weights, "v.blk.23.attn_qkv.bias" : view<13824xi8>
%w_v_blk_23_attn_out_weight = command.parameter %weights, "v.blk.23.attn_out.weight" : view<2654208xi8>
%w_v_blk_23_attn_out_bias = command.parameter %weights, "v.blk.23.attn_out.bias" : view<4608xi8>
%w_v_blk_23_ln2_weight = command.parameter %weights, "v.blk.23.ln2.weight" : view<4608xi8>
%w_v_blk_23_ln2_bias = command.parameter %weights, "v.blk.23.ln2.bias" : view<4608xi8>
%w_v_blk_23_ffn_up_weight = command.parameter %weights, "v.blk.23.ffn_up.weight" : view<9916416xi8>
%w_v_blk_23_ffn_up_bias = command.parameter %weights, "v.blk.23.ffn_up.bias" : view<17216xi8>
%w_v_blk_23_ffn_down_weight = command.parameter %weights, "v.blk.23.ffn_down.weight" : view<9916416xi8>
%w_v_blk_23_ffn_down_bias = command.parameter %weights, "v.blk.23.ffn_down.bias" : view<4608xi8>
%w_v_blk_24_ln1_weight = command.parameter %weights, "v.blk.24.ln1.weight" : view<4608xi8>
%w_v_blk_24_ln1_bias = command.parameter %weights, "v.blk.24.ln1.bias" : view<4608xi8>
%w_v_blk_24_attn_qkv_weight = command.parameter %weights, "v.blk.24.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_24_attn_qkv_bias = command.parameter %weights, "v.blk.24.attn_qkv.bias" : view<13824xi8>
%w_v_blk_24_attn_out_weight = command.parameter %weights, "v.blk.24.attn_out.weight" : view<2654208xi8>
%w_v_blk_24_attn_out_bias = command.parameter %weights, "v.blk.24.attn_out.bias" : view<4608xi8>
%w_v_blk_24_ln2_weight = command.parameter %weights, "v.blk.24.ln2.weight" : view<4608xi8>
%w_v_blk_24_ln2_bias = command.parameter %weights, "v.blk.24.ln2.bias" : view<4608xi8>
%w_v_blk_24_ffn_up_weight = command.parameter %weights, "v.blk.24.ffn_up.weight" : view<9916416xi8>
%w_v_blk_24_ffn_up_bias = command.parameter %weights, "v.blk.24.ffn_up.bias" : view<17216xi8>
%w_v_blk_24_ffn_down_weight = command.parameter %weights, "v.blk.24.ffn_down.weight" : view<9916416xi8>
%w_v_blk_24_ffn_down_bias = command.parameter %weights, "v.blk.24.ffn_down.bias" : view<4608xi8>
%w_v_blk_25_ln1_weight = command.parameter %weights, "v.blk.25.ln1.weight" : view<4608xi8>
%w_v_blk_25_ln1_bias = command.parameter %weights, "v.blk.25.ln1.bias" : view<4608xi8>
%w_v_blk_25_attn_qkv_weight = command.parameter %weights, "v.blk.25.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_25_attn_qkv_bias = command.parameter %weights, "v.blk.25.attn_qkv.bias" : view<13824xi8>
%w_v_blk_25_attn_out_weight = command.parameter %weights, "v.blk.25.attn_out.weight" : view<2654208xi8>
%w_v_blk_25_attn_out_bias = command.parameter %weights, "v.blk.25.attn_out.bias" : view<4608xi8>
%w_v_blk_25_ln2_weight = command.parameter %weights, "v.blk.25.ln2.weight" : view<4608xi8>
%w_v_blk_25_ln2_bias = command.parameter %weights, "v.blk.25.ln2.bias" : view<4608xi8>
%w_v_blk_25_ffn_up_weight = command.parameter %weights, "v.blk.25.ffn_up.weight" : view<9916416xi8>
%w_v_blk_25_ffn_up_bias = command.parameter %weights, "v.blk.25.ffn_up.bias" : view<17216xi8>
%w_v_blk_25_ffn_down_weight = command.parameter %weights, "v.blk.25.ffn_down.weight" : view<9916416xi8>
%w_v_blk_25_ffn_down_bias = command.parameter %weights, "v.blk.25.ffn_down.bias" : view<4608xi8>
%w_v_blk_26_ln1_weight = command.parameter %weights, "v.blk.26.ln1.weight" : view<4608xi8>
%w_v_blk_26_ln1_bias = command.parameter %weights, "v.blk.26.ln1.bias" : view<4608xi8>
%w_v_blk_26_attn_qkv_weight = command.parameter %weights, "v.blk.26.attn_qkv.weight" : view<7962624xi8>
%w_v_blk_26_attn_qkv_bias = command.parameter %weights, "v.blk.26.attn_qkv.bias" : view<13824xi8>
%w_v_blk_26_attn_out_weight = command.parameter %weights, "v.blk.26.attn_out.weight" : view<2654208xi8>
%w_v_blk_26_attn_out_bias = command.parameter %weights, "v.blk.26.attn_out.bias" : view<4608xi8>
%w_v_blk_26_ln2_weight = command.parameter %weights, "v.blk.26.ln2.weight" : view<4608xi8>
%w_v_blk_26_ln2_bias = command.parameter %weights, "v.blk.26.ln2.bias" : view<4608xi8>
%w_v_blk_26_ffn_up_weight = command.parameter %weights, "v.blk.26.ffn_up.weight" : view<9916416xi8>
%w_v_blk_26_ffn_up_bias = command.parameter %weights, "v.blk.26.ffn_up.bias" : view<17216xi8>
%w_v_blk_26_ffn_down_weight = command.parameter %weights, "v.blk.26.ffn_down.weight" : view<9916416xi8>
%w_v_blk_26_ffn_down_bias = command.parameter %weights, "v.blk.26.ffn_down.bias" : view<4608xi8>
%w_v_post_ln_weight = command.parameter %weights, "v.post_ln.weight" : view<4608xi8>
%w_v_post_ln_bias = command.parameter %weights, "v.post_ln.bias" : view<4608xi8>
%w_mm_0_weight = command.parameter %weights, "mm.0.weight" : view<42467328xi8>
%w_mm_0_bias = command.parameter %weights, "mm.0.bias" : view<18432xi8>
%w_mm_2_weight = command.parameter %weights, "mm.2.weight" : view<18874368xi8>
%w_mm_2_bias = command.parameter %weights, "mm.2.bias" : view<8192xi8>
kernel.launch @v_pixels_0p5_0p5_0p5[%n_pix](%n_pix, %pix, %pixf) : [index](index, buffer, buffer)
kernel.launch @v_patch_16_1152_f16_block2_dual_bias[%T](%T, %gw, %img_w, %pixf, %w_v_patch_embd_weight, %w_v_patch_embd_weight_1, %w_v_patch_embd_bias, %h) : [index](index, index, index, buffer, view<1769472xi8>, view<1769472xi8>, view<4608xi8>, buffer)
kernel.launch @v_pos_interp_1152_48_block2_ac[%T](%T, %gw, %gh, %w_v_position_embd_weight, %h) : [index](index, index, index, view<10616832xi8>, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_0_ln1_weight, %w_v_blk_0_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
%k1152 = index.constant 1152 : index
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
%k3456 = index.constant 3456 : index
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_0_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_0_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_0_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_0_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
%p_T_1152 = index.mul %T, %k1152 : index
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_0_ln2_weight, %w_v_blk_0_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
%k4304 = index.constant 4304 : index
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_0_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_0_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_0_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_0_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_1_ln1_weight, %w_v_blk_1_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_1_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_1_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_1_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_1_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_1_ln2_weight, %w_v_blk_1_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_1_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_1_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_1_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_1_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_2_ln1_weight, %w_v_blk_2_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_2_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_2_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_2_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_2_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_2_ln2_weight, %w_v_blk_2_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_2_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_2_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_2_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_2_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_3_ln1_weight, %w_v_blk_3_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_3_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_3_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_3_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_3_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_3_ln2_weight, %w_v_blk_3_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_3_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_3_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_3_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_3_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_4_ln1_weight, %w_v_blk_4_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_4_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_4_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_4_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_4_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_4_ln2_weight, %w_v_blk_4_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_4_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_4_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_4_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_4_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_5_ln1_weight, %w_v_blk_5_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_5_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_5_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_5_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_5_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_5_ln2_weight, %w_v_blk_5_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_5_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_5_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_5_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_5_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_6_ln1_weight, %w_v_blk_6_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_6_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_6_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_6_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_6_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_6_ln2_weight, %w_v_blk_6_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_6_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_6_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_6_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_6_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_7_ln1_weight, %w_v_blk_7_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_7_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_7_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_7_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_7_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_7_ln2_weight, %w_v_blk_7_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_7_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_7_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_7_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_7_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_8_ln1_weight, %w_v_blk_8_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_8_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_8_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_8_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_8_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_8_ln2_weight, %w_v_blk_8_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_8_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_8_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_8_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_8_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_9_ln1_weight, %w_v_blk_9_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_9_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_9_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_9_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_9_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_9_ln2_weight, %w_v_blk_9_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_9_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_9_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_9_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_9_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_10_ln1_weight, %w_v_blk_10_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_10_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_10_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_10_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_10_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_10_ln2_weight, %w_v_blk_10_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_10_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_10_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_10_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_10_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_11_ln1_weight, %w_v_blk_11_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_11_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_11_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_11_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_11_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_11_ln2_weight, %w_v_blk_11_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_11_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_11_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_11_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_11_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_12_ln1_weight, %w_v_blk_12_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_12_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_12_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_12_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_12_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_12_ln2_weight, %w_v_blk_12_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_12_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_12_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_12_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_12_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_13_ln1_weight, %w_v_blk_13_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_13_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_13_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_13_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_13_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_13_ln2_weight, %w_v_blk_13_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_13_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_13_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_13_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_13_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_14_ln1_weight, %w_v_blk_14_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_14_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_14_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_14_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_14_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_14_ln2_weight, %w_v_blk_14_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_14_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_14_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_14_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_14_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_15_ln1_weight, %w_v_blk_15_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_15_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_15_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_15_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_15_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_15_ln2_weight, %w_v_blk_15_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_15_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_15_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_15_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_15_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_16_ln1_weight, %w_v_blk_16_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_16_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_16_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_16_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_16_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_16_ln2_weight, %w_v_blk_16_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_16_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_16_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_16_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_16_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_17_ln1_weight, %w_v_blk_17_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_17_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_17_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_17_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_17_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_17_ln2_weight, %w_v_blk_17_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_17_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_17_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_17_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_17_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_18_ln1_weight, %w_v_blk_18_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_18_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_18_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_18_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_18_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_18_ln2_weight, %w_v_blk_18_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_18_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_18_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_18_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_18_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_19_ln1_weight, %w_v_blk_19_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_19_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_19_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_19_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_19_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_19_ln2_weight, %w_v_blk_19_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_19_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_19_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_19_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_19_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_20_ln1_weight, %w_v_blk_20_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_20_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_20_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_20_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_20_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_20_ln2_weight, %w_v_blk_20_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_20_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_20_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_20_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_20_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_21_ln1_weight, %w_v_blk_21_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_21_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_21_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_21_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_21_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_21_ln2_weight, %w_v_blk_21_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_21_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_21_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_21_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_21_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_22_ln1_weight, %w_v_blk_22_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_22_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_22_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_22_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_22_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_22_ln2_weight, %w_v_blk_22_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_22_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_22_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_22_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_22_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_23_ln1_weight, %w_v_blk_23_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_23_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_23_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_23_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_23_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_23_ln2_weight, %w_v_blk_23_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_23_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_23_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_23_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_23_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_24_ln1_weight, %w_v_blk_24_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_24_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_24_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_24_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_24_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_24_ln2_weight, %w_v_blk_24_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_24_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_24_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_24_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_24_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_25_ln1_weight, %w_v_blk_25_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_25_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_25_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_25_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_25_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_25_ln2_weight, %w_v_blk_25_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_25_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_25_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_25_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_25_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_26_ln1_weight, %w_v_blk_26_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_26_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer)
kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_26_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer)
kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_26_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_26_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_26_ln2_weight, %w_v_blk_26_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_26_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_26_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer)
kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_26_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer)
kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_26_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer)
kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer)
kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_post_ln_weight, %w_v_post_ln_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer)
%k4608 = index.constant 4608 : index
kernel.launch @v_row_scale_f16[%Mp](%Mp, %k4608, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
kernel.launch @v_gemm_f16_store_4608x4608[%Mp, %k4608](%Mp, %k4608, %k4608, %x16, %sc, %w_mm_0_weight, %m1) : [index, index](index, index, index, buffer, buffer, view<42467328xi8>, buffer)
kernel.launch @v_bias_gelu_tanh_4608[%M](%M, %m1, %w_mm_0_bias, %m1) : [index](index, buffer, view<18432xi8>, buffer)
kernel.launch @v_row_scale_f16[%Mp](%Mp, %k4608, %m1, %sc, %x16) : [index](index, index, buffer, buffer, buffer)
%k2048 = index.constant 2048 : index
kernel.launch @v_gemm_f16_store_4608x2048[%Mp, %k2048](%Mp, %k4608, %k2048, %x16, %sc, %w_mm_2_weight, %out) : [index, index](index, index, index, buffer, buffer, view<18874368xi8>, buffer)
kernel.launch @v_bias_none_2048[%M](%M, %out, %w_mm_2_bias, %out) : [index](index, buffer, view<8192xi8>, buffer)
%out_rows = index.constant 2048 : index
%final_n = index.mul %M, %out_rows : index
kernel.launch @v_copy[%final_n](%final_n, %out, %output) : [index](index, buffer, buffer)
command.return
}
kernel.def export("v_row_scale_f16") @v_row_scale_f16(%rows: index) {
%c1 = index.constant 1 : index
%c256 = index.constant 256 : index
kernel.launch.config workgroups(%rows, %c1, %c1) workgroup_size(%c256, %c1, %c1) : index
} launch(%rows_arg: index, %k_arg: index, %x: buffer, %scales: buffer, %x16: buffer) {
%rows = index.assume %rows_arg [range(%rows_arg, 1, 2097152)] : index
%k = index.assume %k_arg [range(%k_arg, 8, 65536), multiple_of(%k_arg, 8)] : index
%row_id = kernel.workgroup.id<x> : index
%row = index.assume %row_id [lt(%row_id, %rows)] : index
%lane = kernel.workitem.id<x> : index
%zero = index.constant 0 : offset
%c0 = index.constant 0 : index
%c1 = index.constant 1 : index
%c256 = index.constant 256 : index
%zero_f = scalar.constant 0.0 : f32
%x_view = buffer.view %x[%zero] : buffer -> view<[%rows]x[%k]xf32>
%h_view = buffer.view %x16[%zero] : buffer -> view<[%rows]x[%k]xf16>
%local = scf.for %c = [%lane to %k step %c256](%m = %zero_f : f32) -> (f32) {
%bc = index.assume %c [lt(%c, %k)] : index
%v = view.load %x_view[%row, %bc] : view<[%rows]x[%k]xf32> -> f32
%a = scalar.absf %v : f32
%n = scalar.maxnumf %m, %a : f32
scf.yield %n : f32
}
%maximum = kernel.workgroup.reduce<maxnumf> %local : f32
%bits = scalar.bitcast %maximum : f32 to i32
%c23 = scalar.constant 23 : i32
%c255 = scalar.constant 255 : i32
%shifted = scalar.shrui %bits, %c23 : i32
%biased = scalar.andi %shifted, %c255 : i32
%c127 = scalar.constant 127 : i32
%c13 = scalar.constant 13 : i32
%c0_i32 = scalar.constant 0 : i32
%c100 = scalar.constant 100 : i32
%unbiased = scalar.subi %biased, %c127 : i32
%raw_e = scalar.subi %unbiased, %c13 : i32
%positive = scalar.maxsi %raw_e, %c0_i32 : i32
%e = scalar.minsi %positive, %c100 : i32
%scale_exp = scalar.subi %c127, %e : i32
%inverse_exp = scalar.addi %c127, %e : i32
%scale_bits = scalar.shli %scale_exp, %c23 : i32
%inverse_bits = scalar.shli %inverse_exp, %c23 : i32
%scale = scalar.bitcast %scale_bits : i32 to f32
%inverse = scalar.bitcast %inverse_bits : i32 to f32
scf.for %c = [%lane to %k step %c256] {
%bc = index.assume %c [lt(%c, %k)] : index
%v = view.load %x_view[%row, %bc] : view<[%rows]x[%k]xf32> -> f32
%sv = scalar.mulf %v, %scale : f32
%hv = scalar.fptrunc %sv : f32 to f16
view.store %hv, %h_view[%row, %bc] : f16, view<[%rows]x[%k]xf16>
}
%first = index.cmp eq, %lane, %c0 : index
scf.if %first {
%s_view = buffer.view %scales[%zero] : buffer -> view<[%rows]x2xf32>
view.store %scale, %s_view[%row, %c0] : f32, view<[%rows]x2xf32>
view.store %inverse, %s_view[%row, %c1] : f32, view<[%rows]x2xf32>
}
kernel.return
}
kernel.def export("v_pixels_0p5_0p5_0p5") @v_pixels_0p5_0p5_0p5(%n: index) {
%c1 = index.constant 1 : index
%c256 = index.constant 256 : index
%c255 = index.constant 255 : index
%n_up = index.add %n, %c255 : index
%groups = index.div %n_up, %c256 : index
kernel.launch.config workgroups(%groups, %c1, %c1) workgroup_size(%c256, %c1, %c1) : index
} launch(%n_arg: index, %src: buffer, %y: buffer) {
%n = index.assume %n_arg [range(%n_arg, 1, 268435456)] : index
%c256 = index.constant 256 : index
%c3 = index.constant 3 : index
%gid = kernel.workgroup.id<x> : index
%lid = kernel.workitem.id<x> : index
%i = index.madd %gid, %c256, %lid : index
%live = index.cmp ult, %i, %n : index
scf.if %live {
%bi = index.assume %i [lt(%i, %n)] : index
%zero = index.constant 0 : offset
%sv = buffer.view %src[%zero] : buffer -> view<[%n]xi8>
%raw = view.load %sv[%bi] : view<[%n]xi8> -> i8
%u = scalar.extui %raw : i8 to i32
%f = scalar.uitofp %u : i32 to f32
%ch = index.rem %bi, %c3 : index
%c0 = index.constant 0 : index
%c1 = index.constant 1 : index
%is0 = index.cmp eq, %ch, %c0 : index
%is1 = index.cmp eq, %ch, %c1 : index
%a0 = scalar.constant 0.00784313725490196 : f32
%a1 = scalar.constant 0.00784313725490196 : f32
%a2 = scalar.constant 0.00784313725490196 : f32
%b0 = scalar.constant -1.0 : f32
%b1 = scalar.constant -1.0 : f32
%b2 = scalar.constant -1.0 : f32
%a12 = scf.select %is1, %a1, %a2 : f32
%a = scf.select %is0, %a0, %a12 : f32
%b12 = scf.select %is1, %b1, %b2 : f32
%b = scf.select %is0, %b0, %b12 : f32
%r = scalar.fmaf %f, %a, %b : f32
%yv = buffer.view %y[%zero] : buffer -> view<[%n]xf32>
view.store %r, %yv[%bi] : f32, view<[%n]xf32>