kernel.def export("v_row_scale_f16") @v_row_scale_f16(%rows: index) { %c1 = index.constant 1 : index %c256 = index.constant 256 : index kernel.launch.config workgroups(%rows, %c1, %c1) workgroup_size(%c256, %c1, %c1) : index } launch(%rows_arg: index, %k_arg: index, %x: buffer, %scales: buffer, %x16: buffer) { %rows = index.assume %rows_arg [range(%rows_arg, 1, 2097152)] : index %k = index.assume %k_arg [range(%k_arg, 8, 65536), multiple_of(%k_arg, 8)] : index %row_id = kernel.workgroup.id : index %row = index.assume %row_id [lt(%row_id, %rows)] : index %lane = kernel.workitem.id : index %zero = index.constant 0 : offset %c0 = index.constant 0 : index %c1 = index.constant 1 : index %c256 = index.constant 256 : index %zero_f = scalar.constant 0.0 : f32 %x_view = buffer.view %x[%zero] : buffer -> view<[%rows]x[%k]xf32> %h_view = buffer.view %x16[%zero] : buffer -> view<[%rows]x[%k]xf16> %local = scf.for %c = [%lane to %k step %c256](%m = %zero_f : f32) -> (f32) { %bc = index.assume %c [lt(%c, %k)] : index %v = view.load %x_view[%row, %bc] : view<[%rows]x[%k]xf32> -> f32 %a = scalar.absf %v : f32 %n = scalar.maxnumf %m, %a : f32 scf.yield %n : f32 } %maximum = kernel.workgroup.reduce %local : f32 %bits = scalar.bitcast %maximum : f32 to i32 %c23 = scalar.constant 23 : i32 %c255 = scalar.constant 255 : i32 %shifted = scalar.shrui %bits, %c23 : i32 %biased = scalar.andi %shifted, %c255 : i32 %c127 = scalar.constant 127 : i32 %c13 = scalar.constant 13 : i32 %c0_i32 = scalar.constant 0 : i32 %c100 = scalar.constant 100 : i32 %unbiased = scalar.subi %biased, %c127 : i32 %raw_e = scalar.subi %unbiased, %c13 : i32 %positive = scalar.maxsi %raw_e, %c0_i32 : i32 %e = scalar.minsi %positive, %c100 : i32 %scale_exp = scalar.subi %c127, %e : i32 %inverse_exp = scalar.addi %c127, %e : i32 %scale_bits = scalar.shli %scale_exp, %c23 : i32 %inverse_bits = scalar.shli %inverse_exp, %c23 : i32 %scale = scalar.bitcast %scale_bits : i32 to f32 %inverse = scalar.bitcast %inverse_bits : i32 to f32 scf.for %c = [%lane to %k step %c256] { %bc = index.assume %c [lt(%c, %k)] : index %v = view.load %x_view[%row, %bc] : view<[%rows]x[%k]xf32> -> f32 %sv = scalar.mulf %v, %scale : f32 %hv = scalar.fptrunc %sv : f32 to f16 view.store %hv, %h_view[%row, %bc] : f16, view<[%rows]x[%k]xf16> } %first = index.cmp eq, %lane, %c0 : index scf.if %first { %s_view = buffer.view %scales[%zero] : buffer -> view<[%rows]x2xf32> view.store %scale, %s_view[%row, %c0] : f32, view<[%rows]x2xf32> view.store %inverse, %s_view[%row, %c1] : f32, view<[%rows]x2xf32> } kernel.return } kernel.def export("v_pixels_0p5_0p5_0p5") @v_pixels_0p5_0p5_0p5(%n: index) { %c1 = index.constant 1 : index %c256 = index.constant 256 : index %c255 = index.constant 255 : index %n_up = index.add %n, %c255 : index %groups = index.div %n_up, %c256 : index kernel.launch.config workgroups(%groups, %c1, %c1) workgroup_size(%c256, %c1, %c1) : index } launch(%n_arg: index, %src: buffer, %y: buffer) { %n = index.assume %n_arg [range(%n_arg, 1, 268435456)] : index %c256 = index.constant 256 : index %c3 = index.constant 3 : index %gid = kernel.workgroup.id : index %lid = kernel.workitem.id : index %i = index.madd %gid, %c256, %lid : index %live = index.cmp ult, %i, %n : index scf.if %live { %bi = index.assume %i [lt(%i, %n)] : index %zero = index.constant 0 : offset %sv = buffer.view %src[%zero] : buffer -> view<[%n]xi8> %raw = view.load %sv[%bi] : view<[%n]xi8> -> i8 %u = scalar.extui %raw : i8 to i32 %f = scalar.uitofp %u : i32 to f32 %ch = index.rem %bi, %c3 : index %c0 = index.constant 0 : index %c1 = index.constant 1 : index %is0 = index.cmp eq, %ch, %c0 : index %is1 = index.cmp eq, %ch, %c1 : index %a0 = scalar.constant 0.00784313725490196 : f32 %a1 = scalar.constant 0.00784313725490196 : f32 %a2 = scalar.constant 0.00784313725490196 : f32 %b0 = scalar.constant -1.0 : f32 %b1 = scalar.constant -1.0 : f32 %b2 = scalar.constant -1.0 : f32 %a12 = scf.select %is1, %a1, %a2 : f32 %a = scf.select %is0, %a0, %a12 : f32 %b12 = scf.select %is1, %b1, %b2 : f32 %b = scf.select %is0, %b0, %b12 : f32 %r = scalar.fmaf %f, %a, %b : f32 %yv = buffer.view %y[%zero] : buffer -> view<[%n]xf32> view.store %r, %yv[%bi] : f32, view<[%n]xf32> } kernel.return } kernel.def export("v_patch_16_1152_f16_block2_dual_bias") @v_patch_16_1152_f16_block2_dual_bias(%tokens: index) { %c1 = index.constant 1 : index %c256 = index.constant 256 : index kernel.launch.config workgroups(%tokens, %c1, %c1) workgroup_size(%c256, %c1, %c1) : index } launch(%tokens_arg: index, %gw_arg: index, %img_w_arg: index, %pix: buffer, %w: buffer, %w2: buffer, %b: buffer, %y: buffer) { %tokens = index.assume %tokens_arg [range(%tokens_arg, 1, 65536)] : index %gw = index.assume %gw_arg [range(%gw_arg, 1, 1024)] : index %img_w = index.assume %img_w_arg [range(%img_w_arg, 1, 16384)] : index %t_id = kernel.workgroup.id : index %t = index.assume %t_id [lt(%t_id, %tokens)] : index %lid = kernel.workitem.id : index %lane = index.assume %lid [range(%lid, 0, 255)] : index %zero = index.constant 0 : offset %c0 = index.constant 0 : index %c1 = index.constant 1 : index %c3 = index.constant 3 : index %c256 = index.constant 256 : index %cP = index.constant 16 : index %cPP = index.constant 256 : index %cK = index.constant 768 : index %cKP = index.constant 768 : index %cE = index.constant 1152 : index %zf = scalar.constant 0.0 : f32 %c2o = index.constant 2 : index %c4o = index.constant 4 : index %bw0 = index.div %gw, %c2o : index %bw1 = index.max %bw0, %c1 : index %bw = index.assume %bw1 [range(%bw1, 1, 4096)] : index %blk = index.div %t, %c4o : index %sub = index.rem %t, %c4o : index %ty = index.div %blk, %bw : index %tx = index.rem %blk, %bw : index %dy = index.div %sub, %c2o : index %dx = index.rem %sub, %c2o : index %py = index.madd %ty, %c2o, %dy : index %px = index.madd %tx, %c2o, %dx : index %stage_bytes = index.constant 3072 : offset %stage = buffer.alloca align(16) %stage_bytes : buffer %sv = buffer.view %stage[%zero] : buffer -> view<768xf32> %rows_px = index.mul %tokens, %c1 : index %pix_n = index.constant 268435456 : index %pv = buffer.view %pix[%zero] : buffer -> view<[%pix_n]xf32> %y0 = index.mul %py, %cP : index %x0 = index.mul %px, %cP : index scf.for %i = [%lane to %cKP step %c256] { %bi = index.assume %i [lt(%i, %cKP)] : index %inside = index.cmp ult, %i, %cK : index %v = scf.if %inside -> (f32) { %ch = index.div %i, %cPP : index %r = index.rem %i, %cPP : index %ky = index.div %r, %cP : index %kx = index.rem %r, %cP : index %yy = index.add %y0, %ky : index %xx = index.add %x0, %kx : index %rowpix = index.mul %yy, %img_w : index %pixel = index.add %rowpix, %xx : index %el = index.madd %pixel, %c3, %ch : index %bel = index.assume %el [lt(%el, %pix_n)] : index %pval = view.load %pv[%bel] : view<[%pix_n]xf32> -> f32 scf.yield %pval : f32 } else { scf.yield %zf : f32 } view.store %v, %sv[%bi] : f32, view<768xf32> } kernel.barrier scope(workgroup) ordering(acq_rel) %wn = index.constant 884736 : index %wv = buffer.view %w[%zero] : buffer -> view<[%wn]xf16> %wv2 = buffer.view %w2[%zero] : buffer -> view<[%wn]xf16> %yv = buffer.view %y[%zero] : buffer -> view<[%tokens]x1152xf32> scf.for %e = [%lane to %cE step %c256] { %be = index.assume %e [lt(%e, %cE)] : index %wrow = index.mul %be, %cK : index %acc = scf.for %k = [%c0 to %cK step %c1](%a = %zf : f32) -> (f32) { %bk = index.assume %k [lt(%k, %cKP)] : index %xk = view.load %sv[%bk] : view<768xf32> -> f32 %wi = index.add %wrow, %k : index %bwi = index.assume %wi [lt(%wi, %wn)] : index %wk_h = view.load %wv[%bwi] : view<[%wn]xf16> -> f16 %wk = scalar.extf %wk_h : f16 to f32 %wk2_h = view.load %wv2[%bwi] : view<[%wn]xf16> -> f16 %wk2 = scalar.extf %wk2_h : f16 to f32 %wks = scalar.addf %wk, %wk2 : f32 %na = scalar.fmaf %wks, %xk, %a : f32 scf.yield %na : f32 } %bview = buffer.view %b[%zero] : buffer -> view<1152xf32> %bb = view.load %bview[%be] : view<1152xf32> -> f32 %out = scalar.addf %acc, %bb : f32 view.store %out, %yv[%t, %be] : f32, view<[%tokens]x1152xf32> } kernel.return } kernel.def export("v_pos_interp_1152_48_block2_ac") @v_pos_interp_1152_48_block2_ac(%tokens: index) { %c1 = index.constant 1 : index %c256 = index.constant 256 : index kernel.launch.config workgroups(%tokens, %c1, %c1) workgroup_size(%c256, %c1, %c1) : index } launch(%tokens_arg: index, %gw_arg: index, %gh_arg: index, %tab: buffer, %y: buffer) { %tokens = index.assume %tokens_arg [range(%tokens_arg, 1, 65536)] : index %gw = index.assume %gw_arg [range(%gw_arg, 1, 1024)] : index %gh = index.assume %gh_arg [range(%gh_arg, 1, 1024)] : index %t_id = kernel.workgroup.id : index %t = index.assume %t_id [lt(%t_id, %tokens)] : index %lid = kernel.workitem.id : index %lane = index.assume %lid [range(%lid, 0, 255)] : index %zero = index.constant 0 : offset %c0 = index.constant 0 : index %c1 = index.constant 1 : index %c256 = index.constant 256 : index %cE = index.constant 1152 : index %nm1 = index.constant 47 : index %c0i = scalar.constant 0 : i32 %nm1i = scalar.constant 47 : i32 %zf = scalar.constant 0.0 : f32 %onef = scalar.constant 1.0 : f32 %halff = scalar.constant 0.5 : f32 %nf = scalar.constant 48.0 : f32 %nm1f = scalar.constant 47.0 : f32 %c2o = index.constant 2 : index %c4o = index.constant 4 : index %bw0 = index.div %gw, %c2o : index %bw1 = index.max %bw0, %c1 : index %bw = index.assume %bw1 [range(%bw1, 1, 4096)] : index %blk = index.div %t, %c4o : index %sub = index.rem %t, %c4o : index %ty = index.div %blk, %bw : index %tx = index.rem %blk, %bw : index %dy = index.div %sub, %c2o : index %dx = index.rem %sub, %c2o : index %py = index.madd %ty, %c2o, %dy : index %px = index.madd %tx, %c2o, %dx : index %py_of_i = index.cast %py : index to i32 %py_of = scalar.sitofp %py_of_i : i32 to f32 %gh_m1 = index.sub %gh, %c1 : index %gh_m1f_i = index.cast %gh_m1 : index to i32 %gh_m1f = scalar.sitofp %gh_m1f_i : i32 to f32 %gh_den = scalar.maxnumf %gh_m1f, %onef : f32 %py_s0 = scalar.mulf %py_of, %nm1f : f32 %py_s = scalar.divf %py_s0, %gh_den : f32 %px_of_i = index.cast %px : index to i32 %px_of = scalar.sitofp %px_of_i : i32 to f32 %gw_m1 = index.sub %gw, %c1 : index %gw_m1f_i = index.cast %gw_m1 : index to i32 %gw_m1f = scalar.sitofp %gw_m1f_i : i32 to f32 %gw_den = scalar.maxnumf %gw_m1f, %onef : f32 %px_s0 = scalar.mulf %px_of, %nm1f : f32 %px_s = scalar.divf %px_s0, %gw_den : f32 %py_fl = scalar.floorf %py_s : f32 %py_fr = scalar.subf %py_s, %py_fl : f32 %py_i0r = scalar.fptosi %py_fl : f32 to i32 %py_i0c = scalar.maxsi %py_i0r, %c0i : i32 %py_i0d = scalar.minsi %py_i0c, %nm1i : i32 %py_i0a = scalar.assume %py_i0d [range(%py_i0d, 0, 47)] : i32 %py_i0 = index.cast %py_i0a : i32 to index %py_i1r = index.add %py_i0, %c1 : index %py_i1m = index.min %py_i1r, %nm1 : index %py_i1 = index.assume %py_i1m [range(%py_i1m, 0, 47)] : index %px_fl = scalar.floorf %px_s : f32 %px_fr = scalar.subf %px_s, %px_fl : f32 %px_i0r = scalar.fptosi %px_fl : f32 to i32 %px_i0c = scalar.maxsi %px_i0r, %c0i : i32 %px_i0d = scalar.minsi %px_i0c, %nm1i : i32 %px_i0a = scalar.assume %px_i0d [range(%px_i0d, 0, 47)] : i32 %px_i0 = index.cast %px_i0a : i32 to index %px_i1r = index.add %px_i0, %c1 : index %px_i1m = index.min %px_i1r, %nm1 : index %px_i1 = index.assume %px_i1m [range(%px_i1m, 0, 47)] : index %fy1 = scalar.addf %py_fr, %zf : f32 %fy0 = scalar.subf %onef, %py_fr : f32 %fx1 = scalar.addf %px_fr, %zf : f32 %fx0 = scalar.subf %onef, %px_fr : f32 %tv = buffer.view %tab[%zero] : buffer -> view<48x48x1152xf32> %yv = buffer.view %y[%zero] : buffer -> view<[%tokens]x1152xf32> scf.for %e = [%lane to %cE step %c256] { %be = index.assume %e [lt(%e, %cE)] : index %v00 = view.load %tv[%py_i0, %px_i0, %be] : view<48x48x1152xf32> -> f32 %v01 = view.load %tv[%py_i0, %px_i1, %be] : view<48x48x1152xf32> -> f32 %v10 = view.load %tv[%py_i1, %px_i0, %be] : view<48x48x1152xf32> -> f32 %v11 = view.load %tv[%py_i1, %px_i1, %be] : view<48x48x1152xf32> -> f32 %r0a = scalar.mulf %v00, %fx0 : f32 %r0 = scalar.fmaf %v01, %fx1, %r0a : f32 %r1a = scalar.mulf %v10, %fx0 : f32 %r1 = scalar.fmaf %v11, %fx1, %r1a : f32 %ra = scalar.mulf %r0, %fy0 : f32 %r = scalar.fmaf %r1, %fy1, %ra : f32 %a = view.load %yv[%t, %be] : view<[%tokens]x1152xf32> -> f32 %s = scalar.addf %a, %r : f32 view.store %s, %yv[%t, %be] : f32, view<[%tokens]x1152xf32> } kernel.return } kernel.def export("v_norm_ln_1152") @v_norm_ln_1152(%rows: index) { %c1 = index.constant 1 : index %c256 = index.constant 256 : index kernel.launch.config workgroups(%rows, %c1, %c1) workgroup_size(%c256, %c1, %c1) : index } launch(%rows_arg: index, %x: buffer, %w: buffer, %b: buffer, %y: buffer) { %rows = index.assume %rows_arg [range(%rows_arg, 1, 65536)] : index %row_id = kernel.workgroup.id : index %row = index.assume %row_id [lt(%row_id, %rows)] : index %lane_id = kernel.workitem.id : index %lane = index.assume %lane_id [range(%lane_id, 0, 255)] : index %zero = index.constant 0 : offset %c0 = index.constant 0 : index %groups = index.constant 36 : index %zero_f = scalar.constant 0.0 : f32 %zero32 = vector.constant 0.0 : vector<32xf32> %count = scalar.constant 1152.0 : f32 %active = index.cmp ult, %lane, %groups : index %x_view = buffer.view %x[%zero] : buffer -> view<[%rows]x36x32xf32> %xv = scf.if %active -> (vector<32xf32>) { %bl = index.assume %lane [range(%lane, 0, 35)] : index %a = vector.load %x_view[%row, %bl, %c0] : view<[%rows]x36x32xf32> -> vector<32xf32> scf.yield %a : vector<32xf32> } else { scf.yield %zero32 : vector<32xf32> } %xlocal = vector.reduce %xv, %zero_f : vector<32xf32>, f32 %xsum = kernel.workgroup.reduce %xlocal : f32 %xmean = scalar.divf %xsum, %count : f32 %xmeans = vector.splat %xmean : vector<32xf32> %xd = vector.subf %xv, %xmeans : vector<32xf32> %xc = scf.select %active, %xd, %zero32 : vector<32xf32> %sq = vector.mulf %xc, %xc : vector<32xf32> %local = vector.reduce %sq, %zero_f : vector<32xf32>, f32 %sum = kernel.workgroup.reduce %local : f32 %mean = scalar.divf %sum, %count : f32 %eps = scalar.constant 9.999999974752427e-07 : f32 %stable = scalar.addf %mean, %eps : f32 %inv = scalar.rsqrtf %stable : f32 %invs = vector.splat %inv : vector<32xf32> scf.if %active { %bl = index.assume %lane [range(%lane, 0, 35)] : index %xn = vector.mulf %xc, %invs : vector<32xf32> %w_view = buffer.view %w[%zero] : buffer -> view<36x32xf32> %b_view = buffer.view %b[%zero] : buffer -> view<36x32xf32> %wv = vector.load %w_view[%bl, %c0] : view<36x32xf32> -> vector<32xf32> %bv = vector.load %b_view[%bl, %c0] : view<36x32xf32> -> vector<32xf32> %xw = vector.mulf %xn, %wv : vector<32xf32> %r = vector.addf %xw, %bv : vector<32xf32> %y_view = buffer.view %y[%zero] : buffer -> view<[%rows]x36x32xf32> vector.store %r, %y_view[%row, %bl, %c0] : vector<32xf32>, view<[%rows]x36x32xf32> } kernel.return } kernel.def export("v_gemm_f16_store_1152x3456") @v_gemm_f16_store_1152x3456(%rows: index, %n: index) { %c1 = index.constant 1 : index %c64 = index.constant 64 : index %c256 = index.constant 256 : index %row_tiles = index.div %rows, %c64 : index %n_const = index.constant 3456 : index %out_tiles = index.div %n_const, %c64 : index kernel.launch.config workgroups(%row_tiles, %out_tiles, %c1) workgroup_size(%c256, %c1, %c1) : index } launch(%rows_arg: index, %k_arg: index, %n_arg: index, %x: buffer, %scales: buffer, %w: buffer, %y: buffer) { %rows = index.assume %rows_arg [range(%rows_arg, 64, 65536), multiple_of(%rows_arg, 64)] : index %k = index.constant 1152 : index %n = index.constant 3456 : index %origin = index.constant 0 : offset %two = index.constant 2 : offset %sixteen = index.constant 16 : offset %block_bytes = index.constant 64 : offset %c0 = index.constant 0 : index %c1 = index.constant 1 : index %c2 = index.constant 2 : index %c4 = index.constant 4 : index %c8 = index.constant 8 : index %c16 = index.constant 16 : index %c32 = index.constant 32 : index %c64 = index.constant 64 : index %c256 = index.constant 256 : index %nibble_mask = vector.constant 252645135 : vector<4xi32> %stage_bytes = index.constant 9216 : offset %row_tile_count = index.div %rows, %c64 : index %out_tile_count = index.div %n, %c64 : index %row_group_id = kernel.workgroup.id : index %out_group_id = kernel.workgroup.id : index %row_group = index.assume %row_group_id [lt(%row_group_id, %row_tile_count)] : index %out_group = index.assume %out_group_id [lt(%out_group_id, %out_tile_count)] : index %workitem_id = kernel.workitem.id : index %workitem = index.assume %workitem_id [range(%workitem_id, 0, 255)] : index %subgroup = kernel.subgroup.id : index %lane = kernel.subgroup.lane.id : index %lane_row = index.rem %lane, %c16 : index %bounded_subgroup = index.assume %subgroup [range(%subgroup, 0, 7)] : index %wave_row_group = index.div %bounded_subgroup, %c2 : index %wave_out_group = index.rem %bounded_subgroup, %c2 : index %wave_row = index.mul %wave_row_group, %c16 : index %wave_out = index.mul %wave_out_group, %c32 : index %w_row0 = index.add %wave_out, %lane_row : index %w_row1 = index.add %w_row0, %c16 : index %row_origin = index.mul %row_group, %c64 : index %out_origin = index.mul %out_group, %c64 : index %result_row = index.add %row_origin, %wave_row : index %result_col0 = index.add %out_origin, %wave_out : index %result_col1 = index.add %result_col0, %c16 : index %blocks = index.div %k, %c32 : index %row_bytes = index.scale %blocks, %block_bytes : index, offset -> offset %x_view = buffer.view %x[%origin] : buffer -> view<[%rows]x[%k]xf16> // 64x64 output tile with the full row stride n. %y_layout = encoding.layout.strided [%n, %c1] : encoding %tile_first = index.madd %row_origin, %n, %out_origin : index %four = index.constant 4 : offset %tile_offset = index.scale %tile_first, %four : index, offset -> offset %y_view = buffer.view %y[%tile_offset] : buffer -> view<64x64xf32, %y_layout> %bounded_wave_row = index.assume %wave_row [range(%wave_row, 0, 48)] : index %bounded_wave_out = index.assume %wave_out [range(%wave_out, 0, 32)] : index %wave_out1 = index.add %bounded_wave_out, %c16 : index %x_stage = buffer.alloca align(16) %stage_bytes : buffer %x_stage_view = buffer.view %x_stage[%origin] : buffer -> view<64x72xf16> %w_stage = buffer.alloca align(16) %stage_bytes : buffer %w_stage_view = buffer.view %w_stage[%origin] : buffer -> view<64x72xf16> // Thread t stages 16 values: row t/4, columns (t%4)*16. %stage_row = index.div %workitem, %c4 : index %stage_quarter = index.rem %workitem, %c4 : index %stage_col = index.mul %stage_quarter, %c16 : index %x_row = index.add %row_origin, %stage_row : index %bounded_x_row = index.assume %x_row [lt(%x_row, %rows)] : index %w_out_row = index.add %out_origin, %stage_row : index %bounded_w_out_row = index.assume %w_out_row [lt(%w_out_row, %n)] : index %w_row_origin = index.scale %bounded_w_out_row, %row_bytes : index, offset -> offset %zero8 = vector.constant 0.0 : vector<8xf32> %init = vector.fragment %zero8 shape [%c16, %c16] : vector<8xf32> %k_steps = index.div %k, %c64 : index %s_view = buffer.view %scales[%origin] : buffer -> view<[%rows]x2xf32> %x_scale = view.load %s_view[%bounded_x_row, %c0] : view<[%rows]x2xf32> -> f32 %x_scales = vector.splat %x_scale : vector<16xf32> %a0_raw, %a1_raw = scf.for %step = [%c0 to %k_steps step %c1](%a0_acc = %init : vector<8xf32>, %a1_acc = %init : vector<8xf32>) -> (vector<8xf32>, vector<8xf32>) { %k_origin = index.mul %step, %c64 : index %x_col = index.add %k_origin, %stage_col : index %x_half = vector.load %x_view[%bounded_x_row, %x_col] : view<[%rows]x[%k]xf16> -> vector<16xf16> vector.store %x_half, %x_stage_view[%stage_row, %stage_col] : vector<16xf16>, view<64x72xf16> %w_col = index.add %k_origin, %stage_col : index %deq0_two_b = index.constant 2 : offset %deq0_col_bytes = index.scale %w_col, %deq0_two_b : index, offset -> offset %deq0_origin = index.add %w_row_origin, %deq0_col_bytes : offset %deq0_view = buffer.view %w[%deq0_origin] : buffer -> view<16xf16> %deq0 = vector.load %deq0_view[0] : view<16xf16> -> vector<16xf16> vector.store %deq0, %w_stage_view[%stage_row, %stage_col] : vector<16xf16>, view<64x72xf16> kernel.barrier scope(workgroup) ordering(acq_rel) %fc0 = index.constant 0 : index %lhs0 = vector.fragment.load %x_stage_view[%wave_row, %fc0] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16> %rv000 = vector.load %w_stage_view[%w_row0, %fc0] : view<64x72xf16> -> vector<16xf16> %rhs000 = vector.fragment %rv000 shape [%c16, %c16] : vector<16xf16> %na0_0 = vector.mma %lhs0, %rhs000, %a0_acc : vector<16xf16>, vector<16xf16>, vector<8xf32> %rv010 = vector.load %w_stage_view[%w_row1, %fc0] : view<64x72xf16> -> vector<16xf16> %rhs010 = vector.fragment %rv010 shape [%c16, %c16] : vector<16xf16> %na1_0 = vector.mma %lhs0, %rhs010, %a1_acc : vector<16xf16>, vector<16xf16>, vector<8xf32> %fc1 = index.constant 16 : index %lhs1 = vector.fragment.load %x_stage_view[%wave_row, %fc1] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16> %rv001 = vector.load %w_stage_view[%w_row0, %fc1] : view<64x72xf16> -> vector<16xf16> %rhs001 = vector.fragment %rv001 shape [%c16, %c16] : vector<16xf16> %na0_1 = vector.mma %lhs1, %rhs001, %na0_0 : vector<16xf16>, vector<16xf16>, vector<8xf32> %rv011 = vector.load %w_stage_view[%w_row1, %fc1] : view<64x72xf16> -> vector<16xf16> %rhs011 = vector.fragment %rv011 shape [%c16, %c16] : vector<16xf16> %na1_1 = vector.mma %lhs1, %rhs011, %na1_0 : vector<16xf16>, vector<16xf16>, vector<8xf32> %fc2 = index.constant 32 : index %lhs2 = vector.fragment.load %x_stage_view[%wave_row, %fc2] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16> %rv002 = vector.load %w_stage_view[%w_row0, %fc2] : view<64x72xf16> -> vector<16xf16> %rhs002 = vector.fragment %rv002 shape [%c16, %c16] : vector<16xf16> %na0_2 = vector.mma %lhs2, %rhs002, %na0_1 : vector<16xf16>, vector<16xf16>, vector<8xf32> %rv012 = vector.load %w_stage_view[%w_row1, %fc2] : view<64x72xf16> -> vector<16xf16> %rhs012 = vector.fragment %rv012 shape [%c16, %c16] : vector<16xf16> %na1_2 = vector.mma %lhs2, %rhs012, %na1_1 : vector<16xf16>, vector<16xf16>, vector<8xf32> %fc3 = index.constant 48 : index %lhs3 = vector.fragment.load %x_stage_view[%wave_row, %fc3] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16> %rv003 = vector.load %w_stage_view[%w_row0, %fc3] : view<64x72xf16> -> vector<16xf16> %rhs003 = vector.fragment %rv003 shape [%c16, %c16] : vector<16xf16> %na0_3 = vector.mma %lhs3, %rhs003, %na0_2 : vector<16xf16>, vector<16xf16>, vector<8xf32> %rv013 = vector.load %w_stage_view[%w_row1, %fc3] : view<64x72xf16> -> vector<16xf16> %rhs013 = vector.fragment %rv013 shape [%c16, %c16] : vector<16xf16> %na1_3 = vector.mma %lhs3, %rhs013, %na1_2 : vector<16xf16>, vector<16xf16>, vector<8xf32> kernel.barrier scope(workgroup) ordering(acq_rel) scf.yield %na0_3, %na1_3 : vector<8xf32>, vector<8xf32> } %inv_bytes = index.constant 4096 : offset %inv_tile = buffer.alloca align(16) %inv_bytes : buffer %inv_view = buffer.view %inv_tile[%origin] : buffer -> view<64x16xf32> %writer = index.cmp ult, %workitem, %c64 : index scf.if %writer { %inv_row_value = index.add %row_origin, %workitem : index %inv_row = index.assume %inv_row_value [lt(%inv_row_value, %rows)] : index %inverse = view.load %s_view[%inv_row, %c1] : view<[%rows]x2xf32> -> f32 %inverse16 = vector.splat %inverse : vector<16xf32> %bw = index.assume %workitem [range(%workitem, 0, 63)] : index vector.store %inverse16, %inv_view[%bw, %c0] : vector<16xf32>, view<64x16xf32> } kernel.barrier scope(workgroup) ordering(acq_rel) %inv_frag = vector.fragment.load %inv_view[%wave_row, %c0] shape [%c16, %c16] : view<64x16xf32> -> vector<8xf32> %a0 = vector.mulf %a0_raw, %inv_frag : vector<8xf32> %a1 = vector.mulf %a1_raw, %inv_frag : vector<8xf32> vector.fragment.store %a0, %y_view[%bounded_wave_row, %bounded_wave_out] shape [%c16, %c16] : vector<8xf32>, view<64x64xf32, %y_layout> vector.fragment.store %a1, %y_view[%bounded_wave_row, %wave_out1] shape [%c16, %c16] : vector<8xf32>, view<64x64xf32, %y_layout> kernel.return } kernel.def export("v_bias_none_3456") @v_bias_none_3456(%rows: index) { %c1 = index.constant 1 : index %c256 = index.constant 256 : index %cols = index.constant 3456 : index %cg = index.constant 14 : index kernel.launch.config workgroups(%rows, %cg, %c1) workgroup_size(%c256, %c1, %c1) : index } launch(%rows_arg: index, %x: buffer, %bias: buffer, %y: buffer) { %rows = index.assume %rows_arg [range(%rows_arg, 1, 65536)] : index %c256 = index.constant 256 : index %cols = index.constant 3456 : index %row_id = kernel.workgroup.id : index %row = index.assume %row_id [lt(%row_id, %rows)] : index %cg = kernel.workgroup.id : index %lid = kernel.workitem.id : index %c = index.madd %cg, %c256, %lid : index %live = index.cmp ult, %c, %cols : index scf.if %live { %bc = index.assume %c [lt(%c, %cols)] : index %zero = index.constant 0 : offset %zf = scalar.constant 0.0 : f32 %xv = buffer.view %x[%zero] : buffer -> view<[%rows]x3456xf32> %bv = buffer.view %bias[%zero] : buffer -> view<3456xf32> %x0 = view.load %xv[%row, %bc] : view<[%rows]x3456xf32> -> f32 %b0 = view.load %bv[%bc] : view<3456xf32> -> f32 %a = scalar.addf %x0, %b0 : f32 %r = scalar.addf %a, %zf : f32 %yv = buffer.view %y[%zero] : buffer -> view<[%rows]x3456xf32> view.store %r, %yv[%row, %bc] : f32, view<[%rows]x3456xf32> } kernel.return } kernel.def export("v_rope_qwen_vision_72x16_s3456_o0_block2") @v_rope_qwen_vision_72x16_s3456_o0_block2(%tokens: index) { %c1 = index.constant 1 : index %c256 = index.constant 256 : index kernel.launch.config workgroups(%tokens, %c1, %c1) workgroup_size(%c256, %c1, %c1) : index } launch(%tokens_arg: index, %gw_arg: index, %x: buffer) { %tokens = index.assume %tokens_arg [range(%tokens_arg, 1, 65536)] : index %gw = index.assume %gw_arg [range(%gw_arg, 1, 4096)] : index %t_id = kernel.workgroup.id : index %t = index.assume %t_id [lt(%t_id, %tokens)] : index %lid = kernel.workitem.id : index %lane = index.assume %lid [range(%lid, 0, 255)] : index %zero = index.constant 0 : offset %c0 = index.constant 0 : index %c1 = index.constant 1 : index %c256 = index.constant 256 : index %cNP = index.constant 36 : index %cWORK = index.constant 576 : index %cD = index.constant 72 : index %cO = index.constant 0 : index %c2o = index.constant 2 : index %c4o = index.constant 4 : index %bw0 = index.div %gw, %c2o : index %bw1 = index.max %bw0, %c1 : index %bw = index.assume %bw1 [range(%bw1, 1, 4096)] : index %blk = index.div %t, %c4o : index %sub = index.rem %t, %c4o : index %ty = index.div %blk, %bw : index %tx = index.rem %blk, %bw : index %dy = index.div %sub, %c2o : index %dx = index.rem %sub, %c2o : index %py = index.madd %ty, %c2o, %dy : index %px = index.madd %tx, %c2o, %dx : index %pxi = index.cast %px : index to i32 %pyi = index.cast %py : index to i32 %offi = scalar.constant 0 : i32 %pxo = scalar.addi %pxi, %offi : i32 %pyo = scalar.addi %pyi, %offi : i32 %pxf = scalar.sitofp %pxo : i32 to f32 %pyf = scalar.sitofp %pyo : i32 to f32 %two_pi = scalar.constant 6.283185307179586 : f32 %inv_two_pi = scalar.constant 0.15915494309189535 : f32 %xv = buffer.view %x[%zero] : buffer -> view<[%tokens]x3456xf32> %cq = index.constant 18 : index %chalf = index.constant 36 : index %lnth = scalar.constant 18.420680743952367 : f32 %halff = scalar.constant 36.0 : f32 scf.for %w = [%lane to %cWORK step %c256] { %h = index.div %w, %cNP : index %j0 = index.rem %w, %cNP : index %j = index.assume %j0 [range(%j0, 0, 35)] : index %isx = index.cmp uge, %j, %cq : index %true = scalar.constant true : i1 %isy = scalar.xori %isx, %true : i1 %jj = index.rem %j, %cq : index %lo_i = index.add %j, %c0 : index %hi_i = index.add %j, %chalf : index %ii = index.cast %jj : index to i32 %if0 = scalar.sitofp %ii : i32 to f32 %if1 = scalar.mulf %if0, %lnth : f32 %if2 = scalar.divf %if1, %halff : f32 %if3 = scalar.negf %if2 : f32 %invf = scalar.expf %if3 : f32 %pos = scf.select %isy, %pyf, %pxf : f32 %ang0 = scalar.mulf %pos, %invf : f32 %tr = scalar.mulf %ang0, %inv_two_pi : f32 %trf = scalar.floorf %tr : f32 %wrap = scalar.mulf %trf, %two_pi : f32 %ang = scalar.subf %ang0, %wrap : f32 %cs = scalar.cosf %ang : f32 %sn = scalar.sinf %ang : f32 %hcol = index.madd %h, %cD, %cO : index %clo0 = index.add %hcol, %lo_i : index %chi0 = index.add %hcol, %hi_i : index %clo = index.assume %clo0 [lt(%clo0, 3456)] : index %chi = index.assume %chi0 [lt(%chi0, 3456)] : index %a = view.load %xv[%t, %clo] : view<[%tokens]x3456xf32> -> f32 %b = view.load %xv[%t, %chi] : view<[%tokens]x3456xf32> -> f32 %acs = scalar.mulf %a, %cs : f32 %bsn = scalar.mulf %b, %sn : f32 %ra = scalar.subf %acs, %bsn : f32 %asn = scalar.mulf %a, %sn : f32 %rb = scalar.fmaf %b, %cs, %asn : f32 view.store %ra, %xv[%t, %clo] : f32, view<[%tokens]x3456xf32> view.store %rb, %xv[%t, %chi] : f32, view<[%tokens]x3456xf32> } kernel.return } kernel.def export("v_rope_qwen_vision_72x16_s3456_o1152_block2") @v_rope_qwen_vision_72x16_s3456_o1152_block2(%tokens: index) { %c1 = index.constant 1 : index %c256 = index.constant 256 : index kernel.launch.config workgroups(%tokens, %c1, %c1) workgroup_size(%c256, %c1, %c1) : index } launch(%tokens_arg: index, %gw_arg: index, %x: buffer) { %tokens = index.assume %tokens_arg [range(%tokens_arg, 1, 65536)] : index %gw = index.assume %gw_arg [range(%gw_arg, 1, 4096)] : index %t_id = kernel.workgroup.id : index %t = index.assume %t_id [lt(%t_id, %tokens)] : index %lid = kernel.workitem.id : index %lane = index.assume %lid [range(%lid, 0, 255)] : index %zero = index.constant 0 : offset %c0 = index.constant 0 : index %c1 = index.constant 1 : index %c256 = index.constant 256 : index %cNP = index.constant 36 : index %cWORK = index.constant 576 : index %cD = index.constant 72 : index %cO = index.constant 1152 : index %c2o = index.constant 2 : index %c4o = index.constant 4 : index %bw0 = index.div %gw, %c2o : index %bw1 = index.max %bw0, %c1 : index %bw = index.assume %bw1 [range(%bw1, 1, 4096)] : index %blk = index.div %t, %c4o : index %sub = index.rem %t, %c4o : index %ty = index.div %blk, %bw : index %tx = index.rem %blk, %bw : index %dy = index.div %sub, %c2o : index %dx = index.rem %sub, %c2o : index %py = index.madd %ty, %c2o, %dy : index %px = index.madd %tx, %c2o, %dx : index %pxi = index.cast %px : index to i32 %pyi = index.cast %py : index to i32 %offi = scalar.constant 0 : i32 %pxo = scalar.addi %pxi, %offi : i32 %pyo = scalar.addi %pyi, %offi : i32 %pxf = scalar.sitofp %pxo : i32 to f32 %pyf = scalar.sitofp %pyo : i32 to f32 %two_pi = scalar.constant 6.283185307179586 : f32 %inv_two_pi = scalar.constant 0.15915494309189535 : f32 %xv = buffer.view %x[%zero] : buffer -> view<[%tokens]x3456xf32> %cq = index.constant 18 : index %chalf = index.constant 36 : index %lnth = scalar.constant 18.420680743952367 : f32 %halff = scalar.constant 36.0 : f32 scf.for %w = [%lane to %cWORK step %c256] { %h = index.div %w, %cNP : index %j0 = index.rem %w, %cNP : index %j = index.assume %j0 [range(%j0, 0, 35)] : index %isx = index.cmp uge, %j, %cq : index %true = scalar.constant true : i1 %isy = scalar.xori %isx, %true : i1 %jj = index.rem %j, %cq : index %lo_i = index.add %j, %c0 : index %hi_i = index.add %j, %chalf : index %ii = index.cast %jj : index to i32 %if0 = scalar.sitofp %ii : i32 to f32 %if1 = scalar.mulf %if0, %lnth : f32 %if2 = scalar.divf %if1, %halff : f32 %if3 = scalar.negf %if2 : f32 %invf = scalar.expf %if3 : f32 %pos = scf.select %isy, %pyf, %pxf : f32 %ang0 = scalar.mulf %pos, %invf : f32 %tr = scalar.mulf %ang0, %inv_two_pi : f32 %trf = scalar.floorf %tr : f32 %wrap = scalar.mulf %trf, %two_pi : f32 %ang = scalar.subf %ang0, %wrap : f32 %cs = scalar.cosf %ang : f32 %sn = scalar.sinf %ang : f32 %hcol = index.madd %h, %cD, %cO : index %clo0 = index.add %hcol, %lo_i : index %chi0 = index.add %hcol, %hi_i : index %clo = index.assume %clo0 [lt(%clo0, 3456)] : index %chi = index.assume %chi0 [lt(%chi0, 3456)] : index %a = view.load %xv[%t, %clo] : view<[%tokens]x3456xf32> -> f32 %b = view.load %xv[%t, %chi] : view<[%tokens]x3456xf32> -> f32 %acs = scalar.mulf %a, %cs : f32 %bsn = scalar.mulf %b, %sn : f32 %ra = scalar.subf %acs, %bsn : f32 %asn = scalar.mulf %a, %sn : f32 %rb = scalar.fmaf %b, %cs, %asn : f32 view.store %ra, %xv[%t, %clo] : f32, view<[%tokens]x3456xf32> view.store %rb, %xv[%t, %chi] : f32, view<[%tokens]x3456xf32> } kernel.return } kernel.def export("v_attn_72x16_s3456_0_1152_2304") @v_attn_72x16_s3456_0_1152_2304(%tokens: index) { %c1 = index.constant 1 : index %c32 = index.constant 32 : index %heads = index.constant 16 : index kernel.launch.config workgroups(%tokens, %heads, %c1) workgroup_size(%c32, %c1, %c1) : index } launch(%tokens_arg: index, %gw_arg: index, %q: buffer, %k: buffer, %v: buffer, %o: buffer) { %tokens = index.assume %tokens_arg [range(%tokens_arg, 1, 65536)] : index %gw = index.assume %gw_arg [range(%gw_arg, 1, 4096)] : index %t_id = kernel.workgroup.id : index %t = index.assume %t_id [lt(%t_id, %tokens)] : index %h_id = kernel.workgroup.id : index %h = index.assume %h_id [range(%h_id, 0, 15)] : index %lid = kernel.workitem.id : index %lane = index.assume %lid [range(%lid, 0, 31)] : index %zero = index.constant 0 : offset %c0 = index.constant 0 : index %c1 = index.constant 1 : index %c32 = index.constant 32 : index %cD = index.constant 72 : index %zf = scalar.constant 0.0 : f32 %neg_big = scalar.constant -3.0e38 : f32 %scale = scalar.constant 0.11785113019775792 : f32 %zero32 = vector.constant 0.0 : vector<8xf32> %qv = buffer.view %q[%zero] : buffer -> view<[%tokens]x3456xf32> %kv = buffer.view %k[%zero] : buffer -> view<[%tokens]x3456xf32> %vv = buffer.view %v[%zero] : buffer -> view<[%tokens]x3456xf32> %cOQ = index.constant 0 : index %cOK = index.constant 1152 : index %cOV = index.constant 2304 : index %qcol = index.madd %h, %cD, %cOQ : index %kcol = index.madd %h, %cD, %cOK : index %vcol = index.madd %h, %cD, %cOV : index %qc00 = index.constant 0 : index %qc01 = index.add %qcol, %qc00 : index %qc0 = index.assume %qc01 [range(%qc01, 0, 3448)] : index %qr0 = vector.load %qv[%t, %qc0] : view<[%tokens]x3456xf32> -> vector<8xf32> %scv0 = vector.splat %scale : vector<8xf32> %q0 = vector.mulf %qr0, %scv0 : vector<8xf32> %qc10 = index.constant 8 : index %qc11 = index.add %qcol, %qc10 : index %qc1 = index.assume %qc11 [range(%qc11, 0, 3448)] : index %qr1 = vector.load %qv[%t, %qc1] : view<[%tokens]x3456xf32> -> vector<8xf32> %scv1 = vector.splat %scale : vector<8xf32> %q1 = vector.mulf %qr1, %scv1 : vector<8xf32> %qc20 = index.constant 16 : index %qc21 = index.add %qcol, %qc20 : index %qc2 = index.assume %qc21 [range(%qc21, 0, 3448)] : index %qr2 = vector.load %qv[%t, %qc2] : view<[%tokens]x3456xf32> -> vector<8xf32> %scv2 = vector.splat %scale : vector<8xf32> %q2 = vector.mulf %qr2, %scv2 : vector<8xf32> %qc30 = index.constant 24 : index %qc31 = index.add %qcol, %qc30 : index %qc3 = index.assume %qc31 [range(%qc31, 0, 3448)] : index %qr3 = vector.load %qv[%t, %qc3] : view<[%tokens]x3456xf32> -> vector<8xf32> %scv3 = vector.splat %scale : vector<8xf32> %q3 = vector.mulf %qr3, %scv3 : vector<8xf32> %qc40 = index.constant 32 : index %qc41 = index.add %qcol, %qc40 : index %qc4 = index.assume %qc41 [range(%qc41, 0, 3448)] : index %qr4 = vector.load %qv[%t, %qc4] : view<[%tokens]x3456xf32> -> vector<8xf32> %scv4 = vector.splat %scale : vector<8xf32> %q4 = vector.mulf %qr4, %scv4 : vector<8xf32> %qc50 = index.constant 40 : index %qc51 = index.add %qcol, %qc50 : index %qc5 = index.assume %qc51 [range(%qc51, 0, 3448)] : index %qr5 = vector.load %qv[%t, %qc5] : view<[%tokens]x3456xf32> -> vector<8xf32> %scv5 = vector.splat %scale : vector<8xf32> %q5 = vector.mulf %qr5, %scv5 : vector<8xf32> %qc60 = index.constant 48 : index %qc61 = index.add %qcol, %qc60 : index %qc6 = index.assume %qc61 [range(%qc61, 0, 3448)] : index %qr6 = vector.load %qv[%t, %qc6] : view<[%tokens]x3456xf32> -> vector<8xf32> %scv6 = vector.splat %scale : vector<8xf32> %q6 = vector.mulf %qr6, %scv6 : vector<8xf32> %qc70 = index.constant 56 : index %qc71 = index.add %qcol, %qc70 : index %qc7 = index.assume %qc71 [range(%qc71, 0, 3448)] : index %qr7 = vector.load %qv[%t, %qc7] : view<[%tokens]x3456xf32> -> vector<8xf32> %scv7 = vector.splat %scale : vector<8xf32> %q7 = vector.mulf %qr7, %scv7 : vector<8xf32> %qc80 = index.constant 64 : index %qc81 = index.add %qcol, %qc80 : index %qc8 = index.assume %qc81 [range(%qc81, 0, 3448)] : index %qr8 = vector.load %qv[%t, %qc8] : view<[%tokens]x3456xf32> -> vector<8xf32> %scv8 = vector.splat %scale : vector<8xf32> %q8 = vector.mulf %qr8, %scv8 : vector<8xf32> %m_l, %l_l, %acc0_l, %acc1_l, %acc2_l, %acc3_l, %acc4_l, %acc5_l, %acc6_l, %acc7_l, %acc8_l = scf.for %j = [%lane to %tokens step %c32](%m_c = %neg_big : f32, %l_c = %zf : f32, %acc0_c = %zero32 : vector<8xf32>, %acc1_c = %zero32 : vector<8xf32>, %acc2_c = %zero32 : vector<8xf32>, %acc3_c = %zero32 : vector<8xf32>, %acc4_c = %zero32 : vector<8xf32>, %acc5_c = %zero32 : vector<8xf32>, %acc6_c = %zero32 : vector<8xf32>, %acc7_c = %zero32 : vector<8xf32>, %acc8_c = %zero32 : vector<8xf32>) -> (f32, f32, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>) { %bj = index.assume %j [lt(%j, %tokens)] : index %allowed = index.cmp ult, %bj, %tokens : index %m_n, %l_n, %acc0_n, %acc1_n, %acc2_n, %acc3_n, %acc4_n, %acc5_n, %acc6_n, %acc7_n, %acc8_n = scf.if %allowed -> (f32, f32, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>) { %kc00 = index.constant 0 : index %kc01 = index.add %kcol, %kc00 : index %kc0 = index.assume %kc01 [range(%kc01, 0, 3448)] : index %kr0 = vector.load %kv[%bj, %kc0] : view<[%tokens]x3456xf32> -> vector<8xf32> %vc01 = index.add %vcol, %kc00 : index %vc0 = index.assume %vc01 [range(%vc01, 0, 3448)] : index %vr0 = vector.load %vv[%bj, %vc0] : view<[%tokens]x3456xf32> -> vector<8xf32> %pd0 = vector.mulf %q0, %kr0 : vector<8xf32> %ps0 = vector.reduce %pd0, %zf : vector<8xf32>, f32 %kc10 = index.constant 8 : index %kc11 = index.add %kcol, %kc10 : index %kc1 = index.assume %kc11 [range(%kc11, 0, 3448)] : index %kr1 = vector.load %kv[%bj, %kc1] : view<[%tokens]x3456xf32> -> vector<8xf32> %vc11 = index.add %vcol, %kc10 : index %vc1 = index.assume %vc11 [range(%vc11, 0, 3448)] : index %vr1 = vector.load %vv[%bj, %vc1] : view<[%tokens]x3456xf32> -> vector<8xf32> %pd1 = vector.mulf %q1, %kr1 : vector<8xf32> %ps1 = vector.reduce %pd1, %zf : vector<8xf32>, f32 %kc20 = index.constant 16 : index %kc21 = index.add %kcol, %kc20 : index %kc2 = index.assume %kc21 [range(%kc21, 0, 3448)] : index %kr2 = vector.load %kv[%bj, %kc2] : view<[%tokens]x3456xf32> -> vector<8xf32> %vc21 = index.add %vcol, %kc20 : index %vc2 = index.assume %vc21 [range(%vc21, 0, 3448)] : index %vr2 = vector.load %vv[%bj, %vc2] : view<[%tokens]x3456xf32> -> vector<8xf32> %pd2 = vector.mulf %q2, %kr2 : vector<8xf32> %ps2 = vector.reduce %pd2, %zf : vector<8xf32>, f32 %kc30 = index.constant 24 : index %kc31 = index.add %kcol, %kc30 : index %kc3 = index.assume %kc31 [range(%kc31, 0, 3448)] : index %kr3 = vector.load %kv[%bj, %kc3] : view<[%tokens]x3456xf32> -> vector<8xf32> %vc31 = index.add %vcol, %kc30 : index %vc3 = index.assume %vc31 [range(%vc31, 0, 3448)] : index %vr3 = vector.load %vv[%bj, %vc3] : view<[%tokens]x3456xf32> -> vector<8xf32> %pd3 = vector.mulf %q3, %kr3 : vector<8xf32> %ps3 = vector.reduce %pd3, %zf : vector<8xf32>, f32 %kc40 = index.constant 32 : index %kc41 = index.add %kcol, %kc40 : index %kc4 = index.assume %kc41 [range(%kc41, 0, 3448)] : index %kr4 = vector.load %kv[%bj, %kc4] : view<[%tokens]x3456xf32> -> vector<8xf32> %vc41 = index.add %vcol, %kc40 : index %vc4 = index.assume %vc41 [range(%vc41, 0, 3448)] : index %vr4 = vector.load %vv[%bj, %vc4] : view<[%tokens]x3456xf32> -> vector<8xf32> %pd4 = vector.mulf %q4, %kr4 : vector<8xf32> %ps4 = vector.reduce %pd4, %zf : vector<8xf32>, f32 %kc50 = index.constant 40 : index %kc51 = index.add %kcol, %kc50 : index %kc5 = index.assume %kc51 [range(%kc51, 0, 3448)] : index %kr5 = vector.load %kv[%bj, %kc5] : view<[%tokens]x3456xf32> -> vector<8xf32> %vc51 = index.add %vcol, %kc50 : index %vc5 = index.assume %vc51 [range(%vc51, 0, 3448)] : index %vr5 = vector.load %vv[%bj, %vc5] : view<[%tokens]x3456xf32> -> vector<8xf32> %pd5 = vector.mulf %q5, %kr5 : vector<8xf32> %ps5 = vector.reduce %pd5, %zf : vector<8xf32>, f32 %kc60 = index.constant 48 : index %kc61 = index.add %kcol, %kc60 : index %kc6 = index.assume %kc61 [range(%kc61, 0, 3448)] : index %kr6 = vector.load %kv[%bj, %kc6] : view<[%tokens]x3456xf32> -> vector<8xf32> %vc61 = index.add %vcol, %kc60 : index %vc6 = index.assume %vc61 [range(%vc61, 0, 3448)] : index %vr6 = vector.load %vv[%bj, %vc6] : view<[%tokens]x3456xf32> -> vector<8xf32> %pd6 = vector.mulf %q6, %kr6 : vector<8xf32> %ps6 = vector.reduce %pd6, %zf : vector<8xf32>, f32 %kc70 = index.constant 56 : index %kc71 = index.add %kcol, %kc70 : index %kc7 = index.assume %kc71 [range(%kc71, 0, 3448)] : index %kr7 = vector.load %kv[%bj, %kc7] : view<[%tokens]x3456xf32> -> vector<8xf32> %vc71 = index.add %vcol, %kc70 : index %vc7 = index.assume %vc71 [range(%vc71, 0, 3448)] : index %vr7 = vector.load %vv[%bj, %vc7] : view<[%tokens]x3456xf32> -> vector<8xf32> %pd7 = vector.mulf %q7, %kr7 : vector<8xf32> %ps7 = vector.reduce %pd7, %zf : vector<8xf32>, f32 %kc80 = index.constant 64 : index %kc81 = index.add %kcol, %kc80 : index %kc8 = index.assume %kc81 [range(%kc81, 0, 3448)] : index %kr8 = vector.load %kv[%bj, %kc8] : view<[%tokens]x3456xf32> -> vector<8xf32> %vc81 = index.add %vcol, %kc80 : index %vc8 = index.assume %vc81 [range(%vc81, 0, 3448)] : index %vr8 = vector.load %vv[%bj, %vc8] : view<[%tokens]x3456xf32> -> vector<8xf32> %pd8 = vector.mulf %q8, %kr8 : vector<8xf32> %ps8 = vector.reduce %pd8, %zf : vector<8xf32>, f32 %ss1 = scalar.addf %ps0, %ps1 : f32 %ss2 = scalar.addf %ss1, %ps2 : f32 %ss3 = scalar.addf %ss2, %ps3 : f32 %ss4 = scalar.addf %ss3, %ps4 : f32 %ss5 = scalar.addf %ss4, %ps5 : f32 %ss6 = scalar.addf %ss5, %ps6 : f32 %ss7 = scalar.addf %ss6, %ps7 : f32 %ss8 = scalar.addf %ss7, %ps8 : f32 %s = scalar.addf %ss8, %zf : f32 %mx = scalar.maxnumf %m_c, %s : f32 %dm = scalar.subf %m_c, %mx : f32 %corr = scalar.expf %dm : f32 %ds = scalar.subf %s, %mx : f32 %p = scalar.expf %ds : f32 %lc = scalar.mulf %l_c, %corr : f32 %ln = scalar.addf %lc, %p : f32 %corrv = vector.splat %corr : vector<8xf32> %pv = vector.splat %p : vector<8xf32> %ac0 = vector.mulf %acc0_c, %corrv : vector<8xf32> %an0 = vector.fmaf %vr0, %pv, %ac0 : vector<8xf32> %ac1 = vector.mulf %acc1_c, %corrv : vector<8xf32> %an1 = vector.fmaf %vr1, %pv, %ac1 : vector<8xf32> %ac2 = vector.mulf %acc2_c, %corrv : vector<8xf32> %an2 = vector.fmaf %vr2, %pv, %ac2 : vector<8xf32> %ac3 = vector.mulf %acc3_c, %corrv : vector<8xf32> %an3 = vector.fmaf %vr3, %pv, %ac3 : vector<8xf32> %ac4 = vector.mulf %acc4_c, %corrv : vector<8xf32> %an4 = vector.fmaf %vr4, %pv, %ac4 : vector<8xf32> %ac5 = vector.mulf %acc5_c, %corrv : vector<8xf32> %an5 = vector.fmaf %vr5, %pv, %ac5 : vector<8xf32> %ac6 = vector.mulf %acc6_c, %corrv : vector<8xf32> %an6 = vector.fmaf %vr6, %pv, %ac6 : vector<8xf32> %ac7 = vector.mulf %acc7_c, %corrv : vector<8xf32> %an7 = vector.fmaf %vr7, %pv, %ac7 : vector<8xf32> %ac8 = vector.mulf %acc8_c, %corrv : vector<8xf32> %an8 = vector.fmaf %vr8, %pv, %ac8 : vector<8xf32> scf.yield %mx, %ln, %an0, %an1, %an2, %an3, %an4, %an5, %an6, %an7, %an8 : f32, f32, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32> } else { scf.yield %m_c, %l_c, %acc0_c, %acc1_c, %acc2_c, %acc3_c, %acc4_c, %acc5_c, %acc6_c, %acc7_c, %acc8_c : f32, f32, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32> } scf.yield %m_n, %l_n, %acc0_n, %acc1_n, %acc2_n, %acc3_n, %acc4_n, %acc5_n, %acc6_n, %acc7_n, %acc8_n : f32, f32, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32> } %mg = kernel.subgroup.reduce %m_l : f32 %dml = scalar.subf %m_l, %mg : f32 %cl = scalar.expf %dml : f32 %lw = scalar.mulf %l_l, %cl : f32 %lg = kernel.subgroup.reduce %lw : f32 %one = scalar.constant 1.0 : f32 %invl = scalar.divf %one, %lg : f32 %clv = vector.splat %cl : vector<8xf32> %stage_bytes = index.constant 9216 : offset %stage = buffer.alloca align(16) %stage_bytes : buffer %st = buffer.view %stage[%zero] : buffer -> view<32x72xf32> %aw0 = vector.mulf %acc0_l, %clv : vector<8xf32> %sc0 = index.constant 0 : index vector.store %aw0, %st[%lane, %sc0] : vector<8xf32>, view<32x72xf32> %aw1 = vector.mulf %acc1_l, %clv : vector<8xf32> %sc1 = index.constant 8 : index vector.store %aw1, %st[%lane, %sc1] : vector<8xf32>, view<32x72xf32> %aw2 = vector.mulf %acc2_l, %clv : vector<8xf32> %sc2 = index.constant 16 : index vector.store %aw2, %st[%lane, %sc2] : vector<8xf32>, view<32x72xf32> %aw3 = vector.mulf %acc3_l, %clv : vector<8xf32> %sc3 = index.constant 24 : index vector.store %aw3, %st[%lane, %sc3] : vector<8xf32>, view<32x72xf32> %aw4 = vector.mulf %acc4_l, %clv : vector<8xf32> %sc4 = index.constant 32 : index vector.store %aw4, %st[%lane, %sc4] : vector<8xf32>, view<32x72xf32> %aw5 = vector.mulf %acc5_l, %clv : vector<8xf32> %sc5 = index.constant 40 : index vector.store %aw5, %st[%lane, %sc5] : vector<8xf32>, view<32x72xf32> %aw6 = vector.mulf %acc6_l, %clv : vector<8xf32> %sc6 = index.constant 48 : index vector.store %aw6, %st[%lane, %sc6] : vector<8xf32>, view<32x72xf32> %aw7 = vector.mulf %acc7_l, %clv : vector<8xf32> %sc7 = index.constant 56 : index vector.store %aw7, %st[%lane, %sc7] : vector<8xf32>, view<32x72xf32> %aw8 = vector.mulf %acc8_l, %clv : vector<8xf32> %sc8 = index.constant 64 : index vector.store %aw8, %st[%lane, %sc8] : vector<8xf32>, view<32x72xf32> kernel.barrier scope(workgroup) ordering(acq_rel) %ov = buffer.view %o[%zero] : buffer -> view<[%tokens]x1152xf32> %ocol0 = index.mul %h, %cD : index scf.for %col = [%lane to %cD step %c32] { %bcol = index.assume %col [lt(%col, %cD)] : index %tot = scf.for %r = [%c0 to %c32 step %c1](%a = %zf : f32) -> (f32) { %br = index.assume %r [range(%r, 0, 31)] : index %x = view.load %st[%br, %bcol] : view<32x72xf32> -> f32 %na = scalar.addf %a, %x : f32 scf.yield %na : f32 } %res = scalar.mulf %tot, %invl : f32 %oc0 = index.add %ocol0, %bcol : index %oc = index.assume %oc0 [lt(%oc0, 1152)] : index view.store %res, %ov[%t, %oc] : f32, view<[%tokens]x1152xf32> } kernel.return } kernel.def export("v_gemm_f16_store_1152x1152") @v_gemm_f16_store_1152x1152(%rows: index, %n: index) { %c1 = index.constant 1 : index %c64 = index.constant 64 : index %c256 = index.constant 256 : index %row_tiles = index.div %rows, %c64 : index %n_const = index.constant 1152 : index %out_tiles = index.div %n_const, %c64 : index kernel.launch.config workgroups(%row_tiles, %out_tiles, %c1) workgroup_size(%c256, %c1, %c1) : index } launch(%rows_arg: index, %k_arg: index, %n_arg: index, %x: buffer, %scales: buffer, %w: buffer, %y: buffer) { %rows = index.assume %rows_arg [range(%rows_arg, 64, 65536), multiple_of(%rows_arg, 64)] : index %k = index.constant 1152 : index %n = index.constant 1152 : index %origin = index.constant 0 : offset %two = index.constant 2 : offset %sixteen = index.constant 16 : offset %block_bytes = index.constant 64 : offset %c0 = index.constant 0 : index %c1 = index.constant 1 : index %c2 = index.constant 2 : index %c4 = index.constant 4 : index %c8 = index.constant 8 : index %c16 = index.constant 16 : index %c32 = index.constant 32 : index %c64 = index.constant 64 : index %c256 = index.constant 256 : index %nibble_mask = vector.constant 252645135 : vector<4xi32> %stage_bytes = index.constant 9216 : offset %row_tile_count = index.div %rows, %c64 : index %out_tile_count = index.div %n, %c64 : index %row_group_id = kernel.workgroup.id : index %out_group_id = kernel.workgroup.id : index %row_group = index.assume %row_group_id [lt(%row_group_id, %row_tile_count)] : index %out_group = index.assume %out_group_id [lt(%out_group_id, %out_tile_count)] : index %workitem_id = kernel.workitem.id : index %workitem = index.assume %workitem_id [range(%workitem_id, 0, 255)] : index %subgroup = kernel.subgroup.id : index %lane = kernel.subgroup.lane.id : index %lane_row = index.rem %lane, %c16 : index %bounded_subgroup = index.assume %subgroup [range(%subgroup, 0, 7)] : index %wave_row_group = index.div %bounded_subgroup, %c2 : index %wave_out_group = index.rem %bounded_subgroup, %c2 : index %wave_row = index.mul %wave_row_group, %c16 : index %wave_out = index.mul %wave_out_group, %c32 : index %w_row0 = index.add %wave_out, %lane_row : index %w_row1 = index.add %w_row0, %c16 : index %row_origin = index.mul %row_group, %c64 : index %out_origin = index.mul %out_group, %c64 : index %result_row = index.add %row_origin, %wave_row : index %result_col0 = index.add %out_origin, %wave_out : index %result_col1 = index.add %result_col0, %c16 : index %blocks = index.div %k, %c32 : index %row_bytes = index.scale %blocks, %block_bytes : index, offset -> offset %x_view = buffer.view %x[%origin] : buffer -> view<[%rows]x[%k]xf16> // 64x64 output tile with the full row stride n. %y_layout = encoding.layout.strided [%n, %c1] : encoding %tile_first = index.madd %row_origin, %n, %out_origin : index %four = index.constant 4 : offset %tile_offset = index.scale %tile_first, %four : index, offset -> offset %y_view = buffer.view %y[%tile_offset] : buffer -> view<64x64xf32, %y_layout> %bounded_wave_row = index.assume %wave_row [range(%wave_row, 0, 48)] : index %bounded_wave_out = index.assume %wave_out [range(%wave_out, 0, 32)] : index %wave_out1 = index.add %bounded_wave_out, %c16 : index %x_stage = buffer.alloca align(16) %stage_bytes : buffer %x_stage_view = buffer.view %x_stage[%origin] : buffer -> view<64x72xf16> %w_stage = buffer.alloca align(16) %stage_bytes : buffer %w_stage_view = buffer.view %w_stage[%origin] : buffer -> view<64x72xf16> // Thread t stages 16 values: row t/4, columns (t%4)*16. %stage_row = index.div %workitem, %c4 : index %stage_quarter = index.rem %workitem, %c4 : index %stage_col = index.mul %stage_quarter, %c16 : index %x_row = index.add %row_origin, %stage_row : index %bounded_x_row = index.assume %x_row [lt(%x_row, %rows)] : index %w_out_row = index.add %out_origin, %stage_row : index %bounded_w_out_row = index.assume %w_out_row [lt(%w_out_row, %n)] : index %w_row_origin = index.scale %bounded_w_out_row, %row_bytes : index, offset -> offset %zero8 = vector.constant 0.0 : vector<8xf32> %init = vector.fragment %zero8 shape [%c16, %c16] : vector<8xf32> %k_steps = index.div %k, %c64 : index %s_view = buffer.view %scales[%origin] : buffer -> view<[%rows]x2xf32> %x_scale = view.load %s_view[%bounded_x_row, %c0] : view<[%rows]x2xf32> -> f32 %x_scales = vector.splat %x_scale : vector<16xf32> %a0_raw, %a1_raw = scf.for %step = [%c0 to %k_steps step %c1](%a0_acc = %init : vector<8xf32>, %a1_acc = %init : vector<8xf32>) -> (vector<8xf32>, vector<8xf32>) { %k_origin = index.mul %step, %c64 : index %x_col = index.add %k_origin, %stage_col : index %x_half = vector.load %x_view[%bounded_x_row, %x_col] : view<[%rows]x[%k]xf16> -> vector<16xf16> vector.store %x_half, %x_stage_view[%stage_row, %stage_col] : vector<16xf16>, view<64x72xf16> %w_col = index.add %k_origin, %stage_col : index %deq0_two_b = index.constant 2 : offset %deq0_col_bytes = index.scale %w_col, %deq0_two_b : index, offset -> offset %deq0_origin = index.add %w_row_origin, %deq0_col_bytes : offset %deq0_view = buffer.view %w[%deq0_origin] : buffer -> view<16xf16> %deq0 = vector.load %deq0_view[0] : view<16xf16> -> vector<16xf16> vector.store %deq0, %w_stage_view[%stage_row, %stage_col] : vector<16xf16>, view<64x72xf16> kernel.barrier scope(workgroup) ordering(acq_rel) %fc0 = index.constant 0 : index %lhs0 = vector.fragment.load %x_stage_view[%wave_row, %fc0] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16> %rv000 = vector.load %w_stage_view[%w_row0, %fc0] : view<64x72xf16> -> vector<16xf16> %rhs000 = vector.fragment %rv000 shape [%c16, %c16] : vector<16xf16> %na0_0 = vector.mma %lhs0, %rhs000, %a0_acc : vector<16xf16>, vector<16xf16>, vector<8xf32> %rv010 = vector.load %w_stage_view[%w_row1, %fc0] : view<64x72xf16> -> vector<16xf16> %rhs010 = vector.fragment %rv010 shape [%c16, %c16] : vector<16xf16> %na1_0 = vector.mma %lhs0, %rhs010, %a1_acc : vector<16xf16>, vector<16xf16>, vector<8xf32> %fc1 = index.constant 16 : index %lhs1 = vector.fragment.load %x_stage_view[%wave_row, %fc1] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16> %rv001 = vector.load %w_stage_view[%w_row0, %fc1] : view<64x72xf16> -> vector<16xf16> %rhs001 = vector.fragment %rv001 shape [%c16, %c16] : vector<16xf16> %na0_1 = vector.mma %lhs1, %rhs001, %na0_0 : vector<16xf16>, vector<16xf16>, vector<8xf32> %rv011 = vector.load %w_stage_view[%w_row1, %fc1] : view<64x72xf16> -> vector<16xf16> %rhs011 = vector.fragment %rv011 shape [%c16, %c16] : vector<16xf16> %na1_1 = vector.mma %lhs1, %rhs011, %na1_0 : vector<16xf16>, vector<16xf16>, vector<8xf32> %fc2 = index.constant 32 : index %lhs2 = vector.fragment.load %x_stage_view[%wave_row, %fc2] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16> %rv002 = vector.load %w_stage_view[%w_row0, %fc2] : view<64x72xf16> -> vector<16xf16> %rhs002 = vector.fragment %rv002 shape [%c16, %c16] : vector<16xf16> %na0_2 = vector.mma %lhs2, %rhs002, %na0_1 : vector<16xf16>, vector<16xf16>, vector<8xf32> %rv012 = vector.load %w_stage_view[%w_row1, %fc2] : view<64x72xf16> -> vector<16xf16> %rhs012 = vector.fragment %rv012 shape [%c16, %c16] : vector<16xf16> %na1_2 = vector.mma %lhs2, %rhs012, %na1_1 : vector<16xf16>, vector<16xf16>, vector<8xf32> %fc3 = index.constant 48 : index %lhs3 = vector.fragment.load %x_stage_view[%wave_row, %fc3] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16> %rv003 = vector.load %w_stage_view[%w_row0, %fc3] : view<64x72xf16> -> vector<16xf16> %rhs003 = vector.fragment %rv003 shape [%c16, %c16] : vector<16xf16> %na0_3 = vector.mma %lhs3, %rhs003, %na0_2 : vector<16xf16>, vector<16xf16>, vector<8xf32> %rv013 = vector.load %w_stage_view[%w_row1, %fc3] : view<64x72xf16> -> vector<16xf16> %rhs013 = vector.fragment %rv013 shape [%c16, %c16] : vector<16xf16> %na1_3 = vector.mma %lhs3, %rhs013, %na1_2 : vector<16xf16>, vector<16xf16>, vector<8xf32> kernel.barrier scope(workgroup) ordering(acq_rel) scf.yield %na0_3, %na1_3 : vector<8xf32>, vector<8xf32> } %inv_bytes = index.constant 4096 : offset %inv_tile = buffer.alloca align(16) %inv_bytes : buffer %inv_view = buffer.view %inv_tile[%origin] : buffer -> view<64x16xf32> %writer = index.cmp ult, %workitem, %c64 : index scf.if %writer { %inv_row_value = index.add %row_origin, %workitem : index %inv_row = index.assume %inv_row_value [lt(%inv_row_value, %rows)] : index %inverse = view.load %s_view[%inv_row, %c1] : view<[%rows]x2xf32> -> f32 %inverse16 = vector.splat %inverse : vector<16xf32> %bw = index.assume %workitem [range(%workitem, 0, 63)] : index vector.store %inverse16, %inv_view[%bw, %c0] : vector<16xf32>, view<64x16xf32> } kernel.barrier scope(workgroup) ordering(acq_rel) %inv_frag = vector.fragment.load %inv_view[%wave_row, %c0] shape [%c16, %c16] : view<64x16xf32> -> vector<8xf32> %a0 = vector.mulf %a0_raw, %inv_frag : vector<8xf32> %a1 = vector.mulf %a1_raw, %inv_frag : vector<8xf32> vector.fragment.store %a0, %y_view[%bounded_wave_row, %bounded_wave_out] shape [%c16, %c16] : vector<8xf32>, view<64x64xf32, %y_layout> vector.fragment.store %a1, %y_view[%bounded_wave_row, %wave_out1] shape [%c16, %c16] : vector<8xf32>, view<64x64xf32, %y_layout> kernel.return } kernel.def export("v_bias_none_1152") @v_bias_none_1152(%rows: index) { %c1 = index.constant 1 : index %c256 = index.constant 256 : index %cols = index.constant 1152 : index %cg = index.constant 5 : index kernel.launch.config workgroups(%rows, %cg, %c1) workgroup_size(%c256, %c1, %c1) : index } launch(%rows_arg: index, %x: buffer, %bias: buffer, %y: buffer) { %rows = index.assume %rows_arg [range(%rows_arg, 1, 65536)] : index %c256 = index.constant 256 : index %cols = index.constant 1152 : index %row_id = kernel.workgroup.id : index %row = index.assume %row_id [lt(%row_id, %rows)] : index %cg = kernel.workgroup.id : index %lid = kernel.workitem.id : index %c = index.madd %cg, %c256, %lid : index %live = index.cmp ult, %c, %cols : index scf.if %live { %bc = index.assume %c [lt(%c, %cols)] : index %zero = index.constant 0 : offset %zf = scalar.constant 0.0 : f32 %xv = buffer.view %x[%zero] : buffer -> view<[%rows]x1152xf32> %bv = buffer.view %bias[%zero] : buffer -> view<1152xf32> %x0 = view.load %xv[%row, %bc] : view<[%rows]x1152xf32> -> f32 %b0 = view.load %bv[%bc] : view<1152xf32> -> f32 %a = scalar.addf %x0, %b0 : f32 %r = scalar.addf %a, %zf : f32 %yv = buffer.view %y[%zero] : buffer -> view<[%rows]x1152xf32> view.store %r, %yv[%row, %bc] : f32, view<[%rows]x1152xf32> } kernel.return } kernel.def export("v_ew_add") @v_ew_add(%n: index) { %c1 = index.constant 1 : index %c256 = index.constant 256 : index %c255 = index.constant 255 : index %n_up = index.add %n, %c255 : index %groups = index.div %n_up, %c256 : index kernel.launch.config workgroups(%groups, %c1, %c1) workgroup_size(%c256, %c1, %c1) : index } launch(%n_arg: index, %x: buffer, %b: buffer, %y: buffer) { %n = index.assume %n_arg [range(%n_arg, 1, 268435456)] : index %c256 = index.constant 256 : index %gid = kernel.workgroup.id : index %lid = kernel.workitem.id : index %i = index.madd %gid, %c256, %lid : index %live = index.cmp ult, %i, %n : index scf.if %live { %bi = index.assume %i [lt(%i, %n)] : index %zero = index.constant 0 : offset %xv = buffer.view %x[%zero] : buffer -> view<[%n]xf32> %a = view.load %xv[%bi] : view<[%n]xf32> -> f32 %bview = buffer.view %b[%zero] : buffer -> view<[%n]xf32> %bv = view.load %bview[%bi] : view<[%n]xf32> -> f32 %r = scalar.addf %a, %bv : f32 %yv = buffer.view %y[%zero] : buffer -> view<[%n]xf32> view.store %r, %yv[%bi] : f32, view<[%n]xf32> } kernel.return } kernel.def export("v_gemm_any_f16_1152x4304") @v_gemm_any_f16_1152x4304(%rows: index, %n: index) { %c1 = index.constant 1 : index %c15 = index.constant 15 : index %c16 = index.constant 16 : index %c63 = index.constant 63 : index %c64 = index.constant 64 : index %rp = index.add %rows, %c15 : index %row_tiles = index.div %rp, %c16 : index %ncst = index.constant 4304 : index %np = index.add %ncst, %c63 : index %col_tiles = index.div %np, %c64 : index kernel.launch.config workgroups(%row_tiles, %col_tiles, %c1) workgroup_size(%c64, %c1, %c1) : index } launch(%rows_arg: index, %k_arg: index, %n_arg: index, %x: buffer, %scales: buffer, %w: buffer, %y: buffer) { %rows = index.assume %rows_arg [range(%rows_arg, 1, 65536)] : index %zero = index.constant 0 : offset %c0 = index.constant 0 : index %c1 = index.constant 1 : index %c8 = index.constant 8 : index %c16 = index.constant 16 : index %c64 = index.constant 64 : index %ck = index.constant 1152 : index %cn = index.constant 4304 : index %zf = scalar.constant 0.0 : f32 %rt_id = kernel.workgroup.id : index %ct_id = kernel.workgroup.id : index %lid = kernel.workitem.id : index %lane = index.assume %lid [range(%lid, 0, 63)] : index %col0 = index.mul %ct_id, %c64 : index %col = index.add %col0, %lane : index %col_live = index.cmp ult, %col, %cn : index %nlast = index.constant 4303 : index %colc = index.min %col, %nlast : index %wcol = index.assume %colc [range(%colc, 0, 4303)] : index %row0 = index.mul %rt_id, %c16 : index %x_view = buffer.view %x[%zero] : buffer -> view<[%rows]x1152xf16> %w_view = buffer.view %w[%zero] : buffer -> view<4304x1152xf16> %s_view = buffer.view %scales[%zero] : buffer -> view<[%rows]x2xf32> %rlast = index.sub %rows, %c1 : index %r0, %r1, %r2, %r3, %r4, %r5, %r6, %r7, %r8, %r9, %r10, %r11, %r12, %r13, %r14, %r15 = scf.for %kk = [%c0 to %ck step %c8](%a0 = %zf : f32, %a1 = %zf : f32, %a2 = %zf : f32, %a3 = %zf : f32, %a4 = %zf : f32, %a5 = %zf : f32, %a6 = %zf : f32, %a7 = %zf : f32, %a8 = %zf : f32, %a9 = %zf : f32, %a10 = %zf : f32, %a11 = %zf : f32, %a12 = %zf : f32, %a13 = %zf : f32, %a14 = %zf : f32, %a15 = %zf : f32) -> (f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32) { %bk = index.assume %kk [range(%kk, 0, 1144), multiple_of(%kk, 8)] : index %wh = vector.load %w_view[%wcol, %bk] : view<4304x1152xf16> -> vector<8xf16> %wv = vector.extf %wh : vector<8xf16> to vector<8xf32> %ri0 = index.constant 0 : index %rr0 = index.add %row0, %ri0 : index %rc0 = index.min %rr0, %rlast : index %rb0 = index.assume %rc0 [lt(%rc0, %rows)] : index %xh0 = vector.load %x_view[%rb0, %bk] : view<[%rows]x1152xf16> -> vector<8xf16> %xv0 = vector.extf %xh0 : vector<8xf16> to vector<8xf32> %d0 = vector.dotf %wv, %xv0, %zf : vector<8xf32>, vector<8xf32>, f32 %n0 = scalar.addf %a0, %d0 : f32 %ri1 = index.constant 1 : index %rr1 = index.add %row0, %ri1 : index %rc1 = index.min %rr1, %rlast : index %rb1 = index.assume %rc1 [lt(%rc1, %rows)] : index %xh1 = vector.load %x_view[%rb1, %bk] : view<[%rows]x1152xf16> -> vector<8xf16> %xv1 = vector.extf %xh1 : vector<8xf16> to vector<8xf32> %d1 = vector.dotf %wv, %xv1, %zf : vector<8xf32>, vector<8xf32>, f32 %n1 = scalar.addf %a1, %d1 : f32 %ri2 = index.constant 2 : index %rr2 = index.add %row0, %ri2 : index %rc2 = index.min %rr2, %rlast : index %rb2 = index.assume %rc2 [lt(%rc2, %rows)] : index %xh2 = vector.load %x_view[%rb2, %bk] : view<[%rows]x1152xf16> -> vector<8xf16> %xv2 = vector.extf %xh2 : vector<8xf16> to vector<8xf32> %d2 = vector.dotf %wv, %xv2, %zf : vector<8xf32>, vector<8xf32>, f32 %n2 = scalar.addf %a2, %d2 : f32 %ri3 = index.constant 3 : index %rr3 = index.add %row0, %ri3 : index %rc3 = index.min %rr3, %rlast : index %rb3 = index.assume %rc3 [lt(%rc3, %rows)] : index %xh3 = vector.load %x_view[%rb3, %bk] : view<[%rows]x1152xf16> -> vector<8xf16> %xv3 = vector.extf %xh3 : vector<8xf16> to vector<8xf32> %d3 = vector.dotf %wv, %xv3, %zf : vector<8xf32>, vector<8xf32>, f32 %n3 = scalar.addf %a3, %d3 : f32 %ri4 = index.constant 4 : index %rr4 = index.add %row0, %ri4 : index %rc4 = index.min %rr4, %rlast : index %rb4 = index.assume %rc4 [lt(%rc4, %rows)] : index %xh4 = vector.load %x_view[%rb4, %bk] : view<[%rows]x1152xf16> -> vector<8xf16> %xv4 = vector.extf %xh4 : vector<8xf16> to vector<8xf32> %d4 = vector.dotf %wv, %xv4, %zf : vector<8xf32>, vector<8xf32>, f32 %n4 = scalar.addf %a4, %d4 : f32 %ri5 = index.constant 5 : index %rr5 = index.add %row0, %ri5 : index %rc5 = index.min %rr5, %rlast : index %rb5 = index.assume %rc5 [lt(%rc5, %rows)] : index %xh5 = vector.load %x_view[%rb5, %bk] : view<[%rows]x1152xf16> -> vector<8xf16> %xv5 = vector.extf %xh5 : vector<8xf16> to vector<8xf32> %d5 = vector.dotf %wv, %xv5, %zf : vector<8xf32>, vector<8xf32>, f32 %n5 = scalar.addf %a5, %d5 : f32 %ri6 = index.constant 6 : index %rr6 = index.add %row0, %ri6 : index %rc6 = index.min %rr6, %rlast : index %rb6 = index.assume %rc6 [lt(%rc6, %rows)] : index %xh6 = vector.load %x_view[%rb6, %bk] : view<[%rows]x1152xf16> -> vector<8xf16> %xv6 = vector.extf %xh6 : vector<8xf16> to vector<8xf32> %d6 = vector.dotf %wv, %xv6, %zf : vector<8xf32>, vector<8xf32>, f32 %n6 = scalar.addf %a6, %d6 : f32 %ri7 = index.constant 7 : index %rr7 = index.add %row0, %ri7 : index %rc7 = index.min %rr7, %rlast : index %rb7 = index.assume %rc7 [lt(%rc7, %rows)] : index %xh7 = vector.load %x_view[%rb7, %bk] : view<[%rows]x1152xf16> -> vector<8xf16> %xv7 = vector.extf %xh7 : vector<8xf16> to vector<8xf32> %d7 = vector.dotf %wv, %xv7, %zf : vector<8xf32>, vector<8xf32>, f32 %n7 = scalar.addf %a7, %d7 : f32 %ri8 = index.constant 8 : index %rr8 = index.add %row0, %ri8 : index %rc8 = index.min %rr8, %rlast : index %rb8 = index.assume %rc8 [lt(%rc8, %rows)] : index %xh8 = vector.load %x_view[%rb8, %bk] : view<[%rows]x1152xf16> -> vector<8xf16> %xv8 = vector.extf %xh8 : vector<8xf16> to vector<8xf32> %d8 = vector.dotf %wv, %xv8, %zf : vector<8xf32>, vector<8xf32>, f32 %n8 = scalar.addf %a8, %d8 : f32 %ri9 = index.constant 9 : index %rr9 = index.add %row0, %ri9 : index %rc9 = index.min %rr9, %rlast : index %rb9 = index.assume %rc9 [lt(%rc9, %rows)] : index %xh9 = vector.load %x_view[%rb9, %bk] : view<[%rows]x1152xf16> -> vector<8xf16> %xv9 = vector.extf %xh9 : vector<8xf16> to vector<8xf32> %d9 = vector.dotf %wv, %xv9, %zf : vector<8xf32>, vector<8xf32>, f32 %n9 = scalar.addf %a9, %d9 : f32 %ri10 = index.constant 10 : index %rr10 = index.add %row0, %ri10 : index %rc10 = index.min %rr10, %rlast : index %rb10 = index.assume %rc10 [lt(%rc10, %rows)] : index %xh10 = vector.load %x_view[%rb10, %bk] : view<[%rows]x1152xf16> -> vector<8xf16> %xv10 = vector.extf %xh10 : vector<8xf16> to vector<8xf32> %d10 = vector.dotf %wv, %xv10, %zf : vector<8xf32>, vector<8xf32>, f32 %n10 = scalar.addf %a10, %d10 : f32 %ri11 = index.constant 11 : index %rr11 = index.add %row0, %ri11 : index %rc11 = index.min %rr11, %rlast : index %rb11 = index.assume %rc11 [lt(%rc11, %rows)] : index %xh11 = vector.load %x_view[%rb11, %bk] : view<[%rows]x1152xf16> -> vector<8xf16> %xv11 = vector.extf %xh11 : vector<8xf16> to vector<8xf32> %d11 = vector.dotf %wv, %xv11, %zf : vector<8xf32>, vector<8xf32>, f32 %n11 = scalar.addf %a11, %d11 : f32 %ri12 = index.constant 12 : index %rr12 = index.add %row0, %ri12 : index %rc12 = index.min %rr12, %rlast : index %rb12 = index.assume %rc12 [lt(%rc12, %rows)] : index %xh12 = vector.load %x_view[%rb12, %bk] : view<[%rows]x1152xf16> -> vector<8xf16> %xv12 = vector.extf %xh12 : vector<8xf16> to vector<8xf32> %d12 = vector.dotf %wv, %xv12, %zf : vector<8xf32>, vector<8xf32>, f32 %n12 = scalar.addf %a12, %d12 : f32 %ri13 = index.constant 13 : index %rr13 = index.add %row0, %ri13 : index %rc13 = index.min %rr13, %rlast : index %rb13 = index.assume %rc13 [lt(%rc13, %rows)] : index %xh13 = vector.load %x_view[%rb13, %bk] : view<[%rows]x1152xf16> -> vector<8xf16> %xv13 = vector.extf %xh13 : vector<8xf16> to vector<8xf32> %d13 = vector.dotf %wv, %xv13, %zf : vector<8xf32>, vector<8xf32>, f32 %n13 = scalar.addf %a13, %d13 : f32 %ri14 = index.constant 14 : index %rr14 = index.add %row0, %ri14 : index %rc14 = index.min %rr14, %rlast : index %rb14 = index.assume %rc14 [lt(%rc14, %rows)] : index %xh14 = vector.load %x_view[%rb14, %bk] : view<[%rows]x1152xf16> -> vector<8xf16> %xv14 = vector.extf %xh14 : vector<8xf16> to vector<8xf32> %d14 = vector.dotf %wv, %xv14, %zf : vector<8xf32>, vector<8xf32>, f32 %n14 = scalar.addf %a14, %d14 : f32 %ri15 = index.constant 15 : index %rr15 = index.add %row0, %ri15 : index %rc15 = index.min %rr15, %rlast : index %rb15 = index.assume %rc15 [lt(%rc15, %rows)] : index %xh15 = vector.load %x_view[%rb15, %bk] : view<[%rows]x1152xf16> -> vector<8xf16> %xv15 = vector.extf %xh15 : vector<8xf16> to vector<8xf32> %d15 = vector.dotf %wv, %xv15, %zf : vector<8xf32>, vector<8xf32>, f32 %n15 = scalar.addf %a15, %d15 : f32 scf.yield %n0, %n1, %n2, %n3, %n4, %n5, %n6, %n7, %n8, %n9, %n10, %n11, %n12, %n13, %n14, %n15 : f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32 } %y_view = buffer.view %y[%zero] : buffer -> view<[%rows]x4304xf32> %ro0 = index.constant 0 : index %orow0 = index.add %row0, %ro0 : index %olive0 = index.cmp ult, %orow0, %rows : index %ok0 = scalar.andi %olive0, %col_live : i1 scf.if %ok0 { %ob0 = index.assume %orow0 [lt(%orow0, %rows)] : index %inv0 = view.load %s_view[%ob0, %c1] : view<[%rows]x2xf32> -> f32 %val0 = scalar.mulf %r0, %inv0 : f32 view.store %val0, %y_view[%ob0, %wcol] : f32, view<[%rows]x4304xf32> } %ro1 = index.constant 1 : index %orow1 = index.add %row0, %ro1 : index %olive1 = index.cmp ult, %orow1, %rows : index %ok1 = scalar.andi %olive1, %col_live : i1 scf.if %ok1 { %ob1 = index.assume %orow1 [lt(%orow1, %rows)] : index %inv1 = view.load %s_view[%ob1, %c1] : view<[%rows]x2xf32> -> f32 %val1 = scalar.mulf %r1, %inv1 : f32 view.store %val1, %y_view[%ob1, %wcol] : f32, view<[%rows]x4304xf32> } %ro2 = index.constant 2 : index %orow2 = index.add %row0, %ro2 : index %olive2 = index.cmp ult, %orow2, %rows : index %ok2 = scalar.andi %olive2, %col_live : i1 scf.if %ok2 { %ob2 = index.assume %orow2 [lt(%orow2, %rows)] : index %inv2 = view.load %s_view[%ob2, %c1] : view<[%rows]x2xf32> -> f32 %val2 = scalar.mulf %r2, %inv2 : f32 view.store %val2, %y_view[%ob2, %wcol] : f32, view<[%rows]x4304xf32> } %ro3 = index.constant 3 : index %orow3 = index.add %row0, %ro3 : index %olive3 = index.cmp ult, %orow3, %rows : index %ok3 = scalar.andi %olive3, %col_live : i1 scf.if %ok3 { %ob3 = index.assume %orow3 [lt(%orow3, %rows)] : index %inv3 = view.load %s_view[%ob3, %c1] : view<[%rows]x2xf32> -> f32 %val3 = scalar.mulf %r3, %inv3 : f32 view.store %val3, %y_view[%ob3, %wcol] : f32, view<[%rows]x4304xf32> } %ro4 = index.constant 4 : index %orow4 = index.add %row0, %ro4 : index %olive4 = index.cmp ult, %orow4, %rows : index %ok4 = scalar.andi %olive4, %col_live : i1 scf.if %ok4 { %ob4 = index.assume %orow4 [lt(%orow4, %rows)] : index %inv4 = view.load %s_view[%ob4, %c1] : view<[%rows]x2xf32> -> f32 %val4 = scalar.mulf %r4, %inv4 : f32 view.store %val4, %y_view[%ob4, %wcol] : f32, view<[%rows]x4304xf32> } %ro5 = index.constant 5 : index %orow5 = index.add %row0, %ro5 : index %olive5 = index.cmp ult, %orow5, %rows : index %ok5 = scalar.andi %olive5, %col_live : i1 scf.if %ok5 { %ob5 = index.assume %orow5 [lt(%orow5, %rows)] : index %inv5 = view.load %s_view[%ob5, %c1] : view<[%rows]x2xf32> -> f32 %val5 = scalar.mulf %r5, %inv5 : f32 view.store %val5, %y_view[%ob5, %wcol] : f32, view<[%rows]x4304xf32> } %ro6 = index.constant 6 : index %orow6 = index.add %row0, %ro6 : index %olive6 = index.cmp ult, %orow6, %rows : index %ok6 = scalar.andi %olive6, %col_live : i1 scf.if %ok6 { %ob6 = index.assume %orow6 [lt(%orow6, %rows)] : index %inv6 = view.load %s_view[%ob6, %c1] : view<[%rows]x2xf32> -> f32 %val6 = scalar.mulf %r6, %inv6 : f32 view.store %val6, %y_view[%ob6, %wcol] : f32, view<[%rows]x4304xf32> } %ro7 = index.constant 7 : index %orow7 = index.add %row0, %ro7 : index %olive7 = index.cmp ult, %orow7, %rows : index %ok7 = scalar.andi %olive7, %col_live : i1 scf.if %ok7 { %ob7 = index.assume %orow7 [lt(%orow7, %rows)] : index %inv7 = view.load %s_view[%ob7, %c1] : view<[%rows]x2xf32> -> f32 %val7 = scalar.mulf %r7, %inv7 : f32 view.store %val7, %y_view[%ob7, %wcol] : f32, view<[%rows]x4304xf32> } %ro8 = index.constant 8 : index %orow8 = index.add %row0, %ro8 : index %olive8 = index.cmp ult, %orow8, %rows : index %ok8 = scalar.andi %olive8, %col_live : i1 scf.if %ok8 { %ob8 = index.assume %orow8 [lt(%orow8, %rows)] : index %inv8 = view.load %s_view[%ob8, %c1] : view<[%rows]x2xf32> -> f32 %val8 = scalar.mulf %r8, %inv8 : f32 view.store %val8, %y_view[%ob8, %wcol] : f32, view<[%rows]x4304xf32> } %ro9 = index.constant 9 : index %orow9 = index.add %row0, %ro9 : index %olive9 = index.cmp ult, %orow9, %rows : index %ok9 = scalar.andi %olive9, %col_live : i1 scf.if %ok9 { %ob9 = index.assume %orow9 [lt(%orow9, %rows)] : index %inv9 = view.load %s_view[%ob9, %c1] : view<[%rows]x2xf32> -> f32 %val9 = scalar.mulf %r9, %inv9 : f32 view.store %val9, %y_view[%ob9, %wcol] : f32, view<[%rows]x4304xf32> } %ro10 = index.constant 10 : index %orow10 = index.add %row0, %ro10 : index %olive10 = index.cmp ult, %orow10, %rows : index %ok10 = scalar.andi %olive10, %col_live : i1 scf.if %ok10 { %ob10 = index.assume %orow10 [lt(%orow10, %rows)] : index %inv10 = view.load %s_view[%ob10, %c1] : view<[%rows]x2xf32> -> f32 %val10 = scalar.mulf %r10, %inv10 : f32 view.store %val10, %y_view[%ob10, %wcol] : f32, view<[%rows]x4304xf32> } %ro11 = index.constant 11 : index %orow11 = index.add %row0, %ro11 : index %olive11 = index.cmp ult, %orow11, %rows : index %ok11 = scalar.andi %olive11, %col_live : i1 scf.if %ok11 { %ob11 = index.assume %orow11 [lt(%orow11, %rows)] : index %inv11 = view.load %s_view[%ob11, %c1] : view<[%rows]x2xf32> -> f32 %val11 = scalar.mulf %r11, %inv11 : f32 view.store %val11, %y_view[%ob11, %wcol] : f32, view<[%rows]x4304xf32> } %ro12 = index.constant 12 : index %orow12 = index.add %row0, %ro12 : index %olive12 = index.cmp ult, %orow12, %rows : index %ok12 = scalar.andi %olive12, %col_live : i1 scf.if %ok12 { %ob12 = index.assume %orow12 [lt(%orow12, %rows)] : index %inv12 = view.load %s_view[%ob12, %c1] : view<[%rows]x2xf32> -> f32 %val12 = scalar.mulf %r12, %inv12 : f32 view.store %val12, %y_view[%ob12, %wcol] : f32, view<[%rows]x4304xf32> } %ro13 = index.constant 13 : index %orow13 = index.add %row0, %ro13 : index %olive13 = index.cmp ult, %orow13, %rows : index %ok13 = scalar.andi %olive13, %col_live : i1 scf.if %ok13 { %ob13 = index.assume %orow13 [lt(%orow13, %rows)] : index %inv13 = view.load %s_view[%ob13, %c1] : view<[%rows]x2xf32> -> f32 %val13 = scalar.mulf %r13, %inv13 : f32 view.store %val13, %y_view[%ob13, %wcol] : f32, view<[%rows]x4304xf32> } %ro14 = index.constant 14 : index %orow14 = index.add %row0, %ro14 : index %olive14 = index.cmp ult, %orow14, %rows : index %ok14 = scalar.andi %olive14, %col_live : i1 scf.if %ok14 { %ob14 = index.assume %orow14 [lt(%orow14, %rows)] : index %inv14 = view.load %s_view[%ob14, %c1] : view<[%rows]x2xf32> -> f32 %val14 = scalar.mulf %r14, %inv14 : f32 view.store %val14, %y_view[%ob14, %wcol] : f32, view<[%rows]x4304xf32> } %ro15 = index.constant 15 : index %orow15 = index.add %row0, %ro15 : index %olive15 = index.cmp ult, %orow15, %rows : index %ok15 = scalar.andi %olive15, %col_live : i1 scf.if %ok15 { %ob15 = index.assume %orow15 [lt(%orow15, %rows)] : index %inv15 = view.load %s_view[%ob15, %c1] : view<[%rows]x2xf32> -> f32 %val15 = scalar.mulf %r15, %inv15 : f32 view.store %val15, %y_view[%ob15, %wcol] : f32, view<[%rows]x4304xf32> } kernel.return } kernel.def export("v_bias_gelu_tanh_4304") @v_bias_gelu_tanh_4304(%rows: index) { %c1 = index.constant 1 : index %c256 = index.constant 256 : index %cols = index.constant 4304 : index %cg = index.constant 17 : index kernel.launch.config workgroups(%rows, %cg, %c1) workgroup_size(%c256, %c1, %c1) : index } launch(%rows_arg: index, %x: buffer, %bias: buffer, %y: buffer) { %rows = index.assume %rows_arg [range(%rows_arg, 1, 65536)] : index %c256 = index.constant 256 : index %cols = index.constant 4304 : index %row_id = kernel.workgroup.id : index %row = index.assume %row_id [lt(%row_id, %rows)] : index %cg = kernel.workgroup.id : index %lid = kernel.workitem.id : index %c = index.madd %cg, %c256, %lid : index %live = index.cmp ult, %c, %cols : index scf.if %live { %bc = index.assume %c [lt(%c, %cols)] : index %zero = index.constant 0 : offset %zf = scalar.constant 0.0 : f32 %xv = buffer.view %x[%zero] : buffer -> view<[%rows]x4304xf32> %bv = buffer.view %bias[%zero] : buffer -> view<4304xf32> %x0 = view.load %xv[%row, %bc] : view<[%rows]x4304xf32> -> f32 %b0 = view.load %bv[%bc] : view<4304xf32> -> f32 %a = scalar.addf %x0, %b0 : f32 %x3 = scalar.mulf %a, %a : f32 %x3b = scalar.mulf %x3, %a : f32 %k1 = scalar.constant 0.044715 : f32 %inner0 = scalar.mulf %x3b, %k1 : f32 %inner1 = scalar.addf %inner0, %a : f32 %k2 = scalar.constant 1.5957691216057308 : f32 %z = scalar.mulf %inner1, %k2 : f32 %nz = scalar.negf %z : f32 %ez = scalar.expf %nz : f32 %one = scalar.constant 1.0 : f32 %den = scalar.addf %ez, %one : f32 %r = scalar.divf %a, %den : f32 %yv = buffer.view %y[%zero] : buffer -> view<[%rows]x4304xf32> view.store %r, %yv[%row, %bc] : f32, view<[%rows]x4304xf32> } kernel.return } kernel.def export("v_gemm_any_f16_4304x1152") @v_gemm_any_f16_4304x1152(%rows: index, %n: index) { %c1 = index.constant 1 : index %c15 = index.constant 15 : index %c16 = index.constant 16 : index %c63 = index.constant 63 : index %c64 = index.constant 64 : index %rp = index.add %rows, %c15 : index %row_tiles = index.div %rp, %c16 : index %ncst = index.constant 1152 : index %np = index.add %ncst, %c63 : index %col_tiles = index.div %np, %c64 : index kernel.launch.config workgroups(%row_tiles, %col_tiles, %c1) workgroup_size(%c64, %c1, %c1) : index } launch(%rows_arg: index, %k_arg: index, %n_arg: index, %x: buffer, %scales: buffer, %w: buffer, %y: buffer) { %rows = index.assume %rows_arg [range(%rows_arg, 1, 65536)] : index %zero = index.constant 0 : offset %c0 = index.constant 0 : index %c1 = index.constant 1 : index %c8 = index.constant 8 : index %c16 = index.constant 16 : index %c64 = index.constant 64 : index %ck = index.constant 4304 : index %cn = index.constant 1152 : index %zf = scalar.constant 0.0 : f32 %rt_id = kernel.workgroup.id : index %ct_id = kernel.workgroup.id : index %lid = kernel.workitem.id : index %lane = index.assume %lid [range(%lid, 0, 63)] : index %col0 = index.mul %ct_id, %c64 : index %col = index.add %col0, %lane : index %col_live = index.cmp ult, %col, %cn : index %nlast = index.constant 1151 : index %colc = index.min %col, %nlast : index %wcol = index.assume %colc [range(%colc, 0, 1151)] : index %row0 = index.mul %rt_id, %c16 : index %x_view = buffer.view %x[%zero] : buffer -> view<[%rows]x4304xf16> %w_view = buffer.view %w[%zero] : buffer -> view<1152x4304xf16> %s_view = buffer.view %scales[%zero] : buffer -> view<[%rows]x2xf32> %rlast = index.sub %rows, %c1 : index %r0, %r1, %r2, %r3, %r4, %r5, %r6, %r7, %r8, %r9, %r10, %r11, %r12, %r13, %r14, %r15 = scf.for %kk = [%c0 to %ck step %c8](%a0 = %zf : f32, %a1 = %zf : f32, %a2 = %zf : f32, %a3 = %zf : f32, %a4 = %zf : f32, %a5 = %zf : f32, %a6 = %zf : f32, %a7 = %zf : f32, %a8 = %zf : f32, %a9 = %zf : f32, %a10 = %zf : f32, %a11 = %zf : f32, %a12 = %zf : f32, %a13 = %zf : f32, %a14 = %zf : f32, %a15 = %zf : f32) -> (f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32) { %bk = index.assume %kk [range(%kk, 0, 4296), multiple_of(%kk, 8)] : index %wh = vector.load %w_view[%wcol, %bk] : view<1152x4304xf16> -> vector<8xf16> %wv = vector.extf %wh : vector<8xf16> to vector<8xf32> %ri0 = index.constant 0 : index %rr0 = index.add %row0, %ri0 : index %rc0 = index.min %rr0, %rlast : index %rb0 = index.assume %rc0 [lt(%rc0, %rows)] : index %xh0 = vector.load %x_view[%rb0, %bk] : view<[%rows]x4304xf16> -> vector<8xf16> %xv0 = vector.extf %xh0 : vector<8xf16> to vector<8xf32> %d0 = vector.dotf %wv, %xv0, %zf : vector<8xf32>, vector<8xf32>, f32 %n0 = scalar.addf %a0, %d0 : f32 %ri1 = index.constant 1 : index %rr1 = index.add %row0, %ri1 : index %rc1 = index.min %rr1, %rlast : index %rb1 = index.assume %rc1 [lt(%rc1, %rows)] : index %xh1 = vector.load %x_view[%rb1, %bk] : view<[%rows]x4304xf16> -> vector<8xf16> %xv1 = vector.extf %xh1 : vector<8xf16> to vector<8xf32> %d1 = vector.dotf %wv, %xv1, %zf : vector<8xf32>, vector<8xf32>, f32 %n1 = scalar.addf %a1, %d1 : f32 %ri2 = index.constant 2 : index %rr2 = index.add %row0, %ri2 : index %rc2 = index.min %rr2, %rlast : index %rb2 = index.assume %rc2 [lt(%rc2, %rows)] : index %xh2 = vector.load %x_view[%rb2, %bk] : view<[%rows]x4304xf16> -> vector<8xf16> %xv2 = vector.extf %xh2 : vector<8xf16> to vector<8xf32> %d2 = vector.dotf %wv, %xv2, %zf : vector<8xf32>, vector<8xf32>, f32 %n2 = scalar.addf %a2, %d2 : f32 %ri3 = index.constant 3 : index %rr3 = index.add %row0, %ri3 : index %rc3 = index.min %rr3, %rlast : index %rb3 = index.assume %rc3 [lt(%rc3, %rows)] : index %xh3 = vector.load %x_view[%rb3, %bk] : view<[%rows]x4304xf16> -> vector<8xf16> %xv3 = vector.extf %xh3 : vector<8xf16> to vector<8xf32> %d3 = vector.dotf %wv, %xv3, %zf : vector<8xf32>, vector<8xf32>, f32 %n3 = scalar.addf %a3, %d3 : f32 %ri4 = index.constant 4 : index %rr4 = index.add %row0, %ri4 : index %rc4 = index.min %rr4, %rlast : index %rb4 = index.assume %rc4 [lt(%rc4, %rows)] : index %xh4 = vector.load %x_view[%rb4, %bk] : view<[%rows]x4304xf16> -> vector<8xf16> %xv4 = vector.extf %xh4 : vector<8xf16> to vector<8xf32> %d4 = vector.dotf %wv, %xv4, %zf : vector<8xf32>, vector<8xf32>, f32 %n4 = scalar.addf %a4, %d4 : f32 %ri5 = index.constant 5 : index %rr5 = index.add %row0, %ri5 : index %rc5 = index.min %rr5, %rlast : index %rb5 = index.assume %rc5 [lt(%rc5, %rows)] : index %xh5 = vector.load %x_view[%rb5, %bk] : view<[%rows]x4304xf16> -> vector<8xf16> %xv5 = vector.extf %xh5 : vector<8xf16> to vector<8xf32> %d5 = vector.dotf %wv, %xv5, %zf : vector<8xf32>, vector<8xf32>, f32 %n5 = scalar.addf %a5, %d5 : f32 %ri6 = index.constant 6 : index %rr6 = index.add %row0, %ri6 : index %rc6 = index.min %rr6, %rlast : index %rb6 = index.assume %rc6 [lt(%rc6, %rows)] : index %xh6 = vector.load %x_view[%rb6, %bk] : view<[%rows]x4304xf16> -> vector<8xf16> %xv6 = vector.extf %xh6 : vector<8xf16> to vector<8xf32> %d6 = vector.dotf %wv, %xv6, %zf : vector<8xf32>, vector<8xf32>, f32 %n6 = scalar.addf %a6, %d6 : f32 %ri7 = index.constant 7 : index %rr7 = index.add %row0, %ri7 : index %rc7 = index.min %rr7, %rlast : index %rb7 = index.assume %rc7 [lt(%rc7, %rows)] : index %xh7 = vector.load %x_view[%rb7, %bk] : view<[%rows]x4304xf16> -> vector<8xf16> %xv7 = vector.extf %xh7 : vector<8xf16> to vector<8xf32> %d7 = vector.dotf %wv, %xv7, %zf : vector<8xf32>, vector<8xf32>, f32 %n7 = scalar.addf %a7, %d7 : f32 %ri8 = index.constant 8 : index %rr8 = index.add %row0, %ri8 : index %rc8 = index.min %rr8, %rlast : index %rb8 = index.assume %rc8 [lt(%rc8, %rows)] : index %xh8 = vector.load %x_view[%rb8, %bk] : view<[%rows]x4304xf16> -> vector<8xf16> %xv8 = vector.extf %xh8 : vector<8xf16> to vector<8xf32> %d8 = vector.dotf %wv, %xv8, %zf : vector<8xf32>, vector<8xf32>, f32 %n8 = scalar.addf %a8, %d8 : f32 %ri9 = index.constant 9 : index %rr9 = index.add %row0, %ri9 : index %rc9 = index.min %rr9, %rlast : index %rb9 = index.assume %rc9 [lt(%rc9, %rows)] : index %xh9 = vector.load %x_view[%rb9, %bk] : view<[%rows]x4304xf16> -> vector<8xf16> %xv9 = vector.extf %xh9 : vector<8xf16> to vector<8xf32> %d9 = vector.dotf %wv, %xv9, %zf : vector<8xf32>, vector<8xf32>, f32 %n9 = scalar.addf %a9, %d9 : f32 %ri10 = index.constant 10 : index %rr10 = index.add %row0, %ri10 : index %rc10 = index.min %rr10, %rlast : index %rb10 = index.assume %rc10 [lt(%rc10, %rows)] : index %xh10 = vector.load %x_view[%rb10, %bk] : view<[%rows]x4304xf16> -> vector<8xf16> %xv10 = vector.extf %xh10 : vector<8xf16> to vector<8xf32> %d10 = vector.dotf %wv, %xv10, %zf : vector<8xf32>, vector<8xf32>, f32 %n10 = scalar.addf %a10, %d10 : f32 %ri11 = index.constant 11 : index %rr11 = index.add %row0, %ri11 : index %rc11 = index.min %rr11, %rlast : index %rb11 = index.assume %rc11 [lt(%rc11, %rows)] : index %xh11 = vector.load %x_view[%rb11, %bk] : view<[%rows]x4304xf16> -> vector<8xf16> %xv11 = vector.extf %xh11 : vector<8xf16> to vector<8xf32> %d11 = vector.dotf %wv, %xv11, %zf : vector<8xf32>, vector<8xf32>, f32 %n11 = scalar.addf %a11, %d11 : f32 %ri12 = index.constant 12 : index %rr12 = index.add %row0, %ri12 : index %rc12 = index.min %rr12, %rlast : index %rb12 = index.assume %rc12 [lt(%rc12, %rows)] : index %xh12 = vector.load %x_view[%rb12, %bk] : view<[%rows]x4304xf16> -> vector<8xf16> %xv12 = vector.extf %xh12 : vector<8xf16> to vector<8xf32> %d12 = vector.dotf %wv, %xv12, %zf : vector<8xf32>, vector<8xf32>, f32 %n12 = scalar.addf %a12, %d12 : f32 %ri13 = index.constant 13 : index %rr13 = index.add %row0, %ri13 : index %rc13 = index.min %rr13, %rlast : index %rb13 = index.assume %rc13 [lt(%rc13, %rows)] : index %xh13 = vector.load %x_view[%rb13, %bk] : view<[%rows]x4304xf16> -> vector<8xf16> %xv13 = vector.extf %xh13 : vector<8xf16> to vector<8xf32> %d13 = vector.dotf %wv, %xv13, %zf : vector<8xf32>, vector<8xf32>, f32 %n13 = scalar.addf %a13, %d13 : f32 %ri14 = index.constant 14 : index %rr14 = index.add %row0, %ri14 : index %rc14 = index.min %rr14, %rlast : index %rb14 = index.assume %rc14 [lt(%rc14, %rows)] : index %xh14 = vector.load %x_view[%rb14, %bk] : view<[%rows]x4304xf16> -> vector<8xf16> %xv14 = vector.extf %xh14 : vector<8xf16> to vector<8xf32> %d14 = vector.dotf %wv, %xv14, %zf : vector<8xf32>, vector<8xf32>, f32 %n14 = scalar.addf %a14, %d14 : f32 %ri15 = index.constant 15 : index %rr15 = index.add %row0, %ri15 : index %rc15 = index.min %rr15, %rlast : index %rb15 = index.assume %rc15 [lt(%rc15, %rows)] : index %xh15 = vector.load %x_view[%rb15, %bk] : view<[%rows]x4304xf16> -> vector<8xf16> %xv15 = vector.extf %xh15 : vector<8xf16> to vector<8xf32> %d15 = vector.dotf %wv, %xv15, %zf : vector<8xf32>, vector<8xf32>, f32 %n15 = scalar.addf %a15, %d15 : f32 scf.yield %n0, %n1, %n2, %n3, %n4, %n5, %n6, %n7, %n8, %n9, %n10, %n11, %n12, %n13, %n14, %n15 : f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32 } %y_view = buffer.view %y[%zero] : buffer -> view<[%rows]x1152xf32> %ro0 = index.constant 0 : index %orow0 = index.add %row0, %ro0 : index %olive0 = index.cmp ult, %orow0, %rows : index %ok0 = scalar.andi %olive0, %col_live : i1 scf.if %ok0 { %ob0 = index.assume %orow0 [lt(%orow0, %rows)] : index %inv0 = view.load %s_view[%ob0, %c1] : view<[%rows]x2xf32> -> f32 %val0 = scalar.mulf %r0, %inv0 : f32 view.store %val0, %y_view[%ob0, %wcol] : f32, view<[%rows]x1152xf32> } %ro1 = index.constant 1 : index %orow1 = index.add %row0, %ro1 : index %olive1 = index.cmp ult, %orow1, %rows : index %ok1 = scalar.andi %olive1, %col_live : i1 scf.if %ok1 { %ob1 = index.assume %orow1 [lt(%orow1, %rows)] : index %inv1 = view.load %s_view[%ob1, %c1] : view<[%rows]x2xf32> -> f32 %val1 = scalar.mulf %r1, %inv1 : f32 view.store %val1, %y_view[%ob1, %wcol] : f32, view<[%rows]x1152xf32> } %ro2 = index.constant 2 : index %orow2 = index.add %row0, %ro2 : index %olive2 = index.cmp ult, %orow2, %rows : index %ok2 = scalar.andi %olive2, %col_live : i1 scf.if %ok2 { %ob2 = index.assume %orow2 [lt(%orow2, %rows)] : index %inv2 = view.load %s_view[%ob2, %c1] : view<[%rows]x2xf32> -> f32 %val2 = scalar.mulf %r2, %inv2 : f32 view.store %val2, %y_view[%ob2, %wcol] : f32, view<[%rows]x1152xf32> } %ro3 = index.constant 3 : index %orow3 = index.add %row0, %ro3 : index %olive3 = index.cmp ult, %orow3, %rows : index %ok3 = scalar.andi %olive3, %col_live : i1 scf.if %ok3 { %ob3 = index.assume %orow3 [lt(%orow3, %rows)] : index %inv3 = view.load %s_view[%ob3, %c1] : view<[%rows]x2xf32> -> f32 %val3 = scalar.mulf %r3, %inv3 : f32 view.store %val3, %y_view[%ob3, %wcol] : f32, view<[%rows]x1152xf32> } %ro4 = index.constant 4 : index %orow4 = index.add %row0, %ro4 : index %olive4 = index.cmp ult, %orow4, %rows : index %ok4 = scalar.andi %olive4, %col_live : i1 scf.if %ok4 { %ob4 = index.assume %orow4 [lt(%orow4, %rows)] : index %inv4 = view.load %s_view[%ob4, %c1] : view<[%rows]x2xf32> -> f32 %val4 = scalar.mulf %r4, %inv4 : f32 view.store %val4, %y_view[%ob4, %wcol] : f32, view<[%rows]x1152xf32> } %ro5 = index.constant 5 : index %orow5 = index.add %row0, %ro5 : index %olive5 = index.cmp ult, %orow5, %rows : index %ok5 = scalar.andi %olive5, %col_live : i1 scf.if %ok5 { %ob5 = index.assume %orow5 [lt(%orow5, %rows)] : index %inv5 = view.load %s_view[%ob5, %c1] : view<[%rows]x2xf32> -> f32 %val5 = scalar.mulf %r5, %inv5 : f32 view.store %val5, %y_view[%ob5, %wcol] : f32, view<[%rows]x1152xf32> } %ro6 = index.constant 6 : index %orow6 = index.add %row0, %ro6 : index %olive6 = index.cmp ult, %orow6, %rows : index %ok6 = scalar.andi %olive6, %col_live : i1 scf.if %ok6 { %ob6 = index.assume %orow6 [lt(%orow6, %rows)] : index %inv6 = view.load %s_view[%ob6, %c1] : view<[%rows]x2xf32> -> f32 %val6 = scalar.mulf %r6, %inv6 : f32 view.store %val6, %y_view[%ob6, %wcol] : f32, view<[%rows]x1152xf32> } %ro7 = index.constant 7 : index %orow7 = index.add %row0, %ro7 : index %olive7 = index.cmp ult, %orow7, %rows : index %ok7 = scalar.andi %olive7, %col_live : i1 scf.if %ok7 { %ob7 = index.assume %orow7 [lt(%orow7, %rows)] : index %inv7 = view.load %s_view[%ob7, %c1] : view<[%rows]x2xf32> -> f32 %val7 = scalar.mulf %r7, %inv7 : f32 view.store %val7, %y_view[%ob7, %wcol] : f32, view<[%rows]x1152xf32> } %ro8 = index.constant 8 : index %orow8 = index.add %row0, %ro8 : index %olive8 = index.cmp ult, %orow8, %rows : index %ok8 = scalar.andi %olive8, %col_live : i1 scf.if %ok8 { %ob8 = index.assume %orow8 [lt(%orow8, %rows)] : index %inv8 = view.load %s_view[%ob8, %c1] : view<[%rows]x2xf32> -> f32 %val8 = scalar.mulf %r8, %inv8 : f32 view.store %val8, %y_view[%ob8, %wcol] : f32, view<[%rows]x1152xf32> } %ro9 = index.constant 9 : index %orow9 = index.add %row0, %ro9 : index %olive9 = index.cmp ult, %orow9, %rows : index %ok9 = scalar.andi %olive9, %col_live : i1 scf.if %ok9 { %ob9 = index.assume %orow9 [lt(%orow9, %rows)] : index %inv9 = view.load %s_view[%ob9, %c1] : view<[%rows]x2xf32> -> f32 %val9 = scalar.mulf %r9, %inv9 : f32 view.store %val9, %y_view[%ob9, %wcol] : f32, view<[%rows]x1152xf32> } %ro10 = index.constant 10 : index %orow10 = index.add %row0, %ro10 : index %olive10 = index.cmp ult, %orow10, %rows : index %ok10 = scalar.andi %olive10, %col_live : i1 scf.if %ok10 { %ob10 = index.assume %orow10 [lt(%orow10, %rows)] : index %inv10 = view.load %s_view[%ob10, %c1] : view<[%rows]x2xf32> -> f32 %val10 = scalar.mulf %r10, %inv10 : f32 view.store %val10, %y_view[%ob10, %wcol] : f32, view<[%rows]x1152xf32> } %ro11 = index.constant 11 : index %orow11 = index.add %row0, %ro11 : index %olive11 = index.cmp ult, %orow11, %rows : index %ok11 = scalar.andi %olive11, %col_live : i1 scf.if %ok11 { %ob11 = index.assume %orow11 [lt(%orow11, %rows)] : index %inv11 = view.load %s_view[%ob11, %c1] : view<[%rows]x2xf32> -> f32 %val11 = scalar.mulf %r11, %inv11 : f32 view.store %val11, %y_view[%ob11, %wcol] : f32, view<[%rows]x1152xf32> } %ro12 = index.constant 12 : index %orow12 = index.add %row0, %ro12 : index %olive12 = index.cmp ult, %orow12, %rows : index %ok12 = scalar.andi %olive12, %col_live : i1 scf.if %ok12 { %ob12 = index.assume %orow12 [lt(%orow12, %rows)] : index %inv12 = view.load %s_view[%ob12, %c1] : view<[%rows]x2xf32> -> f32 %val12 = scalar.mulf %r12, %inv12 : f32 view.store %val12, %y_view[%ob12, %wcol] : f32, view<[%rows]x1152xf32> } %ro13 = index.constant 13 : index %orow13 = index.add %row0, %ro13 : index %olive13 = index.cmp ult, %orow13, %rows : index %ok13 = scalar.andi %olive13, %col_live : i1 scf.if %ok13 { %ob13 = index.assume %orow13 [lt(%orow13, %rows)] : index %inv13 = view.load %s_view[%ob13, %c1] : view<[%rows]x2xf32> -> f32 %val13 = scalar.mulf %r13, %inv13 : f32 view.store %val13, %y_view[%ob13, %wcol] : f32, view<[%rows]x1152xf32> } %ro14 = index.constant 14 : index %orow14 = index.add %row0, %ro14 : index %olive14 = index.cmp ult, %orow14, %rows : index %ok14 = scalar.andi %olive14, %col_live : i1 scf.if %ok14 { %ob14 = index.assume %orow14 [lt(%orow14, %rows)] : index %inv14 = view.load %s_view[%ob14, %c1] : view<[%rows]x2xf32> -> f32 %val14 = scalar.mulf %r14, %inv14 : f32 view.store %val14, %y_view[%ob14, %wcol] : f32, view<[%rows]x1152xf32> } %ro15 = index.constant 15 : index %orow15 = index.add %row0, %ro15 : index %olive15 = index.cmp ult, %orow15, %rows : index %ok15 = scalar.andi %olive15, %col_live : i1 scf.if %ok15 { %ob15 = index.assume %orow15 [lt(%orow15, %rows)] : index %inv15 = view.load %s_view[%ob15, %c1] : view<[%rows]x2xf32> -> f32 %val15 = scalar.mulf %r15, %inv15 : f32 view.store %val15, %y_view[%ob15, %wcol] : f32, view<[%rows]x1152xf32> } kernel.return } kernel.def export("v_gemm_f16_store_4608x4608") @v_gemm_f16_store_4608x4608(%rows: index, %n: index) { %c1 = index.constant 1 : index %c64 = index.constant 64 : index %c256 = index.constant 256 : index %row_tiles = index.div %rows, %c64 : index %n_const = index.constant 4608 : index %out_tiles = index.div %n_const, %c64 : index kernel.launch.config workgroups(%row_tiles, %out_tiles, %c1) workgroup_size(%c256, %c1, %c1) : index } launch(%rows_arg: index, %k_arg: index, %n_arg: index, %x: buffer, %scales: buffer, %w: buffer, %y: buffer) { %rows = index.assume %rows_arg [range(%rows_arg, 64, 65536), multiple_of(%rows_arg, 64)] : index %k = index.constant 4608 : index %n = index.constant 4608 : index %origin = index.constant 0 : offset %two = index.constant 2 : offset %sixteen = index.constant 16 : offset %block_bytes = index.constant 64 : offset %c0 = index.constant 0 : index %c1 = index.constant 1 : index %c2 = index.constant 2 : index %c4 = index.constant 4 : index %c8 = index.constant 8 : index %c16 = index.constant 16 : index %c32 = index.constant 32 : index %c64 = index.constant 64 : index %c256 = index.constant 256 : index %nibble_mask = vector.constant 252645135 : vector<4xi32> %stage_bytes = index.constant 9216 : offset %row_tile_count = index.div %rows, %c64 : index %out_tile_count = index.div %n, %c64 : index %row_group_id = kernel.workgroup.id : index %out_group_id = kernel.workgroup.id : index %row_group = index.assume %row_group_id [lt(%row_group_id, %row_tile_count)] : index %out_group = index.assume %out_group_id [lt(%out_group_id, %out_tile_count)] : index %workitem_id = kernel.workitem.id : index %workitem = index.assume %workitem_id [range(%workitem_id, 0, 255)] : index %subgroup = kernel.subgroup.id : index %lane = kernel.subgroup.lane.id : index %lane_row = index.rem %lane, %c16 : index %bounded_subgroup = index.assume %subgroup [range(%subgroup, 0, 7)] : index %wave_row_group = index.div %bounded_subgroup, %c2 : index %wave_out_group = index.rem %bounded_subgroup, %c2 : index %wave_row = index.mul %wave_row_group, %c16 : index %wave_out = index.mul %wave_out_group, %c32 : index %w_row0 = index.add %wave_out, %lane_row : index %w_row1 = index.add %w_row0, %c16 : index %row_origin = index.mul %row_group, %c64 : index %out_origin = index.mul %out_group, %c64 : index %result_row = index.add %row_origin, %wave_row : index %result_col0 = index.add %out_origin, %wave_out : index %result_col1 = index.add %result_col0, %c16 : index %blocks = index.div %k, %c32 : index %row_bytes = index.scale %blocks, %block_bytes : index, offset -> offset %x_view = buffer.view %x[%origin] : buffer -> view<[%rows]x[%k]xf16> // 64x64 output tile with the full row stride n. %y_layout = encoding.layout.strided [%n, %c1] : encoding %tile_first = index.madd %row_origin, %n, %out_origin : index %four = index.constant 4 : offset %tile_offset = index.scale %tile_first, %four : index, offset -> offset %y_view = buffer.view %y[%tile_offset] : buffer -> view<64x64xf32, %y_layout> %bounded_wave_row = index.assume %wave_row [range(%wave_row, 0, 48)] : index %bounded_wave_out = index.assume %wave_out [range(%wave_out, 0, 32)] : index %wave_out1 = index.add %bounded_wave_out, %c16 : index %x_stage = buffer.alloca align(16) %stage_bytes : buffer %x_stage_view = buffer.view %x_stage[%origin] : buffer -> view<64x72xf16> %w_stage = buffer.alloca align(16) %stage_bytes : buffer %w_stage_view = buffer.view %w_stage[%origin] : buffer -> view<64x72xf16> // Thread t stages 16 values: row t/4, columns (t%4)*16. %stage_row = index.div %workitem, %c4 : index %stage_quarter = index.rem %workitem, %c4 : index %stage_col = index.mul %stage_quarter, %c16 : index %x_row = index.add %row_origin, %stage_row : index %bounded_x_row = index.assume %x_row [lt(%x_row, %rows)] : index %w_out_row = index.add %out_origin, %stage_row : index %bounded_w_out_row = index.assume %w_out_row [lt(%w_out_row, %n)] : index %w_row_origin = index.scale %bounded_w_out_row, %row_bytes : index, offset -> offset %zero8 = vector.constant 0.0 : vector<8xf32> %init = vector.fragment %zero8 shape [%c16, %c16] : vector<8xf32> %k_steps = index.div %k, %c64 : index %s_view = buffer.view %scales[%origin] : buffer -> view<[%rows]x2xf32> %x_scale = view.load %s_view[%bounded_x_row, %c0] : view<[%rows]x2xf32> -> f32 %x_scales = vector.splat %x_scale : vector<16xf32> %a0_raw, %a1_raw = scf.for %step = [%c0 to %k_steps step %c1](%a0_acc = %init : vector<8xf32>, %a1_acc = %init : vector<8xf32>) -> (vector<8xf32>, vector<8xf32>) { %k_origin = index.mul %step, %c64 : index %x_col = index.add %k_origin, %stage_col : index %x_half = vector.load %x_view[%bounded_x_row, %x_col] : view<[%rows]x[%k]xf16> -> vector<16xf16> vector.store %x_half, %x_stage_view[%stage_row, %stage_col] : vector<16xf16>, view<64x72xf16> %w_col = index.add %k_origin, %stage_col : index %deq0_two_b = index.constant 2 : offset %deq0_col_bytes = index.scale %w_col, %deq0_two_b : index, offset -> offset %deq0_origin = index.add %w_row_origin, %deq0_col_bytes : offset %deq0_view = buffer.view %w[%deq0_origin] : buffer -> view<16xf16> %deq0 = vector.load %deq0_view[0] : view<16xf16> -> vector<16xf16> vector.store %deq0, %w_stage_view[%stage_row, %stage_col] : vector<16xf16>, view<64x72xf16> kernel.barrier scope(workgroup) ordering(acq_rel) %fc0 = index.constant 0 : index %lhs0 = vector.fragment.load %x_stage_view[%wave_row, %fc0] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16> %rv000 = vector.load %w_stage_view[%w_row0, %fc0] : view<64x72xf16> -> vector<16xf16> %rhs000 = vector.fragment %rv000 shape [%c16, %c16] : vector<16xf16> %na0_0 = vector.mma %lhs0, %rhs000, %a0_acc : vector<16xf16>, vector<16xf16>, vector<8xf32> %rv010 = vector.load %w_stage_view[%w_row1, %fc0] : view<64x72xf16> -> vector<16xf16> %rhs010 = vector.fragment %rv010 shape [%c16, %c16] : vector<16xf16> %na1_0 = vector.mma %lhs0, %rhs010, %a1_acc : vector<16xf16>, vector<16xf16>, vector<8xf32> %fc1 = index.constant 16 : index %lhs1 = vector.fragment.load %x_stage_view[%wave_row, %fc1] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16> %rv001 = vector.load %w_stage_view[%w_row0, %fc1] : view<64x72xf16> -> vector<16xf16> %rhs001 = vector.fragment %rv001 shape [%c16, %c16] : vector<16xf16> %na0_1 = vector.mma %lhs1, %rhs001, %na0_0 : vector<16xf16>, vector<16xf16>, vector<8xf32> %rv011 = vector.load %w_stage_view[%w_row1, %fc1] : view<64x72xf16> -> vector<16xf16> %rhs011 = vector.fragment %rv011 shape [%c16, %c16] : vector<16xf16> %na1_1 = vector.mma %lhs1, %rhs011, %na1_0 : vector<16xf16>, vector<16xf16>, vector<8xf32> %fc2 = index.constant 32 : index %lhs2 = vector.fragment.load %x_stage_view[%wave_row, %fc2] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16> %rv002 = vector.load %w_stage_view[%w_row0, %fc2] : view<64x72xf16> -> vector<16xf16> %rhs002 = vector.fragment %rv002 shape [%c16, %c16] : vector<16xf16> %na0_2 = vector.mma %lhs2, %rhs002, %na0_1 : vector<16xf16>, vector<16xf16>, vector<8xf32> %rv012 = vector.load %w_stage_view[%w_row1, %fc2] : view<64x72xf16> -> vector<16xf16> %rhs012 = vector.fragment %rv012 shape [%c16, %c16] : vector<16xf16> %na1_2 = vector.mma %lhs2, %rhs012, %na1_1 : vector<16xf16>, vector<16xf16>, vector<8xf32> %fc3 = index.constant 48 : index %lhs3 = vector.fragment.load %x_stage_view[%wave_row, %fc3] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16> %rv003 = vector.load %w_stage_view[%w_row0, %fc3] : view<64x72xf16> -> vector<16xf16> %rhs003 = vector.fragment %rv003 shape [%c16, %c16] : vector<16xf16> %na0_3 = vector.mma %lhs3, %rhs003, %na0_2 : vector<16xf16>, vector<16xf16>, vector<8xf32> %rv013 = vector.load %w_stage_view[%w_row1, %fc3] : view<64x72xf16> -> vector<16xf16> %rhs013 = vector.fragment %rv013 shape [%c16, %c16] : vector<16xf16> %na1_3 = vector.mma %lhs3, %rhs013, %na1_2 : vector<16xf16>, vector<16xf16>, vector<8xf32> kernel.barrier scope(workgroup) ordering(acq_rel) scf.yield %na0_3, %na1_3 : vector<8xf32>, vector<8xf32> } %inv_bytes = index.constant 4096 : offset %inv_tile = buffer.alloca align(16) %inv_bytes : buffer %inv_view = buffer.view %inv_tile[%origin] : buffer -> view<64x16xf32> %writer = index.cmp ult, %workitem, %c64 : index scf.if %writer { %inv_row_value = index.add %row_origin, %workitem : index %inv_row = index.assume %inv_row_value [lt(%inv_row_value, %rows)] : index %inverse = view.load %s_view[%inv_row, %c1] : view<[%rows]x2xf32> -> f32 %inverse16 = vector.splat %inverse : vector<16xf32> %bw = index.assume %workitem [range(%workitem, 0, 63)] : index vector.store %inverse16, %inv_view[%bw, %c0] : vector<16xf32>, view<64x16xf32> } kernel.barrier scope(workgroup) ordering(acq_rel) %inv_frag = vector.fragment.load %inv_view[%wave_row, %c0] shape [%c16, %c16] : view<64x16xf32> -> vector<8xf32> %a0 = vector.mulf %a0_raw, %inv_frag : vector<8xf32> %a1 = vector.mulf %a1_raw, %inv_frag : vector<8xf32> vector.fragment.store %a0, %y_view[%bounded_wave_row, %bounded_wave_out] shape [%c16, %c16] : vector<8xf32>, view<64x64xf32, %y_layout> vector.fragment.store %a1, %y_view[%bounded_wave_row, %wave_out1] shape [%c16, %c16] : vector<8xf32>, view<64x64xf32, %y_layout> kernel.return } kernel.def export("v_bias_gelu_tanh_4608") @v_bias_gelu_tanh_4608(%rows: index) { %c1 = index.constant 1 : index %c256 = index.constant 256 : index %cols = index.constant 4608 : index %cg = index.constant 18 : index kernel.launch.config workgroups(%rows, %cg, %c1) workgroup_size(%c256, %c1, %c1) : index } launch(%rows_arg: index, %x: buffer, %bias: buffer, %y: buffer) { %rows = index.assume %rows_arg [range(%rows_arg, 1, 65536)] : index %c256 = index.constant 256 : index %cols = index.constant 4608 : index %row_id = kernel.workgroup.id : index %row = index.assume %row_id [lt(%row_id, %rows)] : index %cg = kernel.workgroup.id : index %lid = kernel.workitem.id : index %c = index.madd %cg, %c256, %lid : index %live = index.cmp ult, %c, %cols : index scf.if %live { %bc = index.assume %c [lt(%c, %cols)] : index %zero = index.constant 0 : offset %zf = scalar.constant 0.0 : f32 %xv = buffer.view %x[%zero] : buffer -> view<[%rows]x4608xf32> %bv = buffer.view %bias[%zero] : buffer -> view<4608xf32> %x0 = view.load %xv[%row, %bc] : view<[%rows]x4608xf32> -> f32 %b0 = view.load %bv[%bc] : view<4608xf32> -> f32 %a = scalar.addf %x0, %b0 : f32 %x3 = scalar.mulf %a, %a : f32 %x3b = scalar.mulf %x3, %a : f32 %k1 = scalar.constant 0.044715 : f32 %inner0 = scalar.mulf %x3b, %k1 : f32 %inner1 = scalar.addf %inner0, %a : f32 %k2 = scalar.constant 1.5957691216057308 : f32 %z = scalar.mulf %inner1, %k2 : f32 %nz = scalar.negf %z : f32 %ez = scalar.expf %nz : f32 %one = scalar.constant 1.0 : f32 %den = scalar.addf %ez, %one : f32 %r = scalar.divf %a, %den : f32 %yv = buffer.view %y[%zero] : buffer -> view<[%rows]x4608xf32> view.store %r, %yv[%row, %bc] : f32, view<[%rows]x4608xf32> } kernel.return } kernel.def export("v_gemm_f16_store_4608x2048") @v_gemm_f16_store_4608x2048(%rows: index, %n: index) { %c1 = index.constant 1 : index %c64 = index.constant 64 : index %c256 = index.constant 256 : index %row_tiles = index.div %rows, %c64 : index %n_const = index.constant 2048 : index %out_tiles = index.div %n_const, %c64 : index kernel.launch.config workgroups(%row_tiles, %out_tiles, %c1) workgroup_size(%c256, %c1, %c1) : index } launch(%rows_arg: index, %k_arg: index, %n_arg: index, %x: buffer, %scales: buffer, %w: buffer, %y: buffer) { %rows = index.assume %rows_arg [range(%rows_arg, 64, 65536), multiple_of(%rows_arg, 64)] : index %k = index.constant 4608 : index %n = index.constant 2048 : index %origin = index.constant 0 : offset %two = index.constant 2 : offset %sixteen = index.constant 16 : offset %block_bytes = index.constant 64 : offset %c0 = index.constant 0 : index %c1 = index.constant 1 : index %c2 = index.constant 2 : index %c4 = index.constant 4 : index %c8 = index.constant 8 : index %c16 = index.constant 16 : index %c32 = index.constant 32 : index %c64 = index.constant 64 : index %c256 = index.constant 256 : index %nibble_mask = vector.constant 252645135 : vector<4xi32> %stage_bytes = index.constant 9216 : offset %row_tile_count = index.div %rows, %c64 : index %out_tile_count = index.div %n, %c64 : index %row_group_id = kernel.workgroup.id : index %out_group_id = kernel.workgroup.id : index %row_group = index.assume %row_group_id [lt(%row_group_id, %row_tile_count)] : index %out_group = index.assume %out_group_id [lt(%out_group_id, %out_tile_count)] : index %workitem_id = kernel.workitem.id : index %workitem = index.assume %workitem_id [range(%workitem_id, 0, 255)] : index %subgroup = kernel.subgroup.id : index %lane = kernel.subgroup.lane.id : index %lane_row = index.rem %lane, %c16 : index %bounded_subgroup = index.assume %subgroup [range(%subgroup, 0, 7)] : index %wave_row_group = index.div %bounded_subgroup, %c2 : index %wave_out_group = index.rem %bounded_subgroup, %c2 : index %wave_row = index.mul %wave_row_group, %c16 : index %wave_out = index.mul %wave_out_group, %c32 : index %w_row0 = index.add %wave_out, %lane_row : index %w_row1 = index.add %w_row0, %c16 : index %row_origin = index.mul %row_group, %c64 : index %out_origin = index.mul %out_group, %c64 : index %result_row = index.add %row_origin, %wave_row : index %result_col0 = index.add %out_origin, %wave_out : index %result_col1 = index.add %result_col0, %c16 : index %blocks = index.div %k, %c32 : index %row_bytes = index.scale %blocks, %block_bytes : index, offset -> offset %x_view = buffer.view %x[%origin] : buffer -> view<[%rows]x[%k]xf16> // 64x64 output tile with the full row stride n. %y_layout = encoding.layout.strided [%n, %c1] : encoding %tile_first = index.madd %row_origin, %n, %out_origin : index %four = index.constant 4 : offset %tile_offset = index.scale %tile_first, %four : index, offset -> offset %y_view = buffer.view %y[%tile_offset] : buffer -> view<64x64xf32, %y_layout> %bounded_wave_row = index.assume %wave_row [range(%wave_row, 0, 48)] : index %bounded_wave_out = index.assume %wave_out [range(%wave_out, 0, 32)] : index %wave_out1 = index.add %bounded_wave_out, %c16 : index %x_stage = buffer.alloca align(16) %stage_bytes : buffer %x_stage_view = buffer.view %x_stage[%origin] : buffer -> view<64x72xf16> %w_stage = buffer.alloca align(16) %stage_bytes : buffer %w_stage_view = buffer.view %w_stage[%origin] : buffer -> view<64x72xf16> // Thread t stages 16 values: row t/4, columns (t%4)*16. %stage_row = index.div %workitem, %c4 : index %stage_quarter = index.rem %workitem, %c4 : index %stage_col = index.mul %stage_quarter, %c16 : index %x_row = index.add %row_origin, %stage_row : index %bounded_x_row = index.assume %x_row [lt(%x_row, %rows)] : index %w_out_row = index.add %out_origin, %stage_row : index %bounded_w_out_row = index.assume %w_out_row [lt(%w_out_row, %n)] : index %w_row_origin = index.scale %bounded_w_out_row, %row_bytes : index, offset -> offset %zero8 = vector.constant 0.0 : vector<8xf32> %init = vector.fragment %zero8 shape [%c16, %c16] : vector<8xf32> %k_steps = index.div %k, %c64 : index %s_view = buffer.view %scales[%origin] : buffer -> view<[%rows]x2xf32> %x_scale = view.load %s_view[%bounded_x_row, %c0] : view<[%rows]x2xf32> -> f32 %x_scales = vector.splat %x_scale : vector<16xf32> %a0_raw, %a1_raw = scf.for %step = [%c0 to %k_steps step %c1](%a0_acc = %init : vector<8xf32>, %a1_acc = %init : vector<8xf32>) -> (vector<8xf32>, vector<8xf32>) { %k_origin = index.mul %step, %c64 : index %x_col = index.add %k_origin, %stage_col : index %x_half = vector.load %x_view[%bounded_x_row, %x_col] : view<[%rows]x[%k]xf16> -> vector<16xf16> vector.store %x_half, %x_stage_view[%stage_row, %stage_col] : vector<16xf16>, view<64x72xf16> %w_col = index.add %k_origin, %stage_col : index %deq0_two_b = index.constant 2 : offset %deq0_col_bytes = index.scale %w_col, %deq0_two_b : index, offset -> offset %deq0_origin = index.add %w_row_origin, %deq0_col_bytes : offset %deq0_view = buffer.view %w[%deq0_origin] : buffer -> view<16xf16> %deq0 = vector.load %deq0_view[0] : view<16xf16> -> vector<16xf16> vector.store %deq0, %w_stage_view[%stage_row, %stage_col] : vector<16xf16>, view<64x72xf16> kernel.barrier scope(workgroup) ordering(acq_rel) %fc0 = index.constant 0 : index %lhs0 = vector.fragment.load %x_stage_view[%wave_row, %fc0] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16> %rv000 = vector.load %w_stage_view[%w_row0, %fc0] : view<64x72xf16> -> vector<16xf16> %rhs000 = vector.fragment %rv000 shape [%c16, %c16] : vector<16xf16> %na0_0 = vector.mma %lhs0, %rhs000, %a0_acc : vector<16xf16>, vector<16xf16>, vector<8xf32> %rv010 = vector.load %w_stage_view[%w_row1, %fc0] : view<64x72xf16> -> vector<16xf16> %rhs010 = vector.fragment %rv010 shape [%c16, %c16] : vector<16xf16> %na1_0 = vector.mma %lhs0, %rhs010, %a1_acc : vector<16xf16>, vector<16xf16>, vector<8xf32> %fc1 = index.constant 16 : index %lhs1 = vector.fragment.load %x_stage_view[%wave_row, %fc1] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16> %rv001 = vector.load %w_stage_view[%w_row0, %fc1] : view<64x72xf16> -> vector<16xf16> %rhs001 = vector.fragment %rv001 shape [%c16, %c16] : vector<16xf16> %na0_1 = vector.mma %lhs1, %rhs001, %na0_0 : vector<16xf16>, vector<16xf16>, vector<8xf32> %rv011 = vector.load %w_stage_view[%w_row1, %fc1] : view<64x72xf16> -> vector<16xf16> %rhs011 = vector.fragment %rv011 shape [%c16, %c16] : vector<16xf16> %na1_1 = vector.mma %lhs1, %rhs011, %na1_0 : vector<16xf16>, vector<16xf16>, vector<8xf32> %fc2 = index.constant 32 : index %lhs2 = vector.fragment.load %x_stage_view[%wave_row, %fc2] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16> %rv002 = vector.load %w_stage_view[%w_row0, %fc2] : view<64x72xf16> -> vector<16xf16> %rhs002 = vector.fragment %rv002 shape [%c16, %c16] : vector<16xf16> %na0_2 = vector.mma %lhs2, %rhs002, %na0_1 : vector<16xf16>, vector<16xf16>, vector<8xf32> %rv012 = vector.load %w_stage_view[%w_row1, %fc2] : view<64x72xf16> -> vector<16xf16> %rhs012 = vector.fragment %rv012 shape [%c16, %c16] : vector<16xf16> %na1_2 = vector.mma %lhs2, %rhs012, %na1_1 : vector<16xf16>, vector<16xf16>, vector<8xf32> %fc3 = index.constant 48 : index %lhs3 = vector.fragment.load %x_stage_view[%wave_row, %fc3] shape [%c16, %c16] : view<64x72xf16> -> vector<16xf16> %rv003 = vector.load %w_stage_view[%w_row0, %fc3] : view<64x72xf16> -> vector<16xf16> %rhs003 = vector.fragment %rv003 shape [%c16, %c16] : vector<16xf16> %na0_3 = vector.mma %lhs3, %rhs003, %na0_2 : vector<16xf16>, vector<16xf16>, vector<8xf32> %rv013 = vector.load %w_stage_view[%w_row1, %fc3] : view<64x72xf16> -> vector<16xf16> %rhs013 = vector.fragment %rv013 shape [%c16, %c16] : vector<16xf16> %na1_3 = vector.mma %lhs3, %rhs013, %na1_2 : vector<16xf16>, vector<16xf16>, vector<8xf32> kernel.barrier scope(workgroup) ordering(acq_rel) scf.yield %na0_3, %na1_3 : vector<8xf32>, vector<8xf32> } %inv_bytes = index.constant 4096 : offset %inv_tile = buffer.alloca align(16) %inv_bytes : buffer %inv_view = buffer.view %inv_tile[%origin] : buffer -> view<64x16xf32> %writer = index.cmp ult, %workitem, %c64 : index scf.if %writer { %inv_row_value = index.add %row_origin, %workitem : index %inv_row = index.assume %inv_row_value [lt(%inv_row_value, %rows)] : index %inverse = view.load %s_view[%inv_row, %c1] : view<[%rows]x2xf32> -> f32 %inverse16 = vector.splat %inverse : vector<16xf32> %bw = index.assume %workitem [range(%workitem, 0, 63)] : index vector.store %inverse16, %inv_view[%bw, %c0] : vector<16xf32>, view<64x16xf32> } kernel.barrier scope(workgroup) ordering(acq_rel) %inv_frag = vector.fragment.load %inv_view[%wave_row, %c0] shape [%c16, %c16] : view<64x16xf32> -> vector<8xf32> %a0 = vector.mulf %a0_raw, %inv_frag : vector<8xf32> %a1 = vector.mulf %a1_raw, %inv_frag : vector<8xf32> vector.fragment.store %a0, %y_view[%bounded_wave_row, %bounded_wave_out] shape [%c16, %c16] : vector<8xf32>, view<64x64xf32, %y_layout> vector.fragment.store %a1, %y_view[%bounded_wave_row, %wave_out1] shape [%c16, %c16] : vector<8xf32>, view<64x64xf32, %y_layout> kernel.return } kernel.def export("v_bias_none_2048") @v_bias_none_2048(%rows: index) { %c1 = index.constant 1 : index %c256 = index.constant 256 : index %cols = index.constant 2048 : index %cg = index.constant 8 : index kernel.launch.config workgroups(%rows, %cg, %c1) workgroup_size(%c256, %c1, %c1) : index } launch(%rows_arg: index, %x: buffer, %bias: buffer, %y: buffer) { %rows = index.assume %rows_arg [range(%rows_arg, 1, 65536)] : index %c256 = index.constant 256 : index %cols = index.constant 2048 : index %row_id = kernel.workgroup.id : index %row = index.assume %row_id [lt(%row_id, %rows)] : index %cg = kernel.workgroup.id : index %lid = kernel.workitem.id : index %c = index.madd %cg, %c256, %lid : index %live = index.cmp ult, %c, %cols : index scf.if %live { %bc = index.assume %c [lt(%c, %cols)] : index %zero = index.constant 0 : offset %zf = scalar.constant 0.0 : f32 %xv = buffer.view %x[%zero] : buffer -> view<[%rows]x2048xf32> %bv = buffer.view %bias[%zero] : buffer -> view<2048xf32> %x0 = view.load %xv[%row, %bc] : view<[%rows]x2048xf32> -> f32 %b0 = view.load %bv[%bc] : view<2048xf32> -> f32 %a = scalar.addf %x0, %b0 : f32 %r = scalar.addf %a, %zf : f32 %yv = buffer.view %y[%zero] : buffer -> view<[%rows]x2048xf32> view.store %r, %yv[%row, %bc] : f32, view<[%rows]x2048xf32> } kernel.return } kernel.def export("v_resize_bicubic_h_hdr") @v_resize_bicubic_h_hdr(%out_w: index, %max_rows: index) { %count = index.mul %out_w, %max_rows : index %c1 = index.constant 1 : index %c256 = index.constant 256 : index %c255 = index.constant 255 : index %n_up = index.add %count, %c255 : index %groups = index.div %n_up, %c256 : index kernel.launch.config workgroups(%groups, %c1, %c1) workgroup_size(%c256, %c1, %c1) : index } launch(%out_n_arg: index, %other_arg: index, %src: buffer, %dst: buffer) { %zero_h = index.constant 0 : offset %hv = buffer.view %src[%zero_h] : buffer -> view<16xi32> %in_w32 = view.load %hv[0] : view<16xi32> -> i32 %rows32 = view.load %hv[1] : view<16xi32> -> i32 %cin32 = view.load %hv[2] : view<16xi32> -> i32 %pix32 = view.load %hv[3] : view<16xi32> -> i32 %in_b = scalar.assume %in_w32 [range(%in_w32, 1, 16384)] : i32 %in_n = index.cast %in_b : i32 to index %rows_b = scalar.assume %rows32 [range(%rows32, 1, 16384)] : i32 %rows_i = index.cast %rows_b : i32 to index %cin_b = scalar.assume %cin32 [range(%cin32, 3, 4)] : i32 %cin_i = index.cast %cin_b : i32 to index %pix_b = scalar.assume %pix32 [range(%pix32, 0, 2147483647)] : i32 %pix_i = index.cast %pix_b : i32 to index %out_n = index.assume %out_n_arg [range(%out_n_arg, 1, 65536)] : index %other = index.assume %other_arg [range(%other_arg, 1, 65536)] : index %count = index.mul %rows_i, %out_n : index %c256 = index.constant 256 : index %gid = kernel.workgroup.id : index %lid = kernel.workitem.id : index %i = index.madd %gid, %c256, %lid : index %live = index.cmp ult, %i, %count : index scf.if %live { %zero = index.constant 0 : offset %c0 = index.constant 0 : index %c1 = index.constant 1 : index %c3 = index.constant 3 : index %ccin = index.add %cin_i, %c0 : index %zf = scalar.constant 0.0 : f32 %onef = scalar.constant 1.0 : f32 %halff = scalar.constant 0.5 : f32 %twof = scalar.constant 2.0 : f32 %threef = scalar.constant 3.0 : f32 %m3f = scalar.constant -3.0 : f32 %fourf = scalar.constant 4.0 : f32 %fivef = scalar.constant 5.0 : f32 %eightf = scalar.constant 8.0 : f32 %c1p5 = scalar.constant 1.5 : f32 %c2p5 = scalar.constant 2.5 : f32 %mhalf = scalar.constant -0.5 : f32 %pif = scalar.constant 3.141592653589793 : f32 %tinyf = scalar.constant 1.0e-12 : f32 %fxp = scalar.constant 4194304.0 : f32 %o = index.rem %i, %out_n : index %row = index.div %i, %out_n : index %in_i = index.cast %in_n : index to i32 %out_i = index.cast %out_n : index to i32 %in_f = scalar.sitofp %in_i : i32 to f32 %out_f = scalar.sitofp %out_i : i32 to f32 %scale = scalar.divf %in_f, %out_f : f32 %fs = scalar.maxnumf %scale, %onef : f32 %supc = scalar.constant 2.0 : f32 %support = scalar.mulf %supc, %fs : f32 %ss = scalar.divf %onef, %fs : f32 %oi = index.cast %o : index to i32 %of = scalar.sitofp %oi : i32 to f32 %oh = scalar.addf %of, %halff : f32 %center = scalar.mulf %oh, %scale : f32 %lo0 = scalar.subf %center, %support : f32 %lo1 = scalar.addf %lo0, %halff : f32 %lo2 = scalar.fptosi %lo1 : f32 to i32 %zi = scalar.constant 0 : i32 %lo3 = scalar.maxsi %lo2, %zi : i32 %hi0 = scalar.addf %center, %support : f32 %hi1 = scalar.addf %hi0, %halff : f32 %hi2 = scalar.fptosi %hi1 : f32 to i32 %hi3 = scalar.minsi %hi2, %in_i : i32 %lo4 = scalar.assume %lo3 [range(%lo3, 0, 65536)] : i32 %hi4 = scalar.assume %hi3 [range(%hi3, 0, 65536)] : i32 %xlo = index.cast %lo4 : i32 to index %xhi = index.cast %hi4 : i32 to index %ww = scf.for %xi = [%xlo to %xhi step %c1](%acc = %zf : f32) -> (f32) { %xii = index.cast %xi : index to i32 %xf = scalar.sitofp %xii : i32 to f32 %d0 = scalar.subf %xf, %center : f32 %d1 = scalar.addf %d0, %halff : f32 %d = scalar.mulf %d1, %ss : f32 %wv_x = scalar.absf %d : f32 %wv_lt1 = scalar.cmpf olt, %wv_x, %onef : f32 %wv_lt2 = scalar.cmpf olt, %wv_x, %twof : f32 %wv_p1a = scalar.mulf %wv_x, %c1p5 : f32 %wv_p1b = scalar.subf %wv_p1a, %c2p5 : f32 %wv_p1c = scalar.mulf %wv_p1b, %wv_x : f32 %wv_p1d = scalar.mulf %wv_p1c, %wv_x : f32 %wv_p1 = scalar.addf %wv_p1d, %onef : f32 %wv_p2a = scalar.subf %wv_x, %fivef : f32 %wv_p2b = scalar.mulf %wv_p2a, %wv_x : f32 %wv_p2c = scalar.addf %wv_p2b, %eightf : f32 %wv_p2d = scalar.mulf %wv_p2c, %wv_x : f32 %wv_p2e = scalar.subf %wv_p2d, %fourf : f32 %wv_p2 = scalar.mulf %wv_p2e, %mhalf : f32 %wv_q = scf.select %wv_lt2, %wv_p2, %zf : f32 %wv = scf.select %wv_lt1, %wv_p1, %wv_q : f32 %na = scalar.addf %acc, %wv : f32 scf.yield %na : f32 } %wnz = scalar.cmpf one, %ww, %zf : f32 %wdiv = scf.select %wnz, %ww, %onef : f32 %srcn = index.constant 268435456 : index %sv = buffer.view %src[%zero] : buffer -> view<[%srcn]xi8> %bias = scalar.constant 2097152 : i32 %s0, %s1, %s2 = scf.for %xi = [%xlo to %xhi step %c1](%a0 = %bias : i32, %a1 = %bias : i32, %a2 = %bias : i32) -> (i32, i32, i32) { %xii = index.cast %xi : index to i32 %xf = scalar.sitofp %xii : i32 to f32 %d0 = scalar.subf %xf, %center : f32 %d1 = scalar.addf %d0, %halff : f32 %d = scalar.mulf %d1, %ss : f32 %wv_x = scalar.absf %d : f32 %wv_lt1 = scalar.cmpf olt, %wv_x, %onef : f32 %wv_lt2 = scalar.cmpf olt, %wv_x, %twof : f32 %wv_p1a = scalar.mulf %wv_x, %c1p5 : f32 %wv_p1b = scalar.subf %wv_p1a, %c2p5 : f32 %wv_p1c = scalar.mulf %wv_p1b, %wv_x : f32 %wv_p1d = scalar.mulf %wv_p1c, %wv_x : f32 %wv_p1 = scalar.addf %wv_p1d, %onef : f32 %wv_p2a = scalar.subf %wv_x, %fivef : f32 %wv_p2b = scalar.mulf %wv_p2a, %wv_x : f32 %wv_p2c = scalar.addf %wv_p2b, %eightf : f32 %wv_p2d = scalar.mulf %wv_p2c, %wv_x : f32 %wv_p2e = scalar.subf %wv_p2d, %fourf : f32 %wv_p2 = scalar.mulf %wv_p2e, %mhalf : f32 %wv_q = scf.select %wv_lt2, %wv_p2, %zf : f32 %wv = scf.select %wv_lt1, %wv_p1, %wv_q : f32 %wn = scalar.divf %wv, %wdiv : f32 %wfx = scalar.mulf %wn, %fxp : f32 %neg = scalar.cmpf olt, %wn, %zf : f32 %rnd = scf.select %neg, %mhalf, %halff : f32 %wr = scalar.addf %wfx, %rnd : f32 %wi = scalar.fptosi %wr : f32 to i32 %src_idx = index.madd %row, %in_n, %xi : index %el_rel = index.mul %src_idx, %ccin : index %el0 = index.add %el_rel, %pix_i : index %el = index.assume %el0 [lt(%el0, %srcn)] : index %el1r = index.add %el0, %c1 : index %el1 = index.assume %el1r [lt(%el1r, %srcn)] : index %el2r = index.add %el1r, %c1 : index %el2 = index.assume %el2r [lt(%el2r, %srcn)] : index %p0r = view.load %sv[%el] : view<[%srcn]xi8> -> i8 %p1r = view.load %sv[%el1] : view<[%srcn]xi8> -> i8 %p2r = view.load %sv[%el2] : view<[%srcn]xi8> -> i8 %p0 = scalar.extui %p0r : i8 to i32 %p1 = scalar.extui %p1r : i8 to i32 %p2 = scalar.extui %p2r : i8 to i32 %m0 = scalar.muli %p0, %wi : i32 %m1 = scalar.muli %p1, %wi : i32 %m2 = scalar.muli %p2, %wi : i32 %n0 = scalar.addi %a0, %m0 : i32 %n1 = scalar.addi %a1, %m1 : i32 %n2 = scalar.addi %a2, %m2 : i32 scf.yield %n0, %n1, %n2 : i32, i32, i32 } %sh = scalar.constant 22 : i32 %c255i = scalar.constant 255 : i32 %dst_idx = index.madd %row, %out_n, %o : index %dstn = index.constant 268435456 : index %dv = buffer.view %dst[%zero] : buffer -> view<[%dstn]xi8> %de0 = index.mul %dst_idx, %c3 : index %c2x = index.constant 2 : index %q0a = scalar.shrsi %s0, %sh : i32 %q0b = scalar.maxsi %q0a, %zi : i32 %q0c = scalar.minsi %q0b, %c255i : i32 %q0 = scalar.trunci %q0c : i32 to i8 %dc0r = index.add %de0, %c0 : index %dc0 = index.assume %dc0r [lt(%dc0r, %dstn)] : index view.store %q0, %dv[%dc0] : i8, view<[%dstn]xi8> %q1a = scalar.shrsi %s1, %sh : i32 %q1b = scalar.maxsi %q1a, %zi : i32 %q1c = scalar.minsi %q1b, %c255i : i32 %q1 = scalar.trunci %q1c : i32 to i8 %dc1r = index.add %de0, %c1 : index %dc1 = index.assume %dc1r [lt(%dc1r, %dstn)] : index view.store %q1, %dv[%dc1] : i8, view<[%dstn]xi8> %q2a = scalar.shrsi %s2, %sh : i32 %q2b = scalar.maxsi %q2a, %zi : i32 %q2c = scalar.minsi %q2b, %c255i : i32 %q2 = scalar.trunci %q2c : i32 to i8 %dc2r = index.add %de0, %c2x : index %dc2 = index.assume %dc2r [lt(%dc2r, %dstn)] : index view.store %q2, %dv[%dc2] : i8, view<[%dstn]xi8> } kernel.return } kernel.def export("v_resize_bicubic_v_hdr") @v_resize_bicubic_v_hdr(%out_h: index, %cols: index) { %count = index.mul %out_h, %cols : index %c1 = index.constant 1 : index %c256 = index.constant 256 : index %c255 = index.constant 255 : index %n_up = index.add %count, %c255 : index %groups = index.div %n_up, %c256 : index kernel.launch.config workgroups(%groups, %c1, %c1) workgroup_size(%c256, %c1, %c1) : index } launch(%out_n_arg: index, %other_arg: index, %hdr: buffer, %src: buffer, %dst: buffer) { %zero_h = index.constant 0 : offset %hv = buffer.view %hdr[%zero_h] : buffer -> view<16xi32> %in_w32 = view.load %hv[1] : view<16xi32> -> i32 %rows32 = scalar.constant 1 : i32 %cin32 = scalar.constant 3 : i32 %pix32 = scalar.constant 0 : i32 %in_b = scalar.assume %in_w32 [range(%in_w32, 1, 16384)] : i32 %in_n = index.cast %in_b : i32 to index %rows_b = scalar.assume %rows32 [range(%rows32, 1, 16384)] : i32 %rows_i = index.cast %rows_b : i32 to index %cin_b = scalar.assume %cin32 [range(%cin32, 3, 4)] : i32 %cin_i = index.cast %cin_b : i32 to index %pix_b = scalar.assume %pix32 [range(%pix32, 0, 2147483647)] : i32 %pix_i = index.cast %pix_b : i32 to index %out_n = index.assume %out_n_arg [range(%out_n_arg, 1, 65536)] : index %other = index.assume %other_arg [range(%other_arg, 1, 65536)] : index %count = index.mul %other, %out_n : index %c256 = index.constant 256 : index %gid = kernel.workgroup.id : index %lid = kernel.workitem.id : index %i = index.madd %gid, %c256, %lid : index %live = index.cmp ult, %i, %count : index scf.if %live { %zero = index.constant 0 : offset %c0 = index.constant 0 : index %c1 = index.constant 1 : index %c3 = index.constant 3 : index %ccin = index.constant 3 : index %zf = scalar.constant 0.0 : f32 %onef = scalar.constant 1.0 : f32 %halff = scalar.constant 0.5 : f32 %twof = scalar.constant 2.0 : f32 %threef = scalar.constant 3.0 : f32 %m3f = scalar.constant -3.0 : f32 %fourf = scalar.constant 4.0 : f32 %fivef = scalar.constant 5.0 : f32 %eightf = scalar.constant 8.0 : f32 %c1p5 = scalar.constant 1.5 : f32 %c2p5 = scalar.constant 2.5 : f32 %mhalf = scalar.constant -0.5 : f32 %pif = scalar.constant 3.141592653589793 : f32 %tinyf = scalar.constant 1.0e-12 : f32 %fxp = scalar.constant 4194304.0 : f32 %o_v = index.div %i, %other : index %col = index.rem %i, %other : index %in_i = index.cast %in_n : index to i32 %out_i = index.cast %out_n : index to i32 %in_f = scalar.sitofp %in_i : i32 to f32 %out_f = scalar.sitofp %out_i : i32 to f32 %scale = scalar.divf %in_f, %out_f : f32 %fs = scalar.maxnumf %scale, %onef : f32 %supc = scalar.constant 2.0 : f32 %support = scalar.mulf %supc, %fs : f32 %ss = scalar.divf %onef, %fs : f32 %oi = index.cast %o_v : index to i32 %of = scalar.sitofp %oi : i32 to f32 %oh = scalar.addf %of, %halff : f32 %center = scalar.mulf %oh, %scale : f32 %lo0 = scalar.subf %center, %support : f32 %lo1 = scalar.addf %lo0, %halff : f32 %lo2 = scalar.fptosi %lo1 : f32 to i32 %zi = scalar.constant 0 : i32 %lo3 = scalar.maxsi %lo2, %zi : i32 %hi0 = scalar.addf %center, %support : f32 %hi1 = scalar.addf %hi0, %halff : f32 %hi2 = scalar.fptosi %hi1 : f32 to i32 %hi3 = scalar.minsi %hi2, %in_i : i32 %lo4 = scalar.assume %lo3 [range(%lo3, 0, 65536)] : i32 %hi4 = scalar.assume %hi3 [range(%hi3, 0, 65536)] : i32 %xlo = index.cast %lo4 : i32 to index %xhi = index.cast %hi4 : i32 to index %ww = scf.for %xi = [%xlo to %xhi step %c1](%acc = %zf : f32) -> (f32) { %xii = index.cast %xi : index to i32 %xf = scalar.sitofp %xii : i32 to f32 %d0 = scalar.subf %xf, %center : f32 %d1 = scalar.addf %d0, %halff : f32 %d = scalar.mulf %d1, %ss : f32 %wv_x = scalar.absf %d : f32 %wv_lt1 = scalar.cmpf olt, %wv_x, %onef : f32 %wv_lt2 = scalar.cmpf olt, %wv_x, %twof : f32 %wv_p1a = scalar.mulf %wv_x, %c1p5 : f32 %wv_p1b = scalar.subf %wv_p1a, %c2p5 : f32 %wv_p1c = scalar.mulf %wv_p1b, %wv_x : f32 %wv_p1d = scalar.mulf %wv_p1c, %wv_x : f32 %wv_p1 = scalar.addf %wv_p1d, %onef : f32 %wv_p2a = scalar.subf %wv_x, %fivef : f32 %wv_p2b = scalar.mulf %wv_p2a, %wv_x : f32 %wv_p2c = scalar.addf %wv_p2b, %eightf : f32 %wv_p2d = scalar.mulf %wv_p2c, %wv_x : f32 %wv_p2e = scalar.subf %wv_p2d, %fourf : f32 %wv_p2 = scalar.mulf %wv_p2e, %mhalf : f32 %wv_q = scf.select %wv_lt2, %wv_p2, %zf : f32 %wv = scf.select %wv_lt1, %wv_p1, %wv_q : f32 %na = scalar.addf %acc, %wv : f32 scf.yield %na : f32 } %wnz = scalar.cmpf one, %ww, %zf : f32 %wdiv = scf.select %wnz, %ww, %onef : f32 %srcn = index.constant 268435456 : index %sv = buffer.view %src[%zero] : buffer -> view<[%srcn]xi8> %bias = scalar.constant 2097152 : i32 %s0, %s1, %s2 = scf.for %xi = [%xlo to %xhi step %c1](%a0 = %bias : i32, %a1 = %bias : i32, %a2 = %bias : i32) -> (i32, i32, i32) { %xii = index.cast %xi : index to i32 %xf = scalar.sitofp %xii : i32 to f32 %d0 = scalar.subf %xf, %center : f32 %d1 = scalar.addf %d0, %halff : f32 %d = scalar.mulf %d1, %ss : f32 %wv_x = scalar.absf %d : f32 %wv_lt1 = scalar.cmpf olt, %wv_x, %onef : f32 %wv_lt2 = scalar.cmpf olt, %wv_x, %twof : f32 %wv_p1a = scalar.mulf %wv_x, %c1p5 : f32 %wv_p1b = scalar.subf %wv_p1a, %c2p5 : f32 %wv_p1c = scalar.mulf %wv_p1b, %wv_x : f32 %wv_p1d = scalar.mulf %wv_p1c, %wv_x : f32 %wv_p1 = scalar.addf %wv_p1d, %onef : f32 %wv_p2a = scalar.subf %wv_x, %fivef : f32 %wv_p2b = scalar.mulf %wv_p2a, %wv_x : f32 %wv_p2c = scalar.addf %wv_p2b, %eightf : f32 %wv_p2d = scalar.mulf %wv_p2c, %wv_x : f32 %wv_p2e = scalar.subf %wv_p2d, %fourf : f32 %wv_p2 = scalar.mulf %wv_p2e, %mhalf : f32 %wv_q = scf.select %wv_lt2, %wv_p2, %zf : f32 %wv = scf.select %wv_lt1, %wv_p1, %wv_q : f32 %wn = scalar.divf %wv, %wdiv : f32 %wfx = scalar.mulf %wn, %fxp : f32 %neg = scalar.cmpf olt, %wn, %zf : f32 %rnd = scf.select %neg, %mhalf, %halff : f32 %wr = scalar.addf %wfx, %rnd : f32 %wi = scalar.fptosi %wr : f32 to i32 %src_idx = index.madd %xi, %other, %col : index %el0 = index.mul %src_idx, %ccin : index %el = index.assume %el0 [lt(%el0, %srcn)] : index %el1r = index.add %el0, %c1 : index %el1 = index.assume %el1r [lt(%el1r, %srcn)] : index %el2r = index.add %el1r, %c1 : index %el2 = index.assume %el2r [lt(%el2r, %srcn)] : index %p0r = view.load %sv[%el] : view<[%srcn]xi8> -> i8 %p1r = view.load %sv[%el1] : view<[%srcn]xi8> -> i8 %p2r = view.load %sv[%el2] : view<[%srcn]xi8> -> i8 %p0 = scalar.extui %p0r : i8 to i32 %p1 = scalar.extui %p1r : i8 to i32 %p2 = scalar.extui %p2r : i8 to i32 %m0 = scalar.muli %p0, %wi : i32 %m1 = scalar.muli %p1, %wi : i32 %m2 = scalar.muli %p2, %wi : i32 %n0 = scalar.addi %a0, %m0 : i32 %n1 = scalar.addi %a1, %m1 : i32 %n2 = scalar.addi %a2, %m2 : i32 scf.yield %n0, %n1, %n2 : i32, i32, i32 } %sh = scalar.constant 22 : i32 %c255i = scalar.constant 255 : i32 %dst_idx = index.madd %o_v, %other, %col : index %dstn = index.constant 268435456 : index %dv = buffer.view %dst[%zero] : buffer -> view<[%dstn]xi8> %de0 = index.mul %dst_idx, %c3 : index %c2x = index.constant 2 : index %q0a = scalar.shrsi %s0, %sh : i32 %q0b = scalar.maxsi %q0a, %zi : i32 %q0c = scalar.minsi %q0b, %c255i : i32 %q0 = scalar.trunci %q0c : i32 to i8 %dc0r = index.add %de0, %c0 : index %dc0 = index.assume %dc0r [lt(%dc0r, %dstn)] : index view.store %q0, %dv[%dc0] : i8, view<[%dstn]xi8> %q1a = scalar.shrsi %s1, %sh : i32 %q1b = scalar.maxsi %q1a, %zi : i32 %q1c = scalar.minsi %q1b, %c255i : i32 %q1 = scalar.trunci %q1c : i32 to i8 %dc1r = index.add %de0, %c1 : index %dc1 = index.assume %dc1r [lt(%dc1r, %dstn)] : index view.store %q1, %dv[%dc1] : i8, view<[%dstn]xi8> %q2a = scalar.shrsi %s2, %sh : i32 %q2b = scalar.maxsi %q2a, %zi : i32 %q2c = scalar.minsi %q2b, %c255i : i32 %q2 = scalar.trunci %q2c : i32 to i8 %dc2r = index.add %de0, %c2x : index %dc2 = index.assume %dc2r [lt(%dc2r, %dstn)] : index view.store %q2, %dv[%dc2] : i8, view<[%dstn]xi8> } kernel.return } kernel.def export("v_copy") @v_copy(%n: index) { %c1 = index.constant 1 : index %c256 = index.constant 256 : index %c255 = index.constant 255 : index %n_up = index.add %n, %c255 : index %groups = index.div %n_up, %c256 : index kernel.launch.config workgroups(%groups, %c1, %c1) workgroup_size(%c256, %c1, %c1) : index } launch(%n_arg: index, %x: buffer, %y: buffer) { %n = index.assume %n_arg [range(%n_arg, 1, 268435456)] : index %c256 = index.constant 256 : index %gid = kernel.workgroup.id : index %lid = kernel.workitem.id : index %i = index.madd %gid, %c256, %lid : index %live = index.cmp ult, %i, %n : index scf.if %live { %bi = index.assume %i [lt(%i, %n)] : index %zero = index.constant 0 : offset %xv = buffer.view %x[%zero] : buffer -> view<[%n]xf32> %a = view.load %xv[%bi] : view<[%n]xf32> -> f32 %yv = buffer.view %y[%zero] : buffer -> view<[%n]xf32> view.store %a, %yv[%bi] : f32, view<[%n]xf32> } kernel.return } config.decl @vision.height : %value: index where [range(%value, 1, 16384)] config.decl @vision.width : %value: index where [range(%value, 1, 16384)] command.program.def public @vision_encode() launch(%weights: buffer, %input: buffer, %output: buffer) { %img_h = config.get @vision.height : index %img_w = config.get @vision.width : index %cP = index.constant 16 : index %cm = index.constant 2 : index %c1 = index.constant 1 : index %c3 = index.constant 3 : index %c63 = index.constant 63 : index %c64 = index.constant 64 : index %c255 = index.constant 255 : index %c256 = index.constant 256 : index %gh = index.div %img_h, %cP : index %gw = index.div %img_w, %cP : index %T = index.mul %gh, %gw : index %mh = index.div %gh, %cm : index %mw = index.div %gw, %cm : index %M = index.mul %mh, %mw : index %T_up = index.add %T, %c255 : index %T_t = index.div %T_up, %c256 : index %Tp = index.mul %T_t, %c256 : index %M_up = index.add %M, %c63 : index %M_t = index.div %M_up, %c64 : index %Mp = index.mul %M_t, %c64 : index %NH = index.constant 16 : index %img_px = index.mul %img_h, %img_w : index %n_pix = index.mul %img_px, %c3 : index %max_rows = index.constant 4096 : index %hdr_bytes = index.constant 64 : offset %zero_o = index.constant 0 : offset %h_w = index.constant 1152 : index %h_n = index.mul %Tp, %h_w : index %h_bpe = index.constant 4 : offset %h_size = index.scale %h_n, %h_bpe : index, offset -> offset %h = buffer.alloca align(256) %h_size : buffer %x_w = index.constant 4608 : index %x_n = index.mul %Tp, %x_w : index %x_bpe = index.constant 4 : offset %x_size = index.scale %x_n, %x_bpe : index, offset -> offset %x = buffer.alloca align(256) %x_size : buffer %q_w = index.constant 3456 : index %q_n = index.mul %Tp, %q_w : index %q_bpe = index.constant 4 : offset %q_size = index.scale %q_n, %q_bpe : index, offset -> offset %q = buffer.alloca align(256) %q_size : buffer %k_w = index.constant 1152 : index %k_n = index.mul %Tp, %k_w : index %k_bpe = index.constant 4 : offset %k_size = index.scale %k_n, %k_bpe : index, offset -> offset %k = buffer.alloca align(256) %k_size : buffer %v_w = index.constant 1152 : index %v_n = index.mul %Tp, %v_w : index %v_bpe = index.constant 4 : offset %v_size = index.scale %v_n, %v_bpe : index, offset -> offset %v = buffer.alloca align(256) %v_size : buffer %o_w = index.constant 1152 : index %o_n = index.mul %Tp, %o_w : index %o_bpe = index.constant 4 : offset %o_size = index.scale %o_n, %o_bpe : index, offset -> offset %o = buffer.alloca align(256) %o_size : buffer %a_w = index.constant 1152 : index %a_n = index.mul %Tp, %a_w : index %a_bpe = index.constant 4 : offset %a_size = index.scale %a_n, %a_bpe : index, offset -> offset %a = buffer.alloca align(256) %a_size : buffer %f_w = index.constant 4304 : index %f_n = index.mul %Tp, %f_w : index %f_bpe = index.constant 4 : offset %f_size = index.scale %f_n, %f_bpe : index, offset -> offset %f = buffer.alloca align(256) %f_size : buffer %g_w = index.constant 4304 : index %g_n = index.mul %Tp, %g_w : index %g_bpe = index.constant 4 : offset %g_size = index.scale %g_n, %g_bpe : index, offset -> offset %g = buffer.alloca align(256) %g_size : buffer %xs_w = index.constant 4608 : index %xs_n = index.mul %Tp, %xs_w : index %xs_bpe = index.constant 4 : offset %xs_size = index.scale %xs_n, %xs_bpe : index, offset -> offset %xs = buffer.alloca align(256) %xs_size : buffer %x16_w = index.constant 4608 : index %x16_n = index.mul %Tp, %x16_w : index %x16_bpe = index.constant 2 : offset %x16_size = index.scale %x16_n, %x16_bpe : index, offset -> offset %x16 = buffer.alloca align(256) %x16_size : buffer %sc_w = index.constant 2 : index %sc_n = index.mul %Tp, %sc_w : index %sc_bpe = index.constant 4 : offset %sc_size = index.scale %sc_n, %sc_bpe : index, offset -> offset %sc = buffer.alloca align(256) %sc_size : buffer %m_in_w = index.constant 4608 : index %m_in_n = index.mul %Mp, %m_in_w : index %m_in_bpe = index.constant 4 : offset %m_in_size = index.scale %m_in_n, %m_in_bpe : index, offset -> offset %m_in = buffer.alloca align(256) %m_in_size : buffer %m1_w = index.constant 4608 : index %m1_n = index.mul %Mp, %m1_w : index %m1_bpe = index.constant 4 : offset %m1_size = index.scale %m1_n, %m1_bpe : index, offset -> offset %m1 = buffer.alloca align(256) %m1_size : buffer %m2_w = index.constant 4608 : index %m2_n = index.mul %Mp, %m2_w : index %m2_bpe = index.constant 4 : offset %m2_size = index.scale %m2_n, %m2_bpe : index, offset -> offset %m2 = buffer.alloca align(256) %m2_size : buffer %out_w = index.constant 2048 : index %out_n = index.mul %Mp, %out_w : index %out_bpe = index.constant 4 : offset %out_size = index.scale %out_n, %out_bpe : index, offset -> offset %out = buffer.alloca align(256) %out_size : buffer %wide_n = index.mul %max_rows, %img_w : index %wide_n3 = index.mul %wide_n, %c3 : index %wide_bpe = index.constant 1 : offset %wide_size = index.scale %wide_n3, %wide_bpe : index, offset -> offset %wide = buffer.alloca align(256) %wide_size : buffer %pix_size = index.scale %n_pix, %wide_bpe : index, offset -> offset %pix = buffer.alloca align(256) %pix_size : buffer %pixf_bpe = index.constant 4 : offset %pixf_size = index.scale %n_pix, %pixf_bpe : index, offset -> offset %pixf = buffer.alloca align(256) %pixf_size : buffer kernel.launch @v_resize_bicubic_h_hdr[%img_w, %max_rows](%img_w, %max_rows, %input, %wide) : [index, index](index, index, buffer, buffer) kernel.launch @v_resize_bicubic_v_hdr[%img_h, %img_w](%img_h, %img_w, %input, %wide, %pix) : [index, index](index, index, buffer, buffer, buffer) %w_v_patch_embd_weight = command.parameter %weights, "v.patch_embd.weight" : view<1769472xi8> %w_v_patch_embd_weight_1 = command.parameter %weights, "v.patch_embd.weight.1" : view<1769472xi8> %w_v_patch_embd_bias = command.parameter %weights, "v.patch_embd.bias" : view<4608xi8> %w_v_position_embd_weight = command.parameter %weights, "v.position_embd.weight" : view<10616832xi8> %w_v_blk_0_ln1_weight = command.parameter %weights, "v.blk.0.ln1.weight" : view<4608xi8> %w_v_blk_0_ln1_bias = command.parameter %weights, "v.blk.0.ln1.bias" : view<4608xi8> %w_v_blk_0_attn_qkv_weight = command.parameter %weights, "v.blk.0.attn_qkv.weight" : view<7962624xi8> %w_v_blk_0_attn_qkv_bias = command.parameter %weights, "v.blk.0.attn_qkv.bias" : view<13824xi8> %w_v_blk_0_attn_out_weight = command.parameter %weights, "v.blk.0.attn_out.weight" : view<2654208xi8> %w_v_blk_0_attn_out_bias = command.parameter %weights, "v.blk.0.attn_out.bias" : view<4608xi8> %w_v_blk_0_ln2_weight = command.parameter %weights, "v.blk.0.ln2.weight" : view<4608xi8> %w_v_blk_0_ln2_bias = command.parameter %weights, "v.blk.0.ln2.bias" : view<4608xi8> %w_v_blk_0_ffn_up_weight = command.parameter %weights, "v.blk.0.ffn_up.weight" : view<9916416xi8> %w_v_blk_0_ffn_up_bias = command.parameter %weights, "v.blk.0.ffn_up.bias" : view<17216xi8> %w_v_blk_0_ffn_down_weight = command.parameter %weights, "v.blk.0.ffn_down.weight" : view<9916416xi8> %w_v_blk_0_ffn_down_bias = command.parameter %weights, "v.blk.0.ffn_down.bias" : view<4608xi8> %w_v_blk_1_ln1_weight = command.parameter %weights, "v.blk.1.ln1.weight" : view<4608xi8> %w_v_blk_1_ln1_bias = command.parameter %weights, "v.blk.1.ln1.bias" : view<4608xi8> %w_v_blk_1_attn_qkv_weight = command.parameter %weights, "v.blk.1.attn_qkv.weight" : view<7962624xi8> %w_v_blk_1_attn_qkv_bias = command.parameter %weights, "v.blk.1.attn_qkv.bias" : view<13824xi8> %w_v_blk_1_attn_out_weight = command.parameter %weights, "v.blk.1.attn_out.weight" : view<2654208xi8> %w_v_blk_1_attn_out_bias = command.parameter %weights, "v.blk.1.attn_out.bias" : view<4608xi8> %w_v_blk_1_ln2_weight = command.parameter %weights, "v.blk.1.ln2.weight" : view<4608xi8> %w_v_blk_1_ln2_bias = command.parameter %weights, "v.blk.1.ln2.bias" : view<4608xi8> %w_v_blk_1_ffn_up_weight = command.parameter %weights, "v.blk.1.ffn_up.weight" : view<9916416xi8> %w_v_blk_1_ffn_up_bias = command.parameter %weights, "v.blk.1.ffn_up.bias" : view<17216xi8> %w_v_blk_1_ffn_down_weight = command.parameter %weights, "v.blk.1.ffn_down.weight" : view<9916416xi8> %w_v_blk_1_ffn_down_bias = command.parameter %weights, "v.blk.1.ffn_down.bias" : view<4608xi8> %w_v_blk_2_ln1_weight = command.parameter %weights, "v.blk.2.ln1.weight" : view<4608xi8> %w_v_blk_2_ln1_bias = command.parameter %weights, "v.blk.2.ln1.bias" : view<4608xi8> %w_v_blk_2_attn_qkv_weight = command.parameter %weights, "v.blk.2.attn_qkv.weight" : view<7962624xi8> %w_v_blk_2_attn_qkv_bias = command.parameter %weights, "v.blk.2.attn_qkv.bias" : view<13824xi8> %w_v_blk_2_attn_out_weight = command.parameter %weights, "v.blk.2.attn_out.weight" : view<2654208xi8> %w_v_blk_2_attn_out_bias = command.parameter %weights, "v.blk.2.attn_out.bias" : view<4608xi8> %w_v_blk_2_ln2_weight = command.parameter %weights, "v.blk.2.ln2.weight" : view<4608xi8> %w_v_blk_2_ln2_bias = command.parameter %weights, "v.blk.2.ln2.bias" : view<4608xi8> %w_v_blk_2_ffn_up_weight = command.parameter %weights, "v.blk.2.ffn_up.weight" : view<9916416xi8> %w_v_blk_2_ffn_up_bias = command.parameter %weights, "v.blk.2.ffn_up.bias" : view<17216xi8> %w_v_blk_2_ffn_down_weight = command.parameter %weights, "v.blk.2.ffn_down.weight" : view<9916416xi8> %w_v_blk_2_ffn_down_bias = command.parameter %weights, "v.blk.2.ffn_down.bias" : view<4608xi8> %w_v_blk_3_ln1_weight = command.parameter %weights, "v.blk.3.ln1.weight" : view<4608xi8> %w_v_blk_3_ln1_bias = command.parameter %weights, "v.blk.3.ln1.bias" : view<4608xi8> %w_v_blk_3_attn_qkv_weight = command.parameter %weights, "v.blk.3.attn_qkv.weight" : view<7962624xi8> %w_v_blk_3_attn_qkv_bias = command.parameter %weights, "v.blk.3.attn_qkv.bias" : view<13824xi8> %w_v_blk_3_attn_out_weight = command.parameter %weights, "v.blk.3.attn_out.weight" : view<2654208xi8> %w_v_blk_3_attn_out_bias = command.parameter %weights, "v.blk.3.attn_out.bias" : view<4608xi8> %w_v_blk_3_ln2_weight = command.parameter %weights, "v.blk.3.ln2.weight" : view<4608xi8> %w_v_blk_3_ln2_bias = command.parameter %weights, "v.blk.3.ln2.bias" : view<4608xi8> %w_v_blk_3_ffn_up_weight = command.parameter %weights, "v.blk.3.ffn_up.weight" : view<9916416xi8> %w_v_blk_3_ffn_up_bias = command.parameter %weights, "v.blk.3.ffn_up.bias" : view<17216xi8> %w_v_blk_3_ffn_down_weight = command.parameter %weights, "v.blk.3.ffn_down.weight" : view<9916416xi8> %w_v_blk_3_ffn_down_bias = command.parameter %weights, "v.blk.3.ffn_down.bias" : view<4608xi8> %w_v_blk_4_ln1_weight = command.parameter %weights, "v.blk.4.ln1.weight" : view<4608xi8> %w_v_blk_4_ln1_bias = command.parameter %weights, "v.blk.4.ln1.bias" : view<4608xi8> %w_v_blk_4_attn_qkv_weight = command.parameter %weights, "v.blk.4.attn_qkv.weight" : view<7962624xi8> %w_v_blk_4_attn_qkv_bias = command.parameter %weights, "v.blk.4.attn_qkv.bias" : view<13824xi8> %w_v_blk_4_attn_out_weight = command.parameter %weights, "v.blk.4.attn_out.weight" : view<2654208xi8> %w_v_blk_4_attn_out_bias = command.parameter %weights, "v.blk.4.attn_out.bias" : view<4608xi8> %w_v_blk_4_ln2_weight = command.parameter %weights, "v.blk.4.ln2.weight" : view<4608xi8> %w_v_blk_4_ln2_bias = command.parameter %weights, "v.blk.4.ln2.bias" : view<4608xi8> %w_v_blk_4_ffn_up_weight = command.parameter %weights, "v.blk.4.ffn_up.weight" : view<9916416xi8> %w_v_blk_4_ffn_up_bias = command.parameter %weights, "v.blk.4.ffn_up.bias" : view<17216xi8> %w_v_blk_4_ffn_down_weight = command.parameter %weights, "v.blk.4.ffn_down.weight" : view<9916416xi8> %w_v_blk_4_ffn_down_bias = command.parameter %weights, "v.blk.4.ffn_down.bias" : view<4608xi8> %w_v_blk_5_ln1_weight = command.parameter %weights, "v.blk.5.ln1.weight" : view<4608xi8> %w_v_blk_5_ln1_bias = command.parameter %weights, "v.blk.5.ln1.bias" : view<4608xi8> %w_v_blk_5_attn_qkv_weight = command.parameter %weights, "v.blk.5.attn_qkv.weight" : view<7962624xi8> %w_v_blk_5_attn_qkv_bias = command.parameter %weights, "v.blk.5.attn_qkv.bias" : view<13824xi8> %w_v_blk_5_attn_out_weight = command.parameter %weights, "v.blk.5.attn_out.weight" : view<2654208xi8> %w_v_blk_5_attn_out_bias = command.parameter %weights, "v.blk.5.attn_out.bias" : view<4608xi8> %w_v_blk_5_ln2_weight = command.parameter %weights, "v.blk.5.ln2.weight" : view<4608xi8> %w_v_blk_5_ln2_bias = command.parameter %weights, "v.blk.5.ln2.bias" : view<4608xi8> %w_v_blk_5_ffn_up_weight = command.parameter %weights, "v.blk.5.ffn_up.weight" : view<9916416xi8> %w_v_blk_5_ffn_up_bias = command.parameter %weights, "v.blk.5.ffn_up.bias" : view<17216xi8> %w_v_blk_5_ffn_down_weight = command.parameter %weights, "v.blk.5.ffn_down.weight" : view<9916416xi8> %w_v_blk_5_ffn_down_bias = command.parameter %weights, "v.blk.5.ffn_down.bias" : view<4608xi8> %w_v_blk_6_ln1_weight = command.parameter %weights, "v.blk.6.ln1.weight" : view<4608xi8> %w_v_blk_6_ln1_bias = command.parameter %weights, "v.blk.6.ln1.bias" : view<4608xi8> %w_v_blk_6_attn_qkv_weight = command.parameter %weights, "v.blk.6.attn_qkv.weight" : view<7962624xi8> %w_v_blk_6_attn_qkv_bias = command.parameter %weights, "v.blk.6.attn_qkv.bias" : view<13824xi8> %w_v_blk_6_attn_out_weight = command.parameter %weights, "v.blk.6.attn_out.weight" : view<2654208xi8> %w_v_blk_6_attn_out_bias = command.parameter %weights, "v.blk.6.attn_out.bias" : view<4608xi8> %w_v_blk_6_ln2_weight = command.parameter %weights, "v.blk.6.ln2.weight" : view<4608xi8> %w_v_blk_6_ln2_bias = command.parameter %weights, "v.blk.6.ln2.bias" : view<4608xi8> %w_v_blk_6_ffn_up_weight = command.parameter %weights, "v.blk.6.ffn_up.weight" : view<9916416xi8> %w_v_blk_6_ffn_up_bias = command.parameter %weights, "v.blk.6.ffn_up.bias" : view<17216xi8> %w_v_blk_6_ffn_down_weight = command.parameter %weights, "v.blk.6.ffn_down.weight" : view<9916416xi8> %w_v_blk_6_ffn_down_bias = command.parameter %weights, "v.blk.6.ffn_down.bias" : view<4608xi8> %w_v_blk_7_ln1_weight = command.parameter %weights, "v.blk.7.ln1.weight" : view<4608xi8> %w_v_blk_7_ln1_bias = command.parameter %weights, "v.blk.7.ln1.bias" : view<4608xi8> %w_v_blk_7_attn_qkv_weight = command.parameter %weights, "v.blk.7.attn_qkv.weight" : view<7962624xi8> %w_v_blk_7_attn_qkv_bias = command.parameter %weights, "v.blk.7.attn_qkv.bias" : view<13824xi8> %w_v_blk_7_attn_out_weight = command.parameter %weights, "v.blk.7.attn_out.weight" : view<2654208xi8> %w_v_blk_7_attn_out_bias = command.parameter %weights, "v.blk.7.attn_out.bias" : view<4608xi8> %w_v_blk_7_ln2_weight = command.parameter %weights, "v.blk.7.ln2.weight" : view<4608xi8> %w_v_blk_7_ln2_bias = command.parameter %weights, "v.blk.7.ln2.bias" : view<4608xi8> %w_v_blk_7_ffn_up_weight = command.parameter %weights, "v.blk.7.ffn_up.weight" : view<9916416xi8> %w_v_blk_7_ffn_up_bias = command.parameter %weights, "v.blk.7.ffn_up.bias" : view<17216xi8> %w_v_blk_7_ffn_down_weight = command.parameter %weights, "v.blk.7.ffn_down.weight" : view<9916416xi8> %w_v_blk_7_ffn_down_bias = command.parameter %weights, "v.blk.7.ffn_down.bias" : view<4608xi8> %w_v_blk_8_ln1_weight = command.parameter %weights, "v.blk.8.ln1.weight" : view<4608xi8> %w_v_blk_8_ln1_bias = command.parameter %weights, "v.blk.8.ln1.bias" : view<4608xi8> %w_v_blk_8_attn_qkv_weight = command.parameter %weights, "v.blk.8.attn_qkv.weight" : view<7962624xi8> %w_v_blk_8_attn_qkv_bias = command.parameter %weights, "v.blk.8.attn_qkv.bias" : view<13824xi8> %w_v_blk_8_attn_out_weight = command.parameter %weights, "v.blk.8.attn_out.weight" : view<2654208xi8> %w_v_blk_8_attn_out_bias = command.parameter %weights, "v.blk.8.attn_out.bias" : view<4608xi8> %w_v_blk_8_ln2_weight = command.parameter %weights, "v.blk.8.ln2.weight" : view<4608xi8> %w_v_blk_8_ln2_bias = command.parameter %weights, "v.blk.8.ln2.bias" : view<4608xi8> %w_v_blk_8_ffn_up_weight = command.parameter %weights, "v.blk.8.ffn_up.weight" : view<9916416xi8> %w_v_blk_8_ffn_up_bias = command.parameter %weights, "v.blk.8.ffn_up.bias" : view<17216xi8> %w_v_blk_8_ffn_down_weight = command.parameter %weights, "v.blk.8.ffn_down.weight" : view<9916416xi8> %w_v_blk_8_ffn_down_bias = command.parameter %weights, "v.blk.8.ffn_down.bias" : view<4608xi8> %w_v_blk_9_ln1_weight = command.parameter %weights, "v.blk.9.ln1.weight" : view<4608xi8> %w_v_blk_9_ln1_bias = command.parameter %weights, "v.blk.9.ln1.bias" : view<4608xi8> %w_v_blk_9_attn_qkv_weight = command.parameter %weights, "v.blk.9.attn_qkv.weight" : view<7962624xi8> %w_v_blk_9_attn_qkv_bias = command.parameter %weights, "v.blk.9.attn_qkv.bias" : view<13824xi8> %w_v_blk_9_attn_out_weight = command.parameter %weights, "v.blk.9.attn_out.weight" : view<2654208xi8> %w_v_blk_9_attn_out_bias = command.parameter %weights, "v.blk.9.attn_out.bias" : view<4608xi8> %w_v_blk_9_ln2_weight = command.parameter %weights, "v.blk.9.ln2.weight" : view<4608xi8> %w_v_blk_9_ln2_bias = command.parameter %weights, "v.blk.9.ln2.bias" : view<4608xi8> %w_v_blk_9_ffn_up_weight = command.parameter %weights, "v.blk.9.ffn_up.weight" : view<9916416xi8> %w_v_blk_9_ffn_up_bias = command.parameter %weights, "v.blk.9.ffn_up.bias" : view<17216xi8> %w_v_blk_9_ffn_down_weight = command.parameter %weights, "v.blk.9.ffn_down.weight" : view<9916416xi8> %w_v_blk_9_ffn_down_bias = command.parameter %weights, "v.blk.9.ffn_down.bias" : view<4608xi8> %w_v_blk_10_ln1_weight = command.parameter %weights, "v.blk.10.ln1.weight" : view<4608xi8> %w_v_blk_10_ln1_bias = command.parameter %weights, "v.blk.10.ln1.bias" : view<4608xi8> %w_v_blk_10_attn_qkv_weight = command.parameter %weights, "v.blk.10.attn_qkv.weight" : view<7962624xi8> %w_v_blk_10_attn_qkv_bias = command.parameter %weights, "v.blk.10.attn_qkv.bias" : view<13824xi8> %w_v_blk_10_attn_out_weight = command.parameter %weights, "v.blk.10.attn_out.weight" : view<2654208xi8> %w_v_blk_10_attn_out_bias = command.parameter %weights, "v.blk.10.attn_out.bias" : view<4608xi8> %w_v_blk_10_ln2_weight = command.parameter %weights, "v.blk.10.ln2.weight" : view<4608xi8> %w_v_blk_10_ln2_bias = command.parameter %weights, "v.blk.10.ln2.bias" : view<4608xi8> %w_v_blk_10_ffn_up_weight = command.parameter %weights, "v.blk.10.ffn_up.weight" : view<9916416xi8> %w_v_blk_10_ffn_up_bias = command.parameter %weights, "v.blk.10.ffn_up.bias" : view<17216xi8> %w_v_blk_10_ffn_down_weight = command.parameter %weights, "v.blk.10.ffn_down.weight" : view<9916416xi8> %w_v_blk_10_ffn_down_bias = command.parameter %weights, "v.blk.10.ffn_down.bias" : view<4608xi8> %w_v_blk_11_ln1_weight = command.parameter %weights, "v.blk.11.ln1.weight" : view<4608xi8> %w_v_blk_11_ln1_bias = command.parameter %weights, "v.blk.11.ln1.bias" : view<4608xi8> %w_v_blk_11_attn_qkv_weight = command.parameter %weights, "v.blk.11.attn_qkv.weight" : view<7962624xi8> %w_v_blk_11_attn_qkv_bias = command.parameter %weights, "v.blk.11.attn_qkv.bias" : view<13824xi8> %w_v_blk_11_attn_out_weight = command.parameter %weights, "v.blk.11.attn_out.weight" : view<2654208xi8> %w_v_blk_11_attn_out_bias = command.parameter %weights, "v.blk.11.attn_out.bias" : view<4608xi8> %w_v_blk_11_ln2_weight = command.parameter %weights, "v.blk.11.ln2.weight" : view<4608xi8> %w_v_blk_11_ln2_bias = command.parameter %weights, "v.blk.11.ln2.bias" : view<4608xi8> %w_v_blk_11_ffn_up_weight = command.parameter %weights, "v.blk.11.ffn_up.weight" : view<9916416xi8> %w_v_blk_11_ffn_up_bias = command.parameter %weights, "v.blk.11.ffn_up.bias" : view<17216xi8> %w_v_blk_11_ffn_down_weight = command.parameter %weights, "v.blk.11.ffn_down.weight" : view<9916416xi8> %w_v_blk_11_ffn_down_bias = command.parameter %weights, "v.blk.11.ffn_down.bias" : view<4608xi8> %w_v_blk_12_ln1_weight = command.parameter %weights, "v.blk.12.ln1.weight" : view<4608xi8> %w_v_blk_12_ln1_bias = command.parameter %weights, "v.blk.12.ln1.bias" : view<4608xi8> %w_v_blk_12_attn_qkv_weight = command.parameter %weights, "v.blk.12.attn_qkv.weight" : view<7962624xi8> %w_v_blk_12_attn_qkv_bias = command.parameter %weights, "v.blk.12.attn_qkv.bias" : view<13824xi8> %w_v_blk_12_attn_out_weight = command.parameter %weights, "v.blk.12.attn_out.weight" : view<2654208xi8> %w_v_blk_12_attn_out_bias = command.parameter %weights, "v.blk.12.attn_out.bias" : view<4608xi8> %w_v_blk_12_ln2_weight = command.parameter %weights, "v.blk.12.ln2.weight" : view<4608xi8> %w_v_blk_12_ln2_bias = command.parameter %weights, "v.blk.12.ln2.bias" : view<4608xi8> %w_v_blk_12_ffn_up_weight = command.parameter %weights, "v.blk.12.ffn_up.weight" : view<9916416xi8> %w_v_blk_12_ffn_up_bias = command.parameter %weights, "v.blk.12.ffn_up.bias" : view<17216xi8> %w_v_blk_12_ffn_down_weight = command.parameter %weights, "v.blk.12.ffn_down.weight" : view<9916416xi8> %w_v_blk_12_ffn_down_bias = command.parameter %weights, "v.blk.12.ffn_down.bias" : view<4608xi8> %w_v_blk_13_ln1_weight = command.parameter %weights, "v.blk.13.ln1.weight" : view<4608xi8> %w_v_blk_13_ln1_bias = command.parameter %weights, "v.blk.13.ln1.bias" : view<4608xi8> %w_v_blk_13_attn_qkv_weight = command.parameter %weights, "v.blk.13.attn_qkv.weight" : view<7962624xi8> %w_v_blk_13_attn_qkv_bias = command.parameter %weights, "v.blk.13.attn_qkv.bias" : view<13824xi8> %w_v_blk_13_attn_out_weight = command.parameter %weights, "v.blk.13.attn_out.weight" : view<2654208xi8> %w_v_blk_13_attn_out_bias = command.parameter %weights, "v.blk.13.attn_out.bias" : view<4608xi8> %w_v_blk_13_ln2_weight = command.parameter %weights, "v.blk.13.ln2.weight" : view<4608xi8> %w_v_blk_13_ln2_bias = command.parameter %weights, "v.blk.13.ln2.bias" : view<4608xi8> %w_v_blk_13_ffn_up_weight = command.parameter %weights, "v.blk.13.ffn_up.weight" : view<9916416xi8> %w_v_blk_13_ffn_up_bias = command.parameter %weights, "v.blk.13.ffn_up.bias" : view<17216xi8> %w_v_blk_13_ffn_down_weight = command.parameter %weights, "v.blk.13.ffn_down.weight" : view<9916416xi8> %w_v_blk_13_ffn_down_bias = command.parameter %weights, "v.blk.13.ffn_down.bias" : view<4608xi8> %w_v_blk_14_ln1_weight = command.parameter %weights, "v.blk.14.ln1.weight" : view<4608xi8> %w_v_blk_14_ln1_bias = command.parameter %weights, "v.blk.14.ln1.bias" : view<4608xi8> %w_v_blk_14_attn_qkv_weight = command.parameter %weights, "v.blk.14.attn_qkv.weight" : view<7962624xi8> %w_v_blk_14_attn_qkv_bias = command.parameter %weights, "v.blk.14.attn_qkv.bias" : view<13824xi8> %w_v_blk_14_attn_out_weight = command.parameter %weights, "v.blk.14.attn_out.weight" : view<2654208xi8> %w_v_blk_14_attn_out_bias = command.parameter %weights, "v.blk.14.attn_out.bias" : view<4608xi8> %w_v_blk_14_ln2_weight = command.parameter %weights, "v.blk.14.ln2.weight" : view<4608xi8> %w_v_blk_14_ln2_bias = command.parameter %weights, "v.blk.14.ln2.bias" : view<4608xi8> %w_v_blk_14_ffn_up_weight = command.parameter %weights, "v.blk.14.ffn_up.weight" : view<9916416xi8> %w_v_blk_14_ffn_up_bias = command.parameter %weights, "v.blk.14.ffn_up.bias" : view<17216xi8> %w_v_blk_14_ffn_down_weight = command.parameter %weights, "v.blk.14.ffn_down.weight" : view<9916416xi8> %w_v_blk_14_ffn_down_bias = command.parameter %weights, "v.blk.14.ffn_down.bias" : view<4608xi8> %w_v_blk_15_ln1_weight = command.parameter %weights, "v.blk.15.ln1.weight" : view<4608xi8> %w_v_blk_15_ln1_bias = command.parameter %weights, "v.blk.15.ln1.bias" : view<4608xi8> %w_v_blk_15_attn_qkv_weight = command.parameter %weights, "v.blk.15.attn_qkv.weight" : view<7962624xi8> %w_v_blk_15_attn_qkv_bias = command.parameter %weights, "v.blk.15.attn_qkv.bias" : view<13824xi8> %w_v_blk_15_attn_out_weight = command.parameter %weights, "v.blk.15.attn_out.weight" : view<2654208xi8> %w_v_blk_15_attn_out_bias = command.parameter %weights, "v.blk.15.attn_out.bias" : view<4608xi8> %w_v_blk_15_ln2_weight = command.parameter %weights, "v.blk.15.ln2.weight" : view<4608xi8> %w_v_blk_15_ln2_bias = command.parameter %weights, "v.blk.15.ln2.bias" : view<4608xi8> %w_v_blk_15_ffn_up_weight = command.parameter %weights, "v.blk.15.ffn_up.weight" : view<9916416xi8> %w_v_blk_15_ffn_up_bias = command.parameter %weights, "v.blk.15.ffn_up.bias" : view<17216xi8> %w_v_blk_15_ffn_down_weight = command.parameter %weights, "v.blk.15.ffn_down.weight" : view<9916416xi8> %w_v_blk_15_ffn_down_bias = command.parameter %weights, "v.blk.15.ffn_down.bias" : view<4608xi8> %w_v_blk_16_ln1_weight = command.parameter %weights, "v.blk.16.ln1.weight" : view<4608xi8> %w_v_blk_16_ln1_bias = command.parameter %weights, "v.blk.16.ln1.bias" : view<4608xi8> %w_v_blk_16_attn_qkv_weight = command.parameter %weights, "v.blk.16.attn_qkv.weight" : view<7962624xi8> %w_v_blk_16_attn_qkv_bias = command.parameter %weights, "v.blk.16.attn_qkv.bias" : view<13824xi8> %w_v_blk_16_attn_out_weight = command.parameter %weights, "v.blk.16.attn_out.weight" : view<2654208xi8> %w_v_blk_16_attn_out_bias = command.parameter %weights, "v.blk.16.attn_out.bias" : view<4608xi8> %w_v_blk_16_ln2_weight = command.parameter %weights, "v.blk.16.ln2.weight" : view<4608xi8> %w_v_blk_16_ln2_bias = command.parameter %weights, "v.blk.16.ln2.bias" : view<4608xi8> %w_v_blk_16_ffn_up_weight = command.parameter %weights, "v.blk.16.ffn_up.weight" : view<9916416xi8> %w_v_blk_16_ffn_up_bias = command.parameter %weights, "v.blk.16.ffn_up.bias" : view<17216xi8> %w_v_blk_16_ffn_down_weight = command.parameter %weights, "v.blk.16.ffn_down.weight" : view<9916416xi8> %w_v_blk_16_ffn_down_bias = command.parameter %weights, "v.blk.16.ffn_down.bias" : view<4608xi8> %w_v_blk_17_ln1_weight = command.parameter %weights, "v.blk.17.ln1.weight" : view<4608xi8> %w_v_blk_17_ln1_bias = command.parameter %weights, "v.blk.17.ln1.bias" : view<4608xi8> %w_v_blk_17_attn_qkv_weight = command.parameter %weights, "v.blk.17.attn_qkv.weight" : view<7962624xi8> %w_v_blk_17_attn_qkv_bias = command.parameter %weights, "v.blk.17.attn_qkv.bias" : view<13824xi8> %w_v_blk_17_attn_out_weight = command.parameter %weights, "v.blk.17.attn_out.weight" : view<2654208xi8> %w_v_blk_17_attn_out_bias = command.parameter %weights, "v.blk.17.attn_out.bias" : view<4608xi8> %w_v_blk_17_ln2_weight = command.parameter %weights, "v.blk.17.ln2.weight" : view<4608xi8> %w_v_blk_17_ln2_bias = command.parameter %weights, "v.blk.17.ln2.bias" : view<4608xi8> %w_v_blk_17_ffn_up_weight = command.parameter %weights, "v.blk.17.ffn_up.weight" : view<9916416xi8> %w_v_blk_17_ffn_up_bias = command.parameter %weights, "v.blk.17.ffn_up.bias" : view<17216xi8> %w_v_blk_17_ffn_down_weight = command.parameter %weights, "v.blk.17.ffn_down.weight" : view<9916416xi8> %w_v_blk_17_ffn_down_bias = command.parameter %weights, "v.blk.17.ffn_down.bias" : view<4608xi8> %w_v_blk_18_ln1_weight = command.parameter %weights, "v.blk.18.ln1.weight" : view<4608xi8> %w_v_blk_18_ln1_bias = command.parameter %weights, "v.blk.18.ln1.bias" : view<4608xi8> %w_v_blk_18_attn_qkv_weight = command.parameter %weights, "v.blk.18.attn_qkv.weight" : view<7962624xi8> %w_v_blk_18_attn_qkv_bias = command.parameter %weights, "v.blk.18.attn_qkv.bias" : view<13824xi8> %w_v_blk_18_attn_out_weight = command.parameter %weights, "v.blk.18.attn_out.weight" : view<2654208xi8> %w_v_blk_18_attn_out_bias = command.parameter %weights, "v.blk.18.attn_out.bias" : view<4608xi8> %w_v_blk_18_ln2_weight = command.parameter %weights, "v.blk.18.ln2.weight" : view<4608xi8> %w_v_blk_18_ln2_bias = command.parameter %weights, "v.blk.18.ln2.bias" : view<4608xi8> %w_v_blk_18_ffn_up_weight = command.parameter %weights, "v.blk.18.ffn_up.weight" : view<9916416xi8> %w_v_blk_18_ffn_up_bias = command.parameter %weights, "v.blk.18.ffn_up.bias" : view<17216xi8> %w_v_blk_18_ffn_down_weight = command.parameter %weights, "v.blk.18.ffn_down.weight" : view<9916416xi8> %w_v_blk_18_ffn_down_bias = command.parameter %weights, "v.blk.18.ffn_down.bias" : view<4608xi8> %w_v_blk_19_ln1_weight = command.parameter %weights, "v.blk.19.ln1.weight" : view<4608xi8> %w_v_blk_19_ln1_bias = command.parameter %weights, "v.blk.19.ln1.bias" : view<4608xi8> %w_v_blk_19_attn_qkv_weight = command.parameter %weights, "v.blk.19.attn_qkv.weight" : view<7962624xi8> %w_v_blk_19_attn_qkv_bias = command.parameter %weights, "v.blk.19.attn_qkv.bias" : view<13824xi8> %w_v_blk_19_attn_out_weight = command.parameter %weights, "v.blk.19.attn_out.weight" : view<2654208xi8> %w_v_blk_19_attn_out_bias = command.parameter %weights, "v.blk.19.attn_out.bias" : view<4608xi8> %w_v_blk_19_ln2_weight = command.parameter %weights, "v.blk.19.ln2.weight" : view<4608xi8> %w_v_blk_19_ln2_bias = command.parameter %weights, "v.blk.19.ln2.bias" : view<4608xi8> %w_v_blk_19_ffn_up_weight = command.parameter %weights, "v.blk.19.ffn_up.weight" : view<9916416xi8> %w_v_blk_19_ffn_up_bias = command.parameter %weights, "v.blk.19.ffn_up.bias" : view<17216xi8> %w_v_blk_19_ffn_down_weight = command.parameter %weights, "v.blk.19.ffn_down.weight" : view<9916416xi8> %w_v_blk_19_ffn_down_bias = command.parameter %weights, "v.blk.19.ffn_down.bias" : view<4608xi8> %w_v_blk_20_ln1_weight = command.parameter %weights, "v.blk.20.ln1.weight" : view<4608xi8> %w_v_blk_20_ln1_bias = command.parameter %weights, "v.blk.20.ln1.bias" : view<4608xi8> %w_v_blk_20_attn_qkv_weight = command.parameter %weights, "v.blk.20.attn_qkv.weight" : view<7962624xi8> %w_v_blk_20_attn_qkv_bias = command.parameter %weights, "v.blk.20.attn_qkv.bias" : view<13824xi8> %w_v_blk_20_attn_out_weight = command.parameter %weights, "v.blk.20.attn_out.weight" : view<2654208xi8> %w_v_blk_20_attn_out_bias = command.parameter %weights, "v.blk.20.attn_out.bias" : view<4608xi8> %w_v_blk_20_ln2_weight = command.parameter %weights, "v.blk.20.ln2.weight" : view<4608xi8> %w_v_blk_20_ln2_bias = command.parameter %weights, "v.blk.20.ln2.bias" : view<4608xi8> %w_v_blk_20_ffn_up_weight = command.parameter %weights, "v.blk.20.ffn_up.weight" : view<9916416xi8> %w_v_blk_20_ffn_up_bias = command.parameter %weights, "v.blk.20.ffn_up.bias" : view<17216xi8> %w_v_blk_20_ffn_down_weight = command.parameter %weights, "v.blk.20.ffn_down.weight" : view<9916416xi8> %w_v_blk_20_ffn_down_bias = command.parameter %weights, "v.blk.20.ffn_down.bias" : view<4608xi8> %w_v_blk_21_ln1_weight = command.parameter %weights, "v.blk.21.ln1.weight" : view<4608xi8> %w_v_blk_21_ln1_bias = command.parameter %weights, "v.blk.21.ln1.bias" : view<4608xi8> %w_v_blk_21_attn_qkv_weight = command.parameter %weights, "v.blk.21.attn_qkv.weight" : view<7962624xi8> %w_v_blk_21_attn_qkv_bias = command.parameter %weights, "v.blk.21.attn_qkv.bias" : view<13824xi8> %w_v_blk_21_attn_out_weight = command.parameter %weights, "v.blk.21.attn_out.weight" : view<2654208xi8> %w_v_blk_21_attn_out_bias = command.parameter %weights, "v.blk.21.attn_out.bias" : view<4608xi8> %w_v_blk_21_ln2_weight = command.parameter %weights, "v.blk.21.ln2.weight" : view<4608xi8> %w_v_blk_21_ln2_bias = command.parameter %weights, "v.blk.21.ln2.bias" : view<4608xi8> %w_v_blk_21_ffn_up_weight = command.parameter %weights, "v.blk.21.ffn_up.weight" : view<9916416xi8> %w_v_blk_21_ffn_up_bias = command.parameter %weights, "v.blk.21.ffn_up.bias" : view<17216xi8> %w_v_blk_21_ffn_down_weight = command.parameter %weights, "v.blk.21.ffn_down.weight" : view<9916416xi8> %w_v_blk_21_ffn_down_bias = command.parameter %weights, "v.blk.21.ffn_down.bias" : view<4608xi8> %w_v_blk_22_ln1_weight = command.parameter %weights, "v.blk.22.ln1.weight" : view<4608xi8> %w_v_blk_22_ln1_bias = command.parameter %weights, "v.blk.22.ln1.bias" : view<4608xi8> %w_v_blk_22_attn_qkv_weight = command.parameter %weights, "v.blk.22.attn_qkv.weight" : view<7962624xi8> %w_v_blk_22_attn_qkv_bias = command.parameter %weights, "v.blk.22.attn_qkv.bias" : view<13824xi8> %w_v_blk_22_attn_out_weight = command.parameter %weights, "v.blk.22.attn_out.weight" : view<2654208xi8> %w_v_blk_22_attn_out_bias = command.parameter %weights, "v.blk.22.attn_out.bias" : view<4608xi8> %w_v_blk_22_ln2_weight = command.parameter %weights, "v.blk.22.ln2.weight" : view<4608xi8> %w_v_blk_22_ln2_bias = command.parameter %weights, "v.blk.22.ln2.bias" : view<4608xi8> %w_v_blk_22_ffn_up_weight = command.parameter %weights, "v.blk.22.ffn_up.weight" : view<9916416xi8> %w_v_blk_22_ffn_up_bias = command.parameter %weights, "v.blk.22.ffn_up.bias" : view<17216xi8> %w_v_blk_22_ffn_down_weight = command.parameter %weights, "v.blk.22.ffn_down.weight" : view<9916416xi8> %w_v_blk_22_ffn_down_bias = command.parameter %weights, "v.blk.22.ffn_down.bias" : view<4608xi8> %w_v_blk_23_ln1_weight = command.parameter %weights, "v.blk.23.ln1.weight" : view<4608xi8> %w_v_blk_23_ln1_bias = command.parameter %weights, "v.blk.23.ln1.bias" : view<4608xi8> %w_v_blk_23_attn_qkv_weight = command.parameter %weights, "v.blk.23.attn_qkv.weight" : view<7962624xi8> %w_v_blk_23_attn_qkv_bias = command.parameter %weights, "v.blk.23.attn_qkv.bias" : view<13824xi8> %w_v_blk_23_attn_out_weight = command.parameter %weights, "v.blk.23.attn_out.weight" : view<2654208xi8> %w_v_blk_23_attn_out_bias = command.parameter %weights, "v.blk.23.attn_out.bias" : view<4608xi8> %w_v_blk_23_ln2_weight = command.parameter %weights, "v.blk.23.ln2.weight" : view<4608xi8> %w_v_blk_23_ln2_bias = command.parameter %weights, "v.blk.23.ln2.bias" : view<4608xi8> %w_v_blk_23_ffn_up_weight = command.parameter %weights, "v.blk.23.ffn_up.weight" : view<9916416xi8> %w_v_blk_23_ffn_up_bias = command.parameter %weights, "v.blk.23.ffn_up.bias" : view<17216xi8> %w_v_blk_23_ffn_down_weight = command.parameter %weights, "v.blk.23.ffn_down.weight" : view<9916416xi8> %w_v_blk_23_ffn_down_bias = command.parameter %weights, "v.blk.23.ffn_down.bias" : view<4608xi8> %w_v_blk_24_ln1_weight = command.parameter %weights, "v.blk.24.ln1.weight" : view<4608xi8> %w_v_blk_24_ln1_bias = command.parameter %weights, "v.blk.24.ln1.bias" : view<4608xi8> %w_v_blk_24_attn_qkv_weight = command.parameter %weights, "v.blk.24.attn_qkv.weight" : view<7962624xi8> %w_v_blk_24_attn_qkv_bias = command.parameter %weights, "v.blk.24.attn_qkv.bias" : view<13824xi8> %w_v_blk_24_attn_out_weight = command.parameter %weights, "v.blk.24.attn_out.weight" : view<2654208xi8> %w_v_blk_24_attn_out_bias = command.parameter %weights, "v.blk.24.attn_out.bias" : view<4608xi8> %w_v_blk_24_ln2_weight = command.parameter %weights, "v.blk.24.ln2.weight" : view<4608xi8> %w_v_blk_24_ln2_bias = command.parameter %weights, "v.blk.24.ln2.bias" : view<4608xi8> %w_v_blk_24_ffn_up_weight = command.parameter %weights, "v.blk.24.ffn_up.weight" : view<9916416xi8> %w_v_blk_24_ffn_up_bias = command.parameter %weights, "v.blk.24.ffn_up.bias" : view<17216xi8> %w_v_blk_24_ffn_down_weight = command.parameter %weights, "v.blk.24.ffn_down.weight" : view<9916416xi8> %w_v_blk_24_ffn_down_bias = command.parameter %weights, "v.blk.24.ffn_down.bias" : view<4608xi8> %w_v_blk_25_ln1_weight = command.parameter %weights, "v.blk.25.ln1.weight" : view<4608xi8> %w_v_blk_25_ln1_bias = command.parameter %weights, "v.blk.25.ln1.bias" : view<4608xi8> %w_v_blk_25_attn_qkv_weight = command.parameter %weights, "v.blk.25.attn_qkv.weight" : view<7962624xi8> %w_v_blk_25_attn_qkv_bias = command.parameter %weights, "v.blk.25.attn_qkv.bias" : view<13824xi8> %w_v_blk_25_attn_out_weight = command.parameter %weights, "v.blk.25.attn_out.weight" : view<2654208xi8> %w_v_blk_25_attn_out_bias = command.parameter %weights, "v.blk.25.attn_out.bias" : view<4608xi8> %w_v_blk_25_ln2_weight = command.parameter %weights, "v.blk.25.ln2.weight" : view<4608xi8> %w_v_blk_25_ln2_bias = command.parameter %weights, "v.blk.25.ln2.bias" : view<4608xi8> %w_v_blk_25_ffn_up_weight = command.parameter %weights, "v.blk.25.ffn_up.weight" : view<9916416xi8> %w_v_blk_25_ffn_up_bias = command.parameter %weights, "v.blk.25.ffn_up.bias" : view<17216xi8> %w_v_blk_25_ffn_down_weight = command.parameter %weights, "v.blk.25.ffn_down.weight" : view<9916416xi8> %w_v_blk_25_ffn_down_bias = command.parameter %weights, "v.blk.25.ffn_down.bias" : view<4608xi8> %w_v_blk_26_ln1_weight = command.parameter %weights, "v.blk.26.ln1.weight" : view<4608xi8> %w_v_blk_26_ln1_bias = command.parameter %weights, "v.blk.26.ln1.bias" : view<4608xi8> %w_v_blk_26_attn_qkv_weight = command.parameter %weights, "v.blk.26.attn_qkv.weight" : view<7962624xi8> %w_v_blk_26_attn_qkv_bias = command.parameter %weights, "v.blk.26.attn_qkv.bias" : view<13824xi8> %w_v_blk_26_attn_out_weight = command.parameter %weights, "v.blk.26.attn_out.weight" : view<2654208xi8> %w_v_blk_26_attn_out_bias = command.parameter %weights, "v.blk.26.attn_out.bias" : view<4608xi8> %w_v_blk_26_ln2_weight = command.parameter %weights, "v.blk.26.ln2.weight" : view<4608xi8> %w_v_blk_26_ln2_bias = command.parameter %weights, "v.blk.26.ln2.bias" : view<4608xi8> %w_v_blk_26_ffn_up_weight = command.parameter %weights, "v.blk.26.ffn_up.weight" : view<9916416xi8> %w_v_blk_26_ffn_up_bias = command.parameter %weights, "v.blk.26.ffn_up.bias" : view<17216xi8> %w_v_blk_26_ffn_down_weight = command.parameter %weights, "v.blk.26.ffn_down.weight" : view<9916416xi8> %w_v_blk_26_ffn_down_bias = command.parameter %weights, "v.blk.26.ffn_down.bias" : view<4608xi8> %w_v_post_ln_weight = command.parameter %weights, "v.post_ln.weight" : view<4608xi8> %w_v_post_ln_bias = command.parameter %weights, "v.post_ln.bias" : view<4608xi8> %w_mm_0_weight = command.parameter %weights, "mm.0.weight" : view<42467328xi8> %w_mm_0_bias = command.parameter %weights, "mm.0.bias" : view<18432xi8> %w_mm_2_weight = command.parameter %weights, "mm.2.weight" : view<18874368xi8> %w_mm_2_bias = command.parameter %weights, "mm.2.bias" : view<8192xi8> kernel.launch @v_pixels_0p5_0p5_0p5[%n_pix](%n_pix, %pix, %pixf) : [index](index, buffer, buffer) kernel.launch @v_patch_16_1152_f16_block2_dual_bias[%T](%T, %gw, %img_w, %pixf, %w_v_patch_embd_weight, %w_v_patch_embd_weight_1, %w_v_patch_embd_bias, %h) : [index](index, index, index, buffer, view<1769472xi8>, view<1769472xi8>, view<4608xi8>, buffer) kernel.launch @v_pos_interp_1152_48_block2_ac[%T](%T, %gw, %gh, %w_v_position_embd_weight, %h) : [index](index, index, index, view<10616832xi8>, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_0_ln1_weight, %w_v_blk_0_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) %k1152 = index.constant 1152 : index kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) %k3456 = index.constant 3456 : index kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_0_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_0_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_0_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_0_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) %p_T_1152 = index.mul %T, %k1152 : index kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_0_ln2_weight, %w_v_blk_0_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) %k4304 = index.constant 4304 : index kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_0_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_0_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_0_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_0_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_1_ln1_weight, %w_v_blk_1_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_1_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_1_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_1_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_1_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_1_ln2_weight, %w_v_blk_1_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_1_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_1_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_1_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_1_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_2_ln1_weight, %w_v_blk_2_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_2_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_2_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_2_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_2_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_2_ln2_weight, %w_v_blk_2_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_2_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_2_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_2_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_2_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_3_ln1_weight, %w_v_blk_3_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_3_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_3_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_3_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_3_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_3_ln2_weight, %w_v_blk_3_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_3_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_3_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_3_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_3_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_4_ln1_weight, %w_v_blk_4_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_4_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_4_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_4_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_4_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_4_ln2_weight, %w_v_blk_4_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_4_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_4_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_4_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_4_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_5_ln1_weight, %w_v_blk_5_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_5_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_5_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_5_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_5_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_5_ln2_weight, %w_v_blk_5_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_5_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_5_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_5_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_5_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_6_ln1_weight, %w_v_blk_6_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_6_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_6_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_6_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_6_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_6_ln2_weight, %w_v_blk_6_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_6_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_6_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_6_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_6_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_7_ln1_weight, %w_v_blk_7_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_7_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_7_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_7_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_7_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_7_ln2_weight, %w_v_blk_7_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_7_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_7_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_7_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_7_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_8_ln1_weight, %w_v_blk_8_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_8_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_8_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_8_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_8_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_8_ln2_weight, %w_v_blk_8_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_8_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_8_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_8_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_8_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_9_ln1_weight, %w_v_blk_9_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_9_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_9_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_9_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_9_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_9_ln2_weight, %w_v_blk_9_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_9_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_9_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_9_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_9_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_10_ln1_weight, %w_v_blk_10_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_10_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_10_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_10_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_10_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_10_ln2_weight, %w_v_blk_10_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_10_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_10_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_10_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_10_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_11_ln1_weight, %w_v_blk_11_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_11_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_11_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_11_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_11_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_11_ln2_weight, %w_v_blk_11_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_11_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_11_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_11_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_11_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_12_ln1_weight, %w_v_blk_12_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_12_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_12_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_12_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_12_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_12_ln2_weight, %w_v_blk_12_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_12_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_12_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_12_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_12_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_13_ln1_weight, %w_v_blk_13_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_13_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_13_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_13_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_13_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_13_ln2_weight, %w_v_blk_13_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_13_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_13_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_13_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_13_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_14_ln1_weight, %w_v_blk_14_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_14_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_14_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_14_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_14_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_14_ln2_weight, %w_v_blk_14_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_14_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_14_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_14_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_14_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_15_ln1_weight, %w_v_blk_15_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_15_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_15_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_15_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_15_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_15_ln2_weight, %w_v_blk_15_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_15_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_15_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_15_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_15_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_16_ln1_weight, %w_v_blk_16_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_16_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_16_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_16_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_16_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_16_ln2_weight, %w_v_blk_16_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_16_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_16_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_16_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_16_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_17_ln1_weight, %w_v_blk_17_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_17_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_17_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_17_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_17_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_17_ln2_weight, %w_v_blk_17_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_17_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_17_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_17_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_17_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_18_ln1_weight, %w_v_blk_18_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_18_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_18_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_18_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_18_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_18_ln2_weight, %w_v_blk_18_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_18_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_18_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_18_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_18_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_19_ln1_weight, %w_v_blk_19_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_19_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_19_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_19_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_19_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_19_ln2_weight, %w_v_blk_19_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_19_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_19_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_19_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_19_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_20_ln1_weight, %w_v_blk_20_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_20_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_20_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_20_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_20_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_20_ln2_weight, %w_v_blk_20_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_20_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_20_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_20_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_20_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_21_ln1_weight, %w_v_blk_21_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_21_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_21_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_21_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_21_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_21_ln2_weight, %w_v_blk_21_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_21_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_21_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_21_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_21_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_22_ln1_weight, %w_v_blk_22_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_22_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_22_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_22_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_22_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_22_ln2_weight, %w_v_blk_22_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_22_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_22_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_22_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_22_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_23_ln1_weight, %w_v_blk_23_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_23_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_23_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_23_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_23_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_23_ln2_weight, %w_v_blk_23_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_23_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_23_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_23_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_23_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_24_ln1_weight, %w_v_blk_24_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_24_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_24_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_24_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_24_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_24_ln2_weight, %w_v_blk_24_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_24_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_24_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_24_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_24_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_25_ln1_weight, %w_v_blk_25_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_25_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_25_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_25_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_25_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_25_ln2_weight, %w_v_blk_25_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_25_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_25_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_25_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_25_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_26_ln1_weight, %w_v_blk_26_ln1_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x3456[%Tp, %k3456](%Tp, %k1152, %k3456, %x16, %sc, %w_v_blk_26_attn_qkv_weight, %q) : [index, index](index, index, index, buffer, buffer, view<7962624xi8>, buffer) kernel.launch @v_bias_none_3456[%T](%T, %q, %w_v_blk_26_attn_qkv_bias, %q) : [index](index, buffer, view<13824xi8>, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o0_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_rope_qwen_vision_72x16_s3456_o1152_block2[%T](%T, %gw, %q) : [index](index, index, buffer) kernel.launch @v_attn_72x16_s3456_0_1152_2304[%T](%T, %gw, %q, %q, %q, %o) : [index](index, index, buffer, buffer, buffer, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %o, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_1152x1152[%Tp, %k1152](%Tp, %k1152, %k1152, %x16, %sc, %w_v_blk_26_attn_out_weight, %a) : [index, index](index, index, index, buffer, buffer, view<2654208xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_26_attn_out_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_blk_26_ln2_weight, %w_v_blk_26_ln2_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k1152, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_1152x4304[%Tp, %k4304](%Tp, %k1152, %k4304, %x16, %sc, %w_v_blk_26_ffn_up_weight, %f) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4304[%T](%T, %f, %w_v_blk_26_ffn_up_bias, %f) : [index](index, buffer, view<17216xi8>, buffer) kernel.launch @v_row_scale_f16[%Tp](%Tp, %k4304, %f, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_any_f16_4304x1152[%Tp, %k1152](%Tp, %k4304, %k1152, %x16, %sc, %w_v_blk_26_ffn_down_weight, %a) : [index, index](index, index, index, buffer, buffer, view<9916416xi8>, buffer) kernel.launch @v_bias_none_1152[%T](%T, %a, %w_v_blk_26_ffn_down_bias, %a) : [index](index, buffer, view<4608xi8>, buffer) kernel.launch @v_ew_add[%p_T_1152](%p_T_1152, %h, %a, %h) : [index](index, buffer, buffer, buffer) kernel.launch @v_norm_ln_1152[%T](%T, %h, %w_v_post_ln_weight, %w_v_post_ln_bias, %x) : [index](index, buffer, view<4608xi8>, view<4608xi8>, buffer) %k4608 = index.constant 4608 : index kernel.launch @v_row_scale_f16[%Mp](%Mp, %k4608, %x, %sc, %x16) : [index](index, index, buffer, buffer, buffer) kernel.launch @v_gemm_f16_store_4608x4608[%Mp, %k4608](%Mp, %k4608, %k4608, %x16, %sc, %w_mm_0_weight, %m1) : [index, index](index, index, index, buffer, buffer, view<42467328xi8>, buffer) kernel.launch @v_bias_gelu_tanh_4608[%M](%M, %m1, %w_mm_0_bias, %m1) : [index](index, buffer, view<18432xi8>, buffer) kernel.launch @v_row_scale_f16[%Mp](%Mp, %k4608, %m1, %sc, %x16) : [index](index, index, buffer, buffer, buffer) %k2048 = index.constant 2048 : index kernel.launch @v_gemm_f16_store_4608x2048[%Mp, %k2048](%Mp, %k4608, %k2048, %x16, %sc, %w_mm_2_weight, %out) : [index, index](index, index, index, buffer, buffer, view<18874368xi8>, buffer) kernel.launch @v_bias_none_2048[%M](%M, %out, %w_mm_2_bias, %out) : [index](index, buffer, view<8192xi8>, buffer) %out_rows = index.constant 2048 : index %final_n = index.mul %M, %out_rows : index kernel.launch @v_copy[%final_n](%final_n, %out, %output) : [index](index, buffer, buffer) command.return }