jeremyfowers's picture
Promote from loom-fun-staging/InternScience-Sage-A1.5-loom@b3be1155be73
f666a0a verified
Raw History Blame Contribute Delete
17 kB
config.def @llm.card = false : i1
// logits:begin
// A declaration: the one definition (default false) is in llm.loom; a second config.def of the key links ambiguously (LINK/MATERIALIZE).
config.decl @llm.logits : i1
// logits:end
func.decl import("input") @require(%condition: i32, %diagnostic: buffer)
func.decl import("json") @members(%json: buffer) -> (buffer, i32, i64)
func.decl import("prepare") @stage(%root: buffer, %tag: i64) -> (i32)
func.decl import("prepare") @config_i64(%stage: i32, %key: buffer, %value: i64)
func.decl import("prepare") @source(%directory: buffer, %path: buffer) -> (i32)
func.decl import("prepare") @parameter(%stage: i32, %binding: i32, %input: i32, %source: i32, %policy: buffer)
func.decl import("prepare") @source_scope(%scope: buffer) -> (i32)
// "model"
global.rodata.def @model_scope = bytes("6d6f64656c")
// "llm_advance"
global.rodata.def @root = bytes("6c6c6d5f616476616e6365")
// "llm.context", "llm.spans", "llm.tokens", "llm.pool"
global.rodata.def @context_key = bytes("6c6c6d2e636f6e74657874")
global.rodata.def @spans_key = bytes("6c6c6d2e7370616e73")
global.rodata.def @tokens_key = bytes("6c6c6d2e746f6b656e73")
global.rodata.def @pool_key = bytes("6c6c6d2e706f6f6c")
// logits:begin
// "llm.logit_slots": the rows of the logits region a stage may write (min(spans, rows)), set only with the capability.
global.rodata.def @logit_slots_key = bytes("6c6c6d2e6c6f6769745f736c6f7473")
// logits:end
// "llm_verify", "llm_mtp_warm", "llm_mtp_propose", "llm.width", "llm.gated", "llm.epoch_spans", "llm.epoch_tokens"
global.rodata.def @verify_root = bytes("6c6c6d5f766572696679")
global.rodata.def @warm_root = bytes("6c6c6d5f6d74705f7761726d")
global.rodata.def @propose_root = bytes("6c6c6d5f6d74705f70726f706f7365")
global.rodata.def @width_key = bytes("6c6c6d2e7769647468")
global.rodata.def @gated_key = bytes("6c6c6d2e6761746564")
global.rodata.def @epoch_spans_key = bytes("6c6c6d2e65706f63685f7370616e73")
global.rodata.def @epoch_tokens_key = bytes("6c6c6d2e65706f63685f746f6b656e73")
// "draft"
global.rodata.def @draft_scope = bytes("6472616674")
// "weights.loom"
global.rodata.def @policy = bytes("776569676874732e6c6f6f6d")
// "<|im_end|>" is the service terminal marker.
global.rodata.def @terminal = bytes("3c7c696d5f656e647c3e")
global.rodata.def @empty = bytes("")
// "llm requires pooled epochs without MTP, at most 128 shapes and a context of at most 262144"
global.rodata.def @unsupported = bytes("6c6c6d20726571756972657320706f6f6c65642065706f63687320776974686f7574204d54502c206174206d6f7374203132382073686170657320616e64206120636f6e74657874206f66206174206d6f737420323632313434")
func.decl @llm_storage(%maximum: i64, %span_capacity: i64, %context: i64, %pool: i64, %rows: i64, %pins: i64, %drafting: i64) -> (buffer, buffer, buffer, buffer, buffer)
// Control state layout: [0, 8) unused, [8, 88) two transfer records,
// [88, 128) layout recipe, [128, 640) stage per packing shape, then a 16-byte
// sampling record per resident row.
func.def public @prepare(%prefill: i64, %context: i64, %pool: i64, %rows: i64, %pins: i64, %draft_depth: i32, %shapes: buffer, %shape_count: i64, %weights: buffer, %options: buffer) -> (i64, i32, buffer, buffer, buffer, buffer, buffer, buffer, buffer) {
%c0_source = scalar.constant 0 : i32
%origin = index.constant 0 : offset
%c0 = index.constant 0 : index
%c1 = index.constant 1 : index
%c0_i32 = scalar.constant 0 : i32
%c0_i64 = scalar.constant 0 : i64
%c64_i64 = scalar.constant 128 : i64
%c4096_i64 = scalar.constant 262144 : i64
%pooled = scalar.cmpi ne, %pool, %c0_i64 : i64
%packed = scalar.cmpi ne, %shape_count, %c0_i64 : i64
%few_shapes = scalar.cmpi ule, %shape_count, %c64_i64 : i64
%short_context = scalar.cmpi ule, %context, %c4096_i64 : i64
%no_draft = scalar.cmpi eq, %draft_depth, %c0_i32 : i32
%has_epochs = scalar.andi %pooled, %packed : i1
%c7_i32 = scalar.constant 7 : i32
%depth_ok = scalar.cmpi ule, %draft_depth, %c7_i32 : i32
%mode_ok = scalar.andi %has_epochs, %depth_ok : i1
%limits_ok = scalar.andi %few_shapes, %short_context : i1
%supported = scalar.andi %mode_ok, %limits_ok : i1
%supported32 = scalar.extui %supported : i1 to i32
%error = global.load @unsupported : buffer
func.call @require(%supported32, %error) : (i32, buffer)
%empty = global.load @empty : buffer
%policy = global.load @policy : buffer
%card = config.get @llm.card : i1
%checkpoint = scf.if %card -> (i32) {
%scope = global.load @model_scope : buffer
%scoped = func.call @source_scope(%scope) : (buffer) -> (i32)
scf.yield %scoped : i32
} else {
%direct = func.call @source(%empty, %weights) : (buffer, buffer) -> (i32)
scf.yield %direct : i32
}
%bounded_count = scalar.assume %shape_count [range(%shape_count, 1, 128)] : i64
%count = index.cast %bounded_count : i64 to index
%c4_off = index.constant 4 : offset
%stage_slots = index.constant 128 : index
%stage_bytes = index.scale %stage_slots, %c4_off : index, offset -> offset
%header_bytes = index.constant 128 : offset
%control_bytes = index.constant 2048 : offset
// Then one 16-byte sampling record per resident row (control.loom @bind_request).
%bounded_rows = scalar.assume %rows [range(%rows, 1, 1048576)] : i64
%row_count = index.cast %bounded_rows : i64 to index
%record_bytes = index.constant 16 : offset
%sampling_bytes = index.scale %row_count, %record_bytes : index, offset -> offset
%state_bytes = index.add %control_bytes, %sampling_bytes : offset
%state = buffer.alloca<private> align(8) %state_bytes : buffer
buffer.fill %c0_i32, %state[%origin], %state_bytes : i32
%stage_view = buffer.view %state[%header_bytes] : buffer -> view<[%count]xi32>
%verify_at = index.constant 640 : offset
%warm_at = index.constant 1152 : offset
%verify_view = buffer.view %state[%verify_at] : buffer -> view<[%count]xi32>
%warm_view = buffer.view %state[%warm_at] : buffer -> view<[%count]xi32>
%drafting = scalar.cmpi ne, %draft_depth, %c0_i32 : i32
%depth64 = scalar.extui %draft_depth : i32 to i64
%c1_w = scalar.constant 1 : i64
%width = scalar.addi %depth64, %c1_w : i64
%verify_root = global.load @verify_root : buffer
%warm_root = global.load @warm_root : buffer
%width_key = global.load @width_key : buffer
%gated_key = global.load @gated_key : buffer
%gated_off = scalar.constant 0 : i64
%gated_on = scalar.constant 1 : i64
%root = global.load @root : buffer
%context_key = global.load @context_key : buffer
%spans_key = global.load @spans_key : buffer
%tokens_key = global.load @tokens_key : buffer
%pool_key = global.load @pool_key : buffer
// logits:begin
%logits_cfg = config.get @llm.logits : i1
%logit_slots_key = global.load @logit_slots_key : buffer
// logits:end
%shape_view = buffer.view %shapes[%origin] : buffer -> view<[%count]x2xi64>
%maximum, %span_capacity = scf.for %i = [%c0 to %count step %c1](%previous = %c0_i64 : i64, %previous_spans = %c0_i64 : i64) -> (i64, i64) {
%tokens = view.load %shape_view[%i, 0] : view<[%count]x2xi64> -> i64
%spans = view.load %shape_view[%i, 1] : view<[%count]x2xi64> -> i64
%tag = index.cast %i : index to i64
%stage = func.call @stage(%root, %tag) : (buffer, i64) -> (i32)
func.call @config_i64(%stage, %context_key, %context) : (i32, buffer, i64)
func.call @config_i64(%stage, %pool_key, %pool) : (i32, buffer, i64)
func.call @config_i64(%stage, %tokens_key, %tokens) : (i32, buffer, i64)
func.call @config_i64(%stage, %spans_key, %spans) : (i32, buffer, i64)
// logits:begin
scf.if %logits_cfg {
%logit_slots = scalar.minsi %spans, %rows : i64
func.call @config_i64(%stage, %logit_slots_key, %logit_slots) : (i32, buffer, i64)
}
// logits:end
func.call @parameter(%stage, %c0_i32, %c0_source, %checkpoint, %policy) : (i32, i32, i32, i32, buffer)
view.store %stage, %stage_view[%i] : i32, view<[%count]xi32>
%larger = scalar.cmpi ugt, %tokens, %previous : i64
%next = scf.select %larger, %tokens, %previous : i64
%more_spans = scalar.cmpi ugt, %spans, %previous_spans : i64
%next_spans = scf.select %more_spans, %spans, %previous_spans : i64
scf.yield %next, %next_spans : i64, i64
}
// Verify and catch-up stages per shape, declared after every advance stage (the first shape_count
// stages are the shared target stages), then proposal stages for cohorts of 1, 2, 4, ... spans.
scf.if %drafting {
%propose_root = global.load @propose_root : buffer
%epoch_spans_key = global.load @epoch_spans_key : buffer
%epoch_tokens_key = global.load @epoch_tokens_key : buffer
%span32 = scalar.trunci %span_capacity : i64 to i32
%one32 = scalar.constant 1 : i32
%bits32 = scalar.constant 32 : i32
%below = scalar.subi %span32, %one32 : i32
%leading = scalar.ctlzi %below : i32
%last_level = scalar.subi %bits32, %leading : i32
%levels_all = scalar.addi %last_level, %one32 : i32
%level_cap = scalar.constant 14 : i32
%levels32 = scalar.minui %levels_all, %level_cap : i32
%levels_b = scalar.assume %levels32 [range(%levels32, 1, 14)] : i32
%levels = index.cast %levels_b : i32 to index
%c1_64 = scalar.constant 1 : i64
%first_propose = scf.for %level = [%c0 to %levels step %c1](%first = %c0_i32 : i32) -> (i32) {
%shift = index.cast %level : index to i64
%cap = scalar.shli %c1_64, %shift : i64
%pstage = func.call @stage(%propose_root, %shift) : (buffer, i64) -> (i32)
func.call @config_i64(%pstage, %context_key, %context) : (i32, buffer, i64)
func.call @config_i64(%pstage, %pool_key, %pool) : (i32, buffer, i64)
// A DFlash block is [anchor, <mask> x depth] (llama.cpp's n_max + 1 tokens), capped at the trained block size:
// 4 streams at depth 3 propose 16 tokens (one int8 matvec pass per drafter weight) instead of 64 (GEMM).
%block_full = scalar.constant 1 : i64
%block_tokens = scalar.minui %block_full, %width : i64
%cap_tokens = scalar.muli %cap, %block_tokens : i64
func.call @config_i64(%pstage, %tokens_key, %cap_tokens) : (i32, buffer, i64)
func.call @config_i64(%pstage, %spans_key, %cap) : (i32, buffer, i64)
func.call @config_i64(%pstage, %width_key, %width) : (i32, buffer, i64)
func.call @config_i64(%pstage, %epoch_spans_key, %span_capacity) : (i32, buffer, i64)
func.call @config_i64(%pstage, %epoch_tokens_key, %maximum) : (i32, buffer, i64)
%pstage_b0 = scalar.constant 0 : i32
func.call @parameter(%pstage, %pstage_b0, %c0_source, %checkpoint, %policy) : (i32, i32, i32, i32, buffer)
%pstage_b1 = scalar.constant 1 : i32
func.call @parameter(%pstage, %pstage_b1, %c0_source, %checkpoint, %policy) : (i32, i32, i32, i32, buffer)
%pstage_b2 = scalar.constant 2 : i32
func.call @parameter(%pstage, %pstage_b2, %c0_source, %checkpoint, %policy) : (i32, i32, i32, i32, buffer)
%initial = index.cmp eq, %level, %c0 : index
%next_first = scf.select %initial, %pstage, %first : i32
scf.yield %next_first : i32
}
scf.for %i = [%c0 to %count step %c1] {
%tokens = view.load %shape_view[%i, 0] : view<[%count]x2xi64> -> i64
%spans = view.load %shape_view[%i, 1] : view<[%count]x2xi64> -> i64
%tag = index.cast %i : index to i64
%vstage = func.call @stage(%verify_root, %tag) : (buffer, i64) -> (i32)
func.call @config_i64(%vstage, %context_key, %context) : (i32, buffer, i64)
func.call @config_i64(%vstage, %pool_key, %pool) : (i32, buffer, i64)
func.call @config_i64(%vstage, %tokens_key, %tokens) : (i32, buffer, i64)
func.call @config_i64(%vstage, %spans_key, %spans) : (i32, buffer, i64)
func.call @config_i64(%vstage, %width_key, %width) : (i32, buffer, i64)
func.call @config_i64(%vstage, %gated_key, %gated_off) : (i32, buffer, i64)
func.call @parameter(%vstage, %c0_i32, %c0_source, %checkpoint, %policy) : (i32, i32, i32, i32, buffer)
view.store %vstage, %verify_view[%i] : i32, view<[%count]xi32>
%wstage = func.call @stage(%warm_root, %tag) : (buffer, i64) -> (i32)
func.call @config_i64(%wstage, %context_key, %context) : (i32, buffer, i64)
func.call @config_i64(%wstage, %pool_key, %pool) : (i32, buffer, i64)
func.call @config_i64(%wstage, %tokens_key, %tokens) : (i32, buffer, i64)
func.call @config_i64(%wstage, %spans_key, %spans) : (i32, buffer, i64)
%wstage_b0 = scalar.constant 0 : i32
func.call @parameter(%wstage, %wstage_b0, %c0_source, %checkpoint, %policy) : (i32, i32, i32, i32, buffer)
%wstage_b1 = scalar.constant 1 : i32
func.call @parameter(%wstage, %wstage_b1, %c0_source, %checkpoint, %policy) : (i32, i32, i32, i32, buffer)
%wstage_b2 = scalar.constant 2 : i32
func.call @parameter(%wstage, %wstage_b2, %c0_source, %checkpoint, %policy) : (i32, i32, i32, i32, buffer)
%wstage_b3 = scalar.constant 3 : i32
func.call @parameter(%wstage, %wstage_b3, %c0_source, %checkpoint, %policy) : (i32, i32, i32, i32, buffer)
%wstage_b4 = scalar.constant 4 : i32
func.call @parameter(%wstage, %wstage_b4, %c0_source, %checkpoint, %policy) : (i32, i32, i32, i32, buffer)
%wstage_b5 = scalar.constant 5 : i32
func.call @parameter(%wstage, %wstage_b5, %c0_source, %checkpoint, %policy) : (i32, i32, i32, i32, buffer)
%wstage_b6 = scalar.constant 6 : i32
func.call @parameter(%wstage, %wstage_b6, %c0_source, %checkpoint, %policy) : (i32, i32, i32, i32, buffer)
%wstage_b7 = scalar.constant 7 : i32
func.call @parameter(%wstage, %wstage_b7, %c0_source, %checkpoint, %policy) : (i32, i32, i32, i32, buffer)
%wstage_b8 = scalar.constant 8 : i32
func.call @parameter(%wstage, %wstage_b8, %c0_source, %checkpoint, %policy) : (i32, i32, i32, i32, buffer)
view.store %wstage, %warm_view[%i] : i32, view<[%count]xi32>
%same_shape = scalar.cmpi eq, %tokens, %spans : i64
%gate_slots = scalar.constant 32 : i64
%few_spans = scalar.cmpi ult, %spans, %gate_slots : i64
%gate_ok = scalar.andi %same_shape, %few_spans : i1
scf.if %gate_ok {
%gate_at = index.constant 1700 : offset
%gate_view = buffer.view %state[%gate_at] : buffer -> view<32xi32>
%gate_slot = index.cast %spans : i64 to index
%gate_i32 = index.cast %i : index to i32
%gate_one = scalar.constant 1 : i32
%gate_idx1 = scalar.addi %gate_i32, %gate_one : i32
%gtag_base = scalar.constant 256 : i64
%gtag = scalar.addi %tag, %gtag_base : i64
%gvstage = func.call @stage(%verify_root, %gtag) : (buffer, i64) -> (i32)
func.call @config_i64(%gvstage, %context_key, %context) : (i32, buffer, i64)
func.call @config_i64(%gvstage, %pool_key, %pool) : (i32, buffer, i64)
func.call @config_i64(%gvstage, %tokens_key, %tokens) : (i32, buffer, i64)
func.call @config_i64(%gvstage, %spans_key, %spans) : (i32, buffer, i64)
func.call @config_i64(%gvstage, %width_key, %width) : (i32, buffer, i64)
func.call @config_i64(%gvstage, %gated_key, %gated_on) : (i32, buffer, i64)
func.call @parameter(%gvstage, %c0_i32, %c0_source, %checkpoint, %policy) : (i32, i32, i32, i32, buffer)
%gate_shift = scalar.constant 8 : i32
%gv_high = scalar.shli %gvstage, %gate_shift : i32
%gate_tag = scalar.ori %gate_idx1, %gv_high : i32
view.store %gate_tag, %gate_view[%gate_slot] : i32, view<32xi32>
}
}
%draft_at = index.constant 1664 : offset
%draft_view = buffer.view %state[%draft_at] : buffer -> view<3xi32>
view.store %first_propose, %draft_view[0] : i32, view<3xi32>
view.store %levels32, %draft_view[1] : i32, view<3xi32>
%kind32 = scalar.constant 1 : i32
view.store %kind32, %draft_view[2] : i32, view<3xi32>
}
%smaller = scalar.cmpi ult, %maximum, %prefill : i64
%effective = scf.select %smaller, %maximum, %prefill : i64
%layout_origin = index.constant 88 : offset
%layout = buffer.view %state[%layout_origin] : buffer -> view<5xi64>
view.store %maximum, %layout[0] : i64, view<5xi64>
view.store %span_capacity, %layout[1] : i64, view<5xi64>
view.store %context, %layout[2] : i64, view<5xi64>
view.store %pool, %layout[3] : i64, view<5xi64>
view.store %pins, %layout[4] : i64, view<5xi64>
%stage_count32 = index.cast %count : index to i32
%drafting64 = scalar.extui %drafting : i1 to i64
%allocations, %views, %origins, %draft, %geometry = func.call @llm_storage(%maximum, %span_capacity, %context, %pool, %rows, %pins, %drafting64) : (i64, i64, i64, i64, i64, i64, i64) -> (buffer, buffer, buffer, buffer, buffer)
%terminal = global.load @terminal : buffer
func.return %effective, %stage_count32, %state, %terminal, %allocations, %views, %origins, %draft, %geometry : i64, i32, buffer, buffer, buffer, buffer, buffer, buffer, buffer
}