File size: 17,040 Bytes
85f86bf
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
# Independent float references; runGrid tests never use the production CPU VM.
import nim "std/[os, tables, json, math, random, strutils]"
import nim "vm_interp"
import nim "backend"
import nim "quant"
import "../../other/formers.n/src/formers/ir/fir_parse.bybylang"
import "../../other/formers.n/src/formers/ir/fir_exec.bybylang"
import "../../other/formers.n/src/formers/ir/fir_graph.bybylang"
import "../../other/formers.n/src/formers/ir/fir_graph_kernels.bybylang"
import "../../other/formers.n/src/formers/io/safetensors.bybylang"
import "../../other/formers.n/src/formers/arch/detect.bybylang"

function fgClose(actual, expected: openArray[float32], label: string)
    doAssert actual.len == expected.len, label & " length"
    for i in 0 ..< actual.len:
        if classify(actual[i]) in {fcNan, fcInf, fcNegInf} or abs(actual[i]-expected[i]) > 1e-4'f32:
            raise newException(ValueError, label & " index=" & $i & ": " & $actual[i] & " != " & $expected[i])

function fgRandom(rng: var Rand, count: int, scale: float32 = 0.3'f32): seq[float32]
    result = newSeq[float32](count)
    for value in result.mitems:
        value = (rng.rand(2.0'f32)-1)*scale

function refNorm(x, gamma, beta: seq[float32], eps: float32): seq[float32]
    var mean = 0'f32
    for value in x:
        mean += value
    mean /= float32(x.len)
    var variance = 0'f32
    for value in x:
        variance += (value-mean)*(value-mean)
    let inverse = 1'f32/sqrt(variance/float32(x.len)+eps)
    for i, value in x:
        result.add((value-mean)*inverse*gamma[i]+beta[i])

function refAdd(a, b: seq[float32]): seq[float32]
    for i in 0 ..< a.len:
        result.add(a[i]+b[i])

function refGelu(x: seq[float32]): seq[float32]
    # Independent tanh form of gelu_new (equivalent to the sigmoid builder).
    for value in x:
        result.add(0.5'f32*value*(1'f32+tanh(sqrt(2'f32/PI.float32)*(value+0.044715'f32*value*value*value))))

function refLinear(x, w, bias: seq[float32]): seq[float32]
    result = bias
    for j in 0 ..< bias.len:
        for i in 0 ..< x.len:
            result[j] += x[i]*w[i*bias.len+j]

function refAttention(qkv: seq[float32], keys, values: var seq[float32], position, hidden, heads: int): seq[float32]
    for i in 0 ..< hidden:
        keys[position*hidden+i] = qkv[hidden+i]
        values[position*hidden+i] = qkv[2*hidden+i]
    let width = hidden div heads
    result = newSeq[float32](hidden)
    for head in 0 ..< heads:
        var scores = newSeq[float32](position+1)
        for t in 0 .. position:
            for d in 0 ..< width:
                scores[t] += qkv[head*width+d]*keys[t*hidden+head*width+d]
            scores[t] /= sqrt(float32(width))
        var peak = scores[0]
        for score in scores:
            peak = max(peak, score)
        var total = 0'f32
        for score in scores.mitems:
            score = exp(score-peak)
            total += score
        for t in 0 .. position:
            for d in 0 ..< width:
                result[head*width+d] += scores[t]/total*values[t*hidden+head*width+d]

function checkMicro(graph: FirProgram)
    let registry = firDefaultRegistry()
    let bindings = firGraphBindings(graph)
    firValidate(graph, registry, bindings)
    var rng = initRand(7293)
    for node in graph.nodes:
        if node.kind != "micro":
            continue
        let ins = node.inputs.splitWhitespace()
        let output = node.outputs.splitWhitespace()[0]
        var memory = fgRandom(rng, bindings.memoryFloats, 4'f32)
        var expected: seq[float32]
        if node.op == "g2.attn_decode":
            var keys = fgRandom(rng, graph.buffers["key_cache"].elements)
            var values = fgRandom(rng, keys.len)
            for position in 0 ..< graph.symbols["positions"]:
                let qkv = fgRandom(rng, graph.buffers["qkv"].elements, 3'f32)
                for i, value in qkv:
                    memory[bindings.memory["qkv"]+i] = value
                memory[bindings.memory["position"]] = float32(position)
                # Future cache entries are poison; causal decode must not read them.
                for i in position*graph.symbols["hidden"] ..< keys.len:
                    memory[bindings.memory["key_cache"]+i] = 999'f32
                    memory[bindings.memory["value_cache"]+i] = -999'f32
                expected = refAttention(qkv, keys, values, position, graph.symbols["hidden"], graph.symbols["nheads"])
                let kernel = firMicroKernel(node, graph, registry, bindings)
                runGrid(kernel.program, memory, kernel.threads, kernel.tgSize)
                let offset = bindings.memory["attn"]
                fgClose(memory[offset ..< offset+expected.len], expected, "attention")
                let count = (position+1)*graph.symbols["hidden"]
                for cache in ["key_cache", "value_cache"]:
                    let base = bindings.memory[cache]
                    let reference = if cache == "key_cache": keys else: values
                    fgClose(memory[base ..< base+count], reference[0 ..< count], cache)
                for head in 0 ..< graph.symbols["nheads"]:
                    var total = 0'f32
                    for t in 0 .. position:
                        total += memory[bindings.memory["scores"]+head*graph.symbols["positions"]+t]
                    fgClose(@[total], @[1'f32], "softmax sum")
            continue
        let xoff = bindings.memory[ins[0]]
        let count = graph.buffers[ins[0]].elements
        let x = memory[xoff ..< xoff+count]
        if node.op == "g2.layernorm":
            let goff = bindings.memory[ins[1]]
            let boff = bindings.memory[ins[2]]
            expected = refNorm(x, memory[goff ..< goff+count], memory[boff ..< boff+count], graph.epsilon)
        elif node.op == "g2.gelu_new":
            expected = refGelu(x)
        else:
            let boff = bindings.memory[ins[1]]
            expected = refAdd(x, memory[boff ..< boff+count])
        let kernel = firMicroKernel(node, graph, registry, bindings)
        runGrid(kernel.program, memory, kernel.threads, kernel.tgSize)
        let offset = bindings.memory[output]
        fgClose(memory[offset ..< offset+expected.len], expected, node.name)
    echo "PASS GPT-2 all micro nodes through runGrid: layernorm, gelu_new, bias/residual, causal attention/softmax and KV (tol 1e-4)"

function refForward(graph: FirProgram, weights: Table[string, seq[float32]], tokens: seq[int]): seq[float32]
    let hidden = graph.symbols["hidden"]
    let capacity = graph.symbols["positions"]
    let heads = graph.symbols["nheads"]
    var keys = newSeq[seq[float32]](graph.symbols["layers"])
    var values = newSeq[seq[float32]](keys.len)
    for i in 0 ..< keys.len:
        keys[i] = newSeq[float32](capacity*hidden)
        values[i] = newSeq[float32](capacity*hidden)
    for position, token in tokens:
        let wte = weights["wte.weight"]
        let wpe = weights["wpe.weight"]
        var x = refAdd(wte[token*hidden ..< (token+1)*hidden], wpe[position*hidden ..< (position+1)*hidden])
        for layer in 0 ..< keys.len:
            let p = "h." & $layer & "."
            let norm1 = refNorm(x, weights[p & "ln_1.weight"], weights[p & "ln_1.bias"], graph.epsilon)
            let qkv = refLinear(norm1, weights[p & "attn.c_attn.weight"], weights[p & "attn.c_attn.bias"])
            let attn = refAttention(qkv, keys[layer], values[layer], position, hidden, heads)
            x = refAdd(x, refLinear(attn, weights[p & "attn.c_proj.weight"], weights[p & "attn.c_proj.bias"]))
            let norm2 = refNorm(x, weights[p & "ln_2.weight"], weights[p & "ln_2.bias"], graph.epsilon)
            let fc = refLinear(norm2, weights[p & "mlp.c_fc.weight"], weights[p & "mlp.c_fc.bias"])
            x = refAdd(x, refLinear(refGelu(fc), weights[p & "mlp.c_proj.weight"], weights[p & "mlp.c_proj.bias"]))
        x = refNorm(x, weights["ln_f.weight"], weights["ln_f.bias"], graph.epsilon)
        result = newSeq[float32](graph.symbols["vocab"])
        for token in 0 ..< result.len:
            for i in 0 ..< hidden:
                result[token] += x[i]*wte[token*hidden+i]

function writeSafe(path: string, weights: Table[string, seq[float32]], shapes: Table[string, seq[int]])
    var header = newJObject()
    var offset = 0
    var names: seq[string]
    for name, data in weights:
        names.add(name)
        header[name] = %*{"dtype": "F32", "shape": shapes[name], "data_offsets": [offset, offset+data.len*4]}
        offset += data.len*4
    var text = $header
    while text.len mod 8 != 0:
        text.add(' ')
    var f = open(path, fmWrite)
    defer: f.close()
    var length = uint64(text.len)
    discard f.writeBuffer(addr length, 8)
    f.write(text)
    for name in names:
        let data = weights[name]
        discard f.writeBuffer(unsafeAddr data[0], data.len*4)

function checkFull(graph: FirProgram, root: string)
    var weights = initTable[string, seq[float32]]()
    var shapes = initTable[string, seq[int]]()
    var rng = initRand(8343)
    for name, buffer in graph.buffers:
        if buffer.kind != "weight":
            continue
        for layer in 0 ..< (if buffer.tensorName.contains("{i}"): graph.symbols["layers"] else: 1):
            let tensor = buffer.tensorName.replace("{i}", $layer)
            weights[tensor] = fgRandom(rng, buffer.elements)
            shapes[tensor] = buffer.dims
    let safePath = root / "model.safetensors"
    writeSafe(safePath, weights, shapes)
    var model = firLoadGraph(firF32Weights(graph), safePath, newBackend("cpu"))
    defer: firFreeGraph(model)
    var tokens: seq[int]
    for i in 0 ..< graph.symbols["positions"]:
        let token = rng.rand(graph.symbols["vocab"]-1)
        tokens.add(token)
        let actual = firGraphStep(model, token)
        let expected = refForward(graph, weights, tokens)
        fgClose(actual, expected, "full graph at position " & $i)
    doAssert model.dispatched > 0
    doAssert model.sessions == tokens.len and model.logitsReads == tokens.len
    firGraphReset(model)
    fgClose(firGraphStep(model, tokens[0]), refForward(graph, weights, @[tokens[0]]), "fp32 reset")
    # Independently dequantize the framework's packed Q4, transpose back to HF
    # layout and compare the whole resident graph with scalar float formulas.
    var rounded = weights
    for name, buffer in graph.buffers:
        if buffer.dtype != "q4":
            continue
        let k = buffer.dims[0]
        let n = buffer.dims[1]
        for layer in 0 ..< graph.symbols["layers"]:
            let tensor = buffer.tensorName.replace("{i}", $layer)
            var transposed = newSeq[float32](buffer.elements)
            for j in 0 ..< n:
                for i in 0 ..< k:
                    transposed[j*k+i] = weights[tensor][i*n+j]
            let packed = quantizeInt4AsymmetricGrouped(transposed, @[n, k])
            doAssert packed.kind == qkInt4Asymmetric and packed.groupSize == 32
            let restored = dequantizeTensor(packed)
            var hf = newSeq[float32](buffer.elements)
            for j in 0 ..< n:
                for i in 0 ..< k:
                    hf[i*n+j] = restored[j*k+i]
            rounded[tensor] = hf
    var resident = firNewGraph(graph, weights, newBackend("cpu"), firDefaultRegistry())
    defer: firFreeGraph(resident)
    doAssert resident.q4.len == 4*graph.symbols["layers"]
    doAssert resident.fp32.len == 2, "tied head must reuse wte"
    var prefix: seq[int]
    for token in tokens:
        prefix.add(token)
        fgClose(firGraphStep(resident, token), refForward(graph, rounded, prefix), "Q4 resident graph")
    doAssert resident.sessions == tokens.len and resident.logitsReads == tokens.len
    firGraphReset(resident)
    fgClose(firGraphStep(resident, tokens[0]), refForward(graph, rounded, @[tokens[0]]), "Q4 reset")
    echo "PASS resident Q4 full graph vs independently dequantized scalar reference; transpose, tied head, one session/read per token, reset (tol 1e-4)"
    for i in 1 ..< tokens.len:
        discard firGraphStep(model, tokens[i])
    var rejected = false
    try:
        discard firGraphStep(model, 0)
    except ValueError:
        rejected = true
    doAssert rejected, "context overflow accepted"
    # Header-only routing, then malformed shape and truncated payload rejection.
    writeFile(root / "model.fir", readFile(graph.sourcePath))
    writeFile(root / "tokenizer.json", "{}")
    var opts = fmDefaultOptions()
    opts.modelsDir = root
    opts.backend = "cpu"
    let detected = fmDetect(opts)
    doAssert detected.supported and detected.route == frFir, detected.errors.join("; ")
    let header = fmReadSafeHeader(safePath)
    rejected = false
    try:
        discard fmLoadSafeF32(safePath, header, "wte.weight", @[1, 1])
    except ValueError:
        rejected = true
    doAssert rejected, "wrong safetensors shape accepted"
    let valid = readFile(safePath)
    writeFile(safePath, valid[0 ..< valid.len-4])
    rejected = false
    try:
        discard fmReadSafeHeader(safePath)
    except ValueError:
        rejected = true
    doAssert rejected, "truncated safetensors accepted"
    echo "PASS GPT-2 tiny full graph through production CPU VM/backend vs independent reference, all positions (tol 1e-4); safetensors, detection, context guards"

function checkQuant()
    var rng = initRand(425)
    let rows = 3
    let cols = 35 # Odd row width and a partial second group.
    var input = fgRandom(rng, rows*cols, 5'f32)
    for c in 0 ..< cols:
        input[c] = 2.5'f32 # Constant groups must retain their nonzero value.
    let packed = quantizeInt4AsymmetricGrouped(input, @[rows, cols])
    doAssert packed.data.len == rows*((cols+1) div 2)
    doAssert packed.scale.len == rows*2 and packed.zero_point.len == rows*2
    let restored = dequantizeTensor(packed)
    for r in 0 ..< rows:
        for c in 0 ..< cols:
            let sc = packed.scale[r*2+c div 32]
            doAssert abs(restored[r*cols+c]-input[r*cols+c]) <= sc*0.501'f32+1e-6'f32
    fgClose(restored[0 ..< cols], input[0 ..< cols], "constant Q4 groups")
    let ctx = newBackend("cpu")
    var w = beResidentUploadQ4(ctx, packed.data, packed.scale, packed.zero_point, cols, rows, 32, 2)
    defer: beResidentFreeQ4(w)
    let x = fgRandom(rng, cols)
    var expected = newSeq[float32](rows)
    for r in 0 ..< rows:
        for c in 0 ..< cols:
            expected[r] += x[c]*restored[r*cols+c]
    fgClose(beResidentLinearQ4(ctx, x, w, 1), expected, "odd grouped resident GEMV")
    echo "PASS grouped asymmetric Q4: odd row padding, partial groups, nonzero constants and resident GEMV (tol 1e-4)"

function checkBadKernels(root: string)
    # Kernel blocks in a model file must fail loudly: no silent wrong programs.
    let source = readFile("models/gpt2-125m/model.fir")
    var cases: seq[(string, string, string)]
    cases.add(("unknown mnemonic", "  mulconst 1 1 1/in0.size", "  bogus 1 1"))
    cases.add(("register out of range", "  add 1 1 2\n  ustoref32 1 0 @out0", "  add 1 1 40\n  ustoref32 1 0 @out0"))
    cases.add(("unclosed loop", "  forend\n  mulconst 1 1 1/in0.size", "  mulconst 1 1 1/in0.size"))
    cases.add(("unknown name", "mulconst 2 2 0.044715", "mulconst 2 2 not_a_symbol"))
    cases.add(("wrong buffer size", "in1:1,", "in1:2,"))
    cases.add(("wrong arity", "kernel g2.add in=2 out=1", "kernel g2.add in=3 out=1"))
    cases.add(("wrong argument count", "  mul 2 1 1\n  mul 2 2 1", "  mul 2 1\n  mul 2 2 1"))
    for (label, find, replacement) in cases:
        if not source.contains(find):
            raise newException(ValueError, "bad-kernel fixture does not match model.fir: " & label)
        let bad = root / "bad.fir"
        writeFile(bad, source.replace(find, replacement))
        var rejected = false
        try:
            let graph = firParse(bad)
            firValidate(graph, firDefaultRegistry(), firGraphBindings(graph))
        except ValueError:
            rejected = true
        if not rejected:
            raise newException(ValueError, "bad kernel accepted: " & label)
    echo "PASS FIR kernel blocks reject unknown mnemonics, bad registers, unbalanced loops, unknown names, wrong sizes and arities"

checkQuant()
let root = getTempDir() / ("fir_graph_" & $getCurrentProcessId())
createDir(root)
try:
    let path = root / "config.json"
    for (hidden, heads) in [(8, 2), (15, 3)]:
        let cfg = %*{"model_type": "gpt2", "n_embd": hidden, "n_head": heads, "n_layer": 2, "n_positions": 5, "vocab_size": 19, "layer_norm_epsilon": 0.0003, "activation_function": "gelu_new"}
        writeFile(path, $cfg)
        let graph = firParse("models/gpt2-125m/model.fir", path)
        checkMicro(graph)
        checkFull(graph, root)
    let large = firParse("models/gpt2-125m/model.fir")
    firValidate(large, firDefaultRegistry(), firGraphBindings(large))
    echo "PASS GPT-2 real config FIR validation, every micro program built without weights"
    checkBadKernels(root)
finally:
    removeDir(root)