NOT-OMEGA commited on
Commit
750cd73
·
verified ·
1 Parent(s): 2f36b88

Update inference.cpp

Browse files
Files changed (1) hide show
  1. inference.cpp +271 -523
inference.cpp CHANGED
@@ -1,7 +1,13 @@
1
  /*
2
  * ============================================================
3
- * NanoMind — Optimized GPT-2 Inference Engine v3.0 (Batched)
4
  * ============================================================
 
 
 
 
 
 
5
  */
6
  #include <iostream>
7
  #include <cstdio>
@@ -12,609 +18,351 @@
12
  #include <algorithm>
13
  #include <string>
14
  #include <vector>
15
- #include <unordered_map>
16
  #include <unordered_set>
17
- #include <immintrin.h> // AVX2 + FMA
 
 
 
 
 
18
  #ifdef _OPENMP
19
  #include <omp.h>
20
  #endif
21
  #ifdef _WIN32
22
  #include <windows.h>
23
- static double get_ms() {
24
- LARGE_INTEGER f, c;
25
- QueryPerformanceFrequency(&f); QueryPerformanceCounter(&c);
26
- return (double)c.QuadPart / f.QuadPart * 1000.0;
27
- }
28
  #else
29
  #include <sys/time.h>
30
- static double get_ms() {
31
- struct timeval tv; gettimeofday(&tv, NULL);
32
- return tv.tv_sec * 1000.0 + tv.tv_usec / 1000.0;
33
- }
34
  #endif
35
 
 
36
  static thread_local uint64_t tl_rng_state = 0;
37
  static inline float rng_float() {
38
- if (!tl_rng_state) {
39
- tl_rng_state = (uint64_t)get_ms() ^ (uint64_t)(uintptr_t)&tl_rng_state;
40
- if (!tl_rng_state) tl_rng_state = 0xDEADBEEFCAFEBABEULL;
41
- }
42
- tl_rng_state ^= tl_rng_state << 13;
43
- tl_rng_state ^= tl_rng_state >> 7;
44
- tl_rng_state ^= tl_rng_state << 17;
45
  return (float)(tl_rng_state >> 11) * (1.0f / (float)(1ULL << 53));
46
  }
47
 
48
- struct Config {
49
- int n_layer, n_head, n_embd, block_size, vocab_size;
50
- };
51
  struct Weights {
52
- float *wte, *wpe;
53
- float **ln1_w, **ln1_b;
54
- float **c_attn_w, **c_attn_b;
55
- float **c_proj_w, **c_proj_b;
56
- float **ln2_w, **ln2_b;
57
- float **fc_w, **fc_b;
58
- float **mlp_proj_w, **mlp_proj_b;
59
- float *ln_f_w, *ln_f_b;
60
- float *lm_head_w;
61
  };
62
- static Config cfg;
63
  static Weights W;
64
  static float* g_data = nullptr;
65
 
66
- struct Session {
67
- float* k_cache = nullptr;
68
- float* v_cache = nullptr;
69
- int pos = 0;
70
- double last_use = 0.0;
71
- };
72
- static const int MAX_SESSIONS = 50;
73
- static std::unordered_map<std::string, Session> g_sessions;
74
-
75
- static float *g_x, *g_buf, *g_qkv, *g_attn_buf;
76
- static float *g_ff, *g_logits, *g_tmp_out;
77
- static std::pair<float,int>* g_topk_pairs = nullptr;
78
-
79
- // ── BATCH ARRAYS ──
80
  const int MAX_BATCH = 16;
81
  static float *g_x_b[MAX_BATCH], *g_buf_b[MAX_BATCH], *g_qkv_b[MAX_BATCH];
82
  static float *g_attn_buf_b[MAX_BATCH], *g_ff_b[MAX_BATCH], *g_tmp_out_b[MAX_BATCH], *g_logits_b[MAX_BATCH];
 
83
 
84
- // ── Math Kernels ──
85
- static void layer_norm(float* out, const float* x, const float* w, const float* b, int N) {
86
- __m256 vsum = _mm256_setzero_ps();
87
- for (int i = 0; i < N; i += 8) vsum = _mm256_add_ps(vsum, _mm256_loadu_ps(x + i));
88
- float tmp[8]; _mm256_storeu_ps(tmp, vsum);
89
- float mean = (tmp[0]+tmp[1]+tmp[2]+tmp[3]+tmp[4]+tmp[5]+tmp[6]+tmp[7]) / (float)N;
90
-
91
- __m256 vmean = _mm256_set1_ps(mean);
92
- __m256 vvar = _mm256_setzero_ps();
93
- for (int i = 0; i < N; i += 8) {
94
- __m256 d = _mm256_sub_ps(_mm256_loadu_ps(x + i), vmean);
95
- vvar = _mm256_fmadd_ps(d, d, vvar);
96
- }
97
- _mm256_storeu_ps(tmp, vvar);
98
- float var = (tmp[0]+tmp[1]+tmp[2]+tmp[3]+tmp[4]+tmp[5]+tmp[6]+tmp[7]) / (float)N;
99
- __m256 vsc = _mm256_set1_ps(1.f / sqrtf(var + 1e-5f));
100
-
101
- for (int i = 0; i < N; i += 8) {
102
- __m256 d = _mm256_sub_ps(_mm256_loadu_ps(x + i), vmean);
103
- __m256 norm = _mm256_mul_ps(d, vsc);
104
- __m256 result = _mm256_fmadd_ps(norm, _mm256_loadu_ps(w + i), _mm256_loadu_ps(b + i));
105
- _mm256_storeu_ps(out + i, result);
106
- }
107
- float sc = 1.f / sqrtf(var + 1e-5f);
108
- for (int i = (N & ~7); i < N; i++) out[i] = (x[i] - mean) * sc * w[i] + b[i];
109
- }
110
 
111
- static void matmul_vec(float* __restrict__ out, const float* __restrict__ mat, const float* __restrict__ x, int M, int K) {
112
- const int OMP_THRESHOLD = 64;
113
- if (M < OMP_THRESHOLD) {
114
- for (int i = 0; i < M; i++) {
115
- const float* row = mat + (long long)i * K;
116
- __m256 acc = _mm256_setzero_ps();
117
- int j = 0;
118
- for (; j <= K-8; j += 8) acc = _mm256_fmadd_ps(_mm256_loadu_ps(row+j), _mm256_loadu_ps(x+j), acc);
119
- float t[8]; _mm256_storeu_ps(t, acc);
120
- float s = t[0]+t[1]+t[2]+t[3]+t[4]+t[5]+t[6]+t[7];
121
- for (; j < K; j++) s += row[j] * x[j];
122
- out[i] = s;
123
- }
124
- return;
125
- }
126
- #pragma omp parallel for schedule(static)
127
- for (int i = 0; i < M; i++) {
128
- const float* row = mat + (long long)i * K;
129
- __m256 acc = _mm256_setzero_ps();
130
- int j = 0;
131
- for (; j <= K-8; j += 8) acc = _mm256_fmadd_ps(_mm256_loadu_ps(row+j), _mm256_loadu_ps(x+j), acc);
132
- float t[8]; _mm256_storeu_ps(t, acc);
133
- float s = t[0]+t[1]+t[2]+t[3]+t[4]+t[5]+t[6]+t[7];
134
- for (; j < K; j++) s += row[j] * x[j];
135
- out[i] = s;
136
- }
137
- }
138
 
139
- static inline float dot_avx2(const float* __restrict__ a, const float* __restrict__ b, int n) {
140
- __m256 acc = _mm256_setzero_ps();
141
- int i = 0;
142
- for (; i <= n-8; i += 8) acc = _mm256_fmadd_ps(_mm256_loadu_ps(a+i), _mm256_loadu_ps(b+i), acc);
143
- float tmp[8]; _mm256_storeu_ps(tmp, acc);
144
- float s = tmp[0]+tmp[1]+tmp[2]+tmp[3]+tmp[4]+tmp[5]+tmp[6]+tmp[7];
145
- for (; i < n; i++) s += a[i]*b[i];
146
- return s;
147
  }
148
 
149
- static inline void weighted_acc_avx2(float* __restrict__ out, const float* __restrict__ v, float w, int n) {
150
- __m256 wv = _mm256_set1_ps(w);
151
- int i = 0;
152
- for (; i <= n-8; i += 8)
153
- _mm256_storeu_ps(out+i, _mm256_fmadd_ps(wv, _mm256_loadu_ps(v+i), _mm256_loadu_ps(out+i)));
154
- for (; i < n; i++) out[i] += w * v[i];
155
  }
156
 
157
- static inline void add_bias(float* x, const float* b, int N) {
158
- #pragma omp parallel for
159
- for (int i = 0; i < N; i++) x[i] += b[i];
160
  }
161
 
162
- static inline void residual_add(float* x, const float* y, int N) {
163
- #pragma omp parallel for
164
- for (int i = 0; i < N; i++) x[i] += y[i];
 
 
 
 
 
 
 
165
  }
166
-
167
- static void gelu_inplace(float* x, int N) {
168
- const __m256 scale = _mm256_set1_ps(12102203.0f);
169
- const __m256 vbias = _mm256_set1_ps(1064807168.0f);
170
- const __m256 neg_c = _mm256_set1_ps(-1.702f);
171
- const __m256 vone = _mm256_set1_ps(1.0f);
172
- const __m256 vtwo = _mm256_set1_ps(2.0f);
173
- const __m256 vlo = _mm256_set1_ps(-88.0f);
174
- const __m256 vhi = _mm256_set1_ps(88.0f);
175
- int i = 0;
176
- for (; i <= N-8; i += 8) {
177
- __m256 v = _mm256_loadu_ps(x + i);
178
- __m256 t = _mm256_mul_ps(neg_c, v);
179
- t = _mm256_max_ps(t, vlo); t = _mm256_min_ps(t, vhi);
180
- __m256i ti = _mm256_cvttps_epi32(_mm256_fmadd_ps(t, scale, vbias));
181
- __m256 et = _mm256_castsi256_ps(ti);
182
- __m256 denom = _mm256_add_ps(vone, et);
183
- __m256 r = _mm256_rcp_ps(denom);
184
- r = _mm256_mul_ps(r, _mm256_fnmadd_ps(denom, r, vtwo));
185
- _mm256_storeu_ps(x + i, _mm256_mul_ps(v, r));
186
  }
187
- for (; i < N; i++) x[i] = x[i] / (1.0f + expf(-1.702f * x[i]));
188
  }
189
-
190
- static void softmax_inplace(float* x, int N) {
191
- float mx = x[0];
192
- for (int i = 1; i < N; i++) if (x[i] > mx) mx = x[i];
193
- float s = 0.f;
194
- for (int i = 0; i < N; i++) { x[i] = expf(x[i]-mx); s += x[i]; }
195
- for (int i = 0; i < N; i++) x[i] /= s;
196
  }
197
-
198
- static void forward(int token_id, int pos, float* k_cache, float* v_cache) {
199
- const int C = cfg.n_embd;
200
- const int H = cfg.n_head;
201
- const int hs = C / H;
202
-
203
- float* te = W.wte + (long long)token_id * C;
204
- float* pe = W.wpe + (long long)pos * C;
205
- #pragma omp parallel for
206
- for (int i = 0; i < C; i++) g_x[i] = te[i] + pe[i];
207
-
208
- for (int l = 0; l < cfg.n_layer; l++) {
209
- layer_norm(g_buf, g_x, W.ln1_w[l], W.ln1_b[l], C);
210
- matmul_vec(g_qkv, W.c_attn_w[l], g_buf, 3*C, C);
211
- add_bias(g_qkv, W.c_attn_b[l], 3*C);
212
-
213
- float* q = g_qkv;
214
- float* k = g_qkv + C;
215
- float* v = g_qkv + 2*C;
216
- float* kc = k_cache + (long long)l * cfg.block_size * C;
217
- float* vc = v_cache + (long long)l * cfg.block_size * C;
218
- memcpy(kc + (long long)pos*C, k, C*sizeof(float));
219
- memcpy(vc + (long long)pos*C, v, C*sizeof(float));
220
-
221
- #pragma omp parallel for schedule(static)
222
- for (int h = 0; h < H; h++) {
223
- float* qh = q + h*hs;
224
- float scale = 1.f / sqrtf((float)hs);
225
- float* attn = g_attn_buf + h*cfg.block_size;
226
- for (int t = 0; t <= pos; t++) {
227
- float* kh = kc + (long long)t*C + h*hs;
228
- attn[t] = dot_avx2(qh, kh, hs) * scale;
229
- }
230
- softmax_inplace(attn, pos+1);
231
- float* oh = g_buf + h*hs;
232
- memset(oh, 0, hs*sizeof(float));
233
- for (int t = 0; t <= pos; t++) {
234
- float* vh = vc + (long long)t*C + h*hs;
235
- weighted_acc_avx2(oh, vh, attn[t], hs);
236
- }
237
- }
238
-
239
- matmul_vec(g_tmp_out, W.c_proj_w[l], g_buf, C, C);
240
- add_bias(g_tmp_out, W.c_proj_b[l], C);
241
- residual_add(g_x, g_tmp_out, C);
242
-
243
- layer_norm(g_buf, g_x, W.ln2_w[l], W.ln2_b[l], C);
244
- matmul_vec(g_ff, W.fc_w[l], g_buf, 4*C, C);
245
- add_bias(g_ff, W.fc_b[l], 4*C);
246
- gelu_inplace(g_ff, 4*C);
247
- matmul_vec(g_tmp_out, W.mlp_proj_w[l], g_ff, C, 4*C);
248
- add_bias(g_tmp_out, W.mlp_proj_b[l], C);
249
- residual_add(g_x, g_tmp_out, C);
250
- }
251
-
252
- layer_norm(g_buf, g_x, W.ln_f_w, W.ln_f_b, C);
253
- matmul_vec(g_logits, W.lm_head_w, g_buf, cfg.vocab_size, C);
254
  }
 
 
255
 
256
- static void map_weights(float* data, long wbytes) {
257
- float* p = data;
258
- float* end = data + wbytes / sizeof(float);
259
- const int C = cfg.n_embd, L = cfg.n_layer;
260
- W.wte = p; p += (long long)cfg.vocab_size * C;
261
- W.wpe = p; p += (long long)cfg.block_size * C;
262
- #define ARR(f) W.f = (float**)malloc(L*sizeof(float*))
263
- ARR(ln1_w); ARR(ln1_b); ARR(c_attn_w); ARR(c_attn_b);
264
- ARR(c_proj_w); ARR(c_proj_b); ARR(ln2_w); ARR(ln2_b);
265
- ARR(fc_w); ARR(fc_b); ARR(mlp_proj_w); ARR(mlp_proj_b);
266
- #undef ARR
267
- for (int l = 0; l < L; l++) {
268
- W.ln1_w[l] = p; p += C;
269
- W.ln1_b[l] = p; p += C;
270
- W.c_attn_w[l] = p; p += 3LL*C*C;
271
- W.c_attn_b[l] = p; p += 3LL*C;
272
- W.c_proj_w[l] = p; p += 1LL*C*C;
273
- W.c_proj_b[l] = p; p += C;
274
- W.ln2_w[l] = p; p += C;
275
- W.ln2_b[l] = p; p += C;
276
- W.fc_w[l] = p; p += 4LL*C*C;
277
- W.fc_b[l] = p; p += 4LL*C;
278
- W.mlp_proj_w[l] = p; p += 1LL*C*4*C;
279
- W.mlp_proj_b[l] = p; p += C;
280
- }
281
- W.ln_f_w = p; p += C;
282
- W.ln_f_b = p; p += C;
283
- W.lm_head_w = p; p += (long long)cfg.vocab_size * C;
284
-
285
- if (p > end) {
286
- printf("ERROR model_truncated\n"); fflush(stdout);
287
- exit(1);
288
  }
 
289
  }
290
-
291
- static long long kv_bytes() { return (long long)cfg.n_layer * cfg.block_size * cfg.n_embd * sizeof(float); }
292
- static void free_session(Session& s) { free(s.k_cache); free(s.v_cache); s.k_cache = nullptr; s.v_cache = nullptr; s.pos = 0; }
293
- static void evict_oldest() {
294
- if (g_sessions.empty()) return;
295
- std::string oid; double ot = 1e300;
296
- for (auto& kv : g_sessions) if (kv.second.last_use < ot) { ot = kv.second.last_use; oid = kv.first; }
297
- free_session(g_sessions[oid]); g_sessions.erase(oid);
298
- }
299
- static Session& get_or_create(const std::string& id) {
300
- auto it = g_sessions.find(id);
301
- if (it != g_sessions.end()) { it->second.last_use = get_ms(); return it->second; }
302
- if ((int)g_sessions.size() >= MAX_SESSIONS) evict_oldest();
303
- Session s;
304
- long long nb = kv_bytes();
305
- s.k_cache = (float*)calloc(nb, 1);
306
- s.v_cache = (float*)calloc(nb, 1);
307
- s.pos = 0; s.last_use = get_ms();
308
- g_sessions[id] = s;
309
- return g_sessions[id];
310
- }
311
-
312
- static int sample_topk(float temperature, int top_k) {
313
- for (int v = 0; v < cfg.vocab_size; v++) g_logits[v] /= temperature;
314
- int K = std::min(top_k, cfg.vocab_size);
315
- for (int v = 0; v < cfg.vocab_size; v++) g_topk_pairs[v] = {g_logits[v], v};
316
- std::partial_sort(g_topk_pairs, g_topk_pairs + K, g_topk_pairs + cfg.vocab_size,
317
- [](const auto& a, const auto& b){ return a.first > b.first; });
318
- float sum = 0.f;
319
- for (int j = 0; j < K; j++) {
320
- g_topk_pairs[j].first = expf(g_topk_pairs[j].first); sum += g_topk_pairs[j].first;
321
- }
322
- for (int j = 0; j < K; j++) g_topk_pairs[j].first /= sum;
323
- float r = rng_float(), cum = 0.f;
324
- int best = g_topk_pairs[0].second;
325
- for (int j = 0; j < K; j++) {
326
- cum += g_topk_pairs[j].first;
327
- if (r < cum) { best = g_topk_pairs[j].second; break; }
328
- }
329
- return best;
330
  }
331
 
332
  static int sample_topk_batch(float* logits, float temperature, int top_k) {
333
- for (int v = 0; v < cfg.vocab_size; v++) logits[v] /= temperature;
334
  int K = std::min(top_k, cfg.vocab_size);
335
- for (int v = 0; v < cfg.vocab_size; v++) g_topk_pairs[v] = {logits[v], v};
336
- std::partial_sort(g_topk_pairs, g_topk_pairs + K, g_topk_pairs + cfg.vocab_size,
337
- [](const auto& a, const auto& b){ return a.first > b.first; });
338
- float sum = 0.f;
339
- for (int j = 0; j < K; j++) {
340
- g_topk_pairs[j].first = expf(g_topk_pairs[j].first); sum += g_topk_pairs[j].first;
341
- }
342
- for (int j = 0; j < K; j++) g_topk_pairs[j].first /= sum;
343
- float r = rng_float(), cum = 0.f;
344
- int best = g_topk_pairs[0].second;
345
- for (int j = 0; j < K; j++) {
346
- cum += g_topk_pairs[j].first;
347
- if (r < cum) { best = g_topk_pairs[j].second; break; }
348
- }
349
  return best;
350
  }
351
 
352
- static std::vector<std::string> split(const std::string& s, char d) {
353
- std::vector<std::string> out; std::string cur;
354
- for (char c : s) { if (c==d){out.push_back(cur);cur.clear();}else cur+=c; }
355
- out.push_back(cur); return out;
356
- }
357
- static std::vector<int> parse_ints(const std::string& s) {
358
- std::vector<int> out;
359
- for (auto& t : split(s,',')) if (!t.empty()) out.push_back(atoi(t.c_str()));
360
- return out;
 
 
 
 
361
  }
362
 
363
- static void handle_request(const std::string& line) {
364
- auto parts = split(line, '|');
365
- if (parts.size() < 7) { printf("ERROR bad_request_format\n"); fflush(stdout); return; }
366
- std::string sess_id = parts[1];
367
- auto new_tokens = parse_ints(parts[2]);
368
- int max_new = atoi(parts[3].c_str());
369
- float temp = (float)atof(parts[4].c_str());
370
- int top_k = atoi(parts[5].c_str());
371
- auto stop_list = parse_ints(parts[6]);
372
-
373
- temp = std::max(temp, 0.01f);
374
- top_k = std::clamp(top_k, 1, cfg.vocab_size);
375
- max_new = std::max(max_new, 1);
376
-
377
- std::unordered_set<int> stop_ids(stop_list.begin(), stop_list.end());
378
- stop_ids.insert(50256);
379
-
380
- Session& sess = get_or_create(sess_id);
381
-
382
- for (int tok : new_tokens) {
383
- if (sess.pos >= cfg.block_size) { printf("ERROR context_window_full\n"); fflush(stdout); return; }
384
- forward(tok, sess.pos, sess.k_cache, sess.v_cache);
385
- sess.pos++;
386
- }
387
 
388
- double t0 = get_ms();
389
- int gen = 0;
390
- for (int i = 0; i < max_new; i++) {
391
- if (sess.pos >= cfg.block_size) break;
392
- int next = sample_topk(temp, top_k);
393
- printf("TOKEN %d %.2f\n", next, get_ms()-t0);
394
- fflush(stdout);
395
- gen++;
396
- if (stop_ids.count(next)) break;
397
- forward(next, sess.pos, sess.k_cache, sess.v_cache);
398
- sess.pos++;
399
- }
400
 
401
- printf("DONE %d %.2f\n", gen, get_ms()-t0);
402
- fflush(stdout);
403
- }
 
404
 
405
- // ── NEW BATCH HANDLERS ──
406
- static void handle_prefill(const std::string& line) {
407
- auto parts = split(line, '|');
408
- if (parts.size() < 3) return;
409
- std::string sess_id = parts[1];
410
- auto tokens = parse_ints(parts[2]);
411
- Session& sess = get_or_create(sess_id);
412
- for (int tok : tokens) {
413
- if (sess.pos >= cfg.block_size) break;
414
- forward(tok, sess.pos, sess.k_cache, sess.v_cache);
415
- sess.pos++;
 
 
416
  }
417
- printf("PREFILL_OK\n"); fflush(stdout);
418
  }
419
 
420
- static void handle_batch_step(const std::string& line) {
421
- auto parts = split(line, '|');
422
- if (parts.size() < 3) return;
 
 
423
 
424
- float temp = 0.7f; int top_k = 40;
425
- auto cfg_p = split(parts.back(), ':');
426
- if (cfg_p.size() == 2) {
427
- temp = std::max(0.01f, (float)atof(cfg_p[0].c_str()));
428
- top_k = std::clamp(atoi(cfg_p[1].c_str()), 1, ::cfg.vocab_size);
 
429
  }
 
430
 
431
- int B = parts.size() - 2;
432
- if (B > MAX_BATCH) B = MAX_BATCH;
433
 
434
- std::vector<std::string> session_ids(B);
435
- std::vector<int> tokens(B);
436
- std::vector<int> positions(B);
437
- std::vector<Session*> sessions(B);
438
 
439
- for (int b = 0; b < B; b++) {
440
- auto entry = split(parts[b + 1], ':');
441
- session_ids[b] = entry[0];
442
- tokens[b] = atoi(entry[1].c_str());
443
- positions[b] = atoi(entry[2].c_str());
444
- sessions[b] = &get_or_create(session_ids[b]);
445
- }
 
 
 
 
 
446
 
447
- const int C = cfg.n_embd;
448
- const int H = cfg.n_head;
449
- const int hs = C / H;
 
 
 
 
 
 
 
 
 
450
 
451
- for (int b = 0; b < B; b++) {
452
- float* te = W.wte + (long long)tokens[b] * C;
453
- float* pe = W.wpe + (long long)positions[b] * C;
454
- for (int i = 0; i < C; i++) g_x_b[b][i] = te[i] + pe[i];
455
- }
456
 
457
- for (int l = 0; l < cfg.n_layer; l++) {
458
- for (int b = 0; b < B; b++) layer_norm(g_buf_b[b], g_x_b[b], W.ln1_w[l], W.ln1_b[l], C);
459
 
460
- // Cache optimized QKV
461
  for (int b = 0; b < B; b++) {
462
- #pragma omp parallel for schedule(static)
463
- for (int i = 0; i < 3 * C; i++) {
464
- const float* row = W.c_attn_w[l] + (long long)i * C;
465
- g_qkv_b[b][i] = dot_avx2(row, g_buf_b[b], C) + W.c_attn_b[l][i];
 
 
 
466
  }
467
- }
468
 
469
- for (int b = 0; b < B; b++) {
470
- float* kc = sessions[b]->k_cache + (long long)l * cfg.block_size * C;
471
- float* vc = sessions[b]->v_cache + (long long)l * cfg.block_size * C;
472
- memcpy(kc + (long long)positions[b] * C, g_qkv_b[b] + C, C * sizeof(float));
473
- memcpy(vc + (long long)positions[b] * C, g_qkv_b[b] + 2 * C, C * sizeof(float));
 
 
 
 
 
474
  }
475
 
476
- for (int b = 0; b < B; b++) {
477
- int pos = positions[b];
478
- float* kc = sessions[b]->k_cache + (long long)l * cfg.block_size * C;
479
- float* vc = sessions[b]->v_cache + (long long)l * cfg.block_size * C;
480
-
481
- #pragma omp parallel for schedule(static)
482
- for (int h = 0; h < H; h++) {
483
- float* qh = g_qkv_b[b] + h * hs;
484
- float scale = 1.f / sqrtf((float)hs);
485
- float* attn = g_attn_buf_b[b] + h * cfg.block_size;
486
- for (int t = 0; t <= pos; t++) {
487
- float* kh = kc + (long long)t * C + h * hs;
488
- attn[t] = dot_avx2(qh, kh, hs) * scale;
489
- }
490
- softmax_inplace(attn, pos + 1);
 
 
 
 
 
 
 
 
491
 
492
- float* oh = g_buf_b[b] + h * hs;
493
- memset(oh, 0, hs * sizeof(float));
494
- for (int t = 0; t <= pos; t++) {
495
- float* vh = vc + (long long)t * C + h * hs;
496
- weighted_acc_avx2(oh, vh, attn[t], hs);
 
 
 
 
 
 
 
 
 
 
 
 
497
  }
498
  }
499
- }
500
-
501
- // Cache optimized C_PROJ
502
- for (int b = 0; b < B; b++) {
503
- #pragma omp parallel for schedule(static)
504
- for (int i = 0; i < C; i++) {
505
- const float* row = W.c_proj_w[l] + (long long)i * C;
506
- g_tmp_out_b[b][i] = dot_avx2(row, g_buf_b[b], C) + W.c_proj_b[l][i];
507
  }
508
- }
509
- for (int b = 0; b < B; b++) residual_add(g_x_b[b], g_tmp_out_b[b], C);
510
-
511
- for (int b = 0; b < B; b++) layer_norm(g_buf_b[b], g_x_b[b], W.ln2_w[l], W.ln2_b[l], C);
512
-
513
- // Cache optimized FC
514
- for (int b = 0; b < B; b++) {
515
- #pragma omp parallel for schedule(static)
516
- for (int i = 0; i < 4 * C; i++) {
517
- const float* row = W.fc_w[l] + (long long)i * C;
518
- g_ff_b[b][i] = dot_avx2(row, g_buf_b[b], C) + W.fc_b[l][i];
519
  }
 
 
 
 
 
 
520
  }
521
- for (int b = 0; b < B; b++) gelu_inplace(g_ff_b[b], 4 * C);
522
 
523
- // Cache optimized MLP_PROJ
 
524
  for (int b = 0; b < B; b++) {
525
- #pragma omp parallel for schedule(static)
526
- for (int i = 0; i < C; i++) {
527
- const float* row = W.mlp_proj_w[l] + (long long)i * 4 * C;
528
- g_tmp_out_b[b][i] = dot_avx2(row, g_ff_b[b], 4 * C) + W.mlp_proj_b[l][i];
 
 
 
 
 
 
 
 
 
 
 
 
 
 
529
  }
 
530
  }
531
- for (int b = 0; b < B; b++) residual_add(g_x_b[b], g_tmp_out_b[b], C);
532
- }
533
-
534
- for (int b = 0; b < B; b++) layer_norm(g_buf_b[b], g_x_b[b], W.ln_f_w, W.ln_f_b, C);
535
-
536
- std::string response = "TOKENS";
537
- for (int b = 0; b < B; b++) {
538
- #pragma omp parallel for schedule(static)
539
- for (int i = 0; i < cfg.vocab_size; i++) {
540
- const float* row = W.lm_head_w + (long long)i * C;
541
- g_logits_b[b][i] = dot_avx2(row, g_buf_b[b], C);
542
- }
543
- int next_tok = sample_topk_batch(g_logits_b[b], temp, top_k);
544
- response += " " + session_ids[b] + ":" + std::to_string(next_tok);
545
- }
546
- printf("%s\n", response.c_str());
547
- fflush(stdout);
548
- }
549
 
550
- static void handle_reset(const std::string& line) {
551
- auto parts = split(line, '|');
552
- if (parts.size() >= 2) {
553
- auto it = g_sessions.find(parts[1]);
554
- if (it != g_sessions.end()) {
555
- free_session(it->second); g_sessions.erase(it);
556
  }
557
  }
558
- printf("RESET_OK\n"); fflush(stdout);
559
- }
560
-
561
- int main() {
562
- FILE* f = fopen("model.bin", "rb");
563
- if (!f) { printf("ERROR model.bin_not_found\n"); fflush(stdout); return 1; }
564
-
565
- fread(&cfg, sizeof(int), 5, f);
566
- fseek(f, 0, SEEK_END); long fsize = ftell(f);
567
- fseek(f, 5*(long)sizeof(int), SEEK_SET);
568
- long wbytes = fsize - 5*(long)sizeof(int);
569
-
570
- g_data = (float*)malloc(wbytes);
571
- if (!g_data) { printf("ERROR oom\n"); fflush(stdout); return 1; }
572
- fread(g_data, 1, wbytes, f);
573
- fclose(f);
574
-
575
- map_weights(g_data, wbytes);
576
-
577
- const int C = cfg.n_embd;
578
- g_x = (float*)malloc(C * sizeof(float));
579
- g_buf = (float*)malloc(C * sizeof(float));
580
- g_qkv = (float*)malloc(3*C * sizeof(float));
581
- g_attn_buf = (float*)malloc((long long)cfg.n_head * cfg.block_size * sizeof(float));
582
- g_ff = (float*)malloc(4*C * sizeof(float));
583
- g_logits = (float*)malloc((long long)cfg.vocab_size * sizeof(float));
584
- g_tmp_out = (float*)malloc(C * sizeof(float));
585
- g_topk_pairs = (std::pair<float,int>*)malloc((long long)cfg.vocab_size * sizeof(std::pair<float,int>));
586
-
587
- for (int b = 0; b < MAX_BATCH; b++) {
588
- g_x_b[b] = (float*)malloc(C * sizeof(float));
589
- g_buf_b[b] = (float*)malloc(C * sizeof(float));
590
- g_qkv_b[b] = (float*)malloc(3*C * sizeof(float));
591
- g_attn_buf_b[b] = (float*)malloc((long long)cfg.n_head * cfg.block_size * sizeof(float));
592
- g_ff_b[b] = (float*)malloc(4*C * sizeof(float));
593
- g_tmp_out_b[b] = (float*)malloc(C * sizeof(float));
594
- g_logits_b[b] = (float*)malloc((long long)cfg.vocab_size * sizeof(float));
595
- }
596
-
597
- printf("READY\n"); fflush(stdout);
598
-
599
- std::string line;
600
- while (std::getline(std::cin, line)) {
601
- if (!line.empty() && line.back()=='\r') line.pop_back();
602
- if (line.empty()) continue;
603
- if (line == "QUIT") break;
604
- else if (line.rfind("RESET|", 0)==0) handle_reset(line);
605
- else if (line.rfind("REQUEST|", 0)==0) handle_request(line);
606
- else if (line.rfind("PREFILL|", 0)==0) handle_prefill(line);
607
- else if (line.rfind("BATCH_STEP|",0)==0) handle_batch_step(line);
608
- else { printf("ERROR unknown_cmd\n"); fflush(stdout); }
609
- }
610
-
611
- for (auto& kv : g_sessions) free_session(kv.second);
612
- free(g_data);
613
- free(g_x); free(g_buf); free(g_qkv); free(g_attn_buf);
614
- free(g_ff); free(g_logits); free(g_tmp_out); free(g_topk_pairs);
615
- for (int b = 0; b < MAX_BATCH; b++) {
616
- free(g_x_b[b]); free(g_buf_b[b]); free(g_qkv_b[b]);
617
- free(g_attn_buf_b[b]); free(g_ff_b[b]); free(g_tmp_out_b[b]); free(g_logits_b[b]);
618
- }
619
  return 0;
620
  }
 
1
  /*
2
  * ============================================================
3
+ * NanoMind — v7.0 (vLLM-Style Paged KV & Continuous Batching)
4
  * ============================================================
5
+ * Features:
6
+ * - Paged KV Cache (Global Memory Pool, No Fragmentation)
7
+ * - Block Allocator (O(1) allocation/free)
8
+ * - Unified Forward Pass (Prefill and Decode run in parallel)
9
+ * - Continuous Slot Refill (Zero batch starvation)
10
+ * - Thread-safe IO & Async Orchestration
11
  */
12
  #include <iostream>
13
  #include <cstdio>
 
18
  #include <algorithm>
19
  #include <string>
20
  #include <vector>
 
21
  #include <unordered_set>
22
+ #include <queue>
23
+ #include <thread>
24
+ #include <mutex>
25
+ #include <atomic>
26
+ #include <immintrin.h>
27
+
28
  #ifdef _OPENMP
29
  #include <omp.h>
30
  #endif
31
  #ifdef _WIN32
32
  #include <windows.h>
33
+ static double get_ms() { LARGE_INTEGER f,c; QueryPerformanceFrequency(&f); QueryPerformanceCounter(&c); return (double)c.QuadPart/f.QuadPart*1000.0; }
 
 
 
 
34
  #else
35
  #include <sys/time.h>
36
+ static double get_ms() { struct timeval tv; gettimeofday(&tv, NULL); return tv.tv_sec*1000.0 + tv.tv_usec/1000.0; }
 
 
 
37
  #endif
38
 
39
+ // ── Globals & RNG ──
40
  static thread_local uint64_t tl_rng_state = 0;
41
  static inline float rng_float() {
42
+ if (!tl_rng_state) { tl_rng_state = (uint64_t)get_ms() ^ (uint64_t)(uintptr_t)&tl_rng_state; if (!tl_rng_state) tl_rng_state = 0xDEADBEEFCAFEBABEULL; }
43
+ tl_rng_state ^= tl_rng_state << 13; tl_rng_state ^= tl_rng_state >> 7; tl_rng_state ^= tl_rng_state << 17;
 
 
 
 
 
44
  return (float)(tl_rng_state >> 11) * (1.0f / (float)(1ULL << 53));
45
  }
46
 
47
+ struct Config { int n_layer, n_head, n_embd, block_size, vocab_size; };
 
 
48
  struct Weights {
49
+ float *wte, *wpe, **ln1_w, **ln1_b, **c_attn_w, **c_attn_b;
50
+ float **c_proj_w, **c_proj_b, **ln2_w, **ln2_b, **fc_w, **fc_b;
51
+ float **mlp_proj_w, **mlp_proj_b, *ln_f_w, *ln_f_b, *lm_head_w;
 
 
 
 
 
 
52
  };
53
+ static Config cfg;
54
  static Weights W;
55
  static float* g_data = nullptr;
56
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
57
  const int MAX_BATCH = 16;
58
  static float *g_x_b[MAX_BATCH], *g_buf_b[MAX_BATCH], *g_qkv_b[MAX_BATCH];
59
  static float *g_attn_buf_b[MAX_BATCH], *g_ff_b[MAX_BATCH], *g_tmp_out_b[MAX_BATCH], *g_logits_b[MAX_BATCH];
60
+ static std::pair<float,int>* g_topk_pairs = nullptr;
61
 
62
+ // ── Paged KV Cache (The Core Breakthrough) ──
63
+ const int PAGE_SIZE = 16;
64
+ const int MAX_PAGES = 4096; // ~6.4 GB total memory footprint for KV Pool
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
65
 
66
+ static float* g_k_pool = nullptr;
67
+ static float* g_v_pool = nullptr;
68
+ static std::queue<int> g_free_pages;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
69
 
70
+ void init_kv_pool() {
71
+ long long page_bytes = (long long)cfg.n_layer * PAGE_SIZE * cfg.n_embd * sizeof(float);
72
+ g_k_pool = (float*)malloc(MAX_PAGES * page_bytes);
73
+ g_v_pool = (float*)malloc(MAX_PAGES * page_bytes);
74
+ for(int i = 0; i < MAX_PAGES; i++) g_free_pages.push(i);
 
 
 
75
  }
76
 
77
+ int alloc_page() {
78
+ if (g_free_pages.empty()) return -1; // OOM
79
+ int p = g_free_pages.front(); g_free_pages.pop();
80
+ return p;
 
 
81
  }
82
 
83
+ void free_pages(std::vector<int>& pages) {
84
+ for (int p : pages) g_free_pages.push(p);
85
+ pages.clear();
86
  }
87
 
88
+ // ── AVX Math Kernels ──
89
+ static void layer_norm(float* out, const float* x, const float* w, const float* b, int N) {
90
+ __m256 vsum = _mm256_setzero_ps();
91
+ for (int i=0; i<N; i+=8) vsum = _mm256_add_ps(vsum, _mm256_loadu_ps(x+i));
92
+ float tmp[8]; _mm256_storeu_ps(tmp, vsum); float mean = (tmp[0]+tmp[1]+tmp[2]+tmp[3]+tmp[4]+tmp[5]+tmp[6]+tmp[7]) / (float)N;
93
+ __m256 vmean = _mm256_set1_ps(mean), vvar = _mm256_setzero_ps();
94
+ for (int i=0; i<N; i+=8) { __m256 d = _mm256_sub_ps(_mm256_loadu_ps(x+i), vmean); vvar = _mm256_fmadd_ps(d, d, vvar); }
95
+ _mm256_storeu_ps(tmp, vvar); float var = (tmp[0]+tmp[1]+tmp[2]+tmp[3]+tmp[4]+tmp[5]+tmp[6]+tmp[7]) / (float)N;
96
+ __m256 vsc = _mm256_set1_ps(1.f / sqrtf(var + 1e-5f));
97
+ for (int i=0; i<N; i+=8) _mm256_storeu_ps(out+i, _mm256_fmadd_ps(_mm256_mul_ps(_mm256_sub_ps(_mm256_loadu_ps(x+i), vmean), vsc), _mm256_loadu_ps(w+i), _mm256_loadu_ps(b+i)));
98
  }
99
+ static void matmul_vec(float* out, const float* mat, const float* x, int M, int K) {
100
+ #pragma omp parallel for schedule(static) if(M>=64)
101
+ for (int i=0; i<M; i++) {
102
+ const float* row = mat + (long long)i*K; __m256 acc = _mm256_setzero_ps(); int j=0;
103
+ for (; j<=K-8; j+=8) acc = _mm256_fmadd_ps(_mm256_loadu_ps(row+j), _mm256_loadu_ps(x+j), acc);
104
+ float t[8]; _mm256_storeu_ps(t, acc); float s = t[0]+t[1]+t[2]+t[3]+t[4]+t[5]+t[6]+t[7];
105
+ for (; j<K; j++) s += row[j]*x[j]; out[i] = s;
 
 
 
 
 
 
 
 
 
 
 
 
 
106
  }
 
107
  }
108
+ static inline float dot_avx2(const float* a, const float* b, int n) {
109
+ __m256 acc = _mm256_setzero_ps(); int i=0;
110
+ for (; i<=n-8; i+=8) acc = _mm256_fmadd_ps(_mm256_loadu_ps(a+i), _mm256_loadu_ps(b+i), acc);
111
+ float t[8]; _mm256_storeu_ps(t, acc); float s = t[0]+t[1]+t[2]+t[3]+t[4]+t[5]+t[6]+t[7];
112
+ for (; i<n; i++) s += a[i]*b[i]; return s;
 
 
113
  }
114
+ static inline void weighted_acc_avx2(float* out, const float* v, float w, int n) {
115
+ __m256 wv = _mm256_set1_ps(w); int i=0;
116
+ for (; i<=n-8; i+=8) _mm256_storeu_ps(out+i, _mm256_fmadd_ps(wv, _mm256_loadu_ps(v+i), _mm256_loadu_ps(out+i)));
117
+ for (; i<n; i++) out[i] += w * v[i];
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
118
  }
119
+ static inline void residual_add(float* x, const float* y, int N) { #pragma omp parallel for
120
+ for (int i=0; i<N; i++) x[i] += y[i]; }
121
 
122
+ static void gelu_inplace(float* x, int N) {
123
+ const __m256 scale=_mm256_set1_ps(12102203.0f), vbias=_mm256_set1_ps(1064807168.0f), neg_c=_mm256_set1_ps(-1.702f), vone=_mm256_set1_ps(1.0f), vtwo=_mm256_set1_ps(2.0f), vlo=_mm256_set1_ps(-88.0f), vhi=_mm256_set1_ps(88.0f);
124
+ int i=0;
125
+ for (; i<=N-8; i+=8) {
126
+ __m256 v = _mm256_loadu_ps(x+i), t = _mm256_min_ps(_mm256_max_ps(_mm256_mul_ps(neg_c, v), vlo), vhi);
127
+ __m256i ti = _mm256_cvttps_epi32(_mm256_fmadd_ps(t, scale, vbias)); __m256 et = _mm256_castsi256_ps(ti);
128
+ __m256 denom = _mm256_add_ps(vone, et), r = _mm256_rcp_ps(denom);
129
+ _mm256_storeu_ps(x+i, _mm256_mul_ps(v, _mm256_mul_ps(r, _mm256_fnmadd_ps(denom, r, vtwo))));
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
130
  }
131
+ for (; i<N; i++) x[i] = x[i] / (1.0f + expf(-1.702f * x[i]));
132
  }
133
+ static void softmax_inplace(float* x, int N) {
134
+ float mx = x[0]; for (int i=1; i<N; i++) if(x[i]>mx) mx=x[i];
135
+ float s = 0.f; for (int i=0; i<N; i++) { x[i]=expf(x[i]-mx); s+=x[i]; }
136
+ for (int i=0; i<N; i++) x[i]/=s;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
137
  }
138
 
139
  static int sample_topk_batch(float* logits, float temperature, int top_k) {
140
+ for(int v=0; v<cfg.vocab_size; v++) logits[v] /= temperature;
141
  int K = std::min(top_k, cfg.vocab_size);
142
+ for(int v=0; v<cfg.vocab_size; v++) g_topk_pairs[v] = {logits[v], v};
143
+ std::partial_sort(g_topk_pairs, g_topk_pairs+K, g_topk_pairs+cfg.vocab_size, [](const auto& a, const auto& b){ return a.first > b.first; });
144
+ float sum=0.f; for(int j=0; j<K; j++) { g_topk_pairs[j].first=expf(g_topk_pairs[j].first); sum+=g_topk_pairs[j].first; }
145
+ for(int j=0; j<K; j++) g_topk_pairs[j].first/=sum;
146
+ float r = rng_float(), cum = 0.f; int best = g_topk_pairs[0].second;
147
+ for(int j=0; j<K; j++) { cum+=g_topk_pairs[j].first; if(r<cum) { best=g_topk_pairs[j].second; break; } }
 
 
 
 
 
 
 
 
148
  return best;
149
  }
150
 
151
+ static void map_weights(float* data, long wbytes) {
152
+ float* p=data; const int C=cfg.n_embd, L=cfg.n_layer;
153
+ W.wte=p; p+=cfg.vocab_size*C; W.wpe=p; p+=cfg.block_size*C;
154
+ #define ARR(f) W.f=(float**)malloc(L*sizeof(float*))
155
+ ARR(ln1_w); ARR(ln1_b); ARR(c_attn_w); ARR(c_attn_b); ARR(c_proj_w); ARR(c_proj_b); ARR(ln2_w); ARR(ln2_b); ARR(fc_w); ARR(fc_b); ARR(mlp_proj_w); ARR(mlp_proj_b);
156
+ #undef ARR
157
+ for(int l=0; l<L; l++) {
158
+ W.ln1_w[l]=p; p+=C; W.ln1_b[l]=p; p+=C; W.c_attn_w[l]=p; p+=3LL*C*C; W.c_attn_b[l]=p; p+=3LL*C;
159
+ W.c_proj_w[l]=p; p+=1LL*C*C; W.c_proj_b[l]=p; p+=C; W.ln2_w[l]=p; p+=C; W.ln2_b[l]=p; p+=C;
160
+ W.fc_w[l]=p; p+=4LL*C*C; W.fc_b[l]=p; p+=4LL*C; W.mlp_proj_w[l]=p; p+=1LL*C*4*C; W.mlp_proj_b[l]=p; p+=C;
161
+ }
162
+ W.ln_f_w=p; p+=C; W.ln_f_b=p; p+=C; W.lm_head_w=p; p+=cfg.vocab_size*C;
163
+ if(p > data + wbytes/sizeof(float)) { printf("ERROR|SYSTEM|model_truncated\n"); exit(1); }
164
  }
165
 
166
+ static std::vector<std::string> split(const std::string& s, char d) { std::vector<std::string> out; std::string cur; for(char c:s){if(c==d){out.push_back(cur);cur.clear();}else cur+=c;} out.push_back(cur); return out; }
167
+ static std::vector<int> parse_ints(const std::string& s) { std::vector<int> out; for(auto& t:split(s,',')) if(!t.empty()) out.push_back(atoi(t.c_str())); return out; }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
168
 
169
+ // ── Multi-Threading & Scheduler State ──
170
+ struct Request { std::string id; std::vector<int> tokens; int max_new; float temp; int top_k; };
171
+ struct ActiveSeq {
172
+ std::string id; std::vector<int> prompt; int max_new; float temp; int top_k;
173
+ int generated; double t0; int pos; int prefill_cursor;
174
+ std::vector<int> page_table; bool is_prefilling;
175
+ };
 
 
 
 
 
176
 
177
+ std::mutex g_mtx;
178
+ std::vector<Request> g_pending;
179
+ std::unordered_set<std::string> g_cancels;
180
+ std::atomic<bool> g_running{true};
181
 
182
+ void io_thread_func() {
183
+ std::string line;
184
+ while (std::getline(std::cin, line)) {
185
+ if (line.empty()) continue;
186
+ if (line == "QUIT") { g_running = false; break; }
187
+ auto parts = split(line, '|');
188
+ if (parts[0] == "ENQUEUE" && parts.size() >= 6) {
189
+ Request req; req.id = parts[1]; req.tokens = parse_ints(parts[2]); req.max_new = atoi(parts[3].c_str());
190
+ req.temp = std::max(0.01f, (float)atof(parts[4].c_str())); req.top_k = std::clamp(atoi(parts[5].c_str()), 1, cfg.vocab_size);
191
+ std::lock_guard<std::mutex> lock(g_mtx); g_pending.push_back(req);
192
+ } else if (parts[0] == "CANCEL" && parts.size() >= 2) {
193
+ std::lock_guard<std::mutex> lock(g_mtx); g_cancels.insert(parts[1]);
194
+ }
195
  }
 
196
  }
197
 
198
+ int main() {
199
+ FILE* f = fopen("model.bin", "rb"); if(!f) return 1;
200
+ fread(&cfg, sizeof(int), 5, f); fseek(f, 0, SEEK_END); long fsize = ftell(f); fseek(f, 5*(long)sizeof(int), SEEK_SET);
201
+ long wbytes = fsize - 5*(long)sizeof(int); g_data = (float*)malloc(wbytes); fread(g_data, 1, wbytes, f); fclose(f);
202
+ map_weights(g_data, wbytes);
203
 
204
+ const int C = cfg.n_embd;
205
+ g_topk_pairs = (std::pair<float,int>*)malloc(cfg.vocab_size*sizeof(std::pair<float,int>));
206
+ for (int b=0; b<MAX_BATCH; b++) {
207
+ g_x_b[b] = (float*)malloc(C*sizeof(float)); g_buf_b[b] = (float*)malloc(C*sizeof(float)); g_qkv_b[b] = (float*)malloc(3*C*sizeof(float));
208
+ g_attn_buf_b[b] = (float*)malloc(cfg.n_head*cfg.block_size*sizeof(float)); g_ff_b[b] = (float*)malloc(4*C*sizeof(float));
209
+ g_tmp_out_b[b] = (float*)malloc(C*sizeof(float)); g_logits_b[b] = (float*)malloc(cfg.vocab_size*sizeof(float));
210
  }
211
+ init_kv_pool();
212
 
213
+ std::thread io_thread(io_thread_func);
214
+ std::vector<ActiveSeq> active;
215
 
216
+ printf("READY\n"); fflush(stdout);
 
 
 
217
 
218
+ while(g_running) {
219
+ // 1. Safe Teardown & Admissions
220
+ {
221
+ std::lock_guard<std::mutex> lock(g_mtx);
222
+ for(const auto& cid : g_cancels) {
223
+ for (auto it = active.begin(); it != active.end(); ) {
224
+ if (it->id == cid) { free_pages(it->page_table); it = active.erase(it); }
225
+ else ++it;
226
+ }
227
+ g_pending.erase(std::remove_if(g_pending.begin(), g_pending.end(), [&](auto& req){ return req.id == cid; }), g_pending.end());
228
+ }
229
+ g_cancels.clear();
230
 
231
+ // Continuous Slot Refill
232
+ while(active.size() < MAX_BATCH && !g_pending.empty()) {
233
+ auto req = g_pending.front(); g_pending.erase(g_pending.begin());
234
+ ActiveSeq s; s.id = req.id; s.prompt = req.tokens; s.max_new = req.max_new; s.temp = req.temp; s.top_k = req.top_k;
235
+ s.generated = 0; s.pos = 0; s.prefill_cursor = 0; s.t0 = get_ms(); s.is_prefilling = true;
236
+
237
+ int p = alloc_page();
238
+ if (p == -1) { printf("ERROR|%s|OOM\n", s.id.c_str()); fflush(stdout); continue; }
239
+ s.page_table.push_back(p);
240
+ active.push_back(s);
241
+ }
242
+ }
243
 
244
+ if (active.empty()) { std::this_thread::sleep_for(std::chrono::milliseconds(1)); continue; }
 
 
 
 
245
 
246
+ int B = active.size();
247
+ std::vector<int> current_tokens(B);
248
 
249
+ // 2. Prepare Unified Batch Inputs (No Sequential Prefill Blocks)
250
  for (int b = 0; b < B; b++) {
251
+ auto& seq = active[b];
252
+
253
+ // Check page boundaries before step
254
+ if (seq.pos > 0 && seq.pos % PAGE_SIZE == 0) {
255
+ int p = alloc_page();
256
+ if (p == -1) { printf("ERROR|%s|OOM\n", seq.id.c_str()); fflush(stdout); seq.max_new = -1; continue; }
257
+ seq.page_table.push_back(p);
258
  }
 
259
 
260
+ int tok;
261
+ if (seq.is_prefilling) {
262
+ tok = seq.prompt[seq.prefill_cursor];
263
+ } else {
264
+ tok = seq.prompt.back(); // Latest generated token appended here later
265
+ }
266
+ current_tokens[b] = tok;
267
+
268
+ float* te = W.wte + (long long)tok * C; float* pe = W.wpe + (long long)seq.pos * C;
269
+ for (int i = 0; i < C; i++) g_x_b[b][i] = te[i] + pe[i];
270
  }
271
 
272
+ // 3. Batched Transformer Execution with Paged Attention
273
+ const int H = cfg.n_head; const int hs = C / H;
274
+ long long page_stride = (long long)cfg.n_layer * PAGE_SIZE * C;
275
+ long long layer_stride = PAGE_SIZE * C;
276
+ long long tok_stride = C;
277
+
278
+ for (int l = 0; l < cfg.n_layer; l++) {
279
+ for (int b = 0; b < B; b++) layer_norm(g_buf_b[b], g_x_b[b], W.ln1_w[l], W.ln1_b[l], C);
280
+ for (int b = 0; b < B; b++) {
281
+ #pragma omp parallel for schedule(static)
282
+ for (int i = 0; i < 3*C; i++) g_qkv_b[b][i] = dot_avx2(W.c_attn_w[l] + (long long)i*C, g_buf_b[b], C) + W.c_attn_b[l][i];
283
+ }
284
+
285
+ // Write to Paged KV & Attention
286
+ for (int b = 0; b < B; b++) {
287
+ auto& seq = active[b];
288
+ if (seq.max_new == -1) continue; // OOMed
289
+
290
+ int phys_p = seq.page_table[seq.pos / PAGE_SIZE];
291
+ int offset = seq.pos % PAGE_SIZE;
292
+ float* kc = g_k_pool + phys_p * page_stride + l * layer_stride + offset * tok_stride;
293
+ float* vc = g_v_pool + phys_p * page_stride + l * layer_stride + offset * tok_stride;
294
+ memcpy(kc, g_qkv_b[b]+C, C*sizeof(float)); memcpy(vc, g_qkv_b[b]+2*C, C*sizeof(float));
295
 
296
+ #pragma omp parallel for schedule(static)
297
+ for (int h = 0; h < H; h++) {
298
+ float scale = 1.f / sqrtf((float)hs), *attn = g_attn_buf_b[b] + h*cfg.block_size;
299
+ for (int t = 0; t <= seq.pos; t++) {
300
+ int read_p = seq.page_table[t / PAGE_SIZE];
301
+ int read_off = t % PAGE_SIZE;
302
+ float* kh = g_k_pool + read_p * page_stride + l * layer_stride + read_off * tok_stride + h*hs;
303
+ attn[t] = dot_avx2(g_qkv_b[b]+h*hs, kh, hs) * scale;
304
+ }
305
+ softmax_inplace(attn, seq.pos+1);
306
+ memset(g_buf_b[b]+h*hs, 0, hs*sizeof(float));
307
+ for (int t = 0; t <= seq.pos; t++) {
308
+ int read_p = seq.page_table[t / PAGE_SIZE];
309
+ int read_off = t % PAGE_SIZE;
310
+ float* vh = g_v_pool + read_p * page_stride + l * layer_stride + read_off * tok_stride + h*hs;
311
+ weighted_acc_avx2(g_buf_b[b]+h*hs, vh, attn[t], hs);
312
+ }
313
  }
314
  }
315
+ for (int b = 0; b < B; b++) {
316
+ #pragma omp parallel for schedule(static)
317
+ for (int i = 0; i < C; i++) g_tmp_out_b[b][i] = dot_avx2(W.c_proj_w[l] + (long long)i*C, g_buf_b[b], C) + W.c_proj_b[l][i];
 
 
 
 
 
318
  }
319
+ for (int b = 0; b < B; b++) residual_add(g_x_b[b], g_tmp_out_b[b], C);
320
+ for (int b = 0; b < B; b++) layer_norm(g_buf_b[b], g_x_b[b], W.ln2_w[l], W.ln2_b[l], C);
321
+ for (int b = 0; b < B; b++) {
322
+ #pragma omp parallel for schedule(static)
323
+ for (int i = 0; i < 4*C; i++) g_ff_b[b][i] = dot_avx2(W.fc_w[l] + (long long)i*C, g_buf_b[b], C) + W.fc_b[l][i];
 
 
 
 
 
 
324
  }
325
+ for (int b = 0; b < B; b++) gelu_inplace(g_ff_b[b], 4*C);
326
+ for (int b = 0; b < B; b++) {
327
+ #pragma omp parallel for schedule(static)
328
+ for (int i = 0; i < C; i++) g_tmp_out_b[b][i] = dot_avx2(W.mlp_proj_w[l] + (long long)i*4*C, g_ff_b[b], 4*C) + W.mlp_proj_b[l][i];
329
+ }
330
+ for (int b = 0; b < B; b++) residual_add(g_x_b[b], g_tmp_out_b[b], C);
331
  }
332
+ for (int b = 0; b < B; b++) layer_norm(g_buf_b[b], g_x_b[b], W.ln_f_w, W.ln_f_b, C);
333
 
334
+ // 4. Update State, Output & Teardown
335
+ std::vector<int> to_remove;
336
  for (int b = 0; b < B; b++) {
337
+ auto& seq = active[b];
338
+ if (seq.max_new == -1) { to_remove.push_back(b); continue; }
339
+
340
+ if (seq.is_prefilling) {
341
+ seq.prefill_cursor++;
342
+ if (seq.prefill_cursor >= seq.prompt.size()) seq.is_prefilling = false;
343
+ } else {
344
+ #pragma omp parallel for schedule(static)
345
+ for (int i = 0; i < cfg.vocab_size; i++) g_logits_b[b][i] = dot_avx2(W.lm_head_w + (long long)i*C, g_buf_b[b], C);
346
+ int tok = sample_topk_batch(g_logits_b[b], seq.temp, seq.top_k);
347
+ seq.prompt.push_back(tok);
348
+ seq.generated++;
349
+ printf("TOKEN|%s|%d\n", seq.id.c_str(), tok);
350
+
351
+ if (tok == 50256 || seq.generated >= seq.max_new || seq.pos >= cfg.block_size - 2) {
352
+ printf("DONE|%s|%d|%.2f\n", seq.id.c_str(), seq.generated, get_ms() - seq.t0);
353
+ to_remove.push_back(b);
354
+ }
355
  }
356
+ seq.pos++;
357
  }
358
+ fflush(stdout);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
359
 
360
+ for (int i = to_remove.size() - 1; i >= 0; i--) {
361
+ free_pages(active[to_remove[i]].page_table);
362
+ active.erase(active.begin() + to_remove[i]);
 
 
 
363
  }
364
  }
365
+
366
+ g_running = false; io_thread.join();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
367
  return 0;
368
  }