NOT-OMEGA commited on
Commit
bba2621
Β·
verified Β·
1 Parent(s): 8ec75f5

Update inference.cpp

Browse files
Files changed (1) hide show
  1. inference.cpp +739 -566
inference.cpp CHANGED
@@ -1,581 +1,754 @@
1
- /*
2
- * ============================================================
3
- * NanoMind β€” Optimized GPT-2 Inference Engine v2.0
4
- * ============================================================
5
- *
6
- * Changes from v1.0:
7
- * βœ“ Thread-local Xorshift64 RNG (replaces non-thread-safe rand())
8
- * βœ“ Pre-allocated g_topk_pairs (eliminates 400KB malloc per token)
9
- * βœ“ AVX2 layer_norm (was scalar; called 33Γ—/token)
10
- * βœ“ Sigmoid GeLU with Schraudolph fast exp (replaces tanhf cubic)
11
- * βœ“ matmul_vec skips OpenMP for M < 64 (avoids thread-fork overhead)
12
- * βœ“ map_weights bounds check (detect truncated model.bin early)
13
- *
14
- * ── STDIN PROTOCOL ──────────────────────────────────────────
15
- * REQUEST|<sess>|<tokens_csv>|<max_new>|<temp>|<top_k>|<stop_csv>
16
- * RESET|<sess>
17
- * QUIT
18
- *
19
- * ── STDOUT PROTOCOL ─────────────────────────────────────────
20
- * READY
21
- * TOKEN <id> <elapsed_ms>
22
- * DONE <count> <total_ms>
23
- * RESET_OK
24
- * ERROR <message>
25
- *
26
- * ── COMPILE ─────────────────────────────────────────────────
27
- * g++ -O3 -march=native -fopenmp -ffast-math -std=c++17 \
28
- * -o inference inference.cpp -lm
29
- * ============================================================
30
- */
31
- #include <iostream>
32
- #include <cstdio>
33
- #include <cstdlib>
34
- #include <cmath>
35
- #include <cstring>
36
- #include <ctime>
37
- #include <algorithm>
38
- #include <string>
39
- #include <vector>
40
- #include <unordered_map>
41
- #include <unordered_set>
42
- #include <immintrin.h> // AVX2 + FMA
43
- #ifdef _OPENMP
44
- #include <omp.h>
45
- #endif
46
- #ifdef _WIN32
47
- #include <windows.h>
48
- static double get_ms() {
49
- LARGE_INTEGER f, c;
50
- QueryPerformanceFrequency(&f); QueryPerformanceCounter(&c);
51
- return (double)c.QuadPart / f.QuadPart * 1000.0;
52
- }
53
- #else
54
- #include <sys/time.h>
55
- static double get_ms() {
56
- struct timeval tv; gettimeofday(&tv, NULL);
57
- return tv.tv_sec * 1000.0 + tv.tv_usec / 1000.0;
58
- }
59
- #endif
60
-
61
- // ─────────────────────────────────────────────────────────────────────────
62
- // Thread-local Xorshift64 RNG (replaces non-thread-safe rand())
63
- // Each OpenMP thread gets its own independent RNG state β€” no contention,
64
- // no corruption if sampling is ever parallelised in the future.
65
- // ─────────────────────────────────────────────────────────────────────────
66
- static thread_local uint64_t tl_rng_state = 0;
67
- static inline float rng_float() {
68
- if (!tl_rng_state) {
69
- tl_rng_state = (uint64_t)get_ms() ^ (uint64_t)(uintptr_t)&tl_rng_state;
70
- if (!tl_rng_state) tl_rng_state = 0xDEADBEEFCAFEBABEULL;
71
- }
72
- tl_rng_state ^= tl_rng_state << 13;
73
- tl_rng_state ^= tl_rng_state >> 7;
74
- tl_rng_state ^= tl_rng_state << 17;
75
- return (float)(tl_rng_state >> 11) * (1.0f / (float)(1ULL << 53));
76
- }
77
-
78
- // ─────────────────────────────────────────────────────────────────────────
79
- // Config & Weights
80
- // ─────────────────────────────────────────────────────────────────────────
81
- struct Config {
82
- int n_layer, n_head, n_embd, block_size, vocab_size;
83
- };
84
- struct Weights {
85
- float *wte, *wpe;
86
- float **ln1_w, **ln1_b;
87
- float **c_attn_w, **c_attn_b;
88
- float **c_proj_w, **c_proj_b;
89
- float **ln2_w, **ln2_b;
90
- float **fc_w, **fc_b;
91
- float **mlp_proj_w, **mlp_proj_b;
92
- float *ln_f_w, *ln_f_b;
93
- float *lm_head_w;
94
- };
95
- static Config cfg;
96
- static Weights W;
97
- static float* g_data = nullptr;
98
-
99
- // ─────────────────────────────────────────────────────────────────────────
100
- // Session (per-session KV-cache + position)
101
- // ─────────────────────────────────────────────────────────────────────────
102
- struct Session {
103
- float* k_cache = nullptr;
104
- float* v_cache = nullptr;
105
- int pos = 0;
106
- double last_use = 0.0;
107
- };
108
- static const int MAX_SESSIONS = 20;
109
- static std::unordered_map<std::string, Session> g_sessions;
110
-
111
- // ─────────────────────────────────────────────────────────────────────────
112
- // Working Buffers β€” pre-allocated, NO stack VLAs
113
- // ─────────────────────────────────────────────────────────────────────────
114
- static float *g_x, *g_buf, *g_qkv, *g_attn_buf;
115
- static float *g_ff, *g_logits, *g_tmp_out;
116
- // Pre-allocated top-k pair buffer β€” eliminates ~400KB heap alloc per token
117
- static std::pair<float,int>* g_topk_pairs = nullptr;
118
-
119
- // ─────────────────────────────────────────────────────────────────────────
120
- // Math Kernels
121
- // ─────────────────────────────────────────────────────────────────────────
122
-
123
- // AVX2 layer norm.
124
- // v1.0 was pure scalar; this version vectorises all 3 passes.
125
- // Called 33Γ—/token (2Γ—n_layer + 1), so the speedup compounds heavily.
126
- // Assumes N divisible by 8 β€” holds for n_embd=768 and 4Γ—n_embd=3072.
127
- static void layer_norm(float* out, const float* x, const float* w,
128
- const float* b, int N) {
129
- // ── Pass 1: mean ──────────────────────────────────────────────────
130
- __m256 vsum = _mm256_setzero_ps();
131
- for (int i = 0; i < N; i += 8)
132
- vsum = _mm256_add_ps(vsum, _mm256_loadu_ps(x + i));
133
- float tmp[8]; _mm256_storeu_ps(tmp, vsum);
134
- float mean = (tmp[0]+tmp[1]+tmp[2]+tmp[3]+tmp[4]+tmp[5]+tmp[6]+tmp[7]) / (float)N;
135
-
136
- // ── Pass 2: variance ─────────────────────────────────────────────
137
- __m256 vmean = _mm256_set1_ps(mean);
138
- __m256 vvar = _mm256_setzero_ps();
139
- for (int i = 0; i < N; i += 8) {
140
- __m256 d = _mm256_sub_ps(_mm256_loadu_ps(x + i), vmean);
141
- vvar = _mm256_fmadd_ps(d, d, vvar);
142
- }
143
- _mm256_storeu_ps(tmp, vvar);
144
- float var = (tmp[0]+tmp[1]+tmp[2]+tmp[3]+tmp[4]+tmp[5]+tmp[6]+tmp[7]) / (float)N;
145
- __m256 vsc = _mm256_set1_ps(1.f / sqrtf(var + 1e-5f));
146
-
147
- // ── Pass 3: normalize + affine (scale + bias) ─────────────────────
148
- for (int i = 0; i < N; i += 8) {
149
- __m256 d = _mm256_sub_ps(_mm256_loadu_ps(x + i), vmean);
150
- __m256 norm = _mm256_mul_ps(d, vsc);
151
- __m256 result = _mm256_fmadd_ps(norm, _mm256_loadu_ps(w + i),
152
- _mm256_loadu_ps(b + i));
153
- _mm256_storeu_ps(out + i, result);
154
- }
155
- // Scalar tail β€” safety for non-multiple-of-8 N (normally unreachable)
156
- float sc = 1.f / sqrtf(var + 1e-5f);
157
- for (int i = (N & ~7); i < N; i++)
158
- out[i] = (x[i] - mean) * sc * w[i] + b[i];
159
- }
160
-
161
- // AVX2 + FMA matmul: out[M] = mat[M,K] Β· x[K]
162
- // Skips OpenMP thread-fork for M < 64 β€” the fork/join overhead (~2-5Β΅s)
163
- // dominates latency for small projections like the bias add paths.
164
- static void matmul_vec(float* __restrict__ out,
165
- const float* __restrict__ mat,
166
- const float* __restrict__ x,
167
- int M, int K) {
168
- const int OMP_THRESHOLD = 64;
169
- if (M < OMP_THRESHOLD) {
170
- for (int i = 0; i < M; i++) {
171
- const float* row = mat + (long long)i * K;
172
- __m256 acc = _mm256_setzero_ps();
173
- int j = 0;
174
- for (; j <= K-8; j += 8)
175
- acc = _mm256_fmadd_ps(_mm256_loadu_ps(row+j),
176
- _mm256_loadu_ps(x+j), acc);
177
- float t[8]; _mm256_storeu_ps(t, acc);
178
- float s = t[0]+t[1]+t[2]+t[3]+t[4]+t[5]+t[6]+t[7];
179
- for (; j < K; j++) s += row[j] * x[j];
180
- out[i] = s;
181
- }
182
- return;
183
- }
184
- #pragma omp parallel for schedule(static)
185
- for (int i = 0; i < M; i++) {
186
- const float* row = mat + (long long)i * K;
187
- __m256 acc = _mm256_setzero_ps();
188
- int j = 0;
189
- for (; j <= K-8; j += 8)
190
- acc = _mm256_fmadd_ps(_mm256_loadu_ps(row+j),
191
- _mm256_loadu_ps(x+j), acc);
192
- float t[8]; _mm256_storeu_ps(t, acc);
193
- float s = t[0]+t[1]+t[2]+t[3]+t[4]+t[5]+t[6]+t[7];
194
- for (; j < K; j++) s += row[j] * x[j];
195
- out[i] = s;
196
- }
197
- }
198
-
199
- // AVX2 dot product β€” used in attention inner loop (hs=64 β†’ 8 AVX iters)
200
- static inline float dot_avx2(const float* __restrict__ a,
201
- const float* __restrict__ b, int n) {
202
- __m256 acc = _mm256_setzero_ps();
203
- int i = 0;
204
- for (; i <= n-8; i += 8)
205
- acc = _mm256_fmadd_ps(_mm256_loadu_ps(a+i),
206
- _mm256_loadu_ps(b+i), acc);
207
- float tmp[8]; _mm256_storeu_ps(tmp, acc);
208
- float s = tmp[0]+tmp[1]+tmp[2]+tmp[3]+tmp[4]+tmp[5]+tmp[6]+tmp[7];
209
- for (; i < n; i++) s += a[i]*b[i];
210
- return s;
211
- }
212
-
213
- // AVX2 weighted accumulate: out += w * v
214
- static inline void weighted_acc_avx2(float* __restrict__ out,
215
- const float* __restrict__ v,
216
- float w, int n) {
217
- __m256 wv = _mm256_set1_ps(w);
218
- int i = 0;
219
- for (; i <= n-8; i += 8)
220
- _mm256_storeu_ps(out+i,
221
- _mm256_fmadd_ps(wv, _mm256_loadu_ps(v+i),
222
- _mm256_loadu_ps(out+i)));
223
- for (; i < n; i++) out[i] += w * v[i];
224
- }
225
-
226
- static inline void add_bias(float* x, const float* b, int N) {
227
- #pragma omp parallel for
228
- for (int i = 0; i < N; i++) x[i] += b[i];
229
- }
230
-
231
- static inline void residual_add(float* x, const float* y, int N) {
232
- #pragma omp parallel for
233
- for (int i = 0; i < N; i++) x[i] += y[i];
234
- }
235
-
236
- // GeLU via sigmoid approximation with Schraudolph fast exp.
237
- //
238
- // GeLU(x) β‰ˆ x * sigmoid(1.702 * x) = x / (1 + exp(-1.702 * x))
239
- //
240
- // v1.0 used the tanh cubic form: x * 0.5 * (1 + tanh(sqrt(2/Ο€)*(x + 0.044715*xΒ³)))
241
- // which requires an expensive tanhf() call per element.
242
- //
243
- // This version uses:
244
- // exp(t) β‰ˆ reinterpret_cast<float>(int(t * 2^23/ln2 + 127*2^23)) [Schraudolph 1999]
245
- // 1/(1+exp) via rcp_ps + one Newton-Raphson step (faster than div_ps)
246
- //
247
- // Error vs true GeLU: < 0.01 across typical activation range β€” negligible for inference.
248
- static void gelu_inplace(float* x, int N) {
249
- const __m256 scale = _mm256_set1_ps(12102203.0f); // 2^23 / ln(2)
250
- const __m256 vbias = _mm256_set1_ps(1064807168.0f); // 127 * 2^23
251
- const __m256 neg_c = _mm256_set1_ps(-1.702f);
252
- const __m256 vone = _mm256_set1_ps(1.0f);
253
- const __m256 vtwo = _mm256_set1_ps(2.0f);
254
- const __m256 vlo = _mm256_set1_ps(-88.0f);
255
- const __m256 vhi = _mm256_set1_ps(88.0f);
256
- int i = 0;
257
- for (; i <= N-8; i += 8) {
258
- __m256 v = _mm256_loadu_ps(x + i);
259
- __m256 t = _mm256_mul_ps(neg_c, v);
260
- t = _mm256_max_ps(t, vlo);
261
- t = _mm256_min_ps(t, vhi);
262
- // fast exp(t) via Schraudolph bit-reinterpretation trick
263
- __m256i ti = _mm256_cvttps_epi32(_mm256_fmadd_ps(t, scale, vbias));
264
- __m256 et = _mm256_castsi256_ps(ti);
265
- // sigmoid via rcp_ps + Newton-Raphson (faster than div_ps)
266
- __m256 denom = _mm256_add_ps(vone, et);
267
- __m256 r = _mm256_rcp_ps(denom);
268
- r = _mm256_mul_ps(r, _mm256_fnmadd_ps(denom, r, vtwo));
269
- _mm256_storeu_ps(x + i, _mm256_mul_ps(v, r));
270
- }
271
- // Scalar tail
272
- for (; i < N; i++) {
273
- float v = x[i];
274
- x[i] = v / (1.0f + expf(-1.702f * v));
275
- }
276
- }
277
-
278
- static void softmax_inplace(float* x, int N) {
279
- float mx = x[0];
280
- for (int i = 1; i < N; i++) if (x[i] > mx) mx = x[i];
281
- float s = 0.f;
282
- for (int i = 0; i < N; i++) { x[i] = expf(x[i]-mx); s += x[i]; }
283
- for (int i = 0; i < N; i++) x[i] /= s;
284
- }
285
-
286
- // ─────────────────────────────────────────────────────────────────────────
287
- // Forward Pass (single token at position pos)
288
- // ─────────────────────────────────────────────────────────────────────────
289
- static void forward(int token_id, int pos, float* k_cache, float* v_cache) {
290
- const int C = cfg.n_embd;
291
- const int H = cfg.n_head;
292
- const int hs = C / H;
293
-
294
- float* te = W.wte + (long long)token_id * C;
295
- float* pe = W.wpe + (long long)pos * C;
296
- #pragma omp parallel for
297
- for (int i = 0; i < C; i++) g_x[i] = te[i] + pe[i];
298
-
299
- for (int l = 0; l < cfg.n_layer; l++) {
300
- // ── Self-Attention ────────────────────────────────────────────────
301
- layer_norm(g_buf, g_x, W.ln1_w[l], W.ln1_b[l], C);
302
- matmul_vec(g_qkv, W.c_attn_w[l], g_buf, 3*C, C);
303
- add_bias(g_qkv, W.c_attn_b[l], 3*C);
304
-
305
- float* q = g_qkv;
306
- float* k = g_qkv + C;
307
- float* v = g_qkv + 2*C;
308
- float* kc = k_cache + (long long)l * cfg.block_size * C;
309
- float* vc = v_cache + (long long)l * cfg.block_size * C;
310
- memcpy(kc + (long long)pos*C, k, C*sizeof(float));
311
- memcpy(vc + (long long)pos*C, v, C*sizeof(float));
312
-
313
- #pragma omp parallel for schedule(static)
314
- for (int h = 0; h < H; h++) {
315
- float* qh = q + h*hs;
316
- float scale = 1.f / sqrtf((float)hs);
317
- float* attn = g_attn_buf + h*cfg.block_size;
318
- for (int t = 0; t <= pos; t++) {
319
- float* kh = kc + (long long)t*C + h*hs;
320
- attn[t] = dot_avx2(qh, kh, hs) * scale;
321
- }
322
- softmax_inplace(attn, pos+1);
323
- float* oh = g_buf + h*hs;
324
- memset(oh, 0, hs*sizeof(float));
325
- for (int t = 0; t <= pos; t++) {
326
- float* vh = vc + (long long)t*C + h*hs;
327
- weighted_acc_avx2(oh, vh, attn[t], hs);
328
  }
329
- }
330
-
331
- matmul_vec(g_tmp_out, W.c_proj_w[l], g_buf, C, C);
332
- add_bias(g_tmp_out, W.c_proj_b[l], C);
333
- residual_add(g_x, g_tmp_out, C);
334
-
335
- // ── MLP ───────────────────────────────────────────────────────────
336
- layer_norm(g_buf, g_x, W.ln2_w[l], W.ln2_b[l], C);
337
- matmul_vec(g_ff, W.fc_w[l], g_buf, 4*C, C);
338
- add_bias(g_ff, W.fc_b[l], 4*C);
339
- gelu_inplace(g_ff, 4*C);
340
- matmul_vec(g_tmp_out, W.mlp_proj_w[l], g_ff, C, 4*C);
341
- add_bias(g_tmp_out, W.mlp_proj_b[l], C);
342
- residual_add(g_x, g_tmp_out, C);
343
- }
344
-
345
- layer_norm(g_buf, g_x, W.ln_f_w, W.ln_f_b, C);
346
- matmul_vec(g_logits, W.lm_head_w, g_buf, cfg.vocab_size, C);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
347
  }
348
 
349
- // ─────────────────────────────────────────────────────────────────────────
350
- // Weight Mapping
351
- // Now takes wbytes to perform a bounds check β€” detects truncated model.bin
352
- // before it causes a silent segfault or garbage outputs.
353
- // ─────────────────────────────────────────────────────────────────────────
354
- static void map_weights(float* data, long wbytes) {
355
- float* p = data;
356
- float* end = data + wbytes / sizeof(float);
357
- const int C = cfg.n_embd, L = cfg.n_layer;
358
- W.wte = p; p += (long long)cfg.vocab_size * C;
359
- W.wpe = p; p += (long long)cfg.block_size * C;
360
- #define ARR(f) W.f = (float**)malloc(L*sizeof(float*))
361
- ARR(ln1_w); ARR(ln1_b); ARR(c_attn_w); ARR(c_attn_b);
362
- ARR(c_proj_w); ARR(c_proj_b); ARR(ln2_w); ARR(ln2_b);
363
- ARR(fc_w); ARR(fc_b); ARR(mlp_proj_w); ARR(mlp_proj_b);
364
- #undef ARR
365
- for (int l = 0; l < L; l++) {
366
- W.ln1_w[l] = p; p += C;
367
- W.ln1_b[l] = p; p += C;
368
- W.c_attn_w[l] = p; p += 3LL*C*C;
369
- W.c_attn_b[l] = p; p += 3LL*C;
370
- W.c_proj_w[l] = p; p += 1LL*C*C;
371
- W.c_proj_b[l] = p; p += C;
372
- W.ln2_w[l] = p; p += C;
373
- W.ln2_b[l] = p; p += C;
374
- W.fc_w[l] = p; p += 4LL*C*C;
375
- W.fc_b[l] = p; p += 4LL*C;
376
- W.mlp_proj_w[l] = p; p += 1LL*C*4*C;
377
- W.mlp_proj_b[l] = p; p += C;
378
- }
379
- W.ln_f_w = p; p += C;
380
- W.ln_f_b = p; p += C;
381
- W.lm_head_w = p; p += (long long)cfg.vocab_size * C;
382
-
383
- // ── Bounds check: catch truncated / corrupt model.bin immediately ──
384
- if (p > end) {
385
- long long needed = (p - data) * (long long)sizeof(float);
386
- long long got = (end - data) * (long long)sizeof(float);
387
- fprintf(stderr,
388
- "FATAL: model.bin too small β€” need %lld bytes, got %lld.\n"
389
- " Re-download the file from HuggingFace.\n",
390
- needed, got);
391
- fflush(stderr);
392
- printf("ERROR model_truncated\n"); fflush(stdout);
393
- exit(1);
394
- }
395
- }
396
 
397
- // ─────────────────────────────────────────────────────────────────────────
398
- // Session Management (LRU eviction)
399
- // ─────────────────────────────────────────────────────────────────────────
400
- static long long kv_bytes() {
401
- return (long long)cfg.n_layer * cfg.block_size * cfg.n_embd * sizeof(float);
402
- }
403
- static void free_session(Session& s) {
404
- free(s.k_cache); free(s.v_cache);
405
- s.k_cache = nullptr; s.v_cache = nullptr; s.pos = 0;
406
- }
407
- static void evict_oldest() {
408
- if (g_sessions.empty()) return;
409
- std::string oid; double ot = 1e300;
410
- for (auto& kv : g_sessions)
411
- if (kv.second.last_use < ot) { ot = kv.second.last_use; oid = kv.first; }
412
- free_session(g_sessions[oid]);
413
- g_sessions.erase(oid);
414
- }
415
- static Session& get_or_create(const std::string& id) {
416
- auto it = g_sessions.find(id);
417
- if (it != g_sessions.end()) {
418
- it->second.last_use = get_ms();
419
- return it->second;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
420
  }
421
- if ((int)g_sessions.size() >= MAX_SESSIONS) evict_oldest();
422
- Session s;
423
- long long nb = kv_bytes();
424
- s.k_cache = (float*)calloc(nb, 1);
425
- s.v_cache = (float*)calloc(nb, 1);
426
- s.pos = 0;
427
- s.last_use = get_ms();
428
- g_sessions[id] = s;
429
- return g_sessions[id];
430
- }
431
 
432
- // ─────────────────────────────────────────────────────────────────────────
433
- // Top-K Sampler
434
- // Uses pre-allocated g_topk_pairs β€” eliminates ~400KB heap alloc per token.
435
- // Uses thread-local rng_float() β€” replaces non-thread-safe rand().
436
- // ─────────────────────────────────────────────────────────────────────────
437
- static int sample_topk(float temperature, int top_k) {
438
- for (int v = 0; v < cfg.vocab_size; v++) g_logits[v] /= temperature;
439
- int K = std::min(top_k, cfg.vocab_size);
440
- for (int v = 0; v < cfg.vocab_size; v++) g_topk_pairs[v] = {g_logits[v], v};
441
- std::partial_sort(g_topk_pairs, g_topk_pairs + K, g_topk_pairs + cfg.vocab_size,
442
- [](const auto& a, const auto& b){ return a.first > b.first; });
443
- float sum = 0.f;
444
- for (int j = 0; j < K; j++) {
445
- g_topk_pairs[j].first = expf(g_topk_pairs[j].first);
446
- sum += g_topk_pairs[j].first;
447
- }
448
- for (int j = 0; j < K; j++) g_topk_pairs[j].first /= sum;
449
- float r = rng_float(), cum = 0.f;
450
- int best = g_topk_pairs[0].second;
451
- for (int j = 0; j < K; j++) {
452
- cum += g_topk_pairs[j].first;
453
- if (r < cum) { best = g_topk_pairs[j].second; break; }
454
- }
455
- return best;
456
- }
457
-
458
- // ─────────────────────────────────────────────────────────────────────────
459
- // Helpers
460
- // ─────────────────────────────────────────────────────────────────────────
461
- static std::vector<std::string> split(const std::string& s, char d) {
462
- std::vector<std::string> out; std::string cur;
463
- for (char c : s) { if (c==d){out.push_back(cur);cur.clear();}else cur+=c; }
464
- out.push_back(cur); return out;
465
- }
466
- static std::vector<int> parse_ints(const std::string& s) {
467
- std::vector<int> out;
468
- for (auto& t : split(s,',')) if (!t.empty()) out.push_back(atoi(t.c_str()));
469
- return out;
470
- }
471
 
472
- // ─────────────────────────────────────────────────────────────────────────
473
- // Command Handlers
474
- // ─────────────────────────────────────────────────────────────────────────
475
- static void handle_request(const std::string& line) {
476
- auto parts = split(line, '|');
477
- if (parts.size() < 7) {
478
- printf("ERROR bad_request_format\n"); fflush(stdout); return;
479
- }
480
- std::string sess_id = parts[1];
481
- auto new_tokens = parse_ints(parts[2]);
482
- int max_new = atoi(parts[3].c_str());
483
- float temp = (float)atof(parts[4].c_str());
484
- int top_k = atoi(parts[5].c_str());
485
- auto stop_list = parse_ints(parts[6]);
486
-
487
- temp = std::max(temp, 0.01f);
488
- top_k = std::clamp(top_k, 1, cfg.vocab_size);
489
- max_new = std::max(max_new, 1);
490
-
491
- std::unordered_set<int> stop_ids(stop_list.begin(), stop_list.end());
492
- stop_ids.insert(50256); // <|endoftext|>
493
-
494
- Session& sess = get_or_create(sess_id);
495
-
496
- // Prefill
497
- for (int tok : new_tokens) {
498
- if (sess.pos >= cfg.block_size) {
499
- printf("ERROR context_window_full\n"); fflush(stdout); return;
500
- }
501
- forward(tok, sess.pos, sess.k_cache, sess.v_cache);
502
- sess.pos++;
503
  }
504
 
505
- // Autoregressive generation
506
- double t0 = get_ms();
507
- int gen = 0;
508
- for (int i = 0; i < max_new; i++) {
509
- if (sess.pos >= cfg.block_size) break;
510
- int next = sample_topk(temp, top_k);
511
- printf("TOKEN %d %.2f\n", next, get_ms()-t0);
512
- fflush(stdout);
513
- gen++;
514
- if (stop_ids.count(next)) break;
515
- forward(next, sess.pos, sess.k_cache, sess.v_cache);
516
- sess.pos++;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
517
  }
518
 
519
- printf("DONE %d %.2f\n", gen, get_ms()-t0);
520
- fflush(stdout);
521
- }
522
 
523
- static void handle_reset(const std::string& line) {
524
- auto parts = split(line, '|');
525
- if (parts.size() >= 2) {
526
- auto it = g_sessions.find(parts[1]);
527
- if (it != g_sessions.end()) {
528
- free_session(it->second); g_sessions.erase(it);
529
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
530
  }
531
- printf("RESET_OK\n"); fflush(stdout);
532
- }
533
 
534
- // ─────────────────────────────────────────────────────────────────────────
535
- // main
536
- // ─────────────────────────────────────────────────────────────────────────
537
- int main() {
538
- FILE* f = fopen("model.bin", "rb");
539
- if (!f) { printf("ERROR model.bin_not_found\n"); fflush(stdout); return 1; }
540
-
541
- fread(&cfg, sizeof(int), 5, f);
542
- fseek(f, 0, SEEK_END); long fsize = ftell(f);
543
- fseek(f, 5*(long)sizeof(int), SEEK_SET);
544
- long wbytes = fsize - 5*(long)sizeof(int);
545
-
546
- g_data = (float*)malloc(wbytes);
547
- if (!g_data) { printf("ERROR oom\n"); fflush(stdout); return 1; }
548
- fread(g_data, 1, wbytes, f);
549
- fclose(f);
550
-
551
- map_weights(g_data, wbytes); // bounds check β€” exits on truncated file
552
-
553
- const int C = cfg.n_embd;
554
- g_x = (float*)malloc(C * sizeof(float));
555
- g_buf = (float*)malloc(C * sizeof(float));
556
- g_qkv = (float*)malloc(3*C * sizeof(float));
557
- g_attn_buf = (float*)malloc((long long)cfg.n_head * cfg.block_size * sizeof(float));
558
- g_ff = (float*)malloc(4*C * sizeof(float));
559
- g_logits = (float*)malloc((long long)cfg.vocab_size * sizeof(float));
560
- g_tmp_out = (float*)malloc(C * sizeof(float));
561
- g_topk_pairs = (std::pair<float,int>*)malloc(
562
- (long long)cfg.vocab_size * sizeof(std::pair<float,int>));
563
-
564
- printf("READY\n"); fflush(stdout);
565
-
566
- std::string line;
567
- while (std::getline(std::cin, line)) {
568
- if (!line.empty() && line.back()=='\r') line.pop_back();
569
- if (line.empty()) continue;
570
- if (line == "QUIT") break;
571
- else if (line.rfind("RESET|", 0)==0) handle_reset(line);
572
- else if (line.rfind("REQUEST|", 0)==0) handle_request(line);
573
- else { printf("ERROR unknown_cmd\n"); fflush(stdout); }
574
- }
575
 
576
- for (auto& kv : g_sessions) free_session(kv.second);
577
- free(g_data);
578
- free(g_x); free(g_buf); free(g_qkv); free(g_attn_buf);
579
- free(g_ff); free(g_logits); free(g_tmp_out); free(g_topk_pairs);
580
- return 0;
581
- }
 
1
+ """
2
+ NanoMind β€” FastAPI Backend v1.5
3
+ 152M GPT-2 Β· AVX2 + OpenMP Β· Persistent KV-Cache
4
+
5
+ Changes over v1.4:
6
+ 1. MAX_GEN_CEILING lowered to 256 (was 500) β€” MAX_SESSION_TOKENS raised
7
+ to 744, giving ~50% more history before a context reset triggers.
8
+ 2. MAX_INPUT_CHARS = 2000 guard added β€” rejects absurdly long messages
9
+ before they hit the tokenizer and blow the context window.
10
+ 3. SessionData.rebuild_with_sliding_window() β€” on context overflow, keeps
11
+ as many recent conversation turns as fit rather than discarding all
12
+ history. Users no longer experience sudden amnesia mid-conversation.
13
+ 4. CRITICAL: tokens_in_engine is now updated in event_stream's finally
14
+ block, not after the async-for loop. Previously a client disconnect
15
+ (CancelledError) skipped the update entirely, leaving a stale count
16
+ that caused premature or missed context resets on subsequent turns.
17
+ 5. tokens_generated is incremented on every TOKEN chunk so the finally
18
+ block has an accurate partial count even on mid-stream disconnects.
19
+ 6. STOP_STRINGS matching now also catches patterns without a leading
20
+ space (e.g. "\\nUser:" and "\\n User:"), reducing role-bleed leakage.
21
+
22
+ Fixes carried forward from v1.4:
23
+ 7. _download_file no longer passes local_dir β€” downloads to HF cache
24
+ and copies manually to avoid cross-device rename failures.
25
+ 8. Byte-level UTF-8 buffer in InferenceEngine.generate for correct
26
+ multi-byte character decoding (no more β—† replacement chars).
27
+ """
28
+ import asyncio
29
+ import json
30
+ import os
31
+ import shutil
32
+ import time
33
+ import uuid
34
+ from contextlib import asynccontextmanager
35
+ from pathlib import Path
36
+
37
+ import psutil
38
+ import tiktoken
39
+ from fastapi import FastAPI, HTTPException, Response
40
+ from fastapi.middleware.cors import CORSMiddleware
41
+ from fastapi.responses import FileResponse, StreamingResponse
42
+ from pydantic import BaseModel, Field
43
+ from huggingface_hub import hf_hub_download
44
+
45
+ # ─────────────────────────────────────────────────────────────────────────
46
+ # Config
47
+ # ─────────────────────────────────────────────────────────────────────────
48
+ BASE_DIR = Path(__file__).parent
49
+ INFERENCE_EXE = BASE_DIR / "inference"
50
+ MODEL_BIN = BASE_DIR / "model.bin"
51
+ TOKENIZER_BIN = BASE_DIR / "tokenizer.bin"
52
+ HF_REPO_ID = os.environ.get("HF_REPO_ID", "NOT-OMEGA/NanoMind")
53
+
54
+ SYSTEM_TOKEN = "System:"
55
+ USER_TOKEN = "User:"
56
+ ASST_TOKEN = "Assistant:"
57
+ SEP = "\n"
58
+
59
+ BLOCK_SIZE = 1024
60
+ MAX_GEN_CEILING = 256 # lowered from 500 β€” frees 244 extra history tokens
61
+ SAFETY_MARGIN = 24
62
+ # 1024 - 256 - 24 = 744 (was 500 with old ceiling)
63
+ MAX_SESSION_TOKENS = BLOCK_SIZE - MAX_GEN_CEILING - SAFETY_MARGIN
64
+
65
+ # Hard cap on raw input characters β€” rejects giant pastes before tokenising
66
+ MAX_INPUT_CHARS = 2000
67
+
68
+ N_ENGINES = int(os.environ.get("N_ENGINES", "1"))
69
+ OMP_PER_ENGINE = int(os.environ.get("OMP_NUM_THREADS", "2"))
70
+ SESSION_TTL_S = int(os.environ.get("SESSION_TTL", "1800"))
71
+
72
+ enc = tiktoken.get_encoding("gpt2")
73
+ STOP_TOKEN_IDS = [50256]
74
+ STOP_STRINGS = ["User:", "System:"]
75
+
76
+ # ─────────────────────────────────────────────────────────────────────────
77
+ # Inference Engine
78
+ # ─────────────────────────────────────────────────────────────────────────
79
+ class InferenceEngine:
80
+ def __init__(self, eid: int):
81
+ self.eid = eid
82
+ self._proc = None
83
+ self._ready = False
84
+
85
+ async def start(self):
86
+ if not INFERENCE_EXE.exists():
87
+ raise RuntimeError(f"inference binary not found: {INFERENCE_EXE}")
88
+ if not MODEL_BIN.exists():
89
+ raise RuntimeError(f"model.bin not found: {MODEL_BIN}")
90
+
91
+ print(f"[engine-{self.eid}] launching process…")
92
+ env = os.environ.copy()
93
+ env["OMP_NUM_THREADS"] = str(OMP_PER_ENGINE)
94
+
95
+ self._proc = await asyncio.create_subprocess_exec(
96
+ str(INFERENCE_EXE),
97
+ stdin=asyncio.subprocess.PIPE,
98
+ stdout=asyncio.subprocess.PIPE,
99
+ stderr=asyncio.subprocess.PIPE,
100
+ cwd=str(BASE_DIR),
101
+ env=env,
102
+ )
103
+
104
+ try:
105
+ async with asyncio.timeout(120):
106
+ while True:
107
+ raw = await self._proc.stdout.readline()
108
+ if not raw:
109
+ stderr_out = await self._proc.stderr.read()
110
+ raise RuntimeError(
111
+ f"[engine-{self.eid}] process exited before READY. "
112
+ f"stderr: {stderr_out.decode(errors='replace')[:500]}"
113
+ )
114
+ line = raw.decode().strip()
115
+ if line:
116
+ print(f"[engine-{self.eid}] startup: {line}")
117
+ if line == "READY":
118
+ self._ready = True
119
+ print(f"[engine-{self.eid}] READY "
120
+ f"pid={self._proc.pid} "
121
+ f"OMP_NUM_THREADS={OMP_PER_ENGINE}")
122
+ break
123
+ if line.startswith("ERROR"):
124
+ raise RuntimeError(
125
+ f"[engine-{self.eid}] startup error: {line}"
126
+ )
127
+ except TimeoutError:
128
+ self._proc.kill()
129
+ raise RuntimeError(
130
+ f"[engine-{self.eid}] timed out (120s) waiting for READY"
131
+ )
132
+
133
+ async def stop(self):
134
+ self._ready = False
135
+ if self._proc:
136
+ try:
137
+ self._proc.stdin.write(b"QUIT\n")
138
+ await self._proc.stdin.drain()
139
+ await asyncio.wait_for(self._proc.wait(), timeout=3.0)
140
+ except Exception:
141
+ self._proc.kill()
142
+ self._proc = None
143
+
144
+ async def reset_session(self, session_id: str):
145
+ if not self._ready or self._proc is None:
146
+ return
147
+ self._proc.stdin.write(f"RESET|{session_id}\n".encode())
148
+ await self._proc.stdin.drain()
149
+ try:
150
+ async with asyncio.timeout(5):
151
+ while True:
152
+ raw = await self._proc.stdout.readline()
153
+ if not raw:
154
+ break
155
+ if raw.decode().strip() == "RESET_OK":
156
+ break
157
+ except TimeoutError:
158
+ pass
159
+
160
+ async def generate(self, session_id, new_token_ids, max_new, temperature, top_k):
161
+ if not self._ready or self._proc is None:
162
+ yield {"type": "error", "message": f"Engine-{self.eid} not ready"}
163
+ return
164
+
165
+ tokens_csv = ",".join(map(str, new_token_ids))
166
+ stop_csv = ",".join(map(str, STOP_TOKEN_IDS))
167
+ cmd = (
168
+ f"REQUEST|{session_id}|{tokens_csv}|"
169
+ f"{max_new}|{temperature}|{top_k}|{stop_csv}\n"
170
+ )
171
+ self._proc.stdin.write(cmd.encode())
172
+ await self._proc.stdin.drain()
173
+
174
+ # Byte-level buffer for correct multi-byte UTF-8 decoding.
175
+ # GPT-2 BPE tokens can be partial UTF-8 byte sequences β€” decoding
176
+ # each token individually produces replacement chars for non-ASCII.
177
+ byte_buffer = bytearray()
178
+
179
+ try:
180
+ while True:
181
+ raw = await self._proc.stdout.readline()
182
+ if not raw:
183
+ self._ready = False
184
+ yield {"type": "error",
185
+ "message": "Engine process died unexpectedly"}
186
+ return
187
+
188
+ line = raw.decode("utf-8", errors="replace").strip()
189
+ if not line:
190
+ continue
191
+
192
+ if line.startswith("TOKEN"):
193
+ parts = line.split()
194
+ token_id = int(parts[1])
195
+ elapsed = float(parts[2])
196
+ byte_buffer.extend(enc.decode_bytes([token_id]))
197
+ try:
198
+ decoded_text = byte_buffer.decode("utf-8")
199
+ byte_buffer.clear()
200
+ except UnicodeDecodeError:
201
+ continue
202
+ if decoded_text:
203
+ yield {
204
+ "type": "token",
205
+ "id": token_id,
206
+ "text": decoded_text,
207
+ "elapsed_ms": elapsed,
208
+ }
209
+
210
+ elif line.startswith("DONE"):
211
+ if byte_buffer:
212
+ leftover = byte_buffer.decode("utf-8", errors="replace")
213
+ byte_buffer.clear()
214
+ if leftover:
215
+ yield {
216
+ "type": "token",
217
+ "id": -1,
218
+ "text": leftover,
219
+ "elapsed_ms": 0.0,
220
+ }
221
+ parts = line.split()
222
+ total_t = int(parts[1])
223
+ total_ms = float(parts[2])
224
+ tps = round(total_t / (total_ms / 1000.0), 2) \
225
+ if total_ms > 0 else 0
226
+ yield {
227
+ "type": "done",
228
+ "total_tokens": total_t,
229
+ "total_ms": total_ms,
230
+ "tps": tps,
231
+ }
232
+ return
233
+
234
+ elif line.startswith("ERROR"):
235
+ yield {"type": "error", "message": line}
236
+ return
237
+
238
+ except asyncio.CancelledError:
239
+ try:
240
+ async with asyncio.timeout(5):
241
+ while True:
242
+ raw = await self._proc.stdout.readline()
243
+ if not raw:
244
+ break
245
+ if raw.decode().strip().startswith(("DONE", "ERROR")):
246
+ break
247
+ except Exception:
248
+ pass
249
+ raise
250
+
251
+ @property
252
+ def pid(self):
253
+ return self._proc.pid if self._proc else None
254
+
255
+
256
+ # ─────────────────────────────────────────────────────────────────────────
257
+ # Engine Pool
258
+ # ─────────────────────────────────────────────────────────────────────────
259
+ class EnginePool:
260
+ def __init__(self, n: int):
261
+ self.n = n
262
+ self.engines = [InferenceEngine(i) for i in range(n)]
263
+ self._locks: list[asyncio.Lock] = []
264
+ self._session_map: dict[str, int] = {}
265
+ self._engine_load: list[int] = []
266
+ self._map_lock: asyncio.Lock | None = None
267
+
268
+ async def start(self):
269
+ self._map_lock = asyncio.Lock()
270
+ self._locks = [asyncio.Lock() for _ in range(self.n)]
271
+ self._engine_load = [0] * self.n
272
+ await asyncio.gather(*(e.start() for e in self.engines))
273
+ print(f"[pool] {self.n} engine(s) ready "
274
+ f"(OMP_NUM_THREADS={OMP_PER_ENGINE} each)")
275
+
276
+ async def stop(self):
277
+ await asyncio.gather(
278
+ *(e.stop() for e in self.engines), return_exceptions=True
279
+ )
280
+
281
+ async def _assign(self, session_id: str) -> int:
282
+ async with self._map_lock:
283
+ if session_id not in self._session_map:
284
+ idx = min(range(self.n), key=lambda i: self._engine_load[i])
285
+ self._session_map[session_id] = idx
286
+ self._engine_load[idx] += 1
287
+ return self._session_map[session_id]
288
+
289
+ async def _drop(self, session_id: str):
290
+ async with self._map_lock:
291
+ if session_id in self._session_map:
292
+ idx = self._session_map.pop(session_id)
293
+ self._engine_load[idx] = max(0, self._engine_load[idx] - 1)
294
+
295
+ async def generate(self, session_id, new_token_ids, max_new, temp, top_k):
296
+ idx = await self._assign(session_id)
297
+ async with self._locks[idx]:
298
+ async for chunk in self.engines[idx].generate(
299
+ session_id, new_token_ids, max_new, temp, top_k
300
+ ):
301
+ yield chunk
302
+
303
+ async def reset_session(self, session_id: str):
304
+ if self._map_lock is None:
305
+ return
306
+ async with self._map_lock:
307
+ idx = self._session_map.get(session_id)
308
+ if idx is not None:
309
+ async with self._locks[idx]:
310
+ await self.engines[idx].reset_session(session_id)
311
+ await self._drop(session_id)
312
+
313
+ def get_all_pids(self) -> list:
314
+ return [e.pid for e in self.engines if e.pid]
315
+
316
+ def status(self) -> list:
317
+ return [
318
+ {
319
+ "engine_id": i,
320
+ "pid": self.engines[i].pid,
321
+ "sessions": self._engine_load[i],
322
+ "busy": self._locks[i].locked() if self._locks else False,
323
+ "ready": self.engines[i]._ready,
 
 
 
 
324
  }
325
+ for i in range(self.n)
326
+ ]
327
+
328
+
329
+ pool = EnginePool(N_ENGINES)
330
+
331
+ # ───────────────────────────────────────────────────────────────────���─────
332
+ # Session State
333
+ # ─────────────────────────────────────────────────────────────────────────
334
+ class SessionData:
335
+ __slots__ = ("system_prompt", "history", "tokens_in_engine", "last_active")
336
+
337
+ def __init__(self, system_prompt: str):
338
+ self.system_prompt = system_prompt
339
+ self.history: list = []
340
+ self.tokens_in_engine = 0
341
+ self.last_active = time.monotonic()
342
+
343
+ def touch(self):
344
+ self.last_active = time.monotonic()
345
+
346
+ def append_user(self, content: str):
347
+ self.history.append({"role": "user", "content": content})
348
+
349
+ def append_assistant(self, content: str):
350
+ self.history.append({"role": "assistant", "content": content})
351
+
352
+ def new_turn_tokens(self, user_msg: str) -> list[int]:
353
+ """Tokens for a normal (non-overflow) turn."""
354
+ if self.tokens_in_engine == 0:
355
+ text = (
356
+ f"{SYSTEM_TOKEN} {self.system_prompt}{SEP}"
357
+ f"{USER_TOKEN} {user_msg}{SEP}{ASST_TOKEN} "
358
+ )
359
+ else:
360
+ text = f"{USER_TOKEN} {user_msg}{SEP}{ASST_TOKEN} "
361
+ return enc.encode_ordinary(text)
362
+
363
+ def rebuild_with_sliding_window(self, user_msg: str, token_budget: int) -> list[int]:
364
+ """
365
+ Rebuild a full prompt that fits within token_budget.
366
+ Called after a context overflow reset.
367
+ Packs: system prompt + as many recent history turns as fit
368
+ (most-recent-first selection) + current user message.
369
+ This replaces the v1.4 behaviour of discarding all history on reset,
370
+ which caused sudden conversation amnesia for multi-turn sessions.
371
+ """
372
+ prefix = enc.encode_ordinary(f"{SYSTEM_TOKEN} {self.system_prompt}{SEP}")
373
+ suffix = enc.encode_ordinary(f"{USER_TOKEN} {user_msg}{SEP}{ASST_TOKEN} ")
374
+ budget = token_budget - len(prefix) - len(suffix)
375
+
376
+ selected: list[list[int]] = []
377
+ for turn in reversed(self.history):
378
+ role = USER_TOKEN if turn["role"] == "user" else ASST_TOKEN
379
+ chunk = enc.encode_ordinary(f"{role} {turn['content']}{SEP}")
380
+ if len(chunk) > budget:
381
+ break
382
+ selected.insert(0, chunk)
383
+ budget -= len(chunk)
384
+
385
+ tokens: list[int] = prefix[:]
386
+ for chunk in selected:
387
+ tokens += chunk
388
+ tokens += suffix
389
+ return tokens
390
+
391
+
392
+ sessions: dict[str, SessionData] = {}
393
+ _metrics_lock: asyncio.Lock | None = None
394
+ metrics = {
395
+ "total_requests": 0,
396
+ "total_tokens": 0,
397
+ "total_ms": 0.0,
398
+ "errors": 0,
399
+ "start_time": time.time(),
400
+ "sessions_evicted": 0,
401
  }
402
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
403
 
404
+ def get_total_ram_mb() -> float:
405
+ try:
406
+ total = psutil.Process(os.getpid()).memory_info().rss
407
+ for pid in pool.get_all_pids():
408
+ try:
409
+ total += psutil.Process(pid).memory_info().rss
410
+ except psutil.NoSuchProcess:
411
+ pass
412
+ return round(total / 1_000_000, 1)
413
+ except Exception:
414
+ return 0.0
415
+
416
+
417
+ # ─────────────────────────────────────────────────────────────────────────
418
+ # Background Tasks
419
+ # ─────────────────────────────────────────────────────────────────────────
420
+ async def session_gc_loop():
421
+ while True:
422
+ await asyncio.sleep(300)
423
+ if _metrics_lock is None:
424
+ continue
425
+ now = time.monotonic()
426
+ expired = [
427
+ sid for sid, s in list(sessions.items())
428
+ if now - s.last_active > SESSION_TTL_S
429
+ ]
430
+ for sid in expired:
431
+ sessions.pop(sid, None)
432
+ await pool.reset_session(sid)
433
+ async with _metrics_lock:
434
+ metrics["sessions_evicted"] += 1
435
+ if expired:
436
+ print(f"[GC] Evicted {len(expired)} idle sessions")
437
+
438
+
439
+ def _download_file(repo_id: str, filename: str, dest_dir: Path) -> None:
440
+ """Blocking HF download β€” runs in a thread executor.
441
+ Downloads to HF cache (same filesystem), then shutil.copy2() to dest.
442
+ Avoids the cross-device os.rename() failure from passing local_dir
443
+ directly to hf_hub_download in newer huggingface_hub versions.
444
+ """
445
+ dest = dest_dir / filename
446
+ print(f"[HF HUB] Downloading {filename} from {repo_id} …")
447
+ cached = Path(hf_hub_download(repo_id=repo_id, filename=filename))
448
+ if not cached.exists():
449
+ raise FileNotFoundError(
450
+ f"hf_hub_download returned a path that does not exist: {cached}"
451
+ )
452
+ print(f"[HF HUB] Copying {filename} β†’ {dest}")
453
+ shutil.copy2(cached, dest)
454
+ if not dest.exists():
455
+ raise FileNotFoundError(
456
+ f"Copy appeared to succeed but {dest} is missing. "
457
+ "Possible causes: disk full, /app not writable."
458
+ )
459
+ size_mb = dest.stat().st_size // 1_000_000
460
+ print(f"[HF HUB] {filename} ready ({size_mb} MB)")
461
+
462
+
463
+ async def _startup_background():
464
+ loop = asyncio.get_running_loop()
465
+
466
+ if MODEL_BIN.exists():
467
+ print(f"[startup] model.bin present "
468
+ f"({MODEL_BIN.stat().st_size // 1_000_000} MB) β€” skip download")
469
+ else:
470
+ try:
471
+ await loop.run_in_executor(
472
+ None, _download_file, HF_REPO_ID, "model.bin", BASE_DIR
473
+ )
474
+ except Exception as e:
475
+ print(f"[ERROR] model.bin download failed: {e}")
476
+ return
477
+
478
+ if TOKENIZER_BIN.exists():
479
+ print("[startup] tokenizer.bin present β€” skip download")
480
+ else:
481
+ try:
482
+ await loop.run_in_executor(
483
+ None, _download_file, HF_REPO_ID, "tokenizer.bin", BASE_DIR
484
+ )
485
+ except Exception as e:
486
+ print(f"[ERROR] tokenizer.bin download failed: {e}")
487
+ return
488
+
489
+ if not MODEL_BIN.exists():
490
+ print("[ERROR] model.bin missing after download β€” aborting pool start")
491
+ return
492
+ if not TOKENIZER_BIN.exists():
493
+ print("[ERROR] tokenizer.bin missing after download β€” aborting pool start")
494
+ return
495
+
496
+ print(f"[startup] All files OK β€” starting {N_ENGINES} engine(s)…")
497
+ try:
498
+ await pool.start()
499
+ except Exception as e:
500
+ print(f"[ERROR] Pool start failed: {e}")
501
+
502
+
503
+ # ─────────────────────────────────────────────────────────────────────────
504
+ # App Lifecycle
505
+ # ─────────────────────────────────────────────────────────────────────────
506
+ @asynccontextmanager
507
+ async def lifespan(app: FastAPI):
508
+ global _metrics_lock
509
+ _metrics_lock = asyncio.Lock()
510
+ asyncio.create_task(_startup_background())
511
+ asyncio.create_task(session_gc_loop())
512
+ yield
513
+ await pool.stop()
514
+
515
+
516
+ app = FastAPI(title="NanoMind", version="1.5", lifespan=lifespan)
517
+ app.add_middleware(
518
+ CORSMiddleware,
519
+ allow_origins=["*"], allow_methods=["*"], allow_headers=["*"],
520
+ )
521
+
522
+ # ─────────────────────────────────────────────────────────────────────────
523
+ # Request Models
524
+ # ─────────────────────────────────────────────────────────────────────────
525
+ class ChatRequest(BaseModel):
526
+ message: str
527
+ session_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
528
+ system_prompt: str = "You are a helpful, thoughtful, and articulate AI assistant."
529
+ max_new_tokens: int = Field(default=200, ge=1, le=MAX_GEN_CEILING)
530
+ temperature: float = Field(default=0.7, ge=0.01, le=2.0)
531
+ top_k: int = Field(default=40, ge=1, le=200)
532
+
533
+
534
+ class ResetRequest(BaseModel):
535
+ session_id: str
536
+
537
+
538
+ # ─────────────────────────────────────────────────────────────────────────
539
+ # Routes
540
+ # ─────────────────────────────────────────────────────────────────────────
541
+ @app.get("/")
542
+ async def serve_ui():
543
+ return FileResponse(BASE_DIR / "index.html")
544
+
545
+
546
+ @app.get("/health")
547
+ async def health(response: Response):
548
+ mem = psutil.virtual_memory()
549
+ ready_count = sum(1 for e in pool.engines if e._ready)
550
+ is_ready = ready_count > 0
551
+ if not is_ready:
552
+ response.status_code = 503
553
+ return {
554
+ "status": "ok" if is_ready else "starting",
555
+ "engines_ready": ready_count,
556
+ "engines_total": N_ENGINES,
557
+ "omp_threads_per_engine": OMP_PER_ENGINE,
558
+ "speed_mode": N_ENGINES == 1,
559
+ "active_sessions": len(sessions),
560
+ "sessions_evicted": metrics["sessions_evicted"],
561
+ "process_ram_mb": get_total_ram_mb(),
562
+ "system_ram_used_pct": mem.percent,
563
+ "uptime_seconds": round(time.time() - metrics["start_time"], 1),
564
  }
 
 
 
 
 
 
 
 
 
 
565
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
566
 
567
+ @app.get("/pool/status")
568
+ async def pool_status():
569
+ return {
570
+ "n_engines": N_ENGINES,
571
+ "omp_per_engine": OMP_PER_ENGINE,
572
+ "engines": pool.status(),
573
+ "sessions": len(sessions),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
574
  }
575
 
576
+
577
+ @app.post("/chat")
578
+ async def chat(req: ChatRequest):
579
+ if not any(e._ready for e in pool.engines):
580
+ raise HTTPException(503, "Engine loading… retry in ~30s")
581
+
582
+ # ── Input length guard ────────────────────────────────────────────────
583
+ if len(req.message) > MAX_INPUT_CHARS:
584
+ raise HTTPException(
585
+ 400,
586
+ f"Message too long ({len(req.message)} chars). "
587
+ f"Maximum is {MAX_INPUT_CHARS} characters."
588
+ )
589
+
590
+ sess = sessions.get(req.session_id)
591
+ if sess is None:
592
+ sess = SessionData(req.system_prompt)
593
+ sessions[req.session_id] = sess
594
+
595
+ sess.touch()
596
+ new_tokens = sess.new_turn_tokens(req.message)
597
+
598
+ # ── Context window overflow β€” sliding window rebuild ──────────────────
599
+ # v1.4: hard reset discarded all history (conversation amnesia).
600
+ # v1.5: rebuild keeps as many recent turns as fit in the budget.
601
+ if sess.tokens_in_engine + len(new_tokens) + req.max_new_tokens \
602
+ > MAX_SESSION_TOKENS:
603
+ await pool.reset_session(req.session_id)
604
+ sess.tokens_in_engine = 0
605
+ budget = MAX_SESSION_TOKENS - req.max_new_tokens
606
+ new_tokens = sess.rebuild_with_sliding_window(req.message, budget)
607
+
608
+ sess.append_user(req.message)
609
+ async with _metrics_lock:
610
+ metrics["total_requests"] += 1
611
+
612
+ async def event_stream():
613
+ parts: list[str] = []
614
+ t0 = time.perf_counter()
615
+ stopped_early = False
616
+ final_chunk = None
617
+ tokens_generated = 0 # tracked throughout β€” used in finally even on disconnect
618
+
619
+ try:
620
+ async for chunk in pool.generate(
621
+ req.session_id, new_tokens,
622
+ req.max_new_tokens, req.temperature, req.top_k,
623
+ ):
624
+ if chunk["type"] == "token":
625
+ # Count every token regardless of stopped_early so the
626
+ # finally block has an accurate partial count on disconnect.
627
+ tokens_generated += 1
628
+ if not stopped_early:
629
+ parts.append(chunk["text"])
630
+ joined = "".join(parts)
631
+ for s in STOP_STRINGS:
632
+ # Match both "\nUser:" and "\n User:" (no leading space)
633
+ for pattern in (f"\n{s}", f"\n {s}"):
634
+ idx = joined.find(pattern)
635
+ if idx != -1:
636
+ parts = [joined[:idx]]
637
+ stopped_early = True
638
+ break
639
+ if stopped_early:
640
+ break
641
+ if not stopped_early:
642
+ yield f"data: {json.dumps(chunk)}\n\n"
643
+ # stopped_early: silently consume remaining tokens until DONE
644
+
645
+ elif chunk["type"] == "done":
646
+ final_chunk = chunk
647
+ tokens_generated = chunk["total_tokens"] # authoritative count
648
+
649
+ elif chunk["type"] == "error":
650
+ async with _metrics_lock:
651
+ metrics["errors"] += 1
652
+ yield f"data: {json.dumps(chunk)}\n\n"
653
+
654
+ # ── Normal completion path ────────────────────────────────────
655
+ reply = "".join(parts).strip()
656
+ elapsed = (time.perf_counter() - t0) * 1000
657
+
658
+ if final_chunk is not None and not stopped_early:
659
+ tok_out = final_chunk["total_tokens"]
660
+ total_ms = final_chunk["total_ms"]
661
+ tps = final_chunk["tps"]
662
+ else:
663
+ tok_out = len(enc.encode_ordinary(reply)) if reply else 0
664
+ total_ms = round(elapsed, 2)
665
+ tps = round(tok_out / (total_ms / 1000.0), 2) \
666
+ if total_ms > 0 else 0
667
+ tokens_generated = tok_out
668
+
669
+ sess.append_assistant(reply)
670
+ async with _metrics_lock:
671
+ metrics["total_tokens"] += tok_out
672
+ metrics["total_ms"] += elapsed
673
+
674
+ done_payload = json.dumps({
675
+ "type": "done",
676
+ "total_tokens": tok_out,
677
+ "total_ms": total_ms,
678
+ "tps": tps,
679
+ "session_id": req.session_id,
680
+ "full_response": reply,
681
+ })
682
+ yield f"data: {done_payload}\n\n"
683
+
684
+ except Exception as e:
685
+ async with _metrics_lock:
686
+ metrics["errors"] += 1
687
+ yield f"data: {json.dumps({'type': 'error', 'message': str(e)})}\n\n"
688
+
689
+ finally:
690
+ # Always update token count β€” runs even on CancelledError (client disconnect).
691
+ # v1.4 updated tokens_in_engine after the async-for loop, which was
692
+ # skipped entirely on disconnect, leaving a permanently stale count.
693
+ sess.tokens_in_engine += len(new_tokens) + tokens_generated
694
+ yield "data: [DONE]\n\n"
695
+
696
+ return StreamingResponse(
697
+ event_stream(),
698
+ media_type="text/event-stream",
699
+ headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
700
+ )
701
+
702
+
703
+ @app.post("/chat/reset")
704
+ async def reset_chat(req: ResetRequest):
705
+ sessions.pop(req.session_id, None)
706
+ await pool.reset_session(req.session_id)
707
+ return {"status": "ok", "session_id": req.session_id}
708
+
709
+
710
+ @app.get("/chat/history")
711
+ async def get_history(session_id: str):
712
+ sess = sessions.get(session_id)
713
+ if not sess:
714
+ return {"session_id": session_id, "turns": 0, "history": []}
715
+ return {
716
+ "session_id": session_id,
717
+ "turns": sum(1 for m in sess.history if m["role"] == "user"),
718
+ "tokens_in_engine": sess.tokens_in_engine,
719
+ "last_active_ago_s": round(time.monotonic() - sess.last_active, 1),
720
+ "history": sess.history,
721
  }
722
 
 
 
 
723
 
724
+ @app.get("/metrics")
725
+ async def get_metrics():
726
+ async with _metrics_lock:
727
+ n = metrics["total_requests"]
728
+ tok = metrics["total_tokens"]
729
+ ms = metrics["total_ms"]
730
+ err = metrics["errors"]
731
+ evicted = metrics["sessions_evicted"]
732
+ mem = psutil.virtual_memory()
733
+ return {
734
+ "total_requests": n,
735
+ "total_tokens": tok,
736
+ "total_errors": err,
737
+ "avg_tps": round(tok / (ms / 1000), 2) if ms > 0 else 0,
738
+ "active_sessions": len(sessions),
739
+ "sessions_evicted_total": evicted,
740
+ "n_engines": N_ENGINES,
741
+ "omp_per_engine": OMP_PER_ENGINE,
742
+ "engines_ready": sum(1 for e in pool.engines if e._ready),
743
+ "engines_busy": sum(1 for lk in pool._locks if lk.locked())
744
+ if pool._locks else 0,
745
+ "process_ram_mb": get_total_ram_mb(),
746
+ "system_ram_used_pct": mem.percent,
747
+ "system_ram_total_gb": round(mem.total / 1_000_000_000, 1),
748
+ "uptime_s": round(time.time() - metrics["start_time"], 1),
749
  }
 
 
750
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
751
 
752
+ if __name__ == "__main__":
753
+ import uvicorn
754
+ uvicorn.run("main:app", host="0.0.0.0", port=7860, reload=False)