NOT-OMEGA commited on
Commit
60fde4b
Β·
verified Β·
1 Parent(s): 5f5b2c4

Update inference.cpp

Browse files
Files changed (1) hide show
  1. inference.cpp +676 -318
inference.cpp CHANGED
@@ -1,329 +1,687 @@
1
- <!DOCTYPE html>
2
- <html lang="en">
3
- <head>
4
- <meta charset="UTF-8" />
5
- <meta name="viewport" content="width=device-width, initial-scale=1.0" />
6
- <title>OmegaMind 152M β€” C++17 AVX2 Inference Telemetry</title>
7
- <script src="https://cdn.tailwindcss.com"></script>
8
- <script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
9
- <style>
10
- body { background: #070b12; color: #e2e8f0; font-family: 'Inter', system-ui, sans-serif; }
11
- .prose pre { background: #0f172a; padding: 12px; border-radius: 8px; overflow-x: auto; border: 1px solid #1e293b; margin: 8px 0; }
12
- .prose code { color: #38bdf8; font-size: 0.9em; }
13
- ::-webkit-scrollbar { width: 6px; }
14
- ::-webkit-scrollbar-thumb { background: #334155; border-radius: 3px; }
15
- </style>
16
- </head>
17
- <body class="h-screen flex flex-col overflow-hidden">
18
-
19
- <!-- Top Live Telemetry Dashboard -->
20
- <header class="border-b border-slate-800 bg-slate-900/90 backdrop-blur px-5 py-3">
21
- <div class="max-w-7xl mx-auto flex flex-col gap-3">
22
-
23
- <!-- Brand Row -->
24
- <div class="flex items-center justify-between">
25
- <div class="flex items-center gap-2.5">
26
- <div class="h-3 w-3 rounded-full bg-emerald-400 animate-pulse"></div>
27
- <h1 class="text-base font-bold tracking-tight text-white">
28
- OmegaMind <span class="text-xs font-mono bg-sky-500/20 text-sky-400 px-2 py-0.5 rounded ml-1">152.8M β€’ C++17 AVX2/FMA</span>
29
- </h1>
30
- </div>
31
- <button onclick="resetChat()" class="text-xs font-mono bg-slate-800 hover:bg-rose-950/70 hover:text-rose-300 border border-slate-700 px-3 py-1.5 rounded-lg transition">
32
- Reset KV Cache
33
- </button>
34
- </div>
35
-
36
- <!-- 6 Telemetry Cards -->
37
- <div class="grid grid-cols-2 sm:grid-cols-3 lg:grid-cols-6 gap-2.5 text-xs font-mono">
38
-
39
- <!-- 1. TTFT -->
40
- <div class="bg-slate-950/90 border border-slate-800 rounded-lg px-3 py-2">
41
- <div class="text-slate-400 text-[10px] uppercase">1st Token (TTFT)</div>
42
- <div id="stat-ttft" class="text-emerald-400 text-sm font-bold mt-0.5">β€” ms</div>
43
- </div>
44
-
45
- <!-- 2. Total Time -->
46
- <div class="bg-slate-950/90 border border-slate-800 rounded-lg px-3 py-2">
47
- <div class="text-slate-400 text-[10px] uppercase">Output Time</div>
48
- <div id="stat-total-time" class="text-purple-400 text-sm font-bold mt-0.5">β€” ms</div>
49
- </div>
50
-
51
- <!-- 3. Avg Speed per Token -->
52
- <div class="bg-slate-950/90 border border-slate-800 rounded-lg px-3 py-2">
53
- <div class="text-slate-400 text-[10px] uppercase">Avg Token Speed</div>
54
- <div id="stat-speed" class="text-sky-400 text-sm font-bold mt-0.5">β€” tok/s</div>
55
- <div id="stat-ms-tok" class="text-slate-400 text-[10px]">β€” ms/tok</div>
56
- </div>
57
-
58
- <!-- 4. CPU Usage (%) -->
59
- <div class="bg-slate-950/90 border border-slate-800 rounded-lg px-3 py-2">
60
- <div class="flex justify-between text-[10px] text-slate-400 uppercase">
61
- <span>CPU Usage</span>
62
- <span id="stat-cpu" class="text-amber-400 font-bold">0.0%</span>
63
- </div>
64
- <div class="w-full bg-slate-800 h-2 rounded-full mt-1.5 overflow-hidden">
65
- <div id="bar-cpu" class="bg-amber-400 h-full transition-all duration-200" style="width: 0%"></div>
66
- </div>
67
- </div>
68
-
69
- <!-- 5. RAM Usage (%) -->
70
- <div class="bg-slate-950/90 border border-slate-800 rounded-lg px-3 py-2">
71
- <div class="flex justify-between text-[10px] text-slate-400 uppercase">
72
- <span>RAM Usage</span>
73
- <span id="stat-ram" class="text-rose-400 font-bold">0.0%</span>
74
- </div>
75
- <div class="w-full bg-slate-800 h-2 rounded-full mt-1.5 overflow-hidden">
76
- <div id="bar-ram" class="bg-rose-500 h-full transition-all duration-200" style="width: 0%"></div>
77
- </div>
78
- <div id="stat-ram-gb" class="text-slate-400 text-[10px] mt-0.5">β€” / β€” GB</div>
79
- </div>
80
-
81
- <!-- 6. KV Cache -->
82
- <div class="bg-slate-950/90 border border-slate-800 rounded-lg px-3 py-2">
83
- <div class="text-slate-400 text-[10px] uppercase">KV Cache Pos</div>
84
- <div id="stat-kv" class="text-teal-400 text-sm font-bold mt-0.5">0 / 1024</div>
85
- </div>
86
-
87
- </div>
88
- </div>
89
- </header>
90
-
91
- <!-- Main Content -->
92
- <div class="flex-1 flex overflow-hidden">
93
-
94
- <!-- Chat Messages -->
95
- <main class="flex-1 flex flex-col">
96
- <div id="chat-box" class="flex-1 overflow-y-auto p-6 space-y-5">
97
- <div class="max-w-3xl mx-auto bg-slate-900/90 border border-slate-800 rounded-xl p-4 text-sm text-slate-300">
98
- πŸ‘‹ <b>OmegaMind (152.8M)</b> is live. Every response tracks <b>TTFT (ms)</b>, <b>Total Output Time</b>, <b>Avg Speed per Token (tok/s & ms/tok)</b>, <b>CPU %</b>, and <b>RAM %</b> in real time.
99
- </div>
100
- </div>
101
-
102
- <!-- Input Bar -->
103
- <div class="border-t border-slate-800 bg-slate-900/60 p-4">
104
- <form onsubmit="sendMessage(event)" class="max-w-3xl mx-auto flex gap-3">
105
- <input
106
- id="user-input"
107
- type="text"
108
- placeholder="Ask OmegaMind anything (e.g., Write a Python function to reverse a string)..."
109
- class="flex-1 bg-slate-950 border border-slate-800 rounded-xl px-4 py-3 text-sm text-white focus:outline-none focus:border-sky-500"
110
- autocomplete="off"
111
- />
112
- <button
113
- id="send-btn"
114
- type="submit"
115
- class="bg-sky-600 hover:bg-sky-500 text-white font-medium px-6 py-3 rounded-xl text-sm transition disabled:opacity-40"
116
- >
117
- Send
118
- </button>
119
- </form>
120
- </div>
121
- </main>
122
-
123
- <!-- Sampling Sidebar -->
124
- <aside class="w-72 border-l border-slate-800 bg-slate-900/50 p-5 hidden lg:flex flex-col gap-5 text-xs">
125
- <h2 class="font-semibold text-slate-200 uppercase tracking-wider">Sampling Controls</h2>
126
-
127
- <div>
128
- <div class="flex justify-between mb-1"><span>Temperature</span><span id="val-temp" class="font-mono text-sky-400">0.25</span></div>
129
- <input id="inp-temp" type="range" min="0.05" max="1.0" step="0.05" value="0.25" oninput="document.getElementById('val-temp').innerText=this.value" class="w-full accent-sky-500" />
130
- </div>
131
-
132
- <div>
133
- <div class="flex justify-between mb-1"><span>Top-K</span><span id="val-topk" class="font-mono text-sky-400">8</span></div>
134
- <input id="inp-topk" type="range" min="1" max="50" step="1" value="8" oninput="document.getElementById('val-topk').innerText=this.value" class="w-full accent-sky-500" />
135
- </div>
136
-
137
- <div>
138
- <div class="flex justify-between mb-1"><span>Top-P (Nucleus)</span><span id="val-topp" class="font-mono text-sky-400">0.85</span></div>
139
- <input id="inp-topp" type="range" min="0.10" max="1.0" step="0.05" value="0.85" oninput="document.getElementById('val-topp').innerText=this.value" class="w-full accent-sky-500" />
140
- </div>
141
-
142
- <div>
143
- <div class="flex justify-between mb-1"><span>Repetition Penalty</span><span id="val-rep" class="font-mono text-sky-400">1.18</span></div>
144
- <input id="inp-rep" type="range" min="1.00" max="1.40" step="0.02" value="1.18" oninput="document.getElementById('val-rep').innerText=this.value" class="w-full accent-sky-500" />
145
- </div>
146
-
147
- <div>
148
- <div class="flex justify-between mb-1"><span>Max Output Tokens</span><span id="val-max" class="font-mono text-sky-400">256</span></div>
149
- <input id="inp-max" type="range" min="64" max="512" step="32" value="256" oninput="document.getElementById('val-max').innerText=this.value" class="w-full accent-sky-500" />
150
- </div>
151
-
152
- <div class="mt-auto p-3 rounded-lg bg-slate-950 border border-slate-800 text-slate-400 space-y-1">
153
- <div><b>Repo:</b> NOT-OMEGA/OmegaMind</div>
154
- <div><b>Context:</b> 1024 tokens</div>
155
- <div><b>SIMD:</b> AVX2 + FMA3 + OpenMP</div>
156
- </div>
157
- </aside>
158
- </div>
159
-
160
- <script>
161
- const sessionId = "sess_" + Math.random().toString(36).substring(2, 10);
162
- let isFirstTurn = true;
163
-
164
- function updateDashboard(d) {
165
- if (d.ttft_ms !== undefined) {
166
- document.getElementById("stat-ttft").innerText = `${d.ttft_ms} ms`;
167
- }
168
- if (d.total_ms !== undefined) {
169
- const sec = (d.total_ms / 1000).toFixed(2);
170
- document.getElementById("stat-total-time").innerText = `${d.total_ms} ms (${sec}s)`;
171
- }
172
- if (d.tok_s !== undefined) {
173
- document.getElementById("stat-speed").innerText = `${d.tok_s} tok/s`;
174
- }
175
- if (d.ms_per_tok !== undefined) {
176
- document.getElementById("stat-ms-tok").innerText = `${d.ms_per_tok} ms / token`;
177
- }
178
- if (d.cpu_pct !== undefined) {
179
- document.getElementById("stat-cpu").innerText = `${d.cpu_pct}%`;
180
- document.getElementById("bar-cpu").style.width = `${Math.min(d.cpu_pct, 100)}%`;
181
- }
182
- if (d.ram_pct !== undefined) {
183
- document.getElementById("stat-ram").innerText = `${d.ram_pct}%`;
184
- document.getElementById("bar-ram").style.width = `${Math.min(d.ram_pct, 100)}%`;
185
- }
186
- if (d.ram_used_gb !== undefined && d.ram_total_gb !== undefined) {
187
- document.getElementById("stat-ram-gb").innerText = `${d.ram_used_gb} / ${d.ram_total_gb} GB`;
188
- }
189
- if (d.kv_pos !== undefined) {
190
- document.getElementById("stat-kv").innerText = `${d.kv_pos} / 1024`;
191
- }
192
  }
193
-
194
- // Background poll CPU% and RAM% every 2 seconds
195
- setInterval(async () => {
196
- try {
197
- const r = await fetch("/api/metrics");
198
- if (r.ok) updateDashboard(await r.json());
199
- } catch (_) {}
200
- }, 2000);
201
-
202
- function createMessagePair(userText) {
203
- const box = document.getElementById("chat-box");
204
-
205
- // User Bubble
206
- const uWrap = document.createElement("div");
207
- uWrap.className = "max-w-3xl mx-auto flex justify-end";
208
- const uBubble = document.createElement("div");
209
- uBubble.className = "bg-sky-600 text-white px-4 py-2.5 rounded-2xl rounded-br-sm max-w-xl text-sm";
210
- uBubble.textContent = userText;
211
- uWrap.appendChild(uBubble);
212
- box.appendChild(uWrap);
213
-
214
- // Bot Container (Bubble + Telemetry Footer)
215
- const bWrap = document.createElement("div");
216
- bWrap.className = "max-w-3xl mx-auto flex flex-col items-start gap-1.5";
217
-
218
- const bBubble = document.createElement("div");
219
- bBubble.className = "bg-slate-900 border border-slate-800 text-slate-200 px-5 py-3.5 rounded-2xl rounded-bl-sm max-w-2xl text-sm prose prose-invert w-full";
220
- bBubble.innerHTML = "<span class='text-slate-500 animate-pulse'>Prefilling KV cache...</span>";
221
-
222
- const bStats = document.createElement("div");
223
- bStats.className = "text-[11px] font-mono text-slate-400 bg-slate-900/70 border border-slate-800/80 rounded-lg px-3 py-1.5 flex flex-wrap gap-x-4 gap-y-1";
224
- bStats.innerHTML = "<span>⏳ Measuring telemetry...</span>";
225
-
226
- bWrap.appendChild(bBubble);
227
- bWrap.appendChild(bStats);
228
- box.appendChild(bWrap);
229
- box.scrollTop = box.scrollHeight;
230
-
231
- return { bBubble, bStats };
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
232
  }
233
 
234
- async function sendMessage(e) {
235
- e.preventDefault();
236
- const input = document.getElementById("user-input");
237
- const btn = document.getElementById("send-btn");
238
- const text = input.value.trim();
239
- if (!text) return;
240
-
241
- input.value = "";
242
- btn.disabled = true;
243
- const { bBubble, bStats } = createMessagePair(text);
244
-
245
- const payload = {
246
- session_id: sessionId,
247
- message: text,
248
- max_tokens: parseInt(document.getElementById("inp-max").value),
249
- temperature: parseFloat(document.getElementById("inp-temp").value),
250
- top_k: parseInt(document.getElementById("inp-topk").value),
251
- top_p: parseFloat(document.getElementById("inp-topp").value),
252
- repetition_penalty: parseFloat(document.getElementById("inp-rep").value),
253
- is_first_turn: isFirstTurn
254
- };
255
- isFirstTurn = false;
256
-
257
- let rawMarkdown = "";
258
- try {
259
- const res = await fetch("/api/chat", {
260
- method: "POST",
261
- headers: { "Content-Type": "application/json" },
262
- body: JSON.stringify(payload)
263
- });
264
-
265
- const reader = res.body.getReader();
266
- const decoder = new TextDecoder();
267
- let buffer = "";
268
-
269
- while (true) {
270
- const { value, done } = await reader.read();
271
- if (done) break;
272
- buffer += decoder.decode(value, { stream: true });
273
-
274
- const lines = buffer.split("\n\n");
275
- buffer = lines.pop();
276
-
277
- for (const line of lines) {
278
- if (!line.startsWith("data: ")) continue;
279
- const data = JSON.parse(line.slice(6));
280
-
281
- if (data.error) {
282
- bBubble.innerHTML = `<span class="text-rose-400">Error: ${data.error}</span>`;
283
- break;
284
  }
285
- if (data.token) {
286
- rawMarkdown += data.token;
287
- bBubble.innerHTML = marked.parse(rawMarkdown);
288
- document.getElementById("chat-box").scrollTop = document.getElementById("chat-box").scrollHeight;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
289
  }
 
 
 
 
 
 
 
 
 
 
 
 
290
 
291
- updateDashboard(data);
292
-
293
- if (data.ttft_ms !== undefined && data.total_ms !== undefined) {
294
- const totalSec = (data.total_ms / 1000).toFixed(2);
295
- bStats.innerHTML = `
296
- <span>⚑ <b>TTFT:</b> <span class="text-emerald-400">${data.ttft_ms} ms</span></span>
297
- <span>⏱️ <b>Total Time:</b> <span class="text-purple-400">${data.total_ms} ms (${totalSec}s)</span></span>
298
- <span>πŸš€ <b>Avg Speed:</b> <span class="text-sky-400">${data.tok_s} tok/s</span> (${data.ms_per_tok} ms/tok)</span>
299
- <span>πŸ”’ <b>Tokens:</b> ${data.tokens || 0}</span>
300
- <span>πŸ–₯️ <b>CPU:</b> <span class="text-amber-400">${data.cpu_pct}%</span></span>
301
- <span>🧠 <b>RAM:</b> <span class="text-rose-400">${data.ram_pct}%</span></span>
302
- `;
 
 
 
 
 
 
303
  }
304
- }
305
  }
306
- } catch (err) {
307
- bBubble.innerHTML = `<span class="text-rose-400">Connection error: ${err.message}</span>`;
308
- } finally {
309
- btn.disabled = false;
310
- input.focus();
311
- }
312
  }
313
 
314
- async function resetChat() {
315
- await fetch("/api/reset", {
316
- method: "POST",
317
- headers: { "Content-Type": "application/json" },
318
- body: JSON.stringify({ session_id: sessionId })
319
- });
320
- isFirstTurn = true;
321
- document.getElementById("stat-kv").innerText = "0 / 1024";
322
- document.getElementById("chat-box").innerHTML = `
323
- <div class="max-w-3xl mx-auto bg-slate-900/90 border border-slate-800 rounded-xl p-4 text-sm text-emerald-400">
324
- βœ… KV Cache cleared! Ready for a new conversation.
325
- </div>`;
326
  }
327
- </script>
328
- </body>
329
- </html>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /*
2
+ * ============================================================
3
+ * OmegaMind β€” Zero-Dependency C++17 AVX2/FMA Inference Engine
4
+ * Compatible with NOT-OMEGA/OmegaMind (model.bin & tokenizer.bin)
5
+ * Features:
6
+ * - Exact 5x int32 header mapping (n_layer, n_head, n_embd, block_size, vocab_size)
7
+ * - Batch Prefill (OpenMP + AVX2/FMA)
8
+ * - Multi-session LRU KV-Cache with auto-sliding window
9
+ * - Repetition Penalty + Temperature + Top-K + Top-P (Nucleus) Sampling
10
+ * ============================================================
11
+ */
12
+ #include <iostream>
13
+ #include <cstdio>
14
+ #include <cstdlib>
15
+ #include <cmath>
16
+ #include <cstring>
17
+ #include <ctime>
18
+ #include <cstdint>
19
+ #include <algorithm>
20
+ #include <string>
21
+ #include <vector>
22
+ #include <unordered_map>
23
+ #include <unordered_set>
24
+ #include <immintrin.h>
25
+
26
+ #ifdef _OPENMP
27
+ #include <omp.h>
28
+ #endif
29
+
30
+ #ifdef _WIN32
31
+ #include <windows.h>
32
+ static double get_ms() {
33
+ LARGE_INTEGER f, c;
34
+ QueryPerformanceFrequency(&f); QueryPerformanceCounter(&c);
35
+ return (double)c.QuadPart / f.QuadPart * 1000.0;
36
+ }
37
+ #else
38
+ #include <sys/time.h>
39
+ #include <sys/mman.h>
40
+ #include <sys/stat.h>
41
+ #include <fcntl.h>
42
+ #include <unistd.h>
43
+ static double get_ms() {
44
+ struct timeval tv; gettimeofday(&tv, NULL);
45
+ return tv.tv_sec * 1000.0 + tv.tv_usec / 1000.0;
46
+ }
47
+ #endif
48
+
49
+ static void* mmap_file(const char* filepath, size_t* size_out) {
50
+ #ifdef _WIN32
51
+ HANDLE hFile = CreateFileA(filepath, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
52
+ if (hFile == INVALID_HANDLE_VALUE) return nullptr;
53
+ LARGE_INTEGER liSize;
54
+ if (!GetFileSizeEx(hFile, &liSize)) { CloseHandle(hFile); return nullptr; }
55
+ *size_out = liSize.QuadPart;
56
+ HANDLE hMap = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, 0, NULL);
57
+ if (!hMap) { CloseHandle(hFile); return nullptr; }
58
+ void* ptr = MapViewOfFile(hMap, FILE_MAP_READ, 0, 0, 0);
59
+ CloseHandle(hMap);
60
+ CloseHandle(hFile);
61
+ return ptr;
62
+ #else
63
+ int fd = open(filepath, O_RDONLY);
64
+ if (fd < 0) return nullptr;
65
+ struct stat sb;
66
+ if (fstat(fd, &sb) < 0) { close(fd); return nullptr; }
67
+ *size_out = sb.st_size;
68
+ void* ptr = mmap(NULL, sb.st_size, PROT_READ, MAP_SHARED, fd, 0);
69
+ close(fd);
70
+ if (ptr == MAP_FAILED) return nullptr;
71
+ return ptr;
72
+ #endif
73
+ }
74
+
75
+ static thread_local uint64_t tl_rng_state = 0;
76
+ static inline float rng_float() {
77
+ if (!tl_rng_state) {
78
+ uint64_t tid = 0;
79
+ #ifdef _OPENMP
80
+ tid = (uint64_t)omp_get_thread_num();
81
+ #endif
82
+ tl_rng_state = (uint64_t)get_ms() ^ (uint64_t)(uintptr_t)&tl_rng_state ^ (tid * 6364136223846793005ULL + 1);
83
+ if (!tl_rng_state) tl_rng_state = 0xDEADBEEFCAFEBABEULL;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
84
  }
85
+ tl_rng_state ^= tl_rng_state << 13;
86
+ tl_rng_state ^= tl_rng_state >> 7;
87
+ tl_rng_state ^= tl_rng_state << 17;
88
+ return (float)(tl_rng_state >> 11) * (1.0f / (float)(1ULL << 53));
89
+ }
90
+
91
+ struct Config {
92
+ int n_layer;
93
+ int n_head;
94
+ int n_embd;
95
+ int block_size;
96
+ int vocab_size;
97
+ };
98
+
99
+ struct Weights {
100
+ float *wte, *wpe;
101
+ float **ln1_w, **ln1_b, **c_attn_w, **c_attn_b;
102
+ float **c_proj_w, **c_proj_b, **ln2_w, **ln2_b;
103
+ float **fc_w, **fc_b, **mlp_proj_w, **mlp_proj_b;
104
+ float *ln_f_w, *ln_f_b, *lm_head_w;
105
+ };
106
+
107
+ static Config cfg;
108
+ static Weights W;
109
+ static float* g_data = nullptr;
110
+
111
+ struct Session {
112
+ float* k_cache = nullptr;
113
+ float* v_cache = nullptr;
114
+ int pos = 0;
115
+ double last_use = 0.0;
116
+ std::vector<int> history;
117
+ };
118
+
119
+ static const int MAX_SESSIONS = 16;
120
+ static std::unordered_map<std::string, Session> g_sessions;
121
+
122
+ /* Single-token generation buffers */
123
+ static float *g_x, *g_buf, *g_qkv, *g_attn_buf;
124
+ static float *g_ff, *g_logits, *g_tmp_out;
125
+ static std::pair<float,int>* g_topk_pairs = nullptr;
126
+
127
+ /* Batch-prefill workspace */
128
+ static float *g_px = nullptr;
129
+ static float *g_pbuf = nullptr;
130
+ static float *g_pqkv = nullptr;
131
+
132
+ static thread_local float* tl_tmp = nullptr;
133
+ static thread_local float* tl_ff = nullptr;
134
+
135
+ static void ensure_thread_locals() {
136
+ if (!tl_tmp) {
137
+ tl_tmp = (float*)malloc(cfg.n_embd * sizeof(float));
138
+ tl_ff = (float*)malloc(4 * cfg.n_embd * sizeof(float));
139
+ if (!tl_tmp || !tl_ff) {
140
+ fprintf(stderr, "FATAL: thread-local buffer OOM\n");
141
+ exit(1);
142
+ }
143
+ }
144
+ }
145
+
146
+ /* ════════════════════════════════════════════════════
147
+ Core AVX2 / FMA Math Primitives
148
+ ════════════════════════════════════════════════════ */
149
+
150
+ static void layer_norm(float* out, const float* x, const float* w, const float* b, int N) {
151
+ __m256 vsum = _mm256_setzero_ps();
152
+ for (int i = 0; i < N; i += 8)
153
+ vsum = _mm256_add_ps(vsum, _mm256_loadu_ps(x + i));
154
+ float tmp[8]; _mm256_storeu_ps(tmp, vsum);
155
+ float mean = (tmp[0]+tmp[1]+tmp[2]+tmp[3]+tmp[4]+tmp[5]+tmp[6]+tmp[7]) / (float)N;
156
+
157
+ __m256 vmean = _mm256_set1_ps(mean);
158
+ __m256 vvar = _mm256_setzero_ps();
159
+ for (int i = 0; i < N; i += 8) {
160
+ __m256 d = _mm256_sub_ps(_mm256_loadu_ps(x + i), vmean);
161
+ vvar = _mm256_fmadd_ps(d, d, vvar);
162
+ }
163
+ _mm256_storeu_ps(tmp, vvar);
164
+ float var = (tmp[0]+tmp[1]+tmp[2]+tmp[3]+tmp[4]+tmp[5]+tmp[6]+tmp[7]) / (float)N;
165
+ __m256 vsc = _mm256_set1_ps(1.f / sqrtf(var + 1e-5f));
166
+
167
+ for (int i = 0; i < N; i += 8) {
168
+ __m256 d = _mm256_sub_ps(_mm256_loadu_ps(x + i), vmean);
169
+ __m256 norm = _mm256_mul_ps(d, vsc);
170
+ __m256 result = _mm256_fmadd_ps(norm, _mm256_loadu_ps(w + i), _mm256_loadu_ps(b + i));
171
+ _mm256_storeu_ps(out + i, result);
172
+ }
173
+ float sc = 1.f / sqrtf(var + 1e-5f);
174
+ for (int i = (N & ~7); i < N; i++)
175
+ out[i] = (x[i] - mean) * sc * w[i] + b[i];
176
+ }
177
+
178
+ static void matmul_vec(float* __restrict__ out, const float* __restrict__ mat,
179
+ const float* __restrict__ x, int M, int K) {
180
+ #pragma omp parallel for schedule(static) if(M >= 64)
181
+ for (int i = 0; i < M; i++) {
182
+ const float* row = mat + (long long)i * K;
183
+ __m256 acc = _mm256_setzero_ps();
184
+ int j = 0;
185
+ for (; j <= K-8; j += 8)
186
+ acc = _mm256_fmadd_ps(_mm256_loadu_ps(row+j), _mm256_loadu_ps(x+j), acc);
187
+ float t[8]; _mm256_storeu_ps(t, acc);
188
+ float s = t[0]+t[1]+t[2]+t[3]+t[4]+t[5]+t[6]+t[7];
189
+ for (; j < K; j++) s += row[j] * x[j];
190
+ out[i] = s;
191
+ }
192
+ }
193
+
194
+ static inline void matmul_vec_serial(float* __restrict__ out,
195
+ const float* __restrict__ mat,
196
+ const float* __restrict__ x,
197
+ int M, int K) {
198
+ for (int i = 0; i < M; i++) {
199
+ const float* row = mat + (long long)i * K;
200
+ __m256 acc = _mm256_setzero_ps();
201
+ int j = 0;
202
+ for (; j <= K-8; j += 8)
203
+ acc = _mm256_fmadd_ps(_mm256_loadu_ps(row+j), _mm256_loadu_ps(x+j), acc);
204
+ float t[8]; _mm256_storeu_ps(t, acc);
205
+ float s = t[0]+t[1]+t[2]+t[3]+t[4]+t[5]+t[6]+t[7];
206
+ for (; j < K; j++) s += row[j] * x[j];
207
+ out[i] = s;
208
+ }
209
+ }
210
+
211
+ static inline float dot_avx2(const float* __restrict__ a, const float* __restrict__ b, int n) {
212
+ __m256 acc = _mm256_setzero_ps();
213
+ int i = 0;
214
+ for (; i <= n-8; i += 8)
215
+ acc = _mm256_fmadd_ps(_mm256_loadu_ps(a+i), _mm256_loadu_ps(b+i), acc);
216
+ float tmp[8]; _mm256_storeu_ps(tmp, acc);
217
+ float s = tmp[0]+tmp[1]+tmp[2]+tmp[3]+tmp[4]+tmp[5]+t[6]+tmp[7];
218
+ for (; i < n; i++) s += a[i]*b[i];
219
+ return s;
220
+ }
221
+
222
+ static inline float dot_avx2_safe(const float* __restrict__ a, const float* __restrict__ b, int n) {
223
+ __m256 acc = _mm256_setzero_ps();
224
+ int i = 0;
225
+ for (; i <= n-8; i += 8)
226
+ acc = _mm256_fmadd_ps(_mm256_loadu_ps(a+i), _mm256_loadu_ps(b+i), acc);
227
+ float tmp[8]; _mm256_storeu_ps(tmp, acc);
228
+ float s = tmp[0]+tmp[1]+tmp[2]+tmp[3]+tmp[4]+tmp[5]+tmp[6]+tmp[7];
229
+ for (; i < n; i++) s += a[i]*b[i];
230
+ return s;
231
+ }
232
+
233
+ static inline void weighted_acc_avx2(float* __restrict__ out, const float* __restrict__ v, float w, int n) {
234
+ __m256 wv = _mm256_set1_ps(w);
235
+ int i = 0;
236
+ for (; i <= n-8; i += 8)
237
+ _mm256_storeu_ps(out+i, _mm256_fmadd_ps(wv, _mm256_loadu_ps(v+i), _mm256_loadu_ps(out+i)));
238
+ for (; i < n; i++) out[i] += w * v[i];
239
+ }
240
+
241
+ static inline void add_bias(float* x, const float* b, int N) {
242
+ for (int i = 0; i < N; i++) x[i] += b[i];
243
+ }
244
+
245
+ static inline void residual_add(float* x, const float* y, int N) {
246
+ for (int i = 0; i < N; i++) x[i] += y[i];
247
+ }
248
+
249
+ static void gelu_inplace(float* x, int N) {
250
+ for (int i = 0; i < N; i++) {
251
+ float v = x[i];
252
+ x[i] = 0.5f * v * (1.0f + tanhf(0.7978845608f * (v + 0.044715f * v * v * v)));
253
+ }
254
+ }
255
+
256
+ static void softmax_inplace(float* x, int N) {
257
+ float mx = x[0];
258
+ for (int i = 1; i < N; i++) if (x[i] > mx) mx = x[i];
259
+ float s = 0.f;
260
+ for (int i = 0; i < N; i++) { x[i] = expf(x[i]-mx); s += x[i]; }
261
+ for (int i = 0; i < N; i++) x[i] /= s;
262
+ }
263
+
264
+ /* ════════════════════════════════════════════════════
265
+ Forward Passes (Single-Token & Batch Prefill)
266
+ ════════════════════════════════════════════════════ */
267
+ static void forward(int token_id, int pos, float* k_cache, float* v_cache) {
268
+ const int C = cfg.n_embd, H = cfg.n_head, hs = C / H;
269
+ float* te = W.wte + (long long)token_id * C;
270
+ float* pe = W.wpe + (long long)pos * C;
271
+ #pragma omp parallel for
272
+ for (int i = 0; i < C; i++) g_x[i] = te[i] + pe[i];
273
+
274
+ for (int l = 0; l < cfg.n_layer; l++) {
275
+ layer_norm(g_buf, g_x, W.ln1_w[l], W.ln1_b[l], C);
276
+ matmul_vec(g_qkv, W.c_attn_w[l], g_buf, 3*C, C);
277
+ add_bias(g_qkv, W.c_attn_b[l], 3*C);
278
+
279
+ float* q = g_qkv;
280
+ float* k = g_qkv + C;
281
+ float* v = g_qkv + 2*C;
282
+ float* kc = k_cache + (long long)l * cfg.block_size * C;
283
+ float* vc = v_cache + (long long)l * cfg.block_size * C;
284
+ memcpy(kc + (long long)pos*C, k, C*sizeof(float));
285
+ memcpy(vc + (long long)pos*C, v, C*sizeof(float));
286
+
287
+ #pragma omp parallel for schedule(static)
288
+ for (int h = 0; h < H; h++) {
289
+ float* qh = q + h*hs;
290
+ float scale = 1.f / sqrtf((float)hs);
291
+ float* attn = g_attn_buf + h*cfg.block_size;
292
+ for (int t = 0; t <= pos; t++) {
293
+ float* kh = kc + (long long)t*C + h*hs;
294
+ attn[t] = dot_avx2_safe(qh, kh, hs) * scale;
295
+ }
296
+ softmax_inplace(attn, pos+1);
297
+ float* oh = g_buf + h*hs;
298
+ memset(oh, 0, hs*sizeof(float));
299
+ for (int t = 0; t <= pos; t++) {
300
+ float* vh = vc + (long long)t*C + h*hs;
301
+ weighted_acc_avx2(oh, vh, attn[t], hs);
302
+ }
303
+ }
304
+ matmul_vec(g_tmp_out, W.c_proj_w[l], g_buf, C, C);
305
+ add_bias(g_tmp_out, W.c_proj_b[l], C);
306
+ residual_add(g_x, g_tmp_out, C);
307
+
308
+ layer_norm(g_buf, g_x, W.ln2_w[l], W.ln2_b[l], C);
309
+ matmul_vec(g_ff, W.fc_w[l], g_buf, 4*C, C);
310
+ add_bias(g_ff, W.fc_b[l], 4*C);
311
+ gelu_inplace(g_ff, 4*C);
312
+ matmul_vec(g_tmp_out, W.mlp_proj_w[l], g_ff, C, 4*C);
313
+ add_bias(g_tmp_out, W.mlp_proj_b[l], C);
314
+ residual_add(g_x, g_tmp_out, C);
315
+ }
316
+ layer_norm(g_buf, g_x, W.ln_f_w, W.ln_f_b, C);
317
+ matmul_vec(g_logits, W.lm_head_w, g_buf, cfg.vocab_size, C);
318
+ }
319
+
320
+ static void forward_prefill_batch(const int* tokens, int T, int start_pos,
321
+ float* k_cache, float* v_cache) {
322
+ const int C = cfg.n_embd, H = cfg.n_head, hs = C / H;
323
+
324
+ for (int t = 0; t < T; t++) {
325
+ float* dst = g_px + (long long)t * C;
326
+ float* te = W.wte + (long long)tokens[t] * C;
327
+ float* pe = W.wpe + (long long)(start_pos + t) * C;
328
+ for (int i = 0; i < C; i++) dst[i] = te[i] + pe[i];
329
  }
330
 
331
+ for (int l = 0; l < cfg.n_layer; l++) {
332
+ float* kc = k_cache + (long long)l * cfg.block_size * C;
333
+ float* vc = v_cache + (long long)l * cfg.block_size * C;
334
+
335
+ for (int t = 0; t < T; t++)
336
+ layer_norm(g_pbuf + (long long)t*C, g_px + (long long)t*C, W.ln1_w[l], W.ln1_b[l], C);
337
+
338
+ #pragma omp parallel for schedule(static)
339
+ for (int t = 0; t < T; t++) {
340
+ float* qkv = g_pqkv + (long long)t * 3 * C;
341
+ matmul_vec_serial(qkv, W.c_attn_w[l], g_pbuf + (long long)t*C, 3*C, C);
342
+ const float* bias = W.c_attn_b[l];
343
+ for (int i = 0; i < 3*C; i++) qkv[i] += bias[i];
344
+ memcpy(kc + (long long)(start_pos + t)*C, qkv + C, C*sizeof(float));
345
+ memcpy(vc + (long long)(start_pos + t)*C, qkv + 2*C, C*sizeof(float));
346
+ }
347
+
348
+ memset(g_pbuf, 0, (long long)T * C * sizeof(float));
349
+ const float scale = 1.f / sqrtf((float)hs);
350
+
351
+ #pragma omp parallel for schedule(static)
352
+ for (int h = 0; h < H; h++) {
353
+ std::vector<float> attn_scores_vec(cfg.block_size);
354
+ float* attn_scores = attn_scores_vec.data();
355
+ for (int t = 0; t < T; t++) {
356
+ float* qh = g_pqkv + (long long)t*3*C + h*hs;
357
+ int seq_len = start_pos + t + 1;
358
+ for (int s = 0; s < seq_len; s++)
359
+ attn_scores[s] = dot_avx2_safe(qh, kc + (long long)s*C + h*hs, hs) * scale;
360
+ softmax_inplace(attn_scores, seq_len);
361
+ float* oh = g_pbuf + (long long)t*C + h*hs;
362
+ for (int s = 0; s < seq_len; s++)
363
+ weighted_acc_avx2(oh, vc + (long long)s*C + h*hs, attn_scores[s], hs);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
364
  }
365
+ }
366
+
367
+ #pragma omp parallel for schedule(static)
368
+ for (int t = 0; t < T; t++) {
369
+ ensure_thread_locals();
370
+ float* xt = g_px + (long long)t * C;
371
+ float* bt = g_pbuf + (long long)t * C;
372
+ matmul_vec_serial(tl_tmp, W.c_proj_w[l], bt, C, C);
373
+ const float* bias = W.c_proj_b[l];
374
+ for (int i = 0; i < C; i++) xt[i] += tl_tmp[i] + bias[i];
375
+ }
376
+
377
+ for (int t = 0; t < T; t++)
378
+ layer_norm(g_pbuf + (long long)t*C, g_px + (long long)t*C, W.ln2_w[l], W.ln2_b[l], C);
379
+
380
+ #pragma omp parallel for schedule(static)
381
+ for (int t = 0; t < T; t++) {
382
+ ensure_thread_locals();
383
+ float* xt = g_px + (long long)t * C;
384
+ float* bt = g_pbuf + (long long)t * C;
385
+ matmul_vec_serial(tl_ff, W.fc_w[l], bt, 4*C, C);
386
+ const float* bias_fc = W.fc_b[l];
387
+ for (int i = 0; i < 4*C; i++) tl_ff[i] += bias_fc[i];
388
+ gelu_inplace(tl_ff, 4*C);
389
+ matmul_vec_serial(tl_tmp, W.mlp_proj_w[l], tl_ff, C, 4*C);
390
+ const float* bias_p = W.mlp_proj_b[l];
391
+ for (int i = 0; i < C; i++) xt[i] += tl_tmp[i] + bias_p[i];
392
+ }
393
+ }
394
+
395
+ layer_norm(g_buf, g_px + (long long)(T-1)*C, W.ln_f_w, W.ln_f_b, C);
396
+ matmul_vec(g_logits, W.lm_head_w, g_buf, cfg.vocab_size, C);
397
+ }
398
+
399
+ /* ════════════════════════════════════════════════════
400
+ Weight Mapping
401
+ ════════════════════════════════════════════════════ */
402
+ static void map_weights(float* data, long long wbytes) {
403
+ float* p = data;
404
+ float* end = data + wbytes / sizeof(float);
405
+ const int C = cfg.n_embd, L = cfg.n_layer;
406
+
407
+ W.wte = p; p += (long long)cfg.vocab_size * C;
408
+ W.wpe = p; p += (long long)cfg.block_size * C;
409
+
410
+ #define ARR(f) W.f = (float**)malloc(L*sizeof(float*))
411
+ ARR(ln1_w); ARR(ln1_b); ARR(c_attn_w); ARR(c_attn_b);
412
+ ARR(c_proj_w); ARR(c_proj_b); ARR(ln2_w); ARR(ln2_b);
413
+ ARR(fc_w); ARR(fc_b); ARR(mlp_proj_w); ARR(mlp_proj_b);
414
+ #undef ARR
415
+
416
+ for (int l = 0; l < L; l++) {
417
+ W.ln1_w[l] = p; p += C;
418
+ W.ln1_b[l] = p; p += C;
419
+ W.c_attn_w[l] = p; p += 3LL*C*C;
420
+ W.c_attn_b[l] = p; p += 3LL*C;
421
+ W.c_proj_w[l] = p; p += 1LL*C*C;
422
+ W.c_proj_b[l] = p; p += C;
423
+ W.ln2_w[l] = p; p += C;
424
+ W.ln2_b[l] = p; p += C;
425
+ W.fc_w[l] = p; p += 4LL*C*C;
426
+ W.fc_b[l] = p; p += 4LL*C;
427
+ W.mlp_proj_w[l] = p; p += 1LL*C*4*C;
428
+ W.mlp_proj_b[l] = p; p += C;
429
+ }
430
+ W.ln_f_w = p; p += C;
431
+ W.ln_f_b = p; p += C;
432
+ W.lm_head_w = p; p += (long long)cfg.vocab_size * C;
433
+
434
+ if (p > end) {
435
+ long long needed = (p - data) * (long long)sizeof(float);
436
+ long long got = (end - data) * (long long)sizeof(float);
437
+ fprintf(stderr, "FATAL: model.bin truncated β€” needed %lld bytes, got %lld.\n", needed, got);
438
+ printf("ERROR model_truncated\n"); fflush(stdout);
439
+ exit(1);
440
+ }
441
+ }
442
+
443
+ /* ════════════════════════════════════════════════════
444
+ Session Management
445
+ ════════════════════════════════════════════════════ */
446
+ static long long kv_bytes() {
447
+ return (long long)cfg.n_layer * cfg.block_size * cfg.n_embd * sizeof(float);
448
+ }
449
+ static void free_session(Session& s) {
450
+ free(s.k_cache); free(s.v_cache);
451
+ s.k_cache = nullptr; s.v_cache = nullptr; s.pos = 0;
452
+ s.history.clear();
453
+ }
454
+ static void evict_oldest() {
455
+ if (g_sessions.empty()) return;
456
+ std::string oid; double ot = 1e300;
457
+ for (auto& kv : g_sessions)
458
+ if (kv.second.last_use < ot) { ot = kv.second.last_use; oid = kv.first; }
459
+ free_session(g_sessions[oid]);
460
+ g_sessions.erase(oid);
461
+ }
462
+ static Session& get_or_create(const std::string& id) {
463
+ auto it = g_sessions.find(id);
464
+ if (it != g_sessions.end()) { it->second.last_use = get_ms(); return it->second; }
465
+ if ((int)g_sessions.size() >= MAX_SESSIONS) evict_oldest();
466
+ long long nb = kv_bytes();
467
+ float* kc = (float*)calloc(nb, 1);
468
+ float* vc = (float*)calloc(nb, 1);
469
+ if (!kc || !vc) {
470
+ free(kc); free(vc);
471
+ printf("ERROR oom_kv_cache\n"); fflush(stdout);
472
+ static Session dummy;
473
+ return dummy;
474
+ }
475
+ auto pair = g_sessions.emplace(id, Session{});
476
+ auto ins_it = pair.first;
477
+ ins_it->second.k_cache = kc;
478
+ ins_it->second.v_cache = vc;
479
+ ins_it->second.pos = 0;
480
+ ins_it->second.last_use = get_ms();
481
+ return ins_it->second;
482
+ }
483
+
484
+ /* ════════════════════════════════════════════════════
485
+ Sampling: Repetition Penalty + Temp + Top-K + Top-P
486
+ ════════════════════════════════════════════════════ */
487
+ static int sample_token(float temperature, int top_k, float top_p, float rep_penalty,
488
+ const std::vector<int>& history) {
489
+ // 1. Apply Repetition Penalty on recent 64 tokens
490
+ if (rep_penalty > 1.0f && !history.empty()) {
491
+ int start_idx = std::max(0, (int)history.size() - 64);
492
+ std::unordered_set<int> recent(history.begin() + start_idx, history.end());
493
+ for (int tok_id : recent) {
494
+ if (tok_id >= 0 && tok_id < cfg.vocab_size) {
495
+ if (g_logits[tok_id] > 0.0f) g_logits[tok_id] /= rep_penalty;
496
+ else g_logits[tok_id] *= rep_penalty;
497
  }
498
+ }
499
+ }
500
+
501
+ // 2. Temperature scaling & Top-K partial sort
502
+ float inv_temp = 1.0f / std::max(temperature, 1e-4f);
503
+ for (int v = 0; v < cfg.vocab_size; v++) {
504
+ g_topk_pairs[v] = {g_logits[v] * inv_temp, v};
505
+ }
506
+
507
+ int K = std::min(std::max(top_k, 1), cfg.vocab_size);
508
+ std::partial_sort(g_topk_pairs, g_topk_pairs + K, g_topk_pairs + cfg.vocab_size,
509
+ [](const auto& a, const auto& b){ return a.first > b.first; });
510
 
511
+ // 3. Softmax over Top-K candidates
512
+ float max_logit = g_topk_pairs[0].first;
513
+ float sum = 0.f;
514
+ for (int j = 0; j < K; j++) {
515
+ g_topk_pairs[j].first = expf(g_topk_pairs[j].first - max_logit);
516
+ sum += g_topk_pairs[j].first;
517
+ }
518
+ for (int j = 0; j < K; j++) g_topk_pairs[j].first /= sum;
519
+
520
+ // 4. Top-P (Nucleus) cutoff inside Top-K
521
+ int active_k = K;
522
+ if (top_p > 0.0f && top_p < 1.0f) {
523
+ float cum_prob = 0.0f;
524
+ for (int j = 0; j < K; j++) {
525
+ cum_prob += g_topk_pairs[j].first;
526
+ if (cum_prob >= top_p) {
527
+ active_k = j + 1;
528
+ break;
529
  }
 
530
  }
531
+ float p_sum = 0.0f;
532
+ for (int j = 0; j < active_k; j++) p_sum += g_topk_pairs[j].first;
533
+ for (int j = 0; j < active_k; j++) g_topk_pairs[j].first /= p_sum;
 
 
 
534
  }
535
 
536
+ // 5. Multinomial sample
537
+ float r = rng_float(), cum = 0.f;
538
+ int best = g_topk_pairs[0].second;
539
+ for (int j = 0; j < active_k; j++) {
540
+ cum += g_topk_pairs[j].first;
541
+ if (r < cum) { best = g_topk_pairs[j].second; break; }
 
 
 
 
 
 
542
  }
543
+ return best;
544
+ }
545
+
546
+ /* ════════════════════════════════════════════════════
547
+ Request Protocol
548
+ Format: REQUEST|sess_id|tokens|max_new|temp|top_k|stop_ids|top_p|rep_penalty
549
+ ════════════════════════════════════════════════════ */
550
+ static std::vector<std::string> split(const std::string& s, char d) {
551
+ std::vector<std::string> out; std::string cur;
552
+ for (char c : s) { if (c==d){out.push_back(cur);cur.clear();}else cur+=c; }
553
+ out.push_back(cur); return out;
554
+ }
555
+ static std::vector<int> parse_ints(const std::string& s) {
556
+ std::vector<int> out;
557
+ for (auto& t : split(s,',')) if (!t.empty()) out.push_back(atoi(t.c_str()));
558
+ return out;
559
+ }
560
+
561
+ static void handle_request(const std::string& line) {
562
+ auto parts = split(line, '|');
563
+ if (parts.size() < 7) { printf("ERROR bad_request_format\n"); fflush(stdout); return; }
564
+
565
+ std::string sess_id = parts[1];
566
+ auto new_tokens = parse_ints(parts[2]);
567
+ int max_new = atoi(parts[3].c_str());
568
+ float temp = (float)atof(parts[4].c_str());
569
+ int top_k = atoi(parts[5].c_str());
570
+ auto stop_list = parse_ints(parts[6]);
571
+ float top_p = (parts.size() >= 8) ? (float)atof(parts[7].c_str()) : 0.85f;
572
+ float rep_penalty = (parts.size() >= 9) ? (float)atof(parts[8].c_str()) : 1.18f;
573
+
574
+ temp = std::max(temp, 0.01f);
575
+ top_k = std::min(std::max(top_k, 1), cfg.vocab_size);
576
+ max_new = std::max(max_new, 1);
577
+ top_p = std::min(std::max(top_p, 0.05f), 1.0f);
578
+ rep_penalty = std::max(rep_penalty, 1.0f);
579
+
580
+ std::unordered_set<int> stop_ids(stop_list.begin(), stop_list.end());
581
+ stop_ids.insert(50256);
582
+
583
+ Session& sess = get_or_create(sess_id);
584
+ if (!sess.k_cache) return;
585
+ if (new_tokens.empty()) { printf("ERROR no_tokens\n"); fflush(stdout); return; }
586
+
587
+ // If single prompt exceeds block_size, keep the most recent tokens
588
+ if ((int)new_tokens.size() >= cfg.block_size - 4) {
589
+ new_tokens.erase(new_tokens.begin(), new_tokens.end() - (cfg.block_size - max_new - 8));
590
+ }
591
+
592
+ // Auto-reset session KV position if context window fills up
593
+ if (sess.pos + (int)new_tokens.size() >= cfg.block_size) {
594
+ sess.pos = 0;
595
+ sess.history.clear();
596
+ }
597
+
598
+ for (int t : new_tokens) sess.history.push_back(t);
599
+
600
+ double t0 = get_ms();
601
+ forward_prefill_batch(new_tokens.data(), (int)new_tokens.size(),
602
+ sess.pos, sess.k_cache, sess.v_cache);
603
+ sess.pos += (int)new_tokens.size();
604
+
605
+ int gen = 0;
606
+ for (int i = 0; i < max_new; i++) {
607
+ if (sess.pos >= cfg.block_size) break;
608
+ int next = sample_token(temp, top_k, top_p, rep_penalty, sess.history);
609
+ sess.history.push_back(next);
610
+
611
+ printf("TOKEN %d %.2f %d\n", next, get_ms() - t0, sess.pos);
612
+ fflush(stdout);
613
+ gen++;
614
+
615
+ if (stop_ids.count(next)) break;
616
+ forward(next, sess.pos, sess.k_cache, sess.v_cache);
617
+ sess.pos++;
618
+ }
619
+ printf("DONE %d %.2f %d\n", gen, get_ms() - t0, sess.pos);
620
+ fflush(stdout);
621
+ }
622
+
623
+ static void handle_reset(const std::string& line) {
624
+ auto parts = split(line, '|');
625
+ if (parts.size() >= 2) {
626
+ auto it = g_sessions.find(parts[1]);
627
+ if (it != g_sessions.end()) { free_session(it->second); g_sessions.erase(it); }
628
+ }
629
+ printf("RESET_OK\n"); fflush(stdout);
630
+ }
631
+
632
+ int main() {
633
+ size_t mapped_size = 0;
634
+ void* file_data = mmap_file("model.bin", &mapped_size);
635
+ if (!file_data) { printf("ERROR model.bin_not_found\n"); fflush(stdout); return 1; }
636
+
637
+ /* FIX: Read all 5 int32 values directly into Config (n_layer, n_head, n_embd, block_size, vocab_size) */
638
+ constexpr size_t MODEL_HEADER_BYTES = 5 * sizeof(int);
639
+ if (mapped_size < MODEL_HEADER_BYTES) {
640
+ printf("ERROR model.bin_header_truncated\n"); fflush(stdout); return 1;
641
+ }
642
+
643
+ memcpy(&cfg, (char*)file_data, MODEL_HEADER_BYTES);
644
+ fprintf(stderr, "[OmegaMind C++] Loaded Header: layers=%d, heads=%d, embd=%d, block=%d, vocab=%d\n",
645
+ cfg.n_layer, cfg.n_head, cfg.n_embd, cfg.block_size, cfg.vocab_size);
646
+
647
+ long long wbytes = (long long)mapped_size - (long long)MODEL_HEADER_BYTES;
648
+ g_data = (float*)((char*)file_data + MODEL_HEADER_BYTES);
649
+ map_weights(g_data, wbytes);
650
+
651
+ const int C = cfg.n_embd;
652
+ g_x = (float*)malloc(C * sizeof(float));
653
+ g_buf = (float*)malloc(C * sizeof(float));
654
+ g_qkv = (float*)malloc(3*C * sizeof(float));
655
+ g_attn_buf = (float*)malloc((long long)cfg.n_head * cfg.block_size * sizeof(float));
656
+ g_ff = (float*)malloc(4*C * sizeof(float));
657
+ g_logits = (float*)malloc((long long)cfg.vocab_size * sizeof(float));
658
+ g_tmp_out = (float*)malloc(C * sizeof(float));
659
+ g_topk_pairs = (std::pair<float,int>*)malloc((long long)cfg.vocab_size * sizeof(std::pair<float,int>));
660
+
661
+ g_px = (float*)malloc((long long)cfg.block_size * C * sizeof(float));
662
+ g_pbuf = (float*)malloc((long long)cfg.block_size * C * sizeof(float));
663
+ g_pqkv = (float*)malloc((long long)cfg.block_size * 3 * C * sizeof(float));
664
+
665
+ if (!g_x || !g_buf || !g_qkv || !g_attn_buf || !g_ff || !g_logits ||
666
+ !g_tmp_out || !g_topk_pairs || !g_px || !g_pbuf || !g_pqkv) {
667
+ printf("ERROR oom_buffers\n"); fflush(stdout); return 1;
668
+ }
669
+
670
+ printf("READY\n"); fflush(stdout);
671
+
672
+ std::string line;
673
+ while (std::getline(std::cin, line)) {
674
+ if (!line.empty() && line.back()=='\r') line.pop_back();
675
+ if (line.empty()) continue;
676
+ if (line == "QUIT") break;
677
+ else if (line.rfind("RESET|", 0)==0) handle_reset(line);
678
+ else if (line.rfind("REQUEST|", 0)==0) handle_request(line);
679
+ else { printf("ERROR unknown_cmd\n"); fflush(stdout); }
680
+ }
681
+
682
+ for (auto& kv : g_sessions) free_session(kv.second);
683
+ free(g_x); free(g_buf); free(g_qkv); free(g_attn_buf);
684
+ free(g_ff); free(g_logits); free(g_tmp_out); free(g_topk_pairs);
685
+ free(g_px); free(g_pbuf); free(g_pqkv);
686
+ return 0;
687
+ }