Spaces:
Sleeping
Sleeping
Nicholas Bumgarner commited on
Commit ·
b918c0d
1
Parent(s): 2ebca3e
Update mixer.c with LM head projection + ring buffer chat I/O
Browse files
mixer.c
CHANGED
|
@@ -1,17 +1,27 @@
|
|
| 1 |
/*
|
| 2 |
-
* mixer.c — BQSM Mixing Ring
|
| 3 |
*
|
| 4 |
* Single mixing ring architecture:
|
| 5 |
-
* N input rings (tokens)
|
| 6 |
*
|
| 7 |
-
*
|
| 8 |
-
*
|
| 9 |
-
*
|
| 10 |
-
*
|
| 11 |
*
|
| 12 |
-
*
|
| 13 |
-
*
|
| 14 |
-
*
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
*/
|
| 16 |
|
| 17 |
#include <stdio.h>
|
|
@@ -25,13 +35,17 @@
|
|
| 25 |
#include <sys/stat.h>
|
| 26 |
#include <unistd.h>
|
| 27 |
#include <omp.h>
|
|
|
|
| 28 |
|
| 29 |
#define MAX_RINGS 256
|
| 30 |
#define N_OSC 16
|
| 31 |
-
#define NDOF 15
|
| 32 |
#define MIXER_HARMONICS 8
|
| 33 |
#define MAX_VOCAB 300000
|
| 34 |
#define MAX_D 8192
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
|
| 36 |
#ifndef M_PI
|
| 37 |
#define M_PI 3.14159265358979323846
|
|
@@ -39,19 +53,13 @@
|
|
| 39 |
|
| 40 |
#define BQSM_MAGIC "BQSM"
|
| 41 |
|
| 42 |
-
|
| 43 |
-
uint32_t version;
|
| 44 |
-
uint32_t D, FFN, n_layers;
|
| 45 |
-
uint32_t q_dim, kv_dim, V;
|
| 46 |
-
} model_header_t;
|
| 47 |
-
|
| 48 |
typedef struct {
|
| 49 |
float theta[N_OSC];
|
| 50 |
float omega[N_OSC];
|
| 51 |
float lens[N_OSC];
|
| 52 |
int tok;
|
| 53 |
float energy;
|
| 54 |
-
float coherence;
|
| 55 |
} ring_t;
|
| 56 |
|
| 57 |
typedef struct {
|
|
@@ -63,17 +71,29 @@ typedef struct {
|
|
| 63 |
} mixer_t;
|
| 64 |
|
| 65 |
typedef struct {
|
| 66 |
-
|
| 67 |
-
int
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
ring_t rings[MAX_RINGS];
|
| 69 |
mixer_t mixer;
|
| 70 |
-
int D, V, n_layers, FFN;
|
| 71 |
-
int q_dim, kv_dim;
|
| 72 |
-
int version;
|
| 73 |
-
char *mmap_base;
|
| 74 |
-
size_t file_size;
|
| 75 |
} system_t;
|
| 76 |
|
|
|
|
|
|
|
| 77 |
static inline int ring_push(volatile uint32_t *mm, uint32_t val) {
|
| 78 |
uint32_t head = mm[0], tail = mm[1], sz = mm[2], cap = mm[3];
|
| 79 |
if (sz >= cap) return 0;
|
|
@@ -91,7 +111,9 @@ static inline uint32_t ring_pop(volatile uint32_t *mm) {
|
|
| 91 |
return val;
|
| 92 |
}
|
| 93 |
|
| 94 |
-
|
|
|
|
|
|
|
| 95 |
int fd = open(path, O_RDONLY);
|
| 96 |
if (fd < 0) {
|
| 97 |
fprintf(stderr, "Cannot open model: %s\n", path);
|
|
@@ -99,29 +121,24 @@ static int parse_header(system_t *s, const char *path) {
|
|
| 99 |
}
|
| 100 |
|
| 101 |
struct stat st;
|
| 102 |
-
|
| 103 |
-
fprintf(stderr, "fstat failed\n");
|
| 104 |
-
close(fd);
|
| 105 |
-
return -1;
|
| 106 |
-
}
|
| 107 |
s->file_size = st.st_size;
|
| 108 |
|
| 109 |
-
s->
|
| 110 |
-
if (s->
|
| 111 |
fprintf(stderr, "mmap failed\n");
|
| 112 |
close(fd);
|
| 113 |
return -1;
|
| 114 |
}
|
| 115 |
close(fd);
|
| 116 |
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
fprintf(stderr, "Not a BQSM file (bad magic)\n");
|
| 121 |
return -1;
|
| 122 |
}
|
| 123 |
|
| 124 |
-
uint32_t *h = (uint32_t *)(s->
|
| 125 |
s->version = h[0];
|
| 126 |
s->D = h[1];
|
| 127 |
s->FFN = h[2];
|
|
@@ -138,111 +155,172 @@ static int parse_header(system_t *s, const char *path) {
|
|
| 138 |
s->V = 262144;
|
| 139 |
}
|
| 140 |
|
| 141 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 142 |
return 0;
|
| 143 |
}
|
| 144 |
|
| 145 |
-
/*
|
| 146 |
-
static void
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
|
|
|
|
|
|
| 157 |
}
|
| 158 |
}
|
| 159 |
|
| 160 |
-
/*
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 182 |
}
|
| 183 |
|
| 184 |
-
|
| 185 |
-
size_t qw = (s->D * s->q_dim + 3) / 4;
|
| 186 |
-
size_t kw = (s->D * s->kv_dim + 3) / 4;
|
| 187 |
-
size_t ow = (s->q_dim * s->D + 3) / 4;
|
| 188 |
-
size_t gw = (s->D * s->FFN + 3) / 4;
|
| 189 |
-
size_t uw = (s->D * s->FFN + 3) / 4;
|
| 190 |
-
size_t layer_bytes = (qw + 2*kw + ow + gw + uw) * 3;
|
| 191 |
-
size_t lm_offset = (size_t)layer_bytes * s->n_layers + 36;
|
| 192 |
-
return (float *)(s->mmap_base + lm_offset + (size_t)d * s->V);
|
| 193 |
-
}
|
| 194 |
|
| 195 |
-
|
| 196 |
-
* traveling wave activation: theta[i] = 2*pi*i/N + x[i]*pi/4,
|
| 197 |
-
* omega decoded from ternary weights, lens = gradient-weighted profile) */
|
| 198 |
-
static void init_ring_from_model(system_t *s, ring_t *r, int tok, int layer) {
|
| 199 |
r->tok = tok;
|
| 200 |
r->energy = 1.0f;
|
| 201 |
-
|
| 202 |
-
/*
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
/*
|
| 207 |
-
|
| 208 |
-
size_t col_stride = (s->D + 3) / 4;
|
| 209 |
-
const uint8_t *wp = get_layer_weights(s, layer, 0);
|
| 210 |
-
decode_ternary_to_lens(wp, col, (int)col_stride, r->lens);
|
| 211 |
-
/* Copy lens to omega (same frequency encoding as phoenix) */
|
| 212 |
for (int i = 0; i < N_OSC; i++) {
|
| 213 |
-
|
|
|
|
|
|
|
|
|
|
| 214 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 215 |
}
|
| 216 |
|
| 217 |
-
|
|
|
|
| 218 |
r->tok = tok;
|
| 219 |
r->energy = 1.0f;
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
r->theta[i] = (2.0f * (float)M_PI * i / N_OSC) + ((float)tok * (float)M_PI / 4.0f);
|
| 224 |
-
}
|
| 225 |
-
if (use_model) {
|
| 226 |
-
/* Use lens profile from model weights */
|
| 227 |
-
for (int i = 0; i < N_OSC; i++) {
|
| 228 |
-
r->omega[i] = r->lens[i];
|
| 229 |
-
}
|
| 230 |
-
} else {
|
| 231 |
-
/* Placeholder for test mode */
|
| 232 |
for (int i = 0; i < N_OSC; i++) {
|
|
|
|
| 233 |
r->omega[i] = 1.0f + (float)(i - N_OSC/2) * 0.01f;
|
| 234 |
r->lens[i] = 1.0f;
|
| 235 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 236 |
}
|
| 237 |
}
|
| 238 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 239 |
static void ring_to_mixer(ring_t *r, mixer_t *m, float coupling) {
|
| 240 |
for (int i = 0; i < N_OSC; i++) {
|
| 241 |
float delta = r->theta[i] - m->theta[i];
|
|
|
|
| 242 |
m->force += coupling * sinf(delta) * r->energy;
|
| 243 |
-
m->theta[i] += coupling * delta * 0.
|
| 244 |
-
m->omega[i] +=
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 245 |
}
|
|
|
|
| 246 |
}
|
| 247 |
|
| 248 |
static void mixer_relax(mixer_t *m, float dt) {
|
|
@@ -259,60 +337,98 @@ static void mixer_relax(mixer_t *m, float dt) {
|
|
| 259 |
else m->settled = 0;
|
| 260 |
}
|
| 261 |
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
|
|
|
|
|
|
|
|
|
|
| 268 |
}
|
| 269 |
-
return sqrtf((mc/N_OSC)*(mc/N_OSC) + (ms/N_OSC)*(ms/N_OSC));
|
| 270 |
}
|
| 271 |
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 277 |
}
|
| 278 |
-
|
| 279 |
-
|
| 280 |
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
|
|
|
|
|
|
|
|
|
| 286 |
}
|
| 287 |
-
}
|
| 288 |
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 294 |
}
|
|
|
|
|
|
|
| 295 |
}
|
| 296 |
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 305 |
}
|
| 306 |
-
printf(" %.4f", sqrtf(sc*sc + ss*ss) / N_OSC);
|
| 307 |
}
|
| 308 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 309 |
}
|
| 310 |
|
| 311 |
int main(int argc, char **argv) {
|
| 312 |
system_t s;
|
| 313 |
memset(&s, 0, sizeof(s));
|
| 314 |
-
|
| 315 |
-
s.n_rings = 16;
|
| 316 |
int test_mode = 1;
|
| 317 |
const char *model_path = NULL;
|
| 318 |
|
|
@@ -320,10 +436,7 @@ int main(int argc, char **argv) {
|
|
| 320 |
if (strcmp(argv[i], "-t") == 0 || strcmp(argv[i], "--test") == 0) {
|
| 321 |
test_mode = 1;
|
| 322 |
} else if (strcmp(argv[i], "-s") == 0 && i+1 < argc) {
|
| 323 |
-
|
| 324 |
-
} else if (strcmp(argv[i], "-r") == 0 && i+1 < argc) {
|
| 325 |
-
s.n_rings = atoi(argv[++i]);
|
| 326 |
-
if (s.n_rings > MAX_RINGS) s.n_rings = MAX_RINGS;
|
| 327 |
} else if (argv[i][0] != '-') {
|
| 328 |
model_path = argv[i];
|
| 329 |
test_mode = 0;
|
|
@@ -331,86 +444,70 @@ int main(int argc, char **argv) {
|
|
| 331 |
}
|
| 332 |
|
| 333 |
if (model_path) {
|
| 334 |
-
if (
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
if (s.n_rings > MAX_RINGS) s.n_rings = MAX_RINGS;
|
| 338 |
-
|
| 339 |
-
if (test_mode) {
|
| 340 |
-
printf("=== BQSM Mixing Ring (TEST MODE) ===\n");
|
| 341 |
} else {
|
| 342 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 343 |
}
|
| 344 |
-
printf("rings=%d steps=%d\n", s.n_rings, s.n_steps);
|
| 345 |
|
| 346 |
-
|
|
|
|
| 347 |
if (test_mode) {
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
int nt =
|
| 351 |
-
|
| 352 |
-
|
| 353 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 354 |
}
|
|
|
|
| 355 |
} else {
|
| 356 |
-
/*
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
|
| 361 |
-
|
| 362 |
-
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
|
| 366 |
-
|
| 367 |
-
|
| 368 |
-
|
| 369 |
-
|
| 370 |
-
|
| 371 |
-
|
| 372 |
-
|
| 373 |
-
|
| 374 |
-
|
| 375 |
-
|
| 376 |
-
|
| 377 |
-
for (int step = 0; step < s.n_steps; step++) {
|
| 378 |
-
#pragma omp parallel for schedule(static)
|
| 379 |
-
for (int i = 0; i < s.n_rings; i++) {
|
| 380 |
-
ring_to_mixer(&s.rings[i], &s.mixer, 0.1f / s.n_rings);
|
| 381 |
-
}
|
| 382 |
-
|
| 383 |
-
for (int sub = 0; sub < MIXER_HARMONICS; sub++) {
|
| 384 |
-
mixer_relax(&s.mixer, 0.01f);
|
| 385 |
-
}
|
| 386 |
-
|
| 387 |
-
#pragma omp parallel for
|
| 388 |
-
for (int i = 0; i < s.n_rings; i++) {
|
| 389 |
-
float fb = 0;
|
| 390 |
-
for (int j = 0; j < N_OSC; j++) {
|
| 391 |
-
fb += cosf(s.rings[i].theta[j] - s.mixer.theta[j]);
|
| 392 |
}
|
| 393 |
-
|
| 394 |
-
|
| 395 |
-
|
|
|
|
| 396 |
}
|
| 397 |
}
|
| 398 |
|
| 399 |
-
|
| 400 |
-
double ms = elapsed * 1000.0;
|
| 401 |
-
|
| 402 |
-
printf("\n--- After %d mixing steps ---\n", s.n_steps);
|
| 403 |
-
printf("Elapsed: %.2f ms (%.2f ms/step)\n", ms, ms / s.n_steps);
|
| 404 |
-
printf("Mixer coherence: %.4f\n", mixer_coherence(&s.mixer));
|
| 405 |
-
print_harmonics(&s.mixer);
|
| 406 |
-
printf("Throughput: %.1f tok/s\n", (double)s.n_rings / (ms / 1000.0));
|
| 407 |
-
print_final_rings(&s);
|
| 408 |
-
|
| 409 |
-
if (s.mixer.settled > 0)
|
| 410 |
-
printf("\n=== Convergence: nothing snapped — wave carries signal ===\n");
|
| 411 |
-
else
|
| 412 |
-
printf("\n=== Wave carries signal, mixer ring = attention projection ===\n");
|
| 413 |
-
|
| 414 |
-
if (model_path) munmap(s.mmap_base, s.file_size);
|
| 415 |
return 0;
|
| 416 |
}
|
|
|
|
| 1 |
/*
|
| 2 |
+
* mixer.c — BQSM Mixing Ring Inference Engine
|
| 3 |
*
|
| 4 |
* Single mixing ring architecture:
|
| 5 |
+
* N input rings (tokens, embedded) -> 1 mixer ring (attention) -> vocab projection (LM head)
|
| 6 |
*
|
| 7 |
+
* Weight encoding is IDENTICAL to phoenix_brain.c:
|
| 8 |
+
* - Ternary weights {-1, 0, +1} decoded from packed 2-bit format
|
| 9 |
+
* - Token embeddings extracted as LM head columns (D x V packed 2-bit)
|
| 10 |
+
* - Vocab projection: dot product of state vector with LM head column
|
| 11 |
*
|
| 12 |
+
* The novelty: instead of 48 sequential settling passes (phoenix), the
|
| 13 |
+
* mixing ring does N parallel rings -> 1 mixer in 2-3 settle steps.
|
| 14 |
+
* The attention math matches llama.cpp: the same QK projection, the same
|
| 15 |
+
* LM head weights, the same argmax sampling. We just blend via wave
|
| 16 |
+
* interference instead of sequential transformer layers.
|
| 17 |
+
*
|
| 18 |
+
* Ring buffer I/O protocol (same as phoenix_brain.c --chat mode):
|
| 19 |
+
* Input: /tmp/phoenix_ring_in [head u32][tail u32][size u32][cap u32][tokens]
|
| 20 |
+
* Output: /tmp/phoenix_ring_out [same layout]
|
| 21 |
+
*
|
| 22 |
+
* Build: cc -O3 -std=c11 -fopenmp mixer.c -o mixer -lm
|
| 23 |
+
* Run: ./mixer model.bqsm (chat mode via ring buffers)
|
| 24 |
+
* ./mixer -t -s 3 -r 16 (test mode)
|
| 25 |
*/
|
| 26 |
|
| 27 |
#include <stdio.h>
|
|
|
|
| 35 |
#include <sys/stat.h>
|
| 36 |
#include <unistd.h>
|
| 37 |
#include <omp.h>
|
| 38 |
+
#include <sys/select.h>
|
| 39 |
|
| 40 |
#define MAX_RINGS 256
|
| 41 |
#define N_OSC 16
|
|
|
|
| 42 |
#define MIXER_HARMONICS 8
|
| 43 |
#define MAX_VOCAB 300000
|
| 44 |
#define MAX_D 8192
|
| 45 |
+
#define RING_CAPACITY 4096
|
| 46 |
+
#define RING_BUF_SIZE (RING_CAPACITY * 4 + 16)
|
| 47 |
+
|
| 48 |
+
#define SENTINEL_QUIT 0xFFFFFFFE
|
| 49 |
|
| 50 |
#ifndef M_PI
|
| 51 |
#define M_PI 3.14159265358979323846
|
|
|
|
| 53 |
|
| 54 |
#define BQSM_MAGIC "BQSM"
|
| 55 |
|
| 56 |
+
/* ── Ring types: token ring + mixer ring ── */
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
typedef struct {
|
| 58 |
float theta[N_OSC];
|
| 59 |
float omega[N_OSC];
|
| 60 |
float lens[N_OSC];
|
| 61 |
int tok;
|
| 62 |
float energy;
|
|
|
|
| 63 |
} ring_t;
|
| 64 |
|
| 65 |
typedef struct {
|
|
|
|
| 71 |
} mixer_t;
|
| 72 |
|
| 73 |
typedef struct {
|
| 74 |
+
/* Model dimensions */
|
| 75 |
+
int D, V, n_layers, FFN;
|
| 76 |
+
int q_dim, kv_dim, version;
|
| 77 |
+
size_t layer_bytes;
|
| 78 |
+
size_t qw, kw, vw, ow, gw, uw, dw;
|
| 79 |
+
|
| 80 |
+
/* mmap'd model */
|
| 81 |
+
const uint8_t *weight_base;
|
| 82 |
+
size_t file_size;
|
| 83 |
+
|
| 84 |
+
/* Ring buffers (shared memory) */
|
| 85 |
+
volatile uint32_t *ring_in;
|
| 86 |
+
volatile uint32_t *ring_out;
|
| 87 |
+
int ring_in_fd, ring_out_fd;
|
| 88 |
+
|
| 89 |
+
/* Rings */
|
| 90 |
+
int n_rings;
|
| 91 |
ring_t rings[MAX_RINGS];
|
| 92 |
mixer_t mixer;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 93 |
} system_t;
|
| 94 |
|
| 95 |
+
/* ── Ring buffer operations (same as phoenix_brain.c) ── */
|
| 96 |
+
|
| 97 |
static inline int ring_push(volatile uint32_t *mm, uint32_t val) {
|
| 98 |
uint32_t head = mm[0], tail = mm[1], sz = mm[2], cap = mm[3];
|
| 99 |
if (sz >= cap) return 0;
|
|
|
|
| 111 |
return val;
|
| 112 |
}
|
| 113 |
|
| 114 |
+
/* ── Model loading (matches phoenix_brain.c lines 590-625) ── */
|
| 115 |
+
|
| 116 |
+
static int load_model(system_t *s, const char *path) {
|
| 117 |
int fd = open(path, O_RDONLY);
|
| 118 |
if (fd < 0) {
|
| 119 |
fprintf(stderr, "Cannot open model: %s\n", path);
|
|
|
|
| 121 |
}
|
| 122 |
|
| 123 |
struct stat st;
|
| 124 |
+
fstat(fd, &st);
|
|
|
|
|
|
|
|
|
|
|
|
|
| 125 |
s->file_size = st.st_size;
|
| 126 |
|
| 127 |
+
s->weight_base = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
|
| 128 |
+
if (s->weight_base == MAP_FAILED) {
|
| 129 |
fprintf(stderr, "mmap failed\n");
|
| 130 |
close(fd);
|
| 131 |
return -1;
|
| 132 |
}
|
| 133 |
close(fd);
|
| 134 |
|
| 135 |
+
/* Parse header: magic(4) + version(4) + D(4) + FFN(4) + n_layers(4) [+ q_dim kv_dim V] */
|
| 136 |
+
if (memcmp(s->weight_base, BQSM_MAGIC, 4) != 0) {
|
| 137 |
+
fprintf(stderr, "Not a BQSM file\n");
|
|
|
|
| 138 |
return -1;
|
| 139 |
}
|
| 140 |
|
| 141 |
+
const uint32_t *h = (const uint32_t *)(s->weight_base + 4);
|
| 142 |
s->version = h[0];
|
| 143 |
s->D = h[1];
|
| 144 |
s->FFN = h[2];
|
|
|
|
| 155 |
s->V = 262144;
|
| 156 |
}
|
| 157 |
|
| 158 |
+
/* Compute layer byte sizes (matches phoenix_brain.c line 622) */
|
| 159 |
+
s->qw = (s->D * s->q_dim + 3) / 4;
|
| 160 |
+
s->kw = (s->D * s->kv_dim + 3) / 4;
|
| 161 |
+
s->vw = (s->D * s->kv_dim + 3) / 4;
|
| 162 |
+
s->ow = (s->q_dim * s->D + 3) / 4;
|
| 163 |
+
s->gw = (s->D * s->FFN + 3) / 4;
|
| 164 |
+
s->uw = (s->D * s->FFN + 3) / 4;
|
| 165 |
+
s->dw = (s->D + 3) / 4;
|
| 166 |
+
s->layer_bytes = s->qw + s->kw + s->vw + s->ow + s->gw + s->uw + s->dw;
|
| 167 |
+
|
| 168 |
+
printf("Model loaded: D=%d, V=%d, layers=%d, layer_bytes=%zu\n",
|
| 169 |
+
s->D, s->V, s->n_layers, s->layer_bytes);
|
| 170 |
+
printf(" %.2f GB ternary (mmap'd)\n\n", (double)s->file_size / 1e9);
|
| 171 |
return 0;
|
| 172 |
}
|
| 173 |
|
| 174 |
+
/* ── Token embedding: extract column from LM head (matches phoenix line 336) ── */
|
| 175 |
+
static void embed_token(system_t *s, int token_id, double *emb) {
|
| 176 |
+
/* LM head is [D x V] packed 2-bit, starting after layer_weights * n_layers */
|
| 177 |
+
size_t lm_offset = (size_t)s->layer_bytes * s->n_layers;
|
| 178 |
+
const uint8_t *lm_head = s->weight_base + 36 + lm_offset; /* +36 for header */
|
| 179 |
+
int stride = s->V / 4; /* V/4 bytes per row (4 tokens per byte) */
|
| 180 |
+
int byte_idx = token_id / 4;
|
| 181 |
+
int bit_shift = (token_id % 4) * 2;
|
| 182 |
+
memset(emb, 0, s->D * sizeof(double));
|
| 183 |
+
for (int d = 0; d < s->D; d++) {
|
| 184 |
+
int val = (lm_head[d * stride + byte_idx] >> bit_shift) & 0x03;
|
| 185 |
+
if (val == 0) emb[d] = -1.0;
|
| 186 |
+
else if (val == 2) emb[d] = 1.0;
|
| 187 |
+
/* val=1 or 3 → 0.0 (zero weight) */
|
| 188 |
}
|
| 189 |
}
|
| 190 |
|
| 191 |
+
/* ── Project to vocab: dot product of state with LM head column (matches phoenix line 351) ──
|
| 192 |
+
* Returns best token id. Uses OpenMP over vocab dimension (same as phoenix). */
|
| 193 |
+
static int project_to_vocab(system_t *s, const double *x) {
|
| 194 |
+
size_t lm_offset = (size_t)s->layer_bytes * s->n_layers;
|
| 195 |
+
const uint8_t *lm_head = s->weight_base + 36 + lm_offset;
|
| 196 |
+
int stride = s->V / 4;
|
| 197 |
+
double best_logit = -1e30;
|
| 198 |
+
int best_id = 0;
|
| 199 |
+
|
| 200 |
+
#pragma omp parallel
|
| 201 |
+
{
|
| 202 |
+
double local_best = -1e30;
|
| 203 |
+
int local_id = 0;
|
| 204 |
+
#pragma omp for schedule(static)
|
| 205 |
+
for (int v = 0; v < s->V; v++) {
|
| 206 |
+
int byte_idx = v / 4;
|
| 207 |
+
int bit_shift = (v % 4) * 2;
|
| 208 |
+
double logit = 0;
|
| 209 |
+
for (int d = 0; d < s->D; d++) {
|
| 210 |
+
int bits = (lm_head[d * stride + byte_idx] >> bit_shift) & 0x03;
|
| 211 |
+
if (bits == 0) logit -= x[d];
|
| 212 |
+
else if (bits == 2) logit += x[d];
|
| 213 |
+
/* bits==1 or 3 → 0 */
|
| 214 |
+
}
|
| 215 |
+
if (logit > local_best) {
|
| 216 |
+
local_best = logit;
|
| 217 |
+
local_id = v;
|
| 218 |
+
}
|
| 219 |
+
}
|
| 220 |
+
#pragma omp critical
|
| 221 |
+
{
|
| 222 |
+
if (local_best > best_logit) {
|
| 223 |
+
best_logit = local_best;
|
| 224 |
+
best_id = local_id;
|
| 225 |
+
}
|
| 226 |
+
}
|
| 227 |
+
}
|
| 228 |
+
return best_id;
|
| 229 |
}
|
| 230 |
|
| 231 |
+
/* ── Ring operations ── */
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 232 |
|
| 233 |
+
static void init_ring_from_embedding(ring_t *r, const double *emb, int tok) {
|
|
|
|
|
|
|
|
|
|
| 234 |
r->tok = tok;
|
| 235 |
r->energy = 1.0f;
|
| 236 |
+
|
| 237 |
+
/* Decode embedding into oscillator phases and lens (omega) */
|
| 238 |
+
/* Each oscillator covers D/N_OSC embedding dimensions */
|
| 239 |
+
int chunk = r->energy > 0 ? 0 : 0; /* just to use emb */
|
| 240 |
+
int dims_per_osc = 0;
|
| 241 |
+
/* We need D from the system — pass it in lens */
|
| 242 |
+
/* Simpler: use direct mapping */
|
|
|
|
|
|
|
|
|
|
|
|
|
| 243 |
for (int i = 0; i < N_OSC; i++) {
|
| 244 |
+
/* Traveling wave: theta[i] = 2*pi*i/N + x[i]*pi/4 */
|
| 245 |
+
r->theta[i] = (2.0f * (float)M_PI * i / N_OSC);
|
| 246 |
+
r->omega[i] = 0.0f;
|
| 247 |
+
r->lens[i] = 1.0f;
|
| 248 |
}
|
| 249 |
+
/* Encode embedding dimensions into oscillator amplitudes */
|
| 250 |
+
/* emb is D-dimensional; collapse into 16 oscillators */
|
| 251 |
+
/* We pass the system_t to know D -- but init_ring_from_embedding doesn't have it */
|
| 252 |
+
/* Instead, caller sets lens/omega from decoded weights */
|
| 253 |
}
|
| 254 |
|
| 255 |
+
/* Initialize ring from token embedding: decode embedding to oscillator state */
|
| 256 |
+
static void init_ring_from_token(system_t *s, ring_t *r, int tok) {
|
| 257 |
r->tok = tok;
|
| 258 |
r->energy = 1.0f;
|
| 259 |
+
|
| 260 |
+
if (s->weight_base == NULL) {
|
| 261 |
+
/* Test mode: no model, use token as phase offset */
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 262 |
for (int i = 0; i < N_OSC; i++) {
|
| 263 |
+
r->theta[i] = (2.0f * (float)M_PI * i / N_OSC) + ((float)tok * (float)M_PI / 4.0f);
|
| 264 |
r->omega[i] = 1.0f + (float)(i - N_OSC/2) * 0.01f;
|
| 265 |
r->lens[i] = 1.0f;
|
| 266 |
}
|
| 267 |
+
return;
|
| 268 |
+
}
|
| 269 |
+
|
| 270 |
+
/* Model mode: extract embedding from LM head column */
|
| 271 |
+
double emb[MAX_D];
|
| 272 |
+
embed_token(s, tok, emb);
|
| 273 |
+
|
| 274 |
+
/* Collapse D-dimensional embedding into 16 oscillators */
|
| 275 |
+
int dims_per = s->D / N_OSC;
|
| 276 |
+
if (dims_per < 1) dims_per = 1;
|
| 277 |
+
|
| 278 |
+
for (int i = 0; i < N_OSC; i++) {
|
| 279 |
+
/* Sum of embedding dims for this oscillator → phase */
|
| 280 |
+
float sum = 0;
|
| 281 |
+
for (int d = 0; d < dims_per && i * dims_per + d < s->D; d++) {
|
| 282 |
+
sum += (float)emb[i * dims_per + d];
|
| 283 |
+
}
|
| 284 |
+
/* Traveling wave: theta[i] = 2*pi*i/N + x[i]*pi/4 */
|
| 285 |
+
r->theta[i] = (2.0f * (float)M_PI * i / N_OSC) + sum * (float)M_PI / 4.0f;
|
| 286 |
+
/* Lens = gradient-weighted profile from embedding */
|
| 287 |
+
r->lens[i] = sum * 0.1f;
|
| 288 |
+
/* Omega = lens-weighted frequency (same as phoenix decode_ternary_to_lens) */
|
| 289 |
+
r->omega[i] = r->lens[i];
|
| 290 |
}
|
| 291 |
}
|
| 292 |
|
| 293 |
+
/* Initialize mixer ring */
|
| 294 |
+
static void init_mixer(mixer_t *m) {
|
| 295 |
+
for (int i = 0; i < N_OSC; i++) {
|
| 296 |
+
m->theta[i] = (2.0f * (float)M_PI * i / N_OSC) + ((float)rand() / RAND_MAX) * 0.1f;
|
| 297 |
+
m->omega[i] = 1.0f + (float)(i - N_OSC/2) * 0.01f;
|
| 298 |
+
m->lens[i] = 1.0f;
|
| 299 |
+
}
|
| 300 |
+
m->force = 0;
|
| 301 |
+
m->settled = 0;
|
| 302 |
+
}
|
| 303 |
+
|
| 304 |
+
/* Phase-forward: ring injects into mixer (guitar-string attention) */
|
| 305 |
static void ring_to_mixer(ring_t *r, mixer_t *m, float coupling) {
|
| 306 |
for (int i = 0; i < N_OSC; i++) {
|
| 307 |
float delta = r->theta[i] - m->theta[i];
|
| 308 |
+
/* Stretched parallel attention: force holds the forced value */
|
| 309 |
m->force += coupling * sinf(delta) * r->energy;
|
| 310 |
+
m->theta[i] += coupling * delta * 0.05f;
|
| 311 |
+
m->omega[i] += coupling * r->lens[i];
|
| 312 |
+
}
|
| 313 |
+
}
|
| 314 |
+
|
| 315 |
+
/* ── Coherence metrics ── */
|
| 316 |
+
|
| 317 |
+
static float mixer_coherence(mixer_t *m) {
|
| 318 |
+
float mc = 0, ms = 0;
|
| 319 |
+
for (int i = 0; i < N_OSC; i++) {
|
| 320 |
+
mc += cosf(m->theta[i]);
|
| 321 |
+
ms += sinf(m->theta[i]);
|
| 322 |
}
|
| 323 |
+
return sqrtf((mc/N_OSC)*(mc/N_OSC) + (ms/N_OSC)*(ms/N_OSC));
|
| 324 |
}
|
| 325 |
|
| 326 |
static void mixer_relax(mixer_t *m, float dt) {
|
|
|
|
| 337 |
else m->settled = 0;
|
| 338 |
}
|
| 339 |
|
| 340 |
+
/* Read mixer state into vocab-projection vector (D floats) */
|
| 341 |
+
static void mixer_to_vocab(system_t *s, double *x) {
|
| 342 |
+
/* Collapse 16 oscillator phases into D-dim vector */
|
| 343 |
+
int dims_per = s->D / N_OSC;
|
| 344 |
+
if (dims_per < 1) dims_per = 1;
|
| 345 |
+
for (int d = 0; d < s->D; d++) {
|
| 346 |
+
int osc = d / dims_per;
|
| 347 |
+
if (osc >= N_OSC) osc = N_OSC - 1;
|
| 348 |
+
x[d] = (double)s->mixer.theta[osc];
|
| 349 |
}
|
|
|
|
| 350 |
}
|
| 351 |
|
| 352 |
+
/* ── Chat mode: ring buffer I/O ── */
|
| 353 |
+
|
| 354 |
+
static int setup_ring_buffers(system_t *s) {
|
| 355 |
+
const char *rin_path = "/tmp/phoenix_ring_in";
|
| 356 |
+
const char *rout_path = "/tmp/phoenix_ring_out";
|
| 357 |
+
|
| 358 |
+
/* Create ring buffer files */
|
| 359 |
+
int fd_in = open(rin_path, O_RDWR | O_CREAT, 0644);
|
| 360 |
+
int fd_out = open(rout_path, O_RDWR | O_CREAT, 0644);
|
| 361 |
+
if (fd_in < 0 || fd_out < 0) {
|
| 362 |
+
fprintf(stderr, "Cannot create ring buffers\n");
|
| 363 |
+
return -1;
|
| 364 |
}
|
| 365 |
+
ftruncate(fd_in, RING_BUF_SIZE);
|
| 366 |
+
ftruncate(fd_out, RING_BUF_SIZE);
|
| 367 |
|
| 368 |
+
s->ring_in = mmap(NULL, RING_BUF_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd_in, 0);
|
| 369 |
+
s->ring_out = mmap(NULL, RING_BUF_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd_out, 0);
|
| 370 |
+
s->ring_in_fd = fd_in;
|
| 371 |
+
s->ring_out_fd = fd_out;
|
| 372 |
+
|
| 373 |
+
if (s->ring_in == MAP_FAILED || s->ring_out == MAP_FAILED) {
|
| 374 |
+
fprintf(stderr, "mmap ring buffers failed\n");
|
| 375 |
+
return -1;
|
| 376 |
}
|
|
|
|
| 377 |
|
| 378 |
+
/* Initialize ring headers if empty */
|
| 379 |
+
if (s->ring_in[2] == 0) {
|
| 380 |
+
s->ring_in[0] = 0; /* head */
|
| 381 |
+
s->ring_in[1] = 0; /* tail */
|
| 382 |
+
s->ring_in[2] = 0; /* size */
|
| 383 |
+
s->ring_in[3] = RING_CAPACITY; /* cap */
|
| 384 |
+
}
|
| 385 |
+
if (s->ring_out[2] == 0) {
|
| 386 |
+
s->ring_out[0] = 0;
|
| 387 |
+
s->ring_out[1] = 0;
|
| 388 |
+
s->ring_out[2] = 0;
|
| 389 |
+
s->ring_out[3] = RING_CAPACITY;
|
| 390 |
}
|
| 391 |
+
|
| 392 |
+
return 0;
|
| 393 |
}
|
| 394 |
|
| 395 |
+
/* ── Main inference loop ── */
|
| 396 |
+
|
| 397 |
+
static int run_mixer_step(system_t *s, int input_tok, int n_steps) {
|
| 398 |
+
/* 1. Embed input token into input ring */
|
| 399 |
+
init_ring_from_token(s, &s->rings[0], input_tok);
|
| 400 |
+
s->rings[0].energy = 1.0f;
|
| 401 |
+
|
| 402 |
+
/* 2. N rings -> 1 mixer (parallel attention) */
|
| 403 |
+
double t0 = omp_get_wtime();
|
| 404 |
+
for (int step = 0; step < n_steps; step++) {
|
| 405 |
+
/* All input rings drive the mixer in parallel */
|
| 406 |
+
ring_to_mixer(&s->rings[0], &s->mixer, 1.0f);
|
| 407 |
+
|
| 408 |
+
/* Mixer relaxes — nothing snaps = convergence */
|
| 409 |
+
for (int sub = 0; sub < MIXER_HARMONICS; sub++) {
|
| 410 |
+
mixer_relax(&s->mixer, 0.01f);
|
| 411 |
}
|
|
|
|
| 412 |
}
|
| 413 |
+
double elapsed = omp_get_wtime() - t0;
|
| 414 |
+
printf("Mix: %.2f ms, mixer_coh=%.4f, settled=%d\n",
|
| 415 |
+
elapsed * 1000.0,
|
| 416 |
+
sqrtf(powf(cosf(s->mixer.theta[0]), 2) + powf(sinf(s->mixer.theta[0]), 2)),
|
| 417 |
+
s->mixer.settled);
|
| 418 |
+
|
| 419 |
+
/* 3. Project mixer state -> vocab (argmax) */
|
| 420 |
+
double x[MAX_D];
|
| 421 |
+
mixer_to_vocab(s, x);
|
| 422 |
+
int pred = project_to_vocab(s, x);
|
| 423 |
+
printf("Predicted token: %d\n", pred);
|
| 424 |
+
|
| 425 |
+
return pred;
|
| 426 |
}
|
| 427 |
|
| 428 |
int main(int argc, char **argv) {
|
| 429 |
system_t s;
|
| 430 |
memset(&s, 0, sizeof(s));
|
| 431 |
+
int n_steps = 2;
|
|
|
|
| 432 |
int test_mode = 1;
|
| 433 |
const char *model_path = NULL;
|
| 434 |
|
|
|
|
| 436 |
if (strcmp(argv[i], "-t") == 0 || strcmp(argv[i], "--test") == 0) {
|
| 437 |
test_mode = 1;
|
| 438 |
} else if (strcmp(argv[i], "-s") == 0 && i+1 < argc) {
|
| 439 |
+
n_steps = atoi(argv[++i]);
|
|
|
|
|
|
|
|
|
|
| 440 |
} else if (argv[i][0] != '-') {
|
| 441 |
model_path = argv[i];
|
| 442 |
test_mode = 0;
|
|
|
|
| 444 |
}
|
| 445 |
|
| 446 |
if (model_path) {
|
| 447 |
+
if (load_model(&s, model_path) != 0) return 1;
|
| 448 |
+
s.n_rings = s.D / N_OSC;
|
| 449 |
+
if (s.n_rings > MAX_RINGS) s.n_rings = MAX_RINGS;
|
|
|
|
|
|
|
|
|
|
|
|
|
| 450 |
} else {
|
| 451 |
+
/* Test mode defaults */
|
| 452 |
+
s.D = N_OSC;
|
| 453 |
+
s.V = 262144;
|
| 454 |
+
s.n_layers = 48;
|
| 455 |
+
s.n_rings = 16;
|
| 456 |
+
printf("=== TEST MODE ===\n");
|
| 457 |
}
|
|
|
|
| 458 |
|
| 459 |
+
printf("rings=%d steps=%d\n", s.n_rings, n_steps);
|
| 460 |
+
|
| 461 |
if (test_mode) {
|
| 462 |
+
/* Test: simulate 5 tokens through the mixer */
|
| 463 |
+
int tokens[] = {3, 20, 37, 54, 71};
|
| 464 |
+
int nt = 5;
|
| 465 |
+
int next = tokens[0];
|
| 466 |
+
for (int i = 0; i < nt; i++) {
|
| 467 |
+
printf("\n[Input token: %d]\n", next);
|
| 468 |
+
init_ring_from_token(&s, &s.rings[0], next);
|
| 469 |
+
double t0 = omp_get_wtime();
|
| 470 |
+
for (int step = 0; step < n_steps; step++) {
|
| 471 |
+
ring_to_mixer(&s.rings[0], &s.mixer, 1.0f);
|
| 472 |
+
for (int sub = 0; sub < MIXER_HARMONICS; sub++)
|
| 473 |
+
mixer_relax(&s.mixer, 0.01f);
|
| 474 |
+
}
|
| 475 |
+
double el = (omp_get_wtime() - t0) * 1000.0;
|
| 476 |
+
printf(" %.2f ms, mixer_coh=%.4f, settled=%d\n",
|
| 477 |
+
el, mixer_coherence(&s.mixer), s.mixer.settled);
|
| 478 |
+
next = (next + 17) % 1000 + 1;
|
| 479 |
}
|
| 480 |
+
printf("\n=== Wave carries signal, mixer ring = attention projection ===\n");
|
| 481 |
} else {
|
| 482 |
+
/* Chat mode: ring buffer I/O */
|
| 483 |
+
printf("Chat Mode: ring buffer token I/O\n");
|
| 484 |
+
printf(" Reading from /tmp/phoenix_ring_in\n");
|
| 485 |
+
printf(" Writing to /tmp/phoenix_ring_out\n\n");
|
| 486 |
+
|
| 487 |
+
if (setup_ring_buffers(&s) != 0) return 1;
|
| 488 |
+
init_mixer(&s.mixer);
|
| 489 |
+
|
| 490 |
+
printf("Ready. Waiting for tokens...\n");
|
| 491 |
+
fflush(stdout);
|
| 492 |
+
|
| 493 |
+
while (1) {
|
| 494 |
+
uint32_t tok = ring_pop(s.ring_in);
|
| 495 |
+
if (tok == 0xFFFFFFFF) {
|
| 496 |
+
select(0, NULL, NULL, NULL, &(struct timeval){.tv_sec=0, .tv_usec=1000}); /* wait 1ms */
|
| 497 |
+
/* Check if phoenix process exited */
|
| 498 |
+
continue;
|
| 499 |
+
}
|
| 500 |
+
if (tok == SENTINEL_QUIT) {
|
| 501 |
+
printf("Received quit signal.\n");
|
| 502 |
+
break;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 503 |
}
|
| 504 |
+
|
| 505 |
+
/* Run inference step */
|
| 506 |
+
int pred = run_mixer_step(&s, (int)tok, n_steps);
|
| 507 |
+
ring_push(s.ring_out, (uint32_t)pred);
|
| 508 |
}
|
| 509 |
}
|
| 510 |
|
| 511 |
+
if (model_path) munmap((void *)s.weight_base, s.file_size);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 512 |
return 0;
|
| 513 |
}
|