Spaces:
Sleeping
Sleeping
Nicholas Bumgarner commited on
Commit ·
2ebca3e
1
Parent(s): 18881f2
Update mixer.c with actual ternary weight decoding from model
Browse files
mixer.c
CHANGED
|
@@ -142,6 +142,39 @@ static int parse_header(system_t *s, const char *path) {
|
|
| 142 |
return 0;
|
| 143 |
}
|
| 144 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 145 |
static float *get_embedding(system_t *s, int t) {
|
| 146 |
size_t emb_size = (size_t)s->V * s->D * sizeof(float);
|
| 147 |
char *emb_ptr = s->mmap_base + s->file_size - emb_size;
|
|
@@ -159,14 +192,47 @@ static float *get_lm_head_row(system_t *s, int d) {
|
|
| 159 |
return (float *)(s->mmap_base + lm_offset + (size_t)d * s->V);
|
| 160 |
}
|
| 161 |
|
| 162 |
-
|
|
|
|
|
|
|
|
|
|
| 163 |
r->tok = tok;
|
| 164 |
r->energy = 1.0f;
|
| 165 |
r->coherence = 1.0f;
|
|
|
|
| 166 |
for (int i = 0; i < N_OSC; i++) {
|
| 167 |
r->theta[i] = (2.0f * (float)M_PI * i / N_OSC) + ((float)tok * (float)M_PI / 4.0f);
|
| 168 |
-
|
| 169 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 170 |
}
|
| 171 |
}
|
| 172 |
|
|
@@ -284,11 +350,12 @@ int main(int argc, char **argv) {
|
|
| 284 |
int nt = sizeof(tokens)/sizeof(tokens[0]);
|
| 285 |
for (int i = 0; i < s.n_rings; i++) {
|
| 286 |
int tok = tokens[i % nt] + (i * 17) % 1000;
|
| 287 |
-
init_ring(&s.rings[i], tok);
|
| 288 |
}
|
| 289 |
} else {
|
|
|
|
| 290 |
for (int i = 0; i < s.n_rings; i++) {
|
| 291 |
-
|
| 292 |
}
|
| 293 |
}
|
| 294 |
|
|
|
|
| 142 |
return 0;
|
| 143 |
}
|
| 144 |
|
| 145 |
+
/* Decode packed ternary weight to oscillator lens (omega) — matches phoenix_brain.c */
|
| 146 |
+
static void decode_ternary_to_lens(const uint8_t *packed, int col, int stride, float *omega) {
|
| 147 |
+
memset(omega, 0, N_OSC * sizeof(float));
|
| 148 |
+
const uint8_t *w = packed + (size_t)col * stride;
|
| 149 |
+
for (int i = 0; i < N_OSC; i++) {
|
| 150 |
+
int byte_idx = i / 4;
|
| 151 |
+
int bit_idx = (i % 4) * 2;
|
| 152 |
+
if (byte_idx >= stride) break;
|
| 153 |
+
int val = (w[byte_idx] >> bit_idx) & 0x03;
|
| 154 |
+
if (val == 0) omega[i] = -0.5f; /* counter-rotating */
|
| 155 |
+
else if (val == 2) omega[i] = 0.5f; /* co-rotating */
|
| 156 |
+
/* val=1 or val=3 → 0.0 (zero weight, drops out) */
|
| 157 |
+
}
|
| 158 |
+
}
|
| 159 |
+
|
| 160 |
+
/* Get weight bytes for layer l, column c — matches phoenix_brain.c layout */
|
| 161 |
+
static const uint8_t *get_layer_weights(system_t *s, int layer, int col) {
|
| 162 |
+
/* Packed ternaries: 2 bits per weight, 4 per byte */
|
| 163 |
+
/* stride = ceil(D / 4) bytes per column */
|
| 164 |
+
size_t qw = (s->D * s->q_dim + 3) / 4;
|
| 165 |
+
size_t kw = (s->D * s->kv_dim + 3) / 4;
|
| 166 |
+
size_t ow = (s->q_dim * s->D + 3) / 4;
|
| 167 |
+
size_t gw = (s->D * s->FFN + 3) / 4;
|
| 168 |
+
size_t uw = (s->D * s->FFN + 3) / 4;
|
| 169 |
+
size_t layer_bytes = (qw + 2*kw + ow + gw + uw) * 3;
|
| 170 |
+
|
| 171 |
+
/* Weights start at offset 36 (32-byte header + 4-byte n_layers) */
|
| 172 |
+
char *base = s->mmap_base + 36;
|
| 173 |
+
/* Q weights: layer_bytes * layer, offset = col * (D/4) */
|
| 174 |
+
size_t col_stride = (s->D + 3) / 4; /* bytes per column */
|
| 175 |
+
return (const uint8_t *)(base + (size_t)layer * layer_bytes + (size_t)col * col_stride);
|
| 176 |
+
}
|
| 177 |
+
|
| 178 |
static float *get_embedding(system_t *s, int t) {
|
| 179 |
size_t emb_size = (size_t)s->V * s->D * sizeof(float);
|
| 180 |
char *emb_ptr = s->mmap_base + s->file_size - emb_size;
|
|
|
|
| 192 |
return (float *)(s->mmap_base + lm_offset + (size_t)d * s->V);
|
| 193 |
}
|
| 194 |
|
| 195 |
+
/* Initialize a ring from model weights at a given layer (matches phoenix_brain.c
|
| 196 |
+
* traveling wave activation: theta[i] = 2*pi*i/N + x[i]*pi/4,
|
| 197 |
+
* omega decoded from ternary weights, lens = gradient-weighted profile) */
|
| 198 |
+
static void init_ring_from_model(system_t *s, ring_t *r, int tok, int layer) {
|
| 199 |
r->tok = tok;
|
| 200 |
r->energy = 1.0f;
|
| 201 |
r->coherence = 1.0f;
|
| 202 |
+
/* Traveling wave activation */
|
| 203 |
for (int i = 0; i < N_OSC; i++) {
|
| 204 |
r->theta[i] = (2.0f * (float)M_PI * i / N_OSC) + ((float)tok * (float)M_PI / 4.0f);
|
| 205 |
+
}
|
| 206 |
+
/* Decode ternary weights from model into omega (lens profile) */
|
| 207 |
+
int col = layer % (s->D / N_OSC); /* which weight column for this layer */
|
| 208 |
+
size_t col_stride = (s->D + 3) / 4;
|
| 209 |
+
const uint8_t *wp = get_layer_weights(s, layer, 0);
|
| 210 |
+
decode_ternary_to_lens(wp, col, (int)col_stride, r->lens);
|
| 211 |
+
/* Copy lens to omega (same frequency encoding as phoenix) */
|
| 212 |
+
for (int i = 0; i < N_OSC; i++) {
|
| 213 |
+
r->omega[i] = r->lens[i];
|
| 214 |
+
}
|
| 215 |
+
}
|
| 216 |
+
|
| 217 |
+
static void init_ring(ring_t *r, int tok, int use_model) {
|
| 218 |
+
r->tok = tok;
|
| 219 |
+
r->energy = 1.0f;
|
| 220 |
+
r->coherence = 1.0f;
|
| 221 |
+
/* Traveling wave activation: theta[i] = 2*pi*i/N + x[i]*pi/4 */
|
| 222 |
+
for (int i = 0; i < N_OSC; i++) {
|
| 223 |
+
r->theta[i] = (2.0f * (float)M_PI * i / N_OSC) + ((float)tok * (float)M_PI / 4.0f);
|
| 224 |
+
}
|
| 225 |
+
if (use_model) {
|
| 226 |
+
/* Use lens profile from model weights */
|
| 227 |
+
for (int i = 0; i < N_OSC; i++) {
|
| 228 |
+
r->omega[i] = r->lens[i];
|
| 229 |
+
}
|
| 230 |
+
} else {
|
| 231 |
+
/* Placeholder for test mode */
|
| 232 |
+
for (int i = 0; i < N_OSC; i++) {
|
| 233 |
+
r->omega[i] = 1.0f + (float)(i - N_OSC/2) * 0.01f;
|
| 234 |
+
r->lens[i] = 1.0f;
|
| 235 |
+
}
|
| 236 |
}
|
| 237 |
}
|
| 238 |
|
|
|
|
| 350 |
int nt = sizeof(tokens)/sizeof(tokens[0]);
|
| 351 |
for (int i = 0; i < s.n_rings; i++) {
|
| 352 |
int tok = tokens[i % nt] + (i * 17) % 1000;
|
| 353 |
+
init_ring(&s.rings[i], tok, 0);
|
| 354 |
}
|
| 355 |
} else {
|
| 356 |
+
/* Model mode: decode ternary weights from model for each ring */
|
| 357 |
for (int i = 0; i < s.n_rings; i++) {
|
| 358 |
+
init_ring_from_model(&s, &s.rings[i], i * (s.V / s.n_rings), i);
|
| 359 |
}
|
| 360 |
}
|
| 361 |
|