File size: 7,472 Bytes
c41b3ce
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
df6d041
c41b3ce
 
 
 
 
 
 
 
 
 
df6d041
c41b3ce
df6d041
 
c41b3ce
 
 
 
 
 
 
df6d041
 
 
 
 
 
 
 
 
 
 
 
c41b3ce
 
 
 
 
 
 
 
2bff5be
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
c41b3ce
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2bff5be
 
 
 
 
 
 
 
c41b3ce
 
2bff5be
 
 
 
 
 
 
 
 
 
 
 
c41b3ce
 
 
 
 
 
 
2bff5be
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
#include <wasm_simd128.h>
typedef unsigned char u8;
typedef signed char i8;
typedef unsigned int u32;
static inline float maximum(float a,float b){return a>b?a:b;}

static inline float scale_at(const u8 *p) {
    float f;
    __builtin_memcpy(&f,p,4);
    return f;
}

/* Q4 rows are groups of 32. Each byte packs columns j and j+16. */
void unpack(const u8 *w,float *out,int n) {
    for(int i=0;i<n;i+=32,w+=20) {
        float scale=scale_at(w);
        for(int k=0;k<16;k++) {
            out[i+k]=scale*((w[4+k]&15)-8);
            out[i+k+16]=scale*((w[4+k]>>4)-8);
        }
    }
}

static inline v128_t dot16(v128_t a,v128_t b) {
    v128_t low=wasm_i32x4_dot_i16x8(wasm_i16x8_extend_low_i8x16(a),wasm_i16x8_extend_low_i8x16(b));
    v128_t high=wasm_i32x4_dot_i16x8(wasm_i16x8_extend_high_i8x16(a),wasm_i16x8_extend_high_i8x16(b));
    return wasm_i32x4_add(low,high);
}

/* Quantize activations per group; a GEMV reads only the selected bank. */
static inline __attribute__((always_inline)) void matvec(const u8 *w,const float *x,float *y,int rows,int cols,i8 *activ,float *scales, int bits) {
    for(int j=0;j<cols;j+=32) {
        float peak=1e-12f;
        for(int k=0;k<32;k++) peak=maximum(peak,__builtin_fabsf(x[j+k]));
        float scale=peak/127.0f;
        scales[j/32]=scale;
        for(int k=0;k<32;k++) activ[j+k]=(i8)__builtin_nearbyintf(x[j+k]/scale);
    }
    const v128_t mask=wasm_i8x16_splat(15), zero=wasm_i8x16_splat(8);
    for(int i=0;i<rows;i++) {
        v128_t acc=wasm_f32x4_splat(0);
        for(int j=0;j<cols;j+=32,w+=(bits==4?20:36)) {
            v128_t p=wasm_v128_load(w+4);
            v128_t lo=bits==4?wasm_i8x16_sub(wasm_v128_and(p,mask),zero):p;
            v128_t hi=bits==4?wasm_i8x16_sub(wasm_u8x16_shr(p,4),zero):wasm_v128_load(w+20);
            v128_t d=wasm_i32x4_add(dot16(lo,wasm_v128_load(activ+j)),dot16(hi,wasm_v128_load(activ+j+16)));
            v128_t s=wasm_f32x4_splat(scale_at(w)*scales[j/32]);
            acc=wasm_f32x4_add(acc,wasm_f32x4_mul(s,wasm_f32x4_convert_i32x4(d)));
        }
        y[i]=wasm_f32x4_extract_lane(acc,0)+wasm_f32x4_extract_lane(acc,1)+wasm_f32x4_extract_lane(acc,2)+wasm_f32x4_extract_lane(acc,3);
    }
}
void gemv(const u8 *w,const float *x,float *y,int rows,int cols,i8 *activ,float *scales) {
    matvec(w,x,y,rows,cols,activ,scales,4);
}
void gemv8(const u8 *w,const float *x,float *y,int rows,int cols,i8 *activ,float *scales) {
    matvec(w,x,y,rows,cols,activ,scales,8);
}
void unpack8(const u8 *w,float *out,int n) {
    for(int j=0;j<n;j+=32,w+=36){
        float s=scale_at(w);
        for(int k=0;k<32;k++)out[j+k]=s*(float)((const i8*)(w+4))[k];
    }
}
void rms(const float *x,const float *g,float *out,int n) {
    float sum=0;
    for(int j=0;j<n;j++)sum+=x[j]*x[j];
    float s=1/__builtin_sqrtf(sum/n+1e-5f);
    for(int j=0;j<n;j++)out[j]=x[j]*g[j]*s;
}
void add(float *x,const float *v,int n){for(int j=0;j<n;j++)x[j]+=v[j];}

/* The two nonlinear kernels need exp(x) only for x <= 0. Reduce x to a
 * half-log(2) interval and evaluate exp(r) through the sixth-order term.
 * Splitting log(2) keeps the residual accurate near the FP32 exponent floor. */
static inline v128_t exp_nonpositive(v128_t x) {
    x = wasm_f32x4_max(x, wasm_f32x4_splat(-87.0f));
    v128_t exponent = wasm_f32x4_nearest(wasm_f32x4_mul(x, wasm_f32x4_splat(1.44269504089f)));
    v128_t r = wasm_f32x4_sub(x, wasm_f32x4_mul(exponent, wasm_f32x4_splat(0.693359375f)));
    r = wasm_f32x4_sub(r, wasm_f32x4_mul(exponent, wasm_f32x4_splat(-0.00021219444f)));
    v128_t p = wasm_f32x4_splat(1.0f/720);
    p = wasm_f32x4_add(wasm_f32x4_mul(p,r), wasm_f32x4_splat(1.0f/120));
    p = wasm_f32x4_add(wasm_f32x4_mul(p,r), wasm_f32x4_splat(1.0f/24));
    p = wasm_f32x4_add(wasm_f32x4_mul(p,r), wasm_f32x4_splat(1.0f/6));
    p = wasm_f32x4_add(wasm_f32x4_mul(p,r), wasm_f32x4_splat(0.5f));
    p = wasm_f32x4_add(wasm_f32x4_mul(p,r), wasm_f32x4_splat(1.0f));
    p = wasm_f32x4_add(wasm_f32x4_mul(p,r), wasm_f32x4_splat(1.0f));
    v128_t bits=wasm_i32x4_add(wasm_i32x4_trunc_sat_f32x4(exponent),wasm_i32x4_splat(127));
    return wasm_f32x4_mul(p,wasm_i32x4_shl(bits,23));
}

static inline float sum_lanes(v128_t v) {
    return wasm_f32x4_extract_lane(v,0) + wasm_f32x4_extract_lane(v,1)
         + wasm_f32x4_extract_lane(v,2) + wasm_f32x4_extract_lane(v,3);
}

void swiglu(float *h, int n) {
    int i=0;
    const v128_t one=wasm_f32x4_splat(1.f);
    for (;i+4<=n;i+=4) {
        v128_t x=wasm_v128_load(h+i);
        v128_t z=exp_nonpositive(wasm_f32x4_neg(wasm_f32x4_abs(x)));
        v128_t numerator=wasm_v128_bitselect(one,z,wasm_f32x4_ge(x,wasm_f32x4_splat(0)));
        v128_t sig=wasm_f32x4_div(numerator,wasm_f32x4_add(one,z));
        wasm_v128_store(h+i,wasm_f32x4_mul(wasm_f32x4_mul(x,sig),wasm_v128_load(h+n+i)));
    }
    for (;i<n;i++) {
        float x=h[i], z=wasm_f32x4_extract_lane(exp_nonpositive(wasm_f32x4_splat(-__builtin_fabsf(x))),0);
        h[i]=x*h[n+i]*(x<0?z:1.f)/(1.f+z);
    }
}

void rotary(float *x,const float *c,const float *s,int heads,int hd) {
    for(int h=0;h<heads;h++,x+=hd) for(int j=0;j<hd;j+=2){
        float even=x[j],odd=x[j+1];
        x[j]=even*c[j/2]-odd*s[j/2];x[j+1]=even*s[j/2]+odd*c[j/2];
    }
}

/* Cached keys/values are [time,kvhead,head_dim]. */
void attend(const float *q,const float *keys,const float *values,float *out,float *scores,int n,int heads,int kv,int hd) {
    float norm=1/__builtin_sqrtf((float)hd);
    for(int h=0;h<heads;h++) {
        int kh=h/(heads/kv);
        float max=-1e30f;
        for(int t=0;t<n;t++){
            const float *key=keys+(t*kv+kh)*hd;
            const float *query=q+h*hd;
            v128_t accum=wasm_f32x4_splat(0);
            int j=0;
            for (;j+4<=hd;j+=4)
                accum=wasm_f32x4_add(accum,wasm_f32x4_mul(wasm_v128_load(query+j),wasm_v128_load(key+j)));
            float v=sum_lanes(accum);
            for (;j<hd;j++)v+=query[j]*key[j];
            scores[t]=v*norm;max=maximum(max,scores[t]);
        }
        v128_t chunks=wasm_f32x4_splat(0), peak=wasm_f32x4_splat(max);
        int t=0;
        for (;t+4<=n;t+=4) {
            v128_t mass=exp_nonpositive(wasm_f32x4_sub(wasm_v128_load(scores+t),peak));
            wasm_v128_store(scores+t,mass);
            chunks=wasm_f32x4_add(chunks,mass);
        }
        float z=sum_lanes(chunks);
        for (;t<n;t++) {
            scores[t]=wasm_f32x4_extract_lane(exp_nonpositive(wasm_f32x4_splat(scores[t]-max)),0);
            z+=scores[t];
        }
        for(int j=0;j<hd;j++)out[h*hd+j]=0;
        for(int t=0;t<n;t++){
            float a=scores[t]/z;
            for(int j=0;j<hd;j++)out[h*hd+j]+=a*values[(t*kv+kh)*hd+j];
        }
    }
}

/* Small FP32 readout matrices have no quantization stage. */
void linear_f32(const float *weights, const float *x, const float *bias, float *out, int rows, int cols) {
    for (int i = 0; i < rows; i++) {
        const float *row = weights + i * cols;
        v128_t sum = wasm_f32x4_splat(0);
        int j = 0;
        for (; j + 4 <= cols; j += 4)
            sum = wasm_f32x4_add(sum, wasm_f32x4_mul(wasm_v128_load(row + j), wasm_v128_load(x + j)));
        float value = wasm_f32x4_extract_lane(sum, 0) + wasm_f32x4_extract_lane(sum, 1)
                    + wasm_f32x4_extract_lane(sum, 2) + wasm_f32x4_extract_lane(sum, 3);
        for (; j < cols; j++) value += row[j] * x[j];
        out[i] = value + (bias ? bias[i] : 0);
    }
}