Scikit-learn
human-activity-recognition
wearable
wrist
time-series
cpu
scikit-learn
WISP / src /wisp_fast /cis_conv.c
Zipeng365's picture
wisp-har 0.2.0: accelerated implementation with speed (exact/fast) and HMM sequence (subject/recording) options
ac7f7cb verified
Raw History Blame Contribute Delete
38.6 kB
/* Packed random convolution, preserving the tap accumulation order. The
multichannel exact path reuses the BLAS already linked into NumPy's
core, including its choice of fused operations. */
#define _GNU_SOURCE
#define PY_SSIZE_T_CLEAN
#include <Python.h>
#include <math.h>
#include <float.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#ifndef _WIN32
#include <dlfcn.h>
#endif
typedef void (*gemv64_t)(int,int,int64_t,int64_t,double,const double*,int64_t,
const double*,int64_t,double,double*,int64_t);
static gemv64_t np_gemv;
static int attempted;
static int bind_numpy(void) {
if (!attempted) {
attempted = 1;
#ifndef _WIN32
PyObject *core = PyImport_ImportModule("numpy._core._multiarray_umath");
if (!core) { PyErr_Clear(); return 0; }
PyObject *file = PyModule_GetFilenameObject(core);
Py_DECREF(core);
if (file) {
const char *path = PyUnicode_AsUTF8(file);
void *h = path ? dlopen(path, RTLD_LAZY | RTLD_LOCAL) : NULL;
if (h) {
void *symbol = dlsym(h, "scipy_cblas_dgemv64_");
if (symbol) memcpy(&np_gemv, &symbol, sizeof np_gemv);
/* The NumPy module keeps this dependency loaded. */
dlclose(h);
}
Py_DECREF(file);
}
if (PyErr_Occurred()) PyErr_Clear();
#endif
}
return np_gemv != NULL;
}
static PyObject *has_blas(PyObject *self,PyObject *args) {
return PyBool_FromLong(bind_numpy());
}
/* This order is specific to the inspected NumPy OpenBLAS SkylakeX kernel
shipped with the pinned Linux x86-64 wheel. Python additionally verifies
the library's full SHA-256. Other builds use actual calls to GEMV. */
#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) && !defined(_WIN32)
#include <immintrin.h>
#define HAVE_FAST_X86 1
/* Four vectors share weight broadcasts. Keep pertap contributions in four
registers and only materialize responses once, after the last tap. */
__attribute__((target("avx2,fma")))
static void skylakex_project(double *r,const double *base,Py_ssize_t lda,Py_ssize_t frames,
const double *weights,int c,int64_t len,int64_t dilation) {
Py_ssize_t j=0;
for (;j+16<=frames;j+=16) {
__m256d acc[4]={_mm256_setzero_pd(),_mm256_setzero_pd(),_mm256_setzero_pd(),_mm256_setzero_pd()};
for(int h=0;h<len;h++){
const double *v=weights+3*h,*a=base+j+h*dilation;
__m256d w0=_mm256_set1_pd(v[0]),w1=_mm256_set1_pd(v[1]),w2=_mm256_set1_pd(v[2]);
for(int q=0;q<4;q++) {
__m256d dot=_mm256_mul_pd(_mm256_loadu_pd(a+4*q+lda),w1);
dot=_mm256_fmadd_pd(_mm256_loadu_pd(a+4*q),w0,dot);
if(c==3) dot=_mm256_add_pd(dot,_mm256_mul_pd(_mm256_loadu_pd(a+4*q+2*lda),w2));
acc[q]=_mm256_add_pd(acc[q],dot);
}
}
for(int q=0;q<4;q++) _mm256_storeu_pd(r+j+4*q,acc[q]);
}
for(;j+4<=frames;j+=4){
__m256d acc=_mm256_setzero_pd();
for(int h=0;h<len;h++){
const double *v=weights+3*h,*a=base+j+h*dilation;
__m256d dot=_mm256_mul_pd(_mm256_loadu_pd(a+lda),_mm256_set1_pd(v[1]));
dot=_mm256_fmadd_pd(_mm256_loadu_pd(a),_mm256_set1_pd(v[0]),dot);
if(c==3) dot=_mm256_add_pd(dot,_mm256_mul_pd(_mm256_loadu_pd(a+2*lda),_mm256_set1_pd(v[2])));
acc=_mm256_add_pd(acc,dot);
}
_mm256_storeu_pd(r+j,acc);
}
for(;j<frames;j++){
double acc=0;
for(int h=0;h<len;h++){
const double *v=weights+3*h,*a=base+j+h*dilation;
double dot=fma(a[lda],v[1],a[0]*v[0]);
if(c==3)dot=fma(a[2*lda],v[2],dot);
acc+=dot;
}
r[j]=acc;
}
}
/* Specialization by channel count hoists branches out of the hot loop.
Each coefficient broadcast is shared by four eight-position tiles. */
__attribute__((target("avx512f,fma"),always_inline))
static inline void tile_project(double *r,const double *base,Py_ssize_t lda,Py_ssize_t frames,
const double *weights,int c,int64_t len,int64_t dilation){
Py_ssize_t j=0;
for(;j+32<=frames;j+=32){
__m512d x[4]={_mm512_setzero_pd(),_mm512_setzero_pd(),_mm512_setzero_pd(),_mm512_setzero_pd()};
for(int h=0;h<len;h++){
const double *v=weights+3*h,*s=base+j+h*dilation;
__m512d w0=_mm512_set1_pd(v[0]),w1=_mm512_set1_pd(v[1]),w2=_mm512_set1_pd(v[2]);
for(int q=0;q<4;q++){
const double *a=s+8*q;
__m512d av=_mm512_loadu_pd(a),dot;
if(c==1)dot=_mm512_mul_pd(av,w0);
else{
dot=_mm512_fmadd_pd(av,w0,_mm512_mul_pd(_mm512_loadu_pd(a+lda),w1));
if(c==3)dot=_mm512_add_pd(dot,_mm512_mul_pd(_mm512_loadu_pd(a+2*lda),w2));
}
x[q]=_mm512_add_pd(x[q],dot);
}
}
for(int q=0;q<4;q++)_mm512_storeu_pd(r+j+8*q,x[q]);
}
for(;j+8<=frames;j+=8){
__m512d x=_mm512_setzero_pd();
for(int h=0;h<len;h++){
const double *v=weights+3*h,*a=base+j+h*dilation;
__m512d av=_mm512_loadu_pd(a),w0=_mm512_set1_pd(v[0]),dot;
if(c==1)dot=_mm512_mul_pd(av,w0);
else{
dot=_mm512_fmadd_pd(av,w0,_mm512_mul_pd(_mm512_loadu_pd(a+lda),_mm512_set1_pd(v[1])));
if(c==3)dot=_mm512_add_pd(dot,_mm512_mul_pd(_mm512_loadu_pd(a+2*lda),_mm512_set1_pd(v[2])));
}
x=_mm512_add_pd(x,dot);
}
_mm512_storeu_pd(r+j,x);
}
if(c>1)skylakex_project(r+j,base+j,lda,frames-j,weights,c,len,dilation);
else for(;j<frames;j++){
double a=0.;
for(int h=0;h<len;h++)a+=base[j+h*dilation]*weights[3*h];
r[j]=a;
}
}
__attribute__((target("avx512f,fma")))
static void wide_project(double *r,const double *base,Py_ssize_t lda,Py_ssize_t frames,
const double *weights,int c,int64_t len,int64_t dilation){
if(c==1)tile_project(r,base,lda,frames,weights,1,len,dilation);
else if(c==2)tile_project(r,base,lda,frames,weights,2,len,dilation);
else tile_project(r,base,lda,frames,weights,3,len,dilation);
}
/* Fixed-size response tile: a whole eight-vector strip shares each tap.
Mask the short suffix and explicitly retain the special 1--3 frame BLAS
contraction at the very end of a multi-channel projection. */
__attribute__((target("avx512f,fma"),always_inline))
static inline void project_strip(double *r,const double *base,Py_ssize_t lda,int frames,
const double *weights,int c,int len,int dilation) {
int j=0;
for(;j+64<=frames;j+=64){
__m512d acc[8];for(int q=0;q<8;q++) acc[q]=_mm512_setzero_pd();
for(int h=0;h<len;h++){
const double *a=base+j+h*dilation,*v=weights+3*h;
__m512d w0=_mm512_set1_pd(v[0]),w1=_mm512_set1_pd(v[1]),w2=_mm512_set1_pd(v[2]);
for(int q=0;q<8;q++){
__m512d x=_mm512_loadu_pd(a+8*q);
__m512d dot=c==1?_mm512_mul_pd(x,w0):
_mm512_fmadd_pd(x,w0,_mm512_mul_pd(_mm512_loadu_pd(a+lda+8*q),w1));
if(c==3)dot=_mm512_add_pd(dot,_mm512_mul_pd(_mm512_loadu_pd(a+2*lda+8*q),w2));
acc[q]=_mm512_add_pd(acc[q],dot);
}
}
for(int q=0;q<8;q++)_mm512_storeu_pd(r+j+8*q,acc[q]);
}
tile_project(r+j,base+j,lda,frames-j,weights,c,len,dilation);
}
#define STRIP_FUNC(C,L) \
__attribute__((target("avx512f,fma"))) static void strip_##C##_##L( \
double *r,const double *x,Py_ssize_t ld,int n,const double *w,int d) { \
project_strip(r,x,ld,n,w,C,L,d); }
#define STRIP_L(C) STRIP_FUNC(C,7) STRIP_FUNC(C,9) STRIP_FUNC(C,11) STRIP_FUNC(C,15)
STRIP_L(1) STRIP_L(2) STRIP_L(3)
typedef void (*strip_fn)(double*,const double*,Py_ssize_t,int,const double*,int);
#define STRIP_ROW(C) {strip_##C##_7,strip_##C##_9,strip_##C##_11,strip_##C##_15}
static strip_fn strips[3][4]={STRIP_ROW(1),STRIP_ROW(2),STRIP_ROW(3)};
/* NumPy sums each <=128-value leaf in eight lanes, then combines pairs.
Fuse both traversals: bias + max + PPV + mean; squared deviations + sum.
Larger inputs recurse at the same eight-aligned split as pair_sum. */
__attribute__((target("avx512f,popcnt"),always_inline))
static inline double leaf_reduce(double *r,Py_ssize_t n,double shift,int central,
Py_ssize_t *pos,double *maximum,int *special) {
__m512d sum=_mm512_setzero_pd(),hi=_mm512_set1_pd(-INFINITY);
const __m512d s=_mm512_set1_pd(shift);
Py_ssize_t j=0;
for(;j+8<=n;j+=8) {
__m512d v=_mm512_loadu_pd(r+j);
if(central){
v=_mm512_sub_pd(v,s);v=_mm512_mul_pd(v,v);
}else{
v=_mm512_add_pd(v,s);_mm512_storeu_pd(r+j,v);
*pos+=__builtin_popcount((unsigned)_mm512_cmp_pd_mask(v,_mm512_setzero_pd(),_CMP_GT_OQ));
*special|=_mm512_cmp_pd_mask(v,v,_CMP_UNORD_Q);
hi=_mm512_max_pd(hi,v);
}
sum=j?_mm512_add_pd(sum,v):v;
}
double total=-0.;
if(j){
double a[8];_mm512_storeu_pd(a,sum);
total=((a[0]+a[1])+(a[2]+a[3]))+((a[4]+a[5])+(a[6]+a[7]));
if(!central){
double peak=_mm512_reduce_max_pd(hi);
if(peak>=*maximum)*maximum=peak;
}
}
for(;j<n;j++){
double v=r[j];
if(central){v-=shift;v*=v;}
else{
v+=shift;r[j]=v;*pos+=v>0.;*special|=isnan(v);
if(v>=*maximum)*maximum=v;
}
total+=v;
}
return total;
}
__attribute__((target("avx512f,popcnt")))
static double reduce_window(double *r,Py_ssize_t n,double shift,int central,
Py_ssize_t *pos,double *maximum,int *special) {
if(n<=128)return leaf_reduce(r,n,shift,central,pos,maximum,special);
Py_ssize_t m=(n/2)/8*8;
double lo=reduce_window(r,m,shift,central,pos,maximum,special);
return lo+reduce_window(r+m,n-m,shift,central,pos,maximum,special);
}
/* A SIMD lane owns an independent window. This allows reductions to follow
the reference's exact time order without horizontal-vector reductions. */
__attribute__((target("avx512f"),always_inline))
static inline __m512d batch_sum(const double *r,int n){
if(n<8){
__m512d acc=_mm512_set1_pd(-0.);
for(int j=0;j<n;j++)acc=_mm512_add_pd(acc,_mm512_loadu_pd(r+8*j));
return acc;
}
__m512d lane[8];
for(int q=0;q<8;q++)lane[q]=_mm512_loadu_pd(r+8*q);
int j=8;
for(;j+8<=n;j+=8)for(int q=0;q<8;q++)lane[q]=_mm512_add_pd(lane[q],_mm512_loadu_pd(r+8*(j+q)));
__m512d x=_mm512_add_pd(_mm512_add_pd(_mm512_add_pd(lane[0],lane[1]),_mm512_add_pd(lane[2],lane[3])),
_mm512_add_pd(_mm512_add_pd(lane[4],lane[5]),_mm512_add_pd(lane[6],lane[7])));
for(;j<n;j++)x=_mm512_add_pd(x,_mm512_loadu_pd(r+j*8));
return x;
}
/* At lengths 249..255 the right leaf itself exceeds 128. */
__attribute__((target("avx512f")))
static __m512d batch_reduce(const double *r,int n){
if(n<=128)return batch_sum(r,n);
int cut=n/16*8;
return _mm512_add_pd(batch_reduce(r,cut),batch_reduce(r+cut*8,n-cut));
}
__attribute__((target("avx512f"))) static void finish_projection(double *r,int nf,double *out,Py_ssize_t outjump){
__m512d m=batch_reduce(r,nf);
m=_mm512_div_pd(m,_mm512_set1_pd(nf));
__m512d mx=_mm512_set1_pd(-INFINITY),pos=_mm512_setzero_pd(),one=_mm512_set1_pd(1.);
__mmask8 nan=0;
for(int j=0;j<nf;j++){
__m512d v=_mm512_loadu_pd(r+j*8);nan|=_mm512_cmp_pd_mask(v,v,_CMP_UNORD_Q);
mx=_mm512_max_pd(mx,v);pos=_mm512_mask_add_pd(pos,_mm512_cmp_pd_mask(v,_mm512_setzero_pd(),_CMP_GT_OQ),pos,one);
v=_mm512_sub_pd(v,m);_mm512_storeu_pd(r+j*8,_mm512_mul_pd(v,v));
}
__m512d sum=batch_reduce(r,nf);
double d[4][8];
_mm512_storeu_pd(d[0],_mm512_div_pd(pos,_mm512_set1_pd(nf)));_mm512_storeu_pd(d[1],_mm512_mask_mov_pd(mx,nan,_mm512_set1_pd(NAN)));
_mm512_storeu_pd(d[2],m);_mm512_storeu_pd(d[3],_mm512_sqrt_pd(_mm512_div_pd(sum,_mm512_set1_pd(nf))));
for(int lane=0;lane<8;lane++)for(int q=0;q<4;q++)out[lane*outjump+q]=d[q][lane];
}
/* Sort a transient index of the fitted bank by shape, without reordering
the output columns. Paired kernels share input vectors only. */
typedef struct {int64_t geom[6], idx;} BankItem;
static int cmp_bank(const void *x,const void *y){
const BankItem *a=x,*b=y;return memcmp(a->geom,b->geom,sizeof a->geom);
}
__attribute__((target("avx512f,fma"),always_inline))
static inline void pair_kernel(const double *s0,const double *s1,const double *s2,const double *w,const double *w2,
double bias,double bias2,int c,int len,int dil,int nf,double *r,double *r2,double *out,double *out2,Py_ssize_t jump){
int main=nf/8*8,j=0;
for(;j<main;j+=8){
__m512d v[8],u[8];for(int q=0;q<8;++q)v[q]=u[q]=_mm512_setzero_pd();
for(int h=0;h<len;++h){
int off=(j+h*dil)*8;
__m512d wa=_mm512_set1_pd(w[3*h]),wb=_mm512_set1_pd(w[3*h+1]),wc=_mm512_set1_pd(w[3*h+2]);
__m512d xa=_mm512_set1_pd(w2[3*h]),xb=_mm512_set1_pd(w2[3*h+1]),xc=_mm512_set1_pd(w2[3*h+2]);
for(int q=0;q<8;++q,off+=8){
__m512d a=_mm512_loadu_pd(s0+off), b=_mm512_loadu_pd(s1+off),e=_mm512_loadu_pd(s2+off);
__m512d p=a*wa,z=a*xa; if(c>1){p=_mm512_fmadd_pd(a,wa,b*wb);z=_mm512_fmadd_pd(a,xa,b*xb);}
if(c==3){p+=e*wc;z+=e*xc;}v[q]+=p;u[q]+=z;
}
}
for(int q=0;q<8;++q){_mm512_storeu_pd(r+8*(j+q),v[q]+bias);_mm512_storeu_pd(r2+8*(j+q),u[q]+bias2);}
}
for(;j<nf;++j){
__m512d v=_mm512_setzero_pd(),u=v;
for(int h=0;h<len;++h){int off=(j+h*dil)*8;
__m512d a=_mm512_loadu_pd(s0+off),b=_mm512_loadu_pd(s1+off),e=_mm512_loadu_pd(s2+off);
__m512d p=a*w[3*h], z=a*w2[3*h];
if(c>1){
if(j<((nf/4)*4)){p=_mm512_fmadd_pd(a,_mm512_set1_pd(w[3*h]),b*w[3*h+1]);z=_mm512_fmadd_pd(a,_mm512_set1_pd(w2[3*h]),b*w2[3*h+1]);}
else {p=_mm512_fmadd_pd(b,_mm512_set1_pd(w[3*h+1]),p);z=_mm512_fmadd_pd(b,_mm512_set1_pd(w2[3*h+1]),z);}
}
if(c>2){
if(j<((nf/4)*4)){p+=e*w[3*h+2];z+=e*w2[3*h+2];}
else{p=_mm512_fmadd_pd(e,_mm512_set1_pd(w[3*h+2]),p);z=_mm512_fmadd_pd(e,_mm512_set1_pd(w2[3*h+2]),z);}
}
v+=p;u+=z;
}
_mm512_storeu_pd(r+j*8,v+bias);_mm512_storeu_pd(r2+j*8,u+bias2);
}
finish_projection(r,nf,out,jump);finish_projection(r2,nf,out2,jump);
}
__attribute__((target("avx512f,fma"),always_inline))
static inline void batch_kernel(const double *s0,const double *s1,const double *s2,
const double *w,double bias,int c,int len,int dil,int nf,
double *r,double *out,Py_ssize_t outjump){
int main=nf/8*8,j=0;
for(;j<main;j+=8){
__m512d v[8];for(int q=0;q<8;q++)v[q]=_mm512_setzero_pd();
for(int h=0;h<len;h++){
int off=(j+h*dil)*8;
__m512d w0=_mm512_set1_pd(w[3*h]),w1=_mm512_set1_pd(w[3*h+1]),w2=_mm512_set1_pd(w[3*h+2]);
for(int q=0;q<8;q++,off+=8){
__m512d a=_mm512_loadu_pd(s0+off);
__m512d p=c==1?_mm512_mul_pd(a,w0):_mm512_fmadd_pd(a,w0,_mm512_mul_pd(_mm512_loadu_pd(s1+off),w1));
if(c==3)p=_mm512_add_pd(p,_mm512_mul_pd(_mm512_loadu_pd(s2+off),w2));
v[q]=_mm512_add_pd(v[q],p);
}
}
for(int q=0;q<8;q++)_mm512_storeu_pd(r+8*(j+q),_mm512_add_pd(v[q],_mm512_set1_pd(bias)));
}
if(j+4<=nf){
__m512d acc[4];for(int q=0;q<4;q++)acc[q]=_mm512_setzero_pd();
for(int h=0;h<len;h++){
int off=(j+h*dil)*8;__m512d w0=_mm512_set1_pd(w[3*h]),w1=_mm512_set1_pd(w[3*h+1]),w2=_mm512_set1_pd(w[3*h+2]);
for(int q=0;q<4;q++,off+=8){
__m512d u=_mm512_loadu_pd(s0+off),z=c==1?_mm512_mul_pd(u,w0):_mm512_fmadd_pd(u,w0,_mm512_mul_pd(_mm512_loadu_pd(s1+off),w1));
if(c==3)z=_mm512_add_pd(z,_mm512_mul_pd(_mm512_loadu_pd(s2+off),w2));
acc[q]=_mm512_add_pd(acc[q],z);
}
}
for(int q=0;q<4;q++)_mm512_storeu_pd(r+8*(j+q),_mm512_add_pd(acc[q],_mm512_set1_pd(bias)));
j+=4;
}
for(;j<nf;j++){
__m512d v=_mm512_setzero_pd();
for(int h=0;h<len;h++){
int off=(j+h*dil)*8;
__m512d p=_mm512_mul_pd(_mm512_loadu_pd(s0+off),_mm512_set1_pd(w[3*h]));
if(c>1)p=_mm512_fmadd_pd(_mm512_loadu_pd(s1+off),_mm512_set1_pd(w[3*h+1]),p);
if(c>2)p=_mm512_fmadd_pd(_mm512_loadu_pd(s2+off),_mm512_set1_pd(w[3*h+2]),p);
v=_mm512_add_pd(v,p);
}
_mm512_storeu_pd(r+j*8,_mm512_add_pd(v,_mm512_set1_pd(bias)));
}
finish_projection(r,nf,out,outjump);
}
__attribute__((target("avx512f,fma")))
static void batch_bank(const double *x,Py_ssize_t ch,int t,const int64_t *rows,const double *weights,
const double *bias,Py_ssize_t k,int maxlen,double *out,double *packed,double *r,const BankItem *items){
for(Py_ssize_t c=0;c<ch;c++)for(int j=0;j<t;j++)for(int a=0;a<8;a++)
packed[(c*t+j)*8+a]=x[(a*ch+c)*t+j];
for(Py_ssize_t item=0;item<k;item++){
Py_ssize_t z=items[item].idx;
const int64_t *s=rows+6*z;int len=s[0],d=s[1],c=s[2],nf=t-(len-1)*d;
const double *a=packed+8*t*s[3],*b=c>1?packed+8*t*s[4]:a,*e=c>2?packed+8*t*s[5]:a;
if(item+1<k && !memcmp(items[item].geom,items[item+1].geom,6*sizeof(int64_t))){
Py_ssize_t zz=items[++item].idx;
#define PK(C) pair_kernel(a,b,e,weights+z*maxlen*3,weights+zz*maxlen*3,bias[z],bias[zz],C,len,d,nf,r,r+8*t,out+z*4,out+zz*4,4*k)
if(c==1)PK(1);else if(c==2)PK(2);else PK(3);
#undef PK
continue;
}
#define BK(C) batch_kernel(a,b,e,weights+z*maxlen*3,bias[z],C,len,d,nf,r,out+z*4,4*k)
if(c==1)BK(1);else if(c==2)BK(2);else BK(3);
#undef BK
}
}
/* Optional FP32 projections for already fitted FP64 banks. Sixteen retained
time positions share a vector. The two-pass centered variance avoids the
cancellation of E[X^2]-E[X]^2 near a constant. */
__attribute__((target("avx512f,fma,popcnt"),always_inline))
static inline void small_kernel(const float *x,const float *y,const float *z,
const float *w,int c,int len,int d,int nf,float bias,
float *scratch,double *out){
int count=0;__m512 mx=_mm512_set1_ps(-INFINITY);
for(int j=0;j<nf;j+=64){
int left=nf-j;
__m512 acc[4];__mmask16 masks[4];
for(int q=0;q<4;q++){
int take=left-16*q;
masks[q]=take<=0?0:take>=16?0xffff:(1u<<take)-1u;
acc[q]=_mm512_setzero_ps();
}
for(int h=0;h<len;h++){
__m512 w0=_mm512_set1_ps(w[3*h]),w1=_mm512_set1_ps(w[3*h+1]),w2=_mm512_set1_ps(w[3*h+2]);
for(int q=0;q<4;q++){
int off=j+d*h+q*16;
/* Input rows are padded by 64 floats, allowing these full
loads even on the last partial tile. */
__m512 u=_mm512_loadu_ps(x+off);
if(c==1)acc[q]=_mm512_fmadd_ps(u,w0,acc[q]);
else {
__m512 v=_mm512_fmadd_ps(u,w0,_mm512_mul_ps(_mm512_loadu_ps(y+off),w1));
if(c>2)v=_mm512_fmadd_ps(_mm512_loadu_ps(z+off),w2,v);
acc[q]=_mm512_add_ps(acc[q],v);
}
}
}
for(int q=0;q<4;q++){
__m512 v=_mm512_add_ps(acc[q],_mm512_set1_ps(bias));
_mm512_storeu_ps(scratch+j+16*q,v);
__mmask16 mask=masks[q];
mx=_mm512_mask_max_ps(mx,mask,mx,v);
count+=__builtin_popcount((unsigned)(mask&_mm512_cmp_ps_mask(v,_mm512_setzero_ps(),_CMP_GT_OQ)));
}
}
/* The sixteen-window path sums in time order. Preserve that order for
residual windows to make predictions independent of input chunking. */
float sum=0.f,ss=0.f;
for(int j=0;j<nf;j++)sum+=scratch[j];
float mu=sum/nf;
for(int j=0;j<nf;j++){float u=scratch[j]-mu;ss=fmaf(u,u,ss);}
out[0]=(double)count/nf;out[1]=_mm512_reduce_max_ps(mx);
out[2]=mu;out[3]=sqrtf(ss/nf);
}
__attribute__((target("avx512f,fma,popcnt")))
static void small_bank(const double *x,Py_ssize_t ch,int t,const int64_t *rows,const float *w,
const double *bias,Py_ssize_t k,int maxlen,float *buf,double *out){
int pitch=t+64;
float *scratch=buf+ch*pitch;
for(Py_ssize_t c=0;c<ch;c++)
for(int j=0;j<t;j++)buf[c*pitch+j]=(float)x[c*t+j];
for(Py_ssize_t i=0;i<k;i++){
const int64_t *s=rows+6*i;int len=s[0],d=s[1],c=s[2],nf=t-(len-1)*d;
const float *a=buf+s[3]*pitch,*b=c>1?buf+s[4]*pitch:a,*e=c>2?buf+s[5]*pitch:a;
#define WK(C) small_kernel(a,b,e,w+i*3*maxlen,C,len,d,nf,(float)bias[i],scratch,out+4*i)
if(c==1)WK(1);else if(c==2)WK(2);else WK(3);
#undef WK
}
}
/* Window lanes reuse each coefficient over 16 windows by 8 positions.
Pool each finished tile directly, retaining only the response scratch
required for centered variance. */
__attribute__((target("avx512f,fma"),always_inline))
static inline void small_lanes(const float *a,const float *b,const float *e,
const float *w,int c,int len,int d,int nf,float bias,float *work,double *out,int stride){
__m512 hi=_mm512_set1_ps(-INFINITY),sum=_mm512_setzero_ps(),pos=sum;
__m512 one=_mm512_set1_ps(1.),zero=_mm512_setzero_ps(),v_bias=_mm512_set1_ps(bias);
for(int j=0;j<nf;j+=8){
int take=nf-j<8?nf-j:8;__m512 v[8];
for(int q=0;q<8;q++)v[q]=zero;
for(int h=0;h<len;h++){
int start=(j+h*d)*16;
__m512 w0=_mm512_set1_ps(w[3*h]),w1=_mm512_set1_ps(w[3*h+1]),w2=_mm512_set1_ps(w[3*h+2]);
for(int q=0;q<8;q++){
int off=start+16*q;__m512 x=_mm512_loadu_ps(a+off);
if(c==1)v[q]=_mm512_fmadd_ps(x,w0,v[q]);
else{
__m512 dot=_mm512_fmadd_ps(x,w0,_mm512_mul_ps(_mm512_loadu_ps(b+off),w1));
if(c==3)dot=_mm512_fmadd_ps(_mm512_loadu_ps(e+off),w2,dot);
v[q]=_mm512_add_ps(v[q],dot);
}
}
}
for(int q=0;q<take;q++){
__m512 u=_mm512_add_ps(v[q],v_bias);
_mm512_storeu_ps(work+(j+q)*16,u);
hi=_mm512_max_ps(hi,u);sum=_mm512_add_ps(sum,u);
pos=_mm512_mask_add_ps(pos,_mm512_cmp_ps_mask(u,zero,_CMP_GT_OQ),pos,one);
}
}
__m512 avg=_mm512_div_ps(sum,_mm512_set1_ps(nf)),ss=zero;
for(int j=0;j<nf;j++){
__m512 u=_mm512_sub_ps(_mm512_loadu_ps(work+j*16),avg);
ss=_mm512_fmadd_ps(u,u,ss);
}
float pooled[4][16];
_mm512_storeu_ps(pooled[0],pos);_mm512_storeu_ps(pooled[1],hi);
_mm512_storeu_ps(pooled[2],avg);_mm512_storeu_ps(pooled[3],_mm512_sqrt_ps(_mm512_div_ps(ss,_mm512_set1_ps(nf))));
for(int j=0;j<16;j++){
double *dst=out+j*stride;dst[0]=(double)pooled[0][j]/nf;
for(int h=1;h<4;h++)dst[h]=pooled[h][j];
}
}
__attribute__((target("avx512f,fma")))
static void small_batch(const double *x,int ch,int t,const int64_t *rows,const float *w,
const double *bias,int k,int maxlen,float *buf,double *out){
int pitch=(t+8)*16;
float *scratch=buf+ch*pitch;
for(int cc=0;cc<ch;cc++)for(int j=0;j<t;j++)for(int h=0;h<16;h++)
buf[cc*pitch+j*16+h]=(float)x[(h*ch+cc)*t+j];
for(int i=0;i<k;i++){
const int64_t *s=rows+6*i;
int c=s[2],len=s[0],d=s[1],nf=t-(len-1)*d;
const float *a=buf+pitch*s[3],*b=c>1?buf+pitch*s[4]:a,*e=c>2?buf+pitch*s[5]:a;
#define SB(C) small_lanes(a,b,e,w+i*3*maxlen,C,len,d,nf,(float)bias[i],scratch,out+4*i,4*k)
if(c==1)SB(1);else if(c==2)SB(2);else SB(3);
#undef SB
}
}
static int can_fuse(void){
if(!bind_numpy() || !__builtin_cpu_supports("avx2") || !__builtin_cpu_supports("fma"))return 0;
Dl_info info;
if(!dladdr((void*)np_gemv,&info))return 0;
void *handle=dlopen(info.dli_fname,RTLD_LAZY|RTLD_LOCAL);
if(!handle)return 0;
const char *(*core)(void)=NULL;
void *fn=dlsym(handle,"scipy_openblas_get_corename64_");
if(fn)memcpy(&core,&fn,sizeof core);
int ok=core && !strcmp(core(),"SkylakeX");
dlclose(handle);
return ok;
}
#else
#define HAVE_FAST_X86 0
#endif
static PyObject *fused_library(PyObject *self,PyObject *args) {
#if HAVE_FAST_X86
Dl_info info;
if(can_fuse() && dladdr((void*)np_gemv,&info))
return PyUnicode_DecodeFSDefault(info.dli_fname);
#endif
Py_RETURN_NONE;
}
static int view(PyObject *obj, Py_buffer *v, int rank, int writable, int integer) {
int flags=PyBUF_FORMAT|PyBUF_C_CONTIGUOUS|(writable?PyBUF_WRITABLE:0);
if (PyObject_GetBuffer(obj,v,flags)<0) return -1;
int fmt=integer ? (!strcmp(v->format,"q") || !strcmp(v->format,"l"))
: !strcmp(v->format,"d");
if (!fmt || v->ndim!=rank || v->itemsize!=8 || (uintptr_t)v->buf%8) {
PyErr_SetString(PyExc_ValueError,"expected aligned contiguous int64 or float64 array of specified rank");
return -1;
}
return 0;
}
/* Eight independent partial sums and a balanced combination reproduce the
NumPy pairwise reduction tree used by contiguous float64 sums. */
static double pair_sum(const double *x, Py_ssize_t n) {
if(n<8) {
double s=-0.;
for(Py_ssize_t i=0;i<n;i++) s+=x[i];
return s;
}
if(n>128) {
Py_ssize_t m=(n/2)/8*8;
return pair_sum(x,m)+pair_sum(x+m,n-m);
}
double a[8];
memcpy(a,x,sizeof a);
Py_ssize_t i=8;
for(;i+8<=n;i+=8) for(int j=0;j<8;j++) a[j]+=x[i+j];
double s=((a[0]+a[1])+(a[2]+a[3]))+((a[4]+a[5])+(a[6]+a[7]));
for(;i<n;i++) s+=x[i];
return s;
}
#if HAVE_FAST_X86 && !defined(WISP_SCALAR_PHASE)
/* Each SIMD lane is an independent retained response. Keep tap order and the
unfused per-channel arithmetic of the scalar approximation. Mask the final
load as well as its store: a phase row need not contain eight extra values. */
__attribute__((target("avx512f")))
static inline __m512d phase_dot(const double *a,Py_ssize_t b,Py_ssize_t g,
__m512d w0,__m512d w1,__m512d w2,int c,__mmask8 mask){
__m512d v=_mm512_mul_pd(_mm512_maskz_loadu_pd(mask,a),w0);
if(c>1)v=_mm512_add_pd(v,_mm512_mul_pd(_mm512_maskz_loadu_pd(mask,a+b),w1));
if(c>2)v=_mm512_add_pd(v,_mm512_mul_pd(_mm512_maskz_loadu_pd(mask,a+g),w2));
return v;
}
__attribute__((target("avx512f")))
static void wide_phase_project(double *r,const double *phase,Py_ssize_t pitch,
Py_ssize_t stride,Py_ssize_t nf,const int64_t *s,
const double *weights){
const int len=(int)s[0],d=(int)s[1],c=(int)s[2];
const double *x=phase+s[3]*stride*pitch;
const Py_ssize_t b=c>1?(s[4]-s[3])*stride*pitch:0;
const Py_ssize_t g=c>2?(s[5]-s[3])*stride*pitch:0;
const double *tap[32];
/* Divide once for the entire filter, including non-power-of-two strides. */
const Py_ssize_t delta=d%stride, jump=delta*pitch+d/stride;
Py_ssize_t residue=0,off=0,j=0;
for(int h=0;h<len;h++){
tap[h]=x+off;
off+=jump;residue+=delta;
if(residue>=stride){off+=1-stride*pitch;residue-=stride;}
}
for(;j+32<=nf;j+=32){
__m512d a0=_mm512_setzero_pd(),a1=a0,a2=a0,a3=a0;
for(int h=0;h<len;h++){
const double *v=weights+3*h,*a=tap[h]+j;
__m512d w0=_mm512_set1_pd(v[0]),w1=_mm512_set1_pd(v[1]),w2=_mm512_set1_pd(v[2]);
a0=_mm512_add_pd(a0,phase_dot(a,b,g,w0,w1,w2,c,255));
a1=_mm512_add_pd(a1,phase_dot(a+8,b,g,w0,w1,w2,c,255));
a2=_mm512_add_pd(a2,phase_dot(a+16,b,g,w0,w1,w2,c,255));
a3=_mm512_add_pd(a3,phase_dot(a+24,b,g,w0,w1,w2,c,255));
}
_mm512_storeu_pd(r+j,a0);_mm512_storeu_pd(r+j+8,a1);
_mm512_storeu_pd(r+j+16,a2);_mm512_storeu_pd(r+j+24,a3);
}
for(;j<nf;j+=8){
const __mmask8 mask=(nf-j>=8)?255:(__mmask8)((1u<<(nf-j))-1u);
__m512d sum=_mm512_setzero_pd();
for(int h=0;h<len;h++){
const double *v=weights+3*h;
sum=_mm512_add_pd(sum,phase_dot(tap[h]+j,b,g,_mm512_set1_pd(v[0]),
_mm512_set1_pd(v[1]),_mm512_set1_pd(v[2]),c,mask));
}
_mm512_mask_storeu_pd(r+j,mask,sum);
}
}
#endif
/* Subsampled response starts are adjacent after grouping equal time-index
residues. Repack once per window and reuse this layout for every filter. */
static void phase_project(double *r,const double *phase,Py_ssize_t pitch,Py_ssize_t stride,
Py_ssize_t nf,const int64_t *s,const double *weights) {
#if HAVE_FAST_X86 && !defined(WISP_SCALAR_PHASE)
if(s[0]<=32 && __builtin_cpu_supports("avx512f")){
wide_phase_project(r,phase,pitch,stride,nf,s,weights);
return;
}
#endif
const int len=(int)s[0],d=(int)s[1],c=(int)s[2];
const double *x=phase+s[3]*stride*pitch;
const Py_ssize_t b=c>1?(s[4]-s[3])*stride*pitch:0;
const Py_ssize_t g=c>2?(s[5]-s[3])*stride*pitch:0;
memset(r,0,nf*sizeof(double));
for(int h=0;h<len;h++){
const int off=h*d;
const double *a=x+(off%stride)*pitch+off/stride, *v=weights+3*h;
if(c==1) for(Py_ssize_t j=0;j<nf;j++)r[j]+=a[j]*v[0];
else if(c==2) for(Py_ssize_t j=0;j<nf;j++)r[j]+=a[j]*v[0]+a[j+b]*v[1];
else for(Py_ssize_t j=0;j<nf;j++)r[j]+=(a[j]*v[0]+a[j+b]*v[1])+a[j+g]*v[2];
}
}
static PyObject *transform(PyObject *self,PyObject *args) {
PyObject *xo,*mo,*wo,*bo,*yo;
Py_ssize_t stride;
int approximate, fused, polyphase=0, low=0;
float *small=NULL,*smallw=NULL;
Py_buffer x={0},m={0},w={0},b={0},y={0};
double *work=NULL,*phase=NULL,*bt=NULL;
if (!PyArg_ParseTuple(args,"OOOOnppO|pp",&xo,&mo,&wo,&bo,&stride,&approximate,&fused,&yo,&polyphase,&low)) return NULL;
if (view(xo,&x,3,0,0)||view(mo,&m,2,0,1)||view(wo,&w,3,0,0)||
view(bo,&b,1,0,0)||view(yo,&y,3,1,0)) goto finish;
Py_ssize_t n=x.shape[0],ch=x.shape[1],t=x.shape[2],k=m.shape[0],maxlen=w.shape[1];
const int64_t *rows=m.buf;
if (t<1 || t>8192 || n<1 || ch<1 || k<1 || m.shape[1]!=6 ||
w.shape[0]!=k || w.shape[2]!=3 || maxlen<1 || b.shape[0]!=k ||
y.shape[0]!=n || y.shape[1]!=k || y.shape[2]!=4 || stride<1 || stride>t ||
(!approximate && stride!=1) || (low && (approximate || stride!=1))) {
PyErr_SetString(PyExc_ValueError,"invalid bank dimensions or stride"); goto finish;
}
Py_buffer *inputs[]={&x,&m,&w,&b};
for(int i=0;i<4;i++) {
uintptr_t p=(uintptr_t)inputs[i]->buf,q=(uintptr_t)y.buf;
if(p<q+y.len && q<p+inputs[i]->len) {
PyErr_SetString(PyExc_ValueError,"output overlaps input"); goto finish;
}
}
for(Py_ssize_t z=0;z<k;z++) {
const int64_t *meta=rows+6*z;
int64_t len=meta[0],d=meta[1],c=meta[2];
if(len<1 || len>maxlen || d<1 || len-1>(t-1)/d || c<1 || c>3) {
PyErr_SetString(PyExc_ValueError,"invalid kernel geometry"); goto finish;
}
if(!approximate && c>1 && (!bind_numpy() || t-(len-1)*d<=1)) {
PyErr_SetString(PyExc_ValueError,"unsupported exact BLAS geometry"); goto finish;
}
for(int j=0;j<c;j++) if(meta[3+j]<0 || meta[3+j]>=ch || (j && meta[3+j]<=meta[2+j])) {
PyErr_SetString(PyExc_ValueError,"invalid kernel channels"); goto finish;
}
}
#if HAVE_FAST_X86
int wide = __builtin_cpu_supports("avx512f") && __builtin_cpu_supports("popcnt");
fused = fused && !approximate && (__builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma")) && t>=16;
#endif
work=PyMem_Malloc(5*t*sizeof(double));
if (!work) { PyErr_NoMemory(); goto finish; }
if(low){
/* Unusual custom coefficients or unnormalized signals can exceed FP32.
The bound covers convolution, reduction and centered squared sums. */
double xmax=0.,wmax=0.,bmax=0.;
const double *values[3]={x.buf,w.buf,b.buf};
Py_ssize_t sizes[3]={x.len/8,w.len/8,b.len/8};
double *maxima[3]={&xmax,&wmax,&bmax};
for(int j=0;j<3;j++)for(Py_ssize_t i=0;i<sizes[j];i++){
double v=fabs(values[j][i]);if(!isfinite(v)){low=0;break;}
if(v>*maxima[j])*maxima[j]=v;
}
if(xmax>FLT_MAX || wmax>FLT_MAX || bmax>FLT_MAX ||
3.*maxlen*xmax*wmax+bmax>sqrt((double)FLT_MAX/(16.*t)))low=0;
}
if(low){
#if HAVE_FAST_X86
if(!wide || !__builtin_cpu_supports("fma")){
#endif
PyErr_SetString(PyExc_ValueError,"conv32 requires an AVX-512F/FMA/POPCNT CPU");goto finish;
#if HAVE_FAST_X86
}
if((size_t)(ch+1)>SIZE_MAX/((size_t)t+64)/(16*sizeof(float))){
PyErr_NoMemory();goto finish;
}
small=PyMem_Calloc(((size_t)ch+1)*(t+64)*(n>=16?16:1),sizeof(float));smallw=PyMem_Malloc(w.len/2);
if(!small||!smallw){PyErr_NoMemory();goto finish;}
for(Py_ssize_t i=0;i<w.len/8;i++)smallw[i]=(float)((double*)w.buf)[i];
#endif
}
double *r=work,*tmp=work+t,*gather=work+2*t;
const double *X=x.buf,*W=w.buf,*bias=b.buf;
double *Y=y.buf;
Py_ssize_t pitch=(t+stride-1)/stride;
if(polyphase && approximate && stride>1 && ch<=64){
phase=PyMem_Malloc(ch*stride*pitch*sizeof(double));
if(!phase){PyErr_NoMemory();goto finish;}
}
Py_ssize_t first=0,block=phase?1:12;
#if HAVE_FAST_X86
BankItem *items=NULL;
if(!low && wide && fused && n>=8 && ch<=8 && t<=256){
bt=PyMem_Malloc((ch+2)*t*8*sizeof(double));if(!bt){PyErr_NoMemory();goto finish;}
items=PyMem_Malloc(k*sizeof(*items));if(!items){PyErr_NoMemory();goto finish;}
for(Py_ssize_t z=0;z<k;++z){memcpy(items[z].geom,rows+6*z,6*sizeof(int64_t));items[z].idx=z;}
qsort(items,k,sizeof(*items),cmp_bank);
}
#endif
Py_BEGIN_ALLOW_THREADS
#if HAVE_FAST_X86
if(low)for(;first+16<=n;first+=16)small_batch(X+first*ch*t,ch,t,rows,smallw,bias,k,maxlen,small,Y+first*k*4);
if(low)for(;first<n;first++)small_bank(X+first*ch*t,ch,t,rows,smallw,bias,k,maxlen,small,Y+first*k*4);
if(bt) for(;first+8<=n;first+=8)batch_bank(X+first*ch*t,ch,t,rows,W,bias,k,maxlen,Y+first*k*4,bt,bt+ch*t*8,items);
#endif
for(Py_ssize_t tile=first;tile<n;tile+=block){
Py_ssize_t stop=tile+block<n?tile+block:n;
if(phase) for(Py_ssize_t h=0;h<ch;h++) for(Py_ssize_t q=0;q<t;q++)
phase[(h*stride+q%stride)*pitch+q/stride]=X[(tile*ch+h)*t+q];
for(Py_ssize_t z=0;z<k;z++){
for(Py_ssize_t i=tile;i<stop;i++){
const int64_t *s=rows+6*z;
int64_t len=s[0],d=s[1],c=s[2],a0=s[3],lda=c>1?t*(s[4]-a0):t;
Py_ssize_t nf=1+(t-1-(len-1)*d)/stride;
const double *base=X+(i*ch+a0)*t;
if (!phase && c>2 && s[5]-s[4]!=s[4]-a0) {
for(int j=0;j<c;j++) memcpy(gather+j*t,X+(i*ch+s[3+j])*t,t*sizeof(double));
base=gather; lda=t;
}
if(phase) phase_project(r,phase,pitch,stride,nf,s,W+z*maxlen*3); else
#if HAVE_FAST_X86
if(wide && !approximate && (fused || c==1) && nf>=16) {
int ix=len==7?0:len==9?1:len==11?2:len==15?3:-1;
if(ix>=0)strips[c-1][ix](r,base,lda,(int)nf,W+z*maxlen*3,(int)d);
else wide_project(r,base,lda,nf,W+z*maxlen*3,c,len,d);
} else if(fused && c>1) {
skylakex_project(r,base,lda,nf,W+z*maxlen*3,c,len,d);
} else
#endif
{
memset(r,0,nf*sizeof(double));
for(int64_t h=0;h<len;h++){
const double *v=W+(z*maxlen+h)*3,*src=base+h*d;
if(c==1){
if(stride==1) for(Py_ssize_t j=0;j<nf;j++) r[j]+=src[j]*v[0];
else for(Py_ssize_t j=0;j<nf;j++) r[j]+=src[j*stride]*v[0];
} else if(!approximate) {
np_gemv(102,111,nf,c,1.,src,lda,v,1,0.,tmp,1);
for(Py_ssize_t j=0;j<nf;j++) r[j]+=tmp[j];
} else if(c==2) {
for(Py_ssize_t j=0;j<nf;j++) r[j]+=src[j*stride]*v[0]+src[lda+j*stride]*v[1];
} else {
for(Py_ssize_t j=0;j<nf;j++) r[j]+=(src[j*stride]*v[0]+src[lda+j*stride]*v[1])+src[2*lda+j*stride]*v[2];
}
}
}
Py_ssize_t pos=0;
double mx=-INFINITY;
#if HAVE_FAST_X86
if(wide && nf>=16) {
int special=0;
double mu=reduce_window(r,nf,bias[z],0,&pos,&mx,&special)/nf;
if(special || mx==0.){
mx=-INFINITY;
for(Py_ssize_t j=0;j<nf;j++)if(r[j]>=mx || isnan(r[j]))mx=r[j];
}
double *dst=Y+(i*k+z)*4;
dst[0]=(double)pos/nf;dst[1]=mx;dst[2]=mu;
dst[3]=sqrt(reduce_window(r,nf,mu,1,&pos,&mx,&special)/nf);
continue;
}
#endif
for(Py_ssize_t j=0;j<nf;j++) {
r[j]+=bias[z]; pos+=r[j]>0;
if(r[j]>=mx || isnan(r[j])) mx=r[j];
}
double mu=pair_sum(r,nf)/nf;
for(Py_ssize_t j=0;j<nf;j++) { double v=r[j]-mu;tmp[j]=v*v; }
double *dst=Y+(i*k+z)*4;
dst[0]=(double)pos/nf;dst[1]=mx;dst[2]=mu;dst[3]=sqrt(pair_sum(tmp,nf)/nf);
}
}
}
Py_END_ALLOW_THREADS
#if HAVE_FAST_X86
PyMem_Free(items);
#endif
finish:
PyMem_Free(small);PyMem_Free(smallw);
PyMem_Free(bt);PyMem_Free(phase);
PyMem_Free(work);
if(x.obj)PyBuffer_Release(&x);
if(m.obj)PyBuffer_Release(&m);
if(w.obj)PyBuffer_Release(&w);
if(b.obj)PyBuffer_Release(&b);
if(y.obj)PyBuffer_Release(&y);
if(PyErr_Occurred())return NULL;
Py_RETURN_NONE;
}
static PyMethodDef methods[]={{"has_blas",has_blas,METH_NOARGS,"Whether the exact multichannel NumPy BLAS is accessible."},
{"fused_library",fused_library,METH_NOARGS,"Candidate library path for the verified SkylakeX arithmetic profile."},
{"transform",transform,METH_VARARGS,"Apply a packed bank on N,C,T inputs."},{NULL,NULL,0,NULL}};
static struct PyModuleDef mod={PyModuleDef_HEAD_INIT,"_cis_conv",NULL,-1,methods};
PyMODINIT_FUNC PyInit__cis_conv(void){return PyModule_Create(&mod);}