/* Packed random convolution, preserving the tap accumulation order. The multichannel exact path reuses the BLAS already linked into NumPy's core, including its choice of fused operations. */ #define _GNU_SOURCE #define PY_SSIZE_T_CLEAN #include #include #include #include #include #include #ifndef _WIN32 #include #endif typedef void (*gemv64_t)(int,int,int64_t,int64_t,double,const double*,int64_t, const double*,int64_t,double,double*,int64_t); static gemv64_t np_gemv; static int attempted; static int bind_numpy(void) { if (!attempted) { attempted = 1; #ifndef _WIN32 PyObject *core = PyImport_ImportModule("numpy._core._multiarray_umath"); if (!core) { PyErr_Clear(); return 0; } PyObject *file = PyModule_GetFilenameObject(core); Py_DECREF(core); if (file) { const char *path = PyUnicode_AsUTF8(file); void *h = path ? dlopen(path, RTLD_LAZY | RTLD_LOCAL) : NULL; if (h) { void *symbol = dlsym(h, "scipy_cblas_dgemv64_"); if (symbol) memcpy(&np_gemv, &symbol, sizeof np_gemv); /* The NumPy module keeps this dependency loaded. */ dlclose(h); } Py_DECREF(file); } if (PyErr_Occurred()) PyErr_Clear(); #endif } return np_gemv != NULL; } static PyObject *has_blas(PyObject *self,PyObject *args) { return PyBool_FromLong(bind_numpy()); } /* This order is specific to the inspected NumPy OpenBLAS SkylakeX kernel shipped with the pinned Linux x86-64 wheel. Python additionally verifies the library's full SHA-256. Other builds use actual calls to GEMV. */ #if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) && !defined(_WIN32) #include #define HAVE_FAST_X86 1 /* Four vectors share weight broadcasts. Keep pertap contributions in four registers and only materialize responses once, after the last tap. */ __attribute__((target("avx2,fma"))) static void skylakex_project(double *r,const double *base,Py_ssize_t lda,Py_ssize_t frames, const double *weights,int c,int64_t len,int64_t dilation) { Py_ssize_t j=0; for (;j+16<=frames;j+=16) { __m256d acc[4]={_mm256_setzero_pd(),_mm256_setzero_pd(),_mm256_setzero_pd(),_mm256_setzero_pd()}; for(int h=0;h1)skylakex_project(r+j,base+j,lda,frames-j,weights,c,len,dilation); else for(;j=*maximum)*maximum=peak; } } for(;j0.;*special|=isnan(v); if(v>=*maximum)*maximum=v; } total+=v; } return total; } __attribute__((target("avx512f,popcnt"))) static double reduce_window(double *r,Py_ssize_t n,double shift,int central, Py_ssize_t *pos,double *maximum,int *special) { if(n<=128)return leaf_reduce(r,n,shift,central,pos,maximum,special); Py_ssize_t m=(n/2)/8*8; double lo=reduce_window(r,m,shift,central,pos,maximum,special); return lo+reduce_window(r+m,n-m,shift,central,pos,maximum,special); } /* A SIMD lane owns an independent window. This allows reductions to follow the reference's exact time order without horizontal-vector reductions. */ __attribute__((target("avx512f"),always_inline)) static inline __m512d batch_sum(const double *r,int n){ if(n<8){ __m512d acc=_mm512_set1_pd(-0.); for(int j=0;jgeom,b->geom,sizeof a->geom); } __attribute__((target("avx512f,fma"),always_inline)) static inline void pair_kernel(const double *s0,const double *s1,const double *s2,const double *w,const double *w2, double bias,double bias2,int c,int len,int dil,int nf,double *r,double *r2,double *out,double *out2,Py_ssize_t jump){ int main=nf/8*8,j=0; for(;j1){p=_mm512_fmadd_pd(a,wa,b*wb);z=_mm512_fmadd_pd(a,xa,b*xb);} if(c==3){p+=e*wc;z+=e*xc;}v[q]+=p;u[q]+=z; } } for(int q=0;q<8;++q){_mm512_storeu_pd(r+8*(j+q),v[q]+bias);_mm512_storeu_pd(r2+8*(j+q),u[q]+bias2);} } for(;j1){ if(j<((nf/4)*4)){p=_mm512_fmadd_pd(a,_mm512_set1_pd(w[3*h]),b*w[3*h+1]);z=_mm512_fmadd_pd(a,_mm512_set1_pd(w2[3*h]),b*w2[3*h+1]);} else {p=_mm512_fmadd_pd(b,_mm512_set1_pd(w[3*h+1]),p);z=_mm512_fmadd_pd(b,_mm512_set1_pd(w2[3*h+1]),z);} } if(c>2){ if(j<((nf/4)*4)){p+=e*w[3*h+2];z+=e*w2[3*h+2];} else{p=_mm512_fmadd_pd(e,_mm512_set1_pd(w[3*h+2]),p);z=_mm512_fmadd_pd(e,_mm512_set1_pd(w2[3*h+2]),z);} } v+=p;u+=z; } _mm512_storeu_pd(r+j*8,v+bias);_mm512_storeu_pd(r2+j*8,u+bias2); } finish_projection(r,nf,out,jump);finish_projection(r2,nf,out2,jump); } __attribute__((target("avx512f,fma"),always_inline)) static inline void batch_kernel(const double *s0,const double *s1,const double *s2, const double *w,double bias,int c,int len,int dil,int nf, double *r,double *out,Py_ssize_t outjump){ int main=nf/8*8,j=0; for(;j1)p=_mm512_fmadd_pd(_mm512_loadu_pd(s1+off),_mm512_set1_pd(w[3*h+1]),p); if(c>2)p=_mm512_fmadd_pd(_mm512_loadu_pd(s2+off),_mm512_set1_pd(w[3*h+2]),p); v=_mm512_add_pd(v,p); } _mm512_storeu_pd(r+j*8,_mm512_add_pd(v,_mm512_set1_pd(bias))); } finish_projection(r,nf,out,outjump); } __attribute__((target("avx512f,fma"))) static void batch_bank(const double *x,Py_ssize_t ch,int t,const int64_t *rows,const double *weights, const double *bias,Py_ssize_t k,int maxlen,double *out,double *packed,double *r,const BankItem *items){ for(Py_ssize_t c=0;c1?packed+8*t*s[4]:a,*e=c>2?packed+8*t*s[5]:a; if(item+1=16?0xffff:(1u<2)v=_mm512_fmadd_ps(_mm512_loadu_ps(z+off),w2,v); acc[q]=_mm512_add_ps(acc[q],v); } } } for(int q=0;q<4;q++){ __m512 v=_mm512_add_ps(acc[q],_mm512_set1_ps(bias)); _mm512_storeu_ps(scratch+j+16*q,v); __mmask16 mask=masks[q]; mx=_mm512_mask_max_ps(mx,mask,mx,v); count+=__builtin_popcount((unsigned)(mask&_mm512_cmp_ps_mask(v,_mm512_setzero_ps(),_CMP_GT_OQ))); } } /* The sixteen-window path sums in time order. Preserve that order for residual windows to make predictions independent of input chunking. */ float sum=0.f,ss=0.f; for(int j=0;j1?buf+s[4]*pitch:a,*e=c>2?buf+s[5]*pitch:a; #define WK(C) small_kernel(a,b,e,w+i*3*maxlen,C,len,d,nf,(float)bias[i],scratch,out+4*i) if(c==1)WK(1);else if(c==2)WK(2);else WK(3); #undef WK } } /* Window lanes reuse each coefficient over 16 windows by 8 positions. Pool each finished tile directly, retaining only the response scratch required for centered variance. */ __attribute__((target("avx512f,fma"),always_inline)) static inline void small_lanes(const float *a,const float *b,const float *e, const float *w,int c,int len,int d,int nf,float bias,float *work,double *out,int stride){ __m512 hi=_mm512_set1_ps(-INFINITY),sum=_mm512_setzero_ps(),pos=sum; __m512 one=_mm512_set1_ps(1.),zero=_mm512_setzero_ps(),v_bias=_mm512_set1_ps(bias); for(int j=0;j1?buf+pitch*s[4]:a,*e=c>2?buf+pitch*s[5]:a; #define SB(C) small_lanes(a,b,e,w+i*3*maxlen,C,len,d,nf,(float)bias[i],scratch,out+4*i,4*k) if(c==1)SB(1);else if(c==2)SB(2);else SB(3); #undef SB } } static int can_fuse(void){ if(!bind_numpy() || !__builtin_cpu_supports("avx2") || !__builtin_cpu_supports("fma"))return 0; Dl_info info; if(!dladdr((void*)np_gemv,&info))return 0; void *handle=dlopen(info.dli_fname,RTLD_LAZY|RTLD_LOCAL); if(!handle)return 0; const char *(*core)(void)=NULL; void *fn=dlsym(handle,"scipy_openblas_get_corename64_"); if(fn)memcpy(&core,&fn,sizeof core); int ok=core && !strcmp(core(),"SkylakeX"); dlclose(handle); return ok; } #else #define HAVE_FAST_X86 0 #endif static PyObject *fused_library(PyObject *self,PyObject *args) { #if HAVE_FAST_X86 Dl_info info; if(can_fuse() && dladdr((void*)np_gemv,&info)) return PyUnicode_DecodeFSDefault(info.dli_fname); #endif Py_RETURN_NONE; } static int view(PyObject *obj, Py_buffer *v, int rank, int writable, int integer) { int flags=PyBUF_FORMAT|PyBUF_C_CONTIGUOUS|(writable?PyBUF_WRITABLE:0); if (PyObject_GetBuffer(obj,v,flags)<0) return -1; int fmt=integer ? (!strcmp(v->format,"q") || !strcmp(v->format,"l")) : !strcmp(v->format,"d"); if (!fmt || v->ndim!=rank || v->itemsize!=8 || (uintptr_t)v->buf%8) { PyErr_SetString(PyExc_ValueError,"expected aligned contiguous int64 or float64 array of specified rank"); return -1; } return 0; } /* Eight independent partial sums and a balanced combination reproduce the NumPy pairwise reduction tree used by contiguous float64 sums. */ static double pair_sum(const double *x, Py_ssize_t n) { if(n<8) { double s=-0.; for(Py_ssize_t i=0;i128) { Py_ssize_t m=(n/2)/8*8; return pair_sum(x,m)+pair_sum(x+m,n-m); } double a[8]; memcpy(a,x,sizeof a); Py_ssize_t i=8; for(;i+8<=n;i+=8) for(int j=0;j<8;j++) a[j]+=x[i+j]; double s=((a[0]+a[1])+(a[2]+a[3]))+((a[4]+a[5])+(a[6]+a[7])); for(;i1)v=_mm512_add_pd(v,_mm512_mul_pd(_mm512_maskz_loadu_pd(mask,a+b),w1)); if(c>2)v=_mm512_add_pd(v,_mm512_mul_pd(_mm512_maskz_loadu_pd(mask,a+g),w2)); return v; } __attribute__((target("avx512f"))) static void wide_phase_project(double *r,const double *phase,Py_ssize_t pitch, Py_ssize_t stride,Py_ssize_t nf,const int64_t *s, const double *weights){ const int len=(int)s[0],d=(int)s[1],c=(int)s[2]; const double *x=phase+s[3]*stride*pitch; const Py_ssize_t b=c>1?(s[4]-s[3])*stride*pitch:0; const Py_ssize_t g=c>2?(s[5]-s[3])*stride*pitch:0; const double *tap[32]; /* Divide once for the entire filter, including non-power-of-two strides. */ const Py_ssize_t delta=d%stride, jump=delta*pitch+d/stride; Py_ssize_t residue=0,off=0,j=0; for(int h=0;h=stride){off+=1-stride*pitch;residue-=stride;} } for(;j+32<=nf;j+=32){ __m512d a0=_mm512_setzero_pd(),a1=a0,a2=a0,a3=a0; for(int h=0;h=8)?255:(__mmask8)((1u<<(nf-j))-1u); __m512d sum=_mm512_setzero_pd(); for(int h=0;h1?(s[4]-s[3])*stride*pitch:0; const Py_ssize_t g=c>2?(s[5]-s[3])*stride*pitch:0; memset(r,0,nf*sizeof(double)); for(int h=0;h8192 || n<1 || ch<1 || k<1 || m.shape[1]!=6 || w.shape[0]!=k || w.shape[2]!=3 || maxlen<1 || b.shape[0]!=k || y.shape[0]!=n || y.shape[1]!=k || y.shape[2]!=4 || stride<1 || stride>t || (!approximate && stride!=1) || (low && (approximate || stride!=1))) { PyErr_SetString(PyExc_ValueError,"invalid bank dimensions or stride"); goto finish; } Py_buffer *inputs[]={&x,&m,&w,&b}; for(int i=0;i<4;i++) { uintptr_t p=(uintptr_t)inputs[i]->buf,q=(uintptr_t)y.buf; if(plen) { PyErr_SetString(PyExc_ValueError,"output overlaps input"); goto finish; } } for(Py_ssize_t z=0;zmaxlen || d<1 || len-1>(t-1)/d || c<1 || c>3) { PyErr_SetString(PyExc_ValueError,"invalid kernel geometry"); goto finish; } if(!approximate && c>1 && (!bind_numpy() || t-(len-1)*d<=1)) { PyErr_SetString(PyExc_ValueError,"unsupported exact BLAS geometry"); goto finish; } for(int j=0;j=ch || (j && meta[3+j]<=meta[2+j])) { PyErr_SetString(PyExc_ValueError,"invalid kernel channels"); goto finish; } } #if HAVE_FAST_X86 int wide = __builtin_cpu_supports("avx512f") && __builtin_cpu_supports("popcnt"); fused = fused && !approximate && (__builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma")) && t>=16; #endif work=PyMem_Malloc(5*t*sizeof(double)); if (!work) { PyErr_NoMemory(); goto finish; } if(low){ /* Unusual custom coefficients or unnormalized signals can exceed FP32. The bound covers convolution, reduction and centered squared sums. */ double xmax=0.,wmax=0.,bmax=0.; const double *values[3]={x.buf,w.buf,b.buf}; Py_ssize_t sizes[3]={x.len/8,w.len/8,b.len/8}; double *maxima[3]={&xmax,&wmax,&bmax}; for(int j=0;j<3;j++)for(Py_ssize_t i=0;i*maxima[j])*maxima[j]=v; } if(xmax>FLT_MAX || wmax>FLT_MAX || bmax>FLT_MAX || 3.*maxlen*xmax*wmax+bmax>sqrt((double)FLT_MAX/(16.*t)))low=0; } if(low){ #if HAVE_FAST_X86 if(!wide || !__builtin_cpu_supports("fma")){ #endif PyErr_SetString(PyExc_ValueError,"conv32 requires an AVX-512F/FMA/POPCNT CPU");goto finish; #if HAVE_FAST_X86 } if((size_t)(ch+1)>SIZE_MAX/((size_t)t+64)/(16*sizeof(float))){ PyErr_NoMemory();goto finish; } small=PyMem_Calloc(((size_t)ch+1)*(t+64)*(n>=16?16:1),sizeof(float));smallw=PyMem_Malloc(w.len/2); if(!small||!smallw){PyErr_NoMemory();goto finish;} for(Py_ssize_t i=0;i1 && ch<=64){ phase=PyMem_Malloc(ch*stride*pitch*sizeof(double)); if(!phase){PyErr_NoMemory();goto finish;} } Py_ssize_t first=0,block=phase?1:12; #if HAVE_FAST_X86 BankItem *items=NULL; if(!low && wide && fused && n>=8 && ch<=8 && t<=256){ bt=PyMem_Malloc((ch+2)*t*8*sizeof(double));if(!bt){PyErr_NoMemory();goto finish;} items=PyMem_Malloc(k*sizeof(*items));if(!items){PyErr_NoMemory();goto finish;} for(Py_ssize_t z=0;z1?t*(s[4]-a0):t; Py_ssize_t nf=1+(t-1-(len-1)*d)/stride; const double *base=X+(i*ch+a0)*t; if (!phase && c>2 && s[5]-s[4]!=s[4]-a0) { for(int j=0;j=16) { int ix=len==7?0:len==9?1:len==11?2:len==15?3:-1; if(ix>=0)strips[c-1][ix](r,base,lda,(int)nf,W+z*maxlen*3,(int)d); else wide_project(r,base,lda,nf,W+z*maxlen*3,c,len,d); } else if(fused && c>1) { skylakex_project(r,base,lda,nf,W+z*maxlen*3,c,len,d); } else #endif { memset(r,0,nf*sizeof(double)); for(int64_t h=0;h=16) { int special=0; double mu=reduce_window(r,nf,bias[z],0,&pos,&mx,&special)/nf; if(special || mx==0.){ mx=-INFINITY; for(Py_ssize_t j=0;j=mx || isnan(r[j]))mx=r[j]; } double *dst=Y+(i*k+z)*4; dst[0]=(double)pos/nf;dst[1]=mx;dst[2]=mu; dst[3]=sqrt(reduce_window(r,nf,mu,1,&pos,&mx,&special)/nf); continue; } #endif for(Py_ssize_t j=0;j0; if(r[j]>=mx || isnan(r[j])) mx=r[j]; } double mu=pair_sum(r,nf)/nf; for(Py_ssize_t j=0;j