Instructions to use Zipeng365/WISP with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Scikit-learn
How to use Zipeng365/WISP with Scikit-learn:
from huggingface_hub import hf_hub_download import joblib model = joblib.load( hf_hub_download("Zipeng365/WISP", "sklearn_model.joblib") ) # only load pickle files from sources you trust # read more about it here https://skops.readthedocs.io/en/stable/persistence.html - Notebooks
- Google Colab
- Kaggle
Download src/wisp_fast/cis_conv.c from Zipeng365/WISP: direct link, hf CLI and curl.
- Browser
- Download file 38.6 kB
-
https://huggingface.co/Zipeng365/WISP/resolve/main/src/wisp_fast/cis_conv.c
- Command line
-
hf download hf://Zipeng365/WISP/src/wisp_fast/cis_conv.c
-
curl -L -o cis_conv.c https://huggingface.co/Zipeng365/WISP/resolve/main/src/wisp_fast/cis_conv.c
38.6 kB
| /* Packed random convolution, preserving the tap accumulation order. The | |
| multichannel exact path reuses the BLAS already linked into NumPy's | |
| core, including its choice of fused operations. */ | |
| typedef void (*gemv64_t)(int,int,int64_t,int64_t,double,const double*,int64_t, | |
| const double*,int64_t,double,double*,int64_t); | |
| static gemv64_t np_gemv; | |
| static int attempted; | |
| static int bind_numpy(void) { | |
| if (!attempted) { | |
| attempted = 1; | |
| PyObject *core = PyImport_ImportModule("numpy._core._multiarray_umath"); | |
| if (!core) { PyErr_Clear(); return 0; } | |
| PyObject *file = PyModule_GetFilenameObject(core); | |
| Py_DECREF(core); | |
| if (file) { | |
| const char *path = PyUnicode_AsUTF8(file); | |
| void *h = path ? dlopen(path, RTLD_LAZY | RTLD_LOCAL) : NULL; | |
| if (h) { | |
| void *symbol = dlsym(h, "scipy_cblas_dgemv64_"); | |
| if (symbol) memcpy(&np_gemv, &symbol, sizeof np_gemv); | |
| /* The NumPy module keeps this dependency loaded. */ | |
| dlclose(h); | |
| } | |
| Py_DECREF(file); | |
| } | |
| if (PyErr_Occurred()) PyErr_Clear(); | |
| } | |
| return np_gemv != NULL; | |
| } | |
| static PyObject *has_blas(PyObject *self,PyObject *args) { | |
| return PyBool_FromLong(bind_numpy()); | |
| } | |
| /* This order is specific to the inspected NumPy OpenBLAS SkylakeX kernel | |
| shipped with the pinned Linux x86-64 wheel. Python additionally verifies | |
| the library's full SHA-256. Other builds use actual calls to GEMV. */ | |
| /* Four vectors share weight broadcasts. Keep pertap contributions in four | |
| registers and only materialize responses once, after the last tap. */ | |
| __attribute__((target("avx2,fma"))) | |
| static void skylakex_project(double *r,const double *base,Py_ssize_t lda,Py_ssize_t frames, | |
| const double *weights,int c,int64_t len,int64_t dilation) { | |
| Py_ssize_t j=0; | |
| for (;j+16<=frames;j+=16) { | |
| __m256d acc[4]={_mm256_setzero_pd(),_mm256_setzero_pd(),_mm256_setzero_pd(),_mm256_setzero_pd()}; | |
| for(int h=0;h<len;h++){ | |
| const double *v=weights+3*h,*a=base+j+h*dilation; | |
| __m256d w0=_mm256_set1_pd(v[0]),w1=_mm256_set1_pd(v[1]),w2=_mm256_set1_pd(v[2]); | |
| for(int q=0;q<4;q++) { | |
| __m256d dot=_mm256_mul_pd(_mm256_loadu_pd(a+4*q+lda),w1); | |
| dot=_mm256_fmadd_pd(_mm256_loadu_pd(a+4*q),w0,dot); | |
| if(c==3) dot=_mm256_add_pd(dot,_mm256_mul_pd(_mm256_loadu_pd(a+4*q+2*lda),w2)); | |
| acc[q]=_mm256_add_pd(acc[q],dot); | |
| } | |
| } | |
| for(int q=0;q<4;q++) _mm256_storeu_pd(r+j+4*q,acc[q]); | |
| } | |
| for(;j+4<=frames;j+=4){ | |
| __m256d acc=_mm256_setzero_pd(); | |
| for(int h=0;h<len;h++){ | |
| const double *v=weights+3*h,*a=base+j+h*dilation; | |
| __m256d dot=_mm256_mul_pd(_mm256_loadu_pd(a+lda),_mm256_set1_pd(v[1])); | |
| dot=_mm256_fmadd_pd(_mm256_loadu_pd(a),_mm256_set1_pd(v[0]),dot); | |
| if(c==3) dot=_mm256_add_pd(dot,_mm256_mul_pd(_mm256_loadu_pd(a+2*lda),_mm256_set1_pd(v[2]))); | |
| acc=_mm256_add_pd(acc,dot); | |
| } | |
| _mm256_storeu_pd(r+j,acc); | |
| } | |
| for(;j<frames;j++){ | |
| double acc=0; | |
| for(int h=0;h<len;h++){ | |
| const double *v=weights+3*h,*a=base+j+h*dilation; | |
| double dot=fma(a[lda],v[1],a[0]*v[0]); | |
| if(c==3)dot=fma(a[2*lda],v[2],dot); | |
| acc+=dot; | |
| } | |
| r[j]=acc; | |
| } | |
| } | |
| /* Specialization by channel count hoists branches out of the hot loop. | |
| Each coefficient broadcast is shared by four eight-position tiles. */ | |
| __attribute__((target("avx512f,fma"),always_inline)) | |
| static inline void tile_project(double *r,const double *base,Py_ssize_t lda,Py_ssize_t frames, | |
| const double *weights,int c,int64_t len,int64_t dilation){ | |
| Py_ssize_t j=0; | |
| for(;j+32<=frames;j+=32){ | |
| __m512d x[4]={_mm512_setzero_pd(),_mm512_setzero_pd(),_mm512_setzero_pd(),_mm512_setzero_pd()}; | |
| for(int h=0;h<len;h++){ | |
| const double *v=weights+3*h,*s=base+j+h*dilation; | |
| __m512d w0=_mm512_set1_pd(v[0]),w1=_mm512_set1_pd(v[1]),w2=_mm512_set1_pd(v[2]); | |
| for(int q=0;q<4;q++){ | |
| const double *a=s+8*q; | |
| __m512d av=_mm512_loadu_pd(a),dot; | |
| if(c==1)dot=_mm512_mul_pd(av,w0); | |
| else{ | |
| dot=_mm512_fmadd_pd(av,w0,_mm512_mul_pd(_mm512_loadu_pd(a+lda),w1)); | |
| if(c==3)dot=_mm512_add_pd(dot,_mm512_mul_pd(_mm512_loadu_pd(a+2*lda),w2)); | |
| } | |
| x[q]=_mm512_add_pd(x[q],dot); | |
| } | |
| } | |
| for(int q=0;q<4;q++)_mm512_storeu_pd(r+j+8*q,x[q]); | |
| } | |
| for(;j+8<=frames;j+=8){ | |
| __m512d x=_mm512_setzero_pd(); | |
| for(int h=0;h<len;h++){ | |
| const double *v=weights+3*h,*a=base+j+h*dilation; | |
| __m512d av=_mm512_loadu_pd(a),w0=_mm512_set1_pd(v[0]),dot; | |
| if(c==1)dot=_mm512_mul_pd(av,w0); | |
| else{ | |
| dot=_mm512_fmadd_pd(av,w0,_mm512_mul_pd(_mm512_loadu_pd(a+lda),_mm512_set1_pd(v[1]))); | |
| if(c==3)dot=_mm512_add_pd(dot,_mm512_mul_pd(_mm512_loadu_pd(a+2*lda),_mm512_set1_pd(v[2]))); | |
| } | |
| x=_mm512_add_pd(x,dot); | |
| } | |
| _mm512_storeu_pd(r+j,x); | |
| } | |
| if(c>1)skylakex_project(r+j,base+j,lda,frames-j,weights,c,len,dilation); | |
| else for(;j<frames;j++){ | |
| double a=0.; | |
| for(int h=0;h<len;h++)a+=base[j+h*dilation]*weights[3*h]; | |
| r[j]=a; | |
| } | |
| } | |
| __attribute__((target("avx512f,fma"))) | |
| static void wide_project(double *r,const double *base,Py_ssize_t lda,Py_ssize_t frames, | |
| const double *weights,int c,int64_t len,int64_t dilation){ | |
| if(c==1)tile_project(r,base,lda,frames,weights,1,len,dilation); | |
| else if(c==2)tile_project(r,base,lda,frames,weights,2,len,dilation); | |
| else tile_project(r,base,lda,frames,weights,3,len,dilation); | |
| } | |
| /* Fixed-size response tile: a whole eight-vector strip shares each tap. | |
| Mask the short suffix and explicitly retain the special 1--3 frame BLAS | |
| contraction at the very end of a multi-channel projection. */ | |
| __attribute__((target("avx512f,fma"),always_inline)) | |
| static inline void project_strip(double *r,const double *base,Py_ssize_t lda,int frames, | |
| const double *weights,int c,int len,int dilation) { | |
| int j=0; | |
| for(;j+64<=frames;j+=64){ | |
| __m512d acc[8];for(int q=0;q<8;q++) acc[q]=_mm512_setzero_pd(); | |
| for(int h=0;h<len;h++){ | |
| const double *a=base+j+h*dilation,*v=weights+3*h; | |
| __m512d w0=_mm512_set1_pd(v[0]),w1=_mm512_set1_pd(v[1]),w2=_mm512_set1_pd(v[2]); | |
| for(int q=0;q<8;q++){ | |
| __m512d x=_mm512_loadu_pd(a+8*q); | |
| __m512d dot=c==1?_mm512_mul_pd(x,w0): | |
| _mm512_fmadd_pd(x,w0,_mm512_mul_pd(_mm512_loadu_pd(a+lda+8*q),w1)); | |
| if(c==3)dot=_mm512_add_pd(dot,_mm512_mul_pd(_mm512_loadu_pd(a+2*lda+8*q),w2)); | |
| acc[q]=_mm512_add_pd(acc[q],dot); | |
| } | |
| } | |
| for(int q=0;q<8;q++)_mm512_storeu_pd(r+j+8*q,acc[q]); | |
| } | |
| tile_project(r+j,base+j,lda,frames-j,weights,c,len,dilation); | |
| } | |
| STRIP_L(1) STRIP_L(2) STRIP_L(3) | |
| typedef void (*strip_fn)(double*,const double*,Py_ssize_t,int,const double*,int); | |
| static strip_fn strips[3][4]={STRIP_ROW(1),STRIP_ROW(2),STRIP_ROW(3)}; | |
| /* NumPy sums each <=128-value leaf in eight lanes, then combines pairs. | |
| Fuse both traversals: bias + max + PPV + mean; squared deviations + sum. | |
| Larger inputs recurse at the same eight-aligned split as pair_sum. */ | |
| __attribute__((target("avx512f,popcnt"),always_inline)) | |
| static inline double leaf_reduce(double *r,Py_ssize_t n,double shift,int central, | |
| Py_ssize_t *pos,double *maximum,int *special) { | |
| __m512d sum=_mm512_setzero_pd(),hi=_mm512_set1_pd(-INFINITY); | |
| const __m512d s=_mm512_set1_pd(shift); | |
| Py_ssize_t j=0; | |
| for(;j+8<=n;j+=8) { | |
| __m512d v=_mm512_loadu_pd(r+j); | |
| if(central){ | |
| v=_mm512_sub_pd(v,s);v=_mm512_mul_pd(v,v); | |
| }else{ | |
| v=_mm512_add_pd(v,s);_mm512_storeu_pd(r+j,v); | |
| *pos+=__builtin_popcount((unsigned)_mm512_cmp_pd_mask(v,_mm512_setzero_pd(),_CMP_GT_OQ)); | |
| *special|=_mm512_cmp_pd_mask(v,v,_CMP_UNORD_Q); | |
| hi=_mm512_max_pd(hi,v); | |
| } | |
| sum=j?_mm512_add_pd(sum,v):v; | |
| } | |
| double total=-0.; | |
| if(j){ | |
| double a[8];_mm512_storeu_pd(a,sum); | |
| total=((a[0]+a[1])+(a[2]+a[3]))+((a[4]+a[5])+(a[6]+a[7])); | |
| if(!central){ | |
| double peak=_mm512_reduce_max_pd(hi); | |
| if(peak>=*maximum)*maximum=peak; | |
| } | |
| } | |
| for(;j<n;j++){ | |
| double v=r[j]; | |
| if(central){v-=shift;v*=v;} | |
| else{ | |
| v+=shift;r[j]=v;*pos+=v>0.;*special|=isnan(v); | |
| if(v>=*maximum)*maximum=v; | |
| } | |
| total+=v; | |
| } | |
| return total; | |
| } | |
| __attribute__((target("avx512f,popcnt"))) | |
| static double reduce_window(double *r,Py_ssize_t n,double shift,int central, | |
| Py_ssize_t *pos,double *maximum,int *special) { | |
| if(n<=128)return leaf_reduce(r,n,shift,central,pos,maximum,special); | |
| Py_ssize_t m=(n/2)/8*8; | |
| double lo=reduce_window(r,m,shift,central,pos,maximum,special); | |
| return lo+reduce_window(r+m,n-m,shift,central,pos,maximum,special); | |
| } | |
| /* A SIMD lane owns an independent window. This allows reductions to follow | |
| the reference's exact time order without horizontal-vector reductions. */ | |
| __attribute__((target("avx512f"),always_inline)) | |
| static inline __m512d batch_sum(const double *r,int n){ | |
| if(n<8){ | |
| __m512d acc=_mm512_set1_pd(-0.); | |
| for(int j=0;j<n;j++)acc=_mm512_add_pd(acc,_mm512_loadu_pd(r+8*j)); | |
| return acc; | |
| } | |
| __m512d lane[8]; | |
| for(int q=0;q<8;q++)lane[q]=_mm512_loadu_pd(r+8*q); | |
| int j=8; | |
| for(;j+8<=n;j+=8)for(int q=0;q<8;q++)lane[q]=_mm512_add_pd(lane[q],_mm512_loadu_pd(r+8*(j+q))); | |
| __m512d x=_mm512_add_pd(_mm512_add_pd(_mm512_add_pd(lane[0],lane[1]),_mm512_add_pd(lane[2],lane[3])), | |
| _mm512_add_pd(_mm512_add_pd(lane[4],lane[5]),_mm512_add_pd(lane[6],lane[7]))); | |
| for(;j<n;j++)x=_mm512_add_pd(x,_mm512_loadu_pd(r+j*8)); | |
| return x; | |
| } | |
| /* At lengths 249..255 the right leaf itself exceeds 128. */ | |
| __attribute__((target("avx512f"))) | |
| static __m512d batch_reduce(const double *r,int n){ | |
| if(n<=128)return batch_sum(r,n); | |
| int cut=n/16*8; | |
| return _mm512_add_pd(batch_reduce(r,cut),batch_reduce(r+cut*8,n-cut)); | |
| } | |
| __attribute__((target("avx512f"))) static void finish_projection(double *r,int nf,double *out,Py_ssize_t outjump){ | |
| __m512d m=batch_reduce(r,nf); | |
| m=_mm512_div_pd(m,_mm512_set1_pd(nf)); | |
| __m512d mx=_mm512_set1_pd(-INFINITY),pos=_mm512_setzero_pd(),one=_mm512_set1_pd(1.); | |
| __mmask8 nan=0; | |
| for(int j=0;j<nf;j++){ | |
| __m512d v=_mm512_loadu_pd(r+j*8);nan|=_mm512_cmp_pd_mask(v,v,_CMP_UNORD_Q); | |
| mx=_mm512_max_pd(mx,v);pos=_mm512_mask_add_pd(pos,_mm512_cmp_pd_mask(v,_mm512_setzero_pd(),_CMP_GT_OQ),pos,one); | |
| v=_mm512_sub_pd(v,m);_mm512_storeu_pd(r+j*8,_mm512_mul_pd(v,v)); | |
| } | |
| __m512d sum=batch_reduce(r,nf); | |
| double d[4][8]; | |
| _mm512_storeu_pd(d[0],_mm512_div_pd(pos,_mm512_set1_pd(nf)));_mm512_storeu_pd(d[1],_mm512_mask_mov_pd(mx,nan,_mm512_set1_pd(NAN))); | |
| _mm512_storeu_pd(d[2],m);_mm512_storeu_pd(d[3],_mm512_sqrt_pd(_mm512_div_pd(sum,_mm512_set1_pd(nf)))); | |
| for(int lane=0;lane<8;lane++)for(int q=0;q<4;q++)out[lane*outjump+q]=d[q][lane]; | |
| } | |
| /* Sort a transient index of the fitted bank by shape, without reordering | |
| the output columns. Paired kernels share input vectors only. */ | |
| typedef struct {int64_t geom[6], idx;} BankItem; | |
| static int cmp_bank(const void *x,const void *y){ | |
| const BankItem *a=x,*b=y;return memcmp(a->geom,b->geom,sizeof a->geom); | |
| } | |
| __attribute__((target("avx512f,fma"),always_inline)) | |
| static inline void pair_kernel(const double *s0,const double *s1,const double *s2,const double *w,const double *w2, | |
| double bias,double bias2,int c,int len,int dil,int nf,double *r,double *r2,double *out,double *out2,Py_ssize_t jump){ | |
| int main=nf/8*8,j=0; | |
| for(;j<main;j+=8){ | |
| __m512d v[8],u[8];for(int q=0;q<8;++q)v[q]=u[q]=_mm512_setzero_pd(); | |
| for(int h=0;h<len;++h){ | |
| int off=(j+h*dil)*8; | |
| __m512d wa=_mm512_set1_pd(w[3*h]),wb=_mm512_set1_pd(w[3*h+1]),wc=_mm512_set1_pd(w[3*h+2]); | |
| __m512d xa=_mm512_set1_pd(w2[3*h]),xb=_mm512_set1_pd(w2[3*h+1]),xc=_mm512_set1_pd(w2[3*h+2]); | |
| for(int q=0;q<8;++q,off+=8){ | |
| __m512d a=_mm512_loadu_pd(s0+off), b=_mm512_loadu_pd(s1+off),e=_mm512_loadu_pd(s2+off); | |
| __m512d p=a*wa,z=a*xa; if(c>1){p=_mm512_fmadd_pd(a,wa,b*wb);z=_mm512_fmadd_pd(a,xa,b*xb);} | |
| if(c==3){p+=e*wc;z+=e*xc;}v[q]+=p;u[q]+=z; | |
| } | |
| } | |
| for(int q=0;q<8;++q){_mm512_storeu_pd(r+8*(j+q),v[q]+bias);_mm512_storeu_pd(r2+8*(j+q),u[q]+bias2);} | |
| } | |
| for(;j<nf;++j){ | |
| __m512d v=_mm512_setzero_pd(),u=v; | |
| for(int h=0;h<len;++h){int off=(j+h*dil)*8; | |
| __m512d a=_mm512_loadu_pd(s0+off),b=_mm512_loadu_pd(s1+off),e=_mm512_loadu_pd(s2+off); | |
| __m512d p=a*w[3*h], z=a*w2[3*h]; | |
| if(c>1){ | |
| if(j<((nf/4)*4)){p=_mm512_fmadd_pd(a,_mm512_set1_pd(w[3*h]),b*w[3*h+1]);z=_mm512_fmadd_pd(a,_mm512_set1_pd(w2[3*h]),b*w2[3*h+1]);} | |
| else {p=_mm512_fmadd_pd(b,_mm512_set1_pd(w[3*h+1]),p);z=_mm512_fmadd_pd(b,_mm512_set1_pd(w2[3*h+1]),z);} | |
| } | |
| if(c>2){ | |
| if(j<((nf/4)*4)){p+=e*w[3*h+2];z+=e*w2[3*h+2];} | |
| else{p=_mm512_fmadd_pd(e,_mm512_set1_pd(w[3*h+2]),p);z=_mm512_fmadd_pd(e,_mm512_set1_pd(w2[3*h+2]),z);} | |
| } | |
| v+=p;u+=z; | |
| } | |
| _mm512_storeu_pd(r+j*8,v+bias);_mm512_storeu_pd(r2+j*8,u+bias2); | |
| } | |
| finish_projection(r,nf,out,jump);finish_projection(r2,nf,out2,jump); | |
| } | |
| __attribute__((target("avx512f,fma"),always_inline)) | |
| static inline void batch_kernel(const double *s0,const double *s1,const double *s2, | |
| const double *w,double bias,int c,int len,int dil,int nf, | |
| double *r,double *out,Py_ssize_t outjump){ | |
| int main=nf/8*8,j=0; | |
| for(;j<main;j+=8){ | |
| __m512d v[8];for(int q=0;q<8;q++)v[q]=_mm512_setzero_pd(); | |
| for(int h=0;h<len;h++){ | |
| int off=(j+h*dil)*8; | |
| __m512d w0=_mm512_set1_pd(w[3*h]),w1=_mm512_set1_pd(w[3*h+1]),w2=_mm512_set1_pd(w[3*h+2]); | |
| for(int q=0;q<8;q++,off+=8){ | |
| __m512d a=_mm512_loadu_pd(s0+off); | |
| __m512d p=c==1?_mm512_mul_pd(a,w0):_mm512_fmadd_pd(a,w0,_mm512_mul_pd(_mm512_loadu_pd(s1+off),w1)); | |
| if(c==3)p=_mm512_add_pd(p,_mm512_mul_pd(_mm512_loadu_pd(s2+off),w2)); | |
| v[q]=_mm512_add_pd(v[q],p); | |
| } | |
| } | |
| for(int q=0;q<8;q++)_mm512_storeu_pd(r+8*(j+q),_mm512_add_pd(v[q],_mm512_set1_pd(bias))); | |
| } | |
| if(j+4<=nf){ | |
| __m512d acc[4];for(int q=0;q<4;q++)acc[q]=_mm512_setzero_pd(); | |
| for(int h=0;h<len;h++){ | |
| int off=(j+h*dil)*8;__m512d w0=_mm512_set1_pd(w[3*h]),w1=_mm512_set1_pd(w[3*h+1]),w2=_mm512_set1_pd(w[3*h+2]); | |
| for(int q=0;q<4;q++,off+=8){ | |
| __m512d u=_mm512_loadu_pd(s0+off),z=c==1?_mm512_mul_pd(u,w0):_mm512_fmadd_pd(u,w0,_mm512_mul_pd(_mm512_loadu_pd(s1+off),w1)); | |
| if(c==3)z=_mm512_add_pd(z,_mm512_mul_pd(_mm512_loadu_pd(s2+off),w2)); | |
| acc[q]=_mm512_add_pd(acc[q],z); | |
| } | |
| } | |
| for(int q=0;q<4;q++)_mm512_storeu_pd(r+8*(j+q),_mm512_add_pd(acc[q],_mm512_set1_pd(bias))); | |
| j+=4; | |
| } | |
| for(;j<nf;j++){ | |
| __m512d v=_mm512_setzero_pd(); | |
| for(int h=0;h<len;h++){ | |
| int off=(j+h*dil)*8; | |
| __m512d p=_mm512_mul_pd(_mm512_loadu_pd(s0+off),_mm512_set1_pd(w[3*h])); | |
| if(c>1)p=_mm512_fmadd_pd(_mm512_loadu_pd(s1+off),_mm512_set1_pd(w[3*h+1]),p); | |
| if(c>2)p=_mm512_fmadd_pd(_mm512_loadu_pd(s2+off),_mm512_set1_pd(w[3*h+2]),p); | |
| v=_mm512_add_pd(v,p); | |
| } | |
| _mm512_storeu_pd(r+j*8,_mm512_add_pd(v,_mm512_set1_pd(bias))); | |
| } | |
| finish_projection(r,nf,out,outjump); | |
| } | |
| __attribute__((target("avx512f,fma"))) | |
| static void batch_bank(const double *x,Py_ssize_t ch,int t,const int64_t *rows,const double *weights, | |
| const double *bias,Py_ssize_t k,int maxlen,double *out,double *packed,double *r,const BankItem *items){ | |
| for(Py_ssize_t c=0;c<ch;c++)for(int j=0;j<t;j++)for(int a=0;a<8;a++) | |
| packed[(c*t+j)*8+a]=x[(a*ch+c)*t+j]; | |
| for(Py_ssize_t item=0;item<k;item++){ | |
| Py_ssize_t z=items[item].idx; | |
| const int64_t *s=rows+6*z;int len=s[0],d=s[1],c=s[2],nf=t-(len-1)*d; | |
| const double *a=packed+8*t*s[3],*b=c>1?packed+8*t*s[4]:a,*e=c>2?packed+8*t*s[5]:a; | |
| if(item+1<k && !memcmp(items[item].geom,items[item+1].geom,6*sizeof(int64_t))){ | |
| Py_ssize_t zz=items[++item].idx; | |
| if(c==1)PK(1);else if(c==2)PK(2);else PK(3); | |
| continue; | |
| } | |
| if(c==1)BK(1);else if(c==2)BK(2);else BK(3); | |
| } | |
| } | |
| /* Optional FP32 projections for already fitted FP64 banks. Sixteen retained | |
| time positions share a vector. The two-pass centered variance avoids the | |
| cancellation of E[X^2]-E[X]^2 near a constant. */ | |
| __attribute__((target("avx512f,fma,popcnt"),always_inline)) | |
| static inline void small_kernel(const float *x,const float *y,const float *z, | |
| const float *w,int c,int len,int d,int nf,float bias, | |
| float *scratch,double *out){ | |
| int count=0;__m512 mx=_mm512_set1_ps(-INFINITY); | |
| for(int j=0;j<nf;j+=64){ | |
| int left=nf-j; | |
| __m512 acc[4];__mmask16 masks[4]; | |
| for(int q=0;q<4;q++){ | |
| int take=left-16*q; | |
| masks[q]=take<=0?0:take>=16?0xffff:(1u<<take)-1u; | |
| acc[q]=_mm512_setzero_ps(); | |
| } | |
| for(int h=0;h<len;h++){ | |
| __m512 w0=_mm512_set1_ps(w[3*h]),w1=_mm512_set1_ps(w[3*h+1]),w2=_mm512_set1_ps(w[3*h+2]); | |
| for(int q=0;q<4;q++){ | |
| int off=j+d*h+q*16; | |
| /* Input rows are padded by 64 floats, allowing these full | |
| loads even on the last partial tile. */ | |
| __m512 u=_mm512_loadu_ps(x+off); | |
| if(c==1)acc[q]=_mm512_fmadd_ps(u,w0,acc[q]); | |
| else { | |
| __m512 v=_mm512_fmadd_ps(u,w0,_mm512_mul_ps(_mm512_loadu_ps(y+off),w1)); | |
| if(c>2)v=_mm512_fmadd_ps(_mm512_loadu_ps(z+off),w2,v); | |
| acc[q]=_mm512_add_ps(acc[q],v); | |
| } | |
| } | |
| } | |
| for(int q=0;q<4;q++){ | |
| __m512 v=_mm512_add_ps(acc[q],_mm512_set1_ps(bias)); | |
| _mm512_storeu_ps(scratch+j+16*q,v); | |
| __mmask16 mask=masks[q]; | |
| mx=_mm512_mask_max_ps(mx,mask,mx,v); | |
| count+=__builtin_popcount((unsigned)(mask&_mm512_cmp_ps_mask(v,_mm512_setzero_ps(),_CMP_GT_OQ))); | |
| } | |
| } | |
| /* The sixteen-window path sums in time order. Preserve that order for | |
| residual windows to make predictions independent of input chunking. */ | |
| float sum=0.f,ss=0.f; | |
| for(int j=0;j<nf;j++)sum+=scratch[j]; | |
| float mu=sum/nf; | |
| for(int j=0;j<nf;j++){float u=scratch[j]-mu;ss=fmaf(u,u,ss);} | |
| out[0]=(double)count/nf;out[1]=_mm512_reduce_max_ps(mx); | |
| out[2]=mu;out[3]=sqrtf(ss/nf); | |
| } | |
| __attribute__((target("avx512f,fma,popcnt"))) | |
| static void small_bank(const double *x,Py_ssize_t ch,int t,const int64_t *rows,const float *w, | |
| const double *bias,Py_ssize_t k,int maxlen,float *buf,double *out){ | |
| int pitch=t+64; | |
| float *scratch=buf+ch*pitch; | |
| for(Py_ssize_t c=0;c<ch;c++) | |
| for(int j=0;j<t;j++)buf[c*pitch+j]=(float)x[c*t+j]; | |
| for(Py_ssize_t i=0;i<k;i++){ | |
| const int64_t *s=rows+6*i;int len=s[0],d=s[1],c=s[2],nf=t-(len-1)*d; | |
| const float *a=buf+s[3]*pitch,*b=c>1?buf+s[4]*pitch:a,*e=c>2?buf+s[5]*pitch:a; | |
| if(c==1)WK(1);else if(c==2)WK(2);else WK(3); | |
| } | |
| } | |
| /* Window lanes reuse each coefficient over 16 windows by 8 positions. | |
| Pool each finished tile directly, retaining only the response scratch | |
| required for centered variance. */ | |
| __attribute__((target("avx512f,fma"),always_inline)) | |
| static inline void small_lanes(const float *a,const float *b,const float *e, | |
| const float *w,int c,int len,int d,int nf,float bias,float *work,double *out,int stride){ | |
| __m512 hi=_mm512_set1_ps(-INFINITY),sum=_mm512_setzero_ps(),pos=sum; | |
| __m512 one=_mm512_set1_ps(1.),zero=_mm512_setzero_ps(),v_bias=_mm512_set1_ps(bias); | |
| for(int j=0;j<nf;j+=8){ | |
| int take=nf-j<8?nf-j:8;__m512 v[8]; | |
| for(int q=0;q<8;q++)v[q]=zero; | |
| for(int h=0;h<len;h++){ | |
| int start=(j+h*d)*16; | |
| __m512 w0=_mm512_set1_ps(w[3*h]),w1=_mm512_set1_ps(w[3*h+1]),w2=_mm512_set1_ps(w[3*h+2]); | |
| for(int q=0;q<8;q++){ | |
| int off=start+16*q;__m512 x=_mm512_loadu_ps(a+off); | |
| if(c==1)v[q]=_mm512_fmadd_ps(x,w0,v[q]); | |
| else{ | |
| __m512 dot=_mm512_fmadd_ps(x,w0,_mm512_mul_ps(_mm512_loadu_ps(b+off),w1)); | |
| if(c==3)dot=_mm512_fmadd_ps(_mm512_loadu_ps(e+off),w2,dot); | |
| v[q]=_mm512_add_ps(v[q],dot); | |
| } | |
| } | |
| } | |
| for(int q=0;q<take;q++){ | |
| __m512 u=_mm512_add_ps(v[q],v_bias); | |
| _mm512_storeu_ps(work+(j+q)*16,u); | |
| hi=_mm512_max_ps(hi,u);sum=_mm512_add_ps(sum,u); | |
| pos=_mm512_mask_add_ps(pos,_mm512_cmp_ps_mask(u,zero,_CMP_GT_OQ),pos,one); | |
| } | |
| } | |
| __m512 avg=_mm512_div_ps(sum,_mm512_set1_ps(nf)),ss=zero; | |
| for(int j=0;j<nf;j++){ | |
| __m512 u=_mm512_sub_ps(_mm512_loadu_ps(work+j*16),avg); | |
| ss=_mm512_fmadd_ps(u,u,ss); | |
| } | |
| float pooled[4][16]; | |
| _mm512_storeu_ps(pooled[0],pos);_mm512_storeu_ps(pooled[1],hi); | |
| _mm512_storeu_ps(pooled[2],avg);_mm512_storeu_ps(pooled[3],_mm512_sqrt_ps(_mm512_div_ps(ss,_mm512_set1_ps(nf)))); | |
| for(int j=0;j<16;j++){ | |
| double *dst=out+j*stride;dst[0]=(double)pooled[0][j]/nf; | |
| for(int h=1;h<4;h++)dst[h]=pooled[h][j]; | |
| } | |
| } | |
| __attribute__((target("avx512f,fma"))) | |
| static void small_batch(const double *x,int ch,int t,const int64_t *rows,const float *w, | |
| const double *bias,int k,int maxlen,float *buf,double *out){ | |
| int pitch=(t+8)*16; | |
| float *scratch=buf+ch*pitch; | |
| for(int cc=0;cc<ch;cc++)for(int j=0;j<t;j++)for(int h=0;h<16;h++) | |
| buf[cc*pitch+j*16+h]=(float)x[(h*ch+cc)*t+j]; | |
| for(int i=0;i<k;i++){ | |
| const int64_t *s=rows+6*i; | |
| int c=s[2],len=s[0],d=s[1],nf=t-(len-1)*d; | |
| const float *a=buf+pitch*s[3],*b=c>1?buf+pitch*s[4]:a,*e=c>2?buf+pitch*s[5]:a; | |
| if(c==1)SB(1);else if(c==2)SB(2);else SB(3); | |
| } | |
| } | |
| static int can_fuse(void){ | |
| if(!bind_numpy() || !__builtin_cpu_supports("avx2") || !__builtin_cpu_supports("fma"))return 0; | |
| Dl_info info; | |
| if(!dladdr((void*)np_gemv,&info))return 0; | |
| void *handle=dlopen(info.dli_fname,RTLD_LAZY|RTLD_LOCAL); | |
| if(!handle)return 0; | |
| const char *(*core)(void)=NULL; | |
| void *fn=dlsym(handle,"scipy_openblas_get_corename64_"); | |
| if(fn)memcpy(&core,&fn,sizeof core); | |
| int ok=core && !strcmp(core(),"SkylakeX"); | |
| dlclose(handle); | |
| return ok; | |
| } | |
| static PyObject *fused_library(PyObject *self,PyObject *args) { | |
| Dl_info info; | |
| if(can_fuse() && dladdr((void*)np_gemv,&info)) | |
| return PyUnicode_DecodeFSDefault(info.dli_fname); | |
| Py_RETURN_NONE; | |
| } | |
| static int view(PyObject *obj, Py_buffer *v, int rank, int writable, int integer) { | |
| int flags=PyBUF_FORMAT|PyBUF_C_CONTIGUOUS|(writable?PyBUF_WRITABLE:0); | |
| if (PyObject_GetBuffer(obj,v,flags)<0) return -1; | |
| int fmt=integer ? (!strcmp(v->format,"q") || !strcmp(v->format,"l")) | |
| : !strcmp(v->format,"d"); | |
| if (!fmt || v->ndim!=rank || v->itemsize!=8 || (uintptr_t)v->buf%8) { | |
| PyErr_SetString(PyExc_ValueError,"expected aligned contiguous int64 or float64 array of specified rank"); | |
| return -1; | |
| } | |
| return 0; | |
| } | |
| /* Eight independent partial sums and a balanced combination reproduce the | |
| NumPy pairwise reduction tree used by contiguous float64 sums. */ | |
| static double pair_sum(const double *x, Py_ssize_t n) { | |
| if(n<8) { | |
| double s=-0.; | |
| for(Py_ssize_t i=0;i<n;i++) s+=x[i]; | |
| return s; | |
| } | |
| if(n>128) { | |
| Py_ssize_t m=(n/2)/8*8; | |
| return pair_sum(x,m)+pair_sum(x+m,n-m); | |
| } | |
| double a[8]; | |
| memcpy(a,x,sizeof a); | |
| Py_ssize_t i=8; | |
| for(;i+8<=n;i+=8) for(int j=0;j<8;j++) a[j]+=x[i+j]; | |
| double s=((a[0]+a[1])+(a[2]+a[3]))+((a[4]+a[5])+(a[6]+a[7])); | |
| for(;i<n;i++) s+=x[i]; | |
| return s; | |
| } | |
| /* Each SIMD lane is an independent retained response. Keep tap order and the | |
| unfused per-channel arithmetic of the scalar approximation. Mask the final | |
| load as well as its store: a phase row need not contain eight extra values. */ | |
| __attribute__((target("avx512f"))) | |
| static inline __m512d phase_dot(const double *a,Py_ssize_t b,Py_ssize_t g, | |
| __m512d w0,__m512d w1,__m512d w2,int c,__mmask8 mask){ | |
| __m512d v=_mm512_mul_pd(_mm512_maskz_loadu_pd(mask,a),w0); | |
| if(c>1)v=_mm512_add_pd(v,_mm512_mul_pd(_mm512_maskz_loadu_pd(mask,a+b),w1)); | |
| if(c>2)v=_mm512_add_pd(v,_mm512_mul_pd(_mm512_maskz_loadu_pd(mask,a+g),w2)); | |
| return v; | |
| } | |
| __attribute__((target("avx512f"))) | |
| static void wide_phase_project(double *r,const double *phase,Py_ssize_t pitch, | |
| Py_ssize_t stride,Py_ssize_t nf,const int64_t *s, | |
| const double *weights){ | |
| const int len=(int)s[0],d=(int)s[1],c=(int)s[2]; | |
| const double *x=phase+s[3]*stride*pitch; | |
| const Py_ssize_t b=c>1?(s[4]-s[3])*stride*pitch:0; | |
| const Py_ssize_t g=c>2?(s[5]-s[3])*stride*pitch:0; | |
| const double *tap[32]; | |
| /* Divide once for the entire filter, including non-power-of-two strides. */ | |
| const Py_ssize_t delta=d%stride, jump=delta*pitch+d/stride; | |
| Py_ssize_t residue=0,off=0,j=0; | |
| for(int h=0;h<len;h++){ | |
| tap[h]=x+off; | |
| off+=jump;residue+=delta; | |
| if(residue>=stride){off+=1-stride*pitch;residue-=stride;} | |
| } | |
| for(;j+32<=nf;j+=32){ | |
| __m512d a0=_mm512_setzero_pd(),a1=a0,a2=a0,a3=a0; | |
| for(int h=0;h<len;h++){ | |
| const double *v=weights+3*h,*a=tap[h]+j; | |
| __m512d w0=_mm512_set1_pd(v[0]),w1=_mm512_set1_pd(v[1]),w2=_mm512_set1_pd(v[2]); | |
| a0=_mm512_add_pd(a0,phase_dot(a,b,g,w0,w1,w2,c,255)); | |
| a1=_mm512_add_pd(a1,phase_dot(a+8,b,g,w0,w1,w2,c,255)); | |
| a2=_mm512_add_pd(a2,phase_dot(a+16,b,g,w0,w1,w2,c,255)); | |
| a3=_mm512_add_pd(a3,phase_dot(a+24,b,g,w0,w1,w2,c,255)); | |
| } | |
| _mm512_storeu_pd(r+j,a0);_mm512_storeu_pd(r+j+8,a1); | |
| _mm512_storeu_pd(r+j+16,a2);_mm512_storeu_pd(r+j+24,a3); | |
| } | |
| for(;j<nf;j+=8){ | |
| const __mmask8 mask=(nf-j>=8)?255:(__mmask8)((1u<<(nf-j))-1u); | |
| __m512d sum=_mm512_setzero_pd(); | |
| for(int h=0;h<len;h++){ | |
| const double *v=weights+3*h; | |
| sum=_mm512_add_pd(sum,phase_dot(tap[h]+j,b,g,_mm512_set1_pd(v[0]), | |
| _mm512_set1_pd(v[1]),_mm512_set1_pd(v[2]),c,mask)); | |
| } | |
| _mm512_mask_storeu_pd(r+j,mask,sum); | |
| } | |
| } | |
| /* Subsampled response starts are adjacent after grouping equal time-index | |
| residues. Repack once per window and reuse this layout for every filter. */ | |
| static void phase_project(double *r,const double *phase,Py_ssize_t pitch,Py_ssize_t stride, | |
| Py_ssize_t nf,const int64_t *s,const double *weights) { | |
| if(s[0]<=32 && __builtin_cpu_supports("avx512f")){ | |
| wide_phase_project(r,phase,pitch,stride,nf,s,weights); | |
| return; | |
| } | |
| const int len=(int)s[0],d=(int)s[1],c=(int)s[2]; | |
| const double *x=phase+s[3]*stride*pitch; | |
| const Py_ssize_t b=c>1?(s[4]-s[3])*stride*pitch:0; | |
| const Py_ssize_t g=c>2?(s[5]-s[3])*stride*pitch:0; | |
| memset(r,0,nf*sizeof(double)); | |
| for(int h=0;h<len;h++){ | |
| const int off=h*d; | |
| const double *a=x+(off%stride)*pitch+off/stride, *v=weights+3*h; | |
| if(c==1) for(Py_ssize_t j=0;j<nf;j++)r[j]+=a[j]*v[0]; | |
| else if(c==2) for(Py_ssize_t j=0;j<nf;j++)r[j]+=a[j]*v[0]+a[j+b]*v[1]; | |
| else for(Py_ssize_t j=0;j<nf;j++)r[j]+=(a[j]*v[0]+a[j+b]*v[1])+a[j+g]*v[2]; | |
| } | |
| } | |
| static PyObject *transform(PyObject *self,PyObject *args) { | |
| PyObject *xo,*mo,*wo,*bo,*yo; | |
| Py_ssize_t stride; | |
| int approximate, fused, polyphase=0, low=0; | |
| float *small=NULL,*smallw=NULL; | |
| Py_buffer x={0},m={0},w={0},b={0},y={0}; | |
| double *work=NULL,*phase=NULL,*bt=NULL; | |
| if (!PyArg_ParseTuple(args,"OOOOnppO|pp",&xo,&mo,&wo,&bo,&stride,&approximate,&fused,&yo,&polyphase,&low)) return NULL; | |
| if (view(xo,&x,3,0,0)||view(mo,&m,2,0,1)||view(wo,&w,3,0,0)|| | |
| view(bo,&b,1,0,0)||view(yo,&y,3,1,0)) goto finish; | |
| Py_ssize_t n=x.shape[0],ch=x.shape[1],t=x.shape[2],k=m.shape[0],maxlen=w.shape[1]; | |
| const int64_t *rows=m.buf; | |
| if (t<1 || t>8192 || n<1 || ch<1 || k<1 || m.shape[1]!=6 || | |
| w.shape[0]!=k || w.shape[2]!=3 || maxlen<1 || b.shape[0]!=k || | |
| y.shape[0]!=n || y.shape[1]!=k || y.shape[2]!=4 || stride<1 || stride>t || | |
| (!approximate && stride!=1) || (low && (approximate || stride!=1))) { | |
| PyErr_SetString(PyExc_ValueError,"invalid bank dimensions or stride"); goto finish; | |
| } | |
| Py_buffer *inputs[]={&x,&m,&w,&b}; | |
| for(int i=0;i<4;i++) { | |
| uintptr_t p=(uintptr_t)inputs[i]->buf,q=(uintptr_t)y.buf; | |
| if(p<q+y.len && q<p+inputs[i]->len) { | |
| PyErr_SetString(PyExc_ValueError,"output overlaps input"); goto finish; | |
| } | |
| } | |
| for(Py_ssize_t z=0;z<k;z++) { | |
| const int64_t *meta=rows+6*z; | |
| int64_t len=meta[0],d=meta[1],c=meta[2]; | |
| if(len<1 || len>maxlen || d<1 || len-1>(t-1)/d || c<1 || c>3) { | |
| PyErr_SetString(PyExc_ValueError,"invalid kernel geometry"); goto finish; | |
| } | |
| if(!approximate && c>1 && (!bind_numpy() || t-(len-1)*d<=1)) { | |
| PyErr_SetString(PyExc_ValueError,"unsupported exact BLAS geometry"); goto finish; | |
| } | |
| for(int j=0;j<c;j++) if(meta[3+j]<0 || meta[3+j]>=ch || (j && meta[3+j]<=meta[2+j])) { | |
| PyErr_SetString(PyExc_ValueError,"invalid kernel channels"); goto finish; | |
| } | |
| } | |
| int wide = __builtin_cpu_supports("avx512f") && __builtin_cpu_supports("popcnt"); | |
| fused = fused && !approximate && (__builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma")) && t>=16; | |
| work=PyMem_Malloc(5*t*sizeof(double)); | |
| if (!work) { PyErr_NoMemory(); goto finish; } | |
| if(low){ | |
| /* Unusual custom coefficients or unnormalized signals can exceed FP32. | |
| The bound covers convolution, reduction and centered squared sums. */ | |
| double xmax=0.,wmax=0.,bmax=0.; | |
| const double *values[3]={x.buf,w.buf,b.buf}; | |
| Py_ssize_t sizes[3]={x.len/8,w.len/8,b.len/8}; | |
| double *maxima[3]={&xmax,&wmax,&bmax}; | |
| for(int j=0;j<3;j++)for(Py_ssize_t i=0;i<sizes[j];i++){ | |
| double v=fabs(values[j][i]);if(!isfinite(v)){low=0;break;} | |
| if(v>*maxima[j])*maxima[j]=v; | |
| } | |
| if(xmax>FLT_MAX || wmax>FLT_MAX || bmax>FLT_MAX || | |
| 3.*maxlen*xmax*wmax+bmax>sqrt((double)FLT_MAX/(16.*t)))low=0; | |
| } | |
| if(low){ | |
| if(!wide || !__builtin_cpu_supports("fma")){ | |
| PyErr_SetString(PyExc_ValueError,"conv32 requires an AVX-512F/FMA/POPCNT CPU");goto finish; | |
| } | |
| if((size_t)(ch+1)>SIZE_MAX/((size_t)t+64)/(16*sizeof(float))){ | |
| PyErr_NoMemory();goto finish; | |
| } | |
| small=PyMem_Calloc(((size_t)ch+1)*(t+64)*(n>=16?16:1),sizeof(float));smallw=PyMem_Malloc(w.len/2); | |
| if(!small||!smallw){PyErr_NoMemory();goto finish;} | |
| for(Py_ssize_t i=0;i<w.len/8;i++)smallw[i]=(float)((double*)w.buf)[i]; | |
| } | |
| double *r=work,*tmp=work+t,*gather=work+2*t; | |
| const double *X=x.buf,*W=w.buf,*bias=b.buf; | |
| double *Y=y.buf; | |
| Py_ssize_t pitch=(t+stride-1)/stride; | |
| if(polyphase && approximate && stride>1 && ch<=64){ | |
| phase=PyMem_Malloc(ch*stride*pitch*sizeof(double)); | |
| if(!phase){PyErr_NoMemory();goto finish;} | |
| } | |
| Py_ssize_t first=0,block=phase?1:12; | |
| BankItem *items=NULL; | |
| if(!low && wide && fused && n>=8 && ch<=8 && t<=256){ | |
| bt=PyMem_Malloc((ch+2)*t*8*sizeof(double));if(!bt){PyErr_NoMemory();goto finish;} | |
| items=PyMem_Malloc(k*sizeof(*items));if(!items){PyErr_NoMemory();goto finish;} | |
| for(Py_ssize_t z=0;z<k;++z){memcpy(items[z].geom,rows+6*z,6*sizeof(int64_t));items[z].idx=z;} | |
| qsort(items,k,sizeof(*items),cmp_bank); | |
| } | |
| Py_BEGIN_ALLOW_THREADS | |
| if(low)for(;first+16<=n;first+=16)small_batch(X+first*ch*t,ch,t,rows,smallw,bias,k,maxlen,small,Y+first*k*4); | |
| if(low)for(;first<n;first++)small_bank(X+first*ch*t,ch,t,rows,smallw,bias,k,maxlen,small,Y+first*k*4); | |
| if(bt) for(;first+8<=n;first+=8)batch_bank(X+first*ch*t,ch,t,rows,W,bias,k,maxlen,Y+first*k*4,bt,bt+ch*t*8,items); | |
| for(Py_ssize_t tile=first;tile<n;tile+=block){ | |
| Py_ssize_t stop=tile+block<n?tile+block:n; | |
| if(phase) for(Py_ssize_t h=0;h<ch;h++) for(Py_ssize_t q=0;q<t;q++) | |
| phase[(h*stride+q%stride)*pitch+q/stride]=X[(tile*ch+h)*t+q]; | |
| for(Py_ssize_t z=0;z<k;z++){ | |
| for(Py_ssize_t i=tile;i<stop;i++){ | |
| const int64_t *s=rows+6*z; | |
| int64_t len=s[0],d=s[1],c=s[2],a0=s[3],lda=c>1?t*(s[4]-a0):t; | |
| Py_ssize_t nf=1+(t-1-(len-1)*d)/stride; | |
| const double *base=X+(i*ch+a0)*t; | |
| if (!phase && c>2 && s[5]-s[4]!=s[4]-a0) { | |
| for(int j=0;j<c;j++) memcpy(gather+j*t,X+(i*ch+s[3+j])*t,t*sizeof(double)); | |
| base=gather; lda=t; | |
| } | |
| if(phase) phase_project(r,phase,pitch,stride,nf,s,W+z*maxlen*3); else | |
| if(wide && !approximate && (fused || c==1) && nf>=16) { | |
| int ix=len==7?0:len==9?1:len==11?2:len==15?3:-1; | |
| if(ix>=0)strips[c-1][ix](r,base,lda,(int)nf,W+z*maxlen*3,(int)d); | |
| else wide_project(r,base,lda,nf,W+z*maxlen*3,c,len,d); | |
| } else if(fused && c>1) { | |
| skylakex_project(r,base,lda,nf,W+z*maxlen*3,c,len,d); | |
| } else | |
| { | |
| memset(r,0,nf*sizeof(double)); | |
| for(int64_t h=0;h<len;h++){ | |
| const double *v=W+(z*maxlen+h)*3,*src=base+h*d; | |
| if(c==1){ | |
| if(stride==1) for(Py_ssize_t j=0;j<nf;j++) r[j]+=src[j]*v[0]; | |
| else for(Py_ssize_t j=0;j<nf;j++) r[j]+=src[j*stride]*v[0]; | |
| } else if(!approximate) { | |
| np_gemv(102,111,nf,c,1.,src,lda,v,1,0.,tmp,1); | |
| for(Py_ssize_t j=0;j<nf;j++) r[j]+=tmp[j]; | |
| } else if(c==2) { | |
| for(Py_ssize_t j=0;j<nf;j++) r[j]+=src[j*stride]*v[0]+src[lda+j*stride]*v[1]; | |
| } else { | |
| for(Py_ssize_t j=0;j<nf;j++) r[j]+=(src[j*stride]*v[0]+src[lda+j*stride]*v[1])+src[2*lda+j*stride]*v[2]; | |
| } | |
| } | |
| } | |
| Py_ssize_t pos=0; | |
| double mx=-INFINITY; | |
| if(wide && nf>=16) { | |
| int special=0; | |
| double mu=reduce_window(r,nf,bias[z],0,&pos,&mx,&special)/nf; | |
| if(special || mx==0.){ | |
| mx=-INFINITY; | |
| for(Py_ssize_t j=0;j<nf;j++)if(r[j]>=mx || isnan(r[j]))mx=r[j]; | |
| } | |
| double *dst=Y+(i*k+z)*4; | |
| dst[0]=(double)pos/nf;dst[1]=mx;dst[2]=mu; | |
| dst[3]=sqrt(reduce_window(r,nf,mu,1,&pos,&mx,&special)/nf); | |
| continue; | |
| } | |
| for(Py_ssize_t j=0;j<nf;j++) { | |
| r[j]+=bias[z]; pos+=r[j]>0; | |
| if(r[j]>=mx || isnan(r[j])) mx=r[j]; | |
| } | |
| double mu=pair_sum(r,nf)/nf; | |
| for(Py_ssize_t j=0;j<nf;j++) { double v=r[j]-mu;tmp[j]=v*v; } | |
| double *dst=Y+(i*k+z)*4; | |
| dst[0]=(double)pos/nf;dst[1]=mx;dst[2]=mu;dst[3]=sqrt(pair_sum(tmp,nf)/nf); | |
| } | |
| } | |
| } | |
| Py_END_ALLOW_THREADS | |
| PyMem_Free(items); | |
| finish: | |
| PyMem_Free(small);PyMem_Free(smallw); | |
| PyMem_Free(bt);PyMem_Free(phase); | |
| PyMem_Free(work); | |
| if(x.obj)PyBuffer_Release(&x); | |
| if(m.obj)PyBuffer_Release(&m); | |
| if(w.obj)PyBuffer_Release(&w); | |
| if(b.obj)PyBuffer_Release(&b); | |
| if(y.obj)PyBuffer_Release(&y); | |
| if(PyErr_Occurred())return NULL; | |
| Py_RETURN_NONE; | |
| } | |
| static PyMethodDef methods[]={{"has_blas",has_blas,METH_NOARGS,"Whether the exact multichannel NumPy BLAS is accessible."}, | |
| {"fused_library",fused_library,METH_NOARGS,"Candidate library path for the verified SkylakeX arithmetic profile."}, | |
| {"transform",transform,METH_VARARGS,"Apply a packed bank on N,C,T inputs."},{NULL,NULL,0,NULL}}; | |
| static struct PyModuleDef mod={PyModuleDef_HEAD_INIT,"_cis_conv",NULL,-1,methods}; | |
| PyMODINIT_FUNC PyInit__cis_conv(void){return PyModule_Create(&mod);} | |