From be09551cdf2efdddb3d671c1355c46560e4610f2 Mon Sep 17 00:00:00 2001 From: AbdelRauf Date: Thu, 29 Aug 2019 23:22:23 +0000 Subject: [PATCH 1/4] aligned --- kernel/power/caxpy.c | 7 +++---- kernel/power/cdot.c | 6 +++--- kernel/power/cgemv_n.c | 6 +++--- kernel/power/cgemv_t.c | 6 +++--- kernel/power/dgemv_n.c | 2 +- kernel/power/dgemv_t.c | 6 +++--- kernel/power/sgemv_n.c | 3 ++- kernel/power/sgemv_n_8.c | 3 ++- kernel/power/sgemv_t.c | 7 +++---- kernel/power/sgemv_t_8.c | 4 ++-- kernel/power/zgemv_n_4.c | 4 ++-- kernel/power/zgemv_t_4.c | 4 ++-- 12 files changed, 29 insertions(+), 29 deletions(-) diff --git a/kernel/power/caxpy.c b/kernel/power/caxpy.c index 4bdf13c34..00f2ec5e0 100644 --- a/kernel/power/caxpy.c +++ b/kernel/power/caxpy.c @@ -24,12 +24,11 @@ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. *****************************************************************************/ - #include "common.h" - - #ifndef HAVE_ASM_KERNEL #include +static const unsigned char __attribute__((aligned(16))) swap_mask_arr[]={ 4,5,6,7,0,1,2,3, 12,13,14,15, 8,9,10,11}; + static void caxpy_kernel_16(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT alpha_r, FLOAT alpha_i) { @@ -43,7 +42,7 @@ static void caxpy_kernel_16(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT alpha_r, FLOAT register __vector float valpha_i = {alpha_i, alpha_i,alpha_i, alpha_i}; #endif - __vector unsigned char swap_mask = { 4,5,6,7,0,1,2,3, 12,13,14,15, 8,9,10,11}; + __vector unsigned char swap_mask = *((__vector unsigned char*)swap_mask_arr); register __vector float *vy = (__vector float *) y; register __vector float *vx = (__vector float *) x; BLASLONG i=0; diff --git a/kernel/power/cdot.c b/kernel/power/cdot.c index f86a33f22..51d341ada 100644 --- a/kernel/power/cdot.c +++ b/kernel/power/cdot.c @@ -25,12 +25,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. *****************************************************************************/ #include "common.h" - #ifndef HAVE_KERNEL_8 #include +static const unsigned char __attribute__((aligned(16))) swap_mask_arr[]={ 4,5,6,7,0,1,2,3, 12,13,14,15, 8,9,10,11}; static void cdot_kernel_8(BLASLONG n, FLOAT *x, FLOAT *y, float *dot) { - __vector unsigned char swap_mask = { 4,5,6,7,0,1,2,3, 12,13,14,15, 8,9,10,11}; + __vector unsigned char swap_mask = *((__vector unsigned char*)swap_mask_arr); register __vector float *vy = (__vector float *) y; register __vector float *vx = (__vector float *) x; BLASLONG i = 0; @@ -96,7 +96,7 @@ OPENBLAS_COMPLEX_FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLA BLASLONG i = 0; BLASLONG ix=0, iy=0; OPENBLAS_COMPLEX_FLOAT result; - FLOAT dot[4] __attribute__ ((aligned(16))) = {0.0, 0.0, 0.0, 0.0}; + FLOAT dot[4] __attribute__((aligned(16))) = {0.0, 0.0, 0.0, 0.0}; if (n <= 0) { CREAL(result) = 0.0; diff --git a/kernel/power/cgemv_n.c b/kernel/power/cgemv_n.c index cb01e196e..6a195d6d1 100644 --- a/kernel/power/cgemv_n.c +++ b/kernel/power/cgemv_n.c @@ -32,7 +32,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define NBMAX 1024 -static const unsigned char swap_mask_arr[]={ 4,5,6,7,0,1,2,3, 12,13,14,15, 8,9,10,11}; +static const unsigned char __attribute__((aligned(16))) swap_mask_arr[]={ 4,5,6,7,0,1,2,3, 12,13,14,15, 8,9,10,11}; static void cgemv_kernel_4x4(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOAT *y) { @@ -247,8 +247,8 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r, FLOAT alpha_i, BLASLONG m2; BLASLONG m3; BLASLONG n2; - - FLOAT xbuffer[8], *ybuffer; + FLOAT xbuffer[8] __attribute__((aligned(16))); + FLOAT *ybuffer; if (m < 1) return (0); if (n < 1) return (0); diff --git a/kernel/power/cgemv_t.c b/kernel/power/cgemv_t.c index c646618cf..68bbdd60a 100644 --- a/kernel/power/cgemv_t.c +++ b/kernel/power/cgemv_t.c @@ -29,7 +29,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define NBMAX 1024 #include -static const unsigned char swap_mask_arr[]={ 4,5,6,7,0,1,2,3, 12,13,14,15, 8,9,10,11}; +static const unsigned char __attribute__((aligned(16))) swap_mask_arr[]={ 4,5,6,7,0,1,2,3, 12,13,14,15, 8,9,10,11}; static void cgemv_kernel_4x4(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOAT *y, FLOAT alpha_r, FLOAT alpha_i) { BLASLONG i; @@ -260,8 +260,8 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r, FLOAT alpha_i, BLASLONG m2; BLASLONG m3; BLASLONG n2; - - FLOAT ybuffer[8], *xbuffer; + FLOAT ybuffer[8] __attribute__((aligned(16))); + FLOAT *xbuffer; if (m < 1) return (0); if (n < 1) return (0); diff --git a/kernel/power/dgemv_n.c b/kernel/power/dgemv_n.c index b458e11fc..1a3d7669c 100644 --- a/kernel/power/dgemv_n.c +++ b/kernel/power/dgemv_n.c @@ -145,7 +145,7 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLO BLASLONG m3; BLASLONG n2; BLASLONG lda4 = lda << 2; - FLOAT xbuffer[8] __attribute__ ((aligned (16)));; + FLOAT xbuffer[8] __attribute__ ((aligned (16))); FLOAT *ybuffer; if ( m < 1 ) return(0); diff --git a/kernel/power/dgemv_t.c b/kernel/power/dgemv_t.c index b8589a131..d05d7b7d3 100644 --- a/kernel/power/dgemv_t.c +++ b/kernel/power/dgemv_t.c @@ -581,9 +581,9 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLO BLASLONG m1; BLASLONG m2; BLASLONG m3; - BLASLONG n2; - - FLOAT ybuffer[8], *xbuffer; + BLASLONG n2; + FLOAT ybuffer[8] __attribute__((aligned(16))); + FLOAT *xbuffer; if (m < 1) return (0); if (n < 1) return (0); diff --git a/kernel/power/sgemv_n.c b/kernel/power/sgemv_n.c index 9704757fe..81ac031a3 100644 --- a/kernel/power/sgemv_n.c +++ b/kernel/power/sgemv_n.c @@ -174,7 +174,8 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLO BLASLONG n2; BLASLONG lda4 = lda << 2; BLASLONG lda8 = lda << 3; - FLOAT xbuffer[8],*ybuffer; + FLOAT xbuffer[8] __attribute__((aligned(16))); + FLOAT *ybuffer; if ( m < 1 ) return(0); if ( n < 1 ) return(0); diff --git a/kernel/power/sgemv_n_8.c b/kernel/power/sgemv_n_8.c index 9bc93ced6..64696236a 100644 --- a/kernel/power/sgemv_n_8.c +++ b/kernel/power/sgemv_n_8.c @@ -213,7 +213,8 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLO BLASLONG n2; BLASLONG lda4 = lda << 2; BLASLONG lda8 = lda << 3; - FLOAT xbuffer[8],*ybuffer; + FLOAT xbuffer[8] __attribute__((aligned(16))); + FLOAT *ybuffer; if ( m < 1 ) return(0); if ( n < 1 ) return(0); diff --git a/kernel/power/sgemv_t.c b/kernel/power/sgemv_t.c index 96434a13f..3d8a442dc 100644 --- a/kernel/power/sgemv_t.c +++ b/kernel/power/sgemv_t.c @@ -177,10 +177,9 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLO BLASLONG m1; BLASLONG m2; BLASLONG m3; - BLASLONG n2; - - FLOAT ybuffer[8], *xbuffer; - + BLASLONG n2; + FLOAT ybuffer[8] __attribute__((aligned(16))); + FLOAT *xbuffer; if (m < 1) return (0); if (n < 1) return (0); diff --git a/kernel/power/sgemv_t_8.c b/kernel/power/sgemv_t_8.c index 5e9cd63ac..b90512162 100644 --- a/kernel/power/sgemv_t_8.c +++ b/kernel/power/sgemv_t_8.c @@ -204,8 +204,8 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLO BLASLONG m3; BLASLONG n2; - FLOAT ybuffer[8], *xbuffer; - + FLOAT ybuffer[8] __attribute__((aligned(16))); + FLOAT *xbuffer; if (m < 1) return (0); if (n < 1) return (0); diff --git a/kernel/power/zgemv_n_4.c b/kernel/power/zgemv_n_4.c index 167b0a158..ba019d6a5 100644 --- a/kernel/power/zgemv_n_4.c +++ b/kernel/power/zgemv_n_4.c @@ -614,8 +614,8 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r, FLOAT alpha_i, BLASLONG m2; BLASLONG m3; BLASLONG n2; - - FLOAT xbuffer[8], *ybuffer; + FLOAT xbuffer[8] __attribute__((aligned(16))); + FLOAT *ybuffer; if (m < 1) return (0); if (n < 1) return (0); diff --git a/kernel/power/zgemv_t_4.c b/kernel/power/zgemv_t_4.c index 20a0812dd..b34199af6 100644 --- a/kernel/power/zgemv_t_4.c +++ b/kernel/power/zgemv_t_4.c @@ -532,8 +532,8 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r, FLOAT alpha_i, BLASLONG m2; BLASLONG m3; BLASLONG n2; - - FLOAT ybuffer[8], *xbuffer; + FLOAT ybuffer[8] __attribute__((aligned(16))); + FLOAT *xbuffer; if (m < 1) return (0); if (n < 1) return (0); From e79712d96941c099d2f5e4b11544b2a20d97fbdf Mon Sep 17 00:00:00 2001 From: AbdelRauf Date: Fri, 30 Aug 2019 02:52:04 +0000 Subject: [PATCH 2/4] cgemv using vec_vsx_ld instead of letting gcc to decide --- kernel/power/cgemv_n.c | 97 +++++++++++++++++++++++------------------- kernel/power/cgemv_t.c | 97 +++++++++++++++++++++++++++--------------- 2 files changed, 115 insertions(+), 79 deletions(-) diff --git a/kernel/power/cgemv_n.c b/kernel/power/cgemv_n.c index 6a195d6d1..e85517ffa 100644 --- a/kernel/power/cgemv_n.c +++ b/kernel/power/cgemv_n.c @@ -62,23 +62,24 @@ static void cgemv_kernel_4x4(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOA register __vector float vx3_r = {x[6], -x[6],x[6], -x[6]}; register __vector float vx3_i = {x[7], x[7],x[7], x[7]}; #endif - register __vector float *vy = (__vector float *) y; + register __vector float *vptr_y = (__vector float *) y; register __vector float *vptr_a0 = (__vector float *) a0; register __vector float *vptr_a1 = (__vector float *) a1; register __vector float *vptr_a2 = (__vector float *) a2; register __vector float *vptr_a3 = (__vector float *) a3; BLASLONG i = 0; - for (;i< n / 2; i+=2) { - register __vector float vy_0 = vy[i]; - register __vector float vy_1 = vy[i + 1]; - register __vector float va0 = vptr_a0[i]; - register __vector float va1 = vptr_a1[i]; - register __vector float va2 = vptr_a2[i]; - register __vector float va3 = vptr_a3[i]; - register __vector float va0_1 = vptr_a0[i + 1]; - register __vector float va1_1 = vptr_a1[i + 1]; - register __vector float va2_1 = vptr_a2[i + 1]; - register __vector float va3_1 = vptr_a3[i + 1]; + BLASLONG i2=16; + for (;i< n * 8; i+=32,i2+=32) { + register __vector float vy_0 = vec_vsx_ld(i,vptr_y); + register __vector float vy_1 = vec_vsx_ld(i2,vptr_y); + register __vector float va0 = vec_vsx_ld(i,vptr_a0); + register __vector float va1 = vec_vsx_ld(i, vptr_a1); + register __vector float va2 = vec_vsx_ld(i ,vptr_a2); + register __vector float va3 = vec_vsx_ld(i ,vptr_a3); + register __vector float va0_1 = vec_vsx_ld(i2 ,vptr_a0); + register __vector float va1_1 = vec_vsx_ld(i2 ,vptr_a1); + register __vector float va2_1 = vec_vsx_ld(i2 ,vptr_a2); + register __vector float va3_1 = vec_vsx_ld(i2 ,vptr_a3); vy_0 += va0*vx0_r + va1*vx1_r + va2*vx2_r + va3*vx3_r; vy_1 += va0_1*vx0_r + va1_1*vx1_r + va2_1*vx2_r + va3_1*vx3_r; @@ -93,8 +94,8 @@ static void cgemv_kernel_4x4(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOA vy_0 += va0*vx0_i + va1*vx1_i + va2*vx2_i + va3*vx3_i; vy_1 += va0_1*vx0_i + va1_1*vx1_i + va2_1*vx2_i + va3_1*vx3_i; - vy[i] = vy_0; - vy[i + 1] = vy_1; + vec_vsx_st(vy_0 ,i, vptr_y); + vec_vsx_st(vy_1,i2,vptr_y); } } @@ -118,17 +119,19 @@ static void cgemv_kernel_4x2(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOA register __vector float vx1_r = {x[2], -x[2],x[2], -x[2]}; register __vector float vx1_i = {x[3], x[3],x[3], x[3]}; #endif - register __vector float *vy = (__vector float *) y; + register __vector float *vptr_y = (__vector float *) y; register __vector float *vptr_a0 = (__vector float *) a0; register __vector float *vptr_a1 = (__vector float *) a1; - BLASLONG i = 0; - for (;i< n / 2; i+=2) { - register __vector float vy_0 = vy[i]; - register __vector float vy_1 = vy[i + 1]; - register __vector float va0 = vptr_a0[i]; - register __vector float va1 = vptr_a1[i]; - register __vector float va0_1 = vptr_a0[i + 1]; - register __vector float va1_1 = vptr_a1[i + 1]; + BLASLONG i = 0; + BLASLONG i2 = 16; + for (;i< n * 8; i+=32, i2+=32) { + register __vector float vy_0 = vec_vsx_ld(i,vptr_y); + register __vector float vy_1 = vec_vsx_ld(i2,vptr_y); + register __vector float va0 = vec_vsx_ld(i,vptr_a0); + register __vector float va1 = vec_vsx_ld(i, vptr_a1); + register __vector float va0_1 = vec_vsx_ld(i2 ,vptr_a0); + register __vector float va1_1 = vec_vsx_ld(i2 ,vptr_a1); + register __vector float va0x = vec_perm(va0, va0,swap_mask); register __vector float va0x_1 = vec_perm(va0_1, va0_1,swap_mask); register __vector float va1x = vec_perm(va1, va1,swap_mask); @@ -136,8 +139,8 @@ static void cgemv_kernel_4x2(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOA vy_0 += va0*vx0_r + va1*vx1_r + va0x*vx0_i + va1x*vx1_i; vy_1 += va0_1*vx0_r + va1_1*vx1_r + va0x_1*vx0_i + va1x_1*vx1_i; - vy[i] = vy_0; - vy[i + 1] = vy_1; + vec_vsx_st(vy_0 ,i, vptr_y); + vec_vsx_st(vy_1,i2,vptr_y); } } @@ -154,21 +157,23 @@ static void cgemv_kernel_4x1(BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y) { register __vector float vx0_r = {x[0], -x[0],x[0], -x[0]}; register __vector float vx0_i = {x[1], x[1],x[1], x[1]}; #endif - register __vector float *vy = (__vector float *) y; + register __vector float *vptr_y = (__vector float *) y; register __vector float *vptr_a0 = (__vector float *) ap; BLASLONG i = 0; - for (;i< n / 2; i+=2) { - register __vector float vy_0 = vy[i]; - register __vector float vy_1 = vy[i + 1]; - register __vector float va0 = vptr_a0[i]; - register __vector float va0_1 = vptr_a0[i + 1]; + BLASLONG i2 = 16; + for (;i< n * 8; i+=32, i2+=32) { + register __vector float vy_0 = vec_vsx_ld(i,vptr_y); + register __vector float vy_1 = vec_vsx_ld(i2,vptr_y); + register __vector float va0 = vec_vsx_ld(i,vptr_a0); + register __vector float va0_1 = vec_vsx_ld(i2 ,vptr_a0); + register __vector float va0x = vec_perm(va0, va0,swap_mask); register __vector float va0x_1 = vec_perm(va0_1, va0_1,swap_mask); vy_0 += va0*vx0_r + va0x*vx0_i; vy_1 += va0_1*vx0_r + va0x_1*vx0_i; - vy[i] = vy_0; - vy[i + 1] = vy_1; + vec_vsx_st(vy_0 ,i, vptr_y); + vec_vsx_st(vy_1,i2,vptr_y); } } @@ -213,20 +218,24 @@ static void add_y(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_dest, FLOAT register __vector float *vptr_src = (__vector float *) src; register __vector float *vptr_y = (__vector float *) dest; - for (i = 0; i < n/2; i += 2 ){ - register __vector float vy_0 = vptr_y[i]; - register __vector float vy_1 = vptr_y[i +1]; + BLASLONG i2 = 16; + for (;i< n * 8; i+=32, i2+=32) { + register __vector float vy_0 = vec_vsx_ld(i,vptr_y); + register __vector float vy_1 = vec_vsx_ld(i2,vptr_y); - register __vector float vsrc = vptr_src[i]; - register __vector float vsrc_1 = vptr_src[i + 1]; - register __vector float vsrcx = vec_perm(vsrc, vsrc, swap_mask); - register __vector float vsrcx_1 = vec_perm(vsrc_1, vsrc_1, swap_mask); - vy_0 += vsrc*valpha_r + vsrcx*valpha_i; - vy_1 += vsrc_1*valpha_r + vsrcx_1*valpha_i; - vptr_y[i] = vy_0; - vptr_y[i+1 ] = vy_1; + register __vector float vsrc = vec_vsx_ld(i,vptr_src); + register __vector float vsrc_1 = vec_vsx_ld(i2,vptr_src); + + register __vector float vsrcx = vec_perm(vsrc, vsrc, swap_mask); + register __vector float vsrcx_1 = vec_perm(vsrc_1, vsrc_1, swap_mask); + + vy_0 += vsrc*valpha_r + vsrcx*valpha_i; + vy_1 += vsrc_1*valpha_r + vsrcx_1*valpha_i; + + vec_vsx_st(vy_0 ,i, vptr_y); + vec_vsx_st(vy_1,i2,vptr_y); } diff --git a/kernel/power/cgemv_t.c b/kernel/power/cgemv_t.c index 68bbdd60a..57eb066b0 100644 --- a/kernel/power/cgemv_t.c +++ b/kernel/power/cgemv_t.c @@ -32,7 +32,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. static const unsigned char __attribute__((aligned(16))) swap_mask_arr[]={ 4,5,6,7,0,1,2,3, 12,13,14,15, 8,9,10,11}; static void cgemv_kernel_4x4(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOAT *y, FLOAT alpha_r, FLOAT alpha_i) { - BLASLONG i; + FLOAT *a0, *a1, *a2, *a3; a0 = ap; a1 = ap + lda; @@ -48,26 +48,39 @@ static void cgemv_kernel_4x4(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOA register __vector float vtemp2_r = {0.0, 0.0,0.0,0.0}; register __vector float vtemp3_p = {0.0, 0.0,0.0,0.0}; register __vector float vtemp3_r = {0.0, 0.0,0.0,0.0}; - __vector float* va0 = (__vector float*) a0; - __vector float* va1 = (__vector float*) a1; - __vector float* va2 = (__vector float*) a2; - __vector float* va3 = (__vector float*) a3; + __vector float* vptr_a0 = (__vector float*) a0; + __vector float* vptr_a1 = (__vector float*) a1; + __vector float* vptr_a2 = (__vector float*) a2; + __vector float* vptr_a3 = (__vector float*) a3; __vector float* v_x = (__vector float*) x; - for (i = 0; i < n / 2; i+=2) { - register __vector float vx_0 = v_x[i]; - register __vector float vx_1 = v_x[i+1]; + BLASLONG i = 0; + BLASLONG i2 = 16; + for (;i< n * 8; i+=32, i2+=32) { + register __vector float vx_0 = vec_vsx_ld( i,v_x) ; + register __vector float vx_1 = vec_vsx_ld(i2, v_x); + register __vector float vxr_0 = vec_perm(vx_0, vx_0, swap_mask); register __vector float vxr_1 = vec_perm(vx_1, vx_1, swap_mask); - vtemp0_p += vx_0*va0[i] + vx_1*va0[i+1] ; - vtemp0_r += vxr_0*va0[i] + vxr_1*va0[i+1]; - vtemp1_p += vx_0*va1[i] + vx_1*va1[i+1]; - vtemp1_r += vxr_0*va1[i] + vxr_1*va1[i+1]; - vtemp2_p += vx_0*va2[i] + vx_1*va2[i+1]; - vtemp2_r += vxr_0*va2[i] + vxr_1*va2[i+1]; - vtemp3_p += vx_0*va3[i] + vx_1*va3[i+1]; - vtemp3_r += vxr_0*va3[i] + vxr_1*va3[i+1]; + register __vector float va0 = vec_vsx_ld(i,vptr_a0); + register __vector float va1 = vec_vsx_ld(i, vptr_a1); + register __vector float va2 = vec_vsx_ld(i ,vptr_a2); + register __vector float va3 = vec_vsx_ld(i ,vptr_a3); + register __vector float va0_1 = vec_vsx_ld(i2 ,vptr_a0); + register __vector float va1_1 = vec_vsx_ld(i2 ,vptr_a1); + register __vector float va2_1 = vec_vsx_ld(i2 ,vptr_a2); + register __vector float va3_1 = vec_vsx_ld(i2 ,vptr_a3); + + + vtemp0_p += vx_0*va0 + vx_1*va0_1 ; + vtemp0_r += vxr_0*va0 + vxr_1*va0_1; + vtemp1_p += vx_0*va1 + vx_1*va1_1; + vtemp1_r += vxr_0*va1 + vxr_1*va1_1; + vtemp2_p += vx_0*va2 + vx_1*va2_1; + vtemp2_r += vxr_0*va2 + vxr_1*va2_1; + vtemp3_p += vx_0*va3 + vx_1*va3_1; + vtemp3_r += vxr_0*va3 + vxr_1*va3_1; } @@ -128,7 +141,7 @@ static void cgemv_kernel_4x4(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOA static void cgemv_kernel_4x2(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOAT *y, FLOAT alpha_r, FLOAT alpha_i) { - BLASLONG i; + FLOAT *a0, *a1; a0 = ap; a1 = ap + lda; @@ -138,23 +151,33 @@ static void cgemv_kernel_4x2(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOA register __vector float vtemp0_r = {0.0, 0.0,0.0,0.0}; register __vector float vtemp1_p = {0.0, 0.0,0.0,0.0}; register __vector float vtemp1_r = {0.0, 0.0,0.0,0.0}; - __vector float* va0 = (__vector float*) a0; - __vector float* va1 = (__vector float*) a1; + + + __vector float* vptr_a0 = (__vector float*) a0; + __vector float* vptr_a1 = (__vector float*) a1; __vector float* v_x = (__vector float*) x; - for (i = 0; i < n / 2; i+=2) { - register __vector float vx_0 = v_x[i]; - register __vector float vx_1 = v_x[i+1]; + BLASLONG i = 0; + BLASLONG i2 = 16; + for (;i< n * 8; i+=32, i2+=32) { + register __vector float vx_0 = vec_vsx_ld( i,v_x) ; + register __vector float vx_1 = vec_vsx_ld(i2, v_x); + register __vector float vxr_0 = vec_perm(vx_0, vx_0, swap_mask); register __vector float vxr_1 = vec_perm(vx_1, vx_1, swap_mask); - vtemp0_p += vx_0*va0[i] + vx_1*va0[i+1] ; - vtemp0_r += vxr_0*va0[i] + vxr_1*va0[i+1]; - vtemp1_p += vx_0*va1[i] + vx_1*va1[i+1]; - vtemp1_r += vxr_0*va1[i] + vxr_1*va1[i+1]; + register __vector float va0 = vec_vsx_ld(i,vptr_a0); + register __vector float va1 = vec_vsx_ld(i, vptr_a1); + register __vector float va0_1 = vec_vsx_ld(i2 ,vptr_a0); + register __vector float va1_1 = vec_vsx_ld(i2 ,vptr_a1); + + + vtemp0_p += vx_0*va0 + vx_1*va0_1 ; + vtemp0_r += vxr_0*va0 + vxr_1*va0_1; + vtemp1_p += vx_0*va1 + vx_1*va1_1; + vtemp1_r += vxr_0*va1 + vxr_1*va1_1; } - #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) ) register FLOAT temp_r0 = vtemp0_p[0] - vtemp0_p[1] + vtemp0_p[2] - vtemp0_p[3]; @@ -193,23 +216,27 @@ static void cgemv_kernel_4x2(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOA static void cgemv_kernel_4x1(BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y, FLOAT alpha_r, FLOAT alpha_i) { - BLASLONG i; + __vector unsigned char swap_mask = *((__vector unsigned char*)swap_mask_arr); //p for positive(real*real,image*image,real*real,image*image) r for image (real*image,image*real,real*image,image*real) register __vector float vtemp0_p = {0.0, 0.0,0.0,0.0}; register __vector float vtemp0_r = {0.0, 0.0,0.0,0.0}; - __vector float* va0 = (__vector float*) ap; + __vector float* vptr_a0 = (__vector float*) ap; __vector float* v_x = (__vector float*) x; - - for (i = 0; i < n / 2; i+=2) { - register __vector float vx_0 = v_x[i]; - register __vector float vx_1 = v_x[i+1]; + BLASLONG i = 0; + BLASLONG i2 = 16; + for (;i< n * 8; i+=32, i2+=32) { + register __vector float vx_0 = vec_vsx_ld( i,v_x) ; + register __vector float vx_1 = vec_vsx_ld(i2, v_x); + register __vector float vxr_0 = vec_perm(vx_0, vx_0, swap_mask); register __vector float vxr_1 = vec_perm(vx_1, vx_1, swap_mask); - vtemp0_p += vx_0*va0[i] + vx_1*va0[i+1] ; - vtemp0_r += vxr_0*va0[i] + vxr_1*va0[i+1]; + register __vector float va0 = vec_vsx_ld(i,vptr_a0); + register __vector float va0_1 = vec_vsx_ld(i2 ,vptr_a0); + vtemp0_p += vx_0*va0 + vx_1*va0_1 ; + vtemp0_r += vxr_0*va0 + vxr_1*va0_1; } #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) ) From 4c22828812a9d5f0962c836d4c8bf486fde4d9cb Mon Sep 17 00:00:00 2001 From: AbdelRauf Date: Fri, 30 Aug 2019 04:09:15 +0000 Subject: [PATCH 3/4] caxpy and cdot are using vec_vsx_ld --- kernel/power/caxpy.c | 67 ++++++++++++++++++++++++++------------------ kernel/power/cdot.c | 48 +++++++++++++++++-------------- 2 files changed, 67 insertions(+), 48 deletions(-) diff --git a/kernel/power/caxpy.c b/kernel/power/caxpy.c index 00f2ec5e0..0545766b1 100644 --- a/kernel/power/caxpy.c +++ b/kernel/power/caxpy.c @@ -27,6 +27,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "common.h" #ifndef HAVE_ASM_KERNEL #include + +#define offset_0 0 +#define offset_1 16 +#define offset_2 32 +#define offset_3 48 +#define offset_4 64 +#define offset_5 80 +#define offset_6 96 +#define offset_7 112 + static const unsigned char __attribute__((aligned(16))) swap_mask_arr[]={ 4,5,6,7,0,1,2,3, 12,13,14,15, 8,9,10,11}; static void caxpy_kernel_16(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT alpha_r, FLOAT alpha_i) @@ -43,27 +53,28 @@ static void caxpy_kernel_16(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT alpha_r, FLOAT #endif __vector unsigned char swap_mask = *((__vector unsigned char*)swap_mask_arr); - register __vector float *vy = (__vector float *) y; - register __vector float *vx = (__vector float *) x; + register __vector float *vptr_y = (__vector float *) y; + register __vector float *vptr_x = (__vector float *) x; BLASLONG i=0; - for (; i < n/2; i += 8) { + for(;i + +#define offset_0 0 +#define offset_1 16 +#define offset_2 32 +#define offset_3 48 + + + static const unsigned char __attribute__((aligned(16))) swap_mask_arr[]={ 4,5,6,7,0,1,2,3, 12,13,14,15, 8,9,10,11}; static void cdot_kernel_8(BLASLONG n, FLOAT *x, FLOAT *y, float *dot) { __vector unsigned char swap_mask = *((__vector unsigned char*)swap_mask_arr); - register __vector float *vy = (__vector float *) y; - register __vector float *vx = (__vector float *) x; - BLASLONG i = 0; + register __vector float *vptr_y = (__vector float *) y; + register __vector float *vptr_x = (__vector float *) x; register __vector float vd_0 = { 0 }; register __vector float vd_1 = { 0 }; register __vector float vd_2 = { 0 }; @@ -41,26 +48,23 @@ static void cdot_kernel_8(BLASLONG n, FLOAT *x, FLOAT *y, float *dot) register __vector float vdd_0 = { 0 }; register __vector float vdd_1 = { 0 }; register __vector float vdd_2 = { 0 }; - register __vector float vdd_3 = { 0 }; - for (; i < n/2; i += 4) { + register __vector float vdd_3 = { 0 }; + BLASLONG i=0; + for(;i Date: Fri, 30 Aug 2019 11:14:55 +0000 Subject: [PATCH 4/4] fix uninitialized variables i --- kernel/power/cgemv_n.c | 4 ++-- kernel/power/cgemv_t.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/power/cgemv_n.c b/kernel/power/cgemv_n.c index e85517ffa..eec3fa37c 100644 --- a/kernel/power/cgemv_n.c +++ b/kernel/power/cgemv_n.c @@ -181,7 +181,7 @@ static void cgemv_kernel_4x1(BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y) { static void add_y(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_dest, FLOAT alpha_r, FLOAT alpha_i) { - BLASLONG i; + BLASLONG i=0; if (inc_dest != 2) { @@ -246,7 +246,7 @@ static void add_y(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_dest, FLOAT int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r, FLOAT alpha_i, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT * buffer) { - BLASLONG i; + BLASLONG i=0; FLOAT *a_ptr; FLOAT *x_ptr; FLOAT *y_ptr; diff --git a/kernel/power/cgemv_t.c b/kernel/power/cgemv_t.c index 57eb066b0..691f7a3d3 100644 --- a/kernel/power/cgemv_t.c +++ b/kernel/power/cgemv_t.c @@ -276,8 +276,8 @@ static void copy_x(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_src) { } int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r, FLOAT alpha_i, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer) { - BLASLONG i; - BLASLONG j; + BLASLONG i=0; + BLASLONG j=0; FLOAT *a_ptr; FLOAT *x_ptr; FLOAT *y_ptr;