From 7d380f7d79abe1a8d7ed6efd56efe677135c2415 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 28 Aug 2019 11:31:20 +0200 Subject: [PATCH 01/10] Fix PGI build options (again) for #2237 --- Makefile.system | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.system b/Makefile.system index fe6be79db..2cf1322a9 100644 --- a/Makefile.system +++ b/Makefile.system @@ -699,7 +699,7 @@ endif ifeq ($(C_COMPILER), PGI) ifdef BINARY64 -CCOMMON_OPT += -tp p7-64 +CCOMMON_OPT += -tp p7-64 -D__MMX__ -Mnollvm else CCOMMON_OPT += -tp p7 endif From 3a55dca2dceafef421c6198d7dd1876f4bcc5663 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 28 Aug 2019 11:35:31 +0200 Subject: [PATCH 02/10] Make x86_64 zdot compile with PGI and Sun C again broken by #2222 as CREAL,CIMAG do not expand to a valid lvalue with these compilers --- kernel/x86_64/zdot.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/kernel/x86_64/zdot.c b/kernel/x86_64/zdot.c index 48f855b0e..d11cb764f 100644 --- a/kernel/x86_64/zdot.c +++ b/kernel/x86_64/zdot.c @@ -181,10 +181,10 @@ OPENBLAS_COMPLEX_FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLA #if defined(SMP) int nthreads; FLOAT dummy_alpha; + FLOAT zdotr=0., zdoti=0.; #endif OPENBLAS_COMPLEX_FLOAT zdot; - CREAL(zdot) = 0.0; - CIMAG(zdot) = 0.0; + zdot=OPENBLAS_MAKE_COMPLEX_FLOAT(0.0,0.0); #if defined(SMP) if (inc_x == 0 || inc_y == 0 || n <= 10000) @@ -211,15 +211,17 @@ OPENBLAS_COMPLEX_FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLA ptr = (OPENBLAS_COMPLEX_FLOAT *)result; for (i = 0; i < nthreads; i++) { - CREAL(zdot) = CREAL(zdot) + CREAL(*ptr); - CIMAG(zdot) = CIMAG(zdot) + CIMAG(*ptr); + zdotr += CREAL(*ptr); + zdoti += CIMAG(*ptr); +// CREAL(zdot) = CREAL(zdot) + CREAL(*ptr); +// CIMAG(zdot) = CIMAG(zdot) + CIMAG(*ptr); ptr = (void *)(((char *)ptr) + sizeof(double) * 2); } + zdot = OPENBLAS_MAKE_COMPLEX_FLOAT(zdotr,zdoti); } #else zdot_compute(n, x, inc_x, y, inc_y, &zdot); #endif - return zdot; } From bf0d92a3105ae0ed67117dcf0977a164ac9e2e7a Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 28 Aug 2019 17:35:56 +0200 Subject: [PATCH 03/10] Add arch data for cross-compiling to CORE2 for #2235 --- cmake/prebuild.cmake | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/cmake/prebuild.cmake b/cmake/prebuild.cmake index e508a46c2..c2600bd0d 100644 --- a/cmake/prebuild.cmake +++ b/cmake/prebuild.cmake @@ -106,7 +106,25 @@ if (DEFINED CORE AND CMAKE_CROSSCOMPILING AND NOT (${HOST_OS} STREQUAL "WINDOWSS file(APPEND ${TARGET_CONF_TEMP} "#define ${TCORE}\n" "#define CHAR_CORENAME \"${TCORE}\"\n") - if ("${TCORE}" STREQUAL "ARMV7") + if ("${TCORE}" STREQUAL "CORE2") + file(APPEND ${TARGET_CONF_TEMP} + "#define L1_DATA_SIZE\t32768\n" + "#define L1_DATA_LINESIZE\t64\n" + "#define L2_SIZE\t1048576\n" + "#define L2_LINESIZE\t64\n" + "#define DTB_DEFAULT_ENTRIES\t256\n" + "#define DTB_SIZE\t4096\n" + "#define HAVE_CMOV\n" + "#define HAVE_MMX\n" + "#define HAVE_SSE\n" + "#define HAVE_SSE2\n" + "#define HAVE_SSE3\n" + "#define HAVE_SSSE3\n") + set(SGEMM_UNROLL_M 8) + set(SGEMM_UNROLL_N 4) + set(DGEMM_UNROLL_M 4) + set(DGEMM_UNROLL_N 4) + elseif ("${TCORE}" STREQUAL "ARMV7") file(APPEND ${TARGET_CONF_TEMP} "#define L1_DATA_SIZE\t65536\n" "#define L1_DATA_LINESIZE\t32\n" From 11c59acfb1a061f35ec88f11c2176f60b4916e93 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 28 Aug 2019 18:07:44 +0200 Subject: [PATCH 04/10] Keep both PGI/SUN and default code paths to avoid breaking Clang/WIndows --- kernel/x86_64/zdot.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/kernel/x86_64/zdot.c b/kernel/x86_64/zdot.c index d11cb764f..01169e8e6 100644 --- a/kernel/x86_64/zdot.c +++ b/kernel/x86_64/zdot.c @@ -181,11 +181,19 @@ OPENBLAS_COMPLEX_FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLA #if defined(SMP) int nthreads; FLOAT dummy_alpha; +#if defined(C_PGI) || defined(C_SUN) FLOAT zdotr=0., zdoti=0.; +#endif #endif + OPENBLAS_COMPLEX_FLOAT zdot; +#if defined(C_PGI) || defined(C_SUN) zdot=OPENBLAS_MAKE_COMPLEX_FLOAT(0.0,0.0); - +#else + CREAL(zdot) = 0.0; + CIMAG(zdot) = 0.0; +#endif + #if defined(SMP) if (inc_x == 0 || inc_y == 0 || n <= 10000) nthreads = 1; @@ -211,17 +219,23 @@ OPENBLAS_COMPLEX_FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLA ptr = (OPENBLAS_COMPLEX_FLOAT *)result; for (i = 0; i < nthreads; i++) { +#if defined(C_PGI) || defined(C_SUN) zdotr += CREAL(*ptr); zdoti += CIMAG(*ptr); -// CREAL(zdot) = CREAL(zdot) + CREAL(*ptr); -// CIMAG(zdot) = CIMAG(zdot) + CIMAG(*ptr); +#else + CREAL(zdot) = CREAL(zdot) + CREAL(*ptr); + CIMAG(zdot) = CIMAG(zdot) + CIMAG(*ptr); +#endif ptr = (void *)(((char *)ptr) + sizeof(double) * 2); } +#if defined(C_PGI) || defined(C_SUN) zdot = OPENBLAS_MAKE_COMPLEX_FLOAT(zdotr,zdoti); +#endif } #else zdot_compute(n, x, inc_x, y, inc_y, &zdot); #endif + return zdot; } From be09551cdf2efdddb3d671c1355c46560e4610f2 Mon Sep 17 00:00:00 2001 From: AbdelRauf Date: Thu, 29 Aug 2019 23:22:23 +0000 Subject: [PATCH 05/10] aligned --- kernel/power/caxpy.c | 7 +++---- kernel/power/cdot.c | 6 +++--- kernel/power/cgemv_n.c | 6 +++--- kernel/power/cgemv_t.c | 6 +++--- kernel/power/dgemv_n.c | 2 +- kernel/power/dgemv_t.c | 6 +++--- kernel/power/sgemv_n.c | 3 ++- kernel/power/sgemv_n_8.c | 3 ++- kernel/power/sgemv_t.c | 7 +++---- kernel/power/sgemv_t_8.c | 4 ++-- kernel/power/zgemv_n_4.c | 4 ++-- kernel/power/zgemv_t_4.c | 4 ++-- 12 files changed, 29 insertions(+), 29 deletions(-) diff --git a/kernel/power/caxpy.c b/kernel/power/caxpy.c index 4bdf13c34..00f2ec5e0 100644 --- a/kernel/power/caxpy.c +++ b/kernel/power/caxpy.c @@ -24,12 +24,11 @@ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. *****************************************************************************/ - #include "common.h" - - #ifndef HAVE_ASM_KERNEL #include +static const unsigned char __attribute__((aligned(16))) swap_mask_arr[]={ 4,5,6,7,0,1,2,3, 12,13,14,15, 8,9,10,11}; + static void caxpy_kernel_16(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT alpha_r, FLOAT alpha_i) { @@ -43,7 +42,7 @@ static void caxpy_kernel_16(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT alpha_r, FLOAT register __vector float valpha_i = {alpha_i, alpha_i,alpha_i, alpha_i}; #endif - __vector unsigned char swap_mask = { 4,5,6,7,0,1,2,3, 12,13,14,15, 8,9,10,11}; + __vector unsigned char swap_mask = *((__vector unsigned char*)swap_mask_arr); register __vector float *vy = (__vector float *) y; register __vector float *vx = (__vector float *) x; BLASLONG i=0; diff --git a/kernel/power/cdot.c b/kernel/power/cdot.c index f86a33f22..51d341ada 100644 --- a/kernel/power/cdot.c +++ b/kernel/power/cdot.c @@ -25,12 +25,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. *****************************************************************************/ #include "common.h" - #ifndef HAVE_KERNEL_8 #include +static const unsigned char __attribute__((aligned(16))) swap_mask_arr[]={ 4,5,6,7,0,1,2,3, 12,13,14,15, 8,9,10,11}; static void cdot_kernel_8(BLASLONG n, FLOAT *x, FLOAT *y, float *dot) { - __vector unsigned char swap_mask = { 4,5,6,7,0,1,2,3, 12,13,14,15, 8,9,10,11}; + __vector unsigned char swap_mask = *((__vector unsigned char*)swap_mask_arr); register __vector float *vy = (__vector float *) y; register __vector float *vx = (__vector float *) x; BLASLONG i = 0; @@ -96,7 +96,7 @@ OPENBLAS_COMPLEX_FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLA BLASLONG i = 0; BLASLONG ix=0, iy=0; OPENBLAS_COMPLEX_FLOAT result; - FLOAT dot[4] __attribute__ ((aligned(16))) = {0.0, 0.0, 0.0, 0.0}; + FLOAT dot[4] __attribute__((aligned(16))) = {0.0, 0.0, 0.0, 0.0}; if (n <= 0) { CREAL(result) = 0.0; diff --git a/kernel/power/cgemv_n.c b/kernel/power/cgemv_n.c index cb01e196e..6a195d6d1 100644 --- a/kernel/power/cgemv_n.c +++ b/kernel/power/cgemv_n.c @@ -32,7 +32,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define NBMAX 1024 -static const unsigned char swap_mask_arr[]={ 4,5,6,7,0,1,2,3, 12,13,14,15, 8,9,10,11}; +static const unsigned char __attribute__((aligned(16))) swap_mask_arr[]={ 4,5,6,7,0,1,2,3, 12,13,14,15, 8,9,10,11}; static void cgemv_kernel_4x4(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOAT *y) { @@ -247,8 +247,8 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r, FLOAT alpha_i, BLASLONG m2; BLASLONG m3; BLASLONG n2; - - FLOAT xbuffer[8], *ybuffer; + FLOAT xbuffer[8] __attribute__((aligned(16))); + FLOAT *ybuffer; if (m < 1) return (0); if (n < 1) return (0); diff --git a/kernel/power/cgemv_t.c b/kernel/power/cgemv_t.c index c646618cf..68bbdd60a 100644 --- a/kernel/power/cgemv_t.c +++ b/kernel/power/cgemv_t.c @@ -29,7 +29,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define NBMAX 1024 #include -static const unsigned char swap_mask_arr[]={ 4,5,6,7,0,1,2,3, 12,13,14,15, 8,9,10,11}; +static const unsigned char __attribute__((aligned(16))) swap_mask_arr[]={ 4,5,6,7,0,1,2,3, 12,13,14,15, 8,9,10,11}; static void cgemv_kernel_4x4(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOAT *y, FLOAT alpha_r, FLOAT alpha_i) { BLASLONG i; @@ -260,8 +260,8 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r, FLOAT alpha_i, BLASLONG m2; BLASLONG m3; BLASLONG n2; - - FLOAT ybuffer[8], *xbuffer; + FLOAT ybuffer[8] __attribute__((aligned(16))); + FLOAT *xbuffer; if (m < 1) return (0); if (n < 1) return (0); diff --git a/kernel/power/dgemv_n.c b/kernel/power/dgemv_n.c index b458e11fc..1a3d7669c 100644 --- a/kernel/power/dgemv_n.c +++ b/kernel/power/dgemv_n.c @@ -145,7 +145,7 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLO BLASLONG m3; BLASLONG n2; BLASLONG lda4 = lda << 2; - FLOAT xbuffer[8] __attribute__ ((aligned (16)));; + FLOAT xbuffer[8] __attribute__ ((aligned (16))); FLOAT *ybuffer; if ( m < 1 ) return(0); diff --git a/kernel/power/dgemv_t.c b/kernel/power/dgemv_t.c index b8589a131..d05d7b7d3 100644 --- a/kernel/power/dgemv_t.c +++ b/kernel/power/dgemv_t.c @@ -581,9 +581,9 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLO BLASLONG m1; BLASLONG m2; BLASLONG m3; - BLASLONG n2; - - FLOAT ybuffer[8], *xbuffer; + BLASLONG n2; + FLOAT ybuffer[8] __attribute__((aligned(16))); + FLOAT *xbuffer; if (m < 1) return (0); if (n < 1) return (0); diff --git a/kernel/power/sgemv_n.c b/kernel/power/sgemv_n.c index 9704757fe..81ac031a3 100644 --- a/kernel/power/sgemv_n.c +++ b/kernel/power/sgemv_n.c @@ -174,7 +174,8 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLO BLASLONG n2; BLASLONG lda4 = lda << 2; BLASLONG lda8 = lda << 3; - FLOAT xbuffer[8],*ybuffer; + FLOAT xbuffer[8] __attribute__((aligned(16))); + FLOAT *ybuffer; if ( m < 1 ) return(0); if ( n < 1 ) return(0); diff --git a/kernel/power/sgemv_n_8.c b/kernel/power/sgemv_n_8.c index 9bc93ced6..64696236a 100644 --- a/kernel/power/sgemv_n_8.c +++ b/kernel/power/sgemv_n_8.c @@ -213,7 +213,8 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLO BLASLONG n2; BLASLONG lda4 = lda << 2; BLASLONG lda8 = lda << 3; - FLOAT xbuffer[8],*ybuffer; + FLOAT xbuffer[8] __attribute__((aligned(16))); + FLOAT *ybuffer; if ( m < 1 ) return(0); if ( n < 1 ) return(0); diff --git a/kernel/power/sgemv_t.c b/kernel/power/sgemv_t.c index 96434a13f..3d8a442dc 100644 --- a/kernel/power/sgemv_t.c +++ b/kernel/power/sgemv_t.c @@ -177,10 +177,9 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLO BLASLONG m1; BLASLONG m2; BLASLONG m3; - BLASLONG n2; - - FLOAT ybuffer[8], *xbuffer; - + BLASLONG n2; + FLOAT ybuffer[8] __attribute__((aligned(16))); + FLOAT *xbuffer; if (m < 1) return (0); if (n < 1) return (0); diff --git a/kernel/power/sgemv_t_8.c b/kernel/power/sgemv_t_8.c index 5e9cd63ac..b90512162 100644 --- a/kernel/power/sgemv_t_8.c +++ b/kernel/power/sgemv_t_8.c @@ -204,8 +204,8 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLO BLASLONG m3; BLASLONG n2; - FLOAT ybuffer[8], *xbuffer; - + FLOAT ybuffer[8] __attribute__((aligned(16))); + FLOAT *xbuffer; if (m < 1) return (0); if (n < 1) return (0); diff --git a/kernel/power/zgemv_n_4.c b/kernel/power/zgemv_n_4.c index 167b0a158..ba019d6a5 100644 --- a/kernel/power/zgemv_n_4.c +++ b/kernel/power/zgemv_n_4.c @@ -614,8 +614,8 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r, FLOAT alpha_i, BLASLONG m2; BLASLONG m3; BLASLONG n2; - - FLOAT xbuffer[8], *ybuffer; + FLOAT xbuffer[8] __attribute__((aligned(16))); + FLOAT *ybuffer; if (m < 1) return (0); if (n < 1) return (0); diff --git a/kernel/power/zgemv_t_4.c b/kernel/power/zgemv_t_4.c index 20a0812dd..b34199af6 100644 --- a/kernel/power/zgemv_t_4.c +++ b/kernel/power/zgemv_t_4.c @@ -532,8 +532,8 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r, FLOAT alpha_i, BLASLONG m2; BLASLONG m3; BLASLONG n2; - - FLOAT ybuffer[8], *xbuffer; + FLOAT ybuffer[8] __attribute__((aligned(16))); + FLOAT *xbuffer; if (m < 1) return (0); if (n < 1) return (0); From e79712d96941c099d2f5e4b11544b2a20d97fbdf Mon Sep 17 00:00:00 2001 From: AbdelRauf Date: Fri, 30 Aug 2019 02:52:04 +0000 Subject: [PATCH 06/10] cgemv using vec_vsx_ld instead of letting gcc to decide --- kernel/power/cgemv_n.c | 97 +++++++++++++++++++++++------------------- kernel/power/cgemv_t.c | 97 +++++++++++++++++++++++++++--------------- 2 files changed, 115 insertions(+), 79 deletions(-) diff --git a/kernel/power/cgemv_n.c b/kernel/power/cgemv_n.c index 6a195d6d1..e85517ffa 100644 --- a/kernel/power/cgemv_n.c +++ b/kernel/power/cgemv_n.c @@ -62,23 +62,24 @@ static void cgemv_kernel_4x4(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOA register __vector float vx3_r = {x[6], -x[6],x[6], -x[6]}; register __vector float vx3_i = {x[7], x[7],x[7], x[7]}; #endif - register __vector float *vy = (__vector float *) y; + register __vector float *vptr_y = (__vector float *) y; register __vector float *vptr_a0 = (__vector float *) a0; register __vector float *vptr_a1 = (__vector float *) a1; register __vector float *vptr_a2 = (__vector float *) a2; register __vector float *vptr_a3 = (__vector float *) a3; BLASLONG i = 0; - for (;i< n / 2; i+=2) { - register __vector float vy_0 = vy[i]; - register __vector float vy_1 = vy[i + 1]; - register __vector float va0 = vptr_a0[i]; - register __vector float va1 = vptr_a1[i]; - register __vector float va2 = vptr_a2[i]; - register __vector float va3 = vptr_a3[i]; - register __vector float va0_1 = vptr_a0[i + 1]; - register __vector float va1_1 = vptr_a1[i + 1]; - register __vector float va2_1 = vptr_a2[i + 1]; - register __vector float va3_1 = vptr_a3[i + 1]; + BLASLONG i2=16; + for (;i< n * 8; i+=32,i2+=32) { + register __vector float vy_0 = vec_vsx_ld(i,vptr_y); + register __vector float vy_1 = vec_vsx_ld(i2,vptr_y); + register __vector float va0 = vec_vsx_ld(i,vptr_a0); + register __vector float va1 = vec_vsx_ld(i, vptr_a1); + register __vector float va2 = vec_vsx_ld(i ,vptr_a2); + register __vector float va3 = vec_vsx_ld(i ,vptr_a3); + register __vector float va0_1 = vec_vsx_ld(i2 ,vptr_a0); + register __vector float va1_1 = vec_vsx_ld(i2 ,vptr_a1); + register __vector float va2_1 = vec_vsx_ld(i2 ,vptr_a2); + register __vector float va3_1 = vec_vsx_ld(i2 ,vptr_a3); vy_0 += va0*vx0_r + va1*vx1_r + va2*vx2_r + va3*vx3_r; vy_1 += va0_1*vx0_r + va1_1*vx1_r + va2_1*vx2_r + va3_1*vx3_r; @@ -93,8 +94,8 @@ static void cgemv_kernel_4x4(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOA vy_0 += va0*vx0_i + va1*vx1_i + va2*vx2_i + va3*vx3_i; vy_1 += va0_1*vx0_i + va1_1*vx1_i + va2_1*vx2_i + va3_1*vx3_i; - vy[i] = vy_0; - vy[i + 1] = vy_1; + vec_vsx_st(vy_0 ,i, vptr_y); + vec_vsx_st(vy_1,i2,vptr_y); } } @@ -118,17 +119,19 @@ static void cgemv_kernel_4x2(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOA register __vector float vx1_r = {x[2], -x[2],x[2], -x[2]}; register __vector float vx1_i = {x[3], x[3],x[3], x[3]}; #endif - register __vector float *vy = (__vector float *) y; + register __vector float *vptr_y = (__vector float *) y; register __vector float *vptr_a0 = (__vector float *) a0; register __vector float *vptr_a1 = (__vector float *) a1; - BLASLONG i = 0; - for (;i< n / 2; i+=2) { - register __vector float vy_0 = vy[i]; - register __vector float vy_1 = vy[i + 1]; - register __vector float va0 = vptr_a0[i]; - register __vector float va1 = vptr_a1[i]; - register __vector float va0_1 = vptr_a0[i + 1]; - register __vector float va1_1 = vptr_a1[i + 1]; + BLASLONG i = 0; + BLASLONG i2 = 16; + for (;i< n * 8; i+=32, i2+=32) { + register __vector float vy_0 = vec_vsx_ld(i,vptr_y); + register __vector float vy_1 = vec_vsx_ld(i2,vptr_y); + register __vector float va0 = vec_vsx_ld(i,vptr_a0); + register __vector float va1 = vec_vsx_ld(i, vptr_a1); + register __vector float va0_1 = vec_vsx_ld(i2 ,vptr_a0); + register __vector float va1_1 = vec_vsx_ld(i2 ,vptr_a1); + register __vector float va0x = vec_perm(va0, va0,swap_mask); register __vector float va0x_1 = vec_perm(va0_1, va0_1,swap_mask); register __vector float va1x = vec_perm(va1, va1,swap_mask); @@ -136,8 +139,8 @@ static void cgemv_kernel_4x2(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOA vy_0 += va0*vx0_r + va1*vx1_r + va0x*vx0_i + va1x*vx1_i; vy_1 += va0_1*vx0_r + va1_1*vx1_r + va0x_1*vx0_i + va1x_1*vx1_i; - vy[i] = vy_0; - vy[i + 1] = vy_1; + vec_vsx_st(vy_0 ,i, vptr_y); + vec_vsx_st(vy_1,i2,vptr_y); } } @@ -154,21 +157,23 @@ static void cgemv_kernel_4x1(BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y) { register __vector float vx0_r = {x[0], -x[0],x[0], -x[0]}; register __vector float vx0_i = {x[1], x[1],x[1], x[1]}; #endif - register __vector float *vy = (__vector float *) y; + register __vector float *vptr_y = (__vector float *) y; register __vector float *vptr_a0 = (__vector float *) ap; BLASLONG i = 0; - for (;i< n / 2; i+=2) { - register __vector float vy_0 = vy[i]; - register __vector float vy_1 = vy[i + 1]; - register __vector float va0 = vptr_a0[i]; - register __vector float va0_1 = vptr_a0[i + 1]; + BLASLONG i2 = 16; + for (;i< n * 8; i+=32, i2+=32) { + register __vector float vy_0 = vec_vsx_ld(i,vptr_y); + register __vector float vy_1 = vec_vsx_ld(i2,vptr_y); + register __vector float va0 = vec_vsx_ld(i,vptr_a0); + register __vector float va0_1 = vec_vsx_ld(i2 ,vptr_a0); + register __vector float va0x = vec_perm(va0, va0,swap_mask); register __vector float va0x_1 = vec_perm(va0_1, va0_1,swap_mask); vy_0 += va0*vx0_r + va0x*vx0_i; vy_1 += va0_1*vx0_r + va0x_1*vx0_i; - vy[i] = vy_0; - vy[i + 1] = vy_1; + vec_vsx_st(vy_0 ,i, vptr_y); + vec_vsx_st(vy_1,i2,vptr_y); } } @@ -213,20 +218,24 @@ static void add_y(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_dest, FLOAT register __vector float *vptr_src = (__vector float *) src; register __vector float *vptr_y = (__vector float *) dest; - for (i = 0; i < n/2; i += 2 ){ - register __vector float vy_0 = vptr_y[i]; - register __vector float vy_1 = vptr_y[i +1]; + BLASLONG i2 = 16; + for (;i< n * 8; i+=32, i2+=32) { + register __vector float vy_0 = vec_vsx_ld(i,vptr_y); + register __vector float vy_1 = vec_vsx_ld(i2,vptr_y); - register __vector float vsrc = vptr_src[i]; - register __vector float vsrc_1 = vptr_src[i + 1]; - register __vector float vsrcx = vec_perm(vsrc, vsrc, swap_mask); - register __vector float vsrcx_1 = vec_perm(vsrc_1, vsrc_1, swap_mask); - vy_0 += vsrc*valpha_r + vsrcx*valpha_i; - vy_1 += vsrc_1*valpha_r + vsrcx_1*valpha_i; - vptr_y[i] = vy_0; - vptr_y[i+1 ] = vy_1; + register __vector float vsrc = vec_vsx_ld(i,vptr_src); + register __vector float vsrc_1 = vec_vsx_ld(i2,vptr_src); + + register __vector float vsrcx = vec_perm(vsrc, vsrc, swap_mask); + register __vector float vsrcx_1 = vec_perm(vsrc_1, vsrc_1, swap_mask); + + vy_0 += vsrc*valpha_r + vsrcx*valpha_i; + vy_1 += vsrc_1*valpha_r + vsrcx_1*valpha_i; + + vec_vsx_st(vy_0 ,i, vptr_y); + vec_vsx_st(vy_1,i2,vptr_y); } diff --git a/kernel/power/cgemv_t.c b/kernel/power/cgemv_t.c index 68bbdd60a..57eb066b0 100644 --- a/kernel/power/cgemv_t.c +++ b/kernel/power/cgemv_t.c @@ -32,7 +32,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. static const unsigned char __attribute__((aligned(16))) swap_mask_arr[]={ 4,5,6,7,0,1,2,3, 12,13,14,15, 8,9,10,11}; static void cgemv_kernel_4x4(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOAT *y, FLOAT alpha_r, FLOAT alpha_i) { - BLASLONG i; + FLOAT *a0, *a1, *a2, *a3; a0 = ap; a1 = ap + lda; @@ -48,26 +48,39 @@ static void cgemv_kernel_4x4(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOA register __vector float vtemp2_r = {0.0, 0.0,0.0,0.0}; register __vector float vtemp3_p = {0.0, 0.0,0.0,0.0}; register __vector float vtemp3_r = {0.0, 0.0,0.0,0.0}; - __vector float* va0 = (__vector float*) a0; - __vector float* va1 = (__vector float*) a1; - __vector float* va2 = (__vector float*) a2; - __vector float* va3 = (__vector float*) a3; + __vector float* vptr_a0 = (__vector float*) a0; + __vector float* vptr_a1 = (__vector float*) a1; + __vector float* vptr_a2 = (__vector float*) a2; + __vector float* vptr_a3 = (__vector float*) a3; __vector float* v_x = (__vector float*) x; - for (i = 0; i < n / 2; i+=2) { - register __vector float vx_0 = v_x[i]; - register __vector float vx_1 = v_x[i+1]; + BLASLONG i = 0; + BLASLONG i2 = 16; + for (;i< n * 8; i+=32, i2+=32) { + register __vector float vx_0 = vec_vsx_ld( i,v_x) ; + register __vector float vx_1 = vec_vsx_ld(i2, v_x); + register __vector float vxr_0 = vec_perm(vx_0, vx_0, swap_mask); register __vector float vxr_1 = vec_perm(vx_1, vx_1, swap_mask); - vtemp0_p += vx_0*va0[i] + vx_1*va0[i+1] ; - vtemp0_r += vxr_0*va0[i] + vxr_1*va0[i+1]; - vtemp1_p += vx_0*va1[i] + vx_1*va1[i+1]; - vtemp1_r += vxr_0*va1[i] + vxr_1*va1[i+1]; - vtemp2_p += vx_0*va2[i] + vx_1*va2[i+1]; - vtemp2_r += vxr_0*va2[i] + vxr_1*va2[i+1]; - vtemp3_p += vx_0*va3[i] + vx_1*va3[i+1]; - vtemp3_r += vxr_0*va3[i] + vxr_1*va3[i+1]; + register __vector float va0 = vec_vsx_ld(i,vptr_a0); + register __vector float va1 = vec_vsx_ld(i, vptr_a1); + register __vector float va2 = vec_vsx_ld(i ,vptr_a2); + register __vector float va3 = vec_vsx_ld(i ,vptr_a3); + register __vector float va0_1 = vec_vsx_ld(i2 ,vptr_a0); + register __vector float va1_1 = vec_vsx_ld(i2 ,vptr_a1); + register __vector float va2_1 = vec_vsx_ld(i2 ,vptr_a2); + register __vector float va3_1 = vec_vsx_ld(i2 ,vptr_a3); + + + vtemp0_p += vx_0*va0 + vx_1*va0_1 ; + vtemp0_r += vxr_0*va0 + vxr_1*va0_1; + vtemp1_p += vx_0*va1 + vx_1*va1_1; + vtemp1_r += vxr_0*va1 + vxr_1*va1_1; + vtemp2_p += vx_0*va2 + vx_1*va2_1; + vtemp2_r += vxr_0*va2 + vxr_1*va2_1; + vtemp3_p += vx_0*va3 + vx_1*va3_1; + vtemp3_r += vxr_0*va3 + vxr_1*va3_1; } @@ -128,7 +141,7 @@ static void cgemv_kernel_4x4(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOA static void cgemv_kernel_4x2(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOAT *y, FLOAT alpha_r, FLOAT alpha_i) { - BLASLONG i; + FLOAT *a0, *a1; a0 = ap; a1 = ap + lda; @@ -138,23 +151,33 @@ static void cgemv_kernel_4x2(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOA register __vector float vtemp0_r = {0.0, 0.0,0.0,0.0}; register __vector float vtemp1_p = {0.0, 0.0,0.0,0.0}; register __vector float vtemp1_r = {0.0, 0.0,0.0,0.0}; - __vector float* va0 = (__vector float*) a0; - __vector float* va1 = (__vector float*) a1; + + + __vector float* vptr_a0 = (__vector float*) a0; + __vector float* vptr_a1 = (__vector float*) a1; __vector float* v_x = (__vector float*) x; - for (i = 0; i < n / 2; i+=2) { - register __vector float vx_0 = v_x[i]; - register __vector float vx_1 = v_x[i+1]; + BLASLONG i = 0; + BLASLONG i2 = 16; + for (;i< n * 8; i+=32, i2+=32) { + register __vector float vx_0 = vec_vsx_ld( i,v_x) ; + register __vector float vx_1 = vec_vsx_ld(i2, v_x); + register __vector float vxr_0 = vec_perm(vx_0, vx_0, swap_mask); register __vector float vxr_1 = vec_perm(vx_1, vx_1, swap_mask); - vtemp0_p += vx_0*va0[i] + vx_1*va0[i+1] ; - vtemp0_r += vxr_0*va0[i] + vxr_1*va0[i+1]; - vtemp1_p += vx_0*va1[i] + vx_1*va1[i+1]; - vtemp1_r += vxr_0*va1[i] + vxr_1*va1[i+1]; + register __vector float va0 = vec_vsx_ld(i,vptr_a0); + register __vector float va1 = vec_vsx_ld(i, vptr_a1); + register __vector float va0_1 = vec_vsx_ld(i2 ,vptr_a0); + register __vector float va1_1 = vec_vsx_ld(i2 ,vptr_a1); + + + vtemp0_p += vx_0*va0 + vx_1*va0_1 ; + vtemp0_r += vxr_0*va0 + vxr_1*va0_1; + vtemp1_p += vx_0*va1 + vx_1*va1_1; + vtemp1_r += vxr_0*va1 + vxr_1*va1_1; } - #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) ) register FLOAT temp_r0 = vtemp0_p[0] - vtemp0_p[1] + vtemp0_p[2] - vtemp0_p[3]; @@ -193,23 +216,27 @@ static void cgemv_kernel_4x2(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOA static void cgemv_kernel_4x1(BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y, FLOAT alpha_r, FLOAT alpha_i) { - BLASLONG i; + __vector unsigned char swap_mask = *((__vector unsigned char*)swap_mask_arr); //p for positive(real*real,image*image,real*real,image*image) r for image (real*image,image*real,real*image,image*real) register __vector float vtemp0_p = {0.0, 0.0,0.0,0.0}; register __vector float vtemp0_r = {0.0, 0.0,0.0,0.0}; - __vector float* va0 = (__vector float*) ap; + __vector float* vptr_a0 = (__vector float*) ap; __vector float* v_x = (__vector float*) x; - - for (i = 0; i < n / 2; i+=2) { - register __vector float vx_0 = v_x[i]; - register __vector float vx_1 = v_x[i+1]; + BLASLONG i = 0; + BLASLONG i2 = 16; + for (;i< n * 8; i+=32, i2+=32) { + register __vector float vx_0 = vec_vsx_ld( i,v_x) ; + register __vector float vx_1 = vec_vsx_ld(i2, v_x); + register __vector float vxr_0 = vec_perm(vx_0, vx_0, swap_mask); register __vector float vxr_1 = vec_perm(vx_1, vx_1, swap_mask); - vtemp0_p += vx_0*va0[i] + vx_1*va0[i+1] ; - vtemp0_r += vxr_0*va0[i] + vxr_1*va0[i+1]; + register __vector float va0 = vec_vsx_ld(i,vptr_a0); + register __vector float va0_1 = vec_vsx_ld(i2 ,vptr_a0); + vtemp0_p += vx_0*va0 + vx_1*va0_1 ; + vtemp0_r += vxr_0*va0 + vxr_1*va0_1; } #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) ) From 4c22828812a9d5f0962c836d4c8bf486fde4d9cb Mon Sep 17 00:00:00 2001 From: AbdelRauf Date: Fri, 30 Aug 2019 04:09:15 +0000 Subject: [PATCH 07/10] caxpy and cdot are using vec_vsx_ld --- kernel/power/caxpy.c | 67 ++++++++++++++++++++++++++------------------ kernel/power/cdot.c | 48 +++++++++++++++++-------------- 2 files changed, 67 insertions(+), 48 deletions(-) diff --git a/kernel/power/caxpy.c b/kernel/power/caxpy.c index 00f2ec5e0..0545766b1 100644 --- a/kernel/power/caxpy.c +++ b/kernel/power/caxpy.c @@ -27,6 +27,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "common.h" #ifndef HAVE_ASM_KERNEL #include + +#define offset_0 0 +#define offset_1 16 +#define offset_2 32 +#define offset_3 48 +#define offset_4 64 +#define offset_5 80 +#define offset_6 96 +#define offset_7 112 + static const unsigned char __attribute__((aligned(16))) swap_mask_arr[]={ 4,5,6,7,0,1,2,3, 12,13,14,15, 8,9,10,11}; static void caxpy_kernel_16(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT alpha_r, FLOAT alpha_i) @@ -43,27 +53,28 @@ static void caxpy_kernel_16(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT alpha_r, FLOAT #endif __vector unsigned char swap_mask = *((__vector unsigned char*)swap_mask_arr); - register __vector float *vy = (__vector float *) y; - register __vector float *vx = (__vector float *) x; + register __vector float *vptr_y = (__vector float *) y; + register __vector float *vptr_x = (__vector float *) x; BLASLONG i=0; - for (; i < n/2; i += 8) { + for(;i + +#define offset_0 0 +#define offset_1 16 +#define offset_2 32 +#define offset_3 48 + + + static const unsigned char __attribute__((aligned(16))) swap_mask_arr[]={ 4,5,6,7,0,1,2,3, 12,13,14,15, 8,9,10,11}; static void cdot_kernel_8(BLASLONG n, FLOAT *x, FLOAT *y, float *dot) { __vector unsigned char swap_mask = *((__vector unsigned char*)swap_mask_arr); - register __vector float *vy = (__vector float *) y; - register __vector float *vx = (__vector float *) x; - BLASLONG i = 0; + register __vector float *vptr_y = (__vector float *) y; + register __vector float *vptr_x = (__vector float *) x; register __vector float vd_0 = { 0 }; register __vector float vd_1 = { 0 }; register __vector float vd_2 = { 0 }; @@ -41,26 +48,23 @@ static void cdot_kernel_8(BLASLONG n, FLOAT *x, FLOAT *y, float *dot) register __vector float vdd_0 = { 0 }; register __vector float vdd_1 = { 0 }; register __vector float vdd_2 = { 0 }; - register __vector float vdd_3 = { 0 }; - for (; i < n/2; i += 4) { + register __vector float vdd_3 = { 0 }; + BLASLONG i=0; + for(;i Date: Fri, 30 Aug 2019 11:14:55 +0000 Subject: [PATCH 08/10] fix uninitialized variables i --- kernel/power/cgemv_n.c | 4 ++-- kernel/power/cgemv_t.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/power/cgemv_n.c b/kernel/power/cgemv_n.c index e85517ffa..eec3fa37c 100644 --- a/kernel/power/cgemv_n.c +++ b/kernel/power/cgemv_n.c @@ -181,7 +181,7 @@ static void cgemv_kernel_4x1(BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y) { static void add_y(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_dest, FLOAT alpha_r, FLOAT alpha_i) { - BLASLONG i; + BLASLONG i=0; if (inc_dest != 2) { @@ -246,7 +246,7 @@ static void add_y(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_dest, FLOAT int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r, FLOAT alpha_i, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT * buffer) { - BLASLONG i; + BLASLONG i=0; FLOAT *a_ptr; FLOAT *x_ptr; FLOAT *y_ptr; diff --git a/kernel/power/cgemv_t.c b/kernel/power/cgemv_t.c index 57eb066b0..691f7a3d3 100644 --- a/kernel/power/cgemv_t.c +++ b/kernel/power/cgemv_t.c @@ -276,8 +276,8 @@ static void copy_x(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_src) { } int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r, FLOAT alpha_i, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer) { - BLASLONG i; - BLASLONG j; + BLASLONG i=0; + BLASLONG j=0; FLOAT *a_ptr; FLOAT *x_ptr; FLOAT *y_ptr; From b5af7b9c7808f06d6c10c2b2db90a054ab746970 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sat, 31 Aug 2019 18:06:12 +0200 Subject: [PATCH 09/10] Disable ppc64le test environment on Travis CI as this semi-official beta option has suddenly reverted to a standard x86_64 environment causing spurious failures --- .travis.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index a92bb0687..27ecba6c8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,14 +25,14 @@ matrix: - TARGET_BOX=LINUX64 - BTYPE="BINARY=64" - - <<: *test-ubuntu - os: linux-ppc64le - before_script: - - COMMON_FLAGS="DYNAMIC_ARCH=1 TARGET=POWER8 NUM_THREADS=32" - env: - # for matrix annotation only - - TARGET_BOX=PPC64LE_LINUX - - BTYPE="BINARY=64 USE_OPENMP=1" + # - <<: *test-ubuntu + # os: linux-ppc64le + # before_script: + # - COMMON_FLAGS="DYNAMIC_ARCH=1 TARGET=POWER8 NUM_THREADS=32" + # env: + # # for matrix annotation only + # - TARGET_BOX=PPC64LE_LINUX + # - BTYPE="BINARY=64 USE_OPENMP=1" - <<: *test-ubuntu env: From 1fec0570f6b0561a52d72e5d37bbae5fb8d467cb Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Mon, 2 Sep 2019 15:03:45 +0200 Subject: [PATCH 10/10] Add cgemm and zgemm unroll factors for core2 --- cmake/prebuild.cmake | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmake/prebuild.cmake b/cmake/prebuild.cmake index c2600bd0d..2fe168a1c 100644 --- a/cmake/prebuild.cmake +++ b/cmake/prebuild.cmake @@ -124,6 +124,10 @@ if (DEFINED CORE AND CMAKE_CROSSCOMPILING AND NOT (${HOST_OS} STREQUAL "WINDOWSS set(SGEMM_UNROLL_N 4) set(DGEMM_UNROLL_M 4) set(DGEMM_UNROLL_N 4) + set(CGEMM_DEFAULT_UNROLL_M 4) + set(CGEMM_DEFAULT_UNROLL_N 2) + set(ZGEMM_DEFAULT_UNROLL_M 2) + set(ZGEMM_DEFAULT_UNROLL_N 2) elseif ("${TCORE}" STREQUAL "ARMV7") file(APPEND ${TARGET_CONF_TEMP} "#define L1_DATA_SIZE\t65536\n"