From 2359c7c1a9378b74396302880a306d5382d277ac Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sat, 24 Feb 2018 17:50:13 +0100 Subject: [PATCH] Use .p2align instead of .align for portability The OSX assembler apparently mishandles the argument to decimal .align, leading to a significant loss of performance as observed in #730, #901 and most recently #1470 --- kernel/x86_64/caxpy_microk_haswell-2.c | 8 ++++---- kernel/x86_64/cdot_microk_haswell-2.c | 2 +- kernel/x86_64/cscal_microk_haswell-2.c | 8 ++++---- kernel/x86_64/daxpy_microk_haswell-2.c | 2 +- kernel/x86_64/ddot_microk_haswell-2.c | 2 +- kernel/x86_64/dscal_microk_haswell-2.c | 4 ++-- kernel/x86_64/dsymv_L_microk_haswell-2.c | 2 +- kernel/x86_64/dsymv_U_microk_haswell-2.c | 2 +- kernel/x86_64/dtrmm_kernel_4x8_haswell.c | 2 +- kernel/x86_64/dtrsm_kernel_RN_haswell.c | 2 +- kernel/x86_64/saxpy_microk_haswell-2.c | 2 +- kernel/x86_64/sdot_microk_haswell-2.c | 2 +- kernel/x86_64/ssymv_L_microk_haswell-2.c | 2 +- kernel/x86_64/ssymv_U_microk_haswell-2.c | 2 +- kernel/x86_64/zaxpy_microk_haswell-2.c | 8 ++++---- kernel/x86_64/zdot_microk_haswell-2.c | 4 ++-- kernel/x86_64/zscal_microk_haswell-2.c | 8 ++++---- 17 files changed, 31 insertions(+), 31 deletions(-) diff --git a/kernel/x86_64/caxpy_microk_haswell-2.c b/kernel/x86_64/caxpy_microk_haswell-2.c index 7a9fc1b95..00e2e6a42 100644 --- a/kernel/x86_64/caxpy_microk_haswell-2.c +++ b/kernel/x86_64/caxpy_microk_haswell-2.c @@ -50,11 +50,11 @@ static void caxpy_kernel_8( BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *alpha) "vmulps (%5), %%ymm0 , %%ymm0 \n\t" #endif - ".align 16 \n\t" + ".p2align 4 \n\t" "1: \n\t" "vmovups (%2,%0,4), %%ymm5 \n\t" // 4 complex values from x - ".align 2 \n\t" + ".p2align 1 \n\t" "vmovups 32(%2,%0,4), %%ymm7 \n\t" // 4 complex values from x "vmovups 64(%2,%0,4), %%ymm9 \n\t" // 4 complex values from x "vmovups 96(%2,%0,4), %%ymm11 \n\t" // 4 complex values from x @@ -70,7 +70,7 @@ static void caxpy_kernel_8( BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *alpha) "vpermilps $0xb1 , %%ymm11, %%ymm10 \n\t" // exchange real and imag part "vfmadd213ps (%3,%0,4), %%ymm0 , %%ymm5 \n\t" - ".align 2 \n\t" + ".p2align 1 \n\t" "vfmadd213ps 32(%3,%0,4), %%ymm0 , %%ymm7 \n\t" "vfmadd213ps 64(%3,%0,4), %%ymm0 , %%ymm9 \n\t" "vfmadd213ps 96(%3,%0,4), %%ymm0 , %%ymm11 \n\t" @@ -96,7 +96,7 @@ static void caxpy_kernel_8( BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *alpha) "vfmadd231ps %%ymm1 , %%ymm10, %%ymm15 \n\t" "vmovups %%ymm5 , (%3,%0,4) \n\t" - ".align 2 \n\t" + ".p2align 1 \n\t" "vmovups %%ymm7 , 32(%3,%0,4) \n\t" "vmovups %%ymm9 , 64(%3,%0,4) \n\t" "vmovups %%ymm11, 96(%3,%0,4) \n\t" diff --git a/kernel/x86_64/cdot_microk_haswell-2.c b/kernel/x86_64/cdot_microk_haswell-2.c index fc76b138a..fe195a63b 100644 --- a/kernel/x86_64/cdot_microk_haswell-2.c +++ b/kernel/x86_64/cdot_microk_haswell-2.c @@ -46,7 +46,7 @@ static void cdot_kernel_16( BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *dot) "vxorps %%ymm6, %%ymm6, %%ymm6 \n\t" "vxorps %%ymm7, %%ymm7, %%ymm7 \n\t" - ".align 16 \n\t" + ".p2align 4 \n\t" "1: \n\t" "vmovups (%2,%0,4), %%ymm8 \n\t" // 2 * x "vmovups 32(%2,%0,4), %%ymm9 \n\t" // 2 * x diff --git a/kernel/x86_64/cscal_microk_haswell-2.c b/kernel/x86_64/cscal_microk_haswell-2.c index 48e339501..0a4eb683c 100644 --- a/kernel/x86_64/cscal_microk_haswell-2.c +++ b/kernel/x86_64/cscal_microk_haswell-2.c @@ -54,7 +54,7 @@ static void cscal_kernel_16( BLASLONG n, FLOAT *alpha, FLOAT *x) "subq $16, %0 \n\t" "jz 2f \n\t" - ".align 16 \n\t" + ".p2align 4 \n\t" "1: \n\t" //"prefetcht0 128(%1) \n\t" @@ -156,7 +156,7 @@ static void cscal_kernel_16_zero_r( BLASLONG n, FLOAT *alpha, FLOAT *x) "subq $16, %0 \n\t" "jz 2f \n\t" - ".align 16 \n\t" + ".p2align 4 \n\t" "1: \n\t" //"prefetcht0 128(%1) \n\t" @@ -245,7 +245,7 @@ static void cscal_kernel_16_zero_i( BLASLONG n, FLOAT *alpha, FLOAT *x) "subq $16, %0 \n\t" "jz 2f \n\t" - ".align 16 \n\t" + ".p2align 4 \n\t" "1: \n\t" //"prefetcht0 128(%1) \n\t" @@ -312,7 +312,7 @@ static void cscal_kernel_16_zero( BLASLONG n, FLOAT *alpha, FLOAT *x) "addq $128, %1 \n\t" - ".align 16 \n\t" + ".p2align 4 \n\t" "1: \n\t" //"prefetcht0 128(%1) \n\t" diff --git a/kernel/x86_64/daxpy_microk_haswell-2.c b/kernel/x86_64/daxpy_microk_haswell-2.c index db117a8ba..bbe8b9550 100644 --- a/kernel/x86_64/daxpy_microk_haswell-2.c +++ b/kernel/x86_64/daxpy_microk_haswell-2.c @@ -38,7 +38,7 @@ static void daxpy_kernel_8( BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *alpha) ( "vbroadcastsd (%4), %%ymm0 \n\t" // alpha - ".align 16 \n\t" + ".p2align 4 \n\t" "1: \n\t" "vmovups (%3,%0,8), %%ymm12 \n\t" // 4 * y diff --git a/kernel/x86_64/ddot_microk_haswell-2.c b/kernel/x86_64/ddot_microk_haswell-2.c index d36577af3..365737363 100644 --- a/kernel/x86_64/ddot_microk_haswell-2.c +++ b/kernel/x86_64/ddot_microk_haswell-2.c @@ -41,7 +41,7 @@ static void ddot_kernel_8( BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *dot) "vxorpd %%ymm6, %%ymm6, %%ymm6 \n\t" "vxorpd %%ymm7, %%ymm7, %%ymm7 \n\t" - ".align 16 \n\t" + ".p2align 4 \n\t" "1: \n\t" "vmovups (%2,%0,8), %%ymm12 \n\t" // 2 * x "vmovups 32(%2,%0,8), %%ymm13 \n\t" // 2 * x diff --git a/kernel/x86_64/dscal_microk_haswell-2.c b/kernel/x86_64/dscal_microk_haswell-2.c index 07a9c804c..e732a2718 100644 --- a/kernel/x86_64/dscal_microk_haswell-2.c +++ b/kernel/x86_64/dscal_microk_haswell-2.c @@ -58,7 +58,7 @@ static void dscal_kernel_8( BLASLONG n, FLOAT *alpha, FLOAT *x) "subq $1 , %0 \n\t" "jz 2f \n\t" - ".align 16 \n\t" + ".p2align 4 \n\t" "1: \n\t" // "prefetcht0 640(%1) \n\t" @@ -156,7 +156,7 @@ static void dscal_kernel_8_zero( BLASLONG n, FLOAT *alpha, FLOAT *x) "cmpq $0, %0 \n\t" "je 2f \n\t" - ".align 16 \n\t" + ".p2align 4 \n\t" "1: \n\t" "vmovups %%xmm0 ,-128(%1) \n\t" diff --git a/kernel/x86_64/dsymv_L_microk_haswell-2.c b/kernel/x86_64/dsymv_L_microk_haswell-2.c index bc5ec6b87..866782ee6 100644 --- a/kernel/x86_64/dsymv_L_microk_haswell-2.c +++ b/kernel/x86_64/dsymv_L_microk_haswell-2.c @@ -44,7 +44,7 @@ static void dsymv_kernel_4x4(BLASLONG from, BLASLONG to, FLOAT **a, FLOAT *x, FL "vbroadcastsd 16(%8), %%ymm6 \n\t" // temp1[1] "vbroadcastsd 24(%8), %%ymm7 \n\t" // temp1[1] - ".align 16 \n\t" + ".p2align 4 \n\t" "1: \n\t" "vmovups (%3,%0,8), %%ymm9 \n\t" // 2 * y diff --git a/kernel/x86_64/dsymv_U_microk_haswell-2.c b/kernel/x86_64/dsymv_U_microk_haswell-2.c index 6ce384f93..d83d20f8e 100644 --- a/kernel/x86_64/dsymv_U_microk_haswell-2.c +++ b/kernel/x86_64/dsymv_U_microk_haswell-2.c @@ -46,7 +46,7 @@ static void dsymv_kernel_4x4(BLASLONG n, FLOAT *a0, FLOAT *a1, FLOAT *a2, FLOAT "vbroadcastsd 24(%8), %%ymm7 \n\t" // temp1[1] "xorq %0,%0 \n\t" - ".align 16 \n\t" + ".p2align 4 \n\t" "1: \n\t" "vmovups (%3,%0,8), %%ymm9 \n\t" // 2 * y diff --git a/kernel/x86_64/dtrmm_kernel_4x8_haswell.c b/kernel/x86_64/dtrmm_kernel_4x8_haswell.c index 289af772e..651736b89 100644 --- a/kernel/x86_64/dtrmm_kernel_4x8_haswell.c +++ b/kernel/x86_64/dtrmm_kernel_4x8_haswell.c @@ -24,7 +24,7 @@ static void dtrmm_kernel_4x8( BLASLONG n, FLOAT *alpha ,FLOAT *a, FLOAT *b, FLOA " cmp $0, %1 \n\t" " jz 2f \n\t" - " .align 16 \n\t" + " .p2align 4 \n\t" "1: \n\t" " vmovups (%2,%0,4) , %%ymm0 \n\t" " vmovups (%3,%0,8) , %%ymm1 \n\t" diff --git a/kernel/x86_64/dtrsm_kernel_RN_haswell.c b/kernel/x86_64/dtrsm_kernel_RN_haswell.c index da90e40c7..fcab8e2c7 100644 --- a/kernel/x86_64/dtrsm_kernel_RN_haswell.c +++ b/kernel/x86_64/dtrsm_kernel_RN_haswell.c @@ -128,7 +128,7 @@ static void dtrsm_RN_solve_opt(BLASLONG n, FLOAT *a, FLOAT *b, FLOAT *c, BLASLON " cmpq %1, %0 \n\t" " je 21f \n\t" - " .align 16 \n\t" + " .p2align 4 \n\t" "1: \n\t" " vmovups (%2,%1,4), %%ymm4 \n\t" // read a diff --git a/kernel/x86_64/saxpy_microk_haswell-2.c b/kernel/x86_64/saxpy_microk_haswell-2.c index 496424207..3a743d64c 100644 --- a/kernel/x86_64/saxpy_microk_haswell-2.c +++ b/kernel/x86_64/saxpy_microk_haswell-2.c @@ -38,7 +38,7 @@ static void saxpy_kernel_16( BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *alpha) ( "vbroadcastss (%4), %%ymm0 \n\t" // alpha - ".align 16 \n\t" + ".p2align 4 \n\t" "1: \n\t" "vmovups (%3,%0,4), %%ymm12 \n\t" // 8 * y diff --git a/kernel/x86_64/sdot_microk_haswell-2.c b/kernel/x86_64/sdot_microk_haswell-2.c index 3248c408c..df367b61f 100644 --- a/kernel/x86_64/sdot_microk_haswell-2.c +++ b/kernel/x86_64/sdot_microk_haswell-2.c @@ -41,7 +41,7 @@ static void sdot_kernel_16( BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *dot) "vxorps %%ymm6, %%ymm6, %%ymm6 \n\t" "vxorps %%ymm7, %%ymm7, %%ymm7 \n\t" - ".align 16 \n\t" + ".p2align 4 \n\t" "1: \n\t" "vmovups (%2,%0,4), %%ymm12 \n\t" // 2 * x "vmovups 32(%2,%0,4), %%ymm13 \n\t" // 2 * x diff --git a/kernel/x86_64/ssymv_L_microk_haswell-2.c b/kernel/x86_64/ssymv_L_microk_haswell-2.c index 516524528..69db008b6 100644 --- a/kernel/x86_64/ssymv_L_microk_haswell-2.c +++ b/kernel/x86_64/ssymv_L_microk_haswell-2.c @@ -43,7 +43,7 @@ static void ssymv_kernel_4x4(BLASLONG from, BLASLONG to, FLOAT **a, FLOAT *x, FL "vbroadcastss 8(%8), %%xmm6 \n\t" // temp1[1] "vbroadcastss 12(%8), %%xmm7 \n\t" // temp1[1] - ".align 16 \n\t" + ".p2align 4 \n\t" "1: \n\t" "vmovups (%3,%0,4), %%xmm9 \n\t" // 2 * y diff --git a/kernel/x86_64/ssymv_U_microk_haswell-2.c b/kernel/x86_64/ssymv_U_microk_haswell-2.c index 42f801c96..a32e59b44 100644 --- a/kernel/x86_64/ssymv_U_microk_haswell-2.c +++ b/kernel/x86_64/ssymv_U_microk_haswell-2.c @@ -46,7 +46,7 @@ static void ssymv_kernel_4x4(BLASLONG n, FLOAT *a0, FLOAT *a1, FLOAT *a2, FLOAT "vbroadcastss 12(%8), %%ymm7 \n\t" // temp1[1] "xorq %0,%0 \n\t" - ".align 16 \n\t" + ".p2align 4 \n\t" "1: \n\t" "vmovups (%3,%0,4), %%ymm9 \n\t" // 2 * y diff --git a/kernel/x86_64/zaxpy_microk_haswell-2.c b/kernel/x86_64/zaxpy_microk_haswell-2.c index e7e559502..30e8b1955 100644 --- a/kernel/x86_64/zaxpy_microk_haswell-2.c +++ b/kernel/x86_64/zaxpy_microk_haswell-2.c @@ -50,11 +50,11 @@ static void zaxpy_kernel_4( BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *alpha) "vmulpd (%5), %%ymm0 , %%ymm0 \n\t" #endif - ".align 16 \n\t" + ".p2align 4 \n\t" "1: \n\t" "vmovups (%2,%0,8), %%ymm5 \n\t" // 2 complex values from x - ".align 2 \n\t" + ".p2align 1 \n\t" "vmovups 32(%2,%0,8), %%ymm7 \n\t" // 2 complex values from x "vmovups 64(%2,%0,8), %%ymm9 \n\t" // 2 complex values from x "vmovups 96(%2,%0,8), %%ymm11 \n\t" // 2 complex values from x @@ -70,7 +70,7 @@ static void zaxpy_kernel_4( BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *alpha) "vpermilpd $0x5 , %%ymm11, %%ymm10 \n\t" // exchange real and imag part "vfmadd213pd (%3,%0,8), %%ymm0 , %%ymm5 \n\t" - ".align 2 \n\t" + ".p2align 1 \n\t" "vfmadd213pd 32(%3,%0,8), %%ymm0 , %%ymm7 \n\t" "vfmadd213pd 64(%3,%0,8), %%ymm0 , %%ymm9 \n\t" "vfmadd213pd 96(%3,%0,8), %%ymm0 , %%ymm11 \n\t" @@ -96,7 +96,7 @@ static void zaxpy_kernel_4( BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *alpha) "vfmadd231pd %%ymm1 , %%ymm10, %%ymm15 \n\t" "vmovups %%ymm5 , (%3,%0,8) \n\t" - ".align 2 \n\t" + ".p2align 1 \n\t" "vmovups %%ymm7 , 32(%3,%0,8) \n\t" "vmovups %%ymm9 , 64(%3,%0,8) \n\t" "vmovups %%ymm11, 96(%3,%0,8) \n\t" diff --git a/kernel/x86_64/zdot_microk_haswell-2.c b/kernel/x86_64/zdot_microk_haswell-2.c index 810cb4439..11056a3c1 100644 --- a/kernel/x86_64/zdot_microk_haswell-2.c +++ b/kernel/x86_64/zdot_microk_haswell-2.c @@ -50,7 +50,7 @@ static void zdot_kernel_8( BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *dot) "vxorpd %%ymm6, %%ymm6, %%ymm6 \n\t" "vxorpd %%ymm7, %%ymm7, %%ymm7 \n\t" - ".align 16 \n\t" + ".p2align 4 \n\t" "1: \n\t" "vmovups (%2,%0,8), %%ymm8 \n\t" // 2 * x "vmovups 32(%2,%0,8), %%ymm9 \n\t" // 2 * x @@ -131,7 +131,7 @@ static void zdot_kernel_8( BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *dot) "vxorpd %%ymm6, %%ymm6, %%ymm6 \n\t" "vxorpd %%ymm7, %%ymm7, %%ymm7 \n\t" - ".align 16 \n\t" + ".p2align 4 \n\t" "1: \n\t" "prefetcht0 512(%2,%0,8) \n\t" "vmovups (%2,%0,8), %%ymm8 \n\t" // 2 * x diff --git a/kernel/x86_64/zscal_microk_haswell-2.c b/kernel/x86_64/zscal_microk_haswell-2.c index b1a34c1dd..d9253c1ed 100644 --- a/kernel/x86_64/zscal_microk_haswell-2.c +++ b/kernel/x86_64/zscal_microk_haswell-2.c @@ -54,7 +54,7 @@ static void zscal_kernel_8( BLASLONG n, FLOAT *alpha, FLOAT *x) "subq $8 , %0 \n\t" "jz 2f \n\t" - ".align 16 \n\t" + ".p2align 4 \n\t" "1: \n\t" //"prefetcht0 128(%1) \n\t" @@ -156,7 +156,7 @@ static void zscal_kernel_8_zero_r( BLASLONG n, FLOAT *alpha, FLOAT *x) "subq $8 , %0 \n\t" "jz 2f \n\t" - ".align 16 \n\t" + ".p2align 4 \n\t" "1: \n\t" //"prefetcht0 128(%1) \n\t" @@ -245,7 +245,7 @@ static void zscal_kernel_8_zero_i( BLASLONG n, FLOAT *alpha, FLOAT *x) "subq $8 , %0 \n\t" "jz 2f \n\t" - ".align 16 \n\t" + ".p2align 4 \n\t" "1: \n\t" //"prefetcht0 128(%1) \n\t" @@ -312,7 +312,7 @@ static void zscal_kernel_8_zero( BLASLONG n, FLOAT *alpha, FLOAT *x) "addq $128, %1 \n\t" - ".align 16 \n\t" + ".p2align 4 \n\t" "1: \n\t" //"prefetcht0 128(%1) \n\t"