From 553e2754077eb0b2cb8782e8ecdb5e6eb9c8366b Mon Sep 17 00:00:00 2001 From: wernsaar Date: Sun, 7 Sep 2014 20:53:30 +0200 Subject: [PATCH] optimized sgemv_n kernel for sandybridge --- kernel/x86_64/sgemv_n_microk_sandy-4.c | 112 +++++++++++++++++-------- 1 file changed, 79 insertions(+), 33 deletions(-) diff --git a/kernel/x86_64/sgemv_n_microk_sandy-4.c b/kernel/x86_64/sgemv_n_microk_sandy-4.c index b4caca630..44c2b3f2b 100644 --- a/kernel/x86_64/sgemv_n_microk_sandy-4.c +++ b/kernel/x86_64/sgemv_n_microk_sandy-4.c @@ -29,9 +29,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define HAVE_KERNEL_4x8 1 -static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLONG lda4) __attribute__ ((noinline)); +static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLONG lda4, FLOAT *alpha) __attribute__ ((noinline)); -static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLONG lda4) +static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLONG lda4, FLOAT *alpha) { BLASLONG register i = 0; @@ -48,61 +48,75 @@ static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLO "vbroadcastss 24(%2), %%ymm2 \n\t" // x6 "vbroadcastss 28(%2), %%ymm3 \n\t" // x7 + "vbroadcastss (%9), %%ymm6 \n\t" // alpha + "testq $0x04, %1 \n\t" "jz .L08LABEL%= \n\t" - "vmovups (%3,%0,4), %%xmm4 \n\t" // 4 * y + "vxorps %%xmm4 , %%xmm4 , %%xmm4 \n\t" + "vxorps %%xmm5 , %%xmm5 , %%xmm5 \n\t" + "vmovups (%3,%0,4), %%xmm7 \n\t" // 4 * y "vmulps (%4,%0,4), %%xmm12, %%xmm8 \n\t" "vmulps (%5,%0,4), %%xmm13, %%xmm10 \n\t" "vmulps (%6,%0,4), %%xmm14, %%xmm9 \n\t" "vmulps (%7,%0,4), %%xmm15, %%xmm11 \n\t" "vaddps %%xmm4, %%xmm8 , %%xmm4 \n\t" - "vaddps %%xmm4, %%xmm10, %%xmm4 \n\t" + "vaddps %%xmm5, %%xmm10, %%xmm5 \n\t" "vaddps %%xmm4, %%xmm9 , %%xmm4 \n\t" - "vaddps %%xmm4, %%xmm11, %%xmm4 \n\t" + "vaddps %%xmm5, %%xmm11, %%xmm5 \n\t" "vmulps (%4,%8,4), %%xmm0 , %%xmm8 \n\t" "vmulps (%5,%8,4), %%xmm1 , %%xmm10 \n\t" "vmulps (%6,%8,4), %%xmm2 , %%xmm9 \n\t" "vmulps (%7,%8,4), %%xmm3 , %%xmm11 \n\t" "vaddps %%xmm4, %%xmm8 , %%xmm4 \n\t" - "vaddps %%xmm4, %%xmm10, %%xmm4 \n\t" + "vaddps %%xmm5, %%xmm10, %%xmm5 \n\t" "vaddps %%xmm4, %%xmm9 , %%xmm4 \n\t" - "vaddps %%xmm4, %%xmm11, %%xmm4 \n\t" + "vaddps %%xmm5, %%xmm11, %%xmm5 \n\t" - "vmovups %%xmm4, (%3,%0,4) \n\t" // 4 * y + "vaddps %%xmm5, %%xmm4 , %%xmm4 \n\t" + "vmulps %%xmm6, %%xmm4 , %%xmm5 \n\t" + "vaddps %%xmm5, %%xmm7 , %%xmm5 \n\t" + + "vmovups %%xmm5, (%3,%0,4) \n\t" // 4 * y "addq $4, %8 \n\t" "addq $4, %0 \n\t" "subq $4, %1 \n\t" - ".L08LABEL%=: \n\t" + ".L08LABEL%=: \n\t" "testq $0x08, %1 \n\t" "jz .L16LABEL%= \n\t" - "vmovups (%3,%0,4), %%ymm4 \n\t" // 8 * y + "vxorps %%ymm4 , %%ymm4 , %%ymm4 \n\t" + "vxorps %%ymm5 , %%ymm5 , %%ymm5 \n\t" + "vmovups (%3,%0,4), %%ymm7 \n\t" // 8 * y "vmulps (%4,%0,4), %%ymm12, %%ymm8 \n\t" "vmulps (%5,%0,4), %%ymm13, %%ymm10 \n\t" "vmulps (%6,%0,4), %%ymm14, %%ymm9 \n\t" "vmulps (%7,%0,4), %%ymm15, %%ymm11 \n\t" "vaddps %%ymm4, %%ymm8 , %%ymm4 \n\t" - "vaddps %%ymm4, %%ymm10, %%ymm4 \n\t" + "vaddps %%ymm5, %%ymm10, %%ymm5 \n\t" "vaddps %%ymm4, %%ymm9 , %%ymm4 \n\t" - "vaddps %%ymm4, %%ymm11, %%ymm4 \n\t" + "vaddps %%ymm5, %%ymm11, %%ymm5 \n\t" "vmulps (%4,%8,4), %%ymm0 , %%ymm8 \n\t" "vmulps (%5,%8,4), %%ymm1 , %%ymm10 \n\t" "vmulps (%6,%8,4), %%ymm2 , %%ymm9 \n\t" "vmulps (%7,%8,4), %%ymm3 , %%ymm11 \n\t" "vaddps %%ymm4, %%ymm8 , %%ymm4 \n\t" - "vaddps %%ymm4, %%ymm10, %%ymm4 \n\t" + "vaddps %%ymm5, %%ymm10, %%ymm5 \n\t" "vaddps %%ymm4, %%ymm9 , %%ymm4 \n\t" - "vaddps %%ymm4, %%ymm11, %%ymm4 \n\t" + "vaddps %%ymm5, %%ymm11, %%ymm5 \n\t" - "vmovups %%ymm4, (%3,%0,4) \n\t" // 8 * y + "vaddps %%ymm5, %%ymm4 , %%ymm4 \n\t" + "vmulps %%ymm6, %%ymm4 , %%ymm5 \n\t" + "vaddps %%ymm5, %%ymm7 , %%ymm5 \n\t" + + "vmovups %%ymm5, (%3,%0,4) \n\t" // 8 * y "addq $8, %8 \n\t" "addq $8, %0 \n\t" @@ -117,8 +131,8 @@ static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLO ".align 16 \n\t" ".L01LOOP%=: \n\t" - "vmovups (%3,%0,4), %%ymm4 \n\t" // 8 * y - "vmovups 32(%3,%0,4), %%ymm5 \n\t" // 8 * y + "vxorps %%ymm4 , %%ymm4 , %%ymm4 \n\t" + "vxorps %%ymm5 , %%ymm5 , %%ymm5 \n\t" "prefetcht0 192(%4,%0,4) \n\t" "vmulps (%4,%0,4), %%ymm12, %%ymm8 \n\t" @@ -164,6 +178,12 @@ static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLO "vaddps %%ymm4, %%ymm10, %%ymm4 \n\t" "vaddps %%ymm5, %%ymm11, %%ymm5 \n\t" + "vmulps %%ymm6, %%ymm4 , %%ymm4 \n\t" + "vmulps %%ymm6, %%ymm5 , %%ymm5 \n\t" + + "vaddps (%3,%0,4), %%ymm4 , %%ymm4 \n\t" // 8 * y + "vaddps 32(%3,%0,4), %%ymm5 , %%ymm5 \n\t" // 8 * y + "vmovups %%ymm4, (%3,%0,4) \n\t" // 8 * y "vmovups %%ymm5, 32(%3,%0,4) \n\t" // 8 * y @@ -185,11 +205,13 @@ static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLO "r" (ap[1]), // 5 "r" (ap[2]), // 6 "r" (ap[3]), // 7 - "r" (lda4) // 8 + "r" (lda4), // 8 + "r" (alpha) // 9 : "cc", "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", + "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" @@ -201,9 +223,9 @@ static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLO #define HAVE_KERNEL_4x4 1 -static void sgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) __attribute__ ((noinline)); +static void sgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT *alpha) __attribute__ ((noinline)); -static void sgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) +static void sgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT *alpha) { BLASLONG register i = 0; @@ -216,21 +238,29 @@ static void sgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) "vbroadcastss 8(%2), %%ymm14 \n\t" // x2 "vbroadcastss 12(%2), %%ymm15 \n\t" // x3 + "vbroadcastss (%8), %%ymm6 \n\t" // alpha + "testq $0x04, %1 \n\t" "jz .L08LABEL%= \n\t" - "vmovups (%3,%0,4), %%xmm4 \n\t" // 4 * y + "vxorps %%ymm4 , %%ymm4 , %%ymm4 \n\t" + "vxorps %%ymm5 , %%ymm5 , %%ymm5 \n\t" + "vmovups (%3,%0,4), %%xmm7 \n\t" // 4 * y "vmulps (%4,%0,4), %%xmm12, %%xmm8 \n\t" "vmulps (%5,%0,4), %%xmm13, %%xmm10 \n\t" "vmulps (%6,%0,4), %%xmm14, %%xmm9 \n\t" "vmulps (%7,%0,4), %%xmm15, %%xmm11 \n\t" "vaddps %%xmm4, %%xmm8 , %%xmm4 \n\t" - "vaddps %%xmm4, %%xmm10, %%xmm4 \n\t" + "vaddps %%xmm5, %%xmm10, %%xmm5 \n\t" "vaddps %%xmm4, %%xmm9 , %%xmm4 \n\t" - "vaddps %%xmm4, %%xmm11, %%xmm4 \n\t" + "vaddps %%xmm5, %%xmm11, %%xmm5 \n\t" - "vmovups %%xmm4, (%3,%0,4) \n\t" // 4 * y + "vaddps %%xmm5, %%xmm4 , %%xmm4 \n\t" + "vmulps %%xmm6, %%xmm4 , %%xmm5 \n\t" + "vaddps %%xmm5, %%xmm7 , %%xmm5 \n\t" + + "vmovups %%xmm5, (%3,%0,4) \n\t" // 4 * y "addq $4, %0 \n\t" "subq $4, %1 \n\t" @@ -240,18 +270,24 @@ static void sgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) "testq $0x08, %1 \n\t" "jz .L16LABEL%= \n\t" - "vmovups (%3,%0,4), %%ymm4 \n\t" // 8 * y + "vxorps %%ymm4 , %%ymm4 , %%ymm4 \n\t" + "vxorps %%ymm5 , %%ymm5 , %%ymm5 \n\t" + "vmovups (%3,%0,4), %%ymm7 \n\t" // 8 * y "vmulps (%4,%0,4), %%ymm12, %%ymm8 \n\t" "vmulps (%5,%0,4), %%ymm13, %%ymm10 \n\t" "vmulps (%6,%0,4), %%ymm14, %%ymm9 \n\t" "vmulps (%7,%0,4), %%ymm15, %%ymm11 \n\t" "vaddps %%ymm4, %%ymm8 , %%ymm4 \n\t" - "vaddps %%ymm4, %%ymm10, %%ymm4 \n\t" + "vaddps %%ymm5, %%ymm10, %%ymm5 \n\t" "vaddps %%ymm4, %%ymm9 , %%ymm4 \n\t" - "vaddps %%ymm4, %%ymm11, %%ymm4 \n\t" + "vaddps %%ymm5, %%ymm11, %%ymm5 \n\t" - "vmovups %%ymm4, (%3,%0,4) \n\t" // 8 * y + "vaddps %%ymm5, %%ymm4 , %%ymm4 \n\t" + "vmulps %%ymm6, %%ymm4 , %%ymm5 \n\t" + "vaddps %%ymm5, %%ymm7 , %%ymm5 \n\t" + + "vmovups %%ymm5, (%3,%0,4) \n\t" // 8 * y "addq $8, %0 \n\t" "subq $8, %1 \n\t" @@ -265,8 +301,10 @@ static void sgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) ".align 16 \n\t" ".L01LOOP%=: \n\t" - "vmovups (%3,%0,4), %%ymm4 \n\t" // 8 * y - "vmovups 32(%3,%0,4), %%ymm5 \n\t" // 8 * y + "vxorps %%ymm4 , %%ymm4 , %%ymm4 \n\t" + "vxorps %%ymm5 , %%ymm5 , %%ymm5 \n\t" + "vmovups (%3,%0,4), %%ymm0 \n\t" // 8 * y + "vmovups 32(%3,%0,4), %%ymm1 \n\t" // 8 * y "prefetcht0 192(%4,%0,4) \n\t" "vmulps (%4,%0,4), %%ymm12, %%ymm8 \n\t" @@ -290,8 +328,14 @@ static void sgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) "vaddps %%ymm4, %%ymm10, %%ymm4 \n\t" "vaddps %%ymm5, %%ymm11, %%ymm5 \n\t" - "vmovups %%ymm4, (%3,%0,4) \n\t" // 8 * y - "vmovups %%ymm5, 32(%3,%0,4) \n\t" // 8 * y + "vmulps %%ymm6, %%ymm4 , %%ymm4 \n\t" + "vmulps %%ymm6, %%ymm5 , %%ymm5 \n\t" + + "vaddps %%ymm4, %%ymm0 , %%ymm0 \n\t" + "vaddps %%ymm5, %%ymm1 , %%ymm1 \n\t" + + "vmovups %%ymm0, (%3,%0,4) \n\t" // 8 * y + "vmovups %%ymm1, 32(%3,%0,4) \n\t" // 8 * y "addq $16, %0 \n\t" "subq $16, %1 \n\t" @@ -309,8 +353,10 @@ static void sgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) "r" (ap[0]), // 4 "r" (ap[1]), // 5 "r" (ap[2]), // 6 - "r" (ap[3]) // 7 + "r" (ap[3]), // 7 + "r" (alpha) // 8 : "cc", + "%xmm0", "%xmm1", "%xmm4", "%xmm5", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15",