floraachy/OpenBLAS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added optimized zaxpy-kernels

Browse Source
This commit is contained in:
Werner Saar
2015-04-16 11:19:37 +02:00
parent 37b9033c90
commit 6d0db0151f
9 changed files with 666 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kernel/x86_64/zaxpy_microk_bulldozer-2.c
+122 -57
View File
@@ -31,89 +31,87 @@ static void zaxpy_kernel_4( BLASLONG n, FLOAT *x, FLOAT *y , FLOAT *alpha) __att
static void zaxpy_kernel_4( BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *alpha)
{
#if !defined(CONJ)
FLOAT mvec[2] = { -1.0, 1.0 };
#else
FLOAT mvec[2] = { 1.0, -1.0 };
#endif
BLASLONG register i = 0;
if ( n < 384 )
{
__asm__ __volatile__
(
"vzeroupper \n\t"
"vmovddup (%4), %%xmm0 \n\t" // real part of alpha
"vmovddup 8(%4), %%xmm1 \n\t" // imag part of alpha
#if !defined(CONJ)
"vmulpd (%5), %%xmm1 , %%xmm1 \n\t"
#else
"vmulpd (%5), %%xmm0 , %%xmm0 \n\t"
#endif
".align 16 \n\t"
"1: \n\t"
"prefetcht0 768(%2,%0,8) \n\t"
"vmovups (%2,%0,8), %%xmm5 \n\t" // 1 complex values from x
".align 2 \n\t"
"vmovups 16(%2,%0,8), %%xmm7 \n\t" // 1 complex values from x
"vmovups 32(%2,%0,8), %%xmm9 \n\t" // 1 complex values from x
"vmovups 48(%2,%0,8), %%xmm11 \n\t" // 1 complex values from x
"prefetcht0 768(%3,%0,8) \n\t"
#if !defined(CONJ)
"vfmaddpd (%3,%0,8), %%xmm0 , %%xmm5, %%xmm12 \n\t"
"vmovups 64(%2,%0,8), %%xmm12 \n\t" // 1 complex values from x
"vmovups 80(%2,%0,8), %%xmm13 \n\t" // 1 complex values from x
"vmovups 96(%2,%0,8), %%xmm14 \n\t" // 1 complex values from x
"vmovups 112(%2,%0,8), %%xmm15 \n\t" // 1 complex values from x
"vpermilpd $0x1 , %%xmm5 , %%xmm4 \n\t" // exchange real and imag part
"vmulpd %%xmm1, %%xmm4 , %%xmm4 \n\t"
"vfmaddpd 16(%3,%0,8), %%xmm0 , %%xmm7, %%xmm13 \n\t"
"vpermilpd $0x1 , %%xmm7 , %%xmm6 \n\t" // exchange real and imag part
"vmulpd %%xmm1, %%xmm6 , %%xmm6 \n\t"
"vfmaddpd 32(%3,%0,8), %%xmm0 , %%xmm9, %%xmm14 \n\t"
"vpermilpd $0x1 , %%xmm9 , %%xmm8 \n\t" // exchange real and imag part
"vmulpd %%xmm1, %%xmm8 , %%xmm8 \n\t"
"vfmaddpd 48(%3,%0,8), %%xmm0 , %%xmm11,%%xmm15 \n\t"
"vpermilpd $0x1 , %%xmm11, %%xmm10 \n\t" // exchange real and imag part
"vmulpd %%xmm1, %%xmm10, %%xmm10 \n\t"
"vaddsubpd %%xmm4, %%xmm12, %%xmm12 \n\t"
"vaddsubpd %%xmm6, %%xmm13, %%xmm13 \n\t"
"vaddsubpd %%xmm8, %%xmm14, %%xmm14 \n\t"
"vaddsubpd %%xmm10,%%xmm15, %%xmm15 \n\t"
"vfmaddpd (%3,%0,8), %%xmm0 , %%xmm5, %%xmm5 \n\t"
".align 2 \n\t"
"vfmaddpd 16(%3,%0,8), %%xmm0 , %%xmm7, %%xmm7 \n\t"
"vfmaddpd 32(%3,%0,8), %%xmm0 , %%xmm9, %%xmm9 \n\t"
"vfmaddpd 48(%3,%0,8), %%xmm0 , %%xmm11,%%xmm11 \n\t"
#else
"vfmaddpd %%xmm5 , %%xmm1 , %%xmm4 , %%xmm5 \n\t"
"vfmaddpd %%xmm7 , %%xmm1 , %%xmm6 , %%xmm7 \n\t"
"vfmaddpd %%xmm9 , %%xmm1 , %%xmm8 , %%xmm9 \n\t"
"vfmaddpd %%xmm11, %%xmm1 , %%xmm10, %%xmm11 \n\t"
"vmulpd %%xmm0, %%xmm5, %%xmm4 \n\t" // a_r*x_r, a_r*x_i
"vmulpd %%xmm1, %%xmm5, %%xmm5 \n\t" // a_i*x_r, a_i*x_i
"vmulpd %%xmm0, %%xmm7, %%xmm6 \n\t" // a_r*x_r, a_r*x_i
"vmulpd %%xmm1, %%xmm7, %%xmm7 \n\t" // a_i*x_r, a_i*x_i
"vmulpd %%xmm0, %%xmm9, %%xmm8 \n\t" // a_r*x_r, a_r*x_i
"vmulpd %%xmm1, %%xmm9, %%xmm9 \n\t" // a_i*x_r, a_i*x_i
"vmulpd %%xmm0, %%xmm11, %%xmm10 \n\t" // a_r*x_r, a_r*x_i
"vmulpd %%xmm1, %%xmm11, %%xmm11 \n\t" // a_i*x_r, a_i*x_i
"vpermilpd $0x1 , %%xmm12, %%xmm4 \n\t" // exchange real and imag part
"vpermilpd $0x1 , %%xmm13, %%xmm6 \n\t" // exchange real and imag part
"vpermilpd $0x1 , %%xmm14, %%xmm8 \n\t" // exchange real and imag part
"vpermilpd $0x1 , %%xmm15, %%xmm10 \n\t" // exchange real and imag part
"vpermilpd $0x1 , %%xmm4 , %%xmm4 \n\t" // exchange real and imag part
"vaddsubpd %%xmm4 ,%%xmm5 , %%xmm4 \n\t"
"vpermilpd $0x1 , %%xmm4 , %%xmm4 \n\t" // exchange real and imag part
"vfmaddpd 64(%3,%0,8), %%xmm0 , %%xmm12, %%xmm12 \n\t"
"vfmaddpd 80(%3,%0,8), %%xmm0 , %%xmm13, %%xmm13 \n\t"
"vfmaddpd 96(%3,%0,8), %%xmm0 , %%xmm14, %%xmm14 \n\t"
"vfmaddpd 112(%3,%0,8), %%xmm0 , %%xmm15, %%xmm15 \n\t"
"vpermilpd $0x1 , %%xmm6 , %%xmm6 \n\t" // exchange real and imag part
"vaddsubpd %%xmm6 ,%%xmm7 , %%xmm6 \n\t"
"vpermilpd $0x1 , %%xmm6 , %%xmm6 \n\t" // exchange real and imag part
"vfmaddpd %%xmm12, %%xmm1 , %%xmm4 , %%xmm12 \n\t"
"vfmaddpd %%xmm13, %%xmm1 , %%xmm6 , %%xmm13 \n\t"
"vfmaddpd %%xmm14, %%xmm1 , %%xmm8 , %%xmm14 \n\t"
"vfmaddpd %%xmm15, %%xmm1 , %%xmm10, %%xmm15 \n\t"
"vpermilpd $0x1 , %%xmm8 , %%xmm8 \n\t" // exchange real and imag part
"vaddsubpd %%xmm8 ,%%xmm9 , %%xmm8 \n\t"
"vpermilpd $0x1 , %%xmm8 , %%xmm8 \n\t" // exchange real and imag part
"vmovups %%xmm5 , (%3,%0,8) \n\t"
".align 2 \n\t"
"vmovups %%xmm7 , 16(%3,%0,8) \n\t"
"vmovups %%xmm9 , 32(%3,%0,8) \n\t"
"vmovups %%xmm11, 48(%3,%0,8) \n\t"
"vmovups %%xmm12, 64(%3,%0,8) \n\t"
"vmovups %%xmm13, 80(%3,%0,8) \n\t"
"vmovups %%xmm14, 96(%3,%0,8) \n\t"
"vmovups %%xmm15,112(%3,%0,8) \n\t"
"vpermilpd $0x1 , %%xmm10, %%xmm10 \n\t" // exchange real and imag part
"vaddsubpd %%xmm10,%%xmm11, %%xmm10 \n\t"
"vpermilpd $0x1 , %%xmm10, %%xmm10 \n\t" // exchange real and imag part
"vaddpd (%3,%0,8) ,%%xmm4 , %%xmm12 \n\t"
"vaddpd 16(%3,%0,8) ,%%xmm6 , %%xmm13 \n\t"
"vaddpd 32(%3,%0,8) ,%%xmm8 , %%xmm14 \n\t"
"vaddpd 48(%3,%0,8) ,%%xmm10, %%xmm15 \n\t"
#endif
"vmovups %%xmm12, (%3,%0,8) \n\t"
"vmovups %%xmm13, 16(%3,%0,8) \n\t"
"vmovups %%xmm14, 32(%3,%0,8) \n\t"
"vmovups %%xmm15, 48(%3,%0,8) \n\t"
"addq $8 , %0 \n\t"
"subq $4 , %1 \n\t"
"addq $16, %0 \n\t"
"subq $8 , %1 \n\t"
"jnz 1b \n\t"
"vzeroupper \n\t"
:
:
@@ -121,7 +119,8 @@ static void zaxpy_kernel_4( BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *alpha)
"r" (n), // 1
"r" (x), // 2
"r" (y), // 3
"r" (alpha) // 4
"r" (alpha), // 4
"r" (mvec) // 5
: "cc",
"%xmm0", "%xmm1",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
@@ -129,7 +128,73 @@ static void zaxpy_kernel_4( BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *alpha)
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
return;
}
__asm__ __volatile__
(
"vzeroupper \n\t"
"vmovddup (%4), %%xmm0 \n\t" // real part of alpha
"vmovddup 8(%4), %%xmm1 \n\t" // imag part of alpha
#if !defined(CONJ)
"vmulpd (%5), %%xmm1 , %%xmm1 \n\t"
#else
"vmulpd (%5), %%xmm0 , %%xmm0 \n\t"
#endif
".align 16 \n\t"
"1: \n\t"
"prefetcht0 512(%2,%0,8) \n\t"
"vmovups (%2,%0,8), %%xmm5 \n\t" // 1 complex values from x
".align 2 \n\t"
"vmovups 16(%2,%0,8), %%xmm7 \n\t" // 1 complex values from x
"vmovups 32(%2,%0,8), %%xmm9 \n\t" // 1 complex values from x
"vmovups 48(%2,%0,8), %%xmm11 \n\t" // 1 complex values from x
"vpermilpd $0x1 , %%xmm5 , %%xmm4 \n\t" // exchange real and imag part
"vpermilpd $0x1 , %%xmm7 , %%xmm6 \n\t" // exchange real and imag part
"vpermilpd $0x1 , %%xmm9 , %%xmm8 \n\t" // exchange real and imag part
"vpermilpd $0x1 , %%xmm11, %%xmm10 \n\t" // exchange real and imag part
"prefetcht0 512(%3,%0,8) \n\t"
"vfmaddpd (%3,%0,8), %%xmm0 , %%xmm5, %%xmm5 \n\t"
".align 2 \n\t"
"vfmaddpd 16(%3,%0,8), %%xmm0 , %%xmm7, %%xmm7 \n\t"
"vfmaddpd 32(%3,%0,8), %%xmm0 , %%xmm9, %%xmm9 \n\t"
"vfmaddpd 48(%3,%0,8), %%xmm0 , %%xmm11,%%xmm11 \n\t"
"vfmaddpd %%xmm5 , %%xmm1 , %%xmm4 , %%xmm5 \n\t"
"vfmaddpd %%xmm7 , %%xmm1 , %%xmm6 , %%xmm7 \n\t"
"vfmaddpd %%xmm9 , %%xmm1 , %%xmm8 , %%xmm9 \n\t"
"vfmaddpd %%xmm11, %%xmm1 , %%xmm10, %%xmm11 \n\t"
"vmovups %%xmm5 , (%3,%0,8) \n\t"
".align 2 \n\t"
"vmovups %%xmm7 , 16(%3,%0,8) \n\t"
"vmovups %%xmm9 , 32(%3,%0,8) \n\t"
"vmovups %%xmm11, 48(%3,%0,8) \n\t"
"addq $8 , %0 \n\t"
"subq $4, %1 \n\t"
"jnz 1b \n\t"
"vzeroupper \n\t"
:
:
"r" (i), // 0
"r" (n), // 1
"r" (x), // 2
"r" (y), // 3
"r" (alpha), // 4
"r" (mvec) // 5
: "cc",
"%xmm0", "%xmm1",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"memory"
);
}