From 74c9465672f8df5da6ca47cc2c1dd5dedb921e2b Mon Sep 17 00:00:00 2001
From: wernsaar <wernsaar@googlemail.com>
Date: Mon, 18 Aug 2014 12:18:10 +0200
Subject: [PATCH] added optimized dsymv_U kernel for bulldozer

---
 kernel/x86_64/KERNEL.BULLDOZER             |   2 +
 kernel/x86_64/dsymv_U.c                    | 208 +++++++++++++++++++++
 kernel/x86_64/dsymv_U_microk_bulldozer-2.c | 117 ++++++++++++
 3 files changed, 327 insertions(+)
 create mode 100644 kernel/x86_64/dsymv_U.c
 create mode 100644 kernel/x86_64/dsymv_U_microk_bulldozer-2.c

diff --git a/kernel/x86_64/KERNEL.BULLDOZER b/kernel/x86_64/KERNEL.BULLDOZER
index 19bf7fd32..03925cc19 100644
--- a/kernel/x86_64/KERNEL.BULLDOZER
+++ b/kernel/x86_64/KERNEL.BULLDOZER
@@ -1,3 +1,5 @@
+DSYMV_U_KERNEL = dsymv_U.c
+
 SGEMVNKERNEL = sgemv_n.c
 SGEMVTKERNEL = sgemv_t.c
 
diff --git a/kernel/x86_64/dsymv_U.c b/kernel/x86_64/dsymv_U.c
new file mode 100644
index 000000000..1f22abe8d
--- /dev/null
+++ b/kernel/x86_64/dsymv_U.c
@@ -0,0 +1,208 @@
+/***************************************************************************
+Copyright (c) 2013, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+
+#include "common.h"
+
+#if defined(BULLDOZER)
+#include "dsymv_U_microk_bulldozer-2.c"
+#endif
+
+
+#ifndef HAVE_KERNEL_8x2
+
+static void dsymv_kernel_8x2(BLASLONG n, FLOAT *a0, FLOAT *a1, FLOAT *xp, FLOAT *yp, FLOAT *temp1, FLOAT *temp2)
+{
+	FLOAT at0,at1,at2,at3;
+	FLOAT tmp2[2] = { 0.0, 0.0 };
+	FLOAT tp0;
+	FLOAT tp1;
+	BLASLONG i;
+
+	tp0 = temp1[0];
+	tp1 = temp1[1];
+	
+	for (i=0; i<n; i+=2)
+	{
+		at0     = a0[i];
+		at1     = a1[i];
+		at2     = a0[i+1];
+		at3     = a1[i+1];
+		yp[i]   += tp0 * at0 + tp1 *at1;
+		yp[i+1] += tp0 * at2 + tp1 *at3;
+		tmp2[0] += at0 * xp[i] + at2 * xp[i+1];
+		tmp2[1] += at1 * xp[i] + at3 * xp[i+1];
+
+	}
+	temp2[0] = tmp2[0];
+	temp2[1] = tmp2[1];
+
+}
+
+#endif
+
+static void dsymv_kernel_8x1(BLASLONG n, FLOAT *a0, FLOAT *xp, FLOAT *yp, FLOAT *temp1, FLOAT *temp2)
+{
+	FLOAT at0,at1,at2,at3;
+	FLOAT temp = 0.0;
+	FLOAT t1   = *temp1;
+	BLASLONG i;
+	
+	for (i=0; i<(n/4)*4; i+=4)
+	{
+		at0     = a0[i];
+		at1     = a0[i+1];
+		at2     = a0[i+2];
+		at3     = a0[i+3];
+
+		yp[i]   += t1    * at0;
+		temp    += at0   * xp[i];
+		yp[i+1] += t1    * at1;
+		temp    += at1   * xp[i+1];
+
+		yp[i+2] += t1    * at2;
+		temp    += at2   * xp[i+2];
+		yp[i+3] += t1    * at3;
+		temp    += at3   * xp[i+3];
+
+	}
+	*temp2 = temp;
+}
+
+int CNAME(BLASLONG m, BLASLONG offset, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer)
+{
+	BLASLONG i;
+	BLASLONG ix,iy;
+	BLASLONG jx,jy;
+	BLASLONG j;
+	BLASLONG m2;
+	FLOAT temp1;
+	FLOAT temp2;
+	FLOAT *xp, *yp;
+	FLOAT *a0,*a1;
+	FLOAT tmp1[2];
+	FLOAT tmp2[2];
+
+#if 0
+	if( m != offset )
+		printf("Symv_U: m=%d offset=%d\n",m,offset);
+#endif
+
+	BLASLONG m1 = m - offset;
+	BLASLONG mrange = m -m1;
+
+	if ( (inc_x!=1) || (inc_y!=1) || (mrange<16) )
+	{
+
+		jx = m1 * inc_x;
+		jy = m1 * inc_y;
+
+		for (j=m1; j<m; j++)
+		{
+			temp1 = alpha * x[jx];
+			temp2 = 0.0;
+			iy = 0;
+			ix = 0;
+			for (i=0; i<j; i++)
+			{
+				y[iy] += temp1 * a[j*lda+i];
+				temp2 += a[j*lda+i] * x[ix];
+				ix += inc_x;
+				iy += inc_y;
+			
+			}
+			y[jy] += temp1 * a[j*lda+j] + alpha * temp2;
+			jx    += inc_x;
+			jy    += inc_y;
+		}
+		return(0);
+	}
+
+	xp = x;
+	yp = y;
+
+	m2 = m - ( mrange % 2 );
+
+	for (j=m1; j<m2; j+=2)
+	{
+		tmp1[0] = alpha * xp[j];
+		tmp1[1] = alpha * xp[j+1];
+		tmp2[0] = 0.0;
+		tmp2[1] = 0.0;
+		a0    = &a[j*lda];
+		a1    = a0+lda;
+		FLOAT at0,at1;
+		BLASLONG j1 = (j/8)*8;		
+		if ( j1 )
+			dsymv_kernel_8x2(j1, a0, a1, xp, yp, tmp1, tmp2);
+
+		for (i=j1; i<j; i++)
+		{
+			at0     = a0[i];
+			at1     = a1[i];
+			yp[i]   += tmp1[0] * at0 + tmp1[1] *at1;
+			tmp2[0] += at0 * xp[i];
+			tmp2[1] += at1 * xp[i];
+			
+		}
+
+		at1     = a1[j];
+		yp[j]   += tmp1[1] * at1;
+		tmp2[1] += at1 * xp[j];
+
+		yp[j]   += tmp1[0] * a0[j]   + alpha * tmp2[0];
+		yp[j+1] += tmp1[1] * a1[j+1] + alpha * tmp2[1];
+	}
+
+	for ( ; j<m; j++)
+	{
+		temp1 = alpha * xp[j];
+		temp2 = 0.0;
+		a0    = &a[j*lda];
+		FLOAT at0;
+		BLASLONG j1 = (j/8)*8;		
+
+		if ( j1 )
+			dsymv_kernel_8x1(j1, a0, xp, yp, &temp1, &temp2);
+
+		for (i=j1 ; i<j; i++)
+		{
+			at0     = a0[i];
+			yp[i] += temp1 * at0;
+			temp2 += at0 * xp[i];
+			
+		}
+
+		yp[j] += temp1 * a0[j] + alpha * temp2;
+	}
+
+	return(0);
+	
+
+}
+
+
diff --git a/kernel/x86_64/dsymv_U_microk_bulldozer-2.c b/kernel/x86_64/dsymv_U_microk_bulldozer-2.c
new file mode 100644
index 000000000..3b035222b
--- /dev/null
+++ b/kernel/x86_64/dsymv_U_microk_bulldozer-2.c
@@ -0,0 +1,117 @@
+/***************************************************************************
+Copyright (c) 2014, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#define HAVE_KERNEL_8x2 1
+static void dsymv_kernel_8x2( BLASLONG n, FLOAT *a1, FLOAT *a2, FLOAT *x, FLOAT *y, FLOAT *temp1, FLOAT *temp2) __attribute__ ((noinline));
+
+static void dsymv_kernel_8x2(BLASLONG n, FLOAT *a0, FLOAT *a1, FLOAT *x, FLOAT *y, FLOAT *temp1, FLOAT *temp2)
+{
+
+	BLASLONG register i = 0;
+
+	__asm__  __volatile__
+	(
+	"vxorpd		%%xmm0 , %%xmm0 , %%xmm0     \n\t"	// temp2[0]
+	"vxorpd		%%xmm1 , %%xmm1 , %%xmm1     \n\t"	// temp2[1]
+	"vmovddup	(%6),    %%xmm2	             \n\t"	// temp1[0]
+	"vmovddup      8(%6),    %%xmm3	             \n\t"	// temp1[1]
+
+	"xorq		%0,%0			     \n\t"
+
+	".align 16				 \n\t"
+	".L01LOOP%=:				 \n\t"
+
+	"prefetcht0	 192(%4,%0,8)		       \n\t"
+	"vmovups	(%4,%0,8), %%xmm4	 \n\t"	// 2 * a0
+	"vmovups      16(%4,%0,8), %%xmm5	 \n\t"	// 2 * a0
+	"prefetcht0	 192(%2,%0,8)		       \n\t"
+	"vmovups	(%2,%0,8), %%xmm8	 \n\t"	// 2 * x
+	"vmovups      16(%2,%0,8), %%xmm9	 \n\t"	// 2 * x
+	"prefetcht0	 192(%3,%0,8)		       \n\t"
+	"vmovups      32(%4,%0,8), %%xmm6	 \n\t"	// 2 * a0
+	"vmovups      48(%4,%0,8), %%xmm7	 \n\t"	// 2 * a0
+	"vmovups      32(%2,%0,8), %%xmm10	 \n\t"	// 2 * x
+	"vmovups      48(%2,%0,8), %%xmm11	 \n\t"	// 2 * x
+
+	"prefetcht0	 192(%5,%0,8)		            \n\t"
+	"vfmaddpd    (%3,%0,8), %%xmm2 , %%xmm4 , %%xmm12   \n\t" // y += temp1 * a0
+	"vfmaddpd      %%xmm0 , %%xmm8 , %%xmm4 , %%xmm0    \n\t" // temp2 += a0 * x
+	"vfmaddpd  16(%3,%0,8), %%xmm2 , %%xmm5 , %%xmm13   \n\t" // y += temp1 * a0
+	"vmovups	(%5,%0,8), %%xmm4	            \n\t"	// 2 * a1
+	"vfmaddpd      %%xmm0 , %%xmm9 , %%xmm5 , %%xmm0    \n\t" // temp2 += a0 * x
+	"vfmaddpd  32(%3,%0,8), %%xmm2 , %%xmm6 , %%xmm14   \n\t" // y += temp1 * a0
+	"vmovups      16(%5,%0,8), %%xmm5	            \n\t"	// 2 * a1
+	"vfmaddpd      %%xmm0 , %%xmm10, %%xmm6 , %%xmm0    \n\t" // temp2 += a0 * x
+	"vfmaddpd  48(%3,%0,8), %%xmm2 , %%xmm7 , %%xmm15   \n\t" // y += temp1 * a0
+	"vmovups      32(%5,%0,8), %%xmm6	            \n\t"	// 2 * a1
+	"vfmaddpd      %%xmm0 , %%xmm11, %%xmm7 , %%xmm0    \n\t" // temp2 += a0 * x
+	"vmovups      48(%5,%0,8), %%xmm7	            \n\t"	// 2 * a1
+
+	"vfmaddpd %%xmm12, %%xmm3 , %%xmm4 , %%xmm12   \n\t" // y += temp1 * a1
+	"vfmaddpd %%xmm13, %%xmm3 , %%xmm5 , %%xmm13   \n\t" // y += temp1 * a1
+	"vmovups  %%xmm12,   (%3,%0,8)		       \n\t"	// 2 * y
+	"vfmaddpd %%xmm14, %%xmm3 , %%xmm6 , %%xmm14   \n\t" // y += temp1 * a1
+	"vmovups  %%xmm13, 16(%3,%0,8)		       \n\t"	// 2 * y
+	"vfmaddpd %%xmm15, %%xmm3 , %%xmm7 , %%xmm15   \n\t" // y += temp1 * a1
+	"vmovups  %%xmm14, 32(%3,%0,8)		       \n\t"	// 2 * y
+
+	"vfmaddpd %%xmm1 , %%xmm8 , %%xmm4 , %%xmm1   \n\t" // temp2 += a1 * x
+	"vfmaddpd %%xmm1 , %%xmm9 , %%xmm5 , %%xmm1   \n\t" // temp2 += a1 * x
+	"vmovups  %%xmm15, 48(%3,%0,8)		      \n\t"	// 2 * y
+	"vfmaddpd %%xmm1 , %%xmm10, %%xmm6 , %%xmm1   \n\t" // temp2 += a1 * x
+	"vfmaddpd %%xmm1 , %%xmm11, %%xmm7 , %%xmm1   \n\t" // temp2 += a1 * x
+
+        "addq		$8, %0	  	 	      \n\t"
+	"subq	        $8, %1			      \n\t"		
+	"jnz		.L01LOOP%=		      \n\t"
+
+	"vhaddpd        %%xmm0, %%xmm0, %%xmm0  \n\t"
+	"vhaddpd        %%xmm1, %%xmm1, %%xmm1  \n\t"
+	"vmovsd         %%xmm0 , (%7)		\n\t"	// save temp2
+	"vmovsd         %%xmm1 ,8(%7)		\n\t"	// save temp2
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (n),  	// 1
+          "r" (x),      // 2
+          "r" (y),      // 3
+          "r" (a0),  // 4
+          "r" (a1),  // 5
+          "r" (temp1),  // 6
+          "r" (temp2)   // 7
+	: "cc", 
+	  "%xmm0", "%xmm1", "%xmm2", "%xmm3", 
+	  "%xmm4", "%xmm5", "%xmm6", "%xmm7", 
+	  "%xmm8", "%xmm9", "%xmm10", "%xmm11", 
+	  "%xmm12", "%xmm13", "%xmm14", "%xmm15",
+	  "memory"
+	);
+
+} 
+
+