diff --git a/Makefile.system b/Makefile.system index 727b08960..9663322fa 100644 --- a/Makefile.system +++ b/Makefile.system @@ -324,16 +324,14 @@ ifeq ($(ARCH), x86) DYNAMIC_CORE = KATMAI COPPERMINE NORTHWOOD PRESCOTT BANIAS \ CORE2 PENRYN DUNNINGTON NEHALEM ATHLON OPTERON OPTERON_SSE3 BARCELONA BOBCAT ATOM NANO ifneq ($(NO_AVX), 1) -DYNAMIC_CORE += SANDYBRIDGE -#BULLDOZER PILEDRIVER +DYNAMIC_CORE += SANDYBRIDGE BULLDOZER PILEDRIVER endif endif ifeq ($(ARCH), x86_64) DYNAMIC_CORE = PRESCOTT CORE2 PENRYN DUNNINGTON NEHALEM OPTERON OPTERON_SSE3 BARCELONA BOBCAT ATOM NANO ifneq ($(NO_AVX), 1) -DYNAMIC_CORE += SANDYBRIDGE -#BULLDOZER PILEDRIVER +DYNAMIC_CORE += SANDYBRIDGE BULLDOZER PILEDRIVER endif endif diff --git a/cpuid.h b/cpuid.h index 4311ce95e..2cbbd4539 100644 --- a/cpuid.h +++ b/cpuid.h @@ -105,8 +105,8 @@ #define CORE_NANO 19 #define CORE_SANDYBRIDGE 20 #define CORE_BOBCAT 21 -#define CORE_BULLDOZER CORE_BARCELONA -#define CORE_PILEDRIVER CORE_BARCELONA +#define CORE_BULLDOZER 22 +#define CORE_PILEDRIVER 23 #define CORE_HASWELL CORE_SANDYBRIDGE #define HAVE_SSE (1 << 0) @@ -198,8 +198,8 @@ typedef struct { #define CPUTYPE_NANO 43 #define CPUTYPE_SANDYBRIDGE 44 #define CPUTYPE_BOBCAT 45 -#define CPUTYPE_BULLDOZER CPUTYPE_BARCELONA -#define CPUTYPE_PILEDRIVER CPUTYPE_BARCELONA +#define CPUTYPE_BULLDOZER 46 +#define CPUTYPE_PILEDRIVER 47 // this define is because BLAS doesn't have haswell specific optimizations yet #define CPUTYPE_HASWELL CPUTYPE_SANDYBRIDGE diff --git a/driver/others/dynamic.c b/driver/others/dynamic.c index bf60efb19..197cc2b2d 100644 --- a/driver/others/dynamic.c +++ b/driver/others/dynamic.c @@ -63,16 +63,14 @@ extern gotoblas_t gotoblas_BARCELONA; extern gotoblas_t gotoblas_BOBCAT; #ifndef NO_AVX extern gotoblas_t gotoblas_SANDYBRIDGE; -//extern gotoblas_t gotoblas_BULLDOZER; -//extern gotoblas_t gotoblas_PILEDRIVER; +extern gotoblas_t gotoblas_BULLDOZER; +extern gotoblas_t gotoblas_PILEDRIVER; #else //Use NEHALEM kernels for sandy bridge #define gotoblas_SANDYBRIDGE gotoblas_NEHALEM -#endif - #define gotoblas_BULLDOZER gotoblas_BARCELONA #define gotoblas_PILEDRIVER gotoblas_BARCELONA - +#endif //Use sandy bridge kernels for haswell. #define gotoblas_HASWELL gotoblas_SANDYBRIDGE diff --git a/driver/others/init.c b/driver/others/init.c index 5da71cec1..cbcf229fa 100644 --- a/driver/others/init.c +++ b/driver/others/init.c @@ -494,7 +494,7 @@ static void disable_affinity(void) { #ifndef USE_OPENMP for(i=0; i< count; i++){ - lprocmask[i] &= ((unsigned long *)&cpu_orig_mask[0])[i]; + lprocmask[i] &= common->avail[i]; } #endif @@ -754,7 +754,7 @@ void gotoblas_affinity_init(void) { if (common -> num_nodes > 1) numa_mapping(); common -> final_num_procs = 0; - for(i = 0; i < common -> avail_count; i++) common -> final_num_procs += popcount(common -> avail[i]); + for(i = 0; i < common -> avail_count; i++) common -> final_num_procs += rcount(common -> avail[i]) + 1; //Make the max cpu number. for (cpu = 0; cpu < common -> final_num_procs; cpu ++) common -> cpu_use[cpu] = 0; diff --git a/getarch.c b/getarch.c index ec9bb75a8..3ffda6244 100644 --- a/getarch.c +++ b/getarch.c @@ -354,7 +354,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define CORENAME "OPTERON" #endif -#if defined(FORCE_BARCELONA) || defined(FORCE_SHANGHAI) || defined(FORCE_ISTANBUL) || defined (FORCE_PILEDRIVER) || defined (FORCE_BULLDOZER) +#if defined(FORCE_BARCELONA) || defined(FORCE_SHANGHAI) || defined(FORCE_ISTANBUL) #define FORCE #define FORCE_INTEL #define ARCHITECTURE "X86" @@ -384,7 +384,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define CORENAME "BOBCAT" #endif -#if 0 +#if defined (FORCE_BULLDOZER) #define FORCE #define FORCE_INTEL #define ARCHITECTURE "X86" @@ -400,7 +400,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define CORENAME "BULLDOZER" #endif -#if 0 +#if defined (FORCE_PILEDRIVER) #define FORCE #define FORCE_INTEL #define ARCHITECTURE "X86" diff --git a/kernel/Makefile.L3 b/kernel/Makefile.L3 index 2dcae0b9d..f8152ac50 100644 --- a/kernel/Makefile.L3 +++ b/kernel/Makefile.L3 @@ -582,6 +582,24 @@ $(KDIR)ztrmm_kernel_RR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMKERNEL) $(KDIR)ztrmm_kernel_RC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMKERNEL) $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -DTRANSA -DCONJ -DNC $< -o $@ + +else + +ifdef STRMMKERNEL + +$(KDIR)strmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(STRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -DLEFT -UTRANSA $< -o $@ + +$(KDIR)strmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(STRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -DLEFT -DTRANSA $< -o $@ + +$(KDIR)strmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(STRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -UTRANSA $< -o $@ + +$(KDIR)strmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(STRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o $@ + + else $(KDIR)strmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -DLEFT -UTRANSA $< -o $@ @@ -595,17 +613,79 @@ $(KDIR)strmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) $(KDIR)strmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o $@ +endif + +ifdef DTRMMKERNEL + +ifdef DTRMMKERNEL_LN +$(KDIR)dtrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DTRMMKERNEL_LN) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -DLEFT -UTRANSA $< -o $@ +else +$(KDIR)dtrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -DLEFT -UTRANSA $< -o $@ +endif + +ifdef DTRMMKERNEL_LT +$(KDIR)dtrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DTRMMKERNEL_LT) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -DLEFT -DTRANSA $< -o $@ +else +$(KDIR)dtrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -DLEFT -DTRANSA $< -o $@ +endif + +ifdef DTRMMKERNEL_RN +$(KDIR)dtrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DTRMMKERNEL_RN) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -ULEFT -UTRANSA $< -o $@ +else +$(KDIR)dtrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -ULEFT -UTRANSA $< -o $@ +endif + +ifdef DTRMMKERNEL_RT +$(KDIR)dtrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DTRMMKERNEL_RT) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o $@ +else +$(KDIR)dtrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o $@ +endif + +else + +ifdef DTRMMKERNEL_LN +$(KDIR)dtrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL_LN) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -DLEFT -UTRANSA $< -o $@ +else $(KDIR)dtrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -DLEFT -UTRANSA $< -o $@ +endif +ifdef DTRMMKERNEL_LT +$(KDIR)dtrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL_LT) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -DLEFT -DTRANSA $< -o $@ +else $(KDIR)dtrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -DLEFT -DTRANSA $< -o $@ +endif +ifdef DTRMMKERNEL_RN +$(KDIR)dtrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL_RN) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -ULEFT -UTRANSA $< -o $@ +else $(KDIR)dtrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -ULEFT -UTRANSA $< -o $@ +endif +ifdef DTRMMKERNEL_RT +$(KDIR)dtrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL_RT) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o $@ +else $(KDIR)dtrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o $@ +endif + +endif + +ifdef QTRMMKERNEL $(KDIR)qtrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(QGEMMKERNEL) $(CC) $(CFLAGS) -c -DTRMMKERNEL -DXDOUBLE -UCOMPLEX -DLEFT -UTRANSA $< -o $@ @@ -619,6 +699,50 @@ $(KDIR)qtrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(QGEMMKERNEL) $(KDIR)qtrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(QGEMMKERNEL) $(CC) $(CFLAGS) -c -DTRMMKERNEL -DXDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o $@ +else + +$(KDIR)qtrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(QGEMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DXDOUBLE -UCOMPLEX -DLEFT -UTRANSA $< -o $@ + +$(KDIR)qtrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(QGEMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DXDOUBLE -UCOMPLEX -DLEFT -DTRANSA $< -o $@ + +$(KDIR)qtrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(QGEMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DXDOUBLE -UCOMPLEX -ULEFT -UTRANSA $< -o $@ + +$(KDIR)qtrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(QGEMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DXDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o $@ + +endif + +ifdef CTRMMKERNEL + +$(KDIR)ctrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -UTRANSA -UCONJ -DNN $< -o $@ + +$(KDIR)ctrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -DTRANSA -UCONJ -DNN $< -o $@ + +$(KDIR)ctrmm_kernel_LR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -UTRANSA -DCONJ -DCN $< -o $@ + +$(KDIR)ctrmm_kernel_LC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -DTRANSA -DCONJ -DCN $< -o $@ + +$(KDIR)ctrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -UTRANSA -UCONJ -DNN $< -o $@ + +$(KDIR)ctrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -DTRANSA -UCONJ -DNN $< -o $@ + +$(KDIR)ctrmm_kernel_RR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -UTRANSA -DCONJ -DNC $< -o $@ + +$(KDIR)ctrmm_kernel_RC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -DTRANSA -DCONJ -DNC $< -o $@ + +else + $(KDIR)ctrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -UTRANSA -UCONJ -DNN $< -o $@ @@ -643,6 +767,37 @@ $(KDIR)ctrmm_kernel_RR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(KDIR)ctrmm_kernel_RC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -DTRANSA -DCONJ -DNC $< -o $@ +endif + +ifdef ZTRMMKERNEL + +$(KDIR)ztrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -UTRANSA -UCONJ -DNN $< -o $@ + +$(KDIR)ztrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -DTRANSA -UCONJ -DNN $< -o $@ + +$(KDIR)ztrmm_kernel_LR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -UTRANSA -DCONJ -DCN $< -o $@ + +$(KDIR)ztrmm_kernel_LC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -DTRANSA -DCONJ -DCN $< -o $@ + +$(KDIR)ztrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -UTRANSA -UCONJ -DNN $< -o $@ + +$(KDIR)ztrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -DTRANSA -UCONJ -DNN $< -o $@ + +$(KDIR)ztrmm_kernel_RR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -UTRANSA -DCONJ -DNC $< -o $@ + +$(KDIR)ztrmm_kernel_RC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -DTRANSA -DCONJ -DNC $< -o $@ + + +else + $(KDIR)ztrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -UTRANSA -UCONJ -DNN $< -o $@ @@ -666,7 +821,37 @@ $(KDIR)ztrmm_kernel_RR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(KDIR)ztrmm_kernel_RC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -DTRANSA -DCONJ -DNC $< -o $@ + endif +endif + +ifdef XTRMMKERNEL + +$(KDIR)xtrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(XTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DXDOUBLE -DCOMPLEX -DLEFT -UTRANSA -UCONJ -DNN $< -o $@ + +$(KDIR)xtrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(XTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DXDOUBLE -DCOMPLEX -DLEFT -DTRANSA -UCONJ -DNN $< -o $@ + +$(KDIR)xtrmm_kernel_LR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(XTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DXDOUBLE -DCOMPLEX -DLEFT -UTRANSA -DCONJ -DCN $< -o $@ + +$(KDIR)xtrmm_kernel_LC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(XTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DXDOUBLE -DCOMPLEX -DLEFT -DTRANSA -DCONJ -DCN $< -o $@ + +$(KDIR)xtrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(XTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DXDOUBLE -DCOMPLEX -ULEFT -UTRANSA -UCONJ -DNN $< -o $@ + +$(KDIR)xtrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(XTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DXDOUBLE -DCOMPLEX -ULEFT -DTRANSA -UCONJ -DNN $< -o $@ + +$(KDIR)xtrmm_kernel_RR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(XTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DXDOUBLE -DCOMPLEX -ULEFT -UTRANSA -DCONJ -DNC $< -o $@ + +$(KDIR)xtrmm_kernel_RC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(XTRMMKERNEL) + $(CC) $(CFLAGS) -c -DTRMMKERNEL -DXDOUBLE -DCOMPLEX -ULEFT -DTRANSA -DCONJ -DNC $< -o $@ + +else $(KDIR)xtrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(XGEMMKERNEL) $(CC) $(CFLAGS) -c -DTRMMKERNEL -DXDOUBLE -DCOMPLEX -DLEFT -UTRANSA -UCONJ -DNN $< -o $@ @@ -692,6 +877,9 @@ $(KDIR)xtrmm_kernel_RR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(XGEMMKERNEL) $(KDIR)xtrmm_kernel_RC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(XGEMMKERNEL) $(CC) $(CFLAGS) -c -DTRMMKERNEL -DXDOUBLE -DCOMPLEX -ULEFT -DTRANSA -DCONJ -DNC $< -o $@ +endif + + $(KDIR)cgemm3m_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM3MKERNEL) $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNN $< -o $@ diff --git a/kernel/generic/trmmkernel_16x2.c b/kernel/generic/trmmkernel_16x2.c new file mode 100644 index 000000000..437fa0950 --- /dev/null +++ b/kernel/generic/trmmkernel_16x2.c @@ -0,0 +1,1151 @@ +#include "common.h" + +int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc ,BLASLONG offset) +{ + + BLASLONG i,j,k; + FLOAT *C0,*C1,*ptrba,*ptrbb; + + FLOAT res0_0; + FLOAT res0_1; + FLOAT res0_2; + FLOAT res0_3; + FLOAT res0_4; + FLOAT res0_5; + FLOAT res0_6; + FLOAT res0_7; + + FLOAT res0_8; + FLOAT res0_9; + FLOAT res0_10; + FLOAT res0_11; + FLOAT res0_12; + FLOAT res0_13; + FLOAT res0_14; + FLOAT res0_15; + + FLOAT res1_0; + FLOAT res1_1; + FLOAT res1_2; + FLOAT res1_3; + FLOAT res1_4; + FLOAT res1_5; + FLOAT res1_6; + FLOAT res1_7; + + FLOAT res1_8; + FLOAT res1_9; + FLOAT res1_10; + FLOAT res1_11; + FLOAT res1_12; + FLOAT res1_13; + FLOAT res1_14; + FLOAT res1_15; + + FLOAT a0; + FLOAT a1; + + FLOAT b0; + FLOAT b1; + + BLASLONG off, temp; + +#if !defined(LEFT) + off = -offset; +#endif + + + + for (j=0; j> 3) + je .L2_20 + + ALIGN_4 + +.L2_11: + + leaq BUFFER1, BO // first buffer to BO + addq $4 * SIZE, BO + + vzeroall + + movq K, %rax + + andq $-8, %rax // K = K - ( K % 8 ) + je .L2_16 + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + + salq $3, %rax // rax = rax * 8 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L2_12: + + prefetcht0 B_PR1(BO,BI,8) + KERNEL8x2_1(xxx) + KERNEL8x2_2(xxx) + KERNEL8x2_3(xxx) + KERNEL8x2_4(xxx) + + prefetcht0 B_PR1(BO,BI,8) + KERNEL8x2_1(xxx) + KERNEL8x2_2(xxx) + KERNEL8x2_3(xxx) + KERNEL8x2_4(xxx) + + je .L2_16 + + prefetcht0 B_PR1(BO,BI,8) + KERNEL8x2_1(xxx) + KERNEL8x2_2(xxx) + KERNEL8x2_3(xxx) + KERNEL8x2_4(xxx) + + prefetcht0 B_PR1(BO,BI,8) + KERNEL8x2_1(xxx) + KERNEL8x2_2(xxx) + KERNEL8x2_3(xxx) + KERNEL8x2_4(xxx) + + je .L2_16 + + jmp .L2_12 + ALIGN_4 + +.L2_16: + movq K, %rax + + andq $7, %rax # if (k & 1) + je .L2_19 + + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + + salq $3, %rax // rax = rax * 8 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L2_17: + + KERNEL8x2_SUB(xxx) + addq $2, BI + addq $8, %rax + jl .L2_17 + ALIGN_4 + + +.L2_19: + + vmovddup ALPHA, %xmm0 + + vfmaddpd (CO1),%xmm0, %xmm4,%xmm4 + vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7 + vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10 + vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13 + + vfmaddpd (CO1, LDC),%xmm0, %xmm5,%xmm5 + vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8 + vfmaddpd 4 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11 + vfmaddpd 6 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14 + + vmovups %xmm4 , (CO1) + vmovups %xmm7 , 2 * SIZE(CO1) + vmovups %xmm10, 4 * SIZE(CO1) + vmovups %xmm13, 6 * SIZE(CO1) + + vmovups %xmm5 , (CO1, LDC) + vmovups %xmm8 , 2 * SIZE(CO1, LDC) + vmovups %xmm11, 4 * SIZE(CO1, LDC) + vmovups %xmm14, 6 * SIZE(CO1, LDC) + + + addq $8 * SIZE, CO1 # coffset += 8 + decq I # i -- + jg .L2_11 + ALIGN_4 + +/************************************************************************** +* Rest of M +***************************************************************************/ +.L2_20: + // Test rest of M + + testq $7, M + jz .L2_60 // to next 2 lines of N + + testq $4, M + jz .L2_30 + + ALIGN_4 + +.L2_21: + leaq BUFFER1, BO // first buffer to BO + addq $4 * SIZE, BO + + vzeroall + + movq K, %rax + + andq $-8, %rax + je .L2_26 + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + + salq $2, %rax // rax = rax * 4 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L2_22: + + prefetcht0 B_PR1(BO,BI,8) + KERNEL4x2_1(xxx) + KERNEL4x2_2(xxx) + KERNEL4x2_3(xxx) + KERNEL4x2_4(xxx) + + prefetcht0 B_PR1(BO,BI,8) + KERNEL4x2_1(xxx) + KERNEL4x2_2(xxx) + KERNEL4x2_3(xxx) + KERNEL4x2_4(xxx) + + je .L2_26 + + prefetcht0 B_PR1(BO,BI,8) + KERNEL4x2_1(xxx) + KERNEL4x2_2(xxx) + KERNEL4x2_3(xxx) + KERNEL4x2_4(xxx) + + prefetcht0 B_PR1(BO,BI,8) + KERNEL4x2_1(xxx) + KERNEL4x2_2(xxx) + KERNEL4x2_3(xxx) + KERNEL4x2_4(xxx) + + je .L2_26 + + jmp .L2_22 + ALIGN_4 + +.L2_26: + movq K, %rax + + andq $7, %rax # if (k & 1) + je .L2_29 + + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + + salq $2, %rax // rax = rax * 4 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L2_27: + + KERNEL4x2_SUB(xxx) + addq $2, BI + addq $4, %rax + jl .L2_27 + ALIGN_4 + + +.L2_29: + + vmovddup ALPHA, %xmm0 + + vfmaddpd (CO1),%xmm0, %xmm4,%xmm4 + vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7 + + vfmaddpd (CO1, LDC),%xmm0, %xmm5,%xmm5 + vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8 + + vmovups %xmm4 , (CO1) + vmovups %xmm7 , 2 * SIZE(CO1) + + vmovups %xmm5 , (CO1, LDC) + vmovups %xmm8 , 2 * SIZE(CO1, LDC) + + addq $4 * SIZE, CO1 # coffset += 4 + ALIGN_4 + + +.L2_30: + testq $2, M + jz .L2_40 + + ALIGN_4 + +.L2_31: + leaq BUFFER1, BO // first buffer to BO + addq $4 * SIZE, BO + + vzeroall + + movq K, %rax + + andq $-8, %rax + je .L2_36 + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + + salq $1, %rax // rax = rax *2 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L2_32: + + prefetcht0 B_PR1(BO,BI,8) + KERNEL2x2_1(xxx) + KERNEL2x2_2(xxx) + KERNEL2x2_3(xxx) + KERNEL2x2_4(xxx) + + prefetcht0 B_PR1(BO,BI,8) + KERNEL2x2_1(xxx) + KERNEL2x2_2(xxx) + KERNEL2x2_3(xxx) + KERNEL2x2_4(xxx) + + je .L2_36 + + prefetcht0 B_PR1(BO,BI,8) + KERNEL2x2_1(xxx) + KERNEL2x2_2(xxx) + KERNEL2x2_3(xxx) + KERNEL2x2_4(xxx) + + prefetcht0 B_PR1(BO,BI,8) + KERNEL2x2_1(xxx) + KERNEL2x2_2(xxx) + KERNEL2x2_3(xxx) + KERNEL2x2_4(xxx) + + je .L2_36 + + jmp .L2_32 + ALIGN_4 + +.L2_36: + movq K, %rax + + andq $7, %rax # if (k & 1) + je .L2_39 + + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + + salq $1, %rax // rax = rax *2 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L2_37: + + KERNEL2x2_SUB(xxx) + addq $2, BI + addq $2, %rax + jl .L2_37 + ALIGN_4 + + +.L2_39: + + vmovddup ALPHA, %xmm0 + + vfmaddpd (CO1),%xmm0, %xmm4,%xmm4 + vfmaddpd (CO1, LDC),%xmm0, %xmm5,%xmm5 + + vmovups %xmm4 , (CO1) + vmovups %xmm5 , (CO1, LDC) + + addq $2 * SIZE, CO1 # coffset += 2 + ALIGN_4 + + +.L2_40: + testq $1, M + jz .L2_60 // to next 2 lines of N + + ALIGN_4 + +.L2_41: + leaq BUFFER1, BO // first buffer to BO + addq $4 * SIZE, BO + + vzeroall + + movq K, %rax + + andq $-8, %rax + je .L2_46 + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L2_42: + + prefetcht0 B_PR1(BO,BI,8) + KERNEL1x2_1(xxx) + KERNEL1x2_2(xxx) + KERNEL1x2_3(xxx) + KERNEL1x2_4(xxx) + + prefetcht0 B_PR1(BO,BI,8) + KERNEL1x2_1(xxx) + KERNEL1x2_2(xxx) + KERNEL1x2_3(xxx) + KERNEL1x2_4(xxx) + + je .L2_46 + + prefetcht0 B_PR1(BO,BI,8) + KERNEL1x2_1(xxx) + KERNEL1x2_2(xxx) + KERNEL1x2_3(xxx) + KERNEL1x2_4(xxx) + + prefetcht0 B_PR1(BO,BI,8) + KERNEL1x2_1(xxx) + KERNEL1x2_2(xxx) + KERNEL1x2_3(xxx) + KERNEL1x2_4(xxx) + + je .L2_46 + + jmp .L2_42 + ALIGN_4 + +.L2_46: + movq K, %rax + + andq $7, %rax # if (k & 1) + je .L2_49 + + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L2_47: + + KERNEL1x2_SUB(xxx) + addq $2, BI + addq $1, %rax + jl .L2_47 + ALIGN_4 + + +.L2_49: + + vmovddup ALPHA, %xmm0 + + vfmaddsd (CO1),%xmm0, %xmm4,%xmm4 + vfmaddsd (CO1, LDC),%xmm0, %xmm5,%xmm5 + + vmovsd %xmm4 , (CO1) + vmovsd %xmm5 , (CO1, LDC) + + + addq $1 * SIZE, CO1 # coffset += 1 + ALIGN_4 + + + +.L2_60: + + decq J // j -- + jg .L2_01 // next 2 lines of N + + + +.L1_0: + +/************************************************************************************************ +* Loop for Nmod6 % 2 > 0 +*************************************************************************************************/ + + movq Nmod6, J + andq $1, J // j % 2 + je .L999 + ALIGN_4 + +.L1_01: + // copy to sub buffer + movq B, BO1 + leaq BUFFER1, BO // first buffer to BO + movq K, %rax + ALIGN_4 + +.L1_02b: + + vmovsd (BO1), %xmm0 + vmovsd %xmm0, (BO) + addq $1*SIZE,BO1 + addq $1*SIZE,BO + decq %rax + jnz .L1_02b + +.L1_02c: + + movq BO1, B // next offset of B + +.L1_10: + movq C, CO1 + leaq (C, LDC, 1), C // c += 1 * ldc + + movq A, AO // aoffset = a + addq $16 * SIZE, AO + + movq M, I + sarq $3, I // i = (m >> 3) + je .L1_20 + + ALIGN_4 + +.L1_11: + leaq BUFFER1, BO // first buffer to BO + addq $2 * SIZE, BO + + vzeroall + + movq K, %rax + + andq $-8, %rax // K = K - ( K % 8 ) + je .L1_16 + movq %rax, BI // Index for BO + + salq $3, %rax // rax = rax * 8 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L1_12: + + prefetcht0 B_PR1(BO,BI,8) + KERNEL8x1_1(xxx) + KERNEL8x1_2(xxx) + KERNEL8x1_3(xxx) + KERNEL8x1_4(xxx) + + KERNEL8x1_1(xxx) + KERNEL8x1_2(xxx) + KERNEL8x1_3(xxx) + KERNEL8x1_4(xxx) + + je .L1_16 + + prefetcht0 B_PR1(BO,BI,8) + KERNEL8x1_1(xxx) + KERNEL8x1_2(xxx) + KERNEL8x1_3(xxx) + KERNEL8x1_4(xxx) + + KERNEL8x1_1(xxx) + KERNEL8x1_2(xxx) + KERNEL8x1_3(xxx) + KERNEL8x1_4(xxx) + + je .L1_16 + + jmp .L1_12 + ALIGN_4 + +.L1_16: + movq K, %rax + + andq $7, %rax # if (k & 1) + je .L1_19 + + movq %rax, BI // Index for BO + + salq $3, %rax // rax = rax * 8 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L1_17: + + KERNEL8x1_SUB(xxx) + addq $1, BI + addq $8, %rax + jl .L1_17 + ALIGN_4 + + +.L1_19: + + vmovddup ALPHA, %xmm0 + + vfmaddpd (CO1),%xmm0, %xmm4,%xmm4 + vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7 + vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10 + vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13 + + vmovups %xmm4 , (CO1) + vmovups %xmm7 , 2 * SIZE(CO1) + vmovups %xmm10, 4 * SIZE(CO1) + vmovups %xmm13, 6 * SIZE(CO1) + + addq $8 * SIZE, CO1 # coffset += 8 + decq I # i -- + jg .L1_11 + ALIGN_4 + +/************************************************************************** +* Rest of M +***************************************************************************/ +.L1_20: + // Test rest of M + + testq $7, M + jz .L999 + + testq $4, M + jz .L1_30 + + ALIGN_4 + +.L1_21: + leaq BUFFER1, BO // first buffer to BO + addq $2 * SIZE, BO + + + vzeroall + + movq K, %rax + + andq $-8, %rax + je .L1_26 + movq %rax, BI // Index for BO + + salq $2, %rax // rax = rax * 4 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L1_22: + + prefetcht0 B_PR1(BO,BI,8) + KERNEL4x1_1(xxx) + KERNEL4x1_2(xxx) + KERNEL4x1_3(xxx) + KERNEL4x1_4(xxx) + + KERNEL4x1_1(xxx) + KERNEL4x1_2(xxx) + KERNEL4x1_3(xxx) + KERNEL4x1_4(xxx) + + je .L1_26 + + prefetcht0 B_PR1(BO,BI,8) + KERNEL4x1_1(xxx) + KERNEL4x1_2(xxx) + KERNEL4x1_3(xxx) + KERNEL4x1_4(xxx) + + KERNEL4x1_1(xxx) + KERNEL4x1_2(xxx) + KERNEL4x1_3(xxx) + KERNEL4x1_4(xxx) + + je .L1_26 + + jmp .L1_22 + ALIGN_4 + +.L1_26: + movq K, %rax + + andq $7, %rax # if (k & 1) + je .L1_29 + + movq %rax, BI // Index for BO + + salq $2, %rax // rax = rax * 4 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L1_27: + + KERNEL4x1_SUB(xxx) + addq $1, BI + addq $4, %rax + jl .L1_27 + ALIGN_4 + + +.L1_29: + + vmovddup ALPHA, %xmm0 + + vfmaddpd (CO1),%xmm0, %xmm4,%xmm4 + vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7 + + vmovups %xmm4 , (CO1) + vmovups %xmm7 , 2 * SIZE(CO1) + + addq $4 * SIZE, CO1 # coffset += 4 + ALIGN_4 + + +.L1_30: + testq $2, M + jz .L1_40 + + ALIGN_4 + +.L1_31: + leaq BUFFER1, BO // first buffer to BO + addq $2 * SIZE, BO + + + vzeroall + + movq K, %rax + + andq $-8, %rax + je .L1_36 + movq %rax, BI // Index for BO + + salq $1, %rax // rax = rax *2 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L1_32: + + prefetcht0 B_PR1(BO,BI,8) + KERNEL2x1_1(xxx) + KERNEL2x1_2(xxx) + KERNEL2x1_3(xxx) + KERNEL2x1_4(xxx) + + KERNEL2x1_1(xxx) + KERNEL2x1_2(xxx) + KERNEL2x1_3(xxx) + KERNEL2x1_4(xxx) + + je .L1_36 + + KERNEL2x1_1(xxx) + KERNEL2x1_2(xxx) + KERNEL2x1_3(xxx) + KERNEL2x1_4(xxx) + + KERNEL2x1_1(xxx) + KERNEL2x1_2(xxx) + KERNEL2x1_3(xxx) + KERNEL2x1_4(xxx) + + je .L1_36 + + jmp .L1_32 + ALIGN_4 + +.L1_36: + movq K, %rax + + andq $7, %rax # if (k & 1) + je .L1_39 + + movq %rax, BI // Index for BO + + salq $1, %rax // rax = rax *2 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L1_37: + + KERNEL2x1_SUB(xxx) + addq $1, BI + addq $2, %rax + jl .L1_37 + ALIGN_4 + + +.L1_39: + + vmovddup ALPHA, %xmm0 + + vfmaddpd (CO1),%xmm0, %xmm4,%xmm4 + + vmovups %xmm4 , (CO1) + + addq $2 * SIZE, CO1 # coffset += 2 + ALIGN_4 + + +.L1_40: + testq $1, M + jz .L999 + + ALIGN_4 + +.L1_41: + leaq BUFFER1, BO // first buffer to BO + addq $2 * SIZE, BO + + vzeroall + + movq K, %rax + + andq $-8, %rax + je .L1_46 + movq %rax, BI // Index for BO + + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L1_42: + + prefetcht0 B_PR1(BO,BI,8) + KERNEL1x1_1(xxx) + KERNEL1x1_2(xxx) + KERNEL1x1_3(xxx) + KERNEL1x1_4(xxx) + + KERNEL1x1_1(xxx) + KERNEL1x1_2(xxx) + KERNEL1x1_3(xxx) + KERNEL1x1_4(xxx) + + je .L1_46 + + prefetcht0 B_PR1(BO,BI,8) + KERNEL1x1_1(xxx) + KERNEL1x1_2(xxx) + KERNEL1x1_3(xxx) + KERNEL1x1_4(xxx) + + KERNEL1x1_1(xxx) + KERNEL1x1_2(xxx) + KERNEL1x1_3(xxx) + KERNEL1x1_4(xxx) + + je .L1_46 + + jmp .L1_42 + ALIGN_4 + +.L1_46: + movq K, %rax + + andq $7, %rax # if (k & 1) + je .L1_49 + + movq %rax, BI // Index for BO + + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L1_47: + + KERNEL1x1_SUB(xxx) + addq $1, BI + addq $1, %rax + jl .L1_47 + ALIGN_4 + + +.L1_49: + + vmovddup ALPHA, %xmm0 + + vfmaddsd (CO1),%xmm0, %xmm4,%xmm4 + + vmovsd %xmm4 , (CO1) + + addq $1 * SIZE, CO1 # coffset += 1 + ALIGN_4 + + +.L999: + movq SP, %rsp + movq (%rsp), %rbx + movq 8(%rsp), %rbp + movq 16(%rsp), %r12 + movq 24(%rsp), %r13 + movq 32(%rsp), %r14 + movq 40(%rsp), %r15 + +#ifdef WINDOWS_ABI + movq 48(%rsp), %rdi + movq 56(%rsp), %rsi + movups 64(%rsp), %xmm6 + movups 80(%rsp), %xmm7 + movups 96(%rsp), %xmm8 + movups 112(%rsp), %xmm9 + movups 128(%rsp), %xmm10 + movups 144(%rsp), %xmm11 + movups 160(%rsp), %xmm12 + movups 176(%rsp), %xmm13 + movups 192(%rsp), %xmm14 + movups 208(%rsp), %xmm15 +#endif + + addq $STACKSIZE, %rsp + ret + + EPILOGUE + + +#else +/************************************************************************************* +* TRMM Kernel +*************************************************************************************/ + PROLOGUE + PROFCODE + + subq $STACKSIZE, %rsp + movq %rbx, (%rsp) + movq %rbp, 8(%rsp) + movq %r12, 16(%rsp) + movq %r13, 24(%rsp) + movq %r14, 32(%rsp) + movq %r15, 40(%rsp) + + vzeroupper + +#ifdef WINDOWS_ABI + movq %rdi, 48(%rsp) + movq %rsi, 56(%rsp) + movups %xmm6, 64(%rsp) + movups %xmm7, 80(%rsp) + movups %xmm8, 96(%rsp) + movups %xmm9, 112(%rsp) + movups %xmm10, 128(%rsp) + movups %xmm11, 144(%rsp) + movups %xmm12, 160(%rsp) + movups %xmm13, 176(%rsp) + movups %xmm14, 192(%rsp) + movups %xmm15, 208(%rsp) + + movq ARG1, OLD_M + movq ARG2, OLD_N + movq ARG3, OLD_K + movq OLD_A, A + movq OLD_B, B + movq OLD_C, C + movq OLD_LDC, LDC +#ifdef TRMMKERNEL + movsd OLD_OFFSET, %xmm12 +#endif + vmovaps %xmm3, %xmm0 + +#else + movq STACKSIZE + 8(%rsp), LDC +#ifdef TRMMKERNEL + movsd STACKSIZE + 16(%rsp), %xmm12 +#endif + +#endif + + movq %rsp, SP # save old stack + subq $128 + L_BUFFER_SIZE, %rsp + andq $-4096, %rsp # align stack + + STACK_TOUCH + + cmpq $0, OLD_M + je .L999 + + cmpq $0, OLD_N + je .L999 + + cmpq $0, OLD_K + je .L999 + + movq OLD_M, M + movq OLD_N, N + movq OLD_K, K + + vmovsd %xmm0, ALPHA + + salq $BASE_SHIFT, LDC + + movq N, %rax + xorq %rdx, %rdx + movq $2, %rdi + divq %rdi // N / 2 + movq %rax, Ndiv6 // N / 2 + movq %rdx, Nmod6 // N % 2 + + + +#ifdef TRMMKERNEL + vmovsd %xmm12, OFFSET + vmovsd %xmm12, KK +#ifndef LEFT + negq KK +#endif +#endif + + movq Ndiv6, J + cmpq $0, J + je .L1_0 + ALIGN_4 + +.L2_0: + .L2_01: // copy to sub buffer movq B, BO1 @@ -3205,6 +3731,9 @@ .L2_60: +#if defined(TRMMKERNEL) && !defined(LEFT) + addq $2, KK +#endif decq J // j -- jg .L2_01 // next 2 lines of N @@ -3878,3 +4407,7 @@ ret EPILOGUE + + + +#endif diff --git a/kernel/x86_64/dgemv_t_bulldozer.S b/kernel/x86_64/dgemv_t_bulldozer.S index 487ff77ad..36ae2b9df 100644 --- a/kernel/x86_64/dgemv_t_bulldozer.S +++ b/kernel/x86_64/dgemv_t_bulldozer.S @@ -40,7 +40,6 @@ #include "common.h" #include "l2param.h" -// #undef ALIGNED_ACCESS #define A_PRE 256 @@ -111,11 +110,7 @@ #define Y1 %rbp #define X1 %r15 -#ifdef ALIGNED_ACCESS -#define MM INCX -#else #define MM M -#endif #define ALPHA %xmm15 @@ -216,23 +211,6 @@ movq BUFFER, X1 -#ifdef ALIGNED_ACCESS - testq $SIZE, A - je .L01 - - vmovsd (X), %xmm0 - addq INCX, X - - vmovsd %xmm0, 1 * SIZE(BUFFER) - addq $1 * SIZE, BUFFER - addq $2 * SIZE, X1 - decq M - jle .L10 - ALIGN_4 - -.L01: -#endif - movq M, I sarq $3, I jle .L05 @@ -287,10 +265,6 @@ .L10: movq Y, Y1 -#ifdef ALIGNED_ACCESS - testq $SIZE, LDA - jne .L50 -#endif #if GEMV_UNROLL >= 8 cmpq $8, N @@ -316,41 +290,6 @@ vxorps %xmm7 , %xmm7, %xmm7 -#ifdef ALIGNED_ACCESS - testq $SIZE, A - je .L1X - - vmovsd -16 * SIZE(X1), %xmm12 - - vmovsd -16 * SIZE(A1), %xmm8 - vmovsd -16 * SIZE(A1, LDA), %xmm9 - vmovsd -16 * SIZE(A1, LDA, 2), %xmm10 - vmovsd -16 * SIZE(A1, LDA3), %xmm11 - - vfmaddpd %xmm0, %xmm8 , %xmm12, %xmm0 - vfmaddpd %xmm1, %xmm9 , %xmm12, %xmm1 - vfmaddpd %xmm2, %xmm10, %xmm12, %xmm2 - vfmaddpd %xmm3, %xmm11, %xmm12, %xmm3 - - vmovsd -16 * SIZE(A2), %xmm8 - vmovsd -16 * SIZE(A2, LDA), %xmm9 - vmovsd -16 * SIZE(A2, LDA, 2), %xmm10 - vmovsd -16 * SIZE(A2, LDA3), %xmm11 - - vfmaddpd %xmm4, %xmm8 , %xmm12, %xmm4 - vfmaddpd %xmm5, %xmm9 , %xmm12, %xmm5 - vfmaddpd %xmm6, %xmm10, %xmm12, %xmm6 - vfmaddpd %xmm7, %xmm11, %xmm12, %xmm7 - - - addq $SIZE, A1 - addq $SIZE, A2 - addq $SIZE, X1 - ALIGN_3 - -.L1X: -#endif - movq M, I sarq $3, I jle .L15 @@ -671,31 +610,6 @@ vxorps %xmm3 , %xmm3, %xmm3 -#ifdef ALIGNED_ACCESS - testq $SIZE, A - je .L2X - - vmovsd -16 * SIZE(X1), %xmm12 - - vmovsd -16 * SIZE(A1), %xmm8 - vmovsd -16 * SIZE(A1, LDA), %xmm9 - vmovsd -16 * SIZE(A2), %xmm10 - vmovsd -16 * SIZE(A2, LDA), %xmm11 - - - vfmaddpd %xmm0, %xmm8 , %xmm12, %xmm0 - vfmaddpd %xmm1, %xmm9 , %xmm12, %xmm1 - vfmaddpd %xmm2, %xmm10, %xmm12, %xmm2 - vfmaddpd %xmm3, %xmm11, %xmm12, %xmm3 - - addq $SIZE, A1 - addq $SIZE, A2 - addq $SIZE, X1 - ALIGN_3 - -.L2X: -#endif - movq M, I sarq $3, I jle .L25 @@ -924,26 +838,6 @@ vxorps %xmm3 , %xmm3, %xmm3 -#ifdef ALIGNED_ACCESS - testq $SIZE, A - je .L3X - - vmovsd -16 * SIZE(X1), %xmm12 - - vmovsd -16 * SIZE(A1), %xmm8 - vmovsd -16 * SIZE(A2), %xmm9 - - vfmaddpd %xmm0, %xmm8 , %xmm12, %xmm0 - vfmaddpd %xmm1, %xmm9 , %xmm12, %xmm1 - - addq $SIZE, A1 - addq $SIZE, A2 - addq $SIZE, X1 - ALIGN_3 - -.L3X: -#endif - movq M, I sarq $3, I jle .L35 @@ -1100,21 +994,6 @@ vxorps %xmm3 , %xmm3, %xmm3 -#ifdef ALIGNED_ACCESS - testq $SIZE, A - je .L4X - - movsd -16 * SIZE(X1), %xmm12 - movsd -16 * SIZE(A1), %xmm8 - - vfmaddpd %xmm0, %xmm8 , %xmm12, %xmm0 - - addq $SIZE, A1 - addq $SIZE, X1 - ALIGN_3 - -.L4X: -#endif movq M, I sarq $3, I @@ -1223,683 +1102,6 @@ vmovlpd %xmm0, (Y1) addq INCY, Y1 -#ifdef ALIGNED_ACCESS - jmp .L999 - ALIGN_4 - -.L50: -#if GEMV_UNROLL >= 4 - - cmpq $4, N - jl .L60 - ALIGN_3 - -.L51: - subq $4, N - - leaq 16 * SIZE(BUFFER), X1 - - movq A, A1 - leaq (A1, LDA, 2), A2 - leaq (A1, LDA, 4), A - - vxorps %xmm0 , %xmm0, %xmm0 - vxorps %xmm1 , %xmm1, %xmm1 - vxorps %xmm2 , %xmm2, %xmm2 - vxorps %xmm3 , %xmm3, %xmm3 - - - -#ifdef ALIGNED_ACCESS - testq $SIZE, A - je .L5X - - vmovsd -16 * SIZE(X1), %xmm12 - - vmovsd -16 * SIZE(A1), %xmm4 - vmovsd -16 * SIZE(A1, LDA), %xmm5 - vmovsd -16 * SIZE(A2), %xmm6 - vmovsd -16 * SIZE(A2, LDA), %xmm7 - - vfmaddpd %xmm0, %xmm4 , %xmm12, %xmm0 - vfmaddpd %xmm1, %xmm5 , %xmm12, %xmm1 - vfmaddpd %xmm2, %xmm6 , %xmm12, %xmm2 - vfmaddpd %xmm3, %xmm7 , %xmm12, %xmm3 - - - addq $SIZE, A1 - addq $SIZE, A2 - addq $SIZE, X1 - ALIGN_3 - -.L5X: -#endif - - vxorps %xmm8 , %xmm8, %xmm8 - vxorps %xmm9 , %xmm9, %xmm9 - vmovhpd -16 * SIZE(A1, LDA), %xmm8 , %xmm8 - vmovhpd -16 * SIZE(A2, LDA), %xmm9 , %xmm9 - - movq M, I - sarq $3, I - jle .L55 - - VMOVUPS_A2(-15 * SIZE, A1, LDA, 1, %xmm5) - VMOVUPS_A2(-15 * SIZE, A2, LDA, 1, %xmm7) - - VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) - VMOVUPS_XL1(-14 * SIZE, X1, %xmm13) - - decq I - jle .L53 - ALIGN_4 - -.L52: - - vfmaddpd %xmm0 , -16 * SIZE(A1) , %xmm12 , %xmm0 - - vshufpd $1, %xmm5, %xmm8 , %xmm8 - vfmaddpd %xmm1 , %xmm8 , %xmm12 , %xmm1 - VMOVUPS_A2(-13 * SIZE, A1, LDA, 1, %xmm8) - - vfmaddpd %xmm2 , -16 * SIZE(A2) , %xmm12 , %xmm2 - - vshufpd $1, %xmm7, %xmm9 , %xmm9 - vfmaddpd %xmm3 , %xmm9 , %xmm12 , %xmm3 - VMOVUPS_XL1(-12 * SIZE, X1, %xmm12) - VMOVUPS_A2(-13 * SIZE, A2, LDA, 1, %xmm9) - - vfmaddpd %xmm0 , -14 * SIZE(A1) , %xmm13 , %xmm0 - - VMOVUPS_A1(-12 * SIZE, A1, %xmm4) - vshufpd $1, %xmm8, %xmm5 , %xmm5 - vfmaddpd %xmm1 , %xmm5 , %xmm13 , %xmm1 - VMOVUPS_A2(-11 * SIZE, A1, LDA, 1, %xmm5) - - vfmaddpd %xmm2 , -14 * SIZE(A2) , %xmm13 , %xmm2 - - vshufpd $1, %xmm9, %xmm7 , %xmm7 - vfmaddpd %xmm3 , %xmm7 , %xmm13 , %xmm3 - VMOVUPS_XL1(-10 * SIZE, X1, %xmm13) - VMOVUPS_A2(-11 * SIZE, A2, LDA, 1, %xmm7) - - vfmaddpd %xmm0 , -12 * SIZE(A1) , %xmm12 , %xmm0 - - vshufpd $1, %xmm5, %xmm8 , %xmm8 - vfmaddpd %xmm1 , %xmm8 , %xmm12 , %xmm1 - VMOVUPS_A2( -9 * SIZE, A1, LDA, 1, %xmm8) - - vfmaddpd %xmm2 , -12 * SIZE(A2) , %xmm12 , %xmm2 - - vshufpd $1, %xmm7, %xmm9 , %xmm9 - vfmaddpd %xmm3 , %xmm9 , %xmm12 , %xmm3 - VMOVUPS_XL1(-8 * SIZE, X1, %xmm12) - VMOVUPS_A2( -9 * SIZE, A2, LDA, 1, %xmm9) - - vfmaddpd %xmm0 , -10 * SIZE(A1) , %xmm13 , %xmm0 - - vshufpd $1, %xmm8, %xmm5 , %xmm5 - vfmaddpd %xmm1 , %xmm5 , %xmm13 , %xmm1 - VMOVUPS_A2(-7 * SIZE, A1, LDA, 1, %xmm5) - - vfmaddpd %xmm2 , -10 * SIZE(A2) , %xmm13 , %xmm2 - - vshufpd $1, %xmm9, %xmm7 , %xmm7 - vfmaddpd %xmm3 , %xmm7 , %xmm13 , %xmm3 - VMOVUPS_XL1(-6 * SIZE, X1, %xmm13) - VMOVUPS_A2(-7 * SIZE, A2, LDA, 1, %xmm7) - - addq $8 * SIZE, A1 - addq $8 * SIZE, A2 - addq $8 * SIZE, X1 - - decq I - jg .L52 - ALIGN_4 - -.L53: - - vfmaddpd %xmm0 , -16 * SIZE(A1) , %xmm12 , %xmm0 - - vshufpd $1, %xmm5, %xmm8 , %xmm8 - vfmaddpd %xmm1 , %xmm8 , %xmm12 , %xmm1 - VMOVUPS_A2(-13 * SIZE, A1, LDA, 1, %xmm8) - - vfmaddpd %xmm2 , -16 * SIZE(A2) , %xmm12 , %xmm2 - - vshufpd $1, %xmm7, %xmm9 , %xmm9 - vfmaddpd %xmm3 , %xmm9 , %xmm12 , %xmm3 - VMOVUPS_XL1(-12 * SIZE, X1, %xmm12) - VMOVUPS_A2(-13 * SIZE, A2, LDA, 1, %xmm9) - - vfmaddpd %xmm0 , -14 * SIZE(A1) , %xmm13 , %xmm0 - - vshufpd $1, %xmm8, %xmm5 , %xmm5 - vfmaddpd %xmm1 , %xmm5 , %xmm13 , %xmm1 - VMOVUPS_A2(-11 * SIZE, A1, LDA, 1, %xmm5) - - vfmaddpd %xmm2 , -14 * SIZE(A2) , %xmm13 , %xmm2 - - vshufpd $1, %xmm9, %xmm7 , %xmm7 - vfmaddpd %xmm3 , %xmm7 , %xmm13 , %xmm3 - VMOVUPS_XL1(-10 * SIZE, X1, %xmm13) - VMOVUPS_A2(-11 * SIZE, A2, LDA, 1, %xmm7) - - vfmaddpd %xmm0 , -12 * SIZE(A1) , %xmm12 , %xmm0 - - vshufpd $1, %xmm5, %xmm8 , %xmm8 - vfmaddpd %xmm1 , %xmm8 , %xmm12 , %xmm1 - VMOVUPS_A2( -9 * SIZE, A1, LDA, 1, %xmm8) - - vfmaddpd %xmm2 , -12 * SIZE(A2) , %xmm12 , %xmm2 - - vshufpd $1, %xmm7, %xmm9 , %xmm9 - vfmaddpd %xmm3 , %xmm9 , %xmm12 , %xmm3 - VMOVUPS_XL1(-8 * SIZE, X1, %xmm12) - VMOVUPS_A2( -9 * SIZE, A2, LDA, 1, %xmm9) - - vfmaddpd %xmm0 , -10 * SIZE(A1) , %xmm13 , %xmm0 - - vshufpd $1, %xmm8, %xmm5 , %xmm5 - vfmaddpd %xmm1 , %xmm5 , %xmm13 , %xmm1 - VMOVUPS_A2(-7 * SIZE, A1, LDA, 1, %xmm5) - - vfmaddpd %xmm2 , -10 * SIZE(A2) , %xmm13 , %xmm2 - - vshufpd $1, %xmm9, %xmm7 , %xmm7 - vfmaddpd %xmm3 , %xmm7 , %xmm13 , %xmm3 - VMOVUPS_XL1(-6 * SIZE, X1, %xmm13) - VMOVUPS_A2(-7 * SIZE, A2, LDA, 1, %xmm7) - - addq $8 * SIZE, A1 - addq $8 * SIZE, A2 - addq $8 * SIZE, X1 - ALIGN_4 - -.L55: - testq $4, M - jle .L56 - - VMOVUPS_A2(-15 * SIZE, A1, LDA, 1, %xmm5) - VMOVUPS_A2(-15 * SIZE, A2, LDA, 1, %xmm7) - - VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) - VMOVUPS_XL1(-14 * SIZE, X1, %xmm13) - - - vfmaddpd %xmm0 , -16 * SIZE(A1) , %xmm12 , %xmm0 - - vshufpd $1, %xmm5, %xmm8 , %xmm8 - vfmaddpd %xmm1 , %xmm8 , %xmm12 , %xmm1 - VMOVUPS_A2(-13 * SIZE, A1, LDA, 1, %xmm8) - - vfmaddpd %xmm2 , -16 * SIZE(A2) , %xmm12 , %xmm2 - - vshufpd $1, %xmm7, %xmm9 , %xmm9 - vfmaddpd %xmm3 , %xmm9 , %xmm12 , %xmm3 - VMOVUPS_A2(-13 * SIZE, A2, LDA, 1, %xmm9) - - vfmaddpd %xmm0 , -14 * SIZE(A1) , %xmm13 , %xmm0 - - vshufpd $1, %xmm8, %xmm5 , %xmm5 - vfmaddpd %xmm1 , %xmm5 , %xmm13 , %xmm1 - - vfmaddpd %xmm2 , -14 * SIZE(A2) , %xmm13 , %xmm2 - - vshufpd $1, %xmm9, %xmm7 , %xmm7 - vfmaddpd %xmm3 , %xmm7 , %xmm13 , %xmm3 - - - addq $4 * SIZE, A1 - addq $4 * SIZE, A2 - addq $4 * SIZE, X1 - ALIGN_4 - -.L56: - testq $2, M - jle .L57 - - VMOVUPS_A2(-15 * SIZE, A1, LDA, 1, %xmm5) - VMOVUPS_A2(-15 * SIZE, A2, LDA, 1, %xmm7) - - VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) - - - vfmaddpd %xmm0 , -16 * SIZE(A1) , %xmm12 , %xmm0 - - vshufpd $1, %xmm5, %xmm8 , %xmm8 - vfmaddpd %xmm1 , %xmm8 , %xmm12 , %xmm1 - - vfmaddpd %xmm2 , -16 * SIZE(A2) , %xmm12 , %xmm2 - - vshufpd $1, %xmm7, %xmm9 , %xmm9 - vfmaddpd %xmm3 , %xmm9 , %xmm12 , %xmm3 - - addq $2 * SIZE, A1 - addq $2 * SIZE, A2 - addq $2 * SIZE, X1 - ALIGN_4 - -.L57: - testq $1, M - je .L58 - - vmovsd -16 * SIZE(X1), %xmm12 - - vmovsd -16 * SIZE(A1), %xmm4 - vmovsd -16 * SIZE(A2), %xmm6 - - vfmaddpd %xmm0 , %xmm4 , %xmm12 , %xmm0 - - vshufpd $1, %xmm8, %xmm8 , %xmm8 - vfmaddpd %xmm1 , %xmm8 , %xmm12 , %xmm1 - - vfmaddpd %xmm2 , %xmm6 , %xmm12 , %xmm2 - - vshufpd $1, %xmm9, %xmm9 , %xmm9 - vfmaddpd %xmm3 , %xmm9 , %xmm12 , %xmm3 - - ALIGN_4 - -.L58: - vhaddpd %xmm1, %xmm0 , %xmm0 - vhaddpd %xmm3, %xmm2 , %xmm2 - - vmulpd ALPHA, %xmm0 , %xmm0 - vmulpd ALPHA, %xmm2 , %xmm2 - - cmpq $SIZE, INCY - jne .L59 - - vmovups 0 * SIZE(Y), %xmm4 - vmovups 2 * SIZE(Y), %xmm5 - addq $4 * SIZE, Y - - vaddpd %xmm4, %xmm0 , %xmm0 - vaddpd %xmm5, %xmm2 , %xmm2 - - vmovups %xmm0, 0 * SIZE(Y1) - vmovups %xmm2, 2 * SIZE(Y1) - addq $4 * SIZE, Y1 - - cmpq $4, N - jge .L51 - jmp .L60 - ALIGN_4 - -.L59: - vmovsd (Y), %xmm4 - addq INCY, Y - vmovhpd (Y), %xmm4 , %xmm4 - addq INCY, Y - vmovsd (Y), %xmm5 - addq INCY, Y - vmovhpd (Y), %xmm5 , %xmm5 - addq INCY, Y - - vaddpd %xmm4, %xmm0 , %xmm0 - vaddpd %xmm5, %xmm2 , %xmm2 - - vmovlpd %xmm0, (Y1) - addq INCY, Y1 - vmovhpd %xmm0, (Y1) - addq INCY, Y1 - vmovlpd %xmm2, (Y1) - addq INCY, Y1 - vmovhpd %xmm2, (Y1) - addq INCY, Y1 - cmpq $4, N - jge .L51 - ALIGN_4 - -.L60: -#endif - -#if GEMV_UNROLL >= 2 - - cmpq $2, N - jl .L70 - -#if GEMV_UNROLL == 2 - ALIGN_3 - -.L61: -#endif - - subq $2, N - - leaq 16 * SIZE(BUFFER), X1 - - movq A, A1 - leaq (A1, LDA), A2 - leaq (A1, LDA, 2), A - - vxorps %xmm0 , %xmm0, %xmm0 - vxorps %xmm1 , %xmm1, %xmm1 - vxorps %xmm2 , %xmm2, %xmm2 - vxorps %xmm3 , %xmm3, %xmm3 - -#ifdef ALIGNED_ACCESS - testq $SIZE, A - je .L6X - - vmovsd -16 * SIZE(X1), %xmm12 - - vmovsd -16 * SIZE(A1), %xmm4 - vmovsd -16 * SIZE(A2), %xmm5 - - vfmaddpd %xmm0 , %xmm4 , %xmm12 , %xmm0 - vfmaddpd %xmm1 , %xmm5 , %xmm12 , %xmm1 - - addq $SIZE, A1 - addq $SIZE, A2 - addq $SIZE, X1 - ALIGN_3 - -.L6X: -#endif - - vxorps %xmm8 , %xmm8, %xmm8 - vmovhpd -16 * SIZE(A2), %xmm8 , %xmm8 - - movq M, I - sarq $3, I - jle .L65 - - VMOVUPS_A1(-15 * SIZE, A2, %xmm5) - VMOVUPS_A1(-13 * SIZE, A2, %xmm7) - - VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) - VMOVUPS_XL1(-14 * SIZE, X1, %xmm13) - - decq I - jle .L63 - ALIGN_4 - -.L62: - - vfmaddpd %xmm0 , -16 * SIZE(A1) , %xmm12 , %xmm0 - - vshufpd $1, %xmm5, %xmm8 , %xmm8 - vfmaddpd %xmm1 , %xmm8 , %xmm12 , %xmm1 - VMOVUPS_XL1(-12 * SIZE, X1, %xmm12) - VMOVUPS_A1(-11 * SIZE, A2, %xmm9) - - vfmaddpd %xmm0 , -14 * SIZE(A1) , %xmm13 , %xmm0 - - vshufpd $1, %xmm7, %xmm5 , %xmm5 - vfmaddpd %xmm1 , %xmm5 , %xmm13 , %xmm1 - VMOVUPS_XL1(-10 * SIZE, X1, %xmm13) - VMOVUPS_A1( -9 * SIZE, A2, %xmm8) - - vfmaddpd %xmm0 , -12 * SIZE(A1) , %xmm12 , %xmm0 - - vshufpd $1, %xmm9, %xmm7 , %xmm7 - vfmaddpd %xmm1 , %xmm7 , %xmm12 , %xmm1 - VMOVUPS_XL1(-8 * SIZE, X1, %xmm12) - VMOVUPS_A1(-7 * SIZE, A2, %xmm5) - - vfmaddpd %xmm0 , -10 * SIZE(A1) , %xmm13 , %xmm0 - - vshufpd $1, %xmm8, %xmm9 , %xmm9 - vfmaddpd %xmm1 , %xmm9 , %xmm13 , %xmm1 - VMOVUPS_XL1(-6 * SIZE, X1, %xmm13) - VMOVUPS_A1(-5 * SIZE, A2, %xmm7) - - addq $8 * SIZE, A1 - addq $8 * SIZE, A2 - addq $8 * SIZE, X1 - - decq I - jg .L62 - ALIGN_4 - -.L63: - - vfmaddpd %xmm0 , -16 * SIZE(A1) , %xmm12 , %xmm0 - - vshufpd $1, %xmm5, %xmm8 , %xmm8 - vfmaddpd %xmm1 , %xmm8 , %xmm12 , %xmm1 - VMOVUPS_XL1(-12 * SIZE, X1, %xmm12) - VMOVUPS_A1(-11 * SIZE, A2, %xmm9) - - vfmaddpd %xmm0 , -14 * SIZE(A1) , %xmm13 , %xmm0 - - vshufpd $1, %xmm7, %xmm5 , %xmm5 - vfmaddpd %xmm1 , %xmm5 , %xmm13 , %xmm1 - VMOVUPS_XL1(-10 * SIZE, X1, %xmm13) - VMOVUPS_A1( -9 * SIZE, A2, %xmm8) - - vfmaddpd %xmm0 , -12 * SIZE(A1) , %xmm12 , %xmm0 - - vshufpd $1, %xmm9, %xmm7 , %xmm7 - vfmaddpd %xmm1 , %xmm7 , %xmm12 , %xmm1 - - vfmaddpd %xmm0 , -10 * SIZE(A1) , %xmm13 , %xmm0 - - vshufpd $1, %xmm8, %xmm9 , %xmm9 - vfmaddpd %xmm1 , %xmm9 , %xmm13 , %xmm1 - - - addq $8 * SIZE, A1 - addq $8 * SIZE, A2 - addq $8 * SIZE, X1 - ALIGN_4 - -.L65: - testq $4, M - jle .L66 - - VMOVUPS_A1(-15 * SIZE, A2, %xmm5) - VMOVUPS_A1(-13 * SIZE, A2, %xmm7) - - VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) - VMOVUPS_XL1(-14 * SIZE, X1, %xmm13) - - vfmaddpd %xmm0 , -16 * SIZE(A1) , %xmm12 , %xmm0 - - vshufpd $1, %xmm5, %xmm8 , %xmm8 - vfmaddpd %xmm1 , %xmm8 , %xmm12 , %xmm1 - - vfmaddpd %xmm0 , -14 * SIZE(A1) , %xmm13 , %xmm0 - - vshufpd $1, %xmm7, %xmm5 , %xmm5 - vmovups %xmm7, %xmm8 - vfmaddpd %xmm1 , %xmm5 , %xmm13 , %xmm1 - - addq $4 * SIZE, A1 - addq $4 * SIZE, A2 - addq $4 * SIZE, X1 - ALIGN_4 - -.L66: - testq $2, M - jle .L67 - - VMOVUPS_A1(-15 * SIZE, A2, %xmm5) - - VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) - - vfmaddpd %xmm0 , -16 * SIZE(A1) , %xmm12 , %xmm0 - - vshufpd $1, %xmm5, %xmm8 , %xmm8 - vfmaddpd %xmm1 , %xmm8 , %xmm12 , %xmm1 - movaps %xmm5, %xmm8 - - addq $2 * SIZE, A1 - addq $2 * SIZE, A2 - addq $2 * SIZE, X1 - ALIGN_4 - -.L67: - testq $1, M - je .L68 - - vmovsd -16 * SIZE(X1), %xmm12 - - vfmaddpd %xmm0 , -16 * SIZE(A1) , %xmm12 , %xmm0 - vshufpd $1, %xmm8, %xmm8 , %xmm8 - vfmaddpd %xmm1 , %xmm8 , %xmm12 , %xmm1 - ALIGN_4 - -.L68: - vaddpd %xmm2, %xmm0 , %xmm0 - vaddpd %xmm3, %xmm1 , %xmm1 - - vhaddpd %xmm1, %xmm0 , %xmm0 - - vmulpd ALPHA, %xmm0 , %xmm0 - - vmovsd (Y), %xmm4 - addq INCY, Y - vmovhpd (Y), %xmm4 , %xmm4 - addq INCY, Y - - vaddpd %xmm4, %xmm0 , %xmm0 - - vmovlpd %xmm0, (Y1) - addq INCY, Y1 - vmovhpd %xmm0, (Y1) - addq INCY, Y1 - -#if GEMV_UNROLL == 2 - cmpq $2, N - jge .L61 -#endif - ALIGN_4 - -.L70: - cmpq $1, N - jl .L999 - -#endif - - leaq 16 * SIZE(BUFFER), X1 - - movq A, A1 - - vxorps %xmm0 , %xmm0, %xmm0 - vxorps %xmm1 , %xmm1, %xmm1 - vxorps %xmm2 , %xmm2, %xmm2 - vxorps %xmm3 , %xmm3, %xmm3 - -#ifdef ALIGNED_ACCESS - testq $SIZE, A - je .L7X - - vmovsd -16 * SIZE(X1), %xmm12 - vmovsd -16 * SIZE(A1), %xmm4 - - vfmaddpd %xmm0 , %xmm4 , %xmm12 , %xmm0 - - addq $SIZE, A1 - addq $SIZE, X1 - ALIGN_3 - -.L7X: -#endif - movq M, I - sarq $3, I - jle .L75 - - - VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) - VMOVUPS_XL1(-14 * SIZE, X1, %xmm13) - - decq I - jle .L73 - ALIGN_4 - -.L72: - - vfmaddpd %xmm0 , -16 * SIZE(A1) , %xmm12 , %xmm0 - vfmaddpd %xmm2 , -14 * SIZE(A1) , %xmm13 , %xmm2 - - VMOVUPS_XL1(-12 * SIZE, X1, %xmm12) - VMOVUPS_XL1(-10 * SIZE, X1, %xmm13) - - vfmaddpd %xmm0 , -12 * SIZE(A1) , %xmm12 , %xmm0 - vfmaddpd %xmm2 , -10 * SIZE(A1) , %xmm13 , %xmm2 - - VMOVUPS_XL1( -8 * SIZE, X1, %xmm12) - VMOVUPS_XL1( -6 * SIZE, X1, %xmm13) - - addq $8 * SIZE, A1 - addq $8 * SIZE, X1 - - decq I - jg .L72 - ALIGN_4 - -.L73: - - vfmaddpd %xmm0 , -16 * SIZE(A1) , %xmm12 , %xmm0 - vfmaddpd %xmm2 , -14 * SIZE(A1) , %xmm13 , %xmm2 - - VMOVUPS_XL1(-12 * SIZE, X1, %xmm12) - VMOVUPS_XL1(-10 * SIZE, X1, %xmm13) - - vfmaddpd %xmm0 , -12 * SIZE(A1) , %xmm12 , %xmm0 - vfmaddpd %xmm2 , -10 * SIZE(A1) , %xmm13 , %xmm2 - - addq $8 * SIZE, A1 - addq $8 * SIZE, X1 - ALIGN_4 - -.L75: - testq $4, M - jle .L76 - - VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) - VMOVUPS_XL1(-14 * SIZE, X1, %xmm13) - - vfmaddpd %xmm0 , -16 * SIZE(A1) , %xmm12 , %xmm0 - vfmaddpd %xmm2 , -14 * SIZE(A1) , %xmm13 , %xmm2 - - addq $4 * SIZE, A1 - addq $4 * SIZE, X1 - ALIGN_4 - -.L76: - testq $2, M - jle .L77 - - VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) - - vfmaddpd %xmm0 , -16 * SIZE(A1) , %xmm12 , %xmm0 - - addq $2 * SIZE, A1 - addq $2 * SIZE, X1 - ALIGN_4 - -.L77: - testq $1, M - je .L78 - - vmovsd -16 * SIZE(X1), %xmm12 - vmovsd -16 * SIZE(A1), %xmm4 - - vfmaddpd %xmm0 , %xmm4 , %xmm12 , %xmm0 - ALIGN_4 - -.L78: - vaddpd %xmm2, %xmm0 , %xmm0 - vaddpd %xmm3, %xmm1 , %xmm1 - - vaddpd %xmm1, %xmm0 , %xmm0 - - vhaddpd %xmm1, %xmm0 , %xmm0 - - vmulsd ALPHA, %xmm0 , %xmm0 - - vmovsd (Y), %xmm4 - addq INCY, Y - - vaddsd %xmm4, %xmm0 , %xmm0 - - vmovlpd %xmm0, (Y1) - addq INCY, Y1 -#endif ALIGN_4 .L999: diff --git a/kernel/x86_64/dtrsm_kernel_LT_8x2_bulldozer.S b/kernel/x86_64/dtrsm_kernel_LT_8x2_bulldozer.S new file mode 100644 index 000000000..374f45096 --- /dev/null +++ b/kernel/x86_64/dtrsm_kernel_LT_8x2_bulldozer.S @@ -0,0 +1,1281 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#define ASSEMBLER +#include "common.h" + +#define OLD_M %rdi +#define OLD_N %rsi +#define M %r13 +#define N %r14 +#define K %rdx + +#define A %rcx +#define B %r8 +#define C %r9 +#define LDC %r10 + +#define I %r11 +#define AO %rdi +#define BO %rsi +#define CO1 %r15 +#define CO2 %r12 +#define BB %rbp +#define J %rbx + +#ifndef WINDOWS_ABI + +#define STACKSIZE 96 + +#define OFFSET 48(%rsp) +#define AORIG 56(%rsp) +#define KK 64(%rsp) +#define KKK 72(%rsp) + +#else + +#define STACKSIZE 256 + +#define OLD_A 40 + STACKSIZE(%rsp) +#define OLD_B 48 + STACKSIZE(%rsp) +#define OLD_C 56 + STACKSIZE(%rsp) +#define OLD_LDC 64 + STACKSIZE(%rsp) +#define OLD_OFFSET 72 + STACKSIZE(%rsp) + +#define OFFSET 224(%rsp) +#define AORIG 232(%rsp) +#define KK 240(%rsp) +#define KKK 248(%rsp) + +#endif + +#define A_PR1 384 +#define B_PR1 192 + + +.macro KERNEL8x2_SUB + vmovddup -16*SIZE(BO,%rax,2), %xmm1 + vmovddup -15*SIZE(BO,%rax,2), %xmm2 + vmovups -16*SIZE(AO,%rax,8), %xmm0 + vfmaddpd %xmm8 , %xmm0 , %xmm1 , %xmm8 + vfmaddpd %xmm9 , %xmm0 , %xmm2 , %xmm9 + vmovups -14*SIZE(AO,%rax,8), %xmm4 + vfmaddpd %xmm10, %xmm4 , %xmm1 , %xmm10 + vfmaddpd %xmm11, %xmm4 , %xmm2 , %xmm11 + vmovups -12*SIZE(AO,%rax,8), %xmm5 + vfmaddpd %xmm12, %xmm5 , %xmm1 , %xmm12 + vfmaddpd %xmm13, %xmm5 , %xmm2 , %xmm13 + vmovups -10*SIZE(AO,%rax,8), %xmm6 + vfmaddpd %xmm14, %xmm6 , %xmm1 , %xmm14 + vfmaddpd %xmm15, %xmm6 , %xmm2 , %xmm15 + addq $SIZE, %rax +.endm + +.macro SOLVE_8x2 + + vmovups %xmm8 , %xmm1 + vunpcklpd %xmm9 , %xmm8 , %xmm8 + vunpckhpd %xmm9 , %xmm1 , %xmm1 + + vmovups -16 * SIZE(BO), %xmm0 + vsubpd %xmm8 , %xmm0 , %xmm0 + vmovups -14 * SIZE(BO), %xmm8 + vsubpd %xmm1 , %xmm8 , %xmm1 + + vmovups %xmm10, %xmm3 + vunpcklpd %xmm11, %xmm10 , %xmm10 + vunpckhpd %xmm11, %xmm3 , %xmm3 + + vmovups -12 * SIZE(BO), %xmm8 + vmovups -10 * SIZE(BO), %xmm9 + vsubpd %xmm10, %xmm8 , %xmm2 + vsubpd %xmm3 , %xmm9 , %xmm3 + + vmovups %xmm12, %xmm5 + vunpcklpd %xmm13, %xmm12 , %xmm12 + vunpckhpd %xmm13, %xmm5 , %xmm5 + + vmovups -8 * SIZE(BO), %xmm8 + vmovups -6 * SIZE(BO), %xmm9 + vsubpd %xmm12, %xmm8 , %xmm4 + vsubpd %xmm5 , %xmm9 , %xmm5 + + vmovups %xmm14, %xmm7 + vunpcklpd %xmm15, %xmm14 , %xmm14 + vunpckhpd %xmm15, %xmm7 , %xmm7 + + vmovups -4 * SIZE(BO), %xmm8 + vmovups -2 * SIZE(BO), %xmm9 + vsubpd %xmm14, %xmm8 , %xmm6 + vsubpd %xmm7 , %xmm9 , %xmm7 + + vmovddup -16 * SIZE(AO), %xmm8 + vmulpd %xmm0 , %xmm8 , %xmm0 + vmovddup -15 * SIZE(AO), %xmm9 + vfnmaddpd %xmm1 , %xmm0 , %xmm9 , %xmm1 + vmovddup -14 * SIZE(AO), %xmm10 + vfnmaddpd %xmm2 , %xmm0 , %xmm10, %xmm2 + vmovddup -13 * SIZE(AO), %xmm11 + vfnmaddpd %xmm3 , %xmm0 , %xmm11, %xmm3 + vmovddup -12 * SIZE(AO), %xmm8 + vfnmaddpd %xmm4 , %xmm0 , %xmm8 , %xmm4 + vmovddup -11 * SIZE(AO), %xmm9 + vfnmaddpd %xmm5 , %xmm0 , %xmm9 , %xmm5 + vmovddup -10 * SIZE(AO), %xmm10 + vfnmaddpd %xmm6 , %xmm0 , %xmm10, %xmm6 + vmovddup -9 * SIZE(AO), %xmm11 + vfnmaddpd %xmm7 , %xmm0 , %xmm11, %xmm7 + + vmovddup -7 * SIZE(AO), %xmm8 + vmulpd %xmm1 , %xmm8 , %xmm1 + vmovddup -6 * SIZE(AO), %xmm10 + vfnmaddpd %xmm2 , %xmm1 , %xmm10, %xmm2 + vmovddup -5 * SIZE(AO), %xmm11 + vfnmaddpd %xmm3 , %xmm1 , %xmm11, %xmm3 + vmovddup -4 * SIZE(AO), %xmm8 + vfnmaddpd %xmm4 , %xmm1 , %xmm8 , %xmm4 + vmovddup -3 * SIZE(AO), %xmm9 + vfnmaddpd %xmm5 , %xmm1 , %xmm9 , %xmm5 + vmovddup -2 * SIZE(AO), %xmm10 + vfnmaddpd %xmm6 , %xmm1 , %xmm10, %xmm6 + vmovddup -1 * SIZE(AO), %xmm11 + vfnmaddpd %xmm7 , %xmm1 , %xmm11, %xmm7 + + vmovddup 2 * SIZE(AO), %xmm8 + vmulpd %xmm2 , %xmm8 , %xmm2 + vmovddup 3 * SIZE(AO), %xmm11 + vfnmaddpd %xmm3 , %xmm2 , %xmm11, %xmm3 + vmovddup 4 * SIZE(AO), %xmm8 + vfnmaddpd %xmm4 , %xmm2 , %xmm8 , %xmm4 + vmovddup 5 * SIZE(AO), %xmm9 + vfnmaddpd %xmm5 , %xmm2 , %xmm9 , %xmm5 + vmovddup 6 * SIZE(AO), %xmm10 + vfnmaddpd %xmm6 , %xmm2 , %xmm10, %xmm6 + vmovddup 7 * SIZE(AO), %xmm11 + vfnmaddpd %xmm7 , %xmm2 , %xmm11, %xmm7 + + vmovddup 11 * SIZE(AO), %xmm8 + vmulpd %xmm3 , %xmm8 , %xmm3 + vmovddup 12 * SIZE(AO), %xmm11 + vfnmaddpd %xmm4 , %xmm3 , %xmm11, %xmm4 + vmovddup 13 * SIZE(AO), %xmm9 + vfnmaddpd %xmm5 , %xmm3 , %xmm9 , %xmm5 + vmovddup 14 * SIZE(AO), %xmm10 + vfnmaddpd %xmm6 , %xmm3 , %xmm10, %xmm6 + vmovddup 15 * SIZE(AO), %xmm11 + vfnmaddpd %xmm7 , %xmm3 , %xmm11, %xmm7 + + vmovddup 20 * SIZE(AO), %xmm8 + vmulpd %xmm4 , %xmm8 , %xmm4 + vmovddup 21 * SIZE(AO), %xmm9 + vfnmaddpd %xmm5 , %xmm4 , %xmm9 , %xmm5 + vmovddup 22 * SIZE(AO), %xmm10 + vfnmaddpd %xmm6 , %xmm4 , %xmm10, %xmm6 + vmovddup 23 * SIZE(AO), %xmm11 + vfnmaddpd %xmm7 , %xmm4 , %xmm11, %xmm7 + + vmovddup 29 * SIZE(AO), %xmm8 + vmulpd %xmm5 , %xmm8 , %xmm5 + vmovddup 30 * SIZE(AO), %xmm10 + vfnmaddpd %xmm6 , %xmm5 , %xmm10, %xmm6 + vmovddup 31 * SIZE(AO), %xmm11 + vfnmaddpd %xmm7 , %xmm5 , %xmm11, %xmm7 + + vmovddup 38 * SIZE(AO), %xmm8 + vmulpd %xmm6 , %xmm8 , %xmm6 + vmovddup 39 * SIZE(AO), %xmm11 + vfnmaddpd %xmm7 , %xmm6 , %xmm11, %xmm7 + + vmovddup 47 * SIZE(AO), %xmm8 + vmulpd %xmm7 , %xmm8 , %xmm7 + + + vmovsd %xmm0 , 0 * SIZE(CO1) + vmovsd %xmm1 , 1 * SIZE(CO1) + vmovsd %xmm2 , 2 * SIZE(CO1) + vmovsd %xmm3 , 3 * SIZE(CO1) + vmovsd %xmm4 , 4 * SIZE(CO1) + vmovsd %xmm5 , 5 * SIZE(CO1) + vmovsd %xmm6 , 6 * SIZE(CO1) + vmovsd %xmm7 , 7 * SIZE(CO1) + + vmovhpd %xmm0 , 0 * SIZE(CO2) + vmovhpd %xmm1 , 1 * SIZE(CO2) + vmovhpd %xmm2 , 2 * SIZE(CO2) + vmovhpd %xmm3 , 3 * SIZE(CO2) + vmovhpd %xmm4 , 4 * SIZE(CO2) + vmovhpd %xmm5 , 5 * SIZE(CO2) + vmovhpd %xmm6 , 6 * SIZE(CO2) + vmovhpd %xmm7 , 7 * SIZE(CO2) + + vmovups %xmm0 , -16 * SIZE(BO) + vmovups %xmm1 , -14 * SIZE(BO) + vmovups %xmm2 , -12 * SIZE(BO) + vmovups %xmm3 , -10 * SIZE(BO) + vmovups %xmm4 , -8 * SIZE(BO) + vmovups %xmm5 , -6 * SIZE(BO) + vmovups %xmm6 , -4 * SIZE(BO) + vmovups %xmm7 , -2 * SIZE(BO) + +.endm + + + +.macro KERNEL4x2_SUB + vmovddup -16*SIZE(BO,%rax,2), %xmm1 + vmovddup -15*SIZE(BO,%rax,2), %xmm2 + vmovups -16*SIZE(AO,%rax,4), %xmm0 + vfmaddpd %xmm8 , %xmm0 , %xmm1 , %xmm8 + vfmaddpd %xmm9 , %xmm0 , %xmm2 , %xmm9 + vmovups -14*SIZE(AO,%rax,4), %xmm0 + vfmaddpd %xmm10, %xmm0 , %xmm1 , %xmm10 + vfmaddpd %xmm11, %xmm0 , %xmm2 , %xmm11 + addq $SIZE, %rax +.endm + + +.macro SOLVE_4x2 + + vmovups %xmm8 , %xmm1 + vunpcklpd %xmm9 , %xmm8 , %xmm8 + vunpckhpd %xmm9 , %xmm1 , %xmm1 + + vmovups -16 * SIZE(BO), %xmm0 + vsubpd %xmm8 , %xmm0 , %xmm0 + vmovups -14 * SIZE(BO), %xmm8 + vsubpd %xmm1 , %xmm8 , %xmm1 + + vmovups %xmm10, %xmm3 + vunpcklpd %xmm11, %xmm10 , %xmm10 + vunpckhpd %xmm11, %xmm3 , %xmm3 + + vmovups -12 * SIZE(BO), %xmm8 + vmovups -10 * SIZE(BO), %xmm9 + vsubpd %xmm10, %xmm8 , %xmm2 + vsubpd %xmm3 , %xmm9 , %xmm3 + + vmovddup -16 * SIZE(AO), %xmm8 + vmulpd %xmm0 , %xmm8 , %xmm0 + vmovddup -15 * SIZE(AO), %xmm9 + vfnmaddpd %xmm1 , %xmm0 , %xmm9 , %xmm1 + vmovddup -14 * SIZE(AO), %xmm10 + vfnmaddpd %xmm2 , %xmm0 , %xmm10, %xmm2 + vmovddup -13 * SIZE(AO), %xmm11 + vfnmaddpd %xmm3 , %xmm0 , %xmm11, %xmm3 + + vmovddup -11 * SIZE(AO), %xmm8 + vmulpd %xmm1 , %xmm8 , %xmm1 + vmovddup -10 * SIZE(AO), %xmm10 + vfnmaddpd %xmm2 , %xmm1 , %xmm10, %xmm2 + vmovddup -9 * SIZE(AO), %xmm11 + vfnmaddpd %xmm3 , %xmm1 , %xmm11, %xmm3 + + vmovddup -6 * SIZE(AO), %xmm8 + vmulpd %xmm2 , %xmm8 , %xmm2 + vmovddup -5 * SIZE(AO), %xmm11 + vfnmaddpd %xmm3 , %xmm2 , %xmm11, %xmm3 + + vmovddup -1 * SIZE(AO), %xmm8 + vmulpd %xmm3 , %xmm8 , %xmm3 + + vmovsd %xmm0 , 0 * SIZE(CO1) + vmovsd %xmm1 , 1 * SIZE(CO1) + vmovsd %xmm2 , 2 * SIZE(CO1) + vmovsd %xmm3 , 3 * SIZE(CO1) + + vmovhpd %xmm0 , 0 * SIZE(CO2) + vmovhpd %xmm1 , 1 * SIZE(CO2) + vmovhpd %xmm2 , 2 * SIZE(CO2) + vmovhpd %xmm3 , 3 * SIZE(CO2) + + vmovups %xmm0 , -16 * SIZE(BO) + vmovups %xmm1 , -14 * SIZE(BO) + vmovups %xmm2 , -12 * SIZE(BO) + vmovups %xmm3 , -10 * SIZE(BO) + +.endm + + + +.macro KERNEL2x2_SUB + vmovddup -16*SIZE(BO,%rax,2), %xmm1 + vmovddup -15*SIZE(BO,%rax,2), %xmm2 + vmovups -16*SIZE(AO,%rax,2), %xmm0 + vfmaddpd %xmm8 , %xmm0 , %xmm1 , %xmm8 + vfmaddpd %xmm9 , %xmm0 , %xmm2 , %xmm9 + addq $SIZE, %rax +.endm + + +.macro SOLVE_2x2 + + vmovups %xmm8 , %xmm1 + vunpcklpd %xmm9 , %xmm8 , %xmm8 + vunpckhpd %xmm9 , %xmm1 , %xmm1 + + vmovups -16 * SIZE(BO), %xmm0 + vsubpd %xmm8 , %xmm0 , %xmm0 + vmovups -14 * SIZE(BO), %xmm8 + vsubpd %xmm1 , %xmm8 , %xmm1 + + vmovddup -16 * SIZE(AO), %xmm8 + vmulpd %xmm0 , %xmm8 , %xmm0 + vmovddup -15 * SIZE(AO), %xmm9 + vfnmaddpd %xmm1 , %xmm0 , %xmm9 , %xmm1 + + vmovddup -13 * SIZE(AO), %xmm8 + vmulpd %xmm1 , %xmm8 , %xmm1 + + vmovsd %xmm0 , 0 * SIZE(CO1) + vmovsd %xmm1 , 1 * SIZE(CO1) + + vmovhpd %xmm0 , 0 * SIZE(CO2) + vmovhpd %xmm1 , 1 * SIZE(CO2) + + vmovups %xmm0 , -16 * SIZE(BO) + vmovups %xmm1 , -14 * SIZE(BO) + +.endm + + + +.macro KERNEL1x2_SUB + vmovups -16*SIZE(BO,%rax,2), %xmm1 + vmovddup -16*SIZE(AO,%rax,1), %xmm0 + vfmaddpd %xmm8 , %xmm0 , %xmm1 , %xmm8 + addq $SIZE, %rax +.endm + +.macro SOLVE_1x2 + + vmovups -16 * SIZE(BO), %xmm0 + vsubpd %xmm8 , %xmm0 , %xmm0 + + vmovddup -16 * SIZE(AO), %xmm8 + vmulpd %xmm0 , %xmm8 , %xmm0 + + vmovsd %xmm0 , 0 * SIZE(CO1) + + vmovhpd %xmm0 , 0 * SIZE(CO2) + + vmovups %xmm0 , -16 * SIZE(BO) + +.endm + + +/******************************************************************************************/ + + +.macro KERNEL8x1_SUB + vmovddup -16*SIZE(BO,%rax,1), %xmm1 + vmovups -16*SIZE(AO,%rax,8), %xmm0 + vfmaddpd %xmm8 , %xmm0 , %xmm1 , %xmm8 + vmovups -14*SIZE(AO,%rax,8), %xmm0 + vfmaddpd %xmm9 , %xmm0 , %xmm1 , %xmm9 + vmovups -12*SIZE(AO,%rax,8), %xmm0 + vfmaddpd %xmm10, %xmm0 , %xmm1 , %xmm10 + vmovups -10*SIZE(AO,%rax,8), %xmm0 + vfmaddpd %xmm11, %xmm0 , %xmm1 , %xmm11 + addq $SIZE, %rax +.endm + +.macro SOLVE_8x1 + + vmovups -16 * SIZE(BO), %xmm1 + vmovups -14 * SIZE(BO), %xmm3 + vmovups -12 * SIZE(BO), %xmm5 + vmovups -10 * SIZE(BO), %xmm7 + + vsubpd %xmm8 , %xmm1 , %xmm1 + vsubpd %xmm9 , %xmm3 , %xmm3 + vsubpd %xmm10, %xmm5 , %xmm5 + vsubpd %xmm11, %xmm7 , %xmm7 + + vmovups %xmm1 , %xmm0 + vunpckhpd %xmm1 , %xmm1 , %xmm1 + + vmovups %xmm3 , %xmm2 + vunpckhpd %xmm3 , %xmm3 , %xmm3 + + vmovups %xmm5 , %xmm4 + vunpckhpd %xmm5 , %xmm5 , %xmm5 + + vmovups %xmm7 , %xmm6 + vunpckhpd %xmm7 , %xmm7 , %xmm7 + + vmulsd -16 * SIZE(AO), %xmm0 , %xmm0 + vfnmaddsd %xmm1 ,-15 * SIZE(AO), %xmm0 , %xmm1 + vfnmaddsd %xmm2 ,-14 * SIZE(AO), %xmm0 , %xmm2 + vfnmaddsd %xmm3 ,-13 * SIZE(AO), %xmm0 , %xmm3 + vfnmaddsd %xmm4 ,-12 * SIZE(AO), %xmm0 , %xmm4 + vfnmaddsd %xmm5 ,-11 * SIZE(AO), %xmm0 , %xmm5 + vfnmaddsd %xmm6 ,-10 * SIZE(AO), %xmm0 , %xmm6 + vfnmaddsd %xmm7 , -9 * SIZE(AO), %xmm0 , %xmm7 + + vmulsd -7 * SIZE(AO), %xmm1 , %xmm1 + vfnmaddsd %xmm2 , -6 * SIZE(AO), %xmm1 , %xmm2 + vfnmaddsd %xmm3 , -5 * SIZE(AO), %xmm1 , %xmm3 + vfnmaddsd %xmm4 , -4 * SIZE(AO), %xmm1 , %xmm4 + vfnmaddsd %xmm5 , -3 * SIZE(AO), %xmm1 , %xmm5 + vfnmaddsd %xmm6 , -2 * SIZE(AO), %xmm1 , %xmm6 + vfnmaddsd %xmm7 , -1 * SIZE(AO), %xmm1 , %xmm7 + + vmulsd 2 * SIZE(AO), %xmm2 , %xmm2 + vfnmaddsd %xmm3 , 3 * SIZE(AO), %xmm2 , %xmm3 + vfnmaddsd %xmm4 , 4 * SIZE(AO), %xmm2 , %xmm4 + vfnmaddsd %xmm5 , 5 * SIZE(AO), %xmm2 , %xmm5 + vfnmaddsd %xmm6 , 6 * SIZE(AO), %xmm2 , %xmm6 + vfnmaddsd %xmm7 , 7 * SIZE(AO), %xmm2 , %xmm7 + + vmulsd 11 * SIZE(AO), %xmm3 , %xmm3 + vfnmaddsd %xmm4 , 12 * SIZE(AO), %xmm3 , %xmm4 + vfnmaddsd %xmm5 , 13 * SIZE(AO), %xmm3 , %xmm5 + vfnmaddsd %xmm6 , 14 * SIZE(AO), %xmm3 , %xmm6 + vfnmaddsd %xmm7 , 15 * SIZE(AO), %xmm3 , %xmm7 + + vmulsd 20 * SIZE(AO), %xmm4 , %xmm4 + vfnmaddsd %xmm5 , 21 * SIZE(AO), %xmm4 , %xmm5 + vfnmaddsd %xmm6 , 22 * SIZE(AO), %xmm4 , %xmm6 + vfnmaddsd %xmm7 , 23 * SIZE(AO), %xmm4 , %xmm7 + + vmulsd 29 * SIZE(AO), %xmm5 , %xmm5 + vfnmaddsd %xmm6 , 30 * SIZE(AO), %xmm5 , %xmm6 + vfnmaddsd %xmm7 , 31 * SIZE(AO), %xmm5 , %xmm7 + + vmulsd 38 * SIZE(AO), %xmm6 , %xmm6 + vfnmaddsd %xmm7 , 39 * SIZE(AO), %xmm6 , %xmm7 + + vmulsd 47 * SIZE(AO), %xmm7 , %xmm7 + + vmovsd %xmm0 , 0 * SIZE(CO1) + vmovsd %xmm1 , 1 * SIZE(CO1) + vmovsd %xmm2 , 2 * SIZE(CO1) + vmovsd %xmm3 , 3 * SIZE(CO1) + vmovsd %xmm4 , 4 * SIZE(CO1) + vmovsd %xmm5 , 5 * SIZE(CO1) + vmovsd %xmm6 , 6 * SIZE(CO1) + vmovsd %xmm7 , 7 * SIZE(CO1) + + vmovsd %xmm0 , -16 * SIZE(BO) + vmovsd %xmm1 , -15 * SIZE(BO) + vmovsd %xmm2 , -14 * SIZE(BO) + vmovsd %xmm3 , -13 * SIZE(BO) + vmovsd %xmm4 , -12 * SIZE(BO) + vmovsd %xmm5 , -11 * SIZE(BO) + vmovsd %xmm6 , -10 * SIZE(BO) + vmovsd %xmm7 , -9 * SIZE(BO) + +.endm + + + +.macro KERNEL4x1_SUB + vmovddup -16*SIZE(BO,%rax,1), %xmm1 + vmovups -16*SIZE(AO,%rax,4), %xmm0 + vfmaddpd %xmm8 , %xmm0 , %xmm1 , %xmm8 + vmovups -14*SIZE(AO,%rax,4), %xmm0 + vfmaddpd %xmm9 , %xmm0 , %xmm1 , %xmm9 + addq $SIZE, %rax +.endm + + +.macro SOLVE_4x1 + + vmovups -16 * SIZE(BO), %xmm1 + vmovups -14 * SIZE(BO), %xmm3 + + vsubpd %xmm8 , %xmm1 , %xmm1 + vsubpd %xmm9 , %xmm3 , %xmm3 + + vmovups %xmm1 , %xmm0 + vunpckhpd %xmm1 , %xmm1 , %xmm1 + + vmovups %xmm3 , %xmm2 + vunpckhpd %xmm3 , %xmm3 , %xmm3 + + vmulsd -16 * SIZE(AO), %xmm0 , %xmm0 + vfnmaddsd %xmm1 ,-15 * SIZE(AO), %xmm0 , %xmm1 + vfnmaddsd %xmm2 ,-14 * SIZE(AO), %xmm0 , %xmm2 + vfnmaddsd %xmm3 ,-13 * SIZE(AO), %xmm0 , %xmm3 + + vmulsd -11 * SIZE(AO), %xmm1 , %xmm1 + vfnmaddsd %xmm2 ,-10 * SIZE(AO), %xmm1 , %xmm2 + vfnmaddsd %xmm3 , -9 * SIZE(AO), %xmm1 , %xmm3 + + vmulsd -6 * SIZE(AO), %xmm2 , %xmm2 + vfnmaddsd %xmm3 , -5 * SIZE(AO), %xmm2 , %xmm3 + + vmulsd -1 * SIZE(AO), %xmm3 , %xmm3 + + vmovsd %xmm0 , 0 * SIZE(CO1) + vmovsd %xmm1 , 1 * SIZE(CO1) + vmovsd %xmm2 , 2 * SIZE(CO1) + vmovsd %xmm3 , 3 * SIZE(CO1) + + vmovsd %xmm0 , -16 * SIZE(BO) + vmovsd %xmm1 , -15 * SIZE(BO) + vmovsd %xmm2 , -14 * SIZE(BO) + vmovsd %xmm3 , -13 * SIZE(BO) + +.endm + + + +.macro KERNEL2x1_SUB + vmovddup -16*SIZE(BO,%rax,1), %xmm1 + vmovups -16*SIZE(AO,%rax,2), %xmm0 + vfmaddpd %xmm8 , %xmm0 , %xmm1 , %xmm8 + addq $SIZE, %rax +.endm + + +.macro SOLVE_2x1 + + vmovups -16 * SIZE(BO), %xmm1 + + vsubpd %xmm8 , %xmm1 , %xmm1 + + vmovups %xmm1 , %xmm0 + vunpckhpd %xmm1 , %xmm1 , %xmm1 + + vmulsd -16 * SIZE(AO), %xmm0 , %xmm0 + vfnmaddsd %xmm1 ,-15 * SIZE(AO), %xmm0 , %xmm1 + + vmulsd -13 * SIZE(AO), %xmm1 , %xmm1 + + vmovsd %xmm0 , 0 * SIZE(CO1) + vmovsd %xmm1 , 1 * SIZE(CO1) + + vmovsd %xmm0 , -16 * SIZE(BO) + vmovsd %xmm1 , -15 * SIZE(BO) + +.endm + + + +.macro KERNEL1x1_SUB + vmovsd -16*SIZE(BO,%rax,1), %xmm1 + vmovsd -16*SIZE(AO,%rax,1), %xmm0 + vfmaddsd %xmm8 , %xmm0 , %xmm1 , %xmm8 + addq $SIZE, %rax +.endm + +.macro SOLVE_1x1 + + vmovsd -16 * SIZE(BO), %xmm1 + + vsubsd %xmm8 , %xmm1 , %xmm1 + + vmulsd -16 * SIZE(AO), %xmm1 , %xmm1 + + vmovsd %xmm1 , 0 * SIZE(CO1) + + vmovsd %xmm1 , -16 * SIZE(BO) +.endm + + + + + +/***************************************************************************************************************/ + + + PROLOGUE + PROFCODE + + subq $STACKSIZE, %rsp + movq %rbx, (%rsp) + movq %rbp, 8(%rsp) + movq %r12, 16(%rsp) + movq %r13, 24(%rsp) + movq %r14, 32(%rsp) + movq %r15, 40(%rsp) + +#ifdef WINDOWS_ABI + movq %rdi, 48(%rsp) + movq %rsi, 56(%rsp) + movups %xmm6, 64(%rsp) + movups %xmm7, 80(%rsp) + movups %xmm8, 96(%rsp) + movups %xmm9, 112(%rsp) + movups %xmm10, 128(%rsp) + movups %xmm11, 144(%rsp) + movups %xmm12, 160(%rsp) + movups %xmm13, 176(%rsp) + movups %xmm14, 192(%rsp) + movups %xmm15, 208(%rsp) + + movq ARG1, OLD_M + movq ARG2, OLD_N + movq ARG3, K + movq OLD_A, A + movq OLD_B, B + movq OLD_C, C + movq OLD_LDC, LDC + movsd OLD_OFFSET, %xmm12 +#else + movq STACKSIZE + 8(%rsp), LDC + movsd STACKSIZE + 16(%rsp), %xmm12 +#endif + + movq OLD_M, M + movq OLD_N, N + + subq $-16 * SIZE, A + subq $-16 * SIZE, B + + movsd %xmm12, OFFSET + movsd %xmm12, KK + + leaq (, LDC, SIZE), LDC + + + movq N, J + sarq $1, J # j = (n >> 1) + jle .L80 + ALIGN_4 + +.L01: + + movq A, AO + + movq C, CO1 # coffset1 = c + leaq (C, LDC, 1), CO2 # coffset2 = c + ldc + leaq (C, LDC, 2), C + + movq OFFSET, %rax + movq %rax, KK + + movq M, I + sarq $3, I # i = (m >> 3) + jle .L50_A + ALIGN_4 +/*********************************************************************************/ +.L51: + + movq B, BO + + vxorpd %xmm8 , %xmm8 , %xmm8 + vxorpd %xmm9 , %xmm9 , %xmm9 + vxorpd %xmm10, %xmm10, %xmm10 + vxorpd %xmm11, %xmm11, %xmm11 + vxorpd %xmm12, %xmm12, %xmm12 + vxorpd %xmm13, %xmm13, %xmm13 + vxorpd %xmm14, %xmm14, %xmm14 + vxorpd %xmm15, %xmm15, %xmm15 + + + movq KK, %rax + andq $-4, %rax + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 8), AO + leaq (BO, %rax, 2), BO + negq %rax + + je .L56 + ALIGN_4 + +.L52: + prefetcht0 A_PR1(AO,%rax,8) + prefetcht0 B_PR1(BO,%rax,2) + KERNEL8x2_SUB + prefetcht0 A_PR1(AO,%rax,8) + KERNEL8x2_SUB + prefetcht0 A_PR1(AO,%rax,8) + KERNEL8x2_SUB + prefetcht0 A_PR1(AO,%rax,8) + KERNEL8x2_SUB + + jl .L52 + ALIGN_4 + +.L56: + movq KK, %rax + andq $3, %rax # if (k & 1) + je .L59 + + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 8), AO + leaq (BO, %rax, 2), BO + negq %rax + ALIGN_4 + +.L57: + KERNEL8x2_SUB + + jl .L57 + ALIGN_4 + +.L59: + + SOLVE_8x2 + + addq $8 * SIZE, CO1 + addq $8 * SIZE, CO2 + + movq K, %rax + subq KK, %rax + leaq (,%rax, SIZE), %rax + leaq (AO, %rax, 8), AO + leaq (BO, %rax, 2), BO + + addq $8, KK + + decq I # i -- + jg .L51 + ALIGN_4 + +/*********************************************************************************/ + +.L50_A: + testq $4, M + je .L60 + +.L51_A: + + movq B, BO + + vxorpd %xmm8 , %xmm8 , %xmm8 + vxorpd %xmm9 , %xmm9 , %xmm9 + vxorpd %xmm10, %xmm10, %xmm10 + vxorpd %xmm11, %xmm11, %xmm11 + + movq KK, %rax + andq $-4, %rax + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 4), AO + leaq (BO, %rax, 2), BO + negq %rax + + je .L56_A + ALIGN_4 + +.L52_A: + + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + jl .L52_A + ALIGN_4 + +.L56_A: + movq KK, %rax + andq $3, %rax # if (k & 1) + je .L59_A + + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 4), AO + leaq (BO, %rax, 2), BO + negq %rax + ALIGN_4 + +.L57_A: + + KERNEL4x2_SUB + + jl .L57_A + ALIGN_4 + +.L59_A: + + SOLVE_4x2 + + addq $4 * SIZE, CO1 + addq $4 * SIZE, CO2 + + movq K, %rax + subq KK, %rax + leaq (,%rax, SIZE), %rax + leaq (AO, %rax, 4), AO + leaq (BO, %rax, 2), BO + + addq $4, KK + + ALIGN_4 + +/*********************************************************************************/ + + +.L60: + testq $2, M + je .L70 + +.L61: + movq B, BO + + vxorpd %xmm8, %xmm8 , %xmm8 + vxorpd %xmm9, %xmm9 , %xmm9 + + movq KK, %rax + andq $-4, %rax + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 2), AO + leaq (BO, %rax, 2), BO + negq %rax + + je .L66 + ALIGN_4 + +.L62: + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + jl .L62 + ALIGN_4 + +.L66: + movq KK, %rax + andq $3, %rax # if (k & 1) + je .L69 + + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 2), AO + leaq (BO, %rax, 2), BO + negq %rax + ALIGN_4 + +.L67: + + KERNEL2x2_SUB + + jl .L67 + ALIGN_4 + +.L69: + + SOLVE_2x2 + + addq $2 * SIZE, CO1 + addq $2 * SIZE, CO2 + + movq K, %rax + subq KK, %rax + leaq (,%rax, SIZE), %rax + leaq (AO, %rax, 2), AO + leaq (BO, %rax, 2), BO + + addq $2, KK + + ALIGN_4 +/********************************************************************************/ +.L70: + testq $1, M + je .L79 + ALIGN_4 + +.L71: + movq B, BO + + vxorpd %xmm8, %xmm8 , %xmm8 + + movq KK, %rax + andq $-4, %rax + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 1), AO + leaq (BO, %rax, 2), BO + negq %rax + + je .L76 + ALIGN_4 + +.L72: + + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + jl .L72 + ALIGN_4 + +.L76: + movq KK, %rax + andq $3, %rax # if (k & 1) + je .L78 + + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 1), AO + leaq (BO, %rax, 2), BO + negq %rax + ALIGN_4 + +.L77: + + KERNEL1x2_SUB + + jl .L77 + ALIGN_4 + +.L78: + + SOLVE_1x2 + + addq $1 * SIZE, CO1 + addq $1 * SIZE, CO2 + + movq K, %rax + subq KK, %rax + leaq (,%rax, SIZE), %rax + leaq (AO, %rax, 1), AO + leaq (BO, %rax, 2), BO + + addq $1, KK + + ALIGN_4 + +.L79: + + movq BO, B + + decq J # j -- + jg .L01 + ALIGN_4 +/***************************************************************************************/ +.L80: + testq $1, N + je .L999 + + movq A, AO + movq C, CO1 # coffset1 = c + + movq OFFSET, %rax + movq %rax, KK + + movq M, I + sarq $3, I # i = (m >> 3) + jle .L90_A + ALIGN_4 +/*************************************************************************************/ +.L91: + + movq B, BO + + vxorpd %xmm8, %xmm8 , %xmm8 + vxorpd %xmm9, %xmm9 , %xmm9 + vxorpd %xmm10, %xmm10, %xmm10 + vxorpd %xmm11, %xmm11, %xmm11 + + + movq KK, %rax + andq $-4, %rax + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 8), AO + leaq (BO, %rax, 1), BO + negq %rax + + je .L96 + ALIGN_4 + +.L92: + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + + jl .L92 + ALIGN_4 + +.L96: + movq KK, %rax + andq $3, %rax # if (k & 1) + je .L99 + + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 8), AO + leaq (BO, %rax, 1), BO + negq %rax + ALIGN_4 + +.L97: + KERNEL8x1_SUB + + jl .L97 + ALIGN_4 +.L99: + + SOLVE_8x1 + + addq $8 * SIZE, CO1 + + movq K, %rax + subq KK, %rax + leaq (,%rax, SIZE), %rax + leaq (AO, %rax, 8), AO + addq %rax, BO + + addq $8, KK + + + decq I # i -- + jg .L91 + ALIGN_4 + +/*****************************************************************************/ +.L90_A: + testq $4, M + je .L100 + +.L91_A: + movq B, BO + + vxorpd %xmm8, %xmm8 , %xmm8 + vxorpd %xmm9, %xmm9 , %xmm9 + + movq KK, %rax + andq $-4, %rax + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 4), AO + leaq (BO, %rax, 1), BO + negq %rax + + je .L96_A + ALIGN_4 + +.L92_A: + + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + jl .L92_A + ALIGN_4 + +.L96_A: + movq KK, %rax + andq $3, %rax # if (k & 1) + je .L99_A + + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 4), AO + leaq (BO, %rax, 1), BO + negq %rax + ALIGN_4 + +.L97_A: + + KERNEL4x1_SUB + + jl .L97_A + ALIGN_4 +.L99_A: + + SOLVE_4x1 + + addq $4 * SIZE, CO1 + + movq K, %rax + subq KK, %rax + leaq (,%rax, SIZE), %rax + leaq (AO, %rax, 4), AO + addq %rax, BO + + addq $4, KK + + + ALIGN_4 + +/*************************************************************************************/ +.L100: + testq $2, M + je .L110 + + + + movq B, BO + + vxorpd %xmm8, %xmm8 , %xmm8 + + movq KK, %rax + andq $-4, %rax + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 2), AO + leaq (BO, %rax, 1), BO + negq %rax + + je .L106 + ALIGN_4 + +.L102: + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + jl .L102 + ALIGN_4 + +.L106: + movq KK, %rax + andq $3, %rax # if (k & 1) + je .L109 + + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 2), AO + leaq (BO, %rax, 1), BO + negq %rax + ALIGN_4 + +.L107: + + KERNEL2x1_SUB + + jl .L107 + ALIGN_4 + +.L109: + + SOLVE_2x1 + + addq $2 * SIZE, CO1 + + movq K, %rax + subq KK, %rax + leaq (,%rax, SIZE), %rax + leaq (AO, %rax, 2), AO + addq %rax, BO + + addq $2, KK + + ALIGN_4 + +.L110: + testq $1, M + je .L119 + ALIGN_4 + +.L111: + movq B, BO + + vxorpd %xmm8, %xmm8 , %xmm8 + + movq KK, %rax + andq $-4, %rax + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 1), AO + leaq (BO, %rax, 1), BO + negq %rax + + je .L116 + ALIGN_4 + +.L112: + + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + jl .L112 + ALIGN_4 + +.L116: + movq KK, %rax + andq $3, %rax # if (k & 1) + je .L118 + + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 1), AO + leaq (BO, %rax, 1), BO + negq %rax + ALIGN_4 + +.L117: + + KERNEL1x1_SUB + + jl .L117 + ALIGN_4 + +.L118: + + SOLVE_1x1 + + addq $1 * SIZE, CO1 + + movq K, %rax + subq KK, %rax + leaq (,%rax, SIZE), %rax + addq %rax, AO + addq %rax, BO + + addq $1, KK + + ALIGN_4 + +.L119: + + movq BO, B + + + ALIGN_4 + + +.L999: + movq (%rsp), %rbx + movq 8(%rsp), %rbp + movq 16(%rsp), %r12 + movq 24(%rsp), %r13 + movq 32(%rsp), %r14 + movq 40(%rsp), %r15 + +#ifdef WINDOWS_ABI + movq 48(%rsp), %rdi + movq 56(%rsp), %rsi + movups 64(%rsp), %xmm6 + movups 80(%rsp), %xmm7 + movups 96(%rsp), %xmm8 + movups 112(%rsp), %xmm9 + movups 128(%rsp), %xmm10 + movups 144(%rsp), %xmm11 + movups 160(%rsp), %xmm12 + movups 176(%rsp), %xmm13 + movups 192(%rsp), %xmm14 + movups 208(%rsp), %xmm15 +#endif + + addq $STACKSIZE, %rsp + ret + + EPILOGUE diff --git a/kernel/x86_64/dtrsm_kernel_RN_8x2_bulldozer.S b/kernel/x86_64/dtrsm_kernel_RN_8x2_bulldozer.S new file mode 100644 index 000000000..8fa53efa7 --- /dev/null +++ b/kernel/x86_64/dtrsm_kernel_RN_8x2_bulldozer.S @@ -0,0 +1,1075 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#define ASSEMBLER +#include "common.h" + +#define OLD_M %rdi +#define OLD_N %rsi +#define M %r13 +#define N %r14 +#define K %rdx + +#define A %rcx +#define B %r8 +#define C %r9 +#define LDC %r10 + +#define I %r11 +#define AO %rdi +#define BO %rsi +#define CO1 %r15 +#define CO2 %r12 +#define BB %rbp +#define J %rbx + +#ifndef WINDOWS_ABI + +#define STACKSIZE 96 + +#define OFFSET 48(%rsp) +#define AORIG 56(%rsp) +#define KK 64(%rsp) +#define KKK 72(%rsp) + +#else + +#define STACKSIZE 256 + +#define OLD_A 40 + STACKSIZE(%rsp) +#define OLD_B 48 + STACKSIZE(%rsp) +#define OLD_C 56 + STACKSIZE(%rsp) +#define OLD_LDC 64 + STACKSIZE(%rsp) +#define OLD_OFFSET 72 + STACKSIZE(%rsp) + +#define OFFSET 224(%rsp) +#define AORIG 232(%rsp) +#define KK 240(%rsp) +#define KKK 248(%rsp) + +#endif + +#define A_PR1 384 +#define B_PR1 192 + + +.macro KERNEL8x2_SUB + vmovddup -16*SIZE(BO,%rax,2), %xmm1 + vmovddup -15*SIZE(BO,%rax,2), %xmm2 + vmovups -16*SIZE(AO,%rax,8), %xmm0 + vfmaddpd %xmm8 , %xmm0 , %xmm1 , %xmm8 + vfmaddpd %xmm9 , %xmm0 , %xmm2 , %xmm9 + vmovups -14*SIZE(AO,%rax,8), %xmm4 + vfmaddpd %xmm10, %xmm4 , %xmm1 , %xmm10 + vfmaddpd %xmm11, %xmm4 , %xmm2 , %xmm11 + vmovups -12*SIZE(AO,%rax,8), %xmm5 + vfmaddpd %xmm12, %xmm5 , %xmm1 , %xmm12 + vfmaddpd %xmm13, %xmm5 , %xmm2 , %xmm13 + vmovups -10*SIZE(AO,%rax,8), %xmm6 + vfmaddpd %xmm14, %xmm6 , %xmm1 , %xmm14 + vfmaddpd %xmm15, %xmm6 , %xmm2 , %xmm15 + addq $SIZE, %rax +.endm + +.macro SOLVE_8x2 + + vmovups -16 * SIZE(AO), %xmm0 + vmovups -14 * SIZE(AO), %xmm1 + vmovups -12 * SIZE(AO), %xmm2 + vmovups -10 * SIZE(AO), %xmm3 + vmovups -8 * SIZE(AO), %xmm4 + vmovups -6 * SIZE(AO), %xmm5 + vmovups -4 * SIZE(AO), %xmm6 + vmovups -2 * SIZE(AO), %xmm7 + + vsubpd %xmm8 , %xmm0 , %xmm0 + vsubpd %xmm10, %xmm1 , %xmm1 + vsubpd %xmm12, %xmm2 , %xmm2 + vsubpd %xmm14, %xmm3 , %xmm3 + vsubpd %xmm9 , %xmm4 , %xmm4 + vsubpd %xmm11, %xmm5 , %xmm5 + vsubpd %xmm13, %xmm6 , %xmm6 + vsubpd %xmm15, %xmm7 , %xmm7 + + vmovddup -16 * SIZE(BO), %xmm8 + vmulpd %xmm0 , %xmm8 , %xmm0 + vmulpd %xmm1 , %xmm8 , %xmm1 + vmulpd %xmm2 , %xmm8 , %xmm2 + vmulpd %xmm3 , %xmm8 , %xmm3 + + vmovddup -15 * SIZE(BO), %xmm9 + vfnmaddpd %xmm4 , %xmm0 , %xmm9 , %xmm4 + vfnmaddpd %xmm5 , %xmm1 , %xmm9 , %xmm5 + vfnmaddpd %xmm6 , %xmm2 , %xmm9 , %xmm6 + vfnmaddpd %xmm7 , %xmm3 , %xmm9 , %xmm7 + + vmovddup -13 * SIZE(BO), %xmm10 + vmulpd %xmm4 , %xmm10, %xmm4 + vmulpd %xmm5 , %xmm10, %xmm5 + vmulpd %xmm6 , %xmm10, %xmm6 + vmulpd %xmm7 , %xmm10, %xmm7 + + vmovups %xmm0 , 0 * SIZE(CO1) + vmovups %xmm1 , 2 * SIZE(CO1) + vmovups %xmm2 , 4 * SIZE(CO1) + vmovups %xmm3 , 6 * SIZE(CO1) + + vmovups %xmm4 , 0 * SIZE(CO2) + vmovups %xmm5 , 2 * SIZE(CO2) + vmovups %xmm6 , 4 * SIZE(CO2) + vmovups %xmm7 , 6 * SIZE(CO2) + + vmovups %xmm0 , -16 * SIZE(AO) + vmovups %xmm1 , -14 * SIZE(AO) + vmovups %xmm2 , -12 * SIZE(AO) + vmovups %xmm3 , -10 * SIZE(AO) + vmovups %xmm4 , -8 * SIZE(AO) + vmovups %xmm5 , -6 * SIZE(AO) + vmovups %xmm6 , -4 * SIZE(AO) + vmovups %xmm7 , -2 * SIZE(AO) + + +.endm + + + +.macro KERNEL4x2_SUB + vmovddup -16*SIZE(BO,%rax,2), %xmm1 + vmovddup -15*SIZE(BO,%rax,2), %xmm2 + vmovups -16*SIZE(AO,%rax,4), %xmm0 + vfmaddpd %xmm8 , %xmm0 , %xmm1 , %xmm8 + vfmaddpd %xmm9 , %xmm0 , %xmm2 , %xmm9 + vmovups -14*SIZE(AO,%rax,4), %xmm0 + vfmaddpd %xmm10, %xmm0 , %xmm1 , %xmm10 + vfmaddpd %xmm11, %xmm0 , %xmm2 , %xmm11 + addq $SIZE, %rax +.endm + + +.macro SOLVE_4x2 + + vmovups -16 * SIZE(AO), %xmm0 + vmovups -14 * SIZE(AO), %xmm1 + vmovups -12 * SIZE(AO), %xmm2 + vmovups -10 * SIZE(AO), %xmm3 + + vsubpd %xmm8 , %xmm0 , %xmm0 + vsubpd %xmm10, %xmm1 , %xmm1 + vsubpd %xmm9 , %xmm2 , %xmm2 + vsubpd %xmm11, %xmm3 , %xmm3 + + vmovddup -16 * SIZE(BO), %xmm8 + vmulpd %xmm0 , %xmm8 , %xmm0 + vmulpd %xmm1 , %xmm8 , %xmm1 + + vmovddup -15 * SIZE(BO), %xmm9 + vfnmaddpd %xmm2 , %xmm0 , %xmm9 , %xmm2 + vfnmaddpd %xmm3 , %xmm1 , %xmm9 , %xmm3 + + vmovddup -13 * SIZE(BO), %xmm10 + vmulpd %xmm2 , %xmm10, %xmm2 + vmulpd %xmm3 , %xmm10, %xmm3 + + vmovups %xmm0 , 0 * SIZE(CO1) + vmovups %xmm1 , 2 * SIZE(CO1) + + vmovups %xmm2 , 0 * SIZE(CO2) + vmovups %xmm3 , 2 * SIZE(CO2) + + vmovups %xmm0 , -16 * SIZE(AO) + vmovups %xmm1 , -14 * SIZE(AO) + vmovups %xmm2 , -12 * SIZE(AO) + vmovups %xmm3 , -10 * SIZE(AO) + +.endm + + + +.macro KERNEL2x2_SUB + vmovddup -16*SIZE(BO,%rax,2), %xmm1 + vmovddup -15*SIZE(BO,%rax,2), %xmm2 + vmovups -16*SIZE(AO,%rax,2), %xmm0 + vfmaddpd %xmm8 , %xmm0 , %xmm1 , %xmm8 + vfmaddpd %xmm9 , %xmm0 , %xmm2 , %xmm9 + addq $SIZE, %rax +.endm + + +.macro SOLVE_2x2 + + vmovups -16 * SIZE(AO), %xmm0 + vmovups -14 * SIZE(AO), %xmm2 + + vsubpd %xmm8 , %xmm0 , %xmm0 + vsubpd %xmm9 , %xmm2 , %xmm2 + + vmovddup -16 * SIZE(BO), %xmm8 + vmulpd %xmm0 , %xmm8 , %xmm0 + + vmovddup -15 * SIZE(BO), %xmm9 + vfnmaddpd %xmm2 , %xmm0 , %xmm9 , %xmm2 + + vmovddup -13 * SIZE(BO), %xmm10 + vmulpd %xmm2 , %xmm10, %xmm2 + + vmovups %xmm0 , 0 * SIZE(CO1) + + vmovups %xmm2 , 0 * SIZE(CO2) + + vmovups %xmm0 , -16 * SIZE(AO) + vmovups %xmm2 , -14 * SIZE(AO) + +.endm + + + +.macro KERNEL1x2_SUB + vmovups -16*SIZE(BO,%rax,2), %xmm1 + vmovddup -16*SIZE(AO,%rax,1), %xmm0 + vfmaddpd %xmm8 , %xmm0 , %xmm1 , %xmm8 + addq $SIZE, %rax +.endm + +.macro SOLVE_1x2 + + vmovups -16 * SIZE(AO), %xmm2 + + vsubpd %xmm8 , %xmm2 , %xmm2 + + vmovups %xmm2 , %xmm0 + vunpckhpd %xmm0 , %xmm0 , %xmm0 + + vmovsd -16 * SIZE(BO), %xmm8 + vmulsd %xmm2 , %xmm8 , %xmm2 + + vmovsd -15 * SIZE(BO), %xmm9 + vfnmaddsd %xmm0 , %xmm2 , %xmm9 , %xmm0 + + vmovsd -13 * SIZE(BO), %xmm10 + vmulsd %xmm0 , %xmm10, %xmm0 + + vmovsd %xmm2 , 0 * SIZE(CO1) + + vmovsd %xmm0 , 0 * SIZE(CO2) + + vmovsd %xmm2 , -16 * SIZE(AO) + vmovsd %xmm0 , -15 * SIZE(AO) + +.endm + + +/******************************************************************************************/ + + +.macro KERNEL8x1_SUB + vmovddup -16*SIZE(BO,%rax,1), %xmm1 + vmovups -16*SIZE(AO,%rax,8), %xmm0 + vfmaddpd %xmm8 , %xmm0 , %xmm1 , %xmm8 + vmovups -14*SIZE(AO,%rax,8), %xmm0 + vfmaddpd %xmm9 , %xmm0 , %xmm1 , %xmm9 + vmovups -12*SIZE(AO,%rax,8), %xmm0 + vfmaddpd %xmm10, %xmm0 , %xmm1 , %xmm10 + vmovups -10*SIZE(AO,%rax,8), %xmm0 + vfmaddpd %xmm11, %xmm0 , %xmm1 , %xmm11 + addq $SIZE, %rax +.endm + +.macro SOLVE_8x1 + + vmovups -16 * SIZE(AO), %xmm0 + vmovups -14 * SIZE(AO), %xmm1 + vmovups -12 * SIZE(AO), %xmm2 + vmovups -10 * SIZE(AO), %xmm3 + + vsubpd %xmm8 , %xmm0 , %xmm0 + vsubpd %xmm9 , %xmm1 , %xmm1 + vsubpd %xmm10, %xmm2 , %xmm2 + vsubpd %xmm11, %xmm3 , %xmm3 + + vmovddup -16 * SIZE(BO), %xmm8 + vmulpd %xmm0 , %xmm8 , %xmm0 + vmulpd %xmm1 , %xmm8 , %xmm1 + vmulpd %xmm2 , %xmm8 , %xmm2 + vmulpd %xmm3 , %xmm8 , %xmm3 + + vmovups %xmm0 , 0 * SIZE(CO1) + vmovups %xmm1 , 2 * SIZE(CO1) + vmovups %xmm2 , 4 * SIZE(CO1) + vmovups %xmm3 , 6 * SIZE(CO1) + + vmovups %xmm0 , -16 * SIZE(AO) + vmovups %xmm1 , -14 * SIZE(AO) + vmovups %xmm2 , -12 * SIZE(AO) + vmovups %xmm3 , -10 * SIZE(AO) + +.endm + + + +.macro KERNEL4x1_SUB + vmovddup -16*SIZE(BO,%rax,1), %xmm1 + vmovups -16*SIZE(AO,%rax,4), %xmm0 + vfmaddpd %xmm8 , %xmm0 , %xmm1 , %xmm8 + vmovups -14*SIZE(AO,%rax,4), %xmm0 + vfmaddpd %xmm9 , %xmm0 , %xmm1 , %xmm9 + addq $SIZE, %rax +.endm + + +.macro SOLVE_4x1 + + vmovups -16 * SIZE(AO), %xmm0 + vmovups -14 * SIZE(AO), %xmm1 + + vsubpd %xmm8 , %xmm0 , %xmm0 + vsubpd %xmm9 , %xmm1 , %xmm1 + + vmovddup -16 * SIZE(BO), %xmm8 + vmulpd %xmm0 , %xmm8 , %xmm0 + vmulpd %xmm1 , %xmm8 , %xmm1 + + vmovups %xmm0 , 0 * SIZE(CO1) + vmovups %xmm1 , 2 * SIZE(CO1) + + vmovups %xmm0 , -16 * SIZE(AO) + vmovups %xmm1 , -14 * SIZE(AO) + +.endm + + + +.macro KERNEL2x1_SUB + vmovddup -16*SIZE(BO,%rax,1), %xmm1 + vmovups -16*SIZE(AO,%rax,2), %xmm0 + vfmaddpd %xmm8 , %xmm0 , %xmm1 , %xmm8 + addq $SIZE, %rax +.endm + + +.macro SOLVE_2x1 + + vmovups -16 * SIZE(AO), %xmm1 + + vsubpd %xmm8 , %xmm1 , %xmm1 + + vmovddup -16 * SIZE(BO), %xmm8 + vmulpd %xmm1 , %xmm8 , %xmm1 + + vmovups %xmm1 , 0 * SIZE(CO1) + + vmovups %xmm1 , -16 * SIZE(AO) + +.endm + + + +.macro KERNEL1x1_SUB + vmovsd -16*SIZE(BO,%rax,1), %xmm1 + vmovsd -16*SIZE(AO,%rax,1), %xmm0 + vfmaddsd %xmm8 , %xmm0 , %xmm1 , %xmm8 + addq $SIZE, %rax +.endm + +.macro SOLVE_1x1 + + vmovsd -16 * SIZE(AO), %xmm1 + + vsubsd %xmm8 , %xmm1 , %xmm1 + + vmulsd -16 * SIZE(BO), %xmm1 , %xmm1 + + vmovsd %xmm1 , 0 * SIZE(CO1) + + vmovsd %xmm1 , -16 * SIZE(AO) + +.endm + + + + + +/***************************************************************************************************************/ + + + PROLOGUE + PROFCODE + + subq $STACKSIZE, %rsp + movq %rbx, (%rsp) + movq %rbp, 8(%rsp) + movq %r12, 16(%rsp) + movq %r13, 24(%rsp) + movq %r14, 32(%rsp) + movq %r15, 40(%rsp) + +#ifdef WINDOWS_ABI + movq %rdi, 48(%rsp) + movq %rsi, 56(%rsp) + movups %xmm6, 64(%rsp) + movups %xmm7, 80(%rsp) + movups %xmm8, 96(%rsp) + movups %xmm9, 112(%rsp) + movups %xmm10, 128(%rsp) + movups %xmm11, 144(%rsp) + movups %xmm12, 160(%rsp) + movups %xmm13, 176(%rsp) + movups %xmm14, 192(%rsp) + movups %xmm15, 208(%rsp) + + movq ARG1, OLD_M + movq ARG2, OLD_N + movq ARG3, K + movq OLD_A, A + movq OLD_B, B + movq OLD_C, C + movq OLD_LDC, LDC + movsd OLD_OFFSET, %xmm12 +#else + movq STACKSIZE + 8(%rsp), LDC + movsd STACKSIZE + 16(%rsp), %xmm12 +#endif + + movq OLD_M, M + movq OLD_N, N + + subq $-16 * SIZE, A + subq $-16 * SIZE, B + + movsd %xmm12, OFFSET + movsd %xmm12, KK + + negq KK // for RN Kernel + + leaq (, LDC, SIZE), LDC + + + movq N, J + sarq $1, J # j = (n >> 1) + jle .L80 + ALIGN_4 + +.L01: + + movq A, AO + + movq C, CO1 # coffset1 = c + leaq (C, LDC, 1), CO2 # coffset2 = c + ldc + leaq (C, LDC, 2), C + + movq M, I + sarq $3, I # i = (m >> 3) + jle .L50_A + ALIGN_4 +/*********************************************************************************/ +.L51: + + movq B, BO + + vxorpd %xmm8 , %xmm8 , %xmm8 + vxorpd %xmm9 , %xmm9 , %xmm9 + vxorpd %xmm10, %xmm10, %xmm10 + vxorpd %xmm11, %xmm11, %xmm11 + vxorpd %xmm12, %xmm12, %xmm12 + vxorpd %xmm13, %xmm13, %xmm13 + vxorpd %xmm14, %xmm14, %xmm14 + vxorpd %xmm15, %xmm15, %xmm15 + + + movq KK, %rax + andq $-4, %rax + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 8), AO + leaq (BO, %rax, 2), BO + negq %rax + + je .L56 + ALIGN_4 + +.L52: + prefetcht0 A_PR1(AO,%rax,8) + prefetcht0 B_PR1(BO,%rax,2) + KERNEL8x2_SUB + prefetcht0 A_PR1(AO,%rax,8) + KERNEL8x2_SUB + prefetcht0 A_PR1(AO,%rax,8) + KERNEL8x2_SUB + prefetcht0 A_PR1(AO,%rax,8) + KERNEL8x2_SUB + + jl .L52 + ALIGN_4 + +.L56: + movq KK, %rax + andq $3, %rax # if (k & 1) + je .L59 + + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 8), AO + leaq (BO, %rax, 2), BO + negq %rax + ALIGN_4 + +.L57: + KERNEL8x2_SUB + + jl .L57 + ALIGN_4 + +.L59: + + SOLVE_8x2 + + addq $8 * SIZE, CO1 + addq $8 * SIZE, CO2 + + movq K, %rax + subq KK, %rax + leaq (,%rax, SIZE), %rax + leaq (AO, %rax, 8), AO + leaq (BO, %rax, 2), BO + + decq I # i -- + jg .L51 + ALIGN_4 + +/*********************************************************************************/ + +.L50_A: + testq $4, M + je .L60 + +.L51_A: + + movq B, BO + + vxorpd %xmm8 , %xmm8 , %xmm8 + vxorpd %xmm9 , %xmm9 , %xmm9 + vxorpd %xmm10, %xmm10, %xmm10 + vxorpd %xmm11, %xmm11, %xmm11 + + movq KK, %rax + andq $-4, %rax + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 4), AO + leaq (BO, %rax, 2), BO + negq %rax + + je .L56_A + ALIGN_4 + +.L52_A: + + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + jl .L52_A + ALIGN_4 + +.L56_A: + movq KK, %rax + andq $3, %rax # if (k & 1) + je .L59_A + + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 4), AO + leaq (BO, %rax, 2), BO + negq %rax + ALIGN_4 + +.L57_A: + + KERNEL4x2_SUB + + jl .L57_A + ALIGN_4 + +.L59_A: + + SOLVE_4x2 + + addq $4 * SIZE, CO1 + addq $4 * SIZE, CO2 + + movq K, %rax + subq KK, %rax + leaq (,%rax, SIZE), %rax + leaq (AO, %rax, 4), AO + leaq (BO, %rax, 2), BO + + ALIGN_4 + +/*********************************************************************************/ + + +.L60: + testq $2, M + je .L70 + +.L61: + movq B, BO + + vxorpd %xmm8, %xmm8 , %xmm8 + vxorpd %xmm9, %xmm9 , %xmm9 + + movq KK, %rax + andq $-4, %rax + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 2), AO + leaq (BO, %rax, 2), BO + negq %rax + + je .L66 + ALIGN_4 + +.L62: + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + jl .L62 + ALIGN_4 + +.L66: + movq KK, %rax + andq $3, %rax # if (k & 1) + je .L69 + + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 2), AO + leaq (BO, %rax, 2), BO + negq %rax + ALIGN_4 + +.L67: + + KERNEL2x2_SUB + + jl .L67 + ALIGN_4 + +.L69: + + SOLVE_2x2 + + addq $2 * SIZE, CO1 + addq $2 * SIZE, CO2 + + movq K, %rax + subq KK, %rax + leaq (,%rax, SIZE), %rax + leaq (AO, %rax, 2), AO + leaq (BO, %rax, 2), BO + + ALIGN_4 +/********************************************************************************/ +.L70: + testq $1, M + je .L79 + ALIGN_4 + +.L71: + movq B, BO + + vxorpd %xmm8, %xmm8 , %xmm8 + + movq KK, %rax + andq $-4, %rax + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 1), AO + leaq (BO, %rax, 2), BO + negq %rax + + je .L76 + ALIGN_4 + +.L72: + + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + jl .L72 + ALIGN_4 + +.L76: + movq KK, %rax + andq $3, %rax # if (k & 1) + je .L78 + + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 1), AO + leaq (BO, %rax, 2), BO + negq %rax + ALIGN_4 + +.L77: + + KERNEL1x2_SUB + + jl .L77 + ALIGN_4 + +.L78: + + SOLVE_1x2 + + addq $1 * SIZE, CO1 + addq $1 * SIZE, CO2 + + movq K, %rax + subq KK, %rax + leaq (,%rax, SIZE), %rax + leaq (AO, %rax, 1), AO + leaq (BO, %rax, 2), BO + + ALIGN_4 + +.L79: + + addq $2, KK // number of values in B # only for RN Kernel + + movq BO, B + + decq J # j -- + jg .L01 + ALIGN_4 +/***************************************************************************************/ +.L80: + testq $1, N + je .L999 + + movq A, AO + movq C, CO1 # coffset1 = c + + movq M, I + sarq $3, I # i = (m >> 3) + jle .L90_A + ALIGN_4 +/*************************************************************************************/ +.L91: + + movq B, BO + + vxorpd %xmm8, %xmm8 , %xmm8 + vxorpd %xmm9, %xmm9 , %xmm9 + vxorpd %xmm10, %xmm10, %xmm10 + vxorpd %xmm11, %xmm11, %xmm11 + + + movq KK, %rax + andq $-4, %rax + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 8), AO + leaq (BO, %rax, 1), BO + negq %rax + + je .L96 + ALIGN_4 + +.L92: + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + + jl .L92 + ALIGN_4 + +.L96: + movq KK, %rax + andq $3, %rax # if (k & 1) + je .L99 + + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 8), AO + leaq (BO, %rax, 1), BO + negq %rax + ALIGN_4 + +.L97: + KERNEL8x1_SUB + + jl .L97 + ALIGN_4 +.L99: + + SOLVE_8x1 + + addq $8 * SIZE, CO1 + + movq K, %rax + subq KK, %rax + leaq (,%rax, SIZE), %rax + leaq (AO, %rax, 8), AO + addq %rax, BO + + + decq I # i -- + jg .L91 + ALIGN_4 + +/*****************************************************************************/ +.L90_A: + testq $4, M + je .L100 + +.L91_A: + movq B, BO + + vxorpd %xmm8, %xmm8 , %xmm8 + vxorpd %xmm9, %xmm9 , %xmm9 + + movq KK, %rax + andq $-4, %rax + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 4), AO + leaq (BO, %rax, 1), BO + negq %rax + + je .L96_A + ALIGN_4 + +.L92_A: + + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + jl .L92_A + ALIGN_4 + +.L96_A: + movq KK, %rax + andq $3, %rax # if (k & 1) + je .L99_A + + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 4), AO + leaq (BO, %rax, 1), BO + negq %rax + ALIGN_4 + +.L97_A: + + KERNEL4x1_SUB + + jl .L97_A + ALIGN_4 +.L99_A: + + SOLVE_4x1 + + addq $4 * SIZE, CO1 + + movq K, %rax + subq KK, %rax + leaq (,%rax, SIZE), %rax + leaq (AO, %rax, 4), AO + addq %rax, BO + + ALIGN_4 + +/*************************************************************************************/ +.L100: + testq $2, M + je .L110 + + + + movq B, BO + + vxorpd %xmm8, %xmm8 , %xmm8 + + movq KK, %rax + andq $-4, %rax + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 2), AO + leaq (BO, %rax, 1), BO + negq %rax + + je .L106 + ALIGN_4 + +.L102: + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + jl .L102 + ALIGN_4 + +.L106: + movq KK, %rax + andq $3, %rax # if (k & 1) + je .L109 + + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 2), AO + leaq (BO, %rax, 1), BO + negq %rax + ALIGN_4 + +.L107: + + KERNEL2x1_SUB + + jl .L107 + ALIGN_4 + +.L109: + + SOLVE_2x1 + + addq $2 * SIZE, CO1 + + movq K, %rax + subq KK, %rax + leaq (,%rax, SIZE), %rax + leaq (AO, %rax, 2), AO + addq %rax, BO + + ALIGN_4 + +.L110: + testq $1, M + je .L119 + ALIGN_4 + +.L111: + movq B, BO + + vxorpd %xmm8, %xmm8 , %xmm8 + + movq KK, %rax + andq $-4, %rax + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 1), AO + leaq (BO, %rax, 1), BO + negq %rax + + je .L116 + ALIGN_4 + +.L112: + + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + jl .L112 + ALIGN_4 + +.L116: + movq KK, %rax + andq $3, %rax # if (k & 1) + je .L118 + + leaq (, %rax, SIZE), %rax + leaq (AO, %rax, 1), AO + leaq (BO, %rax, 1), BO + negq %rax + ALIGN_4 + +.L117: + + KERNEL1x1_SUB + + jl .L117 + ALIGN_4 + +.L118: + + SOLVE_1x1 + + addq $1 * SIZE, CO1 + + movq K, %rax + subq KK, %rax + leaq (,%rax, SIZE), %rax + addq %rax, AO + addq %rax, BO + + ALIGN_4 + +.L119: + + addq $1 , KK // number of values in B # only for RN Kernel + + movq BO, B + + + ALIGN_4 + + +.L999: + movq (%rsp), %rbx + movq 8(%rsp), %rbp + movq 16(%rsp), %r12 + movq 24(%rsp), %r13 + movq 32(%rsp), %r14 + movq 40(%rsp), %r15 + +#ifdef WINDOWS_ABI + movq 48(%rsp), %rdi + movq 56(%rsp), %rsi + movups 64(%rsp), %xmm6 + movups 80(%rsp), %xmm7 + movups 96(%rsp), %xmm8 + movups 112(%rsp), %xmm9 + movups 128(%rsp), %xmm10 + movups 144(%rsp), %xmm11 + movups 160(%rsp), %xmm12 + movups 176(%rsp), %xmm13 + movups 192(%rsp), %xmm14 + movups 208(%rsp), %xmm15 +#endif + + addq $STACKSIZE, %rsp + ret + + EPILOGUE diff --git a/kernel/x86_64/sgemm_kernel_16x2_bulldozer.S b/kernel/x86_64/sgemm_kernel_16x2_bulldozer.S index f02a1dfa5..2a034f056 100644 --- a/kernel/x86_64/sgemm_kernel_16x2_bulldozer.S +++ b/kernel/x86_64/sgemm_kernel_16x2_bulldozer.S @@ -981,6 +981,8 @@ /*******************************************************************************************/ +#if !defined(TRMMKERNEL) + PROLOGUE PROFCODE @@ -1016,16 +1018,11 @@ movq OLD_B, B movq OLD_C, C movq OLD_LDC, LDC -#ifdef TRMMKERNEL - movsd OLD_OFFSET, %xmm12 -#endif + vmovaps %xmm3, %xmm0 #else movq STACKSIZE + 8(%rsp), LDC -#ifdef TRMMKERNEL - movsd STACKSIZE + 16(%rsp), %xmm12 -#endif #endif @@ -1059,15 +1056,6 @@ movq %rax, Ndiv6 // N / 6 movq %rdx, Nmod6 // N % 6 - - -#ifdef TRMMKERNEL - vmovsd %xmm12, OFFSET - vmovsd %xmm12, KK -#ifndef LEFT - negq KK -#endif -#endif movq Ndiv6, J cmpq $0, J @@ -1248,10 +1236,6 @@ leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc -#if defined(TRMMKERNEL) && defined(LEFT) - movq OFFSET, %rax - movq %rax, KK -#endif movq A, AO // aoffset = a addq $32 * SIZE, AO @@ -1263,40 +1247,12 @@ ALIGN_4 .L6_11: -#if !defined(TRMMKERNEL) || \ - (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO -#else - movq KK, %rax - leaq BUFFER1, BO // first buffer to BO - addq $6 * SIZE, BO - movq %rax, BI // Index for BO - leaq (BI,BI,2), BI // BI = BI * 3 ; number of values - leaq (BO, BI, SIZE), BO - salq $4, %rax // rax = rax * 16 ; number of values - leaq (AO, %rax, SIZE), AO -#endif - vzeroall -#ifndef TRMMKERNEL movq K, %rax -#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) - movq K, %rax - subq KK, %rax - movq %rax, KKK -#else - movq KK, %rax -#ifdef LEFT - addq $16, %rax // number of values in AO -#else - addq $3, %rax // number of values in BO -#endif - movq %rax, KKK -#endif andq $-8, %rax // K = K - ( K % 8 ) je .L6_16 @@ -1343,11 +1299,7 @@ ALIGN_4 .L6_16: -#ifndef TRMMKERNEL movq K, %rax -#else - movq KKK, %rax -#endif andq $7, %rax # if (k & 1) je .L6_19 @@ -1375,8 +1327,6 @@ vbroadcastss ALPHA, %xmm0 -#ifndef TRMMKERNEL - vfmaddps (CO1),%xmm0, %xmm4,%xmm4 vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7 vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10 @@ -1392,23 +1342,6 @@ vfmaddps 8 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12 vfmaddps 12 * SIZE(CO1, LDC, 2),%xmm0, %xmm15,%xmm15 -#else - vmulps %xmm0, %xmm4,%xmm4 - vmulps %xmm0, %xmm7,%xmm7 - vmulps %xmm0, %xmm10,%xmm10 - vmulps %xmm0, %xmm13,%xmm13 - - vmulps %xmm0, %xmm5,%xmm5 - vmulps %xmm0, %xmm8,%xmm8 - vmulps %xmm0, %xmm11,%xmm11 - vmulps %xmm0, %xmm14,%xmm14 - - vmulps %xmm0, %xmm6,%xmm6 - vmulps %xmm0, %xmm9,%xmm9 - vmulps %xmm0, %xmm12,%xmm12 - vmulps %xmm0, %xmm15,%xmm15 - -#endif vmovups %xmm4 , (CO1) vmovups %xmm7 , 4 * SIZE(CO1) @@ -1425,21 +1358,6 @@ vmovups %xmm12, 8 * SIZE(CO1, LDC, 2) vmovups %xmm15,12 * SIZE(CO1, LDC, 2) -#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq K, %rax - subq KKK, %rax - movq %rax, BI // Index for BO - leaq (BI,BI,2), BI // BI = BI * 3 ; number of values - leaq (BO, BI, SIZE), BO - salq $4, %rax // rax = rax * 16 ; number of values - leaq (AO, %rax, SIZE), AO -#endif - - -#if defined(TRMMKERNEL) && defined(LEFT) - addq $16, KK -#endif addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- @@ -1462,41 +1380,12 @@ /**************************************************************************/ .L6_20_1: -#if !defined(TRMMKERNEL) || \ - (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO -#else - movq KK, %rax - leaq BUFFER1, BO // first buffer to BO - addq $6 * SIZE, BO - movq %rax, BI // Index for BO - leaq (BI,BI,2), BI // BI = BI * 3 ; number of values - leaq (BO, BI, SIZE), BO - salq $3, %rax // rax = rax * 8 ; number of values - leaq (AO, %rax, SIZE), AO -#endif - vzeroall -#ifndef TRMMKERNEL movq K, %rax -#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) - movq K, %rax - subq KK, %rax - movq %rax, KKK -#else - movq KK, %rax -#ifdef LEFT - addq $8, %rax // number of values in A -#else - addq $3, %rax // number of values in BO -#endif - movq %rax, KKK -#endif - andq $-8, %rax je .L6_20_6 @@ -1543,11 +1432,7 @@ ALIGN_4 .L6_20_6: -#ifndef TRMMKERNEL movq K, %rax -#else - movq KKK, %rax -#endif andq $7, %rax # if (k & 1) je .L6_20_9 @@ -1575,8 +1460,6 @@ vbroadcastss ALPHA, %xmm0 -#ifndef TRMMKERNEL - vfmaddps (CO1),%xmm0, %xmm4,%xmm4 vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7 @@ -1586,17 +1469,6 @@ vfmaddps (CO1, LDC, 2),%xmm0, %xmm6,%xmm6 vfmaddps 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9 -#else - vmulps %xmm0, %xmm4,%xmm4 - vmulps %xmm0, %xmm7,%xmm7 - - vmulps %xmm0, %xmm5,%xmm5 - vmulps %xmm0, %xmm8,%xmm8 - - vmulps %xmm0, %xmm6,%xmm6 - vmulps %xmm0, %xmm9,%xmm9 - -#endif vmovups %xmm4 , (CO1) vmovups %xmm7 , 4 * SIZE(CO1) @@ -1608,22 +1480,6 @@ vmovups %xmm9 , 4 * SIZE(CO1, LDC, 2) -#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq K, %rax - subq KKK, %rax - movq %rax, BI // Index for BO - leaq (BI,BI,2), BI // BI = BI * 3 ; number of values - leaq (BO, BI, SIZE), BO - salq $3, %rax // rax = rax * 8 ; number of values - leaq (AO, %rax, SIZE), AO -#endif - - -#if defined(TRMMKERNEL) && defined(LEFT) - addq $8, KK -#endif - addq $8 * SIZE, CO1 # coffset += 8 ALIGN_4 @@ -1638,41 +1494,12 @@ ALIGN_4 .L6_21: -#if !defined(TRMMKERNEL) || \ - (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO -#else - movq KK, %rax - leaq BUFFER1, BO // first buffer to BO - addq $6 * SIZE, BO - movq %rax, BI // Index for BO - leaq (BI,BI,2), BI // BI = BI * 3 ; number of values - leaq (BO, BI, SIZE), BO - salq $2, %rax // rax = rax * 4 ; number of values - leaq (AO, %rax, SIZE), AO -#endif - vzeroall -#ifndef TRMMKERNEL movq K, %rax -#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) - movq K, %rax - subq KK, %rax - movq %rax, KKK -#else - movq KK, %rax -#ifdef LEFT - addq $4, %rax // number of values in A -#else - addq $3, %rax // number of values in BO -#endif - movq %rax, KKK -#endif - andq $-8, %rax je .L6_26 @@ -1719,11 +1546,7 @@ ALIGN_4 .L6_26: -#ifndef TRMMKERNEL movq K, %rax -#else - movq KKK, %rax -#endif andq $7, %rax # if (k & 1) je .L6_29 @@ -1751,40 +1574,15 @@ vbroadcastss ALPHA, %xmm0 -#ifndef TRMMKERNEL - vfmaddps (CO1),%xmm0, %xmm4,%xmm4 vfmaddps (CO1, LDC),%xmm0, %xmm5,%xmm5 vfmaddps (CO1, LDC, 2),%xmm0, %xmm6,%xmm6 -#else - vmulps %xmm0, %xmm4,%xmm4 - vmulps %xmm0, %xmm5,%xmm5 - vmulps %xmm0, %xmm6,%xmm6 - -#endif vmovups %xmm4 , (CO1) vmovups %xmm5 , (CO1, LDC) vmovups %xmm6 , (CO1, LDC, 2) - -#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq K, %rax - subq KKK, %rax - movq %rax, BI // Index for BO - leaq (BI,BI,2), BI // BI = BI * 3 ; number of values - leaq (BO, BI, SIZE), BO - salq $2, %rax // rax = rax * 4 ; number of values - leaq (AO, %rax, SIZE), AO -#endif - - -#if defined(TRMMKERNEL) && defined(LEFT) - addq $4, KK -#endif - addq $4 * SIZE, CO1 # coffset += 4 ALIGN_4 @@ -1796,41 +1594,12 @@ ALIGN_4 .L6_31: -#if !defined(TRMMKERNEL) || \ - (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO -#else - movq KK, %rax - leaq BUFFER1, BO // first buffer to BO - addq $6 * SIZE, BO - movq %rax, BI // Index for BO - leaq (BI,BI,2), BI // BI = BI * 3 ; number of values - leaq (BO, BI, SIZE), BO - salq $1, %rax // rax = rax * 2 ; number of values - leaq (AO, %rax, SIZE), AO -#endif - vzeroall -#ifndef TRMMKERNEL movq K, %rax -#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) - movq K, %rax - subq KK, %rax - movq %rax, KKK -#else - movq KK, %rax -#ifdef LEFT - addq $2, %rax // number of values in AO -#else - addq $3, %rax // number of values in BO -#endif - movq %rax, KKK -#endif - andq $-8, %rax je .L6_36 @@ -1877,11 +1646,7 @@ ALIGN_4 .L6_36: -#ifndef TRMMKERNEL movq K, %rax -#else - movq KKK, %rax -#endif andq $7, %rax # if (k & 1) je .L6_39 @@ -1909,8 +1674,6 @@ vmovss ALPHA, %xmm0 -#ifndef TRMMKERNEL - vfmaddss (CO1),%xmm0, %xmm4,%xmm4 vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8 vfmaddss (CO1, LDC),%xmm0, %xmm5,%xmm5 @@ -1918,15 +1681,6 @@ vfmaddss (CO1, LDC, 2),%xmm0, %xmm6,%xmm6 vfmaddss 1 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12 -#else - vmulss %xmm0, %xmm4,%xmm4 - vmulss %xmm0, %xmm8,%xmm8 - vmulss %xmm0, %xmm5,%xmm5 - vmulss %xmm0, %xmm10,%xmm10 - vmulss %xmm0, %xmm6,%xmm6 - vmulss %xmm0, %xmm12,%xmm12 - -#endif vmovss %xmm4 , (CO1) vmovss %xmm8 , 1 * SIZE(CO1) @@ -1935,22 +1689,6 @@ vmovss %xmm6 , (CO1, LDC, 2) vmovss %xmm12, 1 * SIZE(CO1, LDC, 2) -#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq K, %rax - subq KKK, %rax - movq %rax, BI // Index for BO - leaq (BI,BI,2), BI // BI = BI * 3 ; number of values - leaq (BO, BI, SIZE), BO - salq $1, %rax // rax = rax * 2 ; number of values - leaq (AO, %rax, SIZE), AO -#endif - - -#if defined(TRMMKERNEL) && defined(LEFT) - addq $2, KK -#endif - addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 @@ -1961,39 +1699,12 @@ ALIGN_4 .L6_41: -#if !defined(TRMMKERNEL) || \ - (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO -#else - movq KK, %rax - leaq BUFFER1, BO // first buffer to BO - addq $6 * SIZE, BO - movq %rax, BI // Index for BO - leaq (BI,BI,2), BI // BI = BI * 3 ; number of values - leaq (BO, BI, SIZE), BO - leaq (AO, %rax, SIZE), AO -#endif - vzeroall -#ifndef TRMMKERNEL movq K, %rax -#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) - movq K, %rax - subq KK, %rax - movq %rax, KKK -#else - movq KK, %rax -#ifdef LEFT - addq $1, %rax // number of values in AO -#else - addq $3, %rax // number of values in BO -#endif - movq %rax, KKK -#endif andq $-8, %rax je .L6_46 @@ -2036,11 +1747,7 @@ ALIGN_4 .L6_46: -#ifndef TRMMKERNEL movq K, %rax -#else - movq KKK, %rax -#endif andq $7, %rax # if (k & 1) je .L6_49 @@ -2067,38 +1774,15 @@ vmovss ALPHA, %xmm0 -#ifndef TRMMKERNEL - vfmaddss (CO1),%xmm0, %xmm4,%xmm4 vfmaddss (CO1, LDC),%xmm0, %xmm5,%xmm5 vfmaddss (CO1, LDC, 2),%xmm0, %xmm6,%xmm6 -#else - vmulss %xmm0, %xmm4,%xmm4 - vmulss %xmm0, %xmm5,%xmm5 - vmulss %xmm0, %xmm6,%xmm6 - -#endif vmovss %xmm4 , (CO1) vmovss %xmm5 , (CO1, LDC) vmovss %xmm6 , (CO1, LDC, 2) -#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq K, %rax - subq KKK, %rax - movq %rax, BI // Index for BO - leaq (BI,BI,2), BI // BI = BI * 3 ; number of values - leaq (BO, BI, SIZE), BO - leaq (AO, %rax, SIZE), AO -#endif - - -#if defined(TRMMKERNEL) && defined(LEFT) - addq $1, KK -#endif - addq $1 * SIZE, CO1 # coffset += 1 ALIGN_4 @@ -2112,10 +1796,6 @@ leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc -#if defined(TRMMKERNEL) && defined(LEFT) - movq OFFSET, %rax - movq %rax, KK -#endif movq A, AO // aoffset = a addq $32 * SIZE, AO @@ -2127,40 +1807,12 @@ ALIGN_4 .L7_11: -#if !defined(TRMMKERNEL) || \ - (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO -#else - movq KK, %rax - leaq BUFFER2, BO // second buffer to BO - addq $6 * SIZE, BO - movq %rax, BI // Index for BO - leaq (BI,BI,2), BI // BI = BI * 3 ; number of values - leaq (BO, BI, SIZE), BO - salq $4, %rax // rax = rax * 16 ; number of values - leaq (AO, %rax, SIZE), AO -#endif - vzeroall -#ifndef TRMMKERNEL movq K, %rax -#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) - movq K, %rax - subq KK, %rax - movq %rax, KKK -#else - movq KK, %rax -#ifdef LEFT - addq $16, %rax // number of values in AO -#else - addq $3, %rax // number of values in BO -#endif - movq %rax, KKK -#endif andq $-8, %rax // K = K - ( K % 8 ) je .L7_16 @@ -2207,11 +1859,7 @@ ALIGN_4 .L7_16: -#ifndef TRMMKERNEL movq K, %rax -#else - movq KKK, %rax -#endif andq $7, %rax # if (k & 1) je .L7_19 @@ -2239,8 +1887,6 @@ vbroadcastss ALPHA, %xmm0 -#ifndef TRMMKERNEL - vfmaddps (CO1),%xmm0, %xmm4,%xmm4 vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7 vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10 @@ -2256,23 +1902,6 @@ vfmaddps 8 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12 vfmaddps 12 * SIZE(CO1, LDC, 2),%xmm0, %xmm15,%xmm15 -#else - vmulps %xmm0, %xmm4,%xmm4 - vmulps %xmm0, %xmm7,%xmm7 - vmulps %xmm0, %xmm10,%xmm10 - vmulps %xmm0, %xmm13,%xmm13 - - vmulps %xmm0, %xmm5,%xmm5 - vmulps %xmm0, %xmm8,%xmm8 - vmulps %xmm0, %xmm11,%xmm11 - vmulps %xmm0, %xmm14,%xmm14 - - vmulps %xmm0, %xmm6,%xmm6 - vmulps %xmm0, %xmm9,%xmm9 - vmulps %xmm0, %xmm12,%xmm12 - vmulps %xmm0, %xmm15,%xmm15 - -#endif vmovups %xmm4 , (CO1) vmovups %xmm7 , 4 * SIZE(CO1) @@ -2289,21 +1918,6 @@ vmovups %xmm12, 8 * SIZE(CO1, LDC, 2) vmovups %xmm15,12 * SIZE(CO1, LDC, 2) -#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq K, %rax - subq KKK, %rax - movq %rax, BI // Index for BO - leaq (BI,BI,2), BI // BI = BI * 3 ; number of values - leaq (BO, BI, SIZE), BO - salq $4, %rax // rax = rax * 16 ; number of values - leaq (AO, %rax, SIZE), AO -#endif - - -#if defined(TRMMKERNEL) && defined(LEFT) - addq $16, KK -#endif addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- @@ -2326,41 +1940,12 @@ /**************************************************************************/ .L7_20_1: -#if !defined(TRMMKERNEL) || \ - (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) leaq BUFFER2, BO // first buffer to BO addq $6 * SIZE, BO -#else - movq KK, %rax - leaq BUFFER2, BO // first buffer to BO - addq $6 * SIZE, BO - movq %rax, BI // Index for BO - leaq (BI,BI,2), BI // BI = BI * 3 ; number of values - leaq (BO, BI, SIZE), BO - salq $3, %rax // rax = rax * 8 ; number of values - leaq (AO, %rax, SIZE), AO -#endif - vzeroall -#ifndef TRMMKERNEL movq K, %rax -#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) - movq K, %rax - subq KK, %rax - movq %rax, KKK -#else - movq KK, %rax -#ifdef LEFT - addq $8, %rax // number of values in A -#else - addq $3, %rax // number of values in BO -#endif - movq %rax, KKK -#endif - andq $-8, %rax je .L7_20_6 @@ -2407,11 +1992,7 @@ ALIGN_4 .L7_20_6: -#ifndef TRMMKERNEL movq K, %rax -#else - movq KKK, %rax -#endif andq $7, %rax # if (k & 1) je .L7_20_9 @@ -2434,12 +2015,10 @@ jl .L7_20_7 ALIGN_4 - .L7_20_9: - vbroadcastss ALPHA, %xmm0 -#ifndef TRMMKERNEL + vbroadcastss ALPHA, %xmm0 vfmaddps (CO1),%xmm0, %xmm4,%xmm4 vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7 @@ -2450,18 +2029,6 @@ vfmaddps (CO1, LDC, 2),%xmm0, %xmm6,%xmm6 vfmaddps 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9 -#else - vmulps %xmm0, %xmm4,%xmm4 - vmulps %xmm0, %xmm7,%xmm7 - - vmulps %xmm0, %xmm5,%xmm5 - vmulps %xmm0, %xmm8,%xmm8 - - vmulps %xmm0, %xmm6,%xmm6 - vmulps %xmm0, %xmm9,%xmm9 - -#endif - vmovups %xmm4 , (CO1) vmovups %xmm7 , 4 * SIZE(CO1) @@ -2472,22 +2039,6 @@ vmovups %xmm9 , 4 * SIZE(CO1, LDC, 2) -#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq K, %rax - subq KKK, %rax - movq %rax, BI // Index for BO - leaq (BI,BI,2), BI // BI = BI * 3 ; number of values - leaq (BO, BI, SIZE), BO - salq $3, %rax // rax = rax * 8 ; number of values - leaq (AO, %rax, SIZE), AO -#endif - - -#if defined(TRMMKERNEL) && defined(LEFT) - addq $8, KK -#endif - addq $8 * SIZE, CO1 # coffset += 8 ALIGN_4 @@ -2502,41 +2053,12 @@ ALIGN_4 .L7_21: -#if !defined(TRMMKERNEL) || \ - (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO -#else - movq KK, %rax - leaq BUFFER2, BO // second buffer to BO - addq $6 * SIZE, BO - movq %rax, BI // Index for BO - leaq (BI,BI,2), BI // BI = BI * 3 ; number of values - leaq (BO, BI, SIZE), BO - salq $2, %rax // rax = rax * 4 ; number of values - leaq (AO, %rax, SIZE), AO -#endif - vzeroall -#ifndef TRMMKERNEL movq K, %rax -#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) - movq K, %rax - subq KK, %rax - movq %rax, KKK -#else - movq KK, %rax -#ifdef LEFT - addq $4, %rax // number of values in A -#else - addq $3, %rax // number of values in BO -#endif - movq %rax, KKK -#endif - andq $-8, %rax je .L7_26 @@ -2583,11 +2105,7 @@ ALIGN_4 .L7_26: -#ifndef TRMMKERNEL movq K, %rax -#else - movq KKK, %rax -#endif andq $7, %rax # if (k & 1) je .L7_29 @@ -2615,40 +2133,14 @@ vbroadcastss ALPHA, %xmm0 -#ifndef TRMMKERNEL - vfmaddps (CO1),%xmm0, %xmm4,%xmm4 vfmaddps (CO1, LDC),%xmm0, %xmm5,%xmm5 vfmaddps (CO1, LDC, 2),%xmm0, %xmm6 ,%xmm6 -#else - vmulps %xmm0, %xmm4,%xmm4 - vmulps %xmm0, %xmm5,%xmm5 - vmulps %xmm0, %xmm6,%xmm6 - -#endif - vmovups %xmm4 , (CO1) vmovups %xmm5 , (CO1, LDC) vmovups %xmm6 , (CO1, LDC, 2) - -#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq K, %rax - subq KKK, %rax - movq %rax, BI // Index for BO - leaq (BI,BI,2), BI // BI = BI * 3 ; number of values - leaq (BO, BI, SIZE), BO - salq $2, %rax // rax = rax * 4 ; number of values - leaq (AO, %rax, SIZE), AO -#endif - - -#if defined(TRMMKERNEL) && defined(LEFT) - addq $4, KK -#endif - addq $4 * SIZE, CO1 # coffset += 4 ALIGN_4 @@ -2660,41 +2152,12 @@ ALIGN_4 .L7_31: -#if !defined(TRMMKERNEL) || \ - (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO -#else - movq KK, %rax - leaq BUFFER2, BO // second buffer to BO - addq $6 * SIZE, BO - movq %rax, BI // Index for BO - leaq (BI,BI,2), BI // BI = BI * 3 ; number of values - leaq (BO, BI, SIZE), BO - salq $1, %rax // rax = rax * 2 ; number of values - leaq (AO, %rax, SIZE), AO -#endif - vzeroall -#ifndef TRMMKERNEL movq K, %rax -#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) - movq K, %rax - subq KK, %rax - movq %rax, KKK -#else - movq KK, %rax -#ifdef LEFT - addq $2, %rax // number of values in AO -#else - addq $3, %rax // number of values in BO -#endif - movq %rax, KKK -#endif - andq $-8, %rax je .L7_36 @@ -2741,11 +2204,7 @@ ALIGN_4 .L7_36: -#ifndef TRMMKERNEL movq K, %rax -#else - movq KKK, %rax -#endif andq $7, %rax # if (k & 1) je .L7_39 @@ -2773,8 +2232,6 @@ vmovss ALPHA, %xmm0 -#ifndef TRMMKERNEL - vfmaddss (CO1),%xmm0, %xmm4,%xmm4 vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8 vfmaddss (CO1, LDC),%xmm0, %xmm5,%xmm5 @@ -2782,16 +2239,6 @@ vfmaddss (CO1, LDC, 2),%xmm0, %xmm6,%xmm6 vfmaddss 1 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12 -#else - vmulss %xmm0, %xmm4,%xmm4 - vmulss %xmm0, %xmm8,%xmm8 - vmulss %xmm0, %xmm5,%xmm5 - vmulss %xmm0, %xmm10,%xmm10 - vmulss %xmm0, %xmm6,%xmm6 - vmulss %xmm0, %xmm12,%xmm12 - -#endif - vmovss %xmm4 , (CO1) vmovss %xmm8 , 1 * SIZE(CO1) vmovss %xmm5 , (CO1, LDC) @@ -2799,22 +2246,6 @@ vmovss %xmm6 , (CO1, LDC, 2) vmovss %xmm12, 1 * SIZE(CO1, LDC, 2) -#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq K, %rax - subq KKK, %rax - movq %rax, BI // Index for BO - leaq (BI,BI,2), BI // BI = BI * 3 ; number of values - leaq (BO, BI, SIZE), BO - salq $1, %rax // rax = rax * 2 ; number of values - leaq (AO, %rax, SIZE), AO -#endif - - -#if defined(TRMMKERNEL) && defined(LEFT) - addq $2, KK -#endif - addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 @@ -2825,39 +2256,12 @@ ALIGN_4 .L7_41: -#if !defined(TRMMKERNEL) || \ - (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO -#else - movq KK, %rax - leaq BUFFER2, BO // second buffer to BO - addq $6 * SIZE, BO - movq %rax, BI // Index for BO - leaq (BI,BI,2), BI // BI = BI * 3 ; number of values - leaq (BO, BI, SIZE), BO - leaq (AO, %rax, SIZE), AO -#endif - vzeroall -#ifndef TRMMKERNEL movq K, %rax -#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) - movq K, %rax - subq KK, %rax - movq %rax, KKK -#else - movq KK, %rax -#ifdef LEFT - addq $1, %rax // number of values in AO -#else - addq $3, %rax // number of values in BO -#endif - movq %rax, KKK -#endif andq $-8, %rax je .L7_46 @@ -2900,11 +2304,7 @@ ALIGN_4 .L7_46: -#ifndef TRMMKERNEL movq K, %rax -#else - movq KKK, %rax -#endif andq $7, %rax # if (k & 1) je .L7_49 @@ -2931,38 +2331,14 @@ vmovss ALPHA, %xmm0 -#ifndef TRMMKERNEL - vfmaddss (CO1),%xmm0, %xmm4,%xmm4 vfmaddss (CO1, LDC),%xmm0, %xmm5,%xmm5 vfmaddss (CO1, LDC, 2),%xmm0, %xmm6,%xmm6 -#else - vmulss %xmm0, %xmm4,%xmm4 - vmulss %xmm0, %xmm5,%xmm5 - vmulss %xmm0, %xmm6,%xmm6 - -#endif - vmovss %xmm4 , (CO1) vmovss %xmm5 , (CO1, LDC) vmovss %xmm6 , (CO1, LDC, 2) -#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq K, %rax - subq KKK, %rax - movq %rax, BI // Index for BO - leaq (BI,BI,2), BI // BI = BI * 3 ; number of values - leaq (BO, BI, SIZE), BO - leaq (AO, %rax, SIZE), AO -#endif - - -#if defined(TRMMKERNEL) && defined(LEFT) - addq $1, KK -#endif - addq $1 * SIZE, CO1 # coffset += 1 ALIGN_4 @@ -3007,6 +2383,1195 @@ movq BO1, B // next offset of B +.L2_10: + movq C, CO1 + leaq (C, LDC, 2), C // c += 2 * ldc + + + movq A, AO // aoffset = a + addq $32 * SIZE, AO + + movq M, I + sarq $4, I // i = (m >> 4) + je .L2_20 + + ALIGN_4 + +.L2_11: + leaq BUFFER1, BO // first buffer to BO + addq $4 * SIZE, BO + + vzeroall + + movq K, %rax + + andq $-8, %rax // K = K - ( K % 8 ) + je .L2_16 + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + + salq $4, %rax // rax = rax * 16 ; number of values + leaq (AO, %rax, SIZE), AO + leaq (BO, BI, SIZE), BO + negq BI + negq %rax + ALIGN_4 + +.L2_12: + + prefetcht0 B_PR1(BO,BI, SIZE) + KERNEL16x2_1(xxx) + KERNEL16x2_2(xxx) + KERNEL16x2_3(xxx) + KERNEL16x2_4(xxx) + + KERNEL16x2_1(xxx) + KERNEL16x2_2(xxx) + KERNEL16x2_3(xxx) + KERNEL16x2_4(xxx) + + je .L2_16 + + prefetcht0 B_PR1(BO,BI, SIZE) + KERNEL16x2_1(xxx) + KERNEL16x2_2(xxx) + KERNEL16x2_3(xxx) + KERNEL16x2_4(xxx) + + KERNEL16x2_1(xxx) + KERNEL16x2_2(xxx) + KERNEL16x2_3(xxx) + KERNEL16x2_4(xxx) + + je .L2_16 + + jmp .L2_12 + ALIGN_4 + +.L2_16: + movq K, %rax + + andq $7, %rax # if (k & 1) + je .L2_19 + + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + + salq $4, %rax // rax = rax * 16 ; number of values + leaq (AO, %rax, SIZE), AO + leaq (BO, BI, SIZE), BO + negq BI + negq %rax + ALIGN_4 + +.L2_17: + + KERNEL16x2_SUB(xxx) + addq $2, BI + addq $16, %rax + jl .L2_17 + ALIGN_4 + + +.L2_19: + + vbroadcastss ALPHA, %xmm0 + + vfmaddps (CO1),%xmm0, %xmm4,%xmm4 + vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7 + vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10 + vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13 + + vfmaddps (CO1, LDC),%xmm0, %xmm5,%xmm5 + vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8 + vfmaddps 8 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11 + vfmaddps 12 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14 + + vmovups %xmm4 , (CO1) + vmovups %xmm7 , 4 * SIZE(CO1) + vmovups %xmm10, 8 * SIZE(CO1) + vmovups %xmm13,12 * SIZE(CO1) + + vmovups %xmm5 , (CO1, LDC) + vmovups %xmm8 , 4 * SIZE(CO1, LDC) + vmovups %xmm11, 8 * SIZE(CO1, LDC) + vmovups %xmm14,12 * SIZE(CO1, LDC) + + addq $16 * SIZE, CO1 # coffset += 16 + decq I # i -- + jg .L2_11 + ALIGN_4 + +/************************************************************************** +* Rest of M +***************************************************************************/ +.L2_20: + // Test rest of M + + testq $15, M + jz .L2_60 // to next 3 lines of N + + testq $8, M + jz .L2_21pre + ALIGN_4 + +/**************************************************************************/ + +.L2_20_1: + leaq BUFFER1, BO // first buffer to BO + addq $4 * SIZE, BO + + vzeroall + + movq K, %rax + + andq $-8, %rax + je .L2_20_6 + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + + salq $3, %rax // rax = rax * 8 ; number of values + leaq (AO, %rax, SIZE), AO + leaq (BO, BI, SIZE), BO + negq BI + negq %rax + ALIGN_4 + +.L2_20_2: + + prefetcht0 B_PR1(BO,BI, SIZE) + KERNEL8x2_1(xxx) + KERNEL8x2_2(xxx) + KERNEL8x2_3(xxx) + KERNEL8x2_4(xxx) + + KERNEL8x2_1(xxx) + KERNEL8x2_2(xxx) + KERNEL8x2_3(xxx) + KERNEL8x2_4(xxx) + + je .L2_20_6 + + prefetcht0 B_PR1(BO,BI, SIZE) + KERNEL8x2_1(xxx) + KERNEL8x2_2(xxx) + KERNEL8x2_3(xxx) + KERNEL8x2_4(xxx) + + KERNEL8x2_1(xxx) + KERNEL8x2_2(xxx) + KERNEL8x2_3(xxx) + KERNEL8x2_4(xxx) + + je .L2_20_6 + + jmp .L2_20_2 + ALIGN_4 + +.L2_20_6: + movq K, %rax + + andq $7, %rax # if (k & 1) + je .L2_20_9 + + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + + salq $3, %rax // rax = rax * 8 ; number of values + leaq (AO, %rax, SIZE), AO + leaq (BO, BI, SIZE), BO + negq BI + negq %rax + ALIGN_4 + +.L2_20_7: + + KERNEL8x2_SUB(xxx) + addq $2, BI + addq $8, %rax + jl .L2_20_7 + ALIGN_4 + + +.L2_20_9: + + vbroadcastss ALPHA, %xmm0 + + vfmaddps (CO1),%xmm0, %xmm4,%xmm4 + vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7 + + vfmaddps (CO1, LDC),%xmm0, %xmm5,%xmm5 + vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8 + + vmovups %xmm4 , (CO1) + vmovups %xmm7 , 4 * SIZE(CO1) + + vmovups %xmm5 , (CO1, LDC) + vmovups %xmm8 , 4 * SIZE(CO1, LDC) + + + addq $8 * SIZE, CO1 # coffset += 8 + ALIGN_4 + + + +/**************************************************************************/ + +.L2_21pre: + + testq $4, M + jz .L2_30 + ALIGN_4 + +.L2_21: + leaq BUFFER1, BO // first buffer to BO + addq $4 * SIZE, BO + + vzeroall + + movq K, %rax + + andq $-8, %rax + je .L2_26 + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 1 ; number of values + + salq $2, %rax // rax = rax * 4 ; number of values + leaq (AO, %rax, SIZE), AO + leaq (BO, BI, SIZE), BO + negq BI + negq %rax + ALIGN_4 + +.L2_22: + + prefetcht0 B_PR1(BO,BI, SIZE) + KERNEL4x2_1(xxx) + KERNEL4x2_2(xxx) + KERNEL4x2_3(xxx) + KERNEL4x2_4(xxx) + + KERNEL4x2_1(xxx) + KERNEL4x2_2(xxx) + KERNEL4x2_3(xxx) + KERNEL4x2_4(xxx) + + je .L2_26 + + prefetcht0 B_PR1(BO,BI, SIZE) + KERNEL4x2_1(xxx) + KERNEL4x2_2(xxx) + KERNEL4x2_3(xxx) + KERNEL4x2_4(xxx) + + KERNEL4x2_1(xxx) + KERNEL4x2_2(xxx) + KERNEL4x2_3(xxx) + KERNEL4x2_4(xxx) + + je .L2_26 + + jmp .L2_22 + ALIGN_4 + +.L2_26: + movq K, %rax + + andq $7, %rax # if (k & 1) + je .L2_29 + + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + + salq $2, %rax // rax = rax * 4 ; number of values + leaq (AO, %rax, SIZE), AO + leaq (BO, BI, SIZE), BO + negq BI + negq %rax + ALIGN_4 + +.L2_27: + + KERNEL4x2_SUB(xxx) + addq $2, BI + addq $4, %rax + jl .L2_27 + ALIGN_4 + + +.L2_29: + + vbroadcastss ALPHA, %xmm0 + + vfmaddps (CO1),%xmm0, %xmm4,%xmm4 + vfmaddps (CO1, LDC),%xmm0, %xmm5,%xmm5 + + vmovups %xmm4 , (CO1) + vmovups %xmm5 , (CO1, LDC) + + + addq $4 * SIZE, CO1 # coffset += 4 + ALIGN_4 + + +.L2_30: + testq $2, M + jz .L2_40 + + ALIGN_4 + +.L2_31: + leaq BUFFER1, BO // first buffer to BO + addq $4 * SIZE, BO + + vzeroall + + movq K, %rax + + andq $-8, %rax + je .L2_36 + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + + salq $1, %rax // rax = rax *2 ; number of values + leaq (AO, %rax, SIZE), AO + leaq (BO, BI, SIZE), BO + negq BI + negq %rax + ALIGN_4 + +.L2_32: + + prefetcht0 B_PR1(BO,BI,SIZE) + KERNEL2x2_1(xxx) + KERNEL2x2_2(xxx) + KERNEL2x2_3(xxx) + KERNEL2x2_4(xxx) + + KERNEL2x2_1(xxx) + KERNEL2x2_2(xxx) + KERNEL2x2_3(xxx) + KERNEL2x2_4(xxx) + + je .L2_36 + + prefetcht0 B_PR1(BO,BI,SIZE) + KERNEL2x2_1(xxx) + KERNEL2x2_2(xxx) + KERNEL2x2_3(xxx) + KERNEL2x2_4(xxx) + + KERNEL2x2_1(xxx) + KERNEL2x2_2(xxx) + KERNEL2x2_3(xxx) + KERNEL2x2_4(xxx) + + je .L2_36 + + jmp .L2_32 + ALIGN_4 + +.L2_36: + movq K, %rax + + andq $7, %rax # if (k & 1) + je .L2_39 + + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + + salq $1, %rax // rax = rax *2 ; number of values + leaq (AO, %rax, SIZE), AO + leaq (BO, BI, SIZE), BO + negq BI + negq %rax + ALIGN_4 + +.L2_37: + + KERNEL2x2_SUB(xxx) + addq $2, BI + addq $2, %rax + jl .L2_37 + ALIGN_4 + + +.L2_39: + + vmovss ALPHA, %xmm0 + + vfmaddss (CO1),%xmm0, %xmm4,%xmm4 + vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8 + vfmaddss (CO1, LDC),%xmm0, %xmm5,%xmm5 + vfmaddss 1 * SIZE(CO1, LDC),%xmm0, %xmm10,%xmm10 + + vmovss %xmm4 , (CO1) + vmovss %xmm8 , 1 * SIZE(CO1) + vmovss %xmm5 , (CO1, LDC) + vmovss %xmm10, 1 * SIZE(CO1, LDC) + + addq $2 * SIZE, CO1 # coffset += 2 + ALIGN_4 + +.L2_40: + testq $1, M + jz .L2_60 // to next 2 lines of N + + ALIGN_4 + +.L2_41: + leaq BUFFER1, BO // first buffer to BO + addq $4 * SIZE, BO + + vzeroall + + movq K, %rax + + andq $-8, %rax + je .L2_46 + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + + leaq (AO, %rax, SIZE), AO + leaq (BO, BI, SIZE), BO + negq BI + negq %rax + ALIGN_4 + +.L2_42: + + KERNEL1x2_1(xxx) + KERNEL1x2_2(xxx) + KERNEL1x2_3(xxx) + KERNEL1x2_4(xxx) + + KERNEL1x2_1(xxx) + KERNEL1x2_2(xxx) + KERNEL1x2_3(xxx) + KERNEL1x2_4(xxx) + + je .L2_46 + + KERNEL1x2_1(xxx) + KERNEL1x2_2(xxx) + KERNEL1x2_3(xxx) + KERNEL1x2_4(xxx) + + KERNEL1x2_1(xxx) + KERNEL1x2_2(xxx) + KERNEL1x2_3(xxx) + KERNEL1x2_4(xxx) + + je .L2_46 + + jmp .L2_42 + ALIGN_4 + +.L2_46: + movq K, %rax + + andq $7, %rax # if (k & 1) + je .L2_49 + + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + + leaq (AO, %rax, SIZE), AO + leaq (BO, BI, SIZE), BO + negq BI + negq %rax + ALIGN_4 + +.L2_47: + + KERNEL1x2_SUB(xxx) + addq $2, BI + addq $1, %rax + jl .L2_47 + ALIGN_4 + + +.L2_49: + + vmovss ALPHA, %xmm0 + + vfmaddss (CO1),%xmm0, %xmm4,%xmm4 + vfmaddss (CO1, LDC),%xmm0, %xmm5,%xmm5 + + vmovss %xmm4 , (CO1) + vmovss %xmm5 , (CO1, LDC) + + addq $1 * SIZE, CO1 # coffset += 1 + ALIGN_4 + +.L2_60: + + decq J // j -- + jg .L2_01 // next 2 lines of N + + + +.L1_0: + +/************************************************************************************************ +* Loop for Nmod6 % 2 > 0 +*************************************************************************************************/ + + movq Nmod6, J + andq $1, J // j % 2 + je .L999 + ALIGN_4 + +.L1_01: + // copy to sub buffer + movq B, BO1 + leaq BUFFER1, BO // first buffer to BO + movq K, %rax + ALIGN_4 + +.L1_02b: + + vmovss (BO1), %xmm0 + vmovss %xmm0, (BO) + addq $1*SIZE,BO1 + addq $1*SIZE,BO + decq %rax + jnz .L1_02b + +.L1_02c: + + movq BO1, B // next offset of B + +.L1_10: + movq C, CO1 + leaq (C, LDC, 1), C // c += 1 * ldc + + + movq A, AO // aoffset = a + addq $32 * SIZE, AO + + movq M, I + sarq $4, I // i = (m >> 4) + je .L1_20 + + ALIGN_4 + +.L1_11: + leaq BUFFER1, BO // first buffer to BO + addq $2 * SIZE, BO + + vzeroall + + movq K, %rax + + andq $-8, %rax // K = K - ( K % 8 ) + je .L1_16 + movq %rax, BI // Index for BO + + salq $4, %rax // rax = rax * 16 ; number of values + leaq (AO, %rax, SIZE), AO + leaq (BO, BI, SIZE), BO + negq BI + negq %rax + ALIGN_4 + +.L1_12: + + prefetcht0 B_PR1(BO,BI, SIZE) + KERNEL16x1_1(xxx) + KERNEL16x1_2(xxx) + KERNEL16x1_3(xxx) + KERNEL16x1_4(xxx) + + KERNEL16x1_1(xxx) + KERNEL16x1_2(xxx) + KERNEL16x1_3(xxx) + KERNEL16x1_4(xxx) + + je .L1_16 + + KERNEL16x1_1(xxx) + KERNEL16x1_2(xxx) + KERNEL16x1_3(xxx) + KERNEL16x1_4(xxx) + + KERNEL16x1_1(xxx) + KERNEL16x1_2(xxx) + KERNEL16x1_3(xxx) + KERNEL16x1_4(xxx) + + je .L1_16 + + jmp .L1_12 + ALIGN_4 + +.L1_16: + movq K, %rax + + andq $7, %rax # if (k & 1) + je .L1_19 + + movq %rax, BI // Index for BO + + salq $4, %rax // rax = rax * 16 ; number of values + leaq (AO, %rax, SIZE), AO + leaq (BO, BI, SIZE), BO + negq BI + negq %rax + ALIGN_4 + +.L1_17: + + KERNEL16x1_SUB(xxx) + addq $1, BI + addq $16, %rax + jl .L1_17 + ALIGN_4 + + +.L1_19: + + vbroadcastss ALPHA, %xmm0 + + vfmaddps (CO1),%xmm0, %xmm4,%xmm4 + vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7 + vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10 + vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13 + + vmovups %xmm4 , (CO1) + vmovups %xmm7 , 4 * SIZE(CO1) + vmovups %xmm10, 8 * SIZE(CO1) + vmovups %xmm13,12 * SIZE(CO1) + + addq $16 * SIZE, CO1 # coffset += 16 + decq I # i -- + jg .L1_11 + ALIGN_4 + +/************************************************************************** +* Rest of M +***************************************************************************/ +.L1_20: + // Test rest of M + + testq $15, M + jz .L999 + + testq $8, M + jz .L1_21pre + ALIGN_4 + +/**************************************************************************/ + +.L1_20_1: + leaq BUFFER1, BO // first buffer to BO + addq $2 * SIZE, BO + + vzeroall + + movq K, %rax + + andq $-8, %rax + je .L1_20_6 + movq %rax, BI // Index for BO + + salq $3, %rax // rax = rax * 8 ; number of values + leaq (AO, %rax, SIZE), AO + leaq (BO, BI, SIZE), BO + negq BI + negq %rax + ALIGN_4 + +.L1_20_2: + + prefetcht0 B_PR1(BO,BI, SIZE) + KERNEL8x1_1(xxx) + KERNEL8x1_2(xxx) + KERNEL8x1_3(xxx) + KERNEL8x1_4(xxx) + + KERNEL8x1_1(xxx) + KERNEL8x1_2(xxx) + KERNEL8x1_3(xxx) + KERNEL8x1_4(xxx) + + je .L1_20_6 + + KERNEL8x1_1(xxx) + KERNEL8x1_2(xxx) + KERNEL8x1_3(xxx) + KERNEL8x1_4(xxx) + + KERNEL8x1_1(xxx) + KERNEL8x1_2(xxx) + KERNEL8x1_3(xxx) + KERNEL8x1_4(xxx) + + je .L1_20_6 + + jmp .L1_20_2 + ALIGN_4 + +.L1_20_6: + movq K, %rax + + andq $7, %rax # if (k & 1) + je .L1_20_9 + + movq %rax, BI // Index for BO + + salq $3, %rax // rax = rax * 8 ; number of values + leaq (AO, %rax, SIZE), AO + leaq (BO, BI, SIZE), BO + negq BI + negq %rax + ALIGN_4 + +.L1_20_7: + + KERNEL8x1_SUB(xxx) + addq $1, BI + addq $8, %rax + jl .L1_20_7 + ALIGN_4 + + +.L1_20_9: + + vbroadcastss ALPHA, %xmm0 + + vfmaddps (CO1),%xmm0, %xmm4,%xmm4 + vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7 + + vmovups %xmm4 , (CO1) + vmovups %xmm7 , 4 * SIZE(CO1) + + addq $8 * SIZE, CO1 # coffset += 8 + ALIGN_4 + + + +/**************************************************************************/ + +.L1_21pre: + + testq $4, M + jz .L1_30 + ALIGN_4 + +.L1_21: + leaq BUFFER1, BO // first buffer to BO + addq $2 * SIZE, BO + + vzeroall + + movq K, %rax + + andq $-8, %rax + je .L1_26 + movq %rax, BI // Index for BO + + salq $2, %rax // rax = rax * 4 ; number of values + leaq (AO, %rax, SIZE), AO + leaq (BO, BI, SIZE), BO + negq BI + negq %rax + ALIGN_4 + +.L1_22: + + prefetcht0 B_PR1(BO,BI, SIZE) + KERNEL4x1_1(xxx) + KERNEL4x1_2(xxx) + KERNEL4x1_3(xxx) + KERNEL4x1_4(xxx) + + KERNEL4x1_1(xxx) + KERNEL4x1_2(xxx) + KERNEL4x1_3(xxx) + KERNEL4x1_4(xxx) + + je .L1_26 + + KERNEL4x1_1(xxx) + KERNEL4x1_2(xxx) + KERNEL4x1_3(xxx) + KERNEL4x1_4(xxx) + + KERNEL4x1_1(xxx) + KERNEL4x1_2(xxx) + KERNEL4x1_3(xxx) + KERNEL4x1_4(xxx) + + je .L1_26 + + jmp .L1_22 + ALIGN_4 + +.L1_26: + movq K, %rax + + andq $7, %rax # if (k & 1) + je .L1_29 + + movq %rax, BI // Index for BO + + salq $2, %rax // rax = rax * 4 ; number of values + leaq (AO, %rax, SIZE), AO + leaq (BO, BI, SIZE), BO + negq BI + negq %rax + ALIGN_4 + +.L1_27: + + KERNEL4x1_SUB(xxx) + addq $1, BI + addq $4, %rax + jl .L1_27 + ALIGN_4 + + +.L1_29: + + vbroadcastss ALPHA, %xmm0 + + vfmaddps (CO1),%xmm0, %xmm4,%xmm4 + + vmovups %xmm4 , (CO1) + + addq $4 * SIZE, CO1 # coffset += 4 + ALIGN_4 + + +.L1_30: + testq $2, M + jz .L1_40 + + ALIGN_4 + +.L1_31: + leaq BUFFER1, BO // first buffer to BO + addq $2 * SIZE, BO + + vzeroall + + movq K, %rax + + andq $-8, %rax + je .L1_36 + movq %rax, BI // Index for BO + + salq $1, %rax // rax = rax *2 ; number of values + leaq (AO, %rax, SIZE), AO + leaq (BO, BI, SIZE), BO + negq BI + negq %rax + ALIGN_4 + +.L1_32: + + prefetcht0 B_PR1(BO,BI,SIZE) + KERNEL2x1_1(xxx) + KERNEL2x1_2(xxx) + KERNEL2x1_3(xxx) + KERNEL2x1_4(xxx) + + KERNEL2x1_1(xxx) + KERNEL2x1_2(xxx) + KERNEL2x1_3(xxx) + KERNEL2x1_4(xxx) + + je .L1_36 + + KERNEL2x1_1(xxx) + KERNEL2x1_2(xxx) + KERNEL2x1_3(xxx) + KERNEL2x1_4(xxx) + + KERNEL2x1_1(xxx) + KERNEL2x1_2(xxx) + KERNEL2x1_3(xxx) + KERNEL2x1_4(xxx) + + je .L1_36 + + jmp .L1_32 + ALIGN_4 + +.L1_36: + movq K, %rax + + andq $7, %rax # if (k & 1) + je .L1_39 + + movq %rax, BI // Index for BO + + salq $1, %rax // rax = rax *2 ; number of values + leaq (AO, %rax, SIZE), AO + leaq (BO, BI, SIZE), BO + negq BI + negq %rax + ALIGN_4 + +.L1_37: + + KERNEL2x1_SUB(xxx) + addq $1, BI + addq $2, %rax + jl .L1_37 + ALIGN_4 + + +.L1_39: + + vmovss ALPHA, %xmm0 + + vfmaddss (CO1),%xmm0, %xmm4,%xmm4 + vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8 + + vmovss %xmm4 , (CO1) + vmovss %xmm8 , 1 * SIZE(CO1) + + addq $2 * SIZE, CO1 # coffset += 2 + ALIGN_4 + +.L1_40: + testq $1, M + jz .L999 + + ALIGN_4 + +.L1_41: + leaq BUFFER1, BO // first buffer to BO + addq $2 * SIZE, BO + + vzeroall + + movq K, %rax + + andq $-8, %rax + je .L1_46 + movq %rax, BI // Index for BO + + leaq (AO, %rax, SIZE), AO + leaq (BO, BI, SIZE), BO + negq BI + negq %rax + ALIGN_4 + +.L1_42: + + KERNEL1x1_1(xxx) + KERNEL1x1_2(xxx) + KERNEL1x1_3(xxx) + KERNEL1x1_4(xxx) + + KERNEL1x1_1(xxx) + KERNEL1x1_2(xxx) + KERNEL1x1_3(xxx) + KERNEL1x1_4(xxx) + + je .L1_46 + + KERNEL1x1_1(xxx) + KERNEL1x1_2(xxx) + KERNEL1x1_3(xxx) + KERNEL1x1_4(xxx) + + KERNEL1x1_1(xxx) + KERNEL1x1_2(xxx) + KERNEL1x1_3(xxx) + KERNEL1x1_4(xxx) + + je .L1_46 + + jmp .L1_42 + ALIGN_4 + +.L1_46: + movq K, %rax + + andq $7, %rax # if (k & 1) + je .L1_49 + + movq %rax, BI // Index for BO + + leaq (AO, %rax, SIZE), AO + leaq (BO, BI, SIZE), BO + negq BI + negq %rax + ALIGN_4 + +.L1_47: + + KERNEL1x1_SUB(xxx) + addq $1, BI + addq $1, %rax + jl .L1_47 + ALIGN_4 + + +.L1_49: + + vmovss ALPHA, %xmm0 + + vfmaddss (CO1),%xmm0, %xmm4,%xmm4 + + vmovss %xmm4 , (CO1) + + addq $1 * SIZE, CO1 # coffset += 1 + ALIGN_4 + + +.L999: + movq SP, %rsp + movq (%rsp), %rbx + movq 8(%rsp), %rbp + movq 16(%rsp), %r12 + movq 24(%rsp), %r13 + movq 32(%rsp), %r14 + movq 40(%rsp), %r15 + +#ifdef WINDOWS_ABI + movq 48(%rsp), %rdi + movq 56(%rsp), %rsi + movups 64(%rsp), %xmm6 + movups 80(%rsp), %xmm7 + movups 96(%rsp), %xmm8 + movups 112(%rsp), %xmm9 + movups 128(%rsp), %xmm10 + movups 144(%rsp), %xmm11 + movups 160(%rsp), %xmm12 + movups 176(%rsp), %xmm13 + movups 192(%rsp), %xmm14 + movups 208(%rsp), %xmm15 +#endif + + addq $STACKSIZE, %rsp + ret + + EPILOGUE + + +#else +/************************************************************************************* +* TRMM Kernel +*************************************************************************************/ + + + PROLOGUE + PROFCODE + + subq $STACKSIZE, %rsp + movq %rbx, (%rsp) + movq %rbp, 8(%rsp) + movq %r12, 16(%rsp) + movq %r13, 24(%rsp) + movq %r14, 32(%rsp) + movq %r15, 40(%rsp) + + vzeroupper + +#ifdef WINDOWS_ABI + movq %rdi, 48(%rsp) + movq %rsi, 56(%rsp) + movups %xmm6, 64(%rsp) + movups %xmm7, 80(%rsp) + movups %xmm8, 96(%rsp) + movups %xmm9, 112(%rsp) + movups %xmm10, 128(%rsp) + movups %xmm11, 144(%rsp) + movups %xmm12, 160(%rsp) + movups %xmm13, 176(%rsp) + movups %xmm14, 192(%rsp) + movups %xmm15, 208(%rsp) + + movq ARG1, OLD_M + movq ARG2, OLD_N + movq ARG3, OLD_K + movq OLD_A, A + movq OLD_B, B + movq OLD_C, C + movq OLD_LDC, LDC +#ifdef TRMMKERNEL + movsd OLD_OFFSET, %xmm12 +#endif + vmovaps %xmm3, %xmm0 + +#else + movq STACKSIZE + 8(%rsp), LDC +#ifdef TRMMKERNEL + movsd STACKSIZE + 16(%rsp), %xmm12 +#endif + +#endif + + movq %rsp, SP # save old stack + subq $128 + L_BUFFER_SIZE, %rsp + andq $-4096, %rsp # align stack + + STACK_TOUCH + + cmpq $0, OLD_M + je .L999 + + cmpq $0, OLD_N + je .L999 + + cmpq $0, OLD_K + je .L999 + + movq OLD_M, M + movq OLD_N, N + movq OLD_K, K + + vmovsd %xmm0, ALPHA + + salq $BASE_SHIFT, LDC + + movq N, %rax + xorq %rdx, %rdx + movq $2, %rdi + divq %rdi // N / 6 + movq %rax, Ndiv6 // N / 6 + movq %rdx, Nmod6 // N % 6 + + + +#ifdef TRMMKERNEL + vmovsd %xmm12, OFFSET + vmovsd %xmm12, KK +#ifndef LEFT + negq KK +#endif +#endif + + movq Ndiv6, J + cmpq $0, J + je .L1_0 + ALIGN_4 + +.L2_01: + // copy to sub buffer + movq B, BO1 + leaq BUFFER1, BO // first buffer to BO + movq K, %rax + ALIGN_4 + +.L2_02b: + + vmovsd (BO1), %xmm0 + vmovsd %xmm0, (BO) + addq $2*SIZE,BO1 + addq $2*SIZE,BO + decq %rax + jnz .L2_02b + +.L2_02c: + + movq BO1, B // next offset of B + .L2_10: movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -3831,6 +4396,9 @@ .L2_60: +#if defined(TRMMKERNEL) && !defined(LEFT) + addq $2, KK +#endif decq J // j -- jg .L2_01 // next 2 lines of N @@ -4655,3 +5223,9 @@ ret EPILOGUE + + + + + +#endif diff --git a/kernel/x86_64/sgemm_kernel_8x4_bulldozer.S b/kernel/x86_64/sgemm_kernel_8x4_bulldozer.S new file mode 100644 index 000000000..268d3ae7e --- /dev/null +++ b/kernel/x86_64/sgemm_kernel_8x4_bulldozer.S @@ -0,0 +1,3087 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#define ASSEMBLER +#include "common.h" + +#define OLD_M %rdi +#define OLD_N %rsi +#define M %r13 +#define N %r14 +#define K %rdx + +#define A %rcx +#define B %r8 +#define C %r9 +#define LDC %r10 + +#define I %r11 +#define AO %rdi +#define BO %rsi +#define CO1 %r15 +#define CO2 %r12 + +#ifndef WINDOWS_ABI + +#define STACKSIZE 64 + +#else + +#define STACKSIZE 256 + +#define OLD_A 40 + STACKSIZE(%rsp) +#define OLD_B 48 + STACKSIZE(%rsp) +#define OLD_C 56 + STACKSIZE(%rsp) +#define OLD_LDC 64 + STACKSIZE(%rsp) +#define OLD_OFFSET 72 + STACKSIZE(%rsp) + +#endif + +#define ALPHA 0(%rsp) +#define J 16(%rsp) +#define OFFSET 24(%rsp) +#define KK 32(%rsp) +#define KKK 40(%rsp) +#define BUFFER 128(%rsp) + +#define PREFETCH prefetch +#define PREFETCHSIZE (16 * 17 + 0) + +#define RPREFETCHSIZE (16 * 4 + 0) +#define WPREFETCHSIZE (16 * 9 + 0) + +#define KERNEL1(xx) \ + vfmaddps %xmm8,%xmm1,%xmm0,%xmm8 ;\ + vmovaps %xmm2, %xmm0 ;\ + vmovups -28 * SIZE(AO, %rax, 4),%xmm2 ;\ + vfmaddps %xmm12,%xmm2, %xmm1, %xmm12 ;\ + vmovups -24 * SIZE(BO, %rax, 8), %xmm1 ;\ + vfmaddps %xmm9,%xmm3, %xmm0, %xmm9 ;\ + vfmaddps %xmm13,%xmm2, %xmm3, %xmm13 ;\ + vmovups -20 * SIZE(BO, %rax, 8), %xmm3 ;\ + vfmaddps %xmm10,%xmm1, %xmm0, %xmm10 ;\ + vfmaddps %xmm14,%xmm2, %xmm1, %xmm14 ;\ + vfmaddps %xmm11,%xmm3, %xmm0, %xmm11 ;\ + vfmaddps %xmm15,%xmm2, %xmm3, %xmm15 ;\ + vmovups -24 * SIZE(AO, %rax, 4), %xmm0 ;\ + vmovups -16 * SIZE(BO, %rax, 8), %xmm1 ;\ + vmovups -12 * SIZE(BO, %rax, 8), %xmm3 ;\ + vmovaps %xmm0, %xmm2 + + +#define KERNEL2(xx) \ + vfmaddps %xmm8,%xmm1,%xmm0,%xmm8 ;\ + vmovaps %xmm2, %xmm0 ;\ + vmovups -20 * SIZE(AO, %rax, 4),%xmm2 ;\ + vfmaddps %xmm12,%xmm2, %xmm1, %xmm12 ;\ + vmovups -8 * SIZE(BO, %rax, 8), %xmm1 ;\ + vfmaddps %xmm9,%xmm3, %xmm0, %xmm9 ;\ + vfmaddps %xmm13,%xmm2, %xmm3, %xmm13 ;\ + vmovups -4 * SIZE(BO, %rax, 8), %xmm3 ;\ + vfmaddps %xmm10,%xmm1, %xmm0, %xmm10 ;\ + vfmaddps %xmm14,%xmm2, %xmm1, %xmm14 ;\ + vfmaddps %xmm11,%xmm3, %xmm0, %xmm11 ;\ + vfmaddps %xmm15,%xmm2, %xmm3, %xmm15 ;\ + vmovups 4 * SIZE(BO, %rax, 8), %xmm3 ;\ + vmovaps %xmm4, %xmm2 + + + +#define KERNEL3(xx) \ + vfmaddps %xmm8,%xmm5,%xmm4,%xmm8 ;\ + vmovups -12 * SIZE(AO, %rax, 4),%xmm2 ;\ + vfmaddps %xmm12,%xmm2, %xmm5, %xmm12 ;\ + vmovups 32 * SIZE(BO, %rax, 8), %xmm1 ;\ + vmovups 8 * SIZE(BO, %rax, 8), %xmm5 ;\ + vfmaddps %xmm9,%xmm3, %xmm4, %xmm9 ;\ + vfmaddps %xmm13,%xmm2, %xmm3, %xmm13 ;\ + vmovups 12 * SIZE(BO, %rax, 8), %xmm3 ;\ + vfmaddps %xmm10,%xmm5, %xmm4, %xmm10 ;\ + vfmaddps %xmm14,%xmm2, %xmm5, %xmm14 ;\ + vfmaddps %xmm11,%xmm3, %xmm4, %xmm11 ;\ + vfmaddps %xmm15,%xmm2, %xmm3, %xmm15 ;\ + vmovups -8 * SIZE(AO, %rax, 4), %xmm4 ;\ + vmovups 16 * SIZE(BO, %rax, 8), %xmm5 ;\ + vmovups 20 * SIZE(BO, %rax, 8), %xmm3 ;\ + vmovaps %xmm4, %xmm2 + +#define KERNEL4(xx) \ + vfmaddps %xmm8,%xmm5, %xmm4, %xmm8 ;\ + vmovups -4 * SIZE(AO, %rax, 4),%xmm2 ;\ + vfmaddps %xmm12,%xmm2, %xmm5, %xmm12 ;\ + vmovups 24 * SIZE(BO, %rax, 8), %xmm5 ;\ + vfmaddps %xmm9,%xmm3, %xmm4, %xmm9 ;\ + vfmaddps %xmm13,%xmm2, %xmm3, %xmm13 ;\ + vmovups 28 * SIZE(BO, %rax, 8), %xmm3 ;\ + vfmaddps %xmm10,%xmm5, %xmm4, %xmm10 ;\ + vfmaddps %xmm14,%xmm2, %xmm5, %xmm14 ;\ + vmovups 64 * SIZE(BO, %rax, 8), %xmm5 ;\ + vfmaddps %xmm11,%xmm3, %xmm4, %xmm11 ;\ + vfmaddps %xmm15,%xmm2, %xmm3, %xmm15 ;\ + vmovups (AO, %rax, 4), %xmm6 ;\ + vmovups 36 * SIZE(BO, %rax, 8), %xmm3 ;\ + vmovaps %xmm6, %xmm2 + +#define KERNEL5(xx) \ + vfmaddps %xmm8,%xmm1, %xmm6, %xmm8 ;\ + vmovups 4 * SIZE(AO, %rax, 4),%xmm2 ;\ + vfmaddps %xmm12,%xmm2, %xmm1, %xmm12 ;\ + vmovups 40 * SIZE(BO, %rax, 8), %xmm1 ;\ + vfmaddps %xmm9,%xmm3, %xmm6, %xmm9 ;\ + vfmaddps %xmm13,%xmm2, %xmm3, %xmm13 ;\ + vmovups 16 * SIZE(AO, %rax, 4), %xmm7 ;\ + vmovups 44 * SIZE(BO, %rax, 8), %xmm3 ;\ + vfmaddps %xmm10,%xmm1, %xmm6, %xmm10 ;\ + vfmaddps %xmm14,%xmm2, %xmm1, %xmm14 ;\ + vfmaddps %xmm11,%xmm3, %xmm6, %xmm11 ;\ + vfmaddps %xmm15,%xmm2, %xmm3, %xmm15 ;\ + vmovups 8 * SIZE(AO, %rax, 4), %xmm6 ;\ + vmovups 48 * SIZE(BO, %rax, 8), %xmm1 ;\ + vmovups 52 * SIZE(BO, %rax, 8), %xmm3 ;\ + vmovaps %xmm6, %xmm2 + +#define KERNEL6(xx) \ + vfmaddps %xmm8,%xmm1, %xmm6, %xmm8 ;\ + vmovups 12 * SIZE(AO, %rax, 4),%xmm2 ;\ + vfmaddps %xmm12,%xmm2, %xmm1, %xmm12 ;\ + vmovups 56 * SIZE(BO, %rax, 8), %xmm1 ;\ + vfmaddps %xmm9,%xmm3, %xmm6, %xmm9 ;\ + vfmaddps %xmm13,%xmm2, %xmm3, %xmm13 ;\ + vmovups 60 * SIZE(BO, %rax, 8), %xmm3 ;\ + vfmaddps %xmm10,%xmm1, %xmm6, %xmm10 ;\ + vfmaddps %xmm14,%xmm2, %xmm1, %xmm14 ;\ + vfmaddps %xmm11,%xmm3, %xmm6, %xmm11 ;\ + vfmaddps %xmm15,%xmm2, %xmm3, %xmm15 ;\ + vmovups 32 * SIZE(AO, %rax, 4), %xmm0 ;\ + vmovups 68 * SIZE(BO, %rax, 8), %xmm3 ;\ + vmovaps %xmm7, %xmm2 + +#define KERNEL7(xx) \ + vfmaddps %xmm8,%xmm5, %xmm7, %xmm8 ;\ + vmovups 20 * SIZE(AO, %rax, 4),%xmm2 ;\ + vfmaddps %xmm12,%xmm2, %xmm5, %xmm12 ;\ + vmovups 96 * SIZE(BO, %rax, 8), %xmm1 ;\ + vmovups 72 * SIZE(BO, %rax, 8), %xmm5 ;\ + vfmaddps %xmm9,%xmm3, %xmm7, %xmm9 ;\ + vfmaddps %xmm13,%xmm2, %xmm3, %xmm13 ;\ + vmovups 76 * SIZE(BO, %rax, 8), %xmm3 ;\ + vfmaddps %xmm10,%xmm5, %xmm7, %xmm10 ;\ + vfmaddps %xmm14,%xmm2, %xmm5, %xmm14 ;\ + vfmaddps %xmm11,%xmm3, %xmm7, %xmm11 ;\ + vfmaddps %xmm15,%xmm2, %xmm3, %xmm15 ;\ + vmovups 24 * SIZE(AO, %rax, 4), %xmm7 ;\ + vmovups 80 * SIZE(BO, %rax, 8), %xmm5 ;\ + vmovups 84 * SIZE(BO, %rax, 8), %xmm3 ;\ + movaps %xmm7, %xmm2 + +#define KERNEL8(xx) \ + vfmaddps %xmm8,%xmm5, %xmm7, %xmm8 ;\ + vmovups 28 * SIZE(AO, %rax, 4),%xmm2 ;\ + vfmaddps %xmm12,%xmm2, %xmm5, %xmm12 ;\ + vmovups 88 * SIZE(BO, %rax, 8), %xmm5 ;\ + vfmaddps %xmm9, %xmm3, %xmm7, %xmm9 ;\ + vfmaddps %xmm13,%xmm2, %xmm3, %xmm13 ;\ + vmovups 92 * SIZE(BO, %rax, 8), %xmm3 ;\ + vfmaddps %xmm10,%xmm5, %xmm7, %xmm10 ;\ + vfmaddps %xmm14,%xmm2, %xmm5, %xmm14 ;\ + vmovups 48 * SIZE(AO, %rax, 4), %xmm4 ;\ + vmovups 128 * SIZE(BO, %rax, 8), %xmm5 ;\ + vfmaddps %xmm11,%xmm3, %xmm7, %xmm11 ;\ + vfmaddps %xmm15,%xmm2, %xmm3, %xmm15 ;\ + vmovups 100 * SIZE(BO, %rax, 8), %xmm3 ;\ + vmovaps %xmm0, %xmm2 ;\ + addq $16 * SIZE, %rax + +#define KERNEL_SUB1(xx) \ + mulps %xmm1, %xmm0 ;\ + mulps -28 * SIZE(AO, %rax, 4), %xmm1 ;\ + addps %xmm0, %xmm8 ;\ + movaps %xmm2, %xmm0 ;\ + addps %xmm1, %xmm12 ;\ + movaps -24 * SIZE(BO, %rax, 8), %xmm1 ;\ + mulps %xmm3, %xmm2 ;\ + mulps -28 * SIZE(AO, %rax, 4), %xmm3 ;\ + addps %xmm2, %xmm9 ;\ + movaps %xmm0, %xmm2 ;\ + addps %xmm3, %xmm13 ;\ + movaps -20 * SIZE(BO, %rax, 8), %xmm3 ;\ + mulps %xmm1, %xmm0 ;\ + mulps -28 * SIZE(AO, %rax, 4), %xmm1 ;\ + addps %xmm0, %xmm10 ;\ + movaps -24 * SIZE(AO, %rax, 4), %xmm0 ;\ + addps %xmm1, %xmm14 ;\ + movaps -16 * SIZE(BO, %rax, 8), %xmm1 ;\ + mulps %xmm3, %xmm2 ;\ + mulps -28 * SIZE(AO, %rax, 4), %xmm3 ;\ + addps %xmm2, %xmm11 ;\ + addps %xmm3, %xmm15 ;\ + movaps -12 * SIZE(BO, %rax, 8), %xmm3 ;\ + movaps %xmm0, %xmm2 + +#define KERNEL_SUB2(xx) \ + mulps %xmm1, %xmm0 ;\ + mulps -20 * SIZE(AO, %rax, 4), %xmm1 ;\ + addps %xmm0, %xmm8 ;\ + movaps %xmm2, %xmm0 ;\ + addps %xmm1, %xmm12 ;\ + movaps -8 * SIZE(BO, %rax, 8), %xmm1 ;\ + mulps %xmm3, %xmm2 ;\ + mulps -20 * SIZE(AO, %rax, 4), %xmm3 ;\ + addps %xmm2, %xmm9 ;\ + movaps %xmm0, %xmm2 ;\ + addps %xmm3, %xmm13 ;\ + movaps -4 * SIZE(BO, %rax, 8), %xmm3 ;\ + mulps %xmm1, %xmm0 ;\ + mulps -20 * SIZE(AO, %rax, 4), %xmm1 ;\ + addps %xmm0, %xmm10 ;\ + movaps (AO, %rax, 4), %xmm0 ;\ + addps %xmm1, %xmm14 ;\ + movaps 32 * SIZE(BO, %rax, 8), %xmm1 ;\ + mulps %xmm3, %xmm2 ;\ + mulps -20 * SIZE(AO, %rax, 4), %xmm3 ;\ + addps %xmm2, %xmm11 ;\ + addps %xmm3, %xmm15 ;\ + movaps 4 * SIZE(BO, %rax, 8), %xmm3 ;\ + movaps %xmm4, %xmm2 + +#define KERNEL_SUB3(xx) \ + mulps %xmm5, %xmm4 ;\ + mulps -12 * SIZE(AO, %rax, 4), %xmm5 ;\ + addps %xmm4, %xmm8 ;\ + movaps %xmm2, %xmm4 ;\ + addps %xmm5, %xmm12 ;\ + movaps 8 * SIZE(BO, %rax, 8), %xmm5 ;\ + mulps %xmm3, %xmm2 ;\ + mulps -12 * SIZE(AO, %rax, 4), %xmm3 ;\ + addps %xmm2, %xmm9 ;\ + movaps %xmm4, %xmm2 ;\ + addps %xmm3, %xmm13 ;\ + movaps 12 * SIZE(BO, %rax, 8), %xmm3 ;\ + mulps %xmm5, %xmm4 ;\ + mulps -12 * SIZE(AO, %rax, 4), %xmm5 ;\ + addps %xmm4, %xmm10 ;\ + movaps -8 * SIZE(AO, %rax, 4), %xmm4 ;\ + addps %xmm5, %xmm14 ;\ + movaps 16 * SIZE(BO, %rax, 8), %xmm5 ;\ + mulps %xmm3, %xmm2 ;\ + mulps -12 * SIZE(AO, %rax, 4), %xmm3 ;\ + addps %xmm2, %xmm11 ;\ + addps %xmm3, %xmm15 ;\ + movaps 20 * SIZE(BO, %rax, 8), %xmm3 ;\ + movaps %xmm4, %xmm2 + +#define KERNEL_SUB4(xx) \ + mulps %xmm5, %xmm4 ;\ + mulps -4 * SIZE(AO, %rax, 4), %xmm5 ;\ + addps %xmm4, %xmm8 ;\ + movaps %xmm2, %xmm4 ;\ + addps %xmm5, %xmm12 ;\ + movaps 24 * SIZE(BO, %rax, 8), %xmm5 ;\ + mulps %xmm3, %xmm2 ;\ + mulps -4 * SIZE(AO, %rax, 4), %xmm3 ;\ + addps %xmm2, %xmm9 ;\ + movaps %xmm4, %xmm2 ;\ + addps %xmm3, %xmm13 ;\ + movaps 28 * SIZE(BO, %rax, 8), %xmm3 ;\ + mulps %xmm5, %xmm4 ;\ + mulps -4 * SIZE(AO, %rax, 4), %xmm5 ;\ + addps %xmm4, %xmm10 ;\ + addps %xmm5, %xmm14 ;\ + mulps %xmm3, %xmm2 ;\ + mulps -4 * SIZE(AO, %rax, 4), %xmm3 ;\ + addps %xmm2, %xmm11 ;\ + addps %xmm3, %xmm15 ;\ + movaps 36 * SIZE(BO, %rax, 8), %xmm3 ;\ + movaps %xmm0, %xmm2 + +#if defined(OS_LINUX) && defined(CORE_BULLDOZER) && !defined(TRMMKERNEL) + .align 32768 +#endif + PROLOGUE + PROFCODE + + subq $STACKSIZE, %rsp + + movq %rbx, 0(%rsp) + movq %rbp, 8(%rsp) + movq %r12, 16(%rsp) + movq %r13, 24(%rsp) + movq %r14, 32(%rsp) + movq %r15, 40(%rsp) + +#ifdef WINDOWS_ABI + movq %rdi, 48(%rsp) + movq %rsi, 56(%rsp) + movups %xmm6, 64(%rsp) + movups %xmm7, 80(%rsp) + movups %xmm8, 96(%rsp) + movups %xmm9, 112(%rsp) + movups %xmm10, 128(%rsp) + movups %xmm11, 144(%rsp) + movups %xmm12, 160(%rsp) + movups %xmm13, 176(%rsp) + movups %xmm14, 192(%rsp) + movups %xmm15, 208(%rsp) + + movq ARG1, OLD_M + movq ARG2, OLD_N + movq ARG3, K + movq OLD_A, A + movq OLD_B, B + movq OLD_C, C + movq OLD_LDC, LDC +#ifdef TRMMKERNEL + movsd OLD_OFFSET, %xmm12 +#endif + movaps %xmm3, %xmm0 + +#else + movq 72(%rsp), LDC +#ifdef TRMMKERNEL + movsd 80(%rsp), %xmm12 +#endif + +#endif + + movq %rsp, %rbx # save old stack + subq $128 + LOCAL_BUFFER_SIZE, %rsp + andq $-4096, %rsp # align stack + + STACK_TOUCHING + + movq OLD_M, M + movq OLD_N, N + + shufps $0, %xmm0, %xmm0 + movaps %xmm0, ALPHA + +#ifdef TRMMKERNEL + movsd %xmm12, OFFSET + movsd %xmm12, KK +#ifndef LEFT + negq KK +#endif +#endif + + subq $-32 * SIZE, A + + leaq (, LDC, SIZE), LDC + + movq N, J + sarq $2, J # j = (n >> 2) + jle .L50 + +.L01: +#if defined(TRMMKERNEL) && defined(LEFT) + movq OFFSET, %rax + movq %rax, KK +#endif + +/* Copying to Sub Buffer */ + leaq BUFFER, BO + + movq K, %rax + sarq $2, %rax + jle .L03 + ALIGN_4 + +.L02: + + prefetcht0 192(B) + prefetcht0 256(B) + prefetcht0 192(BO) + prefetcht0 256(BO) + movaps 0 * SIZE(B), %xmm3 + movaps 0 * SIZE(B), %xmm3 + movaps 4 * SIZE(B), %xmm7 + movaps 8 * SIZE(B), %xmm11 + movaps 12 * SIZE(B), %xmm15 + + + pshufd $0x00, %xmm3, %xmm0 + pshufd $0x55, %xmm3, %xmm1 + pshufd $0xaa, %xmm3, %xmm2 + pshufd $0xff, %xmm3, %xmm3 + + + pshufd $0x00, %xmm7, %xmm4 + pshufd $0x55, %xmm7, %xmm5 + pshufd $0xaa, %xmm7, %xmm6 + pshufd $0xff, %xmm7, %xmm7 + + movaps %xmm0, 0 * SIZE(BO) + movaps %xmm1, 4 * SIZE(BO) + movaps %xmm2, 8 * SIZE(BO) + movaps %xmm3, 12 * SIZE(BO) + movaps %xmm4, 16 * SIZE(BO) + movaps %xmm5, 20 * SIZE(BO) + movaps %xmm6, 24 * SIZE(BO) + movaps %xmm7, 28 * SIZE(BO) + + + pshufd $0x00, %xmm11, %xmm0 + pshufd $0x55, %xmm11, %xmm1 + pshufd $0xaa, %xmm11, %xmm2 + pshufd $0xff, %xmm11, %xmm3 + + + pshufd $0x00, %xmm15, %xmm4 + pshufd $0x55, %xmm15, %xmm5 + pshufd $0xaa, %xmm15, %xmm6 + pshufd $0xff, %xmm15, %xmm7 + + movaps %xmm0, 32 * SIZE(BO) + movaps %xmm1, 36 * SIZE(BO) + movaps %xmm2, 40 * SIZE(BO) + movaps %xmm3, 44 * SIZE(BO) + movaps %xmm4, 48 * SIZE(BO) + movaps %xmm5, 52 * SIZE(BO) + movaps %xmm6, 56 * SIZE(BO) + movaps %xmm7, 60 * SIZE(BO) + + addq $16 * SIZE, B + addq $64 * SIZE, BO + + decq %rax + jne .L02 + ALIGN_4 + +.L03: + movq K, %rax + andq $3, %rax + BRANCH + jle .L10 + ALIGN_4 + +.L04: + movaps 0 * SIZE(B), %xmm3 + + pshufd $0x00, %xmm3, %xmm0 + pshufd $0x55, %xmm3, %xmm1 + pshufd $0xaa, %xmm3, %xmm2 + pshufd $0xff, %xmm3, %xmm3 + + movaps %xmm0, 0 * SIZE(BO) + movaps %xmm1, 4 * SIZE(BO) + movaps %xmm2, 8 * SIZE(BO) + movaps %xmm3, 12 * SIZE(BO) + + addq $ 4 * SIZE, B + addq $16 * SIZE, BO + decq %rax + jne .L04 + ALIGN_4 + +.L10: + movq C, CO1 + leaq (C, LDC, 1), CO2 + movq A, AO + + + movq M, I + sarq $3, I # i = (m >> 3) + jle .L20 + ALIGN_4 + +.L11: +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + + leaq 32 * SIZE + BUFFER, BO +#else + leaq 32 * SIZE + BUFFER, BO + movq KK, %rax + leaq (, %rax, 8), %rax + leaq (AO, %rax, 4), AO + leaq (BO, %rax, 8), BO +#endif + + movaps -32 * SIZE(AO), %xmm0 + movaps -32 * SIZE(BO), %xmm1 + xorps %xmm8, %xmm8 + movaps -28 * SIZE(BO), %xmm3 + xorps %xmm9, %xmm9 + movaps -16 * SIZE(AO), %xmm4 + xorps %xmm10, %xmm10 + movaps 0 * SIZE(BO), %xmm5 + xorps %xmm11, %xmm11 + + + xorps %xmm12, %xmm12 + xorps %xmm13, %xmm13 + xorps %xmm14, %xmm14 + xorps %xmm15, %xmm15 + movaps %xmm0, %xmm2 + prefetcht0 (CO1) + prefetcht0 (CO1,LDC, 2) + prefetcht0 (CO2) + prefetcht0 (CO2,LDC, 2) + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $8, %rax +#else + addq $4, %rax +#endif + movq %rax, KKK +#endif + andq $-8, %rax + + leaq (, %rax, 8), %rax + leaq (AO, %rax, 4), AO + leaq (BO, %rax, 8), BO + negq %rax + NOBRANCH + je .L15 + ALIGN_3 + +.L12: + KERNEL1(32 * 0) + KERNEL2(32 * 0) + KERNEL3(32 * 0) + KERNEL4(32 * 0) + KERNEL5(32 * 0) + KERNEL6(32 * 0) + KERNEL7(32 * 0) + KERNEL8(32 * 0) + NOBRANCH + je .L15 + KERNEL1(32 * 0) + KERNEL2(32 * 0) + KERNEL3(32 * 0) + KERNEL4(32 * 0) + KERNEL5(32 * 0) + KERNEL6(32 * 0) + KERNEL7(32 * 0) + KERNEL8(32 * 0) + NOBRANCH + je .L15 + KERNEL1(32 * 0) + KERNEL2(32 * 0) + KERNEL3(32 * 0) + KERNEL4(32 * 0) + KERNEL5(32 * 0) + KERNEL6(32 * 0) + KERNEL7(32 * 0) + KERNEL8(32 * 0) + NOBRANCH + je .L15 + KERNEL1(32 * 0) + KERNEL2(32 * 0) + KERNEL3(32 * 0) + KERNEL4(32 * 0) + KERNEL5(32 * 0) + KERNEL6(32 * 0) + KERNEL7(32 * 0) + KERNEL8(32 * 0) + NOBRANCH + je .L15 + KERNEL1(32 * 0) + KERNEL2(32 * 0) + KERNEL3(32 * 0) + KERNEL4(32 * 0) + KERNEL5(32 * 0) + KERNEL6(32 * 0) + KERNEL7(32 * 0) + KERNEL8(32 * 0) + NOBRANCH + je .L15 + KERNEL1(32 * 0) + KERNEL2(32 * 0) + KERNEL3(32 * 0) + KERNEL4(32 * 0) + KERNEL5(32 * 0) + KERNEL6(32 * 0) + KERNEL7(32 * 0) + KERNEL8(32 * 0) + NOBRANCH + je .L15 + KERNEL1(32 * 0) + KERNEL2(32 * 0) + KERNEL3(32 * 0) + KERNEL4(32 * 0) + KERNEL5(32 * 0) + KERNEL6(32 * 0) + KERNEL7(32 * 0) + KERNEL8(32 * 0) + NOBRANCH + je .L15 + KERNEL1(32 * 0) + KERNEL2(32 * 0) + KERNEL3(32 * 0) + KERNEL4(32 * 0) + KERNEL5(32 * 0) + KERNEL6(32 * 0) + KERNEL7(32 * 0) + KERNEL8(32 * 0) + BRANCH + jl .L12 + ALIGN_4 + +.L15: + + movaps ALPHA, %xmm7 + +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + testq $4, %rax + je .L16 + xorq %rax, %rax + ALIGN_3 + + KERNEL_SUB1(32 * 0) + KERNEL_SUB2(32 * 0) + KERNEL_SUB3(32 * 0) + KERNEL_SUB4(32 * 0) + + addq $32 * SIZE, AO + addq $64 * SIZE, BO + ALIGN_3 + +.L16: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + andq $3, %rax # if (k & 1) + je .L18 + + leaq (, %rax, 8), %rax + leaq (AO, %rax, 4), AO + leaq (BO, %rax, 8), BO + negq %rax + ALIGN_4 + +.L17: + mulps %xmm1, %xmm0 + mulps -28 * SIZE(AO, %rax, 4), %xmm1 + addps %xmm0, %xmm8 + movaps %xmm2, %xmm0 + addps %xmm1, %xmm12 + movaps -24 * SIZE(BO, %rax, 8), %xmm1 + mulps %xmm3, %xmm2 + mulps -28 * SIZE(AO, %rax, 4), %xmm3 + addps %xmm2, %xmm9 + movaps %xmm0, %xmm2 + addps %xmm3, %xmm13 + movaps -20 * SIZE(BO, %rax, 8), %xmm3 + mulps %xmm1, %xmm0 + mulps -28 * SIZE(AO, %rax, 4), %xmm1 + addps %xmm0, %xmm10 + movaps -24 * SIZE(AO, %rax, 4), %xmm0 + addps %xmm1, %xmm14 + movaps -16 * SIZE(BO, %rax, 8), %xmm1 + mulps %xmm3, %xmm2 + mulps -28 * SIZE(AO, %rax, 4), %xmm3 + addps %xmm2, %xmm11 + addps %xmm3, %xmm15 + movaps -12 * SIZE(BO, %rax, 8), %xmm3 + movaps %xmm0, %xmm2 + + addq $SIZE * 2, %rax + jl .L17 + ALIGN_4 + +.L18: +#ifndef TRMMKERNEL + + vfmaddps 0 * SIZE(CO1),%xmm7, %xmm8, %xmm8 + vfmaddps 4 * SIZE(CO1),%xmm7, %xmm12, %xmm12 + vfmaddps 0 * SIZE(CO2),%xmm7, %xmm9, %xmm9 + vfmaddps 4 * SIZE(CO2),%xmm7, %xmm13, %xmm13 + vfmaddps 0 * SIZE(CO1, LDC, 2),%xmm7, %xmm10, %xmm10 + vfmaddps 4 * SIZE(CO1, LDC, 2),%xmm7, %xmm14, %xmm14 + vfmaddps 0 * SIZE(CO2, LDC, 2),%xmm7, %xmm11, %xmm11 + vfmaddps 4 * SIZE(CO2, LDC, 2),%xmm7, %xmm15, %xmm15 + +#else + + vmulps %xmm7, %xmm8, %xmm8 + vmulps %xmm7, %xmm9, %xmm9 + vmulps %xmm7, %xmm10, %xmm10 + vmulps %xmm7, %xmm11, %xmm11 + + vmulps %xmm7, %xmm12,%xmm12 + vmulps %xmm7, %xmm13,%xmm13 + vmulps %xmm7, %xmm14,%xmm14 + vmulps %xmm7, %xmm15,%xmm15 + +#endif + + + vmovups %xmm8, 0 * SIZE(CO1) + vmovups %xmm12, 4 * SIZE(CO1) + vmovups %xmm9, 0 * SIZE(CO2) + vmovups %xmm13, 4 * SIZE(CO2) + vmovups %xmm10, 0 * SIZE(CO1, LDC, 2) + vmovups %xmm14, 4 * SIZE(CO1, LDC, 2) + vmovups %xmm11, 0 * SIZE(CO2, LDC, 2) + vmovups %xmm15, 4 * SIZE(CO2, LDC, 2) + prefetcht0 64(CO1) + prefetcht0 64(CO1,LDC, 2) + prefetcht0 64(CO2) + prefetcht0 64(CO2,LDC, 2) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + leaq (,%rax, 8), %rax + leaq (AO, %rax, 4), AO + leaq (BO, %rax, 8), BO +#endif + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $8, KK +#endif + + addq $8 * SIZE, CO1 # coffset += 4 + addq $8 * SIZE, CO2 # coffset += 4 + decq I # i -- + jg .L11 + ALIGN_4 + +.L20: + testq $4, M + je .L30 + +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + + leaq BUFFER, BO +#else + leaq BUFFER, BO + movq KK, %rax + leaq (, %rax, 8), %rax + leaq (AO, %rax, 2), AO + leaq (BO, %rax, 8), BO +#endif + + movaps -32 * SIZE(AO), %xmm8 + movaps -16 * SIZE(AO), %xmm10 + + movaps 0 * SIZE(BO), %xmm9 + movaps 16 * SIZE(BO), %xmm11 + movaps 32 * SIZE(BO), %xmm13 + movaps 48 * SIZE(BO), %xmm15 + + xorps %xmm0, %xmm0 + xorps %xmm1, %xmm1 + xorps %xmm2, %xmm2 + xorps %xmm3, %xmm3 + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $4, %rax +#else + addq $4, %rax +#endif + movq %rax, KKK +#endif + sarq $3, %rax + je .L25 + ALIGN_4 + +.L22: + mulps %xmm8, %xmm9 + addps %xmm9, %xmm0 + movaps 4 * SIZE(BO), %xmm9 + mulps %xmm8, %xmm9 + addps %xmm9, %xmm1 + movaps 8 * SIZE(BO), %xmm9 + mulps %xmm8, %xmm9 + mulps 12 * SIZE(BO), %xmm8 + addps %xmm9, %xmm2 + movaps 64 * SIZE(BO), %xmm9 + addps %xmm8, %xmm3 + movaps -28 * SIZE(AO), %xmm8 + + mulps %xmm8, %xmm11 + addps %xmm11, %xmm0 + movaps 20 * SIZE(BO), %xmm11 + mulps %xmm8, %xmm11 + addps %xmm11, %xmm1 + movaps 24 * SIZE(BO), %xmm11 + mulps %xmm8, %xmm11 + mulps 28 * SIZE(BO), %xmm8 + addps %xmm11, %xmm2 + movaps 80 * SIZE(BO), %xmm11 + addps %xmm8, %xmm3 + movaps -24 * SIZE(AO), %xmm8 + + mulps %xmm8, %xmm13 + addps %xmm13, %xmm0 + movaps 36 * SIZE(BO), %xmm13 + mulps %xmm8, %xmm13 + addps %xmm13, %xmm1 + movaps 40 * SIZE(BO), %xmm13 + mulps %xmm8, %xmm13 + mulps 44 * SIZE(BO), %xmm8 + addps %xmm13, %xmm2 + movaps 96 * SIZE(BO), %xmm13 + addps %xmm8, %xmm3 + movaps -20 * SIZE(AO), %xmm8 + + mulps %xmm8, %xmm15 + addps %xmm15, %xmm0 + movaps 52 * SIZE(BO), %xmm15 + mulps %xmm8, %xmm15 + addps %xmm15, %xmm1 + movaps 56 * SIZE(BO), %xmm15 + mulps %xmm8, %xmm15 + mulps 60 * SIZE(BO), %xmm8 + addps %xmm15, %xmm2 + movaps 112 * SIZE(BO), %xmm15 + addps %xmm8, %xmm3 + movaps 0 * SIZE(AO), %xmm8 + + mulps %xmm10, %xmm9 + addps %xmm9, %xmm0 + movaps 68 * SIZE(BO), %xmm9 + mulps %xmm10, %xmm9 + addps %xmm9, %xmm1 + movaps 72 * SIZE(BO), %xmm9 + mulps %xmm10, %xmm9 + mulps 76 * SIZE(BO), %xmm10 + addps %xmm9, %xmm2 + movaps 128 * SIZE(BO), %xmm9 + addps %xmm10, %xmm3 + movaps -12 * SIZE(AO), %xmm10 + + mulps %xmm10, %xmm11 + addps %xmm11, %xmm0 + movaps 84 * SIZE(BO), %xmm11 + mulps %xmm10, %xmm11 + addps %xmm11, %xmm1 + movaps 88 * SIZE(BO), %xmm11 + mulps %xmm10, %xmm11 + mulps 92 * SIZE(BO), %xmm10 + addps %xmm11, %xmm2 + movaps 144 * SIZE(BO), %xmm11 + addps %xmm10, %xmm3 + movaps -8 * SIZE(AO), %xmm10 + + mulps %xmm10, %xmm13 + addps %xmm13, %xmm0 + movaps 100 * SIZE(BO), %xmm13 + mulps %xmm10, %xmm13 + addps %xmm13, %xmm1 + movaps 104 * SIZE(BO), %xmm13 + mulps %xmm10, %xmm13 + mulps 108 * SIZE(BO), %xmm10 + addps %xmm13, %xmm2 + movaps 160 * SIZE(BO), %xmm13 + addps %xmm10, %xmm3 + movaps -4 * SIZE(AO), %xmm10 + + mulps %xmm10, %xmm15 + addps %xmm15, %xmm0 + movaps 116 * SIZE(BO), %xmm15 + mulps %xmm10, %xmm15 + addps %xmm15, %xmm1 + movaps 120 * SIZE(BO), %xmm15 + mulps %xmm10, %xmm15 + mulps 124 * SIZE(BO), %xmm10 + addps %xmm15, %xmm2 + movaps 176 * SIZE(BO), %xmm15 + addps %xmm10, %xmm3 + movaps 16 * SIZE(AO), %xmm10 + + addq $ 32 * SIZE, AO + addq $128 * SIZE, BO + decq %rax + jne .L22 + ALIGN_4 + +.L25: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + movaps ALPHA, %xmm15 + andq $7, %rax # if (k & 1) + BRANCH + je .L28 + ALIGN_4 + +.L26: + mulps %xmm8, %xmm9 + addps %xmm9, %xmm0 + movaps 4 * SIZE(BO), %xmm9 + mulps %xmm8, %xmm9 + addps %xmm9, %xmm1 + movaps 8 * SIZE(BO), %xmm9 + mulps %xmm8, %xmm9 + mulps 12 * SIZE(BO), %xmm8 + addps %xmm9, %xmm2 + movaps 16 * SIZE(BO), %xmm9 + addps %xmm8, %xmm3 + movaps -28 * SIZE(AO), %xmm8 + + addq $ 4 * SIZE, AO # aoffset += 4 + addq $16 * SIZE, BO # boffset1 += 8 + decq %rax + jg .L26 + ALIGN_4 + +.L28: + mulps %xmm15, %xmm0 + mulps %xmm15, %xmm1 + mulps %xmm15, %xmm2 + mulps %xmm15, %xmm3 + +#ifndef TRMMKERNEL + movsd 0 * SIZE(CO1), %xmm8 + movhps 2 * SIZE(CO1), %xmm8 + movsd 0 * SIZE(CO2), %xmm10 + movhps 2 * SIZE(CO2), %xmm10 + + movsd 0 * SIZE(CO1, LDC, 2), %xmm12 + movhps 2 * SIZE(CO1, LDC, 2), %xmm12 + movsd 0 * SIZE(CO2, LDC, 2), %xmm14 + movhps 2 * SIZE(CO2, LDC, 2), %xmm14 + + addps %xmm8, %xmm0 + addps %xmm10, %xmm1 + addps %xmm12, %xmm2 + addps %xmm14, %xmm3 +#endif + + vmovups %xmm0, 0 * SIZE(CO1) + vmovups %xmm1, 0 * SIZE(CO2) + + vmovups %xmm2, 0 * SIZE(CO1, LDC, 2) + vmovups %xmm3, 0 * SIZE(CO2, LDC, 2) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + leaq (,%rax, 8), %rax + leaq (AO, %rax, 2), AO + leaq (BO, %rax, 8), BO +#endif + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $4, KK +#endif + + addq $4 * SIZE, CO1 # coffset += 4 + addq $4 * SIZE, CO2 # coffset += 4 + ALIGN_4 + +.L30: + testq $2, M + je .L40 + +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + + leaq BUFFER, BO +#else + leaq BUFFER, BO + movq KK, %rax + leaq (, %rax, 8), %rax + leaq (AO, %rax, 1), AO + leaq (BO, %rax, 8), BO +#endif + + movaps -32 * SIZE(AO), %xmm8 + movaps -24 * SIZE(AO), %xmm10 + + movaps 0 * SIZE(BO), %xmm9 + movaps 16 * SIZE(BO), %xmm11 + movaps 32 * SIZE(BO), %xmm13 + movaps 48 * SIZE(BO), %xmm15 + + xorps %xmm0, %xmm0 + xorps %xmm1, %xmm1 + xorps %xmm2, %xmm2 + xorps %xmm3, %xmm3 + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $2, %rax +#else + addq $4, %rax +#endif + movq %rax, KKK +#endif + sarq $3, %rax + je .L35 + ALIGN_4 + +.L32: + mulps %xmm8, %xmm9 + addps %xmm9, %xmm0 + movsd 4 * SIZE(BO), %xmm9 + mulps %xmm8, %xmm9 + addps %xmm9, %xmm1 + movsd 8 * SIZE(BO), %xmm9 + mulps %xmm8, %xmm9 + addps %xmm9, %xmm2 + movsd 12 * SIZE(BO), %xmm9 + mulps %xmm8, %xmm9 + movsd -30 * SIZE(AO), %xmm8 + addps %xmm9, %xmm3 + movsd 64 * SIZE(BO), %xmm9 + + mulps %xmm8, %xmm11 + addps %xmm11, %xmm0 + movsd 20 * SIZE(BO), %xmm11 + mulps %xmm8, %xmm11 + addps %xmm11, %xmm1 + movsd 24 * SIZE(BO), %xmm11 + mulps %xmm8, %xmm11 + addps %xmm11, %xmm2 + movsd 28 * SIZE(BO), %xmm11 + mulps %xmm8, %xmm11 + movsd -28 * SIZE(AO), %xmm8 + addps %xmm11, %xmm3 + movsd 80 * SIZE(BO), %xmm11 + + mulps %xmm8, %xmm13 + addps %xmm13, %xmm0 + movsd 36 * SIZE(BO), %xmm13 + mulps %xmm8, %xmm13 + addps %xmm13, %xmm1 + movsd 40 * SIZE(BO), %xmm13 + mulps %xmm8, %xmm13 + addps %xmm13, %xmm2 + movsd 44 * SIZE(BO), %xmm13 + mulps %xmm8, %xmm13 + movsd -26 * SIZE(AO), %xmm8 + addps %xmm13, %xmm3 + movsd 96 * SIZE(BO), %xmm13 + + mulps %xmm8, %xmm15 + addps %xmm15, %xmm0 + movsd 52 * SIZE(BO), %xmm15 + mulps %xmm8, %xmm15 + addps %xmm15, %xmm1 + movsd 56 * SIZE(BO), %xmm15 + mulps %xmm8, %xmm15 + addps %xmm15, %xmm2 + movsd 60 * SIZE(BO), %xmm15 + mulps %xmm8, %xmm15 + movsd -16 * SIZE(AO), %xmm8 + addps %xmm15, %xmm3 + movsd 112 * SIZE(BO), %xmm15 + + mulps %xmm10, %xmm9 + addps %xmm9, %xmm0 + movsd 68 * SIZE(BO), %xmm9 + mulps %xmm10, %xmm9 + addps %xmm9, %xmm1 + movsd 72 * SIZE(BO), %xmm9 + mulps %xmm10, %xmm9 + addps %xmm9, %xmm2 + movsd 76 * SIZE(BO), %xmm9 + mulps %xmm10, %xmm9 + movsd -22 * SIZE(AO), %xmm10 + addps %xmm9, %xmm3 + movsd 128 * SIZE(BO), %xmm9 + + mulps %xmm10, %xmm11 + addps %xmm11, %xmm0 + movsd 84 * SIZE(BO), %xmm11 + mulps %xmm10, %xmm11 + addps %xmm11, %xmm1 + movsd 88 * SIZE(BO), %xmm11 + mulps %xmm10, %xmm11 + addps %xmm11, %xmm2 + movsd 92 * SIZE(BO), %xmm11 + mulps %xmm10, %xmm11 + movsd -20 * SIZE(AO), %xmm10 + addps %xmm11, %xmm3 + movsd 144 * SIZE(BO), %xmm11 + + mulps %xmm10, %xmm13 + addps %xmm13, %xmm0 + movsd 100 * SIZE(BO), %xmm13 + mulps %xmm10, %xmm13 + addps %xmm13, %xmm1 + movsd 104 * SIZE(BO), %xmm13 + mulps %xmm10, %xmm13 + addps %xmm13, %xmm2 + movsd 108 * SIZE(BO), %xmm13 + mulps %xmm10, %xmm13 + movsd -18 * SIZE(AO), %xmm10 + addps %xmm13, %xmm3 + movsd 160 * SIZE(BO), %xmm13 + + mulps %xmm10, %xmm15 + addps %xmm15, %xmm0 + movsd 116 * SIZE(BO), %xmm15 + mulps %xmm10, %xmm15 + addps %xmm15, %xmm1 + movsd 120 * SIZE(BO), %xmm15 + mulps %xmm10, %xmm15 + addps %xmm15, %xmm2 + movsd 124 * SIZE(BO), %xmm15 + mulps %xmm10, %xmm15 + movsd -8 * SIZE(AO), %xmm10 + addps %xmm15, %xmm3 + movsd 176 * SIZE(BO), %xmm15 + + addq $ 16 * SIZE, AO + addq $128 * SIZE, BO + decq %rax + jne .L32 + ALIGN_4 + +.L35: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + movaps ALPHA, %xmm15 + andq $7, %rax # if (k & 1) + BRANCH + je .L38 + ALIGN_4 + +.L36: + mulps %xmm8, %xmm9 + addps %xmm9, %xmm0 + movsd 4 * SIZE(BO), %xmm9 + mulps %xmm8, %xmm9 + addps %xmm9, %xmm1 + movsd 8 * SIZE(BO), %xmm9 + mulps %xmm8, %xmm9 + addps %xmm9, %xmm2 + movsd 12 * SIZE(BO), %xmm9 + mulps %xmm8, %xmm9 + movsd -30 * SIZE(AO), %xmm8 + addps %xmm9, %xmm3 + movsd 16 * SIZE(BO), %xmm9 + + addq $ 2 * SIZE, AO # aoffset += 4 + addq $16 * SIZE, BO # boffset1 += 8 + decq %rax + jg .L36 + ALIGN_4 + +.L38: + mulps %xmm15, %xmm0 + mulps %xmm15, %xmm1 + mulps %xmm15, %xmm2 + mulps %xmm15, %xmm3 + +#ifndef TRMMKERNEL + movsd 0 * SIZE(CO1), %xmm8 + movsd 0 * SIZE(CO2), %xmm10 + movsd 0 * SIZE(CO1, LDC, 2), %xmm12 + movsd 0 * SIZE(CO2, LDC, 2), %xmm14 + + addps %xmm8, %xmm0 + addps %xmm10, %xmm1 + addps %xmm12, %xmm2 + addps %xmm14, %xmm3 +#endif + + movsd %xmm0, 0 * SIZE(CO1) + movsd %xmm1, 0 * SIZE(CO2) + movsd %xmm2, 0 * SIZE(CO1, LDC, 2) + movsd %xmm3, 0 * SIZE(CO2, LDC, 2) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + leaq (,%rax, 8), %rax + leaq (AO, %rax, 1), AO + leaq (BO, %rax, 8), BO +#endif + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $2, KK +#endif + + addq $2 * SIZE, CO1 # coffset += 4 + addq $2 * SIZE, CO2 # coffset += 4 + ALIGN_4 + +.L40: + testq $1, M + je .L49 + +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + + leaq BUFFER, BO +#else + leaq BUFFER, BO + movq KK, %rax + leaq (, %rax, 4), %rax + leaq (AO, %rax, 1), AO + leaq (BO, %rax, 8), BO + leaq (BO, %rax, 8), BO +#endif + + movss -32 * SIZE(AO), %xmm8 + movss -28 * SIZE(AO), %xmm10 + + movss 0 * SIZE(BO), %xmm9 + movss 16 * SIZE(BO), %xmm11 + movss 32 * SIZE(BO), %xmm13 + movss 48 * SIZE(BO), %xmm15 + + xorps %xmm0, %xmm0 + xorps %xmm1, %xmm1 + xorps %xmm2, %xmm2 + xorps %xmm3, %xmm3 + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $1, %rax +#else + addq $4, %rax +#endif + movq %rax, KKK +#endif + sarq $3, %rax + je .L45 + ALIGN_4 + +.L42: + mulss %xmm8, %xmm9 + addss %xmm9, %xmm0 + movss 4 * SIZE(BO), %xmm9 + mulss %xmm8, %xmm9 + addss %xmm9, %xmm1 + movss 8 * SIZE(BO), %xmm9 + mulss %xmm8, %xmm9 + addss %xmm9, %xmm2 + movss 12 * SIZE(BO), %xmm9 + mulss %xmm8, %xmm9 + movss -31 * SIZE(AO), %xmm8 + addss %xmm9, %xmm3 + movss 64 * SIZE(BO), %xmm9 + + mulss %xmm8, %xmm11 + addss %xmm11, %xmm0 + movss 20 * SIZE(BO), %xmm11 + mulss %xmm8, %xmm11 + addss %xmm11, %xmm1 + movss 24 * SIZE(BO), %xmm11 + mulss %xmm8, %xmm11 + addss %xmm11, %xmm2 + movss 28 * SIZE(BO), %xmm11 + mulss %xmm8, %xmm11 + movss -30 * SIZE(AO), %xmm8 + addss %xmm11, %xmm3 + movss 80 * SIZE(BO), %xmm11 + + mulss %xmm8, %xmm13 + addss %xmm13, %xmm0 + movss 36 * SIZE(BO), %xmm13 + mulss %xmm8, %xmm13 + addss %xmm13, %xmm1 + movss 40 * SIZE(BO), %xmm13 + mulss %xmm8, %xmm13 + addss %xmm13, %xmm2 + movss 44 * SIZE(BO), %xmm13 + mulss %xmm8, %xmm13 + movss -29 * SIZE(AO), %xmm8 + addss %xmm13, %xmm3 + movss 96 * SIZE(BO), %xmm13 + + mulss %xmm8, %xmm15 + addss %xmm15, %xmm0 + movss 52 * SIZE(BO), %xmm15 + mulss %xmm8, %xmm15 + addss %xmm15, %xmm1 + movss 56 * SIZE(BO), %xmm15 + mulss %xmm8, %xmm15 + addss %xmm15, %xmm2 + movss 60 * SIZE(BO), %xmm15 + mulss %xmm8, %xmm15 + movss -24 * SIZE(AO), %xmm8 + addss %xmm15, %xmm3 + movss 112 * SIZE(BO), %xmm15 + + mulss %xmm10, %xmm9 + addss %xmm9, %xmm0 + movss 68 * SIZE(BO), %xmm9 + mulss %xmm10, %xmm9 + addss %xmm9, %xmm1 + movss 72 * SIZE(BO), %xmm9 + mulss %xmm10, %xmm9 + addss %xmm9, %xmm2 + movss 76 * SIZE(BO), %xmm9 + mulss %xmm10, %xmm9 + movss -27 * SIZE(AO), %xmm10 + addss %xmm9, %xmm3 + movss 128 * SIZE(BO), %xmm9 + + mulss %xmm10, %xmm11 + addss %xmm11, %xmm0 + movss 84 * SIZE(BO), %xmm11 + mulss %xmm10, %xmm11 + addss %xmm11, %xmm1 + movss 88 * SIZE(BO), %xmm11 + mulss %xmm10, %xmm11 + addss %xmm11, %xmm2 + movss 92 * SIZE(BO), %xmm11 + mulss %xmm10, %xmm11 + movss -26 * SIZE(AO), %xmm10 + addss %xmm11, %xmm3 + movss 144 * SIZE(BO), %xmm11 + + mulss %xmm10, %xmm13 + addss %xmm13, %xmm0 + movss 100 * SIZE(BO), %xmm13 + mulss %xmm10, %xmm13 + addss %xmm13, %xmm1 + movss 104 * SIZE(BO), %xmm13 + mulss %xmm10, %xmm13 + addss %xmm13, %xmm2 + movss 108 * SIZE(BO), %xmm13 + mulss %xmm10, %xmm13 + movss -25 * SIZE(AO), %xmm10 + addss %xmm13, %xmm3 + movss 160 * SIZE(BO), %xmm13 + + mulss %xmm10, %xmm15 + addss %xmm15, %xmm0 + movss 116 * SIZE(BO), %xmm15 + mulss %xmm10, %xmm15 + addss %xmm15, %xmm1 + movss 120 * SIZE(BO), %xmm15 + mulss %xmm10, %xmm15 + addss %xmm15, %xmm2 + movss 124 * SIZE(BO), %xmm15 + mulss %xmm10, %xmm15 + movss -20 * SIZE(AO), %xmm10 + addss %xmm15, %xmm3 + movss 176 * SIZE(BO), %xmm15 + + addq $ 8 * SIZE, AO + addq $128 * SIZE, BO + decq %rax + jne .L42 + ALIGN_4 + +.L45: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + movaps ALPHA, %xmm15 + andq $7, %rax # if (k & 1) + BRANCH + je .L48 + ALIGN_4 + +.L46: + mulps %xmm8, %xmm9 + addps %xmm9, %xmm0 + movss 4 * SIZE(BO), %xmm9 + mulps %xmm8, %xmm9 + addps %xmm9, %xmm1 + movss 8 * SIZE(BO), %xmm9 + mulps %xmm8, %xmm9 + addps %xmm9, %xmm2 + movss 12 * SIZE(BO), %xmm9 + mulps %xmm8, %xmm9 + movss -31 * SIZE(AO), %xmm8 + addps %xmm9, %xmm3 + movss 16 * SIZE(BO), %xmm9 + + addq $ 1 * SIZE, AO # aoffset += 4 + addq $16 * SIZE, BO # boffset1 += 8 + decq %rax + jg .L46 + ALIGN_4 + +.L48: + mulss %xmm15, %xmm0 + mulss %xmm15, %xmm1 + mulss %xmm15, %xmm2 + mulss %xmm15, %xmm3 + +#ifndef TRMMKERNEL + movss 0 * SIZE(CO1), %xmm8 + movss 0 * SIZE(CO2), %xmm10 + movss 0 * SIZE(CO1, LDC, 2), %xmm12 + movss 0 * SIZE(CO2, LDC, 2), %xmm14 + + addss %xmm8, %xmm0 + addss %xmm10, %xmm1 + addss %xmm12, %xmm2 + addss %xmm14, %xmm3 +#endif + + movss %xmm0, 0 * SIZE(CO1) + movss %xmm1, 0 * SIZE(CO2) + movss %xmm2, 0 * SIZE(CO1, LDC, 2) + movss %xmm3, 0 * SIZE(CO2, LDC, 2) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + leaq (,%rax, 4), %rax + leaq (AO, %rax, 1), AO + leaq (BO, %rax, 8), BO + leaq (BO, %rax, 8), BO +#endif + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $1, KK +#endif + ALIGN_4 + +.L49: +#if defined(TRMMKERNEL) && !defined(LEFT) + addl $4, KK +#endif + leaq (C, LDC, 4), C # c += 4 * ldc + decq J # j -- + jg .L01 + +.L50: + testq $2, N + je .L100 + +.L51: +#if defined(TRMMKERNEL) && defined(LEFT) + movq OFFSET, %rax + movq %rax, KK +#endif + +/* Copying to Sub Buffer */ + leaq BUFFER, BO + + movq K, %rax + sarq $2, %rax + jle .L53 + ALIGN_4 + +.L52: + + movaps 0 * SIZE(B), %xmm3 + movaps 4 * SIZE(B), %xmm7 + + + pshufd $0x00, %xmm3, %xmm0 + pshufd $0x55, %xmm3, %xmm1 + pshufd $0xaa, %xmm3, %xmm2 + pshufd $0xff, %xmm3, %xmm3 + + + pshufd $0x00, %xmm7, %xmm4 + pshufd $0x55, %xmm7, %xmm5 + pshufd $0xaa, %xmm7, %xmm6 + pshufd $0xff, %xmm7, %xmm7 + + movaps %xmm0, 0 * SIZE(BO) + movaps %xmm1, 4 * SIZE(BO) + movaps %xmm2, 8 * SIZE(BO) + movaps %xmm3, 12 * SIZE(BO) + movaps %xmm4, 16 * SIZE(BO) + movaps %xmm5, 20 * SIZE(BO) + movaps %xmm6, 24 * SIZE(BO) + movaps %xmm7, 28 * SIZE(BO) + + addq $ 8 * SIZE, B + addq $32 * SIZE, BO + + decq %rax + jne .L52 + ALIGN_4 + +.L53: + movq K, %rax + andq $3, %rax + BRANCH + jle .L60 + ALIGN_4 + +.L54: + movsd 0 * SIZE(B), %xmm3 + + pshufd $0x00, %xmm3, %xmm0 + pshufd $0x55, %xmm3, %xmm1 + + pshufd $0x00, %xmm7, %xmm4 + pshufd $0x55, %xmm7, %xmm5 + pshufd $0xaa, %xmm7, %xmm6 + pshufd $0xff, %xmm7, %xmm7 + + movaps %xmm0, 0 * SIZE(BO) + movaps %xmm1, 4 * SIZE(BO) + + addq $ 2 * SIZE, B + addq $ 8 * SIZE, BO + decq %rax + jne .L54 + ALIGN_4 + +.L60: + movq C, CO1 # coffset1 = c + leaq (C, LDC, 1), CO2 # coffset2 = c + ldc + movq A, AO # aoffset = a + + movq M, I + sarq $3, I # i = (m >> 3) + jle .L70 + ALIGN_4 + +.L61: +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + + leaq BUFFER, BO +#else + leaq BUFFER, BO + movq KK, %rax + leaq (, %rax, 8), %rax + leaq (AO, %rax, 4), AO + leaq (BO, %rax, 4), BO +#endif + + movaps -32 * SIZE(AO), %xmm8 + movaps -16 * SIZE(AO), %xmm10 + movaps 0 * SIZE(AO), %xmm12 + movaps 16 * SIZE(AO), %xmm14 + + movaps 0 * SIZE(BO), %xmm9 + movaps 16 * SIZE(BO), %xmm11 + movaps 32 * SIZE(BO), %xmm13 + movaps 48 * SIZE(BO), %xmm15 + + xorps %xmm0, %xmm0 + xorps %xmm1, %xmm1 + + prefetchw 4 * SIZE(CO1) + xorps %xmm4, %xmm4 + prefetchw 4 * SIZE(CO2) + xorps %xmm5, %xmm5 + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $8, %rax +#else + addq $2, %rax +#endif + movq %rax, KKK +#endif + sarq $3, %rax + je .L65 + ALIGN_4 + +.L62: + mulps %xmm8, %xmm9 + mulps 4 * SIZE(BO), %xmm8 + addps %xmm9, %xmm0 + movaps 0 * SIZE(BO), %xmm9 + addps %xmm8, %xmm1 + movaps -28 * SIZE(AO), %xmm8 + mulps %xmm8, %xmm9 + mulps 4 * SIZE(BO), %xmm8 + addps %xmm9, %xmm4 + movaps 8 * SIZE(BO), %xmm9 + addps %xmm8, %xmm5 + movaps -24 * SIZE(AO), %xmm8 + + mulps %xmm8, %xmm9 + mulps 12 * SIZE(BO), %xmm8 + addps %xmm9, %xmm0 + movaps 8 * SIZE(BO), %xmm9 + addps %xmm8, %xmm1 + movaps -20 * SIZE(AO), %xmm8 + mulps %xmm8, %xmm9 + mulps 12 * SIZE(BO), %xmm8 + addps %xmm9, %xmm4 + movaps 64 * SIZE(BO), %xmm9 + addps %xmm8, %xmm5 + movaps 32 * SIZE(AO), %xmm8 + + mulps %xmm10, %xmm11 + mulps 20 * SIZE(BO), %xmm10 + addps %xmm11, %xmm0 + movaps 16 * SIZE(BO), %xmm11 + addps %xmm10, %xmm1 + movaps -12 * SIZE(AO), %xmm10 + mulps %xmm10, %xmm11 + mulps 20 * SIZE(BO), %xmm10 + addps %xmm11, %xmm4 + movaps 24 * SIZE(BO), %xmm11 + addps %xmm10, %xmm5 + movaps -8 * SIZE(AO), %xmm10 + + mulps %xmm10, %xmm11 + mulps 28 * SIZE(BO), %xmm10 + addps %xmm11, %xmm0 + movaps 24 * SIZE(BO), %xmm11 + addps %xmm10, %xmm1 + movaps -4 * SIZE(AO), %xmm10 + mulps %xmm10, %xmm11 + mulps 28 * SIZE(BO), %xmm10 + addps %xmm11, %xmm4 + movaps 80 * SIZE(BO), %xmm11 + addps %xmm10, %xmm5 + movaps 48 * SIZE(AO), %xmm10 + + mulps %xmm12, %xmm13 + mulps 36 * SIZE(BO), %xmm12 + addps %xmm13, %xmm0 + movaps 32 * SIZE(BO), %xmm13 + addps %xmm12, %xmm1 + movaps 4 * SIZE(AO), %xmm12 + mulps %xmm12, %xmm13 + mulps 36 * SIZE(BO), %xmm12 + addps %xmm13, %xmm4 + movaps 40 * SIZE(BO), %xmm13 + addps %xmm12, %xmm5 + movaps 8 * SIZE(AO), %xmm12 + + mulps %xmm12, %xmm13 + mulps 44 * SIZE(BO), %xmm12 + addps %xmm13, %xmm0 + movaps 40 * SIZE(BO), %xmm13 + addps %xmm12, %xmm1 + movaps 12 * SIZE(AO), %xmm12 + mulps %xmm12, %xmm13 + mulps 44 * SIZE(BO), %xmm12 + addps %xmm13, %xmm4 + movaps 96 * SIZE(BO), %xmm13 + addps %xmm12, %xmm5 + movaps 64 * SIZE(AO), %xmm12 + + mulps %xmm14, %xmm15 + mulps 52 * SIZE(BO), %xmm14 + addps %xmm15, %xmm0 + movaps 48 * SIZE(BO), %xmm15 + addps %xmm14, %xmm1 + movaps 20 * SIZE(AO), %xmm14 + mulps %xmm14, %xmm15 + mulps 52 * SIZE(BO), %xmm14 + addps %xmm15, %xmm4 + movaps 56 * SIZE(BO), %xmm15 + addps %xmm14, %xmm5 + movaps 24 * SIZE(AO), %xmm14 + + mulps %xmm14, %xmm15 + mulps 60 * SIZE(BO), %xmm14 + addps %xmm15, %xmm0 + movaps 56 * SIZE(BO), %xmm15 + addps %xmm14, %xmm1 + movaps 28 * SIZE(AO), %xmm14 + mulps %xmm14, %xmm15 + mulps 60 * SIZE(BO), %xmm14 + addps %xmm15, %xmm4 + movaps 112 * SIZE(BO), %xmm15 + addps %xmm14, %xmm5 + movaps 80 * SIZE(AO), %xmm14 + + addq $64 * SIZE, AO + addq $64 * SIZE, BO + decq %rax + jne .L62 + ALIGN_4 + +.L65: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + movaps ALPHA, %xmm15 + andq $7, %rax # if (k & 1) + BRANCH + je .L68 + ALIGN_4 + +.L66: + mulps %xmm8, %xmm9 + mulps 4 * SIZE(BO), %xmm8 + addps %xmm9, %xmm0 + movaps 0 * SIZE(BO), %xmm9 + addps %xmm8, %xmm1 + movaps -28 * SIZE(AO), %xmm8 + mulps %xmm8, %xmm9 + mulps 4 * SIZE(BO), %xmm8 + addps %xmm9, %xmm4 + movaps 8 * SIZE(BO), %xmm9 + addps %xmm8, %xmm5 + movaps -24 * SIZE(AO), %xmm8 + + addq $8 * SIZE, AO # aoffset += 4 + addq $8 * SIZE, BO # boffset1 += 8 + decq %rax + jg .L66 + ALIGN_4 + +.L68: +#ifndef TRMMKERNEL + movsd 0 * SIZE(CO1), %xmm8 + movhps 2 * SIZE(CO1), %xmm8 + movsd 4 * SIZE(CO1), %xmm9 + movhps 6 * SIZE(CO1), %xmm9 + + movsd 0 * SIZE(CO2), %xmm10 + movhps 2 * SIZE(CO2), %xmm10 + movsd 4 * SIZE(CO2), %xmm11 + movhps 6 * SIZE(CO2), %xmm11 +#endif + + mulps %xmm15, %xmm0 + mulps %xmm15, %xmm4 + mulps %xmm15, %xmm1 + mulps %xmm15, %xmm5 + +#ifndef TRMMKERNEL + addps %xmm8, %xmm0 + addps %xmm9, %xmm4 + addps %xmm10, %xmm1 + addps %xmm11, %xmm5 +#endif + + vmovups %xmm0, 0 * SIZE(CO1) + vmovups %xmm4, 4 * SIZE(CO1) + + vmovups %xmm1, 0 * SIZE(CO2) + vmovups %xmm5, 4 * SIZE(CO2) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + leaq (,%rax, 8), %rax + leaq (AO, %rax, 4), AO + leaq (BO, %rax, 4), BO +#endif + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $8, KK +#endif + + addq $8 * SIZE, CO1 # coffset += 4 + addq $8 * SIZE, CO2 # coffset += 4 + decq I # i -- + jg .L61 + ALIGN_4 + +.L70: + testq $4, M + je .L80 + + +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + + leaq BUFFER, BO +#else + leaq BUFFER, BO + movq KK, %rax + leaq (, %rax, 8), %rax + leaq (AO, %rax, 2), AO + leaq (BO, %rax, 4), BO +#endif + + movaps -32 * SIZE(AO), %xmm8 + movaps -16 * SIZE(AO), %xmm10 + + movaps 0 * SIZE(BO), %xmm9 + movaps 16 * SIZE(BO), %xmm11 + movaps 32 * SIZE(BO), %xmm13 + movaps 48 * SIZE(BO), %xmm15 + + xorps %xmm0, %xmm0 + xorps %xmm1, %xmm1 + xorps %xmm2, %xmm2 + xorps %xmm3, %xmm3 + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $4, %rax +#else + addq $2, %rax +#endif + movq %rax, KKK +#endif + sarq $3, %rax + je .L75 + ALIGN_4 + +.L72: + mulps %xmm8, %xmm9 + + mulps 4 * SIZE(BO), %xmm8 + addps %xmm9, %xmm0 + movaps 8 * SIZE(BO), %xmm9 + addps %xmm8, %xmm1 + movaps -28 * SIZE(AO), %xmm8 + + mulps %xmm8, %xmm9 + mulps 12 * SIZE(BO), %xmm8 + addps %xmm9, %xmm2 + movaps 64 * SIZE(BO), %xmm9 + addps %xmm8, %xmm3 + movaps -24 * SIZE(AO), %xmm8 + + mulps %xmm8, %xmm11 + mulps 20 * SIZE(BO), %xmm8 + addps %xmm11, %xmm0 + movaps 24 * SIZE(BO), %xmm11 + addps %xmm8, %xmm1 + movaps -20 * SIZE(AO), %xmm8 + + mulps %xmm8, %xmm11 + mulps 28 * SIZE(BO), %xmm8 + addps %xmm11, %xmm2 + movaps 80 * SIZE(BO), %xmm11 + addps %xmm8, %xmm3 + movaps 0 * SIZE(AO), %xmm8 + + mulps %xmm10, %xmm13 + mulps 36 * SIZE(BO), %xmm10 + addps %xmm13, %xmm0 + movaps 40 * SIZE(BO), %xmm13 + addps %xmm10, %xmm1 + movaps -12 * SIZE(AO), %xmm10 + + mulps %xmm10, %xmm13 + mulps 44 * SIZE(BO), %xmm10 + addps %xmm13, %xmm2 + movaps 96 * SIZE(BO), %xmm13 + addps %xmm10, %xmm3 + movaps -8 * SIZE(AO), %xmm10 + + mulps %xmm10, %xmm15 + mulps 52 * SIZE(BO), %xmm10 + addps %xmm15, %xmm0 + movaps 56 * SIZE(BO), %xmm15 + addps %xmm10, %xmm1 + movaps -4 * SIZE(AO), %xmm10 + + mulps %xmm10, %xmm15 + mulps 60 * SIZE(BO), %xmm10 + addps %xmm15, %xmm2 + movaps 112 * SIZE(BO), %xmm15 + addps %xmm10, %xmm3 + movaps 16 * SIZE(AO), %xmm10 + + addq $32 * SIZE, AO + addq $64 * SIZE, BO + decq %rax + jne .L72 + ALIGN_4 + +.L75: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + movaps ALPHA, %xmm15 + andq $7, %rax # if (k & 1) + BRANCH + je .L78 + ALIGN_4 + +.L76: + mulps %xmm8, %xmm9 + mulps 4 * SIZE(BO), %xmm8 + addps %xmm9, %xmm0 + movaps 8 * SIZE(BO), %xmm9 + addps %xmm8, %xmm1 + movaps -28 * SIZE(AO), %xmm8 + + addq $4 * SIZE, AO # aoffset += 4 + addq $8 * SIZE, BO # boffset1 += 8 + decq %rax + jg .L76 + ALIGN_4 + +.L78: +#ifndef TRMMKERNEL + movsd 0 * SIZE(CO1), %xmm8 + movhps 2 * SIZE(CO1), %xmm8 + movsd 0 * SIZE(CO2), %xmm10 + movhps 2 * SIZE(CO2), %xmm10 +#endif + + addps %xmm2, %xmm0 + addps %xmm3, %xmm1 + + mulps %xmm15, %xmm0 + mulps %xmm15, %xmm1 + +#ifndef TRMMKERNEL + addps %xmm8, %xmm0 + addps %xmm10, %xmm1 +#endif + + vmovups %xmm0, 0 * SIZE(CO1) + vmovups %xmm1, 0 * SIZE(CO2) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + leaq (,%rax, 8), %rax + leaq (AO, %rax, 2), AO + leaq (BO, %rax, 4), BO +#endif + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $4, KK +#endif + + addq $4 * SIZE, CO1 # coffset += 4 + addq $4 * SIZE, CO2 # coffset += 4 + ALIGN_4 + +.L80: + testq $2, M + je .L90 + +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + + leaq BUFFER, BO +#else + leaq BUFFER, BO + movq KK, %rax + leaq (, %rax, 8), %rax + leaq (AO, %rax, 1), AO + leaq (BO, %rax, 4), BO +#endif + + movaps -32 * SIZE(AO), %xmm8 + movaps -24 * SIZE(AO), %xmm10 + + movaps 0 * SIZE(BO), %xmm9 + movaps 16 * SIZE(BO), %xmm11 + movaps 32 * SIZE(BO), %xmm13 + movaps 48 * SIZE(BO), %xmm15 + + xorps %xmm0, %xmm0 + xorps %xmm1, %xmm1 + xorps %xmm2, %xmm2 + xorps %xmm3, %xmm3 + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $2, %rax +#else + addq $2, %rax +#endif + movq %rax, KKK +#endif + sarq $3, %rax + je .L85 + ALIGN_4 + +.L82: + mulps %xmm8, %xmm9 + addps %xmm9, %xmm0 + movsd 4 * SIZE(BO), %xmm9 + mulps %xmm8, %xmm9 + movsd -30 * SIZE(AO), %xmm8 + addps %xmm9, %xmm1 + movsd 8 * SIZE(BO), %xmm9 + + mulps %xmm8, %xmm9 + addps %xmm9, %xmm2 + movsd 12 * SIZE(BO), %xmm9 + mulps %xmm8, %xmm9 + movsd -28 * SIZE(AO), %xmm8 + addps %xmm9, %xmm3 + movsd 64 * SIZE(BO), %xmm9 + + mulps %xmm8, %xmm11 + addps %xmm11, %xmm0 + movsd 20 * SIZE(BO), %xmm11 + mulps %xmm8, %xmm11 + movsd -26 * SIZE(AO), %xmm8 + addps %xmm11, %xmm1 + movsd 24 * SIZE(BO), %xmm11 + + mulps %xmm8, %xmm11 + addps %xmm11, %xmm2 + movsd 28 * SIZE(BO), %xmm11 + mulps %xmm8, %xmm11 + movsd -16 * SIZE(AO), %xmm8 + addps %xmm11, %xmm3 + movsd 80 * SIZE(BO), %xmm11 + + mulps %xmm10, %xmm13 + addps %xmm13, %xmm0 + movsd 36 * SIZE(BO), %xmm13 + mulps %xmm10, %xmm13 + movsd -22 * SIZE(AO), %xmm10 + addps %xmm13, %xmm1 + movsd 40 * SIZE(BO), %xmm13 + + mulps %xmm10, %xmm13 + addps %xmm13, %xmm2 + movsd 44 * SIZE(BO), %xmm13 + mulps %xmm10, %xmm13 + movsd -20 * SIZE(AO), %xmm10 + addps %xmm13, %xmm3 + movsd 96 * SIZE(BO), %xmm13 + + mulps %xmm10, %xmm15 + addps %xmm15, %xmm0 + movsd 52 * SIZE(BO), %xmm15 + mulps %xmm10, %xmm15 + movsd -18 * SIZE(AO), %xmm10 + addps %xmm15, %xmm1 + movsd 56 * SIZE(BO), %xmm15 + + mulps %xmm10, %xmm15 + addps %xmm15, %xmm2 + movsd 60 * SIZE(BO), %xmm15 + mulps %xmm10, %xmm15 + movsd -8 * SIZE(AO), %xmm10 + addps %xmm15, %xmm3 + movsd 112 * SIZE(BO), %xmm15 + + addq $16 * SIZE, AO + addq $64 * SIZE, BO + decq %rax + jne .L82 + ALIGN_4 + +.L85: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + movaps ALPHA, %xmm15 + andq $7, %rax # if (k & 1) + BRANCH + je .L88 + ALIGN_4 + +.L86: + mulps %xmm8, %xmm9 + addps %xmm9, %xmm0 + movsd 4 * SIZE(BO), %xmm9 + mulps %xmm8, %xmm9 + movsd -30 * SIZE(AO), %xmm8 + addps %xmm9, %xmm1 + movsd 8 * SIZE(BO), %xmm9 + + addq $2 * SIZE, AO # aoffset += 4 + addq $8 * SIZE, BO # boffset1 += 8 + decq %rax + jg .L86 + ALIGN_4 + +.L88: +#ifndef TRMMKERNEL + movsd 0 * SIZE(CO1), %xmm8 + movsd 0 * SIZE(CO2), %xmm10 +#endif + + addps %xmm2, %xmm0 + addps %xmm3, %xmm1 + + mulps %xmm15, %xmm0 + mulps %xmm15, %xmm1 + +#ifndef TRMMKERNEL + addps %xmm8, %xmm0 + addps %xmm10, %xmm1 +#endif + + movsd %xmm0, 0 * SIZE(CO1) + movsd %xmm1, 0 * SIZE(CO2) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + leaq (,%rax, 8), %rax + leaq (AO, %rax, 1), AO + leaq (BO, %rax, 4), BO +#endif + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $2, KK +#endif + + addq $2 * SIZE, CO1 # coffset += 4 + addq $2 * SIZE, CO2 # coffset += 4 + ALIGN_4 + +.L90: + testq $1, M + je .L99 + +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + + leaq BUFFER, BO +#else + leaq BUFFER, BO + movq KK, %rax + leaq (, %rax, 4), %rax + leaq (AO, %rax, 1), AO + leaq (BO, %rax, 8), BO +#endif + + movss -32 * SIZE(AO), %xmm8 + movss -28 * SIZE(AO), %xmm10 + + movss 0 * SIZE(BO), %xmm9 + movss 16 * SIZE(BO), %xmm11 + movss 32 * SIZE(BO), %xmm13 + movss 48 * SIZE(BO), %xmm15 + + xorps %xmm0, %xmm0 + xorps %xmm1, %xmm1 + xorps %xmm2, %xmm2 + xorps %xmm3, %xmm3 + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $1, %rax +#else + addq $2, %rax +#endif + movq %rax, KKK +#endif + sarq $3, %rax + je .L95 + ALIGN_4 + +.L92: + mulps %xmm8, %xmm9 + addps %xmm9, %xmm0 + movss 4 * SIZE(BO), %xmm9 + mulps %xmm8, %xmm9 + movss -31 * SIZE(AO), %xmm8 + addps %xmm9, %xmm1 + movss 8 * SIZE(BO), %xmm9 + + mulps %xmm8, %xmm9 + addps %xmm9, %xmm2 + movss 12 * SIZE(BO), %xmm9 + mulps %xmm8, %xmm9 + movss -30 * SIZE(AO), %xmm8 + addps %xmm9, %xmm3 + movss 64 * SIZE(BO), %xmm9 + + mulps %xmm8, %xmm11 + addps %xmm11, %xmm0 + movss 20 * SIZE(BO), %xmm11 + mulps %xmm8, %xmm11 + movss -29 * SIZE(AO), %xmm8 + addps %xmm11, %xmm1 + movss 24 * SIZE(BO), %xmm11 + + mulps %xmm8, %xmm11 + addps %xmm11, %xmm2 + movss 28 * SIZE(BO), %xmm11 + mulps %xmm8, %xmm11 + movss -24 * SIZE(AO), %xmm8 + addps %xmm11, %xmm3 + movss 80 * SIZE(BO), %xmm11 + + mulps %xmm10, %xmm13 + addps %xmm13, %xmm0 + movss 36 * SIZE(BO), %xmm13 + mulps %xmm10, %xmm13 + movss -27 * SIZE(AO), %xmm10 + addps %xmm13, %xmm1 + movss 40 * SIZE(BO), %xmm13 + + mulps %xmm10, %xmm13 + addps %xmm13, %xmm2 + movss 44 * SIZE(BO), %xmm13 + mulps %xmm10, %xmm13 + movss -26 * SIZE(AO), %xmm10 + addps %xmm13, %xmm3 + movss 96 * SIZE(BO), %xmm13 + + mulps %xmm10, %xmm15 + addps %xmm15, %xmm0 + movss 52 * SIZE(BO), %xmm15 + mulps %xmm10, %xmm15 + movss -25 * SIZE(AO), %xmm10 + addps %xmm15, %xmm1 + movss 56 * SIZE(BO), %xmm15 + + mulps %xmm10, %xmm15 + addps %xmm15, %xmm2 + movss 60 * SIZE(BO), %xmm15 + mulps %xmm10, %xmm15 + movss -20 * SIZE(AO), %xmm10 + addps %xmm15, %xmm3 + movss 112 * SIZE(BO), %xmm15 + + addq $ 8 * SIZE, AO + addq $64 * SIZE, BO + decq %rax + jne .L92 + ALIGN_4 + +.L95: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + movaps ALPHA, %xmm15 + andq $7, %rax # if (k & 1) + BRANCH + je .L98 + ALIGN_4 + +.L96: + mulps %xmm8, %xmm9 + addps %xmm9, %xmm0 + movss 4 * SIZE(BO), %xmm9 + mulps %xmm8, %xmm9 + movss -31 * SIZE(AO), %xmm8 + addps %xmm9, %xmm1 + movss 8 * SIZE(BO), %xmm9 + + addq $1 * SIZE, AO # aoffset += 4 + addq $8 * SIZE, BO # boffset1 += 8 + decq %rax + jg .L96 + ALIGN_4 + +.L98: +#ifndef TRMMKERNEL + movss 0 * SIZE(CO1), %xmm8 + movss 0 * SIZE(CO2), %xmm10 +#endif + + addss %xmm2, %xmm0 + addss %xmm3, %xmm1 + mulss %xmm15, %xmm0 + mulss %xmm15, %xmm1 + +#ifndef TRMMKERNEL + addss %xmm8, %xmm0 + addss %xmm10, %xmm1 +#endif + + movss %xmm0, 0 * SIZE(CO1) + movss %xmm1, 0 * SIZE(CO2) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + leaq (,%rax, 4), %rax + leaq (AO, %rax, 1), AO + leaq (BO, %rax, 8), BO +#endif + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $1, KK +#endif + ALIGN_4 + +.L99: +#if defined(TRMMKERNEL) && !defined(LEFT) + addl $2, KK +#endif + leaq (C, LDC, 2), C # c += 4 * ldc + ALIGN_4 + + +.L100: + testq $1, N + je .L999 + +.L101: +#if defined(TRMMKERNEL) && defined(LEFT) + movq OFFSET, %rax + movq %rax, KK +#endif + +/* Copying to Sub Buffer */ + leaq BUFFER, BO + + movq K, %rax + sarq $3, %rax + jle .L103 + ALIGN_4 + + +.L102: + + movups 0 * SIZE(B), %xmm3 + movups 4 * SIZE(B), %xmm7 + + + pshufd $0x00, %xmm3, %xmm0 + pshufd $0x55, %xmm3, %xmm1 + pshufd $0xaa, %xmm3, %xmm2 + pshufd $0xff, %xmm3, %xmm3 + + + pshufd $0x00, %xmm7, %xmm4 + pshufd $0x55, %xmm7, %xmm5 + pshufd $0xaa, %xmm7, %xmm6 + pshufd $0xff, %xmm7, %xmm7 + + movaps %xmm0, 0 * SIZE(BO) + movaps %xmm1, 4 * SIZE(BO) + movaps %xmm2, 8 * SIZE(BO) + movaps %xmm3, 12 * SIZE(BO) + movaps %xmm4, 16 * SIZE(BO) + movaps %xmm5, 20 * SIZE(BO) + movaps %xmm6, 24 * SIZE(BO) + movaps %xmm7, 28 * SIZE(BO) + + addq $ 8 * SIZE, B + addq $32 * SIZE, BO + + decq %rax + jne .L102 + ALIGN_4 + +.L103: + movq K, %rax + andq $7, %rax + BRANCH + jle .L110 + ALIGN_4 + +.L104: + movss 0 * SIZE(B), %xmm3 + + pshufd $0x00, %xmm3, %xmm0 + + movaps %xmm0, 0 * SIZE(BO) + + addq $ 1 * SIZE, B + addq $ 4 * SIZE, BO + decq %rax + jne .L104 + ALIGN_4 + +.L110: + movq C, CO1 # coffset1 = c + movq A, AO # aoffset = a + + movq M, I + sarq $3, I # i = (m >> 3) + jle .L120 + ALIGN_4 + +.L111: +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + + leaq BUFFER, BO +#else + leaq BUFFER, BO + movq KK, %rax + leaq (, %rax, 8), %rax + leaq (AO, %rax, 4), AO + leaq (BO, %rax, 2), BO +#endif + + movaps -32 * SIZE(AO), %xmm8 + movaps -16 * SIZE(AO), %xmm10 + movaps 0 * SIZE(AO), %xmm12 + movaps 16 * SIZE(AO), %xmm14 + + movaps 0 * SIZE(BO), %xmm9 + movaps 16 * SIZE(BO), %xmm11 + movaps 32 * SIZE(BO), %xmm13 + movaps 48 * SIZE(BO), %xmm15 + + xorps %xmm0, %xmm0 + xorps %xmm1, %xmm1 + + prefetchw 4 * SIZE(CO1) + xorps %xmm4, %xmm4 + xorps %xmm5, %xmm5 + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $8, %rax +#else + addq $1, %rax +#endif + movq %rax, KKK +#endif + sarq $3, %rax + je .L115 + ALIGN_4 + +.L112: + mulps %xmm9, %xmm8 + + mulps -28 * SIZE(AO), %xmm9 + addps %xmm8, %xmm0 + movaps -24 * SIZE(AO), %xmm8 + addps %xmm9, %xmm4 + movaps 4 * SIZE(BO), %xmm9 + + mulps %xmm9, %xmm8 + mulps -20 * SIZE(AO), %xmm9 + addps %xmm8, %xmm0 + movaps 32 * SIZE(AO), %xmm8 + addps %xmm9, %xmm4 + movaps 8 * SIZE(BO), %xmm9 + + mulps %xmm9, %xmm10 + mulps -12 * SIZE(AO), %xmm9 + addps %xmm10, %xmm0 + movaps -8 * SIZE(AO), %xmm10 + addps %xmm9, %xmm4 + movaps 12 * SIZE(BO), %xmm9 + + mulps %xmm9, %xmm10 + mulps -4 * SIZE(AO), %xmm9 + addps %xmm10, %xmm0 + movaps 48 * SIZE(AO), %xmm10 + addps %xmm9, %xmm4 + movaps 32 * SIZE(BO), %xmm9 + + mulps %xmm11, %xmm12 + mulps 4 * SIZE(AO), %xmm11 + addps %xmm12, %xmm0 + movaps 8 * SIZE(AO), %xmm12 + addps %xmm11, %xmm4 + movaps 20 * SIZE(BO), %xmm11 + + mulps %xmm11, %xmm12 + mulps 12 * SIZE(AO), %xmm11 + addps %xmm12, %xmm0 + movaps 64 * SIZE(AO), %xmm12 + addps %xmm11, %xmm4 + movaps 24 * SIZE(BO), %xmm11 + + mulps %xmm11, %xmm14 + mulps 20 * SIZE(AO), %xmm11 + addps %xmm14, %xmm0 + movaps 24 * SIZE(AO), %xmm14 + addps %xmm11, %xmm4 + movaps 28 * SIZE(BO), %xmm11 + + mulps %xmm11, %xmm14 + mulps 28 * SIZE(AO), %xmm11 + addps %xmm14, %xmm0 + movaps 80 * SIZE(AO), %xmm14 + addps %xmm11, %xmm4 + movaps 48 * SIZE(BO), %xmm11 + + addq $64 * SIZE, AO + addq $32 * SIZE, BO + decq %rax + jne .L112 + ALIGN_4 + +.L115: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + movaps ALPHA, %xmm15 + andq $7, %rax # if (k & 1) + BRANCH + je .L118 + ALIGN_4 + +.L116: + mulps %xmm9, %xmm8 + mulps -28 * SIZE(AO), %xmm9 + addps %xmm8, %xmm0 + movaps -24 * SIZE(AO), %xmm8 + addps %xmm9, %xmm4 + movaps 4 * SIZE(BO), %xmm9 + + addq $8 * SIZE, AO # aoffset += 4 + addq $4 * SIZE, BO # boffset1 += 8 + decq %rax + jg .L116 + ALIGN_4 + +.L118: +#ifndef TRMMKERNEL + movsd 0 * SIZE(CO1), %xmm8 + movhps 2 * SIZE(CO1), %xmm8 + movsd 4 * SIZE(CO1), %xmm9 + movhps 6 * SIZE(CO1), %xmm9 +#endif + + mulps %xmm15, %xmm0 + mulps %xmm15, %xmm4 +#ifndef TRMMKERNEL + addps %xmm8, %xmm0 + addps %xmm9, %xmm4 +#endif + + vmovups %xmm0, 0 * SIZE(CO1) + vmovups %xmm4, 4 * SIZE(CO1) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + leaq (,%rax, 8), %rax + leaq (AO, %rax, 4), AO + leaq (BO, %rax, 2), BO +#endif + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $8, KK +#endif + + addq $8 * SIZE, CO1 # coffset += 4 + decq I # i -- + jg .L111 + ALIGN_4 + +.L120: + testq $4, M + je .L130 + +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + + leaq BUFFER, BO +#else + leaq BUFFER, BO + movq KK, %rax + leaq (, %rax, 8), %rax + leaq (AO, %rax, 2), AO + leaq (BO, %rax, 2), BO +#endif + + movaps -32 * SIZE(AO), %xmm8 + movaps -16 * SIZE(AO), %xmm10 + + movaps 0 * SIZE(BO), %xmm9 + movaps 16 * SIZE(BO), %xmm11 + + xorps %xmm0, %xmm0 + xorps %xmm1, %xmm1 + xorps %xmm2, %xmm2 + xorps %xmm3, %xmm3 + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $4, %rax +#else + addq $1, %rax +#endif + movq %rax, KKK +#endif + sarq $3, %rax + je .L125 + ALIGN_4 + +.L122: + mulps %xmm8, %xmm9 + movaps -28 * SIZE(AO), %xmm8 + mulps 4 * SIZE(BO), %xmm8 + addps %xmm9, %xmm0 + movaps 32 * SIZE(BO), %xmm9 + addps %xmm8, %xmm1 + movaps -24 * SIZE(AO), %xmm8 + mulps 8 * SIZE(BO), %xmm8 + addps %xmm8, %xmm2 + movaps -20 * SIZE(AO), %xmm8 + mulps 12 * SIZE(BO), %xmm8 + addps %xmm8, %xmm3 + movaps 0 * SIZE(AO), %xmm8 + + mulps %xmm10, %xmm11 + movaps -12 * SIZE(AO), %xmm10 + mulps 20 * SIZE(BO), %xmm10 + addps %xmm11, %xmm0 + movaps 48 * SIZE(BO), %xmm11 + addps %xmm10, %xmm1 + movaps -8 * SIZE(AO), %xmm10 + mulps 24 * SIZE(BO), %xmm10 + addps %xmm10, %xmm2 + movaps -4 * SIZE(AO), %xmm10 + mulps 28 * SIZE(BO), %xmm10 + addps %xmm10, %xmm3 + movaps 16 * SIZE(AO), %xmm10 + + addq $32 * SIZE, AO + addq $32 * SIZE, BO + decq %rax + jne .L122 + ALIGN_4 + +.L125: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + movaps ALPHA, %xmm15 + andq $7, %rax # if (k & 1) + BRANCH + je .L128 + ALIGN_4 + +.L126: + mulps %xmm8, %xmm9 + movaps -28 * SIZE(AO), %xmm8 + addps %xmm9, %xmm0 + movaps 4 * SIZE(BO), %xmm9 + + addq $4 * SIZE, AO # aoffset += 4 + addq $4 * SIZE, BO # boffset1 += 8 + decq %rax + jg .L126 + ALIGN_4 + +.L128: +#ifndef TRMMKERNEL + movsd 0 * SIZE(CO1), %xmm8 + movhps 2 * SIZE(CO1), %xmm8 +#endif + + addps %xmm1, %xmm0 + addps %xmm3, %xmm2 + addps %xmm2, %xmm0 + + mulps %xmm15, %xmm0 +#ifndef TRMMKERNEL + addps %xmm8, %xmm0 +#endif + + vmovups %xmm0, 0 * SIZE(CO1) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + leaq (,%rax, 8), %rax + leaq (AO, %rax, 2), AO + leaq (BO, %rax, 2), BO +#endif + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $4, KK +#endif + + addq $4 * SIZE, CO1 # coffset += 4 + ALIGN_4 + +.L130: + testq $2, M + je .L140 + +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + + leaq BUFFER, BO +#else + leaq BUFFER, BO + movq KK, %rax + leaq (, %rax, 8), %rax + leaq (AO, %rax, 1), AO + leaq (BO, %rax, 2), BO +#endif + + movaps -32 * SIZE(AO), %xmm8 + movaps -24 * SIZE(AO), %xmm10 + + movaps 0 * SIZE(BO), %xmm9 + movaps 16 * SIZE(BO), %xmm11 + + xorps %xmm0, %xmm0 + xorps %xmm1, %xmm1 + xorps %xmm2, %xmm2 + xorps %xmm3, %xmm3 + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $2, %rax +#else + addq $1, %rax +#endif + movq %rax, KKK +#endif + sarq $3, %rax + je .L135 + ALIGN_4 + +.L132: + mulps %xmm8, %xmm9 + movsd -30 * SIZE(AO), %xmm8 + addps %xmm9, %xmm0 + movsd 4 * SIZE(BO), %xmm9 + mulps %xmm8, %xmm9 + movsd -28 * SIZE(AO), %xmm8 + addps %xmm9, %xmm1 + movsd 8 * SIZE(BO), %xmm9 + + mulps %xmm8, %xmm9 + movsd -26 * SIZE(AO), %xmm8 + addps %xmm9, %xmm0 + movsd 12 * SIZE(BO), %xmm9 + + mulps %xmm8, %xmm9 + movsd -16 * SIZE(AO), %xmm8 + addps %xmm9, %xmm1 + movsd 32 * SIZE(BO), %xmm9 + + mulps %xmm10, %xmm11 + movsd -22 * SIZE(AO), %xmm10 + addps %xmm11, %xmm0 + movsd 20 * SIZE(BO), %xmm11 + + mulps %xmm10, %xmm11 + movsd -20 * SIZE(AO), %xmm10 + addps %xmm11, %xmm1 + movsd 24 * SIZE(BO), %xmm11 + + mulps %xmm10, %xmm11 + movsd -18 * SIZE(AO), %xmm10 + addps %xmm11, %xmm0 + movsd 28 * SIZE(BO), %xmm11 + + mulps %xmm10, %xmm11 + movsd -8 * SIZE(AO), %xmm10 + addps %xmm11, %xmm1 + movsd 48 * SIZE(BO), %xmm11 + + addq $16 * SIZE, AO + addq $32 * SIZE, BO + decq %rax + jne .L132 + ALIGN_4 + +.L135: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + movaps ALPHA, %xmm15 + andq $7, %rax # if (k & 1) + BRANCH + je .L138 + ALIGN_4 + +.L136: + mulps %xmm8, %xmm9 + movsd -30 * SIZE(AO), %xmm8 + addps %xmm9, %xmm0 + movsd 4 * SIZE(BO), %xmm9 + + addq $2 * SIZE, AO # aoffset += 4 + addq $4 * SIZE, BO # boffset1 += 8 + decq %rax + jg .L136 + ALIGN_4 + +.L138: + addps %xmm1, %xmm0 + mulps %xmm15, %xmm0 + +#ifndef TRMMKERNEL + movsd 0 * SIZE(CO1), %xmm8 + addps %xmm8, %xmm0 +#endif + + movsd %xmm0, 0 * SIZE(CO1) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + leaq (,%rax, 8), %rax + leaq (AO, %rax, 1), AO + leaq (BO, %rax, 2), BO +#endif + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $2, KK +#endif + + addq $2 * SIZE, CO1 # coffset += 4 + ALIGN_4 + +.L140: + testq $1, M + je .L999 + +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + + leaq BUFFER, BO +#else + leaq BUFFER, BO + movq KK, %rax + leaq (, %rax, 4), %rax + leaq (AO, %rax, 1), AO + leaq (BO, %rax, 4), BO +#endif + + movss -32 * SIZE(AO), %xmm8 + movss -28 * SIZE(AO), %xmm10 + + movss 0 * SIZE(BO), %xmm9 + movss 16 * SIZE(BO), %xmm11 + + xorps %xmm0, %xmm0 + xorps %xmm1, %xmm1 + xorps %xmm2, %xmm2 + xorps %xmm3, %xmm3 + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $1, %rax +#else + addq $1, %rax +#endif + movq %rax, KKK +#endif + sarq $3, %rax + je .L145 + ALIGN_4 + +.L142: + mulss %xmm8, %xmm9 + movss -31 * SIZE(AO), %xmm8 + mulss 4 * SIZE(BO), %xmm8 + addss %xmm9, %xmm0 + movss 32 * SIZE(BO), %xmm9 + addss %xmm8, %xmm1 + movss -30 * SIZE(AO), %xmm8 + mulss 8 * SIZE(BO), %xmm8 + addss %xmm8, %xmm2 + movss -29 * SIZE(AO), %xmm8 + mulss 12 * SIZE(BO), %xmm8 + addss %xmm8, %xmm3 + movss -24 * SIZE(AO), %xmm8 + mulss %xmm10, %xmm11 + movss -27 * SIZE(AO), %xmm10 + mulss 20 * SIZE(BO), %xmm10 + addss %xmm11, %xmm0 + movss 48 * SIZE(BO), %xmm11 + addss %xmm10, %xmm1 + movss -26 * SIZE(AO), %xmm10 + mulss 24 * SIZE(BO), %xmm10 + addss %xmm10, %xmm2 + movss -25 * SIZE(AO), %xmm10 + mulss 28 * SIZE(BO), %xmm10 + addss %xmm10, %xmm3 + movss -20 * SIZE(AO), %xmm10 + + addq $ 8 * SIZE, AO + addq $32 * SIZE, BO + decq %rax + jne .L142 + ALIGN_4 + +.L145: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + movss ALPHA, %xmm15 + andq $7, %rax # if (k & 1) + BRANCH + je .L148 + ALIGN_4 + +.L146: + mulss %xmm8, %xmm9 + movss -31 * SIZE(AO), %xmm8 + addss %xmm9, %xmm0 + movss 4 * SIZE(BO), %xmm9 + + addq $1 * SIZE, AO + addq $4 * SIZE, BO + decq %rax + jg .L146 + ALIGN_4 + +.L148: + addss %xmm1, %xmm0 + addss %xmm3, %xmm2 + addss %xmm2, %xmm0 + + mulss %xmm15, %xmm0 + +#ifndef TRMMKERNEL + movss 0 * SIZE(CO1), %xmm8 + addss %xmm8, %xmm0 +#endif + movss %xmm0, 0 * SIZE(CO1) + ALIGN_4 + +.L999: + movq %rbx, %rsp + movq 0(%rsp), %rbx + movq 8(%rsp), %rbp + movq 16(%rsp), %r12 + movq 24(%rsp), %r13 + movq 32(%rsp), %r14 + movq 40(%rsp), %r15 + +#ifdef WINDOWS_ABI + movq 48(%rsp), %rdi + movq 56(%rsp), %rsi + movups 64(%rsp), %xmm6 + movups 80(%rsp), %xmm7 + movups 96(%rsp), %xmm8 + movups 112(%rsp), %xmm9 + movups 128(%rsp), %xmm10 + movups 144(%rsp), %xmm11 + movups 160(%rsp), %xmm12 + movups 176(%rsp), %xmm13 + movups 192(%rsp), %xmm14 + movups 208(%rsp), %xmm15 +#endif + + addq $STACKSIZE, %rsp + ret + + EPILOGUE diff --git a/kernel/x86_64/zgemm_kernel_2x2_bulldozer.S b/kernel/x86_64/zgemm_kernel_2x2_bulldozer.S index fa02bbe84..ea214aba0 100644 --- a/kernel/x86_64/zgemm_kernel_2x2_bulldozer.S +++ b/kernel/x86_64/zgemm_kernel_2x2_bulldozer.S @@ -949,6 +949,9 @@ .L2_60: +#if defined(TRMMKERNEL) && !defined(LEFT) + addq $2, KK +#endif decq J // j -- jg .L2_01 // next 2 lines of N