diff --git a/driver/level3/level3.c b/driver/level3/level3.c index 20e811cd0..27e503b0e 100644 --- a/driver/level3/level3.c +++ b/driver/level3/level3.c @@ -332,7 +332,20 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, #else for(jjs = js; jjs < js + min_j; jjs += min_jj){ min_jj = min_j + js - jjs; - if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N; + +#if defined(BULLDOZER) && defined(ARCH_X86_64) && defined(DOUBLE) && !defined(COMPLEX) + if (min_jj >= 12*GEMM_UNROLL_N) min_jj = 12*GEMM_UNROLL_N; + else + if (min_jj >= 6*GEMM_UNROLL_N) min_jj = 6*GEMM_UNROLL_N; + else + if (min_jj >= 3*GEMM_UNROLL_N) min_jj = 3*GEMM_UNROLL_N; + else + if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N; +#else + + if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N; +#endif + START_RPCC(); diff --git a/driver/level3/level3_thread.c b/driver/level3/level3_thread.c index 000d42397..87a32898c 100644 --- a/driver/level3/level3_thread.c +++ b/driver/level3/level3_thread.c @@ -360,8 +360,20 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, for(jjs = xxx; jjs < MIN(n_to, xxx + div_n); jjs += min_jj){ min_jj = MIN(n_to, xxx + div_n) - jjs; + +#if defined(BULLDOZER) && defined(ARCH_X86_64) && defined(DOUBLE) && !defined(COMPLEX) + if (min_jj >= 12*GEMM_UNROLL_N) min_jj = 12*GEMM_UNROLL_N; + else + if (min_jj >= 6*GEMM_UNROLL_N) min_jj = 6*GEMM_UNROLL_N; + else + if (min_jj >= 3*GEMM_UNROLL_N) min_jj = 3*GEMM_UNROLL_N; + else + if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N; +#else + if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N; - +#endif + START_RPCC(); OCOPY_OPERATION(min_l, min_jj, b, ldb, ls, jjs, @@ -634,7 +646,7 @@ static int gemm_driver(blas_arg_t *args, BLASLONG *range_m, BLASLONG num_cpu_n ++; } - + for (j = 0; j < num_cpu_m; j++) { for (i = 0; i < num_cpu_m; i++) { for (k = 0; k < DIVIDE_RATE; k++) { diff --git a/kernel/x86_64/KERNEL.BULLDOZER b/kernel/x86_64/KERNEL.BULLDOZER index 2ac035fe0..70ae51f6d 100644 --- a/kernel/x86_64/KERNEL.BULLDOZER +++ b/kernel/x86_64/KERNEL.BULLDOZER @@ -10,13 +10,13 @@ SGEMMINCOPYOBJ = sgemm_incopy$(TSUFFIX).$(SUFFIX) SGEMMITCOPYOBJ = sgemm_itcopy$(TSUFFIX).$(SUFFIX) SGEMMONCOPYOBJ = sgemm_oncopy$(TSUFFIX).$(SUFFIX) SGEMMOTCOPYOBJ = sgemm_otcopy$(TSUFFIX).$(SUFFIX) -DGEMMKERNEL = dgemm_kernel_4x4_bulldozer.S -DGEMMINCOPY = -DGEMMITCOPY = -DGEMMONCOPY = ../generic/gemm_ncopy_4.c -DGEMMOTCOPY = ../generic/gemm_tcopy_4.c -DGEMMINCOPYOBJ = -DGEMMITCOPYOBJ = +DGEMMKERNEL = dgemm_kernel_8x2_bulldozer.S +DGEMMINCOPY = ../generic/gemm_ncopy_8.c +DGEMMITCOPY = ../generic/gemm_tcopy_8.c +DGEMMONCOPY = ../generic/gemm_ncopy_2.c +DGEMMOTCOPY = ../generic/gemm_tcopy_2.c +DGEMMINCOPYOBJ = dgemm_incopy$(TSUFFIX).$(SUFFIX) +DGEMMITCOPYOBJ = dgemm_itcopy$(TSUFFIX).$(SUFFIX) DGEMMONCOPYOBJ = dgemm_oncopy$(TSUFFIX).$(SUFFIX) DGEMMOTCOPYOBJ = dgemm_otcopy$(TSUFFIX).$(SUFFIX) CGEMMKERNEL = zgemm_kernel_4x2_barcelona.S @@ -38,25 +38,27 @@ ZGEMMITCOPYOBJ = ZGEMMONCOPYOBJ = zgemm_oncopy$(TSUFFIX).$(SUFFIX) ZGEMMOTCOPYOBJ = zgemm_otcopy$(TSUFFIX).$(SUFFIX) -STRSMKERNEL_LN = trsm_kernel_LN_8x4_sse.S -STRSMKERNEL_LT = trsm_kernel_LT_8x4_sse.S -STRSMKERNEL_RN = trsm_kernel_LT_8x4_sse.S -STRSMKERNEL_RT = trsm_kernel_RT_8x4_sse.S - -DTRSMKERNEL_LN = trsm_kernel_LN_4x4_barcelona.S -DTRSMKERNEL_LT = trsm_kernel_LT_4x4_barcelona.S -DTRSMKERNEL_RN = trsm_kernel_LT_4x4_barcelona.S -DTRSMKERNEL_RT = trsm_kernel_RT_4x4_barcelona.S - -CTRSMKERNEL_LN = ztrsm_kernel_LN_4x2_sse.S -CTRSMKERNEL_LT = ztrsm_kernel_LT_4x2_sse.S -CTRSMKERNEL_RN = ztrsm_kernel_LT_4x2_sse.S -CTRSMKERNEL_RT = ztrsm_kernel_RT_4x2_sse.S - -ZTRSMKERNEL_LN = ztrsm_kernel_LN_2x2_sse2.S -ZTRSMKERNEL_LT = ztrsm_kernel_LT_2x2_sse2.S -ZTRSMKERNEL_RN = ztrsm_kernel_LT_2x2_sse2.S -ZTRSMKERNEL_RT = ztrsm_kernel_RT_2x2_sse2.S - CGEMM3MKERNEL = zgemm3m_kernel_8x4_barcelona.S ZGEMM3MKERNEL = zgemm3m_kernel_4x4_barcelona.S + +STRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +STRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +STRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +STRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +DTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +DTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +DTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +DTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +CTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +CTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +CTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +CTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +ZTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +ZTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +ZTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +ZTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + + diff --git a/kernel/x86_64/dgemm_kernel_4x4_bulldozer.S b/kernel/x86_64/dgemm_kernel_4x4_bulldozer.S deleted file mode 100644 index 91cd49291..000000000 --- a/kernel/x86_64/dgemm_kernel_4x4_bulldozer.S +++ /dev/null @@ -1,1959 +0,0 @@ -/*********************************************************************/ -/* Copyright 2009, 2010 The University of Texas at Austin. */ -/* All rights reserved. */ -/* */ -/* Redistribution and use in source and binary forms, with or */ -/* without modification, are permitted provided that the following */ -/* conditions are met: */ -/* */ -/* 1. Redistributions of source code must retain the above */ -/* copyright notice, this list of conditions and the following */ -/* disclaimer. */ -/* */ -/* 2. Redistributions in binary form must reproduce the above */ -/* copyright notice, this list of conditions and the following */ -/* disclaimer in the documentation and/or other materials */ -/* provided with the distribution. */ -/* */ -/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ -/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ -/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ -/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ -/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ -/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ -/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ -/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ -/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ -/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ -/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ -/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ -/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ -/* POSSIBILITY OF SUCH DAMAGE. */ -/* */ -/* The views and conclusions contained in the software and */ -/* documentation are those of the authors and should not be */ -/* interpreted as representing official policies, either expressed */ -/* or implied, of The University of Texas at Austin. */ -/*********************************************************************/ - -/********************************************************************* -* Changelog: -* -* 2013/04/15 Saar -* Prefetch for A and B -* unroll of inner Loop -* using generic versions for ncopy and tcopy -* moved vmovddup ALPHA, %xmm7 down -* define A_PR1 192 -* define B_PR1 512 -* -* 2013/04/27 Saar -* define A_PR1 224 -* define B_PR1 224 -* created 2 different Kernels -**********************************************************************/ - -/********************************************************************* -* 2013/04/12 Saar -* Performance: -* 3584x3584 89 GFLOPS with 8 threads on 4 modules -* 76 GFLOPS with 4 threads on 4 modules -* 53 GFLOPS with 4 threads on 2 modules -* 46 GFLOPS with 2 threads on 2 modules -* 28 GFLOPS with 2 threads on 1 module -* 23,1 GFLOPS with 1 thread on 1 module -*********************************************************************/ - -#define ASSEMBLER -#include "common.h" - -#define OLD_M %rdi -#define OLD_N %rsi -#define M %r13 -#define N %r14 -#define K %rdx - -#define A %rcx -#define B %r8 -#define C %r9 -#define LDC %r10 - -#define I %r11 -#define AO %rdi -#define BO %rsi -#define CO1 %r15 -#define CO2 %r12 -#define BB %rbp -#define J %rbx - -#ifndef WINDOWS_ABI - -#define STACKSIZE 96 - -#define ALPHA 48(%rsp) -#define OFFSET 56(%rsp) -#define KK 64(%rsp) -#define KKK 72(%rsp) - -#else - -#define STACKSIZE 256 - -#define OLD_A 40 + STACKSIZE(%rsp) -#define OLD_B 48 + STACKSIZE(%rsp) -#define OLD_C 56 + STACKSIZE(%rsp) -#define OLD_LDC 64 + STACKSIZE(%rsp) -#define OLD_OFFSET 72 + STACKSIZE(%rsp) - -#define ALPHA 224(%rsp) -#define OFFSET 232(%rsp) -#define KK 240(%rsp) -#define KKK 248(%rsp) - -#endif - -#define movapd movaps -#define movupd movups - -#define A_PR1 224 -#define B_PR1 224 - - -#if defined(OPTBYMODULE) || !defined(SMP) - -#define KERNEL1(xx) \ - vmovups -14 * SIZE(AO, %rax, 4),%xmm2 ;\ - vfmaddpd %xmm8,%xmm6,%xmm7,%xmm8 ;\ - vmovddup -14 * SIZE(BO, %rax, 4), %xmm5 ;\ - vfmaddpd %xmm12,%xmm2,%xmm7,%xmm12 ;\ - vmovddup -13 * SIZE(BO, %rax, 4), %xmm4 ;\ - vfmaddpd %xmm9,%xmm6,%xmm3,%xmm9 ;\ - vmovddup -12 * SIZE(BO, %rax, 4), %xmm1 ;\ - vfmaddpd %xmm13,%xmm2,%xmm3,%xmm13 ;\ - vfmaddpd %xmm14,%xmm2, %xmm5,%xmm14 ;\ - vmovddup -11 * SIZE(BO, %rax, 4), %xmm3 ;\ - vfmaddpd %xmm15,%xmm2, %xmm4,%xmm15 ;\ - vmovups -12 * SIZE(AO, %rax, 4),%xmm0 ;\ - vfmaddpd %xmm10,%xmm6,%xmm5,%xmm10 ;\ - vfmaddpd %xmm11,%xmm6,%xmm4,%xmm11 ;\ - -#define KERNEL2(xx) \ - vmovups -10 * SIZE(AO, %rax, 4), %xmm2 ;\ - vfmaddpd %xmm8, %xmm0, %xmm1,%xmm8 ;\ - vmovups -8 * SIZE(AO, %rax, 4),%xmm6 ;\ - vfmaddpd %xmm12,%xmm2, %xmm1,%xmm12 ;\ - vmovddup -10 * SIZE(BO, %rax, 4), %xmm5 ;\ - vfmaddpd %xmm9, %xmm0, %xmm3,%xmm9 ;\ - vfmaddpd %xmm13,%xmm2, %xmm3,%xmm13 ;\ - vmovddup -9 * SIZE(BO, %rax, 4), %xmm4 ;\ - vfmaddpd %xmm10,%xmm0, %xmm5,%xmm10 ;\ - vmovddup -8 * SIZE(BO, %rax, 4), %xmm7 ;\ - vfmaddpd %xmm14,%xmm2, %xmm5,%xmm14 ;\ - vmovddup -7 * SIZE(BO, %rax, 4), %xmm3 ;\ - vfmaddpd %xmm11,%xmm0, %xmm4,%xmm11 ;\ - vfmaddpd %xmm15,%xmm2, %xmm4,%xmm15 ;\ - -#define KERNEL3(xx) \ - vmovups -6 * SIZE(AO, %rax, 4),%xmm2 ;\ - vfmaddpd %xmm8, %xmm6, %xmm7, %xmm8 ;\ - vmovddup -6 * SIZE(BO, %rax, 4), %xmm5 ;\ - vfmaddpd %xmm12,%xmm2, %xmm7,%xmm12 ;\ - vmovddup -5 * SIZE(BO, %rax, 4), %xmm4 ;\ - vfmaddpd %xmm9, %xmm6, %xmm3,%xmm9 ;\ - vfmaddpd %xmm13,%xmm2, %xmm3,%xmm13 ;\ - vmovddup -4 * SIZE(BO, %rax, 4), %xmm1 ;\ - vfmaddpd %xmm14,%xmm2, %xmm5,%xmm14 ;\ - vmovddup -3 * SIZE(BO, %rax, 4), %xmm3 ;\ - vfmaddpd %xmm15,%xmm2, %xmm4,%xmm15 ;\ - vmovups -4 * SIZE(AO, %rax, 4),%xmm0 ;\ - vfmaddpd %xmm10,%xmm6, %xmm5,%xmm10 ;\ - vfmaddpd %xmm11,%xmm6, %xmm4, %xmm11 ;\ - -#define KERNEL4(xx) \ - vmovups -2 * SIZE(AO, %rax, 4), %xmm2 ;\ - vfmaddpd %xmm8,%xmm0, %xmm1,%xmm8 ;\ - vmovups (AO, %rax, 4), %xmm6 ;\ - vfmaddpd %xmm12,%xmm2, %xmm1 ,%xmm12;\ - vmovddup -2 * SIZE(BO, %rax, 4), %xmm5 ;\ - vfmaddpd %xmm9,%xmm0, %xmm3,%xmm9 ;\ - vfmaddpd %xmm13,%xmm2, %xmm3,%xmm13 ;\ - vmovddup -1 * SIZE(BO, %rax, 4), %xmm4 ;\ - vfmaddpd %xmm10,%xmm0, %xmm5,%xmm10 ;\ - vmovddup (BO, %rax, 4), %xmm7 ;\ - vfmaddpd %xmm14,%xmm2, %xmm5,%xmm14 ;\ - vmovddup 1 * SIZE(BO, %rax, 4), %xmm3 ;\ - vfmaddpd %xmm11,%xmm0, %xmm4,%xmm11 ;\ - vfmaddpd %xmm15,%xmm2, %xmm4,%xmm15 ;\ - -#define KERNEL5(xx) \ - vmovups 2 * SIZE(AO, %rax, 4),%xmm2 ;\ - vfmaddpd %xmm8,%xmm6, %xmm7,%xmm8 ;\ - vmovddup 2 * SIZE(BO, %rax, 4), %xmm5 ;\ - vfmaddpd %xmm12,%xmm2, %xmm7,%xmm12 ;\ - vmovddup 3 * SIZE(BO, %rax, 4), %xmm4 ;\ - vfmaddpd %xmm9,%xmm6, %xmm3,%xmm9 ;\ - vfmaddpd %xmm13,%xmm2, %xmm3,%xmm13 ;\ - vmovddup 4 * SIZE(BO, %rax, 4), %xmm1 ;\ - vfmaddpd %xmm14,%xmm2, %xmm5,%xmm14 ;\ - vmovddup 5 * SIZE(BO, %rax, 4), %xmm3 ;\ - vfmaddpd %xmm15,%xmm2, %xmm4,%xmm15 ;\ - vmovups 4 * SIZE(AO, %rax, 4),%xmm0 ;\ - vfmaddpd %xmm10,%xmm6, %xmm5,%xmm10 ;\ - vfmaddpd %xmm11,%xmm6, %xmm4,%xmm11 ;\ - -#define KERNEL6(xx) \ - vmovups 6 * SIZE(AO, %rax, 4), %xmm2 ;\ - vfmaddpd %xmm8,%xmm0, %xmm1,%xmm8 ;\ - vmovups 8 * SIZE(AO, %rax, 4), %xmm6 ;\ - vfmaddpd %xmm12,%xmm2, %xmm1,%xmm12 ;\ - vmovddup 6 * SIZE(BO, %rax, 4), %xmm5 ;\ - vfmaddpd %xmm9,%xmm0, %xmm3,%xmm9 ;\ - vfmaddpd %xmm13,%xmm2, %xmm3,%xmm13 ;\ - vmovddup 7 * SIZE(BO, %rax, 4), %xmm4 ;\ - vfmaddpd %xmm10,%xmm0, %xmm5,%xmm10 ;\ - vmovddup 8 * SIZE(BO, %rax, 4), %xmm7 ;\ - vfmaddpd %xmm14,%xmm2, %xmm5,%xmm14 ;\ - vmovddup 9 * SIZE(BO, %rax, 4), %xmm3 ;\ - vfmaddpd %xmm11,%xmm0, %xmm4,%xmm11 ;\ - vfmaddpd %xmm15,%xmm2, %xmm4,%xmm15 ;\ - -#define KERNEL7(xx) \ - vmovups 10 * SIZE(AO, %rax, 4),%xmm2 ;\ - vfmaddpd %xmm8,%xmm6, %xmm7,%xmm8 ;\ - vmovddup 10 * SIZE(BO, %rax, 4), %xmm5 ;\ - vfmaddpd %xmm12,%xmm2, %xmm7,%xmm12 ;\ - vmovddup 11 * SIZE(BO, %rax, 4), %xmm4 ;\ - vfmaddpd %xmm9,%xmm6, %xmm3,%xmm9 ;\ - vfmaddpd %xmm13,%xmm2, %xmm3,%xmm13 ;\ - vmovddup 12 * SIZE(BO, %rax, 4), %xmm1 ;\ - vfmaddpd %xmm14,%xmm2, %xmm5,%xmm14 ;\ - vmovddup 13 * SIZE(BO, %rax, 4), %xmm3 ;\ - vfmaddpd %xmm15,%xmm2, %xmm4,%xmm15 ;\ - vmovups 12 * SIZE(AO, %rax, 4), %xmm0 ;\ - vfmaddpd %xmm10,%xmm6, %xmm5,%xmm10 ;\ - vfmaddpd %xmm11,%xmm6, %xmm4,%xmm11 ;\ - -#define KERNEL8(xx) \ - vmovups 14 * SIZE(AO, %rax, 4), %xmm2 ;\ - vfmaddpd %xmm8,%xmm0, %xmm1,%xmm8 ;\ - vmovups 16 * SIZE(AO, %rax, 4),%xmm6 ;\ - vfmaddpd %xmm12,%xmm2, %xmm1,%xmm12 ;\ - vmovddup 14 * SIZE(BO, %rax, 4), %xmm5 ;\ - vfmaddpd %xmm9,%xmm0, %xmm3,%xmm9 ;\ - vfmaddpd %xmm13, %xmm2, %xmm3,%xmm13 ;\ - vmovddup 15 * SIZE(BO, %rax, 4), %xmm4 ;\ - vfmaddpd %xmm10,%xmm0, %xmm5,%xmm10 ;\ - vmovddup 16 * SIZE(BO, %rax, 4), %xmm7 ;\ - vfmaddpd %xmm14,%xmm2, %xmm5,%xmm14 ;\ - vmovddup 17 * SIZE(BO, %rax, 4), %xmm3 ;\ - vfmaddpd %xmm11,%xmm0, %xmm4,%xmm11 ;\ - vfmaddpd %xmm15,%xmm2, %xmm4,%xmm15 ;\ - -#else - -#define KERNEL1(xx) \ - vmovups -14 * SIZE(AO, %rax, 4),%xmm2 ;\ - vfmaddpd %xmm8,%xmm6,%xmm7,%xmm8 ;\ - vmovddup -14 * SIZE(BO, %rax, 4), %xmm5 ;\ - vfmaddpd %xmm12,%xmm2,%xmm7,%xmm12 ;\ - vmovddup -13 * SIZE(BO, %rax, 4), %xmm4 ;\ - vfmaddpd %xmm9,%xmm6,%xmm3,%xmm9 ;\ - vmovddup -12 * SIZE(BO, %rax, 4), %xmm1 ;\ - vfmaddpd %xmm13,%xmm2,%xmm3,%xmm13 ;\ - vmovddup -8 * SIZE(BO, %rax, 4), %xmm7 ;\ - vfmaddpd %xmm10,%xmm6,%xmm5,%xmm10 ;\ - vmovddup -11 * SIZE(BO, %rax, 4), %xmm3 ;\ - vfmaddpd %xmm11,%xmm6,%xmm4,%xmm11 ;\ - vmovups -12 * SIZE(AO, %rax, 4),%xmm0 ;\ - vfmaddpd %xmm15,%xmm2, %xmm4,%xmm15 ;\ - vmovups -8 * SIZE(AO, %rax, 4),%xmm6 ;\ - vfmaddpd %xmm14,%xmm2, %xmm5,%xmm14 ;\ - -#define KERNEL2(xx) \ - vmovups -10 * SIZE(AO, %rax, 4), %xmm2 ;\ - vfmaddpd %xmm8, %xmm0, %xmm1,%xmm8 ;\ - vfmaddpd %xmm12,%xmm2, %xmm1,%xmm12 ;\ - vmovddup -10 * SIZE(BO, %rax, 4), %xmm5 ;\ - vfmaddpd %xmm9, %xmm0, %xmm3,%xmm9 ;\ - vfmaddpd %xmm13,%xmm2, %xmm3,%xmm13 ;\ - vmovddup -9 * SIZE(BO, %rax, 4), %xmm4 ;\ - vfmaddpd %xmm10,%xmm0, %xmm5,%xmm10 ;\ - vfmaddpd %xmm14,%xmm2, %xmm5,%xmm14 ;\ - vmovddup -7 * SIZE(BO, %rax, 4), %xmm3 ;\ - vfmaddpd %xmm11,%xmm0, %xmm4,%xmm11 ;\ - vfmaddpd %xmm15,%xmm2, %xmm4,%xmm15 ;\ - -#define KERNEL3(xx) \ - vmovups -6 * SIZE(AO, %rax, 4),%xmm2 ;\ - vfmaddpd %xmm8, %xmm6, %xmm7, %xmm8 ;\ - vmovddup -6 * SIZE(BO, %rax, 4), %xmm5 ;\ - vfmaddpd %xmm12,%xmm2, %xmm7,%xmm12 ;\ - vmovddup -5 * SIZE(BO, %rax, 4), %xmm4 ;\ - vfmaddpd %xmm9, %xmm6, %xmm3,%xmm9 ;\ - vmovddup (BO, %rax, 4), %xmm7 ;\ - vfmaddpd %xmm13,%xmm2, %xmm3,%xmm13 ;\ - vmovddup -4 * SIZE(BO, %rax, 4), %xmm1 ;\ - vfmaddpd %xmm10,%xmm6, %xmm5,%xmm10 ;\ - vmovddup -3 * SIZE(BO, %rax, 4), %xmm3 ;\ - vfmaddpd %xmm11,%xmm6, %xmm4, %xmm11 ;\ - vmovups -4 * SIZE(AO, %rax, 4),%xmm0 ;\ - vfmaddpd %xmm14,%xmm2, %xmm5,%xmm14 ;\ - vmovups (AO, %rax, 4), %xmm6 ;\ - vfmaddpd %xmm15,%xmm2, %xmm4,%xmm15 ;\ - -#define KERNEL4(xx) \ - vmovups -2 * SIZE(AO, %rax, 4), %xmm2 ;\ - vfmaddpd %xmm8,%xmm0, %xmm1,%xmm8 ;\ - vfmaddpd %xmm12,%xmm2, %xmm1 ,%xmm12;\ - vmovddup -2 * SIZE(BO, %rax, 4), %xmm5 ;\ - vfmaddpd %xmm9,%xmm0, %xmm3,%xmm9 ;\ - vfmaddpd %xmm13,%xmm2, %xmm3,%xmm13 ;\ - vmovddup -1 * SIZE(BO, %rax, 4), %xmm4 ;\ - vfmaddpd %xmm10,%xmm0, %xmm5,%xmm10 ;\ - vfmaddpd %xmm14,%xmm2, %xmm5,%xmm14 ;\ - vmovddup 1 * SIZE(BO, %rax, 4), %xmm3 ;\ - vfmaddpd %xmm11,%xmm0, %xmm4,%xmm11 ;\ - vfmaddpd %xmm15,%xmm2, %xmm4,%xmm15 ;\ - -#define KERNEL5(xx) \ - vmovups 2 * SIZE(AO, %rax, 4),%xmm2 ;\ - vfmaddpd %xmm8,%xmm6, %xmm7,%xmm8 ;\ - vmovddup 2 * SIZE(BO, %rax, 4), %xmm5 ;\ - vfmaddpd %xmm12,%xmm2, %xmm7,%xmm12 ;\ - vmovddup 3 * SIZE(BO, %rax, 4), %xmm4 ;\ - vfmaddpd %xmm9,%xmm6, %xmm3,%xmm9 ;\ - vmovddup 8 * SIZE(BO, %rax, 4), %xmm7 ;\ - vfmaddpd %xmm13,%xmm2, %xmm3,%xmm13 ;\ - vmovddup 4 * SIZE(BO, %rax, 4), %xmm1 ;\ - vfmaddpd %xmm10,%xmm6, %xmm5,%xmm10 ;\ - vmovddup 5 * SIZE(BO, %rax, 4), %xmm3 ;\ - vfmaddpd %xmm11,%xmm6, %xmm4,%xmm11 ;\ - vmovups 4 * SIZE(AO, %rax, 4),%xmm0 ;\ - vfmaddpd %xmm14,%xmm2, %xmm5,%xmm14 ;\ - vmovups 8 * SIZE(AO, %rax, 4), %xmm6 ;\ - vfmaddpd %xmm15,%xmm2, %xmm4,%xmm15 ;\ - -#define KERNEL6(xx) \ - vmovups 6 * SIZE(AO, %rax, 4), %xmm2 ;\ - vfmaddpd %xmm8,%xmm0, %xmm1,%xmm8 ;\ - vfmaddpd %xmm12,%xmm2, %xmm1,%xmm12 ;\ - vmovddup 6 * SIZE(BO, %rax, 4), %xmm5 ;\ - vfmaddpd %xmm9,%xmm0, %xmm3,%xmm9 ;\ - vfmaddpd %xmm13,%xmm2, %xmm3,%xmm13 ;\ - vmovddup 7 * SIZE(BO, %rax, 4), %xmm4 ;\ - vfmaddpd %xmm10,%xmm0, %xmm5,%xmm10 ;\ - vfmaddpd %xmm14,%xmm2, %xmm5,%xmm14 ;\ - vmovddup 9 * SIZE(BO, %rax, 4), %xmm3 ;\ - vfmaddpd %xmm11,%xmm0, %xmm4,%xmm11 ;\ - vfmaddpd %xmm15,%xmm2, %xmm4,%xmm15 ;\ - -#define KERNEL7(xx) \ - vmovups 10 * SIZE(AO, %rax, 4),%xmm2 ;\ - vfmaddpd %xmm8,%xmm6, %xmm7,%xmm8 ;\ - vmovddup 10 * SIZE(BO, %rax, 4), %xmm5 ;\ - vfmaddpd %xmm12,%xmm2, %xmm7,%xmm12 ;\ - vmovddup 11 * SIZE(BO, %rax, 4), %xmm4 ;\ - vfmaddpd %xmm9,%xmm6, %xmm3,%xmm9 ;\ - vmovddup 16 * SIZE(BO, %rax, 4), %xmm7 ;\ - vfmaddpd %xmm13,%xmm2, %xmm3,%xmm13 ;\ - vmovddup 12 * SIZE(BO, %rax, 4), %xmm1 ;\ - vfmaddpd %xmm10,%xmm6, %xmm5,%xmm10 ;\ - vmovddup 13 * SIZE(BO, %rax, 4), %xmm3 ;\ - vfmaddpd %xmm11,%xmm6, %xmm4,%xmm11 ;\ - vmovups 12 * SIZE(AO, %rax, 4), %xmm0 ;\ - vfmaddpd %xmm14,%xmm2, %xmm5,%xmm14 ;\ - vmovups 16 * SIZE(AO, %rax, 4),%xmm6 ;\ - vfmaddpd %xmm15,%xmm2, %xmm4,%xmm15 ;\ - -#define KERNEL8(xx) \ - vmovups 14 * SIZE(AO, %rax, 4), %xmm2 ;\ - vfmaddpd %xmm8,%xmm0, %xmm1,%xmm8 ;\ - vfmaddpd %xmm12,%xmm2, %xmm1,%xmm12 ;\ - vmovddup 14 * SIZE(BO, %rax, 4), %xmm5 ;\ - vfmaddpd %xmm9,%xmm0, %xmm3,%xmm9 ;\ - vfmaddpd %xmm13, %xmm2, %xmm3,%xmm13 ;\ - vmovddup 15 * SIZE(BO, %rax, 4), %xmm4 ;\ - vfmaddpd %xmm10,%xmm0, %xmm5,%xmm10 ;\ - vfmaddpd %xmm14,%xmm2, %xmm5,%xmm14 ;\ - vmovddup 17 * SIZE(BO, %rax, 4), %xmm3 ;\ - vfmaddpd %xmm11,%xmm0, %xmm4,%xmm11 ;\ - vfmaddpd %xmm15,%xmm2, %xmm4,%xmm15 ;\ - -#endif - -#define KERNEL_SUB1(xx) \ - vmovups -16 * SIZE(AO),%xmm0 ;\ - vmovups -14 * SIZE(AO),%xmm2 ;\ - vmovddup -16 * SIZE(BO), %xmm1 ;\ - vmovddup -15 * SIZE(BO), %xmm3 ;\ - vfmaddpd %xmm8, %xmm0, %xmm1,%xmm8 ;\ - vfmaddpd %xmm9, %xmm0, %xmm3,%xmm9 ;\ - vfmaddpd %xmm12, %xmm2, %xmm1,%xmm12 ;\ - vfmaddpd %xmm13, %xmm2, %xmm3,%xmm13 ;\ - vmovddup -14 * SIZE(BO), %xmm1 ;\ - vmovddup -13 * SIZE(BO), %xmm3 ;\ - vfmaddpd %xmm10, %xmm0, %xmm1,%xmm10 ;\ - vfmaddpd %xmm11, %xmm0, %xmm3,%xmm11 ;\ - vfmaddpd %xmm14, %xmm2, %xmm1,%xmm14 ;\ - vfmaddpd %xmm15, %xmm2, %xmm3,%xmm15 ;\ - - -#define KERNEL_SUB2(xx) \ - vmovups -12 * SIZE(AO), %xmm0 ;\ - vmovups -10 * SIZE(AO), %xmm2 ;\ - vmovddup -12 * SIZE(BO), %xmm1 ;\ - vmovddup -11 * SIZE(BO), %xmm3 ;\ - vfmaddpd %xmm8, %xmm0, %xmm1,%xmm8 ;\ - vfmaddpd %xmm9, %xmm0, %xmm3,%xmm9 ;\ - vfmaddpd %xmm12,%xmm2, %xmm1,%xmm12 ;\ - vfmaddpd %xmm13,%xmm2, %xmm3,%xmm13 ;\ - vmovddup -10 * SIZE(BO), %xmm1 ;\ - vmovddup -9 * SIZE(BO), %xmm3 ;\ - vfmaddpd %xmm10,%xmm0, %xmm1,%xmm10 ;\ - vfmaddpd %xmm11,%xmm0, %xmm3,%xmm11 ;\ - vfmaddpd %xmm14,%xmm2, %xmm1,%xmm14 ;\ - vfmaddpd %xmm15,%xmm2, %xmm3,%xmm15 ;\ - -#define KERNEL_SUB3(xx) \ - vmovups -8 * SIZE(AO),%xmm0 ;\ - vmovups -6 * SIZE(AO),%xmm2 ;\ - vmovddup -8 * SIZE(BO), %xmm1 ;\ - vmovddup -7 * SIZE(BO), %xmm3 ;\ - vfmaddpd %xmm8, %xmm0, %xmm1,%xmm8 ;\ - vfmaddpd %xmm9, %xmm0, %xmm3,%xmm9 ;\ - vfmaddpd %xmm12,%xmm2, %xmm1,%xmm12 ;\ - vfmaddpd %xmm13,%xmm2, %xmm3,%xmm13 ;\ - vmovddup -6 * SIZE(BO), %xmm1 ;\ - vmovddup -5 * SIZE(BO), %xmm3 ;\ - vfmaddpd %xmm10,%xmm0, %xmm1,%xmm10 ;\ - vfmaddpd %xmm11,%xmm0, %xmm3,%xmm11 ;\ - vfmaddpd %xmm14,%xmm2, %xmm1,%xmm14 ;\ - vfmaddpd %xmm15,%xmm2, %xmm3,%xmm15 ;\ - -#define KERNEL_SUB4(xx) \ - vmovups -4 * SIZE(AO), %xmm0 ;\ - vmovups -2 * SIZE(AO), %xmm2 ;\ - vmovddup -4 * SIZE(BO), %xmm1 ;\ - vmovddup -3 * SIZE(BO), %xmm3 ;\ - vfmaddpd %xmm8, %xmm0, %xmm1,%xmm8 ;\ - vfmaddpd %xmm9, %xmm0, %xmm3,%xmm9 ;\ - vfmaddpd %xmm12,%xmm2, %xmm1,%xmm12 ;\ - vfmaddpd %xmm13,%xmm2, %xmm3,%xmm13 ;\ - vmovddup -2 * SIZE(BO), %xmm1 ;\ - vmovddup -1 * SIZE(BO), %xmm3 ;\ - vfmaddpd %xmm10,%xmm0, %xmm1,%xmm10 ;\ - vfmaddpd %xmm11,%xmm0, %xmm3,%xmm11 ;\ - vfmaddpd %xmm14,%xmm2, %xmm1,%xmm14 ;\ - vfmaddpd %xmm15,%xmm2, %xmm3,%xmm15 ;\ - vmovups (AO), %xmm0 ;\ - vmovddup (BO), %xmm1 ;\ - vmovddup 1 * SIZE(BO), %xmm3 ;\ - vmovaps %xmm0, %xmm2 - - PROLOGUE - PROFCODE - - subq $STACKSIZE, %rsp - movq %rbx, (%rsp) - movq %rbp, 8(%rsp) - movq %r12, 16(%rsp) - movq %r13, 24(%rsp) - movq %r14, 32(%rsp) - movq %r15, 40(%rsp) - - vzeroupper - -#ifdef WINDOWS_ABI - movq %rdi, 48(%rsp) - movq %rsi, 56(%rsp) - movups %xmm6, 64(%rsp) - movups %xmm7, 80(%rsp) - movups %xmm8, 96(%rsp) - movups %xmm9, 112(%rsp) - movups %xmm10, 128(%rsp) - movups %xmm11, 144(%rsp) - movups %xmm12, 160(%rsp) - movups %xmm13, 176(%rsp) - movups %xmm14, 192(%rsp) - movups %xmm15, 208(%rsp) - - movq ARG1, OLD_M - movq ARG2, OLD_N - movq ARG3, K - movq OLD_A, A - movq OLD_B, B - movq OLD_C, C - movq OLD_LDC, LDC -#ifdef TRMMKERNEL - movsd OLD_OFFSET, %xmm12 -#endif - vmovaps %xmm3, %xmm0 - -#else - movq STACKSIZE + 8(%rsp), LDC -#ifdef TRMMKERNEL - movsd STACKSIZE + 16(%rsp), %xmm12 -#endif - -#endif - - movq OLD_M, M - movq OLD_N, N - - subq $-16 * SIZE, A - subq $-16 * SIZE, B - - vmovsd %xmm0, ALPHA - - salq $BASE_SHIFT, LDC # LDC << 3 # LDC * 8 - -#ifdef TRMMKERNEL - vmovsd %xmm12, OFFSET - vmovsd %xmm12, KK -#ifndef LEFT - negq KK -#endif -#endif - movq N, J - sarq $2, J # j = (n >> 2) # j = n / 4 - jle .L40 - ALIGN_4 - -.L01: - movq C, CO1 # coffset1 = c - leaq (C, LDC, 2), CO2 # coffset2 = c + ldc - - leaq (C, LDC, 4), C # c += 4 * ldc - -#if defined(TRMMKERNEL) && defined(LEFT) - movq OFFSET, %rax - movq %rax, KK -#endif - - movq A, AO # aoffset = a - - movq K, %rax - salq $BASE_SHIFT + 2, %rax # k << 5 # K * 32 - leaq (B, %rax), BB - - movq M, I - sarq $2, I # i = (m >> 2) - jle .L20 - ALIGN_4 - - .align 16 -.L11: -#if !defined(TRMMKERNEL) || \ - (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq B, BO -#else - movq KK, %rax - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 4), AO - leaq (B, %rax, 4), BO -#endif - - - - vxorpd %xmm8, %xmm8,%xmm8 - vxorpd %xmm9, %xmm9,%xmm9 - vxorpd %xmm10, %xmm10,%xmm10 - vxorpd %xmm11, %xmm11,%xmm11 - vxorpd %xmm12, %xmm12,%xmm12 - vxorpd %xmm13, %xmm13,%xmm13 - vxorpd %xmm14, %xmm14,%xmm14 - vxorpd %xmm15, %xmm15,%xmm15 - - prefetchw (CO1) - // prefetchw (CO1,LDC) - // prefetchw (CO2) - // prefetchw (CO2,LDC) - -#ifndef TRMMKERNEL - movq K, %rax -#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) - movq K, %rax - subq KK, %rax - movq %rax, KKK -#else - movq KK, %rax -#ifdef LEFT - addq $4, %rax -#else - addq $4, %rax -#endif - movq %rax, KKK -#endif - - andq $-8, %rax - salq $BASE_SHIFT, %rax - leaq (AO, %rax, 4), AO - leaq (BO, %rax, 4), BO - negq %rax - je .L15 - // ALIGN_4 - - vmovups -16 * SIZE(AO, %rax, 4),%xmm6 - vmovddup -16 * SIZE(BO, %rax, 4), %xmm7 - vmovddup -15 * SIZE(BO, %rax, 4), %xmm3 - - .align 32 - -.L12: - -#if defined(OPTBYMODULE) || !defined(SMP) - - prefetcht0 A_PR1(AO,%rax,4) - prefetcht0 B_PR1(BO,%rax,4) - KERNEL1(16 * 0) - KERNEL2(16 * 0) - prefetcht0 A_PR1+64(AO,%rax,4) - prefetcht0 B_PR1+64(BO,%rax,4) - KERNEL3(16 * 0) - KERNEL4(16 * 0) - prefetcht0 A_PR1+128(AO,%rax,4) - prefetcht0 B_PR1+128(BO,%rax,4) - KERNEL5(16 * 0) - KERNEL6(16 * 0) - prefetcht0 A_PR1+192(AO,%rax,4) - prefetcht0 B_PR1+192(BO,%rax,4) - KERNEL7(16 * 0) - KERNEL8(16 * 0) - - addq $8 * SIZE, %rax - je .L15 - jmp .L12 - .align 16 - -#else - KERNEL1(16 * 0) - KERNEL2(16 * 0) - KERNEL3(16 * 0) - KERNEL4(16 * 0) - KERNEL5(16 * 0) - KERNEL6(16 * 0) - KERNEL7(16 * 0) - KERNEL8(16 * 0) - - addq $8 * SIZE, %rax - je .L15 - jmp .L12 - .align 16 - -#endif - - -.L15: - -#ifndef TRMMKERNEL - movq K, %rax -#else - movq KKK, %rax -#endif - testq $4, %rax - je .L16 - ALIGN_4 - - KERNEL_SUB1(16 * 0) - KERNEL_SUB2(16 * 0) - KERNEL_SUB3(16 * 0) - KERNEL_SUB4(16 * 0) - - subq $-16 * SIZE, BO - subq $-16 * SIZE, AO - ALIGN_4 - -.L16: -#ifndef TRMMKERNEL - movq K, %rax -#else - movq KKK, %rax -#endif - vmovddup ALPHA, %xmm7 - andq $3, %rax # if (k & 1) - je .L19 - - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 4), AO - leaq (BO, %rax, 4), BO - negq %rax - ALIGN_4 - -.L17: - vmovups -16 * SIZE(AO, %rax, 4), %xmm0 - vmovups -14 * SIZE(AO, %rax, 4), %xmm2 - vmovddup -16 * SIZE(BO, %rax, 4), %xmm1 - vmovddup -15 * SIZE(BO, %rax, 4), %xmm3 - vfmaddpd %xmm8,%xmm0, %xmm1,%xmm8 - vfmaddpd %xmm9,%xmm0, %xmm3,%xmm9 - vfmaddpd %xmm12,%xmm2, %xmm1,%xmm12 - vfmaddpd %xmm13,%xmm2, %xmm3,%xmm13 - vmovddup -14 * SIZE(BO, %rax, 4), %xmm1 - vmovddup -13 * SIZE(BO, %rax, 4), %xmm3 - vfmaddpd %xmm10,%xmm0, %xmm1,%xmm10 - vfmaddpd %xmm11,%xmm0, %xmm3,%xmm11 - vfmaddpd %xmm14,%xmm2, %xmm1,%xmm14 - vfmaddpd %xmm15,%xmm2, %xmm3,%xmm15 -/* - vmovups -12 * SIZE(AO, %rax, 4), %xmm0 - vmovddup -12 * SIZE(BO, %rax, 4), %xmm1 - vmovddup -11 * SIZE(BO, %rax, 4), %xmm3 - vmovaps %xmm0, %xmm2 -*/ - addq $SIZE, %rax - jl .L17 - ALIGN_4 - -.L19: - // prefetch -8 * SIZE(BB) - subq $-16 * SIZE, BB - -#ifndef TRMMKERNEL - - vfmaddpd (CO1),%xmm7, %xmm8,%xmm8 - vfmaddpd 2 * SIZE(CO1),%xmm7, %xmm12,%xmm12 - vfmaddpd (CO1, LDC),%xmm7, %xmm9,%xmm9 - vfmaddpd 2 * SIZE(CO1, LDC),%xmm7, %xmm13,%xmm13 - vfmaddpd (CO2),%xmm7, %xmm10,%xmm10 - vfmaddpd 2 * SIZE(CO2),%xmm7, %xmm14,%xmm14 - vfmaddpd (CO2, LDC),%xmm7, %xmm11,%xmm11 - vfmaddpd 2 * SIZE(CO2, LDC),%xmm7, %xmm15,%xmm15 - -#else - vmulpd %xmm7, %xmm8,%xmm8 - vmulpd %xmm7, %xmm12,%xmm12 - vmulpd %xmm7, %xmm9,%xmm9 - vmulpd %xmm7, %xmm13,%xmm13 - vmulpd %xmm7, %xmm10,%xmm10 - vmulpd %xmm7, %xmm14,%xmm14 - vmulpd %xmm7, %xmm11,%xmm11 - vmulpd %xmm7, %xmm15,%xmm15 - -#endif - - vmovups %xmm8, (CO1) - vmovups %xmm12, 2 * SIZE(CO1) - vmovups %xmm9, (CO1, LDC) - vmovups %xmm13, 2 * SIZE(CO1, LDC) - vmovups %xmm10, (CO2) - vmovups %xmm14, 2 * SIZE(CO2) - vmovups %xmm11, (CO2, LDC) - vmovups %xmm15, 2 * SIZE(CO2, LDC) - -#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq K, %rax - subq KKK, %rax - leaq (,%rax, SIZE), %rax - leaq (AO, %rax, 4), AO - leaq (BO, %rax, 4), BO -#endif - -#if defined(TRMMKERNEL) && defined(LEFT) - addq $4, KK -#endif - - addq $4 * SIZE, CO1 # coffset += 4 - addq $4 * SIZE, CO2 # coffset += 4 - decq I # i -- - BRANCH - jg .L11 - ALIGN_4 - -.L20: - testq $3, M - je .L39 - - testq $2, M - je .L30 - ALIGN_4 - -.L21: -#if !defined(TRMMKERNEL) || \ - (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq B, BO -#else - movq KK, %rax - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 2), AO - leaq (B, %rax, 4), BO -#endif - - vmovups -16 * SIZE(AO), %xmm0 - vxorps %xmm8, %xmm8, %xmm8 - vmovups -12 * SIZE(AO), %xmm2 - vxorps %xmm9, %xmm9 ,%xmm9 - vmovddup -16 * SIZE(BO), %xmm1 - vxorps %xmm10, %xmm10, %xmm10 - vmovddup -15 * SIZE(BO), %xmm5 - vxorps %xmm11, %xmm11, %xmm11 - vmovddup -8 * SIZE(BO), %xmm3 - -#ifndef TRMMKERNEL - movq K, %rax -#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) - movq K, %rax - subq KK, %rax - movq %rax, KKK -#else - movq KK, %rax -#ifdef LEFT - addq $2, %rax -#else - addq $4, %rax -#endif - movq %rax, KKK -#endif - - andq $-4, %rax - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 2), AO - leaq (BO, %rax, 4), BO - negq %rax - NOBRANCH - je .L26 - ALIGN_4 - -.L22: - vfmaddpd %xmm8,%xmm0, %xmm1,%xmm8 - vfmaddpd %xmm9,%xmm0, %xmm5,%xmm9 - vmovddup -14 * SIZE(BO, %rax, 4), %xmm1 - vfmaddpd %xmm10,%xmm0, %xmm1,%xmm10 - vmovddup -13 * SIZE(BO, %rax, 4), %xmm5 - vfmaddpd %xmm11,%xmm0, %xmm5,%xmm11 - vmovups -14 * SIZE(AO, %rax, 2), %xmm0 - vmovddup -12 * SIZE(BO, %rax, 4), %xmm1 - vmovddup -11 * SIZE(BO, %rax, 4), %xmm5 - vfmaddpd %xmm8,%xmm0, %xmm1,%xmm8 - vfmaddpd %xmm9,%xmm0, %xmm5,%xmm9 - vmovddup -10 * SIZE(BO, %rax, 4), %xmm1 - vmovddup -9 * SIZE(BO, %rax, 4), %xmm5 - vfmaddpd %xmm10,%xmm0, %xmm1,%xmm10 - vfmaddpd %xmm11,%xmm0, %xmm5,%xmm11 - vmovddup (BO, %rax, 4), %xmm1 - vmovddup -7 * SIZE(BO, %rax, 4), %xmm5 - vmovups -8 * SIZE(AO, %rax, 2), %xmm0 - vfmaddpd %xmm8,%xmm2, %xmm3,%xmm8 - vfmaddpd %xmm9,%xmm2, %xmm5,%xmm9 - vmovddup -6 * SIZE(BO, %rax, 4), %xmm3 - vmovddup -5 * SIZE(BO, %rax, 4), %xmm5 - vfmaddpd %xmm10,%xmm2, %xmm3,%xmm10 - vfmaddpd %xmm11,%xmm2, %xmm5,%xmm11 - vmovups -10 * SIZE(AO, %rax, 2), %xmm2 - vmovddup -4 * SIZE(BO, %rax, 4), %xmm3 - vmovddup -3 * SIZE(BO, %rax, 4), %xmm5 - vfmaddpd %xmm8,%xmm2, %xmm3,%xmm8 - vfmaddpd %xmm9,%xmm2, %xmm5,%xmm9 - vmovddup -2 * SIZE(BO, %rax, 4), %xmm3 - vmovddup -1 * SIZE(BO, %rax, 4), %xmm5 - vfmaddpd %xmm10,%xmm2, %xmm3,%xmm10 - vfmaddpd %xmm11,%xmm2, %xmm5,%xmm11 - vmovddup 8 * SIZE(BO, %rax, 4), %xmm3 - vmovups -4 * SIZE(AO, %rax, 2), %xmm2 - vmovddup 1 * SIZE(BO, %rax, 4), %xmm5 - - addq $4 * SIZE, %rax - BRANCH - jl .L22 - ALIGN_4 - -.L26: - vmovddup ALPHA, %xmm7 - -#ifndef TRMMKERNEL - movq K, %rax -#else - movq KKK, %rax -#endif - andq $3, %rax # if (k & 1) - je .L29 - - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 2), AO - leaq (BO, %rax, 4), BO - negq %rax - ALIGN_4 - -.L27: - vfmaddpd %xmm8,%xmm0, %xmm1,%xmm8 - vmovddup -14 * SIZE(BO, %rax, 4), %xmm1 - vfmaddpd %xmm9,%xmm0, %xmm5,%xmm9 - vmovddup -13 * SIZE(BO, %rax, 4), %xmm5 - vfmaddpd %xmm10,%xmm0, %xmm1,%xmm10 - vmovddup -12 * SIZE(BO, %rax, 4), %xmm1 - vfmaddpd %xmm11,%xmm0, %xmm5,%xmm11 - vmovups -14 * SIZE(AO, %rax, 2), %xmm0 - vmovddup -11 * SIZE(BO, %rax, 4), %xmm5 - - addq $SIZE, %rax - jl .L27 - ALIGN_4 - -.L29: -#ifndef TRMMKERNEL - - vfmaddpd (CO1),%xmm7, %xmm8,%xmm8 - vfmaddpd (CO1, LDC),%xmm7, %xmm9,%xmm9 - vfmaddpd (CO2),%xmm7, %xmm10,%xmm10 - vfmaddpd (CO2, LDC),%xmm7, %xmm11,%xmm11 - -#else - vmulpd %xmm7, %xmm8,%xmm8 - vmulpd %xmm7, %xmm9,%xmm9 - vmulpd %xmm7, %xmm10,%xmm10 - vmulpd %xmm7, %xmm11,%xmm11 - -#endif - - vmovups %xmm8, (CO1) - vmovups %xmm9, (CO1, LDC) - - vmovups %xmm10, (CO2) - vmovups %xmm11, (CO2, LDC) - -#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq K, %rax - subq KKK, %rax - leaq (,%rax, SIZE), %rax - leaq (AO, %rax, 2), AO - leaq (BO, %rax, 4), BO -#endif - -#if defined(TRMMKERNEL) && defined(LEFT) - addq $2, KK -#endif - - addq $2 * SIZE, CO1 - addq $2 * SIZE, CO2 - ALIGN_4 - -.L30: - testq $1, M - je .L39 - -#if !defined(TRMMKERNEL) || \ - (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq B, BO -#else - movq KK, %rax - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 1), AO - leaq (B, %rax, 4), BO -#endif - - vmovddup -16 * SIZE(AO), %xmm0 - vxorps %xmm8, %xmm8, %xmm8 - vmovddup -14 * SIZE(AO), %xmm2 - vxorps %xmm9, %xmm9, %xmm9 - vmovddup -15 * SIZE(AO), %xmm4 - vxorps %xmm10, %xmm10,%xmm10 - vmovups -16 * SIZE(BO), %xmm1 - vxorps %xmm11, %xmm11,%xmm11 - vmovups -8 * SIZE(BO), %xmm3 - -#ifndef TRMMKERNEL - movq K, %rax -#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) - movq K, %rax - subq KK, %rax - movq %rax, KKK -#else - movq KK, %rax -#ifdef LEFT - addq $1, %rax -#else - addq $4, %rax -#endif - movq %rax, KKK -#endif - - andq $-4, %rax - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 1), AO - leaq (BO, %rax, 4), BO - negq %rax - NOBRANCH - je .L36 - ALIGN_4 - -.L32: - vfmaddpd %xmm8,%xmm0, %xmm1,%xmm8 - vfmaddpd %xmm9,-14 * SIZE(BO, %rax, 4), %xmm0,%xmm9 - vmovups -12 * SIZE(BO, %rax, 4), %xmm1 - vmovddup -12 * SIZE(AO, %rax, 1), %xmm0 - vfmaddpd %xmm10,%xmm4, %xmm1,%xmm10 - vfmaddpd %xmm11,-10 * SIZE(BO, %rax, 4), %xmm4,%xmm11 - vmovups (BO, %rax, 4), %xmm1 - vmovddup -11 * SIZE(AO, %rax, 1), %xmm4 - vfmaddpd %xmm8,%xmm2, %xmm3,%xmm8 - vfmaddpd %xmm9,-6 * SIZE(BO, %rax, 4), %xmm2,%xmm9 - vmovups -4 * SIZE(BO, %rax, 4), %xmm3 - vmovddup -13 * SIZE(AO, %rax, 1), %xmm2 - vfmaddpd %xmm10,%xmm2, %xmm3,%xmm10 - vfmaddpd %xmm11,-2 * SIZE(BO, %rax, 4), %xmm2,%xmm11 - vmovups 8 * SIZE(BO, %rax, 4), %xmm3 - vmovddup -10 * SIZE(AO, %rax, 1), %xmm2 - - addq $4 * SIZE, %rax - BRANCH - jl .L32 - ALIGN_4 - -.L36: - vmovddup ALPHA, %xmm7 - -#ifndef TRMMKERNEL - movq K, %rax -#else - movq KKK, %rax -#endif - andq $3, %rax # if (k & 1) - je .L38 - - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 1), AO - leaq (BO, %rax, 4), BO - negq %rax - ALIGN_4 - -.L37: - vfmaddpd %xmm8,%xmm0, %xmm1,%xmm8 - vfmaddpd %xmm9,-14 * SIZE(BO, %rax, 4), %xmm0,%xmm9 - vmovups -12 * SIZE(BO, %rax, 4), %xmm1 - vmovddup -15 * SIZE(AO, %rax, 1), %xmm0 - - addq $SIZE, %rax - jl .L37 - ALIGN_4 - -.L38: - vaddpd %xmm10, %xmm8,%xmm8 - vaddpd %xmm11, %xmm9,%xmm9 - -#ifndef TRMMKERNEL - vmovsd (CO1), %xmm0 - vmovhpd (CO1, LDC), %xmm0,%xmm0 - vmovsd (CO2), %xmm1 - vmovhpd (CO2, LDC), %xmm1,%xmm1 - - - vfmaddpd %xmm0, %xmm7,%xmm8,%xmm8 - vfmaddpd %xmm1, %xmm7,%xmm9,%xmm9 -#else - - vmulpd %xmm7, %xmm8,%xmm8 - vmulpd %xmm7, %xmm9,%xmm9 - -#endif - - vmovsd %xmm8, (CO1) - vmovhpd %xmm8, (CO1, LDC) - vmovsd %xmm9, (CO2) - vmovhpd %xmm9, (CO2, LDC) - -#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq K, %rax - subq KKK, %rax - leaq (,%rax, SIZE), %rax - leaq (AO, %rax, 1), AO - leaq (BO, %rax, 4), BO -#endif - -#if defined(TRMMKERNEL) && defined(LEFT) - addq $1, KK -#endif - ALIGN_4 - -.L39: -#if defined(TRMMKERNEL) && !defined(LEFT) - addq $4, KK -#endif - - movq BO, B - - decq J # j -- - jg .L01 - ALIGN_4 - -.L40: # N % 4 - testq $3, N # N % 4 == 3 - je .L999 # Jump to end if N % 4 == 0 - - testq $2, N # N % 4 == 2 - je .L80 - ALIGN_4 - -.L41: # N % 4 > 1 -#if defined(TRMMKERNEL) && defined(LEFT) - movq OFFSET, %rax - movq %rax, KK -#endif - - movq C, CO1 # coffset1 = c - leaq (C, LDC, 1), CO2 # coffset2 = c + ldc - movq A, AO # aoffset = a - - movq K, %rax - salq $BASE_SHIFT + 1, %rax # k << 4 - leaq (B, %rax), BB - - movq M, I - sarq $2, I # i = (m >> 2) - jle .L60 - ALIGN_4 - -.L51: -#if !defined(TRMMKERNEL) || \ - (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq B, BO -#else - movq KK, %rax - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 4), AO - leaq (B, %rax, 2), BO -#endif - - vmovddup -16 * SIZE(BO), %xmm1 - vmovddup -15 * SIZE(BO), %xmm5 - vmovddup -12 * SIZE(BO), %xmm3 - vxorps %xmm8, %xmm8,%xmm8 - vxorps %xmm9, %xmm9,%xmm9 - vxorps %xmm12, %xmm12,%xmm12 - vxorps %xmm13, %xmm13,%xmm13 - vmovups -16 * SIZE(AO), %xmm0 - vmovups -8 * SIZE(AO), %xmm4 - vmovups %xmm0, %xmm2 - subq $-8 * SIZE, BB - -#ifndef TRMMKERNEL - movq K, %rax -#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) - movq K, %rax - subq KK, %rax - movq %rax, KKK -#else - movq KK, %rax -#ifdef LEFT - addq $4, %rax -#else - addq $2, %rax -#endif - movq %rax, KKK -#endif - - - andq $-4, %rax - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 4), AO - leaq (BO, %rax, 2), BO - negq %rax - NOBRANCH - je .L56 - ALIGN_4 - -.L52: # Loop for (N % 4) == 2 - vfmaddpd %xmm8,%xmm1, %xmm0,%xmm8 - vfmaddpd %xmm9,%xmm5, %xmm2,%xmm9 - vmovups -14 * SIZE(AO, %rax, 4),%xmm2 - vfmaddpd %xmm12,%xmm2, %xmm1,%xmm12 - vmovups -12 * SIZE(AO, %rax, 4), %xmm0 - vmovddup -14 * SIZE(BO, %rax, 2), %xmm1 - vfmaddpd %xmm13,%xmm2, %xmm5,%xmm13 - vmovddup -13 * SIZE(BO, %rax, 2), %xmm5 - vmovups -10 * SIZE(AO, %rax, 4), %xmm2 - vfmaddpd %xmm8,%xmm1, %xmm0,%xmm8 - vfmaddpd %xmm12,%xmm2, %xmm1,%xmm12 - vfmaddpd %xmm9,%xmm5, %xmm0,%xmm9 - vmovups (AO, %rax, 4), %xmm0 - vmovddup -8 * SIZE(BO, %rax, 2), %xmm1 - vfmaddpd %xmm13,%xmm2, %xmm5,%xmm13 - vmovddup -11 * SIZE(BO, %rax, 2), %xmm5 - vmovups -6 * SIZE(AO, %rax, 4), %xmm2 - vfmaddpd %xmm8,%xmm3, %xmm4,%xmm8 - vfmaddpd %xmm12,%xmm2, %xmm3,%xmm12 - vfmaddpd %xmm9,%xmm5, %xmm4,%xmm9 - vmovups -4 * SIZE(AO, %rax, 4), %xmm4 - vmovddup -10 * SIZE(BO, %rax, 2), %xmm3 - vfmaddpd %xmm13,%xmm2, %xmm5,%xmm13 - vmovddup -9 * SIZE(BO, %rax, 2), %xmm5 - vmovups -2 * SIZE(AO, %rax, 4), %xmm2 - vfmaddpd %xmm8,%xmm3, %xmm4,%xmm8 - vfmaddpd %xmm12,%xmm2, %xmm3,%xmm12 - vfmaddpd %xmm9,%xmm5, %xmm4,%xmm9 - vfmaddpd %xmm13,%xmm2, %xmm5,%xmm13 - vmovups 8 * SIZE(AO, %rax, 4), %xmm4 - vmovddup -4 * SIZE(BO, %rax, 2), %xmm3 - vmovddup -7 * SIZE(BO, %rax, 2), %xmm5 - vmovaps %xmm0, %xmm2 - - addq $4 * SIZE, %rax - BRANCH - jl .L52 - ALIGN_4 - -.L56: - vmovddup ALPHA, %xmm7 - -#ifndef TRMMKERNEL - movq K, %rax -#else - movq KKK, %rax -#endif - andq $3, %rax # if (k & 1) - je .L59 - - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 4), AO - leaq (BO, %rax, 2), BO - negq %rax - ALIGN_4 - -.L57: - vfmaddpd %xmm8,%xmm1, %xmm0,%xmm8 - vfmaddpd %xmm9,%xmm5, %xmm2,%xmm9 - vmovups -14 * SIZE(AO, %rax, 4),%xmm2 - vfmaddpd %xmm12,%xmm2, %xmm1,%xmm12 - vmovups -12 * SIZE(AO, %rax, 4), %xmm0 - vmovddup -14 * SIZE(BO, %rax, 2), %xmm1 - vfmaddpd %xmm13,%xmm2, %xmm5,%xmm13 - vmovddup -13 * SIZE(BO, %rax, 2), %xmm5 - vmovaps %xmm0, %xmm2 - - addq $SIZE, %rax - jl .L57 - ALIGN_4 - -.L59: -#ifndef TRMMKERNEL - vfmaddpd (CO1),%xmm7, %xmm8, %xmm8 - vfmaddpd 2 * SIZE(CO1),%xmm7, %xmm12, %xmm12 - vfmaddpd (CO2),%xmm7, %xmm9, %xmm9 - vfmaddpd 2 * SIZE(CO2),%xmm7, %xmm13, %xmm13 - -#else - vmulpd %xmm7, %xmm8,%xmm8 - vmulpd %xmm7, %xmm9,%xmm9 - vmulpd %xmm7, %xmm12,%xmm12 - vmulpd %xmm7, %xmm13,%xmm13 - -#endif - - vmovups %xmm8, (CO1) - vmovups %xmm12, 2 * SIZE(CO1) - - vmovups %xmm9, (CO2) - vmovups %xmm13, 2 * SIZE(CO2) - -#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq K, %rax - subq KKK, %rax - leaq (,%rax, SIZE), %rax - leaq (AO, %rax, 4), AO - leaq (BO, %rax, 2), BO -#endif - -#if defined(TRMMKERNEL) && defined(LEFT) - addq $4, KK -#endif - - addq $4 * SIZE, CO1 # coffset += 4 - addq $4 * SIZE, CO2 # coffset += 4 - decq I # i -- - jg .L51 - ALIGN_4 - -.L60: - testq $2, M - je .L70 - ALIGN_4 - -.L61: -#if !defined(TRMMKERNEL) || \ - (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq B, BO -#else - movq KK, %rax - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 2), AO - leaq (B, %rax, 2), BO -#endif - - vmovups -16 * SIZE(AO), %xmm0 - vxorps %xmm8, %xmm8,%xmm8 - vmovups -12 * SIZE(AO), %xmm2 - vxorps %xmm9, %xmm9,%xmm9 - vmovddup -16 * SIZE(BO), %xmm1 - vxorps %xmm10, %xmm10,%xmm10 - vmovddup -15 * SIZE(BO), %xmm3 - vxorps %xmm11, %xmm11,%xmm11 - -#ifndef TRMMKERNEL - movq K, %rax -#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) - movq K, %rax - subq KK, %rax - movq %rax, KKK -#else - movq KK, %rax -#ifdef LEFT - addq $2, %rax -#else - addq $2, %rax -#endif - movq %rax, KKK -#endif - - andq $-4, %rax - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 2), AO - leaq (BO, %rax, 2), BO - negq %rax - NOBRANCH - je .L66 - ALIGN_4 - -.L62: - vfmaddpd %xmm8,%xmm0, %xmm1,%xmm8 - vmovddup -14 * SIZE(BO, %rax, 2), %xmm1 - vfmaddpd %xmm9,%xmm0, %xmm3,%xmm9 - vmovups -14 * SIZE(AO, %rax, 2), %xmm0 - vmovddup -13 * SIZE(BO, %rax, 2), %xmm3 - vfmaddpd %xmm10,%xmm0, %xmm1,%xmm10 - vmovddup -12 * SIZE(BO, %rax, 2), %xmm1 - vfmaddpd %xmm11,%xmm0, %xmm3,%xmm11 - vmovups -8 * SIZE(AO, %rax, 2), %xmm0 - vmovddup -11 * SIZE(BO, %rax, 2), %xmm3 - vfmaddpd %xmm8,%xmm2, %xmm1,%xmm8 - vmovddup -10 * SIZE(BO, %rax, 2), %xmm1 - vfmaddpd %xmm9,%xmm2, %xmm3,%xmm9 - vmovups -10 * SIZE(AO, %rax, 2), %xmm2 - vmovddup -9 * SIZE(BO, %rax, 2), %xmm3 - vfmaddpd %xmm10,%xmm2, %xmm1,%xmm10 - vmovddup -8 * SIZE(BO, %rax, 2), %xmm1 - vfmaddpd %xmm11,%xmm2, %xmm3,%xmm11 - vmovups -4 * SIZE(AO, %rax, 2), %xmm2 - vmovddup -7 * SIZE(BO, %rax, 2), %xmm3 - - addq $4 * SIZE, %rax - BRANCH - jl .L62 - ALIGN_4 - -.L66: - vmovddup ALPHA, %xmm7 - -#ifndef TRMMKERNEL - movq K, %rax -#else - movq KKK, %rax -#endif - andq $3, %rax # if (k & 1) - je .L69 - - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 2), AO - leaq (BO, %rax, 2), BO - negq %rax - ALIGN_4 - -.L67: - vfmaddpd %xmm8,%xmm0, %xmm1,%xmm8 - vmovddup -14 * SIZE(BO, %rax, 2), %xmm1 - vfmaddpd %xmm9,%xmm0, %xmm3,%xmm9 - vmovups -14 * SIZE(AO, %rax, 2), %xmm0 - vmovddup -13 * SIZE(BO, %rax, 2), %xmm3 - - addq $SIZE, %rax - jl .L67 - ALIGN_4 - -.L69: - vaddpd %xmm10, %xmm8,%xmm8 - vaddpd %xmm11, %xmm9,%xmm9 - -#ifndef TRMMKERNEL - - vfmaddpd (CO1),%xmm7, %xmm8,%xmm8 - vfmaddpd (CO2),%xmm7, %xmm9,%xmm9 - -#else - - vmulpd %xmm7, %xmm8,%xmm8 - vmulpd %xmm7, %xmm9,%xmm9 - -#endif - - vmovups %xmm8, (CO1) - vmovups %xmm9, (CO2) - -#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq K, %rax - subq KKK, %rax - leaq (,%rax, SIZE), %rax - leaq (AO, %rax, 2), AO - leaq (BO, %rax, 2), BO -#endif - -#if defined(TRMMKERNEL) && defined(LEFT) - addq $2, KK -#endif - - addq $2 * SIZE, CO1 # coffset += 4 - addq $2 * SIZE, CO2 # coffset += 4 - ALIGN_4 - -.L70: - testq $1, M - je .L79 - ALIGN_4 - -.L71: -#if !defined(TRMMKERNEL) || \ - (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq B, BO -#else - movq KK, %rax - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 1), AO - leaq (B, %rax, 2), BO -#endif - - vmovddup -16 * SIZE(AO), %xmm0 - vxorps %xmm8, %xmm8,%xmm8 - vmovddup -15 * SIZE(AO), %xmm1 - vxorps %xmm9, %xmm9,%xmm9 - vmovddup -14 * SIZE(AO), %xmm2 - vxorps %xmm10, %xmm10,%xmm10 - vmovddup -13 * SIZE(AO), %xmm3 - vxorps %xmm11, %xmm11,%xmm11 - -#ifndef TRMMKERNEL - movq K, %rax -#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) - movq K, %rax - subq KK, %rax - movq %rax, KKK -#else - movq KK, %rax -#ifdef LEFT - addq $1, %rax -#else - addq $2, %rax -#endif - movq %rax, KKK -#endif - - andq $-4, %rax - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 1), AO - leaq (BO, %rax, 2), BO - negq %rax - NOBRANCH - je .L76 - ALIGN_4 - -.L72: - vfmaddpd %xmm8,-16 * SIZE(BO, %rax, 2), %xmm0,%xmm8 - vmovddup -12 * SIZE(AO, %rax, 1), %xmm0 - - vfmaddpd %xmm9,-14 * SIZE(BO, %rax, 2), %xmm1,%xmm9 - vmovddup -11 * SIZE(AO, %rax, 1), %xmm1 - - vfmaddpd %xmm10,-12 * SIZE(BO, %rax, 2), %xmm2,%xmm10 - vmovddup -10 * SIZE(AO, %rax, 1), %xmm2 - - vfmaddpd %xmm11,-10 * SIZE(BO, %rax, 2), %xmm3,%xmm11 - vmovddup -9 * SIZE(AO, %rax, 1), %xmm3 - - addq $4 * SIZE, %rax - BRANCH - jl .L72 - ALIGN_4 - -.L76: - vmovddup ALPHA, %xmm7 - -#ifndef TRMMKERNEL - movq K, %rax -#else - movq KKK, %rax -#endif - andq $3, %rax # if (k & 1) - je .L78 - - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 1), AO - leaq (BO, %rax, 2), BO - negq %rax - ALIGN_4 - -.L77: - vfmaddpd %xmm8,-16 * SIZE(BO, %rax, 2), %xmm0,%xmm8 - vmovddup -15 * SIZE(AO, %rax, 1), %xmm0 - - addq $SIZE, %rax - jl .L77 - ALIGN_4 - -.L78: - vaddpd %xmm9, %xmm8,%xmm8 - vaddpd %xmm11, %xmm10,%xmm10 - vaddpd %xmm10, %xmm8,%xmm8 - -#ifndef TRMMKERNEL - vmovsd (CO1), %xmm0 - vmovhpd (CO2), %xmm0,%xmm0 -#endif - - vmulpd %xmm7, %xmm8,%xmm8 - -#ifndef TRMMKERNEL - vaddpd %xmm0, %xmm8,%xmm8 -#endif - - vmovsd %xmm8, (CO1) - vmovhpd %xmm8, (CO2) - -#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq K, %rax - subq KKK, %rax - leaq (,%rax, SIZE), %rax - leaq (AO, %rax, 1), AO - leaq (BO, %rax, 2), BO -#endif - -#if defined(TRMMKERNEL) && defined(LEFT) - addq $1, KK -#endif - ALIGN_4 - -.L79: -#if defined(TRMMKERNEL) && !defined(LEFT) - addq $2, KK -#endif - - movq BO, B - - leaq (C, LDC, 2), C - ALIGN_4 - -.L80: - testq $1, N # N % 4 == 1 - je .L999 # Jump to end if N % 4 == 0 - ALIGN_4 - -.L81: -#if defined(TRMMKERNEL) && defined(LEFT) - movq OFFSET, %rax - movq %rax, KK -#endif - - movq C, CO1 # coffset1 = c - movq A, AO # aoffset = a - - movq M, I - sarq $2, I # i = (m >> 2) - jle .L100 - ALIGN_4 - -.L91: -#if !defined(TRMMKERNEL) || \ - (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq B, BO -#else - movq KK, %rax - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 4), AO - leaq (B, %rax, 1), BO -#endif - - vmovups -8 * SIZE(AO), %xmm2 - vxorps %xmm8, %xmm8,%xmm8 - vmovups -16 * SIZE(AO), %xmm0 - vxorps %xmm9, %xmm9,%xmm9 - vmovddup -16 * SIZE(BO), %xmm1 - vxorps %xmm12, %xmm12,%xmm12 - vmovddup -14 * SIZE(BO), %xmm3 - vxorps %xmm13, %xmm13,%xmm13 - vmovddup -15 * SIZE(BO), %xmm5 - - // prefetchw 3 * SIZE(CO1) - -#ifndef TRMMKERNEL - movq K, %rax -#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) - movq K, %rax - subq KK, %rax - movq %rax, KKK -#else - movq KK, %rax -#ifdef LEFT - addq $4, %rax -#else - addq $1, %rax -#endif - movq %rax, KKK -#endif - - andq $-4, %rax - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 4), AO - leaq (BO, %rax, 1), BO - negq %rax - NOBRANCH - je .L96 - ALIGN_4 - -.L92: - vfmaddpd %xmm8,%xmm1, %xmm0,%xmm8 - vfmaddpd %xmm12,-14 * SIZE(AO, %rax, 4), %xmm1,%xmm12 - vmovapd -12 * SIZE(AO, %rax, 4), %xmm0 - vmovddup -12 * SIZE(BO, %rax, 1), %xmm1 - vfmaddpd %xmm9,%xmm5, %xmm0,%xmm9 - vfmaddpd %xmm13,-10 * SIZE(AO, %rax, 4), %xmm5,%xmm13 - vmovapd (AO, %rax, 4), %xmm0 - vmovddup -13 * SIZE(BO, %rax, 1), %xmm5 - vfmaddpd %xmm8,%xmm3, %xmm2,%xmm8 - vfmaddpd %xmm12,-6 * SIZE(AO, %rax, 4), %xmm3,%xmm12 - vmovapd -4 * SIZE(AO, %rax, 4), %xmm2 - vmovddup -10 * SIZE(BO, %rax, 1), %xmm3 - vfmaddpd %xmm9,%xmm5, %xmm2,%xmm9 - vfmaddpd %xmm13,-2 * SIZE(AO, %rax, 4), %xmm5,%xmm13 - vmovapd 8 * SIZE(AO, %rax, 4), %xmm2 - vmovddup -11 * SIZE(BO, %rax, 1), %xmm5 - - addq $4 * SIZE, %rax - BRANCH - jl .L92 - ALIGN_4 - -.L96: - vmovddup ALPHA, %xmm7 - -#ifndef TRMMKERNEL - movq K, %rax -#else - movq KKK, %rax -#endif - andq $3, %rax # if (k & 1) - je .L99 - - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 4), AO - leaq (BO, %rax, 1), BO - negq %rax - ALIGN_4 - -.L97: - vfmaddpd %xmm8,%xmm1, %xmm0,%xmm8 - vfmaddpd %xmm12,-14 * SIZE(AO, %rax, 4), %xmm1,%xmm12 - vmovups -12 * SIZE(AO, %rax, 4), %xmm0 - vmovddup -15 * SIZE(BO, %rax, 1), %xmm1 - - addq $SIZE, %rax - jl .L97 - ALIGN_4 - -.L99: - vaddpd %xmm9, %xmm8,%xmm8 - vaddpd %xmm13, %xmm12,%xmm12 - -#ifndef TRMMKERNEL - - vfmaddpd (CO1),%xmm7, %xmm8,%xmm8 - vfmaddpd 2 * SIZE(CO1),%xmm7,%xmm12,%xmm12 - -#else - vmulpd %xmm7, %xmm8,%xmm8 - vmulpd %xmm7, %xmm12,%xmm12 - -#endif - - vmovups %xmm8, (CO1) - vmovups %xmm12, 2 * SIZE(CO1) - -#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq K, %rax - subq KKK, %rax - leaq (,%rax, SIZE), %rax - leaq (AO, %rax, 4), AO - leaq (BO, %rax, 1), BO -#endif - -#if defined(TRMMKERNEL) && defined(LEFT) - addq $4, KK -#endif - - addq $4 * SIZE, CO1 # coffset += 4 - decq I # i -- - jg .L91 - ALIGN_4 - -.L100: - testq $2, M - je .L110 - ALIGN_4 - -.L101: -#if !defined(TRMMKERNEL) || \ - (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq B, BO -#else - movq KK, %rax - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 2), AO - leaq (B, %rax, 1), BO -#endif - - vmovddup -16 * SIZE(BO), %xmm0 - vxorps %xmm8, %xmm8,%xmm8 - vmovddup -15 * SIZE(BO), %xmm1 - vxorps %xmm9, %xmm9,%xmm9 - vmovddup -14 * SIZE(BO), %xmm2 - vxorps %xmm10, %xmm10,%xmm10 - vmovddup -13 * SIZE(BO), %xmm3 - vxorps %xmm11, %xmm11,%xmm11 - -#ifndef TRMMKERNEL - movq K, %rax -#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) - movq K, %rax - subq KK, %rax - movq %rax, KKK -#else - movq KK, %rax -#ifdef LEFT - addq $2, %rax -#else - addq $1, %rax -#endif - movq %rax, KKK -#endif - - andq $-4, %rax - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 2), AO - leaq (BO, %rax, 1), BO - negq %rax - NOBRANCH - je .L106 - ALIGN_4 - -.L102: - vfmaddpd %xmm8,-16 * SIZE(AO, %rax, 2), %xmm0,%xmm8 - vmovddup -12 * SIZE(BO, %rax, 1), %xmm0 - - vfmaddpd %xmm9,-14 * SIZE(AO, %rax, 2), %xmm1,%xmm9 - vmovddup -11 * SIZE(BO, %rax, 1), %xmm1 - - vfmaddpd %xmm10,-12 * SIZE(AO, %rax, 2), %xmm2,%xmm10 - vmovddup -10 * SIZE(BO, %rax, 1), %xmm2 - - vfmaddpd %xmm11,-10 * SIZE(AO, %rax, 2), %xmm3,%xmm11 - vmovddup -9 * SIZE(BO, %rax, 1), %xmm3 - - addq $4 * SIZE, %rax - BRANCH - jl .L102 - ALIGN_4 - -.L106: - vmovddup ALPHA, %xmm7 - -#ifndef TRMMKERNEL - movq K, %rax -#else - movq KKK, %rax -#endif - andq $3, %rax # if (k & 1) - je .L109 - - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 2), AO - leaq (BO, %rax, 1), BO - negq %rax - ALIGN_4 - -.L107: - vmovddup -16 * SIZE(BO, %rax, 1), %xmm0 - vfmaddpd %xmm8,-16 * SIZE(AO, %rax, 2), %xmm0,%xmm8 - - addq $SIZE, %rax - jl .L107 - ALIGN_4 - -.L109: - vaddpd %xmm9, %xmm8,%xmm8 - vaddpd %xmm11, %xmm10,%xmm10 - vaddpd %xmm10, %xmm8,%xmm8 - -#ifndef TRMMKERNEL - - vfmaddpd (CO1),%xmm7, %xmm8,%xmm8 -#else - vmulpd %xmm7, %xmm8,%xmm8 - -#endif - - vmovups %xmm8, (CO1) - -#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq K, %rax - subq KKK, %rax - leaq (,%rax, SIZE), %rax - leaq (AO, %rax, 2), AO - leaq (BO, %rax, 1), BO -#endif - -#if defined(TRMMKERNEL) && defined(LEFT) - addq $2, KK -#endif - - addq $2 * SIZE, CO1 # coffset += 4 - - ALIGN_4 - -.L110: - testq $1, M - je .L999 - ALIGN_4 - -.L111: -#if !defined(TRMMKERNEL) || \ - (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ - (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) - movq B, BO -#else - movq KK, %rax - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 1), AO - leaq (B, %rax, 1), BO -#endif - - vmovups -16 * SIZE(AO), %xmm0 - vxorps %xmm8, %xmm8,%xmm8 - movups -14 * SIZE(AO), %xmm1 - vxorps %xmm9, %xmm9,%xmm9 - -#ifndef TRMMKERNEL - movq K, %rax -#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) - movq K, %rax - subq KK, %rax - movq %rax, KKK -#else - movq KK, %rax -#ifdef LEFT - addq $1, %rax -#else - addq $1, %rax -#endif - movq %rax, KKK -#endif - - andq $-4, %rax - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 1), AO - leaq (BO, %rax, 1), BO - negq %rax - NOBRANCH - je .L116 - ALIGN_4 - -.L112: - vfmaddpd %xmm8,-16 * SIZE(BO, %rax, 1), %xmm0,%xmm8 - vmovups -12 * SIZE(AO, %rax, 1), %xmm0 - - vfmaddpd %xmm9,-14 * SIZE(BO, %rax, 1), %xmm1,%xmm9 - vmovups -10 * SIZE(AO, %rax, 1), %xmm1 - - addq $4 * SIZE, %rax - BRANCH - jl .L112 - ALIGN_4 - -.L116: - vmovddup ALPHA, %xmm7 - -#ifndef TRMMKERNEL - movq K, %rax -#else - movq KKK, %rax -#endif - andq $3, %rax # if (k & 1) - je .L118 - - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 1), AO - leaq (BO, %rax, 1), BO - negq %rax - ALIGN_4 - -.L117: - vmulsd -16 * SIZE(BO, %rax, 1), %xmm0,%xmm0 - vaddsd %xmm0, %xmm8,%xmm8 - vmovsd -15 * SIZE(AO, %rax, 1), %xmm0 - - addq $SIZE, %rax - jl .L117 - ALIGN_4 - -.L118: - vaddpd %xmm9, %xmm8,%xmm8 - vhaddpd %xmm8, %xmm8,%xmm8 - -#ifndef TRMMKERNEL - vmovsd (CO1), %xmm0 -#endif - - vmulsd %xmm7, %xmm8,%xmm8 - -#ifndef TRMMKERNEL - vaddsd %xmm0, %xmm8,%xmm8 -#endif - - vmovsd %xmm8, (CO1) - ALIGN_4 - -.L999: - movq (%rsp), %rbx - movq 8(%rsp), %rbp - movq 16(%rsp), %r12 - movq 24(%rsp), %r13 - movq 32(%rsp), %r14 - movq 40(%rsp), %r15 - -#ifdef WINDOWS_ABI - movq 48(%rsp), %rdi - movq 56(%rsp), %rsi - movups 64(%rsp), %xmm6 - movups 80(%rsp), %xmm7 - movups 96(%rsp), %xmm8 - movups 112(%rsp), %xmm9 - movups 128(%rsp), %xmm10 - movups 144(%rsp), %xmm11 - movups 160(%rsp), %xmm12 - movups 176(%rsp), %xmm13 - movups 192(%rsp), %xmm14 - movups 208(%rsp), %xmm15 -#endif - - addq $STACKSIZE, %rsp - ret - - EPILOGUE diff --git a/kernel/x86_64/dgemm_kernel_8x2_bulldozer.S b/kernel/x86_64/dgemm_kernel_8x2_bulldozer.S new file mode 100644 index 000000000..dc980cbe7 --- /dev/null +++ b/kernel/x86_64/dgemm_kernel_8x2_bulldozer.S @@ -0,0 +1,3854 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +/********************************************************************* +* 2013/06/02 Saar +* +* Parameter: +* UNROLL_M 8 +* UNROLL_N 2 +* DGEMM_P 360 +* DGEMM_Q 160 +* +* Performance at m x n without prefetch of BO: +* +* 5760x5760 93.4 GFLOPS with 8 threads on 4 modules (ACML: 90.8 GFLOPS) +* 5760x5760 84.2 GFLOPS with 4 threads on 4 modules (ACML: 82.4 GFLOPS) +* 3840x3840 50.3 GFLOPS with 2 threads on 2 modules (ACML: 49.5 GFLOPS) +* +* 5760x5760 56.4 GFLOPS with 4 threads on 2 modules (ACML: 58.5 GFLOPS) +* 3840x3840 29.0 GFLOPS with 2 threads on 1 modules (ACML: 30.2 GFLOPS) +* 3840x3840 26.1 GFLOPS with 1 threads on 1 modules (ACML: 25.9 GFLOPS) +* +*********************************************************************/ + +/********************************************************************* +* 2013/06/03 Saar +* +* Parameter: +* UNROLL_M 8 +* UNROLL_N 2 +* DGEMM_P 336 +* DGEMM_Q 168 +* NO_WARMUP 1 +* NO_AFFINITY 1 +* GEMM_MULTITHREAD_THRESHOLD 4 +* +* Performance at m x n with prefetch of BO: +* +* 8064x3840 93.7 GFLOPS with 8 threads on 4 modules (ACML: 93.6 GFLOPS) +* 6048x2880 85.1 GFLOPS with 4 threads on 4 modules (ACML: 84.2 GFLOPS) +* 6048x2880 52.0 GFLOPS with 2 threads on 2 modules (ACML: 50.0 GFLOPS) +* +* 6048x2880 56.3 GFLOPS with 4 threads on 2 modules (ACML: 57.6 GFLOPS) +* 4032x1920 29.5 GFLOPS with 2 threads on 1 modules (ACML: 30.5 GFLOPS) +* 4032x1920 26.9 GFLOPS with 1 threads on 1 modules (ACML: 26.1 GFLOPS) +* +*********************************************************************/ + +/********************************************************************* +* 2013/06/04 Saar +* +* Parameter: +* UNROLL_M 8 +* UNROLL_N 2 +* DGEMM_P 384 +* DGEMM_Q 168 +* NO_WARMUP 1 +* NO_AFFINITY 1 +* GEMM_MULTITHREAD_THRESHOLD 4 +* +* Performance at m x n with prefetch of BO: +* +* 6144x5376 94.6 GFLOPS with 8 threads on 4 modules (ACML: 90.5 GFLOPS) +* 6144x5376 86.0 GFLOPS with 4 threads on 4 modules (ACML: 81.5 GFLOPS) +* 4608x4032 52.0 GFLOPS with 2 threads on 2 modules (ACML: 47.5 GFLOPS) +* +* 6144x5376 57.3 GFLOPS with 4 threads on 2 modules (ACML: 56.5 GFLOPS) +* 4608x4032 29.6 GFLOPS with 2 threads on 1 modules (ACML: 30.2 GFLOPS) +* 4608x4032 26.9 GFLOPS with 1 threads on 1 modules (ACML: 25.6 GFLOPS) +* +*********************************************************************/ + + + +#define ASSEMBLER +#include "common.h" + +#define OLD_M %rdi +#define OLD_N %rsi +#define M %r13 +#define J %r14 +#define OLD_K %rdx + +#define A %rcx +#define B %r8 +#define C %r9 +#define LDC %r10 + +#define I %r11 +#define AO %rdi +#define BO %rsi +#define CO1 %r15 +#define K %r12 +#define BI %rbp +#define SP %rbx + +#define BO1 %rdi +#define BO2 %r15 + +#ifndef WINDOWS_ABI + +#define STACKSIZE 96 + +#else + +#define STACKSIZE 256 + +#define OLD_A 40 + STACKSIZE(%rsp) +#define OLD_B 48 + STACKSIZE(%rsp) +#define OLD_C 56 + STACKSIZE(%rsp) +#define OLD_LDC 64 + STACKSIZE(%rsp) +#define OLD_OFFSET 72 + STACKSIZE(%rsp) + +#endif + +#define L_BUFFER_SIZE 512*8*4 +#define LB2_OFFSET 512*8*2 + +#define Ndiv6 24(%rsp) +#define Nmod6 32(%rsp) +#define N 40(%rsp) +#define ALPHA 48(%rsp) +#define OFFSET 56(%rsp) +#define KK 64(%rsp) +#define KKK 72(%rsp) +#define BUFFER1 128(%rsp) +#define BUFFER2 LB2_OFFSET+128(%rsp) + + + +#define A_PR1 384 +#define B_PR1 192 + +#define KERNEL8x3_1(xx) \ + prefetcht0 A_PR1(AO,%rax,8) ;\ + vmovddup -6 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -16 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup -5 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovddup -4 * SIZE(BO, BI, 8), %xmm3 ;\ + vfmaddpd %xmm6,%xmm3,%xmm0,%xmm6 ;\ + vmovups -14 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + vfmaddpd %xmm8,%xmm2,%xmm0,%xmm8 ;\ + vfmaddpd %xmm9,%xmm3,%xmm0,%xmm9 ;\ + vmovups -12 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm10,%xmm1,%xmm0,%xmm10 ;\ + vfmaddpd %xmm11,%xmm2,%xmm0,%xmm11 ;\ + vfmaddpd %xmm12,%xmm3,%xmm0,%xmm12 ;\ + vmovups -10 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm13,%xmm1,%xmm0,%xmm13 ;\ + vfmaddpd %xmm14,%xmm2,%xmm0,%xmm14 ;\ + vfmaddpd %xmm15,%xmm3,%xmm0,%xmm15 ;\ + +#define KERNEL8x3_2(xx) \ + prefetcht0 A_PR1+64(AO,%rax,8) ;\ + vmovddup -3 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -8 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup -2 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovddup -1 * SIZE(BO, BI, 8), %xmm3 ;\ + vfmaddpd %xmm6,%xmm3,%xmm0,%xmm6 ;\ + vmovups -6 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + vfmaddpd %xmm8,%xmm2,%xmm0,%xmm8 ;\ + vfmaddpd %xmm9,%xmm3,%xmm0,%xmm9 ;\ + vmovups -4 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm10,%xmm1,%xmm0,%xmm10 ;\ + vfmaddpd %xmm11,%xmm2,%xmm0,%xmm11 ;\ + vfmaddpd %xmm12,%xmm3,%xmm0,%xmm12 ;\ + vmovups -2 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm13,%xmm1,%xmm0,%xmm13 ;\ + vfmaddpd %xmm14,%xmm2,%xmm0,%xmm14 ;\ + vfmaddpd %xmm15,%xmm3,%xmm0,%xmm15 ;\ + +#define KERNEL8x3_3(xx) \ + prefetcht0 A_PR1+128(AO,%rax,8) ;\ + vmovddup 0 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups 0 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup 1 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovddup 2 * SIZE(BO, BI, 8), %xmm3 ;\ + vfmaddpd %xmm6,%xmm3,%xmm0,%xmm6 ;\ + vmovups 2 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + vfmaddpd %xmm8,%xmm2,%xmm0,%xmm8 ;\ + vfmaddpd %xmm9,%xmm3,%xmm0,%xmm9 ;\ + vmovups 4 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm10,%xmm1,%xmm0,%xmm10 ;\ + vfmaddpd %xmm11,%xmm2,%xmm0,%xmm11 ;\ + vfmaddpd %xmm12,%xmm3,%xmm0,%xmm12 ;\ + vmovups 6 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm13,%xmm1,%xmm0,%xmm13 ;\ + vfmaddpd %xmm14,%xmm2,%xmm0,%xmm14 ;\ + vfmaddpd %xmm15,%xmm3,%xmm0,%xmm15 ;\ + +#define KERNEL8x3_4(xx) \ + prefetcht0 A_PR1+192(AO,%rax,8) ;\ + vmovddup 3 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups 8 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup 4 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovddup 5 * SIZE(BO, BI, 8), %xmm3 ;\ + vfmaddpd %xmm6,%xmm3,%xmm0,%xmm6 ;\ + vmovups 10 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + vfmaddpd %xmm8,%xmm2,%xmm0,%xmm8 ;\ + vfmaddpd %xmm9,%xmm3,%xmm0,%xmm9 ;\ + vmovups 12 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm10,%xmm1,%xmm0,%xmm10 ;\ + vfmaddpd %xmm11,%xmm2,%xmm0,%xmm11 ;\ + vfmaddpd %xmm12,%xmm3,%xmm0,%xmm12 ;\ + vmovups 14 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm13,%xmm1,%xmm0,%xmm13 ;\ + vfmaddpd %xmm14,%xmm2,%xmm0,%xmm14 ;\ + vfmaddpd %xmm15,%xmm3,%xmm0,%xmm15 ;\ + addq $12, BI ;\ + addq $32, %rax ;\ + +#define KERNEL8x3_SUB(xx) \ + vmovddup -6 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -16 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup -5 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovddup -4 * SIZE(BO, BI, 8), %xmm3 ;\ + vfmaddpd %xmm6,%xmm3,%xmm0,%xmm6 ;\ + vmovups -14 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + vfmaddpd %xmm8,%xmm2,%xmm0,%xmm8 ;\ + vfmaddpd %xmm9,%xmm3,%xmm0,%xmm9 ;\ + vmovups -12 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm10,%xmm1,%xmm0,%xmm10 ;\ + vfmaddpd %xmm11,%xmm2,%xmm0,%xmm11 ;\ + vfmaddpd %xmm12,%xmm3,%xmm0,%xmm12 ;\ + vmovups -10 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm13,%xmm1,%xmm0,%xmm13 ;\ + vfmaddpd %xmm14,%xmm2,%xmm0,%xmm14 ;\ + vfmaddpd %xmm15,%xmm3,%xmm0,%xmm15 ;\ + + +/*******************************************************************************************/ + +#define KERNEL4x3_1(xx) \ + vmovddup -6 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -16 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup -5 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovddup -4 * SIZE(BO, BI, 8), %xmm3 ;\ + vfmaddpd %xmm6,%xmm3,%xmm0,%xmm6 ;\ + vmovups -14 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + vfmaddpd %xmm8,%xmm2,%xmm0,%xmm8 ;\ + vfmaddpd %xmm9,%xmm3,%xmm0,%xmm9 ;\ + +#define KERNEL4x3_2(xx) \ + vmovddup -3 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -12 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup -2 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovddup -1 * SIZE(BO, BI, 8), %xmm3 ;\ + vfmaddpd %xmm6,%xmm3,%xmm0,%xmm6 ;\ + vmovups -10 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + vfmaddpd %xmm8,%xmm2,%xmm0,%xmm8 ;\ + vfmaddpd %xmm9,%xmm3,%xmm0,%xmm9 ;\ + +#define KERNEL4x3_3(xx) \ + vmovddup 0 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -8 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup 1 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovddup 2 * SIZE(BO, BI, 8), %xmm3 ;\ + vfmaddpd %xmm6,%xmm3,%xmm0,%xmm6 ;\ + vmovups -6 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + vfmaddpd %xmm8,%xmm2,%xmm0,%xmm8 ;\ + vfmaddpd %xmm9,%xmm3,%xmm0,%xmm9 ;\ + +#define KERNEL4x3_4(xx) \ + vmovddup 3 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -4 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup 4 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovddup 5 * SIZE(BO, BI, 8), %xmm3 ;\ + vfmaddpd %xmm6,%xmm3,%xmm0,%xmm6 ;\ + vmovups -2 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + vfmaddpd %xmm8,%xmm2,%xmm0,%xmm8 ;\ + vfmaddpd %xmm9,%xmm3,%xmm0,%xmm9 ;\ + addq $12, BI ;\ + addq $16, %rax ;\ + +#define KERNEL4x3_SUB(xx) \ + vmovddup -6 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -16 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup -5 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovddup -4 * SIZE(BO, BI, 8), %xmm3 ;\ + vfmaddpd %xmm6,%xmm3,%xmm0,%xmm6 ;\ + vmovups -14 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + vfmaddpd %xmm8,%xmm2,%xmm0,%xmm8 ;\ + vfmaddpd %xmm9,%xmm3,%xmm0,%xmm9 ;\ + + + + + +/*******************************************************************************************/ + +#define KERNEL2x3_1(xx) \ + vmovddup -6 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -16 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup -5 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovddup -4 * SIZE(BO, BI, 8), %xmm3 ;\ + vfmaddpd %xmm6,%xmm3,%xmm0,%xmm6 ;\ + +#define KERNEL2x3_2(xx) \ + vmovddup -3 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -14 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup -2 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovddup -1 * SIZE(BO, BI, 8), %xmm3 ;\ + vfmaddpd %xmm6,%xmm3,%xmm0,%xmm6 ;\ + +#define KERNEL2x3_3(xx) \ + vmovddup 0 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -12 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup 1 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovddup 2 * SIZE(BO, BI, 8), %xmm3 ;\ + vfmaddpd %xmm6,%xmm3,%xmm0,%xmm6 ;\ + +#define KERNEL2x3_4(xx) \ + vmovddup 3 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -10 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup 4 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovddup 5 * SIZE(BO, BI, 8), %xmm3 ;\ + vfmaddpd %xmm6,%xmm3,%xmm0,%xmm6 ;\ + addq $12, BI ;\ + addq $8, %rax ;\ + +#define KERNEL2x3_SUB(xx) \ + vmovddup -6 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -16 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup -5 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovddup -4 * SIZE(BO, BI, 8), %xmm3 ;\ + vfmaddpd %xmm6,%xmm3,%xmm0,%xmm6 ;\ + +/*******************************************************************************************/ + +#define KERNEL1x3_1(xx) \ + vmovsd -6 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovsd -16 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddsd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovsd -5 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddsd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovsd -4 * SIZE(BO, BI, 8), %xmm3 ;\ + vfmaddsd %xmm6,%xmm3,%xmm0,%xmm6 ;\ + +#define KERNEL1x3_2(xx) \ + vmovsd -3 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovsd -15 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddsd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovsd -2 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddsd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovsd -1 * SIZE(BO, BI, 8), %xmm3 ;\ + vfmaddsd %xmm6,%xmm3,%xmm0,%xmm6 ;\ + +#define KERNEL1x3_3(xx) \ + vmovsd 0 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovsd -14 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddsd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovsd 1 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddsd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovsd 2 * SIZE(BO, BI, 8), %xmm3 ;\ + vfmaddsd %xmm6,%xmm3,%xmm0,%xmm6 ;\ + +#define KERNEL1x3_4(xx) \ + vmovsd 3 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovsd -13 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddsd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovsd 4 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddsd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovsd 5 * SIZE(BO, BI, 8), %xmm3 ;\ + vfmaddsd %xmm6,%xmm3,%xmm0,%xmm6 ;\ + addq $12, BI ;\ + addq $4, %rax ;\ + +#define KERNEL1x3_SUB(xx) \ + vmovsd -6 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovsd -16 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddsd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovsd -5 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddsd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovsd -4 * SIZE(BO, BI, 8), %xmm3 ;\ + vfmaddsd %xmm6,%xmm3,%xmm0,%xmm6 ;\ + + + +/******************************************************************************************* +* 2 lines of N +*******************************************************************************************/ + +#define KERNEL8x2_1(xx) \ + prefetcht0 A_PR1(AO,%rax,8) ;\ + vmovddup -4 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -16 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup -3 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovups -14 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + vfmaddpd %xmm8,%xmm2,%xmm0,%xmm8 ;\ + vmovups -12 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm10,%xmm1,%xmm0,%xmm10 ;\ + vfmaddpd %xmm11,%xmm2,%xmm0,%xmm11 ;\ + vmovups -10 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm13,%xmm1,%xmm0,%xmm13 ;\ + vfmaddpd %xmm14,%xmm2,%xmm0,%xmm14 ;\ + +#define KERNEL8x2_2(xx) \ + prefetcht0 A_PR1+64(AO,%rax,8) ;\ + vmovddup -2 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -8 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup -1 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovups -6 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + vfmaddpd %xmm8,%xmm2,%xmm0,%xmm8 ;\ + vmovups -4 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm10,%xmm1,%xmm0,%xmm10 ;\ + vfmaddpd %xmm11,%xmm2,%xmm0,%xmm11 ;\ + vmovups -2 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm13,%xmm1,%xmm0,%xmm13 ;\ + vfmaddpd %xmm14,%xmm2,%xmm0,%xmm14 ;\ + +#define KERNEL8x2_3(xx) \ + prefetcht0 A_PR1+128(AO,%rax,8) ;\ + vmovddup 0 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups 0 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup 1 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovups 2 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + vfmaddpd %xmm8,%xmm2,%xmm0,%xmm8 ;\ + vmovups 4 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm10,%xmm1,%xmm0,%xmm10 ;\ + vfmaddpd %xmm11,%xmm2,%xmm0,%xmm11 ;\ + vmovups 6 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm13,%xmm1,%xmm0,%xmm13 ;\ + vfmaddpd %xmm14,%xmm2,%xmm0,%xmm14 ;\ + +#define KERNEL8x2_4(xx) \ + prefetcht0 A_PR1+192(AO,%rax,8) ;\ + vmovddup 2 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups 8 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup 3 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovups 10 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + vfmaddpd %xmm8,%xmm2,%xmm0,%xmm8 ;\ + vmovups 12 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm10,%xmm1,%xmm0,%xmm10 ;\ + vfmaddpd %xmm11,%xmm2,%xmm0,%xmm11 ;\ + vmovups 14 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm13,%xmm1,%xmm0,%xmm13 ;\ + vfmaddpd %xmm14,%xmm2,%xmm0,%xmm14 ;\ + addq $8, BI ;\ + addq $32, %rax ;\ + +#define KERNEL8x2_SUB(xx) \ + vmovddup -4 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -16 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup -3 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovups -14 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + vfmaddpd %xmm8,%xmm2,%xmm0,%xmm8 ;\ + vmovups -12 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm10,%xmm1,%xmm0,%xmm10 ;\ + vfmaddpd %xmm11,%xmm2,%xmm0,%xmm11 ;\ + vmovups -10 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm13,%xmm1,%xmm0,%xmm13 ;\ + vfmaddpd %xmm14,%xmm2,%xmm0,%xmm14 ;\ + + +/*******************************************************************************************/ + +#define KERNEL4x2_1(xx) \ + vmovddup -4 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -16 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup -3 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovups -14 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + vfmaddpd %xmm8,%xmm2,%xmm0,%xmm8 ;\ + +#define KERNEL4x2_2(xx) \ + vmovddup -2 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -12 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup -1 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovups -10 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + vfmaddpd %xmm8,%xmm2,%xmm0,%xmm8 ;\ + +#define KERNEL4x2_3(xx) \ + vmovddup 0 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -8 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup 1 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovups -6 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + vfmaddpd %xmm8,%xmm2,%xmm0,%xmm8 ;\ + +#define KERNEL4x2_4(xx) \ + vmovddup 2 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -4 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup 3 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovups -2 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + vfmaddpd %xmm8,%xmm2,%xmm0,%xmm8 ;\ + addq $8, BI ;\ + addq $16, %rax ;\ + +#define KERNEL4x2_SUB(xx) \ + vmovddup -4 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -16 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup -3 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + vmovups -14 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + vfmaddpd %xmm8,%xmm2,%xmm0,%xmm8 ;\ + + +/*******************************************************************************************/ + +#define KERNEL2x2_1(xx) \ + vmovddup -4 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -16 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup -3 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + +#define KERNEL2x2_2(xx) \ + vmovddup -2 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -14 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup -1 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + +#define KERNEL2x2_3(xx) \ + vmovddup 0 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -12 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup 1 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + +#define KERNEL2x2_4(xx) \ + vmovddup 2 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -10 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup 3 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + addq $8, BI ;\ + addq $8, %rax ;\ + +#define KERNEL2x2_SUB(xx) \ + vmovddup -4 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -16 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovddup -3 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddpd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + +/*******************************************************************************************/ + +#define KERNEL1x2_1(xx) \ + vmovsd -4 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovsd -16 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddsd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovsd -3 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddsd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + +#define KERNEL1x2_2(xx) \ + vmovsd -2 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovsd -15 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddsd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovsd -1 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddsd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + +#define KERNEL1x2_3(xx) \ + vmovsd 0 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovsd -14 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddsd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovsd 1 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddsd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + +#define KERNEL1x2_4(xx) \ + vmovsd 2 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovsd -13 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddsd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovsd 3 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddsd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + addq $8, BI ;\ + addq $4, %rax ;\ + +#define KERNEL1x2_SUB(xx) \ + vmovsd -4 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovsd -16 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddsd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovsd -3 * SIZE(BO, BI, 8), %xmm2 ;\ + vfmaddsd %xmm5,%xmm2,%xmm0,%xmm5 ;\ + + + +/******************************************************************************************* +* 1 line of N +*******************************************************************************************/ + +#define KERNEL8x1_1(xx) \ + prefetcht0 A_PR1(AO,%rax,8) ;\ + vmovddup -2 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -16 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovups -14 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + vmovups -12 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm10,%xmm1,%xmm0,%xmm10 ;\ + vmovups -10 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm13,%xmm1,%xmm0,%xmm13 ;\ + +#define KERNEL8x1_2(xx) \ + prefetcht0 A_PR1+64(AO,%rax,8) ;\ + vmovddup -1 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -8 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovups -6 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + vmovups -4 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm10,%xmm1,%xmm0,%xmm10 ;\ + vmovups -2 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm13,%xmm1,%xmm0,%xmm13 ;\ + +#define KERNEL8x1_3(xx) \ + prefetcht0 A_PR1+128(AO,%rax,8) ;\ + vmovddup 0 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups 0 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovups 2 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + vmovups 4 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm10,%xmm1,%xmm0,%xmm10 ;\ + vmovups 6 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm13,%xmm1,%xmm0,%xmm13 ;\ + +#define KERNEL8x1_4(xx) \ + prefetcht0 A_PR1+192(AO,%rax,8) ;\ + vmovddup 1 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups 8 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovups 10 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + vmovups 12 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm10,%xmm1,%xmm0,%xmm10 ;\ + vmovups 14 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm13,%xmm1,%xmm0,%xmm13 ;\ + addq $4, BI ;\ + addq $32, %rax ;\ + +#define KERNEL8x1_SUB(xx) \ + vmovddup -2 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -16 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovups -14 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + vmovups -12 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm10,%xmm1,%xmm0,%xmm10 ;\ + vmovups -10 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm13,%xmm1,%xmm0,%xmm13 ;\ + + +/*******************************************************************************************/ + +#define KERNEL4x1_1(xx) \ + vmovddup -2 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -16 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovups -14 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + +#define KERNEL4x1_2(xx) \ + vmovddup -1 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -12 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovups -10 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + +#define KERNEL4x1_3(xx) \ + vmovddup 0 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -8 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovups -6 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + +#define KERNEL4x1_4(xx) \ + vmovddup 1 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -4 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovups -2 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + addq $4, BI ;\ + addq $16, %rax ;\ + +#define KERNEL4x1_SUB(xx) \ + vmovddup -2 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -16 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + vmovups -14 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm7,%xmm1,%xmm0,%xmm7 ;\ + + +/*******************************************************************************************/ + +#define KERNEL2x1_1(xx) \ + vmovddup -2 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -16 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + +#define KERNEL2x1_2(xx) \ + vmovddup -1 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -14 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + +#define KERNEL2x1_3(xx) \ + vmovddup 0 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -12 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + +#define KERNEL2x1_4(xx) \ + vmovddup 1 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -10 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + addq $4, BI ;\ + addq $8, %rax ;\ + +#define KERNEL2x1_SUB(xx) \ + vmovddup -2 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovups -16 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddpd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + +/*******************************************************************************************/ + +#define KERNEL1x1_1(xx) \ + vmovsd -2 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovsd -16 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddsd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + +#define KERNEL1x1_2(xx) \ + vmovsd -1 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovsd -15 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddsd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + +#define KERNEL1x1_3(xx) \ + vmovsd 0 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovsd -14 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddsd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + +#define KERNEL1x1_4(xx) \ + vmovsd 1 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovsd -13 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddsd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + addq $4, BI ;\ + addq $4, %rax ;\ + +#define KERNEL1x1_SUB(xx) \ + vmovsd -2 * SIZE(BO, BI, 8), %xmm1 ;\ + vmovsd -16 * SIZE(AO, %rax, 8), %xmm0 ;\ + vfmaddsd %xmm4,%xmm1,%xmm0,%xmm4 ;\ + + +/*******************************************************************************************/ + + + + + + PROLOGUE + PROFCODE + + subq $STACKSIZE, %rsp + movq %rbx, (%rsp) + movq %rbp, 8(%rsp) + movq %r12, 16(%rsp) + movq %r13, 24(%rsp) + movq %r14, 32(%rsp) + movq %r15, 40(%rsp) + + vzeroupper + +#ifdef WINDOWS_ABI + movq %rdi, 48(%rsp) + movq %rsi, 56(%rsp) + movups %xmm6, 64(%rsp) + movups %xmm7, 80(%rsp) + movups %xmm8, 96(%rsp) + movups %xmm9, 112(%rsp) + movups %xmm10, 128(%rsp) + movups %xmm11, 144(%rsp) + movups %xmm12, 160(%rsp) + movups %xmm13, 176(%rsp) + movups %xmm14, 192(%rsp) + movups %xmm15, 208(%rsp) + + movq ARG1, OLD_M + movq ARG2, OLD_N + movq ARG3, OLD_K + movq OLD_A, A + movq OLD_B, B + movq OLD_C, C + movq OLD_LDC, LDC +#ifdef TRMMKERNEL + movsd OLD_OFFSET, %xmm12 +#endif + vmovaps %xmm3, %xmm0 + +#else + movq STACKSIZE + 8(%rsp), LDC +#ifdef TRMMKERNEL + movsd STACKSIZE + 16(%rsp), %xmm12 +#endif + +#endif + + movq %rsp, SP # save old stack + subq $128 + L_BUFFER_SIZE, %rsp + andq $-4096, %rsp # align stack + + STACK_TOUCHING + + cmpq $0, OLD_M + je .L999 + + cmpq $0, OLD_N + je .L999 + + cmpq $0, OLD_K + je .L999 + + movq OLD_M, M + movq OLD_N, N + movq OLD_K, K + + vmovsd %xmm0, ALPHA + + salq $BASE_SHIFT, LDC + + movq N, %rax + xorq %rdx, %rdx + movq $6, %rdi + divq %rdi // N / 6 + movq %rax, Ndiv6 // N / 6 + movq %rdx, Nmod6 // N % 6 + + + +#ifdef TRMMKERNEL + vmovsd %xmm12, OFFSET + vmovsd %xmm12, KK +#ifndef LEFT + negq KK +#endif +#endif + + movq Ndiv6, J + cmpq $0, J + je .L2_0 + ALIGN_4 + +.L6_01: + // copy to sub buffer + movq K, %rax + salq $1,%rax // K * 2 + movq B, BO1 + leaq (B,%rax,8), BO2 // next offset to BO2 + leaq BUFFER1, BO // first buffer to BO + movq K, %rax + sarq $2, %rax // K / 4 + jz .L6_02a + ALIGN_4 + +.L6_02: + prefetcht0 512(BO1) + prefetcht0 512(BO2) + prefetchw 512(BO) + vmovups (BO1), %xmm0 + vmovups 2*SIZE(BO1), %xmm2 + vmovups 4*SIZE(BO1), %xmm4 + vmovups 6*SIZE(BO1), %xmm6 + vmovsd (BO2), %xmm1 + vmovsd 2*SIZE(BO2), %xmm3 + vmovsd 4*SIZE(BO2), %xmm5 + vmovsd 6*SIZE(BO2), %xmm7 + vmovups %xmm0, (BO) + vmovsd %xmm1, 2*SIZE(BO) + vmovups %xmm2, 3*SIZE(BO) + vmovsd %xmm3, 5*SIZE(BO) + vmovups %xmm4, 6*SIZE(BO) + vmovsd %xmm5, 8*SIZE(BO) + vmovups %xmm6, 9*SIZE(BO) + vmovsd %xmm7,11*SIZE(BO) + addq $8*SIZE,BO1 + addq $8*SIZE,BO2 + addq $12*SIZE,BO + decq %rax + jnz .L6_02 + +.L6_02a: + + movq K, %rax + andq $3, %rax // K % 4 + jz .L6_02c + ALIGN_4 + +.L6_02b: + + vmovups (BO1), %xmm0 + vmovsd (BO2), %xmm1 + vmovups %xmm0, (BO) + vmovsd %xmm1, 2*SIZE(BO) + addq $2*SIZE,BO1 + addq $2*SIZE,BO2 + addq $3*SIZE,BO + decq %rax + jnz .L6_02b + +.L6_02c: + + movq K, %rax + salq $1,%rax // K * 2 + leaq (B,%rax,8), BO1 // next offset to BO1 + leaq (BO1,%rax,8), BO2 // next offset to BO1 + leaq BUFFER2, BO // second buffer to BO + movq K, %rax + sarq $2, %rax // k / 4 + jz .L6_03a + ALIGN_4 + + +.L6_03: + + prefetcht0 512(BO2) + prefetchw 512(BO) + vmovups (BO2), %xmm0 + vmovups 2*SIZE(BO2), %xmm2 + vmovups 4*SIZE(BO2), %xmm4 + vmovups 6*SIZE(BO2), %xmm6 + vmovsd 1*SIZE(BO1), %xmm1 + vmovsd 3*SIZE(BO1), %xmm3 + vmovsd 5*SIZE(BO1), %xmm5 + vmovsd 7*SIZE(BO1), %xmm7 + vmovsd %xmm1, 0*SIZE(BO) + vmovups %xmm0, 1*SIZE(BO) + vmovsd %xmm3, 3*SIZE(BO) + vmovups %xmm2, 4*SIZE(BO) + vmovsd %xmm5, 6*SIZE(BO) + vmovups %xmm4, 7*SIZE(BO) + vmovsd %xmm7, 9*SIZE(BO) + vmovups %xmm6,10*SIZE(BO) + addq $8*SIZE,BO1 + addq $8*SIZE,BO2 + addq $12*SIZE,BO + decq %rax + jnz .L6_03 + +.L6_03a: + + movq K, %rax + andq $3, %rax // K % 4 + jz .L6_03c + ALIGN_4 + + +.L6_03b: + + vmovsd 1*SIZE(BO1), %xmm0 + vmovups (BO2), %xmm1 + vmovsd %xmm0, (BO) + vmovups %xmm1, 1*SIZE(BO) + addq $2*SIZE,BO1 + addq $2*SIZE,BO2 + addq $3*SIZE,BO + decq %rax + jnz .L6_03b + + +.L6_03c: + + movq BO2, B // next offset of B + +.L6_10: + movq C, CO1 + leaq (C, LDC, 2), C + leaq (C, LDC, 1), C // c += 3 * ldc + +#if defined(TRMMKERNEL) && defined(LEFT) + movq OFFSET, %rax + movq %rax, KK +#endif + + movq A, AO // aoffset = a + addq $16 * SIZE, AO + + movq M, I + sarq $3, I // i = (m >> 3) + je .L6_20 + + ALIGN_4 + +.L6_11: +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + leaq BUFFER1, BO // first buffer to BO + addq $6 * SIZE, BO +#else + movq KK, %rax + leaq BUFFER1, BO // first buffer to BO + addq $6 * SIZE, BO + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + leaq (BO, BI, 8), BO + salq $3, %rax // rax = rax * 8 ; number of values + leaq (AO, %rax, 8), AO +#endif + + + vzeroall + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $8, %rax // number of values in AO +#else + addq $3, %rax // number of values in BO +#endif + movq %rax, KKK +#endif + + andq $-8, %rax // K = K - ( K % 8 ) + je .L6_16 + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + + salq $3, %rax // rax = rax * 8 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L6_12: + + prefetcht0 B_PR1(BO,BI,8) + KERNEL8x3_1(xxx) + KERNEL8x3_2(xxx) + KERNEL8x3_3(xxx) + prefetcht0 B_PR1+64(BO,BI,8) + KERNEL8x3_4(xxx) + + KERNEL8x3_1(xxx) + KERNEL8x3_2(xxx) + prefetcht0 B_PR1+32(BO,BI,8) + KERNEL8x3_3(xxx) + KERNEL8x3_4(xxx) + + je .L6_16 + + prefetcht0 B_PR1(BO,BI,8) + KERNEL8x3_1(xxx) + KERNEL8x3_2(xxx) + KERNEL8x3_3(xxx) + prefetcht0 B_PR1+64(BO,BI,8) + KERNEL8x3_4(xxx) + + KERNEL8x3_1(xxx) + KERNEL8x3_2(xxx) + prefetcht0 B_PR1+32(BO,BI,8) + KERNEL8x3_3(xxx) + KERNEL8x3_4(xxx) + + je .L6_16 + + jmp .L6_12 + ALIGN_4 + +.L6_16: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + + andq $7, %rax # if (k & 1) + je .L6_19 + + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + + salq $3, %rax // rax = rax * 8 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L6_17: + + KERNEL8x3_SUB(xxx) + addq $3, BI + addq $8, %rax + jl .L6_17 + ALIGN_4 + + +.L6_19: + + vmovddup ALPHA, %xmm0 + +#ifndef TRMMKERNEL + + vfmaddpd (CO1),%xmm0, %xmm4,%xmm4 + vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7 + vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10 + vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13 + + vfmaddpd (CO1, LDC),%xmm0, %xmm5,%xmm5 + vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8 + vfmaddpd 4 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11 + vfmaddpd 6 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14 + + vfmaddpd (CO1, LDC, 2),%xmm0, %xmm6,%xmm6 + vfmaddpd 2 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9 + vfmaddpd 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12 + vfmaddpd 6 * SIZE(CO1, LDC, 2),%xmm0, %xmm15,%xmm15 + +#else + vmulpd %xmm0, %xmm4,%xmm4 + vmulpd %xmm0, %xmm7,%xmm7 + vmulpd %xmm0, %xmm10,%xmm10 + vmulpd %xmm0, %xmm13,%xmm13 + + vmulpd %xmm0, %xmm5,%xmm5 + vmulpd %xmm0, %xmm8,%xmm8 + vmulpd %xmm0, %xmm11,%xmm11 + vmulpd %xmm0, %xmm14,%xmm14 + + vmulpd %xmm0, %xmm6,%xmm6 + vmulpd %xmm0, %xmm9,%xmm9 + vmulpd %xmm0, %xmm12,%xmm12 + vmulpd %xmm0, %xmm15,%xmm15 + +#endif + + vmovups %xmm4 , (CO1) + vmovups %xmm7 , 2 * SIZE(CO1) + vmovups %xmm10, 4 * SIZE(CO1) + vmovups %xmm13, 6 * SIZE(CO1) + + vmovups %xmm5 , (CO1, LDC) + vmovups %xmm8 , 2 * SIZE(CO1, LDC) + vmovups %xmm11, 4 * SIZE(CO1, LDC) + vmovups %xmm14, 6 * SIZE(CO1, LDC) + + vmovups %xmm6 , (CO1, LDC, 2) + vmovups %xmm9 , 2 * SIZE(CO1, LDC, 2) + vmovups %xmm12, 4 * SIZE(CO1, LDC, 2) + vmovups %xmm15, 6 * SIZE(CO1, LDC, 2) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + leaq (BO, BI, 8), BO + salq $3, %rax // rax = rax * 8 ; number of values + leaq (AO, %rax, 8), AO +#endif + + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $8, KK +#endif + + addq $8 * SIZE, CO1 # coffset += 8 + decq I # i -- + jg .L6_11 + ALIGN_4 + +/************************************************************************** +* Rest of M +***************************************************************************/ +.L6_20: + // Test rest of M + + testq $7, M + jz .L7_10 // to next 3 lines of N + + testq $4, M + jz .L6_30 + + ALIGN_4 + +.L6_21: +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + leaq BUFFER1, BO // first buffer to BO + addq $6 * SIZE, BO +#else + movq KK, %rax + leaq BUFFER1, BO // first buffer to BO + addq $6 * SIZE, BO + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + leaq (BO, BI, 8), BO + salq $2, %rax // rax = rax * 4 ; number of values + leaq (AO, %rax, 8), AO +#endif + + + vzeroall + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $4, %rax // number of values in A +#else + addq $3, %rax // number of values in BO +#endif + movq %rax, KKK +#endif + + + andq $-8, %rax + je .L6_26 + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + + salq $2, %rax // rax = rax * 4 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L6_22: + + prefetcht0 B_PR1(BO,BI,8) + KERNEL4x3_1(xxx) + KERNEL4x3_2(xxx) + KERNEL4x3_3(xxx) + prefetcht0 B_PR1+64(BO,BI,8) + KERNEL4x3_4(xxx) + + KERNEL4x3_1(xxx) + KERNEL4x3_2(xxx) + prefetcht0 B_PR1+32(BO,BI,8) + KERNEL4x3_3(xxx) + KERNEL4x3_4(xxx) + + je .L6_26 + + prefetcht0 B_PR1(BO,BI,8) + KERNEL4x3_1(xxx) + KERNEL4x3_2(xxx) + KERNEL4x3_3(xxx) + prefetcht0 B_PR1+64(BO,BI,8) + KERNEL4x3_4(xxx) + + KERNEL4x3_1(xxx) + KERNEL4x3_2(xxx) + prefetcht0 B_PR1+32(BO,BI,8) + KERNEL4x3_3(xxx) + KERNEL4x3_4(xxx) + + je .L6_26 + + jmp .L6_22 + ALIGN_4 + +.L6_26: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + + andq $7, %rax # if (k & 1) + je .L6_29 + + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + + salq $2, %rax // rax = rax * 4 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L6_27: + + KERNEL4x3_SUB(xxx) + addq $3, BI + addq $4, %rax + jl .L6_27 + ALIGN_4 + + +.L6_29: + + vmovddup ALPHA, %xmm0 + +#ifndef TRMMKERNEL + + vfmaddpd (CO1),%xmm0, %xmm4,%xmm4 + vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7 + + vfmaddpd (CO1, LDC),%xmm0, %xmm5,%xmm5 + vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8 + + vfmaddpd (CO1, LDC, 2),%xmm0, %xmm6,%xmm6 + vfmaddpd 2 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9 + +#else + vmulpd %xmm0, %xmm4,%xmm4 + vmulpd %xmm0, %xmm7,%xmm7 + + vmulpd %xmm0, %xmm5,%xmm5 + vmulpd %xmm0, %xmm8,%xmm8 + + vmulpd %xmm0, %xmm6,%xmm6 + vmulpd %xmm0, %xmm9,%xmm9 + +#endif + + vmovups %xmm4 , (CO1) + vmovups %xmm7 , 2 * SIZE(CO1) + + vmovups %xmm5 , (CO1, LDC) + vmovups %xmm8 , 2 * SIZE(CO1, LDC) + + vmovups %xmm6 , (CO1, LDC, 2) + vmovups %xmm9 , 2 * SIZE(CO1, LDC, 2) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + leaq (BO, BI, 8), BO + salq $2, %rax // rax = rax * 4 ; number of values + leaq (AO, %rax, 8), AO +#endif + + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $4, KK +#endif + + addq $4 * SIZE, CO1 # coffset += 4 + ALIGN_4 + + +.L6_30: + testq $2, M + jz .L6_40 + + ALIGN_4 + +.L6_31: +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + leaq BUFFER1, BO // first buffer to BO + addq $6 * SIZE, BO +#else + movq KK, %rax + leaq BUFFER1, BO // first buffer to BO + addq $6 * SIZE, BO + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + leaq (BO, BI, 8), BO + salq $1, %rax // rax = rax * 2 ; number of values + leaq (AO, %rax, 8), AO +#endif + + + vzeroall + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $2, %rax // number of values in AO +#else + addq $3, %rax // number of values in BO +#endif + movq %rax, KKK +#endif + + + andq $-8, %rax + je .L6_36 + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + + salq $1, %rax // rax = rax *2 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L6_32: + + prefetcht0 B_PR1(BO,BI,8) + KERNEL2x3_1(xxx) + KERNEL2x3_2(xxx) + KERNEL2x3_3(xxx) + prefetcht0 B_PR1+64(BO,BI,8) + KERNEL2x3_4(xxx) + + KERNEL2x3_1(xxx) + KERNEL2x3_2(xxx) + prefetcht0 B_PR1+32(BO,BI,8) + KERNEL2x3_3(xxx) + KERNEL2x3_4(xxx) + + je .L6_36 + + prefetcht0 B_PR1(BO,BI,8) + KERNEL2x3_1(xxx) + KERNEL2x3_2(xxx) + KERNEL2x3_3(xxx) + prefetcht0 B_PR1+64(BO,BI,8) + KERNEL2x3_4(xxx) + + KERNEL2x3_1(xxx) + KERNEL2x3_2(xxx) + prefetcht0 B_PR1+32(BO,BI,8) + KERNEL2x3_3(xxx) + KERNEL2x3_4(xxx) + + je .L6_36 + + jmp .L6_32 + ALIGN_4 + +.L6_36: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + + andq $7, %rax # if (k & 1) + je .L6_39 + + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + + salq $1, %rax // rax = rax *2 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L6_37: + + KERNEL2x3_SUB(xxx) + addq $3, BI + addq $2, %rax + jl .L6_37 + ALIGN_4 + + +.L6_39: + + vmovddup ALPHA, %xmm0 + +#ifndef TRMMKERNEL + + vfmaddpd (CO1),%xmm0, %xmm4,%xmm4 + vfmaddpd (CO1, LDC),%xmm0, %xmm5,%xmm5 + vfmaddpd (CO1, LDC, 2),%xmm0, %xmm6,%xmm6 + +#else + vmulpd %xmm0, %xmm4,%xmm4 + vmulpd %xmm0, %xmm5,%xmm5 + vmulpd %xmm0, %xmm6,%xmm6 + +#endif + + vmovups %xmm4 , (CO1) + vmovups %xmm5 , (CO1, LDC) + vmovups %xmm6 , (CO1, LDC, 2) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + leaq (BO, BI, 8), BO + salq $1, %rax // rax = rax * 2 ; number of values + leaq (AO, %rax, 8), AO +#endif + + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $2, KK +#endif + + addq $2 * SIZE, CO1 # coffset += 2 + ALIGN_4 + +.L6_40: + testq $1, M + jz .L7_10 // to next 3 lines of N + + ALIGN_4 + +.L6_41: +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + leaq BUFFER1, BO // first buffer to BO + addq $6 * SIZE, BO +#else + movq KK, %rax + leaq BUFFER1, BO // first buffer to BO + addq $6 * SIZE, BO + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + leaq (BO, BI, 8), BO + leaq (AO, %rax, 8), AO +#endif + + + vzeroall + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $1, %rax // number of values in AO +#else + addq $3, %rax // number of values in BO +#endif + movq %rax, KKK +#endif + + andq $-8, %rax + je .L6_46 + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L6_42: + + prefetcht0 B_PR1(BO,BI,8) + KERNEL1x3_1(xxx) + KERNEL1x3_2(xxx) + KERNEL1x3_3(xxx) + prefetcht0 B_PR1+64(BO,BI,8) + KERNEL1x3_4(xxx) + + KERNEL1x3_1(xxx) + KERNEL1x3_2(xxx) + prefetcht0 B_PR1+32(BO,BI,8) + KERNEL1x3_3(xxx) + KERNEL1x3_4(xxx) + + je .L6_46 + + prefetcht0 B_PR1(BO,BI,8) + KERNEL1x3_1(xxx) + KERNEL1x3_2(xxx) + KERNEL1x3_3(xxx) + prefetcht0 B_PR1+64(BO,BI,8) + KERNEL1x3_4(xxx) + + KERNEL1x3_1(xxx) + KERNEL1x3_2(xxx) + prefetcht0 B_PR1+32(BO,BI,8) + KERNEL1x3_3(xxx) + KERNEL1x3_4(xxx) + + je .L6_46 + + jmp .L6_42 + ALIGN_4 + +.L6_46: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + + andq $7, %rax # if (k & 1) + je .L6_49 + + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L6_47: + + KERNEL1x3_SUB(xxx) + addq $3, BI + addq $1, %rax + jl .L6_47 + ALIGN_4 + + +.L6_49: + + vmovddup ALPHA, %xmm0 + +#ifndef TRMMKERNEL + + vfmaddsd (CO1),%xmm0, %xmm4,%xmm4 + vfmaddsd (CO1, LDC),%xmm0, %xmm5,%xmm5 + vfmaddsd (CO1, LDC, 2),%xmm0, %xmm6,%xmm6 + +#else + vmulsd %xmm0, %xmm4,%xmm4 + vmulsd %xmm0, %xmm5,%xmm5 + vmulsd %xmm0, %xmm6,%xmm6 + +#endif + + vmovsd %xmm4 , (CO1) + vmovsd %xmm5 , (CO1, LDC) + vmovsd %xmm6 , (CO1, LDC, 2) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + leaq (BO, BI, 8), BO + leaq (AO, %rax, 8), AO +#endif + + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $1, KK +#endif + + addq $1 * SIZE, CO1 # coffset += 1 + ALIGN_4 + + + + +/***************************************************************************************************************/ + +.L7_10: + movq C, CO1 + leaq (C, LDC, 2), C + leaq (C, LDC, 1), C // c += 3 * ldc + +#if defined(TRMMKERNEL) && defined(LEFT) + movq OFFSET, %rax + movq %rax, KK +#endif + + movq A, AO // aoffset = a + addq $16 * SIZE, AO + + movq M, I + sarq $3, I // i = (m >> 3) + je .L7_20 + ALIGN_4 + +.L7_11: +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + leaq BUFFER2, BO // second buffer to BO + addq $6 * SIZE, BO +#else + movq KK, %rax + leaq BUFFER2, BO // second buffer to BO + addq $6 * SIZE, BO + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + leaq (BO, BI, 8), BO + salq $3, %rax // rax = rax * 8 ; number of values + leaq (AO, %rax, 8), AO +#endif + + + vzeroall + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $8, %rax // number of values in AO +#else + addq $3, %rax // number of values in BO +#endif + movq %rax, KKK +#endif + + + andq $-8, %rax + je .L7_16 + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + + salq $3, %rax // rax = rax * 8 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + + ALIGN_4 + +.L7_12: + + prefetcht0 B_PR1(BO,BI,8) + KERNEL8x3_1(xxx) + KERNEL8x3_2(xxx) + KERNEL8x3_3(xxx) + prefetcht0 B_PR1+64(BO,BI,8) + KERNEL8x3_4(xxx) + + KERNEL8x3_1(xxx) + KERNEL8x3_2(xxx) + prefetcht0 B_PR1+32(BO,BI,8) + KERNEL8x3_3(xxx) + KERNEL8x3_4(xxx) + + je .L7_16 + + prefetcht0 B_PR1(BO,BI,8) + KERNEL8x3_1(xxx) + KERNEL8x3_2(xxx) + KERNEL8x3_3(xxx) + prefetcht0 B_PR1+64(BO,BI,8) + KERNEL8x3_4(xxx) + + KERNEL8x3_1(xxx) + KERNEL8x3_2(xxx) + prefetcht0 B_PR1+32(BO,BI,8) + KERNEL8x3_3(xxx) + KERNEL8x3_4(xxx) + + je .L7_16 + + jmp .L7_12 + ALIGN_4 + +.L7_16: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + + andq $7, %rax # if (k & 1) + je .L7_19 + + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + + salq $3, %rax // rax = rax * 8 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L7_17: + + KERNEL8x3_SUB(xxx) + addq $3, BI + addq $8, %rax + jl .L7_17 + ALIGN_4 + + +.L7_19: + + vmovddup ALPHA, %xmm0 + +#ifndef TRMMKERNEL + + vfmaddpd (CO1),%xmm0, %xmm4,%xmm4 + vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7 + vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10 + vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13 + + vfmaddpd (CO1, LDC),%xmm0, %xmm5,%xmm5 + vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8 + vfmaddpd 4 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11 + vfmaddpd 6 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14 + + vfmaddpd (CO1, LDC, 2),%xmm0, %xmm6,%xmm6 + vfmaddpd 2 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9 + vfmaddpd 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12 + vfmaddpd 6 * SIZE(CO1, LDC, 2),%xmm0, %xmm15,%xmm15 + +#else + vmulpd %xmm0, %xmm4,%xmm4 + vmulpd %xmm0, %xmm7,%xmm7 + vmulpd %xmm0, %xmm10,%xmm10 + vmulpd %xmm0, %xmm13,%xmm13 + + vmulpd %xmm0, %xmm5,%xmm5 + vmulpd %xmm0, %xmm8,%xmm8 + vmulpd %xmm0, %xmm11,%xmm11 + vmulpd %xmm0, %xmm14,%xmm14 + + vmulpd %xmm0, %xmm6,%xmm6 + vmulpd %xmm0, %xmm9,%xmm9 + vmulpd %xmm0, %xmm12,%xmm12 + vmulpd %xmm0, %xmm15,%xmm15 + +#endif + + vmovups %xmm4 , (CO1) + vmovups %xmm7 , 2 * SIZE(CO1) + vmovups %xmm10, 4 * SIZE(CO1) + vmovups %xmm13, 6 * SIZE(CO1) + + vmovups %xmm5 , (CO1, LDC) + vmovups %xmm8 , 2 * SIZE(CO1, LDC) + vmovups %xmm11, 4 * SIZE(CO1, LDC) + vmovups %xmm14, 6 * SIZE(CO1, LDC) + + vmovups %xmm6 , (CO1, LDC, 2) + vmovups %xmm9 , 2 * SIZE(CO1, LDC, 2) + vmovups %xmm12, 4 * SIZE(CO1, LDC, 2) + vmovups %xmm15, 6 * SIZE(CO1, LDC, 2) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + leaq (BO, BI, 8), BO + salq $3, %rax // rax = rax * 8 ; number of values + leaq (AO, %rax, 8), AO +#endif + + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $8, KK +#endif + + addq $8 * SIZE, CO1 # coffset += 8 + decq I # i -- + jg .L7_11 + ALIGN_4 + +.L7_20: + // Test rest of M + + testq $7, M + jz .L7_60 // to next 6 lines of N + + testq $4, M + jz .L7_30 + + ALIGN_4 + +.L7_21: +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + leaq BUFFER2, BO // second buffer to BO + addq $6 * SIZE, BO +#else + movq KK, %rax + leaq BUFFER2, BO // second buffer to BO + addq $6 * SIZE, BO + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + leaq (BO, BI, 8), BO + salq $2, %rax // rax = rax * 4 ; number of values + leaq (AO, %rax, 8), AO +#endif + + + vzeroall + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $4, %rax // number of values in AO +#else + addq $3, %rax // number of values in BO +#endif + movq %rax, KKK +#endif + + + andq $-8, %rax + je .L7_26 + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + + salq $2, %rax // rax = rax * 4 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L7_22: + + prefetcht0 B_PR1(BO,BI,8) + KERNEL4x3_1(xxx) + KERNEL4x3_2(xxx) + KERNEL4x3_3(xxx) + prefetcht0 B_PR1+64(BO,BI,8) + KERNEL4x3_4(xxx) + + KERNEL4x3_1(xxx) + KERNEL4x3_2(xxx) + prefetcht0 B_PR1+32(BO,BI,8) + KERNEL4x3_3(xxx) + KERNEL4x3_4(xxx) + + je .L7_26 + + prefetcht0 B_PR1(BO,BI,8) + KERNEL4x3_1(xxx) + KERNEL4x3_2(xxx) + KERNEL4x3_3(xxx) + prefetcht0 B_PR1+64(BO,BI,8) + KERNEL4x3_4(xxx) + + KERNEL4x3_1(xxx) + KERNEL4x3_2(xxx) + prefetcht0 B_PR1+32(BO,BI,8) + KERNEL4x3_3(xxx) + KERNEL4x3_4(xxx) + + je .L7_26 + + jmp .L7_22 + ALIGN_4 + +.L7_26: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + + andq $7, %rax # if (k & 1) + je .L7_29 + + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + + salq $2, %rax // rax = rax * 4 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L7_27: + + KERNEL4x3_SUB(xxx) + addq $3, BI + addq $4, %rax + jl .L7_27 + ALIGN_4 + + +.L7_29: + + vmovddup ALPHA, %xmm0 + +#ifndef TRMMKERNEL + + vfmaddpd (CO1),%xmm0, %xmm4,%xmm4 + vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7 + + vfmaddpd (CO1, LDC),%xmm0, %xmm5,%xmm5 + vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8 + + vfmaddpd (CO1, LDC, 2),%xmm0, %xmm6,%xmm6 + vfmaddpd 2 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9 + +#else + vmulpd %xmm0, %xmm4,%xmm4 + vmulpd %xmm0, %xmm7,%xmm7 + + vmulpd %xmm0, %xmm5,%xmm5 + vmulpd %xmm0, %xmm8,%xmm8 + + vmulpd %xmm0, %xmm6,%xmm6 + vmulpd %xmm0, %xmm9,%xmm9 + +#endif + + vmovups %xmm4 , (CO1) + vmovups %xmm7 , 2 * SIZE(CO1) + + vmovups %xmm5 , (CO1, LDC) + vmovups %xmm8 , 2 * SIZE(CO1, LDC) + + vmovups %xmm6 , (CO1, LDC, 2) + vmovups %xmm9 , 2 * SIZE(CO1, LDC, 2) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + leaq (BO, BI, 8), BO + salq $2, %rax // rax = rax * 4 ; number of values + leaq (AO, %rax, 8), AO +#endif + + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $4, KK +#endif + + addq $4 * SIZE, CO1 # coffset += 4 + ALIGN_4 + + +.L7_30: + testq $2, M + jz .L7_40 + + ALIGN_4 + +.L7_31: +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + leaq BUFFER2, BO // second buffer to BO + addq $6 * SIZE, BO +#else + movq KK, %rax + leaq BUFFER2, BO // second buffer to BO + addq $6 * SIZE, BO + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + leaq (BO, BI, 8), BO + salq $1, %rax // rax = rax * 2 ; number of values + leaq (AO, %rax, 8), AO +#endif + + vzeroall + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $2, %rax // number of values in AO +#else + addq $3, %rax // number of values in BO +#endif + movq %rax, KKK +#endif + + + andq $-8, %rax + je .L7_36 + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + + salq $1, %rax // rax = rax *2 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L7_32: + + prefetcht0 B_PR1(BO,BI,8) + KERNEL2x3_1(xxx) + KERNEL2x3_2(xxx) + KERNEL2x3_3(xxx) + prefetcht0 B_PR1+64(BO,BI,8) + KERNEL2x3_4(xxx) + + KERNEL2x3_1(xxx) + KERNEL2x3_2(xxx) + prefetcht0 B_PR1+32(BO,BI,8) + KERNEL2x3_3(xxx) + KERNEL2x3_4(xxx) + + je .L7_36 + + prefetcht0 B_PR1(BO,BI,8) + KERNEL2x3_1(xxx) + KERNEL2x3_2(xxx) + KERNEL2x3_3(xxx) + prefetcht0 B_PR1+64(BO,BI,8) + KERNEL2x3_4(xxx) + + KERNEL2x3_1(xxx) + KERNEL2x3_2(xxx) + prefetcht0 B_PR1+32(BO,BI,8) + KERNEL2x3_3(xxx) + KERNEL2x3_4(xxx) + + je .L7_36 + + jmp .L7_32 + ALIGN_4 + +.L7_36: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + + andq $7, %rax # if (k & 1) + je .L7_39 + + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + + salq $1, %rax // rax = rax *2 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L7_37: + + KERNEL2x3_SUB(xxx) + addq $3, BI + addq $2, %rax + jl .L7_37 + ALIGN_4 + + +.L7_39: + + vmovddup ALPHA, %xmm0 + +#ifndef TRMMKERNEL + + vfmaddpd (CO1),%xmm0, %xmm4,%xmm4 + vfmaddpd (CO1, LDC),%xmm0, %xmm5,%xmm5 + vfmaddpd (CO1, LDC, 2),%xmm0, %xmm6,%xmm6 + +#else + vmulpd %xmm0, %xmm4,%xmm4 + vmulpd %xmm0, %xmm5,%xmm5 + vmulpd %xmm0, %xmm6,%xmm6 + +#endif + + vmovups %xmm4 , (CO1) + vmovups %xmm5 , (CO1, LDC) + vmovups %xmm6 , (CO1, LDC, 2) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + leaq (BO, BI, 8), BO + salq $1, %rax // rax = rax * 2 ; number of values + leaq (AO, %rax, 8), AO +#endif + + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $2, KK +#endif + + addq $2 * SIZE, CO1 # coffset += 2 + ALIGN_4 + + + + + +.L7_40: + testq $1, M + jz .L7_60 // to next 6 lines of N + + ALIGN_4 + +.L7_41: +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + leaq BUFFER2, BO // second buffer to BO + addq $6 * SIZE, BO +#else + movq KK, %rax + leaq BUFFER2, BO // second buffer to BO + addq $6 * SIZE, BO + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + leaq (BO, BI, 8), BO + leaq (AO, %rax, 8), AO +#endif + + + vzeroall + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $1, %rax // number of values in AO +#else + addq $3, %rax // number of values in BO +#endif + movq %rax, KKK +#endif + + + andq $-8, %rax + je .L7_46 + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L7_42: + + prefetcht0 B_PR1(BO,BI,8) + KERNEL1x3_1(xxx) + KERNEL1x3_2(xxx) + KERNEL1x3_3(xxx) + prefetcht0 B_PR1+64(BO,BI,8) + KERNEL1x3_4(xxx) + + KERNEL1x3_1(xxx) + KERNEL1x3_2(xxx) + prefetcht0 B_PR1+32(BO,BI,8) + KERNEL1x3_3(xxx) + KERNEL1x3_4(xxx) + + je .L7_46 + + prefetcht0 B_PR1(BO,BI,8) + KERNEL1x3_1(xxx) + KERNEL1x3_2(xxx) + KERNEL1x3_3(xxx) + prefetcht0 B_PR1+64(BO,BI,8) + KERNEL1x3_4(xxx) + + KERNEL1x3_1(xxx) + KERNEL1x3_2(xxx) + prefetcht0 B_PR1+32(BO,BI,8) + KERNEL1x3_3(xxx) + KERNEL1x3_4(xxx) + + je .L7_46 + + jmp .L7_42 + ALIGN_4 + +.L7_46: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + + andq $7, %rax # if (k & 1) + je .L7_49 + + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L7_47: + + KERNEL1x3_SUB(xxx) + addq $3, BI + addq $1, %rax + jl .L7_47 + ALIGN_4 + + +.L7_49: + + vmovddup ALPHA, %xmm0 + +#ifndef TRMMKERNEL + + vfmaddsd (CO1),%xmm0, %xmm4,%xmm4 + vfmaddsd (CO1, LDC),%xmm0, %xmm5,%xmm5 + vfmaddsd (CO1, LDC, 2),%xmm0, %xmm6,%xmm6 + +#else + vmulsd %xmm0, %xmm4,%xmm4 + vmulsd %xmm0, %xmm5,%xmm5 + vmulsd %xmm0, %xmm6,%xmm6 + +#endif + + vmovsd %xmm4 , (CO1) + vmovsd %xmm5 , (CO1, LDC) + vmovsd %xmm6 , (CO1, LDC, 2) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + movq %rax, BI // Index for BO + leaq (BI,BI,2), BI // BI = BI * 3 ; number of values + leaq (BO, BI, 8), BO + leaq (AO, %rax, 8), AO +#endif + + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $1, KK +#endif + + addq $1 * SIZE, CO1 # coffset += 1 + +.L7_60: + + decq J // j -- + jg .L6_01 + + +.L2_0: + cmpq $0, Nmod6 // N % 6 == 0 + je .L999 + +/************************************************************************************************ +* Loop for Nmod6 / 2 > 0 +*************************************************************************************************/ + + movq Nmod6, J + sarq $1, J // j = j / 2 + je .L1_0 + ALIGN_4 + +.L2_01: + // copy to sub buffer + movq B, BO1 + leaq BUFFER1, BO // first buffer to BO + movq K, %rax + ALIGN_4 + +.L2_02b: + + vmovups (BO1), %xmm0 + vmovups %xmm0, (BO) + addq $2*SIZE,BO1 + addq $2*SIZE,BO + decq %rax + jnz .L2_02b + +.L2_02c: + + movq BO1, B // next offset of B + +.L2_10: + movq C, CO1 + leaq (C, LDC, 2), C // c += 2 * ldc + +#if defined(TRMMKERNEL) && defined(LEFT) + movq OFFSET, %rax + movq %rax, KK +#endif + + movq A, AO // aoffset = a + addq $16 * SIZE, AO + + movq M, I + sarq $3, I // i = (m >> 3) + je .L2_20 + + ALIGN_4 + +.L2_11: + +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + leaq BUFFER1, BO // first buffer to BO + addq $4 * SIZE, BO +#else + movq KK, %rax + leaq BUFFER1, BO // first buffer to BO + addq $4 * SIZE, BO + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + leaq (BO, BI, 8), BO + salq $3, %rax // rax = rax * 8 ; number of values + leaq (AO, %rax, 8), AO +#endif + + vzeroall + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $8, %rax // number of values in AO +#else + addq $2, %rax // number of values in BO +#endif + movq %rax, KKK +#endif + + + andq $-8, %rax // K = K - ( K % 8 ) + je .L2_16 + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + + salq $3, %rax // rax = rax * 8 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L2_12: + + prefetcht0 B_PR1(BO,BI,8) + KERNEL8x2_1(xxx) + KERNEL8x2_2(xxx) + KERNEL8x2_3(xxx) + KERNEL8x2_4(xxx) + + prefetcht0 B_PR1(BO,BI,8) + KERNEL8x2_1(xxx) + KERNEL8x2_2(xxx) + KERNEL8x2_3(xxx) + KERNEL8x2_4(xxx) + + je .L2_16 + + prefetcht0 B_PR1(BO,BI,8) + KERNEL8x2_1(xxx) + KERNEL8x2_2(xxx) + KERNEL8x2_3(xxx) + KERNEL8x2_4(xxx) + + prefetcht0 B_PR1(BO,BI,8) + KERNEL8x2_1(xxx) + KERNEL8x2_2(xxx) + KERNEL8x2_3(xxx) + KERNEL8x2_4(xxx) + + je .L2_16 + + jmp .L2_12 + ALIGN_4 + +.L2_16: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + + andq $7, %rax # if (k & 1) + je .L2_19 + + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + + salq $3, %rax // rax = rax * 8 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L2_17: + + KERNEL8x2_SUB(xxx) + addq $2, BI + addq $8, %rax + jl .L2_17 + ALIGN_4 + + +.L2_19: + + vmovddup ALPHA, %xmm0 + +#ifndef TRMMKERNEL + + vfmaddpd (CO1),%xmm0, %xmm4,%xmm4 + vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7 + vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10 + vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13 + + vfmaddpd (CO1, LDC),%xmm0, %xmm5,%xmm5 + vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8 + vfmaddpd 4 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11 + vfmaddpd 6 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14 + + +#else + vmulpd %xmm0, %xmm4,%xmm4 + vmulpd %xmm0, %xmm7,%xmm7 + vmulpd %xmm0, %xmm10,%xmm10 + vmulpd %xmm0, %xmm13,%xmm13 + + vmulpd %xmm0, %xmm5,%xmm5 + vmulpd %xmm0, %xmm8,%xmm8 + vmulpd %xmm0, %xmm11,%xmm11 + vmulpd %xmm0, %xmm14,%xmm14 + +#endif + + vmovups %xmm4 , (CO1) + vmovups %xmm7 , 2 * SIZE(CO1) + vmovups %xmm10, 4 * SIZE(CO1) + vmovups %xmm13, 6 * SIZE(CO1) + + vmovups %xmm5 , (CO1, LDC) + vmovups %xmm8 , 2 * SIZE(CO1, LDC) + vmovups %xmm11, 4 * SIZE(CO1, LDC) + vmovups %xmm14, 6 * SIZE(CO1, LDC) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + leaq (BO, BI, 8), BO + salq $3, %rax // rax = rax * 8 ; number of values + leaq (AO, %rax, 8), AO +#endif + + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $8, KK +#endif + + addq $8 * SIZE, CO1 # coffset += 8 + decq I # i -- + jg .L2_11 + ALIGN_4 + +/************************************************************************** +* Rest of M +***************************************************************************/ +.L2_20: + // Test rest of M + + testq $7, M + jz .L2_60 // to next 2 lines of N + + testq $4, M + jz .L2_30 + + ALIGN_4 + +.L2_21: +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + leaq BUFFER1, BO // first buffer to BO + addq $4 * SIZE, BO +#else + movq KK, %rax + leaq BUFFER1, BO // first buffer to BO + addq $4 * SIZE, BO + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + leaq (BO, BI, 8), BO + salq $2, %rax // rax = rax * 4 ; number of values + leaq (AO, %rax, 8), AO +#endif + + vzeroall + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $4, %rax // number of values in AO +#else + addq $2, %rax // number of values in BO +#endif + movq %rax, KKK +#endif + + + andq $-8, %rax + je .L2_26 + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + + salq $2, %rax // rax = rax * 4 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L2_22: + + prefetcht0 B_PR1(BO,BI,8) + KERNEL4x2_1(xxx) + KERNEL4x2_2(xxx) + KERNEL4x2_3(xxx) + KERNEL4x2_4(xxx) + + prefetcht0 B_PR1(BO,BI,8) + KERNEL4x2_1(xxx) + KERNEL4x2_2(xxx) + KERNEL4x2_3(xxx) + KERNEL4x2_4(xxx) + + je .L2_26 + + prefetcht0 B_PR1(BO,BI,8) + KERNEL4x2_1(xxx) + KERNEL4x2_2(xxx) + KERNEL4x2_3(xxx) + KERNEL4x2_4(xxx) + + prefetcht0 B_PR1(BO,BI,8) + KERNEL4x2_1(xxx) + KERNEL4x2_2(xxx) + KERNEL4x2_3(xxx) + KERNEL4x2_4(xxx) + + je .L2_26 + + jmp .L2_22 + ALIGN_4 + +.L2_26: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + + andq $7, %rax # if (k & 1) + je .L2_29 + + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + + salq $2, %rax // rax = rax * 4 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L2_27: + + KERNEL4x2_SUB(xxx) + addq $2, BI + addq $4, %rax + jl .L2_27 + ALIGN_4 + + +.L2_29: + + vmovddup ALPHA, %xmm0 + +#ifndef TRMMKERNEL + + vfmaddpd (CO1),%xmm0, %xmm4,%xmm4 + vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7 + + vfmaddpd (CO1, LDC),%xmm0, %xmm5,%xmm5 + vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8 + + +#else + vmulpd %xmm0, %xmm4,%xmm4 + vmulpd %xmm0, %xmm7,%xmm7 + + vmulpd %xmm0, %xmm5,%xmm5 + vmulpd %xmm0, %xmm8,%xmm8 + +#endif + + vmovups %xmm4 , (CO1) + vmovups %xmm7 , 2 * SIZE(CO1) + + vmovups %xmm5 , (CO1, LDC) + vmovups %xmm8 , 2 * SIZE(CO1, LDC) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + leaq (BO, BI, 8), BO + salq $2, %rax // rax = rax * 4 ; number of values + leaq (AO, %rax, 8), AO +#endif + + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $4, KK +#endif + + addq $4 * SIZE, CO1 # coffset += 4 + ALIGN_4 + + +.L2_30: + testq $2, M + jz .L2_40 + + ALIGN_4 + +.L2_31: +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + leaq BUFFER1, BO // first buffer to BO + addq $4 * SIZE, BO +#else + movq KK, %rax + leaq BUFFER1, BO // first buffer to BO + addq $4 * SIZE, BO + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + leaq (BO, BI, 8), BO + salq $1, %rax // rax = rax * 2 ; number of values + leaq (AO, %rax, 8), AO +#endif + + + vzeroall + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $2, %rax // number of values in AO +#else + addq $2, %rax // number of values in BO +#endif + movq %rax, KKK +#endif + + + andq $-8, %rax + je .L2_36 + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + + salq $1, %rax // rax = rax *2 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L2_32: + + prefetcht0 B_PR1(BO,BI,8) + KERNEL2x2_1(xxx) + KERNEL2x2_2(xxx) + KERNEL2x2_3(xxx) + KERNEL2x2_4(xxx) + + prefetcht0 B_PR1(BO,BI,8) + KERNEL2x2_1(xxx) + KERNEL2x2_2(xxx) + KERNEL2x2_3(xxx) + KERNEL2x2_4(xxx) + + je .L2_36 + + prefetcht0 B_PR1(BO,BI,8) + KERNEL2x2_1(xxx) + KERNEL2x2_2(xxx) + KERNEL2x2_3(xxx) + KERNEL2x2_4(xxx) + + prefetcht0 B_PR1(BO,BI,8) + KERNEL2x2_1(xxx) + KERNEL2x2_2(xxx) + KERNEL2x2_3(xxx) + KERNEL2x2_4(xxx) + + je .L2_36 + + jmp .L2_32 + ALIGN_4 + +.L2_36: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + + andq $7, %rax # if (k & 1) + je .L2_39 + + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + + salq $1, %rax // rax = rax *2 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L2_37: + + KERNEL2x2_SUB(xxx) + addq $2, BI + addq $2, %rax + jl .L2_37 + ALIGN_4 + + +.L2_39: + + vmovddup ALPHA, %xmm0 + +#ifndef TRMMKERNEL + + vfmaddpd (CO1),%xmm0, %xmm4,%xmm4 + vfmaddpd (CO1, LDC),%xmm0, %xmm5,%xmm5 + +#else + vmulpd %xmm0, %xmm4,%xmm4 + vmulpd %xmm0, %xmm5,%xmm5 + +#endif + + vmovups %xmm4 , (CO1) + vmovups %xmm5 , (CO1, LDC) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + leaq (BO, BI, 8), BO + salq $1, %rax // rax = rax * 2 ; number of values + leaq (AO, %rax, 8), AO +#endif + + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $2, KK +#endif + + addq $2 * SIZE, CO1 # coffset += 2 + ALIGN_4 + + +.L2_40: + testq $1, M + jz .L2_60 // to next 2 lines of N + + ALIGN_4 + +.L2_41: +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + leaq BUFFER1, BO // first buffer to BO + addq $4 * SIZE, BO +#else + movq KK, %rax + leaq BUFFER1, BO // first buffer to BO + addq $4 * SIZE, BO + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + leaq (BO, BI, 8), BO + leaq (AO, %rax, 8), AO +#endif + + vzeroall + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $1, %rax // number of values in AO +#else + addq $2, %rax // number of values in BO +#endif + movq %rax, KKK +#endif + + andq $-8, %rax + je .L2_46 + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L2_42: + + prefetcht0 B_PR1(BO,BI,8) + KERNEL1x2_1(xxx) + KERNEL1x2_2(xxx) + KERNEL1x2_3(xxx) + KERNEL1x2_4(xxx) + + prefetcht0 B_PR1(BO,BI,8) + KERNEL1x2_1(xxx) + KERNEL1x2_2(xxx) + KERNEL1x2_3(xxx) + KERNEL1x2_4(xxx) + + je .L2_46 + + prefetcht0 B_PR1(BO,BI,8) + KERNEL1x2_1(xxx) + KERNEL1x2_2(xxx) + KERNEL1x2_3(xxx) + KERNEL1x2_4(xxx) + + prefetcht0 B_PR1(BO,BI,8) + KERNEL1x2_1(xxx) + KERNEL1x2_2(xxx) + KERNEL1x2_3(xxx) + KERNEL1x2_4(xxx) + + je .L2_46 + + jmp .L2_42 + ALIGN_4 + +.L2_46: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + + andq $7, %rax # if (k & 1) + je .L2_49 + + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L2_47: + + KERNEL1x2_SUB(xxx) + addq $2, BI + addq $1, %rax + jl .L2_47 + ALIGN_4 + + +.L2_49: + + vmovddup ALPHA, %xmm0 + +#ifndef TRMMKERNEL + + vfmaddsd (CO1),%xmm0, %xmm4,%xmm4 + vfmaddsd (CO1, LDC),%xmm0, %xmm5,%xmm5 + +#else + vmulsd %xmm0, %xmm4,%xmm4 + vmulsd %xmm0, %xmm5,%xmm5 + +#endif + + vmovsd %xmm4 , (CO1) + vmovsd %xmm5 , (CO1, LDC) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + movq %rax, BI // Index for BO + leaq (BI,BI,1), BI // BI = BI * 2 ; number of values + leaq (BO, BI, 8), BO + leaq (AO, %rax, 8), AO +#endif + + + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $1, KK +#endif + + addq $1 * SIZE, CO1 # coffset += 1 + ALIGN_4 + + + +.L2_60: + + decq J // j -- + jg .L2_01 // next 2 lines of N + + + +.L1_0: + +/************************************************************************************************ +* Loop for Nmod6 % 2 > 0 +*************************************************************************************************/ + + movq Nmod6, J + andq $1, J // j % 2 + je .L999 + ALIGN_4 + +.L1_01: + // copy to sub buffer + movq B, BO1 + leaq BUFFER1, BO // first buffer to BO + movq K, %rax + ALIGN_4 + +.L1_02b: + + vmovsd (BO1), %xmm0 + vmovsd %xmm0, (BO) + addq $1*SIZE,BO1 + addq $1*SIZE,BO + decq %rax + jnz .L1_02b + +.L1_02c: + + movq BO1, B // next offset of B + +.L1_10: + movq C, CO1 + leaq (C, LDC, 1), C // c += 1 * ldc + +#if defined(TRMMKERNEL) && defined(LEFT) + movq OFFSET, %rax + movq %rax, KK +#endif + + movq A, AO // aoffset = a + addq $16 * SIZE, AO + + movq M, I + sarq $3, I // i = (m >> 3) + je .L1_20 + + ALIGN_4 + +.L1_11: +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + leaq BUFFER1, BO // first buffer to BO + addq $2 * SIZE, BO +#else + movq KK, %rax + leaq BUFFER1, BO // first buffer to BO + addq $2 * SIZE, BO + movq %rax, BI // Index for BO + leaq (BO, BI, 8), BO + salq $3, %rax // rax = rax * 8 ; number of values + leaq (AO, %rax, 8), AO +#endif + + + vzeroall + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $8, %rax // number of values in AO +#else + addq $1, %rax // number of values in BO +#endif + movq %rax, KKK +#endif + + andq $-8, %rax // K = K - ( K % 8 ) + je .L1_16 + movq %rax, BI // Index for BO + + salq $3, %rax // rax = rax * 8 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L1_12: + + prefetcht0 B_PR1(BO,BI,8) + KERNEL8x1_1(xxx) + KERNEL8x1_2(xxx) + KERNEL8x1_3(xxx) + KERNEL8x1_4(xxx) + + KERNEL8x1_1(xxx) + KERNEL8x1_2(xxx) + KERNEL8x1_3(xxx) + KERNEL8x1_4(xxx) + + je .L1_16 + + prefetcht0 B_PR1(BO,BI,8) + KERNEL8x1_1(xxx) + KERNEL8x1_2(xxx) + KERNEL8x1_3(xxx) + KERNEL8x1_4(xxx) + + KERNEL8x1_1(xxx) + KERNEL8x1_2(xxx) + KERNEL8x1_3(xxx) + KERNEL8x1_4(xxx) + + je .L1_16 + + jmp .L1_12 + ALIGN_4 + +.L1_16: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + + andq $7, %rax # if (k & 1) + je .L1_19 + + movq %rax, BI // Index for BO + + salq $3, %rax // rax = rax * 8 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L1_17: + + KERNEL8x1_SUB(xxx) + addq $1, BI + addq $8, %rax + jl .L1_17 + ALIGN_4 + + +.L1_19: + + vmovddup ALPHA, %xmm0 + +#ifndef TRMMKERNEL + + vfmaddpd (CO1),%xmm0, %xmm4,%xmm4 + vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7 + vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10 + vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13 + +#else + vmulpd %xmm0, %xmm4,%xmm4 + vmulpd %xmm0, %xmm7,%xmm7 + vmulpd %xmm0, %xmm10,%xmm10 + vmulpd %xmm0, %xmm13,%xmm13 + +#endif + + vmovups %xmm4 , (CO1) + vmovups %xmm7 , 2 * SIZE(CO1) + vmovups %xmm10, 4 * SIZE(CO1) + vmovups %xmm13, 6 * SIZE(CO1) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + movq %rax, BI // Index for BO + leaq (BO, BI, 8), BO + salq $3, %rax // rax = rax * 8 ; number of values + leaq (AO, %rax, 8), AO +#endif + + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $8, KK +#endif + addq $8 * SIZE, CO1 # coffset += 8 + decq I # i -- + jg .L1_11 + ALIGN_4 + +/************************************************************************** +* Rest of M +***************************************************************************/ +.L1_20: + // Test rest of M + + testq $7, M + jz .L999 + + testq $4, M + jz .L1_30 + + ALIGN_4 + +.L1_21: +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + leaq BUFFER1, BO // first buffer to BO + addq $2 * SIZE, BO +#else + movq KK, %rax + leaq BUFFER1, BO // first buffer to BO + addq $2 * SIZE, BO + movq %rax, BI // Index for BO + leaq (BO, BI, 8), BO + salq $2, %rax // rax = rax * 4 ; number of values + leaq (AO, %rax, 8), AO +#endif + + + vzeroall + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $4, %rax // number of values in AO +#else + addq $1, %rax // number of values in BO +#endif + movq %rax, KKK +#endif + + + andq $-8, %rax + je .L1_26 + movq %rax, BI // Index for BO + + salq $2, %rax // rax = rax * 4 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L1_22: + + prefetcht0 B_PR1(BO,BI,8) + KERNEL4x1_1(xxx) + KERNEL4x1_2(xxx) + KERNEL4x1_3(xxx) + KERNEL4x1_4(xxx) + + KERNEL4x1_1(xxx) + KERNEL4x1_2(xxx) + KERNEL4x1_3(xxx) + KERNEL4x1_4(xxx) + + je .L1_26 + + prefetcht0 B_PR1(BO,BI,8) + KERNEL4x1_1(xxx) + KERNEL4x1_2(xxx) + KERNEL4x1_3(xxx) + KERNEL4x1_4(xxx) + + KERNEL4x1_1(xxx) + KERNEL4x1_2(xxx) + KERNEL4x1_3(xxx) + KERNEL4x1_4(xxx) + + je .L1_26 + + jmp .L1_22 + ALIGN_4 + +.L1_26: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + + andq $7, %rax # if (k & 1) + je .L1_29 + + movq %rax, BI // Index for BO + + salq $2, %rax // rax = rax * 4 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L1_27: + + KERNEL4x1_SUB(xxx) + addq $1, BI + addq $4, %rax + jl .L1_27 + ALIGN_4 + + +.L1_29: + + vmovddup ALPHA, %xmm0 + +#ifndef TRMMKERNEL + + vfmaddpd (CO1),%xmm0, %xmm4,%xmm4 + vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7 + +#else + vmulpd %xmm0, %xmm4,%xmm4 + vmulpd %xmm0, %xmm7,%xmm7 + +#endif + + vmovups %xmm4 , (CO1) + vmovups %xmm7 , 2 * SIZE(CO1) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + movq %rax, BI // Index for BO + leaq (BO, BI, 8), BO + salq $2, %rax // rax = rax * 4 ; number of values + leaq (AO, %rax, 8), AO +#endif + + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $4, KK +#endif + + addq $4 * SIZE, CO1 # coffset += 4 + ALIGN_4 + + +.L1_30: + testq $2, M + jz .L1_40 + + ALIGN_4 + +.L1_31: +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + leaq BUFFER1, BO // first buffer to BO + addq $2 * SIZE, BO +#else + movq KK, %rax + leaq BUFFER1, BO // first buffer to BO + addq $2 * SIZE, BO + movq %rax, BI // Index for BO + leaq (BO, BI, 8), BO + salq $1, %rax // rax = rax * 2 ; number of values + leaq (AO, %rax, 8), AO +#endif + + + vzeroall + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $2, %rax // number of values in AO +#else + addq $1, %rax // number of values in BO +#endif + movq %rax, KKK +#endif + + + andq $-8, %rax + je .L1_36 + movq %rax, BI // Index for BO + + salq $1, %rax // rax = rax *2 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L1_32: + + prefetcht0 B_PR1(BO,BI,8) + KERNEL2x1_1(xxx) + KERNEL2x1_2(xxx) + KERNEL2x1_3(xxx) + KERNEL2x1_4(xxx) + + KERNEL2x1_1(xxx) + KERNEL2x1_2(xxx) + KERNEL2x1_3(xxx) + KERNEL2x1_4(xxx) + + je .L1_36 + + KERNEL2x1_1(xxx) + KERNEL2x1_2(xxx) + KERNEL2x1_3(xxx) + KERNEL2x1_4(xxx) + + KERNEL2x1_1(xxx) + KERNEL2x1_2(xxx) + KERNEL2x1_3(xxx) + KERNEL2x1_4(xxx) + + je .L1_36 + + jmp .L1_32 + ALIGN_4 + +.L1_36: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + + andq $7, %rax # if (k & 1) + je .L1_39 + + movq %rax, BI // Index for BO + + salq $1, %rax // rax = rax *2 ; number of values + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L1_37: + + KERNEL2x1_SUB(xxx) + addq $1, BI + addq $2, %rax + jl .L1_37 + ALIGN_4 + + +.L1_39: + + vmovddup ALPHA, %xmm0 + +#ifndef TRMMKERNEL + + vfmaddpd (CO1),%xmm0, %xmm4,%xmm4 + +#else + vmulpd %xmm0, %xmm4,%xmm4 + +#endif + + vmovups %xmm4 , (CO1) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + movq %rax, BI // Index for BO + leaq (BO, BI, 8), BO + salq $1, %rax // rax = rax * 2 ; number of values + leaq (AO, %rax, 8), AO +#endif + + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $2, KK +#endif + addq $2 * SIZE, CO1 # coffset += 2 + ALIGN_4 + + +.L1_40: + testq $1, M + jz .L999 + + ALIGN_4 + +.L1_41: +#if !defined(TRMMKERNEL) || \ + (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + leaq BUFFER1, BO // first buffer to BO + addq $2 * SIZE, BO +#else + movq KK, %rax + leaq BUFFER1, BO // first buffer to BO + addq $2 * SIZE, BO + movq %rax, BI // Index for BO + leaq (BO, BI, 8), BO + leaq (AO, %rax, 8), AO +#endif + + + vzeroall + +#ifndef TRMMKERNEL + movq K, %rax +#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + movq K, %rax + subq KK, %rax + movq %rax, KKK +#else + movq KK, %rax +#ifdef LEFT + addq $1, %rax // number of values in AO +#else + addq $1, %rax // number of values in BO +#endif + movq %rax, KKK +#endif + + andq $-8, %rax + je .L1_46 + movq %rax, BI // Index for BO + + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L1_42: + + prefetcht0 B_PR1(BO,BI,8) + KERNEL1x1_1(xxx) + KERNEL1x1_2(xxx) + KERNEL1x1_3(xxx) + KERNEL1x1_4(xxx) + + KERNEL1x1_1(xxx) + KERNEL1x1_2(xxx) + KERNEL1x1_3(xxx) + KERNEL1x1_4(xxx) + + je .L1_46 + + prefetcht0 B_PR1(BO,BI,8) + KERNEL1x1_1(xxx) + KERNEL1x1_2(xxx) + KERNEL1x1_3(xxx) + KERNEL1x1_4(xxx) + + KERNEL1x1_1(xxx) + KERNEL1x1_2(xxx) + KERNEL1x1_3(xxx) + KERNEL1x1_4(xxx) + + je .L1_46 + + jmp .L1_42 + ALIGN_4 + +.L1_46: +#ifndef TRMMKERNEL + movq K, %rax +#else + movq KKK, %rax +#endif + + andq $7, %rax # if (k & 1) + je .L1_49 + + movq %rax, BI // Index for BO + + leaq (AO, %rax, 8), AO + leaq (BO, BI, 8), BO + negq BI + negq %rax + ALIGN_4 + +.L1_47: + + KERNEL1x1_SUB(xxx) + addq $1, BI + addq $1, %rax + jl .L1_47 + ALIGN_4 + + +.L1_49: + + vmovddup ALPHA, %xmm0 + +#ifndef TRMMKERNEL + + vfmaddsd (CO1),%xmm0, %xmm4,%xmm4 + +#else + vmulsd %xmm0, %xmm4,%xmm4 + +#endif + + vmovsd %xmm4 , (CO1) + +#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ + (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) + movq K, %rax + subq KKK, %rax + movq %rax, BI // Index for BO + leaq (BO, BI, 8), BO + leaq (AO, %rax, 8), AO +#endif + + +#if defined(TRMMKERNEL) && defined(LEFT) + addq $1, KK +#endif + + addq $1 * SIZE, CO1 # coffset += 1 + ALIGN_4 + + +.L999: + movq SP, %rsp + movq (%rsp), %rbx + movq 8(%rsp), %rbp + movq 16(%rsp), %r12 + movq 24(%rsp), %r13 + movq 32(%rsp), %r14 + movq 40(%rsp), %r15 + +#ifdef WINDOWS_ABI + movq 48(%rsp), %rdi + movq 56(%rsp), %rsi + movups 64(%rsp), %xmm6 + movups 80(%rsp), %xmm7 + movups 96(%rsp), %xmm8 + movups 112(%rsp), %xmm9 + movups 128(%rsp), %xmm10 + movups 144(%rsp), %xmm11 + movups 160(%rsp), %xmm12 + movups 176(%rsp), %xmm13 + movups 192(%rsp), %xmm14 + movups 208(%rsp), %xmm15 +#endif + + addq $STACKSIZE, %rsp + ret + + EPILOGUE diff --git a/param.h b/param.h index d6c3a9b00..95b8a699d 100644 --- a/param.h +++ b/param.h @@ -153,7 +153,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define GEMM_DEFAULT_ALIGN 0x0fffUL #define SGEMM_DEFAULT_UNROLL_N 4 +#if defined(BULLDOZER) && defined(ARCH_X86_64) && !defined(COMPLEX) +#define DGEMM_DEFAULT_UNROLL_N 2 +#else #define DGEMM_DEFAULT_UNROLL_N 4 +#endif #define QGEMM_DEFAULT_UNROLL_N 2 #define CGEMM_DEFAULT_UNROLL_N 2 #define ZGEMM_DEFAULT_UNROLL_N 2 @@ -161,14 +165,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifdef ARCH_X86 #define SGEMM_DEFAULT_UNROLL_M 4 -#define DGEMM_DEFAULT_UNROLL_M 2 +#define DGEMM_DEFAULT_UNROLL_M 4 #define QGEMM_DEFAULT_UNROLL_M 2 #define CGEMM_DEFAULT_UNROLL_M 2 #define ZGEMM_DEFAULT_UNROLL_M 1 #define XGEMM_DEFAULT_UNROLL_M 1 #else #define SGEMM_DEFAULT_UNROLL_M 8 +#if defined(BULLDOZER) && !defined(COMPLEX) +#define DGEMM_DEFAULT_UNROLL_M 8 +#else #define DGEMM_DEFAULT_UNROLL_M 4 +#endif #define QGEMM_DEFAULT_UNROLL_M 2 #define CGEMM_DEFAULT_UNROLL_M 4 #define ZGEMM_DEFAULT_UNROLL_M 2 @@ -193,26 +201,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #else #define SGEMM_DEFAULT_P 448 - -#if defined(BULLDOZER) && defined(ARCH_X86_64) -#define DGEMM_DEFAULT_P 248 +#if defined(BULLDOZER) && defined(ARCH_X86_64) && !defined(COMPLEX) +#define DGEMM_DEFAULT_P 384 #else #define DGEMM_DEFAULT_P 224 #endif - #define QGEMM_DEFAULT_P 112 #define CGEMM_DEFAULT_P 224 #define ZGEMM_DEFAULT_P 112 #define XGEMM_DEFAULT_P 56 #define SGEMM_DEFAULT_Q 224 - -#if defined(BULLDOZER) && defined(ARCH_X86_64) -#define DGEMM_DEFAULT_Q 248 +#if defined(BULLDOZER) && defined(ARCH_X86_64) && !defined(COMPLEX) +#define DGEMM_DEFAULT_Q 168 #else #define DGEMM_DEFAULT_Q 224 #endif - #define QGEMM_DEFAULT_Q 224 #define CGEMM_DEFAULT_Q 224 #define ZGEMM_DEFAULT_Q 224 @@ -230,7 +234,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define SYMV_P 16 #define HAVE_EXCLUSIVE_CACHE -#define GEMM_THREAD gemm_thread_mn +#define GEMM_THREAD gemm_thread_m #endif