From 564aa60fec0bf6e89df4faaf6659013cbf16ee91 Mon Sep 17 00:00:00 2001 From: wernsaar Date: Sat, 3 Aug 2013 15:40:51 +0200 Subject: [PATCH] removed dtrsm_kernel_LT_8x2_bulldozer.S --- kernel/x86_64/KERNEL.BULLDOZER | 18 + kernel/x86_64/dtrsm_kernel_LT_8x2_bulldozer.S | 1395 ----------------- 2 files changed, 18 insertions(+), 1395 deletions(-) delete mode 100644 kernel/x86_64/dtrsm_kernel_LT_8x2_bulldozer.S diff --git a/kernel/x86_64/KERNEL.BULLDOZER b/kernel/x86_64/KERNEL.BULLDOZER index 8ebd42244..b19f14244 100644 --- a/kernel/x86_64/KERNEL.BULLDOZER +++ b/kernel/x86_64/KERNEL.BULLDOZER @@ -53,6 +53,7 @@ STRSMKERNEL_RN = ../generic/trsm_kernel_RN.c STRSMKERNEL_RT = ../generic/trsm_kernel_RT.c DTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +#DTRSMKERNEL_LT = dtrsm_kernel_LT_8x2_bulldozer.S DTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c DTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c DTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c @@ -67,4 +68,21 @@ ZTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c ZTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c ZTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c +#STRMMKERNEL = ../generic/trmmkernel_16x2.c +STRMMKERNEL = sgemm_kernel_16x2_bulldozer.S +#STRMMKERNEL_RT = ../generic/trmmkernel_16x2.c +#STRMMKERNEL_RN = ../generic/trmmkernel_16x2.c + +DTRMMKERNEL = dgemm_kernel_8x2_bulldozer.S +#DTRMMKERNEL_RT = ../generic/trmmkernel_8x2.c +#DTRMMKERNEL_RN = ../generic/trmmkernel_8x2.c + +CTRMMKERNEL = cgemm_kernel_4x2_bulldozer.S + +ZTRMMKERNEL = zgemm_kernel_2x2_bulldozer.S +#ZTRMMKERNEL = ../generic/ztrmmkernel_4x2.c +#ZTRMMKERNEL_RR = ../generic/ztrmmkernel_2x2.c +#ZTRMMKERNEL_RC = ../generic/ztrmmkernel_2x2.c + + diff --git a/kernel/x86_64/dtrsm_kernel_LT_8x2_bulldozer.S b/kernel/x86_64/dtrsm_kernel_LT_8x2_bulldozer.S deleted file mode 100644 index af7f18f21..000000000 --- a/kernel/x86_64/dtrsm_kernel_LT_8x2_bulldozer.S +++ /dev/null @@ -1,1395 +0,0 @@ -/*********************************************************************/ -/* Copyright 2009, 2010 The University of Texas at Austin. */ -/* All rights reserved. */ -/* */ -/* Redistribution and use in source and binary forms, with or */ -/* without modification, are permitted provided that the following */ -/* conditions are met: */ -/* */ -/* 1. Redistributions of source code must retain the above */ -/* copyright notice, this list of conditions and the following */ -/* disclaimer. */ -/* */ -/* 2. Redistributions in binary form must reproduce the above */ -/* copyright notice, this list of conditions and the following */ -/* disclaimer in the documentation and/or other materials */ -/* provided with the distribution. */ -/* */ -/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ -/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ -/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ -/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ -/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ -/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ -/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ -/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ -/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ -/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ -/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ -/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ -/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ -/* POSSIBILITY OF SUCH DAMAGE. */ -/* */ -/* The views and conclusions contained in the software and */ -/* documentation are those of the authors and should not be */ -/* interpreted as representing official policies, either expressed */ -/* or implied, of The University of Texas at Austin. */ -/*********************************************************************/ - -#define ASSEMBLER -#include "common.h" - -#define OLD_M %rdi -#define OLD_N %rsi -#define M %r13 -#define N %r14 -#define K %rdx - -#define A %rcx -#define B %r8 -#define C %r9 -#define LDC %r10 - -#define I %r11 -#define AO %rdi -#define BO %rsi -#define CO1 %r15 -#define CO2 %r12 -#define BB %rbp -#define J %rbx - -#ifndef WINDOWS_ABI - -#define STACKSIZE 96 - -#define OFFSET 48(%rsp) -#define AORIG 56(%rsp) -#define KK 64(%rsp) -#define KKK 72(%rsp) - -#else - -#define STACKSIZE 256 - -#define OLD_A 40 + STACKSIZE(%rsp) -#define OLD_B 48 + STACKSIZE(%rsp) -#define OLD_C 56 + STACKSIZE(%rsp) -#define OLD_LDC 64 + STACKSIZE(%rsp) -#define OLD_OFFSET 72 + STACKSIZE(%rsp) - -#define OFFSET 224(%rsp) -#define AORIG 232(%rsp) -#define KK 240(%rsp) -#define KKK 248(%rsp) - -#endif - -#define PREFETCH prefetch -#define PREFETCHSIZE (8 * 7 + 0) - -#define movlpd vmovsd -#define movapd vmovups -#define movupd vmovups - -.macro KERNEL8x2_SUB - vmovddup -16*SIZE(BO,%rax,2), %xmm1 - vmovddup -15*SIZE(BO,%rax,2), %xmm2 - vmovups -16*SIZE(AO,%rax,8), %xmm0 - vfmaddpd %xmm8 , %xmm0 , %xmm1 , %xmm8 - vfmaddpd %xmm9 , %xmm0 , %xmm2 , %xmm9 - vmovups -14*SIZE(AO,%rax,8), %xmm0 - vfmaddpd %xmm10, %xmm0 , %xmm1 , %xmm10 - vfmaddpd %xmm11, %xmm0 , %xmm2 , %xmm11 - vmovups -12*SIZE(AO,%rax,8), %xmm0 - vfmaddpd %xmm12, %xmm0 , %xmm1 , %xmm12 - vfmaddpd %xmm13, %xmm0 , %xmm2 , %xmm13 - vmovups -10*SIZE(AO,%rax,8), %xmm0 - vfmaddpd %xmm14, %xmm0 , %xmm1 , %xmm14 - vfmaddpd %xmm15, %xmm0 , %xmm2 , %xmm15 - addq $SIZE, %rax -.endm - -.macro KERNEL8x1_SUB - vmovddup -16*SIZE(BO,%rax,1), %xmm1 - vmovups -16*SIZE(AO,%rax,8), %xmm0 - vfmaddpd %xmm8 , %xmm0 , %xmm1 , %xmm8 - vmovups -14*SIZE(AO,%rax,8), %xmm0 - vfmaddpd %xmm9 , %xmm0 , %xmm1 , %xmm9 - vmovups -12*SIZE(AO,%rax,8), %xmm0 - vfmaddpd %xmm10, %xmm0 , %xmm1 , %xmm10 - vmovups -10*SIZE(AO,%rax,8), %xmm0 - vfmaddpd %xmm11, %xmm0 , %xmm1 , %xmm11 - addq $SIZE, %rax -.endm - - - - - PROLOGUE - PROFCODE - - subq $STACKSIZE, %rsp - movq %rbx, (%rsp) - movq %rbp, 8(%rsp) - movq %r12, 16(%rsp) - movq %r13, 24(%rsp) - movq %r14, 32(%rsp) - movq %r15, 40(%rsp) - -#ifdef WINDOWS_ABI - movq %rdi, 48(%rsp) - movq %rsi, 56(%rsp) - movups %xmm6, 64(%rsp) - movups %xmm7, 80(%rsp) - movups %xmm8, 96(%rsp) - movups %xmm9, 112(%rsp) - movups %xmm10, 128(%rsp) - movups %xmm11, 144(%rsp) - movups %xmm12, 160(%rsp) - movups %xmm13, 176(%rsp) - movups %xmm14, 192(%rsp) - movups %xmm15, 208(%rsp) - - movq ARG1, OLD_M - movq ARG2, OLD_N - movq ARG3, K - movq OLD_A, A - movq OLD_B, B - movq OLD_C, C - movq OLD_LDC, LDC - movsd OLD_OFFSET, %xmm12 -#else - movq STACKSIZE + 8(%rsp), LDC - movsd STACKSIZE + 16(%rsp), %xmm12 -#endif - - movq OLD_M, M - movq OLD_N, N - - subq $-16 * SIZE, A - subq $-16 * SIZE, B - - movsd %xmm12, OFFSET - movsd %xmm12, KK - - leaq (, LDC, SIZE), LDC - - - movq N, J - sarq $1, J # j = (n >> 1) - jle .L80 - ALIGN_4 - -.L01: - - movq A, AO - - movq C, CO1 # coffset1 = c - leaq (C, LDC, 1), CO2 # coffset2 = c + ldc - leaq (C, LDC, 2), C - - movq OFFSET, %rax - movq %rax, KK - - movq M, I - sarq $3, I # i = (m >> 3) - jle .L50_A - ALIGN_4 -/*********************************************************************************/ -.L51: - - movq B, BO - - vxorpd %xmm8 , %xmm8 , %xmm8 - vxorpd %xmm9 , %xmm9 , %xmm9 - vxorpd %xmm10, %xmm10, %xmm10 - vxorpd %xmm11, %xmm11, %xmm11 - vxorpd %xmm12, %xmm12, %xmm12 - vxorpd %xmm13, %xmm13, %xmm13 - vxorpd %xmm14, %xmm14, %xmm14 - vxorpd %xmm15, %xmm15, %xmm15 - - - movq KK, %rax - andq $-4, %rax - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 8), AO - leaq (BO, %rax, 2), BO - negq %rax - NOBRANCH - je .L56 - ALIGN_4 - -.L52: - KERNEL8x2_SUB - KERNEL8x2_SUB - KERNEL8x2_SUB - KERNEL8x2_SUB - - jl .L52 - ALIGN_4 - -.L56: - movq KK, %rax - andq $3, %rax # if (k & 1) - je .L59 - - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 8), AO - leaq (BO, %rax, 2), BO - negq %rax - ALIGN_4 - -.L57: - KERNEL8x2_SUB - - jl .L57 - ALIGN_4 - -.L59: - - vmovups %xmm8 , %xmm1 - unpcklpd %xmm9 , %xmm8 - unpckhpd %xmm9 , %xmm1 - - vmovups -16 * SIZE(BO), %xmm0 - vsubpd %xmm8 , %xmm0 , %xmm0 - vmovups -14 * SIZE(BO), %xmm8 - vsubpd %xmm1 , %xmm8 , %xmm1 - - vmovups %xmm10, %xmm3 - unpcklpd %xmm11, %xmm10 - unpckhpd %xmm11, %xmm3 - - vmovups -12 * SIZE(BO), %xmm8 - vmovups -10 * SIZE(BO), %xmm9 - vsubpd %xmm10, %xmm8 , %xmm2 - vsubpd %xmm3 , %xmm9 , %xmm3 - - vmovups %xmm12, %xmm5 - unpcklpd %xmm13, %xmm12 - unpckhpd %xmm13, %xmm5 - - vmovups -8 * SIZE(BO), %xmm8 - vmovups -6 * SIZE(BO), %xmm9 - vsubpd %xmm12, %xmm8 , %xmm4 - vsubpd %xmm5 , %xmm9 , %xmm5 - - vmovups %xmm14, %xmm7 - unpcklpd %xmm15, %xmm14 - unpckhpd %xmm15, %xmm7 - - vmovups -4 * SIZE(BO), %xmm8 - vmovups -2 * SIZE(BO), %xmm9 - vsubpd %xmm14, %xmm8 , %xmm6 - vsubpd %xmm7 , %xmm9 , %xmm7 - - vmovddup -16 * SIZE(AO), %xmm8 - vmulpd %xmm0 , %xmm8 , %xmm0 - vmovddup -15 * SIZE(AO), %xmm9 - vfnmaddpd %xmm1 , %xmm0 , %xmm9 , %xmm1 - vmovddup -14 * SIZE(AO), %xmm10 - vfnmaddpd %xmm2 , %xmm0 , %xmm10, %xmm2 - vmovddup -13 * SIZE(AO), %xmm11 - vfnmaddpd %xmm3 , %xmm0 , %xmm11, %xmm3 - vmovddup -12 * SIZE(AO), %xmm8 - vfnmaddpd %xmm4 , %xmm0 , %xmm8 , %xmm4 - vmovddup -11 * SIZE(AO), %xmm9 - vfnmaddpd %xmm5 , %xmm0 , %xmm9 , %xmm5 - vmovddup -10 * SIZE(AO), %xmm10 - vfnmaddpd %xmm6 , %xmm0 , %xmm10, %xmm6 - vmovddup -9 * SIZE(AO), %xmm11 - vfnmaddpd %xmm7 , %xmm0 , %xmm11, %xmm7 - - vmovddup -7 * SIZE(AO), %xmm8 - vmulpd %xmm1 , %xmm8 , %xmm1 - vmovddup -6 * SIZE(AO), %xmm10 - vfnmaddpd %xmm2 , %xmm1 , %xmm10, %xmm2 - vmovddup -5 * SIZE(AO), %xmm11 - vfnmaddpd %xmm3 , %xmm1 , %xmm11, %xmm3 - vmovddup -4 * SIZE(AO), %xmm8 - vfnmaddpd %xmm4 , %xmm1 , %xmm8 , %xmm4 - vmovddup -3 * SIZE(AO), %xmm9 - vfnmaddpd %xmm5 , %xmm1 , %xmm9 , %xmm5 - vmovddup -2 * SIZE(AO), %xmm10 - vfnmaddpd %xmm6 , %xmm1 , %xmm10, %xmm6 - vmovddup -1 * SIZE(AO), %xmm11 - vfnmaddpd %xmm7 , %xmm1 , %xmm11, %xmm7 - - vmovddup 2 * SIZE(AO), %xmm8 - vmulpd %xmm2 , %xmm8 , %xmm2 - vmovddup 3 * SIZE(AO), %xmm11 - vfnmaddpd %xmm3 , %xmm2 , %xmm11, %xmm3 - vmovddup 4 * SIZE(AO), %xmm8 - vfnmaddpd %xmm4 , %xmm2 , %xmm8 , %xmm4 - vmovddup 5 * SIZE(AO), %xmm9 - vfnmaddpd %xmm5 , %xmm2 , %xmm9 , %xmm5 - vmovddup 6 * SIZE(AO), %xmm10 - vfnmaddpd %xmm6 , %xmm2 , %xmm10, %xmm6 - vmovddup 7 * SIZE(AO), %xmm11 - vfnmaddpd %xmm7 , %xmm2 , %xmm11, %xmm7 - - vmovddup 11 * SIZE(AO), %xmm8 - vmulpd %xmm3 , %xmm8 , %xmm3 - vmovddup 12 * SIZE(AO), %xmm11 - vfnmaddpd %xmm4 , %xmm3 , %xmm11, %xmm4 - vmovddup 13 * SIZE(AO), %xmm9 - vfnmaddpd %xmm5 , %xmm3 , %xmm9 , %xmm5 - vmovddup 14 * SIZE(AO), %xmm10 - vfnmaddpd %xmm6 , %xmm3 , %xmm10, %xmm6 - vmovddup 15 * SIZE(AO), %xmm11 - vfnmaddpd %xmm7 , %xmm3 , %xmm11, %xmm7 - - vmovddup 20 * SIZE(AO), %xmm8 - vmulpd %xmm4 , %xmm8 , %xmm4 - vmovddup 21 * SIZE(AO), %xmm9 - vfnmaddpd %xmm5 , %xmm4 , %xmm9 , %xmm5 - vmovddup 22 * SIZE(AO), %xmm10 - vfnmaddpd %xmm6 , %xmm4 , %xmm10, %xmm6 - vmovddup 23 * SIZE(AO), %xmm11 - vfnmaddpd %xmm7 , %xmm4 , %xmm11, %xmm7 - - vmovddup 29 * SIZE(AO), %xmm8 - vmulpd %xmm5 , %xmm8 , %xmm5 - vmovddup 30 * SIZE(AO), %xmm10 - vfnmaddpd %xmm6 , %xmm5 , %xmm10, %xmm6 - vmovddup 31 * SIZE(AO), %xmm11 - vfnmaddpd %xmm7 , %xmm5 , %xmm11, %xmm7 - - vmovddup 38 * SIZE(AO), %xmm8 - vmulpd %xmm6 , %xmm8 , %xmm6 - vmovddup 39 * SIZE(AO), %xmm11 - vfnmaddpd %xmm7 , %xmm6 , %xmm11, %xmm7 - - vmovddup 47 * SIZE(AO), %xmm8 - vmulpd %xmm7 , %xmm8 , %xmm7 - - - vmovsd %xmm0 , 0 * SIZE(CO1) - vmovsd %xmm1 , 1 * SIZE(CO1) - vmovsd %xmm2 , 2 * SIZE(CO1) - vmovsd %xmm3 , 3 * SIZE(CO1) - vmovsd %xmm4 , 4 * SIZE(CO1) - vmovsd %xmm5 , 5 * SIZE(CO1) - vmovsd %xmm6 , 6 * SIZE(CO1) - vmovsd %xmm7 , 7 * SIZE(CO1) - - vmovhpd %xmm0 , 0 * SIZE(CO2) - vmovhpd %xmm1 , 1 * SIZE(CO2) - vmovhpd %xmm2 , 2 * SIZE(CO2) - vmovhpd %xmm3 , 3 * SIZE(CO2) - vmovhpd %xmm4 , 4 * SIZE(CO2) - vmovhpd %xmm5 , 5 * SIZE(CO2) - vmovhpd %xmm6 , 6 * SIZE(CO2) - vmovhpd %xmm7 , 7 * SIZE(CO2) - - vmovups %xmm0 , -16 * SIZE(BO) - vmovups %xmm1 , -14 * SIZE(BO) - vmovups %xmm2 , -12 * SIZE(BO) - vmovups %xmm3 , -10 * SIZE(BO) - vmovups %xmm4 , -8 * SIZE(BO) - vmovups %xmm5 , -6 * SIZE(BO) - vmovups %xmm6 , -4 * SIZE(BO) - vmovups %xmm7 , -2 * SIZE(BO) - - addq $8 * SIZE, CO1 - addq $8 * SIZE, CO2 - - movq K, %rax - subq KK, %rax - leaq (,%rax, SIZE), %rax - leaq (AO, %rax, 8), AO - leaq (BO, %rax, 2), BO - - addq $8, KK - - decq I # i -- - jg .L51 - ALIGN_4 - -/*********************************************************************************/ - -.L50_A: - testq $4, M - je .L80 - -.L51_A: - - movq B, BO - - movddup -16 * SIZE(BO), %xmm1 - movddup -15 * SIZE(BO), %xmm5 - pxor %xmm8, %xmm8 - movddup -12 * SIZE(BO), %xmm3 - pxor %xmm9, %xmm9 - movapd -16 * SIZE(AO), %xmm0 - pxor %xmm12, %xmm12 - movapd -8 * SIZE(AO), %xmm4 - pxor %xmm13, %xmm13 - - movapd %xmm0, %xmm2 - - movq KK, %rax - andq $-4, %rax - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 4), AO - leaq (BO, %rax, 2), BO - negq %rax - NOBRANCH - je .L56_A - ALIGN_4 - -.L52_A: - mulpd %xmm1, %xmm0 - mulpd -14 * SIZE(AO, %rax, 4), %xmm1 - addpd %xmm0, %xmm8 - movapd -12 * SIZE(AO, %rax, 4), %xmm0 - addpd %xmm1, %xmm12 - movddup -14 * SIZE(BO, %rax, 2), %xmm1 - mulpd %xmm5, %xmm2 - mulpd -14 * SIZE(AO, %rax, 4), %xmm5 - addpd %xmm2, %xmm9 - addpd %xmm5, %xmm13 - movddup -13 * SIZE(BO, %rax, 2), %xmm5 - movapd %xmm0, %xmm2 - mulpd %xmm1, %xmm0 - mulpd -10 * SIZE(AO, %rax, 4), %xmm1 - addpd %xmm0, %xmm8 - movapd (AO, %rax, 4), %xmm0 - addpd %xmm1, %xmm12 - movddup -8 * SIZE(BO, %rax, 2), %xmm1 - mulpd %xmm5, %xmm2 - mulpd -10 * SIZE(AO, %rax, 4), %xmm5 - addpd %xmm2, %xmm9 - addpd %xmm5, %xmm13 - movddup -11 * SIZE(BO, %rax, 2), %xmm5 - movapd %xmm4, %xmm2 - mulpd %xmm3, %xmm4 - mulpd -6 * SIZE(AO, %rax, 4), %xmm3 - addpd %xmm4, %xmm8 - movapd -4 * SIZE(AO, %rax, 4), %xmm4 - addpd %xmm3, %xmm12 - movddup -10 * SIZE(BO, %rax, 2), %xmm3 - mulpd %xmm5, %xmm2 - mulpd -6 * SIZE(AO, %rax, 4), %xmm5 - addpd %xmm2, %xmm9 - addpd %xmm5, %xmm13 - movddup -9 * SIZE(BO, %rax, 2), %xmm5 - movapd %xmm4, %xmm2 - mulpd %xmm3, %xmm4 - mulpd -2 * SIZE(AO, %rax, 4), %xmm3 - addpd %xmm4, %xmm8 - movapd 8 * SIZE(AO, %rax, 4), %xmm4 - addpd %xmm3, %xmm12 - movddup -4 * SIZE(BO, %rax, 2), %xmm3 - mulpd %xmm5, %xmm2 - mulpd -2 * SIZE(AO, %rax, 4), %xmm5 - addpd %xmm2, %xmm9 - addpd %xmm5, %xmm13 - movddup -7 * SIZE(BO, %rax, 2), %xmm5 - movapd %xmm0, %xmm2 - - addq $4 * SIZE, %rax - BRANCH - jl .L52_A - ALIGN_4 - -.L56_A: - movq KK, %rax - andq $3, %rax # if (k & 1) - je .L59_A - - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 4), AO - leaq (BO, %rax, 2), BO - negq %rax - ALIGN_4 - -.L57_A: - mulpd %xmm1, %xmm0 - mulpd -14 * SIZE(AO, %rax, 4), %xmm1 - addpd %xmm0, %xmm8 - movapd -12 * SIZE(AO, %rax, 4), %xmm0 - addpd %xmm1, %xmm12 - movddup -14 * SIZE(BO, %rax, 2), %xmm1 - mulpd %xmm5, %xmm2 - mulpd -14 * SIZE(AO, %rax, 4), %xmm5 - addpd %xmm2, %xmm9 - addpd %xmm5, %xmm13 - movddup -13 * SIZE(BO, %rax, 2), %xmm5 - movapd %xmm0, %xmm2 - - addq $SIZE, %rax - jl .L57_A - ALIGN_4 - -.L59_A: - - movapd %xmm8, %xmm0 - unpcklpd %xmm9, %xmm8 - unpckhpd %xmm9, %xmm0 - - movapd %xmm12, %xmm4 - unpcklpd %xmm13, %xmm12 - unpckhpd %xmm13, %xmm4 - - movapd -16 * SIZE(BO), %xmm9 - movapd -14 * SIZE(BO), %xmm13 - movapd -12 * SIZE(BO), %xmm1 - movapd -10 * SIZE(BO), %xmm5 - - subpd %xmm8, %xmm9 - subpd %xmm0, %xmm13 - subpd %xmm12, %xmm1 - subpd %xmm4, %xmm5 - - movddup -16 * SIZE(AO), %xmm8 - mulpd %xmm8, %xmm9 - movddup -15 * SIZE(AO), %xmm10 - mulpd %xmm9, %xmm10 - subpd %xmm10, %xmm13 - movddup -14 * SIZE(AO), %xmm12 - mulpd %xmm9, %xmm12 - subpd %xmm12, %xmm1 - movddup -13 * SIZE(AO), %xmm14 - mulpd %xmm9, %xmm14 - subpd %xmm14, %xmm5 - - - movddup -11 * SIZE(AO), %xmm8 - mulpd %xmm8, %xmm13 - - movddup -10 * SIZE(AO), %xmm10 - mulpd %xmm13, %xmm10 - subpd %xmm10, %xmm1 - movddup -9 * SIZE(AO), %xmm12 - mulpd %xmm13, %xmm12 - subpd %xmm12, %xmm5 - - movddup -6 * SIZE(AO), %xmm8 - mulpd %xmm8, %xmm1 - movddup -5 * SIZE(AO), %xmm10 - mulpd %xmm1, %xmm10 - subpd %xmm10, %xmm5 - - movddup -1 * SIZE(AO), %xmm8 - mulpd %xmm8, %xmm5 - - movlpd %xmm9, 0 * SIZE(CO1) - movlpd %xmm13, 1 * SIZE(CO1) - movlpd %xmm1, 2 * SIZE(CO1) - movlpd %xmm5, 3 * SIZE(CO1) - - movhpd %xmm9, 0 * SIZE(CO2) - movhpd %xmm13, 1 * SIZE(CO2) - movhpd %xmm1, 2 * SIZE(CO2) - movhpd %xmm5, 3 * SIZE(CO2) - - movaps %xmm9, -16 * SIZE(BO) - movaps %xmm13,-14 * SIZE(BO) - movaps %xmm1, -12 * SIZE(BO) - movaps %xmm5, -10 * SIZE(BO) - - addq $4 * SIZE, CO1 - addq $4 * SIZE, CO2 - - movq K, %rax - subq KK, %rax - leaq (,%rax, SIZE), %rax - leaq (AO, %rax, 4), AO - leaq (BO, %rax, 2), BO - - addq $4, KK - - ALIGN_4 - -/*********************************************************************************/ - - -.L60: - testq $2, M - je .L70 - -.L61: - movq B, BO - - movapd -16 * SIZE(AO), %xmm0 - pxor %xmm8, %xmm8 - movapd -12 * SIZE(AO), %xmm2 - pxor %xmm9, %xmm9 - movddup -16 * SIZE(BO), %xmm1 - pxor %xmm10, %xmm10 - movddup -15 * SIZE(BO), %xmm3 - pxor %xmm11, %xmm11 - - movq KK, %rax - andq $-4, %rax - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 2), AO - leaq (BO, %rax, 2), BO - negq %rax - NOBRANCH - je .L66 - ALIGN_4 - -.L62: - mulpd %xmm0, %xmm1 - addpd %xmm1, %xmm8 - movddup -14 * SIZE(BO, %rax, 2), %xmm1 - mulpd %xmm0, %xmm3 - movapd -14 * SIZE(AO, %rax, 2), %xmm0 - addpd %xmm3, %xmm9 - movddup -13 * SIZE(BO, %rax, 2), %xmm3 - mulpd %xmm0, %xmm1 - addpd %xmm1, %xmm10 - movddup -12 * SIZE(BO, %rax, 2), %xmm1 - mulpd %xmm0, %xmm3 - movapd -8 * SIZE(AO, %rax, 2), %xmm0 - addpd %xmm3, %xmm11 - movddup -11 * SIZE(BO, %rax, 2), %xmm3 - mulpd %xmm2, %xmm1 - addpd %xmm1, %xmm8 - movddup -10 * SIZE(BO, %rax, 2), %xmm1 - mulpd %xmm2, %xmm3 - movapd -10 * SIZE(AO, %rax, 2), %xmm2 - addpd %xmm3, %xmm9 - movddup -9 * SIZE(BO, %rax, 2), %xmm3 - mulpd %xmm2, %xmm1 - addpd %xmm1, %xmm10 - movddup -8 * SIZE(BO, %rax, 2), %xmm1 - mulpd %xmm2, %xmm3 - movapd -4 * SIZE(AO, %rax, 2), %xmm2 - addpd %xmm3, %xmm11 - movddup -7 * SIZE(BO, %rax, 2), %xmm3 - - addq $4 * SIZE, %rax - BRANCH - jl .L62 - ALIGN_4 - -.L66: - movq KK, %rax - andq $3, %rax # if (k & 1) - je .L69 - - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 2), AO - leaq (BO, %rax, 2), BO - negq %rax - ALIGN_4 - -.L67: - mulpd %xmm0, %xmm1 - addpd %xmm1, %xmm8 - movddup -14 * SIZE(BO, %rax, 2), %xmm1 - mulpd %xmm0, %xmm3 - movapd -14 * SIZE(AO, %rax, 2), %xmm0 - addpd %xmm3, %xmm9 - movddup -13 * SIZE(BO, %rax, 2), %xmm3 - - addq $SIZE, %rax - jl .L67 - ALIGN_4 - -.L69: - addpd %xmm10, %xmm8 - addpd %xmm11, %xmm9 - - movapd %xmm8, %xmm0 - unpcklpd %xmm9, %xmm8 - unpckhpd %xmm9, %xmm0 - - movapd -16 * SIZE(BO), %xmm9 - movapd -14 * SIZE(BO), %xmm13 - - subpd %xmm8, %xmm9 - subpd %xmm0, %xmm13 - - movddup -16 * SIZE(AO), %xmm8 - mulpd %xmm8, %xmm9 - - movddup -15 * SIZE(AO), %xmm10 - mulpd %xmm9, %xmm10 - subpd %xmm10, %xmm13 - - movddup -13 * SIZE(AO), %xmm8 - mulpd %xmm8, %xmm13 - - movlpd %xmm9, 0 * SIZE(CO1) - movlpd %xmm13, 1 * SIZE(CO1) - - movhpd %xmm9, 0 * SIZE(CO2) - movhpd %xmm13, 1 * SIZE(CO2) - - movaps %xmm9, -16 * SIZE(BO) - movaps %xmm13, -14 * SIZE(BO) - - addq $2 * SIZE, CO1 - addq $2 * SIZE, CO2 - - movq K, %rax - subq KK, %rax - leaq (,%rax, SIZE), %rax - leaq (AO, %rax, 2), AO - leaq (BO, %rax, 2), BO - - addq $2, KK - - ALIGN_4 -/********************************************************************************/ -.L70: - testq $1, M - je .L79 - ALIGN_4 - -.L71: - movq B, BO - - movddup -16 * SIZE(AO), %xmm0 - pxor %xmm8, %xmm8 - movddup -15 * SIZE(AO), %xmm1 - pxor %xmm9, %xmm9 - movddup -14 * SIZE(AO), %xmm2 - pxor %xmm10, %xmm10 - movddup -13 * SIZE(AO), %xmm3 - pxor %xmm11, %xmm11 - - movq KK, %rax - andq $-4, %rax - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 1), AO - leaq (BO, %rax, 2), BO - negq %rax - NOBRANCH - je .L76 - ALIGN_4 - -.L72: - mulpd -16 * SIZE(BO, %rax, 2), %xmm0 - addpd %xmm0, %xmm8 - movddup -12 * SIZE(AO, %rax, 1), %xmm0 - - mulpd -14 * SIZE(BO, %rax, 2), %xmm1 - addpd %xmm1, %xmm9 - movddup -11 * SIZE(AO, %rax, 1), %xmm1 - - mulpd -12 * SIZE(BO, %rax, 2), %xmm2 - addpd %xmm2, %xmm10 - movddup -10 * SIZE(AO, %rax, 1), %xmm2 - - mulpd -10 * SIZE(BO, %rax, 2), %xmm3 - addpd %xmm3, %xmm11 - movddup -9 * SIZE(AO, %rax, 1), %xmm3 - - addq $4 * SIZE, %rax - BRANCH - jl .L72 - ALIGN_4 - -.L76: - movq KK, %rax - andq $3, %rax # if (k & 1) - je .L78 - - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 1), AO - leaq (BO, %rax, 2), BO - negq %rax - ALIGN_4 - -.L77: - mulpd -16 * SIZE(BO, %rax, 2), %xmm0 - addpd %xmm0, %xmm8 - movddup -15 * SIZE(AO, %rax, 1), %xmm0 - - addq $SIZE, %rax - jl .L77 - ALIGN_4 - -.L78: - addpd %xmm9, %xmm8 - addpd %xmm11, %xmm10 - addpd %xmm10, %xmm8 - - movapd -16 * SIZE(BO), %xmm2 - - subpd %xmm8, %xmm2 - - movddup -16 * SIZE(AO), %xmm0 - mulpd %xmm0, %xmm2 - - movlpd %xmm2, 0 * SIZE(CO1) - movhpd %xmm2, 0 * SIZE(CO2) - - movaps %xmm2, -16 * SIZE(BO) - - addq $1 * SIZE, CO1 - addq $1 * SIZE, CO2 - - movq K, %rax - subq KK, %rax - leaq (,%rax, SIZE), %rax - leaq (AO, %rax, 1), AO - leaq (BO, %rax, 2), BO - - addq $1, KK - - ALIGN_4 - -.L79: - - movq BO, B - - decq J # j -- - jg .L01 - ALIGN_4 -/***************************************************************************************/ -.L80: - testq $1, N - je .L999 - - movq A, AO - movq C, CO1 # coffset1 = c - - movq OFFSET, %rax - movq %rax, KK - - movq M, I - sarq $3, I # i = (m >> 3) - jle .L90_A - ALIGN_4 -/*************************************************************************************/ -.L91: - - movq B, BO - - pxor %xmm8, %xmm8 - pxor %xmm9, %xmm9 - pxor %xmm10, %xmm10 - pxor %xmm11, %xmm11 - - - movq KK, %rax - andq $-4, %rax - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 8), AO - leaq (BO, %rax, 1), BO - negq %rax - NOBRANCH - je .L96 - ALIGN_4 - -.L92: - KERNEL8x1_SUB - KERNEL8x1_SUB - KERNEL8x1_SUB - KERNEL8x1_SUB - - jl .L92 - ALIGN_4 - -.L96: - movq KK, %rax - andq $3, %rax # if (k & 1) - je .L99 - - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 8), AO - leaq (BO, %rax, 1), BO - negq %rax - ALIGN_4 - -.L97: - KERNEL8x1_SUB - - jl .L97 - ALIGN_4 -.L99: - - movapd -16 * SIZE(BO), %xmm1 - movapd -14 * SIZE(BO), %xmm3 - movapd -12 * SIZE(BO), %xmm5 - movapd -10 * SIZE(BO), %xmm7 - - subpd %xmm8 , %xmm1 - subpd %xmm9 , %xmm3 - subpd %xmm10, %xmm5 - subpd %xmm11, %xmm7 - - vmovups %xmm1 , %xmm0 - unpckhpd %xmm1 , %xmm1 - - vmovups %xmm3 , %xmm2 - unpckhpd %xmm3 , %xmm3 - - vmovups %xmm5 , %xmm4 - unpckhpd %xmm5 , %xmm5 - - vmovups %xmm7 , %xmm6 - unpckhpd %xmm7 , %xmm7 - - vmulsd -16 * SIZE(AO), %xmm0 , %xmm0 - vfnmaddsd %xmm1 ,-15 * SIZE(AO), %xmm0 , %xmm1 - vfnmaddsd %xmm2 ,-14 * SIZE(AO), %xmm0 , %xmm2 - vfnmaddsd %xmm3 ,-13 * SIZE(AO), %xmm0 , %xmm3 - vfnmaddsd %xmm4 ,-12 * SIZE(AO), %xmm0 , %xmm4 - vfnmaddsd %xmm5 ,-11 * SIZE(AO), %xmm0 , %xmm5 - vfnmaddsd %xmm6 ,-10 * SIZE(AO), %xmm0 , %xmm6 - vfnmaddsd %xmm7 , -9 * SIZE(AO), %xmm0 , %xmm7 - - vmulsd -7 * SIZE(AO), %xmm1 , %xmm1 - vfnmaddsd %xmm2 , -6 * SIZE(AO), %xmm1 , %xmm2 - vfnmaddsd %xmm3 , -5 * SIZE(AO), %xmm1 , %xmm3 - vfnmaddsd %xmm4 , -4 * SIZE(AO), %xmm1 , %xmm4 - vfnmaddsd %xmm5 , -3 * SIZE(AO), %xmm1 , %xmm5 - vfnmaddsd %xmm6 , -2 * SIZE(AO), %xmm1 , %xmm6 - vfnmaddsd %xmm7 , -1 * SIZE(AO), %xmm1 , %xmm7 - - vmulsd 2 * SIZE(AO), %xmm2 , %xmm2 - vfnmaddsd %xmm3 , 3 * SIZE(AO), %xmm2 , %xmm3 - vfnmaddsd %xmm4 , 4 * SIZE(AO), %xmm2 , %xmm4 - vfnmaddsd %xmm5 , 5 * SIZE(AO), %xmm2 , %xmm5 - vfnmaddsd %xmm6 , 6 * SIZE(AO), %xmm2 , %xmm6 - vfnmaddsd %xmm7 , 7 * SIZE(AO), %xmm2 , %xmm7 - - vmulsd 11 * SIZE(AO), %xmm3 , %xmm3 - vfnmaddsd %xmm4 , 12 * SIZE(AO), %xmm3 , %xmm4 - vfnmaddsd %xmm5 , 13 * SIZE(AO), %xmm3 , %xmm5 - vfnmaddsd %xmm6 , 14 * SIZE(AO), %xmm3 , %xmm6 - vfnmaddsd %xmm7 , 15 * SIZE(AO), %xmm3 , %xmm7 - - vmulsd 20 * SIZE(AO), %xmm4 , %xmm4 - vfnmaddsd %xmm5 , 21 * SIZE(AO), %xmm4 , %xmm5 - vfnmaddsd %xmm6 , 22 * SIZE(AO), %xmm4 , %xmm6 - vfnmaddsd %xmm7 , 23 * SIZE(AO), %xmm4 , %xmm7 - - vmulsd 29 * SIZE(AO), %xmm5 , %xmm5 - vfnmaddsd %xmm6 , 30 * SIZE(AO), %xmm5 , %xmm6 - vfnmaddsd %xmm7 , 31 * SIZE(AO), %xmm5 , %xmm7 - - vmulsd 38 * SIZE(AO), %xmm6 , %xmm6 - vfnmaddsd %xmm7 , 39 * SIZE(AO), %xmm6 , %xmm7 - - vmulsd 47 * SIZE(AO), %xmm7 , %xmm7 - - - vmovsd %xmm0 , 0 * SIZE(CO1) - vmovsd %xmm1 , 1 * SIZE(CO1) - vmovsd %xmm2 , 2 * SIZE(CO1) - vmovsd %xmm3 , 3 * SIZE(CO1) - vmovsd %xmm4 , 4 * SIZE(CO1) - vmovsd %xmm5 , 5 * SIZE(CO1) - vmovsd %xmm6 , 6 * SIZE(CO1) - vmovsd %xmm7 , 7 * SIZE(CO1) - - vmovsd %xmm0 , -16 * SIZE(BO) - vmovsd %xmm1 , -15 * SIZE(BO) - vmovsd %xmm2 , -14 * SIZE(BO) - vmovsd %xmm3 , -13 * SIZE(BO) - vmovsd %xmm4 , -12 * SIZE(BO) - vmovsd %xmm5 , -11 * SIZE(BO) - vmovsd %xmm6 , -10 * SIZE(BO) - vmovsd %xmm7 , -9 * SIZE(BO) - - - - addq $8 * SIZE, CO1 - - movq K, %rax - subq KK, %rax - leaq (,%rax, SIZE), %rax - leaq (AO, %rax, 8), AO - addq %rax, BO - - addq $8, KK - - - decq I # i -- - jg .L91 - ALIGN_4 - -/*****************************************************************************/ -.L90_A: - testq $4, M - je .L100 - -.L91_A: - movq B, BO - - movapd -16 * SIZE(AO), %xmm0 - pxor %xmm8, %xmm8 - movapd -8 * SIZE(AO), %xmm2 - pxor %xmm9, %xmm9 - movddup -16 * SIZE(BO), %xmm1 - pxor %xmm10, %xmm10 - movddup -15 * SIZE(BO), %xmm5 - pxor %xmm11, %xmm11 - movddup -14 * SIZE(BO), %xmm3 - - movq KK, %rax - andq $-4, %rax - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 4), AO - leaq (BO, %rax, 1), BO - negq %rax - NOBRANCH - je .L96_A - ALIGN_4 - -.L92_A: - mulpd %xmm1, %xmm0 - mulpd -14 * SIZE(AO, %rax, 4), %xmm1 - addpd %xmm0, %xmm8 - movapd -12 * SIZE(AO, %rax, 4), %xmm0 - addpd %xmm1, %xmm9 - movddup -12 * SIZE(BO, %rax, 1), %xmm1 - mulpd %xmm5, %xmm0 - mulpd -10 * SIZE(AO, %rax, 4), %xmm5 - addpd %xmm0, %xmm10 - movapd (AO, %rax, 4), %xmm0 - addpd %xmm5, %xmm11 - movddup -13 * SIZE(BO, %rax, 1), %xmm5 - mulpd %xmm3, %xmm2 - mulpd -6 * SIZE(AO, %rax, 4), %xmm3 - addpd %xmm2, %xmm8 - movapd -4 * SIZE(AO, %rax, 4), %xmm2 - addpd %xmm3, %xmm9 - movddup -10 * SIZE(BO, %rax, 1), %xmm3 - mulpd %xmm5, %xmm2 - mulpd -2 * SIZE(AO, %rax, 4), %xmm5 - addpd %xmm2, %xmm10 - movapd 8 * SIZE(AO, %rax, 4), %xmm2 - addpd %xmm5, %xmm11 - movddup -11 * SIZE(BO, %rax, 1), %xmm5 - - addq $4 * SIZE, %rax - BRANCH - jl .L92_A - ALIGN_4 - -.L96_A: - movq KK, %rax - andq $3, %rax # if (k & 1) - je .L99_A - - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 4), AO - leaq (BO, %rax, 1), BO - negq %rax - ALIGN_4 - -.L97_A: - mulpd %xmm1, %xmm0 - mulpd -14 * SIZE(AO, %rax, 4), %xmm1 - addpd %xmm0, %xmm8 - movapd -12 * SIZE(AO, %rax, 4), %xmm0 - addpd %xmm1, %xmm9 - movddup -15 * SIZE(BO, %rax, 1), %xmm1 - - addq $SIZE, %rax - jl .L97_A - ALIGN_4 -.L99_A: - addpd %xmm10, %xmm8 - addpd %xmm11, %xmm9 - - - movapd -16 * SIZE(BO), %xmm10 - movapd -14 * SIZE(BO), %xmm11 - - subpd %xmm8, %xmm10 - subpd %xmm9, %xmm11 - - movapd %xmm10, %xmm8 - unpckhpd %xmm8, %xmm8 - - movapd %xmm11, %xmm9 - unpckhpd %xmm9, %xmm9 - - movsd -16 * SIZE(AO), %xmm12 - mulsd %xmm12, %xmm10 - - movsd -15 * SIZE(AO), %xmm13 - mulsd %xmm10, %xmm13 - subsd %xmm13, %xmm8 - movsd -14 * SIZE(AO), %xmm14 - mulsd %xmm10, %xmm14 - subsd %xmm14, %xmm11 - movsd -13 * SIZE(AO), %xmm15 - mulsd %xmm10, %xmm15 - subsd %xmm15, %xmm9 - - movsd -11 * SIZE(AO), %xmm12 - mulsd %xmm12, %xmm8 - - movsd -10 * SIZE(AO), %xmm13 - mulsd %xmm8, %xmm13 - subsd %xmm13, %xmm11 - movsd -9 * SIZE(AO), %xmm14 - mulsd %xmm8, %xmm14 - subsd %xmm14, %xmm9 - - movsd -6 * SIZE(AO), %xmm12 - mulsd %xmm12, %xmm11 - - movsd -5 * SIZE(AO), %xmm13 - mulsd %xmm11, %xmm13 - subsd %xmm13, %xmm9 - - movsd -1 * SIZE(AO), %xmm12 - mulsd %xmm12, %xmm9 - - unpcklpd %xmm8, %xmm10 - unpcklpd %xmm9, %xmm11 - - movlpd %xmm10, 0 * SIZE(CO1) - movhpd %xmm10, 1 * SIZE(CO1) - movlpd %xmm11, 2 * SIZE(CO1) - movhpd %xmm11, 3 * SIZE(CO1) - - movaps %xmm10, -16 * SIZE(BO) - movaps %xmm11, -14 * SIZE(BO) - - addq $4 * SIZE, CO1 - - movq K, %rax - subq KK, %rax - leaq (,%rax, SIZE), %rax - leaq (AO, %rax, 4), AO - addq %rax, BO - - addq $4, KK - - - ALIGN_4 - -/*************************************************************************************/ -.L100: - testq $2, M - je .L110 - - - - movq B, BO - - movddup -16 * SIZE(BO), %xmm0 - pxor %xmm8, %xmm8 - movddup -15 * SIZE(BO), %xmm1 - pxor %xmm9, %xmm9 - movddup -14 * SIZE(BO), %xmm2 - pxor %xmm10, %xmm10 - movddup -13 * SIZE(BO), %xmm3 - pxor %xmm11, %xmm11 - - movq KK, %rax - andq $-4, %rax - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 2), AO - leaq (BO, %rax, 1), BO - negq %rax - NOBRANCH - je .L106 - ALIGN_4 - -.L102: - mulpd -16 * SIZE(AO, %rax, 2), %xmm0 - addpd %xmm0, %xmm8 - movddup -12 * SIZE(BO, %rax, 1), %xmm0 - - mulpd -14 * SIZE(AO, %rax, 2), %xmm1 - addpd %xmm1, %xmm9 - movddup -11 * SIZE(BO, %rax, 1), %xmm1 - - mulpd -12 * SIZE(AO, %rax, 2), %xmm2 - addpd %xmm2, %xmm10 - movddup -10 * SIZE(BO, %rax, 1), %xmm2 - - mulpd -10 * SIZE(AO, %rax, 2), %xmm3 - addpd %xmm3, %xmm11 - movddup -9 * SIZE(BO, %rax, 1), %xmm3 - - addq $4 * SIZE, %rax - BRANCH - jl .L102 - ALIGN_4 - -.L106: - movq KK, %rax - andq $3, %rax # if (k & 1) - je .L109 - - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 2), AO - leaq (BO, %rax, 1), BO - negq %rax - ALIGN_4 - -.L107: - movddup -16 * SIZE(BO, %rax, 1), %xmm0 - mulpd -16 * SIZE(AO, %rax, 2), %xmm0 - addpd %xmm0, %xmm8 - - addq $SIZE, %rax - jl .L107 - ALIGN_4 - -.L109: - addpd %xmm9, %xmm8 - addpd %xmm11, %xmm10 - addpd %xmm10, %xmm8 - - - movapd -16 * SIZE(BO), %xmm10 - subpd %xmm8, %xmm10 - - - movapd %xmm10, %xmm8 - unpckhpd %xmm8, %xmm8 - - movsd -16 * SIZE(AO), %xmm12 - mulsd %xmm12, %xmm10 - - movsd -15 * SIZE(AO), %xmm13 - mulsd %xmm10, %xmm13 - subsd %xmm13, %xmm8 - - movsd -13 * SIZE(AO), %xmm12 - mulsd %xmm12, %xmm8 - - unpcklpd %xmm8, %xmm10 - - movlpd %xmm10, 0 * SIZE(CO1) - movhpd %xmm10, 1 * SIZE(CO1) - - movaps %xmm10, -16 * SIZE(BO) - - addq $2 * SIZE, CO1 - - movq K, %rax - subq KK, %rax - leaq (,%rax, SIZE), %rax - leaq (AO, %rax, 2), AO - addq %rax, BO - - addq $2, KK - - ALIGN_4 - -.L110: - testq $1, M - je .L119 - ALIGN_4 - -.L111: - movq B, BO - - movapd -16 * SIZE(AO), %xmm0 - pxor %xmm8, %xmm8 - movapd -14 * SIZE(AO), %xmm1 - pxor %xmm9, %xmm9 - - movq KK, %rax - andq $-4, %rax - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 1), AO - leaq (BO, %rax, 1), BO - negq %rax - NOBRANCH - je .L116 - ALIGN_4 - -.L112: - mulpd -16 * SIZE(BO, %rax, 1), %xmm0 - addpd %xmm0, %xmm8 - movapd -12 * SIZE(AO, %rax, 1), %xmm0 - - mulpd -14 * SIZE(BO, %rax, 1), %xmm1 - addpd %xmm1, %xmm9 - movapd -10 * SIZE(AO, %rax, 1), %xmm1 - - addq $4 * SIZE, %rax - BRANCH - jl .L112 - ALIGN_4 - -.L116: - movq KK, %rax - andq $3, %rax # if (k & 1) - je .L118 - - leaq (, %rax, SIZE), %rax - leaq (AO, %rax, 1), AO - leaq (BO, %rax, 1), BO - negq %rax - ALIGN_4 - -.L117: - mulsd -16 * SIZE(BO, %rax, 1), %xmm0 - addsd %xmm0, %xmm8 - movsd -15 * SIZE(AO, %rax, 1), %xmm0 - - addq $SIZE, %rax - jl .L117 - ALIGN_4 - -.L118: - addpd %xmm9, %xmm8 - haddpd %xmm8, %xmm8 - - movsd -16 * SIZE(BO), %xmm10 - subsd %xmm8, %xmm10 - - movsd -16 * SIZE(AO), %xmm12 - mulsd %xmm12, %xmm10 - - movsd %xmm10, 0 * SIZE(CO1) - - movlpd %xmm10, -16 * SIZE(BO) - - addq $1 * SIZE, CO1 - - movq K, %rax - subq KK, %rax - leaq (,%rax, SIZE), %rax - addq %rax, AO - addq %rax, BO - - addq $1, KK - - ALIGN_4 - -.L119: - - movq BO, B - - - ALIGN_4 - - -.L999: - movq (%rsp), %rbx - movq 8(%rsp), %rbp - movq 16(%rsp), %r12 - movq 24(%rsp), %r13 - movq 32(%rsp), %r14 - movq 40(%rsp), %r15 - -#ifdef WINDOWS_ABI - movq 48(%rsp), %rdi - movq 56(%rsp), %rsi - movups 64(%rsp), %xmm6 - movups 80(%rsp), %xmm7 - movups 96(%rsp), %xmm8 - movups 112(%rsp), %xmm9 - movups 128(%rsp), %xmm10 - movups 144(%rsp), %xmm11 - movups 160(%rsp), %xmm12 - movups 176(%rsp), %xmm13 - movups 192(%rsp), %xmm14 - movups 208(%rsp), %xmm15 -#endif - - addq $STACKSIZE, %rsp - ret - - EPILOGUE