3241 lines
79 KiB
ArmAsm
3241 lines
79 KiB
ArmAsm
/*****************************************************************************
|
|
Copyright (c) 2011-2014, The OpenBLAS Project
|
|
All rights reserved.
|
|
|
|
Redistribution and use in source and binary forms, with or without
|
|
modification, are permitted provided that the following conditions are
|
|
met:
|
|
|
|
1. Redistributions of source code must retain the above copyright
|
|
notice, this list of conditions and the following disclaimer.
|
|
|
|
2. Redistributions in binary form must reproduce the above copyright
|
|
notice, this list of conditions and the following disclaimer in
|
|
the documentation and/or other materials provided with the
|
|
distribution.
|
|
3. Neither the name of the OpenBLAS project nor the names of
|
|
its contributors may be used to endorse or promote products
|
|
derived from this software without specific prior written
|
|
permission.
|
|
|
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
|
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
|
|
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
**********************************************************************************/
|
|
|
|
#define ASSEMBLER
|
|
#include "common.h"
|
|
|
|
#define old_bm %rdi
|
|
#define old_bn %rsi
|
|
#define old_bk %rdx
|
|
|
|
#define bm %r13
|
|
#define bn %r14
|
|
#define bk %r15
|
|
|
|
#define ALPHA %xmm0
|
|
#define ba %rcx
|
|
#define bb %r8
|
|
#define C %r9
|
|
#define ldc %r10
|
|
|
|
#define i %r11
|
|
#define k %rax
|
|
|
|
#define ptrba %rdi
|
|
#define ptrbb %rsi
|
|
#define C0 %rbx
|
|
#define C1 %rbp
|
|
|
|
#define prebb %r12
|
|
|
|
#ifndef WINDOWS_ABI
|
|
|
|
#define STACKSIZE 128
|
|
|
|
#define old_ldc 8+STACKSIZE(%rsp)
|
|
#define old_offset 16+STACKSIZE(%rsp)
|
|
|
|
#define MEMALPHA_R 48(%rsp)
|
|
#define MEMALPHA_I 56(%rsp)
|
|
#define j 64(%rsp)
|
|
#define OFFSET 72(%rsp)
|
|
#define kk 80(%rsp)
|
|
#define kkk 88(%rsp)
|
|
|
|
#else
|
|
#define STACKSIZE 512
|
|
|
|
#define OLD_ALPHA_I 40 + STACKSIZE(%rsp)
|
|
#define OLD_A 48 + STACKSIZE(%rsp)
|
|
#define OLD_B 56 + STACKSIZE(%rsp)
|
|
#define OLD_C 64 + STACKSIZE(%rsp)
|
|
#define old_ldc 72 + STACKSIZE(%rsp)
|
|
#define old_offset 80 + STACKSIZE(%rsp)
|
|
|
|
#define MEMALPHA_R 224(%rsp)
|
|
#define MEMALPHA_I 232(%rsp)
|
|
#define j 240(%rsp)
|
|
#define OFFSET 248(%rsp)
|
|
#define kk 256(%rsp)
|
|
#define kkk 264(%rsp)
|
|
|
|
#endif
|
|
|
|
#define PREFETCH0 prefetcht0
|
|
#define PREFETCH1 prefetcht0
|
|
#define PREFETCH2 prefetcht0
|
|
#define PRESIZE 64
|
|
|
|
#define xvec0 %xmm0
|
|
#define xvec1 %xmm1
|
|
#define xvec2 %xmm2
|
|
#define xvec3 %xmm3
|
|
#define xvec4 %xmm4
|
|
#define xvec5 %xmm5
|
|
#define xvec6 %xmm6
|
|
#define xvec7 %xmm7
|
|
#define xvec8 %xmm8
|
|
#define xvec9 %xmm9
|
|
#define xvec10 %xmm10
|
|
#define xvec11 %xmm11
|
|
#define xvec12 %xmm12
|
|
#define xvec13 %xmm13
|
|
#define xvec14 %xmm14
|
|
#define xvec15 %xmm15
|
|
|
|
#define yvec0 %ymm0
|
|
#define yvec1 %ymm1
|
|
#define yvec2 %ymm2
|
|
#define yvec3 %ymm3
|
|
#define yvec4 %ymm4
|
|
#define yvec5 %ymm5
|
|
#define yvec6 %ymm6
|
|
#define yvec7 %ymm7
|
|
#define yvec8 %ymm8
|
|
#define yvec9 %ymm9
|
|
#define yvec10 %ymm10
|
|
#define yvec11 %ymm11
|
|
#define yvec12 %ymm12
|
|
#define yvec13 %ymm13
|
|
#define yvec14 %ymm14
|
|
#define yvec15 %ymm15
|
|
|
|
#define LEAQ leaq
|
|
#define ADDQ addq
|
|
#define MULQ imulq
|
|
#define SARQ sarq
|
|
#define SALQ salq
|
|
#define ANDQ andq
|
|
#define SUBQ subq
|
|
#define DECQ decq
|
|
#define JG jg
|
|
#define JLE jle
|
|
#define TEST testq
|
|
#define OR orq
|
|
#define JNE jne
|
|
#define JMP jmp
|
|
#define NOP
|
|
#define XOR xorpd
|
|
#undef MOVQ
|
|
#define MOVQ movq
|
|
|
|
#define XOR_DY vxorpd
|
|
#define XOR_DX vxorpd
|
|
|
|
#define LD_DY vmovapd
|
|
#define LD_DX vmovapd
|
|
#define LDL_DY vmovlpd
|
|
#define LDL_DX vmovlpd
|
|
#define LDH_DY vmovhpd
|
|
#define LDH_DX vmovhpd
|
|
|
|
#define ST_DY vmovapd
|
|
#define ST_DX vmovapd
|
|
#define STL_DY vmovlpd
|
|
#define STL_DX vmovlpd
|
|
#define STH_DY vmovhpd
|
|
#define STH_DX vmovhpd
|
|
|
|
#define EDUP_DY vmovddup
|
|
|
|
#define ADD_DY vaddpd
|
|
#define ADD_DX vaddpd
|
|
#define SUB_DY vsubpd
|
|
#define SUB_DX vsubpd
|
|
|
|
#define ADDSUB_DY vaddsubpd
|
|
#define ADDSUB_DX vaddsubpd
|
|
|
|
#define MUL_DY vmulpd
|
|
#define MUL_DX vmulpd
|
|
|
|
#define SHUF_DY vperm2f128
|
|
#define SHUF_DX vpshufd
|
|
|
|
#define VPERMILP_DY vpermilpd
|
|
|
|
#define BROAD_DY vbroadcastsd
|
|
#define BROAD_DX vmovddup
|
|
|
|
#define MOV_DY vmovapd
|
|
#define MOV_DX vmovapd
|
|
|
|
#define REVS_DY vshufpd
|
|
#define REVS_DX vmovsd
|
|
|
|
#define EXTRA_DY vextractf128
|
|
|
|
|
|
#if defined(NN) || defined(NT) || defined(TN) || defined(TT)
|
|
#define ADD1_DX ADD_DX
|
|
#define ADD1_DY ADD_DY
|
|
#define ADD2_DY ADDSUB_DY
|
|
#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
|
|
#define ADD1_DX SUB_DX
|
|
#define ADD1_DY SUB_DY
|
|
#define ADD2_DY ADDSUB_DY
|
|
#elif defined(RN) || defined(RT) || defined(CN) || defined(CT)
|
|
#define ADD1_DX SUB_DX
|
|
#define ADD1_DY SUB_DY
|
|
#define ADD2_DY ADDSUB_DY
|
|
#else
|
|
#define ADD1_DX ADD_DX
|
|
#define ADD1_DY ADD_DY
|
|
#define ADD2_DY ADDSUB_DY
|
|
#endif
|
|
|
|
PROLOGUE
|
|
|
|
subq $STACKSIZE, %rsp;
|
|
movq %rbx, 0(%rsp);
|
|
movq %rbp, 8(%rsp);
|
|
movq %r12, 16(%rsp);
|
|
movq %r13, 24(%rsp);
|
|
movq %r14, 32(%rsp);
|
|
movq %r15, 40(%rsp);
|
|
|
|
#ifdef WINDOWS_ABI
|
|
movq %rdi, 48(%rsp)
|
|
movq %rsi, 56(%rsp)
|
|
movups %xmm6, 64(%rsp)
|
|
movups %xmm7, 80(%rsp)
|
|
movups %xmm8, 96(%rsp)
|
|
movups %xmm9, 112(%rsp)
|
|
movups %xmm10, 128(%rsp)
|
|
movups %xmm11, 144(%rsp)
|
|
movups %xmm12, 160(%rsp)
|
|
movups %xmm13, 176(%rsp)
|
|
movups %xmm14, 192(%rsp)
|
|
movups %xmm15, 208(%rsp)
|
|
|
|
movq ARG1, old_bm
|
|
movq ARG2, old_bn
|
|
movq ARG3, old_bk
|
|
movq OLD_A, ba
|
|
movq OLD_B, bb
|
|
movq OLD_C, C
|
|
movq old_ldc, ldc
|
|
#ifdef TRMMKERNEL
|
|
movq old_offset, %r11
|
|
#endif
|
|
movaps %xmm3, %xmm0
|
|
movsd OLD_ALPHA_I, %xmm1
|
|
#else
|
|
|
|
movq old_ldc, ldc
|
|
#ifdef TRMMKERNEL
|
|
movq old_offset, %r11;
|
|
#endif
|
|
#endif
|
|
|
|
vzeroupper
|
|
|
|
vmovlps %xmm0, MEMALPHA_R
|
|
vmovlps %xmm1, MEMALPHA_I
|
|
movq old_bm, bm
|
|
movq old_bn, bn
|
|
movq old_bk, bk
|
|
salq $ZBASE_SHIFT, ldc
|
|
#ifdef TRMMKERNEL
|
|
movq %r11, OFFSET
|
|
#ifndef LEFT
|
|
negq %r11;
|
|
#endif
|
|
movq %r11, kk;
|
|
#endif
|
|
|
|
MOVQ bn,j;
|
|
SARQ $2,j; # Rn = 4
|
|
JLE .L0_loopE;
|
|
ALIGN_5;
|
|
.L0_bodyB:;
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
MOVQ OFFSET, %rax;
|
|
MOVQ %rax, kk;
|
|
#endif
|
|
MOVQ C,C0;
|
|
LEAQ (C,ldc,2),C1;
|
|
MOVQ bk, k;
|
|
SALQ $6, k;
|
|
LEAQ (bb, k, 1), prebb; # Rn=4 SIZE=8 COMPLEX=2
|
|
MOVQ ba,ptrba;
|
|
MOVQ bm,i;
|
|
SARQ $2,i; # Rm = 4
|
|
JLE .L1_loopE;
|
|
ALIGN_5;
|
|
.L1_bodyB:;
|
|
#if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA))
|
|
MOVQ bb,ptrbb;
|
|
#else
|
|
MOVQ bb, ptrbb;
|
|
MOVQ kk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 4), ptrba;
|
|
LEAQ (ptrbb, %rax, 4), ptrbb;
|
|
#endif
|
|
|
|
PREFETCH0 0*SIZE(prebb);
|
|
PREFETCH0 8*SIZE(prebb);
|
|
PREFETCH0 16*SIZE(prebb)
|
|
ADDQ $24*SIZE, prebb;
|
|
# Initial Results Register
|
|
XOR_DY yvec15, yvec15, yvec15;
|
|
XOR_DY yvec14, yvec14, yvec14;
|
|
EDUP_DY 0*SIZE(ptrbb), yvec2; # Br1, Br1, Br2, Br2
|
|
XOR_DY yvec13, yvec13, yvec13;
|
|
XOR_DY yvec12, yvec12, yvec12;
|
|
EDUP_DY 4*SIZE(ptrbb), yvec3; # Br3, Br3, Br4, Br4
|
|
PREFETCH2 3*SIZE(C0);
|
|
PREFETCH2 3*SIZE(C1);
|
|
XOR_DY yvec11, yvec11, yvec11;
|
|
XOR_DY yvec10, yvec10, yvec10;
|
|
LD_DY 0*SIZE(ptrba), yvec0; # Ar1, Ai1, Ar2, Ai2
|
|
PREFETCH2 7*SIZE(C0, ldc, 1);
|
|
PREFETCH2 7*SIZE(C1, ldc, 1);
|
|
XOR_DY yvec9, yvec9, yvec9;
|
|
XOR_DY yvec8, yvec8, yvec8;
|
|
#ifndef TRMMKERNEL
|
|
MOVQ bk,k;
|
|
#elif (defined(LEFT)&&!defined(TRANSA))||(!defined(LEFT)&&defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kk, %rax;
|
|
MOVQ %rax, kkk;
|
|
#else
|
|
MOVQ kk, %rax;
|
|
#ifdef LEFT
|
|
ADDQ $4, %rax;
|
|
#else
|
|
ADDQ $4, %rax;
|
|
#endif
|
|
MOVQ %rax, kkk;
|
|
#endif
|
|
SARQ $2,k; # Unroll 4 times
|
|
JLE .L2_loopE;
|
|
ALIGN_5;
|
|
.L2_bodyB:;
|
|
#### Computing kernel ####
|
|
|
|
#### Unroll time 1 ####
|
|
LD_DY 4*SIZE(ptrba), yvec1;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4; # Br2, Br2, Br1, Br1
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5; # Br4, Br4, Br3, Br3
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
ADD1_DY yvec7, yvec11, yvec11;
|
|
|
|
PREFETCH0 PRESIZE*SIZE(ptrba);
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
EDUP_DY 1*SIZE(ptrbb), yvec2; # Bi1, Bi1, Bi2, Bi2
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
EDUP_DY 5*SIZE(ptrbb), yvec3; # Bi3, Bi3, Bi4, Bi4
|
|
ADD1_DY yvec6, yvec14, yvec14;
|
|
ADD1_DY yvec7, yvec10, yvec10;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
VPERMILP_DY $0x05, yvec0, yvec0; # Ai1, Ar1, Ai2, Ar2
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
ADD1_DY yvec7, yvec9, yvec9;
|
|
|
|
MUL_DY yvec1, yvec4, yvec6;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4; # Bi2, Bi2, Bi1, Bi1
|
|
MUL_DY yvec1, yvec5, yvec7;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5; # Bi4, Bi4, Bi3, Bi3
|
|
ADD1_DY yvec6, yvec12, yvec12;
|
|
ADD1_DY yvec7, yvec8, yvec8;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec1; # Ai3, Ar3, Ai4, Ar4
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
ADD2_DY yvec7, yvec11, yvec11;
|
|
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
EDUP_DY 8*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
EDUP_DY 12*SIZE(ptrbb), yvec3;
|
|
ADD2_DY yvec6, yvec14, yvec14;
|
|
ADD2_DY yvec7, yvec10, yvec10;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
LD_DY 8*SIZE(ptrba), yvec0;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
ADD2_DY yvec7, yvec9, yvec9;
|
|
|
|
MUL_DY yvec1, yvec4, yvec6;
|
|
MUL_DY yvec1, yvec5, yvec7;
|
|
ADD2_DY yvec6, yvec12, yvec12;
|
|
ADD2_DY yvec7, yvec8, yvec8;
|
|
|
|
#### Unroll time 2 ####
|
|
LD_DY 12*SIZE(ptrba), yvec1;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5; # Br4, Br4, Br3, Br3
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
ADD1_DY yvec7, yvec11, yvec11;
|
|
|
|
PREFETCH0 (PRESIZE+8)*SIZE(ptrba);
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
EDUP_DY 9*SIZE(ptrbb), yvec2; # Bi1, Bi1, Bi2, Bi2
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
EDUP_DY 13*SIZE(ptrbb), yvec3; # Bi3, Bi3, Bi4, Bi4
|
|
ADD1_DY yvec6, yvec14, yvec14;
|
|
ADD1_DY yvec7, yvec10, yvec10;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
VPERMILP_DY $0x05, yvec0, yvec0; # Ai1, Ar1, Ai2, Ar2
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
ADD1_DY yvec7, yvec9, yvec9;
|
|
|
|
MUL_DY yvec1, yvec4, yvec6;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4; # Bi2, Bi2, Bi1, Bi1
|
|
MUL_DY yvec1, yvec5, yvec7;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5; # Bi4, Bi4, Bi3, Bi3
|
|
ADD1_DY yvec6, yvec12, yvec12;
|
|
ADD1_DY yvec7, yvec8, yvec8;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec1; # Ai3, Ar3, Ai4, Ar4
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
ADD2_DY yvec7, yvec11, yvec11;
|
|
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
EDUP_DY 16*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
EDUP_DY 20*SIZE(ptrbb), yvec3;
|
|
ADD2_DY yvec6, yvec14, yvec14;
|
|
ADD2_DY yvec7, yvec10, yvec10;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
LD_DY 16*SIZE(ptrba), yvec0;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
ADD2_DY yvec7, yvec9, yvec9;
|
|
|
|
MUL_DY yvec1, yvec4, yvec6;
|
|
MUL_DY yvec1, yvec5, yvec7;
|
|
ADD2_DY yvec6, yvec12, yvec12;
|
|
ADD2_DY yvec7, yvec8, yvec8;
|
|
|
|
#### Unroll time 3 ####
|
|
LD_DY 20*SIZE(ptrba), yvec1;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5; # Br4, Br4, Br3, Br3
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
ADD1_DY yvec7, yvec11, yvec11;
|
|
|
|
PREFETCH0 (PRESIZE+16)*SIZE(ptrba);
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
EDUP_DY 17*SIZE(ptrbb), yvec2; # Bi1, Bi1, Bi2, Bi2
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
EDUP_DY 21*SIZE(ptrbb), yvec3; # Bi3, Bi3, Bi4, Bi4
|
|
ADD1_DY yvec6, yvec14, yvec14;
|
|
ADD1_DY yvec7, yvec10, yvec10;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
VPERMILP_DY $0x05, yvec0, yvec0; # Ai1, Ar1, Ai2, Ar2
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
ADD1_DY yvec7, yvec9, yvec9;
|
|
|
|
MUL_DY yvec1, yvec4, yvec6;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4; # Bi2, Bi2, Bi1, Bi1
|
|
MUL_DY yvec1, yvec5, yvec7;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5; # Bi4, Bi4, Bi3, Bi3
|
|
ADD1_DY yvec6, yvec12, yvec12;
|
|
ADD1_DY yvec7, yvec8, yvec8;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec1; # Ai3, Ar3, Ai4, Ar4
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
ADD2_DY yvec7, yvec11, yvec11;
|
|
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
EDUP_DY 24*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
EDUP_DY 28*SIZE(ptrbb), yvec3;
|
|
ADD2_DY yvec6, yvec14, yvec14;
|
|
ADD2_DY yvec7, yvec10, yvec10;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
LD_DY 24*SIZE(ptrba), yvec0;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
ADD2_DY yvec7, yvec9, yvec9;
|
|
|
|
MUL_DY yvec1, yvec4, yvec6;
|
|
MUL_DY yvec1, yvec5, yvec7;
|
|
ADD2_DY yvec6, yvec12, yvec12;
|
|
ADD2_DY yvec7, yvec8, yvec8;
|
|
|
|
#### Unroll time 4 ####
|
|
LD_DY 28*SIZE(ptrba), yvec1;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5; # Br4, Br4, Br3, Br3
|
|
ADDQ $32*SIZE, ptrba;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
ADD1_DY yvec7, yvec11, yvec11;
|
|
|
|
PREFETCH0 (PRESIZE+24)*SIZE(ptrba);
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
EDUP_DY 25*SIZE(ptrbb), yvec2; # Bi1, Bi1, Bi2, Bi2
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
EDUP_DY 29*SIZE(ptrbb), yvec3; # Bi3, Bi3, Bi4, Bi4
|
|
ADD1_DY yvec6, yvec14, yvec14;
|
|
ADD1_DY yvec7, yvec10, yvec10;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
VPERMILP_DY $0x05, yvec0, yvec0; # Ai1, Ar1, Ai2, Ar2
|
|
ADDQ $32*SIZE, ptrbb;
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
ADD1_DY yvec7, yvec9, yvec9;
|
|
|
|
MUL_DY yvec1, yvec4, yvec6;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4; # Bi2, Bi2, Bi1, Bi1
|
|
MUL_DY yvec1, yvec5, yvec7;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5; # Bi4, Bi4, Bi3, Bi3
|
|
ADD1_DY yvec6, yvec12, yvec12;
|
|
ADD1_DY yvec7, yvec8, yvec8;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec1; # Ai3, Ar3, Ai4, Ar4
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
ADD2_DY yvec7, yvec11, yvec11;
|
|
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
EDUP_DY 0*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
EDUP_DY 4*SIZE(ptrbb), yvec3;
|
|
ADD2_DY yvec6, yvec14, yvec14;
|
|
ADD2_DY yvec7, yvec10, yvec10;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
LD_DY 0*SIZE(ptrba), yvec0;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
ADD2_DY yvec7, yvec9, yvec9;
|
|
|
|
MUL_DY yvec1, yvec4, yvec6;
|
|
MUL_DY yvec1, yvec5, yvec7;
|
|
ADD2_DY yvec6, yvec12, yvec12;
|
|
ADD2_DY yvec7, yvec8, yvec8;
|
|
DECQ k;
|
|
JG .L2_bodyB;
|
|
ALIGN_5
|
|
.L2_loopE:;
|
|
#ifndef TRMMKERNEL
|
|
TEST $2, bk;
|
|
#else
|
|
TEST $2, kkk;
|
|
#endif
|
|
JLE .L3_loopE;
|
|
ALIGN_5
|
|
.L3_bodyB:
|
|
#### Unroll time 1 ####
|
|
LD_DY 4*SIZE(ptrba), yvec1;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4; # Br2, Br2, Br1, Br1
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5; # Br4, Br4, Br3, Br3
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
ADD1_DY yvec7, yvec11, yvec11;
|
|
|
|
PREFETCH0 PRESIZE*SIZE(ptrba);
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
EDUP_DY 1*SIZE(ptrbb), yvec2; # Bi1, Bi1, Bi2, Bi2
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
EDUP_DY 5*SIZE(ptrbb), yvec3; # Bi3, Bi3, Bi4, Bi4
|
|
ADD1_DY yvec6, yvec14, yvec14;
|
|
ADD1_DY yvec7, yvec10, yvec10;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
VPERMILP_DY $0x05, yvec0, yvec0; # Ai1, Ar1, Ai2, Ar2
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
ADD1_DY yvec7, yvec9, yvec9;
|
|
|
|
MUL_DY yvec1, yvec4, yvec6;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4; # Bi2, Bi2, Bi1, Bi1
|
|
MUL_DY yvec1, yvec5, yvec7;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5; # Bi4, Bi4, Bi3, Bi3
|
|
ADD1_DY yvec6, yvec12, yvec12;
|
|
ADD1_DY yvec7, yvec8, yvec8;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec1; # Ai3, Ar3, Ai4, Ar4
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
ADD2_DY yvec7, yvec11, yvec11;
|
|
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
EDUP_DY 8*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
EDUP_DY 12*SIZE(ptrbb), yvec3;
|
|
ADD2_DY yvec6, yvec14, yvec14;
|
|
ADD2_DY yvec7, yvec10, yvec10;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
LD_DY 8*SIZE(ptrba), yvec0;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
ADD2_DY yvec7, yvec9, yvec9;
|
|
|
|
MUL_DY yvec1, yvec4, yvec6;
|
|
MUL_DY yvec1, yvec5, yvec7;
|
|
ADD2_DY yvec6, yvec12, yvec12;
|
|
ADD2_DY yvec7, yvec8, yvec8;
|
|
|
|
#### Unroll time 2 ####
|
|
LD_DY 12*SIZE(ptrba), yvec1;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5; # Br4, Br4, Br3, Br3
|
|
ADDQ $16*SIZE, ptrba
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
ADD1_DY yvec7, yvec11, yvec11;
|
|
|
|
PREFETCH0 (PRESIZE+8)*SIZE(ptrba);
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
EDUP_DY 9*SIZE(ptrbb), yvec2; # Bi1, Bi1, Bi2, Bi2
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
EDUP_DY 13*SIZE(ptrbb), yvec3; # Bi3, Bi3, Bi4, Bi4
|
|
ADD1_DY yvec6, yvec14, yvec14;
|
|
ADD1_DY yvec7, yvec10, yvec10;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
VPERMILP_DY $0x05, yvec0, yvec0; # Ai1, Ar1, Ai2, Ar2
|
|
ADDQ $16*SIZE, ptrbb
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
ADD1_DY yvec7, yvec9, yvec9;
|
|
|
|
MUL_DY yvec1, yvec4, yvec6;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4; # Bi2, Bi2, Bi1, Bi1
|
|
MUL_DY yvec1, yvec5, yvec7;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5; # Bi4, Bi4, Bi3, Bi3
|
|
ADD1_DY yvec6, yvec12, yvec12;
|
|
ADD1_DY yvec7, yvec8, yvec8;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec1; # Ai3, Ar3, Ai4, Ar4
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
ADD2_DY yvec7, yvec11, yvec11;
|
|
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
EDUP_DY 0*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
EDUP_DY 4*SIZE(ptrbb), yvec3;
|
|
ADD2_DY yvec6, yvec14, yvec14;
|
|
ADD2_DY yvec7, yvec10, yvec10;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
LD_DY 0*SIZE(ptrba), yvec0;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
ADD2_DY yvec7, yvec9, yvec9;
|
|
|
|
MUL_DY yvec1, yvec4, yvec6;
|
|
MUL_DY yvec1, yvec5, yvec7;
|
|
ADD2_DY yvec6, yvec12, yvec12;
|
|
ADD2_DY yvec7, yvec8, yvec8;
|
|
.L3_loopE:;
|
|
#ifndef TRMMKERNEL
|
|
TEST $1, bk;
|
|
#else
|
|
TEST $1, kkk;
|
|
#endif
|
|
JLE .L4_loopE;
|
|
ALIGN_5
|
|
.L4_loopB:;
|
|
#### Unroll time 1 ####
|
|
PREFETCH0 PRESIZE*SIZE(ptrba);
|
|
LD_DY 4*SIZE(ptrba), yvec1;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5; # Br4, Br4, Br3, Br3
|
|
ADDQ $8*SIZE, ptrba;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
ADD1_DY yvec7, yvec11, yvec11;
|
|
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
EDUP_DY 1*SIZE(ptrbb), yvec2; # Bi1, Bi1, Bi2, Bi2
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
EDUP_DY 5*SIZE(ptrbb), yvec3; # Bi3, Bi3, Bi4, Bi4
|
|
ADD1_DY yvec6, yvec14, yvec14;
|
|
ADD1_DY yvec7, yvec10, yvec10;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
VPERMILP_DY $0x05, yvec0, yvec0; # Ai1, Ar1, Ai2, Ar2
|
|
ADDQ $8*SIZE, ptrbb;
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
ADD1_DY yvec7, yvec9, yvec9;
|
|
|
|
MUL_DY yvec1, yvec4, yvec6;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4; # Bi2, Bi2, Bi1, Bi1
|
|
MUL_DY yvec1, yvec5, yvec7;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5; # Bi4, Bi4, Bi3, Bi3
|
|
ADD1_DY yvec6, yvec12, yvec12;
|
|
ADD1_DY yvec7, yvec8, yvec8;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec1; # Ai3, Ar3, Ai4, Ar4
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
ADD2_DY yvec7, yvec11, yvec11;
|
|
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
ADD2_DY yvec6, yvec14, yvec14;
|
|
ADD2_DY yvec7, yvec10, yvec10;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
ADD2_DY yvec7, yvec9, yvec9;
|
|
|
|
MUL_DY yvec1, yvec4, yvec6;
|
|
MUL_DY yvec1, yvec5, yvec7;
|
|
ADD2_DY yvec6, yvec12, yvec12;
|
|
ADD2_DY yvec7, yvec8, yvec8;
|
|
.L4_loopE:;
|
|
#### Handle ####
|
|
XOR_DY yvec7, yvec7, yvec7;
|
|
#if defined(RN) || defined(RT) || defined(CN) || defined(CT)
|
|
ADDSUB_DY yvec15, yvec7, yvec15;
|
|
ADDSUB_DY yvec14, yvec7, yvec14;
|
|
ADDSUB_DY yvec13, yvec7, yvec13;
|
|
ADDSUB_DY yvec12, yvec7, yvec12;
|
|
ADDSUB_DY yvec11, yvec7, yvec11;
|
|
ADDSUB_DY yvec10, yvec7, yvec10;
|
|
ADDSUB_DY yvec9, yvec7, yvec9;
|
|
ADDSUB_DY yvec8, yvec7, yvec8;
|
|
#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
|
|
SUB_DY yvec15, yvec7, yvec15;
|
|
SUB_DY yvec14, yvec7, yvec14;
|
|
SUB_DY yvec13, yvec7, yvec13;
|
|
SUB_DY yvec12, yvec7, yvec12;
|
|
SUB_DY yvec11, yvec7, yvec11;
|
|
SUB_DY yvec10, yvec7, yvec10;
|
|
SUB_DY yvec9, yvec7, yvec9;
|
|
SUB_DY yvec8, yvec7, yvec8;
|
|
#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
|
|
VPERMILP_DY $0x05, yvec15, yvec15;
|
|
VPERMILP_DY $0x05, yvec14, yvec14;
|
|
VPERMILP_DY $0x05, yvec13, yvec13;
|
|
VPERMILP_DY $0x05, yvec12, yvec12;
|
|
VPERMILP_DY $0x05, yvec11, yvec11;
|
|
VPERMILP_DY $0x05, yvec10, yvec10;
|
|
VPERMILP_DY $0x05, yvec9, yvec9;
|
|
VPERMILP_DY $0x05, yvec8, yvec8;
|
|
ADDSUB_DY yvec15, yvec7, yvec15;
|
|
ADDSUB_DY yvec14, yvec7, yvec14;
|
|
ADDSUB_DY yvec13, yvec7, yvec13;
|
|
ADDSUB_DY yvec12, yvec7, yvec12;
|
|
ADDSUB_DY yvec11, yvec7, yvec11;
|
|
ADDSUB_DY yvec10, yvec7, yvec10;
|
|
ADDSUB_DY yvec9, yvec7, yvec9;
|
|
ADDSUB_DY yvec8, yvec7, yvec8;
|
|
VPERMILP_DY $0x05, yvec15, yvec15;
|
|
VPERMILP_DY $0x05, yvec14, yvec14;
|
|
VPERMILP_DY $0x05, yvec13, yvec13;
|
|
VPERMILP_DY $0x05, yvec12, yvec12;
|
|
VPERMILP_DY $0x05, yvec11, yvec11;
|
|
VPERMILP_DY $0x05, yvec10, yvec10;
|
|
VPERMILP_DY $0x05, yvec9, yvec9;
|
|
VPERMILP_DY $0x05, yvec8, yvec8;
|
|
#endif
|
|
#### Load Alpha ####
|
|
BROAD_DY MEMALPHA_R,yvec7;
|
|
BROAD_DY MEMALPHA_I,yvec6;
|
|
#### Multiply Alpha ####
|
|
VPERMILP_DY $0x05, yvec15, yvec5;
|
|
MUL_DY yvec7, yvec15, yvec15;
|
|
MUL_DY yvec6, yvec5, yvec5;
|
|
ADDSUB_DY yvec5, yvec15, yvec15;
|
|
VPERMILP_DY $0x05, yvec14, yvec4;
|
|
MUL_DY yvec7, yvec14, yvec14;
|
|
MUL_DY yvec6, yvec4, yvec4;
|
|
ADDSUB_DY yvec4, yvec14, yvec14;
|
|
VPERMILP_DY $0x05, yvec13, yvec3;
|
|
MUL_DY yvec7, yvec13, yvec13;
|
|
MUL_DY yvec6, yvec3, yvec3;
|
|
ADDSUB_DY yvec3, yvec13, yvec13;
|
|
VPERMILP_DY $0x05,yvec12, yvec2;
|
|
MUL_DY yvec7, yvec12, yvec12;
|
|
MUL_DY yvec6, yvec2, yvec2;
|
|
ADDSUB_DY yvec2, yvec12, yvec12;
|
|
VPERMILP_DY $0x05, yvec11, yvec1;
|
|
MUL_DY yvec7, yvec11, yvec11;
|
|
MUL_DY yvec6, yvec1, yvec1;
|
|
ADDSUB_DY yvec1, yvec11, yvec11;
|
|
VPERMILP_DY $0x05,yvec10, yvec0;
|
|
MUL_DY yvec7, yvec10, yvec10;
|
|
MUL_DY yvec6, yvec0, yvec0;
|
|
ADDSUB_DY yvec0, yvec10, yvec10;
|
|
VPERMILP_DY $0x05, yvec9, yvec5;
|
|
MUL_DY yvec7, yvec9, yvec9;
|
|
MUL_DY yvec6, yvec5, yvec5;
|
|
ADDSUB_DY yvec5, yvec9, yvec9;
|
|
VPERMILP_DY $0x05, yvec8, yvec4;
|
|
MUL_DY yvec7, yvec8, yvec8;
|
|
MUL_DY yvec6, yvec4, yvec4;
|
|
ADDSUB_DY yvec4, yvec8, yvec8;
|
|
#### Testing Alignment ####
|
|
MOVQ C0, %rax;
|
|
OR ldc, %rax;
|
|
TEST $15, %rax;
|
|
JNE .L4_loopEx;
|
|
ALIGN_5
|
|
#### Store Back ####
|
|
EXTRA_DY $1,yvec15,xvec7;
|
|
EXTRA_DY $1,yvec14,xvec6;
|
|
EXTRA_DY $1,yvec13,xvec5;
|
|
EXTRA_DY $1,yvec12,xvec4;
|
|
EXTRA_DY $1,yvec11,xvec3;
|
|
EXTRA_DY $1,yvec10,xvec2;
|
|
EXTRA_DY $1,yvec9,xvec1;
|
|
EXTRA_DY $1,yvec8,xvec0;
|
|
#ifndef TRMMKERNEL
|
|
ADD_DY 0*SIZE(C0),xvec15, xvec15;
|
|
ADD_DY 2*SIZE(C0,ldc,1), xvec7, xvec7;
|
|
ADD_DY 4*SIZE(C0),xvec14, xvec14;
|
|
ADD_DY 6*SIZE(C0,ldc,1),xvec6, xvec6;
|
|
ADD_DY 0*SIZE(C0,ldc,1),xvec13, xvec13;
|
|
ADD_DY 2*SIZE(C0),xvec5, xvec5;
|
|
ADD_DY 4*SIZE(C0,ldc,1),xvec12, xvec12;
|
|
ADD_DY 6*SIZE(C0),xvec4, xvec4;
|
|
ADD_DY 0*SIZE(C1),xvec11, xvec11;
|
|
ADD_DY 2*SIZE(C1,ldc,1),xvec3, xvec3;
|
|
ADD_DY 4*SIZE(C1),xvec10, xvec10;
|
|
ADD_DY 6*SIZE(C1,ldc,1),xvec2, xvec2;
|
|
ADD_DY 0*SIZE(C1,ldc,1),xvec9, xvec9;
|
|
ADD_DY 2*SIZE(C1),xvec1, xvec1;
|
|
ADD_DY 4*SIZE(C1,ldc,1),xvec8, xvec8;
|
|
ADD_DY 6*SIZE(C1),xvec0, xvec0;
|
|
#endif
|
|
ST_DY xvec15,0*SIZE(C0);
|
|
ST_DY xvec7,2*SIZE(C0,ldc,1);
|
|
ST_DY xvec14,4*SIZE(C0);
|
|
ST_DY xvec6,6*SIZE(C0,ldc,1);
|
|
ST_DY xvec13,0*SIZE(C0,ldc,1);
|
|
ST_DY xvec5,2*SIZE(C0);
|
|
ST_DY xvec12,4*SIZE(C0,ldc,1);
|
|
ST_DY xvec4,6*SIZE(C0);
|
|
ST_DY xvec11,0*SIZE(C1);
|
|
ST_DY xvec3,2*SIZE(C1,ldc,1);
|
|
ST_DY xvec10,4*SIZE(C1);
|
|
ST_DY xvec2,6*SIZE(C1,ldc,1);
|
|
ST_DY xvec9,0*SIZE(C1,ldc,1);
|
|
ST_DY xvec1,2*SIZE(C1);
|
|
ST_DY xvec8,4*SIZE(C1,ldc,1);
|
|
ST_DY xvec0,6*SIZE(C1);
|
|
#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kkk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 4), ptrba;
|
|
LEAQ (ptrbb, %rax, 4), ptrbb;
|
|
#endif
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
ADDQ $4, kk;
|
|
#endif
|
|
ADDQ $8*SIZE,C0;
|
|
ADDQ $8*SIZE,C1;
|
|
.L1_bodyE:;
|
|
DECQ i;
|
|
JG .L1_bodyB;
|
|
JMP .L1_loopE;
|
|
ALIGN_5
|
|
.L4_loopEx:
|
|
EXTRA_DY $1, yvec15, xvec7;
|
|
EXTRA_DY $1, yvec14, xvec6;
|
|
#ifndef TRMMKERNEL
|
|
LDL_DY 0*SIZE(C0), xvec0, xvec0;
|
|
LDH_DY 1*SIZE(C0), xvec0, xvec0;
|
|
LDL_DY 2*SIZE(C0, ldc, 1), xvec1, xvec1;
|
|
LDH_DY 3*SIZE(C0, ldc, 1), xvec1, xvec1;
|
|
LDL_DY 4*SIZE(C0), xvec2, xvec2;
|
|
LDH_DY 5*SIZE(C0), xvec2, xvec2;
|
|
LDL_DY 6*SIZE(C0, ldc, 1), xvec3, xvec3;
|
|
LDH_DY 7*SIZE(C0, ldc, 1), xvec3, xvec3;
|
|
ADD_DY xvec0, xvec15, xvec15;
|
|
ADD_DY xvec1, xvec7, xvec7;
|
|
ADD_DY xvec2, xvec14, xvec14;
|
|
ADD_DY xvec3, xvec6, xvec6;
|
|
#endif
|
|
STL_DY xvec15, 0*SIZE(C0);
|
|
STH_DY xvec15, 1*SIZE(C0);
|
|
STL_DY xvec7, 2*SIZE(C0, ldc, 1);
|
|
STH_DY xvec7, 3*SIZE(C0, ldc, 1);
|
|
STL_DY xvec14, 4*SIZE(C0);
|
|
STH_DY xvec14, 5*SIZE(C0);
|
|
STL_DY xvec6, 6*SIZE(C0, ldc, 1);
|
|
STH_DY xvec6, 7*SIZE(C0, ldc, 1);
|
|
EXTRA_DY $1, yvec13, xvec5;
|
|
EXTRA_DY $1, yvec12, xvec4;
|
|
#ifndef TRMMKERNEL
|
|
LDL_DY 0*SIZE(C0, ldc, 1), xvec3, xvec3;
|
|
LDH_DY 1*SIZE(C0, ldc, 1), xvec3, xvec3;
|
|
LDL_DY 2*SIZE(C0), xvec2, xvec2;
|
|
LDH_DY 3*SIZE(C0), xvec2, xvec2;
|
|
LDL_DY 4*SIZE(C0, ldc, 1), xvec1, xvec1;
|
|
LDH_DY 5*SIZE(C0, ldc, 1), xvec1, xvec1;
|
|
LDL_DY 6*SIZE(C0), xvec0, xvec0;
|
|
LDH_DY 7*SIZE(C0), xvec0, xvec0;
|
|
ADD_DY xvec3, xvec13, xvec13;
|
|
ADD_DY xvec2, xvec5, xvec5;
|
|
ADD_DY xvec1, xvec12, xvec12;
|
|
ADD_DY xvec0, xvec4, xvec4;
|
|
#endif
|
|
STL_DY xvec13, 0*SIZE(C0, ldc, 1);
|
|
STH_DY xvec13, 1*SIZE(C0, ldc, 1);
|
|
STL_DY xvec5, 2*SIZE(C0);
|
|
STH_DY xvec5, 3*SIZE(C0);
|
|
STL_DY xvec12, 4*SIZE(C0, ldc, 1);
|
|
STH_DY xvec12, 5*SIZE(C0, ldc, 1);
|
|
STL_DY xvec4, 6*SIZE(C0);
|
|
STH_DY xvec4, 7*SIZE(C0);
|
|
EXTRA_DY $1, yvec11, xvec3;
|
|
EXTRA_DY $1, yvec10, xvec2;
|
|
#ifndef TRMMKERNEL
|
|
LDL_DY 0*SIZE(C1), xvec7, xvec7;
|
|
LDH_DY 1*SIZE(C1), xvec7, xvec7;
|
|
LDL_DY 2*SIZE(C1, ldc, 1), xvec6, xvec6;
|
|
LDH_DY 3*SIZE(C1, ldc, 1), xvec6, xvec6;
|
|
LDL_DY 4*SIZE(C1), xvec5, xvec5;
|
|
LDH_DY 5*SIZE(C1), xvec5, xvec5;
|
|
LDL_DY 6*SIZE(C1, ldc, 1), xvec4, xvec4;
|
|
LDH_DY 7*SIZE(C1, ldc, 1), xvec4, xvec4;
|
|
ADD_DY xvec7, xvec11, xvec11;
|
|
ADD_DY xvec6, xvec3, xvec3;
|
|
ADD_DY xvec5, xvec10, xvec10;
|
|
ADD_DY xvec4, xvec2, xvec2;
|
|
#endif
|
|
STL_DY xvec11, 0*SIZE(C1);
|
|
STH_DY xvec11, 1*SIZE(C1);
|
|
STL_DY xvec3, 2*SIZE(C1, ldc, 1);
|
|
STH_DY xvec3, 3*SIZE(C1, ldc, 1);
|
|
STL_DY xvec10, 4*SIZE(C1);
|
|
STH_DY xvec10, 5*SIZE(C1);
|
|
STL_DY xvec2, 6*SIZE(C1, ldc, 1);
|
|
STH_DY xvec2, 7*SIZE(C1, ldc, 1);
|
|
EXTRA_DY $1, yvec9, xvec1;
|
|
EXTRA_DY $1, yvec8, xvec0;
|
|
#ifndef TRMMKERNEL
|
|
LDL_DY 0*SIZE(C1, ldc, 1), xvec5, xvec5;
|
|
LDH_DY 1*SIZE(C1, ldc, 1), xvec5, xvec5;
|
|
LDL_DY 2*SIZE(C1), xvec4, xvec4;
|
|
LDH_DY 3*SIZE(C1), xvec4, xvec4;
|
|
LDL_DY 4*SIZE(C1, ldc, 1), xvec3, xvec3;
|
|
LDH_DY 5*SIZE(C1, ldc, 1), xvec3, xvec3;
|
|
LDL_DY 6*SIZE(C1), xvec2, xvec2;
|
|
LDH_DY 7*SIZE(C1), xvec2, xvec2;
|
|
ADD_DY xvec5, xvec9, xvec9;
|
|
ADD_DY xvec4, xvec1, xvec1;
|
|
ADD_DY xvec3, xvec8, xvec8;
|
|
ADD_DY xvec2, xvec0, xvec0;
|
|
#endif
|
|
STL_DY xvec9, 0*SIZE(C1, ldc, 1);
|
|
STH_DY xvec9, 1*SIZE(C1, ldc, 1);
|
|
STL_DY xvec1, 2*SIZE(C1);
|
|
STH_DY xvec1, 3*SIZE(C1);
|
|
STL_DY xvec8, 4*SIZE(C1, ldc, 1);
|
|
STH_DY xvec8, 5*SIZE(C1, ldc, 1);
|
|
STL_DY xvec0, 6*SIZE(C1);
|
|
STH_DY xvec0, 7*SIZE(C1);
|
|
#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kkk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 4), ptrba;
|
|
LEAQ (ptrbb, %rax, 4), ptrbb;
|
|
#endif
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
ADDQ $4, kk;
|
|
#endif
|
|
ADDQ $8*SIZE, C0;
|
|
ADDQ $8*SIZE, C1;
|
|
DECQ i;
|
|
JG .L1_bodyB;
|
|
ALIGN_5;
|
|
.L1_loopE:;
|
|
TEST $2, bm;
|
|
JLE .L5_loopE;
|
|
ALIGN_5
|
|
.L5_bodyB:
|
|
#if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA))
|
|
MOVQ bb,ptrbb;
|
|
#else
|
|
MOVQ bb, ptrbb;
|
|
MOVQ kk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 2), ptrba;
|
|
LEAQ (ptrbb, %rax, 4), ptrbb;
|
|
#endif
|
|
XOR_DY yvec15, yvec15, yvec15;
|
|
XOR_DY yvec14, yvec14, yvec14;
|
|
XOR_DY yvec13, yvec13, yvec13;
|
|
XOR_DY yvec12, yvec12, yvec12;
|
|
#ifndef TRMMKERNEL
|
|
MOVQ bk,k;
|
|
#elif (defined(LEFT)&&!defined(TRANSA))||(!defined(LEFT)&&defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kk, %rax;
|
|
MOVQ %rax, kkk;
|
|
#else
|
|
MOVQ kk, %rax;
|
|
#ifdef LEFT
|
|
ADDQ $2, %rax;
|
|
#else
|
|
ADDQ $4, %rax;
|
|
#endif
|
|
MOVQ %rax, kkk;
|
|
#endif
|
|
SARQ $2, k;
|
|
JLE .L7_loopE;
|
|
ALIGN_5
|
|
.L7_bodyB:
|
|
#### Compute kernel ####
|
|
#### Unroll times 1 ####
|
|
LD_DY 0*SIZE(ptrba), yvec0;
|
|
EDUP_DY 0*SIZE(ptrbb), yvec2;
|
|
EDUP_DY 4*SIZE(ptrbb), yvec3;
|
|
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
EDUP_DY 1*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
ADD1_DY yvec7 ,yvec12, yvec12;
|
|
EDUP_DY 5*SIZE(ptrbb), yvec3
|
|
|
|
VPERMILP_DY $0x05, yvec0, yvec0;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
ADD2_DY yvec7, yvec12, yvec12;
|
|
|
|
#### Unroll time 2 ####
|
|
LD_DY 4*SIZE(ptrba), yvec0;
|
|
EDUP_DY 8*SIZE(ptrbb), yvec2;
|
|
EDUP_DY 12*SIZE(ptrbb), yvec3;
|
|
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
EDUP_DY 9*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
ADD1_DY yvec7 ,yvec12, yvec12;
|
|
EDUP_DY 13*SIZE(ptrbb), yvec3
|
|
|
|
VPERMILP_DY $0x05, yvec0, yvec0;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
ADD2_DY yvec7, yvec12, yvec12;
|
|
|
|
#### Unroll time 3 ####
|
|
LD_DY 8*SIZE(ptrba), yvec0;
|
|
EDUP_DY 16*SIZE(ptrbb), yvec2;
|
|
EDUP_DY 20*SIZE(ptrbb), yvec3;
|
|
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
EDUP_DY 17*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
ADD1_DY yvec7 ,yvec12, yvec12;
|
|
EDUP_DY 21*SIZE(ptrbb), yvec3
|
|
|
|
VPERMILP_DY $0x05, yvec0, yvec0;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
ADD2_DY yvec7, yvec12, yvec12;
|
|
|
|
#### Unroll time 4 ####
|
|
LD_DY 12*SIZE(ptrba), yvec0;
|
|
EDUP_DY 24*SIZE(ptrbb), yvec2;
|
|
EDUP_DY 28*SIZE(ptrbb), yvec3;
|
|
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
EDUP_DY 25*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
ADD1_DY yvec7 ,yvec12, yvec12;
|
|
EDUP_DY 29*SIZE(ptrbb), yvec3
|
|
|
|
VPERMILP_DY $0x05, yvec0, yvec0;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
ADDQ $16*SIZE, ptrba;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
ADD2_DY yvec7, yvec12, yvec12;
|
|
ADDQ $32*SIZE, ptrbb;
|
|
DECQ k;
|
|
JG .L7_bodyB;
|
|
ALIGN_5
|
|
.L7_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $2, bk;
|
|
#else
|
|
TEST $2, kkk;
|
|
#endif
|
|
JLE .L8_loopE;
|
|
ALIGN_5
|
|
.L8_bodyB:
|
|
#### Unroll times 1 ####
|
|
LD_DY 0*SIZE(ptrba), yvec0;
|
|
EDUP_DY 0*SIZE(ptrbb), yvec2;
|
|
EDUP_DY 4*SIZE(ptrbb), yvec3;
|
|
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
EDUP_DY 1*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
ADD1_DY yvec7 ,yvec12, yvec12;
|
|
EDUP_DY 5*SIZE(ptrbb), yvec3
|
|
|
|
VPERMILP_DY $0x05, yvec0, yvec0;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
ADD2_DY yvec7, yvec12, yvec12;
|
|
|
|
#### Unroll time 2 ####
|
|
LD_DY 4*SIZE(ptrba), yvec0;
|
|
EDUP_DY 8*SIZE(ptrbb), yvec2;
|
|
EDUP_DY 12*SIZE(ptrbb), yvec3;
|
|
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
EDUP_DY 9*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
ADD1_DY yvec7 ,yvec12, yvec12;
|
|
EDUP_DY 13*SIZE(ptrbb), yvec3
|
|
|
|
VPERMILP_DY $0x05, yvec0, yvec0;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
ADDQ $8*SIZE, ptrba;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
ADD2_DY yvec7, yvec12, yvec12;
|
|
ADDQ $16*SIZE, ptrbb;
|
|
.L8_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $1, bk;
|
|
#else
|
|
TEST $1, kkk;
|
|
#endif
|
|
JLE .L9_loopE;
|
|
ALIGN_5
|
|
.L9_bodyB:
|
|
#### Unroll times 1 ####
|
|
LD_DY 0*SIZE(ptrba), yvec0;
|
|
EDUP_DY 0*SIZE(ptrbb), yvec2;
|
|
EDUP_DY 4*SIZE(ptrbb), yvec3;
|
|
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
EDUP_DY 1*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
ADD1_DY yvec7 ,yvec12, yvec12;
|
|
EDUP_DY 5*SIZE(ptrbb), yvec3
|
|
|
|
VPERMILP_DY $0x05, yvec0, yvec0;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_DY yvec0, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
MUL_DY yvec0, yvec5, yvec7;
|
|
ADD2_DY yvec7, yvec12, yvec12;
|
|
ADDQ $4*SIZE, ptrba;
|
|
ADDQ $8*SIZE, ptrbb;
|
|
|
|
.L9_loopE:
|
|
#### Handle ####
|
|
XOR_DY yvec7, yvec7, yvec7;
|
|
#if defined(RN) || defined(RT) || defined(CN) || defined(CT)
|
|
ADDSUB_DY yvec15, yvec7, yvec15;
|
|
ADDSUB_DY yvec14, yvec7, yvec14;
|
|
ADDSUB_DY yvec13, yvec7, yvec13;
|
|
ADDSUB_DY yvec12, yvec7, yvec12;
|
|
#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
|
|
SUB_DY yvec15, yvec7, yvec15;
|
|
SUB_DY yvec14, yvec7, yvec14;
|
|
SUB_DY yvec13, yvec7, yvec13;
|
|
SUB_DY yvec12, yvec7, yvec12;
|
|
#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
|
|
VPERMILP_DY $0x05, yvec15, yvec15;
|
|
VPERMILP_DY $0x05, yvec14, yvec14;
|
|
VPERMILP_DY $0x05, yvec13, yvec13;
|
|
VPERMILP_DY $0x05, yvec12, yvec12;
|
|
ADDSUB_DY yvec15, yvec7, yvec15;
|
|
ADDSUB_DY yvec14, yvec7, yvec14;
|
|
ADDSUB_DY yvec13, yvec7, yvec13;
|
|
ADDSUB_DY yvec12, yvec7, yvec12;
|
|
VPERMILP_DY $0x05, yvec15, yvec15;
|
|
VPERMILP_DY $0x05, yvec14, yvec14;
|
|
VPERMILP_DY $0x05, yvec13, yvec13;
|
|
VPERMILP_DY $0x05, yvec12, yvec12;
|
|
#endif
|
|
#### Load Alpha ####
|
|
BROAD_DY MEMALPHA_R, yvec7;
|
|
BROAD_DY MEMALPHA_I, yvec6;
|
|
#### Multiply Alpha ####
|
|
VPERMILP_DY $0x05, yvec15, yvec5;
|
|
MUL_DY yvec7, yvec15, yvec15;
|
|
MUL_DY yvec6, yvec5, yvec5;
|
|
ADD2_DY yvec5, yvec15, yvec15;
|
|
VPERMILP_DY $0x05, yvec14, yvec4;
|
|
MUL_DY yvec7, yvec14, yvec14;
|
|
MUL_DY yvec6, yvec4, yvec4;
|
|
ADD2_DY yvec4, yvec14, yvec14;
|
|
VPERMILP_DY $0x05, yvec13, yvec3;
|
|
MUL_DY yvec7, yvec13, yvec13;
|
|
MUL_DY yvec6, yvec3, yvec3;
|
|
ADD2_DY yvec3, yvec13, yvec13;
|
|
VPERMILP_DY $0x05,yvec12, yvec2;
|
|
MUL_DY yvec7, yvec12, yvec12;
|
|
MUL_DY yvec6, yvec2, yvec2;
|
|
ADD2_DY yvec2, yvec12, yvec12;
|
|
#### Testing Alignment ####
|
|
MOVQ C0, %rax;
|
|
OR ldc, %rax;
|
|
TEST $15, %rax;
|
|
JNE .L9_loopEx;
|
|
ALIGN_5
|
|
#### Writing back ####
|
|
EXTRA_DY $1, yvec15, xvec7;
|
|
EXTRA_DY $1, yvec14, xvec6;
|
|
EXTRA_DY $1, yvec13, xvec5;
|
|
EXTRA_DY $1, yvec12, xvec4;
|
|
#ifndef TRMMKERNEL
|
|
ADD_DX 0*SIZE(C0), xvec15, xvec15;
|
|
ADD_DX 2*SIZE(C0, ldc, 1), xvec7, xvec7;
|
|
ADD_DX 0*SIZE(C0, ldc, 1), xvec13, xvec13;
|
|
ADD_DX 2*SIZE(C0), xvec5, xvec5;
|
|
ADD_DX 0*SIZE(C1), xvec14, xvec14;
|
|
ADD_DX 2*SIZE(C1, ldc, 1), xvec6, xvec6;
|
|
ADD_DX 0*SIZE(C1, ldc, 1), xvec12, xvec12;
|
|
ADD_DX 2*SIZE(C1), xvec4, xvec4;
|
|
#endif
|
|
ST_DX xvec15, 0*SIZE(C0);
|
|
ST_DX xvec7, 2*SIZE(C0, ldc, 1);
|
|
ST_DX xvec13, 0*SIZE(C0, ldc, 1);
|
|
ST_DX xvec5, 2*SIZE(C0);
|
|
ST_DX xvec14, 0*SIZE(C1);
|
|
ST_DX xvec6, 2*SIZE(C1, ldc, 1);
|
|
ST_DX xvec12, 0*SIZE(C1, ldc, 1);
|
|
ST_DX xvec4, 2*SIZE(C1);
|
|
#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kkk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 2), ptrba;
|
|
LEAQ (ptrbb, %rax, 4), ptrbb;
|
|
#endif
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
ADDQ $2, kk;
|
|
#endif
|
|
ADDQ $4*SIZE, C0;
|
|
ADDQ $4*SIZE, C1;
|
|
JMP .L5_loopE;
|
|
ALIGN_5
|
|
.L9_loopEx:
|
|
EXTRA_DY $1, yvec15, xvec7;
|
|
EXTRA_DY $1, yvec14, xvec6;
|
|
EXTRA_DY $1, yvec13, xvec5;
|
|
EXTRA_DY $1, yvec12, xvec4;
|
|
#ifndef TRMMKERNEL
|
|
LDL_DX 0*SIZE(C0), xvec0, xvec0;
|
|
LDH_DX 1*SIZE(C0), xvec0, xvec0;
|
|
LDL_DX 2*SIZE(C0, ldc, 1), xvec1, xvec1;
|
|
LDH_DX 3*SIZE(C0, ldc, 1), xvec1, xvec1;
|
|
LDL_DX 0*SIZE(C0, ldc, 1), xvec2, xvec2;
|
|
LDH_DX 1*SIZE(C0, ldc, 1), xvec2, xvec2;
|
|
LDL_DX 2*SIZE(C0), xvec3, xvec3;
|
|
LDH_DX 3*SIZE(C0), xvec3, xvec3;
|
|
ADD_DX xvec0, xvec15, xvec15;
|
|
ADD_DX xvec1, xvec7, xvec7;
|
|
ADD_DX xvec2, xvec13, xvec13;
|
|
ADD_DX xvec3, xvec5, xvec5;
|
|
#endif
|
|
STL_DX xvec15, 0*SIZE(C0);
|
|
STH_DX xvec15, 1*SIZE(C0);
|
|
STL_DX xvec7, 2*SIZE(C0, ldc, 1);
|
|
STH_DX xvec7, 3*SIZE(C0, ldc, 1);
|
|
STL_DX xvec13, 0*SIZE(C0, ldc, 1);
|
|
STH_DX xvec13, 1*SIZE(C0, ldc, 1);
|
|
STL_DX xvec5, 2*SIZE(C0);
|
|
STH_DX xvec5, 3*SIZE(C0);
|
|
#ifndef TRMMKERNEL
|
|
LDL_DX 0*SIZE(C1), xvec0, xvec0;
|
|
LDH_DX 1*SIZE(C1), xvec0, xvec0;
|
|
LDL_DX 2*SIZE(C1, ldc, 1), xvec1, xvec1;
|
|
LDH_DX 3*SIZE(C1, ldc, 1), xvec1, xvec1;
|
|
LDL_DX 0*SIZE(C1, ldc, 1), xvec2, xvec2;
|
|
LDH_DX 1*SIZE(C1, ldc, 1), xvec2, xvec2;
|
|
LDL_DX 2*SIZE(C1), xvec3, xvec3;
|
|
LDH_DX 3*SIZE(C1), xvec3, xvec3;
|
|
ADD_DX xvec0, xvec14, xvec14;
|
|
ADD_DX xvec1, xvec6, xvec6;
|
|
ADD_DX xvec2, xvec12, xvec12;
|
|
ADD_DX xvec3, xvec4, xvec4;
|
|
#endif
|
|
STL_DX xvec14, 0*SIZE(C1);
|
|
STH_DX xvec14, 1*SIZE(C1);
|
|
STL_DX xvec6, 2*SIZE(C1, ldc, 1);
|
|
STH_DX xvec6, 3*SIZE(C1, ldc, 1);
|
|
STL_DX xvec12, 0*SIZE(C1, ldc, 1);
|
|
STH_DX xvec12, 1*SIZE(C1, ldc, 1);
|
|
STL_DX xvec4, 2*SIZE(C1);
|
|
STH_DX xvec4, 3*SIZE(C1);
|
|
#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kkk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 2), ptrba;
|
|
LEAQ (ptrbb, %rax, 4), ptrbb;
|
|
#endif
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
ADDQ $2, kk;
|
|
#endif
|
|
ADDQ $4*SIZE, C0;
|
|
ADDQ $4*SIZE, C1;
|
|
.L5_loopE:
|
|
TEST $1, bm;
|
|
JLE .L6_loopE;
|
|
ALIGN_5
|
|
.L6_bodyB:
|
|
#if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA))
|
|
MOVQ bb,ptrbb;
|
|
#else
|
|
MOVQ bb, ptrbb;
|
|
MOVQ kk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
ADDQ %rax, ptrba;
|
|
LEAQ (ptrbb, %rax, 4), ptrbb;
|
|
#endif
|
|
XOR_DY yvec15, yvec15, yvec15;
|
|
XOR_DY yvec14, yvec14, yvec14;
|
|
#ifndef TRMMKERNEL
|
|
MOVQ bk,k;
|
|
#elif (defined(LEFT)&&!defined(TRANSA))||(!defined(LEFT)&&defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kk, %rax;
|
|
MOVQ %rax, kkk;
|
|
#else
|
|
MOVQ kk, %rax;
|
|
#ifdef LEFT
|
|
ADDQ $1, %rax;
|
|
#else
|
|
ADDQ $4, %rax;
|
|
#endif
|
|
MOVQ %rax, kkk;
|
|
#endif
|
|
SARQ $2, k;
|
|
JLE .L10_loopE;
|
|
ALIGN_5
|
|
.L10_bodyB:
|
|
LD_DY 0*SIZE(ptrba), yvec0; #### A1r A1i A2r A2i
|
|
EDUP_DY 0*SIZE(ptrbb), yvec2;
|
|
EDUP_DY 4*SIZE(ptrbb), yvec3;
|
|
|
|
SHUF_DY $0x20, yvec0, yvec0, yvec1;
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec4;
|
|
EDUP_DY 1*SIZE(ptrbb), yvec2;
|
|
EDUP_DY 5*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec4, yvec2, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
MUL_DY yvec4, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
|
|
SHUF_DY $0x31, yvec0, yvec0, yvec1;
|
|
EDUP_DY 8*SIZE(ptrbb), yvec2;
|
|
EDUP_DY 12*SIZE(ptrbb), yvec3;
|
|
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec4;
|
|
EDUP_DY 9*SIZE(ptrbb), yvec2;
|
|
EDUP_DY 13*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec4, yvec2, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
MUL_DY yvec4, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
|
|
LD_DY 4*SIZE(ptrba), yvec0;
|
|
EDUP_DY 16*SIZE(ptrbb), yvec2;
|
|
EDUP_DY 20*SIZE(ptrbb), yvec3;
|
|
|
|
SHUF_DY $0x20, yvec0, yvec0, yvec1;
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec4;
|
|
EDUP_DY 17*SIZE(ptrbb), yvec2;
|
|
EDUP_DY 21*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec4, yvec2, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
MUL_DY yvec4, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
|
|
SHUF_DY $0x31, yvec0, yvec0, yvec1;
|
|
EDUP_DY 24*SIZE(ptrbb), yvec2;
|
|
EDUP_DY 28*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec4;
|
|
EDUP_DY 25*SIZE(ptrbb), yvec2;
|
|
EDUP_DY 29*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec4, yvec2, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
MUL_DY yvec4, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14
|
|
ADDQ $8*SIZE, ptrba;
|
|
ADDQ $32*SIZE, ptrbb;
|
|
DECQ k;
|
|
JG .L10_bodyB;
|
|
ALIGN_5
|
|
.L10_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $2, bk;
|
|
#else
|
|
TEST $2, kkk;
|
|
#endif
|
|
JLE .L11_loopE;
|
|
ALIGN_5
|
|
.L11_bodyB:
|
|
LD_DY 0*SIZE(ptrba), yvec0; #### A1r A1i A2r A2i
|
|
EDUP_DY 0*SIZE(ptrbb), yvec2;
|
|
EDUP_DY 4*SIZE(ptrbb), yvec3;
|
|
|
|
SHUF_DY $0x20, yvec0, yvec0, yvec1;
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec4;
|
|
EDUP_DY 1*SIZE(ptrbb), yvec2;
|
|
EDUP_DY 5*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec4, yvec2, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
MUL_DY yvec4, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
|
|
SHUF_DY $0x31, yvec0, yvec0, yvec1;
|
|
EDUP_DY 8*SIZE(ptrbb), yvec2;
|
|
EDUP_DY 12*SIZE(ptrbb), yvec3;
|
|
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec4;
|
|
EDUP_DY 9*SIZE(ptrbb), yvec2;
|
|
EDUP_DY 13*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec4, yvec2, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
MUL_DY yvec4, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
ADDQ $4*SIZE, ptrba;
|
|
ADDQ $16*SIZE, ptrbb;
|
|
|
|
.L11_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $1, bk;
|
|
#else
|
|
TEST $1, kkk;
|
|
#endif
|
|
JLE .L12_loopE;
|
|
ALIGN_5
|
|
.L12_bodyB:
|
|
LD_DY 0*SIZE(ptrba), yvec0; #### A1r A1i A2r A2i
|
|
EDUP_DY 0*SIZE(ptrbb), yvec2;
|
|
EDUP_DY 4*SIZE(ptrbb), yvec3;
|
|
|
|
SHUF_DY $0x20, yvec0, yvec0, yvec1;
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec4;
|
|
EDUP_DY 1*SIZE(ptrbb), yvec2;
|
|
EDUP_DY 5*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec4, yvec2, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
MUL_DY yvec4, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
ADDQ $2*SIZE, ptrba;
|
|
ADDQ $8*SIZE, ptrbb;
|
|
|
|
.L12_loopE:
|
|
#### Handle ####
|
|
XOR_DY yvec7, yvec7, yvec7;
|
|
#if defined(RN) || defined(RT) || defined(CN) || defined(CT)
|
|
ADDSUB_DY yvec15, yvec7, yvec15;
|
|
ADDSUB_DY yvec14, yvec7, yvec14;
|
|
#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
|
|
SUB_DY yvec15, yvec7, yvec15;
|
|
SUB_DY yvec14, yvec7, yvec14;
|
|
#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
|
|
VPERMILP_DY $0x05, yvec15, yvec15;
|
|
VPERMILP_DY $0x05, yvec14, yvec14;
|
|
ADDSUB_DY yvec15, yvec7, yvec15;
|
|
ADDSUB_DY yvec14, yvec7, yvec14;
|
|
VPERMILP_DY $0x05, yvec15, yvec15;
|
|
VPERMILP_DY $0x05, yvec14, yvec14;
|
|
#endif
|
|
#### Multiply Alpha ####
|
|
BROAD_DY MEMALPHA_R, yvec7;
|
|
BROAD_DY MEMALPHA_I, yvec6;
|
|
VPERMILP_DY $0x05, yvec15, yvec5;
|
|
MUL_DY yvec7, yvec15, yvec15;
|
|
MUL_DY yvec6, yvec5, yvec5;
|
|
ADD2_DY yvec5, yvec15, yvec15;
|
|
VPERMILP_DY $0x05, yvec14, yvec4;
|
|
MUL_DY yvec7, yvec14, yvec14;
|
|
MUL_DY yvec6, yvec4, yvec4;
|
|
ADD2_DY yvec4, yvec14, yvec14;
|
|
#### Writing Back ####
|
|
EXTRA_DY $1, yvec15, xvec7;
|
|
EXTRA_DY $1, yvec14, xvec6;
|
|
#ifndef TRMMKERNEL
|
|
LDL_DX 0*SIZE(C0), xvec0, xvec0;
|
|
LDH_DX 1*SIZE(C0), xvec0, xvec0;
|
|
LDL_DX 0*SIZE(C0, ldc, 1), xvec1, xvec1;
|
|
LDH_DX 1*SIZE(C0, ldc, 1), xvec1, xvec1;
|
|
LDL_DX 0*SIZE(C1), xvec2, xvec2;
|
|
LDH_DX 1*SIZE(C1), xvec2, xvec2;
|
|
LDL_DX 0*SIZE(C1, ldc, 1), xvec3, xvec3;
|
|
LDH_DX 1*SIZE(C1, ldc, 1), xvec3, xvec3;
|
|
ADD_DX xvec0, xvec15, xvec15;
|
|
ADD_DX xvec1, xvec7, xvec7;
|
|
ADD_DX xvec2, xvec14, xvec14;
|
|
ADD_DX xvec3, xvec6, xvec6;
|
|
#endif
|
|
STL_DX xvec15, 0*SIZE(C0);
|
|
STH_DX xvec15, 1*SIZE(C0);
|
|
STL_DX xvec7, 0*SIZE(C0, ldc, 1);
|
|
STH_DX xvec7, 1*SIZE(C0, ldc, 1);
|
|
STL_DX xvec14, 0*SIZE(C1);
|
|
STH_DX xvec14, 1*SIZE(C1);
|
|
STL_DX xvec6, 0*SIZE(C1, ldc, 1);
|
|
STH_DX xvec6, 1*SIZE(C1, ldc, 1);
|
|
#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kkk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
ADDQ %rax, ptrba;
|
|
LEAQ (ptrbb, %rax, 4), ptrbb;
|
|
#endif
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
ADDQ $1, kk;
|
|
#endif
|
|
ADDQ $2*SIZE, C0;
|
|
ADDQ $2*SIZE, C1;
|
|
.L6_loopE:
|
|
#if defined(TRMMKERNEL) && !defined(LEFT)
|
|
ADDQ $4, kk;
|
|
#endif
|
|
MOVQ bk,k;
|
|
SALQ $6,k;
|
|
ADDQ k,bb;
|
|
LEAQ (C,ldc,4),C;
|
|
.L0_bodyE:;
|
|
DECQ j;
|
|
JG .L0_bodyB;
|
|
ALIGN_5;
|
|
.L0_loopE:;
|
|
TEST $2, bn;
|
|
JLE .L20_loopE;
|
|
ALIGN_5
|
|
.L20_bodyB:
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
MOVQ OFFSET, %rax;
|
|
MOVQ %rax, kk;
|
|
#endif
|
|
MOVQ C, C0;
|
|
LEAQ (C, ldc, 1), C1;
|
|
MOVQ ba, ptrba;
|
|
MOVQ bm, i;
|
|
SARQ $2, i;
|
|
JLE .L21_loopE;
|
|
ALIGN_5
|
|
.L21_bodyB:
|
|
#if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA))
|
|
MOVQ bb,ptrbb;
|
|
#else
|
|
MOVQ bb, ptrbb;
|
|
MOVQ kk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 4), ptrba;
|
|
LEAQ (ptrbb, %rax, 2), ptrbb;
|
|
#endif
|
|
XOR_DY yvec15, yvec15, yvec15;
|
|
XOR_DY yvec14, yvec14, yvec14;
|
|
XOR_DY yvec13, yvec13, yvec13;
|
|
XOR_DY yvec12, yvec12, yvec12;
|
|
#ifndef TRMMKERNEL
|
|
MOVQ bk,k;
|
|
#elif (defined(LEFT)&&!defined(TRANSA))||(!defined(LEFT)&&defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kk, %rax;
|
|
MOVQ %rax, kkk;
|
|
#else
|
|
MOVQ kk, %rax;
|
|
#ifdef LEFT
|
|
ADDQ $4, %rax;
|
|
#else
|
|
ADDQ $2, %rax;
|
|
#endif
|
|
MOVQ %rax, kkk;
|
|
#endif
|
|
SARQ $2, k;
|
|
JLE .L211_loopE;
|
|
ALIGN_5
|
|
.L211_bodyB:
|
|
#### Unroll time 1 ####
|
|
EDUP_DY 0*SIZE(ptrbb), yvec2;
|
|
LD_DY 0*SIZE(ptrba), yvec0;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
LD_DY 4*SIZE(ptrba), yvec1;
|
|
MUL_DY yvec1, yvec2, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
|
|
EDUP_DY 1*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
VPERMILP_DY $0x05, yvec0, yvec0;
|
|
MUL_DY yvec1, yvec4, yvec7;
|
|
ADD1_DY yvec7, yvec12, yvec12;
|
|
VPERMILP_DY $0x05, yvec1, yvec1;
|
|
|
|
MUL_DY yvec0, yvec3, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
|
|
MUL_DY yvec0, yvec5, yvec6;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
MUL_DY yvec1, yvec5, yvec7;
|
|
ADD2_DY yvec7, yvec12, yvec12;
|
|
|
|
#### Unroll time 2 ####
|
|
EDUP_DY 4*SIZE(ptrbb), yvec2;
|
|
LD_DY 8*SIZE(ptrba), yvec0;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
LD_DY 12*SIZE(ptrba), yvec1;
|
|
MUL_DY yvec1, yvec2, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
|
|
EDUP_DY 5*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
VPERMILP_DY $0x05, yvec0, yvec0;
|
|
MUL_DY yvec1, yvec4, yvec7;
|
|
ADD1_DY yvec7, yvec12, yvec12;
|
|
VPERMILP_DY $0x05, yvec1, yvec1;
|
|
|
|
MUL_DY yvec0, yvec3, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
|
|
MUL_DY yvec0, yvec5, yvec6;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
MUL_DY yvec1, yvec5, yvec7;
|
|
ADD2_DY yvec7, yvec12, yvec12;
|
|
|
|
#### Unroll time 3 ####
|
|
EDUP_DY 8*SIZE(ptrbb), yvec2;
|
|
LD_DY 16*SIZE(ptrba), yvec0;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
LD_DY 20*SIZE(ptrba), yvec1;
|
|
MUL_DY yvec1, yvec2, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
|
|
EDUP_DY 9*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
VPERMILP_DY $0x05, yvec0, yvec0;
|
|
MUL_DY yvec1, yvec4, yvec7;
|
|
ADD1_DY yvec7, yvec12, yvec12;
|
|
VPERMILP_DY $0x05, yvec1, yvec1;
|
|
|
|
MUL_DY yvec0, yvec3, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
|
|
MUL_DY yvec0, yvec5, yvec6;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
MUL_DY yvec1, yvec5, yvec7;
|
|
ADD2_DY yvec7, yvec12, yvec12;
|
|
|
|
#### Unroll time 4 ####
|
|
EDUP_DY 12*SIZE(ptrbb), yvec2;
|
|
LD_DY 24*SIZE(ptrba), yvec0;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
LD_DY 28*SIZE(ptrba), yvec1;
|
|
MUL_DY yvec1, yvec2, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
|
|
EDUP_DY 13*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
VPERMILP_DY $0x05, yvec0, yvec0;
|
|
MUL_DY yvec1, yvec4, yvec7;
|
|
ADD1_DY yvec7, yvec12, yvec12;
|
|
VPERMILP_DY $0x05, yvec1, yvec1;
|
|
|
|
MUL_DY yvec0, yvec3, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
ADDQ $16*SIZE, ptrbb;
|
|
|
|
MUL_DY yvec0, yvec5, yvec6;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
MUL_DY yvec1, yvec5, yvec7;
|
|
ADD2_DY yvec7, yvec12, yvec12;
|
|
ADDQ $32*SIZE, ptrba;
|
|
DECQ k;
|
|
JG .L211_bodyB;
|
|
ALIGN_5
|
|
.L211_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $2, bk;
|
|
#else
|
|
TEST $2, kkk;
|
|
#endif
|
|
JLE .L212_loopE;
|
|
ALIGN_5
|
|
.L212_bodyB:
|
|
#### Unroll time 1 ####
|
|
EDUP_DY 0*SIZE(ptrbb), yvec2;
|
|
LD_DY 0*SIZE(ptrba), yvec0;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
LD_DY 4*SIZE(ptrba), yvec1;
|
|
MUL_DY yvec1, yvec2, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
|
|
EDUP_DY 1*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
VPERMILP_DY $0x05, yvec0, yvec0;
|
|
MUL_DY yvec1, yvec4, yvec7;
|
|
ADD1_DY yvec7, yvec12, yvec12;
|
|
VPERMILP_DY $0x05, yvec1, yvec1;
|
|
|
|
MUL_DY yvec0, yvec3, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
|
|
MUL_DY yvec0, yvec5, yvec6;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
MUL_DY yvec1, yvec5, yvec7;
|
|
ADD2_DY yvec7, yvec12, yvec12;
|
|
|
|
#### Unroll time 2 ####
|
|
EDUP_DY 4*SIZE(ptrbb), yvec2;
|
|
LD_DY 8*SIZE(ptrba), yvec0;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
LD_DY 12*SIZE(ptrba), yvec1;
|
|
MUL_DY yvec1, yvec2, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
|
|
EDUP_DY 5*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
VPERMILP_DY $0x05, yvec0, yvec0;
|
|
MUL_DY yvec1, yvec4, yvec7;
|
|
ADD1_DY yvec7, yvec12, yvec12;
|
|
VPERMILP_DY $0x05, yvec1, yvec1;
|
|
|
|
MUL_DY yvec0, yvec3, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
|
|
MUL_DY yvec0, yvec5, yvec6;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
MUL_DY yvec1, yvec5, yvec7;
|
|
ADD2_DY yvec7, yvec12, yvec12;
|
|
|
|
ADDQ $8*SIZE, ptrbb;
|
|
ADDQ $16*SIZE, ptrba;
|
|
|
|
.L212_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $1, bk;
|
|
#else
|
|
TEST $1, kkk;
|
|
#endif
|
|
JLE .L213_loopE;
|
|
ALIGN_5
|
|
.L213_bodyB:
|
|
#### Unroll time 1 ####
|
|
EDUP_DY 0*SIZE(ptrbb), yvec2;
|
|
LD_DY 0*SIZE(ptrba), yvec0;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
LD_DY 4*SIZE(ptrba), yvec1;
|
|
MUL_DY yvec1, yvec2, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
|
|
EDUP_DY 1*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
VPERMILP_DY $0x05, yvec0, yvec0;
|
|
MUL_DY yvec1, yvec4, yvec7;
|
|
ADD1_DY yvec7, yvec12, yvec12;
|
|
VPERMILP_DY $0x05, yvec1, yvec1;
|
|
|
|
MUL_DY yvec0, yvec3, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
|
|
MUL_DY yvec0, yvec5, yvec6;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
MUL_DY yvec1, yvec5, yvec7;
|
|
ADD2_DY yvec7, yvec12, yvec12;
|
|
ADDQ $4*SIZE, ptrbb;
|
|
ADDQ $8*SIZE, ptrba;
|
|
|
|
.L213_loopE:
|
|
#### Handle ####
|
|
XOR_DY yvec7, yvec7, yvec7;
|
|
#if defined(RN) || defined(RT) || defined(CN) || defined(CT)
|
|
ADDSUB_DY yvec15, yvec7, yvec15;
|
|
ADDSUB_DY yvec14, yvec7, yvec14;
|
|
ADDSUB_DY yvec13, yvec7, yvec13;
|
|
ADDSUB_DY yvec12, yvec7, yvec12;
|
|
#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
|
|
SUB_DY yvec15, yvec7, yvec15;
|
|
SUB_DY yvec14, yvec7, yvec14;
|
|
SUB_DY yvec13, yvec7, yvec13;
|
|
SUB_DY yvec12, yvec7, yvec12;
|
|
#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
|
|
VPERMILP_DY $0x05, yvec15, yvec15;
|
|
VPERMILP_DY $0x05, yvec14, yvec14;
|
|
VPERMILP_DY $0x05, yvec13, yvec13;
|
|
VPERMILP_DY $0x05, yvec12, yvec12;
|
|
ADDSUB_DY yvec15, yvec7, yvec15;
|
|
ADDSUB_DY yvec14, yvec7, yvec14;
|
|
ADDSUB_DY yvec13, yvec7, yvec13;
|
|
ADDSUB_DY yvec12, yvec7, yvec12;
|
|
VPERMILP_DY $0x05, yvec15, yvec15;
|
|
VPERMILP_DY $0x05, yvec14, yvec14;
|
|
VPERMILP_DY $0x05, yvec13, yvec13;
|
|
VPERMILP_DY $0x05, yvec12, yvec12;
|
|
#endif
|
|
#### Load Alpha ####
|
|
BROAD_DY MEMALPHA_R,yvec7;
|
|
BROAD_DY MEMALPHA_I,yvec6;
|
|
#### Multiply Alpha ####
|
|
VPERMILP_DY $0x05, yvec15, yvec5;
|
|
MUL_DY yvec7, yvec15, yvec15;
|
|
MUL_DY yvec6, yvec5, yvec5;
|
|
ADD2_DY yvec5, yvec15, yvec15;
|
|
VPERMILP_DY $0x05, yvec14, yvec4;
|
|
MUL_DY yvec7, yvec14, yvec14;
|
|
MUL_DY yvec6, yvec4, yvec4;
|
|
ADD2_DY yvec4, yvec14, yvec14;
|
|
VPERMILP_DY $0x05, yvec13, yvec3;
|
|
MUL_DY yvec7, yvec13, yvec13;
|
|
MUL_DY yvec6, yvec3, yvec3;
|
|
ADD2_DY yvec3, yvec13, yvec13;
|
|
VPERMILP_DY $0x05,yvec12, yvec2;
|
|
MUL_DY yvec7, yvec12, yvec12;
|
|
MUL_DY yvec6, yvec2, yvec2;
|
|
ADD2_DY yvec2, yvec12, yvec12;
|
|
EXTRA_DY $1, yvec15, xvec7;
|
|
EXTRA_DY $1, yvec14, xvec6;
|
|
EXTRA_DY $1, yvec13, xvec5;
|
|
EXTRA_DY $1, yvec12, xvec4;
|
|
#### Testing Alignment ####
|
|
MOVQ C0, %rax;
|
|
OR ldc, %rax;
|
|
TEST $15, %rax;
|
|
JNE .L213_loopEx;
|
|
ALIGN_5
|
|
#### Writing back ####
|
|
#ifndef TRMMKERNEL
|
|
ADD_DX 0*SIZE(C0), xvec15, xvec15;
|
|
ADD_DX 2*SIZE(C1), xvec7, xvec7;
|
|
ADD_DX 4*SIZE(C0), xvec14, xvec14;
|
|
ADD_DX 6*SIZE(C1), xvec6, xvec6;
|
|
ADD_DX 0*SIZE(C1), xvec13, xvec13;
|
|
ADD_DX 2*SIZE(C0), xvec5, xvec5;
|
|
ADD_DX 4*SIZE(C1), xvec12, xvec12;
|
|
ADD_DX 6*SIZE(C0), xvec4, xvec4;
|
|
#endif
|
|
ST_DX xvec15,0*SIZE(C0);
|
|
ST_DX xvec7,2*SIZE(C1);
|
|
ST_DX xvec14,4*SIZE(C0);
|
|
ST_DX xvec6,6*SIZE(C1);
|
|
ST_DX xvec13,0*SIZE(C1);
|
|
ST_DX xvec5,2*SIZE(C0);
|
|
ST_DX xvec12,4*SIZE(C1);
|
|
ST_DX xvec4,6*SIZE(C0);
|
|
#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kkk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 4), ptrba;
|
|
LEAQ (ptrbb, %rax, 2), ptrbb;
|
|
#endif
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
ADDQ $4, kk;
|
|
#endif
|
|
ADDQ $8*SIZE, C0;
|
|
ADDQ $8*SIZE, C1;
|
|
DECQ i;
|
|
JG .L21_bodyB;
|
|
JMP .L21_loopE;
|
|
ALIGN_5
|
|
.L213_loopEx:
|
|
#ifndef TRMMKERNEL
|
|
LDL_DX 0*SIZE(C0), xvec0, xvec0;
|
|
LDH_DX 1*SIZE(C0), xvec0, xvec0;
|
|
LDL_DX 2*SIZE(C1), xvec1, xvec1;
|
|
LDH_DX 3*SIZE(C1), xvec1, xvec1;
|
|
LDL_DX 4*SIZE(C0), xvec2, xvec2;
|
|
LDH_DX 5*SIZE(C0), xvec2, xvec2;
|
|
LDL_DX 6*SIZE(C1), xvec3, xvec3;
|
|
LDH_DX 7*SIZE(C1), xvec3, xvec3;
|
|
ADD_DX xvec0, xvec15, xvec15;
|
|
ADD_DX xvec1, xvec7, xvec7;
|
|
ADD_DX xvec2, xvec14, xvec14;
|
|
ADD_DX xvec3, xvec6, xvec6;
|
|
#endif
|
|
STL_DX xvec15, 0*SIZE(C0);
|
|
STH_DX xvec15, 1*SIZE(C0);
|
|
STL_DX xvec7, 2*SIZE(C1);
|
|
STH_DX xvec7, 3*SIZE(C1);
|
|
STL_DX xvec14, 4*SIZE(C0);
|
|
STH_DX xvec14, 5*SIZE(C0);
|
|
STL_DX xvec6, 6*SIZE(C1);
|
|
STH_DX xvec6, 7*SIZE(C1);
|
|
#ifndef TRMMKERNEL
|
|
LDL_DX 0*SIZE(C1), xvec3, xvec3;
|
|
LDH_DX 1*SIZE(C1), xvec3, xvec3;
|
|
LDL_DX 2*SIZE(C0), xvec2, xvec2;
|
|
LDH_DX 3*SIZE(C0), xvec2, xvec2;
|
|
LDL_DX 4*SIZE(C1), xvec1, xvec1;
|
|
LDH_DX 5*SIZE(C1), xvec1, xvec1;
|
|
LDL_DX 6*SIZE(C0), xvec0, xvec0;
|
|
LDH_DX 7*SIZE(C0), xvec0, xvec0;
|
|
ADD_DX xvec3, xvec13, xvec13;
|
|
ADD_DX xvec2, xvec5, xvec5;
|
|
ADD_DX xvec1, xvec12, xvec12;
|
|
ADD_DX xvec0, xvec4, xvec4;
|
|
#endif
|
|
STL_DX xvec13, 0*SIZE(C1);
|
|
STH_DX xvec13, 1*SIZE(C1);
|
|
STL_DX xvec5, 2*SIZE(C0);
|
|
STH_DX xvec5, 3*SIZE(C0);
|
|
STL_DX xvec12, 4*SIZE(C1);
|
|
STH_DX xvec12, 5*SIZE(C1);
|
|
STL_DX xvec4, 6*SIZE(C0);
|
|
STH_DX xvec4, 7*SIZE(C0);
|
|
#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kkk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 4), ptrba;
|
|
LEAQ (ptrbb, %rax, 2), ptrbb;
|
|
#endif
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
ADDQ $4, kk;
|
|
#endif
|
|
ADDQ $8*SIZE, C0;
|
|
ADDQ $8*SIZE, C1;
|
|
DECQ i;
|
|
JG .L21_bodyB;
|
|
ALIGN_5
|
|
.L21_loopE:
|
|
TEST $2, bm;
|
|
JLE .L22_loopE;
|
|
ALIGN_5
|
|
.L22_bodyB:
|
|
#if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA))
|
|
MOVQ bb,ptrbb;
|
|
#else
|
|
MOVQ bb, ptrbb;
|
|
MOVQ kk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 2), ptrba;
|
|
LEAQ (ptrbb, %rax, 2), ptrbb;
|
|
#endif
|
|
XOR_DY yvec15, yvec15, yvec15;
|
|
XOR_DY yvec14, yvec14, yvec13;
|
|
#ifndef TRMMKERNEL
|
|
MOVQ bk,k;
|
|
#elif (defined(LEFT)&&!defined(TRANSA))||(!defined(LEFT)&&defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kk, %rax;
|
|
MOVQ %rax, kkk;
|
|
#else
|
|
MOVQ kk, %rax;
|
|
#ifdef LEFT
|
|
ADDQ $2, %rax;
|
|
#else
|
|
ADDQ $2, %rax;
|
|
#endif
|
|
MOVQ %rax, kkk;
|
|
#endif
|
|
SARQ $2, k;
|
|
JLE .L221_loopE;
|
|
ALIGN_5
|
|
.L221_bodyB:
|
|
#### Unroll time 1 ####
|
|
EDUP_DY 0*SIZE(ptrbb), yvec2;
|
|
LD_DY 0*SIZE(ptrba), yvec0;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
|
|
EDUP_DY 1*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
VPERMILP_DY $0x05, yvec0, yvec0;
|
|
|
|
MUL_DY yvec0, yvec3, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
|
|
MUL_DY yvec0, yvec5, yvec6;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
|
|
#### Unroll time 2 ####
|
|
EDUP_DY 4*SIZE(ptrbb), yvec2;
|
|
LD_DY 4*SIZE(ptrba), yvec0;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
|
|
EDUP_DY 5*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
VPERMILP_DY $0x05, yvec0, yvec0;
|
|
|
|
MUL_DY yvec0, yvec3, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
|
|
MUL_DY yvec0, yvec5, yvec6;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
|
|
#### Unroll time 3 ####
|
|
EDUP_DY 8*SIZE(ptrbb), yvec2;
|
|
LD_DY 8*SIZE(ptrba), yvec0;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
|
|
EDUP_DY 9*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
VPERMILP_DY $0x05, yvec0, yvec0;
|
|
|
|
MUL_DY yvec0, yvec3, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
|
|
MUL_DY yvec0, yvec5, yvec6;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
|
|
#### Unroll time 4 ####
|
|
EDUP_DY 12*SIZE(ptrbb), yvec2;
|
|
LD_DY 12*SIZE(ptrba), yvec0;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
|
|
EDUP_DY 13*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
VPERMILP_DY $0x05, yvec0, yvec0;
|
|
|
|
MUL_DY yvec0, yvec3, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
ADDQ $16*SIZE, ptrbb;
|
|
|
|
MUL_DY yvec0, yvec5, yvec6;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
ADDQ $16*SIZE, ptrba;
|
|
DECQ k;
|
|
JG .L221_bodyB;
|
|
ALIGN_5
|
|
.L221_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $2, bk;
|
|
#else
|
|
TEST $2, kkk;
|
|
#endif
|
|
JLE .L222_loopE;
|
|
ALIGN_5
|
|
.L222_bodyB:
|
|
#### Unroll time 1 ####
|
|
EDUP_DY 0*SIZE(ptrbb), yvec2;
|
|
LD_DY 0*SIZE(ptrba), yvec0;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
|
|
EDUP_DY 1*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
VPERMILP_DY $0x05, yvec0, yvec0;
|
|
|
|
MUL_DY yvec0, yvec3, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
|
|
MUL_DY yvec0, yvec5, yvec6;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
|
|
#### Unroll time 2 ####
|
|
EDUP_DY 4*SIZE(ptrbb), yvec2;
|
|
LD_DY 4*SIZE(ptrba), yvec0;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
|
|
EDUP_DY 5*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
VPERMILP_DY $0x05, yvec0, yvec0;
|
|
|
|
MUL_DY yvec0, yvec3, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
|
|
MUL_DY yvec0, yvec5, yvec6;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
ADDQ $8*SIZE, ptrba;
|
|
ADDQ $8*SIZE, ptrbb;
|
|
|
|
.L222_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $1, bk;
|
|
#else
|
|
TEST $1, kkk;
|
|
#endif
|
|
JLE .L223_loopE;
|
|
ALIGN_5
|
|
.L223_bodyB:
|
|
#### Unroll time 1 ####
|
|
EDUP_DY 0*SIZE(ptrbb), yvec2;
|
|
LD_DY 0*SIZE(ptrba), yvec0;
|
|
SHUF_DY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
|
|
EDUP_DY 1*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec0, yvec4, yvec6;
|
|
ADD1_DY yvec6, yvec13, yvec13;
|
|
VPERMILP_DY $0x05, yvec0, yvec0;
|
|
|
|
MUL_DY yvec0, yvec3, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
SHUF_DY $0x03, yvec3, yvec3, yvec5;
|
|
|
|
MUL_DY yvec0, yvec5, yvec6;
|
|
ADD2_DY yvec6, yvec13, yvec13;
|
|
ADDQ $4*SIZE, ptrba;
|
|
ADDQ $4*SIZE, ptrbb;
|
|
|
|
.L223_loopE:
|
|
#### Handle ####
|
|
XOR_DY yvec7, yvec7, yvec7;
|
|
#if defined(RN) || defined(RT) || defined(CN) || defined(CT)
|
|
ADDSUB_DY yvec15, yvec7, yvec15;
|
|
ADDSUB_DY yvec13, yvec7, yvec13;
|
|
#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
|
|
SUB_DY yvec15, yvec7, yvec15;
|
|
SUB_DY yvec13, yvec7, yvec13;
|
|
#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
|
|
VPERMILP_DY $0x05, yvec15, yvec15;
|
|
VPERMILP_DY $0x05, yvec13, yvec13;
|
|
ADDSUB_DY yvec15, yvec7, yvec15;
|
|
ADDSUB_DY yvec13, yvec7, yvec13;
|
|
VPERMILP_DY $0x05, yvec15, yvec15;
|
|
VPERMILP_DY $0x05, yvec13, yvec13;
|
|
#endif
|
|
|
|
#### Load Alpha ####
|
|
BROAD_DY MEMALPHA_R,yvec7;
|
|
BROAD_DY MEMALPHA_I,yvec6;
|
|
#### Multiply Alpha ####
|
|
VPERMILP_DY $0x05, yvec15, yvec5;
|
|
MUL_DY yvec7, yvec15, yvec15;
|
|
MUL_DY yvec6, yvec5, yvec5;
|
|
ADD2_DY yvec5, yvec15, yvec15;
|
|
VPERMILP_DY $0x05, yvec13, yvec3;
|
|
MUL_DY yvec7, yvec13, yvec13;
|
|
MUL_DY yvec6, yvec3, yvec3;
|
|
ADD2_DY yvec3, yvec13, yvec13;
|
|
EXTRA_DY $1, yvec15, xvec7;
|
|
EXTRA_DY $1, yvec13, xvec5;
|
|
#### Write back ####
|
|
#ifndef TRMMKERNEL
|
|
LDL_DX 0*SIZE(C0), xvec0, xvec0;
|
|
LDH_DX 1*SIZE(C0), xvec0, xvec0;
|
|
LDL_DX 2*SIZE(C1), xvec1, xvec1;
|
|
LDH_DX 3*SIZE(C1), xvec1, xvec1;
|
|
LDL_DX 0*SIZE(C1), xvec2, xvec2;
|
|
LDH_DX 1*SIZE(C1), xvec2, xvec2;
|
|
LDL_DX 2*SIZE(C0), xvec3, xvec3;
|
|
LDH_DX 3*SIZE(C0), xvec3, xvec3;
|
|
ADD_DX xvec0, xvec15, xvec15;
|
|
ADD_DX xvec1, xvec7, xvec7;
|
|
ADD_DX xvec2, xvec13, xvec13;
|
|
ADD_DX xvec3, xvec5, xvec5;
|
|
#endif
|
|
STL_DX xvec15, 0*SIZE(C0);
|
|
STH_DX xvec15, 1*SIZE(C0);
|
|
STL_DX xvec7, 2*SIZE(C1);
|
|
STH_DX xvec7, 3*SIZE(C1);
|
|
STL_DX xvec13, 0*SIZE(C1);
|
|
STH_DX xvec13, 1*SIZE(C1);
|
|
STL_DX xvec5, 2*SIZE(C0);
|
|
STH_DX xvec5, 3*SIZE(C0);
|
|
#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kkk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 2), ptrba;
|
|
LEAQ (ptrbb, %rax, 2), ptrbb;
|
|
#endif
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
ADDQ $2, kk;
|
|
#endif
|
|
ADDQ $4*SIZE, C0;
|
|
ADDQ $4*SIZE, C1;
|
|
|
|
.L22_loopE:
|
|
TEST $1, bm;
|
|
JLE .L23_loopE;
|
|
ALIGN_5
|
|
.L23_bodyB:
|
|
#if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA))
|
|
MOVQ bb,ptrbb;
|
|
#else
|
|
MOVQ bb, ptrbb;
|
|
MOVQ kk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
ADDQ %rax, ptrba;
|
|
LEAQ (ptrbb, %rax, 2), ptrbb;
|
|
#endif
|
|
XOR_DY yvec15, yvec15, yvec15;
|
|
#ifndef TRMMKERNEL
|
|
MOVQ bk,k;
|
|
#elif (defined(LEFT)&&!defined(TRANSA))||(!defined(LEFT)&&defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kk, %rax;
|
|
MOVQ %rax, kkk;
|
|
#else
|
|
MOVQ kk, %rax;
|
|
#ifdef LEFT
|
|
ADDQ $1, %rax;
|
|
#else
|
|
ADDQ $2, %rax;
|
|
#endif
|
|
MOVQ %rax, kkk;
|
|
#endif
|
|
SARQ $2, k;
|
|
JLE .L231_loopE;
|
|
ALIGN_5
|
|
.L231_bodyB:
|
|
LD_DY 0*SIZE(ptrba), yvec0; #### A1r A1i A2r A2i
|
|
EDUP_DY 0*SIZE(ptrbb), yvec2;
|
|
|
|
SHUF_DY $0x20, yvec0, yvec0, yvec1;
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec4;
|
|
EDUP_DY 1*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec4, yvec2, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
|
|
SHUF_DY $0x31, yvec0, yvec0, yvec1;
|
|
EDUP_DY 4*SIZE(ptrbb), yvec2;
|
|
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec4;
|
|
EDUP_DY 5*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec4, yvec2, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
|
|
LD_DY 4*SIZE(ptrba), yvec0;
|
|
EDUP_DY 8*SIZE(ptrbb), yvec2;
|
|
|
|
SHUF_DY $0x20, yvec0, yvec0, yvec1;
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec4;
|
|
EDUP_DY 9*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec4, yvec2, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
|
|
SHUF_DY $0x31, yvec0, yvec0, yvec1;
|
|
EDUP_DY 12*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec4;
|
|
EDUP_DY 13*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec4, yvec2, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
ADDQ $8*SIZE, ptrba;
|
|
ADDQ $16*SIZE, ptrbb;
|
|
DECQ k;
|
|
JG .L231_bodyB;
|
|
ALIGN_5
|
|
.L231_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $2, bk;
|
|
#else
|
|
TEST $2, kkk;
|
|
#endif
|
|
JLE .L232_loopE;
|
|
ALIGN_5
|
|
.L232_bodyB:
|
|
LD_DY 0*SIZE(ptrba), yvec0; #### A1r A1i A2r A2i
|
|
EDUP_DY 0*SIZE(ptrbb), yvec2;
|
|
|
|
SHUF_DY $0x20, yvec0, yvec0, yvec1;
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec4;
|
|
EDUP_DY 1*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec4, yvec2, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
|
|
SHUF_DY $0x31, yvec0, yvec0, yvec1;
|
|
EDUP_DY 4*SIZE(ptrbb), yvec2;
|
|
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec4;
|
|
EDUP_DY 5*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec4, yvec2, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
ADDQ $4*SIZE, ptrba;
|
|
ADDQ $8*SIZE, ptrbb;
|
|
|
|
.L232_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $1, bk;
|
|
#else
|
|
TEST $1, kkk;
|
|
#endif
|
|
JLE .L233_loopE;
|
|
ALIGN_5
|
|
.L233_bodyB:
|
|
LD_DY 0*SIZE(ptrba), yvec0; #### A1r A1i A2r A2i
|
|
EDUP_DY 0*SIZE(ptrbb), yvec2;
|
|
|
|
SHUF_DY $0x20, yvec0, yvec0, yvec1;
|
|
MUL_DY yvec1, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec4;
|
|
EDUP_DY 1*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec4, yvec2, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
ADDQ $2*SIZE, ptrba;
|
|
ADDQ $4*SIZE, ptrbb;
|
|
|
|
.L233_loopE:
|
|
#### Handle ####
|
|
XOR_DY yvec7, yvec7, yvec7;
|
|
#if defined(RN) || defined(RT) || defined(CN) || defined(CT)
|
|
ADDSUB_DY yvec15, yvec7, yvec15;
|
|
#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
|
|
SUB_DY yvec15, yvec7, yvec15;
|
|
#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
|
|
VPERMILP_DY $0x05, yvec15, yvec15;
|
|
ADDSUB_DY yvec15, yvec7, yvec15;
|
|
VPERMILP_DY $0x05, yvec15, yvec15;
|
|
#endif
|
|
|
|
#### Multiply Alpha ####
|
|
BROAD_DY MEMALPHA_R, yvec7;
|
|
BROAD_DY MEMALPHA_I, yvec6;
|
|
#### Writing Back ####
|
|
VPERMILP_DY $0x05, yvec15, yvec5;
|
|
MUL_DY yvec7, yvec15, yvec15;
|
|
MUL_DY yvec6, yvec5, yvec5;
|
|
ADD2_DY yvec5, yvec15, yvec15;
|
|
EXTRA_DY $1, yvec15, xvec7;
|
|
#### Writing Back ####
|
|
#ifndef TRMMKERNEL
|
|
LDL_DX 0*SIZE(C0), xvec0, xvec0;
|
|
LDH_DX 1*SIZE(C0), xvec0, xvec0;
|
|
LDL_DX 0*SIZE(C1), xvec1, xvec1;
|
|
LDH_DX 1*SIZE(C1), xvec1, xvec1;
|
|
ADD_DX xvec0, xvec15, xvec15;
|
|
ADD_DX xvec1, xvec7, xvec7;
|
|
#endif
|
|
STL_DX xvec15, 0*SIZE(C0);
|
|
STH_DX xvec15, 1*SIZE(C0);
|
|
STL_DX xvec7, 0*SIZE(C1);
|
|
STH_DX xvec7, 1*SIZE(C1);
|
|
#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kkk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
ADDQ %rax, ptrba;
|
|
LEAQ (ptrbb, %rax, 2), ptrbb;
|
|
#endif
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
ADDQ $1, kk;
|
|
#endif
|
|
ADDQ $2*SIZE, C0;
|
|
ADDQ $2*SIZE, C0;
|
|
.L23_loopE:
|
|
#if defined(TRMMKERNEL) && !defined(LEFT)
|
|
ADDQ $2, kk;
|
|
#endif
|
|
MOVQ bk, k;
|
|
SALQ $5, k;
|
|
ADDQ k, bb;
|
|
LEAQ (C, ldc, 2), C;
|
|
.L20_loopE:
|
|
TEST $1, bn;
|
|
JLE .L30_loopE;
|
|
ALIGN_5
|
|
.L30_bodyB:
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
MOVQ OFFSET, %rax;
|
|
MOVQ %rax, kk;
|
|
#endif
|
|
MOVQ ba, ptrba;
|
|
MOVQ C, C0;
|
|
MOVQ bm, i;
|
|
SARQ $2, i;
|
|
JLE .L31_loopE;
|
|
ALIGN_5
|
|
.L31_bodyB:
|
|
#if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA))
|
|
MOVQ bb,ptrbb;
|
|
#else
|
|
MOVQ bb, ptrbb;
|
|
MOVQ kk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 4), ptrba;
|
|
ADDQ %rax, ptrbb;
|
|
#endif
|
|
XOR_DY yvec15, yvec15, yvec15;
|
|
XOR_DY yvec14, yvec14, yvec14;
|
|
#ifndef TRMMKERNEL
|
|
MOVQ bk,k;
|
|
#elif (defined(LEFT)&&!defined(TRANSA))||(!defined(LEFT)&&defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kk, %rax;
|
|
MOVQ %rax, kkk;
|
|
#else
|
|
MOVQ kk, %rax;
|
|
#ifdef LEFT
|
|
ADDQ $4, %rax;
|
|
#else
|
|
ADDQ $1, %rax;
|
|
#endif
|
|
MOVQ %rax, kkk;
|
|
#endif
|
|
SARQ $2, k;
|
|
JLE .L311_loopE;
|
|
ALIGN_5
|
|
.L311_bodyB:
|
|
LD_DY 0*SIZE(ptrba), yvec0;
|
|
BROAD_DY 0*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
|
|
LD_DY 4*SIZE(ptrba), yvec1;
|
|
MUL_DY yvec1, yvec2, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
|
|
VPERMILP_DY $0x05, yvec0, yvec4;
|
|
BROAD_DY 1*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec4, yvec3, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec5;
|
|
MUL_DY yvec5, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
|
|
LD_DY 8*SIZE(ptrba), yvec0;
|
|
BROAD_DY 2*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
|
|
LD_DY 12*SIZE(ptrba), yvec1;
|
|
MUL_DY yvec1, yvec2, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
|
|
VPERMILP_DY $0x05, yvec0, yvec4;
|
|
BROAD_DY 3*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec4, yvec3, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec5;
|
|
MUL_DY yvec5, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
|
|
LD_DY 16*SIZE(ptrba), yvec0;
|
|
BROAD_DY 4*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
|
|
LD_DY 20*SIZE(ptrba), yvec1;
|
|
MUL_DY yvec1, yvec2, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
|
|
VPERMILP_DY $0x05, yvec0, yvec4;
|
|
BROAD_DY 5*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec4, yvec3, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec5;
|
|
MUL_DY yvec5, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
|
|
LD_DY 24*SIZE(ptrba), yvec0;
|
|
BROAD_DY 6*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
|
|
LD_DY 28*SIZE(ptrba), yvec1;
|
|
MUL_DY yvec1, yvec2, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
|
|
VPERMILP_DY $0x05, yvec0, yvec4;
|
|
BROAD_DY 7*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec4, yvec3, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec5;
|
|
MUL_DY yvec5, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
ADDQ $32*SIZE, ptrba;
|
|
ADDQ $8*SIZE, ptrbb;
|
|
DECQ k;
|
|
JG .L311_bodyB;
|
|
ALIGN_5
|
|
.L311_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $2, bk;
|
|
#else
|
|
TEST $2, kkk;
|
|
#endif
|
|
JLE .L312_loopE;
|
|
ALIGN_5
|
|
.L312_bodyB:
|
|
LD_DY 0*SIZE(ptrba), yvec0;
|
|
BROAD_DY 0*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
|
|
LD_DY 4*SIZE(ptrba), yvec1;
|
|
MUL_DY yvec1, yvec2, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
|
|
VPERMILP_DY $0x05, yvec0, yvec4;
|
|
BROAD_DY 1*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec4, yvec3, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec5;
|
|
MUL_DY yvec5, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
|
|
LD_DY 8*SIZE(ptrba), yvec0;
|
|
BROAD_DY 2*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
|
|
LD_DY 12*SIZE(ptrba), yvec1;
|
|
MUL_DY yvec1, yvec2, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
|
|
VPERMILP_DY $0x05, yvec0, yvec4;
|
|
BROAD_DY 3*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec4, yvec3, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec5;
|
|
MUL_DY yvec5, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
ADDQ $16*SIZE, ptrba;
|
|
ADDQ $4*SIZE, ptrbb;
|
|
|
|
.L312_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $1, bk;
|
|
#else
|
|
TEST $1, kkk;
|
|
#endif
|
|
JLE .L313_loopE;
|
|
ALIGN_5
|
|
.L313_bodyB:
|
|
LD_DY 0*SIZE(ptrba), yvec0;
|
|
BROAD_DY 0*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
|
|
LD_DY 4*SIZE(ptrba), yvec1;
|
|
MUL_DY yvec1, yvec2, yvec7;
|
|
ADD1_DY yvec7, yvec14, yvec14;
|
|
|
|
VPERMILP_DY $0x05, yvec0, yvec4;
|
|
BROAD_DY 1*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec4, yvec3, yvec6;
|
|
ADD2_DY yvec6, yvec15, yvec15;
|
|
|
|
VPERMILP_DY $0x05, yvec1, yvec5;
|
|
MUL_DY yvec5, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec14, yvec14;
|
|
ADDQ $8*SIZE, ptrba;
|
|
ADDQ $2*SIZE, ptrbb;
|
|
|
|
.L313_loopE:
|
|
#### Handle ####
|
|
XOR_DY yvec7, yvec7, yvec7;
|
|
#if defined(RN) || defined(RT) || defined(CN) || defined(CT)
|
|
ADDSUB_DY yvec15, yvec7, yvec15;
|
|
ADDSUB_DY yvec14, yvec7, yvec14;
|
|
#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
|
|
SUB_DY yvec15, yvec7, yvec15;
|
|
SUB_DY yvec14, yvec7, yvec14;
|
|
#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
|
|
VPERMILP_DY $0x05, yvec15, yvec15;
|
|
VPERMILP_DY $0x05, yvec14, yvec14;
|
|
ADDSUB_DY yvec15, yvec7, yvec15;
|
|
ADDSUB_DY yvec14, yvec7, yvec14;
|
|
VPERMILP_DY $0x05, yvec15, yvec15;
|
|
VPERMILP_DY $0x05, yvec14, yvec14;
|
|
#endif
|
|
|
|
#### Load Alpha ####
|
|
BROAD_DY MEMALPHA_R,yvec7;
|
|
BROAD_DY MEMALPHA_I,yvec6;
|
|
#### Multiply Alpha ####
|
|
VPERMILP_DY $0x05, yvec15, yvec5;
|
|
MUL_DY yvec7, yvec15, yvec15;
|
|
MUL_DY yvec6, yvec5, yvec5;
|
|
ADD2_DY yvec5, yvec15, yvec15;
|
|
VPERMILP_DY $0x05, yvec14, yvec4;
|
|
MUL_DY yvec7, yvec14, yvec14;
|
|
MUL_DY yvec6, yvec4, yvec4;
|
|
ADD2_DY yvec4, yvec14, yvec14;
|
|
EXTRA_DY $1, yvec15, xvec7;
|
|
EXTRA_DY $1, yvec14, xvec6;
|
|
#### Writing Back ####
|
|
#ifndef TRMMKERNEL
|
|
LDL_DX 0*SIZE(C0), xvec0, xvec0;
|
|
LDH_DX 1*SIZE(C0), xvec0, xvec0;
|
|
LDL_DX 2*SIZE(C0), xvec1, xvec1;
|
|
LDH_DX 3*SIZE(C0), xvec1, xvec1;
|
|
LDL_DX 4*SIZE(C0), xvec2, xvec2;
|
|
LDH_DX 5*SIZE(C0), xvec2, xvec2;
|
|
LDL_DX 6*SIZE(C0), xvec3, xvec3;
|
|
LDH_DX 7*SIZE(C0), xvec3, xvec3;
|
|
ADD_DX xvec0, xvec15, xvec15;
|
|
ADD_DX xvec1, xvec7, xvec7;
|
|
ADD_DX xvec2, xvec14, xvec14;
|
|
ADD_DX xvec3, xvec6, xvec6;
|
|
#endif
|
|
STL_DX xvec15, 0*SIZE(C0);
|
|
STH_DX xvec15, 1*SIZE(C0);
|
|
STL_DX xvec7, 2*SIZE(C0);
|
|
STH_DX xvec7, 3*SIZE(C0);
|
|
STL_DX xvec14, 4*SIZE(C0);
|
|
STH_DX xvec14, 5*SIZE(C0);
|
|
STL_DX xvec6, 6*SIZE(C0);
|
|
STH_DX xvec6, 7*SIZE(C0);
|
|
#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kkk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 4), ptrba;
|
|
ADDQ %rax, ptrbb;
|
|
#endif
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
ADDQ $4, kk;
|
|
#endif
|
|
ADDQ $8*SIZE, C0;
|
|
DECQ i;
|
|
JG .L31_bodyB;
|
|
ALIGN_5
|
|
.L31_loopE:
|
|
TEST $2, bm;
|
|
JLE .L32_loopE;
|
|
ALIGN_5
|
|
.L32_bodyB:
|
|
#if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA))
|
|
MOVQ bb,ptrbb;
|
|
#else
|
|
MOVQ bb, ptrbb;
|
|
MOVQ kk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 2), ptrba;
|
|
ADDQ %rax, ptrbb;
|
|
#endif
|
|
XOR_DY yvec15, yvec15, yvec15;
|
|
#ifndef TRMMKERNEL
|
|
MOVQ bk,k;
|
|
#elif (defined(LEFT)&&!defined(TRANSA))||(!defined(LEFT)&&defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kk, %rax;
|
|
MOVQ %rax, kkk;
|
|
#else
|
|
MOVQ kk, %rax;
|
|
#ifdef LEFT
|
|
ADDQ $2, %rax;
|
|
#else
|
|
ADDQ $1, %rax;
|
|
#endif
|
|
MOVQ %rax, kkk;
|
|
#endif
|
|
SARQ $2, k;
|
|
JLE .L321_loopE;
|
|
ALIGN_5
|
|
.L321_bodyB:
|
|
LD_DY 0*SIZE(ptrba), yvec0;
|
|
BROAD_DY 0*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
VPERMILP_DY $0x05, yvec0, yvec1;
|
|
BROAD_DY 1*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec15, yvec15;
|
|
|
|
LD_DY 4*SIZE(ptrba), yvec0;
|
|
BROAD_DY 2*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
VPERMILP_DY $0x05, yvec0, yvec1;
|
|
BROAD_DY 3*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec15, yvec15;
|
|
|
|
LD_DY 8*SIZE(ptrba), yvec0;
|
|
BROAD_DY 4*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
VPERMILP_DY $0x05, yvec0, yvec1;
|
|
BROAD_DY 5*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec15, yvec15;
|
|
|
|
LD_DY 12*SIZE(ptrba), yvec0;
|
|
BROAD_DY 6*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
VPERMILP_DY $0x05, yvec0, yvec1;
|
|
BROAD_DY 7*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec15, yvec15;
|
|
ADDQ $16*SIZE, ptrba;
|
|
ADDQ $8*SIZE, ptrbb;
|
|
DECQ k;
|
|
JG .L321_bodyB;
|
|
ALIGN_5
|
|
.L321_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $2, bk;
|
|
#else
|
|
TEST $2, kkk;
|
|
#endif
|
|
JLE .L322_loopE;
|
|
ALIGN_5
|
|
.L322_bodyB:
|
|
LD_DY 0*SIZE(ptrba), yvec0;
|
|
BROAD_DY 0*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
VPERMILP_DY $0x05, yvec0, yvec1;
|
|
BROAD_DY 1*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec15, yvec15;
|
|
|
|
LD_DY 4*SIZE(ptrba), yvec0;
|
|
BROAD_DY 2*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
VPERMILP_DY $0x05, yvec0, yvec1;
|
|
BROAD_DY 3*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec15, yvec15;
|
|
ADDQ $8*SIZE, ptrba;
|
|
ADDQ $4*SIZE, ptrbb;
|
|
|
|
.L322_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $1, bk;
|
|
#else
|
|
TEST $1, kkk;
|
|
#endif
|
|
JLE .L323_loopE;
|
|
ALIGN_5
|
|
.L323_bodyB:
|
|
LD_DY 0*SIZE(ptrba), yvec0;
|
|
BROAD_DY 0*SIZE(ptrbb), yvec2;
|
|
MUL_DY yvec0, yvec2, yvec6;
|
|
ADD1_DY yvec6, yvec15, yvec15;
|
|
VPERMILP_DY $0x05, yvec0, yvec1;
|
|
BROAD_DY 1*SIZE(ptrbb), yvec3;
|
|
MUL_DY yvec1, yvec3, yvec7;
|
|
ADD2_DY yvec7, yvec15, yvec15;
|
|
ADDQ $4*SIZE, ptrba;
|
|
ADDQ $2*SIZE, ptrbb;
|
|
.L323_loopE:
|
|
#### Handle ####
|
|
XOR_DY yvec7, yvec7, yvec7;
|
|
#if defined(RN) || defined(RT) || defined(CN) || defined(CT)
|
|
ADDSUB_DY yvec15, yvec7, yvec15;
|
|
#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
|
|
SUB_DY yvec15, yvec7, yvec15;
|
|
#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
|
|
VPERMILP_DY $0x05, yvec15, yvec15;
|
|
ADDSUB_DY yvec15, yvec7, yvec15;
|
|
VPERMILP_DY $0x05, yvec15, yvec15;
|
|
#endif
|
|
|
|
#### Load Alpha ####
|
|
BROAD_DY MEMALPHA_R,yvec7;
|
|
BROAD_DY MEMALPHA_I,yvec6;
|
|
#### Multiply Alpha ####
|
|
VPERMILP_DY $0x05, yvec15, yvec5;
|
|
MUL_DY yvec7, yvec15, yvec15;
|
|
MUL_DY yvec6, yvec5, yvec5;
|
|
ADD2_DY yvec5, yvec15, yvec15;
|
|
EXTRA_DY $1, yvec15, xvec7;
|
|
#### Writing Back ####
|
|
#ifndef TRMMKERNEL
|
|
LDL_DX 0*SIZE(C0), xvec0, xvec0;
|
|
LDH_DX 1*SIZE(C0), xvec0, xvec0;
|
|
LDL_DX 2*SIZE(C0), xvec1, xvec1;
|
|
LDH_DX 3*SIZE(C0), xvec1, xvec1;
|
|
ADD_DX xvec0, xvec15, xvec15;
|
|
ADD_DX xvec1, xvec7, xvec7;
|
|
#endif
|
|
STL_DX xvec15, 0*SIZE(C0);
|
|
STH_DX xvec15, 1*SIZE(C0);
|
|
STL_DX xvec7, 2*SIZE(C0);
|
|
STH_DX xvec7, 3*SIZE(C0);
|
|
#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kkk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 2), ptrba;
|
|
ADDQ %rax, ptrbb;
|
|
#endif
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
ADDQ $2, kk;
|
|
#endif
|
|
ADDQ $4*SIZE, C0;
|
|
.L32_loopE:
|
|
TEST $1, bm;
|
|
JLE .L33_loopE;
|
|
ALIGN_5
|
|
.L33_bodyB:
|
|
#if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA))
|
|
MOVQ bb,ptrbb;
|
|
#else
|
|
MOVQ bb, ptrbb;
|
|
MOVQ kk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
ADDQ %rax, ptrba;
|
|
ADDQ %rax, ptrbb;
|
|
#endif
|
|
XOR_DY yvec15, yvec15, yvec15;
|
|
#ifndef TRMMKERNEL
|
|
MOVQ bk,k;
|
|
#elif (defined(LEFT)&&!defined(TRANSA))||(!defined(LEFT)&&defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kk, %rax;
|
|
MOVQ %rax, kkk;
|
|
#else
|
|
MOVQ kk, %rax;
|
|
#ifdef LEFT
|
|
ADDQ $1, %rax;
|
|
#else
|
|
ADDQ $1, %rax;
|
|
#endif
|
|
MOVQ %rax, kkk;
|
|
#endif
|
|
SARQ $2, k;
|
|
JLE .L331_loopE;
|
|
ALIGN_5
|
|
.L331_bodyB:
|
|
LD_DX 0*SIZE(ptrba), xvec0;
|
|
BROAD_DX 0*SIZE(ptrbb), xvec2;
|
|
MUL_DX xvec0, xvec2, xvec2;
|
|
ADD1_DX xvec2, xvec15, xvec15;
|
|
|
|
SHUF_DX $0x4e, xvec0, xvec1;
|
|
BROAD_DX 1*SIZE(ptrbb), xvec3;
|
|
MUL_DX xvec1, xvec3, xvec3;
|
|
ADDSUB_DX xvec3, xvec15, xvec15;
|
|
|
|
LD_DX 2*SIZE(ptrba), xvec0;
|
|
BROAD_DX 2*SIZE(ptrbb), xvec2;
|
|
MUL_DX xvec0, xvec2, xvec2;
|
|
ADD1_DX xvec2, xvec15, xvec15;
|
|
|
|
SHUF_DX $0x4e, xvec0, xvec1;
|
|
BROAD_DX 3*SIZE(ptrbb), xvec3;
|
|
MUL_DX xvec1, xvec3, xvec3;
|
|
ADDSUB_DX xvec3, xvec15, xvec15;
|
|
|
|
LD_DX 4*SIZE(ptrba), xvec0;
|
|
BROAD_DX 4*SIZE(ptrbb), xvec2;
|
|
MUL_DX xvec0, xvec2, xvec2;
|
|
ADD1_DX xvec2, xvec15, xvec15;
|
|
|
|
SHUF_DX $0x4e, xvec0, xvec1;
|
|
BROAD_DX 5*SIZE(ptrbb), xvec3;
|
|
MUL_DX xvec1, xvec3, xvec3;
|
|
ADDSUB_DX xvec3, xvec15, xvec15;
|
|
|
|
LD_DX 6*SIZE(ptrba), xvec0;
|
|
BROAD_DX 6*SIZE(ptrbb), xvec2;
|
|
MUL_DX xvec0, xvec2, xvec2;
|
|
ADD1_DX xvec2, xvec15, xvec15;
|
|
|
|
SHUF_DX $0x4e, xvec0, xvec1;
|
|
BROAD_DX 7*SIZE(ptrbb), xvec3;
|
|
MUL_DX xvec1, xvec3, xvec3;
|
|
ADDSUB_DX xvec3, xvec15, xvec15;
|
|
ADDQ $8*SIZE, ptrba;
|
|
ADDQ $8*SIZE, ptrbb;
|
|
DECQ k;
|
|
JG .L331_bodyB;
|
|
ALIGN_5
|
|
.L331_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $2, bk;
|
|
#else
|
|
TEST $2, kkk;
|
|
#endif
|
|
JLE .L332_loopE;
|
|
ALIGN_5
|
|
.L332_bodyB:
|
|
LD_DX 0*SIZE(ptrba), xvec0;
|
|
BROAD_DX 0*SIZE(ptrbb), xvec2;
|
|
MUL_DX xvec0, xvec2, xvec2;
|
|
ADD1_DX xvec2, xvec15, xvec15;
|
|
|
|
SHUF_DX $0x4e, xvec0, xvec1;
|
|
BROAD_DX 1*SIZE(ptrbb), xvec3;
|
|
MUL_DX xvec1, xvec3, xvec3;
|
|
ADDSUB_DX xvec3, xvec15, xvec15;
|
|
|
|
LD_DX 2*SIZE(ptrba), xvec0;
|
|
BROAD_DX 2*SIZE(ptrbb), xvec2;
|
|
MUL_DX xvec0, xvec2, xvec2;
|
|
ADD1_DX xvec2, xvec15, xvec15;
|
|
|
|
SHUF_DX $0x4e, xvec0, xvec1;
|
|
BROAD_DX 3*SIZE(ptrbb), xvec3;
|
|
MUL_DX xvec1, xvec3, xvec3;
|
|
ADDSUB_DX xvec3, xvec15, xvec15;
|
|
ADDQ $4*SIZE, ptrba;
|
|
ADDQ $4*SIZE, ptrbb;
|
|
|
|
.L332_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $1, bk;
|
|
#else
|
|
TEST $1, kkk;
|
|
#endif
|
|
JLE .L333_loopE;
|
|
ALIGN_5
|
|
.L333_bodyB:
|
|
LD_DX 0*SIZE(ptrba), xvec0;
|
|
BROAD_DX 0*SIZE(ptrbb), xvec2;
|
|
MUL_DX xvec0, xvec2, xvec2;
|
|
ADD1_DX xvec2, xvec15, xvec15;
|
|
|
|
SHUF_DX $0x4e, xvec0, xvec1;
|
|
BROAD_DX 1*SIZE(ptrbb), xvec3;
|
|
MUL_DX xvec1, xvec3, xvec3;
|
|
ADDSUB_DX xvec3, xvec15, xvec15;
|
|
ADDQ $2*SIZE, ptrba;
|
|
ADDQ $2*SIZE, ptrbb;
|
|
|
|
.L333_loopE:
|
|
#### Handle ####
|
|
XOR_DY yvec7, yvec7, yvec7;
|
|
#if defined(RN) || defined(RT) || defined(CN) || defined(CT)
|
|
ADDSUB_DX xvec15, xvec7, xvec7;
|
|
MOV_DX xvec7, xvec15;
|
|
#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
|
|
SUB_DX xvec15, xvec7, xvec7;
|
|
MOV_DX xvec7, xvec15;
|
|
#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
|
|
SHUF_DX $0x4e, xvec15, xvec15;
|
|
ADDSUB_DX xvec15, xvec7, xvec7;
|
|
MOV_DX xvec7, xvec15;
|
|
SHUF_DX $0x4e, xvec15, xvec15;
|
|
#endif
|
|
|
|
#### Load Alpha ####
|
|
BROAD_DX MEMALPHA_R,xvec7;
|
|
BROAD_DX MEMALPHA_I,xvec6;
|
|
#### Multiply Alpha ####
|
|
SHUF_DX $0x4e, xvec15, xvec5;
|
|
MUL_DX xvec7, xvec15, xvec15;
|
|
MUL_DX xvec6, xvec5, xvec5;
|
|
ADDSUB_DX xvec5, xvec15, xvec15;
|
|
#### Writing back ####
|
|
#ifndef TRMMKERNEL
|
|
LDL_DX 0*SIZE(C0), xvec0, xvec0;
|
|
LDH_DX 1*SIZE(C0), xvec0, xvec0;
|
|
ADD_DX xvec0, xvec15, xvec15;
|
|
#endif
|
|
STL_DX xvec15, 0*SIZE(C0);
|
|
STH_DX xvec15, 1*SIZE(C0);
|
|
#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kkk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
ADDQ %rax, ptrba;
|
|
ADDQ %rax, ptrbb;
|
|
#endif
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
ADDQ $1, kk;
|
|
#endif
|
|
ADDQ $2*SIZE, C0;
|
|
.L33_loopE:
|
|
#if defined(TRMMKERNEL) && !defined(LEFT)
|
|
ADDQ $1, kk;
|
|
#endif
|
|
MOVQ bk, k;
|
|
SALQ $4*SIZE, k;
|
|
ADDQ k, bb;
|
|
LEAQ (C, ldc, 1), C;
|
|
.L30_loopE:
|
|
movq 0(%rsp), %rbx;
|
|
movq 8(%rsp), %rbp;
|
|
movq 16(%rsp), %r12;
|
|
movq 24(%rsp), %r13;
|
|
movq 32(%rsp), %r14;
|
|
movq 40(%rsp), %r15;
|
|
|
|
|
|
vzeroupper
|
|
|
|
#ifdef WINDOWS_ABI
|
|
movq 48(%rsp), %rdi
|
|
movq 56(%rsp), %rsi
|
|
movups 64(%rsp), %xmm6
|
|
movups 80(%rsp), %xmm7
|
|
movups 96(%rsp), %xmm8
|
|
movups 112(%rsp), %xmm9
|
|
movups 128(%rsp), %xmm10
|
|
movups 144(%rsp), %xmm11
|
|
movups 160(%rsp), %xmm12
|
|
movups 176(%rsp), %xmm13
|
|
movups 192(%rsp), %xmm14
|
|
movups 208(%rsp), %xmm15
|
|
#endif
|
|
|
|
|
|
addq $STACKSIZE, %rsp;
|
|
ret
|
|
|
|
EPILOGUE
|