4460 lines
108 KiB
ArmAsm
4460 lines
108 KiB
ArmAsm
/*****************************************************************************
|
|
Copyright (c) 2011-2014, The OpenBLAS Project
|
|
All rights reserved.
|
|
|
|
Redistribution and use in source and binary forms, with or without
|
|
modification, are permitted provided that the following conditions are
|
|
met:
|
|
|
|
1. Redistributions of source code must retain the above copyright
|
|
notice, this list of conditions and the following disclaimer.
|
|
|
|
2. Redistributions in binary form must reproduce the above copyright
|
|
notice, this list of conditions and the following disclaimer in
|
|
the documentation and/or other materials provided with the
|
|
distribution.
|
|
3. Neither the name of the OpenBLAS project nor the names of
|
|
its contributors may be used to endorse or promote products
|
|
derived from this software without specific prior written
|
|
permission.
|
|
|
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
|
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
|
|
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
**********************************************************************************/
|
|
|
|
#define ASSEMBLER
|
|
#include "common.h"
|
|
|
|
#define old_bm %rdi
|
|
#define old_bn %rsi
|
|
#define old_bk %rdx
|
|
|
|
#define bm %r13
|
|
#define bn %r14
|
|
#define bk %r15
|
|
|
|
#define ALPHA %xmm0
|
|
#define ba %rcx
|
|
#define bb %r8
|
|
#define C %r9
|
|
#define ldc %r10
|
|
|
|
#define i %r11
|
|
#define k %rax
|
|
|
|
#define ptrba %rdi
|
|
#define ptrbb %rsi
|
|
#define C0 %rbx
|
|
#define C1 %rbp
|
|
|
|
#define prebb %r12
|
|
|
|
#ifndef WINDOWS_ABI
|
|
|
|
#define STACKSIZE 128
|
|
|
|
#define old_ldc 8+STACKSIZE(%rsp)
|
|
#define old_offset 16+STACKSIZE(%rsp)
|
|
|
|
#define MEMALPHA_R 48(%rsp)
|
|
#define MEMALPHA_I 56(%rsp)
|
|
#define j 64(%rsp)
|
|
#define OFFSET 72(%rsp)
|
|
#define kk 80(%rsp)
|
|
#define kkk 88(%rsp)
|
|
|
|
#else
|
|
|
|
#define STACKSIZE 512
|
|
|
|
#define OLD_ALPHA_I 40 + STACKSIZE(%rsp)
|
|
#define OLD_A 48 + STACKSIZE(%rsp)
|
|
#define OLD_B 56 + STACKSIZE(%rsp)
|
|
#define OLD_C 64 + STACKSIZE(%rsp)
|
|
#define old_ldc 72 + STACKSIZE(%rsp)
|
|
#define old_offset 80 + STACKSIZE(%rsp)
|
|
|
|
#define MEMALPHA_R 224(%rsp)
|
|
#define MEMALPHA_I 232(%rsp)
|
|
#define j 240(%rsp)
|
|
#define OFFSET 248(%rsp)
|
|
#define kk 256(%rsp)
|
|
#define kkk 264(%rsp)
|
|
|
|
#endif
|
|
|
|
#define PREFETCH0 prefetcht0
|
|
#define PREFETCH1 prefetcht0
|
|
#define PREFETCH2 prefetcht0
|
|
#define PRESIZE 64
|
|
|
|
#define xvec0 %xmm0
|
|
#define xvec1 %xmm1
|
|
#define xvec2 %xmm2
|
|
#define xvec3 %xmm3
|
|
#define xvec4 %xmm4
|
|
#define xvec5 %xmm5
|
|
#define xvec6 %xmm6
|
|
#define xvec7 %xmm7
|
|
#define xvec8 %xmm8
|
|
#define xvec9 %xmm9
|
|
#define xvec10 %xmm10
|
|
#define xvec11 %xmm11
|
|
#define xvec12 %xmm12
|
|
#define xvec13 %xmm13
|
|
#define xvec14 %xmm14
|
|
#define xvec15 %xmm15
|
|
|
|
#define yvec0 %ymm0
|
|
#define yvec1 %ymm1
|
|
#define yvec2 %ymm2
|
|
#define yvec3 %ymm3
|
|
#define yvec4 %ymm4
|
|
#define yvec5 %ymm5
|
|
#define yvec6 %ymm6
|
|
#define yvec7 %ymm7
|
|
#define yvec8 %ymm8
|
|
#define yvec9 %ymm9
|
|
#define yvec10 %ymm10
|
|
#define yvec11 %ymm11
|
|
#define yvec12 %ymm12
|
|
#define yvec13 %ymm13
|
|
#define yvec14 %ymm14
|
|
#define yvec15 %ymm15
|
|
|
|
#define LEAQ leaq
|
|
#define ADDQ addq
|
|
#define MULQ imulq
|
|
#define SARQ sarq
|
|
#define SALQ salq
|
|
#define ANDQ andq
|
|
#define SUBQ subq
|
|
#define DECQ decq
|
|
#define JG jg
|
|
#define JLE jle
|
|
#define TEST testq
|
|
#define OR orq
|
|
#define JNE jne
|
|
#define JMP jmp
|
|
#define NOP
|
|
#define XOR xorpd
|
|
#undef MOVQ
|
|
#define MOVQ movq
|
|
|
|
#define XOR_SY vxorps
|
|
#define XOR_SX vxorps
|
|
|
|
#define LD_SY vmovaps
|
|
#define LD_SX vmovaps
|
|
#define LDL_SX vmovlps
|
|
#define LDL_SY vmovlps
|
|
#define LDH_SX vmovhps
|
|
#define LDH_SY vmovhps
|
|
|
|
#define ST_SY vmovaps
|
|
#define ST_SX vmovaps
|
|
#define STL_SX vmovlps
|
|
#define STL_SY vmovlps
|
|
#define STH_SX vmovhps
|
|
#define STH_SY vmovhps
|
|
|
|
#define EDUP_SY vmovsldup
|
|
#define ODUP_SY vmovshdup
|
|
#define EDUP_SX vmovsldup
|
|
#define ODUP_SX vmovshdup
|
|
|
|
#define ADD_SY vaddps
|
|
#define ADD_SX vaddps
|
|
#define SUB_SY vsubps
|
|
#define SUB_SX vsubps
|
|
|
|
#define ADDSUB_SY vaddsubps
|
|
#define ADDSUB_SX vaddsubps
|
|
|
|
#define MUL_SY vmulps
|
|
#define MUL_SX vmulps
|
|
|
|
#define SHUF_SY vperm2f128
|
|
#define SHUF_SX vpshufd
|
|
|
|
#define VPERMILP_SY vpermilps
|
|
#define VPERMILP_SX vpermilps
|
|
|
|
#define BROAD_SY vbroadcastss
|
|
#define BROAD_SX vbroadcastss
|
|
|
|
#define MOV_SY vmovaps
|
|
#define MOV_SX vmovaps
|
|
|
|
#define REVS_SY vshufps
|
|
#define REVS_SX vshufps
|
|
|
|
#define EXTRA_SY vextractf128
|
|
|
|
#if defined(NN) || defined(NT) || defined(TN) || defined(TT)
|
|
#define ADD1_SY ADD_SY
|
|
#define ADD2_SY ADDSUB_SY
|
|
#define ADD1_SX ADD_SX
|
|
#define ADD2_SX ADDSUB_SX
|
|
#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
|
|
#define ADD1_SY SUB_SY
|
|
#define ADD2_SY ADDSUB_SY
|
|
#define ADD1_SX SUB_SX
|
|
#define ADD2_SX ADDSUB_SX
|
|
#elif defined(RN) || defined(RT) || defined(CN) || defined(CT)
|
|
#define ADD1_SY SUB_SY
|
|
#define ADD2_SY ADDSUB_SY
|
|
#define ADD1_SX SUB_SX
|
|
#define ADD2_SX ADDSUB_SX
|
|
#else
|
|
#define ADD1_SY ADD_SY
|
|
#define ADD2_SY ADDSUB_SY
|
|
#define ADD1_SX ADD_SX
|
|
#define ADD2_SX ADDSUB_SX
|
|
#endif
|
|
|
|
PROLOGUE
|
|
|
|
subq $STACKSIZE, %rsp;
|
|
movq %rbx, 0(%rsp);
|
|
movq %rbp, 8(%rsp);
|
|
movq %r12, 16(%rsp);
|
|
movq %r13, 24(%rsp);
|
|
movq %r14, 32(%rsp);
|
|
movq %r15, 40(%rsp);
|
|
|
|
#ifdef WINDOWS_ABI
|
|
movq %rdi, 48(%rsp)
|
|
movq %rsi, 56(%rsp)
|
|
movups %xmm6, 64(%rsp)
|
|
movups %xmm7, 80(%rsp)
|
|
movups %xmm8, 96(%rsp)
|
|
movups %xmm9, 112(%rsp)
|
|
movups %xmm10, 128(%rsp)
|
|
movups %xmm11, 144(%rsp)
|
|
movups %xmm12, 160(%rsp)
|
|
movups %xmm13, 176(%rsp)
|
|
movups %xmm14, 192(%rsp)
|
|
movups %xmm15, 208(%rsp)
|
|
|
|
movq ARG1, old_bm
|
|
movq ARG2, old_bn
|
|
movq ARG3, old_bk
|
|
movq OLD_A, ba
|
|
movq OLD_B, bb
|
|
movq OLD_C, C
|
|
movq old_ldc, ldc
|
|
#ifdef TRMMKERNEL
|
|
movq old_offset, %r11
|
|
#endif
|
|
movaps %xmm3, %xmm0
|
|
movsd OLD_ALPHA_I, %xmm1
|
|
#else
|
|
movq old_ldc, ldc
|
|
#ifdef TRMMKERNEL
|
|
movq old_offset, %r11;
|
|
#endif
|
|
#endif
|
|
|
|
vzeroupper
|
|
|
|
vmovlps %xmm0, MEMALPHA_R
|
|
vmovlps %xmm1, MEMALPHA_I
|
|
movq old_bm, bm
|
|
movq old_bn, bn
|
|
movq old_bk, bk
|
|
salq $ZBASE_SHIFT, ldc
|
|
#ifdef TRMMKERNEL
|
|
movq %r11, OFFSET
|
|
#ifndef LEFT
|
|
negq %r11;
|
|
#endif
|
|
movq %r11, kk;
|
|
#endif
|
|
|
|
MOVQ bn,j;
|
|
SARQ $2,j; # Rn = 4
|
|
JLE .L0_loopE;
|
|
ALIGN_5;
|
|
.L0_bodyB:;
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
MOVQ OFFSET, %rax;
|
|
MOVQ %rax, kk;
|
|
#endif
|
|
MOVQ C,C0;
|
|
LEAQ (C,ldc,2),C1;
|
|
MOVQ bk, k;
|
|
SALQ $5, k;
|
|
LEAQ (bb, k, 1), prebb; # Rn=4, SIZE=4 COMPLEX=2
|
|
MOVQ ba,ptrba;
|
|
MOVQ bm,i;
|
|
SARQ $3,i; # Rm = 8
|
|
JLE .L1_loopE;
|
|
ALIGN_5;
|
|
.L1_bodyB:;
|
|
#if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA))
|
|
MOVQ bb,ptrbb;
|
|
#else
|
|
MOVQ bb, ptrbb;
|
|
MOVQ kk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 8), ptrba;
|
|
LEAQ (ptrbb, %rax, 4), ptrbb;
|
|
#endif
|
|
# Initial results register
|
|
PREFETCH0 0*SIZE(prebb);
|
|
XOR_SY yvec15, yvec15, yvec15;
|
|
PREFETCH0 16*SIZE(prebb);
|
|
ADDQ $32*SIZE, prebb;
|
|
XOR_SY yvec14, yvec14, yvec14;
|
|
PREFETCH2 3*SIZE(C0);
|
|
XOR_SY yvec13, yvec13, yvec13;
|
|
PREFETCH2 3*SIZE(C0, ldc, 1);
|
|
XOR_SY yvec12, yvec12, yvec12;
|
|
PREFETCH2 3*SIZE(C1);
|
|
EDUP_SY 0*SIZE(ptrbb), yvec2; # Br0, Br1, Br2, Br3
|
|
PREFETCH2 3*SIZE(C1, ldc, 1);
|
|
XOR_SY yvec11, yvec11, yvec11;
|
|
XOR_SY yvec10, yvec10, yvec10;
|
|
LD_SY 0*SIZE(ptrba), yvec0; # Ar0, Ai0, Ar1, Ai1..
|
|
XOR_SY yvec9, yvec9, yvec9;
|
|
XOR_SY yvec8, yvec8, yvec8;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3; # Br2, Br3, Br0, Br1
|
|
#ifndef TRMMKERNEL
|
|
MOVQ bk,k;
|
|
#elif (defined(LEFT)&&!defined(TRANSA))||(!defined(LEFT)&&defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kk, %rax;
|
|
MOVQ %rax, kkk;
|
|
#else
|
|
MOVQ kk, %rax;
|
|
#ifdef LEFT
|
|
ADDQ $8, %rax;
|
|
#else
|
|
ADDQ $4, %rax;
|
|
#endif
|
|
MOVQ %rax, kkk;
|
|
#endif
|
|
SARQ $2,k; # Unroll 4 times
|
|
JLE .L2_loopE;
|
|
ALIGN_5;
|
|
.L2_bodyB:;
|
|
# Computing kernel
|
|
|
|
######### Unroll 1 ##################
|
|
PREFETCH0 PRESIZE*SIZE(ptrba);
|
|
LD_SY 8*SIZE(ptrba), yvec1; # Ar4, Ai4, Ar5, Ai5..
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4; # Br1, Br0, Br3, Br2
|
|
MUL_SY yvec0, yvec3, yvec7;
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5; # Br3, Br2, Br1, Br0
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
ADD1_SY yvec7, yvec13, yvec13;
|
|
|
|
MUL_SY yvec1, yvec2, yvec6;
|
|
ODUP_SY 0*SIZE(ptrbb), yvec2; # Bi0, Bi1, Bi2, Bi3
|
|
MUL_SY yvec1, yvec3, yvec7;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3; # Bi2, Bi3, Bi0, Bi1
|
|
ADD1_SY yvec6, yvec14, yvec14;
|
|
ADD1_SY yvec7, yvec12, yvec12;
|
|
|
|
MUL_SY yvec0, yvec4, yvec6;
|
|
MUL_SY yvec0, yvec5, yvec7;
|
|
VPERMILP_SY $0xb1, yvec0, yvec0; # Ai0, Ar0, Ai1, Ar1..
|
|
ADD1_SY yvec6, yvec11, yvec11;
|
|
ADD1_SY yvec7, yvec9, yvec9;
|
|
|
|
MUL_SY yvec1, yvec4, yvec6;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4; # Br1, Br0, Br3, Br2
|
|
MUL_SY yvec1, yvec5, yvec7;
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5; # Br3, Br2, Br1, Br0
|
|
ADD1_SY yvec6, yvec10, yvec10;
|
|
ADD1_SY yvec7, yvec8, yvec8;
|
|
|
|
VPERMILP_SY $0xb1, yvec1, yvec1;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
MUL_SY yvec0, yvec3, yvec7;
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
ADD2_SY yvec7, yvec13, yvec13;
|
|
|
|
MUL_SY yvec1, yvec2, yvec6;
|
|
EDUP_SY 8*SIZE(ptrbb), yvec2;
|
|
MUL_SY yvec1, yvec3, yvec7;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3;
|
|
ADD2_SY yvec6, yvec14, yvec14;
|
|
ADD2_SY yvec7, yvec12, yvec12;
|
|
|
|
MUL_SY yvec0, yvec4, yvec6;
|
|
MUL_SY yvec0, yvec5, yvec7;
|
|
LD_SY 16*SIZE(ptrba), yvec0;
|
|
ADD2_SY yvec6, yvec11, yvec11;
|
|
ADD2_SY yvec7, yvec9, yvec9;
|
|
|
|
MUL_SY yvec1, yvec4, yvec6;
|
|
MUL_SY yvec1, yvec5, yvec7;
|
|
ADD2_SY yvec6, yvec10, yvec10;
|
|
ADD2_SY yvec7, yvec8, yvec8;
|
|
|
|
######### Unroll 2 ##################
|
|
PREFETCH0 (PRESIZE+16)*SIZE(ptrba);
|
|
LD_SY 24*SIZE(ptrba), yvec1; # Ar4, Ai4, Ar5, Ai5..
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4; # Br1, Br0, Br3, Br2
|
|
MUL_SY yvec0, yvec3, yvec7;
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5; # Br3, Br2, Br1, Br0
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
ADD1_SY yvec7, yvec13, yvec13;
|
|
|
|
MUL_SY yvec1, yvec2, yvec6;
|
|
ODUP_SY 8*SIZE(ptrbb), yvec2; # Bi0, Bi1, Bi2, Bi3
|
|
MUL_SY yvec1, yvec3, yvec7;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3; # Bi2, Bi3, Bi0, Bi1
|
|
ADD1_SY yvec6, yvec14, yvec14;
|
|
ADD1_SY yvec7, yvec12, yvec12;
|
|
|
|
MUL_SY yvec0, yvec4, yvec6;
|
|
MUL_SY yvec0, yvec5, yvec7;
|
|
VPERMILP_SY $0xb1, yvec0, yvec0; # Ai0, Ar0, Ai1, Ar1..
|
|
ADD1_SY yvec6, yvec11, yvec11;
|
|
ADD1_SY yvec7, yvec9, yvec9;
|
|
|
|
MUL_SY yvec1, yvec4, yvec6;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4; # Br1, Br0, Br3, Br2
|
|
MUL_SY yvec1, yvec5, yvec7;
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5; # Br3, Br2, Br1, Br0
|
|
ADD1_SY yvec6, yvec10, yvec10;
|
|
ADD1_SY yvec7, yvec8, yvec8;
|
|
|
|
VPERMILP_SY $0xb1, yvec1, yvec1;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
MUL_SY yvec0, yvec3, yvec7;
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
ADD2_SY yvec7, yvec13, yvec13;
|
|
|
|
MUL_SY yvec1, yvec2, yvec6;
|
|
EDUP_SY 16*SIZE(ptrbb), yvec2;
|
|
MUL_SY yvec1, yvec3, yvec7;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3;
|
|
ADD2_SY yvec6, yvec14, yvec14;
|
|
ADD2_SY yvec7, yvec12, yvec12;
|
|
|
|
MUL_SY yvec0, yvec4, yvec6;
|
|
MUL_SY yvec0, yvec5, yvec7;
|
|
LD_SY 32*SIZE(ptrba), yvec0;
|
|
ADD2_SY yvec6, yvec11, yvec11;
|
|
ADD2_SY yvec7, yvec9, yvec9;
|
|
|
|
MUL_SY yvec1, yvec4, yvec6;
|
|
MUL_SY yvec1, yvec5, yvec7;
|
|
ADD2_SY yvec6, yvec10, yvec10;
|
|
ADD2_SY yvec7, yvec8, yvec8;
|
|
|
|
######### Unroll 3 ##################
|
|
PREFETCH0 (PRESIZE+32)*SIZE(ptrba);
|
|
LD_SY 40*SIZE(ptrba), yvec1; # Ar4, Ai4, Ar5, Ai5..
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4; # Br1, Br0, Br3, Br2
|
|
MUL_SY yvec0, yvec3, yvec7;
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5; # Br3, Br2, Br1, Br0
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
ADD1_SY yvec7, yvec13, yvec13;
|
|
|
|
MUL_SY yvec1, yvec2, yvec6;
|
|
ODUP_SY 16*SIZE(ptrbb), yvec2; # Bi0, Bi1, Bi2, Bi3
|
|
MUL_SY yvec1, yvec3, yvec7;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3; # Bi2, Bi3, Bi0, Bi1
|
|
ADD1_SY yvec6, yvec14, yvec14;
|
|
ADD1_SY yvec7, yvec12, yvec12;
|
|
|
|
MUL_SY yvec0, yvec4, yvec6;
|
|
MUL_SY yvec0, yvec5, yvec7;
|
|
VPERMILP_SY $0xb1, yvec0, yvec0; # Ai0, Ar0, Ai1, Ar1..
|
|
ADD1_SY yvec6, yvec11, yvec11;
|
|
ADD1_SY yvec7, yvec9, yvec9;
|
|
|
|
MUL_SY yvec1, yvec4, yvec6;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4; # Br1, Br0, Br3, Br2
|
|
MUL_SY yvec1, yvec5, yvec7;
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5; # Br3, Br2, Br1, Br0
|
|
ADD1_SY yvec6, yvec10, yvec10;
|
|
ADD1_SY yvec7, yvec8, yvec8;
|
|
|
|
VPERMILP_SY $0xb1, yvec1, yvec1;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
MUL_SY yvec0, yvec3, yvec7;
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
ADD2_SY yvec7, yvec13, yvec13;
|
|
|
|
MUL_SY yvec1, yvec2, yvec6;
|
|
EDUP_SY 24*SIZE(ptrbb), yvec2;
|
|
MUL_SY yvec1, yvec3, yvec7;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3;
|
|
ADD2_SY yvec6, yvec14, yvec14;
|
|
ADD2_SY yvec7, yvec12, yvec12;
|
|
|
|
MUL_SY yvec0, yvec4, yvec6;
|
|
MUL_SY yvec0, yvec5, yvec7;
|
|
LD_SY 48*SIZE(ptrba), yvec0;
|
|
ADD2_SY yvec6, yvec11, yvec11;
|
|
ADD2_SY yvec7, yvec9, yvec9;
|
|
|
|
MUL_SY yvec1, yvec4, yvec6;
|
|
MUL_SY yvec1, yvec5, yvec7;
|
|
ADD2_SY yvec6, yvec10, yvec10;
|
|
ADD2_SY yvec7, yvec8, yvec8;
|
|
|
|
######### Unroll 4 ##################
|
|
PREFETCH0 (PRESIZE+48)*SIZE(ptrba);
|
|
LD_SY 56*SIZE(ptrba), yvec1; # Ar4, Ai4, Ar5, Ai5..
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4; # Br1, Br0, Br3, Br2
|
|
MUL_SY yvec0, yvec3, yvec7;
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5; # Br3, Br2, Br1, Br0
|
|
ADDQ $64*SIZE, ptrba;
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
ADD1_SY yvec7, yvec13, yvec13;
|
|
|
|
MUL_SY yvec1, yvec2, yvec6;
|
|
ODUP_SY 24*SIZE(ptrbb), yvec2; # Bi0, Bi1, Bi2, Bi3
|
|
MUL_SY yvec1, yvec3, yvec7;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3; # Bi2, Bi3, Bi0, Bi1
|
|
ADDQ $32*SIZE, ptrbb;
|
|
ADD1_SY yvec6, yvec14, yvec14;
|
|
ADD1_SY yvec7, yvec12, yvec12;
|
|
|
|
MUL_SY yvec0, yvec4, yvec6;
|
|
MUL_SY yvec0, yvec5, yvec7;
|
|
VPERMILP_SY $0xb1, yvec0, yvec0; # Ai0, Ar0, Ai1, Ar1..
|
|
ADD1_SY yvec6, yvec11, yvec11;
|
|
ADD1_SY yvec7, yvec9, yvec9;
|
|
|
|
MUL_SY yvec1, yvec4, yvec6;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4; # Br1, Br0, Br3, Br2
|
|
MUL_SY yvec1, yvec5, yvec7;
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5; # Br3, Br2, Br1, Br0
|
|
ADD1_SY yvec6, yvec10, yvec10;
|
|
ADD1_SY yvec7, yvec8, yvec8;
|
|
|
|
VPERMILP_SY $0xb1, yvec1, yvec1;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
MUL_SY yvec0, yvec3, yvec7;
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
ADD2_SY yvec7, yvec13, yvec13;
|
|
|
|
MUL_SY yvec1, yvec2, yvec6;
|
|
EDUP_SY 0*SIZE(ptrbb), yvec2;
|
|
MUL_SY yvec1, yvec3, yvec7;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3;
|
|
ADD2_SY yvec6, yvec14, yvec14;
|
|
ADD2_SY yvec7, yvec12, yvec12;
|
|
|
|
MUL_SY yvec0, yvec4, yvec6;
|
|
MUL_SY yvec0, yvec5, yvec7;
|
|
LD_SY 0*SIZE(ptrba), yvec0;
|
|
ADD2_SY yvec6, yvec11, yvec11;
|
|
ADD2_SY yvec7, yvec9, yvec9;
|
|
|
|
MUL_SY yvec1, yvec4, yvec6;
|
|
MUL_SY yvec1, yvec5, yvec7;
|
|
ADD2_SY yvec6, yvec10, yvec10;
|
|
ADD2_SY yvec7, yvec8, yvec8;
|
|
.L2_bodyE:;
|
|
DECQ k;
|
|
JG .L2_bodyB;
|
|
ALIGN_5
|
|
.L2_loopE:;
|
|
#ifndef TRMMKERNEL
|
|
TEST $2, bk;
|
|
#else
|
|
TEST $2, kkk;
|
|
#endif
|
|
JLE .L3_loopE;
|
|
ALIGN_5
|
|
.L3_loopB:
|
|
######### Unroll 1 ##################
|
|
PREFETCH0 PRESIZE*SIZE(ptrba)
|
|
LD_SY 8*SIZE(ptrba), yvec1; # Ar4, Ai4, Ar5, Ai5..
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
MUL_SY yvec0, yvec3, yvec7;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4; # Br1, Br0, Br3, Br2
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
ADD1_SY yvec7, yvec13, yvec13;
|
|
|
|
MUL_SY yvec1, yvec2, yvec6;
|
|
MUL_SY yvec1, yvec3, yvec7;
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5; # Br3, Br2, Br1, Br0
|
|
ADD1_SY yvec6, yvec14, yvec14;
|
|
ADD1_SY yvec7, yvec12, yvec12;
|
|
|
|
ODUP_SY 0*SIZE(ptrbb), yvec2; # Bi0, Bi1, Bi2, Bi3
|
|
MUL_SY yvec0, yvec4, yvec6;
|
|
MUL_SY yvec0, yvec5, yvec7;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3; # Bi2, Bi3, Bi0, Bi1
|
|
ADD1_SY yvec6, yvec11, yvec11;
|
|
ADD1_SY yvec7, yvec9, yvec9;
|
|
|
|
MUL_SY yvec1, yvec4, yvec6;
|
|
MUL_SY yvec1, yvec5, yvec7;
|
|
VPERMILP_SY $0xb1, yvec0, yvec0; # Ai0, Ar0, Ai1, Ar1..
|
|
ADD1_SY yvec6, yvec10, yvec10;
|
|
ADD1_SY yvec7, yvec8, yvec8;
|
|
|
|
VPERMILP_SY $0xb1, yvec1, yvec1;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
MUL_SY yvec0, yvec3, yvec7;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4; # Br1, Br0, Br3, Br2
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
ADD2_SY yvec7, yvec13, yvec13;
|
|
|
|
MUL_SY yvec1, yvec2, yvec6;
|
|
MUL_SY yvec1, yvec3, yvec7;
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5; # Br3, Br2, Br1, Br0
|
|
ADD2_SY yvec6, yvec14, yvec14;
|
|
ADD2_SY yvec7, yvec12, yvec12;
|
|
|
|
EDUP_SY 8*SIZE(ptrbb), yvec2;
|
|
MUL_SY yvec0, yvec4, yvec6;
|
|
MUL_SY yvec0, yvec5, yvec7;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3;
|
|
ADD2_SY yvec6, yvec11, yvec11;
|
|
ADD2_SY yvec7, yvec9, yvec9;
|
|
|
|
LD_SY 16*SIZE(ptrba), yvec0;
|
|
MUL_SY yvec1, yvec4, yvec6;
|
|
MUL_SY yvec1, yvec5, yvec7;
|
|
ADD2_SY yvec6, yvec10, yvec10;
|
|
ADD2_SY yvec7, yvec8, yvec8;
|
|
|
|
######### Unroll 2 ##################
|
|
PREFETCH0 (PRESIZE+16)*SIZE(ptrba)
|
|
LD_SY 24*SIZE(ptrba), yvec1; # Ar4, Ai4, Ar5, Ai5..
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
MUL_SY yvec0, yvec3, yvec7;
|
|
ADDQ $32*SIZE, ptrba
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4; # Br1, Br0, Br3, Br2
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
ADD1_SY yvec7, yvec13, yvec13;
|
|
|
|
MUL_SY yvec1, yvec2, yvec6;
|
|
MUL_SY yvec1, yvec3, yvec7;
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5; # Br3, Br2, Br1, Br0
|
|
ADD1_SY yvec6, yvec14, yvec14;
|
|
ADD1_SY yvec7, yvec12, yvec12;
|
|
|
|
ODUP_SY 8*SIZE(ptrbb), yvec2; # Bi0, Bi1, Bi2, Bi3
|
|
MUL_SY yvec0, yvec4, yvec6;
|
|
MUL_SY yvec0, yvec5, yvec7;
|
|
ADDQ $16*SIZE, ptrbb;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3; # Bi2, Bi3, Bi0, Bi1
|
|
ADD1_SY yvec6, yvec11, yvec11;
|
|
ADD1_SY yvec7, yvec9, yvec9;
|
|
|
|
MUL_SY yvec1, yvec4, yvec6;
|
|
MUL_SY yvec1, yvec5, yvec7;
|
|
VPERMILP_SY $0xb1, yvec0, yvec0; # Ai0, Ar0, Ai1, Ar1..
|
|
ADD1_SY yvec6, yvec10, yvec10;
|
|
ADD1_SY yvec7, yvec8, yvec8;
|
|
|
|
VPERMILP_SY $0xb1, yvec1, yvec1;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
MUL_SY yvec0, yvec3, yvec7;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4; # Br1, Br0, Br3, Br2
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
ADD2_SY yvec7, yvec13, yvec13;
|
|
|
|
MUL_SY yvec1, yvec2, yvec6;
|
|
MUL_SY yvec1, yvec3, yvec7;
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5; # Br3, Br2, Br1, Br0
|
|
ADD2_SY yvec6, yvec14, yvec14;
|
|
ADD2_SY yvec7, yvec12, yvec12;
|
|
|
|
EDUP_SY 0*SIZE(ptrbb), yvec2;
|
|
MUL_SY yvec0, yvec4, yvec6;
|
|
MUL_SY yvec0, yvec5, yvec7;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3;
|
|
ADD2_SY yvec6, yvec11, yvec11;
|
|
ADD2_SY yvec7, yvec9, yvec9;
|
|
|
|
LD_SY 0*SIZE(ptrba), yvec0;
|
|
MUL_SY yvec1, yvec4, yvec6;
|
|
MUL_SY yvec1, yvec5, yvec7;
|
|
ADD2_SY yvec6, yvec10, yvec10;
|
|
ADD2_SY yvec7, yvec8, yvec8;
|
|
.L3_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $1, bk;
|
|
#else
|
|
TEST $1, kkk;
|
|
#endif
|
|
JLE .L4_loopE;
|
|
ALIGN_5
|
|
.L4_loopB:;
|
|
######### Unroll 1 ##################
|
|
PREFETCH0 PRESIZE*SIZE(ptrba)
|
|
LD_SY 8*SIZE(ptrba), yvec1; # Ar4, Ai4, Ar5, Ai5..
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
MUL_SY yvec0, yvec3, yvec7;
|
|
ADDQ $16*SIZE, ptrba;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4; # Br1, Br0, Br3, Br2
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
ADD1_SY yvec7, yvec13, yvec13;
|
|
|
|
MUL_SY yvec1, yvec2, yvec6;
|
|
MUL_SY yvec1, yvec3, yvec7;
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5; # Br3, Br2, Br1, Br0
|
|
ADD1_SY yvec6, yvec14, yvec14;
|
|
ADD1_SY yvec7, yvec12, yvec12;
|
|
|
|
ODUP_SY 0*SIZE(ptrbb), yvec2; # Bi0, Bi1, Bi2, Bi3
|
|
MUL_SY yvec0, yvec4, yvec6;
|
|
MUL_SY yvec0, yvec5, yvec7;
|
|
ADDQ $8*SIZE, ptrbb;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3; # Bi2, Bi3, Bi0, Bi1
|
|
ADD1_SY yvec6, yvec11, yvec11;
|
|
ADD1_SY yvec7, yvec9, yvec9;
|
|
|
|
MUL_SY yvec1, yvec4, yvec6;
|
|
MUL_SY yvec1, yvec5, yvec7;
|
|
VPERMILP_SY $0xb1, yvec0, yvec0; # Ai0, Ar0, Ai1, Ar1..
|
|
ADD1_SY yvec6, yvec10, yvec10;
|
|
ADD1_SY yvec7, yvec8, yvec8;
|
|
|
|
VPERMILP_SY $0xb1, yvec1, yvec1;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
MUL_SY yvec0, yvec3, yvec7;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4; # Br1, Br0, Br3, Br2
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
ADD2_SY yvec7, yvec13, yvec13;
|
|
|
|
MUL_SY yvec1, yvec2, yvec6;
|
|
ADD2_SY yvec6, yvec14, yvec14;
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5; # Br3, Br2, Br1, Br0
|
|
MUL_SY yvec1, yvec3, yvec7;
|
|
ADD2_SY yvec7, yvec12, yvec12;
|
|
|
|
MUL_SY yvec0, yvec4, yvec6;
|
|
MUL_SY yvec0, yvec5, yvec7;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3;
|
|
ADD2_SY yvec6, yvec11, yvec11;
|
|
ADD2_SY yvec7, yvec9, yvec9;
|
|
|
|
MUL_SY yvec1, yvec4, yvec6;
|
|
MUL_SY yvec1, yvec5, yvec7;
|
|
ADD2_SY yvec6, yvec10, yvec10;
|
|
ADD2_SY yvec7, yvec8, yvec8;
|
|
|
|
.L4_loopE:;
|
|
#### Handle ####
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
#if defined(RN) || defined(RT) || defined(CN) || defined(CT)
|
|
ADDSUB_SY yvec15, yvec7, yvec15;
|
|
ADDSUB_SY yvec14, yvec7, yvec14;
|
|
ADDSUB_SY yvec13, yvec7, yvec13;
|
|
ADDSUB_SY yvec12, yvec7, yvec12;
|
|
ADDSUB_SY yvec11, yvec7, yvec11;
|
|
ADDSUB_SY yvec10, yvec7, yvec10;
|
|
ADDSUB_SY yvec9, yvec7, yvec9;
|
|
ADDSUB_SY yvec8, yvec7, yvec8;
|
|
#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
|
|
SUB_SY yvec15, yvec7, yvec15;
|
|
SUB_SY yvec14, yvec7, yvec14;
|
|
SUB_SY yvec13, yvec7, yvec13;
|
|
SUB_SY yvec12, yvec7, yvec12;
|
|
SUB_SY yvec11, yvec7, yvec11;
|
|
SUB_SY yvec10, yvec7, yvec10;
|
|
SUB_SY yvec9, yvec7, yvec9;
|
|
SUB_SY yvec8, yvec7, yvec8;
|
|
#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
|
|
VPERMILP_SY $0xb1, yvec15, yvec15;
|
|
VPERMILP_SY $0xb1, yvec14, yvec14;
|
|
VPERMILP_SY $0xb1, yvec13, yvec13;
|
|
VPERMILP_SY $0xb1, yvec12, yvec12;
|
|
VPERMILP_SY $0xb1, yvec11, yvec11;
|
|
VPERMILP_SY $0xb1, yvec10, yvec10;
|
|
VPERMILP_SY $0xb1, yvec9, yvec9;
|
|
VPERMILP_SY $0xb1, yvec8, yvec8;
|
|
ADDSUB_SY yvec15, yvec7, yvec15;
|
|
ADDSUB_SY yvec14, yvec7, yvec14;
|
|
ADDSUB_SY yvec13, yvec7, yvec13;
|
|
ADDSUB_SY yvec12, yvec7, yvec12;
|
|
ADDSUB_SY yvec11, yvec7, yvec11;
|
|
ADDSUB_SY yvec10, yvec7, yvec10;
|
|
ADDSUB_SY yvec9, yvec7, yvec9;
|
|
ADDSUB_SY yvec8, yvec7, yvec8;
|
|
VPERMILP_SY $0xb1, yvec15, yvec15;
|
|
VPERMILP_SY $0xb1, yvec14, yvec14;
|
|
VPERMILP_SY $0xb1, yvec13, yvec13;
|
|
VPERMILP_SY $0xb1, yvec12, yvec12;
|
|
VPERMILP_SY $0xb1, yvec11, yvec11;
|
|
VPERMILP_SY $0xb1, yvec10, yvec10;
|
|
VPERMILP_SY $0xb1, yvec9, yvec9;
|
|
VPERMILP_SY $0xb1, yvec8, yvec8;
|
|
#endif
|
|
##### Load Alpha ####
|
|
BROAD_SY MEMALPHA_R,yvec7;
|
|
BROAD_SY MEMALPHA_I,yvec6;
|
|
##### Multiply Alpha ####
|
|
VPERMILP_SY $0xb1,yvec15, yvec5;
|
|
MUL_SY yvec15, yvec7, yvec15;
|
|
MUL_SY yvec5, yvec6, yvec5;
|
|
ADDSUB_SY yvec5, yvec15, yvec15;
|
|
VPERMILP_SY $0xb1,yvec14, yvec4;
|
|
MUL_SY yvec14, yvec7, yvec14;
|
|
MUL_SY yvec4, yvec6, yvec4;
|
|
ADDSUB_SY yvec4, yvec14, yvec14;
|
|
VPERMILP_SY $0xb1,yvec13, yvec3;
|
|
MUL_SY yvec13, yvec7, yvec13;
|
|
MUL_SY yvec3, yvec6, yvec3;
|
|
ADDSUB_SY yvec3, yvec13, yvec13;
|
|
VPERMILP_SY $0xb1,yvec12, yvec2;
|
|
MUL_SY yvec12, yvec7, yvec12;
|
|
MUL_SY yvec2, yvec6, yvec2;
|
|
ADDSUB_SY yvec2, yvec12, yvec12;
|
|
VPERMILP_SY $0xb1,yvec11, yvec1;
|
|
MUL_SY yvec11, yvec7, yvec11;
|
|
MUL_SY yvec1, yvec6, yvec1;
|
|
ADDSUB_SY yvec1, yvec11, yvec11;
|
|
VPERMILP_SY $0xb1,yvec10, yvec0;
|
|
MUL_SY yvec10, yvec7, yvec10;
|
|
MUL_SY yvec0, yvec6, yvec0;
|
|
ADDSUB_SY yvec0, yvec10, yvec10;
|
|
VPERMILP_SY $0xb1,yvec9, yvec5;
|
|
MUL_SY yvec9, yvec7, yvec9;
|
|
MUL_SY yvec5, yvec6, yvec5;
|
|
ADDSUB_SY yvec5, yvec9, yvec9;
|
|
VPERMILP_SY $0xb1,yvec8, yvec4;
|
|
MUL_SY yvec8, yvec7, yvec8;
|
|
MUL_SY yvec4, yvec6, yvec4;
|
|
ADDSUB_SY yvec4, yvec8, yvec8;
|
|
#### Shuffle Results ####
|
|
MOV_SY yvec15,yvec7;
|
|
REVS_SY $0xe4,yvec13,yvec15,yvec15;
|
|
REVS_SY $0xe4,yvec7,yvec13,yvec13;
|
|
MOV_SY yvec14,yvec7;
|
|
REVS_SY $0xe4,yvec12,yvec14,yvec14;
|
|
REVS_SY $0xe4,yvec7,yvec12,yvec12;
|
|
MOV_SY yvec11,yvec7;
|
|
REVS_SY $0xe4,yvec9,yvec11,yvec11;
|
|
REVS_SY $0xe4,yvec7,yvec9,yvec9;
|
|
MOV_SY yvec10,yvec7;
|
|
REVS_SY $0xe4,yvec8,yvec10,yvec10;
|
|
REVS_SY $0xe4,yvec7,yvec8,yvec8;
|
|
#### Store Back ####
|
|
#### Testing alignment ####
|
|
MOVQ C0, %rax;
|
|
OR ldc, %rax;
|
|
TEST $15, %rax;
|
|
JNE .L4_loopEx;
|
|
ALIGN_5
|
|
EXTRA_SY $1,yvec15,xvec7;
|
|
EXTRA_SY $1,yvec14,xvec6;
|
|
EXTRA_SY $1,yvec13,xvec5;
|
|
EXTRA_SY $1,yvec12,xvec4;
|
|
EXTRA_SY $1,yvec11,xvec3;
|
|
EXTRA_SY $1,yvec10,xvec2;
|
|
EXTRA_SY $1,yvec9,xvec1;
|
|
EXTRA_SY $1,yvec8,xvec0;
|
|
#ifndef TRMMKERNEL
|
|
ADD_SY 0*SIZE(C0),xvec15, xvec15;
|
|
ADD_SY 4*SIZE(C1),xvec7, xvec7;
|
|
ADD_SY 8*SIZE(C0),xvec14, xvec14;
|
|
ADD_SY 12*SIZE(C1),xvec6, xvec6;
|
|
ADD_SY 0*SIZE(C0,ldc,1),xvec13, xvec13;
|
|
ADD_SY 4*SIZE(C1,ldc,1),xvec5, xvec5;
|
|
ADD_SY 8*SIZE(C0,ldc,1),xvec12, xvec12;
|
|
ADD_SY 12*SIZE(C1,ldc,1),xvec4, xvec4;
|
|
ADD_SY 0*SIZE(C1),xvec11, xvec11;
|
|
ADD_SY 4*SIZE(C0),xvec3, xvec3;
|
|
ADD_SY 8*SIZE(C1),xvec10, xvec10;
|
|
ADD_SY 12*SIZE(C0),xvec2, xvec2;
|
|
ADD_SY 0*SIZE(C1,ldc,1),xvec9, xvec9;
|
|
ADD_SY 4*SIZE(C0,ldc,1),xvec1, xvec1;
|
|
ADD_SY 8*SIZE(C1,ldc,1),xvec8, xvec8;
|
|
ADD_SY 12*SIZE(C0,ldc,1),xvec0, xvec0;
|
|
#endif
|
|
ST_SY xvec15,0*SIZE(C0);
|
|
ST_SY xvec7,4*SIZE(C1);
|
|
ST_SY xvec14,8*SIZE(C0);
|
|
ST_SY xvec6,12*SIZE(C1);
|
|
ST_SY xvec13,0*SIZE(C0,ldc,1);
|
|
ST_SY xvec5,4*SIZE(C1,ldc,1);
|
|
ST_SY xvec12,8*SIZE(C0,ldc,1);
|
|
ST_SY xvec4,12*SIZE(C1,ldc,1);
|
|
ST_SY xvec11,0*SIZE(C1);
|
|
ST_SY xvec3,4*SIZE(C0);
|
|
ST_SY xvec10,8*SIZE(C1);
|
|
ST_SY xvec2,12*SIZE(C0);
|
|
ST_SY xvec9,0*SIZE(C1,ldc,1);
|
|
ST_SY xvec1,4*SIZE(C0,ldc,1);
|
|
ST_SY xvec8,8*SIZE(C1,ldc,1);
|
|
ST_SY xvec0,12*SIZE(C0,ldc,1);
|
|
#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kkk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 8), ptrba;
|
|
LEAQ (ptrbb, %rax, 4), ptrbb;
|
|
#endif
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
ADDQ $8, kk;
|
|
#endif
|
|
ADDQ $16*SIZE,C0;
|
|
ADDQ $16*SIZE,C1;
|
|
.L1_bodyE:;
|
|
DECQ i;
|
|
JG .L1_bodyB;
|
|
JMP .L1_loopE;
|
|
ALIGN_5
|
|
.L4_loopEx:
|
|
EXTRA_SY $1, yvec15, xvec7;
|
|
#ifndef TRMMKERNEL
|
|
LDL_SY 0*SIZE(C0), xvec6, xvec6;
|
|
LDH_SY 2*SIZE(C0), xvec6, xvec6;
|
|
ADD_SY xvec6, xvec15, xvec15;
|
|
#endif
|
|
STL_SY xvec15, 0*SIZE(C0);
|
|
STH_SY xvec15, 2*SIZE(C0);
|
|
#ifndef TRMMKERNEL
|
|
LDL_SY 4*SIZE(C1), xvec5, xvec5;
|
|
LDH_SY 6*SIZE(C1), xvec5, xvec5;
|
|
ADD_SY xvec5, xvec7, xvec7;
|
|
#endif
|
|
STL_SY xvec7, 4*SIZE(C1);
|
|
STH_SY xvec7, 6*SIZE(C1);
|
|
|
|
EXTRA_SY $1, yvec14, xvec6;
|
|
#ifndef TRMMKERNEL
|
|
LDL_SY 8*SIZE(C0), xvec5, xvec5;
|
|
LDH_SY 10*SIZE(C0), xvec5, xvec5;
|
|
ADD_SY xvec5, xvec14, xvec14;
|
|
#endif
|
|
STL_SY xvec14, 8*SIZE(C0);
|
|
STH_SY xvec14, 10*SIZE(C0);
|
|
#ifndef TRMMKERNEL
|
|
LDL_SY 12*SIZE(C1), xvec4, xvec4;
|
|
LDH_SY 14*SIZE(C1), xvec4, xvec4;
|
|
ADD_SY xvec4, xvec6, xvec6;
|
|
#endif
|
|
STL_SY xvec6, 12*SIZE(C1);
|
|
STH_SY xvec6, 14*SIZE(C1);
|
|
|
|
EXTRA_SY $1, yvec13, xvec5;
|
|
#ifndef TRMMKERNEL
|
|
LDL_SY 0*SIZE(C0, ldc, 1), xvec4, xvec4;
|
|
LDH_SY 2*SIZE(C0, ldc, 1), xvec4, xvec4;
|
|
ADD_SY xvec4, xvec13, xvec13;
|
|
#endif
|
|
STL_SY xvec13, 0*SIZE(C0, ldc, 1);
|
|
STH_SY xvec13, 2*SIZE(C0, ldc, 1);
|
|
#ifndef TRMMKERNEL
|
|
LDL_SY 4*SIZE(C1, ldc, 1), xvec3, xvec3;
|
|
LDH_SY 6*SIZE(C1, ldc, 1), xvec3, xvec3;
|
|
ADD_SY xvec3, xvec5, xvec5;
|
|
#endif
|
|
STL_SY xvec5, 4*SIZE(C1, ldc, 1);
|
|
STH_SX xvec5, 6*SIZE(C1, ldc, 1);
|
|
|
|
EXTRA_SY $1, yvec12, xvec4;
|
|
#ifndef TRMMKERNEL
|
|
LDL_SY 8*SIZE(C0, ldc, 1), xvec3, xvec3;
|
|
LDH_SY 10*SIZE(C0, ldc, 1), xvec3, xvec3;
|
|
ADD_SY xvec3, xvec12, xvec12;
|
|
#endif
|
|
STL_SY xvec12, 8*SIZE(C0, ldc, 1);
|
|
STH_SY xvec12, 10*SIZE(C0, ldc, 1);
|
|
#ifndef TRMMKERNEL
|
|
LDL_SY 12*SIZE(C1, ldc, 1), xvec2, xvec2;
|
|
LDH_SY 14*SIZE(C1, ldc, 1), xvec2, xvec2;
|
|
ADD_SY xvec2, xvec4, xvec4;
|
|
#endif
|
|
STL_SY xvec4, 12*SIZE(C1, ldc, 1);
|
|
STH_SY xvec4, 14*SIZE(C1, ldc, 1);
|
|
|
|
EXTRA_SY $1, yvec11, xvec3;
|
|
#ifndef TRMMKERNEL
|
|
LDL_SY 0*SIZE(C1), xvec2, xvec2;
|
|
LDH_SY 2*SIZE(C1), xvec2, xvec2;
|
|
ADD_SY xvec2, xvec11, xvec11;
|
|
#endif
|
|
STL_SY xvec11, 0*SIZE(C1);
|
|
STH_SY xvec11, 2*SIZE(C1);
|
|
#ifndef TRMMKERNEL
|
|
LDL_SY 4*SIZE(C0), xvec1, xvec1;
|
|
LDH_SY 6*SIZE(C0), xvec1, xvec1;
|
|
ADD_SY xvec1, xvec3, xvec3;
|
|
#endif
|
|
STL_SY xvec3, 4*SIZE(C0);
|
|
STH_SY xvec3, 6*SIZE(C0);
|
|
|
|
EXTRA_SY $1, yvec10, xvec2;
|
|
#ifndef TRMMKERNEL
|
|
LDL_SY 8*SIZE(C1), xvec1, xvec1;
|
|
LDH_SY 10*SIZE(C1), xvec1, xvec1;
|
|
ADD_SY xvec1, xvec10, xvec10;
|
|
#endif
|
|
STL_SY xvec10, 8*SIZE(C1);
|
|
STH_SY xvec10, 10*SIZE(C1);
|
|
#ifndef TRMMKERNEL
|
|
LDL_SY 12*SIZE(C0), xvec0, xvec0;
|
|
LDH_SY 14*SIZE(C0), xvec0, xvec0;
|
|
ADD_SY xvec0, xvec2, xvec2;
|
|
#endif
|
|
STL_SY xvec2, 12*SIZE(C0);
|
|
STH_SY xvec2, 14*SIZE(C0);
|
|
|
|
EXTRA_SY $1, yvec9, xvec1;
|
|
#ifndef TRMMKERNEL
|
|
LDL_SY 0*SIZE(C1, ldc, 1), xvec7, xvec7;
|
|
LDH_SY 2*SIZE(C1, ldc, 1), xvec7, xvec7;
|
|
ADD_SY xvec7, xvec9, xvec9;
|
|
#endif
|
|
STL_SY xvec9, 0*SIZE(C1, ldc, 1);
|
|
STH_SY xvec9, 2*SIZE(C1, ldc, 1);
|
|
#ifndef TRMMKERNEL
|
|
LDL_SY 4*SIZE(C0, ldc, 1), xvec6, xvec6;
|
|
LDH_SY 6*SIZE(C0, ldc, 1), xvec6, xvec6;
|
|
ADD_SY xvec6, xvec1, xvec1;
|
|
#endif
|
|
STL_SY xvec1, 4*SIZE(C0, ldc, 1);
|
|
STH_SY xvec1, 6*SIZE(C0, ldc, 1);
|
|
|
|
EXTRA_SY $1, yvec8, xvec0;
|
|
#ifndef TRMMKERNEL
|
|
LDL_SY 8*SIZE(C1, ldc, 1), xvec6, xvec6;
|
|
LDH_SY 10*SIZE(C1, ldc, 1), xvec6, xvec6;
|
|
ADD_SY xvec6, xvec8, xvec8;
|
|
#endif
|
|
STL_SY xvec8, 8*SIZE(C1, ldc, 1);
|
|
STH_SY xvec8, 10*SIZE(C1, ldc, 1);
|
|
#ifndef TRMMKERNEL
|
|
LDL_SY 12*SIZE(C0, ldc, 1), xvec5, xvec5;
|
|
LDH_SY 14*SIZE(C0, ldc, 1), xvec5, xvec5;
|
|
ADD_SY xvec5, xvec0, xvec0;
|
|
#endif
|
|
STL_SY xvec0, 12*SIZE(C0, ldc, 1);
|
|
STH_SY xvec0, 14*SIZE(C0, ldc, 1);
|
|
#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kkk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 8), ptrba;
|
|
LEAQ (ptrbb, %rax, 4), ptrbb;
|
|
#endif
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
ADDQ $8, kk;
|
|
#endif
|
|
ADDQ $16*SIZE, C0;
|
|
ADDQ $16*SIZE, C1;
|
|
DECQ i;
|
|
JG .L1_bodyB;
|
|
ALIGN_5;
|
|
.L1_loopE:;
|
|
TEST $4, bm;
|
|
JLE .L5_loopE;
|
|
ALIGN_5
|
|
.L5_bodyB:
|
|
#if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA))
|
|
MOVQ bb,ptrbb;
|
|
#else
|
|
MOVQ bb, ptrbb;
|
|
MOVQ kk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 4), ptrba;
|
|
LEAQ (ptrbb, %rax, 4), ptrbb;
|
|
#endif
|
|
XOR_SY yvec15, yvec15, yvec15;
|
|
XOR_SY yvec13, yvec13, yvec13;
|
|
XOR_SY yvec11, yvec11, yvec11;
|
|
XOR_SY yvec9, yvec9, yvec9;
|
|
#ifndef TRMMKERNEL
|
|
MOVQ bk,k;
|
|
#elif (defined(LEFT)&&!defined(TRANSA))||(!defined(LEFT)&&defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kk, %rax;
|
|
MOVQ %rax, kkk;
|
|
#else
|
|
MOVQ kk, %rax;
|
|
#ifdef LEFT
|
|
ADDQ $4, %rax;
|
|
#else
|
|
ADDQ $4, %rax;
|
|
#endif
|
|
MOVQ %rax, kkk;
|
|
#endif
|
|
SARQ $2, k;
|
|
JLE .L8_loopE;
|
|
ALIGN_5
|
|
.L8_bodyB:
|
|
#### Unroll times 1 ####
|
|
LD_SY 0*SIZE(ptrba), yvec0;
|
|
VPERMILP_SY $0xb1, yvec0, yvec1;
|
|
EDUP_SY 0*SIZE(ptrbb), yvec2;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_SY yvec0, yvec3, yvec7;
|
|
ADD1_SY yvec7, yvec13, yvec13;
|
|
|
|
ODUP_SY 0*SIZE(ptrbb), yvec2;
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5;
|
|
MUL_SY yvec0, yvec4, yvec6;
|
|
ADD1_SY yvec6, yvec11, yvec11;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3;
|
|
MUL_SY yvec0, yvec5, yvec7;
|
|
ADD1_SY yvec7, yvec9, yvec9;
|
|
|
|
MUL_SY yvec1, yvec2, yvec6;
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_SY yvec1, yvec3, yvec7;
|
|
ADD2_SY yvec7, yvec13, yvec13;
|
|
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5;
|
|
MUL_SY yvec1, yvec4, yvec6;
|
|
ADD2_SY yvec6, yvec11, yvec11;
|
|
MUL_SY yvec1, yvec5, yvec7;
|
|
ADD2_SY yvec7, yvec9, yvec9;
|
|
|
|
#### Unroll time 2 ####
|
|
LD_SY 8*SIZE(ptrba), yvec0;
|
|
VPERMILP_SY $0xb1, yvec0, yvec1;
|
|
EDUP_SY 8*SIZE(ptrbb), yvec2;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_SY yvec0, yvec3, yvec7;
|
|
ADD1_SY yvec7, yvec13, yvec13;
|
|
|
|
ODUP_SY 8*SIZE(ptrbb), yvec2;
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5;
|
|
MUL_SY yvec0, yvec4, yvec6;
|
|
ADD1_SY yvec6, yvec11, yvec11;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3;
|
|
MUL_SY yvec0, yvec5, yvec7;
|
|
ADD1_SY yvec7, yvec9, yvec9;
|
|
|
|
MUL_SY yvec1, yvec2, yvec6;
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_SY yvec1, yvec3, yvec7;
|
|
ADD2_SY yvec7, yvec13, yvec13;
|
|
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5;
|
|
MUL_SY yvec1, yvec4, yvec6;
|
|
ADD2_SY yvec6, yvec11, yvec11;
|
|
MUL_SY yvec1, yvec5, yvec7;
|
|
ADD2_SY yvec7, yvec9, yvec9;
|
|
|
|
#### Unroll time 3 ####
|
|
LD_SY 16*SIZE(ptrba), yvec0;
|
|
VPERMILP_SY $0xb1, yvec0, yvec1;
|
|
EDUP_SY 16*SIZE(ptrbb), yvec2;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_SY yvec0, yvec3, yvec7;
|
|
ADD1_SY yvec7, yvec13, yvec13;
|
|
|
|
ODUP_SY 16*SIZE(ptrbb), yvec2;
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5;
|
|
MUL_SY yvec0, yvec4, yvec6;
|
|
ADD1_SY yvec6, yvec11, yvec11;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3;
|
|
MUL_SY yvec0, yvec5, yvec7;
|
|
ADD1_SY yvec7, yvec9, yvec9;
|
|
|
|
MUL_SY yvec1, yvec2, yvec6;
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_SY yvec1, yvec3, yvec7;
|
|
ADD2_SY yvec7, yvec13, yvec13;
|
|
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5;
|
|
MUL_SY yvec1, yvec4, yvec6;
|
|
ADD2_SY yvec6, yvec11, yvec11;
|
|
MUL_SY yvec1, yvec5, yvec7;
|
|
ADD2_SY yvec7, yvec9, yvec9;
|
|
|
|
#### Unroll time 3 ####
|
|
LD_SY 24*SIZE(ptrba), yvec0;
|
|
VPERMILP_SY $0xb1, yvec0, yvec1;
|
|
EDUP_SY 24*SIZE(ptrbb), yvec2;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_SY yvec0, yvec3, yvec7;
|
|
ADD1_SY yvec7, yvec13, yvec13;
|
|
|
|
ODUP_SY 24*SIZE(ptrbb), yvec2;
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5;
|
|
MUL_SY yvec0, yvec4, yvec6;
|
|
ADD1_SY yvec6, yvec11, yvec11;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3;
|
|
MUL_SY yvec0, yvec5, yvec7;
|
|
ADD1_SY yvec7, yvec9, yvec9;
|
|
|
|
MUL_SY yvec1, yvec2, yvec6;
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_SY yvec1, yvec3, yvec7;
|
|
ADD2_SY yvec7, yvec13, yvec13;
|
|
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5;
|
|
MUL_SY yvec1, yvec4, yvec6;
|
|
ADD2_SY yvec6, yvec11, yvec11;
|
|
MUL_SY yvec1, yvec5, yvec7;
|
|
ADD2_SY yvec7, yvec9, yvec9;
|
|
ADDQ $32*SIZE, ptrba;
|
|
ADDQ $32*SIZE, ptrbb;
|
|
DECQ k;
|
|
JG .L8_bodyB;
|
|
ALIGN_5
|
|
.L8_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $2, bk;
|
|
#else
|
|
TEST $2, kkk;
|
|
#endif
|
|
JLE .L9_loopE;
|
|
ALIGN_5
|
|
.L9_bodyB:
|
|
#### Unroll times 1 ####
|
|
LD_SY 0*SIZE(ptrba), yvec0;
|
|
VPERMILP_SY $0xb1, yvec0, yvec1;
|
|
EDUP_SY 0*SIZE(ptrbb), yvec2;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_SY yvec0, yvec3, yvec7;
|
|
ADD1_SY yvec7, yvec13, yvec13;
|
|
|
|
ODUP_SY 0*SIZE(ptrbb), yvec2;
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5;
|
|
MUL_SY yvec0, yvec4, yvec6;
|
|
ADD1_SY yvec6, yvec11, yvec11;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3;
|
|
MUL_SY yvec0, yvec5, yvec7;
|
|
ADD1_SY yvec7, yvec9, yvec9;
|
|
|
|
MUL_SY yvec1, yvec2, yvec6;
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_SY yvec1, yvec3, yvec7;
|
|
ADD2_SY yvec7, yvec13, yvec13;
|
|
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5;
|
|
MUL_SY yvec1, yvec4, yvec6;
|
|
ADD2_SY yvec6, yvec11, yvec11;
|
|
MUL_SY yvec1, yvec5, yvec7;
|
|
ADD2_SY yvec7, yvec9, yvec9;
|
|
|
|
#### Unroll time 2 ####
|
|
LD_SY 8*SIZE(ptrba), yvec0;
|
|
VPERMILP_SY $0xb1, yvec0, yvec1;
|
|
EDUP_SY 8*SIZE(ptrbb), yvec2;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_SY yvec0, yvec3, yvec7;
|
|
ADD1_SY yvec7, yvec13, yvec13;
|
|
|
|
ODUP_SY 8*SIZE(ptrbb), yvec2;
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5;
|
|
MUL_SY yvec0, yvec4, yvec6;
|
|
ADD1_SY yvec6, yvec11, yvec11;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3;
|
|
MUL_SY yvec0, yvec5, yvec7;
|
|
ADD1_SY yvec7, yvec9, yvec9;
|
|
|
|
MUL_SY yvec1, yvec2, yvec6;
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_SY yvec1, yvec3, yvec7;
|
|
ADD2_SY yvec7, yvec13, yvec13;
|
|
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5;
|
|
MUL_SY yvec1, yvec4, yvec6;
|
|
ADD2_SY yvec6, yvec11, yvec11;
|
|
MUL_SY yvec1, yvec5, yvec7;
|
|
ADD2_SY yvec7, yvec9, yvec9;
|
|
ADDQ $16*SIZE, ptrba;
|
|
ADDQ $16*SIZE, ptrbb;
|
|
|
|
.L9_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $1, bk;
|
|
#else
|
|
TEST $1, kkk;
|
|
#endif
|
|
JLE .L10_loopE;
|
|
ALIGN_5
|
|
.L10_bodyB:
|
|
#### Unroll times 1 ####
|
|
LD_SY 0*SIZE(ptrba), yvec0;
|
|
VPERMILP_SY $0xb1, yvec0, yvec1;
|
|
EDUP_SY 0*SIZE(ptrbb), yvec2;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_SY yvec0, yvec3, yvec7;
|
|
ADD1_SY yvec7, yvec13, yvec13;
|
|
|
|
ODUP_SY 0*SIZE(ptrbb), yvec2;
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5;
|
|
MUL_SY yvec0, yvec4, yvec6;
|
|
ADD1_SY yvec6, yvec11, yvec11;
|
|
VPERMILP_SY $0x4e, yvec2, yvec3;
|
|
MUL_SY yvec0, yvec5, yvec7;
|
|
ADD1_SY yvec7, yvec9, yvec9;
|
|
|
|
MUL_SY yvec1, yvec2, yvec6;
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
SHUF_SY $0x03, yvec2, yvec2, yvec4;
|
|
MUL_SY yvec1, yvec3, yvec7;
|
|
ADD2_SY yvec7, yvec13, yvec13;
|
|
|
|
SHUF_SY $0x03, yvec3, yvec3, yvec5;
|
|
MUL_SY yvec1, yvec4, yvec6;
|
|
ADD2_SY yvec6, yvec11, yvec11;
|
|
MUL_SY yvec1, yvec5, yvec7;
|
|
ADD2_SY yvec7, yvec9, yvec9;
|
|
ADDQ $8*SIZE, ptrba;
|
|
ADDQ $8*SIZE, ptrbb;
|
|
|
|
.L10_loopE:
|
|
#### Handle ####
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
#if defined(RN) || defined(RT) || defined(CN) || defined(CT)
|
|
ADDSUB_SY yvec15, yvec7, yvec15;
|
|
ADDSUB_SY yvec13, yvec7, yvec13;
|
|
ADDSUB_SY yvec11, yvec7, yvec11;
|
|
ADDSUB_SY yvec9, yvec7, yvec9;
|
|
#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
|
|
SUB_SY yvec15, yvec7, yvec15;
|
|
SUB_SY yvec13, yvec7, yvec13;
|
|
SUB_SY yvec11, yvec7, yvec11;
|
|
SUB_SY yvec9, yvec7, yvec9;
|
|
#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
|
|
VPERMILP_SY $0xb1, yvec15, yvec15;
|
|
VPERMILP_SY $0xb1, yvec13, yvec13;
|
|
VPERMILP_SY $0xb1, yvec11, yvec11;
|
|
VPERMILP_SY $0xb1, yvec9, yvec9;
|
|
ADDSUB_SY yvec15, yvec7, yvec15;
|
|
ADDSUB_SY yvec13, yvec7, yvec13;
|
|
ADDSUB_SY yvec11, yvec7, yvec11;
|
|
ADDSUB_SY yvec9, yvec7, yvec9;
|
|
VPERMILP_SY $0xb1, yvec15, yvec15;
|
|
VPERMILP_SY $0xb1, yvec13, yvec13;
|
|
VPERMILP_SY $0xb1, yvec11, yvec11;
|
|
VPERMILP_SY $0xb1, yvec9, yvec9;
|
|
#endif
|
|
##### Load Alpha ####
|
|
BROAD_SY MEMALPHA_R,yvec7;
|
|
BROAD_SY MEMALPHA_I,yvec6;
|
|
##### Multiply Alpha ####
|
|
VPERMILP_SY $0xb1,yvec15, yvec5;
|
|
MUL_SY yvec15, yvec7, yvec15;
|
|
MUL_SY yvec5, yvec6, yvec5;
|
|
ADDSUB_SY yvec5, yvec15, yvec15;
|
|
VPERMILP_SY $0xb1,yvec13, yvec3;
|
|
MUL_SY yvec13, yvec7, yvec13;
|
|
MUL_SY yvec3, yvec6, yvec3;
|
|
ADDSUB_SY yvec3, yvec13, yvec13;
|
|
VPERMILP_SY $0xb1,yvec11, yvec1;
|
|
MUL_SY yvec11, yvec7, yvec11;
|
|
MUL_SY yvec1, yvec6, yvec1;
|
|
ADDSUB_SY yvec1, yvec11, yvec11;
|
|
VPERMILP_SY $0xb1,yvec9, yvec5;
|
|
MUL_SY yvec9, yvec7, yvec9;
|
|
MUL_SY yvec5, yvec6, yvec5;
|
|
ADDSUB_SY yvec5, yvec9, yvec9;
|
|
#### Writing back ####
|
|
#### Shuffle Results ####
|
|
MOV_SY yvec15,yvec7;
|
|
REVS_SY $0xe4,yvec13,yvec15,yvec15;
|
|
REVS_SY $0xe4,yvec7,yvec13,yvec13;
|
|
MOV_SY yvec11,yvec7;
|
|
REVS_SY $0xe4,yvec9,yvec11,yvec11;
|
|
REVS_SY $0xe4,yvec7,yvec9,yvec9;
|
|
#### Writing back ####
|
|
EXTRA_SY $1, yvec15, xvec7;
|
|
#ifndef TRMMKERNEL
|
|
LDL_SX 0*SIZE(C0), xvec6, xvec6;
|
|
LDH_SX 2*SIZE(C0), xvec6, xvec6;
|
|
ADD_SX xvec6, xvec15, xvec15;
|
|
#endif
|
|
STL_SX xvec15, 0*SIZE(C0);
|
|
STH_SX xvec15, 2*SIZE(C0);
|
|
#ifndef TRMMKERNEL
|
|
LDL_SX 4*SIZE(C1), xvec4, xvec4;
|
|
LDH_SX 6*SIZE(C1), xvec4, xvec4;
|
|
ADD_SX xvec4, xvec7, xvec7;
|
|
#endif
|
|
STL_SX xvec7, 4*SIZE(C1);
|
|
STH_SX xvec7, 6*SIZE(C1);
|
|
|
|
EXTRA_SY $1, yvec13, xvec5;
|
|
#ifndef TRMMKERNEL
|
|
LDL_SX 0*SIZE(C0, ldc, 1), xvec4, xvec4;
|
|
LDH_SX 2*SIZE(C0, ldc, 1), xvec4, xvec4;
|
|
ADD_SX xvec4, xvec13, xvec13;
|
|
#endif
|
|
STL_SX xvec13, 0*SIZE(C0, ldc, 1);
|
|
STH_SX xvec13, 2*SIZE(C0, ldc, 1);
|
|
#ifndef TRMMKERNEL
|
|
LDL_SX 4*SIZE(C1, ldc, 1), xvec2, xvec2;
|
|
LDH_SX 6*SIZE(C1, ldc, 1), xvec2, xvec2;
|
|
ADD_SX xvec2, xvec5, xvec5;
|
|
#endif
|
|
STL_SX xvec5, 4*SIZE(C1, ldc, 1);
|
|
STH_SX xvec5, 6*SIZE(C1, ldc, 1);
|
|
|
|
EXTRA_SY $1, yvec11, xvec3;
|
|
#ifndef TRMMKERNEL
|
|
LDL_SX 0*SIZE(C1), xvec2, xvec2;
|
|
LDH_SX 2*SIZE(C1), xvec2, xvec2;
|
|
ADD_SX xvec2, xvec11, xvec11;
|
|
#endif
|
|
STL_SX xvec11, 0*SIZE(C1);
|
|
STH_SX xvec11, 2*SIZE(C1);
|
|
#ifndef TRMMKERNEL
|
|
LDL_SX 4*SIZE(C0), xvec0, xvec0;
|
|
LDH_SX 6*SIZE(C0), xvec0, xvec0;
|
|
ADD_SX xvec0, xvec3, xvec3;
|
|
#endif
|
|
STL_SX xvec3, 4*SIZE(C0);
|
|
STH_SX xvec3, 6*SIZE(C0);
|
|
|
|
EXTRA_SY $1, yvec9, xvec1;
|
|
#ifndef TRMMKERNEL
|
|
LDL_SX 0*SIZE(C1, ldc, 1), xvec0, xvec0;
|
|
LDH_SX 2*SIZE(C1, ldc, 1), xvec0, xvec0;
|
|
ADD_SX xvec0, xvec9, xvec9;
|
|
#endif
|
|
STL_SX xvec9, 0*SIZE(C1, ldc, 1);
|
|
STH_SX xvec9, 2*SIZE(C1, ldc, 1);
|
|
#ifndef TRMMKERNEL
|
|
LDL_SX 4*SIZE(C0, ldc, 1), xvec6, xvec6;
|
|
LDH_SX 6*SIZE(C0, ldc, 1), xvec6, xvec6;
|
|
ADD_SX xvec6, xvec1, xvec1;
|
|
#endif
|
|
STL_SX xvec1, 4*SIZE(C0, ldc, 1);
|
|
STH_SX xvec1, 6*SIZE(C0, ldc, 1);
|
|
#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kkk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 4), ptrba;
|
|
LEAQ (ptrbb, %rax, 4), ptrbb;
|
|
#endif
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
ADDQ $4, kk;
|
|
#endif
|
|
|
|
ADDQ $8*SIZE, C0;
|
|
ADDQ $8*SIZE, C1;
|
|
|
|
.L5_loopE:
|
|
TEST $2, bm;
|
|
JLE .L6_loopE;
|
|
ALIGN_5
|
|
.L6_bodyB:
|
|
#if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA))
|
|
MOVQ bb,ptrbb;
|
|
#else
|
|
MOVQ bb, ptrbb;
|
|
MOVQ kk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 2), ptrba;
|
|
LEAQ (ptrbb, %rax, 4), ptrbb;
|
|
#endif
|
|
#### Initial Results Register ####
|
|
XOR_SY yvec15, yvec15, yvec15;
|
|
XOR_SY yvec14, yvec14, yvec14;
|
|
XOR_SY yvec13, yvec13, yvec13;
|
|
XOR_SY yvec12, yvec12, yvec12;
|
|
#ifndef TRMMKERNEL
|
|
MOVQ bk,k;
|
|
#elif (defined(LEFT)&&!defined(TRANSA))||(!defined(LEFT)&&defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kk, %rax;
|
|
MOVQ %rax, kkk;
|
|
#else
|
|
MOVQ kk, %rax;
|
|
#ifdef LEFT
|
|
ADDQ $2, %rax;
|
|
#else
|
|
ADDQ $4, %rax;
|
|
#endif
|
|
MOVQ %rax, kkk;
|
|
#endif
|
|
SARQ $2, k;
|
|
JLE .L11_loopE;
|
|
ALIGN_5
|
|
.L11_bodyB:
|
|
LD_SX 0*SIZE(ptrba), xvec0; # ar1, ai1, ar2, ai2
|
|
EDUP_SX 0*SIZE(ptrbb), xvec2; # br1, br1, br2, br2
|
|
SHUF_SX $0x4e, xvec2, xvec3; # br3, br3, br4, br4
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec0, xvec3, xvec3;
|
|
ADD1_SX xvec3, xvec14, xvec14;
|
|
|
|
EDUP_SX 4*SIZE(ptrbb), xvec4;
|
|
SHUF_SX $0x4e, xvec4, xvec5;
|
|
MUL_SX xvec0, xvec4, xvec4;
|
|
ADD1_SX xvec4, xvec13, xvec13;
|
|
MUL_SX xvec0, xvec5, xvec5;
|
|
ADD1_SX xvec5, xvec12, xvec12;
|
|
|
|
SHUF_SX $0xb1, xvec0, xvec1;
|
|
ODUP_SX 0*SIZE(ptrbb), xvec2;
|
|
SHUF_SX $0x4e, xvec2, xvec3;
|
|
MUL_SX xvec1, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec14, xvec14;
|
|
|
|
ODUP_SX 4*SIZE(ptrbb), xvec4;
|
|
SHUF_SX $0x4e, xvec4, xvec5;
|
|
MUL_SX xvec1, xvec4, xvec4;
|
|
ADD2_SX xvec4, xvec13, xvec13;
|
|
MUL_SX xvec1, xvec5, xvec5;
|
|
ADD2_SX xvec5, xvec12, xvec12;
|
|
|
|
LD_SX 4*SIZE(ptrba), xvec0; # ar1, ai1, ar2, ai2
|
|
EDUP_SX 8*SIZE(ptrbb), xvec2; # br1, br1, br2, br2
|
|
SHUF_SX $0x4e, xvec2, xvec3; # br3, br3, br4, br4
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec0, xvec3, xvec3;
|
|
ADD1_SX xvec3, xvec14, xvec14;
|
|
|
|
EDUP_SX 12*SIZE(ptrbb), xvec4;
|
|
SHUF_SX $0x4e, xvec4, xvec5;
|
|
MUL_SX xvec0, xvec4, xvec4;
|
|
ADD1_SX xvec4, xvec13, xvec13;
|
|
MUL_SX xvec0, xvec5, xvec5;
|
|
ADD1_SX xvec5, xvec12, xvec12;
|
|
|
|
SHUF_SX $0xb1, xvec0, xvec1;
|
|
ODUP_SX 8*SIZE(ptrbb), xvec2;
|
|
SHUF_SX $0x4e, xvec2, xvec3;
|
|
MUL_SX xvec1, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec14, xvec14;
|
|
|
|
ODUP_SX 12*SIZE(ptrbb), xvec4;
|
|
SHUF_SX $0x4e, xvec4, xvec5;
|
|
MUL_SX xvec1, xvec4, xvec4;
|
|
ADD2_SX xvec4, xvec13, xvec13;
|
|
MUL_SX xvec1, xvec5, xvec5;
|
|
ADD2_SX xvec5, xvec12, xvec12;
|
|
|
|
LD_SX 8*SIZE(ptrba), xvec0; # ar1, ai1, ar2, ai2
|
|
EDUP_SX 16*SIZE(ptrbb), xvec2; # br1, br1, br2, br2
|
|
SHUF_SX $0x4e, xvec2, xvec3; # br3, br3, br4, br4
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec0, xvec3, xvec3;
|
|
ADD1_SX xvec3, xvec14, xvec14;
|
|
|
|
EDUP_SX 20*SIZE(ptrbb), xvec4;
|
|
SHUF_SX $0x4e, xvec4, xvec5;
|
|
MUL_SX xvec0, xvec4, xvec4;
|
|
ADD1_SX xvec4, xvec13, xvec13;
|
|
MUL_SX xvec0, xvec5, xvec5;
|
|
ADD1_SX xvec5, xvec12, xvec12;
|
|
|
|
SHUF_SX $0xb1, xvec0, xvec1;
|
|
ODUP_SX 16*SIZE(ptrbb), xvec2;
|
|
SHUF_SX $0x4e, xvec2, xvec3;
|
|
MUL_SX xvec1, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec14, xvec14;
|
|
|
|
ODUP_SX 20*SIZE(ptrbb), xvec4;
|
|
SHUF_SX $0x4e, xvec4, xvec5;
|
|
MUL_SX xvec1, xvec4, xvec4;
|
|
ADD2_SX xvec4, xvec13, xvec13;
|
|
MUL_SX xvec1, xvec5, xvec5;
|
|
ADD2_SX xvec5, xvec12, xvec12;
|
|
|
|
LD_SX 12*SIZE(ptrba), xvec0; # ar1, ai1, ar2, ai2
|
|
EDUP_SX 24*SIZE(ptrbb), xvec2; # br1, br1, br2, br2
|
|
SHUF_SX $0x4e, xvec2, xvec3; # br3, br3, br4, br4
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec0, xvec3, xvec3;
|
|
ADD1_SX xvec3, xvec14, xvec14;
|
|
|
|
EDUP_SX 28*SIZE(ptrbb), xvec4;
|
|
SHUF_SX $0x4e, xvec4, xvec5;
|
|
MUL_SX xvec0, xvec4, xvec4;
|
|
ADD1_SX xvec4, xvec13, xvec13;
|
|
MUL_SX xvec0, xvec5, xvec5;
|
|
ADD1_SX xvec5, xvec12, xvec12;
|
|
|
|
SHUF_SX $0xb1, xvec0, xvec1;
|
|
ODUP_SX 24*SIZE(ptrbb), xvec2;
|
|
SHUF_SX $0x4e, xvec2, xvec3;
|
|
MUL_SX xvec1, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec14, xvec14;
|
|
|
|
ODUP_SX 28*SIZE(ptrbb), xvec4;
|
|
SHUF_SX $0x4e, xvec4, xvec5;
|
|
MUL_SX xvec1, xvec4, xvec4;
|
|
ADD2_SX xvec4, xvec13, xvec13;
|
|
MUL_SX xvec1, xvec5, xvec5;
|
|
ADD2_SX xvec5, xvec12, xvec12;
|
|
ADDQ $16*SIZE, ptrba;
|
|
ADDQ $32*SIZE, ptrbb;
|
|
DECQ k;
|
|
JG .L11_bodyB;
|
|
ALIGN_5
|
|
.L11_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $2, bk;
|
|
#else
|
|
TEST $2, kkk;
|
|
#endif
|
|
JLE .L12_loopE;
|
|
ALIGN_5
|
|
.L12_bodyB:
|
|
LD_SX 0*SIZE(ptrba), xvec0; # ar1, ai1, ar2, ai2
|
|
EDUP_SX 0*SIZE(ptrbb), xvec2; # br1, br1, br2, br2
|
|
SHUF_SX $0x4e, xvec2, xvec3; # br3, br3, br4, br4
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec0, xvec3, xvec3;
|
|
ADD1_SX xvec3, xvec14, xvec14;
|
|
|
|
EDUP_SX 4*SIZE(ptrbb), xvec4;
|
|
SHUF_SX $0x4e, xvec4, xvec5;
|
|
MUL_SX xvec0, xvec4, xvec4;
|
|
ADD1_SX xvec4, xvec13, xvec13;
|
|
MUL_SX xvec0, xvec5, xvec5;
|
|
ADD1_SX xvec5, xvec12, xvec12;
|
|
|
|
SHUF_SX $0xb1, xvec0, xvec1;
|
|
ODUP_SX 0*SIZE(ptrbb), xvec2;
|
|
SHUF_SX $0x4e, xvec2, xvec3;
|
|
MUL_SX xvec1, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec14, xvec14;
|
|
|
|
ODUP_SX 4*SIZE(ptrbb), xvec4;
|
|
SHUF_SX $0x4e, xvec4, xvec5;
|
|
MUL_SX xvec1, xvec4, xvec4;
|
|
ADD2_SX xvec4, xvec13, xvec13;
|
|
MUL_SX xvec1, xvec5, xvec5;
|
|
ADD2_SX xvec5, xvec12, xvec12;
|
|
|
|
LD_SX 4*SIZE(ptrba), xvec0; # ar1, ai1, ar2, ai2
|
|
EDUP_SX 8*SIZE(ptrbb), xvec2; # br1, br1, br2, br2
|
|
SHUF_SX $0x4e, xvec2, xvec3; # br3, br3, br4, br4
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec0, xvec3, xvec3;
|
|
ADD1_SX xvec3, xvec14, xvec14;
|
|
|
|
EDUP_SX 12*SIZE(ptrbb), xvec4;
|
|
SHUF_SX $0x4e, xvec4, xvec5;
|
|
MUL_SX xvec0, xvec4, xvec4;
|
|
ADD1_SX xvec4, xvec13, xvec13;
|
|
MUL_SX xvec0, xvec5, xvec5;
|
|
ADD1_SX xvec5, xvec12, xvec12;
|
|
|
|
SHUF_SX $0xb1, xvec0, xvec1;
|
|
ODUP_SX 8*SIZE(ptrbb), xvec2;
|
|
SHUF_SX $0x4e, xvec2, xvec3;
|
|
MUL_SX xvec1, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec14, xvec14;
|
|
|
|
ODUP_SX 12*SIZE(ptrbb), xvec4;
|
|
SHUF_SX $0x4e, xvec4, xvec5;
|
|
MUL_SX xvec1, xvec4, xvec4;
|
|
ADD2_SX xvec4, xvec13, xvec13;
|
|
MUL_SX xvec1, xvec5, xvec5;
|
|
ADD2_SX xvec5, xvec12, xvec12;
|
|
ADDQ $8*SIZE, ptrba;
|
|
ADDQ $16*SIZE, ptrbb;
|
|
|
|
.L12_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $1, bk;
|
|
#else
|
|
TEST $1, kkk;
|
|
#endif
|
|
JLE .L13_loopE;
|
|
ALIGN_5
|
|
.L13_bodyB:
|
|
LD_SX 0*SIZE(ptrba), xvec0; # ar1, ai1, ar2, ai2
|
|
EDUP_SX 0*SIZE(ptrbb), xvec2; # br1, br1, br2, br2
|
|
SHUF_SX $0x4e, xvec2, xvec3; # br3, br3, br4, br4
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec0, xvec3, xvec3;
|
|
ADD1_SX xvec3, xvec14, xvec14;
|
|
|
|
EDUP_SX 4*SIZE(ptrbb), xvec4;
|
|
SHUF_SX $0x4e, xvec4, xvec5;
|
|
MUL_SX xvec0, xvec4, xvec4;
|
|
ADD1_SX xvec4, xvec13, xvec13;
|
|
MUL_SX xvec0, xvec5, xvec5;
|
|
ADD1_SX xvec5, xvec12, xvec12;
|
|
|
|
SHUF_SX $0xb1, xvec0, xvec1;
|
|
ODUP_SX 0*SIZE(ptrbb), xvec2;
|
|
SHUF_SX $0x4e, xvec2, xvec3;
|
|
MUL_SX xvec1, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec14, xvec14;
|
|
|
|
ODUP_SX 4*SIZE(ptrbb), xvec4;
|
|
SHUF_SX $0x4e, xvec4, xvec5;
|
|
MUL_SX xvec1, xvec4, xvec4;
|
|
ADD2_SX xvec4, xvec13, xvec13;
|
|
MUL_SX xvec1, xvec5, xvec5;
|
|
ADD2_SX xvec5, xvec12, xvec12;
|
|
ADDQ $4*SIZE, ptrba;
|
|
ADDQ $8*SIZE, ptrbb;
|
|
|
|
.L13_loopE:
|
|
#### Handle ####
|
|
#if defined(RN) || defined(RT) || defined(CN) || defined(CT)
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec15, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec15;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec14, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec14;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec13, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec13;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec12, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec12;
|
|
#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
SUB_SX xvec15, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec15;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
SUB_SX xvec14, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec14;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
SUB_SX xvec13, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec13;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
SUB_SX xvec12, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec12;
|
|
#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
|
|
SHUF_SX $0xb1, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec14, xvec14;
|
|
SHUF_SX $0xb1, xvec13, xvec13;
|
|
SHUF_SX $0xb1, xvec12, xvec12;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec15, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec15;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec14, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec14;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec13, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec13;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec12, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec12;
|
|
SHUF_SX $0xb1, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec14, xvec14;
|
|
SHUF_SX $0xb1, xvec13, xvec13;
|
|
SHUF_SX $0xb1, xvec12, xvec12;
|
|
#endif
|
|
##### Load Alpha ####
|
|
BROAD_SX MEMALPHA_R,xvec7;
|
|
BROAD_SX MEMALPHA_I,xvec6;
|
|
##### Multiply Alpha ####
|
|
VPERMILP_SX $0xb1,xvec15, xvec5;
|
|
MUL_SX xvec7, xvec15, xvec15;
|
|
MUL_SX xvec6, xvec5, xvec5;
|
|
ADDSUB_SX xvec5, xvec15, xvec15;
|
|
VPERMILP_SX $0xb1,xvec14, xvec4;
|
|
MUL_SX xvec7, xvec14, xvec14;
|
|
MUL_SX xvec6, xvec4, xvec4;
|
|
ADDSUB_SX xvec4, xvec14, xvec14;
|
|
VPERMILP_SX $0xb1,xvec13, xvec3;
|
|
MUL_SX xvec7, xvec13, xvec13;
|
|
MUL_SX xvec6, xvec3, xvec3;
|
|
ADDSUB_SX xvec3, xvec13, xvec13;
|
|
VPERMILP_SX $0xb1,xvec12, xvec2;
|
|
MUL_SX xvec7, xvec12, xvec12;
|
|
MUL_SX xvec6, xvec2, xvec2;
|
|
ADDSUB_SX xvec2, xvec12, xvec12;
|
|
#### Writing back ####
|
|
#ifndef TRMMKERNEL
|
|
LDL_SX 0*SIZE(C0), xvec0, xvec0;
|
|
LDH_SX 2*SIZE(C0, ldc,1), xvec0, xvec0;
|
|
LDL_SX 0*SIZE(C0, ldc,1), xvec1, xvec1;
|
|
LDH_SX 2*SIZE(C0), xvec1, xvec1;
|
|
LDL_SX 0*SIZE(C1), xvec2, xvec2;
|
|
LDH_SX 2*SIZE(C1, ldc, 1), xvec2, xvec2;
|
|
LDL_SX 0*SIZE(C1, ldc, 1), xvec3, xvec3;
|
|
LDH_SX 2*SIZE(C1), xvec3, xvec3;
|
|
ADD_SX xvec0, xvec15, xvec15;
|
|
ADD_SX xvec1, xvec14, xvec14;
|
|
ADD_SX xvec2, xvec13, xvec13;
|
|
ADD_SX xvec3, xvec12, xvec12;
|
|
#endif
|
|
STL_SX xvec15, 0*SIZE(C0);
|
|
STH_SX xvec15, 2*SIZE(C0, ldc, 1);
|
|
STL_SX xvec14, 0*SIZE(C0, ldc, 1);
|
|
STH_SX xvec14, 2*SIZE(C0);
|
|
STL_SX xvec13, 0*SIZE(C1);
|
|
STH_SX xvec13, 2*SIZE(C1, ldc, 1);
|
|
STL_SX xvec12, 0*SIZE(C1, ldc, 1);
|
|
STH_SX xvec12, 2*SIZE(C1);
|
|
#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kkk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 2), ptrba;
|
|
LEAQ (ptrbb, %rax, 4), ptrbb;
|
|
#endif
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
ADDQ $2, kk;
|
|
#endif
|
|
|
|
ADDQ $4*SIZE, C0;
|
|
ADDQ $4*SIZE, C1;
|
|
|
|
.L6_loopE:
|
|
TEST $1, bm;
|
|
JLE .L7_loopE;
|
|
ALIGN_5
|
|
.L7_bodyB:
|
|
#if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA))
|
|
MOVQ bb,ptrbb;
|
|
#else
|
|
MOVQ bb, ptrbb;
|
|
MOVQ kk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
ADDQ %rax, ptrba;
|
|
LEAQ (ptrbb, %rax, 4), ptrbb;
|
|
#endif
|
|
XOR_SY yvec15, yvec15, yvec15;
|
|
XOR_SY yvec14, yvec14, yvec14;
|
|
#ifndef TRMMKERNEL
|
|
MOVQ bk,k;
|
|
#elif (defined(LEFT)&&!defined(TRANSA))||(!defined(LEFT)&&defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kk, %rax;
|
|
MOVQ %rax, kkk;
|
|
#else
|
|
MOVQ kk, %rax;
|
|
#ifdef LEFT
|
|
ADDQ $1, %rax;
|
|
#else
|
|
ADDQ $4, %rax;
|
|
#endif
|
|
MOVQ %rax, kkk;
|
|
#endif
|
|
SARQ $2, k;
|
|
JLE .L14_loopE;
|
|
ALIGN_5
|
|
.L14_bodyB:
|
|
BROAD_SX 0*SIZE(ptrba), xvec0;
|
|
LD_SX 0*SIZE(ptrbb), xvec2;
|
|
SHUF_SX $0xb1, xvec2, xvec3;
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
|
|
LD_SX 4*SIZE(ptrbb), xvec4;
|
|
SHUF_SX $0xb1, xvec4, xvec5;
|
|
MUL_SX xvec0, xvec4, xvec4;
|
|
ADD1_SX xvec4, xvec14, xvec14;
|
|
|
|
BROAD_SX 1*SIZE(ptrba), xvec1;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec15, xvec15;
|
|
MUL_SX xvec1, xvec5, xvec5;
|
|
ADD2_SX xvec5, xvec14, xvec14;
|
|
|
|
BROAD_SX 2*SIZE(ptrba), xvec0;
|
|
LD_SX 8*SIZE(ptrbb), xvec2;
|
|
SHUF_SX $0xb1, xvec2, xvec3;
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
|
|
LD_SX 12*SIZE(ptrbb), xvec4;
|
|
SHUF_SX $0xb1, xvec4, xvec5;
|
|
MUL_SX xvec0, xvec4, xvec4;
|
|
ADD1_SX xvec4, xvec14, xvec14;
|
|
|
|
BROAD_SX 3*SIZE(ptrba), xvec1;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec15, xvec15;
|
|
MUL_SX xvec1, xvec5, xvec5;
|
|
ADD2_SX xvec5, xvec14, xvec14;
|
|
|
|
BROAD_SX 4*SIZE(ptrba), xvec0;
|
|
LD_SX 16*SIZE(ptrbb), xvec2;
|
|
SHUF_SX $0xb1, xvec2, xvec3;
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
|
|
LD_SX 20*SIZE(ptrbb), xvec4;
|
|
SHUF_SX $0xb1, xvec4, xvec5;
|
|
MUL_SX xvec0, xvec4, xvec4;
|
|
ADD1_SX xvec4, xvec14, xvec14;
|
|
|
|
BROAD_SX 5*SIZE(ptrba), xvec1;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec15, xvec15;
|
|
MUL_SX xvec1, xvec5, xvec5;
|
|
ADD2_SX xvec5, xvec14, xvec14;
|
|
|
|
BROAD_SX 6*SIZE(ptrba), xvec0;
|
|
LD_SX 24*SIZE(ptrbb), xvec2;
|
|
SHUF_SX $0xb1, xvec2, xvec3;
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
|
|
LD_SX 28*SIZE(ptrbb), xvec4;
|
|
SHUF_SX $0xb1, xvec4, xvec5;
|
|
MUL_SX xvec0, xvec4, xvec4;
|
|
ADD1_SX xvec4, xvec14, xvec14;
|
|
|
|
BROAD_SX 7*SIZE(ptrba), xvec1;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec15, xvec15;
|
|
MUL_SX xvec1, xvec5, xvec5;
|
|
ADD2_SX xvec5, xvec14, xvec14;
|
|
ADDQ $8*SIZE, ptrba;
|
|
ADDQ $32*SIZE, ptrbb;
|
|
DECQ k;
|
|
JG .L14_bodyB;
|
|
ALIGN_5
|
|
.L14_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $2, bk;
|
|
#else
|
|
TEST $2, kkk;
|
|
#endif
|
|
JLE .L15_loopE;
|
|
ALIGN_5
|
|
.L15_bodyB:
|
|
BROAD_SX 0*SIZE(ptrba), xvec0;
|
|
LD_SX 0*SIZE(ptrbb), xvec2;
|
|
SHUF_SX $0xb1, xvec2, xvec3;
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
|
|
LD_SX 4*SIZE(ptrbb), xvec4;
|
|
SHUF_SX $0xb1, xvec4, xvec5;
|
|
MUL_SX xvec0, xvec4, xvec4;
|
|
ADD1_SX xvec4, xvec14, xvec14;
|
|
|
|
BROAD_SX 1*SIZE(ptrba), xvec1;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec15, xvec15;
|
|
MUL_SX xvec1, xvec5, xvec5;
|
|
ADD2_SX xvec5, xvec14, xvec14;
|
|
|
|
BROAD_SX 2*SIZE(ptrba), xvec0;
|
|
LD_SX 8*SIZE(ptrbb), xvec2;
|
|
SHUF_SX $0xb1, xvec2, xvec3;
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
|
|
LD_SX 12*SIZE(ptrbb), xvec4;
|
|
SHUF_SX $0xb1, xvec4, xvec5;
|
|
MUL_SX xvec0, xvec4, xvec4;
|
|
ADD1_SX xvec4, xvec14, xvec14;
|
|
|
|
BROAD_SX 3*SIZE(ptrba), xvec1;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec15, xvec15;
|
|
MUL_SX xvec1, xvec5, xvec5;
|
|
ADD2_SX xvec5, xvec14, xvec14;
|
|
ADDQ $4*SIZE, ptrba;
|
|
ADDQ $16*SIZE, ptrbb;
|
|
|
|
.L15_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $1, bk;
|
|
#else
|
|
TEST $1, kkk;
|
|
#endif
|
|
JLE .L16_loopE;
|
|
ALIGN_5
|
|
.L16_bodyB:
|
|
BROAD_SX 0*SIZE(ptrba), xvec0;
|
|
LD_SX 0*SIZE(ptrbb), xvec2;
|
|
SHUF_SX $0xb1, xvec2, xvec3;
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
|
|
LD_SX 4*SIZE(ptrbb), xvec4;
|
|
SHUF_SX $0xb1, xvec4, xvec5;
|
|
MUL_SX xvec0, xvec4, xvec4;
|
|
ADD1_SX xvec4, xvec14, xvec14;
|
|
|
|
BROAD_SX 1*SIZE(ptrba), xvec1;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec15, xvec15;
|
|
MUL_SX xvec1, xvec5, xvec5;
|
|
ADD2_SX xvec5, xvec14, xvec14;
|
|
ADDQ $2*SIZE, ptrba;
|
|
ADDQ $8*SIZE, ptrbb;
|
|
|
|
.L16_loopE:
|
|
#### Handle ####
|
|
#if defined(NR) || defined(NC) || defined(TR) || defined(TC)
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec15, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec15;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec14, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec14;
|
|
#elif defined(RN) || defined(RT) || defined(CN) || defined(CT)
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
SUB_SX xvec15, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec15;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
SUB_SX xvec14, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec14;
|
|
#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
|
|
SHUF_SX $0xb1, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec14, xvec14;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec15, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec15;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec14, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec14;
|
|
SHUF_SX $0xb1, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec14, xvec14;
|
|
#endif
|
|
##### Load Alpha ####
|
|
BROAD_SX MEMALPHA_R,xvec7;
|
|
BROAD_SX MEMALPHA_I,xvec6;
|
|
##### Multiply Alpha ####
|
|
VPERMILP_SX $0xb1,xvec15, xvec5;
|
|
MUL_SX xvec7, xvec15, xvec15;
|
|
MUL_SX xvec6, xvec5, xvec5;
|
|
ADDSUB_SX xvec5, xvec15, xvec15;
|
|
VPERMILP_SX $0xb1,xvec14, xvec4;
|
|
MUL_SX xvec7, xvec14, xvec14;
|
|
MUL_SX xvec6, xvec4, xvec4;
|
|
ADDSUB_SX xvec4, xvec14, xvec14;
|
|
#### Writing back ####
|
|
#ifndef TRMMKERNEL
|
|
LDL_SX 0*SIZE(C0), xvec0, xvec0;
|
|
LDH_SX 0*SIZE(C0, ldc, 1), xvec0, xvec0;
|
|
LDL_SX 0*SIZE(C1), xvec1, xvec1;
|
|
LDH_SX 0*SIZE(C1, ldc, 1), xvec1, xvec1;
|
|
ADD_SX xvec0, xvec15, xvec15;
|
|
ADD_SX xvec1, xvec14, xvec14;
|
|
#endif
|
|
STL_SX xvec15, 0*SIZE(C0);
|
|
STH_SX xvec15, 0*SIZE(C0, ldc, 1);
|
|
STL_SX xvec14, 0*SIZE(C1);
|
|
STH_SX xvec14, 0*SIZE(C1, ldc, 1);
|
|
#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kkk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
ADDQ %rax, ptrba;
|
|
LEAQ (ptrbb, %rax, 4), ptrbb;
|
|
#endif
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
ADDQ $1, kk;
|
|
#endif
|
|
|
|
ADDQ $2*SIZE, C0;
|
|
ADDQ $2*SIZE, C1;
|
|
.L7_loopE:
|
|
#if defined(TRMMKERNEL) && !defined(LEFT)
|
|
ADDQ $4, kk;
|
|
#endif
|
|
MOVQ bk,k;
|
|
SALQ $5,k;
|
|
ADDQ k,bb;
|
|
LEAQ (C,ldc,4),C;
|
|
.L0_bodyE:;
|
|
DECQ j;
|
|
JG .L0_bodyB;
|
|
ALIGN_5;
|
|
.L0_loopE:;
|
|
TEST $2, bn;
|
|
JLE .L20_loopE;
|
|
ALIGN_5
|
|
.L20_bodyB:
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
MOVQ OFFSET, %rax;
|
|
MOVQ %rax, kk;
|
|
#endif
|
|
MOVQ C, C0;
|
|
LEAQ (C, ldc, 1), C1;
|
|
MOVQ ba, ptrba;
|
|
MOVQ bm, i;
|
|
SARQ $3, i;
|
|
JLE .L21_loopE;
|
|
ALIGN_5
|
|
.L21_bodyB:
|
|
#if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA))
|
|
MOVQ bb,ptrbb;
|
|
#else
|
|
MOVQ bb, ptrbb;
|
|
MOVQ kk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 8), ptrba;
|
|
LEAQ (ptrbb, %rax, 2), ptrbb;
|
|
#endif
|
|
XOR_SY yvec15, yvec15, yvec15;
|
|
XOR_SY yvec14, yvec14, yvec14;
|
|
XOR_SY yvec13, yvec13, yvec13;
|
|
XOR_SY yvec12, yvec12, yvec12;
|
|
XOR_SY yvec11, yvec11, yvec11;
|
|
XOR_SY yvec10, yvec10, yvec10;
|
|
XOR_SY yvec9, yvec9, yvec9;
|
|
XOR_SY yvec8, yvec8, yvec8;
|
|
#ifndef TRMMKERNEL
|
|
MOVQ bk,k;
|
|
#elif (defined(LEFT)&&!defined(TRANSA))||(!defined(LEFT)&&defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kk, %rax;
|
|
MOVQ %rax, kkk;
|
|
#else
|
|
MOVQ kk, %rax;
|
|
#ifdef LEFT
|
|
ADDQ $8, %rax;
|
|
#else
|
|
ADDQ $2, %rax;
|
|
#endif
|
|
MOVQ %rax, kkk;
|
|
#endif
|
|
SARQ $2, k;
|
|
JLE .L211_loopE;
|
|
ALIGN_5
|
|
.L211_bodyB:
|
|
EDUP_SX 0*SIZE(ptrbb), xvec4;
|
|
ODUP_SX 0*SIZE(ptrbb), xvec5;
|
|
SHUF_SX $0x4e, xvec4, xvec6;
|
|
SHUF_SX $0x4e, xvec5, xvec7;
|
|
|
|
LD_SX 0*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec11, xvec11;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec11, xvec11;
|
|
|
|
LD_SX 4*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec14, xvec14;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec10, xvec10;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec14, xvec14;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec10, xvec10;
|
|
|
|
LD_SX 8*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec13, xvec13;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec9, xvec9;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec13, xvec13;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec9, xvec9;
|
|
|
|
LD_SX 12*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec12, xvec12;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec8, xvec8;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec12, xvec12;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec8, xvec8;
|
|
|
|
EDUP_SX 4*SIZE(ptrbb), xvec4;
|
|
ODUP_SX 4*SIZE(ptrbb), xvec5;
|
|
SHUF_SX $0x4e, xvec4, xvec6;
|
|
SHUF_SX $0x4e, xvec5, xvec7;
|
|
|
|
LD_SX 16*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec11, xvec11;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec11, xvec11;
|
|
|
|
LD_SX 20*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec14, xvec14;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec10, xvec10;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec14, xvec14;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec10, xvec10;
|
|
|
|
LD_SX 24*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec13, xvec13;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec9, xvec9;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec13, xvec13;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec9, xvec9;
|
|
|
|
LD_SX 28*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec12, xvec12;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec8, xvec8;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec12, xvec12;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec8, xvec8;
|
|
|
|
EDUP_SX 8*SIZE(ptrbb), xvec4;
|
|
ODUP_SX 8*SIZE(ptrbb), xvec5;
|
|
SHUF_SX $0x4e, xvec4, xvec6;
|
|
SHUF_SX $0x4e, xvec5, xvec7;
|
|
|
|
LD_SX 32*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec11, xvec11;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec11, xvec11;
|
|
|
|
LD_SX 36*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec14, xvec14;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec10, xvec10;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec14, xvec14;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec10, xvec10;
|
|
|
|
LD_SX 40*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec13, xvec13;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec9, xvec9;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec13, xvec13;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec9, xvec9;
|
|
|
|
LD_SX 44*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec12, xvec12;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec8, xvec8;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec12, xvec12;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec8, xvec8;
|
|
|
|
EDUP_SX 12*SIZE(ptrbb), xvec4;
|
|
ODUP_SX 12*SIZE(ptrbb), xvec5;
|
|
SHUF_SX $0x4e, xvec4, xvec6;
|
|
SHUF_SX $0x4e, xvec5, xvec7;
|
|
|
|
LD_SX 48*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec11, xvec11;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec11, xvec11;
|
|
|
|
LD_SX 52*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec14, xvec14;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec10, xvec10;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec14, xvec14;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec10, xvec10;
|
|
|
|
LD_SX 56*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec13, xvec13;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec9, xvec9;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec13, xvec13;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec9, xvec9;
|
|
|
|
LD_SX 60*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec12, xvec12;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec8, xvec8;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec12, xvec12;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec8, xvec8;
|
|
ADDQ $64*SIZE, ptrba;
|
|
ADDQ $16*SIZE, ptrbb;
|
|
DECQ k;
|
|
JG .L211_bodyB;
|
|
ALIGN_5
|
|
.L211_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $2, bk;
|
|
#else
|
|
TEST $2, kkk;
|
|
#endif
|
|
JLE .L212_loopE;
|
|
ALIGN_5
|
|
.L212_bodyB:
|
|
EDUP_SX 0*SIZE(ptrbb), xvec4;
|
|
ODUP_SX 0*SIZE(ptrbb), xvec5;
|
|
SHUF_SX $0x4e, xvec4, xvec6;
|
|
SHUF_SX $0x4e, xvec5, xvec7;
|
|
|
|
LD_SX 0*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec11, xvec11;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec11, xvec11;
|
|
|
|
LD_SX 4*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec14, xvec14;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec10, xvec10;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec14, xvec14;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec10, xvec10;
|
|
|
|
LD_SX 8*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec13, xvec13;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec9, xvec9;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec13, xvec13;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec9, xvec9;
|
|
|
|
LD_SX 12*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec12, xvec12;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec8, xvec8;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec12, xvec12;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec8, xvec8;
|
|
|
|
EDUP_SX 4*SIZE(ptrbb), xvec4;
|
|
ODUP_SX 4*SIZE(ptrbb), xvec5;
|
|
SHUF_SX $0x4e, xvec4, xvec6;
|
|
SHUF_SX $0x4e, xvec5, xvec7;
|
|
|
|
LD_SX 16*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec11, xvec11;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec11, xvec11;
|
|
|
|
LD_SX 20*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec14, xvec14;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec10, xvec10;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec14, xvec14;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec10, xvec10;
|
|
|
|
LD_SX 24*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec13, xvec13;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec9, xvec9;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec13, xvec13;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec9, xvec9;
|
|
|
|
LD_SX 28*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec12, xvec12;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec8, xvec8;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec12, xvec12;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec8, xvec8;
|
|
ADDQ $32*SIZE, ptrba;
|
|
ADDQ $8*SIZE, ptrbb;
|
|
|
|
.L212_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $1, bk;
|
|
#else
|
|
TEST $1, kkk;
|
|
#endif
|
|
JLE .L213_loopE;
|
|
ALIGN_5
|
|
.L213_bodyB:
|
|
EDUP_SX 0*SIZE(ptrbb), xvec4;
|
|
ODUP_SX 0*SIZE(ptrbb), xvec5;
|
|
SHUF_SX $0x4e, xvec4, xvec6;
|
|
SHUF_SX $0x4e, xvec5, xvec7;
|
|
|
|
LD_SX 0*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec11, xvec11;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec11, xvec11;
|
|
|
|
LD_SX 4*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec14, xvec14;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec10, xvec10;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec14, xvec14;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec10, xvec10;
|
|
|
|
LD_SX 8*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec13, xvec13;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec9, xvec9;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec13, xvec13;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec9, xvec9;
|
|
|
|
LD_SX 12*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec12, xvec12;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec8, xvec8;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec12, xvec12;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec8, xvec8;
|
|
ADDQ $16*SIZE, ptrba;
|
|
ADDQ $4*SIZE, ptrbb
|
|
|
|
.L213_loopE:
|
|
#### Handle ####
|
|
#if defined(RN) || defined(RT) || defined(CN) || defined(CT)
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec15, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec15;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec14, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec14;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec13, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec13;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec12, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec12;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec11, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec11;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec10, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec10;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec9, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec9;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec8, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec8;
|
|
#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
SUB_SX xvec15, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec15;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
SUB_SX xvec14, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec14;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
SUB_SX xvec13, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec13;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
SUB_SX xvec12, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec12;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
SUB_SX xvec11, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec11;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
SUB_SX xvec10, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec10;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
SUB_SX xvec9, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec9;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
SUB_SX xvec8, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec8;
|
|
#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
|
|
SHUF_SX $0xb1, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec14, xvec14;
|
|
SHUF_SX $0xb1, xvec13, xvec13;
|
|
SHUF_SX $0xb1, xvec12, xvec12;
|
|
SHUF_SX $0xb1, xvec11, xvec11;
|
|
SHUF_SX $0xb1, xvec10, xvec10;
|
|
SHUF_SX $0xb1, xvec9, xvec9;
|
|
SHUF_SX $0xb1, xvec8, xvec8;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec15, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec15;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec14, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec14;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec13, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec13;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec12, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec12;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec11, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec11;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec10, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec10;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec9, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec9;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec8, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec8;
|
|
SHUF_SX $0xb1, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec14, xvec14;
|
|
SHUF_SX $0xb1, xvec13, xvec13;
|
|
SHUF_SX $0xb1, xvec12, xvec12;
|
|
SHUF_SX $0xb1, xvec11, xvec11;
|
|
SHUF_SX $0xb1, xvec10, xvec10;
|
|
SHUF_SX $0xb1, xvec9, xvec9;
|
|
SHUF_SX $0xb1, xvec8, xvec8;
|
|
#endif
|
|
#### Mulitply Alpha ####
|
|
BROAD_SX MEMALPHA_R, xvec7;
|
|
BROAD_SX MEMALPHA_I, xvec6;
|
|
#### Writng back ####
|
|
VPERMILP_SX $0xb1,xvec15, xvec5;
|
|
MUL_SX xvec7, xvec15, xvec15;
|
|
MUL_SX xvec6, xvec5, xvec5;
|
|
ADDSUB_SX xvec5, xvec15, xvec15;
|
|
VPERMILP_SX $0xb1,xvec14, xvec4;
|
|
MUL_SX xvec7, xvec14, xvec14;
|
|
MUL_SX xvec6, xvec4, xvec4;
|
|
ADDSUB_SX xvec4, xvec14, xvec14;
|
|
VPERMILP_SX $0xb1,xvec13, xvec3;
|
|
MUL_SX xvec7, xvec13, xvec13;
|
|
MUL_SX xvec6, xvec3, xvec3;
|
|
ADDSUB_SX xvec3, xvec13, xvec13;
|
|
VPERMILP_SX $0xb1,xvec12, xvec2;
|
|
MUL_SX xvec7, xvec12, xvec12;
|
|
MUL_SX xvec6, xvec2, xvec2;
|
|
ADDSUB_SX xvec2, xvec12, xvec12;
|
|
VPERMILP_SX $0xb1,xvec11, xvec1;
|
|
MUL_SX xvec7, xvec11, xvec11;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADDSUB_SX xvec1, xvec11, xvec11;
|
|
VPERMILP_SX $0xb1,xvec10, xvec0;
|
|
MUL_SX xvec7, xvec10, xvec10;
|
|
MUL_SX xvec6, xvec0, xvec0;
|
|
ADDSUB_SX xvec0, xvec10, xvec10;
|
|
VPERMILP_SX $0xb1,xvec9, xvec5;
|
|
MUL_SX xvec7, xvec9, xvec9;
|
|
MUL_SX xvec6, xvec5, xvec5;
|
|
ADDSUB_SX xvec5, xvec9, xvec9;
|
|
VPERMILP_SX $0xb1,xvec8, xvec4;
|
|
MUL_SX xvec7, xvec8, xvec8;
|
|
MUL_SX xvec6, xvec4, xvec4;
|
|
ADDSUB_SX xvec4, xvec8, xvec8;
|
|
#ifndef TRMMKERNEL
|
|
LDL_SX 0*SIZE(C0), xvec0, xvec0;
|
|
LDH_SX 2*SIZE(C1), xvec0, xvec0;
|
|
LDL_SX 4*SIZE(C0), xvec1, xvec1;
|
|
LDH_SX 6*SIZE(C1), xvec1, xvec1;
|
|
LDL_SX 8*SIZE(C0), xvec2, xvec2;
|
|
LDH_SX 10*SIZE(C1), xvec2, xvec2;
|
|
LDL_SX 12*SIZE(C0), xvec3, xvec3;
|
|
LDH_SX 14*SIZE(C1), xvec3, xvec3;
|
|
ADD_SX xvec0, xvec15, xvec15;
|
|
ADD_SX xvec1, xvec14, xvec14;
|
|
ADD_SX xvec2, xvec13, xvec13;
|
|
ADD_SX xvec3, xvec12, xvec12;
|
|
#endif
|
|
STL_SX xvec15, 0*SIZE(C0);
|
|
STH_SX xvec15, 2*SIZE(C1);
|
|
STL_SX xvec14, 4*SIZE(C0);
|
|
STH_SX xvec14, 6*SIZE(C1);
|
|
STL_SX xvec13, 8*SIZE(C0);
|
|
STH_SX xvec13, 10*SIZE(C1);
|
|
STL_SX xvec12, 12*SIZE(C0);
|
|
STH_SX xvec12, 14*SIZE(C1);
|
|
#ifndef TRMMKERNEL
|
|
LDL_SX 0*SIZE(C1), xvec4, xvec4;
|
|
LDH_SX 2*SIZE(C0), xvec4, xvec4;
|
|
LDL_SX 4*SIZE(C1), xvec5, xvec5;
|
|
LDH_SX 6*SIZE(C0), xvec5, xvec5;
|
|
LDL_SX 8*SIZE(C1), xvec6, xvec6;
|
|
LDH_SX 10*SIZE(C0), xvec6, xvec6;
|
|
LDL_SX 12*SIZE(C1), xvec7, xvec7;
|
|
LDH_SX 14*SIZE(C0), xvec7, xvec7;
|
|
ADD_SX xvec4, xvec11, xvec11;
|
|
ADD_SX xvec5, xvec10, xvec10;
|
|
ADD_SX xvec6, xvec9, xvec9;
|
|
ADD_SX xvec7, xvec8, xvec8;
|
|
#endif
|
|
STL_SX xvec11, 0*SIZE(C1);
|
|
STH_SX xvec11, 2*SIZE(C0);
|
|
STL_SX xvec10, 4*SIZE(C1);
|
|
STH_SX xvec10, 6*SIZE(C0);
|
|
STL_SX xvec9, 8*SIZE(C1);
|
|
STH_SX xvec9, 10*SIZE(C0);
|
|
STL_SX xvec8, 12*SIZE(C1);
|
|
STH_SX xvec8, 14*SIZE(C0);
|
|
#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kkk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 8), ptrba;
|
|
LEAQ (ptrbb, %rax, 2), ptrbb;
|
|
#endif
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
ADDQ $8, kk;
|
|
#endif
|
|
|
|
ADDQ $16*SIZE, C0;
|
|
ADDQ $16*SIZE, C1;
|
|
DECQ i;
|
|
JG .L21_bodyB;
|
|
ALIGN_5
|
|
.L21_loopE:
|
|
TEST $4, bm;
|
|
JLE .L22_loopE;
|
|
ALIGN_5
|
|
.L22_bodyB:
|
|
#if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA))
|
|
MOVQ bb,ptrbb;
|
|
#else
|
|
MOVQ bb, ptrbb;
|
|
MOVQ kk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 4), ptrba;
|
|
LEAQ (ptrbb, %rax, 2), ptrbb;
|
|
#endif
|
|
XOR_SY yvec15, yvec15, yvec15;
|
|
XOR_SY yvec14, yvec14, yvec14;
|
|
XOR_SY yvec11, yvec11, yvec11;
|
|
XOR_SY yvec10, yvec10, yvec10;
|
|
#ifndef TRMMKERNEL
|
|
MOVQ bk,k;
|
|
#elif (defined(LEFT)&&!defined(TRANSA))||(!defined(LEFT)&&defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kk, %rax;
|
|
MOVQ %rax, kkk;
|
|
#else
|
|
MOVQ kk, %rax;
|
|
#ifdef LEFT
|
|
ADDQ $4, %rax;
|
|
#else
|
|
ADDQ $2, %rax;
|
|
#endif
|
|
MOVQ %rax, kkk;
|
|
#endif
|
|
|
|
SARQ $2, k;
|
|
JLE .L221_loopE;
|
|
ALIGN_5
|
|
.L221_bodyB:
|
|
EDUP_SX 0*SIZE(ptrbb), xvec4;
|
|
ODUP_SX 0*SIZE(ptrbb), xvec5;
|
|
SHUF_SX $0x4e, xvec4, xvec6;
|
|
SHUF_SX $0x4e, xvec5, xvec7;
|
|
|
|
LD_SX 0*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec11, xvec11;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec11, xvec11;
|
|
|
|
LD_SX 4*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec14, xvec14;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec10, xvec10;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec14, xvec14;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec10, xvec10;
|
|
|
|
#### Unroll 2 #####
|
|
EDUP_SX 4*SIZE(ptrbb), xvec4;
|
|
ODUP_SX 4*SIZE(ptrbb), xvec5;
|
|
SHUF_SX $0x4e, xvec4, xvec6;
|
|
SHUF_SX $0x4e, xvec5, xvec7;
|
|
|
|
LD_SX 8*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec11, xvec11;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec11, xvec11;
|
|
|
|
LD_SX 12*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec14, xvec14;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec10, xvec10;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec14, xvec14;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec10, xvec10;
|
|
|
|
#### Unroll 3 ####
|
|
EDUP_SX 8*SIZE(ptrbb), xvec4;
|
|
ODUP_SX 8*SIZE(ptrbb), xvec5;
|
|
SHUF_SX $0x4e, xvec4, xvec6;
|
|
SHUF_SX $0x4e, xvec5, xvec7;
|
|
|
|
LD_SX 16*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec11, xvec11;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec11, xvec11;
|
|
|
|
LD_SX 20*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec14, xvec14;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec10, xvec10;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec14, xvec14;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec10, xvec10;
|
|
|
|
#### Unroll 4 ####
|
|
EDUP_SX 12*SIZE(ptrbb), xvec4;
|
|
ODUP_SX 12*SIZE(ptrbb), xvec5;
|
|
SHUF_SX $0x4e, xvec4, xvec6;
|
|
SHUF_SX $0x4e, xvec5, xvec7;
|
|
|
|
LD_SX 24*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec11, xvec11;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec11, xvec11;
|
|
|
|
LD_SX 28*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec14, xvec14;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec10, xvec10;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec14, xvec14;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec10, xvec10;
|
|
ADDQ $32*SIZE, ptrba;
|
|
ADDQ $16*SIZE, ptrbb;
|
|
DECQ k;
|
|
JG .L221_bodyB;
|
|
ALIGN_5
|
|
.L221_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $2, bk;
|
|
#else
|
|
TEST $2, kkk;
|
|
#endif
|
|
JLE .L222_loopE;
|
|
ALIGN_5
|
|
.L222_bodyB:
|
|
EDUP_SX 0*SIZE(ptrbb), xvec4;
|
|
ODUP_SX 0*SIZE(ptrbb), xvec5;
|
|
SHUF_SX $0x4e, xvec4, xvec6;
|
|
SHUF_SX $0x4e, xvec5, xvec7;
|
|
|
|
LD_SX 0*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec11, xvec11;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec11, xvec11;
|
|
|
|
LD_SX 4*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec14, xvec14;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec10, xvec10;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec14, xvec14;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec10, xvec10;
|
|
|
|
#### Unroll 2 #####
|
|
EDUP_SX 4*SIZE(ptrbb), xvec4;
|
|
ODUP_SX 4*SIZE(ptrbb), xvec5;
|
|
SHUF_SX $0x4e, xvec4, xvec6;
|
|
SHUF_SX $0x4e, xvec5, xvec7;
|
|
|
|
LD_SX 8*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec11, xvec11;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec11, xvec11;
|
|
|
|
LD_SX 12*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec14, xvec14;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec10, xvec10;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec14, xvec14;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec10, xvec10;
|
|
ADDQ $16*SIZE, ptrba;
|
|
ADDQ $8*SIZE, ptrbb;
|
|
|
|
|
|
.L222_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $1, bk;
|
|
#else
|
|
TEST $1, kkk;
|
|
#endif
|
|
JLE .L223_loopE;
|
|
ALIGN_5
|
|
.L223_bodyB:
|
|
EDUP_SX 0*SIZE(ptrbb), xvec4;
|
|
ODUP_SX 0*SIZE(ptrbb), xvec5;
|
|
SHUF_SX $0x4e, xvec4, xvec6;
|
|
SHUF_SX $0x4e, xvec5, xvec7;
|
|
|
|
LD_SX 0*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec11, xvec11;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec11, xvec11;
|
|
|
|
LD_SX 4*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec14, xvec14;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec10, xvec10;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec14, xvec14;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec10, xvec10;
|
|
ADDQ $8*SIZE, ptrba;
|
|
ADDQ $4*SIZE, ptrbb;
|
|
|
|
.L223_loopE:
|
|
#### Handle ####
|
|
#if defined(RN) || defined(RT) || defined(CN) || defined(CT)
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec15, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec15;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec14, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec14;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec11, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec11;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec10, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec10;
|
|
#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
SUB_SX xvec15, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec15;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
SUB_SX xvec14, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec14;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
SUB_SX xvec11, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec11;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
SUB_SX xvec10, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec10;
|
|
#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
|
|
SHUF_SX $0xb1, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec14, xvec14;
|
|
SHUF_SX $0xb1, xvec11, xvec11;
|
|
SHUF_SX $0xb1, xvec10, xvec10;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec15, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec15;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec14, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec14;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec11, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec11;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec10, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec10;
|
|
SHUF_SX $0xb1, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec14, xvec14;
|
|
SHUF_SX $0xb1, xvec11, xvec11;
|
|
SHUF_SX $0xb1, xvec10, xvec10;
|
|
#endif
|
|
#### Mulitply Alpha ####
|
|
BROAD_SX MEMALPHA_R, xvec7;
|
|
BROAD_SX MEMALPHA_I, xvec6;
|
|
#### Writng back ####
|
|
VPERMILP_SX $0xb1,xvec15, xvec5;
|
|
MUL_SX xvec7, xvec15, xvec15;
|
|
MUL_SX xvec6, xvec5, xvec5;
|
|
ADDSUB_SX xvec5, xvec15, xvec15;
|
|
VPERMILP_SX $0xb1,xvec14, xvec4;
|
|
MUL_SX xvec7, xvec14, xvec14;
|
|
MUL_SX xvec6, xvec4, xvec4;
|
|
ADDSUB_SX xvec4, xvec14, xvec14;
|
|
VPERMILP_SX $0xb1,xvec11, xvec1;
|
|
MUL_SX xvec7, xvec11, xvec11;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADDSUB_SX xvec1, xvec11, xvec11;
|
|
VPERMILP_SX $0xb1,xvec10, xvec0;
|
|
MUL_SX xvec7, xvec10, xvec10;
|
|
MUL_SX xvec6, xvec0, xvec0;
|
|
ADDSUB_SX xvec0, xvec10, xvec10;
|
|
#ifndef TRMMKERNEL
|
|
LDL_SX 0*SIZE(C0), xvec0, xvec0;
|
|
LDH_SX 2*SIZE(C1), xvec0, xvec0;
|
|
LDL_SX 4*SIZE(C0), xvec1, xvec1;
|
|
LDH_SX 6*SIZE(C1), xvec1, xvec1;
|
|
ADD_SX xvec0, xvec15, xvec15;
|
|
ADD_SX xvec1, xvec14, xvec14;
|
|
#endif
|
|
STL_SX xvec15, 0*SIZE(C0);
|
|
STH_SX xvec15, 2*SIZE(C1);
|
|
STL_SX xvec14, 4*SIZE(C0);
|
|
STH_SX xvec14, 6*SIZE(C1);
|
|
#ifndef TRMMKERNEL
|
|
LDL_SX 0*SIZE(C1), xvec4, xvec4;
|
|
LDH_SX 2*SIZE(C0), xvec4, xvec4;
|
|
LDL_SX 4*SIZE(C1), xvec5, xvec5;
|
|
LDH_SX 6*SIZE(C0), xvec5, xvec5;
|
|
ADD_SX xvec4, xvec11, xvec11;
|
|
ADD_SX xvec5, xvec10, xvec10;
|
|
#endif
|
|
STL_SX xvec11, 0*SIZE(C1);
|
|
STH_SX xvec11, 2*SIZE(C0);
|
|
STL_SX xvec10, 4*SIZE(C1);
|
|
STH_SX xvec10, 6*SIZE(C0);
|
|
#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kkk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 4), ptrba;
|
|
LEAQ (ptrbb, %rax, 2), ptrbb;
|
|
#endif
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
ADDQ $4, kk;
|
|
#endif
|
|
|
|
ADDQ $8*SIZE, C0;
|
|
ADDQ $8*SIZE, C1;
|
|
|
|
.L22_loopE:
|
|
TEST $2, bm;
|
|
JLE .L23_loopE;
|
|
ALIGN_5
|
|
.L23_bodyB:
|
|
#if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA))
|
|
MOVQ bb,ptrbb;
|
|
#else
|
|
MOVQ bb, ptrbb;
|
|
MOVQ kk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 2), ptrba;
|
|
LEAQ (ptrbb, %rax, 2), ptrbb;
|
|
#endif
|
|
XOR_SY yvec15, yvec15, yvec15;
|
|
XOR_SY yvec11, yvec11, yvec11;
|
|
#ifndef TRMMKERNEL
|
|
MOVQ bk,k;
|
|
#elif (defined(LEFT)&&!defined(TRANSA))||(!defined(LEFT)&&defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kk, %rax;
|
|
MOVQ %rax, kkk;
|
|
#else
|
|
MOVQ kk, %rax;
|
|
#ifdef LEFT
|
|
ADDQ $2, %rax;
|
|
#else
|
|
ADDQ $2, %rax;
|
|
#endif
|
|
MOVQ %rax, kkk;
|
|
#endif
|
|
SARQ $2, k;
|
|
JLE .L231_loopE;
|
|
ALIGN_5
|
|
.L231_bodyB:
|
|
EDUP_SX 0*SIZE(ptrbb), xvec4;
|
|
ODUP_SX 0*SIZE(ptrbb), xvec5;
|
|
SHUF_SX $0x4e, xvec4, xvec6;
|
|
SHUF_SX $0x4e, xvec5, xvec7;
|
|
|
|
LD_SX 0*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec11, xvec11;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec11, xvec11;
|
|
|
|
#### Unroll 2 #####
|
|
EDUP_SX 4*SIZE(ptrbb), xvec4;
|
|
ODUP_SX 4*SIZE(ptrbb), xvec5;
|
|
SHUF_SX $0x4e, xvec4, xvec6;
|
|
SHUF_SX $0x4e, xvec5, xvec7;
|
|
|
|
LD_SX 4*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec11, xvec11;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec11, xvec11;
|
|
|
|
#### Unroll 3 ####
|
|
EDUP_SX 8*SIZE(ptrbb), xvec4;
|
|
ODUP_SX 8*SIZE(ptrbb), xvec5;
|
|
SHUF_SX $0x4e, xvec4, xvec6;
|
|
SHUF_SX $0x4e, xvec5, xvec7;
|
|
|
|
LD_SX 8*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec11, xvec11;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec11, xvec11;
|
|
|
|
#### Unroll 4 ####
|
|
EDUP_SX 12*SIZE(ptrbb), xvec4;
|
|
ODUP_SX 12*SIZE(ptrbb), xvec5;
|
|
SHUF_SX $0x4e, xvec4, xvec6;
|
|
SHUF_SX $0x4e, xvec5, xvec7;
|
|
|
|
LD_SX 12*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec11, xvec11;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec11, xvec11;
|
|
ADDQ $16*SIZE, ptrba;
|
|
ADDQ $16*SIZE, ptrbb;
|
|
DECQ k;
|
|
JG .L231_bodyB;
|
|
ALIGN_5
|
|
.L231_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $2, bk;
|
|
#else
|
|
TEST $2, kkk;
|
|
#endif
|
|
JLE .L232_loopE;
|
|
ALIGN_5
|
|
.L232_bodyB:
|
|
EDUP_SX 0*SIZE(ptrbb), xvec4;
|
|
ODUP_SX 0*SIZE(ptrbb), xvec5;
|
|
SHUF_SX $0x4e, xvec4, xvec6;
|
|
SHUF_SX $0x4e, xvec5, xvec7;
|
|
|
|
LD_SX 0*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec11, xvec11;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec11, xvec11;
|
|
|
|
#### Unroll 2 #####
|
|
EDUP_SX 4*SIZE(ptrbb), xvec4;
|
|
ODUP_SX 4*SIZE(ptrbb), xvec5;
|
|
SHUF_SX $0x4e, xvec4, xvec6;
|
|
SHUF_SX $0x4e, xvec5, xvec7;
|
|
|
|
LD_SX 4*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec11, xvec11;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec11, xvec11;
|
|
ADDQ $8*SIZE, ptrba;
|
|
ADDQ $8*SIZE, ptrbb;
|
|
|
|
.L232_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $1, bk;
|
|
#else
|
|
TEST $1, kkk;
|
|
#endif
|
|
JLE .L233_loopE;
|
|
ALIGN_5
|
|
.L233_bodyB:
|
|
EDUP_SX 0*SIZE(ptrbb), xvec4;
|
|
ODUP_SX 0*SIZE(ptrbb), xvec5;
|
|
SHUF_SX $0x4e, xvec4, xvec6;
|
|
SHUF_SX $0x4e, xvec5, xvec7;
|
|
|
|
LD_SX 0*SIZE(ptrba), xvec0;
|
|
MOV_SX xvec0, xvec1;
|
|
MUL_SX xvec4, xvec0, xvec0;
|
|
ADD1_SX xvec0, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec1, xvec2;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADD1_SX xvec1, xvec11, xvec11;
|
|
|
|
MOV_SX xvec2, xvec3;
|
|
MUL_SX xvec5, xvec2, xvec2;
|
|
ADD2_SX xvec2, xvec15, xvec15;
|
|
MUL_SX xvec7, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec11, xvec11;
|
|
ADDQ $4*SIZE, ptrba;
|
|
ADDQ $4*SIZE, ptrbb;
|
|
|
|
.L233_loopE:
|
|
#### Handle ####
|
|
#if defined(RN) || defined(RT) || defined(CN) || defined(CT)
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec15, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec15;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec11, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec11;
|
|
#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
SUB_SX xvec15, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec15;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
SUB_SX xvec11, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec11;
|
|
#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
|
|
SHUF_SX $0xb1, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec11, xvec11;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec15, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec15;
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
ADDSUB_SX xvec11, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec11;
|
|
SHUF_SX $0xb1, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec11, xvec11;
|
|
#endif
|
|
#### Mulitply Alpha ####
|
|
BROAD_SX MEMALPHA_R, xvec7;
|
|
BROAD_SX MEMALPHA_I, xvec6;
|
|
#### Writng back ####
|
|
VPERMILP_SX $0xb1,xvec15, xvec5;
|
|
MUL_SX xvec7, xvec15, xvec15;
|
|
MUL_SX xvec6, xvec5, xvec5;
|
|
ADDSUB_SX xvec5, xvec15, xvec15;
|
|
VPERMILP_SX $0xb1,xvec11, xvec1;
|
|
MUL_SX xvec7, xvec11, xvec11;
|
|
MUL_SX xvec6, xvec1, xvec1;
|
|
ADDSUB_SX xvec1, xvec11, xvec11;
|
|
#ifndef TRMMKERNEL
|
|
LDL_SX 0*SIZE(C0), xvec0, xvec0;
|
|
LDH_SX 2*SIZE(C1), xvec0, xvec0;
|
|
ADD_SX xvec0, xvec15, xvec15;
|
|
#endif
|
|
STL_SX xvec15, 0*SIZE(C0);
|
|
STH_SX xvec15, 2*SIZE(C1);
|
|
#ifndef TRMMKERNEL
|
|
LDL_SX 0*SIZE(C1), xvec4, xvec4;
|
|
LDH_SX 2*SIZE(C0), xvec4, xvec4;
|
|
ADD_SX xvec4, xvec11, xvec11;
|
|
#endif
|
|
STL_SX xvec11, 0*SIZE(C1);
|
|
STH_SX xvec11, 2*SIZE(C0);
|
|
#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kkk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 2), ptrba;
|
|
LEAQ (ptrbb, %rax, 2), ptrbb;
|
|
#endif
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
ADDQ $2, kk;
|
|
#endif
|
|
|
|
ADDQ $4*SIZE, C0;
|
|
ADDQ $4*SIZE, C1;
|
|
|
|
.L23_loopE:
|
|
TEST $1, bm;
|
|
JLE .L24_loopE;
|
|
ALIGN_5
|
|
.L24_bodyB:
|
|
#if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA))
|
|
MOVQ bb,ptrbb;
|
|
#else
|
|
MOVQ bb, ptrbb;
|
|
MOVQ kk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
ADDQ %rax, ptrba;
|
|
LEAQ (ptrbb, %rax, 2), ptrbb;
|
|
#endif
|
|
XOR_SY yvec15, yvec15, yvec15;
|
|
#ifndef TRMMKERNEL
|
|
MOVQ bk,k;
|
|
#elif (defined(LEFT)&&!defined(TRANSA))||(!defined(LEFT)&&defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kk, %rax;
|
|
MOVQ %rax, kkk;
|
|
#else
|
|
MOVQ kk, %rax;
|
|
#ifdef LEFT
|
|
ADDQ $1, %rax;
|
|
#else
|
|
ADDQ $2, %rax;
|
|
#endif
|
|
MOVQ %rax, kkk;
|
|
#endif
|
|
SARQ $2, k;
|
|
JLE .L241_loopE;
|
|
ALIGN_5
|
|
.L241_bodyB:
|
|
BROAD_SX 0*SIZE(ptrba), xvec0;
|
|
LD_SX 0*SIZE(ptrbb), xvec2;
|
|
SHUF_SX $0xb1, xvec2, xvec3;
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
|
|
BROAD_SX 1*SIZE(ptrba), xvec1;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec15, xvec15;
|
|
|
|
BROAD_SX 2*SIZE(ptrba), xvec0;
|
|
LD_SX 4*SIZE(ptrbb), xvec2;
|
|
SHUF_SX $0xb1, xvec2, xvec3;
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
|
|
BROAD_SX 3*SIZE(ptrba), xvec1;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec15, xvec15;
|
|
|
|
BROAD_SX 4*SIZE(ptrba), xvec0;
|
|
LD_SX 8*SIZE(ptrbb), xvec2;
|
|
SHUF_SX $0xb1, xvec2, xvec3;
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
|
|
BROAD_SX 5*SIZE(ptrba), xvec1;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec15, xvec15;
|
|
|
|
BROAD_SX 6*SIZE(ptrba), xvec0;
|
|
LD_SX 12*SIZE(ptrbb), xvec2;
|
|
SHUF_SX $0xb1, xvec2, xvec3;
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
|
|
BROAD_SX 7*SIZE(ptrba), xvec1;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec15, xvec15;
|
|
ADDQ $8*SIZE, ptrba;
|
|
ADDQ $16*SIZE, ptrbb;
|
|
DECQ k;
|
|
JG .L241_bodyB;
|
|
ALIGN_5
|
|
.L241_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $2, bk;
|
|
#else
|
|
TEST $2, kkk;
|
|
#endif
|
|
JLE .L242_loopE;
|
|
ALIGN_5
|
|
.L242_bodyB:
|
|
BROAD_SX 0*SIZE(ptrba), xvec0;
|
|
LD_SX 0*SIZE(ptrbb), xvec2;
|
|
SHUF_SX $0xb1, xvec2, xvec3;
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
|
|
BROAD_SX 1*SIZE(ptrba), xvec1;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec15, xvec15;
|
|
|
|
BROAD_SX 2*SIZE(ptrba), xvec0;
|
|
LD_SX 4*SIZE(ptrbb), xvec2;
|
|
SHUF_SX $0xb1, xvec2, xvec3;
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
|
|
BROAD_SX 3*SIZE(ptrba), xvec1;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec15, xvec15;
|
|
ADDQ $4*SIZE, ptrba;
|
|
ADDQ $8*SIZE, ptrbb;
|
|
|
|
.L242_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $1, bk;
|
|
#else
|
|
TEST $1, kkk;
|
|
#endif
|
|
JLE .L243_loopE;
|
|
ALIGN_5
|
|
.L243_bodyB:
|
|
BROAD_SX 0*SIZE(ptrba), xvec0;
|
|
LD_SX 0*SIZE(ptrbb), xvec2;
|
|
SHUF_SX $0xb1, xvec2, xvec3;
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
|
|
BROAD_SX 1*SIZE(ptrba), xvec1;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec15, xvec15;
|
|
ADDQ $2*SIZE, ptrba;
|
|
ADDQ $4*SIZE, ptrbb;
|
|
|
|
.L243_loopE:
|
|
#### Handle ####
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
#if defined(NR) || defined(NC) || defined(TR) || defined(TC)
|
|
ADDSUB_SX xvec15, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec15;
|
|
#elif defined(RN) || defined(RT) || defined(CN) || defined(CT)
|
|
SUB_SX xvec15, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec15;
|
|
#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
|
|
SHUF_SX $0xb1, xvec15, xvec15;
|
|
ADDSUB_SX xvec15, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec15;
|
|
SHUF_SX $0xb1, xvec15, xvec15;
|
|
#endif
|
|
##### Load Alpha ####
|
|
BROAD_SX MEMALPHA_R,xvec7;
|
|
BROAD_SX MEMALPHA_I,xvec6;
|
|
##### Multiply Alpha ####
|
|
VPERMILP_SX $0xb1,xvec15, xvec5;
|
|
MUL_SX xvec7, xvec15, xvec15;
|
|
MUL_SX xvec6, xvec5, xvec5;
|
|
ADDSUB_SX xvec5, xvec15, xvec15;
|
|
#### Writing back ####
|
|
#ifndef TRMMKERNEL
|
|
LDL_SX 0*SIZE(C0), xvec0, xvec0;
|
|
LDH_SX 0*SIZE(C1), xvec0, xvec0;
|
|
ADD_SX xvec0, xvec15, xvec15;
|
|
#endif
|
|
STL_SX xvec15, 0*SIZE(C0);
|
|
STH_SX xvec15, 0*SIZE(C1);
|
|
#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kkk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
ADDQ %rax, ptrba;
|
|
LEAQ (ptrbb, %rax, 2), ptrbb;
|
|
#endif
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
ADDQ $1, kk;
|
|
#endif
|
|
ADDQ $2*SIZE, C0;
|
|
ADDQ $2*SIZE, C1;
|
|
.L24_loopE:
|
|
#if defined(TRMMKERNEL) && !defined(LEFT)
|
|
ADDQ $2, kk;
|
|
#endif
|
|
MOVQ bk, k;
|
|
SALQ $4, k;
|
|
ADDQ k, bb;
|
|
LEAQ (C, ldc, 2), C;
|
|
.L20_loopE:
|
|
TEST $1, bn;
|
|
JLE .L30_loopE;
|
|
ALIGN_5
|
|
.L30_bodyB:
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
MOVQ OFFSET, %rax;
|
|
MOVQ %rax, kk;
|
|
#endif
|
|
MOVQ C, C0;
|
|
MOVQ ba, ptrba;
|
|
MOVQ bm, i;
|
|
SARQ $3, i;
|
|
JLE .L31_loopE;
|
|
ALIGN_5
|
|
.L31_bodyB:
|
|
MOVQ bb, ptrbb;
|
|
#if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA))
|
|
MOVQ bb,ptrbb;
|
|
#else
|
|
MOVQ bb, ptrbb;
|
|
MOVQ kk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 8), ptrba;
|
|
ADDQ %rax, ptrbb;
|
|
#endif
|
|
XOR_SY yvec15, yvec15, yvec15;
|
|
XOR_SY yvec14, yvec14, yvec14;
|
|
MOVQ bk, k;
|
|
#ifndef TRMMKERNEL
|
|
MOVQ bk,k;
|
|
#elif (defined(LEFT)&&!defined(TRANSA))||(!defined(LEFT)&&defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kk, %rax;
|
|
MOVQ %rax, kkk;
|
|
#else
|
|
MOVQ kk, %rax;
|
|
#ifdef LEFT
|
|
ADDQ $8, %rax;
|
|
#else
|
|
ADDQ $1, %rax;
|
|
#endif
|
|
MOVQ %rax, kkk;
|
|
#endif
|
|
SARQ $2, k;
|
|
JLE .L311_loopE;
|
|
ALIGN_5
|
|
.L311_bodyB:
|
|
#### Unroll 1 ####
|
|
LD_SY 0*SIZE(ptrba), yvec0;
|
|
LD_SY 8*SIZE(ptrba), yvec1;
|
|
BROAD_SY 0*SIZE(ptrbb), yvec2;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
MUL_SY yvec1, yvec2, yvec7;
|
|
ADD1_SY yvec7, yvec14, yvec14;
|
|
|
|
BROAD_SY 1*SIZE(ptrbb), yvec3;
|
|
VPERMILP_SY $0xb1, yvec0, yvec4;
|
|
VPERMILP_SY $0xb1, yvec1, yvec5;
|
|
MUL_SY yvec4, yvec3, yvec6;
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
MUL_SY yvec5, yvec3, yvec7;
|
|
ADD2_SY yvec7, yvec14, yvec14;
|
|
|
|
#### Unroll 2 ####
|
|
LD_SY 16*SIZE(ptrba), yvec0;
|
|
LD_SY 24*SIZE(ptrba), yvec1;
|
|
BROAD_SY 2*SIZE(ptrbb), yvec2;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
MUL_SY yvec1, yvec2, yvec7;
|
|
ADD1_SY yvec7, yvec14, yvec14;
|
|
|
|
BROAD_SY 3*SIZE(ptrbb), yvec3;
|
|
VPERMILP_SY $0xb1, yvec0, yvec4;
|
|
VPERMILP_SY $0xb1, yvec1, yvec5;
|
|
MUL_SY yvec4, yvec3, yvec6;
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
MUL_SY yvec5, yvec3, yvec7;
|
|
ADD2_SY yvec7, yvec14, yvec14;
|
|
|
|
#### Unroll 3 ####
|
|
LD_SY 32*SIZE(ptrba), yvec0;
|
|
LD_SY 40*SIZE(ptrba), yvec1;
|
|
BROAD_SY 4*SIZE(ptrbb), yvec2;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
MUL_SY yvec1, yvec2, yvec7;
|
|
ADD1_SY yvec7, yvec14, yvec14;
|
|
|
|
BROAD_SY 5*SIZE(ptrbb), yvec3;
|
|
VPERMILP_SY $0xb1, yvec0, yvec4;
|
|
VPERMILP_SY $0xb1, yvec1, yvec5;
|
|
MUL_SY yvec4, yvec3, yvec6;
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
MUL_SY yvec5, yvec3, yvec7;
|
|
ADD2_SY yvec7, yvec14, yvec14;
|
|
|
|
#### Unroll 4 ####
|
|
LD_SY 48*SIZE(ptrba), yvec0;
|
|
LD_SY 56*SIZE(ptrba), yvec1;
|
|
BROAD_SY 6*SIZE(ptrbb), yvec2;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
MUL_SY yvec1, yvec2, yvec7;
|
|
ADD1_SY yvec7, yvec14, yvec14;
|
|
|
|
BROAD_SY 7*SIZE(ptrbb), yvec3;
|
|
VPERMILP_SY $0xb1, yvec0, yvec4;
|
|
VPERMILP_SY $0xb1, yvec1, yvec5;
|
|
MUL_SY yvec4, yvec3, yvec6;
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
MUL_SY yvec5, yvec3, yvec7;
|
|
ADD2_SY yvec7, yvec14, yvec14;
|
|
ADDQ $64*SIZE, ptrba;
|
|
ADDQ $8*SIZE, ptrbb;
|
|
DECQ k;
|
|
JG .L311_bodyB;
|
|
ALIGN_5
|
|
.L311_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $2, bk;
|
|
#else
|
|
TEST $2, kkk;
|
|
#endif
|
|
JLE .L312_loopE;
|
|
ALIGN_5
|
|
.L312_bodyB:
|
|
#### Unroll 1 ####
|
|
LD_SY 0*SIZE(ptrba), yvec0;
|
|
LD_SY 8*SIZE(ptrba), yvec1;
|
|
BROAD_SY 0*SIZE(ptrbb), yvec2;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
MUL_SY yvec1, yvec2, yvec7;
|
|
ADD1_SY yvec7, yvec14, yvec14;
|
|
|
|
BROAD_SY 1*SIZE(ptrbb), yvec3;
|
|
VPERMILP_SY $0xb1, yvec0, yvec4;
|
|
VPERMILP_SY $0xb1, yvec1, yvec5;
|
|
MUL_SY yvec4, yvec3, yvec6;
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
MUL_SY yvec5, yvec3, yvec7;
|
|
ADD2_SY yvec7, yvec14, yvec14;
|
|
|
|
#### Unroll 2 ####
|
|
LD_SY 16*SIZE(ptrba), yvec0;
|
|
LD_SY 24*SIZE(ptrba), yvec1;
|
|
BROAD_SY 2*SIZE(ptrbb), yvec2;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
MUL_SY yvec1, yvec2, yvec7;
|
|
ADD1_SY yvec7, yvec14, yvec14;
|
|
|
|
BROAD_SY 3*SIZE(ptrbb), yvec3;
|
|
VPERMILP_SY $0xb1, yvec0, yvec4;
|
|
VPERMILP_SY $0xb1, yvec1, yvec5;
|
|
MUL_SY yvec4, yvec3, yvec6;
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
MUL_SY yvec5, yvec3, yvec7;
|
|
ADD2_SY yvec7, yvec14, yvec14;
|
|
ADDQ $32*SIZE, ptrba;
|
|
ADDQ $4*SIZE, ptrbb;
|
|
|
|
.L312_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $1, bk;
|
|
#else
|
|
TEST $1, kkk;
|
|
#endif
|
|
JLE .L313_loopE;
|
|
ALIGN_5
|
|
.L313_bodyB:
|
|
#### Unroll 1 ####
|
|
LD_SY 0*SIZE(ptrba), yvec0;
|
|
LD_SY 8*SIZE(ptrba), yvec1;
|
|
BROAD_SY 0*SIZE(ptrbb), yvec2;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
MUL_SY yvec1, yvec2, yvec7;
|
|
ADD1_SY yvec7, yvec14, yvec14;
|
|
|
|
BROAD_SY 1*SIZE(ptrbb), yvec3;
|
|
VPERMILP_SY $0xb1, yvec0, yvec4;
|
|
VPERMILP_SY $0xb1, yvec1, yvec5;
|
|
MUL_SY yvec4, yvec3, yvec6;
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
MUL_SY yvec5, yvec3, yvec7;
|
|
ADD2_SY yvec7, yvec14, yvec14;
|
|
ADDQ $16*SIZE, ptrba;
|
|
ADDQ $2*SIZE, ptrbb;
|
|
|
|
.L313_loopE:
|
|
#### Handle ####
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
#if defined(RN) || defined(RT) || defined(CN) || defined(CT)
|
|
ADDSUB_SY yvec15, yvec7, yvec15;
|
|
ADDSUB_SY yvec14, yvec7, yvec14;
|
|
#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
|
|
SUB_SY yvec15, yvec7, yvec15;
|
|
SUB_SY yvec14, yvec7, yvec14;
|
|
#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
|
|
VPERMILP_SY $0xb1, yvec15, yvec15;
|
|
VPERMILP_SY $0xb1, yvec14, yvec14;
|
|
ADDSUB_SY yvec15, yvec7, yvec15;
|
|
ADDSUB_SY yvec14, yvec7, yvec14;
|
|
VPERMILP_SY $0xb1, yvec15, yvec15;
|
|
VPERMILP_SY $0xb1, yvec14, yvec14;
|
|
#endif
|
|
##### Load Alpha ####
|
|
BROAD_SY MEMALPHA_R,yvec7;
|
|
BROAD_SY MEMALPHA_I,yvec6;
|
|
##### Multiply Alpha ####
|
|
VPERMILP_SY $0xb1,yvec15, yvec5;
|
|
MUL_SY yvec15, yvec7, yvec15;
|
|
MUL_SY yvec5, yvec6, yvec5;
|
|
ADDSUB_SY yvec5, yvec15, yvec15;
|
|
VPERMILP_SY $0xb1,yvec14, yvec4;
|
|
MUL_SY yvec14, yvec7, yvec14;
|
|
MUL_SY yvec4, yvec6, yvec4;
|
|
ADDSUB_SY yvec4, yvec14, yvec14;
|
|
#### Writing back ####
|
|
EXTRA_SY $1, yvec15, xvec7;
|
|
EXTRA_SY $1, yvec14, xvec6;
|
|
#ifndef TRMMKERNEL
|
|
LDL_SX 0*SIZE(C0), xvec0, xvec0;
|
|
LDH_SX 2*SIZE(C0), xvec0, xvec0;
|
|
LDL_SX 4*SIZE(C0), xvec1, xvec1;
|
|
LDH_SX 6*SIZE(C0), xvec1, xvec1;
|
|
LDL_SX 8*SIZE(C0), xvec2, xvec2;
|
|
LDH_SX 10*SIZE(C0), xvec2, xvec2;
|
|
LDL_SX 12*SIZE(C0), xvec3, xvec3;
|
|
LDH_SX 14*SIZE(C0), xvec3, xvec3;
|
|
ADD_SX xvec0, xvec15, xvec15;
|
|
ADD_SX xvec1, xvec7, xvec7;
|
|
ADD_SX xvec2, xvec14, xvec14;
|
|
ADD_SX xvec3, xvec6, xvec6;
|
|
#endif
|
|
STL_SX xvec15, 0*SIZE(C0);
|
|
STH_SX xvec15, 2*SIZE(C0);
|
|
STL_SX xvec7, 4*SIZE(C0);
|
|
STH_SX xvec7, 6*SIZE(C0);
|
|
STL_SX xvec14, 8*SIZE(C0);
|
|
STH_SX xvec14, 10*SIZE(C0);
|
|
STL_SX xvec6, 12*SIZE(C0);
|
|
STH_SX xvec6, 14*SIZE(C0);
|
|
#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kkk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 8), ptrba;
|
|
ADDQ %rax, ptrbb;
|
|
#endif
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
ADDQ $8, kk;
|
|
#endif
|
|
ADDQ $16*SIZE, C0;
|
|
DECQ i;
|
|
JG .L31_bodyB;
|
|
ALIGN_5
|
|
.L31_loopE:
|
|
TEST $4, bm;
|
|
JLE .L32_loopE;
|
|
ALIGN_5
|
|
.L32_bodyB:
|
|
#if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA))
|
|
MOVQ bb,ptrbb;
|
|
#else
|
|
MOVQ bb, ptrbb;
|
|
MOVQ kk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 4), ptrba;
|
|
ADDQ %rax, ptrbb;
|
|
#endif
|
|
XOR_SY yvec15, yvec15, yvec15;
|
|
#ifndef TRMMKERNEL
|
|
MOVQ bk,k;
|
|
#elif (defined(LEFT)&&!defined(TRANSA))||(!defined(LEFT)&&defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kk, %rax;
|
|
MOVQ %rax, kkk;
|
|
#else
|
|
MOVQ kk, %rax;
|
|
#ifdef LEFT
|
|
ADDQ $4, %rax;
|
|
#else
|
|
ADDQ $1, %rax;
|
|
#endif
|
|
MOVQ %rax, kkk;
|
|
#endif
|
|
SARQ $2, k;
|
|
JLE .L321_loopE;
|
|
ALIGN_5
|
|
.L321_bodyB:
|
|
#### Unroll 1 ####
|
|
LD_SY 0*SIZE(ptrba), yvec0;
|
|
BROAD_SY 0*SIZE(ptrbb), yvec2;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
|
|
BROAD_SY 1*SIZE(ptrbb), yvec3;
|
|
VPERMILP_SY $0xb1, yvec0, yvec4;
|
|
MUL_SY yvec4, yvec3, yvec6;
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
|
|
#### Unroll 2 ####
|
|
LD_SY 8*SIZE(ptrba), yvec0;
|
|
BROAD_SY 2*SIZE(ptrbb), yvec2;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
|
|
BROAD_SY 3*SIZE(ptrbb), yvec3;
|
|
VPERMILP_SY $0xb1, yvec0, yvec4;
|
|
MUL_SY yvec4, yvec3, yvec6;
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
|
|
#### Unroll 3 ####
|
|
LD_SY 16*SIZE(ptrba), yvec0;
|
|
BROAD_SY 4*SIZE(ptrbb), yvec2;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
|
|
BROAD_SY 5*SIZE(ptrbb), yvec3;
|
|
VPERMILP_SY $0xb1, yvec0, yvec4;
|
|
MUL_SY yvec4, yvec3, yvec6;
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
|
|
#### Unroll 4 ####
|
|
LD_SY 24*SIZE(ptrba), yvec0;
|
|
BROAD_SY 6*SIZE(ptrbb), yvec2;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
|
|
BROAD_SY 7*SIZE(ptrbb), yvec3;
|
|
VPERMILP_SY $0xb1, yvec0, yvec4;
|
|
MUL_SY yvec4, yvec3, yvec6;
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
ADDQ $32*SIZE, ptrba;
|
|
ADDQ $8*SIZE, ptrbb;
|
|
DECQ k;
|
|
JG .L321_bodyB;
|
|
ALIGN_5
|
|
.L321_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $2, bk;
|
|
#else
|
|
TEST $2, kkk;
|
|
#endif
|
|
JLE .L322_loopE;
|
|
ALIGN_5
|
|
.L322_bodyB:
|
|
#### Unroll 1 ####
|
|
LD_SY 0*SIZE(ptrba), yvec0;
|
|
BROAD_SY 0*SIZE(ptrbb), yvec2;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
|
|
BROAD_SY 1*SIZE(ptrbb), yvec3;
|
|
VPERMILP_SY $0xb1, yvec0, yvec4;
|
|
MUL_SY yvec4, yvec3, yvec6;
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
|
|
#### Unroll 2 ####
|
|
LD_SY 8*SIZE(ptrba), yvec0;
|
|
BROAD_SY 2*SIZE(ptrbb), yvec2;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
|
|
BROAD_SY 3*SIZE(ptrbb), yvec3;
|
|
VPERMILP_SY $0xb1, yvec0, yvec4;
|
|
MUL_SY yvec4, yvec3, yvec6;
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
ADDQ $16*SIZE, ptrba;
|
|
ADDQ $4*SIZE, ptrbb;
|
|
|
|
.L322_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $1, bk;
|
|
#else
|
|
TEST $1, kkk;
|
|
#endif
|
|
JLE .L323_loopE;
|
|
ALIGN_5
|
|
.L323_bodyB:
|
|
#### Unroll 1 ####
|
|
LD_SY 0*SIZE(ptrba), yvec0;
|
|
BROAD_SY 0*SIZE(ptrbb), yvec2;
|
|
MUL_SY yvec0, yvec2, yvec6;
|
|
ADD1_SY yvec6, yvec15, yvec15;
|
|
|
|
BROAD_SY 1*SIZE(ptrbb), yvec3;
|
|
VPERMILP_SY $0xb1, yvec0, yvec4;
|
|
MUL_SY yvec4, yvec3, yvec6;
|
|
ADD2_SY yvec6, yvec15, yvec15;
|
|
ADDQ $8*SIZE, ptrba;
|
|
ADDQ $2*SIZE, ptrbb;
|
|
|
|
.L323_loopE:
|
|
#### Handle ####
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
#if defined(RN) || defined(RT) || defined(CN) || defined(CT)
|
|
ADDSUB_SY yvec15, yvec7, yvec15;
|
|
#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
|
|
SUB_SY yvec15, yvec7, yvec15;
|
|
#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
|
|
VPERMILP_SY $0xb1, yvec15, yvec15;
|
|
ADDSUB_SY yvec15, yvec7, yvec15;
|
|
VPERMILP_SY $0xb1, yvec15, yvec15;
|
|
#endif
|
|
##### Load Alpha ####
|
|
BROAD_SY MEMALPHA_R,yvec7;
|
|
BROAD_SY MEMALPHA_I,yvec6;
|
|
##### Multiply Alpha ####
|
|
VPERMILP_SY $0xb1,yvec15, yvec5;
|
|
MUL_SY yvec15, yvec7, yvec15;
|
|
MUL_SY yvec5, yvec6, yvec5;
|
|
ADDSUB_SY yvec5, yvec15, yvec15;
|
|
#### Writing back ####
|
|
EXTRA_SY $1, yvec15, xvec7;
|
|
#ifndef TRMMKERNEL
|
|
LDL_SX 0*SIZE(C0), xvec0, xvec0;
|
|
LDH_SX 2*SIZE(C0), xvec0, xvec0;
|
|
LDL_SX 4*SIZE(C0), xvec1, xvec1;
|
|
LDH_SX 6*SIZE(C0), xvec1, xvec1;
|
|
ADD_SX xvec0, xvec15, xvec15;
|
|
ADD_SX xvec1, xvec7, xvec7;
|
|
#endif
|
|
STL_SX xvec15, 0*SIZE(C0);
|
|
STH_SX xvec15, 2*SIZE(C0);
|
|
STL_SX xvec7, 4*SIZE(C0);
|
|
STH_SX xvec7, 6*SIZE(C0);
|
|
#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kkk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 4), ptrba;
|
|
ADDQ %rax, ptrbb;
|
|
#endif
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
ADDQ $4, kk;
|
|
#endif
|
|
ADDQ $8*SIZE, C0;
|
|
|
|
.L32_loopE:
|
|
TEST $2, bm;
|
|
JLE .L33_loopE;
|
|
ALIGN_5
|
|
.L33_bodyB:
|
|
#if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA))
|
|
MOVQ bb,ptrbb;
|
|
#else
|
|
MOVQ bb, ptrbb;
|
|
MOVQ kk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 2), ptrba;
|
|
ADDQ %rax, ptrbb;
|
|
#endif
|
|
XOR_SY yvec15, yvec15, yvec15;
|
|
#ifndef TRMMKERNEL
|
|
MOVQ bk,k;
|
|
#elif (defined(LEFT)&&!defined(TRANSA))||(!defined(LEFT)&&defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kk, %rax;
|
|
MOVQ %rax, kkk;
|
|
#else
|
|
MOVQ kk, %rax;
|
|
#ifdef LEFT
|
|
ADDQ $2, %rax;
|
|
#else
|
|
ADDQ $1, %rax;
|
|
#endif
|
|
MOVQ %rax, kkk;
|
|
#endif
|
|
SARQ $2, k;
|
|
JLE .L331_loopE;
|
|
ALIGN_5
|
|
.L331_bodyB:
|
|
#### Unroll 1 ####
|
|
LD_SX 0*SIZE(ptrba), xvec0;
|
|
BROAD_SX 0*SIZE(ptrbb), xvec2;
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec0, xvec1;
|
|
BROAD_SX 1*SIZE(ptrbb), xvec3;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec15, xvec15;
|
|
|
|
#### Unroll 2 ####
|
|
LD_SX 4*SIZE(ptrba), xvec0;
|
|
BROAD_SX 2*SIZE(ptrbb), xvec2;
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec0, xvec1;
|
|
BROAD_SX 3*SIZE(ptrbb), xvec3;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec15, xvec15;
|
|
|
|
#### Unroll 3 ####
|
|
LD_SX 8*SIZE(ptrba), xvec0;
|
|
BROAD_SX 4*SIZE(ptrbb), xvec2;
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec0, xvec1;
|
|
BROAD_SX 5*SIZE(ptrbb), xvec3;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec15, xvec15;
|
|
|
|
#### Unroll 4 ####
|
|
LD_SX 12*SIZE(ptrba), xvec0;
|
|
BROAD_SX 6*SIZE(ptrbb), xvec2;
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec0, xvec1;
|
|
BROAD_SX 7*SIZE(ptrbb), xvec3;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec15, xvec15;
|
|
ADDQ $16*SIZE, ptrba;
|
|
ADDQ $8*SIZE, ptrbb;
|
|
DECQ k;
|
|
JG .L331_bodyB;
|
|
ALIGN_5
|
|
.L331_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $2, bk;
|
|
#else
|
|
TEST $2, kkk;
|
|
#endif
|
|
JLE .L332_loopE;
|
|
ALIGN_5
|
|
.L332_bodyB:
|
|
#### Unroll 1 ####
|
|
LD_SX 0*SIZE(ptrba), xvec0;
|
|
BROAD_SX 0*SIZE(ptrbb), xvec2;
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec0, xvec1;
|
|
BROAD_SX 1*SIZE(ptrbb), xvec3;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec15, xvec15;
|
|
|
|
#### Unroll 2 ####
|
|
LD_SX 4*SIZE(ptrba), xvec0;
|
|
BROAD_SX 2*SIZE(ptrbb), xvec2;
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec0, xvec1;
|
|
BROAD_SX 3*SIZE(ptrbb), xvec3;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec15, xvec15;
|
|
ADDQ $8*SIZE, ptrba;
|
|
ADDQ $4*SIZE, ptrbb;
|
|
|
|
.L332_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $1, bk;
|
|
#else
|
|
TEST $1, kkk;
|
|
#endif
|
|
JLE .L333_loopE;
|
|
ALIGN_5
|
|
.L333_bodyB:
|
|
#### Unroll 1 ####
|
|
LD_SX 0*SIZE(ptrba), xvec0;
|
|
BROAD_SX 0*SIZE(ptrbb), xvec2;
|
|
MUL_SX xvec0, xvec2, xvec2;
|
|
ADD1_SX xvec2, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec0, xvec1;
|
|
BROAD_SX 1*SIZE(ptrbb), xvec3;
|
|
MUL_SX xvec1, xvec3, xvec3;
|
|
ADD2_SX xvec3, xvec15, xvec15;
|
|
ADDQ $4*SIZE, ptrba;
|
|
ADDQ $2*SIZE, ptrbb;
|
|
|
|
.L333_loopE:
|
|
#### Handle ####
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
#if defined(RN) || defined(RT) || defined(CN) || defined(CT)
|
|
ADDSUB_SX xvec15, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec15;
|
|
#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
|
|
SUB_SX xvec15, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec15;
|
|
#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
|
|
SHUF_SX $0xb1, xvec15, xvec15;
|
|
ADDSUB_SX xvec15, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec15;
|
|
SHUF_SX $0xb1, xvec15, xvec15;
|
|
#endif
|
|
#### Mulitply Alpha ####
|
|
BROAD_SX MEMALPHA_R, xvec7;
|
|
BROAD_SX MEMALPHA_I, xvec6;
|
|
#### Writng back ####
|
|
VPERMILP_SX $0xb1,xvec15, xvec5;
|
|
MUL_SX xvec7, xvec15, xvec15;
|
|
MUL_SX xvec6, xvec5, xvec5;
|
|
ADDSUB_SX xvec5, xvec15, xvec15;
|
|
#ifndef TRMMKERNEL
|
|
LDL_SX 0*SIZE(C0), xvec0, xvec0;
|
|
LDH_SX 2*SIZE(C0), xvec0, xvec0;
|
|
ADD_SX xvec0, xvec15, xvec15;
|
|
#endif
|
|
STL_SX xvec15, 0*SIZE(C0);
|
|
STH_SX xvec15, 2*SIZE(C0);
|
|
#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kkk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
LEAQ (ptrba, %rax, 2), ptrba;
|
|
ADDQ %rax, ptrbb;
|
|
#endif
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
ADDQ $2, kk;
|
|
#endif
|
|
ADDQ $4*SIZE, C0;
|
|
|
|
.L33_loopE:
|
|
TEST $1, bm;
|
|
JLE .L34_loopE;
|
|
ALIGN_5
|
|
.L34_bodyB:
|
|
#if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA))
|
|
MOVQ bb,ptrbb;
|
|
#else
|
|
MOVQ bb, ptrbb;
|
|
MOVQ kk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
ADDQ %rax, ptrba;
|
|
ADDQ %rax, ptrbb;
|
|
#endif
|
|
XOR_SY yvec15, yvec15, yvec15;
|
|
#ifndef TRMMKERNEL
|
|
MOVQ bk,k;
|
|
#elif (defined(LEFT)&&!defined(TRANSA))||(!defined(LEFT)&&defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kk, %rax;
|
|
MOVQ %rax, kkk;
|
|
#else
|
|
MOVQ kk, %rax;
|
|
#ifdef LEFT
|
|
ADDQ $1, %rax;
|
|
#else
|
|
ADDQ $1, %rax;
|
|
#endif
|
|
MOVQ %rax, kkk;
|
|
#endif
|
|
SARQ $2, k;
|
|
JLE .L341_loopE;
|
|
ALIGN_5
|
|
.L341_bodyB:
|
|
LD_SX 0*SIZE(ptrba), xvec0;
|
|
LD_SX 0*SIZE(ptrbb), xvec2;
|
|
SHUF_SX $0xa0, xvec2, xvec3;
|
|
MUL_SX xvec0, xvec3, xvec3;
|
|
ADD1_SX xvec3, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec0, xvec1;
|
|
SHUF_SX $0xf5, xvec2, xvec4;
|
|
MUL_SX xvec1, xvec4, xvec4;
|
|
ADD2_SX xvec4, xvec15, xvec15;
|
|
|
|
LD_SX 4*SIZE(ptrba), xvec0;
|
|
LD_SX 4*SIZE(ptrbb), xvec2;
|
|
SHUF_SX $0xa0, xvec2, xvec3;
|
|
MUL_SX xvec0, xvec3, xvec3;
|
|
ADD1_SX xvec3, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec0, xvec1;
|
|
SHUF_SX $0xf5, xvec2, xvec4;
|
|
MUL_SX xvec1, xvec4, xvec4;
|
|
ADD2_SX xvec4, xvec15, xvec15;
|
|
ADDQ $8*SIZE, ptrba;
|
|
ADDQ $8*SIZE, ptrbb;
|
|
DECQ k;
|
|
JG .L341_bodyB;
|
|
ALIGN_5
|
|
.L341_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $2, bk;
|
|
#else
|
|
TEST $2, kkk;
|
|
#endif
|
|
JLE .L342_loopE;
|
|
ALIGN_5
|
|
.L342_bodyB:
|
|
LD_SX 0*SIZE(ptrba), xvec0;
|
|
LD_SX 0*SIZE(ptrbb), xvec2;
|
|
SHUF_SX $0xa0, xvec2, xvec3;
|
|
MUL_SX xvec0, xvec3, xvec3;
|
|
ADD1_SX xvec3, xvec15, xvec15;
|
|
SHUF_SX $0xb1, xvec0, xvec1;
|
|
SHUF_SX $0xf5, xvec2, xvec4;
|
|
MUL_SX xvec1, xvec4, xvec4;
|
|
ADD2_SX xvec4, xvec15, xvec15;
|
|
ADDQ $4*SIZE, ptrba;
|
|
ADDQ $4*SIZE, ptrbb;
|
|
|
|
.L342_loopE:
|
|
#ifndef TRMMKERNEL
|
|
TEST $1, bk;
|
|
#else
|
|
TEST $1, kkk;
|
|
#endif
|
|
JLE .L343_loopE;
|
|
ALIGN_5
|
|
.L343_bodyB:
|
|
XOR_SY yvec0, yvec0, yvec0;
|
|
XOR_SY yvec2, yvec2, yvec2;
|
|
LDL_SX 0*SIZE(ptrba), xvec0, xvec0;
|
|
LDL_SX 0*SIZE(ptrbb), xvec2, xvec2;
|
|
SHUF_SX $0xe0, xvec2, xvec3;
|
|
MUL_SX xvec0, xvec3, xvec3;
|
|
ADD1_SX xvec3, xvec15, xvec15;
|
|
SHUF_SX $0xe1, xvec0, xvec1;
|
|
SHUF_SX $0xe5, xvec2, xvec4;
|
|
MUL_SX xvec1, xvec4, xvec4;
|
|
ADD2_SX xvec4, xvec15, xvec15;
|
|
ADDQ $2*SIZE, ptrba;
|
|
ADDQ $2*SIZE, ptrbb;
|
|
|
|
.L343_loopE:
|
|
#### Handle ####
|
|
XOR_SY yvec7, yvec7, yvec7;
|
|
#if defined(RN) || defined(RT) || defined(CN) || defined(CT)
|
|
ADDSUB_SX xvec15, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec15;
|
|
#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
|
|
SUB_SX xvec15, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec15;
|
|
#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
|
|
SHUF_SX $0xb1, xvec15, xvec15;
|
|
ADDSUB_SX xvec15, xvec7, xvec7;
|
|
MOV_SX xvec7, xvec15;
|
|
SHUF_SX $0xb1, xvec15, xvec15;
|
|
#endif
|
|
BROAD_SX MEMALPHA_R, xvec7;
|
|
BROAD_SX MEMALPHA_I, xvec6;
|
|
VPERMILP_SX $0xb1, xvec15, xvec5;
|
|
MUL_SX xvec7, xvec15, xvec15;
|
|
MUL_SX xvec6, xvec5, xvec5;
|
|
ADDSUB_SX xvec5, xvec15, xvec15;
|
|
SHUF_SX $0x44, xvec15, xvec14;
|
|
SHUF_SX $0xee, xvec15, xvec13;
|
|
ADD_SX xvec13, xvec14, xvec14;
|
|
#ifndef TRMMKERNEL
|
|
LDL_SX 0*SIZE(C0), xvec0, xvec0;
|
|
ADD_SX xvec0, xvec14, xvec14;
|
|
#endif
|
|
STL_SX xvec14, 0*SIZE(C0);
|
|
#if (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
|
|
MOVQ bk, %rax;
|
|
SUBQ kkk, %rax;
|
|
SALQ $ZBASE_SHIFT, %rax;
|
|
ADDQ %rax, ptrba;
|
|
ADDQ %rax, ptrbb;
|
|
#endif
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
ADDQ $1, kk;
|
|
#endif
|
|
ADDQ $2*SIZE, C0;
|
|
|
|
.L34_loopE:
|
|
#if defined(TRMMKERNEL) && !defined(LEFT)
|
|
ADDQ $1, kk;
|
|
#endif
|
|
|
|
MOVQ bk, k;
|
|
SALQ $3, k;
|
|
ADDQ k, bb;
|
|
ADDQ ldc, C;
|
|
.L30_loopE:
|
|
movq 0(%rsp), %rbx;
|
|
movq 8(%rsp), %rbp;
|
|
movq 16(%rsp), %r12;
|
|
movq 24(%rsp), %r13;
|
|
movq 32(%rsp), %r14;
|
|
movq 40(%rsp), %r15;
|
|
|
|
vzeroupper
|
|
|
|
#ifdef WINDOWS_ABI
|
|
movq 48(%rsp), %rdi
|
|
movq 56(%rsp), %rsi
|
|
movups 64(%rsp), %xmm6
|
|
movups 80(%rsp), %xmm7
|
|
movups 96(%rsp), %xmm8
|
|
movups 112(%rsp), %xmm9
|
|
movups 128(%rsp), %xmm10
|
|
movups 144(%rsp), %xmm11
|
|
movups 160(%rsp), %xmm12
|
|
movups 176(%rsp), %xmm13
|
|
movups 192(%rsp), %xmm14
|
|
movups 208(%rsp), %xmm15
|
|
#endif
|
|
|
|
addq $STACKSIZE, %rsp;
|
|
ret
|
|
|
|
EPILOGUE
|