floraachy
/
OpenBLAS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
OpenBLAS/kernel/power/dgemm_macros_power9.S

3623 lines
61 KiB
ArmAsm
Raw Permalink Blame History

/***************************************************************************
Copyright (c) 2013-2019, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
/**************************************************************************************
* Abdelrauf(quickwritereader@googlemail.com)
* BLASTEST : OK
* CTEST : OK
* TEST : OK
* LAPACK-TEST : OK
**************************************************************************************/
/*********************************************************************
* Macros for N=4, M=16 *
*********************************************************************/
.macro LOAD4x16_1
LOAD4x16 1
.endm
.macro LOAD4x16_0
LOAD4x16 0
.endm
.macro LOAD4x16 Zero
lxv vs24, 0(BO)
lxv vs26, 16(BO)
xxpermdi vs25, vs24, vs24,2
xxpermdi vs27, vs26, vs26,2
lxv vs0, 0(AO)
lxv vs1, 16(AO)
lxv vs2, 32(AO)
lxv vs3, 48(AO)
lxv vs4, 64(AO)
lxv vs5, 80(AO)
lxv vs6, 96(AO)
lxv vs7, 112(AO)
.if \Zero==1
xxlxor vs32,vs32,vs32
xxlxor vs33,vs33,vs33
xxlxor vs34,vs34,vs34
xxlxor vs35,vs35,vs35
xxlxor vs36,vs36,vs36
xxlxor vs37,vs37,vs37
xxlxor vs38,vs38,vs38
xxlxor vs39,vs39,vs39
xxlxor vs40, vs40, vs40
xxlxor vs41, vs41, vs41
xxlxor vs42, vs42, vs42
xxlxor vs43, vs43, vs43
xxlxor vs44, vs44, vs44
xxlxor vs45, vs45, vs45
xxlxor vs46, vs46, vs46
xxlxor vs47, vs47, vs47
xxlxor vs48, vs48, vs48
xxlxor vs49, vs49, vs49
xxlxor vs50, vs50, vs50
xxlxor vs51, vs51, vs51
xxlxor vs52, vs52, vs52
xxlxor vs53, vs53, vs53
xxlxor vs54, vs54, vs54
xxlxor vs55, vs55, vs55
xxlxor vs56, vs56, vs56
xxlxor vs57, vs57, vs57
xxlxor vs58, vs58, vs58
xxlxor vs59, vs59, vs59
xxlxor vs60, vs60, vs60
xxlxor vs61, vs61, vs61
xxlxor vs62, vs62, vs62
xxlxor vs63, vs63, vs63
.endif
.endm
#define unit_size 8
#define DISP32(ind,disp) (ind*unit_size*32+disp)
#define DISP16(ind,disp) (ind*unit_size*16+disp)
#define DISP8(ind,disp) (ind*unit_size*8+disp)
#define DISP4(ind,disp) (ind*unit_size*4+disp)
#define DISP2(ind,disp) (ind*unit_size*2+disp)
#define DISP1(ind,disp) (ind*unit_size+disp)
.macro KERNEL4x16_L1_L2 Index,IsLast
KERNEL4x16_L1_L2_I AO,BO, 0,0,0, \Index,\IsLast,0
.endm
.macro KERNEL4x16_I1_L2 OffsetA,OffsetB, Index,IsLast
KERNEL4x16_L1_L2_I AO,BO,1,\OffsetA,\OffsetB,\Index,\IsLast,0
.endm
.macro KERNEL4x16_I1_L2_2 OffsetA,OffsetB, Index,IsLast
KERNEL4x16_L1_L2_I AO,BO, 0,\OffsetA,\OffsetB,\Index,\IsLast,0
.endm
.macro KERNEL4x16_I1_L2_3 OffsetA,OffsetB, Index,IsLast
KERNEL4x16_L1_L2_I AO,BO, 0,\OffsetA,\OffsetB,\Index,\IsLast,1
.endm
.macro KERNEL4x16_I2_L2 AREG,BREG,OffsetA,OffsetB, Index,IsLast
KERNEL4x16_L1_L2_I \AREG,\BREG,1,\OffsetA,\OffsetB,\Index,\IsLast,0
.endm
.macro KERNEL4x16_I2_L2_2 AREG,BREG,OffsetA,OffsetB, Index,IsLast
KERNEL4x16_L1_L2_I \AREG,\BREG, 0,\OffsetA,\OffsetB,\Index,\IsLast,0
.endm
.macro KERNEL4x16_I2_L2_3 AREG,BREG,OffsetA,OffsetB, Index,IsLast
KERNEL4x16_L1_L2_I \AREG,\BREG, 0,\OffsetA,\OffsetB,\Index,\IsLast,1
.endm
.macro KERNEL4x16_L1_L2_I AREG,BREG, First, OffsetA,OffsetB, Index,IsLast ,Complete
.if \First ==1
xvmuldp vs32, vs0, vs24
xvmuldp vs33, vs1, vs24
xvmuldp vs34, vs2, vs24
xvmuldp vs35, vs3, vs24
.else
xvmaddadp vs32, vs0, vs24
xvmaddadp vs33, vs1, vs24
xvmaddadp vs34, vs2, vs24
xvmaddadp vs35, vs3, vs24
.endif
lxv vs8, DISP32(\Index,0+\OffsetA)(\AREG)
lxv vs9, DISP32(\Index,16+\OffsetA)(\AREG)
lxv vs10, DISP32(\Index,32+\OffsetA)(\AREG)
lxv vs11, DISP32(\Index,48+\OffsetA)(\AREG)
.if \First ==1
xvmuldp vs36, vs4, vs24
xvmuldp vs37, vs5, vs24
xvmuldp vs38, vs6, vs24
xvmuldp vs39, vs7, vs24
.else
xvmaddadp vs36, vs4, vs24
xvmaddadp vs37, vs5, vs24
xvmaddadp vs38, vs6, vs24
xvmaddadp vs39, vs7, vs24
.endif
lxv vs28, DISP8(\Index,0 +\OffsetB)(\BREG)
lxv vs30, DISP8(\Index,16 +\OffsetB)(\BREG)
xxpermdi vs29, vs28, vs28,2
xxpermdi vs31, vs30, vs30,2
.if \First ==1
xvmuldp vs40, vs0, vs25
xvmuldp vs41, vs1, vs25
xvmuldp vs42, vs2, vs25
xvmuldp vs43, vs3, vs25
xvmuldp vs44, vs4, vs25
xvmuldp vs45, vs5, vs25
xvmuldp vs46, vs6, vs25
xvmuldp vs47, vs7, vs25
xvmuldp vs48, vs0, vs26
xvmuldp vs49, vs1, vs26
xvmuldp vs50, vs2, vs26
xvmuldp vs51, vs3, vs26
.else
xvmaddadp vs40, vs0, vs25
xvmaddadp vs41, vs1, vs25
xvmaddadp vs42, vs2, vs25
xvmaddadp vs43, vs3, vs25
xvmaddadp vs44, vs4, vs25
xvmaddadp vs45, vs5, vs25
xvmaddadp vs46, vs6, vs25
xvmaddadp vs47, vs7, vs25
xvmaddadp vs48, vs0, vs26
xvmaddadp vs49, vs1, vs26
xvmaddadp vs50, vs2, vs26
xvmaddadp vs51, vs3, vs26
.endif
lxv vs12, DISP32(\Index,64+\OffsetA)(\AREG)
lxv vs13, DISP32(\Index,80+\OffsetA)(\AREG)
.if \First ==1
xvmuldp vs52, vs4, vs26
xvmuldp vs53, vs5, vs26
xvmuldp vs54, vs6, vs26
xvmuldp vs55, vs7, vs26
.else
xvmaddadp vs52, vs4, vs26
xvmaddadp vs53, vs5, vs26
xvmaddadp vs54, vs6, vs26
xvmaddadp vs55, vs7, vs26
.endif
lxv vs14, DISP32(\Index,96+\OffsetA)(\AREG)
lxv vs15, DISP32(\Index,112+\OffsetA)(\AREG)
.if \First ==1
xvmuldp vs56, vs0, vs27
xvmuldp vs57, vs1, vs27
xvmuldp vs58, vs2, vs27
xvmuldp vs59, vs3, vs27
xvmuldp vs60, vs4, vs27
xvmuldp vs61, vs5, vs27
xvmuldp vs62, vs6, vs27
xvmuldp vs63, vs7, vs27
.else
xvmaddadp vs56, vs0, vs27
xvmaddadp vs57, vs1, vs27
xvmaddadp vs58, vs2, vs27
xvmaddadp vs59, vs3, vs27
xvmaddadp vs60, vs4, vs27
xvmaddadp vs61, vs5, vs27
xvmaddadp vs62, vs6, vs27
xvmaddadp vs63, vs7, vs27
.endif
xvmaddadp vs32, vs8, vs28
xvmaddadp vs33, vs9, vs28
xvmaddadp vs34, vs10, vs28
xvmaddadp vs35, vs11, vs28
.if \Complete==0
lxv vs0, DISP32(\Index,128+\OffsetA)(\AREG)
lxv vs1, DISP32(\Index,144+\OffsetA)(\AREG)
.endif
xvmaddadp vs36, vs12, vs28
xvmaddadp vs37, vs13, vs28
xvmaddadp vs38, vs14, vs28
xvmaddadp vs39, vs15, vs28
.if \Complete==0
lxv vs24, DISP8(\Index,32 +\OffsetB)(\BREG)
lxv vs26, DISP8(\Index,48 +\OffsetB)(\BREG)
xxpermdi vs25, vs24, vs24,2
xxpermdi vs27, vs26, vs26,2
.endif
xvmaddadp vs40, vs8, vs29
xvmaddadp vs41, vs9, vs29
xvmaddadp vs42, vs10, vs29
xvmaddadp vs43, vs11, vs29
.if \Complete==0
lxv vs2, DISP32(\Index,160+\OffsetA)(\AREG)
lxv vs3, DISP32(\Index,176+\OffsetA)(\AREG)
.endif
xvmaddadp vs44, vs12, vs29
xvmaddadp vs45, vs13, vs29
xvmaddadp vs46, vs14, vs29
xvmaddadp vs47, vs15, vs29
xvmaddadp vs48, vs8, vs30
xvmaddadp vs49, vs9, vs30
xvmaddadp vs50, vs10, vs30
xvmaddadp vs51, vs11, vs30
.if \Complete==0
lxv vs4, DISP32(\Index,192+\OffsetA)(\AREG)
lxv vs5, DISP32(\Index,208+\OffsetA)(\AREG)
.endif
xvmaddadp vs52, vs12, vs30
xvmaddadp vs53, vs13, vs30
xvmaddadp vs54, vs14, vs30
xvmaddadp vs55, vs15, vs30
.if \Complete==0
lxv vs6, DISP32(\Index,224+\OffsetA)(\AREG)
lxv vs7, DISP32(\Index,240+\OffsetA)(\AREG)
.endif
xvmaddadp vs56, vs8, vs31
xvmaddadp vs57, vs9, vs31
xvmaddadp vs58, vs10, vs31
xvmaddadp vs59, vs11, vs31
xvmaddadp vs60, vs12, vs31
xvmaddadp vs61, vs13, vs31
xvmaddadp vs62, vs14, vs31
xvmaddadp vs63, vs15, vs31
.if \IsLast==1
.if \Complete==1
addi \AREG, \AREG, DISP32(\Index,128+\OffsetA)
addi \BREG, \BREG, DISP8(\Index,32+\OffsetB)
.else
addi \AREG, \AREG, DISP32(\Index,256)
addi \BREG, \BREG, DISP8(\Index,64)
.endif
.endif
.endm
.macro KERNEL4x16 First
lxv vs24, 0(BO)
lxv vs26, 16(BO)
xxpermdi vs25, vs24, vs24,2
xxpermdi vs27, vs26, vs26,2
lxv vs0, 0(AO)
lxv vs1, 16(AO)
lxv vs2, 32(AO)
lxv vs3, 48(AO)
lxv vs4, 64(AO)
lxv vs5, 80(AO)
lxv vs6, 96(AO)
lxv vs7, 112(AO)
addi BO, BO, 32
addi AO, AO, 128
.if \First==1
xvmuldp vs32, vs0, vs24
xvmuldp vs33, vs1, vs24
xvmuldp vs34, vs2, vs24
xvmuldp vs35, vs3, vs24
xvmuldp vs36, vs4, vs24
xvmuldp vs37, vs5, vs24
xvmuldp vs38, vs6, vs24
xvmuldp vs39, vs7, vs24
xvmuldp vs40, vs0, vs25
xvmuldp vs41, vs1, vs25
xvmuldp vs42, vs2, vs25
xvmuldp vs43, vs3, vs25
xvmuldp vs44, vs4, vs25
xvmuldp vs45, vs5, vs25
xvmuldp vs46, vs6, vs25
xvmuldp vs47, vs7, vs25
xvmuldp vs48, vs0, vs26
xvmuldp vs49, vs1, vs26
xvmuldp vs50, vs2, vs26
xvmuldp vs51, vs3, vs26
xvmuldp vs52, vs4, vs26
xvmuldp vs53, vs5, vs26
xvmuldp vs54, vs6, vs26
xvmuldp vs55, vs7, vs26
xvmuldp vs56, vs0, vs27
xvmuldp vs57, vs1, vs27
xvmuldp vs58, vs2, vs27
xvmuldp vs59, vs3, vs27
xvmuldp vs60, vs4, vs27
xvmuldp vs61, vs5, vs27
xvmuldp vs62, vs6, vs27
xvmuldp vs63, vs7, vs27
.else
xvmaddadp vs32, vs0, vs24
xvmaddadp vs33, vs1, vs24
xvmaddadp vs34, vs2, vs24
xvmaddadp vs35, vs3, vs24
xvmaddadp vs36, vs4, vs24
xvmaddadp vs37, vs5, vs24
xvmaddadp vs38, vs6, vs24
xvmaddadp vs39, vs7, vs24
xvmaddadp vs40, vs0, vs25
xvmaddadp vs41, vs1, vs25
xvmaddadp vs42, vs2, vs25
xvmaddadp vs43, vs3, vs25
xvmaddadp vs44, vs4, vs25
xvmaddadp vs45, vs5, vs25
xvmaddadp vs46, vs6, vs25
xvmaddadp vs47, vs7, vs25
xvmaddadp vs48, vs0, vs26
xvmaddadp vs49, vs1, vs26
xvmaddadp vs50, vs2, vs26
xvmaddadp vs51, vs3, vs26
xvmaddadp vs52, vs4, vs26
xvmaddadp vs53, vs5, vs26
xvmaddadp vs54, vs6, vs26
xvmaddadp vs55, vs7, vs26
xvmaddadp vs56, vs0, vs27
xvmaddadp vs57, vs1, vs27
xvmaddadp vs58, vs2, vs27
xvmaddadp vs59, vs3, vs27
xvmaddadp vs60, vs4, vs27
xvmaddadp vs61, vs5, vs27
xvmaddadp vs62, vs6, vs27
xvmaddadp vs63, vs7, vs27
.endif
.endm
.macro SAVE4x16_REGS
add C2, CO, LDC
add C3, C2, LDC
add C4, C3, LDC
.endm
.macro SAVE4x16
#ifndef TRMMKERNEL
lxv vs0, 0(CO)
lxv vs2, 16(CO)
lxv vs4, 32(CO)
lxv vs6, 48(CO)
#endif
xxpermdi vs8, vs40,vs32,1
xxpermdi vs9 ,vs32,vs40,1
#ifndef TRMMKERNEL
lxv vs24, 64(CO)
lxv vs26, 80(CO)
lxv vs28, 96(CO)
lxv vs30, 112(CO)
#endif
xxpermdi vs10, vs41,vs33,1
xxpermdi vs11 ,vs33,vs41,1
#ifndef TRMMKERNEL
lxv vs1, 0(C2)
lxv vs3, 16(C2)
lxv vs5, 32(C2)
lxv vs7, 48(C2)
#endif
xxpermdi vs12, vs42,vs34,1
xxpermdi vs13 ,vs34,vs42,1
#ifndef TRMMKERNEL
lxv vs25, 64(C2)
lxv vs27, 80(C2)
#endif
xxpermdi vs14, vs43,vs35,1
xxpermdi vs15 ,vs35,vs43,1
#ifndef TRMMKERNEL
lxv vs29, 96(C2)
lxv vs31, 112(C2)
#endif
#ifndef TRMMKERNEL
xvmaddadp vs0, vs8, alpha_r
xvmaddadp vs1, vs9, alpha_r
xvmaddadp vs2, vs10, alpha_r
xvmaddadp vs3, vs11, alpha_r
#else
xvmuldp vs0, vs8, alpha_r
xvmuldp vs1, vs9, alpha_r
xvmuldp vs2, vs10, alpha_r
xvmuldp vs3, vs11, alpha_r
#endif
xxpermdi vs8, vs44,vs36,1
xxpermdi vs9 ,vs36,vs44,1
xxpermdi vs10, vs45,vs37,1
xxpermdi vs11 ,vs37,vs45,1
#ifndef TRMMKERNEL
xvmaddadp vs4, vs12, alpha_r
xvmaddadp vs5, vs13, alpha_r
xvmaddadp vs6, vs14, alpha_r
xvmaddadp vs7, vs15, alpha_r
#else
xvmuldp vs4, vs12, alpha_r
xvmuldp vs5, vs13, alpha_r
xvmuldp vs6, vs14, alpha_r
xvmuldp vs7, vs15, alpha_r
#endif
xxpermdi vs12, vs46,vs38,1
xxpermdi vs13 ,vs38,vs46,1
xxpermdi vs14, vs47,vs39,1
xxpermdi vs15 ,vs39,vs47,1
#ifndef TRMMKERNEL
xvmaddadp vs24, vs8, alpha_r
xvmaddadp vs25, vs9, alpha_r
xvmaddadp vs26, vs10, alpha_r
xvmaddadp vs27, vs11, alpha_r
xvmaddadp vs28, vs12, alpha_r
xvmaddadp vs29, vs13, alpha_r
xvmaddadp vs30, vs14, alpha_r
xvmaddadp vs31, vs15, alpha_r
#else
xvmuldp vs24, vs8, alpha_r
xvmuldp vs25, vs9, alpha_r
xvmuldp vs26, vs10, alpha_r
xvmuldp vs27, vs11, alpha_r
xvmuldp vs28, vs12, alpha_r
xvmuldp vs29, vs13, alpha_r
xvmuldp vs30, vs14, alpha_r
xvmuldp vs31, vs15, alpha_r
#endif
stxv vs0, 0(CO)
stxv vs2, 16(CO)
stxv vs4, 32(CO)
stxv vs6, 48(CO)
stxv vs24, 64(CO)
stxv vs26, 80(CO)
stxv vs28, 96(CO)
stxv vs30, 112(CO)
stxv vs1, 0(C2)
stxv vs3, 16(C2)
stxv vs5, 32(C2)
stxv vs7, 48(C2)
stxv vs25, 64(C2)
stxv vs27, 80(C2)
stxv vs29, 96(C2)
stxv vs31, 112(C2)
#ifndef TRMMKERNEL
lxv vs0, 0(C3)
lxv vs2, 16(C3)
lxv vs4, 32(C3)
lxv vs6, 48(C3)
#endif
xxpermdi vs8, vs56,vs48,1
xxpermdi vs9 ,vs48,vs56,1
#ifndef TRMMKERNEL
lxv vs24, 64(C3)
lxv vs26, 80(C3)
#endif
xxpermdi vs10, vs57,vs49,1
xxpermdi vs11 ,vs49,vs57,1
#ifndef TRMMKERNEL
lxv vs28, 96(C3)
lxv vs30, 112(C3)
#endif
xxpermdi vs12, vs58,vs50,1
xxpermdi vs13 ,vs50,vs58,1
#ifndef TRMMKERNEL
lxv vs1, 0(C4)
lxv vs3, 16(C4)
#endif
xxpermdi vs14, vs59,vs51,1
xxpermdi vs15 ,vs51,vs59,1
#ifndef TRMMKERNEL
lxv vs5, 32(C4)
lxv vs7, 48(C4)
lxv vs25, 64(C4)
lxv vs27, 80(C4)
lxv vs29, 96(C4)
lxv vs31, 112(C4)
#endif
#ifndef TRMMKERNEL
xvmaddadp vs0, vs8, alpha_r
xvmaddadp vs1, vs9, alpha_r
xvmaddadp vs2, vs10, alpha_r
xvmaddadp vs3, vs11, alpha_r
#else
xvmuldp vs0, vs8, alpha_r
xvmuldp vs1, vs9, alpha_r
xvmuldp vs2, vs10, alpha_r
xvmuldp vs3, vs11, alpha_r
#endif
xxpermdi vs8, vs60,vs52,1
xxpermdi vs9 ,vs52,vs60,1
xxpermdi vs10, vs61,vs53,1
xxpermdi vs11 ,vs53,vs61,1
#ifndef TRMMKERNEL
xvmaddadp vs4, vs12, alpha_r
xvmaddadp vs5, vs13, alpha_r
xvmaddadp vs6, vs14, alpha_r
xvmaddadp vs7, vs15, alpha_r
#else
xvmuldp vs4, vs12, alpha_r
xvmuldp vs5, vs13, alpha_r
xvmuldp vs6, vs14, alpha_r
xvmuldp vs7, vs15, alpha_r
#endif
xxpermdi vs12, vs62,vs54,1
xxpermdi vs13 ,vs54,vs62,1
xxpermdi vs14, vs63,vs55,1
xxpermdi vs15 ,vs55,vs63,1
#ifndef TRMMKERNEL
xvmaddadp vs24, vs8, alpha_r
xvmaddadp vs25, vs9, alpha_r
xvmaddadp vs26, vs10, alpha_r
xvmaddadp vs27, vs11, alpha_r
xvmaddadp vs28, vs12, alpha_r
xvmaddadp vs29, vs13, alpha_r
xvmaddadp vs30, vs14, alpha_r
xvmaddadp vs31, vs15, alpha_r
#else
xvmuldp vs24, vs8, alpha_r
xvmuldp vs25, vs9, alpha_r
xvmuldp vs26, vs10, alpha_r
xvmuldp vs27, vs11, alpha_r
xvmuldp vs28, vs12, alpha_r
xvmuldp vs29, vs13, alpha_r
xvmuldp vs30, vs14, alpha_r
xvmuldp vs31, vs15, alpha_r
#endif
stxv vs0, 0(C3)
stxv vs2, 16(C3)
stxv vs4, 32(C3)
stxv vs6, 48(C3)
stxv vs24, 64(C3)
stxv vs26, 80(C3)
stxv vs28, 96(C3)
stxv vs30, 112(C3)
stxv vs1, 0(C4)
stxv vs3, 16(C4)
stxv vs5, 32(C4)
stxv vs7, 48(C4)
stxv vs25, 64(C4)
stxv vs27, 80(C4)
stxv vs29, 96(C4)
stxv vs31, 112(C4)
addi CO, CO, 128
.endm
/*********************************************************************
* Macros for N=4, M=8 *
*********************************************************************/
.macro LOAD4x8_1
LOAD4x8 1
.endm
.macro LOAD4x8_0
LOAD4x8 0
.endm
.macro LOAD4x8 Zero
lxv vs24, 0(BO)
lxv vs26, 16(BO)
xxpermdi vs25, vs24, vs24,2
xxpermdi vs27, vs26, vs26,2
lxv vs0, 0(AO)
lxv vs1, 16(AO)
lxv vs2, 32(AO)
lxv vs3, 48(AO)
.if \Zero==1
xxlxor vs32,vs32,vs32
xxlxor vs33,vs33,vs33
xxlxor vs34,vs34,vs34
xxlxor vs35,vs35,vs35
xxlxor vs40, vs40, vs40
xxlxor vs41, vs41, vs41
xxlxor vs42, vs42, vs42
xxlxor vs43, vs43, vs43
xxlxor vs48, vs48, vs48
xxlxor vs49, vs49, vs49
xxlxor vs50, vs50, vs50
xxlxor vs51, vs51, vs51
xxlxor vs56, vs56, vs56
xxlxor vs57, vs57, vs57
xxlxor vs58, vs58, vs58
xxlxor vs59, vs59, vs59
.endif
.endm
.macro KERNEL4x8_L1_L2 Index,IsLast
KERNEL4x8_L1_L2_I 0,0,0, \Index,\IsLast,0
.endm
.macro KERNEL4x8_I1_L2 OffsetA,OffsetB, Index,IsLast
KERNEL4x8_L1_L2_I 1,\OffsetA,\OffsetB,\Index,\IsLast,0
.endm
.macro KERNEL4x8_I1_L2_2 OffsetA,OffsetB, Index,IsLast
KERNEL4x8_L1_L2_I 0,\OffsetA,\OffsetB,\Index,\IsLast,0
.endm
.macro KERNEL4x8_I1_L2_3 OffsetA,OffsetB, Index,IsLast
KERNEL4x8_L1_L2_I 0,\OffsetA,\OffsetB,\Index,\IsLast,1
.endm
.macro KERNEL4x8_L1_L2_I First, OffsetA,OffsetB, Index,IsLast ,Complete
lxv vs8, DISP16(\Index,0+\OffsetA)(AO)
lxv vs9, DISP16(\Index,16+\OffsetA)(AO)
.if \First ==1
xvmuldp vs32, vs0, vs24
xvmuldp vs33, vs1, vs24
xvmuldp vs34, vs2, vs24
xvmuldp vs35, vs3, vs24
.else
xvmaddadp vs32, vs0, vs24
xvmaddadp vs33, vs1, vs24
xvmaddadp vs34, vs2, vs24
xvmaddadp vs35, vs3, vs24
.endif
lxv vs10, DISP16(\Index,32+\OffsetA)(AO)
lxv vs11, DISP16(\Index,48+\OffsetA)(AO)
.if \First ==1
xvmuldp vs40, vs0, vs25
xvmuldp vs41, vs1, vs25
xvmuldp vs42, vs2, vs25
xvmuldp vs43, vs3, vs25
xvmuldp vs48, vs0, vs26
xvmuldp vs49, vs1, vs26
xvmuldp vs50, vs2, vs26
xvmuldp vs51, vs3, vs26
.else
lxv vs28, DISP8(\Index,0 +\OffsetB)(BO)
lxv vs30, DISP8(\Index,16 +\OffsetB)(BO)
xvmaddadp vs40, vs0, vs25
xvmaddadp vs41, vs1, vs25
xvmaddadp vs42, vs2, vs25
xvmaddadp vs43, vs3, vs25
xvmaddadp vs48, vs0, vs26
xvmaddadp vs49, vs1, vs26
xvmaddadp vs50, vs2, vs26
xvmaddadp vs51, vs3, vs26
.endif
xxpermdi vs29, vs28, vs28,2
xxpermdi vs31, vs30, vs30,2
.if \First ==1
xvmuldp vs56, vs0, vs27
xvmuldp vs57, vs1, vs27
xvmuldp vs58, vs2, vs27
xvmuldp vs59, vs3, vs27
.else
xvmaddadp vs56, vs0, vs27
xvmaddadp vs57, vs1, vs27
xvmaddadp vs58, vs2, vs27
xvmaddadp vs59, vs3, vs27
.endif
xvmaddadp vs32, vs8, vs28
xvmaddadp vs33, vs9, vs28
xvmaddadp vs34, vs10, vs28
xvmaddadp vs35, vs11, vs28
.if \Complete==0
lxv vs0, DISP16(\Index,64+\OffsetA)(AO)
lxv vs1, DISP16(\Index,80+\OffsetA)(AO)
.endif
xvmaddadp vs40, vs8, vs29
xvmaddadp vs41, vs9, vs29
xvmaddadp vs42, vs10, vs29
xvmaddadp vs43, vs11, vs29
.if \Complete==0
lxv vs2, DISP16(\Index,96+\OffsetA)(AO)
lxv vs3, DISP16(\Index,112+\OffsetA)(AO)
.endif
xvmaddadp vs48, vs8, vs30
xvmaddadp vs49, vs9, vs30
xvmaddadp vs50, vs10, vs30
xvmaddadp vs51, vs11, vs30
.if \Complete==0
lxv vs24, DISP8(\Index,32 +\OffsetB)(BO)
lxv vs26, DISP8(\Index,48 +\OffsetB)(BO)
.endif
xvmaddadp vs56, vs8, vs31
xvmaddadp vs57, vs9, vs31
xvmaddadp vs58, vs10, vs31
xvmaddadp vs59, vs11, vs31
.if \Complete==0
xxpermdi vs25, vs24, vs24,2
xxpermdi vs27, vs26, vs26,2
.endif
.if \IsLast==1
.if \Complete==1
addi AO, AO, DISP16(\Index,64+\OffsetA)
addi BO, BO, DISP8(\Index,32+\OffsetB)
.else
addi AO, AO, DISP16(\Index,128)
addi BO, BO, DISP8(\Index,64)
.endif
.endif
.endm
.macro KERNEL4x8 First
lxv vs24, 0(BO)
lxv vs26, 16(BO)
xxpermdi vs25, vs24, vs24,2
xxpermdi vs27, vs26, vs26,2
lxv vs0, 0(AO)
lxv vs1, 16(AO)
lxv vs2, 32(AO)
lxv vs3, 48(AO)
addi BO, BO, 32
addi AO, AO, 64
.if \First==1
xvmuldp vs32, vs0, vs24
xvmuldp vs33, vs1, vs24
xvmuldp vs34, vs2, vs24
xvmuldp vs35, vs3, vs24
xvmuldp vs40, vs0, vs25
xvmuldp vs41, vs1, vs25
xvmuldp vs42, vs2, vs25
xvmuldp vs43, vs3, vs25
xvmuldp vs48, vs0, vs26
xvmuldp vs49, vs1, vs26
xvmuldp vs50, vs2, vs26
xvmuldp vs51, vs3, vs26
xvmuldp vs56, vs0, vs27
xvmuldp vs57, vs1, vs27
xvmuldp vs58, vs2, vs27
xvmuldp vs59, vs3, vs27
.else
xvmaddadp vs32, vs0, vs24
xvmaddadp vs33, vs1, vs24
xvmaddadp vs34, vs2, vs24
xvmaddadp vs35, vs3, vs24
xvmaddadp vs40, vs0, vs25
xvmaddadp vs41, vs1, vs25
xvmaddadp vs42, vs2, vs25
xvmaddadp vs43, vs3, vs25
xvmaddadp vs48, vs0, vs26
xvmaddadp vs49, vs1, vs26
xvmaddadp vs50, vs2, vs26
xvmaddadp vs51, vs3, vs26
xvmaddadp vs56, vs0, vs27
xvmaddadp vs57, vs1, vs27
xvmaddadp vs58, vs2, vs27
xvmaddadp vs59, vs3, vs27
.endif
.endm
.macro SAVE4x8
add T2, CO, LDC
add T3, T2, LDC
add T4, T3, LDC
#ifndef TRMMKERNEL
lxv vs0, 0(CO)
lxv vs2, 16(CO)
#endif
xxpermdi vs8, vs40,vs32,1
xxpermdi vs9 ,vs32,vs40,1
#ifndef TRMMKERNEL
lxv vs4, 32(CO)
lxv vs6, 48(CO)
#endif
xxpermdi vs10, vs41,vs33,1
xxpermdi vs11 ,vs33,vs41,1
#ifndef TRMMKERNEL
lxv vs1, 0(T2)
lxv vs3, 16(T2)
#endif
xxpermdi vs12, vs42,vs34,1
xxpermdi vs13 ,vs34,vs42,1
#ifndef TRMMKERNEL
lxv vs5, 32(T2)
lxv vs7, 48(T2)
#endif
xxpermdi vs14, vs43,vs35,1
xxpermdi vs15 ,vs35,vs43,1
#ifndef TRMMKERNEL
xvmaddadp vs0, vs8, alpha_r
xvmaddadp vs1, vs9, alpha_r
xvmaddadp vs2, vs10, alpha_r
xvmaddadp vs3, vs11, alpha_r
xvmaddadp vs4, vs12, alpha_r
xvmaddadp vs5, vs13, alpha_r
xvmaddadp vs6, vs14, alpha_r
xvmaddadp vs7, vs15, alpha_r
#else
xvmuldp vs0, vs8, alpha_r
xvmuldp vs1, vs9, alpha_r
xvmuldp vs2, vs10, alpha_r
xvmuldp vs3, vs11, alpha_r
xvmuldp vs4, vs12, alpha_r
xvmuldp vs5, vs13, alpha_r
xvmuldp vs6, vs14, alpha_r
xvmuldp vs7, vs15, alpha_r
#endif
stxv vs0, 0(CO)
stxv vs2, 16(CO)
stxv vs4, 32(CO)
stxv vs6, 48(CO)
stxv vs1, 0(T2)
stxv vs3, 16(T2)
stxv vs5, 32(T2)
stxv vs7, 48(T2)
xxpermdi vs8, vs56,vs48,1
xxpermdi vs9 ,vs48,vs56,1
#ifndef TRMMKERNEL
lxv vs0, 0(T3)
lxv vs2, 16(T3)
#endif
xxpermdi vs10, vs57,vs49,1
xxpermdi vs11 ,vs49,vs57,1
#ifndef TRMMKERNEL
lxv vs4, 32(T3)
lxv vs6, 48(T3)
#endif
xxpermdi vs12, vs58,vs50,1
xxpermdi vs13 ,vs50,vs58,1
#ifndef TRMMKERNEL
lxv vs1, 0(T4)
lxv vs3, 16(T4)
#endif
xxpermdi vs14, vs59,vs51,1
xxpermdi vs15 ,vs51,vs59,1
#ifndef TRMMKERNEL
lxv vs5, 32(T4)
lxv vs7, 48(T4)
xvmaddadp vs0, vs8, alpha_r
xvmaddadp vs1, vs9, alpha_r
xvmaddadp vs2, vs10, alpha_r
xvmaddadp vs3, vs11, alpha_r
xvmaddadp vs4, vs12, alpha_r
xvmaddadp vs5, vs13, alpha_r
xvmaddadp vs6, vs14, alpha_r
xvmaddadp vs7, vs15, alpha_r
#else
xvmuldp vs0, vs8, alpha_r
xvmuldp vs1, vs9, alpha_r
xvmuldp vs2, vs10, alpha_r
xvmuldp vs3, vs11, alpha_r
xvmuldp vs4, vs12, alpha_r
xvmuldp vs5, vs13, alpha_r
xvmuldp vs6, vs14, alpha_r
xvmuldp vs7, vs15, alpha_r
#endif
stxv vs0, 0(T3)
stxv vs2, 16(T3)
stxv vs4, 32(T3)
stxv vs6, 48(T3)
stxv vs1, 0(T4)
stxv vs3, 16(T4)
stxv vs5, 32(T4)
stxv vs7, 48(T4)
addi CO, CO, 64
.endm
/*********************************************************************
* Macros for N=4, M=4 *
*********************************************************************/
.macro LOAD4x4_1
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvdsx vs24, 0, BO
lxvdsx vs25, o8, BO
lxvdsx vs26, o16, BO
lxvdsx vs27, o24, BO
addi AO, AO, 32
addi BO, BO, 32
.endm
.macro KERNEL4x4_I1
lxvd2x vs8, 0, AO
lxvd2x vs9, o16, AO
lxvdsx vs28, 0, BO
lxvdsx vs29, o8, BO
lxvdsx vs30, o16, BO
lxvdsx vs31, o24, BO
addi AO, AO, 32
addi BO, BO, 32
xvmuldp vs32, vs0, vs24
xvmuldp vs33, vs1, vs24
xvmuldp vs40, vs0, vs25
xvmuldp vs41, vs1, vs25
xvmuldp vs48, vs0, vs26
xvmuldp vs49, vs1, vs26
xvmuldp vs56, vs0, vs27
xvmuldp vs57, vs1, vs27
.endm
.macro KERNEL4x4_1
lxvd2x vs8, 0, AO
lxvd2x vs9, o16, AO
lxvdsx vs28, 0, BO
lxvdsx vs29, o8, BO
lxvdsx vs30, o16, BO
lxvdsx vs31, o24, BO
addi AO, AO, 32
addi BO, BO, 32
xvmaddadp vs32, vs0, vs24
xvmaddadp vs33, vs1, vs24
xvmaddadp vs40, vs0, vs25
xvmaddadp vs41, vs1, vs25
xvmaddadp vs48, vs0, vs26
xvmaddadp vs49, vs1, vs26
xvmaddadp vs56, vs0, vs27
xvmaddadp vs57, vs1, vs27
.endm
.macro KERNEL4x4_2
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvdsx vs24, 0, BO
lxvdsx vs25, o8, BO
lxvdsx vs26, o16, BO
lxvdsx vs27, o24, BO
addi AO, AO, 32
addi BO, BO, 32
xvmaddadp vs32, vs8, vs28
xvmaddadp vs33, vs9, vs28
xvmaddadp vs40, vs8, vs29
xvmaddadp vs41, vs9, vs29
xvmaddadp vs48, vs8, vs30
xvmaddadp vs49, vs9, vs30
xvmaddadp vs56, vs8, vs31
xvmaddadp vs57, vs9, vs31
.endm
.macro KERNEL4x4_E2
xvmaddadp vs32, vs8, vs28
xvmaddadp vs33, vs9, vs28
xvmaddadp vs40, vs8, vs29
xvmaddadp vs41, vs9, vs29
xvmaddadp vs48, vs8, vs30
xvmaddadp vs49, vs9, vs30
xvmaddadp vs56, vs8, vs31
xvmaddadp vs57, vs9, vs31
.endm
.macro KERNEL4x4_SUBI1
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvdsx vs24, 0, BO
lxvdsx vs25, o8, BO
lxvdsx vs26, o16, BO
lxvdsx vs27, o24, BO
addi AO, AO, 32
addi BO, BO, 32
xvmuldp vs32, vs0, vs24
xvmuldp vs33, vs1, vs24
xvmuldp vs40, vs0, vs25
xvmuldp vs41, vs1, vs25
xvmuldp vs48, vs0, vs26
xvmuldp vs49, vs1, vs26
xvmuldp vs56, vs0, vs27
xvmuldp vs57, vs1, vs27
.endm
.macro KERNEL4x4_SUB1
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvdsx vs24, 0, BO
lxvdsx vs25, o8, BO
lxvdsx vs26, o16, BO
lxvdsx vs27, o24, BO
addi AO, AO, 32
addi BO, BO, 32
xvmaddadp vs32, vs0, vs24
xvmaddadp vs33, vs1, vs24
xvmaddadp vs40, vs0, vs25
xvmaddadp vs41, vs1, vs25
xvmaddadp vs48, vs0, vs26
xvmaddadp vs49, vs1, vs26
xvmaddadp vs56, vs0, vs27
xvmaddadp vs57, vs1, vs27
.endm
.macro SAVE4x4
mr T1, CO
#ifndef TRMMKERNEL
lxvd2x vs0, 0, T1
lxvd2x vs1, o16, T1
#endif
#ifndef TRMMKERNEL
xvmaddadp vs0, vs32, alpha_r
xvmaddadp vs1, vs33, alpha_r
#else
xvmuldp vs0, vs32, alpha_r
xvmuldp vs1, vs33, alpha_r
#endif
stxvd2x vs0, 0, T1
stxvd2x vs1, o16, T1
add T1, T1, LDC
#ifndef TRMMKERNEL
lxvd2x vs8, 0, T1
lxvd2x vs9, o16, T1
#endif
#ifndef TRMMKERNEL
xvmaddadp vs8, vs40, alpha_r
xvmaddadp vs9, vs41, alpha_r
#else
xvmuldp vs8, vs40, alpha_r
xvmuldp vs9, vs41, alpha_r
#endif
stxvd2x vs8, 0, T1
stxvd2x vs9, o16, T1
add T1, T1, LDC
#ifndef TRMMKERNEL
lxvd2x vs0, 0, T1
lxvd2x vs1, o16, T1
#endif
#ifndef TRMMKERNEL
xvmaddadp vs0, vs48, alpha_r
xvmaddadp vs1, vs49, alpha_r
#else
xvmuldp vs0, vs48, alpha_r
xvmuldp vs1, vs49, alpha_r
#endif
stxvd2x vs0, 0, T1
stxvd2x vs1, o16, T1
add T1, T1, LDC
#ifndef TRMMKERNEL
lxvd2x vs8, 0, T1
lxvd2x vs9, o16, T1
#endif
#ifndef TRMMKERNEL
xvmaddadp vs8, vs56, alpha_r
xvmaddadp vs9, vs57, alpha_r
#else
xvmuldp vs8, vs56, alpha_r
xvmuldp vs9, vs57, alpha_r
#endif
stxvd2x vs8, 0, T1
stxvd2x vs9, o16, T1
addi CO, CO, 32
.endm
/*********************************************************************
* Macros for N=4, M=2 *
*********************************************************************/
.macro LOAD4x2_1
lxvd2x vs0, 0, AO
lxvdsx vs24, 0, BO
lxvdsx vs25, o8, BO
lxvdsx vs26, o16, BO
lxvdsx vs27, o24, BO
addi AO, AO, 16
addi BO, BO, 32
.endm
.macro KERNEL4x2_I1
lxvd2x vs8, 0, AO
lxvdsx vs28, 0, BO
lxvdsx vs29, o8, BO
lxvdsx vs30, o16, BO
lxvdsx vs31, o24, BO
addi AO, AO, 16
addi BO, BO, 32
xvmuldp vs32, vs0, vs24
xvmuldp vs40, vs0, vs25
xvmuldp vs48, vs0, vs26
xvmuldp vs56, vs0, vs27
.endm
.macro KERNEL4x2_1
lxvd2x vs8, 0, AO
lxvdsx vs28, 0, BO
lxvdsx vs29, o8, BO
lxvdsx vs30, o16, BO
lxvdsx vs31, o24, BO
addi AO, AO, 16
addi BO, BO, 32
xvmaddadp vs32, vs0, vs24
xvmaddadp vs40, vs0, vs25
xvmaddadp vs48, vs0, vs26
xvmaddadp vs56, vs0, vs27
.endm
.macro KERNEL4x2_2
lxvd2x vs0, 0, AO
lxvdsx vs24, 0, BO
lxvdsx vs25, o8, BO
lxvdsx vs26, o16, BO
lxvdsx vs27, o24, BO
addi AO, AO, 16
addi BO, BO, 32
xvmaddadp vs32, vs8, vs28
xvmaddadp vs40, vs8, vs29
xvmaddadp vs48, vs8, vs30
xvmaddadp vs56, vs8, vs31
.endm
.macro KERNEL4x2_E2
xvmaddadp vs32, vs8, vs28
xvmaddadp vs40, vs8, vs29
xvmaddadp vs48, vs8, vs30
xvmaddadp vs56, vs8, vs31
.endm
.macro KERNEL4x2_SUBI1
lxvd2x vs0, 0, AO
lxvdsx vs24, 0, BO
lxvdsx vs25, o8, BO
lxvdsx vs26, o16, BO
lxvdsx vs27, o24, BO
addi AO, AO, 16
addi BO, BO, 32
xvmuldp vs32, vs0, vs24
xvmuldp vs40, vs0, vs25
xvmuldp vs48, vs0, vs26
xvmuldp vs56, vs0, vs27
.endm
.macro KERNEL4x2_SUB1
lxvd2x vs0, 0, AO
lxvdsx vs24, 0, BO
lxvdsx vs25, o8, BO
lxvdsx vs26, o16, BO
lxvdsx vs27, o24, BO
addi AO, AO, 16
addi BO, BO, 32
xvmaddadp vs32, vs0, vs24
xvmaddadp vs40, vs0, vs25
xvmaddadp vs48, vs0, vs26
xvmaddadp vs56, vs0, vs27
.endm
.macro SAVE4x2
mr T1, CO
#ifndef TRMMKERNEL
lxvd2x vs0, 0, T1
#endif
#ifndef TRMMKERNEL
xvmaddadp vs0, vs32, alpha_r
#else
xvmuldp vs0, vs32, alpha_r
#endif
stxvd2x vs0, 0, T1
add T1, T1, LDC
#ifndef TRMMKERNEL
lxvd2x vs8, 0, T1
#endif
#ifndef TRMMKERNEL
xvmaddadp vs8, vs40, alpha_r
#else
xvmuldp vs8, vs40, alpha_r
#endif
stxvd2x vs8, 0, T1
add T1, T1, LDC
#ifndef TRMMKERNEL
lxvd2x vs0, 0, T1
#endif
#ifndef TRMMKERNEL
xvmaddadp vs0, vs48, alpha_r
#else
xvmuldp vs0, vs48, alpha_r
#endif
stxvd2x vs0, 0, T1
add T1, T1, LDC
#ifndef TRMMKERNEL
lxvd2x vs8, 0, T1
#endif
#ifndef TRMMKERNEL
xvmaddadp vs8, vs56, alpha_r
#else
xvmuldp vs8, vs56, alpha_r
#endif
stxvd2x vs8, 0, T1
addi CO, CO, 16
.endm
/*********************************************************************
* Macros for N=4, M=1 *
*********************************************************************/
.macro LOAD4x1_1
lxsdx vs0, 0, AO
lxsdx vs24, 0, BO
lxsdx vs25, o8, BO
lxsdx vs26, o16, BO
lxsdx vs27, o24, BO
addi AO, AO, 8
addi BO, BO, 32
.endm
.macro KERNEL4x1_I1
lxsdx vs8, 0, AO
lxsdx vs28, 0, BO
lxsdx vs29, o8, BO
lxsdx vs30, o16, BO
lxsdx vs31, o24, BO
addi AO, AO, 8
addi BO, BO, 32
xsmuldp vs32, vs0, vs24
xsmuldp vs40, vs0, vs25
xsmuldp vs48, vs0, vs26
xsmuldp vs56, vs0, vs27
.endm
.macro KERNEL4x1_1
lxsdx vs8, 0, AO
lxsdx vs28, 0, BO
lxsdx vs29, o8, BO
lxsdx vs30, o16, BO
lxsdx vs31, o24, BO
addi AO, AO, 8
addi BO, BO, 32
xsmaddadp vs32, vs0, vs24
xsmaddadp vs40, vs0, vs25
xsmaddadp vs48, vs0, vs26
xsmaddadp vs56, vs0, vs27
.endm
.macro KERNEL4x1_2
lxsdx vs0, 0, AO
lxsdx vs24, 0, BO
lxsdx vs25, o8, BO
lxsdx vs26, o16, BO
lxsdx vs27, o24, BO
addi AO, AO, 8
addi BO, BO, 32
xsmaddadp vs32, vs8, vs28
xsmaddadp vs40, vs8, vs29
xsmaddadp vs48, vs8, vs30
xsmaddadp vs56, vs8, vs31
.endm
.macro KERNEL4x1_E2
xsmaddadp vs32, vs8, vs28
xsmaddadp vs40, vs8, vs29
xsmaddadp vs48, vs8, vs30
xsmaddadp vs56, vs8, vs31
.endm
.macro KERNEL4x1_SUBI1
lxsdx vs0, 0, AO
lxsdx vs24, 0, BO
lxsdx vs25, o8, BO
lxsdx vs26, o16, BO
lxsdx vs27, o24, BO
addi AO, AO, 8
addi BO, BO, 32
xsmuldp vs32, vs0, vs24
xsmuldp vs40, vs0, vs25
xsmuldp vs48, vs0, vs26
xsmuldp vs56, vs0, vs27
.endm
.macro KERNEL4x1_SUB1
lxsdx vs0, 0, AO
lxsdx vs24, 0, BO
lxsdx vs25, o8, BO
lxsdx vs26, o16, BO
lxsdx vs27, o24, BO
addi AO, AO, 8
addi BO, BO, 32
xsmaddadp vs32, vs0, vs24
xsmaddadp vs40, vs0, vs25
xsmaddadp vs48, vs0, vs26
xsmaddadp vs56, vs0, vs27
.endm
.macro SAVE4x1
mr T1, CO
#ifndef TRMMKERNEL
lxsdx vs0, 0, T1
#endif
#ifndef TRMMKERNEL
xsmaddadp vs0, vs32, alpha_r
#else
xsmuldp vs0, vs32, alpha_r
#endif
stxsdx vs0, 0, T1
add T1, T1, LDC
#ifndef TRMMKERNEL
lxsdx vs8, 0, T1
#endif
#ifndef TRMMKERNEL
xsmaddadp vs8, vs40, alpha_r
#else
xsmuldp vs8, vs40, alpha_r
#endif
stxsdx vs8, 0, T1
add T1, T1, LDC
#ifndef TRMMKERNEL
lxsdx vs0, 0, T1
#endif
#ifndef TRMMKERNEL
xsmaddadp vs0, vs48, alpha_r
#else
xsmuldp vs0, vs48, alpha_r
#endif
stxsdx vs0, 0, T1
add T1, T1, LDC
#ifndef TRMMKERNEL
lxsdx vs8, 0, T1
#endif
#ifndef TRMMKERNEL
xsmaddadp vs8, vs56, alpha_r
#else
xsmuldp vs8, vs56, alpha_r
#endif
stxsdx vs8, 0, T1
addi CO, CO, 8
.endm
/*********************************************************************
* Macros for N=2, M=16 *
*********************************************************************/
.macro LOAD2x16_1
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvd2x vs2, o32, AO
lxvd2x vs3, o48, AO
lxvdsx vs24, 0, BO
lxvdsx vs25, o8, BO
addi AO, AO, 64
addi BO, BO, 16
lxvd2x vs4, 0, AO
lxvd2x vs5, o16, AO
lxvd2x vs6, o32, AO
lxvd2x vs7, o48, AO
addi AO, AO, 64
.endm
.macro KERNEL2x16_I1
lxvd2x vs8, 0, AO
lxvd2x vs9, o16, AO
lxvd2x vs10, o32, AO
lxvd2x vs11, o48, AO
lxvdsx vs28, 0, BO
lxvdsx vs29, o8, BO
addi AO, AO, 64
addi BO, BO, 16
lxvd2x vs12, 0, AO
lxvd2x vs13, o16, AO
lxvd2x vs14, o32, AO
lxvd2x vs15, o48, AO
addi AO, AO, 64
xvmuldp vs32, vs0, vs24
xvmuldp vs33, vs1, vs24
xvmuldp vs34, vs2, vs24
xvmuldp vs35, vs3, vs24
xvmuldp vs36, vs4, vs24
xvmuldp vs37, vs5, vs24
xvmuldp vs38, vs6, vs24
xvmuldp vs39, vs7, vs24
xvmuldp vs40, vs0, vs25
xvmuldp vs41, vs1, vs25
xvmuldp vs42, vs2, vs25
xvmuldp vs43, vs3, vs25
xvmuldp vs44, vs4, vs25
xvmuldp vs45, vs5, vs25
xvmuldp vs46, vs6, vs25
xvmuldp vs47, vs7, vs25
.endm
.macro KERNEL2x16_1
lxvd2x vs8, 0, AO
lxvd2x vs9, o16, AO
lxvd2x vs10, o32, AO
lxvd2x vs11, o48, AO
lxvdsx vs28, 0, BO
lxvdsx vs29, o8, BO
addi AO, AO, 64
addi BO, BO, 16
lxvd2x vs12, 0, AO
lxvd2x vs13, o16, AO
lxvd2x vs14, o32, AO
lxvd2x vs15, o48, AO
addi AO, AO, 64
xvmaddadp vs32, vs0, vs24
xvmaddadp vs33, vs1, vs24
xvmaddadp vs34, vs2, vs24
xvmaddadp vs35, vs3, vs24
xvmaddadp vs36, vs4, vs24
xvmaddadp vs37, vs5, vs24
xvmaddadp vs38, vs6, vs24
xvmaddadp vs39, vs7, vs24
xvmaddadp vs40, vs0, vs25
xvmaddadp vs41, vs1, vs25
xvmaddadp vs42, vs2, vs25
xvmaddadp vs43, vs3, vs25
xvmaddadp vs44, vs4, vs25
xvmaddadp vs45, vs5, vs25
xvmaddadp vs46, vs6, vs25
xvmaddadp vs47, vs7, vs25
.endm
.macro KERNEL2x16_2
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvd2x vs2, o32, AO
lxvd2x vs3, o48, AO
lxvdsx vs24, 0, BO
lxvdsx vs25, o8, BO
addi AO, AO, 64
addi BO, BO, 16
lxvd2x vs4, 0, AO
lxvd2x vs5, o16, AO
lxvd2x vs6, o32, AO
lxvd2x vs7, o48, AO
addi AO, AO, 64
xvmaddadp vs32, vs8, vs28
xvmaddadp vs33, vs9, vs28
xvmaddadp vs34, vs10, vs28
xvmaddadp vs35, vs11, vs28
xvmaddadp vs36, vs12, vs28
xvmaddadp vs37, vs13, vs28
xvmaddadp vs38, vs14, vs28
xvmaddadp vs39, vs15, vs28
xvmaddadp vs40, vs8, vs29
xvmaddadp vs41, vs9, vs29
xvmaddadp vs42, vs10, vs29
xvmaddadp vs43, vs11, vs29
xvmaddadp vs44, vs12, vs29
xvmaddadp vs45, vs13, vs29
xvmaddadp vs46, vs14, vs29
xvmaddadp vs47, vs15, vs29
.endm
.macro KERNEL2x16_E2
xvmaddadp vs32, vs8, vs28
xvmaddadp vs33, vs9, vs28
xvmaddadp vs34, vs10, vs28
xvmaddadp vs35, vs11, vs28
xvmaddadp vs36, vs12, vs28
xvmaddadp vs37, vs13, vs28
xvmaddadp vs38, vs14, vs28
xvmaddadp vs39, vs15, vs28
xvmaddadp vs40, vs8, vs29
xvmaddadp vs41, vs9, vs29
xvmaddadp vs42, vs10, vs29
xvmaddadp vs43, vs11, vs29
xvmaddadp vs44, vs12, vs29
xvmaddadp vs45, vs13, vs29
xvmaddadp vs46, vs14, vs29
xvmaddadp vs47, vs15, vs29
.endm
.macro KERNEL2x16_SUBI1
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvd2x vs2, o32, AO
lxvd2x vs3, o48, AO
lxvdsx vs24, 0, BO
lxvdsx vs25, o8, BO
addi AO, AO, 64
addi BO, BO, 16
lxvd2x vs4, 0, AO
lxvd2x vs5, o16, AO
lxvd2x vs6, o32, AO
lxvd2x vs7, o48, AO
addi AO, AO, 64
xvmuldp vs32, vs0, vs24
xvmuldp vs33, vs1, vs24
xvmuldp vs34, vs2, vs24
xvmuldp vs35, vs3, vs24
xvmuldp vs36, vs4, vs24
xvmuldp vs37, vs5, vs24
xvmuldp vs38, vs6, vs24
xvmuldp vs39, vs7, vs24
xvmuldp vs40, vs0, vs25
xvmuldp vs41, vs1, vs25
xvmuldp vs42, vs2, vs25
xvmuldp vs43, vs3, vs25
xvmuldp vs44, vs4, vs25
xvmuldp vs45, vs5, vs25
xvmuldp vs46, vs6, vs25
xvmuldp vs47, vs7, vs25
.endm
.macro KERNEL2x16_SUB1
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvd2x vs2, o32, AO
lxvd2x vs3, o48, AO
lxvdsx vs24, 0, BO
lxvdsx vs25, o8, BO
addi AO, AO, 64
addi BO, BO, 16
lxvd2x vs4, 0, AO
lxvd2x vs5, o16, AO
lxvd2x vs6, o32, AO
lxvd2x vs7, o48, AO
addi AO, AO, 64
xvmaddadp vs32, vs0, vs24
xvmaddadp vs33, vs1, vs24
xvmaddadp vs34, vs2, vs24
xvmaddadp vs35, vs3, vs24
xvmaddadp vs36, vs4, vs24
xvmaddadp vs37, vs5, vs24
xvmaddadp vs38, vs6, vs24
xvmaddadp vs39, vs7, vs24
xvmaddadp vs40, vs0, vs25
xvmaddadp vs41, vs1, vs25
xvmaddadp vs42, vs2, vs25
xvmaddadp vs43, vs3, vs25
xvmaddadp vs44, vs4, vs25
xvmaddadp vs45, vs5, vs25
xvmaddadp vs46, vs6, vs25
xvmaddadp vs47, vs7, vs25
.endm
.macro SAVE2x16
mr T1, CO
addi T2, T1, 64
#ifndef TRMMKERNEL
lxvd2x vs0, 0, T1
lxvd2x vs1, o16, T1
lxvd2x vs2, o32, T1
lxvd2x vs3, o48, T1
lxvd2x vs4, 0, T2
lxvd2x vs5, o16, T2
lxvd2x vs6, o32, T2
lxvd2x vs7, o48, T2
#endif
#ifndef TRMMKERNEL
xvmaddadp vs0, vs32, alpha_r
xvmaddadp vs1, vs33, alpha_r
xvmaddadp vs2, vs34, alpha_r
xvmaddadp vs3, vs35, alpha_r
xvmaddadp vs4, vs36, alpha_r
xvmaddadp vs5, vs37, alpha_r
xvmaddadp vs6, vs38, alpha_r
xvmaddadp vs7, vs39, alpha_r
#else
xvmuldp vs0, vs32, alpha_r
xvmuldp vs1, vs33, alpha_r
xvmuldp vs2, vs34, alpha_r
xvmuldp vs3, vs35, alpha_r
xvmuldp vs4, vs36, alpha_r
xvmuldp vs5, vs37, alpha_r
xvmuldp vs6, vs38, alpha_r
xvmuldp vs7, vs39, alpha_r
#endif
stxvd2x vs0, 0, T1
stxvd2x vs1, o16, T1
stxvd2x vs2, o32, T1
stxvd2x vs3, o48, T1
stxvd2x vs4, 0, T2
stxvd2x vs5, o16, T2
stxvd2x vs6, o32, T2
stxvd2x vs7, o48, T2
add T1, T1, LDC
add T2, T2, LDC
#ifndef TRMMKERNEL
lxvd2x vs8, 0, T1
lxvd2x vs9, o16, T1
lxvd2x vs10, o32, T1
lxvd2x vs11, o48, T1
lxvd2x vs12, 0, T2
lxvd2x vs13, o16, T2
lxvd2x vs14, o32, T2
lxvd2x vs15, o48, T2
#endif
#ifndef TRMMKERNEL
xvmaddadp vs8, vs40, alpha_r
xvmaddadp vs9, vs41, alpha_r
xvmaddadp vs10, vs42, alpha_r
xvmaddadp vs11, vs43, alpha_r
xvmaddadp vs12, vs44, alpha_r
xvmaddadp vs13, vs45, alpha_r
xvmaddadp vs14, vs46, alpha_r
xvmaddadp vs15, vs47, alpha_r
#else
xvmuldp vs8, vs40, alpha_r
xvmuldp vs9, vs41, alpha_r
xvmuldp vs10, vs42, alpha_r
xvmuldp vs11, vs43, alpha_r
xvmuldp vs12, vs44, alpha_r
xvmuldp vs13, vs45, alpha_r
xvmuldp vs14, vs46, alpha_r
xvmuldp vs15, vs47, alpha_r
#endif
stxvd2x vs8, 0, T1
stxvd2x vs9, o16, T1
stxvd2x vs10, o32, T1
stxvd2x vs11, o48, T1
stxvd2x vs12, 0, T2
stxvd2x vs13, o16, T2
stxvd2x vs14, o32, T2
stxvd2x vs15, o48, T2
addi CO, CO, 128
.endm
/*********************************************************************
* Macros for N=4, M=8 *
*********************************************************************/
.macro LOAD2x8_1
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvd2x vs2, o32, AO
lxvd2x vs3, o48, AO
lxvdsx vs24, 0, BO
lxvdsx vs25, o8, BO
addi AO, AO, 64
addi BO, BO, 16
.endm
.macro KERNEL2x8_I1
lxvd2x vs8, 0, AO
lxvd2x vs9, o16, AO
lxvd2x vs10, o32, AO
lxvd2x vs11, o48, AO
lxvdsx vs28, 0, BO
lxvdsx vs29, o8, BO
addi AO, AO, 64
addi BO, BO, 16
xvmuldp vs32, vs0, vs24
xvmuldp vs33, vs1, vs24
xvmuldp vs34, vs2, vs24
xvmuldp vs35, vs3, vs24
xvmuldp vs40, vs0, vs25
xvmuldp vs41, vs1, vs25
xvmuldp vs42, vs2, vs25
xvmuldp vs43, vs3, vs25
.endm
.macro KERNEL2x8_1
lxvd2x vs8, 0, AO
lxvd2x vs9, o16, AO
lxvd2x vs10, o32, AO
lxvd2x vs11, o48, AO
lxvdsx vs28, 0, BO
lxvdsx vs29, o8, BO
addi AO, AO, 64
addi BO, BO, 16
xvmaddadp vs32, vs0, vs24
xvmaddadp vs33, vs1, vs24
xvmaddadp vs34, vs2, vs24
xvmaddadp vs35, vs3, vs24
xvmaddadp vs40, vs0, vs25
xvmaddadp vs41, vs1, vs25
xvmaddadp vs42, vs2, vs25
xvmaddadp vs43, vs3, vs25
.endm
.macro KERNEL2x8_2
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvd2x vs2, o32, AO
lxvd2x vs3, o48, AO
lxvdsx vs24, 0, BO
lxvdsx vs25, o8, BO
addi AO, AO, 64
addi BO, BO, 16
xvmaddadp vs32, vs8, vs28
xvmaddadp vs33, vs9, vs28
xvmaddadp vs34, vs10, vs28
xvmaddadp vs35, vs11, vs28
xvmaddadp vs40, vs8, vs29
xvmaddadp vs41, vs9, vs29
xvmaddadp vs42, vs10, vs29
xvmaddadp vs43, vs11, vs29
.endm
.macro KERNEL2x8_E2
xvmaddadp vs32, vs8, vs28
xvmaddadp vs33, vs9, vs28
xvmaddadp vs34, vs10, vs28
xvmaddadp vs35, vs11, vs28
xvmaddadp vs40, vs8, vs29
xvmaddadp vs41, vs9, vs29
xvmaddadp vs42, vs10, vs29
xvmaddadp vs43, vs11, vs29
.endm
.macro KERNEL2x8_SUBI1
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvd2x vs2, o32, AO
lxvd2x vs3, o48, AO
lxvdsx vs24, 0, BO
lxvdsx vs25, o8, BO
addi AO, AO, 64
addi BO, BO, 16
xvmuldp vs32, vs0, vs24
xvmuldp vs33, vs1, vs24
xvmuldp vs34, vs2, vs24
xvmuldp vs35, vs3, vs24
xvmuldp vs40, vs0, vs25
xvmuldp vs41, vs1, vs25
xvmuldp vs42, vs2, vs25
xvmuldp vs43, vs3, vs25
.endm
.macro KERNEL2x8_SUB1
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvd2x vs2, o32, AO
lxvd2x vs3, o48, AO
lxvdsx vs24, 0, BO
lxvdsx vs25, o8, BO
addi AO, AO, 64
addi BO, BO, 16
xvmaddadp vs32, vs0, vs24
xvmaddadp vs33, vs1, vs24
xvmaddadp vs34, vs2, vs24
xvmaddadp vs35, vs3, vs24
xvmaddadp vs40, vs0, vs25
xvmaddadp vs41, vs1, vs25
xvmaddadp vs42, vs2, vs25
xvmaddadp vs43, vs3, vs25
.endm
.macro SAVE2x8
mr T1, CO
#ifndef TRMMKERNEL
lxvd2x vs0, 0, T1
lxvd2x vs1, o16, T1
lxvd2x vs2, o32, T1
lxvd2x vs3, o48, T1
#endif
#ifndef TRMMKERNEL
xvmaddadp vs0, vs32, alpha_r
xvmaddadp vs1, vs33, alpha_r
xvmaddadp vs2, vs34, alpha_r
xvmaddadp vs3, vs35, alpha_r
#else
xvmuldp vs0, vs32, alpha_r
xvmuldp vs1, vs33, alpha_r
xvmuldp vs2, vs34, alpha_r
xvmuldp vs3, vs35, alpha_r
#endif
stxvd2x vs0, 0, T1
stxvd2x vs1, o16, T1
stxvd2x vs2, o32, T1
stxvd2x vs3, o48, T1
add T1, T1, LDC
#ifndef TRMMKERNEL
lxvd2x vs8, 0, T1
lxvd2x vs9, o16, T1
lxvd2x vs10, o32, T1
lxvd2x vs11, o48, T1
#endif
#ifndef TRMMKERNEL
xvmaddadp vs8, vs40, alpha_r
xvmaddadp vs9, vs41, alpha_r
xvmaddadp vs10, vs42, alpha_r
xvmaddadp vs11, vs43, alpha_r
#else
xvmuldp vs8, vs40, alpha_r
xvmuldp vs9, vs41, alpha_r
xvmuldp vs10, vs42, alpha_r
xvmuldp vs11, vs43, alpha_r
#endif
stxvd2x vs8, 0, T1
stxvd2x vs9, o16, T1
stxvd2x vs10, o32, T1
stxvd2x vs11, o48, T1
addi CO, CO, 64
.endm
/*********************************************************************
* Macros for N=2, M=4 *
*********************************************************************/
.macro LOAD2x4_1
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvdsx vs24, 0, BO
lxvdsx vs25, o8, BO
addi AO, AO, 32
addi BO, BO, 16
.endm
.macro KERNEL2x4_I1
lxvd2x vs8, 0, AO
lxvd2x vs9, o16, AO
lxvdsx vs28, 0, BO
lxvdsx vs29, o8, BO
addi AO, AO, 32
addi BO, BO, 16
xvmuldp vs32, vs0, vs24
xvmuldp vs33, vs1, vs24
xvmuldp vs40, vs0, vs25
xvmuldp vs41, vs1, vs25
.endm
.macro KERNEL2x4_1
lxvd2x vs8, 0, AO
lxvd2x vs9, o16, AO
lxvdsx vs28, 0, BO
lxvdsx vs29, o8, BO
addi AO, AO, 32
addi BO, BO, 16
xvmaddadp vs32, vs0, vs24
xvmaddadp vs33, vs1, vs24
xvmaddadp vs40, vs0, vs25
xvmaddadp vs41, vs1, vs25
.endm
.macro KERNEL2x4_2
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvdsx vs24, 0, BO
lxvdsx vs25, o8, BO
addi AO, AO, 32
addi BO, BO, 16
xvmaddadp vs32, vs8, vs28
xvmaddadp vs33, vs9, vs28
xvmaddadp vs40, vs8, vs29
xvmaddadp vs41, vs9, vs29
.endm
.macro KERNEL2x4_E2
xvmaddadp vs32, vs8, vs28
xvmaddadp vs33, vs9, vs28
xvmaddadp vs40, vs8, vs29
xvmaddadp vs41, vs9, vs29
.endm
.macro KERNEL2x4_SUBI1
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvdsx vs24, 0, BO
lxvdsx vs25, o8, BO
addi AO, AO, 32
addi BO, BO, 16
xvmuldp vs32, vs0, vs24
xvmuldp vs33, vs1, vs24
xvmuldp vs40, vs0, vs25
xvmuldp vs41, vs1, vs25
.endm
.macro KERNEL2x4_SUB1
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvdsx vs24, 0, BO
lxvdsx vs25, o8, BO
addi AO, AO, 32
addi BO, BO, 16
xvmaddadp vs32, vs0, vs24
xvmaddadp vs33, vs1, vs24
xvmaddadp vs40, vs0, vs25
xvmaddadp vs41, vs1, vs25
.endm
.macro SAVE2x4
mr T1, CO
#ifndef TRMMKERNEL
lxvd2x vs0, 0, T1
lxvd2x vs1, o16, T1
#endif
#ifndef TRMMKERNEL
xvmaddadp vs0, vs32, alpha_r
xvmaddadp vs1, vs33, alpha_r
#else
xvmuldp vs0, vs32, alpha_r
xvmuldp vs1, vs33, alpha_r
#endif
stxvd2x vs0, 0, T1
stxvd2x vs1, o16, T1
add T1, T1, LDC
#ifndef TRMMKERNEL
lxvd2x vs8, 0, T1
lxvd2x vs9, o16, T1
#endif
#ifndef TRMMKERNEL
xvmaddadp vs8, vs40, alpha_r
xvmaddadp vs9, vs41, alpha_r
#else
xvmuldp vs8, vs40, alpha_r
xvmuldp vs9, vs41, alpha_r
#endif
stxvd2x vs8, 0, T1
stxvd2x vs9, o16, T1
addi CO, CO, 32
.endm
/*********************************************************************
* Macros for N=2, M=2 *
*********************************************************************/
.macro LOAD2x2_1
lxvd2x vs0, 0, AO
lxvdsx vs24, 0, BO
lxvdsx vs25, o8, BO
addi AO, AO, 16
addi BO, BO, 16
.endm
.macro KERNEL2x2_I1
lxvd2x vs8, 0, AO
lxvdsx vs28, 0, BO
lxvdsx vs29, o8, BO
addi AO, AO, 16
addi BO, BO, 16
xvmuldp vs32, vs0, vs24
xvmuldp vs40, vs0, vs25
.endm
.macro KERNEL2x2_1
lxvd2x vs8, 0, AO
lxvdsx vs28, 0, BO
lxvdsx vs29, o8, BO
addi AO, AO, 16
addi BO, BO, 16
xvmaddadp vs32, vs0, vs24
xvmaddadp vs40, vs0, vs25
.endm
.macro KERNEL2x2_2
lxvd2x vs0, 0, AO
lxvdsx vs24, 0, BO
lxvdsx vs25, o8, BO
addi AO, AO, 16
addi BO, BO, 16
xvmaddadp vs32, vs8, vs28
xvmaddadp vs40, vs8, vs29
.endm
.macro KERNEL2x2_E2
xvmaddadp vs32, vs8, vs28
xvmaddadp vs40, vs8, vs29
.endm
.macro KERNEL2x2_SUBI1
lxvd2x vs0, 0, AO
lxvdsx vs24, 0, BO
lxvdsx vs25, o8, BO
addi AO, AO, 16
addi BO, BO, 16
xvmuldp vs32, vs0, vs24
xvmuldp vs40, vs0, vs25
.endm
.macro KERNEL2x2_SUB1
lxvd2x vs0, 0, AO
lxvdsx vs24, 0, BO
lxvdsx vs25, o8, BO
addi AO, AO, 16
addi BO, BO, 16
xvmaddadp vs32, vs0, vs24
xvmaddadp vs40, vs0, vs25
.endm
.macro SAVE2x2
mr T1, CO
#ifndef TRMMKERNEL
lxvd2x vs0, 0, T1
#endif
#ifndef TRMMKERNEL
xvmaddadp vs0, vs32, alpha_r
#else
xvmuldp vs0, vs32, alpha_r
#endif
stxvd2x vs0, 0, T1
add T1, T1, LDC
#ifndef TRMMKERNEL
lxvd2x vs8, 0, T1
#endif
#ifndef TRMMKERNEL
xvmaddadp vs8, vs40, alpha_r
#else
xvmuldp vs8, vs40, alpha_r
#endif
stxvd2x vs8, 0, T1
addi CO, CO, 16
.endm
/*********************************************************************
* Macros for N=2, M=1 *
*********************************************************************/
.macro LOAD2x1_1
lxsdx vs0, 0, AO
lxsdx vs24, 0, BO
lxsdx vs25, o8, BO
addi AO, AO, 8
addi BO, BO, 16
.endm
.macro KERNEL2x1_I1
lxsdx vs8, 0, AO
lxsdx vs28, 0, BO
lxsdx vs29, o8, BO
addi AO, AO, 8
addi BO, BO, 16
xsmuldp vs32, vs0, vs24
xsmuldp vs40, vs0, vs25
.endm
.macro KERNEL2x1_1
lxsdx vs8, 0, AO
lxsdx vs28, 0, BO
lxsdx vs29, o8, BO
addi AO, AO, 8
addi BO, BO, 16
xsmaddadp vs32, vs0, vs24
xsmaddadp vs40, vs0, vs25
.endm
.macro KERNEL2x1_2
lxsdx vs0, 0, AO
lxsdx vs24, 0, BO
lxsdx vs25, o8, BO
addi AO, AO, 8
addi BO, BO, 16
xsmaddadp vs32, vs8, vs28
xsmaddadp vs40, vs8, vs29
.endm
.macro KERNEL2x1_E2
xsmaddadp vs32, vs8, vs28
xsmaddadp vs40, vs8, vs29
.endm
.macro KERNEL2x1_SUBI1
lxsdx vs0, 0, AO
lxsdx vs24, 0, BO
lxsdx vs25, o8, BO
addi AO, AO, 8
addi BO, BO, 16
xsmuldp vs32, vs0, vs24
xsmuldp vs40, vs0, vs25
.endm
.macro KERNEL2x1_SUB1
lxsdx vs0, 0, AO
lxsdx vs24, 0, BO
lxsdx vs25, o8, BO
addi AO, AO, 8
addi BO, BO, 16
xsmaddadp vs32, vs0, vs24
xsmaddadp vs40, vs0, vs25
.endm
.macro SAVE2x1
mr T1, CO
#ifndef TRMMKERNEL
lxsdx vs0, 0, T1
#endif
#ifndef TRMMKERNEL
xsmaddadp vs0, vs32, alpha_r
#else
xsmuldp vs0, vs32, alpha_r
#endif
stxsdx vs0, 0, T1
add T1, T1, LDC
#ifndef TRMMKERNEL
lxsdx vs8, 0, T1
#endif
#ifndef TRMMKERNEL
xsmaddadp vs8, vs40, alpha_r
#else
xsmuldp vs8, vs40, alpha_r
#endif
stxsdx vs8, 0, T1
addi CO, CO, 8
.endm
/*********************************************************************
* Macros for N=1, M=16 *
*********************************************************************/
.macro LOAD1x16_1
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvd2x vs2, o32, AO
lxvd2x vs3, o48, AO
lxvdsx vs24, 0, BO
addi AO, AO, 64
addi BO, BO, 8
lxvd2x vs4, 0, AO
lxvd2x vs5, o16, AO
lxvd2x vs6, o32, AO
lxvd2x vs7, o48, AO
addi AO, AO, 64
.endm
.macro KERNEL1x16_I1
lxvd2x vs8, 0, AO
lxvd2x vs9, o16, AO
lxvd2x vs10, o32, AO
lxvd2x vs11, o48, AO
lxvdsx vs28, 0, BO
addi AO, AO, 64
addi BO, BO, 8
lxvd2x vs12, 0, AO
lxvd2x vs13, o16, AO
lxvd2x vs14, o32, AO
lxvd2x vs15, o48, AO
addi AO, AO, 64
xvmuldp vs32, vs0, vs24
xvmuldp vs33, vs1, vs24
xvmuldp vs34, vs2, vs24
xvmuldp vs35, vs3, vs24
xvmuldp vs36, vs4, vs24
xvmuldp vs37, vs5, vs24
xvmuldp vs38, vs6, vs24
xvmuldp vs39, vs7, vs24
.endm
.macro KERNEL1x16_1
lxvd2x vs8, 0, AO
lxvd2x vs9, o16, AO
lxvd2x vs10, o32, AO
lxvd2x vs11, o48, AO
lxvdsx vs28, 0, BO
addi AO, AO, 64
addi BO, BO, 8
lxvd2x vs12, 0, AO
lxvd2x vs13, o16, AO
lxvd2x vs14, o32, AO
lxvd2x vs15, o48, AO
addi AO, AO, 64
xvmaddadp vs32, vs0, vs24
xvmaddadp vs33, vs1, vs24
xvmaddadp vs34, vs2, vs24
xvmaddadp vs35, vs3, vs24
xvmaddadp vs36, vs4, vs24
xvmaddadp vs37, vs5, vs24
xvmaddadp vs38, vs6, vs24
xvmaddadp vs39, vs7, vs24
.endm
.macro KERNEL1x16_2
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvd2x vs2, o32, AO
lxvd2x vs3, o48, AO
lxvdsx vs24, 0, BO
addi AO, AO, 64
addi BO, BO, 8
lxvd2x vs4, 0, AO
lxvd2x vs5, o16, AO
lxvd2x vs6, o32, AO
lxvd2x vs7, o48, AO
addi AO, AO, 64
xvmaddadp vs32, vs8, vs28
xvmaddadp vs33, vs9, vs28
xvmaddadp vs34, vs10, vs28
xvmaddadp vs35, vs11, vs28
xvmaddadp vs36, vs12, vs28
xvmaddadp vs37, vs13, vs28
xvmaddadp vs38, vs14, vs28
xvmaddadp vs39, vs15, vs28
.endm
.macro KERNEL1x16_E2
xvmaddadp vs32, vs8, vs28
xvmaddadp vs33, vs9, vs28
xvmaddadp vs34, vs10, vs28
xvmaddadp vs35, vs11, vs28
xvmaddadp vs36, vs12, vs28
xvmaddadp vs37, vs13, vs28
xvmaddadp vs38, vs14, vs28
xvmaddadp vs39, vs15, vs28
.endm
.macro KERNEL1x16_SUBI1
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvd2x vs2, o32, AO
lxvd2x vs3, o48, AO
lxvdsx vs24, 0, BO
addi AO, AO, 64
addi BO, BO, 8
lxvd2x vs4, 0, AO
lxvd2x vs5, o16, AO
lxvd2x vs6, o32, AO
lxvd2x vs7, o48, AO
addi AO, AO, 64
xvmuldp vs32, vs0, vs24
xvmuldp vs33, vs1, vs24
xvmuldp vs34, vs2, vs24
xvmuldp vs35, vs3, vs24
xvmuldp vs36, vs4, vs24
xvmuldp vs37, vs5, vs24
xvmuldp vs38, vs6, vs24
xvmuldp vs39, vs7, vs24
.endm
.macro KERNEL1x16_SUB1
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvd2x vs2, o32, AO
lxvd2x vs3, o48, AO
lxvdsx vs24, 0, BO
addi AO, AO, 64
addi BO, BO, 8
lxvd2x vs4, 0, AO
lxvd2x vs5, o16, AO
lxvd2x vs6, o32, AO
lxvd2x vs7, o48, AO
addi AO, AO, 64
xvmaddadp vs32, vs0, vs24
xvmaddadp vs33, vs1, vs24
xvmaddadp vs34, vs2, vs24
xvmaddadp vs35, vs3, vs24
xvmaddadp vs36, vs4, vs24
xvmaddadp vs37, vs5, vs24
xvmaddadp vs38, vs6, vs24
xvmaddadp vs39, vs7, vs24
.endm
.macro SAVE1x16
mr T1, CO
addi T2, T1, 64
#ifndef TRMMKERNEL
lxvd2x vs0, 0, T1
lxvd2x vs1, o16, T1
lxvd2x vs2, o32, T1
lxvd2x vs3, o48, T1
lxvd2x vs4, 0, T2
lxvd2x vs5, o16, T2
lxvd2x vs6, o32, T2
lxvd2x vs7, o48, T2
#endif
#ifndef TRMMKERNEL
xvmaddadp vs0, vs32, alpha_r
xvmaddadp vs1, vs33, alpha_r
xvmaddadp vs2, vs34, alpha_r
xvmaddadp vs3, vs35, alpha_r
xvmaddadp vs4, vs36, alpha_r
xvmaddadp vs5, vs37, alpha_r
xvmaddadp vs6, vs38, alpha_r
xvmaddadp vs7, vs39, alpha_r
#else
xvmuldp vs0, vs32, alpha_r
xvmuldp vs1, vs33, alpha_r
xvmuldp vs2, vs34, alpha_r
xvmuldp vs3, vs35, alpha_r
xvmuldp vs4, vs36, alpha_r
xvmuldp vs5, vs37, alpha_r
xvmuldp vs6, vs38, alpha_r
xvmuldp vs7, vs39, alpha_r
#endif
stxvd2x vs0, 0, T1
stxvd2x vs1, o16, T1
stxvd2x vs2, o32, T1
stxvd2x vs3, o48, T1
stxvd2x vs4, 0, T2
stxvd2x vs5, o16, T2
stxvd2x vs6, o32, T2
stxvd2x vs7, o48, T2
addi CO, CO, 128
.endm
/*********************************************************************
* Macros for N=4, M=8 *
*********************************************************************/
.macro LOAD1x8_1
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvd2x vs2, o32, AO
lxvd2x vs3, o48, AO
lxvdsx vs24, 0, BO
addi AO, AO, 64
addi BO, BO, 8
.endm
.macro KERNEL1x8_I1
lxvd2x vs8, 0, AO
lxvd2x vs9, o16, AO
lxvd2x vs10, o32, AO
lxvd2x vs11, o48, AO
lxvdsx vs28, 0, BO
addi AO, AO, 64
addi BO, BO, 8
xvmuldp vs32, vs0, vs24
xvmuldp vs33, vs1, vs24
xvmuldp vs34, vs2, vs24
xvmuldp vs35, vs3, vs24
.endm
.macro KERNEL1x8_1
lxvd2x vs8, 0, AO
lxvd2x vs9, o16, AO
lxvd2x vs10, o32, AO
lxvd2x vs11, o48, AO
lxvdsx vs28, 0, BO
addi AO, AO, 64
addi BO, BO, 8
xvmaddadp vs32, vs0, vs24
xvmaddadp vs33, vs1, vs24
xvmaddadp vs34, vs2, vs24
xvmaddadp vs35, vs3, vs24
.endm
.macro KERNEL1x8_2
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvd2x vs2, o32, AO
lxvd2x vs3, o48, AO
lxvdsx vs24, 0, BO
addi AO, AO, 64
addi BO, BO, 8
xvmaddadp vs32, vs8, vs28
xvmaddadp vs33, vs9, vs28
xvmaddadp vs34, vs10, vs28
xvmaddadp vs35, vs11, vs28
.endm
.macro KERNEL1x8_E2
xvmaddadp vs32, vs8, vs28
xvmaddadp vs33, vs9, vs28
xvmaddadp vs34, vs10, vs28
xvmaddadp vs35, vs11, vs28
.endm
.macro KERNEL1x8_SUBI1
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvd2x vs2, o32, AO
lxvd2x vs3, o48, AO
lxvdsx vs24, 0, BO
addi AO, AO, 64
addi BO, BO, 8
xvmuldp vs32, vs0, vs24
xvmuldp vs33, vs1, vs24
xvmuldp vs34, vs2, vs24
xvmuldp vs35, vs3, vs24
.endm
.macro KERNEL1x8_SUB1
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvd2x vs2, o32, AO
lxvd2x vs3, o48, AO
lxvdsx vs24, 0, BO
addi AO, AO, 64
addi BO, BO, 8
xvmaddadp vs32, vs0, vs24
xvmaddadp vs33, vs1, vs24
xvmaddadp vs34, vs2, vs24
xvmaddadp vs35, vs3, vs24
.endm
.macro SAVE1x8
mr T1, CO
#ifndef TRMMKERNEL
lxvd2x vs0, 0, T1
lxvd2x vs1, o16, T1
lxvd2x vs2, o32, T1
lxvd2x vs3, o48, T1
#endif
#ifndef TRMMKERNEL
xvmaddadp vs0, vs32, alpha_r
xvmaddadp vs1, vs33, alpha_r
xvmaddadp vs2, vs34, alpha_r
xvmaddadp vs3, vs35, alpha_r
#else
xvmuldp vs0, vs32, alpha_r
xvmuldp vs1, vs33, alpha_r
xvmuldp vs2, vs34, alpha_r
xvmuldp vs3, vs35, alpha_r
#endif
stxvd2x vs0, 0, T1
stxvd2x vs1, o16, T1
stxvd2x vs2, o32, T1
stxvd2x vs3, o48, T1
addi CO, CO, 64
.endm
/*********************************************************************
* Macros for N=1, M=4 *
*********************************************************************/
.macro LOAD1x4_1
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvdsx vs24, 0, BO
addi AO, AO, 32
addi BO, BO, 8
.endm
.macro KERNEL1x4_I1
lxvd2x vs8, 0, AO
lxvd2x vs9, o16, AO
lxvdsx vs28, 0, BO
addi AO, AO, 32
addi BO, BO, 8
xvmuldp vs32, vs0, vs24
xvmuldp vs33, vs1, vs24
.endm
.macro KERNEL1x4_1
lxvd2x vs8, 0, AO
lxvd2x vs9, o16, AO
lxvdsx vs28, 0, BO
addi AO, AO, 32
addi BO, BO, 8
xvmaddadp vs32, vs0, vs24
xvmaddadp vs33, vs1, vs24
.endm
.macro KERNEL1x4_2
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvdsx vs24, 0, BO
addi AO, AO, 32
addi BO, BO, 8
xvmaddadp vs32, vs8, vs28
xvmaddadp vs33, vs9, vs28
.endm
.macro KERNEL1x4_E2
xvmaddadp vs32, vs8, vs28
xvmaddadp vs33, vs9, vs28
.endm
.macro KERNEL1x4_SUBI1
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvdsx vs24, 0, BO
addi AO, AO, 32
addi BO, BO, 8
xvmuldp vs32, vs0, vs24
xvmuldp vs33, vs1, vs24
.endm
.macro KERNEL1x4_SUB1
lxvd2x vs0, 0, AO
lxvd2x vs1, o16, AO
lxvdsx vs24, 0, BO
addi AO, AO, 32
addi BO, BO, 8
xvmaddadp vs32, vs0, vs24
xvmaddadp vs33, vs1, vs24
.endm
.macro SAVE1x4
mr T1, CO
#ifndef TRMMKERNEL
lxvd2x vs0, 0, T1
lxvd2x vs1, o16, T1
#endif
#ifndef TRMMKERNEL
xvmaddadp vs0, vs32, alpha_r
xvmaddadp vs1, vs33, alpha_r
#else
xvmuldp vs0, vs32, alpha_r
xvmuldp vs1, vs33, alpha_r
#endif
stxvd2x vs0, 0, T1
stxvd2x vs1, o16, T1
addi CO, CO, 32
.endm
/*********************************************************************
* Macros for N=1, M=2 *
*********************************************************************/
.macro LOAD1x2_1
lxvd2x vs0, 0, AO
lxvdsx vs24, 0, BO
addi AO, AO, 16
addi BO, BO, 8
.endm
.macro KERNEL1x2_I1
lxvd2x vs8, 0, AO
lxvdsx vs28, 0, BO
addi AO, AO, 16
addi BO, BO, 8
xvmuldp vs32, vs0, vs24
.endm
.macro KERNEL1x2_1
lxvd2x vs8, 0, AO
lxvdsx vs28, 0, BO
addi AO, AO, 16
addi BO, BO, 8
xvmaddadp vs32, vs0, vs24
.endm
.macro KERNEL1x2_2
lxvd2x vs0, 0, AO
lxvdsx vs24, 0, BO
addi AO, AO, 16
addi BO, BO, 8
xvmaddadp vs32, vs8, vs28
.endm
.macro KERNEL1x2_E2
xvmaddadp vs32, vs8, vs28
.endm
.macro KERNEL1x2_SUBI1
lxvd2x vs0, 0, AO
lxvdsx vs24, 0, BO
addi AO, AO, 16
addi BO, BO, 8
xvmuldp vs32, vs0, vs24
.endm
.macro KERNEL1x2_SUB1
lxvd2x vs0, 0, AO
lxvdsx vs24, 0, BO
addi AO, AO, 16
addi BO, BO, 8
xvmaddadp vs32, vs0, vs24
.endm
.macro SAVE1x2
mr T1, CO
#ifndef TRMMKERNEL
lxvd2x vs0, 0, T1
#endif
#ifndef TRMMKERNEL
xvmaddadp vs0, vs32, alpha_r
#else
xvmuldp vs0, vs32, alpha_r
#endif
stxvd2x vs0, 0, T1
addi CO, CO, 16
.endm
/*********************************************************************
* Macros for N=1, M=1 *
*********************************************************************/
.macro LOAD1x1_1
lxsdx vs0, 0, AO
lxsdx vs24, 0, BO
addi AO, AO, 8
addi BO, BO, 8
.endm
.macro KERNEL1x1_I1
lxsdx vs8, 0, AO
lxsdx vs28, 0, BO
addi AO, AO, 8
addi BO, BO, 8
xsmuldp vs32, vs0, vs24
.endm
.macro KERNEL1x1_1
lxsdx vs8, 0, AO
lxsdx vs28, 0, BO
addi AO, AO, 8
addi BO, BO, 8
xsmaddadp vs32, vs0, vs24
.endm
.macro KERNEL1x1_2
lxsdx vs0, 0, AO
lxsdx vs24, 0, BO
addi AO, AO, 8
addi BO, BO, 8
xsmaddadp vs32, vs8, vs28
.endm
.macro KERNEL1x1_E2
xsmaddadp vs32, vs8, vs28
.endm
.macro KERNEL1x1_SUBI1
lxsdx vs0, 0, AO
lxsdx vs24, 0, BO
addi AO, AO, 8
addi BO, BO, 8
xsmuldp vs32, vs0, vs24
.endm
.macro KERNEL1x1_SUB1
lxsdx vs0, 0, AO
lxsdx vs24, 0, BO
addi AO, AO, 8
addi BO, BO, 8
xsmaddadp vs32, vs0, vs24
.endm
.macro SAVE1x1
mr T1, CO
#ifndef TRMMKERNEL
lxsdx vs0, 0, T1
#endif
#ifndef TRMMKERNEL
xsmaddadp vs0, vs32, alpha_r
#else
xsmuldp vs0, vs32, alpha_r
#endif
stxsdx vs0, 0, T1
addi CO, CO, 8
.endm
/****************************TRMM POINTER REFRESH MACROSES*************************/
.macro SHIFT_REG REG1,REG2,SHIFT_VAL
.if \SHIFT_VAL==16
slwi \REG1, \REG2, 7
.elseif \SHIFT_VAL==8
slwi \REG1, \REG2, 6
.elseif \SHIFT_VAL==4
slwi \REG1, \REG2, 5
.elseif \SHIFT_VAL==2
slwi \REG1, \REG2, 4
.elseif \SHIFT_VAL==1
slwi \REG1, \REG2, 3
.endif
.endm
/*
//#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
// ptrbb = bb;
// #else
// ptrba += off*16;
// ptrbb = bb + off*2;
// #endif
*/
.macro REFRESH_POINTERS PTR_A,PTR_B,OFF_VAL,B_VAL,C_A,C_B
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
/* ptrbb = bb;*/
mr \PTR_B,\B_VAL /* refresh BPOINT */
#else
/*
// ptrba =ptrba+ off*C_A;
// ptrbb = bb + off*C_B;
*/
SHIFT_REG T4,\OFF_VAL,\C_B /* Number of values in B shifted */
SHIFT_REG T2,\OFF_VAL,\C_A /* Number of values in A shifted */
add \PTR_B, \B_VAL , T4 /* Add values to BO */
add \PTR_A, \PTR_A, T2 /* Add values to AO */
#endif
.endm
/*
// #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
// temp = bk-off;
// #elif defined(LEFT)
// temp = off+16; // number of values in A
// #else
// temp = off+2; // number of values in B
// #endif
*/
.macro REFRESH_TEMP_BK TEMP_BK,BK_VAL,OFF_VAL,INCR_A,INCR_B
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
/* temp = bk-off;*/
sub \TEMP_BK,\BK_VAL,\OFF_VAL
#elif defined(LEFT)
/* temp = off+INCR_A; // number of values in A */
addi \TEMP_BK, \OFF_VAL, \INCR_A
#else
/* temp = off+INCR_B // number of values in B*/
addi \TEMP_BK,\OFF_VAL, \INCR_B
#endif
.endm
/*
// #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
// temp = bk - off;
// #ifdef LEFT
// temp -= 16; // number of values in A
// #else
// temp -= 2; // number of values in B
// #endif
// ptrba += temp*16;
// ptrbb += temp*2;
// #endif
// #ifdef LEFT
// off += 16; // number of values in A
// #endif
*/
.macro REFRESH_AFTER_SAVE TEMP_BK,BK_VAL,OFF_VAL,PTR_B,PTR_A,C_A,C_B
#if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
/*temp = bk - off;*/
sub \TEMP_BK,\BK_VAL,\OFF_VAL
#ifdef LEFT
/*temp -= 8; // number of values in A*/
addi \TEMP_BK,\TEMP_BK,-\C_A
#else
/*temp -= 4; // number of values in B*/
addi \TEMP_BK,\TEMP_BK,-\C_B
#endif
/*ptrba += temp*C_A;
ptrbb += temp*C_B;*/
SHIFT_REG T4,\TEMP_BK,\C_A
SHIFT_REG T2,\TEMP_BK,\C_B
add \PTR_A, \PTR_A,T4/*ptrba+temp*C_A*/
add \PTR_B, \PTR_B,T2
#endif
#ifdef LEFT
/*off += 8; // number of values in A*/
addi \OFF_VAL,\OFF_VAL,\C_A
#endif
.endm
Reference in New Issue View Git Blame Copy Permalink