floraachy
/
OpenBLAS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
OpenBLAS/kernel/power/zgemm_macros_power9.S

1825 lines
47 KiB
ArmAsm
Raw Permalink Blame History

/***************************************************************************
Copyright (c) 2013-2019, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#define unit_size 16
#define DISP32(ind,disp) (ind*unit_size*32+disp)
#define DISP16(ind,disp) (ind*unit_size*16+disp)
#define DISP8(ind,disp) (ind*unit_size*8+disp)
#define DISP4(ind,disp) (ind*unit_size*4+disp)
#define DISP2(ind,disp) (ind*unit_size*2+disp)
#define DISP1(ind,disp) (ind*unit_size+disp)
#define DISPX(disp) (disp)
/* HELPERS FOR SAVE */
/* {r0,i0} and {r1,i1} into {r0,r1} {i0,i1} */
.macro LOAD_COUPLE_AS_RR_II VS_OUT1,VS_OUT2,VS_TEMP1,VS_TEMP2,REG,LOFFSET
#ifndef TRMMKERNEL
lxv \VS_TEMP1, DISPX(\LOFFSET)(\REG)
lxv \VS_TEMP2, DISPX(\LOFFSET+16)(\REG)
xxmrgld \VS_OUT1,\VS_TEMP1,\VS_TEMP2
xxmrghd \VS_OUT2,\VS_TEMP1,\VS_TEMP2
#endif
.endm
/*from 2 result {a0r*br,a0i*bi} and {a1r*br,a1i*bi} pack into {a0r*br,a1r*br} and {a0i*bi,a1i*bi}*/
.macro RESULT_INTO_REALREAL_IMAGEIMAGE VSIN1,VSIN2,VSOUT1,VSOUT2
xxmrgld \VSOUT1, \VSIN1,\VSIN2 /* real*real from 2 results*/
xxmrghd \VSOUT2, \VSIN1,\VSIN2 /* imag*imag from 2 results*/
.endm
/*from 2 result {a0r*bi,a0i*br} and {a1r*bi,a1i*br} pack into {a0r*bi,a1r*bi} and {a0i*br,a1i*br}*/
.macro RESULT_INTO_REALIMAG_IMAGREAL VSIN1,VSIN2,VSOUT1,VSOUT2
xxmrgld \VSOUT1, \VSIN1,\VSIN2 /* real*imag */
xxmrghd \VSOUT2, \VSIN1,\VSIN2 /* imag*real*/
.endm
/* {a0r*br op a0i*bi ,a1r*br op a1i*bi} ~ {r0,r1}; {a0r*bi op a0i*br ,a1r*bi op a1i*br} ~ {i0,i1}*/
.macro AGGREGATE_REALS_IMAGES VSINR_OUT1,VSINR,VSINI_OUT2,VSINI
#if defined(NN) || defined(NT) || defined(TN) || defined(TT)
xvsubdp \VSINR_OUT1,\VSINR_OUT1,\VSINR
xvadddp \VSINI_OUT2,\VSINI_OUT2,\VSINI
#elif defined(CN) || defined(CT) || defined(RN) || defined(RT)
xvadddp \VSINR_OUT1,\VSINR_OUT1,\VSINR
xvsubdp \VSINI_OUT2,\VSINI_OUT2,\VSINI
#elif defined(NC) || defined(TC) || defined(NR) || defined(TR)
xvadddp \VSINR_OUT1,\VSINR_OUT1,\VSINR
xvsubdp \VSINI_OUT2,\VSINI,\VSINI_OUT2
#else // CC || CR || RC || RR
/*we will assume {-alpha_r,-alpha_i} for this case */
/*i1i2-r1r2 so we will negate alpha real instead to fix sign*/
xvsubdp \VSINR_OUT1,\VSINR,\VSINR_OUT1
/*we will negate alpha image instead instead to fix sign*/
xvadddp \VSINI_OUT2,\VSINI_OUT2,\VSINI
#endif
.endm
/* {i0,i1} * {alpha_i,alpha_i} - VSOUT1 ;VSOUT2 + {r0,r1}*{alpha_i,alpha_i} */
.macro MULT_APLHA_PART1 VSINRR,VSINII,VSOUT1,VSOUT2
#ifndef TRMMKERNEL
xvmsubadp \VSOUT1,\VSINII, alpha_i
xvmaddadp \VSOUT2,\VSINRR, alpha_i
#else
xvmuldp \VSOUT1,\VSINII, alpha_i
xvmuldp \VSOUT2,\VSINRR, alpha_i
#endif
.endm
/* {r0,r1} * {alpha_r,alpha_r} - VSOUT1 ;VSOUT2 + {i0,i1} * {alpha_r,alpha_r} */
.macro MULT_APLHA_PART2 VSINRR,VSINII,VSOUT1,VSOUT2
xvmsubadp \VSOUT1,\VSINRR, alpha_r
xvmaddadp \VSOUT2,\VSINII, alpha_r
.endm
/* unpack to store 2{r,r} {i,i} into {r,i} {r,i} (big endian because of stxv) */
.macro UNPACK_FOR_STORE VSIN1,VSIN2,VSOUT1,VSOUT2
xxmrghd \VSOUT1,\VSIN2,\VSIN1
xxmrgld \VSOUT2,\VSIN2,\VSIN1
.endm
.macro STORE_COUPLE REG,LOFFSET,VSIN1,VSIN2
stxv \VSIN1, DISPX(\LOFFSET)(\REG)
stxv \VSIN2, DISPX(\LOFFSET+16)(\REG)
.endm
.macro SAVE8 VSRes1,VSRes2,VSRes3,VSRes4,VSRes5,VSRes6,VSRes7,VSRes8,VSRes9,VSRes10,VSRes11,VSRes12,VSRes13,VSRes14,VSRes15,VSRes16,BASE_REG,LOFFSET
RESULT_INTO_REALREAL_IMAGEIMAGE \VSRes1,\VSRes3,vs2,vs3
LOAD_COUPLE_AS_RR_II vs14,vs15,vs18,vs19,\BASE_REG,\LOFFSET
RESULT_INTO_REALIMAG_IMAGREAL \VSRes2,\VSRes4,vs4,vs5
LOAD_COUPLE_AS_RR_II vs16,vs17,vs20,vs21,\BASE_REG,(\LOFFSET+32)
RESULT_INTO_REALREAL_IMAGEIMAGE \VSRes5,\VSRes7,vs6,vs7
LOAD_COUPLE_AS_RR_II vs24,vs25,vs18,vs19,\BASE_REG,(\LOFFSET +64)
RESULT_INTO_REALIMAG_IMAGREAL \VSRes6,\VSRes8,vs8,vs9
LOAD_COUPLE_AS_RR_II vs26,vs27,vs20,vs21,\BASE_REG,(\LOFFSET+96)
RESULT_INTO_REALREAL_IMAGEIMAGE \VSRes9,\VSRes11,vs10,vs11
AGGREGATE_REALS_IMAGES vs2,vs3,vs4,vs5
RESULT_INTO_REALIMAG_IMAGREAL \VSRes10,\VSRes12,vs12,vs13
AGGREGATE_REALS_IMAGES vs6,vs7,vs8,vs9
RESULT_INTO_REALREAL_IMAGEIMAGE \VSRes13,\VSRes15,\VSRes1,\VSRes2
MULT_APLHA_PART1 vs2,vs4, vs14,vs15
RESULT_INTO_REALIMAG_IMAGREAL \VSRes14,\VSRes16,\VSRes3,\VSRes4
MULT_APLHA_PART1 vs6,vs8,vs16,vs17
MULT_APLHA_PART2 vs2,vs4,vs14,vs15
AGGREGATE_REALS_IMAGES vs10,vs11,vs12,vs13
MULT_APLHA_PART2 vs6,vs8,vs16,vs17
AGGREGATE_REALS_IMAGES \VSRes1,\VSRes2,\VSRes3,\VSRes4
UNPACK_FOR_STORE vs14,vs15,vs7,vs9
MULT_APLHA_PART1 vs10,vs12, vs24,vs25
UNPACK_FOR_STORE vs16,vs17,vs3,vs5
MULT_APLHA_PART1 \VSRes1,\VSRes3, vs26,vs27
STORE_COUPLE \BASE_REG,\LOFFSET,vs7,vs9
MULT_APLHA_PART2 vs10,vs12,vs24,vs25
STORE_COUPLE \BASE_REG,(\LOFFSET+32),vs3,vs5
MULT_APLHA_PART2 \VSRes1,\VSRes3, vs26,vs27
UNPACK_FOR_STORE vs24,vs25,vs10,vs12
UNPACK_FOR_STORE vs26,vs27,\VSRes1,\VSRes3
STORE_COUPLE \BASE_REG,(\LOFFSET +64),vs10,vs12
STORE_COUPLE \BASE_REG,(\LOFFSET+96),\VSRes1,\VSRes3
.endm
.macro SAVE4 VSRes1,VSRes2,VSRes3,VSRes4,VSRes5,VSRes6,VSRes7,VSRes8,BASE_REG,LOFFSET
RESULT_INTO_REALREAL_IMAGEIMAGE \VSRes1,\VSRes3,vs2,vs3
LOAD_COUPLE_AS_RR_II vs14,vs15,vs18,vs19,\BASE_REG,\LOFFSET
RESULT_INTO_REALIMAG_IMAGREAL \VSRes2,\VSRes4,vs4,vs5
LOAD_COUPLE_AS_RR_II vs16,vs17,vs20,vs21,\BASE_REG,(\LOFFSET+32)
RESULT_INTO_REALREAL_IMAGEIMAGE \VSRes5,\VSRes7,vs6,vs7
RESULT_INTO_REALIMAG_IMAGREAL \VSRes6,\VSRes8,vs8,vs9
AGGREGATE_REALS_IMAGES vs2,vs3,vs4,vs5
AGGREGATE_REALS_IMAGES vs6,vs7,vs8,vs9
MULT_APLHA_PART1 vs2,vs4, vs14,vs15
MULT_APLHA_PART1 vs6,vs8, vs16,vs17
MULT_APLHA_PART2 vs2,vs4, vs14,vs15
MULT_APLHA_PART2 vs6,vs8,vs16,vs17
UNPACK_FOR_STORE vs14,vs15,vs7,vs9
UNPACK_FOR_STORE vs16,vs17,vs3,vs5
STORE_COUPLE \BASE_REG,\LOFFSET,vs7,vs9
STORE_COUPLE \BASE_REG,(\LOFFSET+32),vs3,vs5
.endm
.macro SAVE2 VSRes1,VSRes2,VSRes3,VSRes4,BASE_REG,LOFFSET
RESULT_INTO_REALREAL_IMAGEIMAGE \VSRes1,\VSRes3,vs2,vs3
LOAD_COUPLE_AS_RR_II vs14,vs15,vs18,vs19,\BASE_REG,\LOFFSET
RESULT_INTO_REALIMAG_IMAGREAL \VSRes2,\VSRes4,vs4,vs5
AGGREGATE_REALS_IMAGES vs2,vs3,vs4,vs5
MULT_APLHA_PART1 vs2,vs4, vs14,vs15
MULT_APLHA_PART2 vs2,vs4, vs14,vs15
UNPACK_FOR_STORE vs14,vs15,vs7,vs9
STORE_COUPLE \BASE_REG,\LOFFSET,vs7,vs9
.endm
.macro SAVE1 VSRes1,VSRes2,BASE_REG,LOFFSET
RESULT_INTO_REALREAL_IMAGEIMAGE \VSRes1,\VSRes1,vs2,vs3
#ifndef TRMMKERNEL
lxv vs18, (\LOFFSET)(\BASE_REG)
xxmrgld vs14,vs18,vs18
xxmrghd vs15,vs18,vs18
#endif
RESULT_INTO_REALIMAG_IMAGREAL \VSRes2,\VSRes2,vs4,vs5
AGGREGATE_REALS_IMAGES vs2,vs3,vs4,vs5
MULT_APLHA_PART1 vs2,vs4, vs14,vs15
MULT_APLHA_PART2 vs2,vs4, vs14,vs15
UNPACK_FOR_STORE vs14,vs15,vs7,vs9
xxmrghd vs7,vs15,vs14
stxv vs7, (\LOFFSET)(\BASE_REG)
.endm
/**********************************************************************************************
*
.macros for N=2 and M=8
**********************************************************************************************/
.macro Zero2x8
xxlxor vs32, vs32, vs32
xxlxor vs33, vs33, vs33
xxlxor vs34, vs34, vs34
xxlxor vs35, vs35, vs35
xxlxor vs36, vs36, vs36
xxlxor vs37, vs37, vs37
xxlxor vs38, vs38, vs38
xxlxor vs39, vs39, vs39
xxlxor vs40, vs40, vs40
xxlxor vs41, vs41, vs41
xxlxor vs42, vs42, vs42
xxlxor vs43, vs43, vs43
xxlxor vs44, vs44, vs44
xxlxor vs45, vs45, vs45
xxlxor vs46, vs46, vs46
xxlxor vs47, vs47, vs47
xxlxor vs48, vs48, vs48
xxlxor vs49, vs49, vs49
xxlxor vs50, vs50, vs50
xxlxor vs51, vs51, vs51
xxlxor vs52, vs52, vs52
xxlxor vs53, vs53, vs53
xxlxor vs54, vs54, vs54
xxlxor vs55, vs55, vs55
xxlxor vs56, vs56, vs56
xxlxor vs57, vs57, vs57
xxlxor vs58, vs58, vs58
xxlxor vs59, vs59, vs59
xxlxor vs60, vs60, vs60
xxlxor vs61, vs61, vs61
xxlxor vs62, vs62, vs62
xxlxor vs63, vs63, vs63
.endm
.macro LOAD2x8
LOAD2x8O 0,0
.endm
.macro LOAD2x8O OffsetA,OffsetB
lxv vs16,(\OffsetB+ 0)(BO) // load real imag from B
lxv vs18, (\OffsetB+16)(BO) // load real,imag from B
xxswapd vs17, vs16
xxswapd vs19, vs18
lxv vs0, (0+\OffsetA)(AO) // load real,imag from A
lxv vs1, (16+\OffsetA)(AO) // load real,imag from A
lxv vs2, (32+\OffsetA)(AO) // load real,imag from A
lxv vs3, (48+\OffsetA)(AO) // load real,imag from A
lxv vs4, (64+\OffsetA)(AO) // load real,imag from A
lxv vs5, (80+\OffsetA)(AO) // load real,imag from A
lxv vs6, (96+\OffsetA)(AO) // load real,imag from A
lxv vs7, (112+\OffsetA)(AO) // load real,imag from A
.endm
.macro END2x8_NORMAL
END2x8 AO,BO,128,32
.endm
.macro END2x8_WITHOUT_ADD
END2x8 AO,BO,0,0
.endm
.macro END2x8 AREG, BREG, OffsetA, OffsetB
.if \OffsetB != 0
addi \BREG, \BREG, \OffsetB
.endif
.if \OffsetA != 0
addi \AREG, \AREG, \OffsetA
.endif
xvmaddadp vs32, vs0, vs16
xvmaddadp vs48, vs0, vs18
xvmaddadp vs33, vs0, vs17
xvmaddadp vs49, vs0, vs19
xvmaddadp vs34, vs1, vs16
xvmaddadp vs50, vs1, vs18
xvmaddadp vs35, vs1, vs17
xvmaddadp vs51, vs1, vs19
xvmaddadp vs36, vs2, vs16
xvmaddadp vs52, vs2, vs18
xvmaddadp vs37, vs2, vs17
xvmaddadp vs53, vs2, vs19
xvmaddadp vs38, vs3, vs16
xvmaddadp vs54, vs3, vs18
xvmaddadp vs39, vs3, vs17
xvmaddadp vs55, vs3, vs19
xvmaddadp vs40, vs4, vs16
xvmaddadp vs56, vs4, vs18
xvmaddadp vs41, vs4, vs17
xvmaddadp vs57, vs4, vs19
xvmaddadp vs42, vs5, vs16
xvmaddadp vs58, vs5, vs18
xvmaddadp vs43, vs5, vs17
xvmaddadp vs59, vs5, vs19
xvmaddadp vs44, vs6, vs16
xvmaddadp vs60, vs6, vs18
xvmaddadp vs45, vs6, vs17
xvmaddadp vs61, vs6, vs19
xvmaddadp vs46, vs7, vs16
xvmaddadp vs62, vs7, vs18
xvmaddadp vs47, vs7, vs17
xvmaddadp vs63, vs7, vs19
.endm
.macro LOAD2x8_2
LOAD2x8_2O 0,0
.endm
.macro LOAD2x8_2O OffsetA,OffsetB
lxv vs16,(\OffsetB+ 0)(BO) // load real imag from B
lxv vs18, (\OffsetB+16)(BO) // load real,imag from B
lxv vs20, (\OffsetB+32)(BO) // load real,imag from B
lxv vs22, (\OffsetB+48)(BO) // load real,imag from B
xxswapd vs17, vs16
xxswapd vs19, vs18
lxv vs0, (0+\OffsetA)(AO) // load real,imag from A
lxv vs1, (16+\OffsetA)(AO) // load real,imag from A
lxv vs2, (32+\OffsetA)(AO) // load real,imag from A
lxv vs3, (48+\OffsetA)(AO) // load real,imag from A
lxv vs4, (64+\OffsetA)(AO) // load real,imag from A
lxv vs5, (80+\OffsetA)(AO) // load real,imag from A
lxv vs6, (96+\OffsetA)(AO) // load real,imag from A
lxv vs7, (112+\OffsetA)(AO) // load real,imag from A
lxv vs8, (128+0+\OffsetA)(AO) // load real,imag from A
lxv vs9, (128+16+\OffsetA)(AO) // load real,imag from A
lxv vs10, (128+32+\OffsetA)(AO) // load real,imag from A
lxv vs11, (128+48+\OffsetA)(AO) // load real,imag from A
lxv vs12, (128+64+\OffsetA)(AO) // load real,imag from A
lxv vs13, (128+80+\OffsetA)(AO) // load real,imag from A
lxv vs14, (128+96+\OffsetA)(AO) // load real,imag from A
lxv vs15, (128+112+\OffsetA)(AO) // load real,imag from A
.endm
.macro END2x8_2
/*for load2 offset will be 256 and 64*/
KERNEL2x8_2 AO,BO, 256,64,0 ,1,1
.endm
.macro KERNEL2x8_E2 OffsetA,OffsetB, Index,IsLast
KERNEL2x8_2 AO,BO, \OffsetA,\OffsetB, \Index,\IsLast ,1
.endm
.macro KERNEL2x8_L2 OffsetA,OffsetB, Index,IsLast
KERNEL2x8_2 AO,BO, \OffsetA,\OffsetB, \Index,\IsLast ,0
.endm
.macro KERNEL2x8_2 AREG,BREG, OffsetA,OffsetB, Index,IsLast ,Complete
xvmaddadp vs32, vs0, vs16
xvmaddadp vs48, vs0, vs18
xvmaddadp vs33, vs0, vs17
xvmaddadp vs49, vs0, vs19
xxswapd vs21, vs20
xxswapd vs23, vs22
xvmaddadp vs34, vs1, vs16
xvmaddadp vs50, vs1, vs18
xvmaddadp vs35, vs1, vs17
xvmaddadp vs51, vs1, vs19
.if \Complete==0
lxv vs0, DISP16(\Index, 0 + \OffsetA)(\AREG) // load real,imag from A
lxv vs1, DISP16(\Index,16 + \OffsetA)(\AREG) // load real,imag from A
.endif
xvmaddadp vs36, vs2, vs16
xvmaddadp vs52, vs2, vs18
xvmaddadp vs37, vs2, vs17
xvmaddadp vs53, vs2, vs19
xvmaddadp vs38, vs3, vs16
xvmaddadp vs54, vs3, vs18
xvmaddadp vs39, vs3, vs17
xvmaddadp vs55, vs3, vs19
.if \Complete==0
lxv vs2, DISP16(\Index,32 + \OffsetA)(\AREG) // load real,imag from A
lxv vs3, DISP16(\Index,48 + \OffsetA)(\AREG) // load real,imag from A
.endif
xvmaddadp vs40, vs4, vs16
xvmaddadp vs56, vs4, vs18
xvmaddadp vs41, vs4, vs17
xvmaddadp vs57, vs4, vs19
xvmaddadp vs42, vs5, vs16
xvmaddadp vs58, vs5, vs18
xvmaddadp vs43, vs5, vs17
xvmaddadp vs59, vs5, vs19
.if \Complete==0
lxv vs4, DISP16(\Index,64+ \OffsetA)(\AREG) // load real,imag from A
lxv vs5, DISP16(\Index,64+16 + \OffsetA)(\AREG) // load real,imag from A
.endif
xvmaddadp vs44, vs6, vs16
xvmaddadp vs60, vs6, vs18
xvmaddadp vs45, vs6, vs17
xvmaddadp vs61, vs6, vs19
xvmaddadp vs46, vs7, vs16
xvmaddadp vs62, vs7, vs18
xvmaddadp vs47, vs7, vs17
xvmaddadp vs63, vs7, vs19
.if \Complete==0
lxv vs16, DISP4(\Index, 0+\OffsetB)(\BREG) // load real imag from B
lxv vs18, DISP4(\Index, 16+\OffsetB)(\BREG) // load real,imag from B
.endif
xvmaddadp vs32, vs8, vs20
xvmaddadp vs48, vs8, vs22
.if \Complete==0
lxv vs6, DISP16(\Index,64+32 + \OffsetA)(\AREG) // load real,imag from A
lxv vs7, DISP16(\Index,64+48 + \OffsetA)(\AREG) // load real,imag from A
.endif
xvmaddadp vs33, vs8, vs21
xvmaddadp vs49, vs8, vs23
.if \Complete==0
xxswapd vs17, vs16
xxswapd vs19, vs18
.endif
xvmaddadp vs34, vs9, vs20
xvmaddadp vs50, vs9, vs22
xvmaddadp vs35, vs9, vs21
xvmaddadp vs51, vs9, vs23
.if \Complete==0
lxv vs8, DISP16(\Index,128+ + \OffsetA)(\AREG) // load real,imag from A
lxv vs9, DISP16(\Index,128+16 + \OffsetA)(\AREG) // load real,imag from A
.endif
xvmaddadp vs36, vs10, vs20
xvmaddadp vs52, vs10, vs22
xvmaddadp vs37, vs10, vs21
xvmaddadp vs53, vs10, vs23
xvmaddadp vs38, vs11, vs20
xvmaddadp vs54, vs11, vs22
xvmaddadp vs39, vs11, vs21
xvmaddadp vs55, vs11, vs23
.if \Complete==0
lxv vs10, DISP16(\Index,128+32 + \OffsetA)(\AREG) // load real,imag from A
lxv vs11, DISP16(\Index,128+48 + \OffsetA)(\AREG) // load real,imag from A
.endif
xvmaddadp vs40, vs12, vs20
xvmaddadp vs56, vs12, vs22
xvmaddadp vs41, vs12, vs21
xvmaddadp vs57, vs12, vs23
xvmaddadp vs42, vs13, vs20
xvmaddadp vs58, vs13, vs22
xvmaddadp vs43, vs13, vs21
xvmaddadp vs59, vs13, vs23
.if \Complete==0
lxv vs12, DISP16(\Index, 192 + \OffsetA)(\AREG) // load real,imag from A
lxv vs13, DISP16(\Index,192 +16 + \OffsetA)(\AREG) // load real,imag from A
.endif
xvmaddadp vs44, vs14, vs20
xvmaddadp vs60, vs14, vs22
xvmaddadp vs45, vs14, vs21
xvmaddadp vs61, vs14, vs23
xvmaddadp vs46, vs15, vs20
xvmaddadp vs62, vs15, vs22
xvmaddadp vs47, vs15, vs21
xvmaddadp vs63, vs15, vs23
.if \Complete==0
lxv vs14, DISP16(\Index,192 +32 + \OffsetA)(\AREG) // load real,imag from A
lxv vs15, DISP16(\Index,192 +48 + \OffsetA)(\AREG) // load real,imag from A
lxv vs20, DISP4(\Index, 32+\OffsetB)(\BREG) // load real,imag from B
lxv vs22, DISP4(\Index, 48+\OffsetB)(\BREG) // load real,imag from B
.endif
.if \IsLast==1
.if \Complete==1
addi \AREG, \AREG, DISP16(\Index,\OffsetA)
addi \BREG, \BREG, DISP4(\Index,\OffsetB)
.else
addi \AREG, \AREG, DISP16(\Index,256)
addi \BREG, \BREG, DISP4(\Index,64)
.endif
.endif
.endm
.macro KERNEL2x8
LOAD2x8
END2x8 AO, BO, 128,32
.endm
.macro SAVE2x8
add T1, CO ,LDC
SAVE8 vs32,vs33,vs34,vs35,vs36,vs37,vs38,vs39,vs40,vs41,vs42,vs43,vs44,vs45,vs46,vs47,CO,0
SAVE8 vs48,vs49,vs50,vs51,vs52,vs53,vs54,vs55,vs56,vs57,vs58,vs59,vs60,vs61,vs62,vs63,T1,0
addi CO, CO, 128
.endm
/**********************************************************************************************
*
.macros for N=2 and M=4
**********************************************************************************************/
.macro Zero2x4
xxlxor vs32, vs32, vs32
xxlxor vs33, vs33, vs33
xxlxor vs34, vs34, vs34
xxlxor vs35, vs35, vs35
xxlxor vs36, vs36, vs36
xxlxor vs37, vs37, vs37
xxlxor vs38, vs38, vs38
xxlxor vs39, vs39, vs39
xxlxor vs40, vs40, vs40
xxlxor vs41, vs41, vs41
xxlxor vs42, vs42, vs42
xxlxor vs43, vs43, vs43
xxlxor vs44, vs44, vs44
xxlxor vs45, vs45, vs45
xxlxor vs46, vs46, vs46
xxlxor vs47, vs47, vs47
.endm
.macro LOAD2x4
LOAD2x4O 0,0
.endm
.macro LOAD2x4O OffsetA,OffsetB
lxv vs16,(\OffsetB+ 0)(BO) // load real imag from B
lxv vs18, (\OffsetB+16)(BO) // load real,imag from B
xxswapd vs17, vs16
xxswapd vs19, vs18
lxv vs0, (0+\OffsetA)(AO) // load real,imag from A
lxv vs1, (16+\OffsetA)(AO) // load real,imag from A
lxv vs2, (32+\OffsetA)(AO) // load real,imag from A
lxv vs3, (48+\OffsetA)(AO) // load real,imag from A
.endm
.macro END2x4_NORMAL
END2x4 AO,BO,64,32
.endm
.macro END2x4_WITHOUT_ADD
END2x4 AO,BO,0,0
.endm
.macro END2x4 AREG, BREG, OffsetA, OffsetB
.if \OffsetB != 0
addi \BREG, \BREG, \OffsetB
.endif
.if \OffsetA != 0
addi \AREG, \AREG, \OffsetA
.endif
xvmaddadp vs32, vs0, vs16
xvmaddadp vs40, vs0, vs18
xvmaddadp vs33, vs0, vs17
xvmaddadp vs41, vs0, vs19
xvmaddadp vs34, vs1, vs16
xvmaddadp vs42, vs1, vs18
xvmaddadp vs35, vs1, vs17
xvmaddadp vs43, vs1, vs19
xvmaddadp vs36, vs2, vs16
xvmaddadp vs44, vs2, vs18
xvmaddadp vs37, vs2, vs17
xvmaddadp vs45, vs2, vs19
xvmaddadp vs38, vs3, vs16
xvmaddadp vs46, vs3, vs18
xvmaddadp vs39, vs3, vs17
xvmaddadp vs47, vs3, vs19
.endm
.macro LOAD2x4_2
LOAD2x4_2O 0,0
.endm
.macro LOAD2x4_2O OffsetA,OffsetB
lxv vs16,(\OffsetB+ 0)(BO) // load real imag from B
lxv vs18, (\OffsetB+16)(BO) // load real,imag from B
lxv vs20, (\OffsetB+32)(BO) // load real,imag from B
lxv vs22, (\OffsetB+48)(BO) // load real,imag from B
xxswapd vs17, vs16
xxswapd vs19, vs18
lxv vs0, (0+\OffsetA)(AO) // load real,imag from A
lxv vs1, (16+\OffsetA)(AO) // load real,imag from A
lxv vs2, (32+\OffsetA)(AO) // load real,imag from A
lxv vs3, (48+\OffsetA)(AO) // load real,imag from A
lxv vs8, (64+\OffsetA)(AO) // load real,imag from A
lxv vs9, (80+\OffsetA)(AO) // load real,imag from A
lxv vs10, (96+\OffsetA)(AO) // load real,imag from A
lxv vs11, (112+\OffsetA)(AO) // load real,imag from A
.endm
.macro END2x4_2
/*for load2 offset will be 128 and 64*/
KERNEL2x4_2 AO,BO, 128,64,0 ,1,1
.endm
.macro KERNEL2x4_E2 OffsetA,OffsetB, Index,IsLast
KERNEL2x4_2 AO,BO, \OffsetA,\OffsetB, \Index,\IsLast ,1
.endm
.macro KERNEL2x4_L2 OffsetA,OffsetB, Index,IsLast
KERNEL2x4_2 AO,BO, \OffsetA,\OffsetB, \Index,\IsLast ,0
.endm
.macro KERNEL2x4_2 AREG,BREG, OffsetA,OffsetB, Index,IsLast ,Complete
xvmaddadp vs32, vs0, vs16
xvmaddadp vs40, vs0, vs18
xvmaddadp vs33, vs0, vs17
xvmaddadp vs41, vs0, vs19
xxswapd vs21, vs20
xxswapd vs23, vs22
xvmaddadp vs34, vs1, vs16
xvmaddadp vs42, vs1, vs18
xvmaddadp vs35, vs1, vs17
xvmaddadp vs43, vs1, vs19
.if \Complete==0
lxv vs0, DISP8(\Index, 0 + \OffsetA)(\AREG) // load real,imag from A
lxv vs1, DISP8(\Index,16 + \OffsetA)(\AREG) // load real,imag from A
.endif
xvmaddadp vs36, vs2, vs16
xvmaddadp vs44, vs2, vs18
xvmaddadp vs37, vs2, vs17
xvmaddadp vs45, vs2, vs19
xvmaddadp vs38, vs3, vs16
xvmaddadp vs46, vs3, vs18
xvmaddadp vs39, vs3, vs17
xvmaddadp vs47, vs3, vs19
.if \Complete==0
lxv vs2, DISP8(\Index,32 + \OffsetA)(\AREG) // load real,imag from A
lxv vs3, DISP8(\Index,48 + \OffsetA)(\AREG) // load real,imag from A
.endif
.if \Complete==0
lxv vs16, DISP4(\Index, 0+\OffsetB)(\BREG) // load real imag from B
lxv vs18, DISP4(\Index, 16+\OffsetB)(\BREG) // load real,imag from B
.endif
xvmaddadp vs32, vs8, vs20
xvmaddadp vs40, vs8, vs22
xvmaddadp vs33, vs8, vs21
xvmaddadp vs41, vs8, vs23
.if \Complete==0
xxswapd vs17, vs16
xxswapd vs19, vs18
.endif
xvmaddadp vs34, vs9, vs20
xvmaddadp vs42, vs9, vs22
xvmaddadp vs35, vs9, vs21
xvmaddadp vs43, vs9, vs23
.if \Complete==0
lxv vs8, DISP8(\Index,64+0+ \OffsetA)(\AREG) // load real,imag from A
lxv vs9, DISP8(\Index,64+16 + \OffsetA)(\AREG) // load real,imag from A
.endif
xvmaddadp vs36, vs10, vs20
xvmaddadp vs44, vs10, vs22
xvmaddadp vs37, vs10, vs21
xvmaddadp vs45, vs10, vs23
xvmaddadp vs38, vs11, vs20
xvmaddadp vs46, vs11, vs22
xvmaddadp vs39, vs11, vs21
xvmaddadp vs47, vs11, vs23
.if \Complete==0
lxv vs10, DISP8(\Index,64+32 + \OffsetA)(\AREG) // load real,imag from A
lxv vs11, DISP8(\Index,64+48 + \OffsetA)(\AREG) // load real,imag from A
.endif
.if \Complete==0
lxv vs20, DISP4(\Index, 32+\OffsetB)(\BREG) // load real,imag from B
lxv vs22, DISP4(\Index, 48+\OffsetB)(\BREG) // load real,imag from B
.endif
.if \IsLast==1
.if \Complete==1
addi \AREG, \AREG, DISP8(\Index,\OffsetA)
addi \BREG, \BREG, DISP4(\Index,\OffsetB)
.else
addi \AREG, \AREG, DISP8(\Index,128)
addi \BREG, \BREG, DISP4(\Index,64)
.endif
.endif
.endm
.macro KERNEL2x4
LOAD2x4
END2x4 AO, BO, 64,32
.endm
.macro SAVE2x4
add T1, CO ,LDC
SAVE4 vs32,vs33,vs34,vs35,vs36,vs37,vs38,vs39,CO,0
SAVE4 vs40,vs41,vs42,vs43,vs44,vs45,vs46,vs47,T1,0
addi CO, CO, 64
.endm
/**********************************************************************************************
*
.macros for N=2 and M=2
**********************************************************************************************/
.macro Zero2x2
xxlxor vs32, vs32, vs32
xxlxor vs33, vs33, vs33
xxlxor vs34, vs34, vs34
xxlxor vs35, vs35, vs35
xxlxor vs36, vs36, vs36
xxlxor vs37, vs37, vs37
xxlxor vs38, vs38, vs38
xxlxor vs39, vs39, vs39
.endm
.macro LOAD2x2
LOAD2x2O 0,0
.endm
.macro LOAD2x2O OffsetA,OffsetB
lxv vs16,(\OffsetB+ 0)(BO) // load real imag from B
lxv vs18, (\OffsetB+16)(BO) // load real,imag from B
xxswapd vs17, vs16
xxswapd vs19, vs18
lxv vs0, (0+\OffsetA)(AO) // load real,imag from A
lxv vs1, (16+\OffsetA)(AO) // load real,imag from A
.endm
.macro END2x2_NORMAL
END2x2 AO,BO,32,32
.endm
.macro END2x2_WITHOUT_ADD
END2x2 AO,BO,0,0
.endm
.macro END2x2 AREG, BREG, OffsetA, OffsetB
.if \OffsetB != 0
addi \BREG, \BREG, \OffsetB
.endif
.if \OffsetA != 0
addi \AREG, \AREG, \OffsetA
.endif
xvmaddadp vs32, vs0, vs16
xvmaddadp vs36, vs0, vs18
xvmaddadp vs33, vs0, vs17
xvmaddadp vs37, vs0, vs19
xvmaddadp vs34, vs1, vs16
xvmaddadp vs38, vs1, vs18
xvmaddadp vs35, vs1, vs17
xvmaddadp vs39, vs1, vs19
.endm
.macro LOAD2x2_2
LOAD2x2_2O 0,0
.endm
.macro LOAD2x2_2O OffsetA,OffsetB
lxv vs16,(\OffsetB+ 0)(BO) // load real imag from B
lxv vs18, (\OffsetB+16)(BO) // load real,imag from B
lxv vs20, (\OffsetB+32)(BO) // load real,imag from B
lxv vs22, (\OffsetB+48)(BO) // load real,imag from B
xxswapd vs17, vs16
xxswapd vs19, vs18
lxv vs0, (0+\OffsetA)(AO) // load real,imag from A
lxv vs1, (16+\OffsetA)(AO) // load real,imag from A
lxv vs8, (32+\OffsetA)(AO) // load real,imag from A
lxv vs9, (48+\OffsetA)(AO) // load real,imag from A
.endm
.macro END2x2_2
/*for load2 offset will be 64 and 64*/
KERNEL2x2_2 AO,BO, 64,64,0 ,1,1
.endm
.macro KERNEL2x2_E2 OffsetA,OffsetB, Index,IsLast
KERNEL2x2_2 AO,BO, \OffsetA,\OffsetB, \Index,\IsLast ,1
.endm
.macro KERNEL2x2_L2 OffsetA,OffsetB, Index,IsLast
KERNEL2x2_2 AO,BO, \OffsetA,\OffsetB, \Index,\IsLast ,0
.endm
.macro KERNEL2x2_2 AREG,BREG, OffsetA,OffsetB, Index,IsLast ,Complete
xvmaddadp vs32, vs0, vs16
xvmaddadp vs36, vs0, vs18
xvmaddadp vs33, vs0, vs17
xvmaddadp vs37, vs0, vs19
xxswapd vs21, vs20
xxswapd vs23, vs22
xvmaddadp vs34, vs1, vs16
xvmaddadp vs38, vs1, vs18
xvmaddadp vs35, vs1, vs17
xvmaddadp vs39, vs1, vs19
.if \Complete==0
lxv vs0, DISP4(\Index, 0 + \OffsetA)(\AREG) // load real,imag from A
lxv vs1, DISP4(\Index,16 + \OffsetA)(\AREG) // load real,imag from A
.endif
.if \Complete==0
lxv vs16, DISP4(\Index, 0+\OffsetB)(\BREG) // load real imag from B
lxv vs18, DISP4(\Index, 16+\OffsetB)(\BREG) // load real,imag from B
.endif
xvmaddadp vs32, vs8, vs20
xvmaddadp vs36, vs8, vs22
xvmaddadp vs33, vs8, vs21
xvmaddadp vs37, vs8, vs23
.if \Complete==0
xxswapd vs17, vs16
xxswapd vs19, vs18
.endif
xvmaddadp vs34, vs9, vs20
xvmaddadp vs38, vs9, vs22
xvmaddadp vs35, vs9, vs21
xvmaddadp vs39, vs9, vs23
.if \Complete==0
lxv vs20, DISP4(\Index, 32+\OffsetB)(\BREG) // load real,imag from B
lxv vs22, DISP4(\Index, 48+\OffsetB)(\BREG) // load real,imag from B
.endif
.if \Complete==0
lxv vs8, DISP4(\Index,32+0+ \OffsetA)(\AREG) // load real,imag from A
lxv vs9, DISP4(\Index,32+16 + \OffsetA)(\AREG) // load real,imag from A
.endif
.if \IsLast==1
.if \Complete==1
addi \AREG, \AREG, DISP4(\Index,\OffsetA)
addi \BREG, \BREG, DISP4(\Index,\OffsetB)
.else
addi \AREG, \AREG, DISP4(\Index,64)
addi \BREG, \BREG, DISP4(\Index,64)
.endif
.endif
.endm
.macro KERNEL2x2
LOAD2x2
END2x2 AO, BO, 32,32
.endm
.macro SAVE2x2
add T1, CO ,LDC
SAVE2 vs32,vs33,vs34,vs35,CO,0
SAVE2 vs36,vs37,vs38,vs39,T1,0
addi CO, CO, 32
.endm
/**********************************************************************************************
*
.macros for N=2 and M=1
**********************************************************************************************/
.macro Zero2x1
xxlxor vs32, vs32, vs32
xxlxor vs33, vs33, vs33
xxlxor vs34, vs34, vs34
xxlxor vs35, vs35, vs35
.endm
.macro LOAD2x1
LOAD2x1O 0,0
.endm
.macro LOAD2x1O OffsetA,OffsetB
lxv vs16,(\OffsetB+ 0)(BO) // load real imag from B
lxv vs18, (\OffsetB+16)(BO) // load real,imag from B
xxswapd vs17, vs16
xxswapd vs19, vs18
lxv vs0, (0+\OffsetA)(AO) // load real,imag from A
.endm
.macro END2x1_NORMAL
END2x1 AO,BO,16,32
.endm
.macro END2x1_WITHOUT_ADD
END2x1 AO,BO,0,0
.endm
.macro END2x1 AREG, BREG, OffsetA, OffsetB
.if \OffsetB != 0
addi \BREG, \BREG, \OffsetB
.endif
.if \OffsetA != 0
addi \AREG, \AREG, \OffsetA
.endif
xvmaddadp vs32, vs0, vs16
xvmaddadp vs34, vs0, vs18
xvmaddadp vs33, vs0, vs17
xvmaddadp vs35, vs0, vs19
.endm
.macro LOAD2x1_2
LOAD2x1_2O 0,0
.endm
.macro LOAD2x1_2O OffsetA,OffsetB
lxv vs16,(\OffsetB+ 0)(BO) // load real imag from B
lxv vs18, (\OffsetB+16)(BO) // load real,imag from B
lxv vs20, (\OffsetB+32)(BO) // load real,imag from B
lxv vs22, (\OffsetB+48)(BO) // load real,imag from B
xxswapd vs17, vs16
xxswapd vs19, vs18
lxv vs0, (0+\OffsetA)(AO) // load real,imag from A
lxv vs8, (16+\OffsetA)(AO) // load real,imag from A
.endm
.macro END2x1_2
/*for load2 offset will be 32 and 64*/
KERNEL2x1_2 AO,BO, 32,64,0 ,1,1
.endm
.macro KERNEL2x1_E2 OffsetA,OffsetB, Index,IsLast
KERNEL2x1_2 AO,BO, \OffsetA,\OffsetB, \Index,\IsLast ,1
.endm
.macro KERNEL2x1_L2 OffsetA,OffsetB, Index,IsLast
KERNEL2x1_2 AO,BO, \OffsetA,\OffsetB, \Index,\IsLast ,0
.endm
.macro KERNEL2x1_2 AREG,BREG, OffsetA,OffsetB, Index,IsLast ,Complete
xxswapd vs21, vs20
xxswapd vs23, vs22
xvmaddadp vs32, vs0, vs16
xvmaddadp vs34, vs0, vs18
xvmaddadp vs33, vs0, vs17
xvmaddadp vs35, vs0, vs19
.if \Complete==0
lxv vs0, DISP2(\Index, 0 + \OffsetA)(\AREG) // load real,imag from A
.endif
.if \Complete==0
lxv vs16, DISP4(\Index, 0+\OffsetB)(\BREG) // load real imag from B
lxv vs18, DISP4(\Index, 16+\OffsetB)(\BREG) // load real,imag from B
.endif
.if \Complete==0
xxswapd vs17, vs16
xxswapd vs19, vs18
.endif
xvmaddadp vs32, vs8, vs20
xvmaddadp vs34, vs8, vs22
xvmaddadp vs33, vs8, vs21
xvmaddadp vs35, vs8, vs23
.if \Complete==0
lxv vs8, DISP2(\Index,16+0+ \OffsetA)(\AREG) // load real,imag from A
.endif
.if \Complete==0
lxv vs20, DISP4(\Index, 32+\OffsetB)(\BREG) // load real,imag from B
lxv vs22, DISP4(\Index, 48+\OffsetB)(\BREG) // load real,imag from B
.endif
.if \IsLast==1
.if \Complete==1
addi \AREG, \AREG, DISP2(\Index,\OffsetA)
addi \BREG, \BREG, DISP4(\Index,\OffsetB)
.else
addi \AREG, \AREG, DISP2(\Index,32)
addi \BREG, \BREG, DISP4(\Index,64)
.endif
.endif
.endm
.macro KERNEL2x1
LOAD2x1
END2x1 AO, BO, 16,32
.endm
.macro SAVE2x1
add T1, CO ,LDC
SAVE1 vs32,vs33,CO,0
SAVE1 vs34,vs35,T1,0
addi CO, CO, 16
.endm
/**********************************************************************************************
*
.macros for N=1 and M=8
**********************************************************************************************/
.macro Zero1x8
xxlxor vs32, vs32, vs32
xxlxor vs33, vs33, vs33
xxlxor vs34, vs34, vs34
xxlxor vs35, vs35, vs35
xxlxor vs36, vs36, vs36
xxlxor vs37, vs37, vs37
xxlxor vs38, vs38, vs38
xxlxor vs39, vs39, vs39
xxlxor vs40, vs40, vs40
xxlxor vs41, vs41, vs41
xxlxor vs42, vs42, vs42
xxlxor vs43, vs43, vs43
xxlxor vs44, vs44, vs44
xxlxor vs45, vs45, vs45
xxlxor vs46, vs46, vs46
xxlxor vs47, vs47, vs47
xxlxor vs48, vs48, vs48
.endm
.macro LOAD1x8
LOAD1x8O 0,0
.endm
.macro LOAD1x8O OffsetA,OffsetB
lxv vs16,(\OffsetB+ 0)(BO) // load real imag from B
xxswapd vs17, vs16
lxv vs0, (0+\OffsetA)(AO) // load real,imag from A
lxv vs1, (16+\OffsetA)(AO) // load real,imag from A
lxv vs2, (32+\OffsetA)(AO) // load real,imag from A
lxv vs3, (48+\OffsetA)(AO) // load real,imag from A
lxv vs4, (64+\OffsetA)(AO) // load real,imag from A
lxv vs5, (80+\OffsetA)(AO) // load real,imag from A
lxv vs6, (96+\OffsetA)(AO) // load real,imag from A
lxv vs7, (112+\OffsetA)(AO) // load real,imag from A
.endm
.macro END1x8_NORMAL
END1x8 AO,BO,128,16
.endm
.macro END1x8_WITHOUT_ADD
END1x8 AO,BO,0,0
.endm
.macro END1x8 AREG, BREG, OffsetA, OffsetB
.if \OffsetB != 0
addi \BREG, \BREG, \OffsetB
.endif
.if \OffsetA != 0
addi \AREG, \AREG, \OffsetA
.endif
xvmaddadp vs32, vs0, vs16
xvmaddadp vs33, vs0, vs17
xvmaddadp vs34, vs1, vs16
xvmaddadp vs35, vs1, vs17
xvmaddadp vs36, vs2, vs16
xvmaddadp vs37, vs2, vs17
xvmaddadp vs38, vs3, vs16
xvmaddadp vs39, vs3, vs17
xvmaddadp vs40, vs4, vs16
xvmaddadp vs41, vs4, vs17
xvmaddadp vs42, vs5, vs16
xvmaddadp vs43, vs5, vs17
xvmaddadp vs44, vs6, vs16
xvmaddadp vs45, vs6, vs17
xvmaddadp vs46, vs7, vs16
xvmaddadp vs47, vs7, vs17
.endm
.macro LOAD1x8_2
LOAD1x8_2O 0,0
.endm
.macro LOAD1x8_2O OffsetA,OffsetB
lxv vs16,(\OffsetB+ 0)(BO) // load real imag from B
lxv vs20, (\OffsetB+16)(BO) // load real,imag from B
xxswapd vs17, vs16
lxv vs0, (0+\OffsetA)(AO) // load real,imag from A
lxv vs1, (16+\OffsetA)(AO) // load real,imag from A
lxv vs2, (32+\OffsetA)(AO) // load real,imag from A
lxv vs3, (48+\OffsetA)(AO) // load real,imag from A
lxv vs4, (64+\OffsetA)(AO) // load real,imag from A
lxv vs5, (80+\OffsetA)(AO) // load real,imag from A
lxv vs6, (96+\OffsetA)(AO) // load real,imag from A
lxv vs7, (112+\OffsetA)(AO) // load real,imag from A
lxv vs8, (128+0+\OffsetA)(AO) // load real,imag from A
lxv vs9, (128+16+\OffsetA)(AO) // load real,imag from A
lxv vs10, (128+32+\OffsetA)(AO) // load real,imag from A
lxv vs11, (128+48+\OffsetA)(AO) // load real,imag from A
lxv vs12, (128+64+\OffsetA)(AO) // load real,imag from A
lxv vs13, (128+80+\OffsetA)(AO) // load real,imag from A
lxv vs14, (128+96+\OffsetA)(AO) // load real,imag from A
lxv vs15, (128+112+\OffsetA)(AO) // load real,imag from A
.endm
.macro END1x8_2
/*for load2 offset will be 256 and 32*/
KERNEL1x8_2 AO,BO, 256,32,0 ,1,1
.endm
.macro KERNEL1x8_E2 OffsetA,OffsetB, Index,IsLast
KERNEL1x8_2 AO,BO, \OffsetA,\OffsetB, \Index,\IsLast ,1
.endm
.macro KERNEL1x8_L2 OffsetA,OffsetB, Index,IsLast
KERNEL1x8_2 AO,BO, \OffsetA,\OffsetB, \Index,\IsLast ,0
.endm
.macro KERNEL1x8_2 AREG,BREG, OffsetA,OffsetB, Index,IsLast ,Complete
xvmaddadp vs32, vs0, vs16
xvmaddadp vs33, vs0, vs17
xxswapd vs21, vs20
xvmaddadp vs34, vs1, vs16
xvmaddadp vs35, vs1, vs17
.if \Complete==0
lxv vs0, DISP16(\Index, 0 + \OffsetA)(\AREG) // load real,imag from A
lxv vs1, DISP16(\Index,16 + \OffsetA)(\AREG) // load real,imag from A
.endif
xvmaddadp vs36, vs2, vs16
xvmaddadp vs37, vs2, vs17
xvmaddadp vs38, vs3, vs16
xvmaddadp vs39, vs3, vs17
.if \Complete==0
lxv vs2, DISP16(\Index,32 + \OffsetA)(\AREG) // load real,imag from A
lxv vs3, DISP16(\Index,48 + \OffsetA)(\AREG) // load real,imag from A
.endif
xvmaddadp vs40, vs4, vs16
xvmaddadp vs41, vs4, vs17
xvmaddadp vs42, vs5, vs16
xvmaddadp vs43, vs5, vs17
.if \Complete==0
lxv vs4, DISP16(\Index,64+ \OffsetA)(\AREG) // load real,imag from A
lxv vs5, DISP16(\Index,64+16 + \OffsetA)(\AREG) // load real,imag from A
.endif
xvmaddadp vs44, vs6, vs16
xvmaddadp vs45, vs6, vs17
xvmaddadp vs46, vs7, vs16
xvmaddadp vs47, vs7, vs17
.if \Complete==0
lxv vs16, DISP2(\Index, 0+\OffsetB)(\BREG) // load real imag from B
.endif
.if \Complete==0
xxswapd vs17, vs16
.endif
xvmaddadp vs32, vs8, vs20
xvmaddadp vs33, vs8, vs21
.if \Complete==0
lxv vs6, DISP16(\Index,64+32 + \OffsetA)(\AREG) // load real,imag from A
lxv vs7, DISP16(\Index,64+48 + \OffsetA)(\AREG) // load real,imag from A
.endif
xvmaddadp vs34, vs9, vs20
xvmaddadp vs35, vs9, vs21
.if \Complete==0
lxv vs8, DISP16(\Index,128+ + \OffsetA)(\AREG) // load real,imag from A
lxv vs9, DISP16(\Index,128+16 + \OffsetA)(\AREG) // load real,imag from A
.endif
xvmaddadp vs36, vs10, vs20
xvmaddadp vs37, vs10, vs21
xvmaddadp vs38, vs11, vs20
xvmaddadp vs39, vs11, vs21
.if \Complete==0
lxv vs10, DISP16(\Index,128+32 + \OffsetA)(\AREG) // load real,imag from A
lxv vs11, DISP16(\Index,128+48 + \OffsetA)(\AREG) // load real,imag from A
.endif
xvmaddadp vs40, vs12, vs20
xvmaddadp vs41, vs12, vs21
xvmaddadp vs42, vs13, vs20
xvmaddadp vs43, vs13, vs21
.if \Complete==0
lxv vs12, DISP16(\Index, 192 + \OffsetA)(\AREG) // load real,imag from A
lxv vs13, DISP16(\Index,192 +16 + \OffsetA)(\AREG) // load real,imag from A
.endif
xvmaddadp vs44, vs14, vs20
xvmaddadp vs45, vs14, vs21
xvmaddadp vs46, vs15, vs20
xvmaddadp vs47, vs15, vs21
.if \Complete==0
lxv vs14, DISP16(\Index,192 +32 + \OffsetA)(\AREG) // load real,imag from A
lxv vs15, DISP16(\Index,192 +48 + \OffsetA)(\AREG) // load real,imag from A
lxv vs20, DISP2(\Index, 16+\OffsetB)(\BREG) // load real,imag from B
.endif
.if \IsLast==1
.if \Complete==1
addi \AREG, \AREG, DISP16(\Index,\OffsetA)
addi \BREG, \BREG, DISP2(\Index,\OffsetB)
.else
addi \AREG, \AREG, DISP16(\Index,256)
addi \BREG, \BREG, DISP2(\Index,32)
.endif
.endif
.endm
.macro KERNEL1x8
LOAD1x8
END1x8 AO, BO, 128,16
.endm
.macro SAVE1x8
SAVE8 vs32,vs33,vs34,vs35,vs36,vs37,vs38,vs39,vs40,vs41,vs42,vs43,vs44,vs45,vs46,vs47,CO,0
addi CO, CO, 128
.endm
/**********************************************************************************************
*
.macros for N=2 and M=4
**********************************************************************************************/
.macro Zero1x4
xxlxor vs32, vs32, vs32
xxlxor vs33, vs33, vs33
xxlxor vs34, vs34, vs34
xxlxor vs35, vs35, vs35
xxlxor vs36, vs36, vs36
xxlxor vs37, vs37, vs37
xxlxor vs38, vs38, vs38
xxlxor vs39, vs39, vs39
.endm
.macro LOAD1x4
LOAD1x4O 0,0
.endm
.macro LOAD1x4O OffsetA,OffsetB
lxv vs16,(\OffsetB+ 0)(BO) // load real imag from B
xxswapd vs17, vs16
lxv vs0, (0+\OffsetA)(AO) // load real,imag from A
lxv vs1, (16+\OffsetA)(AO) // load real,imag from A
lxv vs2, (32+\OffsetA)(AO) // load real,imag from A
lxv vs3, (48+\OffsetA)(AO) // load real,imag from A
.endm
.macro END1x4_NORMAL
END1x4 AO,BO,64,16
.endm
.macro END1x4_WITHOUT_ADD
END1x4 AO,BO,0,0
.endm
.macro END1x4 AREG, BREG, OffsetA, OffsetB
.if \OffsetB != 0
addi \BREG, \BREG, \OffsetB
.endif
.if \OffsetA != 0
addi \AREG, \AREG, \OffsetA
.endif
xvmaddadp vs32, vs0, vs16
xvmaddadp vs33, vs0, vs17
xvmaddadp vs34, vs1, vs16
xvmaddadp vs35, vs1, vs17
xvmaddadp vs36, vs2, vs16
xvmaddadp vs37, vs2, vs17
xvmaddadp vs38, vs3, vs16
xvmaddadp vs39, vs3, vs17
.endm
.macro LOAD1x4_2
LOAD1x4_2O 0,0
.endm
.macro LOAD1x4_2O OffsetA,OffsetB
lxv vs16,(\OffsetB+ 0)(BO) // load real imag from B
lxv vs20, (\OffsetB+16)(BO) // load real,imag from B
xxswapd vs17, vs16
lxv vs0, (0+\OffsetA)(AO) // load real,imag from A
lxv vs1, (16+\OffsetA)(AO) // load real,imag from A
lxv vs2, (32+\OffsetA)(AO) // load real,imag from A
lxv vs3, (48+\OffsetA)(AO) // load real,imag from A
lxv vs8, (64+\OffsetA)(AO) // load real,imag from A
lxv vs9, (80+\OffsetA)(AO) // load real,imag from A
lxv vs10, (96+\OffsetA)(AO) // load real,imag from A
lxv vs11, (112+\OffsetA)(AO) // load real,imag from A
.endm
.macro END1x4_2
/*for load2 offset will be 128 and 32*/
KERNEL1x4_2 AO,BO, 128,32,0 ,1,1
.endm
.macro KERNEL1x4_E2 OffsetA,OffsetB, Index,IsLast
KERNEL1x4_2 AO,BO, \OffsetA,\OffsetB, \Index,\IsLast ,1
.endm
.macro KERNEL1x4_L2 OffsetA,OffsetB, Index,IsLast
KERNEL1x4_2 AO,BO, \OffsetA,\OffsetB, \Index,\IsLast ,0
.endm
.macro KERNEL1x4_2 AREG,BREG, OffsetA,OffsetB, Index,IsLast ,Complete
xvmaddadp vs32, vs0, vs16
xvmaddadp vs33, vs0, vs17
xxswapd vs21, vs20
xvmaddadp vs34, vs1, vs16
xvmaddadp vs35, vs1, vs17
.if \Complete==0
lxv vs0, DISP8(\Index, 0 + \OffsetA)(\AREG) // load real,imag from A
lxv vs1, DISP8(\Index,16 + \OffsetA)(\AREG) // load real,imag from A
.endif
xvmaddadp vs36, vs2, vs16
xvmaddadp vs37, vs2, vs17
xvmaddadp vs38, vs3, vs16
xvmaddadp vs39, vs3, vs17
.if \Complete==0
lxv vs2, DISP8(\Index,32 + \OffsetA)(\AREG) // load real,imag from A
lxv vs3, DISP8(\Index,48 + \OffsetA)(\AREG) // load real,imag from A
.endif
.if \Complete==0
lxv vs16, DISP2(\Index, 0+\OffsetB)(\BREG) // load real imag from B
.endif
xvmaddadp vs32, vs8, vs20
xvmaddadp vs33, vs8, vs21
.if \Complete==0
xxswapd vs17, vs16
.endif
xvmaddadp vs34, vs9, vs20
xvmaddadp vs35, vs9, vs21
.if \Complete==0
lxv vs8, DISP8(\Index,64+0+ \OffsetA)(\AREG) // load real,imag from A
lxv vs9, DISP8(\Index,64+16 + \OffsetA)(\AREG) // load real,imag from A
.endif
xvmaddadp vs36, vs10, vs20
xvmaddadp vs37, vs10, vs21
xvmaddadp vs38, vs11, vs20
xvmaddadp vs39, vs11, vs21
.if \Complete==0
lxv vs10, DISP8(\Index,64+32 + \OffsetA)(\AREG) // load real,imag from A
lxv vs11, DISP8(\Index,64+48 + \OffsetA)(\AREG) // load real,imag from A
.endif
.if \Complete==0
lxv vs20, DISP2(\Index, 16+\OffsetB)(\BREG) // load real,imag from B
.endif
.if \IsLast==1
.if \Complete==1
addi \AREG, \AREG, DISP8(\Index,\OffsetA)
addi \BREG, \BREG, DISP2(\Index,\OffsetB)
.else
addi \AREG, \AREG, DISP8(\Index,128)
addi \BREG, \BREG, DISP2(\Index,32)
.endif
.endif
.endm
.macro KERNEL1x4
LOAD1x4
END1x4 AO, BO, 64,16
.endm
.macro SAVE1x4
SAVE4 vs32,vs33,vs34,vs35,vs36,vs37,vs38,vs39,CO,0
addi CO, CO, 64
.endm
/**********************************************************************************************
*
.macros for N=2 and M=2
**********************************************************************************************/
.macro Zero1x2
xxlxor vs32, vs32, vs32
xxlxor vs33, vs33, vs33
xxlxor vs34, vs34, vs34
xxlxor vs35, vs35, vs35
.endm
.macro LOAD1x2
LOAD1x2O 0,0
.endm
.macro LOAD1x2O OffsetA,OffsetB
lxv vs16,(\OffsetB+ 0)(BO) // load real imag from B
xxswapd vs17, vs16
lxv vs0, (0+\OffsetA)(AO) // load real,imag from A
lxv vs1, (16+\OffsetA)(AO) // load real,imag from A
.endm
.macro END1x2_NORMAL
END1x2 AO,BO,32,16
.endm
.macro END1x2_WITHOUT_ADD
END1x2 AO,BO,0,0
.endm
.macro END1x2 AREG, BREG, OffsetA, OffsetB
.if \OffsetB != 0
addi \BREG, \BREG, \OffsetB
.endif
.if \OffsetA != 0
addi \AREG, \AREG, \OffsetA
.endif
xvmaddadp vs32, vs0, vs16
xvmaddadp vs33, vs0, vs17
xvmaddadp vs34, vs1, vs16
xvmaddadp vs35, vs1, vs17
.endm
.macro LOAD1x2_2
LOAD1x2_2O 0,0
.endm
.macro LOAD1x2_2O OffsetA,OffsetB
lxv vs16,(\OffsetB+ 0)(BO) // load real imag from B
lxv vs20, (\OffsetB+16)(BO) // load real,imag from B
xxswapd vs17, vs16
lxv vs0, (0+\OffsetA)(AO) // load real,imag from A
lxv vs1, (16+\OffsetA)(AO) // load real,imag from A
lxv vs8, (32+\OffsetA)(AO) // load real,imag from A
lxv vs9, (48+\OffsetA)(AO) // load real,imag from A
.endm
.macro END1x2_2
/*for load2 offset will be 64 and 32*/
KERNEL1x2_2 AO,BO, 64,32,0 ,1,1
.endm
.macro KERNEL1x2_E2 OffsetA,OffsetB, Index,IsLast
KERNEL1x2_2 AO,BO, \OffsetA,\OffsetB, \Index,\IsLast ,1
.endm
.macro KERNEL1x2_L2 OffsetA,OffsetB, Index,IsLast
KERNEL1x2_2 AO,BO, \OffsetA,\OffsetB, \Index,\IsLast ,0
.endm
.macro KERNEL1x2_2 AREG,BREG, OffsetA,OffsetB, Index,IsLast ,Complete
xvmaddadp vs32, vs0, vs16
xvmaddadp vs33, vs0, vs17
xxswapd vs21, vs20
xvmaddadp vs34, vs1, vs16
xvmaddadp vs35, vs1, vs17
.if \Complete==0
lxv vs0, DISP4(\Index, 0 + \OffsetA)(\AREG) // load real,imag from A
lxv vs1, DISP4(\Index,16 + \OffsetA)(\AREG) // load real,imag from A
.endif
.if \Complete==0
lxv vs16, DISP2(\Index, 0+\OffsetB)(\BREG) // load real imag from B
.endif
xvmaddadp vs32, vs8, vs20
xvmaddadp vs33, vs8, vs21
.if \Complete==0
xxswapd vs17, vs16
.endif
xvmaddadp vs34, vs9, vs20
xvmaddadp vs35, vs9, vs21
.if \Complete==0
lxv vs20, DISP2(\Index, 16+\OffsetB)(\BREG) // load real,imag from B
.endif
.if \Complete==0
lxv vs8, DISP4(\Index,32+0+ \OffsetA)(\AREG) // load real,imag from A
lxv vs9, DISP4(\Index,32+16 + \OffsetA)(\AREG) // load real,imag from A
.endif
.if \IsLast==1
.if \Complete==1
addi \AREG, \AREG, DISP4(\Index,\OffsetA)
addi \BREG, \BREG, DISP2(\Index,\OffsetB)
.else
addi \AREG, \AREG, DISP4(\Index,64)
addi \BREG, \BREG, DISP2(\Index,32)
.endif
.endif
.endm
.macro KERNEL1x2
LOAD1x2
END1x2 AO, BO, 32,16
.endm
.macro SAVE1x2
SAVE2 vs32,vs33,vs34,vs35,CO,0
addi CO, CO, 32
.endm
/**********************************************************************************************
*
.macros for N=2 and M=1
**********************************************************************************************/
.macro Zero1x1
xxlxor vs32, vs32, vs32
xxlxor vs33, vs33, vs33
.endm
.macro LOAD1x1
LOAD1x1O 0,0
.endm
.macro LOAD1x1O OffsetA,OffsetB
lxv vs16,(\OffsetB+ 0)(BO) // load real imag from B
lxv vs0, (0+\OffsetA)(AO) // load real,imag from A
xxswapd vs17, vs16
.endm
.macro END1x1_NORMAL
END1x1 AO,BO,16,16
.endm
.macro END1x1_WITHOUT_ADD
END1x1 AO,BO,0,0
.endm
.macro END1x1 AREG, BREG, OffsetA, OffsetB
.if \OffsetB != 0
addi \BREG, \BREG, \OffsetB
.endif
.if \OffsetA != 0
addi \AREG, \AREG, \OffsetA
.endif
xvmaddadp vs32, vs0, vs16
xvmaddadp vs33, vs0, vs17
.endm
.macro LOAD1x1_2
LOAD1x1_2O 0,0
.endm
.macro LOAD1x1_2O OffsetA,OffsetB
lxv vs16,(\OffsetB+ 0)(BO) // load real imag from B
lxv vs20, (\OffsetB+16)(BO) // load real,imag from B
xxswapd vs17, vs16
lxv vs0, (0+\OffsetA)(AO) // load real,imag from A
lxv vs8, (16+\OffsetA)(AO) // load real,imag from A
.endm
.macro END1x1_2
/*for load2 offset will be 32 and 32*/
KERNEL1x1_2 AO,BO, 32,32,0 ,1,1
.endm
.macro KERNEL1x1_E2 OffsetA,OffsetB, Index,IsLast
KERNEL1x1_2 AO,BO, \OffsetA,\OffsetB, \Index,\IsLast ,1
.endm
.macro KERNEL1x1_L2 OffsetA,OffsetB, Index,IsLast
KERNEL1x1_2 AO,BO, \OffsetA,\OffsetB, \Index,\IsLast ,0
.endm
.macro KERNEL1x1_2 AREG,BREG, OffsetA,OffsetB, Index,IsLast ,Complete
xxswapd vs21, vs20
xvmaddadp vs32, vs0, vs16
xvmaddadp vs33, vs0, vs17
.if \Complete==0
lxv vs0, DISP2(\Index, 0 + \OffsetA)(\AREG) // load real,imag from A
.endif
.if \Complete==0
lxv vs16, DISP2(\Index, 0+\OffsetB)(\BREG) // load real imag from B
.endif
.if \Complete==0
xxswapd vs17, vs16
.endif
xvmaddadp vs32, vs8, vs20
xvmaddadp vs33, vs8, vs21
.if \Complete==0
lxv vs8, DISP2(\Index,16+0+ \OffsetA)(\AREG) // load real,imag from A
.endif
.if \Complete==0
lxv vs20, DISP2(\Index, 16+\OffsetB)(\BREG) // load real,imag from B
.endif
.if \IsLast==1
.if \Complete==1
addi \AREG, \AREG, DISP2(\Index,\OffsetA)
addi \BREG, \BREG, DISP2(\Index,\OffsetB)
.else
addi \AREG, \AREG, DISP2(\Index,32)
addi \BREG, \BREG, DISP2(\Index,32)
.endif
.endif
.endm
.macro KERNEL1x1
LOAD1x1
END1x1 AO, BO, 16,16
.endm
.macro SAVE1x1
SAVE1 vs32,vs33,CO,0
addi CO, CO, 16
.endm
/****************************TRMM POINTER REFRESH
.macroSES*************************/
.macro SHIFT_REG REG1,REG2,SHIFT_VAL
.if \SHIFT_VAL==16
slwi \REG1, \REG2, 8
.elseif \SHIFT_VAL==8
slwi \REG1, \REG2, 7
.elseif \SHIFT_VAL==4
slwi \REG1, \REG2, 6
.elseif \SHIFT_VAL==2
slwi \REG1, \REG2, 5
.elseif \SHIFT_VAL==1
slwi \REG1, \REG2, 4
.endif
.endm
/*
//#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
// ptrbb = bb;
// #else
// ptrba += off*16;
// ptrbb = bb + off*2;
// #endif
*/
.macro REFRESH_POINTERS PTR_A,PTR_B,OFF_VAL,B_VAL,C_A,C_B
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
/* ptrbb = bb;*/
mr \PTR_B,\B_VAL /* refresh BPOINT */
#else
/*
// ptrba =ptrba+ off*C_A;
// ptrbb = bb + off*C_B;
*/
SHIFT_REG T4,\OFF_VAL,\C_B /* Number of values in B shifted */
SHIFT_REG T2,\OFF_VAL,\C_A /* Number of values in A shifted */
add \PTR_B, \B_VAL , T4 /* Add values to BO */
add \PTR_A, \PTR_A, T2 /* Add values to AO */
#endif
.endm
/*
// #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
// temp = bk-off;
// #elif defined(LEFT)
// temp = off+16; // number of values in A
// #else
// temp = off+2; // number of values in B
// #endif
*/
.macro REFRESH_TEMP_BK TEMP_BK,BK_VAL,OFF_VAL,INCR_A,INCR_B
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
/* temp = bk-off;*/
sub \TEMP_BK,\BK_VAL,\OFF_VAL
#elif defined(LEFT)
/* temp = off+INCR_A; // number of values in A */
addi \TEMP_BK, \OFF_VAL, \INCR_A
#else
/* temp = off+INCR_B // number of values in B*/
addi \TEMP_BK,\OFF_VAL, \INCR_B
#endif
.endm
/*
// #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
// temp = bk - off;
// #ifdef LEFT
// temp -= 16; // number of values in A
// #else
// temp -= 2; // number of values in B
// #endif
// ptrba += temp*16;
// ptrbb += temp*2;
// #endif
// #ifdef LEFT
// off += 16; // number of values in A
// #endif
*/
.macro REFRESH_AFTER_SAVE TEMP_BK,BK_VAL,OFF_VAL,PTR_B,PTR_A,C_A,C_B
#if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
/*temp = bk - off;*/
sub \TEMP_BK,\BK_VAL,\OFF_VAL
#ifdef LEFT
/*temp -= 8; // number of values in A*/
addi \TEMP_BK,\TEMP_BK,-\C_A
#else
/*temp -= 4; // number of values in B*/
addi \TEMP_BK,\TEMP_BK,-\C_B
#endif
/*ptrba += temp*C_A;
ptrbb += temp*C_B;*/
SHIFT_REG T4,\TEMP_BK,\C_A
SHIFT_REG T2,\TEMP_BK,\C_B
add \PTR_A, \PTR_A,T4/*ptrba+temp*C_A*/
add \PTR_B, \PTR_B,T2
#endif
#ifdef LEFT
/*off += 8; // number of values in A*/
addi \OFF_VAL,\OFF_VAL,\C_A
#endif
.endm
Reference in New Issue View Git Blame Copy Permalink