floraachy
/
OpenBLAS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
OpenBLAS/kernel/arm64/cgemm_kernel_8x4.S

2163 lines
43 KiB
ArmAsm
Raw Permalink Blame History

/*******************************************************************************
Copyright (c) 2015, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*******************************************************************************/
#define ASSEMBLER
#include "common.h"
/* X0 X1 X2 s0 X3 x4 x5 x6 */
/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha0,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc */
#define origM x0
#define origN x1
#define origK x2
#define origPA x3
#define origPB x4
#define pC x5
#define LDC x6
#define temp x7
#define counterL x8
#define counterI x9
#define counterJ x10
#define pB x11
#define pCRow0 x12
#define pCRow1 x13
#define pCRow2 x14
#define pCRow3 x15
#define pA x16
#define alphaR w17
#define alphaI w19
#define alpha0_R s10
#define alphaV0_R v10.s[0]
#define alpha0_I s11
#define alphaV0_I v11.s[0]
#define A_PRE_SIZE 2560
#define B_PRE_SIZE 448
#define C_PRE_SIZE 128
#if defined(NN) || defined(NT) || defined(TN) || defined(TT)
#define OP_rr fmla
#define OP_ii fmls
#define OP_ri fmla
#define OP_ir fmla
#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
#define OP_rr fmla
#define OP_ii fmla
#define OP_ri fmls
#define OP_ir fmla
#elif defined(RN) || defined(RT) || defined(CN) || defined(CT)
#define OP_rr fmla
#define OP_ii fmla
#define OP_ri fmla
#define OP_ir fmls
#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
#define OP_rr fmla
#define OP_ii fmls
#define OP_ri fmls
#define OP_ir fmls
#endif
// 00 origM
// 01 origN
// 02 origK
// 03 origPA
// 04 origPB
// 05 pC
// 06 origLDC -> LDC
// 07 offset -> temp
// 08 counterL
// 09 counterI
// 10 counterJ
// 11 pB
// 12 pCRow0
// 13 pCRow1
// 14 pCRow2
// 15 pCRow3
// 16 pA
// 17
// 18 must save
// 19 must save
// 20 must save
// 21 must save
// 22 must save
// 23 must save
// 24 must save
// 25 must save
// 26 must save
// 27 must save
// 28 must save
// 29 frame
// 30 link
// 31 sp
//v00 ALPHA_R -> pA0_00_R, pA0_01_R, pA0_02_R, pA0_03_R
//v01 ALPHA_I -> pA0_00_I, pA0_01_I, pA0_02_I, pA0_03_I
//v02 pA0_04_R, pA0_05_R, pA0_06_R, pA0_07_R
//v03 pA0_04_I, pA0_05_I, pA0_06_I, pA0_07_I
//v04 pA1_00_R, pA1_01_R, pA1_02_R, pA1_03_R
//v05 pA1_00_I, pA1_01_I, pA1_02_I, pA1_03_I
//v06 pA1_04_R, pA1_05_R, pA1_06_R, pA1_07_R
//v07 pA1_04_I, pA1_05_I, pA1_06_I, pA1_07_I
//v08 must save pB0_00_R, pB0_01_R
//v09 must save pB0_00_I, pB0_01_I
//v10 must save pB0_02_R, pB0_03_R --> ALPHA0_R
//v11 must save pB0_02_I, pB0_03_I --> ALPHA0_I
//v12 must save pB1_00_R, pB1_01_R
//v13 must save pB1_00_I, pB1_01_I
//v14 must save pB1_02_R, pB1_03_R
//v15 must save pB1_02_I, pB1_03_I
//v16 must save pC_00_R, pC_01_R, pC_02_R, pC_03_R
//v17 must save pC_00_I, pC_01_I, pC_02_I, pC_03_I
//v18 pC_04_R, pC_05_R, pC_06_R, pC_07_R
//v19 pC_04_I, pC_05_I, pC_06_I, pC_07_I
//v20 pC_08_R, pC_09_R, pC_10_R, pC_11_R
//v21 pC_08_I, pC_09_I, pC_10_I, pC_11_I
//v22 pC_12_R, pC_13_R, pC_14_R, pC_15_R
//v23 pC_12_I, pC_13_I, pC_14_I, pC_15_I
//v24 pC_16_R, pC_17_R, pC_18_R, pC_19_R
//v25 pC_16_I, pC_17_I, pC_18_I, pC_19_I
//v26 pC_20_R, pC_21_R, pC_22_R, pC_23_R
//v27 pC_20_I, pC_21_I, pC_22_I, pC_23_I
//v28 pC_24_R, pC_25_R, pC_26_R, pC_27_R
//v29 pC_24_I, pC_25_I, pC_26_I, pC_27_I
//v30 pC_28_R, pC_29_R, pC_30_R, pC_31_R
//v31 pC_28_I, pC_29_I, pC_30_I, pC_31_I
/*******************************************************************************
* Macro definitions
*******************************************************************************/
.macro INIT8x4
fmov s16, wzr
fmov s17, wzr
fmov s18, wzr
fmov s19, s16
fmov s20, wzr
fmov s21, s16
fmov s22, s17
fmov s23, s18
fmov s24, wzr
fmov s25, s16
fmov s26, s17
fmov s27, s18
fmov s28, wzr
fmov s29, s16
fmov s30, s17
fmov s31, s18
.endm
.macro KERNEL8x4_I
ld2 {v8.2s, v9.2s}, [pB]
add pB, pB, #16
ld2 {v0.4s, v1.4s}, [pA]
add pA, pA, #32
ld2 {v2.4s, v3.4s}, [pA]
add pA, pA, #32
fmul v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.s[0]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v17.16b, v17.16b, v17.16b
fmls v17.4s, v0.4s, v9.s[0]
#else
fmul v17.4s, v0.4s, v9.s[0]
#endif
OP_ir v17.4s, v1.4s, v8.s[0]
ld2 {v10.2s, v11.2s}, [pB]
add pB, pB, #16
fmul v18.4s, v2.4s, v8.s[0]
OP_ii v18.4s, v3.4s, v9.s[0]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v19.16b, v19.16b, v19.16b
fmls v19.4s, v2.4s, v9.s[0]
#else
fmul v19.4s, v2.4s, v9.s[0]
#endif
OP_ir v19.4s, v3.4s, v8.s[0]
ld2 {v12.2s, v13.2s}, [pB]
add pB, pB, #16
fmul v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.s[1]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v21.16b, v21.16b, v21.16b
fmls v21.4s, v0.4s, v9.s[1]
#else
fmul v21.4s, v0.4s, v9.s[1]
#endif
OP_ir v21.4s, v1.4s, v8.s[1]
ld2 {v14.2s, v15.2s}, [pB]
add pB, pB, #16
fmul v22.4s, v2.4s, v8.s[1]
OP_ii v22.4s, v3.4s, v9.s[1]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v23.16b, v23.16b, v23.16b
fmls v23.4s, v2.4s, v9.s[1]
#else
fmul v23.4s, v2.4s, v9.s[1]
#endif
OP_ir v23.4s, v3.4s, v8.s[1]
ld2 {v4.4s, v5.4s}, [pA]
add pA, pA, #32
fmul v24.4s, v0.4s, v10.s[0]
OP_ii v24.4s, v1.4s, v11.s[0]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v25.16b, v25.16b, v25.16b
fmls v25.4s, v0.4s, v11.s[0]
#else
fmul v25.4s, v0.4s, v11.s[0]
#endif
OP_ir v25.4s, v1.4s, v10.s[0]
ld2 {v6.4s, v7.4s}, [pA]
add pA, pA, #32
fmul v26.4s, v2.4s, v10.s[0]
OP_ii v26.4s, v3.4s, v11.s[0]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v27.16b, v27.16b, v27.16b
fmls v27.4s, v2.4s, v11.s[0]
#else
fmul v27.4s, v2.4s, v11.s[0]
#endif
OP_ir v27.4s, v3.4s, v10.s[0]
prfm PLDL1KEEP, [pA, #A_PRE_SIZE]
fmul v28.4s, v0.4s, v10.s[1]
OP_ii v28.4s, v1.4s, v11.s[1]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v29.16b, v29.16b, v29.16b
fmls v29.4s, v0.4s, v11.s[1]
#else
fmul v29.4s, v0.4s, v11.s[1]
#endif
OP_ir v29.4s, v1.4s, v10.s[1]
prfm PLDL1KEEP, [pA, #A_PRE_SIZE+64]
fmul v30.4s, v2.4s, v10.s[1]
OP_ii v30.4s, v3.4s, v11.s[1]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v31.16b, v31.16b, v31.16b
fmls v31.4s, v2.4s, v11.s[1]
#else
fmul v31.4s, v2.4s, v11.s[1]
#endif
OP_ir v31.4s, v3.4s, v10.s[1]
.endm
.macro KERNEL8x4_M1
OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.s[0]
ld2 {v12.2s, v13.2s}, [pB]
add pB, pB, #16
OP_rr v18.4s, v2.4s, v8.s[0]
OP_ii v18.4s, v3.4s, v9.s[0]
OP_ri v19.4s, v2.4s, v9.s[0]
OP_ir v19.4s, v3.4s, v8.s[0]
ld2 {v4.4s, v5.4s}, [pA]
add pA, pA, #32
OP_rr v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.s[1]
OP_ri v21.4s, v0.4s, v9.s[1]
OP_ir v21.4s, v1.4s, v8.s[1]
ld2 {v6.4s, v7.4s}, [pA]
add pA, pA, #32
OP_rr v22.4s, v2.4s, v8.s[1]
OP_ii v22.4s, v3.4s, v9.s[1]
OP_ri v23.4s, v2.4s, v9.s[1]
OP_ir v23.4s, v3.4s, v8.s[1]
ld2 {v14.2s, v15.2s}, [pB]
add pB, pB, #16
OP_rr v24.4s, v0.4s, v10.s[0]
OP_ii v24.4s, v1.4s, v11.s[0]
OP_ri v25.4s, v0.4s, v11.s[0]
OP_ir v25.4s, v1.4s, v10.s[0]
prfm PLDL1KEEP, [pA, #A_PRE_SIZE]
OP_rr v26.4s, v2.4s, v10.s[0]
OP_ii v26.4s, v3.4s, v11.s[0]
OP_ri v27.4s, v2.4s, v11.s[0]
OP_ir v27.4s, v3.4s, v10.s[0]
prfm PLDL1KEEP, [pA, #A_PRE_SIZE+64]
OP_rr v28.4s, v0.4s, v10.s[1]
OP_ii v28.4s, v1.4s, v11.s[1]
OP_ri v29.4s, v0.4s, v11.s[1]
OP_ir v29.4s, v1.4s, v10.s[1]
OP_rr v30.4s, v2.4s, v10.s[1]
OP_ii v30.4s, v3.4s, v11.s[1]
OP_ri v31.4s, v2.4s, v11.s[1]
OP_ir v31.4s, v3.4s, v10.s[1]
.endm
.macro KERNEL8x4_M2
OP_rr v16.4s, v4.4s, v12.s[0]
OP_ii v16.4s, v5.4s, v13.s[0]
OP_ri v17.4s, v4.4s, v13.s[0]
OP_ir v17.4s, v5.4s, v12.s[0]
ld2 {v8.2s, v9.2s}, [pB]
add pB, pB, #16
OP_rr v18.4s, v6.4s, v12.s[0]
OP_ii v18.4s, v7.4s, v13.s[0]
OP_ri v19.4s, v6.4s, v13.s[0]
OP_ir v19.4s, v7.4s, v12.s[0]
ld2 {v0.4s, v1.4s}, [pA]
add pA, pA, #32
OP_rr v20.4s, v4.4s, v12.s[1]
OP_ii v20.4s, v5.4s, v13.s[1]
OP_ri v21.4s, v4.4s, v13.s[1]
OP_ir v21.4s, v5.4s, v12.s[1]
ld2 {v2.4s, v3.4s}, [pA]
add pA, pA, #32
OP_rr v22.4s, v6.4s, v12.s[1]
OP_ii v22.4s, v7.4s, v13.s[1]
OP_ri v23.4s, v6.4s, v13.s[1]
OP_ir v23.4s, v7.4s, v12.s[1]
ld2 {v10.2s, v11.2s}, [pB]
add pB, pB, #16
OP_rr v24.4s, v4.4s, v14.s[0]
OP_ii v24.4s, v5.4s, v15.s[0]
OP_ri v25.4s, v4.4s, v15.s[0]
OP_ir v25.4s, v5.4s, v14.s[0]
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
OP_rr v26.4s, v6.4s, v14.s[0]
OP_ii v26.4s, v7.4s, v15.s[0]
OP_ri v27.4s, v6.4s, v15.s[0]
OP_ir v27.4s, v7.4s, v14.s[0]
OP_rr v28.4s, v4.4s, v14.s[1]
OP_ii v28.4s, v5.4s, v15.s[1]
OP_ri v29.4s, v4.4s, v15.s[1]
OP_ir v29.4s, v5.4s, v14.s[1]
OP_rr v30.4s, v6.4s, v14.s[1]
OP_ii v30.4s, v7.4s, v15.s[1]
OP_ri v31.4s, v6.4s, v15.s[1]
OP_ir v31.4s, v7.4s, v14.s[1]
.endm
.macro KERNEL8x4_E
OP_rr v16.4s, v4.4s, v12.s[0]
OP_ii v16.4s, v5.4s, v13.s[0]
OP_ri v17.4s, v4.4s, v13.s[0]
OP_ir v17.4s, v5.4s, v12.s[0]
OP_rr v18.4s, v6.4s, v12.s[0]
OP_ii v18.4s, v7.4s, v13.s[0]
OP_ri v19.4s, v6.4s, v13.s[0]
OP_ir v19.4s, v7.4s, v12.s[0]
OP_rr v20.4s, v4.4s, v12.s[1]
OP_ii v20.4s, v5.4s, v13.s[1]
OP_ri v21.4s, v4.4s, v13.s[1]
OP_ir v21.4s, v5.4s, v12.s[1]
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
OP_rr v22.4s, v6.4s, v12.s[1]
OP_ii v22.4s, v7.4s, v13.s[1]
OP_ri v23.4s, v6.4s, v13.s[1]
OP_ir v23.4s, v7.4s, v12.s[1]
OP_rr v24.4s, v4.4s, v14.s[0]
OP_ii v24.4s, v5.4s, v15.s[0]
OP_ri v25.4s, v4.4s, v15.s[0]
OP_ir v25.4s, v5.4s, v14.s[0]
OP_rr v26.4s, v6.4s, v14.s[0]
OP_ii v26.4s, v7.4s, v15.s[0]
OP_ri v27.4s, v6.4s, v15.s[0]
OP_ir v27.4s, v7.4s, v14.s[0]
OP_rr v28.4s, v4.4s, v14.s[1]
OP_ii v28.4s, v5.4s, v15.s[1]
OP_ri v29.4s, v4.4s, v15.s[1]
OP_ir v29.4s, v5.4s, v14.s[1]
OP_rr v30.4s, v6.4s, v14.s[1]
OP_ii v30.4s, v7.4s, v15.s[1]
OP_ri v31.4s, v6.4s, v15.s[1]
OP_ir v31.4s, v7.4s, v14.s[1]
.endm
.macro KERNEL8x4_SUB
ld2 {v8.2s, v9.2s}, [pB]
add pB, pB, #16
ld2 {v0.4s, v1.4s}, [pA]
add pA, pA, #32
OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.s[0]
ld2 {v2.4s, v3.4s}, [pA]
add pA, pA, #32
OP_rr v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.s[1]
OP_ri v21.4s, v0.4s, v9.s[1]
OP_ir v21.4s, v1.4s, v8.s[1]
ld2 {v10.2s, v11.2s}, [pB]
add pB, pB, #16
OP_rr v18.4s, v2.4s, v8.s[0]
OP_ii v18.4s, v3.4s, v9.s[0]
OP_ri v19.4s, v2.4s, v9.s[0]
OP_ir v19.4s, v3.4s, v8.s[0]
prfm PLDL1KEEP, [pA, #A_PRE_SIZE]
OP_rr v22.4s, v2.4s, v8.s[1]
OP_ii v22.4s, v3.4s, v9.s[1]
OP_ri v23.4s, v2.4s, v9.s[1]
OP_ir v23.4s, v3.4s, v8.s[1]
prfm PLDL1KEEP, [pA, #A_PRE_SIZE+64]
OP_rr v24.4s, v0.4s, v10.s[0]
OP_ii v24.4s, v1.4s, v11.s[0]
OP_ri v25.4s, v0.4s, v11.s[0]
OP_ir v25.4s, v1.4s, v10.s[0]
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
OP_rr v26.4s, v2.4s, v10.s[0]
OP_ii v26.4s, v3.4s, v11.s[0]
OP_ri v27.4s, v2.4s, v11.s[0]
OP_ir v27.4s, v3.4s, v10.s[0]
OP_rr v28.4s, v0.4s, v10.s[1]
OP_ii v28.4s, v1.4s, v11.s[1]
OP_ri v29.4s, v0.4s, v11.s[1]
OP_ir v29.4s, v1.4s, v10.s[1]
OP_rr v30.4s, v2.4s, v10.s[1]
OP_ii v30.4s, v3.4s, v11.s[1]
OP_ri v31.4s, v2.4s, v11.s[1]
OP_ir v31.4s, v3.4s, v10.s[1]
.endm
.macro SAVE8x4
fmov alpha0_R, alphaR
fmov alpha0_I, alphaI
prfm PLDL2KEEP, [pCRow0, #C_PRE_SIZE]
ld2 {v0.4s, v1.4s}, [pCRow0]
fmla v0.4s, v16.4s, alphaV0_R
fmls v0.4s, v17.4s, alphaV0_I
fmla v1.4s, v16.4s, alphaV0_I
fmla v1.4s, v17.4s, alphaV0_R
st2 {v0.4s, v1.4s}, [pCRow0]
add pCRow0, pCRow0, #32
ld2 {v2.4s, v3.4s}, [pCRow0]
fmla v2.4s, v18.4s, alphaV0_R
fmls v2.4s, v19.4s, alphaV0_I
fmla v3.4s, v18.4s, alphaV0_I
fmla v3.4s, v19.4s, alphaV0_R
st2 {v2.4s, v3.4s}, [pCRow0]
add pCRow0, pCRow0, #32
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
ld2 {v4.4s, v5.4s}, [pCRow1]
fmla v4.4s, v20.4s, alphaV0_R
fmls v4.4s, v21.4s, alphaV0_I
fmla v5.4s, v20.4s, alphaV0_I
fmla v5.4s, v21.4s, alphaV0_R
st2 {v4.4s, v5.4s}, [pCRow1]
add pCRow1, pCRow1, #32
ld2 {v6.4s, v7.4s}, [pCRow1]
fmla v6.4s, v22.4s, alphaV0_R
fmls v6.4s, v23.4s, alphaV0_I
fmla v7.4s, v22.4s, alphaV0_I
fmla v7.4s, v23.4s, alphaV0_R
st2 {v6.4s, v7.4s}, [pCRow1]
add pCRow1, pCRow1, #32
prfm PLDL2KEEP, [pCRow2, #C_PRE_SIZE]
ld2 {v0.4s, v1.4s}, [pCRow2]
fmla v0.4s, v24.4s, alphaV0_R
fmls v0.4s, v25.4s, alphaV0_I
fmla v1.4s, v24.4s, alphaV0_I
fmla v1.4s, v25.4s, alphaV0_R
st2 {v0.4s, v1.4s}, [pCRow2]
add pCRow2, pCRow2, #32
ld2 {v2.4s, v3.4s}, [pCRow2]
fmla v2.4s, v26.4s, alphaV0_R
fmls v2.4s, v27.4s, alphaV0_I
fmla v3.4s, v26.4s, alphaV0_I
fmla v3.4s, v27.4s, alphaV0_R
st2 {v2.4s, v3.4s}, [pCRow2]
add pCRow2, pCRow2, #32
prfm PLDL2KEEP, [pCRow3, #C_PRE_SIZE]
ld2 {v4.4s, v5.4s}, [pCRow3]
fmla v4.4s, v28.4s, alphaV0_R
fmls v4.4s, v29.4s, alphaV0_I
fmla v5.4s, v28.4s, alphaV0_I
fmla v5.4s, v29.4s, alphaV0_R
st2 {v4.4s, v5.4s}, [pCRow3]
add pCRow3, pCRow3, #32
ld2 {v6.4s, v7.4s}, [pCRow3]
fmla v6.4s, v30.4s, alphaV0_R
fmls v6.4s, v31.4s, alphaV0_I
fmla v7.4s, v30.4s, alphaV0_I
fmla v7.4s, v31.4s, alphaV0_R
st2 {v6.4s, v7.4s}, [pCRow3]
add pCRow3, pCRow3, #32
.endm
/******************************************************************************/
.macro INIT4x4
fmov s16, wzr
fmov s17, s16
fmov s20, s17
fmov s21, s16
fmov s24, s17
fmov s25, s16
fmov s28, s17
fmov s29, s16
.endm
.macro KERNEL4x4_I
ld2 {v8.4s, v9.4s}, [pB]
add pB, pB, #32
ld2 {v0.4s, v1.4s}, [pA]
add pA, pA, #32
fmul v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.s[0]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v17.16b, v17.16b, v17.16b
fmls v17.4s, v0.4s, v9.s[0]
#else
fmul v17.4s, v0.4s, v9.s[0]
#endif
OP_ir v17.4s, v1.4s, v8.s[0]
fmul v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.s[1]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v21.16b, v21.16b, v21.16b
fmls v21.4s, v0.4s, v9.s[1]
#else
fmul v21.4s, v0.4s, v9.s[1]
#endif
OP_ir v21.4s, v1.4s, v8.s[1]
fmul v24.4s, v0.4s, v8.s[2]
OP_ii v24.4s, v1.4s, v9.s[2]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v25.16b, v25.16b, v25.16b
fmls v25.4s, v0.4s, v9.s[2]
#else
fmul v25.4s, v0.4s, v9.s[2]
#endif
OP_ir v25.4s, v1.4s, v8.s[2]
fmul v28.4s, v0.4s, v8.s[3]
OP_ii v28.4s, v1.4s, v9.s[3]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v29.16b, v29.16b, v29.16b
fmls v29.4s, v0.4s, v9.s[3]
#else
fmul v29.4s, v0.4s, v9.s[3]
#endif
OP_ir v29.4s, v1.4s, v8.s[3]
ld2 {v12.4s, v13.4s}, [pB]
add pB, pB, #32
ld2 {v4.4s, v5.4s}, [pA]
add pA, pA, #32
.endm
.macro KERNEL4x4_M1
OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.s[0]
ld2 {v12.4s, v13.4s}, [pB] // For next round
add pB, pB, #32
OP_rr v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.s[1]
OP_ri v21.4s, v0.4s, v9.s[1]
OP_ir v21.4s, v1.4s, v8.s[1]
ld2 {v4.4s, v5.4s}, [pA] // For next round
add pA, pA, #32
OP_rr v24.4s, v0.4s, v8.s[2]
OP_ii v24.4s, v1.4s, v9.s[2]
OP_ri v25.4s, v0.4s, v9.s[2]
OP_ir v25.4s, v1.4s, v8.s[2]
prfm PLDL1KEEP, [pA, #512]
OP_rr v28.4s, v0.4s, v8.s[3]
OP_ii v28.4s, v1.4s, v9.s[3]
OP_ri v29.4s, v0.4s, v9.s[3]
OP_ir v29.4s, v1.4s, v8.s[3]
.endm
.macro KERNEL4x4_M2
OP_rr v16.4s, v4.4s, v12.s[0]
OP_ii v16.4s, v5.4s, v13.s[0]
OP_ri v17.4s, v4.4s, v13.s[0]
OP_ir v17.4s, v5.4s, v12.s[0]
ld2 {v8.4s, v9.4s}, [pB] // For next round
add pB, pB, #32
OP_rr v20.4s, v4.4s, v12.s[1]
OP_ii v20.4s, v5.4s, v13.s[1]
OP_ri v21.4s, v4.4s, v13.s[1]
OP_ir v21.4s, v5.4s, v12.s[1]
ld2 {v0.4s, v1.4s}, [pA] // For next round
add pA, pA, #32
OP_rr v24.4s, v4.4s, v12.s[2]
OP_ii v24.4s, v5.4s, v13.s[2]
OP_ri v25.4s, v4.4s, v13.s[2]
OP_ir v25.4s, v5.4s, v12.s[2]
prfm PLDL1KEEP, [pB, #512]
OP_rr v28.4s, v4.4s, v12.s[3]
OP_ii v28.4s, v5.4s, v13.s[3]
OP_ri v29.4s, v4.4s, v13.s[3]
OP_ir v29.4s, v5.4s, v12.s[3]
.endm
.macro KERNEL4x4_E
OP_rr v16.4s, v4.4s, v12.s[0]
OP_ii v16.4s, v5.4s, v13.s[0]
OP_ri v17.4s, v4.4s, v13.s[0]
OP_ir v17.4s, v5.4s, v12.s[0]
OP_rr v20.4s, v4.4s, v12.s[1]
OP_ii v20.4s, v5.4s, v13.s[1]
OP_ri v21.4s, v4.4s, v13.s[1]
OP_ir v21.4s, v5.4s, v12.s[1]
OP_rr v24.4s, v4.4s, v12.s[2]
OP_ii v24.4s, v5.4s, v13.s[2]
OP_ri v25.4s, v4.4s, v13.s[2]
OP_ir v25.4s, v5.4s, v12.s[2]
OP_rr v28.4s, v4.4s, v12.s[3]
OP_ii v28.4s, v5.4s, v13.s[3]
OP_ri v29.4s, v4.4s, v13.s[3]
OP_ir v29.4s, v5.4s, v12.s[3]
.endm
.macro KERNEL4x4_SUB
ld2 {v8.4s, v9.4s}, [pB]
add pB, pB, #32
ld2 {v0.4s, v1.4s}, [pA]
add pA, pA, #32
OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.s[0]
OP_rr v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.s[1]
OP_ri v21.4s, v0.4s, v9.s[1]
OP_ir v21.4s, v1.4s, v8.s[1]
OP_rr v24.4s, v0.4s, v8.s[2]
OP_ii v24.4s, v1.4s, v9.s[2]
OP_ri v25.4s, v0.4s, v9.s[2]
OP_ir v25.4s, v1.4s, v8.s[2]
OP_rr v28.4s, v0.4s, v8.s[3]
OP_ii v28.4s, v1.4s, v9.s[3]
OP_ri v29.4s, v0.4s, v9.s[3]
OP_ir v29.4s, v1.4s, v8.s[3]
.endm
.macro SAVE4x4
fmov alpha0_R, alphaR
fmov alpha0_I, alphaI
mov pCRow1, pCRow0
ld2 {v0.4s, v1.4s}, [pCRow1]
fmla v0.4s, v16.4s, alphaV0_R
fmls v0.4s, v17.4s, alphaV0_I
fmla v1.4s, v16.4s, alphaV0_I
fmla v1.4s, v17.4s, alphaV0_R
st2 {v0.4s, v1.4s}, [pCRow1]
add pCRow1, pCRow1, LDC
ld2 {v4.4s, v5.4s}, [pCRow1]
fmla v4.4s, v20.4s, alphaV0_R
fmls v4.4s, v21.4s, alphaV0_I
fmla v5.4s, v20.4s, alphaV0_I
fmla v5.4s, v21.4s, alphaV0_R
st2 {v4.4s, v5.4s}, [pCRow1]
add pCRow1, pCRow1, LDC
ld2 {v0.4s, v1.4s}, [pCRow1]
fmla v0.4s, v24.4s, alphaV0_R
fmls v0.4s, v25.4s, alphaV0_I
fmla v1.4s, v24.4s, alphaV0_I
fmla v1.4s, v25.4s, alphaV0_R
st2 {v0.4s, v1.4s}, [pCRow1]
add pCRow1, pCRow1, LDC
ld2 {v4.4s, v5.4s}, [pCRow1]
fmla v4.4s, v28.4s, alphaV0_R
fmls v4.4s, v29.4s, alphaV0_I
fmla v5.4s, v28.4s, alphaV0_I
fmla v5.4s, v29.4s, alphaV0_R
st2 {v4.4s, v5.4s}, [pCRow1]
add pCRow0, pCRow0, #32
.endm
/******************************************************************************/
.macro INIT2x4
fmov s16, wzr
fmov s17, wzr
fmov s20, s16
fmov s21, s17
fmov s24, s16
fmov s25, s17
fmov s28, s16
fmov s29, s17
.endm
.macro KERNEL2x4_SUB
ld2 {v8.4s, v9.4s}, [pB]
add pB, pB, #32
ld2 {v0.2s, v1.2s}, [pA]
add pA, pA, #16
OP_rr v16.2s, v0.2s, v8.s[0]
OP_ii v16.2s, v1.2s, v9.s[0]
OP_ri v17.2s, v0.2s, v9.s[0]
OP_ir v17.2s, v1.2s, v8.s[0]
OP_rr v20.2s, v0.2s, v8.s[1]
OP_ii v20.2s, v1.2s, v9.s[1]
OP_ri v21.2s, v0.2s, v9.s[1]
OP_ir v21.2s, v1.2s, v8.s[1]
OP_rr v24.2s, v0.2s, v8.s[2]
OP_ii v24.2s, v1.2s, v9.s[2]
OP_ri v25.2s, v0.2s, v9.s[2]
OP_ir v25.2s, v1.2s, v8.s[2]
OP_rr v28.2s, v0.2s, v8.s[3]
OP_ii v28.2s, v1.2s, v9.s[3]
OP_ri v29.2s, v0.2s, v9.s[3]
OP_ir v29.2s, v1.2s, v8.s[3]
.endm
.macro SAVE2x4
fmov alpha0_R, alphaR
fmov alpha0_I, alphaI
mov pCRow1, pCRow0
ld2 {v0.2s, v1.2s}, [pCRow1]
fmla v0.2s, v16.2s, alphaV0_R
fmls v0.2s, v17.2s, alphaV0_I
fmla v1.2s, v16.2s, alphaV0_I
fmla v1.2s, v17.2s, alphaV0_R
st2 {v0.2s, v1.2s}, [pCRow1]
add pCRow1, pCRow1, LDC
ld2 {v4.2s, v5.2s}, [pCRow1]
fmla v4.2s, v20.2s, alphaV0_R
fmls v4.2s, v21.2s, alphaV0_I
fmla v5.2s, v20.2s, alphaV0_I
fmla v5.2s, v21.2s, alphaV0_R
st2 {v4.2s, v5.2s}, [pCRow1]
add pCRow1, pCRow1, LDC
ld2 {v0.2s, v1.2s}, [pCRow1]
fmla v0.2s, v24.2s, alphaV0_R
fmls v0.2s, v25.2s, alphaV0_I
fmla v1.2s, v24.2s, alphaV0_I
fmla v1.2s, v25.2s, alphaV0_R
st2 {v0.2s, v1.2s}, [pCRow1]
add pCRow1, pCRow1, LDC
ld2 {v4.2s, v5.2s}, [pCRow1]
fmla v4.2s, v28.2s, alphaV0_R
fmls v4.2s, v29.2s, alphaV0_I
fmla v5.2s, v28.2s, alphaV0_I
fmla v5.2s, v29.2s, alphaV0_R
st2 {v4.2s, v5.2s}, [pCRow1]
add pCRow0, pCRow0, #16
.endm
/******************************************************************************/
.macro INIT1x4
fmov s16, wzr
fmov s17, wzr
fmov s20, s16
fmov s21, s17
fmov s24, s16
fmov s25, s17
fmov s28, s16
fmov s29, s17
.endm
.macro KERNEL1x4_SUB
ld2 {v8.4s, v9.4s}, [pB]
add pB, pB, #32
ld2 {v0.s, v1.s}[0], [pA]
add pA, pA, #8
OP_rr s16, s0, v8.s[0]
OP_ii s16, s1, v9.s[0]
OP_ri s17, s0, v9.s[0]
OP_ir s17, s1, v8.s[0]
OP_rr s20, s0, v8.s[1]
OP_ii s20, s1, v9.s[1]
OP_ri s21, s0, v9.s[1]
OP_ir s21, s1, v8.s[1]
OP_rr s24, s0, v8.s[2]
OP_ii s24, s1, v9.s[2]
OP_ri s25, s0, v9.s[2]
OP_ir s25, s1, v8.s[2]
OP_rr s28, s0, v8.s[3]
OP_ii s28, s1, v9.s[3]
OP_ri s29, s0, v9.s[3]
OP_ir s29, s1, v8.s[3]
.endm
.macro SAVE1x4
fmov alpha0_R, alphaR
fmov alpha0_I, alphaI
mov pCRow1, pCRow0
ld2 {v0.s, v1.s}[0], [pCRow1]
fmla s0, s16, alphaV0_R
fmls s0, s17, alphaV0_I
fmla s1, s16, alphaV0_I
fmla s1, s17, alphaV0_R
st2 {v0.s, v1.s}[0], [pCRow1]
add pCRow1, pCRow1, LDC
ld2 {v4.s, v5.s}[0], [pCRow1]
fmla s4, s20, alphaV0_R
fmls s4, s21, alphaV0_I
fmla s5, s20, alphaV0_I
fmla s5, s21, alphaV0_R
st2 {v4.s, v5.s}[0], [pCRow1]
add pCRow1, pCRow1, LDC
ld2 {v0.s, v1.s}[0], [pCRow1]
fmla s0, s24, alphaV0_R
fmls s0, s25, alphaV0_I
fmla s1, s24, alphaV0_I
fmla s1, s25, alphaV0_R
st2 {v0.s, v1.s}[0], [pCRow1]
add pCRow1, pCRow1, LDC
ld2 {v4.s, v5.s}[0], [pCRow1]
fmla s4, s28, alphaV0_R
fmls s4, s29, alphaV0_I
fmla s5, s28, alphaV0_I
fmla s5, s29, alphaV0_R
st2 {v4.s, v5.s}[0], [pCRow1]
add pCRow0, pCRow0, #8
.endm
/******************************************************************************/
.macro INIT8x2
fmov s16, wzr
fmov s17, wzr
fmov s18, wzr
fmov s19, s16
fmov s20, wzr
fmov s21, s16
fmov s22, s17
fmov s23, s18
.endm
.macro KERNEL8x2_SUB
ld2 {v8.2s, v9.2s}, [pB]
add pB, pB, #16
ld2 {v0.4s, v1.4s}, [pA]
add pA, pA, #32
ld2 {v2.4s, v3.4s}, [pA]
add pA, pA, #32
OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.s[0]
OP_rr v18.4s, v2.4s, v8.s[0]
OP_ii v18.4s, v3.4s, v9.s[0]
OP_ri v19.4s, v2.4s, v9.s[0]
OP_ir v19.4s, v3.4s, v8.s[0]
OP_rr v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.s[1]
OP_ri v21.4s, v0.4s, v9.s[1]
OP_ir v21.4s, v1.4s, v8.s[1]
OP_rr v22.4s, v2.4s, v8.s[1]
OP_ii v22.4s, v3.4s, v9.s[1]
OP_ri v23.4s, v2.4s, v9.s[1]
OP_ir v23.4s, v3.4s, v8.s[1]
.endm
.macro SAVE8x2
fmov alpha0_R, alphaR
fmov alpha0_I, alphaI
mov pCRow1, pCRow0
ld2 {v0.4s, v1.4s}, [pCRow1]
fmla v0.4s, v16.4s, alphaV0_R
fmls v0.4s, v17.4s, alphaV0_I
fmla v1.4s, v16.4s, alphaV0_I
fmla v1.4s, v17.4s, alphaV0_R
st2 {v0.4s, v1.4s}, [pCRow1]
add pCRow2, pCRow1, #32
ld2 {v2.4s, v3.4s}, [pCRow2]
fmla v2.4s, v18.4s, alphaV0_R
fmls v2.4s, v19.4s, alphaV0_I
fmla v3.4s, v18.4s, alphaV0_I
fmla v3.4s, v19.4s, alphaV0_R
st2 {v2.4s, v3.4s}, [pCRow2]
add pCRow1, pCRow1, LDC
ld2 {v4.4s, v5.4s}, [pCRow1]
fmla v4.4s, v20.4s, alphaV0_R
fmls v4.4s, v21.4s, alphaV0_I
fmla v5.4s, v20.4s, alphaV0_I
fmla v5.4s, v21.4s, alphaV0_R
st2 {v4.4s, v5.4s}, [pCRow1]
add pCRow2, pCRow1, #32
ld2 {v6.4s, v7.4s}, [pCRow2]
fmla v6.4s, v22.4s, alphaV0_R
fmls v6.4s, v23.4s, alphaV0_I
fmla v7.4s, v22.4s, alphaV0_I
fmla v7.4s, v23.4s, alphaV0_R
st2 {v6.4s, v7.4s}, [pCRow2]
add pCRow0, pCRow0, #64
.endm
/******************************************************************************/
.macro INIT4x2
fmov s16, wzr
fmov s17, wzr
fmov s20, s16
fmov s21, s17
.endm
.macro KERNEL4x2_SUB
ld2 {v8.2s, v9.2s}, [pB]
add pB, pB, #16
ld2 {v0.4s, v1.4s}, [pA]
add pA, pA, #32
OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.s[0]
OP_rr v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.s[1]
OP_ri v21.4s, v0.4s, v9.s[1]
OP_ir v21.4s, v1.4s, v8.s[1]
.endm
.macro SAVE4x2
fmov alpha0_R, alphaR
fmov alpha0_I, alphaI
mov pCRow1, pCRow0
ld2 {v0.4s, v1.4s}, [pCRow1]
fmla v0.4s, v16.4s, alphaV0_R
fmls v0.4s, v17.4s, alphaV0_I
fmla v1.4s, v16.4s, alphaV0_I
fmla v1.4s, v17.4s, alphaV0_R
st2 {v0.4s, v1.4s}, [pCRow1]
add pCRow1, pCRow1, LDC
ld2 {v4.4s, v5.4s}, [pCRow1]
fmla v4.4s, v20.4s, alphaV0_R
fmls v4.4s, v21.4s, alphaV0_I
fmla v5.4s, v20.4s, alphaV0_I
fmla v5.4s, v21.4s, alphaV0_R
st2 {v4.4s, v5.4s}, [pCRow1]
add pCRow0, pCRow0, #32
.endm
/******************************************************************************/
.macro INIT2x2
fmov s16, wzr
fmov s17, wzr
fmov s20, s16
fmov s21, s17
.endm
.macro KERNEL2x2_SUB
ld2 {v8.2s, v9.2s}, [pB]
add pB, pB, #16
ld2 {v0.2s, v1.2s}, [pA]
add pA, pA, #16
OP_rr v16.2s, v0.2s, v8.s[0]
OP_ii v16.2s, v1.2s, v9.s[0]
OP_ri v17.2s, v0.2s, v9.s[0]
OP_ir v17.2s, v1.2s, v8.s[0]
OP_rr v20.2s, v0.2s, v8.s[1]
OP_ii v20.2s, v1.2s, v9.s[1]
OP_ri v21.2s, v0.2s, v9.s[1]
OP_ir v21.2s, v1.2s, v8.s[1]
.endm
.macro SAVE2x2
fmov alpha0_R, alphaR
fmov alpha0_I, alphaI
mov pCRow1, pCRow0
ld2 {v0.2s, v1.2s}, [pCRow1]
fmla v0.2s, v16.2s, alphaV0_R
fmls v0.2s, v17.2s, alphaV0_I
fmla v1.2s, v16.2s, alphaV0_I
fmla v1.2s, v17.2s, alphaV0_R
st2 {v0.2s, v1.2s}, [pCRow1]
add pCRow1, pCRow1, LDC
ld2 {v4.2s, v5.2s}, [pCRow1]
fmla v4.2s, v20.2s, alphaV0_R
fmls v4.2s, v21.2s, alphaV0_I
fmla v5.2s, v20.2s, alphaV0_I
fmla v5.2s, v21.2s, alphaV0_R
st2 {v4.2s, v5.2s}, [pCRow1]
add pCRow0, pCRow0, #16
.endm
/******************************************************************************/
.macro INIT1x2
fmov s16, wzr
fmov s17, wzr
fmov s20, wzr
fmov s21, wzr
.endm
.macro KERNEL1x2_SUB
ld2 {v8.2s, v9.2s}, [pB]
add pB, pB, #16
ld2 {v0.s, v1.s}[0], [pA]
add pA, pA, #8
OP_rr s16, s0, v8.s[0]
OP_ii s16, s1, v9.s[0]
OP_ri s17, s0, v9.s[0]
OP_ir s17, s1, v8.s[0]
OP_rr s20, s0, v8.s[1]
OP_ii s20, s1, v9.s[1]
OP_ri s21, s0, v9.s[1]
OP_ir s21, s1, v8.s[1]
.endm
.macro SAVE1x2
fmov alpha0_R, alphaR
fmov alpha0_I, alphaI
mov pCRow1, pCRow0
ld2 {v0.s, v1.s}[0], [pCRow1]
fmla s0, s16, alphaV0_R
fmls s0, s17, alphaV0_I
fmla s1, s16, alphaV0_I
fmla s1, s17, alphaV0_R
st2 {v0.s, v1.s}[0], [pCRow1]
add pCRow1, pCRow1, LDC
ld2 {v4.s, v5.s}[0], [pCRow1]
fmla s4, s20, alphaV0_R
fmls s4, s21, alphaV0_I
fmla s5, s20, alphaV0_I
fmla s5, s21, alphaV0_R
st2 {v4.s, v5.s}[0], [pCRow1]
add pCRow0, pCRow0, #8
.endm
/******************************************************************************/
.macro INIT8x1
fmov s16, wzr
fmov s17, wzr
fmov s18, wzr
fmov s19, s16
.endm
.macro KERNEL8x1_SUB
ld1 {v8.2s}, [pB]
add pB, pB, #8
ld2 {v0.4s, v1.4s}, [pA]
add pA, pA, #32
ld2 {v2.4s, v3.4s}, [pA]
add pA, pA, #32
OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v8.s[1]
OP_ri v17.4s, v0.4s, v8.s[1]
OP_ir v17.4s, v1.4s, v8.s[0]
OP_rr v18.4s, v2.4s, v8.s[0]
OP_ii v18.4s, v3.4s, v8.s[1]
OP_ri v19.4s, v2.4s, v8.s[1]
OP_ir v19.4s, v3.4s, v8.s[0]
.endm
.macro SAVE8x1
fmov alpha0_R, alphaR
fmov alpha0_I, alphaI
mov pCRow1, pCRow0
ld2 {v0.4s, v1.4s}, [pCRow1]
fmla v0.4s, v16.4s, alphaV0_R
fmls v0.4s, v17.4s, alphaV0_I
fmla v1.4s, v16.4s, alphaV0_I
fmla v1.4s, v17.4s, alphaV0_R
st2 {v0.4s, v1.4s}, [pCRow1]
add pCRow1, pCRow1, #32
ld2 {v2.4s, v3.4s}, [pCRow1]
fmla v2.4s, v18.4s, alphaV0_R
fmls v2.4s, v19.4s, alphaV0_I
fmla v3.4s, v18.4s, alphaV0_I
fmla v3.4s, v19.4s, alphaV0_R
st2 {v2.4s, v3.4s}, [pCRow1]
add pCRow0, pCRow0, #64
.endm
/******************************************************************************/
.macro INIT4x1
fmov s16, wzr
fmov s17, s16
.endm
.macro KERNEL4x1_SUB
ld2 {v8.s, v9.s}[0], [pB]
add pB, pB, #8
ld2 {v0.4s, v1.4s}, [pA]
add pA, pA, #32
OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.s[0]
.endm
.macro SAVE4x1
fmov alpha0_R, alphaR
fmov alpha0_I, alphaI
mov pCRow1, pCRow0
ld2 {v0.4s, v1.4s}, [pCRow1]
fmla v0.4s, v16.4s, alphaV0_R
fmls v0.4s, v17.4s, alphaV0_I
fmla v1.4s, v16.4s, alphaV0_I
fmla v1.4s, v17.4s, alphaV0_R
st2 {v0.4s, v1.4s}, [pCRow1]
add pCRow0, pCRow0, #32
.endm
/******************************************************************************/
.macro INIT2x1
fmov s16, wzr
fmov s17, wzr
.endm
.macro KERNEL2x1_SUB
ld2 {v8.s, v9.s}[0], [pB]
add pB, pB, #8
ld2 {v0.2s, v1.2s}, [pA]
add pA, pA, #16
OP_rr v16.2s, v0.2s, v8.s[0]
OP_ii v16.2s, v1.2s, v9.s[0]
OP_ri v17.2s, v0.2s, v9.s[0]
OP_ir v17.2s, v1.2s, v8.s[0]
.endm
.macro SAVE2x1
fmov alpha0_R, alphaR
fmov alpha0_I, alphaI
mov pCRow1, pCRow0
ld2 {v0.2s, v1.2s}, [pCRow1]
fmla v0.2s, v16.2s, alphaV0_R
fmls v0.2s, v17.2s, alphaV0_I
fmla v1.2s, v16.2s, alphaV0_I
fmla v1.2s, v17.2s, alphaV0_R
st2 {v0.2s, v1.2s}, [pCRow1]
add pCRow0, pCRow0, #16
.endm
/******************************************************************************/
.macro INIT1x1
fmov s16, wzr
fmov s17, wzr
.endm
.macro KERNEL1x1_SUB
ld2 {v8.s, v9.s}[0], [pB]
add pB, pB, #8
ld2 {v0.s, v1.s}[0], [pA]
add pA, pA, #8
OP_rr s16, s0, v8.s[0]
OP_ii s16, s1, v9.s[0]
OP_ri s17, s0, v9.s[0]
OP_ir s17, s1, v8.s[0]
.endm
.macro SAVE1x1
fmov alpha0_R, alphaR
fmov alpha0_I, alphaI
mov pCRow1, pCRow0
ld2 {v0.s, v1.s}[0], [pCRow1]
fmla s0, s16, alphaV0_R
fmls s0, s17, alphaV0_I
fmla s1, s16, alphaV0_I
fmla s1, s17, alphaV0_R
st2 {v0.s, v1.s}[0], [pCRow1]
add pCRow0, pCRow0, #8
.endm
/*******************************************************************************
* End of macro definitions
*******************************************************************************/
PROLOGUE
.align 5
add sp, sp, #-(11 * 16)
stp d8, d9, [sp, #(0 * 16)]
stp d10, d11, [sp, #(1 * 16)]
stp d12, d13, [sp, #(2 * 16)]
stp d14, d15, [sp, #(3 * 16)]
stp d16, d17, [sp, #(4 * 16)]
stp x18, x19, [sp, #(5 * 16)]
stp x20, x21, [sp, #(6 * 16)]
stp x22, x23, [sp, #(7 * 16)]
stp x24, x25, [sp, #(8 * 16)]
stp x26, x27, [sp, #(9 * 16)]
str x28, [sp, #(10 * 16)]
prfm PLDL1KEEP, [origPB]
prfm PLDL1KEEP, [origPA]
fmov alphaR, s0
fmov alphaI, s1
lsl LDC, LDC, #3 // ldc = ldc * 8
mov pB, origPB
mov counterJ, origN
asr counterJ, counterJ, #2 // J = J / 4
cmp counterJ, #0
ble .Lcgemm_kernel_L2_BEGIN
/******************************************************************************/
.Lcgemm_kernel_L4_BEGIN:
mov pCRow0, pC
add pCRow1, pCRow0, LDC
add pCRow2, pCRow1, LDC
add pCRow3, pCRow2, LDC
add pC, pCRow3, LDC
mov pA, origPA // pA = start of A array
.Lcgemm_kernel_L4_M8_BEGIN:
mov counterI, origM
asr counterI, counterI, #3 // counterI = counterI / 8
cmp counterI, #0
ble .Lcgemm_kernel_L4_M4_BEGIN
.align 5
.Lcgemm_kernel_L4_M8_20:
mov pB, origPB
asr counterL , origK, #3
cmp counterL , #2
blt .Lcgemm_kernel_L4_M8_32
KERNEL8x4_I
KERNEL8x4_M2
KERNEL8x4_M1
KERNEL8x4_M2
KERNEL8x4_M1
KERNEL8x4_M2
KERNEL8x4_M1
KERNEL8x4_M2
subs counterL, counterL, #2 // subtract 2
ble .Lcgemm_kernel_L4_M8_22a
.align 5
.Lcgemm_kernel_L4_M8_22:
KERNEL8x4_M1
KERNEL8x4_M2
KERNEL8x4_M1
KERNEL8x4_M2
KERNEL8x4_M1
KERNEL8x4_M2
KERNEL8x4_M1
KERNEL8x4_M2
subs counterL, counterL, #1
bgt .Lcgemm_kernel_L4_M8_22
.align 5
.Lcgemm_kernel_L4_M8_22a:
KERNEL8x4_M1
KERNEL8x4_M2
KERNEL8x4_M1
KERNEL8x4_M2
KERNEL8x4_M1
KERNEL8x4_M2
KERNEL8x4_M1
KERNEL8x4_E
b .Lcgemm_kernel_L4_M8_44
.align 5
.Lcgemm_kernel_L4_M8_32:
tst counterL, #1
ble .Lcgemm_kernel_L4_M8_40
KERNEL8x4_I
KERNEL8x4_M2
KERNEL8x4_M1
KERNEL8x4_M2
KERNEL8x4_M1
KERNEL8x4_M2
KERNEL8x4_M1
KERNEL8x4_E
b .Lcgemm_kernel_L4_M8_44
.Lcgemm_kernel_L4_M8_40:
INIT8x4
.Lcgemm_kernel_L4_M8_44:
ands counterL , origK, #7
ble .Lcgemm_kernel_L4_M8_100
.align 5
.Lcgemm_kernel_L4_M8_46:
KERNEL8x4_SUB
subs counterL, counterL, #1
bne .Lcgemm_kernel_L4_M8_46
.Lcgemm_kernel_L4_M8_100:
prfm PLDL1KEEP, [pA]
prfm PLDL1KEEP, [pA, #64]
prfm PLDL1KEEP, [origPB]
SAVE8x4
.Lcgemm_kernel_L4_M8_END:
subs counterI, counterI, #1
bne .Lcgemm_kernel_L4_M8_20
.Lcgemm_kernel_L4_M4_BEGIN:
mov counterI, origM
tst counterI , #7
ble .Lcgemm_kernel_L4_END
tst counterI, #4
ble .Lcgemm_kernel_L4_M2_BEGIN
.Lcgemm_kernel_L4_M4_20:
mov pB, origPB
asr counterL , origK, #1 // L = K / 2
cmp counterL , #2 // is there at least 4 to do?
blt .Lcgemm_kernel_L4_M4_32
KERNEL4x4_I // do one in the K
KERNEL4x4_M2 // do another in the K
subs counterL, counterL, #2
ble .Lcgemm_kernel_L4_M4_22a
.align 5
.Lcgemm_kernel_L4_M4_22:
KERNEL4x4_M1
KERNEL4x4_M2
subs counterL, counterL, #1
bgt .Lcgemm_kernel_L4_M4_22
.Lcgemm_kernel_L4_M4_22a:
KERNEL4x4_M1
KERNEL4x4_E
b .Lcgemm_kernel_L4_M4_44
.Lcgemm_kernel_L4_M4_32:
tst counterL, #1
ble .Lcgemm_kernel_L4_M4_40
KERNEL4x4_I
KERNEL4x4_E
b .Lcgemm_kernel_L4_M4_44
.Lcgemm_kernel_L4_M4_40:
INIT4x4
.Lcgemm_kernel_L4_M4_44:
ands counterL , origK, #1
ble .Lcgemm_kernel_L4_M4_100
.Lcgemm_kernel_L4_M4_46:
KERNEL4x4_SUB
.Lcgemm_kernel_L4_M4_100:
SAVE4x4
.Lcgemm_kernel_L4_M4_END:
.Lcgemm_kernel_L4_M2_BEGIN:
mov counterI, origM
tst counterI , #3
ble .Lcgemm_kernel_L4_END
tst counterI, #2 // counterI = counterI / 2
ble .Lcgemm_kernel_L4_M1_BEGIN
.Lcgemm_kernel_L4_M2_20:
INIT2x4
mov pB, origPB
asr counterL , origK, #3 // counterL = counterL / 8
cmp counterL , #0
ble .Lcgemm_kernel_L4_M2_40
.Lcgemm_kernel_L4_M2_22:
KERNEL2x4_SUB
KERNEL2x4_SUB
KERNEL2x4_SUB
KERNEL2x4_SUB
KERNEL2x4_SUB
KERNEL2x4_SUB
KERNEL2x4_SUB
KERNEL2x4_SUB
subs counterL, counterL, #1
bgt .Lcgemm_kernel_L4_M2_22
.Lcgemm_kernel_L4_M2_40:
ands counterL , origK, #7 // counterL = counterL % 8
ble .Lcgemm_kernel_L4_M2_100
.Lcgemm_kernel_L4_M2_42:
KERNEL2x4_SUB
subs counterL, counterL, #1
bgt .Lcgemm_kernel_L4_M2_42
.Lcgemm_kernel_L4_M2_100:
SAVE2x4
.Lcgemm_kernel_L4_M2_END:
.Lcgemm_kernel_L4_M1_BEGIN:
tst counterI, #1 // counterI = counterI % 2
ble .Lcgemm_kernel_L4_END
.Lcgemm_kernel_L4_M1_20:
INIT1x4
mov pB, origPB
asr counterL , origK, #3 // counterL = counterL / 8
cmp counterL , #0
ble .Lcgemm_kernel_L4_M1_40
.Lcgemm_kernel_L4_M1_22:
KERNEL1x4_SUB
KERNEL1x4_SUB
KERNEL1x4_SUB
KERNEL1x4_SUB
KERNEL1x4_SUB
KERNEL1x4_SUB
KERNEL1x4_SUB
KERNEL1x4_SUB
subs counterL, counterL, #1
bgt .Lcgemm_kernel_L4_M1_22
.Lcgemm_kernel_L4_M1_40:
ands counterL , origK, #7 // counterL = counterL % 8
ble .Lcgemm_kernel_L4_M1_100
.Lcgemm_kernel_L4_M1_42:
KERNEL1x4_SUB
subs counterL, counterL, #1
bgt .Lcgemm_kernel_L4_M1_42
.Lcgemm_kernel_L4_M1_100:
SAVE1x4
.Lcgemm_kernel_L4_END:
lsl temp, origK, #5
add origPB, origPB, temp // B = B + K * 4 * 8
subs counterJ, counterJ , #1 // j--
bgt .Lcgemm_kernel_L4_BEGIN
/******************************************************************************/
.Lcgemm_kernel_L2_BEGIN: // less than 2 left in N direction
mov counterJ , origN
tst counterJ , #3
ble .Lcgemm_kernel_L999 // error, N was less than 4?
tst counterJ , #2
ble .Lcgemm_kernel_L1_BEGIN
mov pCRow0, pC // pCRow0 = pC
add pC,pC,LDC, lsl #1
mov pA, origPA // pA = A
.Lcgemm_kernel_L2_M8_BEGIN:
mov counterI, origM
asr counterI, counterI, #3 // counterI = counterI / 8
cmp counterI, #0
ble .Lcgemm_kernel_L2_M4_BEGIN
.Lcgemm_kernel_L2_M8_20:
INIT8x2
mov pB, origPB
asr counterL , origK, #3 // counterL = counterL / 8
cmp counterL,#0
ble .Lcgemm_kernel_L2_M8_40
.align 5
.Lcgemm_kernel_L2_M8_22:
KERNEL8x2_SUB
KERNEL8x2_SUB
KERNEL8x2_SUB
KERNEL8x2_SUB
KERNEL8x2_SUB
KERNEL8x2_SUB
KERNEL8x2_SUB
KERNEL8x2_SUB
subs counterL, counterL, #1
bgt .Lcgemm_kernel_L2_M8_22
.Lcgemm_kernel_L2_M8_40:
ands counterL , origK, #7 // counterL = counterL % 8
ble .Lcgemm_kernel_L2_M8_100
.Lcgemm_kernel_L2_M8_42:
KERNEL8x2_SUB
subs counterL, counterL, #1
bgt .Lcgemm_kernel_L2_M8_42
.Lcgemm_kernel_L2_M8_100:
SAVE8x2
.Lcgemm_kernel_L2_M8_END:
subs counterI, counterI, #1
bgt .Lcgemm_kernel_L2_M8_20
.Lcgemm_kernel_L2_M4_BEGIN:
mov counterI, origM
tst counterI , #7
ble .Lcgemm_kernel_L2_END
tst counterI, #4 // counterI = counterI / 2
ble .Lcgemm_kernel_L2_M2_BEGIN
.Lcgemm_kernel_L2_M4_20:
INIT4x2
mov pB, origPB
asr counterL , origK, #3 // counterL = counterL / 8
cmp counterL,#0
ble .Lcgemm_kernel_L2_M4_40
.align 5
.Lcgemm_kernel_L2_M4_22:
KERNEL4x2_SUB
KERNEL4x2_SUB
KERNEL4x2_SUB
KERNEL4x2_SUB
KERNEL4x2_SUB
KERNEL4x2_SUB
KERNEL4x2_SUB
KERNEL4x2_SUB
subs counterL, counterL, #1
bgt .Lcgemm_kernel_L2_M4_22
.Lcgemm_kernel_L2_M4_40:
ands counterL , origK, #7 // counterL = counterL % 8
ble .Lcgemm_kernel_L2_M4_100
.Lcgemm_kernel_L2_M4_42:
KERNEL4x2_SUB
subs counterL, counterL, #1
bgt .Lcgemm_kernel_L2_M4_42
.Lcgemm_kernel_L2_M4_100:
SAVE4x2
.Lcgemm_kernel_L2_M4_END:
.Lcgemm_kernel_L2_M2_BEGIN:
mov counterI, origM
tst counterI , #3
ble .Lcgemm_kernel_L2_END
tst counterI, #2 // counterI = counterI / 2
ble .Lcgemm_kernel_L2_M1_BEGIN
.Lcgemm_kernel_L2_M2_20:
INIT2x2
mov pB, origPB
asr counterL , origK, #3 // counterL = counterL / 8
cmp counterL,#0
ble .Lcgemm_kernel_L2_M2_40
.Lcgemm_kernel_L2_M2_22:
KERNEL2x2_SUB
KERNEL2x2_SUB
KERNEL2x2_SUB
KERNEL2x2_SUB
KERNEL2x2_SUB
KERNEL2x2_SUB
KERNEL2x2_SUB
KERNEL2x2_SUB
subs counterL, counterL, #1
bgt .Lcgemm_kernel_L2_M2_22
.Lcgemm_kernel_L2_M2_40:
ands counterL , origK, #7 // counterL = counterL % 8
ble .Lcgemm_kernel_L2_M2_100
.Lcgemm_kernel_L2_M2_42:
KERNEL2x2_SUB
subs counterL, counterL, #1
bgt .Lcgemm_kernel_L2_M2_42
.Lcgemm_kernel_L2_M2_100:
SAVE2x2
.Lcgemm_kernel_L2_M2_END:
.Lcgemm_kernel_L2_M1_BEGIN:
tst counterI, #1 // counterI = counterI % 2
ble .Lcgemm_kernel_L2_END
.Lcgemm_kernel_L2_M1_20:
INIT1x2
mov pB, origPB
asr counterL , origK, #3 // counterL = counterL / 8
cmp counterL, #0
ble .Lcgemm_kernel_L2_M1_40
.Lcgemm_kernel_L2_M1_22:
KERNEL1x2_SUB
KERNEL1x2_SUB
KERNEL1x2_SUB
KERNEL1x2_SUB
KERNEL1x2_SUB
KERNEL1x2_SUB
KERNEL1x2_SUB
KERNEL1x2_SUB
subs counterL, counterL, #1
bgt .Lcgemm_kernel_L2_M1_22
.Lcgemm_kernel_L2_M1_40:
ands counterL , origK, #7 // counterL = counterL % 8
ble .Lcgemm_kernel_L2_M1_100
.Lcgemm_kernel_L2_M1_42:
KERNEL1x2_SUB
subs counterL, counterL, #1
bgt .Lcgemm_kernel_L2_M1_42
.Lcgemm_kernel_L2_M1_100:
SAVE1x2
.Lcgemm_kernel_L2_END:
add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8
/******************************************************************************/
.Lcgemm_kernel_L1_BEGIN:
mov counterJ , origN
tst counterJ , #1
ble .Lcgemm_kernel_L999 // done
mov pCRow0, pC // pCRow0 = C
add pC , pC , LDC // Update pC to point to next
mov pA, origPA // pA = A
.Lcgemm_kernel_L1_M8_BEGIN:
mov counterI, origM
asr counterI, counterI, #3 // counterI = counterI / 8
cmp counterI, #0
ble .Lcgemm_kernel_L1_M4_BEGIN
.Lcgemm_kernel_L1_M8_20:
INIT8x1
mov pB, origPB
asr counterL , origK, #3 // counterL = counterL / 8
cmp counterL , #0
ble .Lcgemm_kernel_L1_M8_40
.align 5
.Lcgemm_kernel_L1_M8_22:
KERNEL8x1_SUB
KERNEL8x1_SUB
KERNEL8x1_SUB
KERNEL8x1_SUB
KERNEL8x1_SUB
KERNEL8x1_SUB
KERNEL8x1_SUB
KERNEL8x1_SUB
subs counterL, counterL, #1
bgt .Lcgemm_kernel_L1_M8_22
.Lcgemm_kernel_L1_M8_40:
ands counterL , origK, #7 // counterL = counterL % 8
ble .Lcgemm_kernel_L1_M8_100
.Lcgemm_kernel_L1_M8_42:
KERNEL8x1_SUB
subs counterL, counterL, #1
bgt .Lcgemm_kernel_L1_M8_42
.Lcgemm_kernel_L1_M8_100:
SAVE8x1
.Lcgemm_kernel_L1_M8_END:
subs counterI, counterI, #1
bgt .Lcgemm_kernel_L1_M8_20
.Lcgemm_kernel_L1_M4_BEGIN:
mov counterI, origM
tst counterI , #7
ble .Lcgemm_kernel_L1_END
tst counterI, #4 // counterI = counterI / 2
ble .Lcgemm_kernel_L1_M2_BEGIN
.Lcgemm_kernel_L1_M4_20:
INIT4x1
mov pB, origPB
asr counterL , origK, #3 // counterL = counterL / 8
cmp counterL , #0
ble .Lcgemm_kernel_L1_M4_40
.align 5
.Lcgemm_kernel_L1_M4_22:
KERNEL4x1_SUB
KERNEL4x1_SUB
KERNEL4x1_SUB
KERNEL4x1_SUB
KERNEL4x1_SUB
KERNEL4x1_SUB
KERNEL4x1_SUB
KERNEL4x1_SUB
subs counterL, counterL, #1
bgt .Lcgemm_kernel_L1_M4_22
.Lcgemm_kernel_L1_M4_40:
ands counterL , origK, #7 // counterL = counterL % 8
ble .Lcgemm_kernel_L1_M4_100
.Lcgemm_kernel_L1_M4_42:
KERNEL4x1_SUB
subs counterL, counterL, #1
bgt .Lcgemm_kernel_L1_M4_42
.Lcgemm_kernel_L1_M4_100:
SAVE4x1
.Lcgemm_kernel_L1_M4_END:
.Lcgemm_kernel_L1_M2_BEGIN:
mov counterI, origM
tst counterI , #3
ble .Lcgemm_kernel_L1_END
tst counterI, #2 // counterI = counterI / 2
ble .Lcgemm_kernel_L1_M1_BEGIN
.Lcgemm_kernel_L1_M2_20:
INIT2x1
mov pB, origPB
asr counterL , origK, #3 // counterL = counterL / 8
cmp counterL , #0
ble .Lcgemm_kernel_L1_M2_40
.Lcgemm_kernel_L1_M2_22:
KERNEL2x1_SUB
KERNEL2x1_SUB
KERNEL2x1_SUB
KERNEL2x1_SUB
KERNEL2x1_SUB
KERNEL2x1_SUB
KERNEL2x1_SUB
KERNEL2x1_SUB
subs counterL, counterL, #1
bgt .Lcgemm_kernel_L1_M2_22
.Lcgemm_kernel_L1_M2_40:
ands counterL , origK, #7 // counterL = counterL % 8
ble .Lcgemm_kernel_L1_M2_100
.Lcgemm_kernel_L1_M2_42:
KERNEL2x1_SUB
subs counterL, counterL, #1
bgt .Lcgemm_kernel_L1_M2_42
.Lcgemm_kernel_L1_M2_100:
SAVE2x1
.Lcgemm_kernel_L1_M2_END:
.Lcgemm_kernel_L1_M1_BEGIN:
tst counterI, #1 // counterI = counterI % 2
ble .Lcgemm_kernel_L1_END
.Lcgemm_kernel_L1_M1_20:
INIT1x1
mov pB, origPB
asr counterL , origK, #3 // counterL = counterL / 8
cmp counterL , #0
ble .Lcgemm_kernel_L1_M1_40
.Lcgemm_kernel_L1_M1_22:
KERNEL1x1_SUB
KERNEL1x1_SUB
KERNEL1x1_SUB
KERNEL1x1_SUB
KERNEL1x1_SUB
KERNEL1x1_SUB
KERNEL1x1_SUB
KERNEL1x1_SUB
subs counterL, counterL, #1
bgt .Lcgemm_kernel_L1_M1_22
.Lcgemm_kernel_L1_M1_40:
ands counterL , origK, #7 // counterL = counterL % 8
ble .Lcgemm_kernel_L1_M1_100
.Lcgemm_kernel_L1_M1_42:
KERNEL1x1_SUB
subs counterL, counterL, #1
bgt .Lcgemm_kernel_L1_M1_42
.Lcgemm_kernel_L1_M1_100:
SAVE1x1
.Lcgemm_kernel_L1_END:
.Lcgemm_kernel_L999:
mov x0, #0 // set return value
ldp d8, d9, [sp, #(0 * 16)]
ldp d10, d11, [sp, #(1 * 16)]
ldp d12, d13, [sp, #(2 * 16)]
ldp d14, d15, [sp, #(3 * 16)]
ldp d16, d17, [sp, #(4 * 16)]
ldp x18, x19, [sp, #(5 * 16)]
ldp x20, x21, [sp, #(6 * 16)]
ldp x22, x23, [sp, #(7 * 16)]
ldp x24, x25, [sp, #(8 * 16)]
ldp x26, x27, [sp, #(9 * 16)]
ldr x28, [sp, #(10 * 16)]
add sp, sp, #(11*16)
ret
EPILOGUE
Reference in New Issue View Git Blame Copy Permalink