diff --git a/kernel/arm64/KERNEL.CORTEXA57 b/kernel/arm64/KERNEL.CORTEXA57 index 7c8eeeea7..64666f05b 100644 --- a/kernel/arm64/KERNEL.CORTEXA57 +++ b/kernel/arm64/KERNEL.CORTEXA57 @@ -60,32 +60,55 @@ DGEMVTKERNEL = gemv_t.S CGEMVTKERNEL = zgemv_t.S ZGEMVTKERNEL = zgemv_t.S -STRMMKERNEL = strmm_kernel_4x4.S -DTRMMKERNEL = dtrmm_kernel_4x4.S -CTRMMKERNEL = ctrmm_kernel_4x4.S -ZTRMMKERNEL = ztrmm_kernel_4x4.S - -SGEMMKERNEL = sgemm_kernel_4x4.S -SGEMMONCOPY = ../generic/gemm_ncopy_4.c -SGEMMOTCOPY = ../generic/gemm_tcopy_4.c +SGEMMKERNEL = sgemm_kernel_$(SGEMM_UNROLL_M)x$(SGEMM_UNROLL_N).S +STRMMKERNEL = strmm_kernel_$(SGEMM_UNROLL_M)x$(SGEMM_UNROLL_N).S +ifneq ($(SGEMM_UNROLL_M), $(SGEMM_UNROLL_N)) +SGEMMINCOPY = ../generic/gemm_ncopy_$(SGEMM_UNROLL_M).c +SGEMMITCOPY = ../generic/gemm_tcopy_$(SGEMM_UNROLL_M).c +SGEMMINCOPYOBJ = sgemm_incopy.o +SGEMMITCOPYOBJ = sgemm_itcopy.o +endif +SGEMMONCOPY = ../generic/gemm_ncopy_$(SGEMM_UNROLL_N).c +SGEMMOTCOPY = ../generic/gemm_tcopy_$(SGEMM_UNROLL_N).c SGEMMONCOPYOBJ = sgemm_oncopy.o SGEMMOTCOPYOBJ = sgemm_otcopy.o -DGEMMKERNEL = dgemm_kernel_4x4.S -DGEMMONCOPY = ../generic/gemm_ncopy_4.c -DGEMMOTCOPY = ../generic/gemm_tcopy_4.c +DGEMMKERNEL = dgemm_kernel_$(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N).S +DTRMMKERNEL = dtrmm_kernel_$(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N).S +ifneq ($(DGEMM_UNROLL_M), $(DGEMM_UNROLL_N)) +DGEMMINCOPY = ../generic/gemm_ncopy_$(DGEMM_UNROLL_M).c +DGEMMITCOPY = ../generic/gemm_tcopy_$(DGEMM_UNROLL_M).c +DGEMMINCOPYOBJ = dgemm_incopy.o +DGEMMITCOPYOBJ = dgemm_itcopy.o +endif +DGEMMONCOPY = ../generic/gemm_ncopy_$(DGEMM_UNROLL_N).c +DGEMMOTCOPY = ../generic/gemm_tcopy_$(DGEMM_UNROLL_N).c DGEMMONCOPYOBJ = dgemm_oncopy.o DGEMMOTCOPYOBJ = dgemm_otcopy.o -CGEMMKERNEL = cgemm_kernel_4x4.S -CGEMMONCOPY = ../generic/zgemm_ncopy_4.c -CGEMMOTCOPY = ../generic/zgemm_tcopy_4.c +CGEMMKERNEL = cgemm_kernel_$(CGEMM_UNROLL_M)x$(CGEMM_UNROLL_N).S +CTRMMKERNEL = ctrmm_kernel_$(CGEMM_UNROLL_M)x$(CGEMM_UNROLL_N).S +ifneq ($(CGEMM_UNROLL_M), $(CGEMM_UNROLL_N)) +CGEMMINCOPY = ../generic/zgemm_ncopy_$(CGEMM_UNROLL_M).c +CGEMMITCOPY = ../generic/zgemm_tcopy_$(CGEMM_UNROLL_M).c +CGEMMINCOPYOBJ = cgemm_incopy.o +CGEMMITCOPYOBJ = cgemm_itcopy.o +endif +CGEMMONCOPY = ../generic/zgemm_ncopy_$(CGEMM_UNROLL_N).c +CGEMMOTCOPY = ../generic/zgemm_tcopy_$(CGEMM_UNROLL_N).c CGEMMONCOPYOBJ = cgemm_oncopy.o CGEMMOTCOPYOBJ = cgemm_otcopy.o -ZGEMMKERNEL = zgemm_kernel_4x4.S -ZGEMMONCOPY = ../generic/zgemm_ncopy_4.c -ZGEMMOTCOPY = ../generic/zgemm_tcopy_4.c +ZGEMMKERNEL = zgemm_kernel_$(ZGEMM_UNROLL_M)x$(ZGEMM_UNROLL_N).S +ZTRMMKERNEL = ztrmm_kernel_$(ZGEMM_UNROLL_M)x$(ZGEMM_UNROLL_N).S +ifneq ($(ZGEMM_UNROLL_M), $(ZGEMM_UNROLL_N)) +ZGEMMINCOPY = ../generic/zgemm_ncopy_$(ZGEMM_UNROLL_M).c +ZGEMMITCOPY = ../generic/zgemm_tcopy_$(ZGEMM_UNROLL_M).c +ZGEMMINCOPYOBJ = zgemm_incopy.o +ZGEMMITCOPYOBJ = zgemm_itcopy.o +endif +ZGEMMONCOPY = ../generic/zgemm_ncopy_$(ZGEMM_UNROLL_N).c +ZGEMMOTCOPY = ../generic/zgemm_tcopy_$(ZGEMM_UNROLL_N).c ZGEMMONCOPYOBJ = zgemm_oncopy.o ZGEMMOTCOPYOBJ = zgemm_otcopy.o diff --git a/kernel/arm64/cgemm_kernel_8x4.S b/kernel/arm64/cgemm_kernel_8x4.S new file mode 100755 index 000000000..40b98cee2 --- /dev/null +++ b/kernel/arm64/cgemm_kernel_8x4.S @@ -0,0 +1,2044 @@ +/******************************************************************************* +Copyright (c) 2015, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#define ASSEMBLER +#include "common.h" + +/* X0 X1 X2 s0 X3 x4 x5 x6 */ +/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha0,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc */ + +#define origM x0 +#define origN x1 +#define origK x2 +#define origPA x3 +#define origPB x4 +#define pC x5 +#define LDC x6 +#define temp x7 +#define counterL x8 +#define counterI x9 +#define counterJ x10 +#define pB x11 +#define pCRow0 x12 +#define pCRow1 x13 +#define pCRow2 x14 +#define pA x15 + +#define alpha0_R s10 +#define alphaV0_R v10.s[0] +#define alpha0_I s11 +#define alphaV0_I v11.s[0] + +#define alpha1_R s14 +#define alphaV1_R v14.s[0] +#define alpha1_I s15 +#define alphaV1_I v15.s[0] + +#if defined(NN) || defined(NT) || defined(TN) || defined(TT) +#define OP_rr fmla +#define OP_ii fmls +#define OP_ri fmla +#define OP_ir fmla +#elif defined(NR) || defined(NC) || defined(TR) || defined(TC) +#define OP_rr fmla +#define OP_ii fmla +#define OP_ri fmls +#define OP_ir fmla +#elif defined(RN) || defined(RT) || defined(CN) || defined(CT) +#define OP_rr fmla +#define OP_ii fmla +#define OP_ri fmla +#define OP_ir fmls +#elif defined(RR) || defined(RC) || defined(CR) || defined(CC) +#define OP_rr fmla +#define OP_ii fmls +#define OP_ri fmls +#define OP_ir fmls +#endif + +// 00 origM +// 01 origN +// 02 origK +// 03 origPA +// 04 origPB +// 05 pC +// 06 origLDC -> LDC +// 07 offset -> temp +// 08 counterL +// 09 counterI +// 10 counterJ +// 11 pB +// 12 pCRow0 +// 13 pCRow1 +// 14 pCRow2 +// 15 pA +// 16 +// 17 +// 18 must save +// 19 must save +// 20 must save +// 21 must save +// 22 must save +// 23 must save +// 24 must save +// 25 must save +// 26 must save +// 27 must save +// 28 must save +// 29 frame +// 30 link +// 31 sp + +//v00 ALPHA_R -> pA0_00_R, pA0_01_R, pA0_02_R, pA0_03_R +//v01 ALPHA_I -> pA0_00_I, pA0_01_I, pA0_02_I, pA0_03_I +//v02 pA0_04_R, pA0_05_R, pA0_06_R, pA0_07_R +//v03 pA0_04_I, pA0_05_I, pA0_06_I, pA0_07_I +//v04 pA1_00_R, pA1_01_R, pA1_02_R, pA1_03_R +//v05 pA1_00_I, pA1_01_I, pA1_02_I, pA1_03_I +//v06 pA1_04_R, pA1_05_R, pA1_06_R, pA1_07_R +//v07 pA1_04_I, pA1_05_I, pA1_06_I, pA1_07_I +//v08 must save pB0_00_R, pB0_01_R, pB0_02_R, pB0_03_R +//v09 must save pB0_00_I, pB0_01_I, pB0_02_I, pB0_03_I +//v10 must save ALPHA0_R +//v11 must save ALPHA0_I +//v12 must save pB1_00_R, pB1_01_R, pB1_02_R, pB1_03_R +//v13 must save pB1_00_I, pB1_01_I, pB1_02_I, pB1_03_I +//v14 must save ALPHA1_R +//v15 must save ALPHA1_I +//v16 must save pC_00_R, pC_01_R, pC_02_R, pC_03_R +//v17 must save pC_00_I, pC_01_I, pC_02_I, pC_03_I +//v18 pC_04_R, pC_05_R, pC_06_R, pC_07_R +//v19 pC_04_I, pC_05_I, pC_06_I, pC_07_I +//v20 pC_08_R, pC_09_R, pC_10_R, pC_11_R +//v21 pC_08_I, pC_09_I, pC_10_I, pC_11_I +//v22 pC_12_R, pC_13_R, pC_14_R, pC_15_R +//v23 pC_12_I, pC_13_I, pC_14_I, pC_15_I +//v24 pC_16_R, pC_17_R, pC_18_R, pC_19_R +//v25 pC_16_I, pC_17_I, pC_18_I, pC_19_I +//v26 pC_20_R, pC_21_R, pC_22_R, pC_23_R +//v27 pC_20_I, pC_21_I, pC_22_I, pC_23_I +//v28 pC_24_R, pC_25_R, pC_26_R, pC_27_R +//v29 pC_24_I, pC_25_I, pC_26_I, pC_27_I +//v30 pC_28_R, pC_29_R, pC_30_R, pC_31_R +//v31 pC_28_I, pC_29_I, pC_30_I, pC_31_I + + +/******************************************************************************* +* Macro definitions +*******************************************************************************/ + +.macro INIT8x4 + fmov s16, wzr + fmov s17, wzr + fmov s18, wzr + fmov s19, s16 + fmov s20, wzr + fmov s21, s16 + fmov s22, s17 + fmov s23, s18 + fmov s24, wzr + fmov s25, s16 + fmov s26, s17 + fmov s27, s18 + fmov s28, wzr + fmov s29, s16 + fmov s30, s17 + fmov s31, s18 +.endm + +.macro KERNEL8x4_I + ld2 {v8.4s, v9.4s}, [pB] + add pB, pB, #32 + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + ld2 {v2.4s, v3.4s}, [pA] + add pA, pA, #32 + + fmul v16.4s, v0.4s, v8.4s[0] + OP_ii v16.4s, v1.4s, v9.4s[0] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v17.16b, v17.16b, v17.16b + fmls v17.4s, v0.4s, v9.4s[0] +#else + fmul v17.4s, v0.4s, v9.4s[0] +#endif + OP_ir v17.4s, v1.4s, v8.4s[0] + + fmul v18.4s, v2.4s, v8.4s[0] + OP_ii v18.4s, v3.4s, v9.4s[0] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v19.16b, v19.16b, v19.16b + fmls v19.4s, v2.4s, v9.4s[0] +#else + fmul v19.4s, v2.4s, v9.4s[0] +#endif + OP_ir v19.4s, v3.4s, v8.4s[0] + + fmul v20.4s, v0.4s, v8.4s[1] + OP_ii v20.4s, v1.4s, v9.4s[1] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v21.16b, v21.16b, v21.16b + fmls v21.4s, v0.4s, v9.4s[1] +#else + fmul v21.4s, v0.4s, v9.4s[1] +#endif + OP_ir v21.4s, v1.4s, v8.4s[1] + + fmul v22.4s, v2.4s, v8.4s[1] + OP_ii v22.4s, v3.4s, v9.4s[1] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v23.16b, v23.16b, v23.16b + fmls v23.4s, v2.4s, v9.4s[1] +#else + fmul v23.4s, v2.4s, v9.4s[1] +#endif + OP_ir v23.4s, v3.4s, v8.4s[1] + + fmul v24.4s, v0.4s, v8.4s[2] + OP_ii v24.4s, v1.4s, v9.4s[2] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v25.16b, v25.16b, v25.16b + fmls v25.4s, v0.4s, v9.4s[2] +#else + fmul v25.4s, v0.4s, v9.4s[2] +#endif + OP_ir v25.4s, v1.4s, v8.4s[2] + + fmul v26.4s, v2.4s, v8.4s[2] + OP_ii v26.4s, v3.4s, v9.4s[2] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v27.16b, v27.16b, v27.16b + fmls v27.4s, v2.4s, v9.4s[2] +#else + fmul v27.4s, v2.4s, v9.4s[2] +#endif + OP_ir v27.4s, v3.4s, v8.4s[2] + + fmul v28.4s, v0.4s, v8.4s[3] + OP_ii v28.4s, v1.4s, v9.4s[3] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v29.16b, v29.16b, v29.16b + fmls v29.4s, v0.4s, v9.4s[3] +#else + fmul v29.4s, v0.4s, v9.4s[3] +#endif + OP_ir v29.4s, v1.4s, v8.4s[3] + + fmul v30.4s, v2.4s, v8.4s[3] + OP_ii v30.4s, v3.4s, v9.4s[3] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v31.16b, v31.16b, v31.16b + fmls v31.4s, v2.4s, v9.4s[3] +#else + fmul v31.4s, v2.4s, v9.4s[3] +#endif + OP_ir v31.4s, v3.4s, v8.4s[3] + + ld2 {v12.4s, v13.4s}, [pB] + add pB, pB, #32 + ld2 {v4.4s, v5.4s}, [pA] + add pA, pA, #32 + ld2 {v6.4s, v7.4s}, [pA] + add pA, pA, #32 +.endm + +.macro KERNEL8x4_M1 + OP_rr v16.4s, v0.4s, v8.4s[0] + OP_ii v16.4s, v1.4s, v9.4s[0] + OP_ri v17.4s, v0.4s, v9.4s[0] + OP_ir v17.4s, v1.4s, v8.4s[0] + + OP_rr v18.4s, v2.4s, v8.4s[0] + OP_ii v18.4s, v3.4s, v9.4s[0] + OP_ri v19.4s, v2.4s, v9.4s[0] + OP_ir v19.4s, v3.4s, v8.4s[0] + + OP_rr v20.4s, v0.4s, v8.4s[1] + OP_ii v20.4s, v1.4s, v9.4s[1] + OP_ri v21.4s, v0.4s, v9.4s[1] + OP_ir v21.4s, v1.4s, v8.4s[1] + + OP_rr v22.4s, v2.4s, v8.4s[1] + OP_ii v22.4s, v3.4s, v9.4s[1] + OP_ri v23.4s, v2.4s, v9.4s[1] + OP_ir v23.4s, v3.4s, v8.4s[1] + + OP_rr v24.4s, v0.4s, v8.4s[2] + OP_ii v24.4s, v1.4s, v9.4s[2] + OP_ri v25.4s, v0.4s, v9.4s[2] + OP_ir v25.4s, v1.4s, v8.4s[2] + + OP_rr v26.4s, v2.4s, v8.4s[2] + OP_ii v26.4s, v3.4s, v9.4s[2] + OP_ri v27.4s, v2.4s, v9.4s[2] + OP_ir v27.4s, v3.4s, v8.4s[2] + + OP_rr v28.4s, v0.4s, v8.4s[3] + OP_ii v28.4s, v1.4s, v9.4s[3] + OP_ri v29.4s, v0.4s, v9.4s[3] + OP_ir v29.4s, v1.4s, v8.4s[3] + + OP_rr v30.4s, v2.4s, v8.4s[3] + OP_ii v30.4s, v3.4s, v9.4s[3] + OP_ri v31.4s, v2.4s, v9.4s[3] + OP_ir v31.4s, v3.4s, v8.4s[3] + + ld2 {v12.4s, v13.4s}, [pB] // For next round + add pB, pB, #32 + ld2 {v4.4s, v5.4s}, [pA] // For next round + add pA, pA, #32 + ld2 {v6.4s, v7.4s}, [pA] + add pA, pA, #32 +.endm + +.macro KERNEL8x4_M2 + OP_rr v16.4s, v4.4s, v12.4s[0] + OP_ii v16.4s, v5.4s, v13.4s[0] + OP_ri v17.4s, v4.4s, v13.4s[0] + OP_ir v17.4s, v5.4s, v12.4s[0] + + OP_rr v18.4s, v6.4s, v12.4s[0] + OP_ii v18.4s, v7.4s, v13.4s[0] + OP_ri v19.4s, v6.4s, v13.4s[0] + OP_ir v19.4s, v7.4s, v12.4s[0] + + OP_rr v20.4s, v4.4s, v12.4s[1] + OP_ii v20.4s, v5.4s, v13.4s[1] + OP_ri v21.4s, v4.4s, v13.4s[1] + OP_ir v21.4s, v5.4s, v12.4s[1] + + OP_rr v22.4s, v6.4s, v12.4s[1] + OP_ii v22.4s, v7.4s, v13.4s[1] + OP_ri v23.4s, v6.4s, v13.4s[1] + OP_ir v23.4s, v7.4s, v12.4s[1] + + OP_rr v24.4s, v4.4s, v12.4s[2] + OP_ii v24.4s, v5.4s, v13.4s[2] + OP_ri v25.4s, v4.4s, v13.4s[2] + OP_ir v25.4s, v5.4s, v12.4s[2] + + OP_rr v26.4s, v6.4s, v12.4s[2] + OP_ii v26.4s, v7.4s, v13.4s[2] + OP_ri v27.4s, v6.4s, v13.4s[2] + OP_ir v27.4s, v7.4s, v12.4s[2] + + OP_rr v28.4s, v4.4s, v12.4s[3] + OP_ii v28.4s, v5.4s, v13.4s[3] + OP_ri v29.4s, v4.4s, v13.4s[3] + OP_ir v29.4s, v5.4s, v12.4s[3] + + OP_rr v30.4s, v6.4s, v12.4s[3] + OP_ii v30.4s, v7.4s, v13.4s[3] + OP_ri v31.4s, v6.4s, v13.4s[3] + OP_ir v31.4s, v7.4s, v12.4s[3] + + ld2 {v8.4s, v9.4s}, [pB] + add pB, pB, #32 + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + ld2 {v2.4s, v3.4s}, [pA] + add pA, pA, #32 +.endm + +.macro KERNEL8x4_E + OP_rr v16.4s, v4.4s, v12.4s[0] + OP_ii v16.4s, v5.4s, v13.4s[0] + OP_ri v17.4s, v4.4s, v13.4s[0] + OP_ir v17.4s, v5.4s, v12.4s[0] + + OP_rr v18.4s, v6.4s, v12.4s[0] + OP_ii v18.4s, v7.4s, v13.4s[0] + OP_ri v19.4s, v6.4s, v13.4s[0] + OP_ir v19.4s, v7.4s, v12.4s[0] + + OP_rr v20.4s, v4.4s, v12.4s[1] + OP_ii v20.4s, v5.4s, v13.4s[1] + OP_ri v21.4s, v4.4s, v13.4s[1] + OP_ir v21.4s, v5.4s, v12.4s[1] + + OP_rr v22.4s, v6.4s, v12.4s[1] + OP_ii v22.4s, v7.4s, v13.4s[1] + OP_ri v23.4s, v6.4s, v13.4s[1] + OP_ir v23.4s, v7.4s, v12.4s[1] + + OP_rr v24.4s, v4.4s, v12.4s[2] + OP_ii v24.4s, v5.4s, v13.4s[2] + OP_ri v25.4s, v4.4s, v13.4s[2] + OP_ir v25.4s, v5.4s, v12.4s[2] + + OP_rr v26.4s, v6.4s, v12.4s[2] + OP_ii v26.4s, v7.4s, v13.4s[2] + OP_ri v27.4s, v6.4s, v13.4s[2] + OP_ir v27.4s, v7.4s, v12.4s[2] + + OP_rr v28.4s, v4.4s, v12.4s[3] + OP_ii v28.4s, v5.4s, v13.4s[3] + OP_ri v29.4s, v4.4s, v13.4s[3] + OP_ir v29.4s, v5.4s, v12.4s[3] + + OP_rr v30.4s, v6.4s, v12.4s[3] + OP_ii v30.4s, v7.4s, v13.4s[3] + OP_ri v31.4s, v6.4s, v13.4s[3] + OP_ir v31.4s, v7.4s, v12.4s[3] + +.endm + +.macro KERNEL8x4_SUB + ld2 {v8.4s, v9.4s}, [pB] + add pB, pB, #32 + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + ld2 {v2.4s, v3.4s}, [pA] + add pA, pA, #32 + + OP_rr v16.4s, v0.4s, v8.4s[0] + OP_ii v16.4s, v1.4s, v9.4s[0] + OP_ri v17.4s, v0.4s, v9.4s[0] + OP_ir v17.4s, v1.4s, v8.4s[0] + + OP_rr v18.4s, v2.4s, v8.4s[0] + OP_ii v18.4s, v3.4s, v9.4s[0] + OP_ri v19.4s, v2.4s, v9.4s[0] + OP_ir v19.4s, v3.4s, v8.4s[0] + + OP_rr v20.4s, v0.4s, v8.4s[1] + OP_ii v20.4s, v1.4s, v9.4s[1] + OP_ri v21.4s, v0.4s, v9.4s[1] + OP_ir v21.4s, v1.4s, v8.4s[1] + + OP_rr v22.4s, v2.4s, v8.4s[1] + OP_ii v22.4s, v3.4s, v9.4s[1] + OP_ri v23.4s, v2.4s, v9.4s[1] + OP_ir v23.4s, v3.4s, v8.4s[1] + + OP_rr v24.4s, v0.4s, v8.4s[2] + OP_ii v24.4s, v1.4s, v9.4s[2] + OP_ri v25.4s, v0.4s, v9.4s[2] + OP_ir v25.4s, v1.4s, v8.4s[2] + + OP_rr v26.4s, v2.4s, v8.4s[2] + OP_ii v26.4s, v3.4s, v9.4s[2] + OP_ri v27.4s, v2.4s, v9.4s[2] + OP_ir v27.4s, v3.4s, v8.4s[2] + + OP_rr v28.4s, v0.4s, v8.4s[3] + OP_ii v28.4s, v1.4s, v9.4s[3] + OP_ri v29.4s, v0.4s, v9.4s[3] + OP_ir v29.4s, v1.4s, v8.4s[3] + + OP_rr v30.4s, v2.4s, v8.4s[3] + OP_ii v30.4s, v3.4s, v9.4s[3] + OP_ri v31.4s, v2.4s, v9.4s[3] + OP_ir v31.4s, v3.4s, v8.4s[3] + +.endm + +.macro SAVE8x4 + mov pCRow1, pCRow0 + + ld2 {v0.4s, v1.4s}, [pCRow1] + fmla v0.4s, v16.4s, alphaV0_R + fmls v0.4s, v17.4s, alphaV0_I + fmla v1.4s, v16.4s, alphaV1_I + fmla v1.4s, v17.4s, alphaV1_R + st2 {v0.4s, v1.4s}, [pCRow1] + + add pCRow2, pCRow1, #32 + + ld2 {v2.4s, v3.4s}, [pCRow2] + fmla v2.4s, v18.4s, alphaV0_R + fmls v2.4s, v19.4s, alphaV0_I + fmla v3.4s, v18.4s, alphaV1_I + fmla v3.4s, v19.4s, alphaV1_R + st2 {v2.4s, v3.4s}, [pCRow2] + + add pCRow1, pCRow1, LDC + + ld2 {v4.4s, v5.4s}, [pCRow1] + fmla v4.4s, v20.4s, alphaV0_R + fmls v4.4s, v21.4s, alphaV0_I + fmla v5.4s, v20.4s, alphaV1_I + fmla v5.4s, v21.4s, alphaV1_R + st2 {v4.4s, v5.4s}, [pCRow1] + + add pCRow2, pCRow1, #32 + + ld2 {v6.4s, v7.4s}, [pCRow2] + fmla v6.4s, v22.4s, alphaV0_R + fmls v6.4s, v23.4s, alphaV0_I + fmla v7.4s, v22.4s, alphaV1_I + fmla v7.4s, v23.4s, alphaV1_R + st2 {v6.4s, v7.4s}, [pCRow2] + + add pCRow1, pCRow1, LDC + + ld2 {v0.4s, v1.4s}, [pCRow1] + fmla v0.4s, v24.4s, alphaV0_R + fmls v0.4s, v25.4s, alphaV0_I + fmla v1.4s, v24.4s, alphaV1_I + fmla v1.4s, v25.4s, alphaV1_R + st2 {v0.4s, v1.4s}, [pCRow1] + + add pCRow2, pCRow1, #32 + + ld2 {v2.4s, v3.4s}, [pCRow2] + fmla v2.4s, v26.4s, alphaV0_R + fmls v2.4s, v27.4s, alphaV0_I + fmla v3.4s, v26.4s, alphaV1_I + fmla v3.4s, v27.4s, alphaV1_R + st2 {v2.4s, v3.4s}, [pCRow2] + + add pCRow1, pCRow1, LDC + + ld2 {v4.4s, v5.4s}, [pCRow1] + fmla v4.4s, v28.4s, alphaV0_R + fmls v4.4s, v29.4s, alphaV0_I + fmla v5.4s, v28.4s, alphaV1_I + fmla v5.4s, v29.4s, alphaV1_R + st2 {v4.4s, v5.4s}, [pCRow1] + + add pCRow2, pCRow1, #32 + + ld2 {v6.4s, v7.4s}, [pCRow2] + fmla v6.4s, v30.4s, alphaV0_R + fmls v6.4s, v31.4s, alphaV0_I + fmla v7.4s, v30.4s, alphaV1_I + fmla v7.4s, v31.4s, alphaV1_R + st2 {v6.4s, v7.4s}, [pCRow2] + + add pCRow0, pCRow0, #64 +.endm + +/******************************************************************************/ + +.macro INIT4x4 + fmov s16, wzr + fmov s17, s16 + fmov s20, s17 + fmov s21, s16 + fmov s24, s17 + fmov s25, s16 + fmov s28, s17 + fmov s29, s16 +.endm + +.macro KERNEL4x4_I + ld2 {v8.4s, v9.4s}, [pB] + add pB, pB, #32 + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + + fmul v16.4s, v0.4s, v8.4s[0] + OP_ii v16.4s, v1.4s, v9.4s[0] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v17.16b, v17.16b, v17.16b + fmls v17.4s, v0.4s, v9.4s[0] +#else + fmul v17.4s, v0.4s, v9.4s[0] +#endif + OP_ir v17.4s, v1.4s, v8.4s[0] + + fmul v20.4s, v0.4s, v8.4s[1] + OP_ii v20.4s, v1.4s, v9.4s[1] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v21.16b, v21.16b, v21.16b + fmls v21.4s, v0.4s, v9.4s[1] +#else + fmul v21.4s, v0.4s, v9.4s[1] +#endif + OP_ir v21.4s, v1.4s, v8.4s[1] + + fmul v24.4s, v0.4s, v8.4s[2] + OP_ii v24.4s, v1.4s, v9.4s[2] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v25.16b, v25.16b, v25.16b + fmls v25.4s, v0.4s, v9.4s[2] +#else + fmul v25.4s, v0.4s, v9.4s[2] +#endif + OP_ir v25.4s, v1.4s, v8.4s[2] + + fmul v28.4s, v0.4s, v8.4s[3] + OP_ii v28.4s, v1.4s, v9.4s[3] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v29.16b, v29.16b, v29.16b + fmls v29.4s, v0.4s, v9.4s[3] +#else + fmul v29.4s, v0.4s, v9.4s[3] +#endif + OP_ir v29.4s, v1.4s, v8.4s[3] + + ld2 {v12.4s, v13.4s}, [pB] + add pB, pB, #32 + ld2 {v4.4s, v5.4s}, [pA] + add pA, pA, #32 +.endm + +.macro KERNEL4x4_M1 + OP_rr v16.4s, v0.4s, v8.4s[0] + OP_ii v16.4s, v1.4s, v9.4s[0] + OP_ri v17.4s, v0.4s, v9.4s[0] + OP_ir v17.4s, v1.4s, v8.4s[0] + + ld2 {v12.4s, v13.4s}, [pB] // For next round + add pB, pB, #32 + + OP_rr v20.4s, v0.4s, v8.4s[1] + OP_ii v20.4s, v1.4s, v9.4s[1] + OP_ri v21.4s, v0.4s, v9.4s[1] + OP_ir v21.4s, v1.4s, v8.4s[1] + + ld2 {v4.4s, v5.4s}, [pA] // For next round + add pA, pA, #32 + + OP_rr v24.4s, v0.4s, v8.4s[2] + OP_ii v24.4s, v1.4s, v9.4s[2] + OP_ri v25.4s, v0.4s, v9.4s[2] + OP_ir v25.4s, v1.4s, v8.4s[2] + + prfm PLDL1KEEP, [pA, #512] + + OP_rr v28.4s, v0.4s, v8.4s[3] + OP_ii v28.4s, v1.4s, v9.4s[3] + OP_ri v29.4s, v0.4s, v9.4s[3] + OP_ir v29.4s, v1.4s, v8.4s[3] +.endm + +.macro KERNEL4x4_M2 + OP_rr v16.4s, v4.4s, v12.4s[0] + OP_ii v16.4s, v5.4s, v13.4s[0] + OP_ri v17.4s, v4.4s, v13.4s[0] + OP_ir v17.4s, v5.4s, v12.4s[0] + + ld2 {v8.4s, v9.4s}, [pB] // For next round + add pB, pB, #32 + + OP_rr v20.4s, v4.4s, v12.4s[1] + OP_ii v20.4s, v5.4s, v13.4s[1] + OP_ri v21.4s, v4.4s, v13.4s[1] + OP_ir v21.4s, v5.4s, v12.4s[1] + + ld2 {v0.4s, v1.4s}, [pA] // For next round + add pA, pA, #32 + + OP_rr v24.4s, v4.4s, v12.4s[2] + OP_ii v24.4s, v5.4s, v13.4s[2] + OP_ri v25.4s, v4.4s, v13.4s[2] + OP_ir v25.4s, v5.4s, v12.4s[2] + + prfm PLDL1KEEP, [pB, #512] + + OP_rr v28.4s, v4.4s, v12.4s[3] + OP_ii v28.4s, v5.4s, v13.4s[3] + OP_ri v29.4s, v4.4s, v13.4s[3] + OP_ir v29.4s, v5.4s, v12.4s[3] +.endm + +.macro KERNEL4x4_E + OP_rr v16.4s, v4.4s, v12.4s[0] + OP_ii v16.4s, v5.4s, v13.4s[0] + OP_ri v17.4s, v4.4s, v13.4s[0] + OP_ir v17.4s, v5.4s, v12.4s[0] + + OP_rr v20.4s, v4.4s, v12.4s[1] + OP_ii v20.4s, v5.4s, v13.4s[1] + OP_ri v21.4s, v4.4s, v13.4s[1] + OP_ir v21.4s, v5.4s, v12.4s[1] + + OP_rr v24.4s, v4.4s, v12.4s[2] + OP_ii v24.4s, v5.4s, v13.4s[2] + OP_ri v25.4s, v4.4s, v13.4s[2] + OP_ir v25.4s, v5.4s, v12.4s[2] + + OP_rr v28.4s, v4.4s, v12.4s[3] + OP_ii v28.4s, v5.4s, v13.4s[3] + OP_ri v29.4s, v4.4s, v13.4s[3] + OP_ir v29.4s, v5.4s, v12.4s[3] +.endm + +.macro KERNEL4x4_SUB + ld2 {v8.4s, v9.4s}, [pB] + add pB, pB, #32 + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + + OP_rr v16.4s, v0.4s, v8.4s[0] + OP_ii v16.4s, v1.4s, v9.4s[0] + OP_ri v17.4s, v0.4s, v9.4s[0] + OP_ir v17.4s, v1.4s, v8.4s[0] + + OP_rr v20.4s, v0.4s, v8.4s[1] + OP_ii v20.4s, v1.4s, v9.4s[1] + OP_ri v21.4s, v0.4s, v9.4s[1] + OP_ir v21.4s, v1.4s, v8.4s[1] + + OP_rr v24.4s, v0.4s, v8.4s[2] + OP_ii v24.4s, v1.4s, v9.4s[2] + OP_ri v25.4s, v0.4s, v9.4s[2] + OP_ir v25.4s, v1.4s, v8.4s[2] + + OP_rr v28.4s, v0.4s, v8.4s[3] + OP_ii v28.4s, v1.4s, v9.4s[3] + OP_ri v29.4s, v0.4s, v9.4s[3] + OP_ir v29.4s, v1.4s, v8.4s[3] +.endm + +.macro SAVE4x4 + mov pCRow1, pCRow0 + + ld2 {v0.4s, v1.4s}, [pCRow1] + fmla v0.4s, v16.4s, alphaV0_R + fmls v0.4s, v17.4s, alphaV0_I + fmla v1.4s, v16.4s, alphaV1_I + fmla v1.4s, v17.4s, alphaV1_R + st2 {v0.4s, v1.4s}, [pCRow1] + + add pCRow1, pCRow1, LDC + + ld2 {v4.4s, v5.4s}, [pCRow1] + fmla v4.4s, v20.4s, alphaV0_R + fmls v4.4s, v21.4s, alphaV0_I + fmla v5.4s, v20.4s, alphaV1_I + fmla v5.4s, v21.4s, alphaV1_R + st2 {v4.4s, v5.4s}, [pCRow1] + + add pCRow1, pCRow1, LDC + + ld2 {v0.4s, v1.4s}, [pCRow1] + fmla v0.4s, v24.4s, alphaV0_R + fmls v0.4s, v25.4s, alphaV0_I + fmla v1.4s, v24.4s, alphaV1_I + fmla v1.4s, v25.4s, alphaV1_R + st2 {v0.4s, v1.4s}, [pCRow1] + + add pCRow1, pCRow1, LDC + + ld2 {v4.4s, v5.4s}, [pCRow1] + fmla v4.4s, v28.4s, alphaV0_R + fmls v4.4s, v29.4s, alphaV0_I + fmla v5.4s, v28.4s, alphaV1_I + fmla v5.4s, v29.4s, alphaV1_R + st2 {v4.4s, v5.4s}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT2x4 + fmov s16, wzr + fmov s17, wzr + fmov s20, s16 + fmov s21, s17 + fmov s24, s16 + fmov s25, s17 + fmov s28, s16 + fmov s29, s17 +.endm + +.macro KERNEL2x4_SUB + ld2 {v8.4s, v9.4s}, [pB] + add pB, pB, #32 + ld2 {v0.2s, v1.2s}, [pA] + add pA, pA, #16 + + OP_rr v16.2s, v0.2s, v8.4s[0] + OP_ii v16.2s, v1.2s, v9.4s[0] + OP_ri v17.2s, v0.2s, v9.4s[0] + OP_ir v17.2s, v1.2s, v8.4s[0] + + OP_rr v20.2s, v0.2s, v8.4s[1] + OP_ii v20.2s, v1.2s, v9.4s[1] + OP_ri v21.2s, v0.2s, v9.4s[1] + OP_ir v21.2s, v1.2s, v8.4s[1] + + OP_rr v24.2s, v0.2s, v8.4s[2] + OP_ii v24.2s, v1.2s, v9.4s[2] + OP_ri v25.2s, v0.2s, v9.4s[2] + OP_ir v25.2s, v1.2s, v8.4s[2] + + OP_rr v28.2s, v0.2s, v8.4s[3] + OP_ii v28.2s, v1.2s, v9.4s[3] + OP_ri v29.2s, v0.2s, v9.4s[3] + OP_ir v29.2s, v1.2s, v8.4s[3] +.endm + +.macro SAVE2x4 + mov pCRow1, pCRow0 + + ld2 {v0.2s, v1.2s}, [pCRow1] + fmla v0.2s, v16.2s, alphaV0_R + fmls v0.2s, v17.2s, alphaV0_I + fmla v1.2s, v16.2s, alphaV1_I + fmla v1.2s, v17.2s, alphaV1_R + st2 {v0.2s, v1.2s}, [pCRow1] + + add pCRow1, pCRow1, LDC + + ld2 {v4.2s, v5.2s}, [pCRow1] + fmla v4.2s, v20.2s, alphaV0_R + fmls v4.2s, v21.2s, alphaV0_I + fmla v5.2s, v20.2s, alphaV1_I + fmla v5.2s, v21.2s, alphaV1_R + st2 {v4.2s, v5.2s}, [pCRow1] + + add pCRow1, pCRow1, LDC + + ld2 {v0.2s, v1.2s}, [pCRow1] + fmla v0.2s, v24.2s, alphaV0_R + fmls v0.2s, v25.2s, alphaV0_I + fmla v1.2s, v24.2s, alphaV1_I + fmla v1.2s, v25.2s, alphaV1_R + st2 {v0.2s, v1.2s}, [pCRow1] + + add pCRow1, pCRow1, LDC + + ld2 {v4.2s, v5.2s}, [pCRow1] + fmla v4.2s, v28.2s, alphaV0_R + fmls v4.2s, v29.2s, alphaV0_I + fmla v5.2s, v28.2s, alphaV1_I + fmla v5.2s, v29.2s, alphaV1_R + st2 {v4.2s, v5.2s}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT1x4 + fmov s16, wzr + fmov s17, wzr + fmov s20, s16 + fmov s21, s17 + fmov s24, s16 + fmov s25, s17 + fmov s28, s16 + fmov s29, s17 +.endm + +.macro KERNEL1x4_SUB + ld2 {v8.4s, v9.4s}, [pB] + add pB, pB, #32 + ld2 {v0.s, v1.s}[0], [pA] + add pA, pA, #8 + + OP_rr s16, s0, v8.4s[0] + OP_ii s16, s1, v9.4s[0] + OP_ri s17, s0, v9.4s[0] + OP_ir s17, s1, v8.4s[0] + + OP_rr s20, s0, v8.4s[1] + OP_ii s20, s1, v9.4s[1] + OP_ri s21, s0, v9.4s[1] + OP_ir s21, s1, v8.4s[1] + + OP_rr s24, s0, v8.4s[2] + OP_ii s24, s1, v9.4s[2] + OP_ri s25, s0, v9.4s[2] + OP_ir s25, s1, v8.4s[2] + + OP_rr s28, s0, v8.4s[3] + OP_ii s28, s1, v9.4s[3] + OP_ri s29, s0, v9.4s[3] + OP_ir s29, s1, v8.4s[3] +.endm + +.macro SAVE1x4 + mov pCRow1, pCRow0 + + ld2 {v0.s, v1.s}[0], [pCRow1] + fmla s0, s16, alphaV0_R + fmls s0, s17, alphaV0_I + fmla s1, s16, alphaV1_I + fmla s1, s17, alphaV1_R + st2 {v0.s, v1.s}[0], [pCRow1] + + add pCRow1, pCRow1, LDC + + ld2 {v4.s, v5.s}[0], [pCRow1] + fmla s4, s20, alphaV0_R + fmls s4, s21, alphaV0_I + fmla s5, s20, alphaV1_I + fmla s5, s21, alphaV1_R + st2 {v4.s, v5.s}[0], [pCRow1] + + add pCRow1, pCRow1, LDC + + ld2 {v0.s, v1.s}[0], [pCRow1] + fmla s0, s24, alphaV0_R + fmls s0, s25, alphaV0_I + fmla s1, s24, alphaV1_I + fmla s1, s25, alphaV1_R + st2 {v0.s, v1.s}[0], [pCRow1] + + add pCRow1, pCRow1, LDC + + ld2 {v4.s, v5.s}[0], [pCRow1] + fmla s4, s28, alphaV0_R + fmls s4, s29, alphaV0_I + fmla s5, s28, alphaV1_I + fmla s5, s29, alphaV1_R + st2 {v4.s, v5.s}[0], [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT8x2 + fmov s16, wzr + fmov s17, wzr + fmov s18, wzr + fmov s19, s16 + fmov s20, wzr + fmov s21, s16 + fmov s22, s17 + fmov s23, s18 +.endm + +.macro KERNEL8x2_SUB + ld2 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + ld2 {v2.4s, v3.4s}, [pA] + add pA, pA, #32 + + OP_rr v16.4s, v0.4s, v8.2s[0] + OP_ii v16.4s, v1.4s, v9.2s[0] + OP_ri v17.4s, v0.4s, v9.2s[0] + OP_ir v17.4s, v1.4s, v8.2s[0] + + OP_rr v18.4s, v2.4s, v8.2s[0] + OP_ii v18.4s, v3.4s, v9.2s[0] + OP_ri v19.4s, v2.4s, v9.2s[0] + OP_ir v19.4s, v3.4s, v8.2s[0] + + OP_rr v20.4s, v0.4s, v8.2s[1] + OP_ii v20.4s, v1.4s, v9.2s[1] + OP_ri v21.4s, v0.4s, v9.2s[1] + OP_ir v21.4s, v1.4s, v8.2s[1] + + OP_rr v22.4s, v2.4s, v8.2s[1] + OP_ii v22.4s, v3.4s, v9.2s[1] + OP_ri v23.4s, v2.4s, v9.2s[1] + OP_ir v23.4s, v3.4s, v8.2s[1] +.endm + +.macro SAVE8x2 + mov pCRow1, pCRow0 + + ld2 {v0.4s, v1.4s}, [pCRow1] + fmla v0.4s, v16.4s, alphaV0_R + fmls v0.4s, v17.4s, alphaV0_I + fmla v1.4s, v16.4s, alphaV1_I + fmla v1.4s, v17.4s, alphaV1_R + st2 {v0.4s, v1.4s}, [pCRow1] + + add pCRow2, pCRow1, #32 + + ld2 {v2.4s, v3.4s}, [pCRow2] + fmla v2.4s, v18.4s, alphaV0_R + fmls v2.4s, v19.4s, alphaV0_I + fmla v3.4s, v18.4s, alphaV1_I + fmla v3.4s, v19.4s, alphaV1_R + st2 {v2.4s, v3.4s}, [pCRow2] + + add pCRow1, pCRow1, LDC + + ld2 {v4.4s, v5.4s}, [pCRow1] + fmla v4.4s, v20.4s, alphaV0_R + fmls v4.4s, v21.4s, alphaV0_I + fmla v5.4s, v20.4s, alphaV1_I + fmla v5.4s, v21.4s, alphaV1_R + st2 {v4.4s, v5.4s}, [pCRow1] + + add pCRow2, pCRow1, #32 + + ld2 {v6.4s, v7.4s}, [pCRow2] + fmla v6.4s, v22.4s, alphaV0_R + fmls v6.4s, v23.4s, alphaV0_I + fmla v7.4s, v22.4s, alphaV1_I + fmla v7.4s, v23.4s, alphaV1_R + st2 {v6.4s, v7.4s}, [pCRow2] + + add pCRow0, pCRow0, #64 +.endm + +/******************************************************************************/ + +.macro INIT4x2 + fmov s16, wzr + fmov s17, wzr + fmov s20, s16 + fmov s21, s17 +.endm + +.macro KERNEL4x2_SUB + ld2 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + + OP_rr v16.4s, v0.4s, v8.2s[0] + OP_ii v16.4s, v1.4s, v9.2s[0] + OP_ri v17.4s, v0.4s, v9.2s[0] + OP_ir v17.4s, v1.4s, v8.2s[0] + + OP_rr v20.4s, v0.4s, v8.2s[1] + OP_ii v20.4s, v1.4s, v9.2s[1] + OP_ri v21.4s, v0.4s, v9.2s[1] + OP_ir v21.4s, v1.4s, v8.2s[1] +.endm + +.macro SAVE4x2 + mov pCRow1, pCRow0 + + ld2 {v0.4s, v1.4s}, [pCRow1] + fmla v0.4s, v16.4s, alphaV0_R + fmls v0.4s, v17.4s, alphaV0_I + fmla v1.4s, v16.4s, alphaV1_I + fmla v1.4s, v17.4s, alphaV1_R + st2 {v0.4s, v1.4s}, [pCRow1] + + add pCRow1, pCRow1, LDC + + ld2 {v4.4s, v5.4s}, [pCRow1] + fmla v4.4s, v20.4s, alphaV0_R + fmls v4.4s, v21.4s, alphaV0_I + fmla v5.4s, v20.4s, alphaV1_I + fmla v5.4s, v21.4s, alphaV1_R + st2 {v4.4s, v5.4s}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT2x2 + fmov s16, wzr + fmov s17, wzr + fmov s20, s16 + fmov s21, s17 +.endm + +.macro KERNEL2x2_SUB + ld2 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld2 {v0.2s, v1.2s}, [pA] + add pA, pA, #16 + + OP_rr v16.2s, v0.2s, v8.2s[0] + OP_ii v16.2s, v1.2s, v9.2s[0] + OP_ri v17.2s, v0.2s, v9.2s[0] + OP_ir v17.2s, v1.2s, v8.2s[0] + + OP_rr v20.2s, v0.2s, v8.2s[1] + OP_ii v20.2s, v1.2s, v9.2s[1] + OP_ri v21.2s, v0.2s, v9.2s[1] + OP_ir v21.2s, v1.2s, v8.2s[1] +.endm + +.macro SAVE2x2 + mov pCRow1, pCRow0 + + ld2 {v0.2s, v1.2s}, [pCRow1] + fmla v0.2s, v16.2s, alphaV0_R + fmls v0.2s, v17.2s, alphaV0_I + fmla v1.2s, v16.2s, alphaV1_I + fmla v1.2s, v17.2s, alphaV1_R + st2 {v0.2s, v1.2s}, [pCRow1] + + add pCRow1, pCRow1, LDC + + ld2 {v4.2s, v5.2s}, [pCRow1] + fmla v4.2s, v20.2s, alphaV0_R + fmls v4.2s, v21.2s, alphaV0_I + fmla v5.2s, v20.2s, alphaV1_I + fmla v5.2s, v21.2s, alphaV1_R + st2 {v4.2s, v5.2s}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT1x2 + fmov s16, wzr + fmov s17, wzr + fmov s20, wzr + fmov s21, wzr +.endm + +.macro KERNEL1x2_SUB + ld2 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld2 {v0.s, v1.s}[0], [pA] + add pA, pA, #8 + + OP_rr s16, s0, v8.2s[0] + OP_ii s16, s1, v9.2s[0] + OP_ri s17, s0, v9.2s[0] + OP_ir s17, s1, v8.2s[0] + + OP_rr s20, s0, v8.2s[1] + OP_ii s20, s1, v9.2s[1] + OP_ri s21, s0, v9.2s[1] + OP_ir s21, s1, v8.2s[1] +.endm + +.macro SAVE1x2 + mov pCRow1, pCRow0 + + ld2 {v0.s, v1.s}[0], [pCRow1] + fmla s0, s16, alphaV0_R + fmls s0, s17, alphaV0_I + fmla s1, s16, alphaV1_I + fmla s1, s17, alphaV1_R + st2 {v0.s, v1.s}[0], [pCRow1] + + add pCRow1, pCRow1, LDC + + ld2 {v4.s, v5.s}[0], [pCRow1] + fmla s4, s20, alphaV0_R + fmls s4, s21, alphaV0_I + fmla s5, s20, alphaV1_I + fmla s5, s21, alphaV1_R + st2 {v4.s, v5.s}[0], [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT8x1 + fmov s16, wzr + fmov s17, wzr + fmov s18, wzr + fmov s19, s16 +.endm + +.macro KERNEL8x1_SUB + ld1 {v8.2s}, [pB] + add pB, pB, #8 + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + ld2 {v2.4s, v3.4s}, [pA] + add pA, pA, #32 + + OP_rr v16.4s, v0.4s, v8.4s[0] + OP_ii v16.4s, v1.4s, v8.4s[1] + OP_ri v17.4s, v0.4s, v8.4s[1] + OP_ir v17.4s, v1.4s, v8.4s[0] + + OP_rr v18.4s, v2.4s, v8.4s[0] + OP_ii v18.4s, v3.4s, v8.4s[1] + OP_ri v19.4s, v2.4s, v8.4s[1] + OP_ir v19.4s, v3.4s, v8.4s[0] +.endm + +.macro SAVE8x1 + mov pCRow1, pCRow0 + + ld2 {v0.4s, v1.4s}, [pCRow1] + fmla v0.4s, v16.4s, alphaV0_R + fmls v0.4s, v17.4s, alphaV0_I + fmla v1.4s, v16.4s, alphaV1_I + fmla v1.4s, v17.4s, alphaV1_R + st2 {v0.4s, v1.4s}, [pCRow1] + + add pCRow1, pCRow1, #32 + + ld2 {v2.4s, v3.4s}, [pCRow1] + fmla v2.4s, v18.4s, alphaV0_R + fmls v2.4s, v19.4s, alphaV0_I + fmla v3.4s, v18.4s, alphaV1_I + fmla v3.4s, v19.4s, alphaV1_R + st2 {v2.4s, v3.4s}, [pCRow1] + + add pCRow0, pCRow0, #64 +.endm + + +/******************************************************************************/ + +.macro INIT4x1 + fmov s16, wzr + fmov s17, s16 +.endm + +.macro KERNEL4x1_SUB + ld2 {v8.s, v9.s}[0], [pB] + add pB, pB, #8 + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + + OP_rr v16.4s, v0.4s, v8.s[0] + OP_ii v16.4s, v1.4s, v9.s[0] + OP_ri v17.4s, v0.4s, v9.s[0] + OP_ir v17.4s, v1.4s, v8.s[0] +.endm + +.macro SAVE4x1 + mov pCRow1, pCRow0 + + ld2 {v0.4s, v1.4s}, [pCRow1] + fmla v0.4s, v16.4s, alphaV0_R + fmls v0.4s, v17.4s, alphaV0_I + fmla v1.4s, v16.4s, alphaV1_I + fmla v1.4s, v17.4s, alphaV1_R + st2 {v0.4s, v1.4s}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT2x1 + fmov s16, wzr + fmov s17, wzr +.endm + +.macro KERNEL2x1_SUB + ld2 {v8.s, v9.s}[0], [pB] + add pB, pB, #8 + ld2 {v0.2s, v1.2s}, [pA] + add pA, pA, #16 + + OP_rr v16.2s, v0.2s, v8.s[0] + OP_ii v16.2s, v1.2s, v9.s[0] + OP_ri v17.2s, v0.2s, v9.s[0] + OP_ir v17.2s, v1.2s, v8.s[0] +.endm + +.macro SAVE2x1 + mov pCRow1, pCRow0 + + ld2 {v0.2s, v1.2s}, [pCRow1] + fmla v0.2s, v16.2s, alphaV0_R + fmls v0.2s, v17.2s, alphaV0_I + fmla v1.2s, v16.2s, alphaV1_I + fmla v1.2s, v17.2s, alphaV1_R + st2 {v0.2s, v1.2s}, [pCRow1] + + add pCRow0, pCRow0, #16 + +.endm + +/******************************************************************************/ + +.macro INIT1x1 + fmov s16, wzr + fmov s17, wzr +.endm + +.macro KERNEL1x1_SUB + ld2 {v8.s, v9.s}[0], [pB] + add pB, pB, #8 + ld2 {v0.s, v1.s}[0], [pA] + add pA, pA, #8 + + OP_rr s16, s0, v8.s[0] + OP_ii s16, s1, v9.s[0] + OP_ri s17, s0, v9.s[0] + OP_ir s17, s1, v8.s[0] +.endm + +.macro SAVE1x1 + mov pCRow1, pCRow0 + + ld2 {v0.s, v1.s}[0], [pCRow1] + fmla s0, s16, alphaV0_R + fmls s0, s17, alphaV0_I + fmla s1, s16, alphaV1_I + fmla s1, s17, alphaV1_R + st2 {v0.s, v1.s}[0], [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************* +* End of macro definitions +*******************************************************************************/ + + PROLOGUE + + .align 5 + add sp, sp, #-(11 * 16) + stp d8, d9, [sp, #(0 * 16)] + stp d10, d11, [sp, #(1 * 16)] + stp d12, d13, [sp, #(2 * 16)] + stp d14, d15, [sp, #(3 * 16)] + stp d16, d17, [sp, #(4 * 16)] + stp x18, x19, [sp, #(5 * 16)] + stp x20, x21, [sp, #(6 * 16)] + stp x22, x23, [sp, #(7 * 16)] + stp x24, x25, [sp, #(8 * 16)] + stp x26, x27, [sp, #(9 * 16)] + str x28, [sp, #(10 * 16)] + + fmov alpha0_R, s0 + fmov alpha0_I, s1 + fmov alpha1_R, s0 + fmov alpha1_I, s1 + + lsl LDC, LDC, #3 // ldc = ldc * 8 + + mov pB, origPB + + mov counterJ, origN + asr counterJ, counterJ, #2 // J = J / 4 + cmp counterJ, #0 + ble cgemm_kernel_L2_BEGIN + +/******************************************************************************/ + +cgemm_kernel_L4_BEGIN: + mov pCRow0, pC // pCRow0 = C + add pC, pC, LDC, lsl #2 + + mov pA, origPA // pA = start of A array + +cgemm_kernel_L4_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 // counterI = counterI / 8 + cmp counterI, #0 + ble cgemm_kernel_L4_M4_BEGIN + +cgemm_kernel_L4_M8_20: + + mov pB, origPB + + asr counterL , origK, #1 // L = K / 2 + cmp counterL , #2 // is there at least 4 to do? + blt cgemm_kernel_L4_M8_32 + + KERNEL8x4_I // do one in the K + KERNEL8x4_M2 // do another in the K + + subs counterL, counterL, #2 // subtract 2 + ble cgemm_kernel_L4_M8_22a + .align 5 + +cgemm_kernel_L4_M8_22: + + KERNEL8x4_M1 + KERNEL8x4_M2 + + subs counterL, counterL, #1 + bgt cgemm_kernel_L4_M8_22 + + +cgemm_kernel_L4_M8_22a: + + KERNEL8x4_M1 + KERNEL8x4_E + + b cgemm_kernel_L4_M8_44 + +cgemm_kernel_L4_M8_32: + + tst counterL, #1 + ble cgemm_kernel_L4_M8_40 + + KERNEL8x4_I + + KERNEL8x4_E + + b cgemm_kernel_L4_M8_44 + +cgemm_kernel_L4_M8_40: + + INIT8x4 + +cgemm_kernel_L4_M8_44: + + ands counterL , origK, #1 + ble cgemm_kernel_L4_M8_100 + +cgemm_kernel_L4_M8_46: + + KERNEL8x4_SUB + +cgemm_kernel_L4_M8_100: + + SAVE8x4 + +cgemm_kernel_L4_M8_END: + subs counterI, counterI, #1 + bne cgemm_kernel_L4_M8_20 + +cgemm_kernel_L4_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble cgemm_kernel_L4_END + + tst counterI, #4 + ble cgemm_kernel_L4_M2_BEGIN + + +cgemm_kernel_L4_M4_20: + + mov pB, origPB + + asr counterL , origK, #1 // L = K / 2 + cmp counterL , #2 // is there at least 4 to do? + blt cgemm_kernel_L4_M4_32 + + KERNEL4x4_I // do one in the K + KERNEL4x4_M2 // do another in the K + + subs counterL, counterL, #2 + ble cgemm_kernel_L4_M4_22a + .align 5 + + +cgemm_kernel_L4_M4_22: + + KERNEL4x4_M1 + KERNEL4x4_M2 + + subs counterL, counterL, #1 + bgt cgemm_kernel_L4_M4_22 + +cgemm_kernel_L4_M4_22a: + KERNEL4x4_M1 + KERNEL4x4_E + b cgemm_kernel_L4_M4_44 +cgemm_kernel_L4_M4_32: + tst counterL, #1 + ble cgemm_kernel_L4_M4_40 + KERNEL4x4_I + KERNEL4x4_E + b cgemm_kernel_L4_M4_44 +cgemm_kernel_L4_M4_40: + + INIT4x4 + +cgemm_kernel_L4_M4_44: + ands counterL , origK, #1 + ble cgemm_kernel_L4_M4_100 + +cgemm_kernel_L4_M4_46: + KERNEL4x4_SUB + +cgemm_kernel_L4_M4_100: + + SAVE4x4 + +cgemm_kernel_L4_M4_END: + +cgemm_kernel_L4_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble cgemm_kernel_L4_END + + tst counterI, #2 // counterI = counterI / 2 + ble cgemm_kernel_L4_M1_BEGIN + +cgemm_kernel_L4_M2_20: + + INIT2x4 + + mov pB, origPB + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble cgemm_kernel_L4_M2_40 + +cgemm_kernel_L4_M2_22: + + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L4_M2_22 + + +cgemm_kernel_L4_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble cgemm_kernel_L4_M2_100 + +cgemm_kernel_L4_M2_42: + + KERNEL2x4_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L4_M2_42 + +cgemm_kernel_L4_M2_100: + + SAVE2x4 + +cgemm_kernel_L4_M2_END: + + +cgemm_kernel_L4_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble cgemm_kernel_L4_END + +cgemm_kernel_L4_M1_20: + + INIT1x4 + + mov pB, origPB + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble cgemm_kernel_L4_M1_40 + +cgemm_kernel_L4_M1_22: + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L4_M1_22 + + +cgemm_kernel_L4_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble cgemm_kernel_L4_M1_100 + +cgemm_kernel_L4_M1_42: + + KERNEL1x4_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L4_M1_42 + +cgemm_kernel_L4_M1_100: + + SAVE1x4 + + +cgemm_kernel_L4_END: + + lsl temp, origK, #5 + add origPB, origPB, temp // B = B + K * 4 * 8 + + subs counterJ, counterJ , #1 // j-- + bgt cgemm_kernel_L4_BEGIN + + +/******************************************************************************/ + +cgemm_kernel_L2_BEGIN: // less than 2 left in N direction + + mov counterJ , origN + tst counterJ , #3 + ble cgemm_kernel_L999 // error, N was less than 4? + + tst counterJ , #2 + ble cgemm_kernel_L1_BEGIN + + mov pCRow0, pC // pCRow0 = pC + + add pC,pC,LDC, lsl #1 + + mov pA, origPA // pA = A + + +cgemm_kernel_L2_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 // counterI = counterI / 8 + cmp counterI, #0 + ble cgemm_kernel_L2_M4_BEGIN + +cgemm_kernel_L2_M8_20: + + INIT8x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble cgemm_kernel_L2_M8_40 + .align 5 + +cgemm_kernel_L2_M8_22: + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L2_M8_22 + + +cgemm_kernel_L2_M8_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble cgemm_kernel_L2_M8_100 + +cgemm_kernel_L2_M8_42: + + KERNEL8x2_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L2_M8_42 + +cgemm_kernel_L2_M8_100: + + SAVE8x2 + +cgemm_kernel_L2_M8_END: + + subs counterI, counterI, #1 + bgt cgemm_kernel_L2_M8_20 + +cgemm_kernel_L2_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble cgemm_kernel_L2_END + + tst counterI, #4 // counterI = counterI / 2 + ble cgemm_kernel_L2_M2_BEGIN + +cgemm_kernel_L2_M4_20: + + INIT4x2 + + mov pB, origPB + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble cgemm_kernel_L2_M4_40 + .align 5 + +cgemm_kernel_L2_M4_22: + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L2_M4_22 + + +cgemm_kernel_L2_M4_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble cgemm_kernel_L2_M4_100 + +cgemm_kernel_L2_M4_42: + + KERNEL4x2_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L2_M4_42 + +cgemm_kernel_L2_M4_100: + + SAVE4x2 + +cgemm_kernel_L2_M4_END: + +cgemm_kernel_L2_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble cgemm_kernel_L2_END + + tst counterI, #2 // counterI = counterI / 2 + ble cgemm_kernel_L2_M1_BEGIN + +cgemm_kernel_L2_M2_20: + + INIT2x2 + + mov pB, origPB + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble cgemm_kernel_L2_M2_40 + +cgemm_kernel_L2_M2_22: + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L2_M2_22 + + +cgemm_kernel_L2_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble cgemm_kernel_L2_M2_100 + +cgemm_kernel_L2_M2_42: + + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L2_M2_42 + +cgemm_kernel_L2_M2_100: + + SAVE2x2 + +cgemm_kernel_L2_M2_END: + + +cgemm_kernel_L2_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble cgemm_kernel_L2_END + +cgemm_kernel_L2_M1_20: + + INIT1x2 + + mov pB, origPB + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL, #0 + ble cgemm_kernel_L2_M1_40 + +cgemm_kernel_L2_M1_22: + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L2_M1_22 + + +cgemm_kernel_L2_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble cgemm_kernel_L2_M1_100 + +cgemm_kernel_L2_M1_42: + + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L2_M1_42 + +cgemm_kernel_L2_M1_100: + + SAVE1x2 + + +cgemm_kernel_L2_END: + add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 + +/******************************************************************************/ + +cgemm_kernel_L1_BEGIN: + + mov counterJ , origN + tst counterJ , #1 + ble cgemm_kernel_L999 // done + + + mov pCRow0, pC // pCRow0 = C + add pC , pC , LDC // Update pC to point to next + + mov pA, origPA // pA = A + + +cgemm_kernel_L1_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 // counterI = counterI / 8 + cmp counterI, #0 + ble cgemm_kernel_L1_M4_BEGIN + +cgemm_kernel_L1_M8_20: + + INIT8x1 + + mov pB, origPB + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble cgemm_kernel_L1_M8_40 + .align 5 + +cgemm_kernel_L1_M8_22: + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L1_M8_22 + + +cgemm_kernel_L1_M8_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble cgemm_kernel_L1_M8_100 + +cgemm_kernel_L1_M8_42: + + KERNEL8x1_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L1_M8_42 + +cgemm_kernel_L1_M8_100: + + SAVE8x1 + +cgemm_kernel_L1_M8_END: + + subs counterI, counterI, #1 + bgt cgemm_kernel_L1_M8_20 + +cgemm_kernel_L1_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble cgemm_kernel_L1_END + + tst counterI, #4 // counterI = counterI / 2 + ble cgemm_kernel_L1_M2_BEGIN + + +cgemm_kernel_L1_M4_20: + + INIT4x1 + + mov pB, origPB + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble cgemm_kernel_L1_M4_40 + .align 5 + +cgemm_kernel_L1_M4_22: + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L1_M4_22 + + +cgemm_kernel_L1_M4_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble cgemm_kernel_L1_M4_100 + +cgemm_kernel_L1_M4_42: + + KERNEL4x1_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L1_M4_42 + +cgemm_kernel_L1_M4_100: + + SAVE4x1 + +cgemm_kernel_L1_M4_END: + + +cgemm_kernel_L1_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble cgemm_kernel_L1_END + + tst counterI, #2 // counterI = counterI / 2 + ble cgemm_kernel_L1_M1_BEGIN + +cgemm_kernel_L1_M2_20: + + INIT2x1 + + mov pB, origPB + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble cgemm_kernel_L1_M2_40 + +cgemm_kernel_L1_M2_22: + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L1_M2_22 + + +cgemm_kernel_L1_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble cgemm_kernel_L1_M2_100 + +cgemm_kernel_L1_M2_42: + + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L1_M2_42 + +cgemm_kernel_L1_M2_100: + + SAVE2x1 + +cgemm_kernel_L1_M2_END: + + +cgemm_kernel_L1_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble cgemm_kernel_L1_END + +cgemm_kernel_L1_M1_20: + + INIT1x1 + + mov pB, origPB + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble cgemm_kernel_L1_M1_40 + +cgemm_kernel_L1_M1_22: + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L1_M1_22 + + +cgemm_kernel_L1_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble cgemm_kernel_L1_M1_100 + +cgemm_kernel_L1_M1_42: + + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L1_M1_42 + +cgemm_kernel_L1_M1_100: + + SAVE1x1 + + +cgemm_kernel_L1_END: + + +cgemm_kernel_L999: + mov x0, #0 // set return value + ldp d8, d9, [sp, #(0 * 16)] + ldp d10, d11, [sp, #(1 * 16)] + ldp d12, d13, [sp, #(2 * 16)] + ldp d14, d15, [sp, #(3 * 16)] + ldp d16, d17, [sp, #(4 * 16)] + ldp x18, x19, [sp, #(5 * 16)] + ldp x20, x21, [sp, #(6 * 16)] + ldp x22, x23, [sp, #(7 * 16)] + ldp x24, x25, [sp, #(8 * 16)] + ldp x26, x27, [sp, #(9 * 16)] + ldr x28, [sp, #(10 * 16)] + add sp, sp, #(11*16) + ret + + EPILOGUE + diff --git a/kernel/arm64/ctrmm_kernel_8x4.S b/kernel/arm64/ctrmm_kernel_8x4.S new file mode 100755 index 000000000..3131541d4 --- /dev/null +++ b/kernel/arm64/ctrmm_kernel_8x4.S @@ -0,0 +1,2425 @@ +/******************************************************************************* +Copyright (c) 2015, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#define ASSEMBLER +#include "common.h" + +/* X0 X1 X2 s0 s1 X3 x4 x5 x6 x7*/ +/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha0, FLOAT alpha1,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc, BLASLONG offset */ + +#define origM x0 +#define origN x1 +#define origK x2 +#define origPA x3 +#define origPB x4 +#define pC x5 +#define LDC x6 +#define offset x7 +#define counterL x8 +#define counterI x9 +#define counterJ x10 +#define pB x11 +#define pCRow0 x12 +#define pCRow1 x13 +#define pCRow2 x14 +#define pA x15 +#define temp x16 +#define tempOffset x17 +#define tempK x18 + +#define alpha0_R s10 +#define alphaV0_R v10.s[0] +#define alpha0_I s11 +#define alphaV0_I v11.s[0] + +#define alpha1_R s14 +#define alphaV1_R v14.s[0] +#define alpha1_I s15 +#define alphaV1_I v15.s[0] + +#if defined(NN) || defined(NT) || defined(TN) || defined(TT) +#define OP_rr fmla +#define OP_ii fmls +#define OP_ri fmla +#define OP_ir fmla +#elif defined(NR) || defined(NC) || defined(TR) || defined(TC) +#define OP_rr fmla +#define OP_ii fmla +#define OP_ri fmls +#define OP_ir fmla +#elif defined(RN) || defined(RT) || defined(CN) || defined(CT) +#define OP_rr fmla +#define OP_ii fmla +#define OP_ri fmla +#define OP_ir fmls +#elif defined(RR) || defined(RC) || defined(CR) || defined(CC) +#define OP_rr fmla +#define OP_ii fmls +#define OP_ri fmls +#define OP_ir fmls +#endif + +// 00 origM +// 01 origN +// 02 origK +// 03 origPA +// 04 origPB +// 05 pC +// 06 origLDC -> LDC +// 07 offset +// 08 counterL +// 09 counterI +// 10 counterJ +// 11 pB +// 12 pCRow0 +// 13 pCRow1 +// 14 pCRow2 +// 15 pA +// 16 temp +// 17 tempOffset +// 18 must save tempK +// 19 must save +// 20 must save +// 21 must save +// 22 must save +// 23 must save +// 24 must save +// 25 must save +// 26 must save +// 27 must save +// 28 must save +// 29 frame +// 30 link +// 31 sp + +//v00 ALPHA_R -> pA0_00_R, pA0_01_R, pA0_02_R, pA0_03_R +//v01 ALPHA_I -> pA0_00_I, pA0_01_I, pA0_02_I, pA0_03_I +//v02 pA0_04_R, pA0_05_R, pA0_06_R, pA0_07_R +//v03 pA0_04_I, pA0_05_I, pA0_06_I, pA0_07_I +//v04 pA1_00_R, pA1_01_R, pA1_02_R, pA1_03_R +//v05 pA1_00_I, pA1_01_I, pA1_02_I, pA1_03_I +//v06 pA1_04_R, pA1_05_R, pA1_06_R, pA1_07_R +//v07 pA1_04_I, pA1_05_I, pA1_06_I, pA1_07_I +//v08 must save pB0_00_R, pB0_01_R, pB0_02_R, pB0_03_R +//v09 must save pB0_00_I, pB0_01_I, pB0_02_I, pB0_03_I +//v10 must save ALPHA0_R +//v11 must save ALPHA0_I +//v12 must save pB1_00_R, pB1_01_R, pB1_02_R, pB1_03_R +//v13 must save pB1_00_I, pB1_01_I, pB1_02_I, pB1_03_I +//v14 must save ALPHA1_R +//v15 must save ALPHA1_I +//v16 must save pC_00_R, pC_01_R, pC_02_R, pC_03_R +//v17 must save pC_00_I, pC_01_I, pC_02_I, pC_03_I +//v18 pC_04_R, pC_05_R, pC_06_R, pC_07_R +//v19 pC_04_I, pC_05_I, pC_06_I, pC_07_I +//v20 pC_08_R, pC_09_R, pC_10_R, pC_11_R +//v21 pC_08_I, pC_09_I, pC_10_I, pC_11_I +//v22 pC_12_R, pC_13_R, pC_14_R, pC_15_R +//v23 pC_12_I, pC_13_I, pC_14_I, pC_15_I +//v24 pC_16_R, pC_17_R, pC_18_R, pC_19_R +//v25 pC_16_I, pC_17_I, pC_18_I, pC_19_I +//v26 pC_20_R, pC_21_R, pC_22_R, pC_23_R +//v27 pC_20_I, pC_21_I, pC_22_I, pC_23_I +//v28 pC_24_R, pC_25_R, pC_26_R, pC_27_R +//v29 pC_24_I, pC_25_I, pC_26_I, pC_27_I +//v30 pC_28_R, pC_29_R, pC_30_R, pC_31_R +//v31 pC_28_I, pC_29_I, pC_30_I, pC_31_I + +/******************************************************************************* +* Macro definitions +*******************************************************************************/ + +.macro INIT8x4 + fmov s16, wzr + fmov s17, wzr + fmov s18, wzr + fmov s19, s16 + fmov s20, wzr + fmov s21, s16 + fmov s22, s17 + fmov s23, s18 + fmov s24, wzr + fmov s25, s16 + fmov s26, s17 + fmov s27, s18 + fmov s28, wzr + fmov s29, s16 + fmov s30, s17 + fmov s31, s18 +.endm + +.macro KERNEL8x4_I + ld2 {v8.4s, v9.4s}, [pB] + add pB, pB, #32 + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + ld2 {v2.4s, v3.4s}, [pA] + add pA, pA, #32 + + fmul v16.4s, v0.4s, v8.4s[0] + OP_ii v16.4s, v1.4s, v9.4s[0] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v17.16b, v17.16b, v17.16b + fmls v17.4s, v0.4s, v9.4s[0] +#else + fmul v17.4s, v0.4s, v9.4s[0] +#endif + OP_ir v17.4s, v1.4s, v8.4s[0] + + fmul v18.4s, v2.4s, v8.4s[0] + OP_ii v18.4s, v3.4s, v9.4s[0] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v19.16b, v19.16b, v19.16b + fmls v19.4s, v2.4s, v9.4s[0] +#else + fmul v19.4s, v2.4s, v9.4s[0] +#endif + OP_ir v19.4s, v3.4s, v8.4s[0] + + fmul v20.4s, v0.4s, v8.4s[1] + OP_ii v20.4s, v1.4s, v9.4s[1] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v21.16b, v21.16b, v21.16b + fmls v21.4s, v0.4s, v9.4s[1] +#else + fmul v21.4s, v0.4s, v9.4s[1] +#endif + OP_ir v21.4s, v1.4s, v8.4s[1] + + fmul v22.4s, v2.4s, v8.4s[1] + OP_ii v22.4s, v3.4s, v9.4s[1] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v23.16b, v23.16b, v23.16b + fmls v23.4s, v2.4s, v9.4s[1] +#else + fmul v23.4s, v2.4s, v9.4s[1] +#endif + OP_ir v23.4s, v3.4s, v8.4s[1] + + fmul v24.4s, v0.4s, v8.4s[2] + OP_ii v24.4s, v1.4s, v9.4s[2] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v25.16b, v25.16b, v25.16b + fmls v25.4s, v0.4s, v9.4s[2] +#else + fmul v25.4s, v0.4s, v9.4s[2] +#endif + OP_ir v25.4s, v1.4s, v8.4s[2] + + fmul v26.4s, v2.4s, v8.4s[2] + OP_ii v26.4s, v3.4s, v9.4s[2] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v27.16b, v27.16b, v27.16b + fmls v27.4s, v2.4s, v9.4s[2] +#else + fmul v27.4s, v2.4s, v9.4s[2] +#endif + OP_ir v27.4s, v3.4s, v8.4s[2] + + fmul v28.4s, v0.4s, v8.4s[3] + OP_ii v28.4s, v1.4s, v9.4s[3] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v29.16b, v29.16b, v29.16b + fmls v29.4s, v0.4s, v9.4s[3] +#else + fmul v29.4s, v0.4s, v9.4s[3] +#endif + OP_ir v29.4s, v1.4s, v8.4s[3] + + fmul v30.4s, v2.4s, v8.4s[3] + OP_ii v30.4s, v3.4s, v9.4s[3] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v31.16b, v31.16b, v31.16b + fmls v31.4s, v2.4s, v9.4s[3] +#else + fmul v31.4s, v2.4s, v9.4s[3] +#endif + OP_ir v31.4s, v3.4s, v8.4s[3] + + ld2 {v12.4s, v13.4s}, [pB] + add pB, pB, #32 + ld2 {v4.4s, v5.4s}, [pA] + add pA, pA, #32 + ld2 {v6.4s, v7.4s}, [pA] + add pA, pA, #32 +.endm + +.macro KERNEL8x4_M1 + OP_rr v16.4s, v0.4s, v8.4s[0] + OP_ii v16.4s, v1.4s, v9.4s[0] + OP_ri v17.4s, v0.4s, v9.4s[0] + OP_ir v17.4s, v1.4s, v8.4s[0] + + OP_rr v18.4s, v2.4s, v8.4s[0] + OP_ii v18.4s, v3.4s, v9.4s[0] + OP_ri v19.4s, v2.4s, v9.4s[0] + OP_ir v19.4s, v3.4s, v8.4s[0] + + OP_rr v20.4s, v0.4s, v8.4s[1] + OP_ii v20.4s, v1.4s, v9.4s[1] + OP_ri v21.4s, v0.4s, v9.4s[1] + OP_ir v21.4s, v1.4s, v8.4s[1] + + OP_rr v22.4s, v2.4s, v8.4s[1] + OP_ii v22.4s, v3.4s, v9.4s[1] + OP_ri v23.4s, v2.4s, v9.4s[1] + OP_ir v23.4s, v3.4s, v8.4s[1] + + OP_rr v24.4s, v0.4s, v8.4s[2] + OP_ii v24.4s, v1.4s, v9.4s[2] + OP_ri v25.4s, v0.4s, v9.4s[2] + OP_ir v25.4s, v1.4s, v8.4s[2] + + OP_rr v26.4s, v2.4s, v8.4s[2] + OP_ii v26.4s, v3.4s, v9.4s[2] + OP_ri v27.4s, v2.4s, v9.4s[2] + OP_ir v27.4s, v3.4s, v8.4s[2] + + OP_rr v28.4s, v0.4s, v8.4s[3] + OP_ii v28.4s, v1.4s, v9.4s[3] + OP_ri v29.4s, v0.4s, v9.4s[3] + OP_ir v29.4s, v1.4s, v8.4s[3] + + OP_rr v30.4s, v2.4s, v8.4s[3] + OP_ii v30.4s, v3.4s, v9.4s[3] + OP_ri v31.4s, v2.4s, v9.4s[3] + OP_ir v31.4s, v3.4s, v8.4s[3] + + ld2 {v12.4s, v13.4s}, [pB] // For next round + add pB, pB, #32 + ld2 {v4.4s, v5.4s}, [pA] // For next round + add pA, pA, #32 + ld2 {v6.4s, v7.4s}, [pA] + add pA, pA, #32 +.endm + +.macro KERNEL8x4_M2 + OP_rr v16.4s, v4.4s, v12.4s[0] + OP_ii v16.4s, v5.4s, v13.4s[0] + OP_ri v17.4s, v4.4s, v13.4s[0] + OP_ir v17.4s, v5.4s, v12.4s[0] + + OP_rr v18.4s, v6.4s, v12.4s[0] + OP_ii v18.4s, v7.4s, v13.4s[0] + OP_ri v19.4s, v6.4s, v13.4s[0] + OP_ir v19.4s, v7.4s, v12.4s[0] + + OP_rr v20.4s, v4.4s, v12.4s[1] + OP_ii v20.4s, v5.4s, v13.4s[1] + OP_ri v21.4s, v4.4s, v13.4s[1] + OP_ir v21.4s, v5.4s, v12.4s[1] + + OP_rr v22.4s, v6.4s, v12.4s[1] + OP_ii v22.4s, v7.4s, v13.4s[1] + OP_ri v23.4s, v6.4s, v13.4s[1] + OP_ir v23.4s, v7.4s, v12.4s[1] + + OP_rr v24.4s, v4.4s, v12.4s[2] + OP_ii v24.4s, v5.4s, v13.4s[2] + OP_ri v25.4s, v4.4s, v13.4s[2] + OP_ir v25.4s, v5.4s, v12.4s[2] + + OP_rr v26.4s, v6.4s, v12.4s[2] + OP_ii v26.4s, v7.4s, v13.4s[2] + OP_ri v27.4s, v6.4s, v13.4s[2] + OP_ir v27.4s, v7.4s, v12.4s[2] + + OP_rr v28.4s, v4.4s, v12.4s[3] + OP_ii v28.4s, v5.4s, v13.4s[3] + OP_ri v29.4s, v4.4s, v13.4s[3] + OP_ir v29.4s, v5.4s, v12.4s[3] + + OP_rr v30.4s, v6.4s, v12.4s[3] + OP_ii v30.4s, v7.4s, v13.4s[3] + OP_ri v31.4s, v6.4s, v13.4s[3] + OP_ir v31.4s, v7.4s, v12.4s[3] + + ld2 {v8.4s, v9.4s}, [pB] + add pB, pB, #32 + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + ld2 {v2.4s, v3.4s}, [pA] + add pA, pA, #32 +.endm + +.macro KERNEL8x4_E + OP_rr v16.4s, v4.4s, v12.4s[0] + OP_ii v16.4s, v5.4s, v13.4s[0] + OP_ri v17.4s, v4.4s, v13.4s[0] + OP_ir v17.4s, v5.4s, v12.4s[0] + + OP_rr v18.4s, v6.4s, v12.4s[0] + OP_ii v18.4s, v7.4s, v13.4s[0] + OP_ri v19.4s, v6.4s, v13.4s[0] + OP_ir v19.4s, v7.4s, v12.4s[0] + + OP_rr v20.4s, v4.4s, v12.4s[1] + OP_ii v20.4s, v5.4s, v13.4s[1] + OP_ri v21.4s, v4.4s, v13.4s[1] + OP_ir v21.4s, v5.4s, v12.4s[1] + + OP_rr v22.4s, v6.4s, v12.4s[1] + OP_ii v22.4s, v7.4s, v13.4s[1] + OP_ri v23.4s, v6.4s, v13.4s[1] + OP_ir v23.4s, v7.4s, v12.4s[1] + + OP_rr v24.4s, v4.4s, v12.4s[2] + OP_ii v24.4s, v5.4s, v13.4s[2] + OP_ri v25.4s, v4.4s, v13.4s[2] + OP_ir v25.4s, v5.4s, v12.4s[2] + + OP_rr v26.4s, v6.4s, v12.4s[2] + OP_ii v26.4s, v7.4s, v13.4s[2] + OP_ri v27.4s, v6.4s, v13.4s[2] + OP_ir v27.4s, v7.4s, v12.4s[2] + + OP_rr v28.4s, v4.4s, v12.4s[3] + OP_ii v28.4s, v5.4s, v13.4s[3] + OP_ri v29.4s, v4.4s, v13.4s[3] + OP_ir v29.4s, v5.4s, v12.4s[3] + + OP_rr v30.4s, v6.4s, v12.4s[3] + OP_ii v30.4s, v7.4s, v13.4s[3] + OP_ri v31.4s, v6.4s, v13.4s[3] + OP_ir v31.4s, v7.4s, v12.4s[3] + +.endm + +.macro KERNEL8x4_SUB + ld2 {v8.4s, v9.4s}, [pB] + add pB, pB, #32 + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + ld2 {v2.4s, v3.4s}, [pA] + add pA, pA, #32 + + OP_rr v16.4s, v0.4s, v8.4s[0] + OP_ii v16.4s, v1.4s, v9.4s[0] + OP_ri v17.4s, v0.4s, v9.4s[0] + OP_ir v17.4s, v1.4s, v8.4s[0] + + OP_rr v18.4s, v2.4s, v8.4s[0] + OP_ii v18.4s, v3.4s, v9.4s[0] + OP_ri v19.4s, v2.4s, v9.4s[0] + OP_ir v19.4s, v3.4s, v8.4s[0] + + OP_rr v20.4s, v0.4s, v8.4s[1] + OP_ii v20.4s, v1.4s, v9.4s[1] + OP_ri v21.4s, v0.4s, v9.4s[1] + OP_ir v21.4s, v1.4s, v8.4s[1] + + OP_rr v22.4s, v2.4s, v8.4s[1] + OP_ii v22.4s, v3.4s, v9.4s[1] + OP_ri v23.4s, v2.4s, v9.4s[1] + OP_ir v23.4s, v3.4s, v8.4s[1] + + OP_rr v24.4s, v0.4s, v8.4s[2] + OP_ii v24.4s, v1.4s, v9.4s[2] + OP_ri v25.4s, v0.4s, v9.4s[2] + OP_ir v25.4s, v1.4s, v8.4s[2] + + OP_rr v26.4s, v2.4s, v8.4s[2] + OP_ii v26.4s, v3.4s, v9.4s[2] + OP_ri v27.4s, v2.4s, v9.4s[2] + OP_ir v27.4s, v3.4s, v8.4s[2] + + OP_rr v28.4s, v0.4s, v8.4s[3] + OP_ii v28.4s, v1.4s, v9.4s[3] + OP_ri v29.4s, v0.4s, v9.4s[3] + OP_ir v29.4s, v1.4s, v8.4s[3] + + OP_rr v30.4s, v2.4s, v8.4s[3] + OP_ii v30.4s, v3.4s, v9.4s[3] + OP_ri v31.4s, v2.4s, v9.4s[3] + OP_ir v31.4s, v3.4s, v8.4s[3] + +.endm + +.macro SAVE8x4 + mov pCRow1, pCRow0 + + + fmul v0.4s, v16.4s, alphaV0_R + fmls v0.4s, v17.4s, alphaV0_I + fmul v1.4s, v16.4s, alphaV1_I + fmla v1.4s, v17.4s, alphaV1_R + st2 {v0.4s, v1.4s}, [pCRow1] + + add pCRow2, pCRow1, #32 + + + fmul v2.4s, v18.4s, alphaV0_R + fmls v2.4s, v19.4s, alphaV0_I + fmul v3.4s, v18.4s, alphaV1_I + fmla v3.4s, v19.4s, alphaV1_R + st2 {v2.4s, v3.4s}, [pCRow2] + + add pCRow1, pCRow1, LDC + + + fmul v4.4s, v20.4s, alphaV0_R + fmls v4.4s, v21.4s, alphaV0_I + fmul v5.4s, v20.4s, alphaV1_I + fmla v5.4s, v21.4s, alphaV1_R + st2 {v4.4s, v5.4s}, [pCRow1] + + add pCRow2, pCRow1, #32 + + + fmul v6.4s, v22.4s, alphaV0_R + fmls v6.4s, v23.4s, alphaV0_I + fmul v7.4s, v22.4s, alphaV1_I + fmla v7.4s, v23.4s, alphaV1_R + st2 {v6.4s, v7.4s}, [pCRow2] + + add pCRow1, pCRow1, LDC + + + fmul v0.4s, v24.4s, alphaV0_R + fmls v0.4s, v25.4s, alphaV0_I + fmul v1.4s, v24.4s, alphaV1_I + fmla v1.4s, v25.4s, alphaV1_R + st2 {v0.4s, v1.4s}, [pCRow1] + + add pCRow2, pCRow1, #32 + + + fmul v2.4s, v26.4s, alphaV0_R + fmls v2.4s, v27.4s, alphaV0_I + fmul v3.4s, v26.4s, alphaV1_I + fmla v3.4s, v27.4s, alphaV1_R + st2 {v2.4s, v3.4s}, [pCRow2] + + add pCRow1, pCRow1, LDC + + + fmul v4.4s, v28.4s, alphaV0_R + fmls v4.4s, v29.4s, alphaV0_I + fmul v5.4s, v28.4s, alphaV1_I + fmla v5.4s, v29.4s, alphaV1_R + st2 {v4.4s, v5.4s}, [pCRow1] + + add pCRow2, pCRow1, #32 + + + fmul v6.4s, v30.4s, alphaV0_R + fmls v6.4s, v31.4s, alphaV0_I + fmul v7.4s, v30.4s, alphaV1_I + fmla v7.4s, v31.4s, alphaV1_R + st2 {v6.4s, v7.4s}, [pCRow2] + + add pCRow0, pCRow0, #64 +.endm + +/******************************************************************************/ + +.macro INIT4x4 + fmov s16, wzr + fmov s17, s16 + fmov s20, s17 + fmov s21, s16 + fmov s24, s17 + fmov s25, s16 + fmov s28, s17 + fmov s29, s16 +.endm + +.macro KERNEL4x4_I + ld2 {v8.4s, v9.4s}, [pB] + add pB, pB, #32 + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + + fmul v16.4s, v0.4s, v8.4s[0] + OP_ii v16.4s, v1.4s, v9.4s[0] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v17.16b, v17.16b, v17.16b + fmls v17.4s, v0.4s, v9.4s[0] +#else + fmul v17.4s, v0.4s, v9.4s[0] +#endif + OP_ir v17.4s, v1.4s, v8.4s[0] + + fmul v20.4s, v0.4s, v8.4s[1] + OP_ii v20.4s, v1.4s, v9.4s[1] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v21.16b, v21.16b, v21.16b + fmls v21.4s, v0.4s, v9.4s[1] +#else + fmul v21.4s, v0.4s, v9.4s[1] +#endif + OP_ir v21.4s, v1.4s, v8.4s[1] + + fmul v24.4s, v0.4s, v8.4s[2] + OP_ii v24.4s, v1.4s, v9.4s[2] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v25.16b, v25.16b, v25.16b + fmls v25.4s, v0.4s, v9.4s[2] +#else + fmul v25.4s, v0.4s, v9.4s[2] +#endif + OP_ir v25.4s, v1.4s, v8.4s[2] + + fmul v28.4s, v0.4s, v8.4s[3] + OP_ii v28.4s, v1.4s, v9.4s[3] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v29.16b, v29.16b, v29.16b + fmls v29.4s, v0.4s, v9.4s[3] +#else + fmul v29.4s, v0.4s, v9.4s[3] +#endif + OP_ir v29.4s, v1.4s, v8.4s[3] + + ld2 {v12.4s, v13.4s}, [pB] + add pB, pB, #32 + ld2 {v4.4s, v5.4s}, [pA] + add pA, pA, #32 +.endm + +.macro KERNEL4x4_M1 + OP_rr v16.4s, v0.4s, v8.4s[0] + OP_ii v16.4s, v1.4s, v9.4s[0] + OP_ri v17.4s, v0.4s, v9.4s[0] + OP_ir v17.4s, v1.4s, v8.4s[0] + + ld2 {v12.4s, v13.4s}, [pB] // For next round + add pB, pB, #32 + + OP_rr v20.4s, v0.4s, v8.4s[1] + OP_ii v20.4s, v1.4s, v9.4s[1] + OP_ri v21.4s, v0.4s, v9.4s[1] + OP_ir v21.4s, v1.4s, v8.4s[1] + + ld2 {v4.4s, v5.4s}, [pA] // For next round + add pA, pA, #32 + + OP_rr v24.4s, v0.4s, v8.4s[2] + OP_ii v24.4s, v1.4s, v9.4s[2] + OP_ri v25.4s, v0.4s, v9.4s[2] + OP_ir v25.4s, v1.4s, v8.4s[2] + + prfm PLDL1KEEP, [pA, #512] + + OP_rr v28.4s, v0.4s, v8.4s[3] + OP_ii v28.4s, v1.4s, v9.4s[3] + OP_ri v29.4s, v0.4s, v9.4s[3] + OP_ir v29.4s, v1.4s, v8.4s[3] +.endm + +.macro KERNEL4x4_M2 + OP_rr v16.4s, v4.4s, v12.4s[0] + OP_ii v16.4s, v5.4s, v13.4s[0] + OP_ri v17.4s, v4.4s, v13.4s[0] + OP_ir v17.4s, v5.4s, v12.4s[0] + + ld2 {v8.4s, v9.4s}, [pB] // For next round + add pB, pB, #32 + + OP_rr v20.4s, v4.4s, v12.4s[1] + OP_ii v20.4s, v5.4s, v13.4s[1] + OP_ri v21.4s, v4.4s, v13.4s[1] + OP_ir v21.4s, v5.4s, v12.4s[1] + + ld2 {v0.4s, v1.4s}, [pA] // For next round + add pA, pA, #32 + + OP_rr v24.4s, v4.4s, v12.4s[2] + OP_ii v24.4s, v5.4s, v13.4s[2] + OP_ri v25.4s, v4.4s, v13.4s[2] + OP_ir v25.4s, v5.4s, v12.4s[2] + + prfm PLDL1KEEP, [pB, #512] + + OP_rr v28.4s, v4.4s, v12.4s[3] + OP_ii v28.4s, v5.4s, v13.4s[3] + OP_ri v29.4s, v4.4s, v13.4s[3] + OP_ir v29.4s, v5.4s, v12.4s[3] +.endm + +.macro KERNEL4x4_E + OP_rr v16.4s, v4.4s, v12.4s[0] + OP_ii v16.4s, v5.4s, v13.4s[0] + OP_ri v17.4s, v4.4s, v13.4s[0] + OP_ir v17.4s, v5.4s, v12.4s[0] + + OP_rr v20.4s, v4.4s, v12.4s[1] + OP_ii v20.4s, v5.4s, v13.4s[1] + OP_ri v21.4s, v4.4s, v13.4s[1] + OP_ir v21.4s, v5.4s, v12.4s[1] + + OP_rr v24.4s, v4.4s, v12.4s[2] + OP_ii v24.4s, v5.4s, v13.4s[2] + OP_ri v25.4s, v4.4s, v13.4s[2] + OP_ir v25.4s, v5.4s, v12.4s[2] + + OP_rr v28.4s, v4.4s, v12.4s[3] + OP_ii v28.4s, v5.4s, v13.4s[3] + OP_ri v29.4s, v4.4s, v13.4s[3] + OP_ir v29.4s, v5.4s, v12.4s[3] +.endm + +.macro KERNEL4x4_SUB + ld2 {v8.4s, v9.4s}, [pB] + add pB, pB, #32 + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + + OP_rr v16.4s, v0.4s, v8.4s[0] + OP_ii v16.4s, v1.4s, v9.4s[0] + OP_ri v17.4s, v0.4s, v9.4s[0] + OP_ir v17.4s, v1.4s, v8.4s[0] + + OP_rr v20.4s, v0.4s, v8.4s[1] + OP_ii v20.4s, v1.4s, v9.4s[1] + OP_ri v21.4s, v0.4s, v9.4s[1] + OP_ir v21.4s, v1.4s, v8.4s[1] + + OP_rr v24.4s, v0.4s, v8.4s[2] + OP_ii v24.4s, v1.4s, v9.4s[2] + OP_ri v25.4s, v0.4s, v9.4s[2] + OP_ir v25.4s, v1.4s, v8.4s[2] + + OP_rr v28.4s, v0.4s, v8.4s[3] + OP_ii v28.4s, v1.4s, v9.4s[3] + OP_ri v29.4s, v0.4s, v9.4s[3] + OP_ir v29.4s, v1.4s, v8.4s[3] +.endm + +.macro SAVE4x4 + mov pCRow1, pCRow0 + + + fmul v0.4s, v16.4s, alphaV0_R + fmls v0.4s, v17.4s, alphaV0_I + fmul v1.4s, v16.4s, alphaV1_I + fmla v1.4s, v17.4s, alphaV1_R + st2 {v0.4s, v1.4s}, [pCRow1] + + add pCRow1, pCRow1, LDC + + + fmul v4.4s, v20.4s, alphaV0_R + fmls v4.4s, v21.4s, alphaV0_I + fmul v5.4s, v20.4s, alphaV1_I + fmla v5.4s, v21.4s, alphaV1_R + st2 {v4.4s, v5.4s}, [pCRow1] + + add pCRow1, pCRow1, LDC + + + fmul v0.4s, v24.4s, alphaV0_R + fmls v0.4s, v25.4s, alphaV0_I + fmul v1.4s, v24.4s, alphaV1_I + fmla v1.4s, v25.4s, alphaV1_R + st2 {v0.4s, v1.4s}, [pCRow1] + + add pCRow1, pCRow1, LDC + + + fmul v4.4s, v28.4s, alphaV0_R + fmls v4.4s, v29.4s, alphaV0_I + fmul v5.4s, v28.4s, alphaV1_I + fmla v5.4s, v29.4s, alphaV1_R + st2 {v4.4s, v5.4s}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT2x4 + fmov s16, wzr + fmov s17, wzr + fmov s20, s16 + fmov s21, s17 + fmov s24, s16 + fmov s25, s17 + fmov s28, s16 + fmov s29, s17 +.endm + +.macro KERNEL2x4_SUB + ld2 {v8.4s, v9.4s}, [pB] + add pB, pB, #32 + ld2 {v0.2s, v1.2s}, [pA] + add pA, pA, #16 + + OP_rr v16.2s, v0.2s, v8.4s[0] + OP_ii v16.2s, v1.2s, v9.4s[0] + OP_ri v17.2s, v0.2s, v9.4s[0] + OP_ir v17.2s, v1.2s, v8.4s[0] + + OP_rr v20.2s, v0.2s, v8.4s[1] + OP_ii v20.2s, v1.2s, v9.4s[1] + OP_ri v21.2s, v0.2s, v9.4s[1] + OP_ir v21.2s, v1.2s, v8.4s[1] + + OP_rr v24.2s, v0.2s, v8.4s[2] + OP_ii v24.2s, v1.2s, v9.4s[2] + OP_ri v25.2s, v0.2s, v9.4s[2] + OP_ir v25.2s, v1.2s, v8.4s[2] + + OP_rr v28.2s, v0.2s, v8.4s[3] + OP_ii v28.2s, v1.2s, v9.4s[3] + OP_ri v29.2s, v0.2s, v9.4s[3] + OP_ir v29.2s, v1.2s, v8.4s[3] +.endm + +.macro SAVE2x4 + mov pCRow1, pCRow0 + + + fmul v0.2s, v16.2s, alphaV0_R + fmls v0.2s, v17.2s, alphaV0_I + fmul v1.2s, v16.2s, alphaV1_I + fmla v1.2s, v17.2s, alphaV1_R + st2 {v0.2s, v1.2s}, [pCRow1] + + add pCRow1, pCRow1, LDC + + + fmul v4.2s, v20.2s, alphaV0_R + fmls v4.2s, v21.2s, alphaV0_I + fmul v5.2s, v20.2s, alphaV1_I + fmla v5.2s, v21.2s, alphaV1_R + st2 {v4.2s, v5.2s}, [pCRow1] + + add pCRow1, pCRow1, LDC + + + fmul v0.2s, v24.2s, alphaV0_R + fmls v0.2s, v25.2s, alphaV0_I + fmul v1.2s, v24.2s, alphaV1_I + fmla v1.2s, v25.2s, alphaV1_R + st2 {v0.2s, v1.2s}, [pCRow1] + + add pCRow1, pCRow1, LDC + + + fmul v4.2s, v28.2s, alphaV0_R + fmls v4.2s, v29.2s, alphaV0_I + fmul v5.2s, v28.2s, alphaV1_I + fmla v5.2s, v29.2s, alphaV1_R + st2 {v4.2s, v5.2s}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT1x4 + fmov s16, wzr + fmov s17, wzr + fmov s20, s16 + fmov s21, s17 + fmov s24, s16 + fmov s25, s17 + fmov s28, s16 + fmov s29, s17 +.endm + +.macro KERNEL1x4_SUB + ld2 {v8.4s, v9.4s}, [pB] + add pB, pB, #32 + ld2 {v0.s, v1.s}[0], [pA] + add pA, pA, #8 + + OP_rr s16, s0, v8.4s[0] + OP_ii s16, s1, v9.4s[0] + OP_ri s17, s0, v9.4s[0] + OP_ir s17, s1, v8.4s[0] + + OP_rr s20, s0, v8.4s[1] + OP_ii s20, s1, v9.4s[1] + OP_ri s21, s0, v9.4s[1] + OP_ir s21, s1, v8.4s[1] + + OP_rr s24, s0, v8.4s[2] + OP_ii s24, s1, v9.4s[2] + OP_ri s25, s0, v9.4s[2] + OP_ir s25, s1, v8.4s[2] + + OP_rr s28, s0, v8.4s[3] + OP_ii s28, s1, v9.4s[3] + OP_ri s29, s0, v9.4s[3] + OP_ir s29, s1, v8.4s[3] +.endm + +.macro SAVE1x4 + mov pCRow1, pCRow0 + + + fmul s0, s16, alphaV0_R + fmls s0, s17, alphaV0_I + fmul s1, s16, alphaV1_I + fmla s1, s17, alphaV1_R + st2 {v0.s, v1.s}[0], [pCRow1] + + add pCRow1, pCRow1, LDC + + + fmul s4, s20, alphaV0_R + fmls s4, s21, alphaV0_I + fmul s5, s20, alphaV1_I + fmla s5, s21, alphaV1_R + st2 {v4.s, v5.s}[0], [pCRow1] + + add pCRow1, pCRow1, LDC + + + fmul s0, s24, alphaV0_R + fmls s0, s25, alphaV0_I + fmul s1, s24, alphaV1_I + fmla s1, s25, alphaV1_R + st2 {v0.s, v1.s}[0], [pCRow1] + + add pCRow1, pCRow1, LDC + + + fmul s4, s28, alphaV0_R + fmls s4, s29, alphaV0_I + fmul s5, s28, alphaV1_I + fmla s5, s29, alphaV1_R + st2 {v4.s, v5.s}[0], [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT8x2 + fmov s16, wzr + fmov s17, wzr + fmov s18, wzr + fmov s19, s16 + fmov s20, wzr + fmov s21, s16 + fmov s22, s17 + fmov s23, s18 +.endm + +.macro KERNEL8x2_SUB + ld2 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + ld2 {v2.4s, v3.4s}, [pA] + add pA, pA, #32 + + OP_rr v16.4s, v0.4s, v8.2s[0] + OP_ii v16.4s, v1.4s, v9.2s[0] + OP_ri v17.4s, v0.4s, v9.2s[0] + OP_ir v17.4s, v1.4s, v8.2s[0] + + OP_rr v18.4s, v2.4s, v8.2s[0] + OP_ii v18.4s, v3.4s, v9.2s[0] + OP_ri v19.4s, v2.4s, v9.2s[0] + OP_ir v19.4s, v3.4s, v8.2s[0] + + OP_rr v20.4s, v0.4s, v8.2s[1] + OP_ii v20.4s, v1.4s, v9.2s[1] + OP_ri v21.4s, v0.4s, v9.2s[1] + OP_ir v21.4s, v1.4s, v8.2s[1] + + OP_rr v22.4s, v2.4s, v8.2s[1] + OP_ii v22.4s, v3.4s, v9.2s[1] + OP_ri v23.4s, v2.4s, v9.2s[1] + OP_ir v23.4s, v3.4s, v8.2s[1] +.endm + +.macro SAVE8x2 + mov pCRow1, pCRow0 + + + fmul v0.4s, v16.4s, alphaV0_R + fmls v0.4s, v17.4s, alphaV0_I + fmul v1.4s, v16.4s, alphaV1_I + fmla v1.4s, v17.4s, alphaV1_R + st2 {v0.4s, v1.4s}, [pCRow1] + + add pCRow2, pCRow1, #32 + + + fmul v2.4s, v18.4s, alphaV0_R + fmls v2.4s, v19.4s, alphaV0_I + fmul v3.4s, v18.4s, alphaV1_I + fmla v3.4s, v19.4s, alphaV1_R + st2 {v2.4s, v3.4s}, [pCRow2] + + add pCRow1, pCRow1, LDC + + + fmul v4.4s, v20.4s, alphaV0_R + fmls v4.4s, v21.4s, alphaV0_I + fmul v5.4s, v20.4s, alphaV1_I + fmla v5.4s, v21.4s, alphaV1_R + st2 {v4.4s, v5.4s}, [pCRow1] + + add pCRow2, pCRow1, #32 + + + fmul v6.4s, v22.4s, alphaV0_R + fmls v6.4s, v23.4s, alphaV0_I + fmul v7.4s, v22.4s, alphaV1_I + fmla v7.4s, v23.4s, alphaV1_R + st2 {v6.4s, v7.4s}, [pCRow2] + + add pCRow0, pCRow0, #64 +.endm + +/******************************************************************************/ + +.macro INIT4x2 + fmov s16, wzr + fmov s17, wzr + fmov s20, s16 + fmov s21, s17 +.endm + +.macro KERNEL4x2_SUB + ld2 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + + OP_rr v16.4s, v0.4s, v8.2s[0] + OP_ii v16.4s, v1.4s, v9.2s[0] + OP_ri v17.4s, v0.4s, v9.2s[0] + OP_ir v17.4s, v1.4s, v8.2s[0] + + OP_rr v20.4s, v0.4s, v8.2s[1] + OP_ii v20.4s, v1.4s, v9.2s[1] + OP_ri v21.4s, v0.4s, v9.2s[1] + OP_ir v21.4s, v1.4s, v8.2s[1] +.endm + +.macro SAVE4x2 + mov pCRow1, pCRow0 + + + fmul v0.4s, v16.4s, alphaV0_R + fmls v0.4s, v17.4s, alphaV0_I + fmul v1.4s, v16.4s, alphaV1_I + fmla v1.4s, v17.4s, alphaV1_R + st2 {v0.4s, v1.4s}, [pCRow1] + + add pCRow1, pCRow1, LDC + + + fmul v4.4s, v20.4s, alphaV0_R + fmls v4.4s, v21.4s, alphaV0_I + fmul v5.4s, v20.4s, alphaV1_I + fmla v5.4s, v21.4s, alphaV1_R + st2 {v4.4s, v5.4s}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT2x2 + fmov s16, wzr + fmov s17, wzr + fmov s20, s16 + fmov s21, s17 +.endm + +.macro KERNEL2x2_SUB + ld2 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld2 {v0.2s, v1.2s}, [pA] + add pA, pA, #16 + + OP_rr v16.2s, v0.2s, v8.2s[0] + OP_ii v16.2s, v1.2s, v9.2s[0] + OP_ri v17.2s, v0.2s, v9.2s[0] + OP_ir v17.2s, v1.2s, v8.2s[0] + + OP_rr v20.2s, v0.2s, v8.2s[1] + OP_ii v20.2s, v1.2s, v9.2s[1] + OP_ri v21.2s, v0.2s, v9.2s[1] + OP_ir v21.2s, v1.2s, v8.2s[1] +.endm + +.macro SAVE2x2 + mov pCRow1, pCRow0 + + + fmul v0.2s, v16.2s, alphaV0_R + fmls v0.2s, v17.2s, alphaV0_I + fmul v1.2s, v16.2s, alphaV1_I + fmla v1.2s, v17.2s, alphaV1_R + st2 {v0.2s, v1.2s}, [pCRow1] + + add pCRow1, pCRow1, LDC + + + fmul v4.2s, v20.2s, alphaV0_R + fmls v4.2s, v21.2s, alphaV0_I + fmul v5.2s, v20.2s, alphaV1_I + fmla v5.2s, v21.2s, alphaV1_R + st2 {v4.2s, v5.2s}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT1x2 + fmov s16, wzr + fmov s17, wzr + fmov s20, wzr + fmov s21, wzr +.endm + +.macro KERNEL1x2_SUB + ld2 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld2 {v0.s, v1.s}[0], [pA] + add pA, pA, #8 + + OP_rr s16, s0, v8.2s[0] + OP_ii s16, s1, v9.2s[0] + OP_ri s17, s0, v9.2s[0] + OP_ir s17, s1, v8.2s[0] + + OP_rr s20, s0, v8.2s[1] + OP_ii s20, s1, v9.2s[1] + OP_ri s21, s0, v9.2s[1] + OP_ir s21, s1, v8.2s[1] +.endm + +.macro SAVE1x2 + mov pCRow1, pCRow0 + + + fmul s0, s16, alphaV0_R + fmls s0, s17, alphaV0_I + fmul s1, s16, alphaV1_I + fmla s1, s17, alphaV1_R + st2 {v0.s, v1.s}[0], [pCRow1] + + add pCRow1, pCRow1, LDC + + + fmul s4, s20, alphaV0_R + fmls s4, s21, alphaV0_I + fmul s5, s20, alphaV1_I + fmla s5, s21, alphaV1_R + st2 {v4.s, v5.s}[0], [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT8x1 + fmov s16, wzr + fmov s17, wzr + fmov s18, wzr + fmov s19, s16 +.endm + +.macro KERNEL8x1_SUB + ld1 {v8.2s}, [pB] + add pB, pB, #8 + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + ld2 {v2.4s, v3.4s}, [pA] + add pA, pA, #32 + + OP_rr v16.4s, v0.4s, v8.4s[0] + OP_ii v16.4s, v1.4s, v8.4s[1] + OP_ri v17.4s, v0.4s, v8.4s[1] + OP_ir v17.4s, v1.4s, v8.4s[0] + + OP_rr v18.4s, v2.4s, v8.4s[0] + OP_ii v18.4s, v3.4s, v8.4s[1] + OP_ri v19.4s, v2.4s, v8.4s[1] + OP_ir v19.4s, v3.4s, v8.4s[0] +.endm + +.macro SAVE8x1 + mov pCRow1, pCRow0 + + + fmul v0.4s, v16.4s, alphaV0_R + fmls v0.4s, v17.4s, alphaV0_I + fmul v1.4s, v16.4s, alphaV1_I + fmla v1.4s, v17.4s, alphaV1_R + st2 {v0.4s, v1.4s}, [pCRow1] + + add pCRow1, pCRow1, #32 + + + fmul v2.4s, v18.4s, alphaV0_R + fmls v2.4s, v19.4s, alphaV0_I + fmul v3.4s, v18.4s, alphaV1_I + fmla v3.4s, v19.4s, alphaV1_R + st2 {v2.4s, v3.4s}, [pCRow1] + + add pCRow0, pCRow0, #64 +.endm + + +/******************************************************************************/ + +.macro INIT4x1 + fmov s16, wzr + fmov s17, s16 +.endm + +.macro KERNEL4x1_SUB + ld2 {v8.s, v9.s}[0], [pB] + add pB, pB, #8 + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + + OP_rr v16.4s, v0.4s, v8.s[0] + OP_ii v16.4s, v1.4s, v9.s[0] + OP_ri v17.4s, v0.4s, v9.s[0] + OP_ir v17.4s, v1.4s, v8.s[0] +.endm + +.macro SAVE4x1 + mov pCRow1, pCRow0 + + + fmul v0.4s, v16.4s, alphaV0_R + fmls v0.4s, v17.4s, alphaV0_I + fmul v1.4s, v16.4s, alphaV1_I + fmla v1.4s, v17.4s, alphaV1_R + st2 {v0.4s, v1.4s}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT2x1 + fmov s16, wzr + fmov s17, wzr +.endm + +.macro KERNEL2x1_SUB + ld2 {v8.s, v9.s}[0], [pB] + add pB, pB, #8 + ld2 {v0.2s, v1.2s}, [pA] + add pA, pA, #16 + + OP_rr v16.2s, v0.2s, v8.s[0] + OP_ii v16.2s, v1.2s, v9.s[0] + OP_ri v17.2s, v0.2s, v9.s[0] + OP_ir v17.2s, v1.2s, v8.s[0] +.endm + +.macro SAVE2x1 + mov pCRow1, pCRow0 + + + fmul v0.2s, v16.2s, alphaV0_R + fmls v0.2s, v17.2s, alphaV0_I + fmul v1.2s, v16.2s, alphaV1_I + fmla v1.2s, v17.2s, alphaV1_R + st2 {v0.2s, v1.2s}, [pCRow1] + + add pCRow0, pCRow0, #16 + +.endm + +/******************************************************************************/ + +.macro INIT1x1 + fmov s16, wzr + fmov s17, wzr +.endm + +.macro KERNEL1x1_SUB + ld2 {v8.s, v9.s}[0], [pB] + add pB, pB, #8 + ld2 {v0.s, v1.s}[0], [pA] + add pA, pA, #8 + + OP_rr s16, s0, v8.s[0] + OP_ii s16, s1, v9.s[0] + OP_ri s17, s0, v9.s[0] + OP_ir s17, s1, v8.s[0] +.endm + +.macro SAVE1x1 + mov pCRow1, pCRow0 + + + fmul s0, s16, alphaV0_R + fmls s0, s17, alphaV0_I + fmul s1, s16, alphaV1_I + fmla s1, s17, alphaV1_R + st2 {v0.s, v1.s}[0], [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************* +* End of macro definitions +*******************************************************************************/ + + PROLOGUE + + .align 5 + add sp, sp, #-(11 * 16) + stp d8, d9, [sp, #(0 * 16)] + stp d10, d11, [sp, #(1 * 16)] + stp d12, d13, [sp, #(2 * 16)] + stp d14, d15, [sp, #(3 * 16)] + stp d16, d17, [sp, #(4 * 16)] + stp x18, x19, [sp, #(5 * 16)] + stp x20, x21, [sp, #(6 * 16)] + stp x22, x23, [sp, #(7 * 16)] + stp x24, x25, [sp, #(8 * 16)] + stp x26, x27, [sp, #(9 * 16)] + str x28, [sp, #(10 * 16)] + + fmov alpha0_R, s0 + fmov alpha0_I, s1 + fmov alpha1_R, s0 + fmov alpha1_I, s1 + + lsl LDC, LDC, #3 // ldc = ldc * 8 + +#if !defined(LEFT) + neg tempOffset, offset +#endif + mov pB, origPB + + mov counterJ, origN + asr counterJ, counterJ, #2 // J = J / 4 + cmp counterJ, #0 + ble ctrmm_kernel_L2_BEGIN + +/******************************************************************************/ + +ctrmm_kernel_L4_BEGIN: + mov pCRow0, pC // pCRow0 = C + add pC, pC, LDC, lsl #2 + +#if defined(LEFT) + mov tempOffset, offset +#endif + mov pA, origPA // pA = start of A array + +ctrmm_kernel_L4_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 // counterI = counterI / 8 + cmp counterI, #0 + ble ctrmm_kernel_L4_M4_BEGIN + +ctrmm_kernel_L4_M8_20: + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #6 + add pA, pA, temp + lsl temp, tempOffset, #5 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #8 +#else + add tempK, tempOffset, #4 +#endif + + asr counterL , tempK, #1 // L = K / 2 + cmp counterL , #2 // is there at least 4 to do? + blt ctrmm_kernel_L4_M8_32 + + KERNEL8x4_I // do one in the K + KERNEL8x4_M2 // do another in the K + + subs counterL, counterL, #2 // subtract 2 + ble ctrmm_kernel_L4_M8_22a + .align 5 + +ctrmm_kernel_L4_M8_22: + + KERNEL8x4_M1 + KERNEL8x4_M2 + + subs counterL, counterL, #1 + bgt ctrmm_kernel_L4_M8_22 + + +ctrmm_kernel_L4_M8_22a: + + KERNEL8x4_M1 + KERNEL8x4_E + + b ctrmm_kernel_L4_M8_44 + +ctrmm_kernel_L4_M8_32: + + tst counterL, #1 + ble ctrmm_kernel_L4_M8_40 + + KERNEL8x4_I + + KERNEL8x4_E + + b ctrmm_kernel_L4_M8_44 + +ctrmm_kernel_L4_M8_40: + + INIT8x4 + +ctrmm_kernel_L4_M8_44: + + ands counterL , tempK, #1 + ble ctrmm_kernel_L4_M8_100 + +ctrmm_kernel_L4_M8_46: + + KERNEL8x4_SUB + +ctrmm_kernel_L4_M8_100: + + SAVE8x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #8 +#else + sub tempK, tempK, #4 +#endif + lsl temp, tempK, #6 + add pA, pA, temp + lsl temp, tempK, #5 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #8 +#endif + +ctrmm_kernel_L4_M8_END: + subs counterI, counterI, #1 + bne ctrmm_kernel_L4_M8_20 + +ctrmm_kernel_L4_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble ctrmm_kernel_L4_END + + tst counterI, #4 + ble ctrmm_kernel_L4_M2_BEGIN + +ctrmm_kernel_L4_M4_20: + + INIT4x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #5 + add pB, pB, temp + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #4 +#else + add tempK, tempOffset, #4 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble ctrmm_kernel_L4_M4_40 + +ctrmm_kernel_L4_M4_22: + + KERNEL4x4_SUB + KERNEL4x4_SUB + KERNEL4x4_SUB + KERNEL4x4_SUB + + KERNEL4x4_SUB + KERNEL4x4_SUB + KERNEL4x4_SUB + KERNEL4x4_SUB + + subs counterL, counterL, #1 + bgt ctrmm_kernel_L4_M4_22 + + +ctrmm_kernel_L4_M4_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble ctrmm_kernel_L4_M4_100 + +ctrmm_kernel_L4_M4_42: + + KERNEL4x4_SUB + + subs counterL, counterL, #1 + bgt ctrmm_kernel_L4_M4_42 + +ctrmm_kernel_L4_M4_100: + + SAVE4x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #4 +#else + sub tempK, tempK, #4 +#endif + lsl temp, tempK, #5 + add pA, pA, temp + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #4 +#endif + +ctrmm_kernel_L4_M4_END: + + +ctrmm_kernel_L4_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble ctrmm_kernel_L4_END + + tst counterI, #2 // counterI = counterI / 2 + ble ctrmm_kernel_L4_M1_BEGIN + +ctrmm_kernel_L4_M2_20: + + INIT2x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #4 + add pA, pA, temp + lsl temp, tempOffset, #5 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #2 +#else + add tempK, tempOffset, #4 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble ctrmm_kernel_L4_M2_40 + +ctrmm_kernel_L4_M2_22: + + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + + subs counterL, counterL, #1 + bgt ctrmm_kernel_L4_M2_22 + + +ctrmm_kernel_L4_M2_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble ctrmm_kernel_L4_M2_100 + +ctrmm_kernel_L4_M2_42: + + KERNEL2x4_SUB + + subs counterL, counterL, #1 + bgt ctrmm_kernel_L4_M2_42 + +ctrmm_kernel_L4_M2_100: + + SAVE2x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #2 +#else + sub tempK, tempK, #4 +#endif + lsl temp, tempK, #4 + add pA, pA, temp + lsl temp, tempK, #5 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #2 +#endif + +ctrmm_kernel_L4_M2_END: + + +ctrmm_kernel_L4_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble ctrmm_kernel_L4_END + +ctrmm_kernel_L4_M1_20: + + INIT1x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #5 + add pB, pB, temp + lsl temp, tempOffset, #3 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #1 +#else + add tempK, tempOffset, #4 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble ctrmm_kernel_L4_M1_40 + +ctrmm_kernel_L4_M1_22: + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + + subs counterL, counterL, #1 + bgt ctrmm_kernel_L4_M1_22 + + +ctrmm_kernel_L4_M1_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble ctrmm_kernel_L4_M1_100 + +ctrmm_kernel_L4_M1_42: + + KERNEL1x4_SUB + + subs counterL, counterL, #1 + bgt ctrmm_kernel_L4_M1_42 + +ctrmm_kernel_L4_M1_100: + + SAVE1x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #1 +#else + sub tempK, tempK, #4 +#endif + lsl temp, tempK, #3 + add pA, pA, temp + lsl temp, tempK, #5 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #1 +#endif + +ctrmm_kernel_L4_END: + + lsl temp, origK, #5 + add origPB, origPB, temp // B = B + K * 4 * 8 + +#if !defined(LEFT) + add tempOffset, tempOffset, #4 +#endif + + subs counterJ, counterJ , #1 // j-- + bgt ctrmm_kernel_L4_BEGIN + + +/******************************************************************************/ + +ctrmm_kernel_L2_BEGIN: // less than 2 left in N direction + + mov counterJ , origN + tst counterJ , #3 + ble ctrmm_kernel_L999 // error, N was less than 4? + + tst counterJ , #2 + ble ctrmm_kernel_L1_BEGIN + + mov pCRow0, pC // pCRow0 = pC + + add pC,pC,LDC, lsl #1 + +#if defined(LEFT) + mov tempOffset, offset +#endif + mov pA, origPA // pA = A + +ctrmm_kernel_L2_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 // counterI = counterI / 8 + cmp counterI, #0 + ble ctrmm_kernel_L2_M4_BEGIN + +ctrmm_kernel_L2_M8_20: + + INIT8x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #6 + add pA, pA, temp + lsl temp, tempOffset, #4 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #8 +#else + add tempK, tempOffset, #2 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble ctrmm_kernel_L2_M8_40 + .align 5 + +ctrmm_kernel_L2_M8_22: + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + + subs counterL, counterL, #1 + bgt ctrmm_kernel_L2_M8_22 + + +ctrmm_kernel_L2_M8_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble ctrmm_kernel_L2_M8_100 + +ctrmm_kernel_L2_M8_42: + + KERNEL8x2_SUB + + subs counterL, counterL, #1 + bgt ctrmm_kernel_L2_M8_42 + +ctrmm_kernel_L2_M8_100: + + SAVE8x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #8 +#else + sub tempK, tempK, #2 +#endif + lsl temp, tempK, #6 + add pA, pA, temp + lsl temp, tempK, #4 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #8 +#endif + +ctrmm_kernel_L2_M8_END: + + subs counterI, counterI, #1 + bgt ctrmm_kernel_L2_M8_20 + +ctrmm_kernel_L2_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble ctrmm_kernel_L2_END + + tst counterI, #4 // counterI = counterI / 2 + ble ctrmm_kernel_L2_M2_BEGIN + +ctrmm_kernel_L2_M4_20: + + INIT4x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #4 + add pB, pB, temp + lsl temp, tempOffset, #5 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #4 +#else + add tempK, tempOffset, #2 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble ctrmm_kernel_L2_M4_40 + .align 5 + +ctrmm_kernel_L2_M4_22: + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + subs counterL, counterL, #1 + bgt ctrmm_kernel_L2_M4_22 + + +ctrmm_kernel_L2_M4_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble ctrmm_kernel_L2_M4_100 + +ctrmm_kernel_L2_M4_42: + + KERNEL4x2_SUB + + subs counterL, counterL, #1 + bgt ctrmm_kernel_L2_M4_42 + +ctrmm_kernel_L2_M4_100: + + SAVE4x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #4 +#else + sub tempK, tempK, #2 +#endif + lsl temp, tempK, #5 + add pA, pA, temp + lsl temp, tempK, #4 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #4 +#endif + +ctrmm_kernel_L2_M4_END: + + +ctrmm_kernel_L2_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble ctrmm_kernel_L2_END + + tst counterI, #2 // counterI = counterI / 2 + ble ctrmm_kernel_L2_M1_BEGIN + +ctrmm_kernel_L2_M2_20: + + INIT2x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #4 + add pB, pB, temp + lsl temp, tempOffset, #4 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #2 +#else + add tempK, tempOffset, #2 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble ctrmm_kernel_L2_M2_40 + +ctrmm_kernel_L2_M2_22: + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt ctrmm_kernel_L2_M2_22 + + +ctrmm_kernel_L2_M2_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble ctrmm_kernel_L2_M2_100 + +ctrmm_kernel_L2_M2_42: + + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt ctrmm_kernel_L2_M2_42 + +ctrmm_kernel_L2_M2_100: + + SAVE2x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #2 +#else + sub tempK, tempK, #2 +#endif + lsl temp, tempK, #4 + add pA, pA, temp + lsl temp, tempK, #4 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #2 +#endif + +ctrmm_kernel_L2_M2_END: + + +ctrmm_kernel_L2_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble ctrmm_kernel_L2_END + +ctrmm_kernel_L2_M1_20: + + INIT1x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #4 + add pB, pB, temp + lsl temp, tempOffset, #3 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #1 +#else + add tempK, tempOffset, #2 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL, #0 + ble ctrmm_kernel_L2_M1_40 + +ctrmm_kernel_L2_M1_22: + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt ctrmm_kernel_L2_M1_22 + + +ctrmm_kernel_L2_M1_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble ctrmm_kernel_L2_M1_100 + +ctrmm_kernel_L2_M1_42: + + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt ctrmm_kernel_L2_M1_42 + +ctrmm_kernel_L2_M1_100: + + SAVE1x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #1 +#else + sub tempK, tempK, #2 +#endif + lsl temp, tempK, #3 + add pA, pA, temp + lsl temp, tempK, #4 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #1 +#endif + +ctrmm_kernel_L2_END: +#if !defined(LEFT) + add tempOffset, tempOffset, #2 +#endif + add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 + +/******************************************************************************/ + +ctrmm_kernel_L1_BEGIN: + + mov counterJ , origN + tst counterJ , #1 + ble ctrmm_kernel_L999 // done + + mov pCRow0, pC // pCRow0 = C + add pC , pC , LDC // Update pC to point to next + +#if defined(LEFT) + mov tempOffset, offset +#endif + mov pA, origPA // pA = A + +ctrmm_kernel_L1_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 // counterI = counterI / 8 + cmp counterI, #0 + ble ctrmm_kernel_L1_M4_BEGIN + +ctrmm_kernel_L1_M8_20: + + INIT8x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #6 + add pA, pA, temp + lsl temp, tempOffset, #3 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #8 +#else + add tempK, tempOffset, #1 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble ctrmm_kernel_L1_M8_40 + .align 5 + +ctrmm_kernel_L1_M8_22: + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + + subs counterL, counterL, #1 + bgt ctrmm_kernel_L1_M8_22 + + +ctrmm_kernel_L1_M8_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble ctrmm_kernel_L1_M8_100 + +ctrmm_kernel_L1_M8_42: + + KERNEL8x1_SUB + + subs counterL, counterL, #1 + bgt ctrmm_kernel_L1_M8_42 + +ctrmm_kernel_L1_M8_100: + + SAVE8x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #8 +#else + sub tempK, tempK, #1 +#endif + lsl temp, tempK, #6 + add pA, pA, temp + lsl temp, tempK, #3 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #8 +#endif + +ctrmm_kernel_L1_M8_END: + + subs counterI, counterI, #1 + bgt ctrmm_kernel_L1_M8_20 + +ctrmm_kernel_L1_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble ctrmm_kernel_L1_END + + tst counterI, #4 // counterI = counterI / 2 + ble ctrmm_kernel_L1_M2_BEGIN + +ctrmm_kernel_L1_M4_20: + + INIT4x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #3 + add pB, pB, temp + lsl temp, tempOffset, #5 + add pA, pA, temp +#endif +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #4 +#else + add tempK, tempOffset, #1 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble ctrmm_kernel_L1_M4_40 + .align 5 + +ctrmm_kernel_L1_M4_22: + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + subs counterL, counterL, #1 + bgt ctrmm_kernel_L1_M4_22 + + +ctrmm_kernel_L1_M4_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble ctrmm_kernel_L1_M4_100 + +ctrmm_kernel_L1_M4_42: + + KERNEL4x1_SUB + + subs counterL, counterL, #1 + bgt ctrmm_kernel_L1_M4_42 + +ctrmm_kernel_L1_M4_100: + + SAVE4x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #4 +#else + sub tempK, tempK, #1 +#endif + lsl temp, tempK, #5 + add pA, pA, temp + lsl temp, tempK, #3 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #4 +#endif + +ctrmm_kernel_L1_M4_END: + +ctrmm_kernel_L1_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble ctrmm_kernel_L1_END + + tst counterI, #2 // counterI = counterI / 2 + ble ctrmm_kernel_L1_M1_BEGIN + +ctrmm_kernel_L1_M2_20: + + INIT2x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #3 + add pB, pB, temp + lsl temp, tempOffset, #4 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #2 +#else + add tempK, tempOffset, #1 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble ctrmm_kernel_L1_M2_40 + +ctrmm_kernel_L1_M2_22: + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt ctrmm_kernel_L1_M2_22 + + +ctrmm_kernel_L1_M2_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble ctrmm_kernel_L1_M2_100 + +ctrmm_kernel_L1_M2_42: + + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt ctrmm_kernel_L1_M2_42 + +ctrmm_kernel_L1_M2_100: + + SAVE2x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #2 +#else + sub tempK, tempK, #1 +#endif + lsl temp, tempK, #4 + add pA, pA, temp + lsl temp, tempK, #3 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #2 +#endif + +ctrmm_kernel_L1_M2_END: + + +ctrmm_kernel_L1_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble ctrmm_kernel_L1_END + +ctrmm_kernel_L1_M1_20: + + INIT1x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #3 + add pB, pB, temp + lsl temp, tempOffset, #3 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #1 +#else + add tempK, tempOffset, #1 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble ctrmm_kernel_L1_M1_40 + +ctrmm_kernel_L1_M1_22: + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt ctrmm_kernel_L1_M1_22 + + +ctrmm_kernel_L1_M1_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble ctrmm_kernel_L1_M1_100 + +ctrmm_kernel_L1_M1_42: + + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt ctrmm_kernel_L1_M1_42 + +ctrmm_kernel_L1_M1_100: + + SAVE1x1 + + +ctrmm_kernel_L1_END: + + +ctrmm_kernel_L999: + mov x0, #0 // set return value + ldp d8, d9, [sp, #(0 * 16)] + ldp d10, d11, [sp, #(1 * 16)] + ldp d12, d13, [sp, #(2 * 16)] + ldp d14, d15, [sp, #(3 * 16)] + ldp d16, d17, [sp, #(4 * 16)] + ldp x18, x19, [sp, #(5 * 16)] + ldp x20, x21, [sp, #(6 * 16)] + ldp x22, x23, [sp, #(7 * 16)] + ldp x24, x25, [sp, #(8 * 16)] + ldp x26, x27, [sp, #(9 * 16)] + ldr x28, [sp, #(10 * 16)] + add sp, sp, #(11*16) + ret + + EPILOGUE + diff --git a/kernel/arm64/dgemm_kernel_4x8.S b/kernel/arm64/dgemm_kernel_4x8.S new file mode 100755 index 000000000..88e9a773d --- /dev/null +++ b/kernel/arm64/dgemm_kernel_4x8.S @@ -0,0 +1,1689 @@ +/******************************************************************************* +Copyright (c) 2015, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#define ASSEMBLER +#include "common.h" + +/* X0 X1 X2 s0 X3 x4 x5 x6 */ +/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha0,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc )*/ + +#define origM x0 +#define origN x1 +#define origK x2 +#define origPA x3 +#define origPB x4 +#define pC x5 +#define LDC x6 +#define temp x7 +#define counterL x8 +#define counterI x9 +#define counterJ x10 +#define pB x11 +#define pCRow0 x12 +#define pCRow1 x13 +#define pCRow2 x14 +#define pA x15 + +#define alpha0 d2 +#define alphaV0 v2.d[0] +#define alpha1 d3 +#define alphaV1 v3.d[0] +#define alpha2 d6 +#define alphaV2 v6.d[0] +#define alpha3 d7 +#define alphaV3 v7.d[0] + +// 00 origM +// 01 origN +// 02 origK +// 03 origPA +// 04 origPB +// 05 pC +// 06 origLDC -> LDC +// 07 temp +// 08 counterL +// 09 counterI +// 10 counterJ +// 11 pB +// 12 pCRow0 +// 13 pCRow1 +// 14 pCRow2 +// 15 pA +// 16 +// 17 +// 18 must save +// 19 must save +// 20 must save +// 21 must save +// 22 must save +// 23 must save +// 24 must save +// 25 must save +// 26 must save +// 27 must save +// 28 must save +// 29 frame +// 30 link +// 31 sp + +//v00 ALPHA -> pA00, pA01 +//v01 pA02, pA03 +//v02 ALPHA0 +//v03 ALPHA1 +//v04 pA10, pA11 +//v05 pA12, pA13 +//v06 ALPHA2 +//v07 ALPHA3 +//v08 must save pB0_0, pB0_1 +//v09 must save pB0_2, pB0_3 +//v10 must save pB0_4, pB0_5 +//v11 must save pB0_6, pB0_7 +//v12 must save pB1_0, pB1_1 +//v13 must save pB1_2, pB1_3 +//v14 must save pB1_4, pB1_5 +//v15 must save pB1_6, pB1_7 +//v16 must save C00, C01 +//v17 must save C02, C03 +//v18 C04, C05 +//v19 C06, C07 +//v20 C10, C11 +//v21 C12, C13 +//v22 C14, C15 +//v23 C16, C17 +//v24 C20, C21 +//v25 C22, C23 +//v26 C24, C25 +//v27 C26, C27 +//v28 C30, C31 +//v29 C32, C33 +//v30 C34, C35 +//v31 C36, C37 + +/******************************************************************************* +* Macro definitions +*******************************************************************************/ + +.macro INIT4x8 + fmov d16, xzr + fmov d17, xzr + fmov d18, xzr + fmov d19, d16 + fmov d20, xzr + fmov d21, d16 + fmov d22, d17 + fmov d23, d18 + fmov d24, xzr + fmov d25, d16 + fmov d26, d17 + fmov d27, d18 + fmov d28, xzr + fmov d29, d16 + fmov d30, d17 + fmov d31, d18 +.endm + +.macro KERNEL4x8_I + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + ld1 {v0.2d, v1.2d}, [pA] + add pA, pA, #32 + ld1 {v10.2d, v11.2d}, [pB] + add pB, pB, #32 + + fmul v16.2d, v0.2d, v8.2d[0] + fmul v17.2d, v1.2d, v8.2d[0] + fmul v18.2d, v0.2d, v8.2d[1] + fmul v19.2d, v1.2d, v8.2d[1] + + fmul v20.2d, v0.2d, v9.2d[0] + fmul v21.2d, v1.2d, v9.2d[0] + fmul v22.2d, v0.2d, v9.2d[1] + fmul v23.2d, v1.2d, v9.2d[1] + + fmul v24.2d, v0.2d, v10.2d[0] + fmul v25.2d, v1.2d, v10.2d[0] + fmul v26.2d, v0.2d, v10.2d[1] + fmul v27.2d, v1.2d, v10.2d[1] + + fmul v28.2d, v0.2d, v11.2d[0] + fmul v29.2d, v1.2d, v11.2d[0] + fmul v30.2d, v0.2d, v11.2d[1] + fmul v31.2d, v1.2d, v11.2d[1] + + ld1 {v12.2d, v13.2d}, [pB] + add pB, pB, #32 + ld1 {v4.2d, v5.2d}, [pA] + add pA, pA, #32 + ld1 {v14.2d, v15.2d}, [pB] + add pB, pB, #32 +.endm + +.macro KERNEL4x8_M1 + fmla v16.2d, v0.2d, v8.2d[0] + fmla v17.2d, v1.2d, v8.2d[0] + fmla v18.2d, v0.2d, v8.2d[1] + fmla v19.2d, v1.2d, v8.2d[1] + + fmla v20.2d, v0.2d, v9.2d[0] + fmla v21.2d, v1.2d, v9.2d[0] + fmla v22.2d, v0.2d, v9.2d[1] + fmla v23.2d, v1.2d, v9.2d[1] + + fmla v24.2d, v0.2d, v10.2d[0] + fmla v25.2d, v1.2d, v10.2d[0] + fmla v26.2d, v0.2d, v10.2d[1] + fmla v27.2d, v1.2d, v10.2d[1] + + fmla v28.2d, v0.2d, v11.2d[0] + fmla v29.2d, v1.2d, v11.2d[0] + fmla v30.2d, v0.2d, v11.2d[1] + fmla v31.2d, v1.2d, v11.2d[1] + + ld1 {v12.2d, v13.2d}, [pB] // For next round + add pB, pB, #32 + ld1 {v4.2d, v5.2d}, [pA] // For next round + add pA, pA, #32 + ld1 {v14.2d, v15.2d}, [pB] + add pB, pB, #32 + + prfm PLDL1KEEP, [pA, #512] +.endm + +.macro KERNEL4x8_M2 + fmla v16.2d, v4.2d, v12.2d[0] + fmla v17.2d, v5.2d, v12.2d[0] + fmla v18.2d, v4.2d, v12.2d[1] + fmla v19.2d, v5.2d, v12.2d[1] + + fmla v20.2d, v4.2d, v13.2d[0] + fmla v21.2d, v5.2d, v13.2d[0] + fmla v22.2d, v4.2d, v13.2d[1] + fmla v23.2d, v5.2d, v13.2d[1] + + fmla v24.2d, v4.2d, v14.2d[0] + fmla v25.2d, v5.2d, v14.2d[0] + fmla v26.2d, v4.2d, v14.2d[1] + fmla v27.2d, v5.2d, v14.2d[1] + + fmla v28.2d, v4.2d, v15.2d[0] + fmla v29.2d, v5.2d, v15.2d[0] + fmla v30.2d, v4.2d, v15.2d[1] + fmla v31.2d, v5.2d, v15.2d[1] + + ld1 {v8.2d, v9.2d}, [pB] // For next round + add pB, pB, #32 + ld1 {v0.2d, v1.2d}, [pA] // For next round + add pA, pA, #32 + ld1 {v10.2d, v11.2d}, [pB] + add pB, pB, #32 + + prfm PLDL1KEEP, [pB, #512] +.endm + +.macro KERNEL4x8_E + fmla v16.2d, v4.2d, v12.2d[0] + fmla v17.2d, v5.2d, v12.2d[0] + fmla v18.2d, v4.2d, v12.2d[1] + fmla v19.2d, v5.2d, v12.2d[1] + + fmla v20.2d, v4.2d, v13.2d[0] + fmla v21.2d, v5.2d, v13.2d[0] + fmla v22.2d, v4.2d, v13.2d[1] + fmla v23.2d, v5.2d, v13.2d[1] + + fmla v24.2d, v4.2d, v14.2d[0] + fmla v25.2d, v5.2d, v14.2d[0] + fmla v26.2d, v4.2d, v14.2d[1] + fmla v27.2d, v5.2d, v14.2d[1] + + fmla v28.2d, v4.2d, v15.2d[0] + fmla v29.2d, v5.2d, v15.2d[0] + fmla v30.2d, v4.2d, v15.2d[1] + fmla v31.2d, v5.2d, v15.2d[1] +.endm + +.macro KERNEL4x8_SUB + ld1 {v8.2d, v9.2d}, [pB] // For next round + add pB, pB, #32 + ld1 {v0.2d, v1.2d}, [pA] // For next round + add pA, pA, #32 + ld1 {v10.2d, v11.2d}, [pB] + add pB, pB, #32 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v17.2d, v1.2d, v8.2d[0] + fmla v18.2d, v0.2d, v8.2d[1] + fmla v19.2d, v1.2d, v8.2d[1] + + fmla v20.2d, v0.2d, v9.2d[0] + fmla v21.2d, v1.2d, v9.2d[0] + fmla v22.2d, v0.2d, v9.2d[1] + fmla v23.2d, v1.2d, v9.2d[1] + + fmla v24.2d, v0.2d, v10.2d[0] + fmla v25.2d, v1.2d, v10.2d[0] + fmla v26.2d, v0.2d, v10.2d[1] + fmla v27.2d, v1.2d, v10.2d[1] + + fmla v28.2d, v0.2d, v11.2d[0] + fmla v29.2d, v1.2d, v11.2d[0] + fmla v30.2d, v0.2d, v11.2d[1] + fmla v31.2d, v1.2d, v11.2d[1] +.endm + +.macro SAVE4x8 + add pCRow1, pCRow0, LDC + + ld1 {v8.2d, v9.2d}, [pCRow0] + fmla v8.2d, v16.2d, alphaV0 + fmla v9.2d, v17.2d, alphaV1 + st1 {v8.2d, v9.2d}, [pCRow0] + + add pCRow2, pCRow1, LDC + + ld1 {v10.2d, v11.2d}, [pCRow1] + fmla v10.2d, v18.2d, alphaV2 + fmla v11.2d, v19.2d, alphaV3 + st1 {v10.2d, v11.2d}, [pCRow1] + + add pCRow1, pCRow2, LDC + + ld1 {v12.2d, v13.2d}, [pCRow2] + fmla v12.2d, v20.2d, alphaV0 + fmla v13.2d, v21.2d, alphaV1 + st1 {v12.2d, v13.2d}, [pCRow2] + + add pCRow2, pCRow1, LDC + + ld1 {v14.2d, v15.2d}, [pCRow1] + fmla v14.2d, v22.2d, alphaV2 + fmla v15.2d, v23.2d, alphaV3 + st1 {v14.2d, v15.2d}, [pCRow1] + + add pCRow1, pCRow2, LDC + + ld1 {v8.2d, v9.2d}, [pCRow2] + fmla v8.2d, v24.2d, alphaV0 + fmla v9.2d, v25.2d, alphaV1 + st1 {v8.2d, v9.2d}, [pCRow2] + + add pCRow2, pCRow1, LDC + + ld1 {v10.2d, v11.2d}, [pCRow1] + fmla v10.2d, v26.2d, alphaV2 + fmla v11.2d, v27.2d, alphaV3 + st1 {v10.2d, v11.2d}, [pCRow1] + + add pCRow1, pCRow2, LDC + + ld1 {v12.2d, v13.2d}, [pCRow2] + fmla v12.2d, v28.2d, alphaV0 + fmla v13.2d, v29.2d, alphaV1 + st1 {v12.2d, v13.2d}, [pCRow2] + + ld1 {v14.2d, v15.2d}, [pCRow1] + fmla v14.2d, v30.2d, alphaV2 + fmla v15.2d, v31.2d, alphaV3 + st1 {v14.2d, v15.2d}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT2x8 + fmov d16, xzr + fmov d18, xzr + fmov d20, xzr + fmov d22, d16 + fmov d24, xzr + fmov d26, d16 + fmov d28, xzr + fmov d30, d16 +.endm + +.macro KERNEL2x8_SUB + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + ld1 {v0.2d}, [pA] + add pA, pA, #16 + ld1 {v10.2d, v11.2d}, [pB] + add pB, pB, #32 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v18.2d, v0.2d, v8.2d[1] + + fmla v20.2d, v0.2d, v9.2d[0] + fmla v22.2d, v0.2d, v9.2d[1] + + fmla v24.2d, v0.2d, v10.2d[0] + fmla v26.2d, v0.2d, v10.2d[1] + + fmla v28.2d, v0.2d, v11.2d[0] + fmla v30.2d, v0.2d, v11.2d[1] +.endm + +.macro SAVE2x8 + add pCRow1, pCRow0, LDC + + ld1 {v8.2d}, [pCRow0] + fmla v8.2d, v16.2d, alphaV0 + st1 {v8.2d}, [pCRow0] + + add pCRow2, pCRow1, LDC + + ld1 {v10.2d}, [pCRow1] + fmla v10.2d, v18.2d, alphaV2 + st1 {v10.2d}, [pCRow1] + + add pCRow1, pCRow2, LDC + + ld1 {v12.2d}, [pCRow2] + fmla v12.2d, v20.2d, alphaV0 + st1 {v12.2d}, [pCRow2] + + add pCRow2, pCRow1, LDC + + ld1 {v14.2d}, [pCRow1] + fmla v14.2d, v22.2d, alphaV2 + st1 {v14.2d}, [pCRow1] + + add pCRow1, pCRow2, LDC + + ld1 {v8.2d}, [pCRow2] + fmla v8.2d, v24.2d, alphaV0 + st1 {v8.2d}, [pCRow2] + + add pCRow2, pCRow1, LDC + + ld1 {v10.2d}, [pCRow1] + fmla v10.2d, v26.2d, alphaV2 + st1 {v10.2d}, [pCRow1] + + add pCRow1, pCRow2, LDC + + ld1 {v12.2d}, [pCRow2] + fmla v12.2d, v28.2d, alphaV0 + st1 {v12.2d}, [pCRow2] + + add pCRow2, pCRow1, LDC + + ld1 {v14.2d}, [pCRow1] + fmla v14.2d, v30.2d, alphaV2 + st1 {v14.2d}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT1x8 + fmov d16, xzr + fmov d20, xzr + fmov d24, xzr + fmov d28, xzr +.endm + +.macro KERNEL1x8_SUB + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + ldr d0, [pA] + add pA, pA, #8 + ld1 {v10.2d, v11.2d}, [pB] + add pB, pB, #32 + + fmla v16.2d, v8.2d, v0.d[0] + fmla v20.2d, v9.2d, v0.d[0] + fmla v24.2d, v10.2d, v0.d[0] + fmla v28.2d, v11.2d, v0.d[0] +.endm + +.macro SAVE1x8 + add pCRow1, pCRow0, LDC + + ld1 {v8.d}[0], [pCRow0] + ld1 {v8.d}[1], [pCRow1] + fmla v8.2d, v16.2d, alphaV0 + st1 {v8.d}[0], [pCRow0] + st1 {v8.d}[1], [pCRow1] + + add pCRow2, pCRow1, LDC + add pCRow1, pCRow2, LDC + + ld1 {v10.d}[0], [pCRow2] + ld1 {v10.d}[1], [pCRow1] + fmla v10.2d, v20.2d, alphaV1 + st1 {v10.d}[0], [pCRow2] + st1 {v10.d}[1], [pCRow1] + + add pCRow2, pCRow1, LDC + add pCRow1, pCRow2, LDC + + ld1 {v12.d}[0], [pCRow2] + ld1 {v12.d}[1], [pCRow1] + fmla v12.2d, v24.2d, alphaV2 + st1 {v12.d}[0], [pCRow2] + st1 {v12.d}[1], [pCRow1] + + add pCRow2, pCRow1, LDC + add pCRow1, pCRow2, LDC + + ld1 {v14.d}[0], [pCRow2] + ld1 {v14.d}[1], [pCRow1] + fmla v14.2d, v28.2d, alphaV3 + st1 {v14.d}[0], [pCRow2] + st1 {v14.d}[1], [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT4x4 + fmov d16, xzr + fmov d17, d16 + fmov d20, d17 + fmov d21, d16 + fmov d24, d17 + fmov d25, d16 + fmov d28, d17 + fmov d29, d16 +.endm + +.macro KERNEL4x4_I + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + ld1 {v0.2d, v1.2d}, [pA] + add pA, pA, #32 + + fmul v16.2d, v0.2d, v8.2d[0] + fmul v29.2d, v1.2d, v9.2d[1] + + fmul v20.2d, v0.2d, v8.2d[1] + fmul v25.2d, v1.2d, v9.2d[0] + + fmul v24.2d, v0.2d, v9.2d[0] + fmul v21.2d, v1.2d, v8.2d[1] + + fmul v28.2d, v0.2d, v9.2d[1] + fmul v17.2d, v1.2d, v8.2d[0] + + ld1 {v12.2d, v13.2d}, [pB] + add pB, pB, #32 + ld1 {v4.2d, v5.2d}, [pA] + add pA, pA, #32 +.endm + +.macro KERNEL4x4_M1 + fmla v16.2d, v0.2d, v8.2d[0] + fmla v29.2d, v1.2d, v9.2d[1] + + ld1 {v12.2d, v13.2d}, [pB] // For next round + add pB, pB, #32 + + fmla v20.2d, v0.2d, v8.2d[1] + fmla v25.2d, v1.2d, v9.2d[0] + + ld1 {v4.2d, v5.2d}, [pA] // For next round + add pA, pA, #32 + + fmla v24.2d, v0.2d, v9.2d[0] + fmla v21.2d, v1.2d, v8.2d[1] + + prfm PLDL1KEEP, [pA, #512] + + fmla v28.2d, v0.2d, v9.2d[1] + fmla v17.2d, v1.2d, v8.2d[0] +.endm + +.macro KERNEL4x4_M2 + fmla v16.2d, v4.2d, v12.2d[0] + fmla v29.2d, v5.2d, v13.2d[1] + + ld1 {v8.2d, v9.2d}, [pB] // For next round + add pB, pB, #32 + + fmla v20.2d, v4.2d, v12.2d[1] + fmla v25.2d, v5.2d, v13.2d[0] + + ld1 {v0.2d, v1.2d}, [pA] // For next round + add pA, pA, #32 + + fmla v24.2d, v4.2d, v13.2d[0] + fmla v21.2d, v5.2d, v12.2d[1] + + prfm PLDL1KEEP, [pB, #512] + + fmla v28.2d, v4.2d, v13.2d[1] + fmla v17.2d, v5.2d, v12.2d[0] +.endm + +.macro KERNEL4x4_E + fmla v16.2d, v4.2d, v12.2d[0] + fmla v29.2d, v5.2d, v13.2d[1] + + fmla v20.2d, v4.2d, v12.2d[1] + fmla v25.2d, v5.2d, v13.2d[0] + + fmla v24.2d, v4.2d, v13.2d[0] + fmla v21.2d, v5.2d, v12.2d[1] + + fmla v28.2d, v4.2d, v13.2d[1] + fmla v17.2d, v5.2d, v12.2d[0] +.endm + +.macro KERNEL4x4_SUB + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + ld1 {v0.2d, v1.2d}, [pA] + add pA, pA, #32 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v29.2d, v1.2d, v9.2d[1] + + fmla v20.2d, v0.2d, v8.2d[1] + fmla v25.2d, v1.2d, v9.2d[0] + + fmla v24.2d, v0.2d, v9.2d[0] + fmla v21.2d, v1.2d, v8.2d[1] + + fmla v28.2d, v0.2d, v9.2d[1] + fmla v17.2d, v1.2d, v8.2d[0] +.endm + +.macro SAVE4x4 + ld1 {v8.2d, v9.2d}, [pCRow0] + fmla v8.2d, v16.2d, alphaV0 + fmla v9.2d, v17.2d, alphaV1 + st1 {v8.2d, v9.2d}, [pCRow0] + + add pCRow1, pCRow0, LDC + + ld1 {v12.2d, v13.2d}, [pCRow1] + fmla v12.2d, v20.2d, alphaV2 + fmla v13.2d, v21.2d, alphaV3 + st1 {v12.2d, v13.2d}, [pCRow1] + + add pCRow2, pCRow1, LDC + + ld1 {v8.2d, v9.2d}, [pCRow2] + fmla v8.2d, v24.2d, alphaV0 + fmla v9.2d, v25.2d, alphaV1 + st1 {v8.2d, v9.2d}, [pCRow2] + + add pCRow1, pCRow2, LDC + + ld1 {v12.2d, v13.2d}, [pCRow1] + fmla v12.2d, v28.2d, alphaV2 + fmla v13.2d, v29.2d, alphaV3 + st1 {v12.2d, v13.2d}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT2x4 + fmov d16, xzr + fmov d20, d16 + fmov d24, d20 + fmov d28, d16 +.endm + +.macro KERNEL2x4_SUB + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + ld1 {v0.2d}, [pA] + add pA, pA, #16 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v20.2d, v0.2d, v8.2d[1] + fmla v24.2d, v0.2d, v9.2d[0] + fmla v28.2d, v0.2d, v9.2d[1] +.endm + +.macro SAVE2x4 + ld1 {v8.2d}, [pCRow0] + fmla v8.2d, v16.2d, alphaV0 + st1 {v8.2d}, [pCRow0] + + add pCRow1, pCRow0, LDC + + ld1 {v12.2d}, [pCRow1] + fmla v12.2d, v20.2d, alphaV1 + st1 {v12.2d}, [pCRow1] + + add pCRow2, pCRow1, LDC + + ld1 {v8.2d}, [pCRow2] + fmla v8.2d, v24.2d, alphaV2 + st1 {v8.2d}, [pCRow2] + + add pCRow1, pCRow2, LDC + + ld1 {v12.2d}, [pCRow1] + fmla v12.2d, v28.2d, alphaV3 + st1 {v12.2d}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT1x4 + fmov d16, xzr + fmov d20, d16 +.endm + +.macro KERNEL1x4_SUB + ldr d0, [pA] + add pA, pA, #8 + + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + + fmla v16.2d, v8.2d, v0.d[0] + fmla v20.2d, v9.2d, v0.d[0] +.endm + +.macro SAVE1x4 + add pCRow1, pCRow0, LDC + + ld1 {v8.d}[0], [pCRow0] + ld1 {v8.d}[1], [pCRow1] + fmla v8.2d, v16.2d, alphaV0 + st1 {v8.d}[0], [pCRow0] + st1 {v8.d}[1], [pCRow1] + + add pCRow2, pCRow1, LDC + add pCRow1, pCRow2, LDC + + ld1 {v12.d}[0], [pCRow2] + ld1 {v12.d}[1], [pCRow1] + fmla v12.2d, v20.2d, alphaV1 + st1 {v12.d}[0], [pCRow2] + st1 {v12.d}[1], [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT4x2 + fmov d16, xzr + fmov d17, d16 + fmov d20, d17 + fmov d21, d16 +.endm + +.macro KERNEL4x2_SUB + ld1 {v8.2d}, [pB] + add pB, pB, #16 + ld1 {v0.2d, v1.2d}, [pA] + add pA, pA, #32 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v17.2d, v1.2d, v8.2d[0] + fmla v20.2d, v0.2d, v8.2d[1] + fmla v21.2d, v1.2d, v8.2d[1] +.endm + +.macro SAVE4x2 + ld1 {v8.2d, v9.2d}, [pCRow0] + fmla v8.2d, v16.2d, alphaV0 + fmla v9.2d, v17.2d, alphaV1 + st1 {v8.2d, v9.2d}, [pCRow0] + + add pCRow1, pCRow0, LDC + + ld1 {v12.2d, v13.2d}, [pCRow1] + fmla v12.2d, v20.2d, alphaV2 + fmla v13.2d, v21.2d, alphaV3 + st1 {v12.2d, v13.2d}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT2x2 + fmov d16, xzr + fmov d20, d16 +.endm + +.macro KERNEL2x2_SUB + ld1 {v8.2d}, [pB] + add pB, pB, #16 + + ld1 {v0.2d}, [pA] + add pA, pA, #16 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v20.2d, v0.2d, v8.2d[1] +.endm + +.macro SAVE2x2 + ld1 {v8.2d}, [pCRow0] + fmla v8.2d, v16.2d, alphaV0 + st1 {v8.2d}, [pCRow0] + + add pCRow1 , pCRow0, LDC + + ld1 {v12.2d}, [pCRow1] + fmla v12.2d, v20.2d, alphaV1 + st1 {v12.2d}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT1x2 + fmov d16, xzr +.endm + +.macro KERNEL1x2_SUB + ld1 {v8.2d} , [pB] + add pB , pB, #16 + + ldr d0 , [pA] + add pA, pA, #8 + + fmla v16.2d, v8.2d, v0.2d[0] +.endm + +.macro SAVE1x2 + add pCRow1 , pCRow0, LDC + + ld1 {v8.d}[0], [pCRow0] + ld1 {v8.d}[1], [pCRow1] + fmla v8.2d, v16.2d, alphaV0 + st1 {v8.d}[0], [pCRow0] + st1 {v8.d}[1], [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT4x1 + fmov d16, xzr + fmov d17, d16 +.endm + +.macro KERNEL4x1_SUB + ldr d8, [pB] + add pB , pB, #8 + + ld1 {v0.2d, v1.2d}, [pA] + add pA , pA, #32 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v17.2d, v1.2d, v8.2d[0] +.endm + +.macro SAVE4x1 + ld1 {v8.2d, v9.2d}, [pCRow0] + fmla v8.2d, v16.2d, alphaV0 + fmla v9.2d, v17.2d, alphaV1 + st1 {v8.2d, v9.2d}, [pCRow0] + + add pCRow0, pCRow0, #32 +.endm + + + + +/******************************************************************************/ + +.macro INIT2x1 + fmov d16, xzr +.endm + +.macro KERNEL2x1_SUB + ldr d8, [pB] + add pB , pB, #8 + + ld1 {v0.2d}, [pA] + add pA , pA, #16 + + fmla v16.2d, v0.2d, v8.2d[0] +.endm + +.macro SAVE2x1 + ld1 {v8.2d}, [pCRow0] + fmla v8.2d, v16.2d, alphaV0 + st1 {v8.2d}, [pCRow0] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT1x1 + fmov d16, xzr +.endm + +.macro KERNEL1x1_SUB + ldr d8, [pB] + add pB , pB, #8 + + ldr d0, [pA] + add pA , pA, #8 + + fmadd d16, d0, d8, d16 +.endm + +.macro SAVE1x1 + ldr d8, [pCRow0] + fmadd d8, d16, alpha0, d8 + str d8, [pCRow0] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************* +* End of macro definitions +*******************************************************************************/ + + PROLOGUE + + .align 5 + add sp, sp, #-(11 * 16) + stp d8, d9, [sp, #(0 * 16)] + stp d10, d11, [sp, #(1 * 16)] + stp d12, d13, [sp, #(2 * 16)] + stp d14, d15, [sp, #(3 * 16)] + stp d16, d17, [sp, #(4 * 16)] + stp x18, x19, [sp, #(5 * 16)] + stp x20, x21, [sp, #(6 * 16)] + stp x22, x23, [sp, #(7 * 16)] + stp x24, x25, [sp, #(8 * 16)] + stp x26, x27, [sp, #(9 * 16)] + str x28, [sp, #(10 * 16)] + + fmov alpha0, d0 + fmov alpha1, d0 + fmov alpha2, d0 + fmov alpha3, d0 + + lsl LDC, LDC, #3 // ldc = ldc * 8 + + mov pB, origPB + + mov counterJ, origN + asr counterJ, counterJ, #3 // J = J / 8 + cmp counterJ, #0 + ble dgemm_kernel_L4_BEGIN + +/******************************************************************************/ + +dgemm_kernel_L8_BEGIN: + + mov pCRow0, pC // pCRow0 = C + add pC, pC, LDC, lsl #3 + + mov pA, origPA // pA = start of A array + +dgemm_kernel_L8_M4_BEGIN: + + mov counterI, origM + asr counterI, counterI, #2 // counterI = counterI / 4 + cmp counterI, #0 + ble dgemm_kernel_L8_M2_BEGIN + +dgemm_kernel_L8_M4_20: + + mov pB, origPB + + asr counterL , origK, #1 // L = K / 2 + cmp counterL , #2 // is there at least 4 to do? + blt dgemm_kernel_L8_M4_32 + + KERNEL4x8_I // do one in the K + KERNEL4x8_M2 // do another in the K + + subs counterL, counterL, #2 + ble dgemm_kernel_L8_M4_22a + .align 5 + +dgemm_kernel_L8_M4_22: + + KERNEL4x8_M1 + KERNEL4x8_M2 + + subs counterL, counterL, #1 + bgt dgemm_kernel_L8_M4_22 + + +dgemm_kernel_L8_M4_22a: + + KERNEL4x8_M1 + KERNEL4x8_E + + b dgemm_kernel_L8_M4_44 + +dgemm_kernel_L8_M4_32: + + tst counterL, #1 + ble dgemm_kernel_L8_M4_40 + + KERNEL4x8_I + + KERNEL4x8_E + + b dgemm_kernel_L8_M4_44 + + +dgemm_kernel_L8_M4_40: + + INIT4x8 + +dgemm_kernel_L8_M4_44: + + ands counterL , origK, #1 + ble dgemm_kernel_L8_M4_100 + +dgemm_kernel_L8_M4_46: + + KERNEL4x8_SUB + +dgemm_kernel_L8_M4_100: + + SAVE4x8 + +dgemm_kernel_L8_M4_END: + subs counterI, counterI, #1 + bne dgemm_kernel_L8_M4_20 + +dgemm_kernel_L8_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble dgemm_kernel_L8_END + + tst counterI, #2 // counterI = counterI / 2 + ble dgemm_kernel_L8_M1_BEGIN + +dgemm_kernel_L8_M2_20: + + INIT2x8 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dgemm_kernel_L8_M2_40 + +dgemm_kernel_L8_M2_22: + + KERNEL2x8_SUB + KERNEL2x8_SUB + KERNEL2x8_SUB + KERNEL2x8_SUB + + KERNEL2x8_SUB + KERNEL2x8_SUB + KERNEL2x8_SUB + KERNEL2x8_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L8_M2_22 + + +dgemm_kernel_L8_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L8_M2_100 + +dgemm_kernel_L8_M2_42: + + KERNEL2x8_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L8_M2_42 + +dgemm_kernel_L8_M2_100: + + SAVE2x8 + +dgemm_kernel_L8_M2_END: + + +dgemm_kernel_L8_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble dgemm_kernel_L8_END + +dgemm_kernel_L8_M1_20: + + INIT1x8 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dgemm_kernel_L8_M1_40 + +dgemm_kernel_L8_M1_22: + KERNEL1x8_SUB + KERNEL1x8_SUB + KERNEL1x8_SUB + KERNEL1x8_SUB + + KERNEL1x8_SUB + KERNEL1x8_SUB + KERNEL1x8_SUB + KERNEL1x8_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L8_M1_22 + + +dgemm_kernel_L8_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L8_M1_100 + +dgemm_kernel_L8_M1_42: + + KERNEL1x8_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L8_M1_42 + +dgemm_kernel_L8_M1_100: + + SAVE1x8 + +dgemm_kernel_L8_END: + + lsl temp, origK, #6 + add origPB, origPB, temp // B = B + K * 8 * 8 + + subs counterJ, counterJ , #1 // j-- + bgt dgemm_kernel_L8_BEGIN + + +/******************************************************************************/ + +dgemm_kernel_L4_BEGIN: + + mov counterJ , origN + tst counterJ , #7 + ble dgemm_kernel_L999 + + tst counterJ , #4 + ble dgemm_kernel_L2_BEGIN + + mov pCRow0, pC // pCRow0 = C + add pC, pC, LDC, lsl #2 + + mov pA, origPA // pA = start of A array + +dgemm_kernel_L4_M4_BEGIN: + + mov counterI, origM + asr counterI, counterI, #2 // counterI = counterI / 4 + cmp counterI, #0 + ble dgemm_kernel_L4_M2_BEGIN + +dgemm_kernel_L4_M4_20: + + mov pB, origPB + + asr counterL , origK, #1 // L = K / 2 + cmp counterL , #2 // is there at least 4 to do? + blt dgemm_kernel_L4_M4_32 + + KERNEL4x4_I // do one in the K + KERNEL4x4_M2 // do another in the K + + subs counterL, counterL, #2 + ble dgemm_kernel_L4_M4_22a + .align 5 + +dgemm_kernel_L4_M4_22: + + KERNEL4x4_M1 + KERNEL4x4_M2 + + subs counterL, counterL, #1 + bgt dgemm_kernel_L4_M4_22 + + +dgemm_kernel_L4_M4_22a: + + KERNEL4x4_M1 + KERNEL4x4_E + + b dgemm_kernel_L4_M4_44 + +dgemm_kernel_L4_M4_32: + + tst counterL, #1 + ble dgemm_kernel_L4_M4_40 + + KERNEL4x4_I + + KERNEL4x4_E + + b dgemm_kernel_L4_M4_44 + + +dgemm_kernel_L4_M4_40: + + INIT4x4 + +dgemm_kernel_L4_M4_44: + + ands counterL , origK, #1 + ble dgemm_kernel_L4_M4_100 + +dgemm_kernel_L4_M4_46: + + KERNEL4x4_SUB + +dgemm_kernel_L4_M4_100: + + SAVE4x4 + +dgemm_kernel_L4_M4_END: + subs counterI, counterI, #1 + bne dgemm_kernel_L4_M4_20 + +dgemm_kernel_L4_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble dgemm_kernel_L4_END + + tst counterI, #2 // counterI = counterI / 2 + ble dgemm_kernel_L4_M1_BEGIN + +dgemm_kernel_L4_M2_20: + + INIT2x4 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dgemm_kernel_L4_M2_40 + +dgemm_kernel_L4_M2_22: + + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L4_M2_22 + + +dgemm_kernel_L4_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L4_M2_100 + +dgemm_kernel_L4_M2_42: + + KERNEL2x4_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L4_M2_42 + +dgemm_kernel_L4_M2_100: + + SAVE2x4 + +dgemm_kernel_L4_M2_END: + + +dgemm_kernel_L4_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble dgemm_kernel_L4_END + +dgemm_kernel_L4_M1_20: + + INIT1x4 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dgemm_kernel_L4_M1_40 + +dgemm_kernel_L4_M1_22: + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L4_M1_22 + + +dgemm_kernel_L4_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L4_M1_100 + +dgemm_kernel_L4_M1_42: + + KERNEL1x4_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L4_M1_42 + +dgemm_kernel_L4_M1_100: + + SAVE1x4 + +dgemm_kernel_L4_END: + + lsl temp, origK, #5 + add origPB, origPB, temp // B = B + K * 4 * 8 + +/******************************************************************************/ + +dgemm_kernel_L2_BEGIN: // less than 2 left in N direction + + mov counterJ , origN + tst counterJ , #3 + ble dgemm_kernel_L999 // error, N was less than 4? + + tst counterJ , #2 + ble dgemm_kernel_L1_BEGIN + + mov pCRow0, pC // pCRow0 = pC + + add pC,pC,LDC, lsl #1 + + mov pA, origPA // pA = A + + +dgemm_kernel_L2_M4_BEGIN: + + mov counterI, origM + asr counterI, counterI, #2 // counterI = counterI / 4 + cmp counterI,#0 + ble dgemm_kernel_L2_M2_BEGIN + +dgemm_kernel_L2_M4_20: + + INIT4x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble dgemm_kernel_L2_M4_40 + .align 5 + +dgemm_kernel_L2_M4_22: + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L2_M4_22 + + +dgemm_kernel_L2_M4_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L2_M4_100 + +dgemm_kernel_L2_M4_42: + + KERNEL4x2_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L2_M4_42 + +dgemm_kernel_L2_M4_100: + + SAVE4x2 + +dgemm_kernel_L2_M4_END: + + subs counterI, counterI, #1 + bgt dgemm_kernel_L2_M4_20 + + +dgemm_kernel_L2_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble dgemm_kernel_L2_END + + tst counterI, #2 // counterI = counterI / 2 + ble dgemm_kernel_L2_M1_BEGIN + +dgemm_kernel_L2_M2_20: + + INIT2x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble dgemm_kernel_L2_M2_40 + +dgemm_kernel_L2_M2_22: + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L2_M2_22 + + +dgemm_kernel_L2_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L2_M2_100 + +dgemm_kernel_L2_M2_42: + + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L2_M2_42 + +dgemm_kernel_L2_M2_100: + + SAVE2x2 + +dgemm_kernel_L2_M2_END: + + +dgemm_kernel_L2_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble dgemm_kernel_L2_END + +dgemm_kernel_L2_M1_20: + + INIT1x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL, #0 + ble dgemm_kernel_L2_M1_40 + +dgemm_kernel_L2_M1_22: + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L2_M1_22 + + +dgemm_kernel_L2_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L2_M1_100 + +dgemm_kernel_L2_M1_42: + + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L2_M1_42 + +dgemm_kernel_L2_M1_100: + + SAVE1x2 + +dgemm_kernel_L2_END: + add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 + +/******************************************************************************/ + +dgemm_kernel_L1_BEGIN: + + mov counterJ , origN + tst counterJ , #1 + ble dgemm_kernel_L999 // done + + + mov pCRow0, pC // pCRow0 = C + add pC , pC , LDC // Update pC to point to next + + mov pA, origPA // pA = A + +dgemm_kernel_L1_M4_BEGIN: + + mov counterI, origM + asr counterI, counterI, #2 // counterI = counterI / 4 + cmp counterI, #0 + ble dgemm_kernel_L1_M2_BEGIN + +dgemm_kernel_L1_M4_20: + + INIT4x1 + + mov pB, origPB + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dgemm_kernel_L1_M4_40 + .align 5 + +dgemm_kernel_L1_M4_22: + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L1_M4_22 + + +dgemm_kernel_L1_M4_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L1_M4_100 + +dgemm_kernel_L1_M4_42: + + KERNEL4x1_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L1_M4_42 + +dgemm_kernel_L1_M4_100: + + SAVE4x1 + +dgemm_kernel_L1_M4_END: + + subs counterI, counterI, #1 + bgt dgemm_kernel_L1_M4_20 + + +dgemm_kernel_L1_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble dgemm_kernel_L1_END + + tst counterI, #2 // counterI = counterI / 2 + ble dgemm_kernel_L1_M1_BEGIN + +dgemm_kernel_L1_M2_20: + + INIT2x1 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dgemm_kernel_L1_M2_40 + +dgemm_kernel_L1_M2_22: + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L1_M2_22 + + +dgemm_kernel_L1_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L1_M2_100 + +dgemm_kernel_L1_M2_42: + + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L1_M2_42 + +dgemm_kernel_L1_M2_100: + + SAVE2x1 + +dgemm_kernel_L1_M2_END: + + +dgemm_kernel_L1_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble dgemm_kernel_L1_END + +dgemm_kernel_L1_M1_20: + + INIT1x1 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dgemm_kernel_L1_M1_40 + +dgemm_kernel_L1_M1_22: + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L1_M1_22 + + +dgemm_kernel_L1_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L1_M1_100 + +dgemm_kernel_L1_M1_42: + + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L1_M1_42 + +dgemm_kernel_L1_M1_100: + + SAVE1x1 + + +dgemm_kernel_L1_END: + + +dgemm_kernel_L999: + mov x0, #0 // set return value + ldp d8, d9, [sp, #(0 * 16)] + ldp d10, d11, [sp, #(1 * 16)] + ldp d12, d13, [sp, #(2 * 16)] + ldp d14, d15, [sp, #(3 * 16)] + ldp d16, d17, [sp, #(4 * 16)] + ldp x18, x19, [sp, #(5 * 16)] + ldp x20, x21, [sp, #(6 * 16)] + ldp x22, x23, [sp, #(7 * 16)] + ldp x24, x25, [sp, #(8 * 16)] + ldp x26, x27, [sp, #(9 * 16)] + ldr x28, [sp, #(10 * 16)] + add sp, sp, #(11*16) + ret + + EPILOGUE + diff --git a/kernel/arm64/dgemm_kernel_8x4.S b/kernel/arm64/dgemm_kernel_8x4.S new file mode 100755 index 000000000..a607fecc4 --- /dev/null +++ b/kernel/arm64/dgemm_kernel_8x4.S @@ -0,0 +1,1570 @@ +/******************************************************************************* +Copyright (c) 2015, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#define ASSEMBLER +#include "common.h" + +/* X0 X1 X2 s0 X3 x4 x5 x6 */ +/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha0,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc )*/ + +#define origM x0 +#define origN x1 +#define origK x2 +#define origPA x3 +#define origPB x4 +#define pC x5 +#define LDC x6 +#define temp x7 +#define counterL x8 +#define counterI x9 +#define counterJ x10 +#define pB x11 +#define pCRow0 x12 +#define pCRow1 x13 +#define pCRow2 x14 +#define pCRow3 x15 +#define pA x16 +#define alpha x17 + +#define alpha0 d10 +#define alphaV0 v10.d[0] +#define alpha1 d11 +#define alphaV1 v11.d[0] +#define alpha2 d14 +#define alphaV2 v14.d[0] +#define alpha3 d15 +#define alphaV3 v15.d[0] + +// 00 origM +// 01 origN +// 02 origK +// 03 origPA +// 04 origPB +// 05 pC +// 06 origLDC -> LDC +// 07 temp +// 08 counterL +// 09 counterI +// 10 counterJ +// 11 pB +// 12 pCRow0 +// 13 pCRow1 +// 14 pCRow2 +// 15 pA +// 16 +// 17 +// 18 must save +// 19 must save +// 20 must save +// 21 must save +// 22 must save +// 23 must save +// 24 must save +// 25 must save +// 26 must save +// 27 must save +// 28 must save +// 29 frame +// 30 link +// 31 sp + +//v00 ALPHA -> pA0_0, pA0_1 +//v01 pA0_2, pA0_3 +//v02 pA0_4, pA0_5 +//v03 pA0_6, pA0_7 +//v04 pA1_0, pA1_1 +//v05 pA1_2, pA1_3 +//v06 pA1_4, pA1_5 +//v07 pA1_6, pA1_7 +//v08 must save pB0_0, pB0_1 +//v09 must save pB0_2, pB0_3 +//v10 must save ALPHA0 +//v11 must save ALPHA1 +//v12 must save pB1_0, pB1_1 +//v13 must save pB1_2, pB1_3 +//v14 must save ALPHA2 +//v15 must save ALPHA3 +//v16 must save C00, C01 +//v17 must save C02, C03 +//v18 C04, C05 +//v19 C06, C07 +//v20 C10, C11 +//v21 C12, C13 +//v22 C14, C15 +//v23 C16, C17 +//v24 C20, C21 +//v25 C22, C23 +//v26 C24, C25 +//v27 C26, C27 +//v28 C30, C31 +//v29 C32, C33 +//v30 C34, C35 +//v31 C36, C37 + +/******************************************************************************* +* Macro definitions +*******************************************************************************/ + +.macro INIT8x4 + fmov d16, xzr + fmov d17, xzr + fmov d18, d16 + fmov d19, xzr + fmov d20, xzr + fmov d21, d16 + fmov d22, d17 + fmov d23, d18 + fmov d24, xzr + fmov d25, d16 + fmov d26, d17 + fmov d27, d18 + fmov d28, xzr + fmov d29, d16 + fmov d30, d17 + fmov d31, d18 +.endm + +.macro KERNEL8x4_I + ld1 {v0.2d, v1.2d}, [pA] + add pA, pA, #32 + ld1 {v2.2d, v3.2d}, [pA] + add pA, pA, #32 + ldp d8, d9, [pB] + add pB, pB, #16 + ldp d10, d11, [pB] + add pB, pB, #16 + + fmul v16.2d, v0.2d, v8.2d[0] + fmul v17.2d, v1.2d, v8.2d[0] + + fmul v18.2d, v2.2d, v8.2d[0] + fmul v19.2d, v3.2d, v8.2d[0] + + fmul v20.2d, v0.2d, v9.2d[0] + fmul v21.2d, v1.2d, v9.2d[0] + + fmul v22.2d, v2.2d, v9.2d[0] + fmul v23.2d, v3.2d, v9.2d[0] + + fmul v24.2d, v0.2d, v10.2d[0] + fmul v25.2d, v1.2d, v10.2d[0] + + fmul v26.2d, v2.2d, v10.2d[0] + fmul v27.2d, v3.2d, v10.2d[0] + + fmul v28.2d, v0.2d, v11.2d[0] + fmul v29.2d, v1.2d, v11.2d[0] + + fmul v30.2d, v2.2d, v11.2d[0] + fmul v31.2d, v3.2d, v11.2d[0] + + ld1 {v4.2d, v5.2d}, [pA] + add pA, pA, #32 + ld1 {v6.2d, v7.2d}, [pA] + add pA, pA, #32 + ldp d12, d13, [pB] + add pB, pB, #16 + ldp d14, d15, [pB] + add pB, pB, #16 +.endm + +.macro KERNEL8x4_M1 + fmla v16.2d, v0.2d, v8.2d[0] + fmla v21.2d, v1.2d, v9.2d[0] + fmla v26.2d, v2.2d, v10.2d[0] + fmla v31.2d, v3.2d, v11.2d[0] + + ld1 {v4.2d}, [pA], #16 + + fmla v20.2d, v0.2d, v9.2d[0] + fmla v17.2d, v1.2d, v8.2d[0] + + ld1 {v5.2d}, [pA], #16 + + fmla v30.2d, v2.2d, v11.2d[0] + fmla v27.2d, v3.2d, v10.2d[0] + + ldp d12, d13, [pB] + add pB, pB, #16 + + fmla v28.2d, v0.2d, v11.2d[0] + fmla v25.2d, v1.2d, v10.2d[0] + + ldp d14, d15, [pB] + add pB, pB, #16 + + fmla v18.2d, v2.2d, v8.2d[0] + fmla v23.2d, v3.2d, v9.2d[0] + + ld1 {v6.2d}, [pA], #16 + + fmla v24.2d, v0.2d, v10.2d[0] + fmla v29.2d, v1.2d, v11.2d[0] + + ld1 {v7.2d}, [pA], #16 + + fmla v22.2d, v2.2d, v9.2d[0] + fmla v19.2d, v3.2d, v8.2d[0] + + prfm PLDL1KEEP, [pA, #224] + prfm PLDL1KEEP, [pA, #224+64] +.endm + +.macro KERNEL8x4_M2 + fmla v16.2d, v4.2d, v12.2d[0] + fmla v21.2d, v5.2d, v13.2d[0] + fmla v26.2d, v6.2d, v14.2d[0] + fmla v31.2d, v7.2d, v15.2d[0] + + ld1 {v0.2d}, [pA], #16 + + fmla v20.2d, v4.2d, v13.2d[0] + fmla v17.2d, v5.2d, v12.2d[0] + + ld1 {v1.2d}, [pA], #16 + + fmla v30.2d, v6.2d, v15.2d[0] + fmla v27.2d, v7.2d, v14.2d[0] + + ldp d8, d9, [pB] + add pB, pB, #16 + + fmla v28.2d, v4.2d, v15.2d[0] + fmla v25.2d, v5.2d, v14.2d[0] + + ldp d10, d11, [pB] + add pB, pB, #16 + + fmla v22.2d, v6.2d, v13.2d[0] + fmla v19.2d, v7.2d, v12.2d[0] + + ld1 {v2.2d}, [pA], #16 + + fmla v24.2d, v4.2d, v14.2d[0] + fmla v29.2d, v5.2d, v15.2d[0] + + ld1 {v3.2d}, [pA], #16 + + fmla v18.2d, v6.2d, v12.2d[0] + fmla v23.2d, v7.2d, v13.2d[0] + + prfm PLDL1KEEP, [pB, #640] +.endm + +.macro KERNEL8x4_E + fmla v16.2d, v4.2d, v12.2d[0] + fmla v17.2d, v5.2d, v12.2d[0] + fmla v18.2d, v6.2d, v12.2d[0] + fmla v19.2d, v7.2d, v12.2d[0] + fmla v20.2d, v4.2d, v13.2d[0] + fmla v21.2d, v5.2d, v13.2d[0] + fmla v22.2d, v6.2d, v13.2d[0] + fmla v23.2d, v7.2d, v13.2d[0] + fmla v24.2d, v4.2d, v14.2d[0] + fmla v25.2d, v5.2d, v14.2d[0] + fmla v26.2d, v6.2d, v14.2d[0] + fmla v27.2d, v7.2d, v14.2d[0] + fmla v28.2d, v4.2d, v15.2d[0] + fmla v29.2d, v5.2d, v15.2d[0] + fmla v30.2d, v6.2d, v15.2d[0] + fmla v31.2d, v7.2d, v15.2d[0] +.endm + +.macro KERNEL8x4_SUB + ld1 {v0.2d, v1.2d}, [pA] + add pA, pA, #32 + ld1 {v2.2d, v3.2d}, [pA] + add pA, pA, #32 + ldp d8, d9, [pB] + add pB, pB, #16 + ldp d10, d11, [pB] + add pB, pB, #16 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v17.2d, v1.2d, v8.2d[0] + fmla v18.2d, v2.2d, v8.2d[0] + fmla v19.2d, v3.2d, v8.2d[0] + + fmla v20.2d, v0.2d, v9.2d[0] + fmla v21.2d, v1.2d, v9.2d[0] + fmla v22.2d, v2.2d, v9.2d[0] + fmla v23.2d, v3.2d, v9.2d[0] + + fmla v24.2d, v0.2d, v10.2d[0] + fmla v25.2d, v1.2d, v10.2d[0] + fmla v26.2d, v2.2d, v10.2d[0] + fmla v27.2d, v3.2d, v10.2d[0] + + fmla v28.2d, v0.2d, v11.2d[0] + fmla v29.2d, v1.2d, v11.2d[0] + fmla v30.2d, v2.2d, v11.2d[0] + fmla v31.2d, v3.2d, v11.2d[0] +.endm + +.macro SAVE8x4 + fmov alpha0, alpha + + ld1 {v0.2d, v1.2d}, [pCRow0] + fmla v0.2d, v16.2d, alphaV0 + fmla v1.2d, v17.2d, alphaV0 + st1 {v0.2d, v1.2d}, [pCRow0] + + add pCRow0, pCRow0, #32 + + ld1 {v2.2d, v3.2d}, [pCRow0] + fmla v2.2d, v18.2d, alphaV0 + fmla v3.2d, v19.2d, alphaV0 + st1 {v2.2d, v3.2d}, [pCRow0] + + add pCRow0, pCRow0, #32 + + ld1 {v4.2d, v5.2d}, [pCRow1] + fmla v4.2d, v20.2d, alphaV0 + fmla v5.2d, v21.2d, alphaV0 + st1 {v4.2d, v5.2d}, [pCRow1] + + add pCRow1, pCRow1, #32 + + ld1 {v6.2d, v7.2d}, [pCRow1] + fmla v6.2d, v22.2d, alphaV0 + fmla v7.2d, v23.2d, alphaV0 + st1 {v6.2d, v7.2d}, [pCRow1] + + add pCRow1, pCRow1, #32 + + ld1 {v0.2d, v1.2d}, [pCRow2] + fmla v0.2d, v24.2d, alphaV0 + fmla v1.2d, v25.2d, alphaV0 + st1 {v0.2d, v1.2d}, [pCRow2] + + add pCRow2, pCRow2, #32 + ld1 {v2.2d, v3.2d}, [pCRow2] + fmla v2.2d, v26.2d, alphaV0 + fmla v3.2d, v27.2d, alphaV0 + st1 {v2.2d, v3.2d}, [pCRow2] + + add pCRow2, pCRow2, #32 + + ld1 {v4.2d, v5.2d}, [pCRow3] + fmla v4.2d, v28.2d, alphaV0 + fmla v5.2d, v29.2d, alphaV0 + st1 {v4.2d, v5.2d}, [pCRow3] + + add pCRow3, pCRow3, #32 + + ld1 {v6.2d, v7.2d}, [pCRow3] + fmla v6.2d, v30.2d, alphaV0 + fmla v7.2d, v31.2d, alphaV0 + st1 {v6.2d, v7.2d}, [pCRow3] + + add pCRow3, pCRow3, #32 + + prfm PLDL2KEEP, [pCRow0, #128] + prfm PLDL2KEEP, [pCRow1, #128] + prfm PLDL2KEEP, [pCRow2, #128] + prfm PLDL2KEEP, [pCRow3, #128] +.endm + +/******************************************************************************/ + +.macro INIT4x4 + fmov d16, xzr + fmov d17, d16 + fmov d20, d17 + fmov d21, d16 + fmov d24, d17 + fmov d25, d16 + fmov d28, d17 + fmov d29, d16 +.endm + +.macro KERNEL4x4_SUB + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + ld1 {v0.2d, v1.2d}, [pA] + add pA, pA, #32 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v29.2d, v1.2d, v9.2d[1] + + fmla v20.2d, v0.2d, v8.2d[1] + fmla v25.2d, v1.2d, v9.2d[0] + + fmla v24.2d, v0.2d, v9.2d[0] + fmla v21.2d, v1.2d, v8.2d[1] + + fmla v28.2d, v0.2d, v9.2d[1] + fmla v17.2d, v1.2d, v8.2d[0] +.endm + +.macro SAVE4x4 + ld1 {v8.2d, v9.2d}, [pCRow0] + fmla v8.2d, v16.2d, alphaV0 + fmla v9.2d, v17.2d, alphaV1 + st1 {v8.2d, v9.2d}, [pCRow0] + + add pCRow1, pCRow0, LDC + + ld1 {v12.2d, v13.2d}, [pCRow1] + fmla v12.2d, v20.2d, alphaV2 + fmla v13.2d, v21.2d, alphaV3 + st1 {v12.2d, v13.2d}, [pCRow1] + + add pCRow2, pCRow1, LDC + + ld1 {v8.2d, v9.2d}, [pCRow2] + fmla v8.2d, v24.2d, alphaV0 + fmla v9.2d, v25.2d, alphaV1 + st1 {v8.2d, v9.2d}, [pCRow2] + + add pCRow1, pCRow2, LDC + + ld1 {v12.2d, v13.2d}, [pCRow1] + fmla v12.2d, v28.2d, alphaV2 + fmla v13.2d, v29.2d, alphaV3 + st1 {v12.2d, v13.2d}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + + +.macro INIT2x4 + fmov d16, xzr + fmov d20, d16 + fmov d24, d20 + fmov d28, d16 +.endm + +.macro KERNEL2x4_SUB + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + ld1 {v0.2d}, [pA] + add pA, pA, #16 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v20.2d, v0.2d, v8.2d[1] + fmla v24.2d, v0.2d, v9.2d[0] + fmla v28.2d, v0.2d, v9.2d[1] +.endm + +.macro SAVE2x4 + ld1 {v8.2d}, [pCRow0] + fmla v8.2d, v16.2d, alphaV0 + st1 {v8.2d}, [pCRow0] + + add pCRow1, pCRow0, LDC + + ld1 {v12.2d}, [pCRow1] + fmla v12.2d, v20.2d, alphaV1 + st1 {v12.2d}, [pCRow1] + + add pCRow2, pCRow1, LDC + + ld1 {v8.2d}, [pCRow2] + fmla v8.2d, v24.2d, alphaV2 + st1 {v8.2d}, [pCRow2] + + add pCRow1, pCRow2, LDC + + ld1 {v12.2d}, [pCRow1] + fmla v12.2d, v28.2d, alphaV3 + st1 {v12.2d}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT1x4 + fmov d16, xzr + fmov d20, d16 +.endm + +.macro KERNEL1x4_SUB + ldr d0, [pA] + add pA, pA, #8 + + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + + fmla v16.2d, v8.2d, v0.d[0] + fmla v20.2d, v9.2d, v0.d[0] +.endm + +.macro SAVE1x4 + add pCRow1, pCRow0, LDC + + ld1 {v8.d}[0], [pCRow0] + ld1 {v8.d}[1], [pCRow1] + fmla v8.2d, v16.2d, alphaV0 + st1 {v8.d}[0], [pCRow0] + st1 {v8.d}[1], [pCRow1] + + add pCRow2, pCRow1, LDC + add pCRow1, pCRow2, LDC + + ld1 {v12.d}[0], [pCRow2] + ld1 {v12.d}[1], [pCRow1] + fmla v12.2d, v20.2d, alphaV1 + st1 {v12.d}[0], [pCRow2] + st1 {v12.d}[1], [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT8x2 + fmov d16, xzr + fmov d17, xzr + fmov d18, d16 + fmov d19, d17 + fmov d20, xzr + fmov d21, d16 + fmov d22, d17 + fmov d23, d18 +.endm + +.macro KERNEL8x2_SUB + ld1 {v0.2d, v1.2d}, [pA] + add pA, pA, #32 + ld1 {v8.2d}, [pB] + add pB, pB, #16 + ld1 {v2.2d, v3.2d}, [pA] + add pA, pA, #32 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v17.2d, v1.2d, v8.2d[0] + fmla v18.2d, v2.2d, v8.2d[0] + fmla v19.2d, v3.2d, v8.2d[0] + + fmla v20.2d, v0.2d, v8.2d[1] + fmla v21.2d, v1.2d, v8.2d[1] + fmla v22.2d, v2.2d, v8.2d[1] + fmla v23.2d, v3.2d, v8.2d[1] +.endm + +.macro SAVE8x2 + add pCRow1, pCRow0, LDC + + ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0] + fmla v0.2d, v16.2d, alphaV0 + fmla v1.2d, v17.2d, alphaV1 + fmla v2.2d, v18.2d, alphaV2 + fmla v3.2d, v19.2d, alphaV3 + st1 {v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0] + + ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [pCRow1] + fmla v4.2d, v20.2d, alphaV0 + fmla v5.2d, v21.2d, alphaV1 + fmla v6.2d, v22.2d, alphaV2 + fmla v7.2d, v23.2d, alphaV3 + st1 {v4.2d, v5.2d, v6.2d, v7.2d}, [pCRow1] + + add pCRow0, pCRow0, #64 +.endm + +/******************************************************************************/ + +.macro INIT4x2 + fmov d16, xzr + fmov d17, d16 + fmov d20, d17 + fmov d21, d16 +.endm + +.macro KERNEL4x2_SUB + ld1 {v8.2d}, [pB] + add pB, pB, #16 + ld1 {v0.2d, v1.2d}, [pA] + add pA, pA, #32 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v17.2d, v1.2d, v8.2d[0] + fmla v20.2d, v0.2d, v8.2d[1] + fmla v21.2d, v1.2d, v8.2d[1] +.endm + +.macro SAVE4x2 + ld1 {v8.2d, v9.2d}, [pCRow0] + fmla v8.2d, v16.2d, alphaV0 + fmla v9.2d, v17.2d, alphaV1 + st1 {v8.2d, v9.2d}, [pCRow0] + + add pCRow1, pCRow0, LDC + + ld1 {v12.2d, v13.2d}, [pCRow1] + fmla v12.2d, v20.2d, alphaV2 + fmla v13.2d, v21.2d, alphaV3 + st1 {v12.2d, v13.2d}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT2x2 + fmov d16, xzr + fmov d20, d16 +.endm + +.macro KERNEL2x2_SUB + ld1 {v8.2d}, [pB] + add pB, pB, #16 + + ld1 {v0.2d}, [pA] + add pA, pA, #16 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v20.2d, v0.2d, v8.2d[1] +.endm + +.macro SAVE2x2 + ld1 {v8.2d}, [pCRow0] + fmla v8.2d, v16.2d, alphaV0 + st1 {v8.2d}, [pCRow0] + + add pCRow1 , pCRow0, LDC + + ld1 {v12.2d}, [pCRow1] + fmla v12.2d, v20.2d, alphaV1 + st1 {v12.2d}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT1x2 + fmov d16, xzr +.endm + +.macro KERNEL1x2_SUB + ld1 {v8.2d} , [pB] + add pB , pB, #16 + + ldr d0 , [pA] + add pA, pA, #8 + + fmla v16.2d, v8.2d, v0.2d[0] +.endm + +.macro SAVE1x2 + add pCRow1 , pCRow0, LDC + + ld1 {v8.d}[0], [pCRow0] + ld1 {v8.d}[1], [pCRow1] + fmla v8.2d, v16.2d, alphaV0 + st1 {v8.d}[0], [pCRow0] + st1 {v8.d}[1], [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT8x1 + fmov d16, xzr + fmov d17, xzr + fmov d18, d16 + fmov d19, d17 +.endm + +.macro KERNEL8x1_SUB + ld1 {v0.2d, v1.2d}, [pA] + add pA , pA, #32 + + ldr d8, [pB] + add pB , pB, #8 + + ld1 {v2.2d, v3.2d}, [pA] + add pA, pA, #32 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v17.2d, v1.2d, v8.2d[0] + fmla v18.2d, v2.2d, v8.2d[0] + fmla v19.2d, v3.2d, v8.2d[0] +.endm + +.macro SAVE8x1 + ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0] + fmla v0.2d, v16.2d, alphaV0 + fmla v1.2d, v17.2d, alphaV1 + fmla v2.2d, v18.2d, alphaV2 + fmla v3.2d, v19.2d, alphaV3 + st1 {v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0] + + add pCRow0, pCRow0, #64 +.endm + + +/******************************************************************************/ + +.macro INIT4x1 + fmov d16, xzr + fmov d17, d16 +.endm + +.macro KERNEL4x1_SUB + ldr d8, [pB] + add pB , pB, #8 + + ld1 {v0.2d, v1.2d}, [pA] + add pA , pA, #32 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v17.2d, v1.2d, v8.2d[0] +.endm + +.macro SAVE4x1 + ld1 {v8.2d, v9.2d}, [pCRow0] + fmla v8.2d, v16.2d, alphaV0 + fmla v9.2d, v17.2d, alphaV1 + st1 {v8.2d, v9.2d}, [pCRow0] + + add pCRow0, pCRow0, #32 +.endm + + +/******************************************************************************/ + +.macro INIT2x1 + fmov d16, xzr +.endm + +.macro KERNEL2x1_SUB + ldr d8, [pB] + add pB , pB, #8 + + ld1 {v0.2d}, [pA] + add pA , pA, #16 + + fmla v16.2d, v0.2d, v8.2d[0] +.endm + +.macro SAVE2x1 + ld1 {v8.2d}, [pCRow0] + fmla v8.2d, v16.2d, alphaV0 + st1 {v8.2d}, [pCRow0] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT1x1 + fmov d16, xzr +.endm + +.macro KERNEL1x1_SUB + ldr d8, [pB] + add pB , pB, #8 + + ldr d0, [pA] + add pA , pA, #8 + + fmadd d16, d0, d8, d16 +.endm + +.macro SAVE1x1 + ldr d8, [pCRow0] + fmadd d8, d16, alpha0, d8 + str d8, [pCRow0] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************* +* End of macro definitions +*******************************************************************************/ + + PROLOGUE + + .align 5 + add sp, sp, #-(11 * 16) + stp d8, d9, [sp, #(0 * 16)] + stp d10, d11, [sp, #(1 * 16)] + stp d12, d13, [sp, #(2 * 16)] + stp d14, d15, [sp, #(3 * 16)] + stp d16, d17, [sp, #(4 * 16)] + stp x18, x19, [sp, #(5 * 16)] + stp x20, x21, [sp, #(6 * 16)] + stp x22, x23, [sp, #(7 * 16)] + stp x24, x25, [sp, #(8 * 16)] + stp x26, x27, [sp, #(9 * 16)] + str x28, [sp, #(10 * 16)] + + fmov alpha, d0 + + lsl LDC, LDC, #3 // ldc = ldc * 8 + + mov pB, origPB + + mov counterJ, origN + asr counterJ, counterJ, #2 // J = J / 4 + cmp counterJ, #0 + ble dgemm_kernel_L2_BEGIN + +/******************************************************************************/ + +dgemm_kernel_L4_BEGIN: + mov pCRow0, pC + add pCRow1, pCRow0, LDC + add pCRow2, pCRow1, LDC + add pCRow3, pCRow2, LDC + add pC, pCRow3, LDC + + mov pA, origPA // pA = start of A array + +dgemm_kernel_L4_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 // counterI = counterI / 8 + cmp counterI, #0 + ble dgemm_kernel_L4_M4_BEGIN + +dgemm_kernel_L4_M8_20: + + mov pB, origPB + + asr counterL , origK, #3 // L = K / 8 + cmp counterL , #2 // is there at least 4 to do? + blt dgemm_kernel_L4_M8_32 + + KERNEL8x4_I + KERNEL8x4_M2 + KERNEL8x4_M1 + KERNEL8x4_M2 + KERNEL8x4_M1 + KERNEL8x4_M2 + KERNEL8x4_M1 + KERNEL8x4_M2 + + subs counterL, counterL, #2 // subtract 2 + ble dgemm_kernel_L4_M8_22a + .align 5 + +dgemm_kernel_L4_M8_22: + + KERNEL8x4_M1 + KERNEL8x4_M2 + KERNEL8x4_M1 + KERNEL8x4_M2 + KERNEL8x4_M1 + KERNEL8x4_M2 + KERNEL8x4_M1 + KERNEL8x4_M2 + + subs counterL, counterL, #1 + bgt dgemm_kernel_L4_M8_22 + + +dgemm_kernel_L4_M8_22a: + + KERNEL8x4_M1 + KERNEL8x4_M2 + KERNEL8x4_M1 + KERNEL8x4_M2 + KERNEL8x4_M1 + KERNEL8x4_M2 + KERNEL8x4_M1 + KERNEL8x4_E + + b dgemm_kernel_L4_M8_44 + +dgemm_kernel_L4_M8_32: + + tst counterL, #1 + ble dgemm_kernel_L4_M8_40 + + KERNEL8x4_I + KERNEL8x4_M2 + KERNEL8x4_M1 + KERNEL8x4_M2 + KERNEL8x4_M1 + KERNEL8x4_M2 + KERNEL8x4_M1 + KERNEL8x4_E + + b dgemm_kernel_L4_M8_44 + +dgemm_kernel_L4_M8_40: + + INIT8x4 + +dgemm_kernel_L4_M8_44: + + ands counterL , origK, #7 + ble dgemm_kernel_L4_M8_100 + +dgemm_kernel_L4_M8_46: + + KERNEL8x4_SUB + + subs counterL, counterL, #1 + bne dgemm_kernel_L4_M8_46 + +dgemm_kernel_L4_M8_100: + + SAVE8x4 + +dgemm_kernel_L4_M8_END: + subs counterI, counterI, #1 + bne dgemm_kernel_L4_M8_20 + +dgemm_kernel_L4_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble dgemm_kernel_L4_END + + tst counterI, #4 + ble dgemm_kernel_L4_M2_BEGIN + +dgemm_kernel_L4_M4_20: + + INIT4x4 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dgemm_kernel_L4_M4_40 + +dgemm_kernel_L4_M4_22: + + KERNEL4x4_SUB + KERNEL4x4_SUB + KERNEL4x4_SUB + KERNEL4x4_SUB + + KERNEL4x4_SUB + KERNEL4x4_SUB + KERNEL4x4_SUB + KERNEL4x4_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L4_M4_22 + +dgemm_kernel_L4_M4_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L4_M4_100 + +dgemm_kernel_L4_M4_42: + + KERNEL4x4_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L4_M4_42 + +dgemm_kernel_L4_M4_100: + + SAVE4x4 + +dgemm_kernel_L4_M4_END: + + +dgemm_kernel_L4_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble dgemm_kernel_L4_END + + tst counterI, #2 // counterI = counterI / 2 + ble dgemm_kernel_L4_M1_BEGIN + +dgemm_kernel_L4_M2_20: + + INIT2x4 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dgemm_kernel_L4_M2_40 + +dgemm_kernel_L4_M2_22: + + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L4_M2_22 + + +dgemm_kernel_L4_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L4_M2_100 + +dgemm_kernel_L4_M2_42: + + KERNEL2x4_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L4_M2_42 + +dgemm_kernel_L4_M2_100: + + SAVE2x4 + +dgemm_kernel_L4_M2_END: + + +dgemm_kernel_L4_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble dgemm_kernel_L4_END + +dgemm_kernel_L4_M1_20: + + INIT1x4 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dgemm_kernel_L4_M1_40 + +dgemm_kernel_L4_M1_22: + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L4_M1_22 + + +dgemm_kernel_L4_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L4_M1_100 + +dgemm_kernel_L4_M1_42: + + KERNEL1x4_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L4_M1_42 + +dgemm_kernel_L4_M1_100: + + SAVE1x4 + +dgemm_kernel_L4_END: + + lsl temp, origK, #5 + add origPB, origPB, temp // B = B + K * 4 * 8 + + subs counterJ, counterJ , #1 // j-- + bgt dgemm_kernel_L4_BEGIN + + +/******************************************************************************/ + +dgemm_kernel_L2_BEGIN: // less than 2 left in N direction + + mov counterJ , origN + tst counterJ , #3 + ble dgemm_kernel_L999 // error, N was less than 4? + + tst counterJ , #2 + ble dgemm_kernel_L1_BEGIN + + mov pCRow0, pC // pCRow0 = pC + + add pC,pC,LDC, lsl #1 + + mov pA, origPA // pA = A + +dgemm_kernel_L2_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 // counterI = counterI / 8 + cmp counterI, #0 + ble dgemm_kernel_L2_M4_BEGIN + +dgemm_kernel_L2_M8_20: + + INIT8x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble dgemm_kernel_L2_M8_40 + .align 5 + +dgemm_kernel_L2_M8_22: + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L2_M8_22 + + +dgemm_kernel_L2_M8_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L2_M8_100 + +dgemm_kernel_L2_M8_42: + + KERNEL8x2_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L2_M8_42 + +dgemm_kernel_L2_M8_100: + + SAVE8x2 + +dgemm_kernel_L2_M8_END: + + subs counterI, counterI, #1 + bgt dgemm_kernel_L2_M8_20 + +dgemm_kernel_L2_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble dgemm_kernel_L2_END + + tst counterI, #4 // counterI = counterI / 2 + ble dgemm_kernel_L2_M2_BEGIN + +dgemm_kernel_L2_M4_20: + + INIT4x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble dgemm_kernel_L2_M4_40 + .align 5 + +dgemm_kernel_L2_M4_22: + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L2_M4_22 + + +dgemm_kernel_L2_M4_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L2_M4_100 + +dgemm_kernel_L2_M4_42: + + KERNEL4x2_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L2_M4_42 + +dgemm_kernel_L2_M4_100: + + SAVE4x2 + +dgemm_kernel_L2_M4_END: + + +dgemm_kernel_L2_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble dgemm_kernel_L2_END + + tst counterI, #2 // counterI = counterI / 2 + ble dgemm_kernel_L2_M1_BEGIN + +dgemm_kernel_L2_M2_20: + + INIT2x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble dgemm_kernel_L2_M2_40 + +dgemm_kernel_L2_M2_22: + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L2_M2_22 + + +dgemm_kernel_L2_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L2_M2_100 + +dgemm_kernel_L2_M2_42: + + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L2_M2_42 + +dgemm_kernel_L2_M2_100: + + SAVE2x2 + +dgemm_kernel_L2_M2_END: + + +dgemm_kernel_L2_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble dgemm_kernel_L2_END + +dgemm_kernel_L2_M1_20: + + INIT1x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL, #0 + ble dgemm_kernel_L2_M1_40 + +dgemm_kernel_L2_M1_22: + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L2_M1_22 + + +dgemm_kernel_L2_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L2_M1_100 + +dgemm_kernel_L2_M1_42: + + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L2_M1_42 + +dgemm_kernel_L2_M1_100: + + SAVE1x2 + +dgemm_kernel_L2_END: + add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 + +/******************************************************************************/ + +dgemm_kernel_L1_BEGIN: + + mov counterJ , origN + tst counterJ , #1 + ble dgemm_kernel_L999 // done + + mov pCRow0, pC // pCRow0 = C + add pC , pC , LDC // Update pC to point to next + + mov pA, origPA // pA = A + +dgemm_kernel_L1_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 // counterI = counterI / 8 + cmp counterI, #0 + ble dgemm_kernel_L1_M4_BEGIN + +dgemm_kernel_L1_M8_20: + + INIT8x1 + + mov pB, origPB + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dgemm_kernel_L1_M8_40 + .align 5 + +dgemm_kernel_L1_M8_22: + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L1_M8_22 + + +dgemm_kernel_L1_M8_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L1_M8_100 + +dgemm_kernel_L1_M8_42: + + KERNEL8x1_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L1_M8_42 + +dgemm_kernel_L1_M8_100: + + SAVE8x1 + +dgemm_kernel_L1_M8_END: + + subs counterI, counterI, #1 + bgt dgemm_kernel_L1_M8_20 + +dgemm_kernel_L1_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble dgemm_kernel_L1_END + + tst counterI, #4 // counterI = counterI / 2 + ble dgemm_kernel_L1_M2_BEGIN + +dgemm_kernel_L1_M4_20: + + INIT4x1 + + mov pB, origPB + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dgemm_kernel_L1_M4_40 + .align 5 + +dgemm_kernel_L1_M4_22: + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L1_M4_22 + + +dgemm_kernel_L1_M4_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L1_M4_100 + +dgemm_kernel_L1_M4_42: + + KERNEL4x1_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L1_M4_42 + +dgemm_kernel_L1_M4_100: + + SAVE4x1 + +dgemm_kernel_L1_M4_END: + +dgemm_kernel_L1_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble dgemm_kernel_L1_END + + tst counterI, #2 // counterI = counterI / 2 + ble dgemm_kernel_L1_M1_BEGIN + +dgemm_kernel_L1_M2_20: + + INIT2x1 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dgemm_kernel_L1_M2_40 + +dgemm_kernel_L1_M2_22: + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L1_M2_22 + + +dgemm_kernel_L1_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L1_M2_100 + +dgemm_kernel_L1_M2_42: + + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L1_M2_42 + +dgemm_kernel_L1_M2_100: + + SAVE2x1 + +dgemm_kernel_L1_M2_END: + + +dgemm_kernel_L1_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble dgemm_kernel_L1_END + +dgemm_kernel_L1_M1_20: + + INIT1x1 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dgemm_kernel_L1_M1_40 + +dgemm_kernel_L1_M1_22: + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L1_M1_22 + + +dgemm_kernel_L1_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L1_M1_100 + +dgemm_kernel_L1_M1_42: + + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L1_M1_42 + +dgemm_kernel_L1_M1_100: + + SAVE1x1 + + +dgemm_kernel_L1_END: + + +dgemm_kernel_L999: + mov x0, #0 // set return value + ldp d8, d9, [sp, #(0 * 16)] + ldp d10, d11, [sp, #(1 * 16)] + ldp d12, d13, [sp, #(2 * 16)] + ldp d14, d15, [sp, #(3 * 16)] + ldp d16, d17, [sp, #(4 * 16)] + ldp x18, x19, [sp, #(5 * 16)] + ldp x20, x21, [sp, #(6 * 16)] + ldp x22, x23, [sp, #(7 * 16)] + ldp x24, x25, [sp, #(8 * 16)] + ldp x26, x27, [sp, #(9 * 16)] + ldr x28, [sp, #(10 * 16)] + add sp, sp, #(11*16) + ret + + EPILOGUE + diff --git a/kernel/arm64/dtrmm_kernel_4x8.S b/kernel/arm64/dtrmm_kernel_4x8.S new file mode 100755 index 000000000..eb7397faa --- /dev/null +++ b/kernel/arm64/dtrmm_kernel_4x8.S @@ -0,0 +1,2026 @@ +/******************************************************************************* +Copyright (c) 2015, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#define ASSEMBLER +#include "common.h" + +/* X0 X1 X2 s0 X3 x4 x5 x6 x7*/ +/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha0,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc, BLASLONG offset) */ + +#define origM x0 +#define origN x1 +#define origK x2 +#define origPA x3 +#define origPB x4 +#define pC x5 +#define LDC x6 +#define offset x7 +#define counterL x8 +#define counterI x9 +#define counterJ x10 +#define pB x11 +#define pCRow0 x12 +#define pCRow1 x13 +#define pCRow2 x14 +#define pA x15 +#define temp x16 +#define tempOffset x17 +#define tempK x18 + +#define alpha0 d2 +#define alphaV0 v2.d[0] +#define alpha1 d3 +#define alphaV1 v3.d[0] +#define alpha2 d6 +#define alphaV2 v6.d[0] +#define alpha3 d7 +#define alphaV3 v7.d[0] + +// 00 origM +// 01 origN +// 02 origK +// 03 origPA +// 04 origPB +// 05 pC +// 06 origLDC -> LDC +// 07 offset +// 08 counterL +// 09 counterI +// 10 counterJ +// 11 pB +// 12 pCRow0 +// 13 pCRow1 +// 14 pCRow2 +// 15 pA +// 16 temp +// 17 tempOffset +// 18 must save tempK +// 19 must save +// 20 must save +// 21 must save +// 22 must save +// 23 must save +// 24 must save +// 25 must save +// 26 must save +// 27 must save +// 28 must save +// 29 frame +// 30 link +// 31 sp + +//v00 ALPHA -> pA00, pA01 +//v01 pA02, pA03 +//v02 ALPHA0 +//v03 ALPHA1 +//v04 pA10, pA11 +//v05 pA12, pA13 +//v06 ALPHA2 +//v07 ALPHA3 +//v08 must save pB0_0, pB0_1 +//v09 must save pB0_2, pB0_3 +//v10 must save pB0_4, pB0_5 +//v11 must save pB0_6, pB0_7 +//v12 must save pB1_0, pB1_1 +//v13 must save pB1_2, pB1_3 +//v14 must save pB1_4, pB1_5 +//v15 must save pB1_6, pB1_7 +//v16 must save C00, C01 +//v17 must save C02, C03 +//v18 C04, C05 +//v19 C06, C07 +//v20 C10, C11 +//v21 C12, C13 +//v22 C14, C15 +//v23 C16, C17 +//v24 C20, C21 +//v25 C22, C23 +//v26 C24, C25 +//v27 C26, C27 +//v28 C30, C31 +//v29 C32, C33 +//v30 C34, C35 +//v31 C36, C37 + +/******************************************************************************* +* Macro definitions +*******************************************************************************/ + +.macro INIT4x8 + fmov d16, xzr + fmov d17, xzr + fmov d18, xzr + fmov d19, d16 + fmov d20, xzr + fmov d21, d16 + fmov d22, d17 + fmov d23, d18 + fmov d24, xzr + fmov d25, d16 + fmov d26, d17 + fmov d27, d18 + fmov d28, xzr + fmov d29, d16 + fmov d30, d17 + fmov d31, d18 +.endm + +.macro KERNEL4x8_I + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + ld1 {v0.2d, v1.2d}, [pA] + add pA, pA, #32 + ld1 {v10.2d, v11.2d}, [pB] + add pB, pB, #32 + + fmul v16.2d, v0.2d, v8.2d[0] + fmul v17.2d, v1.2d, v8.2d[0] + fmul v18.2d, v0.2d, v8.2d[1] + fmul v19.2d, v1.2d, v8.2d[1] + + fmul v20.2d, v0.2d, v9.2d[0] + fmul v21.2d, v1.2d, v9.2d[0] + fmul v22.2d, v0.2d, v9.2d[1] + fmul v23.2d, v1.2d, v9.2d[1] + + fmul v24.2d, v0.2d, v10.2d[0] + fmul v25.2d, v1.2d, v10.2d[0] + fmul v26.2d, v0.2d, v10.2d[1] + fmul v27.2d, v1.2d, v10.2d[1] + + fmul v28.2d, v0.2d, v11.2d[0] + fmul v29.2d, v1.2d, v11.2d[0] + fmul v30.2d, v0.2d, v11.2d[1] + fmul v31.2d, v1.2d, v11.2d[1] + + ld1 {v12.2d, v13.2d}, [pB] + add pB, pB, #32 + ld1 {v4.2d, v5.2d}, [pA] + add pA, pA, #32 + ld1 {v14.2d, v15.2d}, [pB] + add pB, pB, #32 +.endm + +.macro KERNEL4x8_M1 + fmla v16.2d, v0.2d, v8.2d[0] + fmla v17.2d, v1.2d, v8.2d[0] + fmla v18.2d, v0.2d, v8.2d[1] + fmla v19.2d, v1.2d, v8.2d[1] + + fmla v20.2d, v0.2d, v9.2d[0] + fmla v21.2d, v1.2d, v9.2d[0] + fmla v22.2d, v0.2d, v9.2d[1] + fmla v23.2d, v1.2d, v9.2d[1] + + fmla v24.2d, v0.2d, v10.2d[0] + fmla v25.2d, v1.2d, v10.2d[0] + fmla v26.2d, v0.2d, v10.2d[1] + fmla v27.2d, v1.2d, v10.2d[1] + + fmla v28.2d, v0.2d, v11.2d[0] + fmla v29.2d, v1.2d, v11.2d[0] + fmla v30.2d, v0.2d, v11.2d[1] + fmla v31.2d, v1.2d, v11.2d[1] + + ld1 {v12.2d, v13.2d}, [pB] // For next round + add pB, pB, #32 + ld1 {v4.2d, v5.2d}, [pA] // For next round + add pA, pA, #32 + ld1 {v14.2d, v15.2d}, [pB] + add pB, pB, #32 + + prfm PLDL1KEEP, [pA, #512] +.endm + +.macro KERNEL4x8_M2 + fmla v16.2d, v4.2d, v12.2d[0] + fmla v17.2d, v5.2d, v12.2d[0] + fmla v18.2d, v4.2d, v12.2d[1] + fmla v19.2d, v5.2d, v12.2d[1] + + fmla v20.2d, v4.2d, v13.2d[0] + fmla v21.2d, v5.2d, v13.2d[0] + fmla v22.2d, v4.2d, v13.2d[1] + fmla v23.2d, v5.2d, v13.2d[1] + + fmla v24.2d, v4.2d, v14.2d[0] + fmla v25.2d, v5.2d, v14.2d[0] + fmla v26.2d, v4.2d, v14.2d[1] + fmla v27.2d, v5.2d, v14.2d[1] + + fmla v28.2d, v4.2d, v15.2d[0] + fmla v29.2d, v5.2d, v15.2d[0] + fmla v30.2d, v4.2d, v15.2d[1] + fmla v31.2d, v5.2d, v15.2d[1] + + ld1 {v8.2d, v9.2d}, [pB] // For next round + add pB, pB, #32 + ld1 {v0.2d, v1.2d}, [pA] // For next round + add pA, pA, #32 + ld1 {v10.2d, v11.2d}, [pB] + add pB, pB, #32 + + prfm PLDL1KEEP, [pB, #512] +.endm + +.macro KERNEL4x8_E + fmla v16.2d, v4.2d, v12.2d[0] + fmla v17.2d, v5.2d, v12.2d[0] + fmla v18.2d, v4.2d, v12.2d[1] + fmla v19.2d, v5.2d, v12.2d[1] + + fmla v20.2d, v4.2d, v13.2d[0] + fmla v21.2d, v5.2d, v13.2d[0] + fmla v22.2d, v4.2d, v13.2d[1] + fmla v23.2d, v5.2d, v13.2d[1] + + fmla v24.2d, v4.2d, v14.2d[0] + fmla v25.2d, v5.2d, v14.2d[0] + fmla v26.2d, v4.2d, v14.2d[1] + fmla v27.2d, v5.2d, v14.2d[1] + + fmla v28.2d, v4.2d, v15.2d[0] + fmla v29.2d, v5.2d, v15.2d[0] + fmla v30.2d, v4.2d, v15.2d[1] + fmla v31.2d, v5.2d, v15.2d[1] +.endm + +.macro KERNEL4x8_SUB + ld1 {v8.2d, v9.2d}, [pB] // For next round + add pB, pB, #32 + ld1 {v0.2d, v1.2d}, [pA] // For next round + add pA, pA, #32 + ld1 {v10.2d, v11.2d}, [pB] + add pB, pB, #32 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v17.2d, v1.2d, v8.2d[0] + fmla v18.2d, v0.2d, v8.2d[1] + fmla v19.2d, v1.2d, v8.2d[1] + + fmla v20.2d, v0.2d, v9.2d[0] + fmla v21.2d, v1.2d, v9.2d[0] + fmla v22.2d, v0.2d, v9.2d[1] + fmla v23.2d, v1.2d, v9.2d[1] + + fmla v24.2d, v0.2d, v10.2d[0] + fmla v25.2d, v1.2d, v10.2d[0] + fmla v26.2d, v0.2d, v10.2d[1] + fmla v27.2d, v1.2d, v10.2d[1] + + fmla v28.2d, v0.2d, v11.2d[0] + fmla v29.2d, v1.2d, v11.2d[0] + fmla v30.2d, v0.2d, v11.2d[1] + fmla v31.2d, v1.2d, v11.2d[1] +.endm + +.macro SAVE4x8 + add pCRow1, pCRow0, LDC + + fmul v8.2d, v16.2d, alphaV0 + fmul v9.2d, v17.2d, alphaV1 + st1 {v8.2d, v9.2d}, [pCRow0] + + add pCRow2, pCRow1, LDC + + fmul v10.2d, v18.2d, alphaV2 + fmul v11.2d, v19.2d, alphaV3 + st1 {v10.2d, v11.2d}, [pCRow1] + + add pCRow1, pCRow2, LDC + + fmul v12.2d, v20.2d, alphaV0 + fmul v13.2d, v21.2d, alphaV1 + st1 {v12.2d, v13.2d}, [pCRow2] + + add pCRow2, pCRow1, LDC + + fmul v14.2d, v22.2d, alphaV2 + fmul v15.2d, v23.2d, alphaV3 + st1 {v14.2d, v15.2d}, [pCRow1] + + add pCRow1, pCRow2, LDC + + fmul v8.2d, v24.2d, alphaV0 + fmul v9.2d, v25.2d, alphaV1 + st1 {v8.2d, v9.2d}, [pCRow2] + + add pCRow2, pCRow1, LDC + + fmul v10.2d, v26.2d, alphaV2 + fmul v11.2d, v27.2d, alphaV3 + st1 {v10.2d, v11.2d}, [pCRow1] + + add pCRow1, pCRow2, LDC + + fmul v12.2d, v28.2d, alphaV0 + fmul v13.2d, v29.2d, alphaV1 + st1 {v12.2d, v13.2d}, [pCRow2] + + fmul v14.2d, v30.2d, alphaV2 + fmul v15.2d, v31.2d, alphaV3 + st1 {v14.2d, v15.2d}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT2x8 + fmov d16, xzr + fmov d18, xzr + fmov d20, xzr + fmov d22, d16 + fmov d24, xzr + fmov d26, d16 + fmov d28, xzr + fmov d30, d16 +.endm + +.macro KERNEL2x8_SUB + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + ld1 {v0.2d}, [pA] + add pA, pA, #16 + ld1 {v10.2d, v11.2d}, [pB] + add pB, pB, #32 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v18.2d, v0.2d, v8.2d[1] + + fmla v20.2d, v0.2d, v9.2d[0] + fmla v22.2d, v0.2d, v9.2d[1] + + fmla v24.2d, v0.2d, v10.2d[0] + fmla v26.2d, v0.2d, v10.2d[1] + + fmla v28.2d, v0.2d, v11.2d[0] + fmla v30.2d, v0.2d, v11.2d[1] +.endm + +.macro SAVE2x8 + add pCRow1, pCRow0, LDC + + fmul v8.2d, v16.2d, alphaV0 + st1 {v8.2d}, [pCRow0] + + add pCRow2, pCRow1, LDC + + fmul v10.2d, v18.2d, alphaV2 + st1 {v10.2d}, [pCRow1] + + add pCRow1, pCRow2, LDC + + fmul v12.2d, v20.2d, alphaV0 + st1 {v12.2d}, [pCRow2] + + add pCRow2, pCRow1, LDC + + fmul v14.2d, v22.2d, alphaV2 + st1 {v14.2d}, [pCRow1] + + add pCRow1, pCRow2, LDC + + fmul v8.2d, v24.2d, alphaV0 + st1 {v8.2d}, [pCRow2] + + add pCRow2, pCRow1, LDC + + fmul v10.2d, v26.2d, alphaV2 + st1 {v10.2d}, [pCRow1] + + add pCRow1, pCRow2, LDC + + fmul v12.2d, v28.2d, alphaV0 + st1 {v12.2d}, [pCRow2] + + add pCRow2, pCRow1, LDC + + fmul v14.2d, v30.2d, alphaV2 + st1 {v14.2d}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT1x8 + fmov d16, xzr + fmov d20, xzr + fmov d24, xzr + fmov d28, xzr +.endm + +.macro KERNEL1x8_SUB + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + ldr d0, [pA] + add pA, pA, #8 + ld1 {v10.2d, v11.2d}, [pB] + add pB, pB, #32 + + fmla v16.2d, v8.2d, v0.d[0] + fmla v20.2d, v9.2d, v0.d[0] + fmla v24.2d, v10.2d, v0.d[0] + fmla v28.2d, v11.2d, v0.d[0] +.endm + +.macro SAVE1x8 + add pCRow1, pCRow0, LDC + + fmul v8.2d, v16.2d, alphaV0 + st1 {v8.d}[0], [pCRow0] + st1 {v8.d}[1], [pCRow1] + + add pCRow2, pCRow1, LDC + add pCRow1, pCRow2, LDC + + fmul v10.2d, v20.2d, alphaV1 + st1 {v10.d}[0], [pCRow2] + st1 {v10.d}[1], [pCRow1] + + add pCRow2, pCRow1, LDC + add pCRow1, pCRow2, LDC + + fmul v12.2d, v24.2d, alphaV2 + st1 {v12.d}[0], [pCRow2] + st1 {v12.d}[1], [pCRow1] + + add pCRow2, pCRow1, LDC + add pCRow1, pCRow2, LDC + + fmul v14.2d, v28.2d, alphaV3 + st1 {v14.d}[0], [pCRow2] + st1 {v14.d}[1], [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT4x4 + fmov d16, xzr + fmov d17, d16 + fmov d20, d17 + fmov d21, d16 + fmov d24, d17 + fmov d25, d16 + fmov d28, d17 + fmov d29, d16 +.endm + +.macro KERNEL4x4_I + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + ld1 {v0.2d, v1.2d}, [pA] + add pA, pA, #32 + + fmul v16.2d, v0.2d, v8.2d[0] + fmul v29.2d, v1.2d, v9.2d[1] + + fmul v20.2d, v0.2d, v8.2d[1] + fmul v25.2d, v1.2d, v9.2d[0] + + fmul v24.2d, v0.2d, v9.2d[0] + fmul v21.2d, v1.2d, v8.2d[1] + + fmul v28.2d, v0.2d, v9.2d[1] + fmul v17.2d, v1.2d, v8.2d[0] + + ld1 {v12.2d, v13.2d}, [pB] + add pB, pB, #32 + ld1 {v4.2d, v5.2d}, [pA] + add pA, pA, #32 +.endm + +.macro KERNEL4x4_M1 + fmla v16.2d, v0.2d, v8.2d[0] + fmla v29.2d, v1.2d, v9.2d[1] + + ld1 {v12.2d, v13.2d}, [pB] // For next round + add pB, pB, #32 + + fmla v20.2d, v0.2d, v8.2d[1] + fmla v25.2d, v1.2d, v9.2d[0] + + ld1 {v4.2d, v5.2d}, [pA] // For next round + add pA, pA, #32 + + fmla v24.2d, v0.2d, v9.2d[0] + fmla v21.2d, v1.2d, v8.2d[1] + + prfm PLDL1KEEP, [pA, #512] + + fmla v28.2d, v0.2d, v9.2d[1] + fmla v17.2d, v1.2d, v8.2d[0] +.endm + +.macro KERNEL4x4_M2 + fmla v16.2d, v4.2d, v12.2d[0] + fmla v29.2d, v5.2d, v13.2d[1] + + ld1 {v8.2d, v9.2d}, [pB] // For next round + add pB, pB, #32 + + fmla v20.2d, v4.2d, v12.2d[1] + fmla v25.2d, v5.2d, v13.2d[0] + + ld1 {v0.2d, v1.2d}, [pA] // For next round + add pA, pA, #32 + + fmla v24.2d, v4.2d, v13.2d[0] + fmla v21.2d, v5.2d, v12.2d[1] + + prfm PLDL1KEEP, [pB, #512] + + fmla v28.2d, v4.2d, v13.2d[1] + fmla v17.2d, v5.2d, v12.2d[0] +.endm + +.macro KERNEL4x4_E + fmla v16.2d, v4.2d, v12.2d[0] + fmla v29.2d, v5.2d, v13.2d[1] + + fmla v20.2d, v4.2d, v12.2d[1] + fmla v25.2d, v5.2d, v13.2d[0] + + fmla v24.2d, v4.2d, v13.2d[0] + fmla v21.2d, v5.2d, v12.2d[1] + + fmla v28.2d, v4.2d, v13.2d[1] + fmla v17.2d, v5.2d, v12.2d[0] +.endm + +.macro KERNEL4x4_SUB + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + ld1 {v0.2d, v1.2d}, [pA] + add pA, pA, #32 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v29.2d, v1.2d, v9.2d[1] + + fmla v20.2d, v0.2d, v8.2d[1] + fmla v25.2d, v1.2d, v9.2d[0] + + fmla v24.2d, v0.2d, v9.2d[0] + fmla v21.2d, v1.2d, v8.2d[1] + + fmla v28.2d, v0.2d, v9.2d[1] + fmla v17.2d, v1.2d, v8.2d[0] +.endm + +.macro SAVE4x4 + fmul v8.2d, v16.2d, alphaV0 + fmul v9.2d, v17.2d, alphaV1 + st1 {v8.2d, v9.2d}, [pCRow0] + + add pCRow1, pCRow0, LDC + + fmul v12.2d, v20.2d, alphaV2 + fmul v13.2d, v21.2d, alphaV3 + st1 {v12.2d, v13.2d}, [pCRow1] + + add pCRow2, pCRow1, LDC + + fmul v8.2d, v24.2d, alphaV0 + fmul v9.2d, v25.2d, alphaV1 + st1 {v8.2d, v9.2d}, [pCRow2] + + add pCRow1, pCRow2, LDC + + fmul v12.2d, v28.2d, alphaV2 + fmul v13.2d, v29.2d, alphaV3 + st1 {v12.2d, v13.2d}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT2x4 + fmov d16, xzr + fmov d20, d16 + fmov d24, d20 + fmov d28, d16 +.endm + +.macro KERNEL2x4_SUB + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + ld1 {v0.2d}, [pA] + add pA, pA, #16 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v20.2d, v0.2d, v8.2d[1] + fmla v24.2d, v0.2d, v9.2d[0] + fmla v28.2d, v0.2d, v9.2d[1] +.endm + +.macro SAVE2x4 + fmul v8.2d, v16.2d, alphaV0 + st1 {v8.2d}, [pCRow0] + + add pCRow1, pCRow0, LDC + + fmul v12.2d, v20.2d, alphaV1 + st1 {v12.2d}, [pCRow1] + + add pCRow2, pCRow1, LDC + + fmul v8.2d, v24.2d, alphaV2 + st1 {v8.2d}, [pCRow2] + + add pCRow1, pCRow2, LDC + + fmul v12.2d, v28.2d, alphaV3 + st1 {v12.2d}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT1x4 + fmov d16, xzr + fmov d20, d16 +.endm + +.macro KERNEL1x4_SUB + ldr d0, [pA] + add pA, pA, #8 + + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + + fmla v16.2d, v8.2d, v0.d[0] + fmla v20.2d, v9.2d, v0.d[0] +.endm + +.macro SAVE1x4 + add pCRow1, pCRow0, LDC + + fmul v8.2d, v16.2d, alphaV0 + st1 {v8.d}[0], [pCRow0] + st1 {v8.d}[1], [pCRow1] + + add pCRow2, pCRow1, LDC + add pCRow1, pCRow2, LDC + + fmul v12.2d, v20.2d, alphaV1 + st1 {v12.d}[0], [pCRow2] + st1 {v12.d}[1], [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT4x2 + fmov d16, xzr + fmov d17, d16 + fmov d20, d17 + fmov d21, d16 +.endm + +.macro KERNEL4x2_SUB + ld1 {v8.2d}, [pB] + add pB, pB, #16 + ld1 {v0.2d, v1.2d}, [pA] + add pA, pA, #32 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v17.2d, v1.2d, v8.2d[0] + fmla v20.2d, v0.2d, v8.2d[1] + fmla v21.2d, v1.2d, v8.2d[1] +.endm + +.macro SAVE4x2 + fmul v8.2d, v16.2d, alphaV0 + fmul v9.2d, v17.2d, alphaV1 + st1 {v8.2d, v9.2d}, [pCRow0] + + add pCRow1, pCRow0, LDC + + fmul v12.2d, v20.2d, alphaV2 + fmul v13.2d, v21.2d, alphaV3 + st1 {v12.2d, v13.2d}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT2x2 + fmov d16, xzr + fmov d20, d16 +.endm + +.macro KERNEL2x2_SUB + ld1 {v8.2d}, [pB] + add pB, pB, #16 + + ld1 {v0.2d}, [pA] + add pA, pA, #16 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v20.2d, v0.2d, v8.2d[1] +.endm + +.macro SAVE2x2 + fmul v8.2d, v16.2d, alphaV0 + st1 {v8.2d}, [pCRow0] + + add pCRow1 , pCRow0, LDC + + fmul v12.2d, v20.2d, alphaV1 + st1 {v12.2d}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT1x2 + fmov d16, xzr +.endm + +.macro KERNEL1x2_SUB + ld1 {v8.2d} , [pB] + add pB , pB, #16 + + ldr d0 , [pA] + add pA, pA, #8 + + fmla v16.2d, v8.2d, v0.2d[0] +.endm + +.macro SAVE1x2 + add pCRow1 , pCRow0, LDC + + fmul v8.2d, v16.2d, alphaV0 + st1 {v8.d}[0], [pCRow0] + st1 {v8.d}[1], [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT4x1 + fmov d16, xzr + fmov d17, d16 +.endm + +.macro KERNEL4x1_SUB + ldr d8, [pB] + add pB , pB, #8 + + ld1 {v0.2d, v1.2d}, [pA] + add pA , pA, #32 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v17.2d, v1.2d, v8.2d[0] +.endm + +.macro SAVE4x1 + fmul v8.2d, v16.2d, alphaV0 + fmul v9.2d, v17.2d, alphaV1 + st1 {v8.2d, v9.2d}, [pCRow0] + + add pCRow0, pCRow0, #32 +.endm + + + + +/******************************************************************************/ + +.macro INIT2x1 + fmov d16, xzr +.endm + +.macro KERNEL2x1_SUB + ldr d8, [pB] + add pB , pB, #8 + + ld1 {v0.2d}, [pA] + add pA , pA, #16 + + fmla v16.2d, v0.2d, v8.2d[0] +.endm + +.macro SAVE2x1 + fmul v8.2d, v16.2d, alphaV0 + st1 {v8.2d}, [pCRow0] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT1x1 + fmov d16, xzr +.endm + +.macro KERNEL1x1_SUB + ldr d8, [pB] + add pB , pB, #8 + + ldr d0, [pA] + add pA , pA, #8 + + fmadd d16, d0, d8, d16 +.endm + +.macro SAVE1x1 + fmul d8, d16, alpha0 + str d8, [pCRow0] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************* +* End of macro definitions +*******************************************************************************/ + + PROLOGUE + + .align 5 + add sp, sp, #-(11 * 16) + stp d8, d9, [sp, #(0 * 16)] + stp d10, d11, [sp, #(1 * 16)] + stp d12, d13, [sp, #(2 * 16)] + stp d14, d15, [sp, #(3 * 16)] + stp d16, d17, [sp, #(4 * 16)] + stp x18, x19, [sp, #(5 * 16)] + stp x20, x21, [sp, #(6 * 16)] + stp x22, x23, [sp, #(7 * 16)] + stp x24, x25, [sp, #(8 * 16)] + stp x26, x27, [sp, #(9 * 16)] + str x28, [sp, #(10 * 16)] + + fmov alpha0, d0 + fmov alpha1, d0 + fmov alpha2, d0 + fmov alpha3, d0 + + lsl LDC, LDC, #3 // ldc = ldc * 8 + +#if !defined(LEFT) + neg tempOffset, offset +#endif + + mov pB, origPB + + mov counterJ, origN + asr counterJ, counterJ, #3 // J = J / 8 + cmp counterJ, #0 + ble dtrmm_kernel_L4_BEGIN + +/******************************************************************************/ + +dtrmm_kernel_L8_BEGIN: + + mov pCRow0, pC // pCRow0 = C + add pC, pC, LDC, lsl #3 + +#if defined(LEFT) + mov tempOffset, offset +#endif + + mov pA, origPA // pA = start of A array + +dtrmm_kernel_L8_M4_BEGIN: + + mov counterI, origM + asr counterI, counterI, #2 // counterI = counterI / 4 + cmp counterI, #0 + ble dtrmm_kernel_L8_M2_BEGIN + +dtrmm_kernel_L8_M4_20: + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #5 + add pA, pA, temp + lsl temp, tempOffset, #6 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #4 +#else + add tempK, tempOffset, #8 +#endif + + asr counterL, tempK, #1 // L = K / 2 + cmp counterL , #2 // is there at least 4 to do? + blt dtrmm_kernel_L8_M4_32 + + KERNEL4x8_I // do one in the K + KERNEL4x8_M2 // do another in the K + + subs counterL, counterL, #2 + ble dtrmm_kernel_L8_M4_22a + .align 5 + +dtrmm_kernel_L8_M4_22: + + KERNEL4x8_M1 + KERNEL4x8_M2 + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L8_M4_22 + + +dtrmm_kernel_L8_M4_22a: + + KERNEL4x8_M1 + KERNEL4x8_E + + b dtrmm_kernel_L8_M4_44 + +dtrmm_kernel_L8_M4_32: + + tst counterL, #1 + ble dtrmm_kernel_L8_M4_40 + + KERNEL4x8_I + + KERNEL4x8_E + + b dtrmm_kernel_L8_M4_44 + + +dtrmm_kernel_L8_M4_40: + + INIT4x8 + +dtrmm_kernel_L8_M4_44: + + ands counterL, tempK, #1 + ble dtrmm_kernel_L8_M4_100 + +dtrmm_kernel_L8_M4_46: + + KERNEL4x8_SUB + +dtrmm_kernel_L8_M4_100: + + SAVE4x8 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #4 +#else + sub tempK, tempK, #8 +#endif + lsl temp, tempK, #5 + add pA, pA, temp + lsl temp, tempK, #6 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #4 +#endif + +dtrmm_kernel_L8_M4_END: + subs counterI, counterI, #1 + bne dtrmm_kernel_L8_M4_20 + +dtrmm_kernel_L8_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble dtrmm_kernel_L8_END + + tst counterI, #2 // counterI = counterI / 2 + ble dtrmm_kernel_L8_M1_BEGIN + +dtrmm_kernel_L8_M2_20: + + INIT2x8 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #4 + add pA, pA, temp + lsl temp, tempOffset, #6 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #2 +#else + add tempK, tempOffset, #8 +#endif + + asr counterL, tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dtrmm_kernel_L8_M2_40 + +dtrmm_kernel_L8_M2_22: + + KERNEL2x8_SUB + KERNEL2x8_SUB + KERNEL2x8_SUB + KERNEL2x8_SUB + + KERNEL2x8_SUB + KERNEL2x8_SUB + KERNEL2x8_SUB + KERNEL2x8_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L8_M2_22 + + +dtrmm_kernel_L8_M2_40: + + ands counterL, tempK, #7 // counterL = counterL % 8 + ble dtrmm_kernel_L8_M2_100 + +dtrmm_kernel_L8_M2_42: + + KERNEL2x8_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L8_M2_42 + +dtrmm_kernel_L8_M2_100: + + SAVE2x8 + + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #2 +#else + sub tempK, tempK, #8 +#endif + lsl temp, tempK, #4 + add pA, pA, temp + lsl temp, tempK, #6 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #2 +#endif + +dtrmm_kernel_L8_M2_END: + + +dtrmm_kernel_L8_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble dtrmm_kernel_L8_END + +dtrmm_kernel_L8_M1_20: + + INIT1x8 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #3 + add pA, pA, temp + lsl temp, tempOffset, #6 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #1 +#else + add tempK, tempOffset, #8 +#endif + + asr counterL, tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dtrmm_kernel_L8_M1_40 + +dtrmm_kernel_L8_M1_22: + KERNEL1x8_SUB + KERNEL1x8_SUB + KERNEL1x8_SUB + KERNEL1x8_SUB + + KERNEL1x8_SUB + KERNEL1x8_SUB + KERNEL1x8_SUB + KERNEL1x8_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L8_M1_22 + + +dtrmm_kernel_L8_M1_40: + + ands counterL, tempK, #7 // counterL = counterL % 8 + ble dtrmm_kernel_L8_M1_100 + +dtrmm_kernel_L8_M1_42: + + KERNEL1x8_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L8_M1_42 + +dtrmm_kernel_L8_M1_100: + + SAVE1x8 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #1 +#else + sub tempK, tempK, #8 +#endif + lsl temp, tempK, #3 + add pA, pA, temp + lsl temp, tempK, #6 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #1 +#endif + +dtrmm_kernel_L8_END: + + lsl temp, origK, #6 + add origPB, origPB, temp // B = B + K * 8 * 8 + +#if !defined(LEFT) + add tempOffset, tempOffset, #8 +#endif + + subs counterJ, counterJ , #1 // j-- + bgt dtrmm_kernel_L8_BEGIN + + +/******************************************************************************/ + +dtrmm_kernel_L4_BEGIN: + + mov counterJ , origN + tst counterJ , #7 + ble dtrmm_kernel_L999 + + tst counterJ , #4 + ble dtrmm_kernel_L2_BEGIN + + mov pCRow0, pC // pCRow0 = C + add pC, pC, LDC, lsl #2 + +#if defined(LEFT) + mov tempOffset, offset +#endif + + mov pA, origPA // pA = start of A array + +dtrmm_kernel_L4_M4_BEGIN: + + mov counterI, origM + asr counterI, counterI, #2 // counterI = counterI / 4 + cmp counterI, #0 + ble dtrmm_kernel_L4_M2_BEGIN + +dtrmm_kernel_L4_M4_20: + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #5 + add pB, pB, temp + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #4 +#else + add tempK, tempOffset, #4 +#endif + + asr counterL, tempK, #1 // L = K / 2 + cmp counterL , #2 // is there at least 4 to do? + blt dtrmm_kernel_L4_M4_32 + + KERNEL4x4_I // do one in the K + KERNEL4x4_M2 // do another in the K + + subs counterL, counterL, #2 + ble dtrmm_kernel_L4_M4_22a + .align 5 + +dtrmm_kernel_L4_M4_22: + + KERNEL4x4_M1 + KERNEL4x4_M2 + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L4_M4_22 + + +dtrmm_kernel_L4_M4_22a: + + KERNEL4x4_M1 + KERNEL4x4_E + + b dtrmm_kernel_L4_M4_44 + +dtrmm_kernel_L4_M4_32: + + tst counterL, #1 + ble dtrmm_kernel_L4_M4_40 + + KERNEL4x4_I + + KERNEL4x4_E + + b dtrmm_kernel_L4_M4_44 + + +dtrmm_kernel_L4_M4_40: + + INIT4x4 + +dtrmm_kernel_L4_M4_44: + + ands counterL , tempK, #1 + ble dtrmm_kernel_L4_M4_100 + +dtrmm_kernel_L4_M4_46: + + KERNEL4x4_SUB + +dtrmm_kernel_L4_M4_100: + + SAVE4x4 +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #4 +#else + sub tempK, tempK, #4 +#endif + lsl temp, tempK, #5 + add pA, pA, temp + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #4 +#endif + +dtrmm_kernel_L4_M4_END: + subs counterI, counterI, #1 + bne dtrmm_kernel_L4_M4_20 + +dtrmm_kernel_L4_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble dtrmm_kernel_L4_END + + tst counterI, #2 // counterI = counterI / 2 + ble dtrmm_kernel_L4_M1_BEGIN + +dtrmm_kernel_L4_M2_20: + + INIT2x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #4 + add pA, pA, temp + lsl temp, tempOffset, #5 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #2 +#else + add tempK, tempOffset, #4 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dtrmm_kernel_L4_M2_40 + +dtrmm_kernel_L4_M2_22: + + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L4_M2_22 + + +dtrmm_kernel_L4_M2_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble dtrmm_kernel_L4_M2_100 + +dtrmm_kernel_L4_M2_42: + + KERNEL2x4_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L4_M2_42 + +dtrmm_kernel_L4_M2_100: + + SAVE2x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #2 +#else + sub tempK, tempK, #4 +#endif + lsl temp, tempK, #4 + add pA, pA, temp + lsl temp, tempK, #5 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #2 +#endif +dtrmm_kernel_L4_M2_END: + + +dtrmm_kernel_L4_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble dtrmm_kernel_L4_END + +dtrmm_kernel_L4_M1_20: + + INIT1x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #5 + add pB, pB, temp + lsl temp, tempOffset, #3 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #1 +#else + add tempK, tempOffset, #4 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dtrmm_kernel_L4_M1_40 + +dtrmm_kernel_L4_M1_22: + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L4_M1_22 + + +dtrmm_kernel_L4_M1_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble dtrmm_kernel_L4_M1_100 + +dtrmm_kernel_L4_M1_42: + + KERNEL1x4_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L4_M1_42 + +dtrmm_kernel_L4_M1_100: + + SAVE1x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #1 +#else + sub tempK, tempK, #4 +#endif + lsl temp, tempK, #3 + add pA, pA, temp + lsl temp, tempK, #5 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #1 +#endif +dtrmm_kernel_L4_END: + + lsl temp, origK, #5 + add origPB, origPB, temp // B = B + K * 4 * 8 +#if !defined(LEFT) + add tempOffset, tempOffset, #4 +#endif + +/******************************************************************************/ + +dtrmm_kernel_L2_BEGIN: // less than 2 left in N direction + + mov counterJ , origN + tst counterJ , #3 + ble dtrmm_kernel_L999 // error, N was less than 4? + + tst counterJ , #2 + ble dtrmm_kernel_L1_BEGIN + + mov pCRow0, pC // pCRow0 = pC + + add pC,pC,LDC, lsl #1 + +#if defined(LEFT) + mov tempOffset, offset +#endif + mov pA, origPA // pA = A + + +dtrmm_kernel_L2_M4_BEGIN: + + mov counterI, origM + asr counterI, counterI, #2 // counterI = counterI / 4 + cmp counterI,#0 + ble dtrmm_kernel_L2_M2_BEGIN + +dtrmm_kernel_L2_M4_20: + + INIT4x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #4 + add pB, pB, temp + lsl temp, tempOffset, #5 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #4 +#else + add tempK, tempOffset, #2 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble dtrmm_kernel_L2_M4_40 + .align 5 + +dtrmm_kernel_L2_M4_22: + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L2_M4_22 + + +dtrmm_kernel_L2_M4_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble dtrmm_kernel_L2_M4_100 + +dtrmm_kernel_L2_M4_42: + + KERNEL4x2_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L2_M4_42 + +dtrmm_kernel_L2_M4_100: + + SAVE4x2 +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #4 +#else + sub tempK, tempK, #2 +#endif + lsl temp, tempK, #5 + add pA, pA, temp + lsl temp, tempK, #4 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #4 +#endif + +dtrmm_kernel_L2_M4_END: + + subs counterI, counterI, #1 + bgt dtrmm_kernel_L2_M4_20 + + +dtrmm_kernel_L2_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble dtrmm_kernel_L2_END + + tst counterI, #2 // counterI = counterI / 2 + ble dtrmm_kernel_L2_M1_BEGIN + +dtrmm_kernel_L2_M2_20: + + INIT2x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #4 + add pB, pB, temp + lsl temp, tempOffset, #4 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #2 +#else + add tempK, tempOffset, #2 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble dtrmm_kernel_L2_M2_40 + +dtrmm_kernel_L2_M2_22: + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L2_M2_22 + + +dtrmm_kernel_L2_M2_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble dtrmm_kernel_L2_M2_100 + +dtrmm_kernel_L2_M2_42: + + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L2_M2_42 + +dtrmm_kernel_L2_M2_100: + + SAVE2x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #2 +#else + sub tempK, tempK, #2 +#endif + lsl temp, tempK, #4 + add pA, pA, temp + lsl temp, tempK, #4 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #2 +#endif +dtrmm_kernel_L2_M2_END: + + +dtrmm_kernel_L2_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble dtrmm_kernel_L2_END + +dtrmm_kernel_L2_M1_20: + + INIT1x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #4 + add pB, pB, temp + lsl temp, tempOffset, #3 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #1 +#else + add tempK, tempOffset, #2 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL, #0 + ble dtrmm_kernel_L2_M1_40 + +dtrmm_kernel_L2_M1_22: + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L2_M1_22 + + +dtrmm_kernel_L2_M1_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble dtrmm_kernel_L2_M1_100 + +dtrmm_kernel_L2_M1_42: + + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L2_M1_42 + +dtrmm_kernel_L2_M1_100: + + SAVE1x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #1 +#else + sub tempK, tempK, #2 +#endif + lsl temp, tempK, #3 + add pA, pA, temp + lsl temp, tempK, #4 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #1 +#endif +dtrmm_kernel_L2_END: +#if !defined(LEFT) + add tempOffset, tempOffset, #2 +#endif + add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 + +/******************************************************************************/ + +dtrmm_kernel_L1_BEGIN: + + mov counterJ , origN + tst counterJ , #1 + ble dtrmm_kernel_L999 // done + + + mov pCRow0, pC // pCRow0 = C + add pC , pC , LDC // Update pC to point to next + +#if defined(LEFT) + mov tempOffset, offset +#endif + mov pA, origPA // pA = A + +dtrmm_kernel_L1_M4_BEGIN: + + mov counterI, origM + asr counterI, counterI, #2 // counterI = counterI / 4 + cmp counterI, #0 + ble dtrmm_kernel_L1_M2_BEGIN + +dtrmm_kernel_L1_M4_20: + + INIT4x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #3 + add pB, pB, temp + lsl temp, tempOffset, #5 + add pA, pA, temp +#endif +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #4 +#else + add tempK, tempOffset, #1 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dtrmm_kernel_L1_M4_40 + .align 5 + +dtrmm_kernel_L1_M4_22: + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L1_M4_22 + + +dtrmm_kernel_L1_M4_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble dtrmm_kernel_L1_M4_100 + +dtrmm_kernel_L1_M4_42: + + KERNEL4x1_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L1_M4_42 + +dtrmm_kernel_L1_M4_100: + + SAVE4x1 +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #4 +#else + sub tempK, tempK, #1 +#endif + lsl temp, tempK, #5 + add pA, pA, temp + lsl temp, tempK, #3 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #4 +#endif + +dtrmm_kernel_L1_M4_END: + + subs counterI, counterI, #1 + bgt dtrmm_kernel_L1_M4_20 + + +dtrmm_kernel_L1_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble dtrmm_kernel_L1_END + + tst counterI, #2 // counterI = counterI / 2 + ble dtrmm_kernel_L1_M1_BEGIN + +dtrmm_kernel_L1_M2_20: + + INIT2x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #3 + add pB, pB, temp + lsl temp, tempOffset, #4 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #2 +#else + add tempK, tempOffset, #1 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dtrmm_kernel_L1_M2_40 + +dtrmm_kernel_L1_M2_22: + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L1_M2_22 + + +dtrmm_kernel_L1_M2_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble dtrmm_kernel_L1_M2_100 + +dtrmm_kernel_L1_M2_42: + + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L1_M2_42 + +dtrmm_kernel_L1_M2_100: + + SAVE2x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #2 +#else + sub tempK, tempK, #1 +#endif + lsl temp, tempK, #4 + add pA, pA, temp + lsl temp, tempK, #3 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #2 +#endif +dtrmm_kernel_L1_M2_END: + + +dtrmm_kernel_L1_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble dtrmm_kernel_L1_END + +dtrmm_kernel_L1_M1_20: + + INIT1x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #3 + add pB, pB, temp + lsl temp, tempOffset, #3 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #1 +#else + add tempK, tempOffset, #1 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dtrmm_kernel_L1_M1_40 + +dtrmm_kernel_L1_M1_22: + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L1_M1_22 + + +dtrmm_kernel_L1_M1_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble dtrmm_kernel_L1_M1_100 + +dtrmm_kernel_L1_M1_42: + + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L1_M1_42 + +dtrmm_kernel_L1_M1_100: + + SAVE1x1 + + +dtrmm_kernel_L1_END: + + +dtrmm_kernel_L999: + mov x0, #0 // set return value + ldp d8, d9, [sp, #(0 * 16)] + ldp d10, d11, [sp, #(1 * 16)] + ldp d12, d13, [sp, #(2 * 16)] + ldp d14, d15, [sp, #(3 * 16)] + ldp d16, d17, [sp, #(4 * 16)] + ldp x18, x19, [sp, #(5 * 16)] + ldp x20, x21, [sp, #(6 * 16)] + ldp x22, x23, [sp, #(7 * 16)] + ldp x24, x25, [sp, #(8 * 16)] + ldp x26, x27, [sp, #(9 * 16)] + ldr x28, [sp, #(10 * 16)] + add sp, sp, #(11*16) + ret + + EPILOGUE + diff --git a/kernel/arm64/dtrmm_kernel_8x4.S b/kernel/arm64/dtrmm_kernel_8x4.S new file mode 100755 index 000000000..6890505bd --- /dev/null +++ b/kernel/arm64/dtrmm_kernel_8x4.S @@ -0,0 +1,1849 @@ +/******************************************************************************* +Copyright (c) 2015, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#define ASSEMBLER +#include "common.h" + +/* X0 X1 X2 s0 X3 x4 x5 x6 x7*/ +/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha0,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc, BLASLONG offset) */ + +#define origM x0 +#define origN x1 +#define origK x2 +#define origPA x3 +#define origPB x4 +#define pC x5 +#define LDC x6 +#define offset x7 +#define counterL x8 +#define counterI x9 +#define counterJ x10 +#define pB x11 +#define pCRow0 x12 +#define pCRow1 x13 +#define pCRow2 x14 +#define pA x15 +#define temp x16 +#define tempOffset x17 +#define tempK x18 + +#define alpha0 d10 +#define alphaV0 v10.d[0] +#define alpha1 d11 +#define alphaV1 v11.d[0] +#define alpha2 d14 +#define alphaV2 v14.d[0] +#define alpha3 d15 +#define alphaV3 v15.d[0] + +// 00 origM +// 01 origN +// 02 origK +// 03 origPA +// 04 origPB +// 05 pC +// 06 origLDC -> LDC +// 07 offset +// 08 counterL +// 09 counterI +// 10 counterJ +// 11 pB +// 12 pCRow0 +// 13 pCRow1 +// 14 pCRow2 +// 15 pA +// 16 temp +// 17 tempOffset +// 18 must save tempK +// 19 must save +// 20 must save +// 21 must save +// 22 must save +// 23 must save +// 24 must save +// 25 must save +// 26 must save +// 27 must save +// 28 must save +// 29 frame +// 30 link +// 31 sp + +//v00 ALPHA -> pA0_0, pA0_1 +//v01 pA0_2, pA0_3 +//v02 pA0_4, pA0_5 +//v03 pA0_6, pA0_7 +//v04 pA1_0, pA1_1 +//v05 pA1_2, pA1_3 +//v06 pA1_4, pA1_5 +//v07 pA1_6, pA1_7 +//v08 must save pB0_0, pB0_1 +//v09 must save pB0_2, pB0_3 +//v10 must save ALPHA0 +//v11 must save ALPHA1 +//v12 must save pB1_0, pB1_1 +//v13 must save pB1_2, pB1_3 +//v14 must save ALPHA2 +//v15 must save ALPHA3 +//v16 must save C00, C01 +//v17 must save C02, C03 +//v18 C04, C05 +//v19 C06, C07 +//v20 C10, C11 +//v21 C12, C13 +//v22 C14, C15 +//v23 C16, C17 +//v24 C20, C21 +//v25 C22, C23 +//v26 C24, C25 +//v27 C26, C27 +//v28 C30, C31 +//v29 C32, C33 +//v30 C34, C35 +//v31 C36, C37 + +/******************************************************************************* +* Macro definitions +*******************************************************************************/ + +.macro INIT8x4 + fmov d16, xzr + fmov d17, xzr + fmov d18, d16 + fmov d19, xzr + fmov d20, xzr + fmov d21, d16 + fmov d22, d17 + fmov d23, d18 + fmov d24, xzr + fmov d25, d16 + fmov d26, d17 + fmov d27, d18 + fmov d28, xzr + fmov d29, d16 + fmov d30, d17 + fmov d31, d18 +.endm + +.macro KERNEL8x4_I + ld1 {v0.2d, v1.2d}, [pA] + add pA, pA, #32 + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + ld1 {v2.2d, v3.2d}, [pA] + add pA, pA, #32 + + fmul v16.2d, v0.2d, v8.2d[0] + fmul v17.2d, v1.2d, v8.2d[0] + fmul v18.2d, v2.2d, v8.2d[0] + fmul v19.2d, v3.2d, v8.2d[0] + + fmul v20.2d, v0.2d, v8.2d[1] + fmul v21.2d, v1.2d, v8.2d[1] + fmul v22.2d, v2.2d, v8.2d[1] + fmul v23.2d, v3.2d, v8.2d[1] + + fmul v24.2d, v0.2d, v9.2d[0] + fmul v25.2d, v1.2d, v9.2d[0] + fmul v26.2d, v2.2d, v9.2d[0] + fmul v27.2d, v3.2d, v9.2d[0] + + fmul v28.2d, v0.2d, v9.2d[1] + fmul v29.2d, v1.2d, v9.2d[1] + fmul v30.2d, v2.2d, v9.2d[1] + fmul v31.2d, v3.2d, v9.2d[1] + + ld1 {v4.2d, v5.2d}, [pA] + add pA, pA, #32 + ld1 {v12.2d, v13.2d}, [pB] + add pB, pB, #32 + ld1 {v6.2d, v7.2d}, [pA] + add pA, pA, #32 +.endm + +.macro KERNEL8x4_M1 + fmla v16.2d, v0.2d, v8.2d[0] + fmla v17.2d, v1.2d, v8.2d[0] + fmla v18.2d, v2.2d, v8.2d[0] + fmla v19.2d, v3.2d, v8.2d[0] + + fmla v20.2d, v0.2d, v8.2d[1] + fmla v21.2d, v1.2d, v8.2d[1] + fmla v22.2d, v2.2d, v8.2d[1] + fmla v23.2d, v3.2d, v8.2d[1] + + fmla v24.2d, v0.2d, v9.2d[0] + fmla v25.2d, v1.2d, v9.2d[0] + fmla v26.2d, v2.2d, v9.2d[0] + fmla v27.2d, v3.2d, v9.2d[0] + + fmla v28.2d, v0.2d, v9.2d[1] + fmla v29.2d, v1.2d, v9.2d[1] + fmla v30.2d, v2.2d, v9.2d[1] + fmla v31.2d, v3.2d, v9.2d[1] + + ld1 {v4.2d, v5.2d}, [pA] + add pA, pA, #32 + ld1 {v12.2d, v13.2d}, [pB] + add pB, pB, #32 + ld1 {v6.2d, v7.2d}, [pA] + add pA, pA, #32 + + prfm PLDL1KEEP, [pA, #512] +.endm + +.macro KERNEL8x4_M2 + fmla v16.2d, v4.2d, v12.2d[0] + fmla v17.2d, v5.2d, v12.2d[0] + fmla v18.2d, v6.2d, v12.2d[0] + fmla v19.2d, v7.2d, v12.2d[0] + + fmla v20.2d, v4.2d, v12.2d[1] + fmla v21.2d, v5.2d, v12.2d[1] + fmla v22.2d, v6.2d, v12.2d[1] + fmla v23.2d, v7.2d, v12.2d[1] + + fmla v24.2d, v4.2d, v13.2d[0] + fmla v25.2d, v5.2d, v13.2d[0] + fmla v26.2d, v6.2d, v13.2d[0] + fmla v27.2d, v7.2d, v13.2d[0] + + fmla v28.2d, v4.2d, v13.2d[1] + fmla v29.2d, v5.2d, v13.2d[1] + fmla v30.2d, v6.2d, v13.2d[1] + fmla v31.2d, v7.2d, v13.2d[1] + + ld1 {v0.2d, v1.2d}, [pA] + add pA, pA, #32 + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + ld1 {v2.2d, v3.2d}, [pA] + add pA, pA, #32 + + prfm PLDL1KEEP, [pB, #512] +.endm + +.macro KERNEL8x4_E + fmla v16.2d, v4.2d, v12.2d[0] + fmla v17.2d, v5.2d, v12.2d[0] + fmla v18.2d, v6.2d, v12.2d[0] + fmla v19.2d, v7.2d, v12.2d[0] + + fmla v20.2d, v4.2d, v12.2d[1] + fmla v21.2d, v5.2d, v12.2d[1] + fmla v22.2d, v6.2d, v12.2d[1] + fmla v23.2d, v7.2d, v12.2d[1] + + fmla v24.2d, v4.2d, v13.2d[0] + fmla v25.2d, v5.2d, v13.2d[0] + fmla v26.2d, v6.2d, v13.2d[0] + fmla v27.2d, v7.2d, v13.2d[0] + + fmla v28.2d, v4.2d, v13.2d[1] + fmla v29.2d, v5.2d, v13.2d[1] + fmla v30.2d, v6.2d, v13.2d[1] + fmla v31.2d, v7.2d, v13.2d[1] +.endm + +.macro KERNEL8x4_SUB + ld1 {v0.2d, v1.2d}, [pA] + add pA, pA, #32 + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + ld1 {v2.2d, v3.2d}, [pA] + add pA, pA, #32 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v17.2d, v1.2d, v8.2d[0] + fmla v18.2d, v2.2d, v8.2d[0] + fmla v19.2d, v3.2d, v8.2d[0] + + fmla v20.2d, v0.2d, v8.2d[1] + fmla v21.2d, v1.2d, v8.2d[1] + fmla v22.2d, v2.2d, v8.2d[1] + fmla v23.2d, v3.2d, v8.2d[1] + + fmla v24.2d, v0.2d, v9.2d[0] + fmla v25.2d, v1.2d, v9.2d[0] + fmla v26.2d, v2.2d, v9.2d[0] + fmla v27.2d, v3.2d, v9.2d[0] + + fmla v28.2d, v0.2d, v9.2d[1] + fmla v29.2d, v1.2d, v9.2d[1] + fmla v30.2d, v2.2d, v9.2d[1] + fmla v31.2d, v3.2d, v9.2d[1] +.endm + +.macro SAVE8x4 + add pCRow1, pCRow0, LDC + + fmul v0.2d, v16.2d, alphaV0 + fmul v1.2d, v17.2d, alphaV1 + fmul v2.2d, v18.2d, alphaV2 + fmul v3.2d, v19.2d, alphaV3 + st1 {v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0] + + add pCRow2, pCRow1, LDC + + fmul v4.2d, v20.2d, alphaV0 + fmul v5.2d, v21.2d, alphaV1 + fmul v6.2d, v22.2d, alphaV2 + fmul v7.2d, v23.2d, alphaV3 + st1 {v4.2d, v5.2d, v6.2d, v7.2d}, [pCRow1] + + add pCRow1, pCRow2, LDC + + fmul v0.2d, v24.2d, alphaV0 + fmul v1.2d, v25.2d, alphaV1 + fmul v2.2d, v26.2d, alphaV2 + fmul v3.2d, v27.2d, alphaV3 + st1 {v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow2] + + fmul v4.2d, v28.2d, alphaV0 + fmul v5.2d, v29.2d, alphaV1 + fmul v6.2d, v30.2d, alphaV2 + fmul v7.2d, v31.2d, alphaV3 + st1 {v4.2d, v5.2d, v6.2d, v7.2d}, [pCRow1] + + add pCRow0, pCRow0, #64 +.endm + +/******************************************************************************/ + +.macro INIT4x4 + fmov d16, xzr + fmov d17, d16 + fmov d20, d17 + fmov d21, d16 + fmov d24, d17 + fmov d25, d16 + fmov d28, d17 + fmov d29, d16 +.endm + +.macro KERNEL4x4_SUB + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + ld1 {v0.2d, v1.2d}, [pA] + add pA, pA, #32 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v29.2d, v1.2d, v9.2d[1] + + fmla v20.2d, v0.2d, v8.2d[1] + fmla v25.2d, v1.2d, v9.2d[0] + + fmla v24.2d, v0.2d, v9.2d[0] + fmla v21.2d, v1.2d, v8.2d[1] + + fmla v28.2d, v0.2d, v9.2d[1] + fmla v17.2d, v1.2d, v8.2d[0] +.endm + +.macro SAVE4x4 + fmul v8.2d, v16.2d, alphaV0 + fmul v9.2d, v17.2d, alphaV1 + st1 {v8.2d, v9.2d}, [pCRow0] + + add pCRow1, pCRow0, LDC + + fmul v12.2d, v20.2d, alphaV2 + fmul v13.2d, v21.2d, alphaV3 + st1 {v12.2d, v13.2d}, [pCRow1] + + add pCRow2, pCRow1, LDC + + fmul v8.2d, v24.2d, alphaV0 + fmul v9.2d, v25.2d, alphaV1 + st1 {v8.2d, v9.2d}, [pCRow2] + + add pCRow1, pCRow2, LDC + + fmul v12.2d, v28.2d, alphaV2 + fmul v13.2d, v29.2d, alphaV3 + st1 {v12.2d, v13.2d}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + + +.macro INIT2x4 + fmov d16, xzr + fmov d20, d16 + fmov d24, d20 + fmov d28, d16 +.endm + +.macro KERNEL2x4_SUB + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + ld1 {v0.2d}, [pA] + add pA, pA, #16 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v20.2d, v0.2d, v8.2d[1] + fmla v24.2d, v0.2d, v9.2d[0] + fmla v28.2d, v0.2d, v9.2d[1] +.endm + +.macro SAVE2x4 + fmul v8.2d, v16.2d, alphaV0 + st1 {v8.2d}, [pCRow0] + + add pCRow1, pCRow0, LDC + + fmul v12.2d, v20.2d, alphaV1 + st1 {v12.2d}, [pCRow1] + + add pCRow2, pCRow1, LDC + + fmul v8.2d, v24.2d, alphaV2 + st1 {v8.2d}, [pCRow2] + + add pCRow1, pCRow2, LDC + + fmul v12.2d, v28.2d, alphaV3 + st1 {v12.2d}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT1x4 + fmov d16, xzr + fmov d20, d16 +.endm + +.macro KERNEL1x4_SUB + ldr d0, [pA] + add pA, pA, #8 + + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + + fmla v16.2d, v8.2d, v0.d[0] + fmla v20.2d, v9.2d, v0.d[0] +.endm + +.macro SAVE1x4 + add pCRow1, pCRow0, LDC + + fmul v8.2d, v16.2d, alphaV0 + st1 {v8.d}[0], [pCRow0] + st1 {v8.d}[1], [pCRow1] + + add pCRow2, pCRow1, LDC + add pCRow1, pCRow2, LDC + + fmul v12.2d, v20.2d, alphaV1 + st1 {v12.d}[0], [pCRow2] + st1 {v12.d}[1], [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT8x2 + fmov d16, xzr + fmov d17, xzr + fmov d18, d16 + fmov d19, d17 + fmov d20, xzr + fmov d21, d16 + fmov d22, d17 + fmov d23, d18 +.endm + +.macro KERNEL8x2_SUB + ld1 {v0.2d, v1.2d}, [pA] + add pA, pA, #32 + ld1 {v8.2d}, [pB] + add pB, pB, #16 + ld1 {v2.2d, v3.2d}, [pA] + add pA, pA, #32 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v17.2d, v1.2d, v8.2d[0] + fmla v18.2d, v2.2d, v8.2d[0] + fmla v19.2d, v3.2d, v8.2d[0] + + fmla v20.2d, v0.2d, v8.2d[1] + fmla v21.2d, v1.2d, v8.2d[1] + fmla v22.2d, v2.2d, v8.2d[1] + fmla v23.2d, v3.2d, v8.2d[1] +.endm + +.macro SAVE8x2 + add pCRow1, pCRow0, LDC + + fmul v0.2d, v16.2d, alphaV0 + fmul v1.2d, v17.2d, alphaV1 + fmul v2.2d, v18.2d, alphaV2 + fmul v3.2d, v19.2d, alphaV3 + st1 {v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0] + + fmul v4.2d, v20.2d, alphaV0 + fmul v5.2d, v21.2d, alphaV1 + fmul v6.2d, v22.2d, alphaV2 + fmul v7.2d, v23.2d, alphaV3 + st1 {v4.2d, v5.2d, v6.2d, v7.2d}, [pCRow1] + + add pCRow0, pCRow0, #64 +.endm + +/******************************************************************************/ + +.macro INIT4x2 + fmov d16, xzr + fmov d17, d16 + fmov d20, d17 + fmov d21, d16 +.endm + +.macro KERNEL4x2_SUB + ld1 {v8.2d}, [pB] + add pB, pB, #16 + ld1 {v0.2d, v1.2d}, [pA] + add pA, pA, #32 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v17.2d, v1.2d, v8.2d[0] + fmla v20.2d, v0.2d, v8.2d[1] + fmla v21.2d, v1.2d, v8.2d[1] +.endm + +.macro SAVE4x2 + fmul v8.2d, v16.2d, alphaV0 + fmul v9.2d, v17.2d, alphaV1 + st1 {v8.2d, v9.2d}, [pCRow0] + + add pCRow1, pCRow0, LDC + + fmul v12.2d, v20.2d, alphaV2 + fmul v13.2d, v21.2d, alphaV3 + st1 {v12.2d, v13.2d}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT2x2 + fmov d16, xzr + fmov d20, d16 +.endm + +.macro KERNEL2x2_SUB + ld1 {v8.2d}, [pB] + add pB, pB, #16 + + ld1 {v0.2d}, [pA] + add pA, pA, #16 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v20.2d, v0.2d, v8.2d[1] +.endm + +.macro SAVE2x2 + fmul v8.2d, v16.2d, alphaV0 + st1 {v8.2d}, [pCRow0] + + add pCRow1 , pCRow0, LDC + + fmul v12.2d, v20.2d, alphaV1 + st1 {v12.2d}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT1x2 + fmov d16, xzr +.endm + +.macro KERNEL1x2_SUB + ld1 {v8.2d} , [pB] + add pB , pB, #16 + + ldr d0 , [pA] + add pA, pA, #8 + + fmla v16.2d, v8.2d, v0.2d[0] +.endm + +.macro SAVE1x2 + add pCRow1 , pCRow0, LDC + + fmul v8.2d, v16.2d, alphaV0 + st1 {v8.d}[0], [pCRow0] + st1 {v8.d}[1], [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT8x1 + fmov d16, xzr + fmov d17, xzr + fmov d18, d16 + fmov d19, d17 +.endm + +.macro KERNEL8x1_SUB + ld1 {v0.2d, v1.2d}, [pA] + add pA , pA, #32 + + ldr d8, [pB] + add pB , pB, #8 + + ld1 {v2.2d, v3.2d}, [pA] + add pA, pA, #32 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v17.2d, v1.2d, v8.2d[0] + fmla v18.2d, v2.2d, v8.2d[0] + fmla v19.2d, v3.2d, v8.2d[0] +.endm + +.macro SAVE8x1 + fmul v0.2d, v16.2d, alphaV0 + fmul v1.2d, v17.2d, alphaV1 + fmul v2.2d, v18.2d, alphaV2 + fmul v3.2d, v19.2d, alphaV3 + st1 {v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0] + + add pCRow0, pCRow0, #64 +.endm + + +/******************************************************************************/ + +.macro INIT4x1 + fmov d16, xzr + fmov d17, d16 +.endm + +.macro KERNEL4x1_SUB + ldr d8, [pB] + add pB , pB, #8 + + ld1 {v0.2d, v1.2d}, [pA] + add pA , pA, #32 + + fmla v16.2d, v0.2d, v8.2d[0] + fmla v17.2d, v1.2d, v8.2d[0] +.endm + +.macro SAVE4x1 + fmul v8.2d, v16.2d, alphaV0 + fmul v9.2d, v17.2d, alphaV1 + st1 {v8.2d, v9.2d}, [pCRow0] + + add pCRow0, pCRow0, #32 +.endm + + +/******************************************************************************/ + +.macro INIT2x1 + fmov d16, xzr +.endm + +.macro KERNEL2x1_SUB + ldr d8, [pB] + add pB , pB, #8 + + ld1 {v0.2d}, [pA] + add pA , pA, #16 + + fmla v16.2d, v0.2d, v8.2d[0] +.endm + +.macro SAVE2x1 + fmul v8.2d, v16.2d, alphaV0 + st1 {v8.2d}, [pCRow0] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT1x1 + fmov d16, xzr +.endm + +.macro KERNEL1x1_SUB + ldr d8, [pB] + add pB , pB, #8 + + ldr d0, [pA] + add pA , pA, #8 + + fmadd d16, d0, d8, d16 +.endm + +.macro SAVE1x1 + fmul d8, d16, alpha0 + str d8, [pCRow0] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************* +* End of macro definitions +*******************************************************************************/ + + PROLOGUE + + .align 5 + add sp, sp, #-(11 * 16) + stp d8, d9, [sp, #(0 * 16)] + stp d10, d11, [sp, #(1 * 16)] + stp d12, d13, [sp, #(2 * 16)] + stp d14, d15, [sp, #(3 * 16)] + stp d16, d17, [sp, #(4 * 16)] + stp x18, x19, [sp, #(5 * 16)] + stp x20, x21, [sp, #(6 * 16)] + stp x22, x23, [sp, #(7 * 16)] + stp x24, x25, [sp, #(8 * 16)] + stp x26, x27, [sp, #(9 * 16)] + str x28, [sp, #(10 * 16)] + + fmov alpha0, d0 + fmov alpha1, d0 + fmov alpha2, d0 + fmov alpha3, d0 + + lsl LDC, LDC, #3 // ldc = ldc * 8 + +#if !defined(LEFT) + neg tempOffset, offset +#endif + mov pB, origPB + + mov counterJ, origN + asr counterJ, counterJ, #2 // J = J / 4 + cmp counterJ, #0 + ble dtrmm_kernel_L2_BEGIN + +/******************************************************************************/ + +dtrmm_kernel_L4_BEGIN: + mov pCRow0, pC // pCRow0 = C + add pC, pC, LDC, lsl #2 + +#if defined(LEFT) + mov tempOffset, offset +#endif + mov pA, origPA // pA = start of A array + +dtrmm_kernel_L4_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 // counterI = counterI / 8 + cmp counterI, #0 + ble dtrmm_kernel_L4_M4_BEGIN + +dtrmm_kernel_L4_M8_20: + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #6 + add pA, pA, temp + lsl temp, tempOffset, #5 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #8 +#else + add tempK, tempOffset, #4 +#endif + + asr counterL , tempK, #1 // L = K / 2 + cmp counterL , #2 // is there at least 4 to do? + blt dtrmm_kernel_L4_M8_32 + + KERNEL8x4_I // do one in the K + KERNEL8x4_M2 // do another in the K + + subs counterL, counterL, #2 // subtract 2 + ble dtrmm_kernel_L4_M8_22a + .align 5 + +dtrmm_kernel_L4_M8_22: + + KERNEL8x4_M1 + KERNEL8x4_M2 + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L4_M8_22 + + +dtrmm_kernel_L4_M8_22a: + + KERNEL8x4_M1 + KERNEL8x4_E + + b dtrmm_kernel_L4_M8_44 + +dtrmm_kernel_L4_M8_32: + + tst counterL, #1 + ble dtrmm_kernel_L4_M8_40 + + KERNEL8x4_I + + KERNEL8x4_E + + b dtrmm_kernel_L4_M8_44 + +dtrmm_kernel_L4_M8_40: + + INIT8x4 + +dtrmm_kernel_L4_M8_44: + + ands counterL , tempK, #1 + ble dtrmm_kernel_L4_M8_100 + +dtrmm_kernel_L4_M8_46: + + KERNEL8x4_SUB + +dtrmm_kernel_L4_M8_100: + + SAVE8x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #8 +#else + sub tempK, tempK, #4 +#endif + lsl temp, tempK, #6 + add pA, pA, temp + lsl temp, tempK, #5 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #8 +#endif + +dtrmm_kernel_L4_M8_END: + subs counterI, counterI, #1 + bne dtrmm_kernel_L4_M8_20 + +dtrmm_kernel_L4_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble dtrmm_kernel_L4_END + + tst counterI, #4 + ble dtrmm_kernel_L4_M2_BEGIN + +dtrmm_kernel_L4_M4_20: + + INIT4x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #5 + add pB, pB, temp + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #4 +#else + add tempK, tempOffset, #4 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dtrmm_kernel_L4_M4_40 + +dtrmm_kernel_L4_M4_22: + + KERNEL4x4_SUB + KERNEL4x4_SUB + KERNEL4x4_SUB + KERNEL4x4_SUB + + KERNEL4x4_SUB + KERNEL4x4_SUB + KERNEL4x4_SUB + KERNEL4x4_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L4_M4_22 + + +dtrmm_kernel_L4_M4_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble dtrmm_kernel_L4_M4_100 + +dtrmm_kernel_L4_M4_42: + + KERNEL4x4_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L4_M4_42 + +dtrmm_kernel_L4_M4_100: + + SAVE4x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #4 +#else + sub tempK, tempK, #4 +#endif + lsl temp, tempK, #5 + add pA, pA, temp + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #4 +#endif + +dtrmm_kernel_L4_M4_END: + + +dtrmm_kernel_L4_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble dtrmm_kernel_L4_END + + tst counterI, #2 // counterI = counterI / 2 + ble dtrmm_kernel_L4_M1_BEGIN + +dtrmm_kernel_L4_M2_20: + + INIT2x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #4 + add pA, pA, temp + lsl temp, tempOffset, #5 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #2 +#else + add tempK, tempOffset, #4 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dtrmm_kernel_L4_M2_40 + +dtrmm_kernel_L4_M2_22: + + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L4_M2_22 + + +dtrmm_kernel_L4_M2_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble dtrmm_kernel_L4_M2_100 + +dtrmm_kernel_L4_M2_42: + + KERNEL2x4_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L4_M2_42 + +dtrmm_kernel_L4_M2_100: + + SAVE2x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #2 +#else + sub tempK, tempK, #4 +#endif + lsl temp, tempK, #4 + add pA, pA, temp + lsl temp, tempK, #5 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #2 +#endif + +dtrmm_kernel_L4_M2_END: + + +dtrmm_kernel_L4_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble dtrmm_kernel_L4_END + +dtrmm_kernel_L4_M1_20: + + INIT1x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #5 + add pB, pB, temp + lsl temp, tempOffset, #3 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #1 +#else + add tempK, tempOffset, #4 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dtrmm_kernel_L4_M1_40 + +dtrmm_kernel_L4_M1_22: + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L4_M1_22 + + +dtrmm_kernel_L4_M1_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble dtrmm_kernel_L4_M1_100 + +dtrmm_kernel_L4_M1_42: + + KERNEL1x4_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L4_M1_42 + +dtrmm_kernel_L4_M1_100: + + SAVE1x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #1 +#else + sub tempK, tempK, #4 +#endif + lsl temp, tempK, #3 + add pA, pA, temp + lsl temp, tempK, #5 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #1 +#endif + +dtrmm_kernel_L4_END: + + lsl temp, origK, #5 + add origPB, origPB, temp // B = B + K * 4 * 8 + +#if !defined(LEFT) + add tempOffset, tempOffset, #4 +#endif + + subs counterJ, counterJ , #1 // j-- + bgt dtrmm_kernel_L4_BEGIN + + +/******************************************************************************/ + +dtrmm_kernel_L2_BEGIN: // less than 2 left in N direction + + mov counterJ , origN + tst counterJ , #3 + ble dtrmm_kernel_L999 // error, N was less than 4? + + tst counterJ , #2 + ble dtrmm_kernel_L1_BEGIN + + mov pCRow0, pC // pCRow0 = pC + + add pC,pC,LDC, lsl #1 + +#if defined(LEFT) + mov tempOffset, offset +#endif + mov pA, origPA // pA = A + +dtrmm_kernel_L2_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 // counterI = counterI / 8 + cmp counterI, #0 + ble dtrmm_kernel_L2_M4_BEGIN + +dtrmm_kernel_L2_M8_20: + + INIT8x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #6 + add pA, pA, temp + lsl temp, tempOffset, #4 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #8 +#else + add tempK, tempOffset, #2 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble dtrmm_kernel_L2_M8_40 + .align 5 + +dtrmm_kernel_L2_M8_22: + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L2_M8_22 + + +dtrmm_kernel_L2_M8_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble dtrmm_kernel_L2_M8_100 + +dtrmm_kernel_L2_M8_42: + + KERNEL8x2_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L2_M8_42 + +dtrmm_kernel_L2_M8_100: + + SAVE8x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #8 +#else + sub tempK, tempK, #2 +#endif + lsl temp, tempK, #6 + add pA, pA, temp + lsl temp, tempK, #4 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #8 +#endif + +dtrmm_kernel_L2_M8_END: + + subs counterI, counterI, #1 + bgt dtrmm_kernel_L2_M8_20 + +dtrmm_kernel_L2_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble dtrmm_kernel_L2_END + + tst counterI, #4 // counterI = counterI / 2 + ble dtrmm_kernel_L2_M2_BEGIN + +dtrmm_kernel_L2_M4_20: + + INIT4x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #4 + add pB, pB, temp + lsl temp, tempOffset, #5 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #4 +#else + add tempK, tempOffset, #2 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble dtrmm_kernel_L2_M4_40 + .align 5 + +dtrmm_kernel_L2_M4_22: + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L2_M4_22 + + +dtrmm_kernel_L2_M4_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble dtrmm_kernel_L2_M4_100 + +dtrmm_kernel_L2_M4_42: + + KERNEL4x2_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L2_M4_42 + +dtrmm_kernel_L2_M4_100: + + SAVE4x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #4 +#else + sub tempK, tempK, #2 +#endif + lsl temp, tempK, #5 + add pA, pA, temp + lsl temp, tempK, #4 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #4 +#endif + +dtrmm_kernel_L2_M4_END: + + +dtrmm_kernel_L2_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble dtrmm_kernel_L2_END + + tst counterI, #2 // counterI = counterI / 2 + ble dtrmm_kernel_L2_M1_BEGIN + +dtrmm_kernel_L2_M2_20: + + INIT2x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #4 + add pB, pB, temp + lsl temp, tempOffset, #4 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #2 +#else + add tempK, tempOffset, #2 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble dtrmm_kernel_L2_M2_40 + +dtrmm_kernel_L2_M2_22: + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L2_M2_22 + + +dtrmm_kernel_L2_M2_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble dtrmm_kernel_L2_M2_100 + +dtrmm_kernel_L2_M2_42: + + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L2_M2_42 + +dtrmm_kernel_L2_M2_100: + + SAVE2x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #2 +#else + sub tempK, tempK, #2 +#endif + lsl temp, tempK, #4 + add pA, pA, temp + lsl temp, tempK, #4 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #2 +#endif + +dtrmm_kernel_L2_M2_END: + + +dtrmm_kernel_L2_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble dtrmm_kernel_L2_END + +dtrmm_kernel_L2_M1_20: + + INIT1x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #4 + add pB, pB, temp + lsl temp, tempOffset, #3 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #1 +#else + add tempK, tempOffset, #2 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL, #0 + ble dtrmm_kernel_L2_M1_40 + +dtrmm_kernel_L2_M1_22: + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L2_M1_22 + + +dtrmm_kernel_L2_M1_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble dtrmm_kernel_L2_M1_100 + +dtrmm_kernel_L2_M1_42: + + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L2_M1_42 + +dtrmm_kernel_L2_M1_100: + + SAVE1x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #1 +#else + sub tempK, tempK, #2 +#endif + lsl temp, tempK, #3 + add pA, pA, temp + lsl temp, tempK, #4 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #1 +#endif + +dtrmm_kernel_L2_END: +#if !defined(LEFT) + add tempOffset, tempOffset, #2 +#endif + add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 + +/******************************************************************************/ + +dtrmm_kernel_L1_BEGIN: + + mov counterJ , origN + tst counterJ , #1 + ble dtrmm_kernel_L999 // done + + mov pCRow0, pC // pCRow0 = C + add pC , pC , LDC // Update pC to point to next + +#if defined(LEFT) + mov tempOffset, offset +#endif + mov pA, origPA // pA = A + +dtrmm_kernel_L1_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 // counterI = counterI / 8 + cmp counterI, #0 + ble dtrmm_kernel_L1_M4_BEGIN + +dtrmm_kernel_L1_M8_20: + + INIT8x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #6 + add pA, pA, temp + lsl temp, tempOffset, #3 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #8 +#else + add tempK, tempOffset, #1 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dtrmm_kernel_L1_M8_40 + .align 5 + +dtrmm_kernel_L1_M8_22: + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L1_M8_22 + + +dtrmm_kernel_L1_M8_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble dtrmm_kernel_L1_M8_100 + +dtrmm_kernel_L1_M8_42: + + KERNEL8x1_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L1_M8_42 + +dtrmm_kernel_L1_M8_100: + + SAVE8x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #8 +#else + sub tempK, tempK, #1 +#endif + lsl temp, tempK, #6 + add pA, pA, temp + lsl temp, tempK, #3 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #8 +#endif + +dtrmm_kernel_L1_M8_END: + + subs counterI, counterI, #1 + bgt dtrmm_kernel_L1_M8_20 + +dtrmm_kernel_L1_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble dtrmm_kernel_L1_END + + tst counterI, #4 // counterI = counterI / 2 + ble dtrmm_kernel_L1_M2_BEGIN + +dtrmm_kernel_L1_M4_20: + + INIT4x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #3 + add pB, pB, temp + lsl temp, tempOffset, #5 + add pA, pA, temp +#endif +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #4 +#else + add tempK, tempOffset, #1 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dtrmm_kernel_L1_M4_40 + .align 5 + +dtrmm_kernel_L1_M4_22: + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L1_M4_22 + + +dtrmm_kernel_L1_M4_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble dtrmm_kernel_L1_M4_100 + +dtrmm_kernel_L1_M4_42: + + KERNEL4x1_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L1_M4_42 + +dtrmm_kernel_L1_M4_100: + + SAVE4x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #4 +#else + sub tempK, tempK, #1 +#endif + lsl temp, tempK, #5 + add pA, pA, temp + lsl temp, tempK, #3 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #4 +#endif + +dtrmm_kernel_L1_M4_END: + +dtrmm_kernel_L1_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble dtrmm_kernel_L1_END + + tst counterI, #2 // counterI = counterI / 2 + ble dtrmm_kernel_L1_M1_BEGIN + +dtrmm_kernel_L1_M2_20: + + INIT2x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #3 + add pB, pB, temp + lsl temp, tempOffset, #4 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #2 +#else + add tempK, tempOffset, #1 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dtrmm_kernel_L1_M2_40 + +dtrmm_kernel_L1_M2_22: + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L1_M2_22 + + +dtrmm_kernel_L1_M2_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble dtrmm_kernel_L1_M2_100 + +dtrmm_kernel_L1_M2_42: + + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L1_M2_42 + +dtrmm_kernel_L1_M2_100: + + SAVE2x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #2 +#else + sub tempK, tempK, #1 +#endif + lsl temp, tempK, #4 + add pA, pA, temp + lsl temp, tempK, #3 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #2 +#endif + +dtrmm_kernel_L1_M2_END: + + +dtrmm_kernel_L1_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble dtrmm_kernel_L1_END + +dtrmm_kernel_L1_M1_20: + + INIT1x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #3 + add pB, pB, temp + lsl temp, tempOffset, #3 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #1 +#else + add tempK, tempOffset, #1 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dtrmm_kernel_L1_M1_40 + +dtrmm_kernel_L1_M1_22: + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L1_M1_22 + + +dtrmm_kernel_L1_M1_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble dtrmm_kernel_L1_M1_100 + +dtrmm_kernel_L1_M1_42: + + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt dtrmm_kernel_L1_M1_42 + +dtrmm_kernel_L1_M1_100: + + SAVE1x1 + + +dtrmm_kernel_L1_END: + + +dtrmm_kernel_L999: + mov x0, #0 // set return value + ldp d8, d9, [sp, #(0 * 16)] + ldp d10, d11, [sp, #(1 * 16)] + ldp d12, d13, [sp, #(2 * 16)] + ldp d14, d15, [sp, #(3 * 16)] + ldp d16, d17, [sp, #(4 * 16)] + ldp x18, x19, [sp, #(5 * 16)] + ldp x20, x21, [sp, #(6 * 16)] + ldp x22, x23, [sp, #(7 * 16)] + ldp x24, x25, [sp, #(8 * 16)] + ldp x26, x27, [sp, #(9 * 16)] + ldr x28, [sp, #(10 * 16)] + add sp, sp, #(11*16) + ret + + EPILOGUE + diff --git a/kernel/arm64/sgemm_kernel_16x4.S b/kernel/arm64/sgemm_kernel_16x4.S new file mode 100644 index 000000000..22b55b01c --- /dev/null +++ b/kernel/arm64/sgemm_kernel_16x4.S @@ -0,0 +1,1987 @@ +/******************************************************************************* +Copyright (c) 2015, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#define ASSEMBLER +#include "common.h" + +/* X0 X1 X2 s0 X3 x4 x5 x6 */ +/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc) */ + +#define origM x0 +#define origN x1 +#define origK x2 +#define origPA x3 +#define origPB x4 +#define pC x5 +#define LDC x6 +#define temp x7 +#define counterL x8 +#define counterI x9 +#define counterJ x10 +#define pB x11 +#define pCRow0 x12 +#define pCRow1 x13 +#define pCRow2 x14 +#define pA x15 + +#define alpha0 s10 +#define alphaV0 v10.s[0] +#define alpha1 s11 +#define alphaV1 v11.s[0] +#define alpha2 s14 +#define alphaV2 v14.s[0] +#define alpha3 s15 +#define alphaV3 v15.s[0] + +// 00 origM +// 01 origN +// 02 origK +// 03 origPA +// 04 origPB +// 05 pC +// 06 origLDC -> LDC +// 07 offset +// 08 counterL +// 09 counterI +// 10 counterJ +// 11 pB +// 12 pCRow0 +// 13 pCRow1 +// 14 pCRow2 +// 15 pA +// 16 temp +// 17 +// 18 must save +// 19 must save +// 20 must save +// 21 must save +// 22 must save +// 23 must save +// 24 must save +// 25 must save +// 26 must save +// 27 must save +// 28 must save +// 29 frame +// 30 link +// 31 sp + +//v00 ALPHA -> pA0_00, pA0_01, pA0_02, pA0_03 +//v01 pA0_04, pA0_05, pA0_06, pA0_07 +//v02 pA0_08, pA0_09, pA0_10, pA0_11 +//v03 pA0_12, pA0_13, pA0_14, pA0_15 +//v04 pA1_00, pA1_01, pA1_02, pA1_03 +//v05 pA1_04, pA1_05, pA1_06, pA1_07 +//v06 pA1_08, pA1_09, pA1_10, pA1_11 +//v07 pA1_12, pA1_13, pA1_14, pA1_15 +//v08 must save pB00, pB01 +//v09 must save pB02, pB03 +//v10 must save ALPHA0 +//v11 must save ALPHA1 +//v12 must save pB10, pB11 +//v13 must save pB12, pB13 +//v14 must save ALPHA2 +//v15 must save ALPHA3 +//v16 must save C00, C01, C02, C03 +//v17 must save C04, C05, C06, C07 +//v18 C08, C09, C10, C11 +//v19 C12, C13, C14, C15 +//v20 C16, C17, C18, C19 +//v21 C20, C21, C22, C23 +//v22 C24, C25, C26, C27 +//v23 C28, C29, C30, C31 +//v24 C32, C33, C34, C35 +//v25 C36, C37, C38, C39 +//v26 C40, C41, C42, C43 +//v27 C44, C45, C46, C47 +//v28 C48, C49, C50, C51 +//v29 C52, C53, C54, C55 +//v30 C56, C57, C58, C59 +//v31 C60, C61, C62, C63 + +/******************************************************************************* +* Macro definitions +*******************************************************************************/ + +.macro INIT16x4 + fmov s16, wzr + fmov s17, wzr + fmov s18, s16 + fmov s19, s17 + fmov s20, wzr + fmov s21, s16 + fmov s22, s17 + fmov s23, s18 + fmov s24, wzr + fmov s25, s16 + fmov s26, s17 + fmov s27, s18 + fmov s28, wzr + fmov s29, s16 + fmov s30, s17 + fmov s31, s18 +.endm + +.macro KERNEL16x4_I + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + ld1 {v2.4s}, [pA] + add pA, pA, #16 + ld1 {v3.4s}, [pA] + add pA, pA, #16 + + fmul v16.4s, v0.4s, v8.2s[0] + fmul v17.4s, v1.4s, v8.2s[0] + fmul v18.4s, v2.4s, v8.2s[0] + fmul v19.4s, v3.4s, v8.2s[0] + + fmul v20.4s, v0.4s, v8.2s[1] + fmul v21.4s, v1.4s, v8.2s[1] + fmul v22.4s, v2.4s, v8.2s[1] + fmul v23.4s, v3.4s, v8.2s[1] + + fmul v24.4s, v0.4s, v9.2s[0] + fmul v25.4s, v1.4s, v9.2s[0] + fmul v26.4s, v2.4s, v9.2s[0] + fmul v27.4s, v3.4s, v9.2s[0] + + fmul v28.4s, v0.4s, v9.2s[1] + fmul v29.4s, v1.4s, v9.2s[1] + fmul v30.4s, v2.4s, v9.2s[1] + fmul v31.4s, v3.4s, v9.2s[1] + + ld1 {v12.2s, v13.2s}, [pB] + add pB, pB, #16 + ld1 {v4.4s}, [pA] + add pA, pA, #16 + ld1 {v5.4s}, [pA] + add pA, pA, #16 + ld1 {v6.4s}, [pA] + add pA, pA, #16 + ld1 {v7.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL16x4_M1 + fmla v16.4s, v0.4s, v8.2s[0] + fmla v17.4s, v1.4s, v8.2s[0] + fmla v18.4s, v2.4s, v8.2s[0] + fmla v19.4s, v3.4s, v8.2s[0] + + fmla v20.4s, v0.4s, v8.2s[1] + fmla v21.4s, v1.4s, v8.2s[1] + fmla v22.4s, v2.4s, v8.2s[1] + fmla v23.4s, v3.4s, v8.2s[1] + + fmla v24.4s, v0.4s, v9.2s[0] + fmla v25.4s, v1.4s, v9.2s[0] + fmla v26.4s, v2.4s, v9.2s[0] + fmla v27.4s, v3.4s, v9.2s[0] + + fmla v28.4s, v0.4s, v9.2s[1] + fmla v29.4s, v1.4s, v9.2s[1] + fmla v30.4s, v2.4s, v9.2s[1] + fmla v31.4s, v3.4s, v9.2s[1] + + ld1 {v12.2s, v13.2s}, [pB] + add pB, pB, #16 + ld1 {v4.4s}, [pA] + add pA, pA, #16 + ld1 {v5.4s}, [pA] + add pA, pA, #16 + ld1 {v6.4s}, [pA] + add pA, pA, #16 + ld1 {v7.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL16x4_M2 + fmla v16.4s, v4.4s, v12.2s[0] + fmla v17.4s, v5.4s, v12.2s[0] + fmla v18.4s, v6.4s, v12.2s[0] + fmla v19.4s, v7.4s, v12.2s[0] + + fmla v20.4s, v4.4s, v12.2s[1] + fmla v21.4s, v5.4s, v12.2s[1] + fmla v22.4s, v6.4s, v12.2s[1] + fmla v23.4s, v7.4s, v12.2s[1] + + fmla v24.4s, v4.4s, v13.2s[0] + fmla v25.4s, v5.4s, v13.2s[0] + fmla v26.4s, v6.4s, v13.2s[0] + fmla v27.4s, v7.4s, v13.2s[0] + + fmla v28.4s, v4.4s, v13.2s[1] + fmla v29.4s, v5.4s, v13.2s[1] + fmla v30.4s, v6.4s, v13.2s[1] + fmla v31.4s, v7.4s, v13.2s[1] + + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + ld1 {v2.4s}, [pA] + add pA, pA, #16 + ld1 {v3.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL16x4_E + fmla v16.4s, v4.4s, v12.2s[0] + fmla v17.4s, v5.4s, v12.2s[0] + fmla v18.4s, v6.4s, v12.2s[0] + fmla v19.4s, v7.4s, v12.2s[0] + + fmla v20.4s, v4.4s, v12.2s[1] + fmla v21.4s, v5.4s, v12.2s[1] + fmla v22.4s, v6.4s, v12.2s[1] + fmla v23.4s, v7.4s, v12.2s[1] + + fmla v24.4s, v4.4s, v13.2s[0] + fmla v25.4s, v5.4s, v13.2s[0] + fmla v26.4s, v6.4s, v13.2s[0] + fmla v27.4s, v7.4s, v13.2s[0] + + fmla v28.4s, v4.4s, v13.2s[1] + fmla v29.4s, v5.4s, v13.2s[1] + fmla v30.4s, v6.4s, v13.2s[1] + fmla v31.4s, v7.4s, v13.2s[1] +.endm + +.macro KERNEL16x4_SUB + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + ld1 {v2.4s}, [pA] + add pA, pA, #16 + ld1 {v3.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v8.2s[0] + fmla v17.4s, v1.4s, v8.2s[0] + fmla v18.4s, v2.4s, v8.2s[0] + fmla v19.4s, v3.4s, v8.2s[0] + + fmla v20.4s, v0.4s, v8.2s[1] + fmla v21.4s, v1.4s, v8.2s[1] + fmla v22.4s, v2.4s, v8.2s[1] + fmla v23.4s, v3.4s, v8.2s[1] + + fmla v24.4s, v0.4s, v9.2s[0] + fmla v25.4s, v1.4s, v9.2s[0] + fmla v26.4s, v2.4s, v9.2s[0] + fmla v27.4s, v3.4s, v9.2s[0] + + fmla v28.4s, v0.4s, v9.2s[1] + fmla v29.4s, v1.4s, v9.2s[1] + fmla v30.4s, v2.4s, v9.2s[1] + fmla v31.4s, v3.4s, v9.2s[1] +.endm + +.macro SAVE16x4 + add pCRow1, pCRow0, LDC + + ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow0] + fmla v0.4s, v16.4s, alphaV0 + fmla v1.4s, v17.4s, alphaV1 + fmla v2.4s, v18.4s, alphaV2 + fmla v3.4s, v19.4s, alphaV3 + st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow0] + + add pCRow2, pCRow1, LDC + + ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [pCRow1] + fmla v4.4s, v20.4s, alphaV0 + fmla v5.4s, v21.4s, alphaV1 + fmla v6.4s, v22.4s, alphaV2 + fmla v7.4s, v23.4s, alphaV3 + st1 {v4.4s, v5.4s, v6.4s, v7.4s}, [pCRow1] + + add pCRow1, pCRow2, LDC + + ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow2] + fmla v0.4s, v24.4s, alphaV0 + fmla v1.4s, v25.4s, alphaV1 + fmla v2.4s, v26.4s, alphaV2 + fmla v3.4s, v27.4s, alphaV3 + st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow2] + + ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [pCRow1] + fmla v4.4s, v28.4s, alphaV0 + fmla v5.4s, v29.4s, alphaV1 + fmla v6.4s, v30.4s, alphaV2 + fmla v7.4s, v31.4s, alphaV3 + st1 {v4.4s, v5.4s, v6.4s, v7.4s}, [pCRow1] + + add pCRow0, pCRow0, #64 +.endm + +/******************************************************************************/ + +.macro INIT8x4 + fmov s16, wzr + fmov s17, wzr + fmov s20, wzr + fmov s21, s16 + fmov s24, wzr + fmov s25, s16 + fmov s28, wzr + fmov s29, s16 +.endm + +.macro KERNEL8x4_I + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + + fmul v16.4s, v0.4s, v8.2s[0] + fmul v17.4s, v1.4s, v8.2s[0] + fmul v20.4s, v0.4s, v8.2s[1] + fmul v21.4s, v1.4s, v8.2s[1] + fmul v24.4s, v0.4s, v9.2s[0] + fmul v25.4s, v1.4s, v9.2s[0] + fmul v28.4s, v0.4s, v9.2s[1] + fmul v29.4s, v1.4s, v9.2s[1] + + ld1 {v12.2s, v13.2s}, [pB] + add pB, pB, #16 + ld1 {v4.4s}, [pA] + add pA, pA, #16 + ld1 {v5.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL8x4_M1 + fmla v16.4s, v0.4s, v8.2s[0] + fmla v17.4s, v1.4s, v8.2s[0] + fmla v20.4s, v0.4s, v8.2s[1] + fmla v21.4s, v1.4s, v8.2s[1] + fmla v24.4s, v0.4s, v9.2s[0] + fmla v25.4s, v1.4s, v9.2s[0] + fmla v28.4s, v0.4s, v9.2s[1] + fmla v29.4s, v1.4s, v9.2s[1] + + ld1 {v12.2s, v13.2s}, [pB] + add pB, pB, #16 + ld1 {v4.4s}, [pA] + add pA, pA, #16 + ld1 {v5.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL8x4_M2 + fmla v16.4s, v4.4s, v12.2s[0] + fmla v17.4s, v5.4s, v12.2s[0] + fmla v20.4s, v4.4s, v12.2s[1] + fmla v21.4s, v5.4s, v12.2s[1] + fmla v24.4s, v4.4s, v13.2s[0] + fmla v25.4s, v5.4s, v13.2s[0] + fmla v28.4s, v4.4s, v13.2s[1] + fmla v29.4s, v5.4s, v13.2s[1] + + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL8x4_E + fmla v16.4s, v4.4s, v12.2s[0] + fmla v17.4s, v5.4s, v12.2s[0] + fmla v20.4s, v4.4s, v12.2s[1] + fmla v21.4s, v5.4s, v12.2s[1] + fmla v24.4s, v4.4s, v13.2s[0] + fmla v25.4s, v5.4s, v13.2s[0] + fmla v28.4s, v4.4s, v13.2s[1] + fmla v29.4s, v5.4s, v13.2s[1] +.endm + +.macro KERNEL8x4_SUB + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v8.2s[0] + fmla v17.4s, v1.4s, v8.2s[0] + fmla v20.4s, v0.4s, v8.2s[1] + fmla v21.4s, v1.4s, v8.2s[1] + fmla v24.4s, v0.4s, v9.2s[0] + fmla v25.4s, v1.4s, v9.2s[0] + fmla v28.4s, v0.4s, v9.2s[1] + fmla v29.4s, v1.4s, v9.2s[1] +.endm + +.macro SAVE8x4 + add pCRow1, pCRow0, LDC + + ld1 {v0.4s, v1.4s}, [pCRow0] + fmla v0.4s, v16.4s, alphaV0 + fmla v1.4s, v17.4s, alphaV1 + st1 {v0.4s, v1.4s}, [pCRow0] + + add pCRow2, pCRow1, LDC + + ld1 {v4.4s, v5.4s}, [pCRow1] + fmla v4.4s, v20.4s, alphaV0 + fmla v5.4s, v21.4s, alphaV1 + st1 {v4.4s, v5.4s}, [pCRow1] + + add pCRow1, pCRow2, LDC + + ld1 {v0.4s, v1.4s}, [pCRow2] + fmla v0.4s, v24.4s, alphaV0 + fmla v1.4s, v25.4s, alphaV1 + st1 {v0.4s, v1.4s}, [pCRow2] + + ld1 {v4.4s, v5.4s}, [pCRow1] + fmla v4.4s, v28.4s, alphaV0 + fmla v5.4s, v29.4s, alphaV1 + st1 {v4.4s, v5.4s}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT4x4 + fmov s16, wzr + fmov s17, s16 + fmov s20, s17 + fmov s21, s16 + fmov s24, s17 + fmov s25, s16 + fmov s28, s17 + fmov s29, s16 +.endm + +.macro KERNEL4x4_I + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.2s, v1.2s}, [pA] + add pA, pA, #16 + + fmul v16.2s, v0.2s, v8.2s[0] + fmul v29.2s, v1.2s, v9.2s[1] + + fmul v20.2s, v0.2s, v8.2s[1] + fmul v25.2s, v1.2s, v9.2s[0] + + fmul v24.2s, v0.2s, v9.2s[0] + fmul v21.2s, v1.2s, v8.2s[1] + + fmul v28.2s, v0.2s, v9.2s[1] + fmul v17.2s, v1.2s, v8.2s[0] + + ld1 {v12.2s, v13.2s}, [pB] + add pB, pB, #16 + ld1 {v4.2s, v5.2s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL4x4_M1 + fmla v16.2s, v0.2s, v8.2s[0] + fmla v29.2s, v1.2s, v9.2s[1] + + ld1 {v12.2s, v13.2s}, [pB] // For next round + add pB, pB, #16 + + fmla v20.2s, v0.2s, v8.2s[1] + fmla v25.2s, v1.2s, v9.2s[0] + + ld1 {v4.2s, v5.2s}, [pA] // For next round + add pA, pA, #16 + + fmla v24.2s, v0.2s, v9.2s[0] + fmla v21.2s, v1.2s, v8.2s[1] + + prfm PLDL1KEEP, [pB, #512] + + fmla v28.2s, v0.2s, v9.2s[1] + fmla v17.2s, v1.2s, v8.2s[0] +.endm + +.macro KERNEL4x4_M2 + fmla v16.2s, v4.2s, v12.2s[0] + fmla v29.2s, v5.2s, v13.2s[1] + + ld1 {v8.2s, v9.2s}, [pB] // For next round + add pB, pB, #16 + + fmla v20.2s, v4.2s, v12.2s[1] + fmla v25.2s, v5.2s, v13.2s[0] + + ld1 {v0.2s, v1.2s}, [pA] // For next round + add pA, pA, #16 + + fmla v24.2s, v4.2s, v13.2s[0] + fmla v21.2s, v5.2s, v12.2s[1] + + prfm PLDL1KEEP, [pA, #512] + + fmla v28.2s, v4.2s, v13.2s[1] + fmla v17.2s, v5.2s, v12.2s[0] +.endm + +.macro KERNEL4x4_E + fmla v16.2s, v4.2s, v12.2s[0] + fmla v29.2s, v5.2s, v13.2s[1] + + fmla v20.2s, v4.2s, v12.2s[1] + fmla v25.2s, v5.2s, v13.2s[0] + + fmla v24.2s, v4.2s, v13.2s[0] + fmla v21.2s, v5.2s, v12.2s[1] + + fmla v28.2s, v4.2s, v13.2s[1] + fmla v17.2s, v5.2s, v12.2s[0] +.endm + +.macro KERNEL4x4_SUB + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.2s, v1.2s}, [pA] + add pA, pA, #16 + + fmla v16.2s, v0.2s, v8.2s[0] + fmla v29.2s, v1.2s, v9.2s[1] + + fmla v20.2s, v0.2s, v8.2s[1] + fmla v25.2s, v1.2s, v9.2s[0] + + fmla v24.2s, v0.2s, v9.2s[0] + fmla v21.2s, v1.2s, v8.2s[1] + + fmla v28.2s, v0.2s, v9.2s[1] + fmla v17.2s, v1.2s, v8.2s[0] +.endm + +.macro SAVE4x4 + ld1 {v8.2s, v9.2s}, [pCRow0] + fmla v8.2s, v16.2s, alphaV0 + fmla v9.2s, v17.2s, alphaV1 + st1 {v8.2s, v9.2s}, [pCRow0] + + add pCRow1, pCRow0, LDC + ld1 {v12.2s, v13.2s}, [pCRow1] + fmla v12.2s, v20.2s, alphaV2 + fmla v13.2s, v21.2s, alphaV3 + st1 {v12.2s, v13.2s}, [pCRow1] + + add pCRow2, pCRow1, LDC + ld1 {v8.2s, v9.2s}, [pCRow2] + fmla v8.2s, v24.2s, alphaV0 + fmla v9.2s, v25.2s, alphaV1 + st1 {v8.2s, v9.2s}, [pCRow2] + + add pCRow1, pCRow2, LDC + ld1 {v12.2s, v13.2s}, [pCRow1] + fmla v12.2s, v28.2s, alphaV2 + fmla v13.2s, v29.2s, alphaV3 + st1 {v12.2s, v13.2s}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT2x4 + fmov s16, wzr + fmov s20, s16 + fmov s24, s20 + fmov s28, s16 +.endm + +.macro KERNEL2x4_SUB + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.2s}, [pA] + add pA, pA, #8 + + fmla v16.2s, v0.2s, v8.2s[0] + fmla v20.2s, v0.2s, v8.2s[1] + fmla v24.2s, v0.2s, v9.2s[0] + fmla v28.2s, v0.2s, v9.2s[1] +.endm + +.macro SAVE2x4 + ld1 {v8.2s}, [pCRow0] + fmla v8.2s, v16.2s, alphaV0 + st1 {v8.2s}, [pCRow0] + + add pCRow1, pCRow0, LDC + ld1 {v12.2s}, [pCRow1] + fmla v12.2s, v20.2s, alphaV1 + st1 {v12.2s}, [pCRow1] + + add pCRow2, pCRow1, LDC + ld1 {v8.2s}, [pCRow2] + fmla v8.2s, v24.2s, alphaV2 + st1 {v8.2s}, [pCRow2] + + add pCRow1, pCRow2, LDC + ld1 {v12.2s}, [pCRow1] + fmla v12.2s, v28.2s, alphaV3 + st1 {v12.2s}, [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT1x4 + fmov s16, wzr + fmov s20, s16 +.endm + +.macro KERNEL1x4_SUB + ldr s0, [pA] + add pA, pA, #4 + + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + + fmla v16.2s, v8.2s, v0.s[0] + fmla v20.2s, v9.2s, v0.s[0] +.endm + +.macro SAVE1x4 + add pCRow1, pCRow0, LDC + ld1 {v8.s}[0], [pCRow0] + ld1 {v8.s}[1], [pCRow1] + fmla v8.2s, v16.2s, alphaV0 + st1 {v8.s}[0], [pCRow0] + st1 {v8.s}[1], [pCRow1] + + add pCRow2, pCRow1, LDC + add pCRow1, pCRow2, LDC + ld1 {v12.s}[0], [pCRow2] + ld1 {v12.s}[1], [pCRow1] + fmla v12.2s, v20.2s, alphaV1 + st1 {v12.s}[0], [pCRow2] + st1 {v12.s}[1], [pCRow1] + + add pCRow0, pCRow0, #4 +.endm + +/******************************************************************************/ + +.macro INIT16x2 + fmov s16, wzr + fmov s17, wzr + fmov s18, wzr + fmov s19, s16 + fmov s20, wzr + fmov s21, s16 + fmov s22, wzr + fmov s23, s16 +.endm + +.macro KERNEL16x2_SUB + ld1 {v8.2s}, [pB] + add pB, pB, #8 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + ld1 {v2.4s}, [pA] + add pA, pA, #16 + ld1 {v3.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v8.2s[0] + fmla v17.4s, v1.4s, v8.2s[0] + fmla v18.4s, v2.4s, v8.2s[0] + fmla v19.4s, v3.4s, v8.2s[0] + + fmla v20.4s, v0.4s, v8.2s[1] + fmla v21.4s, v1.4s, v8.2s[1] + fmla v22.4s, v2.4s, v8.2s[1] + fmla v23.4s, v3.4s, v8.2s[1] +.endm + +.macro SAVE16x2 + add pCRow1, pCRow0, LDC + + ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow0] + fmla v0.4s, v16.4s, alphaV0 + fmla v1.4s, v17.4s, alphaV1 + fmla v2.4s, v18.4s, alphaV2 + fmla v3.4s, v19.4s, alphaV3 + st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow0] + + ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [pCRow1] + fmla v4.4s, v20.4s, alphaV0 + fmla v5.4s, v21.4s, alphaV1 + fmla v6.4s, v22.4s, alphaV2 + fmla v7.4s, v23.4s, alphaV3 + st1 {v4.4s, v5.4s, v6.4s, v7.4s}, [pCRow1] + + add pCRow0, pCRow0, #64 +.endm + +/******************************************************************************/ + +.macro INIT8x2 + fmov s16, wzr + fmov s17, s16 + fmov s20, s17 + fmov s21, s16 +.endm + +.macro KERNEL8x2_SUB + ld1 {v8.2s}, [pB] + add pB, pB, #8 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v8.2s[0] + fmla v17.4s, v1.4s, v8.2s[0] + + fmla v20.4s, v0.4s, v8.2s[1] + fmla v21.4s, v1.4s, v8.2s[1] +.endm + +.macro SAVE8x2 + add pCRow1, pCRow0, LDC + + ld1 {v0.4s, v1.4s}, [pCRow0] + fmla v0.4s, v16.4s, alphaV0 + fmla v1.4s, v17.4s, alphaV1 + st1 {v0.4s, v1.4s}, [pCRow0] + + add pCRow2, pCRow1, LDC + + ld1 {v4.4s, v5.4s}, [pCRow1] + fmla v4.4s, v20.4s, alphaV0 + fmla v5.4s, v21.4s, alphaV1 + st1 {v4.4s, v5.4s}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT4x2 + fmov s16, wzr + fmov s17, s16 + fmov s20, s17 + fmov s21, s16 +.endm + +.macro KERNEL4x2_SUB + ld1 {v8.2s}, [pB] + add pB, pB, #8 + ld1 {v0.2s, v1.2s}, [pA] + add pA, pA, #16 + + fmla v16.2s, v0.2s, v8.2s[0] + fmla v17.2s, v1.2s, v8.2s[0] + fmla v20.2s, v0.2s, v8.2s[1] + fmla v21.2s, v1.2s, v8.2s[1] +.endm + +.macro SAVE4x2 + ld1 {v8.2s, v9.2s}, [pCRow0] + fmla v8.2s, v16.2s, alphaV0 + fmla v9.2s, v17.2s, alphaV1 + st1 {v8.2s, v9.2s}, [pCRow0] + + add pCRow1, pCRow0, LDC + ld1 {v12.2s, v13.2s}, [pCRow1] + fmla v12.2s, v20.2s, alphaV2 + fmla v13.2s, v21.2s, alphaV3 + st1 {v12.2s, v13.2s}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT2x2 + fmov s16, wzr + fmov s20, s16 +.endm + +.macro KERNEL2x2_SUB + ld1 {v8.2s}, [pB] + add pB, pB, #8 + + ld1 {v0.2s}, [pA] + add pA, pA, #8 + + fmla v16.2s, v0.2s, v8.2s[0] + fmla v20.2s, v0.2s, v8.2s[1] +.endm + +.macro SAVE2x2 + ld1 {v8.2s}, [pCRow0] + fmla v8.2s, v16.2s, alphaV0 + st1 {v8.2s}, [pCRow0] + + add pCRow1 , pCRow0, LDC + ld1 {v12.2s}, [pCRow1] + fmla v12.2s, v20.2s, alphaV1 + st1 {v12.2s}, [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT1x2 + fmov s16, wzr +.endm + +.macro KERNEL1x2_SUB + ld1 {v8.2s} , [pB] + add pB , pB, #8 + + ldr s0 , [pA] + add pA, pA, #4 + + fmla v16.2s, v8.2s, v0.2s[0] +.endm + +.macro SAVE1x2 + add pCRow1 , pCRow0, LDC + ld1 {v8.s}[0], [pCRow0] + ld1 {v8.s}[1], [pCRow1] + fmla v8.2s, v16.2s, alphaV0 + st1 {v8.s}[0], [pCRow0] + st1 {v8.s}[1], [pCRow1] + + add pCRow0, pCRow0, #4 +.endm + +/******************************************************************************/ + +.macro INIT16x1 + fmov s16, wzr + fmov s17, wzr + fmov s18, wzr + fmov s19, s16 +.endm + +.macro KERNEL16x1_SUB + ldr s8, [pB] + add pB , pB, #4 + + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + ld1 {v2.4s}, [pA] + add pA, pA, #16 + ld1 {v3.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v8.2s[0] + fmla v17.4s, v1.4s, v8.2s[0] + fmla v18.4s, v2.4s, v8.2s[0] + fmla v19.4s, v3.4s, v8.2s[0] +.endm + +.macro SAVE16x1 + ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow0] + fmla v0.4s, v16.4s, alphaV0 + fmla v1.4s, v17.4s, alphaV1 + fmla v2.4s, v18.4s, alphaV2 + fmla v3.4s, v19.4s, alphaV3 + st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow0] + + add pCRow0, pCRow0, #64 +.endm + +/******************************************************************************/ + +.macro INIT8x1 + fmov s16, wzr + fmov s17, wzr +.endm + +.macro KERNEL8x1_SUB + ldr s8, [pB] + add pB , pB, #4 + + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v8.2s[0] + fmla v17.4s, v1.4s, v8.2s[0] +.endm + +.macro SAVE8x1 + ld1 {v0.4s, v1.4s}, [pCRow0] + fmla v0.4s, v16.4s, alphaV0 + fmla v1.4s, v17.4s, alphaV1 + st1 {v0.4s, v1.4s}, [pCRow0] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT4x1 + fmov s16, wzr + fmov s17, s16 +.endm + +.macro KERNEL4x1_SUB + ldr s8, [pB] + add pB , pB, #4 + + ld1 {v0.2s, v1.2s}, [pA] + add pA , pA, #16 + + fmla v16.2s, v0.2s, v8.2s[0] + fmla v17.2s, v1.2s, v8.2s[0] +.endm + +.macro SAVE4x1 + ld1 {v8.2s, v9.2s}, [pCRow0] + fmla v8.2s, v16.2s, alphaV0 + fmla v9.2s, v17.2s, alphaV1 + st1 {v8.2s, v9.2s}, [pCRow0] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT2x1 + fmov s16, wzr +.endm + +.macro KERNEL2x1_SUB + ldr s8, [pB] + add pB , pB, #4 + + ld1 {v0.2s}, [pA] + add pA , pA, #8 + + fmla v16.2s, v0.2s, v8.2s[0] +.endm + +.macro SAVE2x1 + ld1 {v8.2s}, [pCRow0] + fmla v8.2s, v16.2s, alphaV0 + st1 {v8.2s}, [pCRow0] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT1x1 + fmov s16, wzr +.endm + +.macro KERNEL1x1_SUB + ldr s8, [pB] + add pB , pB, #4 + + ldr s0, [pA] + add pA , pA, #4 + + fmadd s16, s0, s8, s16 +.endm + +.macro SAVE1x1 + ldr s8, [pCRow0] + fmla s8, s16, alphaV0 + str s8, [pCRow0] + + add pCRow0, pCRow0, #4 +.endm + +/******************************************************************************* +* End of macro definitions +*******************************************************************************/ + + PROLOGUE + +sgemm_kernel_begin: + + .align 5 + add sp, sp, #-(11 * 16) + stp d8, d9, [sp, #(0 * 16)] + stp d10, d11, [sp, #(1 * 16)] + stp d12, d13, [sp, #(2 * 16)] + stp d14, d15, [sp, #(3 * 16)] + stp d16, d17, [sp, #(4 * 16)] + stp x18, x19, [sp, #(5 * 16)] + stp x20, x21, [sp, #(6 * 16)] + stp x22, x23, [sp, #(7 * 16)] + stp x24, x25, [sp, #(8 * 16)] + stp x26, x27, [sp, #(9 * 16)] + str x28, [sp, #(10 * 16)] + + fmov alpha0, s0 + fmov alpha1, s0 + fmov alpha2, s0 + fmov alpha3, s0 + + lsl LDC, LDC, #2 // ldc = ldc * 4 + + mov pB, origPB + + mov counterJ, origN + asr counterJ, counterJ, #2 // J = J / 4 + cmp counterJ, #0 + ble sgemm_kernel_L2_BEGIN + +/******************************************************************************/ + +sgemm_kernel_L4_BEGIN: + mov pCRow0, pC // pCRow0 = C + add pC, pC, LDC, lsl #2 + + mov pA, origPA // pA = start of A array + +sgemm_kernel_L4_M16_BEGIN: + + mov counterI, origM + asr counterI, counterI, #4 // counterI = counterI / 16 + cmp counterI, #0 + ble sgemm_kernel_L4_M8_BEGIN + +sgemm_kernel_L4_M16_20: + + mov pB, origPB + + asr counterL , origK, #1 // L = K / 2 + cmp counterL , #2 // is there at least 4 to do? + blt sgemm_kernel_L4_M16_32 + + KERNEL16x4_I // do one in the K + KERNEL16x4_M2 // do another in the K + + subs counterL, counterL, #2 + ble sgemm_kernel_L4_M16_22a + .align 5 + +sgemm_kernel_L4_M16_22: + + KERNEL16x4_M1 + KERNEL16x4_M2 + + subs counterL, counterL, #1 + bgt sgemm_kernel_L4_M16_22 + +sgemm_kernel_L4_M16_22a: + + KERNEL16x4_M1 + KERNEL16x4_E + + b sgemm_kernel_L4_M16_44 + +sgemm_kernel_L4_M16_32: + + tst counterL, #1 + ble sgemm_kernel_L4_M16_40 + + KERNEL16x4_I + KERNEL16x4_E + + b sgemm_kernel_L4_M16_44 + +sgemm_kernel_L4_M16_40: + + INIT16x4 + +sgemm_kernel_L4_M16_44: + + ands counterL , origK, #1 + ble sgemm_kernel_L4_M16_100 + +sgemm_kernel_L4_M16_46: + + KERNEL16x4_SUB + +sgemm_kernel_L4_M16_100: + + SAVE16x4 + +sgemm_kernel_L4_M16_END: + subs counterI, counterI, #1 + bne sgemm_kernel_L4_M16_20 + +//------------------------------------------------------------------------------ + +sgemm_kernel_L4_M8_BEGIN: + + mov counterI, origM + tst counterI , #15 + ble sgemm_kernel_L4_END + + tst counterI, #8 + ble sgemm_kernel_L4_M4_BEGIN + +sgemm_kernel_L4_M8_20: + + mov pB, origPB + + asr counterL , origK, #1 // L = K / 2 + cmp counterL , #2 // is there at least 4 to do? + blt sgemm_kernel_L4_M8_32 + + KERNEL8x4_I // do one in the K + KERNEL8x4_M2 // do another in the K + + subs counterL, counterL, #2 + ble sgemm_kernel_L4_M8_22a + .align 5 + +sgemm_kernel_L4_M8_22: + + KERNEL8x4_M1 + KERNEL8x4_M2 + + subs counterL, counterL, #1 + bgt sgemm_kernel_L4_M8_22 + +sgemm_kernel_L4_M8_22a: + + KERNEL8x4_M1 + KERNEL8x4_E + + b sgemm_kernel_L4_M8_44 + +sgemm_kernel_L4_M8_32: + + tst counterL, #1 + ble sgemm_kernel_L4_M8_40 + + KERNEL8x4_I + KERNEL8x4_E + + b sgemm_kernel_L4_M8_44 + +sgemm_kernel_L4_M8_40: + + INIT8x4 + +sgemm_kernel_L4_M8_44: + + ands counterL , origK, #1 + ble sgemm_kernel_L4_M8_100 + +sgemm_kernel_L4_M8_46: + + KERNEL8x4_SUB + +sgemm_kernel_L4_M8_100: + + SAVE8x4 + +sgemm_kernel_L4_M8_END: + +//------------------------------------------------------------------------------ + +sgemm_kernel_L4_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble sgemm_kernel_L4_END + + tst counterI, #4 + ble sgemm_kernel_L4_M2_BEGIN + +sgemm_kernel_L4_M4_20: + + mov pB, origPB + + asr counterL , origK, #1 // L = K / 2 + cmp counterL , #2 // is there at least 4 to do? + blt sgemm_kernel_L4_M4_32 + + KERNEL4x4_I // do one in the K + KERNEL4x4_M2 // do another in the K + + subs counterL, counterL, #2 + ble sgemm_kernel_L4_M4_22a + .align 5 + +sgemm_kernel_L4_M4_22: + + KERNEL4x4_M1 + KERNEL4x4_M2 + + subs counterL, counterL, #1 + bgt sgemm_kernel_L4_M4_22 + +sgemm_kernel_L4_M4_22a: + + KERNEL4x4_M1 + KERNEL4x4_E + + b sgemm_kernel_L4_M4_44 + +sgemm_kernel_L4_M4_32: + + tst counterL, #1 + ble sgemm_kernel_L4_M4_40 + + KERNEL4x4_I + KERNEL4x4_E + + b sgemm_kernel_L4_M4_44 + +sgemm_kernel_L4_M4_40: + + INIT4x4 + +sgemm_kernel_L4_M4_44: + + ands counterL , origK, #1 + ble sgemm_kernel_L4_M4_100 + +sgemm_kernel_L4_M4_46: + + KERNEL4x4_SUB + +sgemm_kernel_L4_M4_100: + + SAVE4x4 + +sgemm_kernel_L4_M4_END: + +//------------------------------------------------------------------------------ + +sgemm_kernel_L4_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble sgemm_kernel_L4_END + + tst counterI, #2 // counterI = counterI / 2 + ble sgemm_kernel_L4_M1_BEGIN + +sgemm_kernel_L4_M2_20: + + INIT2x4 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble sgemm_kernel_L4_M2_40 + +sgemm_kernel_L4_M2_22: + + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L4_M2_22 + + +sgemm_kernel_L4_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L4_M2_100 + +sgemm_kernel_L4_M2_42: + + KERNEL2x4_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L4_M2_42 + +sgemm_kernel_L4_M2_100: + + SAVE2x4 + +sgemm_kernel_L4_M2_END: + + +sgemm_kernel_L4_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble sgemm_kernel_L4_END + +sgemm_kernel_L4_M1_20: + + INIT1x4 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble sgemm_kernel_L4_M1_40 + +sgemm_kernel_L4_M1_22: + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L4_M1_22 + + +sgemm_kernel_L4_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L4_M1_100 + +sgemm_kernel_L4_M1_42: + + KERNEL1x4_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L4_M1_42 + +sgemm_kernel_L4_M1_100: + + SAVE1x4 + +sgemm_kernel_L4_END: + add origPB, origPB, origK, lsl #4 // B = B + K * 4 * 4 + + subs counterJ, counterJ , #1 // j-- + bgt sgemm_kernel_L4_BEGIN + + +/******************************************************************************/ + +sgemm_kernel_L2_BEGIN: // less than 2 left in N direction + + mov counterJ , origN + tst counterJ , #3 + ble sgemm_kernel_L999 + + tst counterJ , #2 + ble sgemm_kernel_L1_BEGIN + + mov pCRow0, pC // pCRow0 = pC + + add pC,pC,LDC, lsl #1 + + mov pA, origPA // pA = A + +sgemm_kernel_L2_M16_BEGIN: + + mov counterI, origM + asr counterI, counterI, #4 // counterI = counterI / 16 + cmp counterI,#0 + ble sgemm_kernel_L2_M8_BEGIN + +sgemm_kernel_L2_M16_20: + + INIT16x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble sgemm_kernel_L2_M16_40 + .align 5 + +sgemm_kernel_L2_M16_22: + KERNEL16x2_SUB + KERNEL16x2_SUB + KERNEL16x2_SUB + KERNEL16x2_SUB + + KERNEL16x2_SUB + KERNEL16x2_SUB + KERNEL16x2_SUB + KERNEL16x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M16_22 + + +sgemm_kernel_L2_M16_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L2_M16_100 + +sgemm_kernel_L2_M16_42: + + KERNEL16x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M16_42 + +sgemm_kernel_L2_M16_100: + + SAVE16x2 + +sgemm_kernel_L2_M16_END: + + subs counterI, counterI, #1 + bgt sgemm_kernel_L2_M16_20 + +//------------------------------------------------------------------------------ + +sgemm_kernel_L2_M8_BEGIN: + mov counterI, origM + tst counterI , #15 + ble sgemm_kernel_L2_END + + tst counterI, #8 + ble sgemm_kernel_L2_M4_BEGIN + +sgemm_kernel_L2_M8_20: + + INIT8x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble sgemm_kernel_L2_M8_40 + .align 5 + +sgemm_kernel_L2_M8_22: + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M8_22 + + +sgemm_kernel_L2_M8_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L2_M8_100 + +sgemm_kernel_L2_M8_42: + + KERNEL8x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M8_42 + +sgemm_kernel_L2_M8_100: + + SAVE8x2 + +sgemm_kernel_L2_M8_END: + +//------------------------------------------------------------------------------ + +sgemm_kernel_L2_M4_BEGIN: + mov counterI, origM + tst counterI , #7 + ble sgemm_kernel_L2_END + + tst counterI, #4 + ble sgemm_kernel_L2_M2_BEGIN + +sgemm_kernel_L2_M4_20: + + INIT4x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble sgemm_kernel_L2_M4_40 + .align 5 + +sgemm_kernel_L2_M4_22: + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M4_22 + + +sgemm_kernel_L2_M4_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L2_M4_100 + +sgemm_kernel_L2_M4_42: + + KERNEL4x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M4_42 + +sgemm_kernel_L2_M4_100: + + SAVE4x2 + +sgemm_kernel_L2_M4_END: + +//------------------------------------------------------------------------------ + + +sgemm_kernel_L2_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble sgemm_kernel_L2_END + + tst counterI, #2 // counterI = counterI / 2 + ble sgemm_kernel_L2_M1_BEGIN + +sgemm_kernel_L2_M2_20: + + INIT2x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble sgemm_kernel_L2_M2_40 + +sgemm_kernel_L2_M2_22: + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M2_22 + + +sgemm_kernel_L2_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L2_M2_100 + +sgemm_kernel_L2_M2_42: + + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M2_42 + +sgemm_kernel_L2_M2_100: + + SAVE2x2 + +sgemm_kernel_L2_M2_END: + + +sgemm_kernel_L2_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble sgemm_kernel_L2_END + +sgemm_kernel_L2_M1_20: + + INIT1x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL, #0 + ble sgemm_kernel_L2_M1_40 + +sgemm_kernel_L2_M1_22: + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M1_22 + + +sgemm_kernel_L2_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L2_M1_100 + +sgemm_kernel_L2_M1_42: + + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M1_42 + +sgemm_kernel_L2_M1_100: + + SAVE1x2 + +sgemm_kernel_L2_END: + + add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4 + +/******************************************************************************/ + +sgemm_kernel_L1_BEGIN: + + mov counterJ , origN + tst counterJ , #1 + ble sgemm_kernel_L999 // done + + + mov pCRow0, pC // pCRow0 = C + add pC , pC , LDC // Update pC to point to next + + mov pA, origPA // pA = A + +sgemm_kernel_L1_M16_BEGIN: + + mov counterI, origM + asr counterI, counterI, #4 // counterI = counterI / 16 + cmp counterI, #0 + ble sgemm_kernel_L1_M8_BEGIN + +sgemm_kernel_L1_M16_20: + + INIT16x1 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble sgemm_kernel_L1_M16_40 + .align 5 + +sgemm_kernel_L1_M16_22: + KERNEL16x1_SUB + KERNEL16x1_SUB + KERNEL16x1_SUB + KERNEL16x1_SUB + + KERNEL16x1_SUB + KERNEL16x1_SUB + KERNEL16x1_SUB + KERNEL16x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M16_22 + + +sgemm_kernel_L1_M16_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L1_M16_100 + +sgemm_kernel_L1_M16_42: + + KERNEL16x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M16_42 + +sgemm_kernel_L1_M16_100: + + SAVE16x1 + +sgemm_kernel_L1_M16_END: + + subs counterI, counterI, #1 + bgt sgemm_kernel_L1_M16_20 + +//------------------------------------------------------------------------------ + +sgemm_kernel_L1_M8_BEGIN: + + mov counterI, origM + tst counterI , #15 + ble sgemm_kernel_L1_END + + tst counterI, #8 + ble sgemm_kernel_L1_M4_BEGIN + +sgemm_kernel_L1_M8_20: + + INIT8x1 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble sgemm_kernel_L1_M8_40 + .align 5 + +sgemm_kernel_L1_M8_22: + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M8_22 + + +sgemm_kernel_L1_M8_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L1_M8_100 + +sgemm_kernel_L1_M8_42: + + KERNEL8x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M8_42 + +sgemm_kernel_L1_M8_100: + + SAVE8x1 + +sgemm_kernel_L1_M8_END: + +//------------------------------------------------------------------------------ + +sgemm_kernel_L1_M4_BEGIN: + mov counterI, origM + tst counterI , #7 + ble sgemm_kernel_L1_END + + tst counterI, #4 + ble sgemm_kernel_L1_M2_BEGIN + +sgemm_kernel_L1_M4_20: + + INIT4x1 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble sgemm_kernel_L1_M4_40 + .align 5 + +sgemm_kernel_L1_M4_22: + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M4_22 + + +sgemm_kernel_L1_M4_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L1_M4_100 + +sgemm_kernel_L1_M4_42: + + KERNEL4x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M4_42 + +sgemm_kernel_L1_M4_100: + + SAVE4x1 + +sgemm_kernel_L1_M4_END: + +//------------------------------------------------------------------------------ + +sgemm_kernel_L1_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble sgemm_kernel_L1_END + + tst counterI, #2 // counterI = counterI / 2 + ble sgemm_kernel_L1_M1_BEGIN + +sgemm_kernel_L1_M2_20: + + INIT2x1 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble sgemm_kernel_L1_M2_40 + +sgemm_kernel_L1_M2_22: + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M2_22 + + +sgemm_kernel_L1_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L1_M2_100 + +sgemm_kernel_L1_M2_42: + + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M2_42 + +sgemm_kernel_L1_M2_100: + + SAVE2x1 + +sgemm_kernel_L1_M2_END: + + +sgemm_kernel_L1_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble sgemm_kernel_L1_END + +sgemm_kernel_L1_M1_20: + + INIT1x1 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble sgemm_kernel_L1_M1_40 + +sgemm_kernel_L1_M1_22: + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M1_22 + + +sgemm_kernel_L1_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L1_M1_100 + +sgemm_kernel_L1_M1_42: + + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M1_42 + +sgemm_kernel_L1_M1_100: + + SAVE1x1 + +sgemm_kernel_L1_END: + +sgemm_kernel_L999: + mov x0, #0 // set return value + ldp d8, d9, [sp, #(0 * 16)] + ldp d10, d11, [sp, #(1 * 16)] + ldp d12, d13, [sp, #(2 * 16)] + ldp d14, d15, [sp, #(3 * 16)] + ldp d16, d17, [sp, #(4 * 16)] + ldp x18, x19, [sp, #(5 * 16)] + ldp x20, x21, [sp, #(6 * 16)] + ldp x22, x23, [sp, #(7 * 16)] + ldp x24, x25, [sp, #(8 * 16)] + ldp x26, x27, [sp, #(9 * 16)] + ldr x28, [sp, #(10 * 16)] + add sp, sp, #(11*16) + ret + + EPILOGUE + diff --git a/kernel/arm64/sgemm_kernel_8x8.S b/kernel/arm64/sgemm_kernel_8x8.S new file mode 100644 index 000000000..ac690e4d4 --- /dev/null +++ b/kernel/arm64/sgemm_kernel_8x8.S @@ -0,0 +1,2305 @@ +/******************************************************************************* +Copyright (c) 2015, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#define ASSEMBLER +#include "common.h" + +/* X0 X1 X2 s0 X3 x4 x5 x6 */ +/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc) */ + +#define origM x0 +#define origN x1 +#define origK x2 +#define origPA x3 +#define origPB x4 +#define pC x5 +#define LDC x6 +#define offset x7 +#define counterL x8 +#define counterI x9 +#define counterJ x10 +#define pB x11 +#define pCRow0 x12 +#define pCRow1 x13 +#define pCRow2 x14 +#define pA x15 +#define temp x16 + +#define alpha0 s10 +#define alphaV0 v10.s[0] +#define alpha1 s11 +#define alphaV1 v11.s[0] +#define alpha2 s14 +#define alphaV2 v14.s[0] +#define alpha3 s15 +#define alphaV3 v15.s[0] + +// 00 origM +// 01 origN +// 02 origK +// 03 origPA +// 04 origPB +// 05 pC +// 06 origLDC -> LDC +// 07 offset +// 08 counterL +// 09 counterI +// 10 counterJ +// 11 pB +// 12 pCRow0 +// 13 pCRow1 +// 14 pCRow2 +// 15 pA +// 16 temp +// 17 +// 18 must save +// 19 must save +// 20 must save +// 21 must save +// 22 must save +// 23 must save +// 24 must save +// 25 must save +// 26 must save +// 27 must save +// 28 must save +// 29 frame +// 30 link +// 31 sp + +//v00 ALPHA -> pA0_0, pA0_1, pA0_2, pA0_3 +//v01 pA0_4, pA0_5, pA0_6, pA0_7 +//v02 pA1_0, pA1_1, pA1_2, pA1_3 +//v03 pA1_4, pA1_5, pA1_6, pA1_7 +//v04 pB0_0, pB0_1, pB0_2, pB0_3 +//v05 pB0_4, pB0_5, pB0_6, pB0_7 +//v06 pB1_0, pB1_1, pB1_2, pB1_3 +//v07 pB1_4, pB1_5, pB1_6, pB1_7 +//v08 must save +//v09 must save +//v10 must save ALPHA0 +//v11 must save ALPHA1 +//v12 must save +//v13 must save +//v14 must save ALPHA2 +//v15 must save ALPHA3 +//v16 must save C00, C01, C02, C03 +//v17 must save C04, C05, C06, C07 +//v18 C08, C09, C10, C11 +//v19 C12, C13, C14, C15 +//v20 C16, C17, C18, C19 +//v21 C20, C21, C22, C23 +//v22 C24, C25, C26, C27 +//v23 C28, C29, C30, C31 +//v24 C32, C33, C34, C35 +//v25 C36, C37, C38, C39 +//v26 C40, C41, C42, C43 +//v27 C44, C45, C46, C47 +//v28 C48, C49, C50, C51 +//v29 C52, C53, C54, C55 +//v30 C56, C57, C58, C59 +//v31 C60, C61, C62, C63 + +/******************************************************************************* +* Macro definitions +*******************************************************************************/ + +.macro INIT8x8 + fmov s16, wzr + fmov s17, wzr + fmov s18, s16 + fmov s19, s17 + fmov s20, wzr + fmov s21, s16 + fmov s22, s17 + fmov s23, s18 + fmov s24, wzr + fmov s25, s16 + fmov s26, s17 + fmov s27, s18 + fmov s28, wzr + fmov s29, s16 + fmov s30, s17 + fmov s31, s18 +.endm + +.macro KERNEL8x8_I + ld1 {v4.4s}, [pB] + add pB, pB, #16 + ld1 {v5.4s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + + fmul v16.4s, v0.4s, v4.4s[0] + fmul v17.4s, v1.4s, v4.4s[0] + fmul v18.4s, v0.4s, v4.4s[1] + fmul v19.4s, v1.4s, v4.4s[1] + fmul v20.4s, v0.4s, v4.4s[2] + fmul v21.4s, v1.4s, v4.4s[2] + fmul v22.4s, v0.4s, v4.4s[3] + fmul v23.4s, v1.4s, v4.4s[3] + fmul v24.4s, v0.4s, v5.4s[0] + fmul v25.4s, v1.4s, v5.4s[0] + fmul v26.4s, v0.4s, v5.4s[1] + fmul v27.4s, v1.4s, v5.4s[1] + fmul v28.4s, v0.4s, v5.4s[2] + fmul v29.4s, v1.4s, v5.4s[2] + fmul v30.4s, v0.4s, v5.4s[3] + fmul v31.4s, v1.4s, v5.4s[3] + + ld1 {v6.4s}, [pB] + add pB, pB, #16 + ld1 {v7.4s}, [pB] + add pB, pB, #16 + ld1 {v2.4s}, [pA] + add pA, pA, #16 + ld1 {v3.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL8x8_M1 + fmla v16.4s, v0.4s, v4.4s[0] + fmla v17.4s, v1.4s, v4.4s[0] + fmla v18.4s, v0.4s, v4.4s[1] + fmla v19.4s, v1.4s, v4.4s[1] + fmla v20.4s, v0.4s, v4.4s[2] + fmla v21.4s, v1.4s, v4.4s[2] + fmla v22.4s, v0.4s, v4.4s[3] + fmla v23.4s, v1.4s, v4.4s[3] + fmla v24.4s, v0.4s, v5.4s[0] + fmla v25.4s, v1.4s, v5.4s[0] + fmla v26.4s, v0.4s, v5.4s[1] + fmla v27.4s, v1.4s, v5.4s[1] + fmla v28.4s, v0.4s, v5.4s[2] + fmla v29.4s, v1.4s, v5.4s[2] + fmla v30.4s, v0.4s, v5.4s[3] + fmla v31.4s, v1.4s, v5.4s[3] + + ld1 {v6.4s}, [pB] + add pB, pB, #16 + ld1 {v7.4s}, [pB] + add pB, pB, #16 + ld1 {v2.4s}, [pA] + add pA, pA, #16 + ld1 {v3.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL8x8_M2 + fmla v16.4s, v2.4s, v6.4s[0] + fmla v17.4s, v3.4s, v6.4s[0] + fmla v18.4s, v2.4s, v6.4s[1] + fmla v19.4s, v3.4s, v6.4s[1] + fmla v20.4s, v2.4s, v6.4s[2] + fmla v21.4s, v3.4s, v6.4s[2] + fmla v22.4s, v2.4s, v6.4s[3] + fmla v23.4s, v3.4s, v6.4s[3] + fmla v24.4s, v2.4s, v7.4s[0] + fmla v25.4s, v3.4s, v7.4s[0] + fmla v26.4s, v2.4s, v7.4s[1] + fmla v27.4s, v3.4s, v7.4s[1] + fmla v28.4s, v2.4s, v7.4s[2] + fmla v29.4s, v3.4s, v7.4s[2] + fmla v30.4s, v2.4s, v7.4s[3] + fmla v31.4s, v3.4s, v7.4s[3] + + ld1 {v4.4s}, [pB] + add pB, pB, #16 + ld1 {v5.4s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL8x8_E + fmla v16.4s, v2.4s, v6.4s[0] + fmla v17.4s, v3.4s, v6.4s[0] + fmla v18.4s, v2.4s, v6.4s[1] + fmla v19.4s, v3.4s, v6.4s[1] + fmla v20.4s, v2.4s, v6.4s[2] + fmla v21.4s, v3.4s, v6.4s[2] + fmla v22.4s, v2.4s, v6.4s[3] + fmla v23.4s, v3.4s, v6.4s[3] + fmla v24.4s, v2.4s, v7.4s[0] + fmla v25.4s, v3.4s, v7.4s[0] + fmla v26.4s, v2.4s, v7.4s[1] + fmla v27.4s, v3.4s, v7.4s[1] + fmla v28.4s, v2.4s, v7.4s[2] + fmla v29.4s, v3.4s, v7.4s[2] + fmla v30.4s, v2.4s, v7.4s[3] + fmla v31.4s, v3.4s, v7.4s[3] +.endm + +.macro KERNEL8x8_SUB + ld1 {v4.4s}, [pB] + add pB, pB, #16 + ld1 {v5.4s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v4.4s[0] + fmla v17.4s, v1.4s, v4.4s[0] + fmla v18.4s, v0.4s, v4.4s[1] + fmla v19.4s, v1.4s, v4.4s[1] + fmla v20.4s, v0.4s, v4.4s[2] + fmla v21.4s, v1.4s, v4.4s[2] + fmla v22.4s, v0.4s, v4.4s[3] + fmla v23.4s, v1.4s, v4.4s[3] + fmla v24.4s, v0.4s, v5.4s[0] + fmla v25.4s, v1.4s, v5.4s[0] + fmla v26.4s, v0.4s, v5.4s[1] + fmla v27.4s, v1.4s, v5.4s[1] + fmla v28.4s, v0.4s, v5.4s[2] + fmla v29.4s, v1.4s, v5.4s[2] + fmla v30.4s, v0.4s, v5.4s[3] + fmla v31.4s, v1.4s, v5.4s[3] +.endm + +.macro SAVE8x8 + add pCRow1, pCRow0, LDC + + ld1 {v0.4s, v1.4s}, [pCRow0] + fmla v0.4s, v16.4s, alphaV0 + fmla v1.4s, v17.4s, alphaV1 + st1 {v0.4s, v1.4s}, [pCRow0] + + add pCRow2, pCRow1, LDC + + ld1 {v2.4s, v3.4s}, [pCRow1] + fmla v2.4s, v18.4s, alphaV2 + fmla v3.4s, v19.4s, alphaV3 + st1 {v2.4s, v3.4s}, [pCRow1] + + add pCRow1, pCRow2, LDC + + ld1 {v4.4s, v5.4s}, [pCRow2] + fmla v4.4s, v20.4s, alphaV0 + fmla v5.4s, v21.4s, alphaV1 + st1 {v4.4s, v5.4s}, [pCRow2] + + add pCRow2, pCRow1, LDC + + ld1 {v6.4s, v7.4s}, [pCRow1] + fmla v6.4s, v22.4s, alphaV2 + fmla v7.4s, v23.4s, alphaV3 + st1 {v6.4s, v7.4s}, [pCRow1] + + add pCRow1, pCRow2, LDC + + ld1 {v0.4s, v1.4s}, [pCRow2] + fmla v0.4s, v24.4s, alphaV0 + fmla v1.4s, v25.4s, alphaV1 + st1 {v0.4s, v1.4s}, [pCRow2] + + add pCRow2, pCRow1, LDC + + ld1 {v2.4s, v3.4s}, [pCRow1] + fmla v2.4s, v26.4s, alphaV2 + fmla v3.4s, v27.4s, alphaV3 + st1 {v2.4s, v3.4s}, [pCRow1] + + add pCRow1, pCRow2, LDC + + ld1 {v4.4s, v5.4s}, [pCRow2] + fmla v4.4s, v28.4s, alphaV0 + fmla v5.4s, v29.4s, alphaV1 + st1 {v4.4s, v5.4s}, [pCRow2] + + ld1 {v6.4s, v7.4s}, [pCRow1] + fmla v6.4s, v30.4s, alphaV2 + fmla v7.4s, v31.4s, alphaV3 + st1 {v6.4s, v7.4s}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + + +.macro INIT4x8 + fmov s16, wzr + fmov s18, wzr + fmov s20, wzr + fmov s22, s16 + fmov s24, wzr + fmov s26, s16 + fmov s28, s18 + fmov s30, s20 +.endm + +.macro KERNEL4x8_I + ld1 {v4.4s}, [pB] + add pB, pB, #16 + ld1 {v5.4s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + + fmul v16.4s, v0.4s, v4.4s[0] + fmul v18.4s, v0.4s, v4.4s[1] + fmul v20.4s, v0.4s, v4.4s[2] + fmul v22.4s, v0.4s, v4.4s[3] + fmul v24.4s, v0.4s, v5.4s[0] + fmul v26.4s, v0.4s, v5.4s[1] + fmul v28.4s, v0.4s, v5.4s[2] + fmul v30.4s, v0.4s, v5.4s[3] + + ld1 {v6.4s}, [pB] + add pB, pB, #16 + ld1 {v7.4s}, [pB] + add pB, pB, #16 + ld1 {v2.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL4x8_M1 + fmla v16.4s, v0.4s, v4.4s[0] + fmla v18.4s, v0.4s, v4.4s[1] + fmla v20.4s, v0.4s, v4.4s[2] + fmla v22.4s, v0.4s, v4.4s[3] + fmla v24.4s, v0.4s, v5.4s[0] + fmla v26.4s, v0.4s, v5.4s[1] + fmla v28.4s, v0.4s, v5.4s[2] + fmla v30.4s, v0.4s, v5.4s[3] + + ld1 {v6.4s}, [pB] + add pB, pB, #16 + ld1 {v7.4s}, [pB] + add pB, pB, #16 + ld1 {v2.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL4x8_M2 + fmla v16.4s, v2.4s, v6.4s[0] + fmla v18.4s, v2.4s, v6.4s[1] + fmla v20.4s, v2.4s, v6.4s[2] + fmla v22.4s, v2.4s, v6.4s[3] + fmla v24.4s, v2.4s, v7.4s[0] + fmla v26.4s, v2.4s, v7.4s[1] + fmla v28.4s, v2.4s, v7.4s[2] + fmla v30.4s, v2.4s, v7.4s[3] + + ld1 {v4.4s}, [pB] + add pB, pB, #16 + ld1 {v5.4s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL4x8_E + fmla v16.4s, v2.4s, v6.4s[0] + fmla v18.4s, v2.4s, v6.4s[1] + fmla v20.4s, v2.4s, v6.4s[2] + fmla v22.4s, v2.4s, v6.4s[3] + fmla v24.4s, v2.4s, v7.4s[0] + fmla v26.4s, v2.4s, v7.4s[1] + fmla v28.4s, v2.4s, v7.4s[2] + fmla v30.4s, v2.4s, v7.4s[3] +.endm + +.macro KERNEL4x8_SUB + ld1 {v4.4s}, [pB] + add pB, pB, #16 + ld1 {v5.4s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v4.4s[0] + fmla v18.4s, v0.4s, v4.4s[1] + fmla v20.4s, v0.4s, v4.4s[2] + fmla v22.4s, v0.4s, v4.4s[3] + fmla v24.4s, v0.4s, v5.4s[0] + fmla v26.4s, v0.4s, v5.4s[1] + fmla v28.4s, v0.4s, v5.4s[2] + fmla v30.4s, v0.4s, v5.4s[3] +.endm + +.macro SAVE4x8 + add pCRow1, pCRow0, LDC + + ld1 {v0.4s}, [pCRow0] + fmla v0.4s, v16.4s, alphaV0 + st1 {v0.4s}, [pCRow0] + + add pCRow2, pCRow1, LDC + + ld1 {v2.4s}, [pCRow1] + fmla v2.4s, v18.4s, alphaV2 + st1 {v2.4s}, [pCRow1] + + add pCRow1, pCRow2, LDC + + ld1 {v4.4s}, [pCRow2] + fmla v4.4s, v20.4s, alphaV0 + st1 {v4.4s}, [pCRow2] + + add pCRow2, pCRow1, LDC + + ld1 {v6.4s}, [pCRow1] + fmla v6.4s, v22.4s, alphaV2 + st1 {v6.4s}, [pCRow1] + + add pCRow1, pCRow2, LDC + + ld1 {v0.4s}, [pCRow2] + fmla v0.4s, v24.4s, alphaV0 + st1 {v0.4s}, [pCRow2] + + add pCRow2, pCRow1, LDC + + ld1 {v2.4s}, [pCRow1] + fmla v2.4s, v26.4s, alphaV2 + st1 {v2.4s}, [pCRow1] + + add pCRow1, pCRow2, LDC + + ld1 {v4.4s}, [pCRow2] + fmla v4.4s, v28.4s, alphaV0 + st1 {v4.4s}, [pCRow2] + + ld1 {v6.4s}, [pCRow1] + fmla v6.4s, v30.4s, alphaV2 + st1 {v6.4s}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT2x8 + fmov s16, wzr + fmov s18, wzr + fmov s20, wzr + fmov s22, s16 + fmov s24, wzr + fmov s26, s16 + fmov s28, s18 + fmov s30, s20 +.endm + +.macro KERNEL2x8_SUB + ld1 {v4.4s}, [pB] + add pB, pB, #16 + ld1 {v5.4s}, [pB] + add pB, pB, #16 + ld1 {v0.2s}, [pA] + add pA, pA, #8 + + fmla v16.2s, v0.2s, v4.4s[0] + fmla v18.2s, v0.2s, v4.4s[1] + fmla v20.2s, v0.2s, v4.4s[2] + fmla v22.2s, v0.2s, v4.4s[3] + fmla v24.2s, v0.2s, v5.4s[0] + fmla v26.2s, v0.2s, v5.4s[1] + fmla v28.2s, v0.2s, v5.4s[2] + fmla v30.2s, v0.2s, v5.4s[3] +.endm + +.macro SAVE2x8 + add pCRow1, pCRow0, LDC + + ld1 {v0.2s}, [pCRow0] + fmla v0.2s, v16.2s, alphaV0 + st1 {v0.2s}, [pCRow0] + + add pCRow2, pCRow1, LDC + + ld1 {v2.2s}, [pCRow1] + fmla v2.2s, v18.2s, alphaV2 + st1 {v2.2s}, [pCRow1] + + add pCRow1, pCRow2, LDC + + ld1 {v4.2s}, [pCRow2] + fmla v4.2s, v20.2s, alphaV0 + st1 {v4.2s}, [pCRow2] + + add pCRow2, pCRow1, LDC + + ld1 {v6.2s}, [pCRow1] + fmla v6.2s, v22.2s, alphaV2 + st1 {v6.2s}, [pCRow1] + + add pCRow1, pCRow2, LDC + + ld1 {v0.2s}, [pCRow2] + fmla v0.2s, v24.2s, alphaV0 + st1 {v0.2s}, [pCRow2] + + add pCRow2, pCRow1, LDC + + ld1 {v2.2s}, [pCRow1] + fmla v2.2s, v26.2s, alphaV2 + st1 {v2.2s}, [pCRow1] + + add pCRow1, pCRow2, LDC + + ld1 {v4.2s}, [pCRow2] + fmla v4.2s, v28.2s, alphaV0 + st1 {v4.2s}, [pCRow2] + + ld1 {v6.2s}, [pCRow1] + fmla v6.2s, v30.2s, alphaV2 + st1 {v6.2s}, [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT1x8 + fmov s16, wzr + fmov s18, wzr + fmov s20, wzr + fmov s22, s16 + fmov s24, wzr + fmov s26, s16 + fmov s28, s18 + fmov s30, s20 +.endm + +.macro KERNEL1x8_SUB + ld1 {v4.4s}, [pB] + add pB, pB, #16 + ld1 {v5.4s}, [pB] + add pB, pB, #16 + ldr s0, [pA] + add pA, pA, #4 + + fmla s16, s0, v4.4s[0] + fmla s18, s0, v4.4s[1] + fmla s20, s0, v4.4s[2] + fmla s22, s0, v4.4s[3] + fmla s24, s0, v5.4s[0] + fmla s26, s0, v5.4s[1] + fmla s28, s0, v5.4s[2] + fmla s30, s0, v5.4s[3] +.endm + +.macro SAVE1x8 + add pCRow1, pCRow0, LDC + + ldr s0, [pCRow0] + fmla s0, s16, alphaV0 + str s0, [pCRow0] + + add pCRow2, pCRow1, LDC + + ldr s2, [pCRow1] + fmla s2, s18, alphaV2 + str s2, [pCRow1] + + add pCRow1, pCRow2, LDC + + ldr s4, [pCRow2] + fmla s4, s20, alphaV0 + str s4, [pCRow2] + + add pCRow2, pCRow1, LDC + + ldr s6, [pCRow1] + fmla s6, s22, alphaV2 + str s6, [pCRow1] + + add pCRow1, pCRow2, LDC + + ldr s0, [pCRow2] + fmla s0, s24, alphaV0 + str s0, [pCRow2] + + add pCRow2, pCRow1, LDC + + ldr s2, [pCRow1] + fmla s2, s26, alphaV2 + str s2, [pCRow1] + + add pCRow1, pCRow2, LDC + + ldr s4, [pCRow2] + fmla s4, s28, alphaV0 + str s4, [pCRow2] + + ldr s6, [pCRow1] + fmla s6, s30, alphaV2 + str s6, [pCRow1] + + add pCRow0, pCRow0, #4 +.endm + +/******************************************************************************/ + +.macro INIT8x4 + fmov s16, wzr + fmov s17, wzr + fmov s20, wzr + fmov s21, s16 + fmov s24, wzr + fmov s25, s16 + fmov s28, wzr + fmov s29, s16 +.endm + +.macro KERNEL8x4_I + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + + fmul v16.4s, v0.4s, v8.2s[0] + fmul v17.4s, v1.4s, v8.2s[0] + fmul v20.4s, v0.4s, v8.2s[1] + fmul v21.4s, v1.4s, v8.2s[1] + fmul v24.4s, v0.4s, v9.2s[0] + fmul v25.4s, v1.4s, v9.2s[0] + fmul v28.4s, v0.4s, v9.2s[1] + fmul v29.4s, v1.4s, v9.2s[1] + + ld1 {v12.2s, v13.2s}, [pB] + add pB, pB, #16 + ld1 {v4.4s}, [pA] + add pA, pA, #16 + ld1 {v5.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL8x4_M1 + fmla v16.4s, v0.4s, v8.2s[0] + fmla v17.4s, v1.4s, v8.2s[0] + fmla v20.4s, v0.4s, v8.2s[1] + fmla v21.4s, v1.4s, v8.2s[1] + fmla v24.4s, v0.4s, v9.2s[0] + fmla v25.4s, v1.4s, v9.2s[0] + fmla v28.4s, v0.4s, v9.2s[1] + fmla v29.4s, v1.4s, v9.2s[1] + + ld1 {v12.2s, v13.2s}, [pB] + add pB, pB, #16 + ld1 {v4.4s}, [pA] + add pA, pA, #16 + ld1 {v5.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL8x4_M2 + fmla v16.4s, v4.4s, v12.2s[0] + fmla v17.4s, v5.4s, v12.2s[0] + fmla v20.4s, v4.4s, v12.2s[1] + fmla v21.4s, v5.4s, v12.2s[1] + fmla v24.4s, v4.4s, v13.2s[0] + fmla v25.4s, v5.4s, v13.2s[0] + fmla v28.4s, v4.4s, v13.2s[1] + fmla v29.4s, v5.4s, v13.2s[1] + + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL8x4_E + fmla v16.4s, v4.4s, v12.2s[0] + fmla v17.4s, v5.4s, v12.2s[0] + fmla v20.4s, v4.4s, v12.2s[1] + fmla v21.4s, v5.4s, v12.2s[1] + fmla v24.4s, v4.4s, v13.2s[0] + fmla v25.4s, v5.4s, v13.2s[0] + fmla v28.4s, v4.4s, v13.2s[1] + fmla v29.4s, v5.4s, v13.2s[1] +.endm + +.macro KERNEL8x4_SUB + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v8.2s[0] + fmla v17.4s, v1.4s, v8.2s[0] + fmla v20.4s, v0.4s, v8.2s[1] + fmla v21.4s, v1.4s, v8.2s[1] + fmla v24.4s, v0.4s, v9.2s[0] + fmla v25.4s, v1.4s, v9.2s[0] + fmla v28.4s, v0.4s, v9.2s[1] + fmla v29.4s, v1.4s, v9.2s[1] +.endm + +.macro SAVE8x4 + add pCRow1, pCRow0, LDC + + ld1 {v0.4s, v1.4s}, [pCRow0] + fmla v0.4s, v16.4s, alphaV0 + fmla v1.4s, v17.4s, alphaV1 + st1 {v0.4s, v1.4s}, [pCRow0] + + add pCRow2, pCRow1, LDC + + ld1 {v4.4s, v5.4s}, [pCRow1] + fmla v4.4s, v20.4s, alphaV0 + fmla v5.4s, v21.4s, alphaV1 + st1 {v4.4s, v5.4s}, [pCRow1] + + add pCRow1, pCRow2, LDC + + ld1 {v0.4s, v1.4s}, [pCRow2] + fmla v0.4s, v24.4s, alphaV0 + fmla v1.4s, v25.4s, alphaV1 + st1 {v0.4s, v1.4s}, [pCRow2] + + ld1 {v4.4s, v5.4s}, [pCRow1] + fmla v4.4s, v28.4s, alphaV0 + fmla v5.4s, v29.4s, alphaV1 + st1 {v4.4s, v5.4s}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + + +.macro INIT4x4 + fmov s16, wzr + fmov s17, s16 + fmov s20, s17 + fmov s21, s16 + fmov s24, s17 + fmov s25, s16 + fmov s28, s17 + fmov s29, s16 +.endm + +.macro KERNEL4x4_I + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.2s, v1.2s}, [pA] + add pA, pA, #16 + + fmul v16.2s, v0.2s, v8.2s[0] + fmul v29.2s, v1.2s, v9.2s[1] + + fmul v20.2s, v0.2s, v8.2s[1] + fmul v25.2s, v1.2s, v9.2s[0] + + fmul v24.2s, v0.2s, v9.2s[0] + fmul v21.2s, v1.2s, v8.2s[1] + + fmul v28.2s, v0.2s, v9.2s[1] + fmul v17.2s, v1.2s, v8.2s[0] + + ld1 {v12.2s, v13.2s}, [pB] + add pB, pB, #16 + ld1 {v4.2s, v5.2s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL4x4_M1 + fmla v16.2s, v0.2s, v8.2s[0] + fmla v29.2s, v1.2s, v9.2s[1] + + ld1 {v12.2s, v13.2s}, [pB] // For next round + add pB, pB, #16 + + fmla v20.2s, v0.2s, v8.2s[1] + fmla v25.2s, v1.2s, v9.2s[0] + + ld1 {v4.2s, v5.2s}, [pA] // For next round + add pA, pA, #16 + + fmla v24.2s, v0.2s, v9.2s[0] + fmla v21.2s, v1.2s, v8.2s[1] + + prfm PLDL1KEEP, [pB, #512] + + fmla v28.2s, v0.2s, v9.2s[1] + fmla v17.2s, v1.2s, v8.2s[0] +.endm + +.macro KERNEL4x4_M2 + fmla v16.2s, v4.2s, v12.2s[0] + fmla v29.2s, v5.2s, v13.2s[1] + + ld1 {v8.2s, v9.2s}, [pB] // For next round + add pB, pB, #16 + + fmla v20.2s, v4.2s, v12.2s[1] + fmla v25.2s, v5.2s, v13.2s[0] + + ld1 {v0.2s, v1.2s}, [pA] // For next round + add pA, pA, #16 + + fmla v24.2s, v4.2s, v13.2s[0] + fmla v21.2s, v5.2s, v12.2s[1] + + prfm PLDL1KEEP, [pA, #512] + + fmla v28.2s, v4.2s, v13.2s[1] + fmla v17.2s, v5.2s, v12.2s[0] +.endm + +.macro KERNEL4x4_E + fmla v16.2s, v4.2s, v12.2s[0] + fmla v29.2s, v5.2s, v13.2s[1] + + fmla v20.2s, v4.2s, v12.2s[1] + fmla v25.2s, v5.2s, v13.2s[0] + + fmla v24.2s, v4.2s, v13.2s[0] + fmla v21.2s, v5.2s, v12.2s[1] + + fmla v28.2s, v4.2s, v13.2s[1] + fmla v17.2s, v5.2s, v12.2s[0] +.endm + +.macro KERNEL4x4_SUB + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.2s, v1.2s}, [pA] + add pA, pA, #16 + + fmla v16.2s, v0.2s, v8.2s[0] + fmla v29.2s, v1.2s, v9.2s[1] + + fmla v20.2s, v0.2s, v8.2s[1] + fmla v25.2s, v1.2s, v9.2s[0] + + fmla v24.2s, v0.2s, v9.2s[0] + fmla v21.2s, v1.2s, v8.2s[1] + + fmla v28.2s, v0.2s, v9.2s[1] + fmla v17.2s, v1.2s, v8.2s[0] +.endm + +.macro SAVE4x4 + ld1 {v8.2s, v9.2s}, [pCRow0] + fmla v8.2s, v16.2s, alphaV0 + fmla v9.2s, v17.2s, alphaV1 + st1 {v8.2s, v9.2s}, [pCRow0] + + add pCRow1, pCRow0, LDC + ld1 {v12.2s, v13.2s}, [pCRow1] + fmla v12.2s, v20.2s, alphaV2 + fmla v13.2s, v21.2s, alphaV3 + st1 {v12.2s, v13.2s}, [pCRow1] + + add pCRow2, pCRow1, LDC + ld1 {v8.2s, v9.2s}, [pCRow2] + fmla v8.2s, v24.2s, alphaV0 + fmla v9.2s, v25.2s, alphaV1 + st1 {v8.2s, v9.2s}, [pCRow2] + + add pCRow1, pCRow2, LDC + ld1 {v12.2s, v13.2s}, [pCRow1] + fmla v12.2s, v28.2s, alphaV2 + fmla v13.2s, v29.2s, alphaV3 + st1 {v12.2s, v13.2s}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT2x4 + fmov s16, wzr + fmov s20, s16 + fmov s24, s20 + fmov s28, s16 +.endm + +.macro KERNEL2x4_SUB + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.2s}, [pA] + add pA, pA, #8 + + fmla v16.2s, v0.2s, v8.2s[0] + fmla v20.2s, v0.2s, v8.2s[1] + fmla v24.2s, v0.2s, v9.2s[0] + fmla v28.2s, v0.2s, v9.2s[1] +.endm + +.macro SAVE2x4 + ld1 {v8.2s}, [pCRow0] + fmla v8.2s, v16.2s, alphaV0 + st1 {v8.2s}, [pCRow0] + + add pCRow1, pCRow0, LDC + ld1 {v12.2s}, [pCRow1] + fmla v12.2s, v20.2s, alphaV1 + st1 {v12.2s}, [pCRow1] + + add pCRow2, pCRow1, LDC + ld1 {v8.2s}, [pCRow2] + fmla v8.2s, v24.2s, alphaV2 + st1 {v8.2s}, [pCRow2] + + add pCRow1, pCRow2, LDC + ld1 {v12.2s}, [pCRow1] + fmla v12.2s, v28.2s, alphaV3 + st1 {v12.2s}, [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT1x4 + fmov s16, wzr + fmov s20, s16 +.endm + +.macro KERNEL1x4_SUB + ldr s0, [pA] + add pA, pA, #4 + + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + + fmla v16.2s, v8.2s, v0.s[0] + fmla v20.2s, v9.2s, v0.s[0] +.endm + +.macro SAVE1x4 + add pCRow1, pCRow0, LDC + ld1 {v8.s}[0], [pCRow0] + ld1 {v8.s}[1], [pCRow1] + fmla v8.2s, v16.2s, alphaV0 + st1 {v8.s}[0], [pCRow0] + st1 {v8.s}[1], [pCRow1] + + add pCRow2, pCRow1, LDC + add pCRow1, pCRow2, LDC + ld1 {v12.s}[0], [pCRow2] + ld1 {v12.s}[1], [pCRow1] + fmla v12.2s, v20.2s, alphaV1 + st1 {v12.s}[0], [pCRow2] + st1 {v12.s}[1], [pCRow1] + + add pCRow0, pCRow0, #4 +.endm + +/******************************************************************************/ + +.macro INIT8x2 + fmov s16, wzr + fmov s17, s16 + fmov s20, s17 + fmov s21, s16 +.endm + +.macro KERNEL8x2_SUB + ld1 {v8.2s}, [pB] + add pB, pB, #8 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v8.2s[0] + fmla v17.4s, v1.4s, v8.2s[0] + + fmla v20.4s, v0.4s, v8.2s[1] + fmla v21.4s, v1.4s, v8.2s[1] +.endm + +.macro SAVE8x2 + add pCRow1, pCRow0, LDC + + ld1 {v0.4s, v1.4s}, [pCRow0] + fmla v0.4s, v16.4s, alphaV0 + fmla v1.4s, v17.4s, alphaV1 + st1 {v0.4s, v1.4s}, [pCRow0] + + add pCRow2, pCRow1, LDC + + ld1 {v4.4s, v5.4s}, [pCRow1] + fmla v4.4s, v20.4s, alphaV0 + fmla v5.4s, v21.4s, alphaV1 + st1 {v4.4s, v5.4s}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT4x2 + fmov s16, wzr + fmov s17, s16 + fmov s20, s17 + fmov s21, s16 +.endm + +.macro KERNEL4x2_SUB + ld1 {v8.2s}, [pB] + add pB, pB, #8 + ld1 {v0.2s, v1.2s}, [pA] + add pA, pA, #16 + + fmla v16.2s, v0.2s, v8.2s[0] + fmla v17.2s, v1.2s, v8.2s[0] + fmla v20.2s, v0.2s, v8.2s[1] + fmla v21.2s, v1.2s, v8.2s[1] +.endm + +.macro SAVE4x2 + ld1 {v8.2s, v9.2s}, [pCRow0] + fmla v8.2s, v16.2s, alphaV0 + fmla v9.2s, v17.2s, alphaV1 + st1 {v8.2s, v9.2s}, [pCRow0] + + add pCRow1, pCRow0, LDC + ld1 {v12.2s, v13.2s}, [pCRow1] + fmla v12.2s, v20.2s, alphaV2 + fmla v13.2s, v21.2s, alphaV3 + st1 {v12.2s, v13.2s}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT2x2 + fmov s16, wzr + fmov s20, s16 +.endm + +.macro KERNEL2x2_SUB + ld1 {v8.2s}, [pB] + add pB, pB, #8 + + ld1 {v0.2s}, [pA] + add pA, pA, #8 + + fmla v16.2s, v0.2s, v8.2s[0] + fmla v20.2s, v0.2s, v8.2s[1] +.endm + +.macro SAVE2x2 + ld1 {v8.2s}, [pCRow0] + fmla v8.2s, v16.2s, alphaV0 + st1 {v8.2s}, [pCRow0] + + add pCRow1 , pCRow0, LDC + ld1 {v12.2s}, [pCRow1] + fmla v12.2s, v20.2s, alphaV1 + st1 {v12.2s}, [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT1x2 + fmov s16, wzr +.endm + +.macro KERNEL1x2_SUB + ld1 {v8.2s} , [pB] + add pB , pB, #8 + + ldr s0 , [pA] + add pA, pA, #4 + + fmla v16.2s, v8.2s, v0.2s[0] +.endm + +.macro SAVE1x2 + add pCRow1 , pCRow0, LDC + ld1 {v8.s}[0], [pCRow0] + ld1 {v8.s}[1], [pCRow1] + fmla v8.2s, v16.2s, alphaV0 + st1 {v8.s}[0], [pCRow0] + st1 {v8.s}[1], [pCRow1] + + add pCRow0, pCRow0, #4 +.endm + +/******************************************************************************/ + +.macro INIT8x1 + fmov s16, wzr + fmov s17, wzr +.endm + +.macro KERNEL8x1_SUB + ldr s8, [pB] + add pB , pB, #4 + + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v8.2s[0] + fmla v17.4s, v1.4s, v8.2s[0] +.endm + +.macro SAVE8x1 + ld1 {v0.4s, v1.4s}, [pCRow0] + fmla v0.4s, v16.4s, alphaV0 + fmla v1.4s, v17.4s, alphaV1 + st1 {v0.4s, v1.4s}, [pCRow0] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT4x1 + fmov s16, wzr + fmov s17, s16 +.endm + +.macro KERNEL4x1_SUB + ldr s8, [pB] + add pB , pB, #4 + + ld1 {v0.2s, v1.2s}, [pA] + add pA , pA, #16 + + fmla v16.2s, v0.2s, v8.2s[0] + fmla v17.2s, v1.2s, v8.2s[0] +.endm + +.macro SAVE4x1 + ld1 {v8.2s, v9.2s}, [pCRow0] + fmla v8.2s, v16.2s, alphaV0 + fmla v9.2s, v17.2s, alphaV1 + st1 {v8.2s, v9.2s}, [pCRow0] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT2x1 + fmov s16, wzr +.endm + +.macro KERNEL2x1_SUB + ldr s8, [pB] + add pB , pB, #4 + + ld1 {v0.2s}, [pA] + add pA , pA, #8 + + fmla v16.2s, v0.2s, v8.2s[0] +.endm + +.macro SAVE2x1 + ld1 {v8.2s}, [pCRow0] + fmla v8.2s, v16.2s, alphaV0 + st1 {v8.2s}, [pCRow0] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT1x1 + fmov s16, wzr +.endm + +.macro KERNEL1x1_SUB + ldr s8, [pB] + add pB , pB, #4 + + ldr s0, [pA] + add pA , pA, #4 + + fmadd s16, s0, s8, s16 +.endm + +.macro SAVE1x1 + ldr s8, [pCRow0] + fmla s8, s16, alphaV0 + str s8, [pCRow0] + + add pCRow0, pCRow0, #4 +.endm + +/******************************************************************************* +* End of macro definitions +*******************************************************************************/ + + PROLOGUE + +sgemm_kernel_begin: + + .align 5 + add sp, sp, #-(11 * 16) + stp d8, d9, [sp, #(0 * 16)] + stp d10, d11, [sp, #(1 * 16)] + stp d12, d13, [sp, #(2 * 16)] + stp d14, d15, [sp, #(3 * 16)] + stp d16, d17, [sp, #(4 * 16)] + stp x18, x19, [sp, #(5 * 16)] + stp x20, x21, [sp, #(6 * 16)] + stp x22, x23, [sp, #(7 * 16)] + stp x24, x25, [sp, #(8 * 16)] + stp x26, x27, [sp, #(9 * 16)] + str x28, [sp, #(10 * 16)] + + fmov alpha0, s0 + fmov alpha1, s0 + fmov alpha2, s0 + fmov alpha3, s0 + + lsl LDC, LDC, #2 // ldc = ldc * 4 + + mov pB, origPB + + mov counterJ, origN + asr counterJ, counterJ, #3 // J = J / 8 + cmp counterJ, #0 + ble sgemm_kernel_L4_BEGIN + +/******************************************************************************/ +/******************************************************************************/ + +sgemm_kernel_L8_BEGIN: + mov pCRow0, pC // pCRow0 = C + add pC, pC, LDC, lsl #3 + + mov pA, origPA // pA = start of A array + +/******************************************************************************/ + +sgemm_kernel_L8_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 // counterI = counterI / 8 + cmp counterI, #0 + ble sgemm_kernel_L8_M4_BEGIN + +sgemm_kernel_L8_M8_20: + + mov pB, origPB + + asr counterL , origK, #1 // L = K / 2 + cmp counterL , #2 // is there at least 4 to do? + blt sgemm_kernel_L8_M8_32 + + KERNEL8x8_I // do one in the K + KERNEL8x8_M2 // do another in the K + + subs counterL, counterL, #2 + ble sgemm_kernel_L8_M8_22a + .align 5 + +sgemm_kernel_L8_M8_22: + + KERNEL8x8_M1 + KERNEL8x8_M2 + + subs counterL, counterL, #1 + bgt sgemm_kernel_L8_M8_22 + +sgemm_kernel_L8_M8_22a: + + KERNEL8x8_M1 + KERNEL8x8_E + + b sgemm_kernel_L8_M8_44 + +sgemm_kernel_L8_M8_32: + + tst counterL, #1 + ble sgemm_kernel_L8_M8_40 + + KERNEL8x8_I + KERNEL8x8_E + + b sgemm_kernel_L8_M8_44 + +sgemm_kernel_L8_M8_40: + + INIT8x8 + +sgemm_kernel_L8_M8_44: + + ands counterL , origK, #1 + ble sgemm_kernel_L8_M8_100 + +sgemm_kernel_L8_M8_46: + + KERNEL8x8_SUB + +sgemm_kernel_L8_M8_100: + + SAVE8x8 + +sgemm_kernel_L8_M8_END: + subs counterI, counterI, #1 + bne sgemm_kernel_L8_M8_20 + +/******************************************************************************/ + +sgemm_kernel_L8_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble sgemm_kernel_L8_END + + tst counterI, #4 + ble sgemm_kernel_L8_M2_BEGIN + +sgemm_kernel_L8_M4_20: + + mov pB, origPB + + asr counterL , origK, #1 // L = K / 2 + cmp counterL , #2 // is there at least 4 to do? + blt sgemm_kernel_L8_M4_32 + + KERNEL4x8_I // do one in the K + KERNEL4x8_M2 // do another in the K + + subs counterL, counterL, #2 + ble sgemm_kernel_L8_M4_22a + .align 5 + +sgemm_kernel_L8_M4_22: + + KERNEL4x8_M1 + KERNEL4x8_M2 + + subs counterL, counterL, #1 + bgt sgemm_kernel_L8_M4_22 + +sgemm_kernel_L8_M4_22a: + + KERNEL4x8_M1 + KERNEL4x8_E + + b sgemm_kernel_L8_M4_44 + +sgemm_kernel_L8_M4_32: + + tst counterL, #1 + ble sgemm_kernel_L8_M4_40 + + KERNEL4x8_I + KERNEL4x8_E + + b sgemm_kernel_L8_M4_44 + +sgemm_kernel_L8_M4_40: + + INIT4x8 + +sgemm_kernel_L8_M4_44: + + ands counterL , origK, #1 + ble sgemm_kernel_L8_M4_100 + +sgemm_kernel_L8_M4_46: + + KERNEL4x8_SUB + +sgemm_kernel_L8_M4_100: + + SAVE4x8 + +sgemm_kernel_L8_M4_END: + +/******************************************************************************/ + +sgemm_kernel_L8_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble sgemm_kernel_L8_END + + tst counterI, #2 // counterI = counterI / 2 + ble sgemm_kernel_L8_M1_BEGIN + +sgemm_kernel_L8_M2_20: + + INIT2x8 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble sgemm_kernel_L8_M2_40 + +sgemm_kernel_L8_M2_22: + + KERNEL2x8_SUB + KERNEL2x8_SUB + KERNEL2x8_SUB + KERNEL2x8_SUB + + KERNEL2x8_SUB + KERNEL2x8_SUB + KERNEL2x8_SUB + KERNEL2x8_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L8_M2_22 + + +sgemm_kernel_L8_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L8_M2_100 + +sgemm_kernel_L8_M2_42: + + KERNEL2x8_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L8_M2_42 + +sgemm_kernel_L8_M2_100: + + SAVE2x8 + +sgemm_kernel_L8_M2_END: + +/******************************************************************************/ + +sgemm_kernel_L8_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble sgemm_kernel_L8_END + +sgemm_kernel_L8_M1_20: + + INIT1x8 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble sgemm_kernel_L8_M1_40 + +sgemm_kernel_L8_M1_22: + KERNEL1x8_SUB + KERNEL1x8_SUB + KERNEL1x8_SUB + KERNEL1x8_SUB + + KERNEL1x8_SUB + KERNEL1x8_SUB + KERNEL1x8_SUB + KERNEL1x8_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L8_M1_22 + + +sgemm_kernel_L8_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L8_M1_100 + +sgemm_kernel_L8_M1_42: + + KERNEL1x8_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L8_M1_42 + +sgemm_kernel_L8_M1_100: + + SAVE1x8 + +sgemm_kernel_L8_END: + lsl temp, origK, #5 // B = B + K * 4 * 8 + add origPB, origPB, temp + + subs counterJ, counterJ , #1 // j-- + bgt sgemm_kernel_L8_BEGIN + +/******************************************************************************/ +/******************************************************************************/ + +sgemm_kernel_L4_BEGIN: + + mov counterJ , origN + tst counterJ , #7 + ble sgemm_kernel_L999 + + tst counterJ , #4 + ble sgemm_kernel_L2_BEGIN + + mov pCRow0, pC // pCRow0 = pC + + add pC,pC,LDC, lsl #2 + + mov pA, origPA // pA = A + +/******************************************************************************/ + +sgemm_kernel_L4_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 // counterI = counterI / 8 + cmp counterI, #0 + ble sgemm_kernel_L4_M4_BEGIN + +sgemm_kernel_L4_M8_20: + + mov pB, origPB + + asr counterL , origK, #1 // L = K / 2 + cmp counterL , #2 // is there at least 4 to do? + blt sgemm_kernel_L4_M8_32 + + KERNEL8x4_I // do one in the K + KERNEL8x4_M2 // do another in the K + + subs counterL, counterL, #2 + ble sgemm_kernel_L4_M8_22a + .align 5 + +sgemm_kernel_L4_M8_22: + + KERNEL8x4_M1 + KERNEL8x4_M2 + + subs counterL, counterL, #1 + bgt sgemm_kernel_L4_M8_22 + +sgemm_kernel_L4_M8_22a: + + KERNEL8x4_M1 + KERNEL8x4_E + + b sgemm_kernel_L4_M8_44 + +sgemm_kernel_L4_M8_32: + + tst counterL, #1 + ble sgemm_kernel_L4_M8_40 + + KERNEL8x4_I + KERNEL8x4_E + + b sgemm_kernel_L4_M8_44 + +sgemm_kernel_L4_M8_40: + + INIT8x4 + +sgemm_kernel_L4_M8_44: + + ands counterL , origK, #1 + ble sgemm_kernel_L4_M8_100 + +sgemm_kernel_L4_M8_46: + + KERNEL8x4_SUB + +sgemm_kernel_L4_M8_100: + + SAVE8x4 + +sgemm_kernel_L4_M8_END: + subs counterI, counterI, #1 + bne sgemm_kernel_L4_M8_20 + +/******************************************************************************/ + +sgemm_kernel_L4_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble sgemm_kernel_L4_END + + tst counterI, #4 + ble sgemm_kernel_L4_M2_BEGIN + +sgemm_kernel_L4_M4_20: + + mov pB, origPB + + asr counterL , origK, #1 // L = K / 2 + cmp counterL , #2 // is there at least 4 to do? + blt sgemm_kernel_L4_M4_32 + + KERNEL4x4_I // do one in the K + KERNEL4x4_M2 // do another in the K + + subs counterL, counterL, #2 + ble sgemm_kernel_L4_M4_22a + .align 5 + +sgemm_kernel_L4_M4_22: + + KERNEL4x4_M1 + KERNEL4x4_M2 + + subs counterL, counterL, #1 + bgt sgemm_kernel_L4_M4_22 + +sgemm_kernel_L4_M4_22a: + + KERNEL4x4_M1 + KERNEL4x4_E + + b sgemm_kernel_L4_M4_44 + +sgemm_kernel_L4_M4_32: + + tst counterL, #1 + ble sgemm_kernel_L4_M4_40 + + KERNEL4x4_I + KERNEL4x4_E + + b sgemm_kernel_L4_M4_44 + +sgemm_kernel_L4_M4_40: + + INIT4x4 + +sgemm_kernel_L4_M4_44: + + ands counterL , origK, #1 + ble sgemm_kernel_L4_M4_100 + +sgemm_kernel_L4_M4_46: + + KERNEL4x4_SUB + +sgemm_kernel_L4_M4_100: + + SAVE4x4 + +sgemm_kernel_L4_M4_END: + +/******************************************************************************/ + +sgemm_kernel_L4_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble sgemm_kernel_L4_END + + tst counterI, #2 // counterI = counterI / 2 + ble sgemm_kernel_L4_M1_BEGIN + +sgemm_kernel_L4_M2_20: + + INIT2x4 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble sgemm_kernel_L4_M2_40 + +sgemm_kernel_L4_M2_22: + + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L4_M2_22 + + +sgemm_kernel_L4_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L4_M2_100 + +sgemm_kernel_L4_M2_42: + + KERNEL2x4_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L4_M2_42 + +sgemm_kernel_L4_M2_100: + + SAVE2x4 + +sgemm_kernel_L4_M2_END: + +/******************************************************************************/ + +sgemm_kernel_L4_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble sgemm_kernel_L4_END + +sgemm_kernel_L4_M1_20: + + INIT1x4 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble sgemm_kernel_L4_M1_40 + +sgemm_kernel_L4_M1_22: + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L4_M1_22 + + +sgemm_kernel_L4_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L4_M1_100 + +sgemm_kernel_L4_M1_42: + + KERNEL1x4_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L4_M1_42 + +sgemm_kernel_L4_M1_100: + + SAVE1x4 + +sgemm_kernel_L4_END: + add origPB, origPB, origK, lsl #4 // B = B + K * 4 * 4 + +/******************************************************************************/ +/******************************************************************************/ + +sgemm_kernel_L2_BEGIN: // less than 2 left in N direction + + mov counterJ , origN + tst counterJ , #3 + ble sgemm_kernel_L999 + + tst counterJ , #2 + ble sgemm_kernel_L1_BEGIN + + mov pCRow0, pC // pCRow0 = pC + + add pC,pC,LDC, lsl #1 + + mov pA, origPA // pA = A + +/******************************************************************************/ + +sgemm_kernel_L2_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 // counterI = counterI / 8 + cmp counterI,#0 + ble sgemm_kernel_L2_M4_BEGIN + +sgemm_kernel_L2_M8_20: + + INIT8x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble sgemm_kernel_L2_M8_40 + .align 5 + +sgemm_kernel_L2_M8_22: + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M8_22 + + +sgemm_kernel_L2_M8_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L2_M8_100 + +sgemm_kernel_L2_M8_42: + + KERNEL8x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M8_42 + +sgemm_kernel_L2_M8_100: + + SAVE8x2 + +sgemm_kernel_L2_M8_END: + + subs counterI, counterI, #1 + bgt sgemm_kernel_L2_M8_20 + +/******************************************************************************/ + +sgemm_kernel_L2_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble sgemm_kernel_L2_END + + tst counterI, #4 + ble sgemm_kernel_L2_M2_BEGIN + +sgemm_kernel_L2_M4_20: + + INIT4x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble sgemm_kernel_L2_M4_40 + .align 5 + +sgemm_kernel_L2_M4_22: + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M4_22 + + +sgemm_kernel_L2_M4_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L2_M4_100 + +sgemm_kernel_L2_M4_42: + + KERNEL4x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M4_42 + +sgemm_kernel_L2_M4_100: + + SAVE4x2 + +sgemm_kernel_L2_M4_END: + +/******************************************************************************/ + +sgemm_kernel_L2_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble sgemm_kernel_L2_END + + tst counterI, #2 // counterI = counterI / 2 + ble sgemm_kernel_L2_M1_BEGIN + +sgemm_kernel_L2_M2_20: + + INIT2x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble sgemm_kernel_L2_M2_40 + +sgemm_kernel_L2_M2_22: + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M2_22 + + +sgemm_kernel_L2_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L2_M2_100 + +sgemm_kernel_L2_M2_42: + + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M2_42 + +sgemm_kernel_L2_M2_100: + + SAVE2x2 + +sgemm_kernel_L2_M2_END: + +/******************************************************************************/ + +sgemm_kernel_L2_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble sgemm_kernel_L2_END + +sgemm_kernel_L2_M1_20: + + INIT1x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL, #0 + ble sgemm_kernel_L2_M1_40 + +sgemm_kernel_L2_M1_22: + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M1_22 + + +sgemm_kernel_L2_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L2_M1_100 + +sgemm_kernel_L2_M1_42: + + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M1_42 + +sgemm_kernel_L2_M1_100: + + SAVE1x2 + +sgemm_kernel_L2_END: + + add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4 + +/******************************************************************************/ +/******************************************************************************/ + +sgemm_kernel_L1_BEGIN: + + mov counterJ , origN + tst counterJ , #1 + ble sgemm_kernel_L999 // done + + + mov pCRow0, pC // pCRow0 = C + add pC , pC , LDC // Update pC to point to next + + mov pA, origPA // pA = A + +/******************************************************************************/ + +sgemm_kernel_L1_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 + cmp counterI, #0 + ble sgemm_kernel_L1_M4_BEGIN + +sgemm_kernel_L1_M8_20: + + INIT8x1 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble sgemm_kernel_L1_M8_40 + .align 5 + +sgemm_kernel_L1_M8_22: + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M8_22 + + +sgemm_kernel_L1_M8_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L1_M8_100 + +sgemm_kernel_L1_M8_42: + + KERNEL8x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M8_42 + +sgemm_kernel_L1_M8_100: + + SAVE8x1 + +sgemm_kernel_L1_M8_END: + + subs counterI, counterI, #1 + bgt sgemm_kernel_L1_M8_20 + +/******************************************************************************/ + +sgemm_kernel_L1_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble sgemm_kernel_L1_END + + tst counterI, #4 + ble sgemm_kernel_L1_M2_BEGIN + +sgemm_kernel_L1_M4_20: + + INIT4x1 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble sgemm_kernel_L1_M4_40 + .align 5 + +sgemm_kernel_L1_M4_22: + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M4_22 + + +sgemm_kernel_L1_M4_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L1_M4_100 + +sgemm_kernel_L1_M4_42: + + KERNEL4x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M4_42 + +sgemm_kernel_L1_M4_100: + + SAVE4x1 + +sgemm_kernel_L1_M4_END: + +/******************************************************************************/ + +sgemm_kernel_L1_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble sgemm_kernel_L1_END + + tst counterI, #2 // counterI = counterI / 2 + ble sgemm_kernel_L1_M1_BEGIN + +sgemm_kernel_L1_M2_20: + + INIT2x1 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble sgemm_kernel_L1_M2_40 + +sgemm_kernel_L1_M2_22: + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M2_22 + + +sgemm_kernel_L1_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L1_M2_100 + +sgemm_kernel_L1_M2_42: + + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M2_42 + +sgemm_kernel_L1_M2_100: + + SAVE2x1 + +sgemm_kernel_L1_M2_END: + +/******************************************************************************/ + +sgemm_kernel_L1_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble sgemm_kernel_L1_END + +sgemm_kernel_L1_M1_20: + + INIT1x1 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble sgemm_kernel_L1_M1_40 + +sgemm_kernel_L1_M1_22: + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M1_22 + + +sgemm_kernel_L1_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L1_M1_100 + +sgemm_kernel_L1_M1_42: + + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M1_42 + +sgemm_kernel_L1_M1_100: + + SAVE1x1 + +sgemm_kernel_L1_END: + +/******************************************************************************/ + +sgemm_kernel_L999: + mov x0, #0 // set return value + ldp d8, d9, [sp, #(0 * 16)] + ldp d10, d11, [sp, #(1 * 16)] + ldp d12, d13, [sp, #(2 * 16)] + ldp d14, d15, [sp, #(3 * 16)] + ldp d16, d17, [sp, #(4 * 16)] + ldp x18, x19, [sp, #(5 * 16)] + ldp x20, x21, [sp, #(6 * 16)] + ldp x22, x23, [sp, #(7 * 16)] + ldp x24, x25, [sp, #(8 * 16)] + ldp x26, x27, [sp, #(9 * 16)] + ldr x28, [sp, #(10 * 16)] + add sp, sp, #(11*16) + ret + + EPILOGUE + diff --git a/kernel/arm64/strmm_kernel_16x4.S b/kernel/arm64/strmm_kernel_16x4.S new file mode 100755 index 000000000..b99760a03 --- /dev/null +++ b/kernel/arm64/strmm_kernel_16x4.S @@ -0,0 +1,2431 @@ +/******************************************************************************* +Copyright (c) 2015, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#define ASSEMBLER +#include "common.h" + +/* X0 X1 X2 s0 X3 x4 x5 x6 x7 */ +/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc, BLASLONG offset) */ + +#define origM x0 +#define origN x1 +#define origK x2 +#define origPA x3 +#define origPB x4 +#define pC x5 +#define LDC x6 +#define offset x7 +#define counterL x8 +#define counterI x9 +#define counterJ x10 +#define pB x11 +#define pCRow0 x12 +#define pCRow1 x13 +#define pCRow2 x14 +#define pA x15 +#define temp x16 +#define tempOffset x17 +#define tempK x18 + +#define alpha0 s10 +#define alphaV0 v10.s[0] +#define alpha1 s11 +#define alphaV1 v11.s[0] +#define alpha2 s14 +#define alphaV2 v14.s[0] +#define alpha3 s15 +#define alphaV3 v15.s[0] + +// 00 origM +// 01 origN +// 02 origK +// 03 origPA +// 04 origPB +// 05 pC +// 06 origLDC -> LDC +// 07 offset +// 08 counterL +// 09 counterI +// 10 counterJ +// 11 pB +// 12 pCRow0 +// 13 pCRow1 +// 14 pCRow2 +// 15 pA +// 16 temp +// 17 tempOffset +// 18 must save tempK +// 19 must save +// 20 must save +// 21 must save +// 22 must save +// 23 must save +// 24 must save +// 25 must save +// 26 must save +// 27 must save +// 28 must save +// 29 frame +// 30 link +// 31 sp + +//v00 ALPHA -> pA0_00, pA0_01, pA0_02, pA0_03 +//v01 pA0_04, pA0_05, pA0_06, pA0_07 +//v02 pA0_08, pA0_09, pA0_10, pA0_11 +//v03 pA0_12, pA0_13, pA0_14, pA0_15 +//v04 pA1_00, pA1_01, pA1_02, pA1_03 +//v05 pA1_04, pA1_05, pA1_06, pA1_07 +//v06 pA1_08, pA1_09, pA1_10, pA1_11 +//v07 pA1_12, pA1_13, pA1_14, pA1_15 +//v08 must save pB00, pB01 +//v09 must save pB02, pB03 +//v10 must save ALPHA0 +//v11 must save ALPHA1 +//v12 must save pB10, pB11 +//v13 must save pB12, pB13 +//v14 must save ALPHA2 +//v15 must save ALPHA3 +//v16 must save C00, C01, C02, C03 +//v17 must save C04, C05, C06, C07 +//v18 C08, C09, C10, C11 +//v19 C12, C13, C14, C15 +//v20 C16, C17, C18, C19 +//v21 C20, C21, C22, C23 +//v22 C24, C25, C26, C27 +//v23 C28, C29, C30, C31 +//v24 C32, C33, C34, C35 +//v25 C36, C37, C38, C39 +//v26 C40, C41, C42, C43 +//v27 C44, C45, C46, C47 +//v28 C48, C49, C50, C51 +//v29 C52, C53, C54, C55 +//v30 C56, C57, C58, C59 +//v31 C60, C61, C62, C63 + +/******************************************************************************* +* Macro definitions +*******************************************************************************/ + +.macro INIT16x4 + fmov s16, wzr + fmov s17, wzr + fmov s18, s16 + fmov s19, s17 + fmov s20, wzr + fmov s21, s16 + fmov s22, s17 + fmov s23, s18 + fmov s24, wzr + fmov s25, s16 + fmov s26, s17 + fmov s27, s18 + fmov s28, wzr + fmov s29, s16 + fmov s30, s17 + fmov s31, s18 +.endm + +.macro KERNEL16x4_I + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + ld1 {v2.4s}, [pA] + add pA, pA, #16 + ld1 {v3.4s}, [pA] + add pA, pA, #16 + + fmul v16.4s, v0.4s, v8.2s[0] + fmul v17.4s, v1.4s, v8.2s[0] + fmul v18.4s, v2.4s, v8.2s[0] + fmul v19.4s, v3.4s, v8.2s[0] + + fmul v20.4s, v0.4s, v8.2s[1] + fmul v21.4s, v1.4s, v8.2s[1] + fmul v22.4s, v2.4s, v8.2s[1] + fmul v23.4s, v3.4s, v8.2s[1] + + fmul v24.4s, v0.4s, v9.2s[0] + fmul v25.4s, v1.4s, v9.2s[0] + fmul v26.4s, v2.4s, v9.2s[0] + fmul v27.4s, v3.4s, v9.2s[0] + + fmul v28.4s, v0.4s, v9.2s[1] + fmul v29.4s, v1.4s, v9.2s[1] + fmul v30.4s, v2.4s, v9.2s[1] + fmul v31.4s, v3.4s, v9.2s[1] + + ld1 {v12.2s, v13.2s}, [pB] + add pB, pB, #16 + ld1 {v4.4s}, [pA] + add pA, pA, #16 + ld1 {v5.4s}, [pA] + add pA, pA, #16 + ld1 {v6.4s}, [pA] + add pA, pA, #16 + ld1 {v7.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL16x4_M1 + fmla v16.4s, v0.4s, v8.2s[0] + fmla v17.4s, v1.4s, v8.2s[0] + fmla v18.4s, v2.4s, v8.2s[0] + fmla v19.4s, v3.4s, v8.2s[0] + + fmla v20.4s, v0.4s, v8.2s[1] + fmla v21.4s, v1.4s, v8.2s[1] + fmla v22.4s, v2.4s, v8.2s[1] + fmla v23.4s, v3.4s, v8.2s[1] + + fmla v24.4s, v0.4s, v9.2s[0] + fmla v25.4s, v1.4s, v9.2s[0] + fmla v26.4s, v2.4s, v9.2s[0] + fmla v27.4s, v3.4s, v9.2s[0] + + fmla v28.4s, v0.4s, v9.2s[1] + fmla v29.4s, v1.4s, v9.2s[1] + fmla v30.4s, v2.4s, v9.2s[1] + fmla v31.4s, v3.4s, v9.2s[1] + + ld1 {v12.2s, v13.2s}, [pB] + add pB, pB, #16 + ld1 {v4.4s}, [pA] + add pA, pA, #16 + ld1 {v5.4s}, [pA] + add pA, pA, #16 + ld1 {v6.4s}, [pA] + add pA, pA, #16 + ld1 {v7.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL16x4_M2 + fmla v16.4s, v4.4s, v12.2s[0] + fmla v17.4s, v5.4s, v12.2s[0] + fmla v18.4s, v6.4s, v12.2s[0] + fmla v19.4s, v7.4s, v12.2s[0] + + fmla v20.4s, v4.4s, v12.2s[1] + fmla v21.4s, v5.4s, v12.2s[1] + fmla v22.4s, v6.4s, v12.2s[1] + fmla v23.4s, v7.4s, v12.2s[1] + + fmla v24.4s, v4.4s, v13.2s[0] + fmla v25.4s, v5.4s, v13.2s[0] + fmla v26.4s, v6.4s, v13.2s[0] + fmla v27.4s, v7.4s, v13.2s[0] + + fmla v28.4s, v4.4s, v13.2s[1] + fmla v29.4s, v5.4s, v13.2s[1] + fmla v30.4s, v6.4s, v13.2s[1] + fmla v31.4s, v7.4s, v13.2s[1] + + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + ld1 {v2.4s}, [pA] + add pA, pA, #16 + ld1 {v3.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL16x4_E + fmla v16.4s, v4.4s, v12.2s[0] + fmla v17.4s, v5.4s, v12.2s[0] + fmla v18.4s, v6.4s, v12.2s[0] + fmla v19.4s, v7.4s, v12.2s[0] + + fmla v20.4s, v4.4s, v12.2s[1] + fmla v21.4s, v5.4s, v12.2s[1] + fmla v22.4s, v6.4s, v12.2s[1] + fmla v23.4s, v7.4s, v12.2s[1] + + fmla v24.4s, v4.4s, v13.2s[0] + fmla v25.4s, v5.4s, v13.2s[0] + fmla v26.4s, v6.4s, v13.2s[0] + fmla v27.4s, v7.4s, v13.2s[0] + + fmla v28.4s, v4.4s, v13.2s[1] + fmla v29.4s, v5.4s, v13.2s[1] + fmla v30.4s, v6.4s, v13.2s[1] + fmla v31.4s, v7.4s, v13.2s[1] +.endm + +.macro KERNEL16x4_SUB + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + ld1 {v2.4s}, [pA] + add pA, pA, #16 + ld1 {v3.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v8.2s[0] + fmla v17.4s, v1.4s, v8.2s[0] + fmla v18.4s, v2.4s, v8.2s[0] + fmla v19.4s, v3.4s, v8.2s[0] + + fmla v20.4s, v0.4s, v8.2s[1] + fmla v21.4s, v1.4s, v8.2s[1] + fmla v22.4s, v2.4s, v8.2s[1] + fmla v23.4s, v3.4s, v8.2s[1] + + fmla v24.4s, v0.4s, v9.2s[0] + fmla v25.4s, v1.4s, v9.2s[0] + fmla v26.4s, v2.4s, v9.2s[0] + fmla v27.4s, v3.4s, v9.2s[0] + + fmla v28.4s, v0.4s, v9.2s[1] + fmla v29.4s, v1.4s, v9.2s[1] + fmla v30.4s, v2.4s, v9.2s[1] + fmla v31.4s, v3.4s, v9.2s[1] +.endm + +.macro SAVE16x4 + add pCRow1, pCRow0, LDC + + fmul v0.4s, v16.4s, alphaV0 + fmul v1.4s, v17.4s, alphaV1 + fmul v2.4s, v18.4s, alphaV2 + fmul v3.4s, v19.4s, alphaV3 + st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow0] + + add pCRow2, pCRow1, LDC + + fmul v4.4s, v20.4s, alphaV0 + fmul v5.4s, v21.4s, alphaV1 + fmul v6.4s, v22.4s, alphaV2 + fmul v7.4s, v23.4s, alphaV3 + st1 {v4.4s, v5.4s, v6.4s, v7.4s}, [pCRow1] + + add pCRow1, pCRow2, LDC + + fmul v0.4s, v24.4s, alphaV0 + fmul v1.4s, v25.4s, alphaV1 + fmul v2.4s, v26.4s, alphaV2 + fmul v3.4s, v27.4s, alphaV3 + st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow2] + + fmul v4.4s, v28.4s, alphaV0 + fmul v5.4s, v29.4s, alphaV1 + fmul v6.4s, v30.4s, alphaV2 + fmul v7.4s, v31.4s, alphaV3 + st1 {v4.4s, v5.4s, v6.4s, v7.4s}, [pCRow1] + + add pCRow0, pCRow0, #64 +.endm + +/******************************************************************************/ + +.macro INIT8x4 + fmov s16, wzr + fmov s17, wzr + fmov s20, wzr + fmov s21, s16 + fmov s24, wzr + fmov s25, s16 + fmov s28, wzr + fmov s29, s16 +.endm + +.macro KERNEL8x4_I + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + + fmul v16.4s, v0.4s, v8.2s[0] + fmul v17.4s, v1.4s, v8.2s[0] + fmul v20.4s, v0.4s, v8.2s[1] + fmul v21.4s, v1.4s, v8.2s[1] + fmul v24.4s, v0.4s, v9.2s[0] + fmul v25.4s, v1.4s, v9.2s[0] + fmul v28.4s, v0.4s, v9.2s[1] + fmul v29.4s, v1.4s, v9.2s[1] + + ld1 {v12.2s, v13.2s}, [pB] + add pB, pB, #16 + ld1 {v4.4s}, [pA] + add pA, pA, #16 + ld1 {v5.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL8x4_M1 + fmla v16.4s, v0.4s, v8.2s[0] + fmla v17.4s, v1.4s, v8.2s[0] + fmla v20.4s, v0.4s, v8.2s[1] + fmla v21.4s, v1.4s, v8.2s[1] + fmla v24.4s, v0.4s, v9.2s[0] + fmla v25.4s, v1.4s, v9.2s[0] + fmla v28.4s, v0.4s, v9.2s[1] + fmla v29.4s, v1.4s, v9.2s[1] + + ld1 {v12.2s, v13.2s}, [pB] + add pB, pB, #16 + ld1 {v4.4s}, [pA] + add pA, pA, #16 + ld1 {v5.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL8x4_M2 + fmla v16.4s, v4.4s, v12.2s[0] + fmla v17.4s, v5.4s, v12.2s[0] + fmla v20.4s, v4.4s, v12.2s[1] + fmla v21.4s, v5.4s, v12.2s[1] + fmla v24.4s, v4.4s, v13.2s[0] + fmla v25.4s, v5.4s, v13.2s[0] + fmla v28.4s, v4.4s, v13.2s[1] + fmla v29.4s, v5.4s, v13.2s[1] + + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL8x4_E + fmla v16.4s, v4.4s, v12.2s[0] + fmla v17.4s, v5.4s, v12.2s[0] + fmla v20.4s, v4.4s, v12.2s[1] + fmla v21.4s, v5.4s, v12.2s[1] + fmla v24.4s, v4.4s, v13.2s[0] + fmla v25.4s, v5.4s, v13.2s[0] + fmla v28.4s, v4.4s, v13.2s[1] + fmla v29.4s, v5.4s, v13.2s[1] +.endm + +.macro KERNEL8x4_SUB + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v8.2s[0] + fmla v17.4s, v1.4s, v8.2s[0] + fmla v20.4s, v0.4s, v8.2s[1] + fmla v21.4s, v1.4s, v8.2s[1] + fmla v24.4s, v0.4s, v9.2s[0] + fmla v25.4s, v1.4s, v9.2s[0] + fmla v28.4s, v0.4s, v9.2s[1] + fmla v29.4s, v1.4s, v9.2s[1] +.endm + +.macro SAVE8x4 + add pCRow1, pCRow0, LDC + + fmul v0.4s, v16.4s, alphaV0 + fmul v1.4s, v17.4s, alphaV1 + st1 {v0.4s, v1.4s}, [pCRow0] + + add pCRow2, pCRow1, LDC + + fmul v4.4s, v20.4s, alphaV0 + fmul v5.4s, v21.4s, alphaV1 + st1 {v4.4s, v5.4s}, [pCRow1] + + add pCRow1, pCRow2, LDC + + fmul v0.4s, v24.4s, alphaV0 + fmul v1.4s, v25.4s, alphaV1 + st1 {v0.4s, v1.4s}, [pCRow2] + + fmul v4.4s, v28.4s, alphaV0 + fmul v5.4s, v29.4s, alphaV1 + st1 {v4.4s, v5.4s}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT4x4 + fmov s16, wzr + fmov s17, s16 + fmov s20, s17 + fmov s21, s16 + fmov s24, s17 + fmov s25, s16 + fmov s28, s17 + fmov s29, s16 +.endm + +.macro KERNEL4x4_I + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.2s, v1.2s}, [pA] + add pA, pA, #16 + + fmul v16.2s, v0.2s, v8.2s[0] + fmul v29.2s, v1.2s, v9.2s[1] + + fmul v20.2s, v0.2s, v8.2s[1] + fmul v25.2s, v1.2s, v9.2s[0] + + fmul v24.2s, v0.2s, v9.2s[0] + fmul v21.2s, v1.2s, v8.2s[1] + + fmul v28.2s, v0.2s, v9.2s[1] + fmul v17.2s, v1.2s, v8.2s[0] + + ld1 {v12.2s, v13.2s}, [pB] + add pB, pB, #16 + ld1 {v4.2s, v5.2s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL4x4_M1 + fmla v16.2s, v0.2s, v8.2s[0] + fmla v29.2s, v1.2s, v9.2s[1] + + ld1 {v12.2s, v13.2s}, [pB] // For next round + add pB, pB, #16 + + fmla v20.2s, v0.2s, v8.2s[1] + fmla v25.2s, v1.2s, v9.2s[0] + + ld1 {v4.2s, v5.2s}, [pA] // For next round + add pA, pA, #16 + + fmla v24.2s, v0.2s, v9.2s[0] + fmla v21.2s, v1.2s, v8.2s[1] + + prfm PLDL1KEEP, [pB, #512] + + fmla v28.2s, v0.2s, v9.2s[1] + fmla v17.2s, v1.2s, v8.2s[0] +.endm + +.macro KERNEL4x4_M2 + fmla v16.2s, v4.2s, v12.2s[0] + fmla v29.2s, v5.2s, v13.2s[1] + + ld1 {v8.2s, v9.2s}, [pB] // For next round + add pB, pB, #16 + + fmla v20.2s, v4.2s, v12.2s[1] + fmla v25.2s, v5.2s, v13.2s[0] + + ld1 {v0.2s, v1.2s}, [pA] // For next round + add pA, pA, #16 + + fmla v24.2s, v4.2s, v13.2s[0] + fmla v21.2s, v5.2s, v12.2s[1] + + prfm PLDL1KEEP, [pA, #512] + + fmla v28.2s, v4.2s, v13.2s[1] + fmla v17.2s, v5.2s, v12.2s[0] +.endm + +.macro KERNEL4x4_E + fmla v16.2s, v4.2s, v12.2s[0] + fmla v29.2s, v5.2s, v13.2s[1] + + fmla v20.2s, v4.2s, v12.2s[1] + fmla v25.2s, v5.2s, v13.2s[0] + + fmla v24.2s, v4.2s, v13.2s[0] + fmla v21.2s, v5.2s, v12.2s[1] + + fmla v28.2s, v4.2s, v13.2s[1] + fmla v17.2s, v5.2s, v12.2s[0] +.endm + +.macro KERNEL4x4_SUB + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.2s, v1.2s}, [pA] + add pA, pA, #16 + + fmla v16.2s, v0.2s, v8.2s[0] + fmla v29.2s, v1.2s, v9.2s[1] + + fmla v20.2s, v0.2s, v8.2s[1] + fmla v25.2s, v1.2s, v9.2s[0] + + fmla v24.2s, v0.2s, v9.2s[0] + fmla v21.2s, v1.2s, v8.2s[1] + + fmla v28.2s, v0.2s, v9.2s[1] + fmla v17.2s, v1.2s, v8.2s[0] +.endm + +.macro SAVE4x4 + + fmul v8.2s, v16.2s, alphaV0 + fmul v9.2s, v17.2s, alphaV1 + st1 {v8.2s, v9.2s}, [pCRow0] + + add pCRow1, pCRow0, LDC + + fmul v12.2s, v20.2s, alphaV2 + fmul v13.2s, v21.2s, alphaV3 + st1 {v12.2s, v13.2s}, [pCRow1] + + add pCRow2, pCRow1, LDC + + fmul v8.2s, v24.2s, alphaV0 + fmul v9.2s, v25.2s, alphaV1 + st1 {v8.2s, v9.2s}, [pCRow2] + + add pCRow1, pCRow2, LDC + + fmul v12.2s, v28.2s, alphaV2 + fmul v13.2s, v29.2s, alphaV3 + st1 {v12.2s, v13.2s}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT2x4 + fmov s16, wzr + fmov s20, s16 + fmov s24, s20 + fmov s28, s16 +.endm + +.macro KERNEL2x4_SUB + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.2s}, [pA] + add pA, pA, #8 + + fmla v16.2s, v0.2s, v8.2s[0] + fmla v20.2s, v0.2s, v8.2s[1] + fmla v24.2s, v0.2s, v9.2s[0] + fmla v28.2s, v0.2s, v9.2s[1] +.endm + +.macro SAVE2x4 + fmul v8.2s, v16.2s, alphaV0 + st1 {v8.2s}, [pCRow0] + + add pCRow1, pCRow0, LDC + fmul v12.2s, v20.2s, alphaV1 + st1 {v12.2s}, [pCRow1] + + add pCRow2, pCRow1, LDC + fmul v8.2s, v24.2s, alphaV2 + st1 {v8.2s}, [pCRow2] + + add pCRow1, pCRow2, LDC + fmul v12.2s, v28.2s, alphaV3 + st1 {v12.2s}, [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT1x4 + fmov s16, wzr + fmov s20, s16 +.endm + +.macro KERNEL1x4_SUB + ldr s0, [pA] + add pA, pA, #4 + + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + + fmla v16.2s, v8.2s, v0.s[0] + fmla v20.2s, v9.2s, v0.s[0] +.endm + +.macro SAVE1x4 + add pCRow1, pCRow0, LDC + + fmul v8.2s, v16.2s, alphaV0 + st1 {v8.s}[0], [pCRow0] + st1 {v8.s}[1], [pCRow1] + + add pCRow2, pCRow1, LDC + add pCRow1, pCRow2, LDC + + fmul v12.2s, v20.2s, alphaV1 + st1 {v12.s}[0], [pCRow2] + st1 {v12.s}[1], [pCRow1] + + add pCRow0, pCRow0, #4 +.endm + +/******************************************************************************/ + +.macro INIT16x2 + fmov s16, wzr + fmov s17, wzr + fmov s18, wzr + fmov s19, s16 + fmov s20, wzr + fmov s21, s16 + fmov s22, wzr + fmov s23, s16 +.endm + +.macro KERNEL16x2_SUB + ld1 {v8.2s}, [pB] + add pB, pB, #8 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + ld1 {v2.4s}, [pA] + add pA, pA, #16 + ld1 {v3.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v8.2s[0] + fmla v17.4s, v1.4s, v8.2s[0] + fmla v18.4s, v2.4s, v8.2s[0] + fmla v19.4s, v3.4s, v8.2s[0] + + fmla v20.4s, v0.4s, v8.2s[1] + fmla v21.4s, v1.4s, v8.2s[1] + fmla v22.4s, v2.4s, v8.2s[1] + fmla v23.4s, v3.4s, v8.2s[1] +.endm + +.macro SAVE16x2 + add pCRow1, pCRow0, LDC + + fmul v0.4s, v16.4s, alphaV0 + fmul v1.4s, v17.4s, alphaV1 + fmul v2.4s, v18.4s, alphaV2 + fmul v3.4s, v19.4s, alphaV3 + st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow0] + + fmul v4.4s, v20.4s, alphaV0 + fmul v5.4s, v21.4s, alphaV1 + fmul v6.4s, v22.4s, alphaV2 + fmul v7.4s, v23.4s, alphaV3 + st1 {v4.4s, v5.4s, v6.4s, v7.4s}, [pCRow1] + + add pCRow0, pCRow0, #64 +.endm + +/******************************************************************************/ + +.macro INIT8x2 + fmov s16, wzr + fmov s17, s16 + fmov s20, s17 + fmov s21, s16 +.endm + +.macro KERNEL8x2_SUB + ld1 {v8.2s}, [pB] + add pB, pB, #8 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v8.2s[0] + fmla v17.4s, v1.4s, v8.2s[0] + + fmla v20.4s, v0.4s, v8.2s[1] + fmla v21.4s, v1.4s, v8.2s[1] +.endm + +.macro SAVE8x2 + add pCRow1, pCRow0, LDC + + fmul v0.4s, v16.4s, alphaV0 + fmul v1.4s, v17.4s, alphaV1 + st1 {v0.4s, v1.4s}, [pCRow0] + + add pCRow2, pCRow1, LDC + + fmul v4.4s, v20.4s, alphaV0 + fmul v5.4s, v21.4s, alphaV1 + st1 {v4.4s, v5.4s}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT4x2 + fmov s16, wzr + fmov s17, s16 + fmov s20, s17 + fmov s21, s16 +.endm + +.macro KERNEL4x2_SUB + ld1 {v8.2s}, [pB] + add pB, pB, #8 + ld1 {v0.2s, v1.2s}, [pA] + add pA, pA, #16 + + fmla v16.2s, v0.2s, v8.2s[0] + fmla v17.2s, v1.2s, v8.2s[0] + fmla v20.2s, v0.2s, v8.2s[1] + fmla v21.2s, v1.2s, v8.2s[1] +.endm + +.macro SAVE4x2 + + fmul v8.2s, v16.2s, alphaV0 + fmul v9.2s, v17.2s, alphaV1 + st1 {v8.2s, v9.2s}, [pCRow0] + + add pCRow1, pCRow0, LDC + + fmul v12.2s, v20.2s, alphaV2 + fmul v13.2s, v21.2s, alphaV3 + st1 {v12.2s, v13.2s}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT2x2 + fmov s16, wzr + fmov s20, s16 +.endm + +.macro KERNEL2x2_SUB + ld1 {v8.2s}, [pB] + add pB, pB, #8 + + ld1 {v0.2s}, [pA] + add pA, pA, #8 + + fmla v16.2s, v0.2s, v8.2s[0] + fmla v20.2s, v0.2s, v8.2s[1] +.endm + +.macro SAVE2x2 + fmul v8.2s, v16.2s, alphaV0 + st1 {v8.2s}, [pCRow0] + + add pCRow1 , pCRow0, LDC + + fmul v12.2s, v20.2s, alphaV1 + st1 {v12.2s}, [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT1x2 + fmov s16, wzr +.endm + +.macro KERNEL1x2_SUB + ld1 {v8.2s} , [pB] + add pB , pB, #8 + + ldr s0 , [pA] + add pA, pA, #4 + + fmla v16.2s, v8.2s, v0.2s[0] +.endm + +.macro SAVE1x2 + add pCRow1 , pCRow0, LDC + + fmul v8.2s, v16.2s, alphaV0 + st1 {v8.s}[0], [pCRow0] + st1 {v8.s}[1], [pCRow1] + + add pCRow0, pCRow0, #4 +.endm + +/******************************************************************************/ + +.macro INIT16x1 + fmov s16, wzr + fmov s17, wzr + fmov s18, wzr + fmov s19, s16 +.endm + +.macro KERNEL16x1_SUB + ldr s8, [pB] + add pB , pB, #4 + + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + ld1 {v2.4s}, [pA] + add pA, pA, #16 + ld1 {v3.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v8.2s[0] + fmla v17.4s, v1.4s, v8.2s[0] + fmla v18.4s, v2.4s, v8.2s[0] + fmla v19.4s, v3.4s, v8.2s[0] +.endm + +.macro SAVE16x1 + + fmul v0.4s, v16.4s, alphaV0 + fmul v1.4s, v17.4s, alphaV1 + fmul v2.4s, v18.4s, alphaV2 + fmul v3.4s, v19.4s, alphaV3 + st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow0] + + add pCRow0, pCRow0, #64 +.endm + +/******************************************************************************/ + +.macro INIT8x1 + fmov s16, wzr + fmov s17, wzr +.endm + +.macro KERNEL8x1_SUB + ldr s8, [pB] + add pB , pB, #4 + + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v8.2s[0] + fmla v17.4s, v1.4s, v8.2s[0] +.endm + +.macro SAVE8x1 + + fmul v0.4s, v16.4s, alphaV0 + fmul v1.4s, v17.4s, alphaV1 + st1 {v0.4s, v1.4s}, [pCRow0] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT4x1 + fmov s16, wzr + fmov s17, s16 +.endm + +.macro KERNEL4x1_SUB + ldr s8, [pB] + add pB , pB, #4 + + ld1 {v0.2s, v1.2s}, [pA] + add pA , pA, #16 + + fmla v16.2s, v0.2s, v8.2s[0] + fmla v17.2s, v1.2s, v8.2s[0] +.endm + +.macro SAVE4x1 + + fmul v8.2s, v16.2s, alphaV0 + fmul v9.2s, v17.2s, alphaV1 + st1 {v8.2s, v9.2s}, [pCRow0] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT2x1 + fmov s16, wzr +.endm + +.macro KERNEL2x1_SUB + ldr s8, [pB] + add pB , pB, #4 + + ld1 {v0.2s}, [pA] + add pA , pA, #8 + + fmla v16.2s, v0.2s, v8.2s[0] +.endm + +.macro SAVE2x1 + + fmul v8.2s, v16.2s, alphaV0 + st1 {v8.2s}, [pCRow0] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT1x1 + fmov s16, wzr +.endm + +.macro KERNEL1x1_SUB + ldr s8, [pB] + add pB , pB, #4 + + ldr s0, [pA] + add pA , pA, #4 + + fmadd s16, s0, s8, s16 +.endm + +.macro SAVE1x1 + fmul s8, s16, alpha0 + str s8, [pCRow0] + + add pCRow0, pCRow0, #4 +.endm + +/******************************************************************************* +* End of macro definitions +*******************************************************************************/ + + PROLOGUE + +strmm_kernel_begin: + + .align 5 + add sp, sp, #-(11 * 16) + stp d8, d9, [sp, #(0 * 16)] + stp d10, d11, [sp, #(1 * 16)] + stp d12, d13, [sp, #(2 * 16)] + stp d14, d15, [sp, #(3 * 16)] + stp d16, d17, [sp, #(4 * 16)] + stp x18, x19, [sp, #(5 * 16)] + stp x20, x21, [sp, #(6 * 16)] + stp x22, x23, [sp, #(7 * 16)] + stp x24, x25, [sp, #(8 * 16)] + stp x26, x27, [sp, #(9 * 16)] + str x28, [sp, #(10 * 16)] + + fmov alpha0, s0 + fmov alpha1, s0 + fmov alpha2, s0 + fmov alpha3, s0 + + lsl LDC, LDC, #2 // ldc = ldc * 4 + +#if !defined(LEFT) + neg tempOffset, offset +#endif + mov pB, origPB + + mov counterJ, origN + asr counterJ, counterJ, #2 // J = J / 4 + cmp counterJ, #0 + ble strmm_kernel_L2_BEGIN + +/******************************************************************************/ + +strmm_kernel_L4_BEGIN: + mov pCRow0, pC // pCRow0 = C + add pC, pC, LDC, lsl #2 + +#if defined(LEFT) + mov tempOffset, offset +#endif + mov pA, origPA // pA = start of A array + +strmm_kernel_L4_M16_BEGIN: + + mov counterI, origM + asr counterI, counterI, #4 // counterI = counterI / 16 + cmp counterI, #0 + ble strmm_kernel_L4_M8_BEGIN + +strmm_kernel_L4_M16_20: + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #6 + add pA, pA, temp + lsl temp, tempOffset, #4 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #16 +#else + add tempK, tempOffset, #4 +#endif + + asr counterL , tempK, #1 // L = K / 2 + cmp counterL , #2 // is there at least 4 to do? + blt strmm_kernel_L4_M16_32 + + KERNEL16x4_I // do one in the K + KERNEL16x4_M2 // do another in the K + + subs counterL, counterL, #2 + ble strmm_kernel_L4_M16_22a + .align 5 + +strmm_kernel_L4_M16_22: + + KERNEL16x4_M1 + KERNEL16x4_M2 + + subs counterL, counterL, #1 + bgt strmm_kernel_L4_M16_22 + +strmm_kernel_L4_M16_22a: + + KERNEL16x4_M1 + KERNEL16x4_E + + b strmm_kernel_L4_M16_44 + +strmm_kernel_L4_M16_32: + + tst counterL, #1 + ble strmm_kernel_L4_M16_40 + + KERNEL16x4_I + KERNEL16x4_E + + b strmm_kernel_L4_M16_44 + +strmm_kernel_L4_M16_40: + + INIT16x4 + +strmm_kernel_L4_M16_44: + + ands counterL , tempK, #1 + ble strmm_kernel_L4_M16_100 + +strmm_kernel_L4_M16_46: + + KERNEL16x4_SUB + +strmm_kernel_L4_M16_100: + + SAVE16x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #16 +#else + sub tempK, tempK, #4 +#endif + lsl temp, tempK, #6 + add pA, pA, temp + lsl temp, tempK, #4 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #16 +#endif + +strmm_kernel_L4_M16_END: + subs counterI, counterI, #1 + bne strmm_kernel_L4_M16_20 + +//------------------------------------------------------------------------------ + +strmm_kernel_L4_M8_BEGIN: + + mov counterI, origM + tst counterI , #15 + ble strmm_kernel_L4_END + + tst counterI, #8 + ble strmm_kernel_L4_M4_BEGIN + +strmm_kernel_L4_M8_20: + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #5 + add pA, pA, temp + lsl temp, tempOffset, #4 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #8 +#else + add tempK, tempOffset, #4 +#endif + + asr counterL , tempK, #1 // L = K / 2 + cmp counterL , #2 // is there at least 4 to do? + blt strmm_kernel_L4_M8_32 + + KERNEL8x4_I // do one in the K + KERNEL8x4_M2 // do another in the K + + subs counterL, counterL, #2 + ble strmm_kernel_L4_M8_22a + .align 5 + +strmm_kernel_L4_M8_22: + + KERNEL8x4_M1 + KERNEL8x4_M2 + + subs counterL, counterL, #1 + bgt strmm_kernel_L4_M8_22 + +strmm_kernel_L4_M8_22a: + + KERNEL8x4_M1 + KERNEL8x4_E + + b strmm_kernel_L4_M8_44 + +strmm_kernel_L4_M8_32: + + tst counterL, #1 + ble strmm_kernel_L4_M8_40 + + KERNEL8x4_I + KERNEL8x4_E + + b strmm_kernel_L4_M8_44 + +strmm_kernel_L4_M8_40: + + INIT8x4 + +strmm_kernel_L4_M8_44: + + ands counterL , tempK, #1 + ble strmm_kernel_L4_M8_100 + +strmm_kernel_L4_M8_46: + + KERNEL8x4_SUB + +strmm_kernel_L4_M8_100: + + SAVE8x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #8 +#else + sub tempK, tempK, #4 +#endif + lsl temp, tempK, #5 + add pA, pA, temp + lsl temp, tempK, #4 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #8 +#endif + +strmm_kernel_L4_M8_END: + +//------------------------------------------------------------------------------ + +strmm_kernel_L4_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble strmm_kernel_L4_END + + tst counterI, #4 + ble strmm_kernel_L4_M2_BEGIN + +strmm_kernel_L4_M4_20: + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #4 + add pB, pB, temp + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #4 +#else + add tempK, tempOffset, #4 +#endif + asr counterL , tempK, #1 // L = K / 2 + cmp counterL , #2 // is there at least 4 to do? + blt strmm_kernel_L4_M4_32 + + KERNEL4x4_I // do one in the K + KERNEL4x4_M2 // do another in the K + + subs counterL, counterL, #2 + ble strmm_kernel_L4_M4_22a + .align 5 + +strmm_kernel_L4_M4_22: + + KERNEL4x4_M1 + KERNEL4x4_M2 + + subs counterL, counterL, #1 + bgt strmm_kernel_L4_M4_22 + +strmm_kernel_L4_M4_22a: + + KERNEL4x4_M1 + KERNEL4x4_E + + b strmm_kernel_L4_M4_44 + +strmm_kernel_L4_M4_32: + + tst counterL, #1 + ble strmm_kernel_L4_M4_40 + + KERNEL4x4_I + KERNEL4x4_E + + b strmm_kernel_L4_M4_44 + +strmm_kernel_L4_M4_40: + + INIT4x4 + +strmm_kernel_L4_M4_44: + + ands counterL , tempK, #1 + ble strmm_kernel_L4_M4_100 + +strmm_kernel_L4_M4_46: + + KERNEL4x4_SUB + +strmm_kernel_L4_M4_100: + + SAVE4x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #4 +#else + sub tempK, tempK, #4 +#endif + lsl temp, tempK, #4 + add pA, pA, temp + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #4 +#endif +strmm_kernel_L4_M4_END: + +//------------------------------------------------------------------------------ + +strmm_kernel_L4_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble strmm_kernel_L4_END + + tst counterI, #2 // counterI = counterI / 2 + ble strmm_kernel_L4_M1_BEGIN + +strmm_kernel_L4_M2_20: + + INIT2x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #3 + add pA, pA, temp + lsl temp, tempOffset, #4 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #2 +#else + add tempK, tempOffset, #4 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble strmm_kernel_L4_M2_40 + +strmm_kernel_L4_M2_22: + + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L4_M2_22 + + +strmm_kernel_L4_M2_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble strmm_kernel_L4_M2_100 + +strmm_kernel_L4_M2_42: + + KERNEL2x4_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L4_M2_42 + +strmm_kernel_L4_M2_100: + + SAVE2x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #2 +#else + sub tempK, tempK, #4 +#endif + lsl temp, tempK, #3 + add pA, pA, temp + lsl temp, tempK, #4 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #2 +#endif +strmm_kernel_L4_M2_END: + + +strmm_kernel_L4_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble strmm_kernel_L4_END + +strmm_kernel_L4_M1_20: + + INIT1x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #4 + add pB, pB, temp + lsl temp, tempOffset, #2 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #1 +#else + add tempK, tempOffset, #4 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble strmm_kernel_L4_M1_40 + +strmm_kernel_L4_M1_22: + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L4_M1_22 + + +strmm_kernel_L4_M1_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble strmm_kernel_L4_M1_100 + +strmm_kernel_L4_M1_42: + + KERNEL1x4_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L4_M1_42 + +strmm_kernel_L4_M1_100: + + SAVE1x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #1 +#else + sub tempK, tempK, #4 +#endif + lsl temp, tempK, #2 + add pA, pA, temp + lsl temp, tempK, #4 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #1 +#endif +strmm_kernel_L4_END: + add origPB, origPB, origK, lsl #4 // B = B + K * 4 * 4 +#if !defined(LEFT) + add tempOffset, tempOffset, #4 +#endif + + subs counterJ, counterJ , #1 // j-- + bgt strmm_kernel_L4_BEGIN + + +/******************************************************************************/ + +strmm_kernel_L2_BEGIN: // less than 2 left in N direction + + mov counterJ , origN + tst counterJ , #3 + ble strmm_kernel_L999 + + tst counterJ , #2 + ble strmm_kernel_L1_BEGIN + + mov pCRow0, pC // pCRow0 = pC + + add pC,pC,LDC, lsl #1 + +#if defined(LEFT) + mov tempOffset, offset +#endif + mov pA, origPA // pA = A + +strmm_kernel_L2_M16_BEGIN: + + mov counterI, origM + asr counterI, counterI, #4 // counterI = counterI / 16 + cmp counterI,#0 + ble strmm_kernel_L2_M8_BEGIN + +strmm_kernel_L2_M16_20: + + INIT16x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #6 + add pA, pA, temp + lsl temp, tempOffset, #3 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #16 +#else + add tempK, tempOffset, #2 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble strmm_kernel_L2_M16_40 + .align 5 + +strmm_kernel_L2_M16_22: + KERNEL16x2_SUB + KERNEL16x2_SUB + KERNEL16x2_SUB + KERNEL16x2_SUB + + KERNEL16x2_SUB + KERNEL16x2_SUB + KERNEL16x2_SUB + KERNEL16x2_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L2_M16_22 + + +strmm_kernel_L2_M16_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble strmm_kernel_L2_M16_100 + +strmm_kernel_L2_M16_42: + + KERNEL16x2_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L2_M16_42 + +strmm_kernel_L2_M16_100: + + SAVE16x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #16 +#else + sub tempK, tempK, #2 +#endif + lsl temp, tempK, #6 + add pA, pA, temp + lsl temp, tempK, #3 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #16 +#endif + +strmm_kernel_L2_M16_END: + + subs counterI, counterI, #1 + bgt strmm_kernel_L2_M16_20 + +//------------------------------------------------------------------------------ + +strmm_kernel_L2_M8_BEGIN: + mov counterI, origM + tst counterI , #15 + ble strmm_kernel_L2_END + + tst counterI, #8 + ble strmm_kernel_L2_M4_BEGIN + +strmm_kernel_L2_M8_20: + + INIT8x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #5 + add pA, pA, temp + lsl temp, tempOffset, #3 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #8 +#else + add tempK, tempOffset, #2 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble strmm_kernel_L2_M8_40 + .align 5 + +strmm_kernel_L2_M8_22: + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L2_M8_22 + + +strmm_kernel_L2_M8_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble strmm_kernel_L2_M8_100 + +strmm_kernel_L2_M8_42: + + KERNEL8x2_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L2_M8_42 + +strmm_kernel_L2_M8_100: + + SAVE8x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #8 +#else + sub tempK, tempK, #2 +#endif + lsl temp, tempK, #5 + add pA, pA, temp + lsl temp, tempK, #3 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #8 +#endif + +strmm_kernel_L2_M8_END: + +//------------------------------------------------------------------------------ + +strmm_kernel_L2_M4_BEGIN: + mov counterI, origM + tst counterI , #7 + ble strmm_kernel_L2_END + + tst counterI, #4 + ble strmm_kernel_L2_M2_BEGIN + +strmm_kernel_L2_M4_20: + + INIT4x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #3 + add pB, pB, temp + lsl temp, tempOffset, #4 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #4 +#else + add tempK, tempOffset, #2 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble strmm_kernel_L2_M4_40 + .align 5 + +strmm_kernel_L2_M4_22: + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L2_M4_22 + + +strmm_kernel_L2_M4_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble strmm_kernel_L2_M4_100 + +strmm_kernel_L2_M4_42: + + KERNEL4x2_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L2_M4_42 + +strmm_kernel_L2_M4_100: + + SAVE4x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #4 +#else + sub tempK, tempK, #2 +#endif + lsl temp, tempK, #4 + add pA, pA, temp + lsl temp, tempK, #3 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #4 +#endif +strmm_kernel_L2_M4_END: + +//------------------------------------------------------------------------------ + + +strmm_kernel_L2_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble strmm_kernel_L2_END + + tst counterI, #2 // counterI = counterI / 2 + ble strmm_kernel_L2_M1_BEGIN + +strmm_kernel_L2_M2_20: + + INIT2x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #3 + add pB, pB, temp + lsl temp, tempOffset, #3 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #2 +#else + add tempK, tempOffset, #2 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble strmm_kernel_L2_M2_40 + +strmm_kernel_L2_M2_22: + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L2_M2_22 + + +strmm_kernel_L2_M2_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble strmm_kernel_L2_M2_100 + +strmm_kernel_L2_M2_42: + + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L2_M2_42 + +strmm_kernel_L2_M2_100: + + SAVE2x2 +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #2 +#else + sub tempK, tempK, #2 +#endif + lsl temp, tempK, #3 + add pA, pA, temp + lsl temp, tempK, #3 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #2 +#endif + +strmm_kernel_L2_M2_END: + + +strmm_kernel_L2_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble strmm_kernel_L2_END + +strmm_kernel_L2_M1_20: + + INIT1x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #3 + add pB, pB, temp + lsl temp, tempOffset, #2 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #1 +#else + add tempK, tempOffset, #2 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL, #0 + ble strmm_kernel_L2_M1_40 + +strmm_kernel_L2_M1_22: + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L2_M1_22 + + +strmm_kernel_L2_M1_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble strmm_kernel_L2_M1_100 + +strmm_kernel_L2_M1_42: + + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L2_M1_42 + +strmm_kernel_L2_M1_100: + + SAVE1x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #1 +#else + sub tempK, tempK, #2 +#endif + lsl temp, tempK, #2 + add pA, pA, temp + lsl temp, tempK, #3 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #1 +#endif +strmm_kernel_L2_END: +#if !defined(LEFT) + add tempOffset, tempOffset, #2 +#endif + add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4 + +/******************************************************************************/ + +strmm_kernel_L1_BEGIN: + + mov counterJ , origN + tst counterJ , #1 + ble strmm_kernel_L999 // done + + + mov pCRow0, pC // pCRow0 = C + add pC , pC , LDC // Update pC to point to next + +#if defined(LEFT) + mov tempOffset, offset +#endif + mov pA, origPA // pA = A + +strmm_kernel_L1_M16_BEGIN: + + mov counterI, origM + asr counterI, counterI, #4 // counterI = counterI / 16 + cmp counterI, #0 + ble strmm_kernel_L1_M8_BEGIN + +strmm_kernel_L1_M16_20: + + INIT16x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #6 + add pA, pA, temp + lsl temp, tempOffset, #2 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #16 +#else + add tempK, tempOffset, #1 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble strmm_kernel_L1_M16_40 + .align 5 + +strmm_kernel_L1_M16_22: + KERNEL16x1_SUB + KERNEL16x1_SUB + KERNEL16x1_SUB + KERNEL16x1_SUB + + KERNEL16x1_SUB + KERNEL16x1_SUB + KERNEL16x1_SUB + KERNEL16x1_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L1_M16_22 + + +strmm_kernel_L1_M16_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble strmm_kernel_L1_M16_100 + +strmm_kernel_L1_M16_42: + + KERNEL16x1_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L1_M16_42 + +strmm_kernel_L1_M16_100: + + SAVE16x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #16 +#else + sub tempK, tempK, #1 +#endif + lsl temp, tempK, #6 + add pA, pA, temp + lsl temp, tempK, #2 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #16 +#endif + +strmm_kernel_L1_M16_END: + + subs counterI, counterI, #1 + bgt strmm_kernel_L1_M16_20 + +//------------------------------------------------------------------------------ + +strmm_kernel_L1_M8_BEGIN: + + mov counterI, origM + tst counterI , #15 + ble strmm_kernel_L1_END + + tst counterI, #8 + ble strmm_kernel_L1_M4_BEGIN + +strmm_kernel_L1_M8_20: + + INIT8x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #5 + add pA, pA, temp + lsl temp, tempOffset, #2 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #8 +#else + add tempK, tempOffset, #1 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble strmm_kernel_L1_M8_40 + .align 5 + +strmm_kernel_L1_M8_22: + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L1_M8_22 + + +strmm_kernel_L1_M8_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble strmm_kernel_L1_M8_100 + +strmm_kernel_L1_M8_42: + + KERNEL8x1_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L1_M8_42 + +strmm_kernel_L1_M8_100: + + SAVE8x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #8 +#else + sub tempK, tempK, #1 +#endif + lsl temp, tempK, #5 + add pA, pA, temp + lsl temp, tempK, #2 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #8 +#endif + +strmm_kernel_L1_M8_END: + +//------------------------------------------------------------------------------ + +strmm_kernel_L1_M4_BEGIN: + mov counterI, origM + tst counterI , #7 + ble strmm_kernel_L1_END + + tst counterI, #4 + ble strmm_kernel_L1_M2_BEGIN + +strmm_kernel_L1_M4_20: + + INIT4x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #2 + add pB, pB, temp + lsl temp, tempOffset, #4 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #4 +#else + add tempK, tempOffset, #1 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble strmm_kernel_L1_M4_40 + .align 5 + +strmm_kernel_L1_M4_22: + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L1_M4_22 + + +strmm_kernel_L1_M4_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble strmm_kernel_L1_M4_100 + +strmm_kernel_L1_M4_42: + + KERNEL4x1_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L1_M4_42 + +strmm_kernel_L1_M4_100: + + SAVE4x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #4 +#else + sub tempK, tempK, #1 +#endif + lsl temp, tempK, #4 + add pA, pA, temp + lsl temp, tempK, #2 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #4 +#endif +strmm_kernel_L1_M4_END: + +//------------------------------------------------------------------------------ + +strmm_kernel_L1_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble strmm_kernel_L1_END + + tst counterI, #2 // counterI = counterI / 2 + ble strmm_kernel_L1_M1_BEGIN + +strmm_kernel_L1_M2_20: + + INIT2x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #2 + add pB, pB, temp + lsl temp, tempOffset, #3 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #2 +#else + add tempK, tempOffset, #1 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble strmm_kernel_L1_M2_40 + +strmm_kernel_L1_M2_22: + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L1_M2_22 + + +strmm_kernel_L1_M2_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble strmm_kernel_L1_M2_100 + +strmm_kernel_L1_M2_42: + + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L1_M2_42 + +strmm_kernel_L1_M2_100: + + SAVE2x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #2 +#else + sub tempK, tempK, #1 +#endif + lsl temp, tempK, #3 + add pA, pA, temp + lsl temp, tempK, #2 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #2 +#endif +strmm_kernel_L1_M2_END: + + +strmm_kernel_L1_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble strmm_kernel_L1_END + +strmm_kernel_L1_M1_20: + + INIT1x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #2 + add pB, pB, temp + lsl temp, tempOffset, #2 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #1 +#else + add tempK, tempOffset, #1 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble strmm_kernel_L1_M1_40 + +strmm_kernel_L1_M1_22: + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L1_M1_22 + + +strmm_kernel_L1_M1_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble strmm_kernel_L1_M1_100 + +strmm_kernel_L1_M1_42: + + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L1_M1_42 + +strmm_kernel_L1_M1_100: + + SAVE1x1 + +strmm_kernel_L1_END: + +strmm_kernel_L999: + mov x0, #0 // set return value + ldp d8, d9, [sp, #(0 * 16)] + ldp d10, d11, [sp, #(1 * 16)] + ldp d12, d13, [sp, #(2 * 16)] + ldp d14, d15, [sp, #(3 * 16)] + ldp d16, d17, [sp, #(4 * 16)] + ldp x18, x19, [sp, #(5 * 16)] + ldp x20, x21, [sp, #(6 * 16)] + ldp x22, x23, [sp, #(7 * 16)] + ldp x24, x25, [sp, #(8 * 16)] + ldp x26, x27, [sp, #(9 * 16)] + ldr x28, [sp, #(10 * 16)] + add sp, sp, #(11*16) + ret + + EPILOGUE + diff --git a/kernel/arm64/strmm_kernel_8x8.S b/kernel/arm64/strmm_kernel_8x8.S new file mode 100755 index 000000000..98b912934 --- /dev/null +++ b/kernel/arm64/strmm_kernel_8x8.S @@ -0,0 +1,2795 @@ +/******************************************************************************* +Copyright (c) 2015, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#define ASSEMBLER +#include "common.h" + +/* X0 X1 X2 s0 X3 x4 x5 x6 x7 */ +/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc, BLASLONG offset) */ + +#define origM x0 +#define origN x1 +#define origK x2 +#define origPA x3 +#define origPB x4 +#define pC x5 +#define LDC x6 +#define offset x7 +#define counterL x8 +#define counterI x9 +#define counterJ x10 +#define pB x11 +#define pCRow0 x12 +#define pCRow1 x13 +#define pCRow2 x14 +#define pA x15 +#define temp x16 +#define tempOffset x17 +#define tempK x18 + +#define alpha0 s10 +#define alphaV0 v10.s[0] +#define alpha1 s11 +#define alphaV1 v11.s[0] +#define alpha2 s14 +#define alphaV2 v14.s[0] +#define alpha3 s15 +#define alphaV3 v15.s[0] + +// 00 origM +// 01 origN +// 02 origK +// 03 origPA +// 04 origPB +// 05 pC +// 06 origLDC -> LDC +// 07 offset +// 08 counterL +// 09 counterI +// 10 counterJ +// 11 pB +// 12 pCRow0 +// 13 pCRow1 +// 14 pCRow2 +// 15 pA +// 16 temp +// 17 tempOffset +// 18 must save tempK +// 19 must save +// 20 must save +// 21 must save +// 22 must save +// 23 must save +// 24 must save +// 25 must save +// 26 must save +// 27 must save +// 28 must save +// 29 frame +// 30 link +// 31 sp + +//v00 ALPHA -> pA0_0, pA0_1, pA0_2, pA0_3 +//v01 pA0_4, pA0_5, pA0_6, pA0_7 +//v02 pA1_0, pA1_1, pA1_2, pA1_3 +//v03 pA1_4, pA1_5, pA1_6, pA1_7 +//v04 pB0_0, pB0_1, pB0_2, pB0_3 +//v05 pB0_4, pB0_5, pB0_6, pB0_7 +//v06 pB1_0, pB1_1, pB1_2, pB1_3 +//v07 pB1_4, pB1_5, pB1_6, pB1_7 +//v08 must save +//v09 must save +//v10 must save ALPHA0 +//v11 must save ALPHA1 +//v12 must save +//v13 must save +//v14 must save ALPHA2 +//v15 must save ALPHA3 +//v16 must save C00, C01, C02, C03 +//v17 must save C04, C05, C06, C07 +//v18 C08, C09, C10, C11 +//v19 C12, C13, C14, C15 +//v20 C16, C17, C18, C19 +//v21 C20, C21, C22, C23 +//v22 C24, C25, C26, C27 +//v23 C28, C29, C30, C31 +//v24 C32, C33, C34, C35 +//v25 C36, C37, C38, C39 +//v26 C40, C41, C42, C43 +//v27 C44, C45, C46, C47 +//v28 C48, C49, C50, C51 +//v29 C52, C53, C54, C55 +//v30 C56, C57, C58, C59 +//v31 C60, C61, C62, C63 + +/******************************************************************************* +* Macro definitions +*******************************************************************************/ + +.macro INIT8x8 + fmov s16, wzr + fmov s17, wzr + fmov s18, s16 + fmov s19, s17 + fmov s20, wzr + fmov s21, s16 + fmov s22, s17 + fmov s23, s18 + fmov s24, wzr + fmov s25, s16 + fmov s26, s17 + fmov s27, s18 + fmov s28, wzr + fmov s29, s16 + fmov s30, s17 + fmov s31, s18 +.endm + +.macro KERNEL8x8_I + ld1 {v4.4s}, [pB] + add pB, pB, #16 + ld1 {v5.4s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + + fmul v16.4s, v0.4s, v4.4s[0] + fmul v17.4s, v1.4s, v4.4s[0] + fmul v18.4s, v0.4s, v4.4s[1] + fmul v19.4s, v1.4s, v4.4s[1] + fmul v20.4s, v0.4s, v4.4s[2] + fmul v21.4s, v1.4s, v4.4s[2] + fmul v22.4s, v0.4s, v4.4s[3] + fmul v23.4s, v1.4s, v4.4s[3] + fmul v24.4s, v0.4s, v5.4s[0] + fmul v25.4s, v1.4s, v5.4s[0] + fmul v26.4s, v0.4s, v5.4s[1] + fmul v27.4s, v1.4s, v5.4s[1] + fmul v28.4s, v0.4s, v5.4s[2] + fmul v29.4s, v1.4s, v5.4s[2] + fmul v30.4s, v0.4s, v5.4s[3] + fmul v31.4s, v1.4s, v5.4s[3] + + ld1 {v6.4s}, [pB] + add pB, pB, #16 + ld1 {v7.4s}, [pB] + add pB, pB, #16 + ld1 {v2.4s}, [pA] + add pA, pA, #16 + ld1 {v3.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL8x8_M1 + fmla v16.4s, v0.4s, v4.4s[0] + fmla v17.4s, v1.4s, v4.4s[0] + fmla v18.4s, v0.4s, v4.4s[1] + fmla v19.4s, v1.4s, v4.4s[1] + fmla v20.4s, v0.4s, v4.4s[2] + fmla v21.4s, v1.4s, v4.4s[2] + fmla v22.4s, v0.4s, v4.4s[3] + fmla v23.4s, v1.4s, v4.4s[3] + fmla v24.4s, v0.4s, v5.4s[0] + fmla v25.4s, v1.4s, v5.4s[0] + fmla v26.4s, v0.4s, v5.4s[1] + fmla v27.4s, v1.4s, v5.4s[1] + fmla v28.4s, v0.4s, v5.4s[2] + fmla v29.4s, v1.4s, v5.4s[2] + fmla v30.4s, v0.4s, v5.4s[3] + fmla v31.4s, v1.4s, v5.4s[3] + + ld1 {v6.4s}, [pB] + add pB, pB, #16 + ld1 {v7.4s}, [pB] + add pB, pB, #16 + ld1 {v2.4s}, [pA] + add pA, pA, #16 + ld1 {v3.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL8x8_M2 + fmla v16.4s, v2.4s, v6.4s[0] + fmla v17.4s, v3.4s, v6.4s[0] + fmla v18.4s, v2.4s, v6.4s[1] + fmla v19.4s, v3.4s, v6.4s[1] + fmla v20.4s, v2.4s, v6.4s[2] + fmla v21.4s, v3.4s, v6.4s[2] + fmla v22.4s, v2.4s, v6.4s[3] + fmla v23.4s, v3.4s, v6.4s[3] + fmla v24.4s, v2.4s, v7.4s[0] + fmla v25.4s, v3.4s, v7.4s[0] + fmla v26.4s, v2.4s, v7.4s[1] + fmla v27.4s, v3.4s, v7.4s[1] + fmla v28.4s, v2.4s, v7.4s[2] + fmla v29.4s, v3.4s, v7.4s[2] + fmla v30.4s, v2.4s, v7.4s[3] + fmla v31.4s, v3.4s, v7.4s[3] + + ld1 {v4.4s}, [pB] + add pB, pB, #16 + ld1 {v5.4s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL8x8_E + fmla v16.4s, v2.4s, v6.4s[0] + fmla v17.4s, v3.4s, v6.4s[0] + fmla v18.4s, v2.4s, v6.4s[1] + fmla v19.4s, v3.4s, v6.4s[1] + fmla v20.4s, v2.4s, v6.4s[2] + fmla v21.4s, v3.4s, v6.4s[2] + fmla v22.4s, v2.4s, v6.4s[3] + fmla v23.4s, v3.4s, v6.4s[3] + fmla v24.4s, v2.4s, v7.4s[0] + fmla v25.4s, v3.4s, v7.4s[0] + fmla v26.4s, v2.4s, v7.4s[1] + fmla v27.4s, v3.4s, v7.4s[1] + fmla v28.4s, v2.4s, v7.4s[2] + fmla v29.4s, v3.4s, v7.4s[2] + fmla v30.4s, v2.4s, v7.4s[3] + fmla v31.4s, v3.4s, v7.4s[3] +.endm + +.macro KERNEL8x8_SUB + ld1 {v4.4s}, [pB] + add pB, pB, #16 + ld1 {v5.4s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v4.4s[0] + fmla v17.4s, v1.4s, v4.4s[0] + fmla v18.4s, v0.4s, v4.4s[1] + fmla v19.4s, v1.4s, v4.4s[1] + fmla v20.4s, v0.4s, v4.4s[2] + fmla v21.4s, v1.4s, v4.4s[2] + fmla v22.4s, v0.4s, v4.4s[3] + fmla v23.4s, v1.4s, v4.4s[3] + fmla v24.4s, v0.4s, v5.4s[0] + fmla v25.4s, v1.4s, v5.4s[0] + fmla v26.4s, v0.4s, v5.4s[1] + fmla v27.4s, v1.4s, v5.4s[1] + fmla v28.4s, v0.4s, v5.4s[2] + fmla v29.4s, v1.4s, v5.4s[2] + fmla v30.4s, v0.4s, v5.4s[3] + fmla v31.4s, v1.4s, v5.4s[3] +.endm + +.macro SAVE8x8 + add pCRow1, pCRow0, LDC + + fmul v0.4s, v16.4s, alphaV0 + fmul v1.4s, v17.4s, alphaV1 + st1 {v0.4s, v1.4s}, [pCRow0] + + add pCRow2, pCRow1, LDC + + fmul v2.4s, v18.4s, alphaV2 + fmul v3.4s, v19.4s, alphaV3 + st1 {v2.4s, v3.4s}, [pCRow1] + + add pCRow1, pCRow2, LDC + + fmul v4.4s, v20.4s, alphaV0 + fmul v5.4s, v21.4s, alphaV1 + st1 {v4.4s, v5.4s}, [pCRow2] + + add pCRow2, pCRow1, LDC + + fmul v6.4s, v22.4s, alphaV2 + fmul v7.4s, v23.4s, alphaV3 + st1 {v6.4s, v7.4s}, [pCRow1] + + add pCRow1, pCRow2, LDC + + fmul v0.4s, v24.4s, alphaV0 + fmul v1.4s, v25.4s, alphaV1 + st1 {v0.4s, v1.4s}, [pCRow2] + + add pCRow2, pCRow1, LDC + + fmul v2.4s, v26.4s, alphaV2 + fmul v3.4s, v27.4s, alphaV3 + st1 {v2.4s, v3.4s}, [pCRow1] + + add pCRow1, pCRow2, LDC + + fmul v4.4s, v28.4s, alphaV0 + fmul v5.4s, v29.4s, alphaV1 + st1 {v4.4s, v5.4s}, [pCRow2] + + fmul v6.4s, v30.4s, alphaV2 + fmul v7.4s, v31.4s, alphaV3 + st1 {v6.4s, v7.4s}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + + +.macro INIT4x8 + fmov s16, wzr + fmov s18, wzr + fmov s20, wzr + fmov s22, s16 + fmov s24, wzr + fmov s26, s16 + fmov s28, s18 + fmov s30, s20 +.endm + +.macro KERNEL4x8_I + ld1 {v4.4s}, [pB] + add pB, pB, #16 + ld1 {v5.4s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + + fmul v16.4s, v0.4s, v4.4s[0] + fmul v18.4s, v0.4s, v4.4s[1] + fmul v20.4s, v0.4s, v4.4s[2] + fmul v22.4s, v0.4s, v4.4s[3] + fmul v24.4s, v0.4s, v5.4s[0] + fmul v26.4s, v0.4s, v5.4s[1] + fmul v28.4s, v0.4s, v5.4s[2] + fmul v30.4s, v0.4s, v5.4s[3] + + ld1 {v6.4s}, [pB] + add pB, pB, #16 + ld1 {v7.4s}, [pB] + add pB, pB, #16 + ld1 {v2.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL4x8_M1 + fmla v16.4s, v0.4s, v4.4s[0] + fmla v18.4s, v0.4s, v4.4s[1] + fmla v20.4s, v0.4s, v4.4s[2] + fmla v22.4s, v0.4s, v4.4s[3] + fmla v24.4s, v0.4s, v5.4s[0] + fmla v26.4s, v0.4s, v5.4s[1] + fmla v28.4s, v0.4s, v5.4s[2] + fmla v30.4s, v0.4s, v5.4s[3] + + ld1 {v6.4s}, [pB] + add pB, pB, #16 + ld1 {v7.4s}, [pB] + add pB, pB, #16 + ld1 {v2.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL4x8_M2 + fmla v16.4s, v2.4s, v6.4s[0] + fmla v18.4s, v2.4s, v6.4s[1] + fmla v20.4s, v2.4s, v6.4s[2] + fmla v22.4s, v2.4s, v6.4s[3] + fmla v24.4s, v2.4s, v7.4s[0] + fmla v26.4s, v2.4s, v7.4s[1] + fmla v28.4s, v2.4s, v7.4s[2] + fmla v30.4s, v2.4s, v7.4s[3] + + ld1 {v4.4s}, [pB] + add pB, pB, #16 + ld1 {v5.4s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL4x8_E + fmla v16.4s, v2.4s, v6.4s[0] + fmla v18.4s, v2.4s, v6.4s[1] + fmla v20.4s, v2.4s, v6.4s[2] + fmla v22.4s, v2.4s, v6.4s[3] + fmla v24.4s, v2.4s, v7.4s[0] + fmla v26.4s, v2.4s, v7.4s[1] + fmla v28.4s, v2.4s, v7.4s[2] + fmla v30.4s, v2.4s, v7.4s[3] +.endm + +.macro KERNEL4x8_SUB + ld1 {v4.4s}, [pB] + add pB, pB, #16 + ld1 {v5.4s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v4.4s[0] + fmla v18.4s, v0.4s, v4.4s[1] + fmla v20.4s, v0.4s, v4.4s[2] + fmla v22.4s, v0.4s, v4.4s[3] + fmla v24.4s, v0.4s, v5.4s[0] + fmla v26.4s, v0.4s, v5.4s[1] + fmla v28.4s, v0.4s, v5.4s[2] + fmla v30.4s, v0.4s, v5.4s[3] +.endm + +.macro SAVE4x8 + add pCRow1, pCRow0, LDC + + + fmul v0.4s, v16.4s, alphaV0 + st1 {v0.4s}, [pCRow0] + + add pCRow2, pCRow1, LDC + + + fmul v2.4s, v18.4s, alphaV2 + st1 {v2.4s}, [pCRow1] + + add pCRow1, pCRow2, LDC + + + fmul v4.4s, v20.4s, alphaV0 + st1 {v4.4s}, [pCRow2] + + add pCRow2, pCRow1, LDC + + + fmul v6.4s, v22.4s, alphaV2 + st1 {v6.4s}, [pCRow1] + + add pCRow1, pCRow2, LDC + + + fmul v0.4s, v24.4s, alphaV0 + st1 {v0.4s}, [pCRow2] + + add pCRow2, pCRow1, LDC + + + fmul v2.4s, v26.4s, alphaV2 + st1 {v2.4s}, [pCRow1] + + add pCRow1, pCRow2, LDC + + + fmul v4.4s, v28.4s, alphaV0 + st1 {v4.4s}, [pCRow2] + + + fmul v6.4s, v30.4s, alphaV2 + st1 {v6.4s}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT2x8 + fmov s16, wzr + fmov s18, wzr + fmov s20, wzr + fmov s22, s16 + fmov s24, wzr + fmov s26, s16 + fmov s28, s18 + fmov s30, s20 +.endm + +.macro KERNEL2x8_SUB + ld1 {v4.4s}, [pB] + add pB, pB, #16 + ld1 {v5.4s}, [pB] + add pB, pB, #16 + ld1 {v0.2s}, [pA] + add pA, pA, #8 + + fmla v16.2s, v0.2s, v4.4s[0] + fmla v18.2s, v0.2s, v4.4s[1] + fmla v20.2s, v0.2s, v4.4s[2] + fmla v22.2s, v0.2s, v4.4s[3] + fmla v24.2s, v0.2s, v5.4s[0] + fmla v26.2s, v0.2s, v5.4s[1] + fmla v28.2s, v0.2s, v5.4s[2] + fmla v30.2s, v0.2s, v5.4s[3] +.endm + +.macro SAVE2x8 + add pCRow1, pCRow0, LDC + + + fmul v0.2s, v16.2s, alphaV0 + st1 {v0.2s}, [pCRow0] + + add pCRow2, pCRow1, LDC + + + fmul v2.2s, v18.2s, alphaV2 + st1 {v2.2s}, [pCRow1] + + add pCRow1, pCRow2, LDC + + + fmul v4.2s, v20.2s, alphaV0 + st1 {v4.2s}, [pCRow2] + + add pCRow2, pCRow1, LDC + + + fmul v6.2s, v22.2s, alphaV2 + st1 {v6.2s}, [pCRow1] + + add pCRow1, pCRow2, LDC + + + fmul v0.2s, v24.2s, alphaV0 + st1 {v0.2s}, [pCRow2] + + add pCRow2, pCRow1, LDC + + + fmul v2.2s, v26.2s, alphaV2 + st1 {v2.2s}, [pCRow1] + + add pCRow1, pCRow2, LDC + + + fmul v4.2s, v28.2s, alphaV0 + st1 {v4.2s}, [pCRow2] + + + fmul v6.2s, v30.2s, alphaV2 + st1 {v6.2s}, [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT1x8 + fmov s16, wzr + fmov s18, wzr + fmov s20, wzr + fmov s22, s16 + fmov s24, wzr + fmov s26, s16 + fmov s28, s18 + fmov s30, s20 +.endm + +.macro KERNEL1x8_SUB + ld1 {v4.4s}, [pB] + add pB, pB, #16 + ld1 {v5.4s}, [pB] + add pB, pB, #16 + ldr s0, [pA] + add pA, pA, #4 + + fmla s16, s0, v4.4s[0] + fmla s18, s0, v4.4s[1] + fmla s20, s0, v4.4s[2] + fmla s22, s0, v4.4s[3] + fmla s24, s0, v5.4s[0] + fmla s26, s0, v5.4s[1] + fmla s28, s0, v5.4s[2] + fmla s30, s0, v5.4s[3] +.endm + +.macro SAVE1x8 + add pCRow1, pCRow0, LDC + + + fmul s0, s16, alphaV0 + str s0, [pCRow0] + + add pCRow2, pCRow1, LDC + + + fmul s2, s18, alphaV2 + str s2, [pCRow1] + + add pCRow1, pCRow2, LDC + + + fmul s4, s20, alphaV0 + str s4, [pCRow2] + + add pCRow2, pCRow1, LDC + + + fmul s6, s22, alphaV2 + str s6, [pCRow1] + + add pCRow1, pCRow2, LDC + + + fmul s0, s24, alphaV0 + str s0, [pCRow2] + + add pCRow2, pCRow1, LDC + + + fmul s2, s26, alphaV2 + str s2, [pCRow1] + + add pCRow1, pCRow2, LDC + + + fmul s4, s28, alphaV0 + str s4, [pCRow2] + + + fmul s6, s30, alphaV2 + str s6, [pCRow1] + + add pCRow0, pCRow0, #4 +.endm + +/******************************************************************************/ + +.macro INIT8x4 + fmov s16, wzr + fmov s17, wzr + fmov s20, wzr + fmov s21, s16 + fmov s24, wzr + fmov s25, s16 + fmov s28, wzr + fmov s29, s16 +.endm + +.macro KERNEL8x4_I + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + + fmul v16.4s, v0.4s, v8.2s[0] + fmul v17.4s, v1.4s, v8.2s[0] + fmul v20.4s, v0.4s, v8.2s[1] + fmul v21.4s, v1.4s, v8.2s[1] + fmul v24.4s, v0.4s, v9.2s[0] + fmul v25.4s, v1.4s, v9.2s[0] + fmul v28.4s, v0.4s, v9.2s[1] + fmul v29.4s, v1.4s, v9.2s[1] + + ld1 {v12.2s, v13.2s}, [pB] + add pB, pB, #16 + ld1 {v4.4s}, [pA] + add pA, pA, #16 + ld1 {v5.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL8x4_M1 + fmla v16.4s, v0.4s, v8.2s[0] + fmla v17.4s, v1.4s, v8.2s[0] + fmla v20.4s, v0.4s, v8.2s[1] + fmla v21.4s, v1.4s, v8.2s[1] + fmla v24.4s, v0.4s, v9.2s[0] + fmla v25.4s, v1.4s, v9.2s[0] + fmla v28.4s, v0.4s, v9.2s[1] + fmla v29.4s, v1.4s, v9.2s[1] + + ld1 {v12.2s, v13.2s}, [pB] + add pB, pB, #16 + ld1 {v4.4s}, [pA] + add pA, pA, #16 + ld1 {v5.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL8x4_M2 + fmla v16.4s, v4.4s, v12.2s[0] + fmla v17.4s, v5.4s, v12.2s[0] + fmla v20.4s, v4.4s, v12.2s[1] + fmla v21.4s, v5.4s, v12.2s[1] + fmla v24.4s, v4.4s, v13.2s[0] + fmla v25.4s, v5.4s, v13.2s[0] + fmla v28.4s, v4.4s, v13.2s[1] + fmla v29.4s, v5.4s, v13.2s[1] + + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL8x4_E + fmla v16.4s, v4.4s, v12.2s[0] + fmla v17.4s, v5.4s, v12.2s[0] + fmla v20.4s, v4.4s, v12.2s[1] + fmla v21.4s, v5.4s, v12.2s[1] + fmla v24.4s, v4.4s, v13.2s[0] + fmla v25.4s, v5.4s, v13.2s[0] + fmla v28.4s, v4.4s, v13.2s[1] + fmla v29.4s, v5.4s, v13.2s[1] +.endm + +.macro KERNEL8x4_SUB + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v8.2s[0] + fmla v17.4s, v1.4s, v8.2s[0] + fmla v20.4s, v0.4s, v8.2s[1] + fmla v21.4s, v1.4s, v8.2s[1] + fmla v24.4s, v0.4s, v9.2s[0] + fmla v25.4s, v1.4s, v9.2s[0] + fmla v28.4s, v0.4s, v9.2s[1] + fmla v29.4s, v1.4s, v9.2s[1] +.endm + +.macro SAVE8x4 + add pCRow1, pCRow0, LDC + + + fmul v0.4s, v16.4s, alphaV0 + fmul v1.4s, v17.4s, alphaV1 + st1 {v0.4s, v1.4s}, [pCRow0] + + add pCRow2, pCRow1, LDC + + + fmul v4.4s, v20.4s, alphaV0 + fmul v5.4s, v21.4s, alphaV1 + st1 {v4.4s, v5.4s}, [pCRow1] + + add pCRow1, pCRow2, LDC + + + fmul v0.4s, v24.4s, alphaV0 + fmul v1.4s, v25.4s, alphaV1 + st1 {v0.4s, v1.4s}, [pCRow2] + + + fmul v4.4s, v28.4s, alphaV0 + fmul v5.4s, v29.4s, alphaV1 + st1 {v4.4s, v5.4s}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + + +.macro INIT4x4 + fmov s16, wzr + fmov s17, s16 + fmov s20, s17 + fmov s21, s16 + fmov s24, s17 + fmov s25, s16 + fmov s28, s17 + fmov s29, s16 +.endm + +.macro KERNEL4x4_I + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.2s, v1.2s}, [pA] + add pA, pA, #16 + + fmul v16.2s, v0.2s, v8.2s[0] + fmul v29.2s, v1.2s, v9.2s[1] + + fmul v20.2s, v0.2s, v8.2s[1] + fmul v25.2s, v1.2s, v9.2s[0] + + fmul v24.2s, v0.2s, v9.2s[0] + fmul v21.2s, v1.2s, v8.2s[1] + + fmul v28.2s, v0.2s, v9.2s[1] + fmul v17.2s, v1.2s, v8.2s[0] + + ld1 {v12.2s, v13.2s}, [pB] + add pB, pB, #16 + ld1 {v4.2s, v5.2s}, [pA] + add pA, pA, #16 +.endm + +.macro KERNEL4x4_M1 + fmla v16.2s, v0.2s, v8.2s[0] + fmla v29.2s, v1.2s, v9.2s[1] + + ld1 {v12.2s, v13.2s}, [pB] // For next round + add pB, pB, #16 + + fmla v20.2s, v0.2s, v8.2s[1] + fmla v25.2s, v1.2s, v9.2s[0] + + ld1 {v4.2s, v5.2s}, [pA] // For next round + add pA, pA, #16 + + fmla v24.2s, v0.2s, v9.2s[0] + fmla v21.2s, v1.2s, v8.2s[1] + + prfm PLDL1KEEP, [pB, #512] + + fmla v28.2s, v0.2s, v9.2s[1] + fmla v17.2s, v1.2s, v8.2s[0] +.endm + +.macro KERNEL4x4_M2 + fmla v16.2s, v4.2s, v12.2s[0] + fmla v29.2s, v5.2s, v13.2s[1] + + ld1 {v8.2s, v9.2s}, [pB] // For next round + add pB, pB, #16 + + fmla v20.2s, v4.2s, v12.2s[1] + fmla v25.2s, v5.2s, v13.2s[0] + + ld1 {v0.2s, v1.2s}, [pA] // For next round + add pA, pA, #16 + + fmla v24.2s, v4.2s, v13.2s[0] + fmla v21.2s, v5.2s, v12.2s[1] + + prfm PLDL1KEEP, [pA, #512] + + fmla v28.2s, v4.2s, v13.2s[1] + fmla v17.2s, v5.2s, v12.2s[0] +.endm + +.macro KERNEL4x4_E + fmla v16.2s, v4.2s, v12.2s[0] + fmla v29.2s, v5.2s, v13.2s[1] + + fmla v20.2s, v4.2s, v12.2s[1] + fmla v25.2s, v5.2s, v13.2s[0] + + fmla v24.2s, v4.2s, v13.2s[0] + fmla v21.2s, v5.2s, v12.2s[1] + + fmla v28.2s, v4.2s, v13.2s[1] + fmla v17.2s, v5.2s, v12.2s[0] +.endm + +.macro KERNEL4x4_SUB + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.2s, v1.2s}, [pA] + add pA, pA, #16 + + fmla v16.2s, v0.2s, v8.2s[0] + fmla v29.2s, v1.2s, v9.2s[1] + + fmla v20.2s, v0.2s, v8.2s[1] + fmla v25.2s, v1.2s, v9.2s[0] + + fmla v24.2s, v0.2s, v9.2s[0] + fmla v21.2s, v1.2s, v8.2s[1] + + fmla v28.2s, v0.2s, v9.2s[1] + fmla v17.2s, v1.2s, v8.2s[0] +.endm + +.macro SAVE4x4 + + fmul v8.2s, v16.2s, alphaV0 + fmul v9.2s, v17.2s, alphaV1 + st1 {v8.2s, v9.2s}, [pCRow0] + + add pCRow1, pCRow0, LDC + + fmul v12.2s, v20.2s, alphaV2 + fmul v13.2s, v21.2s, alphaV3 + st1 {v12.2s, v13.2s}, [pCRow1] + + add pCRow2, pCRow1, LDC + + fmul v8.2s, v24.2s, alphaV0 + fmul v9.2s, v25.2s, alphaV1 + st1 {v8.2s, v9.2s}, [pCRow2] + + add pCRow1, pCRow2, LDC + + fmul v12.2s, v28.2s, alphaV2 + fmul v13.2s, v29.2s, alphaV3 + st1 {v12.2s, v13.2s}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT2x4 + fmov s16, wzr + fmov s20, s16 + fmov s24, s20 + fmov s28, s16 +.endm + +.macro KERNEL2x4_SUB + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld1 {v0.2s}, [pA] + add pA, pA, #8 + + fmla v16.2s, v0.2s, v8.2s[0] + fmla v20.2s, v0.2s, v8.2s[1] + fmla v24.2s, v0.2s, v9.2s[0] + fmla v28.2s, v0.2s, v9.2s[1] +.endm + +.macro SAVE2x4 + + fmul v8.2s, v16.2s, alphaV0 + st1 {v8.2s}, [pCRow0] + + add pCRow1, pCRow0, LDC + + fmul v12.2s, v20.2s, alphaV1 + st1 {v12.2s}, [pCRow1] + + add pCRow2, pCRow1, LDC + + fmul v8.2s, v24.2s, alphaV2 + st1 {v8.2s}, [pCRow2] + + add pCRow1, pCRow2, LDC + + fmul v12.2s, v28.2s, alphaV3 + st1 {v12.2s}, [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT1x4 + fmov s16, wzr + fmov s20, s16 +.endm + +.macro KERNEL1x4_SUB + ldr s0, [pA] + add pA, pA, #4 + + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + + fmla v16.2s, v8.2s, v0.s[0] + fmla v20.2s, v9.2s, v0.s[0] +.endm + +.macro SAVE1x4 + add pCRow1, pCRow0, LDC + + + fmul v8.2s, v16.2s, alphaV0 + st1 {v8.s}[0], [pCRow0] + st1 {v8.s}[1], [pCRow1] + + add pCRow2, pCRow1, LDC + add pCRow1, pCRow2, LDC + + + fmul v12.2s, v20.2s, alphaV1 + st1 {v12.s}[0], [pCRow2] + st1 {v12.s}[1], [pCRow1] + + add pCRow0, pCRow0, #4 +.endm + +/******************************************************************************/ + +.macro INIT8x2 + fmov s16, wzr + fmov s17, s16 + fmov s20, s17 + fmov s21, s16 +.endm + +.macro KERNEL8x2_SUB + ld1 {v8.2s}, [pB] + add pB, pB, #8 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v8.2s[0] + fmla v17.4s, v1.4s, v8.2s[0] + + fmla v20.4s, v0.4s, v8.2s[1] + fmla v21.4s, v1.4s, v8.2s[1] +.endm + +.macro SAVE8x2 + add pCRow1, pCRow0, LDC + + + fmul v0.4s, v16.4s, alphaV0 + fmul v1.4s, v17.4s, alphaV1 + st1 {v0.4s, v1.4s}, [pCRow0] + + add pCRow2, pCRow1, LDC + + + fmul v4.4s, v20.4s, alphaV0 + fmul v5.4s, v21.4s, alphaV1 + st1 {v4.4s, v5.4s}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT4x2 + fmov s16, wzr + fmov s17, s16 + fmov s20, s17 + fmov s21, s16 +.endm + +.macro KERNEL4x2_SUB + ld1 {v8.2s}, [pB] + add pB, pB, #8 + ld1 {v0.2s, v1.2s}, [pA] + add pA, pA, #16 + + fmla v16.2s, v0.2s, v8.2s[0] + fmla v17.2s, v1.2s, v8.2s[0] + fmla v20.2s, v0.2s, v8.2s[1] + fmla v21.2s, v1.2s, v8.2s[1] +.endm + +.macro SAVE4x2 + + fmul v8.2s, v16.2s, alphaV0 + fmul v9.2s, v17.2s, alphaV1 + st1 {v8.2s, v9.2s}, [pCRow0] + + add pCRow1, pCRow0, LDC + + fmul v12.2s, v20.2s, alphaV2 + fmul v13.2s, v21.2s, alphaV3 + st1 {v12.2s, v13.2s}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT2x2 + fmov s16, wzr + fmov s20, s16 +.endm + +.macro KERNEL2x2_SUB + ld1 {v8.2s}, [pB] + add pB, pB, #8 + + ld1 {v0.2s}, [pA] + add pA, pA, #8 + + fmla v16.2s, v0.2s, v8.2s[0] + fmla v20.2s, v0.2s, v8.2s[1] +.endm + +.macro SAVE2x2 + + fmul v8.2s, v16.2s, alphaV0 + st1 {v8.2s}, [pCRow0] + + add pCRow1 , pCRow0, LDC + + fmul v12.2s, v20.2s, alphaV1 + st1 {v12.2s}, [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT1x2 + fmov s16, wzr +.endm + +.macro KERNEL1x2_SUB + ld1 {v8.2s} , [pB] + add pB , pB, #8 + + ldr s0 , [pA] + add pA, pA, #4 + + fmla v16.2s, v8.2s, v0.2s[0] +.endm + +.macro SAVE1x2 + add pCRow1 , pCRow0, LDC + + + fmul v8.2s, v16.2s, alphaV0 + st1 {v8.s}[0], [pCRow0] + st1 {v8.s}[1], [pCRow1] + + add pCRow0, pCRow0, #4 +.endm + +/******************************************************************************/ + +.macro INIT8x1 + fmov s16, wzr + fmov s17, wzr +.endm + +.macro KERNEL8x1_SUB + ldr s8, [pB] + add pB , pB, #4 + + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v8.2s[0] + fmla v17.4s, v1.4s, v8.2s[0] +.endm + +.macro SAVE8x1 + + fmul v0.4s, v16.4s, alphaV0 + fmul v1.4s, v17.4s, alphaV1 + st1 {v0.4s, v1.4s}, [pCRow0] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT4x1 + fmov s16, wzr + fmov s17, s16 +.endm + +.macro KERNEL4x1_SUB + ldr s8, [pB] + add pB , pB, #4 + + ld1 {v0.2s, v1.2s}, [pA] + add pA , pA, #16 + + fmla v16.2s, v0.2s, v8.2s[0] + fmla v17.2s, v1.2s, v8.2s[0] +.endm + +.macro SAVE4x1 + + fmul v8.2s, v16.2s, alphaV0 + fmul v9.2s, v17.2s, alphaV1 + st1 {v8.2s, v9.2s}, [pCRow0] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT2x1 + fmov s16, wzr +.endm + +.macro KERNEL2x1_SUB + ldr s8, [pB] + add pB , pB, #4 + + ld1 {v0.2s}, [pA] + add pA , pA, #8 + + fmla v16.2s, v0.2s, v8.2s[0] +.endm + +.macro SAVE2x1 + + fmul v8.2s, v16.2s, alphaV0 + st1 {v8.2s}, [pCRow0] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT1x1 + fmov s16, wzr +.endm + +.macro KERNEL1x1_SUB + ldr s8, [pB] + add pB , pB, #4 + + ldr s0, [pA] + add pA , pA, #4 + + fmadd s16, s0, s8, s16 +.endm + +.macro SAVE1x1 + + fmul s8, s16, alpha0 + str s8, [pCRow0] + + add pCRow0, pCRow0, #4 +.endm + +/******************************************************************************* +* End of macro definitions +*******************************************************************************/ + + PROLOGUE + +strmm_kernel_begin: + + .align 5 + add sp, sp, #-(11 * 16) + stp d8, d9, [sp, #(0 * 16)] + stp d10, d11, [sp, #(1 * 16)] + stp d12, d13, [sp, #(2 * 16)] + stp d14, d15, [sp, #(3 * 16)] + stp d16, d17, [sp, #(4 * 16)] + stp x18, x19, [sp, #(5 * 16)] + stp x20, x21, [sp, #(6 * 16)] + stp x22, x23, [sp, #(7 * 16)] + stp x24, x25, [sp, #(8 * 16)] + stp x26, x27, [sp, #(9 * 16)] + str x28, [sp, #(10 * 16)] + + fmov alpha0, s0 + fmov alpha1, s0 + fmov alpha2, s0 + fmov alpha3, s0 + + lsl LDC, LDC, #2 // ldc = ldc * 4 + +#if !defined(LEFT) + neg tempOffset, offset +#endif + mov pB, origPB + + mov counterJ, origN + asr counterJ, counterJ, #3 // J = J / 8 + cmp counterJ, #0 + ble strmm_kernel_L4_BEGIN + +/******************************************************************************/ +/******************************************************************************/ + +strmm_kernel_L8_BEGIN: + mov pCRow0, pC // pCRow0 = C + add pC, pC, LDC, lsl #3 + +#if defined(LEFT) + mov tempOffset, offset +#endif + + mov pA, origPA // pA = start of A array + +/******************************************************************************/ + +strmm_kernel_L8_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 // counterI = counterI / 8 + cmp counterI, #0 + ble strmm_kernel_L8_M4_BEGIN + +strmm_kernel_L8_M8_20: + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #5 + add pA, pA, temp + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #8 +#else + add tempK, tempOffset, #8 +#endif + + asr counterL , tempK, #1 // L = K / 2 + cmp counterL , #2 // is there at least 4 to do? + blt strmm_kernel_L8_M8_32 + + KERNEL8x8_I // do one in the K + KERNEL8x8_M2 // do another in the K + + subs counterL, counterL, #2 + ble strmm_kernel_L8_M8_22a + .align 5 + +strmm_kernel_L8_M8_22: + + KERNEL8x8_M1 + KERNEL8x8_M2 + + subs counterL, counterL, #1 + bgt strmm_kernel_L8_M8_22 + +strmm_kernel_L8_M8_22a: + + KERNEL8x8_M1 + KERNEL8x8_E + + b strmm_kernel_L8_M8_44 + +strmm_kernel_L8_M8_32: + + tst counterL, #1 + ble strmm_kernel_L8_M8_40 + + KERNEL8x8_I + KERNEL8x8_E + + b strmm_kernel_L8_M8_44 + +strmm_kernel_L8_M8_40: + + INIT8x8 + +strmm_kernel_L8_M8_44: + + ands counterL , tempK, #1 + ble strmm_kernel_L8_M8_100 + +strmm_kernel_L8_M8_46: + + KERNEL8x8_SUB + +strmm_kernel_L8_M8_100: + + SAVE8x8 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #8 +#else + sub tempK, tempK, #8 +#endif + lsl temp, tempK, #5 + add pA, pA, temp + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #8 +#endif + +strmm_kernel_L8_M8_END: + subs counterI, counterI, #1 + bne strmm_kernel_L8_M8_20 + +/******************************************************************************/ + +strmm_kernel_L8_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble strmm_kernel_L8_END + + tst counterI, #4 + ble strmm_kernel_L8_M2_BEGIN + +strmm_kernel_L8_M4_20: + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #4 + add pA, pA, temp + lsl temp, tempOffset, #5 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #4 +#else + add tempK, tempOffset, #8 +#endif + + asr counterL , tempK, #1 // L = K / 2 + cmp counterL , #2 // is there at least 4 to do? + blt strmm_kernel_L8_M4_32 + + KERNEL4x8_I // do one in the K + KERNEL4x8_M2 // do another in the K + + subs counterL, counterL, #2 + ble strmm_kernel_L8_M4_22a + .align 5 + +strmm_kernel_L8_M4_22: + + KERNEL4x8_M1 + KERNEL4x8_M2 + + subs counterL, counterL, #1 + bgt strmm_kernel_L8_M4_22 + +strmm_kernel_L8_M4_22a: + + KERNEL4x8_M1 + KERNEL4x8_E + + b strmm_kernel_L8_M4_44 + +strmm_kernel_L8_M4_32: + + tst counterL, #1 + ble strmm_kernel_L8_M4_40 + + KERNEL4x8_I + KERNEL4x8_E + + b strmm_kernel_L8_M4_44 + +strmm_kernel_L8_M4_40: + + INIT4x8 + +strmm_kernel_L8_M4_44: + + ands counterL , tempK, #1 + ble strmm_kernel_L8_M4_100 + +strmm_kernel_L8_M4_46: + + KERNEL4x8_SUB + +strmm_kernel_L8_M4_100: + + SAVE4x8 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #4 +#else + sub tempK, tempK, #8 +#endif + lsl temp, tempK, #4 + add pA, pA, temp + lsl temp, tempK, #5 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #4 +#endif + +strmm_kernel_L8_M4_END: + +/******************************************************************************/ + +strmm_kernel_L8_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble strmm_kernel_L8_END + + tst counterI, #2 // counterI = counterI / 2 + ble strmm_kernel_L8_M1_BEGIN + +strmm_kernel_L8_M2_20: + + INIT2x8 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #3 + add pA, pA, temp + lsl temp, tempOffset, #5 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #2 +#else + add tempK, tempOffset, #8 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble strmm_kernel_L8_M2_40 + +strmm_kernel_L8_M2_22: + + KERNEL2x8_SUB + KERNEL2x8_SUB + KERNEL2x8_SUB + KERNEL2x8_SUB + + KERNEL2x8_SUB + KERNEL2x8_SUB + KERNEL2x8_SUB + KERNEL2x8_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L8_M2_22 + + +strmm_kernel_L8_M2_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble strmm_kernel_L8_M2_100 + +strmm_kernel_L8_M2_42: + + KERNEL2x8_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L8_M2_42 + +strmm_kernel_L8_M2_100: + + SAVE2x8 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #2 +#else + sub tempK, tempK, #8 +#endif + lsl temp, tempK, #3 + add pA, pA, temp + lsl temp, tempK, #5 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #2 +#endif + +strmm_kernel_L8_M2_END: + +/******************************************************************************/ + +strmm_kernel_L8_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble strmm_kernel_L8_END + +strmm_kernel_L8_M1_20: + + INIT1x8 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #2 + add pA, pA, temp + lsl temp, tempOffset, #5 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #1 +#else + add tempK, tempOffset, #8 +#endif + + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble strmm_kernel_L8_M1_40 + +strmm_kernel_L8_M1_22: + KERNEL1x8_SUB + KERNEL1x8_SUB + KERNEL1x8_SUB + KERNEL1x8_SUB + + KERNEL1x8_SUB + KERNEL1x8_SUB + KERNEL1x8_SUB + KERNEL1x8_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L8_M1_22 + + +strmm_kernel_L8_M1_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble strmm_kernel_L8_M1_100 + +strmm_kernel_L8_M1_42: + + KERNEL1x8_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L8_M1_42 + +strmm_kernel_L8_M1_100: + + SAVE1x8 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #1 +#else + sub tempK, tempK, #8 +#endif + lsl temp, tempK, #2 + add pA, pA, temp + lsl temp, tempK, #5 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #1 +#endif + +strmm_kernel_L8_END: + lsl temp, origK, #5 // B = B + K * 4 * 8 + add origPB, origPB, temp + +#if !defined(LEFT) + add tempOffset, tempOffset, #8 +#endif + + subs counterJ, counterJ , #1 // j-- + bgt strmm_kernel_L8_BEGIN + +/******************************************************************************/ +/******************************************************************************/ + +strmm_kernel_L4_BEGIN: + + mov counterJ , origN + tst counterJ , #7 + ble strmm_kernel_L999 + + tst counterJ , #4 + ble strmm_kernel_L2_BEGIN + + mov pCRow0, pC // pCRow0 = pC + + add pC,pC,LDC, lsl #2 + +#if defined(LEFT) + mov tempOffset, offset +#endif + + mov pA, origPA // pA = A + +/******************************************************************************/ + +strmm_kernel_L4_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 // counterI = counterI / 8 + cmp counterI, #0 + ble strmm_kernel_L4_M4_BEGIN + +strmm_kernel_L4_M8_20: + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #5 + add pA, pA, temp + lsl temp, tempOffset, #4 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #8 +#else + add tempK, tempOffset, #4 +#endif + + asr counterL , tempK, #1 // L = K / 2 + cmp counterL , #2 // is there at least 4 to do? + blt strmm_kernel_L4_M8_32 + + KERNEL8x4_I // do one in the K + KERNEL8x4_M2 // do another in the K + + subs counterL, counterL, #2 + ble strmm_kernel_L4_M8_22a + .align 5 + +strmm_kernel_L4_M8_22: + + KERNEL8x4_M1 + KERNEL8x4_M2 + + subs counterL, counterL, #1 + bgt strmm_kernel_L4_M8_22 + +strmm_kernel_L4_M8_22a: + + KERNEL8x4_M1 + KERNEL8x4_E + + b strmm_kernel_L4_M8_44 + +strmm_kernel_L4_M8_32: + + tst counterL, #1 + ble strmm_kernel_L4_M8_40 + + KERNEL8x4_I + KERNEL8x4_E + + b strmm_kernel_L4_M8_44 + +strmm_kernel_L4_M8_40: + + INIT8x4 + +strmm_kernel_L4_M8_44: + + ands counterL , tempK, #1 + ble strmm_kernel_L4_M8_100 + +strmm_kernel_L4_M8_46: + + KERNEL8x4_SUB + +strmm_kernel_L4_M8_100: + + SAVE8x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #8 +#else + sub tempK, tempK, #4 +#endif + lsl temp, tempK, #5 + add pA, pA, temp + lsl temp, tempK, #4 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #8 +#endif +strmm_kernel_L4_M8_END: + subs counterI, counterI, #1 + bne strmm_kernel_L4_M8_20 + +/******************************************************************************/ + +strmm_kernel_L4_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble strmm_kernel_L4_END + + tst counterI, #4 + ble strmm_kernel_L4_M2_BEGIN + +strmm_kernel_L4_M4_20: + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #4 + add pB, pB, temp + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #4 +#else + add tempK, tempOffset, #4 +#endif + asr counterL , tempK, #1 // L = K / 2 + cmp counterL , #2 // is there at least 4 to do? + blt strmm_kernel_L4_M4_32 + + KERNEL4x4_I // do one in the K + KERNEL4x4_M2 // do another in the K + + subs counterL, counterL, #2 + ble strmm_kernel_L4_M4_22a + .align 5 + +strmm_kernel_L4_M4_22: + + KERNEL4x4_M1 + KERNEL4x4_M2 + + subs counterL, counterL, #1 + bgt strmm_kernel_L4_M4_22 + +strmm_kernel_L4_M4_22a: + + KERNEL4x4_M1 + KERNEL4x4_E + + b strmm_kernel_L4_M4_44 + +strmm_kernel_L4_M4_32: + + tst counterL, #1 + ble strmm_kernel_L4_M4_40 + + KERNEL4x4_I + KERNEL4x4_E + + b strmm_kernel_L4_M4_44 + +strmm_kernel_L4_M4_40: + + INIT4x4 + +strmm_kernel_L4_M4_44: + + ands counterL , tempK, #1 + ble strmm_kernel_L4_M4_100 + +strmm_kernel_L4_M4_46: + + KERNEL4x4_SUB + +strmm_kernel_L4_M4_100: + + SAVE4x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #4 +#else + sub tempK, tempK, #4 +#endif + lsl temp, tempK, #4 + add pA, pA, temp + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #4 +#endif +strmm_kernel_L4_M4_END: + +/******************************************************************************/ + +strmm_kernel_L4_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble strmm_kernel_L4_END + + tst counterI, #2 // counterI = counterI / 2 + ble strmm_kernel_L4_M1_BEGIN + +strmm_kernel_L4_M2_20: + + INIT2x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #3 + add pA, pA, temp + lsl temp, tempOffset, #4 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #2 +#else + add tempK, tempOffset, #4 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble strmm_kernel_L4_M2_40 + +strmm_kernel_L4_M2_22: + + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L4_M2_22 + + +strmm_kernel_L4_M2_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble strmm_kernel_L4_M2_100 + +strmm_kernel_L4_M2_42: + + KERNEL2x4_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L4_M2_42 + +strmm_kernel_L4_M2_100: + + SAVE2x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #2 +#else + sub tempK, tempK, #4 +#endif + lsl temp, tempK, #3 + add pA, pA, temp + lsl temp, tempK, #4 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #2 +#endif +strmm_kernel_L4_M2_END: + +/******************************************************************************/ + +strmm_kernel_L4_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble strmm_kernel_L4_END + +strmm_kernel_L4_M1_20: + + INIT1x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #4 + add pB, pB, temp + lsl temp, tempOffset, #2 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #1 +#else + add tempK, tempOffset, #4 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble strmm_kernel_L4_M1_40 + +strmm_kernel_L4_M1_22: + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L4_M1_22 + + +strmm_kernel_L4_M1_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble strmm_kernel_L4_M1_100 + +strmm_kernel_L4_M1_42: + + KERNEL1x4_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L4_M1_42 + +strmm_kernel_L4_M1_100: + + SAVE1x4 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #1 +#else + sub tempK, tempK, #4 +#endif + lsl temp, tempK, #2 + add pA, pA, temp + lsl temp, tempK, #4 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #1 +#endif +strmm_kernel_L4_END: + add origPB, origPB, origK, lsl #4 // B = B + K * 4 * 4 +#if !defined(LEFT) + add tempOffset, tempOffset, #4 +#endif + +/******************************************************************************/ +/******************************************************************************/ + +strmm_kernel_L2_BEGIN: // less than 2 left in N direction + + mov counterJ , origN + tst counterJ , #3 + ble strmm_kernel_L999 + + tst counterJ , #2 + ble strmm_kernel_L1_BEGIN + + mov pCRow0, pC // pCRow0 = pC + + add pC,pC,LDC, lsl #1 + +#if defined(LEFT) + mov tempOffset, offset +#endif + mov pA, origPA // pA = A + +/******************************************************************************/ + +strmm_kernel_L2_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 // counterI = counterI / 8 + cmp counterI,#0 + ble strmm_kernel_L2_M4_BEGIN + +strmm_kernel_L2_M8_20: + + INIT8x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #5 + add pA, pA, temp + lsl temp, tempOffset, #3 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #8 +#else + add tempK, tempOffset, #2 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble strmm_kernel_L2_M8_40 + .align 5 + +strmm_kernel_L2_M8_22: + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L2_M8_22 + + +strmm_kernel_L2_M8_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble strmm_kernel_L2_M8_100 + +strmm_kernel_L2_M8_42: + + KERNEL8x2_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L2_M8_42 + +strmm_kernel_L2_M8_100: + + SAVE8x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #8 +#else + sub tempK, tempK, #2 +#endif + lsl temp, tempK, #5 + add pA, pA, temp + lsl temp, tempK, #3 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #8 +#endif +strmm_kernel_L2_M8_END: + + subs counterI, counterI, #1 + bgt strmm_kernel_L2_M8_20 + +/******************************************************************************/ + +strmm_kernel_L2_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble strmm_kernel_L2_END + + tst counterI, #4 + ble strmm_kernel_L2_M2_BEGIN + +strmm_kernel_L2_M4_20: + + INIT4x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #3 + add pB, pB, temp + lsl temp, tempOffset, #4 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #4 +#else + add tempK, tempOffset, #2 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble strmm_kernel_L2_M4_40 + .align 5 + +strmm_kernel_L2_M4_22: + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L2_M4_22 + + +strmm_kernel_L2_M4_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble strmm_kernel_L2_M4_100 + +strmm_kernel_L2_M4_42: + + KERNEL4x2_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L2_M4_42 + +strmm_kernel_L2_M4_100: + + SAVE4x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #4 +#else + sub tempK, tempK, #2 +#endif + lsl temp, tempK, #4 + add pA, pA, temp + lsl temp, tempK, #3 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #4 +#endif +strmm_kernel_L2_M4_END: + +/******************************************************************************/ + +strmm_kernel_L2_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble strmm_kernel_L2_END + + tst counterI, #2 // counterI = counterI / 2 + ble strmm_kernel_L2_M1_BEGIN + +strmm_kernel_L2_M2_20: + + INIT2x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #3 + add pB, pB, temp + lsl temp, tempOffset, #3 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #2 +#else + add tempK, tempOffset, #2 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble strmm_kernel_L2_M2_40 + +strmm_kernel_L2_M2_22: + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L2_M2_22 + + +strmm_kernel_L2_M2_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble strmm_kernel_L2_M2_100 + +strmm_kernel_L2_M2_42: + + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L2_M2_42 + +strmm_kernel_L2_M2_100: + + SAVE2x2 +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #2 +#else + sub tempK, tempK, #2 +#endif + lsl temp, tempK, #3 + add pA, pA, temp + lsl temp, tempK, #3 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #2 +#endif + +strmm_kernel_L2_M2_END: + +/******************************************************************************/ + +strmm_kernel_L2_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble strmm_kernel_L2_END + +strmm_kernel_L2_M1_20: + + INIT1x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #3 + add pB, pB, temp + lsl temp, tempOffset, #2 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #1 +#else + add tempK, tempOffset, #2 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL, #0 + ble strmm_kernel_L2_M1_40 + +strmm_kernel_L2_M1_22: + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L2_M1_22 + + +strmm_kernel_L2_M1_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble strmm_kernel_L2_M1_100 + +strmm_kernel_L2_M1_42: + + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L2_M1_42 + +strmm_kernel_L2_M1_100: + + SAVE1x2 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #1 +#else + sub tempK, tempK, #2 +#endif + lsl temp, tempK, #2 + add pA, pA, temp + lsl temp, tempK, #3 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #1 +#endif +strmm_kernel_L2_END: +#if !defined(LEFT) + add tempOffset, tempOffset, #2 +#endif + add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4 + +/******************************************************************************/ +/******************************************************************************/ + +strmm_kernel_L1_BEGIN: + + mov counterJ , origN + tst counterJ , #1 + ble strmm_kernel_L999 // done + + + mov pCRow0, pC // pCRow0 = C + add pC , pC , LDC // Update pC to point to next + +#if defined(LEFT) + mov tempOffset, offset +#endif + mov pA, origPA // pA = A + +/******************************************************************************/ + +strmm_kernel_L1_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 + cmp counterI, #0 + ble strmm_kernel_L1_M4_BEGIN + +strmm_kernel_L1_M8_20: + + INIT8x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #5 + add pA, pA, temp + lsl temp, tempOffset, #2 + add pB, pB, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #8 +#else + add tempK, tempOffset, #1 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble strmm_kernel_L1_M8_40 + .align 5 + +strmm_kernel_L1_M8_22: + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L1_M8_22 + + +strmm_kernel_L1_M8_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble strmm_kernel_L1_M8_100 + +strmm_kernel_L1_M8_42: + + KERNEL8x1_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L1_M8_42 + +strmm_kernel_L1_M8_100: + + SAVE8x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #8 +#else + sub tempK, tempK, #1 +#endif + lsl temp, tempK, #5 + add pA, pA, temp + lsl temp, tempK, #2 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #8 +#endif +strmm_kernel_L1_M8_END: + + subs counterI, counterI, #1 + bgt strmm_kernel_L1_M8_20 + +/******************************************************************************/ + +strmm_kernel_L1_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble strmm_kernel_L1_END + + tst counterI, #4 + ble strmm_kernel_L1_M2_BEGIN + +strmm_kernel_L1_M4_20: + + INIT4x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #2 + add pB, pB, temp + lsl temp, tempOffset, #4 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #4 +#else + add tempK, tempOffset, #1 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble strmm_kernel_L1_M4_40 + .align 5 + +strmm_kernel_L1_M4_22: + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L1_M4_22 + + +strmm_kernel_L1_M4_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble strmm_kernel_L1_M4_100 + +strmm_kernel_L1_M4_42: + + KERNEL4x1_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L1_M4_42 + +strmm_kernel_L1_M4_100: + + SAVE4x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #4 +#else + sub tempK, tempK, #1 +#endif + lsl temp, tempK, #4 + add pA, pA, temp + lsl temp, tempK, #2 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #4 +#endif +strmm_kernel_L1_M4_END: + +/******************************************************************************/ + +strmm_kernel_L1_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble strmm_kernel_L1_END + + tst counterI, #2 // counterI = counterI / 2 + ble strmm_kernel_L1_M1_BEGIN + +strmm_kernel_L1_M2_20: + + INIT2x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #2 + add pB, pB, temp + lsl temp, tempOffset, #3 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #2 +#else + add tempK, tempOffset, #1 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble strmm_kernel_L1_M2_40 + +strmm_kernel_L1_M2_22: + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L1_M2_22 + + +strmm_kernel_L1_M2_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble strmm_kernel_L1_M2_100 + +strmm_kernel_L1_M2_42: + + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L1_M2_42 + +strmm_kernel_L1_M2_100: + + SAVE2x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub tempK, origK, tempOffset +#if defined(LEFT) + sub tempK, tempK, #2 +#else + sub tempK, tempK, #1 +#endif + lsl temp, tempK, #3 + add pA, pA, temp + lsl temp, tempK, #2 + add pB, pB, temp +#endif +#if defined(LEFT) + add tempOffset, tempOffset, #2 +#endif +strmm_kernel_L1_M2_END: + +/******************************************************************************/ + +strmm_kernel_L1_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble strmm_kernel_L1_END + +strmm_kernel_L1_M1_20: + + INIT1x1 + +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + mov pB, origPB +#else + mov pB, origPB + lsl temp, tempOffset, #2 + add pB, pB, temp + lsl temp, tempOffset, #2 + add pA, pA, temp +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub tempK, origK, tempOffset +#elif defined(LEFT) + add tempK, tempOffset, #1 +#else + add tempK, tempOffset, #1 +#endif + asr counterL , tempK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble strmm_kernel_L1_M1_40 + +strmm_kernel_L1_M1_22: + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L1_M1_22 + + +strmm_kernel_L1_M1_40: + + ands counterL , tempK, #7 // counterL = counterL % 8 + ble strmm_kernel_L1_M1_100 + +strmm_kernel_L1_M1_42: + + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt strmm_kernel_L1_M1_42 + +strmm_kernel_L1_M1_100: + + SAVE1x1 + +strmm_kernel_L1_END: + +/******************************************************************************/ + +strmm_kernel_L999: + mov x0, #0 // set return value + ldp d8, d9, [sp, #(0 * 16)] + ldp d10, d11, [sp, #(1 * 16)] + ldp d12, d13, [sp, #(2 * 16)] + ldp d14, d15, [sp, #(3 * 16)] + ldp d16, d17, [sp, #(4 * 16)] + ldp x18, x19, [sp, #(5 * 16)] + ldp x20, x21, [sp, #(6 * 16)] + ldp x22, x23, [sp, #(7 * 16)] + ldp x24, x25, [sp, #(8 * 16)] + ldp x26, x27, [sp, #(9 * 16)] + ldr x28, [sp, #(10 * 16)] + add sp, sp, #(11*16) + ret + + EPILOGUE +