diff --git a/driver/others/parameter.c b/driver/others/parameter.c index 60961b7dd..db45dfda4 100644 --- a/driver/others/parameter.c +++ b/driver/others/parameter.c @@ -730,12 +730,22 @@ void blas_set_parameter(void){ #if defined(ARCH_ARM64) +#if defined(VULCAN) +unsigned long vulcan_pre_a; +unsigned long vulcan_pre_b; +unsigned long vulcan_pre_c; +#endif + void blas_set_parameter(void) { #if defined(VULCAN) dgemm_p = 160; dgemm_q = 128; dgemm_r = 4096; + + vulcan_pre_a = 3584; + vulcan_pre_b = 512; + vulcan_pre_c = 128; #endif } diff --git a/kernel/arm64/KERNEL.VULCAN b/kernel/arm64/KERNEL.VULCAN index 3372529f7..24567f573 100644 --- a/kernel/arm64/KERNEL.VULCAN +++ b/kernel/arm64/KERNEL.VULCAN @@ -1,2 +1,4 @@ include $(KERNELDIR)/KERNEL.CORTEXA57 +DGEMMKERNEL = dgemm_kernel_$(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N)_vulcan.S + diff --git a/kernel/arm64/dgemm_kernel_8x4_vulcan.S b/kernel/arm64/dgemm_kernel_8x4_vulcan.S new file mode 100644 index 000000000..d00fbc775 --- /dev/null +++ b/kernel/arm64/dgemm_kernel_8x4_vulcan.S @@ -0,0 +1,1817 @@ +/******************************************************************************* +Copyright (c) 2016, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#define ASSEMBLER +#include "common.h" + +/* X0 X1 X2 s0 X3 x4 x5 x6 */ +/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha0,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc )*/ + +#define origM x0 +#define origN x1 +#define origK x2 +#define origPA x3 +#define origPB x4 +#define pC x5 +#define LDC x6 +#define temp x7 +#define counterL x8 +#define counterI x9 +#define counterJ x10 +#define pB x11 +#define pCRow0 x12 +#define pCRow1 x13 +#define pCRow2 x14 +#define pCRow3 x15 +#define pA x16 +#define alpha x17 + +#define alpha0 d10 +#define alphaV0 v10.d[0] + +#define A_PRE_SIZE x20 +#define B_PRE_SIZE x21 +#define C_PRE_SIZE x22 + +#define A_PRE_SIZE_64 x23 +#define B_PRE_SIZE_64 x24 + + +// 00 origM +// 01 origN +// 02 origK +// 03 origPA +// 04 origPB +// 05 pC +// 06 origLDC -> LDC +// 07 temp +// 08 counterL +// 09 counterI +// 10 counterJ +// 11 pB +// 12 pCRow0 +// 13 pCRow1 +// 14 pCRow2 +// 15 pCRow3 +// 16 pA +// 17 +// 18 must save +// 19 must save +// 20 must save +// 21 must save +// 22 must save +// 23 must save +// 24 must save +// 25 must save +// 26 must save +// 27 must save +// 28 must save +// 29 frame +// 30 link +// 31 sp + +//v00 ALPHA -> pA0_0, pA0_1 +//v01 pA0_2, pA0_3 +//v02 pA0_4, pA0_5 +//v03 pA0_6, pA0_7 +//v04 pA1_0, pA1_1 +//v05 pA1_2, pA1_3 +//v06 pA1_4, pA1_5 +//v07 pA1_6, pA1_7 +//v08 must save pB0_0 +//v09 must save pB0_1 +//v10 must save pB0_2 --> ALPHA0 +//v11 must save pB0_3 +//v12 must save pB1_0 +//v13 must save pB1_1 +//v14 must save pB1_2 +//v15 must save pB1_3 +//v16 must save C00, C01 +//v17 must save C02, C03 +//v18 C04, C05 +//v19 C06, C07 +//v20 C10, C11 +//v21 C12, C13 +//v22 C14, C15 +//v23 C16, C17 +//v24 C20, C21 +//v25 C22, C23 +//v26 C24, C25 +//v27 C26, C27 +//v28 C30, C31 +//v29 C32, C33 +//v30 C34, C35 +//v31 C36, C37 + +/******************************************************************************* +* Macro definitions +*******************************************************************************/ + +.macro INIT8x4 + fmov d16, xzr + fmov d17, xzr + fmov d18, d16 + fmov d19, xzr + fmov d20, xzr + fmov d21, d16 + fmov d22, d17 + fmov d23, d18 + fmov d24, xzr + fmov d25, d16 + fmov d26, d17 + fmov d27, d18 + fmov d28, xzr + fmov d29, d16 + fmov d30, d17 + fmov d31, d18 +.endm + +.macro KERNEL8x4_I + ldur q0, [pA] + ldur q1, [pA, #16] + + ldur q8, [pB] + ldur q9, [pB, #16] + + fmul v16.2d, v0.2d, v8.d[0] + fmul v20.2d, v0.2d, v8.d[1] + + fmul v17.2d, v1.2d, v8.d[0] + fmul v21.2d, v1.2d, v8.d[1] + + ldp q2, q3, [pA, #32] + fmul v24.2d, v0.2d, v9.d[0] + ldp q4, q5, [pA, #64] + fmul v28.2d, v0.2d, v9.d[1] + + fmul v25.2d, v1.2d, v9.d[0] + fmul v29.2d, v1.2d, v9.d[1] + + ldur q12, [pB, #32] + ldur q13, [pB, #48] + + fmul v18.2d, v2.2d, v8.d[0] + fmul v22.2d, v2.2d, v8.d[1] + + ldur q6, [pA, #96] + ldur q7, [pA, #112] + + add pB, pB, #64 + add pA, pA, #128 + + fmul v26.2d, v2.2d, v9.d[0] + fmul v30.2d, v2.2d, v9.d[1] + + fmul v19.2d, v3.2d, v8.d[0] + fmul v27.2d, v3.2d, v9.d[0] + + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + + fmul v31.2d, v3.2d, v9.d[1] + fmul v23.2d, v3.2d, v8.d[1] + + prfm PLDL1KEEP, [pA, A_PRE_SIZE_64] +.endm + +.macro KERNEL8x4_M1_M2 + fmla v16.2d, v0.2d, v8.d[0] + fmla v20.2d, v0.2d, v8.d[1] + + ldp q4, q5, [pA] + + fmla v24.2d, v0.2d, v9.d[0] + fmla v28.2d, v0.2d, v9.d[1] + + ldp q12, q13, [pB] + + fmla v17.2d, v1.2d, v8.d[0] + fmla v25.2d, v1.2d, v9.d[0] + + prfm PLDL1KEEP, [pA, A_PRE_SIZE_64] + + fmla v21.2d, v1.2d, v8.d[1] + fmla v29.2d, v1.2d, v9.d[1] + + fmla v18.2d, v2.2d, v8.d[0] + fmla v22.2d, v2.2d, v8.d[1] + + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + + fmla v26.2d, v2.2d, v9.d[0] + fmla v30.2d, v2.2d, v9.d[1] + fmla v19.2d, v3.2d, v8.d[0] + fmla v23.2d, v3.2d, v8.d[1] + + ldp q6, q7, [pA, #32] + + fmla v27.2d, v3.2d, v9.d[0] + fmla v31.2d, v3.2d, v9.d[1] + + fmla v16.2d, v4.2d, v12.d[0] + fmla v20.2d, v4.2d, v12.d[1] + fmla v24.2d, v4.2d, v13.d[0] + fmla v28.2d, v4.2d, v13.d[1] + + ldp q0, q1, [pA, #64] + + fmla v17.2d, v5.2d, v12.d[0] + fmla v25.2d, v5.2d, v13.d[0] + + ldp q8, q9, [pB, #32] + ldp q2, q3, [pA, #96] + + fmla v21.2d, v5.2d, v12.d[1] + fmla v29.2d, v5.2d, v13.d[1] + + fmla v18.2d, v6.2d, v12.d[0] + fmla v22.2d, v6.2d, v12.d[1] + + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + + fmla v26.2d, v6.2d, v13.d[0] + fmla v30.2d, v6.2d, v13.d[1] + + fmla v19.2d, v7.2d, v12.d[0] + fmla v23.2d, v7.2d, v12.d[1] + + add pB, pB, #64 + add pA, pA, #128 + + fmla v27.2d, v7.2d, v13.d[0] + fmla v31.2d, v7.2d, v13.d[1] +.endm + + +.macro KERNEL8x4_M1 + fmla v16.2d, v0.2d, v8.d[0] + fmla v20.2d, v0.2d, v8.d[1] + + ldp q4, q5, [pA], #32 + + fmla v24.2d, v0.2d, v9.d[0] + fmla v28.2d, v0.2d, v9.d[1] + + ldp q12, q13, [pB] + add pB, pB, #32 + + fmla v17.2d, v1.2d, v8.d[0] + fmla v25.2d, v1.2d, v9.d[0] + + prfm PLDL1KEEP, [pA, A_PRE_SIZE_64] + + fmla v21.2d, v1.2d, v8.d[1] + fmla v29.2d, v1.2d, v9.d[1] + + fmla v18.2d, v2.2d, v8.d[0] + fmla v22.2d, v2.2d, v8.d[1] + + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + + fmla v26.2d, v2.2d, v9.d[0] + fmla v30.2d, v2.2d, v9.d[1] + fmla v19.2d, v3.2d, v8.d[0] + fmla v23.2d, v3.2d, v8.d[1] + + ldp q6, q7, [pA], #32 + + fmla v27.2d, v3.2d, v9.d[0] + fmla v31.2d, v3.2d, v9.d[1] +.endm + +.macro KERNEL8x4_M2 + fmla v16.2d, v4.2d, v12.d[0] + fmla v20.2d, v4.2d, v12.d[1] + fmla v24.2d, v4.2d, v13.d[0] + fmla v28.2d, v4.2d, v13.d[1] + + ldp q0, q1, [pA], #32 + + fmla v17.2d, v5.2d, v12.d[0] + fmla v25.2d, v5.2d, v13.d[0] + + ldp q8, q9, [pB] + add pB, pB, #32 + + fmla v21.2d, v5.2d, v12.d[1] + fmla v29.2d, v5.2d, v13.d[1] + + fmla v18.2d, v6.2d, v12.d[0] + fmla v22.2d, v6.2d, v12.d[1] + + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + + fmla v26.2d, v6.2d, v13.d[0] + fmla v30.2d, v6.2d, v13.d[1] + + fmla v19.2d, v7.2d, v12.d[0] + fmla v23.2d, v7.2d, v12.d[1] + + ldp q2, q3, [pA], #32 + + fmla v27.2d, v7.2d, v13.d[0] + fmla v31.2d, v7.2d, v13.d[1] +.endm + +.macro KERNEL8x4_E + fmla v16.2d, v4.2d, v12.d[0] + fmla v20.2d, v4.2d, v12.d[1] + fmla v24.2d, v4.2d, v13.d[0] + fmla v28.2d, v4.2d, v13.d[1] + + fmla v17.2d, v5.2d, v12.d[0] + fmla v25.2d, v5.2d, v13.d[0] + fmla v21.2d, v5.2d, v12.d[1] + fmla v29.2d, v5.2d, v13.d[1] + + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + + fmla v18.2d, v6.2d, v12.d[0] + fmla v22.2d, v6.2d, v12.d[1] + fmla v26.2d, v6.2d, v13.d[0] + fmla v30.2d, v6.2d, v13.d[1] + + fmla v19.2d, v7.2d, v12.d[0] + fmla v23.2d, v7.2d, v12.d[1] + fmla v27.2d, v7.2d, v13.d[0] + fmla v31.2d, v7.2d, v13.d[1] +.endm + +.macro KERNEL8x4_SUB + ldp q0, q1, [pA], #32 + + ldur q8, [pB] + + fmla v16.2d, v0.2d, v8.d[0] + fmla v20.2d, v0.2d, v8.d[1] + + ldur q9, [pB, #16] + add pB, pB, #32 + + fmla v17.2d, v1.2d, v8.d[0] + fmla v21.2d, v1.2d, v8.d[1] + + ldp q2, q3, [pA], #32 + + fmla v24.2d, v0.2d, v9.d[0] + fmla v28.2d, v0.2d, v9.d[1] + + fmla v25.2d, v1.2d, v9.d[0] + fmla v29.2d, v1.2d, v9.d[1] + + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + + fmla v18.2d, v2.2d, v8.d[0] + fmla v22.2d, v2.2d, v8.d[1] + + prfm PLDL1KEEP, [pA, A_PRE_SIZE_64] + + fmla v26.2d, v2.2d, v9.d[0] + fmla v30.2d, v2.2d, v9.d[1] + + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + + fmla v19.2d, v3.2d, v8.d[0] + fmla v27.2d, v3.2d, v9.d[0] + + fmla v31.2d, v3.2d, v9.d[1] + fmla v23.2d, v3.2d, v8.d[1] +.endm + +.macro SAVE8x4 + fmov alpha0, alpha + ldr q0, [pCRow0] + ldr q1, [pCRow0, #16] + ldr q2, [pCRow0, #32] + ldr q3, [pCRow0, #48] + ldr q4, [pCRow1] + ldr q5, [pCRow1, #16] + ldr q6, [pCRow1, #32] + ldr q7, [pCRow1, #48] + + fmla v0.2d, v16.2d, alphaV0 + fmla v1.2d, v17.2d, alphaV0 + stp q0, q1, [pCRow0] + + fmla v2.2d, v18.2d, alphaV0 + fmla v3.2d, v19.2d, alphaV0 + stp q2, q3, [pCRow0, #32] + ldr q0, [pCRow2] + ldr q1, [pCRow2, #16] + + fmla v4.2d, v20.2d, alphaV0 + fmla v5.2d, v21.2d, alphaV0 + stp q4, q5, [pCRow1] + ldr q2, [pCRow2, #32] + ldr q3, [pCRow2, #48] + + fmla v6.2d, v22.2d, alphaV0 + fmla v7.2d, v23.2d, alphaV0 + stp q6, q7, [pCRow1, #32] + ldr q4, [pCRow3] + ldr q5, [pCRow3, #16] + + fmla v0.2d, v24.2d, alphaV0 + fmla v1.2d, v25.2d, alphaV0 + stp q0, q1, [pCRow2] + ldr q6, [pCRow3, #32] + ldr q7, [pCRow3, #48] + + fmla v2.2d, v26.2d, alphaV0 + fmla v3.2d, v27.2d, alphaV0 + stp q2, q3, [pCRow2, #32] + + fmla v4.2d, v28.2d, alphaV0 + fmla v5.2d, v29.2d, alphaV0 + stp q4, q5, [pCRow3] + + fmla v6.2d, v30.2d, alphaV0 + fmla v7.2d, v31.2d, alphaV0 + stp q6, q7, [pCRow3, #32] + + add pCRow0, pCRow0, #64 + add pCRow1, pCRow1, #64 + add pCRow2, pCRow2, #64 + add pCRow3, pCRow3, #64 +.endm + +/******************************************************************************/ + +.macro INIT4x4 + fmov d16, xzr + fmov d17, d16 + fmov d20, d17 + fmov d21, d16 + fmov d24, d17 + fmov d25, d16 + fmov d28, d17 + fmov d29, d16 +.endm + +.macro KERNEL4x4_SUB + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + ld1 {v0.2d, v1.2d}, [pA] + add pA, pA, #32 + + fmla v16.2d, v0.2d, v8.d[0] + fmla v29.2d, v1.2d, v9.d[1] + + fmla v20.2d, v0.2d, v8.d[1] + fmla v25.2d, v1.2d, v9.d[0] + + fmla v24.2d, v0.2d, v9.d[0] + fmla v21.2d, v1.2d, v8.d[1] + + fmla v28.2d, v0.2d, v9.d[1] + fmla v17.2d, v1.2d, v8.d[0] +.endm + +.macro SAVE4x4 + fmov alpha0, alpha + + ld1 {v8.2d, v9.2d}, [pCRow0] + fmla v8.2d, v16.2d, alphaV0 + fmla v9.2d, v17.2d, alphaV0 + st1 {v8.2d, v9.2d}, [pCRow0] + + prfm PLDL2KEEP, [pCRow0, C_PRE_SIZE] + add pCRow0, pCRow0, #32 + + ld1 {v12.2d, v13.2d}, [pCRow1] + fmla v12.2d, v20.2d, alphaV0 + fmla v13.2d, v21.2d, alphaV0 + st1 {v12.2d, v13.2d}, [pCRow1] + + prfm PLDL2KEEP, [pCRow1, C_PRE_SIZE] + add pCRow1, pCRow1, #32 + + ld1 {v8.2d, v9.2d}, [pCRow2] + fmla v8.2d, v24.2d, alphaV0 + fmla v9.2d, v25.2d, alphaV0 + st1 {v8.2d, v9.2d}, [pCRow2] + + prfm PLDL2KEEP, [pCRow2, C_PRE_SIZE] + add pCRow2, pCRow2, #32 + + ld1 {v12.2d, v13.2d}, [pCRow3] + fmla v12.2d, v28.2d, alphaV0 + fmla v13.2d, v29.2d, alphaV0 + st1 {v12.2d, v13.2d}, [pCRow3] + + prfm PLDL2KEEP, [pCRow3, C_PRE_SIZE] + add pCRow3, pCRow3, #32 +.endm + +/******************************************************************************/ + + +.macro INIT2x4 + fmov d16, xzr + fmov d20, d16 + fmov d24, d20 + fmov d28, d16 +.endm + +.macro KERNEL2x4_SUB + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + ld1 {v0.2d}, [pA] + add pA, pA, #16 + + fmla v16.2d, v0.2d, v8.d[0] + fmla v20.2d, v0.2d, v8.d[1] + fmla v24.2d, v0.2d, v9.d[0] + fmla v28.2d, v0.2d, v9.d[1] +.endm + +.macro SAVE2x4 + fmov alpha0, alpha + + ld1 {v8.2d}, [pCRow0] + fmla v8.2d, v16.2d, alphaV0 + st1 {v8.2d}, [pCRow0] + + prfm PLDL2KEEP, [pCRow0, C_PRE_SIZE] + add pCRow0, pCRow0, #16 + + ld1 {v12.2d}, [pCRow1] + fmla v12.2d, v20.2d, alphaV0 + st1 {v12.2d}, [pCRow1] + + prfm PLDL2KEEP, [pCRow1, C_PRE_SIZE] + add pCRow1, pCRow1, #16 + + ld1 {v8.2d}, [pCRow2] + fmla v8.2d, v24.2d, alphaV0 + st1 {v8.2d}, [pCRow2] + + prfm PLDL2KEEP, [pCRow2, C_PRE_SIZE] + add pCRow2, pCRow2, #16 + + ld1 {v12.2d}, [pCRow3] + fmla v12.2d, v28.2d, alphaV0 + st1 {v12.2d}, [pCRow3] + + prfm PLDL2KEEP, [pCRow3, C_PRE_SIZE] + add pCRow3, pCRow3, #16 +.endm + +/******************************************************************************/ + +.macro INIT1x4 + fmov d16, xzr + fmov d20, d16 +.endm + +.macro KERNEL1x4_SUB + ldr d0, [pA] + add pA, pA, #8 + + ld1 {v8.2d, v9.2d}, [pB] + add pB, pB, #32 + + fmla v16.2d, v8.2d, v0.d[0] + fmla v20.2d, v9.2d, v0.d[0] +.endm + +.macro SAVE1x4 + fmov alpha0, alpha + + ld1 {v8.d}[0], [pCRow0] + ld1 {v8.d}[1], [pCRow1] + fmla v8.2d, v16.2d, alphaV0 + st1 {v8.d}[0], [pCRow0] + st1 {v8.d}[1], [pCRow1] + + prfm PLDL2KEEP, [pCRow0, C_PRE_SIZE] + add pCRow0, pCRow0, #8 + prfm PLDL2KEEP, [pCRow1, C_PRE_SIZE] + add pCRow1, pCRow1, #8 + + ld1 {v12.d}[0], [pCRow2] + ld1 {v12.d}[1], [pCRow3] + fmla v12.2d, v20.2d, alphaV0 + st1 {v12.d}[0], [pCRow2] + st1 {v12.d}[1], [pCRow3] + + prfm PLDL2KEEP, [pCRow2, C_PRE_SIZE] + add pCRow2, pCRow2, #8 + prfm PLDL2KEEP, [pCRow3, C_PRE_SIZE] + add pCRow3, pCRow3, #8 +.endm + +/******************************************************************************/ + +.macro INIT8x2 + fmov d16, xzr + fmov d17, xzr + fmov d18, d16 + fmov d19, d17 + fmov d20, xzr + fmov d21, d16 + fmov d22, d17 + fmov d23, d18 +.endm + +.macro KERNEL8x2_SUB + ld1 {v0.2d, v1.2d}, [pA] + add pA, pA, #32 + ld1 {v8.2d}, [pB] + add pB, pB, #16 + ld1 {v2.2d, v3.2d}, [pA] + add pA, pA, #32 + + fmla v16.2d, v0.2d, v8.d[0] + fmla v17.2d, v1.2d, v8.d[0] + fmla v18.2d, v2.2d, v8.d[0] + fmla v19.2d, v3.2d, v8.d[0] + + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + + fmla v20.2d, v0.2d, v8.d[1] + fmla v21.2d, v1.2d, v8.d[1] + fmla v22.2d, v2.2d, v8.d[1] + fmla v23.2d, v3.2d, v8.d[1] +.endm + +.macro SAVE8x2 + fmov alpha0, alpha + + ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0] + fmla v0.2d, v16.2d, alphaV0 + fmla v1.2d, v17.2d, alphaV0 + fmla v2.2d, v18.2d, alphaV0 + fmla v3.2d, v19.2d, alphaV0 + st1 {v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0] + + prfm PLDL2KEEP, [pCRow0, C_PRE_SIZE] + add pCRow0, pCRow0, #64 + + ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [pCRow1] + fmla v4.2d, v20.2d, alphaV0 + fmla v5.2d, v21.2d, alphaV0 + fmla v6.2d, v22.2d, alphaV0 + fmla v7.2d, v23.2d, alphaV0 + st1 {v4.2d, v5.2d, v6.2d, v7.2d}, [pCRow1] + + prfm PLDL2KEEP, [pCRow1, C_PRE_SIZE] + add pCRow1, pCRow1, #64 +.endm + +/******************************************************************************/ + +.macro INIT4x2 + fmov d16, xzr + fmov d17, d16 + fmov d20, d17 + fmov d21, d16 +.endm + +.macro KERNEL4x2_SUB + ld1 {v8.2d}, [pB] + add pB, pB, #16 + ld1 {v0.2d, v1.2d}, [pA] + add pA, pA, #32 + + fmla v16.2d, v0.2d, v8.d[0] + fmla v17.2d, v1.2d, v8.d[0] + fmla v20.2d, v0.2d, v8.d[1] + fmla v21.2d, v1.2d, v8.d[1] +.endm + +.macro SAVE4x2 + fmov alpha0, alpha + + ld1 {v8.2d, v9.2d}, [pCRow0] + fmla v8.2d, v16.2d, alphaV0 + fmla v9.2d, v17.2d, alphaV0 + st1 {v8.2d, v9.2d}, [pCRow0] + + prfm PLDL2KEEP, [pCRow0, C_PRE_SIZE] + add pCRow0, pCRow0, #32 + + ld1 {v12.2d, v13.2d}, [pCRow1] + fmla v12.2d, v20.2d, alphaV0 + fmla v13.2d, v21.2d, alphaV0 + st1 {v12.2d, v13.2d}, [pCRow1] + + prfm PLDL2KEEP, [pCRow1, C_PRE_SIZE] + add pCRow1, pCRow1, #32 +.endm + +/******************************************************************************/ + +.macro INIT2x2 + fmov d16, xzr + fmov d20, d16 +.endm + +.macro KERNEL2x2_SUB + ld1 {v8.2d}, [pB] + add pB, pB, #16 + + ld1 {v0.2d}, [pA] + add pA, pA, #16 + + fmla v16.2d, v0.2d, v8.d[0] + fmla v20.2d, v0.2d, v8.d[1] +.endm + +.macro SAVE2x2 + fmov alpha0, alpha + + ld1 {v8.2d}, [pCRow0] + fmla v8.2d, v16.2d, alphaV0 + st1 {v8.2d}, [pCRow0] + + prfm PLDL2KEEP, [pCRow0, C_PRE_SIZE] + add pCRow0, pCRow0, #16 + + ld1 {v12.2d}, [pCRow1] + fmla v12.2d, v20.2d, alphaV0 + st1 {v12.2d}, [pCRow1] + + prfm PLDL2KEEP, [pCRow1, C_PRE_SIZE] + add pCRow1, pCRow1, #16 +.endm + +/******************************************************************************/ + +.macro INIT1x2 + fmov d16, xzr +.endm + +.macro KERNEL1x2_SUB + ld1 {v8.2d} , [pB] + add pB , pB, #16 + + ldr d0 , [pA] + add pA, pA, #8 + + fmla v16.2d, v8.2d, v0.d[0] +.endm + +.macro SAVE1x2 + fmov alpha0, alpha + + ld1 {v8.d}[0], [pCRow0] + ld1 {v8.d}[1], [pCRow1] + fmla v8.2d, v16.2d, alphaV0 + st1 {v8.d}[0], [pCRow0] + st1 {v8.d}[1], [pCRow1] + + prfm PLDL2KEEP, [pCRow0, C_PRE_SIZE] + add pCRow0, pCRow0, #8 + prfm PLDL2KEEP, [pCRow1, C_PRE_SIZE] + add pCRow1, pCRow1, #8 +.endm + +/******************************************************************************/ + +.macro INIT8x1 + fmov d16, xzr + fmov d17, xzr + fmov d18, d16 + fmov d19, d17 +.endm + +.macro KERNEL8x1_SUB + ld1 {v0.2d, v1.2d}, [pA] + add pA , pA, #32 + + ldr d8, [pB] + add pB , pB, #8 + + ld1 {v2.2d, v3.2d}, [pA] + add pA, pA, #32 + + fmla v16.2d, v0.2d, v8.d[0] + fmla v17.2d, v1.2d, v8.d[0] + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + fmla v18.2d, v2.2d, v8.d[0] + fmla v19.2d, v3.2d, v8.d[0] +.endm + +.macro SAVE8x1 + fmov alpha0, alpha + + ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0] + fmla v0.2d, v16.2d, alphaV0 + fmla v1.2d, v17.2d, alphaV0 + fmla v2.2d, v18.2d, alphaV0 + fmla v3.2d, v19.2d, alphaV0 + st1 {v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0] + + prfm PLDL2KEEP, [pCRow0, C_PRE_SIZE] + add pCRow0, pCRow0, #64 +.endm + + +/******************************************************************************/ + +.macro INIT4x1 + fmov d16, xzr + fmov d17, d16 +.endm + +.macro KERNEL4x1_SUB + ldr d8, [pB] + add pB , pB, #8 + + ld1 {v0.2d, v1.2d}, [pA] + add pA , pA, #32 + + fmla v16.2d, v0.2d, v8.d[0] + fmla v17.2d, v1.2d, v8.d[0] +.endm + +.macro SAVE4x1 + fmov alpha0, alpha + + ld1 {v8.2d, v9.2d}, [pCRow0] + fmla v8.2d, v16.2d, alphaV0 + fmla v9.2d, v17.2d, alphaV0 + st1 {v8.2d, v9.2d}, [pCRow0] + + prfm PLDL2KEEP, [pCRow0, C_PRE_SIZE] + add pCRow0, pCRow0, #32 +.endm + + +/******************************************************************************/ + +.macro INIT2x1 + fmov d16, xzr +.endm + +.macro KERNEL2x1_SUB + ldr d8, [pB] + add pB , pB, #8 + + ld1 {v0.2d}, [pA] + add pA , pA, #16 + + fmla v16.2d, v0.2d, v8.d[0] +.endm + +.macro SAVE2x1 + fmov alpha0, alpha + + ld1 {v8.2d}, [pCRow0] + fmla v8.2d, v16.2d, alphaV0 + st1 {v8.2d}, [pCRow0] + + prfm PLDL2KEEP, [pCRow0, C_PRE_SIZE] + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT1x1 + fmov d16, xzr +.endm + +.macro KERNEL1x1_SUB + ldr d8, [pB] + add pB , pB, #8 + + ldr d0, [pA] + add pA , pA, #8 + + fmadd d16, d0, d8, d16 +.endm + +.macro SAVE1x1 + fmov alpha0, alpha + ldr d8, [pCRow0] + fmadd d8, d16, alpha0, d8 + str d8, [pCRow0] + + prfm PLDL2KEEP, [pCRow0, C_PRE_SIZE] + add pCRow0, pCRow0, #8 +.endm + +.macro KERNEL8x4_M1_M2_x1 + KERNEL8x4_M1_M2 +.endm + +.macro KERNEL8x4_M1_M2_x2 + KERNEL8x4_M1_M2_x1 + KERNEL8x4_M1_M2_x1 +.endm + +.macro KERNEL8x4_M1_M2_x4 + KERNEL8x4_M1_M2_x2 + KERNEL8x4_M1_M2_x2 +.endm + +.macro KERNEL8x4_M1_M2_x8 + KERNEL8x4_M1_M2_x4 + KERNEL8x4_M1_M2_x4 +.endm + +.macro KERNEL8x4_M1_M2_x16 + KERNEL8x4_M1_M2_x8 + KERNEL8x4_M1_M2_x8 +.endm + +.macro KERNEL8x4_M1_M2_x32 + KERNEL8x4_M1_M2_x16 + KERNEL8x4_M1_M2_x16 +.endm + +.macro KERNEL8x4_M1_M2_x64 + KERNEL8x4_M1_M2_x32 + KERNEL8x4_M1_M2_x32 +.endm + +/******************************************************************************* +* End of macro definitions +*******************************************************************************/ + + PROLOGUE + + .align 5 + add sp, sp, #-(11 * 16) + stp d8, d9, [sp, #(0 * 16)] + stp d10, d11, [sp, #(1 * 16)] + stp d12, d13, [sp, #(2 * 16)] + stp d14, d15, [sp, #(3 * 16)] + stp d16, d17, [sp, #(4 * 16)] + stp x18, x19, [sp, #(5 * 16)] + stp x20, x21, [sp, #(6 * 16)] + stp x22, x23, [sp, #(7 * 16)] + stp x24, x25, [sp, #(8 * 16)] + stp x26, x27, [sp, #(9 * 16)] + str x28, [sp, #(10 * 16)] + + prfm PLDL1KEEP, [origPB] + prfm PLDL1KEEP, [origPA] + + + ldr A_PRE_SIZE, =vulcan_pre_a + ldr A_PRE_SIZE, [A_PRE_SIZE] + ldr B_PRE_SIZE, =vulcan_pre_b + ldr B_PRE_SIZE, [B_PRE_SIZE] + ldr C_PRE_SIZE, =vulcan_pre_c + ldr C_PRE_SIZE, [C_PRE_SIZE] + add A_PRE_SIZE_64, A_PRE_SIZE, #64 + add B_PRE_SIZE_64, B_PRE_SIZE, #64 + + fmov alpha, d0 + + lsl LDC, LDC, #3 // ldc = ldc * 8 + + mov pB, origPB + + mov counterJ, origN + asr counterJ, counterJ, #2 // J = J / 4 + cmp counterJ, #0 + ble dgemm_kernel_L2_BEGIN + +/******************************************************************************/ + + .align 5 +dgemm_kernel_L4_BEGIN: + mov pCRow0, pC + add pCRow1, pCRow0, LDC + add pCRow2, pCRow1, LDC + add pCRow3, pCRow2, LDC + + add pC, pCRow3, LDC + + mov pA, origPA // pA = start of A array + +dgemm_kernel_L4_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 // counterI = counterI / 8 + cmp counterI, #0 + ble dgemm_kernel_L4_M4_BEGIN + + .align 5 +dgemm_kernel_L4_M8_20: + + mov pB, origPB + + asr counterL , origK, #7 // L = K / 128 + cmp counterL , #2 // is there at least 4 to do? + blt dgemm_kernel_L4_M8_32 + + KERNEL8x4_I + KERNEL8x4_M2 + KERNEL8x4_M1_M2_x32 + KERNEL8x4_M1_M2_x16 + KERNEL8x4_M1_M2_x8 + KERNEL8x4_M1_M2_x4 + KERNEL8x4_M1_M2_x2 + KERNEL8x4_M1_M2_x1 + + subs counterL, counterL, #2 // subtract 2 + ble dgemm_kernel_L4_M8_22a + + .align 5 +dgemm_kernel_L4_M8_22: + + KERNEL8x4_M1_M2_x64 + + subs counterL, counterL, #1 + bgt dgemm_kernel_L4_M8_22 + + .align 5 +dgemm_kernel_L4_M8_22a: + + KERNEL8x4_M1_M2_x32 + KERNEL8x4_M1_M2_x16 + KERNEL8x4_M1_M2_x8 + KERNEL8x4_M1_M2_x4 + KERNEL8x4_M1_M2_x2 + KERNEL8x4_M1_M2_x1 + KERNEL8x4_M1 + KERNEL8x4_E + + b dgemm_kernel_L4_M8_44 + + .align 5 +dgemm_kernel_L4_M8_32: + + tst counterL, #1 + ble dgemm_kernel_L4_M8_40 + + KERNEL8x4_I + KERNEL8x4_M2 + KERNEL8x4_M1_M2_x32 + KERNEL8x4_M1_M2_x16 + KERNEL8x4_M1_M2_x8 + KERNEL8x4_M1_M2_x4 + KERNEL8x4_M1_M2_x2 + KERNEL8x4_M1 + KERNEL8x4_E + + b dgemm_kernel_L4_M8_44 + +dgemm_kernel_L4_M8_40: + + INIT8x4 + +dgemm_kernel_L4_M8_44: + + ands counterL , origK, #127 + ble dgemm_kernel_L4_M8_100 + + .align 5 +dgemm_kernel_L4_M8_46: + + KERNEL8x4_SUB + + subs counterL, counterL, #1 + bne dgemm_kernel_L4_M8_46 + +dgemm_kernel_L4_M8_100: + prfm PLDL2KEEP, [pCRow0, C_PRE_SIZE] + prfm PLDL2KEEP, [pCRow1, C_PRE_SIZE] + prfm PLDL2KEEP, [pCRow2, C_PRE_SIZE] + prfm PLDL2KEEP, [pCRow3, C_PRE_SIZE] + prfm PLDL1KEEP, [pA] + prfm PLDL1KEEP, [pA, #64] + prfm PLDL1KEEP, [origPB] + + SAVE8x4 + +dgemm_kernel_L4_M8_END: + subs counterI, counterI, #1 + bne dgemm_kernel_L4_M8_20 + +dgemm_kernel_L4_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble dgemm_kernel_L4_END + + tst counterI, #4 + ble dgemm_kernel_L4_M2_BEGIN + +dgemm_kernel_L4_M4_20: + + INIT4x4 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dgemm_kernel_L4_M4_40 + + .align 5 +dgemm_kernel_L4_M4_22: + + KERNEL4x4_SUB + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + KERNEL4x4_SUB + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + KERNEL4x4_SUB + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + KERNEL4x4_SUB + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + + KERNEL4x4_SUB + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + KERNEL4x4_SUB + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + KERNEL4x4_SUB + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + KERNEL4x4_SUB + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + + subs counterL, counterL, #1 + bgt dgemm_kernel_L4_M4_22 + +dgemm_kernel_L4_M4_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L4_M4_100 + +dgemm_kernel_L4_M4_42: + + KERNEL4x4_SUB + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + + subs counterL, counterL, #1 + bgt dgemm_kernel_L4_M4_42 + +dgemm_kernel_L4_M4_100: + + SAVE4x4 + +dgemm_kernel_L4_M4_END: + +dgemm_kernel_L4_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble dgemm_kernel_L4_END + + tst counterI, #2 // counterI = counterI / 2 + ble dgemm_kernel_L4_M1_BEGIN + +dgemm_kernel_L4_M2_20: + + INIT2x4 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dgemm_kernel_L4_M2_40 + + .align 5 +dgemm_kernel_L4_M2_22: + + KERNEL2x4_SUB + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + KERNEL2x4_SUB + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + KERNEL2x4_SUB + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + KERNEL2x4_SUB + + KERNEL2x4_SUB + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + KERNEL2x4_SUB + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + KERNEL2x4_SUB + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + KERNEL2x4_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L4_M2_22 + + +dgemm_kernel_L4_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L4_M2_100 + + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + prfm PLDL1KEEP, [pA, A_PRE_SIZE_64] +dgemm_kernel_L4_M2_42: + + KERNEL2x4_SUB + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + + subs counterL, counterL, #1 + bgt dgemm_kernel_L4_M2_42 + +dgemm_kernel_L4_M2_100: + + SAVE2x4 + +dgemm_kernel_L4_M2_END: + + +dgemm_kernel_L4_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble dgemm_kernel_L4_END + +dgemm_kernel_L4_M1_20: + + INIT1x4 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dgemm_kernel_L4_M1_40 + + .align 5 +dgemm_kernel_L4_M1_22: + KERNEL1x4_SUB + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + KERNEL1x4_SUB + KERNEL1x4_SUB + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + KERNEL1x4_SUB + + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + + KERNEL1x4_SUB + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + KERNEL1x4_SUB + KERNEL1x4_SUB + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + KERNEL1x4_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L4_M1_22 + + +dgemm_kernel_L4_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L4_M1_100 + + prfm PLDL1KEEP, [pA, A_PRE_SIZE] +dgemm_kernel_L4_M1_42: + + KERNEL1x4_SUB + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + + subs counterL, counterL, #1 + bgt dgemm_kernel_L4_M1_42 + +dgemm_kernel_L4_M1_100: + + SAVE1x4 + +dgemm_kernel_L4_END: + + lsl temp, origK, #5 + add origPB, origPB, temp // B = B + K * 4 * 8 + + subs counterJ, counterJ , #1 // j-- + bgt dgemm_kernel_L4_BEGIN + + +/******************************************************************************/ + +dgemm_kernel_L2_BEGIN: // less than 2 left in N direction + + mov counterJ , origN + tst counterJ , #3 + ble dgemm_kernel_L999 // error, N was less than 4? + + tst counterJ , #2 + ble dgemm_kernel_L1_BEGIN + + mov pCRow0, pC + add pCRow1, pCRow0, LDC + + add pC, pCRow1, LDC + + mov pA, origPA // pA = A + +dgemm_kernel_L2_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 // counterI = counterI / 8 + cmp counterI, #0 + ble dgemm_kernel_L2_M4_BEGIN + + .align 5 +dgemm_kernel_L2_M8_20: + + INIT8x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble dgemm_kernel_L2_M8_40 + + .align 5 +dgemm_kernel_L2_M8_22: + KERNEL8x2_SUB + KERNEL8x2_SUB + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + KERNEL8x2_SUB + KERNEL8x2_SUB + + KERNEL8x2_SUB + KERNEL8x2_SUB + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + KERNEL8x2_SUB + KERNEL8x2_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L2_M8_22 + +dgemm_kernel_L2_M8_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L2_M8_100 + + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + prfm PLDL1KEEP, [pB, B_PRE_SIZE_64] +dgemm_kernel_L2_M8_42: + + KERNEL8x2_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L2_M8_42 + +dgemm_kernel_L2_M8_100: + + SAVE8x2 + +dgemm_kernel_L2_M8_END: + + subs counterI, counterI, #1 + bgt dgemm_kernel_L2_M8_20 + +dgemm_kernel_L2_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble dgemm_kernel_L2_END + + tst counterI, #4 // counterI = counterI / 2 + ble dgemm_kernel_L2_M2_BEGIN + +dgemm_kernel_L2_M4_20: + + INIT4x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble dgemm_kernel_L2_M4_40 + + .align 5 +dgemm_kernel_L2_M4_22: + KERNEL4x2_SUB + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + KERNEL4x2_SUB + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + KERNEL4x2_SUB + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + KERNEL4x2_SUB + + KERNEL4x2_SUB + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + KERNEL4x2_SUB + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + KERNEL4x2_SUB + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + KERNEL4x2_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L2_M4_22 + + +dgemm_kernel_L2_M4_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L2_M4_100 + + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + prfm PLDL1KEEP, [pB, B_PRE_SIZE_64] +dgemm_kernel_L2_M4_42: + + KERNEL4x2_SUB + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + + subs counterL, counterL, #1 + bgt dgemm_kernel_L2_M4_42 + +dgemm_kernel_L2_M4_100: + + SAVE4x2 + +dgemm_kernel_L2_M4_END: + + +dgemm_kernel_L2_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble dgemm_kernel_L2_END + + tst counterI, #2 // counterI = counterI / 2 + ble dgemm_kernel_L2_M1_BEGIN + +dgemm_kernel_L2_M2_20: + + INIT2x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble dgemm_kernel_L2_M2_40 + +dgemm_kernel_L2_M2_22: + + KERNEL2x2_SUB + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + KERNEL2x2_SUB + KERNEL2x2_SUB + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + KERNEL2x2_SUB + + KERNEL2x2_SUB + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + KERNEL2x2_SUB + KERNEL2x2_SUB + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L2_M2_22 + + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + prfm PLDL1KEEP, [pA, A_PRE_SIZE_64] + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + prfm PLDL1KEEP, [pB, B_PRE_SIZE_64] +dgemm_kernel_L2_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L2_M2_100 + +dgemm_kernel_L2_M2_42: + + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L2_M2_42 + +dgemm_kernel_L2_M2_100: + + SAVE2x2 + +dgemm_kernel_L2_M2_END: + + +dgemm_kernel_L2_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble dgemm_kernel_L2_END + +dgemm_kernel_L2_M1_20: + + INIT1x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL, #0 + ble dgemm_kernel_L2_M1_40 + +dgemm_kernel_L2_M1_22: + KERNEL1x2_SUB + KERNEL1x2_SUB + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + KERNEL1x2_SUB + KERNEL1x2_SUB + + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + + KERNEL1x2_SUB + KERNEL1x2_SUB + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + KERNEL1x2_SUB + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L2_M1_22 + + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + prfm PLDL1KEEP, [pB, B_PRE_SIZE_64] +dgemm_kernel_L2_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L2_M1_100 + +dgemm_kernel_L2_M1_42: + + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L2_M1_42 + +dgemm_kernel_L2_M1_100: + + SAVE1x2 + +dgemm_kernel_L2_END: + add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 + +/******************************************************************************/ + +dgemm_kernel_L1_BEGIN: + + mov counterJ , origN + tst counterJ , #1 + ble dgemm_kernel_L999 // done + + mov pCRow0, pC // pCRow0 = C + add pC , pC , LDC // Update pC to point to next + + mov pA, origPA // pA = A + +dgemm_kernel_L1_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 // counterI = counterI / 8 + cmp counterI, #0 + ble dgemm_kernel_L1_M4_BEGIN + + .align 5 +dgemm_kernel_L1_M8_20: + + INIT8x1 + + mov pB, origPB + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dgemm_kernel_L1_M8_40 + + .align 5 +dgemm_kernel_L1_M8_22: + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L1_M8_22 + + +dgemm_kernel_L1_M8_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L1_M8_100 + + prfm PLDL1KEEP, [pB, B_PRE_SIZE] +dgemm_kernel_L1_M8_42: + + KERNEL8x1_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L1_M8_42 + +dgemm_kernel_L1_M8_100: + + SAVE8x1 + +dgemm_kernel_L1_M8_END: + + subs counterI, counterI, #1 + bgt dgemm_kernel_L1_M8_20 + +dgemm_kernel_L1_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble dgemm_kernel_L1_END + + tst counterI, #4 // counterI = counterI / 2 + ble dgemm_kernel_L1_M2_BEGIN + +dgemm_kernel_L1_M4_20: + + INIT4x1 + + mov pB, origPB + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dgemm_kernel_L1_M4_40 + + .align 5 +dgemm_kernel_L1_M4_22: + KERNEL4x1_SUB + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + KERNEL4x1_SUB + KERNEL4x1_SUB + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + KERNEL4x1_SUB + + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + + KERNEL4x1_SUB + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + KERNEL4x1_SUB + KERNEL4x1_SUB + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + KERNEL4x1_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L1_M4_22 + + +dgemm_kernel_L1_M4_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L1_M4_100 + + prfm PLDL1KEEP, [pB, B_PRE_SIZE] +dgemm_kernel_L1_M4_42: + + KERNEL4x1_SUB + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + + subs counterL, counterL, #1 + bgt dgemm_kernel_L1_M4_42 + +dgemm_kernel_L1_M4_100: + + SAVE4x1 + +dgemm_kernel_L1_M4_END: + +dgemm_kernel_L1_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble dgemm_kernel_L1_END + + tst counterI, #2 // counterI = counterI / 2 + ble dgemm_kernel_L1_M1_BEGIN + +dgemm_kernel_L1_M2_20: + + INIT2x1 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dgemm_kernel_L1_M2_40 + +dgemm_kernel_L1_M2_22: + + KERNEL2x1_SUB + KERNEL2x1_SUB + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + KERNEL2x1_SUB + KERNEL2x1_SUB + + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + + KERNEL2x1_SUB + KERNEL2x1_SUB + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + KERNEL2x1_SUB + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L1_M2_22 + + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + prfm PLDL1KEEP, [pA, A_PRE_SIZE_64] + prfm PLDL1KEEP, [pB, B_PRE_SIZE] +dgemm_kernel_L1_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L1_M2_100 + +dgemm_kernel_L1_M2_42: + + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L1_M2_42 + +dgemm_kernel_L1_M2_100: + + SAVE2x1 + +dgemm_kernel_L1_M2_END: + + +dgemm_kernel_L1_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble dgemm_kernel_L1_END + +dgemm_kernel_L1_M1_20: + + INIT1x1 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble dgemm_kernel_L1_M1_40 + + +dgemm_kernel_L1_M1_22: + KERNEL1x1_SUB + KERNEL1x1_SUB + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + KERNEL1x1_SUB + KERNEL1x1_SUB + + KERNEL1x1_SUB + KERNEL1x1_SUB + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + KERNEL1x1_SUB + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L1_M1_22 + + +dgemm_kernel_L1_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble dgemm_kernel_L1_M1_100 + + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + prfm PLDL1KEEP, [pB, B_PRE_SIZE] +dgemm_kernel_L1_M1_42: + + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt dgemm_kernel_L1_M1_42 + +dgemm_kernel_L1_M1_100: + + SAVE1x1 + + +dgemm_kernel_L1_END: + + +dgemm_kernel_L999: + mov x0, #0 // set return value + ldp d8, d9, [sp, #(0 * 16)] + ldp d10, d11, [sp, #(1 * 16)] + ldp d12, d13, [sp, #(2 * 16)] + ldp d14, d15, [sp, #(3 * 16)] + ldp d16, d17, [sp, #(4 * 16)] + ldp x18, x19, [sp, #(5 * 16)] + ldp x20, x21, [sp, #(6 * 16)] + ldp x22, x23, [sp, #(7 * 16)] + ldp x24, x25, [sp, #(8 * 16)] + ldp x26, x27, [sp, #(9 * 16)] + ldr x28, [sp, #(10 * 16)] + add sp, sp, #(11*16) + ret + + EPILOGUE +