From 759f37febaab009240710891245c3e0020dc5df9 Mon Sep 17 00:00:00 2001 From: Ashwin Sekhar T K Date: Wed, 11 Jan 2017 13:17:10 +0530 Subject: [PATCH 01/10] ARM64: Let target VULCAN inherit THUNDERX2T99 properties --- kernel/arm64/KERNEL.THUNDERX2T99 | 8 +++++++- kernel/arm64/KERNEL.VULCAN | 3 +-- ...ernel_8x4_vulcan.S => dgemm_kernel_8x4_thunderx2t99.S} | 0 3 files changed, 8 insertions(+), 3 deletions(-) rename kernel/arm64/{dgemm_kernel_8x4_vulcan.S => dgemm_kernel_8x4_thunderx2t99.S} (100%) diff --git a/kernel/arm64/KERNEL.THUNDERX2T99 b/kernel/arm64/KERNEL.THUNDERX2T99 index 8145e9be9..fa0cc918e 100644 --- a/kernel/arm64/KERNEL.THUNDERX2T99 +++ b/kernel/arm64/KERNEL.THUNDERX2T99 @@ -1,2 +1,8 @@ -include $(KERNELDIR)/KERNEL.VULCAN +include $(KERNELDIR)/KERNEL.CORTEXA57 + +ifeq ($(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N), 8x4) +DGEMMKERNEL = dgemm_kernel_8x4_thunderx2t99.S +else +DGEMMKERNEL = dgemm_kernel_$(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N).S +endif diff --git a/kernel/arm64/KERNEL.VULCAN b/kernel/arm64/KERNEL.VULCAN index 24567f573..8b0273951 100644 --- a/kernel/arm64/KERNEL.VULCAN +++ b/kernel/arm64/KERNEL.VULCAN @@ -1,4 +1,3 @@ -include $(KERNELDIR)/KERNEL.CORTEXA57 +include $(KERNELDIR)/KERNEL.THUNDERX2T99 -DGEMMKERNEL = dgemm_kernel_$(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N)_vulcan.S diff --git a/kernel/arm64/dgemm_kernel_8x4_vulcan.S b/kernel/arm64/dgemm_kernel_8x4_thunderx2t99.S similarity index 100% rename from kernel/arm64/dgemm_kernel_8x4_vulcan.S rename to kernel/arm64/dgemm_kernel_8x4_thunderx2t99.S From f279ff47892df2fcd66dae0e9836d34c1225927b Mon Sep 17 00:00:00 2001 From: Ashwin Sekhar T K Date: Wed, 11 Jan 2017 15:07:11 +0530 Subject: [PATCH 02/10] THUNDERX2T99: Add Optimized SGEMM Implementation --- driver/others/parameter.c | 4 + kernel/arm64/KERNEL.THUNDERX2T99 | 6 + kernel/arm64/sgemm_kernel_16x4_thunderx2t99.S | 2087 +++++++++++++++++ param.h | 12 +- 4 files changed, 2103 insertions(+), 6 deletions(-) create mode 100644 kernel/arm64/sgemm_kernel_16x4_thunderx2t99.S diff --git a/driver/others/parameter.c b/driver/others/parameter.c index a5cdbf96f..ac337b203 100644 --- a/driver/others/parameter.c +++ b/driver/others/parameter.c @@ -743,6 +743,10 @@ void blas_set_parameter(void) dgemm_q = 128; dgemm_r = 4096; + sgemm_p = 128; + sgemm_q = 352; + sgemm_r = 4096; + dgemm_prefetch_size_a = 3584; dgemm_prefetch_size_b = 512; dgemm_prefetch_size_c = 128; diff --git a/kernel/arm64/KERNEL.THUNDERX2T99 b/kernel/arm64/KERNEL.THUNDERX2T99 index fa0cc918e..dcfdc96a8 100644 --- a/kernel/arm64/KERNEL.THUNDERX2T99 +++ b/kernel/arm64/KERNEL.THUNDERX2T99 @@ -6,3 +6,9 @@ else DGEMMKERNEL = dgemm_kernel_$(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N).S endif +ifeq ($(SGEMM_UNROLL_M)x$(SGEMM_UNROLL_N), 16x4) +SGEMMKERNEL = sgemm_kernel_16x4_thunderx2t99.S +else +SGEMMKERNEL = sgemm_kernel_$(SGEMM_UNROLL_M)x$(SGEMM_UNROLL_N).S +endif + diff --git a/kernel/arm64/sgemm_kernel_16x4_thunderx2t99.S b/kernel/arm64/sgemm_kernel_16x4_thunderx2t99.S new file mode 100644 index 000000000..06b48f1a9 --- /dev/null +++ b/kernel/arm64/sgemm_kernel_16x4_thunderx2t99.S @@ -0,0 +1,2087 @@ +/******************************************************************************* +Copyright (c) 2017, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#define ASSEMBLER +#include "common.h" + +/* X0 X1 X2 s0 X3 x4 x5 x6 */ +/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc) */ + +#define origM x0 +#define origN x1 +#define origK x2 +#define origPA x3 +#define origPB x4 +#define pC x5 +#define LDC x6 +#define temp x7 +#define counterL x8 +#define counterI x9 +#define counterJ x10 +#define pB x11 +#define pCRow0 x12 +#define pCRow1 x13 +#define pCRow2 x14 +#define pCRow3 x15 +#define pA x16 +#define alpha w17 + +#define alpha0 s10 +#define alphaV0 v10.s[0] + +#define A_PRE_SIZE 2560 +#define B_PRE_SIZE 224 +#define C_PRE_SIZE 160 + + +// 00 origM +// 01 origN +// 02 origK +// 03 origPA +// 04 origPB +// 05 pC +// 06 origLDC -> LDC +// 07 offset +// 08 counterL +// 09 counterI +// 10 counterJ +// 11 pB +// 12 pCRow0 +// 13 pCRow1 +// 14 pCRow2 +// 15 pA +// 16 temp +// 17 +// 18 must save +// 19 must save +// 20 must save +// 21 must save +// 22 must save +// 23 must save +// 24 must save +// 25 must save +// 26 must save +// 27 must save +// 28 must save +// 29 frame +// 30 link +// 31 sp + +//v00 ALPHA -> pA0_00, pA0_01, pA0_02, pA0_03 +//v01 pA0_04, pA0_05, pA0_06, pA0_07 +//v02 pA0_08, pA0_09, pA0_10, pA0_11 +//v03 pA0_12, pA0_13, pA0_14, pA0_15 +//v04 pA1_00, pA1_01, pA1_02, pA1_03 +//v05 pA1_04, pA1_05, pA1_06, pA1_07 +//v06 pA1_08, pA1_09, pA1_10, pA1_11 +//v07 pA1_12, pA1_13, pA1_14, pA1_15 +//v08 must save pB00 +//v09 must save pB01 +//v10 must save pB02 +//v11 must save pB03 +//v12 must save pB10 +//v13 must save pB11 +//v14 must save pB12 +//v15 must save pB13 +//v16 must save C00, C01, C02, C03 +//v17 must save C04, C05, C06, C07 +//v18 C08, C09, C10, C11 +//v19 C12, C13, C14, C15 +//v20 C16, C17, C18, C19 +//v21 C20, C21, C22, C23 +//v22 C24, C25, C26, C27 +//v23 C28, C29, C30, C31 +//v24 C32, C33, C34, C35 +//v25 C36, C37, C38, C39 +//v26 C40, C41, C42, C43 +//v27 C44, C45, C46, C47 +//v28 C48, C49, C50, C51 +//v29 C52, C53, C54, C55 +//v30 C56, C57, C58, C59 +//v31 C60, C61, C62, C63 + +/******************************************************************************* +* Macro definitions +*******************************************************************************/ + +.macro INIT16x4 + fmov s16, wzr + fmov s17, wzr + fmov s18, s16 + fmov s19, s17 + fmov s20, wzr + fmov s21, s16 + fmov s22, s17 + fmov s23, s18 + fmov s24, wzr + fmov s25, s16 + fmov s26, s17 + fmov s27, s18 + fmov s28, wzr + fmov s29, s16 + fmov s30, s17 + fmov s31, s18 +.endm + +.macro KERNEL16x4_I + ldur q0, [pA] + ldur q1, [pA, #16] + + ldur d8, [pB] + + fmul v16.4s, v0.4s, v8.s[0] + fmul v20.4s, v0.4s, v8.s[1] + + ldur d10, [pB, #8] + + fmul v24.4s, v0.4s, v10.s[0] + fmul v28.4s, v0.4s, v10.s[1] + + ldur q2, [pA, #32] + ldur q3, [pA, #48] + + fmul v17.4s, v1.4s, v8.s[0] + fmul v21.4s, v1.4s, v8.s[1] + + ldur q4, [pA, #64] + ldur q5, [pA, #80] + + fmul v25.4s, v1.4s, v10.s[0] + fmul v29.4s, v1.4s, v10.s[1] + + ldur d12, [pB, #16] + + fmul v18.4s, v2.4s, v8.s[0] + fmul v22.4s, v2.4s, v8.s[1] + + ldur d14, [pB, #24] + add pB, pB, #32 + + fmul v19.4s, v3.4s, v8.s[0] + fmul v23.4s, v3.4s, v8.s[1] + + ldur q6, [pA, #96] + ldur q7, [pA, #112] + add pA, pA, #128 + + fmul v26.4s, v2.4s, v10.s[0] + fmul v30.4s, v2.4s, v10.s[1] + + prfm PLDL1KEEP, [pA, #A_PRE_SIZE] + + fmul v27.4s, v3.4s, v10.s[0] + fmul v31.4s, v3.4s, v10.s[1] + + prfm PLDL1KEEP, [pA, #A_PRE_SIZE+64] +.endm + +.macro KERNEL16x4_M1 + fmla v16.4s, v0.4s, v8.s[0] + fmla v17.4s, v1.4s, v8.s[0] + + ldur q4, [pA] + ldur q5, [pA, #16] + + fmla v18.4s, v2.4s, v8.s[0] + fmla v19.4s, v3.4s, v8.s[0] + + fmla v20.4s, v0.4s, v8.s[1] + fmla v21.4s, v1.4s, v8.s[1] + + ldur d12, [pB] + + fmla v22.4s, v2.4s, v8.s[1] + fmla v23.4s, v3.4s, v8.s[1] + + ldur d14, [pB, #8] + add pB, pB, #16 + + fmla v24.4s, v0.4s, v10.s[0] + fmla v25.4s, v1.4s, v10.s[0] + + prfm PLDL1KEEP, [pA, #A_PRE_SIZE+64] + + fmla v26.4s, v2.4s, v10.s[0] + fmla v27.4s, v3.4s, v10.s[0] + + prfm PLDL1KEEP, [pA, #A_PRE_SIZE] + + fmla v28.4s, v0.4s, v10.s[1] + fmla v29.4s, v1.4s, v10.s[1] + + ldur q6, [pA, #32] + ldur q7, [pA, #48] + add pA, pA, #64 + + fmla v30.4s, v2.4s, v10.s[1] + fmla v31.4s, v3.4s, v10.s[1] +.endm + +.macro KERNEL16x4_M2 + fmla v16.4s, v4.4s, v12.s[0] + fmla v17.4s, v5.4s, v12.s[0] + + ldur q0, [pA] + ldur q1, [pA, #16] + + fmla v18.4s, v6.4s, v12.s[0] + fmla v19.4s, v7.4s, v12.s[0] + + fmla v20.4s, v4.4s, v12.s[1] + fmla v21.4s, v5.4s, v12.s[1] + + ldur d8, [pB] + + fmla v22.4s, v6.4s, v12.s[1] + fmla v23.4s, v7.4s, v12.s[1] + + ldur d10, [pB, #8] + add pB, pB, #16 + + fmla v24.4s, v4.4s, v14.s[0] + fmla v25.4s, v5.4s, v14.s[0] + + prfm PLDL1KEEP, [pB, #B_PRE_SIZE] + + fmla v26.4s, v6.4s, v14.s[0] + fmla v27.4s, v7.4s, v14.s[0] + + ldur q2, [pA, #32] + ldur q3, [pA, #48] + add pA, pA, #64 + + fmla v28.4s, v4.4s, v14.s[1] + fmla v29.4s, v5.4s, v14.s[1] + + fmla v30.4s, v6.4s, v14.s[1] + fmla v31.4s, v7.4s, v14.s[1] +.endm + +.macro KERNEL16x4_E + fmla v16.4s, v4.4s, v12.s[0] + fmla v20.4s, v4.4s, v12.s[1] + fmla v24.4s, v4.4s, v14.s[0] + fmla v28.4s, v4.4s, v14.s[1] + + fmla v17.4s, v5.4s, v12.s[0] + fmla v21.4s, v5.4s, v12.s[1] + fmla v25.4s, v5.4s, v14.s[0] + fmla v29.4s, v5.4s, v14.s[1] + + prfm PLDL1KEEP, [pB, #B_PRE_SIZE] + + fmla v18.4s, v6.4s, v12.s[0] + fmla v22.4s, v6.4s, v12.s[1] + fmla v26.4s, v6.4s, v14.s[0] + fmla v30.4s, v6.4s, v14.s[1] + + fmla v19.4s, v7.4s, v12.s[0] + fmla v23.4s, v7.4s, v12.s[1] + fmla v27.4s, v7.4s, v14.s[0] + fmla v31.4s, v7.4s, v14.s[1] +.endm + +.macro KERNEL16x4_SUB + ldur q0, [pA] + ldur q1, [pA, #16] + ldur d8, [pB] + + fmla v16.4s, v0.4s, v8.s[0] + fmla v20.4s, v0.4s, v8.s[1] + + ldur d10, [pB, #8] + add pB, pB, #16 + + fmla v24.4s, v0.4s, v10.s[0] + fmla v28.4s, v0.4s, v10.s[1] + + ldur q2, [pA, #32] + ldur q3, [pA, #48] + add pA, pA, #64 + + fmla v17.4s, v1.4s, v8.s[0] + fmla v21.4s, v1.4s, v8.s[1] + + fmla v25.4s, v1.4s, v10.s[0] + fmla v29.4s, v1.4s, v10.s[1] + + fmla v18.4s, v2.4s, v8.s[0] + fmla v22.4s, v2.4s, v8.s[1] + + prfm PLDL1KEEP, [pA, #A_PRE_SIZE] + + fmla v19.4s, v3.4s, v8.s[0] + fmla v23.4s, v3.4s, v8.s[1] + + fmla v26.4s, v2.4s, v10.s[0] + fmla v30.4s, v2.4s, v10.s[1] + + prfm PLDL1KEEP, [pB, #B_PRE_SIZE] + + fmla v27.4s, v3.4s, v10.s[0] + fmla v31.4s, v3.4s, v10.s[1] +.endm + +.macro SAVE16x4 + fmov alpha0, alpha + + prfm PLDL2KEEP, [pCRow0, #C_PRE_SIZE] + prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE] + prfm PLDL2KEEP, [pCRow2, #C_PRE_SIZE] + prfm PLDL2KEEP, [pCRow3, #C_PRE_SIZE] + + ldur q0, [pCRow0] + ldur q1, [pCRow0, #16] + ldur q2, [pCRow0, #32] + ldur q3, [pCRow0, #48] + + ldur q4, [pCRow1] + ldur q5, [pCRow1, #16] + ldur q6, [pCRow1, #32] + ldur q7, [pCRow1, #48] + + fmla v0.4s, v16.4s, alphaV0 + fmla v1.4s, v17.4s, alphaV0 + stp q0, q1, [pCRow0] + + fmla v2.4s, v18.4s, alphaV0 + fmla v3.4s, v19.4s, alphaV0 + stp q2, q3, [pCRow0, #32] + ldur q0, [pCRow2] + ldur q1, [pCRow2, #16] + + + fmla v4.4s, v20.4s, alphaV0 + fmla v5.4s, v21.4s, alphaV0 + stp q4, q5, [pCRow1] + ldur q2, [pCRow2, #32] + ldur q3, [pCRow2, #48] + + fmla v6.4s, v22.4s, alphaV0 + fmla v7.4s, v23.4s, alphaV0 + stp q6, q7, [pCRow1, #32] + ldur q4, [pCRow3] + ldur q5, [pCRow3, #16] + + fmla v0.4s, v24.4s, alphaV0 + fmla v1.4s, v25.4s, alphaV0 + stp q0, q1, [pCRow2] + ldur q6, [pCRow3, #32] + ldur q7, [pCRow3, #48] + + fmla v2.4s, v26.4s, alphaV0 + fmla v3.4s, v27.4s, alphaV0 + stp q2, q3, [pCRow2, #32] + + fmla v4.4s, v28.4s, alphaV0 + fmla v5.4s, v29.4s, alphaV0 + stp q4, q5, [pCRow3] + + fmla v6.4s, v30.4s, alphaV0 + fmla v7.4s, v31.4s, alphaV0 + stp q6, q7, [pCRow3, #32] + + add pCRow0, pCRow0, #64 + add pCRow1, pCRow1, #64 + add pCRow2, pCRow2, #64 + add pCRow3, pCRow3, #64 +.endm + +/******************************************************************************/ + +.macro INIT8x4 + fmov s16, wzr + fmov s17, wzr + fmov s20, wzr + fmov s21, s16 + fmov s24, wzr + fmov s25, s16 + fmov s28, wzr + fmov s29, s16 +.endm + +.macro KERNEL8x4_I + ldp s8, s9, [pB], #8 + ldp s10, s11, [pB], #8 + + ldr q0, [pA], #16 + ldr q1, [pA], #16 + + fmul v16.4s, v0.4s, v8.s[0] + fmul v17.4s, v1.4s, v8.s[0] + fmul v20.4s, v0.4s, v9.s[0] + fmul v21.4s, v1.4s, v9.s[0] + fmul v24.4s, v0.4s, v10.s[0] + fmul v25.4s, v1.4s, v10.s[0] + fmul v28.4s, v0.4s, v11.s[0] + fmul v29.4s, v1.4s, v11.s[0] + + ldp s12, s13, [pB], #8 + ldp s14, s15, [pB], #8 + + ldr q4, [pA], #16 + ldr q5, [pA], #16 +.endm + +.macro KERNEL8x4_M1 + fmla v16.4s, v0.4s, v8.s[0] + fmla v17.4s, v1.4s, v8.s[0] + fmla v20.4s, v0.4s, v9.s[0] + fmla v21.4s, v1.4s, v9.s[0] + fmla v24.4s, v0.4s, v10.s[0] + fmla v25.4s, v1.4s, v10.s[0] + fmla v28.4s, v0.4s, v11.s[0] + fmla v29.4s, v1.4s, v11.s[0] + + ldp s12, s13, [pB], #8 + ldp s14, s15, [pB], #8 + + ldr q4, [pA], #16 + ldr q5, [pA], #16 +.endm + +.macro KERNEL8x4_M2 + fmla v16.4s, v4.4s, v12.s[0] + fmla v17.4s, v5.4s, v12.s[0] + fmla v20.4s, v4.4s, v13.s[0] + fmla v21.4s, v5.4s, v13.s[0] + fmla v24.4s, v4.4s, v14.s[0] + fmla v25.4s, v5.4s, v14.s[0] + fmla v28.4s, v4.4s, v15.s[0] + fmla v29.4s, v5.4s, v15.s[0] + + ldp s8, s9, [pB], #8 + ldp s10, s11, [pB], #8 + + ldr q0, [pA], #16 + ldr q1, [pA], #16 +.endm + +.macro KERNEL8x4_E + fmla v16.4s, v4.4s, v12.s[0] + fmla v17.4s, v5.4s, v12.s[0] + fmla v20.4s, v4.4s, v13.s[0] + fmla v21.4s, v5.4s, v13.s[0] + fmla v24.4s, v4.4s, v14.s[0] + fmla v25.4s, v5.4s, v14.s[0] + fmla v28.4s, v4.4s, v15.s[0] + fmla v29.4s, v5.4s, v15.s[0] +.endm + +.macro KERNEL8x4_SUB + ldp s8, s9, [pB], #8 + ldp s10, s11, [pB], #8 + + ldr q0, [pA], #16 + ldr q1, [pA], #16 + + fmla v16.4s, v0.4s, v8.s[0] + fmla v17.4s, v1.4s, v8.s[0] + fmla v20.4s, v0.4s, v9.s[0] + fmla v21.4s, v1.4s, v9.s[0] + fmla v24.4s, v0.4s, v10.s[0] + fmla v25.4s, v1.4s, v10.s[0] + fmla v28.4s, v0.4s, v11.s[0] + fmla v29.4s, v1.4s, v11.s[0] +.endm + +.macro SAVE8x4 + fmov alpha0, alpha + + ldp q0, q1, [pCRow0] + fmla v0.4s, v16.4s, alphaV0 + fmla v1.4s, v17.4s, alphaV0 + stp q0, q1, [pCRow0] + + add pCRow0, pCRow0, #32 + + ldp q2, q3, [pCRow1] + fmla v2.4s, v20.4s, alphaV0 + fmla v3.4s, v21.4s, alphaV0 + stp q2, q3, [pCRow1] + + add pCRow1, pCRow1, #32 + + ldp q4, q5, [pCRow2] + fmla v4.4s, v24.4s, alphaV0 + fmla v5.4s, v25.4s, alphaV0 + stp q4, q5, [pCRow2] + + add pCRow2, pCRow2, #32 + + ldp q6, q7, [pCRow3] + fmla v6.4s, v28.4s, alphaV0 + fmla v7.4s, v29.4s, alphaV0 + stp q6, q7, [pCRow3] + + add pCRow3, pCRow3, #32 +.endm + +/******************************************************************************/ + +.macro INIT4x4 + fmov s16, wzr + fmov s20, wzr + fmov s24, wzr + fmov s28, wzr +.endm + +.macro KERNEL4x4_I + ldp s8, s9, [pB], #8 + ldp s10, s11, [pB], #8 + + ldr q0, [pA], #16 + + fmul v16.4s, v0.4s, v8.s[0] + fmul v20.4s, v0.4s, v9.s[0] + fmul v24.4s, v0.4s, v10.s[0] + fmul v28.4s, v0.4s, v11.s[0] + + ldp s12, s13, [pB], #8 + ldp s14, s15, [pB], #8 + + ldr q1, [pA], #16 +.endm + +.macro KERNEL4x4_M1 + fmla v16.4s, v0.4s, v8.s[0] + fmla v20.4s, v0.4s, v9.s[0] + fmla v24.4s, v0.4s, v10.s[0] + fmla v28.4s, v0.4s, v11.s[0] + + ldp s12, s13, [pB], #8 + ldp s14, s15, [pB], #8 + + ldr q1, [pA], #16 +.endm + +.macro KERNEL4x4_M2 + fmla v16.4s, v1.4s, v12.s[0] + fmla v20.4s, v1.4s, v13.s[0] + fmla v24.4s, v1.4s, v14.s[0] + fmla v28.4s, v1.4s, v15.s[0] + + ldp s8, s9, [pB], #8 + ldp s10, s11, [pB], #8 + + ldr q0, [pA], #16 +.endm + +.macro KERNEL4x4_E + fmla v16.4s, v1.4s, v12.s[0] + fmla v20.4s, v1.4s, v13.s[0] + fmla v24.4s, v1.4s, v14.s[0] + fmla v28.4s, v1.4s, v15.s[0] +.endm + +.macro KERNEL4x4_SUB + ldp s8, s9, [pB], #8 + ldp s10, s11, [pB], #8 + + ldr q0, [pA], #16 + + fmla v16.4s, v0.4s, v8.s[0] + fmla v20.4s, v0.4s, v9.s[0] + fmla v24.4s, v0.4s, v10.s[0] + fmla v28.4s, v0.4s, v11.s[0] +.endm + +.macro SAVE4x4 + fmov alpha0, alpha + + ldr q0, [pCRow0] + fmla v0.4s, v16.4s, alphaV0 + str q0, [pCRow0] + + add pCRow0, pCRow0, #16 + + ldr q1, [pCRow1] + fmla v1.4s, v20.4s, alphaV0 + str q1, [pCRow1] + + add pCRow1, pCRow1, #16 + + ldr q2, [pCRow2] + fmla v2.4s, v24.4s, alphaV0 + str q2, [pCRow2] + + add pCRow2, pCRow2, #16 + + ldr q3, [pCRow3] + fmla v3.4s, v28.4s, alphaV0 + str q3, [pCRow3] + + add pCRow3, pCRow3, #16 +.endm + +/******************************************************************************/ + +.macro INIT2x4 + fmov s16, wzr + fmov s20, s16 + fmov s24, s20 + fmov s28, s16 +.endm + +.macro KERNEL2x4_SUB + ldp s8, s9, [pB], #8 + ldp s10, s11, [pB], #8 + + ldr d0, [pA], #8 + + fmla v16.2s, v0.2s, v8.s[0] + fmla v20.2s, v0.2s, v9.s[0] + fmla v24.2s, v0.2s, v10.s[0] + fmla v28.2s, v0.2s, v11.s[0] +.endm + +.macro SAVE2x4 + fmov alpha0, alpha + + ldr d0, [pCRow0] + fmla v0.2s, v16.2s, alphaV0 + str d0, [pCRow0] + + add pCRow0, pCRow0, #8 + + ldr d1, [pCRow1] + fmla v1.2s, v20.2s, alphaV0 + str d1, [pCRow1] + + add pCRow1, pCRow1, #8 + + ldr d0, [pCRow2] + fmla v0.2s, v24.2s, alphaV0 + str d0, [pCRow2] + + add pCRow2, pCRow2, #8 + + ldr d1, [pCRow3] + fmla v1.2s, v28.2s, alphaV0 + str d1, [pCRow3] + + add pCRow3, pCRow3, #8 +.endm + +/******************************************************************************/ + +.macro INIT1x4 + fmov s16, wzr + fmov s20, s16 +.endm + +.macro KERNEL1x4_SUB + ldr s0, [pA] + add pA, pA, #4 + + ld1 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + + fmla v16.2s, v8.2s, v0.s[0] + fmla v20.2s, v9.2s, v0.s[0] +.endm + +.macro SAVE1x4 + fmov alpha0, alpha + + ld1 {v8.s}[0], [pCRow0] + ld1 {v8.s}[1], [pCRow1] + fmla v8.2s, v16.2s, alphaV0 + st1 {v8.s}[0], [pCRow0] + st1 {v8.s}[1], [pCRow1] + + add pCRow0, pCRow0, #4 + add pCRow1, pCRow1, #4 + + ld1 {v12.s}[0], [pCRow2] + ld1 {v12.s}[1], [pCRow3] + fmla v12.2s, v20.2s, alphaV0 + st1 {v12.s}[0], [pCRow2] + st1 {v12.s}[1], [pCRow3] + + add pCRow2, pCRow2, #4 + add pCRow3, pCRow3, #4 +.endm + +/******************************************************************************/ + +.macro INIT16x2 + fmov s16, wzr + fmov s17, wzr + fmov s18, wzr + fmov s19, s16 + fmov s20, wzr + fmov s21, s16 + fmov s22, wzr + fmov s23, s16 +.endm + +.macro KERNEL16x2_SUB + ld1 {v8.2s}, [pB] + add pB, pB, #8 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + ld1 {v2.4s}, [pA] + add pA, pA, #16 + ld1 {v3.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v8.s[0] + fmla v17.4s, v1.4s, v8.s[0] + fmla v18.4s, v2.4s, v8.s[0] + fmla v19.4s, v3.4s, v8.s[0] + + fmla v20.4s, v0.4s, v8.s[1] + fmla v21.4s, v1.4s, v8.s[1] + fmla v22.4s, v2.4s, v8.s[1] + fmla v23.4s, v3.4s, v8.s[1] +.endm + +.macro SAVE16x2 + fmov alpha0, alpha + + add pCRow1, pCRow0, LDC + + ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow0] + fmla v0.4s, v16.4s, alphaV0 + fmla v1.4s, v17.4s, alphaV0 + fmla v2.4s, v18.4s, alphaV0 + fmla v3.4s, v19.4s, alphaV0 + st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow0] + + ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [pCRow1] + fmla v4.4s, v20.4s, alphaV0 + fmla v5.4s, v21.4s, alphaV0 + fmla v6.4s, v22.4s, alphaV0 + fmla v7.4s, v23.4s, alphaV0 + st1 {v4.4s, v5.4s, v6.4s, v7.4s}, [pCRow1] + + add pCRow0, pCRow0, #64 +.endm + +/******************************************************************************/ + +.macro INIT8x2 + fmov s16, wzr + fmov s17, s16 + fmov s20, s17 + fmov s21, s16 +.endm + +.macro KERNEL8x2_SUB + ld1 {v8.2s}, [pB] + add pB, pB, #8 + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v8.s[0] + fmla v17.4s, v1.4s, v8.s[0] + + fmla v20.4s, v0.4s, v8.s[1] + fmla v21.4s, v1.4s, v8.s[1] +.endm + +.macro SAVE8x2 + fmov alpha0, alpha + + add pCRow1, pCRow0, LDC + + ld1 {v0.4s, v1.4s}, [pCRow0] + fmla v0.4s, v16.4s, alphaV0 + fmla v1.4s, v17.4s, alphaV0 + st1 {v0.4s, v1.4s}, [pCRow0] + + add pCRow2, pCRow1, LDC + + ld1 {v4.4s, v5.4s}, [pCRow1] + fmla v4.4s, v20.4s, alphaV0 + fmla v5.4s, v21.4s, alphaV0 + st1 {v4.4s, v5.4s}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT4x2 + fmov s16, wzr + fmov s17, s16 + fmov s20, s17 + fmov s21, s16 +.endm + +.macro KERNEL4x2_SUB + ld1 {v8.2s}, [pB] + add pB, pB, #8 + ld1 {v0.2s, v1.2s}, [pA] + add pA, pA, #16 + + fmla v16.2s, v0.2s, v8.s[0] + fmla v17.2s, v1.2s, v8.s[0] + fmla v20.2s, v0.2s, v8.s[1] + fmla v21.2s, v1.2s, v8.s[1] +.endm + +.macro SAVE4x2 + fmov alpha0, alpha + + ld1 {v8.2s, v9.2s}, [pCRow0] + fmla v8.2s, v16.2s, alphaV0 + fmla v9.2s, v17.2s, alphaV0 + st1 {v8.2s, v9.2s}, [pCRow0] + + add pCRow1, pCRow0, LDC + ld1 {v12.2s, v13.2s}, [pCRow1] + fmla v12.2s, v20.2s, alphaV0 + fmla v13.2s, v21.2s, alphaV0 + st1 {v12.2s, v13.2s}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT2x2 + fmov s16, wzr + fmov s20, s16 +.endm + +.macro KERNEL2x2_SUB + ld1 {v8.2s}, [pB] + add pB, pB, #8 + + ld1 {v0.2s}, [pA] + add pA, pA, #8 + + fmla v16.2s, v0.2s, v8.s[0] + fmla v20.2s, v0.2s, v8.s[1] +.endm + +.macro SAVE2x2 + fmov alpha0, alpha + + ld1 {v8.2s}, [pCRow0] + fmla v8.2s, v16.2s, alphaV0 + st1 {v8.2s}, [pCRow0] + + add pCRow1 , pCRow0, LDC + ld1 {v12.2s}, [pCRow1] + fmla v12.2s, v20.2s, alphaV0 + st1 {v12.2s}, [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT1x2 + fmov s16, wzr +.endm + +.macro KERNEL1x2_SUB + ld1 {v8.2s} , [pB] + add pB , pB, #8 + + ldr s0 , [pA] + add pA, pA, #4 + + fmla v16.2s, v8.2s, v0.s[0] +.endm + +.macro SAVE1x2 + fmov alpha0, alpha + + add pCRow1 , pCRow0, LDC + ld1 {v8.s}[0], [pCRow0] + ld1 {v8.s}[1], [pCRow1] + fmla v8.2s, v16.2s, alphaV0 + st1 {v8.s}[0], [pCRow0] + st1 {v8.s}[1], [pCRow1] + + add pCRow0, pCRow0, #4 +.endm + +/******************************************************************************/ + +.macro INIT16x1 + fmov s16, wzr + fmov s17, wzr + fmov s18, wzr + fmov s19, s16 +.endm + +.macro KERNEL16x1_SUB + ldr s8, [pB] + add pB , pB, #4 + + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + ld1 {v2.4s}, [pA] + add pA, pA, #16 + ld1 {v3.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v8.s[0] + fmla v17.4s, v1.4s, v8.s[0] + fmla v18.4s, v2.4s, v8.s[0] + fmla v19.4s, v3.4s, v8.s[0] +.endm + +.macro SAVE16x1 + fmov alpha0, alpha + + ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow0] + fmla v0.4s, v16.4s, alphaV0 + fmla v1.4s, v17.4s, alphaV0 + fmla v2.4s, v18.4s, alphaV0 + fmla v3.4s, v19.4s, alphaV0 + st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow0] + + add pCRow0, pCRow0, #64 +.endm + +/******************************************************************************/ + +.macro INIT8x1 + fmov s16, wzr + fmov s17, wzr +.endm + +.macro KERNEL8x1_SUB + ldr s8, [pB] + add pB , pB, #4 + + ld1 {v0.4s}, [pA] + add pA, pA, #16 + ld1 {v1.4s}, [pA] + add pA, pA, #16 + + fmla v16.4s, v0.4s, v8.s[0] + fmla v17.4s, v1.4s, v8.s[0] +.endm + +.macro SAVE8x1 + fmov alpha0, alpha + + ld1 {v0.4s, v1.4s}, [pCRow0] + fmla v0.4s, v16.4s, alphaV0 + fmla v1.4s, v17.4s, alphaV0 + st1 {v0.4s, v1.4s}, [pCRow0] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT4x1 + fmov s16, wzr + fmov s17, s16 +.endm + +.macro KERNEL4x1_SUB + ldr s8, [pB] + add pB , pB, #4 + + ld1 {v0.2s, v1.2s}, [pA] + add pA , pA, #16 + + fmla v16.2s, v0.2s, v8.s[0] + fmla v17.2s, v1.2s, v8.s[0] +.endm + +.macro SAVE4x1 + fmov alpha0, alpha + + ld1 {v8.2s, v9.2s}, [pCRow0] + fmla v8.2s, v16.2s, alphaV0 + fmla v9.2s, v17.2s, alphaV0 + st1 {v8.2s, v9.2s}, [pCRow0] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT2x1 + fmov s16, wzr +.endm + +.macro KERNEL2x1_SUB + ldr s8, [pB] + add pB , pB, #4 + + ld1 {v0.2s}, [pA] + add pA , pA, #8 + + fmla v16.2s, v0.2s, v8.s[0] +.endm + +.macro SAVE2x1 + fmov alpha0, alpha + + ld1 {v8.2s}, [pCRow0] + fmla v8.2s, v16.2s, alphaV0 + st1 {v8.2s}, [pCRow0] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT1x1 + fmov s16, wzr +.endm + +.macro KERNEL1x1_SUB + ldr s8, [pB] + add pB , pB, #4 + + ldr s0, [pA] + add pA , pA, #4 + + fmadd s16, s0, s8, s16 +.endm + +.macro SAVE1x1 + fmov alpha0, alpha + + ldr s8, [pCRow0] + fmla s8, s16, alphaV0 + str s8, [pCRow0] + + add pCRow0, pCRow0, #4 +.endm + +.macro KERNEL16x4_M1_M2_x1 + KERNEL16x4_M1 + KERNEL16x4_M2 +.endm + +.macro KERNEL16x4_M1_M2_x2 + KERNEL16x4_M1_M2_x1 + KERNEL16x4_M1_M2_x1 +.endm + +.macro KERNEL16x4_M1_M2_x4 + KERNEL16x4_M1_M2_x2 + KERNEL16x4_M1_M2_x2 +.endm + +.macro KERNEL16x4_M1_M2_x8 + KERNEL16x4_M1_M2_x4 + KERNEL16x4_M1_M2_x4 +.endm + +.macro KERNEL16x4_M1_M2_x16 + KERNEL16x4_M1_M2_x8 + KERNEL16x4_M1_M2_x8 +.endm + +.macro KERNEL16x4_M1_M2_x32 + KERNEL16x4_M1_M2_x16 + KERNEL16x4_M1_M2_x16 +.endm + +.macro KERNEL16x4_M1_M2_x64 + KERNEL16x4_M1_M2_x32 + KERNEL16x4_M1_M2_x32 +.endm + +/******************************************************************************* +* End of macro definitions +*******************************************************************************/ + + PROLOGUE + +sgemm_kernel_begin: + + .align 5 + add sp, sp, #-(11 * 16) + stp d8, d9, [sp, #(0 * 16)] + stp d10, d11, [sp, #(1 * 16)] + stp d12, d13, [sp, #(2 * 16)] + stp d14, d15, [sp, #(3 * 16)] + stp d16, d17, [sp, #(4 * 16)] + stp x18, x19, [sp, #(5 * 16)] + stp x20, x21, [sp, #(6 * 16)] + stp x22, x23, [sp, #(7 * 16)] + stp x24, x25, [sp, #(8 * 16)] + stp x26, x27, [sp, #(9 * 16)] + str x28, [sp, #(10 * 16)] + + prfm PLDL1KEEP, [origPB] + prfm PLDL1KEEP, [origPA] + + fmov alpha, s0 + + lsl LDC, LDC, #2 // ldc = ldc * 4 + + mov pB, origPB + + mov counterJ, origN + asr counterJ, counterJ, #2 // J = J / 4 + cmp counterJ, #0 + ble sgemm_kernel_L2_BEGIN + +/******************************************************************************/ + +sgemm_kernel_L4_BEGIN: + mov pCRow0, pC + add pCRow1, pCRow0, LDC + add pCRow2, pCRow1, LDC + add pCRow3, pCRow2, LDC + + add pC, pCRow3, LDC + + mov pA, origPA // pA = start of A array + +sgemm_kernel_L4_M16_BEGIN: + + mov counterI, origM + asr counterI, counterI, #4 // counterI = counterI / 16 + cmp counterI, #0 + ble sgemm_kernel_L4_M8_BEGIN + + .align 5 +sgemm_kernel_L4_M16_20: + + mov pB, origPB + + asr counterL , origK, #4 // L = K / 16 + cmp counterL , #2 + blt sgemm_kernel_L4_M16_32 + + KERNEL16x4_I + KERNEL16x4_M2 + KERNEL16x4_M1_M2_x4 + KERNEL16x4_M1_M2_x2 + KERNEL16x4_M1_M2_x1 + + subs counterL, counterL, #2 + ble sgemm_kernel_L4_M16_22a + + .align 5 +sgemm_kernel_L4_M16_22: + + KERNEL16x4_M1_M2_x8 + + subs counterL, counterL, #1 + bgt sgemm_kernel_L4_M16_22 + + .align 5 +sgemm_kernel_L4_M16_22a: + + KERNEL16x4_M1_M2_x4 + KERNEL16x4_M1_M2_x2 + KERNEL16x4_M1_M2_x1 + KERNEL16x4_M1 + KERNEL16x4_E + + b sgemm_kernel_L4_M16_44 + + .align 5 +sgemm_kernel_L4_M16_32: + + tst counterL, #1 + ble sgemm_kernel_L4_M16_40 + + KERNEL16x4_I + KERNEL16x4_M2 + KERNEL16x4_M1_M2_x4 + KERNEL16x4_M1_M2_x2 + KERNEL16x4_M1 + KERNEL16x4_E + + b sgemm_kernel_L4_M16_44 + +sgemm_kernel_L4_M16_40: + + INIT16x4 + +sgemm_kernel_L4_M16_44: + + ands counterL , origK, #15 + ble sgemm_kernel_L4_M16_100 + + .align 5 +sgemm_kernel_L4_M16_46: + + KERNEL16x4_SUB + subs counterL, counterL, #1 + bne sgemm_kernel_L4_M16_46 + +sgemm_kernel_L4_M16_100: + prfm PLDL1KEEP, [pA] + prfm PLDL1KEEP, [pA, #64] + prfm PLDL1KEEP, [origPB] + + SAVE16x4 + +sgemm_kernel_L4_M16_END: + subs counterI, counterI, #1 + bne sgemm_kernel_L4_M16_20 + +//------------------------------------------------------------------------------ + +sgemm_kernel_L4_M8_BEGIN: + + mov counterI, origM + tst counterI , #15 + ble sgemm_kernel_L4_END + + tst counterI, #8 + ble sgemm_kernel_L4_M4_BEGIN + +sgemm_kernel_L4_M8_20: + + mov pB, origPB + + asr counterL , origK, #1 // L = K / 2 + cmp counterL , #2 // is there at least 4 to do? + blt sgemm_kernel_L4_M8_32 + + KERNEL8x4_I // do one in the K + KERNEL8x4_M2 // do another in the K + + subs counterL, counterL, #2 + ble sgemm_kernel_L4_M8_22a + .align 5 + +sgemm_kernel_L4_M8_22: + + KERNEL8x4_M1 + KERNEL8x4_M2 + + subs counterL, counterL, #1 + bgt sgemm_kernel_L4_M8_22 + +sgemm_kernel_L4_M8_22a: + + KERNEL8x4_M1 + KERNEL8x4_E + + b sgemm_kernel_L4_M8_44 + +sgemm_kernel_L4_M8_32: + + tst counterL, #1 + ble sgemm_kernel_L4_M8_40 + + KERNEL8x4_I + KERNEL8x4_E + + b sgemm_kernel_L4_M8_44 + +sgemm_kernel_L4_M8_40: + + INIT8x4 + +sgemm_kernel_L4_M8_44: + + ands counterL , origK, #1 + ble sgemm_kernel_L4_M8_100 + +sgemm_kernel_L4_M8_46: + + KERNEL8x4_SUB + +sgemm_kernel_L4_M8_100: + + SAVE8x4 + +sgemm_kernel_L4_M8_END: + +//------------------------------------------------------------------------------ + +sgemm_kernel_L4_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble sgemm_kernel_L4_END + + tst counterI, #4 + ble sgemm_kernel_L4_M2_BEGIN + +sgemm_kernel_L4_M4_20: + + mov pB, origPB + + asr counterL , origK, #1 // L = K / 2 + cmp counterL , #2 // is there at least 4 to do? + blt sgemm_kernel_L4_M4_32 + + KERNEL4x4_I // do one in the K + KERNEL4x4_M2 // do another in the K + + subs counterL, counterL, #2 + ble sgemm_kernel_L4_M4_22a + .align 5 + +sgemm_kernel_L4_M4_22: + + KERNEL4x4_M1 + KERNEL4x4_M2 + + subs counterL, counterL, #1 + bgt sgemm_kernel_L4_M4_22 + +sgemm_kernel_L4_M4_22a: + + KERNEL4x4_M1 + KERNEL4x4_E + + b sgemm_kernel_L4_M4_44 + +sgemm_kernel_L4_M4_32: + + tst counterL, #1 + ble sgemm_kernel_L4_M4_40 + + KERNEL4x4_I + KERNEL4x4_E + + b sgemm_kernel_L4_M4_44 + +sgemm_kernel_L4_M4_40: + + INIT4x4 + +sgemm_kernel_L4_M4_44: + + ands counterL , origK, #1 + ble sgemm_kernel_L4_M4_100 + +sgemm_kernel_L4_M4_46: + + KERNEL4x4_SUB + +sgemm_kernel_L4_M4_100: + + SAVE4x4 + +sgemm_kernel_L4_M4_END: + +//------------------------------------------------------------------------------ + +sgemm_kernel_L4_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble sgemm_kernel_L4_END + + tst counterI, #2 // counterI = counterI / 2 + ble sgemm_kernel_L4_M1_BEGIN + +sgemm_kernel_L4_M2_20: + + INIT2x4 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble sgemm_kernel_L4_M2_40 + +sgemm_kernel_L4_M2_22: + + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L4_M2_22 + + +sgemm_kernel_L4_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L4_M2_100 + +sgemm_kernel_L4_M2_42: + + KERNEL2x4_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L4_M2_42 + +sgemm_kernel_L4_M2_100: + + SAVE2x4 + +sgemm_kernel_L4_M2_END: + + +sgemm_kernel_L4_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble sgemm_kernel_L4_END + +sgemm_kernel_L4_M1_20: + + INIT1x4 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble sgemm_kernel_L4_M1_40 + +sgemm_kernel_L4_M1_22: + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L4_M1_22 + + +sgemm_kernel_L4_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L4_M1_100 + +sgemm_kernel_L4_M1_42: + + KERNEL1x4_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L4_M1_42 + +sgemm_kernel_L4_M1_100: + + SAVE1x4 + +sgemm_kernel_L4_END: + add origPB, origPB, origK, lsl #4 // B = B + K * 4 * 4 + + subs counterJ, counterJ , #1 // j-- + bgt sgemm_kernel_L4_BEGIN + + +/******************************************************************************/ + +sgemm_kernel_L2_BEGIN: // less than 2 left in N direction + + mov counterJ , origN + tst counterJ , #3 + ble sgemm_kernel_L999 + + tst counterJ , #2 + ble sgemm_kernel_L1_BEGIN + + mov pCRow0, pC // pCRow0 = pC + + add pC,pC,LDC, lsl #1 + + mov pA, origPA // pA = A + +sgemm_kernel_L2_M16_BEGIN: + + mov counterI, origM + asr counterI, counterI, #4 // counterI = counterI / 16 + cmp counterI,#0 + ble sgemm_kernel_L2_M8_BEGIN + +sgemm_kernel_L2_M16_20: + + INIT16x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble sgemm_kernel_L2_M16_40 + .align 5 + +sgemm_kernel_L2_M16_22: + KERNEL16x2_SUB + KERNEL16x2_SUB + KERNEL16x2_SUB + KERNEL16x2_SUB + + KERNEL16x2_SUB + KERNEL16x2_SUB + KERNEL16x2_SUB + KERNEL16x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M16_22 + + +sgemm_kernel_L2_M16_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L2_M16_100 + +sgemm_kernel_L2_M16_42: + + KERNEL16x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M16_42 + +sgemm_kernel_L2_M16_100: + + SAVE16x2 + +sgemm_kernel_L2_M16_END: + + subs counterI, counterI, #1 + bgt sgemm_kernel_L2_M16_20 + +//------------------------------------------------------------------------------ + +sgemm_kernel_L2_M8_BEGIN: + mov counterI, origM + tst counterI , #15 + ble sgemm_kernel_L2_END + + tst counterI, #8 + ble sgemm_kernel_L2_M4_BEGIN + +sgemm_kernel_L2_M8_20: + + INIT8x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble sgemm_kernel_L2_M8_40 + .align 5 + +sgemm_kernel_L2_M8_22: + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M8_22 + + +sgemm_kernel_L2_M8_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L2_M8_100 + +sgemm_kernel_L2_M8_42: + + KERNEL8x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M8_42 + +sgemm_kernel_L2_M8_100: + + SAVE8x2 + +sgemm_kernel_L2_M8_END: + +//------------------------------------------------------------------------------ + +sgemm_kernel_L2_M4_BEGIN: + mov counterI, origM + tst counterI , #7 + ble sgemm_kernel_L2_END + + tst counterI, #4 + ble sgemm_kernel_L2_M2_BEGIN + +sgemm_kernel_L2_M4_20: + + INIT4x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble sgemm_kernel_L2_M4_40 + .align 5 + +sgemm_kernel_L2_M4_22: + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M4_22 + + +sgemm_kernel_L2_M4_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L2_M4_100 + +sgemm_kernel_L2_M4_42: + + KERNEL4x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M4_42 + +sgemm_kernel_L2_M4_100: + + SAVE4x2 + +sgemm_kernel_L2_M4_END: + +//------------------------------------------------------------------------------ + + +sgemm_kernel_L2_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble sgemm_kernel_L2_END + + tst counterI, #2 // counterI = counterI / 2 + ble sgemm_kernel_L2_M1_BEGIN + +sgemm_kernel_L2_M2_20: + + INIT2x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble sgemm_kernel_L2_M2_40 + +sgemm_kernel_L2_M2_22: + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M2_22 + + +sgemm_kernel_L2_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L2_M2_100 + +sgemm_kernel_L2_M2_42: + + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M2_42 + +sgemm_kernel_L2_M2_100: + + SAVE2x2 + +sgemm_kernel_L2_M2_END: + + +sgemm_kernel_L2_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble sgemm_kernel_L2_END + +sgemm_kernel_L2_M1_20: + + INIT1x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL, #0 + ble sgemm_kernel_L2_M1_40 + +sgemm_kernel_L2_M1_22: + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M1_22 + + +sgemm_kernel_L2_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L2_M1_100 + +sgemm_kernel_L2_M1_42: + + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L2_M1_42 + +sgemm_kernel_L2_M1_100: + + SAVE1x2 + +sgemm_kernel_L2_END: + + add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4 + +/******************************************************************************/ + +sgemm_kernel_L1_BEGIN: + + mov counterJ , origN + tst counterJ , #1 + ble sgemm_kernel_L999 // done + + + mov pCRow0, pC // pCRow0 = C + add pC , pC , LDC // Update pC to point to next + + mov pA, origPA // pA = A + +sgemm_kernel_L1_M16_BEGIN: + + mov counterI, origM + asr counterI, counterI, #4 // counterI = counterI / 16 + cmp counterI, #0 + ble sgemm_kernel_L1_M8_BEGIN + +sgemm_kernel_L1_M16_20: + + INIT16x1 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble sgemm_kernel_L1_M16_40 + .align 5 + +sgemm_kernel_L1_M16_22: + KERNEL16x1_SUB + KERNEL16x1_SUB + KERNEL16x1_SUB + KERNEL16x1_SUB + + KERNEL16x1_SUB + KERNEL16x1_SUB + KERNEL16x1_SUB + KERNEL16x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M16_22 + + +sgemm_kernel_L1_M16_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L1_M16_100 + +sgemm_kernel_L1_M16_42: + + KERNEL16x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M16_42 + +sgemm_kernel_L1_M16_100: + + SAVE16x1 + +sgemm_kernel_L1_M16_END: + + subs counterI, counterI, #1 + bgt sgemm_kernel_L1_M16_20 + +//------------------------------------------------------------------------------ + +sgemm_kernel_L1_M8_BEGIN: + + mov counterI, origM + tst counterI , #15 + ble sgemm_kernel_L1_END + + tst counterI, #8 + ble sgemm_kernel_L1_M4_BEGIN + +sgemm_kernel_L1_M8_20: + + INIT8x1 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble sgemm_kernel_L1_M8_40 + .align 5 + +sgemm_kernel_L1_M8_22: + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M8_22 + + +sgemm_kernel_L1_M8_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L1_M8_100 + +sgemm_kernel_L1_M8_42: + + KERNEL8x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M8_42 + +sgemm_kernel_L1_M8_100: + + SAVE8x1 + +sgemm_kernel_L1_M8_END: + +//------------------------------------------------------------------------------ + +sgemm_kernel_L1_M4_BEGIN: + mov counterI, origM + tst counterI , #7 + ble sgemm_kernel_L1_END + + tst counterI, #4 + ble sgemm_kernel_L1_M2_BEGIN + +sgemm_kernel_L1_M4_20: + + INIT4x1 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble sgemm_kernel_L1_M4_40 + .align 5 + +sgemm_kernel_L1_M4_22: + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M4_22 + + +sgemm_kernel_L1_M4_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L1_M4_100 + +sgemm_kernel_L1_M4_42: + + KERNEL4x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M4_42 + +sgemm_kernel_L1_M4_100: + + SAVE4x1 + +sgemm_kernel_L1_M4_END: + +//------------------------------------------------------------------------------ + +sgemm_kernel_L1_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble sgemm_kernel_L1_END + + tst counterI, #2 // counterI = counterI / 2 + ble sgemm_kernel_L1_M1_BEGIN + +sgemm_kernel_L1_M2_20: + + INIT2x1 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble sgemm_kernel_L1_M2_40 + +sgemm_kernel_L1_M2_22: + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M2_22 + + +sgemm_kernel_L1_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L1_M2_100 + +sgemm_kernel_L1_M2_42: + + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M2_42 + +sgemm_kernel_L1_M2_100: + + SAVE2x1 + +sgemm_kernel_L1_M2_END: + + +sgemm_kernel_L1_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble sgemm_kernel_L1_END + +sgemm_kernel_L1_M1_20: + + INIT1x1 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble sgemm_kernel_L1_M1_40 + +sgemm_kernel_L1_M1_22: + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M1_22 + + +sgemm_kernel_L1_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble sgemm_kernel_L1_M1_100 + +sgemm_kernel_L1_M1_42: + + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt sgemm_kernel_L1_M1_42 + +sgemm_kernel_L1_M1_100: + + SAVE1x1 + +sgemm_kernel_L1_END: + +sgemm_kernel_L999: + mov x0, #0 // set return value + ldp d8, d9, [sp, #(0 * 16)] + ldp d10, d11, [sp, #(1 * 16)] + ldp d12, d13, [sp, #(2 * 16)] + ldp d14, d15, [sp, #(3 * 16)] + ldp d16, d17, [sp, #(4 * 16)] + ldp x18, x19, [sp, #(5 * 16)] + ldp x20, x21, [sp, #(6 * 16)] + ldp x22, x23, [sp, #(7 * 16)] + ldp x24, x25, [sp, #(8 * 16)] + ldp x26, x27, [sp, #(9 * 16)] + ldr x28, [sp, #(10 * 16)] + add sp, sp, #(11*16) + ret + + EPILOGUE + diff --git a/param.h b/param.h index 62b4db738..1ceccd52a 100644 --- a/param.h +++ b/param.h @@ -2326,17 +2326,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define ZGEMM_DEFAULT_UNROLL_M 4 #define ZGEMM_DEFAULT_UNROLL_N 4 -#define SGEMM_DEFAULT_P 512 +#define SGEMM_DEFAULT_P sgemm_p #define DGEMM_DEFAULT_P dgemm_p #define CGEMM_DEFAULT_P 256 #define ZGEMM_DEFAULT_P 128 -#define SGEMM_DEFAULT_Q 1024 +#define SGEMM_DEFAULT_Q sgemm_q #define DGEMM_DEFAULT_Q dgemm_q #define CGEMM_DEFAULT_Q 512 #define ZGEMM_DEFAULT_Q 512 -#define SGEMM_DEFAULT_R 4096 +#define SGEMM_DEFAULT_R sgemm_r #define DGEMM_DEFAULT_R dgemm_r #define CGEMM_DEFAULT_R 4096 #define ZGEMM_DEFAULT_R 2048 @@ -2482,17 +2482,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define ZGEMM_DEFAULT_UNROLL_M 4 #define ZGEMM_DEFAULT_UNROLL_N 4 -#define SGEMM_DEFAULT_P 512 +#define SGEMM_DEFAULT_P sgemm_p #define DGEMM_DEFAULT_P dgemm_p #define CGEMM_DEFAULT_P 256 #define ZGEMM_DEFAULT_P 128 -#define SGEMM_DEFAULT_Q 1024 +#define SGEMM_DEFAULT_Q sgemm_q #define DGEMM_DEFAULT_Q dgemm_q #define CGEMM_DEFAULT_Q 512 #define ZGEMM_DEFAULT_Q 512 -#define SGEMM_DEFAULT_R 4096 +#define SGEMM_DEFAULT_R sgemm_r #define DGEMM_DEFAULT_R dgemm_r #define CGEMM_DEFAULT_R 4096 #define ZGEMM_DEFAULT_R 2048 From 981064acc6dd370a6cd5b6d48d9b9b639dea9848 Mon Sep 17 00:00:00 2001 From: Ashwin Sekhar T K Date: Tue, 17 Jan 2017 00:28:54 -0800 Subject: [PATCH 03/10] THUNDERX2T99: Add Optimized DAXPY Implementation --- kernel/arm64/KERNEL.THUNDERX2T99 | 2 + kernel/arm64/daxpy_thunderx2t99.S | 196 ++++++++++++++++++++++++++++++ 2 files changed, 198 insertions(+) create mode 100644 kernel/arm64/daxpy_thunderx2t99.S diff --git a/kernel/arm64/KERNEL.THUNDERX2T99 b/kernel/arm64/KERNEL.THUNDERX2T99 index dcfdc96a8..f169956f9 100644 --- a/kernel/arm64/KERNEL.THUNDERX2T99 +++ b/kernel/arm64/KERNEL.THUNDERX2T99 @@ -1,5 +1,7 @@ include $(KERNELDIR)/KERNEL.CORTEXA57 +DAXPYKERNEL=daxpy_thunderx2t99.S + ifeq ($(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N), 8x4) DGEMMKERNEL = dgemm_kernel_8x4_thunderx2t99.S else diff --git a/kernel/arm64/daxpy_thunderx2t99.S b/kernel/arm64/daxpy_thunderx2t99.S new file mode 100644 index 000000000..5eb2ec0c3 --- /dev/null +++ b/kernel/arm64/daxpy_thunderx2t99.S @@ -0,0 +1,196 @@ +/******************************************************************************* +Copyright (c) 2017, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#define ASSEMBLER +#include "common.h" + +#define N x0 /* vector length */ +#define X x3 /* X vector address */ +#define INC_X x4 /* X stride */ +#define Y x5 /* Y vector address */ +#define INC_Y x6 /* Y stride */ +#define I x1 /* loop variable */ + +/******************************************************************************* +* Macro definitions +*******************************************************************************/ + +#define DA d0 /* scale input value */ +#define TMPX d1 +#define TMPVX {v1.d}[0] +#define TMPY d2 +#define TMPVY {v2.d}[0] +#define SZ 8 + +/******************************************************************************/ + +.macro KERNEL_F1 + ldr TMPX, [X], #SZ + ldr TMPY, [Y] + fmadd TMPY, TMPX, DA, TMPY + str TMPY, [Y], #SZ +.endm + +.macro KERNEL_F16 + ldp q4, q5, [X] + ldp q16, q17, [Y] + + ldp q6, q7, [X, #32] + ldp q18, q19, [Y, #32] + + fmla v16.2d, v4.2d, v0.d[0] + fmla v17.2d, v5.2d, v0.d[0] + + PRFM PLDL1KEEP, [X, #896] + PRFM PLDL1KEEP, [Y, #896] + + stp q16, q17, [Y] + + ldp q20, q21, [X, #64] + ldp q24, q25, [Y, #64] + + fmla v18.2d, v6.2d, v0.d[0] + fmla v19.2d, v7.2d, v0.d[0] + + PRFM PLDL1KEEP, [X, #896+64] + PRFM PLDL1KEEP, [Y, #896+64] + + stp q18, q19, [Y, #32] + + ldp q22, q23, [X, #96] + ldp q26, q27, [Y, #96] + + fmla v24.2d, v20.2d, v0.d[0] + fmla v25.2d, v21.2d, v0.d[0] + + stp q24, q25, [Y, #64] + + fmla v26.2d, v22.2d, v0.d[0] + fmla v27.2d, v23.2d, v0.d[0] + + stp q26, q27, [Y, #96] + + add Y, Y, #128 + add X, X, #128 +.endm + +.macro KERNEL_F32 + KERNEL_F16 + KERNEL_F16 +.endm + +.macro INIT_S + lsl INC_X, INC_X, #3 + lsl INC_Y, INC_Y, #3 +.endm + +.macro KERNEL_S1 + ld1 TMPVX, [X], INC_X + ldr TMPY, [Y] + fmadd TMPY, TMPX, DA, TMPY + st1 TMPVY, [Y], INC_Y +.endm + +/******************************************************************************* +* End of macro definitions +*******************************************************************************/ + + PROLOGUE + + cmp N, xzr + ble axpy_kernel_L999 + + fcmp DA, #0.0 + beq axpy_kernel_L999 + + cmp INC_X, #1 + bne axpy_kernel_S_BEGIN + cmp INC_Y, #1 + bne axpy_kernel_S_BEGIN + +axpy_kernel_F_BEGIN: + + asr I, N, #5 + cmp I, xzr + beq axpy_kernel_F1 + + .align 5 +axpy_kernel_F32: + + KERNEL_F32 + + subs I, I, #1 + bne axpy_kernel_F32 + +axpy_kernel_F1: + + ands I, N, #31 + ble axpy_kernel_L999 + +axpy_kernel_F10: + + KERNEL_F1 + + subs I, I, #1 + bne axpy_kernel_F10 + + b axpy_kernel_L999 + +axpy_kernel_S_BEGIN: + + INIT_S + + asr I, N, #2 + cmp I, xzr + ble axpy_kernel_S1 + +axpy_kernel_S4: + + KERNEL_S1 + KERNEL_S1 + KERNEL_S1 + KERNEL_S1 + + subs I, I, #1 + bne axpy_kernel_S4 + +axpy_kernel_S1: + + ands I, N, #3 + ble axpy_kernel_L999 + +axpy_kernel_S10: + + KERNEL_S1 + + subs I, I, #1 + bne axpy_kernel_S10 + +axpy_kernel_L999: + + mov w0, wzr + ret From 0f1d6e8b392ba84303539ca77009528cd5d7a37f Mon Sep 17 00:00:00 2001 From: Ashwin Sekhar T K Date: Mon, 16 Jan 2017 23:16:23 -0800 Subject: [PATCH 04/10] THUNDERX2T99: Improve DGEMM --- kernel/arm64/dgemm_kernel_8x4_thunderx2t99.S | 171 ++++++++----------- 1 file changed, 71 insertions(+), 100 deletions(-) diff --git a/kernel/arm64/dgemm_kernel_8x4_thunderx2t99.S b/kernel/arm64/dgemm_kernel_8x4_thunderx2t99.S index 3dc90254c..86865d825 100644 --- a/kernel/arm64/dgemm_kernel_8x4_thunderx2t99.S +++ b/kernel/arm64/dgemm_kernel_8x4_thunderx2t99.S @@ -151,187 +151,164 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .endm .macro KERNEL8x4_I - ldur q0, [pA] - ldur q1, [pA, #16] - - ldur q8, [pB] - ldur q9, [pB, #16] + ldp q0, q1, [pA] + ldp q8, q9, [pB] + ldp q2, q3, [pA, #32] + ldp q4, q5, [pA, #64] + ldp q12, q13, [pB, #32] + ldp q6, q7, [pA, #96] fmul v16.2d, v0.2d, v8.d[0] fmul v20.2d, v0.2d, v8.d[1] - fmul v17.2d, v1.2d, v8.d[0] fmul v21.2d, v1.2d, v8.d[1] - ldp q2, q3, [pA, #32] - fmul v24.2d, v0.2d, v9.d[0] - ldp q4, q5, [pA, #64] - fmul v28.2d, v0.2d, v9.d[1] + add pA, pA, #128 + add pB, pB, #64 + fmul v24.2d, v0.2d, v9.d[0] + fmul v28.2d, v0.2d, v9.d[1] fmul v25.2d, v1.2d, v9.d[0] fmul v29.2d, v1.2d, v9.d[1] - ldur q12, [pB, #32] - ldur q13, [pB, #48] + prfm PLDL1KEEP, [pA, A_PRE_SIZE] + prfm PLDL1KEEP, [pA, A_PRE_SIZE_64] fmul v18.2d, v2.2d, v8.d[0] fmul v22.2d, v2.2d, v8.d[1] - - ldur q6, [pA, #96] - ldur q7, [pA, #112] - - add pB, pB, #64 - add pA, pA, #128 - fmul v26.2d, v2.2d, v9.d[0] fmul v30.2d, v2.2d, v9.d[1] fmul v19.2d, v3.2d, v8.d[0] fmul v27.2d, v3.2d, v9.d[0] - - prfm PLDL1KEEP, [pA, A_PRE_SIZE] - fmul v31.2d, v3.2d, v9.d[1] fmul v23.2d, v3.2d, v8.d[1] - - prfm PLDL1KEEP, [pA, A_PRE_SIZE_64] .endm .macro KERNEL8x4_M1_M2 + + ldp q12, q13, [pB] + ldp q4, q5, [pA] + ldp q6, q7, [pA, #32] + fmla v16.2d, v0.2d, v8.d[0] fmla v20.2d, v0.2d, v8.d[1] - - ldp q4, q5, [pA] - fmla v24.2d, v0.2d, v9.d[0] fmla v28.2d, v0.2d, v9.d[1] - ldp q12, q13, [pB] + prfm PLDL1KEEP, [pA, A_PRE_SIZE] fmla v17.2d, v1.2d, v8.d[0] fmla v25.2d, v1.2d, v9.d[0] - - prfm PLDL1KEEP, [pA, A_PRE_SIZE_64] - fmla v21.2d, v1.2d, v8.d[1] fmla v29.2d, v1.2d, v9.d[1] + prfm PLDL1KEEP, [pA, A_PRE_SIZE_64] + fmla v18.2d, v2.2d, v8.d[0] fmla v22.2d, v2.2d, v8.d[1] - - prfm PLDL1KEEP, [pA, A_PRE_SIZE] - fmla v26.2d, v2.2d, v9.d[0] fmla v30.2d, v2.2d, v9.d[1] + + prfm PLDL1KEEP, [pA, #3840] + fmla v19.2d, v3.2d, v8.d[0] fmla v23.2d, v3.2d, v8.d[1] - - ldp q6, q7, [pA, #32] - fmla v27.2d, v3.2d, v9.d[0] fmla v31.2d, v3.2d, v9.d[1] + + ldp q8, q9, [pB, #32] + ldp q0, q1, [pA, #64] + ldp q2, q3, [pA, #96] + fmla v16.2d, v4.2d, v12.d[0] fmla v20.2d, v4.2d, v12.d[1] fmla v24.2d, v4.2d, v13.d[0] fmla v28.2d, v4.2d, v13.d[1] - ldp q0, q1, [pA, #64] + prfm PLDL1KEEP, [pB, B_PRE_SIZE] fmla v17.2d, v5.2d, v12.d[0] fmla v25.2d, v5.2d, v13.d[0] - - ldp q8, q9, [pB, #32] - ldp q2, q3, [pA, #96] - fmla v21.2d, v5.2d, v12.d[1] fmla v29.2d, v5.2d, v13.d[1] fmla v18.2d, v6.2d, v12.d[0] fmla v22.2d, v6.2d, v12.d[1] - - prfm PLDL1KEEP, [pB, B_PRE_SIZE] - fmla v26.2d, v6.2d, v13.d[0] fmla v30.2d, v6.2d, v13.d[1] - fmla v19.2d, v7.2d, v12.d[0] - fmla v23.2d, v7.2d, v12.d[1] - add pB, pB, #64 add pA, pA, #128 + fmla v19.2d, v7.2d, v12.d[0] + fmla v23.2d, v7.2d, v12.d[1] fmla v27.2d, v7.2d, v13.d[0] fmla v31.2d, v7.2d, v13.d[1] .endm .macro KERNEL8x4_M1 + ldp q12, q13, [pB] + ldp q4, q5, [pA] + ldp q6, q7, [pA, #32] + fmla v16.2d, v0.2d, v8.d[0] fmla v20.2d, v0.2d, v8.d[1] - - ldp q4, q5, [pA], #32 - fmla v24.2d, v0.2d, v9.d[0] fmla v28.2d, v0.2d, v9.d[1] - ldp q12, q13, [pB] - add pB, pB, #32 + prfm PLDL1KEEP, [pA, A_PRE_SIZE] fmla v17.2d, v1.2d, v8.d[0] fmla v25.2d, v1.2d, v9.d[0] - - prfm PLDL1KEEP, [pA, A_PRE_SIZE_64] - fmla v21.2d, v1.2d, v8.d[1] fmla v29.2d, v1.2d, v9.d[1] + prfm PLDL1KEEP, [pA, A_PRE_SIZE_64] + fmla v18.2d, v2.2d, v8.d[0] fmla v22.2d, v2.2d, v8.d[1] - - prfm PLDL1KEEP, [pA, A_PRE_SIZE] - fmla v26.2d, v2.2d, v9.d[0] fmla v30.2d, v2.2d, v9.d[1] + + add pB, pB, #32 + add pA, pA, #64 + fmla v19.2d, v3.2d, v8.d[0] fmla v23.2d, v3.2d, v8.d[1] - - ldp q6, q7, [pA], #32 - fmla v27.2d, v3.2d, v9.d[0] fmla v31.2d, v3.2d, v9.d[1] .endm .macro KERNEL8x4_M2 + ldp q8, q9, [pB] + ldp q0, q1, [pA] + ldp q2, q3, [pA, #32] + fmla v16.2d, v4.2d, v12.d[0] fmla v20.2d, v4.2d, v12.d[1] fmla v24.2d, v4.2d, v13.d[0] fmla v28.2d, v4.2d, v13.d[1] - ldp q0, q1, [pA], #32 + prfm PLDL1KEEP, [pB, B_PRE_SIZE] fmla v17.2d, v5.2d, v12.d[0] fmla v25.2d, v5.2d, v13.d[0] - - ldp q8, q9, [pB] - add pB, pB, #32 - fmla v21.2d, v5.2d, v12.d[1] fmla v29.2d, v5.2d, v13.d[1] fmla v18.2d, v6.2d, v12.d[0] fmla v22.2d, v6.2d, v12.d[1] - - prfm PLDL1KEEP, [pB, B_PRE_SIZE] - fmla v26.2d, v6.2d, v13.d[0] fmla v30.2d, v6.2d, v13.d[1] + add pB, pB, #32 + add pA, pA, #64 + fmla v19.2d, v7.2d, v12.d[0] fmla v23.2d, v7.2d, v12.d[1] - - ldp q2, q3, [pA], #32 - fmla v27.2d, v7.2d, v13.d[0] fmla v31.2d, v7.2d, v13.d[1] .endm @@ -342,13 +319,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. fmla v24.2d, v4.2d, v13.d[0] fmla v28.2d, v4.2d, v13.d[1] + prfm PLDL1KEEP, [pB, B_PRE_SIZE] + fmla v17.2d, v5.2d, v12.d[0] fmla v25.2d, v5.2d, v13.d[0] fmla v21.2d, v5.2d, v12.d[1] fmla v29.2d, v5.2d, v13.d[1] - prfm PLDL1KEEP, [pB, B_PRE_SIZE] - fmla v18.2d, v6.2d, v12.d[0] fmla v22.2d, v6.2d, v12.d[1] fmla v26.2d, v6.2d, v13.d[0] @@ -361,42 +338,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .endm .macro KERNEL8x4_SUB - ldp q0, q1, [pA], #32 - - ldur q8, [pB] - - fmla v16.2d, v0.2d, v8.d[0] - fmla v20.2d, v0.2d, v8.d[1] - - ldur q9, [pB, #16] - add pB, pB, #32 - - fmla v17.2d, v1.2d, v8.d[0] - fmla v21.2d, v1.2d, v8.d[1] - - ldp q2, q3, [pA], #32 - - fmla v24.2d, v0.2d, v9.d[0] - fmla v28.2d, v0.2d, v9.d[1] - - fmla v25.2d, v1.2d, v9.d[0] - fmla v29.2d, v1.2d, v9.d[1] + ldp q0, q1, [pA] + ldp q8, q9, [pB] + ldp q2, q3, [pA, #32] prfm PLDL1KEEP, [pA, A_PRE_SIZE] - fmla v18.2d, v2.2d, v8.d[0] - fmla v22.2d, v2.2d, v8.d[1] + fmla v16.2d, v0.2d, v8.d[0] + fmla v20.2d, v0.2d, v8.d[1] + fmla v17.2d, v1.2d, v8.d[0] + fmla v21.2d, v1.2d, v8.d[1] prfm PLDL1KEEP, [pA, A_PRE_SIZE_64] - fmla v26.2d, v2.2d, v9.d[0] - fmla v30.2d, v2.2d, v9.d[1] + fmla v24.2d, v0.2d, v9.d[0] + fmla v28.2d, v0.2d, v9.d[1] + fmla v25.2d, v1.2d, v9.d[0] + fmla v29.2d, v1.2d, v9.d[1] prfm PLDL1KEEP, [pB, B_PRE_SIZE] + fmla v18.2d, v2.2d, v8.d[0] + fmla v22.2d, v2.2d, v8.d[1] + fmla v26.2d, v2.2d, v9.d[0] + fmla v30.2d, v2.2d, v9.d[1] + + add pB, pB, #32 + add pA, pA, #64 + fmla v19.2d, v3.2d, v8.d[0] fmla v27.2d, v3.2d, v9.d[0] - fmla v31.2d, v3.2d, v9.d[1] fmla v23.2d, v3.2d, v8.d[1] .endm From f33fcedb30937cbf6f31492c1136559e4646203c Mon Sep 17 00:00:00 2001 From: Ashwin Sekhar T K Date: Wed, 18 Jan 2017 00:57:11 -0800 Subject: [PATCH 05/10] THUNDERX2T99: Improve SGEMM --- kernel/arm64/sgemm_kernel_16x4_thunderx2t99.S | 100 ++++++++---------- 1 file changed, 47 insertions(+), 53 deletions(-) diff --git a/kernel/arm64/sgemm_kernel_16x4_thunderx2t99.S b/kernel/arm64/sgemm_kernel_16x4_thunderx2t99.S index 06b48f1a9..0ee10e130 100644 --- a/kernel/arm64/sgemm_kernel_16x4_thunderx2t99.S +++ b/kernel/arm64/sgemm_kernel_16x4_thunderx2t99.S @@ -151,15 +151,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ldur q0, [pA] ldur q1, [pA, #16] - ldur d8, [pB] + ldur q8, [pB] fmul v16.4s, v0.4s, v8.s[0] fmul v20.4s, v0.4s, v8.s[1] - ldur d10, [pB, #8] - - fmul v24.4s, v0.4s, v10.s[0] - fmul v28.4s, v0.4s, v10.s[1] + fmul v24.4s, v0.4s, v8.s[2] + fmul v28.4s, v0.4s, v8.s[3] ldur q2, [pA, #32] ldur q3, [pA, #48] @@ -170,31 +168,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ldur q4, [pA, #64] ldur q5, [pA, #80] - fmul v25.4s, v1.4s, v10.s[0] - fmul v29.4s, v1.4s, v10.s[1] + fmul v25.4s, v1.4s, v8.s[2] + fmul v29.4s, v1.4s, v8.s[3] - ldur d12, [pB, #16] + ldur q12, [pB, #16] fmul v18.4s, v2.4s, v8.s[0] fmul v22.4s, v2.4s, v8.s[1] - ldur d14, [pB, #24] - add pB, pB, #32 - fmul v19.4s, v3.4s, v8.s[0] fmul v23.4s, v3.4s, v8.s[1] ldur q6, [pA, #96] ldur q7, [pA, #112] + + add pB, pB, #32 add pA, pA, #128 - fmul v26.4s, v2.4s, v10.s[0] - fmul v30.4s, v2.4s, v10.s[1] + fmul v26.4s, v2.4s, v8.s[2] + fmul v30.4s, v2.4s, v8.s[3] prfm PLDL1KEEP, [pA, #A_PRE_SIZE] - fmul v27.4s, v3.4s, v10.s[0] - fmul v31.4s, v3.4s, v10.s[1] + fmul v27.4s, v3.4s, v8.s[2] + fmul v31.4s, v3.4s, v8.s[3] prfm PLDL1KEEP, [pA, #A_PRE_SIZE+64] .endm @@ -212,33 +209,32 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. fmla v20.4s, v0.4s, v8.s[1] fmla v21.4s, v1.4s, v8.s[1] - ldur d12, [pB] + ldur q12, [pB] fmla v22.4s, v2.4s, v8.s[1] fmla v23.4s, v3.4s, v8.s[1] - ldur d14, [pB, #8] add pB, pB, #16 - fmla v24.4s, v0.4s, v10.s[0] - fmla v25.4s, v1.4s, v10.s[0] + fmla v24.4s, v0.4s, v8.s[2] + fmla v25.4s, v1.4s, v8.s[2] prfm PLDL1KEEP, [pA, #A_PRE_SIZE+64] - fmla v26.4s, v2.4s, v10.s[0] - fmla v27.4s, v3.4s, v10.s[0] + fmla v26.4s, v2.4s, v8.s[2] + fmla v27.4s, v3.4s, v8.s[2] prfm PLDL1KEEP, [pA, #A_PRE_SIZE] - fmla v28.4s, v0.4s, v10.s[1] - fmla v29.4s, v1.4s, v10.s[1] + fmla v28.4s, v0.4s, v8.s[3] + fmla v29.4s, v1.4s, v8.s[3] ldur q6, [pA, #32] ldur q7, [pA, #48] add pA, pA, #64 - fmla v30.4s, v2.4s, v10.s[1] - fmla v31.4s, v3.4s, v10.s[1] + fmla v30.4s, v2.4s, v8.s[3] + fmla v31.4s, v3.4s, v8.s[3] .endm .macro KERNEL16x4_M2 @@ -254,70 +250,68 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. fmla v20.4s, v4.4s, v12.s[1] fmla v21.4s, v5.4s, v12.s[1] - ldur d8, [pB] + ldur q8, [pB] fmla v22.4s, v6.4s, v12.s[1] fmla v23.4s, v7.4s, v12.s[1] - ldur d10, [pB, #8] add pB, pB, #16 - fmla v24.4s, v4.4s, v14.s[0] - fmla v25.4s, v5.4s, v14.s[0] + fmla v24.4s, v4.4s, v12.s[2] + fmla v25.4s, v5.4s, v12.s[2] prfm PLDL1KEEP, [pB, #B_PRE_SIZE] - fmla v26.4s, v6.4s, v14.s[0] - fmla v27.4s, v7.4s, v14.s[0] + fmla v26.4s, v6.4s, v12.s[2] + fmla v27.4s, v7.4s, v12.s[2] ldur q2, [pA, #32] ldur q3, [pA, #48] add pA, pA, #64 - fmla v28.4s, v4.4s, v14.s[1] - fmla v29.4s, v5.4s, v14.s[1] + fmla v28.4s, v4.4s, v12.s[3] + fmla v29.4s, v5.4s, v12.s[3] - fmla v30.4s, v6.4s, v14.s[1] - fmla v31.4s, v7.4s, v14.s[1] + fmla v30.4s, v6.4s, v12.s[3] + fmla v31.4s, v7.4s, v12.s[3] .endm .macro KERNEL16x4_E fmla v16.4s, v4.4s, v12.s[0] fmla v20.4s, v4.4s, v12.s[1] - fmla v24.4s, v4.4s, v14.s[0] - fmla v28.4s, v4.4s, v14.s[1] + fmla v24.4s, v4.4s, v12.s[2] + fmla v28.4s, v4.4s, v12.s[3] fmla v17.4s, v5.4s, v12.s[0] fmla v21.4s, v5.4s, v12.s[1] - fmla v25.4s, v5.4s, v14.s[0] - fmla v29.4s, v5.4s, v14.s[1] + fmla v25.4s, v5.4s, v12.s[2] + fmla v29.4s, v5.4s, v12.s[3] prfm PLDL1KEEP, [pB, #B_PRE_SIZE] fmla v18.4s, v6.4s, v12.s[0] fmla v22.4s, v6.4s, v12.s[1] - fmla v26.4s, v6.4s, v14.s[0] - fmla v30.4s, v6.4s, v14.s[1] + fmla v26.4s, v6.4s, v12.s[2] + fmla v30.4s, v6.4s, v12.s[3] fmla v19.4s, v7.4s, v12.s[0] fmla v23.4s, v7.4s, v12.s[1] - fmla v27.4s, v7.4s, v14.s[0] - fmla v31.4s, v7.4s, v14.s[1] + fmla v27.4s, v7.4s, v12.s[2] + fmla v31.4s, v7.4s, v12.s[3] .endm .macro KERNEL16x4_SUB ldur q0, [pA] ldur q1, [pA, #16] - ldur d8, [pB] + ldur q8, [pB] fmla v16.4s, v0.4s, v8.s[0] fmla v20.4s, v0.4s, v8.s[1] - ldur d10, [pB, #8] add pB, pB, #16 - fmla v24.4s, v0.4s, v10.s[0] - fmla v28.4s, v0.4s, v10.s[1] + fmla v24.4s, v0.4s, v8.s[2] + fmla v28.4s, v0.4s, v8.s[3] ldur q2, [pA, #32] ldur q3, [pA, #48] @@ -326,8 +320,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. fmla v17.4s, v1.4s, v8.s[0] fmla v21.4s, v1.4s, v8.s[1] - fmla v25.4s, v1.4s, v10.s[0] - fmla v29.4s, v1.4s, v10.s[1] + fmla v25.4s, v1.4s, v8.s[2] + fmla v29.4s, v1.4s, v8.s[3] fmla v18.4s, v2.4s, v8.s[0] fmla v22.4s, v2.4s, v8.s[1] @@ -337,13 +331,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. fmla v19.4s, v3.4s, v8.s[0] fmla v23.4s, v3.4s, v8.s[1] - fmla v26.4s, v2.4s, v10.s[0] - fmla v30.4s, v2.4s, v10.s[1] + fmla v26.4s, v2.4s, v8.s[2] + fmla v30.4s, v2.4s, v8.s[3] prfm PLDL1KEEP, [pB, #B_PRE_SIZE] - fmla v27.4s, v3.4s, v10.s[0] - fmla v31.4s, v3.4s, v10.s[1] + fmla v27.4s, v3.4s, v8.s[2] + fmla v31.4s, v3.4s, v8.s[3] .endm .macro SAVE16x4 From 0c07003ccf0a8352cbc5fe50a7586530d522b9a1 Mon Sep 17 00:00:00 2001 From: Ashwin Sekhar T K Date: Thu, 19 Jan 2017 10:53:48 +0530 Subject: [PATCH 06/10] THUNDERX2T99: Add Optimized DDOT Implementation --- kernel/arm64/KERNEL.THUNDERX2T99 | 2 + kernel/arm64/ddot_thunderx2t99.S | 207 +++++++++++++++++++++++++++++++ 2 files changed, 209 insertions(+) create mode 100644 kernel/arm64/ddot_thunderx2t99.S diff --git a/kernel/arm64/KERNEL.THUNDERX2T99 b/kernel/arm64/KERNEL.THUNDERX2T99 index f169956f9..b6143b25f 100644 --- a/kernel/arm64/KERNEL.THUNDERX2T99 +++ b/kernel/arm64/KERNEL.THUNDERX2T99 @@ -2,6 +2,8 @@ include $(KERNELDIR)/KERNEL.CORTEXA57 DAXPYKERNEL=daxpy_thunderx2t99.S +DDOTKERNEL=ddot_thunderx2t99.S + ifeq ($(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N), 8x4) DGEMMKERNEL = dgemm_kernel_8x4_thunderx2t99.S else diff --git a/kernel/arm64/ddot_thunderx2t99.S b/kernel/arm64/ddot_thunderx2t99.S new file mode 100644 index 000000000..5fa39adf6 --- /dev/null +++ b/kernel/arm64/ddot_thunderx2t99.S @@ -0,0 +1,207 @@ +/******************************************************************************* +Copyright (c) 2017, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#define ASSEMBLER +#include "common.h" + +#define N x0 /* vector length */ +#define X x1 /* X vector address */ +#define INC_X x2 /* X stride */ +#define Y x3 /* Y vector address */ +#define INC_Y x4 /* Y stride */ +#define I x5 /* loop variable */ + +/******************************************************************************* +* Macro definitions +*******************************************************************************/ + +#define REG0 xzr +#define DOTF d0 +#define TMPX d16 +#define LD1VX {v16.d}[0] +#define TMPY d24 +#define LD1VY {v24.d}[0] +#define SZ 8 + +/******************************************************************************/ + +.macro KERNEL_F1 + ldr TMPX, [X] + ldr TMPY, [Y] + add X, X, #SZ + add Y, Y, #SZ + fmadd DOTF, TMPX, TMPY, DOTF +.endm + +.macro KERNEL_F16 + ldp q16, q17, [X] + ldp q24, q25, [Y] + + ldp q18, q19, [X, #32] + ldp q26, q27, [Y, #32] + + fmla v0.2d, v16.2d, v24.2d + fmla v1.2d, v17.2d, v25.2d + + ldp q20, q21, [X, #64] + ldp q28, q29, [Y, #64] + + fmla v2.2d, v18.2d, v26.2d + fmla v3.2d, v19.2d, v27.2d + + ldp q22, q23, [X, #96] + ldp q30, q31, [Y, #96] + + add Y, Y, #128 + add X, X, #128 + + fmla v4.2d, v20.2d, v28.2d + fmla v5.2d, v21.2d, v29.2d + + PRFM PLDL1KEEP, [X, #896] + PRFM PLDL1KEEP, [Y, #896] + PRFM PLDL1KEEP, [X, #896+64] + PRFM PLDL1KEEP, [Y, #896+64] + + fmla v6.2d, v22.2d, v30.2d + fmla v7.2d, v23.2d, v31.2d +.endm + +.macro KERNEL_F32 + KERNEL_F16 + KERNEL_F16 +.endm + +.macro KERNEL_F32_FINALIZE + fadd v0.2d, v0.2d, v1.2d + fadd v2.2d, v2.2d, v3.2d + fadd v4.2d, v4.2d, v5.2d + fadd v6.2d, v6.2d, v7.2d + fadd v0.2d, v0.2d, v2.2d + fadd v4.2d, v4.2d, v6.2d + fadd v0.2d, v0.2d, v4.2d + faddp DOTF, v0.2d +.endm + +.macro INIT_S + lsl INC_X, INC_X, #3 + lsl INC_Y, INC_Y, #3 +.endm + +.macro KERNEL_S1 + ld1 LD1VX, [X], INC_X + ld1 LD1VY, [Y], INC_Y + fmadd DOTF, TMPX, TMPY, DOTF +.endm + +/******************************************************************************* +* End of macro definitions +*******************************************************************************/ + + PROLOGUE + + fmov DOTF, REG0 + fmov d1, REG0 + fmov d2, REG0 + fmov d3, REG0 + fmov d4, REG0 + fmov d5, REG0 + fmov d6, REG0 + fmov d7, REG0 + + cmp N, xzr + ble dot_kernel_L999 + + cmp INC_X, #1 + bne dot_kernel_S_BEGIN + cmp INC_Y, #1 + bne dot_kernel_S_BEGIN + +dot_kernel_F_BEGIN: + + asr I, N, #5 + cmp I, xzr + beq dot_kernel_F1 + +dot_kernel_F32: + + KERNEL_F32 + + subs I, I, #1 + bne dot_kernel_F32 + + KERNEL_F32_FINALIZE + +dot_kernel_F1: + + ands I, N, #31 + ble dot_kernel_L999 + +dot_kernel_F10: + + KERNEL_F1 + + subs I, I, #1 + bne dot_kernel_F10 + + ret + +dot_kernel_S_BEGIN: + + INIT_S + + asr I, N, #2 + cmp I, xzr + ble dot_kernel_S1 + +dot_kernel_S4: + + KERNEL_S1 + KERNEL_S1 + KERNEL_S1 + KERNEL_S1 + + subs I, I, #1 + bne dot_kernel_S4 + +dot_kernel_S1: + + ands I, N, #3 + ble dot_kernel_L999 + +dot_kernel_S10: + + KERNEL_S1 + + subs I, I, #1 + bne dot_kernel_S10 + +dot_kernel_L999: + + ret + + EPILOGUE From d0a79ca6e03528d84316b11e6edd8755dcfdf7be Mon Sep 17 00:00:00 2001 From: Ashwin Sekhar T K Date: Thu, 19 Jan 2017 10:56:17 +0530 Subject: [PATCH 07/10] THUNDERX2T99: Add threaded DDOT Implementation --- kernel/arm64/KERNEL.THUNDERX2T99 | 4 + kernel/arm64/ddot_thunderx2t99.c | 258 +++++++++++++++++++++++++++++++ 2 files changed, 262 insertions(+) create mode 100644 kernel/arm64/ddot_thunderx2t99.c diff --git a/kernel/arm64/KERNEL.THUNDERX2T99 b/kernel/arm64/KERNEL.THUNDERX2T99 index b6143b25f..c9d3132fb 100644 --- a/kernel/arm64/KERNEL.THUNDERX2T99 +++ b/kernel/arm64/KERNEL.THUNDERX2T99 @@ -2,7 +2,11 @@ include $(KERNELDIR)/KERNEL.CORTEXA57 DAXPYKERNEL=daxpy_thunderx2t99.S +ifndef SMP DDOTKERNEL=ddot_thunderx2t99.S +else +DDOTKERNEL=ddot_thunderx2t99.c +endif ifeq ($(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N), 8x4) DGEMMKERNEL = dgemm_kernel_8x4_thunderx2t99.S diff --git a/kernel/arm64/ddot_thunderx2t99.c b/kernel/arm64/ddot_thunderx2t99.c new file mode 100644 index 000000000..36a8d6c21 --- /dev/null +++ b/kernel/arm64/ddot_thunderx2t99.c @@ -0,0 +1,258 @@ +/*************************************************************************** +Copyright (c) 2017, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + + +#include "common.h" + +#include + +#define N "x0" /* vector length */ +#define X "x1" /* "X" vector address */ +#define INC_X "x2" /* "X" stride */ +#define Y "x3" /* "Y" vector address */ +#define INC_Y "x4" /* "Y" stride */ +#define J "x5" /* loop variable */ + +#define REG0 "xzr" +#define DOTF "d0" +#define TMPX "d16" +#define LD1VX "{v16.d}[0]" +#define TMPY "d24" +#define LD1VY "{v24.d}[0]" +#define SZ "8" + +extern int blas_level1_thread_with_return_value(int mode, BLASLONG m, BLASLONG n, + BLASLONG k, void *alpha, void *a, BLASLONG lda, void *b, BLASLONG ldb, + void *c, BLASLONG ldc, int (*function)(), int nthreads); + + +static FLOAT ddot_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y) +{ + FLOAT dot = 0.0 ; + + if ( n < 0 ) return(dot); + + __asm__ __volatile__ ( + " mov "N", %[N_] \n" + " mov "X", %[X_] \n" + " mov "INC_X", %[INCX_] \n" + " mov "Y", %[Y_] \n" + " mov "INC_Y", %[INCY_] \n" + " fmov "DOTF", "REG0" \n" + " fmov d1, "REG0" \n" + " fmov d2, "REG0" \n" + " fmov d3, "REG0" \n" + " fmov d4, "REG0" \n" + " fmov d5, "REG0" \n" + " fmov d6, "REG0" \n" + " fmov d7, "REG0" \n" + + " cmp "N", xzr \n" + " ble 9f //dot_kernel_L999 \n" + + " cmp "INC_X", #1 \n" + " bne 5f //dot_kernel_S_BEGIN \n" + " cmp "INC_Y", #1 \n" + " bne 5f //dot_kernel_S_BEGIN \n" + + "1: //dot_kernel_F_BEGIN \n" + " asr "J", "N", #5 \n" + " cmp "J", xzr \n" + " beq 3f //dot_kernel_F1 \n" + + " .align 5 \n" + "2: //dot_kernel_F32 \n" + " ldp q16, q17, ["X"] \n" + " ldp q24, q25, ["Y"] \n" + " ldp q18, q19, ["X", #32] \n" + " ldp q26, q27, ["Y", #32] \n" + " fmla v0.2d, v16.2d, v24.2d \n" + " fmla v1.2d, v17.2d, v25.2d \n" + " ldp q20, q21, ["X", #64] \n" + " ldp q28, q29, ["Y", #64] \n" + " fmla v2.2d, v18.2d, v26.2d \n" + " fmla v3.2d, v19.2d, v27.2d \n" + " ldp q22, q23, ["X", #96] \n" + " ldp q30, q31, ["Y", #96] \n" + " add "Y", "Y", #128 \n" + " add "X", "X", #128 \n" + " fmla v4.2d, v20.2d, v28.2d \n" + " fmla v5.2d, v21.2d, v29.2d \n" + " PRFM PLDL1KEEP, ["X", #896] \n" + " PRFM PLDL1KEEP, ["Y", #896] \n" + " PRFM PLDL1KEEP, ["X", #896+64] \n" + " PRFM PLDL1KEEP, ["Y", #896+64] \n" + " fmla v6.2d, v22.2d, v30.2d \n" + " fmla v7.2d, v23.2d, v31.2d \n" + + " ldp q16, q17, ["X"] \n" + " ldp q24, q25, ["Y"] \n" + " ldp q18, q19, ["X", #32] \n" + " ldp q26, q27, ["Y", #32] \n" + " fmla v0.2d, v16.2d, v24.2d \n" + " fmla v1.2d, v17.2d, v25.2d \n" + " ldp q20, q21, ["X", #64] \n" + " ldp q28, q29, ["Y", #64] \n" + " fmla v2.2d, v18.2d, v26.2d \n" + " fmla v3.2d, v19.2d, v27.2d \n" + " ldp q22, q23, ["X", #96] \n" + " ldp q30, q31, ["Y", #96] \n" + " add "Y", "Y", #128 \n" + " add "X", "X", #128 \n" + " fmla v4.2d, v20.2d, v28.2d \n" + " fmla v5.2d, v21.2d, v29.2d \n" + " PRFM PLDL1KEEP, ["X", #896] \n" + " PRFM PLDL1KEEP, ["Y", #896] \n" + " PRFM PLDL1KEEP, ["X", #896+64] \n" + " PRFM PLDL1KEEP, ["Y", #896+64] \n" + " fmla v6.2d, v22.2d, v30.2d \n" + " fmla v7.2d, v23.2d, v31.2d \n" + + " subs "J", "J", #1 \n" + " bne 2b //dot_kernel_F32 \n" + + " fadd v0.2d, v0.2d, v1.2d \n" + " fadd v2.2d, v2.2d, v3.2d \n" + " fadd v4.2d, v4.2d, v5.2d \n" + " fadd v6.2d, v6.2d, v7.2d \n" + " fadd v0.2d, v0.2d, v2.2d \n" + " fadd v4.2d, v4.2d, v6.2d \n" + " fadd v0.2d, v0.2d, v4.2d \n" + " faddp "DOTF", v0.2d \n" + + "3: //dot_kernel_F1 \n" + " ands "J", "N", #31 \n" + " ble 9f //dot_kernel_L999 \n" + + "4: //dot_kernel_F10 \n" + " ldr "TMPX", ["X"] \n" + " ldr "TMPY", ["Y"] \n" + " add "X", "X", #"SZ" \n" + " add "Y", "Y", #"SZ" \n" + " fmadd "DOTF", "TMPX", "TMPY", "DOTF" \n" + + " subs "J", "J", #1 \n" + " bne 4b //dot_kernel_F10 \n" + + " b 9f //dot_kernel_L999 \n" + + "5: //dot_kernel_S_BEGIN \n" + " lsl "INC_X", "INC_X", #3 \n" + " lsl "INC_Y", "INC_Y", #3 \n" + " asr "J", "N", #2 \n" + " cmp "J", xzr \n" + " ble 7f //dot_kernel_S1 \n" + + "6: //dot_kernel_S4: \n" + " ld1 "LD1VX", ["X"], "INC_X" \n" + " ld1 "LD1VY", ["Y"], "INC_Y" \n" + " fmadd "DOTF", "TMPX", "TMPY", "DOTF" \n" + " ld1 "LD1VX", ["X"], "INC_X" \n" + " ld1 "LD1VY", ["Y"], "INC_Y" \n" + " fmadd "DOTF", "TMPX", "TMPY", "DOTF" \n" + " ld1 "LD1VX", ["X"], "INC_X" \n" + " ld1 "LD1VY", ["Y"], "INC_Y" \n" + " fmadd "DOTF", "TMPX", "TMPY", "DOTF" \n" + " ld1 "LD1VX", ["X"], "INC_X" \n" + " ld1 "LD1VY", ["Y"], "INC_Y" \n" + " fmadd "DOTF", "TMPX", "TMPY", "DOTF" \n" + " subs "J", "J", #1 \n" + " bne 6b //dot_kernel_S4 \n" + + "7: //dot_kernel_S1: \n" + " ands "J", "N", #3 \n" + " ble 9f //dot_kernel_L999 \n" + + "8: //dot_kernel_S10 \n" + " ld1 "LD1VX", ["X"], "INC_X" \n" + " ld1 "LD1VY", ["Y"], "INC_Y" \n" + " fmadd "DOTF", "TMPX", "TMPY", "DOTF" \n" + " subs "J", "J", #1 \n" + " bne 8b //dot_kernel_S10 \n" + + "9: //dot_kernel_L999 \n" + " fmov %[DOT_], "DOTF" \n" + : [DOT_] "=r" (dot) //%0 + : [N_] "r" (n), //%1 + [X_] "r" (x), //%2 + [INCX_] "r" (inc_x), //%3 + [Y_] "r" (y), //%4 + [INCY_] "r" (inc_y) //%5 + : "cc", + "memory", + "x0", "x1", "x2", "x3", "x4", "x5", + "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7" + ); + + return(dot); +} + +static int ddot_thread_function(BLASLONG n, BLASLONG dummy0, + BLASLONG dummy1, FLOAT dummy2, FLOAT *x, BLASLONG inc_x, FLOAT *y, + BLASLONG inc_y, FLOAT *result, BLASLONG dummy3) +{ + *result = ddot_compute(n, x, inc_x, y, inc_y); + + return 0; +} + +FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y) +{ + int nthreads; + FLOAT dot = 0.0; + FLOAT dummy_alpha; + + nthreads = num_cpu_avail(1); + + if (inc_x == 0 || inc_y == 0) + nthreads = 1; + + if (n <= 10000) + nthreads = 1; + + if (nthreads == 1) { + dot = ddot_compute(n, x, inc_x, y, inc_y); + } else { + int mode, i; + char result[MAX_CPU_NUMBER * sizeof(double) * 2]; + FLOAT *ptr; + + mode = BLAS_DOUBLE | BLAS_REAL; + + blas_level1_thread_with_return_value(mode, n, 0, 0, &dummy_alpha, + x, inc_x, y, inc_y, result, 0, + ( void *)ddot_thread_function, nthreads); + + ptr = (FLOAT *)result; + for (i = 0; i < nthreads; i++) { + dot = dot + (*ptr); + ptr = (FLOAT *)(((char *)ptr) + sizeof(double) * 2); + } + } + + return dot; +} From 01e1d853394dc7efce01f6b8483af82775b9eb7c Mon Sep 17 00:00:00 2001 From: Ashwin Sekhar T K Date: Wed, 18 Jan 2017 00:39:04 -0800 Subject: [PATCH 08/10] Update .gitignore --- .gitignore | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.gitignore b/.gitignore index fa2b91d9b..c172e25e9 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,20 @@ lapack-netlib/make.inc lapack-netlib/lapacke/include/lapacke_mangling.h lapack-netlib/TESTING/testing_results.txt lapack-netlib/INSTALL/test* +lapack-netlib/TESTING/xeigtstc +lapack-netlib/TESTING/xeigtstd +lapack-netlib/TESTING/xeigtsts +lapack-netlib/TESTING/xeigtstz +lapack-netlib/TESTING/xlintstc +lapack-netlib/TESTING/xlintstd +lapack-netlib/TESTING/xlintstds +lapack-netlib/TESTING/xlintstrfc +lapack-netlib/TESTING/xlintstrfd +lapack-netlib/TESTING/xlintstrfs +lapack-netlib/TESTING/xlintstrfz +lapack-netlib/TESTING/xlintsts +lapack-netlib/TESTING/xlintstz +lapack-netlib/TESTING/xlintstzc *.so *.so.* *.a @@ -70,3 +84,5 @@ test/zblat3 build build.* *.swp +benchmark/*.goto + From ca0b36b0121947d8ac3f4a5966f04789231630e4 Mon Sep 17 00:00:00 2001 From: Ashwin Sekhar T K Date: Thu, 19 Jan 2017 00:57:02 -0800 Subject: [PATCH 09/10] THUNDERX2T99: Add Optimized SNRM2 Implementation --- benchmark/nrm2.c | 7 +- kernel/arm64/KERNEL.THUNDERX2T99 | 11 +- kernel/arm64/snrm2_thunderx2t99.S | 228 ++++++++++++++++++++++++++++++ 3 files changed, 239 insertions(+), 7 deletions(-) create mode 100644 kernel/arm64/snrm2_thunderx2t99.S diff --git a/benchmark/nrm2.c b/benchmark/nrm2.c index 691f28c34..d3718f9e0 100644 --- a/benchmark/nrm2.c +++ b/benchmark/nrm2.c @@ -149,7 +149,7 @@ int main(int argc, char *argv[]){ srandom(getpid()); #endif - fprintf(stderr, " SIZE Time\n"); + fprintf(stderr, " SIZE Flops\n"); for(m = from; m <= to; m += step) { @@ -180,7 +180,10 @@ int main(int argc, char *argv[]){ timeg /= loops; - fprintf(stderr, " %10.6f secs\n", timeg); + fprintf(stderr, + " %10.2f MFlops %10.6f sec\n", + COMPSIZE * COMPSIZE * 2. * (double)m / timeg * 1.e-6, timeg); + } diff --git a/kernel/arm64/KERNEL.THUNDERX2T99 b/kernel/arm64/KERNEL.THUNDERX2T99 index c9d3132fb..b604a7e39 100644 --- a/kernel/arm64/KERNEL.THUNDERX2T99 +++ b/kernel/arm64/KERNEL.THUNDERX2T99 @@ -1,17 +1,18 @@ include $(KERNELDIR)/KERNEL.CORTEXA57 -DAXPYKERNEL=daxpy_thunderx2t99.S +SNRM2KERNEL = snrm2_thunderx2t99.S +DAXPYKERNEL = daxpy_thunderx2t99.S ifndef SMP -DDOTKERNEL=ddot_thunderx2t99.S +DDOTKERNEL = ddot_thunderx2t99.S else -DDOTKERNEL=ddot_thunderx2t99.c +DDOTKERNEL = ddot_thunderx2t99.c endif ifeq ($(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N), 8x4) -DGEMMKERNEL = dgemm_kernel_8x4_thunderx2t99.S +DGEMMKERNEL = dgemm_kernel_8x4_thunderx2t99.S else -DGEMMKERNEL = dgemm_kernel_$(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N).S +DGEMMKERNEL = dgemm_kernel_$(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N).S endif ifeq ($(SGEMM_UNROLL_M)x$(SGEMM_UNROLL_N), 16x4) diff --git a/kernel/arm64/snrm2_thunderx2t99.S b/kernel/arm64/snrm2_thunderx2t99.S new file mode 100644 index 000000000..d69441d08 --- /dev/null +++ b/kernel/arm64/snrm2_thunderx2t99.S @@ -0,0 +1,228 @@ +/******************************************************************************* +Copyright (c) 2017, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#define ASSEMBLER +#include "common.h" + +#define N x0 /* vector length */ +#define X x1 /* X vector address */ +#define INC_X x2 /* X stride */ +#define I x5 /* loop variable */ + +/******************************************************************************* +* Macro definitions +*******************************************************************************/ + +#define TMPF s16 +#define TMPFD d17 +#define SSQ s0 +#define SSQD d0 +#define TMPVF {v16.s}[0] +#define TMPVFD {v17.s}[0] +#define SZ 4 + +/******************************************************************************/ + +.macro INIT + fmov SSQD, xzr + fmov d1, xzr + fmov d2, xzr + fmov d3, xzr + fmov d4, xzr + fmov d5, xzr + fmov d6, xzr + fmov d7, xzr +.endm + +.macro KERNEL_F1 + ldr TMPF, [X], #SZ + fcvt TMPFD, TMPF + fmadd SSQD, TMPFD, TMPFD, SSQD +.endm + +.macro KERNEL_F32 + ldur q16, [X] + ldur q18, [X, #16] + ldur q20, [X, #32] + ldur q22, [X, #48] + ldur q24, [X, #64] + ldur q26, [X, #80] + ldur q28, [X, #96] + ldur q30, [X, #112] + + add X, X, #128 + + fcvtl2 v17.2d, v16.4s + fcvtl v16.2d, v16.2s + fcvtl2 v19.2d, v18.4s + fcvtl v18.2d, v18.2s + fcvtl2 v21.2d, v20.4s + fcvtl v20.2d, v20.2s + fcvtl2 v23.2d, v22.4s + fcvtl v22.2d, v22.2s + fcvtl2 v25.2d, v24.4s + fcvtl v24.2d, v24.2s + fcvtl2 v27.2d, v26.4s + fcvtl v26.2d, v26.2s + fcvtl2 v29.2d, v28.4s + fcvtl v28.2d, v28.2s + fcvtl2 v31.2d, v30.4s + fcvtl v30.2d, v30.2s + + fmla v0.2d, v16.2d, v16.2d + fmla v1.2d, v17.2d, v17.2d + fmla v2.2d, v18.2d, v18.2d + fmla v3.2d, v19.2d, v19.2d + fmla v4.2d, v20.2d, v20.2d + fmla v5.2d, v21.2d, v21.2d + fmla v6.2d, v22.2d, v22.2d + fmla v7.2d, v23.2d, v23.2d + + fmla v0.2d, v24.2d, v24.2d + fmla v1.2d, v25.2d, v25.2d + fmla v2.2d, v26.2d, v26.2d + fmla v3.2d, v27.2d, v27.2d + fmla v4.2d, v28.2d, v28.2d + fmla v5.2d, v29.2d, v29.2d + fmla v6.2d, v30.2d, v30.2d + fmla v7.2d, v31.2d, v31.2d + + prfm PLDL1KEEP, [X, #1024] + prfm PLDL1KEEP, [X, #1024+64] +.endm + +.macro KERNEL_F32_FINALIZE + fadd v0.2d, v0.2d, v1.2d + fadd v2.2d, v2.2d, v3.2d + fadd v4.2d, v4.2d, v5.2d + fadd v6.2d, v6.2d, v7.2d + + fadd v0.2d, v0.2d, v2.2d + fadd v4.2d, v4.2d, v6.2d + + fadd v0.2d, v0.2d, v4.2d + faddp SSQD, v0.2d +.endm + +.macro INIT_S + lsl INC_X, INC_X, #2 +.endm + +.macro KERNEL_S1 + ldr TMPF, [X] + add X, X, INC_X + fcvt TMPFD, TMPF + fmadd SSQD, TMPFD, TMPFD, SSQD +.endm + +/******************************************************************************* +* End of macro definitions +*******************************************************************************/ + + PROLOGUE + + INIT + + cmp N, xzr + ble nrm2_kernel_zero + cmp INC_X, xzr + ble nrm2_kernel_zero + cmp INC_X, #1 + bne nrm2_kernel_S_BEGIN + +nrm2_kernel_F_BEGIN: + + asr I, N, #6 + cmp I, xzr + beq nrm2_kernel_S_BEGIN + + .align 5 +nrm2_kernel_F64: + + KERNEL_F32 + KERNEL_F32 + + subs I, I, #1 + bne nrm2_kernel_F64 + + KERNEL_F32_FINALIZE + +nrm2_kernel_F1: + + ands I, N, #63 + ble nrm2_kernel_L999 + +nrm2_kernel_F10: + + KERNEL_F1 + + subs I, I, #1 + bne nrm2_kernel_F10 + + b nrm2_kernel_L999 + +nrm2_kernel_S_BEGIN: + + INIT_S + + asr I, N, #2 + cmp I, xzr + ble nrm2_kernel_S1 + +nrm2_kernel_S4: + + KERNEL_S1 + KERNEL_S1 + KERNEL_S1 + KERNEL_S1 + + subs I, I, #1 + bne nrm2_kernel_S4 + +nrm2_kernel_S1: + + ands I, N, #3 + ble nrm2_kernel_L999 + +nrm2_kernel_S10: + + KERNEL_S1 + + subs I, I, #1 + bne nrm2_kernel_S10 + +nrm2_kernel_L999: + fsqrt SSQD, SSQD + fcvt SSQ, SSQD + ret + +nrm2_kernel_zero: + fmov SSQ, wzr + + ret + + EPILOGUE From ee6ea7e988aba065f3c9053e31db6cb1fa9b9af3 Mon Sep 17 00:00:00 2001 From: Ashwin Sekhar T K Date: Thu, 19 Jan 2017 15:57:13 +0530 Subject: [PATCH 10/10] THUNDERX2T99: Add Optimized CNRM2 Implementation --- kernel/arm64/KERNEL.THUNDERX2T99 | 2 + kernel/arm64/cnrm2_thunderx2t99.S | 228 ++++++++++++++++++++++++++++++ 2 files changed, 230 insertions(+) create mode 100644 kernel/arm64/cnrm2_thunderx2t99.S diff --git a/kernel/arm64/KERNEL.THUNDERX2T99 b/kernel/arm64/KERNEL.THUNDERX2T99 index b604a7e39..03e77879e 100644 --- a/kernel/arm64/KERNEL.THUNDERX2T99 +++ b/kernel/arm64/KERNEL.THUNDERX2T99 @@ -1,6 +1,8 @@ include $(KERNELDIR)/KERNEL.CORTEXA57 SNRM2KERNEL = snrm2_thunderx2t99.S +CNRM2KERNEL = cnrm2_thunderx2t99.S + DAXPYKERNEL = daxpy_thunderx2t99.S ifndef SMP diff --git a/kernel/arm64/cnrm2_thunderx2t99.S b/kernel/arm64/cnrm2_thunderx2t99.S new file mode 100644 index 000000000..567219efd --- /dev/null +++ b/kernel/arm64/cnrm2_thunderx2t99.S @@ -0,0 +1,228 @@ +/******************************************************************************* +Copyright (c) 2017, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#define ASSEMBLER +#include "common.h" + +#define N x0 /* vector length */ +#define X x1 /* X vector address */ +#define INC_X x2 /* X stride */ +#define I x5 /* loop variable */ + +/******************************************************************************* +* Macro definitions +*******************************************************************************/ + +#define TMPF d16 +#define SSQ s0 +#define SSQD d0 + +/******************************************************************************/ + +.macro INIT + fmov SSQD, xzr + fmov d1, xzr + fmov d2, xzr + fmov d3, xzr + fmov d4, xzr + fmov d5, xzr + fmov d6, xzr + fmov d7, xzr +.endm + +.macro KERNEL_F1 + ldr TMPF, [X] + add X, X, #8 + fcvtl v16.2d, v16.2s + fmla v0.2d, v16.2d, v16.2d +.endm + +.macro KERNEL_F16 + ldur q16, [X] + ldur q18, [X, #16] + ldur q20, [X, #32] + ldur q22, [X, #48] + ldur q24, [X, #64] + ldur q26, [X, #80] + ldur q28, [X, #96] + ldur q30, [X, #112] + + add X, X, #128 + + fcvtl2 v17.2d, v16.4s + fcvtl v16.2d, v16.2s + fcvtl2 v19.2d, v18.4s + fcvtl v18.2d, v18.2s + fcvtl2 v21.2d, v20.4s + fcvtl v20.2d, v20.2s + fcvtl2 v23.2d, v22.4s + fcvtl v22.2d, v22.2s + fcvtl2 v25.2d, v24.4s + fcvtl v24.2d, v24.2s + fcvtl2 v27.2d, v26.4s + fcvtl v26.2d, v26.2s + fcvtl2 v29.2d, v28.4s + fcvtl v28.2d, v28.2s + fcvtl2 v31.2d, v30.4s + fcvtl v30.2d, v30.2s + + fmla v0.2d, v16.2d, v16.2d + fmla v1.2d, v17.2d, v17.2d + fmla v2.2d, v18.2d, v18.2d + fmla v3.2d, v19.2d, v19.2d + fmla v4.2d, v20.2d, v20.2d + fmla v5.2d, v21.2d, v21.2d + fmla v6.2d, v22.2d, v22.2d + fmla v7.2d, v23.2d, v23.2d + + fmla v0.2d, v24.2d, v24.2d + fmla v1.2d, v25.2d, v25.2d + fmla v2.2d, v26.2d, v26.2d + fmla v3.2d, v27.2d, v27.2d + fmla v4.2d, v28.2d, v28.2d + fmla v5.2d, v29.2d, v29.2d + fmla v6.2d, v30.2d, v30.2d + fmla v7.2d, v31.2d, v31.2d + + prfm PLDL1KEEP, [X, #1024] + prfm PLDL1KEEP, [X, #1024+64] +.endm + +.macro KERNEL_F16_FINALIZE + fadd v0.2d, v0.2d, v1.2d + fadd v2.2d, v2.2d, v3.2d + fadd v4.2d, v4.2d, v5.2d + fadd v6.2d, v6.2d, v7.2d + + fadd v0.2d, v0.2d, v2.2d + fadd v4.2d, v4.2d, v6.2d + + fadd v0.2d, v0.2d, v4.2d +.endm + +.macro KERNEL_FINALIZE + faddp SSQD, v0.2d +.endm + +.macro INIT_S + lsl INC_X, INC_X, #3 +.endm + +.macro KERNEL_S1 + ldr TMPF, [X] + add X, X, INC_X + fcvtl v16.2d, v16.2s + fmla v0.2d, v16.2d, v16.2d +.endm + +/******************************************************************************* +* End of macro definitions +*******************************************************************************/ + + PROLOGUE + + INIT + + cmp N, xzr + ble nrm2_kernel_zero + cmp INC_X, xzr + ble nrm2_kernel_zero + cmp INC_X, #1 + bne nrm2_kernel_S_BEGIN + +nrm2_kernel_F_BEGIN: + + asr I, N, #4 + cmp I, xzr + beq nrm2_kernel_S_BEGIN + + .align 5 +nrm2_kernel_F16: + + KERNEL_F16 + + subs I, I, #1 + bne nrm2_kernel_F16 + + KERNEL_F16_FINALIZE + +nrm2_kernel_F1: + + ands I, N, #15 + ble nrm2_kernel_L999 + +nrm2_kernel_F10: + + KERNEL_F1 + + subs I, I, #1 + bne nrm2_kernel_F10 + + b nrm2_kernel_L999 + +nrm2_kernel_S_BEGIN: + + INIT_S + + asr I, N, #2 + cmp I, xzr + ble nrm2_kernel_S1 + +nrm2_kernel_S4: + + KERNEL_S1 + KERNEL_S1 + KERNEL_S1 + KERNEL_S1 + + subs I, I, #1 + bne nrm2_kernel_S4 + +nrm2_kernel_S1: + + ands I, N, #3 + ble nrm2_kernel_L999 + +nrm2_kernel_S10: + + KERNEL_S1 + + subs I, I, #1 + bne nrm2_kernel_S10 + +nrm2_kernel_L999: + KERNEL_FINALIZE + fsqrt SSQD, SSQD + fcvt SSQ, SSQD + ret + +nrm2_kernel_zero: + fmov SSQ, wzr + + ret + + EPILOGUE