From cde3aee08b9a3ea5bc0c2795b4256e9649df2c6e Mon Sep 17 00:00:00 2001 From: Ashwin Sekhar T K Date: Tue, 24 Jan 2017 14:49:49 +0530 Subject: [PATCH 01/12] ARM64: Rename kernel files to have consistent naming --- kernel/arm64/KERNEL.THUNDERX | 6 +++--- kernel/arm64/KERNEL.THUNDERX2T99 | 2 +- kernel/arm64/{daxpy-thunderx.c => daxpy_thunderx.c} | 0 kernel/arm64/{ddot-thunderx.c => ddot_thunderx.c} | 0 .../{ddot_thunderx2t99.c => ddot_thunderx2t99_threaded.c} | 0 kernel/arm64/{dot-thunderx.c => dot_thunderx.c} | 0 6 files changed, 4 insertions(+), 4 deletions(-) rename kernel/arm64/{daxpy-thunderx.c => daxpy_thunderx.c} (100%) rename kernel/arm64/{ddot-thunderx.c => ddot_thunderx.c} (100%) rename kernel/arm64/{ddot_thunderx2t99.c => ddot_thunderx2t99_threaded.c} (100%) rename kernel/arm64/{dot-thunderx.c => dot_thunderx.c} (100%) diff --git a/kernel/arm64/KERNEL.THUNDERX b/kernel/arm64/KERNEL.THUNDERX index 4e49ca2a0..11b7a2ca8 100644 --- a/kernel/arm64/KERNEL.THUNDERX +++ b/kernel/arm64/KERNEL.THUNDERX @@ -1,6 +1,6 @@ include $(KERNELDIR)/KERNEL.ARMV8 -SDOTKERNEL=dot-thunderx.c -DDOTKERNEL=ddot-thunderx.c -DAXPYKERNEL=daxpy-thunderx.c +SDOTKERNEL=dot_thunderx.c +DDOTKERNEL=ddot_thunderx.c +DAXPYKERNEL=daxpy_thunderx.c diff --git a/kernel/arm64/KERNEL.THUNDERX2T99 b/kernel/arm64/KERNEL.THUNDERX2T99 index 03e77879e..66fb6b36f 100644 --- a/kernel/arm64/KERNEL.THUNDERX2T99 +++ b/kernel/arm64/KERNEL.THUNDERX2T99 @@ -8,7 +8,7 @@ DAXPYKERNEL = daxpy_thunderx2t99.S ifndef SMP DDOTKERNEL = ddot_thunderx2t99.S else -DDOTKERNEL = ddot_thunderx2t99.c +DDOTKERNEL = ddot_thunderx2t99_threaded.c endif ifeq ($(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N), 8x4) diff --git a/kernel/arm64/daxpy-thunderx.c b/kernel/arm64/daxpy_thunderx.c similarity index 100% rename from kernel/arm64/daxpy-thunderx.c rename to kernel/arm64/daxpy_thunderx.c diff --git a/kernel/arm64/ddot-thunderx.c b/kernel/arm64/ddot_thunderx.c similarity index 100% rename from kernel/arm64/ddot-thunderx.c rename to kernel/arm64/ddot_thunderx.c diff --git a/kernel/arm64/ddot_thunderx2t99.c b/kernel/arm64/ddot_thunderx2t99_threaded.c similarity index 100% rename from kernel/arm64/ddot_thunderx2t99.c rename to kernel/arm64/ddot_thunderx2t99_threaded.c diff --git a/kernel/arm64/dot-thunderx.c b/kernel/arm64/dot_thunderx.c similarity index 100% rename from kernel/arm64/dot-thunderx.c rename to kernel/arm64/dot_thunderx.c From 907e286eb6868920fcc13ea814baf90a1972f02c Mon Sep 17 00:00:00 2001 From: Ashwin Sekhar T K Date: Tue, 24 Jan 2017 21:39:29 +0530 Subject: [PATCH 02/12] THUNDERX2T99: Add threaded SNRM2 Implementation --- kernel/arm64/KERNEL.THUNDERX2T99 | 4 + kernel/arm64/nrm2_thunderx2t99_threaded.c | 249 ++++++++++++++++++++++ 2 files changed, 253 insertions(+) create mode 100644 kernel/arm64/nrm2_thunderx2t99_threaded.c diff --git a/kernel/arm64/KERNEL.THUNDERX2T99 b/kernel/arm64/KERNEL.THUNDERX2T99 index 66fb6b36f..3a7fa2f07 100644 --- a/kernel/arm64/KERNEL.THUNDERX2T99 +++ b/kernel/arm64/KERNEL.THUNDERX2T99 @@ -1,6 +1,10 @@ include $(KERNELDIR)/KERNEL.CORTEXA57 +ifndef SMP SNRM2KERNEL = snrm2_thunderx2t99.S +else +SNRM2KERNEL = nrm2_thunderx2t99_threaded.c +endif CNRM2KERNEL = cnrm2_thunderx2t99.S DAXPYKERNEL = daxpy_thunderx2t99.S diff --git a/kernel/arm64/nrm2_thunderx2t99_threaded.c b/kernel/arm64/nrm2_thunderx2t99_threaded.c new file mode 100644 index 000000000..d810c6713 --- /dev/null +++ b/kernel/arm64/nrm2_thunderx2t99_threaded.c @@ -0,0 +1,249 @@ +/*************************************************************************** +Copyright (c) 2017, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + + +#include "common.h" + +#include + +extern int blas_level1_thread_with_return_value(int mode, BLASLONG m, BLASLONG n, + BLASLONG k, void *alpha, void *a, BLASLONG lda, void *b, BLASLONG ldb, + void *c, BLASLONG ldc, int (*function)(), int nthreads); + +#if !defined(DOUBLE) +#define N "x0" /* vector length */ +#define X "x1" /* X vector address */ +#define INC_X "x2" /* X stride */ +#define I "x5" /* loop variable */ + +#define TMPF "s16" +#define TMPFD "d17" +#define SSQD "d0" + +#define KERNEL_F1 \ + "ldr "TMPF", ["X"], #4 \n" \ + "fcvt "TMPFD", "TMPF" \n" \ + "fmadd "SSQD", "TMPFD", "TMPFD", "SSQD"\n" + +#define KERNEL_F32 \ + "ldur q16, ["X"] \n" \ + "ldur q18, ["X", #16] \n" \ + "ldur q20, ["X", #32] \n" \ + "ldur q22, ["X", #48] \n" \ + "ldur q24, ["X", #64] \n" \ + "ldur q26, ["X", #80] \n" \ + "ldur q28, ["X", #96] \n" \ + "ldur q30, ["X", #112] \n" \ + "add "X", "X", #128 \n" \ + "fcvtl2 v17.2d, v16.4s \n" \ + "fcvtl v16.2d, v16.2s \n" \ + "fcvtl2 v19.2d, v18.4s \n" \ + "fcvtl v18.2d, v18.2s \n" \ + "fcvtl2 v21.2d, v20.4s \n" \ + "fcvtl v20.2d, v20.2s \n" \ + "fcvtl2 v23.2d, v22.4s \n" \ + "fcvtl v22.2d, v22.2s \n" \ + "fcvtl2 v25.2d, v24.4s \n" \ + "fcvtl v24.2d, v24.2s \n" \ + "fcvtl2 v27.2d, v26.4s \n" \ + "fcvtl v26.2d, v26.2s \n" \ + "fcvtl2 v29.2d, v28.4s \n" \ + "fcvtl v28.2d, v28.2s \n" \ + "fcvtl2 v31.2d, v30.4s \n" \ + "fcvtl v30.2d, v30.2s \n" \ + "fmla v0.2d, v16.2d, v16.2d \n" \ + "fmla v1.2d, v17.2d, v17.2d \n" \ + "fmla v2.2d, v18.2d, v18.2d \n" \ + "fmla v3.2d, v19.2d, v19.2d \n" \ + "fmla v4.2d, v20.2d, v20.2d \n" \ + "fmla v5.2d, v21.2d, v21.2d \n" \ + "fmla v6.2d, v22.2d, v22.2d \n" \ + "fmla v7.2d, v23.2d, v23.2d \n" \ + "fmla v0.2d, v24.2d, v24.2d \n" \ + "fmla v1.2d, v25.2d, v25.2d \n" \ + "fmla v2.2d, v26.2d, v26.2d \n" \ + "fmla v3.2d, v27.2d, v27.2d \n" \ + "fmla v4.2d, v28.2d, v28.2d \n" \ + "fmla v5.2d, v29.2d, v29.2d \n" \ + "fmla v6.2d, v30.2d, v30.2d \n" \ + "fmla v7.2d, v31.2d, v31.2d \n" \ + "prfm PLDL1KEEP, ["X", #1024] \n" \ + "prfm PLDL1KEEP, ["X", #1024+64] \n" + +#define KERNEL_F32_FINALIZE \ + "fadd v0.2d, v0.2d, v1.2d \n" \ + "fadd v2.2d, v2.2d, v3.2d \n" \ + "fadd v4.2d, v4.2d, v5.2d \n" \ + "fadd v6.2d, v6.2d, v7.2d \n" \ + "fadd v0.2d, v0.2d, v2.2d \n" \ + "fadd v4.2d, v4.2d, v6.2d \n" \ + "fadd v0.2d, v0.2d, v4.2d \n" \ + "faddp "SSQD", v0.2d \n" + +#define KERNEL_S1 \ + "ldr "TMPF", ["X"] \n" \ + "add "X", "X", "INC_X" \n" \ + "fcvt "TMPFD", "TMPF" \n" \ + "fmadd "SSQD", "TMPFD", "TMPFD", "SSQD"\n" + + +static double nrm2_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x) +{ + double ret = 0.0 ; + + if (n <= 0) return ret; + + __asm__ __volatile__ ( + " mov "N", %[N_] \n" + " mov "X", %[X_] \n" + " mov "INC_X", %[INCX_] \n" + " fmov "SSQD", xzr \n" + " fmov d1, xzr \n" + " fmov d2, xzr \n" + " fmov d3, xzr \n" + " fmov d4, xzr \n" + " fmov d5, xzr \n" + " fmov d6, xzr \n" + " fmov d7, xzr \n" + " cmp "N", xzr \n" + " ble 8f //nrm2_kernel_L999 \n" + " cmp "INC_X", xzr \n" + " ble 8f //nrm2_kernel_L999 \n" + " cmp "INC_X", #1 \n" + " bne 5f // nrm2_kernel_S_BEGIN \n" + + "1: //nrm2_kernel_F_BEGIN: \n" + " asr "I", "N", #6 \n" + " cmp "I", xzr \n" + " beq 5f // nrm2_kernel_S_BEGIN \n" + + " .align 5 \n" + "2: //nrm2_kernel_F64: \n" + " "KERNEL_F32" \n" + " "KERNEL_F32" \n" + " subs "I", "I", #1 \n" + " bne 2b //nrm2_kernel_F64 \n" + " "KERNEL_F32_FINALIZE" \n" + + "3: // nrm2_kernel_F1: \n" + " ands "I", "N", #63 \n" + " ble 8f //nrm2_kernel_L999 \n" + + "4: // nrm2_kernel_F10: \n" + " "KERNEL_F1" \n" + " subs "I", "I", #1 \n" + " bne 4b //nrm2_kernel_F10 \n" + " b 8f //nrm2_kernel_L999 \n" + + "5: // nrm2_kernel_S_BEGIN: \n" + " lsl "INC_X", "INC_X", #2 \n" + " asr "I", "N", #2 \n" + " cmp "I", xzr \n" + " ble 6f //nrm2_kernel_S1 \n" + + "4: //nrm2_kernel_S4: \n" + " "KERNEL_S1" \n" + " "KERNEL_S1" \n" + " "KERNEL_S1" \n" + " "KERNEL_S1" \n" + " subs "I", "I", #1 \n" + " bne 4b //nrm2_kernel_S4 \n" + + "6: //nrm2_kernel_S1: \n" + " ands "I", "N", #3 \n" + " ble 8f //nrm2_kernel_L999 \n" + + "7: //nrm2_kernel_S10: \n" + " "KERNEL_S1" \n" + " subs "I", "I", #1 \n" + " bne 7b //nrm2_kernel_S10 \n" + + "8: //nrm2_kernel_L999: \n" + " fmov %[RET_], "SSQD" \n" + + : [RET_] "=r" (ret) //%0 + : [N_] "r" (n), //%1 + [X_] "r" (x), //%2 + [INCX_] "r" (inc_x) //%3 + : "cc", + "memory", + "x0", "x1", "x2", "x3", "x4", "x5", + "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7" + ); + + return ret; +} +#else //!defined(DOUBLE) +#endif + +static int nrm2_thread_function(BLASLONG n, BLASLONG dummy0, + BLASLONG dummy1, FLOAT dummy2, FLOAT *x, BLASLONG inc_x, FLOAT *dummy3, + BLASLONG dummy4, FLOAT *result, BLASLONG dummy5) +{ + *(double *)result = nrm2_compute(n, x, inc_x); + + return 0; +} + +FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) +{ + int nthreads; + FLOAT nrm2 = 0.0; + double nrm2_double = 0.0; + FLOAT dummy_alpha; + + if (n <= 0 || inc_x <= 0) return 0.0; + if (n == 1) return fabs(x[0]); + + nthreads = num_cpu_avail(1); + + if (n <= 10000) + nthreads = 1; + + if (nthreads == 1) { + nrm2_double = nrm2_compute(n, x, inc_x); + } else { + int mode, i; + char result[MAX_CPU_NUMBER * sizeof(double) * 2]; + double *ptr; + + mode = BLAS_SINGLE | BLAS_REAL; + + blas_level1_thread_with_return_value(mode, n, 0, 0, &dummy_alpha, + x, inc_x, NULL, 0, result, 0, + ( void *)nrm2_thread_function, nthreads); + + ptr = (double *)result; + for (i = 0; i < nthreads; i++) { + nrm2_double = nrm2_double + (*ptr) * (*ptr); + ptr = (double *)(((char *)ptr) + sizeof(double) * 2); + } + } + nrm2 = sqrt(nrm2_double); + + return nrm2; +} From 1530e78cfe4c5dfbe14a7925219de26163d499f2 Mon Sep 17 00:00:00 2001 From: Ashwin Sekhar T K Date: Tue, 24 Jan 2017 20:50:23 -0800 Subject: [PATCH 03/12] Benchmarks: Avoid building lapack benchmarks when NO_LAPACK=1 --- benchmark/Makefile | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/benchmark/Makefile b/benchmark/Makefile index e801ce4eb..51e9c64aa 100644 --- a/benchmark/Makefile +++ b/benchmark/Makefile @@ -37,6 +37,18 @@ ESSL=/opt/ibm/lib #LIBESSL = -lesslsmp $(ESSL)/libxlomp_ser.so.1 $(ESSL)/libxlf90_r.so.1 $(ESSL)/libxlfmath.so.1 $(ESSL)/libxlsmp.so.1 /opt/ibm/xlC/13.1.3/lib/libxl.a LIBESSL = -lesslsmp $(ESSL)/libxlf90_r.so.1 $(ESSL)/libxlfmath.so.1 $(ESSL)/libxlsmp.so.1 /opt/ibm/xlC/13.1.3/lib/libxl.a +ifneq ($(NO_LAPACK), 1) +GOTO_LAPACK_TARGETS=slinpack.goto dlinpack.goto clinpack.goto zlinpack.goto \ + scholesky.goto dcholesky.goto ccholesky.goto zcholesky.goto \ + sgesv.goto dgesv.goto cgesv.goto zgesv.goto \ + sgeev.goto dgeev.goto cgeev.goto zgeev.goto \ + csymv.goto zsymv.goto \ + sgetri.goto dgetri.goto cgetri.goto zgetri.goto \ + spotrf.goto dpotrf.goto cpotrf.goto zpotrf.goto +else +GOTO_LAPACK_TARGETS= +endif + ifeq ($(OSNAME), WINNT) goto :: slinpack.goto dlinpack.goto clinpack.goto zlinpack.goto \ @@ -147,9 +159,7 @@ mkl :: slinpack.mkl dlinpack.mkl clinpack.mkl zlinpack.mkl \ else -goto :: slinpack.goto dlinpack.goto clinpack.goto zlinpack.goto \ - scholesky.goto dcholesky.goto ccholesky.goto zcholesky.goto \ - sgemm.goto dgemm.goto cgemm.goto zgemm.goto \ +goto :: sgemm.goto dgemm.goto cgemm.goto zgemm.goto \ strmm.goto dtrmm.goto ctrmm.goto ztrmm.goto \ strsm.goto dtrsm.goto ctrsm.goto ztrsm.goto \ ssyrk.goto dsyrk.goto csyrk.goto zsyrk.goto \ @@ -162,20 +172,16 @@ goto :: slinpack.goto dlinpack.goto clinpack.goto zlinpack.goto \ sswap.goto dswap.goto cswap.goto zswap.goto \ sscal.goto dscal.goto cscal.goto zscal.goto \ sasum.goto dasum.goto casum.goto zasum.goto \ - ssymv.goto dsymv.goto csymv.goto zsymv.goto \ + ssymv.goto dsymv.goto \ chemv.goto zhemv.goto \ chemm.goto zhemm.goto \ cherk.goto zherk.goto \ cher2k.goto zher2k.goto \ sgemv.goto dgemv.goto cgemv.goto zgemv.goto \ - sgesv.goto dgesv.goto cgesv.goto zgesv.goto \ - sgeev.goto dgeev.goto cgeev.goto zgeev.goto \ - sgetri.goto dgetri.goto cgetri.goto zgetri.goto \ - spotrf.goto dpotrf.goto cpotrf.goto zpotrf.goto \ ssymm.goto dsymm.goto csymm.goto zsymm.goto \ smallscaling \ isamax.goto idamax.goto icamax.goto izamax.goto \ - snrm2.goto dnrm2.goto scnrm2.goto dznrm2.goto + snrm2.goto dnrm2.goto scnrm2.goto dznrm2.goto $(GOTO_LAPACK_TARGETS) acml :: slinpack.acml dlinpack.acml clinpack.acml zlinpack.acml \ scholesky.acml dcholesky.acml ccholesky.acml zcholesky.acml \ From efda640723cc7f345b3109bb570355f43c44a407 Mon Sep 17 00:00:00 2001 From: Ashwin Sekhar T K Date: Tue, 24 Jan 2017 23:13:47 -0800 Subject: [PATCH 04/12] Benchmark: Add MFlops print in iamax benchmark --- benchmark/iamax.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/benchmark/iamax.c b/benchmark/iamax.c index c55f41579..034e24ea9 100644 --- a/benchmark/iamax.c +++ b/benchmark/iamax.c @@ -149,7 +149,7 @@ int main(int argc, char *argv[]){ srandom(getpid()); #endif - fprintf(stderr, " SIZE Time\n"); + fprintf(stderr, " SIZE Flops\n"); for(m = from; m <= to; m += step) { @@ -180,7 +180,9 @@ int main(int argc, char *argv[]){ timeg /= loops; - fprintf(stderr, " %10.6f secs\n", timeg); + fprintf(stderr, + " %10.2f MFlops %10.6f sec\n", + COMPSIZE * sizeof(FLOAT) * 1. * (double)m / timeg * 1.e-6, timeg); } From 1de6fa0f505394988a083534e91f152e42ee23c1 Mon Sep 17 00:00:00 2001 From: Ashwin Sekhar T K Date: Tue, 24 Jan 2017 23:14:09 -0800 Subject: [PATCH 05/12] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c172e25e9..a2ec7dd08 100644 --- a/.gitignore +++ b/.gitignore @@ -85,4 +85,5 @@ build build.* *.swp benchmark/*.goto +benchmark/smallscaling From 2757b49767c7cea4e91d7139a76aa9e27626322b Mon Sep 17 00:00:00 2001 From: Ashwin Sekhar T K Date: Wed, 25 Jan 2017 03:14:59 -0800 Subject: [PATCH 06/12] THUNDERX2T99: Add Optimized CGEMM Implementation --- driver/others/parameter.c | 4 + kernel/arm64/KERNEL.THUNDERX2T99 | 7 +- kernel/arm64/cgemm_kernel_8x4_thunderx2t99.S | 2175 ++++++++++++++++++ param.h | 46 +- 4 files changed, 2186 insertions(+), 46 deletions(-) create mode 100644 kernel/arm64/cgemm_kernel_8x4_thunderx2t99.S diff --git a/driver/others/parameter.c b/driver/others/parameter.c index ac337b203..96da0213e 100644 --- a/driver/others/parameter.c +++ b/driver/others/parameter.c @@ -747,6 +747,10 @@ void blas_set_parameter(void) sgemm_q = 352; sgemm_r = 4096; + cgemm_p = 128; + cgemm_q = 224; + cgemm_r = 4096; + dgemm_prefetch_size_a = 3584; dgemm_prefetch_size_b = 512; dgemm_prefetch_size_c = 128; diff --git a/kernel/arm64/KERNEL.THUNDERX2T99 b/kernel/arm64/KERNEL.THUNDERX2T99 index 3a7fa2f07..94132bf60 100644 --- a/kernel/arm64/KERNEL.THUNDERX2T99 +++ b/kernel/arm64/KERNEL.THUNDERX2T99 @@ -17,13 +17,12 @@ endif ifeq ($(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N), 8x4) DGEMMKERNEL = dgemm_kernel_8x4_thunderx2t99.S -else -DGEMMKERNEL = dgemm_kernel_$(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N).S endif ifeq ($(SGEMM_UNROLL_M)x$(SGEMM_UNROLL_N), 16x4) SGEMMKERNEL = sgemm_kernel_16x4_thunderx2t99.S -else -SGEMMKERNEL = sgemm_kernel_$(SGEMM_UNROLL_M)x$(SGEMM_UNROLL_N).S endif +ifeq ($(CGEMM_UNROLL_M)x$(CGEMM_UNROLL_N), 8x4) +CGEMMKERNEL = cgemm_kernel_8x4_thunderx2t99.S +endif diff --git a/kernel/arm64/cgemm_kernel_8x4_thunderx2t99.S b/kernel/arm64/cgemm_kernel_8x4_thunderx2t99.S new file mode 100644 index 000000000..367cd0217 --- /dev/null +++ b/kernel/arm64/cgemm_kernel_8x4_thunderx2t99.S @@ -0,0 +1,2175 @@ +/******************************************************************************* +Copyright (c) 2017, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#define ASSEMBLER +#include "common.h" + +/* X0 X1 X2 s0 X3 x4 x5 x6 */ +/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha0,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc */ + +#define origM x0 +#define origN x1 +#define origK x2 +#define origPA x3 +#define origPB x4 +#define pC x5 +#define LDC x6 +#define temp x7 +#define counterL x8 +#define counterI x9 +#define counterJ x10 +#define pB x11 +#define pCRow0 x12 +#define pCRow1 x13 +#define pCRow2 x14 +#define pCRow3 x15 +#define pA x16 +#define alphaR w17 +#define alphaI w18 + +#define alpha0_R s10 +#define alphaV0_R v10.s[0] +#define alpha0_I s11 +#define alphaV0_I v11.s[0] + +#define A_PRE_SIZE 2560 +#define B_PRE_SIZE 448 +#define C_PRE_SIZE 128 + +#if defined(NN) || defined(NT) || defined(TN) || defined(TT) +#define OP_rr fmla +#define OP_ii fmls +#define OP_ri fmla +#define OP_ir fmla +#elif defined(NR) || defined(NC) || defined(TR) || defined(TC) +#define OP_rr fmla +#define OP_ii fmla +#define OP_ri fmls +#define OP_ir fmla +#elif defined(RN) || defined(RT) || defined(CN) || defined(CT) +#define OP_rr fmla +#define OP_ii fmla +#define OP_ri fmla +#define OP_ir fmls +#elif defined(RR) || defined(RC) || defined(CR) || defined(CC) +#define OP_rr fmla +#define OP_ii fmls +#define OP_ri fmls +#define OP_ir fmls +#endif + +// 00 origM +// 01 origN +// 02 origK +// 03 origPA +// 04 origPB +// 05 pC +// 06 origLDC -> LDC +// 07 offset -> temp +// 08 counterL +// 09 counterI +// 10 counterJ +// 11 pB +// 12 pCRow0 +// 13 pCRow1 +// 14 pCRow2 +// 15 pCRow3 +// 16 pA +// 17 +// 18 must save +// 19 must save +// 20 must save +// 21 must save +// 22 must save +// 23 must save +// 24 must save +// 25 must save +// 26 must save +// 27 must save +// 28 must save +// 29 frame +// 30 link +// 31 sp + +//v00 ALPHA_R -> pA0_00_R, pA0_01_R, pA0_02_R, pA0_03_R +//v01 ALPHA_I -> pA0_00_I, pA0_01_I, pA0_02_I, pA0_03_I +//v02 pA0_04_R, pA0_05_R, pA0_06_R, pA0_07_R +//v03 pA0_04_I, pA0_05_I, pA0_06_I, pA0_07_I +//v04 pA1_00_R, pA1_01_R, pA1_02_R, pA1_03_R +//v05 pA1_00_I, pA1_01_I, pA1_02_I, pA1_03_I +//v06 pA1_04_R, pA1_05_R, pA1_06_R, pA1_07_R +//v07 pA1_04_I, pA1_05_I, pA1_06_I, pA1_07_I +//v08 must save pB0_00_R, pB0_01_R +//v09 must save pB0_00_I, pB0_01_I +//v10 must save pB0_02_R, pB0_03_R --> ALPHA0_R +//v11 must save pB0_02_I, pB0_03_I --> ALPHA0_I +//v12 must save pB1_00_R, pB1_01_R +//v13 must save pB1_00_I, pB1_01_I +//v14 must save pB1_02_R, pB1_03_R +//v15 must save pB1_02_I, pB1_03_I +//v16 must save pC_00_R, pC_01_R, pC_02_R, pC_03_R +//v17 must save pC_00_I, pC_01_I, pC_02_I, pC_03_I +//v18 pC_04_R, pC_05_R, pC_06_R, pC_07_R +//v19 pC_04_I, pC_05_I, pC_06_I, pC_07_I +//v20 pC_08_R, pC_09_R, pC_10_R, pC_11_R +//v21 pC_08_I, pC_09_I, pC_10_I, pC_11_I +//v22 pC_12_R, pC_13_R, pC_14_R, pC_15_R +//v23 pC_12_I, pC_13_I, pC_14_I, pC_15_I +//v24 pC_16_R, pC_17_R, pC_18_R, pC_19_R +//v25 pC_16_I, pC_17_I, pC_18_I, pC_19_I +//v26 pC_20_R, pC_21_R, pC_22_R, pC_23_R +//v27 pC_20_I, pC_21_I, pC_22_I, pC_23_I +//v28 pC_24_R, pC_25_R, pC_26_R, pC_27_R +//v29 pC_24_I, pC_25_I, pC_26_I, pC_27_I +//v30 pC_28_R, pC_29_R, pC_30_R, pC_31_R +//v31 pC_28_I, pC_29_I, pC_30_I, pC_31_I + + +/******************************************************************************* +* Macro definitions +*******************************************************************************/ + +.macro INIT8x4 + fmov s16, wzr + fmov s17, wzr + fmov s18, wzr + fmov s19, s16 + fmov s20, wzr + fmov s21, s16 + fmov s22, s17 + fmov s23, s18 + fmov s24, wzr + fmov s25, s16 + fmov s26, s17 + fmov s27, s18 + fmov s28, wzr + fmov s29, s16 + fmov s30, s17 + fmov s31, s18 +.endm + +.macro KERNEL8x4_I + ldr q8, [pB] + add pB, pB, #16 + + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + ld2 {v2.4s, v3.4s}, [pA] + add pA, pA, #32 + + fmul v16.4s, v0.4s, v8.s[0] + OP_ii v16.4s, v1.4s, v8.s[1] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v17.16b, v17.16b, v17.16b + fmls v17.4s, v0.4s, v8.s[1] +#else + fmul v17.4s, v0.4s, v8.s[1] +#endif + OP_ir v17.4s, v1.4s, v8.s[0] + + ldr q10, [pB] + add pB, pB, #16 + + fmul v18.4s, v2.4s, v8.s[0] + OP_ii v18.4s, v3.4s, v8.s[1] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v19.16b, v19.16b, v19.16b + fmls v19.4s, v2.4s, v8.s[1] +#else + fmul v19.4s, v2.4s, v8.s[1] +#endif + OP_ir v19.4s, v3.4s, v8.s[0] + + ldr q12, [pB] + add pB, pB, #16 + + fmul v20.4s, v0.4s, v8.s[2] + OP_ii v20.4s, v1.4s, v8.s[3] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v21.16b, v21.16b, v21.16b + fmls v21.4s, v0.4s, v8.s[3] +#else + fmul v21.4s, v0.4s, v8.s[3] +#endif + OP_ir v21.4s, v1.4s, v8.s[2] + + ldr q14, [pB] + add pB, pB, #16 + + fmul v22.4s, v2.4s, v8.s[2] + OP_ii v22.4s, v3.4s, v8.s[3] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v23.16b, v23.16b, v23.16b + fmls v23.4s, v2.4s, v8.s[3] +#else + fmul v23.4s, v2.4s, v8.s[3] +#endif + OP_ir v23.4s, v3.4s, v8.s[2] + + ld2 {v4.4s, v5.4s}, [pA] + add pA, pA, #32 + + fmul v24.4s, v0.4s, v10.s[0] + OP_ii v24.4s, v1.4s, v10.s[1] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v25.16b, v25.16b, v25.16b + fmls v25.4s, v0.4s, v10.s[1] +#else + fmul v25.4s, v0.4s, v10.s[1] +#endif + OP_ir v25.4s, v1.4s, v10.s[0] + + ld2 {v6.4s, v7.4s}, [pA] + add pA, pA, #32 + + fmul v26.4s, v2.4s, v10.s[0] + OP_ii v26.4s, v3.4s, v10.s[1] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v27.16b, v27.16b, v27.16b + fmls v27.4s, v2.4s, v10.s[1] +#else + fmul v27.4s, v2.4s, v10.s[1] +#endif + OP_ir v27.4s, v3.4s, v10.s[0] + + prfm PLDL1KEEP, [pA, #A_PRE_SIZE] + + fmul v28.4s, v0.4s, v10.s[2] + OP_ii v28.4s, v1.4s, v10.s[3] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v29.16b, v29.16b, v29.16b + fmls v29.4s, v0.4s, v10.s[3] +#else + fmul v29.4s, v0.4s, v10.s[3] +#endif + OP_ir v29.4s, v1.4s, v10.s[2] + + prfm PLDL1KEEP, [pA, #A_PRE_SIZE+64] + + fmul v30.4s, v2.4s, v10.s[2] + OP_ii v30.4s, v3.4s, v10.s[3] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v31.16b, v31.16b, v31.16b + fmls v31.4s, v2.4s, v10.s[3] +#else + fmul v31.4s, v2.4s, v10.s[3] +#endif + OP_ir v31.4s, v3.4s, v10.s[2] +.endm + +.macro KERNEL8x4_M1 + OP_rr v16.4s, v0.4s, v8.s[0] + OP_ii v16.4s, v1.4s, v8.s[1] + OP_ri v17.4s, v0.4s, v8.s[1] + OP_ir v17.4s, v1.4s, v8.s[0] + + ldr q12, [pB] + add pB, pB, #16 + + OP_rr v18.4s, v2.4s, v8.s[0] + OP_ii v18.4s, v3.4s, v8.s[1] + OP_ri v19.4s, v2.4s, v8.s[1] + OP_ir v19.4s, v3.4s, v8.s[0] + + ld2 {v4.4s, v5.4s}, [pA] + add pA, pA, #32 + + OP_rr v20.4s, v0.4s, v8.s[2] + OP_ii v20.4s, v1.4s, v8.s[3] + OP_ri v21.4s, v0.4s, v8.s[3] + OP_ir v21.4s, v1.4s, v8.s[2] + + ld2 {v6.4s, v7.4s}, [pA] + add pA, pA, #32 + + OP_rr v22.4s, v2.4s, v8.s[2] + OP_ii v22.4s, v3.4s, v8.s[3] + OP_ri v23.4s, v2.4s, v8.s[3] + OP_ir v23.4s, v3.4s, v8.s[2] + + ldr q14, [pB] + add pB, pB, #16 + + OP_rr v24.4s, v0.4s, v10.s[0] + OP_ii v24.4s, v1.4s, v10.s[1] + OP_ri v25.4s, v0.4s, v10.s[1] + OP_ir v25.4s, v1.4s, v10.s[0] + + prfm PLDL1KEEP, [pA, #A_PRE_SIZE] + + OP_rr v26.4s, v2.4s, v10.s[0] + OP_ii v26.4s, v3.4s, v10.s[1] + OP_ri v27.4s, v2.4s, v10.s[1] + OP_ir v27.4s, v3.4s, v10.s[0] + + prfm PLDL1KEEP, [pA, #A_PRE_SIZE+64] + + OP_rr v28.4s, v0.4s, v10.s[2] + OP_ii v28.4s, v1.4s, v10.s[3] + OP_ri v29.4s, v0.4s, v10.s[3] + OP_ir v29.4s, v1.4s, v10.s[2] + + OP_rr v30.4s, v2.4s, v10.s[2] + OP_ii v30.4s, v3.4s, v10.s[3] + OP_ri v31.4s, v2.4s, v10.s[3] + OP_ir v31.4s, v3.4s, v10.s[2] +.endm + +.macro KERNEL8x4_M2 + OP_rr v16.4s, v4.4s, v12.s[0] + OP_ii v16.4s, v5.4s, v12.s[1] + OP_ri v17.4s, v4.4s, v12.s[1] + OP_ir v17.4s, v5.4s, v12.s[0] + + ldr q8, [pB] + add pB, pB, #16 + + OP_rr v18.4s, v6.4s, v12.s[0] + OP_ii v18.4s, v7.4s, v12.s[1] + OP_ri v19.4s, v6.4s, v12.s[1] + OP_ir v19.4s, v7.4s, v12.s[0] + + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + + OP_rr v20.4s, v4.4s, v12.s[2] + OP_ii v20.4s, v5.4s, v12.s[3] + OP_ri v21.4s, v4.4s, v12.s[3] + OP_ir v21.4s, v5.4s, v12.s[2] + + ld2 {v2.4s, v3.4s}, [pA] + add pA, pA, #32 + + OP_rr v22.4s, v6.4s, v12.s[2] + OP_ii v22.4s, v7.4s, v12.s[3] + OP_ri v23.4s, v6.4s, v12.s[3] + OP_ir v23.4s, v7.4s, v12.s[2] + + ldr q10, [pB] + add pB, pB, #16 + + OP_rr v24.4s, v4.4s, v14.s[0] + OP_ii v24.4s, v5.4s, v14.s[1] + OP_ri v25.4s, v4.4s, v14.s[1] + OP_ir v25.4s, v5.4s, v14.s[0] + + prfm PLDL1KEEP, [pB, #B_PRE_SIZE] + + OP_rr v26.4s, v6.4s, v14.s[0] + OP_ii v26.4s, v7.4s, v14.s[1] + OP_ri v27.4s, v6.4s, v14.s[1] + OP_ir v27.4s, v7.4s, v14.s[0] + + OP_rr v28.4s, v4.4s, v14.s[2] + OP_ii v28.4s, v5.4s, v14.s[3] + OP_ri v29.4s, v4.4s, v14.s[3] + OP_ir v29.4s, v5.4s, v14.s[2] + + OP_rr v30.4s, v6.4s, v14.s[2] + OP_ii v30.4s, v7.4s, v14.s[3] + OP_ri v31.4s, v6.4s, v14.s[3] + OP_ir v31.4s, v7.4s, v14.s[2] +.endm + +.macro KERNEL8x4_E + OP_rr v16.4s, v4.4s, v12.s[0] + OP_ii v16.4s, v5.4s, v12.s[1] + OP_ri v17.4s, v4.4s, v12.s[1] + OP_ir v17.4s, v5.4s, v12.s[0] + + OP_rr v18.4s, v6.4s, v12.s[0] + OP_ii v18.4s, v7.4s, v12.s[1] + OP_ri v19.4s, v6.4s, v12.s[1] + OP_ir v19.4s, v7.4s, v12.s[0] + + OP_rr v20.4s, v4.4s, v12.s[2] + OP_ii v20.4s, v5.4s, v12.s[3] + OP_ri v21.4s, v4.4s, v12.s[3] + OP_ir v21.4s, v5.4s, v12.s[2] + + prfm PLDL1KEEP, [pB, #B_PRE_SIZE] + + OP_rr v22.4s, v6.4s, v12.s[2] + OP_ii v22.4s, v7.4s, v12.s[3] + OP_ri v23.4s, v6.4s, v12.s[3] + OP_ir v23.4s, v7.4s, v12.s[2] + + OP_rr v24.4s, v4.4s, v14.s[0] + OP_ii v24.4s, v5.4s, v14.s[1] + OP_ri v25.4s, v4.4s, v14.s[1] + OP_ir v25.4s, v5.4s, v14.s[0] + + OP_rr v26.4s, v6.4s, v14.s[0] + OP_ii v26.4s, v7.4s, v14.s[1] + OP_ri v27.4s, v6.4s, v14.s[1] + OP_ir v27.4s, v7.4s, v14.s[0] + + OP_rr v28.4s, v4.4s, v14.s[2] + OP_ii v28.4s, v5.4s, v14.s[3] + OP_ri v29.4s, v4.4s, v14.s[3] + OP_ir v29.4s, v5.4s, v14.s[2] + + OP_rr v30.4s, v6.4s, v14.s[2] + OP_ii v30.4s, v7.4s, v14.s[3] + OP_ri v31.4s, v6.4s, v14.s[3] + OP_ir v31.4s, v7.4s, v14.s[2] +.endm + +.macro KERNEL8x4_SUB + ldr q8, [pB] + add pB, pB, #16 + + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + + OP_rr v16.4s, v0.4s, v8.s[0] + OP_ii v16.4s, v1.4s, v8.s[1] + OP_ri v17.4s, v0.4s, v8.s[1] + OP_ir v17.4s, v1.4s, v8.s[0] + + ld2 {v2.4s, v3.4s}, [pA] + add pA, pA, #32 + + OP_rr v20.4s, v0.4s, v8.s[2] + OP_ii v20.4s, v1.4s, v8.s[3] + OP_ri v21.4s, v0.4s, v8.s[3] + OP_ir v21.4s, v1.4s, v8.s[2] + + ldr q10, [pB] + add pB, pB, #16 + + OP_rr v18.4s, v2.4s, v8.s[0] + OP_ii v18.4s, v3.4s, v8.s[1] + OP_ri v19.4s, v2.4s, v8.s[1] + OP_ir v19.4s, v3.4s, v8.s[0] + + prfm PLDL1KEEP, [pA, #A_PRE_SIZE] + + OP_rr v22.4s, v2.4s, v8.s[2] + OP_ii v22.4s, v3.4s, v8.s[3] + OP_ri v23.4s, v2.4s, v8.s[3] + OP_ir v23.4s, v3.4s, v8.s[2] + + prfm PLDL1KEEP, [pA, #A_PRE_SIZE+64] + + OP_rr v24.4s, v0.4s, v10.s[0] + OP_ii v24.4s, v1.4s, v10.s[1] + OP_ri v25.4s, v0.4s, v10.s[1] + OP_ir v25.4s, v1.4s, v10.s[0] + + prfm PLDL1KEEP, [pB, #B_PRE_SIZE] + + OP_rr v26.4s, v2.4s, v10.s[0] + OP_ii v26.4s, v3.4s, v10.s[1] + OP_ri v27.4s, v2.4s, v10.s[1] + OP_ir v27.4s, v3.4s, v10.s[0] + + OP_rr v28.4s, v0.4s, v10.s[2] + OP_ii v28.4s, v1.4s, v10.s[3] + OP_ri v29.4s, v0.4s, v10.s[3] + OP_ir v29.4s, v1.4s, v10.s[2] + + OP_rr v30.4s, v2.4s, v10.s[2] + OP_ii v30.4s, v3.4s, v10.s[3] + OP_ri v31.4s, v2.4s, v10.s[3] + OP_ir v31.4s, v3.4s, v10.s[2] +.endm + +.macro SAVE8x4 + fmov alpha0_R, alphaR + fmov alpha0_I, alphaI + + prfm PLDL2KEEP, [pCRow0, #C_PRE_SIZE] + + ld2 {v0.4s, v1.4s}, [pCRow0] + fmla v0.4s, v16.4s, alphaV0_R + fmls v0.4s, v17.4s, alphaV0_I + fmla v1.4s, v16.4s, alphaV0_I + fmla v1.4s, v17.4s, alphaV0_R + st2 {v0.4s, v1.4s}, [pCRow0] + + add pCRow0, pCRow0, #32 + + ld2 {v2.4s, v3.4s}, [pCRow0] + fmla v2.4s, v18.4s, alphaV0_R + fmls v2.4s, v19.4s, alphaV0_I + fmla v3.4s, v18.4s, alphaV0_I + fmla v3.4s, v19.4s, alphaV0_R + st2 {v2.4s, v3.4s}, [pCRow0] + + add pCRow0, pCRow0, #32 + prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE] + + ld2 {v4.4s, v5.4s}, [pCRow1] + fmla v4.4s, v20.4s, alphaV0_R + fmls v4.4s, v21.4s, alphaV0_I + fmla v5.4s, v20.4s, alphaV0_I + fmla v5.4s, v21.4s, alphaV0_R + st2 {v4.4s, v5.4s}, [pCRow1] + + add pCRow1, pCRow1, #32 + + ld2 {v6.4s, v7.4s}, [pCRow1] + fmla v6.4s, v22.4s, alphaV0_R + fmls v6.4s, v23.4s, alphaV0_I + fmla v7.4s, v22.4s, alphaV0_I + fmla v7.4s, v23.4s, alphaV0_R + st2 {v6.4s, v7.4s}, [pCRow1] + + add pCRow1, pCRow1, #32 + prfm PLDL2KEEP, [pCRow2, #C_PRE_SIZE] + + ld2 {v0.4s, v1.4s}, [pCRow2] + fmla v0.4s, v24.4s, alphaV0_R + fmls v0.4s, v25.4s, alphaV0_I + fmla v1.4s, v24.4s, alphaV0_I + fmla v1.4s, v25.4s, alphaV0_R + st2 {v0.4s, v1.4s}, [pCRow2] + + add pCRow2, pCRow2, #32 + + ld2 {v2.4s, v3.4s}, [pCRow2] + fmla v2.4s, v26.4s, alphaV0_R + fmls v2.4s, v27.4s, alphaV0_I + fmla v3.4s, v26.4s, alphaV0_I + fmla v3.4s, v27.4s, alphaV0_R + st2 {v2.4s, v3.4s}, [pCRow2] + + add pCRow2, pCRow2, #32 + prfm PLDL2KEEP, [pCRow3, #C_PRE_SIZE] + + ld2 {v4.4s, v5.4s}, [pCRow3] + fmla v4.4s, v28.4s, alphaV0_R + fmls v4.4s, v29.4s, alphaV0_I + fmla v5.4s, v28.4s, alphaV0_I + fmla v5.4s, v29.4s, alphaV0_R + st2 {v4.4s, v5.4s}, [pCRow3] + + add pCRow3, pCRow3, #32 + + ld2 {v6.4s, v7.4s}, [pCRow3] + fmla v6.4s, v30.4s, alphaV0_R + fmls v6.4s, v31.4s, alphaV0_I + fmla v7.4s, v30.4s, alphaV0_I + fmla v7.4s, v31.4s, alphaV0_R + st2 {v6.4s, v7.4s}, [pCRow3] + + add pCRow3, pCRow3, #32 +.endm + +/******************************************************************************/ + +.macro INIT4x4 + fmov s16, wzr + fmov s17, s16 + fmov s20, s17 + fmov s21, s16 + fmov s24, s17 + fmov s25, s16 + fmov s28, s17 + fmov s29, s16 +.endm + +.macro KERNEL4x4_I + ld2 {v8.4s, v9.4s}, [pB] + add pB, pB, #32 + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + + fmul v16.4s, v0.4s, v8.s[0] + OP_ii v16.4s, v1.4s, v9.s[0] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v17.16b, v17.16b, v17.16b + fmls v17.4s, v0.4s, v9.s[0] +#else + fmul v17.4s, v0.4s, v9.s[0] +#endif + OP_ir v17.4s, v1.4s, v8.s[0] + + fmul v20.4s, v0.4s, v8.s[1] + OP_ii v20.4s, v1.4s, v9.s[1] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v21.16b, v21.16b, v21.16b + fmls v21.4s, v0.4s, v9.s[1] +#else + fmul v21.4s, v0.4s, v9.s[1] +#endif + OP_ir v21.4s, v1.4s, v8.s[1] + + fmul v24.4s, v0.4s, v8.s[2] + OP_ii v24.4s, v1.4s, v9.s[2] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v25.16b, v25.16b, v25.16b + fmls v25.4s, v0.4s, v9.s[2] +#else + fmul v25.4s, v0.4s, v9.s[2] +#endif + OP_ir v25.4s, v1.4s, v8.s[2] + + fmul v28.4s, v0.4s, v8.s[3] + OP_ii v28.4s, v1.4s, v9.s[3] +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ + defined(RR) || defined(RC) || defined(CR) || defined(CC) + eor v29.16b, v29.16b, v29.16b + fmls v29.4s, v0.4s, v9.s[3] +#else + fmul v29.4s, v0.4s, v9.s[3] +#endif + OP_ir v29.4s, v1.4s, v8.s[3] + + ld2 {v12.4s, v13.4s}, [pB] + add pB, pB, #32 + ld2 {v4.4s, v5.4s}, [pA] + add pA, pA, #32 +.endm + +.macro KERNEL4x4_M1 + OP_rr v16.4s, v0.4s, v8.s[0] + OP_ii v16.4s, v1.4s, v9.s[0] + OP_ri v17.4s, v0.4s, v9.s[0] + OP_ir v17.4s, v1.4s, v8.s[0] + + ld2 {v12.4s, v13.4s}, [pB] // For next round + add pB, pB, #32 + + OP_rr v20.4s, v0.4s, v8.s[1] + OP_ii v20.4s, v1.4s, v9.s[1] + OP_ri v21.4s, v0.4s, v9.s[1] + OP_ir v21.4s, v1.4s, v8.s[1] + + ld2 {v4.4s, v5.4s}, [pA] // For next round + add pA, pA, #32 + + OP_rr v24.4s, v0.4s, v8.s[2] + OP_ii v24.4s, v1.4s, v9.s[2] + OP_ri v25.4s, v0.4s, v9.s[2] + OP_ir v25.4s, v1.4s, v8.s[2] + + prfm PLDL1KEEP, [pA, #512] + + OP_rr v28.4s, v0.4s, v8.s[3] + OP_ii v28.4s, v1.4s, v9.s[3] + OP_ri v29.4s, v0.4s, v9.s[3] + OP_ir v29.4s, v1.4s, v8.s[3] +.endm + +.macro KERNEL4x4_M2 + OP_rr v16.4s, v4.4s, v12.s[0] + OP_ii v16.4s, v5.4s, v13.s[0] + OP_ri v17.4s, v4.4s, v13.s[0] + OP_ir v17.4s, v5.4s, v12.s[0] + + ld2 {v8.4s, v9.4s}, [pB] // For next round + add pB, pB, #32 + + OP_rr v20.4s, v4.4s, v12.s[1] + OP_ii v20.4s, v5.4s, v13.s[1] + OP_ri v21.4s, v4.4s, v13.s[1] + OP_ir v21.4s, v5.4s, v12.s[1] + + ld2 {v0.4s, v1.4s}, [pA] // For next round + add pA, pA, #32 + + OP_rr v24.4s, v4.4s, v12.s[2] + OP_ii v24.4s, v5.4s, v13.s[2] + OP_ri v25.4s, v4.4s, v13.s[2] + OP_ir v25.4s, v5.4s, v12.s[2] + + prfm PLDL1KEEP, [pB, #512] + + OP_rr v28.4s, v4.4s, v12.s[3] + OP_ii v28.4s, v5.4s, v13.s[3] + OP_ri v29.4s, v4.4s, v13.s[3] + OP_ir v29.4s, v5.4s, v12.s[3] +.endm + +.macro KERNEL4x4_E + OP_rr v16.4s, v4.4s, v12.s[0] + OP_ii v16.4s, v5.4s, v13.s[0] + OP_ri v17.4s, v4.4s, v13.s[0] + OP_ir v17.4s, v5.4s, v12.s[0] + + OP_rr v20.4s, v4.4s, v12.s[1] + OP_ii v20.4s, v5.4s, v13.s[1] + OP_ri v21.4s, v4.4s, v13.s[1] + OP_ir v21.4s, v5.4s, v12.s[1] + + OP_rr v24.4s, v4.4s, v12.s[2] + OP_ii v24.4s, v5.4s, v13.s[2] + OP_ri v25.4s, v4.4s, v13.s[2] + OP_ir v25.4s, v5.4s, v12.s[2] + + OP_rr v28.4s, v4.4s, v12.s[3] + OP_ii v28.4s, v5.4s, v13.s[3] + OP_ri v29.4s, v4.4s, v13.s[3] + OP_ir v29.4s, v5.4s, v12.s[3] +.endm + +.macro KERNEL4x4_SUB + ld2 {v8.4s, v9.4s}, [pB] + add pB, pB, #32 + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + + OP_rr v16.4s, v0.4s, v8.s[0] + OP_ii v16.4s, v1.4s, v9.s[0] + OP_ri v17.4s, v0.4s, v9.s[0] + OP_ir v17.4s, v1.4s, v8.s[0] + + OP_rr v20.4s, v0.4s, v8.s[1] + OP_ii v20.4s, v1.4s, v9.s[1] + OP_ri v21.4s, v0.4s, v9.s[1] + OP_ir v21.4s, v1.4s, v8.s[1] + + OP_rr v24.4s, v0.4s, v8.s[2] + OP_ii v24.4s, v1.4s, v9.s[2] + OP_ri v25.4s, v0.4s, v9.s[2] + OP_ir v25.4s, v1.4s, v8.s[2] + + OP_rr v28.4s, v0.4s, v8.s[3] + OP_ii v28.4s, v1.4s, v9.s[3] + OP_ri v29.4s, v0.4s, v9.s[3] + OP_ir v29.4s, v1.4s, v8.s[3] +.endm + +.macro SAVE4x4 + fmov alpha0_R, alphaR + fmov alpha0_I, alphaI + + mov pCRow1, pCRow0 + + ld2 {v0.4s, v1.4s}, [pCRow1] + fmla v0.4s, v16.4s, alphaV0_R + fmls v0.4s, v17.4s, alphaV0_I + fmla v1.4s, v16.4s, alphaV0_I + fmla v1.4s, v17.4s, alphaV0_R + st2 {v0.4s, v1.4s}, [pCRow1] + + add pCRow1, pCRow1, LDC + + ld2 {v4.4s, v5.4s}, [pCRow1] + fmla v4.4s, v20.4s, alphaV0_R + fmls v4.4s, v21.4s, alphaV0_I + fmla v5.4s, v20.4s, alphaV0_I + fmla v5.4s, v21.4s, alphaV0_R + st2 {v4.4s, v5.4s}, [pCRow1] + + add pCRow1, pCRow1, LDC + + ld2 {v0.4s, v1.4s}, [pCRow1] + fmla v0.4s, v24.4s, alphaV0_R + fmls v0.4s, v25.4s, alphaV0_I + fmla v1.4s, v24.4s, alphaV0_I + fmla v1.4s, v25.4s, alphaV0_R + st2 {v0.4s, v1.4s}, [pCRow1] + + add pCRow1, pCRow1, LDC + + ld2 {v4.4s, v5.4s}, [pCRow1] + fmla v4.4s, v28.4s, alphaV0_R + fmls v4.4s, v29.4s, alphaV0_I + fmla v5.4s, v28.4s, alphaV0_I + fmla v5.4s, v29.4s, alphaV0_R + st2 {v4.4s, v5.4s}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT2x4 + fmov s16, wzr + fmov s17, wzr + fmov s20, s16 + fmov s21, s17 + fmov s24, s16 + fmov s25, s17 + fmov s28, s16 + fmov s29, s17 +.endm + +.macro KERNEL2x4_SUB + ld2 {v8.4s, v9.4s}, [pB] + add pB, pB, #32 + ld2 {v0.2s, v1.2s}, [pA] + add pA, pA, #16 + + OP_rr v16.2s, v0.2s, v8.s[0] + OP_ii v16.2s, v1.2s, v9.s[0] + OP_ri v17.2s, v0.2s, v9.s[0] + OP_ir v17.2s, v1.2s, v8.s[0] + + OP_rr v20.2s, v0.2s, v8.s[1] + OP_ii v20.2s, v1.2s, v9.s[1] + OP_ri v21.2s, v0.2s, v9.s[1] + OP_ir v21.2s, v1.2s, v8.s[1] + + OP_rr v24.2s, v0.2s, v8.s[2] + OP_ii v24.2s, v1.2s, v9.s[2] + OP_ri v25.2s, v0.2s, v9.s[2] + OP_ir v25.2s, v1.2s, v8.s[2] + + OP_rr v28.2s, v0.2s, v8.s[3] + OP_ii v28.2s, v1.2s, v9.s[3] + OP_ri v29.2s, v0.2s, v9.s[3] + OP_ir v29.2s, v1.2s, v8.s[3] +.endm + +.macro SAVE2x4 + fmov alpha0_R, alphaR + fmov alpha0_I, alphaI + + mov pCRow1, pCRow0 + + ld2 {v0.2s, v1.2s}, [pCRow1] + fmla v0.2s, v16.2s, alphaV0_R + fmls v0.2s, v17.2s, alphaV0_I + fmla v1.2s, v16.2s, alphaV0_I + fmla v1.2s, v17.2s, alphaV0_R + st2 {v0.2s, v1.2s}, [pCRow1] + + add pCRow1, pCRow1, LDC + + ld2 {v4.2s, v5.2s}, [pCRow1] + fmla v4.2s, v20.2s, alphaV0_R + fmls v4.2s, v21.2s, alphaV0_I + fmla v5.2s, v20.2s, alphaV0_I + fmla v5.2s, v21.2s, alphaV0_R + st2 {v4.2s, v5.2s}, [pCRow1] + + add pCRow1, pCRow1, LDC + + ld2 {v0.2s, v1.2s}, [pCRow1] + fmla v0.2s, v24.2s, alphaV0_R + fmls v0.2s, v25.2s, alphaV0_I + fmla v1.2s, v24.2s, alphaV0_I + fmla v1.2s, v25.2s, alphaV0_R + st2 {v0.2s, v1.2s}, [pCRow1] + + add pCRow1, pCRow1, LDC + + ld2 {v4.2s, v5.2s}, [pCRow1] + fmla v4.2s, v28.2s, alphaV0_R + fmls v4.2s, v29.2s, alphaV0_I + fmla v5.2s, v28.2s, alphaV0_I + fmla v5.2s, v29.2s, alphaV0_R + st2 {v4.2s, v5.2s}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT1x4 + fmov s16, wzr + fmov s17, wzr + fmov s20, s16 + fmov s21, s17 + fmov s24, s16 + fmov s25, s17 + fmov s28, s16 + fmov s29, s17 +.endm + +.macro KERNEL1x4_SUB + ld2 {v8.4s, v9.4s}, [pB] + add pB, pB, #32 + ld2 {v0.s, v1.s}[0], [pA] + add pA, pA, #8 + + OP_rr s16, s0, v8.s[0] + OP_ii s16, s1, v9.s[0] + OP_ri s17, s0, v9.s[0] + OP_ir s17, s1, v8.s[0] + + OP_rr s20, s0, v8.s[1] + OP_ii s20, s1, v9.s[1] + OP_ri s21, s0, v9.s[1] + OP_ir s21, s1, v8.s[1] + + OP_rr s24, s0, v8.s[2] + OP_ii s24, s1, v9.s[2] + OP_ri s25, s0, v9.s[2] + OP_ir s25, s1, v8.s[2] + + OP_rr s28, s0, v8.s[3] + OP_ii s28, s1, v9.s[3] + OP_ri s29, s0, v9.s[3] + OP_ir s29, s1, v8.s[3] +.endm + +.macro SAVE1x4 + fmov alpha0_R, alphaR + fmov alpha0_I, alphaI + + mov pCRow1, pCRow0 + + ld2 {v0.s, v1.s}[0], [pCRow1] + fmla s0, s16, alphaV0_R + fmls s0, s17, alphaV0_I + fmla s1, s16, alphaV0_I + fmla s1, s17, alphaV0_R + st2 {v0.s, v1.s}[0], [pCRow1] + + add pCRow1, pCRow1, LDC + + ld2 {v4.s, v5.s}[0], [pCRow1] + fmla s4, s20, alphaV0_R + fmls s4, s21, alphaV0_I + fmla s5, s20, alphaV0_I + fmla s5, s21, alphaV0_R + st2 {v4.s, v5.s}[0], [pCRow1] + + add pCRow1, pCRow1, LDC + + ld2 {v0.s, v1.s}[0], [pCRow1] + fmla s0, s24, alphaV0_R + fmls s0, s25, alphaV0_I + fmla s1, s24, alphaV0_I + fmla s1, s25, alphaV0_R + st2 {v0.s, v1.s}[0], [pCRow1] + + add pCRow1, pCRow1, LDC + + ld2 {v4.s, v5.s}[0], [pCRow1] + fmla s4, s28, alphaV0_R + fmls s4, s29, alphaV0_I + fmla s5, s28, alphaV0_I + fmla s5, s29, alphaV0_R + st2 {v4.s, v5.s}[0], [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT8x2 + fmov s16, wzr + fmov s17, wzr + fmov s18, wzr + fmov s19, s16 + fmov s20, wzr + fmov s21, s16 + fmov s22, s17 + fmov s23, s18 +.endm + +.macro KERNEL8x2_SUB + ld2 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + ld2 {v2.4s, v3.4s}, [pA] + add pA, pA, #32 + + OP_rr v16.4s, v0.4s, v8.s[0] + OP_ii v16.4s, v1.4s, v9.s[0] + OP_ri v17.4s, v0.4s, v9.s[0] + OP_ir v17.4s, v1.4s, v8.s[0] + + OP_rr v18.4s, v2.4s, v8.s[0] + OP_ii v18.4s, v3.4s, v9.s[0] + OP_ri v19.4s, v2.4s, v9.s[0] + OP_ir v19.4s, v3.4s, v8.s[0] + + OP_rr v20.4s, v0.4s, v8.s[1] + OP_ii v20.4s, v1.4s, v9.s[1] + OP_ri v21.4s, v0.4s, v9.s[1] + OP_ir v21.4s, v1.4s, v8.s[1] + + OP_rr v22.4s, v2.4s, v8.s[1] + OP_ii v22.4s, v3.4s, v9.s[1] + OP_ri v23.4s, v2.4s, v9.s[1] + OP_ir v23.4s, v3.4s, v8.s[1] +.endm + +.macro SAVE8x2 + fmov alpha0_R, alphaR + fmov alpha0_I, alphaI + + mov pCRow1, pCRow0 + + ld2 {v0.4s, v1.4s}, [pCRow1] + fmla v0.4s, v16.4s, alphaV0_R + fmls v0.4s, v17.4s, alphaV0_I + fmla v1.4s, v16.4s, alphaV0_I + fmla v1.4s, v17.4s, alphaV0_R + st2 {v0.4s, v1.4s}, [pCRow1] + + add pCRow2, pCRow1, #32 + + ld2 {v2.4s, v3.4s}, [pCRow2] + fmla v2.4s, v18.4s, alphaV0_R + fmls v2.4s, v19.4s, alphaV0_I + fmla v3.4s, v18.4s, alphaV0_I + fmla v3.4s, v19.4s, alphaV0_R + st2 {v2.4s, v3.4s}, [pCRow2] + + add pCRow1, pCRow1, LDC + + ld2 {v4.4s, v5.4s}, [pCRow1] + fmla v4.4s, v20.4s, alphaV0_R + fmls v4.4s, v21.4s, alphaV0_I + fmla v5.4s, v20.4s, alphaV0_I + fmla v5.4s, v21.4s, alphaV0_R + st2 {v4.4s, v5.4s}, [pCRow1] + + add pCRow2, pCRow1, #32 + + ld2 {v6.4s, v7.4s}, [pCRow2] + fmla v6.4s, v22.4s, alphaV0_R + fmls v6.4s, v23.4s, alphaV0_I + fmla v7.4s, v22.4s, alphaV0_I + fmla v7.4s, v23.4s, alphaV0_R + st2 {v6.4s, v7.4s}, [pCRow2] + + add pCRow0, pCRow0, #64 +.endm + +/******************************************************************************/ + +.macro INIT4x2 + fmov s16, wzr + fmov s17, wzr + fmov s20, s16 + fmov s21, s17 +.endm + +.macro KERNEL4x2_SUB + ld2 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + + OP_rr v16.4s, v0.4s, v8.s[0] + OP_ii v16.4s, v1.4s, v9.s[0] + OP_ri v17.4s, v0.4s, v9.s[0] + OP_ir v17.4s, v1.4s, v8.s[0] + + OP_rr v20.4s, v0.4s, v8.s[1] + OP_ii v20.4s, v1.4s, v9.s[1] + OP_ri v21.4s, v0.4s, v9.s[1] + OP_ir v21.4s, v1.4s, v8.s[1] +.endm + +.macro SAVE4x2 + fmov alpha0_R, alphaR + fmov alpha0_I, alphaI + + mov pCRow1, pCRow0 + + ld2 {v0.4s, v1.4s}, [pCRow1] + fmla v0.4s, v16.4s, alphaV0_R + fmls v0.4s, v17.4s, alphaV0_I + fmla v1.4s, v16.4s, alphaV0_I + fmla v1.4s, v17.4s, alphaV0_R + st2 {v0.4s, v1.4s}, [pCRow1] + + add pCRow1, pCRow1, LDC + + ld2 {v4.4s, v5.4s}, [pCRow1] + fmla v4.4s, v20.4s, alphaV0_R + fmls v4.4s, v21.4s, alphaV0_I + fmla v5.4s, v20.4s, alphaV0_I + fmla v5.4s, v21.4s, alphaV0_R + st2 {v4.4s, v5.4s}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT2x2 + fmov s16, wzr + fmov s17, wzr + fmov s20, s16 + fmov s21, s17 +.endm + +.macro KERNEL2x2_SUB + ld2 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld2 {v0.2s, v1.2s}, [pA] + add pA, pA, #16 + + OP_rr v16.2s, v0.2s, v8.s[0] + OP_ii v16.2s, v1.2s, v9.s[0] + OP_ri v17.2s, v0.2s, v9.s[0] + OP_ir v17.2s, v1.2s, v8.s[0] + + OP_rr v20.2s, v0.2s, v8.s[1] + OP_ii v20.2s, v1.2s, v9.s[1] + OP_ri v21.2s, v0.2s, v9.s[1] + OP_ir v21.2s, v1.2s, v8.s[1] +.endm + +.macro SAVE2x2 + fmov alpha0_R, alphaR + fmov alpha0_I, alphaI + + mov pCRow1, pCRow0 + + ld2 {v0.2s, v1.2s}, [pCRow1] + fmla v0.2s, v16.2s, alphaV0_R + fmls v0.2s, v17.2s, alphaV0_I + fmla v1.2s, v16.2s, alphaV0_I + fmla v1.2s, v17.2s, alphaV0_R + st2 {v0.2s, v1.2s}, [pCRow1] + + add pCRow1, pCRow1, LDC + + ld2 {v4.2s, v5.2s}, [pCRow1] + fmla v4.2s, v20.2s, alphaV0_R + fmls v4.2s, v21.2s, alphaV0_I + fmla v5.2s, v20.2s, alphaV0_I + fmla v5.2s, v21.2s, alphaV0_R + st2 {v4.2s, v5.2s}, [pCRow1] + + add pCRow0, pCRow0, #16 +.endm + +/******************************************************************************/ + +.macro INIT1x2 + fmov s16, wzr + fmov s17, wzr + fmov s20, wzr + fmov s21, wzr +.endm + +.macro KERNEL1x2_SUB + ld2 {v8.2s, v9.2s}, [pB] + add pB, pB, #16 + ld2 {v0.s, v1.s}[0], [pA] + add pA, pA, #8 + + OP_rr s16, s0, v8.s[0] + OP_ii s16, s1, v9.s[0] + OP_ri s17, s0, v9.s[0] + OP_ir s17, s1, v8.s[0] + + OP_rr s20, s0, v8.s[1] + OP_ii s20, s1, v9.s[1] + OP_ri s21, s0, v9.s[1] + OP_ir s21, s1, v8.s[1] +.endm + +.macro SAVE1x2 + fmov alpha0_R, alphaR + fmov alpha0_I, alphaI + + mov pCRow1, pCRow0 + + ld2 {v0.s, v1.s}[0], [pCRow1] + fmla s0, s16, alphaV0_R + fmls s0, s17, alphaV0_I + fmla s1, s16, alphaV0_I + fmla s1, s17, alphaV0_R + st2 {v0.s, v1.s}[0], [pCRow1] + + add pCRow1, pCRow1, LDC + + ld2 {v4.s, v5.s}[0], [pCRow1] + fmla s4, s20, alphaV0_R + fmls s4, s21, alphaV0_I + fmla s5, s20, alphaV0_I + fmla s5, s21, alphaV0_R + st2 {v4.s, v5.s}[0], [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +/******************************************************************************/ + +.macro INIT8x1 + fmov s16, wzr + fmov s17, wzr + fmov s18, wzr + fmov s19, s16 +.endm + +.macro KERNEL8x1_SUB + ld1 {v8.2s}, [pB] + add pB, pB, #8 + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + ld2 {v2.4s, v3.4s}, [pA] + add pA, pA, #32 + + OP_rr v16.4s, v0.4s, v8.s[0] + OP_ii v16.4s, v1.4s, v8.s[1] + OP_ri v17.4s, v0.4s, v8.s[1] + OP_ir v17.4s, v1.4s, v8.s[0] + + OP_rr v18.4s, v2.4s, v8.s[0] + OP_ii v18.4s, v3.4s, v8.s[1] + OP_ri v19.4s, v2.4s, v8.s[1] + OP_ir v19.4s, v3.4s, v8.s[0] +.endm + +.macro SAVE8x1 + fmov alpha0_R, alphaR + fmov alpha0_I, alphaI + + mov pCRow1, pCRow0 + + ld2 {v0.4s, v1.4s}, [pCRow1] + fmla v0.4s, v16.4s, alphaV0_R + fmls v0.4s, v17.4s, alphaV0_I + fmla v1.4s, v16.4s, alphaV0_I + fmla v1.4s, v17.4s, alphaV0_R + st2 {v0.4s, v1.4s}, [pCRow1] + + add pCRow1, pCRow1, #32 + + ld2 {v2.4s, v3.4s}, [pCRow1] + fmla v2.4s, v18.4s, alphaV0_R + fmls v2.4s, v19.4s, alphaV0_I + fmla v3.4s, v18.4s, alphaV0_I + fmla v3.4s, v19.4s, alphaV0_R + st2 {v2.4s, v3.4s}, [pCRow1] + + add pCRow0, pCRow0, #64 +.endm + + +/******************************************************************************/ + +.macro INIT4x1 + fmov s16, wzr + fmov s17, s16 +.endm + +.macro KERNEL4x1_SUB + ld2 {v8.s, v9.s}[0], [pB] + add pB, pB, #8 + ld2 {v0.4s, v1.4s}, [pA] + add pA, pA, #32 + + OP_rr v16.4s, v0.4s, v8.s[0] + OP_ii v16.4s, v1.4s, v9.s[0] + OP_ri v17.4s, v0.4s, v9.s[0] + OP_ir v17.4s, v1.4s, v8.s[0] +.endm + +.macro SAVE4x1 + fmov alpha0_R, alphaR + fmov alpha0_I, alphaI + + mov pCRow1, pCRow0 + + ld2 {v0.4s, v1.4s}, [pCRow1] + fmla v0.4s, v16.4s, alphaV0_R + fmls v0.4s, v17.4s, alphaV0_I + fmla v1.4s, v16.4s, alphaV0_I + fmla v1.4s, v17.4s, alphaV0_R + st2 {v0.4s, v1.4s}, [pCRow1] + + add pCRow0, pCRow0, #32 +.endm + +/******************************************************************************/ + +.macro INIT2x1 + fmov s16, wzr + fmov s17, wzr +.endm + +.macro KERNEL2x1_SUB + ld2 {v8.s, v9.s}[0], [pB] + add pB, pB, #8 + ld2 {v0.2s, v1.2s}, [pA] + add pA, pA, #16 + + OP_rr v16.2s, v0.2s, v8.s[0] + OP_ii v16.2s, v1.2s, v9.s[0] + OP_ri v17.2s, v0.2s, v9.s[0] + OP_ir v17.2s, v1.2s, v8.s[0] +.endm + +.macro SAVE2x1 + fmov alpha0_R, alphaR + fmov alpha0_I, alphaI + + mov pCRow1, pCRow0 + + ld2 {v0.2s, v1.2s}, [pCRow1] + fmla v0.2s, v16.2s, alphaV0_R + fmls v0.2s, v17.2s, alphaV0_I + fmla v1.2s, v16.2s, alphaV0_I + fmla v1.2s, v17.2s, alphaV0_R + st2 {v0.2s, v1.2s}, [pCRow1] + + add pCRow0, pCRow0, #16 + +.endm + +/******************************************************************************/ + +.macro INIT1x1 + fmov s16, wzr + fmov s17, wzr +.endm + +.macro KERNEL1x1_SUB + ld2 {v8.s, v9.s}[0], [pB] + add pB, pB, #8 + ld2 {v0.s, v1.s}[0], [pA] + add pA, pA, #8 + + OP_rr s16, s0, v8.s[0] + OP_ii s16, s1, v9.s[0] + OP_ri s17, s0, v9.s[0] + OP_ir s17, s1, v8.s[0] +.endm + +.macro SAVE1x1 + fmov alpha0_R, alphaR + fmov alpha0_I, alphaI + + mov pCRow1, pCRow0 + + ld2 {v0.s, v1.s}[0], [pCRow1] + fmla s0, s16, alphaV0_R + fmls s0, s17, alphaV0_I + fmla s1, s16, alphaV0_I + fmla s1, s17, alphaV0_R + st2 {v0.s, v1.s}[0], [pCRow1] + + add pCRow0, pCRow0, #8 +.endm + +.macro KERNEL8x4_M1_M2_x1 + KERNEL8x4_M1 + KERNEL8x4_M2 +.endm + +.macro KERNEL8x4_M1_M2_x2 + KERNEL8x4_M1_M2_x1 + KERNEL8x4_M1_M2_x1 +.endm + +.macro KERNEL8x4_M1_M2_x4 + KERNEL8x4_M1_M2_x2 + KERNEL8x4_M1_M2_x2 +.endm + +.macro KERNEL8x4_M1_M2_x8 + KERNEL8x4_M1_M2_x4 + KERNEL8x4_M1_M2_x4 +.endm + +.macro KERNEL8x4_M1_M2_x16 + KERNEL8x4_M1_M2_x8 + KERNEL8x4_M1_M2_x8 +.endm + +/******************************************************************************* +* End of macro definitions +*******************************************************************************/ + + PROLOGUE + + .align 5 + add sp, sp, #-(11 * 16) + stp d8, d9, [sp, #(0 * 16)] + stp d10, d11, [sp, #(1 * 16)] + stp d12, d13, [sp, #(2 * 16)] + stp d14, d15, [sp, #(3 * 16)] + stp d16, d17, [sp, #(4 * 16)] + stp x18, x19, [sp, #(5 * 16)] + stp x20, x21, [sp, #(6 * 16)] + stp x22, x23, [sp, #(7 * 16)] + stp x24, x25, [sp, #(8 * 16)] + stp x26, x27, [sp, #(9 * 16)] + str x28, [sp, #(10 * 16)] + + prfm PLDL1KEEP, [origPB] + prfm PLDL1KEEP, [origPA] + + fmov alphaR, s0 + fmov alphaI, s1 + + lsl LDC, LDC, #3 // ldc = ldc * 8 + + mov pB, origPB + + mov counterJ, origN + asr counterJ, counterJ, #2 // J = J / 4 + cmp counterJ, #0 + ble cgemm_kernel_L2_BEGIN + +/******************************************************************************/ + +cgemm_kernel_L4_BEGIN: + mov pCRow0, pC + add pCRow1, pCRow0, LDC + add pCRow2, pCRow1, LDC + add pCRow3, pCRow2, LDC + + add pC, pCRow3, LDC + + mov pA, origPA // pA = start of A array + +cgemm_kernel_L4_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 // counterI = counterI / 8 + cmp counterI, #0 + ble cgemm_kernel_L4_M4_BEGIN + + .align 5 +cgemm_kernel_L4_M8_20: + + mov pB, origPB + + asr counterL , origK, #5 // origK / 32 + cmp counterL , #2 + blt cgemm_kernel_L4_M8_32 + + KERNEL8x4_I + KERNEL8x4_M2 + KERNEL8x4_M1_M2_x1 + KERNEL8x4_M1_M2_x2 + KERNEL8x4_M1_M2_x4 + KERNEL8x4_M1_M2_x8 + + subs counterL, counterL, #2 // subtract 2 + ble cgemm_kernel_L4_M8_22a + + .align 5 +cgemm_kernel_L4_M8_22: + + KERNEL8x4_M1_M2_x16 + + subs counterL, counterL, #1 + bgt cgemm_kernel_L4_M8_22 + + .align 5 +cgemm_kernel_L4_M8_22a: + + KERNEL8x4_M1_M2_x8 + KERNEL8x4_M1_M2_x4 + KERNEL8x4_M1_M2_x2 + KERNEL8x4_M1_M2_x1 + KERNEL8x4_M1 + KERNEL8x4_E + + b cgemm_kernel_L4_M8_44 + + .align 5 +cgemm_kernel_L4_M8_32: + + tst counterL, #1 + ble cgemm_kernel_L4_M8_40 + + KERNEL8x4_I + KERNEL8x4_M2 + KERNEL8x4_M1_M2_x8 + KERNEL8x4_M1_M2_x4 + KERNEL8x4_M1_M2_x2 + KERNEL8x4_M1 + KERNEL8x4_E + + b cgemm_kernel_L4_M8_44 + +cgemm_kernel_L4_M8_40: + + INIT8x4 + +cgemm_kernel_L4_M8_44: + + ands counterL , origK, #31 + ble cgemm_kernel_L4_M8_100 + + .align 5 +cgemm_kernel_L4_M8_46: + + KERNEL8x4_SUB + + subs counterL, counterL, #1 + bne cgemm_kernel_L4_M8_46 + +cgemm_kernel_L4_M8_100: + prfm PLDL1KEEP, [pA] + prfm PLDL1KEEP, [pA, #64] + prfm PLDL1KEEP, [origPB] + + SAVE8x4 + +cgemm_kernel_L4_M8_END: + subs counterI, counterI, #1 + bne cgemm_kernel_L4_M8_20 + +cgemm_kernel_L4_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble cgemm_kernel_L4_END + + tst counterI, #4 + ble cgemm_kernel_L4_M2_BEGIN + + +cgemm_kernel_L4_M4_20: + + mov pB, origPB + + asr counterL , origK, #1 // L = K / 2 + cmp counterL , #2 // is there at least 4 to do? + blt cgemm_kernel_L4_M4_32 + + KERNEL4x4_I // do one in the K + KERNEL4x4_M2 // do another in the K + + subs counterL, counterL, #2 + ble cgemm_kernel_L4_M4_22a + .align 5 + + +cgemm_kernel_L4_M4_22: + + KERNEL4x4_M1 + KERNEL4x4_M2 + + subs counterL, counterL, #1 + bgt cgemm_kernel_L4_M4_22 + +cgemm_kernel_L4_M4_22a: + KERNEL4x4_M1 + KERNEL4x4_E + b cgemm_kernel_L4_M4_44 +cgemm_kernel_L4_M4_32: + tst counterL, #1 + ble cgemm_kernel_L4_M4_40 + KERNEL4x4_I + KERNEL4x4_E + b cgemm_kernel_L4_M4_44 +cgemm_kernel_L4_M4_40: + + INIT4x4 + +cgemm_kernel_L4_M4_44: + ands counterL , origK, #1 + ble cgemm_kernel_L4_M4_100 + +cgemm_kernel_L4_M4_46: + KERNEL4x4_SUB + +cgemm_kernel_L4_M4_100: + + SAVE4x4 + +cgemm_kernel_L4_M4_END: + +cgemm_kernel_L4_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble cgemm_kernel_L4_END + + tst counterI, #2 // counterI = counterI / 2 + ble cgemm_kernel_L4_M1_BEGIN + +cgemm_kernel_L4_M2_20: + + INIT2x4 + + mov pB, origPB + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble cgemm_kernel_L4_M2_40 + +cgemm_kernel_L4_M2_22: + + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + KERNEL2x4_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L4_M2_22 + + +cgemm_kernel_L4_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble cgemm_kernel_L4_M2_100 + +cgemm_kernel_L4_M2_42: + + KERNEL2x4_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L4_M2_42 + +cgemm_kernel_L4_M2_100: + + SAVE2x4 + +cgemm_kernel_L4_M2_END: + + +cgemm_kernel_L4_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble cgemm_kernel_L4_END + +cgemm_kernel_L4_M1_20: + + INIT1x4 + + mov pB, origPB + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble cgemm_kernel_L4_M1_40 + +cgemm_kernel_L4_M1_22: + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + KERNEL1x4_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L4_M1_22 + + +cgemm_kernel_L4_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble cgemm_kernel_L4_M1_100 + +cgemm_kernel_L4_M1_42: + + KERNEL1x4_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L4_M1_42 + +cgemm_kernel_L4_M1_100: + + SAVE1x4 + + +cgemm_kernel_L4_END: + + lsl temp, origK, #5 + add origPB, origPB, temp // B = B + K * 4 * 8 + + subs counterJ, counterJ , #1 // j-- + bgt cgemm_kernel_L4_BEGIN + + +/******************************************************************************/ + +cgemm_kernel_L2_BEGIN: // less than 2 left in N direction + + mov counterJ , origN + tst counterJ , #3 + ble cgemm_kernel_L999 // error, N was less than 4? + + tst counterJ , #2 + ble cgemm_kernel_L1_BEGIN + + mov pCRow0, pC // pCRow0 = pC + + add pC,pC,LDC, lsl #1 + + mov pA, origPA // pA = A + + +cgemm_kernel_L2_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 // counterI = counterI / 8 + cmp counterI, #0 + ble cgemm_kernel_L2_M4_BEGIN + +cgemm_kernel_L2_M8_20: + + INIT8x2 + + mov pB, origPB + + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble cgemm_kernel_L2_M8_40 + .align 5 + +cgemm_kernel_L2_M8_22: + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + KERNEL8x2_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L2_M8_22 + + +cgemm_kernel_L2_M8_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble cgemm_kernel_L2_M8_100 + +cgemm_kernel_L2_M8_42: + + KERNEL8x2_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L2_M8_42 + +cgemm_kernel_L2_M8_100: + + SAVE8x2 + +cgemm_kernel_L2_M8_END: + + subs counterI, counterI, #1 + bgt cgemm_kernel_L2_M8_20 + +cgemm_kernel_L2_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble cgemm_kernel_L2_END + + tst counterI, #4 // counterI = counterI / 2 + ble cgemm_kernel_L2_M2_BEGIN + +cgemm_kernel_L2_M4_20: + + INIT4x2 + + mov pB, origPB + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble cgemm_kernel_L2_M4_40 + .align 5 + +cgemm_kernel_L2_M4_22: + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + KERNEL4x2_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L2_M4_22 + + +cgemm_kernel_L2_M4_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble cgemm_kernel_L2_M4_100 + +cgemm_kernel_L2_M4_42: + + KERNEL4x2_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L2_M4_42 + +cgemm_kernel_L2_M4_100: + + SAVE4x2 + +cgemm_kernel_L2_M4_END: + +cgemm_kernel_L2_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble cgemm_kernel_L2_END + + tst counterI, #2 // counterI = counterI / 2 + ble cgemm_kernel_L2_M1_BEGIN + +cgemm_kernel_L2_M2_20: + + INIT2x2 + + mov pB, origPB + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL,#0 + ble cgemm_kernel_L2_M2_40 + +cgemm_kernel_L2_M2_22: + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L2_M2_22 + + +cgemm_kernel_L2_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble cgemm_kernel_L2_M2_100 + +cgemm_kernel_L2_M2_42: + + KERNEL2x2_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L2_M2_42 + +cgemm_kernel_L2_M2_100: + + SAVE2x2 + +cgemm_kernel_L2_M2_END: + + +cgemm_kernel_L2_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble cgemm_kernel_L2_END + +cgemm_kernel_L2_M1_20: + + INIT1x2 + + mov pB, origPB + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL, #0 + ble cgemm_kernel_L2_M1_40 + +cgemm_kernel_L2_M1_22: + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L2_M1_22 + + +cgemm_kernel_L2_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble cgemm_kernel_L2_M1_100 + +cgemm_kernel_L2_M1_42: + + KERNEL1x2_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L2_M1_42 + +cgemm_kernel_L2_M1_100: + + SAVE1x2 + + +cgemm_kernel_L2_END: + add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 + +/******************************************************************************/ + +cgemm_kernel_L1_BEGIN: + + mov counterJ , origN + tst counterJ , #1 + ble cgemm_kernel_L999 // done + + + mov pCRow0, pC // pCRow0 = C + add pC , pC , LDC // Update pC to point to next + + mov pA, origPA // pA = A + + +cgemm_kernel_L1_M8_BEGIN: + + mov counterI, origM + asr counterI, counterI, #3 // counterI = counterI / 8 + cmp counterI, #0 + ble cgemm_kernel_L1_M4_BEGIN + +cgemm_kernel_L1_M8_20: + + INIT8x1 + + mov pB, origPB + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble cgemm_kernel_L1_M8_40 + .align 5 + +cgemm_kernel_L1_M8_22: + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + KERNEL8x1_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L1_M8_22 + + +cgemm_kernel_L1_M8_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble cgemm_kernel_L1_M8_100 + +cgemm_kernel_L1_M8_42: + + KERNEL8x1_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L1_M8_42 + +cgemm_kernel_L1_M8_100: + + SAVE8x1 + +cgemm_kernel_L1_M8_END: + + subs counterI, counterI, #1 + bgt cgemm_kernel_L1_M8_20 + +cgemm_kernel_L1_M4_BEGIN: + + mov counterI, origM + tst counterI , #7 + ble cgemm_kernel_L1_END + + tst counterI, #4 // counterI = counterI / 2 + ble cgemm_kernel_L1_M2_BEGIN + + +cgemm_kernel_L1_M4_20: + + INIT4x1 + + mov pB, origPB + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble cgemm_kernel_L1_M4_40 + .align 5 + +cgemm_kernel_L1_M4_22: + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + KERNEL4x1_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L1_M4_22 + + +cgemm_kernel_L1_M4_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble cgemm_kernel_L1_M4_100 + +cgemm_kernel_L1_M4_42: + + KERNEL4x1_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L1_M4_42 + +cgemm_kernel_L1_M4_100: + + SAVE4x1 + +cgemm_kernel_L1_M4_END: + + +cgemm_kernel_L1_M2_BEGIN: + + mov counterI, origM + tst counterI , #3 + ble cgemm_kernel_L1_END + + tst counterI, #2 // counterI = counterI / 2 + ble cgemm_kernel_L1_M1_BEGIN + +cgemm_kernel_L1_M2_20: + + INIT2x1 + + mov pB, origPB + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble cgemm_kernel_L1_M2_40 + +cgemm_kernel_L1_M2_22: + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L1_M2_22 + + +cgemm_kernel_L1_M2_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble cgemm_kernel_L1_M2_100 + +cgemm_kernel_L1_M2_42: + + KERNEL2x1_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L1_M2_42 + +cgemm_kernel_L1_M2_100: + + SAVE2x1 + +cgemm_kernel_L1_M2_END: + + +cgemm_kernel_L1_M1_BEGIN: + + tst counterI, #1 // counterI = counterI % 2 + ble cgemm_kernel_L1_END + +cgemm_kernel_L1_M1_20: + + INIT1x1 + + mov pB, origPB + asr counterL , origK, #3 // counterL = counterL / 8 + cmp counterL , #0 + ble cgemm_kernel_L1_M1_40 + +cgemm_kernel_L1_M1_22: + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L1_M1_22 + + +cgemm_kernel_L1_M1_40: + + ands counterL , origK, #7 // counterL = counterL % 8 + ble cgemm_kernel_L1_M1_100 + +cgemm_kernel_L1_M1_42: + + KERNEL1x1_SUB + + subs counterL, counterL, #1 + bgt cgemm_kernel_L1_M1_42 + +cgemm_kernel_L1_M1_100: + + SAVE1x1 + + +cgemm_kernel_L1_END: + + +cgemm_kernel_L999: + mov x0, #0 // set return value + ldp d8, d9, [sp, #(0 * 16)] + ldp d10, d11, [sp, #(1 * 16)] + ldp d12, d13, [sp, #(2 * 16)] + ldp d14, d15, [sp, #(3 * 16)] + ldp d16, d17, [sp, #(4 * 16)] + ldp x18, x19, [sp, #(5 * 16)] + ldp x20, x21, [sp, #(6 * 16)] + ldp x22, x23, [sp, #(7 * 16)] + ldp x24, x25, [sp, #(8 * 16)] + ldp x26, x27, [sp, #(9 * 16)] + ldr x28, [sp, #(10 * 16)] + add sp, sp, #(11*16) + ret + + EPILOGUE + diff --git a/param.h b/param.h index 1ceccd52a..59e5cc895 100644 --- a/param.h +++ b/param.h @@ -2303,44 +2303,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define ZGEMM_DEFAULT_R 4096 -#define SYMV_P 16 -#endif - -#if defined(VULCAN) -#define SNUMOPT 2 -#define DNUMOPT 2 - -#define GEMM_DEFAULT_OFFSET_A 0 -#define GEMM_DEFAULT_OFFSET_B 0 -#define GEMM_DEFAULT_ALIGN 0x03fffUL - -#define SGEMM_DEFAULT_UNROLL_M 16 -#define SGEMM_DEFAULT_UNROLL_N 4 - -#define DGEMM_DEFAULT_UNROLL_M 8 -#define DGEMM_DEFAULT_UNROLL_N 4 - -#define CGEMM_DEFAULT_UNROLL_M 8 -#define CGEMM_DEFAULT_UNROLL_N 4 - -#define ZGEMM_DEFAULT_UNROLL_M 4 -#define ZGEMM_DEFAULT_UNROLL_N 4 - -#define SGEMM_DEFAULT_P sgemm_p -#define DGEMM_DEFAULT_P dgemm_p -#define CGEMM_DEFAULT_P 256 -#define ZGEMM_DEFAULT_P 128 - -#define SGEMM_DEFAULT_Q sgemm_q -#define DGEMM_DEFAULT_Q dgemm_q -#define CGEMM_DEFAULT_Q 512 -#define ZGEMM_DEFAULT_Q 512 - -#define SGEMM_DEFAULT_R sgemm_r -#define DGEMM_DEFAULT_R dgemm_r -#define CGEMM_DEFAULT_R 4096 -#define ZGEMM_DEFAULT_R 2048 - #define SYMV_P 16 #endif @@ -2462,7 +2424,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define SYMV_P 16 #endif -#if defined(THUNDERX2T99) +#if defined(THUNDERX2T99) || defined(VULCAN) #define SNUMOPT 2 #define DNUMOPT 2 @@ -2484,17 +2446,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define SGEMM_DEFAULT_P sgemm_p #define DGEMM_DEFAULT_P dgemm_p -#define CGEMM_DEFAULT_P 256 +#define CGEMM_DEFAULT_P cgemm_p #define ZGEMM_DEFAULT_P 128 #define SGEMM_DEFAULT_Q sgemm_q #define DGEMM_DEFAULT_Q dgemm_q -#define CGEMM_DEFAULT_Q 512 +#define CGEMM_DEFAULT_Q cgemm_q #define ZGEMM_DEFAULT_Q 512 #define SGEMM_DEFAULT_R sgemm_r #define DGEMM_DEFAULT_R dgemm_r -#define CGEMM_DEFAULT_R 4096 +#define CGEMM_DEFAULT_R cgemm_r #define ZGEMM_DEFAULT_R 2048 #define SYMV_P 16 From e0dc5f58c5688df306291c657c1d9115688248d7 Mon Sep 17 00:00:00 2001 From: Ashwin Sekhar T K Date: Fri, 27 Jan 2017 01:11:58 -0800 Subject: [PATCH 07/12] THUNDERX2T99: Remove Duplicate Code --- kernel/arm64/KERNEL.THUNDERX2T99 | 12 +- kernel/arm64/ddot_thunderx2t99.S | 207 ---------------- ...erx2t99_threaded.c => ddot_thunderx2t99.c} | 12 +- kernel/arm64/snrm2_thunderx2t99.S | 228 ------------------ ...rx2t99_threaded.c => snrm2_thunderx2t99.c} | 15 +- 5 files changed, 24 insertions(+), 450 deletions(-) delete mode 100644 kernel/arm64/ddot_thunderx2t99.S rename kernel/arm64/{ddot_thunderx2t99_threaded.c => ddot_thunderx2t99.c} (98%) delete mode 100644 kernel/arm64/snrm2_thunderx2t99.S rename kernel/arm64/{nrm2_thunderx2t99_threaded.c => snrm2_thunderx2t99.c} (98%) diff --git a/kernel/arm64/KERNEL.THUNDERX2T99 b/kernel/arm64/KERNEL.THUNDERX2T99 index 94132bf60..c8668b67f 100644 --- a/kernel/arm64/KERNEL.THUNDERX2T99 +++ b/kernel/arm64/KERNEL.THUNDERX2T99 @@ -1,19 +1,11 @@ include $(KERNELDIR)/KERNEL.CORTEXA57 -ifndef SMP -SNRM2KERNEL = snrm2_thunderx2t99.S -else -SNRM2KERNEL = nrm2_thunderx2t99_threaded.c -endif +SNRM2KERNEL = snrm2_thunderx2t99.c CNRM2KERNEL = cnrm2_thunderx2t99.S DAXPYKERNEL = daxpy_thunderx2t99.S -ifndef SMP -DDOTKERNEL = ddot_thunderx2t99.S -else -DDOTKERNEL = ddot_thunderx2t99_threaded.c -endif +DDOTKERNEL = ddot_thunderx2t99.c ifeq ($(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N), 8x4) DGEMMKERNEL = dgemm_kernel_8x4_thunderx2t99.S diff --git a/kernel/arm64/ddot_thunderx2t99.S b/kernel/arm64/ddot_thunderx2t99.S deleted file mode 100644 index 5fa39adf6..000000000 --- a/kernel/arm64/ddot_thunderx2t99.S +++ /dev/null @@ -1,207 +0,0 @@ -/******************************************************************************* -Copyright (c) 2017, The OpenBLAS Project -All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: -1. Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in -the documentation and/or other materials provided with the -distribution. -3. Neither the name of the OpenBLAS project nor the names of -its contributors may be used to endorse or promote products -derived from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*******************************************************************************/ - -#define ASSEMBLER -#include "common.h" - -#define N x0 /* vector length */ -#define X x1 /* X vector address */ -#define INC_X x2 /* X stride */ -#define Y x3 /* Y vector address */ -#define INC_Y x4 /* Y stride */ -#define I x5 /* loop variable */ - -/******************************************************************************* -* Macro definitions -*******************************************************************************/ - -#define REG0 xzr -#define DOTF d0 -#define TMPX d16 -#define LD1VX {v16.d}[0] -#define TMPY d24 -#define LD1VY {v24.d}[0] -#define SZ 8 - -/******************************************************************************/ - -.macro KERNEL_F1 - ldr TMPX, [X] - ldr TMPY, [Y] - add X, X, #SZ - add Y, Y, #SZ - fmadd DOTF, TMPX, TMPY, DOTF -.endm - -.macro KERNEL_F16 - ldp q16, q17, [X] - ldp q24, q25, [Y] - - ldp q18, q19, [X, #32] - ldp q26, q27, [Y, #32] - - fmla v0.2d, v16.2d, v24.2d - fmla v1.2d, v17.2d, v25.2d - - ldp q20, q21, [X, #64] - ldp q28, q29, [Y, #64] - - fmla v2.2d, v18.2d, v26.2d - fmla v3.2d, v19.2d, v27.2d - - ldp q22, q23, [X, #96] - ldp q30, q31, [Y, #96] - - add Y, Y, #128 - add X, X, #128 - - fmla v4.2d, v20.2d, v28.2d - fmla v5.2d, v21.2d, v29.2d - - PRFM PLDL1KEEP, [X, #896] - PRFM PLDL1KEEP, [Y, #896] - PRFM PLDL1KEEP, [X, #896+64] - PRFM PLDL1KEEP, [Y, #896+64] - - fmla v6.2d, v22.2d, v30.2d - fmla v7.2d, v23.2d, v31.2d -.endm - -.macro KERNEL_F32 - KERNEL_F16 - KERNEL_F16 -.endm - -.macro KERNEL_F32_FINALIZE - fadd v0.2d, v0.2d, v1.2d - fadd v2.2d, v2.2d, v3.2d - fadd v4.2d, v4.2d, v5.2d - fadd v6.2d, v6.2d, v7.2d - fadd v0.2d, v0.2d, v2.2d - fadd v4.2d, v4.2d, v6.2d - fadd v0.2d, v0.2d, v4.2d - faddp DOTF, v0.2d -.endm - -.macro INIT_S - lsl INC_X, INC_X, #3 - lsl INC_Y, INC_Y, #3 -.endm - -.macro KERNEL_S1 - ld1 LD1VX, [X], INC_X - ld1 LD1VY, [Y], INC_Y - fmadd DOTF, TMPX, TMPY, DOTF -.endm - -/******************************************************************************* -* End of macro definitions -*******************************************************************************/ - - PROLOGUE - - fmov DOTF, REG0 - fmov d1, REG0 - fmov d2, REG0 - fmov d3, REG0 - fmov d4, REG0 - fmov d5, REG0 - fmov d6, REG0 - fmov d7, REG0 - - cmp N, xzr - ble dot_kernel_L999 - - cmp INC_X, #1 - bne dot_kernel_S_BEGIN - cmp INC_Y, #1 - bne dot_kernel_S_BEGIN - -dot_kernel_F_BEGIN: - - asr I, N, #5 - cmp I, xzr - beq dot_kernel_F1 - -dot_kernel_F32: - - KERNEL_F32 - - subs I, I, #1 - bne dot_kernel_F32 - - KERNEL_F32_FINALIZE - -dot_kernel_F1: - - ands I, N, #31 - ble dot_kernel_L999 - -dot_kernel_F10: - - KERNEL_F1 - - subs I, I, #1 - bne dot_kernel_F10 - - ret - -dot_kernel_S_BEGIN: - - INIT_S - - asr I, N, #2 - cmp I, xzr - ble dot_kernel_S1 - -dot_kernel_S4: - - KERNEL_S1 - KERNEL_S1 - KERNEL_S1 - KERNEL_S1 - - subs I, I, #1 - bne dot_kernel_S4 - -dot_kernel_S1: - - ands I, N, #3 - ble dot_kernel_L999 - -dot_kernel_S10: - - KERNEL_S1 - - subs I, I, #1 - bne dot_kernel_S10 - -dot_kernel_L999: - - ret - - EPILOGUE diff --git a/kernel/arm64/ddot_thunderx2t99_threaded.c b/kernel/arm64/ddot_thunderx2t99.c similarity index 98% rename from kernel/arm64/ddot_thunderx2t99_threaded.c rename to kernel/arm64/ddot_thunderx2t99.c index 36a8d6c21..86026733e 100644 --- a/kernel/arm64/ddot_thunderx2t99_threaded.c +++ b/kernel/arm64/ddot_thunderx2t99.c @@ -45,9 +45,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define LD1VY "{v24.d}[0]" #define SZ "8" +#if defined(SMP) extern int blas_level1_thread_with_return_value(int mode, BLASLONG m, BLASLONG n, BLASLONG k, void *alpha, void *a, BLASLONG lda, void *b, BLASLONG ldb, void *c, BLASLONG ldc, int (*function)(), int nthreads); +#endif static FLOAT ddot_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y) @@ -211,6 +213,7 @@ static FLOAT ddot_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLO return(dot); } +#if defined(SMP) static int ddot_thread_function(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT dummy2, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *result, BLASLONG dummy3) @@ -219,13 +222,17 @@ static int ddot_thread_function(BLASLONG n, BLASLONG dummy0, return 0; } +#endif FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y) { +#if defined(SMP) int nthreads; - FLOAT dot = 0.0; FLOAT dummy_alpha; +#endif + FLOAT dot = 0.0; +#if defined(SMP) nthreads = num_cpu_avail(1); if (inc_x == 0 || inc_y == 0) @@ -253,6 +260,9 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y) ptr = (FLOAT *)(((char *)ptr) + sizeof(double) * 2); } } +#else + dot = ddot_compute(n, x, inc_x, y, inc_y); +#endif return dot; } diff --git a/kernel/arm64/snrm2_thunderx2t99.S b/kernel/arm64/snrm2_thunderx2t99.S deleted file mode 100644 index d69441d08..000000000 --- a/kernel/arm64/snrm2_thunderx2t99.S +++ /dev/null @@ -1,228 +0,0 @@ -/******************************************************************************* -Copyright (c) 2017, The OpenBLAS Project -All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: -1. Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in -the documentation and/or other materials provided with the -distribution. -3. Neither the name of the OpenBLAS project nor the names of -its contributors may be used to endorse or promote products -derived from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*******************************************************************************/ - -#define ASSEMBLER -#include "common.h" - -#define N x0 /* vector length */ -#define X x1 /* X vector address */ -#define INC_X x2 /* X stride */ -#define I x5 /* loop variable */ - -/******************************************************************************* -* Macro definitions -*******************************************************************************/ - -#define TMPF s16 -#define TMPFD d17 -#define SSQ s0 -#define SSQD d0 -#define TMPVF {v16.s}[0] -#define TMPVFD {v17.s}[0] -#define SZ 4 - -/******************************************************************************/ - -.macro INIT - fmov SSQD, xzr - fmov d1, xzr - fmov d2, xzr - fmov d3, xzr - fmov d4, xzr - fmov d5, xzr - fmov d6, xzr - fmov d7, xzr -.endm - -.macro KERNEL_F1 - ldr TMPF, [X], #SZ - fcvt TMPFD, TMPF - fmadd SSQD, TMPFD, TMPFD, SSQD -.endm - -.macro KERNEL_F32 - ldur q16, [X] - ldur q18, [X, #16] - ldur q20, [X, #32] - ldur q22, [X, #48] - ldur q24, [X, #64] - ldur q26, [X, #80] - ldur q28, [X, #96] - ldur q30, [X, #112] - - add X, X, #128 - - fcvtl2 v17.2d, v16.4s - fcvtl v16.2d, v16.2s - fcvtl2 v19.2d, v18.4s - fcvtl v18.2d, v18.2s - fcvtl2 v21.2d, v20.4s - fcvtl v20.2d, v20.2s - fcvtl2 v23.2d, v22.4s - fcvtl v22.2d, v22.2s - fcvtl2 v25.2d, v24.4s - fcvtl v24.2d, v24.2s - fcvtl2 v27.2d, v26.4s - fcvtl v26.2d, v26.2s - fcvtl2 v29.2d, v28.4s - fcvtl v28.2d, v28.2s - fcvtl2 v31.2d, v30.4s - fcvtl v30.2d, v30.2s - - fmla v0.2d, v16.2d, v16.2d - fmla v1.2d, v17.2d, v17.2d - fmla v2.2d, v18.2d, v18.2d - fmla v3.2d, v19.2d, v19.2d - fmla v4.2d, v20.2d, v20.2d - fmla v5.2d, v21.2d, v21.2d - fmla v6.2d, v22.2d, v22.2d - fmla v7.2d, v23.2d, v23.2d - - fmla v0.2d, v24.2d, v24.2d - fmla v1.2d, v25.2d, v25.2d - fmla v2.2d, v26.2d, v26.2d - fmla v3.2d, v27.2d, v27.2d - fmla v4.2d, v28.2d, v28.2d - fmla v5.2d, v29.2d, v29.2d - fmla v6.2d, v30.2d, v30.2d - fmla v7.2d, v31.2d, v31.2d - - prfm PLDL1KEEP, [X, #1024] - prfm PLDL1KEEP, [X, #1024+64] -.endm - -.macro KERNEL_F32_FINALIZE - fadd v0.2d, v0.2d, v1.2d - fadd v2.2d, v2.2d, v3.2d - fadd v4.2d, v4.2d, v5.2d - fadd v6.2d, v6.2d, v7.2d - - fadd v0.2d, v0.2d, v2.2d - fadd v4.2d, v4.2d, v6.2d - - fadd v0.2d, v0.2d, v4.2d - faddp SSQD, v0.2d -.endm - -.macro INIT_S - lsl INC_X, INC_X, #2 -.endm - -.macro KERNEL_S1 - ldr TMPF, [X] - add X, X, INC_X - fcvt TMPFD, TMPF - fmadd SSQD, TMPFD, TMPFD, SSQD -.endm - -/******************************************************************************* -* End of macro definitions -*******************************************************************************/ - - PROLOGUE - - INIT - - cmp N, xzr - ble nrm2_kernel_zero - cmp INC_X, xzr - ble nrm2_kernel_zero - cmp INC_X, #1 - bne nrm2_kernel_S_BEGIN - -nrm2_kernel_F_BEGIN: - - asr I, N, #6 - cmp I, xzr - beq nrm2_kernel_S_BEGIN - - .align 5 -nrm2_kernel_F64: - - KERNEL_F32 - KERNEL_F32 - - subs I, I, #1 - bne nrm2_kernel_F64 - - KERNEL_F32_FINALIZE - -nrm2_kernel_F1: - - ands I, N, #63 - ble nrm2_kernel_L999 - -nrm2_kernel_F10: - - KERNEL_F1 - - subs I, I, #1 - bne nrm2_kernel_F10 - - b nrm2_kernel_L999 - -nrm2_kernel_S_BEGIN: - - INIT_S - - asr I, N, #2 - cmp I, xzr - ble nrm2_kernel_S1 - -nrm2_kernel_S4: - - KERNEL_S1 - KERNEL_S1 - KERNEL_S1 - KERNEL_S1 - - subs I, I, #1 - bne nrm2_kernel_S4 - -nrm2_kernel_S1: - - ands I, N, #3 - ble nrm2_kernel_L999 - -nrm2_kernel_S10: - - KERNEL_S1 - - subs I, I, #1 - bne nrm2_kernel_S10 - -nrm2_kernel_L999: - fsqrt SSQD, SSQD - fcvt SSQ, SSQD - ret - -nrm2_kernel_zero: - fmov SSQ, wzr - - ret - - EPILOGUE diff --git a/kernel/arm64/nrm2_thunderx2t99_threaded.c b/kernel/arm64/snrm2_thunderx2t99.c similarity index 98% rename from kernel/arm64/nrm2_thunderx2t99_threaded.c rename to kernel/arm64/snrm2_thunderx2t99.c index d810c6713..90595582c 100644 --- a/kernel/arm64/nrm2_thunderx2t99_threaded.c +++ b/kernel/arm64/snrm2_thunderx2t99.c @@ -30,11 +30,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include +#if defined(SMP) extern int blas_level1_thread_with_return_value(int mode, BLASLONG m, BLASLONG n, BLASLONG k, void *alpha, void *a, BLASLONG lda, void *b, BLASLONG ldb, void *c, BLASLONG ldc, int (*function)(), int nthreads); +#endif -#if !defined(DOUBLE) #define N "x0" /* vector length */ #define X "x1" /* X vector address */ #define INC_X "x2" /* X stride */ @@ -197,9 +198,8 @@ static double nrm2_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x) return ret; } -#else //!defined(DOUBLE) -#endif +#if defined(SMP) static int nrm2_thread_function(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT dummy2, FLOAT *x, BLASLONG inc_x, FLOAT *dummy3, BLASLONG dummy4, FLOAT *result, BLASLONG dummy5) @@ -208,17 +208,21 @@ static int nrm2_thread_function(BLASLONG n, BLASLONG dummy0, return 0; } +#endif FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) { +#if defined(SMP) int nthreads; + FLOAT dummy_alpha; +#endif FLOAT nrm2 = 0.0; double nrm2_double = 0.0; - FLOAT dummy_alpha; if (n <= 0 || inc_x <= 0) return 0.0; if (n == 1) return fabs(x[0]); +#if defined(SMP) nthreads = num_cpu_avail(1); if (n <= 10000) @@ -243,6 +247,9 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) ptr = (double *)(((char *)ptr) + sizeof(double) * 2); } } +#else + nrm2_double = nrm2_compute(n, x, inc_x); +#endif nrm2 = sqrt(nrm2_double); return nrm2; From ff6f572f2e74fbe415656f8d8fb1251e32d9a0c0 Mon Sep 17 00:00:00 2001 From: Ashwin Sekhar T K Date: Mon, 30 Jan 2017 12:09:04 +0530 Subject: [PATCH 08/12] THUNDERX2T99: Rename labels in for DDOT and SNRM2 --- kernel/arm64/ddot_thunderx2t99.c | 45 +++++++++++++++--------------- kernel/arm64/snrm2_thunderx2t99.c | 46 +++++++++++++++---------------- 2 files changed, 46 insertions(+), 45 deletions(-) diff --git a/kernel/arm64/ddot_thunderx2t99.c b/kernel/arm64/ddot_thunderx2t99.c index 86026733e..cb894d5ee 100644 --- a/kernel/arm64/ddot_thunderx2t99.c +++ b/kernel/arm64/ddot_thunderx2t99.c @@ -64,7 +64,7 @@ static FLOAT ddot_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLO " mov "INC_X", %[INCX_] \n" " mov "Y", %[Y_] \n" " mov "INC_Y", %[INCY_] \n" - " fmov "DOTF", "REG0" \n" + " fmov "DOTF", "REG0" \n" " fmov d1, "REG0" \n" " fmov d2, "REG0" \n" " fmov d3, "REG0" \n" @@ -74,20 +74,20 @@ static FLOAT ddot_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLO " fmov d7, "REG0" \n" " cmp "N", xzr \n" - " ble 9f //dot_kernel_L999 \n" + " ble .Ldot_kernel_L999 \n" " cmp "INC_X", #1 \n" - " bne 5f //dot_kernel_S_BEGIN \n" + " bne .Ldot_kernel_S_BEGIN \n" " cmp "INC_Y", #1 \n" - " bne 5f //dot_kernel_S_BEGIN \n" + " bne .Ldot_kernel_S_BEGIN \n" - "1: //dot_kernel_F_BEGIN \n" + ".Ldot_kernel_F_BEGIN: \n" " asr "J", "N", #5 \n" " cmp "J", xzr \n" - " beq 3f //dot_kernel_F1 \n" + " beq .Ldot_kernel_F1 \n" " .align 5 \n" - "2: //dot_kernel_F32 \n" + ".Ldot_kernel_F32: \n" " ldp q16, q17, ["X"] \n" " ldp q24, q25, ["Y"] \n" " ldp q18, q19, ["X", #32] \n" @@ -135,7 +135,7 @@ static FLOAT ddot_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLO " fmla v7.2d, v23.2d, v31.2d \n" " subs "J", "J", #1 \n" - " bne 2b //dot_kernel_F32 \n" + " bne .Ldot_kernel_F32 \n" " fadd v0.2d, v0.2d, v1.2d \n" " fadd v2.2d, v2.2d, v3.2d \n" @@ -146,11 +146,11 @@ static FLOAT ddot_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLO " fadd v0.2d, v0.2d, v4.2d \n" " faddp "DOTF", v0.2d \n" - "3: //dot_kernel_F1 \n" + ".Ldot_kernel_F1: \n" " ands "J", "N", #31 \n" - " ble 9f //dot_kernel_L999 \n" + " ble .Ldot_kernel_L999 \n" - "4: //dot_kernel_F10 \n" + ".Ldot_kernel_F10: \n" " ldr "TMPX", ["X"] \n" " ldr "TMPY", ["Y"] \n" " add "X", "X", #"SZ" \n" @@ -158,18 +158,18 @@ static FLOAT ddot_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLO " fmadd "DOTF", "TMPX", "TMPY", "DOTF" \n" " subs "J", "J", #1 \n" - " bne 4b //dot_kernel_F10 \n" + " bne .Ldot_kernel_F10 \n" - " b 9f //dot_kernel_L999 \n" + " b .Ldot_kernel_L999 \n" - "5: //dot_kernel_S_BEGIN \n" + ".Ldot_kernel_S_BEGIN: \n" " lsl "INC_X", "INC_X", #3 \n" " lsl "INC_Y", "INC_Y", #3 \n" " asr "J", "N", #2 \n" " cmp "J", xzr \n" - " ble 7f //dot_kernel_S1 \n" + " ble .Ldot_kernel_S1 \n" - "6: //dot_kernel_S4: \n" + ".Ldot_kernel_S4: \n" " ld1 "LD1VX", ["X"], "INC_X" \n" " ld1 "LD1VY", ["Y"], "INC_Y" \n" " fmadd "DOTF", "TMPX", "TMPY", "DOTF" \n" @@ -183,21 +183,22 @@ static FLOAT ddot_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLO " ld1 "LD1VY", ["Y"], "INC_Y" \n" " fmadd "DOTF", "TMPX", "TMPY", "DOTF" \n" " subs "J", "J", #1 \n" - " bne 6b //dot_kernel_S4 \n" + " bne .Ldot_kernel_S4 \n" - "7: //dot_kernel_S1: \n" + ".Ldot_kernel_S1: \n" " ands "J", "N", #3 \n" - " ble 9f //dot_kernel_L999 \n" + " ble .Ldot_kernel_L999 \n" - "8: //dot_kernel_S10 \n" + ".Ldot_kernel_S10: \n" " ld1 "LD1VX", ["X"], "INC_X" \n" " ld1 "LD1VY", ["Y"], "INC_Y" \n" " fmadd "DOTF", "TMPX", "TMPY", "DOTF" \n" " subs "J", "J", #1 \n" - " bne 8b //dot_kernel_S10 \n" + " bne .Ldot_kernel_S10 \n" - "9: //dot_kernel_L999 \n" + ".Ldot_kernel_L999: \n" " fmov %[DOT_], "DOTF" \n" + : [DOT_] "=r" (dot) //%0 : [N_] "r" (n), //%1 [X_] "r" (x), //%2 diff --git a/kernel/arm64/snrm2_thunderx2t99.c b/kernel/arm64/snrm2_thunderx2t99.c index 90595582c..8b7620c2f 100644 --- a/kernel/arm64/snrm2_thunderx2t99.c +++ b/kernel/arm64/snrm2_thunderx2t99.c @@ -131,59 +131,59 @@ static double nrm2_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x) " fmov d6, xzr \n" " fmov d7, xzr \n" " cmp "N", xzr \n" - " ble 8f //nrm2_kernel_L999 \n" + " ble .Lnrm2_kernel_L999 \n" " cmp "INC_X", xzr \n" - " ble 8f //nrm2_kernel_L999 \n" + " ble .Lnrm2_kernel_L999 \n" " cmp "INC_X", #1 \n" - " bne 5f // nrm2_kernel_S_BEGIN \n" + " bne .Lnrm2_kernel_S_BEGIN \n" - "1: //nrm2_kernel_F_BEGIN: \n" + ".Lnrm2_kernel_F_BEGIN: \n" " asr "I", "N", #6 \n" " cmp "I", xzr \n" - " beq 5f // nrm2_kernel_S_BEGIN \n" + " beq .Lnrm2_kernel_S_BEGIN \n" " .align 5 \n" - "2: //nrm2_kernel_F64: \n" + ".Lnrm2_kernel_F64: \n" " "KERNEL_F32" \n" " "KERNEL_F32" \n" " subs "I", "I", #1 \n" - " bne 2b //nrm2_kernel_F64 \n" + " bne .Lnrm2_kernel_F64 \n" " "KERNEL_F32_FINALIZE" \n" - "3: // nrm2_kernel_F1: \n" + ".Lnrm2_kernel_F1: \n" " ands "I", "N", #63 \n" - " ble 8f //nrm2_kernel_L999 \n" + " ble .Lnrm2_kernel_L999 \n" - "4: // nrm2_kernel_F10: \n" + ".Lnrm2_kernel_F10: \n" " "KERNEL_F1" \n" - " subs "I", "I", #1 \n" - " bne 4b //nrm2_kernel_F10 \n" - " b 8f //nrm2_kernel_L999 \n" + " subs "I", "I", #1 \n" + " bne .Lnrm2_kernel_F10 \n" + " b .Lnrm2_kernel_L999 \n" - "5: // nrm2_kernel_S_BEGIN: \n" + ".Lnrm2_kernel_S_BEGIN: \n" " lsl "INC_X", "INC_X", #2 \n" " asr "I", "N", #2 \n" " cmp "I", xzr \n" - " ble 6f //nrm2_kernel_S1 \n" + " ble .Lnrm2_kernel_S1 \n" - "4: //nrm2_kernel_S4: \n" + ".Lnrm2_kernel_S4: \n" " "KERNEL_S1" \n" " "KERNEL_S1" \n" " "KERNEL_S1" \n" " "KERNEL_S1" \n" " subs "I", "I", #1 \n" - " bne 4b //nrm2_kernel_S4 \n" + " bne .Lnrm2_kernel_S4 \n" - "6: //nrm2_kernel_S1: \n" + ".Lnrm2_kernel_S1: \n" " ands "I", "N", #3 \n" - " ble 8f //nrm2_kernel_L999 \n" + " ble .Lnrm2_kernel_L999 \n" - "7: //nrm2_kernel_S10: \n" + ".Lnrm2_kernel_S10: \n" " "KERNEL_S1" \n" - " subs "I", "I", #1 \n" - " bne 7b //nrm2_kernel_S10 \n" + " subs "I", "I", #1 \n" + " bne .Lnrm2_kernel_S10 \n" - "8: //nrm2_kernel_L999: \n" + ".Lnrm2_kernel_L999: \n" " fmov %[RET_], "SSQD" \n" : [RET_] "=r" (ret) //%0 From 99bd2892bf8fc8c90dc25ae1b35eff2943b454b3 Mon Sep 17 00:00:00 2001 From: Ashwin Sekhar T K Date: Fri, 27 Jan 2017 01:26:00 -0800 Subject: [PATCH 09/12] THUNDERX2T99: Add optimized CASUM Implementation --- kernel/arm64/KERNEL.THUNDERX2T99 | 2 + kernel/arm64/casum_thunderx2t99.c | 268 ++++++++++++++++++++++++++++++ 2 files changed, 270 insertions(+) create mode 100644 kernel/arm64/casum_thunderx2t99.c diff --git a/kernel/arm64/KERNEL.THUNDERX2T99 b/kernel/arm64/KERNEL.THUNDERX2T99 index c8668b67f..8e5d1b0d0 100644 --- a/kernel/arm64/KERNEL.THUNDERX2T99 +++ b/kernel/arm64/KERNEL.THUNDERX2T99 @@ -1,5 +1,7 @@ include $(KERNELDIR)/KERNEL.CORTEXA57 +CASUMKERNEL = casum_thunderx2t99.c + SNRM2KERNEL = snrm2_thunderx2t99.c CNRM2KERNEL = cnrm2_thunderx2t99.S diff --git a/kernel/arm64/casum_thunderx2t99.c b/kernel/arm64/casum_thunderx2t99.c new file mode 100644 index 000000000..4dac2e8ab --- /dev/null +++ b/kernel/arm64/casum_thunderx2t99.c @@ -0,0 +1,268 @@ +/*************************************************************************** +Copyright (c) 2017, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include "common.h" + +#include + +#define N "x0" /* vector length */ +#define X "x1" /* "X" vector address */ +#define INC_X "x2" /* "X" stride */ +#define J "x5" /* loop variable */ + +#define REG0 "wzr" +#define SUMF "s0" +#define SUMFD "d0" + +/******************************************************************************/ + +#define KERNEL_F1 \ + "ldr d1, ["X"] \n" \ + "add "X", "X", #8 \n" \ + "fabs v1.2s, v1.2s \n" \ + "ext v2.8b, v1.8b, v1.8b, #4 \n" \ + "fadd s1, s1, s2 \n" \ + "fadd "SUMF", "SUMF", s1 \n" + +#define KERNEL_F32 \ + "ldr q16, ["X"] \n" \ + "ldr q17, ["X", #16] \n" \ + "ldr q18, ["X", #32] \n" \ + "ldr q19, ["X", #48] \n" \ + "ldp q20, q21, ["X", #64] \n" \ + "ldp q22, q23, ["X", #96] \n" \ + "fabs v16.4s, v16.4s \n" \ + "fabs v17.4s, v17.4s \n" \ + "fabs v18.4s, v18.4s \n" \ + "fabs v19.4s, v19.4s \n" \ + "ldp q24, q25, ["X", #128] \n" \ + "ldp q26, q27, ["X", #160] \n" \ + "fabs v20.4s, v20.4s \n" \ + "fabs v21.4s, v21.4s \n" \ + "fabs v22.4s, v22.4s \n" \ + "fabs v23.4s, v23.4s \n" \ + "fadd v16.4s, v16.4s, v17.4s \n" \ + "fadd v18.4s, v18.4s, v19.4s \n" \ + "ldp q28, q29, ["X", #192] \n" \ + "ldp q30, q31, ["X", #224] \n" \ + "fabs v24.4s, v24.4s \n" \ + "fabs v25.4s, v25.4s \n" \ + "fabs v26.4s, v26.4s \n" \ + "fabs v27.4s, v27.4s \n" \ + "add "X", "X", #256 \n" \ + "fadd v20.4s, v20.4s, v21.4s \n" \ + "fadd v22.4s, v22.4s, v23.4s \n" \ + "fabs v28.4s, v28.4s \n" \ + "fabs v29.4s, v29.4s \n" \ + "fabs v30.4s, v30.4s \n" \ + "fabs v31.4s, v31.4s \n" \ + "PRFM PLDL1KEEP, ["X", #1024] \n" \ + "PRFM PLDL1KEEP, ["X", #1024+64] \n" \ + "fadd v24.4s, v24.4s, v25.4s \n" \ + "fadd v26.4s, v26.4s, v27.4s \n" \ + "fadd v0.4s, v0.4s, v16.4s \n" \ + "fadd v1.4s, v1.4s, v18.4s \n" \ + "fadd v2.4s, v2.4s, v20.4s \n" \ + "fadd v3.4s, v3.4s, v22.4s \n" \ + "PRFM PLDL1KEEP, ["X", #1024+128] \n" \ + "PRFM PLDL1KEEP, ["X", #1024+192] \n" \ + "fadd v28.4s, v28.4s, v29.4s \n" \ + "fadd v30.4s, v30.4s, v31.4s \n" \ + "fadd v4.4s, v4.4s, v24.4s \n" \ + "fadd v5.4s, v5.4s, v26.4s \n" \ + "fadd v6.4s, v6.4s, v28.4s \n" \ + "fadd v7.4s, v7.4s, v30.4s \n" + +#define KERNEL_F32_FINALIZE \ + "fadd v0.4s, v0.4s, v1.4s \n" \ + "fadd v2.4s, v2.4s, v3.4s \n" \ + "fadd v4.4s, v4.4s, v5.4s \n" \ + "fadd v6.4s, v6.4s, v7.4s \n" \ + "fadd v0.4s, v0.4s, v2.4s \n" \ + "fadd v4.4s, v4.4s, v6.4s \n" \ + "fadd v0.4s, v0.4s, v4.4s \n" \ + "ext v1.16b, v0.16b, v0.16b, #8 \n" \ + "fadd v0.2s, v0.2s, v1.2s \n" \ + "faddp "SUMF", v0.2s \n" + +#define INIT_S \ + "lsl "INC_X", "INC_X", #3 \n" + +#define KERNEL_S1 \ + "ldr d1, ["X"] \n" \ + "add "X", "X", "INC_X" \n" \ + "fabs v1.2s, v1.2s \n" \ + "ext v2.8b, v1.8b, v1.8b, #4 \n" \ + "fadd s1, s1, s2 \n" \ + "fadd "SUMF", "SUMF", s1 \n" + + +#if defined(SMP) +extern int blas_level1_thread_with_return_value(int mode, BLASLONG m, BLASLONG n, + BLASLONG k, void *alpha, void *a, BLASLONG lda, void *b, BLASLONG ldb, + void *c, BLASLONG ldc, int (*function)(), int nthreads); +#endif + + +static FLOAT casum_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x) +{ + FLOAT asum = 0.0 ; + + if ( n < 0 ) return(asum); + + __asm__ __volatile__ ( + " mov "N", %[N_] \n" + " mov "X", %[X_] \n" + " mov "INC_X", %[INCX_] \n" + " fmov "SUMF", "REG0" \n" + " fmov s1, "REG0" \n" + " fmov s2, "REG0" \n" + " fmov s3, "REG0" \n" + " fmov s4, "REG0" \n" + " fmov s5, "REG0" \n" + " fmov s6, "REG0" \n" + " fmov s7, "REG0" \n" + " cmp "N", xzr \n" + " ble .Lasum_kernel_L999 \n" + " cmp "INC_X", xzr \n" + " ble .Lasum_kernel_L999 \n" + " cmp "INC_X", #1 \n" + " bne .Lasum_kernel_S_BEGIN \n" + + ".Lasum_kernel_F_BEGIN: \n" + " asr "J", "N", #5 \n" + " cmp "J", xzr \n" + " beq .Lasum_kernel_F1 \n" + + ".Lasum_kernel_F32: \n" + " "KERNEL_F32" \n" + " subs "J", "J", #1 \n" + " bne .Lasum_kernel_F32 \n" + " "KERNEL_F32_FINALIZE" \n" + + ".Lasum_kernel_F1: \n" + " ands "J", "N", #31 \n" + " ble .Lasum_kernel_L999 \n" + + ".Lasum_kernel_F10: \n" + " "KERNEL_F1" \n" + " subs "J", "J", #1 \n" + " bne .Lasum_kernel_F10 \n" + " b .Lasum_kernel_L999 \n" + + ".Lasum_kernel_S_BEGIN: \n" + " "INIT_S" \n" + " asr "J", "N", #2 \n" + " cmp "J", xzr \n" + " ble .Lasum_kernel_S1 \n" + + ".Lasum_kernel_S4: \n" + " "KERNEL_S1" \n" + " "KERNEL_S1" \n" + " "KERNEL_S1" \n" + " "KERNEL_S1" \n" + " subs "J", "J", #1 \n" + " bne .Lasum_kernel_S4 \n" + + ".Lasum_kernel_S1: \n" + " ands "J", "N", #3 \n" + " ble .Lasum_kernel_L999 \n" + + ".Lasum_kernel_S10: \n" + " "KERNEL_S1" \n" + " subs "J", "J", #1 \n" + " bne .Lasum_kernel_S10 \n" + + ".Lasum_kernel_L999: \n" + " fmov %[ASUM_], "SUMFD" \n" + + : [ASUM_] "=r" (asum) //%0 + : [N_] "r" (n), //%1 + [X_] "r" (x), //%2 + [INCX_] "r" (inc_x) //%3 + : "cc", + "memory", + "x0", "x1", "x2", "x3", "x4", "x5", + "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7" + ); + + return asum; +} + +#if defined(SMP) +static int casum_thread_function(BLASLONG n, BLASLONG dummy0, + BLASLONG dummy1, FLOAT dummy2, FLOAT *x, BLASLONG inc_x, FLOAT *y, + BLASLONG inc_y, FLOAT *result, BLASLONG dummy3) +{ + *result = casum_compute(n, x, inc_x); + + return 0; +} +#endif + +FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) +{ +#if defined(SMP) + int nthreads; + FLOAT dummy_alpha; +#endif + FLOAT asum = 0.0; + +#if defined(SMP) + nthreads = num_cpu_avail(1); + + if (inc_x == 0) + nthreads = 1; + + if (n <= 10000) + nthreads = 1; + + if (nthreads == 1) { + asum = casum_compute(n, x, inc_x); + } else { + int mode, i; + char result[MAX_CPU_NUMBER * sizeof(double) * 2]; + FLOAT *ptr; + + mode = BLAS_SINGLE | BLAS_COMPLEX; + + blas_level1_thread_with_return_value(mode, n, 0, 0, &dummy_alpha, + x, inc_x, NULL, 0, result, 0, + ( void *)casum_thread_function, nthreads); + + ptr = (FLOAT *)result; + for (i = 0; i < nthreads; i++) { + asum = asum + (*ptr); + ptr = (FLOAT *)(((char *)ptr) + sizeof(double) * 2); + } + } +#else + asum = casum_compute(n, x, inc_x); +#endif + + return asum; +} From 3918d170259bfa95f29cf8b659aedd8b803e02d2 Mon Sep 17 00:00:00 2001 From: Ashwin Sekhar T K Date: Tue, 31 Jan 2017 11:40:45 +0530 Subject: [PATCH 10/12] LAPACK: Fix lapack-test errors in ARM64 threaded version --- lapack/getrf/getrf_parallel.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lapack/getrf/getrf_parallel.c b/lapack/getrf/getrf_parallel.c index d5af74a7c..db8c836e0 100644 --- a/lapack/getrf/getrf_parallel.c +++ b/lapack/getrf/getrf_parallel.c @@ -284,6 +284,7 @@ static int inner_advanced_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG * } } + MB; for (i = 0; i < args -> nthreads; i++) job[mypos].working[i][CACHE_LINE_SIZE * bufferside] = (BLASLONG)buffer[bufferside]; @@ -324,6 +325,7 @@ static int inner_advanced_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG * sa, (FLOAT *)job[current].working[mypos][CACHE_LINE_SIZE * bufferside], c, lda, is, xxx); + MB; if (is + min_i >= m) { job[current].working[mypos][CACHE_LINE_SIZE * bufferside] = 0; } From e58233460ab3ec3d1fe71b7f0bed96e6dc67f291 Mon Sep 17 00:00:00 2001 From: Ashwin Sekhar T K Date: Tue, 31 Jan 2017 23:25:41 -0800 Subject: [PATCH 11/12] THUDNERX2T99: Add optimized D/C/Z ASUM Implementations --- kernel/arm64/KERNEL.THUNDERX2T99 | 4 + kernel/arm64/dasum_thunderx2t99.c | 263 +++++++++++++++++++++++++++++ kernel/arm64/sasum_thunderx2t99.c | 265 ++++++++++++++++++++++++++++++ kernel/arm64/zasum_thunderx2t99.c | 265 ++++++++++++++++++++++++++++++ 4 files changed, 797 insertions(+) create mode 100644 kernel/arm64/dasum_thunderx2t99.c create mode 100644 kernel/arm64/sasum_thunderx2t99.c create mode 100644 kernel/arm64/zasum_thunderx2t99.c diff --git a/kernel/arm64/KERNEL.THUNDERX2T99 b/kernel/arm64/KERNEL.THUNDERX2T99 index 8e5d1b0d0..6964aabf8 100644 --- a/kernel/arm64/KERNEL.THUNDERX2T99 +++ b/kernel/arm64/KERNEL.THUNDERX2T99 @@ -1,6 +1,9 @@ include $(KERNELDIR)/KERNEL.CORTEXA57 +SASUMKERNEL = sasum_thunderx2t99.c +DASUMKERNEL = dasum_thunderx2t99.c CASUMKERNEL = casum_thunderx2t99.c +ZASUMKERNEL = zasum_thunderx2t99.c SNRM2KERNEL = snrm2_thunderx2t99.c CNRM2KERNEL = cnrm2_thunderx2t99.S @@ -9,6 +12,7 @@ DAXPYKERNEL = daxpy_thunderx2t99.S DDOTKERNEL = ddot_thunderx2t99.c + ifeq ($(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N), 8x4) DGEMMKERNEL = dgemm_kernel_8x4_thunderx2t99.S endif diff --git a/kernel/arm64/dasum_thunderx2t99.c b/kernel/arm64/dasum_thunderx2t99.c new file mode 100644 index 000000000..bd6bb055d --- /dev/null +++ b/kernel/arm64/dasum_thunderx2t99.c @@ -0,0 +1,263 @@ +/*************************************************************************** +Copyright (c) 2017, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include "common.h" + +#include + +#define N "x0" /* vector length */ +#define X "x1" /* "X" vector address */ +#define INC_X "x2" /* "X" stride */ +#define J "x5" /* loop variable */ + +#define REG0 "xzr" +#define SUMF "d0" +#define TMPF "d1" + +/******************************************************************************/ + +#define KERNEL_F1 \ + "ldr "TMPF", ["X"] \n" \ + "add "X", "X", #8 \n" \ + "fabs "TMPF", "TMPF" \n" \ + "fadd "SUMF", "SUMF", "TMPF" \n" + +#define KERNEL_F32 \ + "ldr q16, ["X"] \n" \ + "ldr q17, ["X", #16] \n" \ + "ldr q18, ["X", #32] \n" \ + "ldr q19, ["X", #48] \n" \ + "ldp q20, q21, ["X", #64] \n" \ + "ldp q22, q23, ["X", #96] \n" \ + "fabs v16.2d, v16.2d \n" \ + "fabs v17.2d, v17.2d \n" \ + "fabs v18.2d, v18.2d \n" \ + "fabs v19.2d, v19.2d \n" \ + "ldp q24, q25, ["X", #128] \n" \ + "ldp q26, q27, ["X", #160] \n" \ + "fabs v20.2d, v20.2d \n" \ + "fabs v21.2d, v21.2d \n" \ + "fabs v22.2d, v22.2d \n" \ + "fabs v23.2d, v23.2d \n" \ + "fadd v16.2d, v16.2d, v17.2d \n" \ + "fadd v18.2d, v18.2d, v19.2d \n" \ + "ldp q28, q29, ["X", #192] \n" \ + "ldp q30, q31, ["X", #224] \n" \ + "fabs v24.2d, v24.2d \n" \ + "fabs v25.2d, v25.2d \n" \ + "fabs v26.2d, v26.2d \n" \ + "fabs v27.2d, v27.2d \n" \ + "add "X", "X", #256 \n" \ + "fadd v20.2d, v20.2d, v21.2d \n" \ + "fadd v22.2d, v22.2d, v23.2d \n" \ + "fabs v28.2d, v28.2d \n" \ + "fabs v29.2d, v29.2d \n" \ + "fabs v30.2d, v30.2d \n" \ + "fabs v31.2d, v31.2d \n" \ + "PRFM PLDL1KEEP, ["X", #1024] \n" \ + "PRFM PLDL1KEEP, ["X", #1024+64] \n" \ + "fadd v24.2d, v24.2d, v25.2d \n" \ + "fadd v26.2d, v26.2d, v27.2d \n" \ + "fadd v28.2d, v28.2d, v29.2d \n" \ + "fadd v30.2d, v30.2d, v31.2d \n" \ + "fadd v0.2d, v0.2d, v16.2d \n" \ + "fadd v1.2d, v1.2d, v18.2d \n" \ + "fadd v2.2d, v2.2d, v20.2d \n" \ + "fadd v3.2d, v3.2d, v22.2d \n" \ + "PRFM PLDL1KEEP, ["X", #1024+128] \n" \ + "PRFM PLDL1KEEP, ["X", #1024+192] \n" \ + "fadd v4.2d, v4.2d, v24.2d \n" \ + "fadd v5.2d, v5.2d, v26.2d \n" \ + "fadd v6.2d, v6.2d, v28.2d \n" \ + "fadd v7.2d, v7.2d, v30.2d \n" + +#define KERNEL_F32_FINALIZE \ + "fadd v0.2d, v0.2d, v1.2d \n" \ + "fadd v2.2d, v2.2d, v3.2d \n" \ + "fadd v4.2d, v4.2d, v5.2d \n" \ + "fadd v6.2d, v6.2d, v7.2d \n" \ + "fadd v0.2d, v0.2d, v2.2d \n" \ + "fadd v4.2d, v4.2d, v6.2d \n" \ + "fadd v0.2d, v0.2d, v4.2d \n" \ + "faddp "SUMF", v0.2d \n" + +#define INIT_S \ + "lsl "INC_X", "INC_X", #3 \n" + +#define KERNEL_S1 \ + "ldr "TMPF", ["X"] \n" \ + "add "X", "X", "INC_X" \n" \ + "fabs "TMPF", "TMPF" \n" \ + "fadd "SUMF", "SUMF", "TMPF" \n" + + +#if defined(SMP) +extern int blas_level1_thread_with_return_value(int mode, BLASLONG m, BLASLONG n, + BLASLONG k, void *alpha, void *a, BLASLONG lda, void *b, BLASLONG ldb, + void *c, BLASLONG ldc, int (*function)(), int nthreads); +#endif + + +static FLOAT dasum_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x) +{ + FLOAT asum = 0.0 ; + + if ( n < 0 ) return(asum); + + __asm__ __volatile__ ( + " mov "N", %[N_] \n" + " mov "X", %[X_] \n" + " mov "INC_X", %[INCX_] \n" + " fmov "SUMF", "REG0" \n" + " fmov d1, "REG0" \n" + " fmov d2, "REG0" \n" + " fmov d3, "REG0" \n" + " fmov d4, "REG0" \n" + " fmov d5, "REG0" \n" + " fmov d6, "REG0" \n" + " fmov d7, "REG0" \n" + " cmp "N", xzr \n" + " ble .Lasum_kernel_L999 \n" + " cmp "INC_X", xzr \n" + " ble .Lasum_kernel_L999 \n" + " cmp "INC_X", #1 \n" + " bne .Lasum_kernel_S_BEGIN \n" + + ".Lasum_kernel_F_BEGIN: \n" + " asr "J", "N", #5 \n" + " cmp "J", xzr \n" + " beq .Lasum_kernel_F1 \n" + + ".align 5 \n" + ".Lasum_kernel_F32: \n" + " "KERNEL_F32" \n" + " subs "J", "J", #1 \n" + " bne .Lasum_kernel_F32 \n" + " "KERNEL_F32_FINALIZE" \n" + + ".Lasum_kernel_F1: \n" + " ands "J", "N", #31 \n" + " ble .Lasum_kernel_L999 \n" + + ".Lasum_kernel_F10: \n" + " "KERNEL_F1" \n" + " subs "J", "J", #1 \n" + " bne .Lasum_kernel_F10 \n" + " b .Lasum_kernel_L999 \n" + + ".Lasum_kernel_S_BEGIN: \n" + " "INIT_S" \n" + " asr "J", "N", #2 \n" + " cmp "J", xzr \n" + " ble .Lasum_kernel_S1 \n" + + ".Lasum_kernel_S4: \n" + " "KERNEL_S1" \n" + " "KERNEL_S1" \n" + " "KERNEL_S1" \n" + " "KERNEL_S1" \n" + " subs "J", "J", #1 \n" + " bne .Lasum_kernel_S4 \n" + + ".Lasum_kernel_S1: \n" + " ands "J", "N", #3 \n" + " ble .Lasum_kernel_L999 \n" + + ".Lasum_kernel_S10: \n" + " "KERNEL_S1" \n" + " subs "J", "J", #1 \n" + " bne .Lasum_kernel_S10 \n" + + ".Lasum_kernel_L999: \n" + " fmov %[ASUM_], "SUMF" \n" + + : [ASUM_] "=r" (asum) //%0 + : [N_] "r" (n), //%1 + [X_] "r" (x), //%2 + [INCX_] "r" (inc_x) //%3 + : "cc", + "memory", + "x0", "x1", "x2", "x3", "x4", "x5", + "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7" + ); + + return asum; +} + +#if defined(SMP) +static int dasum_thread_function(BLASLONG n, BLASLONG dummy0, + BLASLONG dummy1, FLOAT dummy2, FLOAT *x, BLASLONG inc_x, FLOAT *y, + BLASLONG inc_y, FLOAT *result, BLASLONG dummy3) +{ + *result = dasum_compute(n, x, inc_x); + + return 0; +} +#endif + +FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) +{ +#if defined(SMP) + int nthreads; + FLOAT dummy_alpha; +#endif + FLOAT asum = 0.0; + +#if defined(SMP) + nthreads = num_cpu_avail(1); + + if (inc_x == 0) + nthreads = 1; + + if (n <= 10000) + nthreads = 1; + + if (nthreads == 1) { + asum = dasum_compute(n, x, inc_x); + } else { + int mode, i; + char result[MAX_CPU_NUMBER * sizeof(double) * 2]; + FLOAT *ptr; + + mode = BLAS_DOUBLE; + + blas_level1_thread_with_return_value(mode, n, 0, 0, &dummy_alpha, + x, inc_x, NULL, 0, result, 0, + ( void *)dasum_thread_function, nthreads); + + ptr = (FLOAT *)result; + for (i = 0; i < nthreads; i++) { + asum = asum + (*ptr); + ptr = (FLOAT *)(((char *)ptr) + sizeof(double) * 2); + } + } +#else + asum = dasum_compute(n, x, inc_x); +#endif + + return asum; +} diff --git a/kernel/arm64/sasum_thunderx2t99.c b/kernel/arm64/sasum_thunderx2t99.c new file mode 100644 index 000000000..767535dae --- /dev/null +++ b/kernel/arm64/sasum_thunderx2t99.c @@ -0,0 +1,265 @@ +/*************************************************************************** +Copyright (c) 2017, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include "common.h" + +#include + +#define N "x0" /* vector length */ +#define X "x1" /* "X" vector address */ +#define INC_X "x2" /* "X" stride */ +#define J "x5" /* loop variable */ + +#define REG0 "wzr" +#define SUMF "s0" +#define SUMFD "d0" + +/******************************************************************************/ + +#define KERNEL_F1 \ + "ldr s1, ["X"] \n" \ + "add "X", "X", #4 \n" \ + "fabs s1, s1 \n" \ + "fadd "SUMF", "SUMF", s1 \n" + +#define KERNEL_F64 \ + "ldr q16, ["X"] \n" \ + "ldr q17, ["X", #16] \n" \ + "ldr q18, ["X", #32] \n" \ + "ldr q19, ["X", #48] \n" \ + "ldp q20, q21, ["X", #64] \n" \ + "ldp q22, q23, ["X", #96] \n" \ + "fabs v16.4s, v16.4s \n" \ + "fabs v17.4s, v17.4s \n" \ + "fabs v18.4s, v18.4s \n" \ + "fabs v19.4s, v19.4s \n" \ + "ldp q24, q25, ["X", #128] \n" \ + "ldp q26, q27, ["X", #160] \n" \ + "fabs v20.4s, v20.4s \n" \ + "fabs v21.4s, v21.4s \n" \ + "fabs v22.4s, v22.4s \n" \ + "fabs v23.4s, v23.4s \n" \ + "fadd v16.4s, v16.4s, v17.4s \n" \ + "fadd v18.4s, v18.4s, v19.4s \n" \ + "ldp q28, q29, ["X", #192] \n" \ + "ldp q30, q31, ["X", #224] \n" \ + "fabs v24.4s, v24.4s \n" \ + "fabs v25.4s, v25.4s \n" \ + "fabs v26.4s, v26.4s \n" \ + "fabs v27.4s, v27.4s \n" \ + "add "X", "X", #256 \n" \ + "fadd v20.4s, v20.4s, v21.4s \n" \ + "fadd v22.4s, v22.4s, v23.4s \n" \ + "fabs v28.4s, v28.4s \n" \ + "fabs v29.4s, v29.4s \n" \ + "fabs v30.4s, v30.4s \n" \ + "fabs v31.4s, v31.4s \n" \ + "PRFM PLDL1KEEP, ["X", #1024] \n" \ + "PRFM PLDL1KEEP, ["X", #1024+64] \n" \ + "fadd v24.4s, v24.4s, v25.4s \n" \ + "fadd v26.4s, v26.4s, v27.4s \n" \ + "fadd v0.4s, v0.4s, v16.4s \n" \ + "fadd v1.4s, v1.4s, v18.4s \n" \ + "fadd v2.4s, v2.4s, v20.4s \n" \ + "fadd v3.4s, v3.4s, v22.4s \n" \ + "PRFM PLDL1KEEP, ["X", #1024+128] \n" \ + "PRFM PLDL1KEEP, ["X", #1024+192] \n" \ + "fadd v28.4s, v28.4s, v29.4s \n" \ + "fadd v30.4s, v30.4s, v31.4s \n" \ + "fadd v4.4s, v4.4s, v24.4s \n" \ + "fadd v5.4s, v5.4s, v26.4s \n" \ + "fadd v6.4s, v6.4s, v28.4s \n" \ + "fadd v7.4s, v7.4s, v30.4s \n" + +#define KERNEL_F64_FINALIZE \ + "fadd v0.4s, v0.4s, v1.4s \n" \ + "fadd v2.4s, v2.4s, v3.4s \n" \ + "fadd v4.4s, v4.4s, v5.4s \n" \ + "fadd v6.4s, v6.4s, v7.4s \n" \ + "fadd v0.4s, v0.4s, v2.4s \n" \ + "fadd v4.4s, v4.4s, v6.4s \n" \ + "fadd v0.4s, v0.4s, v4.4s \n" \ + "ext v1.16b, v0.16b, v0.16b, #8 \n" \ + "fadd v0.2s, v0.2s, v1.2s \n" \ + "faddp "SUMF", v0.2s \n" + +#define INIT_S \ + "lsl "INC_X", "INC_X", #2 \n" + +#define KERNEL_S1 \ + "ldr s1, ["X"] \n" \ + "add "X", "X", "INC_X" \n" \ + "fabs s1, s1 \n" \ + "fadd "SUMF", "SUMF", s1 \n" + + +#if defined(SMP) +extern int blas_level1_thread_with_return_value(int mode, BLASLONG m, BLASLONG n, + BLASLONG k, void *alpha, void *a, BLASLONG lda, void *b, BLASLONG ldb, + void *c, BLASLONG ldc, int (*function)(), int nthreads); +#endif + + +static FLOAT sasum_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x) +{ + FLOAT asum = 0.0 ; + + if ( n < 0 ) return(asum); + + __asm__ __volatile__ ( + " mov "N", %[N_] \n" + " mov "X", %[X_] \n" + " mov "INC_X", %[INCX_] \n" + " fmov "SUMF", "REG0" \n" + " fmov s1, "REG0" \n" + " fmov s2, "REG0" \n" + " fmov s3, "REG0" \n" + " fmov s4, "REG0" \n" + " fmov s5, "REG0" \n" + " fmov s6, "REG0" \n" + " fmov s7, "REG0" \n" + " cmp "N", xzr \n" + " ble .Lasum_kernel_L999 \n" + " cmp "INC_X", xzr \n" + " ble .Lasum_kernel_L999 \n" + " cmp "INC_X", #1 \n" + " bne .Lasum_kernel_S_BEGIN \n" + + ".Lasum_kernel_F_BEGIN: \n" + " asr "J", "N", #6 \n" + " cmp "J", xzr \n" + " beq .Lasum_kernel_F1 \n" + + ".align 5 \n" + ".Lasum_kernel_F64: \n" + " "KERNEL_F64" \n" + " subs "J", "J", #1 \n" + " bne .Lasum_kernel_F64 \n" + " "KERNEL_F64_FINALIZE" \n" + + ".Lasum_kernel_F1: \n" + " ands "J", "N", #63 \n" + " ble .Lasum_kernel_L999 \n" + + ".Lasum_kernel_F10: \n" + " "KERNEL_F1" \n" + " subs "J", "J", #1 \n" + " bne .Lasum_kernel_F10 \n" + " b .Lasum_kernel_L999 \n" + + ".Lasum_kernel_S_BEGIN: \n" + " "INIT_S" \n" + " asr "J", "N", #2 \n" + " cmp "J", xzr \n" + " ble .Lasum_kernel_S1 \n" + + ".Lasum_kernel_S4: \n" + " "KERNEL_S1" \n" + " "KERNEL_S1" \n" + " "KERNEL_S1" \n" + " "KERNEL_S1" \n" + " subs "J", "J", #1 \n" + " bne .Lasum_kernel_S4 \n" + + ".Lasum_kernel_S1: \n" + " ands "J", "N", #3 \n" + " ble .Lasum_kernel_L999 \n" + + ".Lasum_kernel_S10: \n" + " "KERNEL_S1" \n" + " subs "J", "J", #1 \n" + " bne .Lasum_kernel_S10 \n" + + ".Lasum_kernel_L999: \n" + " fmov %[ASUM_], "SUMFD" \n" + + : [ASUM_] "=r" (asum) //%0 + : [N_] "r" (n), //%1 + [X_] "r" (x), //%2 + [INCX_] "r" (inc_x) //%3 + : "cc", + "memory", + "x0", "x1", "x2", "x3", "x4", "x5", + "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7" + ); + + return asum; +} + +#if defined(SMP) +static int sasum_thread_function(BLASLONG n, BLASLONG dummy0, + BLASLONG dummy1, FLOAT dummy2, FLOAT *x, BLASLONG inc_x, FLOAT *y, + BLASLONG inc_y, FLOAT *result, BLASLONG dummy3) +{ + *result = sasum_compute(n, x, inc_x); + + return 0; +} +#endif + +FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) +{ +#if defined(SMP) + int nthreads; + FLOAT dummy_alpha; +#endif + FLOAT asum = 0.0; + +#if defined(SMP) + nthreads = num_cpu_avail(1); + + if (inc_x == 0) + nthreads = 1; + + if (n <= 10000) + nthreads = 1; + + if (nthreads == 1) { + asum = sasum_compute(n, x, inc_x); + } else { + int mode, i; + char result[MAX_CPU_NUMBER * sizeof(double) * 2]; + FLOAT *ptr; + + mode = BLAS_SINGLE; + + blas_level1_thread_with_return_value(mode, n, 0, 0, &dummy_alpha, + x, inc_x, NULL, 0, result, 0, + ( void *)sasum_thread_function, nthreads); + + ptr = (FLOAT *)result; + for (i = 0; i < nthreads; i++) { + asum = asum + (*ptr); + ptr = (FLOAT *)(((char *)ptr) + sizeof(double) * 2); + } + } +#else + asum = sasum_compute(n, x, inc_x); +#endif + + return asum; +} diff --git a/kernel/arm64/zasum_thunderx2t99.c b/kernel/arm64/zasum_thunderx2t99.c new file mode 100644 index 000000000..e0f4ae21a --- /dev/null +++ b/kernel/arm64/zasum_thunderx2t99.c @@ -0,0 +1,265 @@ +/*************************************************************************** +Copyright (c) 2017, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include "common.h" + +#include + +#define N "x0" /* vector length */ +#define X "x1" /* "X" vector address */ +#define INC_X "x2" /* "X" stride */ +#define J "x5" /* loop variable */ + +#define REG0 "xzr" +#define SUMF "d0" +#define TMPF "d1" + +/******************************************************************************/ + +#define KERNEL_F1 \ + "ldr q1, ["X"] \n" \ + "add "X", "X", #16 \n" \ + "fabs v1.2d, v1.2d \n" \ + "faddp d1, v1.2d \n" \ + "fadd "SUMF", "SUMF", d1 \n" + +#define KERNEL_F16 \ + "ldr q16, ["X"] \n" \ + "ldr q17, ["X", #16] \n" \ + "ldr q18, ["X", #32] \n" \ + "ldr q19, ["X", #48] \n" \ + "ldp q20, q21, ["X", #64] \n" \ + "ldp q22, q23, ["X", #96] \n" \ + "fabs v16.2d, v16.2d \n" \ + "fabs v17.2d, v17.2d \n" \ + "fabs v18.2d, v18.2d \n" \ + "fabs v19.2d, v19.2d \n" \ + "ldp q24, q25, ["X", #128] \n" \ + "ldp q26, q27, ["X", #160] \n" \ + "fabs v20.2d, v20.2d \n" \ + "fabs v21.2d, v21.2d \n" \ + "fabs v22.2d, v22.2d \n" \ + "fabs v23.2d, v23.2d \n" \ + "fadd v16.2d, v16.2d, v17.2d \n" \ + "fadd v18.2d, v18.2d, v19.2d \n" \ + "ldp q28, q29, ["X", #192] \n" \ + "ldp q30, q31, ["X", #224] \n" \ + "fabs v24.2d, v24.2d \n" \ + "fabs v25.2d, v25.2d \n" \ + "fabs v26.2d, v26.2d \n" \ + "fabs v27.2d, v27.2d \n" \ + "add "X", "X", #256 \n" \ + "fadd v20.2d, v20.2d, v21.2d \n" \ + "fadd v22.2d, v22.2d, v23.2d \n" \ + "fabs v28.2d, v28.2d \n" \ + "fabs v29.2d, v29.2d \n" \ + "fabs v30.2d, v30.2d \n" \ + "fabs v31.2d, v31.2d \n" \ + "PRFM PLDL1KEEP, ["X", #1024] \n" \ + "PRFM PLDL1KEEP, ["X", #1024+64] \n" \ + "fadd v24.2d, v24.2d, v25.2d \n" \ + "fadd v26.2d, v26.2d, v27.2d \n" \ + "fadd v28.2d, v28.2d, v29.2d \n" \ + "fadd v30.2d, v30.2d, v31.2d \n" \ + "fadd v0.2d, v0.2d, v16.2d \n" \ + "fadd v1.2d, v1.2d, v18.2d \n" \ + "fadd v2.2d, v2.2d, v20.2d \n" \ + "fadd v3.2d, v3.2d, v22.2d \n" \ + "PRFM PLDL1KEEP, ["X", #1024+128] \n" \ + "PRFM PLDL1KEEP, ["X", #1024+192] \n" \ + "fadd v4.2d, v4.2d, v24.2d \n" \ + "fadd v5.2d, v5.2d, v26.2d \n" \ + "fadd v6.2d, v6.2d, v28.2d \n" \ + "fadd v7.2d, v7.2d, v30.2d \n" + +#define KERNEL_F16_FINALIZE \ + "fadd v0.2d, v0.2d, v1.2d \n" \ + "fadd v2.2d, v2.2d, v3.2d \n" \ + "fadd v4.2d, v4.2d, v5.2d \n" \ + "fadd v6.2d, v6.2d, v7.2d \n" \ + "fadd v0.2d, v0.2d, v2.2d \n" \ + "fadd v4.2d, v4.2d, v6.2d \n" \ + "fadd v0.2d, v0.2d, v4.2d \n" \ + "faddp "SUMF", v0.2d \n" + +#define INIT_S \ + "lsl "INC_X", "INC_X", #4 \n" + +#define KERNEL_S1 \ + "ldr q1, ["X"] \n" \ + "add "X", "X", "INC_X" \n" \ + "fabs v1.2d, v1.2d \n" \ + "faddp d1, v1.2d \n" \ + "fadd "SUMF", "SUMF", d1 \n" + + +#if defined(SMP) +extern int blas_level1_thread_with_return_value(int mode, BLASLONG m, BLASLONG n, + BLASLONG k, void *alpha, void *a, BLASLONG lda, void *b, BLASLONG ldb, + void *c, BLASLONG ldc, int (*function)(), int nthreads); +#endif + + +static FLOAT zasum_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x) +{ + FLOAT asum = 0.0 ; + + if ( n < 0 ) return(asum); + + __asm__ __volatile__ ( + " mov "N", %[N_] \n" + " mov "X", %[X_] \n" + " mov "INC_X", %[INCX_] \n" + " fmov "SUMF", "REG0" \n" + " fmov d1, "REG0" \n" + " fmov d2, "REG0" \n" + " fmov d3, "REG0" \n" + " fmov d4, "REG0" \n" + " fmov d5, "REG0" \n" + " fmov d6, "REG0" \n" + " fmov d7, "REG0" \n" + " cmp "N", xzr \n" + " ble .Lasum_kernel_L999 \n" + " cmp "INC_X", xzr \n" + " ble .Lasum_kernel_L999 \n" + " cmp "INC_X", #1 \n" + " bne .Lasum_kernel_S_BEGIN \n" + + ".Lasum_kernel_F_BEGIN: \n" + " asr "J", "N", #4 \n" + " cmp "J", xzr \n" + " beq .Lasum_kernel_F1 \n" + + ".align 5 \n" + ".Lasum_kernel_F16: \n" + " "KERNEL_F16" \n" + " subs "J", "J", #1 \n" + " bne .Lasum_kernel_F16 \n" + " "KERNEL_F16_FINALIZE" \n" + + ".Lasum_kernel_F1: \n" + " ands "J", "N", #15 \n" + " ble .Lasum_kernel_L999 \n" + + ".Lasum_kernel_F10: \n" + " "KERNEL_F1" \n" + " subs "J", "J", #1 \n" + " bne .Lasum_kernel_F10 \n" + " b .Lasum_kernel_L999 \n" + + ".Lasum_kernel_S_BEGIN: \n" + " "INIT_S" \n" + " asr "J", "N", #2 \n" + " cmp "J", xzr \n" + " ble .Lasum_kernel_S1 \n" + + ".Lasum_kernel_S4: \n" + " "KERNEL_S1" \n" + " "KERNEL_S1" \n" + " "KERNEL_S1" \n" + " "KERNEL_S1" \n" + " subs "J", "J", #1 \n" + " bne .Lasum_kernel_S4 \n" + + ".Lasum_kernel_S1: \n" + " ands "J", "N", #3 \n" + " ble .Lasum_kernel_L999 \n" + + ".Lasum_kernel_S10: \n" + " "KERNEL_S1" \n" + " subs "J", "J", #1 \n" + " bne .Lasum_kernel_S10 \n" + + ".Lasum_kernel_L999: \n" + " fmov %[ASUM_], "SUMF" \n" + + : [ASUM_] "=r" (asum) //%0 + : [N_] "r" (n), //%1 + [X_] "r" (x), //%2 + [INCX_] "r" (inc_x) //%3 + : "cc", + "memory", + "x0", "x1", "x2", "x3", "x4", "x5", + "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7" + ); + + return asum; +} + +#if defined(SMP) +static int zasum_thread_function(BLASLONG n, BLASLONG dummy0, + BLASLONG dummy1, FLOAT dummy2, FLOAT *x, BLASLONG inc_x, FLOAT *y, + BLASLONG inc_y, FLOAT *result, BLASLONG dummy3) +{ + *result = zasum_compute(n, x, inc_x); + + return 0; +} +#endif + +FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) +{ +#if defined(SMP) + int nthreads; + FLOAT dummy_alpha; +#endif + FLOAT asum = 0.0; + +#if defined(SMP) + nthreads = num_cpu_avail(1); + + if (inc_x == 0) + nthreads = 1; + + if (n <= 10000) + nthreads = 1; + + if (nthreads == 1) { + asum = zasum_compute(n, x, inc_x); + } else { + int mode, i; + char result[MAX_CPU_NUMBER * sizeof(double) * 2]; + FLOAT *ptr; + + mode = BLAS_DOUBLE | BLAS_COMPLEX; + + blas_level1_thread_with_return_value(mode, n, 0, 0, &dummy_alpha, + x, inc_x, NULL, 0, result, 0, + ( void *)zasum_thread_function, nthreads); + + ptr = (FLOAT *)result; + for (i = 0; i < nthreads; i++) { + asum = asum + (*ptr); + ptr = (FLOAT *)(((char *)ptr) + sizeof(double) * 2); + } + } +#else + asum = zasum_compute(n, x, inc_x); +#endif + + return asum; +} From d09f88192c8569c23932ab1edcc52fcf2ac7fa50 Mon Sep 17 00:00:00 2001 From: Ashwin Sekhar T K Date: Wed, 1 Feb 2017 22:10:35 -0800 Subject: [PATCH 12/12] THUNDERX2T99: Add optimized S/D/C/Z COPY Implementations --- kernel/arm64/KERNEL.THUNDERX2T99 | 5 + kernel/arm64/copy_thunderx2t99.c | 219 +++++++++++++++++++++++++++++++ 2 files changed, 224 insertions(+) create mode 100644 kernel/arm64/copy_thunderx2t99.c diff --git a/kernel/arm64/KERNEL.THUNDERX2T99 b/kernel/arm64/KERNEL.THUNDERX2T99 index 6964aabf8..497768dbf 100644 --- a/kernel/arm64/KERNEL.THUNDERX2T99 +++ b/kernel/arm64/KERNEL.THUNDERX2T99 @@ -5,6 +5,11 @@ DASUMKERNEL = dasum_thunderx2t99.c CASUMKERNEL = casum_thunderx2t99.c ZASUMKERNEL = zasum_thunderx2t99.c +SCOPYKERNEL = copy_thunderx2t99.c +DCOPYKERNEL = copy_thunderx2t99.c +CCOPYKERNEL = copy_thunderx2t99.c +ZCOPYKERNEL = copy_thunderx2t99.c + SNRM2KERNEL = snrm2_thunderx2t99.c CNRM2KERNEL = cnrm2_thunderx2t99.S diff --git a/kernel/arm64/copy_thunderx2t99.c b/kernel/arm64/copy_thunderx2t99.c new file mode 100644 index 000000000..49526a15e --- /dev/null +++ b/kernel/arm64/copy_thunderx2t99.c @@ -0,0 +1,219 @@ +/*************************************************************************** +Copyright (c) 2017, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include "common.h" + +#include +#define N "x0" /* vector length */ +#define X "x1" /* X vector address */ +#define INC_X "x2" /* X stride */ +#define Y "x3" /* Y vector address */ +#define INC_Y "x4" /* Y stride */ +#define J "x5" /* loop variable */ + +/******************************************************************************* +* Macro definitions +*******************************************************************************/ +#if !defined(COMPLEX) +#if !defined(DOUBLE) +#define TMPF "s0" +#define INC_SHIFT "2" +#define N_DIV_SHIFT "2" +#define N_REM_MASK "3" +#else +#define TMPF "d0" +#define INC_SHIFT "3" +#define N_DIV_SHIFT "1" +#define N_REM_MASK "1" +#endif +#else +#if !defined(DOUBLE) +#define TMPF "d0" +#define INC_SHIFT "3" +#define N_DIV_SHIFT "1" +#define N_REM_MASK "1" +#else +#define TMPF "q0" +#define INC_SHIFT "4" +#define N_DIV_SHIFT "0" +#define N_REM_MASK "0" +#endif +#endif + +#define KERNEL_F1 \ + "ldr "TMPF", ["X"] \n" \ + "add "X", "X", "INC_X" \n" \ + "str "TMPF", ["Y"] \n" \ + "add "Y", "Y", "INC_Y" \n" + +#define KERNEL_F \ + "ldr q0, ["X"], #16 \n" \ + "str q0, ["Y"], #16 \n" + +#define INIT \ + "lsl "INC_X", "INC_X", #"INC_SHIFT" \n" \ + "lsl "INC_Y", "INC_Y", #"INC_SHIFT" \n" + + +static int do_copy(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y) +{ + if ( n < 0 ) return 0; + + __asm__ __volatile__ ( + " mov "N", %[N_] \n" + " mov "X", %[X_] \n" + " mov "INC_X", %[INCX_] \n" + " mov "Y", %[Y_] \n" + " mov "INC_Y", %[INCY_] \n" + " cmp "N", xzr \n" + " ble .Lcopy_kernel_L999 \n" + " cmp "INC_X", #1 \n" + " bne .Lcopy_kernel_S_BEGIN \n" + " cmp "INC_Y", #1 \n" + " bne .Lcopy_kernel_S_BEGIN \n" + + ".Lcopy_kernel_F_BEGIN: \n" + " "INIT" \n" + " asr "J", "N", #"N_DIV_SHIFT" \n" + " cmp "J", xzr \n" + " beq .Lcopy_kernel_F1 \n" + " .align 5 \n" + + ".Lcopy_kernel_F: \n" + " "KERNEL_F" \n" + " subs "J", "J", #1 \n" + " bne .Lcopy_kernel_F \n" + + ".Lcopy_kernel_F1: \n" +#if defined(COMPLEX) && defined(DOUBLE) + " b .Lcopy_kernel_L999 \n" +#else + " ands "J", "N", #"N_REM_MASK" \n" + " ble .Lcopy_kernel_L999 \n" +#endif + + ".Lcopy_kernel_F10: \n" + " "KERNEL_F1" \n" + " subs "J", "J", #1 \n" + " bne .Lcopy_kernel_F10 \n" + " b .Lcopy_kernel_L999 \n" + + ".Lcopy_kernel_S_BEGIN: \n" + " "INIT" \n" + " asr "J", "N", #2 \n" + " cmp "J", xzr \n" + " ble .Lcopy_kernel_S1 \n" + + ".Lcopy_kernel_S4: \n" + " "KERNEL_F1" \n" + " "KERNEL_F1" \n" + " "KERNEL_F1" \n" + " "KERNEL_F1" \n" + " subs "J", "J", #1 \n" + " bne .Lcopy_kernel_S4 \n" + + ".Lcopy_kernel_S1: \n" + " ands "J", "N", #3 \n" + " ble .Lcopy_kernel_L999 \n" + + ".Lcopy_kernel_S10: \n" + " "KERNEL_F1" \n" + " subs "J", "J", #1 \n" + " bne .Lcopy_kernel_S10 \n" + + ".Lcopy_kernel_L999: \n" + + : + : [N_] "r" (n), //%1 + [X_] "r" (x), //%2 + [INCX_] "r" (inc_x), //%3 + [Y_] "r" (y), //%4 + [INCY_] "r" (inc_y) //%5 + : "cc", + "memory", + "x0", "x1", "x2", "x3", "x4", "x5", + "d0" + ); + + return 0; +} + +#if defined(SMP) +static int copy_thread_function(BLASLONG n, BLASLONG dummy0, + BLASLONG dummy1, FLOAT dummy2, FLOAT *x, BLASLONG inc_x, FLOAT *y, + BLASLONG inc_y, FLOAT *dummy3, BLASLONG dummy4) +{ + do_copy(n, x, inc_x, y, inc_y); + + return 0; +} +#endif + +int CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y) +{ +#if defined(SMP) + int nthreads; + FLOAT dummy_alpha; +#endif + + if (n <= 0) return 0; + +#if defined(SMP) + nthreads = num_cpu_avail(1); + + if (inc_x == 0) + nthreads = 1; + + if (n <= 10000) + nthreads = 1; + + if (nthreads == 1) { + do_copy(n, x, inc_x, y, inc_y); + } else { + int mode = 0; + +#if !defined(COMPLEX) + mode = BLAS_REAL; +#else + mode = BLAS_COMPLEX; +#endif +#if !defined(DOUBLE) + mode |= BLAS_SINGLE; +#else + mode |= BLAS_DOUBLE; +#endif + + blas_level1_thread(mode, n, 0, 0, &dummy_alpha, + x, inc_x, y, inc_y, NULL, 0, + ( void *)copy_thread_function, nthreads); + } +#else + do_copy(n, x, inc_x, y, inc_y); +#endif + + return 0; +}