From 1ca750471a89d183ad20610cd0fe4db1e8d0f25b Mon Sep 17 00:00:00 2001 From: Werner Saar Date: Sun, 10 Apr 2016 11:28:20 +0200 Subject: [PATCH 1/4] added cholesky benchmarks to Makefile for ESSL --- benchmark/Makefile | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/benchmark/Makefile b/benchmark/Makefile index 8166f3863..0a1359254 100644 --- a/benchmark/Makefile +++ b/benchmark/Makefile @@ -261,7 +261,8 @@ endif essl :: sgemm.essl strmm.essl dgemm.essl dtrmm.essl \ cgemm.essl ctrmm.essl zgemm.essl ztrmm.essl \ - slinpack.essl clinpack.essl dlinpack.essl zlinpack.essl + slinpack.essl clinpack.essl dlinpack.essl zlinpack.essl \ + scholesky.essl ccholesky.essl dcholesky.essl zcholesky.essl veclib :: slinpack.veclib dlinpack.veclib clinpack.veclib zlinpack.veclib \ scholesky.veclib dcholesky.veclib ccholesky.veclib zcholesky.veclib \ @@ -393,6 +394,9 @@ scholesky.mkl : scholesky.$(SUFFIX) scholesky.veclib : scholesky.$(SUFFIX) -$(CC) $(CFLAGS) -o $(@F) $^ $(LIBVECLIB) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) +scholesky.essl : scholesky.$(SUFFIX) + -$(CC) $(CFLAGS) -o $(@F) $^ $(LIBESSL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) + ##################################### Dcholesky ################################################### dcholesky.goto : dcholesky.$(SUFFIX) ../$(LIBNAME) @@ -410,6 +414,9 @@ dcholesky.mkl : dcholesky.$(SUFFIX) dcholesky.veclib : dcholesky.$(SUFFIX) -$(CC) $(CFLAGS) -o $(@F) $^ $(LIBVECLIB) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) +dcholesky.essl : dcholesky.$(SUFFIX) + -$(CC) $(CFLAGS) -o $(@F) $^ $(LIBESSL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) + ##################################### Ccholesky ################################################### ccholesky.goto : ccholesky.$(SUFFIX) ../$(LIBNAME) @@ -427,6 +434,9 @@ ccholesky.mkl : ccholesky.$(SUFFIX) ccholesky.veclib : ccholesky.$(SUFFIX) -$(CC) $(CFLAGS) -o $(@F) $^ $(LIBVECLIB) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) +ccholesky.essl : ccholesky.$(SUFFIX) + -$(CC) $(CFLAGS) -o $(@F) $^ $(LIBESSL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) + ##################################### Zcholesky ################################################### @@ -445,6 +455,9 @@ zcholesky.mkl : zcholesky.$(SUFFIX) zcholesky.veclib : zcholesky.$(SUFFIX) -$(CC) $(CFLAGS) -o $(@F) $^ $(LIBVECLIB) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) +zcholesky.essl : zcholesky.$(SUFFIX) + -$(CC) $(CFLAGS) -o $(@F) $^ $(LIBESSL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) + ##################################### Sgemm #################################################### sgemm.goto : sgemm.$(SUFFIX) ../$(LIBNAME) $(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm From 8037d78eed1c1685a5ded60c388756c2eda6357a Mon Sep 17 00:00:00 2001 From: Werner Saar Date: Mon, 11 Apr 2016 11:21:36 +0200 Subject: [PATCH 2/4] bugfix for arm scal.c and zscal.c --- kernel/arm/scal.c | 4 ++++ kernel/arm/zscal.c | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/kernel/arm/scal.c b/kernel/arm/scal.c index 91ca76569..4ef49e293 100644 --- a/kernel/arm/scal.c +++ b/kernel/arm/scal.c @@ -40,6 +40,10 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS { BLASLONG i=0,j=0; + if ( (n <= 0) || (inc_x <= 0)) + return(0); + + while(j < n) { diff --git a/kernel/arm/zscal.c b/kernel/arm/zscal.c index f543edc04..0521aaa0b 100644 --- a/kernel/arm/zscal.c +++ b/kernel/arm/zscal.c @@ -43,6 +43,10 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r,FLOAT da_i, F BLASLONG ip = 0; FLOAT temp; + if ( (n <= 0) || (inc_x <= 0)) + return(0); + + inc_x2 = 2 * inc_x; for ( i=0; i Date: Tue, 19 Apr 2016 16:08:54 +0200 Subject: [PATCH 3/4] added optimized sgemm_tcopy for power8 --- Makefile.power | 1 - kernel/power/KERNEL.POWER8 | 2 +- kernel/power/sgemm_tcopy_16_power8.S | 176 +++++++++ kernel/power/sgemm_tcopy_logic_16_power8.S | 268 ++++++++++++++ kernel/power/sgemm_tcopy_macros_16_power8.S | 381 ++++++++++++++++++++ param.h | 7 +- 6 files changed, 827 insertions(+), 8 deletions(-) create mode 100644 kernel/power/sgemm_tcopy_16_power8.S create mode 100644 kernel/power/sgemm_tcopy_logic_16_power8.S create mode 100644 kernel/power/sgemm_tcopy_macros_16_power8.S diff --git a/Makefile.power b/Makefile.power index 7e2b47386..cc138e60a 100644 --- a/Makefile.power +++ b/Makefile.power @@ -1,4 +1,3 @@ -# CCOMMON_OPT += -DALLOC_SHM FLAMEPATH = $(HOME)/flame/lib diff --git a/kernel/power/KERNEL.POWER8 b/kernel/power/KERNEL.POWER8 index b37a4213b..fa5cc5e63 100644 --- a/kernel/power/KERNEL.POWER8 +++ b/kernel/power/KERNEL.POWER8 @@ -10,7 +10,7 @@ ZTRMMKERNEL = ztrmm_kernel_8x2_power8.S SGEMMKERNEL = sgemm_kernel_16x8_power8.S SGEMMINCOPY = ../generic/gemm_ncopy_16.c -SGEMMITCOPY = ../generic/gemm_tcopy_16.c +SGEMMITCOPY = sgemm_tcopy_16_power8.S SGEMMONCOPY = ../generic/gemm_ncopy_8.c SGEMMOTCOPY = ../generic/gemm_tcopy_8.c SGEMMINCOPYOBJ = sgemm_incopy.o diff --git a/kernel/power/sgemm_tcopy_16_power8.S b/kernel/power/sgemm_tcopy_16_power8.S new file mode 100644 index 000000000..d690aab8e --- /dev/null +++ b/kernel/power/sgemm_tcopy_16_power8.S @@ -0,0 +1,176 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#define ASSEMBLER +#include "common.h" +#include "def_vsx.h" + +#define M r3 +#define N r4 +#define A r5 +#define LDA r6 +#define B r7 + +#define A0 r8 +#define A1 r9 +#define A2 r10 +#define A3 r11 + +#define J r12 + +#define PREA r14 +#define PREB r15 +#define BO r16 +#define B8 r17 +#define B4 r18 +#define B2 r19 +#define B1 r20 +#define o4 r21 +#define T2 r22 +#define I r23 +#define o16 r24 +#define o32 r25 +#define o48 r26 +#define B16 r29 +#define M16 r30 +#define T1 r31 + +#define o0 0 + +#include "sgemm_tcopy_macros_16_power8.S" + +#define STACKSIZE 384 + + + PROLOGUE + PROFCODE + + addi SP, SP, -STACKSIZE + li r0, 0 + + std r31, 144(SP) + std r30, 152(SP) + std r29, 160(SP) + std r28, 168(SP) + std r27, 176(SP) + std r26, 184(SP) + std r25, 192(SP) + std r24, 200(SP) + std r23, 208(SP) + std r22, 216(SP) + std r21, 224(SP) + std r20, 232(SP) + std r19, 240(SP) + std r18, 248(SP) + std r17, 256(SP) + std r16, 264(SP) + std r15, 272(SP) + std r14, 280(SP) + + cmpwi cr0, M, 0 + ble- L999 + cmpwi cr0, N, 0 + ble- L999 + + slwi LDA, LDA, BASE_SHIFT + slwi M16, M, 4 + BASE_SHIFT + + li T1, -16 + li T2, -8 + li PREA, -4 + li PREB, -2 + + and B8, N, T1 + and B4, N, T2 + and B2, N, PREA + and B1, N, PREB + + mullw B8, B8, M + mullw B4, B4, M + mullw B2, B2, M + mullw B1, B1, M + + slwi B8, B8, BASE_SHIFT + slwi B4, B4, BASE_SHIFT + slwi B2, B2, BASE_SHIFT + slwi B1, B1, BASE_SHIFT + + add B8, B8, B + add B4, B4, B + add B2, B2, B + add B1, B1, B + + li PREA, 384 + li PREB, 384 + + li o4, 4 + li o16, 16 + li o32, 32 + li o48, 48 + +#include "sgemm_tcopy_logic_16_power8.S" + +L999: + + li r3, 0 + + ld r31, 144(SP) + ld r30, 152(SP) + ld r29, 160(SP) + ld r28, 168(SP) + ld r27, 176(SP) + ld r26, 184(SP) + ld r25, 192(SP) + ld r24, 200(SP) + ld r23, 208(SP) + ld r22, 216(SP) + ld r21, 224(SP) + ld r20, 232(SP) + ld r19, 240(SP) + ld r18, 248(SP) + ld r17, 256(SP) + ld r16, 264(SP) + ld r15, 272(SP) + ld r14, 280(SP) + + addi SP, SP, STACKSIZE + + blr + EPILOGUE + + diff --git a/kernel/power/sgemm_tcopy_logic_16_power8.S b/kernel/power/sgemm_tcopy_logic_16_power8.S new file mode 100644 index 000000000..c0969eec6 --- /dev/null +++ b/kernel/power/sgemm_tcopy_logic_16_power8.S @@ -0,0 +1,268 @@ + srawi. I, M, 2 + ble SCOPYT_L2_BEGIN + + +SCOPYT_L4_BEGIN: + + mr A0, A + add A1, A0, LDA + add A2, A1, LDA + add A3, A2, LDA + add A, A3, LDA + mr B16, B + addi B, B, 64*SIZE + + sradi. J, N, 4 + ble SCOPYT_L4x8_BEGIN + + mr BO, B16 + +SCOPYT_L4x16_LOOP: + + COPY_4x16 + + addi A0, A0, 16*SIZE + addi A1, A1, 16*SIZE + addi A2, A2, 16*SIZE + addi A3, A3, 16*SIZE + add BO, BO, M16 + + addic. J, J, -1 + bgt SCOPYT_L4x16_LOOP + +SCOPYT_L4x8_BEGIN: + + andi. T1, N, 8 + ble SCOPYT_L4x4_BEGIN + + mr BO, B8 + + COPY_4x8 + + addi A0, A0, 8*SIZE + addi A1, A1, 8*SIZE + addi A2, A2, 8*SIZE + addi A3, A3, 8*SIZE + + addi B8, B8, 32*SIZE + +SCOPYT_L4x4_BEGIN: + + andi. T1, N, 4 + ble SCOPYT_L4x2_BEGIN + + mr BO, B4 + + COPY_4x4 + + addi A0, A0, 4*SIZE + addi A1, A1, 4*SIZE + addi A2, A2, 4*SIZE + addi A3, A3, 4*SIZE + + addi B4, B4, 16*SIZE + +SCOPYT_L4x2_BEGIN: + + andi. T1, N, 2 + ble SCOPYT_L4x1_BEGIN + + mr BO, B2 + + COPY_4x2 + + addi A0, A0, 2*SIZE + addi A1, A1, 2*SIZE + addi A2, A2, 2*SIZE + addi A3, A3, 2*SIZE + + addi B2, B2, 8*SIZE + +SCOPYT_L4x1_BEGIN: + + andi. T1, N, 1 + ble SCOPYT_L4_END + + mr BO, B1 + + COPY_4x1 + + addi A0, A0, 1*SIZE + addi A1, A1, 1*SIZE + addi A2, A2, 1*SIZE + addi A3, A3, 1*SIZE + + addi B1, B1, 4*SIZE + +SCOPYT_L4_END: + + addic. I, I, -1 + bgt SCOPYT_L4_BEGIN + + + +SCOPYT_L2_BEGIN: + + andi. T1, M, 2 + ble SCOPYT_L1_BEGIN + + mr A0, A + add A1, A0, LDA + add A, A1, LDA + mr B16, B + addi B, B, 32*SIZE + + sradi. J, N, 4 + ble SCOPYT_L2x8_BEGIN + + mr BO, B16 + +SCOPYT_L2x16_LOOP: + + COPY_2x16 + + addi A0, A0, 16*SIZE + addi A1, A1, 16*SIZE + add BO, BO, M16 + + addic. J, J, -1 + bgt SCOPYT_L2x16_LOOP + +SCOPYT_L2x8_BEGIN: + + andi. T1, N, 8 + ble SCOPYT_L2x4_BEGIN + + mr BO, B8 + + COPY_2x8 + + addi A0, A0, 8*SIZE + addi A1, A1, 8*SIZE + + addi B8, B8, 16*SIZE + +SCOPYT_L2x4_BEGIN: + + andi. T1, N, 4 + ble SCOPYT_L2x2_BEGIN + + mr BO, B4 + + COPY_2x4 + + addi A0, A0, 4*SIZE + addi A1, A1, 4*SIZE + + addi B4, B4, 8*SIZE + +SCOPYT_L2x2_BEGIN: + + andi. T1, N, 2 + ble SCOPYT_L2x1_BEGIN + + mr BO, B2 + + COPY_2x2 + + addi A0, A0, 2*SIZE + addi A1, A1, 2*SIZE + + addi B2, B2, 4*SIZE + +SCOPYT_L2x1_BEGIN: + + andi. T1, N, 1 + ble SCOPYT_L2_END + + mr BO, B1 + + COPY_2x1 + + addi A0, A0, 1*SIZE + addi A1, A1, 1*SIZE + + addi B1, B1, 2*SIZE + +SCOPYT_L2_END: + + +SCOPYT_L1_BEGIN: + + andi. T1, M, 1 + ble L999 + + mr A0, A + add A, A0, LDA + mr B16, B + addi B, B, 16*SIZE + + sradi. J, N, 4 + ble SCOPYT_L1x8_BEGIN + + mr BO, B16 + +SCOPYT_L1x16_LOOP: + + COPY_1x16 + + addi A0, A0, 16*SIZE + add BO, BO, M16 + + addic. J, J, -1 + bgt SCOPYT_L1x16_LOOP + +SCOPYT_L1x8_BEGIN: + + andi. T1, N, 8 + ble SCOPYT_L1x4_BEGIN + + mr BO, B8 + + COPY_1x8 + + addi A0, A0, 8*SIZE + + addi B8, B8, 8*SIZE + +SCOPYT_L1x4_BEGIN: + + andi. T1, N, 4 + ble SCOPYT_L1x2_BEGIN + + mr BO, B4 + + COPY_1x4 + + addi A0, A0, 4*SIZE + + addi B4, B4, 4*SIZE + +SCOPYT_L1x2_BEGIN: + + andi. T1, N, 2 + ble SCOPYT_L1x1_BEGIN + + mr BO, B2 + + COPY_1x2 + + addi A0, A0, 2*SIZE + + addi B2, B2, 2*SIZE + +SCOPYT_L1x1_BEGIN: + + andi. T1, N, 1 + ble SCOPYT_L1_END + + mr BO, B1 + + COPY_1x1 + + addi A0, A0, 1*SIZE + + addi B1, B1, 1*SIZE + +SCOPYT_L1_END: + diff --git a/kernel/power/sgemm_tcopy_macros_16_power8.S b/kernel/power/sgemm_tcopy_macros_16_power8.S new file mode 100644 index 000000000..36c4593b5 --- /dev/null +++ b/kernel/power/sgemm_tcopy_macros_16_power8.S @@ -0,0 +1,381 @@ + +/********************************************************************************************** +* Macros for N=4 and M=16 +**********************************************************************************************/ + +.macro COPY_4x16 + + lxvw4x vs32, o0, A0 + lxvw4x vs33, o16, A0 + lxvw4x vs34, o32, A0 + lxvw4x vs35, o48, A0 + + lxvw4x vs36, o0, A1 + lxvw4x vs37, o16, A1 + lxvw4x vs38, o32, A1 + lxvw4x vs39, o48, A1 + + lxvw4x vs40, o0, A2 + lxvw4x vs41, o16, A2 + lxvw4x vs42, o32, A2 + lxvw4x vs43, o48, A2 + + lxvw4x vs44, o0, A3 + lxvw4x vs45, o16, A3 + lxvw4x vs46, o32, A3 + lxvw4x vs47, o48, A3 + + mr T1, BO + + stxvw4x vs32, o0, T1 + stxvw4x vs33, o16, T1 + stxvw4x vs34, o32, T1 + stxvw4x vs35, o48, T1 + + addi T1, T1, 64 + + stxvw4x vs36, o0, T1 + stxvw4x vs37, o16, T1 + stxvw4x vs38, o32, T1 + stxvw4x vs39, o48, T1 + + addi T1, T1, 64 + + stxvw4x vs40, o0, T1 + stxvw4x vs41, o16, T1 + stxvw4x vs42, o32, T1 + stxvw4x vs43, o48, T1 + + addi T1, T1, 64 + + stxvw4x vs44, o0, T1 + stxvw4x vs45, o16, T1 + stxvw4x vs46, o32, T1 + stxvw4x vs47, o48, T1 + +.endm + +/********************************************************************************************** +* Macros for N=4 and M=8 +**********************************************************************************************/ + +.macro COPY_4x8 + + lxvw4x vs32, o0, A0 + lxvw4x vs33, o16, A0 + + lxvw4x vs34, o0, A1 + lxvw4x vs35, o16, A1 + + lxvw4x vs36, o0, A2 + lxvw4x vs37, o16, A2 + + lxvw4x vs38, o0, A3 + lxvw4x vs39, o16, A3 + + mr T1, BO + + stxvw4x vs32, o0, T1 + stxvw4x vs33, o16, T1 + + stxvw4x vs34, o32, T1 + stxvw4x vs35, o48, T1 + + addi T1, T1, 64 + + stxvw4x vs36, o0, T1 + stxvw4x vs37, o16, T1 + + stxvw4x vs38, o32, T1 + stxvw4x vs39, o48, T1 + +.endm + +/********************************************************************************************** +* Macros for N=4 and M=4 +**********************************************************************************************/ + +.macro COPY_4x4 + + lxvw4x vs32, o0, A0 + + lxvw4x vs33, o0, A1 + + lxvw4x vs34, o0, A2 + + lxvw4x vs35, o0, A3 + + mr T1, BO + + stxvw4x vs32, o0, T1 + + stxvw4x vs33, o16, T1 + + stxvw4x vs34, o32, T1 + + stxvw4x vs35, o48, T1 + +.endm + +/********************************************************************************************** +* Macros for N=4 and M=2 +**********************************************************************************************/ + +.macro COPY_4x2 + + lxsspx vs32, o0, A0 + lxsspx vs33, o4, A0 + + lxsspx vs34, o0, A1 + lxsspx vs35, o4, A1 + + lxsspx vs36, o0, A2 + lxsspx vs37, o4, A2 + + lxsspx vs38, o0, A3 + lxsspx vs39, o4, A3 + + mr T1, BO + + stxsspx vs32, o0, T1 + stxsspx vs33, o4, T1 + + addi T1, T1, 8 + + stxsspx vs34, o0, T1 + stxsspx vs35, o4, T1 + + addi T1, T1, 8 + + stxsspx vs36, o0, T1 + stxsspx vs37, o4, T1 + + addi T1, T1, 8 + + stxsspx vs38, o0, T1 + stxsspx vs39, o4, T1 + +.endm + +/********************************************************************************************** +* Macros for N=4 and M=1 +**********************************************************************************************/ + +.macro COPY_4x1 + + lxsspx vs32, o0, A0 + + lxsspx vs33, o0, A1 + + lxsspx vs34, o0, A2 + + lxsspx vs35, o0, A3 + + mr T1, BO + + stxsspx vs32, o0, T1 + + stxsspx vs33, o4, T1 + + addi T1, T1, 8 + + stxsspx vs34, o0, T1 + + stxsspx vs35, o4, T1 + +.endm + +/********************************************************************************************** +* Macros for N=2 and M=16 +**********************************************************************************************/ + +.macro COPY_2x16 + + lxvw4x vs32, o0, A0 + lxvw4x vs33, o16, A0 + lxvw4x vs34, o32, A0 + lxvw4x vs35, o48, A0 + + lxvw4x vs36, o0, A1 + lxvw4x vs37, o16, A1 + lxvw4x vs38, o32, A1 + lxvw4x vs39, o48, A1 + + mr T1, BO + + stxvw4x vs32, o0, T1 + stxvw4x vs33, o16, T1 + stxvw4x vs34, o32, T1 + stxvw4x vs35, o48, T1 + + addi T1, T1, 64 + + stxvw4x vs36, o0, T1 + stxvw4x vs37, o16, T1 + stxvw4x vs38, o32, T1 + stxvw4x vs39, o48, T1 + +.endm + +/********************************************************************************************** +* Macros for N=2 and M=8 +**********************************************************************************************/ + +.macro COPY_2x8 + + lxvw4x vs32, o0, A0 + lxvw4x vs33, o16, A0 + + lxvw4x vs34, o0, A1 + lxvw4x vs35, o16, A1 + + mr T1, BO + + stxvw4x vs32, o0, T1 + stxvw4x vs33, o16, T1 + + stxvw4x vs34, o32, T1 + stxvw4x vs35, o48, T1 + +.endm + +/********************************************************************************************** +* Macros for N=2 and M=4 +**********************************************************************************************/ + +.macro COPY_2x4 + + lxvw4x vs32, o0, A0 + + lxvw4x vs33, o0, A1 + + mr T1, BO + + stxvw4x vs32, o0, T1 + + stxvw4x vs33, o16, T1 + +.endm + +/********************************************************************************************** +* Macros for N=2 and M=2 +**********************************************************************************************/ + +.macro COPY_2x2 + + lxsspx vs32, o0, A0 + lxsspx vs33, o4, A0 + + lxsspx vs34, o0, A1 + lxsspx vs35, o4, A1 + + mr T1, BO + + stxsspx vs32, o0, T1 + stxsspx vs33, o4, T1 + + addi T1, T1, 8 + + stxsspx vs34, o0, T1 + stxsspx vs35, o4, T1 + +.endm + +/********************************************************************************************** +* Macros for N=2 and M=1 +**********************************************************************************************/ + +.macro COPY_2x1 + + lxsspx vs32, o0, A0 + + lxsspx vs33, o0, A1 + + mr T1, BO + + stxsspx vs32, o0, T1 + + stxsspx vs33, o4, T1 + +.endm + +/********************************************************************************************** +* Macros for N=1 and M=16 +**********************************************************************************************/ + +.macro COPY_1x16 + + lxvw4x vs32, o0, A0 + lxvw4x vs33, o16, A0 + lxvw4x vs34, o32, A0 + lxvw4x vs35, o48, A0 + + mr T1, BO + + stxvw4x vs32, o0, T1 + stxvw4x vs33, o16, T1 + stxvw4x vs34, o32, T1 + stxvw4x vs35, o48, T1 + +.endm + +/********************************************************************************************** +* Macros for N=1 and M=8 +**********************************************************************************************/ + +.macro COPY_1x8 + + lxvw4x vs32, o0, A0 + lxvw4x vs33, o16, A0 + + mr T1, BO + + stxvw4x vs32, o0, T1 + stxvw4x vs33, o16, T1 + +.endm + +/********************************************************************************************** +* Macros for N=1 and M=4 +**********************************************************************************************/ + +.macro COPY_1x4 + + lxvw4x vs32, o0, A0 + + mr T1, BO + + stxvw4x vs32, o0, T1 + +.endm + +/********************************************************************************************** +* Macros for N=1 and M=2 +**********************************************************************************************/ + +.macro COPY_1x2 + + lxsspx vs32, o0, A0 + lxsspx vs33, o4, A0 + + mr T1, BO + + stxsspx vs32, o0, T1 + stxsspx vs33, o4, T1 + +.endm + +/********************************************************************************************** +* Macros for N=1 and M=1 +**********************************************************************************************/ + +.macro COPY_1x1 + + lxsspx vs32, o0, A0 + + mr T1, BO + + stxsspx vs32, o0, T1 + +.endm + diff --git a/param.h b/param.h index a6ead4b64..8ecc812dc 100644 --- a/param.h +++ b/param.h @@ -1964,7 +1964,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define SNUMOPT 16 #define DNUMOPT 8 -#define GEMM_DEFAULT_OFFSET_A 4096 +#define GEMM_DEFAULT_OFFSET_A 0 #define GEMM_DEFAULT_OFFSET_B 4096 #define GEMM_DEFAULT_ALIGN 0x03fffUL @@ -1987,11 +1987,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define CGEMM_DEFAULT_Q 720 #define ZGEMM_DEFAULT_Q 720 -#define SGEMM_DEFAULT_R 21600 -#define DGEMM_DEFAULT_R 14400 -#define CGEMM_DEFAULT_R 16200 -#define ZGEMM_DEFAULT_R 21600 - #define SYMV_P 8 #endif From 0001260f4bc4c8bb5509ec29e002ad6f42e84345 Mon Sep 17 00:00:00 2001 From: Werner Saar Date: Wed, 20 Apr 2016 13:06:38 +0200 Subject: [PATCH 4/4] optimized sgemm --- kernel/power/sgemm_logic_16x8_power8.S | 119 ++++++-------- kernel/power/sgemm_macros_16x8_power8.S | 177 +++++++++++++++++---- kernel/power/sgemm_tcopy_16_power8.S | 2 +- kernel/power/sgemm_tcopy_logic_16_power8.S | 20 +++ 4 files changed, 210 insertions(+), 108 deletions(-) diff --git a/kernel/power/sgemm_logic_16x8_power8.S b/kernel/power/sgemm_logic_16x8_power8.S index 06bb79ea3..6ba999024 100644 --- a/kernel/power/sgemm_logic_16x8_power8.S +++ b/kernel/power/sgemm_logic_16x8_power8.S @@ -1,38 +1,3 @@ -/*************************************************************************** -Copyright (c) 2013-2016, The OpenBLAS Project -All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: -1. Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in -the documentation and/or other materials provided with the -distribution. -3. Neither the name of the OpenBLAS project nor the names of -its contributors may be used to endorse or promote products -derived from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*****************************************************************************/ - -/************************************************************************************** -* 2016/04/02 Werner Saar (wernsaar@googlemail.com) -* BLASTEST : OK -* CTEST : OK -* TEST : OK -* LAPACK-TEST : OK -**************************************************************************************/ - srawi. J, N, 3 ble SGEMM_L8_END @@ -40,35 +5,48 @@ SGEMM_L8_BEGIN: mr BO, B mr BBO, BBUFFER - slwi T1, K, 3 + srawi. T1, K, 2 + ble SGEMM_L8_COPYB1 -SGEMM_L8_COPYB: + +SGEMM_L8_COPYB4: + + dcbt BO, PRE dcbtst BBO, PRE + COPYB_4x8 + addic. T1, T1, -1 + ble SGEMM_L8_COPYB1 - lxvw4x vs3, o0, BO - lxvw4x vs11, o16, BO - xxspltw vs4, vs3, 0 - xxspltw vs5, vs3, 1 - xxspltw vs6, vs3, 2 - xxspltw vs7, vs3, 3 - xxspltw vs12, vs11, 0 - xxspltw vs13, vs11, 1 - xxspltw vs14, vs11, 2 - xxspltw vs15, vs11, 3 - stxvw4x vs4, o0, BBO - stxvw4x vs5, o16, BBO - stxvw4x vs6, o32, BBO - stxvw4x vs7, o48, BBO - addi BO, BO, 32 - addi BBO, BBO, 64 - stxvw4x vs12, o0, BBO - stxvw4x vs13, o16, BBO - stxvw4x vs14, o32, BBO - stxvw4x vs15, o48, BBO - addic. T1, T1, -8 - addi BBO, BBO, 64 + dcbtst BBO, PRE + COPYB_4x8 + addic. T1, T1, -1 + ble SGEMM_L8_COPYB1 - bge SGEMM_L8_COPYB + dcbtst BBO, PRE + COPYB_4x8 + addic. T1, T1, -1 + ble SGEMM_L8_COPYB1 + + dcbtst BBO, PRE + COPYB_4x8 + addic. T1, T1, -1 + + bgt SGEMM_L8_COPYB4 + +SGEMM_L8_COPYB1: + + andi. T1, K, 3 + ble SGEMM_L8_COPYB_END + +SGEMM_L8_COPYB1_LOOP: + + + COPYB_1x8 + addic. T1, T1, -1 + + bgt SGEMM_L8_COPYB1_LOOP + +SGEMM_L8_COPYB_END: mr CO, C mr AO, A @@ -93,24 +71,24 @@ SGEMM_L8x16_LOOP_START: LOAD8x16_1 dcbt BO, PRE KERNEL8x16_I1 - dcbt BO, PRE dcbt AO, PRE + dcbt BO, PRE KERNEL8x16_2 dcbt BO, PRE KERNEL8x16_1 - dcbt BO, PRE dcbt AO, PRE + dcbt BO, PRE KERNEL8x16_2 dcbt BO, PRE KERNEL8x16_1 - dcbt BO, PRE dcbt AO, PRE + dcbt BO, PRE KERNEL8x16_2 dcbt BO, PRE KERNEL8x16_1 - dcbt BO, PRE dcbt AO, PRE + dcbt BO, PRE KERNEL8x16_2 addic. L, L, -2 @@ -122,24 +100,24 @@ SGEMM_L8x16_LOOP: dcbt BO, PRE KERNEL8x16_1 - dcbt BO, PRE dcbt AO, PRE + dcbt BO, PRE KERNEL8x16_2 dcbt BO, PRE KERNEL8x16_1 - dcbt BO, PRE dcbt AO, PRE + dcbt BO, PRE KERNEL8x16_2 dcbt BO, PRE KERNEL8x16_1 - dcbt BO, PRE dcbt AO, PRE + dcbt BO, PRE KERNEL8x16_2 dcbt BO, PRE KERNEL8x16_1 - dcbt BO, PRE dcbt AO, PRE + dcbt BO, PRE KERNEL8x16_2 addic. L, L, -1 @@ -149,18 +127,15 @@ SGEMM_L8x16_LOOP_END: dcbt BO, PRE KERNEL8x16_1 - dcbt BO, PRE dcbt AO, PRE + dcbt BO, PRE KERNEL8x16_2 dcbt BO, PRE KERNEL8x16_1 - dcbt BO, PRE dcbt AO, PRE KERNEL8x16_2 - dcbt BO, PRE KERNEL8x16_1 - dcbt BO, PRE dcbt AO, PRE KERNEL8x16_2 KERNEL8x16_1 diff --git a/kernel/power/sgemm_macros_16x8_power8.S b/kernel/power/sgemm_macros_16x8_power8.S index 71dc52979..94e7d7d02 100644 --- a/kernel/power/sgemm_macros_16x8_power8.S +++ b/kernel/power/sgemm_macros_16x8_power8.S @@ -1,38 +1,3 @@ -/*************************************************************************** -Copyright (c) 2013-2016, The OpenBLAS Project -All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: -1. Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in -the documentation and/or other materials provided with the -distribution. -3. Neither the name of the OpenBLAS project nor the names of -its contributors may be used to endorse or promote products -derived from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*****************************************************************************/ - -/************************************************************************************** -* 2016/04/02 Werner Saar (wernsaar@googlemail.com) -* BLASTEST : OK -* CTEST : OK -* TEST : OK -* LAPACK-TEST : OK -**************************************************************************************/ - /********************************************************************************************** * Macros for N=8 and M=16 @@ -5886,3 +5851,145 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .endm + + + + +.macro COPYB_4x8 + + + lxvw4x vs5, o0, BO + xxspltw vs6, vs5, 0 + xxspltw vs7, vs5, 1 + xxspltw vs8, vs5, 2 + xxspltw vs9, vs5, 3 + + lxvw4x vs10, o16, BO + xxspltw vs11, vs10, 0 + xxspltw vs12, vs10, 1 + xxspltw vs13, vs10, 2 + xxspltw vs14, vs10, 3 + + lxvw4x vs15, o32, BO + xxspltw vs16, vs15, 0 + xxspltw vs17, vs15, 1 + xxspltw vs18, vs15, 2 + xxspltw vs19, vs15, 3 + + lxvw4x vs20, o48, BO + xxspltw vs21, vs20, 0 + xxspltw vs22, vs20, 1 + xxspltw vs23, vs20, 2 + xxspltw vs24, vs20, 3 + + addi BO, BO, 64 + lxvw4x vs35, o0, BO + xxspltw vs36, vs35, 0 + xxspltw vs37, vs35, 1 + xxspltw vs38, vs35, 2 + xxspltw vs39, vs35, 3 + + lxvw4x vs40, o16, BO + xxspltw vs41, vs40, 0 + xxspltw vs42, vs40, 1 + xxspltw vs43, vs40, 2 + xxspltw vs44, vs40, 3 + + lxvw4x vs45, o32, BO + xxspltw vs46, vs45, 0 + xxspltw vs47, vs45, 1 + xxspltw vs48, vs45, 2 + xxspltw vs49, vs45, 3 + + lxvw4x vs50, o48, BO + xxspltw vs51, vs50, 0 + xxspltw vs52, vs50, 1 + xxspltw vs53, vs50, 2 + xxspltw vs54, vs50, 3 + + addi BO, BO, 64 + + + stxvw4x vs6, o0, BBO + stxvw4x vs7, o16, BBO + stxvw4x vs8, o32, BBO + stxvw4x vs9, o48, BBO + + addi BBO, BBO, 64 + stxvw4x vs11, o0, BBO + stxvw4x vs12, o16, BBO + stxvw4x vs13, o32, BBO + stxvw4x vs14, o48, BBO + + addi BBO, BBO, 64 + stxvw4x vs16, o0, BBO + stxvw4x vs17, o16, BBO + stxvw4x vs18, o32, BBO + stxvw4x vs19, o48, BBO + + addi BBO, BBO, 64 + stxvw4x vs21, o0, BBO + stxvw4x vs22, o16, BBO + stxvw4x vs23, o32, BBO + stxvw4x vs24, o48, BBO + + addi BBO, BBO, 64 + stxvw4x vs36, o0, BBO + stxvw4x vs37, o16, BBO + stxvw4x vs38, o32, BBO + stxvw4x vs39, o48, BBO + + addi BBO, BBO, 64 + stxvw4x vs41, o0, BBO + stxvw4x vs42, o16, BBO + stxvw4x vs43, o32, BBO + stxvw4x vs44, o48, BBO + + addi BBO, BBO, 64 + stxvw4x vs46, o0, BBO + stxvw4x vs47, o16, BBO + stxvw4x vs48, o32, BBO + stxvw4x vs49, o48, BBO + + addi BBO, BBO, 64 + stxvw4x vs51, o0, BBO + stxvw4x vs52, o16, BBO + stxvw4x vs53, o32, BBO + stxvw4x vs54, o48, BBO + + addi BBO, BBO, 64 +.endm + + +.macro COPYB_1x8 + + + lxvw4x vs5, o0, BO + xxspltw vs6, vs5, 0 + xxspltw vs7, vs5, 1 + xxspltw vs8, vs5, 2 + xxspltw vs9, vs5, 3 + + lxvw4x vs10, o16, BO + xxspltw vs11, vs10, 0 + xxspltw vs12, vs10, 1 + xxspltw vs13, vs10, 2 + xxspltw vs14, vs10, 3 + + + addi BO, BO, 32 + + stxvw4x vs6, o0, BBO + stxvw4x vs7, o16, BBO + stxvw4x vs8, o32, BBO + stxvw4x vs9, o48, BBO + + addi BBO, BBO, 64 + stxvw4x vs11, o0, BBO + stxvw4x vs12, o16, BBO + stxvw4x vs13, o32, BBO + stxvw4x vs14, o48, BBO + + addi BBO, BBO, 64 +.endm + diff --git a/kernel/power/sgemm_tcopy_16_power8.S b/kernel/power/sgemm_tcopy_16_power8.S index d690aab8e..c31784d6f 100644 --- a/kernel/power/sgemm_tcopy_16_power8.S +++ b/kernel/power/sgemm_tcopy_16_power8.S @@ -136,7 +136,7 @@ add B1, B1, B li PREA, 384 - li PREB, 384 + addi PREB, M16, 128 li o4, 4 li o16, 16 diff --git a/kernel/power/sgemm_tcopy_logic_16_power8.S b/kernel/power/sgemm_tcopy_logic_16_power8.S index c0969eec6..5a715c8dd 100644 --- a/kernel/power/sgemm_tcopy_logic_16_power8.S +++ b/kernel/power/sgemm_tcopy_logic_16_power8.S @@ -19,6 +19,26 @@ SCOPYT_L4_BEGIN: SCOPYT_L4x16_LOOP: + dcbtst BO, M16 + dcbtst BO, PREB + dcbt A0, PREA + dcbt A1, PREA + dcbt A2, PREA + dcbt A3, PREA + COPY_4x16 + + addi A0, A0, 16*SIZE + addi A1, A1, 16*SIZE + addi A2, A2, 16*SIZE + addi A3, A3, 16*SIZE + add BO, BO, M16 + + addic. J, J, -1 + ble SCOPYT_L4x8_BEGIN + + + dcbtst BO, M16 + dcbtst BO, PREB COPY_4x16 addi A0, A0, 16*SIZE