floraachy
/
OpenBLAS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix to store results in correct order for POWER10 GEMM kernels 

There is a recent compiler change in __builtin_mma_disassemble_acc() which
affects the order of storing result in POWER10. Also removing new LDFLAG
-mno-power10-stub as it is handled by linker automatically.
This commit is contained in:
Rajalakshmi Srinivasaraghavan 2020-07-24 23:08:11 -05:00
parent 6a2a60038c
commit 9be2688c78
4 changed files with 113 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Makefile.system
View File

@ -617,7 +617,6 @@ DYNAMIC_CORE += POWER8
ifneq ($(C_COMPILER), GCC)
DYNAMIC_CORE += POWER9
DYNAMIC_CORE += POWER10
override LDFLAGS += -Wl,-no-power10-stubs
endif
ifeq ($(C_COMPILER), GCC)
ifeq ($(GCCVERSIONGT5), 1)
@ -627,11 +626,9 @@ $(info, OpenBLAS: Your gcc version is too old to build the POWER9 kernels.)
endif
ifeq ($(GCCVERSIONGTEQ11), 1)
DYNAMIC_CORE += POWER10
override LDFLAGS += -Wl,-no-power10-stubs
else ifeq ($(GCCVERSIONGTEQ10), 1)
ifeq ($(GCCMINORVERSIONGTEQ2), 1)
DYNAMIC_CORE += POWER10
override LDFLAGS += -Wl,-no-power10-stubs
endif
else
$(info, OpenBLAS: Your gcc version is too old to build the POWER10 kernels.)

66
kernel/power/dgemm_kernel_power10.c
View File

@ -27,64 +27,64 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "common.h"
#include <altivec.h>
typedef unsigned char vec_t __attribute__ ((vector_size (16)));
typedef __vector unsigned char vec_t;
typedef FLOAT v4sf_t __attribute__ ((vector_size (16)));
typedef FLOAT v2sf_t __attribute__ ((vector_size (8)));
#ifdef TRMMKERNEL
#define SAVE_ACC(ACC, J) \
__builtin_mma_disassemble_acc (result, ACC); \
__builtin_mma_disassemble_acc ((void *)result, ACC); \
rowC = (v4sf_t *) &CO[0* ldc+J]; \
rowC[0] = result[3] * alpha; \
rowC[0] = result[0] * alpha; \
rowC = (v4sf_t *) &CO[1*ldc+J]; \
rowC[0] = result[2] * alpha; \
rowC[0] = result[1] * alpha; \
rowC = (v4sf_t *) &CO[2*ldc+J]; \
rowC[0] = result[1] * alpha; \
rowC = (v4sf_t *) &CO[3*ldc+J]; \
rowC[0] = result[0] * alpha;
#define SAVE_ACC1(ACC, J) \
__builtin_mma_disassemble_acc (result, ACC); \
rowC = (v4sf_t *) &CO[4* ldc+J]; \
rowC[0] = result[3] * alpha; \
rowC = (v4sf_t *) &CO[5*ldc+J]; \
rowC[0] = result[2] * alpha; \
rowC = (v4sf_t *) &CO[6*ldc+J]; \
rowC = (v4sf_t *) &CO[3*ldc+J]; \
rowC[0] = result[3] * alpha;
#define SAVE_ACC1(ACC, J) \
__builtin_mma_disassemble_acc ((void *)result, ACC); \
rowC = (v4sf_t *) &CO[4* ldc+J]; \
rowC[0] = result[0] * alpha; \
rowC = (v4sf_t *) &CO[5*ldc+J]; \
rowC[0] = result[1] * alpha; \
rowC = (v4sf_t *) &CO[6*ldc+J]; \
rowC[0] = result[2] * alpha; \
rowC = (v4sf_t *) &CO[7*ldc+J]; \
rowC[0] = result[0] * alpha;
rowC[0] = result[3] * alpha;
#define SAVE2x4_ACC(ACC, J) \
__builtin_mma_disassemble_acc (result, ACC); \
__builtin_mma_disassemble_acc ((void *)result, ACC); \
rowC = (v4sf_t *) &CO[0* ldc+J]; \
rowC[0] = result[3] * alpha; \
rowC[0] = result[0] * alpha; \
rowC = (v4sf_t *) &CO[1* ldc+J]; \
rowC[0] = result[2] * alpha;
rowC[0] = result[1] * alpha;
#else
#define SAVE_ACC(ACC, J) \
__builtin_mma_disassemble_acc (result, ACC); \
__builtin_mma_disassemble_acc ((void *)result, ACC); \
rowC = (v4sf_t *) &CO[0* ldc+J]; \
rowC[0] += result[3] * alpha; \
rowC[0] += result[0] * alpha; \
rowC = (v4sf_t *) &CO[1*ldc+J]; \
rowC[0] += result[2] * alpha; \
rowC[0] += result[1] * alpha; \
rowC = (v4sf_t *) &CO[2*ldc+J]; \
rowC[0] += result[1] * alpha; \
rowC = (v4sf_t *) &CO[3*ldc+J]; \
rowC[0] += result[0] * alpha;
#define SAVE_ACC1(ACC, J) \
__builtin_mma_disassemble_acc (result, ACC); \
rowC = (v4sf_t *) &CO[4* ldc+J]; \
rowC[0] += result[3] * alpha; \
rowC = (v4sf_t *) &CO[5*ldc+J]; \
rowC[0] += result[2] * alpha; \
rowC = (v4sf_t *) &CO[6*ldc+J]; \
rowC = (v4sf_t *) &CO[3*ldc+J]; \
rowC[0] += result[3] * alpha;
#define SAVE_ACC1(ACC, J) \
__builtin_mma_disassemble_acc ((void *)result, ACC); \
rowC = (v4sf_t *) &CO[4* ldc+J]; \
rowC[0] += result[0] * alpha; \
rowC = (v4sf_t *) &CO[5*ldc+J]; \
rowC[0] += result[1] * alpha; \
rowC = (v4sf_t *) &CO[6*ldc+J]; \
rowC[0] += result[2] * alpha; \
rowC = (v4sf_t *) &CO[7*ldc+J]; \
rowC[0] += result[0] * alpha;
rowC[0] += result[3] * alpha;
#define SAVE2x4_ACC(ACC, J) \
__builtin_mma_disassemble_acc (result, ACC); \
__builtin_mma_disassemble_acc ((void *)result, ACC); \
rowC = (v4sf_t *) &CO[0* ldc+J]; \
rowC[0] += result[3] * alpha; \
rowC[0] += result[0] * alpha; \
rowC = (v4sf_t *) &CO[1* ldc+J]; \
rowC[0] += result[2] * alpha;
rowC[0] += result[1] * alpha;
#endif
#define SET_ACC_ZERO4() \

106
kernel/power/sgemm_kernel_power10.c
View File

@ -27,103 +27,103 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "common.h"
#include <altivec.h>
typedef unsigned char vec_t __attribute__ ((vector_size (16)));
typedef __vector unsigned char vec_t;
typedef FLOAT v4sf_t __attribute__ ((vector_size (16)));
typedef FLOAT v2sf_t __attribute__ ((vector_size (8)));
#if defined(TRMMKERNEL)
#define SAVE_ACC(ACC, J) \
__builtin_mma_disassemble_acc (result, ACC); \
__builtin_mma_disassemble_acc ((void *)result, ACC); \
rowC = (v4sf_t *) &CO[0* ldc+J]; \
rowC[0] = result[3] * alpha; \
rowC[0] = result[0] * alpha; \
rowC = (v4sf_t *) &CO[1*ldc+J]; \
rowC[0] = result[2] * alpha; \
rowC[0] = result[1] * alpha; \
rowC = (v4sf_t *) &CO[2*ldc+J]; \
rowC[0] = result[1] * alpha; \
rowC[0] = result[2] * alpha; \
rowC = (v4sf_t *) &CO[3*ldc+J]; \
rowC[0] = result[0] * alpha;
rowC[0] = result[3] * alpha;
#define SAVE_ACC1(ACC, J) \
__builtin_mma_disassemble_acc (result, ACC); \
__builtin_mma_disassemble_acc ((void *)result, ACC); \
rowC = (v4sf_t *) &CO[4* ldc+J]; \
rowC[0] = result[3] * alpha; \
rowC[0] = result[0] * alpha; \
rowC = (v4sf_t *) &CO[5*ldc+J]; \
rowC[0] = result[2] * alpha; \
rowC = (v4sf_t *) &CO[6*ldc+J]; \
rowC[0] = result[1] * alpha; \
rowC = (v4sf_t *) &CO[6*ldc+J]; \
rowC[0] = result[2] * alpha; \
rowC = (v4sf_t *) &CO[7*ldc+J]; \
rowC[0] = result[0] * alpha;
rowC[0] = result[3] * alpha;
#define SAVE4x2_ACC(ACC, J) \
__builtin_mma_disassemble_acc (result, ACC); \
__builtin_mma_disassemble_acc ((void *)result, ACC); \
rowC = (v2sf_t *) &CO[0* ldc+J]; \
rowC[0] = result[6] * alpha; \
rowC[0] = result[0] * alpha; \
rowC = (v2sf_t *) &CO[1* ldc+J]; \
rowC[0] = result[4] * alpha; \
rowC[0] = result[2] * alpha; \
rowC = (v2sf_t *) &CO[2* ldc+J]; \
rowC[0] = result[2] * alpha; \
rowC = (v2sf_t *) &CO[3* ldc+J]; \
rowC[0] = result[0] * alpha;
#define SAVE4x2_ACC1(ACC, J) \
__builtin_mma_disassemble_acc (result, ACC); \
rowC = (v2sf_t *) &CO[4* ldc+J]; \
rowC[0] = result[6] * alpha; \
rowC = (v2sf_t *) &CO[5* ldc+J]; \
rowC[0] = result[4] * alpha; \
rowC = (v2sf_t *) &CO[6* ldc+J]; \
rowC = (v2sf_t *) &CO[3* ldc+J]; \
rowC[0] = result[6] * alpha;
#define SAVE4x2_ACC1(ACC, J) \
__builtin_mma_disassemble_acc ((void *)result, ACC); \
rowC = (v2sf_t *) &CO[4* ldc+J]; \
rowC[0] = result[0] * alpha; \
rowC = (v2sf_t *) &CO[5* ldc+J]; \
rowC[0] = result[2] * alpha; \
rowC = (v2sf_t *) &CO[6* ldc+J]; \
rowC[0] = result[4] * alpha; \
rowC = (v2sf_t *) &CO[7* ldc+J]; \
rowC[0] = result[0] * alpha;
rowC[0] = result[6] * alpha;
#define SAVE2x4_ACC(ACC, J) \
__builtin_mma_disassemble_acc (result, ACC); \
__builtin_mma_disassemble_acc ((void *)result, ACC); \
rowC = (v4sf_t *) &CO[0* ldc+J]; \
rowC[0] = result[3] * alpha; \
rowC[0] = result[0] * alpha; \
rowC = (v4sf_t *) &CO[1* ldc+J]; \
rowC[0] = result[2] * alpha;
rowC[0] = result[1] * alpha;
#else
#define SAVE_ACC(ACC, J) \
__builtin_mma_disassemble_acc (result, ACC); \
__builtin_mma_disassemble_acc ((void *)result, ACC); \
rowC = (v4sf_t *) &CO[0* ldc+J]; \
rowC[0] += result[3] * alpha; \
rowC[0] += result[0] * alpha; \
rowC = (v4sf_t *) &CO[1*ldc+J]; \
rowC[0] += result[2] * alpha; \
rowC[0] += result[1] * alpha; \
rowC = (v4sf_t *) &CO[2*ldc+J]; \
rowC[0] += result[1] * alpha; \
rowC[0] += result[2] * alpha; \
rowC = (v4sf_t *) &CO[3*ldc+J]; \
rowC[0] += result[0] * alpha;
rowC[0] += result[3] * alpha;
#define SAVE_ACC1(ACC, J) \
__builtin_mma_disassemble_acc (result, ACC); \
__builtin_mma_disassemble_acc ((void *)result, ACC); \
rowC = (v4sf_t *) &CO[4* ldc+J]; \
rowC[0] += result[3] * alpha; \
rowC[0] += result[0] * alpha; \
rowC = (v4sf_t *) &CO[5*ldc+J]; \
rowC[0] += result[2] * alpha; \
rowC = (v4sf_t *) &CO[6*ldc+J]; \
rowC[0] += result[1] * alpha; \
rowC = (v4sf_t *) &CO[6*ldc+J]; \
rowC[0] += result[2] * alpha; \
rowC = (v4sf_t *) &CO[7*ldc+J]; \
rowC[0] += result[0] * alpha;
rowC[0] += result[3] * alpha;
#define SAVE4x2_ACC(ACC, J) \
__builtin_mma_disassemble_acc (result, ACC); \
__builtin_mma_disassemble_acc ((void *)result, ACC); \
rowC = (v2sf_t *) &CO[0* ldc+J]; \
rowC[0] += result[6] * alpha; \
rowC[0] += result[0] * alpha; \
rowC = (v2sf_t *) &CO[1* ldc+J]; \
rowC[0] += result[4] * alpha; \
rowC[0] += result[2] * alpha; \
rowC = (v2sf_t *) &CO[2* ldc+J]; \
rowC[0] += result[2] * alpha; \
rowC = (v2sf_t *) &CO[3* ldc+J]; \
rowC[0] += result[0] * alpha;
#define SAVE4x2_ACC1(ACC, J) \
__builtin_mma_disassemble_acc (result, ACC); \
rowC = (v2sf_t *) &CO[4* ldc+J]; \
rowC[0] += result[6] * alpha; \
rowC = (v2sf_t *) &CO[5* ldc+J]; \
rowC[0] += result[4] * alpha; \
rowC = (v2sf_t *) &CO[6* ldc+J]; \
rowC = (v2sf_t *) &CO[3* ldc+J]; \
rowC[0] += result[6] * alpha;
#define SAVE4x2_ACC1(ACC, J) \
__builtin_mma_disassemble_acc ((void *)result, ACC); \
rowC = (v2sf_t *) &CO[4* ldc+J]; \
rowC[0] += result[0] * alpha; \
rowC = (v2sf_t *) &CO[5* ldc+J]; \
rowC[0] += result[2] * alpha; \
rowC = (v2sf_t *) &CO[6* ldc+J]; \
rowC[0] += result[4] * alpha; \
rowC = (v2sf_t *) &CO[7* ldc+J]; \
rowC[0] += result[0] * alpha;
rowC[0] += result[6] * alpha;
#define SAVE2x4_ACC(ACC, J) \
__builtin_mma_disassemble_acc (result, ACC); \
__builtin_mma_disassemble_acc ((void *)result, ACC); \
rowC = (v4sf_t *) &CO[0* ldc+J]; \
rowC[0] += result[3] * alpha; \
rowC[0] += result[0] * alpha; \
rowC = (v4sf_t *) &CO[1* ldc+J]; \
rowC[0] += result[2] * alpha;
rowC[0] += result[1] * alpha;
#endif
#define KERNEL(i, j) \
__builtin_mma_xvf32gerpp (&acc0, rowB[i], rowA[j]); \

54
kernel/power/shgemm_kernel_power10.c
View File

@ -45,7 +45,7 @@ bfloat16tof32 (bfloat16 f16)
#define BF16TOF32(x) x
#endif
typedef unsigned char vec_t __attribute__ ((vector_size (16)));
typedef __vector unsigned char vec_t;
typedef FLOAT v4sf_t __attribute__ ((vector_size (16)));
typedef FLOAT v2sf_t __attribute__ ((vector_size (8)));
@ -64,54 +64,54 @@ vector char mask =
#define MERGE_LOW(x, y) (vec_t) vec_mergel ((vector short)x, (vector short)y)
#define SAVE_ACC(ACC, J) \
__builtin_mma_disassemble_acc (result, ACC); \
__builtin_mma_disassemble_acc ((void *)result, ACC); \
rowC = (v4sf_t *) &CO[0* ldc+J]; \
rowC[0] += result[3] * alpha; \
rowC[0] += result[0] * alpha; \
rowC = (v4sf_t *) &CO[1*ldc+J]; \
rowC[0] += result[2] * alpha; \
rowC[0] += result[1] * alpha; \
rowC = (v4sf_t *) &CO[2*ldc+J]; \
rowC[0] += result[1] * alpha; \
rowC[0] += result[2] * alpha; \
rowC = (v4sf_t *) &CO[3*ldc+J]; \
rowC[0] += result[0] * alpha;
rowC[0] += result[3] * alpha;
#define SAVE_ACC1(ACC, J) \
__builtin_mma_disassemble_acc (result, ACC); \
__builtin_mma_disassemble_acc ((void *)result, ACC); \
rowC = (v4sf_t *) &CO[4* ldc+J]; \
rowC[0] += result[3] * alpha; \
rowC[0] += result[0] * alpha; \
rowC = (v4sf_t *) &CO[5*ldc+J]; \
rowC[0] += result[2] * alpha; \
rowC = (v4sf_t *) &CO[6*ldc+J]; \
rowC[0] += result[1] * alpha; \
rowC = (v4sf_t *) &CO[6*ldc+J]; \
rowC[0] += result[2] * alpha; \
rowC = (v4sf_t *) &CO[7*ldc+J]; \
rowC[0] += result[0] * alpha;
rowC[0] += result[3] * alpha;
#define SAVE4x2_ACC(ACC, J) \
__builtin_mma_disassemble_acc (result, ACC); \
__builtin_mma_disassemble_acc ((void *)result, ACC); \
rowC = (v2sf_t *) &CO[0* ldc+J]; \
rowC[0] += result[6] * alpha; \
rowC[0] += result[0] * alpha; \
rowC = (v2sf_t *) &CO[1* ldc+J]; \
rowC[0] += result[4] * alpha; \
rowC[0] += result[2] * alpha; \
rowC = (v2sf_t *) &CO[2* ldc+J]; \
rowC[0] += result[2] * alpha; \
rowC = (v2sf_t *) &CO[3* ldc+J]; \
rowC[0] += result[0] * alpha;
#define SAVE4x2_ACC1(ACC, J) \
__builtin_mma_disassemble_acc (result, ACC); \
rowC = (v2sf_t *) &CO[4* ldc+J]; \
rowC[0] += result[6] * alpha; \
rowC = (v2sf_t *) &CO[5* ldc+J]; \
rowC[0] += result[4] * alpha; \
rowC = (v2sf_t *) &CO[6* ldc+J]; \
rowC = (v2sf_t *) &CO[3* ldc+J]; \
rowC[0] += result[6] * alpha;
#define SAVE4x2_ACC1(ACC, J) \
__builtin_mma_disassemble_acc ((void *)result, ACC); \
rowC = (v2sf_t *) &CO[4* ldc+J]; \
rowC[0] += result[0] * alpha; \
rowC = (v2sf_t *) &CO[5* ldc+J]; \
rowC[0] += result[2] * alpha; \
rowC = (v2sf_t *) &CO[6* ldc+J]; \
rowC[0] += result[4] * alpha; \
rowC = (v2sf_t *) &CO[7* ldc+J]; \
rowC[0] += result[0] * alpha;
rowC[0] += result[6] * alpha;
#define MMA __builtin_mma_xvbf16ger2pp
#define SAVE2x4_ACC(ACC, J) \
__builtin_mma_disassemble_acc (result, ACC); \
__builtin_mma_disassemble_acc ((void *)result, ACC); \
rowC = (v4sf_t *) &CO[0* ldc+J]; \
rowC[0] += result[3] * alpha; \
rowC[0] += result[0] * alpha; \
rowC = (v4sf_t *) &CO[1* ldc+J]; \
rowC[0] += result[2] * alpha;
rowC[0] += result[1] * alpha;
#define SET_ACC_ZERO4() \
__builtin_mma_xxsetaccz (&acc0); \