From 960dec234fad4834f1fcd0c2453f878c35576d41 Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Mon, 2 Mar 2020 00:09:49 +0100
Subject: [PATCH 01/48] Version 0.3.9

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 951271717..eb1543591 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -6,7 +6,7 @@ cmake_minimum_required(VERSION 2.8.5)
 project(OpenBLAS C ASM)
 set(OpenBLAS_MAJOR_VERSION 0)
 set(OpenBLAS_MINOR_VERSION 3)
-set(OpenBLAS_PATCH_VERSION 9.dev)
+set(OpenBLAS_PATCH_VERSION 9)
 set(OpenBLAS_VERSION "${OpenBLAS_MAJOR_VERSION}.${OpenBLAS_MINOR_VERSION}.${OpenBLAS_PATCH_VERSION}")
 
 # Adhere to GNU filesystem layout conventions

From 33f76a6c378681ac2af76b20e55ff682191a1937 Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Mon, 2 Mar 2020 00:10:20 +0100
Subject: [PATCH 02/48] Version 0.3.9

---
 Makefile.rule | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile.rule b/Makefile.rule
index 724a60ec4..a4465e448 100644
--- a/Makefile.rule
+++ b/Makefile.rule
@@ -3,7 +3,7 @@
 #
 
 # This library's version
-VERSION = 0.3.9.dev
+VERSION = 0.3.9
 
 # If you set the suffix, the library name will be libopenblas_$(LIBNAMESUFFIX).a
 # and libopenblas_$(LIBNAMESUFFIX).so. Meanwhile, the soname in shared library

From f07a80354b60121a91f40574ebd6d7c306dcd100 Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Tue, 9 Jun 2020 16:07:03 +0200
Subject: [PATCH 03/48] Apply previously AOCC-specific workaround to all
 versions of flang

---
 Makefile.system | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/Makefile.system b/Makefile.system
index 252c816a9..56e94f2a6 100644
--- a/Makefile.system
+++ b/Makefile.system
@@ -797,11 +797,11 @@ endif
 ifeq ($(USE_OPENMP), 1)
 FCOMMON_OPT += -fopenmp
 endif
-ifeq ($(OSNAME), Linux)
-ifeq ($(ARCH), x86_64)
-FLANG_VENDOR := $(shell expr `$(FC) --version|cut -f 1 -d "."|head -1`)
-endif
-endif
+#ifeq ($(OSNAME), Linux)
+#ifeq ($(ARCH), x86_64)
+#FLANG_VENDOR := $(shell expr `$(FC) --version|cut -f 1 -d "."|head -1`)
+#endif
+#endif
 endif
 
 ifeq ($(F_COMPILER), G77)
@@ -1276,7 +1276,8 @@ endif
 
 override CFLAGS     += $(COMMON_OPT) $(CCOMMON_OPT) -I$(TOPDIR)
 override PFLAGS     += $(COMMON_OPT) $(CCOMMON_OPT) -I$(TOPDIR) -DPROFILE $(COMMON_PROF)
-ifeq ($(FLANG_VENDOR),AOCC)
+#ifeq ($(FLANG_VENDOR),AOCC)
+ifeq ($(F_COMPILER),FLANG)
 override FFLAGS     += $(filter-out -O2 -O3,$(COMMON_OPT)) -O1 $(FCOMMON_OPT)
 else
 override FFLAGS     += $(COMMON_OPT) $(FCOMMON_OPT)

From ba2c5b404d1c0ac5ed6037c44a6adefb385bc73a Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Tue, 9 Jun 2020 16:09:34 +0200
Subject: [PATCH 04/48] When building with flang, use it also for the final
 link step to get dependencies right

---
 exports/Makefile | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/exports/Makefile b/exports/Makefile
index c92d6e996..01a313b35 100644
--- a/exports/Makefile
+++ b/exports/Makefile
@@ -155,8 +155,12 @@ ifeq ($(F_COMPILER), INTEL)
 	-Wl,--whole-archive $< -Wl,--no-whole-archive \
 	-Wl,-soname,$(INTERNALNAME) $(EXTRALIB)
 	$(CC) $(CFLAGS) $(LDFLAGS) -w -o linktest linktest.c ../$(LIBSONAME) $(FEXTRALIB) && echo OK.
+else ifeq ($(F_COMPILER), FLANG)
+	$(FC) $(FFLAGS) $(LDFLAGS) -shared -o ../$(LIBSONAME) \
+	-Wl,--whole-archive $< -Wl,--no-whole-archive \
+	-Wl,-soname,$(INTERNALNAME) $(EXTRALIB)
+	$(CC) $(CFLAGS) $(LDFLAGS) -w -o linktest linktest.c ../$(LIBSONAME) $(FEXTRALIB) && echo OK.
 else
-
 ifneq ($(C_COMPILER), LSB)
 	$(CC) $(CFLAGS) $(LDFLAGS) -shared -o ../$(LIBSONAME) \
 	-Wl,--whole-archive $< -Wl,--no-whole-archive \

From 3ce469a34f435f64d9b3f6ca8ccba69dcfce31b3 Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Tue, 9 Jun 2020 16:11:13 +0200
Subject: [PATCH 05/48] Limit optimization level to O1 for flang and add
 -frecursive

---
 cmake/fc.cmake     | 1 +
 cmake/system.cmake | 9 +++++++++
 2 files changed, 10 insertions(+)

diff --git a/cmake/fc.cmake b/cmake/fc.cmake
index cc330ae2c..9dcedffb2 100644
--- a/cmake/fc.cmake
+++ b/cmake/fc.cmake
@@ -16,6 +16,7 @@ if (${F_COMPILER} STREQUAL "FLANG")
   if (USE_OPENMP)
     set(FCOMMON_OPT "${FCOMMON_OPT} -fopenmp")
   endif ()
+  set(FCOMMON_OPT "${FCOMMON_OPT} -frecursive")
 endif ()
 
 if (${F_COMPILER} STREQUAL "G77")
diff --git a/cmake/system.cmake b/cmake/system.cmake
index 7e7f726c5..c2ae471d2 100644
--- a/cmake/system.cmake
+++ b/cmake/system.cmake
@@ -417,6 +417,15 @@ if (${CMAKE_C_COMPILER} STREQUAL "LSB" OR ${CMAKE_SYSTEM_NAME} STREQUAL "Windows
   set(LAPACK_CFLAGS "${LAPACK_CFLAGS} -DLAPACK_COMPLEX_STRUCTURE")
 endif ()
 
+if ("${CMAKE_BUILD_TYPE}" STREQUAL "Release")
+if ("${F_COMPILER}" STREQUAL "FLANG")
+  set(FILTER_FLAGS "-O2;-O3")
+  foreach (FILTER_FLAG ${FILTER_FLAGS})
+    string(REPLACE ${FILTER_FLAG} "-O1" CMAKE_Fortran_FLAGS_RELEASE ${CMAKE_Fortran_FLAGS_RELEASE})
+  endforeach ()
+endif ()
+endif ()
+
 if (NOT DEFINED SUFFIX)
   set(SUFFIX o)
 endif ()

From bc6fd20a40f0750311311b31872cf3e97f845617 Mon Sep 17 00:00:00 2001
From: ZhangDanfeng <467688405@qq.com>
Date: Wed, 10 Jun 2020 01:01:16 +0800
Subject: [PATCH 06/48] fix INIT8x4

Signed-off-by: ZhangDanfeng <467688405@qq.com>
---
 kernel/arm64/sgemm_kernel_8x8_cortexa53.S | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/kernel/arm64/sgemm_kernel_8x8_cortexa53.S b/kernel/arm64/sgemm_kernel_8x8_cortexa53.S
index fec0c9ae9..628a928ca 100644
--- a/kernel/arm64/sgemm_kernel_8x8_cortexa53.S
+++ b/kernel/arm64/sgemm_kernel_8x8_cortexa53.S
@@ -681,12 +681,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .macro INIT8x4
 	fmov		s16, wzr
 	fmov		s17, wzr
+	fmov		s18, wzr
+	fmov		s19, s16
 	fmov		s20, wzr
 	fmov		s21, s16
-	fmov		s24, wzr
-	fmov		s25, s16
-	fmov		s28, wzr
-	fmov		s29, s16
+	fmov		s22, wzr
+	fmov		s23, s16
 .endm
 
 .macro KERNEL8x4_I
@@ -765,14 +765,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	fmla	v21.4s, v3.4s, v6.s[2]
 	fmla	v22.4s, v2.4s, v6.s[3]
 	fmla	v23.4s, v3.4s, v6.s[3]
-	fmla	v24.4s, v2.4s, v7.s[0]
-	fmla	v25.4s, v3.4s, v7.s[0]
-	fmla	v26.4s, v2.4s, v7.s[1]
-	fmla	v27.4s, v3.4s, v7.s[1]
-	fmla	v28.4s, v2.4s, v7.s[2]
-	fmla	v29.4s, v3.4s, v7.s[2]
-	fmla	v30.4s, v2.4s, v7.s[3]
-	fmla	v31.4s, v3.4s, v7.s[3]
 .endm
 
 .macro KERNEL8x4_SUB

From 9fe930f205c3ad56fe92d9c4c65e48836db33a27 Mon Sep 17 00:00:00 2001
From: Rajalakshmi Srinivasaraghavan <rajis@linux.ibm.com>
Date: Thu, 11 Jun 2020 15:47:20 -0500
Subject: [PATCH 07/48] powerpc: Add support for future processor

This is the initial patch to support build infrastructure
for POWER10 architecture.
---
 Makefile.power                |  10 ++
 Makefile.system               |   7 ++
 TargetList.txt                |   1 +
 cmake/arch.cmake              |   2 +-
 cmake/prebuild.cmake          |   2 +-
 common.h                      |   7 +-
 common_power.h                |   8 +-
 cpuid_power.c                 |  15 ++-
 driver/others/dynamic_power.c |  16 ++-
 getarch.c                     |  13 +++
 kernel/CMakeLists.txt         |   2 +-
 kernel/Makefile.L3            |   4 +
 kernel/power/KERNEL.POWER10   | 214 ++++++++++++++++++++++++++++++++++
 kernel/power/casum.c          |   2 +-
 kernel/power/ccopy.c          |   2 +-
 kernel/power/crot.c           |   2 +-
 kernel/power/cswap.c          |   2 +-
 kernel/power/dasum.c          |   2 +-
 kernel/power/daxpy.c          |   2 +-
 kernel/power/dcopy.c          |   2 +-
 kernel/power/ddot.c           |   2 +-
 kernel/power/dgemv_n.c        |   2 +-
 kernel/power/drot.c           |   2 +-
 kernel/power/dscal.c          |   2 +-
 kernel/power/dswap.c          |   2 +-
 kernel/power/sasum.c          |   2 +-
 kernel/power/scopy.c          |   2 +-
 kernel/power/sdot.c           |   2 +-
 kernel/power/srot.c           |   2 +-
 kernel/power/sscal.c          |   2 +-
 kernel/power/sswap.c          |   2 +-
 kernel/power/zasum.c          |   2 +-
 kernel/power/zaxpy.c          |   2 +-
 kernel/power/zcopy.c          |   2 +-
 kernel/power/zdot.c           |   2 +-
 kernel/power/zscal.c          |   2 +-
 kernel/power/zswap.c          |   2 +-
 param.h                       |   2 +-
 38 files changed, 309 insertions(+), 42 deletions(-)
 create mode 100644 kernel/power/KERNEL.POWER10

diff --git a/Makefile.power b/Makefile.power
index 24d8aa8a7..5c431860f 100644
--- a/Makefile.power
+++ b/Makefile.power
@@ -9,6 +9,16 @@ else
 USE_OPENMP = 1
 endif
 
+ifeq ($(CORE), POWER10)
+ifeq ($(USE_OPENMP), 1)
+COMMON_OPT += -Ofast -mcpu=future -mtune=future -mvsx -malign-power -DUSE_OPENMP -fno-fast-math -fopenmp
+FCOMMON_OPT += -O2 -frecursive -mcpu=future -mtune=future -malign-power -DUSE_OPENMP -fno-fast-math -fopenmp
+else
+COMMON_OPT += -Ofast -mcpu=future -mtune=future -mvsx -malign-power -fno-fast-math
+FCOMMON_OPT += -O2 -frecursive -mcpu=future -mtune=future -malign-power -fno-fast-math
+endif
+endif
+
 ifeq ($(CORE), POWER9)
 ifeq ($(USE_OPENMP), 1)
 COMMON_OPT += -Ofast -mcpu=power9 -mtune=power9 -mvsx -malign-power -DUSE_OPENMP -fno-fast-math -fopenmp
diff --git a/Makefile.system b/Makefile.system
index 56e94f2a6..3decc1457 100644
--- a/Makefile.system
+++ b/Makefile.system
@@ -595,6 +595,7 @@ DYNAMIC_CORE = POWER6
 DYNAMIC_CORE += POWER8
 ifneq ($(C_COMPILER), GCC)
 DYNAMIC_CORE += POWER9
+DYNAMIC_CORE += POWER10
 endif
 ifeq ($(C_COMPILER), GCC)
 GCCVERSIONGT5 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \> 5)
@@ -603,6 +604,12 @@ DYNAMIC_CORE += POWER9
 else
 $(info, OpenBLAS: Your gcc version is too old to build the POWER9 kernels.)
 endif
+GCCVERSIONGTEQ11 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \>= 11)
+ifeq ($(GCCVERSIONGTEQ11), 1)
+DYNAMIC_CORE += POWER10
+else
+$(info, OpenBLAS: Your gcc version is too old to build the POWER10 kernels.)
+endif
 endif
 endif
 
diff --git a/TargetList.txt b/TargetList.txt
index e2d2f4026..4e54e3077 100644
--- a/TargetList.txt
+++ b/TargetList.txt
@@ -49,6 +49,7 @@ POWER6
 POWER7
 POWER8
 POWER9
+POWER10
 PPCG4
 PPC970
 PPC970MP
diff --git a/cmake/arch.cmake b/cmake/arch.cmake
index 9d51f777c..d56ba99cb 100644
--- a/cmake/arch.cmake
+++ b/cmake/arch.cmake
@@ -49,7 +49,7 @@ if (DYNAMIC_ARCH)
   endif ()
   
   if (POWER)
-    set(DYNAMIC_CORE POWER6 POWER8 POWER9)
+	  set(DYNAMIC_CORE POWER6 POWER8 POWER9 POWER10)
   endif ()
   
   if (X86)
diff --git a/cmake/prebuild.cmake b/cmake/prebuild.cmake
index 067b97b4b..30256870c 100644
--- a/cmake/prebuild.cmake
+++ b/cmake/prebuild.cmake
@@ -420,7 +420,7 @@ if (DEFINED CORE AND CMAKE_CROSSCOMPILING AND NOT (${HOST_OS} STREQUAL "WINDOWSS
     set(ZGEMM_UNROLL_M 8)
     set(ZGEMM_UNROLL_N 2)
     set(SYMV_P 8)
-  elseif ("${TCORE}" STREQUAL "POWER9")
+  elseif ("${TCORE}" STREQUAL "POWER9" OR "${TCORE}" STREQUAL "POWER10")
     file(APPEND ${TARGET_CONF_TEMP}
       "#define L1_DATA_SIZE 32768\n"
       "#define L1_DATA_LINESIZE 128\n"
diff --git a/common.h b/common.h
index e2c8cdee5..00b34a3f7 100644
--- a/common.h
+++ b/common.h
@@ -360,13 +360,8 @@ typedef int blasint;
 #endif
 #endif
 
-#ifdef POWER8
-#ifndef YIELDING
-#define YIELDING        __asm__ __volatile__ ("nop;nop;nop;nop;nop;nop;nop;nop;\n");
-#endif
-#endif
 
-#ifdef POWER9
+#if defined(POWER8) || defined(POWER9) || defined(POWER10)
 #ifndef YIELDING
 #define YIELDING        __asm__ __volatile__ ("nop;nop;nop;nop;nop;nop;nop;nop;\n");
 #endif
diff --git a/common_power.h b/common_power.h
index e29d0f382..aa19794b5 100644
--- a/common_power.h
+++ b/common_power.h
@@ -68,7 +68,7 @@
 #endif
 
 
-#if defined(POWER8) || defined(POWER9)
+#if defined(POWER8) || defined(POWER9) || defined(POWER10)
 #define MB		__asm__ __volatile__ ("eieio":::"memory")
 #define WMB		__asm__ __volatile__ ("eieio":::"memory")
 #define RMB		__asm__ __volatile__ ("eieio":::"memory")
@@ -272,7 +272,7 @@ static inline int blas_quickdivide(blasint x, blasint y){
 #define HAVE_PREFETCH
 #endif
 
-#if defined(POWER3) || defined(POWER6) || defined(PPCG4) || defined(CELL) || defined(POWER8) || defined(POWER9) || defined(PPC970)
+#if defined(POWER3) || defined(POWER6) || defined(PPCG4) || defined(CELL) || defined(POWER8) || defined(POWER9) || defined(POWER10) || defined(PPC970)
 #define DCBT_ARG	0
 #else
 #define DCBT_ARG	8
@@ -294,7 +294,7 @@ static inline int blas_quickdivide(blasint x, blasint y){
 #define L1_PREFETCH	dcbtst
 #endif
 
-#if defined(POWER8) || defined(POWER9)
+#if defined(POWER8) || defined(POWER9) || defined(POWER10)
 #define L1_DUALFETCH
 #define L1_PREFETCHSIZE (16 + 128 * 100)
 #define L1_PREFETCH	dcbtst
@@ -843,7 +843,7 @@ Lmcount$lazy_ptr:
 #define BUFFER_SIZE     (  2 << 20)
 #elif defined(PPC440FP2)
 #define BUFFER_SIZE     ( 16 << 20)
-#elif defined(POWER8) || defined(POWER9)
+#elif defined(POWER8) || defined(POWER9) || defined(POWER10)
 #define BUFFER_SIZE     ( 64 << 20)
 #else
 #define BUFFER_SIZE     ( 16 << 20)
diff --git a/cpuid_power.c b/cpuid_power.c
index d5ba6fb2c..b36aa4945 100644
--- a/cpuid_power.c
+++ b/cpuid_power.c
@@ -57,6 +57,7 @@
 #define CPUTYPE_PPCG4	   7
 #define CPUTYPE_POWER8     8
 #define CPUTYPE_POWER9     9
+#define CPUTYPE_POWER10    10
 
 char *cpuname[] = {
   "UNKNOWN",
@@ -68,7 +69,8 @@ char *cpuname[] = {
   "CELL",
   "PPCG4",
   "POWER8",
-  "POWER9"
+  "POWER9",
+  "POWER10"
 };
 
 char *lowercpuname[] = {
@@ -81,7 +83,8 @@ char *lowercpuname[] = {
   "cell",
   "ppcg4",
   "power8",
-  "power9"	
+  "power9",
+  "power10"
 };
 
 char *corename[] = {
@@ -94,7 +97,8 @@ char *corename[] = {
   "CELL",
   "PPCG4",
   "POWER8",
-  "POWER9"   	
+  "POWER9",
+  "POWER10"
 };
 
 int detect(void){
@@ -125,6 +129,7 @@ int detect(void){
   if (!strncasecmp(p, "POWER7", 6)) return CPUTYPE_POWER6;
   if (!strncasecmp(p, "POWER8", 6)) return CPUTYPE_POWER8;
   if (!strncasecmp(p, "POWER9", 6)) return CPUTYPE_POWER9;
+  if (!strncasecmp(p, "POWER10", 7)) return CPUTYPE_POWER10;
   if (!strncasecmp(p, "Cell",   4)) return CPUTYPE_CELL;
   if (!strncasecmp(p, "7447",   4)) return CPUTYPE_PPCG4;
 
@@ -157,6 +162,7 @@ int detect(void){
   if (!strncasecmp(p, "POWER7", 6)) return CPUTYPE_POWER6;
   if (!strncasecmp(p, "POWER8", 6)) return CPUTYPE_POWER8;
   if (!strncasecmp(p, "POWER9", 6)) return CPUTYPE_POWER9;
+  if (!strncasecmp(p, "POWER10", 7)) return CPUTYPE_POWER10;
   if (!strncasecmp(p, "Cell",   4)) return CPUTYPE_CELL;
   if (!strncasecmp(p, "7447",   4)) return CPUTYPE_PPCG4;
   return CPUTYPE_POWER5;
@@ -179,6 +185,9 @@ int detect(void){
 int id;
 __asm __volatile("mfpvr %0" : "=r"(id));
 switch ( id >> 16 ) {
+  case 0x80: // POWER10
+    return CPUTYPE_POWER10;
+    break;
   case 0x4e: // POWER9
     return CPUTYPE_POWER9;
     break;
diff --git a/driver/others/dynamic_power.c b/driver/others/dynamic_power.c
index 8c831b998..811a5fae3 100644
--- a/driver/others/dynamic_power.c
+++ b/driver/others/dynamic_power.c
@@ -6,6 +6,9 @@ extern gotoblas_t gotoblas_POWER8;
 #if (!defined __GNUC__) || ( __GNUC__ >= 6)
 extern gotoblas_t gotoblas_POWER9;
 #endif
+#if (!defined __GNUC__) || ( __GNUC__ >= 11)
+extern gotoblas_t gotoblas_POWER10;
+#endif
 
 extern void openblas_warning(int verbose, const char *msg);
 
@@ -13,7 +16,8 @@ static char *corename[] = {
 	"unknown",
 	"POWER6",
 	"POWER8",
-	"POWER9"
+	"POWER9",
+	"POWER10"
 };
 
 #define NUM_CORETYPES 4
@@ -23,6 +27,9 @@ char *gotoblas_corename(void) {
 	if (gotoblas == &gotoblas_POWER8)	return corename[2];
 #if (!defined __GNUC__) || ( __GNUC__ >= 6)
 	if (gotoblas == &gotoblas_POWER9)	return corename[3];
+#endif
+#if (!defined __GNUC__) || ( __GNUC__ >= 11)
+	if (gotoblas == &gotoblas_POWER10)	return corename[4];
 #endif
 	return corename[0];
 }
@@ -36,6 +43,10 @@ static gotoblas_t *get_coretype(void) {
 #if (!defined __GNUC__) || ( __GNUC__ >= 6)
 	if (__builtin_cpu_is("power9"))
 		return &gotoblas_POWER9;
+#endif
+#if (!defined __GNUC__) || ( __GNUC__ >= 11)
+	if (__builtin_cpu_is("isa_3_1") && __builtin_cpu_supports ("mma"))
+		return &gotoblas_POWER10;
 #endif
 	return NULL;
 }
@@ -61,6 +72,9 @@ static gotoblas_t *force_coretype(char * coretype) {
 	case  2: return (&gotoblas_POWER8);
 #if (!defined __GNUC__) || ( __GNUC__ >= 6)
 	case  3: return (&gotoblas_POWER9);
+#endif
+#if (!defined __GNUC__) || ( __GNUC__ >= 11)
+	case  4: return (&gotoblas_POWER10);
 #endif
 	default: return NULL;
 	}
diff --git a/getarch.c b/getarch.c
index c173d58b8..164947f3e 100644
--- a/getarch.c
+++ b/getarch.c
@@ -650,6 +650,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define CORENAME  "POWER9"
 #endif
 
+#if defined(FORCE_POWER10)
+#define FORCE
+#define ARCHITECTURE    "POWER"
+#define SUBARCHITECTURE "POWER10"
+#define SUBDIRNAME      "power"
+#define ARCHCONFIG   "-DPOWER10 " \
+		     "-DL1_DATA_SIZE=32768 -DL1_DATA_LINESIZE=128 " \
+		     "-DL2_SIZE=4194304 -DL2_LINESIZE=128 " \
+		     "-DDTB_DEFAULT_ENTRIES=128 -DDTB_SIZE=4096 -DL2_ASSOCIATIVE=8 "
+#define LIBNAME   "power10"
+#define CORENAME  "POWER10"
+#endif
+
 #ifdef FORCE_PPCG4
 #define FORCE
 #define ARCHITECTURE    "POWER"
diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt
index b114c6a33..d1349c5f8 100644
--- a/kernel/CMakeLists.txt
+++ b/kernel/CMakeLists.txt
@@ -130,7 +130,7 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
     if (ARM OR ARM64 OR (TARGET_CORE MATCHES LONGSOON3B) OR (TARGET_CORE MATCHES GENERIC) OR (TARGET_CORE MATCHES HASWELL) OR (TARGET_CORE MATCHES ZEN) OR (TARGET_CORE MATCHES SKYLAKEX) )
       set(USE_TRMM true)
     endif ()
-    if (ZARCH OR (TARGET_CORE MATCHES POWER8) OR (TARGET_CORE MATCHES POWER9)) 
+    if (ZARCH OR (TARGET_CORE MATCHES POWER8) OR (TARGET_CORE MATCHES POWER9) OR (TARGET_CORE MATCHES POWER10))
       set(USE_TRMM true)
     endif ()
 
diff --git a/kernel/Makefile.L3 b/kernel/Makefile.L3
index da6c5fd57..0cb02ef85 100644
--- a/kernel/Makefile.L3
+++ b/kernel/Makefile.L3
@@ -51,6 +51,10 @@ ifeq ($(CORE), POWER9)
 USE_TRMM = 1
 endif
 
+ifeq ($(CORE), POWER10)
+USE_TRMM = 1
+endif
+
 ifeq ($(ARCH), zarch)
 USE_TRMM = 1
 endif
diff --git a/kernel/power/KERNEL.POWER10 b/kernel/power/KERNEL.POWER10
new file mode 100644
index 000000000..ab8fbfcd9
--- /dev/null
+++ b/kernel/power/KERNEL.POWER10
@@ -0,0 +1,214 @@
+ifeq ($(__BYTE_ORDER__),__ORDER_BIG_ENDIAN__)
+include $(KERNELDIR)/KERNEL.POWER8
+else
+
+#SGEMM_BETA = ../generic/gemm_beta.c
+#DGEMM_BETA = ../generic/gemm_beta.c
+#CGEMM_BETA = ../generic/zgemm_beta.c
+#ZGEMM_BETA = ../generic/zgemm_beta.c
+
+STRMMKERNEL	= sgemm_kernel_power9.S
+DTRMMKERNEL	= dgemm_kernel_power9.S
+CTRMMKERNEL	= cgemm_kernel_power9.S
+ZTRMMKERNEL	= zgemm_kernel_power9.S
+
+SGEMMKERNEL    =  sgemm_kernel_power9.S
+SGEMMINCOPY    = ../generic/gemm_ncopy_16.c
+SGEMMITCOPY    = sgemm_tcopy_16_power8.S
+SGEMMONCOPY    =  ../generic/gemm_ncopy_8.c
+SGEMMOTCOPY    = sgemm_tcopy_8_power8.S
+SGEMMINCOPYOBJ =  sgemm_incopy$(TSUFFIX).$(SUFFIX)
+SGEMMITCOPYOBJ =  sgemm_itcopy$(TSUFFIX).$(SUFFIX)
+SGEMMONCOPYOBJ =  sgemm_oncopy$(TSUFFIX).$(SUFFIX)
+SGEMMOTCOPYOBJ =  sgemm_otcopy$(TSUFFIX).$(SUFFIX)
+
+DGEMMKERNEL    =  dgemm_kernel_power9.S
+DGEMMINCOPY    = ../generic/gemm_ncopy_16.c
+DGEMMITCOPY    =  dgemm_tcopy_16_power8.S
+DGEMMONCOPY    =  dgemm_ncopy_4_power8.S
+DGEMMOTCOPY    =  ../generic/gemm_tcopy_4.c
+DGEMMINCOPYOBJ =  dgemm_incopy$(TSUFFIX).$(SUFFIX)
+DGEMMITCOPYOBJ =  dgemm_itcopy$(TSUFFIX).$(SUFFIX)
+DGEMMONCOPYOBJ =  dgemm_oncopy$(TSUFFIX).$(SUFFIX)
+DGEMMOTCOPYOBJ =  dgemm_otcopy$(TSUFFIX).$(SUFFIX)
+
+CGEMMKERNEL    = cgemm_kernel_power9.S
+CGEMMINCOPY    = ../generic/zgemm_ncopy_8.c
+CGEMMITCOPY    = ../generic/zgemm_tcopy_8.c
+CGEMMONCOPY    = ../generic/zgemm_ncopy_4.c
+CGEMMOTCOPY    = ../generic/zgemm_tcopy_4.c
+CGEMMONCOPYOBJ =  cgemm_oncopy$(TSUFFIX).$(SUFFIX)
+CGEMMOTCOPYOBJ =  cgemm_otcopy$(TSUFFIX).$(SUFFIX)
+CGEMMINCOPYOBJ =  cgemm_incopy$(TSUFFIX).$(SUFFIX)
+CGEMMITCOPYOBJ =  cgemm_itcopy$(TSUFFIX).$(SUFFIX)
+
+ZGEMMKERNEL    = zgemm_kernel_power9.S
+ZGEMMONCOPY    = ../generic/zgemm_ncopy_2.c
+ZGEMMOTCOPY    = ../generic/zgemm_tcopy_2.c
+ZGEMMINCOPY    = ../generic/zgemm_ncopy_8.c
+ZGEMMITCOPY    = zgemm_tcopy_8_power8.S
+ZGEMMONCOPYOBJ =  zgemm_oncopy$(TSUFFIX).$(SUFFIX)
+ZGEMMOTCOPYOBJ =  zgemm_otcopy$(TSUFFIX).$(SUFFIX)
+ZGEMMINCOPYOBJ =  zgemm_incopy$(TSUFFIX).$(SUFFIX)
+ZGEMMITCOPYOBJ =  zgemm_itcopy$(TSUFFIX).$(SUFFIX)
+
+STRSMKERNEL_LN	=  ../generic/trsm_kernel_LN.c
+STRSMKERNEL_LT	=  ../generic/trsm_kernel_LT.c
+STRSMKERNEL_RN	=  ../generic/trsm_kernel_RN.c
+STRSMKERNEL_RT	=  ../generic/trsm_kernel_RT.c
+
+DTRSMKERNEL_LN	= ../generic/trsm_kernel_LN.c
+DTRSMKERNEL_LT	= dtrsm_kernel_LT_16x4_power8.S
+DTRSMKERNEL_RN	= ../generic/trsm_kernel_RN.c
+DTRSMKERNEL_RT	= ../generic/trsm_kernel_RT.c
+
+CTRSMKERNEL_LN	= ../generic/trsm_kernel_LN.c
+CTRSMKERNEL_LT	= ../generic/trsm_kernel_LT.c
+CTRSMKERNEL_RN	= ../generic/trsm_kernel_RN.c
+CTRSMKERNEL_RT	= ../generic/trsm_kernel_RT.c
+
+ZTRSMKERNEL_LN	= ../generic/trsm_kernel_LN.c
+ZTRSMKERNEL_LT	= ../generic/trsm_kernel_LT.c
+ZTRSMKERNEL_RN	= ../generic/trsm_kernel_RN.c
+ZTRSMKERNEL_RT	= ../generic/trsm_kernel_RT.c
+
+#Todo: CGEMM3MKERNEL should be 4x4 blocksizes.
+#CGEMM3MKERNEL    =  zgemm3m_kernel_8x4_sse3.S
+#ZGEMM3MKERNEL    =  zgemm3m_kernel_4x4_sse3.S
+
+#Pure C for other kernels
+#SAMAXKERNEL  = ../arm/amax.c
+#DAMAXKERNEL  = ../arm/amax.c
+#CAMAXKERNEL  = ../arm/zamax.c
+#ZAMAXKERNEL  = ../arm/zamax.c
+#
+#SAMINKERNEL  = ../arm/amin.c
+#DAMINKERNEL  = ../arm/amin.c
+#CAMINKERNEL  = ../arm/zamin.c
+#ZAMINKERNEL  = ../arm/zamin.c
+#
+#SMAXKERNEL   = ../arm/max.c
+#DMAXKERNEL   = ../arm/max.c
+#
+#SMINKERNEL   = ../arm/min.c
+#DMINKERNEL   = ../arm/min.c
+#
+ifneq ($(GCCVERSIONGTEQ9),1)
+ISAMAXKERNEL = isamax_power9.S
+else
+ISAMAXKERNEL = isamax.c
+endif
+IDAMAXKERNEL = idamax.c
+ifneq ($(GCCVERSIONGTEQ9),1)
+ICAMAXKERNEL = icamax_power9.S
+else
+ICAMAXKERNEL = icamax.c
+endif
+IZAMAXKERNEL = izamax.c
+#
+ifneq ($(GCCVERSIONGTEQ9),1)
+ISAMINKERNEL = isamin_power9.S
+else
+ISAMINKERNEL = isamin.c
+endif
+IDAMINKERNEL = idamin.c
+ifneq ($(GCCVERSIONGTEQ9),1)
+ICAMINKERNEL = icamin_power9.S
+else
+ICAMINKERNEL = icamin.c
+endif
+IZAMINKERNEL = izamin.c
+#
+#ISMAXKERNEL  = ../arm/imax.c
+#IDMAXKERNEL  = ../arm/imax.c
+#
+#ISMINKERNEL  = ../arm/imin.c
+#IDMINKERNEL  = ../arm/imin.c
+#
+SASUMKERNEL  = sasum.c
+DASUMKERNEL  = dasum.c
+CASUMKERNEL  = casum.c
+ZASUMKERNEL  = zasum.c
+#
+SAXPYKERNEL  = saxpy.c
+DAXPYKERNEL  = daxpy.c
+ifneq ($(GCCVERSIONGTEQ9),1)
+CAXPYKERNEL  = caxpy_power9.S
+else
+CAXPYKERNEL  = caxpy.c
+endif
+ZAXPYKERNEL  = zaxpy.c
+#
+SCOPYKERNEL  = scopy.c
+DCOPYKERNEL  = dcopy.c
+CCOPYKERNEL  = ccopy.c
+ZCOPYKERNEL  = zcopy.c
+#
+SDOTKERNEL   =  sdot.c
+DDOTKERNEL   =  ddot.c
+DSDOTKERNEL  =  sdot.c
+ifneq ($(GCCVERSIONGTEQ9),1)
+CDOTKERNEL   =  cdot_power9.S
+else
+CDOTKERNEL   =  cdot.c
+endif
+ZDOTKERNEL   =  zdot.c
+#
+SNRM2KERNEL  = ../arm/nrm2.c
+DNRM2KERNEL  = ../arm/nrm2.c
+CNRM2KERNEL  = ../arm/znrm2.c
+ZNRM2KERNEL  = ../arm/znrm2.c
+#
+SROTKERNEL   = srot.c
+DROTKERNEL   = drot.c
+CROTKERNEL   = crot.c
+ZROTKERNEL   = zrot.c
+#
+SSCALKERNEL  = sscal.c
+DSCALKERNEL  = dscal.c
+CSCALKERNEL  = zscal.c
+ZSCALKERNEL  = zscal.c
+#
+SSWAPKERNEL  = sswap.c
+DSWAPKERNEL  = dswap.c
+CSWAPKERNEL  = cswap.c
+ZSWAPKERNEL  = zswap.c
+#
+
+SGEMVNKERNEL = sgemv_n.c
+DGEMVNKERNEL = dgemv_n.c
+CGEMVNKERNEL = cgemv_n.c
+ZGEMVNKERNEL = zgemv_n_4.c
+#
+SGEMVTKERNEL = sgemv_t.c
+DGEMVTKERNEL = dgemv_t.c
+CGEMVTKERNEL = cgemv_t.c
+ZGEMVTKERNEL = zgemv_t_4.c
+
+
+#SSYMV_U_KERNEL =  ../generic/symv_k.c
+#SSYMV_L_KERNEL =  ../generic/symv_k.c
+#DSYMV_U_KERNEL =  ../generic/symv_k.c
+#DSYMV_L_KERNEL =  ../generic/symv_k.c
+#QSYMV_U_KERNEL =  ../generic/symv_k.c
+#QSYMV_L_KERNEL =  ../generic/symv_k.c
+#CSYMV_U_KERNEL =  ../generic/zsymv_k.c
+#CSYMV_L_KERNEL =  ../generic/zsymv_k.c
+#ZSYMV_U_KERNEL =  ../generic/zsymv_k.c
+#ZSYMV_L_KERNEL =  ../generic/zsymv_k.c
+#XSYMV_U_KERNEL =  ../generic/zsymv_k.c
+#XSYMV_L_KERNEL =  ../generic/zsymv_k.c
+
+#ZHEMV_U_KERNEL =  ../generic/zhemv_k.c
+#ZHEMV_L_KERNEL =  ../generic/zhemv_k.c
+
+LSAME_KERNEL = ../generic/lsame.c
+SCABS_KERNEL	= ../generic/cabs.c
+DCABS_KERNEL	= ../generic/cabs.c
+QCABS_KERNEL	= ../generic/cabs.c
+
+#Dump kernel
+CGEMM3MKERNEL    = ../generic/zgemm3mkernel_dump.c
+ZGEMM3MKERNEL    = ../generic/zgemm3mkernel_dump.c
+
+endif
diff --git a/kernel/power/casum.c b/kernel/power/casum.c
index a9ece0768..3478a39ef 100644
--- a/kernel/power/casum.c
+++ b/kernel/power/casum.c
@@ -46,7 +46,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #endif
 
-#if defined(POWER8)  || defined(POWER9)
+#if defined(POWER8)  || defined(POWER9) || defined(POWER10)
 #include "casum_microk_power8.c"
 #endif
 
diff --git a/kernel/power/ccopy.c b/kernel/power/ccopy.c
index 50df84cc5..cbe5b48d2 100644
--- a/kernel/power/ccopy.c
+++ b/kernel/power/ccopy.c
@@ -35,7 +35,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "common.h"
 
-#if defined(POWER8) || defined(POWER9)
+#if defined(POWER8) || defined(POWER9) || defined(POWER10)
 #include "ccopy_microk_power8.c"
 #endif
 
diff --git a/kernel/power/crot.c b/kernel/power/crot.c
index 2a5835546..5c1d44620 100644
--- a/kernel/power/crot.c
+++ b/kernel/power/crot.c
@@ -27,7 +27,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  
 #include "common.h"
  
-#if defined(POWER8) || defined(POWER9)
+#if defined(POWER8) || defined(POWER9) || defined(POWER10)
 
 static void crot_kernel_8 (long n, float *x, float *y, float c, float s)
 {
diff --git a/kernel/power/cswap.c b/kernel/power/cswap.c
index 31e02fe5a..88cb1d638 100644
--- a/kernel/power/cswap.c
+++ b/kernel/power/cswap.c
@@ -36,7 +36,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "common.h"
 
 
-#if defined(POWER8)  || defined(POWER9)
+#if defined(POWER8)  || defined(POWER9) || defined(POWER10)
 #include "cswap_microk_power8.c"
 #endif
 
diff --git a/kernel/power/dasum.c b/kernel/power/dasum.c
index d0e060977..09e06d909 100644
--- a/kernel/power/dasum.c
+++ b/kernel/power/dasum.c
@@ -46,7 +46,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #endif
 
-#if defined(POWER8) || defined(POWER9)
+#if defined(POWER8) || defined(POWER9) || defined(POWER10)
 #include "dasum_microk_power8.c"
 #endif
 
diff --git a/kernel/power/daxpy.c b/kernel/power/daxpy.c
index f09611ff0..018beafd1 100644
--- a/kernel/power/daxpy.c
+++ b/kernel/power/daxpy.c
@@ -36,7 +36,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "common.h"
 
 
-#if defined(POWER8) || defined(POWER9)
+#if defined(POWER8) || defined(POWER9) || defined(POWER10)
 #include "daxpy_microk_power8.c"
 #endif
 
diff --git a/kernel/power/dcopy.c b/kernel/power/dcopy.c
index 27b39144b..cf203e71e 100644
--- a/kernel/power/dcopy.c
+++ b/kernel/power/dcopy.c
@@ -35,7 +35,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "common.h"
 
-#if defined(POWER8) || defined(POWER9)
+#if defined(POWER8) || defined(POWER9) || defined(POWER10)
 #include "dcopy_microk_power8.c"
 #endif
 
diff --git a/kernel/power/ddot.c b/kernel/power/ddot.c
index f985df1c5..bd9e1fb97 100644
--- a/kernel/power/ddot.c
+++ b/kernel/power/ddot.c
@@ -36,7 +36,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "common.h"
 
 
-#if defined(POWER8)  || defined(POWER9)
+#if defined(POWER8)  || defined(POWER9) || defined(POWER10)
 #include "ddot_microk_power8.c"
 #endif
 
diff --git a/kernel/power/dgemv_n.c b/kernel/power/dgemv_n.c
index 1a3d7669c..b4dfda550 100644
--- a/kernel/power/dgemv_n.c
+++ b/kernel/power/dgemv_n.c
@@ -38,7 +38,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "common.h"
 
 
-#if defined(POWER8) || defined(POWER9)
+#if defined(POWER8) || defined(POWER9) || defined(POWER10)
 #include "dgemv_n_microk_power8.c"
 #endif
 
diff --git a/kernel/power/drot.c b/kernel/power/drot.c
index baeb54205..b808ab566 100644
--- a/kernel/power/drot.c
+++ b/kernel/power/drot.c
@@ -39,7 +39,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #pragma GCC optimize "O1"
 
-#if defined(POWER8) || defined(POWER9)
+#if defined(POWER8) || defined(POWER9) || defined(POWER10)
 #include "drot_microk_power8.c"
 #endif
 
diff --git a/kernel/power/dscal.c b/kernel/power/dscal.c
index 779a08e9c..7e0fe48c0 100644
--- a/kernel/power/dscal.c
+++ b/kernel/power/dscal.c
@@ -35,7 +35,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "common.h"
 
-#if defined(POWER8) || defined(POWER9)
+#if defined(POWER8) || defined(POWER9) || defined(POWER10)
 #include "dscal_microk_power8.c"
 #endif
 
diff --git a/kernel/power/dswap.c b/kernel/power/dswap.c
index 52b7f50da..795bb10b4 100644
--- a/kernel/power/dswap.c
+++ b/kernel/power/dswap.c
@@ -35,7 +35,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "common.h"
 
-#if defined(POWER8) || defined(POWER9)
+#if defined(POWER8) || defined(POWER9) || defined(POWER10)
 #include "dswap_microk_power8.c"
 #endif
 
diff --git a/kernel/power/sasum.c b/kernel/power/sasum.c
index 5908347d3..b259d7d76 100644
--- a/kernel/power/sasum.c
+++ b/kernel/power/sasum.c
@@ -46,7 +46,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #endif
 
-#if defined(POWER8) || defined(POWER9)
+#if defined(POWER8) || defined(POWER9) || defined(POWER10)
 #include "sasum_microk_power8.c"
 #endif
 
diff --git a/kernel/power/scopy.c b/kernel/power/scopy.c
index 5e3fe45a5..5207d386e 100644
--- a/kernel/power/scopy.c
+++ b/kernel/power/scopy.c
@@ -35,7 +35,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "common.h"
 
-#if defined(POWER8) || defined(POWER9)
+#if defined(POWER8) || defined(POWER9) || defined(POWER10)
 #include "scopy_microk_power8.c"
 #endif
 
diff --git a/kernel/power/sdot.c b/kernel/power/sdot.c
index ae527dde9..8de434e41 100644
--- a/kernel/power/sdot.c
+++ b/kernel/power/sdot.c
@@ -35,7 +35,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "common.h"
 
-#if defined(POWER8)  || defined(POWER9)
+#if defined(POWER8)  || defined(POWER9) || defined(POWER10)
 #include "sdot_microk_power8.c"
 #endif
 
diff --git a/kernel/power/srot.c b/kernel/power/srot.c
index 6af813c16..9638a59eb 100644
--- a/kernel/power/srot.c
+++ b/kernel/power/srot.c
@@ -39,7 +39,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #pragma GCC optimize "O1"
 
-#if defined(POWER8) || defined(POWER9)
+#if defined(POWER8) || defined(POWER9) || defined(POWER10)
 #include "srot_microk_power8.c"
 #endif
 
diff --git a/kernel/power/sscal.c b/kernel/power/sscal.c
index 4f3ba5698..ddd5b2c5b 100644
--- a/kernel/power/sscal.c
+++ b/kernel/power/sscal.c
@@ -35,7 +35,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "common.h"
 
-#if defined(POWER8) || defined(POWER9)
+#if defined(POWER8) || defined(POWER9) || defined(POWER10)
 #include "sscal_microk_power8.c"
 #endif
 
diff --git a/kernel/power/sswap.c b/kernel/power/sswap.c
index 23d13280f..a56434444 100644
--- a/kernel/power/sswap.c
+++ b/kernel/power/sswap.c
@@ -35,7 +35,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "common.h"
 
-#if defined(POWER8) || defined(POWER9)
+#if defined(POWER8) || defined(POWER9) || defined(POWER10)
 #include "sswap_microk_power8.c"
 #endif
 
diff --git a/kernel/power/zasum.c b/kernel/power/zasum.c
index f61c62e75..8383e39ab 100644
--- a/kernel/power/zasum.c
+++ b/kernel/power/zasum.c
@@ -46,7 +46,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #endif
 
-#if defined(POWER8) || defined(POWER9)
+#if defined(POWER8) || defined(POWER9) || defined(POWER10)
 #include "zasum_microk_power8.c"
 #endif
 
diff --git a/kernel/power/zaxpy.c b/kernel/power/zaxpy.c
index f0f8c6910..4a7c26c69 100644
--- a/kernel/power/zaxpy.c
+++ b/kernel/power/zaxpy.c
@@ -36,7 +36,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "common.h"
 
 
-#if defined(POWER8) || defined(POWER9)
+#if defined(POWER8) || defined(POWER9) || defined(POWER10)
 #include "zaxpy_microk_power8.c"
 #endif
 
diff --git a/kernel/power/zcopy.c b/kernel/power/zcopy.c
index b21d6ef15..bb80decd2 100644
--- a/kernel/power/zcopy.c
+++ b/kernel/power/zcopy.c
@@ -35,7 +35,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "common.h"
 
-#if defined(POWER8) || defined(POWER9)
+#if defined(POWER8) || defined(POWER9) || defined(POWER10)
 #include "zcopy_microk_power8.c"
 #endif
 
diff --git a/kernel/power/zdot.c b/kernel/power/zdot.c
index fd36c7f44..9086ef35b 100644
--- a/kernel/power/zdot.c
+++ b/kernel/power/zdot.c
@@ -36,7 +36,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "common.h"
 
 
-#if defined(POWER8) || defined(POWER9)
+#if defined(POWER8) || defined(POWER9) || defined(POWER10)
 #include "zdot_microk_power8.c"
 #endif
 
diff --git a/kernel/power/zscal.c b/kernel/power/zscal.c
index a1b441d2c..16b584bca 100644
--- a/kernel/power/zscal.c
+++ b/kernel/power/zscal.c
@@ -38,7 +38,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #pragma GCC optimize "O1"
 
-#if defined(POWER8) || defined(POWER9)
+#if defined(POWER8) || defined(POWER9) || defined(POWER10)
 #if defined(DOUBLE)
 #include "zscal_microk_power8.c"
 #endif
diff --git a/kernel/power/zswap.c b/kernel/power/zswap.c
index 1d8826f41..c6508f032 100644
--- a/kernel/power/zswap.c
+++ b/kernel/power/zswap.c
@@ -36,7 +36,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "common.h"
 
 
-#if defined(POWER8) || defined(POWER9)
+#if defined(POWER8) || defined(POWER9) || defined(POWER10)
 #include "zswap_microk_power8.c"
 #endif
 
diff --git a/param.h b/param.h
index 04928277c..fd0ea7599 100644
--- a/param.h
+++ b/param.h
@@ -2260,7 +2260,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #endif
 
-#if defined(POWER9)
+#if defined(POWER9) || defined(POWER10)
 
 #define SNUMOPT		16
 #define DNUMOPT		8

From 007d9f97d7d53bfb569708ba51f8d9fdd3fba211 Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Sat, 13 Jun 2020 19:25:28 +0200
Subject: [PATCH 08/48] Make gotoblas_corename report the name of the selected
 TARGET rather than its aliases

---
 driver/others/dynamic.c | 50 +++++++++++++++++++++++++++++++++++------
 1 file changed, 43 insertions(+), 7 deletions(-)

diff --git a/driver/others/dynamic.c b/driver/others/dynamic.c
index 2e87e186a..1bf0e4a6d 100644
--- a/driver/others/dynamic.c
+++ b/driver/others/dynamic.c
@@ -764,18 +764,53 @@ char *gotoblas_corename(void) {
   if (gotoblas == &gotoblas_NORTHWOOD)    return corename[ 3];
   if (gotoblas == &gotoblas_PRESCOTT)     return corename[ 4];
   if (gotoblas == &gotoblas_BANIAS)       return corename[ 5];
-  if (gotoblas == &gotoblas_ATOM)         return corename[ 6];
+  if (gotoblas == &gotoblas_ATOM)
+#ifdef DYNAMIC_OLDER
+           return corename[ 6];
+#else
+           return corename[10];
+#endif
   if (gotoblas == &gotoblas_CORE2)        return corename[ 7];
-  if (gotoblas == &gotoblas_PENRYN)       return corename[ 8];
-  if (gotoblas == &gotoblas_DUNNINGTON)   return corename[ 9];
+  if (gotoblas == &gotoblas_PENRYN)
+#ifdef DYNAMIC_OLDER
+           return corename[ 8];
+#else
+           return corename[7];
+#endif
+  if (gotoblas == &gotoblas_DUNNINGTON)
+#ifdef DYNAMIC_OLDER
+           return corename[ 9];
+#else
+           return corename[7];
+#endif
   if (gotoblas == &gotoblas_NEHALEM)      return corename[10];
   if (gotoblas == &gotoblas_ATHLON)       return corename[11];
-  if (gotoblas == &gotoblas_OPTERON_SSE3) return corename[12];
-  if (gotoblas == &gotoblas_OPTERON)      return corename[13];
+  if (gotoblas == &gotoblas_OPTERON_SSE3)
+#ifdef DYNAMIC_OLDER
+           return corename[12];
+#else
+           return corename[7];
+#endif
+  if (gotoblas == &gotoblas_OPTERON)
+#ifdef DYNAMIC_OLDER
+           return corename[13];
+#else
+           return corename[7];
+#endif
   if (gotoblas == &gotoblas_BARCELONA)    return corename[14];
-  if (gotoblas == &gotoblas_NANO)         return corename[15];
+  if (gotoblas == &gotoblas_NANO)
+#ifdef DYNAMIC_OLDER
+           return corename[15];
+#else
+           return corename[10];
+#endif
   if (gotoblas == &gotoblas_SANDYBRIDGE)  return corename[16];
-  if (gotoblas == &gotoblas_BOBCAT)       return corename[17];
+  if (gotoblas == &gotoblas_BOBCAT)
+#ifdef DYNAMIC_OLDER
+           return corename[17];
+#else
+           return corename[7];
+#endif
   if (gotoblas == &gotoblas_BULLDOZER)    return corename[18];
   if (gotoblas == &gotoblas_PILEDRIVER)   return corename[19];
   if (gotoblas == &gotoblas_HASWELL)      return corename[20];
@@ -787,6 +822,7 @@ char *gotoblas_corename(void) {
 }
 
 
+
 static gotoblas_t *force_coretype(char *coretype){
 
 	int i ;

From 41fc6f3cd2c46ff8ed136b2eb03782c434646c00 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20M=C3=A4rtens?= <maertens@stce.rwth-aachen.de>
Date: Sat, 13 Jun 2020 22:37:39 +0200
Subject: [PATCH 09/48] Added missing exported symbols.

---
 exports/gensymbol | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/exports/gensymbol b/exports/gensymbol
index 0a68a3572..73b4be248 100644
--- a/exports/gensymbol
+++ b/exports/gensymbol
@@ -40,15 +40,10 @@
     ztbsv,ztpmv,ztpsv,ztrmm,ztrmv,ztrsm,ztrsv,
     xerbla,
     saxpby,daxpby,caxpby,zaxpby,
+    somatcopy, domatcopy, comatcopy, zomatcopy,
+    simatcopy, dimatcopy, cimatcopy, zimatcopy,
     sgeadd,dgeadd,cgeadd,zgeadd,
-    somatcopy,
-    simatcopy,
-    domatcopy,
-    dimatcopy,
-    comatcopy,
-    cimatcopy,
-    zomatcopy,
-    zimatcopy,
+    ssum, dsum, scsum, dzsum
 );
 
 @halfblasobjs = (shgemm);
@@ -81,7 +76,12 @@
     cblas_saxpby,cblas_daxpby,cblas_caxpby,cblas_zaxpby,
     cblas_somatcopy, cblas_domatcopy, cblas_comatcopy, cblas_zomatcopy,
     cblas_simatcopy, cblas_dimatcopy, cblas_cimatcopy, cblas_zimatcopy,
-    cblas_sgeadd, cblas_dgeadd,cblas_cgeadd, cblas_zgeadd
+    cblas_sgeadd, cblas_dgeadd,cblas_cgeadd, cblas_zgeadd,
+    cblas_isamin, cblas_idamin, cblas_icamin, cblas_izamin,
+    cblas_ismin, cblas_idmin, cblas_icmin, cblas_izmin,
+    cblas_ismax, cblas_idmax, cblas_icmax, cblas_izmax,
+    cblas_ssum, cblas_dsum, cblas_scsum, cblas_dzsum,
+    cblas_xerbla
 );
 
 @halfcblasobjs = (cblas_shgemm);
@@ -3501,9 +3501,12 @@ if ($ARGV[1] eq "x86")    { @underscore_objs = (@underscore_objs, @gemm3mobjs);
 if ($ARGV[1] eq "ia64")   { @underscore_objs = (@underscore_objs, @gemm3mobjs); };
 if ($ARGV[1] eq "MIPS")   { @underscore_objs = (@underscore_objs, @gemm3mobjs); };
 
-
 if ($ARGV[4] == 0) {
     @no_underscore_objs = (@cblasobjs, @misc_no_underscore_objs);
+    if ($ARGV[1] eq "x86_64") { @no_underscore_objs = (@no_underscore_objs, @cblasgemm3mobjs); };
+    if ($ARGV[1] eq "x86")    { @no_underscore_objs = (@no_underscore_objs, @cblasgemm3mobjs); };
+    if ($ARGV[1] eq "ia64")   { @no_underscore_objs = (@no_underscore_objs, @cblasgemm3mobjs); };
+    if ($ARGV[1] eq "MIPS")   { @no_underscore_objs = (@no_underscore_objs, @cblasgemm3mobjs); };
 }else{
     #NO_CBLAS=1
     @no_underscore_objs = (@misc_no_underscore_objs);

From abf670757b2838dd68eede52aba29275b4c6d2cb Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Sat, 13 Jun 2020 23:21:13 +0200
Subject: [PATCH 10/48] Respect predefined defaults for AR, AS, LD and RANLIB

---
 Makefile.system | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile.system b/Makefile.system
index 56e94f2a6..d5e747824 100644
--- a/Makefile.system
+++ b/Makefile.system
@@ -263,10 +263,10 @@ endif
 
 ARFLAGS	=
 CPP	= $(COMPILER) -E
-AR	= $(CROSS_SUFFIX)ar
-AS	= $(CROSS_SUFFIX)as
-LD	= $(CROSS_SUFFIX)ld
-RANLIB	= $(CROSS_SUFFIX)ranlib
+AR	?= $(CROSS_SUFFIX)ar
+AS	?= $(CROSS_SUFFIX)as
+LD	?= $(CROSS_SUFFIX)ld
+RANLIB	?= $(CROSS_SUFFIX)ranlib
 NM	= $(CROSS_SUFFIX)nm
 DLLWRAP = $(CROSS_SUFFIX)dllwrap
 OBJCOPY = $(CROSS_SUFFIX)objcopy

From 0ed2adf0b25f25ddedf8c858010d233b87d615cc Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Sun, 14 Jun 2020 00:01:20 +0200
Subject: [PATCH 11/48] Fix spelling of flang option -Mrecursive and add -Kieee

---
 Makefile.system | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/Makefile.system b/Makefile.system
index 56e94f2a6..31cdd12b2 100644
--- a/Makefile.system
+++ b/Makefile.system
@@ -783,7 +783,7 @@ endif
 
 ifeq ($(F_COMPILER), FLANG)
 CCOMMON_OPT += -DF_INTERFACE_FLANG
-FCOMMON_OPT += -frecursive
+FCOMMON_OPT += -Mrecursive -Kieee
 ifdef BINARY64
 ifdef INTERFACE64
 ifneq ($(INTERFACE64), 0)
@@ -797,11 +797,6 @@ endif
 ifeq ($(USE_OPENMP), 1)
 FCOMMON_OPT += -fopenmp
 endif
-#ifeq ($(OSNAME), Linux)
-#ifeq ($(ARCH), x86_64)
-#FLANG_VENDOR := $(shell expr `$(FC) --version|cut -f 1 -d "."|head -1`)
-#endif
-#endif
 endif
 
 ifeq ($(F_COMPILER), G77)
@@ -1276,7 +1271,6 @@ endif
 
 override CFLAGS     += $(COMMON_OPT) $(CCOMMON_OPT) -I$(TOPDIR)
 override PFLAGS     += $(COMMON_OPT) $(CCOMMON_OPT) -I$(TOPDIR) -DPROFILE $(COMMON_PROF)
-#ifeq ($(FLANG_VENDOR),AOCC)
 ifeq ($(F_COMPILER),FLANG)
 override FFLAGS     += $(filter-out -O2 -O3,$(COMMON_OPT)) -O1 $(FCOMMON_OPT)
 else

From 1dd712131e0e4efcae52f7171cb47d427cd60fff Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Sun, 14 Jun 2020 00:09:31 +0200
Subject: [PATCH 12/48] Fix spelling of flang option -Mrecursive and add -Kieee

---
 cmake/fc.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/fc.cmake b/cmake/fc.cmake
index 9dcedffb2..fc1f9bb22 100644
--- a/cmake/fc.cmake
+++ b/cmake/fc.cmake
@@ -16,7 +16,7 @@ if (${F_COMPILER} STREQUAL "FLANG")
   if (USE_OPENMP)
     set(FCOMMON_OPT "${FCOMMON_OPT} -fopenmp")
   endif ()
-  set(FCOMMON_OPT "${FCOMMON_OPT} -frecursive")
+  set(FCOMMON_OPT "${FCOMMON_OPT} -Mrecursive -Kieee")
 endif ()
 
 if (${F_COMPILER} STREQUAL "G77")

From 18a11137f1be433b88ef34e0fb115e7280a67d12 Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Sun, 14 Jun 2020 10:26:25 +0200
Subject: [PATCH 13/48] Update BLAS tests to correspond to Reference-LAPACK
 3.9.0

replaces calculation of machine precision with call to epsilon intrinsic and removes the requirement for previous output files to be removed before rerunning tests
---
 test/cblat1.f |  83 ++++++++++++++++------
 test/cblat2.f | 188 +++++++++++++++++++++++++++++--------------------
 test/cblat3.f | 187 ++++++++++++++++++++++++++++++------------------
 test/dblat2.f | 186 +++++++++++++++++++++++++++++-------------------
 test/dblat3.f | 168 ++++++++++++++++++++++++++++----------------
 test/sblat2.f | 186 +++++++++++++++++++++++++++++-------------------
 test/sblat3.f | 168 ++++++++++++++++++++++++++++----------------
 test/zblat1.f |  83 ++++++++++++++++------
 test/zblat2.f | 188 +++++++++++++++++++++++++++++--------------------
 test/zblat3.f | 191 ++++++++++++++++++++++++++++++++------------------
 10 files changed, 1036 insertions(+), 592 deletions(-)

diff --git a/test/cblat1.f b/test/cblat1.f
index d6b53d105..ecf2a44cb 100644
--- a/test/cblat1.f
+++ b/test/cblat1.f
@@ -1,7 +1,49 @@
+*> \brief \b CBLAT1
+*
+*  =========== DOCUMENTATION ===========
+*
+* Online html documentation available at
+*            http://www.netlib.org/lapack/explore-html/
+*
+*  Definition:
+*  ===========
+*
+*       PROGRAM CBLAT1
+*
+*
+*> \par Purpose:
+*  =============
+*>
+*> \verbatim
+*>
+*>    Test program for the COMPLEX Level 1 BLAS.
+*>    Based upon the original BLAS test routine together with:
+*>
+*>    F06GAF Example Program Text
+*> \endverbatim
+*
+*  Authors:
+*  ========
+*
+*> \author Univ. of Tennessee
+*> \author Univ. of California Berkeley
+*> \author Univ. of Colorado Denver
+*> \author NAG Ltd.
+*
+*> \date April 2012
+*
+*> \ingroup complex_blas_testing
+*
+*  =====================================================================
       PROGRAM CBLAT1
-*     Test program for the COMPLEX    Level 1 BLAS.
-*     Based upon the original BLAS test routine together with:
-*     F06GAF Example Program Text
+*
+*  -- Reference BLAS test routine (version 3.7.0) --
+*  -- Reference BLAS is a software package provided by Univ. of Tennessee,    --
+*  -- Univ. of California Berkeley, Univ. of Colorado Denver and NAG Ltd..--
+*     April 2012
+*
+*  =====================================================================
+*
 *     .. Parameters ..
       INTEGER          NOUT
       PARAMETER        (NOUT=6)
@@ -114,8 +156,8 @@
      +                  (5.0E0,6.0E0), (5.0E0,6.0E0), (0.1E0,0.1E0),
      +                  (-0.6E0,0.1E0), (0.1E0,-0.3E0), (7.0E0,8.0E0),
      +                  (7.0E0,8.0E0), (7.0E0,8.0E0), (7.0E0,8.0E0),
-     +                  (7.0E0,8.0E0), (0.3E0,0.1E0), (0.1E0,0.4E0),
-     +                  (0.4E0,0.1E0), (0.1E0,0.2E0), (2.0E0,3.0E0),
+     +                  (7.0E0,8.0E0), (0.3E0,0.1E0), (0.5E0,0.0E0),
+     +                  (0.0E0,0.5E0), (0.0E0,0.2E0), (2.0E0,3.0E0),
      +                  (2.0E0,3.0E0), (2.0E0,3.0E0), (2.0E0,3.0E0)/
       DATA              ((CV(I,J,2),I=1,8),J=1,5)/(0.1E0,0.1E0),
      +                  (4.0E0,5.0E0), (4.0E0,5.0E0), (4.0E0,5.0E0),
@@ -129,10 +171,10 @@
      +                  (3.0E0,6.0E0), (-0.6E0,0.1E0), (4.0E0,7.0E0),
      +                  (0.1E0,-0.3E0), (7.0E0,2.0E0), (7.0E0,2.0E0),
      +                  (7.0E0,2.0E0), (0.3E0,0.1E0), (5.0E0,8.0E0),
-     +                  (0.1E0,0.4E0), (6.0E0,9.0E0), (0.4E0,0.1E0),
-     +                  (8.0E0,3.0E0), (0.1E0,0.2E0), (9.0E0,4.0E0)/
-      DATA              STRUE2/0.0E0, 0.5E0, 0.6E0, 0.7E0, 0.7E0/
-      DATA              STRUE4/0.0E0, 0.7E0, 1.0E0, 1.3E0, 1.7E0/
+     +                  (0.5E0,0.0E0), (6.0E0,9.0E0), (0.0E0,0.5E0),
+     +                  (8.0E0,3.0E0), (0.0E0,0.2E0), (9.0E0,4.0E0)/
+      DATA              STRUE2/0.0E0, 0.5E0, 0.6E0, 0.7E0, 0.8E0/
+      DATA              STRUE4/0.0E0, 0.7E0, 1.0E0, 1.3E0, 1.6E0/
       DATA              ((CTRUE5(I,J,1),I=1,8),J=1,5)/(0.1E0,0.1E0),
      +                  (1.0E0,2.0E0), (1.0E0,2.0E0), (1.0E0,2.0E0),
      +                  (1.0E0,2.0E0), (1.0E0,2.0E0), (1.0E0,2.0E0),
@@ -145,8 +187,8 @@
      +                  (0.11E0,-0.03E0), (-0.17E0,0.46E0),
      +                  (-0.17E0,-0.19E0), (7.0E0,8.0E0), (7.0E0,8.0E0),
      +                  (7.0E0,8.0E0), (7.0E0,8.0E0), (7.0E0,8.0E0),
-     +                  (0.19E0,-0.17E0), (0.32E0,0.09E0),
-     +                  (0.23E0,-0.24E0), (0.18E0,0.01E0),
+     +                  (0.19E0,-0.17E0), (0.20E0,-0.35E0),
+     +                  (0.35E0,0.20E0), (0.14E0,0.08E0),
      +                  (2.0E0,3.0E0), (2.0E0,3.0E0), (2.0E0,3.0E0),
      +                  (2.0E0,3.0E0)/
       DATA              ((CTRUE5(I,J,2),I=1,8),J=1,5)/(0.1E0,0.1E0),
@@ -162,9 +204,9 @@
      +                  (-0.17E0,0.46E0), (4.0E0,7.0E0),
      +                  (-0.17E0,-0.19E0), (7.0E0,2.0E0), (7.0E0,2.0E0),
      +                  (7.0E0,2.0E0), (0.19E0,-0.17E0), (5.0E0,8.0E0),
-     +                  (0.32E0,0.09E0), (6.0E0,9.0E0),
-     +                  (0.23E0,-0.24E0), (8.0E0,3.0E0),
-     +                  (0.18E0,0.01E0), (9.0E0,4.0E0)/
+     +                  (0.20E0,-0.35E0), (6.0E0,9.0E0),
+     +                  (0.35E0,0.20E0), (8.0E0,3.0E0),
+     +                  (0.14E0,0.08E0), (9.0E0,4.0E0)/
       DATA              ((CTRUE6(I,J,1),I=1,8),J=1,5)/(0.1E0,0.1E0),
      +                  (1.0E0,2.0E0), (1.0E0,2.0E0), (1.0E0,2.0E0),
      +                  (1.0E0,2.0E0), (1.0E0,2.0E0), (1.0E0,2.0E0),
@@ -177,8 +219,8 @@
      +                  (0.03E0,0.03E0), (-0.18E0,0.03E0),
      +                  (0.03E0,-0.09E0), (7.0E0,8.0E0), (7.0E0,8.0E0),
      +                  (7.0E0,8.0E0), (7.0E0,8.0E0), (7.0E0,8.0E0),
-     +                  (0.09E0,0.03E0), (0.03E0,0.12E0),
-     +                  (0.12E0,0.03E0), (0.03E0,0.06E0), (2.0E0,3.0E0),
+     +                  (0.09E0,0.03E0), (0.15E0,0.00E0),
+     +                  (0.00E0,0.15E0), (0.00E0,0.06E0), (2.0E0,3.0E0),
      +                  (2.0E0,3.0E0), (2.0E0,3.0E0), (2.0E0,3.0E0)/
       DATA              ((CTRUE6(I,J,2),I=1,8),J=1,5)/(0.1E0,0.1E0),
      +                  (4.0E0,5.0E0), (4.0E0,5.0E0), (4.0E0,5.0E0),
@@ -193,8 +235,8 @@
      +                  (-0.18E0,0.03E0), (4.0E0,7.0E0),
      +                  (0.03E0,-0.09E0), (7.0E0,2.0E0), (7.0E0,2.0E0),
      +                  (7.0E0,2.0E0), (0.09E0,0.03E0), (5.0E0,8.0E0),
-     +                  (0.03E0,0.12E0), (6.0E0,9.0E0), (0.12E0,0.03E0),
-     +                  (8.0E0,3.0E0), (0.03E0,0.06E0), (9.0E0,4.0E0)/
+     +                  (0.15E0,0.00E0), (6.0E0,9.0E0), (0.00E0,0.15E0),
+     +                  (8.0E0,3.0E0), (0.00E0,0.06E0), (9.0E0,4.0E0)/
       DATA              ITRUE3/0, 1, 2, 2, 2/
 *     .. Executable Statements ..
       DO 60 INCX = 1, 2
@@ -529,7 +571,8 @@
 *
 *     .. Parameters ..
       INTEGER          NOUT
-      PARAMETER        (NOUT=6)
+      REAL             ZERO
+      PARAMETER        (NOUT=6, ZERO=0.0E0)
 *     .. Scalar Arguments ..
       REAL             SFAC
       INTEGER          LEN
@@ -552,7 +595,7 @@
 *
       DO 40 I = 1, LEN
          SD = SCOMP(I) - STRUE(I)
-         IF (SDIFF(ABS(SSIZE(I))+ABS(SFAC*SD),ABS(SSIZE(I))).EQ.0.0E0)
+         IF (ABS(SFAC*SD) .LE. ABS(SSIZE(I))*EPSILON(ZERO))
      +       GO TO 40
 *
 *                             HERE    SCOMP(I) IS NOT CLOSE TO STRUE(I).
diff --git a/test/cblat2.f b/test/cblat2.f
index 20f188100..8c7bac48e 100644
--- a/test/cblat2.f
+++ b/test/cblat2.f
@@ -1,68 +1,114 @@
+*> \brief \b CBLAT2
+*
+*  =========== DOCUMENTATION ===========
+*
+* Online html documentation available at
+*            http://www.netlib.org/lapack/explore-html/
+*
+*  Definition:
+*  ===========
+*
+*       PROGRAM CBLAT2
+*
+*
+*> \par Purpose:
+*  =============
+*>
+*> \verbatim
+*>
+*> Test program for the COMPLEX          Level 2 Blas.
+*>
+*> The program must be driven by a short data file. The first 18 records
+*> of the file are read using list-directed input, the last 17 records
+*> are read using the format ( A6, L2 ). An annotated example of a data
+*> file can be obtained by deleting the first 3 characters from the
+*> following 35 lines:
+*> 'cblat2.out'      NAME OF SUMMARY OUTPUT FILE
+*> 6                 UNIT NUMBER OF SUMMARY FILE
+*> 'CBLA2T.SNAP'     NAME OF SNAPSHOT OUTPUT FILE
+*> -1                UNIT NUMBER OF SNAPSHOT FILE (NOT USED IF .LT. 0)
+*> F        LOGICAL FLAG, T TO REWIND SNAPSHOT FILE AFTER EACH RECORD.
+*> F        LOGICAL FLAG, T TO STOP ON FAILURES.
+*> T        LOGICAL FLAG, T TO TEST ERROR EXITS.
+*> 16.0     THRESHOLD VALUE OF TEST RATIO
+*> 6                 NUMBER OF VALUES OF N
+*> 0 1 2 3 5 9       VALUES OF N
+*> 4                 NUMBER OF VALUES OF K
+*> 0 1 2 4           VALUES OF K
+*> 4                 NUMBER OF VALUES OF INCX AND INCY
+*> 1 2 -1 -2         VALUES OF INCX AND INCY
+*> 3                 NUMBER OF VALUES OF ALPHA
+*> (0.0,0.0) (1.0,0.0) (0.7,-0.9)       VALUES OF ALPHA
+*> 3                 NUMBER OF VALUES OF BETA
+*> (0.0,0.0) (1.0,0.0) (1.3,-1.1)       VALUES OF BETA
+*> CGEMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> CGBMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> CHEMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> CHBMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> CHPMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> CTRMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> CTBMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> CTPMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> CTRSV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> CTBSV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> CTPSV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> CGERC  T PUT F FOR NO TEST. SAME COLUMNS.
+*> CGERU  T PUT F FOR NO TEST. SAME COLUMNS.
+*> CHER   T PUT F FOR NO TEST. SAME COLUMNS.
+*> CHPR   T PUT F FOR NO TEST. SAME COLUMNS.
+*> CHER2  T PUT F FOR NO TEST. SAME COLUMNS.
+*> CHPR2  T PUT F FOR NO TEST. SAME COLUMNS.
+*>
+*> Further Details
+*> ===============
+*>
+*>    See:
+*>
+*>       Dongarra J. J., Du Croz J. J., Hammarling S.  and Hanson R. J..
+*>       An  extended  set of Fortran  Basic Linear Algebra Subprograms.
+*>
+*>       Technical  Memoranda  Nos. 41 (revision 3) and 81,  Mathematics
+*>       and  Computer Science  Division,  Argonne  National Laboratory,
+*>       9700 South Cass Avenue, Argonne, Illinois 60439, US.
+*>
+*>       Or
+*>
+*>       NAG  Technical Reports TR3/87 and TR4/87,  Numerical Algorithms
+*>       Group  Ltd.,  NAG  Central  Office,  256  Banbury  Road, Oxford
+*>       OX2 7DE, UK,  and  Numerical Algorithms Group Inc.,  1101  31st
+*>       Street,  Suite 100,  Downers Grove,  Illinois 60515-1263,  USA.
+*>
+*>
+*> -- Written on 10-August-1987.
+*>    Richard Hanson, Sandia National Labs.
+*>    Jeremy Du Croz, NAG Central Office.
+*>
+*>    10-9-00:  Change STATUS='NEW' to 'UNKNOWN' so that the testers
+*>              can be run multiple times without deleting generated
+*>              output files (susan)
+*> \endverbatim
+*
+*  Authors:
+*  ========
+*
+*> \author Univ. of Tennessee
+*> \author Univ. of California Berkeley
+*> \author Univ. of Colorado Denver
+*> \author NAG Ltd.
+*
+*> \date April 2012
+*
+*> \ingroup complex_blas_testing
+*
+*  =====================================================================
       PROGRAM CBLAT2
 *
-*  Test program for the COMPLEX          Level 2 Blas.
+*  -- Reference BLAS test routine (version 3.7.0) --
+*  -- Reference BLAS is a software package provided by Univ. of Tennessee,    --
+*  -- Univ. of California Berkeley, Univ. of Colorado Denver and NAG Ltd..--
+*     April 2012
 *
-*  The program must be driven by a short data file. The first 18 records
-*  of the file are read using list-directed input, the last 17 records
-*  are read using the format ( A6, L2 ). An annotated example of a data
-*  file can be obtained by deleting the first 3 characters from the
-*  following 35 lines:
-*  'CBLAT2.SUMM'     NAME OF SUMMARY OUTPUT FILE
-*  6                 UNIT NUMBER OF SUMMARY FILE
-*  'CBLA2T.SNAP'     NAME OF SNAPSHOT OUTPUT FILE
-*  -1                UNIT NUMBER OF SNAPSHOT FILE (NOT USED IF .LT. 0)
-*  F        LOGICAL FLAG, T TO REWIND SNAPSHOT FILE AFTER EACH RECORD.
-*  F        LOGICAL FLAG, T TO STOP ON FAILURES.
-*  T        LOGICAL FLAG, T TO TEST ERROR EXITS.
-*  16.0     THRESHOLD VALUE OF TEST RATIO
-*  6                 NUMBER OF VALUES OF N
-*  0 1 2 3 5 9       VALUES OF N
-*  4                 NUMBER OF VALUES OF K
-*  0 1 2 4           VALUES OF K
-*  4                 NUMBER OF VALUES OF INCX AND INCY
-*  1 2 -1 -2         VALUES OF INCX AND INCY
-*  3                 NUMBER OF VALUES OF ALPHA
-*  (0.0,0.0) (1.0,0.0) (0.7,-0.9)       VALUES OF ALPHA
-*  3                 NUMBER OF VALUES OF BETA
-*  (0.0,0.0) (1.0,0.0) (1.3,-1.1)       VALUES OF BETA
-*  CGEMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  CGBMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  CHEMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  CHBMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  CHPMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  CTRMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  CTBMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  CTPMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  CTRSV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  CTBSV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  CTPSV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  CGERC  T PUT F FOR NO TEST. SAME COLUMNS.
-*  CGERU  T PUT F FOR NO TEST. SAME COLUMNS.
-*  CHER   T PUT F FOR NO TEST. SAME COLUMNS.
-*  CHPR   T PUT F FOR NO TEST. SAME COLUMNS.
-*  CHER2  T PUT F FOR NO TEST. SAME COLUMNS.
-*  CHPR2  T PUT F FOR NO TEST. SAME COLUMNS.
-*
-*     See:
-*
-*        Dongarra J. J., Du Croz J. J., Hammarling S.  and Hanson R. J..
-*        An  extended  set of Fortran  Basic Linear Algebra Subprograms.
-*
-*        Technical  Memoranda  Nos. 41 (revision 3) and 81,  Mathematics
-*        and  Computer Science  Division,  Argonne  National Laboratory,
-*        9700 South Cass Avenue, Argonne, Illinois 60439, US.
-*
-*        Or
-*
-*        NAG  Technical Reports TR3/87 and TR4/87,  Numerical Algorithms
-*        Group  Ltd.,  NAG  Central  Office,  256  Banbury  Road, Oxford
-*        OX2 7DE, UK,  and  Numerical Algorithms Group Inc.,  1101  31st
-*        Street,  Suite 100,  Downers Grove,  Illinois 60515-1263,  USA.
-*
-*
-*  -- Written on 10-August-1987.
-*     Richard Hanson, Sandia National Labs.
-*     Jeremy Du Croz, NAG Central Office.
+*  =====================================================================
 *
 *     .. Parameters ..
       INTEGER            NIN
@@ -71,8 +117,8 @@
       PARAMETER          ( NSUBS = 17 )
       COMPLEX            ZERO, ONE
       PARAMETER          ( ZERO = ( 0.0, 0.0 ), ONE = ( 1.0, 0.0 ) )
-      REAL               RZERO, RHALF, RONE
-      PARAMETER          ( RZERO = 0.0, RHALF = 0.5, RONE = 1.0 )
+      REAL               RZERO
+      PARAMETER          ( RZERO = 0.0 )
       INTEGER            NMAX, INCMAX
       PARAMETER          ( NMAX = 65, INCMAX = 2 )
       INTEGER            NINMAX, NIDMAX, NKBMAX, NALMAX, NBEMAX
@@ -126,7 +172,7 @@
 *
       READ( NIN, FMT = * )SUMMRY
       READ( NIN, FMT = * )NOUT
-      OPEN( NOUT, FILE = SUMMRY, STATUS = 'NEW' )
+      OPEN( NOUT, FILE = SUMMRY, STATUS = 'UNKNOWN' )
       NOUTC = NOUT
 *
 *     Read name and unit number for snapshot output file and open file.
@@ -135,7 +181,7 @@
       READ( NIN, FMT = * )NTRA
       TRACE = NTRA.GE.0
       IF( TRACE )THEN
-         OPEN( NTRA, FILE = SNAPS, STATUS = 'NEW' )
+         OPEN( NTRA, FILE = SNAPS, STATUS = 'UNKNOWN' )
       END IF
 *     Read the flag that directs rewinding of the snapshot file.
       READ( NIN, FMT = * )REWI
@@ -240,14 +286,7 @@
 *
 *     Compute EPS (the machine precision).
 *
-      EPS = RONE
-   90 CONTINUE
-      IF( SDIFF( RONE + EPS, RONE ).EQ.RZERO )
-     $   GO TO 100
-      EPS = RHALF*EPS
-      GO TO 90
-  100 CONTINUE
-      EPS = EPS + EPS
+      EPS = EPSILON(RZERO)
       WRITE( NOUT, FMT = 9998 )EPS
 *
 *     Check the reliability of CMVCH using exact data.
@@ -3079,7 +3118,6 @@
    50    CONTINUE
       END IF
 *
-   60 CONTINUE
       LCERES = .TRUE.
       GO TO 80
    70 CONTINUE
diff --git a/test/cblat3.f b/test/cblat3.f
index 5df1ddd64..a65e1364c 100644
--- a/test/cblat3.f
+++ b/test/cblat3.f
@@ -1,50 +1,96 @@
+*> \brief \b CBLAT3
+*
+*  =========== DOCUMENTATION ===========
+*
+* Online html documentation available at
+*            http://www.netlib.org/lapack/explore-html/
+*
+*  Definition:
+*  ===========
+*
+*       PROGRAM CBLAT3
+*
+*
+*> \par Purpose:
+*  =============
+*>
+*> \verbatim
+*>
+*> Test program for the COMPLEX          Level 3 Blas.
+*>
+*> The program must be driven by a short data file. The first 14 records
+*> of the file are read using list-directed input, the last 9 records
+*> are read using the format ( A6, L2 ). An annotated example of a data
+*> file can be obtained by deleting the first 3 characters from the
+*> following 23 lines:
+*> 'cblat3.out'      NAME OF SUMMARY OUTPUT FILE
+*> 6                 UNIT NUMBER OF SUMMARY FILE
+*> 'CBLAT3.SNAP'     NAME OF SNAPSHOT OUTPUT FILE
+*> -1                UNIT NUMBER OF SNAPSHOT FILE (NOT USED IF .LT. 0)
+*> F        LOGICAL FLAG, T TO REWIND SNAPSHOT FILE AFTER EACH RECORD.
+*> F        LOGICAL FLAG, T TO STOP ON FAILURES.
+*> T        LOGICAL FLAG, T TO TEST ERROR EXITS.
+*> 16.0     THRESHOLD VALUE OF TEST RATIO
+*> 6                 NUMBER OF VALUES OF N
+*> 0 1 2 3 5 9       VALUES OF N
+*> 3                 NUMBER OF VALUES OF ALPHA
+*> (0.0,0.0) (1.0,0.0) (0.7,-0.9)       VALUES OF ALPHA
+*> 3                 NUMBER OF VALUES OF BETA
+*> (0.0,0.0) (1.0,0.0) (1.3,-1.1)       VALUES OF BETA
+*> CGEMM  T PUT F FOR NO TEST. SAME COLUMNS.
+*> CHEMM  T PUT F FOR NO TEST. SAME COLUMNS.
+*> CSYMM  T PUT F FOR NO TEST. SAME COLUMNS.
+*> CTRMM  T PUT F FOR NO TEST. SAME COLUMNS.
+*> CTRSM  T PUT F FOR NO TEST. SAME COLUMNS.
+*> CHERK  T PUT F FOR NO TEST. SAME COLUMNS.
+*> CSYRK  T PUT F FOR NO TEST. SAME COLUMNS.
+*> CHER2K T PUT F FOR NO TEST. SAME COLUMNS.
+*> CSYR2K T PUT F FOR NO TEST. SAME COLUMNS.
+*>
+*> Further Details
+*> ===============
+*>
+*> See:
+*>
+*>    Dongarra J. J., Du Croz J. J., Duff I. S. and Hammarling S.
+*>    A Set of Level 3 Basic Linear Algebra Subprograms.
+*>
+*>    Technical Memorandum No.88 (Revision 1), Mathematics and
+*>    Computer Science Division, Argonne National Laboratory, 9700
+*>    South Cass Avenue, Argonne, Illinois 60439, US.
+*>
+*> -- Written on 8-February-1989.
+*>    Jack Dongarra, Argonne National Laboratory.
+*>    Iain Duff, AERE Harwell.
+*>    Jeremy Du Croz, Numerical Algorithms Group Ltd.
+*>    Sven Hammarling, Numerical Algorithms Group Ltd.
+*>
+*>    10-9-00:  Change STATUS='NEW' to 'UNKNOWN' so that the testers
+*>              can be run multiple times without deleting generated
+*>              output files (susan)
+*> \endverbatim
+*
+*  Authors:
+*  ========
+*
+*> \author Univ. of Tennessee
+*> \author Univ. of California Berkeley
+*> \author Univ. of Colorado Denver
+*> \author NAG Ltd.
+*
+*> \date April 2012
+*
+*> \ingroup complex_blas_testing
+*
+*  =====================================================================
       PROGRAM CBLAT3
 *
-*  Test program for the COMPLEX          Level 3 Blas.
+*  -- Reference BLAS test routine (version 3.7.0) --
+*  -- Reference BLAS is a software package provided by Univ. of Tennessee,    --
+*  -- Univ. of California Berkeley, Univ. of Colorado Denver and NAG Ltd..--
+*     April 2012
 *
-*  The program must be driven by a short data file. The first 14 records
-*  of the file are read using list-directed input, the last 9 records
-*  are read using the format ( A6, L2 ). An annotated example of a data
-*  file can be obtained by deleting the first 3 characters from the
-*  following 23 lines:
-*  'CBLAT3.SUMM'     NAME OF SUMMARY OUTPUT FILE
-*  6                 UNIT NUMBER OF SUMMARY FILE
-*  'CBLAT3.SNAP'     NAME OF SNAPSHOT OUTPUT FILE
-*  -1                UNIT NUMBER OF SNAPSHOT FILE (NOT USED IF .LT. 0)
-*  F        LOGICAL FLAG, T TO REWIND SNAPSHOT FILE AFTER EACH RECORD.
-*  F        LOGICAL FLAG, T TO STOP ON FAILURES.
-*  T        LOGICAL FLAG, T TO TEST ERROR EXITS.
-*  16.0     THRESHOLD VALUE OF TEST RATIO
-*  6                 NUMBER OF VALUES OF N
-*  0 1 2 3 5 9       VALUES OF N
-*  3                 NUMBER OF VALUES OF ALPHA
-*  (0.0,0.0) (1.0,0.0) (0.7,-0.9)       VALUES OF ALPHA
-*  3                 NUMBER OF VALUES OF BETA
-*  (0.0,0.0) (1.0,0.0) (1.3,-1.1)       VALUES OF BETA
-*  CGEMM  T PUT F FOR NO TEST. SAME COLUMNS.
-*  CHEMM  T PUT F FOR NO TEST. SAME COLUMNS.
-*  CSYMM  T PUT F FOR NO TEST. SAME COLUMNS.
-*  CTRMM  T PUT F FOR NO TEST. SAME COLUMNS.
-*  CTRSM  T PUT F FOR NO TEST. SAME COLUMNS.
-*  CHERK  T PUT F FOR NO TEST. SAME COLUMNS.
-*  CSYRK  T PUT F FOR NO TEST. SAME COLUMNS.
-*  CHER2K T PUT F FOR NO TEST. SAME COLUMNS.
-*  CSYR2K T PUT F FOR NO TEST. SAME COLUMNS.
-*
-*  See:
-*
-*     Dongarra J. J., Du Croz J. J., Duff I. S. and Hammarling S.
-*     A Set of Level 3 Basic Linear Algebra Subprograms.
-*
-*     Technical Memorandum No.88 (Revision 1), Mathematics and
-*     Computer Science Division, Argonne National Laboratory, 9700
-*     South Cass Avenue, Argonne, Illinois 60439, US.
-*
-*  -- Written on 8-February-1989.
-*     Jack Dongarra, Argonne National Laboratory.
-*     Iain Duff, AERE Harwell.
-*     Jeremy Du Croz, Numerical Algorithms Group Ltd.
-*     Sven Hammarling, Numerical Algorithms Group Ltd.
+*  =====================================================================
 *
 *     .. Parameters ..
       INTEGER            NIN
@@ -53,8 +99,8 @@
       PARAMETER          ( NSUBS = 9 )
       COMPLEX            ZERO, ONE
       PARAMETER          ( ZERO = ( 0.0, 0.0 ), ONE = ( 1.0, 0.0 ) )
-      REAL               RZERO, RHALF, RONE
-      PARAMETER          ( RZERO = 0.0, RHALF = 0.5, RONE = 1.0 )
+      REAL               RZERO
+      PARAMETER          ( RZERO = 0.0 )
       INTEGER            NMAX
       PARAMETER          ( NMAX = 65 )
       INTEGER            NIDMAX, NALMAX, NBEMAX
@@ -103,7 +149,7 @@
 *
       READ( NIN, FMT = * )SUMMRY
       READ( NIN, FMT = * )NOUT
-      OPEN( NOUT, FILE = SUMMRY, STATUS = 'NEW' )
+      OPEN( NOUT, FILE = SUMMRY )
       NOUTC = NOUT
 *
 *     Read name and unit number for snapshot output file and open file.
@@ -112,7 +158,7 @@
       READ( NIN, FMT = * )NTRA
       TRACE = NTRA.GE.0
       IF( TRACE )THEN
-         OPEN( NTRA, FILE = SNAPS, STATUS = 'NEW' )
+         OPEN( NTRA, FILE = SNAPS )
       END IF
 *     Read the flag that directs rewinding of the snapshot file.
       READ( NIN, FMT = * )REWI
@@ -189,14 +235,7 @@
 *
 *     Compute EPS (the machine precision).
 *
-      EPS = RONE
-   70 CONTINUE
-      IF( SDIFF( RONE + EPS, RONE ).EQ.RZERO )
-     $   GO TO 80
-      EPS = RHALF*EPS
-      GO TO 70
-   80 CONTINUE
-      EPS = EPS + EPS
+      EPS = EPSILON(RZERO)
       WRITE( NOUT, FMT = 9998 )EPS
 *
 *     Check the reliability of CMMCH using exact data.
@@ -1301,8 +1340,6 @@
       NC = 0
       RESET = .TRUE.
       ERRMAX = RZERO
-      RALS = RONE
-      RBETS = RONE
 *
       DO 100 IN = 1, NIDIM
          N = IDIM( IN )
@@ -1948,7 +1985,7 @@
 *
 *  Tests the error exits from the Level 3 Blas.
 *  Requires a special version of the error-handling routine XERBLA.
-*  ALPHA, RALPHA, BETA, RBETA, A, B and C should not need to be defined.
+*  A, B and C should not need to be defined.
 *
 *  Auxiliary routine for test program for Level 3 Blas.
 *
@@ -1958,12 +1995,19 @@
 *     Jeremy Du Croz, Numerical Algorithms Group Ltd.
 *     Sven Hammarling, Numerical Algorithms Group Ltd.
 *
+*  3-19-92:  Initialize ALPHA, BETA, RALPHA, and RBETA  (eca)
+*  3-19-92:  Fix argument 12 in calls to CSYMM and CHEMM
+*            with INFOT = 9  (eca)
+*
 *     .. Scalar Arguments ..
       INTEGER            ISNUM, NOUT
       CHARACTER*6        SRNAMT
 *     .. Scalars in Common ..
       INTEGER            INFOT, NOUTC
       LOGICAL            LERR, OK
+*     .. Parameters ..
+      REAL               ONE, TWO
+      PARAMETER          ( ONE = 1.0E0, TWO = 2.0E0 )
 *     .. Local Scalars ..
       COMPLEX            ALPHA, BETA
       REAL               RALPHA, RBETA
@@ -1981,6 +2025,14 @@
 *     LERR is set to .TRUE. by the special version of XERBLA each time
 *     it is called, and is then tested and re-set by CHKXER.
       LERR = .FALSE.
+*
+*     Initialize ALPHA, BETA, RALPHA, and RBETA.
+*
+      ALPHA = CMPLX( ONE, -ONE )
+      BETA = CMPLX( TWO, -TWO )
+      RALPHA = ONE
+      RBETA = TWO
+*
       GO TO ( 10, 20, 30, 40, 50, 60, 70, 80,
      $        90 )ISNUM
    10 INFOT = 1
@@ -2207,16 +2259,16 @@
       CALL CHEMM( 'R', 'L', 0, 2, ALPHA, A, 1, B, 1, BETA, C, 1 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 9
-      CALL CHEMM( 'L', 'U', 2, 0, ALPHA, A, 2, B, 1, BETA, C, 1 )
+      CALL CHEMM( 'L', 'U', 2, 0, ALPHA, A, 2, B, 1, BETA, C, 2 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 9
-      CALL CHEMM( 'R', 'U', 2, 0, ALPHA, A, 1, B, 1, BETA, C, 1 )
+      CALL CHEMM( 'R', 'U', 2, 0, ALPHA, A, 1, B, 1, BETA, C, 2 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 9
-      CALL CHEMM( 'L', 'L', 2, 0, ALPHA, A, 2, B, 1, BETA, C, 1 )
+      CALL CHEMM( 'L', 'L', 2, 0, ALPHA, A, 2, B, 1, BETA, C, 2 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 9
-      CALL CHEMM( 'R', 'L', 2, 0, ALPHA, A, 1, B, 1, BETA, C, 1 )
+      CALL CHEMM( 'R', 'L', 2, 0, ALPHA, A, 1, B, 1, BETA, C, 2 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 12
       CALL CHEMM( 'L', 'U', 2, 0, ALPHA, A, 2, B, 2, BETA, C, 1 )
@@ -2274,16 +2326,16 @@
       CALL CSYMM( 'R', 'L', 0, 2, ALPHA, A, 1, B, 1, BETA, C, 1 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 9
-      CALL CSYMM( 'L', 'U', 2, 0, ALPHA, A, 2, B, 1, BETA, C, 1 )
+      CALL CSYMM( 'L', 'U', 2, 0, ALPHA, A, 2, B, 1, BETA, C, 2 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 9
-      CALL CSYMM( 'R', 'U', 2, 0, ALPHA, A, 1, B, 1, BETA, C, 1 )
+      CALL CSYMM( 'R', 'U', 2, 0, ALPHA, A, 1, B, 1, BETA, C, 2 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 9
-      CALL CSYMM( 'L', 'L', 2, 0, ALPHA, A, 2, B, 1, BETA, C, 1 )
+      CALL CSYMM( 'L', 'L', 2, 0, ALPHA, A, 2, B, 1, BETA, C, 2 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 9
-      CALL CSYMM( 'R', 'L', 2, 0, ALPHA, A, 1, B, 1, BETA, C, 1 )
+      CALL CSYMM( 'R', 'L', 2, 0, ALPHA, A, 1, B, 1, BETA, C, 2 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 12
       CALL CSYMM( 'L', 'U', 2, 0, ALPHA, A, 2, B, 2, BETA, C, 1 )
@@ -3270,7 +3322,6 @@
    50    CONTINUE
       END IF
 *
-   60 CONTINUE
       LCERES = .TRUE.
       GO TO 80
    70 CONTINUE
diff --git a/test/dblat2.f b/test/dblat2.f
index 4002d4368..9bbbe9792 100644
--- a/test/dblat2.f
+++ b/test/dblat2.f
@@ -1,75 +1,121 @@
+*> \brief \b DBLAT2
+*
+*  =========== DOCUMENTATION ===========
+*
+* Online html documentation available at
+*            http://www.netlib.org/lapack/explore-html/
+*
+*  Definition:
+*  ===========
+*
+*       PROGRAM DBLAT2
+*
+*
+*> \par Purpose:
+*  =============
+*>
+*> \verbatim
+*>
+*> Test program for the DOUBLE PRECISION Level 2 Blas.
+*>
+*> The program must be driven by a short data file. The first 18 records
+*> of the file are read using list-directed input, the last 16 records
+*> are read using the format ( A6, L2 ). An annotated example of a data
+*> file can be obtained by deleting the first 3 characters from the
+*> following 34 lines:
+*> 'dblat2.out'      NAME OF SUMMARY OUTPUT FILE
+*> 6                 UNIT NUMBER OF SUMMARY FILE
+*> 'DBLAT2.SNAP'     NAME OF SNAPSHOT OUTPUT FILE
+*> -1                UNIT NUMBER OF SNAPSHOT FILE (NOT USED IF .LT. 0)
+*> F        LOGICAL FLAG, T TO REWIND SNAPSHOT FILE AFTER EACH RECORD.
+*> F        LOGICAL FLAG, T TO STOP ON FAILURES.
+*> T        LOGICAL FLAG, T TO TEST ERROR EXITS.
+*> 16.0     THRESHOLD VALUE OF TEST RATIO
+*> 6                 NUMBER OF VALUES OF N
+*> 0 1 2 3 5 9       VALUES OF N
+*> 4                 NUMBER OF VALUES OF K
+*> 0 1 2 4           VALUES OF K
+*> 4                 NUMBER OF VALUES OF INCX AND INCY
+*> 1 2 -1 -2         VALUES OF INCX AND INCY
+*> 3                 NUMBER OF VALUES OF ALPHA
+*> 0.0 1.0 0.7       VALUES OF ALPHA
+*> 3                 NUMBER OF VALUES OF BETA
+*> 0.0 1.0 0.9       VALUES OF BETAC
+*> DGEMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> DGBMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> DSYMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> DSBMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> DSPMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> DTRMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> DTBMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> DTPMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> DTRSV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> DTBSV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> DTPSV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> DGER   T PUT F FOR NO TEST. SAME COLUMNS.
+*> DSYR   T PUT F FOR NO TEST. SAME COLUMNS.
+*> DSPR   T PUT F FOR NO TEST. SAME COLUMNS.
+*> DSYR2  T PUT F FOR NO TEST. SAME COLUMNS.
+*> DSPR2  T PUT F FOR NO TEST. SAME COLUMNS.
+*>
+*> Further Details
+*> ===============
+*>
+*>    See:
+*>
+*>       Dongarra J. J., Du Croz J. J., Hammarling S.  and Hanson R. J..
+*>       An  extended  set of Fortran  Basic Linear Algebra Subprograms.
+*>
+*>       Technical  Memoranda  Nos. 41 (revision 3) and 81,  Mathematics
+*>       and  Computer Science  Division,  Argonne  National Laboratory,
+*>       9700 South Cass Avenue, Argonne, Illinois 60439, US.
+*>
+*>       Or
+*>
+*>       NAG  Technical Reports TR3/87 and TR4/87,  Numerical Algorithms
+*>       Group  Ltd.,  NAG  Central  Office,  256  Banbury  Road, Oxford
+*>       OX2 7DE, UK,  and  Numerical Algorithms Group Inc.,  1101  31st
+*>       Street,  Suite 100,  Downers Grove,  Illinois 60515-1263,  USA.
+*>
+*>
+*> -- Written on 10-August-1987.
+*>    Richard Hanson, Sandia National Labs.
+*>    Jeremy Du Croz, NAG Central Office.
+*>
+*>    10-9-00:  Change STATUS='NEW' to 'UNKNOWN' so that the testers
+*>              can be run multiple times without deleting generated
+*>              output files (susan)
+*> \endverbatim
+*
+*  Authors:
+*  ========
+*
+*> \author Univ. of Tennessee
+*> \author Univ. of California Berkeley
+*> \author Univ. of Colorado Denver
+*> \author NAG Ltd.
+*
+*> \date April 2012
+*
+*> \ingroup double_blas_testing
+*
+*  =====================================================================
       PROGRAM DBLAT2
 *
-*  Test program for the DOUBLE PRECISION Level 2 Blas.
+*  -- Reference BLAS test routine (version 3.7.0) --
+*  -- Reference BLAS is a software package provided by Univ. of Tennessee,    --
+*  -- Univ. of California Berkeley, Univ. of Colorado Denver and NAG Ltd..--
+*     April 2012
 *
-*  The program must be driven by a short data file. The first 18 records
-*  of the file are read using list-directed input, the last 16 records
-*  are read using the format ( A6, L2 ). An annotated example of a data
-*  file can be obtained by deleting the first 3 characters from the
-*  following 34 lines:
-*  'DBLAT2.SUMM'     NAME OF SUMMARY OUTPUT FILE
-*  6                 UNIT NUMBER OF SUMMARY FILE
-*  'DBLAT2.SNAP'     NAME OF SNAPSHOT OUTPUT FILE
-*  -1                UNIT NUMBER OF SNAPSHOT FILE (NOT USED IF .LT. 0)
-*  F        LOGICAL FLAG, T TO REWIND SNAPSHOT FILE AFTER EACH RECORD.
-*  F        LOGICAL FLAG, T TO STOP ON FAILURES.
-*  T        LOGICAL FLAG, T TO TEST ERROR EXITS.
-*  16.0     THRESHOLD VALUE OF TEST RATIO
-*  6                 NUMBER OF VALUES OF N
-*  0 1 2 3 5 9       VALUES OF N
-*  4                 NUMBER OF VALUES OF K
-*  0 1 2 4           VALUES OF K
-*  4                 NUMBER OF VALUES OF INCX AND INCY
-*  1 2 -1 -2         VALUES OF INCX AND INCY
-*  3                 NUMBER OF VALUES OF ALPHA
-*  0.0 1.0 0.7       VALUES OF ALPHA
-*  3                 NUMBER OF VALUES OF BETA
-*  0.0 1.0 0.9       VALUES OF BETA
-*  DGEMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  DGBMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  DSYMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  DSBMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  DSPMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  DTRMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  DTBMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  DTPMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  DTRSV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  DTBSV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  DTPSV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  DGER   T PUT F FOR NO TEST. SAME COLUMNS.
-*  DSYR   T PUT F FOR NO TEST. SAME COLUMNS.
-*  DSPR   T PUT F FOR NO TEST. SAME COLUMNS.
-*  DSYR2  T PUT F FOR NO TEST. SAME COLUMNS.
-*  DSPR2  T PUT F FOR NO TEST. SAME COLUMNS.
-*
-*     See:
-*
-*        Dongarra J. J., Du Croz J. J., Hammarling S.  and Hanson R. J..
-*        An  extended  set of Fortran  Basic Linear Algebra Subprograms.
-*
-*        Technical  Memoranda  Nos. 41 (revision 3) and 81,  Mathematics
-*        and  Computer Science  Division,  Argonne  National Laboratory,
-*        9700 South Cass Avenue, Argonne, Illinois 60439, US.
-*
-*        Or
-*
-*        NAG  Technical Reports TR3/87 and TR4/87,  Numerical Algorithms
-*        Group  Ltd.,  NAG  Central  Office,  256  Banbury  Road, Oxford
-*        OX2 7DE, UK,  and  Numerical Algorithms Group Inc.,  1101  31st
-*        Street,  Suite 100,  Downers Grove,  Illinois 60515-1263,  USA.
-*
-*
-*  -- Written on 10-August-1987.
-*     Richard Hanson, Sandia National Labs.
-*     Jeremy Du Croz, NAG Central Office.
+*  =====================================================================
 *
 *     .. Parameters ..
       INTEGER            NIN
       PARAMETER          ( NIN = 5 )
       INTEGER            NSUBS
       PARAMETER          ( NSUBS = 16 )
-      DOUBLE PRECISION   ZERO, HALF, ONE
-      PARAMETER          ( ZERO = 0.0D0, HALF = 0.5D0, ONE = 1.0D0 )
+      DOUBLE PRECISION   ZERO, ONE
+      PARAMETER          ( ZERO = 0.0D0, ONE = 1.0D0 )
       INTEGER            NMAX, INCMAX
       PARAMETER          ( NMAX = 65, INCMAX = 2 )
       INTEGER            NINMAX, NIDMAX, NKBMAX, NALMAX, NBEMAX
@@ -121,7 +167,7 @@
 *
       READ( NIN, FMT = * )SUMMRY
       READ( NIN, FMT = * )NOUT
-      OPEN( NOUT, FILE = SUMMRY, STATUS = 'NEW' )
+      OPEN( NOUT, FILE = SUMMRY, STATUS = 'UNKNOWN' )
       NOUTC = NOUT
 *
 *     Read name and unit number for snapshot output file and open file.
@@ -130,7 +176,7 @@
       READ( NIN, FMT = * )NTRA
       TRACE = NTRA.GE.0
       IF( TRACE )THEN
-         OPEN( NTRA, FILE = SNAPS, STATUS = 'NEW' )
+         OPEN( NTRA, FILE = SNAPS, STATUS = 'UNKNOWN' )
       END IF
 *     Read the flag that directs rewinding of the snapshot file.
       READ( NIN, FMT = * )REWI
@@ -235,14 +281,7 @@
 *
 *     Compute EPS (the machine precision).
 *
-      EPS = ONE
-   90 CONTINUE
-      IF( DDIFF( ONE + EPS, ONE ).EQ.ZERO )
-     $   GO TO 100
-      EPS = HALF*EPS
-      GO TO 90
-  100 CONTINUE
-      EPS = EPS + EPS
+      EPS = EPSILON(ZERO)
       WRITE( NOUT, FMT = 9998 )EPS
 *
 *     Check the reliability of DMVCH using exact data.
@@ -2982,7 +3021,6 @@
    50    CONTINUE
       END IF
 *
-   60 CONTINUE
       LDERES = .TRUE.
       GO TO 80
    70 CONTINUE
diff --git a/test/dblat3.f b/test/dblat3.f
index 082e03e5e..1ebec4ffa 100644
--- a/test/dblat3.f
+++ b/test/dblat3.f
@@ -1,55 +1,101 @@
+*> \brief \b DBLAT3
+*
+*  =========== DOCUMENTATION ===========
+*
+* Online html documentation available at
+*            http://www.netlib.org/lapack/explore-html/
+*
+*  Definition:
+*  ===========
+*
+*       PROGRAM DBLAT3
+*
+*
+*> \par Purpose:
+*  =============
+*>
+*> \verbatim
+*>
+*> Test program for the DOUBLE PRECISION Level 3 Blas.
+*>
+*> The program must be driven by a short data file. The first 14 records
+*> of the file are read using list-directed input, the last 6 records
+*> are read using the format ( A6, L2 ). An annotated example of a data
+*> file can be obtained by deleting the first 3 characters from the
+*> following 20 lines:
+*> 'dblat3.out'      NAME OF SUMMARY OUTPUT FILE
+*> 6                 UNIT NUMBER OF SUMMARY FILE
+*> 'DBLAT3.SNAP'     NAME OF SNAPSHOT OUTPUT FILE
+*> -1                UNIT NUMBER OF SNAPSHOT FILE (NOT USED IF .LT. 0)
+*> F        LOGICAL FLAG, T TO REWIND SNAPSHOT FILE AFTER EACH RECORD.
+*> F        LOGICAL FLAG, T TO STOP ON FAILURES.
+*> T        LOGICAL FLAG, T TO TEST ERROR EXITS.
+*> 16.0     THRESHOLD VALUE OF TEST RATIO
+*> 6                 NUMBER OF VALUES OF N
+*> 0 1 2 3 5 9       VALUES OF N
+*> 3                 NUMBER OF VALUES OF ALPHA
+*> 0.0 1.0 0.7       VALUES OF ALPHA
+*> 3                 NUMBER OF VALUES OF BETA
+*> 0.0 1.0 1.3       VALUES OF BETA
+*> DGEMM  T PUT F FOR NO TEST. SAME COLUMNS.
+*> DSYMM  T PUT F FOR NO TEST. SAME COLUMNS.
+*> DTRMM  T PUT F FOR NO TEST. SAME COLUMNS.
+*> DTRSM  T PUT F FOR NO TEST. SAME COLUMNS.
+*> DSYRK  T PUT F FOR NO TEST. SAME COLUMNS.
+*> DSYR2K T PUT F FOR NO TEST. SAME COLUMNS.
+*>
+*> Further Details
+*> ===============
+*>
+*> See:
+*>
+*>    Dongarra J. J., Du Croz J. J., Duff I. S. and Hammarling S.
+*>    A Set of Level 3 Basic Linear Algebra Subprograms.
+*>
+*>    Technical Memorandum No.88 (Revision 1), Mathematics and
+*>    Computer Science Division, Argonne National Laboratory, 9700
+*>    South Cass Avenue, Argonne, Illinois 60439, US.
+*>
+*> -- Written on 8-February-1989.
+*>    Jack Dongarra, Argonne National Laboratory.
+*>    Iain Duff, AERE Harwell.
+*>    Jeremy Du Croz, Numerical Algorithms Group Ltd.
+*>    Sven Hammarling, Numerical Algorithms Group Ltd.
+*>
+*>    10-9-00:  Change STATUS='NEW' to 'UNKNOWN' so that the testers
+*>              can be run multiple times without deleting generated
+*>              output files (susan)
+*> \endverbatim
+*
+*  Authors:
+*  ========
+*
+*> \author Univ. of Tennessee
+*> \author Univ. of California Berkeley
+*> \author Univ. of Colorado Denver
+*> \author NAG Ltd.
+*
+*> \date April 2012
+*
+*> \ingroup double_blas_testing
+*
+*  =====================================================================
       PROGRAM DBLAT3
 *
-*  Test program for the DOUBLE PRECISION Level 3 Blas.
+*  -- Reference BLAS test routine (version 3.7.0) --
+*  -- Reference BLAS is a software package provided by Univ. of Tennessee,    --
+*  -- Univ. of California Berkeley, Univ. of Colorado Denver and NAG Ltd..--
+*     April 2012
 *
-*  The program must be driven by a short data file. The first 14 records
-*  of the file are read using list-directed input, the last 6 records
-*  are read using the format ( A6, L2 ). An annotated example of a data
-*  file can be obtained by deleting the first 3 characters from the
-*  following 20 lines:
-*  'DBLAT3.SUMM'     NAME OF SUMMARY OUTPUT FILE
-*  6                 UNIT NUMBER OF SUMMARY FILE
-*  'DBLAT3.SNAP'     NAME OF SNAPSHOT OUTPUT FILE
-*  -1                UNIT NUMBER OF SNAPSHOT FILE (NOT USED IF .LT. 0)
-*  F        LOGICAL FLAG, T TO REWIND SNAPSHOT FILE AFTER EACH RECORD.
-*  F        LOGICAL FLAG, T TO STOP ON FAILURES.
-*  T        LOGICAL FLAG, T TO TEST ERROR EXITS.
-*  16.0     THRESHOLD VALUE OF TEST RATIO
-*  6                 NUMBER OF VALUES OF N
-*  0 1 2 3 5 9       VALUES OF N
-*  3                 NUMBER OF VALUES OF ALPHA
-*  0.0 1.0 0.7       VALUES OF ALPHA
-*  3                 NUMBER OF VALUES OF BETA
-*  0.0 1.0 1.3       VALUES OF BETA
-*  DGEMM  T PUT F FOR NO TEST. SAME COLUMNS.
-*  DSYMM  T PUT F FOR NO TEST. SAME COLUMNS.
-*  DTRMM  T PUT F FOR NO TEST. SAME COLUMNS.
-*  DTRSM  T PUT F FOR NO TEST. SAME COLUMNS.
-*  DSYRK  T PUT F FOR NO TEST. SAME COLUMNS.
-*  DSYR2K T PUT F FOR NO TEST. SAME COLUMNS.
-*
-*  See:
-*
-*     Dongarra J. J., Du Croz J. J., Duff I. S. and Hammarling S.
-*     A Set of Level 3 Basic Linear Algebra Subprograms.
-*
-*     Technical Memorandum No.88 (Revision 1), Mathematics and
-*     Computer Science Division, Argonne National Laboratory, 9700
-*     South Cass Avenue, Argonne, Illinois 60439, US.
-*
-*  -- Written on 8-February-1989.
-*     Jack Dongarra, Argonne National Laboratory.
-*     Iain Duff, AERE Harwell.
-*     Jeremy Du Croz, Numerical Algorithms Group Ltd.
-*     Sven Hammarling, Numerical Algorithms Group Ltd.
+*  =====================================================================
 *
 *     .. Parameters ..
       INTEGER            NIN
       PARAMETER          ( NIN = 5 )
       INTEGER            NSUBS
       PARAMETER          ( NSUBS = 6 )
-      DOUBLE PRECISION   ZERO, HALF, ONE
-      PARAMETER          ( ZERO = 0.0D0, HALF = 0.5D0, ONE = 1.0D0 )
+      DOUBLE PRECISION   ZERO, ONE
+      PARAMETER          ( ZERO = 0.0D0, ONE = 1.0D0 )
       INTEGER            NMAX
       PARAMETER          ( NMAX = 65 )
       INTEGER            NIDMAX, NALMAX, NBEMAX
@@ -96,7 +142,7 @@
 *
       READ( NIN, FMT = * )SUMMRY
       READ( NIN, FMT = * )NOUT
-      OPEN( NOUT, FILE = SUMMRY, STATUS = 'NEW' )
+      OPEN( NOUT, FILE = SUMMRY, STATUS = 'UNKNOWN' )
       NOUTC = NOUT
 *
 *     Read name and unit number for snapshot output file and open file.
@@ -105,7 +151,7 @@
       READ( NIN, FMT = * )NTRA
       TRACE = NTRA.GE.0
       IF( TRACE )THEN
-         OPEN( NTRA, FILE = SNAPS, STATUS = 'NEW' )
+         OPEN( NTRA, FILE = SNAPS, STATUS = 'UNKNOWN' )
       END IF
 *     Read the flag that directs rewinding of the snapshot file.
       READ( NIN, FMT = * )REWI
@@ -182,14 +228,7 @@
 *
 *     Compute EPS (the machine precision).
 *
-      EPS = ONE
-   70 CONTINUE
-      IF( DDIFF( ONE + EPS, ONE ).EQ.ZERO )
-     $   GO TO 80
-      EPS = HALF*EPS
-      GO TO 70
-   80 CONTINUE
-      EPS = EPS + EPS
+      EPS = EPSILON(ZERO)
       WRITE( NOUT, FMT = 9998 )EPS
 *
 *     Check the reliability of DMMCH using exact data.
@@ -1802,7 +1841,7 @@
 *
 *  Tests the error exits from the Level 3 Blas.
 *  Requires a special version of the error-handling routine XERBLA.
-*  ALPHA, BETA, A, B and C should not need to be defined.
+*  A, B and C should not need to be defined.
 *
 *  Auxiliary routine for test program for Level 3 Blas.
 *
@@ -1812,12 +1851,18 @@
 *     Jeremy Du Croz, Numerical Algorithms Group Ltd.
 *     Sven Hammarling, Numerical Algorithms Group Ltd.
 *
+*  3-19-92:  Initialize ALPHA and BETA  (eca)
+*  3-19-92:  Fix argument 12 in calls to SSYMM with INFOT = 9  (eca)
+*
 *     .. Scalar Arguments ..
       INTEGER            ISNUM, NOUT
       CHARACTER*6        SRNAMT
 *     .. Scalars in Common ..
       INTEGER            INFOT, NOUTC
       LOGICAL            LERR, OK
+*     .. Parameters ..
+      DOUBLE PRECISION   ONE, TWO
+      PARAMETER          ( ONE = 1.0D0, TWO = 2.0D0 )
 *     .. Local Scalars ..
       DOUBLE PRECISION   ALPHA, BETA
 *     .. Local Arrays ..
@@ -1834,6 +1879,12 @@
 *     LERR is set to .TRUE. by the special version of XERBLA each time
 *     it is called, and is then tested and re-set by CHKXER.
       LERR = .FALSE.
+*
+*     Initialize ALPHA and BETA.
+*
+      ALPHA = ONE
+      BETA = TWO
+*
       GO TO ( 10, 20, 30, 40, 50, 60 )ISNUM
    10 INFOT = 1
       CALL DGEMM( '/', 'N', 0, 0, 0, ALPHA, A, 1, B, 1, BETA, C, 1 )
@@ -1963,16 +2014,16 @@
       CALL DSYMM( 'R', 'L', 0, 2, ALPHA, A, 1, B, 1, BETA, C, 1 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 9
-      CALL DSYMM( 'L', 'U', 2, 0, ALPHA, A, 2, B, 1, BETA, C, 1 )
+      CALL DSYMM( 'L', 'U', 2, 0, ALPHA, A, 2, B, 1, BETA, C, 2 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 9
-      CALL DSYMM( 'R', 'U', 2, 0, ALPHA, A, 1, B, 1, BETA, C, 1 )
+      CALL DSYMM( 'R', 'U', 2, 0, ALPHA, A, 1, B, 1, BETA, C, 2 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 9
-      CALL DSYMM( 'L', 'L', 2, 0, ALPHA, A, 2, B, 1, BETA, C, 1 )
+      CALL DSYMM( 'L', 'L', 2, 0, ALPHA, A, 2, B, 1, BETA, C, 2 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 9
-      CALL DSYMM( 'R', 'L', 2, 0, ALPHA, A, 1, B, 1, BETA, C, 1 )
+      CALL DSYMM( 'R', 'L', 2, 0, ALPHA, A, 1, B, 1, BETA, C, 2 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 12
       CALL DSYMM( 'L', 'U', 2, 0, ALPHA, A, 2, B, 2, BETA, C, 1 )
@@ -2660,7 +2711,6 @@
    50    CONTINUE
       END IF
 *
-   60 CONTINUE
       LDERES = .TRUE.
       GO TO 80
    70 CONTINUE
diff --git a/test/sblat2.f b/test/sblat2.f
index a1074be52..56ead8640 100644
--- a/test/sblat2.f
+++ b/test/sblat2.f
@@ -1,75 +1,121 @@
+*> \brief \b SBLAT2
+*
+*  =========== DOCUMENTATION ===========
+*
+* Online html documentation available at
+*            http://www.netlib.org/lapack/explore-html/
+*
+*  Definition:
+*  ===========
+*
+*       PROGRAM SBLAT2
+*
+*
+*> \par Purpose:
+*  =============
+*>
+*> \verbatim
+*>
+*> Test program for the REAL Level 2 Blas.
+*>
+*> The program must be driven by a short data file. The first 18 records
+*> of the file are read using list-directed input, the last 16 records
+*> are read using the format ( A6, L2 ). An annotated example of a data
+*> file can be obtained by deleting the first 3 characters from the
+*> following 34 lines:
+*> 'sblat2.out'      NAME OF SUMMARY OUTPUT FILE
+*> 6                 UNIT NUMBER OF SUMMARY FILE
+*> 'SBLAT2.SNAP'     NAME OF SNAPSHOT OUTPUT FILE
+*> -1                UNIT NUMBER OF SNAPSHOT FILE (NOT USED IF .LT. 0)
+*> F        LOGICAL FLAG, T TO REWIND SNAPSHOT FILE AFTER EACH RECORD.
+*> F        LOGICAL FLAG, T TO STOP ON FAILURES.
+*> T        LOGICAL FLAG, T TO TEST ERROR EXITS.
+*> 16.0     THRESHOLD VALUE OF TEST RATIO
+*> 6                 NUMBER OF VALUES OF N
+*> 0 1 2 3 5 9       VALUES OF N
+*> 4                 NUMBER OF VALUES OF K
+*> 0 1 2 4           VALUES OF K
+*> 4                 NUMBER OF VALUES OF INCX AND INCY
+*> 1 2 -1 -2         VALUES OF INCX AND INCY
+*> 3                 NUMBER OF VALUES OF ALPHA
+*> 0.0 1.0 0.7       VALUES OF ALPHA
+*> 3                 NUMBER OF VALUES OF BETA
+*> 0.0 1.0 0.9       VALUES OF BETA
+*> SGEMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> SGBMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> SSYMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> SSBMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> SSPMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> STRMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> STBMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> STPMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> STRSV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> STBSV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> STPSV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> SGER   T PUT F FOR NO TEST. SAME COLUMNS.
+*> SSYR   T PUT F FOR NO TEST. SAME COLUMNS.
+*> SSPR   T PUT F FOR NO TEST. SAME COLUMNS.
+*> SSYR2  T PUT F FOR NO TEST. SAME COLUMNS.
+*> SSPR2  T PUT F FOR NO TEST. SAME COLUMNS.
+*>
+*> Further Details
+*> ===============
+*>
+*>    See:
+*>
+*>       Dongarra J. J., Du Croz J. J., Hammarling S.  and Hanson R. J..
+*>       An  extended  set of Fortran  Basic Linear Algebra Subprograms.
+*>
+*>       Technical  Memoranda  Nos. 41 (revision 3) and 81,  Mathematics
+*>       and  Computer Science  Division,  Argonne  National Laboratory,
+*>       9700 South Cass Avenue, Argonne, Illinois 60439, US.
+*>
+*>       Or
+*>
+*>       NAG  Technical Reports TR3/87 and TR4/87,  Numerical Algorithms
+*>       Group  Ltd.,  NAG  Central  Office,  256  Banbury  Road, Oxford
+*>       OX2 7DE, UK,  and  Numerical Algorithms Group Inc.,  1101  31st
+*>       Street,  Suite 100,  Downers Grove,  Illinois 60515-1263,  USA.
+*>
+*>
+*> -- Written on 10-August-1987.
+*>    Richard Hanson, Sandia National Labs.
+*>    Jeremy Du Croz, NAG Central Office.
+*>
+*>    10-9-00:  Change STATUS='NEW' to 'UNKNOWN' so that the testers
+*>              can be run multiple times without deleting generated
+*>              output files (susan)
+*> \endverbatim
+*
+*  Authors:
+*  ========
+*
+*> \author Univ. of Tennessee
+*> \author Univ. of California Berkeley
+*> \author Univ. of Colorado Denver
+*> \author NAG Ltd.
+*
+*> \date April 2012
+*
+*> \ingroup single_blas_testing
+*
+*  =====================================================================
       PROGRAM SBLAT2
 *
-*  Test program for the REAL             Level 2 Blas.
+*  -- Reference BLAS test routine (version 3.7.0) --
+*  -- Reference BLAS is a software package provided by Univ. of Tennessee,    --
+*  -- Univ. of California Berkeley, Univ. of Colorado Denver and NAG Ltd..--
+*     April 2012
 *
-*  The program must be driven by a short data file. The first 18 records
-*  of the file are read using list-directed input, the last 16 records
-*  are read using the format ( A6, L2 ). An annotated example of a data
-*  file can be obtained by deleting the first 3 characters from the
-*  following 34 lines:
-*  'SBLAT2.SUMM'     NAME OF SUMMARY OUTPUT FILE
-*  6                 UNIT NUMBER OF SUMMARY FILE
-*  'SBLAT2.SNAP'     NAME OF SNAPSHOT OUTPUT FILE
-*  -1                UNIT NUMBER OF SNAPSHOT FILE (NOT USED IF .LT. 0)
-*  F        LOGICAL FLAG, T TO REWIND SNAPSHOT FILE AFTER EACH RECORD.
-*  F        LOGICAL FLAG, T TO STOP ON FAILURES.
-*  T        LOGICAL FLAG, T TO TEST ERROR EXITS.
-*  16.0     THRESHOLD VALUE OF TEST RATIO
-*  6                 NUMBER OF VALUES OF N
-*  0 1 2 3 5 9       VALUES OF N
-*  4                 NUMBER OF VALUES OF K
-*  0 1 2 4           VALUES OF K
-*  4                 NUMBER OF VALUES OF INCX AND INCY
-*  1 2 -1 -2         VALUES OF INCX AND INCY
-*  3                 NUMBER OF VALUES OF ALPHA
-*  0.0 1.0 0.7       VALUES OF ALPHA
-*  3                 NUMBER OF VALUES OF BETA
-*  0.0 1.0 0.9       VALUES OF BETA
-*  SGEMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  SGBMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  SSYMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  SSBMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  SSPMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  STRMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  STBMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  STPMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  STRSV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  STBSV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  STPSV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  SGER   T PUT F FOR NO TEST. SAME COLUMNS.
-*  SSYR   T PUT F FOR NO TEST. SAME COLUMNS.
-*  SSPR   T PUT F FOR NO TEST. SAME COLUMNS.
-*  SSYR2  T PUT F FOR NO TEST. SAME COLUMNS.
-*  SSPR2  T PUT F FOR NO TEST. SAME COLUMNS.
-*
-*     See:
-*
-*        Dongarra J. J., Du Croz J. J., Hammarling S.  and Hanson R. J..
-*        An  extended  set of Fortran  Basic Linear Algebra Subprograms.
-*
-*        Technical  Memoranda  Nos. 41 (revision 3) and 81,  Mathematics
-*        and  Computer Science  Division,  Argonne  National Laboratory,
-*        9700 South Cass Avenue, Argonne, Illinois 60439, US.
-*
-*        Or
-*
-*        NAG  Technical Reports TR3/87 and TR4/87,  Numerical Algorithms
-*        Group  Ltd.,  NAG  Central  Office,  256  Banbury  Road, Oxford
-*        OX2 7DE, UK,  and  Numerical Algorithms Group Inc.,  1101  31st
-*        Street,  Suite 100,  Downers Grove,  Illinois 60515-1263,  USA.
-*
-*
-*  -- Written on 10-August-1987.
-*     Richard Hanson, Sandia National Labs.
-*     Jeremy Du Croz, NAG Central Office.
+*  =====================================================================
 *
 *     .. Parameters ..
       INTEGER            NIN
       PARAMETER          ( NIN = 5 )
       INTEGER            NSUBS
       PARAMETER          ( NSUBS = 16 )
-      REAL               ZERO, HALF, ONE
-      PARAMETER          ( ZERO = 0.0, HALF = 0.5, ONE = 1.0 )
+      REAL               ZERO, ONE
+      PARAMETER          ( ZERO = 0.0, ONE = 1.0 )
       INTEGER            NMAX, INCMAX
       PARAMETER          ( NMAX = 65, INCMAX = 2 )
       INTEGER            NINMAX, NIDMAX, NKBMAX, NALMAX, NBEMAX
@@ -121,7 +167,7 @@
 *
       READ( NIN, FMT = * )SUMMRY
       READ( NIN, FMT = * )NOUT
-      OPEN( NOUT, FILE = SUMMRY, STATUS = 'NEW' )
+      OPEN( NOUT, FILE = SUMMRY, STATUS = 'UNKNOWN' )
       NOUTC = NOUT
 *
 *     Read name and unit number for snapshot output file and open file.
@@ -130,7 +176,7 @@
       READ( NIN, FMT = * )NTRA
       TRACE = NTRA.GE.0
       IF( TRACE )THEN
-         OPEN( NTRA, FILE = SNAPS, STATUS = 'NEW' )
+         OPEN( NTRA, FILE = SNAPS, STATUS = 'UNKNOWN' )
       END IF
 *     Read the flag that directs rewinding of the snapshot file.
       READ( NIN, FMT = * )REWI
@@ -235,14 +281,7 @@
 *
 *     Compute EPS (the machine precision).
 *
-      EPS = ONE
-   90 CONTINUE
-      IF( SDIFF( ONE + EPS, ONE ).EQ.ZERO )
-     $   GO TO 100
-      EPS = HALF*EPS
-      GO TO 90
-  100 CONTINUE
-      EPS = EPS + EPS
+      EPS = EPSILON(ZERO)
       WRITE( NOUT, FMT = 9998 )EPS
 *
 *     Check the reliability of SMVCH using exact data.
@@ -2982,7 +3021,6 @@
    50    CONTINUE
       END IF
 *
-   60 CONTINUE
       LSERES = .TRUE.
       GO TO 80
    70 CONTINUE
diff --git a/test/sblat3.f b/test/sblat3.f
index 325a9eb92..66edac14e 100644
--- a/test/sblat3.f
+++ b/test/sblat3.f
@@ -1,55 +1,101 @@
+*> \brief \b SBLAT3
+*
+*  =========== DOCUMENTATION ===========
+*
+* Online html documentation available at
+*            http://www.netlib.org/lapack/explore-html/
+*
+*  Definition:
+*  ===========
+*
+*       PROGRAM SBLAT3
+*
+*
+*> \par Purpose:
+*  =============
+*>
+*> \verbatim
+*>
+*> Test program for the REAL             Level 3 Blas.
+*>
+*> The program must be driven by a short data file. The first 14 records
+*> of the file are read using list-directed input, the last 6 records
+*> are read using the format ( A6, L2 ). An annotated example of a data
+*> file can be obtained by deleting the first 3 characters from the
+*> following 20 lines:
+*> 'sblat3.out'      NAME OF SUMMARY OUTPUT FILE
+*> 6                 UNIT NUMBER OF SUMMARY FILE
+*> 'SBLAT3.SNAP'     NAME OF SNAPSHOT OUTPUT FILE
+*> -1                UNIT NUMBER OF SNAPSHOT FILE (NOT USED IF .LT. 0)
+*> F        LOGICAL FLAG, T TO REWIND SNAPSHOT FILE AFTER EACH RECORD.
+*> F        LOGICAL FLAG, T TO STOP ON FAILURES.
+*> T        LOGICAL FLAG, T TO TEST ERROR EXITS.
+*> 16.0     THRESHOLD VALUE OF TEST RATIO
+*> 6                 NUMBER OF VALUES OF N
+*> 0 1 2 3 5 9       VALUES OF N
+*> 3                 NUMBER OF VALUES OF ALPHA
+*> 0.0 1.0 0.7       VALUES OF ALPHA
+*> 3                 NUMBER OF VALUES OF BETA
+*> 0.0 1.0 1.3       VALUES OF BETA
+*> SGEMM  T PUT F FOR NO TEST. SAME COLUMNS.
+*> SSYMM  T PUT F FOR NO TEST. SAME COLUMNS.
+*> STRMM  T PUT F FOR NO TEST. SAME COLUMNS.
+*> STRSM  T PUT F FOR NO TEST. SAME COLUMNS.
+*> SSYRK  T PUT F FOR NO TEST. SAME COLUMNS.
+*> SSYR2K T PUT F FOR NO TEST. SAME COLUMNS.
+*>
+*> Further Details
+*> ===============
+*>
+*> See:
+*>
+*>    Dongarra J. J., Du Croz J. J., Duff I. S. and Hammarling S.
+*>    A Set of Level 3 Basic Linear Algebra Subprograms.
+*>
+*>    Technical Memorandum No.88 (Revision 1), Mathematics and
+*>    Computer Science Division, Argonne National Laboratory, 9700
+*>    South Cass Avenue, Argonne, Illinois 60439, US.
+*>
+*> -- Written on 8-February-1989.
+*>    Jack Dongarra, Argonne National Laboratory.
+*>    Iain Duff, AERE Harwell.
+*>    Jeremy Du Croz, Numerical Algorithms Group Ltd.
+*>    Sven Hammarling, Numerical Algorithms Group Ltd.
+*>
+*>    10-9-00:  Change STATUS='NEW' to 'UNKNOWN' so that the testers
+*>              can be run multiple times without deleting generated
+*>              output files (susan)
+*> \endverbatim
+*
+*  Authors:
+*  ========
+*
+*> \author Univ. of Tennessee
+*> \author Univ. of California Berkeley
+*> \author Univ. of Colorado Denver
+*> \author NAG Ltd.
+*
+*> \date April 2012
+*
+*> \ingroup single_blas_testing
+*
+*  =====================================================================
       PROGRAM SBLAT3
 *
-*  Test program for the REAL             Level 3 Blas.
+*  -- Reference BLAS test routine (version 3.7.0) --
+*  -- Reference BLAS is a software package provided by Univ. of Tennessee,    --
+*  -- Univ. of California Berkeley, Univ. of Colorado Denver and NAG Ltd..--
+*     April 2012
 *
-*  The program must be driven by a short data file. The first 14 records
-*  of the file are read using list-directed input, the last 6 records
-*  are read using the format ( A6, L2 ). An annotated example of a data
-*  file can be obtained by deleting the first 3 characters from the
-*  following 20 lines:
-*  'SBLAT3.SUMM'     NAME OF SUMMARY OUTPUT FILE
-*  6                 UNIT NUMBER OF SUMMARY FILE
-*  'SBLAT3.SNAP'     NAME OF SNAPSHOT OUTPUT FILE
-*  -1                UNIT NUMBER OF SNAPSHOT FILE (NOT USED IF .LT. 0)
-*  F        LOGICAL FLAG, T TO REWIND SNAPSHOT FILE AFTER EACH RECORD.
-*  F        LOGICAL FLAG, T TO STOP ON FAILURES.
-*  T        LOGICAL FLAG, T TO TEST ERROR EXITS.
-*  16.0     THRESHOLD VALUE OF TEST RATIO
-*  6                 NUMBER OF VALUES OF N
-*  0 1 2 3 5 9       VALUES OF N
-*  3                 NUMBER OF VALUES OF ALPHA
-*  0.0 1.0 0.7       VALUES OF ALPHA
-*  3                 NUMBER OF VALUES OF BETA
-*  0.0 1.0 1.3       VALUES OF BETA
-*  SGEMM  T PUT F FOR NO TEST. SAME COLUMNS.
-*  SSYMM  T PUT F FOR NO TEST. SAME COLUMNS.
-*  STRMM  T PUT F FOR NO TEST. SAME COLUMNS.
-*  STRSM  T PUT F FOR NO TEST. SAME COLUMNS.
-*  SSYRK  T PUT F FOR NO TEST. SAME COLUMNS.
-*  SSYR2K T PUT F FOR NO TEST. SAME COLUMNS.
-*
-*  See:
-*
-*     Dongarra J. J., Du Croz J. J., Duff I. S. and Hammarling S.
-*     A Set of Level 3 Basic Linear Algebra Subprograms.
-*
-*     Technical Memorandum No.88 (Revision 1), Mathematics and
-*     Computer Science Division, Argonne National Laboratory, 9700
-*     South Cass Avenue, Argonne, Illinois 60439, US.
-*
-*  -- Written on 8-February-1989.
-*     Jack Dongarra, Argonne National Laboratory.
-*     Iain Duff, AERE Harwell.
-*     Jeremy Du Croz, Numerical Algorithms Group Ltd.
-*     Sven Hammarling, Numerical Algorithms Group Ltd.
+*  =====================================================================
 *
 *     .. Parameters ..
       INTEGER            NIN
       PARAMETER          ( NIN = 5 )
       INTEGER            NSUBS
       PARAMETER          ( NSUBS = 6 )
-      REAL               ZERO, HALF, ONE
-      PARAMETER          ( ZERO = 0.0, HALF = 0.5, ONE = 1.0 )
+      REAL               ZERO, ONE
+      PARAMETER          ( ZERO = 0.0, ONE = 1.0 )
       INTEGER            NMAX
       PARAMETER          ( NMAX = 65 )
       INTEGER            NIDMAX, NALMAX, NBEMAX
@@ -96,7 +142,7 @@
 *
       READ( NIN, FMT = * )SUMMRY
       READ( NIN, FMT = * )NOUT
-      OPEN( NOUT, FILE = SUMMRY, STATUS = 'NEW' )
+      OPEN( NOUT, FILE = SUMMRY )
       NOUTC = NOUT
 *
 *     Read name and unit number for snapshot output file and open file.
@@ -105,7 +151,7 @@
       READ( NIN, FMT = * )NTRA
       TRACE = NTRA.GE.0
       IF( TRACE )THEN
-         OPEN( NTRA, FILE = SNAPS, STATUS = 'NEW' )
+         OPEN( NTRA, FILE = SNAPS )
       END IF
 *     Read the flag that directs rewinding of the snapshot file.
       READ( NIN, FMT = * )REWI
@@ -182,14 +228,7 @@
 *
 *     Compute EPS (the machine precision).
 *
-      EPS = ONE
-   70 CONTINUE
-      IF( SDIFF( ONE + EPS, ONE ).EQ.ZERO )
-     $   GO TO 80
-      EPS = HALF*EPS
-      GO TO 70
-   80 CONTINUE
-      EPS = EPS + EPS
+      EPS = EPSILON(ZERO)
       WRITE( NOUT, FMT = 9998 )EPS
 *
 *     Check the reliability of SMMCH using exact data.
@@ -1802,7 +1841,7 @@
 *
 *  Tests the error exits from the Level 3 Blas.
 *  Requires a special version of the error-handling routine XERBLA.
-*  ALPHA, BETA, A, B and C should not need to be defined.
+*  A, B and C should not need to be defined.
 *
 *  Auxiliary routine for test program for Level 3 Blas.
 *
@@ -1812,12 +1851,18 @@
 *     Jeremy Du Croz, Numerical Algorithms Group Ltd.
 *     Sven Hammarling, Numerical Algorithms Group Ltd.
 *
+*  3-19-92:  Initialize ALPHA and BETA  (eca)
+*  3-19-92:  Fix argument 12 in calls to SSYMM with INFOT = 9  (eca)
+*
 *     .. Scalar Arguments ..
       INTEGER            ISNUM, NOUT
       CHARACTER*6        SRNAMT
 *     .. Scalars in Common ..
       INTEGER            INFOT, NOUTC
       LOGICAL            LERR, OK
+*     .. Parameters ..
+      REAL               ONE, TWO
+      PARAMETER          ( ONE = 1.0E0, TWO = 2.0E0 )
 *     .. Local Scalars ..
       REAL               ALPHA, BETA
 *     .. Local Arrays ..
@@ -1834,6 +1879,12 @@
 *     LERR is set to .TRUE. by the special version of XERBLA each time
 *     it is called, and is then tested and re-set by CHKXER.
       LERR = .FALSE.
+*
+*     Initialize ALPHA and BETA.
+*
+      ALPHA = ONE
+      BETA = TWO
+*
       GO TO ( 10, 20, 30, 40, 50, 60 )ISNUM
    10 INFOT = 1
       CALL SGEMM( '/', 'N', 0, 0, 0, ALPHA, A, 1, B, 1, BETA, C, 1 )
@@ -1963,16 +2014,16 @@
       CALL SSYMM( 'R', 'L', 0, 2, ALPHA, A, 1, B, 1, BETA, C, 1 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 9
-      CALL SSYMM( 'L', 'U', 2, 0, ALPHA, A, 2, B, 1, BETA, C, 1 )
+      CALL SSYMM( 'L', 'U', 2, 0, ALPHA, A, 2, B, 1, BETA, C, 2 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 9
-      CALL SSYMM( 'R', 'U', 2, 0, ALPHA, A, 1, B, 1, BETA, C, 1 )
+      CALL SSYMM( 'R', 'U', 2, 0, ALPHA, A, 1, B, 1, BETA, C, 2 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 9
-      CALL SSYMM( 'L', 'L', 2, 0, ALPHA, A, 2, B, 1, BETA, C, 1 )
+      CALL SSYMM( 'L', 'L', 2, 0, ALPHA, A, 2, B, 1, BETA, C, 2 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 9
-      CALL SSYMM( 'R', 'L', 2, 0, ALPHA, A, 1, B, 1, BETA, C, 1 )
+      CALL SSYMM( 'R', 'L', 2, 0, ALPHA, A, 1, B, 1, BETA, C, 2 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 12
       CALL SSYMM( 'L', 'U', 2, 0, ALPHA, A, 2, B, 2, BETA, C, 1 )
@@ -2660,7 +2711,6 @@
    50    CONTINUE
       END IF
 *
-   60 CONTINUE
       LSERES = .TRUE.
       GO TO 80
    70 CONTINUE
diff --git a/test/zblat1.f b/test/zblat1.f
index 8b4b8d21e..2d7b88490 100644
--- a/test/zblat1.f
+++ b/test/zblat1.f
@@ -1,7 +1,49 @@
+*> \brief \b ZBLAT1
+*
+*  =========== DOCUMENTATION ===========
+*
+* Online html documentation available at
+*            http://www.netlib.org/lapack/explore-html/
+*
+*  Definition:
+*  ===========
+*
+*       PROGRAM ZBLAT1
+*
+*
+*> \par Purpose:
+*  =============
+*>
+*> \verbatim
+*>
+*>    Test program for the COMPLEX*16 Level 1 BLAS.
+*>
+*>    Based upon the original BLAS test routine together with:
+*>    F06GAF Example Program Text
+*> \endverbatim
+*
+*  Authors:
+*  ========
+*
+*> \author Univ. of Tennessee
+*> \author Univ. of California Berkeley
+*> \author Univ. of Colorado Denver
+*> \author NAG Ltd.
+*
+*> \date April 2012
+*
+*> \ingroup complex16_blas_testing
+*
+*  =====================================================================
       PROGRAM ZBLAT1
-*     Test program for the COMPLEX*16 Level 1 BLAS.
-*     Based upon the original BLAS test routine together with:
-*     F06GAF Example Program Text
+*
+*  -- Reference BLAS test routine (version 3.7.0) --
+*  -- Reference BLAS is a software package provided by Univ. of Tennessee,    --
+*  -- Univ. of California Berkeley, Univ. of Colorado Denver and NAG Ltd..--
+*     April 2012
+*
+*  =====================================================================
+*
 *     .. Parameters ..
       INTEGER          NOUT
       PARAMETER        (NOUT=6)
@@ -114,8 +156,8 @@
      +                  (5.0D0,6.0D0), (5.0D0,6.0D0), (0.1D0,0.1D0),
      +                  (-0.6D0,0.1D0), (0.1D0,-0.3D0), (7.0D0,8.0D0),
      +                  (7.0D0,8.0D0), (7.0D0,8.0D0), (7.0D0,8.0D0),
-     +                  (7.0D0,8.0D0), (0.3D0,0.1D0), (0.1D0,0.4D0),
-     +                  (0.4D0,0.1D0), (0.1D0,0.2D0), (2.0D0,3.0D0),
+     +                  (7.0D0,8.0D0), (0.3D0,0.1D0), (0.5D0,0.0D0),
+     +                  (0.0D0,0.5D0), (0.0D0,0.2D0), (2.0D0,3.0D0),
      +                  (2.0D0,3.0D0), (2.0D0,3.0D0), (2.0D0,3.0D0)/
       DATA              ((CV(I,J,2),I=1,8),J=1,5)/(0.1D0,0.1D0),
      +                  (4.0D0,5.0D0), (4.0D0,5.0D0), (4.0D0,5.0D0),
@@ -129,10 +171,10 @@
      +                  (3.0D0,6.0D0), (-0.6D0,0.1D0), (4.0D0,7.0D0),
      +                  (0.1D0,-0.3D0), (7.0D0,2.0D0), (7.0D0,2.0D0),
      +                  (7.0D0,2.0D0), (0.3D0,0.1D0), (5.0D0,8.0D0),
-     +                  (0.1D0,0.4D0), (6.0D0,9.0D0), (0.4D0,0.1D0),
-     +                  (8.0D0,3.0D0), (0.1D0,0.2D0), (9.0D0,4.0D0)/
-      DATA              STRUE2/0.0D0, 0.5D0, 0.6D0, 0.7D0, 0.7D0/
-      DATA              STRUE4/0.0D0, 0.7D0, 1.0D0, 1.3D0, 1.7D0/
+     +                  (0.5D0,0.0D0), (6.0D0,9.0D0), (0.0D0,0.5D0),
+     +                  (8.0D0,3.0D0), (0.0D0,0.2D0), (9.0D0,4.0D0)/
+      DATA              STRUE2/0.0D0, 0.5D0, 0.6D0, 0.7D0, 0.8D0/
+      DATA              STRUE4/0.0D0, 0.7D0, 1.0D0, 1.3D0, 1.6D0/
       DATA              ((CTRUE5(I,J,1),I=1,8),J=1,5)/(0.1D0,0.1D0),
      +                  (1.0D0,2.0D0), (1.0D0,2.0D0), (1.0D0,2.0D0),
      +                  (1.0D0,2.0D0), (1.0D0,2.0D0), (1.0D0,2.0D0),
@@ -145,8 +187,8 @@
      +                  (0.11D0,-0.03D0), (-0.17D0,0.46D0),
      +                  (-0.17D0,-0.19D0), (7.0D0,8.0D0), (7.0D0,8.0D0),
      +                  (7.0D0,8.0D0), (7.0D0,8.0D0), (7.0D0,8.0D0),
-     +                  (0.19D0,-0.17D0), (0.32D0,0.09D0),
-     +                  (0.23D0,-0.24D0), (0.18D0,0.01D0),
+     +                  (0.19D0,-0.17D0), (0.20D0,-0.35D0),
+     +                  (0.35D0,0.20D0), (0.14D0,0.08D0),
      +                  (2.0D0,3.0D0), (2.0D0,3.0D0), (2.0D0,3.0D0),
      +                  (2.0D0,3.0D0)/
       DATA              ((CTRUE5(I,J,2),I=1,8),J=1,5)/(0.1D0,0.1D0),
@@ -162,9 +204,9 @@
      +                  (-0.17D0,0.46D0), (4.0D0,7.0D0),
      +                  (-0.17D0,-0.19D0), (7.0D0,2.0D0), (7.0D0,2.0D0),
      +                  (7.0D0,2.0D0), (0.19D0,-0.17D0), (5.0D0,8.0D0),
-     +                  (0.32D0,0.09D0), (6.0D0,9.0D0),
-     +                  (0.23D0,-0.24D0), (8.0D0,3.0D0),
-     +                  (0.18D0,0.01D0), (9.0D0,4.0D0)/
+     +                  (0.20D0,-0.35D0), (6.0D0,9.0D0),
+     +                  (0.35D0,0.20D0), (8.0D0,3.0D0),
+     +                  (0.14D0,0.08D0), (9.0D0,4.0D0)/
       DATA              ((CTRUE6(I,J,1),I=1,8),J=1,5)/(0.1D0,0.1D0),
      +                  (1.0D0,2.0D0), (1.0D0,2.0D0), (1.0D0,2.0D0),
      +                  (1.0D0,2.0D0), (1.0D0,2.0D0), (1.0D0,2.0D0),
@@ -177,8 +219,8 @@
      +                  (0.03D0,0.03D0), (-0.18D0,0.03D0),
      +                  (0.03D0,-0.09D0), (7.0D0,8.0D0), (7.0D0,8.0D0),
      +                  (7.0D0,8.0D0), (7.0D0,8.0D0), (7.0D0,8.0D0),
-     +                  (0.09D0,0.03D0), (0.03D0,0.12D0),
-     +                  (0.12D0,0.03D0), (0.03D0,0.06D0), (2.0D0,3.0D0),
+     +                  (0.09D0,0.03D0), (0.15D0,0.00D0),
+     +                  (0.00D0,0.15D0), (0.00D0,0.06D0), (2.0D0,3.0D0),
      +                  (2.0D0,3.0D0), (2.0D0,3.0D0), (2.0D0,3.0D0)/
       DATA              ((CTRUE6(I,J,2),I=1,8),J=1,5)/(0.1D0,0.1D0),
      +                  (4.0D0,5.0D0), (4.0D0,5.0D0), (4.0D0,5.0D0),
@@ -193,8 +235,8 @@
      +                  (-0.18D0,0.03D0), (4.0D0,7.0D0),
      +                  (0.03D0,-0.09D0), (7.0D0,2.0D0), (7.0D0,2.0D0),
      +                  (7.0D0,2.0D0), (0.09D0,0.03D0), (5.0D0,8.0D0),
-     +                  (0.03D0,0.12D0), (6.0D0,9.0D0), (0.12D0,0.03D0),
-     +                  (8.0D0,3.0D0), (0.03D0,0.06D0), (9.0D0,4.0D0)/
+     +                  (0.15D0,0.00D0), (6.0D0,9.0D0), (0.00D0,0.15D0),
+     +                  (8.0D0,3.0D0), (0.00D0,0.06D0), (9.0D0,4.0D0)/
       DATA              ITRUE3/0, 1, 2, 2, 2/
 *     .. Executable Statements ..
       DO 60 INCX = 1, 2
@@ -529,7 +571,8 @@
 *
 *     .. Parameters ..
       INTEGER          NOUT
-      PARAMETER        (NOUT=6)
+      DOUBLE PRECISION ZERO
+      PARAMETER        (NOUT=6, ZERO=0.0D0)
 *     .. Scalar Arguments ..
       DOUBLE PRECISION SFAC
       INTEGER          LEN
@@ -552,7 +595,7 @@
 *
       DO 40 I = 1, LEN
          SD = SCOMP(I) - STRUE(I)
-         IF (SDIFF(ABS(SSIZE(I))+ABS(SFAC*SD),ABS(SSIZE(I))).EQ.0.0D0)
+         IF (ABS(SFAC*SD) .LE. ABS(SSIZE(I))*EPSILON(ZERO))
      +       GO TO 40
 *
 *                             HERE    SCOMP(I) IS NOT CLOSE TO STRUE(I).
diff --git a/test/zblat2.f b/test/zblat2.f
index e65cdcc70..4a20ac567 100644
--- a/test/zblat2.f
+++ b/test/zblat2.f
@@ -1,68 +1,114 @@
+*> \brief \b ZBLAT2
+*
+*  =========== DOCUMENTATION ===========
+*
+* Online html documentation available at
+*            http://www.netlib.org/lapack/explore-html/
+*
+*  Definition:
+*  ===========
+*
+*       PROGRAM ZBLAT2
+*
+*
+*> \par Purpose:
+*  =============
+*>
+*> \verbatim
+*>
+*> Test program for the COMPLEX*16       Level 2 Blas.
+*>
+*> The program must be driven by a short data file. The first 18 records
+*> of the file are read using list-directed input, the last 17 records
+*> are read using the format ( A6, L2 ). An annotated example of a data
+*> file can be obtained by deleting the first 3 characters from the
+*> following 35 lines:
+*> 'zblat2.out'      NAME OF SUMMARY OUTPUT FILE
+*> 6                 UNIT NUMBER OF SUMMARY FILE
+*> 'CBLA2T.SNAP'     NAME OF SNAPSHOT OUTPUT FILE
+*> -1                UNIT NUMBER OF SNAPSHOT FILE (NOT USED IF .LT. 0)
+*> F        LOGICAL FLAG, T TO REWIND SNAPSHOT FILE AFTER EACH RECORD.
+*> F        LOGICAL FLAG, T TO STOP ON FAILURES.
+*> T        LOGICAL FLAG, T TO TEST ERROR EXITS.
+*> 16.0     THRESHOLD VALUE OF TEST RATIO
+*> 6                 NUMBER OF VALUES OF N
+*> 0 1 2 3 5 9       VALUES OF N
+*> 4                 NUMBER OF VALUES OF K
+*> 0 1 2 4           VALUES OF K
+*> 4                 NUMBER OF VALUES OF INCX AND INCY
+*> 1 2 -1 -2         VALUES OF INCX AND INCY
+*> 3                 NUMBER OF VALUES OF ALPHA
+*> (0.0,0.0) (1.0,0.0) (0.7,-0.9)       VALUES OF ALPHA
+*> 3                 NUMBER OF VALUES OF BETA
+*> (0.0,0.0) (1.0,0.0) (1.3,-1.1)       VALUES OF BETA
+*> ZGEMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> ZGBMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> ZHEMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> ZHBMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> ZHPMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> ZTRMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> ZTBMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> ZTPMV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> ZTRSV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> ZTBSV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> ZTPSV  T PUT F FOR NO TEST. SAME COLUMNS.
+*> ZGERC  T PUT F FOR NO TEST. SAME COLUMNS.
+*> ZGERU  T PUT F FOR NO TEST. SAME COLUMNS.
+*> ZHER   T PUT F FOR NO TEST. SAME COLUMNS.
+*> ZHPR   T PUT F FOR NO TEST. SAME COLUMNS.
+*> ZHER2  T PUT F FOR NO TEST. SAME COLUMNS.
+*> ZHPR2  T PUT F FOR NO TEST. SAME COLUMNS.
+*>
+*> Further Details
+*> ===============
+*>
+*>    See:
+*>
+*>       Dongarra J. J., Du Croz J. J., Hammarling S.  and Hanson R. J..
+*>       An  extended  set of Fortran  Basic Linear Algebra Subprograms.
+*>
+*>       Technical  Memoranda  Nos. 41 (revision 3) and 81,  Mathematics
+*>       and  Computer Science  Division,  Argonne  National Laboratory,
+*>       9700 South Cass Avenue, Argonne, Illinois 60439, US.
+*>
+*>       Or
+*>
+*>       NAG  Technical Reports TR3/87 and TR4/87,  Numerical Algorithms
+*>       Group  Ltd.,  NAG  Central  Office,  256  Banbury  Road, Oxford
+*>       OX2 7DE, UK,  and  Numerical Algorithms Group Inc.,  1101  31st
+*>       Street,  Suite 100,  Downers Grove,  Illinois 60515-1263,  USA.
+*>
+*>
+*> -- Written on 10-August-1987.
+*>    Richard Hanson, Sandia National Labs.
+*>    Jeremy Du Croz, NAG Central Office.
+*>
+*>    10-9-00:  Change STATUS='NEW' to 'UNKNOWN' so that the testers
+*>              can be run multiple times without deleting generated
+*>              output files (susan)
+*> \endverbatim
+*
+*  Authors:
+*  ========
+*
+*> \author Univ. of Tennessee
+*> \author Univ. of California Berkeley
+*> \author Univ. of Colorado Denver
+*> \author NAG Ltd.
+*
+*> \date April 2012
+*
+*> \ingroup complex16_blas_testing
+*
+*  =====================================================================
       PROGRAM ZBLAT2
 *
-*  Test program for the COMPLEX*16       Level 2 Blas.
+*  -- Reference BLAS test routine (version 3.7.0) --
+*  -- Reference BLAS is a software package provided by Univ. of Tennessee,    --
+*  -- Univ. of California Berkeley, Univ. of Colorado Denver and NAG Ltd..--
+*     April 2012
 *
-*  The program must be driven by a short data file. The first 18 records
-*  of the file are read using list-directed input, the last 17 records
-*  are read using the format ( A6, L2 ). An annotated example of a data
-*  file can be obtained by deleting the first 3 characters from the
-*  following 35 lines:
-*  'ZBLAT2.SUMM'     NAME OF SUMMARY OUTPUT FILE
-*  6                 UNIT NUMBER OF SUMMARY FILE
-*  'CBLA2T.SNAP'     NAME OF SNAPSHOT OUTPUT FILE
-*  -1                UNIT NUMBER OF SNAPSHOT FILE (NOT USED IF .LT. 0)
-*  F        LOGICAL FLAG, T TO REWIND SNAPSHOT FILE AFTER EACH RECORD.
-*  F        LOGICAL FLAG, T TO STOP ON FAILURES.
-*  T        LOGICAL FLAG, T TO TEST ERROR EXITS.
-*  16.0     THRESHOLD VALUE OF TEST RATIO
-*  6                 NUMBER OF VALUES OF N
-*  0 1 2 3 5 9       VALUES OF N
-*  4                 NUMBER OF VALUES OF K
-*  0 1 2 4           VALUES OF K
-*  4                 NUMBER OF VALUES OF INCX AND INCY
-*  1 2 -1 -2         VALUES OF INCX AND INCY
-*  3                 NUMBER OF VALUES OF ALPHA
-*  (0.0,0.0) (1.0,0.0) (0.7,-0.9)       VALUES OF ALPHA
-*  3                 NUMBER OF VALUES OF BETA
-*  (0.0,0.0) (1.0,0.0) (1.3,-1.1)       VALUES OF BETA
-*  ZGEMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  ZGBMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  ZHEMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  ZHBMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  ZHPMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  ZTRMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  ZTBMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  ZTPMV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  ZTRSV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  ZTBSV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  ZTPSV  T PUT F FOR NO TEST. SAME COLUMNS.
-*  ZGERC  T PUT F FOR NO TEST. SAME COLUMNS.
-*  ZGERU  T PUT F FOR NO TEST. SAME COLUMNS.
-*  ZHER   T PUT F FOR NO TEST. SAME COLUMNS.
-*  ZHPR   T PUT F FOR NO TEST. SAME COLUMNS.
-*  ZHER2  T PUT F FOR NO TEST. SAME COLUMNS.
-*  ZHPR2  T PUT F FOR NO TEST. SAME COLUMNS.
-*
-*     See:
-*
-*        Dongarra J. J., Du Croz J. J., Hammarling S.  and Hanson R. J..
-*        An  extended  set of Fortran  Basic Linear Algebra Subprograms.
-*
-*        Technical  Memoranda  Nos. 41 (revision 3) and 81,  Mathematics
-*        and  Computer Science  Division,  Argonne  National Laboratory,
-*        9700 South Cass Avenue, Argonne, Illinois 60439, US.
-*
-*        Or
-*
-*        NAG  Technical Reports TR3/87 and TR4/87,  Numerical Algorithms
-*        Group  Ltd.,  NAG  Central  Office,  256  Banbury  Road, Oxford
-*        OX2 7DE, UK,  and  Numerical Algorithms Group Inc.,  1101  31st
-*        Street,  Suite 100,  Downers Grove,  Illinois 60515-1263,  USA.
-*
-*
-*  -- Written on 10-August-1987.
-*     Richard Hanson, Sandia National Labs.
-*     Jeremy Du Croz, NAG Central Office.
+*  =====================================================================
 *
 *     .. Parameters ..
       INTEGER            NIN
@@ -72,8 +118,8 @@
       COMPLEX*16         ZERO, ONE
       PARAMETER          ( ZERO = ( 0.0D0, 0.0D0 ),
      $                   ONE = ( 1.0D0, 0.0D0 ) )
-      DOUBLE PRECISION   RZERO, RHALF, RONE
-      PARAMETER          ( RZERO = 0.0D0, RHALF = 0.5D0, RONE = 1.0D0 )
+      DOUBLE PRECISION   RZERO
+      PARAMETER          ( RZERO = 0.0D0 )
       INTEGER            NMAX, INCMAX
       PARAMETER          ( NMAX = 65, INCMAX = 2 )
       INTEGER            NINMAX, NIDMAX, NKBMAX, NALMAX, NBEMAX
@@ -127,7 +173,7 @@
 *
       READ( NIN, FMT = * )SUMMRY
       READ( NIN, FMT = * )NOUT
-      OPEN( NOUT, FILE = SUMMRY, STATUS = 'NEW' )
+      OPEN( NOUT, FILE = SUMMRY, STATUS = 'UNKNOWN' )
       NOUTC = NOUT
 *
 *     Read name and unit number for snapshot output file and open file.
@@ -136,7 +182,7 @@
       READ( NIN, FMT = * )NTRA
       TRACE = NTRA.GE.0
       IF( TRACE )THEN
-         OPEN( NTRA, FILE = SNAPS, STATUS = 'NEW' )
+         OPEN( NTRA, FILE = SNAPS, STATUS = 'UNKNOWN' )
       END IF
 *     Read the flag that directs rewinding of the snapshot file.
       READ( NIN, FMT = * )REWI
@@ -241,14 +287,7 @@
 *
 *     Compute EPS (the machine precision).
 *
-      EPS = RONE
-   90 CONTINUE
-      IF( DDIFF( RONE + EPS, RONE ).EQ.RZERO )
-     $   GO TO 100
-      EPS = RHALF*EPS
-      GO TO 90
-  100 CONTINUE
-      EPS = EPS + EPS
+      EPS = EPSILON(RZERO)
       WRITE( NOUT, FMT = 9998 )EPS
 *
 *     Check the reliability of ZMVCH using exact data.
@@ -3087,7 +3126,6 @@
    50    CONTINUE
       END IF
 *
-   60 CONTINUE
       LZERES = .TRUE.
       GO TO 80
    70 CONTINUE
diff --git a/test/zblat3.f b/test/zblat3.f
index f03b1a617..0e38334e9 100644
--- a/test/zblat3.f
+++ b/test/zblat3.f
@@ -1,50 +1,97 @@
+*> \brief \b ZBLAT3
+*
+*  =========== DOCUMENTATION ===========
+*
+* Online html documentation available at
+*            http://www.netlib.org/lapack/explore-html/
+*
+*  Definition:
+*  ===========
+*
+*       PROGRAM ZBLAT3
+*
+*
+*> \par Purpose:
+*  =============
+*>
+*> \verbatim
+*>
+*> Test program for the COMPLEX*16       Level 3 Blas.
+*>
+*> The program must be driven by a short data file. The first 14 records
+*> of the file are read using list-directed input, the last 9 records
+*> are read using the format ( A6, L2 ). An annotated example of a data
+*> file can be obtained by deleting the first 3 characters from the
+*> following 23 lines:
+*> 'zblat3.out'      NAME OF SUMMARY OUTPUT FILE
+*> 6                 UNIT NUMBER OF SUMMARY FILE
+*> 'ZBLAT3.SNAP'     NAME OF SNAPSHOT OUTPUT FILE
+*> -1                UNIT NUMBER OF SNAPSHOT FILE (NOT USED IF .LT. 0)
+*> F        LOGICAL FLAG, T TO REWIND SNAPSHOT FILE AFTER EACH RECORD.
+*> F        LOGICAL FLAG, T TO STOP ON FAILURES.
+*> T        LOGICAL FLAG, T TO TEST ERROR EXITS.
+*> 16.0     THRESHOLD VALUE OF TEST RATIO
+*> 6                 NUMBER OF VALUES OF N
+*> 0 1 2 3 5 9       VALUES OF N
+*> 3                 NUMBER OF VALUES OF ALPHA
+*> (0.0,0.0) (1.0,0.0) (0.7,-0.9)       VALUES OF ALPHA
+*> 3                 NUMBER OF VALUES OF BETA
+*> (0.0,0.0) (1.0,0.0) (1.3,-1.1)       VALUES OF BETA
+*> ZGEMM  T PUT F FOR NO TEST. SAME COLUMNS.
+*> ZHEMM  T PUT F FOR NO TEST. SAME COLUMNS.
+*> ZSYMM  T PUT F FOR NO TEST. SAME COLUMNS.
+*> ZTRMM  T PUT F FOR NO TEST. SAME COLUMNS.
+*> ZTRSM  T PUT F FOR NO TEST. SAME COLUMNS.
+*> ZHERK  T PUT F FOR NO TEST. SAME COLUMNS.
+*> ZSYRK  T PUT F FOR NO TEST. SAME COLUMNS.
+*> ZHER2K T PUT F FOR NO TEST. SAME COLUMNS.
+*> ZSYR2K T PUT F FOR NO TEST. SAME COLUMNS.
+*>
+*>
+*> Further Details
+*> ===============
+*>
+*> See:
+*>
+*>    Dongarra J. J., Du Croz J. J., Duff I. S. and Hammarling S.
+*>    A Set of Level 3 Basic Linear Algebra Subprograms.
+*>
+*>    Technical Memorandum No.88 (Revision 1), Mathematics and
+*>    Computer Science Division, Argonne National Laboratory, 9700
+*>    South Cass Avenue, Argonne, Illinois 60439, US.
+*>
+*> -- Written on 8-February-1989.
+*>    Jack Dongarra, Argonne National Laboratory.
+*>    Iain Duff, AERE Harwell.
+*>    Jeremy Du Croz, Numerical Algorithms Group Ltd.
+*>    Sven Hammarling, Numerical Algorithms Group Ltd.
+*>
+*>    10-9-00:  Change STATUS='NEW' to 'UNKNOWN' so that the testers
+*>              can be run multiple times without deleting generated
+*>              output files (susan)
+*> \endverbatim
+*
+*  Authors:
+*  ========
+*
+*> \author Univ. of Tennessee
+*> \author Univ. of California Berkeley
+*> \author Univ. of Colorado Denver
+*> \author NAG Ltd.
+*
+*> \date April 2012
+*
+*> \ingroup complex16_blas_testing
+*
+*  =====================================================================
       PROGRAM ZBLAT3
 *
-*  Test program for the COMPLEX*16       Level 3 Blas.
+*  -- Reference BLAS test routine (version 3.7.0) --
+*  -- Reference BLAS is a software package provided by Univ. of Tennessee,    --
+*  -- Univ. of California Berkeley, Univ. of Colorado Denver and NAG Ltd..--
+*     April 2012
 *
-*  The program must be driven by a short data file. The first 14 records
-*  of the file are read using list-directed input, the last 9 records
-*  are read using the format ( A6, L2 ). An annotated example of a data
-*  file can be obtained by deleting the first 3 characters from the
-*  following 23 lines:
-*  'ZBLAT3.SUMM'     NAME OF SUMMARY OUTPUT FILE
-*  6                 UNIT NUMBER OF SUMMARY FILE
-*  'ZBLAT3.SNAP'     NAME OF SNAPSHOT OUTPUT FILE
-*  -1                UNIT NUMBER OF SNAPSHOT FILE (NOT USED IF .LT. 0)
-*  F        LOGICAL FLAG, T TO REWIND SNAPSHOT FILE AFTER EACH RECORD.
-*  F        LOGICAL FLAG, T TO STOP ON FAILURES.
-*  T        LOGICAL FLAG, T TO TEST ERROR EXITS.
-*  16.0     THRESHOLD VALUE OF TEST RATIO
-*  6                 NUMBER OF VALUES OF N
-*  0 1 2 3 5 9       VALUES OF N
-*  3                 NUMBER OF VALUES OF ALPHA
-*  (0.0,0.0) (1.0,0.0) (0.7,-0.9)       VALUES OF ALPHA
-*  3                 NUMBER OF VALUES OF BETA
-*  (0.0,0.0) (1.0,0.0) (1.3,-1.1)       VALUES OF BETA
-*  ZGEMM  T PUT F FOR NO TEST. SAME COLUMNS.
-*  ZHEMM  T PUT F FOR NO TEST. SAME COLUMNS.
-*  ZSYMM  T PUT F FOR NO TEST. SAME COLUMNS.
-*  ZTRMM  T PUT F FOR NO TEST. SAME COLUMNS.
-*  ZTRSM  T PUT F FOR NO TEST. SAME COLUMNS.
-*  ZHERK  T PUT F FOR NO TEST. SAME COLUMNS.
-*  ZSYRK  T PUT F FOR NO TEST. SAME COLUMNS.
-*  ZHER2K T PUT F FOR NO TEST. SAME COLUMNS.
-*  ZSYR2K T PUT F FOR NO TEST. SAME COLUMNS.
-*
-*  See:
-*
-*     Dongarra J. J., Du Croz J. J., Duff I. S. and Hammarling S.
-*     A Set of Level 3 Basic Linear Algebra Subprograms.
-*
-*     Technical Memorandum No.88 (Revision 1), Mathematics and
-*     Computer Science Division, Argonne National Laboratory, 9700
-*     South Cass Avenue, Argonne, Illinois 60439, US.
-*
-*  -- Written on 8-February-1989.
-*     Jack Dongarra, Argonne National Laboratory.
-*     Iain Duff, AERE Harwell.
-*     Jeremy Du Croz, Numerical Algorithms Group Ltd.
-*     Sven Hammarling, Numerical Algorithms Group Ltd.
+*  =====================================================================
 *
 *     .. Parameters ..
       INTEGER            NIN
@@ -54,8 +101,8 @@
       COMPLEX*16         ZERO, ONE
       PARAMETER          ( ZERO = ( 0.0D0, 0.0D0 ),
      $                   ONE = ( 1.0D0, 0.0D0 ) )
-      DOUBLE PRECISION   RZERO, RHALF, RONE
-      PARAMETER          ( RZERO = 0.0D0, RHALF = 0.5D0, RONE = 1.0D0 )
+      DOUBLE PRECISION   RZERO
+      PARAMETER          ( RZERO = 0.0D0 )
       INTEGER            NMAX
       PARAMETER          ( NMAX = 65 )
       INTEGER            NIDMAX, NALMAX, NBEMAX
@@ -104,7 +151,7 @@
 *
       READ( NIN, FMT = * )SUMMRY
       READ( NIN, FMT = * )NOUT
-      OPEN( NOUT, FILE = SUMMRY, STATUS = 'NEW' )
+      OPEN( NOUT, FILE = SUMMRY, STATUS = 'UNKNOWN' )
       NOUTC = NOUT
 *
 *     Read name and unit number for snapshot output file and open file.
@@ -113,7 +160,7 @@
       READ( NIN, FMT = * )NTRA
       TRACE = NTRA.GE.0
       IF( TRACE )THEN
-         OPEN( NTRA, FILE = SNAPS, STATUS = 'NEW' )
+         OPEN( NTRA, FILE = SNAPS, STATUS = 'UNKNOWN' )
       END IF
 *     Read the flag that directs rewinding of the snapshot file.
       READ( NIN, FMT = * )REWI
@@ -190,14 +237,7 @@
 *
 *     Compute EPS (the machine precision).
 *
-      EPS = RONE
-   70 CONTINUE
-      IF( DDIFF( RONE + EPS, RONE ).EQ.RZERO )
-     $   GO TO 80
-      EPS = RHALF*EPS
-      GO TO 70
-   80 CONTINUE
-      EPS = EPS + EPS
+      EPS = EPSILON(RZERO)
       WRITE( NOUT, FMT = 9998 )EPS
 *
 *     Check the reliability of ZMMCH using exact data.
@@ -1303,8 +1343,6 @@
       NC = 0
       RESET = .TRUE.
       ERRMAX = RZERO
-      RALS = RONE
-      RBETS = RONE
 *
       DO 100 IN = 1, NIDIM
          N = IDIM( IN )
@@ -1951,7 +1989,7 @@
 *
 *  Tests the error exits from the Level 3 Blas.
 *  Requires a special version of the error-handling routine XERBLA.
-*  ALPHA, RALPHA, BETA, RBETA, A, B and C should not need to be defined.
+*  A, B and C should not need to be defined.
 *
 *  Auxiliary routine for test program for Level 3 Blas.
 *
@@ -1961,12 +1999,20 @@
 *     Jeremy Du Croz, Numerical Algorithms Group Ltd.
 *     Sven Hammarling, Numerical Algorithms Group Ltd.
 *
+*  3-19-92:  Initialize ALPHA, BETA, RALPHA, and RBETA  (eca)
+*  3-19-92:  Fix argument 12 in calls to ZSYMM and ZHEMM
+*            with INFOT = 9  (eca)
+*  10-9-00:  Declared INTRINSIC DCMPLX (susan)
+*
 *     .. Scalar Arguments ..
       INTEGER            ISNUM, NOUT
       CHARACTER*6        SRNAMT
 *     .. Scalars in Common ..
       INTEGER            INFOT, NOUTC
       LOGICAL            LERR, OK
+*     .. Parameters ..
+      REAL               ONE, TWO
+      PARAMETER          ( ONE = 1.0D0, TWO = 2.0D0 )
 *     .. Local Scalars ..
       COMPLEX*16         ALPHA, BETA
       DOUBLE PRECISION   RALPHA, RBETA
@@ -1975,6 +2021,8 @@
 *     .. External Subroutines ..
       EXTERNAL           ZGEMM, ZHEMM, ZHER2K, ZHERK, CHKXER, ZSYMM,
      $                   ZSYR2K, ZSYRK, ZTRMM, ZTRSM
+*     .. Intrinsic Functions ..
+      INTRINSIC          DCMPLX
 *     .. Common blocks ..
       COMMON             /INFOC/INFOT, NOUTC, OK, LERR
 *     .. Executable Statements ..
@@ -1984,6 +2032,14 @@
 *     LERR is set to .TRUE. by the special version of XERBLA each time
 *     it is called, and is then tested and re-set by CHKXER.
       LERR = .FALSE.
+*
+*     Initialize ALPHA, BETA, RALPHA, and RBETA.
+*
+      ALPHA = DCMPLX( ONE, -ONE )
+      BETA = DCMPLX( TWO, -TWO )
+      RALPHA = ONE
+      RBETA = TWO
+*
       GO TO ( 10, 20, 30, 40, 50, 60, 70, 80,
      $        90 )ISNUM
    10 INFOT = 1
@@ -2210,16 +2266,16 @@
       CALL ZHEMM( 'R', 'L', 0, 2, ALPHA, A, 1, B, 1, BETA, C, 1 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 9
-      CALL ZHEMM( 'L', 'U', 2, 0, ALPHA, A, 2, B, 1, BETA, C, 1 )
+      CALL ZHEMM( 'L', 'U', 2, 0, ALPHA, A, 2, B, 1, BETA, C, 2 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 9
-      CALL ZHEMM( 'R', 'U', 2, 0, ALPHA, A, 1, B, 1, BETA, C, 1 )
+      CALL ZHEMM( 'R', 'U', 2, 0, ALPHA, A, 1, B, 1, BETA, C, 2 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 9
-      CALL ZHEMM( 'L', 'L', 2, 0, ALPHA, A, 2, B, 1, BETA, C, 1 )
+      CALL ZHEMM( 'L', 'L', 2, 0, ALPHA, A, 2, B, 1, BETA, C, 2 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 9
-      CALL ZHEMM( 'R', 'L', 2, 0, ALPHA, A, 1, B, 1, BETA, C, 1 )
+      CALL ZHEMM( 'R', 'L', 2, 0, ALPHA, A, 1, B, 1, BETA, C, 2 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 12
       CALL ZHEMM( 'L', 'U', 2, 0, ALPHA, A, 2, B, 2, BETA, C, 1 )
@@ -2277,16 +2333,16 @@
       CALL ZSYMM( 'R', 'L', 0, 2, ALPHA, A, 1, B, 1, BETA, C, 1 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 9
-      CALL ZSYMM( 'L', 'U', 2, 0, ALPHA, A, 2, B, 1, BETA, C, 1 )
+      CALL ZSYMM( 'L', 'U', 2, 0, ALPHA, A, 2, B, 1, BETA, C, 2 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 9
-      CALL ZSYMM( 'R', 'U', 2, 0, ALPHA, A, 1, B, 1, BETA, C, 1 )
+      CALL ZSYMM( 'R', 'U', 2, 0, ALPHA, A, 1, B, 1, BETA, C, 2 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 9
-      CALL ZSYMM( 'L', 'L', 2, 0, ALPHA, A, 2, B, 1, BETA, C, 1 )
+      CALL ZSYMM( 'L', 'L', 2, 0, ALPHA, A, 2, B, 1, BETA, C, 2 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 9
-      CALL ZSYMM( 'R', 'L', 2, 0, ALPHA, A, 1, B, 1, BETA, C, 1 )
+      CALL ZSYMM( 'R', 'L', 2, 0, ALPHA, A, 1, B, 1, BETA, C, 2 )
       CALL CHKXER( SRNAMT, INFOT, NOUT, LERR, OK )
       INFOT = 12
       CALL ZSYMM( 'L', 'U', 2, 0, ALPHA, A, 2, B, 2, BETA, C, 1 )
@@ -3276,7 +3332,6 @@
    50    CONTINUE
       END IF
 *
-   60 CONTINUE
       LZERES = .TRUE.
       GO TO 80
    70 CONTINUE

From 79cdcde7173cbb8adf231867fb6643afaada5712 Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Sun, 14 Jun 2020 17:18:16 +0200
Subject: [PATCH 14/48] Re-enable higher optimization levels for flang while
 disabling loop unrolling for AOCC flang

---
 Makefile.system | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/Makefile.system b/Makefile.system
index 31cdd12b2..d68353b12 100644
--- a/Makefile.system
+++ b/Makefile.system
@@ -784,6 +784,14 @@ endif
 ifeq ($(F_COMPILER), FLANG)
 CCOMMON_OPT += -DF_INTERFACE_FLANG
 FCOMMON_OPT += -Mrecursive -Kieee
+ifeq ($(OSNAME), Linux)
+ifeq ($(ARCH), x86_64)
+FLANG_VENDOR := $(shell expr `$(FC) --version|cut -f 1 -d "."|head -1`)
+ifeq ($(FLANG_VENDOR),AOCC)
+FCOMMON_OPT += -fno-unroll-loops
+endif
+endif
+endif
 ifdef BINARY64
 ifdef INTERFACE64
 ifneq ($(INTERFACE64), 0)
@@ -1271,11 +1279,7 @@ endif
 
 override CFLAGS     += $(COMMON_OPT) $(CCOMMON_OPT) -I$(TOPDIR)
 override PFLAGS     += $(COMMON_OPT) $(CCOMMON_OPT) -I$(TOPDIR) -DPROFILE $(COMMON_PROF)
-ifeq ($(F_COMPILER),FLANG)
-override FFLAGS     += $(filter-out -O2 -O3,$(COMMON_OPT)) -O1 $(FCOMMON_OPT)
-else
 override FFLAGS     += $(COMMON_OPT) $(FCOMMON_OPT)
-endif
 override FPFLAGS    += $(FCOMMON_OPT) $(COMMON_PROF)
 #MAKEOVERRIDES =
 

From 6876221cf340d3efb71f64e73aef5006b3bc96d6 Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Sun, 14 Jun 2020 17:40:24 +0200
Subject: [PATCH 15/48] Remove optimization level limit for flang again and add
 -fno-unroll-loops for AOCC flang 2.x instead

---
 cmake/system.cmake | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/cmake/system.cmake b/cmake/system.cmake
index c2ae471d2..d8dcc3cf3 100644
--- a/cmake/system.cmake
+++ b/cmake/system.cmake
@@ -419,10 +419,9 @@ endif ()
 
 if ("${CMAKE_BUILD_TYPE}" STREQUAL "Release")
 if ("${F_COMPILER}" STREQUAL "FLANG")
-  set(FILTER_FLAGS "-O2;-O3")
-  foreach (FILTER_FLAG ${FILTER_FLAGS})
-    string(REPLACE ${FILTER_FLAG} "-O1" CMAKE_Fortran_FLAGS_RELEASE ${CMAKE_Fortran_FLAGS_RELEASE})
-  endforeach ()
+if (${CMAKE_Fortran_COMPILER_VERSION} VERSION_LESS_EQUAL 3)
+  set(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -fno-unroll-loops")
+endif ()
 endif ()
 endif ()
 

From 72888497e2ffb6233ffd18ccf0b4d4bb01701b17 Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Sun, 14 Jun 2020 21:55:31 +0200
Subject: [PATCH 16/48] Update with 0.3.10 changes

---
 Changelog.txt | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 73 insertions(+)

diff --git a/Changelog.txt b/Changelog.txt
index 5f924629b..cbf0b50f5 100644
--- a/Changelog.txt
+++ b/Changelog.txt
@@ -1,4 +1,77 @@
 OpenBLAS ChangeLog
+====================================================================
+Version 0.3.10
+ 14-Jun-2020
+
+common:
+	* Improved thread locking behaviour in blas_server and parallel getrf                         
+	* Imported bugfix 394 from LAPACK (spurious reference to "XERBL"
+	  due to overlong lines)
+	* Imported bugfix 403 from LAPACK (compile option "recursive" required
+	  for correctness with Intel and PGI)
+	* Imported bugfix 408 from LAPACK (wrong scaling in ZHEEQUB)
+	* Imported bugfix 411 from LAPACK (infinite loop in LARGV/LARTG/LARTGP)
+	* Fixed mismatches between BUFFERSIZE and GEMM_UNROLL parameters that
+	  could lead to crashes at large matrix sizes  
+	* Restored internal soname in dynamic libraries on FreeBSD and Dragonfly 
+	* Added API (openblas_setaffinity) to set the thread affinity on Linux
+	* Added initial infrastructure for half-precision floating point 
+	  (bfloat16) support with a generic implementation of SHGEMM     
+	* Added CMAKE build system support for building the cblas_Xgemm3m
+	  functions
+	* Fixed CMAKE support for building in a path with embedded spaces
+	* Fixed CMAKE (non)handling of NO_EXPRECISION and MAX_STACK_ALLOC
+	* Fixed GCC version detection in the Makefiles
+	* Allowed overriding the names of AR, AS and LD in Makefile builds
+
+POWER:
+	* Fixed big-endian POWER8 ELFv2 builds on FreeBSD
+	* Fixed GCC version checks and DYNAMIC_ARCH builds on POWER9
+	* Fixed CMAKE build support for POWER9
+	* fixed a potential race condition in the thread buffer allocation
+	* Worked around LAPACK test failures on PPC G4
+
+MIPS:
+	* Fixed a potential race condition in the thread buffer allocation
+	* Added support for MIPS 24K/24KE family based on P5600 kernels
+
+MIPS64:
+	* fixed a potential race condition in the thread buffer allocation
+	* Added TARGET=GENERIC
+
+ARMV7:
+	* Fixed a race condition in the thread buffer allocation
+
+ARMV8:
+	* Fixed a race condition in the thread buffer allocation
+	* Fixed zero initialisation in the assembly for SGEMM and DGEMM BETA
+	* Improved performance of the ThunderX2 DAXPY kernel
+	* Added an optimized SGEMM kernel for Cortex A53    
+	* Fixed Makefile support for INTERFACE64 (8-byte integer)
+
+x86_64:
+	* Fixed a syntax error in the CMAKE setup for SkylakeX
+	* Improved performance of STRSM on Haswell, SkylakeX and Ryzen
+	* Improved SGEMM performance on SGEMM for workloads with ldc a
+	  multiple of 1024
+	* Improved DGEMM performance on Skylake X
+	* Fixed unwanted AVX512-dependency of SGEMM in DYNAMIC_ARCH
+	  builds created on SkylakeX
+	* Removed data alignment requirement in the SSE2 copy kernels
+	  that could cause spurious crashes
+	* Added a workaround for an optimizer bug in AppleClang 11.0.3
+	* Fixed LAPACK test failures due to wrong options for Intel Fortran
+	* Fixed compilation and LAPACK test results with recent Flang
+	  and AMD AOCC
+	* Fixed DYNAMIC_ARCH builds with CMAKE on OS X
+	* Fixed missing exports of cblas_i?amin, cblas_i?min, cblas_i?max,
+	  cblas_?sum, cblas_?gemm3m in the shared library on OS 
+	* Fixed reporting of cpu name in DYNAMIC_ARCH builds (would sometimes
+	  show the name of an older generation chip supported by the same kernels)
+
+IBM Z:
+	* Improved performance of SGEMM/STRMM and DGEMM/DTRMM on Z14
+
 ====================================================================
 Version 0.3.9
  1-Mar-2020

From 1eb197905056afa1b3e6d138d6084fb4d2b46322 Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Sun, 14 Jun 2020 21:57:15 +0200
Subject: [PATCH 17/48] Increment version to 0.3.10.dev

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5118475cc..9d4aa0ca6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -6,7 +6,7 @@ cmake_minimum_required(VERSION 2.8.5)
 project(OpenBLAS C ASM)
 set(OpenBLAS_MAJOR_VERSION 0)
 set(OpenBLAS_MINOR_VERSION 3)
-set(OpenBLAS_PATCH_VERSION 9.dev)
+set(OpenBLAS_PATCH_VERSION 10.dev)
 set(OpenBLAS_VERSION "${OpenBLAS_MAJOR_VERSION}.${OpenBLAS_MINOR_VERSION}.${OpenBLAS_PATCH_VERSION}")
 
 # Adhere to GNU filesystem layout conventions

From 3b673a24b704ab37e89eaf3832971726927e45e9 Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Sun, 14 Jun 2020 21:57:52 +0200
Subject: [PATCH 18/48] Increment version to 0.3.10.dev

---
 Makefile.rule | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile.rule b/Makefile.rule
index 8549e6394..2c12177ee 100644
--- a/Makefile.rule
+++ b/Makefile.rule
@@ -3,7 +3,7 @@
 #
 
 # This library's version
-VERSION = 0.3.9.dev
+VERSION = 0.3.10.dev
 
 # If you set the suffix, the library name will be libopenblas_$(LIBNAMESUFFIX).a
 # and libopenblas_$(LIBNAMESUFFIX).so. Meanwhile, the soname in shared library

From 1c53e1366d5441ee7fa22b77be7bea8c5eabef32 Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Sun, 14 Jun 2020 22:04:37 +0200
Subject: [PATCH 19/48] Increment version to 0.3.10.dev

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 12621d6b8..bb5322a1d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -6,7 +6,7 @@ cmake_minimum_required(VERSION 2.8.5)
 project(OpenBLAS C ASM)
 set(OpenBLAS_MAJOR_VERSION 0)
 set(OpenBLAS_MINOR_VERSION 3)
-set(OpenBLAS_PATCH_VERSION 10)
+set(OpenBLAS_PATCH_VERSION 10.dev)
 
 set(OpenBLAS_VERSION "${OpenBLAS_MAJOR_VERSION}.${OpenBLAS_MINOR_VERSION}.${OpenBLAS_PATCH_VERSION}")
 

From 1bd3cd66c270134d138f7b61cd158407a07086cf Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Sun, 14 Jun 2020 22:05:19 +0200
Subject: [PATCH 20/48] Increment version to 0.3.10.dev

---
 Makefile.rule | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile.rule b/Makefile.rule
index 551c094ca..2c12177ee 100644
--- a/Makefile.rule
+++ b/Makefile.rule
@@ -3,7 +3,7 @@
 #
 
 # This library's version
-VERSION = 0.3.10
+VERSION = 0.3.10.dev
 
 # If you set the suffix, the library name will be libopenblas_$(LIBNAMESUFFIX).a
 # and libopenblas_$(LIBNAMESUFFIX).so. Meanwhile, the soname in shared library

From a2d13ea61183099c05aa31e23ef59e1411d77177 Mon Sep 17 00:00:00 2001
From: Marius Hillenbrand <mhillen@linux.ibm.com>
Date: Tue, 16 Jun 2020 14:40:50 +0200
Subject: [PATCH 21/48] Fix gcc version detection for zarch

Employ common variables for gcc version detection and fix the broken
check for gcc >= 5.2.
Fixes #2668

Signed-off-by: Marius Hillenbrand <mhillen@linux.ibm.com>
---
 Makefile.system | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/Makefile.system b/Makefile.system
index 8d78b420f..5738b14ec 100644
--- a/Makefile.system
+++ b/Makefile.system
@@ -282,9 +282,11 @@ endif
 ifeq ($(C_COMPILER), GCC)
 GCCVERSIONGTEQ4 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \>= 4)
 GCCVERSIONGT4 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \> 4)
+GCCVERSIONEQ5 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` = 5)
 GCCVERSIONGT5 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \> 5)
 GCCVERSIONGTEQ7 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \>= 7)
 GCCVERSIONGTEQ9 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \>= 9)
+GCCMINORVERSIONGTEQ2 := $(shell expr `$(CC) -dumpversion | cut -f2 -d.` \>= 2)
 GCCMINORVERSIONGTEQ7 := $(shell expr `$(CC) -dumpversion | cut -f2 -d.` \>= 7)
 endif
 
@@ -570,20 +572,27 @@ ifeq ($(ARCH), zarch)
 DYNAMIC_CORE = ZARCH_GENERIC
 
 # Z13 is supported since gcc-5.2, gcc-6, and in RHEL 7.3 and newer
-GCC_GE_52 := $(subst 0,,$(shell expr `$(CC) -dumpversion` \>= "5.2"))
-
-ifeq ($(wildcard /etc/redhat-release), /etc/redhat-release)
-RHEL_WITH_Z13 := $(subst 0,,$(shell source /etc/os-release ; expr $$VERSION_ID \>= "7.3"))
+ifeq ($(GCCVERSIONGT5), 1)
+	ZARCH_SUPPORT_Z13 := 1
+else ifeq ($(GCCVERSIONEQ5), 1)
+ifeq ($(GCCMINORVERSIONGTEQ2), 1)
+	ZARCH_SUPPORT_Z13 := 1
+endif
 endif
 
-ifeq ($(or $(GCC_GE_52),$(RHEL_WITH_Z13)), 1)
+ifeq ($(wildcard /etc/redhat-release), /etc/redhat-release)
+ifeq ($(shell source /etc/os-release ; expr $$VERSION_ID \>= "7.3"), 1)
+	ZARCH_SUPPORT_Z13 := 1
+endif
+endif
+
+ifeq ($(ZARCH_SUPPORT_Z13), 1)
 DYNAMIC_CORE += Z13
 else
 $(info OpenBLAS: Not building Z13 kernels because gcc is older than 5.2 or 6.x)
 endif
 
-GCC_MAJOR_GE_7 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \>= 7)
-ifeq ($(GCC_MAJOR_GE_7), 1)
+ifeq ($(GCCVERSIONGTEQ7), 1)
 DYNAMIC_CORE += Z14
 else
 $(info OpenBLAS: Not building Z14 kernels because gcc is older than 7.x)

From 23892917667d87072eef2f18b6120f5d3c029f90 Mon Sep 17 00:00:00 2001
From: Marius Hillenbrand <mhillen@linux.ibm.com>
Date: Tue, 16 Jun 2020 14:45:09 +0200
Subject: [PATCH 22/48] Makefile.system: remove duplicate variable
 GCCVERSIONGT5

... to bring unified gcc version detection with common variables to the
one remaining spot in Makefile.system.

Signed-off-by: Marius Hillenbrand <mhillen@linux.ibm.com>
---
 Makefile.system | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Makefile.system b/Makefile.system
index 5738b14ec..63cdbccd8 100644
--- a/Makefile.system
+++ b/Makefile.system
@@ -606,7 +606,6 @@ ifneq ($(C_COMPILER), GCC)
 DYNAMIC_CORE += POWER9
 endif
 ifeq ($(C_COMPILER), GCC)
-GCCVERSIONGT5 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \> 5)
 ifeq ($(GCCVERSIONGT5), 1)
 DYNAMIC_CORE += POWER9
 else

From cde4690721ad54043239db000a46537a9169ca02 Mon Sep 17 00:00:00 2001
From: Marius Hillenbrand <mhillen@linux.ibm.com>
Date: Tue, 16 Jun 2020 15:45:59 +0200
Subject: [PATCH 23/48] RFC: Use gcc -dumpfullversion to get minor version with
 gcc-7.x

In gcc-7.1, the behavior of -dumpversion changed to be configured
at compile-time. On some distributions it only dumps the major version
(e.g., Ubuntu), so the current checks for the gcc minor version report
false negatives. As a replacement, gcc-7.1 introduced -dumpfullversion
which always prints the full version.

Update the gcc version detection in Makefile.system to employ
-dumpfullversion with gcc-7 and newer.

Posting this patch for discussion, since it emerged from discussions
around issue #2668 and PR #2669. It is not solving a problem right now,
but may be useful in the future.

Signed-off-by: Marius Hillenbrand <mhillen@linux.ibm.com>
---
 Makefile.system | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/Makefile.system b/Makefile.system
index 63cdbccd8..7e0b2757e 100644
--- a/Makefile.system
+++ b/Makefile.system
@@ -286,8 +286,15 @@ GCCVERSIONEQ5 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` = 5)
 GCCVERSIONGT5 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \> 5)
 GCCVERSIONGTEQ7 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \>= 7)
 GCCVERSIONGTEQ9 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \>= 9)
-GCCMINORVERSIONGTEQ2 := $(shell expr `$(CC) -dumpversion | cut -f2 -d.` \>= 2)
-GCCMINORVERSIONGTEQ7 := $(shell expr `$(CC) -dumpversion | cut -f2 -d.` \>= 7)
+# Note that the behavior of -dumpversion is compile-time-configurable for
+# gcc-7.x and newer. Use -dumpfullversion there
+ifeq ($(GCCVERSIONGTEQ7),1)
+	GCCDUMPVERSION_PARAM := -dumpfullversion
+else
+	GCCDUMPVERSION_PARAM := -dumpversion
+endif
+GCCMINORVERSIONGTEQ2 := $(shell expr `$(CC) $(GCCDUMPVERSION_PARAM) | cut -f2 -d.` \>= 2)
+GCCMINORVERSIONGTEQ7 := $(shell expr `$(CC) $(GCCDUMPVERSION_PARAM) | cut -f2 -d.` \>= 7)
 endif
 
 #

From 478898b37a91836a39d046f8c70e26c6c9fc06c7 Mon Sep 17 00:00:00 2001
From: Marius Hillenbrand <mhillen@linux.ibm.com>
Date: Wed, 17 Jun 2020 16:08:48 +0200
Subject: [PATCH 24/48] cpp_thread_test/dgemv: cap concurrency to number of hw
 threads on small systems

... instead of (number of hw threads - 4) to avoid invalid numbers on
smaller systems. Currently, systems with 4 or fewer CPUs (e.g., small CI
VMs) would fail the test. Fixes one of the issues discussed in #2668

Signed-off-by: Marius Hillenbrand <mhillen@linux.ibm.com>
---
 cpp_thread_test/dgemv_thread_safety.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp_thread_test/dgemv_thread_safety.cpp b/cpp_thread_test/dgemv_thread_safety.cpp
index 5411fec29..277594ff0 100644
--- a/cpp_thread_test/dgemv_thread_safety.cpp
+++ b/cpp_thread_test/dgemv_thread_safety.cpp
@@ -18,7 +18,7 @@ int main(int argc, char* argv[]){
 	uint32_t maxHwThreads = omp_get_max_threads();
 	
 	if (maxHwThreads < 52)
-		numConcurrentThreads = maxHwThreads -4;
+		numConcurrentThreads = maxHwThreads;
 	
 	if (argc > 4){
 		std::cout<<"ERROR: too many arguments for thread safety tester"<<std::endl;

From de838c38ef9db98a635de1dcedeba8b578ec87b3 Mon Sep 17 00:00:00 2001
From: Marius Hillenbrand <mhillen@linux.ibm.com>
Date: Wed, 17 Jun 2020 16:15:44 +0200
Subject: [PATCH 25/48] cpp_thread_test/dgemv: fail early if concurrency is
 zero

The two test cases dgemv_tester and dgemm_tester accept the degree of
concurrency as command line argument (amongst others). Fail early if
value 0 has been specified, instead of later with less-clear symptoms.

Signed-off-by: Marius Hillenbrand <mhillen@linux.ibm.com>
---
 cpp_thread_test/cpp_thread_safety_common.h | 8 ++++++++
 cpp_thread_test/dgemm_thread_safety.cpp    | 2 ++
 cpp_thread_test/dgemv_thread_safety.cpp    | 2 ++
 3 files changed, 12 insertions(+)

diff --git a/cpp_thread_test/cpp_thread_safety_common.h b/cpp_thread_test/cpp_thread_safety_common.h
index 60ab5bb2f..8005369a8 100644
--- a/cpp_thread_test/cpp_thread_safety_common.h
+++ b/cpp_thread_test/cpp_thread_safety_common.h
@@ -5,6 +5,14 @@ inline void pauser(){
     std::getline(std::cin, dummy);
 }
 
+void FailIfThreadsAreZero(uint32_t numConcurrentThreads) {
+	if(numConcurrentThreads == 0) {
+		std::cout<<"ERROR: Invalid parameter 0 for number of concurrent calls into OpenBLAS!"<<std::endl;
+		std::cout<<"CBLAS DGEMV thread safety test FAILED!"<<std::endl;
+		exit(-1);
+	}
+}
+
 void FillMatrices(std::vector<std::vector<double>>& matBlock, std::mt19937_64& PRNG, std::uniform_real_distribution<double>& rngdist, const blasint randomMatSize, const uint32_t numConcurrentThreads, const uint32_t numMat){
 	for(uint32_t i=0; i<numMat; i++){
 		for(uint32_t j = 0; j < static_cast<uint32_t>(randomMatSize*randomMatSize); j++){
diff --git a/cpp_thread_test/dgemm_thread_safety.cpp b/cpp_thread_test/dgemm_thread_safety.cpp
index 1c5287524..104c64f2a 100644
--- a/cpp_thread_test/dgemm_thread_safety.cpp
+++ b/cpp_thread_test/dgemm_thread_safety.cpp
@@ -46,6 +46,8 @@ int main(int argc, char* argv[]){
 	std::cout<<"Number of concurrent calls into OpenBLAS : "<<numConcurrentThreads<<'\n';
 	std::cout<<"Number of testing rounds : "<<numTestRounds<<'\n';
 	std::cout<<"This test will need "<<(static_cast<uint64_t>(randomMatSize*randomMatSize)*numConcurrentThreads*3*8)/static_cast<double>(1024*1024)<<" MiB of RAM\n"<<std::endl;
+
+	FailIfThreadsAreZero(numConcurrentThreads);
 	
 	std::cout<<"Initializing random number generator..."<<std::flush;
 	std::mt19937_64 PRNG = InitPRNG();
diff --git a/cpp_thread_test/dgemv_thread_safety.cpp b/cpp_thread_test/dgemv_thread_safety.cpp
index 277594ff0..20ea38138 100644
--- a/cpp_thread_test/dgemv_thread_safety.cpp
+++ b/cpp_thread_test/dgemv_thread_safety.cpp
@@ -47,6 +47,8 @@ int main(int argc, char* argv[]){
 	std::cout<<"Number of concurrent calls into OpenBLAS : "<<numConcurrentThreads<<'\n';
 	std::cout<<"Number of testing rounds : "<<numTestRounds<<'\n';
 	std::cout<<"This test will need "<<((static_cast<uint64_t>(randomMatSize*randomMatSize)*numConcurrentThreads*8)+(static_cast<uint64_t>(randomMatSize)*numConcurrentThreads*8*2))/static_cast<double>(1024*1024)<<" MiB of RAM\n"<<std::endl;
+
+	FailIfThreadsAreZero(numConcurrentThreads);
 	
 	std::cout<<"Initializing random number generator..."<<std::flush;
 	std::mt19937_64 PRNG = InitPRNG();

From 086d87a302ced18853b3cbf4f14cc22fd86837c8 Mon Sep 17 00:00:00 2001
From: wjc404 <52632443+wjc404@users.noreply.github.com>
Date: Sat, 20 Jun 2020 00:07:43 +0800
Subject: [PATCH 26/48] AVX512 dgemm tcopy_16 function

---
 kernel/x86_64/KERNEL.SKYLAKEX           |   5 +-
 kernel/x86_64/dgemm_tcopy_16_skylakex.c | 129 ++++++++++++++++++++++++
 2 files changed, 133 insertions(+), 1 deletion(-)
 create mode 100644 kernel/x86_64/dgemm_tcopy_16_skylakex.c

diff --git a/kernel/x86_64/KERNEL.SKYLAKEX b/kernel/x86_64/KERNEL.SKYLAKEX
index 65f031d03..9b8b84c30 100644
--- a/kernel/x86_64/KERNEL.SKYLAKEX
+++ b/kernel/x86_64/KERNEL.SKYLAKEX
@@ -14,7 +14,7 @@ STRSMKERNEL_RT =  ../generic/trsm_kernel_RT.c
 DGEMMKERNEL    =  dgemm_kernel_16x2_skylakex.c
 DTRMMKERNEL    =  dgemm_kernel_16x2_skylakex.c
 DGEMMINCOPY    =  ../generic/gemm_ncopy_16.c
-DGEMMITCOPY    =  ../generic/gemm_tcopy_16.c
+DGEMMITCOPY    =  dgemm_tcopy_16_skylakex.c
 DGEMMONCOPY    =  ../generic/gemm_ncopy_2.c
 DGEMMOTCOPY    =  ../generic/gemm_tcopy_2.c
 DTRSMKERNEL_RN =  ../generic/trsm_kernel_RN.c
@@ -24,3 +24,6 @@ DGEMM_BETA = dgemm_beta_skylakex.c
 
 CGEMMKERNEL    =  cgemm_kernel_8x2_skylakex.c
 ZGEMMKERNEL    =  zgemm_kernel_4x2_skylakex.c
+
+CSCALKERNEL    = ../arm/zscal.c
+ZSCALKERNEL    = ../arm/zscal.c
diff --git a/kernel/x86_64/dgemm_tcopy_16_skylakex.c b/kernel/x86_64/dgemm_tcopy_16_skylakex.c
new file mode 100644
index 000000000..a1da60f8f
--- /dev/null
+++ b/kernel/x86_64/dgemm_tcopy_16_skylakex.c
@@ -0,0 +1,129 @@
+#include <stdio.h>
+#include "common.h"
+#include <immintrin.h>
+
+int CNAME(BLASLONG dim_second, BLASLONG dim_first, double *src, BLASLONG lead_dim, double *dst){
+  double *src1, *src2, *src3, *src4, *dst1;
+  __m512d z1,z2,z3,z4,z5,z6,z7,z8; __m256d y1,y2,y3,y4; __m128d x1,x2,x3,x4; double s1,s2,s3,s4;
+  BLASLONG dim1_count, dim2_count, src_inc;
+  src_inc = 4 * lead_dim - dim_first;
+  src1 = src; src2 = src + lead_dim; src3 = src2 + lead_dim; src4 = src3 + lead_dim;
+  for(dim2_count=dim_second; dim2_count>3; dim2_count-=4){
+    dst1 = dst + 16 * (dim_second - dim2_count);
+    for(dim1_count=dim_first; dim1_count>15; dim1_count-=16){
+      z1 = _mm512_loadu_pd(src1); z2 = _mm512_loadu_pd(src1+8); src1 += 16;
+      z3 = _mm512_loadu_pd(src2); z4 = _mm512_loadu_pd(src2+8); src2 += 16;
+      z5 = _mm512_loadu_pd(src3); z6 = _mm512_loadu_pd(src3+8); src3 += 16;
+      z7 = _mm512_loadu_pd(src4); z8 = _mm512_loadu_pd(src4+8); src4 += 16;
+      _mm512_storeu_pd(dst1+ 0,z1); _mm512_storeu_pd(dst1+ 8,z2);
+      _mm512_storeu_pd(dst1+16,z3); _mm512_storeu_pd(dst1+24,z4);
+      _mm512_storeu_pd(dst1+32,z5); _mm512_storeu_pd(dst1+40,z6);
+      _mm512_storeu_pd(dst1+48,z7); _mm512_storeu_pd(dst1+56,z8); dst1 += 16 * dim_second;
+    }
+    dst1 -= 8 * (dim_second - dim2_count);
+    if(dim1_count>7){
+      z1 = _mm512_loadu_pd(src1); src1 += 8;
+      z2 = _mm512_loadu_pd(src2); src2 += 8;
+      z3 = _mm512_loadu_pd(src3); src3 += 8;
+      z4 = _mm512_loadu_pd(src4); src4 += 8;
+      _mm512_storeu_pd(dst1+ 0,z1); _mm512_storeu_pd(dst1+ 8,z2);
+      _mm512_storeu_pd(dst1+16,z3); _mm512_storeu_pd(dst1+24,z4); dst1 += 8 * dim_second;
+      dim1_count -= 8;
+    }
+    dst1 -= 4 * (dim_second - dim2_count);
+    if(dim1_count>3){
+      y1 = _mm256_loadu_pd(src1); src1 += 4;
+      y2 = _mm256_loadu_pd(src2); src2 += 4;
+      y3 = _mm256_loadu_pd(src3); src3 += 4;
+      y4 = _mm256_loadu_pd(src4); src4 += 4;
+      _mm256_storeu_pd(dst1+ 0,y1); _mm256_storeu_pd(dst1+ 4,y2);
+      _mm256_storeu_pd(dst1+ 8,y3); _mm256_storeu_pd(dst1+12,y4); dst1 += 4 * dim_second;
+      dim1_count -= 4;
+    }
+    dst1 -= 2 * (dim_second - dim2_count);
+    if(dim1_count>1){
+      x1 = _mm_loadu_pd(src1); src1 += 2;
+      x2 = _mm_loadu_pd(src2); src2 += 2;
+      x3 = _mm_loadu_pd(src3); src3 += 2;
+      x4 = _mm_loadu_pd(src4); src4 += 2;
+      _mm_storeu_pd(dst1+0,x1); _mm_storeu_pd(dst1+2,x2);
+      _mm_storeu_pd(dst1+4,x3); _mm_storeu_pd(dst1+6,x4); dst1 += 2 * dim_second;
+      dim1_count -= 2;
+    }
+    dst1 -= dim_second - dim2_count;
+    if(dim1_count>0){
+      s1 = *src1; src1++; s2 = *src2; src2++; s3 = *src3; src3++; s4 = *src4; src4++;
+      dst1[0] = s1; dst1[1] = s2; dst1[2] = s3; dst1[3] = s4;
+    }
+    src1 += src_inc; src2 += src_inc; src3 += src_inc; src4 += src_inc;
+  }
+  src_inc -= 2 * lead_dim;
+  for(; dim2_count>1; dim2_count-=2){
+    dst1 = dst + 16 * (dim_second - dim2_count);
+    for(dim1_count=dim_first; dim1_count>15; dim1_count-=16){
+      z1 = _mm512_loadu_pd(src1); z2 = _mm512_loadu_pd(src1+8); src1 += 16;
+      z3 = _mm512_loadu_pd(src2); z4 = _mm512_loadu_pd(src2+8); src2 += 16;
+      _mm512_storeu_pd(dst1+ 0,z1); _mm512_storeu_pd(dst1+ 8,z2);
+      _mm512_storeu_pd(dst1+16,z3); _mm512_storeu_pd(dst1+24,z4); dst1 += 16 * dim_second;
+    }
+    dst1 -= 8 * (dim_second - dim2_count);
+    if(dim1_count>7){
+      z1 = _mm512_loadu_pd(src1); src1 += 8;
+      z2 = _mm512_loadu_pd(src2); src2 += 8;
+      _mm512_storeu_pd(dst1+ 0,z1); _mm512_storeu_pd(dst1+ 8,z2); dst1 += 8 * dim_second;
+      dim1_count -= 8;
+    }
+    dst1 -= 4 * (dim_second - dim2_count);
+    if(dim1_count>3){
+      y1 = _mm256_loadu_pd(src1); src1 += 4;
+      y2 = _mm256_loadu_pd(src2); src2 += 4;
+      _mm256_storeu_pd(dst1+ 0,y1); _mm256_storeu_pd(dst1+ 4,y2); dst1 += 4 * dim_second;
+      dim1_count -= 4;
+    }
+    dst1 -= 2 * (dim_second - dim2_count);
+    if(dim1_count>1){
+      x1 = _mm_loadu_pd(src1); src1 += 2;
+      x2 = _mm_loadu_pd(src2); src2 += 2;
+      _mm_storeu_pd(dst1+0,x1); _mm_storeu_pd(dst1+2,x2); dst1 += 2 * dim_second;
+      dim1_count -= 2;
+    }
+    dst1 -= dim_second - dim2_count;
+    if(dim1_count>0){
+      s1 = *src1; src1++; s2 = *src2; src2++;
+      dst1[0] = s1; dst1[1] = s2;
+    }
+    src1 += src_inc; src2 += src_inc;
+  }
+  src_inc -= lead_dim;
+  for(; dim2_count>0; dim2_count--){
+    dst1 = dst + 16 * (dim_second - dim2_count);
+    for(dim1_count=dim_first; dim1_count>15; dim1_count-=16){
+      z1 = _mm512_loadu_pd(src1); z2 = _mm512_loadu_pd(src1+8); src1 += 16;
+      _mm512_storeu_pd(dst1+ 0,z1); _mm512_storeu_pd(dst1+ 8,z2); dst1 += 16 * dim_second;
+    }
+    dst1 -= 8 * (dim_second - dim2_count);
+    if(dim1_count>7){
+      z1 = _mm512_loadu_pd(src1); src1 += 8;
+      _mm512_storeu_pd(dst1+ 0,z1); dst1 += 8 * dim_second;
+      dim1_count -= 8;
+    }
+    dst1 -= 4 * (dim_second - dim2_count);
+    if(dim1_count>3){
+      y1 = _mm256_loadu_pd(src1); src1 += 4;
+      _mm256_storeu_pd(dst1+ 0,y1); dst1 += 4 * dim_second;
+      dim1_count -= 4;
+    }
+    dst1 -= 2 * (dim_second - dim2_count);
+    if(dim1_count>1){
+      x1 = _mm_loadu_pd(src1); src1 += 2;
+      _mm_storeu_pd(dst1+0,x1); dst1 += 2 * dim_second;
+      dim1_count -= 2;
+    }
+    dst1 -= dim_second - dim2_count;
+    if(dim1_count>0){
+      s1 = *src1; src1++;
+      dst1[0] = s1;
+    }
+    src1 += src_inc;
+  }
+}

From e6b92750349e273d6bb7b28673f10c39cff90c26 Mon Sep 17 00:00:00 2001
From: User User-User <user@localhost>
Date: Wed, 24 Jun 2020 09:12:23 +0300
Subject: [PATCH 27/48] address vs2019 C4293

---
 driver/others/dynamic.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/driver/others/dynamic.c b/driver/others/dynamic.c
index 1bf0e4a6d..38eb76643 100644
--- a/driver/others/dynamic.c
+++ b/driver/others/dynamic.c
@@ -332,7 +332,7 @@ int support_avx512(){
   if((ebx & (1<<7)) == 0){
       ret=0;  //OS does not even support AVX2
   }
-  if((ebx & (1<<31)) != 0){
+  if((ebx & (1u<<31)) != 0){
     xgetbv(0, &eax, &edx);
     if((eax & 0xe0) == 0xe0)
       ret=1;  //OS supports AVX512VL
@@ -632,7 +632,7 @@ static gotoblas_t *get_coretype(void){
         cpuid(0x80000000, &eax, &ebx, &ecx, &edx);
         if ( (eax & 0xffff)  >= 0x01) {
             cpuid(0x80000001, &eax, &ebx, &ecx, &edx);
-            if ((edx & (1 << 30)) == 0 || (edx & (1 << 31)) == 0)
+            if ((edx & (1 << 30)) == 0 || (edx & (1u << 31)) == 0)
               return NULL;
           }
         else

From df4ade070f745d5c542067b5fd5bab3e29d39dcf Mon Sep 17 00:00:00 2001
From: Kavana Bhat <Kavana.bhat@in.ibm.com>
Date: Wed, 24 Jun 2020 04:25:47 -0500
Subject: [PATCH 28/48] Fix for #2671

---
 kernel/Makefile.L3 | 94 +++++++++++++++++++++++-----------------------
 1 file changed, 47 insertions(+), 47 deletions(-)

diff --git a/kernel/Makefile.L3 b/kernel/Makefile.L3
index 0cb02ef85..86772cb22 100644
--- a/kernel/Makefile.L3
+++ b/kernel/Makefile.L3
@@ -483,7 +483,7 @@ $(KDIR)$(SHGEMMONCOPYOBJ) : $(KERNELDIR)/$(SHGEMMONCOPY)
 $(KDIR)$(SHGEMMOTCOPYOBJ) : $(KERNELDIR)/$(SHGEMMOTCOPY)
 
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DHALF -UDOUBLE -UCOMPLEX $< -o shgemmotcopy.s
+	$(CC) $(CFLAGS) -S -DHALF -UDOUBLE -UCOMPLEX $< -o - > shgemmotcopy.s
 	m4 shgemmotcopy.s > shgemmotcopy_nomacros.s
 	$(CC) $(CFLAGS) -c -DHALF -UDOUBLE -UCOMPLEX shgemmotcopy_nomacros.s -o $@
 	rm shgemmotcopy.s shgemmotcopy_nomacros.s
@@ -498,7 +498,7 @@ $(KDIR)$(SHGEMMINCOPYOBJ) : $(KERNELDIR)/$(SHGEMMINCOPY)
 
 $(KDIR)$(SHGEMMITCOPYOBJ) : $(KERNELDIR)/$(SHGEMMITCOPY)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DHALF -UDOUBLE -UCOMPLEX $< -o shgemmitcopy.s
+	$(CC) $(CFLAGS) -S -DHALF -UDOUBLE -UCOMPLEX $< -o - > shgemmitcopy.s
 	m4 shgemmitcopy.s > shgemmitcopy_nomacros.s
 	$(CC) $(CFLAGS) -c -DHALF -UDOUBLE -UCOMPLEX shgemmitcopy_nomacros.s -o $@
 	rm shgemmitcopy.s shgemmitcopy_nomacros.s
@@ -514,7 +514,7 @@ $(KDIR)$(SGEMMONCOPYOBJ) : $(KERNELDIR)/$(SGEMMONCOPY)
 
 $(KDIR)$(SGEMMOTCOPYOBJ) : $(KERNELDIR)/$(SGEMMOTCOPY)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -UDOUBLE -UCOMPLEX $< -o sgemmotcopy.s	
+	$(CC) $(CFLAGS) -S -UDOUBLE -UCOMPLEX $< -o - > sgemmotcopy.s	
 	m4 sgemmotcopy.s > sgemmotcopy_nomacros.s
 	$(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX sgemmotcopy_nomacros.s -o $@
 	rm sgemmotcopy.s sgemmotcopy_nomacros.s
@@ -530,7 +530,7 @@ $(KDIR)$(SGEMMINCOPYOBJ) : $(KERNELDIR)/$(SGEMMINCOPY)
 
 $(KDIR)$(SGEMMITCOPYOBJ) : $(KERNELDIR)/$(SGEMMITCOPY)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -UDOUBLE -UCOMPLEX $< -o sgemmitcopy.s	
+	$(CC) $(CFLAGS) -S -UDOUBLE -UCOMPLEX $< -o - > sgemmitcopy.s	
 	m4 sgemmitcopy.s > sgemmitcopy_nomacros.s
 	$(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX sgemmitcopy_nomacros.s -o $@
 	rm sgemmitcopy.s sgemmitcopy_nomacros.s
@@ -542,7 +542,7 @@ endif
 
 $(KDIR)$(DGEMMONCOPYOBJ) : $(KERNELDIR)/$(DGEMMONCOPY)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DDOUBLE -UCOMPLEX $< -o dgemm_ncopy.s
+	$(CC) $(CFLAGS) -S -DDOUBLE -UCOMPLEX $< -o - > dgemm_ncopy.s
 	m4 dgemm_ncopy.s > dgemm_ncopy_nomacros.s
 	$(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX dgemm_ncopy_nomacros.s -o $@
 	rm dgemm_ncopy.s dgemm_ncopy_nomacros.s
@@ -560,7 +560,7 @@ $(KDIR)$(DGEMMINCOPYOBJ) : $(KERNELDIR)/$(DGEMMINCOPY)
 
 $(KDIR)$(DGEMMITCOPYOBJ) : $(KERNELDIR)/$(DGEMMITCOPY)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DDOUBLE -UCOMPLEX $< -o dgemm_itcopy.s
+	$(CC) $(CFLAGS) -S -DDOUBLE -UCOMPLEX $< -o - > dgemm_itcopy.s
 	m4 dgemm_itcopy.s > dgemm_itcopy_nomacros.s
 	$(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX dgemm_itcopy_nomacros.s -o $@
 	rm dgemm_itcopy.s dgemm_itcopy_nomacros.s
@@ -603,7 +603,7 @@ $(KDIR)$(CGEMMINCOPYOBJ) : $(KERNELDIR)/$(CGEMMINCOPY)
 
 $(KDIR)$(CGEMMITCOPYOBJ) : $(KERNELDIR)/$(CGEMMITCOPY)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -UDOUBLE -UCOMPLEX -E $< -o cgemm_itcopy.s
+	$(CC) $(CFLAGS) -UDOUBLE -UCOMPLEX -S $< -o - > cgemm_itcopy.s
 	m4 cgemm_itcopy.s > cgemm_itcopy_nomacros.s
 	$(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX cgemm_itcopy_nomacros.s -o $@
 	rm cgemm_itcopy.s cgemm_itcopy_nomacros.s
@@ -626,7 +626,7 @@ $(KDIR)$(ZGEMMINCOPYOBJ) : $(KERNELDIR)/$(ZGEMMINCOPY)
 
 $(KDIR)$(ZGEMMITCOPYOBJ) : $(KERNELDIR)/$(ZGEMMITCOPY)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DDOUBLE -UCOMPLEX $< -o zgemm_itcopy.s
+	$(CC) $(CFLAGS) -S -DDOUBLE -UCOMPLEX $< -o - > zgemm_itcopy.s
 	m4 zgemm_itcopy.s > zgemm_itcopy_nomacros.s
 	$(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX zgemm_itcopy_nomacros.s -o $@
 	rm zgemm_itcopy.s zgemm_itcopy_nomacros.s
@@ -658,7 +658,7 @@ endif
 
 $(KDIR)sgemm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) $(SGEMMDEPEND)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -UDOUBLE -UCOMPLEX  $< -o sgemm_kernel$(TSUFFIX).s
+	$(CC) $(CFLAGS) -S -UDOUBLE -UCOMPLEX  $< -o - > sgemm_kernel$(TSUFFIX).s
 	m4 sgemm_kernel$(TSUFFIX).s > sgemm_kernel$(TSUFFIX)_nomacros.s
 	$(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX sgemm_kernel$(TSUFFIX)_nomacros.s -o $@
 	rm sgemm_kernel$(TSUFFIX).s sgemm_kernel$(TSUFFIX)_nomacros.s
@@ -670,7 +670,7 @@ ifeq ($(BUILD_HALF), 1)
 
 $(KDIR)shgemm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SHGEMMKERNEL) $(SHGEMMDEPEND)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DHALF -UDOUBLE -UCOMPLEX  $< -o shgemm_kernel$(TSUFFIX).s
+	$(CC) $(CFLAGS) -S -DHALF -UDOUBLE -UCOMPLEX  $< -o - > shgemm_kernel$(TSUFFIX).s
 	m4 shgemm_kernel$(TSUFFIX).s > shgemm_kernel$(TSUFFIX)_nomacros.s
 	$(CC) $(CFLAGS) -c -DHALF -UDOUBLE -UCOMPLEX shgemm_kernel$(TSUFFIX)_nomacros.s -o $@
 	rm shgemm_kernel$(TSUFFIX).s shgemm_kernel$(TSUFFIX)_nomacros.s
@@ -681,7 +681,7 @@ endif
 
 $(KDIR)dgemm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) $(DGEMMDEPEND)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DDOUBLE -UCOMPLEX $< -o dgemm_kernel$(TSUFFIX).s
+	$(CC) $(CFLAGS) -S -DDOUBLE -UCOMPLEX $< -o - > dgemm_kernel$(TSUFFIX).s
 	m4 dgemm_kernel$(TSUFFIX).s > dgemm_kernel$(TSUFFIX)_nomacros.s
 	$(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX dgemm_kernel$(TSUFFIX)_nomacros.s -o $@
 	rm dgemm_kernel$(TSUFFIX).s dgemm_kernel$(TSUFFIX)_nomacros.s
@@ -694,7 +694,7 @@ $(KDIR)qgemm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(QGEMMKERNEL) $(QGEMMDEP
 
 $(KDIR)cgemm_kernel_n$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -UDOUBLE -DCOMPLEX -DNN $< -o cgemm_kernel_n.s
+	$(CC) $(CFLAGS) -S -UDOUBLE -DCOMPLEX -DNN $< -o - > cgemm_kernel_n.s
 	m4 cgemm_kernel_n.s > cgemm_kernel_n_nomacros.s
 	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNN cgemm_kernel_n_nomacros.s -o $@
 	rm cgemm_kernel_n.s cgemm_kernel_n_nomacros.s
@@ -704,7 +704,7 @@ endif
 
 $(KDIR)cgemm_kernel_l$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -UDOUBLE -DCOMPLEX -DCN $< -o cgemm_kernel_l.s
+	$(CC) $(CFLAGS) -S -UDOUBLE -DCOMPLEX -DCN $< -o - > cgemm_kernel_l.s
 	m4 cgemm_kernel_l.s > cgemm_kernel_l_nomacros.s
 	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCN cgemm_kernel_l_nomacros.s -o $@
 	rm cgemm_kernel_l.s cgemm_kernel_l_nomacros.s
@@ -714,7 +714,7 @@ endif
 
 $(KDIR)cgemm_kernel_r$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -UDOUBLE -DCOMPLEX -DNC  $< -o cgemm_kernel_r.s
+	$(CC) $(CFLAGS) -S -UDOUBLE -DCOMPLEX -DNC  $< -o - > cgemm_kernel_r.s
 	m4 cgemm_kernel_r.s > cgemm_kernel_r_nomacros.s
 	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNC cgemm_kernel_r_nomacros.s -o $@
 	rm cgemm_kernel_r.s cgemm_kernel_r_nomacros.s
@@ -724,7 +724,7 @@ endif
 
 $(KDIR)cgemm_kernel_b$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -UDOUBLE -DCOMPLEX -DCC $< -o cgemm_kernel_b.s
+	$(CC) $(CFLAGS) -S -UDOUBLE -DCOMPLEX -DCC $< -o - > cgemm_kernel_b.s
 	m4 cgemm_kernel_b.s > cgemm_kernel_b_nomacros.s
 	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCC cgemm_kernel_b_nomacros.s -o $@
 	rm cgemm_kernel_b.s cgemm_kernel_b_nomacros.s
@@ -734,7 +734,7 @@ endif
 
 $(KDIR)zgemm_kernel_n$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DDOUBLE -DCOMPLEX -DNN $< -o zgemm_kernel_n.s
+	$(CC) $(CFLAGS) -S -DDOUBLE -DCOMPLEX -DNN $< -o - > zgemm_kernel_n.s
 	m4 zgemm_kernel_n.s > zgemm_kernel_n_nomacros.s
 	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNN zgemm_kernel_n_nomacros.s -o $@
 	rm zgemm_kernel_n.s zgemm_kernel_n_nomacros.s
@@ -744,7 +744,7 @@ endif
 
 $(KDIR)zgemm_kernel_l$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DDOUBLE -DCOMPLEX -DCN $< -o zgemm_kernel_l.s
+	$(CC) $(CFLAGS) -S -DDOUBLE -DCOMPLEX -DCN $< -o - > zgemm_kernel_l.s
 	m4 zgemm_kernel_l.s > zgemm_kernel_l_nomacros.s
 	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DCN zgemm_kernel_l_nomacros.s -o $@
 	rm zgemm_kernel_l.s zgemm_kernel_l_nomacros.s
@@ -754,7 +754,7 @@ endif
 
 $(KDIR)zgemm_kernel_r$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DDOUBLE -DCOMPLEX -DNC $< -o zgemm_kernel_r.s
+	$(CC) $(CFLAGS) -S -DDOUBLE -DCOMPLEX -DNC $< -o - > zgemm_kernel_r.s
 	m4 zgemm_kernel_r.s > zgemm_kernel_r_nomacros.s
 	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNC zgemm_kernel_r_nomacros.s -o $@
 	rm zgemm_kernel_r.s zgemm_kernel_r_nomacros.s
@@ -764,7 +764,7 @@ endif
 
 $(KDIR)zgemm_kernel_b$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DDOUBLE -DCOMPLEX -DCC $< -o zgemm_kernel_b.s
+	$(CC) $(CFLAGS) -S -DDOUBLE -DCOMPLEX -DCC $< -o - > zgemm_kernel_b.s
 	m4 zgemm_kernel_b.s > zgemm_kernel_b_nomacros.s
 	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DCC zgemm_kernel_b_nomacros.s -o $@
 	rm zgemm_kernel_b.s zgemm_kernel_b_nomacros.s
@@ -788,7 +788,7 @@ $(KDIR)xgemm_kernel_b$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(XGEMMKERNEL) $(XGEMMD
 ifdef USE_TRMM
 $(KDIR)strmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(STRMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -UDOUBLE -UCOMPLEX -DLEFT -UTRANSA $< -o strmmkernel_ln.s	
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -UDOUBLE -UCOMPLEX -DLEFT -UTRANSA $< -o - > strmmkernel_ln.s	
 	m4 strmmkernel_ln.s > strmmkernel_ln_nomacros.s
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -DLEFT -UTRANSA strmmkernel_ln_nomacros.s -o $@
 	rm strmmkernel_ln.s strmmkernel_ln_nomacros.s
@@ -798,7 +798,7 @@ endif
 
 $(KDIR)strmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(STRMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -UDOUBLE -UCOMPLEX -DLEFT -DTRANSA $< -o strmmkernel_lt.s	
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -UDOUBLE -UCOMPLEX -DLEFT -DTRANSA $< -o - > strmmkernel_lt.s	
 	m4 strmmkernel_lt.s > strmmkernel_lt_nomacros.s
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -DLEFT -DTRANSA strmmkernel_lt_nomacros.s -o $@
 	rm strmmkernel_lt.s strmmkernel_lt_nomacros.s
@@ -808,7 +808,7 @@ endif
 
 $(KDIR)strmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(STRMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -UTRANSA $< -o strmmkernel_rn.s	
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -UTRANSA $< -o - > strmmkernel_rn.s	
 	m4 strmmkernel_rn.s > strmmkernel_rn_nomacros.s
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -UTRANSA strmmkernel_rn_nomacros.s -o $@
 	rm strmmkernel_rn.s strmmkernel_rn_nomacros.s
@@ -818,7 +818,7 @@ endif
 
 $(KDIR)strmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(STRMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o strmm_kernel_rt.s	
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o - > strmm_kernel_rt.s	
 	m4 strmm_kernel_rt.s > strmm_kernel_rt_nomacros.s
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -DTRANSA strmm_kernel_rt_nomacros.s -o $@
 	rm strmm_kernel_rt.s strmm_kernel_rt_nomacros.s
@@ -828,7 +828,7 @@ endif
 
 $(KDIR)dtrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DTRMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -DDOUBLE -UCOMPLEX -DLEFT -UTRANSA $< -o dtrmm_kernel_ln.s
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -DDOUBLE -UCOMPLEX -DLEFT -UTRANSA $< -o - > dtrmm_kernel_ln.s
 	m4 dtrmm_kernel_ln.s > dtrmm_kernel_ln_nomacros.s
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -DLEFT -UTRANSA dtrmm_kernel_ln_nomacros.s -o $@
 	rm dtrmm_kernel_ln.s dtrmm_kernel_ln_nomacros.s
@@ -838,7 +838,7 @@ endif
 
 $(KDIR)dtrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DTRMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -DDOUBLE -UCOMPLEX -DLEFT -DTRANSA $< -o dtrmm_kernel_lt.s
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -DDOUBLE -UCOMPLEX -DLEFT -DTRANSA $< -o - > dtrmm_kernel_lt.s
 	m4 dtrmm_kernel_lt.s > dtrmm_kernel_lt_nomacros.s
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -DLEFT -DTRANSA dtrmm_kernel_lt_nomacros.s -o $@
 	rm dtrmm_kernel_lt.s dtrmm_kernel_lt_nomacros.s
@@ -848,7 +848,7 @@ endif
 
 $(KDIR)dtrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DTRMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -DDOUBLE -UCOMPLEX -ULEFT -UTRANSA $< -o dtrmm_kernel_rn.s
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -DDOUBLE -UCOMPLEX -ULEFT -UTRANSA $< -o - > dtrmm_kernel_rn.s
 	m4 dtrmm_kernel_rn.s > dtrmm_kernel_rn_nomacros.s
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -ULEFT -UTRANSA dtrmm_kernel_rn_nomacros.s -o $@
 	rm dtrmm_kernel_rn.s dtrmm_kernel_rn_nomacros.s
@@ -858,7 +858,7 @@ endif
 
 $(KDIR)dtrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DTRMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -DDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o dtrmm_kernel_rt.s
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -DDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o - > dtrmm_kernel_rt.s
 	m4 dtrmm_kernel_rt.s > dtrmm_kernel_rt_nomacros.s
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -ULEFT -DTRANSA dtrmm_kernel_rt_nomacros.s -o $@
 	rm dtrmm_kernel_rt.s dtrmm_kernel_rt_nomacros.s
@@ -880,7 +880,7 @@ $(KDIR)qtrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(QGEMMKERNEL)
 
 $(KDIR)ctrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -UTRANSA -UCONJ -DNN  $< -o ctrmm_kernel_ln.s
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -UTRANSA -UCONJ -DNN  $< -o - > ctrmm_kernel_ln.s
 	m4 ctrmm_kernel_ln.s > ctrmm_kernel_ln_nomacros.s
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -UTRANSA -UCONJ -DNN ctrmm_kernel_ln_nomacros.s -o $@
 	rm ctrmm_kernel_ln.s ctrmm_kernel_ln_nomacros.s
@@ -890,7 +890,7 @@ endif
 
 $(KDIR)ctrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -DTRANSA -UCONJ -DNN $< -o ctrmm_kernel_lt.s
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -DTRANSA -UCONJ -DNN $< -o - > ctrmm_kernel_lt.s
 	m4 ctrmm_kernel_lt.s > ctrmm_kernel_lt_nomacros.s
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -DTRANSA -UCONJ -DNN ctrmm_kernel_lt_nomacros.s -o $@
 	rm ctrmm_kernel_lt.s ctrmm_kernel_lt_nomacros.s
@@ -900,7 +900,7 @@ endif
 
 $(KDIR)ctrmm_kernel_LR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -UTRANSA -DCONJ -DCN $< -o ctrmm_kernel_lr.s
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -UTRANSA -DCONJ -DCN $< -o - > ctrmm_kernel_lr.s
 	m4 ctrmm_kernel_lr.s > ctrmm_kernel_lr_nomacros.s
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -UTRANSA -DCONJ -DCN  ctrmm_kernel_lr_nomacros.s -o $@
 	rm ctrmm_kernel_lr.s ctrmm_kernel_lr_nomacros.s
@@ -910,7 +910,7 @@ endif
 
 $(KDIR)ctrmm_kernel_LC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -DTRANSA -DCONJ -DCN $< -o ctrmm_kernel_lc.s
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -DTRANSA -DCONJ -DCN $< -o - > ctrmm_kernel_lc.s
 	m4 ctrmm_kernel_lc.s > ctrmm_kernel_lc_nomacros.s
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -DTRANSA -DCONJ -DCN ctrmm_kernel_lc_nomacros.s -o $@
 	rm ctrmm_kernel_lc_nomacros.s ctrmm_kernel_lc.s
@@ -920,7 +920,7 @@ endif
 
 $(KDIR)ctrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -UTRANSA -UCONJ -DNN $< -o ctrmm_kernel_rn.s
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -UTRANSA -UCONJ -DNN $< -o - > ctrmm_kernel_rn.s
 	m4 ctrmm_kernel_rn.s > ctrmm_kernel_rn_nomacros.s
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -UTRANSA -UCONJ -DNN ctrmm_kernel_rn_nomacros.s -o $@
 	rm ctrmm_kernel_rn.s ctrmm_kernel_rn_nomacros.s
@@ -930,7 +930,7 @@ endif
 
 $(KDIR)ctrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -DTRANSA -UCONJ -DNN $< -o ctrmm_kernel_rt.s
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -DTRANSA -UCONJ -DNN $< -o - > ctrmm_kernel_rt.s
 	m4 ctrmm_kernel_rt.s > ctrmm_kernel_rt_nomacros.s
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -DTRANSA -UCONJ -DNN ctrmm_kernel_rt_nomacros.s -o $@
 	rm ctrmm_kernel_rt.s ctrmm_kernel_rt_nomacros.s
@@ -940,7 +940,7 @@ endif
 
 $(KDIR)ctrmm_kernel_RR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -UTRANSA -DCONJ -DNC $< -o ctrmm_kernel_rr.s
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -UTRANSA -DCONJ -DNC $< -o - > ctrmm_kernel_rr.s
 	m4 ctrmm_kernel_rr.s > ctrmm_kernel_rr_nomacros.s
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -UTRANSA -DCONJ -DNC ctrmm_kernel_rr_nomacros.s -o $@
 	rm ctrmm_kernel_rr.s ctrmm_kernel_rr_nomacros.s
@@ -950,7 +950,7 @@ endif
 
 $(KDIR)ctrmm_kernel_RC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -DTRANSA -DCONJ -DNC $< -o ctrmm_kernel_RC.s
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -DTRANSA -DCONJ -DNC $< -o - > ctrmm_kernel_RC.s
 	m4 ctrmm_kernel_RC.s > ctrmm_kernel_RC_nomacros.s
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -DTRANSA -DCONJ -DNC ctrmm_kernel_RC_nomacros.s -o $@
 	rm ctrmm_kernel_RC.s ctrmm_kernel_RC_nomacros.s
@@ -960,7 +960,7 @@ endif
 
 $(KDIR)ztrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -UTRANSA -UCONJ -DNN $< -o ztrmm_kernel_ln.s
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -UTRANSA -UCONJ -DNN $< -o - > ztrmm_kernel_ln.s
 	m4 ztrmm_kernel_ln.s > ztrmm_kernel_ln_nomacros.s
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -UTRANSA -UCONJ -DNN ztrmm_kernel_ln_nomacros.s -o $@
 	rm ztrmm_kernel_ln.s ztrmm_kernel_ln_nomacros.s
@@ -970,7 +970,7 @@ endif
 
 $(KDIR)ztrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -DTRANSA -UCONJ -DNN $< -o ztrmm_kernel_lt.s
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -DTRANSA -UCONJ -DNN $< -o - > ztrmm_kernel_lt.s
 	m4 ztrmm_kernel_lt.s > ztrmm_kernel_lt_nomacros.s
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -DTRANSA -UCONJ -DNN ztrmm_kernel_lt_nomacros.s -o $@
 	rm ztrmm_kernel_lt.s ztrmm_kernel_lt_nomacros.s
@@ -980,7 +980,7 @@ endif
 
 $(KDIR)ztrmm_kernel_LR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -UTRANSA -DCONJ -DCN $< -o ztrmm_kernel_lr.s
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -UTRANSA -DCONJ -DCN $< -o - > ztrmm_kernel_lr.s
 	m4 ztrmm_kernel_lr.s > ztrmm_kernel_lr_nomacros.s
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -UTRANSA -DCONJ -DCN ztrmm_kernel_lr_nomacros.s -o $@
 	rm ztrmm_kernel_lr.s ztrmm_kernel_lr_nomacros.s
@@ -990,7 +990,7 @@ endif
 
 $(KDIR)ztrmm_kernel_LC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -DTRANSA -DCONJ -DCN $< -o ztrmm_kernel_lc.s
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -DTRANSA -DCONJ -DCN $< -o - > ztrmm_kernel_lc.s
 	m4 ztrmm_kernel_lc.s >ztrmm_kernel_lc_nomacros.s
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -DTRANSA -DCONJ -DCN ztrmm_kernel_lc_nomacros.s -o $@
 	rm ztrmm_kernel_lc.s ztrmm_kernel_lc_nomacros.s 
@@ -1000,7 +1000,7 @@ endif
 
 $(KDIR)ztrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -UTRANSA -UCONJ -DNN $< -o ztrmm_kernel_rn.s
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -UTRANSA -UCONJ -DNN $< -o - > ztrmm_kernel_rn.s
 	m4 ztrmm_kernel_rn.s > ztrmm_kernel_rn_nomacros.s
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -UTRANSA -UCONJ -DNN ztrmm_kernel_rn_nomacros.s -o $@
 	rm ztrmm_kernel_rn.s ztrmm_kernel_rn_nomacros.s
@@ -1010,7 +1010,7 @@ endif
 
 $(KDIR)ztrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -DTRANSA -UCONJ -DNN $< -o ztrmm_kernel_rt.s
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -DTRANSA -UCONJ -DNN $< -o - > ztrmm_kernel_rt.s
 	m4 ztrmm_kernel_rt.s > ztrmm_kernel_rt_nomacros.s
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -DTRANSA -UCONJ -DNN ztrmm_kernel_rt_nomacros.s -o $@
 	rm ztrmm_kernel_rt.s ztrmm_kernel_rt_nomacros.s
@@ -1020,7 +1020,7 @@ endif
 
 $(KDIR)ztrmm_kernel_RR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -UTRANSA -DCONJ -DNC $< -o ztrmm_kernel_rr.s
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -UTRANSA -DCONJ -DNC $< -o - > ztrmm_kernel_rr.s
 	m4 ztrmm_kernel_rr.s > ztrmm_kernel_rr_nomacros.s
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -UTRANSA -DCONJ -DNC ztrmm_kernel_rr_nomacros.s -o $@
 	rm ztrmm_kernel_rr.s ztrmm_kernel_rr_nomacros.s
@@ -1030,7 +1030,7 @@ endif
 
 $(KDIR)ztrmm_kernel_RC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -DTRANSA -DCONJ -DNC $< -o ztrmm_kernel_rc.s
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -DTRANSA -DCONJ -DNC $< -o - > ztrmm_kernel_rc.s
 	m4 ztrmm_kernel_rc.s > ztrmm_kernel_rc_nomacros.s
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -DTRANSA -DCONJ -DNC ztrmm_kernel_rc_nomacros.s -o $@
 	rm ztrmm_kernel_rc.s ztrmm_kernel_rc_nomacros.s
@@ -1050,7 +1050,7 @@ $(KDIR)strmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL)
 
 $(KDIR)strmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o strmm_kernel_rt.s	
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o - > strmm_kernel_rt.s	
 	m4 strmm_kernel_rt.s > strmm_kernel_rt_nomacros.s
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -DTRANSA strmm_kernel_rt_nomacros.s -o $@
 	rm strmm_kernel_rt.s strmm_kernel_rt_nomacros.s
@@ -1184,7 +1184,7 @@ $(KDIR)dtrsm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DTRSMKERNEL_LN) $(DT
 
 $(KDIR)dtrsm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DTRSMKERNEL_LT) $(DTRSMDEPEND)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRSMKERNEL -UCOMPLEX -DDOUBLE -UUPPER -DLT -UCONJ $< -o dtrsm_kernel_lt.s
+	$(CC) $(CFLAGS) -S -DTRSMKERNEL -UCOMPLEX -DDOUBLE -UUPPER -DLT -UCONJ $< -o - > dtrsm_kernel_lt.s
 	m4 dtrsm_kernel_lt.s > dtrsm_kernel_lt_nomacros.s
 	$(CC) -c $(CFLAGS) -DTRSMKERNEL -UCOMPLEX -DDOUBLE -UUPPER -DLT -UCONJ dtrsm_kernel_lt_nomacros.s -o $@
 	rm dtrsm_kernel_lt.s dtrsm_kernel_lt_nomacros.s
@@ -2460,7 +2460,7 @@ $(KDIR)cgemm_kernel_l$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMM
 
 $(KDIR)cgemm_kernel_r$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND)
 ifeq ($(OS), AIX)
-	$(CC) $(PFLAGS) -E -UDOUBLE -DCOMPLEX -DNC $< -o cgemm_kernel_r.s
+	$(CC) $(PFLAGS) -S -UDOUBLE -DCOMPLEX -DNC $< -o - > cgemm_kernel_r.s
 	m4 cgemm_kernel_r.s > cgemm_kernel_r_nomacros.s
 	$(CC) $(PFLAGS) -c -UDOUBLE -DCOMPLEX -DNC cgemm_kernel_r_nomacros.s -o $@
 	rm cgemm_kernel_r.s cgemm_kernel_r_nomacros.s 
@@ -2506,7 +2506,7 @@ $(KDIR)strmm_kernel_RN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL)
 
 $(KDIR)strmm_kernel_RT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL)
 ifeq ($(OS), AIX)
-	$(CC) $(CFLAGS) -E -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o strmm_kernel_rt.s	
+	$(CC) $(CFLAGS) -S -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o - > strmm_kernel_rt.s	
 	m4 strmmkernel_rn.s > strmm_kernel_rt_nomacros.s
 	$(CC) $(PFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -DTRANSA strmm_kernel_rt_nomacros.s -o $@
 	rm strmm_kernel_rt.s strmm_kernel_rt_nomacros.s

From 571eadb88063c91ea9b5b1bcb2ae33cd8fbc5762 Mon Sep 17 00:00:00 2001
From: Rajalakshmi Srinivasaraghavan <rajis@linux.ibm.com>
Date: Wed, 24 Jun 2020 14:48:15 -0500
Subject: [PATCH 29/48] powerpc: Optimized SGEMM/DGEMM/CGEMM for POWER10

This patch introduces new optimized version of SGEMM, CGEMM and DGEMM
using power10 Matrix-Multiply Assist (MMA) feature introduced in
POWER ISA v3.1. This patch makes use of new POWER10 compute instructions
for matrix multiplication operation.

Tested on simulator and there are no new test failures.
Cycles count reduced by 30-50%  compared to POWER9 version depending on
M/N/K sizes.
MMA GCC patch for reference:
https://gcc.gnu.org/git/?p=gcc.git;a=commit;h=8ee2640bfdc62f835ec9740278f948034bc7d9f1
---
 kernel/power/KERNEL.POWER10         |   12 +-
 kernel/power/cgemm_kernel_power10.S |  286 +++
 kernel/power/cgemm_logic_power10.S  | 2814 +++++++++++++++++++++++++++
 kernel/power/cgemm_macros_power10.S | 2131 ++++++++++++++++++++
 kernel/power/dgemm_kernel_power10.c |  864 ++++++++
 kernel/power/sgemm_kernel_power10.c | 1334 +++++++++++++
 6 files changed, 7435 insertions(+), 6 deletions(-)
 create mode 100644 kernel/power/cgemm_kernel_power10.S
 create mode 100644 kernel/power/cgemm_logic_power10.S
 create mode 100644 kernel/power/cgemm_macros_power10.S
 create mode 100644 kernel/power/dgemm_kernel_power10.c
 create mode 100644 kernel/power/sgemm_kernel_power10.c

diff --git a/kernel/power/KERNEL.POWER10 b/kernel/power/KERNEL.POWER10
index ab8fbfcd9..00d31f8b6 100644
--- a/kernel/power/KERNEL.POWER10
+++ b/kernel/power/KERNEL.POWER10
@@ -7,12 +7,12 @@ else
 #CGEMM_BETA = ../generic/zgemm_beta.c
 #ZGEMM_BETA = ../generic/zgemm_beta.c
 
-STRMMKERNEL	= sgemm_kernel_power9.S
-DTRMMKERNEL	= dgemm_kernel_power9.S
-CTRMMKERNEL	= cgemm_kernel_power9.S
+STRMMKERNEL	= sgemm_kernel_power10.c
+DTRMMKERNEL	= dgemm_kernel_power10.c
+CTRMMKERNEL	= cgemm_kernel_power10.S
 ZTRMMKERNEL	= zgemm_kernel_power9.S
 
-SGEMMKERNEL    =  sgemm_kernel_power9.S
+SGEMMKERNEL    =  sgemm_kernel_power10.c
 SGEMMINCOPY    = ../generic/gemm_ncopy_16.c
 SGEMMITCOPY    = sgemm_tcopy_16_power8.S
 SGEMMONCOPY    =  ../generic/gemm_ncopy_8.c
@@ -22,7 +22,7 @@ SGEMMITCOPYOBJ =  sgemm_itcopy$(TSUFFIX).$(SUFFIX)
 SGEMMONCOPYOBJ =  sgemm_oncopy$(TSUFFIX).$(SUFFIX)
 SGEMMOTCOPYOBJ =  sgemm_otcopy$(TSUFFIX).$(SUFFIX)
 
-DGEMMKERNEL    =  dgemm_kernel_power9.S
+DGEMMKERNEL    =  dgemm_kernel_power10.c
 DGEMMINCOPY    = ../generic/gemm_ncopy_16.c
 DGEMMITCOPY    =  dgemm_tcopy_16_power8.S
 DGEMMONCOPY    =  dgemm_ncopy_4_power8.S
@@ -32,7 +32,7 @@ DGEMMITCOPYOBJ =  dgemm_itcopy$(TSUFFIX).$(SUFFIX)
 DGEMMONCOPYOBJ =  dgemm_oncopy$(TSUFFIX).$(SUFFIX)
 DGEMMOTCOPYOBJ =  dgemm_otcopy$(TSUFFIX).$(SUFFIX)
 
-CGEMMKERNEL    = cgemm_kernel_power9.S
+CGEMMKERNEL    = cgemm_kernel_power10.S
 CGEMMINCOPY    = ../generic/zgemm_ncopy_8.c
 CGEMMITCOPY    = ../generic/zgemm_tcopy_8.c
 CGEMMONCOPY    = ../generic/zgemm_ncopy_4.c
diff --git a/kernel/power/cgemm_kernel_power10.S b/kernel/power/cgemm_kernel_power10.S
new file mode 100644
index 000000000..e04f948dd
--- /dev/null
+++ b/kernel/power/cgemm_kernel_power10.S
@@ -0,0 +1,286 @@
+/***************************************************************************
+Copyright (c) 2013-2020, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#define ASSEMBLER
+#include "common.h"
+#include "def_vsx.h"
+
+ 
+#define LOAD	ld
+#define STACKSIZE  (512 )  
+#define FLINK_SAVE (STACKSIZE+16) /* 16($r12) */  
+#define	M	r3
+#define	N	r4
+#define	K	r5
+
+
+#define A	r8
+#define	B	r9
+#define	C	r10
+#define	LDC	r6
+#define OFFSET	r7
+
+
+#define alpha_r vs51
+#define alpha_i vs55
+#define save_permute_1 vs59
+#define permute_mask vs63
+#define o0	0
+ 
+
+#define T1	r11
+#define T2	r12
+#define T3	r14
+#define T4	r15
+#define T5	r16
+#define T6	r17
+#define L	r18
+#define T7	r19
+#define T8	r20
+#define TEMP_REG	r21
+#define	I	r22
+#define J	r23
+#define AO	r24
+#define	BO	r25
+#define	CO 	r26
+#define T9	r27
+#define	T10	r28
+#define	PRE	r29
+
+#define T12	r30
+#define T13	r31
+
+#include "cgemm_macros_power10.S"
+
+.equ    perm_const1, 0x0405060700010203
+.equ    perm_const2, 0x0c0d0e0f08090a0b
+.equ save_permute_12, 0x0c0d0e0f1c1d1e1f
+.equ save_permute_11, 0x0405060714151617
+
+
+
+#ifndef NEEDPARAM
+
+	PROLOGUE
+	PROFCODE
+
+
+	addi	SP, SP, -STACKSIZE
+	mflr r0
+
+
+	stfd	f14,    0(SP)
+	stfd	f15,    8(SP)
+	stfd	f16,   16(SP)
+	stfd	f17,   24(SP)
+
+	stfd	f18,   32(SP)
+	stfd	f19,   40(SP)
+	stfd	f20,   48(SP)
+	stfd	f21,   56(SP)
+
+	stfd	f22,   64(SP)
+	stfd	f23,   72(SP)
+	stfd	f24,   80(SP)
+	stfd	f25,   88(SP)
+
+	stfd	f26,   96(SP)
+	stfd	f27,  104(SP)
+	stfd	f28,  112(SP)
+	stfd	f29,  120(SP)
+
+	stfd	f30,  128(SP)
+	stfd	f31,  136(SP)
+
+
+	std	r31,  144(SP)
+	std	r30,  152(SP)
+	std	r29,  160(SP)
+	std	r28,  168(SP)
+	std	r27,  176(SP)
+	std	r26,  184(SP)
+	std	r25,  192(SP)
+	std	r24,  200(SP)
+	std	r23,  208(SP)
+	std	r22,  216(SP)
+	std	r21,  224(SP)
+	std	r20,  232(SP)
+	std	r19,  240(SP)
+	std	r18,  248(SP)
+	std	r17,  256(SP)
+	std	r16,  264(SP)
+	std	r15,  272(SP)
+	std	r14,  280(SP)
+ 
+ 
+  stxv    vs52,  288(SP)
+  stxv    vs53,  304(SP)
+  stxv    vs54,  320(SP)
+  stxv    vs55,  336(SP)
+  stxv    vs56,  352(SP)
+  stxv    vs57,  368(SP)
+  stxv    vs58,  384(SP)
+  stxv    vs59,  400(SP)
+  stxv    vs60,  416(SP)
+  stxv    vs61,  432(SP)
+  stxv    vs62,  448(SP)
+  stxv    vs63,  464(SP)
+  std     r0,   FLINK_SAVE(SP)
+ 
+
+
+	ld	LDC, FRAMESLOT(0) + STACKSIZE(SP)
+
+
+
+#ifdef TRMMKERNEL
+	ld	OFFSET,  FRAMESLOT(1) + STACKSIZE(SP)
+#endif
+   slwi    LDC, LDC, ZBASE_SHIFT
+
+ 
+ 
+	/*alpha is stored in f1. convert to single and splat*/
+    xscvdpspn alpha_r,vs1 
+    xscvdpspn alpha_i,vs2 
+	xxspltw   alpha_r,alpha_r,0 
+	xxspltw   alpha_i,alpha_i,0 
+/*load reverse permute mask for big endian
+  uint128 = 0xc0d0e0f08090a0b0405060700010203
+*/ 
+		
+	lis T2, perm_const2@highest
+	lis T1, perm_const1@highest
+	lis T3, save_permute_12@highest
+	lis T4, save_permute_11@highest
+
+	
+	ori T2, T2, perm_const2@higher
+	ori T1, T1, perm_const1@higher
+	ori T3, T3, save_permute_12@higher
+	ori T4, T4, save_permute_11@higher
+
+	
+	rldicr T2, T2, 32, 31
+	rldicr T1, T1, 32, 31
+	rldicr T3, T3, 32, 31
+	rldicr T4, T4, 32, 31 
+
+	oris T2, T2, perm_const2@h
+	oris T1, T1, perm_const1@h
+	oris T3, T3, save_permute_12@h
+	oris T4, T4, save_permute_11@h
+
+	
+	ori T2, T2, perm_const2@l  
+	ori T1, T1, perm_const1@l
+	ori T3, T3, save_permute_12@l  
+	ori T4, T4, save_permute_11@l
+
+	
+  li r0,0
+  li PRE,512
+
+#if defined(CC) || defined(CR) || defined(RC) || defined(RR) 
+/*negate for this case as we will use addition -1*(a+b) */
+  xvnegsp alpha_r,alpha_r
+  xvnegsp alpha_i,alpha_i
+#endif
+
+	mtvsrdd permute_mask,T2,T1
+	mtvsrdd save_permute_1,T3,T4 	
+
+     /*mask is reverse permute so we have to make it inner permute */
+ 	xxpermdi	permute_mask,	permute_mask,	permute_mask,2 
+
+#include "cgemm_logic_power10.S"
+
+.L999: 
+	lfd	f14,    0(SP)
+	lfd	f15,    8(SP)
+	lfd	f16,   16(SP)
+	lfd	f17,   24(SP)
+
+	lfd	f18,   32(SP)
+	lfd	f19,   40(SP)
+	lfd	f20,   48(SP)
+	lfd	f21,   56(SP)
+
+	lfd	f22,   64(SP)
+	lfd	f23,   72(SP)
+	lfd	f24,   80(SP)
+	lfd	f25,   88(SP)
+
+	lfd	f26,   96(SP)
+	lfd	f27,  104(SP)
+	lfd	f28,  112(SP)
+	lfd	f29,  120(SP)
+
+	lfd	f30,  128(SP)
+	lfd	f31,  136(SP)
+
+	ld	r31,  144(SP)
+	ld	r30,  152(SP)
+	ld	r29,  160(SP)
+	ld	r28,  168(SP)
+	ld	r27,  176(SP)
+	ld	r26,  184(SP)
+	ld	r25,  192(SP)
+	ld	r24,  200(SP)
+	ld	r23,  208(SP)
+	ld	r22,  216(SP)
+	ld	r21,  224(SP)
+	ld	r20,  232(SP)
+	ld	r19,  240(SP)
+	ld	r18,  248(SP)
+	ld	r17,  256(SP)
+	ld	r16,  264(SP)
+	ld	r15,  272(SP)
+	ld	r14,  280(SP)
+
+	ld    r0, 	 FLINK_SAVE(SP)	
+ 
+    lxv    vs52,  288(SP)
+    lxv    vs53,  304(SP)
+    lxv    vs54,  320(SP)
+    lxv    vs55,  336(SP)
+    lxv    vs56,  352(SP)
+    lxv    vs57,  368(SP)
+    lxv    vs58,  384(SP) 
+    lxv    vs59,  400(SP)
+	mtlr r0
+    lxv    vs60,  416(SP)
+    lxv    vs61,  432(SP) 
+    lxv    vs62,  448(SP)
+    lxv    vs63,  464(SP)
+
+	addi	SP, SP, STACKSIZE 
+	blr
+
+
+	EPILOGUE
+#endif
diff --git a/kernel/power/cgemm_logic_power10.S b/kernel/power/cgemm_logic_power10.S
new file mode 100644
index 000000000..3700ac87b
--- /dev/null
+++ b/kernel/power/cgemm_logic_power10.S
@@ -0,0 +1,2814 @@
+/***************************************************************************
+Copyright (c) 2013-2020, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#define MY_ALIGN .align 3
+b CGEMM_L4
+/*                MINI SUBROUTINES                            */      
+/*                4x8 MAIN 128x+2 LOOP                     */      
+
+
+CGEMM_L4x8_LMAIN_SUB:
+/*----------------------------------------*/   
+    mtctr   T8
+    LOAD4x8_2 
+    MY_ALIGN
+CGEMM_L4x8_LOOP:
+/*----------------------------------------*/   
+    dcbt    AO, PRE
+    dcbt    BO, PRE
+    KERNEL4x8_L2 128,64,0,0 
+CGEMM_L4x8_K128:
+/*----------------------------------------*/   
+    KERNEL4x8_L2 128,64,1,0
+    dcbt    AO, T2  
+    KERNEL4x8_L2 128,64,2,0
+    KERNEL4x8_L2 128,64,3,0 
+    dcbt    AO, T3
+    dcbt    BO, T2
+    KERNEL4x8_L2 128,64,4,0
+    KERNEL4x8_L2 128,64,5,0
+    dcbt    AO, T4  
+    KERNEL4x8_L2 128,64,6,0
+    KERNEL4x8_L2 128,64,7,0  
+    dcbt    AO, T5  
+    dcbt    BO, T3
+    KERNEL4x8_L2 128,64,8,0
+    KERNEL4x8_L2 128,64,9,0
+    KERNEL4x8_L2 128,64,10,0
+    KERNEL4x8_L2 128,64,11,0  
+    dcbt    BO, T4
+    KERNEL4x8_L2 128,64,12,0
+    KERNEL4x8_L2 128,64,13,0
+    KERNEL4x8_L2 128,64,14,0
+    KERNEL4x8_L2 128,64,15,0  
+    KERNEL4x8_L2 128,64,16,0
+    KERNEL4x8_L2 128,64,17,0 
+    KERNEL4x8_L2 128,64,18,0
+    KERNEL4x8_L2 128,64,19,0  
+    KERNEL4x8_L2 128,64,20,0
+    KERNEL4x8_L2 128,64,21,0 
+    KERNEL4x8_L2 128,64,22,0
+    KERNEL4x8_L2 128,64,23,0   
+    KERNEL4x8_L2 128,64,24,0
+    KERNEL4x8_L2 128,64,25,0
+    KERNEL4x8_L2 128,64,26,0
+    KERNEL4x8_L2 128,64,27,0  
+    KERNEL4x8_L2 128,64,28,0
+    KERNEL4x8_L2 128,64,29,0
+    KERNEL4x8_L2 128,64,30,0
+    KERNEL4x8_L2 128,64,31,0 
+    KERNEL4x8_L2 128,64,32,0
+    KERNEL4x8_L2 128,64,33,0
+    KERNEL4x8_L2 128,64,34,0
+    KERNEL4x8_L2 128,64,35,0 
+    KERNEL4x8_L2 128,64,36,0
+    KERNEL4x8_L2 128,64,37,0
+    KERNEL4x8_L2 128,64,38,0
+    KERNEL4x8_L2 128,64,39,0  
+    KERNEL4x8_L2 128,64,40,0
+    KERNEL4x8_L2 128,64,41,0
+    KERNEL4x8_L2 128,64,42,0
+    KERNEL4x8_L2 128,64,43,0  
+    KERNEL4x8_L2 128,64,44,0
+    KERNEL4x8_L2 128,64,45,0
+    KERNEL4x8_L2 128,64,46,0
+    KERNEL4x8_L2 128,64,47,0 
+    KERNEL4x8_L2 128,64,48,0
+    KERNEL4x8_L2 128,64,49,0 
+    KERNEL4x8_L2 128,64,50,0
+    KERNEL4x8_L2 128,64,51,0  
+    KERNEL4x8_L2 128,64,52,0
+    KERNEL4x8_L2 128,64,53,0 
+    KERNEL4x8_L2 128,64,54,0
+    KERNEL4x8_L2 128,64,55,0  
+    KERNEL4x8_L2 128,64,56,0
+    KERNEL4x8_L2 128,64,57,0
+    KERNEL4x8_L2 128,64,58,0
+    KERNEL4x8_L2 128,64,59,0  
+    KERNEL4x8_L2 128,64,60,0
+    KERNEL4x8_L2 128,64,61,0
+    KERNEL4x8_L2 128,64,62,0 
+    KERNEL4x8_L2 128,64,63,1  
+    bdnz    CGEMM_L4x8_LOOP
+    MY_ALIGN  
+CGEMM_L4x8_LOOP_END:
+/*----------------------------------------*/   
+    END4x8_2
+    blr
+    MY_ALIGN
+
+
+CGEMM_4x8_L64_SUB:
+/*----------------------------------------*/   
+    LOAD4x8_2  
+    dcbt    AO, PRE
+    dcbt    BO, PRE
+    KERNEL4x8_L2 128,64,0,0 
+    KERNEL4x8_L2 128,64,1,0
+    dcbt    AO, T2  
+    KERNEL4x8_L2 128,64,2,0
+    KERNEL4x8_L2 128,64,3,0 
+    dcbt    AO, T3
+    dcbt    BO, T2
+    KERNEL4x8_L2 128,64,4,0
+    KERNEL4x8_L2 128,64,5,0
+    dcbt    AO, T4  
+    KERNEL4x8_L2 128,64,6,0
+    KERNEL4x8_L2 128,64,7,0  
+    dcbt    AO, T5  
+    dcbt    BO, T3
+    KERNEL4x8_L2 128,64,8,0
+    KERNEL4x8_L2 128,64,9,0
+    KERNEL4x8_L2 128,64,10,0
+    KERNEL4x8_L2 128,64,11,0  
+    dcbt    BO, T4
+    KERNEL4x8_L2 128,64,12,0
+    KERNEL4x8_L2 128,64,13,0
+    KERNEL4x8_L2 128,64,14,0
+    KERNEL4x8_L2 128,64,15,0  
+    KERNEL4x8_L2 128,64,16,0
+    KERNEL4x8_L2 128,64,17,0 
+    KERNEL4x8_L2 128,64,18,0
+    KERNEL4x8_L2 128,64,19,0  
+    KERNEL4x8_L2 128,64,20,0
+    KERNEL4x8_L2 128,64,21,0 
+    KERNEL4x8_L2 128,64,22,0
+    KERNEL4x8_L2 128,64,23,0   
+    KERNEL4x8_L2 128,64,24,0
+    KERNEL4x8_L2 128,64,25,0
+    KERNEL4x8_L2 128,64,26,0
+    KERNEL4x8_L2 128,64,27,0  
+    KERNEL4x8_L2 128,64,28,0
+    KERNEL4x8_L2 128,64,29,0
+    KERNEL4x8_L2 128,64,30,0
+    KERNEL4x8_E2 128,64,31,1
+    blr
+    MY_ALIGN
+
+
+CGEMM_4x8_L32_SUB:
+/*----------------------------------------*/   
+    LOAD4x8_2  
+    dcbt    AO, PRE
+    dcbt    BO, PRE
+    KERNEL4x8_L2 128,64,0,0 
+    KERNEL4x8_L2 128,64,1,0
+    dcbt    AO, T2  
+    KERNEL4x8_L2 128,64,2,0
+    KERNEL4x8_L2 128,64,3,0 
+    dcbt    AO, T3
+    dcbt    BO, T2
+    KERNEL4x8_L2 128,64,4,0
+    KERNEL4x8_L2 128,64,5,0
+    dcbt    AO, T4  
+    KERNEL4x8_L2 128,64,6,0
+    KERNEL4x8_L2 128,64,7,0  
+    dcbt    AO, T5  
+    dcbt    BO, T3
+    KERNEL4x8_L2 128,64,8,0
+    KERNEL4x8_L2 128,64,9,0
+    KERNEL4x8_L2 128,64,10,0
+    KERNEL4x8_L2 128,64,11,0  
+    dcbt    BO, T4
+    KERNEL4x8_L2 128,64,12,0
+    KERNEL4x8_L2 128,64,13,0
+    KERNEL4x8_L2 128,64,14,0
+    KERNEL4x8_E2 128,64,15,1
+    blr
+    MY_ALIGN
+
+
+CGEMM_4x8_L16_SUB:
+/*----------------------------------------*/   
+    LOAD4x8_2 
+    dcbt    AO, PRE
+    dcbt    BO, PRE
+    KERNEL4x8_L2 128,64,0,0 
+    KERNEL4x8_L2 128,64,1,0
+    dcbt    AO, T2  
+    KERNEL4x8_L2 128,64,2,0
+    KERNEL4x8_L2 128,64,3,0 
+    dcbt    AO, T3
+    dcbt    BO, T2
+    KERNEL4x8_L2 128,64,4,0
+    KERNEL4x8_L2 128,64,5,0
+    dcbt    AO, T4  
+    KERNEL4x8_L2 128,64,6,0
+    KERNEL4x8_E2 128,64,7,1
+    blr
+    MY_ALIGN
+
+
+CGEMM_4x4_LMAIN_SUB:
+/*----------------------------------------*/   
+    mtctr   T8
+    LOAD4x4_2  
+    MY_ALIGN
+CGEMM_L4x4_LOOP:
+/*----------------------------------------*/   
+    KERNEL4x4_L2 64,64,0,0
+CGEMM_L4x4_K32:
+/*----------------------------------------*/   
+    KERNEL4x4_L2 64,64,1,0   
+    KERNEL4x4_L2 64,64,2,0
+    KERNEL4x4_L2 64,64,3,0  
+    KERNEL4x4_L2 64,64,4,0
+    KERNEL4x4_L2 64,64,5,0 
+    KERNEL4x4_L2 64,64,6,0
+    KERNEL4x4_L2 64,64,7,0
+    KERNEL4x4_L2 64,64,8,0
+    KERNEL4x4_L2 64,64,9,0   
+    KERNEL4x4_L2 64,64,10,0
+    KERNEL4x4_L2 64,64,11,0  
+    KERNEL4x4_L2 64,64,12,0
+    KERNEL4x4_L2 64,64,13,0 
+    KERNEL4x4_L2 64,64,14,0
+    KERNEL4x4_L2 64,64,15,1    
+    bdnz    CGEMM_L4x4_LOOP
+    MY_ALIGN  
+CGEMM_L4x4_LOOP_END:
+/*----------------------------------------*/   
+    END4x4_2 
+    blr
+    MY_ALIGN
+
+
+CGEMM_4x4_L16_SUB:
+/*----------------------------------------*/   
+    LOAD4x4_2
+    KERNEL4x4_L2 64,64,0,0
+    KERNEL4x4_L2 64,64,1,0   
+    KERNEL4x4_L2 64,64,2,0
+    KERNEL4x4_L2 64,64,3,0  
+    KERNEL4x4_L2 64,64,4,0
+    KERNEL4x4_L2 64,64,5,0 
+    KERNEL4x4_L2 64,64,6,0
+    KERNEL4x4_E2 64,64,7,1
+    blr
+    MY_ALIGN
+
+
+CGEMM_4x4_L8_SUB:
+/*----------------------------------------*/   
+    LOAD4x4_2
+    KERNEL4x4_L2 64,64,0,0
+    KERNEL4x4_L2 64,64,1,0   
+    KERNEL4x4_L2 64,64,2,0
+    KERNEL4x4_E2 64,64,3,1 
+    blr
+
+
+CGEMM_4x2_LMAIN_SUB:
+/*----------------------------------------*/   
+    mtctr   T8
+    LOAD4x2_2  
+    MY_ALIGN 
+CGEMM_L4x2_LOOP:
+/*----------------------------------------*/   
+    KERNEL4x2_L2 32,64,0,0 
+CGEMM_L4x2_K32:
+/*----------------------------------------*/   
+    KERNEL4x2_L2 32,64,1,0  
+    KERNEL4x2_L2 32,64,2,0
+    KERNEL4x2_L2 32,64,3,0  
+    KERNEL4x2_L2 32,64,4,0
+    KERNEL4x2_L2 32,64,5,0 
+    KERNEL4x2_L2 32,64,6,0
+    KERNEL4x2_L2 32,64,7,0
+    KERNEL4x2_L2 32,64,8,0
+    KERNEL4x2_L2 32,64,9,0  
+    KERNEL4x2_L2 32,64,10,0
+    KERNEL4x2_L2 32,64,11,0  
+    KERNEL4x2_L2 32,64,12,0
+    KERNEL4x2_L2 32,64,13,0 
+    KERNEL4x2_L2 32,64,14,0
+    KERNEL4x2_L2 32,64,15,1   
+    bdnz    CGEMM_L4x2_LOOP
+    MY_ALIGN  
+
+
+CGEMM_L4x2_LOOP_END:
+/*----------------------------------------*/   
+    END4x2_2 
+    blr
+    MY_ALIGN
+CGEMM_4x2_L16_SUB:
+/*----------------------------------------*/   
+    LOAD4x2_2
+    KERNEL4x2_L2 32,64,0,0
+    KERNEL4x2_L2 32,64,1,0  
+    KERNEL4x2_L2 32,64,2,0
+    KERNEL4x2_L2 32,64,3,0  
+    KERNEL4x2_L2 32,64,4,0
+    KERNEL4x2_L2 32,64,5,0 
+    KERNEL4x2_L2 32,64,6,0
+    KERNEL4x2_E2 32,64,7,1
+    blr
+    MY_ALIGN
+CGEMM_4x2_L8_SUB:
+/*----------------------------------------*/   
+    LOAD4x2_2
+    KERNEL4x2_L2 32,64,0,0
+    KERNEL4x2_L2 32,64,1,0  
+    KERNEL4x2_L2 32,64,2,0
+    KERNEL4x2_E2 32,64,3,1  
+    blr
+
+
+CGEMM_4x1_LMAIN_SUB:
+/*----------------------------------------*/   
+    mtctr   T8
+    LOAD4x1_2  
+    MY_ALIGN
+CGEMM_L4x1_LOOP:
+/*----------------------------------------*/   
+    KERNEL4x1_L2 16,64,0,0 
+CGEMM_L4x1_K32:
+/*----------------------------------------*/   
+    KERNEL4x1_L2 16,64,1,0  
+    KERNEL4x1_L2 16,64,2,0
+    KERNEL4x1_L2 16,64,3,0  
+    KERNEL4x1_L2 16,64,4,0
+    KERNEL4x1_L2 16,64,5,0 
+    KERNEL4x1_L2 16,64,6,0
+    KERNEL4x1_L2 16,64,7,0
+    KERNEL4x1_L2 16,64,8,0
+    KERNEL4x1_L2 16,64,9,0  
+    KERNEL4x1_L2 16,64,10,0
+    KERNEL4x1_L2 16,64,11,0  
+    KERNEL4x1_L2 16,64,12,0
+    KERNEL4x1_L2 16,64,13,0 
+    KERNEL4x1_L2 16,64,14,0
+    KERNEL4x1_L2 16,64,15,1   
+    bdnz    CGEMM_L4x1_LOOP
+    MY_ALIGN  
+CGEMM_L4x1_LOOP_END:
+/*----------------------------------------*/   
+    END4x1_2 
+    blr
+
+    MY_ALIGN
+CGEMM_4x1_L16_SUB:
+/*----------------------------------------*/   
+    LOAD4x1_2
+    KERNEL4x1_L2 16,64,0,0
+    KERNEL4x1_L2 16,64,1,0  
+    KERNEL4x1_L2 16,64,2,0
+    KERNEL4x1_L2 16,64,3,0  
+    KERNEL4x1_L2 16,64,4,0
+    KERNEL4x1_L2 16,64,5,0 
+    KERNEL4x1_L2 16,64,6,0
+    KERNEL4x1_E2 16,64,7,1
+    blr
+    MY_ALIGN
+
+
+CGEMM_4x1_L8_SUB:
+/*----------------------------------------*/   
+    LOAD4x1_2
+    KERNEL4x1_L2 16,64,0,0
+    KERNEL4x1_L2 16,64,1,0  
+    KERNEL4x1_L2 16,64,2,0
+    KERNEL4x1_E2 16,64,3,1  
+    blr
+
+
+
+/*             MAIN LOOP BEGINS               */   
+    MY_ALIGN
+
+
+CGEMM_L4:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL) && !defined(LEFT)   
+    neg TEMP_REG, OFFSET 
+#endif   
+    /* Pre set value in vs57 as 0xffff0000ffff0000 for masking */
+    vspltisb v24, -1
+    vspltisb v25, 0
+    xxsldwi vs57, vs56, vs57, 1
+    xxpermdi vs57, vs57, vs57, 3
+    srawi.    J,  N,  2
+    ble   CGEMM_L4_END
+
+
+CGEMM_L4_BEGIN:
+/*----------------------------------------*/   
+    mr    CO, C
+    slwi    T1, LDC , 2     
+    add     T2,C,LDC    
+    mr    AO, A  
+    add   C,  C,  T1
+#if defined(TRMMKERNEL) && defined(LEFT)   
+    mr TEMP_REG, OFFSET  /*off = offset;*/
+#endif     
+    srawi.    I,  M,  3
+    ble   CGEMM_L4x8_END
+    dcbt    CO,r0  /*just prefetch*/
+    dcbt    T2,r0    
+
+
+CGEMM_L4x8_BEGIN:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)   
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,8,4
+#else    
+    mr    BO, B  
+    dcbt    B,  r0  
+#endif     
+    dcbt    AO, r0
+#if defined(TRMMKERNEL)   
+    REFRESH_TEMP_BK T6,K,TEMP_REG,8,4
+    mr T1, T6
+/* TEMPS FOR PREFETCH */   
+    li T2, 1024
+    li T3, 1024+512
+    addi T1,T1, -2
+/* TEMPS FOR PREFETCH */     
+    li T4, 2048
+    li T5, 2048+512   
+    srawi.   T8, T1, 7 /**(T1-2) % 128x */
+#else   
+    mr T1, K
+/* TEMPS FOR PREFETCH */   
+    li T2, 1024
+    li T3, 1024+512
+    addi T1,T1, -2
+/* TEMPS FOR PREFETCH */     
+    li T4, 2048
+    li T5, 2048+512 
+    srawi.   T8, T1, 7 /**(K-2) % 128x */
+#endif   
+    ZERO4x8  
+    ble   CGEMM_L4x8_SUB0
+    bl CGEMM_L4x8_LMAIN_SUB
+    andi.   L,  T1, 127
+    ble   CGEMM_L4x8_SAVE
+    b   CGEMM_L4x8_SUB2
+
+
+CGEMM_L4x8_SUB0:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)   
+    andi.   L,  T6, 255
+    cmpwi   T6,129
+#else   
+    andi.   L,  K,  255
+    cmpwi   K,129
+#endif       
+    li T8,1
+    bne CMP4x8_128K
+    addi BO,BO,-32
+    addi AO,AO,-64 
+    LOAD4x8O 64,32 
+    END4x8_WITHOUT_ADD   
+    LOAD4x8_2O  128, 64 
+    mtctr   T8    
+    bl CGEMM_L4x8_K128   
+    b CGEMM_L4x8_SAVE  
+    CMP4x8_128K:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)    
+    cmpwi   T6,128
+#else    
+    cmpwi   K,128
+#endif        
+    bne CGEMM_L4x8_SUB2 
+    MY_ALIGN   
+    mtctr   T8
+    addi BO,BO,-64
+    addi AO,AO,-128   
+    LOAD4x8_2O 128,64
+    bl CGEMM_L4x8_K128   
+    b CGEMM_L4x8_SAVE 
+    MY_ALIGN
+
+
+CGEMM_L4x8_SUB2:
+/*----------------------------------------*/   
+    andi.      T1,L, 64
+    ble CGEMM_L4x8_SUB2_32
+    bl  CGEMM_4x8_L64_SUB
+    MY_ALIGN
+
+
+CGEMM_L4x8_SUB2_32:
+/*----------------------------------------*/   
+    andi.      T1,L, 32
+    ble CGEMM_L4x8_SUB2_16    
+    bl  CGEMM_4x8_L32_SUB
+    MY_ALIGN 
+
+
+CGEMM_L4x8_SUB2_16:
+/*----------------------------------------*/   
+    andi.      T1,L, 16
+    ble CGEMM_L4x8_SUB2_8
+    bl  CGEMM_4x8_L16_SUB  
+    MY_ALIGN    
+
+
+CGEMM_L4x8_SUB2_8:
+/*----------------------------------------*/   
+    andi.      T1,L, 8
+    ble CGEMM_L4x8_SUB2_4
+    LOAD4x8_2
+    KERNEL4x8_L2  128,64, 0,0
+    KERNEL4x8_L2  128,64, 1,0
+    KERNEL4x8_L2  128,64, 2,0
+    KERNEL4x8_E2  128,64, 3,1
+    MY_ALIGN   
+
+
+CGEMM_L4x8_SUB2_4:
+/*----------------------------------------*/   
+    andi.      T1,L, 4
+    ble CGEMM_L4x8_SUB2_2
+    LOAD4x8_2
+    KERNEL4x8_L2  128,64, 0,0
+    KERNEL4x8_E2  128,64, 1,1
+    MY_ALIGN
+
+
+CGEMM_L4x8_SUB2_2:
+/*----------------------------------------*/   
+    andi.      T1,L, 2
+    ble CGEMM_L4x8_SUB2_1
+    LOAD4x8_2 
+    KERNEL4x8_E2  128,64, 0,1
+    MY_ALIGN    
+
+
+CGEMM_L4x8_SUB2_1:
+/*----------------------------------------*/   
+    andi.      T1,L, 1
+    ble CGEMM_L4x8_SAVE 
+    KERNEL4x8
+
+    MY_ALIGN
+CGEMM_L4x8_SAVE:
+/*----------------------------------------*/   
+    addic.    I,  I,  -1
+    MY_ALIGN
+    SAVE4x8
+#if defined(TRMMKERNEL)    
+    REFRESH_AFTER_SAVE T6,K,TEMP_REG,BO,AO,8,4
+#endif     
+    bgt   CGEMM_L4x8_BEGIN
+    andi.   T2, M,  7
+    ble   CGEMM_L4x1_END
+    andi.   T1, M,  4
+    ble   CGEMM_L4x4_END
+    b   CGEMM_L4x4_BEGIN
+    MY_ALIGN 
+
+
+CGEMM_L4x8_END:
+/*----------------------------------------*/   
+
+
+CGEMM_L4x4_BEGIN:
+/*----------------------------------------*/   
+    andi.   T2, M,  7
+    ble   CGEMM_L4x1_END
+    andi.   T1, M,  4
+    ble   CGEMM_L4x4_END
+#if defined(TRMMKERNEL)   
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,4,4
+#else    
+    mr    BO, B   
+#endif        
+#if defined(TRMMKERNEL)   
+    REFRESH_TEMP_BK T6,K,TEMP_REG,4,4
+    mr T1, T6 
+    addi T1,T1, -2 
+    srawi.   T8, T1, 5 /**(T1-2) % 32x */
+#else   
+    mr T1, K 
+    addi T1,T1, -2
+    srawi.   T8, T1, 5 /**(K-2) % 32x */
+#endif     
+    ZERO4x4
+    ble   CGEMM_L4x4_SUB0 
+    bl CGEMM_4x4_LMAIN_SUB
+    andi.   L,  T1, 31
+    ble   CGEMM_L4x4_SAVE
+    b    CGEMM_L4x4_SUB2
+
+
+CGEMM_L4x4_SUB0:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)   
+    andi.   L,  T6, 63
+    cmpwi   T6,33
+#else   
+    andi.   L,  K,  63
+    cmpwi   K,33
+#endif       
+    li T8,1
+    bne CMP4x4_32K
+    addi BO,BO,-32
+    addi AO,AO,-32  
+    LOAD4x4O 32,32 
+    END4x4_WITHOUT_ADD   
+    LOAD4x4_2O  64, 64 
+    mtctr   T8    
+    bl CGEMM_L4x4_K32   
+    b CGEMM_L4x4_SAVE  
+    CMP4x4_32K:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)    
+    cmpwi   T6,32
+#else    
+    cmpwi   K,32
+#endif        
+    bne CGEMM_L4x4_SUB2 
+    MY_ALIGN   
+    mtctr   T8
+    addi BO,BO,-64
+    addi AO,AO,-64   
+    LOAD4x4_2O 64,64
+    bl CGEMM_L4x4_K32   
+    b CGEMM_L4x4_SAVE 
+    MY_ALIGN 
+    MY_ALIGN 
+
+
+CGEMM_L4x4_SUB2:
+/*----------------------------------------*/   
+    andi.      T1,L, 16
+    ble CGEMM_L4x4_SUB2_8
+    bl  CGEMM_4x4_L16_SUB  
+    MY_ALIGN
+
+
+CGEMM_L4x4_SUB2_8:
+/*----------------------------------------*/   
+    andi.      T1,L, 8
+    ble CGEMM_L4x4_SUB2_4
+    bl CGEMM_4x4_L8_SUB
+    MY_ALIGN  
+
+
+CGEMM_L4x4_SUB2_4:
+/*----------------------------------------*/   
+    andi.      T1,L, 4
+    ble CGEMM_L4x4_SUB2_2
+    LOAD4x4_2
+    KERNEL4x4_L2  64,64, 0,0
+    KERNEL4x4_E2  64,64, 1,1
+    MY_ALIGN
+
+
+CGEMM_L4x4_SUB2_2:
+/*----------------------------------------*/   
+    andi.      T1,L, 2
+    ble CGEMM_L4x4_SUB2_1
+    LOAD4x4_2
+    KERNEL4x4_E2  64,64, 0,1
+    MY_ALIGN    
+
+
+CGEMM_L4x4_SUB2_1:
+/*----------------------------------------*/   
+    andi.      T1,L, 1
+    ble CGEMM_L4x4_SAVE 
+    KERNEL4x4
+
+
+CGEMM_L4x4_SAVE:
+/*----------------------------------------*/   
+    SAVE4x4
+#if defined(TRMMKERNEL)    
+    REFRESH_AFTER_SAVE T6,K,TEMP_REG,BO,AO,4,4
+#endif     
+
+
+CGEMM_L4x4_END:
+/*----------------------------------------*/   
+
+
+CGEMM_L4x2_BEGIN:
+/*----------------------------------------*/   
+    andi.   T1, M,  2
+    ble   CGEMM_L4x2_END
+#if defined(TRMMKERNEL)   
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,2,4
+#else    
+    mr    BO, B   
+#endif        
+#if defined(TRMMKERNEL)   
+    REFRESH_TEMP_BK T6,K,TEMP_REG,2,4
+    mr T1, T6 
+    addi T1,T1, -2 
+    srawi.   T8, T1, 5 /**(T1-2) % 32x */
+#else   
+    mr T1, K 
+    addi T1,T1, -2
+    srawi.   T8, T1, 5 /**(K-2) % 32x */
+#endif     
+    ZERO4x2
+    ble   CGEMM_L4x2_SUB0 
+    bl CGEMM_4x2_LMAIN_SUB
+    andi.   L,  T1, 31
+    ble   CGEMM_L4x2_SAVE
+    b   CGEMM_L4x2_SUB2
+
+
+CGEMM_L4x2_SUB0:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)   
+    andi.   L,  T6, 63
+    cmpwi   T6,33
+#else   
+    andi.   L,  K,  63
+    cmpwi   K,33
+#endif       
+    li T8,1
+    bne CMP4x2_32K
+    addi BO,BO,-32
+    addi AO,AO,-16  
+    LOAD4x2O 16,32 
+    END4x2_WITHOUT_ADD   
+    LOAD4x2_2O  32, 64  
+    mtctr   T8    
+    bl CGEMM_L4x2_K32   
+    b CGEMM_L4x2_SAVE  
+    CMP4x2_32K:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)    
+    cmpwi   T6,32
+#else    
+    cmpwi   K,32
+#endif        
+    bne CGEMM_L4x2_SUB2 
+    MY_ALIGN   
+    mtctr   T8
+    addi BO,BO,-64
+    addi AO,AO,-32   
+    LOAD4x2_2O 32,64
+    bl CGEMM_L4x2_K32   
+    b CGEMM_L4x2_SAVE 
+    MY_ALIGN 
+    MY_ALIGN 
+
+
+CGEMM_L4x2_SUB2:
+/*----------------------------------------*/   
+    andi.      T1,L, 16
+    ble CGEMM_L4x2_SUB2_8
+    bl CGEMM_4x2_L16_SUB  
+    MY_ALIGN
+
+
+CGEMM_L4x2_SUB2_8:
+/*----------------------------------------*/   
+    andi.      T1,L, 8
+    ble CGEMM_L4x2_SUB2_4
+    bl CGEMM_4x2_L8_SUB
+    MY_ALIGN  
+
+
+CGEMM_L4x2_SUB2_4:
+/*----------------------------------------*/   
+    andi.      T1,L, 4
+    ble CGEMM_L4x2_SUB2_2
+    LOAD4x2_2
+    KERNEL4x2_L2  32,64, 0,0
+    KERNEL4x2_E2  32,64, 1,1
+    MY_ALIGN
+
+
+CGEMM_L4x2_SUB2_2:
+/*----------------------------------------*/   
+    andi.      T1,L, 2
+    ble CGEMM_L4x2_SUB2_1
+    LOAD4x2_2
+    KERNEL4x2_E2  32,64, 0,1
+    MY_ALIGN    
+
+
+CGEMM_L4x2_SUB2_1:
+/*----------------------------------------*/   
+    andi.      T1,L, 1
+    ble CGEMM_L4x2_SAVE 
+    KERNEL4x2
+
+    MY_ALIGN
+CGEMM_L4x2_SAVE:
+/*----------------------------------------*/   
+    SAVE4x2
+#if defined(TRMMKERNEL)    
+    REFRESH_AFTER_SAVE T6,K,TEMP_REG,BO,AO,2,4
+#endif     
+
+
+CGEMM_L4x2_END:
+/*----------------------------------------*/   
+
+
+CGEMM_L4x1_BEGIN:
+/*----------------------------------------*/   
+    andi.   T1, M,  1
+    ble   CGEMM_L4x1_END
+#if defined(TRMMKERNEL)   
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,1,4
+#else    
+    mr    BO, B   
+#endif        
+#if defined(TRMMKERNEL)   
+    REFRESH_TEMP_BK T6,K,TEMP_REG,1,4
+    mr T1, T6 
+    addi T1,T1, -2 
+    srawi.   T8, T1, 5 /**(T1-2) % 32x */
+#else   
+    mr T1, K 
+    addi T1,T1, -2
+    srawi.   T8, T1, 5 /**(K-2) % 32x */
+#endif     
+    ZERO4x1
+    ble   CGEMM_L4x1_SUB0 
+    bl CGEMM_4x1_LMAIN_SUB
+    andi.   L,  T1, 31
+    ble   CGEMM_L4x1_SAVE
+    b   CGEMM_L4x1_SUB2
+
+
+CGEMM_L4x1_SUB0:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)   
+    andi.   L,  T6, 63
+    cmpwi   T6,33
+#else   
+    andi.   L,  K,  63
+    cmpwi   K,33
+#endif       
+    li T8,1
+    bne CMP4x1_32K
+    addi BO,BO,-32
+    addi AO,AO,-8  
+    LOAD4x1O 8,32 
+    END4x1_WITHOUT_ADD   
+    LOAD4x1_2O  16, 64  
+    mtctr   T8    
+    bl CGEMM_L4x1_K32   
+    b CGEMM_L4x1_SAVE  
+    CMP4x1_32K:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)    
+    cmpwi   T6,32
+#else    
+    cmpwi   K,32
+#endif        
+    bne CGEMM_L4x1_SUB2 
+    MY_ALIGN   
+    mtctr   T8
+    addi BO,BO,-64
+    addi AO,AO,-16   
+    LOAD4x1_2O 16,64
+    bl CGEMM_L4x1_K32   
+    b CGEMM_L4x1_SAVE 
+    MY_ALIGN 
+    MY_ALIGN 
+
+
+CGEMM_L4x1_SUB2:
+/*----------------------------------------*/   
+    andi.      T1,L, 16
+    ble CGEMM_L4x1_SUB2_8
+    bl CGEMM_4x1_L16_SUB  
+    MY_ALIGN
+
+
+CGEMM_L4x1_SUB2_8:
+/*----------------------------------------*/   
+    andi.      T1,L, 8
+    ble CGEMM_L4x1_SUB2_4
+    bl CGEMM_4x1_L8_SUB
+    MY_ALIGN  
+
+
+CGEMM_L4x1_SUB2_4:
+/*----------------------------------------*/   
+    andi.      T1,L, 4
+    ble CGEMM_L4x1_SUB2_2
+    LOAD4x1_2
+    KERNEL4x1_L2  16,64, 0,0
+    KERNEL4x1_E2  16,64, 1,1
+    MY_ALIGN
+
+
+CGEMM_L4x1_SUB2_2:
+/*----------------------------------------*/   
+    andi.      T1,L, 2
+    ble CGEMM_L4x1_SUB2_1
+    LOAD4x1_2
+    KERNEL4x1_E2  16,64, 0,1
+    MY_ALIGN    
+
+
+CGEMM_L4x1_SUB2_1:
+/*----------------------------------------*/   
+    andi.      T1,L, 1
+    ble CGEMM_L4x1_SAVE 
+    KERNEL4x1
+
+    MY_ALIGN
+CGEMM_L4x1_SAVE:
+/*----------------------------------------*/  
+     
+    SAVE4x1
+#if defined(TRMMKERNEL)    
+    REFRESH_AFTER_SAVE T6,K,TEMP_REG,BO,AO,1,4
+#endif   
+
+
+CGEMM_L4x1_END:
+/*----------------------------------------*/   
+    slwi    T1, K,  5
+    addic.    J,  J,  -1
+    add   B,  B,  T1
+#if defined(TRMMKERNEL) && !defined(LEFT)   
+    addi TEMP_REG, TEMP_REG, 4
+#endif   
+    bgt   CGEMM_L4_BEGIN
+
+
+CGEMM_L4_END:
+
+b CGEMM_L2
+/*                MINI SUBROUTINES                            */      
+/*                2x8 MAIN 128x+2 LOOP                     */
+
+
+CGEMM_L2x8_LMAIN_SUB:
+/*----------------------------------------*/   
+    mtctr   T8
+    LOAD2x8_2 
+    MY_ALIGN
+CGEMM_L2x8_LOOP:
+/*----------------------------------------*/   
+    dcbt    AO, PRE
+    dcbt    BO, PRE
+    KERNEL2x8_L2 128,32,0,0 
+CGEMM_L2x8_K128:
+/*----------------------------------------*/   
+    KERNEL2x8_L2 128,32,1,0
+    dcbt    AO, T2  
+    KERNEL2x8_L2 128,32,2,0
+    KERNEL2x8_L2 128,32,3,0 
+    dcbt    AO, T3
+    dcbt    BO, T2
+    KERNEL2x8_L2 128,32,4,0
+    KERNEL2x8_L2 128,32,5,0
+    dcbt    AO, T4  
+    KERNEL2x8_L2 128,32,6,0
+    KERNEL2x8_L2 128,32,7,0  
+    dcbt    AO, T5  
+    dcbt    BO, T3
+    KERNEL2x8_L2 128,32,8,0
+    KERNEL2x8_L2 128,32,9,0
+    KERNEL2x8_L2 128,32,10,0
+    KERNEL2x8_L2 128,32,11,0  
+    dcbt    BO, T4
+    KERNEL2x8_L2 128,32,12,0
+    KERNEL2x8_L2 128,32,13,0
+    KERNEL2x8_L2 128,32,14,0
+    KERNEL2x8_L2 128,32,15,0  
+    KERNEL2x8_L2 128,32,16,0
+    KERNEL2x8_L2 128,32,17,0 
+    KERNEL2x8_L2 128,32,18,0
+    KERNEL2x8_L2 128,32,19,0  
+    KERNEL2x8_L2 128,32,20,0
+    KERNEL2x8_L2 128,32,21,0 
+    KERNEL2x8_L2 128,32,22,0
+    KERNEL2x8_L2 128,32,23,0   
+    KERNEL2x8_L2 128,32,24,0
+    KERNEL2x8_L2 128,32,25,0
+    KERNEL2x8_L2 128,32,26,0
+    KERNEL2x8_L2 128,32,27,0  
+    KERNEL2x8_L2 128,32,28,0
+    KERNEL2x8_L2 128,32,29,0
+    KERNEL2x8_L2 128,32,30,0
+    KERNEL2x8_L2 128,32,31,0 
+    KERNEL2x8_L2 128,32,32,0
+    KERNEL2x8_L2 128,32,33,0
+    KERNEL2x8_L2 128,32,34,0
+    KERNEL2x8_L2 128,32,35,0 
+    KERNEL2x8_L2 128,32,36,0
+    KERNEL2x8_L2 128,32,37,0
+    KERNEL2x8_L2 128,32,38,0
+    KERNEL2x8_L2 128,32,39,0  
+    KERNEL2x8_L2 128,32,40,0
+    KERNEL2x8_L2 128,32,41,0
+    KERNEL2x8_L2 128,32,42,0
+    KERNEL2x8_L2 128,32,43,0  
+    KERNEL2x8_L2 128,32,44,0
+    KERNEL2x8_L2 128,32,45,0
+    KERNEL2x8_L2 128,32,46,0
+    KERNEL2x8_L2 128,32,47,0 
+    KERNEL2x8_L2 128,32,48,0
+    KERNEL2x8_L2 128,32,49,0 
+    KERNEL2x8_L2 128,32,50,0
+    KERNEL2x8_L2 128,32,51,0  
+    KERNEL2x8_L2 128,32,52,0
+    KERNEL2x8_L2 128,32,53,0 
+    KERNEL2x8_L2 128,32,54,0
+    KERNEL2x8_L2 128,32,55,0  
+    KERNEL2x8_L2 128,32,56,0
+    KERNEL2x8_L2 128,32,57,0
+    KERNEL2x8_L2 128,32,58,0
+    KERNEL2x8_L2 128,32,59,0  
+    KERNEL2x8_L2 128,32,60,0
+    KERNEL2x8_L2 128,32,61,0
+    KERNEL2x8_L2 128,32,62,0 
+    KERNEL2x8_L2 128,32,63,1  
+    bdnz    CGEMM_L2x8_LOOP
+    MY_ALIGN  
+CGEMM_L2x8_LOOP_END:
+/*----------------------------------------*/   
+    END2x8_2
+    blr
+    MY_ALIGN
+
+
+CGEMM_2x8_L64_SUB:
+/*----------------------------------------*/   
+    LOAD2x8_2  
+    dcbt    AO, PRE
+    dcbt    BO, PRE
+    KERNEL2x8_L2 128,32,0,0 
+    KERNEL2x8_L2 128,32,1,0
+    dcbt    AO, T2  
+    KERNEL2x8_L2 128,32,2,0
+    KERNEL2x8_L2 128,32,3,0 
+    dcbt    AO, T3
+    dcbt    BO, T2
+    KERNEL2x8_L2 128,32,4,0
+    KERNEL2x8_L2 128,32,5,0
+    dcbt    AO, T4  
+    KERNEL2x8_L2 128,32,6,0
+    KERNEL2x8_L2 128,32,7,0  
+    dcbt    AO, T5  
+    dcbt    BO, T3
+    KERNEL2x8_L2 128,32,8,0
+    KERNEL2x8_L2 128,32,9,0
+    KERNEL2x8_L2 128,32,10,0
+    KERNEL2x8_L2 128,32,11,0  
+    dcbt    BO, T4
+    KERNEL2x8_L2 128,32,12,0
+    KERNEL2x8_L2 128,32,13,0
+    KERNEL2x8_L2 128,32,14,0
+    KERNEL2x8_L2 128,32,15,0  
+    KERNEL2x8_L2 128,32,16,0
+    KERNEL2x8_L2 128,32,17,0 
+    KERNEL2x8_L2 128,32,18,0
+    KERNEL2x8_L2 128,32,19,0  
+    KERNEL2x8_L2 128,32,20,0
+    KERNEL2x8_L2 128,32,21,0 
+    KERNEL2x8_L2 128,32,22,0
+    KERNEL2x8_L2 128,32,23,0   
+    KERNEL2x8_L2 128,32,24,0
+    KERNEL2x8_L2 128,32,25,0
+    KERNEL2x8_L2 128,32,26,0
+    KERNEL2x8_L2 128,32,27,0  
+    KERNEL2x8_L2 128,32,28,0
+    KERNEL2x8_L2 128,32,29,0
+    KERNEL2x8_L2 128,32,30,0
+    KERNEL2x8_E2 128,32,31,1
+    blr
+    MY_ALIGN
+
+
+CGEMM_2x8_L32_SUB:
+/*----------------------------------------*/   
+    LOAD2x8_2  
+    dcbt    AO, PRE
+    dcbt    BO, PRE
+    KERNEL2x8_L2 128,32,0,0 
+    KERNEL2x8_L2 128,32,1,0
+    dcbt    AO, T2  
+    KERNEL2x8_L2 128,32,2,0
+    KERNEL2x8_L2 128,32,3,0 
+    dcbt    AO, T3
+    dcbt    BO, T2
+    KERNEL2x8_L2 128,32,4,0
+    KERNEL2x8_L2 128,32,5,0
+    dcbt    AO, T4  
+    KERNEL2x8_L2 128,32,6,0
+    KERNEL2x8_L2 128,32,7,0  
+    dcbt    AO, T5  
+    dcbt    BO, T3
+    KERNEL2x8_L2 128,32,8,0
+    KERNEL2x8_L2 128,32,9,0
+    KERNEL2x8_L2 128,32,10,0
+    KERNEL2x8_L2 128,32,11,0  
+    dcbt    BO, T4
+    KERNEL2x8_L2 128,32,12,0
+    KERNEL2x8_L2 128,32,13,0
+    KERNEL2x8_L2 128,32,14,0
+    KERNEL2x8_E2 128,32,15,1
+    blr
+    MY_ALIGN
+
+
+CGEMM_2x8_L16_SUB:
+/*----------------------------------------*/   
+    LOAD2x8_2 
+    dcbt    AO, PRE
+    dcbt    BO, PRE
+    KERNEL2x8_L2 128,32,0,0 
+    KERNEL2x8_L2 128,32,1,0
+    dcbt    AO, T2  
+    KERNEL2x8_L2 128,32,2,0
+    KERNEL2x8_L2 128,32,3,0 
+    dcbt    AO, T3
+    dcbt    BO, T2
+    KERNEL2x8_L2 128,32,4,0
+    KERNEL2x8_L2 128,32,5,0
+    dcbt    AO, T4  
+    KERNEL2x8_L2 128,32,6,0
+    KERNEL2x8_E2 128,32,7,1
+    blr
+    MY_ALIGN
+
+
+CGEMM_2x4_LMAIN_SUB:
+/*----------------------------------------*/   
+    mtctr   T8
+    LOAD2x4_2  
+    MY_ALIGN
+CGEMM_L2x4_LOOP:
+/*----------------------------------------*/   
+    KERNEL2x4_L2 64,32,0,0
+CGEMM_L2x4_K32:
+/*----------------------------------------*/   
+    KERNEL2x4_L2 64,32,1,0   
+    KERNEL2x4_L2 64,32,2,0
+    KERNEL2x4_L2 64,32,3,0  
+    KERNEL2x4_L2 64,32,4,0
+    KERNEL2x4_L2 64,32,5,0 
+    KERNEL2x4_L2 64,32,6,0
+    KERNEL2x4_L2 64,32,7,0
+    KERNEL2x4_L2 64,32,8,0
+    KERNEL2x4_L2 64,32,9,0   
+    KERNEL2x4_L2 64,32,10,0
+    KERNEL2x4_L2 64,32,11,0  
+    KERNEL2x4_L2 64,32,12,0
+    KERNEL2x4_L2 64,32,13,0 
+    KERNEL2x4_L2 64,32,14,0
+    KERNEL2x4_L2 64,32,15,1    
+    bdnz    CGEMM_L2x4_LOOP
+    MY_ALIGN  
+CGEMM_L2x4_LOOP_END:
+/*----------------------------------------*/   
+    END2x4_2 
+    blr
+    MY_ALIGN
+
+
+CGEMM_2x4_L16_SUB:
+/*----------------------------------------*/   
+    LOAD2x4_2
+    KERNEL2x4_L2 64,32,0,0
+    KERNEL2x4_L2 64,32,1,0   
+    KERNEL2x4_L2 64,32,2,0
+    KERNEL2x4_L2 64,32,3,0  
+    KERNEL2x4_L2 64,32,4,0
+    KERNEL2x4_L2 64,32,5,0 
+    KERNEL2x4_L2 64,32,6,0
+    KERNEL2x4_E2 64,32,7,1
+    blr
+    MY_ALIGN
+
+
+CGEMM_2x4_L8_SUB:
+/*----------------------------------------*/   
+    LOAD2x4_2
+    KERNEL2x4_L2 64,32,0,0
+    KERNEL2x4_L2 64,32,1,0   
+    KERNEL2x4_L2 64,32,2,0
+    KERNEL2x4_E2 64,32,3,1 
+    blr
+
+
+CGEMM_2x2_LMAIN_SUB:
+/*----------------------------------------*/   
+    mtctr   T8
+    LOAD2x2_2  
+    MY_ALIGN 
+CGEMM_L2x2_LOOP:
+/*----------------------------------------*/   
+    KERNEL2x2_L2 32,32,0,0 
+CGEMM_L2x2_K32:
+/*----------------------------------------*/   
+    KERNEL2x2_L2 32,32,1,0  
+    KERNEL2x2_L2 32,32,2,0
+    KERNEL2x2_L2 32,32,3,0  
+    KERNEL2x2_L2 32,32,4,0
+    KERNEL2x2_L2 32,32,5,0 
+    KERNEL2x2_L2 32,32,6,0
+    KERNEL2x2_L2 32,32,7,0
+    KERNEL2x2_L2 32,32,8,0
+    KERNEL2x2_L2 32,32,9,0  
+    KERNEL2x2_L2 32,32,10,0
+    KERNEL2x2_L2 32,32,11,0  
+    KERNEL2x2_L2 32,32,12,0
+    KERNEL2x2_L2 32,32,13,0 
+    KERNEL2x2_L2 32,32,14,0
+    KERNEL2x2_L2 32,32,15,1   
+    bdnz    CGEMM_L2x2_LOOP
+    MY_ALIGN  
+
+
+CGEMM_L2x2_LOOP_END:
+/*----------------------------------------*/   
+    END2x2_2 
+    blr
+    MY_ALIGN
+CGEMM_2x2_L16_SUB:
+/*----------------------------------------*/   
+    LOAD2x2_2
+    KERNEL2x2_L2 32,32,0,0
+    KERNEL2x2_L2 32,32,1,0  
+    KERNEL2x2_L2 32,32,2,0
+    KERNEL2x2_L2 32,32,3,0  
+    KERNEL2x2_L2 32,32,4,0
+    KERNEL2x2_L2 32,32,5,0 
+    KERNEL2x2_L2 32,32,6,0
+    KERNEL2x2_E2 32,32,7,1
+    blr
+    MY_ALIGN
+CGEMM_2x2_L8_SUB:
+/*----------------------------------------*/   
+    LOAD2x2_2
+    KERNEL2x2_L2 32,32,0,0
+    KERNEL2x2_L2 32,32,1,0  
+    KERNEL2x2_L2 32,32,2,0
+    KERNEL2x2_E2 32,32,3,1  
+    blr
+
+
+CGEMM_2x1_LMAIN_SUB:
+/*----------------------------------------*/   
+    mtctr   T8
+    LOAD2x1_2  
+    MY_ALIGN
+CGEMM_L2x1_LOOP:
+/*----------------------------------------*/   
+    KERNEL2x1_L2 16,32,0,0 
+CGEMM_L2x1_K32:
+/*----------------------------------------*/   
+    KERNEL2x1_L2 16,32,1,0  
+    KERNEL2x1_L2 16,32,2,0
+    KERNEL2x1_L2 16,32,3,0  
+    KERNEL2x1_L2 16,32,4,0
+    KERNEL2x1_L2 16,32,5,0 
+    KERNEL2x1_L2 16,32,6,0
+    KERNEL2x1_L2 16,32,7,0
+    KERNEL2x1_L2 16,32,8,0
+    KERNEL2x1_L2 16,32,9,0  
+    KERNEL2x1_L2 16,32,10,0
+    KERNEL2x1_L2 16,32,11,0  
+    KERNEL2x1_L2 16,32,12,0
+    KERNEL2x1_L2 16,32,13,0 
+    KERNEL2x1_L2 16,32,14,0
+    KERNEL2x1_L2 16,32,15,1   
+    bdnz    CGEMM_L2x1_LOOP
+    MY_ALIGN  
+CGEMM_L2x1_LOOP_END:
+/*----------------------------------------*/   
+    END2x1_2 
+    blr
+
+    MY_ALIGN
+CGEMM_2x1_L16_SUB:
+/*----------------------------------------*/   
+    LOAD2x1_2
+    KERNEL2x1_L2 16,32,0,0
+    KERNEL2x1_L2 16,32,1,0  
+    KERNEL2x1_L2 16,32,2,0
+    KERNEL2x1_L2 16,32,3,0  
+    KERNEL2x1_L2 16,32,4,0
+    KERNEL2x1_L2 16,32,5,0 
+    KERNEL2x1_L2 16,32,6,0
+    KERNEL2x1_E2 16,32,7,1
+    blr
+    MY_ALIGN
+
+
+CGEMM_2x1_L8_SUB:
+/*----------------------------------------*/   
+    LOAD2x1_2
+    KERNEL2x1_L2 16,32,0,0
+    KERNEL2x1_L2 16,32,1,0  
+    KERNEL2x1_L2 16,32,2,0
+    KERNEL2x1_E2 16,32,3,1  
+    blr
+
+
+
+/*             MAIN LOOP BEGINS               */   
+    MY_ALIGN
+
+
+CGEMM_L2:
+/*----------------------------------------*/   
+
+    andi.    J,  N,  2
+    ble   CGEMM_L2_END
+
+
+CGEMM_L2_BEGIN:
+/*----------------------------------------*/   
+    mr    CO, C
+    slwi    T1, LDC , 1     
+    add     T2,C,LDC    
+    mr    AO, A  
+    add   C,  C,  T1
+#if defined(TRMMKERNEL) && defined(LEFT)   
+    mr TEMP_REG, OFFSET  /*off = offset;*/
+#endif     
+    srawi.    I,  M, 3
+    ble   CGEMM_L2x8_END
+    dcbt    CO,r0  /*just prefetch*/
+    dcbt    T2,r0    
+
+
+CGEMM_L2x8_BEGIN:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)   
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,8,2
+#else    
+    mr    BO, B  
+    dcbt    B,  r0  
+#endif     
+    dcbt    AO, r0
+#if defined(TRMMKERNEL)   
+    REFRESH_TEMP_BK T6,K,TEMP_REG,8,2
+    mr T1, T6
+/* TEMPS FOR PREFETCH */   
+    li T2, 1024
+    li T3, 1024+512
+    addi T1,T1, -2
+/* TEMPS FOR PREFETCH */     
+    li T4, 2048
+    li T5, 2048+512   
+    srawi.   T8, T1, 7 /**(T1-2) % 128x */
+#else   
+    mr T1, K
+/* TEMPS FOR PREFETCH */   
+    li T2, 1024
+    li T3, 1024+512
+    addi T1,T1, -2
+/* TEMPS FOR PREFETCH */     
+    li T4, 2048
+    li T5, 2048+512 
+    srawi.   T8, T1, 7 /**(K-2) % 128x */
+#endif   
+    ZERO2x8  
+    ble   CGEMM_L2x8_SUB0
+    bl CGEMM_L2x8_LMAIN_SUB
+    andi.   L,  T1, 127
+    ble   CGEMM_L2x8_SAVE
+    b   CGEMM_L2x8_SUB2
+
+
+CGEMM_L2x8_SUB0:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)   
+    andi.   L,  T6, 255
+    cmpwi   T6,129
+#else   
+    andi.   L,  K,  255
+    cmpwi   K,129
+#endif       
+    li T8,1
+    bne CMP2x8_128K
+    addi BO,BO,-16
+    addi AO,AO,-64 
+    LOAD2x8O 64,16 
+    END2x8_WITHOUT_ADD   
+    LOAD2x8_2O  128, 32 
+    mtctr   T8    
+    bl CGEMM_L2x8_K128   
+    b CGEMM_L2x8_SAVE  
+    CMP2x8_128K:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)    
+    cmpwi   T6,128
+#else    
+    cmpwi   K,128
+#endif        
+    bne CGEMM_L2x8_SUB2 
+    MY_ALIGN   
+    mtctr   T8
+    addi BO,BO,-32
+    addi AO,AO,-128   
+    LOAD2x8_2O 128,32
+    bl CGEMM_L2x8_K128   
+    b CGEMM_L2x8_SAVE 
+    MY_ALIGN
+
+
+CGEMM_L2x8_SUB2:
+/*----------------------------------------*/   
+    andi.      T1,L, 64
+    ble CGEMM_L2x8_SUB2_32
+    bl  CGEMM_2x8_L64_SUB
+    MY_ALIGN
+
+
+CGEMM_L2x8_SUB2_32:
+/*----------------------------------------*/   
+    andi.      T1,L, 32
+    ble CGEMM_L2x8_SUB2_16    
+    bl  CGEMM_2x8_L32_SUB
+    MY_ALIGN 
+
+
+CGEMM_L2x8_SUB2_16:
+/*----------------------------------------*/   
+    andi.      T1,L, 16
+    ble CGEMM_L2x8_SUB2_8
+    bl  CGEMM_2x8_L16_SUB  
+    MY_ALIGN    
+
+
+CGEMM_L2x8_SUB2_8:
+/*----------------------------------------*/   
+    andi.      T1,L, 8
+    ble CGEMM_L2x8_SUB2_4
+    LOAD2x8_2
+    KERNEL2x8_L2  128,32, 0,0
+    KERNEL2x8_L2  128,32, 1,0
+    KERNEL2x8_L2  128,32, 2,0
+    KERNEL2x8_E2  128,32, 3,1
+    MY_ALIGN   
+
+
+CGEMM_L2x8_SUB2_4:
+/*----------------------------------------*/   
+    andi.      T1,L, 4
+    ble CGEMM_L2x8_SUB2_2
+    LOAD2x8_2
+    KERNEL2x8_L2  128,32, 0,0
+    KERNEL2x8_E2  128,32, 1,1
+    MY_ALIGN
+
+
+CGEMM_L2x8_SUB2_2:
+/*----------------------------------------*/   
+    andi.      T1,L, 2
+    ble CGEMM_L2x8_SUB2_1
+    LOAD2x8_2 
+    KERNEL2x8_E2  128,32, 0,1
+    MY_ALIGN    
+
+
+CGEMM_L2x8_SUB2_1:
+/*----------------------------------------*/   
+    andi.      T1,L, 1
+    ble CGEMM_L2x8_SAVE 
+    KERNEL2x8
+
+    MY_ALIGN
+CGEMM_L2x8_SAVE:
+/*----------------------------------------*/   
+    addic.    I,  I,  -1
+    MY_ALIGN
+    SAVE2x8
+#if defined(TRMMKERNEL)    
+    REFRESH_AFTER_SAVE T6,K,TEMP_REG,BO,AO,8,2
+#endif     
+    bgt   CGEMM_L2x8_BEGIN
+    andi.   T2, M,  7
+    ble   CGEMM_L2x1_END
+    andi.   T1, M,  4
+    ble   CGEMM_L2x4_END
+    b   CGEMM_L2x4_BEGIN
+    MY_ALIGN 
+
+
+CGEMM_L2x8_END:
+/*----------------------------------------*/   
+
+
+CGEMM_L2x4_BEGIN:
+/*----------------------------------------*/   
+    andi.   T2, M,  7
+    ble   CGEMM_L2x1_END
+    andi.   T1, M,  4
+    ble   CGEMM_L2x4_END
+#if defined(TRMMKERNEL)   
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,4,2
+#else    
+    mr    BO, B   
+#endif        
+#if defined(TRMMKERNEL)   
+    REFRESH_TEMP_BK T6,K,TEMP_REG,4,2
+    mr T1, T6 
+    addi T1,T1, -2 
+    srawi.   T8, T1, 5 /**(T1-2) % 32x */
+#else   
+    mr T1, K 
+    addi T1,T1, -2
+    srawi.   T8, T1, 5 /**(K-2) % 32x */
+#endif     
+    ZERO2x4
+    ble   CGEMM_L2x4_SUB0 
+    bl CGEMM_2x4_LMAIN_SUB
+    andi.   L,  T1, 31
+    ble   CGEMM_L2x4_SAVE
+    b    CGEMM_L2x4_SUB2
+
+
+CGEMM_L2x4_SUB0:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)   
+    andi.   L,  T6, 63
+    cmpwi   T6,33
+#else   
+    andi.   L,  K,  63
+    cmpwi   K,33
+#endif       
+    li T8,1
+    bne CMP2x4_32K
+    addi BO,BO,-16
+    addi AO,AO,-32  
+    LOAD2x4O 32,16 
+    END2x4_WITHOUT_ADD   
+    LOAD2x4_2O  64, 32 
+    mtctr   T8    
+    bl CGEMM_L2x4_K32   
+    b CGEMM_L2x4_SAVE  
+    CMP2x4_32K:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)    
+    cmpwi   T6,32
+#else    
+    cmpwi   K,32
+#endif        
+    bne CGEMM_L2x4_SUB2 
+    MY_ALIGN   
+    mtctr   T8
+    addi BO,BO,-32
+    addi AO,AO,-64   
+    LOAD2x4_2O 64,32
+    bl CGEMM_L2x4_K32   
+    b CGEMM_L2x4_SAVE 
+    MY_ALIGN 
+    MY_ALIGN 
+
+
+CGEMM_L2x4_SUB2:
+/*----------------------------------------*/   
+    andi.      T1,L, 16
+    ble CGEMM_L2x4_SUB2_8
+    bl  CGEMM_2x4_L16_SUB  
+    MY_ALIGN
+
+
+CGEMM_L2x4_SUB2_8:
+/*----------------------------------------*/   
+    andi.      T1,L, 8
+    ble CGEMM_L2x4_SUB2_4
+    bl CGEMM_2x4_L8_SUB
+    MY_ALIGN  
+
+
+CGEMM_L2x4_SUB2_4:
+/*----------------------------------------*/   
+    andi.      T1,L, 4
+    ble CGEMM_L2x4_SUB2_2
+    LOAD2x4_2
+    KERNEL2x4_L2  64,32, 0,0
+    KERNEL2x4_E2  64,32, 1,1
+    MY_ALIGN
+
+
+CGEMM_L2x4_SUB2_2:
+/*----------------------------------------*/   
+    andi.      T1,L, 2
+    ble CGEMM_L2x4_SUB2_1
+    LOAD2x4_2
+    KERNEL2x4_E2  64,32, 0,1
+    MY_ALIGN    
+
+
+CGEMM_L2x4_SUB2_1:
+/*----------------------------------------*/   
+    andi.      T1,L, 1
+    ble CGEMM_L2x4_SAVE 
+    KERNEL2x4
+
+
+CGEMM_L2x4_SAVE:
+/*----------------------------------------*/   
+    SAVE2x4
+#if defined(TRMMKERNEL)    
+    REFRESH_AFTER_SAVE T6,K,TEMP_REG,BO,AO,4,2
+#endif     
+
+
+CGEMM_L2x4_END:
+/*----------------------------------------*/   
+
+
+CGEMM_L2x2_BEGIN:
+/*----------------------------------------*/   
+    andi.   T1, M,  2
+    ble   CGEMM_L2x2_END
+#if defined(TRMMKERNEL)   
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,2,2
+#else    
+    mr    BO, B   
+#endif        
+#if defined(TRMMKERNEL)   
+    REFRESH_TEMP_BK T6,K,TEMP_REG,2,2
+    mr T1, T6 
+    addi T1,T1, -2 
+    srawi.   T8, T1, 5 /**(T1-2) % 32x */
+#else   
+    mr T1, K 
+    addi T1,T1, -2
+    srawi.   T8, T1, 5 /**(K-2) % 32x */
+#endif     
+    ZERO2x2
+    ble   CGEMM_L2x2_SUB0 
+    bl CGEMM_2x2_LMAIN_SUB
+    andi.   L,  T1, 31
+    ble   CGEMM_L2x2_SAVE
+    b   CGEMM_L2x2_SUB2
+
+
+CGEMM_L2x2_SUB0:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)   
+    andi.   L,  T6, 63
+    cmpwi   T6,33
+#else   
+    andi.   L,  K,  63
+    cmpwi   K,33
+#endif       
+    li T8,1
+    bne CMP2x2_32K
+    addi BO,BO,-16
+    addi AO,AO,-16  
+    LOAD2x2O 16,16 
+    END2x2_WITHOUT_ADD   
+    LOAD2x2_2O  32, 32  
+    mtctr   T8    
+    bl CGEMM_L2x2_K32   
+    b CGEMM_L2x2_SAVE  
+    CMP2x2_32K:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)    
+    cmpwi   T6,32
+#else    
+    cmpwi   K,32
+#endif        
+    bne CGEMM_L2x2_SUB2 
+    MY_ALIGN   
+    mtctr   T8
+    addi BO,BO,-32
+    addi AO,AO,-32   
+    LOAD2x2_2O 32,32
+    bl CGEMM_L2x2_K32   
+    b CGEMM_L2x2_SAVE 
+    MY_ALIGN 
+    MY_ALIGN 
+
+
+CGEMM_L2x2_SUB2:
+/*----------------------------------------*/   
+    andi.      T1,L, 16
+    ble CGEMM_L2x2_SUB2_8
+    bl CGEMM_2x2_L16_SUB  
+    MY_ALIGN
+
+
+CGEMM_L2x2_SUB2_8:
+/*----------------------------------------*/   
+    andi.      T1,L, 8
+    ble CGEMM_L2x2_SUB2_4
+    bl CGEMM_2x2_L8_SUB
+    MY_ALIGN  
+
+
+CGEMM_L2x2_SUB2_4:
+/*----------------------------------------*/   
+    andi.      T1,L, 4
+    ble CGEMM_L2x2_SUB2_2
+    LOAD2x2_2
+    KERNEL2x2_L2  32,32, 0,0
+    KERNEL2x2_E2  32,32, 1,1
+    MY_ALIGN
+
+
+CGEMM_L2x2_SUB2_2:
+/*----------------------------------------*/   
+    andi.      T1,L, 2
+    ble CGEMM_L2x2_SUB2_1
+    LOAD2x2_2
+    KERNEL2x2_E2  32,32, 0,1
+    MY_ALIGN    
+
+
+CGEMM_L2x2_SUB2_1:
+/*----------------------------------------*/   
+    andi.      T1,L, 1
+    ble CGEMM_L2x2_SAVE 
+    KERNEL2x2
+
+    MY_ALIGN
+CGEMM_L2x2_SAVE:
+/*----------------------------------------*/   
+    SAVE2x2
+#if defined(TRMMKERNEL)    
+    REFRESH_AFTER_SAVE T6,K,TEMP_REG,BO,AO,2,2
+#endif     
+
+
+CGEMM_L2x2_END:
+/*----------------------------------------*/   
+
+
+CGEMM_L2x1_BEGIN:
+/*----------------------------------------*/   
+    andi.   T1, M,  1
+    ble   CGEMM_L2x1_END
+#if defined(TRMMKERNEL)   
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,1,2
+#else    
+    mr    BO, B   
+#endif        
+#if defined(TRMMKERNEL)   
+    REFRESH_TEMP_BK T6,K,TEMP_REG,1,2
+    mr T1, T6 
+    addi T1,T1, -2 
+    srawi.   T8, T1, 5 /**(T1-2) % 32x */
+#else   
+    mr T1, K 
+    addi T1,T1, -2
+    srawi.   T8, T1, 5 /**(K-2) % 32x */
+#endif     
+    ZERO2x1
+    ble   CGEMM_L2x1_SUB0 
+    bl CGEMM_2x1_LMAIN_SUB
+    andi.   L,  T1, 31
+    ble   CGEMM_L2x1_SAVE
+    b   CGEMM_L2x1_SUB2
+
+
+CGEMM_L2x1_SUB0:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)   
+    andi.   L,  T6, 63
+    cmpwi   T6,33
+#else   
+    andi.   L,  K,  63
+    cmpwi   K,33
+#endif       
+    li T8,1
+    bne CMP2x1_32K
+    addi BO,BO,-16
+    addi AO,AO,-8  
+    LOAD2x1O 8,16 
+    END2x1_WITHOUT_ADD   
+    LOAD2x1_2O  16, 32  
+    mtctr   T8    
+    bl CGEMM_L2x1_K32   
+    b CGEMM_L2x1_SAVE  
+    CMP2x1_32K:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)    
+    cmpwi   T6,32
+#else    
+    cmpwi   K,32
+#endif        
+    bne CGEMM_L2x1_SUB2 
+    MY_ALIGN   
+    mtctr   T8
+    addi BO,BO,-32
+    addi AO,AO,-16   
+    LOAD2x1_2O 16,32
+    bl CGEMM_L2x1_K32   
+    b CGEMM_L2x1_SAVE 
+    MY_ALIGN 
+    MY_ALIGN 
+
+
+CGEMM_L2x1_SUB2:
+/*----------------------------------------*/   
+    andi.      T1,L, 16
+    ble CGEMM_L2x1_SUB2_8
+    bl CGEMM_2x1_L16_SUB  
+    MY_ALIGN
+
+
+CGEMM_L2x1_SUB2_8:
+/*----------------------------------------*/   
+    andi.      T1,L, 8
+    ble CGEMM_L2x1_SUB2_4
+    bl CGEMM_2x1_L8_SUB
+    MY_ALIGN  
+
+
+CGEMM_L2x1_SUB2_4:
+/*----------------------------------------*/   
+    andi.      T1,L, 4
+    ble CGEMM_L2x1_SUB2_2
+    LOAD2x1_2
+    KERNEL2x1_L2  16,32, 0,0
+    KERNEL2x1_E2  16,32, 1,1
+    MY_ALIGN
+
+
+CGEMM_L2x1_SUB2_2:
+/*----------------------------------------*/   
+    andi.      T1,L, 2
+    ble CGEMM_L2x1_SUB2_1
+    LOAD2x1_2
+    KERNEL2x1_E2  16,32, 0,1
+    MY_ALIGN    
+
+
+CGEMM_L2x1_SUB2_1:
+/*----------------------------------------*/   
+    andi.      T1,L, 1
+    ble CGEMM_L2x1_SAVE 
+    KERNEL2x1
+
+    MY_ALIGN
+CGEMM_L2x1_SAVE:
+/*----------------------------------------*/  
+     
+    SAVE2x1
+#if defined(TRMMKERNEL)    
+    REFRESH_AFTER_SAVE T6,K,TEMP_REG,BO,AO,1,2
+#endif   
+
+
+CGEMM_L2x1_END:
+/*----------------------------------------*/   
+    slwi    T1, K,  4
+
+    add   B,  B,  T1
+#if defined(TRMMKERNEL) && !defined(LEFT)   
+    addi TEMP_REG, TEMP_REG, 2
+#endif   
+
+CGEMM_L2_END:
+
+
+b CGEMM_L1
+/*                MINI SUBROUTINES                            */      
+/*                1x8 MAIN 128x+2 LOOP                     */      
+
+
+CGEMM_L1x8_LMAIN_SUB:
+/*----------------------------------------*/   
+    mtctr   T8
+    LOAD1x8_2 
+    MY_ALIGN
+CGEMM_L1x8_LOOP:
+/*----------------------------------------*/   
+    dcbt    AO, PRE
+    dcbt    BO, PRE
+    KERNEL1x8_L2 128,16,0,0 
+CGEMM_L1x8_K128:
+/*----------------------------------------*/   
+    KERNEL1x8_L2 128,16,1,0
+    dcbt    AO, T2  
+    KERNEL1x8_L2 128,16,2,0
+    KERNEL1x8_L2 128,16,3,0 
+    dcbt    AO, T3
+    dcbt    BO, T2
+    KERNEL1x8_L2 128,16,4,0
+    KERNEL1x8_L2 128,16,5,0
+    dcbt    AO, T4  
+    KERNEL1x8_L2 128,16,6,0
+    KERNEL1x8_L2 128,16,7,0  
+    dcbt    AO, T5  
+    dcbt    BO, T3
+    KERNEL1x8_L2 128,16,8,0
+    KERNEL1x8_L2 128,16,9,0
+    KERNEL1x8_L2 128,16,10,0
+    KERNEL1x8_L2 128,16,11,0  
+    dcbt    BO, T4
+    KERNEL1x8_L2 128,16,12,0
+    KERNEL1x8_L2 128,16,13,0
+    KERNEL1x8_L2 128,16,14,0
+    KERNEL1x8_L2 128,16,15,0  
+    KERNEL1x8_L2 128,16,16,0
+    KERNEL1x8_L2 128,16,17,0 
+    KERNEL1x8_L2 128,16,18,0
+    KERNEL1x8_L2 128,16,19,0  
+    KERNEL1x8_L2 128,16,20,0
+    KERNEL1x8_L2 128,16,21,0 
+    KERNEL1x8_L2 128,16,22,0
+    KERNEL1x8_L2 128,16,23,0   
+    KERNEL1x8_L2 128,16,24,0
+    KERNEL1x8_L2 128,16,25,0
+    KERNEL1x8_L2 128,16,26,0
+    KERNEL1x8_L2 128,16,27,0  
+    KERNEL1x8_L2 128,16,28,0
+    KERNEL1x8_L2 128,16,29,0
+    KERNEL1x8_L2 128,16,30,0
+    KERNEL1x8_L2 128,16,31,0 
+    KERNEL1x8_L2 128,16,32,0
+    KERNEL1x8_L2 128,16,33,0
+    KERNEL1x8_L2 128,16,34,0
+    KERNEL1x8_L2 128,16,35,0 
+    KERNEL1x8_L2 128,16,36,0
+    KERNEL1x8_L2 128,16,37,0
+    KERNEL1x8_L2 128,16,38,0
+    KERNEL1x8_L2 128,16,39,0  
+    KERNEL1x8_L2 128,16,40,0
+    KERNEL1x8_L2 128,16,41,0
+    KERNEL1x8_L2 128,16,42,0
+    KERNEL1x8_L2 128,16,43,0  
+    KERNEL1x8_L2 128,16,44,0
+    KERNEL1x8_L2 128,16,45,0
+    KERNEL1x8_L2 128,16,46,0
+    KERNEL1x8_L2 128,16,47,0 
+    KERNEL1x8_L2 128,16,48,0
+    KERNEL1x8_L2 128,16,49,0 
+    KERNEL1x8_L2 128,16,50,0
+    KERNEL1x8_L2 128,16,51,0  
+    KERNEL1x8_L2 128,16,52,0
+    KERNEL1x8_L2 128,16,53,0 
+    KERNEL1x8_L2 128,16,54,0
+    KERNEL1x8_L2 128,16,55,0  
+    KERNEL1x8_L2 128,16,56,0
+    KERNEL1x8_L2 128,16,57,0
+    KERNEL1x8_L2 128,16,58,0
+    KERNEL1x8_L2 128,16,59,0  
+    KERNEL1x8_L2 128,16,60,0
+    KERNEL1x8_L2 128,16,61,0
+    KERNEL1x8_L2 128,16,62,0 
+    KERNEL1x8_L2 128,16,63,1  
+    bdnz    CGEMM_L1x8_LOOP
+    MY_ALIGN  
+CGEMM_L1x8_LOOP_END:
+/*----------------------------------------*/   
+    END1x8_2
+    blr
+    MY_ALIGN
+
+
+CGEMM_1x8_L64_SUB:
+/*----------------------------------------*/   
+    LOAD1x8_2  
+    dcbt    AO, PRE
+    dcbt    BO, PRE
+    KERNEL1x8_L2 128,16,0,0 
+    KERNEL1x8_L2 128,16,1,0
+    dcbt    AO, T2  
+    KERNEL1x8_L2 128,16,2,0
+    KERNEL1x8_L2 128,16,3,0 
+    dcbt    AO, T3
+    dcbt    BO, T2
+    KERNEL1x8_L2 128,16,4,0
+    KERNEL1x8_L2 128,16,5,0
+    dcbt    AO, T4  
+    KERNEL1x8_L2 128,16,6,0
+    KERNEL1x8_L2 128,16,7,0  
+    dcbt    AO, T5  
+    dcbt    BO, T3
+    KERNEL1x8_L2 128,16,8,0
+    KERNEL1x8_L2 128,16,9,0
+    KERNEL1x8_L2 128,16,10,0
+    KERNEL1x8_L2 128,16,11,0  
+    dcbt    BO, T4
+    KERNEL1x8_L2 128,16,12,0
+    KERNEL1x8_L2 128,16,13,0
+    KERNEL1x8_L2 128,16,14,0
+    KERNEL1x8_L2 128,16,15,0  
+    KERNEL1x8_L2 128,16,16,0
+    KERNEL1x8_L2 128,16,17,0 
+    KERNEL1x8_L2 128,16,18,0
+    KERNEL1x8_L2 128,16,19,0  
+    KERNEL1x8_L2 128,16,20,0
+    KERNEL1x8_L2 128,16,21,0 
+    KERNEL1x8_L2 128,16,22,0
+    KERNEL1x8_L2 128,16,23,0   
+    KERNEL1x8_L2 128,16,24,0
+    KERNEL1x8_L2 128,16,25,0
+    KERNEL1x8_L2 128,16,26,0
+    KERNEL1x8_L2 128,16,27,0  
+    KERNEL1x8_L2 128,16,28,0
+    KERNEL1x8_L2 128,16,29,0
+    KERNEL1x8_L2 128,16,30,0
+    KERNEL1x8_E2 128,16,31,1
+    blr
+    MY_ALIGN
+
+
+CGEMM_1x8_L32_SUB:
+/*----------------------------------------*/   
+    LOAD1x8_2  
+    dcbt    AO, PRE
+    dcbt    BO, PRE
+    KERNEL1x8_L2 128,16,0,0 
+    KERNEL1x8_L2 128,16,1,0
+    dcbt    AO, T2  
+    KERNEL1x8_L2 128,16,2,0
+    KERNEL1x8_L2 128,16,3,0 
+    dcbt    AO, T3
+    dcbt    BO, T2
+    KERNEL1x8_L2 128,16,4,0
+    KERNEL1x8_L2 128,16,5,0
+    dcbt    AO, T4  
+    KERNEL1x8_L2 128,16,6,0
+    KERNEL1x8_L2 128,16,7,0  
+    dcbt    AO, T5  
+    dcbt    BO, T3
+    KERNEL1x8_L2 128,16,8,0
+    KERNEL1x8_L2 128,16,9,0
+    KERNEL1x8_L2 128,16,10,0
+    KERNEL1x8_L2 128,16,11,0  
+    dcbt    BO, T4
+    KERNEL1x8_L2 128,16,12,0
+    KERNEL1x8_L2 128,16,13,0
+    KERNEL1x8_L2 128,16,14,0
+    KERNEL1x8_E2 128,16,15,1
+    blr
+    MY_ALIGN
+
+
+CGEMM_1x8_L16_SUB:
+/*----------------------------------------*/   
+    LOAD1x8_2 
+    dcbt    AO, PRE
+    dcbt    BO, PRE
+    KERNEL1x8_L2 128,16,0,0 
+    KERNEL1x8_L2 128,16,1,0
+    dcbt    AO, T2  
+    KERNEL1x8_L2 128,16,2,0
+    KERNEL1x8_L2 128,16,3,0 
+    dcbt    AO, T3
+    dcbt    BO, T2
+    KERNEL1x8_L2 128,16,4,0
+    KERNEL1x8_L2 128,16,5,0
+    dcbt    AO, T4  
+    KERNEL1x8_L2 128,16,6,0
+    KERNEL1x8_E2 128,16,7,1
+    blr
+    MY_ALIGN
+
+
+CGEMM_1x4_LMAIN_SUB:
+/*----------------------------------------*/   
+    mtctr   T8
+    LOAD1x4_2  
+    MY_ALIGN
+CGEMM_L1x4_LOOP:
+/*----------------------------------------*/   
+    KERNEL1x4_L2 64,16,0,0
+CGEMM_L1x4_K32:
+/*----------------------------------------*/   
+    KERNEL1x4_L2 64,16,1,0   
+    KERNEL1x4_L2 64,16,2,0
+    KERNEL1x4_L2 64,16,3,0  
+    KERNEL1x4_L2 64,16,4,0
+    KERNEL1x4_L2 64,16,5,0 
+    KERNEL1x4_L2 64,16,6,0
+    KERNEL1x4_L2 64,16,7,0
+    KERNEL1x4_L2 64,16,8,0
+    KERNEL1x4_L2 64,16,9,0   
+    KERNEL1x4_L2 64,16,10,0
+    KERNEL1x4_L2 64,16,11,0  
+    KERNEL1x4_L2 64,16,12,0
+    KERNEL1x4_L2 64,16,13,0 
+    KERNEL1x4_L2 64,16,14,0
+    KERNEL1x4_L2 64,16,15,1    
+    bdnz    CGEMM_L1x4_LOOP
+    MY_ALIGN  
+CGEMM_L1x4_LOOP_END:
+/*----------------------------------------*/   
+    END1x4_2 
+    blr
+    MY_ALIGN
+
+
+CGEMM_1x4_L16_SUB:
+/*----------------------------------------*/   
+    LOAD1x4_2
+    KERNEL1x4_L2 64,16,0,0
+    KERNEL1x4_L2 64,16,1,0   
+    KERNEL1x4_L2 64,16,2,0
+    KERNEL1x4_L2 64,16,3,0  
+    KERNEL1x4_L2 64,16,4,0
+    KERNEL1x4_L2 64,16,5,0 
+    KERNEL1x4_L2 64,16,6,0
+    KERNEL1x4_E2 64,16,7,1
+    blr
+    MY_ALIGN
+
+
+CGEMM_1x4_L8_SUB:
+/*----------------------------------------*/   
+    LOAD1x4_2
+    KERNEL1x4_L2 64,16,0,0
+    KERNEL1x4_L2 64,16,1,0   
+    KERNEL1x4_L2 64,16,2,0
+    KERNEL1x4_E2 64,16,3,1 
+    blr
+
+
+CGEMM_1x2_LMAIN_SUB:
+/*----------------------------------------*/   
+    mtctr   T8
+    LOAD1x2_2  
+    MY_ALIGN 
+CGEMM_L1x2_LOOP:
+/*----------------------------------------*/   
+    KERNEL1x2_L2 32,16,0,0 
+CGEMM_L1x2_K32:
+/*----------------------------------------*/   
+    KERNEL1x2_L2 32,16,1,0  
+    KERNEL1x2_L2 32,16,2,0
+    KERNEL1x2_L2 32,16,3,0  
+    KERNEL1x2_L2 32,16,4,0
+    KERNEL1x2_L2 32,16,5,0 
+    KERNEL1x2_L2 32,16,6,0
+    KERNEL1x2_L2 32,16,7,0
+    KERNEL1x2_L2 32,16,8,0
+    KERNEL1x2_L2 32,16,9,0  
+    KERNEL1x2_L2 32,16,10,0
+    KERNEL1x2_L2 32,16,11,0  
+    KERNEL1x2_L2 32,16,12,0
+    KERNEL1x2_L2 32,16,13,0 
+    KERNEL1x2_L2 32,16,14,0
+    KERNEL1x2_L2 32,16,15,1   
+    bdnz    CGEMM_L1x2_LOOP
+    MY_ALIGN  
+
+
+CGEMM_L1x2_LOOP_END:
+/*----------------------------------------*/   
+    END1x2_2 
+    blr
+    MY_ALIGN
+CGEMM_1x2_L16_SUB:
+/*----------------------------------------*/   
+    LOAD1x2_2
+    KERNEL1x2_L2 32,16,0,0
+    KERNEL1x2_L2 32,16,1,0  
+    KERNEL1x2_L2 32,16,2,0
+    KERNEL1x2_L2 32,16,3,0  
+    KERNEL1x2_L2 32,16,4,0
+    KERNEL1x2_L2 32,16,5,0 
+    KERNEL1x2_L2 32,16,6,0
+    KERNEL1x2_E2 32,16,7,1
+    blr
+    MY_ALIGN
+CGEMM_1x2_L8_SUB:
+/*----------------------------------------*/   
+    LOAD1x2_2
+    KERNEL1x2_L2 32,16,0,0
+    KERNEL1x2_L2 32,16,1,0  
+    KERNEL1x2_L2 32,16,2,0
+    KERNEL1x2_E2 32,16,3,1  
+    blr
+
+
+CGEMM_1x1_LMAIN_SUB:
+/*----------------------------------------*/   
+    mtctr   T8
+    LOAD1x1_2  
+    MY_ALIGN
+CGEMM_L1x1_LOOP:
+/*----------------------------------------*/   
+    KERNEL1x1_L2 16,16,0,0 
+CGEMM_L1x1_K32:
+/*----------------------------------------*/   
+    KERNEL1x1_L2 16,16,1,0  
+    KERNEL1x1_L2 16,16,2,0
+    KERNEL1x1_L2 16,16,3,0  
+    KERNEL1x1_L2 16,16,4,0
+    KERNEL1x1_L2 16,16,5,0 
+    KERNEL1x1_L2 16,16,6,0
+    KERNEL1x1_L2 16,16,7,0
+    KERNEL1x1_L2 16,16,8,0
+    KERNEL1x1_L2 16,16,9,0  
+    KERNEL1x1_L2 16,16,10,0
+    KERNEL1x1_L2 16,16,11,0  
+    KERNEL1x1_L2 16,16,12,0
+    KERNEL1x1_L2 16,16,13,0 
+    KERNEL1x1_L2 16,16,14,0
+    KERNEL1x1_L2 16,16,15,1   
+    bdnz    CGEMM_L1x1_LOOP
+    MY_ALIGN  
+CGEMM_L1x1_LOOP_END:
+/*----------------------------------------*/   
+    END1x1_2 
+    blr
+
+    MY_ALIGN
+CGEMM_1x1_L16_SUB:
+/*----------------------------------------*/   
+    LOAD1x1_2
+    KERNEL1x1_L2 16,16,0,0
+    KERNEL1x1_L2 16,16,1,0  
+    KERNEL1x1_L2 16,16,2,0
+    KERNEL1x1_L2 16,16,3,0  
+    KERNEL1x1_L2 16,16,4,0
+    KERNEL1x1_L2 16,16,5,0 
+    KERNEL1x1_L2 16,16,6,0
+    KERNEL1x1_E2 16,16,7,1
+    blr
+    MY_ALIGN
+
+
+CGEMM_1x1_L8_SUB:
+/*----------------------------------------*/   
+    LOAD1x1_2
+    KERNEL1x1_L2 16,16,0,0
+    KERNEL1x1_L2 16,16,1,0  
+    KERNEL1x1_L2 16,16,2,0
+    KERNEL1x1_E2 16,16,3,1  
+    blr
+
+
+
+/*             MAIN LOOP BEGINS               */   
+    MY_ALIGN
+
+
+CGEMM_L1:
+/*----------------------------------------*/   
+
+    andi.    J,  N,  1
+    ble   CGEMM_L1_END
+
+CGEMM_L1_BEGIN:
+/*----------------------------------------*/   
+    mr    CO, C  
+    add     T2,C,LDC    
+    mr    AO, A  
+    add   C,  C,  T1
+#if defined(TRMMKERNEL) && defined(LEFT)   
+    mr TEMP_REG, OFFSET  /*off = offset;*/
+#endif     
+    srawi.    I,  M,  3
+    ble   CGEMM_L1x8_END
+    dcbt    CO,r0  /*just prefetch*/
+    dcbt    T2,r0    
+
+
+CGEMM_L1x8_BEGIN:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)   
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,8,1
+#else    
+    mr    BO, B  
+    dcbt    B,  r0  
+#endif     
+    dcbt    AO, r0
+#if defined(TRMMKERNEL)   
+    REFRESH_TEMP_BK T6,K,TEMP_REG,8,1
+    mr T1, T6
+/* TEMPS FOR PREFETCH */   
+    li T2, 1024
+    li T3, 1024+512
+    addi T1,T1, -2
+/* TEMPS FOR PREFETCH */     
+    li T4, 2048
+    li T5, 2048+512   
+    srawi.   T8, T1, 7 /**(T1-2) % 128x */
+#else   
+    mr T1, K
+/* TEMPS FOR PREFETCH */   
+    li T2, 1024
+    li T3, 1024+512
+    addi T1,T1, -2
+/* TEMPS FOR PREFETCH */     
+    li T4, 2048
+    li T5, 2048+512 
+    srawi.   T8, T1, 7 /**(K-2) % 128x */
+#endif   
+    ZERO1x8  
+    ble   CGEMM_L1x8_SUB0
+    bl CGEMM_L1x8_LMAIN_SUB
+    andi.   L,  T1, 127
+    ble   CGEMM_L1x8_SAVE
+    b   CGEMM_L1x8_SUB2
+
+
+CGEMM_L1x8_SUB0:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)   
+    andi.   L,  T6, 255
+    cmpwi   T6,129
+#else   
+    andi.   L,  K,  255
+    cmpwi   K,129
+#endif       
+    li T8,1
+    bne CMP1x8_128K
+    addi BO,BO,-8
+    addi AO,AO,-64 
+    LOAD1x8O 64,8 
+    END1x8_WITHOUT_ADD   
+    LOAD1x8_2O  128, 16 
+    mtctr   T8    
+    bl CGEMM_L1x8_K128   
+    b CGEMM_L1x8_SAVE  
+    CMP1x8_128K:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)    
+    cmpwi   T6,128
+#else    
+    cmpwi   K,128
+#endif        
+    bne CGEMM_L1x8_SUB2 
+    MY_ALIGN   
+    mtctr   T8
+    addi BO,BO,-16
+    addi AO,AO,-128   
+    LOAD1x8_2O 128,16
+    bl CGEMM_L1x8_K128   
+    b CGEMM_L1x8_SAVE 
+    MY_ALIGN
+
+
+CGEMM_L1x8_SUB2:
+/*----------------------------------------*/   
+    andi.      T1,L, 64
+    ble CGEMM_L1x8_SUB2_32
+    bl  CGEMM_1x8_L64_SUB
+    MY_ALIGN
+
+
+CGEMM_L1x8_SUB2_32:
+/*----------------------------------------*/   
+    andi.      T1,L, 32
+    ble CGEMM_L1x8_SUB2_16    
+    bl  CGEMM_1x8_L32_SUB
+    MY_ALIGN 
+
+
+CGEMM_L1x8_SUB2_16:
+/*----------------------------------------*/   
+    andi.      T1,L, 16
+    ble CGEMM_L1x8_SUB2_8
+    bl  CGEMM_1x8_L16_SUB  
+    MY_ALIGN    
+
+
+CGEMM_L1x8_SUB2_8:
+/*----------------------------------------*/   
+    andi.      T1,L, 8
+    ble CGEMM_L1x8_SUB2_4
+    LOAD1x8_2
+    KERNEL1x8_L2  128,16, 0,0
+    KERNEL1x8_L2  128,16, 1,0
+    KERNEL1x8_L2  128,16, 2,0
+    KERNEL1x8_E2  128,16, 3,1
+    MY_ALIGN   
+
+
+CGEMM_L1x8_SUB2_4:
+/*----------------------------------------*/   
+    andi.      T1,L, 4
+    ble CGEMM_L1x8_SUB2_2
+    LOAD1x8_2
+    KERNEL1x8_L2  128,16, 0,0
+    KERNEL1x8_E2  128,16, 1,1
+    MY_ALIGN
+
+
+CGEMM_L1x8_SUB2_2:
+/*----------------------------------------*/   
+    andi.      T1,L, 2
+    ble CGEMM_L1x8_SUB2_1
+    LOAD1x8_2 
+    KERNEL1x8_E2  128,16, 0,1
+    MY_ALIGN    
+
+
+CGEMM_L1x8_SUB2_1:
+/*----------------------------------------*/   
+    andi.      T1,L, 1
+    ble CGEMM_L1x8_SAVE 
+    KERNEL1x8
+
+    MY_ALIGN
+CGEMM_L1x8_SAVE:
+/*----------------------------------------*/   
+    addic.    I,  I,  -1
+    MY_ALIGN
+    SAVE1x8
+#if defined(TRMMKERNEL)    
+    REFRESH_AFTER_SAVE T6,K,TEMP_REG,BO,AO,8,1
+#endif     
+    bgt   CGEMM_L1x8_BEGIN
+    andi.   T2, M,  7
+    ble   CGEMM_L1x1_END
+    andi.   T1, M,  4
+    ble   CGEMM_L1x4_END
+    b   CGEMM_L1x4_BEGIN
+    MY_ALIGN 
+
+
+CGEMM_L1x8_END:
+/*----------------------------------------*/   
+
+
+CGEMM_L1x4_BEGIN:
+/*----------------------------------------*/   
+    andi.   T2, M,  7
+    ble   CGEMM_L1x1_END
+    andi.   T1, M,  4
+    ble   CGEMM_L1x4_END
+#if defined(TRMMKERNEL)   
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,4,1
+#else    
+    mr    BO, B   
+#endif        
+#if defined(TRMMKERNEL)   
+    REFRESH_TEMP_BK T6,K,TEMP_REG,4,1
+    mr T1, T6 
+    addi T1,T1, -2 
+    srawi.   T8, T1, 5 /**(T1-2) % 31x */
+#else   
+    mr T1, K 
+    addi T1,T1, -2
+    srawi.   T8, T1, 5 /**(K-2) % 31x */
+#endif     
+    ZERO1x4
+    ble   CGEMM_L1x4_SUB0 
+    bl CGEMM_1x4_LMAIN_SUB
+    andi.   L,  T1, 31
+    ble   CGEMM_L1x4_SAVE
+    b    CGEMM_L1x4_SUB2
+
+
+CGEMM_L1x4_SUB0:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)   
+    andi.   L,  T6, 63
+    cmpwi   T6,33
+#else   
+    andi.   L,  K,  63
+    cmpwi   K,33
+#endif       
+    li T8,1
+    bne CMP1x4_32K
+    addi BO,BO,-8
+    addi AO,AO,-32  
+    LOAD1x4O 32,8 
+    END1x4_WITHOUT_ADD   
+    LOAD1x4_2O  64, 16 
+    mtctr   T8    
+    bl CGEMM_L1x4_K32   
+    b CGEMM_L1x4_SAVE  
+    CMP1x4_32K:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)    
+    cmpwi   T6,32
+#else    
+    cmpwi   K,32
+#endif        
+    bne CGEMM_L1x4_SUB2 
+    MY_ALIGN   
+    mtctr   T8
+    addi BO,BO,-16
+    addi AO,AO,-64   
+    LOAD1x4_2O 64,16
+    bl CGEMM_L1x4_K32   
+    b CGEMM_L1x4_SAVE 
+    MY_ALIGN 
+    MY_ALIGN 
+
+
+CGEMM_L1x4_SUB2:
+/*----------------------------------------*/   
+    andi.      T1,L, 16
+    ble CGEMM_L1x4_SUB2_8
+    bl  CGEMM_1x4_L16_SUB  
+    MY_ALIGN
+
+
+CGEMM_L1x4_SUB2_8:
+/*----------------------------------------*/   
+    andi.      T1,L, 8
+    ble CGEMM_L1x4_SUB2_4
+    bl CGEMM_1x4_L8_SUB
+    MY_ALIGN  
+
+
+CGEMM_L1x4_SUB2_4:
+/*----------------------------------------*/   
+    andi.      T1,L, 4
+    ble CGEMM_L1x4_SUB2_2
+    LOAD1x4_2
+    KERNEL1x4_L2  64,16, 0,0
+    KERNEL1x4_E2  64,16, 1,1
+    MY_ALIGN
+
+
+CGEMM_L1x4_SUB2_2:
+/*----------------------------------------*/   
+    andi.      T1,L, 2
+    ble CGEMM_L1x4_SUB2_1
+    LOAD1x4_2
+    KERNEL1x4_E2  64,16, 0,1
+    MY_ALIGN    
+
+
+CGEMM_L1x4_SUB2_1:
+/*----------------------------------------*/   
+    andi.      T1,L, 1
+    ble CGEMM_L1x4_SAVE 
+    KERNEL1x4
+
+
+CGEMM_L1x4_SAVE:
+/*----------------------------------------*/   
+    SAVE1x4
+#if defined(TRMMKERNEL)    
+    REFRESH_AFTER_SAVE T6,K,TEMP_REG,BO,AO,4,1
+#endif     
+
+
+CGEMM_L1x4_END:
+/*----------------------------------------*/   
+
+
+CGEMM_L1x2_BEGIN:
+/*----------------------------------------*/   
+    andi.   T1, M,  2
+    ble   CGEMM_L1x2_END
+#if defined(TRMMKERNEL)   
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,2,1
+#else    
+    mr    BO, B   
+#endif        
+#if defined(TRMMKERNEL)   
+    REFRESH_TEMP_BK T6,K,TEMP_REG,2,1
+    mr T1, T6 
+    addi T1,T1, -2 
+    srawi.   T8, T1, 5 /**(T1-2) % 31x */
+#else   
+    mr T1, K 
+    addi T1,T1, -2
+    srawi.   T8, T1, 5 /**(K-2) % 31x */
+#endif     
+    ZERO1x2
+    ble   CGEMM_L1x2_SUB0 
+    bl CGEMM_1x2_LMAIN_SUB
+    andi.   L,  T1, 31
+    ble   CGEMM_L1x2_SAVE
+    b   CGEMM_L1x2_SUB2
+
+
+CGEMM_L1x2_SUB0:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)   
+    andi.   L,  T6, 63
+    cmpwi   T6,33
+#else   
+    andi.   L,  K,  63
+    cmpwi   K,33
+#endif       
+    li T8,1
+    bne CMP1x2_32K
+    addi BO,BO,-8
+    addi AO,AO,-16  
+    LOAD1x2O 16,8 
+    END1x2_WITHOUT_ADD   
+    LOAD1x2_2O  32, 16  
+    mtctr   T8    
+    bl CGEMM_L1x2_K32   
+    b CGEMM_L1x2_SAVE  
+    CMP1x2_32K:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)    
+    cmpwi   T6,32
+#else    
+    cmpwi   K,32
+#endif        
+    bne CGEMM_L1x2_SUB2 
+    MY_ALIGN   
+    mtctr   T8
+    addi BO,BO,-16
+    addi AO,AO,-32   
+    LOAD1x2_2O 32,16
+    bl CGEMM_L1x2_K32   
+    b CGEMM_L1x2_SAVE 
+    MY_ALIGN 
+    MY_ALIGN 
+
+
+CGEMM_L1x2_SUB2:
+/*----------------------------------------*/   
+    andi.      T1,L, 16
+    ble CGEMM_L1x2_SUB2_8
+    bl CGEMM_1x2_L16_SUB  
+    MY_ALIGN
+
+
+CGEMM_L1x2_SUB2_8:
+/*----------------------------------------*/   
+    andi.      T1,L, 8
+    ble CGEMM_L1x2_SUB2_4
+    bl CGEMM_1x2_L8_SUB
+    MY_ALIGN  
+
+
+CGEMM_L1x2_SUB2_4:
+/*----------------------------------------*/   
+    andi.      T1,L, 4
+    ble CGEMM_L1x2_SUB2_2
+    LOAD1x2_2
+    KERNEL1x2_L2  32,16, 0,0
+    KERNEL1x2_E2  32,16, 1,1
+    MY_ALIGN
+
+
+CGEMM_L1x2_SUB2_2:
+/*----------------------------------------*/   
+    andi.      T1,L, 2
+    ble CGEMM_L1x2_SUB2_1
+    LOAD1x2_2
+    KERNEL1x2_E2  32,16, 0,1
+    MY_ALIGN    
+
+
+CGEMM_L1x2_SUB2_1:
+/*----------------------------------------*/   
+    andi.      T1,L, 1
+    ble CGEMM_L1x2_SAVE 
+    KERNEL1x2
+
+    MY_ALIGN
+CGEMM_L1x2_SAVE:
+/*----------------------------------------*/   
+    SAVE1x2
+#if defined(TRMMKERNEL)    
+    REFRESH_AFTER_SAVE T6,K,TEMP_REG,BO,AO,2,1
+#endif     
+
+
+CGEMM_L1x2_END:
+/*----------------------------------------*/   
+
+
+CGEMM_L1x1_BEGIN:
+/*----------------------------------------*/   
+    andi.   T1, M,  1
+    ble   CGEMM_L1x1_END
+#if defined(TRMMKERNEL)   
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,1,1
+#else    
+    mr    BO, B   
+#endif        
+#if defined(TRMMKERNEL)   
+    REFRESH_TEMP_BK T6,K,TEMP_REG,1,1
+    mr T1, T6 
+    addi T1,T1, -2 
+    srawi.   T8, T1, 5 /**(T1-2) % 31x */
+#else   
+    mr T1, K 
+    addi T1,T1, -2
+    srawi.   T8, T1, 5 /**(K-2) % 31x */
+#endif     
+    ZERO1x1
+    ble   CGEMM_L1x1_SUB0 
+    bl CGEMM_1x1_LMAIN_SUB
+    andi.   L,  T1, 31
+    ble   CGEMM_L1x1_SAVE
+    b   CGEMM_L1x1_SUB2
+
+
+CGEMM_L1x1_SUB0:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)   
+    andi.   L,  T6, 63
+    cmpwi   T6,33
+#else   
+    andi.   L,  K,  63
+    cmpwi   K,33
+#endif       
+    li T8,1
+    bne CMP1x1_32K
+    addi BO,BO,-8
+    addi AO,AO,-8  
+    LOAD1x1O 8,8 
+    END1x1_WITHOUT_ADD   
+    LOAD1x1_2O  16, 16  
+    mtctr   T8    
+    bl CGEMM_L1x1_K32   
+    b CGEMM_L1x1_SAVE  
+    CMP1x1_32K:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)    
+    cmpwi   T6,32
+#else    
+    cmpwi   K,32
+#endif        
+    bne CGEMM_L1x1_SUB2 
+    MY_ALIGN   
+    mtctr   T8
+    addi BO,BO,-16
+    addi AO,AO,-16   
+    LOAD1x1_2O 16,16
+    bl CGEMM_L1x1_K32   
+    b CGEMM_L1x1_SAVE 
+    MY_ALIGN 
+    MY_ALIGN 
+
+
+CGEMM_L1x1_SUB2:
+/*----------------------------------------*/   
+    andi.      T1,L, 16
+    ble CGEMM_L1x1_SUB2_8
+    bl CGEMM_1x1_L16_SUB  
+    MY_ALIGN
+
+
+CGEMM_L1x1_SUB2_8:
+/*----------------------------------------*/   
+    andi.      T1,L, 8
+    ble CGEMM_L1x1_SUB2_4
+    bl CGEMM_1x1_L8_SUB
+    MY_ALIGN  
+
+
+CGEMM_L1x1_SUB2_4:
+/*----------------------------------------*/   
+    andi.      T1,L, 4
+    ble CGEMM_L1x1_SUB2_2
+    LOAD1x1_2
+    KERNEL1x1_L2  16,16, 0,0
+    KERNEL1x1_E2  16,16, 1,1
+    MY_ALIGN
+
+
+CGEMM_L1x1_SUB2_2:
+/*----------------------------------------*/   
+    andi.      T1,L, 2
+    ble CGEMM_L1x1_SUB2_1
+    LOAD1x1_2
+    KERNEL1x1_E2  16,16, 0,1
+    MY_ALIGN    
+
+
+CGEMM_L1x1_SUB2_1:
+/*----------------------------------------*/   
+    andi.      T1,L, 1
+    ble CGEMM_L1x1_SAVE 
+    KERNEL1x1
+
+    MY_ALIGN
+CGEMM_L1x1_SAVE:
+/*----------------------------------------*/  
+     
+    SAVE1x1
+#if defined(TRMMKERNEL)    
+    REFRESH_AFTER_SAVE T6,K,TEMP_REG,BO,AO,1,1
+#endif   
+
+
+CGEMM_L1x1_END:
+/*----------------------------------------*/   
+    slwi    T1, K,  3
+
+    add   B,  B,  T1
+#if defined(TRMMKERNEL) && !defined(LEFT)   
+    addi TEMP_REG, TEMP_REG, 1
+#endif   
+
+CGEMM_L1_END:
+
+
+
+
diff --git a/kernel/power/cgemm_macros_power10.S b/kernel/power/cgemm_macros_power10.S
new file mode 100644
index 000000000..b66e93405
--- /dev/null
+++ b/kernel/power/cgemm_macros_power10.S
@@ -0,0 +1,2131 @@
+/***************************************************************************
+Copyright (c) 2013-2020, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#define unit_size 8
+#define DISP32(ind, disp) (ind*unit_size*32+disp)
+#define DISP16(ind, disp) (ind*unit_size*16+disp)
+#define DISP8(ind, disp) (ind*unit_size*8+disp)
+#define DISP4(ind, disp) (ind*unit_size*4+disp)
+#define DISP2(ind, disp) (ind*unit_size*2+disp)
+#define DISP1(ind, disp) (ind*unit_size+disp)
+#define DISPX(disp)  (disp)
+
+.macro	AGGREGATE_REALS_IMAGES  VSINR_OUT1, VSINR, VSINI_OUT2, VSINI
+#if	defined(NN) || defined(NT) || defined(TN) || defined(TT)
+	xvsubsp  \VSINR_OUT1, \VSINR_OUT1, \VSINR
+	xvaddsp  \VSINI_OUT2, \VSINI_OUT2, \VSINI
+#elif  defined(CN) || defined(CT) || defined(RN) || defined(RT)
+	xvaddsp  \VSINR_OUT1, \VSINR_OUT1, \VSINR
+	xvsubsp  \VSINI_OUT2, \VSINI_OUT2, \VSINI
+#elif  defined(NC) || defined(TC) || defined(NR) || defined(TR)
+	xvaddsp  \VSINR_OUT1, \VSINR_OUT1, \VSINR
+	xvsubsp  \VSINI_OUT2, \VSINI, \VSINI_OUT2
+#else	// CC || CR || RC || RR
+    /*we will assume {-alpha_r,-alpha_i} for this case */
+    /*i1i2-r1r2 so we will negate alpha real instead to fix sign*/
+	xvsubsp  \VSINR_OUT1, \VSINR, \VSINR_OUT1
+    /*we will negate alpha image   instead to fix sign*/
+	xvaddsp  \VSINI_OUT2, \VSINI_OUT2, \VSINI
+#endif
+.endm
+
+.macro	AGGREGATE_REALS_IMAGES_A_PERMUTE  VSINR_OUT1, VSINR, VSINI_OUT2, VSINI
+#if	defined(NN) || defined(NT) || defined(TN) || defined(TT)
+	xvsubsp  \VSINR_OUT1, \VSINR_OUT1, \VSINR
+	xvaddsp  \VSINI_OUT2, \VSINI_OUT2, \VSINI
+#elif  defined(CN) || defined(CT) || defined(RN) || defined(RT)
+	xvaddsp  \VSINR_OUT1, \VSINR_OUT1, \VSINR
+	xvsubsp  \VSINI_OUT2, \VSINI, \VSINI_OUT2
+#elif  defined(NC) || defined(TC) || defined(NR) || defined(TR)
+	xvaddsp  \VSINR_OUT1, \VSINR_OUT1, \VSINR
+	xvsubsp  \VSINI_OUT2, \VSINI_OUT2, \VSINI
+#else	// CC || CR || RC || RR
+    /*we will assume {-alpha_r,-alpha_i} for this case */
+    /*i1i2-r1r2 so we will negate alpha real instead to fix sign*/
+	xvsubsp  \VSINR_OUT1, \VSINR, \VSINR_OUT1
+    /*we will negate alpha image   instead to fix sign*/
+	xvaddsp  \VSINI_OUT2, \VSINI_OUT2, \VSINI
+#endif
+.endm
+
+/* {i0,i1} * {alpha_i,alpha_i} [- VSOUT1] ;[VSOUT2 +] {r0,r1}*{alpha_i,alpha_i} */
+
+.macro MULT_APLHA_PART1  VSINRR, VSINII, VSOUT1, VSOUT2
+	xvmulsp \VSOUT1, \VSINII, alpha_i
+	xvmulsp  \VSOUT2, \VSINRR, alpha_i
+.endm
+
+/*   {r0,r1} * {alpha_r,alpha_r} -  VSOUT1 ;VSOUT2 + {i0,i1} * {alpha_r,alpha_r} */
+
+.macro MULT_APLHA_PART2  VSINRR, VSINII, VSOUT1, VSOUT2
+	xvmsubasp  \VSOUT1, \VSINRR, alpha_r
+	xvmaddasp \VSOUT2, \VSINII, alpha_r
+.endm
+
+.macro	PERMUTE1	OUT, R1, R2, R3, R4
+	xxsel	vs62, \R1, \R2, vs57
+	xxsel	\OUT, \R3, \R4, vs57
+	xxpermdi	\OUT, \OUT, vs62, 1
+.endm
+.macro	PERMUTE2	OUT, R1, R2, R3, R4
+	xxsel	vs62, \R2, \R1, vs57
+	xxsel	\OUT, \R4, \R3, vs57
+	xxpermdi	\OUT, vs62, \OUT, 1
+	xxperm	\OUT, \OUT, permute_mask
+.endm
+.macro PERMUTE3	OUT, R1, R2, R3, R4
+	xxsel	vs62, \R1, \R2, vs57
+	xxsel	\OUT, \R3, \R4, vs57
+	xxpermdi \OUT, vs62, \OUT, 2
+.endm
+.macro PERMUTE4	OUT, R1, R2, R3, R4
+	xxsel	vs62, \R2, \R1, vs57
+	xxsel	\OUT, \R4, \R3, vs57
+	xxpermdi	\OUT, \OUT, vs62, 2
+	xxperm	\OUT, \OUT, permute_mask
+.endm
+.macro	GROUP1
+	xxperm	vs0, vs32, permute_mask
+	xxperm	vs4, vs40, permute_mask
+	xxperm	vs1, vs33, permute_mask
+	xxperm	vs5, vs41, permute_mask
+	xxperm	vs8, vs36, permute_mask
+	xxperm	vs12, vs44, permute_mask
+	xxperm	vs9, vs37, permute_mask
+	xxperm	vs13, vs45, permute_mask
+.endm
+.macro	AGG_GROUP1
+	AGGREGATE_REALS_IMAGES	vs32, vs0, vs40, vs4
+	AGGREGATE_REALS_IMAGES	vs33, vs1, vs41, vs5
+	AGGREGATE_REALS_IMAGES	vs36, vs8, vs44, vs12
+	AGGREGATE_REALS_IMAGES	vs37, vs9, vs45, vs13
+.endm
+.macro	GROUP2
+	xxperm	vs0, vs34, permute_mask
+	xxperm	vs4, vs42, permute_mask
+	xxperm	vs1, vs35, permute_mask
+	xxperm	vs5, vs43, permute_mask
+	xxperm	vs8, vs38, permute_mask
+	xxperm	vs12, vs46, permute_mask
+	xxperm	vs9, vs39, permute_mask
+	xxperm	vs13, vs47, permute_mask
+.endm
+.macro	AGG_GROUP2
+	AGGREGATE_REALS_IMAGES	vs34, vs0, vs42, vs4
+	AGGREGATE_REALS_IMAGES	vs35, vs1, vs43, vs5
+	AGGREGATE_REALS_IMAGES	vs38, vs8, vs46, vs12
+	AGGREGATE_REALS_IMAGES	vs39, vs9, vs47, vs13
+.endm
+.macro	MULTIPLY_GROUP1
+	MULT_APLHA_PART1	vs32, vs40, vs0, vs1
+	MULT_APLHA_PART1	vs33, vs41, vs2, vs3
+	MULT_APLHA_PART1	vs36, vs44, vs8, vs9
+	MULT_APLHA_PART1	vs37, vs45, vs10, vs11
+	MULT_APLHA_PART2	vs32, vs40, vs0, vs1
+	MULT_APLHA_PART2	vs33, vs41, vs2, vs3
+	MULT_APLHA_PART2	vs36, vs44, vs8, vs9
+	MULT_APLHA_PART2	vs37, vs45, vs10, vs11
+.endm
+.macro	MULTIPLY_GROUP2
+	MULT_APLHA_PART1	vs34, vs42, vs4, vs5
+	MULT_APLHA_PART1	vs35, vs43, vs6, vs7
+	MULT_APLHA_PART1	vs38, vs46, vs12, vs13
+	MULT_APLHA_PART1	vs39, vs47, vs14, vs15
+	MULT_APLHA_PART2	vs34, vs42, vs4, vs5
+	MULT_APLHA_PART2	vs35, vs43, vs6, vs7
+	MULT_APLHA_PART2	vs38, vs46, vs12, vs13
+	MULT_APLHA_PART2	vs39, vs47, vs14, vs15
+.endm
+/* reconstruct r, i pairs*/
+.macro	RECONSTRUCT_PAIR1
+	xxperm	vs0, vs1, save_permute_1
+	xxperm	vs2, vs3, save_permute_1
+	xxperm	vs8, vs9, save_permute_1
+	xxperm	vs10, vs11, save_permute_1
+.endm
+.macro	RECONSTRUCT_PAIR2
+	xxperm	vs4, vs5, save_permute_1
+	xxperm	vs6, vs7, save_permute_1
+	xxperm	vs12, vs13, save_permute_1
+	xxperm	vs14, vs15, save_permute_1
+.endm
+.macro	SHUFFLE_ACC	ACC, R0, R1, R2, R3, O1, O2, O3, O4
+	xxmfacc	\ACC
+	PERMUTE1	\O1, \R3, \R2, \R1, \R0
+	PERMUTE2	\O2, \R1, \R0, \R3, \R2
+	PERMUTE3	\O3, \R1, \R0, \R3, \R2
+	PERMUTE4	\O4, \R3, \R2, \R1, \R0
+.endm
+/*                                             macros for N=4 and M=8
+**********************************************************************************************/
+.macro	ZERO4x8
+	xxsetaccz	0
+	xxsetaccz	1
+	xxsetaccz	2
+	xxsetaccz	3
+	xxsetaccz	4
+	xxsetaccz	5
+	xxsetaccz	6
+	xxsetaccz	7
+.endm
+
+.macro	LOAD4x8
+	LOAD4x8O	0, 0
+.endm
+
+.macro	LOAD4x8O  OffsetA, OffsetB
+	lxvp	vs34, (\OffsetB+0)(BO)
+	lxvp	vs32, (\OffsetA+0)(AO)
+	lxvp	vs36, (\OffsetA+32)(AO)
+.endm
+
+.macro	END4x8_NORMAL
+	END4x8	AO, BO, 64, 32
+.endm
+
+.macro	END4x8_WITHOUT_ADD
+	END4x8	AO, BO, 0, 0
+.endm
+
+.macro	END4x8	AREG, BREG, OffsetA, OffsetB
+.if \OffsetB != 0
+	addi	\BREG, \BREG, \OffsetB
+.endif
+.if \OffsetA != 0
+	addi	\AREG, \AREG, \OffsetA
+.endif
+	xvf32gerpp	3, 36, 35
+	xvf32gerpp	2, 37, 35
+	xvf32gerpp	1, 32, 35
+	xvf32gerpp	0, 33, 35
+	xvf32gerpp	7, 36, 34
+	xvf32gerpp	6, 37, 34
+	xvf32gerpp	5, 32, 34
+	xvf32gerpp	4, 33, 34
+.endm
+
+.macro	LOAD4x8_2
+	LOAD4x8_2O	0, 0
+.endm
+
+.macro	LOAD4x8_2O  OffsetA, OffsetB
+	lxvp	vs34, (\OffsetB)(BO)
+	lxvp	vs38, (32+\OffsetB)(BO)
+	lxvp	vs32, (0+\OffsetA)(AO)
+	lxvp	vs36, (32+\OffsetA)(AO)
+	lxvp	vs40, (64+\OffsetA)(AO)
+	lxvp	vs42, (64+32+\OffsetA)(AO)
+.endm
+
+.macro	END4x8_2
+	/*for load2 offset will be 128 and 64*/
+	KERNEL4x8_2	AO, BO, 128, 64, 0, 1, 1
+.endm
+
+.macro	KERNEL4x8_E2	OffsetA, OffsetB, Index, IsLast
+	KERNEL4x8_2	AO, BO, \OffsetA, \OffsetB, \Index, \IsLast, 1
+.endm
+
+.macro	KERNEL4x8_L2	OffsetA, OffsetB, Index, IsLast
+	KERNEL4x8_2	AO, BO, \OffsetA, \OffsetB, \Index, \IsLast, 0
+.endm
+
+.macro	KERNEL4x8_2	AREG, BREG, OffsetA, OffsetB, Index, IsLast, Complete
+	xvf32gerpp	3, 36, 35
+	xvf32gerpp	2, 37, 35
+	xvf32gerpp	1, 32, 35
+	xvf32gerpp	0, 33, 35
+	xvf32gerpp	7, 36, 34
+	xvf32gerpp	6, 37, 34
+	xvf32gerpp	5, 32, 34
+	xvf32gerpp	4, 33, 34
+.if \Complete==0
+	lxvp	vs34, DISP8(\Index, \OffsetB)(\BREG)
+	lxvp	vs32, DISP16(\Index, 0+\OffsetA)(\AREG)
+	lxvp	vs36, DISP16(\Index, 32+\OffsetA)(\AREG)
+.endif
+	xvf32gerpp	3, 42, 39
+	xvf32gerpp	2, 43, 39
+	xvf32gerpp	1, 40, 39
+	xvf32gerpp	0, 41, 39
+	xvf32gerpp	7, 42, 38
+	xvf32gerpp	6, 43, 38
+	xvf32gerpp	5, 40, 38
+	xvf32gerpp	4, 41, 38
+.if \Complete==0
+	lxvp	vs40, DISP16(\Index, 64+\OffsetA)(\AREG)
+	lxvp	vs38, DISP8(\Index, 32+\OffsetB)(\BREG)
+	lxvp	vs42, DISP16(\Index, 64+32+\OffsetA)(\AREG)
+.endif
+.if \IsLast==1
+.if \Complete==1
+	addi	\BREG, \BREG, DISP8(\Index, \OffsetB)
+	addi    \AREG, \AREG, DISP16(\Index, \OffsetA)
+.else
+	addi	\BREG, \BREG, DISP8(\Index, 64)
+	addi    \AREG, \AREG, DISP16(\Index, 128)
+.endif
+.endif
+.endm
+
+.macro	KERNEL4x8
+	LOAD4x8
+	END4x8	AO, BO, 64, 32
+.endm
+
+.macro SAVE4x8
+	SHUFFLE_ACC	0, vs0, vs1, vs2, vs3, vs32, vs40, vs36, vs44
+	SHUFFLE_ACC	1, vs4, vs5, vs6, vs7, vs33, vs41, vs37, vs45
+	SHUFFLE_ACC	2, vs8, vs9, vs10, vs11, vs34, vs42, vs38, vs46
+	SHUFFLE_ACC	3, vs12, vs13, vs14, vs15, vs35, vs43, vs39, vs47
+	SHUFFLE_ACC	4, vs16, vs17, vs18, vs19, vs48, vs56, vs52, vs60
+	SHUFFLE_ACC	5, vs20, vs21, vs22, vs23, vs49, vs16, vs53, vs61
+	SHUFFLE_ACC	7, vs28, vs29, vs30, vs31, vs17, vs19, vs18, vs20
+	SHUFFLE_ACC	6, vs24, vs25, vs26, vs27, vs50, vs58, vs54, vs21
+	add	T4, LDC, LDC
+	add	T1, CO, LDC
+#ifndef TRMMKERNEL
+	lxvp	vs24, 0(CO)
+#endif
+#ifndef TRMMKERNEL
+	lxvp	vs26, 32(CO)
+#endif
+#ifndef TRMMKERNEL
+	lxvp	vs28, 0(T1)
+#endif
+	xxperm	vs2, vs34, permute_mask
+	xxperm	vs6, vs42, permute_mask
+#ifndef TRMMKERNEL
+	lxvp	vs30, 32(T1)
+#endif
+	xxperm	vs3, vs35, permute_mask
+	xxperm	vs7, vs43, permute_mask
+	add	T2, CO, T4
+	add	T3, T1, T4
+	GROUP1
+	AGG_GROUP1
+	AGGREGATE_REALS_IMAGES	vs34, vs2, vs42, vs6
+	xxperm	vs10, vs38, permute_mask
+	xxperm	vs14, vs46, permute_mask
+	AGGREGATE_REALS_IMAGES	vs35, vs3, vs43, vs7
+	xxperm	vs11, vs39, permute_mask
+	xxperm	vs15, vs47, permute_mask
+	xxperm	vs0, vs48, permute_mask
+	xxperm	vs4, vs56, permute_mask
+	xxperm	vs1, vs49, permute_mask
+	xxperm	vs5, vs16, permute_mask
+	AGGREGATE_REALS_IMAGES	vs38, vs10, vs46, vs14
+	xxperm	vs2, vs50, permute_mask
+	xxperm	vs6, vs58, permute_mask
+	AGGREGATE_REALS_IMAGES	vs39, vs11, vs47, vs15
+	xxperm	vs3, vs17, permute_mask
+	xxperm	vs7, vs19, permute_mask
+	AGGREGATE_REALS_IMAGES	vs48, vs0, vs56, vs4
+	xxperm	vs8, vs52, permute_mask
+	xxperm	vs12, vs60, permute_mask
+	AGGREGATE_REALS_IMAGES	vs49, vs1, vs16, vs5
+	xxperm	vs9, vs53, permute_mask
+	xxperm	vs13, vs61, permute_mask
+	AGGREGATE_REALS_IMAGES	vs50, vs2, vs58, vs6
+	xxperm	vs10, vs54, permute_mask
+	xxperm	vs14, vs21, permute_mask
+	AGGREGATE_REALS_IMAGES	vs17, vs3, vs19, vs7
+	xxperm	vs11, vs18, permute_mask
+	xxperm	vs15, vs20, permute_mask
+	AGGREGATE_REALS_IMAGES	vs52, vs8, vs60, vs12
+	AGGREGATE_REALS_IMAGES	vs53, vs9, vs61, vs13
+/*VSINRR, VSINII, VSOUT1, VSOUT2*/
+	MULT_APLHA_PART1    vs32, vs40, vs0, vs1
+	AGGREGATE_REALS_IMAGES	vs54, vs10, vs21, vs14
+	MULT_APLHA_PART1    vs33, vs41, vs2, vs3
+	AGGREGATE_REALS_IMAGES	vs18, vs11, vs20, vs15
+	MULT_APLHA_PART1    vs34, vs42, vs4, vs5
+	MULT_APLHA_PART1    vs35, vs43, vs6, vs7
+	MULT_APLHA_PART2    vs32, vs40, vs0, vs1
+	MULT_APLHA_PART2    vs33, vs41, vs2, vs3
+	MULT_APLHA_PART2    vs34, vs42, vs4, vs5
+	MULT_APLHA_PART2    vs35, vs43, vs6, vs7
+#ifndef TRMMKERNEL
+	lxvp	vs32, 0(T2)
+#endif
+	MULT_APLHA_PART1    vs36, vs44, vs8, vs9
+	MULT_APLHA_PART1    vs37, vs45, vs10, vs11
+#ifndef TRMMKERNEL
+	lxvp	vs40, 32(T2)
+#endif
+	MULT_APLHA_PART1    vs38, vs46, vs12, vs13
+	MULT_APLHA_PART1    vs39, vs47, vs14, vs15
+#ifndef TRMMKERNEL
+	lxvp	vs34, 0(T3)
+#endif
+	MULT_APLHA_PART2    vs36, vs44, vs8, vs9
+	MULT_APLHA_PART2    vs37, vs45, vs10, vs11
+#ifndef TRMMKERNEL
+	lxvp	vs42, 32(T3)
+#endif
+	MULT_APLHA_PART2    vs38, vs46, vs12, vs13
+	MULT_APLHA_PART2    vs39, vs47, vs14, vs15
+	RECONSTRUCT_PAIR1
+	RECONSTRUCT_PAIR2
+#ifndef TRMMKERNEL
+	/* add */
+	xxpermdi	vs1, vs8, vs0, 2
+	xxpermdi	vs3, vs10, vs2, 2
+	xxpermdi	vs5, vs12, vs4, 2
+	xxpermdi	vs7, vs14, vs6, 2
+	xxpermdi	vs9, vs0, vs8, 2
+	xxpermdi	vs11, vs2, vs10, 2
+	xvaddsp	vs24, vs24, vs3
+	xvaddsp	vs25, vs25, vs1
+	xxpermdi	vs13, vs4, vs12, 2
+	xxpermdi	vs15, vs6, vs14, 2
+	xvaddsp	vs26, vs26, vs7
+	xvaddsp	vs27, vs27, vs5
+	xvaddsp	vs28, vs28, vs11
+	xvaddsp	vs29, vs29, vs9
+	xvaddsp	vs30, vs30, vs15
+	xvaddsp	vs31, vs31, vs13
+#else
+	xxpermdi	vs25, vs8, vs0, 2
+	xxpermdi	vs24, vs10, vs2, 2
+	xxpermdi	vs27, vs12, vs4, 2
+	xxpermdi	vs26, vs14, vs6, 2
+	xxpermdi	vs29, vs0, vs8, 2
+	xxpermdi	vs28, vs2, vs10, 2
+	xxpermdi	vs31, vs4, vs12, 2
+	xxpermdi	vs30, vs6, vs14, 2
+#endif
+	stxvp	vs24, 0(CO)
+	MULT_APLHA_PART1    vs48, vs56, vs0, vs1
+	MULT_APLHA_PART1    vs49, vs16, vs2, vs3
+	stxvp	vs26, 32(CO)
+	MULT_APLHA_PART1    vs50, vs58, vs4, vs5
+	MULT_APLHA_PART1    vs17, vs19, vs6, vs7
+	stxvp	vs28, 0(T1)
+	MULT_APLHA_PART2    vs48, vs56, vs0, vs1
+	MULT_APLHA_PART2    vs49, vs16, vs2, vs3
+	stxvp	vs30, 32(T1)
+	MULT_APLHA_PART2    vs50, vs58, vs4, vs5
+	MULT_APLHA_PART2    vs17, vs19, vs6, vs7
+	MULT_APLHA_PART1    vs52, vs60, vs8, vs9
+	MULT_APLHA_PART1    vs53, vs61, vs10, vs11
+	MULT_APLHA_PART1    vs54, vs21, vs12, vs13
+	MULT_APLHA_PART1    vs18, vs20, vs14, vs15
+	MULT_APLHA_PART2    vs52, vs60, vs8, vs9
+	MULT_APLHA_PART2    vs53, vs61, vs10, vs11
+	MULT_APLHA_PART2    vs54, vs21, vs12, vs13
+	MULT_APLHA_PART2    vs18, vs20, vs14, vs15
+	RECONSTRUCT_PAIR1
+	RECONSTRUCT_PAIR2
+#ifndef TRMMKERNEL
+  /* add */
+	xxpermdi	vs1, vs8, vs0, 2
+	xxpermdi	vs3, vs10, vs2, 2
+	xxpermdi	vs5, vs12, vs4, 2
+	xxpermdi	vs7, vs14, vs6, 2
+	xxpermdi	vs9, vs0, vs8, 2
+	xxpermdi	vs11, vs2, vs10, 2
+	xvaddsp	vs32, vs32, vs3
+	xvaddsp	vs33, vs33, vs1
+	xxpermdi	vs13, vs4, vs12, 2
+	xxpermdi	vs15, vs6, vs14, 2
+	xvaddsp	vs40, vs40, vs7
+	xvaddsp vs41, vs41, vs5
+	xvaddsp	vs34, vs34, vs11
+	xvaddsp	vs35, vs35, vs9
+	xvaddsp	vs42, vs42, vs15
+	xvaddsp	vs43, vs43, vs13
+#else
+	xxpermdi	vs33, vs8, vs0, 2
+	xxpermdi	vs32, vs10, vs2, 2
+	xxpermdi	vs41, vs12, vs4, 2
+	xxpermdi	vs40, vs14, vs6, 2
+	xxpermdi	vs35, vs0, vs8, 2
+	xxpermdi	vs34, vs2, vs10, 2
+	xxpermdi	vs43, vs4, vs12, 2
+	xxpermdi	vs42, vs6, vs14, 2
+#endif
+	stxvp	vs32, 0(T2)
+	stxvp	vs40, 32(T2)
+	stxvp	vs34, 0(T3)
+	stxvp	vs42, 32(T3)
+	addi	CO, CO, 64
+.endm
+
+/*                                             macros for N=4 and M=4
+**********************************************************************************************/
+
+.macro	ZERO4x4
+	xxsetaccz	0
+	xxsetaccz	1
+	xxsetaccz	2
+	xxsetaccz	3
+.endm
+
+.macro	LOAD4x4
+	LOAD4x4O 0, 0
+.endm
+
+.macro	LOAD4x4O  OffsetA, OffsetB
+	lxvp	vs34, (\OffsetB+0)(BO)
+	lxvp	vs32, (\OffsetA+0)(AO)
+.endm
+
+.macro	END4x4_NORMAL
+	END4x4 AO, BO, 32, 32
+.endm
+
+.macro	END4x4_WITHOUT_ADD
+	END4x4 AO, BO, 0, 0
+.endm
+
+.macro	END4x4	AREG, BREG, OffsetA, OffsetB
+.if \OffsetB != 0
+	addi	\BREG, \BREG, \OffsetB
+.endif
+.if \OffsetA != 0
+	addi	\AREG, \AREG, \OffsetA
+.endif
+	xvf32gerpp	3, 32, 34
+	xvf32gerpp	2, 33, 34
+	xvf32gerpp	1, 32, 35
+	xvf32gerpp	0, 33, 35
+.endm
+
+.macro	LOAD4x4_2
+	LOAD4x4_2O 0, 0
+.endm
+
+.macro	LOAD4x4_2O  OffsetA, OffsetB
+	lxvp	vs34, (\OffsetB)(BO)
+	lxvp	vs38, (32+\OffsetB)(BO)
+	lxvp	vs32, (0+\OffsetA)(AO)
+	lxvp	vs36, (32+\OffsetA)(AO)
+.endm
+
+.macro	END4x4_2
+  /*for load2 offset will be 64 and 64*/
+	KERNEL4x4_2	AO, BO, 64, 64, 0, 1, 1
+.endm
+
+.macro	KERNEL4x4_E2	OffsetA, OffsetB, Index, IsLast
+	KERNEL4x4_2	AO, BO, \OffsetA, \OffsetB, \Index, \IsLast, 1
+.endm
+
+.macro	KERNEL4x4_L2	OffsetA, OffsetB, Index, IsLast
+	KERNEL4x4_2	AO, BO, \OffsetA, \OffsetB, \Index, \IsLast, 0
+.endm
+
+.macro	KERNEL4x4_2	AREG, BREG, OffsetA, OffsetB, Index, IsLast, Complete
+	xvf32gerpp	3, 32, 34
+	xvf32gerpp	2, 33, 34
+	xvf32gerpp	1, 32, 35
+	xvf32gerpp	0, 33, 35
+.if \Complete==0
+	lxvp	vs34, DISP8(\Index, \OffsetB)(\BREG)
+	lxvp	vs32, DISP8(\Index, 0+\OffsetA)(\AREG)
+.endif
+	xvf32gerpp	3, 36, 38
+	xvf32gerpp	2, 37, 38
+	xvf32gerpp	1, 36, 39
+	xvf32gerpp	0, 37, 39
+.if \Complete==0
+	lxvp	vs38, DISP8(\Index, 32+\OffsetB)(\BREG)
+	lxvp	vs36, DISP8(\Index, 32+\OffsetA)(\AREG)
+.endif
+.if \IsLast==1
+.if \Complete==1
+	addi		\BREG, \BREG, DISP8(\Index, \OffsetB)
+	addi    \AREG, \AREG, DISP8(\Index, \OffsetA)
+.else
+	addi		\BREG, \BREG, DISP8(\Index, 64)
+	addi    \AREG, \AREG, DISP8(\Index, 64)
+.endif
+.endif
+.endm
+
+.macro	KERNEL4x4
+	LOAD4x4
+	END4x4  AO, BO, 32, 32
+.endm
+
+.macro SAVE4x4
+	SHUFFLE_ACC	0, vs0, vs1, vs2, vs3, vs32, vs40, vs36, vs44
+	SHUFFLE_ACC	1, vs4, vs5, vs6, vs7, vs33, vs41, vs37, vs45
+	SHUFFLE_ACC	2, vs8, vs9, vs10, vs11, vs34, vs42, vs38, vs46
+	SHUFFLE_ACC	3, vs12, vs13, vs14, vs15, vs35, vs43, vs39, vs47
+	add	T4, LDC, LDC
+	add	T1, CO, LDC
+#ifndef TRMMKERNEL
+	lxvp	vs24, 0(CO)
+#endif
+	add	T2, CO, T4
+	add	T3, T1, T4
+#ifndef TRMMKERNEL
+	lxvp	vs26, 0(T1)
+#endif
+ #ifndef TRMMKERNEL
+	lxvp	vs28, 0(T2)
+#endif
+#ifndef TRMMKERNEL
+	lxvp	vs30, 0(T3)
+#endif
+	GROUP1
+	AGG_GROUP1
+	GROUP2
+	AGG_GROUP2
+  /*VSINRR, VSINII, VSOUT1, VSOUT2*/
+	MULTIPLY_GROUP1
+	MULTIPLY_GROUP2
+/* reconstruct r, i pairs*/
+	RECONSTRUCT_PAIR1
+	RECONSTRUCT_PAIR2
+#ifndef TRMMKERNEL
+  /* add */
+	xxpermdi	vs1, vs8, vs0, 2
+	xxpermdi	vs3, vs10, vs2, 2
+	xxpermdi	vs9, vs0, vs8, 2
+	xxpermdi	vs11, vs2, vs10, 2
+	xxpermdi	vs5, vs12, vs4, 2
+	xxpermdi	vs7, vs14, vs6, 2
+	xxpermdi	vs13, vs4, vs12, 2
+	xxpermdi	vs15, vs6, vs14, 2
+	xvaddsp	vs24, vs24, vs3
+	xvaddsp	vs25, vs25, vs1
+	xvaddsp	vs26, vs26, vs11
+	xvaddsp	vs27, vs27, vs9
+	xvaddsp	vs28, vs28, vs7
+	xvaddsp	vs29, vs29, vs5
+	xvaddsp	vs30, vs30, vs15
+	xvaddsp	vs31, vs31, vs13
+#else
+	xxpermdi	vs25, vs8, vs0, 2
+	xxpermdi	vs24, vs10, vs2, 2
+	xxpermdi	vs27, vs0, vs8, 2
+	xxpermdi	vs26, vs2, vs10, 2
+	xxpermdi	vs29, vs12, vs4, 2
+	xxpermdi	vs28, vs14, vs6, 2
+	xxpermdi	vs31, vs4, vs12, 2
+	xxpermdi	vs30, vs6, vs14, 2
+#endif
+	stxvp	vs24, 0(CO)
+	stxvp	vs26, 0(T1)
+	stxvp	vs28, 0(T2)
+	stxvp	vs30, 0(T3)
+	addi  CO, CO, 32
+.endm
+
+/*                                             macros for N=4 and M=2
+**********************************************************************************************/
+
+.macro	ZERO4x2
+	xxsetaccz	0
+	xxsetaccz	1
+.endm
+
+.macro	LOAD4x2
+	LOAD4x2O 0, 0
+.endm
+
+.macro	LOAD4x2O  OffsetA, OffsetB
+	lxv	vs32, (\OffsetA+0)(AO)
+	lxvp	vs34, (\OffsetB+0)(BO)
+.endm
+
+.macro	END4x2_NORMAL
+	END4x2 AO, BO, 16, 32
+.endm
+
+.macro	END4x2_WITHOUT_ADD
+	END4x2 AO, BO, 0, 0
+.endm
+
+.macro	END4x2	AREG, BREG, OffsetA, OffsetB
+.if \OffsetB != 0
+	addi	\BREG, \BREG, \OffsetB
+.endif
+
+.if \OffsetA != 0
+	addi	\AREG, \AREG, \OffsetA
+.endif
+	xvf32gerpp	1, 34, 32
+	xvf32gerpp	0, 35, 32
+.endm
+
+.macro	LOAD4x2_2
+	LOAD4x2_2O 0, 0
+.endm
+
+.macro	LOAD4x2_2O  OffsetA, OffsetB
+	lxvp	vs32, (\OffsetA)(AO)
+	lxvp	vs34, (0+\OffsetB)(BO)
+	lxvp	vs36, (32+\OffsetB)(BO)
+.endm
+
+.macro	END4x2_2
+  /*for load2 offset will be 32 and 64*/
+	KERNEL4x2_2	AO, BO, 32, 64, 0, 1, 1
+.endm
+
+.macro	KERNEL4x2_E2	OffsetA, OffsetB, Index, IsLast
+	KERNEL4x2_2	AO, BO, \OffsetA, \OffsetB, \Index, \IsLast, 1
+.endm
+
+.macro	KERNEL4x2_L2	OffsetA, OffsetB, Index, IsLast
+	KERNEL4x2_2	AO, BO, \OffsetA, \OffsetB, \Index, \IsLast, 0
+.endm
+
+.macro	KERNEL4x2_2	AREG, BREG, OffsetA, OffsetB, Index, IsLast, Complete
+	xvf32gerpp	1, 34, 33
+	xvf32gerpp	0, 35, 33
+.if \Complete==0
+	lxvp	vs34, DISP8(\Index, 0+\OffsetB)(\BREG)
+.endif
+	xvf32gerpp	1, 36, 32
+	xvf32gerpp	0, 37, 32
+.if \Complete==0
+	lxvp	vs32, DISP4(\Index, \OffsetA)(\AREG)
+	lxvp	vs36, DISP8(\Index, 32+\OffsetB)(\BREG)
+.endif
+.if \IsLast==1
+.if \Complete==1
+	addi    \AREG, \AREG, DISP4(\Index, \OffsetA)
+	addi		\BREG, \BREG, DISP8(\Index, \OffsetB)
+.else
+	addi    \AREG, \AREG, DISP4(\Index, 32)
+	addi		\BREG, \BREG, DISP8(\Index, 64)
+.endif
+.endif
+.endm
+
+.macro	KERNEL4x2
+	LOAD4x2
+	END4x2  AO, BO, 16, 32
+.endm
+
+.macro SAVE4x2
+	SHUFFLE_ACC	0, vs0, vs1, vs2, vs3, vs32, vs40, vs36, vs44
+	SHUFFLE_ACC	1, vs4, vs5, vs6, vs7, vs33, vs41, vs37, vs45
+	add	T4, LDC, LDC
+	add	T1, CO, LDC
+	add	T2, CO, T4
+	add	T3, T1, T4
+#ifndef TRMMKERNEL
+	lxv	vs24, 0(CO)
+#endif
+#ifndef TRMMKERNEL
+	lxv	vs25, 0(T1)
+#endif
+#ifndef TRMMKERNEL
+	lxv	vs26, 0(T2)
+#endif
+#ifndef TRMMKERNEL
+	lxv	vs27, 0(T3)
+#endif
+	GROUP1
+	AGGREGATE_REALS_IMAGES_A_PERMUTE vs32, vs0, vs40, vs4
+	AGGREGATE_REALS_IMAGES_A_PERMUTE vs33, vs1, vs41, vs5
+	AGGREGATE_REALS_IMAGES_A_PERMUTE vs36, vs8, vs44, vs12
+	AGGREGATE_REALS_IMAGES_A_PERMUTE vs37, vs9, vs45, vs13
+  /*VSINRR, VSINII, VSOUT1, VSOUT2*/
+	MULTIPLY_GROUP1
+/* reconstruct r, i pairs*/
+	RECONSTRUCT_PAIR1
+#ifndef TRMMKERNEL
+  /* add */
+	xxpermdi	vs1, vs8, vs0, 0
+	xxpermdi	vs9, vs10, vs2, 0
+	xxpermdi	vs3, vs0, vs8, 3
+	xxpermdi	vs11, vs2, vs10, 3
+	xvaddsp	vs24, vs24, vs1
+	xvaddsp	vs26, vs26, vs9
+	xvaddsp	vs25, vs25, vs3
+	xvaddsp	vs27, vs27, vs11
+#else
+	xxpermdi	vs24, vs8, vs0, 0
+	xxpermdi	vs26, vs10, vs2, 0
+	xxpermdi	vs25, vs0, vs8, 3
+	xxpermdi	vs27, vs2, vs10, 3
+#endif
+	stxv	vs24, 0(CO)
+	stxv	vs25, 0(T1)
+	stxv	vs26, 0(T2)
+	stxv	vs27, 0(T3)
+	addi  CO, CO, 16
+.endm
+
+/*                                             macros for N=4 and M=2
+**********************************************************************************************/
+
+.macro	ZERO4x1
+	xxsetaccz	0
+	xxsetaccz	1
+.endm
+
+.macro	LOAD4x1
+	LOAD4x1O 0, 0
+.endm
+
+.macro	LOAD4x1O  OffsetA, OffsetB
+	lxsd	v0, (\OffsetA+0)(AO)
+	lxvp	vs34, (\OffsetB+0)(BO)
+.endm
+
+.macro	END4x1_NORMAL
+	END4x1 AO, BO,8, 32
+.endm
+
+.macro	END4x1_WITHOUT_ADD
+	END4x1 AO, BO, 0, 0
+.endm
+
+.macro	END4x1 AREG, BREG, OffsetA, OffsetB
+.if \OffsetB != 0
+	addi  \BREG, \BREG, \OffsetB
+.endif
+
+.if \OffsetA != 0
+	addi  \AREG, \AREG, \OffsetA
+.endif
+	xvf32gerpp	    0, 35, 32
+	xvf32gerpp	    1, 34, 32
+.endm
+
+.macro	LOAD4x1_2
+	LOAD4x1_2O 0, 0
+.endm
+
+.macro	LOAD4x1_2O  OffsetA, OffsetB
+	lxv	vs32, (\OffsetA)(AO)
+	vspltisb        v6, 0
+	xxpermdi        vs33, vs32, vs38, 0
+	xxpermdi        vs32, vs32, vs38, 2
+	lxvp	vs34, (0+\OffsetB)(BO)
+	lxvp	vs36, (32+\OffsetB)(BO)
+.endm
+
+.macro	END4x1_2
+  /*for load2 offset will be 16 and 64*/
+	KERNEL4x1_2  AO, BO, 16, 64, 0, 1, 1
+.endm
+
+.macro	KERNEL4x1_E2 OffsetA, OffsetB, Index, IsLast
+	KERNEL4x1_2 AO, BO, \OffsetA, \OffsetB, \Index, \IsLast, 1
+.endm
+
+.macro	KERNEL4x1_L2 OffsetA, OffsetB, Index, IsLast
+	KERNEL4x1_2 AO, BO, \OffsetA, \OffsetB, \Index, \IsLast, 0
+.endm
+
+.macro	KERNEL4x1_2  AREG, BREG, OffsetA, OffsetB, Index, IsLast, Complete
+	xvf32gerpp	    0, 35, 32
+	xvf32gerpp	    1, 34, 32
+.if \Complete==0
+	lxvp	vs34, DISP8(\Index, 0+\OffsetB)(\BREG)
+.endif
+	xvf32gerpp	    0, 37, 33
+	xvf32gerpp	    1, 36, 33
+.if \Complete==0
+	lxv	vs32, DISP2(\Index, \OffsetA)(\AREG)
+	lxvp	vs36, DISP8(\Index, 32+\OffsetB)(\BREG)
+	xxpermdi        vs33, vs32, vs38, 0
+	xxpermdi        vs32, vs32, vs38, 2
+.endif
+.if \IsLast==1
+.if \Complete==1
+	addi    \AREG, \AREG, DISP2(\Index, \OffsetA)
+	addi    \BREG, \BREG, DISP8(\Index, \OffsetB)
+.else
+	addi    \AREG, \AREG, DISP2(\Index, 16)
+	addi    \BREG, \BREG, DISP8(\Index, 64)
+.endif
+.endif
+.endm
+
+.macro	KERNEL4x1
+	LOAD4x1
+	END4x1  AO, BO, 8, 32
+.endm
+
+.macro SAVE4x1
+	SHUFFLE_ACC	0, vs0, vs1, vs2, vs3, vs32, vs40, vs36, vs44
+	SHUFFLE_ACC	1, vs4, vs5, vs6, vs7, vs33, vs41, vs37, vs45
+	xxpermdi	vs32, vs32, vs36, 1
+	xxpermdi	vs40, vs40, vs44, 1
+	xxpermdi	vs33, vs33, vs37, 1
+	xxpermdi	vs41, vs41, vs45, 1
+	add	T4, LDC, LDC
+	add	T1, CO, LDC
+	add	T2, CO, T4
+	add	T3, T1, T4
+#ifndef TRMMKERNEL
+	lxsd	v4, 0(CO)
+#endif
+#ifndef TRMMKERNEL
+	lxsd	v5, 0(T1)
+#endif
+#ifndef TRMMKERNEL
+	lxsd	v6, 0(T2)
+#endif
+#ifndef TRMMKERNEL
+	lxsd	v7, 0(T3)
+#endif
+	xxperm	vs0, vs32, permute_mask
+	xxperm	vs4, vs40, permute_mask
+	xxperm	vs1, vs33, permute_mask
+	xxperm	vs5, vs41, permute_mask
+	AGGREGATE_REALS_IMAGES_A_PERMUTE vs32, vs0, vs40, vs4
+	AGGREGATE_REALS_IMAGES_A_PERMUTE vs33, vs1, vs41, vs5
+  /*VSINRR, VSINII, VSOUT1, VSOUT2*/
+	MULT_APLHA_PART1    vs32, vs40, vs0, vs1
+	MULT_APLHA_PART1    vs33, vs41, vs2, vs3
+	MULT_APLHA_PART2    vs32, vs40, vs0, vs1
+	MULT_APLHA_PART2    vs33, vs41, vs2, vs3
+/* reconstruct r, i pairs*/
+	xxperm	vs0, vs1, save_permute_1
+	xxperm	vs2, vs3, save_permute_1
+#ifndef TRMMKERNEL
+  /* add */
+	xxspltd vs1, vs0, 0
+	xxspltd vs3, vs0, 1
+	xxspltd vs9, vs2, 0
+	xxspltd vs11, vs2, 1
+ /*--v4==vs36 v5==vs37 v6==vs38 v7==vs39---*/
+	xvaddsp	vs36, vs36, vs1
+	xvaddsp	vs37, vs37, vs3
+	xvaddsp	vs38, vs38, vs9
+	xvaddsp	vs39, vs39, vs11
+#else
+ /*--v4==vs36 v5==vs37 v6==vs38 v7==vs39---*/
+	xxspltd vs36, vs0, 0
+	xxspltd vs37, vs0, 1
+	xxspltd vs38, vs2, 0
+	xxspltd vs39, vs2, 1
+#endif
+	stxsd	v4, 0(CO)
+	stxsd	v5, 0(T1)
+	stxsd	v6, 0(T2)
+	stxsd	v7, 0(T3)
+	addi  CO, CO, 8
+.endm
+
+/*                                             macros for N=2 and M=8
+**********************************************************************************************/
+
+.macro	ZERO2x8
+	xxsetaccz	0
+	xxsetaccz	1
+	xxsetaccz	2
+	xxsetaccz	3
+.endm
+
+.macro	LOAD2x8
+	LOAD2x8O 0, 0
+.endm
+
+.macro	LOAD2x8O  OffsetA, OffsetB
+	lxv	vs34, (\OffsetB+0)(BO)
+	lxvp	vs32, (\OffsetA+0)(AO)
+	lxvp	vs36, (\OffsetA+32)(AO)
+.endm
+
+.macro	END2x8_NORMAL
+	END2x8 AO, BO, 64, 16
+.endm
+
+.macro	END2x8_WITHOUT_ADD
+	END2x8 AO, BO, 0, 0
+.endm
+
+.macro	END2x8 AREG, BREG, OffsetA, OffsetB
+.if \OffsetB != 0
+	addi  \BREG, \BREG, \OffsetB
+.endif
+
+.if \OffsetA != 0
+	addi  \AREG, \AREG, \OffsetA
+.endif
+	xvf32gerpp	2, 37, 34
+	xvf32gerpp	3, 36, 34
+	xvf32gerpp	0, 33, 34
+	xvf32gerpp	1, 32, 34
+.endm
+
+.macro	LOAD2x8_2
+	LOAD2x8_2O 0, 0
+.endm
+
+.macro	LOAD2x8_2O  OffsetA, OffsetB
+	lxvp	vs34, (\OffsetB)(BO)
+	lxvp	vs32, (0+\OffsetA)(AO)
+	lxvp	vs36, (32+\OffsetA)(AO)
+	lxvp	vs38, (64+\OffsetA)(AO)
+	lxvp	vs40, (64+32+\OffsetA)(AO)
+.endm
+
+.macro	END2x8_2
+  /*for load2 offset will be 128 and 32*/
+	KERNEL2x8_2  AO, BO, 128, 32, 0, 1, 1
+.endm
+
+.macro	KERNEL2x8_E2 OffsetA, OffsetB, Index, IsLast
+	KERNEL2x8_2 AO, BO, \OffsetA, \OffsetB, \Index, \IsLast, 1
+.endm
+
+.macro	KERNEL2x8_L2 OffsetA, OffsetB, Index, IsLast
+	KERNEL2x8_2 AO, BO, \OffsetA, \OffsetB, \Index, \IsLast, 0
+.endm
+
+.macro	KERNEL2x8_2  AREG, BREG, OffsetA, OffsetB, Index, IsLast, Complete
+	xvf32gerpp	2, 37, 35
+	xvf32gerpp	3, 36, 35
+	xvf32gerpp	0, 33, 35
+	xvf32gerpp	1, 32, 35
+
+.if \Complete==0
+	lxvp	vs32, DISP16(\Index, 0+\OffsetA)(\AREG)
+	lxvp	vs36, DISP16(\Index, 32+\OffsetA)(\AREG)
+.endif
+	xvf32gerpp	2, 41, 34
+	xvf32gerpp	3, 40, 34
+	xvf32gerpp	0, 39, 34
+	xvf32gerpp	1, 38, 34
+
+.if \Complete==0
+	lxvp	vs34, DISP4(\Index, \OffsetB)(\BREG)
+	lxvp	vs38, DISP16(\Index, 64+\OffsetA)(\AREG)
+	lxvp	vs40, DISP16(\Index, 64+32+\OffsetA)(\AREG)
+.endif
+.if \IsLast==1
+.if \Complete==1
+	addi    \BREG, \BREG, DISP4(\Index, \OffsetB)
+	addi    \AREG, \AREG, DISP16(\Index, \OffsetA)
+.else
+	addi    \BREG, \BREG, DISP4(\Index, 32)
+	addi    \AREG, \AREG, DISP16(\Index, 128)
+.endif
+.endif
+.endm
+
+.macro	KERNEL2x8
+	LOAD2x8
+	END2x8  AO, BO, 64, 16
+.endm
+
+.macro SAVE2x8
+	SHUFFLE_ACC	0, vs0, vs1, vs2, vs3, vs32, vs40, vs36, vs44
+	SHUFFLE_ACC	1, vs4, vs5, vs6, vs7, vs33, vs41, vs37, vs45
+	SHUFFLE_ACC	2, vs8, vs9, vs10, vs11, vs34, vs42, vs38, vs46
+	SHUFFLE_ACC	3, vs12, vs13, vs14, vs15, vs35, vs43, vs39, vs47
+	add	T1, CO, LDC
+#ifndef TRMMKERNEL
+	lxvp	vs24, 0(CO)
+#endif
+#ifndef TRMMKERNEL
+	lxvp	vs26, 32(CO)
+#endif
+#ifndef TRMMKERNEL
+	lxvp	vs28, 0(T1)
+#endif
+#ifndef TRMMKERNEL
+	lxvp	vs30, 32(T1)
+#endif
+	add	T2, CO, T4
+	add	T3, T1, T4
+	GROUP1
+	AGG_GROUP1
+	GROUP2
+	AGG_GROUP2
+  /*VSINRR, VSINII, VSOUT1, VSOUT2*/
+	MULTIPLY_GROUP1
+	MULTIPLY_GROUP2
+/* reconstruct r, i pairs*/
+	RECONSTRUCT_PAIR1
+	RECONSTRUCT_PAIR2
+#ifndef TRMMKERNEL
+  /* add */
+	xxpermdi	vs1, vs8, vs0, 2
+	xxpermdi	vs3, vs10, vs2, 2
+	xxpermdi	vs5, vs12, vs4, 2
+	xxpermdi	vs7, vs14, vs6, 2
+	xxpermdi	vs9, vs0, vs8, 2
+	xxpermdi	vs11, vs2, vs10, 2
+	xvaddsp	vs24, vs24, vs3
+	xvaddsp	vs25, vs25, vs1
+	xxpermdi	vs13, vs4, vs12, 2
+	xxpermdi	vs15, vs6, vs14, 2
+	xvaddsp	vs26, vs26, vs7
+	xvaddsp	vs27, vs27, vs5
+	xvaddsp	vs28, vs28, vs11
+	xvaddsp	vs29, vs29, vs9
+	xvaddsp	vs30, vs30, vs15
+	xvaddsp	vs31, vs31, vs13
+#else
+	xxpermdi	vs25, vs8, vs0, 2
+	xxpermdi	vs24, vs10, vs2, 2
+	xxpermdi	vs27, vs12, vs4, 2
+	xxpermdi	vs26, vs14, vs6, 2
+	xxpermdi	vs29, vs0, vs8, 2
+	xxpermdi	vs28, vs2, vs10, 2
+	xxpermdi	vs31, vs4, vs12, 2
+	xxpermdi	vs30, vs6, vs14, 2
+#endif
+	stxvp	vs24, 0(CO)
+	stxvp	vs26, 32(CO)
+	stxvp	vs28, 0(T1)
+	stxvp	vs30, 32(T1)
+	addi  CO, CO, 64
+.endm
+
+/*                                             macros for N=2 and M=4
+**********************************************************************************************/
+
+.macro	ZERO2x4
+	xxsetaccz	0
+	xxsetaccz	1
+.endm
+
+.macro	LOAD2x4
+	LOAD2x4O 0, 0
+.endm
+
+.macro	LOAD2x4O  OffsetA, OffsetB
+	lxv	vs34, (\OffsetB+0)(BO)
+	lxvp	vs32, (\OffsetA+0)(AO)
+.endm
+
+.macro	END2x4_NORMAL
+	END2x4 AO, BO, 32, 16
+.endm
+
+.macro	END2x4_WITHOUT_ADD
+	END2x4 AO, BO, 0, 0
+.endm
+
+.macro	END2x4 AREG, BREG, OffsetA, OffsetB
+.if \OffsetB != 0
+	addi  \BREG, \BREG, \OffsetB
+.endif
+.if \OffsetA != 0
+	addi  \AREG, \AREG, \OffsetA
+.endif
+	xvf32gerpp	0, 33, 34
+	xvf32gerpp	1, 32, 34
+.endm
+
+.macro	LOAD2x4_2
+	LOAD2x4_2O 0, 0
+.endm
+
+.macro	LOAD2x4_2O  OffsetA, OffsetB
+	lxvp	vs34, (\OffsetB)(BO)
+	lxvp	vs32, (0+\OffsetA)(AO)
+	lxvp	vs36, (32+\OffsetA)(AO)
+.endm
+
+.macro	END2x4_2
+  /*for load2 offset will be 64 and 32*/
+	KERNEL2x4_2  AO, BO, 64, 32, 0, 1, 1
+.endm
+
+.macro	KERNEL2x4_E2 OffsetA, OffsetB, Index, IsLast
+	KERNEL2x4_2 AO, BO, \OffsetA, \OffsetB, \Index, \IsLast, 1
+.endm
+
+.macro	KERNEL2x4_L2 OffsetA, OffsetB, Index, IsLast
+	KERNEL2x4_2 AO, BO, \OffsetA, \OffsetB, \Index, \IsLast, 0
+.endm
+
+.macro	KERNEL2x4_2  AREG, BREG, OffsetA, OffsetB, Index, IsLast, Complete
+	xvf32gerpp	0, 33, 35
+	xvf32gerpp	1, 32, 35
+.if \Complete==0
+	lxvp	vs32, DISP8(\Index, 0+\OffsetA)(\AREG)
+.endif
+	xvf32gerpp	0, 37, 34
+	xvf32gerpp	1, 36, 34
+.if \Complete==0
+	lxvp	vs34, DISP4(\Index, \OffsetB)(\BREG)
+	lxvp	vs36, DISP8(\Index, 32+\OffsetA)(\AREG)
+.endif
+.if \IsLast==1
+.if \Complete==1
+	addi    \BREG, \BREG, DISP4(\Index, \OffsetB)
+	addi    \AREG, \AREG, DISP8(\Index, \OffsetA)
+.else
+	addi    \BREG, \BREG, DISP4(\Index, 32)
+	addi    \AREG, \AREG, DISP8(\Index, 64)
+.endif
+.endif
+.endm
+
+.macro	KERNEL2x4
+	LOAD2x4
+	END2x4  AO, BO, 32, 16
+.endm
+
+.macro SAVE2x4
+	SHUFFLE_ACC	0, vs0, vs1, vs2, vs3, vs32, vs40, vs36, vs44
+	SHUFFLE_ACC	1, vs4, vs5, vs6, vs7, vs33, vs41, vs37, vs45
+	add	T1, CO, LDC
+#ifndef TRMMKERNEL
+	lxvp	vs24, 0(CO)
+#endif
+#ifndef TRMMKERNEL
+	lxvp	vs26, 0(T1)
+#endif
+	GROUP1
+	AGG_GROUP1
+  /*VSINRR, VSINII, VSOUT1, VSOUT2*/
+	MULTIPLY_GROUP1
+/* reconstruct r, i pairs*/
+	RECONSTRUCT_PAIR1
+#ifndef TRMMKERNEL
+  /* add */
+	xxpermdi	vs1, vs8, vs0, 2
+	xxpermdi	vs3, vs10, vs2, 2
+	xxpermdi	vs9, vs0, vs8, 2
+	xxpermdi	vs11, vs2, vs10, 2
+	xvaddsp	vs24, vs24, vs3
+	xvaddsp	vs25, vs25, vs1
+	xvaddsp	vs26, vs26, vs11
+	xvaddsp	vs27, vs27, vs9
+#else
+	xxpermdi	vs25, vs8, vs0, 2
+	xxpermdi	vs24, vs10, vs2, 2
+	xxpermdi	vs27, vs0, vs8, 2
+	xxpermdi	vs26, vs2, vs10, 2
+#endif
+	stxvp	vs24, 0(CO)
+	stxvp	vs26, 0(T1)
+	addi  CO, CO, 32
+.endm
+
+/*                                             macros for N=2 and M=2
+**********************************************************************************************/
+
+.macro	ZERO2x2
+	xxsetaccz	0
+.endm
+
+.macro	LOAD2x2
+	LOAD2x2O 0, 0
+.endm
+
+.macro	LOAD2x2O  OffsetA, OffsetB
+	lxv	vs32, (\OffsetA+0)(AO)
+	lxv	vs34, (\OffsetB+0)(BO)
+.endm
+
+.macro	END2x2_NORMAL
+	END2x2 AO, BO, 16, 16
+.endm
+
+.macro	END2x2_WITHOUT_ADD
+	END2x2 AO, BO, 0, 0
+.endm
+
+.macro	END2x2 AREG, BREG, OffsetA, OffsetB
+.if \OffsetB != 0
+	addi  \BREG, \BREG, \OffsetB
+.endif
+
+.if \OffsetA != 0
+	addi  \AREG, \AREG, \OffsetA
+.endif
+	xvf32gerpp	0, 34, 32
+.endm
+
+.macro	LOAD2x2_2
+	LOAD2x2_2O 0, 0
+.endm
+
+.macro	LOAD2x2_2O  OffsetA, OffsetB
+	lxvp	vs32, (\OffsetA)(AO)
+	lxvp	vs34, (0+\OffsetB)(BO)
+.endm
+
+.macro	END2x2_2
+  /*for load2 offset will be 32 and 32*/
+	KERNEL2x2_2  AO, BO, 32, 32, 0, 1, 1
+.endm
+
+.macro	KERNEL2x2_E2 OffsetA, OffsetB, Index, IsLast
+	KERNEL2x2_2 AO, BO, \OffsetA, \OffsetB, \Index, \IsLast, 1
+.endm
+
+.macro	KERNEL2x2_L2 OffsetA, OffsetB, Index, IsLast
+	KERNEL2x2_2 AO, BO, \OffsetA, \OffsetB, \Index, \IsLast, 0
+.endm
+
+.macro	KERNEL2x2_2  AREG, BREG, OffsetA, OffsetB, Index, IsLast, Complete
+	xvf32gerpp	0, 34, 32
+	xvf32gerpp	0, 35, 33
+.if \Complete==0
+	lxvp	vs32, DISP4(\Index, \OffsetA)(\AREG)
+	lxvp	vs34, DISP4(\Index, \OffsetA)(\BREG)
+.endif
+.if \IsLast==1
+.if \Complete==1
+	addi    \AREG, \AREG, DISP4(\Index, \OffsetA)
+	addi    \BREG, \BREG, DISP4(\Index, \OffsetB)
+.else
+	addi    \AREG, \AREG, DISP4(\Index, 32)
+	addi    \BREG, \BREG, DISP4(\Index, 32)
+.endif
+.endif
+.endm
+
+.macro	KERNEL2x2
+	LOAD2x2
+	END2x2  AO, BO, 16, 16
+.endm
+
+.macro SAVE2x2
+	SHUFFLE_ACC	0, vs0, vs1, vs2, vs3, vs32, vs40, vs36, vs44
+	add	T1, CO, LDC
+#ifndef TRMMKERNEL
+	lxv	vs24, 0(CO)
+#endif
+#ifndef TRMMKERNEL
+	lxv	vs26, 0(T1)
+#endif
+	xxperm	vs0, vs32, permute_mask
+	xxperm	vs4, vs40, permute_mask
+	xxperm	vs8, vs36, permute_mask
+	xxperm	vs12, vs44, permute_mask
+	AGGREGATE_REALS_IMAGES_A_PERMUTE vs32, vs0, vs40, vs4
+	AGGREGATE_REALS_IMAGES_A_PERMUTE vs36, vs8, vs44, vs12
+  /*VSINRR, VSINII, VSOUT1, VSOUT2*/
+	MULT_APLHA_PART1    vs32, vs40, vs0, vs1
+	MULT_APLHA_PART1    vs36, vs44, vs8, vs9
+	MULT_APLHA_PART2    vs32, vs40, vs0, vs1
+	MULT_APLHA_PART2    vs36, vs44, vs8, vs9
+/* reconstruct r, i pairs*/
+	xxperm	vs0, vs1, save_permute_1
+	xxperm	vs8, vs9, save_permute_1
+#ifndef TRMMKERNEL
+  /* add */
+	xxpermdi	vs1, vs8, vs0, 0
+	xxpermdi	vs9, vs0, vs8, 3
+	xvaddsp	vs24, vs24, vs1
+	xvaddsp	vs26, vs26, vs9
+#else
+	xxpermdi	vs24, vs8, vs0, 0
+	xxpermdi	vs26, vs0, vs8, 3
+#endif
+	stxv	vs24, 0(CO)
+	stxv	vs26, 0(T1)
+	addi  CO, CO, 16
+.endm
+
+/*                                             macros for N=2 and M=1
+**********************************************************************************************/
+
+.macro	ZERO2x1
+	xxlxor  vs32, vs32, vs32
+	xxlxor  vs40, vs40, vs40
+.endm
+
+.macro	LOAD2x1
+	LOAD2x1O 0, 0
+.endm
+
+.macro	LOAD2x1O  OffsetA, OffsetB
+	lxsd	v4, (\OffsetA+0)(AO)
+	lxv	vs0, (\OffsetB+0)(BO)
+	xxspltd  vs24, vs36, 0
+	xxperm    vs26, vs24, permute_mask
+.endm
+
+.macro	END2x1_NORMAL
+	END2x1 AO, BO,8, 16
+.endm
+
+.macro	END2x1_WITHOUT_ADD
+	END2x1 AO, BO, 0, 0
+.endm
+
+.macro	END2x1 AREG, BREG, OffsetA, OffsetB
+.if \OffsetB != 0
+	addi  \BREG, \BREG, \OffsetB
+.endif
+
+.if \OffsetA != 0
+	addi  \AREG, \AREG, \OffsetA
+.endif
+	xvmaddasp	vs32, vs0, vs24
+	xvmaddasp	vs40, vs0, vs26
+.endm
+
+.macro	LOAD2x1_2
+	LOAD2x1_2O 0, 0
+.endm
+
+.macro	LOAD2x1_2O  OffsetA, OffsetB
+	lxv	vs27, (\OffsetA)(AO)
+	lxvp	vs4, (0+\OffsetB)(BO)
+	xxspltd  vs8, vs27, 1
+	xxspltd  vs24, vs27, 0
+	xxperm    vs10, vs8, permute_mask
+	xxperm    vs26, vs24, permute_mask
+.endm
+
+.macro	END2x1_2
+  /*for load2 offset will be 16 and 32*/
+	KERNEL2x1_2  AO, BO, 16, 32, 0, 1, 1
+.endm
+
+.macro	KERNEL2x1_E2 OffsetA, OffsetB, Index, IsLast
+	KERNEL2x1_2 AO, BO, \OffsetA, \OffsetB, \Index, \IsLast, 1
+.endm
+
+.macro	KERNEL2x1_L2 OffsetA, OffsetB, Index, IsLast
+	KERNEL2x1_2 AO, BO, \OffsetA, \OffsetB, \Index, \IsLast, 0
+.endm
+
+.macro	KERNEL2x1_2  AREG, BREG, OffsetA, OffsetB, Index, IsLast, Complete
+	xvmaddasp	vs32, vs5, vs8
+	xvmaddasp	vs40, vs5, vs10
+.if \Complete==0
+	lxv	vs27, DISP2(\Index, \OffsetA)(\AREG)
+	xxspltd  vs8, vs27, 1
+.endif
+.if \Complete==0
+	xxperm    vs10, vs8, permute_mask
+.endif
+	xvmaddasp	vs32, vs4, vs24
+	xvmaddasp	vs40, vs4, vs26
+.if \Complete==0
+	xxspltd  vs24, vs27, 0
+	xxperm   vs26, vs24, permute_mask
+.endif
+.if \Complete==0
+	lxvp	vs4, DISP4(\Index, 0+\OffsetB)(\BREG)
+.endif
+.if \IsLast==1
+.if \Complete==1
+	addi    \AREG, \AREG, DISP2(\Index, \OffsetA)
+	addi    \BREG, \BREG, DISP4(\Index, \OffsetB)
+.else
+	addi    \AREG, \AREG, DISP2(\Index, 16)
+	addi    \BREG, \BREG, DISP4(\Index, 32)
+.endif
+.endif
+.endm
+
+.macro	KERNEL2x1
+	LOAD2x1
+	END2x1  AO, BO, 8, 16
+.endm
+
+.macro SAVE2x1
+	add	T1, CO, LDC
+#ifndef TRMMKERNEL
+	lxsd	v4, 0(CO)
+#endif
+#ifndef TRMMKERNEL
+	lxsd	v5, 0(T1)
+#endif
+	xxperm	vs0, vs32, permute_mask
+	xxperm	vs4, vs40, permute_mask
+	AGGREGATE_REALS_IMAGES_A_PERMUTE vs32, vs0, vs40, vs4
+	AGGREGATE_REALS_IMAGES_A_PERMUTE vs33, vs1, vs41, vs5
+  /*VSINRR, VSINII, VSOUT1, VSOUT2*/
+	MULT_APLHA_PART1    vs32, vs40, vs0, vs1
+	MULT_APLHA_PART2    vs32, vs40, vs0, vs1
+/* reconstruct r, i pairs*/
+	xxperm	vs0, vs1, save_permute_1
+#ifndef TRMMKERNEL
+  /* add */
+	xxspltd vs1, vs0, 0
+	xxspltd vs3, vs0, 1
+ /*--v4==vs36 v5==vs37---*/
+	xvaddsp	vs36, vs36, vs1
+	xvaddsp	vs37, vs37, vs3
+#else
+ /*--v4==vs36 v5==vs37---*/
+	xxspltd vs36, vs0, 0
+	xxspltd vs37, vs0, 1
+#endif
+	stxsd	v4, 0(CO)
+	stxsd	v5, 0(T1)
+	addi  CO, CO, 8
+.endm
+
+/*                                             macros for N=1 and M=8
+**********************************************************************************************/
+
+.macro	ZERO1x8
+	xxsetaccz	0
+	xxsetaccz	1
+	xxsetaccz	2
+	xxsetaccz	3
+.endm
+
+.macro	LOAD1x8
+	LOAD1x8O 0, 0
+.endm
+
+.macro	LOAD1x8O  OffsetA, OffsetB
+	lxsd	v2, (\OffsetB+0)(BO)
+	lxvp	vs32, (\OffsetA+0)(AO)
+	lxvp	vs36, (\OffsetA+32)(AO)
+.endm
+
+.macro	END1x8_NORMAL
+	END1x8 AO, BO, 64,8
+.endm
+
+.macro	END1x8_WITHOUT_ADD
+	END1x8 AO, BO, 0, 0
+.endm
+
+.macro	END1x8 AREG, BREG, OffsetA, OffsetB
+.if \OffsetB != 0
+	addi  \BREG, \BREG, \OffsetB
+.endif
+
+.if \OffsetA != 0
+	addi  \AREG, \AREG, \OffsetA
+.endif
+	xvf32gerpp	    0, 34, 33
+	xvf32gerpp	    1, 34, 32
+	xvf32gerpp	    2, 34, 37
+	xvf32gerpp	    3, 34, 36
+.endm
+
+.macro	LOAD1x8_2
+	LOAD1x8_2O 0, 0
+.endm
+
+.macro	LOAD1x8_2O  OffsetA, OffsetB
+	lxv	vs34, (\OffsetB)(BO)
+	lxvp	vs32, (0+\OffsetA)(AO)
+	lxvp	vs36, (32+\OffsetA)(AO)
+	vspltisb        v10, 0
+	xxpermdi        vs35, vs34, vs42, 0
+	xxpermdi        vs34, vs34, vs42, 2
+	lxvp	vs38, (64+\OffsetA)(AO)
+	lxvp	vs40, (64+32+\OffsetA)(AO)
+.endm
+
+.macro	END1x8_2
+  /*for load2 offset will be 128 and 16*/
+	KERNEL1x8_2  AO, BO, 128, 16, 0, 1, 1
+.endm
+
+.macro	KERNEL1x8_E2 OffsetA, OffsetB, Index, IsLast
+	KERNEL1x8_2 AO, BO, \OffsetA, \OffsetB, \Index, \IsLast, 1
+.endm
+
+.macro	KERNEL1x8_L2 OffsetA, OffsetB, Index, IsLast
+	KERNEL1x8_2 AO, BO, \OffsetA, \OffsetB, \Index, \IsLast, 0
+.endm
+
+.macro	KERNEL1x8_2  AREG, BREG, OffsetA, OffsetB, Index, IsLast, Complete
+	xvf32gerpp	    0, 34, 33
+	xvf32gerpp	    1, 34, 32
+.if \Complete==0
+	lxvp	vs32, DISP16(\Index, 0+\OffsetA)(\AREG)
+.endif
+	xvf32gerpp	    2, 34, 37
+	xvf32gerpp	    3, 34, 36
+.if \Complete==0
+	lxvp	vs36, DISP16(\Index, 32+\OffsetA)(\AREG)
+.endif
+	xvf32gerpp	    0, 35, 39
+	xvf32gerpp	    1, 35, 38
+.if \Complete==0
+	lxvp	vs38, DISP16(\Index, 64+\OffsetA)(\AREG)
+.endif
+	xvf32gerpp	    2, 35, 41
+	xvf32gerpp	    3, 35, 40
+.if \Complete==0
+	lxv	vs34, DISP2(\Index, \OffsetB)(\BREG)
+	xxpermdi        vs35, vs34, vs42, 0
+	xxpermdi        vs34, vs34, vs42, 2
+	lxvp	vs40, DISP16(\Index, 64+32+\OffsetA)(\AREG)
+.endif
+.if \IsLast==1
+.if \Complete==1
+	addi    \BREG, \BREG, DISP2(\Index, \OffsetB)
+	addi    \AREG, \AREG, DISP16(\Index, \OffsetA)
+.else
+	addi    \BREG, \BREG, DISP2(\Index, 16)
+	addi    \AREG, \AREG, DISP16(\Index, 128)
+.endif
+.endif
+.endm
+
+.macro	KERNEL1x8
+	LOAD1x8
+	END1x8  AO, BO, 64,8
+.endm
+
+.macro SAVE1x8
+	SHUFFLE_ACC	0, vs0, vs1, vs2, vs3, vs32, vs40, vs36, vs44
+	SHUFFLE_ACC	1, vs4, vs5, vs6, vs7, vs33, vs41, vs37, vs45
+	SHUFFLE_ACC	2, vs8, vs9, vs10, vs11, vs34, vs42, vs38, vs46
+	SHUFFLE_ACC	3, vs12, vs13, vs14, vs15, vs35, vs43, vs39, vs47
+	xxpermdi	vs32, vs32, vs36, 0
+	xxpermdi	vs33, vs33, vs37, 0
+	xxpermdi	vs34, vs34, vs38, 0
+	xxpermdi	vs35, vs35, vs39, 0
+	xxpermdi	vs40, vs40, vs44, 0
+	xxperm vs40, vs40, permute_mask
+	xxpermdi	vs41, vs41, vs45, 0
+	xxperm vs41, vs41, permute_mask
+	xxpermdi	vs42, vs42, vs46, 0
+	xxperm vs42, vs42, permute_mask
+	xxpermdi	vs43, vs43, vs47, 0
+	xxperm vs43, vs43, permute_mask
+#ifndef TRMMKERNEL
+	lxvp	vs24, 0(CO)
+#endif
+	xxperm	vs0, vs32, permute_mask
+	xxperm	vs4, vs40, permute_mask
+#ifndef TRMMKERNEL
+	lxvp	vs26, 32(CO)
+#endif
+	xxperm	vs1, vs33, permute_mask
+	xxperm	vs5, vs41, permute_mask
+	xxperm	vs2, vs34, permute_mask
+	xxperm	vs6, vs42, permute_mask
+	xxperm	vs3, vs35, permute_mask
+	xxperm	vs7, vs43, permute_mask
+	AGGREGATE_REALS_IMAGES	vs32, vs0, vs40, vs4
+	AGGREGATE_REALS_IMAGES	vs33, vs1, vs41, vs5
+	AGGREGATE_REALS_IMAGES	vs34, vs2, vs42, vs6
+	AGGREGATE_REALS_IMAGES	vs35, vs3, vs43, vs7
+  /*inner reverse save_permute and store vs28 */
+	xxpermdi	vs28,save_permute_1,save_permute_1, 2
+  /*VSINRR, VSINII, VSOUT1, VSOUT2*/
+	MULT_APLHA_PART1    vs32, vs40, vs0, vs1
+	MULT_APLHA_PART1    vs33, vs41, vs2, vs3
+	MULT_APLHA_PART1    vs34, vs42, vs4, vs5
+	MULT_APLHA_PART1    vs35, vs43, vs6, vs7
+	MULT_APLHA_PART2    vs32, vs40, vs0, vs1
+	MULT_APLHA_PART2    vs33, vs41, vs2, vs3
+	MULT_APLHA_PART2    vs34, vs42, vs4, vs5
+	MULT_APLHA_PART2    vs35, vs43, vs6, vs7
+/* reconstruct r, i pairs*/
+	xxperm	vs0, vs1, vs28
+	xxperm	vs2, vs3, vs28
+	xxperm	vs4, vs5, vs28
+	xxperm	vs6, vs7, vs28
+#ifndef TRMMKERNEL
+  /* add */
+	xvaddsp	vs24, vs24, vs2
+	xvaddsp	vs25, vs25, vs0
+	xvaddsp	vs26, vs26, vs6
+	xvaddsp	vs27, vs27, vs4
+	stxvp	vs24, 0(CO)
+	stxvp	vs26, 32(CO)
+#else
+/* reconstruct r, i pairs*/
+	stxv	vs0, 0(CO)
+	stxv	vs2, 16(CO)
+	stxv	vs4, 32(CO)
+	stxv	vs6, 48(CO)
+#endif
+	addi  CO, CO, 64
+.endm
+
+/*                                             macros for N=1 and M=4
+**********************************************************************************************/
+
+.macro	ZERO1x4
+	xxsetaccz	0
+	xxsetaccz	1
+.endm
+
+.macro	LOAD1x4
+	LOAD1x4O 0, 0
+.endm
+
+.macro	LOAD1x4O  OffsetA, OffsetB
+	lxsd	v2, (\OffsetB+0)(BO)
+	lxvp	vs32, (\OffsetA+0)(AO)
+.endm
+
+.macro	END1x4_NORMAL
+	END1x4 AO, BO, 32,8
+.endm
+
+.macro	END1x4_WITHOUT_ADD
+	END1x4 AO, BO, 0, 0
+.endm
+
+.macro	END1x4 AREG, BREG, OffsetA, OffsetB
+.if \OffsetB != 0
+	addi  \BREG, \BREG, \OffsetB
+.endif
+
+.if \OffsetA != 0
+	addi  \AREG, \AREG, \OffsetA
+.endif
+	xvf32gerpp	    0, 34, 33
+	xvf32gerpp	    1, 34, 32
+.endm
+
+.macro	LOAD1x4_2
+	LOAD1x4_2O 0, 0
+.endm
+
+.macro	LOAD1x4_2O  OffsetA, OffsetB
+	lxv	vs34, (\OffsetB)(BO)
+	lxvp	vs32, (0+\OffsetA)(AO)
+	vspltisb        v6, 0
+	xxpermdi        vs35, vs34, vs38, 0
+	xxpermdi        vs34, vs34, vs38, 2
+	lxvp	vs36, (32+\OffsetA)(AO)
+.endm
+
+.macro	END1x4_2
+  /*for load2 offset will be 64 and 16*/
+	KERNEL1x4_2  AO, BO, 64, 16, 0, 1, 1
+.endm
+
+.macro	KERNEL1x4_E2 OffsetA, OffsetB, Index, IsLast
+	KERNEL1x4_2 AO, BO, \OffsetA, \OffsetB, \Index, \IsLast, 1
+.endm
+
+.macro	KERNEL1x4_L2 OffsetA, OffsetB, Index, IsLast
+	KERNEL1x4_2 AO, BO, \OffsetA, \OffsetB, \Index, \IsLast, 0
+.endm
+
+.macro	KERNEL1x4_2  AREG, BREG, OffsetA, OffsetB, Index, IsLast, Complete
+	xvf32gerpp	    0, 34, 33
+	xvf32gerpp	    1, 34, 32
+.if \Complete==0
+	lxvp	vs32, DISP8(\Index, 0+\OffsetA)(\AREG)
+.endif
+	xvf32gerpp	    0, 35, 37
+	xvf32gerpp	    1, 35, 36
+.if \Complete==0
+	lxv	vs34, DISP2(\Index, \OffsetB)(\BREG)
+	xxpermdi        vs35, vs34, vs38, 0
+	xxpermdi        vs34, vs34, vs38, 2
+	lxvp	vs36, DISP8(\Index, 32+\OffsetA)(\AREG)
+.endif
+.if \IsLast==1
+.if \Complete==1
+	addi    \BREG, \BREG, DISP2(\Index, \OffsetB)
+	addi    \AREG, \AREG, DISP8(\Index, \OffsetA)
+.else
+	addi    \BREG, \BREG, DISP2(\Index, 16)
+	addi    \AREG, \AREG, DISP8(\Index, 64)
+.endif
+.endif
+.endm
+
+.macro	KERNEL1x4
+	LOAD1x4
+	END1x4	AO, BO, 32,8
+.endm
+
+.macro SAVE1x4
+	SHUFFLE_ACC	0, vs0, vs1, vs2, vs3, vs32, vs40, vs36, vs44
+	SHUFFLE_ACC	1, vs4, vs5, vs6, vs7, vs33, vs41, vs37, vs45
+	xxpermdi	vs32, vs32, vs36, 0
+	xxpermdi	vs40, vs40, vs44, 0
+	xxpermdi	vs33, vs33, vs37, 0
+	xxpermdi	vs41, vs41, vs45, 0
+	xxperm vs40, vs40, permute_mask
+	xxperm vs41, vs41, permute_mask
+#ifndef TRMMKERNEL
+	lxvp	vs24, 0(CO)
+#endif
+	xxperm	vs0, vs32, permute_mask
+	xxperm	vs4, vs40, permute_mask
+	xxperm	vs1, vs33, permute_mask
+	xxperm	vs5, vs41, permute_mask
+	AGGREGATE_REALS_IMAGES	vs32, vs0, vs40, vs4
+	AGGREGATE_REALS_IMAGES	vs33, vs1, vs41, vs5
+  /*inner reverse save_permute and store vs28 */
+	xxpermdi	vs28,save_permute_1,save_permute_1, 2
+  /*VSINRR, VSINII, VSOUT1, VSOUT2*/
+	MULT_APLHA_PART1    vs32, vs40, vs0, vs1
+	MULT_APLHA_PART1    vs33, vs41, vs2, vs3
+	MULT_APLHA_PART2    vs32, vs40, vs0, vs1
+	MULT_APLHA_PART2    vs33, vs41, vs2, vs3
+/* reconstruct r, i pairs*/
+	xxperm	vs0, vs1, vs28
+	xxperm	vs2, vs3, vs28
+#ifndef TRMMKERNEL
+  /* add */
+	xvaddsp	vs24, vs24, vs2
+	xvaddsp	vs25, vs25, vs0
+	stxvp	vs24, 0(CO)
+#else
+/* reconstruct r, i pairs*/
+	stxv	vs0, 0(CO)
+	stxv	vs2, 16(CO)
+#endif
+	addi  CO, CO, 32
+.endm
+
+/*                                             macros for N=1 and M=2
+**********************************************************************************************/
+
+.macro	ZERO1x2
+	xxlxor  vs32, vs32, vs32
+	xxlxor  vs40, vs40, vs40
+.endm
+
+.macro	LOAD1x2
+	LOAD1x2O 0, 0
+.endm
+
+.macro	LOAD1x2O  OffsetA, OffsetB
+	lxsd	vs4, (\OffsetB+0)(BO)
+	lxv	vs0, (\OffsetA+0)(AO)
+	xxspltd   vs24, vs36, 0
+	xxperm    vs26, vs24, permute_mask
+.endm
+
+.macro	END1x2_NORMAL
+	END1x2 AO, BO, 16,8
+.endm
+
+.macro	END1x2_WITHOUT_ADD
+	END1x2 AO, BO, 0, 0
+.endm
+
+.macro	END1x2 AREG, BREG, OffsetA, OffsetB
+.if \OffsetB != 0
+	addi  \BREG, \BREG, \OffsetB
+.endif
+.if \OffsetA != 0
+	addi  \AREG, \AREG, \OffsetA
+.endif
+	xvmaddasp	vs32, vs0, vs24
+	xvmaddasp	vs40, vs0, vs26
+.endm
+
+.macro	LOAD1x2_2
+	LOAD1x2_2O 0, 0
+.endm
+
+.macro	LOAD1x2_2O  OffsetA, OffsetB
+	lxv	vs27, (\OffsetB)(BO)
+	lxvp	vs4, (0+\OffsetA)(AO)
+	xxspltd  vs8, vs27, 1
+	xxspltd  vs24, vs27, 0
+	xxperm    vs10, vs8, permute_mask
+	xxperm    vs26, vs24, permute_mask
+.endm
+
+.macro	END1x2_2
+  /*for load2 offset will be 32 and 16*/
+	KERNEL1x2_2  AO, BO, 32, 16, 0, 1, 1
+.endm
+
+.macro	KERNEL1x2_E2 OffsetA, OffsetB, Index, IsLast
+	KERNEL1x2_2 AO, BO, \OffsetA, \OffsetB, \Index, \IsLast, 1
+.endm
+
+.macro	KERNEL1x2_L2 OffsetA, OffsetB, Index, IsLast
+	KERNEL1x2_2 AO, BO, \OffsetA, \OffsetB, \Index, \IsLast, 0
+.endm
+
+.macro	KERNEL1x2_2  AREG, BREG, OffsetA, OffsetB, Index, IsLast, Complete
+.if \Complete==0
+	lxv	vs27, DISP2(\Index, \OffsetB)(\BREG)
+.endif
+	xvmaddasp	vs32, vs5, vs8
+	xvmaddasp	vs40, vs5, vs10
+
+.if \Complete==0
+	xxspltd  vs8, vs27, 1
+	xxperm    vs10, vs8, permute_mask
+.endif
+	xvmaddasp	vs32, vs4, vs24
+	xvmaddasp	vs40, vs4, vs26
+.if \Complete==0
+	lxvp	vs4, DISP4(\Index, 0+\OffsetA)(\AREG)
+.endif
+
+.if \Complete==0
+	xxspltd  vs24, vs27, 0
+	xxperm    vs26, vs24, permute_mask
+.endif
+.if \IsLast==1
+.if \Complete==1
+	addi    \BREG, \BREG, DISP2(\Index, \OffsetB)
+	addi    \AREG, \AREG, DISP4(\Index, \OffsetA)
+.else
+	addi    \BREG, \BREG, DISP2(\Index, 16)
+	addi    \AREG, \AREG, DISP4(\Index, 32)
+.endif
+.endif
+.endm
+
+.macro	KERNEL1x2
+	LOAD1x2
+	END1x2  AO, BO, 16,8
+.endm
+
+.macro SAVE1x2
+#ifndef TRMMKERNEL
+	lxv	vs24, 0(CO)
+#endif
+	xxperm	vs0, vs32, permute_mask
+	xxperm	vs4, vs40, permute_mask
+	AGGREGATE_REALS_IMAGES	vs32, vs0, vs40, vs4
+  /*inner reverse save_permute and store vs28 */
+	xxpermdi	vs28,save_permute_1,save_permute_1, 2
+  /*VSINRR, VSINII, VSOUT1, VSOUT2*/
+	MULT_APLHA_PART1    vs32, vs40, vs0, vs1
+	MULT_APLHA_PART2    vs32, vs40, vs0, vs1
+/* reconstruct r, i pairs*/
+	xxperm	vs0, vs1, vs28
+#ifndef TRMMKERNEL
+  /* add */
+	xvaddsp	vs24, vs24, vs0
+	stxv	vs24, 0(CO)
+#else
+/* reconstruct r, i pairs*/
+	stxv	vs0, 0(CO)
+#endif
+	addi  CO, CO, 16
+.endm
+
+/*                                             macros for N=1 and M=1
+**********************************************************************************************/
+.macro	ZERO1x1
+	xxlxor  vs32, vs32, vs32
+	xxlxor  vs40, vs40, vs40
+.endm
+
+.macro	LOAD1x1
+	LOAD1x1O 0, 0
+.endm
+
+.macro	LOAD1x1O  OffsetA, OffsetB
+	lxsd	v4, (\OffsetB+0)(BO)
+	lxsd	v5, (\OffsetA+0)(AO)
+	xxperm    vs38, vs36, permute_mask
+.endm
+
+.macro	END1x1_NORMAL
+	END1x1 AO, BO,8,8
+.endm
+
+.macro	END1x1_WITHOUT_ADD
+	END1x1 AO, BO, 0, 0
+.endm
+
+.macro	END1x1 AREG, BREG, OffsetA, OffsetB
+.if \OffsetB != 0
+	addi  \BREG, \BREG, \OffsetB
+.endif
+.if \OffsetA != 0
+	addi  \AREG, \AREG, \OffsetA
+.endif
+	xvmaddasp	vs32, vs37, vs36
+	xvmaddasp	vs40, vs37, vs38
+.endm
+
+.macro	LOAD1x1_2
+	LOAD1x1_2O 0, 0
+.endm
+
+.macro	LOAD1x1_2O  OffsetA, OffsetB
+	lxv	vs8, (\OffsetB)(BO)
+	lxv	vs4, (0+\OffsetA)(AO)
+	xxperm    vs10, vs8, permute_mask
+.endm
+
+.macro	END1x1_2
+  /*for load2 offset will be 16 and 16*/
+	KERNEL1x1_2  AO, BO, 16, 16, 0, 1, 1
+.endm
+
+.macro	KERNEL1x1_E2 OffsetA, OffsetB, Index, IsLast
+	KERNEL1x1_2 AO, BO, \OffsetA, \OffsetB, \Index, \IsLast, 1
+.endm
+
+.macro	KERNEL1x1_L2 OffsetA, OffsetB, Index, IsLast
+	KERNEL1x1_2 AO, BO, \OffsetA, \OffsetB, \Index, \IsLast, 0
+.endm
+
+.macro	KERNEL1x1_2  AREG, BREG, OffsetA, OffsetB, Index, IsLast, Complete
+	xvmaddasp	vs32, vs4, vs8
+	xvmaddasp	vs40, vs4, vs10
+.if \Complete==0
+	lxv	vs8, DISP2(\Index, \OffsetB)(\BREG)
+	lxv	vs4, DISP2(\Index, \OffsetB)(\AREG)
+	xxperm    vs10, vs8, permute_mask
+.endif
+.if \IsLast==1
+.if \Complete==1
+	addi    \BREG, \BREG, DISP2(\Index, \OffsetB)
+	addi    \AREG, \AREG, DISP2(\Index, \OffsetA)
+.else
+	addi    \BREG, \BREG, DISP2(\Index, 16)
+	addi    \AREG, \AREG, DISP2(\Index, 16)
+.endif
+.endif
+.endm
+
+.macro	KERNEL1x1
+	LOAD1x1
+	END1x1  AO, BO, 8,8
+.endm
+
+.macro SAVE1x1
+#ifndef TRMMKERNEL
+	lxsd	v4, 0(CO)
+#endif
+  /*aggregate x2*/
+	xxpermdi	vs33, vs32, vs32, 2
+	xxpermdi	vs41, vs40, vs40, 2
+	xvaddsp	vs32, vs32, vs33
+	xvaddsp	vs40, vs40, vs41
+
+	xxperm	vs0, vs32, permute_mask
+	xxperm	vs4, vs40, permute_mask
+	AGGREGATE_REALS_IMAGES	vs32, vs0, vs40, vs4
+  /*inner reverse save_permute and store vs28 */
+	xxpermdi	vs28,save_permute_1,save_permute_1, 2
+  /*VSINRR, VSINII, VSOUT1, VSOUT2*/
+	MULT_APLHA_PART1    vs32, vs40, vs37, vs1
+	MULT_APLHA_PART2    vs32, vs40, vs37, vs1
+/* reconstruct r, i pairs*/
+	xxperm	vs37, vs1, vs28
+#ifndef TRMMKERNEL
+  /* add */
+	xvaddsp	vs36, vs36, vs37
+	stxsd	v4, 0(CO)
+#else
+/* vs37 is v5 */
+	stxsd	v5, 0(CO)
+#endif
+	addi  CO, CO, 8
+.endm
+
+/****************************TRMM POINTER REFRESH MACROSES*************************/
+.macro SHIFT_REG	REG1,REG2,SHIFT_VAL
+.if \SHIFT_VAL==16
+	slwi		\REG1, \REG2, 7
+.elseif \SHIFT_VAL==8
+	slwi		\REG1, \REG2, 6
+.elseif \SHIFT_VAL==4
+	slwi		\REG1, \REG2, 5
+.elseif \SHIFT_VAL==2
+	slwi		\REG1, \REG2, 4
+.elseif \SHIFT_VAL==1
+	slwi		\REG1, \REG2, 3
+.endif
+.endm
+
+/*
+//#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+// 		ptrbb = bb;
+// #else
+// 		ptrba += off*8;
+// 		ptrbb = bb + off*4;
+// #endif
+*/
+.macro REFRESH_POINTERS  PTR_A,PTR_B, OFF_VAL, B_VAL, C_A, C_B
+#if (defined(LEFT) &&  defined(TRANSA)) ||  (!defined(LEFT) && !defined(TRANSA))
+/* ptrbb = bb;*/
+	mr \PTR_B, \B_VAL     /* refresh BPOINT */
+#else
+/*
+// ptrba  =ptrba+ off*C_A;
+// ptrbb = bb + off*C_B;
+*/
+	SHIFT_REG T4, \OFF_VAL, \C_B	/* Number of values in B shifted  */
+	SHIFT_REG T2, \OFF_VAL, \C_A	/* Number of values in A shifted  */
+	add	\PTR_B, \B_VAL, T4	/* Add values to BO */
+	add	\PTR_A, \PTR_A, T2	/* Add values to AO  */
+#endif
+.endm
+
+/*
+// #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+// 		temp = bk-off;
+// #elif defined(LEFT)
+// 		temp = off+8;	// number of values in A
+// #else
+// 		temp = off+4;	// number of values in B
+// #endif
+*/
+.macro REFRESH_TEMP_BK TEMP_BK, BK_VAL, OFF_VAL, INCR_A, INCR_B
+    #if (defined(LEFT) && !defined(TRANSA)) ||  (!defined(LEFT) && defined(TRANSA))
+	/* temp = bk-off;*/
+	sub \TEMP_BK, \BK_VAL, \OFF_VAL
+    #elif defined(LEFT)
+	/* temp = off+INCR_A;	// number of values in A */
+	addi \TEMP_BK, \OFF_VAL, \INCR_A
+    #else
+	/* temp = off+INCR_B	// number of values in B*/
+	addi \TEMP_BK, \OFF_VAL, \INCR_B
+    #endif
+.endm
+/*
+// #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+// 		temp = bk - off;
+// #ifdef LEFT
+// 		temp -= 8; // number of values in A
+// #else
+// 		temp -= 4; // number of values in B
+// #endif
+// 		ptrba += temp*8;
+// 		ptrbb += temp*4;
+// #endif
+
+// #ifdef LEFT
+// 		off += 8; // number of values in A
+// #endif
+*/
+.macro REFRESH_AFTER_SAVE TEMP_BK, BK_VAL, OFF_VAL,PTR_B,PTR_A, C_A, C_B
+    #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	/*temp = bk - off;*/
+	sub \TEMP_BK, \BK_VAL, \OFF_VAL
+    #ifdef LEFT
+	/*temp -= 8; // number of values in A*/
+	addi \TEMP_BK, \TEMP_BK,-\C_A
+    #else
+	/*temp -= 4; // number of values in B*/
+	addi \TEMP_BK, \TEMP_BK,-\C_B
+    #endif
+	/*ptrba += temp*C_A;
+	ptrbb += temp*C_B;*/
+	SHIFT_REG T4, \TEMP_BK, \C_A
+	SHIFT_REG T2, \TEMP_BK, \C_B
+	add \PTR_A, \PTR_A, T4/*ptrba+temp*C_A*/
+	add \PTR_B, \PTR_B, T2
+    #endif
+    #ifdef LEFT
+	/*off += 8; // number of values in A*/
+	addi \OFF_VAL, \OFF_VAL, \C_A
+    #endif
+.endm
diff --git a/kernel/power/dgemm_kernel_power10.c b/kernel/power/dgemm_kernel_power10.c
new file mode 100644
index 000000000..b3ee301be
--- /dev/null
+++ b/kernel/power/dgemm_kernel_power10.c
@@ -0,0 +1,864 @@
+/*********************************************************************************
+Copyright (c) 2020, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+**********************************************************************************/
+#include "common.h"
+#include <altivec.h>
+
+typedef unsigned char vec_t __attribute__ ((vector_size (16)));
+typedef FLOAT v4sf_t __attribute__ ((vector_size (16)));
+typedef FLOAT v2sf_t __attribute__ ((vector_size (8)));
+
+#ifdef TRMMKERNEL
+#define SAVE_ACC(ACC, J)  \
+          __builtin_mma_disassemble_acc (result, ACC); \
+          rowC = (v4sf_t *) &CO[0* ldc+J]; \
+          rowC[0] = result[3] * alpha; \
+          rowC = (v4sf_t *) &CO[1*ldc+J]; \
+          rowC[0] = result[2] * alpha; \
+          rowC = (v4sf_t *) &CO[2*ldc+J]; \
+          rowC[0] = result[1] * alpha; \
+          rowC = (v4sf_t *) &CO[3*ldc+J]; \
+          rowC[0] = result[0] * alpha;
+#define SAVE_ACC1(ACC, J)  \
+          __builtin_mma_disassemble_acc (result, ACC); \
+          rowC = (v4sf_t *) &CO[4* ldc+J]; \
+          rowC[0] = result[3] * alpha; \
+          rowC = (v4sf_t *) &CO[5*ldc+J]; \
+          rowC[0] = result[2] * alpha; \
+          rowC = (v4sf_t *) &CO[6*ldc+J]; \
+          rowC[0] = result[1] * alpha; \
+          rowC = (v4sf_t *) &CO[7*ldc+J]; \
+          rowC[0] = result[0] * alpha;
+#define  SAVE2x4_ACC(ACC, J)  \
+	  __builtin_mma_disassemble_acc (result, ACC); \
+	  rowC = (v4sf_t *) &CO[0* ldc+J]; \
+          rowC[0] = result[3] * alpha; \
+	  rowC = (v4sf_t *) &CO[1* ldc+J]; \
+          rowC[0] = result[2] * alpha;
+#else
+#define SAVE_ACC(ACC, J)  \
+          __builtin_mma_disassemble_acc (result, ACC); \
+          rowC = (v4sf_t *) &CO[0* ldc+J]; \
+          rowC[0] += result[3] * alpha; \
+          rowC = (v4sf_t *) &CO[1*ldc+J]; \
+          rowC[0] += result[2] * alpha; \
+          rowC = (v4sf_t *) &CO[2*ldc+J]; \
+          rowC[0] += result[1] * alpha; \
+          rowC = (v4sf_t *) &CO[3*ldc+J]; \
+          rowC[0] += result[0] * alpha;
+#define SAVE_ACC1(ACC, J)  \
+          __builtin_mma_disassemble_acc (result, ACC); \
+          rowC = (v4sf_t *) &CO[4* ldc+J]; \
+          rowC[0] += result[3] * alpha; \
+          rowC = (v4sf_t *) &CO[5*ldc+J]; \
+          rowC[0] += result[2] * alpha; \
+          rowC = (v4sf_t *) &CO[6*ldc+J]; \
+          rowC[0] += result[1] * alpha; \
+          rowC = (v4sf_t *) &CO[7*ldc+J]; \
+          rowC[0] += result[0] * alpha;
+#define  SAVE2x4_ACC(ACC, J)  \
+	  __builtin_mma_disassemble_acc (result, ACC); \
+	  rowC = (v4sf_t *) &CO[0* ldc+J]; \
+          rowC[0] += result[3] * alpha; \
+	  rowC = (v4sf_t *) &CO[1* ldc+J]; \
+          rowC[0] += result[2] * alpha;
+#endif
+
+#define SET_ACC_ZERO4() \
+          __builtin_mma_xxsetaccz (&acc0); \
+          __builtin_mma_xxsetaccz (&acc1); \
+          __builtin_mma_xxsetaccz (&acc2); \
+          __builtin_mma_xxsetaccz (&acc3);
+
+#define SET_ACC_ZERO8() \
+          __builtin_mma_xxsetaccz (&acc0); \
+          __builtin_mma_xxsetaccz (&acc1); \
+          __builtin_mma_xxsetaccz (&acc2); \
+          __builtin_mma_xxsetaccz (&acc3); \
+          __builtin_mma_xxsetaccz (&acc4); \
+          __builtin_mma_xxsetaccz (&acc5); \
+          __builtin_mma_xxsetaccz (&acc6); \
+          __builtin_mma_xxsetaccz (&acc7);
+
+#define PREFETCH1(x, y) asm volatile ("dcbt %0, %1" : : "r" (x), "b" (y) : "memory");
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+#define REFRESH_TEMP_BK(x, y) \
+            temp = k - off;
+#elif defined(LEFT)
+#define REFRESH_TEMP_BK(x, y) \
+            temp = off + x;
+#else
+#define REFRESH_TEMP_BK(x, y) \
+            temp = off + y;
+#endif
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+#define REFRESH_POINTERS(x, y) \
+          BO = B; \
+          REFRESH_TEMP_BK(x, y)
+#else
+#define REFRESH_POINTERS(x, y) \
+          AO += off * x; \
+          BO = B + off * y; \
+          REFRESH_TEMP_BK(x, y)
+#endif
+
+#ifdef LEFT
+#define REFRESH_OFF(x) \
+            off += x;
+#else
+#define REFRESH_OFF(x)
+#endif
+
+#ifdef LEFT
+#define UPDATE_TEMP(x, y) \
+            temp -= x;
+#else
+#define UPDATE_TEMP(x, y) \
+            temp -= y;
+#endif
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+#define REFRESH_TMP_AFTER_SAVE(x, y) \
+            temp = k - off; \
+            UPDATE_TEMP(x, y) \
+            AO += temp * x; \
+            BO += temp * y;
+#else
+#define REFRESH_TMP_AFTER_SAVE(x, y)
+#endif
+
+#define REFRESH_AFTER_SAVE(x,y) \
+        REFRESH_TMP_AFTER_SAVE(x, y) \
+        REFRESH_OFF(x)
+/*************************************************************************************
+* GEMM Kernel
+*************************************************************************************/
+int
+CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
+       FLOAT * C, BLASLONG ldc
+#ifdef TRMMKERNEL
+       , BLASLONG offset
+#endif
+  )
+{
+  BLASLONG N = n;
+  BLASLONG i1;
+#if defined(TRMMKERNEL)
+  BLASLONG off;
+#endif
+#if defined(TRMMKERNEL) && !defined(LEFT)
+  off = -offset;
+#endif
+  v4sf_t valpha = { alpha, alpha };
+  N = n >> 2;
+  for (i1 = 0; i1 < N; i1++)
+    {
+      BLASLONG i, j, temp;
+      FLOAT *CO;
+      FLOAT *AO;
+#if defined(TRMMKERNEL) && defined(LEFT)
+      off = offset;
+#endif
+      CO = C;
+      C += ldc << 2;
+      AO = A;
+      PREFETCH1 (A, 128);
+      PREFETCH1 (A, 256);
+      i = m >> 4;
+      for (j = 0; j < i; j++)
+	{
+          FLOAT *BO;
+#if defined(TRMMKERNEL)
+          REFRESH_POINTERS (16, 4);
+#else
+          BO = B;
+          temp = k;
+#endif
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  BLASLONG l = 0;
+	  PREFETCH1 (CO, 0);
+	  PREFETCH1 (CO + ldc, 0);
+	  PREFETCH1 (CO + ldc + ldc, 0);
+	  PREFETCH1 (CO + ldc + ldc + ldc, 0);
+	  PREFETCH1 (CO, 128);
+	  PREFETCH1 (CO + ldc, 128);
+	  PREFETCH1 (CO + ldc + ldc, 128);
+	  PREFETCH1 (CO + ldc + ldc + ldc, 128);
+	  __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7;
+	  SET_ACC_ZERO8 ();
+	  for (l = 0; l < temp; l++)
+	    {
+	      vec_t *rowA = (vec_t *) & AO[l << 4];
+	      __vector_pair rowB;
+	      vec_t *rb = (vec_t *) & BO[l << 2];
+	      __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
+	      __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);
+	      __builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]);
+	      __builtin_mma_xvf64gerpp (&acc2, rowB, rowA[2]);
+	      __builtin_mma_xvf64gerpp (&acc3, rowB, rowA[3]);
+	      __builtin_mma_xvf64gerpp (&acc4, rowB, rowA[4]);
+	      __builtin_mma_xvf64gerpp (&acc5, rowB, rowA[5]);
+	      __builtin_mma_xvf64gerpp (&acc6, rowB, rowA[6]);
+	      __builtin_mma_xvf64gerpp (&acc7, rowB, rowA[7]);
+	    }
+	  SAVE_ACC (&acc0, 0);
+	  SAVE_ACC (&acc2, 4);
+	  SAVE_ACC (&acc1, 2);
+	  SAVE_ACC (&acc3, 6);
+	  SAVE_ACC (&acc4, 8);
+	  SAVE_ACC (&acc6, 12);
+	  SAVE_ACC (&acc5, 10);
+	  SAVE_ACC (&acc7, 14);
+	  AO += temp << 4;
+	  BO += temp << 2;
+#if defined(TRMMKERNEL)
+          REFRESH_AFTER_SAVE (16, 4)
+#endif
+	  CO += 16;
+	}
+      i = (m & 15) >> 3;
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO;
+#if defined(TRMMKERNEL)
+          REFRESH_POINTERS (8, 4);
+#else
+          BO = B;
+          temp = k;
+#endif
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  __vector_quad acc0, acc1, acc2, acc3;
+	  SET_ACC_ZERO4 ();
+	  BLASLONG l = 0;
+	  for (l = 0; l < temp; l++)
+	    {
+	      vec_t *rowA = (vec_t *) & AO[l << 3];
+	      __vector_pair rowB;
+	      vec_t *rb = (vec_t *) & BO[l << 2];
+	      __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
+	      __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);
+	      __builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]);
+	      __builtin_mma_xvf64gerpp (&acc2, rowB, rowA[2]);
+	      __builtin_mma_xvf64gerpp (&acc3, rowB, rowA[3]);
+	    }
+	  SAVE_ACC (&acc0, 0);
+	  SAVE_ACC (&acc2, 4);
+	  SAVE_ACC (&acc1, 2);
+	  SAVE_ACC (&acc3, 6);
+	  CO += 8;
+	  AO += temp << 3;
+	  BO += temp << 2;
+#if defined(TRMMKERNEL)
+          REFRESH_AFTER_SAVE (8, 4)
+#endif
+	}
+      i = (m & 7) >> 2;
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO;
+#if defined(TRMMKERNEL)
+          REFRESH_POINTERS (4, 4);
+#else
+          BO = B;
+          temp = k;
+#endif
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  __vector_quad acc0, acc1;
+	  __builtin_mma_xxsetaccz (&acc0);
+	  __builtin_mma_xxsetaccz (&acc1);
+	  BLASLONG l = 0;
+	  for (l = 0; l < temp; l++)
+	    {
+	      vec_t *rowA = (vec_t *) & AO[l << 2];
+	      __vector_pair rowB;
+	      vec_t *rb = (vec_t *) & BO[l << 2];
+	      __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
+	      __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);
+	      __builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]);
+	    }
+	  SAVE_ACC (&acc0, 0);
+	  SAVE_ACC (&acc1, 2);
+	  CO += 4;
+	  AO += temp << 2;
+	  BO += temp << 2;
+#if defined(TRMMKERNEL)
+          REFRESH_AFTER_SAVE (4, 4)
+#endif
+	}
+      i = (m & 3) >> 1;
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO;
+#if defined(TRMMKERNEL)
+          REFRESH_POINTERS (2, 4);
+#else
+          BO = B;
+          temp = k;
+#endif
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  __vector_quad acc0;
+	  __builtin_mma_xxsetaccz (&acc0);
+	  BLASLONG l = 0;
+	  for (l = 0; l < temp; l++)
+	    {
+	      vec_t *rowA = (vec_t *) & AO[l << 1];
+	      __vector_pair rowB;
+	      vec_t *rb = (vec_t *) & BO[l << 2];
+	      __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
+	      __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);
+	    }
+	  SAVE_ACC (&acc0, 0);
+	  CO += 2;
+	  AO += temp << 1;
+	  BO += temp << 2;
+#if defined(TRMMKERNEL)
+          REFRESH_AFTER_SAVE (2, 4)
+#endif
+	}
+      i = (m & 1) >> 0;
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO;
+#if defined(TRMMKERNEL)
+          REFRESH_POINTERS (1, 4);
+#else
+          BO = B;
+          temp = k;
+#endif
+	  BLASLONG l = 0;
+	  v4sf_t t = { 0, 0 };
+	  v4sf_t t1 = { 0, 0 };
+	  for (l = 0; l < temp; l++)
+	    {
+	      v4sf_t rowA = { AO[l], AO[l] };
+	      v4sf_t rowB = { BO[l << 2], BO[(l << 2) + 1] };
+	      v4sf_t rowB1 = { BO[(l << 2) + 2], BO[(l << 2) + 3] };
+	      t += rowA * rowB;
+	      t1 += rowA * rowB1;
+	    }
+	  t = t * valpha;
+	  t1 = t1 * valpha;
+#if defined(TRMMKERNEL)
+	  CO[0 * ldc] = t[0];
+	  CO[1 * ldc] = t[1];
+	  CO[2 * ldc] = t1[0];
+	  CO[3 * ldc] = t1[1];
+#else
+	  CO[0 * ldc] += t[0];
+	  CO[1 * ldc] += t[1];
+	  CO[2 * ldc] += t1[0];
+	  CO[3 * ldc] += t1[1];
+#endif
+	  CO += 1;
+	  AO += temp;
+	  BO += temp << 2;
+#if defined(TRMMKERNEL)
+          REFRESH_AFTER_SAVE (1, 4)
+#endif
+	}
+#if defined(TRMMKERNEL) && !defined(LEFT)
+      off += 4;                 // number of values in A
+#endif
+      B += k << 2;
+    }
+  N = (n & 3) >> 1;
+  for (i1 = 0; i1 < N; i1++)
+    {
+      BLASLONG i, j, temp;
+#if defined(TRMMKERNEL) && defined(LEFT)
+      off = offset;
+#endif
+      FLOAT *CO;
+      FLOAT *AO;
+      CO = C;
+      C += ldc << 1;
+      AO = A;
+      i = m >> 4;
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO;
+#if defined(TRMMKERNEL)
+          REFRESH_POINTERS (16, 2);
+#else
+          BO = B;
+          temp = k;
+#endif
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7;
+	  SET_ACC_ZERO8 ();
+	  BLASLONG l = 0;
+	  for (l = 0; l < temp; l++)
+	    {
+	      FLOAT t[4] = { 0, 0, 0, 0 };
+	      t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];
+	      __vector_pair rowB;
+	      vec_t *rb = (vec_t *) & t[0];
+	      __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
+	      vec_t *rowA = (vec_t *) & AO[l << 4];
+	      __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);
+	      __builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]);
+	      __builtin_mma_xvf64gerpp (&acc2, rowB, rowA[2]);
+	      __builtin_mma_xvf64gerpp (&acc3, rowB, rowA[3]);
+	      __builtin_mma_xvf64gerpp (&acc4, rowB, rowA[4]);
+	      __builtin_mma_xvf64gerpp (&acc5, rowB, rowA[5]);
+	      __builtin_mma_xvf64gerpp (&acc6, rowB, rowA[6]);
+	      __builtin_mma_xvf64gerpp (&acc7, rowB, rowA[7]);
+	    }
+	  SAVE2x4_ACC (&acc0, 0);
+	  SAVE2x4_ACC (&acc1, 2);
+	  SAVE2x4_ACC (&acc2, 4);
+	  SAVE2x4_ACC (&acc3, 6);
+	  SAVE2x4_ACC (&acc4, 8);
+	  SAVE2x4_ACC (&acc5, 10);
+	  SAVE2x4_ACC (&acc6, 12);
+	  SAVE2x4_ACC (&acc7, 14);
+	  CO += 16;
+	  AO += temp << 4;
+	  BO += temp << 1;
+#if defined(TRMMKERNEL)
+          REFRESH_AFTER_SAVE (16, 2)
+#endif
+	}
+      i = (m & 15) >> 3;
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO;
+#if defined(TRMMKERNEL)
+          REFRESH_POINTERS (8, 2);
+#else
+          BO = B;
+          temp = k;
+#endif
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  __vector_quad acc0, acc1, acc2, acc3;
+	  SET_ACC_ZERO4 ();
+	  BLASLONG l = 0;
+	  for (l = 0; l < temp; l++)
+	    {
+	      FLOAT t[4] = { 0, 0, 0, 0 };
+	      t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];
+	      __vector_pair rowB;
+	      vec_t *rb = (vec_t *) & t[0];
+	      __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
+	      vec_t *rowA = (vec_t *) & AO[l << 3];
+	      __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);
+	      __builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]);
+	      __builtin_mma_xvf64gerpp (&acc2, rowB, rowA[2]);
+	      __builtin_mma_xvf64gerpp (&acc3, rowB, rowA[3]);
+	    }
+	  SAVE2x4_ACC (&acc0, 0);
+	  SAVE2x4_ACC (&acc1, 2);
+	  SAVE2x4_ACC (&acc2, 4);
+	  SAVE2x4_ACC (&acc3, 6);
+	  CO += 8;
+	  AO += temp << 3;
+	  BO += temp << 1;
+#if defined(TRMMKERNEL)
+          REFRESH_AFTER_SAVE (8, 2)
+#endif
+	}
+      i = (m & 7) >> 2;
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO;
+#if defined(TRMMKERNEL)
+          REFRESH_POINTERS (4, 2);
+#else
+          BO = B;
+          temp = k;
+#endif
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  __vector_quad acc0, acc1;
+	  __builtin_mma_xxsetaccz (&acc0);
+	  __builtin_mma_xxsetaccz (&acc1);
+	  BLASLONG l = 0;
+	  for (l = 0; l < temp; l++)
+	    {
+	      FLOAT t[4] = { 0, 0, 0, 0 };
+	      t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];
+	      __vector_pair rowB;
+	      vec_t *rb = (vec_t *) & t[0];
+	      __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
+	      vec_t *rowA = (vec_t *) & AO[l << 2];
+	      __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);
+	      __builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]);
+	    }
+	  SAVE2x4_ACC (&acc0, 0);
+	  SAVE2x4_ACC (&acc1, 2);
+	  CO += 4;
+	  AO += temp << 2;
+	  BO += temp << 1;
+#if defined(TRMMKERNEL)
+          REFRESH_AFTER_SAVE (4, 2)
+#endif
+	}
+      i = (m & 3) >> 1;
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO;
+#if defined(TRMMKERNEL)
+          REFRESH_POINTERS (2, 2);
+#else
+          BO = B;
+          temp = k;
+#endif
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  __vector_quad acc0;
+	  __builtin_mma_xxsetaccz (&acc0);
+	  BLASLONG l = 0;
+	  for (l = 0; l < temp; l++)
+	    {
+	      FLOAT t[4] = { 0, 0, 0, 0 };
+	      t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];
+	      __vector_pair rowB;
+	      vec_t *rb = (vec_t *) & t[0];
+	      __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
+	      vec_t *rowA = (vec_t *) & AO[l << 1];
+	      __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);
+	    }
+	  SAVE2x4_ACC (&acc0, 0);
+	  CO += 2;
+	  AO += temp << 1;
+	  BO += temp << 1;
+#if defined(TRMMKERNEL)
+          REFRESH_AFTER_SAVE (2, 2)
+#endif
+	}
+      i = (m & 1) >> 0;
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO;
+#if defined(TRMMKERNEL)
+          REFRESH_POINTERS (1, 2);
+#else
+          BO = B;
+          temp = k;
+#endif
+	  BLASLONG l = 0;
+	  v4sf_t t = { 0, 0 };
+	  for (l = 0; l < temp; l++)
+	    {
+	      v4sf_t rowA = { AO[l], AO[l] };
+	      v4sf_t rowB = { BO[l << 1], BO[(l << 1) + 1] };
+	      t += rowA * rowB;
+	    }
+	  t = t * valpha;
+#if defined(TRMMKERNEL)
+	  CO[0 * ldc] = t[0];
+	  CO[1 * ldc] = t[1];
+#else
+	  CO[0 * ldc] += t[0];
+	  CO[1 * ldc] += t[1];
+#endif
+	  CO += 1;
+	  AO += temp;
+	  BO += temp << 1;
+#if defined(TRMMKERNEL)
+          REFRESH_AFTER_SAVE (1, 2)
+#endif
+	}
+#if defined(TRMMKERNEL) && !defined(LEFT)
+      off += 2;                 // number of values in A
+#endif
+      B += k << 1;
+    }
+  N = (n & 1) >> 0;
+  for (i1 = 0; i1 < N; i1++)
+    {
+      BLASLONG i, temp;
+#if defined(TRMMKERNEL) && defined(LEFT)
+      off = offset;
+#endif
+      FLOAT *CO;
+      FLOAT *AO;
+      CO = C;
+      C += ldc;
+      AO = A;
+      i = m;
+      while (i >= 16)
+	{
+	  FLOAT *BO;
+#if defined(TRMMKERNEL)
+          REFRESH_POINTERS (16, 1)
+#else
+          BO = B;
+          temp = k;
+#endif
+	  BLASLONG l = 0;
+	  v4sf_t t = { 0, 0 };
+	  v4sf_t t1 = { 0, 0 };
+	  v4sf_t t2 = { 0, 0 };
+	  v4sf_t t3 = { 0, 0 };
+	  v4sf_t t4 = { 0, 0 };
+	  v4sf_t t5 = { 0, 0 };
+	  v4sf_t t6 = { 0, 0 };
+	  v4sf_t t7 = { 0, 0 };
+	  for (l = 0; l < temp; l++)
+	    {
+	      v4sf_t rowB = { BO[l], BO[l] };
+	      v4sf_t rowA = { AO[l << 4], AO[(l << 4) + 1] };
+	      v4sf_t rowA1 = { AO[(l << 4) + 2], AO[(l << 4) + 3] };
+	      v4sf_t rowA2 = { AO[(l << 4) + 4], AO[(l << 4) + 5] };
+	      v4sf_t rowA3 = { AO[(l << 4) + 6], AO[(l << 4) + 7] };
+	      v4sf_t rowA4 = { AO[(l << 4) + 8], AO[(l << 4) + 9] };
+	      v4sf_t rowA5 = { AO[(l << 4) + 10], AO[(l << 4) + 11] };
+	      v4sf_t rowA6 = { AO[(l << 4) + 12], AO[(l << 4) + 13] };
+	      v4sf_t rowA7 = { AO[(l << 4) + 14], AO[(l << 4) + 15] };
+	      t += rowA * rowB;
+	      t1 += rowA1 * rowB;
+	      t2 += rowA2 * rowB;
+	      t3 += rowA3 * rowB;
+	      t4 += rowA4 * rowB;
+	      t5 += rowA5 * rowB;
+	      t6 += rowA6 * rowB;
+	      t7 += rowA7 * rowB;
+	    }
+	  t = t * valpha;
+	  t1 = t1 * valpha;
+	  t2 = t2 * valpha;
+	  t3 = t3 * valpha;
+	  t4 = t4 * valpha;
+	  t5 = t5 * valpha;
+	  t6 = t6 * valpha;
+	  t7 = t7 * valpha;
+#if defined(TRMMKERNEL)
+	  CO[0] = t[0];
+	  CO[1] = t[1];
+	  CO[2] = t1[0];
+	  CO[3] = t1[1];
+	  CO[4] = t2[0];
+	  CO[5] = t2[1];
+	  CO[6] = t3[0];
+	  CO[7] = t3[1];
+	  CO[8] = t4[0];
+	  CO[9] = t4[1];
+	  CO[10] = t5[0];
+	  CO[11] = t5[1];
+	  CO[12] = t6[0];
+	  CO[13] = t6[1];
+	  CO[14] = t7[0];
+	  CO[15] = t7[1];
+#else
+	  CO[0] += t[0];
+	  CO[1] += t[1];
+	  CO[2] += t1[0];
+	  CO[3] += t1[1];
+	  CO[4] += t2[0];
+	  CO[5] += t2[1];
+	  CO[6] += t3[0];
+	  CO[7] += t3[1];
+	  CO[8] += t4[0];
+	  CO[9] += t4[1];
+	  CO[10] += t5[0];
+	  CO[11] += t5[1];
+	  CO[12] += t6[0];
+	  CO[13] += t6[1];
+	  CO[14] += t7[0];
+	  CO[15] += t7[1];
+#endif
+	  AO += temp << 4;
+	  BO += temp;
+	  CO += 16;
+	  i -= 16;
+#if defined(TRMMKERNEL)
+          REFRESH_AFTER_SAVE (16, 1)
+#endif
+	}
+      while (i >= 8)
+	{
+	  FLOAT *BO;
+#if defined(TRMMKERNEL)
+          REFRESH_POINTERS (8, 1)
+#else
+          BO = B;
+          temp = k;
+#endif
+	  BLASLONG l = 0;
+	  v4sf_t t = { 0, 0 };
+	  v4sf_t t1 = { 0, 0 };
+	  v4sf_t t2 = { 0, 0 };
+	  v4sf_t t3 = { 0, 0 };
+	  for (l = 0; l < temp; l++)
+	    {
+	      v4sf_t rowB = { BO[l], BO[l] };
+	      v4sf_t rowA = { AO[l << 3], AO[(l << 3) + 1] };
+	      v4sf_t rowA1 = { AO[(l << 3) + 2], AO[(l << 3) + 3] };
+	      v4sf_t rowA2 = { AO[(l << 3) + 4], AO[(l << 3) + 5] };
+	      v4sf_t rowA3 = { AO[(l << 3) + 6], AO[(l << 3) + 7] };
+	      t += rowA * rowB;
+	      t1 += rowA1 * rowB;
+	      t2 += rowA2 * rowB;
+	      t3 += rowA3 * rowB;
+	    }
+	  t = t * valpha;
+	  t1 = t1 * valpha;
+	  t2 = t2 * valpha;
+	  t3 = t3 * valpha;
+#if defined(TRMMKERNEL)
+	  CO[0] = t[0];
+	  CO[1] = t[1];
+	  CO[2] = t1[0];
+	  CO[3] = t1[1];
+	  CO[4] = t2[0];
+	  CO[5] = t2[1];
+	  CO[6] = t3[0];
+	  CO[7] = t3[1];
+#else
+	  CO[0] += t[0];
+	  CO[1] += t[1];
+	  CO[2] += t1[0];
+	  CO[3] += t1[1];
+	  CO[4] += t2[0];
+	  CO[5] += t2[1];
+	  CO[6] += t3[0];
+	  CO[7] += t3[1];
+#endif
+	  AO += temp << 3;
+	  BO += temp;
+	  CO += 8;
+	  i -= 8;
+#if defined(TRMMKERNEL)
+          REFRESH_AFTER_SAVE (8, 1)
+#endif
+	}
+      while (i >= 4)
+	{
+	  FLOAT *BO;
+#if defined(TRMMKERNEL)
+          REFRESH_POINTERS (4, 1)
+#else
+          BO = B;
+          temp = k;
+#endif
+	  BLASLONG l = 0;
+	  v4sf_t t = { 0, 0 };
+	  v4sf_t t1 = { 0, 0 };
+	  for (l = 0; l < temp; l++)
+	    {
+	      v4sf_t rowB = { BO[l], BO[l] };
+	      v4sf_t rowA = { AO[l << 2], AO[(l << 2) + 1] };
+	      v4sf_t rowA1 = { AO[(l << 2) + 2], AO[(l << 2) + 3] };
+	      t += rowA * rowB;
+	      t1 += rowA1 * rowB;
+	    }
+	  t = t * valpha;
+	  t1 = t1 * valpha;
+#if defined(TRMMKERNEL)
+	  CO[0] = t[0];
+	  CO[1] = t[1];
+	  CO[2] = t1[0];
+	  CO[3] = t1[1];
+#else
+	  CO[0] += t[0];
+	  CO[1] += t[1];
+	  CO[2] += t1[0];
+	  CO[3] += t1[1];
+#endif
+	  AO += temp << 2;
+	  BO += temp;
+	  CO += 4;
+	  i -= 4;
+#if defined(TRMMKERNEL)
+          REFRESH_AFTER_SAVE (4, 1)
+#endif
+	}
+      while (i >= 2)
+	{
+	  FLOAT *BO;
+#if defined(TRMMKERNEL)
+          REFRESH_POINTERS (2, 1)
+#else
+          BO = B;
+          temp = k;
+#endif
+	  BLASLONG l = 0;
+	  v4sf_t t = { 0, 0 };
+	  for (l = 0; l < temp; l++)
+	    {
+	      v4sf_t rowB = { BO[l], BO[l] };
+	      v4sf_t rowA = { AO[l << 1], AO[(l << 1) + 1] };
+	      t += rowA * rowB;
+	    }
+	  t = t * valpha;
+#if defined(TRMMKERNEL)
+	  CO[0] = t[0];
+	  CO[1] = t[1];
+#else
+	  CO[0] += t[0];
+	  CO[1] += t[1];
+#endif
+	  AO += temp << 1;
+	  BO += temp;
+	  CO += 2;
+	  i -= 2;
+#if defined(TRMMKERNEL)
+          REFRESH_AFTER_SAVE (2, 1)
+#endif
+	}
+      while (i >= 1)
+	{
+	  FLOAT *BO;
+#if defined(TRMMKERNEL)
+          REFRESH_POINTERS (1, 1)
+#else
+          BO = B;
+          temp = k;
+#endif
+	  BLASLONG l = 0;
+	  FLOAT t = 0;
+	  for (l = 0; l < temp; l++)
+	    {
+	      t += AO[l] * BO[l];
+	    }
+	  AO += temp;
+	  BO += temp;
+#if defined(TRMMKERNEL)
+	  CO[0] = t * alpha;
+#else
+	  CO[0] += t * alpha;
+#endif
+	  CO += 1;
+	  i -= 1;
+#if defined(TRMMKERNEL)
+          REFRESH_AFTER_SAVE (1, 1)
+#endif
+	}
+#if defined(TRMMKERNEL) && !defined(LEFT)
+      off += 1;                 // number of values in A
+#endif
+      B += k;
+    }
+  return 0;
+}
diff --git a/kernel/power/sgemm_kernel_power10.c b/kernel/power/sgemm_kernel_power10.c
new file mode 100644
index 000000000..01c122c6d
--- /dev/null
+++ b/kernel/power/sgemm_kernel_power10.c
@@ -0,0 +1,1334 @@
+/*********************************************************************************
+Copyright (c) 2020, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+**********************************************************************************/
+#include "common.h"
+#include <altivec.h>
+
+typedef unsigned char vec_t __attribute__ ((vector_size (16)));
+typedef FLOAT v4sf_t __attribute__ ((vector_size (16)));
+typedef FLOAT v2sf_t __attribute__ ((vector_size (8)));
+#if defined(TRMMKERNEL)
+#define SAVE_ACC(ACC, J)  \
+	  __builtin_mma_disassemble_acc (result, ACC); \
+	  rowC = (v4sf_t *) &CO[0* ldc+J]; \
+          rowC[0] = result[3] * alpha; \
+          rowC = (v4sf_t *) &CO[1*ldc+J]; \
+          rowC[0] = result[2] * alpha; \
+          rowC = (v4sf_t *) &CO[2*ldc+J]; \
+          rowC[0] = result[1] * alpha; \
+          rowC = (v4sf_t *) &CO[3*ldc+J]; \
+          rowC[0] = result[0] * alpha;
+#define SAVE_ACC1(ACC, J)  \
+	  __builtin_mma_disassemble_acc (result, ACC); \
+	  rowC = (v4sf_t *) &CO[4* ldc+J]; \
+          rowC[0] = result[3] * alpha; \
+          rowC = (v4sf_t *) &CO[5*ldc+J]; \
+          rowC[0] = result[2] * alpha; \
+          rowC = (v4sf_t *) &CO[6*ldc+J]; \
+          rowC[0] = result[1] * alpha; \
+          rowC = (v4sf_t *) &CO[7*ldc+J]; \
+          rowC[0] = result[0] * alpha;
+#define  SAVE4x2_ACC(ACC, J)  \
+	  __builtin_mma_disassemble_acc (result, ACC); \
+	  rowC = (v2sf_t *) &CO[0* ldc+J]; \
+          rowC[0] = result[6] * alpha; \
+	  rowC = (v2sf_t *) &CO[1* ldc+J]; \
+          rowC[0] = result[4] * alpha; \
+	  rowC = (v2sf_t *) &CO[2* ldc+J]; \
+          rowC[0] = result[2] * alpha; \
+	  rowC = (v2sf_t *) &CO[3* ldc+J]; \
+          rowC[0] = result[0] * alpha;
+#define  SAVE4x2_ACC1(ACC, J)  \
+	  __builtin_mma_disassemble_acc (result, ACC); \
+	  rowC = (v2sf_t *) &CO[4* ldc+J]; \
+          rowC[0] = result[6] * alpha; \
+	  rowC = (v2sf_t *) &CO[5* ldc+J]; \
+          rowC[0] = result[4] * alpha; \
+	  rowC = (v2sf_t *) &CO[6* ldc+J]; \
+          rowC[0] = result[2] * alpha; \
+	  rowC = (v2sf_t *) &CO[7* ldc+J]; \
+          rowC[0] = result[0] * alpha;
+#define  SAVE2x4_ACC(ACC, J)  \
+	  __builtin_mma_disassemble_acc (result, ACC); \
+	  rowC = (v4sf_t *) &CO[0* ldc+J]; \
+          rowC[0] = result[3] * alpha; \
+	  rowC = (v4sf_t *) &CO[1* ldc+J]; \
+          rowC[0] = result[2] * alpha;
+#else
+#define SAVE_ACC(ACC, J)  \
+	  __builtin_mma_disassemble_acc (result, ACC); \
+	  rowC = (v4sf_t *) &CO[0* ldc+J]; \
+          rowC[0] += result[3] * alpha; \
+          rowC = (v4sf_t *) &CO[1*ldc+J]; \
+          rowC[0] += result[2] * alpha; \
+          rowC = (v4sf_t *) &CO[2*ldc+J]; \
+          rowC[0] += result[1] * alpha; \
+          rowC = (v4sf_t *) &CO[3*ldc+J]; \
+          rowC[0] += result[0] * alpha;
+#define SAVE_ACC1(ACC, J)  \
+	  __builtin_mma_disassemble_acc (result, ACC); \
+	  rowC = (v4sf_t *) &CO[4* ldc+J]; \
+          rowC[0] += result[3] * alpha; \
+          rowC = (v4sf_t *) &CO[5*ldc+J]; \
+          rowC[0] += result[2] * alpha; \
+          rowC = (v4sf_t *) &CO[6*ldc+J]; \
+          rowC[0] += result[1] * alpha; \
+          rowC = (v4sf_t *) &CO[7*ldc+J]; \
+          rowC[0] += result[0] * alpha;
+#define  SAVE4x2_ACC(ACC, J)  \
+	  __builtin_mma_disassemble_acc (result, ACC); \
+	  rowC = (v2sf_t *) &CO[0* ldc+J]; \
+          rowC[0] += result[6] * alpha; \
+	  rowC = (v2sf_t *) &CO[1* ldc+J]; \
+          rowC[0] += result[4] * alpha; \
+	  rowC = (v2sf_t *) &CO[2* ldc+J]; \
+          rowC[0] += result[2] * alpha; \
+	  rowC = (v2sf_t *) &CO[3* ldc+J]; \
+          rowC[0] += result[0] * alpha;
+#define  SAVE4x2_ACC1(ACC, J)  \
+	  __builtin_mma_disassemble_acc (result, ACC); \
+	  rowC = (v2sf_t *) &CO[4* ldc+J]; \
+          rowC[0] += result[6] * alpha; \
+	  rowC = (v2sf_t *) &CO[5* ldc+J]; \
+          rowC[0] += result[4] * alpha; \
+	  rowC = (v2sf_t *) &CO[6* ldc+J]; \
+          rowC[0] += result[2] * alpha; \
+	  rowC = (v2sf_t *) &CO[7* ldc+J]; \
+          rowC[0] += result[0] * alpha;
+#define  SAVE2x4_ACC(ACC, J)  \
+	  __builtin_mma_disassemble_acc (result, ACC); \
+	  rowC = (v4sf_t *) &CO[0* ldc+J]; \
+          rowC[0] += result[3] * alpha; \
+	  rowC = (v4sf_t *) &CO[1* ldc+J]; \
+          rowC[0] += result[2] * alpha;
+#endif
+#define KERNEL(i, j) \
+          __builtin_mma_xvf32gerpp (&acc0, rowB[i], rowA[j]); \
+          __builtin_mma_xvf32gerpp (&acc1, rowB[i+1], rowA[j]); \
+          __builtin_mma_xvf32gerpp (&acc2, rowB[i], rowA[j+1]); \
+          __builtin_mma_xvf32gerpp (&acc3, rowB[i+1], rowA[j+1]); \
+          __builtin_mma_xvf32gerpp (&acc4, rowB[i], rowA[j+2]); \
+          __builtin_mma_xvf32gerpp (&acc5, rowB[i+1], rowA[j+2]); \
+          __builtin_mma_xvf32gerpp (&acc6, rowB[i], rowA[j+3]); \
+          __builtin_mma_xvf32gerpp (&acc7, rowB[i+1], rowA[j+3]);
+#define SET_ACC_ZERO4() \
+          __builtin_mma_xxsetaccz (&acc0); \
+          __builtin_mma_xxsetaccz (&acc1); \
+          __builtin_mma_xxsetaccz (&acc2); \
+          __builtin_mma_xxsetaccz (&acc3);
+
+#define SET_ACC_ZERO8() \
+          __builtin_mma_xxsetaccz (&acc0); \
+          __builtin_mma_xxsetaccz (&acc1); \
+          __builtin_mma_xxsetaccz (&acc2); \
+          __builtin_mma_xxsetaccz (&acc3); \
+          __builtin_mma_xxsetaccz (&acc4); \
+          __builtin_mma_xxsetaccz (&acc5); \
+          __builtin_mma_xxsetaccz (&acc6); \
+          __builtin_mma_xxsetaccz (&acc7);
+
+#define PREFETCH1(x, y) asm volatile ("dcbt %0, %1" : : "r" (x), "b" (y) : "memory");
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+#define REFRESH_TEMP_BK(x, y) \
+            temp = k - off;
+#elif defined(LEFT)
+#define REFRESH_TEMP_BK(x, y) \
+            temp = off + x;
+#else
+#define REFRESH_TEMP_BK(x, y) \
+            temp = off + y;
+#endif
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+#define REFRESH_POINTERS(x, y) \
+	  BO = B; \
+          REFRESH_TEMP_BK(x, y)
+#else
+#define REFRESH_POINTERS(x, y) \
+          AO += off * x; \
+          BO = B + off * y; \
+          REFRESH_TEMP_BK(x, y)
+#endif
+
+#ifdef LEFT
+#define REFRESH_OFF(x) \
+            off += x;
+#else
+#define REFRESH_OFF(x)
+#endif
+
+#ifdef LEFT
+#define UPDATE_TEMP(x, y) \
+            temp -= x;
+#else
+#define UPDATE_TEMP(x, y) \
+            temp -= y;
+#endif
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+#define REFRESH_TMP_AFTER_SAVE(x, y) \
+            temp = k - off; \
+            UPDATE_TEMP(x, y) \
+            AO += temp * x; \
+            BO += temp * y;
+#else
+#define REFRESH_TMP_AFTER_SAVE(x, y)
+#endif
+
+#define REFRESH_AFTER_SAVE(x,y) \
+        REFRESH_TMP_AFTER_SAVE(x, y) \
+	REFRESH_OFF(x)
+/*************************************************************************************
+* GEMM Kernel
+*************************************************************************************/
+int
+CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
+       FLOAT * C, BLASLONG ldc
+#ifdef TRMMKERNEL
+       , BLASLONG offset
+#endif
+  )
+{
+  BLASLONG N = n;
+  BLASLONG i1;
+#if defined(TRMMKERNEL)
+  BLASLONG off;
+#endif
+#if defined(TRMMKERNEL) && !defined(LEFT)
+  off = -offset;
+#endif
+
+  v4sf_t valpha = { alpha, alpha, alpha, alpha };
+  N = n >> 3;
+  for (i1 = 0; i1 < N; i1++)
+    {
+      BLASLONG i, j, temp;
+      FLOAT *CO;
+      FLOAT *AO;
+#if defined(TRMMKERNEL) && defined(LEFT)
+      off = offset;
+#endif
+      CO = C;
+      C += ldc << 3;
+      AO = A;
+      PREFETCH1 (A, 128);
+      PREFETCH1 (A, 256);
+      i = m >> 4;
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO;
+#if defined(TRMMKERNEL)
+	  REFRESH_POINTERS (16, 8);
+#else
+	  BO = B;
+	  temp = k;
+#endif
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7;
+	  SET_ACC_ZERO8 ();
+	  BLASLONG l = 0;
+	  BLASLONG K = temp / 64;
+	  for (l = 0; l < K; l++)
+	    {
+	      vec_t *rowA = (vec_t *) & AO[0];
+	      vec_t *rowB = (vec_t *) & BO[0];
+	      KERNEL (0, 0);
+	      KERNEL (2, 4);
+	      KERNEL (4, 8);
+	      KERNEL (6, 12);
+	      KERNEL (8, 16);
+	      KERNEL (10, 20);
+	      KERNEL (12, 24);
+	      KERNEL (14, 28);
+	      KERNEL (16, 32);
+	      KERNEL (18, 36);
+	      KERNEL (20, 40);
+	      KERNEL (22, 44);
+	      KERNEL (24, 48);
+	      KERNEL (26, 52);
+	      KERNEL (28, 56);
+	      KERNEL (30, 60);
+	      KERNEL (32, 64);
+	      KERNEL (34, 68);
+	      KERNEL (36, 72);
+	      KERNEL (38, 76);
+	      KERNEL (40, 80);
+	      KERNEL (42, 84);
+	      KERNEL (44, 88);
+	      KERNEL (46, 92);
+	      KERNEL (48, 96);
+	      KERNEL (50, 100);
+	      KERNEL (52, 104);
+	      KERNEL (54, 108);
+	      KERNEL (56, 112);
+	      KERNEL (58, 116);
+	      KERNEL (60, 120);
+	      KERNEL (62, 124);
+	      KERNEL (64, 128);
+	      KERNEL (66, 132);
+	      KERNEL (68, 136);
+	      KERNEL (70, 140);
+	      KERNEL (72, 144);
+	      KERNEL (74, 148);
+	      KERNEL (76, 152);
+	      KERNEL (78, 156);
+	      KERNEL (80, 160);
+	      KERNEL (82, 164);
+	      KERNEL (84, 168);
+	      KERNEL (86, 172);
+	      KERNEL (88, 176);
+	      KERNEL (90, 180);
+	      KERNEL (92, 184);
+	      KERNEL (94, 188);
+	      KERNEL (96, 192);
+	      KERNEL (98, 196);
+	      KERNEL (100, 200);
+	      KERNEL (102, 204);
+	      KERNEL (104, 208);
+	      KERNEL (106, 212);
+	      KERNEL (108, 216);
+	      KERNEL (110, 220);
+	      KERNEL (112, 224);
+	      KERNEL (114, 228);
+	      KERNEL (116, 232);
+	      KERNEL (118, 236);
+	      KERNEL (120, 240);
+	      KERNEL (122, 244);
+	      KERNEL (124, 248);
+	      KERNEL (126, 252);
+	      AO += 1024;
+	      BO += 512;
+	    }
+	  if ((temp & 63) >> 5)
+	    {
+	      vec_t *rowA = (vec_t *) & AO[0];
+	      vec_t *rowB = (vec_t *) & BO[0];
+	      KERNEL (0, 0);
+	      KERNEL (2, 4);
+	      KERNEL (4, 8);
+	      KERNEL (6, 12);
+	      KERNEL (8, 16);
+	      KERNEL (10, 20);
+	      KERNEL (12, 24);
+	      KERNEL (14, 28);
+	      KERNEL (16, 32);
+	      KERNEL (18, 36);
+	      KERNEL (20, 40);
+	      KERNEL (22, 44);
+	      KERNEL (24, 48);
+	      KERNEL (26, 52);
+	      KERNEL (28, 56);
+	      KERNEL (30, 60);
+	      KERNEL (32, 64);
+	      KERNEL (34, 68);
+	      KERNEL (36, 72);
+	      KERNEL (38, 76);
+	      KERNEL (40, 80);
+	      KERNEL (42, 84);
+	      KERNEL (44, 88);
+	      KERNEL (46, 92);
+	      KERNEL (48, 96);
+	      KERNEL (50, 100);
+	      KERNEL (52, 104);
+	      KERNEL (54, 108);
+	      KERNEL (56, 112);
+	      KERNEL (58, 116);
+	      KERNEL (60, 120);
+	      KERNEL (62, 124);
+	      AO += 512;
+	      BO += 256;
+	    }
+	  if ((temp & 31) >> 4)
+	    {
+	      vec_t *rowA = (vec_t *) & AO[0];
+	      vec_t *rowB = (vec_t *) & BO[0];
+	      KERNEL (0, 0);
+	      KERNEL (2, 4);
+	      KERNEL (4, 8);
+	      KERNEL (6, 12);
+	      KERNEL (8, 16);
+	      KERNEL (10, 20);
+	      KERNEL (12, 24);
+	      KERNEL (14, 28);
+	      KERNEL (16, 32);
+	      KERNEL (18, 36);
+	      KERNEL (20, 40);
+	      KERNEL (22, 44);
+	      KERNEL (24, 48);
+	      KERNEL (26, 52);
+	      KERNEL (28, 56);
+	      KERNEL (30, 60);
+	      AO += 256;
+	      BO += 128;
+	    }
+	  if ((temp & 15) >> 3)
+	    {
+	      vec_t *rowA = (vec_t *) & AO[0];
+	      vec_t *rowB = (vec_t *) & BO[0];
+	      KERNEL (0, 0);
+	      KERNEL (2, 4);
+	      KERNEL (4, 8);
+	      KERNEL (6, 12);
+	      KERNEL (8, 16);
+	      KERNEL (10, 20);
+	      KERNEL (12, 24);
+	      KERNEL (14, 28);
+	      AO += 128;
+	      BO += 64;
+	    }
+	  if ((temp & 7) >> 2)
+	    {
+	      vec_t *rowA = (vec_t *) & AO[0];
+	      vec_t *rowB = (vec_t *) & BO[0];
+	      KERNEL (0, 0);
+	      KERNEL (2, 4);
+	      KERNEL (4, 8);
+	      KERNEL (6, 12);
+	      AO += 64;
+	      BO += 32;
+	    }
+	  if ((temp & 3) >> 1)
+	    {
+	      vec_t *rowA = (vec_t *) & AO[0];
+	      vec_t *rowB = (vec_t *) & BO[0];
+	      KERNEL (0, 0);
+	      KERNEL (2, 4);
+	      AO += 32;
+	      BO += 16;
+	    }
+	  if ((temp & 1) >> 0)
+	    {
+	      vec_t *rowA = (vec_t *) & AO[0];
+	      vec_t *rowB = (vec_t *) & BO[0];
+	      KERNEL (0, 0);
+	      AO += 16;
+	      BO += 8;
+	    }
+	  SAVE_ACC (&acc0, 0);
+	  SAVE_ACC (&acc2, 4);
+	  SAVE_ACC1 (&acc1, 0);
+	  SAVE_ACC1 (&acc3, 4);
+	  SAVE_ACC (&acc4, 8);
+	  SAVE_ACC (&acc6, 12);
+	  SAVE_ACC1 (&acc5, 8);
+	  SAVE_ACC1 (&acc7, 12);
+#if defined(TRMMKERNEL)
+	  REFRESH_AFTER_SAVE (16, 8)
+#endif
+	    CO += 16;
+	}
+      i = (m & 15) >> 3;
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO;
+#if defined(TRMMKERNEL)
+	  REFRESH_POINTERS (8, 8);
+#else
+	  BO = B;
+	  temp = k;
+#endif
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  __vector_quad acc0, acc1, acc2, acc3;
+	  SET_ACC_ZERO4 ();
+	  BLASLONG l = 0;
+	  for (l = 0; l < temp; l++)
+	    {
+	      vec_t *rowA = (vec_t *) & AO[l << 3];
+	      vec_t *rowB = (vec_t *) & BO[l << 3];
+	      __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);
+	      __builtin_mma_xvf32gerpp (&acc1, rowB[1], rowA[0]);
+	      __builtin_mma_xvf32gerpp (&acc2, rowB[0], rowA[1]);
+	      __builtin_mma_xvf32gerpp (&acc3, rowB[1], rowA[1]);
+	    }
+	  SAVE_ACC (&acc0, 0);
+	  SAVE_ACC (&acc2, 4);
+	  SAVE_ACC1 (&acc1, 0);
+	  SAVE_ACC1 (&acc3, 4);
+	  AO += (temp << 3);
+	  BO += (temp << 3);
+	  CO += 8;
+#if defined(TRMMKERNEL)
+	  REFRESH_AFTER_SAVE (8, 8)
+#endif
+	}
+      i = (m & 7) >> 2;
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO;
+#if defined(TRMMKERNEL)
+	  REFRESH_POINTERS (4, 8);
+#else
+	  BO = B;
+	  temp = k;
+#endif
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  __vector_quad acc0, acc1;
+	  __builtin_mma_xxsetaccz (&acc0);
+	  __builtin_mma_xxsetaccz (&acc1);
+	  BLASLONG l = 0;
+	  for (l = 0; l < temp; l++)
+	    {
+	      vec_t *rowA = (vec_t *) & AO[l << 2];
+	      vec_t *rowB = (vec_t *) & BO[l << 3];
+	      __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);
+	      __builtin_mma_xvf32gerpp (&acc1, rowB[1], rowA[0]);
+	    }
+	  SAVE_ACC (&acc0, 0);
+	  SAVE_ACC1 (&acc1, 0);
+	  CO += 4;
+	  AO += (temp << 2);
+	  BO += (temp << 3);
+#if defined(TRMMKERNEL)
+	  REFRESH_AFTER_SAVE (4, 8)
+#endif
+	}
+      i = (m & 3) >> 1;
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO;
+#if defined(TRMMKERNEL)
+	  REFRESH_POINTERS (2, 8);
+#else
+	  BO = B;
+	  temp = k;
+#endif
+
+	  v2sf_t *rowC;
+	  v2sf_t result[8];
+	  __vector_quad acc0, acc1;
+	  __builtin_mma_xxsetaccz (&acc0);
+	  __builtin_mma_xxsetaccz (&acc1);
+	  BLASLONG l = 0;
+	  for (l = 0; l < temp; l++)
+	    {
+	      FLOAT t[4] = { 0 };
+	      t[0] = AO[l << 1], t[1] = AO[(l << 1) + 1];
+	      vec_t *rowA = (vec_t *) & t[0];
+	      vec_t *rowB = (vec_t *) & BO[l << 3];
+	      __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);
+	      __builtin_mma_xvf32gerpp (&acc1, rowB[1], rowA[0]);
+	    }
+	  SAVE4x2_ACC (&acc0, 0);
+	  SAVE4x2_ACC1 (&acc1, 0);
+	  CO += 2;
+	  AO += (temp << 1);
+	  BO += (temp << 3);
+#if defined(TRMMKERNEL)
+	  REFRESH_AFTER_SAVE (2, 8)
+#endif
+	}
+      i = (m & 1) >> 0;
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO;
+#if defined(TRMMKERNEL)
+	  REFRESH_POINTERS (1, 8);
+#else
+	  BO = B;
+	  temp = k;
+#endif
+	  BLASLONG l = 0;
+	  v4sf_t t = { 0, 0, 0, 0 };
+	  v4sf_t t1 = { 0, 0, 0, 0 };
+	  for (l = 0; l < temp; l++)
+	    {
+	      v4sf_t rowA = { AO[l], AO[l], AO[l], AO[l] };
+	      v4sf_t rowB = { BO[l << 3], BO[(l << 3) + 1], BO[(l << 3) + 2],
+		BO[(l << 3) + 3]
+	      };
+	      v4sf_t rowB1 =
+		{ BO[(l << 3) + 4], BO[(l << 3) + 5], BO[(l << 3) + 6],
+		BO[(l << 3) + 7]
+	      };
+	      t += rowA * rowB;
+	      t1 += rowA * rowB1;
+	    }
+	  t = t * valpha;
+	  t1 = t1 * valpha;
+#if defined(TRMMKERNEL)
+	  CO[0 * ldc] = t[0];
+	  CO[1 * ldc] = t[1];
+	  CO[2 * ldc] = t[2];
+	  CO[3 * ldc] = t[3];
+	  CO[4 * ldc] = t1[0];
+	  CO[5 * ldc] = t1[1];
+	  CO[6 * ldc] = t1[2];
+	  CO[7 * ldc] = t1[3];
+#else
+	  CO[0 * ldc] += t[0];
+	  CO[1 * ldc] += t[1];
+	  CO[2 * ldc] += t[2];
+	  CO[3 * ldc] += t[3];
+	  CO[4 * ldc] += t1[0];
+	  CO[5 * ldc] += t1[1];
+	  CO[6 * ldc] += t1[2];
+	  CO[7 * ldc] += t1[3];
+#endif
+	  CO += 1;
+	  AO += temp;
+	  BO += (temp << 3);
+#if defined(TRMMKERNEL)
+	  REFRESH_AFTER_SAVE (1, 8)
+#endif
+	}
+#if defined(TRMMKERNEL) && !defined(LEFT)
+      off += 8;			// number of values in A
+#endif
+
+      B += k << 3;
+    }
+  N = (n & 7) >> 2;
+  for (i1 = 0; i1 < N; i1++)
+    {
+      BLASLONG i, j, temp;
+#if defined(TRMMKERNEL) && defined(LEFT)
+      off = offset;
+#endif
+      FLOAT *CO;
+      FLOAT *AO;
+      CO = C;
+      C += ldc << 2;
+      AO = A;
+#if !defined(TRMMKERNEL)
+      i = m >> 5;
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO = B;
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  FLOAT *A1;
+	  A1 = AO + (16 * k);
+	  __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7;
+	  SET_ACC_ZERO8 ();
+	  BLASLONG l = 0;
+	  for (l = 0; l < k; l++)
+	    {
+	      vec_t *rowA = (vec_t *) & AO[l << 4];
+	      vec_t *rowA1 = (vec_t *) & A1[l << 4];
+	      vec_t *rowB = (vec_t *) & BO[l << 2];
+	      __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);
+	      __builtin_mma_xvf32gerpp (&acc1, rowB[0], rowA[1]);
+	      __builtin_mma_xvf32gerpp (&acc2, rowB[0], rowA[2]);
+	      __builtin_mma_xvf32gerpp (&acc3, rowB[0], rowA[3]);
+	      __builtin_mma_xvf32gerpp (&acc4, rowB[0], rowA1[0]);
+	      __builtin_mma_xvf32gerpp (&acc5, rowB[0], rowA1[1]);
+	      __builtin_mma_xvf32gerpp (&acc6, rowB[0], rowA1[2]);
+	      __builtin_mma_xvf32gerpp (&acc7, rowB[0], rowA1[3]);
+	    }
+
+	  SAVE_ACC (&acc0, 0);
+	  SAVE_ACC (&acc1, 4);
+	  CO += 8;
+	  SAVE_ACC (&acc2, 0);
+	  SAVE_ACC (&acc3, 4);
+	  CO += 8;
+	  SAVE_ACC (&acc4, 0);
+	  SAVE_ACC (&acc5, 4);
+	  CO += 8;
+	  SAVE_ACC (&acc6, 0);
+	  SAVE_ACC (&acc7, 4);
+	  CO += 8;
+	  AO += k << 5;
+	  BO += k << 2;
+	}
+      i = (m & 31) >> 4;
+#else
+      i = m >> 4;
+#endif
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO;
+#if defined(TRMMKERNEL)
+	  REFRESH_POINTERS (16, 4);
+#else
+	  BO = B;
+	  temp = k;
+#endif
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  __vector_quad acc0, acc1, acc2, acc3;
+	  SET_ACC_ZERO4 ();
+	  BLASLONG l = 0;
+	  for (l = 0; l < temp; l++)
+	    {
+	      vec_t *rowA = (vec_t *) & AO[l << 4];
+	      vec_t *rowB = (vec_t *) & BO[l << 2];
+	      __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);
+	      __builtin_mma_xvf32gerpp (&acc1, rowB[0], rowA[1]);
+	      __builtin_mma_xvf32gerpp (&acc2, rowB[0], rowA[2]);
+	      __builtin_mma_xvf32gerpp (&acc3, rowB[0], rowA[3]);
+	    }
+
+	  SAVE_ACC (&acc0, 0);
+	  SAVE_ACC (&acc1, 4);
+	  CO += 8;
+	  SAVE_ACC (&acc2, 0);
+	  SAVE_ACC (&acc3, 4);
+	  CO += 8;
+	  AO += temp << 4;
+	  BO += temp << 2;
+#if defined(TRMMKERNEL)
+	  REFRESH_AFTER_SAVE (16, 4)
+#endif
+	}
+      i = (m & 15) >> 3;
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO;
+#if defined(TRMMKERNEL)
+	  REFRESH_POINTERS (8, 4);
+#else
+	  BO = B;
+	  temp = k;
+#endif
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  __vector_quad acc0, acc1;
+	  __builtin_mma_xxsetaccz (&acc0);
+	  __builtin_mma_xxsetaccz (&acc1);
+	  BLASLONG l = 0;
+	  for (l = 0; l < temp; l++)
+	    {
+	      vec_t *rowA = (vec_t *) & AO[l << 3];
+	      vec_t *rowB = (vec_t *) & BO[l << 2];
+	      __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);
+	      __builtin_mma_xvf32gerpp (&acc1, rowB[0], rowA[1]);
+	    }
+	  SAVE_ACC (&acc0, 0);
+	  SAVE_ACC (&acc1, 4);
+	  CO += 8;
+	  AO += temp << 3;
+	  BO += temp << 2;
+#if defined(TRMMKERNEL)
+	  REFRESH_AFTER_SAVE (8, 4)
+#endif
+	}
+      i = (m & 7) >> 2;
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO;
+#if defined(TRMMKERNEL)
+	  REFRESH_POINTERS (4, 4);
+#else
+	  BO = B;
+	  temp = k;
+#endif
+	  v4sf_t *rowC;
+	  __vector_quad acc0;
+	  v4sf_t result[4];
+	  __builtin_mma_xxsetaccz (&acc0);
+	  BLASLONG l = 0;
+	  for (l = 0; l < temp; l++)
+	    {
+	      vec_t *rowA = (vec_t *) & AO[l << 2];
+	      vec_t *rowB = (vec_t *) & BO[l << 2];
+	      __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);
+	    }
+	  SAVE_ACC (&acc0, 0);
+	  CO += 4;
+	  AO += temp << 2;
+	  BO += temp << 2;
+#if defined(TRMMKERNEL)
+	  REFRESH_AFTER_SAVE (4, 4)
+#endif
+	}
+      i = (m & 3) >> 1;
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO;
+#if defined(TRMMKERNEL)
+	  REFRESH_POINTERS (2, 4);
+#else
+	  BO = B;
+	  temp = k;
+#endif
+	  v2sf_t *rowC;
+	  v2sf_t result[8];
+	  __vector_quad acc0;
+	  __builtin_mma_xxsetaccz (&acc0);
+	  BLASLONG l = 0;
+	  for (l = 0; l < temp; l++)
+	    {
+	      FLOAT t[4] = { 0 };
+	      t[0] = AO[l << 1], t[1] = AO[(l << 1) + 1];
+	      vec_t *rowA = (vec_t *) & t[0];
+	      vec_t *rowB = (vec_t *) & BO[l << 2];
+	      __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);
+	    }
+	  SAVE4x2_ACC (&acc0, 0);
+	  CO += 2;
+	  AO += temp << 1;
+	  BO += temp << 2;
+#if defined(TRMMKERNEL)
+	  REFRESH_AFTER_SAVE (2, 4)
+#endif
+	}
+      i = (m & 1) >> 0;
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO;
+#if defined(TRMMKERNEL)
+	  REFRESH_POINTERS (1, 4)
+#else
+	  BO = B;
+	  temp = k;
+#endif
+	  BLASLONG l = 0;
+	  v4sf_t t = { 0, 0, 0, 0 };
+	  for (l = 0; l < temp; l++)
+	    {
+	      v4sf_t rowA = { AO[l], AO[l], AO[l], AO[l] };
+	      v4sf_t rowB = { BO[l << 2], BO[(l << 2) + 1], BO[(l << 2) + 2],
+		BO[(l << 2) + 3]
+	      };
+	      t += rowA * rowB;
+	    }
+	  t = t * valpha;
+#if defined(TRMMKERNEL)
+	  CO[0 * ldc] = t[0];
+	  CO[1 * ldc] = t[1];
+	  CO[2 * ldc] = t[2];
+	  CO[3 * ldc] = t[3];
+#else
+	  CO[0 * ldc] += t[0];
+	  CO[1 * ldc] += t[1];
+	  CO[2 * ldc] += t[2];
+	  CO[3 * ldc] += t[3];
+#endif
+	  CO += 1;
+	  AO += temp;
+	  BO += temp << 2;
+#if defined(TRMMKERNEL)
+	  REFRESH_AFTER_SAVE (1, 4)
+#endif
+	}
+#if defined(TRMMKERNEL) && !defined(LEFT)
+      off += 4;			// number of values in A
+#endif
+
+      B += k << 2;
+    }
+  N = (n & 3) >> 1;
+  for (i1 = 0; i1 < N; i1++)
+    {
+      BLASLONG i, j, temp;
+#if defined(TRMMKERNEL) && defined(LEFT)
+      off = offset;
+#endif
+      FLOAT *CO;
+      FLOAT *AO;
+      CO = C;
+      C += ldc << 1;
+      AO = A;
+#if !defined(TRMMKERNEL)
+      i = m >> 5;
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO = B;
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  FLOAT *A1;
+	  A1 = AO + (16 * k);
+	  __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7;
+	  SET_ACC_ZERO8 ();
+	  BLASLONG l = 0;
+	  for (l = 0; l < k; l++)
+	    {
+	      FLOAT t[4] = { 0 };
+	      t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];
+	      vec_t *rowB = (vec_t *) & t[0];
+	      vec_t *rowA = (vec_t *) & AO[l << 4];
+	      vec_t *rowA1 = (vec_t *) & A1[l << 4];
+	      __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);
+	      __builtin_mma_xvf32gerpp (&acc1, rowB[0], rowA[1]);
+	      __builtin_mma_xvf32gerpp (&acc2, rowB[0], rowA[2]);
+	      __builtin_mma_xvf32gerpp (&acc3, rowB[0], rowA[3]);
+	      __builtin_mma_xvf32gerpp (&acc4, rowB[0], rowA1[0]);
+	      __builtin_mma_xvf32gerpp (&acc5, rowB[0], rowA1[1]);
+	      __builtin_mma_xvf32gerpp (&acc6, rowB[0], rowA1[2]);
+	      __builtin_mma_xvf32gerpp (&acc7, rowB[0], rowA1[3]);
+	    }
+	  SAVE2x4_ACC (&acc0, 0);
+	  SAVE2x4_ACC (&acc1, 4);
+	  SAVE2x4_ACC (&acc2, 8);
+	  SAVE2x4_ACC (&acc3, 12);
+	  CO += 16;
+	  SAVE2x4_ACC (&acc4, 0);
+	  SAVE2x4_ACC (&acc5, 4);
+	  SAVE2x4_ACC (&acc6, 8);
+	  SAVE2x4_ACC (&acc7, 12);
+	  CO += 16;
+	  AO += k << 5;
+	  BO += k << 1;
+	}
+      i = (m & 31) >> 4;
+#else
+      i = m >> 4;
+#endif
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO;
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  __vector_quad acc0, acc1, acc2, acc3;
+	  SET_ACC_ZERO4 ();
+	  BLASLONG l = 0;
+#if defined(TRMMKERNEL)
+	  REFRESH_POINTERS (16, 2)
+#else
+	  BO = B;
+	  temp = k;
+#endif
+	  for (l = 0; l < temp; l++)
+	    {
+	      FLOAT t[4] = { 0 };
+	      t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];
+	      vec_t *rowB = (vec_t *) & t[0];
+	      vec_t *rowA = (vec_t *) & AO[l << 4];
+	      __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);
+	      __builtin_mma_xvf32gerpp (&acc1, rowB[0], rowA[1]);
+	      __builtin_mma_xvf32gerpp (&acc2, rowB[0], rowA[2]);
+	      __builtin_mma_xvf32gerpp (&acc3, rowB[0], rowA[3]);
+	    }
+	  SAVE2x4_ACC (&acc0, 0);
+	  SAVE2x4_ACC (&acc1, 4);
+	  SAVE2x4_ACC (&acc2, 8);
+	  SAVE2x4_ACC (&acc3, 12);
+	  CO += 16;
+	  AO += temp << 4;
+	  BO += temp << 1;
+#if defined(TRMMKERNEL)
+	  REFRESH_AFTER_SAVE (16, 2)
+#endif
+	}
+      i = (m & 15) >> 3;
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO;
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  __vector_quad acc0, acc1;
+	  __builtin_mma_xxsetaccz (&acc0);
+	  __builtin_mma_xxsetaccz (&acc1);
+#if defined(TRMMKERNEL)
+	  REFRESH_POINTERS (8, 2)
+#else
+	  BO = B;
+	  temp = k;
+#endif
+	  BLASLONG l = 0;
+	  for (l = 0; l < temp; l++)
+	    {
+	      FLOAT t[4] = { 0 };
+	      t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];
+	      vec_t *rowB = (vec_t *) & t[0];
+	      vec_t *rowA = (vec_t *) & AO[l << 3];
+	      __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);
+	      __builtin_mma_xvf32gerpp (&acc1, rowB[0], rowA[1]);
+	    }
+	  SAVE2x4_ACC (&acc0, 0);
+	  SAVE2x4_ACC (&acc1, 4);
+	  CO += 8;
+	  AO += temp << 3;
+	  BO += temp << 1;
+#if defined(TRMMKERNEL)
+	  REFRESH_AFTER_SAVE (8, 2)
+#endif
+	}
+      i = (m & 7) >> 2;
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO;
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  __vector_quad acc0;
+	  __builtin_mma_xxsetaccz (&acc0);
+#if defined(TRMMKERNEL)
+	  REFRESH_POINTERS (4, 2)
+#else
+	  BO = B;
+	  temp = k;
+#endif
+	  BLASLONG l = 0;
+	  for (l = 0; l < temp; l++)
+	    {
+	      FLOAT t[4] = { 0 };
+	      t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];
+	      vec_t *rowB = (vec_t *) & t[0];
+	      vec_t *rowA = (vec_t *) & AO[l << 2];
+	      __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);
+	    }
+	  SAVE2x4_ACC (&acc0, 0);
+	  CO += 4;
+	  AO += temp << 2;
+	  BO += temp << 1;
+#if defined(TRMMKERNEL)
+	  REFRESH_AFTER_SAVE (4, 2)
+#endif
+	}
+      i = (m & 3) >> 1;
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO;
+	  BLASLONG l = 0;
+#if defined(TRMMKERNEL)
+	  REFRESH_POINTERS (2, 2)
+#else
+	  BO = B;
+	  temp = k;
+#endif
+	  v4sf_t t = { 0, 0, 0, 0 };
+	  for (l = 0; l < (temp << 1); l += 2)
+	    {
+	      v4sf_t rowA = { AO[l], AO[l], AO[l + 1], AO[l + 1] };
+	      v4sf_t rowB = { BO[l], BO[l + 1], BO[l], BO[l + 1] };
+	      t += rowA * rowB;
+	    }
+	  t = t * valpha;
+#if defined(TRMMKERNEL)
+	  CO[0 * ldc] = t[0];
+	  CO[1 * ldc] = t[1];
+	  CO[0 * ldc + 1] = t[2];
+	  CO[1 * ldc + 1] = t[3];
+#else
+	  CO[0 * ldc] += t[0];
+	  CO[1 * ldc] += t[1];
+	  CO[0 * ldc + 1] += t[2];
+	  CO[1 * ldc + 1] += t[3];
+#endif
+	  CO += 2;
+	  AO += temp << 1;
+	  BO += temp << 1;
+#if defined(TRMMKERNEL)
+	  REFRESH_AFTER_SAVE (2, 2)
+#endif
+	}
+      i = (m & 1) >> 0;
+      for (j = 0; j < i; j++)
+	{
+	  FLOAT *BO;
+	  BLASLONG l = 0;
+#if defined(TRMMKERNEL)
+	  REFRESH_POINTERS (1, 2)
+#else
+	  BO = B;
+	  temp = k;
+#endif
+	  v4sf_t t = { 0, 0, 0, 0 };
+	  for (l = 0; l < temp; l++)
+	    {
+	      v4sf_t rowA = { AO[l], AO[l], 0, 0 };
+	      v4sf_t rowB = { BO[l << 1], BO[(l << 1) + 1], 0, 0 };
+	      t += rowA * rowB;
+	    }
+	  t = t * valpha;
+#if defined(TRMMKERNEL)
+	  CO[0 * ldc] = t[0];
+	  CO[1 * ldc] = t[1];
+#else
+	  CO[0 * ldc] += t[0];
+	  CO[1 * ldc] += t[1];
+#endif
+	  CO += 1;
+	  AO += temp;
+	  BO += temp << 1;
+#if defined(TRMMKERNEL)
+	  REFRESH_AFTER_SAVE (1, 2)
+#endif
+	}
+#if defined(TRMMKERNEL) && !defined(LEFT)
+      off += 2;			// number of values in A
+#endif
+
+      B += k << 1;
+    }
+  N = (n & 1) >> 0;
+  for (i1 = 0; i1 < N; i1++)
+    {
+      BLASLONG i, temp;
+#if defined(TRMMKERNEL) && defined(LEFT)
+      off = offset;
+#endif
+      FLOAT *CO;
+      FLOAT *AO;
+      CO = C;
+      C += ldc;
+      AO = A;
+      i = m;
+      while (i >= 16)
+	{
+	  FLOAT *BO;
+	  BLASLONG l = 0;
+#if defined(TRMMKERNEL)
+	  REFRESH_POINTERS (16, 1)
+#else
+	  BO = B;
+	  temp = k;
+#endif
+
+	  v4sf_t t = { 0, 0, 0, 0 };
+	  v4sf_t t1 = { 0, 0, 0, 0 };
+	  v4sf_t t2 = { 0, 0, 0, 0 };
+	  v4sf_t t3 = { 0, 0, 0, 0 };
+	  for (l = 0; l < temp; l++)
+	    {
+	      v4sf_t rowB = { BO[l], BO[l], BO[l], BO[l] };
+	      v4sf_t rowA = { AO[l << 4], AO[(l << 4) + 1], AO[(l << 4) + 2],
+		AO[(l << 4) + 3]
+	      };
+	      v4sf_t rowA1 =
+		{ AO[(l << 4) + 4], AO[(l << 4) + 5], AO[(l << 4) + 6],
+		AO[(l << 4) + 7]
+	      };
+	      v4sf_t rowA2 =
+		{ AO[(l << 4) + 8], AO[(l << 4) + 9], AO[(l << 4) + 10],
+		AO[(l << 4) + 11]
+	      };
+	      v4sf_t rowA3 =
+		{ AO[(l << 4) + 12], AO[(l << 4) + 13], AO[(l << 4) + 14],
+		AO[(l << 4) + 15]
+	      };
+	      t += rowA * rowB;
+	      t1 += rowA1 * rowB;
+	      t2 += rowA2 * rowB;
+	      t3 += rowA3 * rowB;
+	    }
+	  t = t * valpha;
+	  t1 = t1 * valpha;
+	  t2 = t2 * valpha;
+	  t3 = t3 * valpha;
+#if defined(TRMMKERNEL)
+	  CO[0] = t[0];
+	  CO[1] = t[1];
+	  CO[2] = t[2];
+	  CO[3] = t[3];
+	  CO[4] = t1[0];
+	  CO[5] = t1[1];
+	  CO[6] = t1[2];
+	  CO[7] = t1[3];
+	  CO[8] = t2[0];
+	  CO[9] = t2[1];
+	  CO[10] = t2[2];
+	  CO[11] = t2[3];
+	  CO[12] = t3[0];
+	  CO[13] = t3[1];
+	  CO[14] = t3[2];
+	  CO[15] = t3[3];
+#else
+	  CO[0] += t[0];
+	  CO[1] += t[1];
+	  CO[2] += t[2];
+	  CO[3] += t[3];
+	  CO[4] += t1[0];
+	  CO[5] += t1[1];
+	  CO[6] += t1[2];
+	  CO[7] += t1[3];
+	  CO[8] += t2[0];
+	  CO[9] += t2[1];
+	  CO[10] += t2[2];
+	  CO[11] += t2[3];
+	  CO[12] += t3[0];
+	  CO[13] += t3[1];
+	  CO[14] += t3[2];
+	  CO[15] += t3[3];
+#endif
+	  AO += temp << 4;
+	  BO += temp;
+	  CO += 16;
+	  i -= 16;
+#if defined(TRMMKERNEL)
+	  REFRESH_AFTER_SAVE (16, 1)
+#endif
+	}
+      while (i >= 8)
+	{
+	  FLOAT *BO;
+	  BLASLONG l = 0;
+	  v4sf_t t = { 0, 0, 0, 0 };
+	  v4sf_t t1 = { 0, 0, 0, 0 };
+#if defined(TRMMKERNEL)
+	  REFRESH_POINTERS (8, 1)
+#else
+	  BO = B;
+	  temp = k;
+#endif
+
+	  for (l = 0; l < temp; l++)
+	    {
+	      v4sf_t rowB = { BO[l], BO[l], BO[l], BO[l] };
+	      v4sf_t rowA = { AO[l << 3], AO[(l << 3) + 1], AO[(l << 3) + 2],
+		AO[(l << 3) + 3]
+	      };
+	      v4sf_t rowA1 =
+		{ AO[(l << 3) + 4], AO[(l << 3) + 5], AO[(l << 3) + 6],
+		AO[(l << 3) + 7]
+	      };
+	      t += rowA * rowB;
+	      t1 += rowA1 * rowB;
+	    }
+	  t = t * valpha;
+	  t1 = t1 * valpha;
+#if defined(TRMMKERNEL)
+	  CO[0] = t[0];
+	  CO[1] = t[1];
+	  CO[2] = t[2];
+	  CO[3] = t[3];
+	  CO[4] = t1[0];
+	  CO[5] = t1[1];
+	  CO[6] = t1[2];
+	  CO[7] = t1[3];
+#else
+	  CO[0] += t[0];
+	  CO[1] += t[1];
+	  CO[2] += t[2];
+	  CO[3] += t[3];
+	  CO[4] += t1[0];
+	  CO[5] += t1[1];
+	  CO[6] += t1[2];
+	  CO[7] += t1[3];
+#endif
+	  AO += temp << 3;
+	  BO += temp;
+	  CO += 8;
+	  i -= 8;
+#if defined(TRMMKERNEL)
+	  REFRESH_AFTER_SAVE (8, 1)
+#endif
+	}
+      while (i >= 4)
+	{
+	  FLOAT *BO;
+	  BLASLONG l = 0;
+	  v4sf_t t = { 0, 0, 0, 0 };
+#if defined(TRMMKERNEL)
+	  REFRESH_POINTERS (4, 1)
+#else
+	  BO = B;
+	  temp = k;
+#endif
+
+	  for (l = 0; l < temp; l++)
+	    {
+	      v4sf_t rowB = { BO[l], BO[l], BO[l], BO[l] };
+	      v4sf_t rowA = { AO[l << 2], AO[(l << 2) + 1], AO[(l << 2) + 2],
+		AO[(l << 2) + 3]
+	      };
+	      t += rowA * rowB;
+	    }
+	  t = t * valpha;
+#if defined(TRMMKERNEL)
+	  CO[0] = t[0];
+	  CO[1] = t[1];
+	  CO[2] = t[2];
+	  CO[3] = t[3];
+#else
+	  CO[0] += t[0];
+	  CO[1] += t[1];
+	  CO[2] += t[2];
+	  CO[3] += t[3];
+#endif
+	  AO += temp << 2;
+	  BO += temp;
+	  CO += 4;
+	  i -= 4;
+#if defined(TRMMKERNEL)
+	  REFRESH_AFTER_SAVE (4, 1)
+#endif
+	}
+      while (i >= 2)
+	{
+	  FLOAT *BO;
+	  BLASLONG l = 0;
+#if defined(TRMMKERNEL)
+	  REFRESH_POINTERS (2, 1)
+#else
+	  BO = B;
+	  temp = k;
+#endif
+
+	  v4sf_t t = { 0, 0, 0, 0 };
+	  for (l = 0; l < temp; l++)
+	    {
+	      v4sf_t rowB = { BO[l], BO[l], 0, 0 };
+	      v4sf_t rowA = { AO[l << 1], AO[(l << 1) + 1], 0, 0 };
+	      t += rowA * rowB;
+	    }
+	  t = t * valpha;
+#if defined(TRMMKERNEL)
+	  CO[0] = t[0];
+	  CO[1] = t[1];
+#else
+	  CO[0] += t[0];
+	  CO[1] += t[1];
+#endif
+	  AO += temp << 1;
+	  BO += temp;
+	  CO += 2;
+	  i -= 2;
+#if defined(TRMMKERNEL)
+	  REFRESH_AFTER_SAVE (2, 1)
+#endif
+	}
+      while (i >= 1)
+	{
+	  FLOAT *BO;
+#if defined(TRMMKERNEL)
+	  REFRESH_POINTERS (1, 1)
+#else
+	  BO = B;
+	  temp = k;
+#endif
+
+	  BLASLONG l = 0;
+	  FLOAT t = 0;
+	  for (l = 0; l < temp; l++)
+	    {
+	      t += AO[l] * BO[l];
+	    }
+	  AO += temp;
+	  BO += temp;
+#if defined(TRMMKERNEL)
+	  CO[0] = t * alpha;
+#else
+	  CO[0] += t * alpha;
+#endif
+	  CO += 1;
+	  i -= 1;
+#if defined(TRMMKERNEL)
+	  REFRESH_AFTER_SAVE (1, 1)
+#endif
+	}
+
+#if defined(TRMMKERNEL) && !defined(LEFT)
+      off += 1;			// number of values in A
+#endif
+      B += k;
+    }
+  return 0;
+}

From bb2f52844bbcd5c786d7b37f8c4d88dbf7a3b89e Mon Sep 17 00:00:00 2001
From: Gordon Fossum <fossum@us.ibm.com>
Date: Wed, 24 Jun 2020 14:50:12 -0500
Subject: [PATCH 30/48] powerpc: Optimized ZGEMM kernel for POWER10

This patch introduces new optimized version of ZGEMM kernel
using power10 Matrix-Multiply Assist (MMA) feature introduced in
POWER ISA v3.1. This patch makes use of new POWER10 compute instructions
for matrix multiplication operation.

Tested on simulator and there are no new test failures.
Cycles count reduced by 30-50%  compared to POWER9 version depending on
M/N/K sizes.
---
 kernel/power/KERNEL.POWER10         |    4 +-
 kernel/power/zgemm_kernel_power10.S |  245 ++++
 kernel/power/zgemm_logic_power10.S  | 1735 +++++++++++++++++++++++++++
 kernel/power/zgemm_macros_power10.S | 1138 ++++++++++++++++++
 4 files changed, 3120 insertions(+), 2 deletions(-)
 create mode 100644 kernel/power/zgemm_kernel_power10.S
 create mode 100644 kernel/power/zgemm_logic_power10.S
 create mode 100644 kernel/power/zgemm_macros_power10.S

diff --git a/kernel/power/KERNEL.POWER10 b/kernel/power/KERNEL.POWER10
index 00d31f8b6..4fc7190b0 100644
--- a/kernel/power/KERNEL.POWER10
+++ b/kernel/power/KERNEL.POWER10
@@ -10,7 +10,7 @@ else
 STRMMKERNEL	= sgemm_kernel_power10.c
 DTRMMKERNEL	= dgemm_kernel_power10.c
 CTRMMKERNEL	= cgemm_kernel_power10.S
-ZTRMMKERNEL	= zgemm_kernel_power9.S
+ZTRMMKERNEL	= zgemm_kernel_power10.S
 
 SGEMMKERNEL    =  sgemm_kernel_power10.c
 SGEMMINCOPY    = ../generic/gemm_ncopy_16.c
@@ -42,7 +42,7 @@ CGEMMOTCOPYOBJ =  cgemm_otcopy$(TSUFFIX).$(SUFFIX)
 CGEMMINCOPYOBJ =  cgemm_incopy$(TSUFFIX).$(SUFFIX)
 CGEMMITCOPYOBJ =  cgemm_itcopy$(TSUFFIX).$(SUFFIX)
 
-ZGEMMKERNEL    = zgemm_kernel_power9.S
+ZGEMMKERNEL    = zgemm_kernel_power10.S
 ZGEMMONCOPY    = ../generic/zgemm_ncopy_2.c
 ZGEMMOTCOPY    = ../generic/zgemm_tcopy_2.c
 ZGEMMINCOPY    = ../generic/zgemm_ncopy_8.c
diff --git a/kernel/power/zgemm_kernel_power10.S b/kernel/power/zgemm_kernel_power10.S
new file mode 100644
index 000000000..fca389e69
--- /dev/null
+++ b/kernel/power/zgemm_kernel_power10.S
@@ -0,0 +1,245 @@
+/***************************************************************************
+Copyright (c) 2013-2020, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+#define ASSEMBLER
+#include "common.h"
+#include "def_vsx.h"
+
+#define LOAD	ld
+ 
+#define STACKSIZE 512
+
+#define FZERO	312+192(SP)
+
+#define FLINK_SAVE (STACKSIZE+16) /* 16($r12) */
+
+#define	M	r3
+#define	N	r4
+#define	K	r5
+
+ 
+#define A	r8
+#define	B	r9
+#define	C	r10
+#define	LDC	r6
+#define OFFSET	r7
+ 
+ 
+
+#define o0	0
+#define alpha_r vs62
+#define alpha_i vs63
+
+#define VECSAVE r11
+
+#define FRAMEPOINTER r12
+
+#define T10 r14
+
+#define L	r15
+#define T8	r16
+#define T5	r17
+#define T2	r19
+#define TEMP_REG	r20
+#define	T6	r21
+#define	I	r22
+#define J	r23
+#define AO	r24
+#define	BO	r25
+#define	CO	r26
+#define T7	r27
+#define	T3	r28
+#define T4	r29
+
+#define PRE	r30
+#define T1  	r31
+
+#ifndef NEEDPARAM
+
+	PROLOGUE
+	PROFCODE
+
+	mr      FRAMEPOINTER, SP
+    addi    SP, SP, -STACKSIZE 
+    mflr    r0
+	stfd	f14,    0(SP)
+	stfd	f15,    8(SP)
+	stfd	f16,   16(SP)
+	stfd	f17,   24(SP)
+
+	stfd	f18,   32(SP)
+	stfd	f19,   40(SP)
+	stfd	f20,   48(SP)
+	stfd	f21,   56(SP)
+
+	stfd	f22,   64(SP)
+	stfd	f23,   72(SP)
+	stfd	f24,   80(SP)
+	stfd	f25,   88(SP)
+
+	stfd	f26,   96(SP)
+	stfd	f27,  104(SP)
+	stfd	f28,  112(SP)
+	stfd	f29,  120(SP)
+
+	stfd	f30,  128(SP)
+	stfd	f31,  136(SP)
+
+    xxspltd  alpha_r,vs1,0  /*copy from register f1 */
+    xxspltd  alpha_i,vs2,0  /*copy from register f2 */
+ 
+	std	r31,  144(SP)
+	std	r30,  152(SP)
+	std	r29,  160(SP)
+	std	r28,  168(SP)
+	std	r27,  176(SP)
+	std	r26,  184(SP)
+	std	r25,  192(SP)
+	std	r24,  200(SP)
+	std	r23,  208(SP)
+	std	r22,  216(SP)
+	std	r21,  224(SP)
+	std	r20,  232(SP)
+	std	r19,  240(SP)
+	std	r18,  248(SP)
+	std	r17,  256(SP)
+	std	r16,  264(SP)
+	std	r15,  272(SP)
+	std	r14,  280(SP)
+ 
+ 
+    stxv    vs20,  288(SP)
+    stxv    vs21,  304(SP)
+    stxv    vs22,  320(SP)
+    stxv    vs23,  336(SP)
+    stxv    vs24,  352(SP)
+    stxv    vs25,  368(SP)
+    stxv    vs26,  384(SP)
+    stxv    vs27,  400(SP)
+    stxv    vs28,  416(SP)
+    stxv    vs29,  432(SP)
+    stxv    vs30,  448(SP)
+    stxv    vs31,  464(SP)
+
+    std    r0, FLINK_SAVE(SP)
+ 
+
+#if defined(linux) || defined(__FreeBSD__)
+	ld	LDC, FRAMESLOT(0) + 0(FRAMEPOINTER)
+#endif
+
+
+#ifdef TRMMKERNEL
+#if (defined(linux) || defined(__FreeBSD__)) && defined(__64BIT__)
+	ld	OFFSET,  FRAMESLOT(1) + 0(FRAMEPOINTER)
+#endif 
+#endif
+
+
+#include "zgemm_macros_power10.S"
+
+ 
+
+	slwi	LDC, LDC, ZBASE_SHIFT
+	li	PRE,  512 
+    li  r0,   0
+ 
+
+#if defined(CC) || defined(CR) || defined(RC) || defined(RR) 
+/*negate for this case as we will use addition -1*(a+b) */
+  xvnegdp alpha_r,alpha_r
+  xvnegdp alpha_i,alpha_i
+#endif
+	.align 4
+
+#include "zgemm_logic_power10.S"
+
+L999:
+ 
+	lfd	f14,    0(SP)
+	lfd	f15,    8(SP)
+	lfd	f16,   16(SP)
+	lfd	f17,   24(SP)
+
+	lfd	f18,   32(SP)
+	lfd	f19,   40(SP)
+	lfd	f20,   48(SP)
+	lfd	f21,   56(SP)
+
+	lfd	f22,   64(SP)
+	lfd	f23,   72(SP)
+	lfd	f24,   80(SP)
+	lfd	f25,   88(SP)
+
+	lfd	f26,   96(SP)
+	lfd	f27,  104(SP)
+	lfd	f28,  112(SP)
+	lfd	f29,  120(SP)
+
+	lfd	f30,  128(SP)
+	lfd	f31,  136(SP)
+
+ 
+	ld	r31,  144(SP)
+	ld	r30,  152(SP)
+	ld	r29,  160(SP)
+	ld	r28,  168(SP)
+	ld	r27,  176(SP)
+	ld	r26,  184(SP)
+	ld	r25,  192(SP)
+	ld	r24,  200(SP)
+	ld	r23,  208(SP)
+	ld	r22,  216(SP)
+	ld	r21,  224(SP)
+	ld	r20,  232(SP)
+	ld	r19,  240(SP)
+	ld	r18,  248(SP)
+	ld	r17,  256(SP)
+	ld	r16,  264(SP)
+	ld	r15,  272(SP)
+	ld	r14,  280(SP)
+
+	ld    r0, 	 FLINK_SAVE(SP)	
+ 
+    lxv    vs20,  288(SP)
+    lxv    vs21,  304(SP)
+    lxv    vs22,  320(SP)
+    lxv    vs23,  336(SP)
+    lxv    vs24,  352(SP)
+    lxv    vs25,  368(SP)
+    lxv    vs26,  384(SP) 
+    lxv    vs27,  400(SP)
+	mtlr r0
+    lxv    vs28,  416(SP)
+    lxv    vs29,  432(SP) 
+    lxv    vs30,  448(SP)
+    lxv    vs31,  464(SP)
+
+	addi	SP, SP, STACKSIZE 
+	blr
+
+	EPILOGUE
+#endif
diff --git a/kernel/power/zgemm_logic_power10.S b/kernel/power/zgemm_logic_power10.S
new file mode 100644
index 000000000..1143733e0
--- /dev/null
+++ b/kernel/power/zgemm_logic_power10.S
@@ -0,0 +1,1735 @@
+/***************************************************************************
+Copyright (c) 2013-2020, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+#define MY_ALIGN .align 3
+b ZGEMM_L2
+/*                MINI SUBROUTINES                            */      
+/*                2x8 MAIN 128x+2 LOOP                     */      
+
+
+ZGEMM_L2x8_LMAIN_SUB:
+/*----------------------------------------*/   
+    mtctr   T8
+    MY_ALIGN
+ZGEMM_L2x8_LOOP:
+/*----------------------------------------*/   
+    dcbt    AO, PRE
+    dcbt    BO, PRE
+    KERNEL2x8_2 0, 0
+ZGEMM_L2x8_K128:
+/*----------------------------------------*/   
+    KERNEL2x8_2 1, 0
+    dcbt    AO, T2  
+    KERNEL2x8_2 2, 0
+    KERNEL2x8_2 3, 0
+    dcbt    AO, T3
+    dcbt    BO, T2
+    KERNEL2x8_2 4, 0
+    KERNEL2x8_2 5, 0
+    dcbt    AO, T4  
+    KERNEL2x8_2 6, 0
+    KERNEL2x8_2 7, 0
+    dcbt    AO, T5  
+    dcbt    BO, T3
+    KERNEL2x8_2 8, 0
+    KERNEL2x8_2 9, 0
+    KERNEL2x8_2 10, 0
+    KERNEL2x8_2 11, 0
+    dcbt    BO, T4
+    KERNEL2x8_2 12, 0
+    KERNEL2x8_2 13, 0
+    KERNEL2x8_2 14, 0
+    KERNEL2x8_2 15, 0
+    KERNEL2x8_2 16, 0
+    KERNEL2x8_2 17, 0
+    KERNEL2x8_2 18, 0
+    KERNEL2x8_2 19, 0
+    KERNEL2x8_2 20, 0
+    KERNEL2x8_2 21, 0
+    KERNEL2x8_2 22, 0
+    KERNEL2x8_2 23, 0
+    KERNEL2x8_2 24, 0
+    KERNEL2x8_2 25, 0
+    KERNEL2x8_2 26, 0
+    KERNEL2x8_2 27, 0
+    KERNEL2x8_2 28, 0
+    KERNEL2x8_2 29, 0
+    KERNEL2x8_2 30, 0
+    KERNEL2x8_2 31, 0
+    KERNEL2x8_2 32, 0
+    KERNEL2x8_2 33, 0
+    KERNEL2x8_2 34, 0
+    KERNEL2x8_2 35, 0
+    KERNEL2x8_2 36, 0
+    KERNEL2x8_2 37, 0
+    KERNEL2x8_2 38, 0
+    KERNEL2x8_2 39, 0
+    KERNEL2x8_2 40, 0
+    KERNEL2x8_2 41, 0
+    KERNEL2x8_2 42, 0
+    KERNEL2x8_2 43, 0
+    KERNEL2x8_2 44, 0
+    KERNEL2x8_2 45, 0
+    KERNEL2x8_2 46, 0
+    KERNEL2x8_2 47, 0
+    KERNEL2x8_2 48, 0
+    KERNEL2x8_2 49, 0
+    KERNEL2x8_2 50, 0
+    KERNEL2x8_2 51, 0
+    KERNEL2x8_2 52, 0
+    KERNEL2x8_2 53, 0
+    KERNEL2x8_2 54, 0
+    KERNEL2x8_2 55, 0
+    KERNEL2x8_2 56, 0
+    KERNEL2x8_2 57, 0
+    KERNEL2x8_2 58, 0
+    KERNEL2x8_2 59, 0
+    KERNEL2x8_2 60, 0
+    KERNEL2x8_2 61, 0
+    KERNEL2x8_2 62, 0
+    KERNEL2x8_2 63, 1
+    bdz     ZGEMM_L2x8_LOOP_END
+    b       ZGEMM_L2x8_LOOP
+    MY_ALIGN  
+
+ZGEMM_L2x8_LOOP_END:
+/*----------------------------------------*/   
+    KERNEL2x8_2 0, 1
+    blr
+    MY_ALIGN
+
+
+ZGEMM_2x4_LMAIN_SUB:
+/*----------------------------------------*/   
+    mtctr   T8
+    MY_ALIGN
+ZGEMM_L2x4_LOOP:
+/*----------------------------------------*/   
+    KERNEL2x4_2 0, 0
+ZGEMM_L2x4_K32:
+/*----------------------------------------*/   
+    KERNEL2x4_2 1, 0
+    KERNEL2x4_2 2, 0
+    KERNEL2x4_2 3, 0
+    KERNEL2x4_2 4, 0
+    KERNEL2x4_2 5, 0
+    KERNEL2x4_2 6, 0
+    KERNEL2x4_2 7, 0
+    KERNEL2x4_2 8, 0
+    KERNEL2x4_2 9, 0
+    KERNEL2x4_2 10, 0
+    KERNEL2x4_2 11, 0
+    KERNEL2x4_2 12, 0
+    KERNEL2x4_2 13, 0
+    KERNEL2x4_2 14, 0
+    KERNEL2x4_2 15, 1
+    bdnz    ZGEMM_L2x4_LOOP
+    MY_ALIGN  
+ZGEMM_L2x4_LOOP_END:
+/*----------------------------------------*/   
+    KERNEL2x4_2 0, 1
+    blr
+    MY_ALIGN
+
+
+ZGEMM_2x2_LMAIN_SUB:
+/*----------------------------------------*/   
+    mtctr   T8
+    MY_ALIGN 
+ZGEMM_L2x2_LOOP:
+/*----------------------------------------*/   
+    KERNEL2x2_2 0, 0 
+ZGEMM_L2x2_K32:
+/*----------------------------------------*/   
+    KERNEL2x2_2 1, 0  
+    KERNEL2x2_2 2, 0
+    KERNEL2x2_2 3, 0  
+    KERNEL2x2_2 4, 0
+    KERNEL2x2_2 5, 0 
+    KERNEL2x2_2 6, 0
+    KERNEL2x2_2 7, 0
+    KERNEL2x2_2 8, 0
+    KERNEL2x2_2 9, 0  
+    KERNEL2x2_2 10, 0
+    KERNEL2x2_2 11, 0  
+    KERNEL2x2_2 12, 0
+    KERNEL2x2_2 13, 0 
+    KERNEL2x2_2 14, 0
+    KERNEL2x2_2 15, 1   
+    bdnz    ZGEMM_L2x2_LOOP
+    MY_ALIGN  
+
+
+ZGEMM_L2x2_LOOP_END:
+/*----------------------------------------*/   
+    KERNEL2x2_2 0, 1
+    blr
+    MY_ALIGN
+
+ZGEMM_2x1_LMAIN_SUB:
+/*----------------------------------------*/   
+    mtctr   T8
+    LOAD2x1_2  
+    MY_ALIGN
+ZGEMM_L2x1_LOOP:
+/*----------------------------------------*/   
+    KERNEL2x1_L2 32, 64, 0, 0 
+ZGEMM_L2x1_K32:
+/*----------------------------------------*/   
+    KERNEL2x1_L2 32, 64, 1, 0  
+    KERNEL2x1_L2 32, 64, 2, 0
+    KERNEL2x1_L2 32, 64, 3, 0  
+    KERNEL2x1_L2 32, 64, 4, 0
+    KERNEL2x1_L2 32, 64, 5, 0 
+    KERNEL2x1_L2 32, 64, 6, 0
+    KERNEL2x1_L2 32, 64, 7, 0
+    KERNEL2x1_L2 32, 64, 8, 0
+    KERNEL2x1_L2 32, 64, 9, 0  
+    KERNEL2x1_L2 32, 64, 10, 0
+    KERNEL2x1_L2 32, 64, 11, 0  
+    KERNEL2x1_L2 32, 64, 12, 0
+    KERNEL2x1_L2 32, 64, 13, 0 
+    KERNEL2x1_L2 32, 64, 14, 0
+    KERNEL2x1_L2 32, 64, 15, 1   
+    bdnz    ZGEMM_L2x1_LOOP
+    MY_ALIGN  
+ZGEMM_L2x1_LOOP_END:
+/*----------------------------------------*/   
+    END2x1_2 
+    blr
+
+    MY_ALIGN
+
+
+/*             MAIN LOOP BEGINS               */   
+    MY_ALIGN
+
+
+ZGEMM_L2:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL) && !defined(LEFT)   
+    neg TEMP_REG, OFFSET 
+#endif   
+    srawi.    J, N, 1
+    bgt   ZGEMM_L2_BEGIN
+    b     ZGEMM_L2_END
+
+ZGEMM_L2_BEGIN:
+/*----------------------------------------*/   
+    mr    CO, C
+    slwi    T1, LDC, 1     
+    add     T2,C,LDC    
+    mr    AO, A  
+    add   C, C, T1
+#if defined(TRMMKERNEL) && defined(LEFT)   
+    mr TEMP_REG, OFFSET  /*off = offset;*/
+#endif     
+    srawi.    I, M, 3
+    bgt   ZGEMM_L2_BEGIN_CONTINUE
+    b     ZGEMM_L2x8_END
+
+ZGEMM_L2_BEGIN_CONTINUE:
+    dcbt    CO,r0  /*just prefetch*/
+    dcbt    T2,r0
+
+
+ZGEMM_L2x8_BEGIN:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)   
+    REFRESH_POINTERS  AO, BO,TEMP_REG, B, 8, 2
+#else    
+    mr    BO, B  
+    dcbt    B, r0  
+#endif     
+    dcbt    AO, r0
+#if defined(TRMMKERNEL)   
+    REFRESH_TEMP_BK T6,K,TEMP_REG, 8, 2
+    mr T1, T6
+#else   
+    mr T1, K
+#endif   
+/* TEMPS FOR PREFETCH */   
+    li T2, 1024
+    li T3, 1024+512
+    addi T1,T1, -2
+/* TEMPS FOR PREFETCH */     
+    li T4, 2048
+    li T5, 2048+512 
+    srawi.   T8, T1, 7 /* T8 <- T1 % 128 */
+
+    KERNEL2x8_PRELOAD
+    KERNEL2x8_ZERO_AND_PRIME_MMA
+    ble   ZGEMM_L2x8_SUB0
+    bl ZGEMM_L2x8_LMAIN_SUB
+    andi.   L, T1, 127
+
+    bgt   ZGEMM_L2x8_BEGIN_CONTINUE
+    b     ZGEMM_L2x8_SAVE
+
+ZGEMM_L2x8_BEGIN_CONTINUE:
+    b   ZGEMM_L2x8_SUB2
+
+
+ZGEMM_L2x8_SUB0:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)   
+    andi.   L, T6, 255
+    cmpwi   T6, 129
+#else   
+    andi.   L, K, 255
+    cmpwi   K, 129
+#endif       
+    li T8, 1
+    bne CMP2x8_128K
+    LOAD_END_2x8 128, 32
+    KERNEL2x8_PRELOAD
+    addi BO, BO, -64
+    addi AO,AO, -256   
+    mtctr   T8    
+    bl ZGEMM_L2x8_K128
+    b ZGEMM_L2x8_SAVE  
+
+CMP2x8_128K:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)    
+    cmpwi   T6, 128
+#else    
+    cmpwi   K, 128
+#endif        
+    bne ZGEMM_L2x8_SUB2 
+    MY_ALIGN   
+    mtctr   T8
+    addi BO, BO, -64
+    addi AO,AO, -256   
+    bl ZGEMM_L2x8_K128
+    b ZGEMM_L2x8_SAVE 
+    MY_ALIGN
+
+
+ZGEMM_L2x8_SUB2:
+/*----------------------------------------*/   
+    andi.      T1,L, 64
+    ble ZGEMM_L2x8_SUB2_32
+    dcbt    AO, PRE
+    dcbt    BO, PRE
+    KERNEL2x8_2 0, 0
+    KERNEL2x8_2 1, 0
+    dcbt    AO, T2  
+    KERNEL2x8_2 2, 0
+    KERNEL2x8_2 3, 0
+    dcbt    AO, T3
+    dcbt    BO, T2
+    KERNEL2x8_2 4, 0
+    KERNEL2x8_2 5, 0
+    dcbt    AO, T4  
+    KERNEL2x8_2 6, 0
+    KERNEL2x8_2 7, 0
+    dcbt    AO, T5  
+    dcbt    BO, T3
+    KERNEL2x8_2 8, 0
+    KERNEL2x8_2 9, 0
+    KERNEL2x8_2 10, 0
+    KERNEL2x8_2 11, 0
+    dcbt    BO, T4
+    KERNEL2x8_2 12, 0
+    KERNEL2x8_2 13, 0
+    KERNEL2x8_2 14, 0
+    KERNEL2x8_2 15, 0
+    KERNEL2x8_2 16, 0
+    KERNEL2x8_2 17, 0
+    KERNEL2x8_2 18, 0
+    KERNEL2x8_2 19, 0
+    KERNEL2x8_2 20, 0
+    KERNEL2x8_2 21, 0
+    KERNEL2x8_2 22, 0
+    KERNEL2x8_2 23, 0
+    KERNEL2x8_2 24, 0
+    KERNEL2x8_2 25, 0
+    KERNEL2x8_2 26, 0
+    KERNEL2x8_2 27, 0
+    KERNEL2x8_2 28, 0
+    KERNEL2x8_2 29, 0
+    KERNEL2x8_2 30, 0
+    KERNEL2x8_2 31, 1
+    MY_ALIGN
+
+
+ZGEMM_L2x8_SUB2_32:
+/*----------------------------------------*/   
+    andi.      T1,L, 32
+    ble ZGEMM_L2x8_SUB2_16    
+    dcbt    AO, PRE
+    dcbt    BO, PRE
+    KERNEL2x8_2 0, 0
+    KERNEL2x8_2 1, 0
+    dcbt    AO, T2  
+    KERNEL2x8_2 2, 0
+    KERNEL2x8_2 3, 0
+    dcbt    AO, T3
+    dcbt    BO, T2
+    KERNEL2x8_2 4, 0
+    KERNEL2x8_2 5, 0
+    dcbt    AO, T4  
+    KERNEL2x8_2 6, 0
+    KERNEL2x8_2 7, 0
+    dcbt    AO, T5  
+    dcbt    BO, T3
+    KERNEL2x8_2 8, 0
+    KERNEL2x8_2 9, 0
+    KERNEL2x8_2 10, 0
+    KERNEL2x8_2 11, 0
+    dcbt    BO, T4
+    KERNEL2x8_2 12, 0
+    KERNEL2x8_2 13, 0
+    KERNEL2x8_2 14, 0
+    KERNEL2x8_2 15, 1
+    MY_ALIGN 
+
+
+ZGEMM_L2x8_SUB2_16:
+/*----------------------------------------*/   
+    andi.      T1,L, 16
+    ble ZGEMM_L2x8_SUB2_8
+    dcbt    AO, PRE
+    dcbt    BO, PRE
+    KERNEL2x8_2 0, 0
+    KERNEL2x8_2 1, 0
+    dcbt    AO, T2  
+    KERNEL2x8_2 2, 0
+    KERNEL2x8_2 3, 0
+    dcbt    AO, T3
+    dcbt    BO, T2
+    KERNEL2x8_2 4, 0
+    KERNEL2x8_2 5, 0
+    dcbt    AO, T4  
+    KERNEL2x8_2 6, 0
+    KERNEL2x8_2 7, 1
+    MY_ALIGN    
+
+
+ZGEMM_L2x8_SUB2_8:
+/*----------------------------------------*/   
+    andi.      T1,L, 8
+    ble ZGEMM_L2x8_SUB2_4
+    KERNEL2x8_2 0, 0
+    KERNEL2x8_2 1, 0
+    KERNEL2x8_2 2, 0
+    KERNEL2x8_2 3, 1
+    MY_ALIGN   
+
+
+ZGEMM_L2x8_SUB2_4:
+/*----------------------------------------*/   
+    andi.      T1,L, 4
+    ble ZGEMM_L2x8_SUB2_2
+    KERNEL2x8_2 0, 0
+    KERNEL2x8_2 1, 1
+    MY_ALIGN
+
+
+ZGEMM_L2x8_SUB2_2:
+/*----------------------------------------*/   
+    andi.      T1,L, 2
+    ble ZGEMM_L2x8_SUB2_1
+    KERNEL2x8_2 0, 1
+    MY_ALIGN    
+
+
+ZGEMM_L2x8_SUB2_1:
+/*----------------------------------------*/   
+    andi.      T1,L, 1
+    ble ZGEMM_L2x8_SAVE 
+    LOAD_END_2x8 128, 32
+
+
+ZGEMM_L2x8_SAVE:
+/*----------------------------------------*/   
+    addic.    I, I, -1
+    KERNEL2x8_UNPRIME_MMA
+    SAVE2x8
+#if defined(TRMMKERNEL)    
+    REFRESH_AFTER_SAVE T6,K,TEMP_REG, BO,AO, 8, 2
+#endif     
+
+    ble   ZGEMM_L2x8_SAVE_CONTINUE
+    b     ZGEMM_L2x8_BEGIN
+
+ZGEMM_L2x8_SAVE_CONTINUE:
+    andi.   T2, M, 7
+    ble   ZGEMM_L2x1_END
+    andi.   T1, M, 4
+    ble   ZGEMM_L2x4_END
+    b   ZGEMM_L2x4_BEGIN
+    MY_ALIGN 
+
+
+ZGEMM_L2x8_END:
+/*----------------------------------------*/   
+
+
+ZGEMM_L2x4_BEGIN:
+/*----------------------------------------*/   
+    andi.   T2, M, 7
+    ble   ZGEMM_L2x1_END
+    andi.   T1, M, 4
+    ble   ZGEMM_L2x4_END
+#if defined(TRMMKERNEL)   
+    REFRESH_POINTERS  AO, BO,TEMP_REG, B, 4, 2
+#else    
+    mr    BO, B   
+#endif        
+#if defined(TRMMKERNEL)   
+    REFRESH_TEMP_BK T6,K,TEMP_REG, 4, 2
+    mr T1, T6 
+    addi T1,T1, -2 
+    srawi.   T8, T1, 5 /**(T11-2) % 32x */
+#else   
+    mr T1, K 
+    addi T1,T1, -2
+    srawi.   T8, T1, 5 /**(K-2) % 32x */
+#endif     
+    KERNEL2x4_PRELOAD
+    KERNEL2x4_ZERO_AND_PRIME_MMA
+    ble   ZGEMM_L2x4_SUB0 
+    bl ZGEMM_2x4_LMAIN_SUB
+    andi.   L, T1, 31
+    ble   ZGEMM_L2x4_SAVE
+    b    ZGEMM_L2x4_SUB2
+
+
+ZGEMM_L2x4_SUB0:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)   
+    andi.   L, T6, 63
+    cmpwi   T6, 33
+#else   
+    andi.   L, K, 63
+    cmpwi   K, 33
+#endif       
+    li T8, 1
+    bne CMP2x4_32K
+    LOAD_END_2x4 64, 32
+    KERNEL2x4_PRELOAD
+    addi BO, BO, -64
+    addi AO,AO, -128
+    mtctr   T8    
+    bl ZGEMM_L2x4_K32   
+    b ZGEMM_L2x4_SAVE  
+    CMP2x4_32K:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)    
+    cmpwi   T6, 32
+#else    
+    cmpwi   K, 32
+#endif        
+    bne ZGEMM_L2x4_SUB2 
+    MY_ALIGN   
+    mtctr   T8
+    addi BO, BO, -64
+    addi AO,AO, -128
+    bl ZGEMM_L2x4_K32   
+    b ZGEMM_L2x4_SAVE 
+    MY_ALIGN 
+    MY_ALIGN 
+
+
+ZGEMM_L2x4_SUB2:
+/*----------------------------------------*/   
+    andi.      T1,L, 16
+    ble ZGEMM_L2x4_SUB2_8
+    KERNEL2x4_2 0, 0
+    KERNEL2x4_2 1, 0
+    KERNEL2x4_2 2, 0
+    KERNEL2x4_2 3, 0
+    KERNEL2x4_2 4, 0
+    KERNEL2x4_2 5, 0
+    KERNEL2x4_2 6, 0
+    KERNEL2x4_2 7, 1
+    MY_ALIGN
+
+
+ZGEMM_L2x4_SUB2_8:
+/*----------------------------------------*/   
+    andi.      T1,L, 8
+    ble ZGEMM_L2x4_SUB2_4
+    KERNEL2x4_2 0, 0
+    KERNEL2x4_2 1, 0
+    KERNEL2x4_2 2, 0
+    KERNEL2x4_2 3, 1
+    MY_ALIGN  
+
+
+ZGEMM_L2x4_SUB2_4:
+/*----------------------------------------*/   
+    andi.      T1,L, 4
+    ble ZGEMM_L2x4_SUB2_2
+    KERNEL2x4_2 0, 0
+    KERNEL2x4_2 1, 1
+    MY_ALIGN
+
+
+ZGEMM_L2x4_SUB2_2:
+/*----------------------------------------*/   
+    andi.      T1,L, 2
+    ble ZGEMM_L2x4_SUB2_1
+    KERNEL2x4_2 0, 1
+    MY_ALIGN    
+
+
+ZGEMM_L2x4_SUB2_1:
+/*----------------------------------------*/   
+    andi.      T1,L, 1
+    ble ZGEMM_L2x4_SAVE 
+    LOAD_END_2x4 64, 32
+
+
+ZGEMM_L2x4_SAVE:
+/*----------------------------------------*/   
+    KERNEL2x4_UNPRIME_MMA
+    SAVE2x4
+#if defined(TRMMKERNEL)    
+    REFRESH_AFTER_SAVE T6,K,TEMP_REG, BO,AO, 4, 2
+#endif     
+
+
+ZGEMM_L2x4_END:
+/*----------------------------------------*/   
+
+
+ZGEMM_L2x2_BEGIN:
+/*----------------------------------------*/   
+    andi.   T1, M, 2
+    ble   ZGEMM_L2x2_END
+#if defined(TRMMKERNEL)   
+    REFRESH_POINTERS  AO, BO,TEMP_REG, B, 2, 2
+#else    
+    mr    BO, B   
+#endif        
+#if defined(TRMMKERNEL)   
+    REFRESH_TEMP_BK T6,K,TEMP_REG, 2, 2
+    mr T1, T6 
+    addi T1,T1, -2 
+    srawi.   T8, T1, 5 /**(T11-2) % 32x */
+#else   
+    mr T1, K 
+    addi T1,T1, -2
+    srawi.   T8, T1, 5 /**(K-2) % 32x */
+#endif     
+    KERNEL2x2_PRELOAD
+    KERNEL2x2_ZERO_AND_PRIME_MMA
+    ble   ZGEMM_L2x2_SUB0 
+    bl ZGEMM_2x2_LMAIN_SUB
+    andi.   L, T1, 31
+    ble   ZGEMM_L2x2_SAVE
+    b   ZGEMM_L2x2_SUB2
+
+
+ZGEMM_L2x2_SUB0:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)   
+    andi.   L, T6, 63
+    cmpwi   T6, 33
+#else   
+    andi.   L, K, 63
+    cmpwi   K, 33
+#endif       
+    li T8, 1
+    bne CMP2x2_32K
+    LOAD_END_2x2 32, 32
+    KERNEL2x2_PRELOAD
+    addi BO, BO, -64
+    addi AO,AO, -64
+    mtctr   T8    
+    bl ZGEMM_L2x2_K32   
+    b ZGEMM_L2x2_SAVE  
+    CMP2x2_32K:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)    
+    cmpwi   T6, 32
+#else    
+    cmpwi   K, 32
+#endif        
+    bne ZGEMM_L2x2_SUB2 
+    MY_ALIGN   
+    mtctr   T8
+    addi BO, BO, -64
+    addi AO,AO, -64
+    bl ZGEMM_L2x2_K32   
+    b ZGEMM_L2x2_SAVE 
+    MY_ALIGN 
+    MY_ALIGN 
+
+
+ZGEMM_L2x2_SUB2:
+/*----------------------------------------*/   
+    andi.      T1,L, 16
+    ble ZGEMM_L2x2_SUB2_8
+    KERNEL2x2_2 0, 0
+    KERNEL2x2_2 1, 0  
+    KERNEL2x2_2 2, 0
+    KERNEL2x2_2 3, 0  
+    KERNEL2x2_2 4, 0
+    KERNEL2x2_2 5, 0 
+    KERNEL2x2_2 6, 0
+    KERNEL2x2_2 7, 1
+    MY_ALIGN
+
+
+ZGEMM_L2x2_SUB2_8:
+/*----------------------------------------*/   
+    andi.      T1,L, 8
+    ble ZGEMM_L2x2_SUB2_4
+    KERNEL2x2_2 0, 0
+    KERNEL2x2_2 1, 0  
+    KERNEL2x2_2 2, 0
+    KERNEL2x2_2 3, 1  
+    MY_ALIGN  
+
+
+ZGEMM_L2x2_SUB2_4:
+/*----------------------------------------*/   
+    andi.      T1,L, 4
+    ble ZGEMM_L2x2_SUB2_2
+    KERNEL2x2_2 0, 0
+    KERNEL2x2_2 1, 1
+    MY_ALIGN
+
+
+ZGEMM_L2x2_SUB2_2:
+/*----------------------------------------*/   
+    andi.      T1,L, 2
+    ble ZGEMM_L2x2_SUB2_1
+    KERNEL2x2_2 0, 1
+    MY_ALIGN    
+
+
+ZGEMM_L2x2_SUB2_1:
+/*----------------------------------------*/   
+    andi.      T1,L, 1
+    ble ZGEMM_L2x2_SAVE 
+    LOAD_END_2x2 32, 32
+
+
+ZGEMM_L2x2_SAVE:
+/*----------------------------------------*/   
+    KERNEL2x2_UNPRIME_MMA
+    SAVE2x2
+#if defined(TRMMKERNEL)    
+    REFRESH_AFTER_SAVE T6,K,TEMP_REG, BO,AO, 2, 2
+#endif     
+
+
+ZGEMM_L2x2_END:
+/*----------------------------------------*/   
+
+
+ZGEMM_L2x1_BEGIN:
+/*----------------------------------------*/   
+    andi.   T1, M, 1
+    ble   ZGEMM_L2x1_END
+#if defined(TRMMKERNEL)   
+    REFRESH_POINTERS  AO, BO,TEMP_REG, B, 1, 2
+#else    
+    mr    BO, B   
+#endif        
+#if defined(TRMMKERNEL)   
+    REFRESH_TEMP_BK T6,K,TEMP_REG, 1, 2
+    mr T1, T6 
+    addi T1,T1, -2 
+    srawi.   T8, T1, 5 /**(T11-2) % 32x */
+#else   
+    mr T1, K 
+    addi T1,T1, -2
+    srawi.   T8, T1, 5 /**(K-2) % 32x */
+#endif     
+    ZERO2x1
+    ble   ZGEMM_L2x1_SUB0 
+    bl ZGEMM_2x1_LMAIN_SUB
+    andi.   L, T1, 31
+    ble   ZGEMM_L2x1_SAVE
+    b   ZGEMM_L2x1_SUB2
+
+
+ZGEMM_L2x1_SUB0:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)   
+    andi.   L, T6, 63
+    cmpwi   T6, 33
+#else   
+    andi.   L, K, 63
+    cmpwi   K, 33
+#endif       
+    li T8, 1
+    bne CMP2x1_32K
+    addi BO, BO, -32
+    addi AO,AO, -16  
+    LOAD2x1O 16, 32 
+    END2x1_WITHOUT_ADD   
+    LOAD2x1_2O  32, 64  
+    mtctr   T8    
+    bl ZGEMM_L2x1_K32   
+    b ZGEMM_L2x1_SAVE  
+    CMP2x1_32K:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)    
+    cmpwi   T6, 32
+#else    
+    cmpwi   K, 32
+#endif        
+    bne ZGEMM_L2x1_SUB2 
+    MY_ALIGN   
+    mtctr   T8
+    addi BO, BO, -64
+    addi AO,AO, -32   
+    LOAD2x1_2O 32, 64
+    bl ZGEMM_L2x1_K32   
+    b ZGEMM_L2x1_SAVE 
+    MY_ALIGN 
+    MY_ALIGN 
+
+
+ZGEMM_L2x1_SUB2:
+/*----------------------------------------*/   
+    andi.      T1,L, 16
+    ble ZGEMM_L2x1_SUB2_8
+    LOAD2x1_2
+    KERNEL2x1_L2 32, 64, 0, 0
+    KERNEL2x1_L2 32, 64, 1, 0  
+    KERNEL2x1_L2 32, 64, 2, 0
+    KERNEL2x1_L2 32, 64, 3, 0  
+    KERNEL2x1_L2 32, 64, 4, 0
+    KERNEL2x1_L2 32, 64, 5, 0 
+    KERNEL2x1_L2 32, 64, 6, 0
+    KERNEL2x1_E2 32, 64, 7, 1
+    MY_ALIGN
+
+
+ZGEMM_L2x1_SUB2_8:
+/*----------------------------------------*/   
+    andi.      T1,L, 8
+    ble ZGEMM_L2x1_SUB2_4
+    LOAD2x1_2
+    KERNEL2x1_L2 32, 64, 0, 0
+    KERNEL2x1_L2 32, 64, 1, 0  
+    KERNEL2x1_L2 32, 64, 2, 0
+    KERNEL2x1_E2 32, 64, 3, 1  
+    MY_ALIGN  
+
+
+ZGEMM_L2x1_SUB2_4:
+/*----------------------------------------*/   
+    andi.      T1,L, 4
+    ble ZGEMM_L2x1_SUB2_2
+    LOAD2x1_2
+    KERNEL2x1_L2  32, 64, 0, 0
+    KERNEL2x1_E2  32, 64, 1, 1
+    MY_ALIGN
+
+
+ZGEMM_L2x1_SUB2_2:
+/*----------------------------------------*/   
+    andi.      T1,L, 2
+    ble ZGEMM_L2x1_SUB2_1
+    LOAD2x1_2
+    KERNEL2x1_E2  32, 64, 0, 1
+    MY_ALIGN    
+
+
+ZGEMM_L2x1_SUB2_1:
+/*----------------------------------------*/   
+    andi.      T1,L, 1
+    ble ZGEMM_L2x1_SAVE 
+    KERNEL2x1
+
+
+ZGEMM_L2x1_SAVE:
+/*----------------------------------------*/   
+    SAVE2x1
+#if defined(TRMMKERNEL)    
+    REFRESH_AFTER_SAVE T6,K,TEMP_REG, BO,AO, 1, 2
+#endif   
+
+
+ZGEMM_L2x1_END:
+/*----------------------------------------*/   
+    slwi    T1, K, 5
+    addic.    J, J, -1
+    add   B, B, T1
+#if defined(TRMMKERNEL) && !defined(LEFT)   
+    addi TEMP_REG, TEMP_REG, 2 
+#endif   
+    ble   ZGEMM_L2_END 
+    b     ZGEMM_L2_BEGIN
+
+ZGEMM_L2_END:
+
+b ZGEMM_L1
+/*                MINI SUBROUTINES                            */      
+/*                1x8 MAIN 128x+2 LOOP                     */      
+
+
+ZGEMM_L1x8_LMAIN_SUB:
+/*----------------------------------------*/   
+    mtctr   T8
+    MY_ALIGN
+ZGEMM_L1x8_LOOP:
+/*----------------------------------------*/   
+    dcbt    AO, PRE
+    dcbt    BO, PRE
+    KERNEL1x8_2 0, 0
+ZGEMM_L1x8_K128:
+/*----------------------------------------*/   
+    KERNEL1x8_2 1, 0
+    dcbt    AO, T2  
+    KERNEL1x8_2 2, 0
+    KERNEL1x8_2 3, 0
+    dcbt    AO, T3
+    dcbt    BO, T2
+    KERNEL1x8_2 4, 0
+    KERNEL1x8_2 5, 0
+    dcbt    AO, T4  
+    KERNEL1x8_2 6, 0
+    KERNEL1x8_2 7, 0
+    dcbt    AO, T5  
+    dcbt    BO, T3
+    KERNEL1x8_2 8, 0
+    KERNEL1x8_2 9, 0
+    KERNEL1x8_2 10, 0
+    KERNEL1x8_2 11, 0
+    dcbt    BO, T4
+    KERNEL1x8_2 12, 0
+    KERNEL1x8_2 13, 0
+    KERNEL1x8_2 14, 0
+    KERNEL1x8_2 15, 0
+    KERNEL1x8_2 16, 0
+    KERNEL1x8_2 17, 0
+    KERNEL1x8_2 18, 0
+    KERNEL1x8_2 19, 0
+    KERNEL1x8_2 20, 0
+    KERNEL1x8_2 21, 0
+    KERNEL1x8_2 22, 0
+    KERNEL1x8_2 23, 0
+    KERNEL1x8_2 24, 0
+    KERNEL1x8_2 25, 0
+    KERNEL1x8_2 26, 0
+    KERNEL1x8_2 27, 0
+    KERNEL1x8_2 28, 0
+    KERNEL1x8_2 29, 0
+    KERNEL1x8_2 30, 0
+    KERNEL1x8_2 31, 0
+    KERNEL1x8_2 32, 0
+    KERNEL1x8_2 33, 0
+    KERNEL1x8_2 34, 0
+    KERNEL1x8_2 35, 0
+    KERNEL1x8_2 36, 0
+    KERNEL1x8_2 37, 0
+    KERNEL1x8_2 38, 0
+    KERNEL1x8_2 39, 0
+    KERNEL1x8_2 40, 0
+    KERNEL1x8_2 41, 0
+    KERNEL1x8_2 42, 0
+    KERNEL1x8_2 43, 0
+    KERNEL1x8_2 44, 0
+    KERNEL1x8_2 45, 0
+    KERNEL1x8_2 46, 0
+    KERNEL1x8_2 47, 0
+    KERNEL1x8_2 48, 0
+    KERNEL1x8_2 49, 0
+    KERNEL1x8_2 50, 0
+    KERNEL1x8_2 51, 0
+    KERNEL1x8_2 52, 0
+    KERNEL1x8_2 53, 0
+    KERNEL1x8_2 54, 0
+    KERNEL1x8_2 55, 0
+    KERNEL1x8_2 56, 0
+    KERNEL1x8_2 57, 0
+    KERNEL1x8_2 58, 0
+    KERNEL1x8_2 59, 0
+    KERNEL1x8_2 60, 0
+    KERNEL1x8_2 61, 0
+    KERNEL1x8_2 62, 0
+    KERNEL1x8_2 63, 1
+    bdnz    ZGEMM_L1x8_LOOP
+    MY_ALIGN  
+ZGEMM_L1x8_LOOP_END:
+/*----------------------------------------*/   
+    KERNEL1x8_2 0, 1
+    blr
+    MY_ALIGN
+
+
+ZGEMM_1x4_LMAIN_SUB:
+/*----------------------------------------*/   
+    mtctr   T8
+    MY_ALIGN
+
+
+ZGEMM_L1x4_LOOP:
+/*----------------------------------------*/   
+    KERNEL1x4_2 0, 0
+
+
+ZGEMM_L1x4_K32:
+/*----------------------------------------*/   
+    KERNEL1x4_2 1, 0
+    KERNEL1x4_2 2, 0
+    KERNEL1x4_2 3, 0
+    KERNEL1x4_2 4, 0
+    KERNEL1x4_2 5, 0
+    KERNEL1x4_2 6, 0
+    KERNEL1x4_2 7, 0
+    KERNEL1x4_2 8, 0
+    KERNEL1x4_2 9, 0
+    KERNEL1x4_2 10, 0
+    KERNEL1x4_2 11, 0
+    KERNEL1x4_2 12, 0
+    KERNEL1x4_2 13, 0
+    KERNEL1x4_2 14, 0
+    KERNEL1x4_2 15, 1
+    bdnz    ZGEMM_L1x4_LOOP
+    MY_ALIGN  
+
+
+ZGEMM_L1x4_LOOP_END:
+/*----------------------------------------*/   
+    KERNEL1x4_2 0, 1
+    blr
+    MY_ALIGN
+
+
+ZGEMM_1x2_LMAIN_SUB:
+/*----------------------------------------*/   
+    mtctr   T8
+    MY_ALIGN
+
+
+ZGEMM_L1x2_LOOP:
+/*----------------------------------------*/   
+    KERNEL1x2_2 0, 0
+
+
+ZGEMM_L1x2_K32:
+/*----------------------------------------*/   
+    KERNEL1x2_2 1, 0
+    KERNEL1x2_2 2, 0
+    KERNEL1x2_2 3, 0
+    KERNEL1x2_2 4, 0
+    KERNEL1x2_2 5, 0
+    KERNEL1x2_2 6, 0
+    KERNEL1x2_2 7, 0
+    KERNEL1x2_2 8, 0
+    KERNEL1x2_2 9, 0
+    KERNEL1x2_2 10, 0
+    KERNEL1x2_2 11, 0
+    KERNEL1x2_2 12, 0
+    KERNEL1x2_2 13, 0
+    KERNEL1x2_2 14, 0
+    KERNEL1x2_2 15, 1
+    bdnz    ZGEMM_L1x2_LOOP
+    MY_ALIGN  
+
+
+ZGEMM_L1x2_LOOP_END:
+/*----------------------------------------*/   
+    KERNEL1x2_2 0, 1
+    blr
+    MY_ALIGN
+
+
+ZGEMM_1x1_LMAIN_SUB:
+/*----------------------------------------*/   
+    mtctr   T8
+    LOAD1x1_2  
+    MY_ALIGN
+
+
+ZGEMM_L1x1_LOOP:
+/*----------------------------------------*/   
+    KERNEL1x1_L2 32, 32, 0, 0
+
+
+ZGEMM_L1x1_K32:
+/*----------------------------------------*/   
+    KERNEL1x1_L2 32, 32, 1, 0  
+    KERNEL1x1_L2 32, 32, 2, 0
+    KERNEL1x1_L2 32, 32, 3, 0  
+    KERNEL1x1_L2 32, 32, 4, 0
+    KERNEL1x1_L2 32, 32, 5, 0 
+    KERNEL1x1_L2 32, 32, 6, 0
+    KERNEL1x1_L2 32, 32, 7, 0
+    KERNEL1x1_L2 32, 32, 8, 0
+    KERNEL1x1_L2 32, 32, 9, 0  
+    KERNEL1x1_L2 32, 32, 10, 0
+    KERNEL1x1_L2 32, 32, 11, 0  
+    KERNEL1x1_L2 32, 32, 12, 0
+    KERNEL1x1_L2 32, 32, 13, 0 
+    KERNEL1x1_L2 32, 32, 14, 0
+    KERNEL1x1_L2 32, 32, 15, 1   
+    bdnz    ZGEMM_L1x1_LOOP
+    MY_ALIGN  
+
+
+ZGEMM_L1x1_LOOP_END:
+/*----------------------------------------*/   
+    END1x1_2 
+    blr
+    MY_ALIGN
+
+
+/*----------------------N1 BEGINS---------*/
+ZGEMM_L1:
+/*----------------------------------------*/   
+    andi.   T1, N, 1
+    ble   ZGEMM_L1_END
+		
+ZGEMM_L1_BEGIN:
+/*----------------------------------------*/   
+    mr    CO, C
+   
+    add     T2,C,LDC    
+    mr    AO, A  
+    add   C, C, T1
+#if defined(TRMMKERNEL) && defined(LEFT)   
+    mr TEMP_REG, OFFSET  /*off = offset;*/
+#endif     
+    srawi.    I, M, 3
+    ble   ZGEMM_L1x8_END
+    dcbt    CO,r0  /*just prefetch*/
+    dcbt    T2,r0    
+
+
+ZGEMM_L1x8_BEGIN:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)   
+    REFRESH_POINTERS  AO, BO,TEMP_REG, B, 8, 1
+#else    
+    mr    BO, B  
+    dcbt    B, r0  
+#endif     
+    dcbt    AO, r0
+#if defined(TRMMKERNEL)   
+    REFRESH_TEMP_BK T6,K,TEMP_REG, 8, 1
+    mr T1, T6
+/* TEMPS FOR PREFETCH */   
+    li T2, 1024
+    li T3, 1024+512
+    addi T1,T1, -2
+/* TEMPS FOR PREFETCH */     
+    li T4, 2048
+    li T5, 2048+512   
+    srawi.   T8, T1, 7 /**(T11-2) % 128x */
+#else   
+    mr T1, K
+/* TEMPS FOR PREFETCH */   
+    li T2, 1024
+    li T3, 1024+512
+    addi T1,T1, -2
+/* TEMPS FOR PREFETCH */     
+    li T4, 2048
+    li T5, 2048+512 
+    srawi.   T8, T1, 7 /**(K-2) % 128x */
+#endif   
+    KERNEL1x8_ZERO_AND_PRIME_MMA
+    ble   ZGEMM_L1x8_SUB0
+    bl ZGEMM_L1x8_LMAIN_SUB
+    andi.   L, T1, 127
+    ble   ZGEMM_L1x8_SAVE
+    b   ZGEMM_L1x8_SUB2
+
+
+ZGEMM_L1x8_SUB0:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)   
+    andi.   L, T6, 255
+    cmpwi   T6, 129
+#else   
+    andi.   L, K, 255
+    cmpwi   K, 129
+#endif       
+    li T8, 1
+    bne CMP1x8_128K
+    LOAD_END_1x8 -128, -16
+    mtctr   T8    
+    bl ZGEMM_L1x8_K128   
+    b ZGEMM_L1x8_SAVE  
+    CMP1x8_128K:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)    
+    cmpwi   T6, 128
+#else    
+    cmpwi   K, 128
+#endif        
+    bne ZGEMM_L1x8_SUB2 
+    MY_ALIGN   
+    mtctr   T8
+    addi BO, BO, -32
+    addi AO,AO, -256   
+    bl ZGEMM_L1x8_K128   
+    b ZGEMM_L1x8_SAVE 
+    MY_ALIGN
+
+
+ZGEMM_L1x8_SUB2:
+/*----------------------------------------*/   
+    andi.      T1,L, 64
+    ble ZGEMM_L1x8_SUB2_32
+    dcbt    AO, PRE
+    dcbt    BO, PRE
+    KERNEL1x8_2 0, 0
+    KERNEL1x8_2 1, 0
+    dcbt    AO, T2  
+    KERNEL1x8_2 2, 0
+    KERNEL1x8_2 3, 0
+    dcbt    AO, T3
+    dcbt    BO, T2
+    KERNEL1x8_2 4, 0
+    KERNEL1x8_2 5, 0
+    dcbt    AO, T4  
+    KERNEL1x8_2 6, 0
+    KERNEL1x8_2 7, 0
+    dcbt    AO, T5  
+    dcbt    BO, T3
+    KERNEL1x8_2 8, 0
+    KERNEL1x8_2 9, 0
+    KERNEL1x8_2 10, 0
+    KERNEL1x8_2 11, 0
+    dcbt    BO, T4
+    KERNEL1x8_2 12, 0
+    KERNEL1x8_2 13, 0
+    KERNEL1x8_2 14, 0
+    KERNEL1x8_2 15, 0
+    KERNEL1x8_2 16, 0
+    KERNEL1x8_2 17, 0
+    KERNEL1x8_2 18, 0
+    KERNEL1x8_2 19, 0
+    KERNEL1x8_2 20, 0
+    KERNEL1x8_2 21, 0
+    KERNEL1x8_2 22, 0
+    KERNEL1x8_2 23, 0
+    KERNEL1x8_2 24, 0
+    KERNEL1x8_2 25, 0
+    KERNEL1x8_2 26, 0
+    KERNEL1x8_2 27, 0
+    KERNEL1x8_2 28, 0
+    KERNEL1x8_2 29, 0
+    KERNEL1x8_2 30, 0
+    KERNEL1x8_2 31, 1
+    MY_ALIGN
+
+
+ZGEMM_L1x8_SUB2_32:
+/*----------------------------------------*/   
+    andi.      T1,L, 32
+    ble ZGEMM_L1x8_SUB2_16    
+    dcbt    AO, PRE
+    dcbt    BO, PRE
+    KERNEL1x8_2 0, 0
+    KERNEL1x8_2 1, 0
+    dcbt    AO, T2  
+    KERNEL1x8_2 2, 0
+    KERNEL1x8_2 3, 0
+    dcbt    AO, T3
+    dcbt    BO, T2
+    KERNEL1x8_2 4, 0
+    KERNEL1x8_2 5, 0
+    dcbt    AO, T4  
+    KERNEL1x8_2 6, 0
+    KERNEL1x8_2 7, 0
+    dcbt    AO, T5  
+    dcbt    BO, T3
+    KERNEL1x8_2 8, 0
+    KERNEL1x8_2 9, 0
+    KERNEL1x8_2 10, 0
+    KERNEL1x8_2 11, 0
+    dcbt    BO, T4
+    KERNEL1x8_2 12, 0
+    KERNEL1x8_2 13, 0
+    KERNEL1x8_2 14, 0
+    KERNEL1x8_2 15, 1
+    MY_ALIGN 
+
+
+ZGEMM_L1x8_SUB2_16:
+/*----------------------------------------*/   
+    andi.      T1,L, 16
+    ble ZGEMM_L1x8_SUB2_8
+    dcbt    AO, PRE
+    dcbt    BO, PRE
+    KERNEL1x8_2 0, 0
+    KERNEL1x8_2 1, 0
+    dcbt    AO, T2  
+    KERNEL1x8_2 2, 0
+    KERNEL1x8_2 3, 0
+    dcbt    AO, T3
+    dcbt    BO, T2
+    KERNEL1x8_2 4, 0
+    KERNEL1x8_2 5, 0
+    dcbt    AO, T4  
+    KERNEL1x8_2 6, 0
+    KERNEL1x8_2 7, 1
+    MY_ALIGN    
+
+
+ZGEMM_L1x8_SUB2_8:
+/*----------------------------------------*/   
+    andi.      T1,L, 8
+    ble ZGEMM_L1x8_SUB2_4
+    KERNEL1x8_2 0, 0
+    KERNEL1x8_2 1, 0
+    KERNEL1x8_2 2, 0
+    KERNEL1x8_2 3, 1
+    MY_ALIGN   
+
+
+ZGEMM_L1x8_SUB2_4:
+/*----------------------------------------*/   
+    andi.      T1,L, 4
+    ble ZGEMM_L1x8_SUB2_2
+    KERNEL1x8_2 0, 0
+    KERNEL1x8_2 1, 1
+    MY_ALIGN
+
+
+ZGEMM_L1x8_SUB2_2:
+/*----------------------------------------*/   
+    andi.      T1,L, 2
+    ble ZGEMM_L1x8_SUB2_1
+    KERNEL1x8_2 0, 1
+    MY_ALIGN    
+
+
+ZGEMM_L1x8_SUB2_1:
+/*----------------------------------------*/   
+    andi.      T1,L, 1
+    ble ZGEMM_L1x8_SAVE 
+    LOAD_END_1x8 128, 16
+
+
+ZGEMM_L1x8_SAVE:
+/*----------------------------------------*/   
+    addic.    I, I, -1
+    KERNEL1x8_UNPRIME_MMA
+    SAVE1x8
+#if defined(TRMMKERNEL)    
+    REFRESH_AFTER_SAVE T6,K,TEMP_REG, BO,AO, 8, 1
+#endif     
+    bgt   ZGEMM_L1x8_BEGIN
+    andi.   T2, M, 7
+    ble   ZGEMM_L1x1_END
+    andi.   T1, M, 4
+    ble   ZGEMM_L1x4_END
+    b   ZGEMM_L1x4_BEGIN
+    MY_ALIGN 
+
+
+ZGEMM_L1x8_END:
+/*----------------------------------------*/   
+
+
+ZGEMM_L1x4_BEGIN:
+/*----------------------------------------*/   
+    andi.   T2, M, 7
+    ble   ZGEMM_L1x1_END
+    andi.   T1, M, 4
+    ble   ZGEMM_L1x4_END
+#if defined(TRMMKERNEL)   
+    REFRESH_POINTERS  AO, BO,TEMP_REG, B, 4, 1
+#else    
+    mr    BO, B   
+#endif        
+#if defined(TRMMKERNEL)   
+    REFRESH_TEMP_BK T6,K,TEMP_REG, 4, 1
+    mr T1, T6 
+    addi T1,T1, -2 
+    srawi.   T8, T1, 5 /**(T11-2) % 32x */
+#else   
+    mr T1, K 
+    addi T1,T1, -2
+    srawi.   T8, T1, 5 /**(K-2) % 32x */
+#endif     
+    KERNEL1x4_ZERO_AND_PRIME_MMA
+    ble   ZGEMM_L1x4_SUB0 
+    bl ZGEMM_1x4_LMAIN_SUB
+    andi.   L, T1, 31
+    ble   ZGEMM_L1x4_SAVE
+    b   ZGEMM_L1x4_SUB2
+
+
+ZGEMM_L1x4_SUB0:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)   
+    andi.   L, T6, 63
+    cmpwi   T6, 33
+#else   
+    andi.   L, K, 63
+    cmpwi   K, 33
+#endif       
+    li T8, 1
+    bne CMP1x4_32K
+    LOAD_END_1x4 -64, -16 
+    mtctr   T8    
+    bl ZGEMM_L1x4_K32   
+    b ZGEMM_L1x4_SAVE  
+    CMP1x4_32K:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)    
+    cmpwi   T6, 32
+#else    
+    cmpwi   K, 32
+#endif        
+    bne ZGEMM_L1x4_SUB2 
+    MY_ALIGN   
+    mtctr   T8
+    addi BO, BO, -32
+    addi AO,AO, -128   
+    bl ZGEMM_L1x4_K32   
+    b ZGEMM_L1x4_SAVE 
+    MY_ALIGN 
+    MY_ALIGN 
+
+
+ZGEMM_L1x4_SUB2:
+/*----------------------------------------*/   
+    andi.      T1,L, 16
+    ble ZGEMM_L1x4_SUB2_8
+    KERNEL1x4_2 0, 0
+    KERNEL1x4_2 1, 0
+    KERNEL1x4_2 2, 0
+    KERNEL1x4_2 3, 0
+    KERNEL1x4_2 4, 0
+    KERNEL1x4_2 5, 0
+    KERNEL1x4_2 6, 0
+    KERNEL1x4_2 7, 1
+    MY_ALIGN
+
+
+ZGEMM_L1x4_SUB2_8:
+/*----------------------------------------*/   
+    andi.      T1,L, 8
+    ble ZGEMM_L1x4_SUB2_4
+    KERNEL1x4_2 0, 0
+    KERNEL1x4_2 1, 0
+    KERNEL1x4_2 2, 0
+    KERNEL1x4_2 3, 1
+    MY_ALIGN  
+
+
+ZGEMM_L1x4_SUB2_4:
+/*----------------------------------------*/   
+    andi.      T1,L, 4
+    ble ZGEMM_L1x4_SUB2_2
+    KERNEL1x4_2 0, 0
+    KERNEL1x4_2 1, 1
+    MY_ALIGN
+
+
+ZGEMM_L1x4_SUB2_2:
+/*----------------------------------------*/   
+    andi.      T1,L, 2
+    ble ZGEMM_L1x4_SUB2_1
+    KERNEL1x4_2 0, 1
+    MY_ALIGN    
+
+
+ZGEMM_L1x4_SUB2_1:
+/*----------------------------------------*/   
+    andi.      T1,L, 1
+    ble ZGEMM_L1x4_SAVE 
+    LOAD_END_1x4 64,16
+
+
+
+ZGEMM_L1x4_SAVE:
+/*----------------------------------------*/   
+    KERNEL1x4_UNPRIME_MMA
+    SAVE1x4
+#if defined(TRMMKERNEL)    
+    REFRESH_AFTER_SAVE T6,K,TEMP_REG, BO,AO, 4, 1
+#endif     
+
+
+ZGEMM_L1x4_END:
+/*----------------------------------------*/   
+
+
+ZGEMM_L1x2_BEGIN:
+/*----------------------------------------*/   
+    andi.   T1, M, 2
+    ble   ZGEMM_L1x2_END
+#if defined(TRMMKERNEL)   
+    REFRESH_POINTERS  AO, BO,TEMP_REG, B, 2, 1
+#else    
+    mr    BO, B   
+#endif        
+#if defined(TRMMKERNEL)   
+    REFRESH_TEMP_BK T6,K,TEMP_REG, 2, 1
+    mr T1, T6 
+    addi T1,T1, -2 
+    srawi.   T8, T1, 5 /**(T11-2) % 32x */
+#else   
+    mr T1, K 
+    addi T1,T1, -2
+    srawi.   T8, T1, 5 /**(K-2) % 32x */
+#endif     
+    KERNEL1x2_ZERO_AND_PRIME_MMA
+    ble   ZGEMM_L1x2_SUB0 
+    bl ZGEMM_1x2_LMAIN_SUB
+    andi.   L, T1, 31
+    ble   ZGEMM_L1x2_SAVE
+    b   ZGEMM_L1x2_SUB2
+
+
+ZGEMM_L1x2_SUB0:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)   
+    andi.   L, T6, 63
+    cmpwi   T6, 33
+#else   
+    andi.   L, K, 63
+    cmpwi   K, 33
+#endif       
+    li T8, 1
+    bne CMP1x2_32K
+    LOAD_END_1x2 -32, -16 
+    mtctr   T8    
+    bl ZGEMM_L1x2_K32   
+    b ZGEMM_L1x2_SAVE  
+    CMP1x2_32K:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)    
+    cmpwi   T6, 32
+#else    
+    cmpwi   K, 32
+#endif        
+    bne ZGEMM_L1x2_SUB2 
+    MY_ALIGN   
+    mtctr   T8
+    addi BO, BO, -32
+    addi AO,AO, -64   
+    bl ZGEMM_L1x2_K32   
+    b ZGEMM_L1x2_SAVE 
+    MY_ALIGN 
+    MY_ALIGN 
+
+
+ZGEMM_L1x2_SUB2:
+/*----------------------------------------*/   
+    andi.      T1,L, 16
+    ble ZGEMM_L1x2_SUB2_8
+    KERNEL1x2_2 0, 0
+    KERNEL1x2_2 1, 0
+    KERNEL1x2_2 2, 0
+    KERNEL1x2_2 3, 0
+    KERNEL1x2_2 4, 0
+    KERNEL1x2_2 5, 0
+    KERNEL1x2_2 6, 0
+    KERNEL1x2_2 7, 1
+    MY_ALIGN
+
+
+ZGEMM_L1x2_SUB2_8:
+/*----------------------------------------*/   
+    andi.      T1,L, 8
+    ble ZGEMM_L1x2_SUB2_4
+    KERNEL1x2_2 0, 0
+    KERNEL1x2_2 1, 0
+    KERNEL1x2_2 2, 0
+    KERNEL1x2_2 3, 1
+    MY_ALIGN  
+
+
+ZGEMM_L1x2_SUB2_4:
+/*----------------------------------------*/   
+    andi.      T1,L, 4
+    ble ZGEMM_L1x2_SUB2_2
+    KERNEL1x2_2 0, 0
+    KERNEL1x2_2 1, 1
+    MY_ALIGN
+
+
+ZGEMM_L1x2_SUB2_2:
+/*----------------------------------------*/   
+    andi.      T1,L, 2
+    ble ZGEMM_L1x2_SUB2_1
+    KERNEL1x2_2 0, 1
+    MY_ALIGN    
+
+
+ZGEMM_L1x2_SUB2_1:
+/*----------------------------------------*/   
+    andi.      T1,L, 1
+    ble ZGEMM_L1x2_SAVE 
+    LOAD_END_1x2 32,16
+
+
+ZGEMM_L1x2_SAVE:
+/*----------------------------------------*/   
+    KERNEL1x2_UNPRIME_MMA
+    SAVE1x2
+#if defined(TRMMKERNEL)    
+    REFRESH_AFTER_SAVE T6,K,TEMP_REG, BO,AO, 2, 1
+#endif     
+
+
+ZGEMM_L1x2_END:
+/*----------------------------------------*/   
+
+
+ZGEMM_L1x1_BEGIN:
+/*----------------------------------------*/   
+    andi.   T1, M, 1
+    ble   ZGEMM_L1x1_END
+#if defined(TRMMKERNEL)   
+    REFRESH_POINTERS  AO, BO,TEMP_REG, B, 1, 1
+#else    
+    mr    BO, B   
+#endif        
+#if defined(TRMMKERNEL)   
+    REFRESH_TEMP_BK T6,K,TEMP_REG, 1, 1
+    mr T1, T6 
+    addi T1,T1, -2 
+    srawi.   T8, T1, 5 /**(T11-2) % 32x */
+#else   
+    mr T1, K 
+    addi T1,T1, -2
+    srawi.   T8, T1, 5 /**(K-2) % 32x */
+#endif     
+    ZERO1x1
+    ble   ZGEMM_L1x1_SUB0 
+    bl ZGEMM_1x1_LMAIN_SUB
+    andi.   L, T1, 31
+    ble   ZGEMM_L1x1_SAVE
+    b   ZGEMM_L1x1_SUB2
+
+
+ZGEMM_L1x1_SUB0:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)   
+    andi.   L, T6, 63
+    cmpwi   T6, 33
+#else   
+    andi.   L, K, 63
+    cmpwi   K, 33
+#endif       
+    li T8, 1
+    bne CMP1x1_32K
+    addi BO, BO, -16
+    addi AO,AO, -16  
+    LOAD1x1O 16, 16 
+    END1x1_WITHOUT_ADD   
+    LOAD1x1_2O  32, 32  
+    mtctr   T8    
+    bl ZGEMM_L1x1_K32   
+    b ZGEMM_L1x1_SAVE  
+    CMP1x1_32K:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL)    
+    cmpwi   T6, 32
+#else    
+    cmpwi   K, 32
+#endif        
+    bne ZGEMM_L1x1_SUB2 
+    MY_ALIGN   
+    mtctr   T8
+    addi BO, BO, -32
+    addi AO,AO, -32   
+    LOAD1x1_2O 32, 32
+    bl ZGEMM_L1x1_K32   
+    b ZGEMM_L1x1_SAVE 
+    MY_ALIGN 
+
+
+ZGEMM_L1x1_SUB2:
+/*----------------------------------------*/   
+    andi.      T1, L, 16
+    ble ZGEMM_L1x1_SUB2_8
+    LOAD1x1_2
+    KERNEL1x1_L2 32, 32, 0, 0
+    KERNEL1x1_L2 32, 32, 1, 0  
+    KERNEL1x1_L2 32, 32, 2, 0
+    KERNEL1x1_L2 32, 32, 3, 0  
+    KERNEL1x1_L2 32, 32, 4, 0
+    KERNEL1x1_L2 32, 32, 5, 0 
+    KERNEL1x1_L2 32, 32, 6, 0
+    KERNEL1x1_E2 32, 32, 7, 1
+    MY_ALIGN
+
+
+ZGEMM_L1x1_SUB2_8:
+/*----------------------------------------*/   
+    andi.      T1, L, 8
+    ble ZGEMM_L1x1_SUB2_4
+    LOAD1x1_2
+    KERNEL1x1_L2 32, 32, 0, 0
+    KERNEL1x1_L2 32, 32, 1, 0  
+    KERNEL1x1_L2 32, 32, 2, 0
+    KERNEL1x1_E2 32, 32, 3, 1  
+    MY_ALIGN  
+
+
+ZGEMM_L1x1_SUB2_4:
+/*----------------------------------------*/   
+    andi.      T1,L, 4
+    ble ZGEMM_L1x1_SUB2_2
+    LOAD1x1_2
+    KERNEL1x1_L2  32, 32, 0, 0
+    KERNEL1x1_E2  32, 32, 1, 1
+    MY_ALIGN
+
+
+ZGEMM_L1x1_SUB2_2:
+/*----------------------------------------*/   
+    andi.      T1,L, 2
+    ble ZGEMM_L1x1_SUB2_1
+    LOAD1x1_2
+    KERNEL1x1_E2  32, 32, 0, 1
+    MY_ALIGN    
+
+
+ZGEMM_L1x1_SUB2_1:
+/*----------------------------------------*/   
+    andi.      T1,L, 1
+    ble ZGEMM_L1x1_SAVE 
+    KERNEL1x1
+
+
+ZGEMM_L1x1_SAVE:
+/*----------------------------------------*/   
+    SAVE1x1
+#if defined(TRMMKERNEL)    
+    REFRESH_AFTER_SAVE T6,K,TEMP_REG, BO,AO, 1, 1
+#endif   
+
+
+ZGEMM_L1x1_END:
+/*----------------------------------------*/   
+#if defined(TRMMKERNEL) && !defined(LEFT)   
+    addi TEMP_REG, TEMP_REG, 1
+#endif   
+
+
+ZGEMM_L1_END:
+/*----------------------------------------*/   
diff --git a/kernel/power/zgemm_macros_power10.S b/kernel/power/zgemm_macros_power10.S
new file mode 100644
index 000000000..42f9c5ad4
--- /dev/null
+++ b/kernel/power/zgemm_macros_power10.S
@@ -0,0 +1,1138 @@
+/***************************************************************************
+Copyright (c) 2013-2020, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#define unit_size 16
+#define DISP32(ind,disp) (ind*unit_size*32+disp)
+#define DISP16(ind,disp) (ind*unit_size*16+disp)
+#define DISP8(ind,disp) (ind*unit_size*8+disp)
+#define DISP4(ind,disp) (ind*unit_size*4+disp)
+#define DISP2(ind,disp) (ind*unit_size*2+disp)
+#define DISP1(ind,disp) (ind*unit_size+disp)
+#define DISPX(disp)  (disp)
+/*	HELPERS FOR SAVE	*/
+/* {r0,i0} and {r1,i1} into  {r0,r1} {i0,i1} */
+
+
+.macro LOAD_COUPLE_AS_RR_II  VS_OUT1,VS_OUT2,VS_TEMP1,VS_TEMP2,REG,LOFFSET 
+#ifndef TRMMKERNEL 
+  lxv	\VS_TEMP1,	DISPX(\LOFFSET)(\REG)
+  lxv	\VS_TEMP2,	DISPX(\LOFFSET+16)(\REG)
+  xxmrgld  \VS_OUT1,\VS_TEMP1,\VS_TEMP2
+  xxmrghd  \VS_OUT2,\VS_TEMP1,\VS_TEMP2	
+#endif	
+.endm
+/*from 2 result {a0r*br,a0i*bi} and {a1r*br,a1i*bi} pack into {a0r*br,a1r*br} and {a0i*bi,a1i*bi}*/
+
+
+.macro RESULT_INTO_REALREAL_IMAGEIMAGE VSIN1,VSIN2,VSOUT1,VSOUT2
+	xxmrgld	\VSOUT1, \VSIN1,\VSIN2 /*  real*real from 2 results*/
+	xxmrghd	\VSOUT2, \VSIN1,\VSIN2 /*  imag*imag from 2 results*/
+.endm 
+/*from 2 result {a0r*bi,a0i*br} and {a1r*bi,a1i*br} pack into {a0r*bi,a1r*bi} and {a0i*br,a1i*br}*/
+
+
+.macro RESULT_INTO_REALIMAG_IMAGREAL VSIN1,VSIN2,VSOUT1,VSOUT2 
+	xxmrgld	\VSOUT1, \VSIN1,\VSIN2 /*  real*imag */
+	xxmrghd	\VSOUT2, \VSIN1,\VSIN2 /*  imag*real*/
+.endm
+/* {a0r*br op a0i*bi ,a1r*br op a1i*bi} ~ {r0,r1}; {a0r*bi op a0i*br ,a1r*bi op a1i*br} ~ {i0,i1}*/
+
+
+.macro  AGGREGATE_REALS_IMAGES  VSINR_OUT1,VSINR,VSINI_OUT2,VSINI
+#if	defined(NN) || defined(NT) || defined(TN) || defined(TT) 
+	xvsubdp  \VSINR_OUT1,\VSINR_OUT1,\VSINR
+	xvadddp  \VSINI_OUT2,\VSINI_OUT2,\VSINI  
+#elif  defined(CN) || defined(CT) || defined(RN) || defined(RT) 
+	xvadddp  \VSINR_OUT1,\VSINR_OUT1,\VSINR
+	xvsubdp  \VSINI_OUT2,\VSINI_OUT2,\VSINI 
+#elif  defined(NC) || defined(TC) || defined(NR) || defined(TR) 
+	xvadddp  \VSINR_OUT1,\VSINR_OUT1,\VSINR
+	xvsubdp  \VSINI_OUT2,\VSINI,\VSINI_OUT2  
+#else	// CC || CR || RC || RR 
+    /*we will assume {-alpha_r,-alpha_i} for this case */
+    /*i1i2-r1r2 so we will negate alpha real instead to fix sign*/
+	xvsubdp  \VSINR_OUT1,\VSINR,\VSINR_OUT1
+    /*we will negate alpha image instead  instead to fix sign*/
+	xvadddp  \VSINI_OUT2,\VSINI_OUT2,\VSINI 
+#endif
+.endm 
+/* {i0,i1} * {alpha_i,alpha_i} - VSOUT1 ;VSOUT2 + {r0,r1}*{alpha_i,alpha_i} */
+
+
+.macro MULT_APLHA_PART1  VSINRR,VSINII,VSOUT1,VSOUT2
+#ifndef TRMMKERNEL  
+	xvmsubadp \VSOUT1,\VSINII, alpha_i
+	xvmaddadp  \VSOUT2,\VSINRR, alpha_i
+#else 
+	xvmuldp \VSOUT1,\VSINII, alpha_i 
+	xvmuldp  \VSOUT2,\VSINRR, alpha_i
+#endif 
+.endm
+/*   {r0,r1} * {alpha_r,alpha_r} -  VSOUT1 ;VSOUT2 + {i0,i1} * {alpha_r,alpha_r} */
+
+
+.macro MULT_APLHA_PART2  VSINRR,VSINII,VSOUT1,VSOUT2 
+	xvmsubadp  \VSOUT1,\VSINRR, alpha_r
+	xvmaddadp \VSOUT2,\VSINII, alpha_r
+.endm
+/* unpack to store 2{r,r} {i,i} into  {r,i} {r,i} (big endian because of stxv) */
+
+
+.macro UNPACK_FOR_STORE VSIN1,VSIN2,VSOUT1,VSOUT2 
+	xxmrghd  \VSOUT1,\VSIN2,\VSIN1
+	xxmrgld  \VSOUT2,\VSIN2,\VSIN1
+.endm
+
+
+.macro STORE_COUPLE REG,LOFFSET,VSIN1,VSIN2
+	stxv	\VSIN1,	DISPX(\LOFFSET)(\REG)
+	stxv	\VSIN2,	DISPX(\LOFFSET+16)(\REG)
+.endm
+
+
+.macro SAVE8 VSRes1,VSRes2,VSRes3,VSRes4,VSRes5,VSRes6,VSRes7,VSRes8,VSRes9,VSRes10,VSRes11,VSRes12,VSRes13,VSRes14,VSRes15,VSRes16,BASE_REG,LOFFSET
+  RESULT_INTO_REALREAL_IMAGEIMAGE \VSRes1,\VSRes3,vs34,vs35
+  LOAD_COUPLE_AS_RR_II	vs46,vs47,vs50,vs51,\BASE_REG,\LOFFSET
+  RESULT_INTO_REALIMAG_IMAGREAL	\VSRes2,\VSRes4,vs36,vs37
+  LOAD_COUPLE_AS_RR_II	vs48,vs49,vs52,vs53,\BASE_REG,(\LOFFSET+32)
+  RESULT_INTO_REALREAL_IMAGEIMAGE \VSRes5,\VSRes7,vs38,vs39
+  LOAD_COUPLE_AS_RR_II	vs56,vs57,vs50,vs51,\BASE_REG,(\LOFFSET +64)
+  RESULT_INTO_REALIMAG_IMAGREAL	\VSRes6,\VSRes8,vs40,vs41 
+  LOAD_COUPLE_AS_RR_II	vs58,vs59,vs52,vs53,\BASE_REG,(\LOFFSET+96)
+  RESULT_INTO_REALREAL_IMAGEIMAGE \VSRes9,\VSRes11,vs42,vs43
+  AGGREGATE_REALS_IMAGES	vs34,vs35,vs36,vs37
+  RESULT_INTO_REALIMAG_IMAGREAL	\VSRes10,\VSRes12,vs44,vs45 
+  AGGREGATE_REALS_IMAGES	vs38,vs39,vs40,vs41  
+  RESULT_INTO_REALREAL_IMAGEIMAGE \VSRes13,\VSRes15,\VSRes1,\VSRes2
+  MULT_APLHA_PART1	vs34,vs36, vs46,vs47
+  RESULT_INTO_REALIMAG_IMAGREAL	\VSRes14,\VSRes16,\VSRes3,\VSRes4
+  MULT_APLHA_PART1	vs38,vs40,vs48,vs49
+  MULT_APLHA_PART2  vs34,vs36,vs46,vs47 
+  AGGREGATE_REALS_IMAGES	vs42,vs43,vs44,vs45
+  MULT_APLHA_PART2	vs38,vs40,vs48,vs49
+  AGGREGATE_REALS_IMAGES	\VSRes1,\VSRes2,\VSRes3,\VSRes4	
+  UNPACK_FOR_STORE	vs46,vs47,vs39,vs41
+  MULT_APLHA_PART1	vs42,vs44, vs56,vs57
+  UNPACK_FOR_STORE	vs48,vs49,vs35,vs37 
+  MULT_APLHA_PART1	\VSRes1,\VSRes3, vs58,vs59
+  STORE_COUPLE	\BASE_REG,\LOFFSET,vs39,vs41
+  MULT_APLHA_PART2	vs42,vs44,vs56,vs57
+  STORE_COUPLE	\BASE_REG,(\LOFFSET+32),vs35,vs37 
+  MULT_APLHA_PART2	\VSRes1,\VSRes3, vs58,vs59
+  UNPACK_FOR_STORE	vs56,vs57,vs42,vs44
+  UNPACK_FOR_STORE	vs58,vs59,\VSRes1,\VSRes3
+  STORE_COUPLE	\BASE_REG,(\LOFFSET +64),vs42,vs44
+  STORE_COUPLE	\BASE_REG,(\LOFFSET+96),\VSRes1,\VSRes3
+.endm
+
+
+.macro SAVE4  VSRes1,VSRes2,VSRes3,VSRes4,VSRes5,VSRes6,VSRes7,VSRes8,BASE_REG,LOFFSET
+  RESULT_INTO_REALREAL_IMAGEIMAGE \VSRes1,\VSRes3,vs34,vs35
+  LOAD_COUPLE_AS_RR_II	vs46,vs47,vs50,vs51,\BASE_REG,\LOFFSET
+  RESULT_INTO_REALIMAG_IMAGREAL	\VSRes2,\VSRes4,vs36,vs37
+  LOAD_COUPLE_AS_RR_II	vs48,vs49,vs52,vs53,\BASE_REG,(\LOFFSET+32)
+  RESULT_INTO_REALREAL_IMAGEIMAGE \VSRes5,\VSRes7,vs38,vs39
+  RESULT_INTO_REALIMAG_IMAGREAL	\VSRes6,\VSRes8,vs40,vs41 
+  AGGREGATE_REALS_IMAGES	vs34,vs35,vs36,vs37	
+  AGGREGATE_REALS_IMAGES	vs38,vs39,vs40,vs41  
+  MULT_APLHA_PART1	vs34,vs36, vs46,vs47
+  MULT_APLHA_PART1	vs38,vs40, vs48,vs49
+  MULT_APLHA_PART2	vs34,vs36, vs46,vs47 
+  MULT_APLHA_PART2	vs38,vs40,vs48,vs49
+  UNPACK_FOR_STORE	vs46,vs47,vs39,vs41
+  UNPACK_FOR_STORE	vs48,vs49,vs35,vs37
+  STORE_COUPLE	\BASE_REG,\LOFFSET,vs39,vs41
+  STORE_COUPLE	\BASE_REG,(\LOFFSET+32),vs35,vs37
+.endm
+
+
+.macro SAVE2  VSRes1,VSRes2,VSRes3,VSRes4,BASE_REG,LOFFSET
+  RESULT_INTO_REALREAL_IMAGEIMAGE \VSRes1,\VSRes3,vs34,vs35
+  LOAD_COUPLE_AS_RR_II	vs46,vs47,vs50,vs51,\BASE_REG,\LOFFSET
+  RESULT_INTO_REALIMAG_IMAGREAL	\VSRes2,\VSRes4,vs36,vs37	
+  AGGREGATE_REALS_IMAGES	vs34,vs35,vs36,vs37	
+  MULT_APLHA_PART1	vs34,vs36, vs46,vs47	
+  MULT_APLHA_PART2	vs34,vs36, vs46,vs47  
+  UNPACK_FOR_STORE	vs46,vs47,vs39,vs41	
+  STORE_COUPLE	\BASE_REG,\LOFFSET,vs39,vs41  
+.endm
+
+
+.macro SAVE1  VSRes1,VSRes2,BASE_REG,LOFFSET
+  RESULT_INTO_REALREAL_IMAGEIMAGE \VSRes1,\VSRes1,vs34,vs35
+#ifndef TRMMKERNEL 
+  lxv	vs50,	(\LOFFSET)(\BASE_REG) 
+  xxmrgld  vs46,vs50,vs50
+  xxmrghd  vs47,vs50,vs50	
+#endif	
+  RESULT_INTO_REALIMAG_IMAGREAL	\VSRes2,\VSRes2,vs36,vs37	
+  AGGREGATE_REALS_IMAGES	vs34,vs35,vs36,vs37	
+  MULT_APLHA_PART1	vs34,vs36, vs46,vs47	
+  MULT_APLHA_PART2	vs34,vs36, vs46,vs47  
+  UNPACK_FOR_STORE	vs46,vs47,vs39,vs41 
+  xxmrghd  vs39,vs47,vs46	
+  stxv	vs39,	(\LOFFSET)(\BASE_REG) 
+.endm
+
+/**********************************************************************************************
+*
+
+.macros for N=2 and M=8
+**********************************************************************************************/
+
+.macro  KERNEL2x8_ZERO_AND_PRIME_MMA
+        /* zero out and prime the MMA accumulators */
+        xxsetaccz 0
+        xxsetaccz 1
+        xxsetaccz 2
+        xxsetaccz 3
+        xxsetaccz 4
+        xxsetaccz 5
+        xxsetaccz 6
+        xxsetaccz 7
+.endm
+
+
+.macro KERNEL2x8_PRELOAD
+	lxvp	vs32,	 0(AO)	// load real,imag from A
+	lxvp	vs34,	32(AO)	// load real,imag from A
+	lxvp	vs36,	64(AO)	// load real,imag from A
+	lxvp	vs38,	96(AO)	// load real,imag from A
+	lxvp	vs48,	 0(BO)	// load real imag from B
+.endm
+
+
+.macro KERNEL2x8_2	Index, IsLast
+	lxvp	vs40,	DISP16(\Index,128)(AO)	// load real,imag from A
+	lxvp	vs42,	DISP16(\Index,160)(AO)	// load real,imag from A
+	lxvp	vs44,	DISP16(\Index,192)(AO)	// load real,imag from A
+	lxvp	vs46,	DISP16(\Index,224)(AO)	// load real,imag from A
+ 	lxvp	vs50,	DISP4(\Index,  32)(BO)	// load real,imag from B
+	xvf64gerpp	0,	vs32,	vs49
+	xvf64gerpp	1,	vs34,	vs49
+	xvf64gerpp	2,	vs36,	vs49
+	xvf64gerpp	3,	vs38,	vs49
+	xvf64gerpp	4,	vs32,	vs48
+	xvf64gerpp	5,	vs34,	vs48
+	xvf64gerpp	6,	vs36,	vs48
+	xvf64gerpp	7,	vs38,	vs48
+	lxvp	vs32,	DISP16(\Index, 256)(AO)	// load real,imag from A
+	lxvp	vs34,	DISP16(\Index, 288)(AO)	// load real,imag from A
+	lxvp	vs36,	DISP16(\Index, 320)(AO)	// load real,imag from A
+	lxvp	vs38,	DISP16(\Index, 352)(AO)	// load real,imag from A
+	lxvp	vs48,	DISP4(\Index,  64)(BO)	// load real imag from B
+	xvf64gerpp	0,	vs40,	vs51
+	xvf64gerpp	1,	vs42,	vs51
+	xvf64gerpp	2,	vs44,	vs51
+	xvf64gerpp	3,	vs46,	vs51
+	xvf64gerpp	4,	vs40,	vs50
+	xvf64gerpp	5,	vs42,	vs50
+	xvf64gerpp	6,	vs44,	vs50
+	xvf64gerpp	7,	vs46,	vs50
+.if \IsLast==1
+	addi	AO, AO,  DISP16(\Index,256)
+	addi	BO, BO,  DISP4(\Index,64)
+.endif 
+.endm
+
+
+.macro LOAD_END_2x8  OffsetA,OffsetB
+	xvf64gerpp	0,	vs32,	vs49
+	xvf64gerpp	1,	vs34,	vs49
+	xvf64gerpp	2,	vs36,	vs49
+	xvf64gerpp	3,	vs38,	vs49
+	xvf64gerpp	4,	vs32,	vs48
+	xvf64gerpp	5,	vs34,	vs48
+	xvf64gerpp	6,	vs36,	vs48
+	xvf64gerpp	7,	vs38,	vs48
+	addi	BO, BO, \OffsetB
+	addi	AO, AO, \OffsetA
+.endm
+
+
+.macro  KERNEL2x8_UNPRIME_MMA
+        /* "unprime" MMA accumulators */
+        xxmfacc 0
+        xxmfacc 1
+        xxmfacc 2
+        xxmfacc 3
+        xxmfacc 4
+        xxmfacc 5
+        xxmfacc 6
+        xxmfacc 7
+.endm
+
+
+.macro SAVE2x8
+	add	T1, CO ,LDC 
+        xxpermdi vs32, vs0, vs1, 0b01
+        xxpermdi vs33, vs0, vs1, 0b10
+        xxpermdi vs34, vs2, vs3, 0b01
+        xxpermdi vs35, vs2, vs3, 0b10
+        xxpermdi vs36, vs4, vs5, 0b01
+        xxpermdi vs37, vs4, vs5, 0b10
+        xxpermdi vs38, vs6, vs7, 0b01
+        xxpermdi vs39, vs6, vs7, 0b10
+        xxpermdi vs40, vs8, vs9, 0b01
+        xxpermdi vs41, vs8, vs9, 0b10
+        xxpermdi vs42, vs10, vs11, 0b01
+        xxpermdi vs43, vs10, vs11, 0b10
+        xxpermdi vs44, vs12, vs13, 0b01
+        xxpermdi vs45, vs12, vs13, 0b10
+        xxpermdi vs46, vs14, vs15, 0b01
+        xxpermdi vs47, vs14, vs15, 0b10
+
+	xxlor vs2, vs32, vs32
+	xxlor vs3, vs33, vs33
+	xxlor vs0, vs34, vs34
+	xxlor vs1, vs35, vs35
+	xxlor vs6, vs36, vs36
+	xxlor vs7, vs37, vs37
+	xxlor vs4, vs38, vs38
+	xxlor vs5, vs39, vs39
+	xxlor vs10, vs40, vs40
+	xxlor vs11, vs41, vs41
+	xxlor vs8, vs42, vs42
+	xxlor vs9, vs43, vs43
+	xxlor vs14, vs44, vs44
+	xxlor vs15, vs45, vs45
+	xxlor vs12, vs46, vs46
+	xxlor vs13, vs47, vs47
+
+        xxpermdi vs32, vs16, vs17, 0b01
+        xxpermdi vs33, vs16, vs17, 0b10
+        xxpermdi vs34, vs18, vs19, 0b01
+        xxpermdi vs35, vs18, vs19, 0b10
+        xxpermdi vs36, vs20, vs21, 0b01
+        xxpermdi vs37, vs20, vs21, 0b10
+        xxpermdi vs38, vs22, vs23, 0b01
+        xxpermdi vs39, vs22, vs23, 0b10
+        xxpermdi vs40, vs24, vs25, 0b01
+        xxpermdi vs41, vs24, vs25, 0b10
+        xxpermdi vs42, vs26, vs27, 0b01
+        xxpermdi vs43, vs26, vs27, 0b10
+        xxpermdi vs44, vs28, vs29, 0b01
+        xxpermdi vs45, vs28, vs29, 0b10
+        xxpermdi vs46, vs30, vs31, 0b01
+        xxpermdi vs47, vs30, vs31, 0b10
+       
+	xxlor vs18, vs32, vs32
+	xxlor vs19, vs33, vs33
+	xxlor vs16, vs34, vs34
+	xxlor vs17, vs35, vs35
+	xxlor vs22, vs36, vs36
+	xxlor vs23, vs37, vs37
+	xxlor vs20, vs38, vs38
+	xxlor vs21, vs39, vs39
+	xxlor vs26, vs40, vs40
+	xxlor vs27, vs41, vs41
+	xxlor vs24, vs42, vs42
+	xxlor vs25, vs43, vs43
+	xxlor vs30, vs44, vs44
+	xxlor vs31, vs45, vs45
+	xxlor vs28, vs46, vs46
+	xxlor vs29, vs47, vs47
+
+	SAVE8  vs0,vs1,vs2,vs3,vs4,vs5,vs6,vs7,vs8,vs9,vs10,vs11,vs12,vs13,vs14,vs15,CO,0
+	SAVE8  vs16,vs17,vs18,vs19,vs20,vs21,vs22,vs23,vs24,vs25,vs26,vs27,vs28,vs29,vs30,vs31,T1,0  
+	addi	CO, CO, 128
+.endm
+
+/**********************************************************************************************
+*
+
+.macros for N=2 and M=4
+**********************************************************************************************/
+
+.macro  KERNEL2x4_ZERO_AND_PRIME_MMA
+        /* zero out and prime the MMA accumulators */
+        xxsetaccz 0
+        xxsetaccz 1
+        xxsetaccz 2
+        xxsetaccz 3
+.endm
+
+
+.macro KERNEL2x4_PRELOAD
+	lxvp	vs32,	 0(AO)	// load real,imag from A
+	lxvp	vs34,	32(AO)	// load real,imag from A
+	lxvp	vs48,	 0(BO)	// load real imag from B
+.endm
+
+
+.macro KERNEL2x4_2 Index, IsLast
+	lxvp	vs40,	DISP8(\Index,  64)(AO)	// load real,imag from A
+	lxvp	vs42,	DISP8(\Index,  96)(AO)	// load real,imag from A
+ 	lxvp	vs50,	DISP4(\Index,  32)(BO)  // load real,imag from B
+        xvf64gerpp      0,      vs32,   vs49
+        xvf64gerpp      1,      vs34,   vs49
+        xvf64gerpp      2,      vs32,   vs48
+        xvf64gerpp      3,      vs34,   vs48
+	lxvp	vs32,	DISP8(\Index, 128)(AO)	// load real,imag from A
+	lxvp	vs34,	DISP8(\Index, 160)(AO)	// load real,imag from A
+ 	lxvp	vs48,	DISP4(\Index,  64)(BO)  // load real,imag from B
+        xvf64gerpp      0,      vs40,   vs51 
+        xvf64gerpp      1,      vs42,   vs51
+        xvf64gerpp      2,      vs40,   vs50
+        xvf64gerpp      3,      vs42,   vs50
+.if \IsLast==1
+	addi	AO, AO, DISP8(\Index,128)
+	addi	BO, BO, DISP4(\Index,64)
+.endif 
+.endm
+ 
+
+.macro LOAD_END_2x4	OffsetA, OffsetB
+        xvf64gerpp      0,      vs32,   vs49
+        xvf64gerpp      1,      vs34,   vs49
+        xvf64gerpp      2,      vs32,   vs48
+        xvf64gerpp      3,      vs34,   vs48
+	addi	BO, BO, \OffsetB
+	addi	AO, AO, \OffsetA
+.endm
+
+
+.macro  KERNEL2x4_UNPRIME_MMA
+        /* "unprime" MMA accumulators */
+        xxmfacc 0
+        xxmfacc 1
+        xxmfacc 2
+        xxmfacc 3
+.endm
+
+
+.macro SAVE2x4 
+	add	T1, CO ,LDC 
+        xxpermdi vs32, vs0, vs1, 0b01
+        xxpermdi vs33, vs0, vs1, 0b10
+        xxpermdi vs34, vs2, vs3, 0b01
+        xxpermdi vs35, vs2, vs3, 0b10
+        xxpermdi vs36, vs4, vs5, 0b01
+        xxpermdi vs37, vs4, vs5, 0b10
+        xxpermdi vs38, vs6, vs7, 0b01
+        xxpermdi vs39, vs6, vs7, 0b10
+        xxpermdi vs40, vs8, vs9, 0b01
+        xxpermdi vs41, vs8, vs9, 0b10
+        xxpermdi vs42, vs10, vs11, 0b01
+        xxpermdi vs43, vs10, vs11, 0b10
+        xxpermdi vs44, vs12, vs13, 0b01
+        xxpermdi vs45, vs12, vs13, 0b10
+        xxpermdi vs46, vs14, vs15, 0b01
+        xxpermdi vs47, vs14, vs15, 0b10
+
+	xxlor vs2, vs32, vs32
+	xxlor vs3, vs33, vs33
+	xxlor vs0, vs34, vs34
+	xxlor vs1, vs35, vs35
+	xxlor vs6, vs36, vs36
+	xxlor vs7, vs37, vs37
+	xxlor vs4, vs38, vs38
+	xxlor vs5, vs39, vs39
+	xxlor vs10, vs40, vs40
+	xxlor vs11, vs41, vs41
+	xxlor vs8, vs42, vs42
+	xxlor vs9, vs43, vs43
+	xxlor vs14, vs44, vs44
+	xxlor vs15, vs45, vs45
+	xxlor vs12, vs46, vs46
+	xxlor vs13, vs47, vs47
+
+	SAVE4  vs0,vs1,vs2,vs3,vs4,vs5,vs6,vs7,CO,0
+	SAVE4  vs8,vs9,vs10,vs11,vs12,vs13,vs14,vs15,T1,0  
+	addi	CO, CO, 64
+.endm
+
+/**********************************************************************************************
+*
+
+.macros for N=2 and M=2
+**********************************************************************************************/
+
+.macro  KERNEL2x2_ZERO_AND_PRIME_MMA
+        /* zero out and prime the MMA accumulators */
+        xxsetaccz 0
+        xxsetaccz 1
+.endm
+
+
+.macro KERNEL2x2_PRELOAD
+	lxvp	vs32,	 0(AO)	// load real,imag from A
+	lxvp	vs48,	 0(BO)	// load real imag from B
+.endm
+
+
+.macro KERNEL2x2_2 Index, IsLast
+	lxvp	vs40,	DISP4(\Index, 32)(AO)	// load real,imag from A
+ 	lxvp	vs50,	DISP4(\Index, 32)(BO)	// load real,imag from B
+        xvf64gerpp      0,      vs32,   vs49
+        xvf64gerpp      1,      vs32,   vs48
+	lxvp	vs32,	DISP4(\Index, 64)(AO)	// load real,imag from A
+	lxvp	vs48,	DISP4(\Index, 64)(BO)	// load real imag from B
+        xvf64gerpp      0,      vs40,   vs51
+        xvf64gerpp      1,      vs40,   vs50
+.if \IsLast==1
+	addi	AO, AO, DISP4(\Index,64)
+	addi	BO, BO, DISP4(\Index,64)
+.endif 
+.endm
+
+ 
+.macro LOAD_END_2x2  OffsetA,OffsetB
+        xvf64gerpp      0,      vs32,   vs49
+        xvf64gerpp      1,      vs32,   vs48
+	addi	BO, BO, \OffsetB
+	addi	AO, AO, \OffsetA
+.endm
+
+
+.macro  KERNEL2x2_UNPRIME_MMA
+        /* "unprime" MMA accumulators */
+        xxmfacc 0
+        xxmfacc 1
+.endm
+
+
+.macro SAVE2x2 
+	add	T1, CO ,LDC 
+        xxpermdi vs32, vs0, vs1, 0b01
+        xxpermdi vs33, vs0, vs1, 0b10
+        xxpermdi vs34, vs2, vs3, 0b01
+        xxpermdi vs35, vs2, vs3, 0b10
+        xxpermdi vs36, vs4, vs5, 0b01
+        xxpermdi vs37, vs4, vs5, 0b10
+        xxpermdi vs38, vs6, vs7, 0b01
+        xxpermdi vs39, vs6, vs7, 0b10
+
+	xxlor vs2, vs32, vs32
+	xxlor vs3, vs33, vs33
+	xxlor vs0, vs34, vs34
+	xxlor vs1, vs35, vs35
+	xxlor vs6, vs36, vs36
+	xxlor vs7, vs37, vs37
+	xxlor vs4, vs38, vs38
+	xxlor vs5, vs39, vs39
+
+	SAVE2  vs0,vs1,vs2,vs3,CO,0
+	SAVE2  vs4,vs5,vs6,vs7,T1,0 
+	addi	CO, CO, 32 
+.endm
+
+/**********************************************************************************************
+*
+
+.macros for N=2 and M=1
+**********************************************************************************************/
+
+.macro ZERO2x1
+	xxlxor	vs0,	vs0,	vs0
+	xxlxor	vs1,	vs1,	vs1
+	xxlxor	vs2,	vs2,	vs2
+	xxlxor	vs3,	vs3,	vs3
+ 
+.endm
+
+
+.macro LOAD2x1   
+	LOAD2x1O 0,0 
+.endm
+
+
+.macro LOAD2x1O  OffsetA,OffsetB
+	lxv	vs48,(\OffsetB+	0)(BO)	// load real imag from B
+	lxv	vs50,	(\OffsetB+16)(BO)	// load real,imag from B 
+	xxswapd	vs49, vs48
+	xxswapd	vs51, vs50
+	lxv	vs32,	(0+\OffsetA)(AO)	// load real,imag from A 
+.endm
+
+
+.macro END2x1_WITHOUT_ADD
+	END2x1 AO,BO,0,0
+.endm
+
+
+.macro END2x1	AREG, BREG, OffsetA, OffsetB
+.if \OffsetB != 0
+	addi	\BREG, \BREG, \OffsetB
+.endif
+.if \OffsetA != 0
+	addi	\AREG, \AREG, \OffsetA
+.endif
+	xvmaddadp	vs0,	vs32,	vs48
+	xvmaddadp	vs2,	vs32,	vs50
+	xvmaddadp	vs1,	vs32,	vs49
+	xvmaddadp	vs3,	vs32,	vs51 
+.endm
+
+
+.macro LOAD2x1_2
+    LOAD2x1_2O 0,0
+.endm	
+
+
+.macro LOAD2x1_2O  OffsetA,OffsetB
+	lxv	vs48,(\OffsetB+	0)(BO)	// load real imag from B
+	lxv	vs50,	(\OffsetB+16)(BO)	// load real,imag from B
+	lxv	vs52,	(\OffsetB+32)(BO)	// load real,imag	from B
+	lxv	vs54,	(\OffsetB+48)(BO)	// load real,imag  from B	
+	xxswapd	vs49, vs48
+	xxswapd	vs51, vs50
+	lxv	vs32,	(0+\OffsetA)(AO)	// load real,imag from A
+	lxv	vs40,	(16+\OffsetA)(AO)	// load real,imag from A 
+.endm	
+
+
+.macro END2x1_2	  
+  /*for load2 offset will be 32 and 64*/
+   KERNEL2x1_2	AO,BO,	32,64,0 ,1,1 
+.endm
+
+
+.macro KERNEL2x1_E2	OffsetA,OffsetB, Index,IsLast 
+  KERNEL2x1_2	AO,BO,	\OffsetA,\OffsetB, \Index,\IsLast ,1 
+.endm
+
+
+.macro KERNEL2x1_L2	OffsetA,OffsetB, Index,IsLast
+  KERNEL2x1_2	AO,BO,	\OffsetA,\OffsetB, \Index,\IsLast ,0 
+.endm
+
+
+.macro KERNEL2x1_2	AREG,BREG,	OffsetA,OffsetB, Index,IsLast ,Complete
+  xxswapd	vs53, vs52
+  xxswapd	vs55, vs54 
+	xvmaddadp	vs0,	vs32,	vs48
+	xvmaddadp	vs2,	vs32,	vs50
+	xvmaddadp	vs1,	vs32,	vs49
+	xvmaddadp	vs3,	vs32,	vs51
+.if \Complete==0	
+	lxv	vs32,	DISP2(\Index, 0 + \OffsetA)(\AREG)	// load real,imag from A 
+.endif	 
+.if \Complete==0		
+	lxv	vs48,	DISP4(\Index, 0+\OffsetB)(\BREG)	// load real imag from B
+	lxv	vs50,	DISP4(\Index, 16+\OffsetB)(\BREG)	// load real,imag from B
+.endif
+.if \Complete==0		
+  xxswapd	vs49, vs48
+  xxswapd	vs51, vs50
+.endif 
+	xvmaddadp	vs0,	vs40,	vs52
+	xvmaddadp	vs2,	vs40,	vs54 
+	xvmaddadp	vs1,	vs40,	vs53
+	xvmaddadp	vs3,	vs40,	vs55
+.if \Complete==0		
+	lxv	vs40,	DISP2(\Index,16+0+ \OffsetA)(\AREG)	// load real,imag from A 
+.endif
+ 
+.if \Complete==0	 
+ 	lxv	vs52,	DISP4(\Index, 32+\OffsetB)(\BREG)	// load real,imag	from B
+	lxv	vs54,	DISP4(\Index, 48+\OffsetB)(\BREG)	// load real,imag  from B
+.endif
+.if \IsLast==1
+.if \Complete==1
+	addi	\AREG, \AREG,  DISP2(\Index,\OffsetA)
+	addi	\BREG, \BREG,  DISP4(\Index,\OffsetB)
+.else
+	addi	\AREG, \AREG, DISP2(\Index,32)
+	addi	\BREG, \BREG,  DISP4(\Index,64)
+.endif
+.endif 
+.endm
+ 
+
+.macro KERNEL2x1
+  LOAD2x1
+  END2x1  AO, BO, 16,32
+.endm
+
+
+.macro SAVE2x1
+	add	T1, CO ,LDC 
+	SAVE1  vs0,vs1,CO,0
+	SAVE1  vs2,vs3,T1,0  
+	addi	CO, CO, 16 
+.endm
+
+/**********************************************************************************************
+*
+
+.macros for N=1 and M=8
+**********************************************************************************************/
+
+.macro  KERNEL1x8_ZERO_AND_PRIME_MMA
+        /* zero out and prime the MMA accumulators */
+        xxsetaccz 0
+        xxsetaccz 1
+        xxsetaccz 2
+        xxsetaccz 3
+.endm
+
+
+.macro KERNEL1x8_2	Index,IsLast
+	lxvp	vs32,	DISP16(\Index,   0)(AO)	// load real,imag from A
+	lxvp	vs34,	DISP16(\Index,  32)(AO)	// load real,imag from A
+	lxvp	vs36,	DISP16(\Index,  64)(AO)	// load real,imag from A
+	lxvp	vs38,	DISP16(\Index,  96)(AO)	// load real,imag from A
+	lxvp	vs40,	DISP16(\Index, 128)(AO)	// load real,imag from A
+	lxvp	vs42,	DISP16(\Index, 160)(AO)	// load real,imag from A
+	lxvp	vs44,	DISP16(\Index, 192)(AO)	// load real,imag from A
+	lxvp	vs46,	DISP16(\Index, 224)(AO)	// load real,imag from A
+	lxvp	vs48,	DISP2(\Index,    0)(BO)	// load real imag from B
+        xvf64gerpp      0,      vs32,   vs49
+        xvf64gerpp      1,      vs34,   vs49
+        xvf64gerpp      2,      vs36,   vs49
+        xvf64gerpp      3,      vs38,   vs49
+        xvf64gerpp      0,      vs40,   vs48
+        xvf64gerpp      1,      vs42,   vs48
+        xvf64gerpp      2,      vs44,   vs48
+        xvf64gerpp      3,      vs46,   vs48
+.if \IsLast==1
+	addi	AO, AO, DISP16(\Index,256)
+	addi	BO, BO,  DISP2(\Index,32)
+.endif 
+.endm
+
+
+.macro LOAD_END_1x8  OffsetA,OffsetB
+	lxvp	vs32,	0(AO)	// load real,imag from A
+	lxvp	vs34,	32(AO)	// load real,imag from A
+	lxvp	vs36,	64(AO)	// load real,imag from A
+	lxvp	vs38,	96(AO)	// load real,imag from A
+	lxv	vs48,	0(BO)	// load real imag from B 
+        xvf64gerpp      0,      vs32,   vs48
+        xvf64gerpp      1,      vs34,   vs48
+        xvf64gerpp      2,      vs36,   vs48
+        xvf64gerpp      3,      vs38,   vs48
+	addi	BO, BO, \OffsetB
+	addi	AO, AO, \OffsetA
+.endm
+
+
+.macro  KERNEL1x8_UNPRIME_MMA
+        /* "unprime" MMA accumulators */
+        xxmfacc 0
+        xxmfacc 1
+        xxmfacc 2
+        xxmfacc 3
+.endm
+
+
+.macro SAVE1x8
+        xxpermdi vs32, vs0, vs1, 0b01
+        xxpermdi vs33, vs0, vs1, 0b10
+        xxpermdi vs34, vs2, vs3, 0b01
+        xxpermdi vs35, vs2, vs3, 0b10
+        xxpermdi vs36, vs4, vs5, 0b01
+        xxpermdi vs37, vs4, vs5, 0b10
+        xxpermdi vs38, vs6, vs7, 0b01
+        xxpermdi vs39, vs6, vs7, 0b10
+        xxpermdi vs40, vs8, vs9, 0b01
+        xxpermdi vs41, vs8, vs9, 0b10
+        xxpermdi vs42, vs10, vs11, 0b01
+        xxpermdi vs43, vs10, vs11, 0b10
+        xxpermdi vs44, vs12, vs13, 0b01
+        xxpermdi vs45, vs12, vs13, 0b10
+        xxpermdi vs46, vs14, vs15, 0b01
+        xxpermdi vs47, vs14, vs15, 0b10
+
+	xxlor vs2, vs32, vs32
+	xxlor vs3, vs33, vs33
+	xxlor vs0, vs34, vs34
+	xxlor vs1, vs35, vs35
+	xxlor vs6, vs36, vs36
+	xxlor vs7, vs37, vs37
+	xxlor vs4, vs38, vs38
+	xxlor vs5, vs39, vs39
+	xxlor vs10, vs40, vs40
+	xxlor vs11, vs41, vs41
+	xxlor vs8, vs42, vs42
+	xxlor vs9, vs43, vs43
+	xxlor vs14, vs44, vs44
+	xxlor vs15, vs45, vs45
+	xxlor vs12, vs46, vs46
+	xxlor vs13, vs47, vs47
+
+	SAVE8  vs0,vs1,vs2,vs3,vs4,vs5,vs6,vs7,vs8,vs9,vs10,vs11,vs12,vs13,vs14,vs15,CO,0
+	addi	CO, CO, 128
+.endm
+
+/**********************************************************************************************
+*
+
+.macros for N=1 and M=4
+**********************************************************************************************/
+
+.macro  KERNEL1x4_ZERO_AND_PRIME_MMA
+        /* zero out and prime the MMA accumulators */
+        xxsetaccz 0
+        xxsetaccz 1
+.endm
+
+
+.macro KERNEL1x4_2	Index,IsLast
+	lxvp	vs32,	DISP8(\Index,  0)(AO)	// load real,imag from A
+	lxvp	vs34,	DISP8(\Index, 32)(AO)	// load real,imag from A
+	lxvp	vs40,	DISP8(\Index, 64)(AO)	// load real,imag from A
+	lxvp	vs42,	DISP8(\Index, 96)(AO)	// load real,imag from A
+	lxvp	vs48,	DISP2(\Index,  0)(BO)	// load real imag from B
+        xvf64gerpp      0,      vs32,   vs49
+        xvf64gerpp      1,      vs34,   vs49
+        xvf64gerpp      0,      vs40,   vs48
+        xvf64gerpp      1,      vs42,   vs48
+.if \IsLast==1
+	addi	AO, AO, DISP8(\Index,128)
+	addi	BO, BO,  DISP2(\Index,32)
+.endif 
+.endm
+ 
+
+.macro LOAD_END_1x4  OffsetA,OffsetB
+	lxvp	vs32,	0(AO)	// load real,imag from A
+	lxvp	vs34,	32(AO)	// load real,imag from A
+	lxv	vs48,	0(BO)	// load real imag from B
+        xvf64gerpp      0,      vs32,   vs48
+        xvf64gerpp      1,      vs34,   vs48
+	addi	BO, BO, \OffsetB
+	addi	AO, AO, \OffsetA
+.endm
+
+
+.macro  KERNEL1x4_UNPRIME_MMA
+        /* "unprime" MMA accumulators */
+        xxmfacc 0
+        xxmfacc 1
+.endm
+
+
+.macro SAVE1x4 
+        xxpermdi vs32, vs0, vs1, 0b01
+        xxpermdi vs33, vs0, vs1, 0b10
+        xxpermdi vs34, vs2, vs3, 0b01
+        xxpermdi vs35, vs2, vs3, 0b10
+        xxpermdi vs36, vs4, vs5, 0b01
+        xxpermdi vs37, vs4, vs5, 0b10
+        xxpermdi vs38, vs6, vs7, 0b01
+        xxpermdi vs39, vs6, vs7, 0b10
+
+	xxlor vs2, vs32, vs32
+	xxlor vs3, vs33, vs33
+	xxlor vs0, vs34, vs34
+	xxlor vs1, vs35, vs35
+	xxlor vs6, vs36, vs36
+	xxlor vs7, vs37, vs37
+	xxlor vs4, vs38, vs38
+	xxlor vs5, vs39, vs39
+
+	SAVE4  vs0,vs1,vs2,vs3,vs4,vs5,vs6,vs7,CO,0
+	addi	CO, CO, 64
+.endm
+
+/**********************************************************************************************
+*
+
+.macros for N=1 and M=2
+**********************************************************************************************/
+
+.macro  KERNEL1x2_ZERO_AND_PRIME_MMA
+        /* zero out and prime the MMA accumulators */
+        xxsetaccz 0
+.endm
+
+
+.macro KERNEL1x2_2	Index,IsLast
+	lxvp	vs32,	DISP4(\Index,  0)(AO)	// load real,imag from A
+	lxvp	vs40,	DISP4(\Index, 32)(AO)	// load real,imag from A
+	lxvp	vs48,	DISP2(\Index,  0)(BO)	// load real imag from B
+        xvf64gerpp      0,      vs32,   vs49
+        xvf64gerpp      0,      vs40,   vs48
+.if \IsLast==1
+	addi	AO, AO, DISP4(\Index,64)
+	addi	BO, BO, DISP2(\Index,32)
+.endif 
+.endm
+ 
+
+.macro LOAD_END_1x2  OffsetA,OffsetB
+	lxvp	vs32,	0(AO)	// load real,imag from A
+	lxv	vs48,	0(BO)	// load real imag from B
+        xvf64gerpp      0,      vs32,   vs48
+	addi	BO, BO, \OffsetB
+	addi	AO, AO, \OffsetA
+.endm
+
+
+.macro  KERNEL1x2_UNPRIME_MMA
+        /* "unprime" MMA accumulators */
+        xxmfacc 0
+.endm
+
+
+.macro SAVE1x2 
+        xxpermdi vs32, vs0, vs1, 0b01
+        xxpermdi vs33, vs0, vs1, 0b10
+        xxpermdi vs34, vs2, vs3, 0b01
+        xxpermdi vs35, vs2, vs3, 0b10
+
+	xxlor vs2, vs32, vs32
+	xxlor vs3, vs33, vs33
+	xxlor vs0, vs34, vs34
+	xxlor vs1, vs35, vs35
+
+	SAVE2  vs0,vs1,vs2,vs3,CO,0
+	addi	CO, CO, 32 
+.endm
+
+/**********************************************************************************************
+*
+
+.macros for N=1 and M=1
+**********************************************************************************************/
+
+.macro ZERO1x1
+	xxlxor	vs0,	vs0,	vs0
+	xxlxor	vs1,	vs1,	vs1 
+.endm
+
+
+.macro LOAD1x1   
+	LOAD1x1O 0,0 
+.endm
+
+
+.macro LOAD1x1O  OffsetA,OffsetB
+	lxv	vs48,(\OffsetB+	0)(BO)	// load real imag from B
+	lxv	vs32,	(0+\OffsetA)(AO)	// load real,imag from A 
+	xxswapd	vs49, vs48
+
+.endm
+
+
+.macro END1x1_WITHOUT_ADD
+	END1x1 AO,BO,0,0
+.endm
+
+
+.macro END1x1	AREG, BREG, OffsetA, OffsetB
+.if \OffsetB != 0
+	addi	\BREG, \BREG, \OffsetB
+.endif
+.if \OffsetA != 0
+	addi	\AREG, \AREG, \OffsetA
+.endif
+	xvmaddadp	vs0,	vs32,	vs48 
+	xvmaddadp	vs1,	vs32,	vs49 
+.endm
+
+
+.macro LOAD1x1_2
+    LOAD1x1_2O 0,0
+.endm	
+
+
+.macro LOAD1x1_2O  OffsetA,OffsetB
+	lxv	vs48,(\OffsetB+	0)(BO)	// load real imag from B
+	lxv	vs52,	(\OffsetB+16)(BO)	// load real,imag	from B
+	xxswapd	vs49, vs48
+
+	lxv	vs32,	(0+\OffsetA)(AO)	// load real,imag from A
+	lxv	vs40,	(16+\OffsetA)(AO)	// load real,imag from A 
+.endm	
+
+
+.macro END1x1_2	  
+  /*for load2 offset will be 32 and 32*/
+   KERNEL1x1_2	AO,BO,	32,32,0 ,1,1 
+.endm
+ 
+
+
+.macro KERNEL1x1_E2	OffsetA,OffsetB, Index,IsLast 
+  KERNEL1x1_2	AO,BO,	\OffsetA,\OffsetB, \Index,\IsLast ,1 
+.endm
+
+
+.macro KERNEL1x1_L2	OffsetA,OffsetB, Index,IsLast
+  KERNEL1x1_2	AO,BO,	\OffsetA,\OffsetB, \Index,\IsLast ,0 
+.endm
+
+
+.macro KERNEL1x1_2	AREG,BREG,	OffsetA,OffsetB, Index,IsLast ,Complete
+  xxswapd	vs53, vs52
+	xvmaddadp	vs0,	vs32,	vs48 
+	xvmaddadp	vs1,	vs32,	vs49 
+.if \Complete==0	
+	lxv	vs32,	DISP2(\Index, 0 + \OffsetA)(\AREG)	// load real,imag from A 
+.endif	 
+.if \Complete==0		
+	lxv	vs48,	DISP2(\Index, 0+\OffsetB)(\BREG)	// load real imag from B
+.endif
+.if \Complete==0		
+  xxswapd	vs49, vs48
+.endif 
+	xvmaddadp	vs0,	vs40,	vs52
+	xvmaddadp	vs1,	vs40,	vs53 
+.if \Complete==0		
+	lxv	vs40,	DISP2(\Index,16+0+ \OffsetA)(\AREG)	// load real,imag from A 
+.endif
+ 
+.if \Complete==0	 
+ 	lxv	vs52,	DISP2(\Index, 16+\OffsetB)(\BREG)	// load real,imag	from B
+.endif
+.if \IsLast==1
+.if \Complete==1
+	addi	\AREG, \AREG,  DISP2(\Index,\OffsetA)
+	addi	\BREG, \BREG,  DISP2(\Index,\OffsetB)
+.else
+	addi	\AREG, \AREG, DISP2(\Index,32)
+	addi	\BREG, \BREG,  DISP2(\Index,32)
+.endif
+.endif 
+.endm
+ 
+
+
+.macro KERNEL1x1
+  LOAD1x1
+  END1x1  AO, BO, 16,16
+.endm
+
+
+
+.macro SAVE1x1
+	SAVE1  vs0,vs1,CO,0
+	addi	CO, CO, 16 
+.endm
+
+/****************************TRMM POINTER REFRESH
+
+.macroSES*************************/
+
+
+.macro SHIFT_REG  REG1,REG2,SHIFT_VAL
+		.if \SHIFT_VAL==16 
+			slwi		\REG1,	\REG2,	8			
+		.elseif \SHIFT_VAL==8  
+			slwi		\REG1,	\REG2,	7			 
+		.elseif \SHIFT_VAL==4
+			slwi		\REG1,	\REG2,	6			  
+		.elseif \SHIFT_VAL==2
+			slwi		\REG1,	\REG2,	5			 
+		.elseif \SHIFT_VAL==1
+			slwi		\REG1,	\REG2,	4			 
+		.endif
+.endm
+/*
+//#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+// 		ptrbb = bb;
+// #else
+// 		ptrba += off*16;
+// 		ptrbb = bb + off*2;
+// #endif
+*/
+
+
+.macro REFRESH_POINTERS  PTR_A,PTR_B,OFF_VAL,B_VAL,C_A,C_B
+    #if (defined(LEFT) &&  defined(TRANSA)) ||  (!defined(LEFT) && !defined(TRANSA))
+        /* ptrbb = bb;*/
+        mr \PTR_B,\B_VAL     /* refresh BPOINT */
+    #else
+		    /*
+        // ptrba  =ptrba+ off*C_A;
+        // ptrbb = bb + off*C_B; 
+				*/
+		SHIFT_REG T4,\OFF_VAL,\C_B		/* Number of values in B shifted  */
+		SHIFT_REG T2,\OFF_VAL,\C_A		/* Number of values in A shifted  */
+		add		\PTR_B,	\B_VAL ,	T4				/* Add values to BO */
+		add		\PTR_A,	\PTR_A,	T2				/* Add values to AO  */
+    #endif 
+.endm
+
+/*
+// #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+// 		temp = bk-off;
+// #elif defined(LEFT)
+// 		temp = off+16;	// number of values in A
+// #else
+// 		temp = off+2;	// number of values in B
+// #endif
+*/
+
+
+.macro REFRESH_TEMP_BK TEMP_BK,BK_VAL,OFF_VAL,INCR_A,INCR_B
+    #if (defined(LEFT) && !defined(TRANSA)) ||  (!defined(LEFT) && defined(TRANSA))
+                            /* temp = bk-off;*/
+           sub \TEMP_BK,\BK_VAL,\OFF_VAL
+    #elif defined(LEFT)
+                            /* temp = off+INCR_A;	// number of values in A */
+           addi \TEMP_BK, \OFF_VAL, \INCR_A
+    #else
+                            /* temp = off+INCR_B	// number of values in B*/
+           addi \TEMP_BK,\OFF_VAL, \INCR_B
+    #endif
+.endm
+/*
+// #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+// 		temp = bk - off;
+// #ifdef LEFT
+// 		temp -= 16; // number of values in A
+// #else
+// 		temp -= 2; // number of values in B
+// #endif
+// 		ptrba += temp*16;
+// 		ptrbb += temp*2;
+// #endif
+// #ifdef LEFT
+// 		off += 16; // number of values in A
+// #endif
+*/
+ 
+
+
+.macro REFRESH_AFTER_SAVE TEMP_BK,BK_VAL,OFF_VAL,PTR_B,PTR_A,C_A,C_B
+    #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+                    /*temp = bk - off;*/
+                sub \TEMP_BK,\BK_VAL,\OFF_VAL
+    #ifdef LEFT
+                    /*temp -= 8; // number of values in A*/
+                addi \TEMP_BK,\TEMP_BK,-\C_A
+    #else
+                    /*temp -= 4; // number of values in B*/
+                addi \TEMP_BK,\TEMP_BK,-\C_B 
+    #endif
+                    /*ptrba += temp*C_A;
+                    ptrbb += temp*C_B;*/ 
+                SHIFT_REG T4,\TEMP_BK,\C_A
+								SHIFT_REG T2,\TEMP_BK,\C_B
+                add \PTR_A, \PTR_A,T4/*ptrba+temp*C_A*/ 
+								add \PTR_B, \PTR_B,T2 
+    #endif
+    #ifdef LEFT
+                    /*off += 8; // number of values in A*/
+                 addi \OFF_VAL,\OFF_VAL,\C_A
+    #endif
+.endm
+

From 2a91452bdd1d735b11156add482b9f35c3d01c69 Mon Sep 17 00:00:00 2001
From: Matthew Treinish <mtreinish@kortar.org>
Date: Thu, 25 Jun 2020 11:32:09 -0400
Subject: [PATCH 31/48] Add cpu detection support for comet lake U

Comet Lake U CPUs have family: 6, model: 6, extended family: 0, and
extended model: 10 were not being correctly detected by GETARCH during
openblas builds and would show CORE=UNKNOWN and LIBCORE=unknown. This
commit adds the necessary information to cpuid_x86 to detect extended
family 10 model 6 and return the proper core information. It's
essentially just a skylake cpu, not skylake x, so I just took the used
the same return fields as skylake.
---
 cpuid_x86.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/cpuid_x86.c b/cpuid_x86.c
index e29adecae..1fe5ca152 100644
--- a/cpuid_x86.c
+++ b/cpuid_x86.c
@@ -1955,6 +1955,19 @@ int get_coretype(void){
 	    return CORE_NEHALEM;
         }
         break;
+      case 10:
+        switch (model) {
+    case 6:
+      // Comet Lake U
+            if(support_avx())
+  #ifndef NO_AVX2
+              return CORE_HASWELL;
+  #else
+          return CORE_SANDYBRIDGE;
+  #endif
+            else
+          return CORE_NEHALEM;
+        }
       case 5:
         switch (model) {
 	case 6:

From f37e941d5270e396ed27e4ad5fd484fb257b742b Mon Sep 17 00:00:00 2001
From: Matthew Treinish <mtreinish@kortar.org>
Date: Thu, 25 Jun 2020 11:56:49 -0400
Subject: [PATCH 32/48] Add support to driver/others/dynamic.c too

---
 driver/others/dynamic.c | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/driver/others/dynamic.c b/driver/others/dynamic.c
index 38eb76643..7677f265a 100644
--- a/driver/others/dynamic.c
+++ b/driver/others/dynamic.c
@@ -618,6 +618,18 @@ static gotoblas_t *get_coretype(void){
 	    return &gotoblas_NEHALEM; //OS doesn't support AVX. Use old kernels.
 	  }
 	}
+      case 10:
+    if (model == 6) {
+	  if(support_avx2())
+	    return &gotoblas_HASWELL;
+	  if(support_avx()) {
+	    openblas_warning(FALLBACK_VERBOSE, SANDYBRIDGE_FALLBACK);
+	    return &gotoblas_SANDYBRIDGE;
+	  } else {
+	    openblas_warning(FALLBACK_VERBOSE, NEHALEM_FALLBACK);
+	    return &gotoblas_NEHALEM; //OS doesn't support AVX. Use old kernels.
+	  }
+    }
 	return NULL;
       }
       case 0xf:

From 2f9c10810c932fc015cb4e5078cab7117bc120b6 Mon Sep 17 00:00:00 2001
From: Matthew Treinish <mtreinish@kortar.org>
Date: Thu, 25 Jun 2020 15:53:56 -0400
Subject: [PATCH 33/48] Also set CPUTYPE in get_cpuname()

---
 cpuid_x86.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/cpuid_x86.c b/cpuid_x86.c
index 1fe5ca152..3538690b9 100644
--- a/cpuid_x86.c
+++ b/cpuid_x86.c
@@ -1406,6 +1406,16 @@ int get_cpuname(void){
 	    return CPUTYPE_SANDYBRIDGE;
           else
 	    return CPUTYPE_NEHALEM;
+    }
+      case 10: //family 6 exmodel 10
+        switch (model) {
+    case 6: // Comet Lake U
+          if(support_avx2())
+            return CPUTYPE_HASWELL;
+          if(support_avx())
+        return CPUTYPE_SANDYBRIDGE;
+          else
+        return CPUTYPE_NEHALEM;
 	}
 	break;    
       }

From d23419accc2f60a27b95cb29f11f76443a82d111 Mon Sep 17 00:00:00 2001
From: Rajalakshmi Srinivasaraghavan <rajis@linux.ibm.com>
Date: Thu, 25 Jun 2020 22:19:08 -0500
Subject: [PATCH 34/48] powerpc: Optimized SHGEMM kernel for POWER10

This patch introduces new optimized version of SHGEMM kernel
using power10 Matrix-Multiply Assist (MMA) feature introduced in
POWER ISA v3.1. This patch makes use of new POWER10 compute instructions
for matrix multiplication operation.

Tested on simulator and there are no new test failures.
---
 kernel/generic/gemm_ncopy_16.c       |   30 +-
 kernel/generic/gemm_ncopy_8.c        |   42 +-
 kernel/generic/gemm_tcopy_16.c       |   24 +-
 kernel/generic/gemm_tcopy_8.c        |   42 +-
 kernel/power/KERNEL.POWER10          |   11 +
 kernel/power/shgemm_kernel_power10.c | 1044 ++++++++++++++++++++++++++
 param.h                              |   13 +
 7 files changed, 1137 insertions(+), 69 deletions(-)
 create mode 100644 kernel/power/shgemm_kernel_power10.c

diff --git a/kernel/generic/gemm_ncopy_16.c b/kernel/generic/gemm_ncopy_16.c
index 5f91d0dbe..d3ab46472 100644
--- a/kernel/generic/gemm_ncopy_16.c
+++ b/kernel/generic/gemm_ncopy_16.c
@@ -39,24 +39,24 @@
 #include <stdio.h>
 #include "common.h"
 
-int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, FLOAT *b){
+int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){
   BLASLONG i, j;
 
-  FLOAT *aoffset;
-  FLOAT *aoffset1, *aoffset2, *aoffset3, *aoffset4;
-  FLOAT *aoffset5, *aoffset6, *aoffset7, *aoffset8;
-  FLOAT *aoffset9, *aoffset10, *aoffset11, *aoffset12;
-  FLOAT *aoffset13, *aoffset14, *aoffset15, *aoffset16;
+  IFLOAT *aoffset;
+  IFLOAT *aoffset1, *aoffset2, *aoffset3, *aoffset4;
+  IFLOAT *aoffset5, *aoffset6, *aoffset7, *aoffset8;
+  IFLOAT *aoffset9, *aoffset10, *aoffset11, *aoffset12;
+  IFLOAT *aoffset13, *aoffset14, *aoffset15, *aoffset16;
 
-  FLOAT *boffset;
-  FLOAT ctemp01, ctemp02, ctemp03, ctemp04;
-  FLOAT ctemp05, ctemp06, ctemp07, ctemp08;
-  FLOAT ctemp09, ctemp10, ctemp11, ctemp12;
-  FLOAT ctemp13, ctemp14, ctemp15, ctemp16;
-  FLOAT ctemp17, ctemp18, ctemp19, ctemp20;
-  FLOAT ctemp21, ctemp22, ctemp23, ctemp24;
-  FLOAT ctemp25, ctemp26, ctemp27, ctemp28;
-  FLOAT ctemp29, ctemp30, ctemp31, ctemp32;
+  IFLOAT *boffset;
+  IFLOAT ctemp01, ctemp02, ctemp03, ctemp04;
+  IFLOAT ctemp05, ctemp06, ctemp07, ctemp08;
+  IFLOAT ctemp09, ctemp10, ctemp11, ctemp12;
+  IFLOAT ctemp13, ctemp14, ctemp15, ctemp16;
+  IFLOAT ctemp17, ctemp18, ctemp19, ctemp20;
+  IFLOAT ctemp21, ctemp22, ctemp23, ctemp24;
+  IFLOAT ctemp25, ctemp26, ctemp27, ctemp28;
+  IFLOAT ctemp29, ctemp30, ctemp31, ctemp32;
 
   aoffset = a;
   boffset = b;
diff --git a/kernel/generic/gemm_ncopy_8.c b/kernel/generic/gemm_ncopy_8.c
index a49a778e6..aaf9c8917 100644
--- a/kernel/generic/gemm_ncopy_8.c
+++ b/kernel/generic/gemm_ncopy_8.c
@@ -39,30 +39,30 @@
 #include <stdio.h>
 #include "common.h"
 
-int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, FLOAT *b){
+int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){
   BLASLONG i, j;
 
-  FLOAT *aoffset;
-  FLOAT *aoffset1, *aoffset2, *aoffset3, *aoffset4;
-  FLOAT *aoffset5, *aoffset6, *aoffset7, *aoffset8;
+  IFLOAT *aoffset;
+  IFLOAT *aoffset1, *aoffset2, *aoffset3, *aoffset4;
+  IFLOAT *aoffset5, *aoffset6, *aoffset7, *aoffset8;
 
-  FLOAT *boffset;
-  FLOAT ctemp01, ctemp02, ctemp03, ctemp04;
-  FLOAT ctemp05, ctemp06, ctemp07, ctemp08;
-  FLOAT ctemp09, ctemp10, ctemp11, ctemp12;
-  FLOAT ctemp13, ctemp14, ctemp15, ctemp16;
-  FLOAT ctemp17, ctemp18, ctemp19, ctemp20;
-  FLOAT ctemp21, ctemp22, ctemp23, ctemp24;
-  FLOAT ctemp25, ctemp26, ctemp27, ctemp28;
-  FLOAT ctemp29, ctemp30, ctemp31, ctemp32;
-  FLOAT ctemp33, ctemp34, ctemp35, ctemp36;
-  FLOAT ctemp37, ctemp38, ctemp39, ctemp40;
-  FLOAT ctemp41, ctemp42, ctemp43, ctemp44;
-  FLOAT ctemp45, ctemp46, ctemp47, ctemp48;
-  FLOAT ctemp49, ctemp50, ctemp51, ctemp52;
-  FLOAT ctemp53, ctemp54, ctemp55, ctemp56;
-  FLOAT ctemp57, ctemp58, ctemp59, ctemp60;
-  FLOAT ctemp61, ctemp62, ctemp63, ctemp64;
+  IFLOAT *boffset;
+  IFLOAT ctemp01, ctemp02, ctemp03, ctemp04;
+  IFLOAT ctemp05, ctemp06, ctemp07, ctemp08;
+  IFLOAT ctemp09, ctemp10, ctemp11, ctemp12;
+  IFLOAT ctemp13, ctemp14, ctemp15, ctemp16;
+  IFLOAT ctemp17, ctemp18, ctemp19, ctemp20;
+  IFLOAT ctemp21, ctemp22, ctemp23, ctemp24;
+  IFLOAT ctemp25, ctemp26, ctemp27, ctemp28;
+  IFLOAT ctemp29, ctemp30, ctemp31, ctemp32;
+  IFLOAT ctemp33, ctemp34, ctemp35, ctemp36;
+  IFLOAT ctemp37, ctemp38, ctemp39, ctemp40;
+  IFLOAT ctemp41, ctemp42, ctemp43, ctemp44;
+  IFLOAT ctemp45, ctemp46, ctemp47, ctemp48;
+  IFLOAT ctemp49, ctemp50, ctemp51, ctemp52;
+  IFLOAT ctemp53, ctemp54, ctemp55, ctemp56;
+  IFLOAT ctemp57, ctemp58, ctemp59, ctemp60;
+  IFLOAT ctemp61, ctemp62, ctemp63, ctemp64;
 
 
   aoffset = a;
diff --git a/kernel/generic/gemm_tcopy_16.c b/kernel/generic/gemm_tcopy_16.c
index 56268ebf2..14252599a 100644
--- a/kernel/generic/gemm_tcopy_16.c
+++ b/kernel/generic/gemm_tcopy_16.c
@@ -39,22 +39,22 @@
 #include <stdio.h>
 #include "common.h"
 
-int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, FLOAT *b){
+int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){
 
   BLASLONG i, j;
 
-  FLOAT *aoffset;
-  FLOAT *aoffset1, *aoffset2;
-  FLOAT *boffset;
+  IFLOAT *aoffset;
+  IFLOAT *aoffset1, *aoffset2;
+  IFLOAT *boffset;
 
-  FLOAT ctemp01, ctemp02, ctemp03, ctemp04;
-  FLOAT ctemp05, ctemp06, ctemp07, ctemp08;
-  FLOAT ctemp09, ctemp10, ctemp11, ctemp12;
-  FLOAT ctemp13, ctemp14, ctemp15, ctemp16;
-  FLOAT ctemp17, ctemp18, ctemp19, ctemp20;
-  FLOAT ctemp21, ctemp22, ctemp23, ctemp24;
-  FLOAT ctemp25, ctemp26, ctemp27, ctemp28;
-  FLOAT ctemp29, ctemp30, ctemp31, ctemp32;
+  IFLOAT ctemp01, ctemp02, ctemp03, ctemp04;
+  IFLOAT ctemp05, ctemp06, ctemp07, ctemp08;
+  IFLOAT ctemp09, ctemp10, ctemp11, ctemp12;
+  IFLOAT ctemp13, ctemp14, ctemp15, ctemp16;
+  IFLOAT ctemp17, ctemp18, ctemp19, ctemp20;
+  IFLOAT ctemp21, ctemp22, ctemp23, ctemp24;
+  IFLOAT ctemp25, ctemp26, ctemp27, ctemp28;
+  IFLOAT ctemp29, ctemp30, ctemp31, ctemp32;
 
   aoffset   = a;
   boffset   = b;
diff --git a/kernel/generic/gemm_tcopy_8.c b/kernel/generic/gemm_tcopy_8.c
index b28f3d219..3e8a839db 100644
--- a/kernel/generic/gemm_tcopy_8.c
+++ b/kernel/generic/gemm_tcopy_8.c
@@ -39,32 +39,32 @@
 #include <stdio.h>
 #include "common.h"
 
-int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, FLOAT *b){
+int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){
 
   BLASLONG i, j;
 
-  FLOAT *aoffset;
-  FLOAT *aoffset1, *aoffset2, *aoffset3, *aoffset4;
-  FLOAT *aoffset5, *aoffset6, *aoffset7, *aoffset8;
+  IFLOAT *aoffset;
+  IFLOAT *aoffset1, *aoffset2, *aoffset3, *aoffset4;
+  IFLOAT *aoffset5, *aoffset6, *aoffset7, *aoffset8;
 
-  FLOAT *boffset,  *boffset1, *boffset2, *boffset3, *boffset4;
+  IFLOAT *boffset,  *boffset1, *boffset2, *boffset3, *boffset4;
 
-  FLOAT ctemp01, ctemp02, ctemp03, ctemp04;
-  FLOAT ctemp05, ctemp06, ctemp07, ctemp08;
-  FLOAT ctemp09, ctemp10, ctemp11, ctemp12;
-  FLOAT ctemp13, ctemp14, ctemp15, ctemp16;
-  FLOAT ctemp17, ctemp18, ctemp19, ctemp20;
-  FLOAT ctemp21, ctemp22, ctemp23, ctemp24;
-  FLOAT ctemp25, ctemp26, ctemp27, ctemp28;
-  FLOAT ctemp29, ctemp30, ctemp31, ctemp32;
-  FLOAT ctemp33, ctemp34, ctemp35, ctemp36;
-  FLOAT ctemp37, ctemp38, ctemp39, ctemp40;
-  FLOAT ctemp41, ctemp42, ctemp43, ctemp44;
-  FLOAT ctemp45, ctemp46, ctemp47, ctemp48;
-  FLOAT ctemp49, ctemp50, ctemp51, ctemp52;
-  FLOAT ctemp53, ctemp54, ctemp55, ctemp56;
-  FLOAT ctemp57, ctemp58, ctemp59, ctemp60;
-  FLOAT ctemp61, ctemp62, ctemp63, ctemp64;
+  IFLOAT ctemp01, ctemp02, ctemp03, ctemp04;
+  IFLOAT ctemp05, ctemp06, ctemp07, ctemp08;
+  IFLOAT ctemp09, ctemp10, ctemp11, ctemp12;
+  IFLOAT ctemp13, ctemp14, ctemp15, ctemp16;
+  IFLOAT ctemp17, ctemp18, ctemp19, ctemp20;
+  IFLOAT ctemp21, ctemp22, ctemp23, ctemp24;
+  IFLOAT ctemp25, ctemp26, ctemp27, ctemp28;
+  IFLOAT ctemp29, ctemp30, ctemp31, ctemp32;
+  IFLOAT ctemp33, ctemp34, ctemp35, ctemp36;
+  IFLOAT ctemp37, ctemp38, ctemp39, ctemp40;
+  IFLOAT ctemp41, ctemp42, ctemp43, ctemp44;
+  IFLOAT ctemp45, ctemp46, ctemp47, ctemp48;
+  IFLOAT ctemp49, ctemp50, ctemp51, ctemp52;
+  IFLOAT ctemp53, ctemp54, ctemp55, ctemp56;
+  IFLOAT ctemp57, ctemp58, ctemp59, ctemp60;
+  IFLOAT ctemp61, ctemp62, ctemp63, ctemp64;
 
   aoffset   = a;
   boffset   = b;
diff --git a/kernel/power/KERNEL.POWER10 b/kernel/power/KERNEL.POWER10
index 4fc7190b0..39f5e9414 100644
--- a/kernel/power/KERNEL.POWER10
+++ b/kernel/power/KERNEL.POWER10
@@ -7,6 +7,17 @@ else
 #CGEMM_BETA = ../generic/zgemm_beta.c
 #ZGEMM_BETA = ../generic/zgemm_beta.c
 
+SHGEMM_BETA = ../generic/gemm_beta.c
+SHGEMMKERNEL    = shgemm_kernel_power10.c
+SHGEMMINCOPY    = ../generic/gemm_ncopy_16.c
+SHGEMMITCOPY    = ../generic/gemm_tcopy_16.c
+SHGEMMONCOPY    = ../generic/gemm_ncopy_8.c
+SHGEMMOTCOPY    = ../generic/gemm_tcopy_8.c
+SHGEMMINCOPYOBJ =  shgemm_incopy$(TSUFFIX).$(SUFFIX)
+SHGEMMITCOPYOBJ =  shgemm_itcopy$(TSUFFIX).$(SUFFIX)
+SHGEMMONCOPYOBJ =  shgemm_oncopy$(TSUFFIX).$(SUFFIX)
+SHGEMMOTCOPYOBJ =  shgemm_otcopy$(TSUFFIX).$(SUFFIX)
+
 STRMMKERNEL	= sgemm_kernel_power10.c
 DTRMMKERNEL	= dgemm_kernel_power10.c
 CTRMMKERNEL	= cgemm_kernel_power10.S
diff --git a/kernel/power/shgemm_kernel_power10.c b/kernel/power/shgemm_kernel_power10.c
new file mode 100644
index 000000000..7455f925c
--- /dev/null
+++ b/kernel/power/shgemm_kernel_power10.c
@@ -0,0 +1,1044 @@
+/*********************************************************************************
+Copyright (c) 2020, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+**********************************************************************************/
+#include "common.h"
+#include <altivec.h>
+#if defined(HALF) && defined(HALFCONVERSION)
+static float
+bfloat16tof32 (bfloat16 f16)
+{
+  float result = 0;
+  unsigned short *q = (unsigned short *) (&result);
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+  q[0] = f16;
+#else
+  q[1] = f16;
+#endif
+  return result;
+}
+
+#define BF16TOF32(x) (bfloat16tof32(x))
+#else
+#define BF16TOF32(x) x
+#endif
+
+typedef unsigned char vec_t __attribute__ ((vector_size (16)));
+typedef FLOAT v4sf_t __attribute__ ((vector_size (16)));
+typedef FLOAT v2sf_t __attribute__ ((vector_size (8)));
+
+vector char mask =
+  { 0x0, 0x1, 0x8, 0x9, 0x2, 0x3, 0xa, 0xb, 0x4, 0x5, 0xc, 0xd, 0x6, 0x7, 0xe,
+  0xf
+};
+
+/* 
+ * BFLOAT16 xvbf16ger2pp instruction needs 4×2 matrix of
+ * bfloat16 floating-point values as input. Hence this
+ * merging is needed on A and B matrices. 
+ */
+#define MERGE_ROW(x) vec_perm(x, x, mask)
+#define MERGE_HIGH(x, y) (vec_t) vec_mergeh ((vector short)x, (vector short)y)
+#define MERGE_LOW(x, y) (vec_t) vec_mergel ((vector short)x, (vector short)y)
+
+#define SAVE_ACC(ACC, J)  \
+	  __builtin_mma_disassemble_acc (result, ACC); \
+	  rowC = (v4sf_t *) &CO[0* ldc+J]; \
+          rowC[0] += result[3] * alpha; \
+          rowC = (v4sf_t *) &CO[1*ldc+J]; \
+          rowC[0] += result[2] * alpha; \
+          rowC = (v4sf_t *) &CO[2*ldc+J]; \
+          rowC[0] += result[1] * alpha; \
+          rowC = (v4sf_t *) &CO[3*ldc+J]; \
+          rowC[0] += result[0] * alpha;
+#define SAVE_ACC1(ACC, J)  \
+	  __builtin_mma_disassemble_acc (result, ACC); \
+	  rowC = (v4sf_t *) &CO[4* ldc+J]; \
+          rowC[0] += result[3] * alpha; \
+          rowC = (v4sf_t *) &CO[5*ldc+J]; \
+          rowC[0] += result[2] * alpha; \
+          rowC = (v4sf_t *) &CO[6*ldc+J]; \
+          rowC[0] += result[1] * alpha; \
+          rowC = (v4sf_t *) &CO[7*ldc+J]; \
+          rowC[0] += result[0] * alpha;
+#define  SAVE4x2_ACC(ACC, J)  \
+	  __builtin_mma_disassemble_acc (result, ACC); \
+	  rowC = (v2sf_t *) &CO[0* ldc+J]; \
+          rowC[0] += result[6] * alpha; \
+	  rowC = (v2sf_t *) &CO[1* ldc+J]; \
+          rowC[0] += result[4] * alpha; \
+	  rowC = (v2sf_t *) &CO[2* ldc+J]; \
+          rowC[0] += result[2] * alpha; \
+	  rowC = (v2sf_t *) &CO[3* ldc+J]; \
+          rowC[0] += result[0] * alpha;
+#define  SAVE4x2_ACC1(ACC, J)  \
+	  __builtin_mma_disassemble_acc (result, ACC); \
+	  rowC = (v2sf_t *) &CO[4* ldc+J]; \
+          rowC[0] += result[6] * alpha; \
+	  rowC = (v2sf_t *) &CO[5* ldc+J]; \
+          rowC[0] += result[4] * alpha; \
+	  rowC = (v2sf_t *) &CO[6* ldc+J]; \
+          rowC[0] += result[2] * alpha; \
+	  rowC = (v2sf_t *) &CO[7* ldc+J]; \
+          rowC[0] += result[0] * alpha;
+
+#define MMA __builtin_mma_xvbf16ger2pp
+
+#define  SAVE2x4_ACC(ACC, J)  \
+	  __builtin_mma_disassemble_acc (result, ACC); \
+	  rowC = (v4sf_t *) &CO[0* ldc+J]; \
+          rowC[0] += result[3] * alpha; \
+	  rowC = (v4sf_t *) &CO[1* ldc+J]; \
+          rowC[0] += result[2] * alpha;
+
+#define SET_ACC_ZERO4() \
+	  __builtin_mma_xxsetaccz (&acc0); \
+	  __builtin_mma_xxsetaccz (&acc1); \
+	  __builtin_mma_xxsetaccz (&acc2); \
+	  __builtin_mma_xxsetaccz (&acc3);
+
+#define SET_ACC_ZERO8() \
+	  __builtin_mma_xxsetaccz (&acc0); \
+	  __builtin_mma_xxsetaccz (&acc1); \
+	  __builtin_mma_xxsetaccz (&acc2); \
+	  __builtin_mma_xxsetaccz (&acc3); \
+	  __builtin_mma_xxsetaccz (&acc4); \
+	  __builtin_mma_xxsetaccz (&acc5); \
+	  __builtin_mma_xxsetaccz (&acc6); \
+	  __builtin_mma_xxsetaccz (&acc7);
+
+#define PREFETCH1(x, y) asm volatile ("dcbt %0, %1" : : "r" (x), "b" (y) : "memory");
+/*************************************************************************************
+* SHGEMM Kernel
+*************************************************************************************/
+int
+CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
+       IFLOAT * B, FLOAT * C, BLASLONG ldc)
+{
+  BLASLONG N = n;
+  BLASLONG i1;
+  v4sf_t valpha = { alpha, alpha, alpha, alpha };
+  vector short vzero = { 0, 0, 0, 0, 0, 0, 0, 0 };
+  N = n >> 3;
+  /* Loop for n >= 8. */
+  for (i1 = 0; i1 < N; i1++)
+    {
+      BLASLONG i, j;
+      FLOAT *CO;
+      IFLOAT *AO;
+      CO = C;
+      C += ldc << 3;
+      AO = A;
+      PREFETCH1 (A, 128);
+      PREFETCH1 (A, 256);
+      i = m >> 4;
+      /* Loop for m >= 16. */
+      for (j = 0; j < i; j++)
+	{
+	  IFLOAT *BO = B;
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7;
+	  SET_ACC_ZERO8 ();
+	  BLASLONG l = 0;
+	  for (l = 0; l < k / 2; l++)
+	    {
+	      vec_t *rowA = (vec_t *) & (AO[l << 5]);
+	      vec_t *rowB = (vec_t *) & (BO[l << 4]);
+	      vec_t rowB_h = MERGE_HIGH (rowB[0], rowB[1]);
+	      vec_t rowB_l = MERGE_LOW (rowB[0], rowB[1]);
+	      vec_t rowA_h = MERGE_HIGH (rowA[0], rowA[2]);
+	      vec_t rowA_l = MERGE_LOW (rowA[0], rowA[2]);
+	      vec_t rowA2_h = MERGE_HIGH (rowA[1], rowA[3]);
+	      vec_t rowA2_l = MERGE_LOW (rowA[1], rowA[3]);
+	      MMA (&acc0, rowB_h, rowA_h);
+	      MMA (&acc1, rowB_l, rowA_h);
+	      MMA (&acc2, rowB_h, rowA_l);
+	      MMA (&acc3, rowB_l, rowA_l);
+	      MMA (&acc4, rowB_h, rowA2_h);
+	      MMA (&acc5, rowB_l, rowA2_h);
+	      MMA (&acc6, rowB_h, rowA2_l);
+	      MMA (&acc7, rowB_l, rowA2_l);
+	    }
+	  if (k % 2 == 1)
+	    {
+	      if (k > 1)
+		l = (k / 2) << 4;
+	      vec_t *rowA = (vec_t *) & (AO[l << 1]);
+	      vec_t *rowB = (vec_t *) & (BO[l]);
+	      vec_t rowB_h = MERGE_HIGH (rowB[0], rowB[1]);
+	      vec_t rowB_l = MERGE_LOW (rowB[0], rowB[1]);
+	      vec_t rowA_h = MERGE_HIGH (rowA[0], vzero);
+	      vec_t rowA_l = MERGE_LOW (rowA[0], vzero);
+	      vec_t rowA2_h = MERGE_HIGH (rowA[1], vzero);
+	      vec_t rowA2_l = MERGE_LOW (rowA[1], vzero);
+	      MMA (&acc0, rowB_h, rowA_h);
+	      MMA (&acc1, rowB_l, rowA_h);
+	      MMA (&acc2, rowB_h, rowA_l);
+	      MMA (&acc3, rowB_l, rowA_l);
+	      MMA (&acc4, rowB_h, rowA2_h);
+	      MMA (&acc5, rowB_l, rowA2_h);
+	      MMA (&acc6, rowB_h, rowA2_l);
+	      MMA (&acc7, rowB_l, rowA2_l);
+	    }
+	  SAVE_ACC (&acc0, 0);
+	  SAVE_ACC (&acc2, 4);
+	  SAVE_ACC1 (&acc1, 0);
+	  SAVE_ACC1 (&acc3, 4);
+	  SAVE_ACC (&acc4, 8);
+	  SAVE_ACC (&acc6, 12);
+	  SAVE_ACC1 (&acc5, 8);
+	  SAVE_ACC1 (&acc7, 12);
+	  CO += 16;
+
+	  AO += (k << 4);
+	  BO += (k << 3);
+	}
+      i = (m & 15) >> 3;
+      /* Loop for m >= 8. */
+      for (j = 0; j < i; j++)
+	{
+	  IFLOAT *BO = B;
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  __vector_quad acc0, acc1, acc2, acc3;
+	  SET_ACC_ZERO4 ();
+	  BLASLONG l = 0;
+	  for (l = 0; l < k / 2; l++)
+	    {
+	      vec_t *rowA = (vec_t *) & (AO[l << 4]);
+	      vec_t *rowB = (vec_t *) & (BO[l << 4]);
+	      vec_t rowB_h = MERGE_HIGH (rowB[0], rowB[1]);
+	      vec_t rowB_l = MERGE_LOW (rowB[0], rowB[1]);
+	      vec_t rowA_h = MERGE_HIGH (rowA[0], rowA[1]);
+	      vec_t rowA_l = MERGE_LOW (rowA[0], rowA[1]);
+	      MMA (&acc0, rowB_h, rowA_h);
+	      MMA (&acc1, rowB_l, rowA_h);
+	      MMA (&acc2, rowB_h, rowA_l);
+	      MMA (&acc3, rowB_l, rowA_l);
+	    }
+	  if (k % 2 == 1)
+	    {
+	      if (k > 1)
+		l = (k / 2) << 4;
+	      vec_t *rowA = (vec_t *) & (AO[l]);
+	      vec_t *rowB = (vec_t *) & (BO[l]);
+	      vec_t rowB_h = MERGE_HIGH (rowB[0], rowB[1]);
+	      vec_t rowB_l = MERGE_LOW (rowB[0], rowB[1]);
+	      vec_t rowA_h = MERGE_HIGH (rowA[0], vzero);
+	      vec_t rowA_l = MERGE_LOW (rowA[0], vzero);
+	      MMA (&acc0, rowB_h, rowA_h);
+	      MMA (&acc1, rowB_l, rowA_h);
+	      MMA (&acc2, rowB_h, rowA_l);
+	      MMA (&acc3, rowB_l, rowA_l);
+	    }
+	  SAVE_ACC (&acc0, 0);
+	  SAVE_ACC (&acc2, 4);
+	  SAVE_ACC1 (&acc1, 0);
+	  SAVE_ACC1 (&acc3, 4);
+	  CO += 8;
+	  AO += (k << 3);
+	  BO += (k << 3);
+	}
+      i = (m & 7) >> 2;
+      /* Loop for m >= 4. */
+      for (j = 0; j < i; j++)
+	{
+	  IFLOAT *BO = B;
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  __vector_quad acc0, acc1;
+	  __builtin_mma_xxsetaccz (&acc0);
+	  __builtin_mma_xxsetaccz (&acc1);
+	  BLASLONG l = 0;
+	  for (l = 0; l < k / 2; l++)
+	    {
+	      vec_t *rowA = (vec_t *) & (AO[l << 3]);
+	      vec_t *rowB = (vec_t *) & (BO[l << 4]);
+	      vec_t rowA_mrg = MERGE_ROW (rowA[0]);
+	      MMA (&acc0, MERGE_HIGH (rowB[0], rowB[1]), rowA_mrg);
+	      MMA (&acc1, MERGE_LOW (rowB[0], rowB[1]), rowA_mrg);
+	    }
+	  if (k % 2 == 1)
+	    {
+	      if (k > 1)
+		l = (k / 2) << 3;
+	      vector short rowA =
+		{ AO[l + 0], 0, AO[l + 1], 0, AO[l + 2], 0, AO[l + 3], 0 };
+	      vec_t *rowB = (vec_t *) & (BO[l << 1]);
+	      MMA (&acc0, MERGE_HIGH (rowB[0], rowB[1]), (vec_t) rowA);
+	      MMA (&acc1, MERGE_LOW (rowB[0], rowB[1]), (vec_t) rowA);
+	    }
+	  SAVE_ACC (&acc0, 0);
+	  SAVE_ACC1 (&acc1, 0);
+	  CO += 4;
+	  AO += (k << 2);
+	  BO += (k << 3);
+	}
+      i = (m & 3) >> 1;
+      /* Loop for m >= 2. */
+      for (j = 0; j < i; j++)
+	{
+	  IFLOAT *BO = B;
+	  v2sf_t *rowC;
+	  v2sf_t result[8];
+	  __vector_quad acc0, acc1;
+	  __builtin_mma_xxsetaccz (&acc0);
+	  __builtin_mma_xxsetaccz (&acc1);
+	  BLASLONG l = 0;
+	  for (l = 0; l < k / 2; l++)
+	    {
+	      vector short rowA =
+		{ AO[(l << 2) + 0], AO[(l << 2) + 2], AO[(l << 2) + 1],
+		AO[(l << 2) + 3],
+		0, 0, 0, 0
+	      };
+	      vec_t *rowB = (vec_t *) & (BO[l << 4]);
+	      MMA (&acc0, MERGE_HIGH (rowB[0], rowB[1]), (vec_t) rowA);
+	      MMA (&acc1, MERGE_LOW (rowB[0], rowB[1]), (vec_t) rowA);
+	    }
+	  if (k % 2 == 1)
+	    {
+	      if (k > 1)
+		l = (k / 2) << 2;
+	      vector short rowA = { AO[l + 0], 0, AO[l + 1], 0, 0, 0, 0, 0 };
+	      vec_t *rowB = (vec_t *) & (BO[(l << 2)]);
+	      MMA (&acc0, MERGE_HIGH (rowB[0], rowB[1]), (vec_t) rowA);
+	      MMA (&acc1, MERGE_LOW (rowB[0], rowB[1]), (vec_t) rowA);
+	    }
+	  SAVE4x2_ACC (&acc0, 0);
+	  SAVE4x2_ACC1 (&acc1, 0);
+	  CO += 2;
+	  AO += (k << 1);
+	  BO += (k << 3);
+	}
+      i = (m & 1) >> 0;
+      /* Loop for m = 1. */
+      for (j = 0; j < i; j++)
+	{
+	  IFLOAT *BO = B;
+	  BLASLONG l = 0;
+	  v4sf_t t = { 0, 0, 0, 0 }
+	  , t1 =
+	  {
+	  0, 0, 0, 0};
+	  for (l = 0; l < k; l++)
+	    {
+	      v4sf_t rowA =
+		{ BF16TOF32 (AO[l]), BF16TOF32 (AO[l]), BF16TOF32 (AO[l]),
+		BF16TOF32 (AO[l])
+	      };
+	      v4sf_t rowB =
+		{ BF16TOF32 (BO[l << 3]), BF16TOF32 (BO[(l << 3) + 1]),
+		BF16TOF32 (BO[(l << 3) + 2]),
+		BF16TOF32 (BO[(l << 3) + 3])
+	      };
+	      v4sf_t rowB1 =
+		{ BF16TOF32 (BO[(l << 3) + 4]), BF16TOF32 (BO[(l << 3) + 5]),
+		BF16TOF32 (BO[(l << 3) + 6]),
+		BF16TOF32 (BO[(l << 3) + 7])
+	      };
+	      t += rowA * rowB;
+	      t1 += rowA * rowB1;
+	    }
+	  t = t * valpha;
+	  t1 = t1 * valpha;
+	  CO[0 * ldc] += t[0];
+	  CO[1 * ldc] += t[1];
+	  CO[2 * ldc] += t[2];
+	  CO[3 * ldc] += t[3];
+	  CO[4 * ldc] += t1[0];
+	  CO[5 * ldc] += t1[1];
+	  CO[6 * ldc] += t1[2];
+	  CO[7 * ldc] += t1[3];
+	  CO += 1;
+	  AO += k;
+	  BO += (k << 3);
+	}
+      B += k << 3;
+    }
+  N = (n & 7) >> 2;
+  /* Loop for n >= 4. */
+  for (i1 = 0; i1 < N; i1++)
+    {
+      BLASLONG i, j;
+      FLOAT *CO;
+      IFLOAT *AO;
+      CO = C;
+      C += ldc << 2;
+      AO = A;
+      i = m >> 5;
+      /* Loop for m >= 32. */
+      for (j = 0; j < i; j++)
+	{
+	  IFLOAT *BO = B;
+	  IFLOAT *A1 = AO + (16 * k);
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7;
+	  SET_ACC_ZERO8 ();
+	  BLASLONG l = 0;
+	  for (l = 0; l < k / 2; l++)
+	    {
+	      vec_t *rowA = (vec_t *) & (AO[l << 5]);
+	      vec_t *rowA1 = (vec_t *) & (A1[l << 5]);
+	      vec_t *rowB = (vec_t *) & (BO[l << 3]);
+	      vec_t rowB_mrg = MERGE_ROW (rowB[0]);
+	      MMA (&acc0, rowB_mrg, MERGE_HIGH (rowA[0], rowA[2]));
+	      MMA (&acc1, rowB_mrg, MERGE_LOW (rowA[0], rowA[2]));
+	      MMA (&acc2, rowB_mrg, MERGE_HIGH (rowA[1], rowA[3]));
+	      MMA (&acc3, rowB_mrg, MERGE_LOW (rowA[1], rowA[3]));
+	      MMA (&acc4, rowB_mrg, MERGE_HIGH (rowA1[0], rowA1[2]));
+	      MMA (&acc5, rowB_mrg, MERGE_LOW (rowA1[0], rowA1[2]));
+	      MMA (&acc6, rowB_mrg, MERGE_HIGH (rowA1[1], rowA1[3]));
+	      MMA (&acc7, rowB_mrg, MERGE_LOW (rowA1[1], rowA1[3]));
+	    }
+	  if (k % 2 == 1)
+	    {
+	      if (k > 1)
+		l = (k / 2) << 3;
+	      vec_t *rowA = (vec_t *) & (AO[(l << 2)]);
+	      vec_t *rowA1 = (vec_t *) & (A1[(l << 2)]);
+	      vec_t *rowB = (vec_t *) & (BO[l]);
+	      vec_t rowB_mrg = MERGE_ROW (rowB[0]);
+	      MMA (&acc0, rowB_mrg, MERGE_HIGH (rowA[0], vzero));
+	      MMA (&acc1, rowB_mrg, MERGE_LOW (rowA[0], vzero));
+	      MMA (&acc2, rowB_mrg, MERGE_HIGH (rowA[1], vzero));
+	      MMA (&acc3, rowB_mrg, MERGE_LOW (rowA[1], vzero));
+	      MMA (&acc4, rowB_mrg, MERGE_HIGH (rowA1[0], vzero));
+	      MMA (&acc5, rowB_mrg, MERGE_LOW (rowA1[0], vzero));
+	      MMA (&acc6, rowB_mrg, MERGE_HIGH (rowA1[1], vzero));
+	      MMA (&acc7, rowB_mrg, MERGE_LOW (rowA1[1], vzero));
+	    }
+
+	  SAVE_ACC (&acc0, 0);
+	  SAVE_ACC (&acc1, 4);
+	  CO += 8;
+	  SAVE_ACC (&acc2, 0);
+	  SAVE_ACC (&acc3, 4);
+	  CO += 8;
+	  SAVE_ACC (&acc4, 0);
+	  SAVE_ACC (&acc5, 4);
+	  CO += 8;
+	  SAVE_ACC (&acc6, 0);
+	  SAVE_ACC (&acc7, 4);
+	  CO += 8;
+	  AO += k << 5;
+	  BO += k << 2;
+	}
+      i = (m & 31) >> 4;
+      /* Loop for m >= 16. */
+      for (j = 0; j < i; j++)
+	{
+	  IFLOAT *BO = B;
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  __vector_quad acc0, acc1, acc2, acc3;
+	  SET_ACC_ZERO4 ();
+	  BLASLONG l = 0;
+	  for (l = 0; l < k / 2; l++)
+	    {
+	      vec_t *rowA = (vec_t *) & (AO[l << 5]);
+	      vec_t *rowB = (vec_t *) & (BO[l << 3]);
+	      vec_t rowB_mrg = MERGE_ROW (rowB[0]);
+	      MMA (&acc0, rowB_mrg, MERGE_HIGH (rowA[0], rowA[2]));
+	      MMA (&acc1, rowB_mrg, MERGE_LOW (rowA[0], rowA[2]));
+	      MMA (&acc2, rowB_mrg, MERGE_HIGH (rowA[1], rowA[3]));
+	      MMA (&acc3, rowB_mrg, MERGE_LOW (rowA[1], rowA[3]));
+	    }
+	  if (k % 2 == 1)
+	    {
+	      if (k > 1)
+		l = (k / 2) << 3;
+	      vec_t *rowA = (vec_t *) & (AO[(l << 2)]);
+	      vec_t *rowB = (vec_t *) & (BO[l]);
+	      vec_t rowB_mrg = MERGE_ROW (rowB[0]);
+	      MMA (&acc0, rowB_mrg, MERGE_HIGH (rowA[0], vzero));
+	      MMA (&acc1, rowB_mrg, MERGE_LOW (rowA[0], vzero));
+	      MMA (&acc2, rowB_mrg, MERGE_HIGH (rowA[1], vzero));
+	      MMA (&acc3, rowB_mrg, MERGE_LOW (rowA[1], vzero));
+	    }
+
+	  SAVE_ACC (&acc0, 0);
+	  SAVE_ACC (&acc1, 4);
+	  CO += 8;
+	  SAVE_ACC (&acc2, 0);
+	  SAVE_ACC (&acc3, 4);
+	  CO += 8;
+	  AO += k << 4;
+	  BO += k << 2;
+	}
+      i = (m & 15) >> 3;
+      /* Loop for m >= 8. */
+      for (j = 0; j < i; j++)
+	{
+	  IFLOAT *BO = B;
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  __vector_quad acc0, acc1;
+	  __builtin_mma_xxsetaccz (&acc0);
+	  __builtin_mma_xxsetaccz (&acc1);
+	  BLASLONG l = 0;
+	  for (l = 0; l < k / 2; l++)
+	    {
+	      vec_t *rowA = (vec_t *) & (AO[l << 4]);
+	      vec_t *rowB = (vec_t *) & (BO[l << 3]);
+	      vec_t rowB_mrg = MERGE_ROW (rowB[0]);
+	      MMA (&acc0, rowB_mrg, MERGE_HIGH (rowA[0], rowA[1]));
+	      MMA (&acc1, rowB_mrg, MERGE_LOW (rowA[0], rowA[1]));
+	    }
+	  if (k % 2 == 1)
+	    {
+	      if (k > 1)
+		l = (k / 2) << 3;
+	      vec_t *rowA = (vec_t *) & (AO[l << 1]);
+	      vec_t *rowB = (vec_t *) & (BO[l]);
+	      vec_t rowB_mrg = MERGE_ROW (rowB[0]);
+	      MMA (&acc0, rowB_mrg, MERGE_HIGH (rowA[0], vzero));
+	      MMA (&acc1, rowB_mrg, MERGE_LOW (rowA[0], vzero));
+	    }
+	  SAVE_ACC (&acc0, 0);
+	  SAVE_ACC (&acc1, 4);
+	  CO += 8;
+	  AO += k << 3;
+	  BO += k << 2;
+	}
+      i = (m & 7) >> 2;
+      /* Loop for m >= 4. */
+      for (j = 0; j < i; j++)
+	{
+	  IFLOAT *BO = B;
+	  v4sf_t *rowC;
+	  __vector_quad acc0;
+	  v4sf_t result[4];
+	  BLASLONG l = 0;
+	  __builtin_mma_xxsetaccz (&acc0);
+	  for (l = 0; l < k / 2; l++)
+	    {
+	      vec_t *rowA = (vec_t *) & (AO[l << 3]);
+	      vec_t *rowB = (vec_t *) & (BO[l << 3]);
+	      MMA (&acc0, MERGE_ROW (rowB[0]), MERGE_ROW (rowA[0]));
+	    }
+	  if (k % 2 == 1)
+	    {
+	      if (k > 1)
+		l = (k / 2) << 3;
+	      vector short rowA =
+		{ AO[l], 0, AO[l + 1], 0, AO[l + 2], 0, AO[l + 3], 0 };
+	      vec_t *rowB = (vec_t *) & (BO[l]);
+	      MMA (&acc0, MERGE_ROW (rowB[0]), (vec_t) rowA);
+	    }
+	  SAVE_ACC (&acc0, 0);
+	  CO += 4;
+	  AO += k << 2;
+	  BO += k << 2;
+	}
+      i = (m & 3) >> 1;
+      /* Loop for m >= 2. */
+      for (j = 0; j < i; j++)
+	{
+	  IFLOAT *BO = B;
+	  v2sf_t *rowC;
+	  v2sf_t result[8];
+	  __vector_quad acc0;
+	  BLASLONG l = 0;
+	  __builtin_mma_xxsetaccz (&acc0);
+	  for (l = 0; l < k / 2; l++)
+	    {
+	      vector short rowA =
+		{ AO[(l << 2) + 0], AO[(l << 2) + 2], AO[(l << 2) + 1],
+		AO[(l << 2) + 3],
+		0, 0, 0, 0
+	      };
+	      vec_t *rowB = (vec_t *) & (BO[l << 3]);
+	      MMA (&acc0, MERGE_ROW (rowB[0]), (vec_t) rowA);
+	    }
+	  if (k % 2 == 1)
+	    {
+	      if (k > 1)
+		l = (k / 2) << 2;
+	      vector short rowA = { AO[l], 0, AO[l + 1], 0, 0, 0, 0, 0 };
+	      vec_t *rowB = (vec_t *) & (BO[l << 1]);
+	      MMA (&acc0, MERGE_ROW (rowB[0]), (vec_t) rowA);
+	    }
+	  SAVE4x2_ACC (&acc0, 0);
+	  CO += 2;
+	  AO += k << 1;
+	  BO += k << 2;
+	}
+      i = (m & 1) >> 0;
+      /* Loop for m = 1. */
+      for (j = 0; j < i; j++)
+	{
+	  IFLOAT *BO = B;
+	  BLASLONG l = 0;
+	  v4sf_t t = { 0, 0, 0, 0 };
+	  for (l = 0; l < k; l++)
+	    {
+	      v4sf_t rowA =
+		{ BF16TOF32 (AO[l]), BF16TOF32 (AO[l]), BF16TOF32 (AO[l]),
+		BF16TOF32 (AO[l])
+	      };
+	      v4sf_t rowB =
+		{ BF16TOF32 (BO[l << 2]), BF16TOF32 (BO[(l << 2) + 1]),
+		BF16TOF32 (BO[(l << 2) + 2]),
+		BF16TOF32 (BO[(l << 2) + 3])
+	      };
+	      t += rowA * rowB;
+	    }
+	  t = t * valpha;
+	  CO[0 * ldc] += t[0];
+	  CO[1 * ldc] += t[1];
+	  CO[2 * ldc] += t[2];
+	  CO[3 * ldc] += t[3];
+	  AO += k;
+	  BO += (k << 2);
+	  CO += 1;
+	}
+
+      B += k << 2;
+    }
+  N = (n & 3) >> 1;
+  /* Loop for n >= 2. */
+  for (i1 = 0; i1 < N; i1++)
+    {
+      BLASLONG i, j;
+      FLOAT *CO;
+      IFLOAT *AO;
+      CO = C;
+      C += ldc << 1;
+      AO = A;
+      i = m >> 5;
+      /* Loop for m >= 32. */
+      for (j = 0; j < i; j++)
+	{
+	  IFLOAT *BO = B;
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  IFLOAT *A1 = AO + (16 * k);
+	  __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7;
+	  SET_ACC_ZERO8 ();
+	  BLASLONG l = 0;
+	  for (l = 0; l < k / 2; l++)
+	    {
+	      vector short rowB =
+		{ BO[(l << 2) + 0], BO[(l << 2) + 2], BO[(l << 2) + 1],
+		BO[(l << 2) + 3],
+		0, 0, 0, 0
+	      };
+	      vec_t *rowA = (vec_t *) & (AO[l << 5]);
+	      vec_t *rowA1 = (vec_t *) & (A1[l << 5]);
+	      MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[2]));
+	      MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[2]));
+	      MMA (&acc2, (vec_t) rowB, MERGE_HIGH (rowA[1], rowA[3]));
+	      MMA (&acc3, (vec_t) rowB, MERGE_LOW (rowA[1], rowA[3]));
+	      MMA (&acc4, (vec_t) rowB, MERGE_HIGH (rowA1[0], rowA1[2]));
+	      MMA (&acc5, (vec_t) rowB, MERGE_LOW (rowA1[0], rowA1[2]));
+	      MMA (&acc6, (vec_t) rowB, MERGE_HIGH (rowA1[1], rowA1[3]));
+	      MMA (&acc7, (vec_t) rowB, MERGE_LOW (rowA1[1], rowA1[3]));
+	    }
+	  if (k % 2 == 1)
+	    {
+	      if (k > 1)
+		l = (k / 2) << 2;
+	      vector short rowB = { BO[l + 0], 0, BO[l + 1], 0, 0, 0, 0, 0 };
+	      vec_t *rowA = (vec_t *) & (AO[l << 3]);
+	      vec_t *rowA1 = (vec_t *) & (A1[l << 3]);
+	      MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[2]));
+	      MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[2]));
+	      MMA (&acc2, (vec_t) rowB, MERGE_HIGH (rowA[1], rowA[3]));
+	      MMA (&acc3, (vec_t) rowB, MERGE_LOW (rowA[1], rowA[3]));
+	      MMA (&acc4, (vec_t) rowB, MERGE_HIGH (rowA1[0], rowA1[2]));
+	      MMA (&acc5, (vec_t) rowB, MERGE_LOW (rowA1[0], rowA1[2]));
+	      MMA (&acc6, (vec_t) rowB, MERGE_HIGH (rowA1[1], rowA1[3]));
+	      MMA (&acc7, (vec_t) rowB, MERGE_LOW (rowA1[1], rowA1[3]));
+	    }
+	  SAVE2x4_ACC (&acc0, 0);
+	  SAVE2x4_ACC (&acc1, 4);
+	  SAVE2x4_ACC (&acc2, 8);
+	  SAVE2x4_ACC (&acc3, 12);
+	  CO += 16;
+	  SAVE2x4_ACC (&acc4, 0);
+	  SAVE2x4_ACC (&acc5, 4);
+	  SAVE2x4_ACC (&acc6, 8);
+	  SAVE2x4_ACC (&acc7, 12);
+	  CO += 16;
+	  AO += k << 5;
+	  BO += k << 1;
+	}
+      i = (m & 31) >> 4;
+      /* Loop for m >= 16. */
+      for (j = 0; j < i; j++)
+	{
+	  IFLOAT *BO = B;
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  __vector_quad acc0, acc1, acc2, acc3;
+	  SET_ACC_ZERO4 ();
+	  BLASLONG l = 0;
+	  for (l = 0; l < k / 2; l++)
+	    {
+	      vector short rowB =
+		{ BO[(l << 2) + 0], BO[(l << 2) + 2], BO[(l << 2) + 1],
+		BO[(l << 2) + 3],
+		0, 0, 0, 0
+	      };
+	      vec_t *rowA = (vec_t *) & (AO[l << 5]);
+	      MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[2]));
+	      MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[2]));
+	      MMA (&acc2, (vec_t) rowB, MERGE_HIGH (rowA[1], rowA[3]));
+	      MMA (&acc3, (vec_t) rowB, MERGE_LOW (rowA[1], rowA[3]));
+	    }
+	  if (k % 2 == 1)
+	    {
+	      if (k > 1)
+		l = (k / 2) << 2;
+	      vector short rowB = { BO[l + 0], 0, BO[l + 1], 0, 0, 0, 0, 0 };
+	      vec_t *rowA = (vec_t *) & (AO[l << 3]);
+	      MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[2]));
+	      MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[2]));
+	      MMA (&acc2, (vec_t) rowB, MERGE_HIGH (rowA[1], rowA[3]));
+	      MMA (&acc3, (vec_t) rowB, MERGE_LOW (rowA[1], rowA[3]));
+	    }
+	  SAVE2x4_ACC (&acc0, 0);
+	  SAVE2x4_ACC (&acc1, 4);
+	  SAVE2x4_ACC (&acc2, 8);
+	  SAVE2x4_ACC (&acc3, 12);
+	  CO += 16;
+	  AO += k << 4;
+	  BO += k << 1;
+	}
+      i = (m & 15) >> 3;
+      /* Loop for m >= 8. */
+      for (j = 0; j < i; j++)
+	{
+	  IFLOAT *BO = B;
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  __vector_quad acc0, acc1;
+	  __builtin_mma_xxsetaccz (&acc0);
+	  __builtin_mma_xxsetaccz (&acc1);
+	  BLASLONG l = 0;
+	  for (l = 0; l < k / 2; l++)
+	    {
+	      vector short rowB =
+		{ BO[(l << 2) + 0], BO[(l << 2) + 2], BO[(l << 2) + 1],
+		BO[(l << 2) + 3],
+		0, 0, 0, 0
+	      };
+	      vec_t *rowA = (vec_t *) & (AO[l << 4]);
+	      MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[1]));
+	      MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[1]));
+	    }
+	  if (k % 2 == 1)
+	    {
+	      if (k > 1)
+		l = (k / 2) << 2;
+	      vector short rowB = { BO[l + 0], 0, BO[l + 1], 0, 0, 0, 0, 0 };
+	      vec_t *rowA = (vec_t *) & (AO[(l << 2)]);
+	      MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[1]));
+	      MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[1]));
+	    }
+	  SAVE2x4_ACC (&acc0, 0);
+	  SAVE2x4_ACC (&acc1, 4);
+	  CO += 8;
+	  AO += k << 3;
+	  BO += k << 1;
+	}
+      i = (m & 7) >> 2;
+      /* Loop for m >= 4. */
+      for (j = 0; j < i; j++)
+	{
+	  IFLOAT *BO = B;
+	  v4sf_t *rowC;
+	  v4sf_t result[4];
+	  __vector_quad acc0;
+	  __builtin_mma_xxsetaccz (&acc0);
+	  BLASLONG l = 0;
+	  for (l = 0; l < k / 2; l++)
+	    {
+	      vector short rowB =
+		{ BO[(l << 2) + 0], BO[(l << 2) + 2], BO[(l << 2) + 1],
+		BO[(l << 2) + 3],
+		0, 0, 0, 0
+	      };
+	      vec_t *rowA = (vec_t *) & (AO[l << 3]);
+	      MMA (&acc0, (vec_t) rowB, MERGE_ROW (rowA[0]));
+	    }
+	  if (k % 2 == 1)
+	    {
+	      if (k > 1)
+		l = (k / 2) << 2;
+	      vector short rowB = { BO[l + 0], 0, BO[l + 1], 0, 0, 0, 0, 0 };
+	      vec_t *rowA = (vec_t *) & (AO[l << 1]);
+	      MMA (&acc0, (vec_t) rowB, MERGE_ROW (rowA[0]));
+	    }
+	  SAVE2x4_ACC (&acc0, 0);
+	  CO += 4;
+	  AO += k << 2;
+	  BO += k << 1;
+	}
+      i = (m & 3) >> 1;
+      /* Loop for m >= 2. */
+      for (j = 0; j < i; j++)
+	{
+	  IFLOAT *BO = B;
+	  BLASLONG l = 0;
+	  v4sf_t t = { 0, 0, 0, 0 };
+	  for (l = 0; l < (k << 1); l += 2)
+	    {
+	      v4sf_t rowA =
+		{ BF16TOF32 (AO[l]), BF16TOF32 (AO[l]), BF16TOF32 (AO[l + 1]),
+		BF16TOF32 (AO[l + 1])
+	      };
+	      v4sf_t rowB =
+		{ BF16TOF32 (BO[l]), BF16TOF32 (BO[l + 1]), BF16TOF32 (BO[l]),
+		BF16TOF32 (BO[l + 1])
+	      };
+	      t += rowA * rowB;
+	    }
+	  t = t * valpha;
+	  CO[0 * ldc] += t[0];
+	  CO[1 * ldc] += t[1];
+	  CO[0 * ldc + 1] += t[2];
+	  CO[1 * ldc + 1] += t[3];
+	  CO += 2;
+	  AO += k << 1;
+	  BO += k << 1;
+	}
+      i = (m & 1) >> 0;
+      /* Loop for m = 1. */
+      for (j = 0; j < i; j++)
+	{
+	  IFLOAT *BO = B;
+	  BLASLONG l = 0;
+	  v4sf_t t = { 0, 0, 0, 0 };
+	  for (l = 0; l < k; l++)
+	    {
+	      v4sf_t rowA = { BF16TOF32 (AO[l]), BF16TOF32 (AO[l]), 0, 0 };
+	      v4sf_t rowB =
+		{ BF16TOF32 (BO[l << 1]), BF16TOF32 (BO[(l << 1) + 1]), 0,
+		0
+	      };
+	      t += rowA * rowB;
+	    }
+	  CO[0 * ldc] += t[0] * alpha;
+	  CO[1 * ldc] += t[1] * alpha;
+	  CO += 1;
+	  AO += k;
+	  BO += k << 1;
+	}
+      B += k << 1;
+    }
+  N = (n & 1) >> 0;
+  /* Loop for n = 1. */
+  for (i1 = 0; i1 < N; i1++)
+    {
+      BLASLONG i;
+      FLOAT *CO;
+      IFLOAT *AO;
+      CO = C;
+      C += ldc;
+      AO = A;
+      i = m;
+      /* Loop for m >= 16. */
+      while (i >= 16)
+	{
+	  IFLOAT *BO = B;
+	  BLASLONG l = 0;
+	  v4sf_t t = { 0, 0, 0, 0 };
+	  v4sf_t t1 = { 0, 0, 0, 0 };
+	  v4sf_t t2 = { 0, 0, 0, 0 };
+	  v4sf_t t3 = { 0, 0, 0, 0 };
+	  for (l = 0; l < k; l++)
+	    {
+	      v4sf_t rowB =
+		{ BF16TOF32 (BO[l]), BF16TOF32 (BO[l]), BF16TOF32 (BO[l]),
+		BF16TOF32 (BO[l])
+	      };
+	      v4sf_t rowA =
+		{ BF16TOF32 (AO[l << 4]), BF16TOF32 (AO[(l << 4) + 1]),
+		BF16TOF32 (AO[(l << 4) + 2]),
+		BF16TOF32 (AO[(l << 4) + 3])
+	      };
+	      v4sf_t rowA1 =
+		{ BF16TOF32 (AO[(l << 4) + 4]), BF16TOF32 (AO[(l << 4) + 5]),
+		BF16TOF32 (AO[(l << 4) + 6]),
+		BF16TOF32 (AO[(l << 4) + 7])
+	      };
+	      v4sf_t rowA2 =
+		{ BF16TOF32 (AO[(l << 4) + 8]), BF16TOF32 (AO[(l << 4) + 9]),
+		BF16TOF32 (AO[(l << 4) + 10]),
+		BF16TOF32 (AO[(l << 4) + 11])
+	      };
+	      v4sf_t rowA3 = { BF16TOF32 (AO[(l << 4) + 12]),
+		BF16TOF32 (AO[(l << 4) + 13]), BF16TOF32 (AO[(l << 4) + 14]),
+		BF16TOF32 (AO[(l << 4) + 15])
+	      };
+	      t += rowA * rowB;
+	      t1 += rowA1 * rowB;
+	      t2 += rowA2 * rowB;
+	      t3 += rowA3 * rowB;
+	    }
+	  t = t * valpha;
+	  t1 = t1 * valpha;
+	  t2 = t2 * valpha;
+	  t3 = t3 * valpha;
+	  CO[0] += t[0];
+	  CO[1] += t[1];
+	  CO[2] += t[2];
+	  CO[3] += t[3];
+	  CO[4] += t1[0];
+	  CO[5] += t1[1];
+	  CO[6] += t1[2];
+	  CO[7] += t1[3];
+	  CO[8] += t2[0];
+	  CO[9] += t2[1];
+	  CO[10] += t2[2];
+	  CO[11] += t2[3];
+	  CO[12] += t3[0];
+	  CO[13] += t3[1];
+	  CO[14] += t3[2];
+	  CO[15] += t3[3];
+	  AO += k << 4;
+	  BO += k;
+	  CO += 16;
+	  i -= 16;
+	}
+      /* Loop for m >= 8. */
+      while (i >= 8)
+	{
+	  IFLOAT *BO = B;
+	  BLASLONG l = 0;
+	  v4sf_t t = { 0, 0, 0, 0 };
+	  v4sf_t t1 = { 0, 0, 0, 0 };
+	  for (l = 0; l < k; l++)
+	    {
+	      v4sf_t rowB =
+		{ BF16TOF32 (BO[l]), BF16TOF32 (BO[l]), BF16TOF32 (BO[l]),
+		BF16TOF32 (BO[l])
+	      };
+	      v4sf_t rowA =
+		{ BF16TOF32 (AO[l << 3]), BF16TOF32 (AO[(l << 3) + 1]),
+		BF16TOF32 (AO[(l << 3) + 2]),
+		BF16TOF32 (AO[(l << 3) + 3])
+	      };
+	      v4sf_t rowA1 =
+		{ BF16TOF32 (AO[(l << 3) + 4]), BF16TOF32 (AO[(l << 3) + 5]),
+		BF16TOF32 (AO[(l << 3) + 6]),
+		BF16TOF32 (AO[(l << 3) + 7])
+	      };
+	      t += rowA * rowB;
+	      t1 += rowA1 * rowB;
+	    }
+	  t = t * valpha;
+	  t1 = t1 * valpha;
+	  CO[0] += t[0];
+	  CO[1] += t[1];
+	  CO[2] += t[2];
+	  CO[3] += t[3];
+	  CO[4] += t1[0];
+	  CO[5] += t1[1];
+	  CO[6] += t1[2];
+	  CO[7] += t1[3];
+	  AO += k << 3;
+	  BO += k;
+	  CO += 8;
+	  i -= 8;
+	}
+      /* Loop for m >= 4. */
+      while (i >= 4)
+	{
+	  IFLOAT *BO = B;
+	  BLASLONG l = 0;
+	  v4sf_t t = { 0, 0, 0, 0 };
+	  for (l = 0; l < k; l++)
+	    {
+	      v4sf_t rowB =
+		{ BF16TOF32 (BO[l]), BF16TOF32 (BO[l]), BF16TOF32 (BO[l]),
+		BF16TOF32 (BO[l])
+	      };
+	      v4sf_t rowA =
+		{ BF16TOF32 (AO[l << 2]), BF16TOF32 (AO[(l << 2) + 1]),
+		BF16TOF32 (AO[(l << 2) + 2]),
+		BF16TOF32 (AO[(l << 2) + 3])
+	      };
+	      t += rowA * rowB;
+	    }
+	  t = t * valpha;
+	  CO[0] += t[0];
+	  CO[1] += t[1];
+	  CO[2] += t[2];
+	  CO[3] += t[3];
+	  AO += k << 2;
+	  BO += k;
+	  CO += 4;
+	  i -= 4;
+	}
+      /* Loop for m >= 2. */
+      while (i >= 2)
+	{
+	  IFLOAT *BO = B;
+	  BLASLONG l = 0;
+	  v4sf_t t = { 0, 0, 0, 0 };
+	  for (l = 0; l < k; l++)
+	    {
+	      v4sf_t rowB = { BF16TOF32 (BO[l]), BF16TOF32 (BO[l]), 0, 0 };
+	      v4sf_t rowA =
+		{ BF16TOF32 (AO[l << 1]), BF16TOF32 (AO[(l << 1) + 1]), 0,
+		0
+	      };
+	      t += rowA * rowB;
+	    }
+	  t = t * valpha;
+	  CO[0] += t[0];
+	  CO[1] += t[1];
+	  AO += k << 1;
+	  BO += k;
+	  CO += 2;
+	  i -= 2;
+	}
+      /* Loop for m = 1. */
+      while (i >= 1)
+	{
+	  IFLOAT *BO = B;
+	  BLASLONG l = 0;
+	  FLOAT t = 0;
+	  for (l = 0; l < k; l++)
+	    {
+	      t += BF16TOF32 (AO[l]) * BF16TOF32 (BO[l]);
+	    }
+	  AO += k;
+	  BO += k;
+	  CO[0] += t * alpha;
+	  CO += 1;
+	  i -= 1;
+	}
+
+      B += k;
+    }
+
+  return 0;
+}
diff --git a/param.h b/param.h
index fd0ea7599..e8cf53f0a 100644
--- a/param.h
+++ b/param.h
@@ -2297,6 +2297,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #endif
 
+#if defined(POWER10)
+#undef SHGEMM_DEFAULT_UNROLL_N
+#undef SHGEMM_DEFAULT_UNROLL_M
+#undef SHGEMM_DEFAULT_P
+#undef SHGEMM_DEFAULT_R
+#undef SHGEMM_DEFAULT_Q
+#define SHGEMM_DEFAULT_UNROLL_M 16
+#define SHGEMM_DEFAULT_UNROLL_N 8
+#define SHGEMM_DEFAULT_P 832
+#define SHGEMM_DEFAULT_Q 1026
+#define SHGEMM_DEFAULT_R 4096
+#endif
+
 #if defined(SPARC) && defined(V7)
 
 #define SNUMOPT		4

From e30ad0e521e77d3b72b8d46c18434cc911374f8d Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Fri, 26 Jun 2020 09:00:43 +0200
Subject: [PATCH 35/48] Strip UTF8 byte order marker from source

---
 kernel/x86_64/sgemm_kernel_8x4_haswell_2.c        | 2 +-
 kernel/x86_64/strsm_kernel_8x4_haswell_LN.c       | 2 +-
 kernel/x86_64/strsm_kernel_8x4_haswell_L_common.h | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/kernel/x86_64/sgemm_kernel_8x4_haswell_2.c b/kernel/x86_64/sgemm_kernel_8x4_haswell_2.c
index 5ab3e6d1f..a2e78c58d 100644
--- a/kernel/x86_64/sgemm_kernel_8x4_haswell_2.c
+++ b/kernel/x86_64/sgemm_kernel_8x4_haswell_2.c
@@ -1,4 +1,4 @@
-/* %0 = "+r"(a_pointer), %1 = "+r"(b_pointer), %2 = "+r"(c_pointer), %3 = "+r"(ldc_in_bytes), %4 for k_count, %5 for c_store, %6 = b_pref */
+/* %0 = "+r"(a_pointer), %1 = "+r"(b_pointer), %2 = "+r"(c_pointer), %3 = "+r"(ldc_in_bytes), %4 for k_count, %5 for c_store, %6 = b_pref */
 /* r10 = tmp, r11 = m_counter, r12 = k << 2(const), r13 = tmp, r14 = b_head_pos(const), r15 = tmp */
 
 /* m = 8 *//* ymm0 for alpha, ymm1-ymm3 for temporary use, ymm4-ymm15 for accumulators */
diff --git a/kernel/x86_64/strsm_kernel_8x4_haswell_LN.c b/kernel/x86_64/strsm_kernel_8x4_haswell_LN.c
index 4131debb1..5410bd4ae 100644
--- a/kernel/x86_64/strsm_kernel_8x4_haswell_LN.c
+++ b/kernel/x86_64/strsm_kernel_8x4_haswell_LN.c
@@ -1,4 +1,4 @@
-#include "common.h"
+#include "common.h"
 #include <stdint.h>
 #include "strsm_kernel_8x4_haswell_L_common.h"
 
diff --git a/kernel/x86_64/strsm_kernel_8x4_haswell_L_common.h b/kernel/x86_64/strsm_kernel_8x4_haswell_L_common.h
index cfa56da97..2862a5b8d 100644
--- a/kernel/x86_64/strsm_kernel_8x4_haswell_L_common.h
+++ b/kernel/x86_64/strsm_kernel_8x4_haswell_L_common.h
@@ -1,4 +1,4 @@
-/* r11 = m_counter, r12 = size_of_k_elements, r13 = kk, r14 = b_head, r15 = a_head */
+/* r11 = m_counter, r12 = size_of_k_elements, r13 = kk, r14 = b_head, r15 = a_head */
 /* register i/o: %0 = a_ptr, %1 = b_ptr, %2 = c_ptr, %3 = c_tmp, %4 = ldc, %5 = k_counter */
 /* memory input: %6 = K, %7 = offset, %8 = {1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0}, %9 = {0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0}, %10 = M */
 

From 584ef8d4ae57d9eda3a8e27b84d2d37c60e8e4a5 Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Sat, 27 Jun 2020 14:36:37 +0200
Subject: [PATCH 36/48] Add support for Comet Lake H & S

---
 driver/others/dynamic.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/driver/others/dynamic.c b/driver/others/dynamic.c
index 7677f265a..c03b0b21d 100644
--- a/driver/others/dynamic.c
+++ b/driver/others/dynamic.c
@@ -619,7 +619,7 @@ static gotoblas_t *get_coretype(void){
 	  }
 	}
       case 10:
-    if (model == 6) {
+    if (model == 5 || model == 6) {
 	  if(support_avx2())
 	    return &gotoblas_HASWELL;
 	  if(support_avx()) {

From 83f47468254c5bca8e86a659e709de3f2cc4ffd4 Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Sat, 27 Jun 2020 14:41:24 +0200
Subject: [PATCH 37/48] Add support for Comet Lake H and S

---
 cpuid_x86.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/cpuid_x86.c b/cpuid_x86.c
index 3538690b9..356800b78 100644
--- a/cpuid_x86.c
+++ b/cpuid_x86.c
@@ -1409,6 +1409,7 @@ int get_cpuname(void){
     }
       case 10: //family 6 exmodel 10
         switch (model) {
+    case 5: // Comet Lake H and S
     case 6: // Comet Lake U
           if(support_avx2())
             return CPUTYPE_HASWELL;
@@ -1967,16 +1968,16 @@ int get_coretype(void){
         break;
       case 10:
         switch (model) {
-    case 6:
-      // Comet Lake U
+	  case 5: // Comet Lake H and S
+    	  case 6: // Comet Lake U
             if(support_avx())
   #ifndef NO_AVX2
               return CORE_HASWELL;
   #else
-          return CORE_SANDYBRIDGE;
+              return CORE_SANDYBRIDGE;
   #endif
             else
-          return CORE_NEHALEM;
+              return CORE_NEHALEM;
         }
       case 5:
         switch (model) {

From 634e1305f9caf640dfa42e61d4da564d8aedf16b Mon Sep 17 00:00:00 2001
From: EGuesnet <51407514+EGuesnet@users.noreply.github.com>
Date: Tue, 30 Jun 2020 15:16:39 +0200
Subject: [PATCH 38/48] Update cgemm_kernel_8x4_power8.S

---
 kernel/power/cgemm_kernel_8x4_power8.S | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/kernel/power/cgemm_kernel_8x4_power8.S b/kernel/power/cgemm_kernel_8x4_power8.S
index 2bc99974f..6be8c128c 100644
--- a/kernel/power/cgemm_kernel_8x4_power8.S
+++ b/kernel/power/cgemm_kernel_8x4_power8.S
@@ -424,7 +424,7 @@ L999:
 	lwz	r16,  204(SP)
 	lwz	r15,  208(SP)
 	lwz	r14,  212(SP)
-        addi    r11, 224
+        addi    r11, SP, 224
 #endif
         lvx     v20, r11, r0
         addi    r11, r11, 16
@@ -459,4 +459,4 @@ L999:
 	blr
 
 	EPILOGUE
-#endif^
+#endif

From 4ab3651591d231c69f0f16dbeae26e2cc7ee819f Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Thu, 2 Jul 2020 17:00:15 +0200
Subject: [PATCH 39/48] Option -mavx2 requires at least gcc 4.7

---
 Makefile.x86_64 | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/Makefile.x86_64 b/Makefile.x86_64
index f2de51ef4..2676bd258 100644
--- a/Makefile.x86_64
+++ b/Makefile.x86_64
@@ -31,14 +31,24 @@ ifeq ($(CORE), HASWELL)
 ifndef DYNAMIC_ARCH
 ifndef NO_AVX2
 ifeq ($(C_COMPILER), GCC)
+# AVX2 support was added in 4.7.0
+GCCVERSIONGTEQ4 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \>= 4)
+GCCMINORVERSIONGTEQ7 := $(shell expr `$(CC) -dumpversion | cut -f2 -d.` \>= 7)
+ifeq ($(GCCVERSIONGTEQ4)$(GCCMINORVERSIONGTEQ7), 11)
 CCOMMON_OPT += -mavx2
 endif
+endif
 ifeq ($(F_COMPILER), GFORTRAN)
+# AVX2 support was added in 4.7.0
+GCCVERSIONGTEQ4 := $(shell expr `$(FC) -dumpversion | cut -f1 -d.` \>= 4)
+GCCMINORVERSIONGTEQ7 := $(shell expr `$(FC) -dumpversion | cut -f2 -d.` \>= 7)
+ifeq ($(GCCVERSIONGTEQ4)$(GCCMINORVERSIONGTEQ7), 11)
 FCOMMON_OPT += -mavx2
 endif
 endif
 endif
 endif
+endif
 
 
 

From 10a2923f640e9b1aa3f8bca34e71481586aa3acd Mon Sep 17 00:00:00 2001
From: Jussi Enkovaara <jussi.enkovaara@csc.fi>
Date: Tue, 7 Jul 2020 13:35:43 +0300
Subject: [PATCH 40/48] fixes #2238

Always obey omp_get_max_threads() when build with USE_OPENMP
---
 common_thread.h | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/common_thread.h b/common_thread.h
index 6ec40e096..ec0c65b22 100644
--- a/common_thread.h
+++ b/common_thread.h
@@ -132,18 +132,18 @@ extern int blas_server_avail;
 static __inline int num_cpu_avail(int level) {
 
 #ifdef USE_OPENMP
-	int openmp_nthreads=0;
+	int openmp_nthreads=omp_get_max_threads();
 #endif
 
+#ifndef USE_OPENMP 
   if (blas_cpu_number == 1
-
-#ifdef USE_OPENMP
-      || omp_in_parallel()
 #endif
-      ) return 1;
+#ifdef USE_OPENMP
+     if (openmp_nthreads == 1 || omp_in_parallel()
+#endif
+      ) return 1;        
 
 #ifdef USE_OPENMP
-  openmp_nthreads=omp_get_max_threads();
   if (blas_cpu_number != openmp_nthreads) {
 	  goto_set_num_threads(openmp_nthreads);
   }

From 45d819ca82f6a562de04cc5cfd3b70fd513fd4b8 Mon Sep 17 00:00:00 2001
From: Rajalakshmi Srinivasaraghavan <rajis@linux.ibm.com>
Date: Tue, 7 Jul 2020 11:25:20 -0500
Subject: [PATCH 41/48] Changing mcpu option as power10

As compiler enabled mcpu option as power10, changing it from future.
---
 Makefile.power                | 8 ++++----
 driver/others/dynamic_power.c | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Makefile.power b/Makefile.power
index 5c431860f..beb311945 100644
--- a/Makefile.power
+++ b/Makefile.power
@@ -11,11 +11,11 @@ endif
 
 ifeq ($(CORE), POWER10)
 ifeq ($(USE_OPENMP), 1)
-COMMON_OPT += -Ofast -mcpu=future -mtune=future -mvsx -malign-power -DUSE_OPENMP -fno-fast-math -fopenmp
-FCOMMON_OPT += -O2 -frecursive -mcpu=future -mtune=future -malign-power -DUSE_OPENMP -fno-fast-math -fopenmp
+COMMON_OPT += -Ofast -mcpu=power10 -mtune=power10 -mvsx -malign-power -DUSE_OPENMP -fno-fast-math -fopenmp
+FCOMMON_OPT += -O2 -frecursive -mcpu=power10 -mtune=power10 -malign-power -DUSE_OPENMP -fno-fast-math -fopenmp
 else
-COMMON_OPT += -Ofast -mcpu=future -mtune=future -mvsx -malign-power -fno-fast-math
-FCOMMON_OPT += -O2 -frecursive -mcpu=future -mtune=future -malign-power -fno-fast-math
+COMMON_OPT += -Ofast -mcpu=power10 -mtune=power10 -mvsx -malign-power -fno-fast-math
+FCOMMON_OPT += -O2 -frecursive -mcpu=power10 -mtune=power10 -malign-power -fno-fast-math
 endif
 endif
 
diff --git a/driver/others/dynamic_power.c b/driver/others/dynamic_power.c
index 811a5fae3..f625b9431 100644
--- a/driver/others/dynamic_power.c
+++ b/driver/others/dynamic_power.c
@@ -45,7 +45,7 @@ static gotoblas_t *get_coretype(void) {
 		return &gotoblas_POWER9;
 #endif
 #if (!defined __GNUC__) || ( __GNUC__ >= 11)
-	if (__builtin_cpu_is("isa_3_1") && __builtin_cpu_supports ("mma"))
+	if (__builtin_cpu_supports ("arch_3_1") && __builtin_cpu_supports ("mma"))
 		return &gotoblas_POWER10;
 #endif
 	return NULL;

From 1d63631afe3da02ade6aa7ca7698b08754c148a8 Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Thu, 9 Jul 2020 11:42:02 +0200
Subject: [PATCH 42/48] Add lapack-test

---
 CMakeLists.txt | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index bb5322a1d..7cdc4181a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -6,8 +6,7 @@ cmake_minimum_required(VERSION 2.8.5)
 project(OpenBLAS C ASM)
 set(OpenBLAS_MAJOR_VERSION 0)
 set(OpenBLAS_MINOR_VERSION 3)
-set(OpenBLAS_PATCH_VERSION 10.dev)
-
+set(OpenBLAS_PATCH_VERSION 9.dev)
 set(OpenBLAS_VERSION "${OpenBLAS_MAJOR_VERSION}.${OpenBLAS_MINOR_VERSION}.${OpenBLAS_PATCH_VERSION}")
 
 # Adhere to GNU filesystem layout conventions
@@ -236,6 +235,10 @@ if (NOT MSVC AND NOT NOFORTRAN)
   endif()
 endif()
 
+if (NOT NOFORTRAN)
+  add_subdirectory(lapack-netlib/TESTING)
+endif()
+
 set_target_properties(${OpenBLAS_LIBNAME} PROPERTIES
   VERSION ${OpenBLAS_MAJOR_VERSION}.${OpenBLAS_MINOR_VERSION}
   SOVERSION ${OpenBLAS_MAJOR_VERSION}

From 60188a8c82398281794956f41c3e7232f0004532 Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Thu, 9 Jul 2020 11:44:31 +0200
Subject: [PATCH 43/48] Append crude hack for enabling lapack tests in the
 OpenBLAS build

---
 lapack-netlib/TESTING/CMakeLists.txt | 393 +++++++++++++++++++++++++++
 1 file changed, 393 insertions(+)

diff --git a/lapack-netlib/TESTING/CMakeLists.txt b/lapack-netlib/TESTING/CMakeLists.txt
index d5ca95013..755826bfe 100644
--- a/lapack-netlib/TESTING/CMakeLists.txt
+++ b/lapack-netlib/TESTING/CMakeLists.txt
@@ -1,3 +1,5 @@
+enable_testing()
+
 if(MSVC_VERSION)
 #  string(REPLACE "/STACK:10000000" "/STACK:900000000000000000"
 #    CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}")
@@ -168,3 +170,394 @@ if(PYTHONINTERP_FOUND)
     COMMAND ${PYTHON_EXECUTABLE} "lapack_testing.py"
     )
 endif()
+
+
+
+# $1 exec, $2 input, $3 output_result
+FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh
+"rm -f $3\n"
+"$1 < $2\n"
+"grep -q FATAL $3\n"
+"if [ $? -eq 0 ]; then\n"
+"echo Error\n"
+"exit 1\n"
+"else\n"
+"exit 0\n"
+"fi\n"
+)
+
+
+add_test(NAME "REAL_LAPACK_linear_equation_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/LIN/xlintsts" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/stest.in" "${CMAKE_CURRENT_BINARY_DIR}/stest.out"
+)
+add_test(NAME "COMPLEX_LAPACK_linear_equation_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/LIN/xlintstc" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/ctest.in" "${CMAKE_CURRENT_BINARY_DIR}/ctest.out"
+)
+add_test(NAME "DOUBLE_PRECISION_LAPACK_linear_equation_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/LIN//xlintstd" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/dtest.in" "${CMAKE_CURRENT_BINARY_DIR}/dtest.out"
+)
+add_test(NAME "COMPLEX16_LAPACK_linear_equation_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/LIN//xlintstz" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/ztest.in" "${CMAKE_CURRENT_BINARY_DIR}/ztest.out"
+)
+
+add_test(NAME "SINGLE-DOUBLE_PRECISION_LAPACK_prototype_linear_equation_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/LIN/xlintstds" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/dstest.in" " ${CMAKE_CURRENT_BINARY_DIR}/dstest.out"
+)
+# ======== COMPLEX-COMPLEX16 LIN TESTS ========================
+
+add_test(NAME "Testing_COMPLEX-COMPLEX16_LAPACK_prototype_linear_equation_routines"
+	 COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/LIN/xlintstzc" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/zctest.in" " ${CMAKE_CURRENT_BINARY_DIR}/zctest.out"
+)
+
+# ======== SINGLE RFP LIN TESTS ========================
+
+add_test(NAME "Testing_REAL_LAPACK_RFP_prototype_linear_equation_routines"
+	 COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/LIN/xlintstrfs" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/stest_rfp.in" "${CMAKE_CURRENT_BINARY_DIR}/stest_rfp.out"
+)
+
+# ======== COMPLEX16 RFP LIN TESTS ========================
+
+add_test(NAME "Testing_DOUBLE_PRECISION_LAPACK_RFP_prototype_linear_equation_routines"
+	 COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/LIN/xlintstrfd" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/dtest_rfp.in" " ${CMAKE_CURRENT_BINARY_DIR}/dtest_rfp.out"
+)
+# ======== COMPLEX16 RFP LIN TESTS ========================
+
+add_test(NAME "Testing_COMPLEX_LAPACK_RFP_prototype_linear_equation_routines"
+	 COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/LIN/xlintstrfc" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/ctest_rfp.in" " ${CMAKE_CURRENT_BINARY_DIR}/ctest_rfp.out"
+)
+
+# ======== COMPLEX16 RFP LIN TESTS ========================
+
+add_test(NAME "Testing_COMPLEX16_LAPACK_RFP_prototype_linear_equation_routines"
+	 COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/LIN/xlintstrfz" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/ztest_rfp.in" " ${CMAKE_CURRENT_BINARY_DIR}/ztest_rfp.out"
+)
+#
+#
+# ======== SINGLE EIG TESTS ===========================
+#
+
+add_test(NAME "SNEP:_Testing_Nonsymmetric_Eigenvalue_Problem_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtsts" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/nep.in" " ${CMAKE_CURRENT_BINARY_DIR}/snep.out"
+)
+
+add_test(NAME "SSEP:_Testing_Symmetric_Eigenvalue_Problem_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtsts" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/sep.in" " ${CMAKE_CURRENT_BINARY_DIR}/ssep.out"
+)
+
+add_test(NAME "SSE2:_Testing_Symmetric_Eigenvalue_Problem_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtsts" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/se2.in" " ${CMAKE_CURRENT_BINARY_DIR}/sse2.out"
+)
+
+add_test(NAME "SSVD:_Testing_Singular_Value_Decomposition_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtsts" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/svd.in" " ${CMAKE_CURRENT_BINARY_DIR}/ssvd.out"
+)
+
+add_test(NAME "SSEC:_Testing_REAL_Eigen_Condition_Routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtsts" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/sec.in" " ${CMAKE_CURRENT_BINARY_DIR}/sec.out"
+)
+
+add_test(NAME "SSEV:_Testing_REAL_Nonsymmetric_Eigenvalue_Driver"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtsts" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/sed.in" " ${CMAKE_CURRENT_BINARY_DIR}/sed.out"
+)
+
+add_test(NAME "SGG:_Testing_REAL_Nonsymmetric_Generalized_Eigenvalue_Problem_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtsts" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/sgg.in" " ${CMAKE_CURRENT_BINARY_DIR}/sgg.out"
+)
+
+add_test(NAME "SGD:_Testing_REAL_Nonsymmetric_Generalized_Eigenvalue_Problem_driver_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtsts" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/sgd.in" " ${CMAKE_CURRENT_BINARY_DIR}/sgd.out"
+)
+
+add_test(NAME "SSB:_Testing_REAL_Symmetric_Eigenvalue_Problem_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtsts" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/ssb.in" " ${CMAKE_CURRENT_BINARY_DIR}/ssb.out"
+)
+
+add_test(NAME "SSG:_Testing_REAL_Symmetric_Generalized_Eigenvalue_Problem_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtsts" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/ssg.in" " ${CMAKE_CURRENT_BINARY_DIR}/ssg.out"
+)
+
+add_test(NAME "SGEBAL:_Testing_the_balancing_of_a_REAL_general_matrix"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtsts" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/sbal.in" " ${CMAKE_CURRENT_BINARY_DIR}/sbal.out"
+)
+
+add_test(NAME "SGEBAK:_Testing_the_back_transformation_of_a_REAL_balanced_matrix"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtsts" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/sbak.in" " ${CMAKE_CURRENT_BINARY_DIR}/sbak.out"
+)
+
+add_test(NAME "SGGBAL:_Testing_the_balancing_of_a_pair_of_REAL_general_matrices"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtsts" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/sgbal.in" " ${CMAKE_CURRENT_BINARY_DIR}/sgbal.out"
+)
+
+add_test(NAME "SGGBAK:_Testing_the_back_transformation_of_a_pair_of_REAL_balanced_matrices"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtsts" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/sgbak.in" " ${CMAKE_CURRENT_BINARY_DIR}/sgbak.out"
+)
+
+add_test(NAME "SBB:_Testing_banded_Singular_Value_Decomposition_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtsts" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/sbb.in" " ${CMAKE_CURRENT_BINARY_DIR}/sbb.out"
+)
+
+add_test(NAME "SGLM:_Testing_Generalized_Linear_Regression_Model_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtsts" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/glm.in" " ${CMAKE_CURRENT_BINARY_DIR}/sglm.out"
+)
+
+add_test(NAME "SGQR:_Testing_Generalized_QR_and_RQ_factorization_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtsts" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/gqr.in" " ${CMAKE_CURRENT_BINARY_DIR}/sgqr.out"
+)
+
+add_test(NAME "SGSV:_Testing_Generalized_Singular_Value_Decomposition_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtsts" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/gsv.in" "${CMAKE_CURRENT_BINARY_DIR}/sgsv.out"
+)
+
+add_test(NAME "SCSD:_Testing_CS_Decomposition_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtsts" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/csd.in" " ${CMAKE_CURRENT_BINARY_DIR}/scsd.out"
+)
+
+add_test(NAME "SLSE:_Testing_Constrained_Linear_Least_Squares_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtsts" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/lse.in" " ${CMAKE_CURRENT_BINARY_DIR}/slse.out"
+)
+
+# ======== COMPLEX EIG TESTS ===========================
+
+add_test(NAME "CNEP:_Testing_Nonsymmetric_Eigenvalue_Problem_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstc" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/nep.in" " ${CMAKE_CURRENT_BINARY_DIR}/cnep.out"
+)
+
+add_test(NAME "CSEP:_Testing_Symmetric_Eigenvalue_Problem_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstc" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/sep.in" " ${CMAKE_CURRENT_BINARY_DIR}/csep.out"
+)
+
+add_test(NAME "CSE2:_Testing_Symmetric_Eigenvalue_Problem_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstc" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/se2.in" " ${CMAKE_CURRENT_BINARY_DIR}/cse2.out"
+)
+
+add_test(NAME "CSVD:_Testing_Singular_Value_Decomposition_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstc" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/svd.in" " ${CMAKE_CURRENT_BINARY_DIR}/csvd.out"
+)
+
+add_test(NAME "CEC:_Testing_COMPLEX_Eigen_Condition_Routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstc" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/cec.in" " ${CMAKE_CURRENT_BINARY_DIR}/cec.out"
+)
+
+add_test(NAME "CES:_Testing_COMPLEX_Nonsymmetric_Schur_Form_Driver"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstc" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/ced.in" " ${CMAKE_CURRENT_BINARY_DIR}/ced.out"
+)
+
+add_test(NAME "CGG:_Testing_COMPLEX_Nonsymmetric_Generalized_Eigenvalue_Problem_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstc" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/cgg.in" " ${CMAKE_CURRENT_BINARY_DIR}/cgg.out"
+)
+
+add_test(NAME "CGD:_Testing_COMPLEX_Nonsymmetric_Generalized_Eigenvalue_Problem_driver_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstc" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/cgd.in" " ${CMAKE_CURRENT_BINARY_DIR}/cgd.out"
+)
+
+add_test(NAME "CHB:_Testing_Hermitian_Eigenvalue_Problem_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstc" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/csb.in" " ${CMAKE_CURRENT_BINARY_DIR}/csb.out"
+)
+
+add_test(NAME "CSG:_Testing_Symmetric_Generalized_Eigenvalue_Problem_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstc" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/csg.in" " ${CMAKE_CURRENT_BINARY_DIR}/csg.out"
+)
+
+add_test(NAME "CGEBAL:_Testing_the_balancing_of_a_COMPLEX_general_matrix"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstc" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/cbal.in" " ${CMAKE_CURRENT_BINARY_DIR}/cbal.out"
+)
+
+add_test(NAME "CGEBAK:_Testing_the_back_transformation_of_a_COMPLEX_balanced_matrix"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstc" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/cbak.in" " ${CMAKE_CURRENT_BINARY_DIR}/cbak.out"
+)
+
+add_test(NAME "CGGBAL:_Testing_the_balancing_of_a_pair_of_COMPLEX_general_matrices"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstc" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/cgbal.in" " ${CMAKE_CURRENT_BINARY_DIR}/cgbal.out"
+)
+
+add_test(NAME "CGGBAK:_Testing_the_back_transformation_of_a_pair_of_COMPLEX_balanced_matrices"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstc" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/cgbak.in" " ${CMAKE_CURRENT_BINARY_DIR}/cgbak.out"
+)
+
+add_test(NAME "CBB:_Testing_banded_Singular_Value_Decomposition_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstc" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/cbb.in" " ${CMAKE_CURRENT_BINARY_DIR}/cbb.out"
+)
+
+add_test(NAME "CGLM:_Testing_Generalized_Linear_Regression_Model_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstc" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/glm.in" " ${CMAKE_CURRENT_BINARY_DIR}/cglm.out"
+)
+
+add_test(NAME "CGQR:_Testing_Generalized_QR_and_RQ_factorization_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstc" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/gqr.in" " ${CMAKE_CURRENT_BINARY_DIR}/cgqr.out"
+)
+
+add_test(NAME "CGSV:_Testing_Generalized_Singular_Value_Decomposition_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstc" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/gsv.in" " ${CMAKE_CURRENT_BINARY_DIR}/cgsv.out"
+)
+
+add_test(NAME "CCSD:_Testing_CS_Decomposition_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstc" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/csd.in" " ${CMAKE_CURRENT_BINARY_DIR}/ccsd.out"
+)
+
+add_test(NAME "CLSE:_Testing_Constrained_Linear_Least_Squares_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstc" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/lse.in" " ${CMAKE_CURRENT_BINARY_DIR}/clse.out"
+)
+
+# ======== DOUBLE EIG TESTS ===========================
+
+add_test(NAME "DNEP:_Testing_Nonsymmetric_Eigenvalue_Problem_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstd" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/nep.in" " ${CMAKE_CURRENT_BINARY_DIR}/dnep.out"
+)
+
+add_test(NAME "DSEP:_Testing_Symmetric_Eigenvalue_Problem_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstd" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/sep.in" " ${CMAKE_CURRENT_BINARY_DIR}/dsep.out"
+)
+
+add_test(NAME "DSE2:_Testing_Symmetric_Eigenvalue_Problem_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstd" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/se2.in" " ${CMAKE_CURRENT_BINARY_DIR}/dse2.out"
+)
+
+add_test(NAME "DSVD:_Testing_Singular_Value_Decomposition_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstd" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/svd.in" " ${CMAKE_CURRENT_BINARY_DIR}/dsvd.out"
+)
+
+add_test(NAME "DEC:_Testing_DOUBLE_PRECISION_Eigen_Condition_Routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstd" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/dec.in" " ${CMAKE_CURRENT_BINARY_DIR}/dec.out"
+)
+
+add_test(NAME "DEV:_Testing_DOUBLE_PRECISION_Nonsymmetric_Eigenvalue_Driver"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstd" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/ded.in" " ${CMAKE_CURRENT_BINARY_DIR}/ded.out"
+)
+
+add_test(NAME "DGG:_Testing_DOUBLE_PRECISION_Nonsymmetric_Generalized_Eigenvalue_Problem_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstd" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/dgg.in" " ${CMAKE_CURRENT_BINARY_DIR}/dgg.out"
+)
+
+add_test(NAME "DGD:_Testing_DOUBLE_PRECISION_Nonsymmetric_Generalized_Eigenvalue_Problem_driver_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstd" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/dgd.in" " ${CMAKE_CURRENT_BINARY_DIR}/dgd.out"
+)
+
+add_test(NAME "DSB:_Testing_DOUBLE_PRECISION_Symmetric_Eigenvalue_Problem_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstd" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/dsb.in" " ${CMAKE_CURRENT_BINARY_DIR}/dsb.out"
+)
+
+add_test(NAME "DSG:_Testing_DOUBLE_PRECISION_Symmetric_Generalized_Eigenvalue_Problem_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstd" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/dsg.in" " ${CMAKE_CURRENT_BINARY_DIR}/dsg.out"
+)
+
+add_test(NAME "DGEBAL:_Testing_the_balancing_of_a_DOUBLE_PRECISION_general_matrix"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstd" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/dbal.in" " ${CMAKE_CURRENT_BINARY_DIR}/dbal.out"
+)
+
+add_test(NAME "DGEBAK:_Testing_the_back_transformation_of_a_DOUBLE_PRECISION_balanced_matrix"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstd" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/dbak.in" " ${CMAKE_CURRENT_BINARY_DIR}/dbak.out"
+)
+
+add_test(NAME "DGGBAL:_Testing_the_balancing_of_a_pair_of_DOUBLE_PRECISION_general_matrices"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstd" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/dgbal.in" " ${CMAKE_CURRENT_BINARY_DIR}/dgbal.out"
+)
+
+add_test(NAME "DGGBAK:_Testing_the_back_transformation_of_a_pair_of_DOUBLE_PRECISION_balanced_matrices"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstd" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/dgbak.in" " ${CMAKE_CURRENT_BINARY_DIR}/dgbak.out"
+)
+
+add_test(NAME "DBB:_Testing_banded_Singular_Value_Decomposition_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstd" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/dbb.in" " ${CMAKE_CURRENT_BINARY_DIR}/dbb.out"
+)
+
+add_test(NAME "DGLM:_Testing_Generalized_Linear_Regression_Model_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstd" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/glm.in" " ${CMAKE_CURRENT_BINARY_DIR}/dglm.out"
+)
+
+add_test(NAME "DGQR:_Testing_Generalized_QR_and_RQ_factorization_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstd" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/gqr.in" " ${CMAKE_CURRENT_BINARY_DIR}/dgqr.out"
+)
+
+add_test(NAME "DGSV:_Testing_Generalized_Singular_Value_Decomposition_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstd" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/gsv.in" " ${CMAKE_CURRENT_BINARY_DIR}/dgsv.out"
+)
+
+add_test(NAME "DCSD:_Testing_CS_Decomposition_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstd" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/csd.in" " ${CMAKE_CURRENT_BINARY_DIR}/dcsd.out"
+)
+
+add_test(NAME "DLSE:_Testing_Constrained_Linear_Least_Squares_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstd" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/lse.in" " ${CMAKE_CURRENT_BINARY_DIR}/dlse.out"
+)
+
+# ======== COMPLEX16 EIG TESTS ===========================
+
+add_test(NAME "ZNEP:_Testing_Nonsymmetric_Eigenvalue_Problem_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstz" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/nep.in" " ${CMAKE_CURRENT_BINARY_DIR}/znep.out"
+)
+
+add_test(NAME "ZSEP:_Testing_Symmetric_Eigenvalue_Problem_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstz" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/sep.in" " ${CMAKE_CURRENT_BINARY_DIR}/zsep.out"
+)
+
+add_test(NAME "ZSE2:_Testing_Symmetric_Eigenvalue_Problem_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstz" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/se2.in" " ${CMAKE_CURRENT_BINARY_DIR}/zse2.out"
+)
+
+add_test(NAME "ZSVD:_Testing_Singular_Value_Decomposition_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstz" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/svd.in" " ${CMAKE_CURRENT_BINARY_DIR}/zsvd.out"
+)
+
+add_test(NAME "ZEC:_Testing_COMPLEX16_Eigen_Condition_Routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstz" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/zec.in" " ${CMAKE_CURRENT_BINARY_DIR}/zec.out"
+)
+
+add_test(NAME "ZES:_Testing_COMPLEX16_Nonsymmetric_Schur_Form_Driver"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstz" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/zed.in" " ${CMAKE_CURRENT_BINARY_DIR}/zed.out"
+)
+
+add_test(NAME "ZGG:_Testing_COMPLEX16_Nonsymmetric_Generalized_Eigenvalue_Problem_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstz" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/zgg.in" " ${CMAKE_CURRENT_BINARY_DIR}/zgg.out"
+)
+
+add_test(NAME "ZGD:_Testing_COMPLEX16_Nonsymmetric_Generalized_Eigenvalue_Problem_driver_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstz" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/zgd.in" " ${CMAKE_CURRENT_BINARY_DIR}/zgd.out"
+)
+
+add_test(NAME "ZHB:_Testing_Hermitian_Eigenvalue_Problem_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstz" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/zsb.in" " ${CMAKE_CURRENT_BINARY_DIR}/zsb.out"
+)
+
+add_test(NAME "ZSG:_Testing_Symmetric_Generalized_Eigenvalue_Problem_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstz" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/zsg.in" " ${CMAKE_CURRENT_BINARY_DIR}/zsg.out"
+)
+
+add_test(NAME "ZGEBAL:_Testing_the_balancing_of_a_COMPLEX16_general_matrix"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstz" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/zbal.in" " ${CMAKE_CURRENT_BINARY_DIR}/zbal.out"
+)
+
+add_test(NAME "ZGEBAK:_Testing_the_back_transformation_of_a_COMPLEX16_balanced_matrix"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstz" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/zbak.in" " ${CMAKE_CURRENT_BINARY_DIR}/zbak.out"
+)
+
+add_test(NAME "ZGGBAL:_Testing_the_balancing_of_a_pair_of_COMPLEX_general_matrices"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstz" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/zgbal.in" " ${CMAKE_CURRENT_BINARY_DIR}/zgbal.out"
+)
+
+add_test(NAME "ZGGBAK:_Testing_the_back_transformation_of_a_pair_of_COMPLEX16_balanced_matrices"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstz" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/zgbak.in" " ${CMAKE_CURRENT_BINARY_DIR}/zgbak.out"
+)
+
+add_test(NAME "ZBB:_Testing_banded_Singular_Value_Decomposition_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstz" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/zbb.in" " ${CMAKE_CURRENT_BINARY_DIR}/zbb.out"
+)
+
+add_test(NAME "ZGLM:_Testing_Generalized_Linear_Regression_Model_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstz" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/glm.in" " ${CMAKE_CURRENT_BINARY_DIR}/zglm.out"
+)
+
+add_test(NAME "ZGQR:_Testing_Generalized_QR_and_RQ_factorization_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstz" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/gqr.in" " ${CMAKE_CURRENT_BINARY_DIR}/zgqr.out"
+)
+
+add_test(NAME "ZGSV:_Testing_Generalized_Singular_Value_Decomposition_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstz" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/gsv.in" " ${CMAKE_CURRENT_BINARY_DIR}/zgsv.out"
+)
+
+add_test(NAME "ZCSD:_Testing_CS_Decomposition_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstz" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/csd.in" " ${CMAKE_CURRENT_BINARY_DIR}/zcsd.out"
+)
+
+add_test(NAME "Constrained_Linear_Least_Squares_routines"
+	COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/test_helper.sh" "${CMAKE_CURRENT_BINARY_DIR}/EIG/xeigtstz" "${PROJECT_SOURCE_DIR}/lapack-netlib/TESTING/lse.in" " ${CMAKE_CURRENT_BINARY_DIR}/zlse.out"
+)

From 29b5887d5f00bd94478fe84ac4518c4cb0391941 Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Thu, 9 Jul 2020 13:12:35 +0200
Subject: [PATCH 44/48] Modify for building with OpenBLAS

---
 lapack-netlib/TESTING/EIG/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lapack-netlib/TESTING/EIG/CMakeLists.txt b/lapack-netlib/TESTING/EIG/CMakeLists.txt
index 20fd25b4a..70eea8443 100644
--- a/lapack-netlib/TESTING/EIG/CMakeLists.txt
+++ b/lapack-netlib/TESTING/EIG/CMakeLists.txt
@@ -98,7 +98,7 @@ set(ZEIGTST zchkee.f
 
 macro(add_eig_executable name)
   add_executable(${name} ${ARGN})
-  target_link_libraries(${name} tmglib ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES})
+  target_link_libraries(${name} openblas)
 endmacro()
 
 if(BUILD_SINGLE)

From c502760befbb25e6a9415dbd6b1e811f711e7cf3 Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Thu, 9 Jul 2020 13:13:16 +0200
Subject: [PATCH 45/48] Modify for building with OpenBLAS

---
 lapack-netlib/TESTING/LIN/CMakeLists.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lapack-netlib/TESTING/LIN/CMakeLists.txt b/lapack-netlib/TESTING/LIN/CMakeLists.txt
index c941d3577..954cab193 100644
--- a/lapack-netlib/TESTING/LIN/CMakeLists.txt
+++ b/lapack-netlib/TESTING/LIN/CMakeLists.txt
@@ -239,7 +239,8 @@ set(ZLINTSTRFP zchkrfp.f zdrvrfp.f zdrvrf1.f zdrvrf2.f zdrvrf3.f zdrvrf4.f zerrr
 
 macro(add_lin_executable name)
   add_executable(${name} ${ARGN})
-  target_link_libraries(${name} tmglib ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES})
+  target_link_libraries(${name} openblas)
+#${TMGLIB} ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES})
 endmacro()
 
 if(BUILD_SINGLE)

From f76602474945ce7d5f930080e2b3fd016e945bc9 Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Thu, 9 Jul 2020 13:44:25 +0200
Subject: [PATCH 46/48] enable fortran for cmake

---
 lapack-netlib/TESTING/CMakeLists.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lapack-netlib/TESTING/CMakeLists.txt b/lapack-netlib/TESTING/CMakeLists.txt
index 755826bfe..80e6b3232 100644
--- a/lapack-netlib/TESTING/CMakeLists.txt
+++ b/lapack-netlib/TESTING/CMakeLists.txt
@@ -1,3 +1,5 @@
+enable_language(Fortran)
+
 enable_testing()
 
 if(MSVC_VERSION)

From d4a0299e166b33ed9d61a488018f3e1bb5491d30 Mon Sep 17 00:00:00 2001
From: Martin Kroeker <martin@ruby.chemie.uni-freiburg.de>
Date: Thu, 9 Jul 2020 13:57:27 +0200
Subject: [PATCH 47/48] Do not build lapack-test on MSVC for now (same as with
 BLAS test)

---
 CMakeLists.txt | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7cdc4181a..7e51e7e38 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -233,9 +233,6 @@ if (NOT MSVC AND NOT NOFORTRAN)
   if(NOT NO_CBLAS)
     add_subdirectory(ctest)
   endif()
-endif()
-
-if (NOT NOFORTRAN)
   add_subdirectory(lapack-netlib/TESTING)
 endif()
 

From af1e140e35cbdbb4d1f98addf3b817b1369460a3 Mon Sep 17 00:00:00 2001
From: Rajalakshmi Srinivasaraghavan <rajis@linux.ibm.com>
Date: Thu, 9 Jul 2020 21:46:06 -0500
Subject: [PATCH 48/48] Change minimum gcc version for POWER10

As the MMA patches for POWER10 are backported to gcc10.2, changing
the minimum gcc version needed to build OpenBLAS for POWER10.
---
 Makefile.system               |  7 ++++++-
 driver/others/dynamic_power.c | 12 ++++++++----
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/Makefile.system b/Makefile.system
index 1b473c59d..61ae264bf 100644
--- a/Makefile.system
+++ b/Makefile.system
@@ -286,6 +286,8 @@ GCCVERSIONEQ5 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` = 5)
 GCCVERSIONGT5 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \> 5)
 GCCVERSIONGTEQ7 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \>= 7)
 GCCVERSIONGTEQ9 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \>= 9)
+GCCVERSIONGTEQ11 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \>= 11)
+GCCVERSIONGTEQ10 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \>= 10)
 # Note that the behavior of -dumpversion is compile-time-configurable for
 # gcc-7.x and newer. Use -dumpfullversion there
 ifeq ($(GCCVERSIONGTEQ7),1)
@@ -619,9 +621,12 @@ DYNAMIC_CORE += POWER9
 else
 $(info, OpenBLAS: Your gcc version is too old to build the POWER9 kernels.)
 endif
-GCCVERSIONGTEQ11 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \>= 11)
 ifeq ($(GCCVERSIONGTEQ11), 1)
 DYNAMIC_CORE += POWER10
+else ifeq ($(GCCVERSIONEQ10), 1)
+ifeq ($(GCCMINORVERSIONGTEQ2), 1)
+DYNAMIC_CORE += POWER10
+endif
 else
 $(info, OpenBLAS: Your gcc version is too old to build the POWER10 kernels.)
 endif
diff --git a/driver/others/dynamic_power.c b/driver/others/dynamic_power.c
index f625b9431..ca1d42408 100644
--- a/driver/others/dynamic_power.c
+++ b/driver/others/dynamic_power.c
@@ -6,7 +6,11 @@ extern gotoblas_t gotoblas_POWER8;
 #if (!defined __GNUC__) || ( __GNUC__ >= 6)
 extern gotoblas_t gotoblas_POWER9;
 #endif
-#if (!defined __GNUC__) || ( __GNUC__ >= 11)
+#if (!defined __GNUC__) || ( __GNUC__ >= 11) \
+     || (__GNUC__ == 10 && __GNUC_MINOR__ >= 2)
+#define HAVE_P10_SUPPORT 1
+#endif
+#ifdef HAVE_P10_SUPPORT
 extern gotoblas_t gotoblas_POWER10;
 #endif
 
@@ -28,7 +32,7 @@ char *gotoblas_corename(void) {
 #if (!defined __GNUC__) || ( __GNUC__ >= 6)
 	if (gotoblas == &gotoblas_POWER9)	return corename[3];
 #endif
-#if (!defined __GNUC__) || ( __GNUC__ >= 11)
+#ifdef HAVE_P10_SUPPORT
 	if (gotoblas == &gotoblas_POWER10)	return corename[4];
 #endif
 	return corename[0];
@@ -44,7 +48,7 @@ static gotoblas_t *get_coretype(void) {
 	if (__builtin_cpu_is("power9"))
 		return &gotoblas_POWER9;
 #endif
-#if (!defined __GNUC__) || ( __GNUC__ >= 11)
+#ifdef HAVE_P10_SUPPORT
 	if (__builtin_cpu_supports ("arch_3_1") && __builtin_cpu_supports ("mma"))
 		return &gotoblas_POWER10;
 #endif
@@ -73,7 +77,7 @@ static gotoblas_t *force_coretype(char * coretype) {
 #if (!defined __GNUC__) || ( __GNUC__ >= 6)
 	case  3: return (&gotoblas_POWER9);
 #endif
-#if (!defined __GNUC__) || ( __GNUC__ >= 11)
+#ifdef HAVE_P10_SUPPORT
 	case  4: return (&gotoblas_POWER10);
 #endif
 	default: return NULL;