diff --git a/kernel/x86_64/sgemm_kernel_16x4_haswell.S b/kernel/x86_64/sgemm_kernel_16x4_haswell.S
index 1c8a37710..5a6a9e690 100644
--- a/kernel/x86_64/sgemm_kernel_16x4_haswell.S
+++ b/kernel/x86_64/sgemm_kernel_16x4_haswell.S
@@ -131,11 +131,11 @@
 #else
 
 .macro VFMADD231PS_ y0,y1,y2
-	vfmadd231pd \y0,\y1,\y2
+	vfmadd231ps \y0,\y1,\y2
 .endm
 
 .macro VFMADD231SS_ x0,x1,x2
-	vfmadd231sd \x0,\x1,\x2
+	vfmadd231ss \x0,\x1,\x2
 .endm
 
 #endif
@@ -859,6 +859,7 @@
 	jz	.L4_01b
 	ALIGN_4
 
+
 .L4_01a:
         prefetcht0 512(BO1)
         prefetchw  512(BO)
@@ -957,7 +958,7 @@
 	andq	$-8, %rax			//  K = K - ( K % 8 )
 	je	.L4_16
 	movq    %rax, BI                        //  Index for BO
-        salq    $4, BI                   	//  BI = BI * 4 ; number of values
+        leaq    (,BI,4) , BI                   	//  BI = BI * 4 ; number of values
 
 	salq	$4, %rax			// rax = rax * 16 ; number of values
 	leaq	(AO, %rax, SIZE), AO
@@ -1026,7 +1027,7 @@
 	je .L4_19
 
 	movq    %rax, BI                        //  Index for BO
-        salq    $2, BI                   	//  BI = BI * 4 ; number of values
+        leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values
 
 	salq	$4, %rax			// rax = rax * 16 ; number of values
 	leaq	(AO, %rax, SIZE), AO
@@ -1124,7 +1125,7 @@
 	andq	$-8, %rax
 	je	.L4_20_6
 	movq    %rax, BI                        //  Index for BO
-        salq    $2, BI                   	//  BI = BI * 4 ; number of values
+        leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values
 
 	salq	$3, %rax			// rax = rax * 8 ; number of values
 	leaq	(AO, %rax, SIZE), AO
@@ -1173,7 +1174,7 @@
 	je .L4_20_9
 
 	movq    %rax, BI                        //  Index for BO
-        salq    $2, BI    	                //  BI = BI * 4 ; number of values
+        leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values
 
 	salq	$3, %rax			// rax = rax * 8 ; number of values
 	leaq	(AO, %rax, SIZE), AO
@@ -1264,7 +1265,7 @@
 	andq	$-8, %rax
 	je	.L4_26
 	movq    %rax, BI                        //  Index for BO
-        salq    $2, BI                   	//  BI = BI * 4 ; number of values
+        leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values
 
 	salq	$2, %rax			// rax = rax * 4 ; number of values
 	leaq	(AO, %rax, SIZE), AO
@@ -1313,7 +1314,7 @@
 	je .L4_29
 
 	movq    %rax, BI                        //  Index for BO
-        salq    $2, BI     	                //  BI = BI * 4 ; number of values
+        leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values
 
 	salq	$2, %rax			// rax = rax * 4 ; number of values
 	leaq	(AO, %rax, SIZE), AO
@@ -1401,7 +1402,7 @@
 	andq	$-8, %rax
 	je	.L4_36
 	movq    %rax, BI                        //  Index for BO
-        salq    $2, BI                          //  BI = BI * 4 ; number of values
+        leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values
 
 	salq	$1, %rax			// rax = rax *2 ; number of values
 	leaq	(AO, %rax, SIZE), AO
@@ -1535,7 +1536,7 @@
 	andq	$-8, %rax
 	je	.L4_46
 	movq    %rax, BI                        //  Index for BO
-        salq    $2, BI                          //  BI = BI * 4 ; number of values
+        leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values
 
 	leaq	(AO, %rax, SIZE), AO
 	leaq	(BO, BI, SIZE), BO
@@ -1583,7 +1584,7 @@
 	je .L4_49
 
 	movq    %rax, BI                        //  Index for BO
-        salq    $2, BI                          //  BI = BI * 4 ; number of values
+        leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values
 
 	leaq	(AO, %rax, SIZE), AO
 	leaq	(BO, BI, SIZE), BO
diff --git a/kernel/x86_64/zgemm_kernel_4x2_haswell.S b/kernel/x86_64/zgemm_kernel_4x2_haswell.S
new file mode 100644
index 000000000..814d45179
--- /dev/null
+++ b/kernel/x86_64/zgemm_kernel_4x2_haswell.S
@@ -0,0 +1,2021 @@
+/*********************************************************************/
+/* Copyright 2009, 2010 The University of Texas at Austin.           */
+/* All rights reserved.                                              */
+/*                                                                   */
+/* Redistribution and use in source and binary forms, with or        */
+/* without modification, are permitted provided that the following   */
+/* conditions are met:                                               */
+/*                                                                   */
+/*   1. Redistributions of source code must retain the above         */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer.                                                  */
+/*                                                                   */
+/*   2. Redistributions in binary form must reproduce the above      */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer in the documentation and/or other materials       */
+/*      provided with the distribution.                              */
+/*                                                                   */
+/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
+/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
+/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
+/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
+/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
+/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
+/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
+/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
+/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */
+/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
+/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
+/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
+/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
+/*    POSSIBILITY OF SUCH DAMAGE.                                    */
+/*                                                                   */
+/* The views and conclusions contained in the software and           */
+/* documentation are those of the authors and should not be          */
+/* interpreted as representing official policies, either expressed   */
+/* or implied, of The University of Texas at Austin.                 */
+/*********************************************************************/
+
+
+#define ASSEMBLER
+#include "common.h"
+ 
+#define OLD_M	%rdi
+#define OLD_N	%rsi
+#define M	%r13
+#define J	%r14
+#define OLD_K	%rdx
+
+#define A	%rcx
+#define B	%r8
+#define C	%r9
+#define LDC	%r10
+	
+#define I	%r11
+#define AO	%rdi
+#define BO	%rsi
+#define	CO1	%r15
+#define K	%r12
+#define BI	%rbp
+#define	SP	%rbx
+
+#define BO1	%rdi
+#define BO2	%r15
+
+#ifndef WINDOWS_ABI
+
+#define STACKSIZE 96
+
+#else
+
+#define STACKSIZE 320
+
+#define OLD_ALPHA_I     40 + STACKSIZE(%rsp)
+#define OLD_A           48 + STACKSIZE(%rsp)
+#define OLD_B           56 + STACKSIZE(%rsp)
+#define OLD_C           64 + STACKSIZE(%rsp)
+#define OLD_LDC         72 + STACKSIZE(%rsp)
+#define OLD_OFFSET      80 + STACKSIZE(%rsp)
+
+#endif
+
+#define L_BUFFER_SIZE 512*8*4
+#define LB2_OFFSET    512*8*2
+
+#define Ndiv6	 24(%rsp)
+#define Nmod6	 32(%rsp)
+#define N	 40(%rsp)
+#define ALPHA_R  48(%rsp)
+#define ALPHA_I  56(%rsp)
+#define OFFSET   64(%rsp)
+#define KK       72(%rsp)
+#define KKK      80(%rsp)
+#define BUFFER1	           128(%rsp)
+#define BUFFER2	LB2_OFFSET+128(%rsp)
+
+#if defined(OS_WINDOWS)
+#if   L_BUFFER_SIZE > 16384
+#define STACK_TOUCH \
+        movl    $0,  4096 * 4(%rsp);\
+        movl    $0,  4096 * 3(%rsp);\
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 12288
+#define STACK_TOUCH \
+        movl    $0,  4096 * 3(%rsp);\
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 8192
+#define STACK_TOUCH \
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 4096
+#define STACK_TOUCH \
+        movl    $0,  4096 * 1(%rsp);
+#else
+#define STACK_TOUCH
+#endif
+#else
+#define STACK_TOUCH
+#endif
+
+#if defined(BULLDOZER)
+
+.macro VFMADD231PD_ y0,y1,y2
+        vfmaddpd \y0,\y1,\y2,\y0
+.endm
+
+.macro VFMADD231SD_ x0,x1,x2
+        vfmaddsd \x0,\x1,\x2,\x0
+.endm
+
+#else
+
+.macro VFMADD231PD_ y0,y1,y2
+        vfmadd231pd \y0,\y1,\y2
+.endm
+
+.macro VFMADD231SD_ x0,x1,x2
+        vfmadd231sd \x0,\x1,\x2
+.endm
+
+#endif
+
+#if defined(BULLDOZER)
+
+#if   defined(NN) || defined(NT) || defined(TN) || defined(TT)
+
+.macro	VFMADDPD_R y0,y1,y2
+        vfmaddpd \y0,\y1,\y2,\y0
+.endm
+
+.macro	VFMADDPD_I y0,y1,y2
+        vfmaddpd \y0,\y1,\y2,\y0
+.endm
+
+#define VFMADD_R    vfmaddpd
+#define VFMADD_I    vfmaddpd
+
+#elif defined(RN) || defined(RT) || defined(CN) || defined(CT)
+
+.macro	VFMADDPD_R y0,y1,y2
+        vfnmaddpd \y0,\y1,\y2,\y0
+.endm
+
+.macro	VFMADDPD_I y0,y1,y2
+        vfmaddpd \y0,\y1,\y2,\y0
+.endm
+
+#define VFMADD_R    vfnmaddpd
+#define VFMADD_I    vfmaddpd
+
+#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+.macro	VFMADDPD_R y0,y1,y2
+        vfmaddpd \y0,\y1,\y2,\y0
+.endm
+
+.macro	VFMADDPD_I y0,y1,y2
+        vfnmaddpd \y0,\y1,\y2,\y0
+.endm
+
+#define VFMADD_R    vfmaddpd
+#define VFMADD_I    vfnmaddpd
+
+#else
+
+.macro	VFMADDPD_R y0,y1,y2
+        vfnmaddpd \y0,\y1,\y2,\y0
+.endm
+
+.macro	VFMADDPD_I y0,y1,y2
+        vfnmaddpd \y0,\y1,\y2,\y0
+.endm
+
+#define VFMADD_R    vfnmaddpd
+#define VFMADD_I    vfnmaddpd
+
+#endif
+
+#else
+
+#if   defined(NN) || defined(NT) || defined(TN) || defined(TT)
+
+.macro	VFMADDPD_R y0,y1,y2
+        vfmadd231pd \y0,\y1,\y2
+.endm
+
+.macro	VFMADDPD_I y0,y1,y2
+        vfmadd231pd \y0,\y1,\y2
+.endm
+
+#define VFMADD_R    vfmadd231pd
+#define VFMADD_I    vfmadd231pd
+
+#elif defined(RN) || defined(RT) || defined(CN) || defined(CT)
+
+.macro	VFMADDPD_R y0,y1,y2
+        vfnmadd231pd \y0,\y1,\y2
+.endm
+
+.macro	VFMADDPD_I y0,y1,y2
+        vfmadd231pd \y0,\y1,\y2
+.endm
+
+#define VFMADD_R    vfnmadd231pd
+#define VFMADD_I    vfmadd231pd
+
+#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+.macro	VFMADDPD_R y0,y1,y2
+        vfmadd231pd \y0,\y1,\y2
+.endm
+
+.macro	VFMADDPD_I y0,y1,y2
+        vfnmadd231pd \y0,\y1,\y2
+.endm
+
+#define VFMADD_R    vfmadd231pd
+#define VFMADD_I    vfnmadd231pd
+
+#else
+
+.macro	VFMADDPD_R y0,y1,y2
+        vfnmadd231pd \y0,\y1,\y2
+.endm
+
+.macro	VFMADDPD_I y0,y1,y2
+        vfnmadd231pd \y0,\y1,\y2
+.endm
+
+#define VFMADD_R    vfnmadd231pd
+#define VFMADD_I    vfnmadd231pd
+
+#endif
+
+#endif
+
+#define	A_PR1	384
+#define	B_PR1	192
+/***************************************************************************************************/
+
+.macro KERNEL4x2_SUB
+        vmovups           -8 * SIZE(AO, %rax, SIZE), %ymm0
+        vmovups           -4 * SIZE(AO, %rax, SIZE), %ymm1
+
+        vbroadcastsd      -8 * SIZE(BO, BI, SIZE),   %ymm4
+        vbroadcastsd      -7 * SIZE(BO, BI, SIZE),   %ymm5
+        VFMADDPD_R        %ymm8 ,%ymm4,%ymm0
+        VFMADDPD_R        %ymm12,%ymm4,%ymm1
+        vbroadcastsd      -6 * SIZE(BO, BI, SIZE),   %ymm6
+        VFMADDPD_I        %ymm9 ,%ymm5,%ymm0
+        VFMADDPD_I        %ymm13,%ymm5,%ymm1
+        vbroadcastsd      -5 * SIZE(BO, BI, SIZE),   %ymm7
+        VFMADDPD_R        %ymm10,%ymm6,%ymm0
+        VFMADDPD_R        %ymm14,%ymm6,%ymm1
+        VFMADDPD_I        %ymm11,%ymm7,%ymm0
+        VFMADDPD_I        %ymm15,%ymm7,%ymm1
+
+        addq    $4, BI                           
+        addq    $8, %rax                         
+.endm
+
+.macro SAVE4x2
+
+	vbroadcastsd	ALPHA_R, %ymm0
+	vbroadcastsd	ALPHA_I, %ymm1
+
+	// swap high and low 8 bytes
+        vshufpd $0x05, %ymm9 , %ymm9, %ymm9
+        vshufpd $0x05, %ymm11, %ymm11, %ymm11
+        vshufpd $0x05, %ymm13, %ymm13, %ymm13
+        vshufpd $0x05, %ymm15, %ymm15, %ymm15
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubpd %ymm9, %ymm8 , %ymm8
+        vaddsubpd %ymm11,%ymm10, %ymm10
+        vaddsubpd %ymm13,%ymm12, %ymm12
+        vaddsubpd %ymm15,%ymm14, %ymm14
+
+        vshufpd $0x05, %ymm8 , %ymm8, %ymm9
+        vshufpd $0x05, %ymm10, %ymm10, %ymm11
+        vshufpd $0x05, %ymm12, %ymm12, %ymm13
+        vshufpd $0x05, %ymm14, %ymm14, %ymm15
+
+#else
+        vaddsubpd %ymm8,  %ymm9 ,%ymm9
+        vaddsubpd %ymm10, %ymm11,%ymm11
+        vaddsubpd %ymm12, %ymm13,%ymm13
+        vaddsubpd %ymm14, %ymm15,%ymm15
+
+        vmovapd   %ymm9,  %ymm8
+        vmovapd   %ymm11, %ymm10
+        vmovapd   %ymm13, %ymm12
+        vmovapd   %ymm15, %ymm14
+
+	// swap high and low 8 bytes
+        vshufpd $0x05, %ymm9 , %ymm9, %ymm9
+        vshufpd $0x05, %ymm11, %ymm11, %ymm11
+        vshufpd $0x05, %ymm13, %ymm13, %ymm13
+        vshufpd $0x05, %ymm15, %ymm15, %ymm15
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulpd  %ymm8 , %ymm0, %ymm8
+        vmulpd  %ymm10, %ymm0, %ymm10
+        vmulpd  %ymm12, %ymm0, %ymm12
+        vmulpd  %ymm14, %ymm0, %ymm14
+
+	// multiply with ALPHA_I
+        vmulpd  %ymm9 , %ymm1, %ymm9
+        vmulpd  %ymm11, %ymm1, %ymm11
+        vmulpd  %ymm13, %ymm1, %ymm13
+        vmulpd  %ymm15, %ymm1, %ymm15
+
+	vaddsubpd %ymm9, %ymm8 , %ymm8
+        vaddsubpd %ymm11,%ymm10, %ymm10
+        vaddsubpd %ymm13,%ymm12, %ymm12
+        vaddsubpd %ymm15,%ymm14, %ymm14
+
+
+
+#ifndef TRMMKERNEL
+
+	vaddpd 	 	(CO1), %ymm8 , %ymm8
+	vaddpd  4 * SIZE(CO1), %ymm12, %ymm12
+
+	vaddpd 	 	(CO1, LDC), %ymm10, %ymm10
+	vaddpd  4 * SIZE(CO1, LDC), %ymm14, %ymm14
+
+#endif
+
+	vmovups	%ymm8 ,  	(CO1)
+	vmovups	%ymm12 , 4 * SIZE(CO1)
+
+	vmovups	%ymm10 ,  	(CO1, LDC)
+	vmovups	%ymm14 , 4 * SIZE(CO1, LDC)
+
+
+
+.endm
+
+/***************************************************************************************************/
+#define KERNEL2x2_1(xx) \
+        prefetcht0      A_PR1(AO,%rax,SIZE)        ;\
+        vmovups          -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vmovddup         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups          -6 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vmovddup         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+        vmovddup         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        VFMADD_R        %xmm14,%xmm6,%xmm1,%xmm14 ;\
+        vmovddup         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+        VFMADD_I        %xmm15,%xmm7,%xmm1,%xmm15 ;\
+
+#define KERNEL2x2_2(xx) \
+        vmovups          -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vmovddup         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups          -2 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vmovddup         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+        vmovddup         -2 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        VFMADD_R        %xmm14,%xmm6,%xmm1,%xmm14 ;\
+        vmovddup         -1 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+        VFMADD_I        %xmm15,%xmm7,%xmm1,%xmm15 ;\
+
+#define KERNEL2x2_3(xx) \
+	prefetcht0      A_PR1+64(AO,%rax,SIZE)     ;\
+        vmovups           0 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vmovddup          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups           2 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vmovddup          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+        vmovddup          2 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        VFMADD_R        %xmm14,%xmm6,%xmm1,%xmm14 ;\
+        vmovddup          3 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+        VFMADD_I        %xmm15,%xmm7,%xmm1,%xmm15 ;\
+
+#define KERNEL2x2_4(xx) \
+        vmovups           4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vmovddup          4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups           6 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vmovddup          5 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+        vmovddup          6 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        VFMADD_R        %xmm14,%xmm6,%xmm1,%xmm14 ;\
+        vmovddup          7 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+        VFMADD_I        %xmm15,%xmm7,%xmm1,%xmm15 ;\
+        addq    $16, BI                            ;\
+        addq    $16, %rax                          ;\
+
+
+#define KERNEL2x2_SUB(xx) \
+        vmovups          -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vmovddup         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups          -6 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vmovddup         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+        vmovddup         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        VFMADD_R        %xmm14,%xmm6,%xmm1,%xmm14 ;\
+        vmovddup         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+        VFMADD_I        %xmm15,%xmm7,%xmm1,%xmm15 ;\
+        addq    $4, BI                            ;\
+        addq    $4, %rax                          ;\
+
+/************************************************************************************************/
+
+/************************************************************************************************/
+
+#define KERNEL1x2_1(xx) \
+        prefetcht0      A_PR1(AO,%rax,SIZE)        ;\
+        vmovups          -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vmovddup         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovddup         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        vmovddup         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        vmovddup         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+
+#define KERNEL1x2_2(xx) \
+        vmovups          -6 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vmovddup         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vmovddup         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        vmovddup         -2 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        vmovddup         -1 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+
+#define KERNEL1x2_3(xx) \
+        vmovups          -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vmovddup          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovddup          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        vmovddup          2 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        vmovddup          3 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+
+#define KERNEL1x2_4(xx) \
+        vmovups          -2 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vmovddup          4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovddup          5 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        vmovddup          6 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        vmovddup          7 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+        addq    $16, BI                            ;\
+        addq    $8 , %rax                          ;\
+
+
+#define KERNEL1x2_SUB(xx) \
+        vmovups          -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vmovddup         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovddup         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        vmovddup         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        vmovddup         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+        addq    $4, BI                            ;\
+        addq    $2, %rax                          ;\
+
+/************************************************************************************************/
+
+.macro KERNEL4x1_SUB
+        vmovups          -8 * SIZE(AO, %rax, SIZE), %ymm0
+        vmovups          -4 * SIZE(AO, %rax, SIZE), %ymm1
+        vbroadcastsd     -4 * SIZE(BO, BI, SIZE)  , %ymm4
+        vbroadcastsd     -3 * SIZE(BO, BI, SIZE)  , %ymm5
+        VFMADDPD_R        %ymm8 ,%ymm4,%ymm0
+        VFMADDPD_R        %ymm12,%ymm4,%ymm1
+        VFMADDPD_I        %ymm9 ,%ymm5,%ymm0
+        VFMADDPD_I        %ymm13,%ymm5,%ymm1
+
+        addq    $2, BI                           
+        addq    $8, %rax                         
+.endm
+
+.macro SAVE4x1
+
+	vbroadcastsd	ALPHA_R, %ymm0
+	vbroadcastsd	ALPHA_I, %ymm1
+
+	// swap high and low 8 bytes
+        vshufpd $0x05, %ymm9 , %ymm9, %ymm9
+        vshufpd $0x05, %ymm13, %ymm13, %ymm13
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubpd %ymm9, %ymm8  , %ymm8
+        vaddsubpd %ymm13,%ymm12 , %ymm12
+
+        vshufpd $0x05, %ymm8 , %ymm8, %ymm9
+        vshufpd $0x05, %ymm12, %ymm12, %ymm13
+
+#else
+        vaddsubpd %ymm8, %ymm9 , %ymm9
+        vaddsubpd %ymm12,%ymm13, %ymm13
+
+        vmovapd   %ymm9,  %ymm8
+        vmovapd   %ymm13, %ymm12
+
+	// swap high and low 8 bytes
+        vshufpd $0x05, %ymm9 , %ymm9, %ymm9
+        vshufpd $0x05, %ymm13, %ymm13, %ymm13
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulpd  %ymm8 , %ymm0, %ymm8
+        vmulpd  %ymm12, %ymm0, %ymm12
+
+	// multiply with ALPHA_I
+        vmulpd  %ymm9 , %ymm1, %ymm9
+        vmulpd  %ymm13, %ymm1, %ymm13
+
+	vaddsubpd %ymm9,  %ymm8 , %ymm8
+        vaddsubpd %ymm13, %ymm12, %ymm12
+
+
+
+#ifndef TRMMKERNEL
+
+	vaddpd 	 	(CO1), %ymm8 , %ymm8
+	vaddpd  4 * SIZE(CO1), %ymm12, %ymm12
+
+#endif
+
+	vmovups	%ymm8 ,  	(CO1)
+	vmovups	%ymm12 ,4 * SIZE(CO1)
+
+.endm
+
+
+
+/************************************************************************************************/
+
+#define KERNEL2x1_1(xx) \
+        prefetcht0      A_PR1(AO,%rax,SIZE)        ;\
+        vmovups          -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vmovddup         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups          -6 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vmovddup         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+
+#define KERNEL2x1_2(xx) \
+        vmovups          -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vmovddup         -2 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups          -2 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vmovddup         -1 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+
+#define KERNEL2x1_3(xx) \
+	prefetcht0      A_PR1+64(AO,%rax,SIZE)     ;\
+        vmovups           0 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vmovddup          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups           2 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vmovddup          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+
+#define KERNEL2x1_4(xx) \
+        vmovups           4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vmovddup          2 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups           6 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vmovddup          3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+        addq    $8, BI                            ;\
+        addq    $16, %rax                          ;\
+
+
+#define KERNEL2x1_SUB(xx) \
+        vmovups          -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vmovddup         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups          -6 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vmovddup         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+        addq    $2, BI                            ;\
+        addq    $4, %rax                          ;\
+
+
+/************************************************************************************************/
+
+#define KERNEL1x1_1(xx) \
+        prefetcht0      A_PR1(AO,%rax,SIZE)        ;\
+        vmovups          -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vmovddup         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovddup         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+
+#define KERNEL1x1_2(xx) \
+        vmovups          -6 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vmovddup         -2 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovddup         -1 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+
+#define KERNEL1x1_3(xx) \
+        vmovups          -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vmovddup          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovddup          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+
+#define KERNEL1x1_4(xx) \
+        vmovups          -2 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vmovddup          2 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovddup          3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        addq    $8, BI                            ;\
+        addq    $8, %rax                          ;\
+
+
+#define KERNEL1x1_SUB(xx) \
+        vmovups          -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vmovddup         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovddup         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        addq    $2, BI                            ;\
+        addq    $2, %rax                          ;\
+
+
+/************************************************************************************************/
+
+
+
+
+	PROLOGUE
+	PROFCODE
+	
+	subq	$STACKSIZE, %rsp
+	movq	%rbx,   (%rsp)
+	movq	%rbp,  8(%rsp)
+	movq	%r12, 16(%rsp)
+	movq	%r13, 24(%rsp)
+	movq	%r14, 32(%rsp)
+	movq	%r15, 40(%rsp)
+
+	vzeroupper
+
+#ifdef WINDOWS_ABI
+	movq	%rdi,    48(%rsp)
+	movq	%rsi,    56(%rsp)
+	movups	%xmm6,   64(%rsp)
+	movups	%xmm7,   80(%rsp)
+	movups	%xmm8,   96(%rsp)
+	movups	%xmm9,  112(%rsp)
+	movups	%xmm10, 128(%rsp)
+	movups	%xmm11, 144(%rsp)
+	movups	%xmm12, 160(%rsp)
+	movups	%xmm13, 176(%rsp)
+	movups	%xmm14, 192(%rsp)
+	movups	%xmm15, 208(%rsp)
+
+	movq	ARG1,      OLD_M
+	movq	ARG2,      OLD_N
+	movq	ARG3,      OLD_K
+	movq	OLD_A,     A
+	movq	OLD_B,     B
+	movq	OLD_C,     C
+	movq	OLD_LDC,   LDC
+#ifdef TRMMKERNEL
+	movsd	OLD_OFFSET, %xmm12
+#endif
+	vmovaps	%xmm3, %xmm0
+
+#else
+	movq	STACKSIZE +  8(%rsp), LDC
+#ifdef TRMMKERNEL
+	movsd	STACKSIZE + 16(%rsp), %xmm12
+#endif
+
+#endif
+
+	movq    %rsp, SP      # save old stack
+        subq    $128 + L_BUFFER_SIZE, %rsp
+        andq    $-4096, %rsp    # align stack
+
+        STACK_TOUCH
+
+	cmpq	$0, OLD_M
+	je	.L999
+
+	cmpq	$0, OLD_N
+	je	.L999
+
+	cmpq	$0, OLD_K
+	je	.L999
+
+	movq	OLD_M, M
+	movq	OLD_N, N
+	movq	OLD_K, K
+
+	vmovsd	 %xmm0, ALPHA_R
+	vmovsd	 %xmm1, ALPHA_I
+
+	salq	$ZBASE_SHIFT, LDC
+
+	movq    N, %rax
+        xorq    %rdx, %rdx
+        movq    $2,  %rdi
+        divq    %rdi                    //    N / 2
+        movq    %rax, Ndiv6             //    N / 2
+        movq    %rdx, Nmod6             //    N % 2
+
+	
+
+#ifdef TRMMKERNEL
+	vmovsd	%xmm12, OFFSET
+	vmovsd	%xmm12, KK
+#ifndef LEFT
+	negq	KK
+#endif	
+#endif
+
+.L2_00_0:
+
+	movq	Ndiv6,  J
+	cmpq	$0, J
+	je	.L1_2_0
+	ALIGN_4
+
+
+
+.L2_00_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L2_00_02b:
+
+	vmovups		(BO1), %xmm0
+	vmovups	2 * SIZE(BO1), %xmm1
+	vmovups	%xmm0,       (BO)
+	vmovups	%xmm1, 2 * SIZE(BO)
+	addq	$4*SIZE,BO1
+	addq	$4*SIZE,BO
+	decq	%rax
+	jnz	.L2_00_02b
+
+.L2_00_02c:
+
+	movq	BO1, B			// next offset of B
+
+
+.L2_00_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		// c += 2 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$8 * SIZE, AO
+
+	movq	M,  I
+	sarq	$2, I			// i = (m >> 2)
+	je	.L2_2_10
+
+	ALIGN_4
+
+/******************************************************************************************************************/
+
+.L2_4_11:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$8 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$8 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L2_4_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_4_12:
+
+        prefetcht0      A_PR1(AO,%rax,SIZE)
+        prefetcht0      B_PR1(BO,BI  ,SIZE)
+	KERNEL4x2_SUB
+        prefetcht0      A_PR1(AO,%rax,SIZE)
+	KERNEL4x2_SUB
+        prefetcht0      A_PR1(AO,%rax,SIZE)
+        prefetcht0      B_PR1(BO,BI  ,SIZE)
+	KERNEL4x2_SUB
+        prefetcht0      A_PR1(AO,%rax,SIZE)
+	KERNEL4x2_SUB
+
+        prefetcht0      A_PR1(AO,%rax,SIZE)
+        prefetcht0      B_PR1(BO,BI  ,SIZE)
+	KERNEL4x2_SUB
+        prefetcht0      A_PR1(AO,%rax,SIZE)
+	KERNEL4x2_SUB
+        prefetcht0      A_PR1(AO,%rax,SIZE)
+        prefetcht0      B_PR1(BO,BI  ,SIZE)
+	KERNEL4x2_SUB
+        prefetcht0      A_PR1(AO,%rax,SIZE)
+	KERNEL4x2_SUB
+
+	je	.L2_4_16
+
+        prefetcht0      A_PR1(AO,%rax,SIZE)
+        prefetcht0      B_PR1(BO,BI  ,SIZE)
+	KERNEL4x2_SUB
+        prefetcht0      A_PR1(AO,%rax,SIZE)
+	KERNEL4x2_SUB
+        prefetcht0      A_PR1(AO,%rax,SIZE)
+        prefetcht0      B_PR1(BO,BI  ,SIZE)
+	KERNEL4x2_SUB
+        prefetcht0      A_PR1(AO,%rax,SIZE)
+	KERNEL4x2_SUB
+
+        prefetcht0      A_PR1(AO,%rax,SIZE)
+        prefetcht0      B_PR1(BO,BI  ,SIZE)
+	KERNEL4x2_SUB
+        prefetcht0      A_PR1(AO,%rax,SIZE)
+	KERNEL4x2_SUB
+        prefetcht0      A_PR1(AO,%rax,SIZE)
+        prefetcht0      B_PR1(BO,BI  ,SIZE)
+	KERNEL4x2_SUB
+        prefetcht0      A_PR1(AO,%rax,SIZE)
+	KERNEL4x2_SUB
+
+	je	.L2_4_16
+
+	jmp	.L2_4_12
+	ALIGN_4
+
+.L2_4_16:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_4_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_4_17:
+
+	KERNEL4x2_SUB
+
+	jl	.L2_4_17
+	ALIGN_4
+
+
+.L2_4_19:
+
+	SAVE4x2
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK
+#endif
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	decq	I			# i --
+	jg	.L2_4_11
+	ALIGN_4	
+
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+
+
+/******************************************************************************************************************/
+.L2_2_10:
+	testq	$2, M		
+	jz	.L2_2_40		// to next 2 lines of N
+
+.L2_2_11:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$8 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$8 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L2_2_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_2_12:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	prefetcht0	B_PR1+64(BO,BI,SIZE)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	prefetcht0	B_PR1+64(BO,BI,SIZE)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	je	.L2_2_16
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	prefetcht0	B_PR1+64(BO,BI,SIZE)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	prefetcht0	B_PR1+64(BO,BI,SIZE)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	je	.L2_2_16
+
+	jmp	.L2_2_12
+	ALIGN_4
+
+.L2_2_16:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_2_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_2_17:
+
+	KERNEL2x2_SUB(xxx)
+	jl	.L2_2_17
+	ALIGN_4
+
+
+.L2_2_19:
+
+	vmovddup	ALPHA_R, %xmm0
+	vmovddup	ALPHA_I, %xmm1
+
+	// swap high and low 64 bytes
+        vshufpd $0x01, %xmm9 , %xmm9, %xmm9
+        vshufpd $0x01, %xmm11, %xmm11, %xmm11
+        vshufpd $0x01, %xmm13, %xmm13, %xmm13
+        vshufpd $0x01, %xmm15, %xmm15, %xmm15
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubpd %xmm9, %xmm8 , %xmm8
+        vaddsubpd %xmm11,%xmm10, %xmm10
+        vaddsubpd %xmm13,%xmm12, %xmm12
+        vaddsubpd %xmm15,%xmm14, %xmm14
+
+        vshufpd $0x01, %xmm8 , %xmm8, %xmm9
+        vshufpd $0x01, %xmm10, %xmm10, %xmm11
+        vshufpd $0x01, %xmm12, %xmm12, %xmm13
+        vshufpd $0x01, %xmm14, %xmm14, %xmm15
+
+#else
+        vaddsubpd %xmm8,  %xmm9 ,%xmm9
+        vaddsubpd %xmm10, %xmm11,%xmm11
+        vaddsubpd %xmm12, %xmm13,%xmm13
+        vaddsubpd %xmm14, %xmm15,%xmm15
+
+        vmovapd   %xmm9,  %xmm8
+        vmovapd   %xmm11, %xmm10
+        vmovapd   %xmm13, %xmm12
+        vmovapd   %xmm15, %xmm14
+
+	// swap high and low 64 bytes
+        vshufpd $0x01, %xmm9 , %xmm9, %xmm9
+        vshufpd $0x01, %xmm11, %xmm11, %xmm11
+        vshufpd $0x01, %xmm13, %xmm13, %xmm13
+        vshufpd $0x01, %xmm15, %xmm15, %xmm15
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulpd  %xmm8 , %xmm0, %xmm8
+        vmulpd  %xmm10, %xmm0, %xmm10
+        vmulpd  %xmm12, %xmm0, %xmm12
+        vmulpd  %xmm14, %xmm0, %xmm14
+
+	// multiply with ALPHA_I
+        vmulpd  %xmm9 , %xmm1, %xmm9
+        vmulpd  %xmm11, %xmm1, %xmm11
+        vmulpd  %xmm13, %xmm1, %xmm13
+        vmulpd  %xmm15, %xmm1, %xmm15
+
+	vaddsubpd %xmm9, %xmm8 , %xmm8
+        vaddsubpd %xmm11,%xmm10, %xmm10
+        vaddsubpd %xmm13,%xmm12, %xmm12
+        vaddsubpd %xmm15,%xmm14, %xmm14
+
+
+
+#ifndef TRMMKERNEL
+
+	vaddpd 	 	(CO1), %xmm8 , %xmm8
+	vaddpd  2 * SIZE(CO1), %xmm12, %xmm12
+
+	vaddpd 	 	(CO1, LDC), %xmm10, %xmm10
+	vaddpd  2 * SIZE(CO1, LDC), %xmm14, %xmm14
+
+#endif
+
+	vmovups	%xmm8 ,  	(CO1)
+	vmovups	%xmm12 , 2 * SIZE(CO1)
+
+	vmovups	%xmm10 ,  	(CO1, LDC)
+	vmovups	%xmm14 , 2 * SIZE(CO1, LDC)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK
+#endif
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4	
+
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L2_2_40:
+	testq	$1, M		
+	jz	.L2_2_60		// to next 2 lines of N
+
+	ALIGN_4
+
+.L2_2_41:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$8 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$8 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$1, %rax			// rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L2_2_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$1, %rax			// rax = rax * 2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_2_42:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	prefetcht0	B_PR1+64(BO,BI,SIZE)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	prefetcht0	B_PR1+64(BO,BI,SIZE)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	je	.L2_2_46
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	prefetcht0	B_PR1+64(BO,BI,SIZE)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	prefetcht0	B_PR1+64(BO,BI,SIZE)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	je	.L2_2_46
+
+	jmp	.L2_2_42
+	ALIGN_4
+
+.L2_2_46:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_2_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$1, %rax			// rax = rax * 2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_2_47:
+
+	KERNEL1x2_SUB(xxx)
+	jl	.L2_2_47
+	ALIGN_4
+
+
+.L2_2_49:
+
+	vmovddup	ALPHA_R, %xmm0
+	vmovddup	ALPHA_I, %xmm1
+
+	// swap high and low 64 bytes
+        vshufpd $0x01, %xmm9 , %xmm9, %xmm9
+        vshufpd $0x01, %xmm11, %xmm11, %xmm11
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubpd %xmm9, %xmm8 , %xmm8
+        vaddsubpd %xmm11,%xmm10, %xmm10
+
+        vshufpd $0x01, %xmm8 , %xmm8, %xmm9
+        vshufpd $0x01, %xmm10, %xmm10, %xmm11
+
+#else
+        vaddsubpd %xmm8, %xmm9, %xmm9
+        vaddsubpd %xmm10,%xmm11, %xmm11
+
+        vmovapd   %xmm9,  %xmm8
+        vmovapd   %xmm11, %xmm10
+
+	// swap high and low 64 bytes
+        vshufpd $0x01, %xmm9 , %xmm9, %xmm9
+        vshufpd $0x01, %xmm11, %xmm11, %xmm11
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulpd  %xmm8 , %xmm0, %xmm8
+        vmulpd  %xmm10, %xmm0, %xmm10
+
+	// multiply with ALPHA_I
+        vmulpd  %xmm9 , %xmm1, %xmm9
+        vmulpd  %xmm11, %xmm1, %xmm11
+
+	vaddsubpd %xmm9, %xmm8 , %xmm8
+        vaddsubpd %xmm11,%xmm10, %xmm10
+
+
+
+#ifndef TRMMKERNEL
+
+	vaddpd 	 	(CO1), %xmm8 , %xmm8
+	vaddpd 	 	(CO1, LDC), %xmm10, %xmm10
+
+#endif
+
+	vmovups	%xmm8 ,  	(CO1)
+	vmovups	%xmm10 ,  	(CO1, LDC)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$1, %rax			// rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK
+#endif
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	decq	I			# i --
+	jg	.L2_2_41
+	ALIGN_4	
+
+
+
+	
+.L2_2_60:
+#if defined(TRMMKERNEL) && !defined(LEFT)
+        addq    $2, KK
+#endif
+
+	decq	J			// j --
+	jg	.L2_00_01			// next 2 lines of N
+
+
+
+.L1_2_0:
+
+/************************************************************************************************
+* Loop for Nmod6 % 2 > 0
+*************************************************************************************************/
+
+	movq	Nmod6, J		
+	andq	$1, J			// j % 2
+	je	.L999
+	ALIGN_4
+
+.L1_00_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L1_00_02b:
+
+	vmovups		(BO1), %xmm0
+	vmovups	%xmm0,       (BO)
+	addq	$2*SIZE,BO1
+	addq	$2*SIZE,BO
+	decq	%rax
+	jnz	.L1_00_02b
+
+.L1_00_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L1_00_10:
+	movq	C, CO1
+	leaq	(C, LDC, 1), C		// c += 1 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$8 * SIZE, AO
+
+	movq	M,  I
+	sarq	$2, I			// i = (m >> 2)
+	je	.L1_2_10
+
+	ALIGN_4
+
+/*******************************************************************************************************/
+
+
+.L1_4_11:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$4 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$4 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L1_4_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_4_12:
+
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+
+	je	.L1_4_16
+
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+
+	je	.L1_4_16
+
+	jmp	.L1_4_12
+	ALIGN_4
+
+.L1_4_16:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_4_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_4_17:
+
+	KERNEL4x1_SUB
+
+	jl	.L1_4_17
+	ALIGN_4
+
+
+.L1_4_19:
+
+	SAVE4x1
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK
+#endif
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	decq	I			# i --
+	jg	.L1_4_11
+	ALIGN_4	
+
+
+
+
+/*******************************************************************************************************/
+.L1_2_10:
+	testq	$2, M		
+	jz	.L1_2_40
+
+
+.L1_2_11:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$4 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$4 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L1_2_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_2_12:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	je	.L1_2_16
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	je	.L1_2_16
+
+	jmp	.L1_2_12
+	ALIGN_4
+
+.L1_2_16:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_2_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_2_17:
+
+	KERNEL2x1_SUB(xxx)
+	jl	.L1_2_17
+	ALIGN_4
+
+
+.L1_2_19:
+
+	vmovddup	ALPHA_R, %xmm0
+	vmovddup	ALPHA_I, %xmm1
+
+	// swap high and low 64 bytes
+        vshufpd $0x01, %xmm9 , %xmm9, %xmm9
+        vshufpd $0x01, %xmm13, %xmm13, %xmm13
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubpd %xmm9, %xmm8  , %xmm8
+        vaddsubpd %xmm13,%xmm12 , %xmm12
+
+        vshufpd $0x01, %xmm8 , %xmm8, %xmm9
+        vshufpd $0x01, %xmm12, %xmm12, %xmm13
+
+#else
+        vaddsubpd %xmm8, %xmm9 , %xmm9
+        vaddsubpd %xmm12,%xmm13, %xmm13
+
+        vmovapd   %xmm9,  %xmm8
+        vmovapd   %xmm13, %xmm12
+
+	// swap high and low 64 bytes
+        vshufpd $0x01, %xmm9 , %xmm9, %xmm9
+        vshufpd $0x01, %xmm13, %xmm13, %xmm13
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulpd  %xmm8 , %xmm0, %xmm8
+        vmulpd  %xmm12, %xmm0, %xmm12
+
+	// multiply with ALPHA_I
+        vmulpd  %xmm9 , %xmm1, %xmm9
+        vmulpd  %xmm13, %xmm1, %xmm13
+
+	vaddsubpd %xmm9,  %xmm8 , %xmm8
+        vaddsubpd %xmm13, %xmm12, %xmm12
+
+
+
+#ifndef TRMMKERNEL
+
+	vaddpd 	 	(CO1), %xmm8 , %xmm8
+	vaddpd  2 * SIZE(CO1), %xmm12, %xmm12
+
+#endif
+
+	vmovups	%xmm8 ,  	(CO1)
+	vmovups	%xmm12 , 2 * SIZE(CO1)
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK
+#endif
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+
+	ALIGN_4	
+
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L1_2_40:
+	testq	$1, M		
+	jz	.L999
+
+	ALIGN_4
+
+.L1_2_41:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$4 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$4 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$1, %rax			// rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L1_2_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+
+	salq	$1, %rax			// rax = rax * 2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_2_42:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	je	.L1_2_46
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	je	.L1_2_46
+
+	jmp	.L1_2_42
+	ALIGN_4
+
+.L1_2_46:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_2_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+
+	salq	$1, %rax			// rax = rax * 2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_2_47:
+
+	KERNEL1x1_SUB(xxx)
+	jl	.L1_2_47
+	ALIGN_4
+
+
+.L1_2_49:
+
+	vmovddup	ALPHA_R, %xmm0
+	vmovddup	ALPHA_I, %xmm1
+
+	// swap high and low 64 bytes
+        vshufpd $0x01, %xmm9 , %xmm9, %xmm9
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubpd %xmm9, %xmm8,  %xmm8
+
+        vshufpd $0x01, %xmm8 , %xmm8, %xmm9
+
+#else
+        vaddsubpd %xmm8, %xmm9,  %xmm9
+
+        vmovapd   %xmm9,  %xmm8
+
+	// swap high and low 64 bytes
+        vshufpd $0x01, %xmm9 , %xmm9, %xmm9
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulpd  %xmm8 , %xmm0, %xmm8
+
+	// multiply with ALPHA_I
+        vmulpd  %xmm9 , %xmm1, %xmm9
+
+	vaddsubpd %xmm9 ,%xmm8,  %xmm8
+
+
+
+#ifndef TRMMKERNEL
+
+	vaddpd 	 	(CO1), %xmm8 , %xmm8
+
+#endif
+
+	vmovups	%xmm8 ,  	(CO1)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$1, %rax			// rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK
+#endif
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	decq	I			# i --
+	jg	.L1_2_41
+	ALIGN_4	
+
+
+
+
+
+
+.L999:
+	movq   		SP, %rsp
+	movq	   (%rsp), %rbx
+	movq	  8(%rsp), %rbp
+	movq	 16(%rsp), %r12
+	movq	 24(%rsp), %r13
+	movq	 32(%rsp), %r14
+	movq	 40(%rsp), %r15
+
+#ifdef WINDOWS_ABI
+	movq	 48(%rsp), %rdi
+	movq	 56(%rsp), %rsi
+	movups	 64(%rsp), %xmm6
+	movups	 80(%rsp), %xmm7
+	movups	 96(%rsp), %xmm8
+	movups	112(%rsp), %xmm9
+	movups	128(%rsp), %xmm10
+	movups	144(%rsp), %xmm11
+	movups	160(%rsp), %xmm12
+	movups	176(%rsp), %xmm13
+	movups	192(%rsp), %xmm14
+	movups	208(%rsp), %xmm15
+#endif
+
+	addq	$STACKSIZE, %rsp
+	ret
+
+	EPILOGUE