diff --git a/kernel/mips64/KERNEL b/kernel/mips64/KERNEL index a14b1cb38..6afb2cf13 100644 --- a/kernel/mips64/KERNEL +++ b/kernel/mips64/KERNEL @@ -123,10 +123,21 @@ ifndef DTRSMKERNEL_RT DTRSMKERNEL_RT = trsm_kernel_RT.S endif +ifndef CTRSMKERNEL_LN CTRSMKERNEL_LN = ztrsm_kernel_LT.S +endif + +ifndef CTRSMKERNEL_LT CTRSMKERNEL_LT = ztrsm_kernel_LT.S +endif + +ifndef CTRSMKERNEL_RN CTRSMKERNEL_RN = ztrsm_kernel_LT.S +endif + +ifndef CTRSMKERNEL_RT CTRSMKERNEL_RT = ztrsm_kernel_RT.S +endif ifndef ZTRSMKERNEL_LN ZTRSMKERNEL_LN = ztrsm_kernel_LT.S diff --git a/kernel/mips64/KERNEL.LOONGSON3A b/kernel/mips64/KERNEL.LOONGSON3A index 706f48128..ebab8e6ea 100644 --- a/kernel/mips64/KERNEL.LOONGSON3A +++ b/kernel/mips64/KERNEL.LOONGSON3A @@ -1,19 +1,25 @@ SAXPYKERNEL=axpy_loongson3a.S DAXPYKERNEL=daxpy_loongson3a_simd.S -SGEMMKERNEL = sgemm_kernel_loongson3a.S +SGEMMKERNEL = sgemm_kernel_loongson3a_4x4.S SGEMMONCOPY = ../generic/gemm_ncopy_4.c SGEMMOTCOPY = ../generic/gemm_tcopy_4.c SGEMMONCOPYOBJ = sgemm_oncopy.o SGEMMOTCOPYOBJ = sgemm_otcopy.o -DGEMMKERNEL = gemm_kernel_loongson3a.S +DGEMMKERNEL = dgemm_kernel_loongson3a_4x4.S DGEMMONCOPY = ../generic/gemm_ncopy_4.c DGEMMOTCOPY = ../generic/gemm_tcopy_4.c DGEMMONCOPYOBJ = dgemm_oncopy.o DGEMMOTCOPYOBJ = dgemm_otcopy.o -ZGEMMKERNEL = zgemm_kernel_loongson3a.S +CGEMMKERNEL = cgemm_kernel_loongson3a_2x2.S +CGEMMONCOPY = ../generic/zgemm_ncopy_2.c +CGEMMOTCOPY = ../generic/zgemm_tcopy_2.c +CGEMMONCOPYOBJ = cgemm_oncopy.o +CGEMMOTCOPYOBJ = cgemm_otcopy.o + +ZGEMMKERNEL = zgemm_kernel_loongson3a_2x2.S ZGEMMONCOPY = ../generic/zgemm_ncopy_2.c ZGEMMOTCOPY = ../generic/zgemm_tcopy_2.c ZGEMMONCOPYOBJ = zgemm_oncopy.o @@ -29,6 +35,11 @@ DTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c DTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c DTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c +CTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +CTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +CTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +CTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + ZTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c ZTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c ZTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c diff --git a/kernel/mips64/cgemm_kernel_loongson3a_2x2.S b/kernel/mips64/cgemm_kernel_loongson3a_2x2.S new file mode 100644 index 000000000..5ded7aed0 --- /dev/null +++ b/kernel/mips64/cgemm_kernel_loongson3a_2x2.S @@ -0,0 +1,1468 @@ +#define ASSEMBLER +#include "common.h" + +#define FETCH ld +#define gsLQC1(base,fq,ft,offset) .word(0x32<<26|base<<21|ft<<16|0x1<<15|offset<<6|0x1<<5|fq) +#define gsSQC1(base,fq,ft,offset) .word(0x3A<<26|base<<21|ft<<16|0x1<<15|offset<<6|0x1<<5|fq) + + +#define STACKSIZE 160 +#define M $4 +#define N $5 +#define K $6 +#define A $9 +#define B $10 +#define C $11 +#define LDC $8 + +#define AO $12 +#define BO $13 + +#define R12 12 +#define R13 13 + +#define I $2 +#define J $3 +#define L $7 + +#define CO1 $14 +#define CO2 $15 +#define PREA $16 +#define PREB $17 + +#if defined(TRMMKERNEL) +#define OFFSET $18 +#define KK $19 +#define TEMP $20 +#endif + +#define a1 $f0 +#define a2 $f1 +#define a3 $f2 +#define a4 $f3 + +#define b1 $f4 +#define b2 $f5 +#define b3 $f6 +#define b4 $f7 + +#define a5 $f8 +#define a6 $f9 +#define a7 $f10 +#define a8 $f11 + +#define b5 $f12 +#define b6 $f13 +#define b7 $f15 +#define b8 $f16 + +#define c11 $f14 +#define c12 $f17 +#define c13 $f18 +#define c14 $f19 +#define c21 $f20 +#define c22 $f21 +#define c23 $f22 +#define c24 $f23 +#define c31 $f24 +#define c32 $f25 +#define c33 $f26 +#define c34 $f27 +#define c41 $f28 +#define c42 $f29 +#define c43 $f30 +#define c44 $f31 + +#define F0 0 +#define F1 1 +#define F2 2 +#define F3 3 +#define F4 4 +#define F5 5 +#define F6 6 +#define F7 7 +#define F8 8 +#define F9 9 +#define F10 10 +#define F11 11 +#define F12 12 +#define F13 13 +#define F14 14 +#define F15 15 +#define F16 16 +#define F17 17 +#define F18 18 +#define F19 19 +#define F20 20 +#define F21 21 +#define F22 22 +#define F23 23 +#define F24 24 +#define F25 25 +#define F26 26 +#define F27 27 +#define F28 28 +#define F29 29 +#define F30 30 +#define F31 31 + +#define ALPHA_R $f15 +#define ALPHA_I $f16 + +################################# +## MADD1 a*c +## MADD2 b*c +## MADD3 a*d +## MADD4 d*b +################################## +#if defined(NN) || defined(NT) || defined(TN) || defined(TT) +#define MADD1 MADD +#define MADD2 MADD +#define MADD3 MADD +#define MADD4 NMSUB +#endif + +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) +#define MADD1 MADD +#define MADD2 MADD +#define MADD3 NMSUB +#define MADD4 MADD +#endif + +#if defined(RN) || defined(RT) || defined(CN) || defined(CT) +#define MADD1 MADD +#define MADD2 NMSUB +#define MADD3 MADD +#define MADD4 MADD +#endif + +#if defined(RR) || defined(RC) || defined(CR) || defined(CC) +#define MADD1 MADD +#define MADD2 NMSUB +#define MADD3 NMSUB +#define MADD4 NMSUB +#endif + + PROLOGUE + + LDARG LDC, 0($sp) + daddiu $sp, $sp, -STACKSIZE + + SDARG $16, 0($sp) + SDARG $17, 8($sp) + sdc1 $f24, 16($sp) + sdc1 $f25, 24($sp) + sdc1 $f26, 32($sp) + sdc1 $f27, 40($sp) + sdc1 $f28, 48($sp) + sdc1 $f29, 56($sp) + +#if defined(TRMMKERNEL) + SDARG $18, 64($sp) + SDARG $19, 72($sp) + SDARG $20, 80($sp) + + LDARG OFFSET, STACKSIZE + 8($sp) +#endif + +#ifndef __64BIT__ + sdc1 $f20, 88($sp) + sdc1 $f21, 96($sp) + sdc1 $f22,104($sp) + sdc1 $f23,112($sp) +#endif + + dsra J, N, 1 # J=N/2 + ST ALPHA_R, 128($sp) # store alpha_r & alpha_i + +#if defined(TRMMKERNEL) && !defined(LEFT) + neg KK, OFFSET +#endif + + dsll LDC, LDC, ZBASE_SHIFT # LDC*SIZE*COMPSIZE + blez J, .L20 + ST ALPHA_I, 136($sp) + + + .align 5 +.L10: +#if defined(TRMMKERNEL) && defined(LEFT) + move KK, OFFSET +#endif + + daddiu J, J, -1 + dsra I, M, 1 # I=M/2 + + dsll PREB, K, 1 + ZBASE_SHIFT # PREA=K*2*2^4 + dsll PREA, K, 1 + ZBASE_SHIFT # PREA=K*2*2^4 + + move CO1, C # Fix pointer Cx + daddu CO2, C, LDC + + move AO, A # Reset AO + blez I, .L30 + daddu PREA, PREA, A # PREA=A+panel size + +.L11: +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + move BO, B +#else + dsll TEMP, KK, 1 + ZBASE_SHIFT + + daddu AO, AO, TEMP + daddu BO, B, TEMP +#endif + + MTC $0, c11 # Clear results regs + LD a1, 0 * SIZE(AO) + MOV c12, c11 + LD a2, 1 * SIZE(AO) + + MOV c13, c11 + LD b1, 0 * SIZE(BO) + MOV c14, c11 + LD b2, 1 * SIZE(BO) + + MOV c21, c11 + LD a3, 2 * SIZE(AO) + MOV c22, c11 + LD a4, 3 * SIZE(AO) + + MOV c23, c11 + LD b3, 2 * SIZE(BO) + MOV c24, c11 + LD b4, 3 * SIZE(BO) + + FETCH $0, 0 * SIZE(CO2) + MOV c31, c11 + MOV c32, c11 + + FETCH $0, 0 * SIZE(CO1) + MOV c33, c11 + MOV c34, c11 + + FETCH $0, 4 * SIZE(CO2) + MOV c41, c11 + MOV c42, c11 + + FETCH $0, 4 * SIZE(CO1) + MOV c43, c11 + MOV c44, c11 + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + dsubu TEMP, K, KK +#elif defined(LEFT) + daddiu TEMP, KK, 2 +#else + daddiu TEMP, KK, 2 +#endif + dsra L, TEMP, 2 + daddu PREB, PREB, B # PREA=A+panel size + blez L, .L15 + NOP + +#else + + dsra L, K, 2 # Unroll K 4 times + move BO, B + + MTC $0, c11 # Clear results regs + LD a1, 0 * SIZE(AO) + MOV c12, c11 + LD a2, 1 * SIZE(AO) + + MOV c13, c11 + LD b1, 0 * SIZE(BO) + MOV c14, c11 + LD b2, 1 * SIZE(BO) + + MOV c21, c11 + LD a3, 2 * SIZE(AO) + MOV c22, c11 + LD a4, 3 * SIZE(AO) + + MOV c23, c11 + LD b3, 2 * SIZE(BO) + MOV c24, c11 + LD b4, 3 * SIZE(BO) + + MOV c31, c11 + MOV c32, c11 + FETCH $0, 0 * SIZE(CO2) + + MOV c33, c11 + MOV c34, c11 + FETCH $0, 0 * SIZE(CO1) + + MOV c41, c11 + MOV c42, c11 + FETCH $0, 4 * SIZE(CO2) + + MOV c43, c11 + NOP + FETCH $0, 4 * SIZE(CO1) + + daddu PREB, PREB, B # PREA=A+panel size + blez L, .L15 + MOV c44, c11 +#endif + + .align 5 + +.L12: + LD a5, 4 * SIZE(AO) + LD a6, 5 * SIZE(AO) + MADD1 c11, c11, a1, b1 # axc A1xB1 + MADD3 c13, c13, a1, b2 # axd + + LD b5, 4 * SIZE(BO) + LD b6, 5 * SIZE(BO) + MADD2 c12, c12, a2, b1 # bxc + MADD4 c14, c14, a2, b2 # bxd + + LD a7, 6 * SIZE(AO) + LD a8, 7 * SIZE(AO) + MADD1 c21, c21, a3, b1 # A2xB1 + MADD3 c23, c23, a3, b2 + + LD b7, 6 * SIZE(BO) + LD b8, 7 * SIZE(BO) + MADD2 c22, c22, a4, b1 + MADD4 c24, c24, a4, b2 + + FETCH $0, 4 * SIZE(PREA) + FETCH $0, 4 * SIZE(PREB) + MADD1 c31, c31, a1, b3 # A1xB2 + MADD3 c33, c33, a1, b4 + + MADD2 c32, c32, a2, b3 + MADD4 c34, c34, a2, b4 + + MADD1 c41, c41, a3, b3 # A2xB2 + MADD3 c43, c43, a3, b4 + MADD2 c42, c42, a4, b3 + MADD4 c44, c44, a4, b4 + + LD a1, 8 * SIZE(AO) + LD a2, 9 * SIZE(AO) + MADD1 c11, c11, a5, b5 # axc A1xB1 + MADD3 c13, c13, a5, b6 # axd + + LD b1, 8 * SIZE(BO) + LD b2, 9 * SIZE(BO) + MADD2 c12, c12, a6, b5 # bxc + MADD4 c14, c14, a6, b6 # bxd + + LD a3, 10 * SIZE(AO) + LD a4, 11 * SIZE(AO) + MADD1 c21, c21, a7, b5 # A2xB1 + MADD3 c23, c23, a7, b6 + + LD b3, 10 * SIZE(BO) + LD b4, 11 * SIZE(BO) + MADD2 c22, c22, a8, b5 + MADD4 c24, c24, a8, b6 + + FETCH $0, 8 * SIZE(PREA) + FETCH $0, 8 * SIZE(PREB) + MADD1 c31, c31, a5, b7 # A1xB2 + MADD3 c33, c33, a5, b8 + + MADD2 c32, c32, a6, b7 + MADD4 c34, c34, a6, b8 + + MADD1 c41, c41, a7, b7 # A2xB2 + MADD3 c43, c43, a7, b8 + MADD2 c42, c42, a8, b7 + MADD4 c44, c44, a8, b8 + + LD a5, 12 * SIZE(AO) + LD a6, 13 * SIZE(AO) + MADD1 c11, c11, a1, b1 # axc A1xB1 + MADD3 c13, c13, a1, b2 # axd + + LD b5, 12 * SIZE(BO) + LD b6, 13 * SIZE(BO) + MADD2 c12, c12, a2, b1 # bxc + MADD4 c14, c14, a2, b2 # bxd + + LD a7, 14 * SIZE(AO) + LD a8, 15 * SIZE(AO) + MADD1 c21, c21, a3, b1 # A2xB1 + MADD3 c23, c23, a3, b2 + + LD b7, 14 * SIZE(BO) + LD b8, 15 * SIZE(BO) + MADD2 c22, c22, a4, b1 + MADD4 c24, c24, a4, b2 + + FETCH $0, 12 * SIZE(PREA) + MADD1 c31, c31, a1, b3 # A1xB2 + MADD3 c33, c33, a1, b4 + daddiu L, L, -1 + + FETCH $0, 12 * SIZE(PREB) + MADD2 c32, c32, a2, b3 + MADD4 c34, c34, a2, b4 + daddiu AO, AO, 16 * SIZE + + daddiu BO, BO, 16 * SIZE # 2nr*4kr*cmpx + MADD1 c41, c41, a3, b3 # A2xB2 + MADD3 c43, c43, a3, b4 + daddu PREA, PREA, 16 * SIZE + + MADD2 c42, c42, a4, b3 + MADD4 c44, c44, a4, b4 + daddu PREB, PREB, 16 * SIZE + + LD a1, 0 * SIZE(AO) + LD a2, 1 * SIZE(AO) + MADD1 c11, c11, a5, b5 # axc A1xB1 + MADD3 c13, c13, a5, b6 # axd + + LD b1, 0 * SIZE(BO) + LD b2, 1 * SIZE(BO) + MADD2 c12, c12, a6, b5 # bxc + MADD4 c14, c14, a6, b6 # bxd + + LD a3, 2 * SIZE(AO) + LD a4, 3 * SIZE(AO) + MADD1 c21, c21, a7, b5 # A2xB1 + MADD3 c23, c23, a7, b6 + + LD b3, 2 * SIZE(BO) + LD b4, 3 * SIZE(BO) + MADD2 c22, c22, a8, b5 + MADD4 c24, c24, a8, b6 + + FETCH $0, 0 * SIZE(PREA) + FETCH $0, 0 * SIZE(PREB) + MADD1 c31, c31, a5, b7 # A1xB2 + MADD3 c33, c33, a5, b8 + + MADD2 c32, c32, a6, b7 + MADD4 c34, c34, a6, b8 + + MADD1 c41, c41, a7, b7 # A2xB2 + MADD3 c43, c43, a7, b8 + + MADD2 c42, c42, a8, b7 + bgtz L, .L12 + MADD4 c44, c44, a8, b8 + + .align 5 + +.L15: +#ifndef TRMMKERNEL + andi L, K, 3 + LD ALPHA_R, 128($sp) +#else + andi L, TEMP, 3 + LD ALPHA_R, 128($sp) +#endif + blez L, .L18 + LD ALPHA_I, 136($sp) + + .align 5 + +.L16: + daddiu BO, BO, 4 * SIZE # 2nr*1kr*cmpx + daddiu AO, AO, 4 * SIZE # 2mr*1kr*cmpx + MADD1 c11, c11, a1, b1 # axc A1xB1 + MADD3 c13, c13, a1, b2 # axd + + daddiu PREA, PREA, 4 * SIZE + daddiu PREB, PREB, 4 * SIZE + MADD2 c12, c12, a2, b1 # bxc + MADD4 c14, c14, a2, b2 # bxd + + MADD1 c21, c21, a3, b1 # A2xB1 + MADD3 c23, c23, a3, b2 + + MADD2 c22, c22, a4, b1 + MADD4 c24, c24, a4, b2 + + FETCH $0, 0 * SIZE(PREA) + MADD1 c31, c31, a1, b3 # A1xB2 + MADD3 c33, c33, a1, b4 + daddiu L, L, -1 + + MADD2 c32, c32, a2, b3 + MADD4 c34, c34, a2, b4 + + FETCH $0, 0 * SIZE(PREB) + MADD1 c41, c41, a3, b3 # A2xB2 + MADD3 c43, c43, a3, b4 + + MADD2 c42, c42, a4, b3 + MADD4 c44, c44, a4, b4 + + LD a1, 0 * SIZE(AO) + LD a2, 1 * SIZE(AO) + LD b1, 0 * SIZE(BO) + LD b2, 1 * SIZE(BO) + LD a3, 2 * SIZE(AO) + LD a4, 3 * SIZE(AO) + LD b3, 2 * SIZE(BO) + LD b4, 3 * SIZE(BO) + bgtz L, .L16 + NOP + +.L18: +#ifndef TRMMKERNEL + ADD c11, c14, c11 + LD a1, 0 * SIZE(CO1) + ADD c12, c13, c12 + LD a2, 1 * SIZE(CO1) + ADD c21, c24, c21 + LD b1, 2 * SIZE(CO1) + ADD c22, c23, c22 + LD b2, 3 * SIZE(CO1) + + ADD c31, c34, c31 + LD a3, 0 * SIZE(CO2) + ADD c32, c33, c32 + LD a4, 1 * SIZE(CO2) + ADD c41, c44, c41 + LD b3, 2 * SIZE(CO2) + ADD c42, c43, c42 + LD b4, 3 * SIZE(CO2) + + daddiu I, I, -1 + MADD a1, a1, ALPHA_R, c11 + MADD a2, a2, ALPHA_R, c12 + MADD b1, b1, ALPHA_R, c21 + MADD b2, b2, ALPHA_R, c22 + + NMSUB a1, a1, ALPHA_I, c12 + MADD a2, a2, ALPHA_I, c11 + NMSUB b1, b1, ALPHA_I, c22 + MADD b2, b2, ALPHA_I, c21 + + MADD a3, a3, ALPHA_R, c31 + MADD a4, a4, ALPHA_R, c32 + ST a1, 0 * SIZE(CO1) + MADD b3, b3, ALPHA_R, c41 + MADD b4, b4, ALPHA_R, c42 + ST a2, 1 * SIZE(CO1) + + NMSUB a3, a3, ALPHA_I, c32 + MADD a4, a4, ALPHA_I, c31 + ST b1, 2 * SIZE(CO1) + + NMSUB b3, b3, ALPHA_I, c42 + MADD b4, b4, ALPHA_I, c41 + ST b2, 3 * SIZE(CO1) + + ST a3, 0 * SIZE(CO2) + ST a4, 1 * SIZE(CO2) + ST b3, 2 * SIZE(CO2) + ST b4, 3 * SIZE(CO2) + +#else + ADD c11, c14, c11 + ADD c12, c13, c12 + ADD c21, c24, c21 + ADD c22, c23, c22 + + ADD c31, c34, c31 + ADD c32, c33, c32 + ADD c41, c44, c41 + ADD c42, c43, c42 + + daddiu I, I, -1 + MUL a1, ALPHA_R, c11 + MUL a2, ALPHA_R, c12 + MUL b1, ALPHA_R, c21 + MUL b2, ALPHA_R, c22 + + NMSUB a1, a1, ALPHA_I, c12 + MADD a2, a2, ALPHA_I, c11 + NMSUB b1, b1, ALPHA_I, c22 + MADD b2, b2, ALPHA_I, c21 + + MUL a3, ALPHA_R, c31 + MUL a4, ALPHA_R, c32 + MUL b3, ALPHA_R, c41 + MUL b4, ALPHA_R, c42 + + NMSUB a3, a3, ALPHA_I, c32 + MADD a4, a4, ALPHA_I, c31 + NMSUB b3, b3, ALPHA_I, c42 + MADD b4, b4, ALPHA_I, c41 + + ST a1, 0 * SIZE(CO1) + ST a2, 1 * SIZE(CO1) + ST b1, 2 * SIZE(CO1) + ST b2, 3 * SIZE(CO1) + + ST a3, 0 * SIZE(CO2) + ST a4, 1 * SIZE(CO2) + ST b3, 2 * SIZE(CO2) + ST b4, 3 * SIZE(CO2) + + +#if ( defined(LEFT) && defined(TRANSA)) || \ + (!defined(LEFT) && !defined(TRANSA)) + dsubu TEMP, K, KK +#ifdef LEFT + daddiu TEMP, TEMP, -2 +#else + daddiu TEMP, TEMP, -2 +#endif + + dsll TEMP, TEMP, 1 + ZBASE_SHIFT + + daddu AO, AO, TEMP + daddu BO, BO, TEMP +#endif + +#ifdef LEFT + daddiu KK, KK, 2 +#endif +#endif + + dsll PREB, K, 1 + ZBASE_SHIFT # PREA=K*2*2^4 + daddiu CO1,CO1, 4 * SIZE + bgtz I, .L11 + daddiu CO2,CO2, 4 * SIZE + + .align 5 +.L30: + andi I, M, 1 + daddu C, C, LDC # Change C to next panel + + daddu PREB, PREB, B # PREA=A+panel size + blez I, .L19 + daddu C, C, LDC # Change C to next panel + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + move BO, B +#else + dsll L, KK, ZBASE_SHIFT # MR=1 + dsll TEMP, KK, 1 + ZBASE_SHIFT # NR=2 + + daddu AO, AO, L + daddu BO, B, TEMP +#endif + + LD a1, 0 * SIZE(AO) + LD a2, 1 * SIZE(AO) + MTC $0, c11 # Clear results regs + MOV c12, c11 + + LD b1, 0 * SIZE(BO) + LD b2, 1 * SIZE(BO) + MOV c13, c11 + MOV c14, c11 + + LD b3, 2 * SIZE(BO) + LD b4, 3 * SIZE(BO) + MOV c31, c11 + MOV c32, c11 + + FETCH $0, 0 * SIZE(PREB) + MOV c33, c11 + MOV c34, c11 + + FETCH $0, 0 * SIZE(CO1) + FETCH $0, 0 * SIZE(CO2) + FETCH $0, 4 * SIZE(CO1) + FETCH $0, 4 * SIZE(CO2) + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + dsubu TEMP, K, KK +#elif defined(LEFT) + daddiu TEMP, KK, 1 # MR=1 +#else + daddiu TEMP, KK, 2 # NR=2 +#endif + dsra L, TEMP, 2 + blez L, .L35 + NOP + +#else + + LD a1, 0 * SIZE(AO) + LD a2, 1 * SIZE(AO) + dsra L, K, 2 # Unroll K 4 times + move BO, B + + LD b1, 0 * SIZE(BO) + LD b2, 1 * SIZE(BO) + MTC $0, c11 # Clear results regs + MOV c12, c11 + + LD b3, 2 * SIZE(BO) + LD b4, 3 * SIZE(BO) + MOV c13, c11 + MOV c14, c11 + + FETCH $0, 0 * SIZE(PREB) + MOV c31, c11 + MOV c32, c11 + + FETCH $0, 0 * SIZE(CO1) + FETCH $0, 0 * SIZE(CO2) + FETCH $0, 4 * SIZE(CO1) + FETCH $0, 4 * SIZE(CO2) + + MOV c33, c11 + blez L, .L35 + MOV c34, c11 +#endif + + .align 5 + +.L32: + LD a3, 2 * SIZE(AO) + LD a4, 3 * SIZE(AO) + MADD1 c11, c11, a1, b1 # axc A1xB1 + MADD3 c13, c13, a1, b2 # axd + + LD b5, 4 * SIZE(BO) + LD b6, 5 * SIZE(BO) + MADD2 c12, c12, a2, b1 # bxc + MADD4 c14, c14, a2, b2 # bxd + + LD b7, 6 * SIZE(BO) + LD b8, 7 * SIZE(BO) + MADD1 c31, c31, a1, b3 # A1xB2 + MADD3 c33, c33, a1, b4 + + FETCH $0, 4 * SIZE(PREB) + MADD2 c32, c32, a2, b3 + MADD4 c34, c34, a2, b4 + NOP + + LD a5, 4 * SIZE(AO) + LD a6, 5 * SIZE(AO) + MADD1 c11, c11, a3, b5 # axc A1xB1 + MADD3 c13, c13, a3, b6 # axd + + LD b1, 8 * SIZE(BO) + LD b2, 9 * SIZE(BO) + MADD2 c12, c12, a4, b5 # bxc + MADD4 c14, c14, a4, b6 # bxd + + LD b3, 10 * SIZE(BO) + LD b4, 11 * SIZE(BO) + MADD1 c31, c31, a3, b7 # A1xB2 + MADD3 c33, c33, a3, b8 + + FETCH $0, 8 * SIZE(PREB) + MADD2 c32, c32, a4, b7 + MADD4 c34, c34, a4, b8 + daddiu L, L, -1 + + LD a7, 6 * SIZE(AO) + LD a8, 7 * SIZE(AO) + MADD1 c11, c11, a5, b1 # axc A1xB1 + MADD3 c13, c13, a5, b2 # axd + + LD b5, 12 * SIZE(BO) + LD b6, 13 * SIZE(BO) + MADD2 c12, c12, a6, b1 # bxc + MADD4 c14, c14, a6, b2 # bxd + + LD b7, 14 * SIZE(BO) + LD b8, 15 * SIZE(BO) + MADD1 c31, c31, a5, b3 # A1xB2 + MADD3 c33, c33, a5, b4 + + daddiu AO, AO, 8 * SIZE # 2mr*4kr*cmpx + daddiu BO, BO, 16 * SIZE # 2nr*4kr*cmpx + + FETCH $0, 12 * SIZE(PREB) + MADD2 c32, c32, a6, b3 + MADD4 c34, c34, a6, b4 + NOP + + LD a1, 0 * SIZE(AO) + LD a2, 1 * SIZE(AO) + MADD1 c11, c11, a7, b5 # axc A1xB1 + MADD3 c13, c13, a7, b6 # axd + + LD b1, 0 * SIZE(BO) + LD b2, 1 * SIZE(BO) + MADD2 c12, c12, a8, b5 # bxc + MADD4 c14, c14, a8, b6 # bxd + + LD b3, 2 * SIZE(BO) + LD b4, 3 * SIZE(BO) + MADD1 c31, c31, a7, b7 # A1xB2 + NOP + + MADD3 c33, c33, a7, b8 + daddiu PREB, PREB, 16 * SIZE + + FETCH $0, 0 * SIZE(PREB) + MADD2 c32, c32, a8, b7 + bgtz L, .L32 + MADD4 c34, c34, a8, b8 + + +.L35: +#ifndef TRMMKERNEL + andi L, K, 3 + LD ALPHA_R, 128($sp) +#else + andi L, TEMP, 3 + LD ALPHA_R, 128($sp) +#endif + blez L, .L38 + LD ALPHA_I, 136($sp) + .align 5 + +.L36: + daddiu L, L, -1 + MADD1 c11, c11, a1, b1 # axc A1xB1 + MADD3 c13, c13, a1, b2 # axd + + daddiu BO, BO, 4 * SIZE # 2nr*1kr*cmpx + MADD2 c12, c12, a2, b1 # bxc + MADD4 c14, c14, a2, b2 # bxd + + daddiu AO, AO, 2 * SIZE # 2mr*1kr*cmpx + MADD1 c31, c31, a1, b3 # A1xB2 + MADD3 c33, c33, a1, b4 + + daddiu PREB, PREB, 4 * SIZE + MADD2 c32, c32, a2, b3 + MADD4 c34, c34, a2, b4 + + LD a1, 0 * SIZE(AO) + LD a2, 1 * SIZE(AO) + + LD b1, 0 * SIZE(BO) + LD b2, 1 * SIZE(BO) + LD b3, 2 * SIZE(BO) + LD b4, 3 * SIZE(BO) + bgtz L, .L36 + NOP + +.L38: +#ifndef TRMMKERNEL + ADD c11, c14, c11 + LD a1, 0 * SIZE(CO1) + ADD c12, c13, c12 + LD a2, 1 * SIZE(CO1) + + ADD c31, c34, c31 + LD a3, 0 * SIZE(CO2) + ADD c32, c33, c32 + LD a4, 1 * SIZE(CO2) + + MADD a1, a1, ALPHA_R, c11 + MADD a2, a2, ALPHA_R, c12 + + MADD a3, a3, ALPHA_R, c31 + MADD a4, a4, ALPHA_R, c32 + + NMSUB a1, a1, ALPHA_I, c12 + MADD a2, a2, ALPHA_I, c11 + + NMSUB a3, a3, ALPHA_I, c32 + MADD a4, a4, ALPHA_I, c31 + + ST a1, 0 * SIZE(CO1) + ST a2, 1 * SIZE(CO1) + + ST a3, 0 * SIZE(CO2) + ST a4, 1 * SIZE(CO2) + + daddiu CO1,CO1, 2 * SIZE + daddiu CO2,CO2, 2 * SIZE + +#else + ADD c11, c14, c11 + ADD c12, c13, c12 + + ADD c31, c34, c31 + ADD c32, c33, c32 + + MUL a1, ALPHA_R, c11 + MUL a2, ALPHA_R, c12 + + MUL a3, ALPHA_R, c31 + MUL a4, ALPHA_R, c32 + + NMSUB a1, a1, ALPHA_I, c12 + MADD a2, a2, ALPHA_I, c11 + + NMSUB a3, a3, ALPHA_I, c32 + MADD a4, a4, ALPHA_I, c31 + + ST a1, 0 * SIZE(CO1) + ST a2, 1 * SIZE(CO1) + + ST a3, 0 * SIZE(CO2) + ST a4, 1 * SIZE(CO2) + + daddiu CO1,CO1, 2 * SIZE + daddiu CO2,CO2, 2 * SIZE + +#if ( defined(LEFT) && defined(TRANSA)) || \ + (!defined(LEFT) && !defined(TRANSA)) + dsubu TEMP, K, KK +#ifdef LEFT + daddiu TEMP, TEMP, -1 +#else + daddiu TEMP, TEMP, -2 +#endif + dsll L, TEMP, ZBASE_SHIFT + dsll TEMP, TEMP, 1 + ZBASE_SHIFT + + daddu AO, AO, L + daddu BO, BO, TEMP +#endif + +#ifdef LEFT + daddiu KK, KK, 1 +#endif +#endif + + .align 5 + +.L19: +#if defined(TRMMKERNEL) && !defined(LEFT) + daddiu KK, KK, 2 +#endif + + bgtz J, .L10 + move B, BO + + .align 5 + +.L20: + andi J, N, 1 + blez J, .L999 + dsll PREA, K, 1+ZBASE_SHIFT # PREA=K*2*2^4 + + dsra I, M, 1 # I=M/2 + move CO1, C + +#if defined(TRMMKERNEL) && defined(LEFT) + move KK, OFFSET +#endif + + move AO, A # Reset AO + blez I, .L29 + daddu PREA, PREA, A + +.L21: +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + move BO, B +#else + dsll L, KK, 1 + ZBASE_SHIFT + dsll TEMP, KK, ZBASE_SHIFT + + daddu AO, AO, L + daddu BO, B, TEMP +#endif + + LD a1, 0 * SIZE(AO) + LD a2, 1 * SIZE(AO) + MTC $0, c11 # Clear results regs + MOV c12, c11 + + LD b1, 0 * SIZE(BO) + LD b2, 1 * SIZE(BO) + MOV c13, c11 + MOV c14, c11 + + LD a3, 2 * SIZE(AO) + LD a4, 3 * SIZE(AO) + MOV c21, c11 + MOV c22, c11 + + FETCH $0, 0 * SIZE(PREA) + MOV c23, c11 + MOV c24, c11 + + FETCH $0, 0 * SIZE(CO1) + FETCH $0, 4 * SIZE(CO1) + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + dsubu TEMP, K, KK +#elif defined(LEFT) + daddiu TEMP, KK, 2 # define Mr=2 +#else + daddiu TEMP, KK, 1 # define NR=1 +#endif + dsra L, TEMP, 2 + blez L, .L25 + NOP + +#else + dsra L, K, 2 # Unroll K 4 times + move BO, B + + LD a1, 0 * SIZE(AO) + LD a2, 1 * SIZE(AO) + MTC $0, c11 # Clear results regs + MOV c12, c11 + + LD b1, 0 * SIZE(BO) + LD b2, 1 * SIZE(BO) + MOV c13, c11 + MOV c14, c11 + + LD a3, 2 * SIZE(AO) + LD a4, 3 * SIZE(AO) + MOV c21, c11 + MOV c22, c11 + + FETCH $0, 0 * SIZE(PREA) + MOV c23, c11 + MOV c24, c11 + + FETCH $0, 0 * SIZE(CO1) + FETCH $0, 4 * SIZE(CO1) + + blez L, .L25 + NOP +#endif + + .align 5 + +.L22: + LD a5, 4 * SIZE(AO) + LD a6, 5 * SIZE(AO) + MADD1 c11, c11, a1, b1 # axc A1xB1 + MADD3 c13, c13, a1, b2 # axd + + LD b3, 2 * SIZE(BO) + LD b4, 3 * SIZE(BO) + MADD2 c12, c12, a2, b1 # bxc + MADD4 c14, c14, a2, b2 # bxd + + LD a7, 6 * SIZE(AO) + LD a8, 7 * SIZE(AO) + MADD1 c21, c21, a3, b1 # A2xB1 + MADD3 c23, c23, a3, b2 + + FETCH $0, 4 * SIZE(PREA) + MADD2 c22, c22, a4, b1 + MADD4 c24, c24, a4, b2 + + LD a1, 8 * SIZE(AO) + LD a2, 9 * SIZE(AO) + MADD1 c11, c11, a5, b3 # axc A1xB1 + MADD3 c13, c13, a5, b4 # axd + + LD b5, 4 * SIZE(BO) + LD b6, 5 * SIZE(BO) + MADD2 c12, c12, a6, b3 # bxc + MADD4 c14, c14, a6, b4 # bxd + + LD a3, 10 * SIZE(AO) + LD a4, 11 * SIZE(AO) + MADD1 c21, c21, a7, b3 # A2xB1 + MADD3 c23, c23, a7, b4 + + FETCH $0, 8 * SIZE(PREA) + MADD2 c22, c22, a8, b3 + MADD4 c24, c24, a8, b4 + daddiu L, L, -1 + + LD a5, 12 * SIZE(AO) + LD a6, 13 * SIZE(AO) + MADD1 c11, c11, a1, b5 # axc A1xB1 + MADD3 c13, c13, a1, b6 # axd + + LD b7, 6 * SIZE(BO) + LD b8, 7 * SIZE(BO) + MADD2 c12, c12, a2, b5 # bxc + MADD4 c14, c14, a2, b6 # bxd + + LD a7, 14 * SIZE(AO) + LD a8, 15 * SIZE(AO) + MADD1 c21, c21, a3, b5 # A2xB1 + MADD3 c23, c23, a3, b6 + + daddiu BO, BO, 8 * SIZE # 1nr*4kr*cmpx + daddiu AO, AO, 16 * SIZE # 2mr*4kr*cmpx + + FETCH $0, 12 * SIZE(PREA) + MADD2 c22, c22, a4, b5 + MADD4 c24, c24, a4, b6 + daddiu PREA, PREA, 16 * SIZE + + LD a1, 0 * SIZE(AO) + LD a2, 1 * SIZE(AO) + MADD1 c11, c11, a5, b7 # axc A1xB1 + MADD3 c13, c13, a5, b8 # axd + + LD b1, 0 * SIZE(BO) + LD b2, 1 * SIZE(BO) + MADD2 c12, c12, a6, b7 # bxc + MADD4 c14, c14, a6, b8 # bxd + + LD a3, 2 * SIZE(AO) + LD a4, 3 * SIZE(AO) + MADD1 c21, c21, a7, b7 # A2xB1 + MADD3 c23, c23, a7, b8 + + FETCH $0, 0 * SIZE(PREA) + MADD2 c22, c22, a8, b7 + bgtz L, .L22 + MADD4 c24, c24, a8, b8 + + +.L25: +#ifndef TRMMKERNEL + andi L, K, 3 + LD ALPHA_R, 128($sp) +#else + andi L, TEMP, 3 + LD ALPHA_R, 128($sp) +#endif + blez L, .L28 + LD ALPHA_I, 136($sp) + .align 3 + +.L26: + daddiu L, L, -1 + MADD1 c11, c11, a1, b1 # axc A1xB1 + MADD3 c13, c13, a1, b2 # axd + + daddiu BO, BO, 2 * SIZE # 2nr*1kr*cmpx + MADD2 c12, c12, a2, b1 # bxc + MADD4 c14, c14, a2, b2 # bxd + + daddiu AO, AO, 4 * SIZE # 2mr*1kr*cmpx + MADD1 c21, c21, a3, b1 # A2xB1 + MADD3 c23, c23, a3, b2 + + daddiu PREA, PREA, 4 * SIZE # 2mr*1kr*cmpx + MADD2 c22, c22, a4, b1 + MADD4 c24, c24, a4, b2 + +# gsLQC1(R12, F1, F0, 0) # R:a1 I:a2 Unroll K=4 +# gsLQC1(R12, F3, F2, 1) # R:a3 I:a4 +# gsLQC1(R13, F5, F4, 0) # R:b1 I:b2 + LD a1, 0 * SIZE(AO) + LD a2, 1 * SIZE(AO) + LD b1, 0 * SIZE(BO) + LD b2, 1 * SIZE(BO) + LD a3, 2 * SIZE(AO) + LD a4, 3 * SIZE(AO) + + bgtz L, .L26 + FETCH $0, 0 * SIZE(PREA) + +.L28: +#ifndef TRMMKERNEL + ADD c11, c14, c11 + LD a1, 0 * SIZE(CO1) + ADD c12, c13, c12 + LD a2, 1 * SIZE(CO1) + ADD c21, c24, c21 + LD b1, 2 * SIZE(CO1) + ADD c22, c23, c22 + LD b2, 3 * SIZE(CO1) + + daddiu I, I, -1 + MADD a1, a1, ALPHA_R, c11 + MADD a2, a2, ALPHA_R, c12 + MADD b1, b1, ALPHA_R, c21 + MADD b2, b2, ALPHA_R, c22 + + NMSUB a1, a1, ALPHA_I, c12 + MADD a2, a2, ALPHA_I, c11 + NMSUB b1, b1, ALPHA_I, c22 + MADD b2, b2, ALPHA_I, c21 + + ST a1, 0 * SIZE(CO1) + ST a2, 1 * SIZE(CO1) + ST b1, 2 * SIZE(CO1) + ST b2, 3 * SIZE(CO1) + +#else + ADD c11, c14, c11 + ADD c12, c13, c12 + ADD c21, c24, c21 + ADD c22, c23, c22 + + daddiu I, I, -1 + MUL a1, ALPHA_R, c11 + MUL a2, ALPHA_R, c12 + MUL b1, ALPHA_R, c21 + MUL b2, ALPHA_R, c22 + + NMSUB a1, a1, ALPHA_I, c12 + MADD a2, a2, ALPHA_I, c11 + NMSUB b1, b1, ALPHA_I, c22 + MADD b2, b2, ALPHA_I, c21 + + ST a1, 0 * SIZE(CO1) + ST a2, 1 * SIZE(CO1) + ST b1, 2 * SIZE(CO1) + ST b2, 3 * SIZE(CO1) + +#if ( defined(LEFT) && defined(TRANSA)) || \ + (!defined(LEFT) && !defined(TRANSA)) + dsubu TEMP, K, KK +#ifdef LEFT + daddiu TEMP, TEMP, -2 +#else + daddiu TEMP, TEMP, -1 +#endif + + dsll L, TEMP, 1 + ZBASE_SHIFT + dsll TEMP, TEMP, ZBASE_SHIFT + + daddu AO, AO, L + daddu BO, BO, TEMP +#endif + +#ifdef LEFT + daddiu KK, KK, 2 +#endif +#endif + daddiu CO1,CO1, 4 * SIZE + bgtz I, .L21 + NOP + +.L29: + andi I, M, 1 + blez I, .L999 + NOP + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + move BO, B +#else + dsll TEMP, KK, ZBASE_SHIFT + + daddu AO, AO, TEMP + daddu BO, B, TEMP +#endif + +# gsLQC1(R12, F1, F0, 0) # R:a1 I:a2 + LD a1, 0 * SIZE(AO) + LD a2, 1 * SIZE(AO) + MTC $0, c11 # Clear results regs + MOV c12, c11 + +# gsLQC1(R13, F5, F4, 0) # R:b1 I:b2 + LD b1, 0 * SIZE(BO) + LD b2, 1 * SIZE(BO) + MOV c13, c11 + MOV c14, c11 + + FETCH $0, 0 * SIZE(PREA) + FETCH $0, 4 * SIZE(PREA) + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + dsubu TEMP, K, KK +#elif defined(LEFT) + daddiu TEMP, KK, 1 +#else + daddiu TEMP, KK, 1 +#endif + dsra L, TEMP, 2 + blez L, .L45 + NOP + +#else + dsra L, K, 2 # Unroll K 4 times + move BO, B + +# gsLQC1(R12, F1, F0, 0) # R:a1 I:a2 + LD a1, 0 * SIZE(AO) + LD a2, 1 * SIZE(AO) + MTC $0, c11 # Clear results regs + MOV c12, c11 + +# gsLQC1(R13, F5, F4, 0) # R:b1 I:b2 + LD b1, 0 * SIZE(BO) + LD b2, 1 * SIZE(BO) + MOV c13, c11 + MOV c14, c11 + + FETCH $0, 0 * SIZE(PREA) + FETCH $0, 4 * SIZE(PREA) + blez L, .L45 + NOP +#endif + + .align 3 + +.L42: +# gsLQC1(R12, F3, F2, 1) # R:a3 I:a4 + LD a3, 2 * SIZE(AO) + LD a4, 3 * SIZE(AO) + MADD1 c11, c11, a1, b1 # axc A1xB1 + MADD3 c13, c13, a1, b2 # axd + +# gsLQC1(R13, F7, F6, 1) # R:b2 I:b3 + LD b3, 2 * SIZE(BO) + LD b4, 3 * SIZE(BO) + MADD2 c12, c12, a2, b1 # bxc + MADD4 c14, c14, a2, b2 # bxd + +# gsLQC1(R12, F9, F8, 2) # Unroll K=1 + LD a5, 4 * SIZE(AO) + LD a6, 5 * SIZE(AO) + MADD1 c11, c11, a3, b3 # axc A1xB1 + MADD3 c13, c13, a3, b4 # axd + +# gsLQC1(R13, F13, F12, 2) + LD b5, 4 * SIZE(BO) + LD b6, 5 * SIZE(BO) + MADD2 c12, c12, a4, b3 # bxc + MADD4 c14, c14, a4, b4 # bxd + +# gsLQC1(R12, F11, F10, 3) + LD a7, 6 * SIZE(AO) + LD a8, 7 * SIZE(AO) + MADD1 c11, c11, a5, b5 # axc A1xB1 + MADD3 c13, c13, a5, b6 # axd + + daddiu L, L, -1 + +# gsLQC1(R13, F16, F15, 3) + LD b7, 6 * SIZE(BO) + LD b8, 7 * SIZE(BO) + MADD2 c12, c12, a6, b5 # bxc + MADD4 c14, c14, a6, b6 # bxd + + daddiu AO, AO, 8 * SIZE # 2mr*4kr*cmpx + daddiu BO, BO, 8 * SIZE # 2nr*4kr*cmpx + +# gsLQC1(R12, F1, F0, 0) # R:a1 I:a2 Unroll K=4 + LD a1, 0 * SIZE(AO) + LD a2, 1 * SIZE(AO) + MADD1 c11, c11, a7, b7 # axc A1xB1 + MADD3 c13, c13, a7, b8 # axd + +# gsLQC1(R13, F5, F4, 0) # R:b1 I:b2 + LD b1, 0 * SIZE(BO) + LD b2, 1 * SIZE(BO) + MADD2 c12, c12, a8, b7 # bxc + MADD4 c14, c14, a8, b8 # bxd + + bgtz L, .L42 + NOP + + + .align 5 + +.L45: +#ifndef TRMMKERNEL + andi L, K, 3 + LD ALPHA_R, 128($sp) +#else + andi L, TEMP, 3 + LD ALPHA_R, 128($sp) +#endif + blez L, .L48 + LD ALPHA_I, 136($sp) + +.L46: + daddiu L, L, -1 + daddiu BO, BO, 1 * SIZE * COMPSIZE # 2nr*1kr*cmpx + daddiu AO, AO, 1 * SIZE * COMPSIZE # 2mr*1kr*cmpx + + MADD1 c11, c11, a1, b1 # axc A1xB1 + MADD3 c13, c13, a1, b2 # axd + MADD2 c12, c12, a2, b1 # bxc + MADD4 c14, c14, a2, b2 # bxd + +# gsLQC1(R12, F1, F0, 0) # R:a1 I:a2 Unroll K=4 +# gsLQC1(R13, F5, F4, 0) # R:b1 I:b2 + LD a1, 0 * SIZE(AO) + LD a2, 1 * SIZE(AO) + LD b1, 0 * SIZE(BO) + LD b2, 1 * SIZE(BO) + bgtz L, .L46 + NOP + +.L48: +#ifndef TRMMKERNEL + ADD c11, c14, c11 + ADD c12, c13, c12 + + LD a1, 0 * SIZE(CO1) + LD a2, 1 * SIZE(CO1) + + MADD a1, a1, ALPHA_R, c11 + MADD a2, a2, ALPHA_R, c12 + + NMSUB a1, a1, ALPHA_I, c12 + MADD a2, a2, ALPHA_I, c11 + + ST a1, 0 * SIZE(CO1) + ST a2, 1 * SIZE(CO1) + +#else + ADD c11, c14, c11 + ADD c12, c13, c12 + + MUL a1, ALPHA_R, c11 + MUL a2, ALPHA_R, c12 + + NMSUB a1, a1, ALPHA_I, c12 + MADD a2, a2, ALPHA_I, c11 + + ST a1, 0 * SIZE(CO1) + ST a2, 1 * SIZE(CO1) + +#if ( defined(LEFT) && defined(TRANSA)) || \ + (!defined(LEFT) && !defined(TRANSA)) + dsubu TEMP, K, KK +#ifdef LEFT + daddiu TEMP, TEMP, -1 +#else + daddiu TEMP, TEMP, -1 +#endif + + dsll TEMP, TEMP, ZBASE_SHIFT + + daddu AO, AO, TEMP + daddu BO, BO, TEMP +#endif + +#ifdef LEFT + daddiu KK, KK, 1 +#endif + + daddiu CO1,CO1, 2 * SIZE +#endif + + + + .align 5 + +.L999: + LDARG $16, 0($sp) + LDARG $17, 8($sp) + ldc1 $f24, 16($sp) + ldc1 $f25, 24($sp) + ldc1 $f26, 32($sp) + ldc1 $f27, 40($sp) + ldc1 $f28, 48($sp) + ldc1 $f29, 56($sp) + +#if defined(TRMMKERNEL) + LDARG $18, 64($sp) + LDARG $19, 72($sp) + LDARG $20, 80($sp) +#endif + +#ifndef __64BIT__ + ldc1 $f20, 88($sp) + ldc1 $f21, 96($sp) + ldc1 $f22,104($sp) + ldc1 $f23,112($sp) +#endif + + j $31 + daddiu $sp, $sp, STACKSIZE + + EPILOGUE diff --git a/kernel/mips64/gemm_kernel_loongson3a.S b/kernel/mips64/dgemm_kernel_loongson3a_4x4.S similarity index 100% rename from kernel/mips64/gemm_kernel_loongson3a.S rename to kernel/mips64/dgemm_kernel_loongson3a_4x4.S diff --git a/kernel/mips64/sgemm_kernel_loongson3a.S b/kernel/mips64/sgemm_kernel_loongson3a_4x4.S similarity index 100% rename from kernel/mips64/sgemm_kernel_loongson3a.S rename to kernel/mips64/sgemm_kernel_loongson3a_4x4.S diff --git a/kernel/mips64/zgemm_kernel_loongson3a.S b/kernel/mips64/zgemm_kernel_loongson3a_2x2.S similarity index 100% rename from kernel/mips64/zgemm_kernel_loongson3a.S rename to kernel/mips64/zgemm_kernel_loongson3a_2x2.S index 4cc396614..a8faad2f6 100644 --- a/kernel/mips64/zgemm_kernel_loongson3a.S +++ b/kernel/mips64/zgemm_kernel_loongson3a_2x2.S @@ -1065,8 +1065,8 @@ daddiu PREA, PREA, 4 * SIZE # 2mr*1kr*cmpx MADD2 c22, c22, a4, b1 MADD4 c24, c24, a4, b2 - gsLQC1(R12, F1, F0, 0) # R:a1 I:a2 Unroll K=4 + gsLQC1(R12, F1, F0, 0) # R:a1 I:a2 Unroll K=4 gsLQC1(R12, F3, F2, 1) # R:a3 I:a4 gsLQC1(R13, F5, F4, 0) # R:b1 I:b2 diff --git a/param.h b/param.h index cab3e68dd..fd399a96f 100644 --- a/param.h +++ b/param.h @@ -1486,25 +1486,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define DGEMM_DEFAULT_UNROLL_M 4 #define DGEMM_DEFAULT_UNROLL_N 4 -#define CGEMM_DEFAULT_UNROLL_M 1 -#define CGEMM_DEFAULT_UNROLL_N 4 +#define CGEMM_DEFAULT_UNROLL_M 2 +#define CGEMM_DEFAULT_UNROLL_N 2 #define ZGEMM_DEFAULT_UNROLL_M 2 #define ZGEMM_DEFAULT_UNROLL_N 2 #define SGEMM_DEFAULT_P 64 #define DGEMM_DEFAULT_P 32 -#define CGEMM_DEFAULT_P 108 +#define CGEMM_DEFAULT_P 64 #define ZGEMM_DEFAULT_P 32 #define SGEMM_DEFAULT_Q 116 #define DGEMM_DEFAULT_Q 116 -#define CGEMM_DEFAULT_Q 144 +#define CGEMM_DEFAULT_Q 100 #define ZGEMM_DEFAULT_Q 80 #define SGEMM_DEFAULT_R 1000 #define DGEMM_DEFAULT_R 1000 -#define CGEMM_DEFAULT_R 2000 +#define CGEMM_DEFAULT_R 1000 #define ZGEMM_DEFAULT_R 1000 #define SYMV_P 16