floraachy
/
OpenBLAS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
OpenBLAS/kernel/loongarch64/cgemm_kernel_8x4_lsx.S

3313 lines
102 KiB
ArmAsm
Raw Blame History

/*******************************************************************************
Copyright (c) 2024, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*******************************************************************************/
#define ASSEMBLER
#include "common.h"
/* Function parameters */
#define M $r4 // param 1: bm
#define N $r5 // param 2: bn
#define K $r6 // param 3: bk
#define ALPHA_R $f0 // param 4: alphar
#define ALPHA_I $f1 // param 5: alphai
#define A $r7 // param 6: ba
#define B $r8 // param 7: bb
#define C $r9 // param 8: bc
#define LDC $r10 // param 9: ldc
#if defined (TRMMKERNEL)
#define OFFSET $r11 // param 10: offset
#endif
#define OFF $r26
#define I $r12
#define J $r13
#define L $r14
#define TL $r15
#define A0 $r16
#define B0 $r17
#define C0 $r18
#define C1 $r19
#define C2 $r20
#define C3 $r23
#define T0 $r24
#define T1 $r25
#define T2 $r26
#define T3 $r27
#define a1 $f2
#define a2 $f3
#define a3 $f4
#define a4 $f5
#define a5 $f6
#define a6 $f7
#define a7 $f8
#define a8 $f9
#define b1 $f10
#define b2 $f11
#define b3 $f12
#define b4 $f13
#define b5 $f14
#define b6 $f15
#define b7 $f16
#define b8 $f17
#define c11 $f18
#define c12 $f19
#define c21 $f20
#define c22 $f21
#define c31 $f22
#define c32 $f23
#define c41 $f24
#define c42 $f25
/* LSX vectors */
#define U0 $vr30
#define U1 $vr31
#define U2 $vr2
#define U3 $vr3
#define U4 $vr4
#define U5 $vr5
#define U6 $vr6
#define U7 $vr7
#define U8 $vr8
#define U9 $vr9
#define U10 $vr10
#define U11 $vr11
#define U12 $vr12
#define U13 $vr13
#define U14 $vr14
#define U15 $vr15
#define D0 $vr16
#define D1 $vr17
#define D2 $vr18
#define D3 $vr19
#define D4 $vr20
#define D5 $vr21
#define D6 $vr22
#define D7 $vr23
#define D8 $vr24
#define D9 $vr25
#define D10 $vr26
#define D11 $vr27
#define D12 $vr28
#define D13 $vr29
#define VALPHAR $vr28
#define VALPHAI $vr29
#if defined(NN) || defined(NT) || defined(TN) || defined(TT)
#define VMADD1 VFMADD
#define VMADD2 VFMADD
#define VMADD3 VNMSUB
#define VMADD4 VFMADD
#define MADD1 MADD
#define MADD2 MADD
#define MADD3 NMSUB
#define MADD4 MADD
#endif
#if defined(NR) || defined(NC) || defined(TR) || defined(TC)
#define VMADD1 VFMADD
#define VMADD2 VFMADD
#define VMADD3 VFMADD
#define VMADD4 VNMSUB
#define MADD1 MADD
#define MADD2 MADD
#define MADD3 MADD
#define MADD4 NMSUB
#endif
#if defined(RN) || defined(RT) || defined(CN) || defined(CT)
#define VMADD1 VFMADD
#define VMADD2 VNMSUB
#define VMADD3 VFMADD
#define VMADD4 VFMADD
#define MADD1 MADD
#define MADD2 NMSUB
#define MADD3 MADD
#define MADD4 MADD
#endif
#if defined(RR) || defined(RC) || defined(CR) || defined(CC)
#define VMADD1 VFMADD
#define VMADD2 VNMSUB
#define VMADD3 VNMSUB
#define VMADD4 VNMSUB
#define MADD1 MADD
#define MADD2 NMSUB
#define MADD3 NMSUB
#define MADD4 NMSUB
#endif
PROLOGUE
addi.d $sp, $sp, -128
SDARG $r23, $sp, 0
SDARG $r24, $sp, 8
SDARG $r25, $sp, 16
SDARG $r26, $sp, 24
SDARG $r27, $sp, 32
ST $f23, $sp, 40
ST $f24, $sp, 48
ST $f25, $sp, 56
ST $f26, $sp, 64
ST $f27, $sp, 72
ST $f28, $sp, 80
ST $f29, $sp, 88
ST $f30, $sp, 96
ST $f31, $sp, 104
ST ALPHA_R,$sp, 112
ST ALPHA_I,$sp, 120
vldrepl.w VALPHAR, $sp, 112
vldrepl.w VALPHAI, $sp, 120
#if defined (TRMMKERNEL) && !defined(LEFT)
sub.d OFF, $r0, OFFSET
#else
xor OFF, OFF, OFF
#endif
slli.d LDC, LDC, 2
move J, $r0
srai.d T0, N, 2 //bn/4
beq J, T0, .L19
.L10: /* for(j=0; j<bn/4; j+=1) */
move C0, C
slli.d TL, LDC, 1
add.d C1, C0, TL
add.d C2, C1, TL
add.d C3, C2, TL
move A0, A //ptrba
#if defined(TRMMKERNEL) && defined(LEFT)
move OFF, OFFSET
#endif
move I, $r0
srai.d T0, M, 3 //bm/8
beq I, T0, .L150
.L11: /* for(i=0; i<bm/8; i+=1) */
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d T3, OFF, 0x06
add.d A0, A0, T3
slli.d T3, OFF, 0x05
add.d B0, B, T3
#endif
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF //temp
#elif defined(LEFT)
addi.d TL, OFF, 8
#else
addi.d TL, OFF, 4
#endif
#endif // #if defined(TRMMKERNEL)
vxor.v U0, U0, U0
vxor.v U1, U1, U1
vxor.v U2, U2, U2
vxor.v U3, U3, U3
vxor.v U4, U4, U4
vxor.v U5, U5, U5
vxor.v U6, U6, U6
vxor.v U7, U7, U7
vxor.v U8, U8, U8
vxor.v U9, U9, U9
vxor.v U10, U10, U10
vxor.v U11, U11, U11
vxor.v U12, U12, U12
vxor.v U13, U13, U13
vxor.v U14, U14, U14
vxor.v U15, U15, U15
move L, $r0 //cycle param k
beq L, TL, .L13
blt TL, L, .L13
.L12: /* for(k=0; k<temp; k+=1) */
vld D0, A0, 0x00 // a0ri a1ri
vld D2, B0, 0x00 // b0ri b1ri
vld D3, B0, 0x10 // b2ri b3ri
vshuf4i.w D4, D0, 0x00 //a0r
vshuf4i.w D5, D0, 0x55 //a0i
vpackev.w D6, D3, D2
vshuf4i.w D6, D6, 0xd8 //b0r b1r b2r b3r
vpackod.w D7, D3, D2
vshuf4i.w D7, D7, 0xd8 //b0i b1i b2i b3i
VMADD1 U0, D4, D6, U0 //00r 10r 20r 30r
VMADD2 U1, D5, D6, U1 //00i 10i 20i 30i
VMADD3 U0, D5, D7, U0
VMADD4 U1, D4, D7, U1
vshuf4i.w D4, D0, 0xaa //a1r
vshuf4i.w D5, D0, 0xff //a1i
VMADD1 U2, D4, D6, U2 //01r 11r 21r 31r
VMADD2 U3, D5, D6, U3 //01i 11i 21i 31i
VMADD3 U2, D5, D7, U2
VMADD4 U3, D4, D7, U3
vld D0, A0, 0x10 // a2ri a3ri
vshuf4i.w D4, D0, 0x00 //a2r
vshuf4i.w D5, D0, 0x55 //a2i
VMADD1 U4, D4, D6, U4 //02r 12r 22r 32r
VMADD2 U5, D5, D6, U5 //02i 12i 22i 32i
VMADD3 U4, D5, D7, U4
VMADD4 U5, D4, D7, U5
vshuf4i.w D4, D0, 0xaa //a3r
vshuf4i.w D5, D0, 0xff //a3i
VMADD1 U6, D4, D6, U6 //03r 13r 23r 33r
VMADD2 U7, D5, D6, U7 //03i 13i 23i 33i
VMADD3 U6, D5, D7, U6
VMADD4 U7, D4, D7, U7
vld D0, A0, 0x20 // a4ri a5ri
vshuf4i.w D4, D0, 0x00 //a4r
vshuf4i.w D5, D0, 0x55 //a4i
VMADD1 U8, D4, D6, U8 //04r 14r 24r 34r
VMADD2 U9, D5, D6, U9 //04i 14i 24i 34i
VMADD3 U8, D5, D7, U8
VMADD4 U9, D4, D7, U9
vshuf4i.w D4, D0, 0xaa //a5r
vshuf4i.w D5, D0, 0xff //a5i
VMADD1 U10, D4, D6, U10 //05r 15r 25r 35r
VMADD2 U11, D5, D6, U11 //05i 15i 25i 35i
VMADD3 U10, D5, D7, U10
VMADD4 U11, D4, D7, U11
vld D0, A0, 0x30 // a6ri a7ri
vshuf4i.w D4, D0, 0x00 //a6r
vshuf4i.w D5, D0, 0x55 //a6i
VMADD1 U12, D4, D6, U12 //06r 16r 26r 36r
VMADD2 U13, D5, D6, U13 //06i 16i 26i 36i
VMADD3 U12, D5, D7, U12
VMADD4 U13, D4, D7, U13
vshuf4i.w D4, D0, 0xaa //a5r
vshuf4i.w D5, D0, 0xff //a5i
VMADD1 U14, D4, D6, U14 //07r 17r 27r 37r
VMADD2 U15, D5, D6, U15 //07i 17i 27i 37i
VMADD3 U14, D5, D7, U14
VMADD4 U15, D4, D7, U15
addi.d A0, A0, 0x40
addi.d B0, B0, 0x20
addi.d L, L, 1
blt L, TL, .L12
.L13:
#if defined(TRMMKERNEL)
//res00 res10 res20 res30
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vld D2, C2, 0x00 //c2: 0 1 2 3
vld D3, C3, 0x00 //c3: 0 1 2 3
vand.v D4, D1, D1
vpermi.w D4, D0, 0x44 //c0:0 1, c1:0 1
vshuf4i.w D6, D4, 0xd8 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D7, D4, 0x8d //c0[1] c1[1] c0[0] c1[0]
vand.v D5, D3, D3
vpermi.w D5, D2, 0x44 //c2:0 1, c3:0 1
vshuf4i.w D8, D5, 0xd8 //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D9, D5, 0x8d //c2[1] c3[1] c2[0] c3[0]
vpermi.w D8, D6, 0x44 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D9, D7, 0x44 //c0[1] c1[1] c2[1] c3[1]
vfmul.s D8, U0, VALPHAR
vfmul.s D9, U1, VALPHAR
VNMSUB D8, U1, VALPHAI, D8
VFMADD D9, U0, VALPHAI, D9
vand.v D10, D9, D9 //c0[1] c1[1] c2[1] c3[1]
vand.v D11, D9, D9 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D10, D8, 0x44 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D10, D10, 0xd8 //c0[0] c0[1] c1[0] c1[1]
vpermi.w D11, D8, 0xee //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D11, D11, 0xd8 //c2[0] c2[1] c3[0] c3[1]
//res01 res11 res21 res31
vand.v D4, D1, D1
vpermi.w D4, D0, 0xee //c0:2 3, c1:2 3
vshuf4i.w D6, D4, 0xd8 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D7, D4, 0x8d //c0[3] c1[3] c0[2] c1[2]
vand.v D5, D3, D3
vpermi.w D5, D2, 0xee //c2:2 3, c3:2 3
vshuf4i.w D8, D5, 0xd8 //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D9, D5, 0x8d //c2[3] c3[3] c2[2] c3[2]
vpermi.w D8, D6, 0x44 //c0[2] c1[2] c2[2] c3[2]
vpermi.w D9, D7, 0x44 //c0[3] c1[3] c2[3] c3[3]
vfmul.s D8, U2, VALPHAR
vfmul.s D9, U3, VALPHAR
VNMSUB D8, U3, VALPHAI, D8
VFMADD D9, U2, VALPHAI, D9
vand.v D4, D9, D9
vpermi.w D4, D8, 0x44 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D4, D4, 0xd8 //c0[2] c0[3] c1[2] c1[3]
vand.v D2, D4, D4
vand.v D5, D9, D9
vpermi.w D5, D8, 0xee //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D5, D5, 0xd8 //c2[2] c2[3] c3[2] c3[3]
vand.v D3, D5, D5
vand.v D0, D10, D10 //c0[0] c0[1] c1[0] c1[1]
vand.v D1, D11, D11 //c2[0] c2[1] c3[0] c3[1]
vpermi.w D4, D0, 0x44 //c0: 0 1 2 3
vpermi.w D2, D0, 0xee //c1: 0 1 2 3
vpermi.w D5, D1, 0x44 //c2: 0 1 2 3
vpermi.w D3, D1, 0xee //c3: 0 1 2 3
vst D4, C0, 0x00
vst D2, C1, 0x00
vst D5, C2, 0x00
vst D3, C3, 0x00
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
addi.d C2, C2, 0x10
addi.d C3, C3, 0x10
//res02 res12 res22 res32
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vld D2, C2, 0x00 //c2: 0 1 2 3
vld D3, C3, 0x00 //c3: 0 1 2 3
vand.v D4, D1, D1
vpermi.w D4, D0, 0x44 //c0:0 1, c1:0 1
vshuf4i.w D6, D4, 0xd8 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D7, D4, 0x8d //c0[1] c1[1] c0[0] c1[0]
vand.v D5, D3, D3
vpermi.w D5, D2, 0x44 //c2:0 1, c3:0 1
vshuf4i.w D8, D5, 0xd8 //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D9, D5, 0x8d //c2[1] c3[1] c2[0] c3[0]
vpermi.w D8, D6, 0x44 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D9, D7, 0x44 //c0[1] c1[1] c2[1] c3[1]
vfmul.s D8, U4, VALPHAR
vfmul.s D9, U5, VALPHAR
VNMSUB D8, U5, VALPHAI, D8
VFMADD D9, U4, VALPHAI, D9
vand.v D10, D9, D9 //c0[1] c1[1] c2[1] c3[1]
vand.v D11, D9, D9 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D10, D8, 0x44 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D10, D10, 0xd8 //c0[0] c0[1] c1[0] c1[1]
vpermi.w D11, D8, 0xee //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D11, D11, 0xd8 //c2[0] c2[1] c3[0] c3[1]
//res03 res13 res23 res33
vand.v D4, D1, D1
vpermi.w D4, D0, 0xee //c0:2 3, c1:2 3
vshuf4i.w D6, D4, 0xd8 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D7, D4, 0x8d //c0[3] c1[3] c0[2] c1[2]
vand.v D5, D3, D3
vpermi.w D5, D2, 0xee //c2:2 3, c3:2 3
vshuf4i.w D8, D5, 0xd8 //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D9, D5, 0x8d //c2[3] c3[3] c2[2] c3[2]
vpermi.w D8, D6, 0x44 //c0[2] c1[2] c2[2] c3[2]
vpermi.w D9, D7, 0x44 //c0[3] c1[3] c2[3] c3[3]
vfmul.s D8, U6, VALPHAR
vfmul.s D9, U7, VALPHAR
VNMSUB D8, U7, VALPHAI, D8
VFMADD D9, U6, VALPHAI, D9
vand.v D4, D9, D9
vpermi.w D4, D8, 0x44 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D4, D4, 0xd8 //c0[2] c0[3] c1[2] c1[3]
vand.v D2, D4, D4
vand.v D5, D9, D9
vpermi.w D5, D8, 0xee //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D5, D5, 0xd8 //c2[2] c2[3] c3[2] c3[3]
vand.v D3, D5, D5
vand.v D0, D10, D10 //c0[0] c0[1] c1[0] c1[1]
vand.v D1, D11, D11 //c2[0] c2[1] c3[0] c3[1]
vpermi.w D4, D0, 0x44 //c0: 0 1 2 3
vpermi.w D2, D0, 0xee //c1: 0 1 2 3
vpermi.w D5, D1, 0x44 //c2: 0 1 2 3
vpermi.w D3, D1, 0xee //c3: 0 1 2 3
vst D4, C0, 0x00
vst D2, C1, 0x00
vst D5, C2, 0x00
vst D3, C3, 0x00
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
addi.d C2, C2, 0x10
addi.d C3, C3, 0x10
//res04 res14 res24 res34
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vld D2, C2, 0x00 //c2: 0 1 2 3
vld D3, C3, 0x00 //c3: 0 1 2 3
vand.v D4, D1, D1
vpermi.w D4, D0, 0x44 //c0:0 1, c1:0 1
vshuf4i.w D6, D4, 0xd8 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D7, D4, 0x8d //c0[1] c1[1] c0[0] c1[0]
vand.v D5, D3, D3
vpermi.w D5, D2, 0x44 //c2:0 1, c3:0 1
vshuf4i.w D8, D5, 0xd8 //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D9, D5, 0x8d //c2[1] c3[1] c2[0] c3[0]
vpermi.w D8, D6, 0x44 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D9, D7, 0x44 //c0[1] c1[1] c2[1] c3[1]
vfmul.s D8, U8, VALPHAR
vfmul.s D9, U9, VALPHAR
VNMSUB D8, U9, VALPHAI, D8
VFMADD D9, U8, VALPHAI, D9
vand.v D10, D9, D9 //c0[1] c1[1] c2[1] c3[1]
vand.v D11, D9, D9 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D10, D8, 0x44 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D10, D10, 0xd8 //c0[0] c0[1] c1[0] c1[1]
vpermi.w D11, D8, 0xee //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D11, D11, 0xd8 //c2[0] c2[1] c3[0] c3[1]
//res05 res15 res25 res35
vand.v D4, D1, D1
vpermi.w D4, D0, 0xee //c0:2 3, c1:2 3
vshuf4i.w D6, D4, 0xd8 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D7, D4, 0x8d //c0[3] c1[3] c0[2] c1[2]
vand.v D5, D3, D3
vpermi.w D5, D2, 0xee //c2:2 3, c3:2 3
vshuf4i.w D8, D5, 0xd8 //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D9, D5, 0x8d //c2[3] c3[3] c2[2] c3[2]
vpermi.w D8, D6, 0x44 //c0[2] c1[2] c2[2] c3[2]
vpermi.w D9, D7, 0x44 //c0[3] c1[3] c2[3] c3[3]
vfmul.s D8, U10, VALPHAR
vfmul.s D9, U11, VALPHAR
VNMSUB D8, U11, VALPHAI, D8
VFMADD D9, U10, VALPHAI, D9
vand.v D4, D9, D9
vpermi.w D4, D8, 0x44 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D4, D4, 0xd8 //c0[2] c0[3] c1[2] c1[3]
vand.v D2, D4, D4
vand.v D5, D9, D9
vpermi.w D5, D8, 0xee //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D5, D5, 0xd8 //c2[2] c2[3] c3[2] c3[3]
vand.v D3, D5, D5
vand.v D0, D10, D10 //c0[0] c0[1] c1[0] c1[1]
vand.v D1, D11, D11 //c2[0] c2[1] c3[0] c3[1]
vpermi.w D4, D0, 0x44 //c0: 0 1 2 3
vpermi.w D2, D0, 0xee //c1: 0 1 2 3
vpermi.w D5, D1, 0x44 //c2: 0 1 2 3
vpermi.w D3, D1, 0xee //c3: 0 1 2 3
vst D4, C0, 0x00
vst D2, C1, 0x00
vst D5, C2, 0x00
vst D3, C3, 0x00
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
addi.d C2, C2, 0x10
addi.d C3, C3, 0x10
//res06 res16 res26 res36
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vld D2, C2, 0x00 //c2: 0 1 2 3
vld D3, C3, 0x00 //c3: 0 1 2 3
vand.v D4, D1, D1
vpermi.w D4, D0, 0x44 //c0:0 1, c1:0 1
vshuf4i.w D6, D4, 0xd8 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D7, D4, 0x8d //c0[1] c1[1] c0[0] c1[0]
vand.v D5, D3, D3
vpermi.w D5, D2, 0x44 //c2:0 1, c3:0 1
vshuf4i.w D8, D5, 0xd8 //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D9, D5, 0x8d //c2[1] c3[1] c2[0] c3[0]
vpermi.w D8, D6, 0x44 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D9, D7, 0x44 //c0[1] c1[1] c2[1] c3[1]
vfmul.s D8, U12, VALPHAR
vfmul.s D9, U13, VALPHAR
VNMSUB D8, U13, VALPHAI, D8
VFMADD D9, U12, VALPHAI, D9
vand.v D10, D9, D9 //c0[1] c1[1] c2[1] c3[1]
vand.v D11, D9, D9 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D10, D8, 0x44 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D10, D10, 0xd8 //c0[0] c0[1] c1[0] c1[1]
vpermi.w D11, D8, 0xee //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D11, D11, 0xd8 //c2[0] c2[1] c3[0] c3[1]
//res07 res17 res27 res37
vand.v D4, D1, D1
vpermi.w D4, D0, 0xee //c0:2 3, c1:2 3
vshuf4i.w D6, D4, 0xd8 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D7, D4, 0x8d //c0[3] c1[3] c0[2] c1[2]
vand.v D5, D3, D3
vpermi.w D5, D2, 0xee //c2:2 3, c3:2 3
vshuf4i.w D8, D5, 0xd8 //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D9, D5, 0x8d //c2[3] c3[3] c2[2] c3[2]
vpermi.w D8, D6, 0x44 //c0[2] c1[2] c2[2] c3[2]
vpermi.w D9, D7, 0x44 //c0[3] c1[3] c2[3] c3[3]
vfmul.s D8, U14, VALPHAR
vfmul.s D9, U15, VALPHAR
VNMSUB D8, U15, VALPHAI, D8
VFMADD D9, U14, VALPHAI, D9
vand.v D4, D9, D9
vpermi.w D4, D8, 0x44 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D4, D4, 0xd8 //c0[2] c0[3] c1[2] c1[3]
vand.v D2, D4, D4
vand.v D5, D9, D9
vpermi.w D5, D8, 0xee //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D5, D5, 0xd8 //c2[2] c2[3] c3[2] c3[3]
vand.v D3, D5, D5
vand.v D0, D10, D10 //c0[0] c0[1] c1[0] c1[1]
vand.v D1, D11, D11 //c2[0] c2[1] c3[0] c3[1]
vpermi.w D4, D0, 0x44 //c0: 0 1 2 3
vpermi.w D2, D0, 0xee //c1: 0 1 2 3
vpermi.w D5, D1, 0x44 //c2: 0 1 2 3
vpermi.w D3, D1, 0xee //c3: 0 1 2 3
vst D4, C0, 0x00
vst D2, C1, 0x00
vst D5, C2, 0x00
vst D3, C3, 0x00
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
addi.d C2, C2, 0x10
addi.d C3, C3, 0x10
#else
//res00 res10 res20 res30
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vld D2, C2, 0x00 //c2: 0 1 2 3
vld D3, C3, 0x00 //c3: 0 1 2 3
vand.v D4, D1, D1
vpermi.w D4, D0, 0x44 //c0:0 1, c1:0 1
vshuf4i.w D6, D4, 0xd8 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D7, D4, 0x8d //c0[1] c1[1] c0[0] c1[0]
vand.v D5, D3, D3
vpermi.w D5, D2, 0x44 //c2:0 1, c3:0 1
vshuf4i.w D8, D5, 0xd8 //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D9, D5, 0x8d //c2[1] c3[1] c2[0] c3[0]
vpermi.w D8, D6, 0x44 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D9, D7, 0x44 //c0[1] c1[1] c2[1] c3[1]
VFMADD D8, U0, VALPHAR, D8
VFMADD D9, U1, VALPHAR, D9
VNMSUB D8, U1, VALPHAI, D8
VFMADD D9, U0, VALPHAI, D9
vand.v D10, D9, D9 //c0[1] c1[1] c2[1] c3[1]
vand.v D11, D9, D9 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D10, D8, 0x44 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D10, D10, 0xd8 //c0[0] c0[1] c1[0] c1[1]
vpermi.w D11, D8, 0xee //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D11, D11, 0xd8 //c2[0] c2[1] c3[0] c3[1]
//res01 res11 res21 res31
vand.v D4, D1, D1
vpermi.w D4, D0, 0xee //c0:2 3, c1:2 3
vshuf4i.w D6, D4, 0xd8 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D7, D4, 0x8d //c0[3] c1[3] c0[2] c1[2]
vand.v D5, D3, D3
vpermi.w D5, D2, 0xee //c2:2 3, c3:2 3
vshuf4i.w D8, D5, 0xd8 //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D9, D5, 0x8d //c2[3] c3[3] c2[2] c3[2]
vpermi.w D8, D6, 0x44 //c0[2] c1[2] c2[2] c3[2]
vpermi.w D9, D7, 0x44 //c0[3] c1[3] c2[3] c3[3]
VFMADD D8, U2, VALPHAR, D8
VFMADD D9, U3, VALPHAR, D9
VNMSUB D8, U3, VALPHAI, D8
VFMADD D9, U2, VALPHAI, D9
vand.v D4, D9, D9
vpermi.w D4, D8, 0x44 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D4, D4, 0xd8 //c0[2] c0[3] c1[2] c1[3]
vand.v D2, D4, D4
vand.v D5, D9, D9
vpermi.w D5, D8, 0xee //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D5, D5, 0xd8 //c2[2] c2[3] c3[2] c3[3]
vand.v D3, D5, D5
vand.v D0, D10, D10 //c0[0] c0[1] c1[0] c1[1]
vand.v D1, D11, D11 //c2[0] c2[1] c3[0] c3[1]
vpermi.w D4, D0, 0x44 //c0: 0 1 2 3
vpermi.w D2, D0, 0xee //c1: 0 1 2 3
vpermi.w D5, D1, 0x44 //c2: 0 1 2 3
vpermi.w D3, D1, 0xee //c3: 0 1 2 3
vst D4, C0, 0x00
vst D2, C1, 0x00
vst D5, C2, 0x00
vst D3, C3, 0x00
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
addi.d C2, C2, 0x10
addi.d C3, C3, 0x10
//res02 res12 res22 res32
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vld D2, C2, 0x00 //c2: 0 1 2 3
vld D3, C3, 0x00 //c3: 0 1 2 3
vand.v D4, D1, D1
vpermi.w D4, D0, 0x44 //c0:0 1, c1:0 1
vshuf4i.w D6, D4, 0xd8 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D7, D4, 0x8d //c0[1] c1[1] c0[0] c1[0]
vand.v D5, D3, D3
vpermi.w D5, D2, 0x44 //c2:0 1, c3:0 1
vshuf4i.w D8, D5, 0xd8 //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D9, D5, 0x8d //c2[1] c3[1] c2[0] c3[0]
vpermi.w D8, D6, 0x44 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D9, D7, 0x44 //c0[1] c1[1] c2[1] c3[1]
VFMADD D8, U4, VALPHAR, D8
VFMADD D9, U5, VALPHAR, D9
VNMSUB D8, U5, VALPHAI, D8
VFMADD D9, U4, VALPHAI, D9
vand.v D10, D9, D9 //c0[1] c1[1] c2[1] c3[1]
vand.v D11, D9, D9 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D10, D8, 0x44 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D10, D10, 0xd8 //c0[0] c0[1] c1[0] c1[1]
vpermi.w D11, D8, 0xee //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D11, D11, 0xd8 //c2[0] c2[1] c3[0] c3[1]
//res03 res13 res23 res33
vand.v D4, D1, D1
vpermi.w D4, D0, 0xee //c0:2 3, c1:2 3
vshuf4i.w D6, D4, 0xd8 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D7, D4, 0x8d //c0[3] c1[3] c0[2] c1[2]
vand.v D5, D3, D3
vpermi.w D5, D2, 0xee //c2:2 3, c3:2 3
vshuf4i.w D8, D5, 0xd8 //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D9, D5, 0x8d //c2[3] c3[3] c2[2] c3[2]
vpermi.w D8, D6, 0x44 //c0[2] c1[2] c2[2] c3[2]
vpermi.w D9, D7, 0x44 //c0[3] c1[3] c2[3] c3[3]
VFMADD D8, U6, VALPHAR, D8
VFMADD D9, U7, VALPHAR, D9
VNMSUB D8, U7, VALPHAI, D8
VFMADD D9, U6, VALPHAI, D9
vand.v D4, D9, D9
vpermi.w D4, D8, 0x44 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D4, D4, 0xd8 //c0[2] c0[3] c1[2] c1[3]
vand.v D2, D4, D4
vand.v D5, D9, D9
vpermi.w D5, D8, 0xee //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D5, D5, 0xd8 //c2[2] c2[3] c3[2] c3[3]
vand.v D3, D5, D5
vand.v D0, D10, D10 //c0[0] c0[1] c1[0] c1[1]
vand.v D1, D11, D11 //c2[0] c2[1] c3[0] c3[1]
vpermi.w D4, D0, 0x44 //c0: 0 1 2 3
vpermi.w D2, D0, 0xee //c1: 0 1 2 3
vpermi.w D5, D1, 0x44 //c2: 0 1 2 3
vpermi.w D3, D1, 0xee //c3: 0 1 2 3
vst D4, C0, 0x00
vst D2, C1, 0x00
vst D5, C2, 0x00
vst D3, C3, 0x00
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
addi.d C2, C2, 0x10
addi.d C3, C3, 0x10
//res04 res14 res24 res34
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vld D2, C2, 0x00 //c2: 0 1 2 3
vld D3, C3, 0x00 //c3: 0 1 2 3
vand.v D4, D1, D1
vpermi.w D4, D0, 0x44 //c0:0 1, c1:0 1
vshuf4i.w D6, D4, 0xd8 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D7, D4, 0x8d //c0[1] c1[1] c0[0] c1[0]
vand.v D5, D3, D3
vpermi.w D5, D2, 0x44 //c2:0 1, c3:0 1
vshuf4i.w D8, D5, 0xd8 //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D9, D5, 0x8d //c2[1] c3[1] c2[0] c3[0]
vpermi.w D8, D6, 0x44 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D9, D7, 0x44 //c0[1] c1[1] c2[1] c3[1]
VFMADD D8, U8, VALPHAR, D8
VFMADD D9, U9, VALPHAR, D9
VNMSUB D8, U9, VALPHAI, D8
VFMADD D9, U8, VALPHAI, D9
vand.v D10, D9, D9 //c0[1] c1[1] c2[1] c3[1]
vand.v D11, D9, D9 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D10, D8, 0x44 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D10, D10, 0xd8 //c0[0] c0[1] c1[0] c1[1]
vpermi.w D11, D8, 0xee //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D11, D11, 0xd8 //c2[0] c2[1] c3[0] c3[1]
//res05 res15 res25 res35
vand.v D4, D1, D1
vpermi.w D4, D0, 0xee //c0:2 3, c1:2 3
vshuf4i.w D6, D4, 0xd8 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D7, D4, 0x8d //c0[3] c1[3] c0[2] c1[2]
vand.v D5, D3, D3
vpermi.w D5, D2, 0xee //c2:2 3, c3:2 3
vshuf4i.w D8, D5, 0xd8 //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D9, D5, 0x8d //c2[3] c3[3] c2[2] c3[2]
vpermi.w D8, D6, 0x44 //c0[2] c1[2] c2[2] c3[2]
vpermi.w D9, D7, 0x44 //c0[3] c1[3] c2[3] c3[3]
VFMADD D8, U10, VALPHAR, D8
VFMADD D9, U11, VALPHAR, D9
VNMSUB D8, U11, VALPHAI, D8
VFMADD D9, U10, VALPHAI, D9
vand.v D4, D9, D9
vpermi.w D4, D8, 0x44 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D4, D4, 0xd8 //c0[2] c0[3] c1[2] c1[3]
vand.v D2, D4, D4
vand.v D5, D9, D9
vpermi.w D5, D8, 0xee //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D5, D5, 0xd8 //c2[2] c2[3] c3[2] c3[3]
vand.v D3, D5, D5
vand.v D0, D10, D10 //c0[0] c0[1] c1[0] c1[1]
vand.v D1, D11, D11 //c2[0] c2[1] c3[0] c3[1]
vpermi.w D4, D0, 0x44 //c0: 0 1 2 3
vpermi.w D2, D0, 0xee //c1: 0 1 2 3
vpermi.w D5, D1, 0x44 //c2: 0 1 2 3
vpermi.w D3, D1, 0xee //c3: 0 1 2 3
vst D4, C0, 0x00
vst D2, C1, 0x00
vst D5, C2, 0x00
vst D3, C3, 0x00
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
addi.d C2, C2, 0x10
addi.d C3, C3, 0x10
//res06 res16 res26 res36
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vld D2, C2, 0x00 //c2: 0 1 2 3
vld D3, C3, 0x00 //c3: 0 1 2 3
vand.v D4, D1, D1
vpermi.w D4, D0, 0x44 //c0:0 1, c1:0 1
vshuf4i.w D6, D4, 0xd8 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D7, D4, 0x8d //c0[1] c1[1] c0[0] c1[0]
vand.v D5, D3, D3
vpermi.w D5, D2, 0x44 //c2:0 1, c3:0 1
vshuf4i.w D8, D5, 0xd8 //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D9, D5, 0x8d //c2[1] c3[1] c2[0] c3[0]
vpermi.w D8, D6, 0x44 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D9, D7, 0x44 //c0[1] c1[1] c2[1] c3[1]
VFMADD D8, U12, VALPHAR, D8
VFMADD D9, U13, VALPHAR, D9
VNMSUB D8, U13, VALPHAI, D8
VFMADD D9, U12, VALPHAI, D9
vand.v D10, D9, D9 //c0[1] c1[1] c2[1] c3[1]
vand.v D11, D9, D9 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D10, D8, 0x44 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D10, D10, 0xd8 //c0[0] c0[1] c1[0] c1[1]
vpermi.w D11, D8, 0xee //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D11, D11, 0xd8 //c2[0] c2[1] c3[0] c3[1]
//res07 res17 res27 res37
vand.v D4, D1, D1
vpermi.w D4, D0, 0xee //c0:2 3, c1:2 3
vshuf4i.w D6, D4, 0xd8 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D7, D4, 0x8d //c0[3] c1[3] c0[2] c1[2]
vand.v D5, D3, D3
vpermi.w D5, D2, 0xee //c2:2 3, c3:2 3
vshuf4i.w D8, D5, 0xd8 //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D9, D5, 0x8d //c2[3] c3[3] c2[2] c3[2]
vpermi.w D8, D6, 0x44 //c0[2] c1[2] c2[2] c3[2]
vpermi.w D9, D7, 0x44 //c0[3] c1[3] c2[3] c3[3]
VFMADD D8, U14, VALPHAR, D8
VFMADD D9, U15, VALPHAR, D9
VNMSUB D8, U15, VALPHAI, D8
VFMADD D9, U14, VALPHAI, D9
vand.v D4, D9, D9
vpermi.w D4, D8, 0x44 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D4, D4, 0xd8 //c0[2] c0[3] c1[2] c1[3]
vand.v D2, D4, D4
vand.v D5, D9, D9
vpermi.w D5, D8, 0xee //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D5, D5, 0xd8 //c2[2] c2[3] c3[2] c3[3]
vand.v D3, D5, D5
vand.v D0, D10, D10 //c0[0] c0[1] c1[0] c1[1]
vand.v D1, D11, D11 //c2[0] c2[1] c3[0] c3[1]
vpermi.w D4, D0, 0x44 //c0: 0 1 2 3
vpermi.w D2, D0, 0xee //c1: 0 1 2 3
vpermi.w D5, D1, 0x44 //c2: 0 1 2 3
vpermi.w D3, D1, 0xee //c3: 0 1 2 3
vst D4, C0, 0x00
vst D2, C1, 0x00
vst D5, C2, 0x00
vst D3, C3, 0x00
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
addi.d C2, C2, 0x10
addi.d C3, C3, 0x10
#endif
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -8
#else
addi.d TL, TL, -4
#endif
slli.d T3, TL, 0x06
add.d A0, A0, T3
slli.d T3, TL, 0x05
add.d B0, B0, T3
#endif
#ifdef LEFT
addi.d OFF, OFF, 8
#endif
#endif // #if defined(TRMMKERNEL)
addi.d I, I, 1
blt I, T0, .L11
.L150:
move I, $r0
andi T0, M, 4
beq I, T0, .L18
.L15: /* if (bm & 4) */
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d T3, OFF, 0x05
add.d A0, A0, T3
slli.d T3, OFF, 0x05
add.d B0, B, T3
#endif
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF
#elif defined(LEFT)
addi.d TL, OFF, 4
#else
addi.d TL, OFF, 4
#endif
#endif // #if defined(TRMMKERNEL)
vxor.v U0, U0, U0
vxor.v U1, U1, U1
vxor.v U2, U2, U2
vxor.v U3, U3, U3
vxor.v U4, U4, U4
vxor.v U5, U5, U5
vxor.v U6, U6, U6
vxor.v U7, U7, U7
move L, $r0 //cycle param k
beq L, TL, .L17
blt TL, L, .L17
.L16: /* for (k=0; k<temp; k++) */
vld D0, A0, 0x00 // a0ri a1ri
vld D2, B0, 0x00 // b0ri b1ri
vld D3, B0, 0x10 // b2ri b3ri
vshuf4i.w D4, D0, 0x00 //a0r
vshuf4i.w D5, D0, 0x55 //a0i
vpackev.w D6, D3, D2
vshuf4i.w D6, D6, 0xd8 //b0r b1r b2r b3r
vpackod.w D7, D3, D2
vshuf4i.w D7, D7, 0xd8 //b0i b1i b2i b3i
VMADD1 U0, D4, D6, U0 //00r 10r 20r 30r
VMADD2 U1, D5, D6, U1 //00i 10i 20i 30i
VMADD3 U0, D5, D7, U0
VMADD4 U1, D4, D7, U1
vshuf4i.w D4, D0, 0xaa //a1r
vshuf4i.w D5, D0, 0xff //a1i
VMADD1 U2, D4, D6, U2 //01r 11r 21r 31r
VMADD2 U3, D5, D6, U3 //01i 11i 21i 31i
VMADD3 U2, D5, D7, U2
VMADD4 U3, D4, D7, U3
vld D0, A0, 0x10 // a2ri a3ri
vshuf4i.w D4, D0, 0x00 //a2r
vshuf4i.w D5, D0, 0x55 //a2i
VMADD1 U4, D4, D6, U4 //02r 12r 22r 32r
VMADD2 U5, D5, D6, U5 //02i 12i 22i 32i
VMADD3 U4, D5, D7, U4
VMADD4 U5, D4, D7, U5
vshuf4i.w D4, D0, 0xaa //a3r
vshuf4i.w D5, D0, 0xff //a3i
VMADD1 U6, D4, D6, U6 //03r 13r 23r 33r
VMADD2 U7, D5, D6, U7 //03i 13i 23i 33i
VMADD3 U6, D5, D7, U6
VMADD4 U7, D4, D7, U7
addi.d A0, A0, 0x20
addi.d B0, B0, 0x20
addi.d L, L, 1
blt L, TL, .L16
.L17:
#if defined(TRMMKERNEL)
//res00 res10 res20 res30
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vld D2, C2, 0x00 //c2: 0 1 2 3
vld D3, C3, 0x00 //c3: 0 1 2 3
vand.v D4, D1, D1
vpermi.w D4, D0, 0x44 //c0:0 1, c1:0 1
vshuf4i.w D6, D4, 0xd8 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D7, D4, 0x8d //c0[1] c1[1] c0[0] c1[0]
vand.v D5, D3, D3
vpermi.w D5, D2, 0x44 //c2:0 1, c3:0 1
vshuf4i.w D8, D5, 0xd8 //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D9, D5, 0x8d //c2[1] c3[1] c2[0] c3[0]
vpermi.w D8, D6, 0x44 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D9, D7, 0x44 //c0[1] c1[1] c2[1] c3[1]
vfmul.s D8, U0, VALPHAR
vfmul.s D9, U1, VALPHAR
VNMSUB D8, U1, VALPHAI, D8
VFMADD D9, U0, VALPHAI, D9
vand.v D10, D9, D9 //c0[1] c1[1] c2[1] c3[1]
vand.v D11, D9, D9 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D10, D8, 0x44 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D10, D10, 0xd8 //c0[0] c0[1] c1[0] c1[1]
vpermi.w D11, D8, 0xee //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D11, D11, 0xd8 //c2[0] c2[1] c3[0] c3[1]
//res01 res11 res21 res31
vand.v D4, D1, D1
vpermi.w D4, D0, 0xee //c0:2 3, c1:2 3
vshuf4i.w D6, D4, 0xd8 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D7, D4, 0x8d //c0[3] c1[3] c0[2] c1[2]
vand.v D5, D3, D3
vpermi.w D5, D2, 0xee //c2:2 3, c3:2 3
vshuf4i.w D8, D5, 0xd8 //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D9, D5, 0x8d //c2[3] c3[3] c2[2] c3[2]
vpermi.w D8, D6, 0x44 //c0[2] c1[2] c2[2] c3[2]
vpermi.w D9, D7, 0x44 //c0[3] c1[3] c2[3] c3[3]
vfmul.s D8, U2, VALPHAR
vfmul.s D9, U3, VALPHAR
VNMSUB D8, U3, VALPHAI, D8
VFMADD D9, U2, VALPHAI, D9
vand.v D4, D9, D9
vpermi.w D4, D8, 0x44 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D4, D4, 0xd8 //c0[2] c0[3] c1[2] c1[3]
vand.v D2, D4, D4
vand.v D5, D9, D9
vpermi.w D5, D8, 0xee //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D5, D5, 0xd8 //c2[2] c2[3] c3[2] c3[3]
vand.v D3, D5, D5
vand.v D0, D10, D10 //c0[0] c0[1] c1[0] c1[1]
vand.v D1, D11, D11 //c2[0] c2[1] c3[0] c3[1]
vpermi.w D4, D0, 0x44 //c0: 0 1 2 3
vpermi.w D2, D0, 0xee //c1: 0 1 2 3
vpermi.w D5, D1, 0x44 //c2: 0 1 2 3
vpermi.w D3, D1, 0xee //c3: 0 1 2 3
vst D4, C0, 0x00
vst D2, C1, 0x00
vst D5, C2, 0x00
vst D3, C3, 0x00
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
addi.d C2, C2, 0x10
addi.d C3, C3, 0x10
//res02 res12 res22 res32
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vld D2, C2, 0x00 //c2: 0 1 2 3
vld D3, C3, 0x00 //c3: 0 1 2 3
vand.v D4, D1, D1
vpermi.w D4, D0, 0x44 //c0:0 1, c1:0 1
vshuf4i.w D6, D4, 0xd8 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D7, D4, 0x8d //c0[1] c1[1] c0[0] c1[0]
vand.v D5, D3, D3
vpermi.w D5, D2, 0x44 //c2:0 1, c3:0 1
vshuf4i.w D8, D5, 0xd8 //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D9, D5, 0x8d //c2[1] c3[1] c2[0] c3[0]
vpermi.w D8, D6, 0x44 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D9, D7, 0x44 //c0[1] c1[1] c2[1] c3[1]
vfmul.s D8, U4, VALPHAR
vfmul.s D9, U5, VALPHAR
VNMSUB D8, U5, VALPHAI, D8
VFMADD D9, U4, VALPHAI, D9
vand.v D10, D9, D9 //c0[1] c1[1] c2[1] c3[1]
vand.v D11, D9, D9 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D10, D8, 0x44 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D10, D10, 0xd8 //c0[0] c0[1] c1[0] c1[1]
vpermi.w D11, D8, 0xee //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D11, D11, 0xd8 //c2[0] c2[1] c3[0] c3[1]
//res03 res13 res23 res33
vand.v D4, D1, D1
vpermi.w D4, D0, 0xee //c0:2 3, c1:2 3
vshuf4i.w D6, D4, 0xd8 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D7, D4, 0x8d //c0[3] c1[3] c0[2] c1[2]
vand.v D5, D3, D3
vpermi.w D5, D2, 0xee //c2:2 3, c3:2 3
vshuf4i.w D8, D5, 0xd8 //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D9, D5, 0x8d //c2[3] c3[3] c2[2] c3[2]
vpermi.w D8, D6, 0x44 //c0[2] c1[2] c2[2] c3[2]
vpermi.w D9, D7, 0x44 //c0[3] c1[3] c2[3] c3[3]
vfmul.s D8, U6, VALPHAR
vfmul.s D9, U7, VALPHAR
VNMSUB D8, U7, VALPHAI, D8
VFMADD D9, U6, VALPHAI, D9
vand.v D4, D9, D9
vpermi.w D4, D8, 0x44 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D4, D4, 0xd8 //c0[2] c0[3] c1[2] c1[3]
vand.v D2, D4, D4
vand.v D5, D9, D9
vpermi.w D5, D8, 0xee //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D5, D5, 0xd8 //c2[2] c2[3] c3[2] c3[3]
vand.v D3, D5, D5
vand.v D0, D10, D10 //c0[0] c0[1] c1[0] c1[1]
vand.v D1, D11, D11 //c2[0] c2[1] c3[0] c3[1]
vpermi.w D4, D0, 0x44 //c0: 0 1 2 3
vpermi.w D2, D0, 0xee //c1: 0 1 2 3
vpermi.w D5, D1, 0x44 //c2: 0 1 2 3
vpermi.w D3, D1, 0xee //c3: 0 1 2 3
vst D4, C0, 0x00
vst D2, C1, 0x00
vst D5, C2, 0x00
vst D3, C3, 0x00
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
addi.d C2, C2, 0x10
addi.d C3, C3, 0x10
#else
//res00 res10 res20 res30
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vld D2, C2, 0x00 //c2: 0 1 2 3
vld D3, C3, 0x00 //c3: 0 1 2 3
vand.v D4, D1, D1
vpermi.w D4, D0, 0x44 //c0:0 1, c1:0 1
vshuf4i.w D6, D4, 0xd8 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D7, D4, 0x8d //c0[1] c1[1] c0[0] c1[0]
vand.v D5, D3, D3
vpermi.w D5, D2, 0x44 //c2:0 1, c3:0 1
vshuf4i.w D8, D5, 0xd8 //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D9, D5, 0x8d //c2[1] c3[1] c2[0] c3[0]
vpermi.w D8, D6, 0x44 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D9, D7, 0x44 //c0[1] c1[1] c2[1] c3[1]
VFMADD D8, U0, VALPHAR, D8
VFMADD D9, U1, VALPHAR, D9
VNMSUB D8, U1, VALPHAI, D8
VFMADD D9, U0, VALPHAI, D9
vst VALPHAR, C0, 0x00
vst VALPHAI, C1, 0x00
vand.v D10, D9, D9 //c0[1] c1[1] c2[1] c3[1]
vand.v D11, D9, D9 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D10, D8, 0x44 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D10, D10, 0xd8 //c0[0] c0[1] c1[0] c1[1]
vpermi.w D11, D8, 0xee //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D11, D11, 0xd8 //c2[0] c2[1] c3[0] c3[1]
//res01 res11 res21 res31
vand.v D4, D1, D1
vpermi.w D4, D0, 0xee //c0:2 3, c1:2 3
vshuf4i.w D6, D4, 0xd8 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D7, D4, 0x8d //c0[3] c1[3] c0[2] c1[2]
vand.v D5, D3, D3
vpermi.w D5, D2, 0xee //c2:2 3, c3:2 3
vshuf4i.w D8, D5, 0xd8 //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D9, D5, 0x8d //c2[3] c3[3] c2[2] c3[2]
vpermi.w D8, D6, 0x44 //c0[2] c1[2] c2[2] c3[2]
vpermi.w D9, D7, 0x44 //c0[3] c1[3] c2[3] c3[3]
VFMADD D8, U2, VALPHAR, D8
VFMADD D9, U3, VALPHAR, D9
VNMSUB D8, U3, VALPHAI, D8
VFMADD D9, U2, VALPHAI, D9
vand.v D4, D9, D9
vpermi.w D4, D8, 0x44 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D4, D4, 0xd8 //c0[2] c0[3] c1[2] c1[3]
vand.v D2, D4, D4
vand.v D5, D9, D9
vpermi.w D5, D8, 0xee //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D5, D5, 0xd8 //c2[2] c2[3] c3[2] c3[3]
vand.v D3, D5, D5
vand.v D0, D10, D10 //c0[0] c0[1] c1[0] c1[1]
vand.v D1, D11, D11 //c2[0] c2[1] c3[0] c3[1]
vpermi.w D4, D0, 0x44 //c0: 0 1 2 3
vpermi.w D2, D0, 0xee //c1: 0 1 2 3
vpermi.w D5, D1, 0x44 //c2: 0 1 2 3
vpermi.w D3, D1, 0xee //c3: 0 1 2 3
// vst VALPHAR,C0, 0x00
// LD $f15, C0, 0x00
// LD $f15, C0, 0x04
// LD $f15, C0, 0x08
// LD $f15, C0, 0x0c
// vst VALPHAI,C0, 0x00
// LD $f15, C0, 0x00
// LD $f15, C0, 0x04
// LD $f15, C0, 0x08
// LD $f15, C0, 0x0c
vst D4, C0, 0x00
vst D2, C1, 0x00
vst D5, C2, 0x00
vst D3, C3, 0x00
// LD $f15, C0, 0x00
// LD $f15, C0, 0x04
// LD $f15, C0, 0x08
// LD $f15, C0, 0x0c
// LD $f15, C1, 0x00
// LD $f15, C1, 0x04
// LD $f15, C1, 0x08
// LD $f15, C1, 0x0c
// LD $f15, C2, 0x00
// LD $f15, C2, 0x04
// LD $f15, C2, 0x08
// LD $f15, C2, 0x0c
// LD $f15, C3, 0x00
// LD $f15, C3, 0x04
// LD $f15, C3, 0x08
// LD $f15, C3, 0x0c
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
addi.d C2, C2, 0x10
addi.d C3, C3, 0x10
//res02 res12 res22 res32
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vld D2, C2, 0x00 //c2: 0 1 2 3
vld D3, C3, 0x00 //c3: 0 1 2 3
vand.v D4, D1, D1
vpermi.w D4, D0, 0x44 //c0:0 1, c1:0 1
vshuf4i.w D6, D4, 0xd8 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D7, D4, 0x8d //c0[1] c1[1] c0[0] c1[0]
vand.v D5, D3, D3
vpermi.w D5, D2, 0x44 //c2:0 1, c3:0 1
vshuf4i.w D8, D5, 0xd8 //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D9, D5, 0x8d //c2[1] c3[1] c2[0] c3[0]
vpermi.w D8, D6, 0x44 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D9, D7, 0x44 //c0[1] c1[1] c2[1] c3[1]
VFMADD D8, U4, VALPHAR, D8
VFMADD D9, U5, VALPHAR, D9
VNMSUB D8, U5, VALPHAI, D8
VFMADD D9, U4, VALPHAI, D9
vand.v D10, D9, D9 //c0[1] c1[1] c2[1] c3[1]
vand.v D11, D9, D9 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D10, D8, 0x44 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D10, D10, 0xd8 //c0[0] c0[1] c1[0] c1[1]
vpermi.w D11, D8, 0xee //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D11, D11, 0xd8 //c2[0] c2[1] c3[0] c3[1]
//res03 res13 res23 res33
vand.v D4, D1, D1
vpermi.w D4, D0, 0xee //c0:2 3, c1:2 3
vshuf4i.w D6, D4, 0xd8 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D7, D4, 0x8d //c0[3] c1[3] c0[2] c1[2]
vand.v D5, D3, D3
vpermi.w D5, D2, 0xee //c2:2 3, c3:2 3
vshuf4i.w D8, D5, 0xd8 //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D9, D5, 0x8d //c2[3] c3[3] c2[2] c3[2]
vpermi.w D8, D6, 0x44 //c0[2] c1[2] c2[2] c3[2]
vpermi.w D9, D7, 0x44 //c0[3] c1[3] c2[3] c3[3]
VFMADD D8, U6, VALPHAR, D8
VFMADD D9, U7, VALPHAR, D9
VNMSUB D8, U7, VALPHAI, D8
VFMADD D9, U6, VALPHAI, D9
vand.v D4, D9, D9
vpermi.w D4, D8, 0x44 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D4, D4, 0xd8 //c0[2] c0[3] c1[2] c1[3]
vand.v D2, D4, D4
vand.v D5, D9, D9
vpermi.w D5, D8, 0xee //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D5, D5, 0xd8 //c2[2] c2[3] c3[2] c3[3]
vand.v D3, D5, D5
vand.v D0, D10, D10 //c0[0] c0[1] c1[0] c1[1]
vand.v D1, D11, D11 //c2[0] c2[1] c3[0] c3[1]
vpermi.w D4, D0, 0x44 //c0: 0 1 2 3
vpermi.w D2, D0, 0xee //c1: 0 1 2 3
vpermi.w D5, D1, 0x44 //c2: 0 1 2 3
vpermi.w D3, D1, 0xee //c3: 0 1 2 3
vst D4, C0, 0x00
vst D2, C1, 0x00
vst D5, C2, 0x00
vst D3, C3, 0x00
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
addi.d C2, C2, 0x10
addi.d C3, C3, 0x10
#endif
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -4
#else
addi.d TL, TL, -4
#endif
slli.d T3, TL, 0x05
add.d A0, A0, T3
add.d B0, B0, T3
#endif
#ifdef LEFT
addi.d OFF, OFF, 4
#endif
#endif // #if defined(TRMMKERNEL)
.L18: /* if (bm & 2) */
move I, $r0
andi T0, M, 2
beq I, T0, .L183
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d T3, OFF, 0x04
add.d A0, A0, T3
slli.d T3, OFF, 0x05
add.d B0, B, T3
#endif
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF
#elif defined(LEFT)
addi.d TL, OFF, 2
#else
addi.d TL, OFF, 4
#endif
#endif // #if defined(TRMMKERNEL)
vxor.v U0, U0, U0
vxor.v U1, U1, U1
vxor.v U2, U2, U2
vxor.v U3, U3, U3
move L, $r0 //cycle param k
beq L, TL, .L182
blt TL, L, .L182
.L181: /* for (k=0; k<temp; k++) */
vld D0, A0, 0x00 // a0ri a1ri
vld D2, B0, 0x00 // b0ri b1ri
vld D3, B0, 0x10 // b2ri b3ri
vshuf4i.w D4, D0, 0x00 //a0r
vshuf4i.w D5, D0, 0x55 //a0i
vpackev.w D6, D3, D2
vshuf4i.w D6, D6, 0xd8 //b0r b1r b2r b3r
vpackod.w D7, D3, D2
vshuf4i.w D7, D7, 0xd8 //b0i b1i b2i b3i
VMADD1 U0, D4, D6, U0 //00r 10r 20r 30r
VMADD2 U1, D5, D6, U1 //00i 10i 20i 30i
VMADD3 U0, D5, D7, U0
VMADD4 U1, D4, D7, U1
vshuf4i.w D4, D0, 0xaa //a1r
vshuf4i.w D5, D0, 0xff //a1i
VMADD1 U2, D4, D6, U2 //01r 11r 21r 31r
VMADD2 U3, D5, D6, U3 //01i 11i 21i 31i
VMADD3 U2, D5, D7, U2
VMADD4 U3, D4, D7, U3
addi.d A0, A0, 0x10
addi.d B0, B0, 0x20
addi.d L, L, 1
blt L, TL, .L181
.L182:
#if defined(TRMMKERNEL)
//res00 res10 res20 res30
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vld D2, C2, 0x00 //c2: 0 1 2 3
vld D3, C3, 0x00 //c3: 0 1 2 3
vand.v D4, D1, D1
vpermi.w D4, D0, 0x44 //c0:0 1, c1:0 1
vshuf4i.w D6, D4, 0xd8 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D7, D4, 0x8d //c0[1] c1[1] c0[0] c1[0]
vand.v D5, D3, D3
vpermi.w D5, D2, 0x44 //c2:0 1, c3:0 1
vshuf4i.w D8, D5, 0xd8 //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D9, D5, 0x8d //c2[1] c3[1] c2[0] c3[0]
vpermi.w D8, D6, 0x44 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D9, D7, 0x44 //c0[1] c1[1] c2[1] c3[1]
vfmul.s D8, U0, VALPHAR
vfmul.s D9, U1, VALPHAR
VNMSUB D8, U1, VALPHAI, D8
VFMADD D9, U0, VALPHAI, D9
vand.v D10, D9, D9 //c0[1] c1[1] c2[1] c3[1]
vand.v D11, D9, D9 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D10, D8, 0x44 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D10, D10, 0xd8 //c0[0] c0[1] c1[0] c1[1]
vpermi.w D11, D8, 0xee //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D11, D11, 0xd8 //c2[0] c2[1] c3[0] c3[1]
//res01 res11 res21 res31
vand.v D4, D1, D1
vpermi.w D4, D0, 0xee //c0:2 3, c1:2 3
vshuf4i.w D6, D4, 0xd8 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D7, D4, 0x8d //c0[3] c1[3] c0[2] c1[2]
vand.v D5, D3, D3
vpermi.w D5, D2, 0xee //c2:2 3, c3:2 3
vshuf4i.w D8, D5, 0xd8 //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D9, D5, 0x8d //c2[3] c3[3] c2[2] c3[2]
vpermi.w D8, D6, 0x44 //c0[2] c1[2] c2[2] c3[2]
vpermi.w D9, D7, 0x44 //c0[3] c1[3] c2[3] c3[3]
vfmul.s D8, U2, VALPHAR
vfmul.s D9, U3, VALPHAR
VNMSUB D8, U3, VALPHAI, D8
VFMADD D9, U2, VALPHAI, D9
vand.v D4, D9, D9
vpermi.w D4, D8, 0x44 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D4, D4, 0xd8 //c0[2] c0[3] c1[2] c1[3]
vand.v D2, D4, D4
vand.v D5, D9, D9
vpermi.w D5, D8, 0xee //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D5, D5, 0xd8 //c2[2] c2[3] c3[2] c3[3]
vand.v D3, D5, D5
vand.v D0, D10, D10 //c0[0] c0[1] c1[0] c1[1]
vand.v D1, D11, D11 //c2[0] c2[1] c3[0] c3[1]
vpermi.w D4, D0, 0x44 //c0: 0 1 2 3
vpermi.w D2, D0, 0xee //c1: 0 1 2 3
vpermi.w D5, D1, 0x44 //c2: 0 1 2 3
vpermi.w D3, D1, 0xee //c3: 0 1 2 3
vst D4, C0, 0x00
vst D2, C1, 0x00
vst D5, C2, 0x00
vst D3, C3, 0x00
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
addi.d C2, C2, 0x10
addi.d C3, C3, 0x10
#else
//res00 res10 res20 res30
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vld D2, C2, 0x00 //c2: 0 1 2 3
vld D3, C3, 0x00 //c3: 0 1 2 3
vand.v D4, D1, D1
vpermi.w D4, D0, 0x44 //c0:0 1, c1:0 1
vshuf4i.w D6, D4, 0xd8 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D7, D4, 0x8d //c0[1] c1[1] c0[0] c1[0]
vand.v D5, D3, D3
vpermi.w D5, D2, 0x44 //c2:0 1, c3:0 1
vshuf4i.w D8, D5, 0xd8 //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D9, D5, 0x8d //c2[1] c3[1] c2[0] c3[0]
vpermi.w D8, D6, 0x44 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D9, D7, 0x44 //c0[1] c1[1] c2[1] c3[1]
VFMADD D8, U0, VALPHAR, D8
VFMADD D9, U1, VALPHAR, D9
VNMSUB D8, U1, VALPHAI, D8
VFMADD D9, U0, VALPHAI, D9
vand.v D10, D9, D9 //c0[1] c1[1] c2[1] c3[1]
vand.v D11, D9, D9 //c0[0] c1[0] c2[0] c3[0]
vpermi.w D10, D8, 0x44 //c0[0] c1[0] c0[1] c1[1]
vshuf4i.w D10, D10, 0xd8 //c0[0] c0[1] c1[0] c1[1]
vpermi.w D11, D8, 0xee //c2[0] c3[0] c2[1] c3[1]
vshuf4i.w D11, D11, 0xd8 //c2[0] c2[1] c3[0] c3[1]
//res01 res11 res21 res31
vand.v D4, D1, D1
vpermi.w D4, D0, 0xee //c0:2 3, c1:2 3
vshuf4i.w D6, D4, 0xd8 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D7, D4, 0x8d //c0[3] c1[3] c0[2] c1[2]
vand.v D5, D3, D3
vpermi.w D5, D2, 0xee //c2:2 3, c3:2 3
vshuf4i.w D8, D5, 0xd8 //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D9, D5, 0x8d //c2[3] c3[3] c2[2] c3[2]
vpermi.w D8, D6, 0x44 //c0[2] c1[2] c2[2] c3[2]
vpermi.w D9, D7, 0x44 //c0[3] c1[3] c2[3] c3[3]
VFMADD D8, U2, VALPHAR, D8
VFMADD D9, U3, VALPHAR, D9
VNMSUB D8, U3, VALPHAI, D8
VFMADD D9, U2, VALPHAI, D9
vand.v D4, D9, D9
vpermi.w D4, D8, 0x44 //c0[2] c1[2] c0[3] c1[3]
vshuf4i.w D4, D4, 0xd8 //c0[2] c0[3] c1[2] c1[3]
vand.v D2, D4, D4
vand.v D5, D9, D9
vpermi.w D5, D8, 0xee //c2[2] c3[2] c2[3] c3[3]
vshuf4i.w D5, D5, 0xd8 //c2[2] c2[3] c3[2] c3[3]
vand.v D3, D5, D5
vand.v D0, D10, D10 //c0[0] c0[1] c1[0] c1[1]
vand.v D1, D11, D11 //c2[0] c2[1] c3[0] c3[1]
vpermi.w D4, D0, 0x44 //c0: 0 1 2 3
vpermi.w D2, D0, 0xee //c1: 0 1 2 3
vpermi.w D5, D1, 0x44 //c2: 0 1 2 3
vpermi.w D3, D1, 0xee //c3: 0 1 2 3
vst D4, C0, 0x00
vst D2, C1, 0x00
vst D5, C2, 0x00
vst D3, C3, 0x00
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
addi.d C2, C2, 0x10
addi.d C3, C3, 0x10
#endif
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -2
#else
addi.d TL, TL, -4
#endif
slli.d T3, TL, 0x04
add.d A0, A0, T3
slli.d T3, TL, 0x05
add.d B0, B0, T3
#endif
#ifdef LEFT
addi.d OFF, OFF, 2
#endif
#endif // #if defined(TRMMKERNEL)
.L183: /* if (bm & 1) */
move I, $r0
andi T0, M, 1
beq I, T0, .L186
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d T3, OFF, 0x03
add.d A0, A0, T3
slli.d T3, OFF, 0x05
add.d B0, B, T3
#endif
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF
#elif defined(LEFT)
addi.d TL, OFF, 1
#else
addi.d TL, OFF, 4
#endif
#endif // #if defined(TRMMKERNEL)
MTC c11, $r0
MTC c12, $r0
MTC c21, $r0
MTC c22, $r0
MTC c31, $r0
MTC c32, $r0
MTC c41, $r0
MTC c42, $r0
move L, $r0 //cycle param k
beq L, TL, .L185
blt TL, L, .L185
.L184: /* for (k=0; k<temp; k++) */
LD a1, A0, 0x00 //a0r
LD a2, A0, 0x04 //a0i
LD b1, B0, 0x00 //b0r
LD b2, B0, 0x04 //b0i
LD b3, B0, 0x08 //b1r
LD b4, B0, 0x0c //b1i
LD b5, B0, 0x10 //b2r
LD b6, B0, 0x14 //b2i
LD b7, B0, 0x18 //b3r
LD b8, B0, 0x1c //b3i
MADD1 c11, a1, b1, c11 //res00r
MADD2 c12, a2, b1, c12 //res00i
MADD3 c11, a2, b2, c11
MADD4 c12, a1, b2, c12
MADD1 c21, a1, b3, c21 //res10r
MADD2 c22, a2, b3, c22 //res10i
MADD3 c21, a2, b4, c21
MADD4 c22, a1, b4, c22
MADD1 c31, a1, b5, c31 //res20r
MADD2 c32, a2, b5, c32 //res20i
MADD3 c31, a2, b6, c31
MADD4 c32, a1, b6, c32
MADD1 c41, a1, b7, c41 //res30r
MADD2 c42, a2, b7, c42 //res30i
MADD3 c41, a2, b8, c41
MADD4 c42, a1, b8, c42
addi.d A0, A0, 0x08
addi.d B0, B0, 0x20
addi.d L, L, 1
blt L, TL, .L184
.L185:
#if defined(TRMMKERNEL)
//res00 res10 res20 res30
LD a5, C0, 0x00 //C0[0]
LD a6, C0, 0x04 //C0[1]
MUL a5, c11, ALPHA_R
MUL a6, c12, ALPHA_R
NMSUB a5, c12, ALPHA_I, a5
MADD a6, c11, ALPHA_I, a6
ST a5, C0, 0x00
ST a6, C0, 0x04
LD a5, C1, 0x00 //C1[0]
LD a6, C1, 0x04 //C1[1]
MUL a5, c21, ALPHA_R
MUL a6, c22, ALPHA_R
NMSUB a5, c22, ALPHA_I, a5
MADD a6, c21, ALPHA_I, a6
ST a5, C1, 0x00
ST a6, C1, 0x04
LD a5, C2, 0x00 //C2[0]
LD a6, C2, 0x04 //C2[1]
MUL a5, c31, ALPHA_R
MUL a6, c32, ALPHA_R
NMSUB a5, c32, ALPHA_I, a5
MADD a6, c31, ALPHA_I, a6
ST a5, C2, 0x00
ST a6, C2, 0x04
LD a5, C3, 0x00 //C3[0]
LD a6, C3, 0x04 //C3[1]
MUL a5, c41, ALPHA_R
MUL a6, c42, ALPHA_R
NMSUB a5, c42, ALPHA_I, a5
MADD a6, c41, ALPHA_I, a6
ST a5, C3, 0x00
ST a6, C3, 0x04
addi.d C0, C0, 0x08
addi.d C1, C1, 0x08
addi.d C2, C2, 0x08
addi.d C3, C3, 0x08
#else
//res00 res10 res20 res30
LD a5, C0, 0x00 //C0[0]
LD a6, C0, 0x04 //C0[1]
MADD a5, c11, ALPHA_R, a5
MADD a6, c12, ALPHA_R, a6
NMSUB a5, c12, ALPHA_I, a5
MADD a6, c11, ALPHA_I, a6
ST a5, C0, 0x00
ST a6, C0, 0x04
LD a5, C1, 0x00 //C1[0]
LD a6, C1, 0x04 //C1[1]
MADD a5, c21, ALPHA_R, a5
MADD a6, c22, ALPHA_R, a6
NMSUB a5, c22, ALPHA_I, a5
MADD a6, c21, ALPHA_I, a6
ST a5, C1, 0x00
ST a6, C1, 0x04
LD a5, C2, 0x00 //C2[0]
LD a6, C2, 0x04 //C2[1]
MADD a5, c31, ALPHA_R, a5
MADD a6, c32, ALPHA_R, a6
NMSUB a5, c32, ALPHA_I, a5
MADD a6, c31, ALPHA_I, a6
ST a5, C2, 0x00
ST a6, C2, 0x04
LD a5, C3, 0x00 //C3[0]
LD a6, C3, 0x04 //C3[1]
MADD a5, c41, ALPHA_R, a5
MADD a6, c42, ALPHA_R, a6
NMSUB a5, c42, ALPHA_I, a5
MADD a6, c41, ALPHA_I, a6
ST a5, C3, 0x00
ST a6, C3, 0x04
addi.d C0, C0, 0x08
addi.d C1, C1, 0x08
addi.d C2, C2, 0x08
addi.d C3, C3, 0x08
#endif
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -1
#else
addi.d TL, TL, -4
#endif
slli.d T3, TL, 0x03
add.d A0, A0, T3
slli.d C3, TL, 0x05
add.d B0, B0, T3
#endif
#ifdef LEFT
addi.d OFF, OFF, 1
#endif
#endif // #if defined(TRMMKERNEL)
.L186:
#if defined(TRMMKERNEL) && !defined(LEFT)
addi.d OFF, OFF, 4
#endif
slli.d L, K, 0x05
add.d B, B, L
slli.d I, LDC, 0x03
add.d C, C, I
addi.d J, J, 1
srai.d T0, N, 2
blt J, T0, .L10
.L19:
move J, $r0
andi T0, N, 2
beq J, T0, .L30
.L20: /* for (j=0; j<(bn&2); j+=2) */
#if defined(TRMMKERNEL) && defined(LEFT)
move OFF, OFFSET
#endif
move C0, C
slli.d TL, LDC, 1
add.d C1, C0, TL
move A0, A //ptrba
move I, $r0
srai.d T0, M, 3 //bm/8
beq I, T0, .L24
.L21: /* for (i=0; i<bm/8; i+=1) */
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d T3, OFF, 0x06
add.d A0, A0, T3
slli.d T3, OFF, 0x04
add.d B0, B, T3
#endif
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF
#elif defined(LEFT)
addi.d TL, OFF, 8
#else
addi.d TL, OFF, 2
#endif
#endif // #if defined(TRMMKERNEL)
vxor.v U0, U0, U0
vxor.v U1, U1, U1
vxor.v U2, U2, U2
vxor.v U3, U3, U3
vxor.v U4, U4, U4
vxor.v U5, U5, U5
vxor.v U6, U6, U6
vxor.v U7, U7, U7
move L, $r0 //cycle param k
beq L, TL, .L23
blt TL, L, .L23
.L22: /* for (k=0; k<temp; k++) */
vld D0, A0, 0x00 // a0ri a1ri
vld D2, B0, 0x00 // b0ri b1ri
vshuf4i.w D4, D0, 0xa0 //a0rr a1rr
vshuf4i.w D5, D0, 0xf5 //a0ii a1ii
vshuf4i.w D6, D2, 0x88 //b0r b1r b0r b1r
vshuf4i.w D7, D2, 0xdd //b0i b1i b0i b1i
VMADD1 U0, D4, D6, U0 //00r 10r 01r 11r
VMADD2 U1, D5, D6, U1 //00i 10i 01i 11i
VMADD3 U0, D5, D7, U0
VMADD4 U1, D4, D7, U1
vld D0, A0, 0x10 // a2ri a3ri
vshuf4i.w D4, D0, 0xa0 //a2rr a3rr
vshuf4i.w D5, D0, 0xf5 //a2ii a3ii
VMADD1 U2, D4, D6, U2 //02r 12r 03r 13r
VMADD2 U3, D5, D6, U3 //02i 12i 03i 13i
VMADD3 U2, D5, D7, U2
VMADD4 U3, D4, D7, U3
vld D0, A0, 0x20 // a4ri a5ri
vshuf4i.w D4, D0, 0xa0 //a4rr a5rr
vshuf4i.w D5, D0, 0xf5 //a4ii a5ii
VMADD1 U4, D4, D6, U4 //04r 14r 05r 15r
VMADD2 U5, D5, D6, U5 //04i 14i 05i 15i
VMADD3 U4, D5, D7, U4
VMADD4 U5, D4, D7, U5
vld D0, A0, 0x30 // a6ri a7ri
vshuf4i.w D4, D0, 0xa0 //a6rr a7rr
vshuf4i.w D5, D0, 0xf5 //a6ii a7ii
VMADD1 U6, D4, D6, U6 //06r 16r 07r 17r
VMADD2 U7, D5, D6, U7 //06i 16i 07i 17i
VMADD3 U6, D5, D7, U6
VMADD4 U7, D4, D7, U7
addi.d A0, A0, 0x40
addi.d B0, B0, 0x10
addi.d L, L, 1
blt L, TL, .L22
.L23:
#if defined(TRMMKERNEL)
//res00 res10 res01 res11
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vpackev.w D2, D1, D0 //0 4 2 6
vpackod.w D3, D1, D0 //1 5 3 7
vfmul.s D2, U0, VALPHAR
vfmul.s D3, U1, VALPHAR
VNMSUB D2, U1, VALPHAI, D2
VFMADD D3, U0, VALPHAI, D3
vpackev.w D4, D3, D2 //0 1 2 3
vpackod.w D5, D3, D2 //4 5 6 7
vst D4, C0, 0x00 //c0: 0 1 2 3
vst D5, C1, 0x00 //c1: 0 1 2 3
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
//res02 res12 res03 res13
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vpackev.w D2, D1, D0 //0 4 2 6
vpackod.w D3, D1, D0 //1 5 3 7
vfmul.s D2, U2, VALPHAR
vfmul.s D3, U3, VALPHAR
VNMSUB D2, U3, VALPHAI, D2
VFMADD D3, U2, VALPHAI, D3
vpackev.w D4, D3, D2 //0 1 2 3
vpackod.w D5, D3, D2 //4 5 6 7
vst D4, C0, 0x00 //c0: 0 1 2 3
vst D5, C1, 0x00 //c1: 0 1 2 3
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
//res04 res14 res05 res15
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vpackev.w D2, D1, D0 //0 4 2 6
vpackod.w D3, D1, D0 //1 5 3 7
vfmul.s D2, U4, VALPHAR
vfmul.s D3, U5, VALPHAR
VNMSUB D2, U5, VALPHAI, D2
VFMADD D3, U4, VALPHAI, D3
vpackev.w D4, D3, D2 //0 1 2 3
vpackod.w D5, D3, D2 //4 5 6 7
vst D4, C0, 0x00 //c0: 0 1 2 3
vst D5, C1, 0x00 //c1: 0 1 2 3
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
//res06 res16 res07 res17
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vpackev.w D2, D1, D0 //0 4 2 6
vpackod.w D3, D1, D0 //1 5 3 7
vfmul.s D2, U6, VALPHAR
vfmul.s D3, U7, VALPHAR
VNMSUB D2, U7, VALPHAI, D2
VFMADD D3, U6, VALPHAI, D3
vpackev.w D4, D3, D2 //0 1 2 3
vpackod.w D5, D3, D2 //4 5 6 7
vst D4, C0, 0x00 //c0: 0 1 2 3
vst D5, C1, 0x00 //c1: 0 1 2 3
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
#else
//res00 res10 res01 res11
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vpackev.w D2, D1, D0 //0 4 2 6
vpackod.w D3, D1, D0 //1 5 3 7
VFMADD D2, U0, VALPHAR, D2
VFMADD D3, U1, VALPHAR, D3
VNMSUB D2, U1, VALPHAI, D2
VFMADD D3, U0, VALPHAI, D3
vpackev.w D4, D3, D2 //0 1 2 3
vpackod.w D5, D3, D2 //4 5 6 7
vst D4, C0, 0x00 //c0: 0 1 2 3
vst D5, C1, 0x00 //c1: 0 1 2 3
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
//res02 res12 res03 res13
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vpackev.w D2, D1, D0 //0 4 2 6
vpackod.w D3, D1, D0 //1 5 3 7
VFMADD D2, U2, VALPHAR, D2
VFMADD D3, U3, VALPHAR, D3
VNMSUB D2, U3, VALPHAI, D2
VFMADD D3, U2, VALPHAI, D3
vpackev.w D4, D3, D2 //0 1 2 3
vpackod.w D5, D3, D2 //4 5 6 7
vst D4, C0, 0x00 //c0: 0 1 2 3
vst D5, C1, 0x00 //c1: 0 1 2 3
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
//res04 res14 res05 res15
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vpackev.w D2, D1, D0 //0 4 2 6
vpackod.w D3, D1, D0 //1 5 3 7
VFMADD D2, U4, VALPHAR, D2
VFMADD D3, U5, VALPHAR, D3
VNMSUB D2, U5, VALPHAI, D2
VFMADD D3, U4, VALPHAI, D3
vpackev.w D4, D3, D2 //0 1 2 3
vpackod.w D5, D3, D2 //4 5 6 7
vst D4, C0, 0x00 //c0: 0 1 2 3
vst D5, C1, 0x00 //c1: 0 1 2 3
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
//res06 res16 res07 res17
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vpackev.w D2, D1, D0 //0 4 2 6
vpackod.w D3, D1, D0 //1 5 3 7
VFMADD D2, U6, VALPHAR, D2
VFMADD D3, U7, VALPHAR, D3
VNMSUB D2, U7, VALPHAI, D2
VFMADD D3, U6, VALPHAI, D3
vpackev.w D4, D3, D2 //0 1 2 3
vpackod.w D5, D3, D2 //4 5 6 7
vst D4, C0, 0x00 //c0: 0 1 2 3
vst D5, C1, 0x00 //c1: 0 1 2 3
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
#endif
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -8
#else
addi.d TL, TL, -2
#endif
slli.d T3, TL, 0x06
add.d A0, A0, T3
slli.d T3, TL, 0x04
add.d B0, B0, T3
#endif
#ifdef LEFT
addi.d OFF, OFF, 8
#endif
#endif // #if defined(TRMMKERNEL)
addi.d I, I, 1
blt I, T0, .L21
.L24: /* if ( bm & 4 ) */
move I, $r0
andi T1, M, 4 //bm&4
beq I, T1, .L280
.L25:
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d T3, OFF, 0x05
add.d A0, A0, T3
slli.d T3, OFF, 0x04
add.d B0, B, T3
#endif
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF
#elif defined(LEFT)
addi.d TL, OFF, 4
#else
addi.d TL, OFF, 2
#endif
#endif // #if defined(TRMMKERNEL)
vxor.v U0, U0, U0
vxor.v U1, U1, U1
vxor.v U2, U2, U2
vxor.v U3, U3, U3
move L, $r0 //cycle param k
beq L, TL, .L27
blt TL, L, .L27
.L26: /* for (k=0; k<temp; k++) */
vld D0, A0, 0x00 // a0ri a1ri
vld D2, B0, 0x00 // b0ri b1ri
vshuf4i.w D4, D0, 0xa0 //a0rr a1rr
vshuf4i.w D5, D0, 0xf5 //a0ii a1ii
vshuf4i.w D6, D2, 0x88 //b0r b1r b0r b1r
vshuf4i.w D7, D2, 0xdd //b0i b1i b0i b1i
VMADD1 U0, D4, D6, U0 //00r 10r 01r 11r
VMADD2 U1, D5, D6, U1 //00i 10i 01i 11i
VMADD3 U0, D5, D7, U0
VMADD4 U1, D4, D7, U1
vld D0, A0, 0x10 // a2ri a3ri
vshuf4i.w D4, D0, 0xa0 //a2rr a3rr
vshuf4i.w D5, D0, 0xf5 //a2ii a3ii
VMADD1 U2, D4, D6, U2 //02r 12r 03r 13r
VMADD2 U3, D5, D6, U3 //02i 12i 03i 13i
VMADD3 U2, D5, D7, U2
VMADD4 U3, D4, D7, U3
addi.d A0, A0, 0x20
addi.d B0, B0, 0x10
addi.d L, L, 1
blt L, TL, .L26
.L27:
#if defined(TRMMKERNEL)
//res00 res10 res01 res11
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vpackev.w D2, D1, D0 //0 4 2 6
vpackod.w D3, D1, D0 //1 5 3 7
vfmul.s D2, U0, VALPHAR
vfmul.s D3, U1, VALPHAR
VNMSUB D2, U1, VALPHAI, D2
VFMADD D3, U0, VALPHAI, D3
vpackev.w D4, D3, D2 //0 1 2 3
vpackod.w D5, D3, D2 //4 5 6 7
vst D4, C0, 0x00 //c0: 0 1 2 3
vst D5, C1, 0x00 //c1: 0 1 2 3
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
//res02 res12 res03 res13
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vpackev.w D2, D1, D0 //0 4 2 6
vpackod.w D3, D1, D0 //1 5 3 7
vfmul.s D2, U2, VALPHAR
vfmul.s D3, U3, VALPHAR
VNMSUB D2, U3, VALPHAI, D2
VFMADD D3, U2, VALPHAI, D3
vpackev.w D4, D3, D2 //0 1 2 3
vpackod.w D5, D3, D2 //4 5 6 7
vst D4, C0, 0x00 //c0: 0 1 2 3
vst D5, C1, 0x00 //c1: 0 1 2 3
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
#else
//res00 res10 res01 res11
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vpackev.w D2, D1, D0 //0 4 2 6
vpackod.w D3, D1, D0 //1 5 3 7
VFMADD D2, U0, VALPHAR, D2
VFMADD D3, U1, VALPHAR, D3
VNMSUB D2, U1, VALPHAI, D2
VFMADD D3, U0, VALPHAI, D3
vpackev.w D4, D3, D2 //0 1 2 3
vpackod.w D5, D3, D2 //4 5 6 7
vst D4, C0, 0x00 //c0: 0 1 2 3
vst D5, C1, 0x00 //c1: 0 1 2 3
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
//res02 res12 res03 res13
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vpackev.w D2, D1, D0 //0 4 2 6
vpackod.w D3, D1, D0 //1 5 3 7
VFMADD D2, U2, VALPHAR, D2
VFMADD D3, U3, VALPHAR, D3
VNMSUB D2, U3, VALPHAI, D2
VFMADD D3, U2, VALPHAI, D3
vpackev.w D4, D3, D2 //0 1 2 3
vpackod.w D5, D3, D2 //4 5 6 7
vst D4, C0, 0x00 //c0: 0 1 2 3
vst D5, C1, 0x00 //c1: 0 1 2 3
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
#endif
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -4
#else
addi.d TL, TL, -2
#endif
slli.d T3, TL, 0x05
add.d A0, A0, T3
slli.d T3, TL, 0x04
add.d B0, B0, T3
#endif
#ifdef LEFT
addi.d OFF, OFF, 4
#endif
#endif // #if defined(TRMMKERNEL)
.L280: /* if ( bm & 2 )*/
move I, $r0
andi T1, M, 2 //bm&2
beq I, T1, .L284
.L281:
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d T3, OFF, 0x04
add.d A0, A0, T3
add.d B0, B, T3
#endif
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF
#elif defined(LEFT)
addi.d TL, OFF, 2
#else
addi.d TL, OFF, 2
#endif
#endif // #if defined(TRMMKERNEL)
vxor.v U0, U0, U0
vxor.v U1, U1, U1
move L, $r0 //cycle param k
beq L, TL, .L283
blt TL, L, .L283
.L282: /* for (k=0; k<temp; k++) */
vld D0, A0, 0x00 // a0ri a1ri
vld D2, B0, 0x00 // b0ri b1ri
vshuf4i.w D4, D0, 0xa0 //a0rr a1rr
vshuf4i.w D5, D0, 0xf5 //a0ii a1ii
vshuf4i.w D6, D2, 0x88 //b0r b1r b0r b1r
vshuf4i.w D7, D2, 0xdd //b0i b1i b0i b1i
VMADD1 U0, D4, D6, U0 //00r 10r 01r 11r
VMADD2 U1, D5, D6, U1 //00i 10i 01i 11i
VMADD3 U0, D5, D7, U0
VMADD4 U1, D4, D7, U1
addi.d A0, A0, 0x10
addi.d B0, B0, 0x10
addi.d L, L, 1
blt L, TL, .L282
.L283:
#if defined(TRMMKERNEL)
//res00 res10 res01 res11
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vpackev.w D2, D1, D0 //0 4 2 6
vpackod.w D3, D1, D0 //1 5 3 7
vfmul.s D2, U0, VALPHAR
vfmul.s D3, U1, VALPHAR
VNMSUB D2, U1, VALPHAI, D2
VFMADD D3, U0, VALPHAI, D3
vpackev.w D4, D3, D2 //0 1 2 3
vpackod.w D5, D3, D2 //4 5 6 7
vst D4, C0, 0x00 //c0: 0 1 2 3
vst D5, C1, 0x00 //c1: 0 1 2 3
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
#else
//res00 res10 res01 res11
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C1, 0x00 //c1: 0 1 2 3
vpackev.w D2, D1, D0 //0 4 2 6
vpackod.w D3, D1, D0 //1 5 3 7
VFMADD D2, U0, VALPHAR, D2
VFMADD D3, U1, VALPHAR, D3
VNMSUB D2, U1, VALPHAI, D2
VFMADD D3, U0, VALPHAI, D3
vpackev.w D4, D3, D2 //0 1 2 3
vpackod.w D5, D3, D2 //4 5 6 7
vst D4, C0, 0x00 //c0: 0 1 2 3
vst D5, C1, 0x00 //c1: 0 1 2 3
addi.d C0, C0, 0x10
addi.d C1, C1, 0x10
#endif
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -2
#else
addi.d TL, TL, -2
#endif
slli.d T3, TL, 0x04
add.d A0, A0, T3
slli.d T3, TL, 0x04
add.d B0, B0, T3
#endif
#ifdef LEFT
addi.d OFF, OFF, 2
#endif
#endif // #if defined(TRMMKERNEL)
.L284: /* if ( bm & 1 )*/
move I, $r0
andi T1, M, 1 //bm&1
beq I, T1, .L288
.L285:
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d T3, OFF, 0x03
add.d A0, A0, T3
slli.d T3, OFF, 0x04
add.d B0, B, T3
#endif
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF
#elif defined(LEFT)
addi.d TL, OFF, 1
#else
addi.d TL, OFF, 2
#endif
#endif // #if defined(TRMMKERNEL)
MTC c11, $r0
MTC c12, $r0
MTC c21, $r0
MTC c22, $r0
move L, $r0 //cycle param k
beq L, TL, .L287
blt TL, L, .L287
.L286: /* for (k=0; k<temp; k++) */
LD a1, A0, 0x00 //a0r
LD a2, A0, 0x04 //a0i
LD b1, B0, 0x00 //b0r
LD b2, B0, 0x04 //b0i
LD b3, B0, 0x08 //b1r
LD b4, B0, 0x0c //b1i
MADD1 c11, a1, b1, c11 //res00r
MADD2 c12, a2, b1, c12 //res00i
MADD3 c11, a2, b2, c11
MADD4 c12, a1, b2, c12
MADD1 c21, a1, b3, c21 //res10r
MADD2 c22, a2, b3, c22 //res10i
MADD3 c21, a2, b4, c21
MADD4 c22, a1, b4, c22
addi.d A0, A0, 0x08
addi.d B0, B0, 0x10
addi.d L, L, 1
blt L, TL, .L286
.L287:
#if defined(TRMMKERNEL)
//res00 res10
LD a5, C0, 0x00 //C0[0]
LD a6, C0, 0x04 //C0[1]
LD a7, C1, 0x00 //C1[0]
LD a8, C1, 0x04 //C1[1]
MUL a5, c11, ALPHA_R
MUL a6, c12, ALPHA_R
NMSUB a5, c12, ALPHA_I, a5
MADD a6, c11, ALPHA_I, a6
MUL a7, c21, ALPHA_R
MUL a8, c22, ALPHA_R
NMSUB a7, c22, ALPHA_I, a7
MADD a8, c21, ALPHA_I, a8
ST a5, C0, 0x00
ST a6, C0, 0x04
ST a7, C1, 0x00
ST a8, C1, 0x04
addi.d C0, C0, 0x08
addi.d C1, C1, 0x08
#else
//res00 res10
LD a5, C0, 0x00 //C0[0]
LD a6, C0, 0x04 //C0[1]
LD a7, C1, 0x00 //C1[0]
LD a8, C1, 0x04 //C1[1]
MADD a5, c11, ALPHA_R, a5
MADD a6, c12, ALPHA_R, a6
NMSUB a5, c12, ALPHA_I, a5
MADD a6, c11, ALPHA_I, a6
MADD a7, c21, ALPHA_R, a7
MADD a8, c22, ALPHA_R, a8
NMSUB a7, c22, ALPHA_I, a7
MADD a8, c21, ALPHA_I, a8
ST a5, C0, 0x00
ST a6, C0, 0x04
ST a7, C1, 0x00
ST a8, C1, 0x04
addi.d C0, C0, 0x08
addi.d C1, C1, 0x08
#endif
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -1
#else
addi.d TL, TL, -2
#endif
slli.d T3, TL, 0x03
add.d A0, A0, T3
slli.d T3, TL, 0x04
add.d B0, B0, T3
#endif
#ifdef LEFT
addi.d OFF, OFF, 1
#endif
#endif // #if defined(TRMMKERNEL)
.L288:
#if defined(TRMMKERNEL) && !defined(LEFT)
addi.d OFF, OFF, 2
#endif
slli.d L, K, 4
add.d B, B, L
slli.d I, LDC, 2
add.d C, C, I
addi.d J, J, 2
andi T0, N, 2
blt J, T0, .L20
.L30:
move J, $r0
andi T0, N, 1
beq J, T0, .L999
.L300: /* for (j=0; j<(bn&1); j+=1) */
#if defined(TRMMKERNEL) && defined(LEFT)
move OFF, OFFSET
#endif
move C0, C
move A0, A //ptrba
move I, $r0
srai.d T0, M, 3 //bm/8
beq I, T0, .L34
.L31: /* for (i=0; i<bm/8; i+=1) */
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d T3, OFF, 0x06
add.d A0, A0, T3
slli.d T3, OFF, 0x03
add.d B0, B, T3
#endif
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF
#elif defined(LEFT)
addi.d TL, OFF, 8
#else
addi.d TL, OFF, 1
#endif
#endif // #if defined(TRMMKERNEL)
vxor.v U0, U0, U0
vxor.v U1, U1, U1
vxor.v U2, U2, U2
vxor.v U3, U3, U3
move L, $r0 //cycle param k
beq L, TL, .L33
blt TL, L, .L33
.L32: /* for (k=0; k<temp; k++) */
vld D0, A0, 0x00 // a0ri a1ri
vld D1, A0, 0x10 // a2ri a3ri
vldrepl.w D2, B0, 0x00 //b0r
vldrepl.w D3, B0, 0x04 //b0i
vpackev.w D4, D1, D0
vshuf4i.w D4, D4, 0xd8 //a0r a1r a2r a3r
vpackod.w D5, D1, D0
vshuf4i.w D5, D5, 0xd8 //a0i a1i a2i a3i
VMADD1 U0, D4, D2, U0 //00r 01r 02r 03r
VMADD2 U1, D5, D2, U1 //00i 01i 02i 03i
VMADD3 U0, D5, D3, U0
VMADD4 U1, D4, D3, U1
vld D0, A0, 0x20 // a4ri a5ri
vld D1, A0, 0x30 // a6ri a7ri
vpackev.w D4, D1, D0
vshuf4i.w D4, D4, 0xd8 //a4r a5r a6r a7r
vpackod.w D5, D1, D0
vshuf4i.w D5, D5, 0xd8 //a4i a5i a6i a7i
VMADD1 U2, D4, D2, U2 //04r 05r 06r 07r
VMADD2 U3, D5, D2, U3 //04i 05i 06i 07i
VMADD3 U2, D5, D3, U2
VMADD4 U3, D4, D3, U3
addi.d A0, A0, 0x40
addi.d B0, B0, 0x08
addi.d L, L, 1
blt L, TL, .L32
.L33:
#if defined(TRMMKERNEL)
//res00 res01 res02 res03
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C0, 0x10 //c0: 4 5 6 7
vpackev.w D2, D1, D0
vshuf4i.w D2, D2, 0xd8 //0 2 4 6
vpackod.w D3, D1, D0
vshuf4i.w D3, D3, 0xd8 //1 3 5 7
vfmul.s D2, U0, VALPHAR
vfmul.s D3, U1, VALPHAR
VNMSUB D2, U1, VALPHAI, D2
VFMADD D3, U0, VALPHAI, D3
vand.v D4, D3, D3 //1 3 5 7
vpermi.w D4, D2, 0x44 //0 2 1 3
vshuf4i.w D4, D4, 0xd8 //0 1 2 3
vand.v D5, D3, D3 //1 3 5 7
vpermi.w D5, D2, 0xee //4 6 5 7
vshuf4i.w D5, D5, 0xd8 //4 5 6 7
vst D4, C0, 0x00
vst D5, C0, 0x10
//res04 res05 res06 res07
vld D0, C0, 0x20 //c0: 8 9 10 11
vld D1, C0, 0x30 //c0: 12 13 14 15
vpackev.w D2, D1, D0
vshuf4i.w D2, D2, 0xd8 //8 10 12 14
vpackod.w D3, D1, D0
vshuf4i.w D3, D3, 0xd8 //9 11 13 15
vfmul.s D2, U2, VALPHAR
vfmul.s D3, U3, VALPHAR
VNMSUB D2, U3, VALPHAI, D2
VFMADD D3, U2, VALPHAI, D3
vand.v D4, D3, D3 //8 10 12 14
vpermi.w D4, D2, 0x44 //8 10 9 11
vshuf4i.w D4, D4, 0xd8 //8 9 10 11
vand.v D5, D3, D3 //9 11 13 15
vpermi.w D5, D2, 0xee //12 14 13 15
vshuf4i.w D5, D5, 0xd8 //12 13 14 15
vst D4, C0, 0x20
vst D5, C0, 0x30
addi.d C0, C0, 0x40
#else
//res00 res01 res02 res03
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C0, 0x10 //c0: 4 5 6 7
vpackev.w D2, D1, D0
vshuf4i.w D2, D2, 0xd8 //0 2 4 6
vpackod.w D3, D1, D0
vshuf4i.w D3, D3, 0xd8 //1 3 5 7
VFMADD D2, U0, VALPHAR, D2
VFMADD D3, U1, VALPHAR, D3
VNMSUB D2, U1, VALPHAI, D2
VFMADD D3, U0, VALPHAI, D3
vand.v D4, D3, D3 //1 3 5 7
vpermi.w D4, D2, 0x44 //0 2 1 3
vshuf4i.w D4, D4, 0xd8 //0 1 2 3
vand.v D5, D3, D3 //1 3 5 7
vpermi.w D5, D2, 0xee //4 6 5 7
vshuf4i.w D5, D5, 0xd8 //4 5 6 7
vst D4, C0, 0x00
vst D5, C0, 0x10
//res04 res05 res06 res07
vld D0, C0, 0x20 //c0: 8 9 10 11
vld D1, C0, 0x30 //c0: 12 13 14 15
vpackev.w D2, D1, D0
vshuf4i.w D2, D2, 0xd8 //8 10 12 14
vpackod.w D3, D1, D0
vshuf4i.w D3, D3, 0xd8 //9 11 13 15
VFMADD D2, U2, VALPHAR, D2
VFMADD D3, U3, VALPHAR, D3
VNMSUB D2, U3, VALPHAI, D2
VFMADD D3, U2, VALPHAI, D3
vand.v D4, D3, D3 //8 10 12 14
vpermi.w D4, D2, 0x44 //8 10 9 11
vshuf4i.w D4, D4, 0xd8 //8 9 10 11
vand.v D5, D3, D3 //9 11 13 15
vpermi.w D5, D2, 0xee //12 14 13 15
vshuf4i.w D5, D5, 0xd8 //12 13 14 15
vst D4, C0, 0x20
vst D5, C0, 0x30
addi.d C0, C0, 0x40
#endif
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -8
#else
addi.d TL, TL, -1
#endif
slli.d T3, TL, 0x06
add.d A0, A0, T3
slli.d T3, TL, 0x03
add.d B0, B0, T3
#endif
#ifdef LEFT
addi.d OFF, OFF, 8
#endif
#endif // #if defined(TRMMKERNEL)
addi.d I, I, 1
blt I, T0, .L31
.L34: /* if ( bm & 4 ) */
move I, $r0
andi T1, M, 4 //bm&4
beq I, T1, .L38
.L35:
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d T3, OFF, 0x05
add.d A0, A0, T3
slli.d T3, OFF, 0x03
add.d B0, B, T3
#endif
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF
#elif defined(LEFT)
addi.d TL, OFF, 4
#else
addi.d TL, OFF, 1
#endif
#endif // #if defined(TRMMKERNEL)
vxor.v U0, U0, U0
vxor.v U1, U1, U1
move L, $r0 //cycle param k
beq L, TL, .L37
blt TL, L, .L37
.L36: /* for (k=0; k<temp; k++) */
vld D0, A0, 0x00 // a0ri a1ri
vld D1, A0, 0x10 // a2ri a3ri
vldrepl.w D2, B0, 0x00 //b0r
vldrepl.w D3, B0, 0x04 //b0i
vpackev.w D4, D1, D0
vshuf4i.w D4, D4, 0xd8 //a0r a1r a2r a3r
vpackod.w D5, D1, D0
vshuf4i.w D5, D5, 0xd8 //a0i a1i a2i a3i
VMADD1 U0, D4, D2, U0 //00r 01r 02r 03r
VMADD2 U1, D5, D2, U1 //00i 01i 02i 03i
VMADD3 U0, D5, D3, U0
VMADD4 U1, D4, D3, U1
addi.d A0, A0, 0x20
addi.d B0, B0, 0x08
addi.d L, L, 1
blt L, TL, .L36
.L37:
#if defined(TRMMKERNEL)
//res00 res01 res02 res03
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C0, 0x10 //c0: 4 5 6 7
vpackev.w D2, D1, D0
vshuf4i.w D2, D2, 0xd8 //0 2 4 6
vpackod.w D3, D1, D0
vshuf4i.w D3, D3, 0xd8 //1 3 5 7
vfmul.s D2, U0, VALPHAR
vfmul.s D3, U1, VALPHAR
VNMSUB D2, U1, VALPHAI, D2
VFMADD D3, U0, VALPHAI, D3
vand.v D4, D3, D3 //1 3 5 7
vpermi.w D4, D2, 0x44 //0 2 1 3
vshuf4i.w D4, D4, 0xd8 //0 1 2 3
vand.v D5, D3, D3 //1 3 5 7
vpermi.w D5, D2, 0xee //4 6 5 7
vshuf4i.w D5, D5, 0xd8 //4 5 6 7
vst D4, C0, 0x00
vst D5, C0, 0x10
addi.d C0, C0, 0x20
#else
//res00 res01 res02 res03
vld D0, C0, 0x00 //c0: 0 1 2 3
vld D1, C0, 0x10 //c0: 4 5 6 7
vpackev.w D2, D1, D0
vshuf4i.w D2, D2, 0xd8 //0 2 4 6
vpackod.w D3, D1, D0
vshuf4i.w D3, D3, 0xd8 //1 3 5 7
VFMADD D2, U0, VALPHAR, D2
VFMADD D3, U1, VALPHAR, D3
VNMSUB D2, U1, VALPHAI, D2
VFMADD D3, U0, VALPHAI, D3
vand.v D4, D3, D3 //1 3 5 7
vpermi.w D4, D2, 0x44 //0 2 1 3
vshuf4i.w D4, D4, 0xd8 //0 1 2 3
vand.v D5, D3, D3 //1 3 5 7
vpermi.w D5, D2, 0xee //4 6 5 7
vshuf4i.w D5, D5, 0xd8 //4 5 6 7
vst D4, C0, 0x00
vst D5, C0, 0x10
addi.d C0, C0, 0x20
#endif
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -4
#else
addi.d TL, TL, -1
#endif
slli.d T3, TL, 0x05
add.d A0, A0, T3
slli.d T3, TL, 0x03
add.d B0, B0, T3
#endif
#ifdef LEFT
addi.d OFF, OFF, 4
#endif
#endif // #if defined(TRMMKERNEL)
.L38: /* if ( bm & 2 ) */
move I, $r0
andi T1, M, 2 //bm&2
beq I, T1, .L312
.L39:
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d T3, OFF, 0x04
add.d A0, A0, T3
slli.d T3, OFF, 0x03
add.d B0, B, T3
#endif
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF
#elif defined(LEFT)
addi.d TL, OFF, 2
#else
addi.d TL, OFF, 1
#endif
#endif // #if defined(TRMMKERNEL)
MTC c11, $r0
MTC c12, $r0
MTC c21, $r0
MTC c22, $r0
move L, $r0 //cycle param k
beq L, TL, .L311
blt TL, L, .L311
.L310: /* for (k=0; k<temp; k++) */
LD a1, A0, 0x00 //a0r
LD a2, A0, 0x04 //a0i
LD a3, A0, 0x08 //a1r
LD a4, A0, 0x0c //a1i
LD b1, B0, 0x00 //b0r
LD b2, B0, 0x04 //b0i
MADD1 c11, a1, b1, c11 //res00r
MADD2 c12, a2, b1, c12 //res00i
MADD3 c11, a2, b2, c11
MADD4 c12, a1, b2, c12
MADD1 c21, a3, b1, c21 //res10r
MADD2 c22, a4, b1, c22 //res10i
MADD3 c21, a4, b2, c21
MADD4 c22, a3, b2, c22
addi.d A0, A0, 0x10
addi.d B0, B0, 0x08
addi.d L, L, 1
blt L, TL, .L310
.L311:
#if defined(TRMMKERNEL)
//res00 res10
LD a5, C0, 0x00 //C0[0]
LD a6, C0, 0x04 //C0[1]
LD a7, C0, 0x08 //C0[2]
LD a8, C0, 0x0c //C0[3]
MUL a5, c11, ALPHA_R
MUL a6, c12, ALPHA_R
NMSUB a5, c12, ALPHA_I, a5
MADD a6, c11, ALPHA_I, a6
MUL a7, c21, ALPHA_R
MUL a8, c22, ALPHA_R
NMSUB a7, c22, ALPHA_I, a7
MADD a8, c21, ALPHA_I, a8
ST a5, C0, 0x00
ST a6, C0, 0x04
ST a7, C0, 0x08
ST a8, C0, 0x0c
addi.d C0, C0, 0x10
#else
//res00 res10
LD a5, C0, 0x00 //C0[0]
LD a6, C0, 0x04 //C0[1]
LD a7, C0, 0x08 //C0[2]
LD a8, C0, 0x0c //C0[3]
MADD a5, c11, ALPHA_R, a5
MADD a6, c12, ALPHA_R, a6
NMSUB a5, c12, ALPHA_I, a5
MADD a6, c11, ALPHA_I, a6
MADD a7, c21, ALPHA_R, a7
MADD a8, c22, ALPHA_R, a8
NMSUB a7, c22, ALPHA_I, a7
MADD a8, c21, ALPHA_I, a8
ST a5, C0, 0x00
ST a6, C0, 0x04
ST a7, C0, 0x08
ST a8, C0, 0x0c
addi.d C0, C0, 0x10
#endif
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -2
#else
addi.d TL, TL, -1
#endif
slli.d T3, TL, 0x04
add.d A0, A0, T3
slli.d T3, TL, 0x03
add.d B0, B0, T3
#endif
#ifdef LEFT
addi.d OFF, OFF, 2
#endif
#endif // #if defined(TRMMKERNEL)
.L312: /* if ( bm & 1 )*/
move I, $r0
andi T1, M, 1 //bm&1
beq I, T1, .L316
.L313:
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d T3, OFF, 0x03
add.d A0, A0, T3
slli.d T3, OFF, 0x03
add.d B0, B, T3
#endif
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF
#elif defined(LEFT)
addi.d TL, OFF, 1
#else
addi.d TL, OFF, 1
#endif
#endif // #if defined(TRMMKERNEL)
MTC c11, $r0
MTC c12, $r0
move L, $r0 //cycle param k
beq L, TL, .L315
blt TL, L, .L315
.L314: /* for (k=0; k<temp; k++) */
LD a1, A0, 0x00
LD a2, A0, 0x04
LD b1, B0, 0x00
LD b2, B0, 0x04
MADD1 c11, a1, b1, c11
MADD2 c12, a2, b1, c12
MADD3 c11, a2, b2, c11
MADD4 c12, a1, b2, c12
addi.d A0, A0, 0x08
addi.d B0, B0, 0x08
addi.d L, L, 1
blt L, TL, .L314
.L315:
#if defined(TRMMKERNEL)
MUL a5, c11, ALPHA_R
MUL a6, c12, ALPHA_I
SUB a5, a5, a6
ST a5, C0, 0x00
MUL a5, c12, ALPHA_R
MUL a6, c11, ALPHA_I
ADD a6, a5, a6
ST a6, C0, 0x04
addi.d C0, C0, 0x08
#else
LD a5, C0, 0x00 //C0[0]
LD a6, C0, 0x04 //C0[1]
MADD a5, c11, ALPHA_R, a5
MADD a6, c12, ALPHA_R, a6
NMSUB a5, c12, ALPHA_I, a5
MADD a6, c11, ALPHA_I, a6
ST a5, C0, 0x00
ST a6, C0, 0x04
addi.d C0, C0, 0x08
#endif
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -1
#else
addi.d TL, TL, -1
#endif
slli.d T3, TL, 0x03
add.d A0, A0, T3
add.d B0, B0, T3
#endif
#ifdef LEFT
addi.d OFF, OFF, 1
#endif
#endif // #if defined(TRMMKERNEL)
.L316:
slli.d L, K, 3
add.d B, B, L
slli.d I, LDC, 1
add.d C, C, I
addi.d J, J, 1
andi T0, N, 1
blt J, T0, .L300
.L999:
LDARG $r23, $sp, 0
LDARG $r24, $sp, 8
LDARG $r25, $sp, 16
LDARG $r26, $sp, 24
LDARG $r27, $sp, 32
LD $f23, $sp, 40
LD $f24, $sp, 48
LD $f25, $sp, 56
LD $f26, $sp, 64
LD $f27, $sp, 72
LD $f28, $sp, 80
LD $f29, $sp, 88
LD $f30, $sp, 96
LD $f31, $sp, 104
addi.d $sp, $sp, 128
jirl $r0, $r1, 0x0
EPILOGUE
Reference in New Issue View Git Blame Copy Permalink