7007 lines
119 KiB
ArmAsm
7007 lines
119 KiB
ArmAsm
/*********************************************************************/
|
|
/* Copyright 2009, 2010 The University of Texas at Austin. */
|
|
/* All rights reserved. */
|
|
/* */
|
|
/* Redistribution and use in source and binary forms, with or */
|
|
/* without modification, are permitted provided that the following */
|
|
/* conditions are met: */
|
|
/* */
|
|
/* 1. Redistributions of source code must retain the above */
|
|
/* copyright notice, this list of conditions and the following */
|
|
/* disclaimer. */
|
|
/* */
|
|
/* 2. Redistributions in binary form must reproduce the above */
|
|
/* copyright notice, this list of conditions and the following */
|
|
/* disclaimer in the documentation and/or other materials */
|
|
/* provided with the distribution. */
|
|
/* */
|
|
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
|
|
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
|
|
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
|
|
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
|
|
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
|
|
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
|
|
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
|
|
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
|
|
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
|
|
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
|
|
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
|
|
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
|
|
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
|
|
/* POSSIBILITY OF SUCH DAMAGE. */
|
|
/* */
|
|
/* The views and conclusions contained in the software and */
|
|
/* documentation are those of the authors and should not be */
|
|
/* interpreted as representing official policies, either expressed */
|
|
/* or implied, of The University of Texas at Austin. */
|
|
/*********************************************************************/
|
|
|
|
#define ASSEMBLER
|
|
#include "common.h"
|
|
|
|
#define ALPHA 0
|
|
#define FZERO 8
|
|
|
|
#define M r3
|
|
#define N r4
|
|
#define K r5
|
|
|
|
#ifdef linux
|
|
#define A r6
|
|
#define B r7
|
|
#define C r8
|
|
#define LDC r9
|
|
#define OFFSET r10
|
|
#endif
|
|
|
|
#define TEMP r11
|
|
#define KK r14
|
|
#define INCM1 r15
|
|
#define INCM3 r16
|
|
#define INCM5 r17
|
|
#define INCM7 r18
|
|
#define INC2 r19
|
|
#define INC r20
|
|
#define INC4 r21
|
|
|
|
#define I r22
|
|
#define J r23
|
|
#define AO r24
|
|
#define BO r25
|
|
#define AO2 r26
|
|
#define BO2 r27
|
|
|
|
#define CO1 r28
|
|
#define CO2 r29
|
|
#define CO3 r30
|
|
#define CO4 r31
|
|
|
|
#ifndef NEEDPARAM
|
|
|
|
#define A1 f16
|
|
#define A2 f17
|
|
#define A3 f18
|
|
#define A4 f19
|
|
#define A5 f20
|
|
#define A6 f21
|
|
#define A7 f22
|
|
#define A8 f23
|
|
#define A9 f24
|
|
#define A10 f25
|
|
|
|
#define B1 f26
|
|
#define B2 f27
|
|
#define B3 f28
|
|
#define B4 f29
|
|
#define B5 f30
|
|
#define B6 f31
|
|
|
|
#define AP B6
|
|
|
|
|
|
PROLOGUE
|
|
PROFCODE
|
|
|
|
li r0, -16
|
|
|
|
stfpdux f14, SP, r0
|
|
stfpdux f15, SP, r0
|
|
stfpdux f16, SP, r0
|
|
stfpdux f17, SP, r0
|
|
stfpdux f18, SP, r0
|
|
stfpdux f19, SP, r0
|
|
stfpdux f20, SP, r0
|
|
stfpdux f21, SP, r0
|
|
stfpdux f22, SP, r0
|
|
stfpdux f23, SP, r0
|
|
stfpdux f24, SP, r0
|
|
stfpdux f25, SP, r0
|
|
stfpdux f26, SP, r0
|
|
stfpdux f27, SP, r0
|
|
stfpdux f28, SP, r0
|
|
stfpdux f29, SP, r0
|
|
stfpdux f30, SP, r0
|
|
stfpdux f31, SP, r0
|
|
|
|
stwu r31, -4(SP)
|
|
stwu r30, -4(SP)
|
|
stwu r29, -4(SP)
|
|
stwu r28, -4(SP)
|
|
|
|
stwu r27, -4(SP)
|
|
stwu r26, -4(SP)
|
|
stwu r25, -4(SP)
|
|
stwu r24, -4(SP)
|
|
|
|
stwu r23, -4(SP)
|
|
stwu r22, -4(SP)
|
|
stwu r21, -4(SP)
|
|
stwu r20, -4(SP)
|
|
|
|
stwu r19, -4(SP)
|
|
stwu r18, -4(SP)
|
|
stwu r17, -4(SP)
|
|
stwu r16, -4(SP)
|
|
|
|
stwu r15, -4(SP)
|
|
stwu r14, -4(SP) # dummy
|
|
|
|
li r0, 0
|
|
|
|
stwu r0, -4(SP)
|
|
stwu r0, -4(SP)
|
|
stfdu f1, -8(SP)
|
|
|
|
slwi LDC, LDC, BASE_SHIFT
|
|
|
|
cmpwi cr0, M, 0
|
|
ble .L999
|
|
cmpwi cr0, N, 0
|
|
ble .L999
|
|
cmpwi cr0, K, 0
|
|
ble .L999
|
|
|
|
li INC, 1 * SIZE
|
|
li INC2, 2 * SIZE
|
|
li INC4, 4 * SIZE
|
|
|
|
#if defined(TRMMKERNEL) && !defined(LEFT)
|
|
neg KK, OFFSET
|
|
#endif
|
|
|
|
andi. r0, C, 2 * SIZE - 1
|
|
bne .L1000
|
|
andi. r0, LDC, 2 * SIZE - 1
|
|
bne .L1000
|
|
|
|
/* High performance version */
|
|
|
|
li INCM3, -2 * SIZE
|
|
li INCM5, -5 * SIZE
|
|
li INCM7, -6 * SIZE
|
|
|
|
addi C, C, - 2 * SIZE
|
|
srawi. J, N, 2
|
|
ble .L50
|
|
.align 4
|
|
|
|
.L10:
|
|
mr CO1, C
|
|
add CO2, C, LDC
|
|
add CO3, CO2, LDC
|
|
add CO4, CO3, LDC
|
|
add C, CO4, LDC
|
|
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
mr KK, OFFSET
|
|
#endif
|
|
|
|
addi AO, A, -4 * SIZE
|
|
|
|
li r0, FZERO
|
|
lfpsx f0, SP, r0
|
|
|
|
srawi. I, M, 3
|
|
ble .L20
|
|
.align 4
|
|
|
|
.L11:
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
|
addi AO2, AO, 2 * SIZE
|
|
fpmr f4, f0
|
|
addi BO, B, - 4 * SIZE
|
|
fpmr f8, f0
|
|
addi BO2, B, - 2 * SIZE
|
|
fpmr f12, f0
|
|
#else
|
|
slwi TEMP, KK, 3 + BASE_SHIFT
|
|
slwi r0, KK, 2 + BASE_SHIFT
|
|
add AO, AO, TEMP
|
|
add BO, B, r0
|
|
|
|
addi AO2, AO, 2 * SIZE
|
|
fpmr f4, f0
|
|
addi BO, BO, - 4 * SIZE
|
|
fpmr f8, f0
|
|
addi BO2, BO, 2 * SIZE
|
|
fpmr f12, f0
|
|
#endif
|
|
|
|
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 8
|
|
#else
|
|
addi TEMP, KK, 4
|
|
#endif
|
|
srawi. TEMP, TEMP, 2
|
|
fpmr f1, f0
|
|
mtspr CTR, TEMP
|
|
ble .L14
|
|
|
|
#else
|
|
addi AO2, AO, 2 * SIZE
|
|
fpmr f4, f0
|
|
addi BO, B, - 4 * SIZE
|
|
fpmr f8, f0
|
|
addi BO2, B, - 2 * SIZE
|
|
fpmr f12, f0
|
|
|
|
srawi. r0, K, 2
|
|
fpmr f1, f0
|
|
mtspr CTR, r0
|
|
ble .L14
|
|
#endif
|
|
|
|
LFPDUX A1, AO, INC4
|
|
fpmr f5, f0
|
|
LFPDUX A3, AO, INC4
|
|
fpmr f9, f0
|
|
LFPDUX B1, BO, INC4
|
|
fpmr f13, f0
|
|
|
|
LFPDUX A5, AO, INC4
|
|
fpmr f2, f0
|
|
LFPDUX A6, AO, INC4
|
|
fpmr f6, f0
|
|
LFPDUX B3, BO, INC4
|
|
fpmr f10, f0
|
|
LFPDUX A7, AO, INC4
|
|
fpmr f14, f0
|
|
|
|
LFPDUX A8, AO, INC4
|
|
fpmr f3, f0
|
|
LFPDUX B5, BO, INC4
|
|
fpmr f7, f0
|
|
LFPDUX A9, AO, INC4
|
|
fpmr f11, f0
|
|
LFPDUX A2, AO2, INC4
|
|
fpmr f15, f0
|
|
LFPDUX B2, BO2, INC4
|
|
bdz- .L13
|
|
.align 4
|
|
|
|
.L12:
|
|
|
|
## 1 ##
|
|
fxcpmadd f0, B1, A1, f0
|
|
nop
|
|
fxcsmadd f4, B1, A1, f4
|
|
nop
|
|
fxcpmadd f8, B2, A1, f8
|
|
LFPDUX B4, BO2, INC4
|
|
fxcsmadd f12, B2, A1, f12
|
|
LFPDUX B6, BO, INC4
|
|
|
|
fxcpmadd f1, B1, A2, f1
|
|
nop
|
|
fxcsmadd f5, B1, A2, f5
|
|
LFPDUX A4, AO2, INC4
|
|
fxcpmadd f9, B2, A2, f9
|
|
LFPDUX A10, AO, INC4
|
|
fxcsmadd f13, B2, A2, f13
|
|
nop
|
|
|
|
fxcpmadd f2, B1, A3, f2
|
|
nop
|
|
fxcsmadd f6, B1, A3, f6
|
|
nop
|
|
fxcpmadd f10, B2, A3, f10
|
|
nop
|
|
fxcsmadd f14, B2, A3, f14
|
|
nop
|
|
|
|
fxcpmadd f3, B1, A4, f3
|
|
nop
|
|
fxcsmadd f7, B1, A4, f7
|
|
LFPDUX A2, AO2, INC4
|
|
fxcpmadd f11, B2, A4, f11
|
|
LFPDUX A1, AO, INC4
|
|
fxcsmadd f15, B2, A4, f15
|
|
nop
|
|
|
|
## 2 ##
|
|
|
|
fxcpmadd f0, B3, A5, f0
|
|
nop
|
|
fxcsmadd f4, B3, A5, f4
|
|
nop
|
|
fxcpmadd f8, B4, A5, f8
|
|
LFPDUX B2, BO2, INC4
|
|
fxcsmadd f12, B4, A5, f12
|
|
LFPDUX B1, BO, INC4
|
|
|
|
fxcpmadd f1, B3, A2, f1
|
|
nop
|
|
fxcsmadd f5, B3, A2, f5
|
|
LFPDUX A4, AO2, INC4
|
|
fxcpmadd f9, B4, A2, f9
|
|
LFPDUX A3, AO, INC4
|
|
fxcsmadd f13, B4, A2, f13
|
|
nop
|
|
|
|
fxcpmadd f2, B3, A6, f2
|
|
nop
|
|
fxcsmadd f6, B3, A6, f6
|
|
nop
|
|
fxcpmadd f10, B4, A6, f10
|
|
nop
|
|
fxcsmadd f14, B4, A6, f14
|
|
nop
|
|
|
|
fxcpmadd f3, B3, A4, f3
|
|
nop
|
|
fxcsmadd f7, B3, A4, f7
|
|
LFPDUX A2, AO2, INC4
|
|
fxcpmadd f11, B4, A4, f11
|
|
LFPDUX A5, AO, INC4
|
|
fxcsmadd f15, B4, A4, f15
|
|
nop
|
|
|
|
## 3 ##
|
|
|
|
fxcpmadd f0, B5, A7, f0
|
|
nop
|
|
fxcsmadd f4, B5, A7, f4
|
|
nop
|
|
fxcpmadd f8, B2, A7, f8
|
|
LFPDUX B4, BO2, INC4
|
|
fxcsmadd f12, B2, A7, f12
|
|
LFPDUX B3, BO, INC4
|
|
|
|
fxcpmadd f1, B5, A2, f1
|
|
nop
|
|
fxcsmadd f5, B5, A2, f5
|
|
LFPDUX A4, AO2, INC4
|
|
fxcpmadd f9, B2, A2, f9
|
|
LFPDUX A6, AO, INC4
|
|
fxcsmadd f13, B2, A2, f13
|
|
nop
|
|
|
|
fxcpmadd f2, B5, A8, f2
|
|
nop
|
|
fxcsmadd f6, B5, A8, f6
|
|
nop
|
|
fxcpmadd f10, B2, A8, f10
|
|
nop
|
|
fxcsmadd f14, B2, A8, f14
|
|
nop
|
|
|
|
fxcpmadd f3, B5, A4, f3
|
|
nop
|
|
fxcsmadd f7, B5, A4, f7
|
|
LFPDUX A2, AO2, INC4
|
|
fxcpmadd f11, B2, A4, f11
|
|
LFPDUX A7, AO, INC4
|
|
fxcsmadd f15, B2, A4, f15
|
|
nop
|
|
|
|
## 4 ##
|
|
fxcpmadd f0, B6, A9, f0
|
|
nop
|
|
fxcsmadd f4, B6, A9, f4
|
|
nop
|
|
fxcpmadd f8, B4, A9, f8
|
|
LFPDUX B2, BO2, INC4
|
|
fxcsmadd f12, B4, A9, f12
|
|
LFPDUX B5, BO, INC4
|
|
|
|
fxcpmadd f1, B6, A2, f1
|
|
nop
|
|
fxcsmadd f5, B6, A2, f5
|
|
LFPDUX A4, AO2, INC4
|
|
fxcpmadd f9, B4, A2, f9
|
|
LFPDUX A8, AO, INC4
|
|
fxcsmadd f13, B4, A2, f13
|
|
nop
|
|
|
|
fxcpmadd f2, B6, A10, f2
|
|
nop
|
|
fxcsmadd f6, B6, A10, f6
|
|
nop
|
|
fxcpmadd f10, B4, A10, f10
|
|
nop
|
|
fxcsmadd f14, B4, A10, f14
|
|
nop
|
|
|
|
fxcpmadd f3, B6, A4, f3
|
|
LFPDUX A2, AO2, INC4
|
|
fxcsmadd f7, B6, A4, f7
|
|
LFPDUX A9, AO, INC4
|
|
fxcpmadd f11, B4, A4, f11
|
|
nop
|
|
fxcsmadd f15, B4, A4, f15
|
|
bdnz+ .L12
|
|
.align 4
|
|
|
|
.L13:
|
|
## 1 ##
|
|
|
|
fxcpmadd f0, B1, A1, f0
|
|
nop
|
|
fxcsmadd f4, B1, A1, f4
|
|
nop
|
|
fxcpmadd f8, B2, A1, f8
|
|
LFPDUX B4, BO2, INC4
|
|
fxcsmadd f12, B2, A1, f12
|
|
LFPDUX B6, BO, INC4
|
|
|
|
fxcpmadd f1, B1, A2, f1
|
|
nop
|
|
fxcsmadd f5, B1, A2, f5
|
|
LFPDUX A4, AO2, INC4
|
|
fxcpmadd f9, B2, A2, f9
|
|
LFPDUX A10, AO, INC4
|
|
fxcsmadd f13, B2, A2, f13
|
|
nop
|
|
|
|
fxcpmadd f2, B1, A3, f2
|
|
nop
|
|
fxcsmadd f6, B1, A3, f6
|
|
nop
|
|
fxcpmadd f10, B2, A3, f10
|
|
nop
|
|
fxcsmadd f14, B2, A3, f14
|
|
nop
|
|
|
|
fxcpmadd f3, B1, A4, f3
|
|
nop
|
|
fxcsmadd f7, B1, A4, f7
|
|
LFPDUX A2, AO2, INC4
|
|
fxcpmadd f11, B2, A4, f11
|
|
#ifndef TRMMKERNEL
|
|
LFPDUX A1, CO1, INC2
|
|
#else
|
|
nop
|
|
#endif
|
|
fxcsmadd f15, B2, A4, f15
|
|
nop
|
|
|
|
## 2 ##
|
|
|
|
fxcpmadd f0, B3, A5, f0
|
|
nop
|
|
fxcsmadd f4, B3, A5, f4
|
|
nop
|
|
fxcpmadd f8, B4, A5, f8
|
|
LFPDUX B2, BO2, INC4
|
|
fxcsmadd f12, B4, A5, f12
|
|
#ifndef TRMMKERNEL
|
|
LFPDUX B1, CO1, INC4
|
|
#else
|
|
nop
|
|
#endif
|
|
|
|
fxcpmadd f1, B3, A2, f1
|
|
nop
|
|
fxcsmadd f5, B3, A2, f5
|
|
LFPDUX A4, AO2, INC4
|
|
fxcpmadd f9, B4, A2, f9
|
|
#ifndef TRMMKERNEL
|
|
LFPDUX A3, CO2, INC2
|
|
#else
|
|
nop
|
|
#endif
|
|
fxcsmadd f13, B4, A2, f13
|
|
nop
|
|
|
|
fxcpmadd f2, B3, A6, f2
|
|
nop
|
|
fxcsmadd f6, B3, A6, f6
|
|
nop
|
|
fxcpmadd f10, B4, A6, f10
|
|
nop
|
|
fxcsmadd f14, B4, A6, f14
|
|
nop
|
|
|
|
fxcpmadd f3, B3, A4, f3
|
|
nop
|
|
fxcsmadd f7, B3, A4, f7
|
|
LFPDUX A2, AO2, INC4
|
|
fxcpmadd f11, B4, A4, f11
|
|
#ifndef TRMMKERNEL
|
|
LFPDUX A5, CO2, INC4
|
|
#else
|
|
nop
|
|
#endif
|
|
fxcsmadd f15, B4, A4, f15
|
|
nop
|
|
|
|
## 3 ##
|
|
|
|
fxcpmadd f0, B5, A7, f0
|
|
nop
|
|
fxcsmadd f4, B5, A7, f4
|
|
nop
|
|
fxcpmadd f8, B2, A7, f8
|
|
LFPDUX B4, BO2, INC4
|
|
fxcsmadd f12, B2, A7, f12
|
|
#ifndef TRMMKERNEL
|
|
LFPDUX B3, CO3, INC2
|
|
#else
|
|
nop
|
|
#endif
|
|
|
|
fxcpmadd f1, B5, A2, f1
|
|
nop
|
|
fxcsmadd f5, B5, A2, f5
|
|
LFPDUX A4, AO2, INC4
|
|
fxcpmadd f9, B2, A2, f9
|
|
#ifndef TRMMKERNEL
|
|
LFPDUX A6, CO3, INC4
|
|
#else
|
|
nop
|
|
#endif
|
|
fxcsmadd f13, B2, A2, f13
|
|
nop
|
|
|
|
fxcpmadd f2, B5, A8, f2
|
|
nop
|
|
fxcsmadd f6, B5, A8, f6
|
|
nop
|
|
fxcpmadd f10, B2, A8, f10
|
|
nop
|
|
fxcsmadd f14, B2, A8, f14
|
|
nop
|
|
|
|
fxcpmadd f3, B5, A4, f3
|
|
nop
|
|
fxcsmadd f7, B5, A4, f7
|
|
LFPDUX A2, AO2, INC4
|
|
fxcpmadd f11, B2, A4, f11
|
|
#ifndef TRMMKERNEL
|
|
LFPDUX A7, CO4, INC2
|
|
#else
|
|
nop
|
|
#endif
|
|
fxcsmadd f15, B2, A4, f15
|
|
nop
|
|
|
|
## 4 ##
|
|
|
|
fxcpmadd f0, B6, A9, f0
|
|
nop
|
|
fxcsmadd f4, B6, A9, f4
|
|
nop
|
|
fxcpmadd f8, B4, A9, f8
|
|
nop
|
|
fxcsmadd f12, B4, A9, f12
|
|
#ifndef TRMMKERNEL
|
|
LFPDUX B2, CO4, INC4
|
|
#else
|
|
nop
|
|
#endif
|
|
|
|
fxcpmadd f1, B6, A2, f1
|
|
nop
|
|
fxcsmadd f5, B6, A2, f5
|
|
LFPDUX A4, AO2, INC4
|
|
fxcpmadd f9, B4, A2, f9
|
|
#ifndef TRMMKERNEL
|
|
LFPDUX B5, CO1, INCM3
|
|
#else
|
|
nop
|
|
#endif
|
|
fxcsmadd f13, B4, A2, f13
|
|
nop
|
|
|
|
fxcpmadd f2, B6, A10, f2
|
|
nop
|
|
fxcsmadd f6, B6, A10, f6
|
|
nop
|
|
fxcpmadd f10, B4, A10, f10
|
|
nop
|
|
fxcsmadd f14, B4, A10, f14
|
|
#ifndef TRMMKERNEL
|
|
LFPDUX A8, CO1, INC4
|
|
#else
|
|
nop
|
|
#endif
|
|
|
|
fxcpmadd f3, B6, A4, f3
|
|
nop
|
|
fxcsmadd f7, B6, A4, f7
|
|
nop
|
|
fxcpmadd f11, B4, A4, f11
|
|
nop
|
|
fxcsmadd f15, B4, A4, f15
|
|
#ifndef TRMMKERNEL
|
|
LFPDUX A9, CO2, INCM3
|
|
#else
|
|
nop
|
|
#endif
|
|
.align 4
|
|
|
|
.L14:
|
|
lfd AP, ALPHA(SP)
|
|
#ifdef TRMMKERNEL
|
|
fsmfp AP, AP
|
|
#endif
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 8
|
|
#else
|
|
addi TEMP, KK, 4
|
|
#endif
|
|
andi. r0, TEMP, 3
|
|
mtspr CTR, r0
|
|
ble+ .L18
|
|
|
|
cmpwi cr0, TEMP, 3
|
|
bgt+ .L15
|
|
#else
|
|
andi. r0, K, 3
|
|
mtspr CTR, r0
|
|
ble+ .L18
|
|
|
|
cmpwi cr0, K, 3
|
|
bgt+ .L15
|
|
#endif
|
|
|
|
#ifndef TRMMKERNEL
|
|
LFPDUX A1, CO1, INC2
|
|
fpmr f5, f0
|
|
LFPDUX B1, CO1, INC4
|
|
fpmr f9, f0
|
|
LFPDUX A3, CO2, INC2
|
|
fpmr f13, f0
|
|
LFPDUX A5, CO2, INC4
|
|
fpmr f2, f0
|
|
|
|
LFPDUX B3, CO3, INC2
|
|
fpmr f6, f0
|
|
LFPDUX A6, CO3, INC4
|
|
fpmr f10, f0
|
|
LFPDUX A7, CO4, INC2
|
|
fpmr f14, f0
|
|
LFPDUX B2, CO4, INC4
|
|
fpmr f3, f0
|
|
|
|
LFPDUX B5, CO1, INCM3
|
|
fpmr f7, f0
|
|
LFPDUX A8, CO1, INC4
|
|
fpmr f11, f0
|
|
LFPDUX A9, CO2, INCM3
|
|
fpmr f15, f0
|
|
#else
|
|
fpmr f5, f0
|
|
fpmr f9, f0
|
|
fpmr f13, f0
|
|
fpmr f2, f0
|
|
|
|
fpmr f6, f0
|
|
fpmr f10, f0
|
|
fpmr f14, f0
|
|
fpmr f3, f0
|
|
|
|
fpmr f7, f0
|
|
fpmr f11, f0
|
|
fpmr f15, f0
|
|
nop
|
|
#endif
|
|
.align 4
|
|
|
|
.L15:
|
|
LFPDUX A2, AO, INC4
|
|
LFPDUX A4, AO2, INC4
|
|
LFPDUX A10, BO, INC4
|
|
LFPDUX B4, BO2, INC4
|
|
bdz- .L17
|
|
.align 4
|
|
|
|
.L16:
|
|
fxcpmadd f0, A10, A2, f0
|
|
fxcsmadd f4, A10, A2, f4
|
|
fxcpmadd f8, B4, A2, f8
|
|
fxcsmadd f12, B4, A2, f12
|
|
LFPDUX A2, AO, INC4
|
|
|
|
fxcpmadd f1, A10, A4, f1
|
|
fxcsmadd f5, A10, A4, f5
|
|
fxcpmadd f9, B4, A4, f9
|
|
fxcsmadd f13, B4, A4, f13
|
|
LFPDUX A4, AO2, INC4
|
|
|
|
fxcpmadd f2, A10, A2, f2
|
|
fxcsmadd f6, A10, A2, f6
|
|
fxcpmadd f10, B4, A2, f10
|
|
fxcsmadd f14, B4, A2, f14
|
|
LFPDUX A2, AO, INC4
|
|
|
|
fxcpmadd f3, A10, A4, f3
|
|
fxcsmadd f7, A10, A4, f7
|
|
LFPDUX A10, BO, INC4
|
|
fxcpmadd f11, B4, A4, f11
|
|
fxcsmadd f15, B4, A4, f15
|
|
LFPDUX A4, AO2, INC4
|
|
LFPDUX B4, BO2, INC4
|
|
bdnz+ .L16
|
|
.align 4
|
|
|
|
.L17:
|
|
fxcpmadd f0, A10, A2, f0
|
|
fxcsmadd f4, A10, A2, f4
|
|
fxcpmadd f8, B4, A2, f8
|
|
fxcsmadd f12, B4, A2, f12
|
|
LFPDUX A2, AO, INC4
|
|
|
|
fxcpmadd f1, A10, A4, f1
|
|
fxcsmadd f5, A10, A4, f5
|
|
fxcpmadd f9, B4, A4, f9
|
|
fxcsmadd f13, B4, A4, f13
|
|
LFPDUX A4, AO2, INC4
|
|
|
|
fxcpmadd f2, A10, A2, f2
|
|
fxcsmadd f6, A10, A2, f6
|
|
fxcpmadd f10, B4, A2, f10
|
|
fxcsmadd f14, B4, A2, f14
|
|
|
|
fxcpmadd f3, A10, A4, f3
|
|
fxcsmadd f7, A10, A4, f7
|
|
fxcpmadd f11, B4, A4, f11
|
|
fxcsmadd f15, B4, A4, f15
|
|
.align 4
|
|
|
|
.L18:
|
|
#ifndef TRMMKERNEL
|
|
fxcpmadd f0, AP, f0, A1
|
|
LFPDUX B4, CO2, INC4
|
|
fxcpmadd f1, AP, f1, B5
|
|
LFPDUX A2, CO3, INCM3
|
|
|
|
fxcpmadd f2, AP, f2, B1
|
|
LFPDUX A4, CO3, INC4
|
|
fxcpmadd f3, AP, f3, A8
|
|
LFPDUX A10, CO4, INCM3
|
|
|
|
fxcpmadd f4, AP, f4, A3
|
|
LFPDUX A1, CO4, INC4
|
|
fxcpmadd f5, AP, f5, A9
|
|
STFPDUX f0, CO1, INCM7
|
|
|
|
fxcpmadd f6, AP, f6, A5
|
|
STFPDUX f1, CO1, INC2
|
|
fxcpmadd f7, AP, f7, B4
|
|
STFPDUX f2, CO1, INC2
|
|
|
|
fxcpmadd f8, AP, f8, B3
|
|
STFPDUX f3, CO1, INC2
|
|
fxcpmadd f9, AP, f9, A2
|
|
STFPDUX f4, CO2, INCM7
|
|
|
|
fxcpmadd f10, AP, f10, A6
|
|
STFPDUX f5, CO2, INC2
|
|
fxcpmadd f11, AP, f11, A4
|
|
STFPDUX f6, CO2, INC2
|
|
|
|
fxcpmadd f12, AP, f12, A7
|
|
STFPDUX f7, CO2, INC2
|
|
fxcpmadd f13, AP, f13, A10
|
|
STFPDUX f8, CO3, INCM7
|
|
|
|
fxcpmadd f14, AP, f14, B2
|
|
STFPDUX f9, CO3, INC2
|
|
fxcpmadd f15, AP, f15, A1
|
|
STFPDUX f10, CO3, INC2
|
|
|
|
STFPDUX f11, CO3, INC2
|
|
STFPDUX f12, CO4, INCM7
|
|
STFPDUX f13, CO4, INC2
|
|
STFPDUX f14, CO4, INC2
|
|
STFPDUX f15, CO4, INC2
|
|
#else
|
|
fpmul f0, AP, f0
|
|
fpmul f1, AP, f1
|
|
fpmul f2, AP, f2
|
|
fpmul f3, AP, f3
|
|
|
|
fpmul f4, AP, f4
|
|
fpmul f5, AP, f5
|
|
STFPDUX f0, CO1, INC2
|
|
|
|
fpmul f6, AP, f6
|
|
STFPDUX f1, CO1, INC2
|
|
fpmul f7, AP, f7
|
|
STFPDUX f2, CO1, INC2
|
|
|
|
fpmul f8, AP, f8
|
|
STFPDUX f3, CO1, INC2
|
|
fpmul f9, AP, f9
|
|
STFPDUX f4, CO2, INC2
|
|
|
|
fpmul f10, AP, f10
|
|
STFPDUX f5, CO2, INC2
|
|
fpmul f11, AP, f11
|
|
STFPDUX f6, CO2, INC2
|
|
|
|
fpmul f12, AP, f12
|
|
STFPDUX f7, CO2, INC2
|
|
fpmul f13, AP, f13
|
|
STFPDUX f8, CO3, INC2
|
|
|
|
fpmul f14, AP, f14
|
|
STFPDUX f9, CO3, INC2
|
|
fpmul f15, AP, f15
|
|
STFPDUX f10, CO3, INC2
|
|
|
|
STFPDUX f11, CO3, INC2
|
|
STFPDUX f12, CO4, INC2
|
|
STFPDUX f13, CO4, INC2
|
|
STFPDUX f14, CO4, INC2
|
|
STFPDUX f15, CO4, INC2
|
|
#endif
|
|
|
|
#ifdef TRMMKERNEL
|
|
#if ( defined(LEFT) && defined(TRANSA)) || \
|
|
(!defined(LEFT) && !defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#ifdef LEFT
|
|
addi TEMP, TEMP, -8
|
|
#else
|
|
addi TEMP, TEMP, -4
|
|
#endif
|
|
slwi r0, TEMP, 3 + BASE_SHIFT
|
|
slwi TEMP, TEMP, 2 + BASE_SHIFT
|
|
add AO, AO, r0
|
|
add BO, BO, TEMP
|
|
#endif
|
|
|
|
#ifdef LEFT
|
|
addi KK, KK, 8
|
|
#endif
|
|
#endif
|
|
|
|
addic. I, I, -1
|
|
li r0, FZERO
|
|
|
|
lfpsx f0, SP, r0
|
|
bgt+ .L11
|
|
.align 4
|
|
|
|
.L20:
|
|
andi. I, M, 4
|
|
beq .L30
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
|
addi AO2, AO, 2 * SIZE
|
|
fpmr f4, f0
|
|
addi BO, B, - 4 * SIZE
|
|
fpmr f8, f0
|
|
addi BO2, B, - 2 * SIZE
|
|
fpmr f12, f0
|
|
#else
|
|
slwi TEMP, KK, 2 + BASE_SHIFT
|
|
slwi r0, KK, 2 + BASE_SHIFT
|
|
add AO, AO, TEMP
|
|
add BO, B, r0
|
|
|
|
addi AO2, AO, 2 * SIZE
|
|
fpmr f4, f0
|
|
addi BO, BO, - 4 * SIZE
|
|
fpmr f8, f0
|
|
addi BO2, BO, 2 * SIZE
|
|
fpmr f12, f0
|
|
#endif
|
|
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 4
|
|
#else
|
|
addi TEMP, KK, 4
|
|
#endif
|
|
|
|
srawi. TEMP, TEMP, 2
|
|
fpmr f1, f0
|
|
fpmr f5, f0
|
|
fpmr f9, f0
|
|
mtspr CTR, TEMP
|
|
fpmr f13, f0
|
|
ble .L24
|
|
#else
|
|
addi AO2, AO, 2 * SIZE
|
|
fpmr f4, f0
|
|
addi BO, B, - 4 * SIZE
|
|
fpmr f8, f0
|
|
addi BO2, B, - 2 * SIZE
|
|
fpmr f12, f0
|
|
|
|
srawi. r0, K, 2
|
|
fpmr f1, f0
|
|
fpmr f5, f0
|
|
fpmr f9, f0
|
|
mtspr CTR, r0
|
|
fpmr f13, f0
|
|
ble .L24
|
|
#endif
|
|
|
|
LFPDUX A1, AO, INC4
|
|
LFPDUX B1, BO, INC4
|
|
LFPDUX A2, AO2, INC4
|
|
LFPDUX B2, BO2, INC4
|
|
LFPDUX A3, AO, INC4
|
|
LFPDUX B3, BO, INC4
|
|
LFPDUX A4, AO2, INC4
|
|
LFPDUX B4, BO2, INC4
|
|
|
|
LFPDUX A5, AO, INC4
|
|
LFPDUX B5, BO, INC4
|
|
LFPDUX A6, AO2, INC4
|
|
LFPDUX B6, BO2, INC4
|
|
LFPDUX A7, AO, INC4
|
|
LFPDUX A9, BO, INC4
|
|
LFPDUX A10, BO2, INC4
|
|
bdz- .L23
|
|
.align 4
|
|
|
|
.L22:
|
|
fxcpmadd f0, B1, A1, f0
|
|
nop
|
|
fxcsmadd f4, B1, A1, f4
|
|
LFPDUX A8, AO2, INC4
|
|
fxcpmadd f8, B2, A1, f8
|
|
nop
|
|
fxcsmadd f12, B2, A1, f12
|
|
LFPDUX A1, AO, INC4
|
|
|
|
fxcpmadd f1, B1, A2, f1
|
|
nop
|
|
fxcsmadd f5, B1, A2, f5
|
|
LFPDUX B1, BO, INC4
|
|
fxcpmadd f9, B2, A2, f9
|
|
nop
|
|
fxcsmadd f13, B2, A2, f13
|
|
LFPDUX B2, BO2, INC4
|
|
|
|
fxcpmadd f0, B3, A3, f0
|
|
nop
|
|
fxcsmadd f4, B3, A3, f4
|
|
LFPDUX A2, AO2, INC4
|
|
fxcpmadd f8, B4, A3, f8
|
|
nop
|
|
fxcsmadd f12, B4, A3, f12
|
|
LFPDUX A3, AO, INC4
|
|
|
|
fxcpmadd f1, B3, A4, f1
|
|
nop
|
|
fxcsmadd f5, B3, A4, f5
|
|
LFPDUX B3, BO, INC4
|
|
fxcpmadd f9, B4, A4, f9
|
|
nop
|
|
fxcsmadd f13, B4, A4, f13
|
|
LFPDUX B4, BO2, INC4
|
|
|
|
fxcpmadd f0, B5, A5, f0
|
|
nop
|
|
fxcsmadd f4, B5, A5, f4
|
|
LFPDUX A4, AO2, INC4
|
|
fxcpmadd f8, B6, A5, f8
|
|
nop
|
|
fxcsmadd f12, B6, A5, f12
|
|
LFPDUX A5, AO, INC4
|
|
|
|
fxcpmadd f1, B5, A6, f1
|
|
nop
|
|
fxcsmadd f5, B5, A6, f5
|
|
LFPDUX B5, BO, INC4
|
|
fxcpmadd f9, B6, A6, f9
|
|
nop
|
|
fxcsmadd f13, B6, A6, f13
|
|
LFPDUX B6, BO2, INC4
|
|
|
|
fxcpmadd f0, A9, A7, f0
|
|
nop
|
|
fxcsmadd f4, A9, A7, f4
|
|
LFPDUX A6, AO2, INC4
|
|
fxcpmadd f8, A10, A7, f8
|
|
nop
|
|
fxcsmadd f12, A10, A7, f12
|
|
LFPDUX A7, AO, INC4
|
|
|
|
fxcpmadd f1, A9, A8, f1
|
|
nop
|
|
fxcsmadd f5, A9, A8, f5
|
|
LFPDUX A9, BO, INC4
|
|
fxcpmadd f9, A10, A8, f9
|
|
nop
|
|
fxcsmadd f13, A10, A8, f13
|
|
LFPDUX A10, BO2, INC4
|
|
bdnz+ .L22
|
|
.align 4
|
|
|
|
.L23:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f4, B1, A1, f4
|
|
LFPDUX A8, AO2, INC4
|
|
fxcpmadd f8, B2, A1, f8
|
|
fxcsmadd f12, B2, A1, f12
|
|
|
|
fxcpmadd f1, B1, A2, f1
|
|
fxcsmadd f5, B1, A2, f5
|
|
fxcpmadd f9, B2, A2, f9
|
|
fxcsmadd f13, B2, A2, f13
|
|
|
|
fxcpmadd f0, B3, A3, f0
|
|
fxcsmadd f4, B3, A3, f4
|
|
fxcpmadd f8, B4, A3, f8
|
|
fxcsmadd f12, B4, A3, f12
|
|
|
|
fxcpmadd f1, B3, A4, f1
|
|
fxcsmadd f5, B3, A4, f5
|
|
fxcpmadd f9, B4, A4, f9
|
|
fxcsmadd f13, B4, A4, f13
|
|
|
|
fxcpmadd f0, B5, A5, f0
|
|
fxcsmadd f4, B5, A5, f4
|
|
fxcpmadd f8, B6, A5, f8
|
|
fxcsmadd f12, B6, A5, f12
|
|
|
|
fxcpmadd f1, B5, A6, f1
|
|
fxcsmadd f5, B5, A6, f5
|
|
fxcpmadd f9, B6, A6, f9
|
|
fxcsmadd f13, B6, A6, f13
|
|
|
|
fxcpmadd f0, A9, A7, f0
|
|
fxcsmadd f4, A9, A7, f4
|
|
fxcpmadd f8, A10, A7, f8
|
|
fxcsmadd f12, A10, A7, f12
|
|
|
|
fxcpmadd f1, A9, A8, f1
|
|
fxcsmadd f5, A9, A8, f5
|
|
fxcpmadd f9, A10, A8, f9
|
|
fxcsmadd f13, A10, A8, f13
|
|
.align 4
|
|
|
|
.L24:
|
|
lfd AP, ALPHA(SP)
|
|
#ifdef TRMMKERNEL
|
|
fsmfp AP, AP
|
|
#endif
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 4
|
|
#else
|
|
addi TEMP, KK, 4
|
|
#endif
|
|
andi. TEMP, TEMP, 3
|
|
mtspr CTR, TEMP
|
|
#else
|
|
andi. r0, K, 3
|
|
mtspr CTR, r0
|
|
#endif
|
|
ble+ .L28
|
|
|
|
LFPDUX A1, AO, INC4
|
|
LFPDUX A2, AO2, INC4
|
|
LFPDUX B1, BO, INC4
|
|
LFPDUX B2, BO2, INC4
|
|
bdz- .L27
|
|
.align 4
|
|
|
|
.L26:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f4, B1, A1, f4
|
|
fxcpmadd f8, B2, A1, f8
|
|
fxcsmadd f12, B2, A1, f12
|
|
LFPDUX A1, AO, INC4
|
|
|
|
fxcpmadd f1, B1, A2, f1
|
|
fxcsmadd f5, B1, A2, f5
|
|
LFPDUX B1, BO, INC4
|
|
fxcpmadd f9, B2, A2, f9
|
|
fxcsmadd f13, B2, A2, f13
|
|
LFPDUX A2, AO2, INC4
|
|
LFPDUX B2, BO2, INC4
|
|
bdnz+ .L26
|
|
.align 4
|
|
|
|
.L27:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f4, B1, A1, f4
|
|
fxcpmadd f8, B2, A1, f8
|
|
fxcsmadd f12, B2, A1, f12
|
|
|
|
fxcpmadd f1, B1, A2, f1
|
|
fxcsmadd f5, B1, A2, f5
|
|
fxcpmadd f9, B2, A2, f9
|
|
fxcsmadd f13, B2, A2, f13
|
|
.align 4
|
|
|
|
.L28:
|
|
#ifndef TRMMKERNEL
|
|
LFPDUX A1, CO1, INC2
|
|
LFPDUX B1, CO1, INC2
|
|
LFPDUX B3, CO2, INC2
|
|
LFPDUX A6, CO2, INC2
|
|
|
|
LFPDUX B5, CO3, INC2
|
|
LFPDUX A8, CO3, INC2
|
|
LFPDUX A2, CO4, INC2
|
|
LFPDUX A4, CO4, INC2
|
|
|
|
fxcpmadd f0, AP, f0, A1
|
|
fxcpmadd f1, AP, f1, B1
|
|
fxcpmadd f4, AP, f4, B3
|
|
fxcpmadd f5, AP, f5, A6
|
|
|
|
fxcpmadd f8, AP, f8, B5
|
|
fxcpmadd f9, AP, f9, A8
|
|
STFPDUX f0, CO1, INCM3
|
|
fxcpmadd f12, AP, f12, A2
|
|
STFPDUX f1, CO1, INC2
|
|
fxcpmadd f13, AP, f13, A4
|
|
STFPDUX f4, CO2, INCM3
|
|
|
|
STFPDUX f5, CO2, INC2
|
|
STFPDUX f8, CO3, INCM3
|
|
STFPDUX f9, CO3, INC2
|
|
STFPDUX f12, CO4, INCM3
|
|
STFPDUX f13, CO4, INC2
|
|
#else
|
|
fpmul f0, AP, f0
|
|
fpmul f1, AP, f1
|
|
fpmul f4, AP, f4
|
|
fpmul f5, AP, f5
|
|
|
|
fpmul f8, AP, f8
|
|
fpmul f9, AP, f9
|
|
STFPDUX f0, CO1, INC2
|
|
fpmul f12, AP, f12
|
|
STFPDUX f1, CO1, INC2
|
|
fpmul f13, AP, f13
|
|
STFPDUX f4, CO2, INC2
|
|
|
|
STFPDUX f5, CO2, INC2
|
|
STFPDUX f8, CO3, INC2
|
|
STFPDUX f9, CO3, INC2
|
|
STFPDUX f12, CO4, INC2
|
|
STFPDUX f13, CO4, INC2
|
|
#endif
|
|
|
|
|
|
#ifdef TRMMKERNEL
|
|
#if ( defined(LEFT) && defined(TRANSA)) || \
|
|
(!defined(LEFT) && !defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#ifdef LEFT
|
|
addi TEMP, TEMP, -4
|
|
#else
|
|
addi TEMP, TEMP, -4
|
|
#endif
|
|
slwi r0, TEMP, 2 + BASE_SHIFT
|
|
slwi TEMP, TEMP, 2 + BASE_SHIFT
|
|
add AO, AO, r0
|
|
add BO, BO, TEMP
|
|
#endif
|
|
|
|
#ifdef LEFT
|
|
addi KK, KK, 4
|
|
#endif
|
|
#endif
|
|
|
|
li r0, FZERO
|
|
lfpsx f0, SP, r0
|
|
.align 4
|
|
|
|
.L30:
|
|
andi. I, M, 2
|
|
beq .L40
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
|
addi AO2, AO, 2 * SIZE
|
|
fpmr f1, f0
|
|
addi BO, B, - 4 * SIZE
|
|
fpmr f2, f0
|
|
addi BO2, B, - 2 * SIZE
|
|
fpmr f3, f0
|
|
#else
|
|
slwi TEMP, KK, 1 + BASE_SHIFT
|
|
slwi r0, KK, 2 + BASE_SHIFT
|
|
add AO, AO, TEMP
|
|
add BO, B, r0
|
|
|
|
addi AO2, AO, 2 * SIZE
|
|
fpmr f1, f0
|
|
addi BO, BO, - 4 * SIZE
|
|
fpmr f2, f0
|
|
addi BO2, BO, 2 * SIZE
|
|
fpmr f3, f0
|
|
#endif
|
|
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 2
|
|
#else
|
|
addi TEMP, KK, 4
|
|
#endif
|
|
|
|
srawi. r0, TEMP, 2
|
|
mtspr CTR, r0
|
|
ble .L34
|
|
|
|
#else
|
|
addi AO2, AO, 2 * SIZE
|
|
fpmr f1, f0
|
|
addi BO, B, - 4 * SIZE
|
|
fpmr f2, f0
|
|
addi BO2, B, - 2 * SIZE
|
|
fpmr f3, f0
|
|
|
|
srawi. r0, K, 2
|
|
mtspr CTR, r0
|
|
ble .L34
|
|
#endif
|
|
|
|
LFPDUX A1, AO, INC4
|
|
LFPDUX B1, BO, INC4
|
|
LFPDUX B2, BO2, INC4
|
|
LFPDUX A2, AO2, INC4
|
|
LFPDUX B3, BO, INC4
|
|
LFPDUX B4, BO2, INC4
|
|
|
|
LFPDUX A3, AO, INC4
|
|
LFPDUX A5, BO, INC4
|
|
LFPDUX A6, BO2, INC4
|
|
LFPDUX A4, AO2, INC4
|
|
LFPDUX A7, BO, INC4
|
|
LFPDUX A8, BO2, INC4
|
|
bdz- .L33
|
|
.align 4
|
|
|
|
.L32:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f1, B1, A1, f1
|
|
LFPDUX B1, BO, INC4
|
|
fxcpmadd f2, B2, A1, f2
|
|
fxcsmadd f3, B2, A1, f3
|
|
LFPDUX B2, BO2, INC4
|
|
LFPDUX A1, AO, INC4
|
|
|
|
fxcpmadd f0, B3, A2, f0
|
|
fxcsmadd f1, B3, A2, f1
|
|
LFPDUX B3, BO, INC4
|
|
fxcpmadd f2, B4, A2, f2
|
|
fxcsmadd f3, B4, A2, f3
|
|
LFPDUX B4, BO2, INC4
|
|
LFPDUX A2, AO2, INC4
|
|
|
|
fxcpmadd f0, A5, A3, f0
|
|
fxcsmadd f1, A5, A3, f1
|
|
LFPDUX A5, BO, INC4
|
|
fxcpmadd f2, A6, A3, f2
|
|
fxcsmadd f3, A6, A3, f3
|
|
LFPDUX A6, BO2, INC4
|
|
LFPDUX A3, AO, INC4
|
|
|
|
fxcpmadd f0, A7, A4, f0
|
|
fxcsmadd f1, A7, A4, f1
|
|
LFPDUX A7, BO, INC4
|
|
fxcpmadd f2, A8, A4, f2
|
|
fxcsmadd f3, A8, A4, f3
|
|
LFPDUX A8, BO2, INC4
|
|
LFPDUX A4, AO2, INC4
|
|
bdnz+ .L32
|
|
.align 4
|
|
|
|
.L33:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f1, B1, A1, f1
|
|
fxcpmadd f2, B2, A1, f2
|
|
fxcsmadd f3, B2, A1, f3
|
|
|
|
fxcpmadd f0, B3, A2, f0
|
|
fxcsmadd f1, B3, A2, f1
|
|
fxcpmadd f2, B4, A2, f2
|
|
fxcsmadd f3, B4, A2, f3
|
|
|
|
fxcpmadd f0, A5, A3, f0
|
|
fxcsmadd f1, A5, A3, f1
|
|
fxcpmadd f2, A6, A3, f2
|
|
fxcsmadd f3, A6, A3, f3
|
|
|
|
fxcpmadd f0, A7, A4, f0
|
|
fxcsmadd f1, A7, A4, f1
|
|
fxcpmadd f2, A8, A4, f2
|
|
fxcsmadd f3, A8, A4, f3
|
|
.align 4
|
|
|
|
.L34:
|
|
lfd AP, ALPHA(SP)
|
|
#ifdef TRMMKERNEL
|
|
fsmfp AP, AP
|
|
#endif
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 2
|
|
#else
|
|
addi TEMP, KK, 4
|
|
#endif
|
|
andi. TEMP, TEMP, 3
|
|
mtspr CTR, TEMP
|
|
#else
|
|
andi. r0, K, 3
|
|
mtspr CTR, r0
|
|
#endif
|
|
ble+ .L38
|
|
|
|
LFPDX A1, AO, INC4
|
|
LFPDUX B1, BO, INC4
|
|
LFPDUX B2, BO2, INC4
|
|
add AO, AO, INC2
|
|
bdz- .L37
|
|
.align 4
|
|
|
|
.L36:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f1, B1, A1, f1
|
|
LFPDUX B1, BO, INC4
|
|
fxcpmadd f2, B2, A1, f2
|
|
fxcsmadd f3, B2, A1, f3
|
|
LFPDX A1, AO, INC4
|
|
LFPDUX B2, BO2, INC4
|
|
add AO, AO, INC2
|
|
bdnz+ .L36
|
|
.align 4
|
|
|
|
.L37:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f1, B1, A1, f1
|
|
fxcpmadd f2, B2, A1, f2
|
|
fxcsmadd f3, B2, A1, f3
|
|
.align 4
|
|
|
|
.L38:
|
|
#ifndef TRMMKERNEL
|
|
LFPDX A1, CO1, INC2
|
|
LFPDX A2, CO2, INC2
|
|
LFPDX A3, CO3, INC2
|
|
LFPDX A4, CO4, INC2
|
|
|
|
fxcpmadd f0, AP, f0, A1
|
|
fxcpmadd f1, AP, f1, A2
|
|
fxcpmadd f2, AP, f2, A3
|
|
fxcpmadd f3, AP, f3, A4
|
|
#else
|
|
fpmul f0, AP, f0
|
|
fpmul f1, AP, f1
|
|
fpmul f2, AP, f2
|
|
fpmul f3, AP, f3
|
|
#endif
|
|
|
|
STFPDUX f0, CO1, INC2
|
|
STFPDUX f1, CO2, INC2
|
|
STFPDUX f2, CO3, INC2
|
|
STFPDUX f3, CO4, INC2
|
|
|
|
#ifdef TRMMKERNEL
|
|
#if ( defined(LEFT) && defined(TRANSA)) || \
|
|
(!defined(LEFT) && !defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#ifdef LEFT
|
|
addi TEMP, TEMP, -2
|
|
#else
|
|
addi TEMP, TEMP, -4
|
|
#endif
|
|
slwi r0, TEMP, 1 + BASE_SHIFT
|
|
slwi TEMP, TEMP, 2 + BASE_SHIFT
|
|
add AO, AO, r0
|
|
add BO, BO, TEMP
|
|
#endif
|
|
|
|
#ifdef LEFT
|
|
addi KK, KK, 2
|
|
#endif
|
|
#endif
|
|
|
|
li r0, FZERO
|
|
lfpsx f0, SP, r0
|
|
.align 4
|
|
|
|
.L40:
|
|
andi. I, M, 1
|
|
beq .L49
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
|
addi AO2, AO, 2 * SIZE
|
|
fpmr f1, f0
|
|
addi BO, B, - 4 * SIZE
|
|
fpmr f2, f0
|
|
addi BO2, B, - 2 * SIZE
|
|
fpmr f3, f0
|
|
#else
|
|
slwi TEMP, KK, 0 + BASE_SHIFT
|
|
slwi r0, KK, 2 + BASE_SHIFT
|
|
add AO, AO, TEMP
|
|
add BO, B, r0
|
|
|
|
addi AO2, AO, 2 * SIZE
|
|
fpmr f1, f0
|
|
addi BO, BO, - 4 * SIZE
|
|
fpmr f2, f0
|
|
addi BO2, BO, 2 * SIZE
|
|
fpmr f3, f0
|
|
#endif
|
|
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 1
|
|
#else
|
|
addi TEMP, KK, 4
|
|
#endif
|
|
srawi. r0, TEMP, 3
|
|
mtspr CTR, r0
|
|
ble .L44
|
|
|
|
#else
|
|
addi AO2, AO, 2 * SIZE
|
|
fpmr f1, f0
|
|
addi BO, B, - 4 * SIZE
|
|
fpmr f2, f0
|
|
addi BO2, B, - 2 * SIZE
|
|
fpmr f3, f0
|
|
|
|
srawi. r0, K, 3
|
|
mtspr CTR, r0
|
|
ble .L44
|
|
#endif
|
|
|
|
LFPDUX A1, AO, INC4
|
|
LFPDUX B1, BO, INC4
|
|
LFPDUX B2, BO2, INC4
|
|
LFPDUX A2, AO2, INC4
|
|
LFPDUX B3, BO, INC4
|
|
LFPDUX B4, BO2, INC4
|
|
|
|
LFPDUX A3, AO, INC4
|
|
LFPDUX A5, BO, INC4
|
|
LFPDUX A6, BO2, INC4
|
|
LFPDUX A4, AO2, INC4
|
|
LFPDUX A7, BO, INC4
|
|
LFPDUX A8, BO2, INC4
|
|
bdz- .L43
|
|
.align 4
|
|
|
|
.L42:
|
|
fxcpmadd f0, A1, B1, f0
|
|
LFPDUX B1, BO, INC4
|
|
fxcpmadd f1, A1, B2, f1
|
|
LFPDUX B2, BO2, INC4
|
|
fxcsmadd f2, A1, B3, f2
|
|
LFPDUX B3, BO, INC4
|
|
fxcsmadd f3, A1, B4, f3
|
|
LFPDUX B4, BO2, INC4
|
|
LFPDUX A1, AO, INC4
|
|
|
|
fxcpmadd f0, A2, A5, f0
|
|
LFPDUX A5, BO, INC4
|
|
fxcpmadd f1, A2, A6, f1
|
|
LFPDUX A6, BO2, INC4
|
|
fxcsmadd f2, A2, A7, f2
|
|
LFPDUX A7, BO, INC4
|
|
fxcsmadd f3, A2, A8, f3
|
|
LFPDUX A8, BO2, INC4
|
|
LFPDUX A2, AO2, INC4
|
|
|
|
fxcpmadd f0, A3, B1, f0
|
|
LFPDUX B1, BO, INC4
|
|
fxcpmadd f1, A3, B2, f1
|
|
LFPDUX B2, BO2, INC4
|
|
fxcsmadd f2, A3, B3, f2
|
|
LFPDUX B3, BO, INC4
|
|
fxcsmadd f3, A3, B4, f3
|
|
LFPDUX B4, BO2, INC4
|
|
LFPDUX A3, AO, INC4
|
|
|
|
fxcpmadd f0, A4, A5, f0
|
|
LFPDUX A5, BO, INC4
|
|
fxcpmadd f1, A4, A6, f1
|
|
LFPDUX A6, BO2, INC4
|
|
fxcsmadd f2, A4, A7, f2
|
|
LFPDUX A7, BO, INC4
|
|
fxcsmadd f3, A4, A8, f3
|
|
LFPDUX A8, BO2, INC4
|
|
LFPDUX A4, AO2, INC4
|
|
bdnz+ .L42
|
|
.align 4
|
|
|
|
.L43:
|
|
fxcpmadd f0, A1, B1, f0
|
|
LFPDUX B1, BO, INC4
|
|
fxcpmadd f1, A1, B2, f1
|
|
LFPDUX B2, BO2, INC4
|
|
fxcsmadd f2, A1, B3, f2
|
|
LFPDUX B3, BO, INC4
|
|
fxcsmadd f3, A1, B4, f3
|
|
LFPDUX B4, BO2, INC4
|
|
|
|
fxcpmadd f0, A2, A5, f0
|
|
LFPDUX A5, BO, INC4
|
|
fxcpmadd f1, A2, A6, f1
|
|
LFPDUX A6, BO2, INC4
|
|
fxcsmadd f2, A2, A7, f2
|
|
LFPDUX A7, BO, INC4
|
|
fxcsmadd f3, A2, A8, f3
|
|
LFPDUX A8, BO2, INC4
|
|
|
|
fxcpmadd f0, A3, B1, f0
|
|
fxcpmadd f1, A3, B2, f1
|
|
fxcsmadd f2, A3, B3, f2
|
|
fxcsmadd f3, A3, B4, f3
|
|
|
|
fxcpmadd f0, A4, A5, f0
|
|
fxcpmadd f1, A4, A6, f1
|
|
fxcsmadd f2, A4, A7, f2
|
|
fxcsmadd f3, A4, A8, f3
|
|
.align 4
|
|
|
|
.L44:
|
|
lfd AP, ALPHA(SP)
|
|
#ifdef TRMMKERNEL
|
|
fsmfp AP, AP
|
|
#endif
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 1
|
|
#else
|
|
addi TEMP, KK, 4
|
|
#endif
|
|
andi. TEMP, TEMP, 7
|
|
mtspr CTR, TEMP
|
|
#else
|
|
andi. r0, K, 7
|
|
mtspr CTR, r0
|
|
#endif
|
|
ble+ .L48
|
|
|
|
LFDX A1, AO, INC4
|
|
LFPDUX B1, BO, INC4
|
|
LFPDUX B2, BO2, INC4
|
|
add AO, AO, INC
|
|
bdz- .L47
|
|
.align 4
|
|
|
|
.L46:
|
|
fxcpmadd f0, A1, B1, f0
|
|
LFPDUX B1, BO, INC4
|
|
fxcpmadd f1, A1, B2, f1
|
|
LFDX A1, AO, INC4
|
|
LFPDUX B2, BO2, INC4
|
|
add AO, AO, INC
|
|
bdnz+ .L46
|
|
.align 4
|
|
|
|
.L47:
|
|
fxcpmadd f0, A1, B1, f0
|
|
fxcpmadd f1, A1, B2, f1
|
|
.align 4
|
|
|
|
.L48:
|
|
#ifndef TRMMKERNEL
|
|
LFDX A1, CO1, INC2
|
|
LFDX A2, CO2, INC2
|
|
LFDX A3, CO3, INC2
|
|
LFDX A4, CO4, INC2
|
|
|
|
fpadd f0, f0, f2
|
|
fpadd f1, f1, f3
|
|
|
|
fsmfp A1, A2
|
|
fsmfp A3, A4
|
|
|
|
fxcpmadd f0, AP, f0, A1
|
|
fxcpmadd f1, AP, f1, A3
|
|
#else
|
|
fpadd f0, f0, f2
|
|
fpadd f1, f1, f3
|
|
|
|
fpmul f0, AP, f0
|
|
fpmul f1, AP, f1
|
|
#endif
|
|
|
|
STFDX f0, CO1, INC2
|
|
STFSDX f0, CO2, INC2
|
|
STFDX f1, CO3, INC2
|
|
STFSDX f1, CO4, INC2
|
|
|
|
#ifdef TRMMKERNEL
|
|
#if ( defined(LEFT) && defined(TRANSA)) || \
|
|
(!defined(LEFT) && !defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#ifdef LEFT
|
|
addi TEMP, TEMP, -1
|
|
#else
|
|
addi TEMP, TEMP, -4
|
|
#endif
|
|
slwi r0, TEMP, 0 + BASE_SHIFT
|
|
slwi TEMP, TEMP, 2 + BASE_SHIFT
|
|
add AO, AO, r0
|
|
add BO, BO, TEMP
|
|
#endif
|
|
|
|
#ifdef LEFT
|
|
addi KK, KK, 1
|
|
#endif
|
|
#endif
|
|
.align 4
|
|
|
|
.L49:
|
|
#if defined(TRMMKERNEL) && !defined(LEFT)
|
|
addi KK, KK, 4
|
|
#endif
|
|
|
|
addi B, BO, 4 * SIZE
|
|
|
|
addic. J, J, -1
|
|
bgt+ .L10
|
|
.align 4
|
|
|
|
.L50:
|
|
andi. J, N, 2
|
|
beq .L90
|
|
|
|
mr CO1, C
|
|
add CO2, C, LDC
|
|
add C, CO2, LDC
|
|
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
mr KK, OFFSET
|
|
#endif
|
|
|
|
addi AO, A, -2 * SIZE
|
|
|
|
li r0, FZERO
|
|
lfpsx f0, SP, r0
|
|
|
|
srawi. I, M, 3
|
|
ble .L60
|
|
.align 4
|
|
|
|
.L51:
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
|
fpmr f4, f0
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
fpmr f5, f0
|
|
fpmr f2, f0
|
|
fpmr f6, f0
|
|
#else
|
|
slwi TEMP, KK, 3 + BASE_SHIFT
|
|
slwi r0, KK, 1 + BASE_SHIFT
|
|
add AO, AO, TEMP
|
|
add BO, B, r0
|
|
|
|
fpmr f4, f0
|
|
addi BO, BO, - 2 * SIZE
|
|
fpmr f1, f0
|
|
fpmr f5, f0
|
|
fpmr f2, f0
|
|
fpmr f6, f0
|
|
#endif
|
|
|
|
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 8
|
|
#else
|
|
addi TEMP, KK, 2
|
|
#endif
|
|
srawi. r0, TEMP, 2
|
|
fpmr f3, f0
|
|
mtspr CTR, r0
|
|
fpmr f7, f0
|
|
ble .L54
|
|
#else
|
|
fpmr f4, f0
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
fpmr f5, f0
|
|
fpmr f2, f0
|
|
fpmr f6, f0
|
|
|
|
srawi. r0, K, 2
|
|
fpmr f3, f0
|
|
mtspr CTR, r0
|
|
fpmr f7, f0
|
|
ble .L54
|
|
#endif
|
|
|
|
LFPDUX B1, BO, INC2
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
LFPDUX A3, AO, INC2
|
|
LFPDUX A4, AO, INC2
|
|
|
|
LFPDUX B3, BO, INC2
|
|
LFPDUX A5, AO, INC2
|
|
LFPDUX A6, AO, INC2
|
|
LFPDUX A7, AO, INC2
|
|
LFPDUX A8, AO, INC2
|
|
bdz- .L53
|
|
.align 4
|
|
|
|
.L52:
|
|
fxcpmadd f0, B1, A1, f0
|
|
LFPDUX B4, BO, INC2
|
|
fxcsmadd f4, B1, A1, f4
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B1, A2, f1
|
|
nop
|
|
fxcsmadd f5, B1, A2, f5
|
|
LFPDUX A2, AO, INC2
|
|
|
|
fxcpmadd f2, B1, A3, f2
|
|
nop
|
|
fxcsmadd f6, B1, A3, f6
|
|
LFPDUX A3, AO, INC2
|
|
fxcpmadd f3, B1, A4, f3
|
|
nop
|
|
fxcsmadd f7, B1, A4, f7
|
|
LFPDUX A4, AO, INC2
|
|
|
|
fxcpmadd f0, B2, A5, f0
|
|
LFPDUX B1, BO, INC2
|
|
fxcsmadd f4, B2, A5, f4
|
|
LFPDUX A5, AO, INC2
|
|
fxcpmadd f1, B2, A6, f1
|
|
nop
|
|
fxcsmadd f5, B2, A6, f5
|
|
LFPDUX A6, AO, INC2
|
|
|
|
fxcpmadd f2, B2, A7, f2
|
|
nop
|
|
fxcsmadd f6, B2, A7, f6
|
|
LFPDUX A7, AO, INC2
|
|
fxcpmadd f3, B2, A8, f3
|
|
nop
|
|
fxcsmadd f7, B2, A8, f7
|
|
LFPDUX A8, AO, INC2
|
|
|
|
fxcpmadd f0, B3, A1, f0
|
|
LFPDUX B2, BO, INC2
|
|
fxcsmadd f4, B3, A1, f4
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B3, A2, f1
|
|
nop
|
|
fxcsmadd f5, B3, A2, f5
|
|
LFPDUX A2, AO, INC2
|
|
|
|
fxcpmadd f2, B3, A3, f2
|
|
nop
|
|
fxcsmadd f6, B3, A3, f6
|
|
LFPDUX A3, AO, INC2
|
|
fxcpmadd f3, B3, A4, f3
|
|
nop
|
|
fxcsmadd f7, B3, A4, f7
|
|
LFPDUX A4, AO, INC2
|
|
|
|
fxcpmadd f0, B4, A5, f0
|
|
LFPDUX B3, BO, INC2
|
|
fxcsmadd f4, B4, A5, f4
|
|
LFPDUX A5, AO, INC2
|
|
fxcpmadd f1, B4, A6, f1
|
|
nop
|
|
fxcsmadd f5, B4, A6, f5
|
|
LFPDUX A6, AO, INC2
|
|
|
|
fxcpmadd f2, B4, A7, f2
|
|
nop
|
|
fxcsmadd f6, B4, A7, f6
|
|
LFPDUX A7, AO, INC2
|
|
fxcpmadd f3, B4, A8, f3
|
|
nop
|
|
fxcsmadd f7, B4, A8, f7
|
|
LFPDUX A8, AO, INC2
|
|
bdnz+ .L52
|
|
.align 4
|
|
|
|
.L53:
|
|
fxcpmadd f0, B1, A1, f0
|
|
LFPDUX B4, BO, INC2
|
|
fxcsmadd f4, B1, A1, f4
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B1, A2, f1
|
|
nop
|
|
fxcsmadd f5, B1, A2, f5
|
|
LFPDUX A2, AO, INC2
|
|
|
|
fxcpmadd f2, B1, A3, f2
|
|
nop
|
|
fxcsmadd f6, B1, A3, f6
|
|
LFPDUX A3, AO, INC2
|
|
fxcpmadd f3, B1, A4, f3
|
|
nop
|
|
fxcsmadd f7, B1, A4, f7
|
|
LFPDUX A4, AO, INC2
|
|
|
|
fxcpmadd f0, B2, A5, f0
|
|
nop
|
|
fxcsmadd f4, B2, A5, f4
|
|
LFPDUX A5, AO, INC2
|
|
fxcpmadd f1, B2, A6, f1
|
|
nop
|
|
fxcsmadd f5, B2, A6, f5
|
|
LFPDUX A6, AO, INC2
|
|
|
|
fxcpmadd f2, B2, A7, f2
|
|
nop
|
|
fxcsmadd f6, B2, A7, f6
|
|
LFPDUX A7, AO, INC2
|
|
fxcpmadd f3, B2, A8, f3
|
|
nop
|
|
fxcsmadd f7, B2, A8, f7
|
|
LFPDUX A8, AO, INC2
|
|
|
|
fxcpmadd f0, B3, A1, f0
|
|
fxcsmadd f4, B3, A1, f4
|
|
fxcpmadd f1, B3, A2, f1
|
|
fxcsmadd f5, B3, A2, f5
|
|
|
|
fxcpmadd f2, B3, A3, f2
|
|
fxcsmadd f6, B3, A3, f6
|
|
fxcpmadd f3, B3, A4, f3
|
|
fxcsmadd f7, B3, A4, f7
|
|
|
|
fxcpmadd f0, B4, A5, f0
|
|
fxcsmadd f4, B4, A5, f4
|
|
fxcpmadd f1, B4, A6, f1
|
|
fxcsmadd f5, B4, A6, f5
|
|
|
|
fxcpmadd f2, B4, A7, f2
|
|
fxcsmadd f6, B4, A7, f6
|
|
fxcpmadd f3, B4, A8, f3
|
|
fxcsmadd f7, B4, A8, f7
|
|
.align 4
|
|
|
|
.L54:
|
|
lfd AP, ALPHA(SP)
|
|
#ifdef TRMMKERNEL
|
|
fsmfp AP, AP
|
|
#endif
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 8
|
|
#else
|
|
addi TEMP, KK, 2
|
|
#endif
|
|
andi. TEMP, TEMP, 3
|
|
mtspr CTR, TEMP
|
|
#else
|
|
andi. r0, K, 3
|
|
mtspr CTR, r0
|
|
#endif
|
|
ble+ .L58
|
|
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX A3, AO, INC2
|
|
LFPDUX A4, AO, INC2
|
|
bdz- .L57
|
|
.align 4
|
|
|
|
.L56:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f4, B1, A1, f4
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B1, A2, f1
|
|
fxcsmadd f5, B1, A2, f5
|
|
LFPDUX A2, AO, INC2
|
|
|
|
fxcpmadd f2, B1, A3, f2
|
|
fxcsmadd f6, B1, A3, f6
|
|
LFPDUX A3, AO, INC2
|
|
fxcpmadd f3, B1, A4, f3
|
|
fxcsmadd f7, B1, A4, f7
|
|
LFPDUX A4, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
bdnz+ .L56
|
|
.align 4
|
|
|
|
.L57:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f4, B1, A1, f4
|
|
fxcpmadd f1, B1, A2, f1
|
|
fxcsmadd f5, B1, A2, f5
|
|
|
|
fxcpmadd f2, B1, A3, f2
|
|
fxcsmadd f6, B1, A3, f6
|
|
fxcpmadd f3, B1, A4, f3
|
|
fxcsmadd f7, B1, A4, f7
|
|
.align 4
|
|
|
|
.L58:
|
|
#ifndef TRMMKERNEL
|
|
LFPDUX A1, CO1, INC2
|
|
LFPDUX B1, CO1, INC2
|
|
LFPDUX A3, CO1, INC2
|
|
LFPDUX A5, CO1, INC2
|
|
|
|
LFPDUX B3, CO2, INC2
|
|
LFPDUX A6, CO2, INC2
|
|
LFPDUX A7, CO2, INC2
|
|
LFPDUX B2, CO2, INC2
|
|
|
|
fxcpmadd f0, AP, f0, A1
|
|
fxcpmadd f1, AP, f1, B1
|
|
fxcpmadd f2, AP, f2, A3
|
|
fxcpmadd f3, AP, f3, A5
|
|
|
|
fxcpmadd f4, AP, f4, B3
|
|
fxcpmadd f5, AP, f5, A6
|
|
STFPDUX f0, CO1, INCM7
|
|
fxcpmadd f6, AP, f6, A7
|
|
STFPDUX f1, CO1, INC2
|
|
fxcpmadd f7, AP, f7, B2
|
|
STFPDUX f2, CO1, INC2
|
|
STFPDUX f3, CO1, INC2
|
|
STFPDUX f4, CO2, INCM7
|
|
|
|
STFPDUX f5, CO2, INC2
|
|
STFPDUX f6, CO2, INC2
|
|
STFPDUX f7, CO2, INC2
|
|
#else
|
|
fpmul f0, AP, f0
|
|
fpmul f1, AP, f1
|
|
fpmul f2, AP, f2
|
|
fpmul f3, AP, f3
|
|
|
|
fpmul f4, AP, f4
|
|
fpmul f5, AP, f5
|
|
STFPDUX f0, CO1, INC2
|
|
fpmul f6, AP, f6
|
|
STFPDUX f1, CO1, INC2
|
|
fpmul f7, AP, f7
|
|
STFPDUX f2, CO1, INC2
|
|
STFPDUX f3, CO1, INC2
|
|
STFPDUX f4, CO2, INC2
|
|
|
|
STFPDUX f5, CO2, INC2
|
|
STFPDUX f6, CO2, INC2
|
|
STFPDUX f7, CO2, INC2
|
|
#endif
|
|
|
|
|
|
#ifdef TRMMKERNEL
|
|
#if ( defined(LEFT) && defined(TRANSA)) || \
|
|
(!defined(LEFT) && !defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#ifdef LEFT
|
|
addi TEMP, TEMP, -8
|
|
#else
|
|
addi TEMP, TEMP, -2
|
|
#endif
|
|
slwi r0, TEMP, 3 + BASE_SHIFT
|
|
slwi TEMP, TEMP, 1 + BASE_SHIFT
|
|
add AO, AO, r0
|
|
add BO, BO, TEMP
|
|
#endif
|
|
|
|
#ifdef LEFT
|
|
addi KK, KK, 8
|
|
#endif
|
|
#endif
|
|
|
|
addic. I, I, -1
|
|
li r0, FZERO
|
|
|
|
lfpsx f0, SP, r0
|
|
bgt+ .L51
|
|
.align 4
|
|
|
|
.L60:
|
|
andi. I, M, 4
|
|
beq .L70
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
#else
|
|
slwi TEMP, KK, 2 + BASE_SHIFT
|
|
slwi r0, KK, 1 + BASE_SHIFT
|
|
add AO, AO, TEMP
|
|
add BO, B, r0
|
|
|
|
addi BO, BO, - 2 * SIZE
|
|
fpmr f1, f0
|
|
#endif
|
|
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 4
|
|
#else
|
|
addi TEMP, KK, 2
|
|
#endif
|
|
fpmr f2, f0
|
|
srawi. r0, TEMP, 2
|
|
mtspr CTR, r0
|
|
fpmr f3, f0
|
|
ble .L64
|
|
#else
|
|
srawi. r0, K, 2
|
|
fpmr f1, f0
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f2, f0
|
|
mtspr CTR, r0
|
|
fpmr f3, f0
|
|
ble .L64
|
|
#endif
|
|
|
|
LFPDUX B1, BO, INC2
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
LFPDUX A3, AO, INC2
|
|
LFPDUX A4, AO, INC2
|
|
|
|
LFPDUX B3, BO, INC2
|
|
LFPDUX A5, AO, INC2
|
|
LFPDUX A6, AO, INC2
|
|
LFPDUX B4, BO, INC2
|
|
LFPDUX A7, AO, INC2
|
|
LFPDUX A8, AO, INC2
|
|
bdz- .L63
|
|
.align 4
|
|
|
|
.L62:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f2, B1, A1, f2
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B1, A2, f1
|
|
fxcsmadd f3, B1, A2, f3
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
|
|
fxcpmadd f0, B2, A3, f0
|
|
fxcsmadd f2, B2, A3, f2
|
|
LFPDUX A3, AO, INC2
|
|
fxcpmadd f1, B2, A4, f1
|
|
fxcsmadd f3, B2, A4, f3
|
|
LFPDUX A4, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
|
|
fxcpmadd f0, B3, A5, f0
|
|
fxcsmadd f2, B3, A5, f2
|
|
LFPDUX A5, AO, INC2
|
|
fxcpmadd f1, B3, A6, f1
|
|
fxcsmadd f3, B3, A6, f3
|
|
LFPDUX A6, AO, INC2
|
|
LFPDUX B3, BO, INC2
|
|
|
|
fxcpmadd f0, B4, A7, f0
|
|
fxcsmadd f2, B4, A7, f2
|
|
LFPDUX A7, AO, INC2
|
|
fxcpmadd f1, B4, A8, f1
|
|
fxcsmadd f3, B4, A8, f3
|
|
LFPDUX A8, AO, INC2
|
|
LFPDUX B4, BO, INC2
|
|
bdnz+ .L62
|
|
.align 4
|
|
|
|
.L63:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f2, B1, A1, f2
|
|
fxcpmadd f1, B1, A2, f1
|
|
fxcsmadd f3, B1, A2, f3
|
|
|
|
fxcpmadd f0, B2, A3, f0
|
|
fxcsmadd f2, B2, A3, f2
|
|
fxcpmadd f1, B2, A4, f1
|
|
fxcsmadd f3, B2, A4, f3
|
|
|
|
fxcpmadd f0, B3, A5, f0
|
|
fxcsmadd f2, B3, A5, f2
|
|
fxcpmadd f1, B3, A6, f1
|
|
fxcsmadd f3, B3, A6, f3
|
|
|
|
fxcpmadd f0, B4, A7, f0
|
|
fxcsmadd f2, B4, A7, f2
|
|
fxcpmadd f1, B4, A8, f1
|
|
fxcsmadd f3, B4, A8, f3
|
|
.align 4
|
|
|
|
.L64:
|
|
lfd AP, ALPHA(SP)
|
|
#ifdef TRMMKERNEL
|
|
fsmfp AP, AP
|
|
#endif
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 4
|
|
#else
|
|
addi TEMP, KK, 2
|
|
#endif
|
|
andi. TEMP, TEMP, 3
|
|
mtspr CTR, TEMP
|
|
#else
|
|
andi. r0, K, 3
|
|
mtspr CTR, r0
|
|
#endif
|
|
ble+ .L68
|
|
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
bdz- .L67
|
|
.align 4
|
|
|
|
.L66:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f2, B1, A1, f2
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B1, A2, f1
|
|
fxcsmadd f3, B1, A2, f3
|
|
LFPDUX B1, BO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
bdnz+ .L66
|
|
.align 4
|
|
|
|
.L67:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f2, B1, A1, f2
|
|
fxcpmadd f1, B1, A2, f1
|
|
fxcsmadd f3, B1, A2, f3
|
|
.align 4
|
|
|
|
.L68:
|
|
#ifndef TRMMKERNEL
|
|
LFPDUX A1, CO1, INC2
|
|
LFPDUX A2, CO1, INC2
|
|
LFPDUX A3, CO2, INC2
|
|
LFPDUX A4, CO2, INC2
|
|
|
|
fxcpmadd f0, AP, f0, A1
|
|
fxcpmadd f1, AP, f1, A2
|
|
fxcpmadd f2, AP, f2, A3
|
|
fxcpmadd f3, AP, f3, A4
|
|
|
|
STFPDUX f0, CO1, INCM3
|
|
STFPDUX f1, CO1, INC2
|
|
STFPDUX f2, CO2, INCM3
|
|
STFPDUX f3, CO2, INC2
|
|
#else
|
|
fpmul f0, AP, f0
|
|
fpmul f1, AP, f1
|
|
fpmul f2, AP, f2
|
|
fpmul f3, AP, f3
|
|
|
|
STFPDUX f0, CO1, INC2
|
|
STFPDUX f1, CO1, INC2
|
|
STFPDUX f2, CO2, INC2
|
|
STFPDUX f3, CO2, INC2
|
|
#endif
|
|
|
|
#ifdef TRMMKERNEL
|
|
#if ( defined(LEFT) && defined(TRANSA)) || \
|
|
(!defined(LEFT) && !defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#ifdef LEFT
|
|
addi TEMP, TEMP, -4
|
|
#else
|
|
addi TEMP, TEMP, -2
|
|
#endif
|
|
slwi r0, TEMP, 2 + BASE_SHIFT
|
|
slwi TEMP, TEMP, 1 + BASE_SHIFT
|
|
add AO, AO, r0
|
|
add BO, BO, TEMP
|
|
#endif
|
|
|
|
#ifdef LEFT
|
|
addi KK, KK, 4
|
|
#endif
|
|
#endif
|
|
|
|
li r0, FZERO
|
|
lfpsx f0, SP, r0
|
|
.align 4
|
|
|
|
.L70:
|
|
andi. I, M, 2
|
|
beq .L80
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
#else
|
|
slwi TEMP, KK, 1 + BASE_SHIFT
|
|
slwi r0, KK, 1 + BASE_SHIFT
|
|
add AO, AO, TEMP
|
|
add BO, B, r0
|
|
|
|
addi BO, BO, - 2 * SIZE
|
|
fpmr f1, f0
|
|
#endif
|
|
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 2
|
|
#else
|
|
addi TEMP, KK, 2
|
|
#endif
|
|
srawi. r0, TEMP, 3
|
|
fpmr f2, f0
|
|
mtspr CTR, r0
|
|
fpmr f3, f0
|
|
ble .L74
|
|
#else
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
|
|
srawi. r0, K, 3
|
|
fpmr f2, f0
|
|
mtspr CTR, r0
|
|
fpmr f3, f0
|
|
ble .L74
|
|
#endif
|
|
|
|
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
LFPDUX A3, AO, INC2
|
|
LFPDUX B3, BO, INC2
|
|
LFPDUX A4, AO, INC2
|
|
LFPDUX B4, BO, INC2
|
|
|
|
LFPDUX A5, AO, INC2
|
|
LFPDUX B5, BO, INC2
|
|
LFPDUX A6, AO, INC2
|
|
LFPDUX B6, BO, INC2
|
|
LFPDUX A7, AO, INC2
|
|
LFPDUX A9, BO, INC2
|
|
LFPDUX A8, AO, INC2
|
|
LFPDUX A10, BO, INC2
|
|
bdz- .L73
|
|
.align 4
|
|
|
|
.L72:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f1, B1, A1, f1
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
fxcpmadd f2, B2, A2, f2
|
|
fxcsmadd f3, B2, A2, f3
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
|
|
fxcpmadd f0, B3, A3, f0
|
|
fxcsmadd f1, B3, A3, f1
|
|
LFPDUX A3, AO, INC2
|
|
LFPDUX B3, BO, INC2
|
|
fxcpmadd f2, B4, A4, f2
|
|
fxcsmadd f3, B4, A4, f3
|
|
LFPDUX A4, AO, INC2
|
|
LFPDUX B4, BO, INC2
|
|
|
|
fxcpmadd f0, B5, A5, f0
|
|
fxcsmadd f1, B5, A5, f1
|
|
LFPDUX A5, AO, INC2
|
|
LFPDUX B5, BO, INC2
|
|
fxcpmadd f2, B6, A6, f2
|
|
fxcsmadd f3, B6, A6, f3
|
|
LFPDUX A6, AO, INC2
|
|
LFPDUX B6, BO, INC2
|
|
|
|
fxcpmadd f0, A9, A7, f0
|
|
fxcsmadd f1, A9, A7, f1
|
|
LFPDUX A7, AO, INC2
|
|
LFPDUX A9, BO, INC2
|
|
fxcpmadd f2, A10, A8, f2
|
|
fxcsmadd f3, A10, A8, f3
|
|
LFPDUX A8, AO, INC2
|
|
LFPDUX A10, BO, INC2
|
|
bdnz+ .L72
|
|
.align 4
|
|
|
|
.L73:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f1, B1, A1, f1
|
|
fxcpmadd f2, B2, A2, f2
|
|
fxcsmadd f3, B2, A2, f3
|
|
|
|
fxcpmadd f0, B3, A3, f0
|
|
fxcsmadd f1, B3, A3, f1
|
|
fxcpmadd f2, B4, A4, f2
|
|
fxcsmadd f3, B4, A4, f3
|
|
|
|
fxcpmadd f0, B5, A5, f0
|
|
fxcsmadd f1, B5, A5, f1
|
|
fxcpmadd f2, B6, A6, f2
|
|
fxcsmadd f3, B6, A6, f3
|
|
|
|
fxcpmadd f0, A9, A7, f0
|
|
fxcsmadd f1, A9, A7, f1
|
|
fxcpmadd f2, A10, A8, f2
|
|
fxcsmadd f3, A10, A8, f3
|
|
.align 4
|
|
|
|
.L74:
|
|
lfd AP, ALPHA(SP)
|
|
#ifdef TRMMKERNEL
|
|
fsmfp AP, AP
|
|
#endif
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 2
|
|
#else
|
|
addi TEMP, KK, 2
|
|
#endif
|
|
andi. TEMP, TEMP, 7
|
|
mtspr CTR, TEMP
|
|
#else
|
|
andi. r0, K, 7
|
|
mtspr CTR, r0
|
|
#endif
|
|
ble+ .L78
|
|
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
bdz- .L77
|
|
.align 4
|
|
|
|
.L76:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f1, B1, A1, f1
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
bdnz+ .L76
|
|
.align 4
|
|
|
|
.L77:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f1, B1, A1, f1
|
|
.align 4
|
|
|
|
.L78:
|
|
#ifndef TRMMKERNEL
|
|
LFPDX A1, CO1, INC2
|
|
LFPDX B3, CO2, INC2
|
|
|
|
fpadd f0, f0, f2
|
|
fpadd f1, f1, f3
|
|
|
|
fxcpmadd f0, AP, f0, A1
|
|
fxcpmadd f1, AP, f1, B3
|
|
#else
|
|
fpadd f0, f0, f2
|
|
fpadd f1, f1, f3
|
|
|
|
fpmul f0, AP, f0
|
|
fpmul f1, AP, f1
|
|
#endif
|
|
|
|
STFPDUX f0, CO1, INC2
|
|
STFPDUX f1, CO2, INC2
|
|
|
|
#ifdef TRMMKERNEL
|
|
#if ( defined(LEFT) && defined(TRANSA)) || \
|
|
(!defined(LEFT) && !defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#ifdef LEFT
|
|
addi TEMP, TEMP, -2
|
|
#else
|
|
addi TEMP, TEMP, -2
|
|
#endif
|
|
slwi r0, TEMP, 1 + BASE_SHIFT
|
|
slwi TEMP, TEMP, 1 + BASE_SHIFT
|
|
add AO, AO, r0
|
|
add BO, BO, TEMP
|
|
#endif
|
|
|
|
#ifdef LEFT
|
|
addi KK, KK, 2
|
|
#endif
|
|
#endif
|
|
|
|
li r0, FZERO
|
|
lfpsx f0, SP, r0
|
|
.align 4
|
|
|
|
.L80:
|
|
andi. I, M, 1
|
|
beq .L89
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
#else
|
|
slwi TEMP, KK, 0 + BASE_SHIFT
|
|
slwi r0, KK, 1 + BASE_SHIFT
|
|
add AO, AO, TEMP
|
|
add BO, B, r0
|
|
|
|
addi BO, BO, - 2 * SIZE
|
|
fpmr f1, f0
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
#endif
|
|
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 1
|
|
#else
|
|
addi TEMP, KK, 2
|
|
#endif
|
|
srawi. r0, TEMP, 3
|
|
mtspr CTR, r0
|
|
ble .L84
|
|
#else
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
|
|
srawi. r0, K, 3
|
|
mtspr CTR, r0
|
|
ble .L84
|
|
#endif
|
|
|
|
LFPDUX B1, BO, INC2
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
|
|
LFPDUX B2, BO, INC2
|
|
LFPDUX A3, AO, INC2
|
|
LFPDUX A4, AO, INC2
|
|
|
|
LFPDUX B3, BO, INC2
|
|
LFPDUX B4, BO, INC2
|
|
bdz- .L83
|
|
.align 4
|
|
|
|
.L82:
|
|
fxcpmadd f0, A1, B1, f0
|
|
LFPDUX B1, BO, INC2
|
|
fxcsmadd f1, A1, B2, f1
|
|
LFPDUX B2, BO, INC2
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f2, A2, B3, f2
|
|
LFPDUX B3, BO, INC2
|
|
fxcsmadd f3, A2, B4, f3
|
|
LFPDUX B4, BO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
|
|
fxcpmadd f0, A3, B1, f0
|
|
LFPDUX B1, BO, INC2
|
|
fxcsmadd f1, A3, B2, f1
|
|
LFPDUX B2, BO, INC2
|
|
LFPDUX A3, AO, INC2
|
|
fxcpmadd f2, A4, B3, f2
|
|
LFPDUX B3, BO, INC2
|
|
fxcsmadd f3, A4, B4, f3
|
|
LFPDUX B4, BO, INC2
|
|
LFPDUX A4, AO, INC2
|
|
bdnz+ .L82
|
|
.align 4
|
|
|
|
.L83:
|
|
fxcpmadd f0, A1, B1, f0
|
|
LFPDUX B1, BO, INC2
|
|
fxcsmadd f1, A1, B2, f1
|
|
LFPDUX B2, BO, INC2
|
|
fxcpmadd f2, A2, B3, f2
|
|
LFPDUX B3, BO, INC2
|
|
fxcsmadd f3, A2, B4, f3
|
|
LFPDUX B4, BO, INC2
|
|
|
|
fxcpmadd f0, A3, B1, f0
|
|
fxcsmadd f1, A3, B2, f1
|
|
fxcpmadd f2, A4, B3, f2
|
|
fxcsmadd f3, A4, B4, f3
|
|
.align 4
|
|
|
|
.L84:
|
|
lfd AP, ALPHA(SP)
|
|
#ifdef TRMMKERNEL
|
|
fsmfp AP, AP
|
|
#endif
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 1
|
|
#else
|
|
addi TEMP, KK, 2
|
|
#endif
|
|
andi. TEMP, TEMP, 7
|
|
mtspr CTR, TEMP
|
|
#else
|
|
andi. r0, K, 7
|
|
mtspr CTR, r0
|
|
#endif
|
|
ble+ .L88
|
|
|
|
LFDX A1, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
add AO, AO, INC
|
|
bdz- .L87
|
|
.align 4
|
|
|
|
.L86:
|
|
fxcpmadd f0, A1, B1, f0
|
|
LFDX A1, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
add AO, AO, INC
|
|
bdnz+ .L86
|
|
.align 4
|
|
|
|
.L87:
|
|
fxcpmadd f0, A1, B1, f0
|
|
.align 4
|
|
|
|
.L88:
|
|
#ifndef TRMMKERNEL
|
|
LFDX A1, CO1, INC2
|
|
LFDX A2, CO2, INC2
|
|
|
|
fpadd f0, f0, f1
|
|
fpadd f2, f2, f3
|
|
fsmfp A1, A2
|
|
fpadd f0, f0, f2
|
|
fxcpmadd f0, AP, f0, A1
|
|
#else
|
|
fpadd f0, f0, f1
|
|
fpadd f2, f2, f3
|
|
fsmfp A1, A2
|
|
fpadd f0, f0, f2
|
|
fpmul f0, AP, f0
|
|
#endif
|
|
|
|
STFDX f0, CO1, INC2
|
|
STFSDX f0, CO2, INC2
|
|
|
|
#ifdef TRMMKERNEL
|
|
#if ( defined(LEFT) && defined(TRANSA)) || \
|
|
(!defined(LEFT) && !defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#ifdef LEFT
|
|
addi TEMP, TEMP, -1
|
|
#else
|
|
addi TEMP, TEMP, -2
|
|
#endif
|
|
slwi r0, TEMP, 0 + BASE_SHIFT
|
|
slwi TEMP, TEMP, 1 + BASE_SHIFT
|
|
add AO, AO, r0
|
|
add BO, BO, TEMP
|
|
#endif
|
|
|
|
#ifdef LEFT
|
|
addi KK, KK, 1
|
|
#endif
|
|
#endif
|
|
.align 4
|
|
|
|
.L89:
|
|
#if defined(TRMMKERNEL) && !defined(LEFT)
|
|
addi KK, KK, 2
|
|
#endif
|
|
|
|
addi B, BO, 2 * SIZE
|
|
.align 4
|
|
|
|
.L90:
|
|
andi. J, N, 1
|
|
beq .L999
|
|
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
mr KK, OFFSET
|
|
#endif
|
|
|
|
mr CO1, C
|
|
addi AO, A, -2 * SIZE
|
|
|
|
li r0, FZERO
|
|
lfpsx f0, SP, r0
|
|
|
|
srawi. I, M, 3
|
|
ble .L100
|
|
.align 4
|
|
|
|
.L91:
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
#else
|
|
slwi TEMP, KK, 3 + BASE_SHIFT
|
|
slwi r0, KK, 0 + BASE_SHIFT
|
|
add AO, AO, TEMP
|
|
add BO, B, r0
|
|
|
|
addi BO, BO, - 2 * SIZE
|
|
fpmr f1, f0
|
|
#endif
|
|
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 8
|
|
#else
|
|
addi TEMP, KK, 1
|
|
#endif
|
|
fpmr f2, f0
|
|
srawi. r0, TEMP, 2
|
|
fpmr f3, f0
|
|
mtspr CTR, r0
|
|
ble .L94
|
|
|
|
#else
|
|
srawi. r0, K, 2
|
|
fpmr f1, f0
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
mtspr CTR, r0
|
|
ble .L94
|
|
#endif
|
|
|
|
LFPDUX B1, BO, INC2
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX A3, AO, INC2
|
|
LFPDUX A4, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
LFPDUX A5, AO, INC2
|
|
LFPDUX A6, AO, INC2
|
|
LFPDUX A7, AO, INC2
|
|
LFPDUX A8, AO, INC2
|
|
bdz- .L93
|
|
.align 4
|
|
|
|
.L92:
|
|
fxcpmadd f0, B1, A1, f0
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B1, A2, f1
|
|
LFPDUX A2, AO, INC2
|
|
fxcpmadd f2, B1, A3, f2
|
|
LFPDUX A3, AO, INC2
|
|
fxcpmadd f3, B1, A4, f3
|
|
LFPDUX A4, AO, INC2
|
|
|
|
fxcsmadd f0, B1, A5, f0
|
|
LFPDUX A5, AO, INC2
|
|
fxcsmadd f1, B1, A6, f1
|
|
LFPDUX A6, AO, INC2
|
|
fxcsmadd f2, B1, A7, f2
|
|
LFPDUX A7, AO, INC2
|
|
fxcsmadd f3, B1, A8, f3
|
|
LFPDUX A8, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
|
|
fxcpmadd f0, B2, A1, f0
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B2, A2, f1
|
|
LFPDUX A2, AO, INC2
|
|
fxcpmadd f2, B2, A3, f2
|
|
LFPDUX A3, AO, INC2
|
|
fxcpmadd f3, B2, A4, f3
|
|
LFPDUX A4, AO, INC2
|
|
|
|
fxcsmadd f0, B2, A5, f0
|
|
LFPDUX A5, AO, INC2
|
|
fxcsmadd f1, B2, A6, f1
|
|
LFPDUX A6, AO, INC2
|
|
fxcsmadd f2, B2, A7, f2
|
|
LFPDUX A7, AO, INC2
|
|
fxcsmadd f3, B2, A8, f3
|
|
LFPDUX A8, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
bdnz+ .L92
|
|
.align 4
|
|
|
|
.L93:
|
|
fxcpmadd f0, B1, A1, f0
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B1, A2, f1
|
|
LFPDUX A2, AO, INC2
|
|
fxcpmadd f2, B1, A3, f2
|
|
LFPDUX A3, AO, INC2
|
|
fxcpmadd f3, B1, A4, f3
|
|
LFPDUX A4, AO, INC2
|
|
|
|
fxcsmadd f0, B1, A5, f0
|
|
LFPDUX A5, AO, INC2
|
|
fxcsmadd f1, B1, A6, f1
|
|
LFPDUX A6, AO, INC2
|
|
fxcsmadd f2, B1, A7, f2
|
|
LFPDUX A7, AO, INC2
|
|
fxcsmadd f3, B1, A8, f3
|
|
LFPDUX A8, AO, INC2
|
|
|
|
fxcpmadd f0, B2, A1, f0
|
|
fxcpmadd f1, B2, A2, f1
|
|
fxcpmadd f2, B2, A3, f2
|
|
fxcpmadd f3, B2, A4, f3
|
|
|
|
fxcsmadd f0, B2, A5, f0
|
|
fxcsmadd f1, B2, A6, f1
|
|
fxcsmadd f2, B2, A7, f2
|
|
fxcsmadd f3, B2, A8, f3
|
|
.align 4
|
|
|
|
.L94:
|
|
lfd AP, ALPHA(SP)
|
|
#ifdef TRMMKERNEL
|
|
fsmfp AP, AP
|
|
#endif
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 8
|
|
#else
|
|
addi TEMP, KK, 1
|
|
#endif
|
|
andi. TEMP, TEMP, 3
|
|
mtspr CTR, TEMP
|
|
#else
|
|
andi. r0, K, 3
|
|
mtspr CTR, r0
|
|
#endif
|
|
ble+ .L98
|
|
|
|
LFDX B1, BO, INC2
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX A3, AO, INC2
|
|
LFPDUX A4, AO, INC2
|
|
add BO, BO, INC
|
|
bdz- .L97
|
|
.align 4
|
|
|
|
.L96:
|
|
fxcpmadd f0, B1, A1, f0
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B1, A2, f1
|
|
LFPDUX A2, AO, INC2
|
|
fxcpmadd f2, B1, A3, f2
|
|
LFPDUX A3, AO, INC2
|
|
fxcpmadd f3, B1, A4, f3
|
|
LFDX B1, BO, INC2
|
|
LFPDUX A4, AO, INC2
|
|
add BO, BO, INC
|
|
bdnz+ .L96
|
|
.align 4
|
|
|
|
.L97:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcpmadd f1, B1, A2, f1
|
|
fxcpmadd f2, B1, A3, f2
|
|
fxcpmadd f3, B1, A4, f3
|
|
.align 4
|
|
|
|
.L98:
|
|
#ifndef TRMMKERNEL
|
|
LFPDUX A1, CO1, INC2
|
|
LFPDUX B1, CO1, INC2
|
|
LFPDUX A3, CO1, INC2
|
|
LFPDUX A5, CO1, INC2
|
|
|
|
fxcpmadd f0, AP, f0, A1
|
|
fxcpmadd f1, AP, f1, B1
|
|
fxcpmadd f2, AP, f2, A3
|
|
fxcpmadd f3, AP, f3, A5
|
|
|
|
STFPDUX f0, CO1, INCM7
|
|
STFPDUX f1, CO1, INC2
|
|
STFPDUX f2, CO1, INC2
|
|
STFPDUX f3, CO1, INC2
|
|
#else
|
|
fpmul f0, AP, f0
|
|
fpmul f1, AP, f1
|
|
fpmul f2, AP, f2
|
|
fpmul f3, AP, f3
|
|
|
|
STFPDUX f0, CO1, INC2
|
|
STFPDUX f1, CO1, INC2
|
|
STFPDUX f2, CO1, INC2
|
|
STFPDUX f3, CO1, INC2
|
|
#endif
|
|
|
|
#ifdef TRMMKERNEL
|
|
#if ( defined(LEFT) && defined(TRANSA)) || \
|
|
(!defined(LEFT) && !defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#ifdef LEFT
|
|
addi TEMP, TEMP, -8
|
|
#else
|
|
addi TEMP, TEMP, -1
|
|
#endif
|
|
slwi r0, TEMP, 3 + BASE_SHIFT
|
|
slwi TEMP, TEMP, 0 + BASE_SHIFT
|
|
add AO, AO, r0
|
|
add BO, BO, TEMP
|
|
#endif
|
|
|
|
#ifdef LEFT
|
|
addi KK, KK, 8
|
|
#endif
|
|
#endif
|
|
|
|
addic. I, I, -1
|
|
li r0, FZERO
|
|
|
|
lfpsx f0, SP, r0
|
|
bgt+ .L91
|
|
.align 4
|
|
|
|
.L100:
|
|
andi. I, M, 4
|
|
beq .L110
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
#else
|
|
slwi TEMP, KK, 2 + BASE_SHIFT
|
|
slwi r0, KK, 0 + BASE_SHIFT
|
|
add AO, AO, TEMP
|
|
add BO, B, r0
|
|
|
|
fpmr f1, f0
|
|
addi BO, BO, - 2 * SIZE
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
#endif
|
|
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 4
|
|
#else
|
|
addi TEMP, KK, 1
|
|
#endif
|
|
srawi. r0, TEMP, 3
|
|
mtspr CTR, r0
|
|
ble .L104
|
|
#else
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
|
|
srawi. r0, K, 3
|
|
mtspr CTR, r0
|
|
ble .L104
|
|
#endif
|
|
|
|
LFPDUX B1, BO, INC2
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX A3, AO, INC2
|
|
LFPDUX A4, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
LFPDUX A5, AO, INC2
|
|
LFPDUX A6, AO, INC2
|
|
LFPDUX A7, AO, INC2
|
|
LFPDUX A8, AO, INC2
|
|
LFPDUX B3, BO, INC2
|
|
LFPDUX B4, BO, INC2
|
|
|
|
bdz- .L103
|
|
.align 4
|
|
|
|
.L102:
|
|
fxcpmadd f0, B1, A1, f0
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B1, A2, f1
|
|
LFPDUX A2, AO, INC2
|
|
fxcsmadd f2, B1, A3, f2
|
|
LFPDUX A3, AO, INC2
|
|
fxcsmadd f3, B1, A4, f3
|
|
LFPDUX A4, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
|
|
fxcpmadd f0, B2, A5, f0
|
|
LFPDUX A5, AO, INC2
|
|
fxcpmadd f1, B2, A6, f1
|
|
LFPDUX A6, AO, INC2
|
|
fxcsmadd f2, B2, A7, f2
|
|
LFPDUX A7, AO, INC2
|
|
fxcsmadd f3, B2, A8, f3
|
|
LFPDUX A8, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
|
|
fxcpmadd f0, B3, A1, f0
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B3, A2, f1
|
|
LFPDUX A2, AO, INC2
|
|
fxcsmadd f2, B3, A3, f2
|
|
LFPDUX A3, AO, INC2
|
|
fxcsmadd f3, B3, A4, f3
|
|
LFPDUX A4, AO, INC2
|
|
LFPDUX B3, BO, INC2
|
|
|
|
fxcpmadd f0, B4, A5, f0
|
|
LFPDUX A5, AO, INC2
|
|
fxcpmadd f1, B4, A6, f1
|
|
LFPDUX A6, AO, INC2
|
|
fxcsmadd f2, B4, A7, f2
|
|
LFPDUX A7, AO, INC2
|
|
fxcsmadd f3, B4, A8, f3
|
|
LFPDUX A8, AO, INC2
|
|
LFPDUX B4, BO, INC2
|
|
bdnz+ .L102
|
|
.align 4
|
|
|
|
.L103:
|
|
fxcpmadd f0, B1, A1, f0
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B1, A2, f1
|
|
LFPDUX A2, AO, INC2
|
|
fxcsmadd f2, B1, A3, f2
|
|
LFPDUX A3, AO, INC2
|
|
fxcsmadd f3, B1, A4, f3
|
|
LFPDUX A4, AO, INC2
|
|
|
|
fxcpmadd f0, B2, A5, f0
|
|
LFPDUX A5, AO, INC2
|
|
fxcpmadd f1, B2, A6, f1
|
|
LFPDUX A6, AO, INC2
|
|
fxcsmadd f2, B2, A7, f2
|
|
LFPDUX A7, AO, INC2
|
|
fxcsmadd f3, B2, A8, f3
|
|
LFPDUX A8, AO, INC2
|
|
|
|
fxcpmadd f0, B3, A1, f0
|
|
fxcpmadd f1, B3, A2, f1
|
|
fxcsmadd f2, B3, A3, f2
|
|
fxcsmadd f3, B3, A4, f3
|
|
|
|
fxcpmadd f0, B4, A5, f0
|
|
fxcpmadd f1, B4, A6, f1
|
|
fxcsmadd f2, B4, A7, f2
|
|
fxcsmadd f3, B4, A8, f3
|
|
.align 4
|
|
|
|
.L104:
|
|
lfd AP, ALPHA(SP)
|
|
#ifdef TRMMKERNEL
|
|
fsmfp AP, AP
|
|
#endif
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 4
|
|
#else
|
|
addi TEMP, KK, 1
|
|
#endif
|
|
andi. TEMP, TEMP, 7
|
|
mtspr CTR, TEMP
|
|
#else
|
|
andi. r0, K, 7
|
|
mtspr CTR, r0
|
|
#endif
|
|
ble+ .L108
|
|
|
|
LFPDUX A1, AO, INC2
|
|
LFDX B1, BO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
add BO, BO, INC
|
|
bdz- .L107
|
|
.align 4
|
|
|
|
.L106:
|
|
fxcpmadd f0, B1, A1, f0
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B1, A2, f1
|
|
LFDX B1, BO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
add BO, BO, INC
|
|
bdnz+ .L106
|
|
.align 4
|
|
|
|
.L107:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcpmadd f1, B1, A2, f1
|
|
.align 4
|
|
|
|
.L108:
|
|
#ifndef TRMMKERNEL
|
|
LFPDUX A1, CO1, INC2
|
|
LFPDUX B1, CO1, INC2
|
|
|
|
fpadd f0, f0, f2
|
|
fpadd f1, f1, f3
|
|
|
|
fxcpmadd f0, AP, f0, A1
|
|
fxcpmadd f1, AP, f1, B1
|
|
|
|
STFPDUX f0, CO1, INCM3
|
|
STFPDUX f1, CO1, INC2
|
|
#else
|
|
fpadd f0, f0, f2
|
|
fpadd f1, f1, f3
|
|
|
|
fpmul f0, AP, f0
|
|
fpmul f1, AP, f1
|
|
|
|
STFPDUX f0, CO1, INC2
|
|
STFPDUX f1, CO1, INC2
|
|
#endif
|
|
|
|
|
|
#ifdef TRMMKERNEL
|
|
#if ( defined(LEFT) && defined(TRANSA)) || \
|
|
(!defined(LEFT) && !defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#ifdef LEFT
|
|
addi TEMP, TEMP, -4
|
|
#else
|
|
addi TEMP, TEMP, -1
|
|
#endif
|
|
slwi r0, TEMP, 2 + BASE_SHIFT
|
|
slwi TEMP, TEMP, 0 + BASE_SHIFT
|
|
add AO, AO, r0
|
|
add BO, BO, TEMP
|
|
#endif
|
|
|
|
#ifdef LEFT
|
|
addi KK, KK, 4
|
|
#endif
|
|
#endif
|
|
|
|
li r0, FZERO
|
|
lfpsx f0, SP, r0
|
|
.align 4
|
|
|
|
.L110:
|
|
andi. I, M, 2
|
|
beq .L120
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
#else
|
|
slwi TEMP, KK, 1 + BASE_SHIFT
|
|
slwi r0, KK, 0 + BASE_SHIFT
|
|
add AO, AO, TEMP
|
|
add BO, B, r0
|
|
|
|
fpmr f1, f0
|
|
addi BO, BO, - 2 * SIZE
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
#endif
|
|
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 2
|
|
#else
|
|
addi TEMP, KK, 1
|
|
#endif
|
|
srawi. r0, TEMP, 3
|
|
mtspr CTR, r0
|
|
ble .L114
|
|
#else
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
|
|
srawi. r0, K, 3
|
|
mtspr CTR, r0
|
|
ble .L114
|
|
#endif
|
|
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
|
|
LFPDUX A3, AO, INC2
|
|
LFPDUX A4, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
|
|
LFPDUX A5, AO, INC2
|
|
LFPDUX A6, AO, INC2
|
|
LFPDUX B3, BO, INC2
|
|
|
|
LFPDUX A7, AO, INC2
|
|
LFPDUX A8, AO, INC2
|
|
LFPDUX B4, BO, INC2
|
|
bdz- .L113
|
|
.align 4
|
|
|
|
.L112:
|
|
fxcpmadd f0, B1, A1, f0
|
|
LFPDUX A1, AO, INC2
|
|
fxcsmadd f1, B1, A2, f1
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
fxcpmadd f2, B2, A3, f2
|
|
LFPDUX A3, AO, INC2
|
|
fxcsmadd f3, B2, A4, f3
|
|
LFPDUX A4, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
fxcpmadd f0, B3, A5, f0
|
|
LFPDUX A5, AO, INC2
|
|
fxcsmadd f1, B3, A6, f1
|
|
LFPDUX A6, AO, INC2
|
|
LFPDUX B3, BO, INC2
|
|
fxcpmadd f2, B4, A7, f2
|
|
LFPDUX A7, AO, INC2
|
|
fxcsmadd f3, B4, A8, f3
|
|
LFPDUX A8, AO, INC2
|
|
LFPDUX B4, BO, INC2
|
|
bdnz+ .L112
|
|
.align 4
|
|
|
|
.L113:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f1, B1, A2, f1
|
|
fxcpmadd f2, B2, A3, f2
|
|
fxcsmadd f3, B2, A4, f3
|
|
fxcpmadd f0, B3, A5, f0
|
|
fxcsmadd f1, B3, A6, f1
|
|
fxcpmadd f2, B4, A7, f2
|
|
fxcsmadd f3, B4, A8, f3
|
|
.align 4
|
|
|
|
.L114:
|
|
lfd AP, ALPHA(SP)
|
|
#ifdef TRMMKERNEL
|
|
fsmfp AP, AP
|
|
#endif
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 2
|
|
#else
|
|
addi TEMP, KK, 1
|
|
#endif
|
|
andi. TEMP, TEMP, 7
|
|
mtspr CTR, TEMP
|
|
#else
|
|
andi. r0, K, 7
|
|
mtspr CTR, r0
|
|
#endif
|
|
ble+ .L118
|
|
|
|
LFPDUX A1, AO, INC2
|
|
LFDX B1, BO, INC2
|
|
add BO, BO, INC
|
|
bdz- .L117
|
|
.align 4
|
|
|
|
.L116:
|
|
fxcpmadd f0, B1, A1, f0
|
|
LFPDUX A1, AO, INC2
|
|
LFDX B1, BO, INC2
|
|
add BO, BO, INC
|
|
bdnz+ .L116
|
|
.align 4
|
|
|
|
.L117:
|
|
fxcpmadd f0, B1, A1, f0
|
|
.align 4
|
|
|
|
.L118:
|
|
#ifndef TRMMKERNEL
|
|
LFPDX A1, CO1, INC2
|
|
|
|
fpadd f0, f0, f1
|
|
fpadd f2, f3, f2
|
|
fpadd f0, f0, f2
|
|
fxcpmadd f1, AP, f0, A1
|
|
|
|
li r0, FZERO
|
|
lfpsx f0, SP, r0
|
|
|
|
STFPDUX f1, CO1, INC2
|
|
#else
|
|
fpadd f0, f0, f1
|
|
fpadd f2, f3, f2
|
|
fpadd f0, f0, f2
|
|
fpmul f1, AP, f0
|
|
|
|
li r0, FZERO
|
|
lfpsx f0, SP, r0
|
|
|
|
STFPDUX f1, CO1, INC2
|
|
#endif
|
|
|
|
|
|
#ifdef TRMMKERNEL
|
|
#if ( defined(LEFT) && defined(TRANSA)) || \
|
|
(!defined(LEFT) && !defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#ifdef LEFT
|
|
addi TEMP, TEMP, -2
|
|
#else
|
|
addi TEMP, TEMP, -1
|
|
#endif
|
|
slwi r0, TEMP, 1 + BASE_SHIFT
|
|
slwi TEMP, TEMP, 0 + BASE_SHIFT
|
|
add AO, AO, r0
|
|
add BO, BO, TEMP
|
|
#endif
|
|
|
|
#ifdef LEFT
|
|
addi KK, KK, 2
|
|
#endif
|
|
#endif
|
|
.align 4
|
|
|
|
.L120:
|
|
andi. I, M, 1
|
|
beq .L999
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
#else
|
|
slwi TEMP, KK, 0 + BASE_SHIFT
|
|
slwi r0, KK, 0 + BASE_SHIFT
|
|
add AO, AO, TEMP
|
|
add BO, B, r0
|
|
|
|
fpmr f1, f0
|
|
addi BO, BO, - 2 * SIZE
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
#endif
|
|
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 1
|
|
#else
|
|
addi TEMP, KK, 1
|
|
#endif
|
|
srawi. r0, TEMP, 3
|
|
mtspr CTR, r0
|
|
ble .L124
|
|
#else
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
|
|
srawi. r0, K, 3
|
|
mtspr CTR, r0
|
|
ble .L124
|
|
#endif
|
|
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
LFPDUX A3, AO, INC2
|
|
LFPDUX B3, BO, INC2
|
|
LFPDUX A4, AO, INC2
|
|
LFPDUX B4, BO, INC2
|
|
bdz- .L123
|
|
.align 4
|
|
|
|
.L122:
|
|
fpmadd f0, A1, B1, f0
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
fpmadd f1, A2, B2, f1
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
fpmadd f2, A3, B3, f2
|
|
LFPDUX A3, AO, INC2
|
|
LFPDUX B3, BO, INC2
|
|
fpmadd f3, A4, B4, f3
|
|
LFPDUX A4, AO, INC2
|
|
LFPDUX B4, BO, INC2
|
|
bdnz+ .L122
|
|
.align 4
|
|
|
|
.L123:
|
|
fpmadd f0, A1, B1, f0
|
|
fpmadd f1, A2, B2, f1
|
|
fpmadd f2, A3, B3, f2
|
|
fpmadd f3, A4, B4, f3
|
|
.align 4
|
|
|
|
.L124:
|
|
lfd AP, ALPHA(SP)
|
|
#ifdef TRMMKERNEL
|
|
fsmfp AP, AP
|
|
#endif
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 1
|
|
#else
|
|
addi TEMP, KK, 1
|
|
#endif
|
|
andi. TEMP, TEMP, 7
|
|
mtspr CTR, TEMP
|
|
#else
|
|
andi. r0, K, 7
|
|
mtspr CTR, r0
|
|
#endif
|
|
ble+ .L128
|
|
|
|
LFDX A1, AO, INC2
|
|
LFDX B1, BO, INC2
|
|
add AO, AO, INC
|
|
add BO, BO, INC
|
|
bdz- .L127
|
|
.align 4
|
|
|
|
.L126:
|
|
fmadd f0, A1, B1, f0
|
|
LFDX A1, AO, INC2
|
|
LFDX B1, BO, INC2
|
|
add AO, AO, INC
|
|
add BO, BO, INC
|
|
bdnz+ .L126
|
|
.align 4
|
|
|
|
.L127:
|
|
fmadd f0, A1, B1, f0
|
|
.align 4
|
|
|
|
.L128:
|
|
#ifndef TRMMKERNEL
|
|
LFDX A1, CO1, INC2
|
|
fpadd f0, f0, f1
|
|
fpadd f2, f2, f3
|
|
fpadd f0, f0, f2
|
|
fsmtp f1, f0
|
|
fadd f0, f0, f1
|
|
fmadd f0, AP, f0, A1
|
|
#else
|
|
fpadd f0, f0, f1
|
|
fpadd f2, f2, f3
|
|
fpadd f0, f0, f2
|
|
fsmtp f1, f0
|
|
fadd f0, f0, f1
|
|
fpmul f0, AP, f0
|
|
#endif
|
|
STFDUX f0, CO1, INC2
|
|
.align 4
|
|
|
|
.L999:
|
|
addi SP, SP, 12
|
|
|
|
lwzu r14, 4(SP)
|
|
lwzu r15, 4(SP)
|
|
|
|
lwzu r16, 4(SP)
|
|
lwzu r17, 4(SP)
|
|
lwzu r18, 4(SP)
|
|
lwzu r19, 4(SP)
|
|
|
|
lwzu r20, 4(SP)
|
|
lwzu r21, 4(SP)
|
|
lwzu r22, 4(SP)
|
|
lwzu r23, 4(SP)
|
|
|
|
lwzu r24, 4(SP)
|
|
lwzu r25, 4(SP)
|
|
lwzu r26, 4(SP)
|
|
lwzu r27, 4(SP)
|
|
|
|
lwzu r28, 4(SP)
|
|
lwzu r29, 4(SP)
|
|
lwzu r30, 4(SP)
|
|
lwzu r31, 4(SP)
|
|
|
|
subi SP, SP, 12
|
|
li r0, 16
|
|
|
|
lfpdux f31, SP, r0
|
|
lfpdux f30, SP, r0
|
|
lfpdux f29, SP, r0
|
|
lfpdux f28, SP, r0
|
|
lfpdux f27, SP, r0
|
|
lfpdux f26, SP, r0
|
|
lfpdux f25, SP, r0
|
|
lfpdux f24, SP, r0
|
|
lfpdux f23, SP, r0
|
|
lfpdux f22, SP, r0
|
|
lfpdux f21, SP, r0
|
|
lfpdux f20, SP, r0
|
|
lfpdux f19, SP, r0
|
|
lfpdux f18, SP, r0
|
|
lfpdux f17, SP, r0
|
|
lfpdux f16, SP, r0
|
|
lfpdux f15, SP, r0
|
|
lfpdux f14, SP, r0
|
|
addi SP, SP, 16
|
|
blr
|
|
.align 4
|
|
|
|
.L1000:
|
|
li INCM1, -1 * SIZE
|
|
li INCM3, -3 * SIZE
|
|
li INCM5, -5 * SIZE
|
|
li INCM7, -7 * SIZE
|
|
|
|
addi C, C, - 1 * SIZE
|
|
srawi. J, N, 2
|
|
ble .L1050
|
|
.align 4
|
|
|
|
.L1010:
|
|
mr CO1, C
|
|
add CO2, C, LDC
|
|
add CO3, CO2, LDC
|
|
add CO4, CO3, LDC
|
|
add C, CO4, LDC
|
|
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
mr KK, OFFSET
|
|
#endif
|
|
|
|
addi AO, A, -4 * SIZE
|
|
|
|
li r0, FZERO
|
|
lfpsx f0, SP, r0
|
|
|
|
srawi. I, M, 3
|
|
ble .L1020
|
|
.align 4
|
|
|
|
.L1011:
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
|
addi AO2, AO, 2 * SIZE
|
|
fpmr f4, f0
|
|
addi BO, B, - 4 * SIZE
|
|
fpmr f8, f0
|
|
addi BO2, B, - 2 * SIZE
|
|
fpmr f12, f0
|
|
#else
|
|
slwi TEMP, KK, 3 + BASE_SHIFT
|
|
slwi r0, KK, 2 + BASE_SHIFT
|
|
add AO, AO, TEMP
|
|
add BO, B, r0
|
|
|
|
addi AO2, AO, 2 * SIZE
|
|
fpmr f4, f0
|
|
addi BO, BO, - 4 * SIZE
|
|
fpmr f8, f0
|
|
addi BO2, BO, 2 * SIZE
|
|
fpmr f12, f0
|
|
#endif
|
|
|
|
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 8
|
|
#else
|
|
addi TEMP, KK, 4
|
|
#endif
|
|
srawi. TEMP, TEMP, 2
|
|
fpmr f1, f0
|
|
mtspr CTR, TEMP
|
|
ble .L1014
|
|
|
|
#else
|
|
addi AO2, AO, 2 * SIZE
|
|
fpmr f4, f0
|
|
addi BO, B, - 4 * SIZE
|
|
fpmr f8, f0
|
|
addi BO2, B, - 2 * SIZE
|
|
fpmr f12, f0
|
|
|
|
srawi. r0, K, 2
|
|
fpmr f1, f0
|
|
mtspr CTR, r0
|
|
ble .L1014
|
|
#endif
|
|
|
|
LFPDUX A1, AO, INC4
|
|
fpmr f5, f0
|
|
LFPDUX A3, AO, INC4
|
|
fpmr f9, f0
|
|
LFPDUX B1, BO, INC4
|
|
fpmr f13, f0
|
|
|
|
LFPDUX A5, AO, INC4
|
|
fpmr f2, f0
|
|
LFPDUX A6, AO, INC4
|
|
fpmr f6, f0
|
|
LFPDUX B3, BO, INC4
|
|
fpmr f10, f0
|
|
LFPDUX A7, AO, INC4
|
|
fpmr f14, f0
|
|
|
|
LFPDUX A8, AO, INC4
|
|
fpmr f3, f0
|
|
LFPDUX B5, BO, INC4
|
|
fpmr f7, f0
|
|
LFPDUX A9, AO, INC4
|
|
fpmr f11, f0
|
|
LFPDUX A2, AO2, INC4
|
|
fpmr f15, f0
|
|
LFPDUX B2, BO2, INC4
|
|
bdz- .L1013
|
|
.align 4
|
|
|
|
.L1012:
|
|
|
|
## 1 ##
|
|
fxcpmadd f0, B1, A1, f0
|
|
nop
|
|
fxcsmadd f4, B1, A1, f4
|
|
nop
|
|
fxcpmadd f8, B2, A1, f8
|
|
LFPDUX B4, BO2, INC4
|
|
fxcsmadd f12, B2, A1, f12
|
|
LFPDUX B6, BO, INC4
|
|
|
|
fxcpmadd f1, B1, A2, f1
|
|
nop
|
|
fxcsmadd f5, B1, A2, f5
|
|
LFPDUX A4, AO2, INC4
|
|
fxcpmadd f9, B2, A2, f9
|
|
LFPDUX A10, AO, INC4
|
|
fxcsmadd f13, B2, A2, f13
|
|
nop
|
|
|
|
fxcpmadd f2, B1, A3, f2
|
|
nop
|
|
fxcsmadd f6, B1, A3, f6
|
|
nop
|
|
fxcpmadd f10, B2, A3, f10
|
|
nop
|
|
fxcsmadd f14, B2, A3, f14
|
|
nop
|
|
|
|
fxcpmadd f3, B1, A4, f3
|
|
nop
|
|
fxcsmadd f7, B1, A4, f7
|
|
LFPDUX A2, AO2, INC4
|
|
fxcpmadd f11, B2, A4, f11
|
|
LFPDUX A1, AO, INC4
|
|
fxcsmadd f15, B2, A4, f15
|
|
nop
|
|
|
|
## 2 ##
|
|
|
|
fxcpmadd f0, B3, A5, f0
|
|
nop
|
|
fxcsmadd f4, B3, A5, f4
|
|
nop
|
|
fxcpmadd f8, B4, A5, f8
|
|
LFPDUX B2, BO2, INC4
|
|
fxcsmadd f12, B4, A5, f12
|
|
LFPDUX B1, BO, INC4
|
|
|
|
fxcpmadd f1, B3, A2, f1
|
|
nop
|
|
fxcsmadd f5, B3, A2, f5
|
|
LFPDUX A4, AO2, INC4
|
|
fxcpmadd f9, B4, A2, f9
|
|
LFPDUX A3, AO, INC4
|
|
fxcsmadd f13, B4, A2, f13
|
|
nop
|
|
|
|
fxcpmadd f2, B3, A6, f2
|
|
nop
|
|
fxcsmadd f6, B3, A6, f6
|
|
nop
|
|
fxcpmadd f10, B4, A6, f10
|
|
nop
|
|
fxcsmadd f14, B4, A6, f14
|
|
nop
|
|
|
|
fxcpmadd f3, B3, A4, f3
|
|
nop
|
|
fxcsmadd f7, B3, A4, f7
|
|
LFPDUX A2, AO2, INC4
|
|
fxcpmadd f11, B4, A4, f11
|
|
LFPDUX A5, AO, INC4
|
|
fxcsmadd f15, B4, A4, f15
|
|
nop
|
|
|
|
## 3 ##
|
|
|
|
fxcpmadd f0, B5, A7, f0
|
|
nop
|
|
fxcsmadd f4, B5, A7, f4
|
|
nop
|
|
fxcpmadd f8, B2, A7, f8
|
|
LFPDUX B4, BO2, INC4
|
|
fxcsmadd f12, B2, A7, f12
|
|
LFPDUX B3, BO, INC4
|
|
|
|
fxcpmadd f1, B5, A2, f1
|
|
nop
|
|
fxcsmadd f5, B5, A2, f5
|
|
LFPDUX A4, AO2, INC4
|
|
fxcpmadd f9, B2, A2, f9
|
|
LFPDUX A6, AO, INC4
|
|
fxcsmadd f13, B2, A2, f13
|
|
nop
|
|
|
|
fxcpmadd f2, B5, A8, f2
|
|
nop
|
|
fxcsmadd f6, B5, A8, f6
|
|
nop
|
|
fxcpmadd f10, B2, A8, f10
|
|
nop
|
|
fxcsmadd f14, B2, A8, f14
|
|
nop
|
|
|
|
fxcpmadd f3, B5, A4, f3
|
|
nop
|
|
fxcsmadd f7, B5, A4, f7
|
|
LFPDUX A2, AO2, INC4
|
|
fxcpmadd f11, B2, A4, f11
|
|
LFPDUX A7, AO, INC4
|
|
fxcsmadd f15, B2, A4, f15
|
|
nop
|
|
|
|
## 4 ##
|
|
fxcpmadd f0, B6, A9, f0
|
|
nop
|
|
fxcsmadd f4, B6, A9, f4
|
|
nop
|
|
fxcpmadd f8, B4, A9, f8
|
|
LFPDUX B2, BO2, INC4
|
|
fxcsmadd f12, B4, A9, f12
|
|
LFPDUX B5, BO, INC4
|
|
|
|
fxcpmadd f1, B6, A2, f1
|
|
nop
|
|
fxcsmadd f5, B6, A2, f5
|
|
LFPDUX A4, AO2, INC4
|
|
fxcpmadd f9, B4, A2, f9
|
|
LFPDUX A8, AO, INC4
|
|
fxcsmadd f13, B4, A2, f13
|
|
nop
|
|
|
|
fxcpmadd f2, B6, A10, f2
|
|
nop
|
|
fxcsmadd f6, B6, A10, f6
|
|
nop
|
|
fxcpmadd f10, B4, A10, f10
|
|
nop
|
|
fxcsmadd f14, B4, A10, f14
|
|
nop
|
|
|
|
fxcpmadd f3, B6, A4, f3
|
|
LFPDUX A2, AO2, INC4
|
|
fxcsmadd f7, B6, A4, f7
|
|
LFPDUX A9, AO, INC4
|
|
fxcpmadd f11, B4, A4, f11
|
|
nop
|
|
fxcsmadd f15, B4, A4, f15
|
|
bdnz+ .L1012
|
|
.align 4
|
|
|
|
.L1013:
|
|
## 1 ##
|
|
|
|
fxcpmadd f0, B1, A1, f0
|
|
nop
|
|
fxcsmadd f4, B1, A1, f4
|
|
nop
|
|
fxcpmadd f8, B2, A1, f8
|
|
LFPDUX B4, BO2, INC4
|
|
fxcsmadd f12, B2, A1, f12
|
|
LFPDUX B6, BO, INC4
|
|
|
|
fxcpmadd f1, B1, A2, f1
|
|
nop
|
|
fxcsmadd f5, B1, A2, f5
|
|
LFPDUX A4, AO2, INC4
|
|
fxcpmadd f9, B2, A2, f9
|
|
LFPDUX A10, AO, INC4
|
|
fxcsmadd f13, B2, A2, f13
|
|
nop
|
|
|
|
fxcpmadd f2, B1, A3, f2
|
|
nop
|
|
fxcsmadd f6, B1, A3, f6
|
|
nop
|
|
fxcpmadd f10, B2, A3, f10
|
|
nop
|
|
fxcsmadd f14, B2, A3, f14
|
|
nop
|
|
|
|
fxcpmadd f3, B1, A4, f3
|
|
nop
|
|
fxcsmadd f7, B1, A4, f7
|
|
LFPDUX A2, AO2, INC4
|
|
fxcpmadd f11, B2, A4, f11
|
|
#ifndef TRMMKERNEL
|
|
LFDUX A1, CO1, INC
|
|
#else
|
|
nop
|
|
#endif
|
|
fxcsmadd f15, B2, A4, f15
|
|
nop
|
|
|
|
## 2 ##
|
|
|
|
fxcpmadd f0, B3, A5, f0
|
|
nop
|
|
fxcsmadd f4, B3, A5, f4
|
|
nop
|
|
fxcpmadd f8, B4, A5, f8
|
|
LFPDUX B2, BO2, INC4
|
|
fxcsmadd f12, B4, A5, f12
|
|
#ifndef TRMMKERNEL
|
|
LFDUX B1, CO1, INC2
|
|
#else
|
|
nop
|
|
#endif
|
|
|
|
fxcpmadd f1, B3, A2, f1
|
|
nop
|
|
fxcsmadd f5, B3, A2, f5
|
|
LFPDUX A4, AO2, INC4
|
|
fxcpmadd f9, B4, A2, f9
|
|
#ifndef TRMMKERNEL
|
|
LFDUX A3, CO1, INC2
|
|
#else
|
|
nop
|
|
#endif
|
|
fxcsmadd f13, B4, A2, f13
|
|
nop
|
|
|
|
fxcpmadd f2, B3, A6, f2
|
|
nop
|
|
fxcsmadd f6, B3, A6, f6
|
|
nop
|
|
fxcpmadd f10, B4, A6, f10
|
|
nop
|
|
fxcsmadd f14, B4, A6, f14
|
|
nop
|
|
|
|
fxcpmadd f3, B3, A4, f3
|
|
nop
|
|
fxcsmadd f7, B3, A4, f7
|
|
LFPDUX A2, AO2, INC4
|
|
fxcpmadd f11, B4, A4, f11
|
|
#ifndef TRMMKERNEL
|
|
LFDUX A5, CO1, INC2
|
|
#else
|
|
nop
|
|
#endif
|
|
fxcsmadd f15, B4, A4, f15
|
|
nop
|
|
|
|
## 3 ##
|
|
|
|
fxcpmadd f0, B5, A7, f0
|
|
nop
|
|
fxcsmadd f4, B5, A7, f4
|
|
nop
|
|
fxcpmadd f8, B2, A7, f8
|
|
LFPDUX B4, BO2, INC4
|
|
fxcsmadd f12, B2, A7, f12
|
|
#ifndef TRMMKERNEL
|
|
LFSDUX A1, CO1, INCM5
|
|
#else
|
|
nop
|
|
#endif
|
|
|
|
fxcpmadd f1, B5, A2, f1
|
|
nop
|
|
fxcsmadd f5, B5, A2, f5
|
|
LFPDUX A4, AO2, INC4
|
|
fxcpmadd f9, B2, A2, f9
|
|
#ifndef TRMMKERNEL
|
|
LFSDUX B1, CO1, INC2
|
|
#else
|
|
nop
|
|
#endif
|
|
fxcsmadd f13, B2, A2, f13
|
|
nop
|
|
|
|
fxcpmadd f2, B5, A8, f2
|
|
nop
|
|
fxcsmadd f6, B5, A8, f6
|
|
nop
|
|
fxcpmadd f10, B2, A8, f10
|
|
nop
|
|
fxcsmadd f14, B2, A8, f14
|
|
nop
|
|
|
|
fxcpmadd f3, B5, A4, f3
|
|
nop
|
|
fxcsmadd f7, B5, A4, f7
|
|
LFPDUX A2, AO2, INC4
|
|
fxcpmadd f11, B2, A4, f11
|
|
#ifndef TRMMKERNEL
|
|
LFSDUX A3, CO1, INC2
|
|
#else
|
|
nop
|
|
#endif
|
|
fxcsmadd f15, B2, A4, f15
|
|
nop
|
|
|
|
## 4 ##
|
|
|
|
fxcpmadd f0, B6, A9, f0
|
|
nop
|
|
fxcsmadd f4, B6, A9, f4
|
|
nop
|
|
fxcpmadd f8, B4, A9, f8
|
|
#ifndef TRMMKERNEL
|
|
LFSDUX A5, CO1, INC2
|
|
#else
|
|
nop
|
|
#endif
|
|
fxcsmadd f12, B4, A9, f12
|
|
#ifndef TRMMKERNEL
|
|
LFDUX B3, CO2, INC
|
|
#else
|
|
nop
|
|
#endif
|
|
|
|
fxcpmadd f1, B6, A2, f1
|
|
nop
|
|
fxcsmadd f5, B6, A2, f5
|
|
LFPDUX A4, AO2, INC4
|
|
fxcpmadd f9, B4, A2, f9
|
|
#ifndef TRMMKERNEL
|
|
LFDUX A6, CO2, INC2
|
|
#else
|
|
nop
|
|
#endif
|
|
fxcsmadd f13, B4, A2, f13
|
|
nop
|
|
|
|
fxcpmadd f2, B6, A10, f2
|
|
nop
|
|
fxcsmadd f6, B6, A10, f6
|
|
nop
|
|
fxcpmadd f10, B4, A10, f10
|
|
nop
|
|
fxcsmadd f14, B4, A10, f14
|
|
#ifndef TRMMKERNEL
|
|
LFDUX A7, CO2, INC2
|
|
#else
|
|
nop
|
|
#endif
|
|
|
|
fxcpmadd f3, B6, A4, f3
|
|
nop
|
|
fxcsmadd f7, B6, A4, f7
|
|
nop
|
|
fxcpmadd f11, B4, A4, f11
|
|
nop
|
|
fxcsmadd f15, B4, A4, f15
|
|
#ifndef TRMMKERNEL
|
|
LFDUX B2, CO2, INC2
|
|
#else
|
|
nop
|
|
#endif
|
|
.align 4
|
|
|
|
.L1014:
|
|
lfd AP, ALPHA(SP)
|
|
#ifdef TRMMKERNEL
|
|
fsmfp AP, AP
|
|
#endif
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 8
|
|
#else
|
|
addi TEMP, KK, 4
|
|
#endif
|
|
andi. r0, TEMP, 3
|
|
mtspr CTR, r0
|
|
ble+ .L1018
|
|
|
|
cmpwi cr0, TEMP, 3
|
|
bgt+ .L1015
|
|
#else
|
|
andi. r0, K, 3
|
|
mtspr CTR, r0
|
|
ble+ .L1018
|
|
|
|
cmpwi cr0, K, 3
|
|
bgt+ .L1015
|
|
#endif
|
|
|
|
#ifndef TRMMKERNEL
|
|
LFDUX A1, CO1, INC
|
|
fpmr f5, f0
|
|
LFDUX B1, CO1, INC2
|
|
fpmr f9, f0
|
|
LFDUX A3, CO1, INC2
|
|
fpmr f13, f0
|
|
LFDUX A5, CO1, INC2
|
|
fpmr f2, f0
|
|
|
|
LFSDUX A1, CO1, INCM5
|
|
fpmr f6, f0
|
|
LFSDUX B1, CO1, INC2
|
|
fpmr f10, f0
|
|
LFSDUX A3, CO1, INC2
|
|
fpmr f14, f0
|
|
LFSDUX A5, CO1, INC2
|
|
fpmr f3, f0
|
|
|
|
LFDUX B3, CO2, INC
|
|
fpmr f7, f0
|
|
LFDUX A6, CO2, INC2
|
|
fpmr f11, f0
|
|
LFDUX A7, CO2, INC2
|
|
fpmr f15, f0
|
|
LFDUX B2, CO2, INC2
|
|
#else
|
|
fpmr f5, f0
|
|
fpmr f9, f0
|
|
fpmr f13, f0
|
|
fpmr f2, f0
|
|
|
|
fpmr f6, f0
|
|
fpmr f10, f0
|
|
fpmr f14, f0
|
|
fpmr f3, f0
|
|
|
|
fpmr f7, f0
|
|
fpmr f11, f0
|
|
fpmr f15, f0
|
|
nop
|
|
#endif
|
|
.align 4
|
|
|
|
.L1015:
|
|
LFPDUX A2, AO, INC4
|
|
LFPDUX A4, AO2, INC4
|
|
LFPDUX A10, BO, INC4
|
|
LFPDUX B4, BO2, INC4
|
|
bdz- .L1017
|
|
.align 4
|
|
|
|
.L1016:
|
|
fxcpmadd f0, A10, A2, f0
|
|
fxcsmadd f4, A10, A2, f4
|
|
fxcpmadd f8, B4, A2, f8
|
|
fxcsmadd f12, B4, A2, f12
|
|
LFPDUX A2, AO, INC4
|
|
|
|
fxcpmadd f1, A10, A4, f1
|
|
fxcsmadd f5, A10, A4, f5
|
|
fxcpmadd f9, B4, A4, f9
|
|
fxcsmadd f13, B4, A4, f13
|
|
LFPDUX A4, AO2, INC4
|
|
|
|
fxcpmadd f2, A10, A2, f2
|
|
fxcsmadd f6, A10, A2, f6
|
|
fxcpmadd f10, B4, A2, f10
|
|
fxcsmadd f14, B4, A2, f14
|
|
LFPDUX A2, AO, INC4
|
|
|
|
fxcpmadd f3, A10, A4, f3
|
|
fxcsmadd f7, A10, A4, f7
|
|
LFPDUX A10, BO, INC4
|
|
fxcpmadd f11, B4, A4, f11
|
|
fxcsmadd f15, B4, A4, f15
|
|
LFPDUX A4, AO2, INC4
|
|
LFPDUX B4, BO2, INC4
|
|
bdnz+ .L1016
|
|
.align 4
|
|
|
|
.L1017:
|
|
fxcpmadd f0, A10, A2, f0
|
|
fxcsmadd f4, A10, A2, f4
|
|
fxcpmadd f8, B4, A2, f8
|
|
fxcsmadd f12, B4, A2, f12
|
|
LFPDUX A2, AO, INC4
|
|
|
|
fxcpmadd f1, A10, A4, f1
|
|
fxcsmadd f5, A10, A4, f5
|
|
fxcpmadd f9, B4, A4, f9
|
|
fxcsmadd f13, B4, A4, f13
|
|
LFPDUX A4, AO2, INC4
|
|
|
|
fxcpmadd f2, A10, A2, f2
|
|
fxcsmadd f6, A10, A2, f6
|
|
fxcpmadd f10, B4, A2, f10
|
|
fxcsmadd f14, B4, A2, f14
|
|
|
|
fxcpmadd f3, A10, A4, f3
|
|
fxcsmadd f7, A10, A4, f7
|
|
fxcpmadd f11, B4, A4, f11
|
|
fxcsmadd f15, B4, A4, f15
|
|
.align 4
|
|
|
|
.L1018:
|
|
#ifndef TRMMKERNEL
|
|
LFSDUX B3, CO2, INCM5
|
|
LFSDUX A6, CO2, INC2
|
|
LFSDUX A7, CO2, INC2
|
|
LFSDUX B2, CO2, INC2
|
|
|
|
LFDUX B5, CO3, INC
|
|
LFDUX A8, CO3, INC2
|
|
LFDUX A9, CO3, INC2
|
|
LFDUX B4, CO3, INC2
|
|
|
|
LFSDUX B5, CO3, INCM5
|
|
LFSDUX A8, CO3, INC2
|
|
LFSDUX A9, CO3, INC2
|
|
LFSDUX B4, CO3, INC2
|
|
|
|
LFDUX A2, CO4, INC
|
|
LFDUX A4, CO4, INC2
|
|
|
|
fxcpmadd f0, AP, f0, A1
|
|
LFDUX A10, CO4, INC2
|
|
LFDUX A1, CO4, INC2
|
|
|
|
fxcpmadd f1, AP, f1, B1
|
|
LFSDUX A2, CO4, INCM5
|
|
LFSDUX A4, CO4, INC2
|
|
|
|
fxcpmadd f2, AP, f2, A3
|
|
LFSDUX A10, CO4, INC2
|
|
LFSDUX A1, CO4, INC2
|
|
|
|
fxcpmadd f3, AP, f3, A5
|
|
STFDUX f0, CO1, INCM7
|
|
STFSDUX f0, CO1, INC
|
|
|
|
fxcpmadd f4, AP, f4, B3
|
|
STFDUX f1, CO1, INC
|
|
STFSDUX f1, CO1, INC
|
|
|
|
fxcpmadd f5, AP, f5, A6
|
|
STFDUX f2, CO1, INC
|
|
STFSDUX f2, CO1, INC
|
|
|
|
fxcpmadd f6, AP, f6, A7
|
|
STFDUX f3, CO1, INC
|
|
STFSDUX f3, CO1, INC
|
|
|
|
fxcpmadd f7, AP, f7, B2
|
|
STFDUX f4, CO2, INCM7
|
|
STFSDUX f4, CO2, INC
|
|
|
|
fxcpmadd f8, AP, f8, B5
|
|
STFDUX f5, CO2, INC
|
|
STFSDUX f5, CO2, INC
|
|
|
|
fxcpmadd f9, AP, f9, A8
|
|
STFDUX f6, CO2, INC
|
|
STFSDUX f6, CO2, INC
|
|
|
|
fxcpmadd f10, AP, f10, A9
|
|
STFDUX f7, CO2, INC
|
|
STFSDUX f7, CO2, INC
|
|
|
|
fxcpmadd f11, AP, f11, B4
|
|
STFDUX f8, CO3, INCM7
|
|
STFSDUX f8, CO3, INC
|
|
|
|
fxcpmadd f12, AP, f12, A2
|
|
STFDUX f9, CO3, INC
|
|
STFSDUX f9, CO3, INC
|
|
|
|
fxcpmadd f13, AP, f13, A4
|
|
STFDUX f10, CO3, INC
|
|
STFSDUX f10, CO3, INC
|
|
|
|
fxcpmadd f14, AP, f14, A10
|
|
STFDUX f11, CO3, INC
|
|
STFSDUX f11, CO3, INC
|
|
|
|
fxcpmadd f15, AP, f15, A1
|
|
STFDUX f12, CO4, INCM7
|
|
#else
|
|
fpmul f0, AP, f0
|
|
fpmul f1, AP, f1
|
|
fpmul f2, AP, f2
|
|
fpmul f3, AP, f3
|
|
|
|
STFDUX f0, CO1, INC
|
|
STFSDUX f0, CO1, INC
|
|
|
|
fpmul f4, AP, f4
|
|
STFDUX f1, CO1, INC
|
|
STFSDUX f1, CO1, INC
|
|
|
|
fpmul f5, AP, f5
|
|
STFDUX f2, CO1, INC
|
|
STFSDUX f2, CO1, INC
|
|
|
|
fpmul f6, AP, f6
|
|
STFDUX f3, CO1, INC
|
|
STFSDUX f3, CO1, INC
|
|
|
|
fpmul f7, AP, f7
|
|
STFDUX f4, CO2, INC
|
|
STFSDUX f4, CO2, INC
|
|
|
|
fpmul f8, AP, f8
|
|
STFDUX f5, CO2, INC
|
|
STFSDUX f5, CO2, INC
|
|
|
|
fpmul f9, AP, f9
|
|
STFDUX f6, CO2, INC
|
|
STFSDUX f6, CO2, INC
|
|
|
|
fpmul f10, AP, f10
|
|
STFDUX f7, CO2, INC
|
|
STFSDUX f7, CO2, INC
|
|
|
|
fpmul f11, AP, f11
|
|
STFDUX f8, CO3, INC
|
|
STFSDUX f8, CO3, INC
|
|
|
|
fpmul f12, AP, f12
|
|
STFDUX f9, CO3, INC
|
|
STFSDUX f9, CO3, INC
|
|
|
|
fpmul f13, AP, f13
|
|
STFDUX f10, CO3, INC
|
|
STFSDUX f10, CO3, INC
|
|
|
|
fpmul f14, AP, f14
|
|
STFDUX f11, CO3, INC
|
|
STFSDUX f11, CO3, INC
|
|
|
|
fpmul f15, AP, f15
|
|
STFDUX f12, CO4, INC
|
|
#endif
|
|
|
|
STFSDUX f12, CO4, INC
|
|
STFDUX f13, CO4, INC
|
|
STFSDUX f13, CO4, INC
|
|
STFDUX f14, CO4, INC
|
|
STFSDUX f14, CO4, INC
|
|
STFDUX f15, CO4, INC
|
|
STFSDUX f15, CO4, INC
|
|
|
|
#ifdef TRMMKERNEL
|
|
#if ( defined(LEFT) && defined(TRANSA)) || \
|
|
(!defined(LEFT) && !defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#ifdef LEFT
|
|
addi TEMP, TEMP, -8
|
|
#else
|
|
addi TEMP, TEMP, -4
|
|
#endif
|
|
slwi r0, TEMP, 3 + BASE_SHIFT
|
|
slwi TEMP, TEMP, 2 + BASE_SHIFT
|
|
add AO, AO, r0
|
|
add BO, BO, TEMP
|
|
#endif
|
|
|
|
#ifdef LEFT
|
|
addi KK, KK, 8
|
|
#endif
|
|
#endif
|
|
|
|
addic. I, I, -1
|
|
li r0, FZERO
|
|
|
|
lfpsx f0, SP, r0
|
|
bgt+ .L1011
|
|
.align 4
|
|
|
|
.L1020:
|
|
andi. I, M, 4
|
|
beq .L1030
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
|
addi AO2, AO, 2 * SIZE
|
|
fpmr f4, f0
|
|
addi BO, B, - 4 * SIZE
|
|
fpmr f8, f0
|
|
addi BO2, B, - 2 * SIZE
|
|
fpmr f12, f0
|
|
#else
|
|
slwi TEMP, KK, 2 + BASE_SHIFT
|
|
slwi r0, KK, 2 + BASE_SHIFT
|
|
add AO, AO, TEMP
|
|
add BO, B, r0
|
|
|
|
addi AO2, AO, 2 * SIZE
|
|
fpmr f4, f0
|
|
addi BO, BO, - 4 * SIZE
|
|
fpmr f8, f0
|
|
addi BO2, BO, 2 * SIZE
|
|
fpmr f12, f0
|
|
#endif
|
|
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 4
|
|
#else
|
|
addi TEMP, KK, 4
|
|
#endif
|
|
|
|
srawi. TEMP, TEMP, 2
|
|
fpmr f1, f0
|
|
fpmr f5, f0
|
|
fpmr f9, f0
|
|
mtspr CTR, TEMP
|
|
fpmr f13, f0
|
|
ble .L1024
|
|
#else
|
|
addi AO2, AO, 2 * SIZE
|
|
fpmr f4, f0
|
|
addi BO, B, - 4 * SIZE
|
|
fpmr f8, f0
|
|
addi BO2, B, - 2 * SIZE
|
|
fpmr f12, f0
|
|
|
|
srawi. r0, K, 2
|
|
fpmr f1, f0
|
|
fpmr f5, f0
|
|
fpmr f9, f0
|
|
mtspr CTR, r0
|
|
fpmr f13, f0
|
|
ble .L1024
|
|
#endif
|
|
|
|
LFPDUX A1, AO, INC4
|
|
LFPDUX B1, BO, INC4
|
|
LFPDUX A2, AO2, INC4
|
|
LFPDUX B2, BO2, INC4
|
|
LFPDUX A3, AO, INC4
|
|
LFPDUX B3, BO, INC4
|
|
LFPDUX A4, AO2, INC4
|
|
LFPDUX B4, BO2, INC4
|
|
|
|
LFPDUX A5, AO, INC4
|
|
LFPDUX B5, BO, INC4
|
|
LFPDUX A6, AO2, INC4
|
|
LFPDUX B6, BO2, INC4
|
|
LFPDUX A7, AO, INC4
|
|
LFPDUX A9, BO, INC4
|
|
LFPDUX A10, BO2, INC4
|
|
bdz- .L1023
|
|
.align 4
|
|
|
|
.L1022:
|
|
fxcpmadd f0, B1, A1, f0
|
|
nop
|
|
fxcsmadd f4, B1, A1, f4
|
|
LFPDUX A8, AO2, INC4
|
|
fxcpmadd f8, B2, A1, f8
|
|
nop
|
|
fxcsmadd f12, B2, A1, f12
|
|
LFPDUX A1, AO, INC4
|
|
|
|
fxcpmadd f1, B1, A2, f1
|
|
nop
|
|
fxcsmadd f5, B1, A2, f5
|
|
LFPDUX B1, BO, INC4
|
|
fxcpmadd f9, B2, A2, f9
|
|
nop
|
|
fxcsmadd f13, B2, A2, f13
|
|
LFPDUX B2, BO2, INC4
|
|
|
|
fxcpmadd f0, B3, A3, f0
|
|
nop
|
|
fxcsmadd f4, B3, A3, f4
|
|
LFPDUX A2, AO2, INC4
|
|
fxcpmadd f8, B4, A3, f8
|
|
nop
|
|
fxcsmadd f12, B4, A3, f12
|
|
LFPDUX A3, AO, INC4
|
|
|
|
fxcpmadd f1, B3, A4, f1
|
|
nop
|
|
fxcsmadd f5, B3, A4, f5
|
|
LFPDUX B3, BO, INC4
|
|
fxcpmadd f9, B4, A4, f9
|
|
nop
|
|
fxcsmadd f13, B4, A4, f13
|
|
LFPDUX B4, BO2, INC4
|
|
|
|
fxcpmadd f0, B5, A5, f0
|
|
nop
|
|
fxcsmadd f4, B5, A5, f4
|
|
LFPDUX A4, AO2, INC4
|
|
fxcpmadd f8, B6, A5, f8
|
|
nop
|
|
fxcsmadd f12, B6, A5, f12
|
|
LFPDUX A5, AO, INC4
|
|
|
|
fxcpmadd f1, B5, A6, f1
|
|
nop
|
|
fxcsmadd f5, B5, A6, f5
|
|
LFPDUX B5, BO, INC4
|
|
fxcpmadd f9, B6, A6, f9
|
|
nop
|
|
fxcsmadd f13, B6, A6, f13
|
|
LFPDUX B6, BO2, INC4
|
|
|
|
fxcpmadd f0, A9, A7, f0
|
|
nop
|
|
fxcsmadd f4, A9, A7, f4
|
|
LFPDUX A6, AO2, INC4
|
|
fxcpmadd f8, A10, A7, f8
|
|
nop
|
|
fxcsmadd f12, A10, A7, f12
|
|
LFPDUX A7, AO, INC4
|
|
|
|
fxcpmadd f1, A9, A8, f1
|
|
nop
|
|
fxcsmadd f5, A9, A8, f5
|
|
LFPDUX A9, BO, INC4
|
|
fxcpmadd f9, A10, A8, f9
|
|
nop
|
|
fxcsmadd f13, A10, A8, f13
|
|
LFPDUX A10, BO2, INC4
|
|
bdnz+ .L1022
|
|
.align 4
|
|
|
|
.L1023:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f4, B1, A1, f4
|
|
LFPDUX A8, AO2, INC4
|
|
fxcpmadd f8, B2, A1, f8
|
|
fxcsmadd f12, B2, A1, f12
|
|
|
|
fxcpmadd f1, B1, A2, f1
|
|
fxcsmadd f5, B1, A2, f5
|
|
fxcpmadd f9, B2, A2, f9
|
|
fxcsmadd f13, B2, A2, f13
|
|
|
|
fxcpmadd f0, B3, A3, f0
|
|
fxcsmadd f4, B3, A3, f4
|
|
fxcpmadd f8, B4, A3, f8
|
|
fxcsmadd f12, B4, A3, f12
|
|
|
|
fxcpmadd f1, B3, A4, f1
|
|
fxcsmadd f5, B3, A4, f5
|
|
fxcpmadd f9, B4, A4, f9
|
|
fxcsmadd f13, B4, A4, f13
|
|
|
|
fxcpmadd f0, B5, A5, f0
|
|
fxcsmadd f4, B5, A5, f4
|
|
fxcpmadd f8, B6, A5, f8
|
|
fxcsmadd f12, B6, A5, f12
|
|
|
|
fxcpmadd f1, B5, A6, f1
|
|
fxcsmadd f5, B5, A6, f5
|
|
fxcpmadd f9, B6, A6, f9
|
|
fxcsmadd f13, B6, A6, f13
|
|
|
|
fxcpmadd f0, A9, A7, f0
|
|
fxcsmadd f4, A9, A7, f4
|
|
fxcpmadd f8, A10, A7, f8
|
|
fxcsmadd f12, A10, A7, f12
|
|
|
|
fxcpmadd f1, A9, A8, f1
|
|
fxcsmadd f5, A9, A8, f5
|
|
fxcpmadd f9, A10, A8, f9
|
|
fxcsmadd f13, A10, A8, f13
|
|
.align 4
|
|
|
|
.L1024:
|
|
lfd AP, ALPHA(SP)
|
|
#ifdef TRMMKERNEL
|
|
fsmfp AP, AP
|
|
#endif
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 4
|
|
#else
|
|
addi TEMP, KK, 4
|
|
#endif
|
|
andi. TEMP, TEMP, 3
|
|
mtspr CTR, TEMP
|
|
#else
|
|
andi. r0, K, 3
|
|
mtspr CTR, r0
|
|
#endif
|
|
ble+ .L1028
|
|
|
|
LFPDUX A1, AO, INC4
|
|
LFPDUX A2, AO2, INC4
|
|
LFPDUX B1, BO, INC4
|
|
LFPDUX B2, BO2, INC4
|
|
bdz- .L1027
|
|
.align 4
|
|
|
|
.L1026:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f4, B1, A1, f4
|
|
fxcpmadd f8, B2, A1, f8
|
|
fxcsmadd f12, B2, A1, f12
|
|
LFPDUX A1, AO, INC4
|
|
|
|
fxcpmadd f1, B1, A2, f1
|
|
fxcsmadd f5, B1, A2, f5
|
|
LFPDUX B1, BO, INC4
|
|
fxcpmadd f9, B2, A2, f9
|
|
fxcsmadd f13, B2, A2, f13
|
|
LFPDUX A2, AO2, INC4
|
|
LFPDUX B2, BO2, INC4
|
|
bdnz+ .L1026
|
|
.align 4
|
|
|
|
.L1027:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f4, B1, A1, f4
|
|
fxcpmadd f8, B2, A1, f8
|
|
fxcsmadd f12, B2, A1, f12
|
|
|
|
fxcpmadd f1, B1, A2, f1
|
|
fxcsmadd f5, B1, A2, f5
|
|
fxcpmadd f9, B2, A2, f9
|
|
fxcsmadd f13, B2, A2, f13
|
|
.align 4
|
|
|
|
.L1028:
|
|
#ifndef TRMMKERNEL
|
|
LFDUX A1, CO1, INC
|
|
LFDUX B1, CO1, INC2
|
|
LFDUX B3, CO2, INC
|
|
LFDUX A6, CO2, INC2
|
|
|
|
LFSDUX A1, CO1, INCM1
|
|
LFSDUX B1, CO1, INC2
|
|
LFSDUX B3, CO2, INCM1
|
|
LFSDUX A6, CO2, INC2
|
|
|
|
LFDUX B5, CO3, INC
|
|
LFDUX A8, CO3, INC2
|
|
LFDUX A2, CO4, INC
|
|
LFDUX A4, CO4, INC2
|
|
|
|
fxcpmadd f0, AP, f0, A1
|
|
LFSDUX B5, CO3, INCM1
|
|
LFSDUX A8, CO3, INC2
|
|
|
|
fxcpmadd f1, AP, f1, B1
|
|
LFSDUX A2, CO4, INCM1
|
|
LFSDUX A4, CO4, INC2
|
|
|
|
fxcpmadd f4, AP, f4, B3
|
|
STFDUX f0, CO1, INCM3
|
|
STFSDUX f0, CO1, INC
|
|
|
|
fxcpmadd f5, AP, f5, A6
|
|
STFDUX f1, CO1, INC
|
|
STFSDUX f1, CO1, INC
|
|
|
|
fxcpmadd f8, AP, f8, B5
|
|
STFDUX f4, CO2, INCM3
|
|
STFSDUX f4, CO2, INC
|
|
|
|
fxcpmadd f9, AP, f9, A8
|
|
STFDUX f5, CO2, INC
|
|
STFSDUX f5, CO2, INC
|
|
|
|
fxcpmadd f12, AP, f12, A2
|
|
STFDUX f8, CO3, INCM3
|
|
STFSDUX f8, CO3, INC
|
|
|
|
fxcpmadd f13, AP, f13, A4
|
|
STFDUX f9, CO3, INC
|
|
STFSDUX f9, CO3, INC
|
|
|
|
STFDUX f12, CO4, INCM3
|
|
STFSDUX f12, CO4, INC
|
|
|
|
STFDUX f13, CO4, INC
|
|
STFSDUX f13, CO4, INC
|
|
#else
|
|
fpmul f0, AP, f0
|
|
fpmul f1, AP, f1
|
|
|
|
fpmul f4, AP, f4
|
|
STFDUX f0, CO1, INC
|
|
STFSDUX f0, CO1, INC
|
|
|
|
fpmul f5, AP, f5
|
|
STFDUX f1, CO1, INC
|
|
STFSDUX f1, CO1, INC
|
|
|
|
fpmul f8, AP, f8
|
|
STFDUX f4, CO2, INC
|
|
STFSDUX f4, CO2, INC
|
|
|
|
fpmul f9, AP, f9
|
|
STFDUX f5, CO2, INC
|
|
STFSDUX f5, CO2, INC
|
|
|
|
fpmul f12, AP, f12
|
|
STFDUX f8, CO3, INC
|
|
STFSDUX f8, CO3, INC
|
|
|
|
fpmul f13, AP, f13
|
|
STFDUX f9, CO3, INC
|
|
STFSDUX f9, CO3, INC
|
|
|
|
STFDUX f12, CO4, INC
|
|
STFSDUX f12, CO4, INC
|
|
|
|
STFDUX f13, CO4, INC
|
|
STFSDUX f13, CO4, INC
|
|
#endif
|
|
|
|
|
|
#ifdef TRMMKERNEL
|
|
#if ( defined(LEFT) && defined(TRANSA)) || \
|
|
(!defined(LEFT) && !defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#ifdef LEFT
|
|
addi TEMP, TEMP, -4
|
|
#else
|
|
addi TEMP, TEMP, -4
|
|
#endif
|
|
slwi r0, TEMP, 2 + BASE_SHIFT
|
|
slwi TEMP, TEMP, 2 + BASE_SHIFT
|
|
add AO, AO, r0
|
|
add BO, BO, TEMP
|
|
#endif
|
|
|
|
#ifdef LEFT
|
|
addi KK, KK, 4
|
|
#endif
|
|
#endif
|
|
|
|
li r0, FZERO
|
|
lfpsx f0, SP, r0
|
|
.align 4
|
|
|
|
.L1030:
|
|
andi. I, M, 2
|
|
beq .L1040
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
|
addi AO2, AO, 2 * SIZE
|
|
fpmr f1, f0
|
|
addi BO, B, - 4 * SIZE
|
|
fpmr f2, f0
|
|
addi BO2, B, - 2 * SIZE
|
|
fpmr f3, f0
|
|
#else
|
|
slwi TEMP, KK, 1 + BASE_SHIFT
|
|
slwi r0, KK, 2 + BASE_SHIFT
|
|
add AO, AO, TEMP
|
|
add BO, B, r0
|
|
|
|
addi AO2, AO, 2 * SIZE
|
|
fpmr f1, f0
|
|
addi BO, BO, - 4 * SIZE
|
|
fpmr f2, f0
|
|
addi BO2, BO, 2 * SIZE
|
|
fpmr f3, f0
|
|
#endif
|
|
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 2
|
|
#else
|
|
addi TEMP, KK, 4
|
|
#endif
|
|
|
|
srawi. r0, TEMP, 2
|
|
mtspr CTR, r0
|
|
ble .L1034
|
|
|
|
#else
|
|
addi AO2, AO, 2 * SIZE
|
|
fpmr f1, f0
|
|
addi BO, B, - 4 * SIZE
|
|
fpmr f2, f0
|
|
addi BO2, B, - 2 * SIZE
|
|
fpmr f3, f0
|
|
|
|
srawi. r0, K, 2
|
|
mtspr CTR, r0
|
|
ble .L1034
|
|
#endif
|
|
|
|
LFPDUX A1, AO, INC4
|
|
LFPDUX B1, BO, INC4
|
|
LFPDUX B2, BO2, INC4
|
|
LFPDUX A2, AO2, INC4
|
|
LFPDUX B3, BO, INC4
|
|
LFPDUX B4, BO2, INC4
|
|
|
|
LFPDUX A3, AO, INC4
|
|
LFPDUX A5, BO, INC4
|
|
LFPDUX A6, BO2, INC4
|
|
LFPDUX A4, AO2, INC4
|
|
LFPDUX A7, BO, INC4
|
|
LFPDUX A8, BO2, INC4
|
|
bdz- .L1033
|
|
.align 4
|
|
|
|
.L1032:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f1, B1, A1, f1
|
|
LFPDUX B1, BO, INC4
|
|
fxcpmadd f2, B2, A1, f2
|
|
fxcsmadd f3, B2, A1, f3
|
|
LFPDUX B2, BO2, INC4
|
|
LFPDUX A1, AO, INC4
|
|
|
|
fxcpmadd f0, B3, A2, f0
|
|
fxcsmadd f1, B3, A2, f1
|
|
LFPDUX B3, BO, INC4
|
|
fxcpmadd f2, B4, A2, f2
|
|
fxcsmadd f3, B4, A2, f3
|
|
LFPDUX B4, BO2, INC4
|
|
LFPDUX A2, AO2, INC4
|
|
|
|
fxcpmadd f0, A5, A3, f0
|
|
fxcsmadd f1, A5, A3, f1
|
|
LFPDUX A5, BO, INC4
|
|
fxcpmadd f2, A6, A3, f2
|
|
fxcsmadd f3, A6, A3, f3
|
|
LFPDUX A6, BO2, INC4
|
|
LFPDUX A3, AO, INC4
|
|
|
|
fxcpmadd f0, A7, A4, f0
|
|
fxcsmadd f1, A7, A4, f1
|
|
LFPDUX A7, BO, INC4
|
|
fxcpmadd f2, A8, A4, f2
|
|
fxcsmadd f3, A8, A4, f3
|
|
LFPDUX A8, BO2, INC4
|
|
LFPDUX A4, AO2, INC4
|
|
bdnz+ .L1032
|
|
.align 4
|
|
|
|
.L1033:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f1, B1, A1, f1
|
|
fxcpmadd f2, B2, A1, f2
|
|
fxcsmadd f3, B2, A1, f3
|
|
|
|
fxcpmadd f0, B3, A2, f0
|
|
fxcsmadd f1, B3, A2, f1
|
|
fxcpmadd f2, B4, A2, f2
|
|
fxcsmadd f3, B4, A2, f3
|
|
|
|
fxcpmadd f0, A5, A3, f0
|
|
fxcsmadd f1, A5, A3, f1
|
|
fxcpmadd f2, A6, A3, f2
|
|
fxcsmadd f3, A6, A3, f3
|
|
|
|
fxcpmadd f0, A7, A4, f0
|
|
fxcsmadd f1, A7, A4, f1
|
|
fxcpmadd f2, A8, A4, f2
|
|
fxcsmadd f3, A8, A4, f3
|
|
.align 4
|
|
|
|
.L1034:
|
|
lfd AP, ALPHA(SP)
|
|
#ifdef TRMMKERNEL
|
|
fsmfp AP, AP
|
|
#endif
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 2
|
|
#else
|
|
addi TEMP, KK, 4
|
|
#endif
|
|
andi. TEMP, TEMP, 3
|
|
mtspr CTR, TEMP
|
|
#else
|
|
andi. r0, K, 3
|
|
mtspr CTR, r0
|
|
#endif
|
|
ble+ .L1038
|
|
|
|
LFPDX A1, AO, INC4
|
|
LFPDUX B1, BO, INC4
|
|
LFPDUX B2, BO2, INC4
|
|
add AO, AO, INC2
|
|
bdz- .L1037
|
|
.align 4
|
|
|
|
.L1036:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f1, B1, A1, f1
|
|
LFPDUX B1, BO, INC4
|
|
fxcpmadd f2, B2, A1, f2
|
|
fxcsmadd f3, B2, A1, f3
|
|
LFPDX A1, AO, INC4
|
|
LFPDUX B2, BO2, INC4
|
|
add AO, AO, INC2
|
|
bdnz+ .L1036
|
|
.align 4
|
|
|
|
.L1037:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f1, B1, A1, f1
|
|
fxcpmadd f2, B2, A1, f2
|
|
fxcsmadd f3, B2, A1, f3
|
|
.align 4
|
|
|
|
.L1038:
|
|
#ifndef TRMMKERNEL
|
|
LFDUX A1, CO1, INC
|
|
LFDUX A2, CO2, INC
|
|
LFDUX A3, CO3, INC
|
|
LFDUX A4, CO4, INC
|
|
|
|
LFSDUX A1, CO1, INC
|
|
LFSDUX A2, CO2, INC
|
|
LFSDUX A3, CO3, INC
|
|
LFSDUX A4, CO4, INC
|
|
|
|
fxcpmadd f0, AP, f0, A1
|
|
fxcpmadd f1, AP, f1, A2
|
|
fxcpmadd f2, AP, f2, A3
|
|
fxcpmadd f3, AP, f3, A4
|
|
|
|
STFDUX f0, CO1, INCM1
|
|
STFSDUX f0, CO1, INC
|
|
|
|
STFDUX f1, CO2, INCM1
|
|
STFSDUX f1, CO2, INC
|
|
|
|
STFDUX f2, CO3, INCM1
|
|
STFSDUX f2, CO3, INC
|
|
|
|
STFDUX f3, CO4, INCM1
|
|
STFSDUX f3, CO4, INC
|
|
#else
|
|
fpmul f0, AP, f0
|
|
fpmul f1, AP, f1
|
|
fpmul f2, AP, f2
|
|
fpmul f3, AP, f3
|
|
|
|
STFDUX f0, CO1, INC
|
|
STFSDUX f0, CO1, INC
|
|
|
|
STFDUX f1, CO2, INC
|
|
STFSDUX f1, CO2, INC
|
|
|
|
STFDUX f2, CO3, INC
|
|
STFSDUX f2, CO3, INC
|
|
|
|
STFDUX f3, CO4, INC
|
|
STFSDUX f3, CO4, INC
|
|
#endif
|
|
|
|
|
|
#ifdef TRMMKERNEL
|
|
#if ( defined(LEFT) && defined(TRANSA)) || \
|
|
(!defined(LEFT) && !defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#ifdef LEFT
|
|
addi TEMP, TEMP, -2
|
|
#else
|
|
addi TEMP, TEMP, -4
|
|
#endif
|
|
slwi r0, TEMP, 1 + BASE_SHIFT
|
|
slwi TEMP, TEMP, 2 + BASE_SHIFT
|
|
add AO, AO, r0
|
|
add BO, BO, TEMP
|
|
#endif
|
|
|
|
#ifdef LEFT
|
|
addi KK, KK, 2
|
|
#endif
|
|
#endif
|
|
|
|
li r0, FZERO
|
|
lfpsx f0, SP, r0
|
|
.align 4
|
|
|
|
.L1040:
|
|
andi. I, M, 1
|
|
beq .L1049
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
|
addi AO2, AO, 2 * SIZE
|
|
fpmr f1, f0
|
|
addi BO, B, - 4 * SIZE
|
|
fpmr f2, f0
|
|
addi BO2, B, - 2 * SIZE
|
|
fpmr f3, f0
|
|
#else
|
|
slwi TEMP, KK, 0 + BASE_SHIFT
|
|
slwi r0, KK, 2 + BASE_SHIFT
|
|
add AO, AO, TEMP
|
|
add BO, B, r0
|
|
|
|
addi AO2, AO, 2 * SIZE
|
|
fpmr f1, f0
|
|
addi BO, BO, - 4 * SIZE
|
|
fpmr f2, f0
|
|
addi BO2, BO, 2 * SIZE
|
|
fpmr f3, f0
|
|
#endif
|
|
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 1
|
|
#else
|
|
addi TEMP, KK, 4
|
|
#endif
|
|
srawi. r0, TEMP, 3
|
|
mtspr CTR, r0
|
|
ble .L1044
|
|
|
|
#else
|
|
addi AO2, AO, 2 * SIZE
|
|
fpmr f1, f0
|
|
addi BO, B, - 4 * SIZE
|
|
fpmr f2, f0
|
|
addi BO2, B, - 2 * SIZE
|
|
fpmr f3, f0
|
|
|
|
srawi. r0, K, 3
|
|
mtspr CTR, r0
|
|
ble .L1044
|
|
#endif
|
|
|
|
LFPDUX A1, AO, INC4
|
|
LFPDUX B1, BO, INC4
|
|
LFPDUX B2, BO2, INC4
|
|
LFPDUX A2, AO2, INC4
|
|
LFPDUX B3, BO, INC4
|
|
LFPDUX B4, BO2, INC4
|
|
|
|
LFPDUX A3, AO, INC4
|
|
LFPDUX A5, BO, INC4
|
|
LFPDUX A6, BO2, INC4
|
|
LFPDUX A4, AO2, INC4
|
|
LFPDUX A7, BO, INC4
|
|
LFPDUX A8, BO2, INC4
|
|
bdz- .L1043
|
|
.align 4
|
|
|
|
.L1042:
|
|
fxcpmadd f0, A1, B1, f0
|
|
LFPDUX B1, BO, INC4
|
|
fxcpmadd f1, A1, B2, f1
|
|
LFPDUX B2, BO2, INC4
|
|
fxcsmadd f2, A1, B3, f2
|
|
LFPDUX B3, BO, INC4
|
|
fxcsmadd f3, A1, B4, f3
|
|
LFPDUX B4, BO2, INC4
|
|
LFPDUX A1, AO, INC4
|
|
|
|
fxcpmadd f0, A2, A5, f0
|
|
LFPDUX A5, BO, INC4
|
|
fxcpmadd f1, A2, A6, f1
|
|
LFPDUX A6, BO2, INC4
|
|
fxcsmadd f2, A2, A7, f2
|
|
LFPDUX A7, BO, INC4
|
|
fxcsmadd f3, A2, A8, f3
|
|
LFPDUX A8, BO2, INC4
|
|
LFPDUX A2, AO2, INC4
|
|
|
|
fxcpmadd f0, A3, B1, f0
|
|
LFPDUX B1, BO, INC4
|
|
fxcpmadd f1, A3, B2, f1
|
|
LFPDUX B2, BO2, INC4
|
|
fxcsmadd f2, A3, B3, f2
|
|
LFPDUX B3, BO, INC4
|
|
fxcsmadd f3, A3, B4, f3
|
|
LFPDUX B4, BO2, INC4
|
|
LFPDUX A3, AO, INC4
|
|
|
|
fxcpmadd f0, A4, A5, f0
|
|
LFPDUX A5, BO, INC4
|
|
fxcpmadd f1, A4, A6, f1
|
|
LFPDUX A6, BO2, INC4
|
|
fxcsmadd f2, A4, A7, f2
|
|
LFPDUX A7, BO, INC4
|
|
fxcsmadd f3, A4, A8, f3
|
|
LFPDUX A8, BO2, INC4
|
|
LFPDUX A4, AO2, INC4
|
|
bdnz+ .L1042
|
|
.align 4
|
|
|
|
.L1043:
|
|
fxcpmadd f0, A1, B1, f0
|
|
LFPDUX B1, BO, INC4
|
|
fxcpmadd f1, A1, B2, f1
|
|
LFPDUX B2, BO2, INC4
|
|
fxcsmadd f2, A1, B3, f2
|
|
LFPDUX B3, BO, INC4
|
|
fxcsmadd f3, A1, B4, f3
|
|
LFPDUX B4, BO2, INC4
|
|
|
|
fxcpmadd f0, A2, A5, f0
|
|
LFPDUX A5, BO, INC4
|
|
fxcpmadd f1, A2, A6, f1
|
|
LFPDUX A6, BO2, INC4
|
|
fxcsmadd f2, A2, A7, f2
|
|
LFPDUX A7, BO, INC4
|
|
fxcsmadd f3, A2, A8, f3
|
|
LFPDUX A8, BO2, INC4
|
|
|
|
fxcpmadd f0, A3, B1, f0
|
|
fxcpmadd f1, A3, B2, f1
|
|
fxcsmadd f2, A3, B3, f2
|
|
fxcsmadd f3, A3, B4, f3
|
|
|
|
fxcpmadd f0, A4, A5, f0
|
|
fxcpmadd f1, A4, A6, f1
|
|
fxcsmadd f2, A4, A7, f2
|
|
fxcsmadd f3, A4, A8, f3
|
|
.align 4
|
|
|
|
.L1044:
|
|
lfd AP, ALPHA(SP)
|
|
#ifdef TRMMKERNEL
|
|
fsmfp AP, AP
|
|
#endif
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 1
|
|
#else
|
|
addi TEMP, KK, 4
|
|
#endif
|
|
andi. TEMP, TEMP, 7
|
|
mtspr CTR, TEMP
|
|
#else
|
|
andi. r0, K, 7
|
|
mtspr CTR, r0
|
|
#endif
|
|
ble+ .L1048
|
|
|
|
LFDX A1, AO, INC4
|
|
LFPDUX B1, BO, INC4
|
|
LFPDUX B2, BO2, INC4
|
|
add AO, AO, INC
|
|
bdz- .L1047
|
|
.align 4
|
|
|
|
.L1046:
|
|
fxcpmadd f0, A1, B1, f0
|
|
LFPDUX B1, BO, INC4
|
|
fxcpmadd f1, A1, B2, f1
|
|
LFDX A1, AO, INC4
|
|
LFPDUX B2, BO2, INC4
|
|
add AO, AO, INC
|
|
bdnz+ .L1046
|
|
.align 4
|
|
|
|
.L1047:
|
|
fxcpmadd f0, A1, B1, f0
|
|
fxcpmadd f1, A1, B2, f1
|
|
.align 4
|
|
|
|
.L1048:
|
|
#ifndef TRMMKERNEL
|
|
LFDX A1, CO1, INC
|
|
LFDX B3, CO3, INC
|
|
LFSDX A1, CO2, INC
|
|
LFSDX B3, CO4, INC
|
|
|
|
fpadd f0, f0, f2
|
|
fpadd f1, f1, f3
|
|
|
|
fxcpmadd f0, AP, f0, A1
|
|
fxcpmadd f1, AP, f1, B3
|
|
#else
|
|
fpadd f0, f0, f2
|
|
fpadd f1, f1, f3
|
|
|
|
fpmul f0, AP, f0
|
|
fpmul f1, AP, f1
|
|
#endif
|
|
|
|
STFDUX f0, CO1, INC
|
|
STFSDUX f0, CO2, INC
|
|
STFDUX f1, CO3, INC
|
|
STFSDUX f1, CO4, INC
|
|
|
|
#ifdef TRMMKERNEL
|
|
#if ( defined(LEFT) && defined(TRANSA)) || \
|
|
(!defined(LEFT) && !defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#ifdef LEFT
|
|
addi TEMP, TEMP, -1
|
|
#else
|
|
addi TEMP, TEMP, -4
|
|
#endif
|
|
slwi r0, TEMP, 0 + BASE_SHIFT
|
|
slwi TEMP, TEMP, 2 + BASE_SHIFT
|
|
add AO, AO, r0
|
|
add BO, BO, TEMP
|
|
#endif
|
|
|
|
#ifdef LEFT
|
|
addi KK, KK, 1
|
|
#endif
|
|
#endif
|
|
.align 4
|
|
|
|
.L1049:
|
|
#if defined(TRMMKERNEL) && !defined(LEFT)
|
|
addi KK, KK, 4
|
|
#endif
|
|
|
|
addi B, BO, 4 * SIZE
|
|
|
|
addic. J, J, -1
|
|
bgt+ .L1010
|
|
.align 4
|
|
|
|
.L1050:
|
|
andi. J, N, 2
|
|
beq .L1090
|
|
|
|
mr CO1, C
|
|
add CO2, C, LDC
|
|
add C, CO2, LDC
|
|
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
mr KK, OFFSET
|
|
#endif
|
|
|
|
addi AO, A, -2 * SIZE
|
|
|
|
li r0, FZERO
|
|
lfpsx f0, SP, r0
|
|
|
|
srawi. I, M, 3
|
|
ble .L1060
|
|
.align 4
|
|
|
|
.L1051:
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
|
fpmr f4, f0
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
fpmr f5, f0
|
|
fpmr f2, f0
|
|
fpmr f6, f0
|
|
#else
|
|
slwi TEMP, KK, 3 + BASE_SHIFT
|
|
slwi r0, KK, 1 + BASE_SHIFT
|
|
add AO, AO, TEMP
|
|
add BO, B, r0
|
|
|
|
fpmr f4, f0
|
|
addi BO, BO, - 2 * SIZE
|
|
fpmr f1, f0
|
|
fpmr f5, f0
|
|
fpmr f2, f0
|
|
fpmr f6, f0
|
|
#endif
|
|
|
|
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 8
|
|
#else
|
|
addi TEMP, KK, 2
|
|
#endif
|
|
srawi. r0, TEMP, 2
|
|
fpmr f3, f0
|
|
mtspr CTR, r0
|
|
fpmr f7, f0
|
|
ble .L1054
|
|
#else
|
|
fpmr f4, f0
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
fpmr f5, f0
|
|
fpmr f2, f0
|
|
fpmr f6, f0
|
|
|
|
srawi. r0, K, 2
|
|
fpmr f3, f0
|
|
mtspr CTR, r0
|
|
fpmr f7, f0
|
|
ble .L1054
|
|
#endif
|
|
|
|
LFPDUX B1, BO, INC2
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
LFPDUX A3, AO, INC2
|
|
LFPDUX A4, AO, INC2
|
|
|
|
LFPDUX B3, BO, INC2
|
|
LFPDUX A5, AO, INC2
|
|
LFPDUX A6, AO, INC2
|
|
LFPDUX A7, AO, INC2
|
|
LFPDUX A8, AO, INC2
|
|
bdz- .L1053
|
|
.align 4
|
|
|
|
.L1052:
|
|
fxcpmadd f0, B1, A1, f0
|
|
LFPDUX B4, BO, INC2
|
|
fxcsmadd f4, B1, A1, f4
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B1, A2, f1
|
|
nop
|
|
fxcsmadd f5, B1, A2, f5
|
|
LFPDUX A2, AO, INC2
|
|
|
|
fxcpmadd f2, B1, A3, f2
|
|
nop
|
|
fxcsmadd f6, B1, A3, f6
|
|
LFPDUX A3, AO, INC2
|
|
fxcpmadd f3, B1, A4, f3
|
|
nop
|
|
fxcsmadd f7, B1, A4, f7
|
|
LFPDUX A4, AO, INC2
|
|
|
|
fxcpmadd f0, B2, A5, f0
|
|
LFPDUX B1, BO, INC2
|
|
fxcsmadd f4, B2, A5, f4
|
|
LFPDUX A5, AO, INC2
|
|
fxcpmadd f1, B2, A6, f1
|
|
nop
|
|
fxcsmadd f5, B2, A6, f5
|
|
LFPDUX A6, AO, INC2
|
|
|
|
fxcpmadd f2, B2, A7, f2
|
|
nop
|
|
fxcsmadd f6, B2, A7, f6
|
|
LFPDUX A7, AO, INC2
|
|
fxcpmadd f3, B2, A8, f3
|
|
nop
|
|
fxcsmadd f7, B2, A8, f7
|
|
LFPDUX A8, AO, INC2
|
|
|
|
fxcpmadd f0, B3, A1, f0
|
|
LFPDUX B2, BO, INC2
|
|
fxcsmadd f4, B3, A1, f4
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B3, A2, f1
|
|
nop
|
|
fxcsmadd f5, B3, A2, f5
|
|
LFPDUX A2, AO, INC2
|
|
|
|
fxcpmadd f2, B3, A3, f2
|
|
nop
|
|
fxcsmadd f6, B3, A3, f6
|
|
LFPDUX A3, AO, INC2
|
|
fxcpmadd f3, B3, A4, f3
|
|
nop
|
|
fxcsmadd f7, B3, A4, f7
|
|
LFPDUX A4, AO, INC2
|
|
|
|
fxcpmadd f0, B4, A5, f0
|
|
LFPDUX B3, BO, INC2
|
|
fxcsmadd f4, B4, A5, f4
|
|
LFPDUX A5, AO, INC2
|
|
fxcpmadd f1, B4, A6, f1
|
|
nop
|
|
fxcsmadd f5, B4, A6, f5
|
|
LFPDUX A6, AO, INC2
|
|
|
|
fxcpmadd f2, B4, A7, f2
|
|
nop
|
|
fxcsmadd f6, B4, A7, f6
|
|
LFPDUX A7, AO, INC2
|
|
fxcpmadd f3, B4, A8, f3
|
|
nop
|
|
fxcsmadd f7, B4, A8, f7
|
|
LFPDUX A8, AO, INC2
|
|
bdnz+ .L1052
|
|
.align 4
|
|
|
|
.L1053:
|
|
fxcpmadd f0, B1, A1, f0
|
|
LFPDUX B4, BO, INC2
|
|
fxcsmadd f4, B1, A1, f4
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B1, A2, f1
|
|
nop
|
|
fxcsmadd f5, B1, A2, f5
|
|
LFPDUX A2, AO, INC2
|
|
|
|
fxcpmadd f2, B1, A3, f2
|
|
nop
|
|
fxcsmadd f6, B1, A3, f6
|
|
LFPDUX A3, AO, INC2
|
|
fxcpmadd f3, B1, A4, f3
|
|
nop
|
|
fxcsmadd f7, B1, A4, f7
|
|
LFPDUX A4, AO, INC2
|
|
|
|
fxcpmadd f0, B2, A5, f0
|
|
nop
|
|
fxcsmadd f4, B2, A5, f4
|
|
LFPDUX A5, AO, INC2
|
|
fxcpmadd f1, B2, A6, f1
|
|
nop
|
|
fxcsmadd f5, B2, A6, f5
|
|
LFPDUX A6, AO, INC2
|
|
|
|
fxcpmadd f2, B2, A7, f2
|
|
nop
|
|
fxcsmadd f6, B2, A7, f6
|
|
LFPDUX A7, AO, INC2
|
|
fxcpmadd f3, B2, A8, f3
|
|
nop
|
|
fxcsmadd f7, B2, A8, f7
|
|
LFPDUX A8, AO, INC2
|
|
|
|
fxcpmadd f0, B3, A1, f0
|
|
fxcsmadd f4, B3, A1, f4
|
|
fxcpmadd f1, B3, A2, f1
|
|
fxcsmadd f5, B3, A2, f5
|
|
|
|
fxcpmadd f2, B3, A3, f2
|
|
fxcsmadd f6, B3, A3, f6
|
|
fxcpmadd f3, B3, A4, f3
|
|
fxcsmadd f7, B3, A4, f7
|
|
|
|
fxcpmadd f0, B4, A5, f0
|
|
fxcsmadd f4, B4, A5, f4
|
|
fxcpmadd f1, B4, A6, f1
|
|
fxcsmadd f5, B4, A6, f5
|
|
|
|
fxcpmadd f2, B4, A7, f2
|
|
fxcsmadd f6, B4, A7, f6
|
|
fxcpmadd f3, B4, A8, f3
|
|
fxcsmadd f7, B4, A8, f7
|
|
.align 4
|
|
|
|
.L1054:
|
|
lfd AP, ALPHA(SP)
|
|
#ifdef TRMMKERNEL
|
|
fsmfp AP, AP
|
|
#endif
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 8
|
|
#else
|
|
addi TEMP, KK, 2
|
|
#endif
|
|
andi. TEMP, TEMP, 3
|
|
mtspr CTR, TEMP
|
|
#else
|
|
andi. r0, K, 3
|
|
mtspr CTR, r0
|
|
#endif
|
|
ble+ .L1058
|
|
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX A3, AO, INC2
|
|
LFPDUX A4, AO, INC2
|
|
bdz- .L1057
|
|
.align 4
|
|
|
|
.L1056:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f4, B1, A1, f4
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B1, A2, f1
|
|
fxcsmadd f5, B1, A2, f5
|
|
LFPDUX A2, AO, INC2
|
|
|
|
fxcpmadd f2, B1, A3, f2
|
|
fxcsmadd f6, B1, A3, f6
|
|
LFPDUX A3, AO, INC2
|
|
fxcpmadd f3, B1, A4, f3
|
|
fxcsmadd f7, B1, A4, f7
|
|
LFPDUX A4, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
bdnz+ .L1056
|
|
.align 4
|
|
|
|
.L1057:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f4, B1, A1, f4
|
|
fxcpmadd f1, B1, A2, f1
|
|
fxcsmadd f5, B1, A2, f5
|
|
|
|
fxcpmadd f2, B1, A3, f2
|
|
fxcsmadd f6, B1, A3, f6
|
|
fxcpmadd f3, B1, A4, f3
|
|
fxcsmadd f7, B1, A4, f7
|
|
.align 4
|
|
|
|
.L1058:
|
|
#ifndef TRMMKERNEL
|
|
LFDUX A1, CO1, INC
|
|
LFDUX B1, CO1, INC2
|
|
LFDUX A3, CO1, INC2
|
|
LFDUX A5, CO1, INC2
|
|
|
|
LFSDUX A1, CO1, INCM5
|
|
LFSDUX B1, CO1, INC2
|
|
LFSDUX A3, CO1, INC2
|
|
LFSDUX A5, CO1, INC2
|
|
|
|
LFDUX B3, CO2, INC
|
|
LFDUX A6, CO2, INC2
|
|
LFDUX A7, CO2, INC2
|
|
LFDUX B2, CO2, INC2
|
|
|
|
fxcpmadd f0, AP, f0, A1
|
|
LFSDUX B3, CO2, INCM5
|
|
LFSDUX A6, CO2, INC2
|
|
fxcpmadd f1, AP, f1, B1
|
|
LFSDUX A7, CO2, INC2
|
|
LFSDUX B2, CO2, INC2
|
|
|
|
fxcpmadd f2, AP, f2, A3
|
|
STFDUX f0, CO1, INCM7
|
|
STFSDUX f0, CO1, INC
|
|
|
|
fxcpmadd f3, AP, f3, A5
|
|
STFDUX f1, CO1, INC
|
|
STFSDUX f1, CO1, INC
|
|
|
|
fxcpmadd f4, AP, f4, B3
|
|
STFDUX f2, CO1, INC
|
|
STFSDUX f2, CO1, INC
|
|
|
|
fxcpmadd f5, AP, f5, A6
|
|
STFDUX f3, CO1, INC
|
|
STFSDUX f3, CO1, INC
|
|
|
|
fxcpmadd f6, AP, f6, A7
|
|
STFDUX f4, CO2, INCM7
|
|
STFSDUX f4, CO2, INC
|
|
|
|
fxcpmadd f7, AP, f7, B2
|
|
STFDUX f5, CO2, INC
|
|
STFSDUX f5, CO2, INC
|
|
|
|
STFDUX f6, CO2, INC
|
|
STFSDUX f6, CO2, INC
|
|
|
|
STFDUX f7, CO2, INC
|
|
STFSDUX f7, CO2, INC
|
|
#else
|
|
fpmul f0, AP, f0
|
|
fpmul f1, AP, f1
|
|
|
|
fpmul f2, AP, f2
|
|
STFDUX f0, CO1, INC
|
|
STFSDUX f0, CO1, INC
|
|
|
|
fpmul f3, AP, f3
|
|
STFDUX f1, CO1, INC
|
|
STFSDUX f1, CO1, INC
|
|
|
|
fpmul f4, AP, f4
|
|
STFDUX f2, CO1, INC
|
|
STFSDUX f2, CO1, INC
|
|
|
|
fpmul f5, AP, f5
|
|
STFDUX f3, CO1, INC
|
|
STFSDUX f3, CO1, INC
|
|
|
|
fpmul f6, AP, f6
|
|
STFDUX f4, CO2, INC
|
|
STFSDUX f4, CO2, INC
|
|
|
|
fpmul f7, AP, f7
|
|
STFDUX f5, CO2, INC
|
|
STFSDUX f5, CO2, INC
|
|
|
|
STFDUX f6, CO2, INC
|
|
STFSDUX f6, CO2, INC
|
|
|
|
STFDUX f7, CO2, INC
|
|
STFSDUX f7, CO2, INC
|
|
#endif
|
|
|
|
|
|
#ifdef TRMMKERNEL
|
|
#if ( defined(LEFT) && defined(TRANSA)) || \
|
|
(!defined(LEFT) && !defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#ifdef LEFT
|
|
addi TEMP, TEMP, -8
|
|
#else
|
|
addi TEMP, TEMP, -2
|
|
#endif
|
|
slwi r0, TEMP, 3 + BASE_SHIFT
|
|
slwi TEMP, TEMP, 1 + BASE_SHIFT
|
|
add AO, AO, r0
|
|
add BO, BO, TEMP
|
|
#endif
|
|
|
|
#ifdef LEFT
|
|
addi KK, KK, 8
|
|
#endif
|
|
#endif
|
|
|
|
addic. I, I, -1
|
|
li r0, FZERO
|
|
|
|
lfpsx f0, SP, r0
|
|
bgt+ .L1051
|
|
.align 4
|
|
|
|
.L1060:
|
|
andi. I, M, 4
|
|
beq .L1070
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
#else
|
|
slwi TEMP, KK, 2 + BASE_SHIFT
|
|
slwi r0, KK, 1 + BASE_SHIFT
|
|
add AO, AO, TEMP
|
|
add BO, B, r0
|
|
|
|
addi BO, BO, - 2 * SIZE
|
|
fpmr f1, f0
|
|
#endif
|
|
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 4
|
|
#else
|
|
addi TEMP, KK, 2
|
|
#endif
|
|
fpmr f2, f0
|
|
srawi. r0, TEMP, 2
|
|
mtspr CTR, r0
|
|
fpmr f3, f0
|
|
ble .L1064
|
|
#else
|
|
srawi. r0, K, 2
|
|
fpmr f1, f0
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f2, f0
|
|
mtspr CTR, r0
|
|
fpmr f3, f0
|
|
ble .L1064
|
|
#endif
|
|
|
|
LFPDUX B1, BO, INC2
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
LFPDUX A3, AO, INC2
|
|
LFPDUX A4, AO, INC2
|
|
|
|
LFPDUX B3, BO, INC2
|
|
LFPDUX A5, AO, INC2
|
|
LFPDUX A6, AO, INC2
|
|
LFPDUX B4, BO, INC2
|
|
LFPDUX A7, AO, INC2
|
|
LFPDUX A8, AO, INC2
|
|
bdz- .L1063
|
|
.align 4
|
|
|
|
.L1062:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f2, B1, A1, f2
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B1, A2, f1
|
|
fxcsmadd f3, B1, A2, f3
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
|
|
fxcpmadd f0, B2, A3, f0
|
|
fxcsmadd f2, B2, A3, f2
|
|
LFPDUX A3, AO, INC2
|
|
fxcpmadd f1, B2, A4, f1
|
|
fxcsmadd f3, B2, A4, f3
|
|
LFPDUX A4, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
|
|
fxcpmadd f0, B3, A5, f0
|
|
fxcsmadd f2, B3, A5, f2
|
|
LFPDUX A5, AO, INC2
|
|
fxcpmadd f1, B3, A6, f1
|
|
fxcsmadd f3, B3, A6, f3
|
|
LFPDUX A6, AO, INC2
|
|
LFPDUX B3, BO, INC2
|
|
|
|
fxcpmadd f0, B4, A7, f0
|
|
fxcsmadd f2, B4, A7, f2
|
|
LFPDUX A7, AO, INC2
|
|
fxcpmadd f1, B4, A8, f1
|
|
fxcsmadd f3, B4, A8, f3
|
|
LFPDUX A8, AO, INC2
|
|
LFPDUX B4, BO, INC2
|
|
bdnz+ .L1062
|
|
.align 4
|
|
|
|
.L1063:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f2, B1, A1, f2
|
|
fxcpmadd f1, B1, A2, f1
|
|
fxcsmadd f3, B1, A2, f3
|
|
|
|
fxcpmadd f0, B2, A3, f0
|
|
fxcsmadd f2, B2, A3, f2
|
|
fxcpmadd f1, B2, A4, f1
|
|
fxcsmadd f3, B2, A4, f3
|
|
|
|
fxcpmadd f0, B3, A5, f0
|
|
fxcsmadd f2, B3, A5, f2
|
|
fxcpmadd f1, B3, A6, f1
|
|
fxcsmadd f3, B3, A6, f3
|
|
|
|
fxcpmadd f0, B4, A7, f0
|
|
fxcsmadd f2, B4, A7, f2
|
|
fxcpmadd f1, B4, A8, f1
|
|
fxcsmadd f3, B4, A8, f3
|
|
.align 4
|
|
|
|
.L1064:
|
|
lfd AP, ALPHA(SP)
|
|
#ifdef TRMMKERNEL
|
|
fsmfp AP, AP
|
|
#endif
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 4
|
|
#else
|
|
addi TEMP, KK, 2
|
|
#endif
|
|
andi. TEMP, TEMP, 3
|
|
mtspr CTR, TEMP
|
|
#else
|
|
andi. r0, K, 3
|
|
mtspr CTR, r0
|
|
#endif
|
|
ble+ .L1068
|
|
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
bdz- .L1067
|
|
.align 4
|
|
|
|
.L1066:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f2, B1, A1, f2
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B1, A2, f1
|
|
fxcsmadd f3, B1, A2, f3
|
|
LFPDUX B1, BO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
bdnz+ .L1066
|
|
.align 4
|
|
|
|
.L1067:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f2, B1, A1, f2
|
|
fxcpmadd f1, B1, A2, f1
|
|
fxcsmadd f3, B1, A2, f3
|
|
.align 4
|
|
|
|
.L1068:
|
|
#ifndef TRMMKERNEL
|
|
LFDUX A1, CO1, INC
|
|
LFDUX A2, CO1, INC2
|
|
LFDUX A3, CO2, INC
|
|
LFDUX A4, CO2, INC2
|
|
|
|
LFSDUX A1, CO1, INCM1
|
|
LFSDUX A2, CO1, INC2
|
|
LFSDUX A3, CO2, INCM1
|
|
LFSDUX A4, CO2, INC2
|
|
|
|
fxcpmadd f0, AP, f0, A1
|
|
fxcpmadd f1, AP, f1, A2
|
|
fxcpmadd f2, AP, f2, A3
|
|
STFDUX f0, CO1, INCM3
|
|
STFSDUX f0, CO1, INC
|
|
|
|
fxcpmadd f3, AP, f3, A4
|
|
STFDUX f1, CO1, INC
|
|
STFSDUX f1, CO1, INC
|
|
|
|
STFDUX f2, CO2, INCM3
|
|
STFSDUX f2, CO2, INC
|
|
|
|
STFDUX f3, CO2, INC
|
|
STFSDUX f3, CO2, INC
|
|
#else
|
|
fpmul f0, AP, f0
|
|
fpmul f1, AP, f1
|
|
fpmul f2, AP, f2
|
|
STFDUX f0, CO1, INC
|
|
STFSDUX f0, CO1, INC
|
|
|
|
fpmul f3, AP, f3
|
|
STFDUX f1, CO1, INC
|
|
STFSDUX f1, CO1, INC
|
|
|
|
STFDUX f2, CO2, INC
|
|
STFSDUX f2, CO2, INC
|
|
|
|
STFDUX f3, CO2, INC
|
|
STFSDUX f3, CO2, INC
|
|
#endif
|
|
|
|
|
|
#ifdef TRMMKERNEL
|
|
#if ( defined(LEFT) && defined(TRANSA)) || \
|
|
(!defined(LEFT) && !defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#ifdef LEFT
|
|
addi TEMP, TEMP, -4
|
|
#else
|
|
addi TEMP, TEMP, -2
|
|
#endif
|
|
slwi r0, TEMP, 2 + BASE_SHIFT
|
|
slwi TEMP, TEMP, 1 + BASE_SHIFT
|
|
add AO, AO, r0
|
|
add BO, BO, TEMP
|
|
#endif
|
|
|
|
#ifdef LEFT
|
|
addi KK, KK, 4
|
|
#endif
|
|
#endif
|
|
|
|
li r0, FZERO
|
|
lfpsx f0, SP, r0
|
|
.align 4
|
|
|
|
.L1070:
|
|
andi. I, M, 2
|
|
beq .L1080
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
#else
|
|
slwi TEMP, KK, 1 + BASE_SHIFT
|
|
slwi r0, KK, 1 + BASE_SHIFT
|
|
add AO, AO, TEMP
|
|
add BO, B, r0
|
|
|
|
addi BO, BO, - 2 * SIZE
|
|
fpmr f1, f0
|
|
#endif
|
|
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 2
|
|
#else
|
|
addi TEMP, KK, 2
|
|
#endif
|
|
srawi. r0, TEMP, 3
|
|
fpmr f2, f0
|
|
mtspr CTR, r0
|
|
fpmr f3, f0
|
|
ble .L1074
|
|
#else
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
|
|
srawi. r0, K, 3
|
|
fpmr f2, f0
|
|
mtspr CTR, r0
|
|
fpmr f3, f0
|
|
ble .L1074
|
|
#endif
|
|
|
|
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
LFPDUX A3, AO, INC2
|
|
LFPDUX B3, BO, INC2
|
|
LFPDUX A4, AO, INC2
|
|
LFPDUX B4, BO, INC2
|
|
|
|
LFPDUX A5, AO, INC2
|
|
LFPDUX B5, BO, INC2
|
|
LFPDUX A6, AO, INC2
|
|
LFPDUX B6, BO, INC2
|
|
LFPDUX A7, AO, INC2
|
|
LFPDUX A9, BO, INC2
|
|
LFPDUX A8, AO, INC2
|
|
LFPDUX A10, BO, INC2
|
|
bdz- .L1073
|
|
.align 4
|
|
|
|
.L1072:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f1, B1, A1, f1
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
fxcpmadd f2, B2, A2, f2
|
|
fxcsmadd f3, B2, A2, f3
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
|
|
fxcpmadd f0, B3, A3, f0
|
|
fxcsmadd f1, B3, A3, f1
|
|
LFPDUX A3, AO, INC2
|
|
LFPDUX B3, BO, INC2
|
|
fxcpmadd f2, B4, A4, f2
|
|
fxcsmadd f3, B4, A4, f3
|
|
LFPDUX A4, AO, INC2
|
|
LFPDUX B4, BO, INC2
|
|
|
|
fxcpmadd f0, B5, A5, f0
|
|
fxcsmadd f1, B5, A5, f1
|
|
LFPDUX A5, AO, INC2
|
|
LFPDUX B5, BO, INC2
|
|
fxcpmadd f2, B6, A6, f2
|
|
fxcsmadd f3, B6, A6, f3
|
|
LFPDUX A6, AO, INC2
|
|
LFPDUX B6, BO, INC2
|
|
|
|
fxcpmadd f0, A9, A7, f0
|
|
fxcsmadd f1, A9, A7, f1
|
|
LFPDUX A7, AO, INC2
|
|
LFPDUX A9, BO, INC2
|
|
fxcpmadd f2, A10, A8, f2
|
|
fxcsmadd f3, A10, A8, f3
|
|
LFPDUX A8, AO, INC2
|
|
LFPDUX A10, BO, INC2
|
|
bdnz+ .L1072
|
|
.align 4
|
|
|
|
.L1073:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f1, B1, A1, f1
|
|
fxcpmadd f2, B2, A2, f2
|
|
fxcsmadd f3, B2, A2, f3
|
|
|
|
fxcpmadd f0, B3, A3, f0
|
|
fxcsmadd f1, B3, A3, f1
|
|
fxcpmadd f2, B4, A4, f2
|
|
fxcsmadd f3, B4, A4, f3
|
|
|
|
fxcpmadd f0, B5, A5, f0
|
|
fxcsmadd f1, B5, A5, f1
|
|
fxcpmadd f2, B6, A6, f2
|
|
fxcsmadd f3, B6, A6, f3
|
|
|
|
fxcpmadd f0, A9, A7, f0
|
|
fxcsmadd f1, A9, A7, f1
|
|
fxcpmadd f2, A10, A8, f2
|
|
fxcsmadd f3, A10, A8, f3
|
|
.align 4
|
|
|
|
.L1074:
|
|
lfd AP, ALPHA(SP)
|
|
#ifdef TRMMKERNEL
|
|
fsmfp AP, AP
|
|
#endif
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 2
|
|
#else
|
|
addi TEMP, KK, 2
|
|
#endif
|
|
andi. TEMP, TEMP, 7
|
|
mtspr CTR, TEMP
|
|
#else
|
|
andi. r0, K, 7
|
|
mtspr CTR, r0
|
|
#endif
|
|
ble+ .L1078
|
|
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
bdz- .L1077
|
|
.align 4
|
|
|
|
.L1076:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f1, B1, A1, f1
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
bdnz+ .L1076
|
|
.align 4
|
|
|
|
.L1077:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f1, B1, A1, f1
|
|
.align 4
|
|
|
|
.L1078:
|
|
#ifndef TRMMKERNEL
|
|
LFDUX A1, CO1, INC
|
|
LFDUX B3, CO2, INC
|
|
LFSDUX A1, CO1, INC
|
|
LFSDUX B3, CO2, INC
|
|
|
|
fpadd f0, f0, f2
|
|
fpadd f1, f1, f3
|
|
|
|
fxcpmadd f0, AP, f0, A1
|
|
fxcpmadd f1, AP, f1, B3
|
|
|
|
STFDUX f0, CO1, INCM1
|
|
STFSDUX f0, CO1, INC
|
|
STFDUX f1, CO2, INCM1
|
|
STFSDUX f1, CO2, INC
|
|
#else
|
|
fpadd f0, f0, f2
|
|
fpadd f1, f1, f3
|
|
|
|
fpmul f0, AP, f0
|
|
fpmul f1, AP, f1
|
|
|
|
STFDUX f0, CO1, INC
|
|
STFSDUX f0, CO1, INC
|
|
STFDUX f1, CO2, INC
|
|
STFSDUX f1, CO2, INC
|
|
#endif
|
|
|
|
|
|
#ifdef TRMMKERNEL
|
|
#if ( defined(LEFT) && defined(TRANSA)) || \
|
|
(!defined(LEFT) && !defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#ifdef LEFT
|
|
addi TEMP, TEMP, -2
|
|
#else
|
|
addi TEMP, TEMP, -2
|
|
#endif
|
|
slwi r0, TEMP, 1 + BASE_SHIFT
|
|
slwi TEMP, TEMP, 1 + BASE_SHIFT
|
|
add AO, AO, r0
|
|
add BO, BO, TEMP
|
|
#endif
|
|
|
|
#ifdef LEFT
|
|
addi KK, KK, 2
|
|
#endif
|
|
#endif
|
|
|
|
li r0, FZERO
|
|
lfpsx f0, SP, r0
|
|
.align 4
|
|
|
|
.L1080:
|
|
andi. I, M, 1
|
|
beq .L1089
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
#else
|
|
slwi TEMP, KK, 0 + BASE_SHIFT
|
|
slwi r0, KK, 1 + BASE_SHIFT
|
|
add AO, AO, TEMP
|
|
add BO, B, r0
|
|
|
|
addi BO, BO, - 2 * SIZE
|
|
fpmr f1, f0
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
#endif
|
|
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 1
|
|
#else
|
|
addi TEMP, KK, 2
|
|
#endif
|
|
srawi. r0, TEMP, 3
|
|
mtspr CTR, r0
|
|
ble .L1084
|
|
#else
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
|
|
srawi. r0, K, 3
|
|
mtspr CTR, r0
|
|
ble .L1084
|
|
#endif
|
|
|
|
LFPDUX B1, BO, INC2
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
|
|
LFPDUX B2, BO, INC2
|
|
LFPDUX A3, AO, INC2
|
|
LFPDUX A4, AO, INC2
|
|
|
|
LFPDUX B3, BO, INC2
|
|
LFPDUX B4, BO, INC2
|
|
bdz- .L1083
|
|
.align 4
|
|
|
|
.L1082:
|
|
fxcpmadd f0, A1, B1, f0
|
|
LFPDUX B1, BO, INC2
|
|
fxcsmadd f1, A1, B2, f1
|
|
LFPDUX B2, BO, INC2
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f2, A2, B3, f2
|
|
LFPDUX B3, BO, INC2
|
|
fxcsmadd f3, A2, B4, f3
|
|
LFPDUX B4, BO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
|
|
fxcpmadd f0, A3, B1, f0
|
|
LFPDUX B1, BO, INC2
|
|
fxcsmadd f1, A3, B2, f1
|
|
LFPDUX B2, BO, INC2
|
|
LFPDUX A3, AO, INC2
|
|
fxcpmadd f2, A4, B3, f2
|
|
LFPDUX B3, BO, INC2
|
|
fxcsmadd f3, A4, B4, f3
|
|
LFPDUX B4, BO, INC2
|
|
LFPDUX A4, AO, INC2
|
|
bdnz+ .L1082
|
|
.align 4
|
|
|
|
.L1083:
|
|
fxcpmadd f0, A1, B1, f0
|
|
LFPDUX B1, BO, INC2
|
|
fxcsmadd f1, A1, B2, f1
|
|
LFPDUX B2, BO, INC2
|
|
fxcpmadd f2, A2, B3, f2
|
|
LFPDUX B3, BO, INC2
|
|
fxcsmadd f3, A2, B4, f3
|
|
LFPDUX B4, BO, INC2
|
|
|
|
fxcpmadd f0, A3, B1, f0
|
|
fxcsmadd f1, A3, B2, f1
|
|
fxcpmadd f2, A4, B3, f2
|
|
fxcsmadd f3, A4, B4, f3
|
|
.align 4
|
|
|
|
.L1084:
|
|
lfd AP, ALPHA(SP)
|
|
#ifdef TRMMKERNEL
|
|
fsmfp AP, AP
|
|
#endif
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 1
|
|
#else
|
|
addi TEMP, KK, 2
|
|
#endif
|
|
andi. TEMP, TEMP, 7
|
|
mtspr CTR, TEMP
|
|
#else
|
|
andi. r0, K, 7
|
|
mtspr CTR, r0
|
|
#endif
|
|
ble+ .L1088
|
|
|
|
LFDX A1, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
add AO, AO, INC
|
|
bdz- .L1087
|
|
.align 4
|
|
|
|
.L1086:
|
|
fxcpmadd f0, A1, B1, f0
|
|
LFDX A1, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
add AO, AO, INC
|
|
bdnz+ .L1086
|
|
.align 4
|
|
|
|
.L1087:
|
|
fxcpmadd f0, A1, B1, f0
|
|
.align 4
|
|
|
|
.L1088:
|
|
#ifndef TRMMKERNEL
|
|
LFDX A1, CO1, INC
|
|
LFDX A2, CO2, INC
|
|
|
|
fpadd f0, f0, f1
|
|
fpadd f2, f2, f3
|
|
fsmfp A1, A2
|
|
fpadd f0, f0, f2
|
|
fxcpmadd f0, AP, f0, A1
|
|
#else
|
|
fpadd f0, f0, f1
|
|
fpadd f2, f2, f3
|
|
fsmfp A1, A2
|
|
fpadd f0, f0, f2
|
|
fpmul f0, AP, f0
|
|
#endif
|
|
|
|
STFDUX f0, CO1, INC
|
|
STFSDUX f0, CO2, INC
|
|
|
|
#ifdef TRMMKERNEL
|
|
#if ( defined(LEFT) && defined(TRANSA)) || \
|
|
(!defined(LEFT) && !defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#ifdef LEFT
|
|
addi TEMP, TEMP, -1
|
|
#else
|
|
addi TEMP, TEMP, -2
|
|
#endif
|
|
slwi r0, TEMP, 0 + BASE_SHIFT
|
|
slwi TEMP, TEMP, 1 + BASE_SHIFT
|
|
add AO, AO, r0
|
|
add BO, BO, TEMP
|
|
#endif
|
|
|
|
#ifdef LEFT
|
|
addi KK, KK, 1
|
|
#endif
|
|
#endif
|
|
.align 4
|
|
|
|
.L1089:
|
|
#if defined(TRMMKERNEL) && !defined(LEFT)
|
|
addi KK, KK, 2
|
|
#endif
|
|
|
|
addi B, BO, 2 * SIZE
|
|
.align 4
|
|
|
|
.L1090:
|
|
andi. J, N, 1
|
|
beq .L10999
|
|
|
|
#if defined(TRMMKERNEL) && defined(LEFT)
|
|
mr KK, OFFSET
|
|
#endif
|
|
|
|
mr CO1, C
|
|
addi AO, A, -2 * SIZE
|
|
|
|
li r0, FZERO
|
|
lfpsx f0, SP, r0
|
|
|
|
srawi. I, M, 3
|
|
ble .L10100
|
|
.align 4
|
|
|
|
.L1091:
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
#else
|
|
slwi TEMP, KK, 3 + BASE_SHIFT
|
|
slwi r0, KK, 0 + BASE_SHIFT
|
|
add AO, AO, TEMP
|
|
add BO, B, r0
|
|
|
|
addi BO, BO, - 2 * SIZE
|
|
fpmr f1, f0
|
|
#endif
|
|
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 8
|
|
#else
|
|
addi TEMP, KK, 1
|
|
#endif
|
|
fpmr f2, f0
|
|
srawi. r0, TEMP, 2
|
|
fpmr f3, f0
|
|
mtspr CTR, r0
|
|
ble .L1094
|
|
|
|
#else
|
|
srawi. r0, K, 2
|
|
fpmr f1, f0
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
mtspr CTR, r0
|
|
ble .L1094
|
|
#endif
|
|
|
|
LFPDUX B1, BO, INC2
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX A3, AO, INC2
|
|
LFPDUX A4, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
LFPDUX A5, AO, INC2
|
|
LFPDUX A6, AO, INC2
|
|
LFPDUX A7, AO, INC2
|
|
LFPDUX A8, AO, INC2
|
|
bdz- .L1093
|
|
.align 4
|
|
|
|
.L1092:
|
|
fxcpmadd f0, B1, A1, f0
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B1, A2, f1
|
|
LFPDUX A2, AO, INC2
|
|
fxcpmadd f2, B1, A3, f2
|
|
LFPDUX A3, AO, INC2
|
|
fxcpmadd f3, B1, A4, f3
|
|
LFPDUX A4, AO, INC2
|
|
|
|
fxcsmadd f0, B1, A5, f0
|
|
LFPDUX A5, AO, INC2
|
|
fxcsmadd f1, B1, A6, f1
|
|
LFPDUX A6, AO, INC2
|
|
fxcsmadd f2, B1, A7, f2
|
|
LFPDUX A7, AO, INC2
|
|
fxcsmadd f3, B1, A8, f3
|
|
LFPDUX A8, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
|
|
fxcpmadd f0, B2, A1, f0
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B2, A2, f1
|
|
LFPDUX A2, AO, INC2
|
|
fxcpmadd f2, B2, A3, f2
|
|
LFPDUX A3, AO, INC2
|
|
fxcpmadd f3, B2, A4, f3
|
|
LFPDUX A4, AO, INC2
|
|
|
|
fxcsmadd f0, B2, A5, f0
|
|
LFPDUX A5, AO, INC2
|
|
fxcsmadd f1, B2, A6, f1
|
|
LFPDUX A6, AO, INC2
|
|
fxcsmadd f2, B2, A7, f2
|
|
LFPDUX A7, AO, INC2
|
|
fxcsmadd f3, B2, A8, f3
|
|
LFPDUX A8, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
bdnz+ .L1092
|
|
.align 4
|
|
|
|
.L1093:
|
|
fxcpmadd f0, B1, A1, f0
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B1, A2, f1
|
|
LFPDUX A2, AO, INC2
|
|
fxcpmadd f2, B1, A3, f2
|
|
LFPDUX A3, AO, INC2
|
|
fxcpmadd f3, B1, A4, f3
|
|
LFPDUX A4, AO, INC2
|
|
|
|
fxcsmadd f0, B1, A5, f0
|
|
LFPDUX A5, AO, INC2
|
|
fxcsmadd f1, B1, A6, f1
|
|
LFPDUX A6, AO, INC2
|
|
fxcsmadd f2, B1, A7, f2
|
|
LFPDUX A7, AO, INC2
|
|
fxcsmadd f3, B1, A8, f3
|
|
LFPDUX A8, AO, INC2
|
|
|
|
fxcpmadd f0, B2, A1, f0
|
|
fxcpmadd f1, B2, A2, f1
|
|
fxcpmadd f2, B2, A3, f2
|
|
fxcpmadd f3, B2, A4, f3
|
|
|
|
fxcsmadd f0, B2, A5, f0
|
|
fxcsmadd f1, B2, A6, f1
|
|
fxcsmadd f2, B2, A7, f2
|
|
fxcsmadd f3, B2, A8, f3
|
|
.align 4
|
|
|
|
.L1094:
|
|
lfd AP, ALPHA(SP)
|
|
#ifdef TRMMKERNEL
|
|
fsmfp AP, AP
|
|
#endif
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 8
|
|
#else
|
|
addi TEMP, KK, 1
|
|
#endif
|
|
andi. TEMP, TEMP, 3
|
|
mtspr CTR, TEMP
|
|
#else
|
|
andi. r0, K, 3
|
|
mtspr CTR, r0
|
|
#endif
|
|
ble+ .L1098
|
|
|
|
LFDX B1, BO, INC2
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX A3, AO, INC2
|
|
LFPDUX A4, AO, INC2
|
|
add BO, BO, INC
|
|
bdz- .L1097
|
|
.align 4
|
|
|
|
.L1096:
|
|
fxcpmadd f0, B1, A1, f0
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B1, A2, f1
|
|
LFPDUX A2, AO, INC2
|
|
fxcpmadd f2, B1, A3, f2
|
|
LFPDUX A3, AO, INC2
|
|
fxcpmadd f3, B1, A4, f3
|
|
LFDX B1, BO, INC2
|
|
LFPDUX A4, AO, INC2
|
|
add BO, BO, INC
|
|
bdnz+ .L1096
|
|
.align 4
|
|
|
|
.L1097:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcpmadd f1, B1, A2, f1
|
|
fxcpmadd f2, B1, A3, f2
|
|
fxcpmadd f3, B1, A4, f3
|
|
.align 4
|
|
|
|
.L1098:
|
|
#ifndef TRMMKERNEL
|
|
LFDUX A1, CO1, INC
|
|
LFDUX B1, CO1, INC2
|
|
LFDUX A3, CO1, INC2
|
|
LFDUX A5, CO1, INC2
|
|
|
|
LFSDUX A1, CO1, INCM5
|
|
LFSDUX B1, CO1, INC2
|
|
LFSDUX A3, CO1, INC2
|
|
LFSDUX A5, CO1, INC2
|
|
|
|
fxcpmadd f0, AP, f0, A1
|
|
fxcpmadd f1, AP, f1, B1
|
|
fxcpmadd f2, AP, f2, A3
|
|
STFDUX f0, CO1, INCM7
|
|
STFSDUX f0, CO1, INC
|
|
|
|
fxcpmadd f3, AP, f3, A5
|
|
#else
|
|
fpmul f0, AP, f0
|
|
fpmul f1, AP, f1
|
|
fpmul f2, AP, f2
|
|
STFDUX f0, CO1, INC
|
|
STFSDUX f0, CO1, INC
|
|
|
|
fpmul f3, AP, f3
|
|
#endif
|
|
|
|
STFDUX f1, CO1, INC
|
|
STFSDUX f1, CO1, INC
|
|
|
|
STFDUX f2, CO1, INC
|
|
STFSDUX f2, CO1, INC
|
|
|
|
STFDUX f3, CO1, INC
|
|
STFSDUX f3, CO1, INC
|
|
|
|
#ifdef TRMMKERNEL
|
|
#if ( defined(LEFT) && defined(TRANSA)) || \
|
|
(!defined(LEFT) && !defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#ifdef LEFT
|
|
addi TEMP, TEMP, -8
|
|
#else
|
|
addi TEMP, TEMP, -1
|
|
#endif
|
|
slwi r0, TEMP, 3 + BASE_SHIFT
|
|
slwi TEMP, TEMP, 0 + BASE_SHIFT
|
|
add AO, AO, r0
|
|
add BO, BO, TEMP
|
|
#endif
|
|
|
|
#ifdef LEFT
|
|
addi KK, KK, 8
|
|
#endif
|
|
#endif
|
|
|
|
addic. I, I, -1
|
|
li r0, FZERO
|
|
|
|
lfpsx f0, SP, r0
|
|
bgt+ .L1091
|
|
.align 4
|
|
|
|
.L10100:
|
|
andi. I, M, 4
|
|
beq .L10110
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
#else
|
|
slwi TEMP, KK, 2 + BASE_SHIFT
|
|
slwi r0, KK, 0 + BASE_SHIFT
|
|
add AO, AO, TEMP
|
|
add BO, B, r0
|
|
|
|
fpmr f1, f0
|
|
addi BO, BO, - 2 * SIZE
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
#endif
|
|
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 4
|
|
#else
|
|
addi TEMP, KK, 1
|
|
#endif
|
|
srawi. r0, TEMP, 3
|
|
mtspr CTR, r0
|
|
ble .L10104
|
|
#else
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
|
|
srawi. r0, K, 3
|
|
mtspr CTR, r0
|
|
ble .L10104
|
|
#endif
|
|
|
|
LFPDUX B1, BO, INC2
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX A3, AO, INC2
|
|
LFPDUX A4, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
LFPDUX A5, AO, INC2
|
|
LFPDUX A6, AO, INC2
|
|
LFPDUX A7, AO, INC2
|
|
LFPDUX A8, AO, INC2
|
|
LFPDUX B3, BO, INC2
|
|
LFPDUX B4, BO, INC2
|
|
|
|
bdz- .L10103
|
|
.align 4
|
|
|
|
.L10102:
|
|
fxcpmadd f0, B1, A1, f0
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B1, A2, f1
|
|
LFPDUX A2, AO, INC2
|
|
fxcsmadd f2, B1, A3, f2
|
|
LFPDUX A3, AO, INC2
|
|
fxcsmadd f3, B1, A4, f3
|
|
LFPDUX A4, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
|
|
fxcpmadd f0, B2, A5, f0
|
|
LFPDUX A5, AO, INC2
|
|
fxcpmadd f1, B2, A6, f1
|
|
LFPDUX A6, AO, INC2
|
|
fxcsmadd f2, B2, A7, f2
|
|
LFPDUX A7, AO, INC2
|
|
fxcsmadd f3, B2, A8, f3
|
|
LFPDUX A8, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
|
|
fxcpmadd f0, B3, A1, f0
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B3, A2, f1
|
|
LFPDUX A2, AO, INC2
|
|
fxcsmadd f2, B3, A3, f2
|
|
LFPDUX A3, AO, INC2
|
|
fxcsmadd f3, B3, A4, f3
|
|
LFPDUX A4, AO, INC2
|
|
LFPDUX B3, BO, INC2
|
|
|
|
fxcpmadd f0, B4, A5, f0
|
|
LFPDUX A5, AO, INC2
|
|
fxcpmadd f1, B4, A6, f1
|
|
LFPDUX A6, AO, INC2
|
|
fxcsmadd f2, B4, A7, f2
|
|
LFPDUX A7, AO, INC2
|
|
fxcsmadd f3, B4, A8, f3
|
|
LFPDUX A8, AO, INC2
|
|
LFPDUX B4, BO, INC2
|
|
bdnz+ .L10102
|
|
.align 4
|
|
|
|
.L10103:
|
|
fxcpmadd f0, B1, A1, f0
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B1, A2, f1
|
|
LFPDUX A2, AO, INC2
|
|
fxcsmadd f2, B1, A3, f2
|
|
LFPDUX A3, AO, INC2
|
|
fxcsmadd f3, B1, A4, f3
|
|
LFPDUX A4, AO, INC2
|
|
|
|
fxcpmadd f0, B2, A5, f0
|
|
LFPDUX A5, AO, INC2
|
|
fxcpmadd f1, B2, A6, f1
|
|
LFPDUX A6, AO, INC2
|
|
fxcsmadd f2, B2, A7, f2
|
|
LFPDUX A7, AO, INC2
|
|
fxcsmadd f3, B2, A8, f3
|
|
LFPDUX A8, AO, INC2
|
|
|
|
fxcpmadd f0, B3, A1, f0
|
|
fxcpmadd f1, B3, A2, f1
|
|
fxcsmadd f2, B3, A3, f2
|
|
fxcsmadd f3, B3, A4, f3
|
|
|
|
fxcpmadd f0, B4, A5, f0
|
|
fxcpmadd f1, B4, A6, f1
|
|
fxcsmadd f2, B4, A7, f2
|
|
fxcsmadd f3, B4, A8, f3
|
|
.align 4
|
|
|
|
.L10104:
|
|
lfd AP, ALPHA(SP)
|
|
#ifdef TRMMKERNEL
|
|
fsmfp AP, AP
|
|
#endif
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 4
|
|
#else
|
|
addi TEMP, KK, 1
|
|
#endif
|
|
andi. TEMP, TEMP, 7
|
|
mtspr CTR, TEMP
|
|
#else
|
|
andi. r0, K, 7
|
|
mtspr CTR, r0
|
|
#endif
|
|
ble+ .L10108
|
|
|
|
LFPDUX A1, AO, INC2
|
|
LFDX B1, BO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
add BO, BO, INC
|
|
bdz- .L10107
|
|
.align 4
|
|
|
|
.L10106:
|
|
fxcpmadd f0, B1, A1, f0
|
|
LFPDUX A1, AO, INC2
|
|
fxcpmadd f1, B1, A2, f1
|
|
LFDX B1, BO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
add BO, BO, INC
|
|
bdnz+ .L10106
|
|
.align 4
|
|
|
|
.L10107:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcpmadd f1, B1, A2, f1
|
|
.align 4
|
|
|
|
.L10108:
|
|
#ifndef TRMMKERNEL
|
|
LFDUX A1, CO1, INC
|
|
LFDUX B1, CO1, INC2
|
|
LFSDUX A1, CO1, INCM1
|
|
LFSDUX B1, CO1, INC2
|
|
|
|
fpadd f0, f0, f2
|
|
fpadd f1, f1, f3
|
|
|
|
fxcpmadd f0, AP, f0, A1
|
|
fxcpmadd f1, AP, f1, B1
|
|
|
|
STFDUX f0, CO1, INCM3
|
|
STFSDUX f0, CO1, INC
|
|
#else
|
|
fpadd f0, f0, f2
|
|
fpadd f1, f1, f3
|
|
|
|
fpmul f0, AP, f0
|
|
fpmul f1, AP, f1
|
|
|
|
STFDUX f0, CO1, INC
|
|
STFSDUX f0, CO1, INC
|
|
#endif
|
|
|
|
STFDUX f1, CO1, INC
|
|
STFSDUX f1, CO1, INC
|
|
|
|
#ifdef TRMMKERNEL
|
|
#if ( defined(LEFT) && defined(TRANSA)) || \
|
|
(!defined(LEFT) && !defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#ifdef LEFT
|
|
addi TEMP, TEMP, -4
|
|
#else
|
|
addi TEMP, TEMP, -1
|
|
#endif
|
|
slwi r0, TEMP, 2 + BASE_SHIFT
|
|
slwi TEMP, TEMP, 0 + BASE_SHIFT
|
|
add AO, AO, r0
|
|
add BO, BO, TEMP
|
|
#endif
|
|
|
|
#ifdef LEFT
|
|
addi KK, KK, 4
|
|
#endif
|
|
#endif
|
|
|
|
li r0, FZERO
|
|
lfpsx f0, SP, r0
|
|
.align 4
|
|
|
|
.L10110:
|
|
andi. I, M, 2
|
|
beq .L10120
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
#else
|
|
slwi TEMP, KK, 1 + BASE_SHIFT
|
|
slwi r0, KK, 0 + BASE_SHIFT
|
|
add AO, AO, TEMP
|
|
add BO, B, r0
|
|
|
|
fpmr f1, f0
|
|
addi BO, BO, - 2 * SIZE
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
#endif
|
|
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 2
|
|
#else
|
|
addi TEMP, KK, 1
|
|
#endif
|
|
srawi. r0, TEMP, 3
|
|
mtspr CTR, r0
|
|
ble .L10114
|
|
#else
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
|
|
srawi. r0, K, 3
|
|
mtspr CTR, r0
|
|
ble .L10114
|
|
#endif
|
|
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
|
|
LFPDUX A3, AO, INC2
|
|
LFPDUX A4, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
|
|
LFPDUX A5, AO, INC2
|
|
LFPDUX A6, AO, INC2
|
|
LFPDUX B3, BO, INC2
|
|
|
|
LFPDUX A7, AO, INC2
|
|
LFPDUX A8, AO, INC2
|
|
LFPDUX B4, BO, INC2
|
|
bdz- .L10113
|
|
.align 4
|
|
|
|
.L10112:
|
|
fxcpmadd f0, B1, A1, f0
|
|
LFPDUX A1, AO, INC2
|
|
fxcsmadd f1, B1, A2, f1
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
fxcpmadd f2, B2, A3, f2
|
|
LFPDUX A3, AO, INC2
|
|
fxcsmadd f3, B2, A4, f3
|
|
LFPDUX A4, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
fxcpmadd f0, B3, A5, f0
|
|
LFPDUX A5, AO, INC2
|
|
fxcsmadd f1, B3, A6, f1
|
|
LFPDUX A6, AO, INC2
|
|
LFPDUX B3, BO, INC2
|
|
fxcpmadd f2, B4, A7, f2
|
|
LFPDUX A7, AO, INC2
|
|
fxcsmadd f3, B4, A8, f3
|
|
LFPDUX A8, AO, INC2
|
|
LFPDUX B4, BO, INC2
|
|
bdnz+ .L10112
|
|
.align 4
|
|
|
|
.L10113:
|
|
fxcpmadd f0, B1, A1, f0
|
|
fxcsmadd f1, B1, A2, f1
|
|
fxcpmadd f2, B2, A3, f2
|
|
fxcsmadd f3, B2, A4, f3
|
|
fxcpmadd f0, B3, A5, f0
|
|
fxcsmadd f1, B3, A6, f1
|
|
fxcpmadd f2, B4, A7, f2
|
|
fxcsmadd f3, B4, A8, f3
|
|
.align 4
|
|
|
|
.L10114:
|
|
lfd AP, ALPHA(SP)
|
|
#ifdef TRMMKERNEL
|
|
fsmfp AP, AP
|
|
#endif
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 2
|
|
#else
|
|
addi TEMP, KK, 1
|
|
#endif
|
|
andi. TEMP, TEMP, 7
|
|
mtspr CTR, TEMP
|
|
#else
|
|
andi. r0, K, 7
|
|
mtspr CTR, r0
|
|
#endif
|
|
ble+ .L10118
|
|
|
|
LFPDUX A1, AO, INC2
|
|
LFDX B1, BO, INC2
|
|
add BO, BO, INC
|
|
bdz- .L10117
|
|
.align 4
|
|
|
|
.L10116:
|
|
fxcpmadd f0, B1, A1, f0
|
|
LFPDUX A1, AO, INC2
|
|
LFDX B1, BO, INC2
|
|
add BO, BO, INC
|
|
bdnz+ .L10116
|
|
.align 4
|
|
|
|
.L10117:
|
|
fxcpmadd f0, B1, A1, f0
|
|
.align 4
|
|
|
|
.L10118:
|
|
#ifndef TRMMKERNEL
|
|
LFDUX A1, CO1, INC
|
|
LFDUX A2, CO1, INC
|
|
|
|
fpadd f0, f0, f1
|
|
fpadd f2, f3, f2
|
|
fsmfp A1, A2
|
|
fpadd f0, f0, f2
|
|
fxcpmadd f1, AP, f0, A1
|
|
|
|
li r0, FZERO
|
|
lfpsx f0, SP, r0
|
|
|
|
STFDUX f1, CO1, INCM1
|
|
STFSDUX f1, CO1, INC
|
|
#else
|
|
fpadd f0, f0, f1
|
|
fpadd f2, f3, f2
|
|
fsmfp A1, A2
|
|
fpadd f0, f0, f2
|
|
fpmul f1, AP, f0
|
|
|
|
li r0, FZERO
|
|
lfpsx f0, SP, r0
|
|
|
|
STFDUX f1, CO1, INC
|
|
STFSDUX f1, CO1, INC
|
|
#endif
|
|
|
|
|
|
#ifdef TRMMKERNEL
|
|
#if ( defined(LEFT) && defined(TRANSA)) || \
|
|
(!defined(LEFT) && !defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#ifdef LEFT
|
|
addi TEMP, TEMP, -2
|
|
#else
|
|
addi TEMP, TEMP, -1
|
|
#endif
|
|
slwi r0, TEMP, 1 + BASE_SHIFT
|
|
slwi TEMP, TEMP, 0 + BASE_SHIFT
|
|
add AO, AO, r0
|
|
add BO, BO, TEMP
|
|
#endif
|
|
|
|
#ifdef LEFT
|
|
addi KK, KK, 2
|
|
#endif
|
|
#endif
|
|
.align 4
|
|
|
|
.L10120:
|
|
andi. I, M, 1
|
|
beq .L10999
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
#else
|
|
slwi TEMP, KK, 0 + BASE_SHIFT
|
|
slwi r0, KK, 0 + BASE_SHIFT
|
|
add AO, AO, TEMP
|
|
add BO, B, r0
|
|
|
|
fpmr f1, f0
|
|
addi BO, BO, - 2 * SIZE
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
#endif
|
|
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 1
|
|
#else
|
|
addi TEMP, KK, 1
|
|
#endif
|
|
srawi. r0, TEMP, 3
|
|
mtspr CTR, r0
|
|
ble .L10124
|
|
#else
|
|
addi BO, B, - 2 * SIZE
|
|
fpmr f1, f0
|
|
fpmr f2, f0
|
|
fpmr f3, f0
|
|
|
|
srawi. r0, K, 3
|
|
mtspr CTR, r0
|
|
ble .L10124
|
|
#endif
|
|
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
LFPDUX A3, AO, INC2
|
|
LFPDUX B3, BO, INC2
|
|
LFPDUX A4, AO, INC2
|
|
LFPDUX B4, BO, INC2
|
|
bdz- .L10123
|
|
.align 4
|
|
|
|
.L10122:
|
|
fpmadd f0, A1, B1, f0
|
|
LFPDUX A1, AO, INC2
|
|
LFPDUX B1, BO, INC2
|
|
fpmadd f1, A2, B2, f1
|
|
LFPDUX A2, AO, INC2
|
|
LFPDUX B2, BO, INC2
|
|
fpmadd f2, A3, B3, f2
|
|
LFPDUX A3, AO, INC2
|
|
LFPDUX B3, BO, INC2
|
|
fpmadd f3, A4, B4, f3
|
|
LFPDUX A4, AO, INC2
|
|
LFPDUX B4, BO, INC2
|
|
bdnz+ .L10122
|
|
.align 4
|
|
|
|
.L10123:
|
|
fpmadd f0, A1, B1, f0
|
|
fpmadd f1, A2, B2, f1
|
|
fpmadd f2, A3, B3, f2
|
|
fpmadd f3, A4, B4, f3
|
|
.align 4
|
|
|
|
.L10124:
|
|
lfd AP, ALPHA(SP)
|
|
#ifdef TRMMKERNEL
|
|
fsmfp AP, AP
|
|
#endif
|
|
|
|
#if defined(TRMMKERNEL)
|
|
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
|
sub TEMP, K, KK
|
|
#elif defined(LEFT)
|
|
addi TEMP, KK, 1
|
|
#else
|
|
addi TEMP, KK, 1
|
|
#endif
|
|
andi. TEMP, TEMP, 7
|
|
mtspr CTR, TEMP
|
|
#else
|
|
andi. r0, K, 7
|
|
mtspr CTR, r0
|
|
#endif
|
|
ble+ .L10128
|
|
|
|
LFDX A1, AO, INC2
|
|
LFDX B1, BO, INC2
|
|
add AO, AO, INC
|
|
add BO, BO, INC
|
|
bdz- .L10127
|
|
.align 4
|
|
|
|
.L10126:
|
|
fmadd f0, A1, B1, f0
|
|
LFDX A1, AO, INC2
|
|
LFDX B1, BO, INC2
|
|
add AO, AO, INC
|
|
add BO, BO, INC
|
|
bdnz+ .L10126
|
|
.align 4
|
|
|
|
.L10127:
|
|
fmadd f0, A1, B1, f0
|
|
.align 4
|
|
|
|
.L10128:
|
|
#ifndef TRMMKERNEL
|
|
LFDX A1, CO1, INC
|
|
fpadd f0, f0, f1
|
|
fpadd f2, f2, f3
|
|
fpadd f0, f0, f2
|
|
fsmtp f1, f0
|
|
fadd f0, f0, f1
|
|
fmadd f0, AP, f0, A1
|
|
STFDUX f0, CO1, INC
|
|
#else
|
|
fpadd f0, f0, f1
|
|
fpadd f2, f2, f3
|
|
fpadd f0, f0, f2
|
|
fsmtp f1, f0
|
|
fadd f0, f0, f1
|
|
fmul f0, AP, f0
|
|
STFDUX f0, CO1, INC
|
|
#endif
|
|
.align 4
|
|
|
|
.L10999:
|
|
addi SP, SP, 12
|
|
|
|
lwzu r14, 4(SP)
|
|
lwzu r15, 4(SP)
|
|
|
|
lwzu r16, 4(SP)
|
|
lwzu r17, 4(SP)
|
|
lwzu r18, 4(SP)
|
|
lwzu r19, 4(SP)
|
|
|
|
lwzu r20, 4(SP)
|
|
lwzu r21, 4(SP)
|
|
lwzu r22, 4(SP)
|
|
lwzu r23, 4(SP)
|
|
|
|
lwzu r24, 4(SP)
|
|
lwzu r25, 4(SP)
|
|
lwzu r26, 4(SP)
|
|
lwzu r27, 4(SP)
|
|
|
|
lwzu r28, 4(SP)
|
|
lwzu r29, 4(SP)
|
|
lwzu r30, 4(SP)
|
|
lwzu r31, 4(SP)
|
|
|
|
subi SP, SP, 12
|
|
li r0, 16
|
|
|
|
lfpdux f31, SP, r0
|
|
lfpdux f30, SP, r0
|
|
lfpdux f29, SP, r0
|
|
lfpdux f28, SP, r0
|
|
lfpdux f27, SP, r0
|
|
lfpdux f26, SP, r0
|
|
lfpdux f25, SP, r0
|
|
lfpdux f24, SP, r0
|
|
lfpdux f23, SP, r0
|
|
lfpdux f22, SP, r0
|
|
lfpdux f21, SP, r0
|
|
lfpdux f20, SP, r0
|
|
lfpdux f19, SP, r0
|
|
lfpdux f18, SP, r0
|
|
lfpdux f17, SP, r0
|
|
lfpdux f16, SP, r0
|
|
lfpdux f15, SP, r0
|
|
lfpdux f14, SP, r0
|
|
addi SP, SP, 16
|
|
blr
|
|
|
|
|
|
EPILOGUE
|
|
#endif
|