Merge pull request #2679 from RajalakshmiSR/P10_GEMM
POWER10 Optimized GEMM kernels
This commit is contained in:
Martin Kroeker
GitHub
commit
dfe819f3bd
|
|
@ -7,12 +7,12 @@ else
|
|||
#CGEMM_BETA = ../generic/zgemm_beta.c
|
||||
#ZGEMM_BETA = ../generic/zgemm_beta.c
|
||||
|
||||
STRMMKERNEL = sgemm_kernel_power9.S
|
||||
DTRMMKERNEL = dgemm_kernel_power9.S
|
||||
CTRMMKERNEL = cgemm_kernel_power9.S
|
||||
ZTRMMKERNEL = zgemm_kernel_power9.S
|
||||
STRMMKERNEL = sgemm_kernel_power10.c
|
||||
DTRMMKERNEL = dgemm_kernel_power10.c
|
||||
CTRMMKERNEL = cgemm_kernel_power10.S
|
||||
ZTRMMKERNEL = zgemm_kernel_power10.S
|
||||
|
||||
SGEMMKERNEL = sgemm_kernel_power9.S
|
||||
SGEMMKERNEL = sgemm_kernel_power10.c
|
||||
SGEMMINCOPY = ../generic/gemm_ncopy_16.c
|
||||
SGEMMITCOPY = sgemm_tcopy_16_power8.S
|
||||
SGEMMONCOPY = ../generic/gemm_ncopy_8.c
|
||||
|
|
@ -22,7 +22,7 @@ SGEMMITCOPYOBJ = sgemm_itcopy$(TSUFFIX).$(SUFFIX)
|
|||
SGEMMONCOPYOBJ = sgemm_oncopy$(TSUFFIX).$(SUFFIX)
|
||||
SGEMMOTCOPYOBJ = sgemm_otcopy$(TSUFFIX).$(SUFFIX)
|
||||
|
||||
DGEMMKERNEL = dgemm_kernel_power9.S
|
||||
DGEMMKERNEL = dgemm_kernel_power10.c
|
||||
DGEMMINCOPY = ../generic/gemm_ncopy_16.c
|
||||
DGEMMITCOPY = dgemm_tcopy_16_power8.S
|
||||
DGEMMONCOPY = dgemm_ncopy_4_power8.S
|
||||
|
|
@ -32,7 +32,7 @@ DGEMMITCOPYOBJ = dgemm_itcopy$(TSUFFIX).$(SUFFIX)
|
|||
DGEMMONCOPYOBJ = dgemm_oncopy$(TSUFFIX).$(SUFFIX)
|
||||
DGEMMOTCOPYOBJ = dgemm_otcopy$(TSUFFIX).$(SUFFIX)
|
||||
|
||||
CGEMMKERNEL = cgemm_kernel_power9.S
|
||||
CGEMMKERNEL = cgemm_kernel_power10.S
|
||||
CGEMMINCOPY = ../generic/zgemm_ncopy_8.c
|
||||
CGEMMITCOPY = ../generic/zgemm_tcopy_8.c
|
||||
CGEMMONCOPY = ../generic/zgemm_ncopy_4.c
|
||||
|
|
@ -42,7 +42,7 @@ CGEMMOTCOPYOBJ = cgemm_otcopy$(TSUFFIX).$(SUFFIX)
|
|||
CGEMMINCOPYOBJ = cgemm_incopy$(TSUFFIX).$(SUFFIX)
|
||||
CGEMMITCOPYOBJ = cgemm_itcopy$(TSUFFIX).$(SUFFIX)
|
||||
|
||||
ZGEMMKERNEL = zgemm_kernel_power9.S
|
||||
ZGEMMKERNEL = zgemm_kernel_power10.S
|
||||
ZGEMMONCOPY = ../generic/zgemm_ncopy_2.c
|
||||
ZGEMMOTCOPY = ../generic/zgemm_tcopy_2.c
|
||||
ZGEMMINCOPY = ../generic/zgemm_ncopy_8.c
|
||||
|
|
|
|||
|
|
@ -0,0 +1,286 @@
|
|||
/***************************************************************************
|
||||
Copyright (c) 2013-2020, The OpenBLAS Project
|
||||
All rights reserved.
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
1. Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in
|
||||
the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
3. Neither the name of the OpenBLAS project nor the names of
|
||||
its contributors may be used to endorse or promote products
|
||||
derived from this software without specific prior written permission.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
|
||||
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
|
||||
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*****************************************************************************/
|
||||
|
||||
#define ASSEMBLER
|
||||
#include "common.h"
|
||||
#include "def_vsx.h"
|
||||
|
||||
|
||||
#define LOAD ld
|
||||
#define STACKSIZE (512 )
|
||||
#define FLINK_SAVE (STACKSIZE+16) /* 16($r12) */
|
||||
#define M r3
|
||||
#define N r4
|
||||
#define K r5
|
||||
|
||||
|
||||
#define A r8
|
||||
#define B r9
|
||||
#define C r10
|
||||
#define LDC r6
|
||||
#define OFFSET r7
|
||||
|
||||
|
||||
#define alpha_r vs51
|
||||
#define alpha_i vs55
|
||||
#define save_permute_1 vs59
|
||||
#define permute_mask vs63
|
||||
#define o0 0
|
||||
|
||||
|
||||
#define T1 r11
|
||||
#define T2 r12
|
||||
#define T3 r14
|
||||
#define T4 r15
|
||||
#define T5 r16
|
||||
#define T6 r17
|
||||
#define L r18
|
||||
#define T7 r19
|
||||
#define T8 r20
|
||||
#define TEMP_REG r21
|
||||
#define I r22
|
||||
#define J r23
|
||||
#define AO r24
|
||||
#define BO r25
|
||||
#define CO r26
|
||||
#define T9 r27
|
||||
#define T10 r28
|
||||
#define PRE r29
|
||||
|
||||
#define T12 r30
|
||||
#define T13 r31
|
||||
|
||||
#include "cgemm_macros_power10.S"
|
||||
|
||||
.equ perm_const1, 0x0405060700010203
|
||||
.equ perm_const2, 0x0c0d0e0f08090a0b
|
||||
.equ save_permute_12, 0x0c0d0e0f1c1d1e1f
|
||||
.equ save_permute_11, 0x0405060714151617
|
||||
|
||||
|
||||
|
||||
#ifndef NEEDPARAM
|
||||
|
||||
PROLOGUE
|
||||
PROFCODE
|
||||
|
||||
|
||||
addi SP, SP, -STACKSIZE
|
||||
mflr r0
|
||||
|
||||
|
||||
stfd f14, 0(SP)
|
||||
stfd f15, 8(SP)
|
||||
stfd f16, 16(SP)
|
||||
stfd f17, 24(SP)
|
||||
|
||||
stfd f18, 32(SP)
|
||||
stfd f19, 40(SP)
|
||||
stfd f20, 48(SP)
|
||||
stfd f21, 56(SP)
|
||||
|
||||
stfd f22, 64(SP)
|
||||
stfd f23, 72(SP)
|
||||
stfd f24, 80(SP)
|
||||
stfd f25, 88(SP)
|
||||
|
||||
stfd f26, 96(SP)
|
||||
stfd f27, 104(SP)
|
||||
stfd f28, 112(SP)
|
||||
stfd f29, 120(SP)
|
||||
|
||||
stfd f30, 128(SP)
|
||||
stfd f31, 136(SP)
|
||||
|
||||
|
||||
std r31, 144(SP)
|
||||
std r30, 152(SP)
|
||||
std r29, 160(SP)
|
||||
std r28, 168(SP)
|
||||
std r27, 176(SP)
|
||||
std r26, 184(SP)
|
||||
std r25, 192(SP)
|
||||
std r24, 200(SP)
|
||||
std r23, 208(SP)
|
||||
std r22, 216(SP)
|
||||
std r21, 224(SP)
|
||||
std r20, 232(SP)
|
||||
std r19, 240(SP)
|
||||
std r18, 248(SP)
|
||||
std r17, 256(SP)
|
||||
std r16, 264(SP)
|
||||
std r15, 272(SP)
|
||||
std r14, 280(SP)
|
||||
|
||||
|
||||
stxv vs52, 288(SP)
|
||||
stxv vs53, 304(SP)
|
||||
stxv vs54, 320(SP)
|
||||
stxv vs55, 336(SP)
|
||||
stxv vs56, 352(SP)
|
||||
stxv vs57, 368(SP)
|
||||
stxv vs58, 384(SP)
|
||||
stxv vs59, 400(SP)
|
||||
stxv vs60, 416(SP)
|
||||
stxv vs61, 432(SP)
|
||||
stxv vs62, 448(SP)
|
||||
stxv vs63, 464(SP)
|
||||
std r0, FLINK_SAVE(SP)
|
||||
|
||||
|
||||
|
||||
ld LDC, FRAMESLOT(0) + STACKSIZE(SP)
|
||||
|
||||
|
||||
|
||||
#ifdef TRMMKERNEL
|
||||
ld OFFSET, FRAMESLOT(1) + STACKSIZE(SP)
|
||||
#endif
|
||||
slwi LDC, LDC, ZBASE_SHIFT
|
||||
|
||||
|
||||
|
||||
/*alpha is stored in f1. convert to single and splat*/
|
||||
xscvdpspn alpha_r,vs1
|
||||
xscvdpspn alpha_i,vs2
|
||||
xxspltw alpha_r,alpha_r,0
|
||||
xxspltw alpha_i,alpha_i,0
|
||||
/*load reverse permute mask for big endian
|
||||
uint128 = 0xc0d0e0f08090a0b0405060700010203
|
||||
*/
|
||||
|
||||
lis T2, perm_const2@highest
|
||||
lis T1, perm_const1@highest
|
||||
lis T3, save_permute_12@highest
|
||||
lis T4, save_permute_11@highest
|
||||
|
||||
|
||||
ori T2, T2, perm_const2@higher
|
||||
ori T1, T1, perm_const1@higher
|
||||
ori T3, T3, save_permute_12@higher
|
||||
ori T4, T4, save_permute_11@higher
|
||||
|
||||
|
||||
rldicr T2, T2, 32, 31
|
||||
rldicr T1, T1, 32, 31
|
||||
rldicr T3, T3, 32, 31
|
||||
rldicr T4, T4, 32, 31
|
||||
|
||||
oris T2, T2, perm_const2@h
|
||||
oris T1, T1, perm_const1@h
|
||||
oris T3, T3, save_permute_12@h
|
||||
oris T4, T4, save_permute_11@h
|
||||
|
||||
|
||||
ori T2, T2, perm_const2@l
|
||||
ori T1, T1, perm_const1@l
|
||||
ori T3, T3, save_permute_12@l
|
||||
ori T4, T4, save_permute_11@l
|
||||
|
||||
|
||||
li r0,0
|
||||
li PRE,512
|
||||
|
||||
#if defined(CC) || defined(CR) || defined(RC) || defined(RR)
|
||||
/*negate for this case as we will use addition -1*(a+b) */
|
||||
xvnegsp alpha_r,alpha_r
|
||||
xvnegsp alpha_i,alpha_i
|
||||
#endif
|
||||
|
||||
mtvsrdd permute_mask,T2,T1
|
||||
mtvsrdd save_permute_1,T3,T4
|
||||
|
||||
/*mask is reverse permute so we have to make it inner permute */
|
||||
xxpermdi permute_mask, permute_mask, permute_mask,2
|
||||
|
||||
#include "cgemm_logic_power10.S"
|
||||
|
||||
.L999:
|
||||
lfd f14, 0(SP)
|
||||
lfd f15, 8(SP)
|
||||
lfd f16, 16(SP)
|
||||
lfd f17, 24(SP)
|
||||
|
||||
lfd f18, 32(SP)
|
||||
lfd f19, 40(SP)
|
||||
lfd f20, 48(SP)
|
||||
lfd f21, 56(SP)
|
||||
|
||||
lfd f22, 64(SP)
|
||||
lfd f23, 72(SP)
|
||||
lfd f24, 80(SP)
|
||||
lfd f25, 88(SP)
|
||||
|
||||
lfd f26, 96(SP)
|
||||
lfd f27, 104(SP)
|
||||
lfd f28, 112(SP)
|
||||
lfd f29, 120(SP)
|
||||
|
||||
lfd f30, 128(SP)
|
||||
lfd f31, 136(SP)
|
||||
|
||||
ld r31, 144(SP)
|
||||
ld r30, 152(SP)
|
||||
ld r29, 160(SP)
|
||||
ld r28, 168(SP)
|
||||
ld r27, 176(SP)
|
||||
ld r26, 184(SP)
|
||||
ld r25, 192(SP)
|
||||
ld r24, 200(SP)
|
||||
ld r23, 208(SP)
|
||||
ld r22, 216(SP)
|
||||
ld r21, 224(SP)
|
||||
ld r20, 232(SP)
|
||||
ld r19, 240(SP)
|
||||
ld r18, 248(SP)
|
||||
ld r17, 256(SP)
|
||||
ld r16, 264(SP)
|
||||
ld r15, 272(SP)
|
||||
ld r14, 280(SP)
|
||||
|
||||
ld r0, FLINK_SAVE(SP)
|
||||
|
||||
lxv vs52, 288(SP)
|
||||
lxv vs53, 304(SP)
|
||||
lxv vs54, 320(SP)
|
||||
lxv vs55, 336(SP)
|
||||
lxv vs56, 352(SP)
|
||||
lxv vs57, 368(SP)
|
||||
lxv vs58, 384(SP)
|
||||
lxv vs59, 400(SP)
|
||||
mtlr r0
|
||||
lxv vs60, 416(SP)
|
||||
lxv vs61, 432(SP)
|
||||
lxv vs62, 448(SP)
|
||||
lxv vs63, 464(SP)
|
||||
|
||||
addi SP, SP, STACKSIZE
|
||||
blr
|
||||
|
||||
|
||||
EPILOGUE
|
||||
#endif
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,864 @@
|
|||
/*********************************************************************************
|
||||
Copyright (c) 2020, The OpenBLAS Project
|
||||
All rights reserved.
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
1. Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in
|
||||
the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
3. Neither the name of the OpenBLAS project nor the names of
|
||||
its contributors may be used to endorse or promote products
|
||||
derived from this software without specific prior written permission.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
|
||||
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
|
||||
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
**********************************************************************************/
|
||||
#include "common.h"
|
||||
#include <altivec.h>
|
||||
|
||||
typedef unsigned char vec_t __attribute__ ((vector_size (16)));
|
||||
typedef FLOAT v4sf_t __attribute__ ((vector_size (16)));
|
||||
typedef FLOAT v2sf_t __attribute__ ((vector_size (8)));
|
||||
|
||||
#ifdef TRMMKERNEL
|
||||
#define SAVE_ACC(ACC, J) \
|
||||
__builtin_mma_disassemble_acc (result, ACC); \
|
||||
rowC = (v4sf_t *) &CO[0* ldc+J]; \
|
||||
rowC[0] = result[3] * alpha; \
|
||||
rowC = (v4sf_t *) &CO[1*ldc+J]; \
|
||||
rowC[0] = result[2] * alpha; \
|
||||
rowC = (v4sf_t *) &CO[2*ldc+J]; \
|
||||
rowC[0] = result[1] * alpha; \
|
||||
rowC = (v4sf_t *) &CO[3*ldc+J]; \
|
||||
rowC[0] = result[0] * alpha;
|
||||
#define SAVE_ACC1(ACC, J) \
|
||||
__builtin_mma_disassemble_acc (result, ACC); \
|
||||
rowC = (v4sf_t *) &CO[4* ldc+J]; \
|
||||
rowC[0] = result[3] * alpha; \
|
||||
rowC = (v4sf_t *) &CO[5*ldc+J]; \
|
||||
rowC[0] = result[2] * alpha; \
|
||||
rowC = (v4sf_t *) &CO[6*ldc+J]; \
|
||||
rowC[0] = result[1] * alpha; \
|
||||
rowC = (v4sf_t *) &CO[7*ldc+J]; \
|
||||
rowC[0] = result[0] * alpha;
|
||||
#define SAVE2x4_ACC(ACC, J) \
|
||||
__builtin_mma_disassemble_acc (result, ACC); \
|
||||
rowC = (v4sf_t *) &CO[0* ldc+J]; \
|
||||
rowC[0] = result[3] * alpha; \
|
||||
rowC = (v4sf_t *) &CO[1* ldc+J]; \
|
||||
rowC[0] = result[2] * alpha;
|
||||
#else
|
||||
#define SAVE_ACC(ACC, J) \
|
||||
__builtin_mma_disassemble_acc (result, ACC); \
|
||||
rowC = (v4sf_t *) &CO[0* ldc+J]; \
|
||||
rowC[0] += result[3] * alpha; \
|
||||
rowC = (v4sf_t *) &CO[1*ldc+J]; \
|
||||
rowC[0] += result[2] * alpha; \
|
||||
rowC = (v4sf_t *) &CO[2*ldc+J]; \
|
||||
rowC[0] += result[1] * alpha; \
|
||||
rowC = (v4sf_t *) &CO[3*ldc+J]; \
|
||||
rowC[0] += result[0] * alpha;
|
||||
#define SAVE_ACC1(ACC, J) \
|
||||
__builtin_mma_disassemble_acc (result, ACC); \
|
||||
rowC = (v4sf_t *) &CO[4* ldc+J]; \
|
||||
rowC[0] += result[3] * alpha; \
|
||||
rowC = (v4sf_t *) &CO[5*ldc+J]; \
|
||||
rowC[0] += result[2] * alpha; \
|
||||
rowC = (v4sf_t *) &CO[6*ldc+J]; \
|
||||
rowC[0] += result[1] * alpha; \
|
||||
rowC = (v4sf_t *) &CO[7*ldc+J]; \
|
||||
rowC[0] += result[0] * alpha;
|
||||
#define SAVE2x4_ACC(ACC, J) \
|
||||
__builtin_mma_disassemble_acc (result, ACC); \
|
||||
rowC = (v4sf_t *) &CO[0* ldc+J]; \
|
||||
rowC[0] += result[3] * alpha; \
|
||||
rowC = (v4sf_t *) &CO[1* ldc+J]; \
|
||||
rowC[0] += result[2] * alpha;
|
||||
#endif
|
||||
|
||||
#define SET_ACC_ZERO4() \
|
||||
__builtin_mma_xxsetaccz (&acc0); \
|
||||
__builtin_mma_xxsetaccz (&acc1); \
|
||||
__builtin_mma_xxsetaccz (&acc2); \
|
||||
__builtin_mma_xxsetaccz (&acc3);
|
||||
|
||||
#define SET_ACC_ZERO8() \
|
||||
__builtin_mma_xxsetaccz (&acc0); \
|
||||
__builtin_mma_xxsetaccz (&acc1); \
|
||||
__builtin_mma_xxsetaccz (&acc2); \
|
||||
__builtin_mma_xxsetaccz (&acc3); \
|
||||
__builtin_mma_xxsetaccz (&acc4); \
|
||||
__builtin_mma_xxsetaccz (&acc5); \
|
||||
__builtin_mma_xxsetaccz (&acc6); \
|
||||
__builtin_mma_xxsetaccz (&acc7);
|
||||
|
||||
#define PREFETCH1(x, y) asm volatile ("dcbt %0, %1" : : "r" (x), "b" (y) : "memory");
|
||||
|
||||
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
|
||||
#define REFRESH_TEMP_BK(x, y) \
|
||||
temp = k - off;
|
||||
#elif defined(LEFT)
|
||||
#define REFRESH_TEMP_BK(x, y) \
|
||||
temp = off + x;
|
||||
#else
|
||||
#define REFRESH_TEMP_BK(x, y) \
|
||||
temp = off + y;
|
||||
#endif
|
||||
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
||||
#define REFRESH_POINTERS(x, y) \
|
||||
BO = B; \
|
||||
REFRESH_TEMP_BK(x, y)
|
||||
#else
|
||||
#define REFRESH_POINTERS(x, y) \
|
||||
AO += off * x; \
|
||||
BO = B + off * y; \
|
||||
REFRESH_TEMP_BK(x, y)
|
||||
#endif
|
||||
|
||||
#ifdef LEFT
|
||||
#define REFRESH_OFF(x) \
|
||||
off += x;
|
||||
#else
|
||||
#define REFRESH_OFF(x)
|
||||
#endif
|
||||
|
||||
#ifdef LEFT
|
||||
#define UPDATE_TEMP(x, y) \
|
||||
temp -= x;
|
||||
#else
|
||||
#define UPDATE_TEMP(x, y) \
|
||||
temp -= y;
|
||||
#endif
|
||||
|
||||
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
|
||||
#define REFRESH_TMP_AFTER_SAVE(x, y) \
|
||||
temp = k - off; \
|
||||
UPDATE_TEMP(x, y) \
|
||||
AO += temp * x; \
|
||||
BO += temp * y;
|
||||
#else
|
||||
#define REFRESH_TMP_AFTER_SAVE(x, y)
|
||||
#endif
|
||||
|
||||
#define REFRESH_AFTER_SAVE(x,y) \
|
||||
REFRESH_TMP_AFTER_SAVE(x, y) \
|
||||
REFRESH_OFF(x)
|
||||
/*************************************************************************************
|
||||
* GEMM Kernel
|
||||
*************************************************************************************/
|
||||
int
|
||||
CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
|
||||
FLOAT * C, BLASLONG ldc
|
||||
#ifdef TRMMKERNEL
|
||||
, BLASLONG offset
|
||||
#endif
|
||||
)
|
||||
{
|
||||
BLASLONG N = n;
|
||||
BLASLONG i1;
|
||||
#if defined(TRMMKERNEL)
|
||||
BLASLONG off;
|
||||
#endif
|
||||
#if defined(TRMMKERNEL) && !defined(LEFT)
|
||||
off = -offset;
|
||||
#endif
|
||||
v4sf_t valpha = { alpha, alpha };
|
||||
N = n >> 2;
|
||||
for (i1 = 0; i1 < N; i1++)
|
||||
{
|
||||
BLASLONG i, j, temp;
|
||||
FLOAT *CO;
|
||||
FLOAT *AO;
|
||||
#if defined(TRMMKERNEL) && defined(LEFT)
|
||||
off = offset;
|
||||
#endif
|
||||
CO = C;
|
||||
C += ldc << 2;
|
||||
AO = A;
|
||||
PREFETCH1 (A, 128);
|
||||
PREFETCH1 (A, 256);
|
||||
i = m >> 4;
|
||||
for (j = 0; j < i; j++)
|
||||
{
|
||||
FLOAT *BO;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_POINTERS (16, 4);
|
||||
#else
|
||||
BO = B;
|
||||
temp = k;
|
||||
#endif
|
||||
v4sf_t *rowC;
|
||||
v4sf_t result[4];
|
||||
BLASLONG l = 0;
|
||||
PREFETCH1 (CO, 0);
|
||||
PREFETCH1 (CO + ldc, 0);
|
||||
PREFETCH1 (CO + ldc + ldc, 0);
|
||||
PREFETCH1 (CO + ldc + ldc + ldc, 0);
|
||||
PREFETCH1 (CO, 128);
|
||||
PREFETCH1 (CO + ldc, 128);
|
||||
PREFETCH1 (CO + ldc + ldc, 128);
|
||||
PREFETCH1 (CO + ldc + ldc + ldc, 128);
|
||||
__vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7;
|
||||
SET_ACC_ZERO8 ();
|
||||
for (l = 0; l < temp; l++)
|
||||
{
|
||||
vec_t *rowA = (vec_t *) & AO[l << 4];
|
||||
__vector_pair rowB;
|
||||
vec_t *rb = (vec_t *) & BO[l << 2];
|
||||
__builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
|
||||
__builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);
|
||||
__builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]);
|
||||
__builtin_mma_xvf64gerpp (&acc2, rowB, rowA[2]);
|
||||
__builtin_mma_xvf64gerpp (&acc3, rowB, rowA[3]);
|
||||
__builtin_mma_xvf64gerpp (&acc4, rowB, rowA[4]);
|
||||
__builtin_mma_xvf64gerpp (&acc5, rowB, rowA[5]);
|
||||
__builtin_mma_xvf64gerpp (&acc6, rowB, rowA[6]);
|
||||
__builtin_mma_xvf64gerpp (&acc7, rowB, rowA[7]);
|
||||
}
|
||||
SAVE_ACC (&acc0, 0);
|
||||
SAVE_ACC (&acc2, 4);
|
||||
SAVE_ACC (&acc1, 2);
|
||||
SAVE_ACC (&acc3, 6);
|
||||
SAVE_ACC (&acc4, 8);
|
||||
SAVE_ACC (&acc6, 12);
|
||||
SAVE_ACC (&acc5, 10);
|
||||
SAVE_ACC (&acc7, 14);
|
||||
AO += temp << 4;
|
||||
BO += temp << 2;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_AFTER_SAVE (16, 4)
|
||||
#endif
|
||||
CO += 16;
|
||||
}
|
||||
i = (m & 15) >> 3;
|
||||
for (j = 0; j < i; j++)
|
||||
{
|
||||
FLOAT *BO;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_POINTERS (8, 4);
|
||||
#else
|
||||
BO = B;
|
||||
temp = k;
|
||||
#endif
|
||||
v4sf_t *rowC;
|
||||
v4sf_t result[4];
|
||||
__vector_quad acc0, acc1, acc2, acc3;
|
||||
SET_ACC_ZERO4 ();
|
||||
BLASLONG l = 0;
|
||||
for (l = 0; l < temp; l++)
|
||||
{
|
||||
vec_t *rowA = (vec_t *) & AO[l << 3];
|
||||
__vector_pair rowB;
|
||||
vec_t *rb = (vec_t *) & BO[l << 2];
|
||||
__builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
|
||||
__builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);
|
||||
__builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]);
|
||||
__builtin_mma_xvf64gerpp (&acc2, rowB, rowA[2]);
|
||||
__builtin_mma_xvf64gerpp (&acc3, rowB, rowA[3]);
|
||||
}
|
||||
SAVE_ACC (&acc0, 0);
|
||||
SAVE_ACC (&acc2, 4);
|
||||
SAVE_ACC (&acc1, 2);
|
||||
SAVE_ACC (&acc3, 6);
|
||||
CO += 8;
|
||||
AO += temp << 3;
|
||||
BO += temp << 2;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_AFTER_SAVE (8, 4)
|
||||
#endif
|
||||
}
|
||||
i = (m & 7) >> 2;
|
||||
for (j = 0; j < i; j++)
|
||||
{
|
||||
FLOAT *BO;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_POINTERS (4, 4);
|
||||
#else
|
||||
BO = B;
|
||||
temp = k;
|
||||
#endif
|
||||
v4sf_t *rowC;
|
||||
v4sf_t result[4];
|
||||
__vector_quad acc0, acc1;
|
||||
__builtin_mma_xxsetaccz (&acc0);
|
||||
__builtin_mma_xxsetaccz (&acc1);
|
||||
BLASLONG l = 0;
|
||||
for (l = 0; l < temp; l++)
|
||||
{
|
||||
vec_t *rowA = (vec_t *) & AO[l << 2];
|
||||
__vector_pair rowB;
|
||||
vec_t *rb = (vec_t *) & BO[l << 2];
|
||||
__builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
|
||||
__builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);
|
||||
__builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]);
|
||||
}
|
||||
SAVE_ACC (&acc0, 0);
|
||||
SAVE_ACC (&acc1, 2);
|
||||
CO += 4;
|
||||
AO += temp << 2;
|
||||
BO += temp << 2;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_AFTER_SAVE (4, 4)
|
||||
#endif
|
||||
}
|
||||
i = (m & 3) >> 1;
|
||||
for (j = 0; j < i; j++)
|
||||
{
|
||||
FLOAT *BO;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_POINTERS (2, 4);
|
||||
#else
|
||||
BO = B;
|
||||
temp = k;
|
||||
#endif
|
||||
v4sf_t *rowC;
|
||||
v4sf_t result[4];
|
||||
__vector_quad acc0;
|
||||
__builtin_mma_xxsetaccz (&acc0);
|
||||
BLASLONG l = 0;
|
||||
for (l = 0; l < temp; l++)
|
||||
{
|
||||
vec_t *rowA = (vec_t *) & AO[l << 1];
|
||||
__vector_pair rowB;
|
||||
vec_t *rb = (vec_t *) & BO[l << 2];
|
||||
__builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
|
||||
__builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);
|
||||
}
|
||||
SAVE_ACC (&acc0, 0);
|
||||
CO += 2;
|
||||
AO += temp << 1;
|
||||
BO += temp << 2;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_AFTER_SAVE (2, 4)
|
||||
#endif
|
||||
}
|
||||
i = (m & 1) >> 0;
|
||||
for (j = 0; j < i; j++)
|
||||
{
|
||||
FLOAT *BO;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_POINTERS (1, 4);
|
||||
#else
|
||||
BO = B;
|
||||
temp = k;
|
||||
#endif
|
||||
BLASLONG l = 0;
|
||||
v4sf_t t = { 0, 0 };
|
||||
v4sf_t t1 = { 0, 0 };
|
||||
for (l = 0; l < temp; l++)
|
||||
{
|
||||
v4sf_t rowA = { AO[l], AO[l] };
|
||||
v4sf_t rowB = { BO[l << 2], BO[(l << 2) + 1] };
|
||||
v4sf_t rowB1 = { BO[(l << 2) + 2], BO[(l << 2) + 3] };
|
||||
t += rowA * rowB;
|
||||
t1 += rowA * rowB1;
|
||||
}
|
||||
t = t * valpha;
|
||||
t1 = t1 * valpha;
|
||||
#if defined(TRMMKERNEL)
|
||||
CO[0 * ldc] = t[0];
|
||||
CO[1 * ldc] = t[1];
|
||||
CO[2 * ldc] = t1[0];
|
||||
CO[3 * ldc] = t1[1];
|
||||
#else
|
||||
CO[0 * ldc] += t[0];
|
||||
CO[1 * ldc] += t[1];
|
||||
CO[2 * ldc] += t1[0];
|
||||
CO[3 * ldc] += t1[1];
|
||||
#endif
|
||||
CO += 1;
|
||||
AO += temp;
|
||||
BO += temp << 2;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_AFTER_SAVE (1, 4)
|
||||
#endif
|
||||
}
|
||||
#if defined(TRMMKERNEL) && !defined(LEFT)
|
||||
off += 4; // number of values in A
|
||||
#endif
|
||||
B += k << 2;
|
||||
}
|
||||
N = (n & 3) >> 1;
|
||||
for (i1 = 0; i1 < N; i1++)
|
||||
{
|
||||
BLASLONG i, j, temp;
|
||||
#if defined(TRMMKERNEL) && defined(LEFT)
|
||||
off = offset;
|
||||
#endif
|
||||
FLOAT *CO;
|
||||
FLOAT *AO;
|
||||
CO = C;
|
||||
C += ldc << 1;
|
||||
AO = A;
|
||||
i = m >> 4;
|
||||
for (j = 0; j < i; j++)
|
||||
{
|
||||
FLOAT *BO;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_POINTERS (16, 2);
|
||||
#else
|
||||
BO = B;
|
||||
temp = k;
|
||||
#endif
|
||||
v4sf_t *rowC;
|
||||
v4sf_t result[4];
|
||||
__vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7;
|
||||
SET_ACC_ZERO8 ();
|
||||
BLASLONG l = 0;
|
||||
for (l = 0; l < temp; l++)
|
||||
{
|
||||
FLOAT t[4] = { 0, 0, 0, 0 };
|
||||
t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];
|
||||
__vector_pair rowB;
|
||||
vec_t *rb = (vec_t *) & t[0];
|
||||
__builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
|
||||
vec_t *rowA = (vec_t *) & AO[l << 4];
|
||||
__builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);
|
||||
__builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]);
|
||||
__builtin_mma_xvf64gerpp (&acc2, rowB, rowA[2]);
|
||||
__builtin_mma_xvf64gerpp (&acc3, rowB, rowA[3]);
|
||||
__builtin_mma_xvf64gerpp (&acc4, rowB, rowA[4]);
|
||||
__builtin_mma_xvf64gerpp (&acc5, rowB, rowA[5]);
|
||||
__builtin_mma_xvf64gerpp (&acc6, rowB, rowA[6]);
|
||||
__builtin_mma_xvf64gerpp (&acc7, rowB, rowA[7]);
|
||||
}
|
||||
SAVE2x4_ACC (&acc0, 0);
|
||||
SAVE2x4_ACC (&acc1, 2);
|
||||
SAVE2x4_ACC (&acc2, 4);
|
||||
SAVE2x4_ACC (&acc3, 6);
|
||||
SAVE2x4_ACC (&acc4, 8);
|
||||
SAVE2x4_ACC (&acc5, 10);
|
||||
SAVE2x4_ACC (&acc6, 12);
|
||||
SAVE2x4_ACC (&acc7, 14);
|
||||
CO += 16;
|
||||
AO += temp << 4;
|
||||
BO += temp << 1;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_AFTER_SAVE (16, 2)
|
||||
#endif
|
||||
}
|
||||
i = (m & 15) >> 3;
|
||||
for (j = 0; j < i; j++)
|
||||
{
|
||||
FLOAT *BO;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_POINTERS (8, 2);
|
||||
#else
|
||||
BO = B;
|
||||
temp = k;
|
||||
#endif
|
||||
v4sf_t *rowC;
|
||||
v4sf_t result[4];
|
||||
__vector_quad acc0, acc1, acc2, acc3;
|
||||
SET_ACC_ZERO4 ();
|
||||
BLASLONG l = 0;
|
||||
for (l = 0; l < temp; l++)
|
||||
{
|
||||
FLOAT t[4] = { 0, 0, 0, 0 };
|
||||
t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];
|
||||
__vector_pair rowB;
|
||||
vec_t *rb = (vec_t *) & t[0];
|
||||
__builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
|
||||
vec_t *rowA = (vec_t *) & AO[l << 3];
|
||||
__builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);
|
||||
__builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]);
|
||||
__builtin_mma_xvf64gerpp (&acc2, rowB, rowA[2]);
|
||||
__builtin_mma_xvf64gerpp (&acc3, rowB, rowA[3]);
|
||||
}
|
||||
SAVE2x4_ACC (&acc0, 0);
|
||||
SAVE2x4_ACC (&acc1, 2);
|
||||
SAVE2x4_ACC (&acc2, 4);
|
||||
SAVE2x4_ACC (&acc3, 6);
|
||||
CO += 8;
|
||||
AO += temp << 3;
|
||||
BO += temp << 1;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_AFTER_SAVE (8, 2)
|
||||
#endif
|
||||
}
|
||||
i = (m & 7) >> 2;
|
||||
for (j = 0; j < i; j++)
|
||||
{
|
||||
FLOAT *BO;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_POINTERS (4, 2);
|
||||
#else
|
||||
BO = B;
|
||||
temp = k;
|
||||
#endif
|
||||
v4sf_t *rowC;
|
||||
v4sf_t result[4];
|
||||
__vector_quad acc0, acc1;
|
||||
__builtin_mma_xxsetaccz (&acc0);
|
||||
__builtin_mma_xxsetaccz (&acc1);
|
||||
BLASLONG l = 0;
|
||||
for (l = 0; l < temp; l++)
|
||||
{
|
||||
FLOAT t[4] = { 0, 0, 0, 0 };
|
||||
t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];
|
||||
__vector_pair rowB;
|
||||
vec_t *rb = (vec_t *) & t[0];
|
||||
__builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
|
||||
vec_t *rowA = (vec_t *) & AO[l << 2];
|
||||
__builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);
|
||||
__builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]);
|
||||
}
|
||||
SAVE2x4_ACC (&acc0, 0);
|
||||
SAVE2x4_ACC (&acc1, 2);
|
||||
CO += 4;
|
||||
AO += temp << 2;
|
||||
BO += temp << 1;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_AFTER_SAVE (4, 2)
|
||||
#endif
|
||||
}
|
||||
i = (m & 3) >> 1;
|
||||
for (j = 0; j < i; j++)
|
||||
{
|
||||
FLOAT *BO;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_POINTERS (2, 2);
|
||||
#else
|
||||
BO = B;
|
||||
temp = k;
|
||||
#endif
|
||||
v4sf_t *rowC;
|
||||
v4sf_t result[4];
|
||||
__vector_quad acc0;
|
||||
__builtin_mma_xxsetaccz (&acc0);
|
||||
BLASLONG l = 0;
|
||||
for (l = 0; l < temp; l++)
|
||||
{
|
||||
FLOAT t[4] = { 0, 0, 0, 0 };
|
||||
t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];
|
||||
__vector_pair rowB;
|
||||
vec_t *rb = (vec_t *) & t[0];
|
||||
__builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
|
||||
vec_t *rowA = (vec_t *) & AO[l << 1];
|
||||
__builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);
|
||||
}
|
||||
SAVE2x4_ACC (&acc0, 0);
|
||||
CO += 2;
|
||||
AO += temp << 1;
|
||||
BO += temp << 1;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_AFTER_SAVE (2, 2)
|
||||
#endif
|
||||
}
|
||||
i = (m & 1) >> 0;
|
||||
for (j = 0; j < i; j++)
|
||||
{
|
||||
FLOAT *BO;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_POINTERS (1, 2);
|
||||
#else
|
||||
BO = B;
|
||||
temp = k;
|
||||
#endif
|
||||
BLASLONG l = 0;
|
||||
v4sf_t t = { 0, 0 };
|
||||
for (l = 0; l < temp; l++)
|
||||
{
|
||||
v4sf_t rowA = { AO[l], AO[l] };
|
||||
v4sf_t rowB = { BO[l << 1], BO[(l << 1) + 1] };
|
||||
t += rowA * rowB;
|
||||
}
|
||||
t = t * valpha;
|
||||
#if defined(TRMMKERNEL)
|
||||
CO[0 * ldc] = t[0];
|
||||
CO[1 * ldc] = t[1];
|
||||
#else
|
||||
CO[0 * ldc] += t[0];
|
||||
CO[1 * ldc] += t[1];
|
||||
#endif
|
||||
CO += 1;
|
||||
AO += temp;
|
||||
BO += temp << 1;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_AFTER_SAVE (1, 2)
|
||||
#endif
|
||||
}
|
||||
#if defined(TRMMKERNEL) && !defined(LEFT)
|
||||
off += 2; // number of values in A
|
||||
#endif
|
||||
B += k << 1;
|
||||
}
|
||||
N = (n & 1) >> 0;
|
||||
for (i1 = 0; i1 < N; i1++)
|
||||
{
|
||||
BLASLONG i, temp;
|
||||
#if defined(TRMMKERNEL) && defined(LEFT)
|
||||
off = offset;
|
||||
#endif
|
||||
FLOAT *CO;
|
||||
FLOAT *AO;
|
||||
CO = C;
|
||||
C += ldc;
|
||||
AO = A;
|
||||
i = m;
|
||||
while (i >= 16)
|
||||
{
|
||||
FLOAT *BO;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_POINTERS (16, 1)
|
||||
#else
|
||||
BO = B;
|
||||
temp = k;
|
||||
#endif
|
||||
BLASLONG l = 0;
|
||||
v4sf_t t = { 0, 0 };
|
||||
v4sf_t t1 = { 0, 0 };
|
||||
v4sf_t t2 = { 0, 0 };
|
||||
v4sf_t t3 = { 0, 0 };
|
||||
v4sf_t t4 = { 0, 0 };
|
||||
v4sf_t t5 = { 0, 0 };
|
||||
v4sf_t t6 = { 0, 0 };
|
||||
v4sf_t t7 = { 0, 0 };
|
||||
for (l = 0; l < temp; l++)
|
||||
{
|
||||
v4sf_t rowB = { BO[l], BO[l] };
|
||||
v4sf_t rowA = { AO[l << 4], AO[(l << 4) + 1] };
|
||||
v4sf_t rowA1 = { AO[(l << 4) + 2], AO[(l << 4) + 3] };
|
||||
v4sf_t rowA2 = { AO[(l << 4) + 4], AO[(l << 4) + 5] };
|
||||
v4sf_t rowA3 = { AO[(l << 4) + 6], AO[(l << 4) + 7] };
|
||||
v4sf_t rowA4 = { AO[(l << 4) + 8], AO[(l << 4) + 9] };
|
||||
v4sf_t rowA5 = { AO[(l << 4) + 10], AO[(l << 4) + 11] };
|
||||
v4sf_t rowA6 = { AO[(l << 4) + 12], AO[(l << 4) + 13] };
|
||||
v4sf_t rowA7 = { AO[(l << 4) + 14], AO[(l << 4) + 15] };
|
||||
t += rowA * rowB;
|
||||
t1 += rowA1 * rowB;
|
||||
t2 += rowA2 * rowB;
|
||||
t3 += rowA3 * rowB;
|
||||
t4 += rowA4 * rowB;
|
||||
t5 += rowA5 * rowB;
|
||||
t6 += rowA6 * rowB;
|
||||
t7 += rowA7 * rowB;
|
||||
}
|
||||
t = t * valpha;
|
||||
t1 = t1 * valpha;
|
||||
t2 = t2 * valpha;
|
||||
t3 = t3 * valpha;
|
||||
t4 = t4 * valpha;
|
||||
t5 = t5 * valpha;
|
||||
t6 = t6 * valpha;
|
||||
t7 = t7 * valpha;
|
||||
#if defined(TRMMKERNEL)
|
||||
CO[0] = t[0];
|
||||
CO[1] = t[1];
|
||||
CO[2] = t1[0];
|
||||
CO[3] = t1[1];
|
||||
CO[4] = t2[0];
|
||||
CO[5] = t2[1];
|
||||
CO[6] = t3[0];
|
||||
CO[7] = t3[1];
|
||||
CO[8] = t4[0];
|
||||
CO[9] = t4[1];
|
||||
CO[10] = t5[0];
|
||||
CO[11] = t5[1];
|
||||
CO[12] = t6[0];
|
||||
CO[13] = t6[1];
|
||||
CO[14] = t7[0];
|
||||
CO[15] = t7[1];
|
||||
#else
|
||||
CO[0] += t[0];
|
||||
CO[1] += t[1];
|
||||
CO[2] += t1[0];
|
||||
CO[3] += t1[1];
|
||||
CO[4] += t2[0];
|
||||
CO[5] += t2[1];
|
||||
CO[6] += t3[0];
|
||||
CO[7] += t3[1];
|
||||
CO[8] += t4[0];
|
||||
CO[9] += t4[1];
|
||||
CO[10] += t5[0];
|
||||
CO[11] += t5[1];
|
||||
CO[12] += t6[0];
|
||||
CO[13] += t6[1];
|
||||
CO[14] += t7[0];
|
||||
CO[15] += t7[1];
|
||||
#endif
|
||||
AO += temp << 4;
|
||||
BO += temp;
|
||||
CO += 16;
|
||||
i -= 16;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_AFTER_SAVE (16, 1)
|
||||
#endif
|
||||
}
|
||||
while (i >= 8)
|
||||
{
|
||||
FLOAT *BO;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_POINTERS (8, 1)
|
||||
#else
|
||||
BO = B;
|
||||
temp = k;
|
||||
#endif
|
||||
BLASLONG l = 0;
|
||||
v4sf_t t = { 0, 0 };
|
||||
v4sf_t t1 = { 0, 0 };
|
||||
v4sf_t t2 = { 0, 0 };
|
||||
v4sf_t t3 = { 0, 0 };
|
||||
for (l = 0; l < temp; l++)
|
||||
{
|
||||
v4sf_t rowB = { BO[l], BO[l] };
|
||||
v4sf_t rowA = { AO[l << 3], AO[(l << 3) + 1] };
|
||||
v4sf_t rowA1 = { AO[(l << 3) + 2], AO[(l << 3) + 3] };
|
||||
v4sf_t rowA2 = { AO[(l << 3) + 4], AO[(l << 3) + 5] };
|
||||
v4sf_t rowA3 = { AO[(l << 3) + 6], AO[(l << 3) + 7] };
|
||||
t += rowA * rowB;
|
||||
t1 += rowA1 * rowB;
|
||||
t2 += rowA2 * rowB;
|
||||
t3 += rowA3 * rowB;
|
||||
}
|
||||
t = t * valpha;
|
||||
t1 = t1 * valpha;
|
||||
t2 = t2 * valpha;
|
||||
t3 = t3 * valpha;
|
||||
#if defined(TRMMKERNEL)
|
||||
CO[0] = t[0];
|
||||
CO[1] = t[1];
|
||||
CO[2] = t1[0];
|
||||
CO[3] = t1[1];
|
||||
CO[4] = t2[0];
|
||||
CO[5] = t2[1];
|
||||
CO[6] = t3[0];
|
||||
CO[7] = t3[1];
|
||||
#else
|
||||
CO[0] += t[0];
|
||||
CO[1] += t[1];
|
||||
CO[2] += t1[0];
|
||||
CO[3] += t1[1];
|
||||
CO[4] += t2[0];
|
||||
CO[5] += t2[1];
|
||||
CO[6] += t3[0];
|
||||
CO[7] += t3[1];
|
||||
#endif
|
||||
AO += temp << 3;
|
||||
BO += temp;
|
||||
CO += 8;
|
||||
i -= 8;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_AFTER_SAVE (8, 1)
|
||||
#endif
|
||||
}
|
||||
while (i >= 4)
|
||||
{
|
||||
FLOAT *BO;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_POINTERS (4, 1)
|
||||
#else
|
||||
BO = B;
|
||||
temp = k;
|
||||
#endif
|
||||
BLASLONG l = 0;
|
||||
v4sf_t t = { 0, 0 };
|
||||
v4sf_t t1 = { 0, 0 };
|
||||
for (l = 0; l < temp; l++)
|
||||
{
|
||||
v4sf_t rowB = { BO[l], BO[l] };
|
||||
v4sf_t rowA = { AO[l << 2], AO[(l << 2) + 1] };
|
||||
v4sf_t rowA1 = { AO[(l << 2) + 2], AO[(l << 2) + 3] };
|
||||
t += rowA * rowB;
|
||||
t1 += rowA1 * rowB;
|
||||
}
|
||||
t = t * valpha;
|
||||
t1 = t1 * valpha;
|
||||
#if defined(TRMMKERNEL)
|
||||
CO[0] = t[0];
|
||||
CO[1] = t[1];
|
||||
CO[2] = t1[0];
|
||||
CO[3] = t1[1];
|
||||
#else
|
||||
CO[0] += t[0];
|
||||
CO[1] += t[1];
|
||||
CO[2] += t1[0];
|
||||
CO[3] += t1[1];
|
||||
#endif
|
||||
AO += temp << 2;
|
||||
BO += temp;
|
||||
CO += 4;
|
||||
i -= 4;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_AFTER_SAVE (4, 1)
|
||||
#endif
|
||||
}
|
||||
while (i >= 2)
|
||||
{
|
||||
FLOAT *BO;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_POINTERS (2, 1)
|
||||
#else
|
||||
BO = B;
|
||||
temp = k;
|
||||
#endif
|
||||
BLASLONG l = 0;
|
||||
v4sf_t t = { 0, 0 };
|
||||
for (l = 0; l < temp; l++)
|
||||
{
|
||||
v4sf_t rowB = { BO[l], BO[l] };
|
||||
v4sf_t rowA = { AO[l << 1], AO[(l << 1) + 1] };
|
||||
t += rowA * rowB;
|
||||
}
|
||||
t = t * valpha;
|
||||
#if defined(TRMMKERNEL)
|
||||
CO[0] = t[0];
|
||||
CO[1] = t[1];
|
||||
#else
|
||||
CO[0] += t[0];
|
||||
CO[1] += t[1];
|
||||
#endif
|
||||
AO += temp << 1;
|
||||
BO += temp;
|
||||
CO += 2;
|
||||
i -= 2;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_AFTER_SAVE (2, 1)
|
||||
#endif
|
||||
}
|
||||
while (i >= 1)
|
||||
{
|
||||
FLOAT *BO;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_POINTERS (1, 1)
|
||||
#else
|
||||
BO = B;
|
||||
temp = k;
|
||||
#endif
|
||||
BLASLONG l = 0;
|
||||
FLOAT t = 0;
|
||||
for (l = 0; l < temp; l++)
|
||||
{
|
||||
t += AO[l] * BO[l];
|
||||
}
|
||||
AO += temp;
|
||||
BO += temp;
|
||||
#if defined(TRMMKERNEL)
|
||||
CO[0] = t * alpha;
|
||||
#else
|
||||
CO[0] += t * alpha;
|
||||
#endif
|
||||
CO += 1;
|
||||
i -= 1;
|
||||
#if defined(TRMMKERNEL)
|
||||
REFRESH_AFTER_SAVE (1, 1)
|
||||
#endif
|
||||
}
|
||||
#if defined(TRMMKERNEL) && !defined(LEFT)
|
||||
off += 1; // number of values in A
|
||||
#endif
|
||||
B += k;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,245 @@
|
|||
/***************************************************************************
|
||||
Copyright (c) 2013-2020, The OpenBLAS Project
|
||||
All rights reserved.
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
1. Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in
|
||||
the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
3. Neither the name of the OpenBLAS project nor the names of
|
||||
its contributors may be used to endorse or promote products
|
||||
derived from this software without specific prior written permission.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
|
||||
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
|
||||
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*****************************************************************************/
|
||||
#define ASSEMBLER
|
||||
#include "common.h"
|
||||
#include "def_vsx.h"
|
||||
|
||||
#define LOAD ld
|
||||
|
||||
#define STACKSIZE 512
|
||||
|
||||
#define FZERO 312+192(SP)
|
||||
|
||||
#define FLINK_SAVE (STACKSIZE+16) /* 16($r12) */
|
||||
|
||||
#define M r3
|
||||
#define N r4
|
||||
#define K r5
|
||||
|
||||
|
||||
#define A r8
|
||||
#define B r9
|
||||
#define C r10
|
||||
#define LDC r6
|
||||
#define OFFSET r7
|
||||
|
||||
|
||||
|
||||
#define o0 0
|
||||
#define alpha_r vs62
|
||||
#define alpha_i vs63
|
||||
|
||||
#define VECSAVE r11
|
||||
|
||||
#define FRAMEPOINTER r12
|
||||
|
||||
#define T10 r14
|
||||
|
||||
#define L r15
|
||||
#define T8 r16
|
||||
#define T5 r17
|
||||
#define T2 r19
|
||||
#define TEMP_REG r20
|
||||
#define T6 r21
|
||||
#define I r22
|
||||
#define J r23
|
||||
#define AO r24
|
||||
#define BO r25
|
||||
#define CO r26
|
||||
#define T7 r27
|
||||
#define T3 r28
|
||||
#define T4 r29
|
||||
|
||||
#define PRE r30
|
||||
#define T1 r31
|
||||
|
||||
#ifndef NEEDPARAM
|
||||
|
||||
PROLOGUE
|
||||
PROFCODE
|
||||
|
||||
mr FRAMEPOINTER, SP
|
||||
addi SP, SP, -STACKSIZE
|
||||
mflr r0
|
||||
stfd f14, 0(SP)
|
||||
stfd f15, 8(SP)
|
||||
stfd f16, 16(SP)
|
||||
stfd f17, 24(SP)
|
||||
|
||||
stfd f18, 32(SP)
|
||||
stfd f19, 40(SP)
|
||||
stfd f20, 48(SP)
|
||||
stfd f21, 56(SP)
|
||||
|
||||
stfd f22, 64(SP)
|
||||
stfd f23, 72(SP)
|
||||
stfd f24, 80(SP)
|
||||
stfd f25, 88(SP)
|
||||
|
||||
stfd f26, 96(SP)
|
||||
stfd f27, 104(SP)
|
||||
stfd f28, 112(SP)
|
||||
stfd f29, 120(SP)
|
||||
|
||||
stfd f30, 128(SP)
|
||||
stfd f31, 136(SP)
|
||||
|
||||
xxspltd alpha_r,vs1,0 /*copy from register f1 */
|
||||
xxspltd alpha_i,vs2,0 /*copy from register f2 */
|
||||
|
||||
std r31, 144(SP)
|
||||
std r30, 152(SP)
|
||||
std r29, 160(SP)
|
||||
std r28, 168(SP)
|
||||
std r27, 176(SP)
|
||||
std r26, 184(SP)
|
||||
std r25, 192(SP)
|
||||
std r24, 200(SP)
|
||||
std r23, 208(SP)
|
||||
std r22, 216(SP)
|
||||
std r21, 224(SP)
|
||||
std r20, 232(SP)
|
||||
std r19, 240(SP)
|
||||
std r18, 248(SP)
|
||||
std r17, 256(SP)
|
||||
std r16, 264(SP)
|
||||
std r15, 272(SP)
|
||||
std r14, 280(SP)
|
||||
|
||||
|
||||
stxv vs20, 288(SP)
|
||||
stxv vs21, 304(SP)
|
||||
stxv vs22, 320(SP)
|
||||
stxv vs23, 336(SP)
|
||||
stxv vs24, 352(SP)
|
||||
stxv vs25, 368(SP)
|
||||
stxv vs26, 384(SP)
|
||||
stxv vs27, 400(SP)
|
||||
stxv vs28, 416(SP)
|
||||
stxv vs29, 432(SP)
|
||||
stxv vs30, 448(SP)
|
||||
stxv vs31, 464(SP)
|
||||
|
||||
std r0, FLINK_SAVE(SP)
|
||||
|
||||
|
||||
#if defined(linux) || defined(__FreeBSD__)
|
||||
ld LDC, FRAMESLOT(0) + 0(FRAMEPOINTER)
|
||||
#endif
|
||||
|
||||
|
||||
#ifdef TRMMKERNEL
|
||||
#if (defined(linux) || defined(__FreeBSD__)) && defined(__64BIT__)
|
||||
ld OFFSET, FRAMESLOT(1) + 0(FRAMEPOINTER)
|
||||
#endif
|
||||
#endif
|
||||
|
||||
|
||||
#include "zgemm_macros_power10.S"
|
||||
|
||||
|
||||
|
||||
slwi LDC, LDC, ZBASE_SHIFT
|
||||
li PRE, 512
|
||||
li r0, 0
|
||||
|
||||
|
||||
#if defined(CC) || defined(CR) || defined(RC) || defined(RR)
|
||||
/*negate for this case as we will use addition -1*(a+b) */
|
||||
xvnegdp alpha_r,alpha_r
|
||||
xvnegdp alpha_i,alpha_i
|
||||
#endif
|
||||
.align 4
|
||||
|
||||
#include "zgemm_logic_power10.S"
|
||||
|
||||
L999:
|
||||
|
||||
lfd f14, 0(SP)
|
||||
lfd f15, 8(SP)
|
||||
lfd f16, 16(SP)
|
||||
lfd f17, 24(SP)
|
||||
|
||||
lfd f18, 32(SP)
|
||||
lfd f19, 40(SP)
|
||||
lfd f20, 48(SP)
|
||||
lfd f21, 56(SP)
|
||||
|
||||
lfd f22, 64(SP)
|
||||
lfd f23, 72(SP)
|
||||
lfd f24, 80(SP)
|
||||
lfd f25, 88(SP)
|
||||
|
||||
lfd f26, 96(SP)
|
||||
lfd f27, 104(SP)
|
||||
lfd f28, 112(SP)
|
||||
lfd f29, 120(SP)
|
||||
|
||||
lfd f30, 128(SP)
|
||||
lfd f31, 136(SP)
|
||||
|
||||
|
||||
ld r31, 144(SP)
|
||||
ld r30, 152(SP)
|
||||
ld r29, 160(SP)
|
||||
ld r28, 168(SP)
|
||||
ld r27, 176(SP)
|
||||
ld r26, 184(SP)
|
||||
ld r25, 192(SP)
|
||||
ld r24, 200(SP)
|
||||
ld r23, 208(SP)
|
||||
ld r22, 216(SP)
|
||||
ld r21, 224(SP)
|
||||
ld r20, 232(SP)
|
||||
ld r19, 240(SP)
|
||||
ld r18, 248(SP)
|
||||
ld r17, 256(SP)
|
||||
ld r16, 264(SP)
|
||||
ld r15, 272(SP)
|
||||
ld r14, 280(SP)
|
||||
|
||||
ld r0, FLINK_SAVE(SP)
|
||||
|
||||
lxv vs20, 288(SP)
|
||||
lxv vs21, 304(SP)
|
||||
lxv vs22, 320(SP)
|
||||
lxv vs23, 336(SP)
|
||||
lxv vs24, 352(SP)
|
||||
lxv vs25, 368(SP)
|
||||
lxv vs26, 384(SP)
|
||||
lxv vs27, 400(SP)
|
||||
mtlr r0
|
||||
lxv vs28, 416(SP)
|
||||
lxv vs29, 432(SP)
|
||||
lxv vs30, 448(SP)
|
||||
lxv vs31, 464(SP)
|
||||
|
||||
addi SP, SP, STACKSIZE
|
||||
blr
|
||||
|
||||
EPILOGUE
|
||||
#endif
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue