floraachy
/
OpenBLAS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimized axpy kernels for CORTEXA57 

Co-Authored-By: Ralph Campbell <ralph.campbell@broadcom.com>
This commit is contained in:
Ashwin Sekhar T K 2015-10-06 12:12:08 +05:30
parent 3e4acedf0e
commit 2690b71b1f
3 changed files with 515 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
kernel/arm64/KERNEL.CORTEXA57
View File

@ -15,3 +15,8 @@ DASUMKERNEL = asum.S
CASUMKERNEL = casum.S CASUMKERNEL = casum.S
ZASUMKERNEL = zasum.S ZASUMKERNEL = zasum.S
SAXPYKERNEL = axpy.S
DAXPYKERNEL = axpy.S
CAXPYKERNEL = zaxpy.S
ZAXPYKERNEL = zaxpy.S

209
kernel/arm64/axpy.S Normal file
View File

@ -0,0 +1,209 @@
/*******************************************************************************
Copyright (c) 2015, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*******************************************************************************/
#define ASSEMBLER
#include "common.h"
#define N x0 /* vector length */
#define X x3 /* X vector address */
#define INC_X x4 /* X stride */
#define Y x5 /* Y vector address */
#define INC_Y x6 /* Y stride */
#define I x1 /* loop variable */
/*******************************************************************************
* Macro definitions
*******************************************************************************/
#if !defined(DOUBLE)
#define DA s0 /* scale input value */
#define TMPX s1
#define TMPVX {v1.s}[0]
#define TMPY s2
#define TMPVY {v2.s}[0]
#define SZ 4
#else
#define DA d0 /* scale input value */
#define TMPX d1
#define TMPVX {v1.d}[0]
#define TMPY d2
#define TMPVY {v2.d}[0]
#define SZ 8
#endif
/******************************************************************************/
.macro KERNEL_F1
ldr TMPX, [X], #SZ
ldr TMPY, [Y]
fmadd TMPY, TMPX, DA, TMPY
str TMPY, [Y], #SZ
.endm
.macro KERNEL_F4
#if !defined(DOUBLE)
ld1 {v1.4s}, [X], #16
ld1 {v2.4s}, [Y]
fmla v2.4s, v1.4s, v0.s[0]
st1 {v2.4s}, [Y], #16
#else // DOUBLE
ld1 {v1.2d, v2.2d}, [X], #32
ld1 {v3.2d, v4.2d}, [Y]
fmla v3.2d, v1.2d, v0.d[0]
fmla v4.2d, v2.2d, v0.d[0]
st1 {v3.2d, v4.2d}, [Y], #32
#endif
.endm
.macro KERNEL_F8
#if !defined(DOUBLE)
ld1 {v1.4s, v2.4s}, [X], #32
ld1 {v3.4s, v4.4s}, [Y]
fmla v3.4s, v1.4s, v0.s[0]
fmla v4.4s, v2.4s, v0.s[0]
st1 {v3.4s, v4.4s}, [Y], #32
#else // DOUBLE
ld1 {v1.2d, v2.2d, v3.2d, v4.2d}, [X], #64
ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [Y]
fmla v16.2d, v1.2d, v0.d[0]
fmla v17.2d, v2.2d, v0.d[0]
fmla v18.2d, v3.2d, v0.d[0]
fmla v19.2d, v4.2d, v0.d[0]
st1 {v16.2d, v17.2d, v18.2d, v19.2d}, [Y], #64
#endif
PRFM PLDL1KEEP, [X, #512]
PRFM PLDL1KEEP, [Y, #512]
.endm
.macro INIT_S
#if !defined(DOUBLE)
lsl INC_X, INC_X, #2
lsl INC_Y, INC_Y, #2
#else
lsl INC_X, INC_X, #3
lsl INC_Y, INC_Y, #3
#endif
.endm
.macro KERNEL_S1
ld1 TMPVX, [X], INC_X
ldr TMPY, [Y]
fmadd TMPY, TMPX, DA, TMPY
st1 TMPVY, [Y], INC_Y
.endm
/*******************************************************************************
* End of macro definitions
*******************************************************************************/
PROLOGUE
cmp N, xzr
ble axpy_kernel_L999
fcmp DA, #0.0
beq axpy_kernel_L999
cmp INC_X, #1
bne axpy_kernel_S_BEGIN
cmp INC_Y, #1
bne axpy_kernel_S_BEGIN
axpy_kernel_F_BEGIN:
asr I, N, #3
cmp I, xzr
beq axpy_kernel_F1
axpy_kernel_F8:
KERNEL_F8
subs I, I, #1
bne axpy_kernel_F8
axpy_kernel_F1:
ands I, N, #7
ble axpy_kernel_L999
axpy_kernel_F10:
KERNEL_F1
subs I, I, #1
bne axpy_kernel_F10
mov w0, wzr
ret
axpy_kernel_S_BEGIN:
INIT_S
asr I, N, #2
cmp I, xzr
ble axpy_kernel_S1
axpy_kernel_S4:
KERNEL_S1
KERNEL_S1
KERNEL_S1
KERNEL_S1
subs I, I, #1
bne axpy_kernel_S4
axpy_kernel_S1:
ands I, N, #3
ble axpy_kernel_L999
axpy_kernel_S10:
KERNEL_S1
subs I, I, #1
bne axpy_kernel_S10
axpy_kernel_L999:
mov w0, wzr
ret

301
kernel/arm64/zaxpy.S Normal file
View File

@ -0,0 +1,301 @@
/*******************************************************************************
Copyright (c) 2015, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*******************************************************************************/
#define ASSEMBLER
#include "common.h"
#define N x0 /* vector length */
#define X x3 /* X vector address */
#define INC_X x4 /* X stride */
#define Y x5 /* Y vector address */
#define INC_Y x6 /* Y stride */
#define I x1 /* loop variable */
#define Y_COPY x7 /* loop variable */
/*******************************************************************************
* Macro definitions
*******************************************************************************/
#if !defined(DOUBLE)
#define DA_R s0 /* scale input value */
#define DA_I s1 /* scale input value */
#define TMPX v2.2s
#define TMPY v3.2s
#define SZ 4
#else
#define DA_R d0 /* scale input value */
#define DA_I d1 /* scale input value */
#define TMPX v2.2d
#define TMPY v3.2d
#define SZ 8
#endif
/******************************************************************************/
.macro INIT
#if !defined(CONJ)
#if !defined(DOUBLE)
ins v0.s[1], v0.s[0] // v0 = DA_R, DA_R
fneg s2, DA_I
ins v1.s[1], v2.s[0] // v1 = -DA_I, DA_I
ext v1.8b, v1.8b, v1.8b, #4 // v1 = DA_I, -DA_I
#else
ins v0.d[1], v0.d[0] // v0 = DA_R, DA_R
fneg d2, DA_I
ins v1.d[1], v2.d[0] // v1 = -DA_I, DA_I
ext v1.16b, v1.16b, v1.16b, #8 // v1 = DA_I, -DA_I
#endif
#else
#if !defined(DOUBLE)
fneg s2, DA_R
ins v0.s[1], v2.s[0] // v0 = -DA_R, DA_R
ins v1.s[1], v1.s[0] // v1 = DA_I, DA_I
#else
fneg d2, DA_R
ins v0.d[1], v2.d[0] // v0 = -DA_R, DA_R
ins v1.d[1], v1.d[0] // v1 = DA_I, DA_I
#endif
#endif
.endm
.macro KERNEL_F1
#if !defined(DOUBLE)
ld1 {v2.2s}, [X], #8 // V2 = X[ix+1], X[ix]; X += 2
ld1 {v3.2s}, [Y] // V3 = Y[iy+1], Y[iy]
ext v4.8b, v2.8b, v2.8b, #4 // V4 = X[ix], X[ix+1]
fmla v3.2s, v0.2s, v2.2s // Y[iy] += DA_R * X[ix]
// Y[iy+1] += +-DA_R * X[ix+1]
fmla v3.2s, v1.2s, v4.2s // Y[iy] += +-DA_I * X[ix+1]
// Y[iy+1] += DA_I * X[ix]
st1 {v3.2s}, [Y], #8
#else
ld1 {v2.2d}, [X], #16 // V2 = X[ix+1], X[ix]; X += 2
ld1 {v3.2d}, [Y] // V3 = Y[iy+1], Y[iy]
ext v4.16b, v2.16b, v2.16b, #8 // V4 = X[ix], X[ix+1]
fmla v3.2d, v0.2d, v2.2d // Y[iy] += DA_R * X[ix]
// Y[iy+1] += +-DA_R * X[ix+1]
fmla v3.2d, v1.2d, v4.2d // Y[iy] += +-DA_I * X[ix+1]
// Y[iy+1] += DA_I * X[ix]
st1 {v3.2d}, [Y], #16
#endif
.endm
.macro KERNEL_INIT_F4
#if !defined(DOUBLE)
// Replicate the lower 2 floats into the upper 2 slots
ins v0.d[1], v0.d[0] // v0 = DA_R, DA_R, DA_R, DA_R
ins v1.d[1], v1.d[0] // v1 = DA_I, DA_I, DA_I, DA_I
#endif
.endm
.macro KERNEL_F4
#if !defined(DOUBLE)
ld1 {v2.4s,v3.4s}, [X], #32 // V2 = X[3], X[2], X[1], X[0]
// V3 = X[7], X[6], X[5], X[4]
ext v6.8b, v2.8b, v2.8b, #4 // V6 = - , - , X[0], X[1]
ins v6.s[2], v2.s[3] // V6 = - , X[3], X[0], X[1]
ins v6.s[3], v2.s[2] // V6 = X[2], X[3], X[0], X[1]
ld1 {v4.4s,v5.4s}, [Y] // V4 = Y[3], Y[2], Y[1], Y[0]
// V5 = Y[7], Y[6], Y[5], Y[4]
ext v7.8b, v3.8b, v3.8b, #4 // V7 = - , - , X[4], X[5]
ins v7.s[2], v3.s[3] // V7 = - , X[7], X[4], X[5]
ins v7.s[3], v3.s[2] // V7 = X[6], X[7], X[4], X[5]
fmla v4.4s, v0.4s, v2.4s // Y[iy] += DA_R * X[ix]
// Y[iy+1] += +-DA_R * X[ix+1]
fmla v4.4s, v1.4s, v6.4s // Y[iy] += +-DA_I * X[ix+1]
// Y[iy+1] += DA_I * X[ix]
st1 {v4.4s}, [Y], #16
fmla v5.4s, v0.4s, v3.4s // Y[iy] += DA_R * X[ix]
fmla v5.4s, v1.4s, v7.4s // Y[iy] += +-DA_I * X[ix+1]
// Y[iy+1] += +-DA_R * X[ix+1]
// Y[iy+1] += DA_I * X[ix]
st1 {v5.4s}, [Y], #16
#else // DOUBLE
ld1 {v2.2d,v3.2d}, [X], #32 // CX0, CX1, CX2, CX3
ext v20.16b, v2.16b, v2.16b, #8 // X[ix], X[ix+1]
ext v21.16b, v3.16b, v3.16b, #8 // X[ix], X[ix+1]
ld1 {v4.2d,v5.2d}, [X], #32 // CX0, CX1, CX2, CX3
ext v22.16b, v4.16b, v4.16b, #8 // X[ix], X[ix+1]
ext v23.16b, v5.16b, v5.16b, #8 // X[ix], X[ix+1]
ld1 {v16.2d,v17.2d}, [Y_COPY], #32 // CY0, CY1, CY2, CY3
fmla v16.2d, v0.2d, v2.2d
fmla v17.2d, v0.2d, v3.2d
ld1 {v18.2d,v19.2d}, [Y_COPY], #32 // CY0, CY1, CY2, CY3
fmla v16.2d, v1.2d, v20.2d
fmla v17.2d, v1.2d, v21.2d
st1 {v16.2d,v17.2d}, [Y], #32
fmla v18.2d, v0.2d, v4.2d
fmla v19.2d, v0.2d, v5.2d
fmla v18.2d, v1.2d, v22.2d
fmla v19.2d, v1.2d, v23.2d
st1 {v18.2d,v19.2d}, [Y], #32
#endif
PRFM PLDL1KEEP, [X, #512]
PRFM PLDL1KEEP, [Y, #512]
.endm
.macro INIT_S
#if !defined(DOUBLE)
lsl INC_X, INC_X, #3
lsl INC_Y, INC_Y, #3
#else
lsl INC_X, INC_X, #4
lsl INC_Y, INC_Y, #4
#endif
.endm
.macro KERNEL_S1
#if !defined(DOUBLE)
ld1 {v2.2s}, [X], INC_X // V2 = X[ix+1], X[ix]; X += 2
ld1 {v3.2s}, [Y] // V3 = Y[iy+1], Y[iy]
ext v4.8b, v2.8b, v2.8b, #4 // V4 = X[ix], X[ix+1]
fmla v3.2s, v0.2s, v2.2s // Y[iy] += DA_R * X[ix]
// Y[iy+1] += +-DA_R * X[ix+1]
fmla v3.2s, v1.2s, v4.2s // Y[iy] += +-DA_I * X[ix+1]
// Y[iy+1] += DA_I * X[ix]
st1 {v3.2s}, [Y], INC_Y
#else
ld1 {v2.2d}, [X], INC_X // V2 = X[ix+1], X[ix]; X += 2
ld1 {v3.2d}, [Y] // V3 = Y[iy+1], Y[iy]
ext v4.16b, v2.16b, v2.16b, #8 // V4 = X[ix], X[ix+1]
fmla v3.2d, v0.2d, v2.2d // Y[iy] += DA_R * X[ix]
// Y[iy+1] += +-DA_R * X[ix+1]
fmla v3.2d, v1.2d, v4.2d // Y[iy] += +-DA_I * X[ix+1]
// Y[iy+1] += DA_I * X[ix]
st1 {v3.2d}, [Y], INC_Y
#endif
.endm
/*******************************************************************************
* End of macro definitions
*******************************************************************************/
PROLOGUE
cmp N, xzr
ble zaxpy_kernel_L999
mov Y_COPY, Y
fcmp DA_R, #0.0
bne .L1
fcmp DA_I, #0.0
beq zaxpy_kernel_L999
.L1:
INIT
cmp INC_X, #1
bne zaxpy_kernel_S_BEGIN
cmp INC_Y, #1
bne zaxpy_kernel_S_BEGIN
zaxpy_kernel_F_BEGIN:
asr I, N, #2
cmp I, xzr
beq zaxpy_kernel_F1
KERNEL_INIT_F4
zaxpy_kernel_F4:
KERNEL_F4
subs I, I, #1
bne zaxpy_kernel_F4
zaxpy_kernel_F1:
ands I, N, #3
ble zaxpy_kernel_L999
zaxpy_kernel_F10:
KERNEL_F1
subs I, I, #1
bne zaxpy_kernel_F10
mov w0, wzr
ret
zaxpy_kernel_S_BEGIN:
INIT_S
asr I, N, #2
cmp I, xzr
ble zaxpy_kernel_S1
zaxpy_kernel_S4:
KERNEL_S1
KERNEL_S1
KERNEL_S1
KERNEL_S1
subs I, I, #1
bne zaxpy_kernel_S4
zaxpy_kernel_S1:
ands I, N, #3
ble zaxpy_kernel_L999
zaxpy_kernel_S10:
KERNEL_S1
subs I, I, #1
bne zaxpy_kernel_S10
zaxpy_kernel_L999:
mov w0, wzr
ret