floraachy
/
OpenBLAS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove prefetches from SVE kernels 

This is a precursor to enabling the SVE kernels for Arm(R) Neoverse(TM)
V1 which has 256-bit SVE. Testing revealed that the SVE kernel was
actually worse in some cases than the existing kernel which seemed odd -
removing these prefetches the underlying architecture seems to do a better job
😸
This commit is contained in:
Chris Sidebottom 2022-12-16 14:36:42 +00:00
parent 042e3c0e7c
commit 1361229291
2 changed files with 0 additions and 226 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

113
kernel/arm64/dgemm_kernel_sve_v2x8.S
View File

@ -189,20 +189,16 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
ld1rd z11.d, p0/z, [pB, 24]
fmla z24.d, p0/m, z0.d, z12.d
fmla z25.d, p0/m, z1.d, z12.d
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE]
ld1rd z12.d, p0/z, [pB, 32]
fmla z26.d, p0/m, z0.d, z13.d
fmla z27.d, p0/m, z1.d, z13.d
prfm PLDL1KEEP, [pA2, #A_PRE_SIZE]
ld1rd z13.d, p0/z, [pB, 40]
fmla z28.d, p0/m, z0.d, z14.d
fmla z29.d, p0/m, z1.d, z14.d
ld1rd z14.d, p0/z, [pB, 48]
fmla z30.d, p0/m, z0.d, z15.d
fmla z31.d, p0/m, z1.d, z15.d
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE+64]
ld1rd z15.d, p0/z, [pB, 56]
prfm PLDL1KEEP, [pA2, #A_PRE_SIZE+64]
add pB, pB, 64
.endm
@ -227,19 +223,15 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
ld1rd z11.d, p0/z, [pB, 24]
fmla z24.d, p0/m, z0.d, z12.d
fmla z25.d, p0/m, z1.d, z12.d
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE]
ld1rd z12.d, p0/z, [pB, 32]
fmla z26.d, p0/m, z0.d, z13.d
fmla z27.d, p0/m, z1.d, z13.d
prfm PLDL1KEEP, [pA2, #A_PRE_SIZE]
ld1rd z13.d, p0/z, [pB, 40]
fmla z28.d, p0/m, z0.d, z14.d
fmla z29.d, p0/m, z1.d, z14.d
ld1rd z14.d, p0/z, [pB, 48]
fmla z30.d, p0/m, z0.d, z15.d
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE+64]
fmla z31.d, p0/m, z1.d, z15.d
prfm PLDL1KEEP, [pA2, #A_PRE_SIZE+64]
ld1rd z15.d, p0/z, [pB, 56]
add pB, pB, 64
@ -265,7 +257,6 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
ld1rd z11.d, p0/z, [pB, 24]
fmla z24.d, p0/m, z2.d, z12.d
fmla z25.d, p0/m, z3.d, z12.d
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
ld1rd z12.d, p0/z, [pB, 32]
fmla z26.d, p0/m, z2.d, z13.d
fmla z27.d, p0/m, z3.d, z13.d
@ -291,7 +282,6 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
fmla z23.d, p0/m, z3.d, z11.d
fmla z24.d, p0/m, z2.d, z12.d
fmla z25.d, p0/m, z3.d, z12.d
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
fmla z26.d, p0/m, z2.d, z13.d
fmla z27.d, p0/m, z3.d, z13.d
fmla z28.d, p0/m, z2.d, z14.d
@ -322,25 +312,21 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
fmla z18.d, p0/m, z0.d, z9.d
fmla z19.d, p0/m, z1.d, z9.d
fmla z20.d, p0/m, z0.d, z10.d
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE]
fmla z21.d, p0/m, z1.d, z10.d
fmla z22.d, p0/m, z0.d, z11.d
fmla z23.d, p0/m, z1.d, z11.d
fmla z24.d, p0/m, z0.d, z12.d
prfm PLDL1KEEP, [pA2, #A_PRE_SIZE]
fmla z25.d, p0/m, z1.d, z12.d
fmla z26.d, p0/m, z0.d, z13.d
fmla z27.d, p0/m, z1.d, z13.d
fmla z28.d, p0/m, z0.d, z14.d
fmla z29.d, p0/m, z1.d, z14.d
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
fmla z30.d, p0/m, z0.d, z15.d
fmla z31.d, p0/m, z1.d, z15.d
.endm
.macro SAVEv2x8
prfm PLDL2KEEP, [pCRow0, #C_PRE_SIZE]
add pCRow1, pCRow0, LDC
ld1d z8.d, p0/z, [pCRow0]
@ -349,7 +335,6 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
fmla z9.d, p0/m, z17.d, alphaZ
st1d z8.d, p0, [pCRow0]
st1d z9.d, p0, [pCRow0, #1, mul vl]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
add pCRow2, pCRow1, LDC
ld1d z10.d, p0/z, [pCRow1]
@ -358,7 +343,6 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
fmla z11.d, p0/m, z19.d, alphaZ
st1d z10.d, p0, [pCRow1]
st1d z11.d, p0, [pCRow1, #1, mul vl]
prfm PLDL2KEEP, [pCRow2, #C_PRE_SIZE]
add pCRow1, pCRow2, LDC
ld1d z12.d, p0/z, [pCRow2]
@ -367,7 +351,6 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
fmla z13.d, p0/m, z21.d, alphaZ
st1d z12.d, p0, [pCRow2]
st1d z13.d, p0, [pCRow2, #1, mul vl]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
add pCRow2, pCRow1, LDC
ld1d z14.d, p0/z, [pCRow1]
@ -376,7 +359,6 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
fmla z15.d, p0/m, z23.d, alphaZ
st1d z14.d, p0, [pCRow1]
st1d z15.d, p0, [pCRow1, #1, mul vl]
prfm PLDL2KEEP, [pCRow2, #C_PRE_SIZE]
add pCRow1, pCRow2, LDC
ld1d z8.d, p0/z, [pCRow2]
@ -385,7 +367,6 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
fmla z9.d, p0/m, z25.d, alphaZ
st1d z8.d, p0, [pCRow2]
st1d z9.d, p0, [pCRow2, #1, mul vl]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
add pCRow2, pCRow1, LDC
ld1d z10.d, p0/z, [pCRow1]
@ -394,7 +375,6 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
fmla z11.d, p0/m, z27.d, alphaZ
st1d z10.d, p0, [pCRow1]
st1d z11.d, p0, [pCRow1, #1, mul vl]
prfm PLDL2KEEP, [pCRow2, #C_PRE_SIZE]
add pCRow1, pCRow2, LDC
ld1d z12.d, p0/z, [pCRow2]
@ -403,7 +383,6 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
fmla z13.d, p0/m, z29.d, alphaZ
st1d z12.d, p0, [pCRow2]
st1d z13.d, p0, [pCRow2, #1, mul vl]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
ld1d z14.d, p0/z, [pCRow1]
ld1d z15.d, p0/z, [pCRow1, #1, mul vl]
@ -443,10 +422,8 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
fmla z16.d, p0/m, z0.d, z8.d
fmla z17.d, p0/m, z1.d, z8.d
fmla z18.d, p0/m, z0.d, z9.d
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE]
fmla z19.d, p0/m, z1.d, z9.d
fmla z20.d, p0/m, z0.d, z10.d
prfm PLDL1KEEP, [pA2, #A_PRE_SIZE]
fmla z21.d, p0/m, z1.d, z10.d
fmla z22.d, p0/m, z0.d, z11.d
fmla z23.d, p0/m, z1.d, z11.d
@ -454,7 +431,6 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
.macro SAVEv2x4
prfm PLDL2KEEP, [pCRow0, #C_PRE_SIZE]
add pCRow1, pCRow0, LDC
ld1d z8.d, p0/z, [pCRow0]
@ -463,7 +439,6 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
fmla z9.d, p0/m, z17.d, alphaZ
st1d z8.d, p0, [pCRow0]
st1d z9.d, p0, [pCRow0, #1, mul vl]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
add pCRow2, pCRow1, LDC
ld1d z10.d, p0/z, [pCRow1]
@ -472,7 +447,6 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
fmla z11.d, p0/m, z19.d, alphaZ
st1d z10.d, p0, [pCRow1]
st1d z11.d, p0, [pCRow1, #1, mul vl]
prfm PLDL2KEEP, [pCRow2, #C_PRE_SIZE]
add pCRow1, pCRow2, LDC
ld1d z12.d, p0/z, [pCRow2]
@ -481,7 +455,6 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
fmla z13.d, p0/m, z21.d, alphaZ
st1d z12.d, p0, [pCRow2]
st1d z13.d, p0, [pCRow2, #1, mul vl]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
ld1d z14.d, p0/z, [pCRow1]
ld1d z15.d, p0/z, [pCRow1, #1, mul vl]
@ -514,15 +487,12 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
fmla z16.d, p0/m, z0.d, z8.d
fmla z17.d, p0/m, z1.d, z8.d
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE]
fmla z18.d, p0/m, z0.d, z9.d
fmla z19.d, p0/m, z1.d, z9.d
prfm PLDL1KEEP, [pA2, #A_PRE_SIZE]
.endm
.macro SAVEv2x2
prfm PLDL2KEEP, [pCRow0, #C_PRE_SIZE]
add pCRow1, pCRow0, LDC
ld1d z8.d, p0/z, [pCRow0]
@ -531,7 +501,6 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
fmla z9.d, p0/m, z17.d, alphaZ
st1d z8.d, p0, [pCRow0]
st1d z9.d, p0, [pCRow0, #1, mul vl]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
ld1d z10.d, p0/z, [pCRow1]
ld1d z11.d, p0/z, [pCRow1, #1, mul vl]
@ -539,7 +508,6 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
fmla z11.d, p0/m, z19.d, alphaZ
st1d z10.d, p0, [pCRow1]
st1d z11.d, p0, [pCRow1, #1, mul vl]
prfm PLDL2KEEP, [pCRow2, #C_PRE_SIZE]
@ -563,12 +531,10 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
fmla z16.d, p0/m, z0.d, z8.d
fmla z17.d, p0/m, z1.d, z8.d
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE]
.endm
.macro SAVEv2x1
prfm PLDL2KEEP, [pCRow0, #C_PRE_SIZE]
add pCRow1, pCRow0, LDC
ld1d z8.d, p0/z, [pCRow0]
@ -618,14 +584,12 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
fmla z19.d, p1/m, z0.d, z11.d
ld1rd z11.d, p0/z, [pB, 24]
fmla z20.d, p1/m, z0.d, z12.d
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE]
ld1rd z12.d, p0/z, [pB, 32]
fmla z21.d, p1/m, z0.d, z13.d
ld1rd z13.d, p0/z, [pB, 40]
fmla z22.d, p1/m, z0.d, z14.d
ld1rd z14.d, p0/z, [pB, 48]
fmla z23.d, p1/m, z0.d, z15.d
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE+64]
ld1rd z15.d, p0/z, [pB, 56]
add pB, pB, 64
@ -644,14 +608,12 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
fmla z19.d, p1/m, z0.d, z11.d
ld1rd z11.d, p0/z, [pB, 24]
fmla z20.d, p1/m, z0.d, z12.d
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE]
ld1rd z12.d, p0/z, [pB, 32]
fmla z21.d, p1/m, z0.d, z13.d
ld1rd z13.d, p0/z, [pB, 40]
fmla z22.d, p1/m, z0.d, z14.d
ld1rd z14.d, p0/z, [pB, 48]
fmla z23.d, p1/m, z0.d, z15.d
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE+64]
ld1rd z15.d, p0/z, [pB, 56]
add pB, pB, 64
@ -671,7 +633,6 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
ld1rd z11.d, p0/z, [pB, 24]
fmla z20.d, p1/m, z1.d, z12.d
ld1rd z12.d, p0/z, [pB, 32]
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
fmla z21.d, p1/m, z1.d, z13.d
ld1rd z13.d, p0/z, [pB, 40]
fmla z22.d, p1/m, z1.d, z14.d
@ -688,7 +649,6 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
fmla z18.d, p1/m, z1.d, z10.d
fmla z19.d, p1/m, z1.d, z11.d
fmla z20.d, p1/m, z1.d, z12.d
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
fmla z21.d, p1/m, z1.d, z13.d
fmla z22.d, p1/m, z1.d, z14.d
fmla z23.d, p1/m, z1.d, z15.d
@ -712,11 +672,9 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
fmla z16.d, p1/m, z0.d, z8.d
fmla z17.d, p1/m, z0.d, z9.d
fmla z18.d, p1/m, z0.d, z10.d
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE]
fmla z19.d, p1/m, z0.d, z11.d
fmla z20.d, p1/m, z0.d, z12.d
fmla z21.d, p1/m, z0.d, z13.d
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
fmla z22.d, p1/m, z0.d, z14.d
fmla z23.d, p1/m, z0.d, z15.d
@ -725,49 +683,41 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
.macro SAVEv1x8
prfm PLDL2KEEP, [pCRow0, #C_PRE_SIZE]
add pCRow1, pCRow0, LDC
ld1d z24.d, p1/z, [pCRow0]
fmla z24.d, p1/m, z16.d, alphaZ
st1d z24.d, p1, [pCRow0]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
add pCRow2, pCRow1, LDC
ld1d z25.d, p1/z, [pCRow1]
fmla z25.d, p1/m, z17.d, alphaZ
st1d z25.d, p1, [pCRow1]
prfm PLDL2KEEP, [pCRow2, #C_PRE_SIZE]
add pCRow1, pCRow2, LDC
ld1d z26.d, p1/z, [pCRow2]
fmla z26.d, p1/m, z18.d, alphaZ
st1d z26.d, p1, [pCRow2]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
add pCRow2, pCRow1, LDC
ld1d z27.d, p1/z, [pCRow1]
fmla z27.d, p1/m, z19.d, alphaZ
st1d z27.d, p1, [pCRow1]
prfm PLDL2KEEP, [pCRow2, #C_PRE_SIZE]
add pCRow1, pCRow2, LDC
ld1d z28.d, p1/z, [pCRow2]
fmla z28.d, p1/m, z20.d, alphaZ
st1d z28.d, p1, [pCRow2]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
add pCRow2, pCRow1, LDC
ld1d z29.d, p1/z, [pCRow1]
fmla z29.d, p1/m, z21.d, alphaZ
st1d z29.d, p1, [pCRow1]
prfm PLDL2KEEP, [pCRow2, #C_PRE_SIZE]
add pCRow1, pCRow2, LDC
ld1d z30.d, p1/z, [pCRow2]
fmla z30.d, p1/m, z22.d, alphaZ
st1d z30.d, p1, [pCRow2]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
ld1d z31.d, p1/z, [pCRow1]
fmla z31.d, p1/m, z23.d, alphaZ
@ -799,7 +749,6 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
fmla z16.d, p1/m, z0.d, z8.d
fmla z17.d, p1/m, z0.d, z9.d
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE]
fmla z18.d, p1/m, z0.d, z10.d
fmla z19.d, p1/m, z0.d, z11.d
@ -807,25 +756,21 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
.macro SAVEv1x4
prfm PLDL2KEEP, [pCRow0, #C_PRE_SIZE]
add pCRow1, pCRow0, LDC
ld1d z24.d, p1/z, [pCRow0]
fmla z24.d, p1/m, z16.d, alphaZ
st1d z24.d, p1, [pCRow0]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
add pCRow2, pCRow1, LDC
ld1d z25.d, p1/z, [pCRow1]
fmla z25.d, p1/m, z17.d, alphaZ
st1d z25.d, p1, [pCRow1]
prfm PLDL2KEEP, [pCRow2, #C_PRE_SIZE]
add pCRow1, pCRow2, LDC
ld1d z26.d, p1/z, [pCRow2]
fmla z26.d, p1/m, z18.d, alphaZ
st1d z26.d, p1, [pCRow2]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
ld1d z27.d, p1/z, [pCRow1]
fmla z27.d, p1/m, z19.d, alphaZ
@ -852,20 +797,17 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
add pB, pB, 16
fmla z16.d, p1/m, z0.d, z8.d
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE]
fmla z17.d, p1/m, z0.d, z9.d
.endm
.macro SAVEv1x2
prfm PLDL2KEEP, [pCRow0, #C_PRE_SIZE]
add pCRow1, pCRow0, LDC
ld1d z24.d, p1/z, [pCRow0]
fmla z24.d, p1/m, z16.d, alphaZ
st1d z24.d, p1, [pCRow0]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
ld1d z25.d, p1/z, [pCRow1]
fmla z25.d, p1/m, z17.d, alphaZ
@ -890,13 +832,11 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
add pB, pB, 8
fmla z16.d, p1/m, z0.d, z8.d
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE]
.endm
.macro SAVEv1x1
prfm PLDL2KEEP, [pCRow0, #C_PRE_SIZE]
ld1d z24.d, p1/z, [pCRow0]
fmla z24.d, p1/m, z16.d, alphaZ
@ -928,8 +868,6 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
stp x26, x27, [sp, #(9 * 16)]
str x28, [sp, #(10 * 16)]
prfm PLDL1KEEP, [origPB]
prfm PLDL1KEEP, [origPA]
fmov alpha, d0
dup alphaZ, alpha
@ -968,7 +906,6 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
/* Until we have at least 2*SVE_LEN iters left in M, we do them with V2*8 kernel */
mul temp, vec_len, origK // generate address of pA2
add pA2, pA1, temp, lsl #3 // pA1 = start of A array
prfm PLDL1KEEP, [pA2]
.align 5
.Ldgemm_kernel_L8_Mv2_20:
@ -1057,11 +994,6 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
bne .Ldgemm_kernel_L8_Mv2_46
.Ldgemm_kernel_L8_Mv2_100:
prfm PLDL1KEEP, [pA1]
prfm PLDL1KEEP, [pA1, #64]
prfm PLDL1KEEP, [pA2]
prfm PLDL1KEEP, [pA2, #64]
prfm PLDL1KEEP, [origPB]
SAVEv2x8
mov pA1, pA2 // pA1 = pA2
@ -1171,9 +1103,6 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
bne .Ldgemm_kernel_L8_Mv1_46
.Ldgemm_kernel_L8_Mv1_100:
prfm PLDL1KEEP, [pA1]
prfm PLDL1KEEP, [pA1, #64]
prfm PLDL1KEEP, [origPB]
SAVEv1x8
@ -1233,16 +1162,12 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
.align 5
.Ldgemm_kernel_L4_Mv2_22:
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv2x4_SUB
KERNELv2x4_SUB
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv2x4_SUB
KERNELv2x4_SUB
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv2x4_SUB
KERNELv2x4_SUB
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv2x4_SUB
KERNELv2x4_SUB
@ -1257,18 +1182,12 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
.align 5
.Ldgemm_kernel_L4_Mv2_46:
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv2x4_SUB
subs counterL, counterL, #1
bne .Ldgemm_kernel_L4_Mv2_46
.Ldgemm_kernel_L4_Mv2_100:
prfm PLDL1KEEP, [pA1]
prfm PLDL1KEEP, [pA1, #64]
prfm PLDL1KEEP, [pA2]
prfm PLDL1KEEP, [pA2, #64]
prfm PLDL1KEEP, [origPB]
SAVEv2x4
mov pA1, pA2 // pA1 = pA2
@ -1304,16 +1223,12 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
.align 5
.Ldgemm_kernel_L4_Mv1_22:
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv1x4_SUB
KERNELv1x4_SUB
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv1x4_SUB
KERNELv1x4_SUB
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv1x4_SUB
KERNELv1x4_SUB
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv1x4_SUB
KERNELv1x4_SUB
@ -1328,16 +1243,12 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
.align 5
.Ldgemm_kernel_L4_Mv1_46:
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv1x4_SUB
subs counterL, counterL, #1
bne .Ldgemm_kernel_L4_Mv1_46
.Ldgemm_kernel_L4_Mv1_100:
prfm PLDL1KEEP, [pA1]
prfm PLDL1KEEP, [pA1, #64]
prfm PLDL1KEEP, [origPB]
SAVEv1x4
@ -1393,12 +1304,10 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
.align 5
.Ldgemm_kernel_L2_Mv2_22:
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv2x2_SUB
KERNELv2x2_SUB
KERNELv2x2_SUB
KERNELv2x2_SUB
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv2x2_SUB
KERNELv2x2_SUB
KERNELv2x2_SUB
@ -1415,18 +1324,12 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
.align 5
.Ldgemm_kernel_L2_Mv2_46:
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv2x2_SUB
subs counterL, counterL, #1
bne .Ldgemm_kernel_L2_Mv2_46
.Ldgemm_kernel_L2_Mv2_100:
prfm PLDL1KEEP, [pA1]
prfm PLDL1KEEP, [pA1, #64]
prfm PLDL1KEEP, [pA2]
prfm PLDL1KEEP, [pA2, #64]
prfm PLDL1KEEP, [origPB]
SAVEv2x2
mov pA1, pA2 // pA1 = pA2
@ -1463,12 +1366,10 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
.align 5
.Ldgemm_kernel_L2_Mv1_22:
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv1x2_SUB
KERNELv1x2_SUB
KERNELv1x2_SUB
KERNELv1x2_SUB
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv1x2_SUB
KERNELv1x2_SUB
KERNELv1x2_SUB
@ -1485,16 +1386,12 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
.align 5
.Ldgemm_kernel_L2_Mv1_46:
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv1x2_SUB
subs counterL, counterL, #1
bne .Ldgemm_kernel_L2_Mv1_46
.Ldgemm_kernel_L2_Mv1_100:
prfm PLDL1KEEP, [pA1]
prfm PLDL1KEEP, [pA1, #64]
prfm PLDL1KEEP, [origPB]
SAVEv1x2
@ -1550,7 +1447,6 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
.align 5
.Ldgemm_kernel_L1_Mv2_22:
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv2x1_SUB
KERNELv2x1_SUB
KERNELv2x1_SUB
@ -1571,16 +1467,12 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
.align 5
.Ldgemm_kernel_L1_Mv2_46:
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv2x1_SUB
subs counterL, counterL, #1
bgt .Ldgemm_kernel_L1_Mv2_46
.Ldgemm_kernel_L1_Mv2_100:
prfm PLDL1KEEP, [pA1]
prfm PLDL1KEEP, [pA1, #64]
prfm PLDL1KEEP, [origPB]
SAVEv2x1
mov pA1, pA2 // pA1 = pA2
@ -1617,7 +1509,6 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
.align 5
.Ldgemm_kernel_L1_Mv1_22:
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv1x1_SUB
KERNELv1x1_SUB
KERNELv1x1_SUB
@ -1638,16 +1529,12 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
.align 5
.Ldgemm_kernel_L1_Mv1_46:
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv1x1_SUB
subs counterL, counterL, #1
bgt .Ldgemm_kernel_L1_Mv1_46
.Ldgemm_kernel_L1_Mv1_100:
prfm PLDL1KEEP, [pA1]
prfm PLDL1KEEP, [pA1, #64]
prfm PLDL1KEEP, [origPB]
SAVEv1x1

113
kernel/arm64/sgemm_kernel_sve_v2x8.S
View File

@ -189,20 +189,16 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
ld1rw z11.s, p0/z, [pB, 12]
fmla z24.s, p0/m, z0.s, z12.s
fmla z25.s, p0/m, z1.s, z12.s
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE]
ld1rw z12.s, p0/z, [pB, 16]
fmla z26.s, p0/m, z0.s, z13.s
fmla z27.s, p0/m, z1.s, z13.s
prfm PLDL1KEEP, [pA2, #A_PRE_SIZE]
ld1rw z13.s, p0/z, [pB, 20]
fmla z28.s, p0/m, z0.s, z14.s
fmla z29.s, p0/m, z1.s, z14.s
ld1rw z14.s, p0/z, [pB, 24]
fmla z30.s, p0/m, z0.s, z15.s
fmla z31.s, p0/m, z1.s, z15.s
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE+64]
ld1rw z15.s, p0/z, [pB, 28]
prfm PLDL1KEEP, [pA2, #A_PRE_SIZE+64]
add pB, pB, 32
.endm
@ -227,19 +223,15 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
ld1rw z11.s, p0/z, [pB, 12]
fmla z24.s, p0/m, z0.s, z12.s
fmla z25.s, p0/m, z1.s, z12.s
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE]
ld1rw z12.s, p0/z, [pB, 16]
fmla z26.s, p0/m, z0.s, z13.s
fmla z27.s, p0/m, z1.s, z13.s
prfm PLDL1KEEP, [pA2, #A_PRE_SIZE]
ld1rw z13.s, p0/z, [pB, 20]
fmla z28.s, p0/m, z0.s, z14.s
fmla z29.s, p0/m, z1.s, z14.s
ld1rw z14.s, p0/z, [pB, 24]
fmla z30.s, p0/m, z0.s, z15.s
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE+64]
fmla z31.s, p0/m, z1.s, z15.s
prfm PLDL1KEEP, [pA2, #A_PRE_SIZE+64]
ld1rw z15.s, p0/z, [pB, 28]
add pB, pB, 32
@ -265,7 +257,6 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
ld1rw z11.s, p0/z, [pB, 12]
fmla z24.s, p0/m, z2.s, z12.s
fmla z25.s, p0/m, z3.s, z12.s
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
ld1rw z12.s, p0/z, [pB, 16]
fmla z26.s, p0/m, z2.s, z13.s
fmla z27.s, p0/m, z3.s, z13.s
@ -291,7 +282,6 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
fmla z23.s, p0/m, z3.s, z11.s
fmla z24.s, p0/m, z2.s, z12.s
fmla z25.s, p0/m, z3.s, z12.s
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
fmla z26.s, p0/m, z2.s, z13.s
fmla z27.s, p0/m, z3.s, z13.s
fmla z28.s, p0/m, z2.s, z14.s
@ -322,25 +312,21 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
fmla z18.s, p0/m, z0.s, z9.s
fmla z19.s, p0/m, z1.s, z9.s
fmla z20.s, p0/m, z0.s, z10.s
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE]
fmla z21.s, p0/m, z1.s, z10.s
fmla z22.s, p0/m, z0.s, z11.s
fmla z23.s, p0/m, z1.s, z11.s
fmla z24.s, p0/m, z0.s, z12.s
prfm PLDL1KEEP, [pA2, #A_PRE_SIZE]
fmla z25.s, p0/m, z1.s, z12.s
fmla z26.s, p0/m, z0.s, z13.s
fmla z27.s, p0/m, z1.s, z13.s
fmla z28.s, p0/m, z0.s, z14.s
fmla z29.s, p0/m, z1.s, z14.s
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
fmla z30.s, p0/m, z0.s, z15.s
fmla z31.s, p0/m, z1.s, z15.s
.endm
.macro SAVEv2x8
prfm PLDL2KEEP, [pCRow0, #C_PRE_SIZE]
add pCRow1, pCRow0, LDC
ld1w z8.s, p0/z, [pCRow0]
@ -349,7 +335,6 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
fmla z9.s, p0/m, z17.s, alphaZ
st1w z8.s, p0, [pCRow0]
st1w z9.s, p0, [pCRow0, #1, mul vl]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
add pCRow2, pCRow1, LDC
ld1w z10.s, p0/z, [pCRow1]
@ -358,7 +343,6 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
fmla z11.s, p0/m, z19.s, alphaZ
st1w z10.s, p0, [pCRow1]
st1w z11.s, p0, [pCRow1, #1, mul vl]
prfm PLDL2KEEP, [pCRow2, #C_PRE_SIZE]
add pCRow1, pCRow2, LDC
ld1w z12.s, p0/z, [pCRow2]
@ -367,7 +351,6 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
fmla z13.s, p0/m, z21.s, alphaZ
st1w z12.s, p0, [pCRow2]
st1w z13.s, p0, [pCRow2, #1, mul vl]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
add pCRow2, pCRow1, LDC
ld1w z14.s, p0/z, [pCRow1]
@ -376,7 +359,6 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
fmla z15.s, p0/m, z23.s, alphaZ
st1w z14.s, p0, [pCRow1]
st1w z15.s, p0, [pCRow1, #1, mul vl]
prfm PLDL2KEEP, [pCRow2, #C_PRE_SIZE]
add pCRow1, pCRow2, LDC
ld1w z8.s, p0/z, [pCRow2]
@ -385,7 +367,6 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
fmla z9.s, p0/m, z25.s, alphaZ
st1w z8.s, p0, [pCRow2]
st1w z9.s, p0, [pCRow2, #1, mul vl]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
add pCRow2, pCRow1, LDC
ld1w z10.s, p0/z, [pCRow1]
@ -394,7 +375,6 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
fmla z11.s, p0/m, z27.s, alphaZ
st1w z10.s, p0, [pCRow1]
st1w z11.s, p0, [pCRow1, #1, mul vl]
prfm PLDL2KEEP, [pCRow2, #C_PRE_SIZE]
add pCRow1, pCRow2, LDC
ld1w z12.s, p0/z, [pCRow2]
@ -403,7 +383,6 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
fmla z13.s, p0/m, z29.s, alphaZ
st1w z12.s, p0, [pCRow2]
st1w z13.s, p0, [pCRow2, #1, mul vl]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
ld1w z14.s, p0/z, [pCRow1]
ld1w z15.s, p0/z, [pCRow1, #1, mul vl]
@ -443,10 +422,8 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
fmla z16.s, p0/m, z0.s, z8.s
fmla z17.s, p0/m, z1.s, z8.s
fmla z18.s, p0/m, z0.s, z9.s
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE]
fmla z19.s, p0/m, z1.s, z9.s
fmla z20.s, p0/m, z0.s, z10.s
prfm PLDL1KEEP, [pA2, #A_PRE_SIZE]
fmla z21.s, p0/m, z1.s, z10.s
fmla z22.s, p0/m, z0.s, z11.s
fmla z23.s, p0/m, z1.s, z11.s
@ -454,7 +431,6 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
.macro SAVEv2x4
prfm PLDL2KEEP, [pCRow0, #C_PRE_SIZE]
add pCRow1, pCRow0, LDC
ld1w z8.s, p0/z, [pCRow0]
@ -463,7 +439,6 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
fmla z9.s, p0/m, z17.s, alphaZ
st1w z8.s, p0, [pCRow0]
st1w z9.s, p0, [pCRow0, #1, mul vl]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
add pCRow2, pCRow1, LDC
ld1w z10.s, p0/z, [pCRow1]
@ -472,7 +447,6 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
fmla z11.s, p0/m, z19.s, alphaZ
st1w z10.s, p0, [pCRow1]
st1w z11.s, p0, [pCRow1, #1, mul vl]
prfm PLDL2KEEP, [pCRow2, #C_PRE_SIZE]
add pCRow1, pCRow2, LDC
ld1w z12.s, p0/z, [pCRow2]
@ -481,7 +455,6 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
fmla z13.s, p0/m, z21.s, alphaZ
st1w z12.s, p0, [pCRow2]
st1w z13.s, p0, [pCRow2, #1, mul vl]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
ld1w z14.s, p0/z, [pCRow1]
ld1w z15.s, p0/z, [pCRow1, #1, mul vl]
@ -514,15 +487,12 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
fmla z16.s, p0/m, z0.s, z8.s
fmla z17.s, p0/m, z1.s, z8.s
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE]
fmla z18.s, p0/m, z0.s, z9.s
fmla z19.s, p0/m, z1.s, z9.s
prfm PLDL1KEEP, [pA2, #A_PRE_SIZE]
.endm
.macro SAVEv2x2
prfm PLDL2KEEP, [pCRow0, #C_PRE_SIZE]
add pCRow1, pCRow0, LDC
ld1w z8.s, p0/z, [pCRow0]
@ -531,7 +501,6 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
fmla z9.s, p0/m, z17.s, alphaZ
st1w z8.s, p0, [pCRow0]
st1w z9.s, p0, [pCRow0, #1, mul vl]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
ld1w z10.s, p0/z, [pCRow1]
ld1w z11.s, p0/z, [pCRow1, #1, mul vl]
@ -539,7 +508,6 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
fmla z11.s, p0/m, z19.s, alphaZ
st1w z10.s, p0, [pCRow1]
st1w z11.s, p0, [pCRow1, #1, mul vl]
prfm PLDL2KEEP, [pCRow2, #C_PRE_SIZE]
@ -563,12 +531,10 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
fmla z16.s, p0/m, z0.s, z8.s
fmla z17.s, p0/m, z1.s, z8.s
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE]
.endm
.macro SAVEv2x1
prfm PLDL2KEEP, [pCRow0, #C_PRE_SIZE]
add pCRow1, pCRow0, LDC
ld1w z8.s, p0/z, [pCRow0]
@ -618,14 +584,12 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
fmla z19.s, p1/m, z0.s, z11.s
ld1rw z11.s, p0/z, [pB, 12]
fmla z20.s, p1/m, z0.s, z12.s
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE]
ld1rw z12.s, p0/z, [pB, 16]
fmla z21.s, p1/m, z0.s, z13.s
ld1rw z13.s, p0/z, [pB, 20]
fmla z22.s, p1/m, z0.s, z14.s
ld1rw z14.s, p0/z, [pB, 24]
fmla z23.s, p1/m, z0.s, z15.s
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE+64]
ld1rw z15.s, p0/z, [pB, 28]
add pB, pB, 32
@ -644,14 +608,12 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
fmla z19.s, p1/m, z0.s, z11.s
ld1rw z11.s, p0/z, [pB, 12]
fmla z20.s, p1/m, z0.s, z12.s
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE]
ld1rw z12.s, p0/z, [pB, 16]
fmla z21.s, p1/m, z0.s, z13.s
ld1rw z13.s, p0/z, [pB, 20]
fmla z22.s, p1/m, z0.s, z14.s
ld1rw z14.s, p0/z, [pB, 24]
fmla z23.s, p1/m, z0.s, z15.s
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE+64]
ld1rw z15.s, p0/z, [pB, 28]
add pB, pB, 32
@ -671,7 +633,6 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
ld1rw z11.s, p0/z, [pB, 12]
fmla z20.s, p1/m, z1.s, z12.s
ld1rw z12.s, p0/z, [pB, 16]
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
fmla z21.s, p1/m, z1.s, z13.s
ld1rw z13.s, p0/z, [pB, 20]
fmla z22.s, p1/m, z1.s, z14.s
@ -688,7 +649,6 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
fmla z18.s, p1/m, z1.s, z10.s
fmla z19.s, p1/m, z1.s, z11.s
fmla z20.s, p1/m, z1.s, z12.s
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
fmla z21.s, p1/m, z1.s, z13.s
fmla z22.s, p1/m, z1.s, z14.s
fmla z23.s, p1/m, z1.s, z15.s
@ -712,11 +672,9 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
fmla z16.s, p1/m, z0.s, z8.s
fmla z17.s, p1/m, z0.s, z9.s
fmla z18.s, p1/m, z0.s, z10.s
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE]
fmla z19.s, p1/m, z0.s, z11.s
fmla z20.s, p1/m, z0.s, z12.s
fmla z21.s, p1/m, z0.s, z13.s
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
fmla z22.s, p1/m, z0.s, z14.s
fmla z23.s, p1/m, z0.s, z15.s
@ -725,49 +683,41 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
.macro SAVEv1x8
prfm PLDL2KEEP, [pCRow0, #C_PRE_SIZE]
add pCRow1, pCRow0, LDC
ld1w z24.s, p1/z, [pCRow0]
fmla z24.s, p1/m, z16.s, alphaZ
st1w z24.s, p1, [pCRow0]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
add pCRow2, pCRow1, LDC
ld1w z25.s, p1/z, [pCRow1]
fmla z25.s, p1/m, z17.s, alphaZ
st1w z25.s, p1, [pCRow1]
prfm PLDL2KEEP, [pCRow2, #C_PRE_SIZE]
add pCRow1, pCRow2, LDC
ld1w z26.s, p1/z, [pCRow2]
fmla z26.s, p1/m, z18.s, alphaZ
st1w z26.s, p1, [pCRow2]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
add pCRow2, pCRow1, LDC
ld1w z27.s, p1/z, [pCRow1]
fmla z27.s, p1/m, z19.s, alphaZ
st1w z27.s, p1, [pCRow1]
prfm PLDL2KEEP, [pCRow2, #C_PRE_SIZE]
add pCRow1, pCRow2, LDC
ld1w z28.s, p1/z, [pCRow2]
fmla z28.s, p1/m, z20.s, alphaZ
st1w z28.s, p1, [pCRow2]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
add pCRow2, pCRow1, LDC
ld1w z29.s, p1/z, [pCRow1]
fmla z29.s, p1/m, z21.s, alphaZ
st1w z29.s, p1, [pCRow1]
prfm PLDL2KEEP, [pCRow2, #C_PRE_SIZE]
add pCRow1, pCRow2, LDC
ld1w z30.s, p1/z, [pCRow2]
fmla z30.s, p1/m, z22.s, alphaZ
st1w z30.s, p1, [pCRow2]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
ld1w z31.s, p1/z, [pCRow1]
fmla z31.s, p1/m, z23.s, alphaZ
@ -799,7 +749,6 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
fmla z16.s, p1/m, z0.s, z8.s
fmla z17.s, p1/m, z0.s, z9.s
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE]
fmla z18.s, p1/m, z0.s, z10.s
fmla z19.s, p1/m, z0.s, z11.s
@ -807,25 +756,21 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
.macro SAVEv1x4
prfm PLDL2KEEP, [pCRow0, #C_PRE_SIZE]
add pCRow1, pCRow0, LDC
ld1w z24.s, p1/z, [pCRow0]
fmla z24.s, p1/m, z16.s, alphaZ
st1w z24.s, p1, [pCRow0]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
add pCRow2, pCRow1, LDC
ld1w z25.s, p1/z, [pCRow1]
fmla z25.s, p1/m, z17.s, alphaZ
st1w z25.s, p1, [pCRow1]
prfm PLDL2KEEP, [pCRow2, #C_PRE_SIZE]
add pCRow1, pCRow2, LDC
ld1w z26.s, p1/z, [pCRow2]
fmla z26.s, p1/m, z18.s, alphaZ
st1w z26.s, p1, [pCRow2]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
ld1w z27.s, p1/z, [pCRow1]
fmla z27.s, p1/m, z19.s, alphaZ
@ -852,20 +797,17 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
add pB, pB, 8
fmla z16.s, p1/m, z0.s, z8.s
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE]
fmla z17.s, p1/m, z0.s, z9.s
.endm
.macro SAVEv1x2
prfm PLDL2KEEP, [pCRow0, #C_PRE_SIZE]
add pCRow1, pCRow0, LDC
ld1w z24.s, p1/z, [pCRow0]
fmla z24.s, p1/m, z16.s, alphaZ
st1w z24.s, p1, [pCRow0]
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
ld1w z25.s, p1/z, [pCRow1]
fmla z25.s, p1/m, z17.s, alphaZ
@ -890,13 +832,11 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
add pB, pB, 4
fmla z16.s, p1/m, z0.s, z8.s
prfm PLDL1KEEP, [pA1, #A_PRE_SIZE]
.endm
.macro SAVEv1x1
prfm PLDL2KEEP, [pCRow0, #C_PRE_SIZE]
ld1w z24.s, p1/z, [pCRow0]
fmla z24.s, p1/m, z16.s, alphaZ
@ -928,8 +868,6 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
stp x26, x27, [sp, #(9 * 16)]
str x28, [sp, #(10 * 16)]
prfm PLDL1KEEP, [origPB]
prfm PLDL1KEEP, [origPA]
fmov alpha, s0
dup alphaZ, alpha
@ -968,7 +906,6 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
/* Until we have at least 2*SVE_LEN iters left in M, we do them with V2*8 kernel */
mul temp, vec_len, origK // generate address of pA2
add pA2, pA1, temp, lsl #2 // pA1 = start of A array
prfm PLDL1KEEP, [pA2]
.align 5
.Lsgemm_kernel_L8_Mv2_20:
@ -1057,11 +994,6 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
bne .Lsgemm_kernel_L8_Mv2_46
.Lsgemm_kernel_L8_Mv2_100:
prfm PLDL1KEEP, [pA1]
prfm PLDL1KEEP, [pA1, #64]
prfm PLDL1KEEP, [pA2]
prfm PLDL1KEEP, [pA2, #64]
prfm PLDL1KEEP, [origPB]
SAVEv2x8
mov pA1, pA2 // pA1 = pA2
@ -1171,9 +1103,6 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
bne .Lsgemm_kernel_L8_Mv1_46
.Lsgemm_kernel_L8_Mv1_100:
prfm PLDL1KEEP, [pA1]
prfm PLDL1KEEP, [pA1, #64]
prfm PLDL1KEEP, [origPB]
SAVEv1x8
@ -1233,16 +1162,12 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
.align 5
.Lsgemm_kernel_L4_Mv2_22:
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv2x4_SUB
KERNELv2x4_SUB
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv2x4_SUB
KERNELv2x4_SUB
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv2x4_SUB
KERNELv2x4_SUB
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv2x4_SUB
KERNELv2x4_SUB
@ -1257,18 +1182,12 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
.align 5
.Lsgemm_kernel_L4_Mv2_46:
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv2x4_SUB
subs counterL, counterL, #1
bne .Lsgemm_kernel_L4_Mv2_46
.Lsgemm_kernel_L4_Mv2_100:
prfm PLDL1KEEP, [pA1]
prfm PLDL1KEEP, [pA1, #64]
prfm PLDL1KEEP, [pA2]
prfm PLDL1KEEP, [pA2, #64]
prfm PLDL1KEEP, [origPB]
SAVEv2x4
mov pA1, pA2 // pA1 = pA2
@ -1304,16 +1223,12 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
.align 5
.Lsgemm_kernel_L4_Mv1_22:
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv1x4_SUB
KERNELv1x4_SUB
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv1x4_SUB
KERNELv1x4_SUB
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv1x4_SUB
KERNELv1x4_SUB
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv1x4_SUB
KERNELv1x4_SUB
@ -1328,16 +1243,12 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
.align 5
.Lsgemm_kernel_L4_Mv1_46:
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv1x4_SUB
subs counterL, counterL, #1
bne .Lsgemm_kernel_L4_Mv1_46
.Lsgemm_kernel_L4_Mv1_100:
prfm PLDL1KEEP, [pA1]
prfm PLDL1KEEP, [pA1, #64]
prfm PLDL1KEEP, [origPB]
SAVEv1x4
@ -1393,12 +1304,10 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
.align 5
.Lsgemm_kernel_L2_Mv2_22:
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv2x2_SUB
KERNELv2x2_SUB
KERNELv2x2_SUB
KERNELv2x2_SUB
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv2x2_SUB
KERNELv2x2_SUB
KERNELv2x2_SUB
@ -1415,18 +1324,12 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
.align 5
.Lsgemm_kernel_L2_Mv2_46:
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv2x2_SUB
subs counterL, counterL, #1
bne .Lsgemm_kernel_L2_Mv2_46
.Lsgemm_kernel_L2_Mv2_100:
prfm PLDL1KEEP, [pA1]
prfm PLDL1KEEP, [pA1, #64]
prfm PLDL1KEEP, [pA2]
prfm PLDL1KEEP, [pA2, #64]
prfm PLDL1KEEP, [origPB]
SAVEv2x2
mov pA1, pA2 // pA1 = pA2
@ -1463,12 +1366,10 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
.align 5
.Lsgemm_kernel_L2_Mv1_22:
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv1x2_SUB
KERNELv1x2_SUB
KERNELv1x2_SUB
KERNELv1x2_SUB
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv1x2_SUB
KERNELv1x2_SUB
KERNELv1x2_SUB
@ -1485,16 +1386,12 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
.align 5
.Lsgemm_kernel_L2_Mv1_46:
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv1x2_SUB
subs counterL, counterL, #1
bne .Lsgemm_kernel_L2_Mv1_46
.Lsgemm_kernel_L2_Mv1_100:
prfm PLDL1KEEP, [pA1]
prfm PLDL1KEEP, [pA1, #64]
prfm PLDL1KEEP, [origPB]
SAVEv1x2
@ -1550,7 +1447,6 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
.align 5
.Lsgemm_kernel_L1_Mv2_22:
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv2x1_SUB
KERNELv2x1_SUB
KERNELv2x1_SUB
@ -1571,16 +1467,12 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
.align 5
.Lsgemm_kernel_L1_Mv2_46:
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv2x1_SUB
subs counterL, counterL, #1
bgt .Lsgemm_kernel_L1_Mv2_46
.Lsgemm_kernel_L1_Mv2_100:
prfm PLDL1KEEP, [pA1]
prfm PLDL1KEEP, [pA1, #64]
prfm PLDL1KEEP, [origPB]
SAVEv2x1
mov pA1, pA2 // pA1 = pA2
@ -1617,7 +1509,6 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
.align 5
.Lsgemm_kernel_L1_Mv1_22:
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv1x1_SUB
KERNELv1x1_SUB
KERNELv1x1_SUB
@ -1638,16 +1529,12 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */
.align 5
.Lsgemm_kernel_L1_Mv1_46:
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
KERNELv1x1_SUB
subs counterL, counterL, #1
bgt .Lsgemm_kernel_L1_Mv1_46
.Lsgemm_kernel_L1_Mv1_100:
prfm PLDL1KEEP, [pA1]
prfm PLDL1KEEP, [pA1, #64]
prfm PLDL1KEEP, [origPB]
SAVEv1x1