floraachy
/
OpenBLAS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cortex-A57: Fix clang compilation errors

This commit is contained in:
Ashwin Sekhar T K 2016-03-24 10:31:28 +05:30
parent 3b5ffb49d3
commit 278511ad2d
18 changed files with 3204 additions and 3204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

480
kernel/arm64/cgemm_kernel_4x4.S
View File

@ -179,93 +179,93 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v2.4s, v3.4s}, [ppA] ld2 {v2.4s, v3.4s}, [ppA]
add ppA, ppA, #32 add ppA, ppA, #32
fmul v16.4s, v0.4s, v8.4s[0] fmul v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.4s[0] OP_ii v16.4s, v1.4s, v9.s[0]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v17.16b, v17.16b, v17.16b eor v17.16b, v17.16b, v17.16b
fmls v17.4s, v0.4s, v9.4s[0] fmls v17.4s, v0.4s, v9.s[0]
#else #else
fmul v17.4s, v0.4s, v9.4s[0] fmul v17.4s, v0.4s, v9.s[0]
#endif #endif
OP_ir v17.4s, v1.4s, v8.4s[0] OP_ir v17.4s, v1.4s, v8.s[0]
fmul v20.4s, v0.4s, v8.4s[1] fmul v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.4s[1] OP_ii v20.4s, v1.4s, v9.s[1]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v21.16b, v21.16b, v21.16b eor v21.16b, v21.16b, v21.16b
fmls v21.4s, v0.4s, v9.4s[1] fmls v21.4s, v0.4s, v9.s[1]
#else #else
fmul v21.4s, v0.4s, v9.4s[1] fmul v21.4s, v0.4s, v9.s[1]
#endif #endif
OP_ir v21.4s, v1.4s, v8.4s[1] OP_ir v21.4s, v1.4s, v8.s[1]
fmul v24.4s, v0.4s, v8.4s[2] fmul v24.4s, v0.4s, v8.s[2]
OP_ii v24.4s, v1.4s, v9.4s[2] OP_ii v24.4s, v1.4s, v9.s[2]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v25.16b, v25.16b, v25.16b eor v25.16b, v25.16b, v25.16b
fmls v25.4s, v0.4s, v9.4s[2] fmls v25.4s, v0.4s, v9.s[2]
#else #else
fmul v25.4s, v0.4s, v9.4s[2] fmul v25.4s, v0.4s, v9.s[2]
#endif #endif
OP_ir v25.4s, v1.4s, v8.4s[2] OP_ir v25.4s, v1.4s, v8.s[2]
fmul v28.4s, v0.4s, v8.4s[3] fmul v28.4s, v0.4s, v8.s[3]
OP_ii v28.4s, v1.4s, v9.4s[3] OP_ii v28.4s, v1.4s, v9.s[3]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v29.16b, v29.16b, v29.16b eor v29.16b, v29.16b, v29.16b
fmls v29.4s, v0.4s, v9.4s[3] fmls v29.4s, v0.4s, v9.s[3]
#else #else
fmul v29.4s, v0.4s, v9.4s[3] fmul v29.4s, v0.4s, v9.s[3]
#endif #endif
OP_ir v29.4s, v1.4s, v8.4s[3] OP_ir v29.4s, v1.4s, v8.s[3]
fmul v18.4s, v2.4s, v8.4s[0] fmul v18.4s, v2.4s, v8.s[0]
OP_ii v18.4s, v3.4s, v9.4s[0] OP_ii v18.4s, v3.4s, v9.s[0]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v19.16b, v19.16b, v19.16b eor v19.16b, v19.16b, v19.16b
fmls v19.4s, v2.4s, v9.4s[0] fmls v19.4s, v2.4s, v9.s[0]
#else #else
fmul v19.4s, v2.4s, v9.4s[0] fmul v19.4s, v2.4s, v9.s[0]
#endif #endif
OP_ir v19.4s, v3.4s, v8.4s[0] OP_ir v19.4s, v3.4s, v8.s[0]
fmul v22.4s, v2.4s, v8.4s[1] fmul v22.4s, v2.4s, v8.s[1]
OP_ii v22.4s, v3.4s, v9.4s[1] OP_ii v22.4s, v3.4s, v9.s[1]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v23.16b, v23.16b, v23.16b eor v23.16b, v23.16b, v23.16b
fmls v23.4s, v2.4s, v9.4s[1] fmls v23.4s, v2.4s, v9.s[1]
#else #else
fmul v23.4s, v2.4s, v9.4s[1] fmul v23.4s, v2.4s, v9.s[1]
#endif #endif
OP_ir v23.4s, v3.4s, v8.4s[1] OP_ir v23.4s, v3.4s, v8.s[1]
fmul v26.4s, v2.4s, v8.4s[2] fmul v26.4s, v2.4s, v8.s[2]
OP_ii v26.4s, v3.4s, v9.4s[2] OP_ii v26.4s, v3.4s, v9.s[2]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v27.16b, v27.16b, v27.16b eor v27.16b, v27.16b, v27.16b
fmls v27.4s, v2.4s, v9.4s[2] fmls v27.4s, v2.4s, v9.s[2]
#else #else
fmul v27.4s, v2.4s, v9.4s[2] fmul v27.4s, v2.4s, v9.s[2]
#endif #endif
OP_ir v27.4s, v3.4s, v8.4s[2] OP_ir v27.4s, v3.4s, v8.s[2]
fmul v30.4s, v2.4s, v8.4s[3] fmul v30.4s, v2.4s, v8.s[3]
OP_ii v30.4s, v3.4s, v9.4s[3] OP_ii v30.4s, v3.4s, v9.s[3]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v31.16b, v31.16b, v31.16b eor v31.16b, v31.16b, v31.16b
fmls v31.4s, v2.4s, v9.4s[3] fmls v31.4s, v2.4s, v9.s[3]
#else #else
fmul v31.4s, v2.4s, v9.4s[3] fmul v31.4s, v2.4s, v9.s[3]
#endif #endif
OP_ir v31.4s, v3.4s, v8.4s[3] OP_ir v31.4s, v3.4s, v8.s[3]
ld2 {v12.4s, v13.4s}, [pB] ld2 {v12.4s, v13.4s}, [pB]
add pB, pB, #32 add pB, pB, #32
@ -276,159 +276,159 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x4_M1 .macro KERNEL8x4_M1
OP_rr v16.4s, v0.4s, v8.4s[0] OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.4s[0] OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.4s[0] OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.4s[0] OP_ir v17.4s, v1.4s, v8.s[0]
ld2 {v12.4s, v13.4s}, [pB] // for next round ld2 {v12.4s, v13.4s}, [pB] // for next round
add pB, pB, #32 add pB, pB, #32
OP_rr v20.4s, v0.4s, v8.4s[1] OP_rr v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.4s[1] OP_ii v20.4s, v1.4s, v9.s[1]
OP_ri v21.4s, v0.4s, v9.4s[1] OP_ri v21.4s, v0.4s, v9.s[1]
OP_ir v21.4s, v1.4s, v8.4s[1] OP_ir v21.4s, v1.4s, v8.s[1]
prfm PLDL1KEEP, [pB, #512] prfm PLDL1KEEP, [pB, #512]
OP_rr v24.4s, v0.4s, v8.4s[2] OP_rr v24.4s, v0.4s, v8.s[2]
OP_ii v24.4s, v1.4s, v9.4s[2] OP_ii v24.4s, v1.4s, v9.s[2]
OP_ri v25.4s, v0.4s, v9.4s[2] OP_ri v25.4s, v0.4s, v9.s[2]
OP_ir v25.4s, v1.4s, v8.4s[2] OP_ir v25.4s, v1.4s, v8.s[2]
ld2 {v4.4s, v5.4s} , [pA] // for next round ld2 {v4.4s, v5.4s} , [pA] // for next round
add pA, pA, #32 add pA, pA, #32
OP_rr v28.4s, v0.4s, v8.4s[3] OP_rr v28.4s, v0.4s, v8.s[3]
OP_ii v28.4s, v1.4s, v9.4s[3] OP_ii v28.4s, v1.4s, v9.s[3]
OP_ri v29.4s, v0.4s, v9.4s[3] OP_ri v29.4s, v0.4s, v9.s[3]
OP_ir v29.4s, v1.4s, v8.4s[3] OP_ir v29.4s, v1.4s, v8.s[3]
prfm PLDL1KEEP, [pA, #512] prfm PLDL1KEEP, [pA, #512]
OP_rr v18.4s, v2.4s, v8.4s[0] OP_rr v18.4s, v2.4s, v8.s[0]
OP_ii v18.4s, v3.4s, v9.4s[0] OP_ii v18.4s, v3.4s, v9.s[0]
OP_ri v19.4s, v2.4s, v9.4s[0] OP_ri v19.4s, v2.4s, v9.s[0]
OP_ir v19.4s, v3.4s, v8.4s[0] OP_ir v19.4s, v3.4s, v8.s[0]
ld2 {v6.4s, v7.4s} , [ppA] // for next round ld2 {v6.4s, v7.4s} , [ppA] // for next round
add ppA, ppA, #32 add ppA, ppA, #32
OP_rr v22.4s, v2.4s, v8.4s[1] OP_rr v22.4s, v2.4s, v8.s[1]
OP_ii v22.4s, v3.4s, v9.4s[1] OP_ii v22.4s, v3.4s, v9.s[1]
OP_ri v23.4s, v2.4s, v9.4s[1] OP_ri v23.4s, v2.4s, v9.s[1]
OP_ir v23.4s, v3.4s, v8.4s[1] OP_ir v23.4s, v3.4s, v8.s[1]
prfm PLDL1KEEP, [ppA, #512] prfm PLDL1KEEP, [ppA, #512]
OP_rr v26.4s, v2.4s, v8.4s[2] OP_rr v26.4s, v2.4s, v8.s[2]
OP_ii v26.4s, v3.4s, v9.4s[2] OP_ii v26.4s, v3.4s, v9.s[2]
OP_ri v27.4s, v2.4s, v9.4s[2] OP_ri v27.4s, v2.4s, v9.s[2]
OP_ir v27.4s, v3.4s, v8.4s[2] OP_ir v27.4s, v3.4s, v8.s[2]
OP_rr v30.4s, v2.4s, v8.4s[3] OP_rr v30.4s, v2.4s, v8.s[3]
OP_ii v30.4s, v3.4s, v9.4s[3] OP_ii v30.4s, v3.4s, v9.s[3]
OP_ri v31.4s, v2.4s, v9.4s[3] OP_ri v31.4s, v2.4s, v9.s[3]
OP_ir v31.4s, v3.4s, v8.4s[3] OP_ir v31.4s, v3.4s, v8.s[3]
.endm .endm
.macro KERNEL8x4_M2 .macro KERNEL8x4_M2
OP_rr v16.4s, v4.4s, v12.4s[0] OP_rr v16.4s, v4.4s, v12.s[0]
OP_ii v16.4s, v5.4s, v13.4s[0] OP_ii v16.4s, v5.4s, v13.s[0]
OP_ri v17.4s, v4.4s, v13.4s[0] OP_ri v17.4s, v4.4s, v13.s[0]
OP_ir v17.4s, v5.4s, v12.4s[0] OP_ir v17.4s, v5.4s, v12.s[0]
ld2 {v8.4s, v9.4s}, [pB] // for next round ld2 {v8.4s, v9.4s}, [pB] // for next round
add pB, pB, #32 add pB, pB, #32
OP_rr v20.4s, v4.4s, v12.4s[1] OP_rr v20.4s, v4.4s, v12.s[1]
OP_ii v20.4s, v5.4s, v13.4s[1] OP_ii v20.4s, v5.4s, v13.s[1]
OP_ri v21.4s, v4.4s, v13.4s[1] OP_ri v21.4s, v4.4s, v13.s[1]
OP_ir v21.4s, v5.4s, v12.4s[1] OP_ir v21.4s, v5.4s, v12.s[1]
prfm PLDL1KEEP, [pA, #512] prfm PLDL1KEEP, [pA, #512]
OP_rr v24.4s, v4.4s, v12.4s[2] OP_rr v24.4s, v4.4s, v12.s[2]
OP_ii v24.4s, v5.4s, v13.4s[2] OP_ii v24.4s, v5.4s, v13.s[2]
OP_ri v25.4s, v4.4s, v13.4s[2] OP_ri v25.4s, v4.4s, v13.s[2]
OP_ir v25.4s, v5.4s, v12.4s[2] OP_ir v25.4s, v5.4s, v12.s[2]
ld2 {v0.4s, v1.4s}, [pA] // for next round ld2 {v0.4s, v1.4s}, [pA] // for next round
add pA, pA, #32 add pA, pA, #32
OP_rr v28.4s, v4.4s, v12.4s[3] OP_rr v28.4s, v4.4s, v12.s[3]
OP_ii v28.4s, v5.4s, v13.4s[3] OP_ii v28.4s, v5.4s, v13.s[3]
OP_ri v29.4s, v4.4s, v13.4s[3] OP_ri v29.4s, v4.4s, v13.s[3]
OP_ir v29.4s, v5.4s, v12.4s[3] OP_ir v29.4s, v5.4s, v12.s[3]
prfm PLDL1KEEP, [ppA, #512] prfm PLDL1KEEP, [ppA, #512]
OP_rr v18.4s, v6.4s, v12.4s[0] OP_rr v18.4s, v6.4s, v12.s[0]
OP_ii v18.4s, v7.4s, v13.4s[0] OP_ii v18.4s, v7.4s, v13.s[0]
OP_ri v19.4s, v6.4s, v13.4s[0] OP_ri v19.4s, v6.4s, v13.s[0]
OP_ir v19.4s, v7.4s, v12.4s[0] OP_ir v19.4s, v7.4s, v12.s[0]
ld2 {v2.4s, v3.4s}, [ppA] // for next round ld2 {v2.4s, v3.4s}, [ppA] // for next round
add ppA, ppA, #32 add ppA, ppA, #32
OP_rr v22.4s, v6.4s, v12.4s[1] OP_rr v22.4s, v6.4s, v12.s[1]
OP_ii v22.4s, v7.4s, v13.4s[1] OP_ii v22.4s, v7.4s, v13.s[1]
OP_ri v23.4s, v6.4s, v13.4s[1] OP_ri v23.4s, v6.4s, v13.s[1]
OP_ir v23.4s, v7.4s, v12.4s[1] OP_ir v23.4s, v7.4s, v12.s[1]
prfm PLDL1KEEP, [pB, #512] prfm PLDL1KEEP, [pB, #512]
OP_rr v26.4s, v6.4s, v12.4s[2] OP_rr v26.4s, v6.4s, v12.s[2]
OP_ii v26.4s, v7.4s, v13.4s[2] OP_ii v26.4s, v7.4s, v13.s[2]
OP_ri v27.4s, v6.4s, v13.4s[2] OP_ri v27.4s, v6.4s, v13.s[2]
OP_ir v27.4s, v7.4s, v12.4s[2] OP_ir v27.4s, v7.4s, v12.s[2]
OP_rr v30.4s, v6.4s, v12.4s[3] OP_rr v30.4s, v6.4s, v12.s[3]
OP_ii v30.4s, v7.4s, v13.4s[3] OP_ii v30.4s, v7.4s, v13.s[3]
OP_ri v31.4s, v6.4s, v13.4s[3] OP_ri v31.4s, v6.4s, v13.s[3]
OP_ir v31.4s, v7.4s, v12.4s[3] OP_ir v31.4s, v7.4s, v12.s[3]
.endm .endm
.macro KERNEL8x4_E .macro KERNEL8x4_E
OP_rr v16.4s, v4.4s, v12.4s[0] OP_rr v16.4s, v4.4s, v12.s[0]
OP_ii v16.4s, v5.4s, v13.4s[0] OP_ii v16.4s, v5.4s, v13.s[0]
OP_ri v17.4s, v4.4s, v13.4s[0] OP_ri v17.4s, v4.4s, v13.s[0]
OP_ir v17.4s, v5.4s, v12.4s[0] OP_ir v17.4s, v5.4s, v12.s[0]
OP_rr v20.4s, v4.4s, v12.4s[1] OP_rr v20.4s, v4.4s, v12.s[1]
OP_ii v20.4s, v5.4s, v13.4s[1] OP_ii v20.4s, v5.4s, v13.s[1]
OP_ri v21.4s, v4.4s, v13.4s[1] OP_ri v21.4s, v4.4s, v13.s[1]
OP_ir v21.4s, v5.4s, v12.4s[1] OP_ir v21.4s, v5.4s, v12.s[1]
OP_rr v24.4s, v4.4s, v12.4s[2] OP_rr v24.4s, v4.4s, v12.s[2]
OP_ii v24.4s, v5.4s, v13.4s[2] OP_ii v24.4s, v5.4s, v13.s[2]
OP_ri v25.4s, v4.4s, v13.4s[2] OP_ri v25.4s, v4.4s, v13.s[2]
OP_ir v25.4s, v5.4s, v12.4s[2] OP_ir v25.4s, v5.4s, v12.s[2]
OP_rr v28.4s, v4.4s, v12.4s[3] OP_rr v28.4s, v4.4s, v12.s[3]
OP_ii v28.4s, v5.4s, v13.4s[3] OP_ii v28.4s, v5.4s, v13.s[3]
OP_ri v29.4s, v4.4s, v13.4s[3] OP_ri v29.4s, v4.4s, v13.s[3]
OP_ir v29.4s, v5.4s, v12.4s[3] OP_ir v29.4s, v5.4s, v12.s[3]
OP_rr v18.4s, v6.4s, v12.4s[0] OP_rr v18.4s, v6.4s, v12.s[0]
OP_ii v18.4s, v7.4s, v13.4s[0] OP_ii v18.4s, v7.4s, v13.s[0]
OP_ri v19.4s, v6.4s, v13.4s[0] OP_ri v19.4s, v6.4s, v13.s[0]
OP_ir v19.4s, v7.4s, v12.4s[0] OP_ir v19.4s, v7.4s, v12.s[0]
OP_rr v22.4s, v6.4s, v12.4s[1] OP_rr v22.4s, v6.4s, v12.s[1]
OP_ii v22.4s, v7.4s, v13.4s[1] OP_ii v22.4s, v7.4s, v13.s[1]
OP_ri v23.4s, v6.4s, v13.4s[1] OP_ri v23.4s, v6.4s, v13.s[1]
OP_ir v23.4s, v7.4s, v12.4s[1] OP_ir v23.4s, v7.4s, v12.s[1]
OP_rr v26.4s, v6.4s, v12.4s[2] OP_rr v26.4s, v6.4s, v12.s[2]
OP_ii v26.4s, v7.4s, v13.4s[2] OP_ii v26.4s, v7.4s, v13.s[2]
OP_ri v27.4s, v6.4s, v13.4s[2] OP_ri v27.4s, v6.4s, v13.s[2]
OP_ir v27.4s, v7.4s, v12.4s[2] OP_ir v27.4s, v7.4s, v12.s[2]
OP_rr v30.4s, v6.4s, v12.4s[3] OP_rr v30.4s, v6.4s, v12.s[3]
OP_ii v30.4s, v7.4s, v13.4s[3] OP_ii v30.4s, v7.4s, v13.s[3]
OP_ri v31.4s, v6.4s, v13.4s[3] OP_ri v31.4s, v6.4s, v13.s[3]
OP_ir v31.4s, v7.4s, v12.4s[3] OP_ir v31.4s, v7.4s, v12.s[3]
.endm .endm
.macro KERNEL8x4_SUB .macro KERNEL8x4_SUB
@ -437,48 +437,48 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.4s, v1.4s}, [pA] ld2 {v0.4s, v1.4s}, [pA]
add pA, pA, #32 add pA, pA, #32
OP_rr v16.4s, v0.4s, v8.4s[0] OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.4s[0] OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.4s[0] OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.4s[0] OP_ir v17.4s, v1.4s, v8.s[0]
OP_rr v20.4s, v0.4s, v8.4s[1] OP_rr v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.4s[1] OP_ii v20.4s, v1.4s, v9.s[1]
OP_ri v21.4s, v0.4s, v9.4s[1] OP_ri v21.4s, v0.4s, v9.s[1]
OP_ir v21.4s, v1.4s, v8.4s[1] OP_ir v21.4s, v1.4s, v8.s[1]
ld2 {v2.4s, v3.4s}, [ppA] ld2 {v2.4s, v3.4s}, [ppA]
add ppA, ppA, #32 add ppA, ppA, #32
OP_rr v24.4s, v0.4s, v8.4s[2] OP_rr v24.4s, v0.4s, v8.s[2]
OP_ii v24.4s, v1.4s, v9.4s[2] OP_ii v24.4s, v1.4s, v9.s[2]
OP_ri v25.4s, v0.4s, v9.4s[2] OP_ri v25.4s, v0.4s, v9.s[2]
OP_ir v25.4s, v1.4s, v8.4s[2] OP_ir v25.4s, v1.4s, v8.s[2]
OP_rr v28.4s, v0.4s, v8.4s[3] OP_rr v28.4s, v0.4s, v8.s[3]
OP_ii v28.4s, v1.4s, v9.4s[3] OP_ii v28.4s, v1.4s, v9.s[3]
OP_ri v29.4s, v0.4s, v9.4s[3] OP_ri v29.4s, v0.4s, v9.s[3]
OP_ir v29.4s, v1.4s, v8.4s[3] OP_ir v29.4s, v1.4s, v8.s[3]
OP_rr v18.4s, v2.4s, v8.4s[0] OP_rr v18.4s, v2.4s, v8.s[0]
OP_ii v18.4s, v3.4s, v9.4s[0] OP_ii v18.4s, v3.4s, v9.s[0]
OP_ri v19.4s, v2.4s, v9.4s[0] OP_ri v19.4s, v2.4s, v9.s[0]
OP_ir v19.4s, v3.4s, v8.4s[0] OP_ir v19.4s, v3.4s, v8.s[0]
OP_rr v22.4s, v2.4s, v8.4s[1] OP_rr v22.4s, v2.4s, v8.s[1]
OP_ii v22.4s, v3.4s, v9.4s[1] OP_ii v22.4s, v3.4s, v9.s[1]
OP_ri v23.4s, v2.4s, v9.4s[1] OP_ri v23.4s, v2.4s, v9.s[1]
OP_ir v23.4s, v3.4s, v8.4s[1] OP_ir v23.4s, v3.4s, v8.s[1]
OP_rr v26.4s, v2.4s, v8.4s[2] OP_rr v26.4s, v2.4s, v8.s[2]
OP_ii v26.4s, v3.4s, v9.4s[2] OP_ii v26.4s, v3.4s, v9.s[2]
OP_ri v27.4s, v2.4s, v9.4s[2] OP_ri v27.4s, v2.4s, v9.s[2]
OP_ir v27.4s, v3.4s, v8.4s[2] OP_ir v27.4s, v3.4s, v8.s[2]
OP_rr v30.4s, v2.4s, v8.4s[3] OP_rr v30.4s, v2.4s, v8.s[3]
OP_ii v30.4s, v3.4s, v9.4s[3] OP_ii v30.4s, v3.4s, v9.s[3]
OP_ri v31.4s, v2.4s, v9.4s[3] OP_ri v31.4s, v2.4s, v9.s[3]
OP_ir v31.4s, v3.4s, v8.4s[3] OP_ir v31.4s, v3.4s, v8.s[3]
.endm .endm
.macro SAVE8x4 .macro SAVE8x4
@ -578,25 +578,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.4s, v1.4s}, [pA] ld2 {v0.4s, v1.4s}, [pA]
add pA, pA, #32 add pA, pA, #32
OP_rr v16.4s, v0.4s, v8.4s[0] OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.4s[0] OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.4s[0] OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.4s[0] OP_ir v17.4s, v1.4s, v8.s[0]
OP_rr v20.4s, v0.4s, v8.4s[1] OP_rr v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.4s[1] OP_ii v20.4s, v1.4s, v9.s[1]
OP_ri v21.4s, v0.4s, v9.4s[1] OP_ri v21.4s, v0.4s, v9.s[1]
OP_ir v21.4s, v1.4s, v8.4s[1] OP_ir v21.4s, v1.4s, v8.s[1]
OP_rr v24.4s, v0.4s, v8.4s[2] OP_rr v24.4s, v0.4s, v8.s[2]
OP_ii v24.4s, v1.4s, v9.4s[2] OP_ii v24.4s, v1.4s, v9.s[2]
OP_ri v25.4s, v0.4s, v9.4s[2] OP_ri v25.4s, v0.4s, v9.s[2]
OP_ir v25.4s, v1.4s, v8.4s[2] OP_ir v25.4s, v1.4s, v8.s[2]
OP_rr v28.4s, v0.4s, v8.4s[3] OP_rr v28.4s, v0.4s, v8.s[3]
OP_ii v28.4s, v1.4s, v9.4s[3] OP_ii v28.4s, v1.4s, v9.s[3]
OP_ri v29.4s, v0.4s, v9.4s[3] OP_ri v29.4s, v0.4s, v9.s[3]
OP_ir v29.4s, v1.4s, v8.4s[3] OP_ir v29.4s, v1.4s, v8.s[3]
.endm .endm
.macro SAVE4x4 .macro SAVE4x4
@ -658,25 +658,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.2s, v1.2s}, [pA] ld2 {v0.2s, v1.2s}, [pA]
add pA, pA, #16 add pA, pA, #16
OP_rr v16.2s, v0.2s, v8.4s[0] OP_rr v16.2s, v0.2s, v8.s[0]
OP_ii v16.2s, v1.2s, v9.4s[0] OP_ii v16.2s, v1.2s, v9.s[0]
OP_ri v17.2s, v0.2s, v9.4s[0] OP_ri v17.2s, v0.2s, v9.s[0]
OP_ir v17.2s, v1.2s, v8.4s[0] OP_ir v17.2s, v1.2s, v8.s[0]
OP_rr v20.2s, v0.2s, v8.4s[1] OP_rr v20.2s, v0.2s, v8.s[1]
OP_ii v20.2s, v1.2s, v9.4s[1] OP_ii v20.2s, v1.2s, v9.s[1]
OP_ri v21.2s, v0.2s, v9.4s[1] OP_ri v21.2s, v0.2s, v9.s[1]
OP_ir v21.2s, v1.2s, v8.4s[1] OP_ir v21.2s, v1.2s, v8.s[1]
OP_rr v24.2s, v0.2s, v8.4s[2] OP_rr v24.2s, v0.2s, v8.s[2]
OP_ii v24.2s, v1.2s, v9.4s[2] OP_ii v24.2s, v1.2s, v9.s[2]
OP_ri v25.2s, v0.2s, v9.4s[2] OP_ri v25.2s, v0.2s, v9.s[2]
OP_ir v25.2s, v1.2s, v8.4s[2] OP_ir v25.2s, v1.2s, v8.s[2]
OP_rr v28.2s, v0.2s, v8.4s[3] OP_rr v28.2s, v0.2s, v8.s[3]
OP_ii v28.2s, v1.2s, v9.4s[3] OP_ii v28.2s, v1.2s, v9.s[3]
OP_ri v29.2s, v0.2s, v9.4s[3] OP_ri v29.2s, v0.2s, v9.s[3]
OP_ir v29.2s, v1.2s, v8.4s[3] OP_ir v29.2s, v1.2s, v8.s[3]
.endm .endm
.macro SAVE2x4 .macro SAVE2x4
@ -738,25 +738,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.s, v1.s}[0], [pA] ld2 {v0.s, v1.s}[0], [pA]
add pA, pA, #8 add pA, pA, #8
OP_rr s16, s0, v8.4s[0] OP_rr s16, s0, v8.s[0]
OP_ii s16, s1, v9.4s[0] OP_ii s16, s1, v9.s[0]
OP_ri s17, s0, v9.4s[0] OP_ri s17, s0, v9.s[0]
OP_ir s17, s1, v8.4s[0] OP_ir s17, s1, v8.s[0]
OP_rr s20, s0, v8.4s[1] OP_rr s20, s0, v8.s[1]
OP_ii s20, s1, v9.4s[1] OP_ii s20, s1, v9.s[1]
OP_ri s21, s0, v9.4s[1] OP_ri s21, s0, v9.s[1]
OP_ir s21, s1, v8.4s[1] OP_ir s21, s1, v8.s[1]
OP_rr s24, s0, v8.4s[2] OP_rr s24, s0, v8.s[2]
OP_ii s24, s1, v9.4s[2] OP_ii s24, s1, v9.s[2]
OP_ri s25, s0, v9.4s[2] OP_ri s25, s0, v9.s[2]
OP_ir s25, s1, v8.4s[2] OP_ir s25, s1, v8.s[2]
OP_rr s28, s0, v8.4s[3] OP_rr s28, s0, v8.s[3]
OP_ii s28, s1, v9.4s[3] OP_ii s28, s1, v9.s[3]
OP_ri s29, s0, v9.4s[3] OP_ri s29, s0, v9.s[3]
OP_ir s29, s1, v8.4s[3] OP_ir s29, s1, v8.s[3]
.endm .endm
.macro SAVE1x4 .macro SAVE1x4
@ -814,15 +814,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.4s, v1.4s}, [pA] ld2 {v0.4s, v1.4s}, [pA]
add pA, pA, #32 add pA, pA, #32
OP_rr v16.4s, v0.4s, v8.2s[0] OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.2s[0] OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.2s[0] OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.2s[0] OP_ir v17.4s, v1.4s, v8.s[0]
OP_rr v20.4s, v0.4s, v8.2s[1] OP_rr v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.2s[1] OP_ii v20.4s, v1.4s, v9.s[1]
OP_ri v21.4s, v0.4s, v9.2s[1] OP_ri v21.4s, v0.4s, v9.s[1]
OP_ir v21.4s, v1.4s, v8.2s[1] OP_ir v21.4s, v1.4s, v8.s[1]
.endm .endm
.macro SAVE4x2 .macro SAVE4x2
@ -862,15 +862,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.2s, v1.2s}, [pA] ld2 {v0.2s, v1.2s}, [pA]
add pA, pA, #16 add pA, pA, #16
OP_rr v16.2s, v0.2s, v8.2s[0] OP_rr v16.2s, v0.2s, v8.s[0]
OP_ii v16.2s, v1.2s, v9.2s[0] OP_ii v16.2s, v1.2s, v9.s[0]
OP_ri v17.2s, v0.2s, v9.2s[0] OP_ri v17.2s, v0.2s, v9.s[0]
OP_ir v17.2s, v1.2s, v8.2s[0] OP_ir v17.2s, v1.2s, v8.s[0]
OP_rr v20.2s, v0.2s, v8.2s[1] OP_rr v20.2s, v0.2s, v8.s[1]
OP_ii v20.2s, v1.2s, v9.2s[1] OP_ii v20.2s, v1.2s, v9.s[1]
OP_ri v21.2s, v0.2s, v9.2s[1] OP_ri v21.2s, v0.2s, v9.s[1]
OP_ir v21.2s, v1.2s, v8.2s[1] OP_ir v21.2s, v1.2s, v8.s[1]
.endm .endm
.macro SAVE2x2 .macro SAVE2x2
@ -910,15 +910,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.s, v1.s}[0], [pA] ld2 {v0.s, v1.s}[0], [pA]
add pA, pA, #8 add pA, pA, #8
OP_rr s16, s0, v8.2s[0] OP_rr s16, s0, v8.s[0]
OP_ii s16, s1, v9.2s[0] OP_ii s16, s1, v9.s[0]
OP_ri s17, s0, v9.2s[0] OP_ri s17, s0, v9.s[0]
OP_ir s17, s1, v8.2s[0] OP_ir s17, s1, v8.s[0]
OP_rr s20, s0, v8.2s[1] OP_rr s20, s0, v8.s[1]
OP_ii s20, s1, v9.2s[1] OP_ii s20, s1, v9.s[1]
OP_ri s21, s0, v9.2s[1] OP_ri s21, s0, v9.s[1]
OP_ir s21, s1, v8.2s[1] OP_ir s21, s1, v8.s[1]
.endm .endm
.macro SAVE1x2 .macro SAVE1x2

664
kernel/arm64/cgemm_kernel_8x4.S Executable file → Normal file
View File

@ -178,93 +178,93 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v2.4s, v3.4s}, [pA] ld2 {v2.4s, v3.4s}, [pA]
add pA, pA, #32 add pA, pA, #32
fmul v16.4s, v0.4s, v8.4s[0] fmul v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.4s[0] OP_ii v16.4s, v1.4s, v9.s[0]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v17.16b, v17.16b, v17.16b eor v17.16b, v17.16b, v17.16b
fmls v17.4s, v0.4s, v9.4s[0] fmls v17.4s, v0.4s, v9.s[0]
#else #else
fmul v17.4s, v0.4s, v9.4s[0] fmul v17.4s, v0.4s, v9.s[0]
#endif #endif
OP_ir v17.4s, v1.4s, v8.4s[0] OP_ir v17.4s, v1.4s, v8.s[0]
fmul v18.4s, v2.4s, v8.4s[0] fmul v18.4s, v2.4s, v8.s[0]
OP_ii v18.4s, v3.4s, v9.4s[0] OP_ii v18.4s, v3.4s, v9.s[0]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v19.16b, v19.16b, v19.16b eor v19.16b, v19.16b, v19.16b
fmls v19.4s, v2.4s, v9.4s[0] fmls v19.4s, v2.4s, v9.s[0]
#else #else
fmul v19.4s, v2.4s, v9.4s[0] fmul v19.4s, v2.4s, v9.s[0]
#endif #endif
OP_ir v19.4s, v3.4s, v8.4s[0] OP_ir v19.4s, v3.4s, v8.s[0]
fmul v20.4s, v0.4s, v8.4s[1] fmul v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.4s[1] OP_ii v20.4s, v1.4s, v9.s[1]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v21.16b, v21.16b, v21.16b eor v21.16b, v21.16b, v21.16b
fmls v21.4s, v0.4s, v9.4s[1] fmls v21.4s, v0.4s, v9.s[1]
#else #else
fmul v21.4s, v0.4s, v9.4s[1] fmul v21.4s, v0.4s, v9.s[1]
#endif #endif
OP_ir v21.4s, v1.4s, v8.4s[1] OP_ir v21.4s, v1.4s, v8.s[1]
fmul v22.4s, v2.4s, v8.4s[1] fmul v22.4s, v2.4s, v8.s[1]
OP_ii v22.4s, v3.4s, v9.4s[1] OP_ii v22.4s, v3.4s, v9.s[1]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v23.16b, v23.16b, v23.16b eor v23.16b, v23.16b, v23.16b
fmls v23.4s, v2.4s, v9.4s[1] fmls v23.4s, v2.4s, v9.s[1]
#else #else
fmul v23.4s, v2.4s, v9.4s[1] fmul v23.4s, v2.4s, v9.s[1]
#endif #endif
OP_ir v23.4s, v3.4s, v8.4s[1] OP_ir v23.4s, v3.4s, v8.s[1]
fmul v24.4s, v0.4s, v8.4s[2] fmul v24.4s, v0.4s, v8.s[2]
OP_ii v24.4s, v1.4s, v9.4s[2] OP_ii v24.4s, v1.4s, v9.s[2]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v25.16b, v25.16b, v25.16b eor v25.16b, v25.16b, v25.16b
fmls v25.4s, v0.4s, v9.4s[2] fmls v25.4s, v0.4s, v9.s[2]
#else #else
fmul v25.4s, v0.4s, v9.4s[2] fmul v25.4s, v0.4s, v9.s[2]
#endif #endif
OP_ir v25.4s, v1.4s, v8.4s[2] OP_ir v25.4s, v1.4s, v8.s[2]
fmul v26.4s, v2.4s, v8.4s[2] fmul v26.4s, v2.4s, v8.s[2]
OP_ii v26.4s, v3.4s, v9.4s[2] OP_ii v26.4s, v3.4s, v9.s[2]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v27.16b, v27.16b, v27.16b eor v27.16b, v27.16b, v27.16b
fmls v27.4s, v2.4s, v9.4s[2] fmls v27.4s, v2.4s, v9.s[2]
#else #else
fmul v27.4s, v2.4s, v9.4s[2] fmul v27.4s, v2.4s, v9.s[2]
#endif #endif
OP_ir v27.4s, v3.4s, v8.4s[2] OP_ir v27.4s, v3.4s, v8.s[2]
fmul v28.4s, v0.4s, v8.4s[3] fmul v28.4s, v0.4s, v8.s[3]
OP_ii v28.4s, v1.4s, v9.4s[3] OP_ii v28.4s, v1.4s, v9.s[3]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v29.16b, v29.16b, v29.16b eor v29.16b, v29.16b, v29.16b
fmls v29.4s, v0.4s, v9.4s[3] fmls v29.4s, v0.4s, v9.s[3]
#else #else
fmul v29.4s, v0.4s, v9.4s[3] fmul v29.4s, v0.4s, v9.s[3]
#endif #endif
OP_ir v29.4s, v1.4s, v8.4s[3] OP_ir v29.4s, v1.4s, v8.s[3]
fmul v30.4s, v2.4s, v8.4s[3] fmul v30.4s, v2.4s, v8.s[3]
OP_ii v30.4s, v3.4s, v9.4s[3] OP_ii v30.4s, v3.4s, v9.s[3]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v31.16b, v31.16b, v31.16b eor v31.16b, v31.16b, v31.16b
fmls v31.4s, v2.4s, v9.4s[3] fmls v31.4s, v2.4s, v9.s[3]
#else #else
fmul v31.4s, v2.4s, v9.4s[3] fmul v31.4s, v2.4s, v9.s[3]
#endif #endif
OP_ir v31.4s, v3.4s, v8.4s[3] OP_ir v31.4s, v3.4s, v8.s[3]
ld2 {v12.4s, v13.4s}, [pB] ld2 {v12.4s, v13.4s}, [pB]
add pB, pB, #32 add pB, pB, #32
@ -275,45 +275,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x4_M1 .macro KERNEL8x4_M1
OP_rr v16.4s, v0.4s, v8.4s[0] OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.4s[0] OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.4s[0] OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.4s[0] OP_ir v17.4s, v1.4s, v8.s[0]
OP_rr v18.4s, v2.4s, v8.4s[0] OP_rr v18.4s, v2.4s, v8.s[0]
OP_ii v18.4s, v3.4s, v9.4s[0] OP_ii v18.4s, v3.4s, v9.s[0]
OP_ri v19.4s, v2.4s, v9.4s[0] OP_ri v19.4s, v2.4s, v9.s[0]
OP_ir v19.4s, v3.4s, v8.4s[0] OP_ir v19.4s, v3.4s, v8.s[0]
OP_rr v20.4s, v0.4s, v8.4s[1] OP_rr v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.4s[1] OP_ii v20.4s, v1.4s, v9.s[1]
OP_ri v21.4s, v0.4s, v9.4s[1] OP_ri v21.4s, v0.4s, v9.s[1]
OP_ir v21.4s, v1.4s, v8.4s[1] OP_ir v21.4s, v1.4s, v8.s[1]
OP_rr v22.4s, v2.4s, v8.4s[1] OP_rr v22.4s, v2.4s, v8.s[1]
OP_ii v22.4s, v3.4s, v9.4s[1] OP_ii v22.4s, v3.4s, v9.s[1]
OP_ri v23.4s, v2.4s, v9.4s[1] OP_ri v23.4s, v2.4s, v9.s[1]
OP_ir v23.4s, v3.4s, v8.4s[1] OP_ir v23.4s, v3.4s, v8.s[1]
OP_rr v24.4s, v0.4s, v8.4s[2] OP_rr v24.4s, v0.4s, v8.s[2]
OP_ii v24.4s, v1.4s, v9.4s[2] OP_ii v24.4s, v1.4s, v9.s[2]
OP_ri v25.4s, v0.4s, v9.4s[2] OP_ri v25.4s, v0.4s, v9.s[2]
OP_ir v25.4s, v1.4s, v8.4s[2] OP_ir v25.4s, v1.4s, v8.s[2]
OP_rr v26.4s, v2.4s, v8.4s[2] OP_rr v26.4s, v2.4s, v8.s[2]
OP_ii v26.4s, v3.4s, v9.4s[2] OP_ii v26.4s, v3.4s, v9.s[2]
OP_ri v27.4s, v2.4s, v9.4s[2] OP_ri v27.4s, v2.4s, v9.s[2]
OP_ir v27.4s, v3.4s, v8.4s[2] OP_ir v27.4s, v3.4s, v8.s[2]
OP_rr v28.4s, v0.4s, v8.4s[3] OP_rr v28.4s, v0.4s, v8.s[3]
OP_ii v28.4s, v1.4s, v9.4s[3] OP_ii v28.4s, v1.4s, v9.s[3]
OP_ri v29.4s, v0.4s, v9.4s[3] OP_ri v29.4s, v0.4s, v9.s[3]
OP_ir v29.4s, v1.4s, v8.4s[3] OP_ir v29.4s, v1.4s, v8.s[3]
OP_rr v30.4s, v2.4s, v8.4s[3] OP_rr v30.4s, v2.4s, v8.s[3]
OP_ii v30.4s, v3.4s, v9.4s[3] OP_ii v30.4s, v3.4s, v9.s[3]
OP_ri v31.4s, v2.4s, v9.4s[3] OP_ri v31.4s, v2.4s, v9.s[3]
OP_ir v31.4s, v3.4s, v8.4s[3] OP_ir v31.4s, v3.4s, v8.s[3]
ld2 {v12.4s, v13.4s}, [pB] // For next round ld2 {v12.4s, v13.4s}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
@ -324,45 +324,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x4_M2 .macro KERNEL8x4_M2
OP_rr v16.4s, v4.4s, v12.4s[0] OP_rr v16.4s, v4.4s, v12.s[0]
OP_ii v16.4s, v5.4s, v13.4s[0] OP_ii v16.4s, v5.4s, v13.s[0]
OP_ri v17.4s, v4.4s, v13.4s[0] OP_ri v17.4s, v4.4s, v13.s[0]
OP_ir v17.4s, v5.4s, v12.4s[0] OP_ir v17.4s, v5.4s, v12.s[0]
OP_rr v18.4s, v6.4s, v12.4s[0] OP_rr v18.4s, v6.4s, v12.s[0]
OP_ii v18.4s, v7.4s, v13.4s[0] OP_ii v18.4s, v7.4s, v13.s[0]
OP_ri v19.4s, v6.4s, v13.4s[0] OP_ri v19.4s, v6.4s, v13.s[0]
OP_ir v19.4s, v7.4s, v12.4s[0] OP_ir v19.4s, v7.4s, v12.s[0]
OP_rr v20.4s, v4.4s, v12.4s[1] OP_rr v20.4s, v4.4s, v12.s[1]
OP_ii v20.4s, v5.4s, v13.4s[1] OP_ii v20.4s, v5.4s, v13.s[1]
OP_ri v21.4s, v4.4s, v13.4s[1] OP_ri v21.4s, v4.4s, v13.s[1]
OP_ir v21.4s, v5.4s, v12.4s[1] OP_ir v21.4s, v5.4s, v12.s[1]
OP_rr v22.4s, v6.4s, v12.4s[1] OP_rr v22.4s, v6.4s, v12.s[1]
OP_ii v22.4s, v7.4s, v13.4s[1] OP_ii v22.4s, v7.4s, v13.s[1]
OP_ri v23.4s, v6.4s, v13.4s[1] OP_ri v23.4s, v6.4s, v13.s[1]
OP_ir v23.4s, v7.4s, v12.4s[1] OP_ir v23.4s, v7.4s, v12.s[1]
OP_rr v24.4s, v4.4s, v12.4s[2] OP_rr v24.4s, v4.4s, v12.s[2]
OP_ii v24.4s, v5.4s, v13.4s[2] OP_ii v24.4s, v5.4s, v13.s[2]
OP_ri v25.4s, v4.4s, v13.4s[2] OP_ri v25.4s, v4.4s, v13.s[2]
OP_ir v25.4s, v5.4s, v12.4s[2] OP_ir v25.4s, v5.4s, v12.s[2]
OP_rr v26.4s, v6.4s, v12.4s[2] OP_rr v26.4s, v6.4s, v12.s[2]
OP_ii v26.4s, v7.4s, v13.4s[2] OP_ii v26.4s, v7.4s, v13.s[2]
OP_ri v27.4s, v6.4s, v13.4s[2] OP_ri v27.4s, v6.4s, v13.s[2]
OP_ir v27.4s, v7.4s, v12.4s[2] OP_ir v27.4s, v7.4s, v12.s[2]
OP_rr v28.4s, v4.4s, v12.4s[3] OP_rr v28.4s, v4.4s, v12.s[3]
OP_ii v28.4s, v5.4s, v13.4s[3] OP_ii v28.4s, v5.4s, v13.s[3]
OP_ri v29.4s, v4.4s, v13.4s[3] OP_ri v29.4s, v4.4s, v13.s[3]
OP_ir v29.4s, v5.4s, v12.4s[3] OP_ir v29.4s, v5.4s, v12.s[3]
OP_rr v30.4s, v6.4s, v12.4s[3] OP_rr v30.4s, v6.4s, v12.s[3]
OP_ii v30.4s, v7.4s, v13.4s[3] OP_ii v30.4s, v7.4s, v13.s[3]
OP_ri v31.4s, v6.4s, v13.4s[3] OP_ri v31.4s, v6.4s, v13.s[3]
OP_ir v31.4s, v7.4s, v12.4s[3] OP_ir v31.4s, v7.4s, v12.s[3]
ld2 {v8.4s, v9.4s}, [pB] ld2 {v8.4s, v9.4s}, [pB]
add pB, pB, #32 add pB, pB, #32
@ -373,45 +373,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x4_E .macro KERNEL8x4_E
OP_rr v16.4s, v4.4s, v12.4s[0] OP_rr v16.4s, v4.4s, v12.s[0]
OP_ii v16.4s, v5.4s, v13.4s[0] OP_ii v16.4s, v5.4s, v13.s[0]
OP_ri v17.4s, v4.4s, v13.4s[0] OP_ri v17.4s, v4.4s, v13.s[0]
OP_ir v17.4s, v5.4s, v12.4s[0] OP_ir v17.4s, v5.4s, v12.s[0]
OP_rr v18.4s, v6.4s, v12.4s[0] OP_rr v18.4s, v6.4s, v12.s[0]
OP_ii v18.4s, v7.4s, v13.4s[0] OP_ii v18.4s, v7.4s, v13.s[0]
OP_ri v19.4s, v6.4s, v13.4s[0] OP_ri v19.4s, v6.4s, v13.s[0]
OP_ir v19.4s, v7.4s, v12.4s[0] OP_ir v19.4s, v7.4s, v12.s[0]
OP_rr v20.4s, v4.4s, v12.4s[1] OP_rr v20.4s, v4.4s, v12.s[1]
OP_ii v20.4s, v5.4s, v13.4s[1] OP_ii v20.4s, v5.4s, v13.s[1]
OP_ri v21.4s, v4.4s, v13.4s[1] OP_ri v21.4s, v4.4s, v13.s[1]
OP_ir v21.4s, v5.4s, v12.4s[1] OP_ir v21.4s, v5.4s, v12.s[1]
OP_rr v22.4s, v6.4s, v12.4s[1] OP_rr v22.4s, v6.4s, v12.s[1]
OP_ii v22.4s, v7.4s, v13.4s[1] OP_ii v22.4s, v7.4s, v13.s[1]
OP_ri v23.4s, v6.4s, v13.4s[1] OP_ri v23.4s, v6.4s, v13.s[1]
OP_ir v23.4s, v7.4s, v12.4s[1] OP_ir v23.4s, v7.4s, v12.s[1]
OP_rr v24.4s, v4.4s, v12.4s[2] OP_rr v24.4s, v4.4s, v12.s[2]
OP_ii v24.4s, v5.4s, v13.4s[2] OP_ii v24.4s, v5.4s, v13.s[2]
OP_ri v25.4s, v4.4s, v13.4s[2] OP_ri v25.4s, v4.4s, v13.s[2]
OP_ir v25.4s, v5.4s, v12.4s[2] OP_ir v25.4s, v5.4s, v12.s[2]
OP_rr v26.4s, v6.4s, v12.4s[2] OP_rr v26.4s, v6.4s, v12.s[2]
OP_ii v26.4s, v7.4s, v13.4s[2] OP_ii v26.4s, v7.4s, v13.s[2]
OP_ri v27.4s, v6.4s, v13.4s[2] OP_ri v27.4s, v6.4s, v13.s[2]
OP_ir v27.4s, v7.4s, v12.4s[2] OP_ir v27.4s, v7.4s, v12.s[2]
OP_rr v28.4s, v4.4s, v12.4s[3] OP_rr v28.4s, v4.4s, v12.s[3]
OP_ii v28.4s, v5.4s, v13.4s[3] OP_ii v28.4s, v5.4s, v13.s[3]
OP_ri v29.4s, v4.4s, v13.4s[3] OP_ri v29.4s, v4.4s, v13.s[3]
OP_ir v29.4s, v5.4s, v12.4s[3] OP_ir v29.4s, v5.4s, v12.s[3]
OP_rr v30.4s, v6.4s, v12.4s[3] OP_rr v30.4s, v6.4s, v12.s[3]
OP_ii v30.4s, v7.4s, v13.4s[3] OP_ii v30.4s, v7.4s, v13.s[3]
OP_ri v31.4s, v6.4s, v13.4s[3] OP_ri v31.4s, v6.4s, v13.s[3]
OP_ir v31.4s, v7.4s, v12.4s[3] OP_ir v31.4s, v7.4s, v12.s[3]
.endm .endm
@ -423,45 +423,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v2.4s, v3.4s}, [pA] ld2 {v2.4s, v3.4s}, [pA]
add pA, pA, #32 add pA, pA, #32
OP_rr v16.4s, v0.4s, v8.4s[0] OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.4s[0] OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.4s[0] OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.4s[0] OP_ir v17.4s, v1.4s, v8.s[0]
OP_rr v18.4s, v2.4s, v8.4s[0] OP_rr v18.4s, v2.4s, v8.s[0]
OP_ii v18.4s, v3.4s, v9.4s[0] OP_ii v18.4s, v3.4s, v9.s[0]
OP_ri v19.4s, v2.4s, v9.4s[0] OP_ri v19.4s, v2.4s, v9.s[0]
OP_ir v19.4s, v3.4s, v8.4s[0] OP_ir v19.4s, v3.4s, v8.s[0]
OP_rr v20.4s, v0.4s, v8.4s[1] OP_rr v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.4s[1] OP_ii v20.4s, v1.4s, v9.s[1]
OP_ri v21.4s, v0.4s, v9.4s[1] OP_ri v21.4s, v0.4s, v9.s[1]
OP_ir v21.4s, v1.4s, v8.4s[1] OP_ir v21.4s, v1.4s, v8.s[1]
OP_rr v22.4s, v2.4s, v8.4s[1] OP_rr v22.4s, v2.4s, v8.s[1]
OP_ii v22.4s, v3.4s, v9.4s[1] OP_ii v22.4s, v3.4s, v9.s[1]
OP_ri v23.4s, v2.4s, v9.4s[1] OP_ri v23.4s, v2.4s, v9.s[1]
OP_ir v23.4s, v3.4s, v8.4s[1] OP_ir v23.4s, v3.4s, v8.s[1]
OP_rr v24.4s, v0.4s, v8.4s[2] OP_rr v24.4s, v0.4s, v8.s[2]
OP_ii v24.4s, v1.4s, v9.4s[2] OP_ii v24.4s, v1.4s, v9.s[2]
OP_ri v25.4s, v0.4s, v9.4s[2] OP_ri v25.4s, v0.4s, v9.s[2]
OP_ir v25.4s, v1.4s, v8.4s[2] OP_ir v25.4s, v1.4s, v8.s[2]
OP_rr v26.4s, v2.4s, v8.4s[2] OP_rr v26.4s, v2.4s, v8.s[2]
OP_ii v26.4s, v3.4s, v9.4s[2] OP_ii v26.4s, v3.4s, v9.s[2]
OP_ri v27.4s, v2.4s, v9.4s[2] OP_ri v27.4s, v2.4s, v9.s[2]
OP_ir v27.4s, v3.4s, v8.4s[2] OP_ir v27.4s, v3.4s, v8.s[2]
OP_rr v28.4s, v0.4s, v8.4s[3] OP_rr v28.4s, v0.4s, v8.s[3]
OP_ii v28.4s, v1.4s, v9.4s[3] OP_ii v28.4s, v1.4s, v9.s[3]
OP_ri v29.4s, v0.4s, v9.4s[3] OP_ri v29.4s, v0.4s, v9.s[3]
OP_ir v29.4s, v1.4s, v8.4s[3] OP_ir v29.4s, v1.4s, v8.s[3]
OP_rr v30.4s, v2.4s, v8.4s[3] OP_rr v30.4s, v2.4s, v8.s[3]
OP_ii v30.4s, v3.4s, v9.4s[3] OP_ii v30.4s, v3.4s, v9.s[3]
OP_ri v31.4s, v2.4s, v9.4s[3] OP_ri v31.4s, v2.4s, v9.s[3]
OP_ir v31.4s, v3.4s, v8.4s[3] OP_ir v31.4s, v3.4s, v8.s[3]
.endm .endm
@ -560,49 +560,49 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.4s, v1.4s}, [pA] ld2 {v0.4s, v1.4s}, [pA]
add pA, pA, #32 add pA, pA, #32
fmul v16.4s, v0.4s, v8.4s[0] fmul v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.4s[0] OP_ii v16.4s, v1.4s, v9.s[0]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v17.16b, v17.16b, v17.16b eor v17.16b, v17.16b, v17.16b
fmls v17.4s, v0.4s, v9.4s[0] fmls v17.4s, v0.4s, v9.s[0]
#else #else
fmul v17.4s, v0.4s, v9.4s[0] fmul v17.4s, v0.4s, v9.s[0]
#endif #endif
OP_ir v17.4s, v1.4s, v8.4s[0] OP_ir v17.4s, v1.4s, v8.s[0]
fmul v20.4s, v0.4s, v8.4s[1] fmul v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.4s[1] OP_ii v20.4s, v1.4s, v9.s[1]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v21.16b, v21.16b, v21.16b eor v21.16b, v21.16b, v21.16b
fmls v21.4s, v0.4s, v9.4s[1] fmls v21.4s, v0.4s, v9.s[1]
#else #else
fmul v21.4s, v0.4s, v9.4s[1] fmul v21.4s, v0.4s, v9.s[1]
#endif #endif
OP_ir v21.4s, v1.4s, v8.4s[1] OP_ir v21.4s, v1.4s, v8.s[1]
fmul v24.4s, v0.4s, v8.4s[2] fmul v24.4s, v0.4s, v8.s[2]
OP_ii v24.4s, v1.4s, v9.4s[2] OP_ii v24.4s, v1.4s, v9.s[2]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v25.16b, v25.16b, v25.16b eor v25.16b, v25.16b, v25.16b
fmls v25.4s, v0.4s, v9.4s[2] fmls v25.4s, v0.4s, v9.s[2]
#else #else
fmul v25.4s, v0.4s, v9.4s[2] fmul v25.4s, v0.4s, v9.s[2]
#endif #endif
OP_ir v25.4s, v1.4s, v8.4s[2] OP_ir v25.4s, v1.4s, v8.s[2]
fmul v28.4s, v0.4s, v8.4s[3] fmul v28.4s, v0.4s, v8.s[3]
OP_ii v28.4s, v1.4s, v9.4s[3] OP_ii v28.4s, v1.4s, v9.s[3]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v29.16b, v29.16b, v29.16b eor v29.16b, v29.16b, v29.16b
fmls v29.4s, v0.4s, v9.4s[3] fmls v29.4s, v0.4s, v9.s[3]
#else #else
fmul v29.4s, v0.4s, v9.4s[3] fmul v29.4s, v0.4s, v9.s[3]
#endif #endif
OP_ir v29.4s, v1.4s, v8.4s[3] OP_ir v29.4s, v1.4s, v8.s[3]
ld2 {v12.4s, v13.4s}, [pB] ld2 {v12.4s, v13.4s}, [pB]
add pB, pB, #32 add pB, pB, #32
@ -611,85 +611,85 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL4x4_M1 .macro KERNEL4x4_M1
OP_rr v16.4s, v0.4s, v8.4s[0] OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.4s[0] OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.4s[0] OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.4s[0] OP_ir v17.4s, v1.4s, v8.s[0]
ld2 {v12.4s, v13.4s}, [pB] // For next round ld2 {v12.4s, v13.4s}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
OP_rr v20.4s, v0.4s, v8.4s[1] OP_rr v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.4s[1] OP_ii v20.4s, v1.4s, v9.s[1]
OP_ri v21.4s, v0.4s, v9.4s[1] OP_ri v21.4s, v0.4s, v9.s[1]
OP_ir v21.4s, v1.4s, v8.4s[1] OP_ir v21.4s, v1.4s, v8.s[1]
ld2 {v4.4s, v5.4s}, [pA] // For next round ld2 {v4.4s, v5.4s}, [pA] // For next round
add pA, pA, #32 add pA, pA, #32
OP_rr v24.4s, v0.4s, v8.4s[2] OP_rr v24.4s, v0.4s, v8.s[2]
OP_ii v24.4s, v1.4s, v9.4s[2] OP_ii v24.4s, v1.4s, v9.s[2]
OP_ri v25.4s, v0.4s, v9.4s[2] OP_ri v25.4s, v0.4s, v9.s[2]
OP_ir v25.4s, v1.4s, v8.4s[2] OP_ir v25.4s, v1.4s, v8.s[2]
prfm PLDL1KEEP, [pA, #512] prfm PLDL1KEEP, [pA, #512]
OP_rr v28.4s, v0.4s, v8.4s[3] OP_rr v28.4s, v0.4s, v8.s[3]
OP_ii v28.4s, v1.4s, v9.4s[3] OP_ii v28.4s, v1.4s, v9.s[3]
OP_ri v29.4s, v0.4s, v9.4s[3] OP_ri v29.4s, v0.4s, v9.s[3]
OP_ir v29.4s, v1.4s, v8.4s[3] OP_ir v29.4s, v1.4s, v8.s[3]
.endm .endm
.macro KERNEL4x4_M2 .macro KERNEL4x4_M2
OP_rr v16.4s, v4.4s, v12.4s[0] OP_rr v16.4s, v4.4s, v12.s[0]
OP_ii v16.4s, v5.4s, v13.4s[0] OP_ii v16.4s, v5.4s, v13.s[0]
OP_ri v17.4s, v4.4s, v13.4s[0] OP_ri v17.4s, v4.4s, v13.s[0]
OP_ir v17.4s, v5.4s, v12.4s[0] OP_ir v17.4s, v5.4s, v12.s[0]
ld2 {v8.4s, v9.4s}, [pB] // For next round ld2 {v8.4s, v9.4s}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
OP_rr v20.4s, v4.4s, v12.4s[1] OP_rr v20.4s, v4.4s, v12.s[1]
OP_ii v20.4s, v5.4s, v13.4s[1] OP_ii v20.4s, v5.4s, v13.s[1]
OP_ri v21.4s, v4.4s, v13.4s[1] OP_ri v21.4s, v4.4s, v13.s[1]
OP_ir v21.4s, v5.4s, v12.4s[1] OP_ir v21.4s, v5.4s, v12.s[1]
ld2 {v0.4s, v1.4s}, [pA] // For next round ld2 {v0.4s, v1.4s}, [pA] // For next round
add pA, pA, #32 add pA, pA, #32
OP_rr v24.4s, v4.4s, v12.4s[2] OP_rr v24.4s, v4.4s, v12.s[2]
OP_ii v24.4s, v5.4s, v13.4s[2] OP_ii v24.4s, v5.4s, v13.s[2]
OP_ri v25.4s, v4.4s, v13.4s[2] OP_ri v25.4s, v4.4s, v13.s[2]
OP_ir v25.4s, v5.4s, v12.4s[2] OP_ir v25.4s, v5.4s, v12.s[2]
prfm PLDL1KEEP, [pB, #512] prfm PLDL1KEEP, [pB, #512]
OP_rr v28.4s, v4.4s, v12.4s[3] OP_rr v28.4s, v4.4s, v12.s[3]
OP_ii v28.4s, v5.4s, v13.4s[3] OP_ii v28.4s, v5.4s, v13.s[3]
OP_ri v29.4s, v4.4s, v13.4s[3] OP_ri v29.4s, v4.4s, v13.s[3]
OP_ir v29.4s, v5.4s, v12.4s[3] OP_ir v29.4s, v5.4s, v12.s[3]
.endm .endm
.macro KERNEL4x4_E .macro KERNEL4x4_E
OP_rr v16.4s, v4.4s, v12.4s[0] OP_rr v16.4s, v4.4s, v12.s[0]
OP_ii v16.4s, v5.4s, v13.4s[0] OP_ii v16.4s, v5.4s, v13.s[0]
OP_ri v17.4s, v4.4s, v13.4s[0] OP_ri v17.4s, v4.4s, v13.s[0]
OP_ir v17.4s, v5.4s, v12.4s[0] OP_ir v17.4s, v5.4s, v12.s[0]
OP_rr v20.4s, v4.4s, v12.4s[1] OP_rr v20.4s, v4.4s, v12.s[1]
OP_ii v20.4s, v5.4s, v13.4s[1] OP_ii v20.4s, v5.4s, v13.s[1]
OP_ri v21.4s, v4.4s, v13.4s[1] OP_ri v21.4s, v4.4s, v13.s[1]
OP_ir v21.4s, v5.4s, v12.4s[1] OP_ir v21.4s, v5.4s, v12.s[1]
OP_rr v24.4s, v4.4s, v12.4s[2] OP_rr v24.4s, v4.4s, v12.s[2]
OP_ii v24.4s, v5.4s, v13.4s[2] OP_ii v24.4s, v5.4s, v13.s[2]
OP_ri v25.4s, v4.4s, v13.4s[2] OP_ri v25.4s, v4.4s, v13.s[2]
OP_ir v25.4s, v5.4s, v12.4s[2] OP_ir v25.4s, v5.4s, v12.s[2]
OP_rr v28.4s, v4.4s, v12.4s[3] OP_rr v28.4s, v4.4s, v12.s[3]
OP_ii v28.4s, v5.4s, v13.4s[3] OP_ii v28.4s, v5.4s, v13.s[3]
OP_ri v29.4s, v4.4s, v13.4s[3] OP_ri v29.4s, v4.4s, v13.s[3]
OP_ir v29.4s, v5.4s, v12.4s[3] OP_ir v29.4s, v5.4s, v12.s[3]
.endm .endm
.macro KERNEL4x4_SUB .macro KERNEL4x4_SUB
@ -698,25 +698,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.4s, v1.4s}, [pA] ld2 {v0.4s, v1.4s}, [pA]
add pA, pA, #32 add pA, pA, #32
OP_rr v16.4s, v0.4s, v8.4s[0] OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.4s[0] OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.4s[0] OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.4s[0] OP_ir v17.4s, v1.4s, v8.s[0]
OP_rr v20.4s, v0.4s, v8.4s[1] OP_rr v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.4s[1] OP_ii v20.4s, v1.4s, v9.s[1]
OP_ri v21.4s, v0.4s, v9.4s[1] OP_ri v21.4s, v0.4s, v9.s[1]
OP_ir v21.4s, v1.4s, v8.4s[1] OP_ir v21.4s, v1.4s, v8.s[1]
OP_rr v24.4s, v0.4s, v8.4s[2] OP_rr v24.4s, v0.4s, v8.s[2]
OP_ii v24.4s, v1.4s, v9.4s[2] OP_ii v24.4s, v1.4s, v9.s[2]
OP_ri v25.4s, v0.4s, v9.4s[2] OP_ri v25.4s, v0.4s, v9.s[2]
OP_ir v25.4s, v1.4s, v8.4s[2] OP_ir v25.4s, v1.4s, v8.s[2]
OP_rr v28.4s, v0.4s, v8.4s[3] OP_rr v28.4s, v0.4s, v8.s[3]
OP_ii v28.4s, v1.4s, v9.4s[3] OP_ii v28.4s, v1.4s, v9.s[3]
OP_ri v29.4s, v0.4s, v9.4s[3] OP_ri v29.4s, v0.4s, v9.s[3]
OP_ir v29.4s, v1.4s, v8.4s[3] OP_ir v29.4s, v1.4s, v8.s[3]
.endm .endm
.macro SAVE4x4 .macro SAVE4x4
@ -778,25 +778,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.2s, v1.2s}, [pA] ld2 {v0.2s, v1.2s}, [pA]
add pA, pA, #16 add pA, pA, #16
OP_rr v16.2s, v0.2s, v8.4s[0] OP_rr v16.2s, v0.2s, v8.s[0]
OP_ii v16.2s, v1.2s, v9.4s[0] OP_ii v16.2s, v1.2s, v9.s[0]
OP_ri v17.2s, v0.2s, v9.4s[0] OP_ri v17.2s, v0.2s, v9.s[0]
OP_ir v17.2s, v1.2s, v8.4s[0] OP_ir v17.2s, v1.2s, v8.s[0]
OP_rr v20.2s, v0.2s, v8.4s[1] OP_rr v20.2s, v0.2s, v8.s[1]
OP_ii v20.2s, v1.2s, v9.4s[1] OP_ii v20.2s, v1.2s, v9.s[1]
OP_ri v21.2s, v0.2s, v9.4s[1] OP_ri v21.2s, v0.2s, v9.s[1]
OP_ir v21.2s, v1.2s, v8.4s[1] OP_ir v21.2s, v1.2s, v8.s[1]
OP_rr v24.2s, v0.2s, v8.4s[2] OP_rr v24.2s, v0.2s, v8.s[2]
OP_ii v24.2s, v1.2s, v9.4s[2] OP_ii v24.2s, v1.2s, v9.s[2]
OP_ri v25.2s, v0.2s, v9.4s[2] OP_ri v25.2s, v0.2s, v9.s[2]
OP_ir v25.2s, v1.2s, v8.4s[2] OP_ir v25.2s, v1.2s, v8.s[2]
OP_rr v28.2s, v0.2s, v8.4s[3] OP_rr v28.2s, v0.2s, v8.s[3]
OP_ii v28.2s, v1.2s, v9.4s[3] OP_ii v28.2s, v1.2s, v9.s[3]
OP_ri v29.2s, v0.2s, v9.4s[3] OP_ri v29.2s, v0.2s, v9.s[3]
OP_ir v29.2s, v1.2s, v8.4s[3] OP_ir v29.2s, v1.2s, v8.s[3]
.endm .endm
.macro SAVE2x4 .macro SAVE2x4
@ -858,25 +858,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.s, v1.s}[0], [pA] ld2 {v0.s, v1.s}[0], [pA]
add pA, pA, #8 add pA, pA, #8
OP_rr s16, s0, v8.4s[0] OP_rr s16, s0, v8.s[0]
OP_ii s16, s1, v9.4s[0] OP_ii s16, s1, v9.s[0]
OP_ri s17, s0, v9.4s[0] OP_ri s17, s0, v9.s[0]
OP_ir s17, s1, v8.4s[0] OP_ir s17, s1, v8.s[0]
OP_rr s20, s0, v8.4s[1] OP_rr s20, s0, v8.s[1]
OP_ii s20, s1, v9.4s[1] OP_ii s20, s1, v9.s[1]
OP_ri s21, s0, v9.4s[1] OP_ri s21, s0, v9.s[1]
OP_ir s21, s1, v8.4s[1] OP_ir s21, s1, v8.s[1]
OP_rr s24, s0, v8.4s[2] OP_rr s24, s0, v8.s[2]
OP_ii s24, s1, v9.4s[2] OP_ii s24, s1, v9.s[2]
OP_ri s25, s0, v9.4s[2] OP_ri s25, s0, v9.s[2]
OP_ir s25, s1, v8.4s[2] OP_ir s25, s1, v8.s[2]
OP_rr s28, s0, v8.4s[3] OP_rr s28, s0, v8.s[3]
OP_ii s28, s1, v9.4s[3] OP_ii s28, s1, v9.s[3]
OP_ri s29, s0, v9.4s[3] OP_ri s29, s0, v9.s[3]
OP_ir s29, s1, v8.4s[3] OP_ir s29, s1, v8.s[3]
.endm .endm
.macro SAVE1x4 .macro SAVE1x4
@ -940,25 +940,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v2.4s, v3.4s}, [pA] ld2 {v2.4s, v3.4s}, [pA]
add pA, pA, #32 add pA, pA, #32
OP_rr v16.4s, v0.4s, v8.2s[0] OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.2s[0] OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.2s[0] OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.2s[0] OP_ir v17.4s, v1.4s, v8.s[0]
OP_rr v18.4s, v2.4s, v8.2s[0] OP_rr v18.4s, v2.4s, v8.s[0]
OP_ii v18.4s, v3.4s, v9.2s[0] OP_ii v18.4s, v3.4s, v9.s[0]
OP_ri v19.4s, v2.4s, v9.2s[0] OP_ri v19.4s, v2.4s, v9.s[0]
OP_ir v19.4s, v3.4s, v8.2s[0] OP_ir v19.4s, v3.4s, v8.s[0]
OP_rr v20.4s, v0.4s, v8.2s[1] OP_rr v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.2s[1] OP_ii v20.4s, v1.4s, v9.s[1]
OP_ri v21.4s, v0.4s, v9.2s[1] OP_ri v21.4s, v0.4s, v9.s[1]
OP_ir v21.4s, v1.4s, v8.2s[1] OP_ir v21.4s, v1.4s, v8.s[1]
OP_rr v22.4s, v2.4s, v8.2s[1] OP_rr v22.4s, v2.4s, v8.s[1]
OP_ii v22.4s, v3.4s, v9.2s[1] OP_ii v22.4s, v3.4s, v9.s[1]
OP_ri v23.4s, v2.4s, v9.2s[1] OP_ri v23.4s, v2.4s, v9.s[1]
OP_ir v23.4s, v3.4s, v8.2s[1] OP_ir v23.4s, v3.4s, v8.s[1]
.endm .endm
.macro SAVE8x2 .macro SAVE8x2
@ -1016,15 +1016,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.4s, v1.4s}, [pA] ld2 {v0.4s, v1.4s}, [pA]
add pA, pA, #32 add pA, pA, #32
OP_rr v16.4s, v0.4s, v8.2s[0] OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.2s[0] OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.2s[0] OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.2s[0] OP_ir v17.4s, v1.4s, v8.s[0]
OP_rr v20.4s, v0.4s, v8.2s[1] OP_rr v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.2s[1] OP_ii v20.4s, v1.4s, v9.s[1]
OP_ri v21.4s, v0.4s, v9.2s[1] OP_ri v21.4s, v0.4s, v9.s[1]
OP_ir v21.4s, v1.4s, v8.2s[1] OP_ir v21.4s, v1.4s, v8.s[1]
.endm .endm
.macro SAVE4x2 .macro SAVE4x2
@ -1064,15 +1064,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.2s, v1.2s}, [pA] ld2 {v0.2s, v1.2s}, [pA]
add pA, pA, #16 add pA, pA, #16
OP_rr v16.2s, v0.2s, v8.2s[0] OP_rr v16.2s, v0.2s, v8.s[0]
OP_ii v16.2s, v1.2s, v9.2s[0] OP_ii v16.2s, v1.2s, v9.s[0]
OP_ri v17.2s, v0.2s, v9.2s[0] OP_ri v17.2s, v0.2s, v9.s[0]
OP_ir v17.2s, v1.2s, v8.2s[0] OP_ir v17.2s, v1.2s, v8.s[0]
OP_rr v20.2s, v0.2s, v8.2s[1] OP_rr v20.2s, v0.2s, v8.s[1]
OP_ii v20.2s, v1.2s, v9.2s[1] OP_ii v20.2s, v1.2s, v9.s[1]
OP_ri v21.2s, v0.2s, v9.2s[1] OP_ri v21.2s, v0.2s, v9.s[1]
OP_ir v21.2s, v1.2s, v8.2s[1] OP_ir v21.2s, v1.2s, v8.s[1]
.endm .endm
.macro SAVE2x2 .macro SAVE2x2
@ -1112,15 +1112,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.s, v1.s}[0], [pA] ld2 {v0.s, v1.s}[0], [pA]
add pA, pA, #8 add pA, pA, #8
OP_rr s16, s0, v8.2s[0] OP_rr s16, s0, v8.s[0]
OP_ii s16, s1, v9.2s[0] OP_ii s16, s1, v9.s[0]
OP_ri s17, s0, v9.2s[0] OP_ri s17, s0, v9.s[0]
OP_ir s17, s1, v8.2s[0] OP_ir s17, s1, v8.s[0]
OP_rr s20, s0, v8.2s[1] OP_rr s20, s0, v8.s[1]
OP_ii s20, s1, v9.2s[1] OP_ii s20, s1, v9.s[1]
OP_ri s21, s0, v9.2s[1] OP_ri s21, s0, v9.s[1]
OP_ir s21, s1, v8.2s[1] OP_ir s21, s1, v8.s[1]
.endm .endm
.macro SAVE1x2 .macro SAVE1x2
@ -1162,15 +1162,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v2.4s, v3.4s}, [pA] ld2 {v2.4s, v3.4s}, [pA]
add pA, pA, #32 add pA, pA, #32
OP_rr v16.4s, v0.4s, v8.4s[0] OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v8.4s[1] OP_ii v16.4s, v1.4s, v8.s[1]
OP_ri v17.4s, v0.4s, v8.4s[1] OP_ri v17.4s, v0.4s, v8.s[1]
OP_ir v17.4s, v1.4s, v8.4s[0] OP_ir v17.4s, v1.4s, v8.s[0]
OP_rr v18.4s, v2.4s, v8.4s[0] OP_rr v18.4s, v2.4s, v8.s[0]
OP_ii v18.4s, v3.4s, v8.4s[1] OP_ii v18.4s, v3.4s, v8.s[1]
OP_ri v19.4s, v2.4s, v8.4s[1] OP_ri v19.4s, v2.4s, v8.s[1]
OP_ir v19.4s, v3.4s, v8.4s[0] OP_ir v19.4s, v3.4s, v8.s[0]
.endm .endm
.macro SAVE8x1 .macro SAVE8x1

280
kernel/arm64/ctrmm_kernel_4x4.S
View File

@ -170,49 +170,49 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.4s, v1.4s}, [pA] ld2 {v0.4s, v1.4s}, [pA]
add pA, pA, #32 add pA, pA, #32
fmul v16.4s, v0.4s, v8.4s[0] fmul v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.4s[0] OP_ii v16.4s, v1.4s, v9.s[0]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v17.16b, v17.16b, v17.16b eor v17.16b, v17.16b, v17.16b
fmls v17.4s, v0.4s, v9.4s[0] fmls v17.4s, v0.4s, v9.s[0]
#else #else
fmul v17.4s, v0.4s, v9.4s[0] fmul v17.4s, v0.4s, v9.s[0]
#endif #endif
OP_ir v17.4s, v1.4s, v8.4s[0] OP_ir v17.4s, v1.4s, v8.s[0]
fmul v20.4s, v0.4s, v8.4s[1] fmul v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.4s[1] OP_ii v20.4s, v1.4s, v9.s[1]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v21.16b, v21.16b, v21.16b eor v21.16b, v21.16b, v21.16b
fmls v21.4s, v0.4s, v9.4s[1] fmls v21.4s, v0.4s, v9.s[1]
#else #else
fmul v21.4s, v0.4s, v9.4s[1] fmul v21.4s, v0.4s, v9.s[1]
#endif #endif
OP_ir v21.4s, v1.4s, v8.4s[1] OP_ir v21.4s, v1.4s, v8.s[1]
fmul v24.4s, v0.4s, v8.4s[2] fmul v24.4s, v0.4s, v8.s[2]
OP_ii v24.4s, v1.4s, v9.4s[2] OP_ii v24.4s, v1.4s, v9.s[2]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v25.16b, v25.16b, v25.16b eor v25.16b, v25.16b, v25.16b
fmls v25.4s, v0.4s, v9.4s[2] fmls v25.4s, v0.4s, v9.s[2]
#else #else
fmul v25.4s, v0.4s, v9.4s[2] fmul v25.4s, v0.4s, v9.s[2]
#endif #endif
OP_ir v25.4s, v1.4s, v8.4s[2] OP_ir v25.4s, v1.4s, v8.s[2]
fmul v28.4s, v0.4s, v8.4s[3] fmul v28.4s, v0.4s, v8.s[3]
OP_ii v28.4s, v1.4s, v9.4s[3] OP_ii v28.4s, v1.4s, v9.s[3]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v29.16b, v29.16b, v29.16b eor v29.16b, v29.16b, v29.16b
fmls v29.4s, v0.4s, v9.4s[3] fmls v29.4s, v0.4s, v9.s[3]
#else #else
fmul v29.4s, v0.4s, v9.4s[3] fmul v29.4s, v0.4s, v9.s[3]
#endif #endif
OP_ir v29.4s, v1.4s, v8.4s[3] OP_ir v29.4s, v1.4s, v8.s[3]
ld2 {v12.4s, v13.4s}, [pB] ld2 {v12.4s, v13.4s}, [pB]
add pB, pB, #32 add pB, pB, #32
@ -221,85 +221,85 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL4x4_M1 .macro KERNEL4x4_M1
OP_rr v16.4s, v0.4s, v8.4s[0] OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.4s[0] OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.4s[0] OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.4s[0] OP_ir v17.4s, v1.4s, v8.s[0]
ld2 {v12.4s, v13.4s}, [pB] // For next round ld2 {v12.4s, v13.4s}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
OP_rr v20.4s, v0.4s, v8.4s[1] OP_rr v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.4s[1] OP_ii v20.4s, v1.4s, v9.s[1]
OP_ri v21.4s, v0.4s, v9.4s[1] OP_ri v21.4s, v0.4s, v9.s[1]
OP_ir v21.4s, v1.4s, v8.4s[1] OP_ir v21.4s, v1.4s, v8.s[1]
ld2 {v4.4s, v5.4s}, [pA] // For next round ld2 {v4.4s, v5.4s}, [pA] // For next round
add pA, pA, #32 add pA, pA, #32
OP_rr v24.4s, v0.4s, v8.4s[2] OP_rr v24.4s, v0.4s, v8.s[2]
OP_ii v24.4s, v1.4s, v9.4s[2] OP_ii v24.4s, v1.4s, v9.s[2]
OP_ri v25.4s, v0.4s, v9.4s[2] OP_ri v25.4s, v0.4s, v9.s[2]
OP_ir v25.4s, v1.4s, v8.4s[2] OP_ir v25.4s, v1.4s, v8.s[2]
prfm PLDL1KEEP, [pA, #512] prfm PLDL1KEEP, [pA, #512]
OP_rr v28.4s, v0.4s, v8.4s[3] OP_rr v28.4s, v0.4s, v8.s[3]
OP_ii v28.4s, v1.4s, v9.4s[3] OP_ii v28.4s, v1.4s, v9.s[3]
OP_ri v29.4s, v0.4s, v9.4s[3] OP_ri v29.4s, v0.4s, v9.s[3]
OP_ir v29.4s, v1.4s, v8.4s[3] OP_ir v29.4s, v1.4s, v8.s[3]
.endm .endm
.macro KERNEL4x4_M2 .macro KERNEL4x4_M2
OP_rr v16.4s, v4.4s, v12.4s[0] OP_rr v16.4s, v4.4s, v12.s[0]
OP_ii v16.4s, v5.4s, v13.4s[0] OP_ii v16.4s, v5.4s, v13.s[0]
OP_ri v17.4s, v4.4s, v13.4s[0] OP_ri v17.4s, v4.4s, v13.s[0]
OP_ir v17.4s, v5.4s, v12.4s[0] OP_ir v17.4s, v5.4s, v12.s[0]
ld2 {v8.4s, v9.4s}, [pB] // For next round ld2 {v8.4s, v9.4s}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
OP_rr v20.4s, v4.4s, v12.4s[1] OP_rr v20.4s, v4.4s, v12.s[1]
OP_ii v20.4s, v5.4s, v13.4s[1] OP_ii v20.4s, v5.4s, v13.s[1]
OP_ri v21.4s, v4.4s, v13.4s[1] OP_ri v21.4s, v4.4s, v13.s[1]
OP_ir v21.4s, v5.4s, v12.4s[1] OP_ir v21.4s, v5.4s, v12.s[1]
ld2 {v0.4s, v1.4s}, [pA] // For next round ld2 {v0.4s, v1.4s}, [pA] // For next round
add pA, pA, #32 add pA, pA, #32
OP_rr v24.4s, v4.4s, v12.4s[2] OP_rr v24.4s, v4.4s, v12.s[2]
OP_ii v24.4s, v5.4s, v13.4s[2] OP_ii v24.4s, v5.4s, v13.s[2]
OP_ri v25.4s, v4.4s, v13.4s[2] OP_ri v25.4s, v4.4s, v13.s[2]
OP_ir v25.4s, v5.4s, v12.4s[2] OP_ir v25.4s, v5.4s, v12.s[2]
prfm PLDL1KEEP, [pB, #512] prfm PLDL1KEEP, [pB, #512]
OP_rr v28.4s, v4.4s, v12.4s[3] OP_rr v28.4s, v4.4s, v12.s[3]
OP_ii v28.4s, v5.4s, v13.4s[3] OP_ii v28.4s, v5.4s, v13.s[3]
OP_ri v29.4s, v4.4s, v13.4s[3] OP_ri v29.4s, v4.4s, v13.s[3]
OP_ir v29.4s, v5.4s, v12.4s[3] OP_ir v29.4s, v5.4s, v12.s[3]
.endm .endm
.macro KERNEL4x4_E .macro KERNEL4x4_E
OP_rr v16.4s, v4.4s, v12.4s[0] OP_rr v16.4s, v4.4s, v12.s[0]
OP_ii v16.4s, v5.4s, v13.4s[0] OP_ii v16.4s, v5.4s, v13.s[0]
OP_ri v17.4s, v4.4s, v13.4s[0] OP_ri v17.4s, v4.4s, v13.s[0]
OP_ir v17.4s, v5.4s, v12.4s[0] OP_ir v17.4s, v5.4s, v12.s[0]
OP_rr v20.4s, v4.4s, v12.4s[1] OP_rr v20.4s, v4.4s, v12.s[1]
OP_ii v20.4s, v5.4s, v13.4s[1] OP_ii v20.4s, v5.4s, v13.s[1]
OP_ri v21.4s, v4.4s, v13.4s[1] OP_ri v21.4s, v4.4s, v13.s[1]
OP_ir v21.4s, v5.4s, v12.4s[1] OP_ir v21.4s, v5.4s, v12.s[1]
OP_rr v24.4s, v4.4s, v12.4s[2] OP_rr v24.4s, v4.4s, v12.s[2]
OP_ii v24.4s, v5.4s, v13.4s[2] OP_ii v24.4s, v5.4s, v13.s[2]
OP_ri v25.4s, v4.4s, v13.4s[2] OP_ri v25.4s, v4.4s, v13.s[2]
OP_ir v25.4s, v5.4s, v12.4s[2] OP_ir v25.4s, v5.4s, v12.s[2]
OP_rr v28.4s, v4.4s, v12.4s[3] OP_rr v28.4s, v4.4s, v12.s[3]
OP_ii v28.4s, v5.4s, v13.4s[3] OP_ii v28.4s, v5.4s, v13.s[3]
OP_ri v29.4s, v4.4s, v13.4s[3] OP_ri v29.4s, v4.4s, v13.s[3]
OP_ir v29.4s, v5.4s, v12.4s[3] OP_ir v29.4s, v5.4s, v12.s[3]
.endm .endm
.macro KERNEL4x4_SUB .macro KERNEL4x4_SUB
@ -308,25 +308,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.4s, v1.4s}, [pA] ld2 {v0.4s, v1.4s}, [pA]
add pA, pA, #32 add pA, pA, #32
OP_rr v16.4s, v0.4s, v8.4s[0] OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.4s[0] OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.4s[0] OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.4s[0] OP_ir v17.4s, v1.4s, v8.s[0]
OP_rr v20.4s, v0.4s, v8.4s[1] OP_rr v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.4s[1] OP_ii v20.4s, v1.4s, v9.s[1]
OP_ri v21.4s, v0.4s, v9.4s[1] OP_ri v21.4s, v0.4s, v9.s[1]
OP_ir v21.4s, v1.4s, v8.4s[1] OP_ir v21.4s, v1.4s, v8.s[1]
OP_rr v24.4s, v0.4s, v8.4s[2] OP_rr v24.4s, v0.4s, v8.s[2]
OP_ii v24.4s, v1.4s, v9.4s[2] OP_ii v24.4s, v1.4s, v9.s[2]
OP_ri v25.4s, v0.4s, v9.4s[2] OP_ri v25.4s, v0.4s, v9.s[2]
OP_ir v25.4s, v1.4s, v8.4s[2] OP_ir v25.4s, v1.4s, v8.s[2]
OP_rr v28.4s, v0.4s, v8.4s[3] OP_rr v28.4s, v0.4s, v8.s[3]
OP_ii v28.4s, v1.4s, v9.4s[3] OP_ii v28.4s, v1.4s, v9.s[3]
OP_ri v29.4s, v0.4s, v9.4s[3] OP_ri v29.4s, v0.4s, v9.s[3]
OP_ir v29.4s, v1.4s, v8.4s[3] OP_ir v29.4s, v1.4s, v8.s[3]
.endm .endm
.macro SAVE4x4 .macro SAVE4x4
@ -384,25 +384,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.2s, v1.2s}, [pA] ld2 {v0.2s, v1.2s}, [pA]
add pA, pA, #16 add pA, pA, #16
OP_rr v16.2s, v0.2s, v8.4s[0] OP_rr v16.2s, v0.2s, v8.s[0]
OP_ii v16.2s, v1.2s, v9.4s[0] OP_ii v16.2s, v1.2s, v9.s[0]
OP_ri v17.2s, v0.2s, v9.4s[0] OP_ri v17.2s, v0.2s, v9.s[0]
OP_ir v17.2s, v1.2s, v8.4s[0] OP_ir v17.2s, v1.2s, v8.s[0]
OP_rr v20.2s, v0.2s, v8.4s[1] OP_rr v20.2s, v0.2s, v8.s[1]
OP_ii v20.2s, v1.2s, v9.4s[1] OP_ii v20.2s, v1.2s, v9.s[1]
OP_ri v21.2s, v0.2s, v9.4s[1] OP_ri v21.2s, v0.2s, v9.s[1]
OP_ir v21.2s, v1.2s, v8.4s[1] OP_ir v21.2s, v1.2s, v8.s[1]
OP_rr v24.2s, v0.2s, v8.4s[2] OP_rr v24.2s, v0.2s, v8.s[2]
OP_ii v24.2s, v1.2s, v9.4s[2] OP_ii v24.2s, v1.2s, v9.s[2]
OP_ri v25.2s, v0.2s, v9.4s[2] OP_ri v25.2s, v0.2s, v9.s[2]
OP_ir v25.2s, v1.2s, v8.4s[2] OP_ir v25.2s, v1.2s, v8.s[2]
OP_rr v28.2s, v0.2s, v8.4s[3] OP_rr v28.2s, v0.2s, v8.s[3]
OP_ii v28.2s, v1.2s, v9.4s[3] OP_ii v28.2s, v1.2s, v9.s[3]
OP_ri v29.2s, v0.2s, v9.4s[3] OP_ri v29.2s, v0.2s, v9.s[3]
OP_ir v29.2s, v1.2s, v8.4s[3] OP_ir v29.2s, v1.2s, v8.s[3]
.endm .endm
.macro SAVE2x4 .macro SAVE2x4
@ -460,25 +460,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.s, v1.s}[0], [pA] ld2 {v0.s, v1.s}[0], [pA]
add pA, pA, #8 add pA, pA, #8
OP_rr s16, s0, v8.4s[0] OP_rr s16, s0, v8.s[0]
OP_ii s16, s1, v9.4s[0] OP_ii s16, s1, v9.s[0]
OP_ri s17, s0, v9.4s[0] OP_ri s17, s0, v9.s[0]
OP_ir s17, s1, v8.4s[0] OP_ir s17, s1, v8.s[0]
OP_rr s20, s0, v8.4s[1] OP_rr s20, s0, v8.s[1]
OP_ii s20, s1, v9.4s[1] OP_ii s20, s1, v9.s[1]
OP_ri s21, s0, v9.4s[1] OP_ri s21, s0, v9.s[1]
OP_ir s21, s1, v8.4s[1] OP_ir s21, s1, v8.s[1]
OP_rr s24, s0, v8.4s[2] OP_rr s24, s0, v8.s[2]
OP_ii s24, s1, v9.4s[2] OP_ii s24, s1, v9.s[2]
OP_ri s25, s0, v9.4s[2] OP_ri s25, s0, v9.s[2]
OP_ir s25, s1, v8.4s[2] OP_ir s25, s1, v8.s[2]
OP_rr s28, s0, v8.4s[3] OP_rr s28, s0, v8.s[3]
OP_ii s28, s1, v9.4s[3] OP_ii s28, s1, v9.s[3]
OP_ri s29, s0, v9.4s[3] OP_ri s29, s0, v9.s[3]
OP_ir s29, s1, v8.4s[3] OP_ir s29, s1, v8.s[3]
.endm .endm
.macro SAVE1x4 .macro SAVE1x4
@ -532,15 +532,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.4s, v1.4s}, [pA] ld2 {v0.4s, v1.4s}, [pA]
add pA, pA, #32 add pA, pA, #32
OP_rr v16.4s, v0.4s, v8.2s[0] OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.2s[0] OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.2s[0] OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.2s[0] OP_ir v17.4s, v1.4s, v8.s[0]
OP_rr v20.4s, v0.4s, v8.2s[1] OP_rr v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.2s[1] OP_ii v20.4s, v1.4s, v9.s[1]
OP_ri v21.4s, v0.4s, v9.2s[1] OP_ri v21.4s, v0.4s, v9.s[1]
OP_ir v21.4s, v1.4s, v8.2s[1] OP_ir v21.4s, v1.4s, v8.s[1]
.endm .endm
.macro SAVE4x2 .macro SAVE4x2
@ -578,15 +578,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.2s, v1.2s}, [pA] ld2 {v0.2s, v1.2s}, [pA]
add pA, pA, #16 add pA, pA, #16
OP_rr v16.2s, v0.2s, v8.2s[0] OP_rr v16.2s, v0.2s, v8.s[0]
OP_ii v16.2s, v1.2s, v9.2s[0] OP_ii v16.2s, v1.2s, v9.s[0]
OP_ri v17.2s, v0.2s, v9.2s[0] OP_ri v17.2s, v0.2s, v9.s[0]
OP_ir v17.2s, v1.2s, v8.2s[0] OP_ir v17.2s, v1.2s, v8.s[0]
OP_rr v20.2s, v0.2s, v8.2s[1] OP_rr v20.2s, v0.2s, v8.s[1]
OP_ii v20.2s, v1.2s, v9.2s[1] OP_ii v20.2s, v1.2s, v9.s[1]
OP_ri v21.2s, v0.2s, v9.2s[1] OP_ri v21.2s, v0.2s, v9.s[1]
OP_ir v21.2s, v1.2s, v8.2s[1] OP_ir v21.2s, v1.2s, v8.s[1]
.endm .endm
.macro SAVE2x2 .macro SAVE2x2
@ -624,15 +624,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.s, v1.s}[0], [pA] ld2 {v0.s, v1.s}[0], [pA]
add pA, pA, #8 add pA, pA, #8
OP_rr s16, s0, v8.2s[0] OP_rr s16, s0, v8.s[0]
OP_ii s16, s1, v9.2s[0] OP_ii s16, s1, v9.s[0]
OP_ri s17, s0, v9.2s[0] OP_ri s17, s0, v9.s[0]
OP_ir s17, s1, v8.2s[0] OP_ir s17, s1, v8.s[0]
OP_rr s20, s0, v8.2s[1] OP_rr s20, s0, v8.s[1]
OP_ii s20, s1, v9.2s[1] OP_ii s20, s1, v9.s[1]
OP_ri s21, s0, v9.2s[1] OP_ri s21, s0, v9.s[1]
OP_ir s21, s1, v8.2s[1] OP_ir s21, s1, v8.s[1]
.endm .endm
.macro SAVE1x2 .macro SAVE1x2

664
kernel/arm64/ctrmm_kernel_8x4.S Executable file → Normal file
View File

@ -180,93 +180,93 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v2.4s, v3.4s}, [pA] ld2 {v2.4s, v3.4s}, [pA]
add pA, pA, #32 add pA, pA, #32
fmul v16.4s, v0.4s, v8.4s[0] fmul v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.4s[0] OP_ii v16.4s, v1.4s, v9.s[0]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v17.16b, v17.16b, v17.16b eor v17.16b, v17.16b, v17.16b
fmls v17.4s, v0.4s, v9.4s[0] fmls v17.4s, v0.4s, v9.s[0]
#else #else
fmul v17.4s, v0.4s, v9.4s[0] fmul v17.4s, v0.4s, v9.s[0]
#endif #endif
OP_ir v17.4s, v1.4s, v8.4s[0] OP_ir v17.4s, v1.4s, v8.s[0]
fmul v18.4s, v2.4s, v8.4s[0] fmul v18.4s, v2.4s, v8.s[0]
OP_ii v18.4s, v3.4s, v9.4s[0] OP_ii v18.4s, v3.4s, v9.s[0]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v19.16b, v19.16b, v19.16b eor v19.16b, v19.16b, v19.16b
fmls v19.4s, v2.4s, v9.4s[0] fmls v19.4s, v2.4s, v9.s[0]
#else #else
fmul v19.4s, v2.4s, v9.4s[0] fmul v19.4s, v2.4s, v9.s[0]
#endif #endif
OP_ir v19.4s, v3.4s, v8.4s[0] OP_ir v19.4s, v3.4s, v8.s[0]
fmul v20.4s, v0.4s, v8.4s[1] fmul v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.4s[1] OP_ii v20.4s, v1.4s, v9.s[1]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v21.16b, v21.16b, v21.16b eor v21.16b, v21.16b, v21.16b
fmls v21.4s, v0.4s, v9.4s[1] fmls v21.4s, v0.4s, v9.s[1]
#else #else
fmul v21.4s, v0.4s, v9.4s[1] fmul v21.4s, v0.4s, v9.s[1]
#endif #endif
OP_ir v21.4s, v1.4s, v8.4s[1] OP_ir v21.4s, v1.4s, v8.s[1]
fmul v22.4s, v2.4s, v8.4s[1] fmul v22.4s, v2.4s, v8.s[1]
OP_ii v22.4s, v3.4s, v9.4s[1] OP_ii v22.4s, v3.4s, v9.s[1]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v23.16b, v23.16b, v23.16b eor v23.16b, v23.16b, v23.16b
fmls v23.4s, v2.4s, v9.4s[1] fmls v23.4s, v2.4s, v9.s[1]
#else #else
fmul v23.4s, v2.4s, v9.4s[1] fmul v23.4s, v2.4s, v9.s[1]
#endif #endif
OP_ir v23.4s, v3.4s, v8.4s[1] OP_ir v23.4s, v3.4s, v8.s[1]
fmul v24.4s, v0.4s, v8.4s[2] fmul v24.4s, v0.4s, v8.s[2]
OP_ii v24.4s, v1.4s, v9.4s[2] OP_ii v24.4s, v1.4s, v9.s[2]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v25.16b, v25.16b, v25.16b eor v25.16b, v25.16b, v25.16b
fmls v25.4s, v0.4s, v9.4s[2] fmls v25.4s, v0.4s, v9.s[2]
#else #else
fmul v25.4s, v0.4s, v9.4s[2] fmul v25.4s, v0.4s, v9.s[2]
#endif #endif
OP_ir v25.4s, v1.4s, v8.4s[2] OP_ir v25.4s, v1.4s, v8.s[2]
fmul v26.4s, v2.4s, v8.4s[2] fmul v26.4s, v2.4s, v8.s[2]
OP_ii v26.4s, v3.4s, v9.4s[2] OP_ii v26.4s, v3.4s, v9.s[2]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v27.16b, v27.16b, v27.16b eor v27.16b, v27.16b, v27.16b
fmls v27.4s, v2.4s, v9.4s[2] fmls v27.4s, v2.4s, v9.s[2]
#else #else
fmul v27.4s, v2.4s, v9.4s[2] fmul v27.4s, v2.4s, v9.s[2]
#endif #endif
OP_ir v27.4s, v3.4s, v8.4s[2] OP_ir v27.4s, v3.4s, v8.s[2]
fmul v28.4s, v0.4s, v8.4s[3] fmul v28.4s, v0.4s, v8.s[3]
OP_ii v28.4s, v1.4s, v9.4s[3] OP_ii v28.4s, v1.4s, v9.s[3]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v29.16b, v29.16b, v29.16b eor v29.16b, v29.16b, v29.16b
fmls v29.4s, v0.4s, v9.4s[3] fmls v29.4s, v0.4s, v9.s[3]
#else #else
fmul v29.4s, v0.4s, v9.4s[3] fmul v29.4s, v0.4s, v9.s[3]
#endif #endif
OP_ir v29.4s, v1.4s, v8.4s[3] OP_ir v29.4s, v1.4s, v8.s[3]
fmul v30.4s, v2.4s, v8.4s[3] fmul v30.4s, v2.4s, v8.s[3]
OP_ii v30.4s, v3.4s, v9.4s[3] OP_ii v30.4s, v3.4s, v9.s[3]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v31.16b, v31.16b, v31.16b eor v31.16b, v31.16b, v31.16b
fmls v31.4s, v2.4s, v9.4s[3] fmls v31.4s, v2.4s, v9.s[3]
#else #else
fmul v31.4s, v2.4s, v9.4s[3] fmul v31.4s, v2.4s, v9.s[3]
#endif #endif
OP_ir v31.4s, v3.4s, v8.4s[3] OP_ir v31.4s, v3.4s, v8.s[3]
ld2 {v12.4s, v13.4s}, [pB] ld2 {v12.4s, v13.4s}, [pB]
add pB, pB, #32 add pB, pB, #32
@ -277,45 +277,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x4_M1 .macro KERNEL8x4_M1
OP_rr v16.4s, v0.4s, v8.4s[0] OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.4s[0] OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.4s[0] OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.4s[0] OP_ir v17.4s, v1.4s, v8.s[0]
OP_rr v18.4s, v2.4s, v8.4s[0] OP_rr v18.4s, v2.4s, v8.s[0]
OP_ii v18.4s, v3.4s, v9.4s[0] OP_ii v18.4s, v3.4s, v9.s[0]
OP_ri v19.4s, v2.4s, v9.4s[0] OP_ri v19.4s, v2.4s, v9.s[0]
OP_ir v19.4s, v3.4s, v8.4s[0] OP_ir v19.4s, v3.4s, v8.s[0]
OP_rr v20.4s, v0.4s, v8.4s[1] OP_rr v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.4s[1] OP_ii v20.4s, v1.4s, v9.s[1]
OP_ri v21.4s, v0.4s, v9.4s[1] OP_ri v21.4s, v0.4s, v9.s[1]
OP_ir v21.4s, v1.4s, v8.4s[1] OP_ir v21.4s, v1.4s, v8.s[1]
OP_rr v22.4s, v2.4s, v8.4s[1] OP_rr v22.4s, v2.4s, v8.s[1]
OP_ii v22.4s, v3.4s, v9.4s[1] OP_ii v22.4s, v3.4s, v9.s[1]
OP_ri v23.4s, v2.4s, v9.4s[1] OP_ri v23.4s, v2.4s, v9.s[1]
OP_ir v23.4s, v3.4s, v8.4s[1] OP_ir v23.4s, v3.4s, v8.s[1]
OP_rr v24.4s, v0.4s, v8.4s[2] OP_rr v24.4s, v0.4s, v8.s[2]
OP_ii v24.4s, v1.4s, v9.4s[2] OP_ii v24.4s, v1.4s, v9.s[2]
OP_ri v25.4s, v0.4s, v9.4s[2] OP_ri v25.4s, v0.4s, v9.s[2]
OP_ir v25.4s, v1.4s, v8.4s[2] OP_ir v25.4s, v1.4s, v8.s[2]
OP_rr v26.4s, v2.4s, v8.4s[2] OP_rr v26.4s, v2.4s, v8.s[2]
OP_ii v26.4s, v3.4s, v9.4s[2] OP_ii v26.4s, v3.4s, v9.s[2]
OP_ri v27.4s, v2.4s, v9.4s[2] OP_ri v27.4s, v2.4s, v9.s[2]
OP_ir v27.4s, v3.4s, v8.4s[2] OP_ir v27.4s, v3.4s, v8.s[2]
OP_rr v28.4s, v0.4s, v8.4s[3] OP_rr v28.4s, v0.4s, v8.s[3]
OP_ii v28.4s, v1.4s, v9.4s[3] OP_ii v28.4s, v1.4s, v9.s[3]
OP_ri v29.4s, v0.4s, v9.4s[3] OP_ri v29.4s, v0.4s, v9.s[3]
OP_ir v29.4s, v1.4s, v8.4s[3] OP_ir v29.4s, v1.4s, v8.s[3]
OP_rr v30.4s, v2.4s, v8.4s[3] OP_rr v30.4s, v2.4s, v8.s[3]
OP_ii v30.4s, v3.4s, v9.4s[3] OP_ii v30.4s, v3.4s, v9.s[3]
OP_ri v31.4s, v2.4s, v9.4s[3] OP_ri v31.4s, v2.4s, v9.s[3]
OP_ir v31.4s, v3.4s, v8.4s[3] OP_ir v31.4s, v3.4s, v8.s[3]
ld2 {v12.4s, v13.4s}, [pB] // For next round ld2 {v12.4s, v13.4s}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
@ -326,45 +326,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x4_M2 .macro KERNEL8x4_M2
OP_rr v16.4s, v4.4s, v12.4s[0] OP_rr v16.4s, v4.4s, v12.s[0]
OP_ii v16.4s, v5.4s, v13.4s[0] OP_ii v16.4s, v5.4s, v13.s[0]
OP_ri v17.4s, v4.4s, v13.4s[0] OP_ri v17.4s, v4.4s, v13.s[0]
OP_ir v17.4s, v5.4s, v12.4s[0] OP_ir v17.4s, v5.4s, v12.s[0]
OP_rr v18.4s, v6.4s, v12.4s[0] OP_rr v18.4s, v6.4s, v12.s[0]
OP_ii v18.4s, v7.4s, v13.4s[0] OP_ii v18.4s, v7.4s, v13.s[0]
OP_ri v19.4s, v6.4s, v13.4s[0] OP_ri v19.4s, v6.4s, v13.s[0]
OP_ir v19.4s, v7.4s, v12.4s[0] OP_ir v19.4s, v7.4s, v12.s[0]
OP_rr v20.4s, v4.4s, v12.4s[1] OP_rr v20.4s, v4.4s, v12.s[1]
OP_ii v20.4s, v5.4s, v13.4s[1] OP_ii v20.4s, v5.4s, v13.s[1]
OP_ri v21.4s, v4.4s, v13.4s[1] OP_ri v21.4s, v4.4s, v13.s[1]
OP_ir v21.4s, v5.4s, v12.4s[1] OP_ir v21.4s, v5.4s, v12.s[1]
OP_rr v22.4s, v6.4s, v12.4s[1] OP_rr v22.4s, v6.4s, v12.s[1]
OP_ii v22.4s, v7.4s, v13.4s[1] OP_ii v22.4s, v7.4s, v13.s[1]
OP_ri v23.4s, v6.4s, v13.4s[1] OP_ri v23.4s, v6.4s, v13.s[1]
OP_ir v23.4s, v7.4s, v12.4s[1] OP_ir v23.4s, v7.4s, v12.s[1]
OP_rr v24.4s, v4.4s, v12.4s[2] OP_rr v24.4s, v4.4s, v12.s[2]
OP_ii v24.4s, v5.4s, v13.4s[2] OP_ii v24.4s, v5.4s, v13.s[2]
OP_ri v25.4s, v4.4s, v13.4s[2] OP_ri v25.4s, v4.4s, v13.s[2]
OP_ir v25.4s, v5.4s, v12.4s[2] OP_ir v25.4s, v5.4s, v12.s[2]
OP_rr v26.4s, v6.4s, v12.4s[2] OP_rr v26.4s, v6.4s, v12.s[2]
OP_ii v26.4s, v7.4s, v13.4s[2] OP_ii v26.4s, v7.4s, v13.s[2]
OP_ri v27.4s, v6.4s, v13.4s[2] OP_ri v27.4s, v6.4s, v13.s[2]
OP_ir v27.4s, v7.4s, v12.4s[2] OP_ir v27.4s, v7.4s, v12.s[2]
OP_rr v28.4s, v4.4s, v12.4s[3] OP_rr v28.4s, v4.4s, v12.s[3]
OP_ii v28.4s, v5.4s, v13.4s[3] OP_ii v28.4s, v5.4s, v13.s[3]
OP_ri v29.4s, v4.4s, v13.4s[3] OP_ri v29.4s, v4.4s, v13.s[3]
OP_ir v29.4s, v5.4s, v12.4s[3] OP_ir v29.4s, v5.4s, v12.s[3]
OP_rr v30.4s, v6.4s, v12.4s[3] OP_rr v30.4s, v6.4s, v12.s[3]
OP_ii v30.4s, v7.4s, v13.4s[3] OP_ii v30.4s, v7.4s, v13.s[3]
OP_ri v31.4s, v6.4s, v13.4s[3] OP_ri v31.4s, v6.4s, v13.s[3]
OP_ir v31.4s, v7.4s, v12.4s[3] OP_ir v31.4s, v7.4s, v12.s[3]
ld2 {v8.4s, v9.4s}, [pB] ld2 {v8.4s, v9.4s}, [pB]
add pB, pB, #32 add pB, pB, #32
@ -375,45 +375,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x4_E .macro KERNEL8x4_E
OP_rr v16.4s, v4.4s, v12.4s[0] OP_rr v16.4s, v4.4s, v12.s[0]
OP_ii v16.4s, v5.4s, v13.4s[0] OP_ii v16.4s, v5.4s, v13.s[0]
OP_ri v17.4s, v4.4s, v13.4s[0] OP_ri v17.4s, v4.4s, v13.s[0]
OP_ir v17.4s, v5.4s, v12.4s[0] OP_ir v17.4s, v5.4s, v12.s[0]
OP_rr v18.4s, v6.4s, v12.4s[0] OP_rr v18.4s, v6.4s, v12.s[0]
OP_ii v18.4s, v7.4s, v13.4s[0] OP_ii v18.4s, v7.4s, v13.s[0]
OP_ri v19.4s, v6.4s, v13.4s[0] OP_ri v19.4s, v6.4s, v13.s[0]
OP_ir v19.4s, v7.4s, v12.4s[0] OP_ir v19.4s, v7.4s, v12.s[0]
OP_rr v20.4s, v4.4s, v12.4s[1] OP_rr v20.4s, v4.4s, v12.s[1]
OP_ii v20.4s, v5.4s, v13.4s[1] OP_ii v20.4s, v5.4s, v13.s[1]
OP_ri v21.4s, v4.4s, v13.4s[1] OP_ri v21.4s, v4.4s, v13.s[1]
OP_ir v21.4s, v5.4s, v12.4s[1] OP_ir v21.4s, v5.4s, v12.s[1]
OP_rr v22.4s, v6.4s, v12.4s[1] OP_rr v22.4s, v6.4s, v12.s[1]
OP_ii v22.4s, v7.4s, v13.4s[1] OP_ii v22.4s, v7.4s, v13.s[1]
OP_ri v23.4s, v6.4s, v13.4s[1] OP_ri v23.4s, v6.4s, v13.s[1]
OP_ir v23.4s, v7.4s, v12.4s[1] OP_ir v23.4s, v7.4s, v12.s[1]
OP_rr v24.4s, v4.4s, v12.4s[2] OP_rr v24.4s, v4.4s, v12.s[2]
OP_ii v24.4s, v5.4s, v13.4s[2] OP_ii v24.4s, v5.4s, v13.s[2]
OP_ri v25.4s, v4.4s, v13.4s[2] OP_ri v25.4s, v4.4s, v13.s[2]
OP_ir v25.4s, v5.4s, v12.4s[2] OP_ir v25.4s, v5.4s, v12.s[2]
OP_rr v26.4s, v6.4s, v12.4s[2] OP_rr v26.4s, v6.4s, v12.s[2]
OP_ii v26.4s, v7.4s, v13.4s[2] OP_ii v26.4s, v7.4s, v13.s[2]
OP_ri v27.4s, v6.4s, v13.4s[2] OP_ri v27.4s, v6.4s, v13.s[2]
OP_ir v27.4s, v7.4s, v12.4s[2] OP_ir v27.4s, v7.4s, v12.s[2]
OP_rr v28.4s, v4.4s, v12.4s[3] OP_rr v28.4s, v4.4s, v12.s[3]
OP_ii v28.4s, v5.4s, v13.4s[3] OP_ii v28.4s, v5.4s, v13.s[3]
OP_ri v29.4s, v4.4s, v13.4s[3] OP_ri v29.4s, v4.4s, v13.s[3]
OP_ir v29.4s, v5.4s, v12.4s[3] OP_ir v29.4s, v5.4s, v12.s[3]
OP_rr v30.4s, v6.4s, v12.4s[3] OP_rr v30.4s, v6.4s, v12.s[3]
OP_ii v30.4s, v7.4s, v13.4s[3] OP_ii v30.4s, v7.4s, v13.s[3]
OP_ri v31.4s, v6.4s, v13.4s[3] OP_ri v31.4s, v6.4s, v13.s[3]
OP_ir v31.4s, v7.4s, v12.4s[3] OP_ir v31.4s, v7.4s, v12.s[3]
.endm .endm
@ -425,45 +425,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v2.4s, v3.4s}, [pA] ld2 {v2.4s, v3.4s}, [pA]
add pA, pA, #32 add pA, pA, #32
OP_rr v16.4s, v0.4s, v8.4s[0] OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.4s[0] OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.4s[0] OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.4s[0] OP_ir v17.4s, v1.4s, v8.s[0]
OP_rr v18.4s, v2.4s, v8.4s[0] OP_rr v18.4s, v2.4s, v8.s[0]
OP_ii v18.4s, v3.4s, v9.4s[0] OP_ii v18.4s, v3.4s, v9.s[0]
OP_ri v19.4s, v2.4s, v9.4s[0] OP_ri v19.4s, v2.4s, v9.s[0]
OP_ir v19.4s, v3.4s, v8.4s[0] OP_ir v19.4s, v3.4s, v8.s[0]
OP_rr v20.4s, v0.4s, v8.4s[1] OP_rr v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.4s[1] OP_ii v20.4s, v1.4s, v9.s[1]
OP_ri v21.4s, v0.4s, v9.4s[1] OP_ri v21.4s, v0.4s, v9.s[1]
OP_ir v21.4s, v1.4s, v8.4s[1] OP_ir v21.4s, v1.4s, v8.s[1]
OP_rr v22.4s, v2.4s, v8.4s[1] OP_rr v22.4s, v2.4s, v8.s[1]
OP_ii v22.4s, v3.4s, v9.4s[1] OP_ii v22.4s, v3.4s, v9.s[1]
OP_ri v23.4s, v2.4s, v9.4s[1] OP_ri v23.4s, v2.4s, v9.s[1]
OP_ir v23.4s, v3.4s, v8.4s[1] OP_ir v23.4s, v3.4s, v8.s[1]
OP_rr v24.4s, v0.4s, v8.4s[2] OP_rr v24.4s, v0.4s, v8.s[2]
OP_ii v24.4s, v1.4s, v9.4s[2] OP_ii v24.4s, v1.4s, v9.s[2]
OP_ri v25.4s, v0.4s, v9.4s[2] OP_ri v25.4s, v0.4s, v9.s[2]
OP_ir v25.4s, v1.4s, v8.4s[2] OP_ir v25.4s, v1.4s, v8.s[2]
OP_rr v26.4s, v2.4s, v8.4s[2] OP_rr v26.4s, v2.4s, v8.s[2]
OP_ii v26.4s, v3.4s, v9.4s[2] OP_ii v26.4s, v3.4s, v9.s[2]
OP_ri v27.4s, v2.4s, v9.4s[2] OP_ri v27.4s, v2.4s, v9.s[2]
OP_ir v27.4s, v3.4s, v8.4s[2] OP_ir v27.4s, v3.4s, v8.s[2]
OP_rr v28.4s, v0.4s, v8.4s[3] OP_rr v28.4s, v0.4s, v8.s[3]
OP_ii v28.4s, v1.4s, v9.4s[3] OP_ii v28.4s, v1.4s, v9.s[3]
OP_ri v29.4s, v0.4s, v9.4s[3] OP_ri v29.4s, v0.4s, v9.s[3]
OP_ir v29.4s, v1.4s, v8.4s[3] OP_ir v29.4s, v1.4s, v8.s[3]
OP_rr v30.4s, v2.4s, v8.4s[3] OP_rr v30.4s, v2.4s, v8.s[3]
OP_ii v30.4s, v3.4s, v9.4s[3] OP_ii v30.4s, v3.4s, v9.s[3]
OP_ri v31.4s, v2.4s, v9.4s[3] OP_ri v31.4s, v2.4s, v9.s[3]
OP_ir v31.4s, v3.4s, v8.4s[3] OP_ir v31.4s, v3.4s, v8.s[3]
.endm .endm
@ -562,49 +562,49 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.4s, v1.4s}, [pA] ld2 {v0.4s, v1.4s}, [pA]
add pA, pA, #32 add pA, pA, #32
fmul v16.4s, v0.4s, v8.4s[0] fmul v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.4s[0] OP_ii v16.4s, v1.4s, v9.s[0]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v17.16b, v17.16b, v17.16b eor v17.16b, v17.16b, v17.16b
fmls v17.4s, v0.4s, v9.4s[0] fmls v17.4s, v0.4s, v9.s[0]
#else #else
fmul v17.4s, v0.4s, v9.4s[0] fmul v17.4s, v0.4s, v9.s[0]
#endif #endif
OP_ir v17.4s, v1.4s, v8.4s[0] OP_ir v17.4s, v1.4s, v8.s[0]
fmul v20.4s, v0.4s, v8.4s[1] fmul v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.4s[1] OP_ii v20.4s, v1.4s, v9.s[1]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v21.16b, v21.16b, v21.16b eor v21.16b, v21.16b, v21.16b
fmls v21.4s, v0.4s, v9.4s[1] fmls v21.4s, v0.4s, v9.s[1]
#else #else
fmul v21.4s, v0.4s, v9.4s[1] fmul v21.4s, v0.4s, v9.s[1]
#endif #endif
OP_ir v21.4s, v1.4s, v8.4s[1] OP_ir v21.4s, v1.4s, v8.s[1]
fmul v24.4s, v0.4s, v8.4s[2] fmul v24.4s, v0.4s, v8.s[2]
OP_ii v24.4s, v1.4s, v9.4s[2] OP_ii v24.4s, v1.4s, v9.s[2]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v25.16b, v25.16b, v25.16b eor v25.16b, v25.16b, v25.16b
fmls v25.4s, v0.4s, v9.4s[2] fmls v25.4s, v0.4s, v9.s[2]
#else #else
fmul v25.4s, v0.4s, v9.4s[2] fmul v25.4s, v0.4s, v9.s[2]
#endif #endif
OP_ir v25.4s, v1.4s, v8.4s[2] OP_ir v25.4s, v1.4s, v8.s[2]
fmul v28.4s, v0.4s, v8.4s[3] fmul v28.4s, v0.4s, v8.s[3]
OP_ii v28.4s, v1.4s, v9.4s[3] OP_ii v28.4s, v1.4s, v9.s[3]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v29.16b, v29.16b, v29.16b eor v29.16b, v29.16b, v29.16b
fmls v29.4s, v0.4s, v9.4s[3] fmls v29.4s, v0.4s, v9.s[3]
#else #else
fmul v29.4s, v0.4s, v9.4s[3] fmul v29.4s, v0.4s, v9.s[3]
#endif #endif
OP_ir v29.4s, v1.4s, v8.4s[3] OP_ir v29.4s, v1.4s, v8.s[3]
ld2 {v12.4s, v13.4s}, [pB] ld2 {v12.4s, v13.4s}, [pB]
add pB, pB, #32 add pB, pB, #32
@ -613,85 +613,85 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL4x4_M1 .macro KERNEL4x4_M1
OP_rr v16.4s, v0.4s, v8.4s[0] OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.4s[0] OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.4s[0] OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.4s[0] OP_ir v17.4s, v1.4s, v8.s[0]
ld2 {v12.4s, v13.4s}, [pB] // For next round ld2 {v12.4s, v13.4s}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
OP_rr v20.4s, v0.4s, v8.4s[1] OP_rr v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.4s[1] OP_ii v20.4s, v1.4s, v9.s[1]
OP_ri v21.4s, v0.4s, v9.4s[1] OP_ri v21.4s, v0.4s, v9.s[1]
OP_ir v21.4s, v1.4s, v8.4s[1] OP_ir v21.4s, v1.4s, v8.s[1]
ld2 {v4.4s, v5.4s}, [pA] // For next round ld2 {v4.4s, v5.4s}, [pA] // For next round
add pA, pA, #32 add pA, pA, #32
OP_rr v24.4s, v0.4s, v8.4s[2] OP_rr v24.4s, v0.4s, v8.s[2]
OP_ii v24.4s, v1.4s, v9.4s[2] OP_ii v24.4s, v1.4s, v9.s[2]
OP_ri v25.4s, v0.4s, v9.4s[2] OP_ri v25.4s, v0.4s, v9.s[2]
OP_ir v25.4s, v1.4s, v8.4s[2] OP_ir v25.4s, v1.4s, v8.s[2]
prfm PLDL1KEEP, [pA, #512] prfm PLDL1KEEP, [pA, #512]
OP_rr v28.4s, v0.4s, v8.4s[3] OP_rr v28.4s, v0.4s, v8.s[3]
OP_ii v28.4s, v1.4s, v9.4s[3] OP_ii v28.4s, v1.4s, v9.s[3]
OP_ri v29.4s, v0.4s, v9.4s[3] OP_ri v29.4s, v0.4s, v9.s[3]
OP_ir v29.4s, v1.4s, v8.4s[3] OP_ir v29.4s, v1.4s, v8.s[3]
.endm .endm
.macro KERNEL4x4_M2 .macro KERNEL4x4_M2
OP_rr v16.4s, v4.4s, v12.4s[0] OP_rr v16.4s, v4.4s, v12.s[0]
OP_ii v16.4s, v5.4s, v13.4s[0] OP_ii v16.4s, v5.4s, v13.s[0]
OP_ri v17.4s, v4.4s, v13.4s[0] OP_ri v17.4s, v4.4s, v13.s[0]
OP_ir v17.4s, v5.4s, v12.4s[0] OP_ir v17.4s, v5.4s, v12.s[0]
ld2 {v8.4s, v9.4s}, [pB] // For next round ld2 {v8.4s, v9.4s}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
OP_rr v20.4s, v4.4s, v12.4s[1] OP_rr v20.4s, v4.4s, v12.s[1]
OP_ii v20.4s, v5.4s, v13.4s[1] OP_ii v20.4s, v5.4s, v13.s[1]
OP_ri v21.4s, v4.4s, v13.4s[1] OP_ri v21.4s, v4.4s, v13.s[1]
OP_ir v21.4s, v5.4s, v12.4s[1] OP_ir v21.4s, v5.4s, v12.s[1]
ld2 {v0.4s, v1.4s}, [pA] // For next round ld2 {v0.4s, v1.4s}, [pA] // For next round
add pA, pA, #32 add pA, pA, #32
OP_rr v24.4s, v4.4s, v12.4s[2] OP_rr v24.4s, v4.4s, v12.s[2]
OP_ii v24.4s, v5.4s, v13.4s[2] OP_ii v24.4s, v5.4s, v13.s[2]
OP_ri v25.4s, v4.4s, v13.4s[2] OP_ri v25.4s, v4.4s, v13.s[2]
OP_ir v25.4s, v5.4s, v12.4s[2] OP_ir v25.4s, v5.4s, v12.s[2]
prfm PLDL1KEEP, [pB, #512] prfm PLDL1KEEP, [pB, #512]
OP_rr v28.4s, v4.4s, v12.4s[3] OP_rr v28.4s, v4.4s, v12.s[3]
OP_ii v28.4s, v5.4s, v13.4s[3] OP_ii v28.4s, v5.4s, v13.s[3]
OP_ri v29.4s, v4.4s, v13.4s[3] OP_ri v29.4s, v4.4s, v13.s[3]
OP_ir v29.4s, v5.4s, v12.4s[3] OP_ir v29.4s, v5.4s, v12.s[3]
.endm .endm
.macro KERNEL4x4_E .macro KERNEL4x4_E
OP_rr v16.4s, v4.4s, v12.4s[0] OP_rr v16.4s, v4.4s, v12.s[0]
OP_ii v16.4s, v5.4s, v13.4s[0] OP_ii v16.4s, v5.4s, v13.s[0]
OP_ri v17.4s, v4.4s, v13.4s[0] OP_ri v17.4s, v4.4s, v13.s[0]
OP_ir v17.4s, v5.4s, v12.4s[0] OP_ir v17.4s, v5.4s, v12.s[0]
OP_rr v20.4s, v4.4s, v12.4s[1] OP_rr v20.4s, v4.4s, v12.s[1]
OP_ii v20.4s, v5.4s, v13.4s[1] OP_ii v20.4s, v5.4s, v13.s[1]
OP_ri v21.4s, v4.4s, v13.4s[1] OP_ri v21.4s, v4.4s, v13.s[1]
OP_ir v21.4s, v5.4s, v12.4s[1] OP_ir v21.4s, v5.4s, v12.s[1]
OP_rr v24.4s, v4.4s, v12.4s[2] OP_rr v24.4s, v4.4s, v12.s[2]
OP_ii v24.4s, v5.4s, v13.4s[2] OP_ii v24.4s, v5.4s, v13.s[2]
OP_ri v25.4s, v4.4s, v13.4s[2] OP_ri v25.4s, v4.4s, v13.s[2]
OP_ir v25.4s, v5.4s, v12.4s[2] OP_ir v25.4s, v5.4s, v12.s[2]
OP_rr v28.4s, v4.4s, v12.4s[3] OP_rr v28.4s, v4.4s, v12.s[3]
OP_ii v28.4s, v5.4s, v13.4s[3] OP_ii v28.4s, v5.4s, v13.s[3]
OP_ri v29.4s, v4.4s, v13.4s[3] OP_ri v29.4s, v4.4s, v13.s[3]
OP_ir v29.4s, v5.4s, v12.4s[3] OP_ir v29.4s, v5.4s, v12.s[3]
.endm .endm
.macro KERNEL4x4_SUB .macro KERNEL4x4_SUB
@ -700,25 +700,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.4s, v1.4s}, [pA] ld2 {v0.4s, v1.4s}, [pA]
add pA, pA, #32 add pA, pA, #32
OP_rr v16.4s, v0.4s, v8.4s[0] OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.4s[0] OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.4s[0] OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.4s[0] OP_ir v17.4s, v1.4s, v8.s[0]
OP_rr v20.4s, v0.4s, v8.4s[1] OP_rr v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.4s[1] OP_ii v20.4s, v1.4s, v9.s[1]
OP_ri v21.4s, v0.4s, v9.4s[1] OP_ri v21.4s, v0.4s, v9.s[1]
OP_ir v21.4s, v1.4s, v8.4s[1] OP_ir v21.4s, v1.4s, v8.s[1]
OP_rr v24.4s, v0.4s, v8.4s[2] OP_rr v24.4s, v0.4s, v8.s[2]
OP_ii v24.4s, v1.4s, v9.4s[2] OP_ii v24.4s, v1.4s, v9.s[2]
OP_ri v25.4s, v0.4s, v9.4s[2] OP_ri v25.4s, v0.4s, v9.s[2]
OP_ir v25.4s, v1.4s, v8.4s[2] OP_ir v25.4s, v1.4s, v8.s[2]
OP_rr v28.4s, v0.4s, v8.4s[3] OP_rr v28.4s, v0.4s, v8.s[3]
OP_ii v28.4s, v1.4s, v9.4s[3] OP_ii v28.4s, v1.4s, v9.s[3]
OP_ri v29.4s, v0.4s, v9.4s[3] OP_ri v29.4s, v0.4s, v9.s[3]
OP_ir v29.4s, v1.4s, v8.4s[3] OP_ir v29.4s, v1.4s, v8.s[3]
.endm .endm
.macro SAVE4x4 .macro SAVE4x4
@ -780,25 +780,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.2s, v1.2s}, [pA] ld2 {v0.2s, v1.2s}, [pA]
add pA, pA, #16 add pA, pA, #16
OP_rr v16.2s, v0.2s, v8.4s[0] OP_rr v16.2s, v0.2s, v8.s[0]
OP_ii v16.2s, v1.2s, v9.4s[0] OP_ii v16.2s, v1.2s, v9.s[0]
OP_ri v17.2s, v0.2s, v9.4s[0] OP_ri v17.2s, v0.2s, v9.s[0]
OP_ir v17.2s, v1.2s, v8.4s[0] OP_ir v17.2s, v1.2s, v8.s[0]
OP_rr v20.2s, v0.2s, v8.4s[1] OP_rr v20.2s, v0.2s, v8.s[1]
OP_ii v20.2s, v1.2s, v9.4s[1] OP_ii v20.2s, v1.2s, v9.s[1]
OP_ri v21.2s, v0.2s, v9.4s[1] OP_ri v21.2s, v0.2s, v9.s[1]
OP_ir v21.2s, v1.2s, v8.4s[1] OP_ir v21.2s, v1.2s, v8.s[1]
OP_rr v24.2s, v0.2s, v8.4s[2] OP_rr v24.2s, v0.2s, v8.s[2]
OP_ii v24.2s, v1.2s, v9.4s[2] OP_ii v24.2s, v1.2s, v9.s[2]
OP_ri v25.2s, v0.2s, v9.4s[2] OP_ri v25.2s, v0.2s, v9.s[2]
OP_ir v25.2s, v1.2s, v8.4s[2] OP_ir v25.2s, v1.2s, v8.s[2]
OP_rr v28.2s, v0.2s, v8.4s[3] OP_rr v28.2s, v0.2s, v8.s[3]
OP_ii v28.2s, v1.2s, v9.4s[3] OP_ii v28.2s, v1.2s, v9.s[3]
OP_ri v29.2s, v0.2s, v9.4s[3] OP_ri v29.2s, v0.2s, v9.s[3]
OP_ir v29.2s, v1.2s, v8.4s[3] OP_ir v29.2s, v1.2s, v8.s[3]
.endm .endm
.macro SAVE2x4 .macro SAVE2x4
@ -860,25 +860,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.s, v1.s}[0], [pA] ld2 {v0.s, v1.s}[0], [pA]
add pA, pA, #8 add pA, pA, #8
OP_rr s16, s0, v8.4s[0] OP_rr s16, s0, v8.s[0]
OP_ii s16, s1, v9.4s[0] OP_ii s16, s1, v9.s[0]
OP_ri s17, s0, v9.4s[0] OP_ri s17, s0, v9.s[0]
OP_ir s17, s1, v8.4s[0] OP_ir s17, s1, v8.s[0]
OP_rr s20, s0, v8.4s[1] OP_rr s20, s0, v8.s[1]
OP_ii s20, s1, v9.4s[1] OP_ii s20, s1, v9.s[1]
OP_ri s21, s0, v9.4s[1] OP_ri s21, s0, v9.s[1]
OP_ir s21, s1, v8.4s[1] OP_ir s21, s1, v8.s[1]
OP_rr s24, s0, v8.4s[2] OP_rr s24, s0, v8.s[2]
OP_ii s24, s1, v9.4s[2] OP_ii s24, s1, v9.s[2]
OP_ri s25, s0, v9.4s[2] OP_ri s25, s0, v9.s[2]
OP_ir s25, s1, v8.4s[2] OP_ir s25, s1, v8.s[2]
OP_rr s28, s0, v8.4s[3] OP_rr s28, s0, v8.s[3]
OP_ii s28, s1, v9.4s[3] OP_ii s28, s1, v9.s[3]
OP_ri s29, s0, v9.4s[3] OP_ri s29, s0, v9.s[3]
OP_ir s29, s1, v8.4s[3] OP_ir s29, s1, v8.s[3]
.endm .endm
.macro SAVE1x4 .macro SAVE1x4
@ -942,25 +942,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v2.4s, v3.4s}, [pA] ld2 {v2.4s, v3.4s}, [pA]
add pA, pA, #32 add pA, pA, #32
OP_rr v16.4s, v0.4s, v8.2s[0] OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.2s[0] OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.2s[0] OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.2s[0] OP_ir v17.4s, v1.4s, v8.s[0]
OP_rr v18.4s, v2.4s, v8.2s[0] OP_rr v18.4s, v2.4s, v8.s[0]
OP_ii v18.4s, v3.4s, v9.2s[0] OP_ii v18.4s, v3.4s, v9.s[0]
OP_ri v19.4s, v2.4s, v9.2s[0] OP_ri v19.4s, v2.4s, v9.s[0]
OP_ir v19.4s, v3.4s, v8.2s[0] OP_ir v19.4s, v3.4s, v8.s[0]
OP_rr v20.4s, v0.4s, v8.2s[1] OP_rr v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.2s[1] OP_ii v20.4s, v1.4s, v9.s[1]
OP_ri v21.4s, v0.4s, v9.2s[1] OP_ri v21.4s, v0.4s, v9.s[1]
OP_ir v21.4s, v1.4s, v8.2s[1] OP_ir v21.4s, v1.4s, v8.s[1]
OP_rr v22.4s, v2.4s, v8.2s[1] OP_rr v22.4s, v2.4s, v8.s[1]
OP_ii v22.4s, v3.4s, v9.2s[1] OP_ii v22.4s, v3.4s, v9.s[1]
OP_ri v23.4s, v2.4s, v9.2s[1] OP_ri v23.4s, v2.4s, v9.s[1]
OP_ir v23.4s, v3.4s, v8.2s[1] OP_ir v23.4s, v3.4s, v8.s[1]
.endm .endm
.macro SAVE8x2 .macro SAVE8x2
@ -1018,15 +1018,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.4s, v1.4s}, [pA] ld2 {v0.4s, v1.4s}, [pA]
add pA, pA, #32 add pA, pA, #32
OP_rr v16.4s, v0.4s, v8.2s[0] OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v9.2s[0] OP_ii v16.4s, v1.4s, v9.s[0]
OP_ri v17.4s, v0.4s, v9.2s[0] OP_ri v17.4s, v0.4s, v9.s[0]
OP_ir v17.4s, v1.4s, v8.2s[0] OP_ir v17.4s, v1.4s, v8.s[0]
OP_rr v20.4s, v0.4s, v8.2s[1] OP_rr v20.4s, v0.4s, v8.s[1]
OP_ii v20.4s, v1.4s, v9.2s[1] OP_ii v20.4s, v1.4s, v9.s[1]
OP_ri v21.4s, v0.4s, v9.2s[1] OP_ri v21.4s, v0.4s, v9.s[1]
OP_ir v21.4s, v1.4s, v8.2s[1] OP_ir v21.4s, v1.4s, v8.s[1]
.endm .endm
.macro SAVE4x2 .macro SAVE4x2
@ -1066,15 +1066,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.2s, v1.2s}, [pA] ld2 {v0.2s, v1.2s}, [pA]
add pA, pA, #16 add pA, pA, #16
OP_rr v16.2s, v0.2s, v8.2s[0] OP_rr v16.2s, v0.2s, v8.s[0]
OP_ii v16.2s, v1.2s, v9.2s[0] OP_ii v16.2s, v1.2s, v9.s[0]
OP_ri v17.2s, v0.2s, v9.2s[0] OP_ri v17.2s, v0.2s, v9.s[0]
OP_ir v17.2s, v1.2s, v8.2s[0] OP_ir v17.2s, v1.2s, v8.s[0]
OP_rr v20.2s, v0.2s, v8.2s[1] OP_rr v20.2s, v0.2s, v8.s[1]
OP_ii v20.2s, v1.2s, v9.2s[1] OP_ii v20.2s, v1.2s, v9.s[1]
OP_ri v21.2s, v0.2s, v9.2s[1] OP_ri v21.2s, v0.2s, v9.s[1]
OP_ir v21.2s, v1.2s, v8.2s[1] OP_ir v21.2s, v1.2s, v8.s[1]
.endm .endm
.macro SAVE2x2 .macro SAVE2x2
@ -1114,15 +1114,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.s, v1.s}[0], [pA] ld2 {v0.s, v1.s}[0], [pA]
add pA, pA, #8 add pA, pA, #8
OP_rr s16, s0, v8.2s[0] OP_rr s16, s0, v8.s[0]
OP_ii s16, s1, v9.2s[0] OP_ii s16, s1, v9.s[0]
OP_ri s17, s0, v9.2s[0] OP_ri s17, s0, v9.s[0]
OP_ir s17, s1, v8.2s[0] OP_ir s17, s1, v8.s[0]
OP_rr s20, s0, v8.2s[1] OP_rr s20, s0, v8.s[1]
OP_ii s20, s1, v9.2s[1] OP_ii s20, s1, v9.s[1]
OP_ri s21, s0, v9.2s[1] OP_ri s21, s0, v9.s[1]
OP_ir s21, s1, v8.2s[1] OP_ir s21, s1, v8.s[1]
.endm .endm
.macro SAVE1x2 .macro SAVE1x2
@ -1164,15 +1164,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v2.4s, v3.4s}, [pA] ld2 {v2.4s, v3.4s}, [pA]
add pA, pA, #32 add pA, pA, #32
OP_rr v16.4s, v0.4s, v8.4s[0] OP_rr v16.4s, v0.4s, v8.s[0]
OP_ii v16.4s, v1.4s, v8.4s[1] OP_ii v16.4s, v1.4s, v8.s[1]
OP_ri v17.4s, v0.4s, v8.4s[1] OP_ri v17.4s, v0.4s, v8.s[1]
OP_ir v17.4s, v1.4s, v8.4s[0] OP_ir v17.4s, v1.4s, v8.s[0]
OP_rr v18.4s, v2.4s, v8.4s[0] OP_rr v18.4s, v2.4s, v8.s[0]
OP_ii v18.4s, v3.4s, v8.4s[1] OP_ii v18.4s, v3.4s, v8.s[1]
OP_ri v19.4s, v2.4s, v8.4s[1] OP_ri v19.4s, v2.4s, v8.s[1]
OP_ir v19.4s, v3.4s, v8.4s[0] OP_ir v19.4s, v3.4s, v8.s[0]
.endm .endm
.macro SAVE8x1 .macro SAVE8x1

204
kernel/arm64/dgemm_kernel_4x4.S
View File

@ -161,150 +161,150 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ldp q0, q1, [pA] ldp q0, q1, [pA]
add pA, pA, #32 add pA, pA, #32
fmul v16.2d, v0.2d, v8.2d[0] fmul v16.2d, v0.2d, v8.d[0]
fmul v29.2d, v1.2d, v11.2d[0] fmul v29.2d, v1.2d, v11.d[0]
ldp q2, q3, [ppA] ldp q2, q3, [ppA]
add ppA, ppA, #32 add ppA, ppA, #32
fmul v20.2d, v0.2d, v9.2d[0] fmul v20.2d, v0.2d, v9.d[0]
fmul v25.2d, v1.2d, v10.2d[0] fmul v25.2d, v1.2d, v10.d[0]
prfm PLDL1KEEP, [pA, #A_PRE_SIZE] prfm PLDL1KEEP, [pA, #A_PRE_SIZE]
fmul v18.2d, v2.2d, v8.2d[0] fmul v18.2d, v2.2d, v8.d[0]
fmul v31.2d, v3.2d, v11.2d[0] fmul v31.2d, v3.2d, v11.d[0]
prfm PLDL1KEEP, [ppA, #A_PRE_SIZE] prfm PLDL1KEEP, [ppA, #A_PRE_SIZE]
fmul v22.2d, v2.2d, v9.2d[0] fmul v22.2d, v2.2d, v9.d[0]
fmul v27.2d, v3.2d, v10.2d[0] fmul v27.2d, v3.2d, v10.d[0]
ldp d12, d13, [pB] ldp d12, d13, [pB]
add pB, pB, #16 add pB, pB, #16
fmul v24.2d, v0.2d, v10.2d[0] fmul v24.2d, v0.2d, v10.d[0]
fmul v21.2d, v1.2d, v9.2d[0] fmul v21.2d, v1.2d, v9.d[0]
ldp q4, q5, [pA] // for next round ldp q4, q5, [pA] // for next round
add pA, pA, #32 add pA, pA, #32
fmul v26.2d, v2.2d, v10.2d[0] fmul v26.2d, v2.2d, v10.d[0]
fmul v23.2d, v3.2d, v9.2d[0] fmul v23.2d, v3.2d, v9.d[0]
ldp q6, q7, [ppA] // for next round ldp q6, q7, [ppA] // for next round
add ppA, ppA, #32 add ppA, ppA, #32
fmul v28.2d, v0.2d, v11.2d[0] fmul v28.2d, v0.2d, v11.d[0]
fmul v17.2d, v1.2d, v8.2d[0] fmul v17.2d, v1.2d, v8.d[0]
ldp d14, d15, [pB] ldp d14, d15, [pB]
add pB, pB, #16 add pB, pB, #16
fmul v30.2d, v2.2d, v11.2d[0] fmul v30.2d, v2.2d, v11.d[0]
fmul v19.2d, v3.2d, v8.2d[0] fmul v19.2d, v3.2d, v8.d[0]
.endm .endm
.macro KERNEL8x4_M2 .macro KERNEL8x4_M2
fmla v16.2d, v4.2d, v12.2d[0] fmla v16.2d, v4.2d, v12.d[0]
fmla v29.2d, v5.2d, v15.2d[0] fmla v29.2d, v5.2d, v15.d[0]
ldp d8, d9, [pB] ldp d8, d9, [pB]
add pB, pB, #16 add pB, pB, #16
fmla v18.2d, v6.2d, v12.2d[0] fmla v18.2d, v6.2d, v12.d[0]
fmla v31.2d, v7.2d, v15.2d[0] fmla v31.2d, v7.2d, v15.d[0]
ldp d10, d11, [pB] ldp d10, d11, [pB]
add pB, pB, #16 add pB, pB, #16
fmla v20.2d, v4.2d, v13.2d[0] fmla v20.2d, v4.2d, v13.d[0]
fmla v25.2d, v5.2d, v14.2d[0] fmla v25.2d, v5.2d, v14.d[0]
prfm PLDL1KEEP, [pB, #B_PRE_SIZE] prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
fmla v22.2d, v6.2d, v13.2d[0] fmla v22.2d, v6.2d, v13.d[0]
fmla v27.2d, v7.2d, v14.2d[0] fmla v27.2d, v7.2d, v14.d[0]
fmla v24.2d, v4.2d, v14.2d[0] fmla v24.2d, v4.2d, v14.d[0]
fmla v21.2d, v5.2d, v13.2d[0] fmla v21.2d, v5.2d, v13.d[0]
ldp q0, q1, [pA] ldp q0, q1, [pA]
add pA, pA, #32 add pA, pA, #32
fmla v26.2d, v6.2d, v14.2d[0] fmla v26.2d, v6.2d, v14.d[0]
fmla v23.2d, v7.2d, v13.2d[0] fmla v23.2d, v7.2d, v13.d[0]
fmla v28.2d, v4.2d, v15.2d[0] fmla v28.2d, v4.2d, v15.d[0]
fmla v17.2d, v5.2d, v12.2d[0] fmla v17.2d, v5.2d, v12.d[0]
ldp q2, q3, [ppA] ldp q2, q3, [ppA]
add ppA, ppA, #32 add ppA, ppA, #32
fmla v30.2d, v6.2d, v15.2d[0] fmla v30.2d, v6.2d, v15.d[0]
fmla v19.2d, v7.2d, v12.2d[0] fmla v19.2d, v7.2d, v12.d[0]
.endm .endm
.macro KERNEL8x4_M1 .macro KERNEL8x4_M1
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v29.2d, v1.2d, v11.2d[0] fmla v29.2d, v1.2d, v11.d[0]
ldp d12, d13, [pB] ldp d12, d13, [pB]
add pB, pB, #16 add pB, pB, #16
fmla v18.2d, v2.2d, v8.2d[0] fmla v18.2d, v2.2d, v8.d[0]
fmla v31.2d, v3.2d, v11.2d[0] fmla v31.2d, v3.2d, v11.d[0]
ldp d14, d15, [pB] ldp d14, d15, [pB]
add pB, pB, #16 add pB, pB, #16
fmla v20.2d, v0.2d, v9.2d[0] fmla v20.2d, v0.2d, v9.d[0]
fmla v25.2d, v1.2d, v10.2d[0] fmla v25.2d, v1.2d, v10.d[0]
prfm PLDL1KEEP, [pA, #A_PRE_SIZE] prfm PLDL1KEEP, [pA, #A_PRE_SIZE]
fmla v22.2d, v2.2d, v9.2d[0] fmla v22.2d, v2.2d, v9.d[0]
fmla v27.2d, v3.2d, v10.2d[0] fmla v27.2d, v3.2d, v10.d[0]
prfm PLDL1KEEP, [ppA, #A_PRE_SIZE] prfm PLDL1KEEP, [ppA, #A_PRE_SIZE]
fmla v24.2d, v0.2d, v10.2d[0] fmla v24.2d, v0.2d, v10.d[0]
fmla v21.2d, v1.2d, v9.2d[0] fmla v21.2d, v1.2d, v9.d[0]
ldp q4, q5, [pA] ldp q4, q5, [pA]
add pA, pA, #32 add pA, pA, #32
fmla v26.2d, v2.2d, v10.2d[0] fmla v26.2d, v2.2d, v10.d[0]
fmla v23.2d, v3.2d, v9.2d[0] fmla v23.2d, v3.2d, v9.d[0]
fmla v28.2d, v0.2d, v11.2d[0] fmla v28.2d, v0.2d, v11.d[0]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
ldp q6, q7, [ppA] ldp q6, q7, [ppA]
add ppA, ppA, #32 add ppA, ppA, #32
fmla v30.2d, v2.2d, v11.2d[0] fmla v30.2d, v2.2d, v11.d[0]
fmla v19.2d, v3.2d, v8.2d[0] fmla v19.2d, v3.2d, v8.d[0]
.endm .endm
.macro KERNEL8x4_E .macro KERNEL8x4_E
fmla v16.2d, v4.2d, v12.2d[0] fmla v16.2d, v4.2d, v12.d[0]
fmla v25.2d, v5.2d, v14.2d[0] fmla v25.2d, v5.2d, v14.d[0]
fmla v18.2d, v6.2d, v12.2d[0] fmla v18.2d, v6.2d, v12.d[0]
fmla v27.2d, v7.2d, v14.2d[0] fmla v27.2d, v7.2d, v14.d[0]
fmla v20.2d, v4.2d, v13.2d[0] fmla v20.2d, v4.2d, v13.d[0]
fmla v29.2d, v5.2d, v15.2d[0] fmla v29.2d, v5.2d, v15.d[0]
fmla v22.2d, v6.2d, v13.2d[0] fmla v22.2d, v6.2d, v13.d[0]
fmla v31.2d, v7.2d, v15.2d[0] fmla v31.2d, v7.2d, v15.d[0]
fmla v24.2d, v4.2d, v14.2d[0] fmla v24.2d, v4.2d, v14.d[0]
fmla v17.2d, v5.2d, v12.2d[0] fmla v17.2d, v5.2d, v12.d[0]
fmla v26.2d, v6.2d, v14.2d[0] fmla v26.2d, v6.2d, v14.d[0]
fmla v19.2d, v7.2d, v12.2d[0] fmla v19.2d, v7.2d, v12.d[0]
fmla v28.2d, v4.2d, v15.2d[0] fmla v28.2d, v4.2d, v15.d[0]
fmla v21.2d, v5.2d, v13.2d[0] fmla v21.2d, v5.2d, v13.d[0]
fmla v30.2d, v6.2d, v15.2d[0] fmla v30.2d, v6.2d, v15.d[0]
fmla v23.2d, v7.2d, v13.2d[0] fmla v23.2d, v7.2d, v13.d[0]
.endm .endm
.macro KERNEL8x4_SUB .macro KERNEL8x4_SUB
@ -315,28 +315,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ldp q0, q1, [pA] ldp q0, q1, [pA]
add pA, pA, #32 add pA, pA, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v29.2d, v1.2d, v11.2d[0] fmla v29.2d, v1.2d, v11.d[0]
fmla v20.2d, v0.2d, v9.2d[0] fmla v20.2d, v0.2d, v9.d[0]
fmla v25.2d, v1.2d, v10.2d[0] fmla v25.2d, v1.2d, v10.d[0]
ldp q2, q3, [ppA] ldp q2, q3, [ppA]
add ppA, ppA, #32 add ppA, ppA, #32
fmla v24.2d, v0.2d, v10.2d[0] fmla v24.2d, v0.2d, v10.d[0]
fmla v21.2d, v1.2d, v9.2d[0] fmla v21.2d, v1.2d, v9.d[0]
fmla v28.2d, v0.2d, v11.2d[0] fmla v28.2d, v0.2d, v11.d[0]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
fmla v18.2d, v2.2d, v8.2d[0] fmla v18.2d, v2.2d, v8.d[0]
fmla v31.2d, v3.2d, v11.2d[0] fmla v31.2d, v3.2d, v11.d[0]
fmla v22.2d, v2.2d, v9.2d[0] fmla v22.2d, v2.2d, v9.d[0]
fmla v27.2d, v3.2d, v10.2d[0] fmla v27.2d, v3.2d, v10.d[0]
fmla v26.2d, v2.2d, v10.2d[0] fmla v26.2d, v2.2d, v10.d[0]
fmla v23.2d, v3.2d, v9.2d[0] fmla v23.2d, v3.2d, v9.d[0]
fmla v30.2d, v2.2d, v11.2d[0] fmla v30.2d, v2.2d, v11.d[0]
fmla v19.2d, v3.2d, v8.2d[0] fmla v19.2d, v3.2d, v8.d[0]
.endm .endm
.macro SAVE8x4 .macro SAVE8x4
@ -422,17 +422,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d, v1.2d}, [pA] ld1 {v0.2d, v1.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v29.2d, v1.2d, v9.2d[1] fmla v29.2d, v1.2d, v9.d[1]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
fmla v25.2d, v1.2d, v9.2d[0] fmla v25.2d, v1.2d, v9.d[0]
fmla v24.2d, v0.2d, v9.2d[0] fmla v24.2d, v0.2d, v9.d[0]
fmla v21.2d, v1.2d, v8.2d[1] fmla v21.2d, v1.2d, v8.d[1]
fmla v28.2d, v0.2d, v9.2d[1] fmla v28.2d, v0.2d, v9.d[1]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
.endm .endm
.macro SAVE4x4 .macro SAVE4x4
@ -482,10 +482,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d}, [pA] ld1 {v0.2d}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
fmla v24.2d, v0.2d, v9.2d[0] fmla v24.2d, v0.2d, v9.d[0]
fmla v28.2d, v0.2d, v9.2d[1] fmla v28.2d, v0.2d, v9.d[1]
.endm .endm
.macro SAVE2x4 .macro SAVE2x4
@ -572,10 +572,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d, v1.2d}, [pA] ld1 {v0.2d, v1.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
fmla v21.2d, v1.2d, v8.2d[1] fmla v21.2d, v1.2d, v8.d[1]
.endm .endm
.macro SAVE4x2 .macro SAVE4x2
@ -610,8 +610,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d}, [pA] ld1 {v0.2d}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
.endm .endm
.macro SAVE2x2 .macro SAVE2x2
@ -643,7 +643,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ldr d0 , [pA] ldr d0 , [pA]
add pA, pA, #8 add pA, pA, #8
fmla v16.2d, v8.2d, v0.2d[0] fmla v16.2d, v8.2d, v0.d[0]
.endm .endm
.macro SAVE1x2 .macro SAVE1x2
@ -674,8 +674,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d, v1.2d}, [pA] ld1 {v0.2d, v1.2d}, [pA]
add pA , pA, #32 add pA , pA, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
.endm .endm
.macro SAVE4x1 .macro SAVE4x1
@ -705,7 +705,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d}, [pA] ld1 {v0.2d}, [pA]
add pA , pA, #16 add pA , pA, #16
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
.endm .endm
.macro SAVE2x1 .macro SAVE2x1

284
kernel/arm64/dgemm_kernel_4x8.S Executable file → Normal file
View File

@ -154,25 +154,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v10.2d, v11.2d}, [pB] ld1 {v10.2d, v11.2d}, [pB]
add pB, pB, #32 add pB, pB, #32
fmul v16.2d, v0.2d, v8.2d[0] fmul v16.2d, v0.2d, v8.d[0]
fmul v17.2d, v1.2d, v8.2d[0] fmul v17.2d, v1.2d, v8.d[0]
fmul v18.2d, v0.2d, v8.2d[1] fmul v18.2d, v0.2d, v8.d[1]
fmul v19.2d, v1.2d, v8.2d[1] fmul v19.2d, v1.2d, v8.d[1]
fmul v20.2d, v0.2d, v9.2d[0] fmul v20.2d, v0.2d, v9.d[0]
fmul v21.2d, v1.2d, v9.2d[0] fmul v21.2d, v1.2d, v9.d[0]
fmul v22.2d, v0.2d, v9.2d[1] fmul v22.2d, v0.2d, v9.d[1]
fmul v23.2d, v1.2d, v9.2d[1] fmul v23.2d, v1.2d, v9.d[1]
fmul v24.2d, v0.2d, v10.2d[0] fmul v24.2d, v0.2d, v10.d[0]
fmul v25.2d, v1.2d, v10.2d[0] fmul v25.2d, v1.2d, v10.d[0]
fmul v26.2d, v0.2d, v10.2d[1] fmul v26.2d, v0.2d, v10.d[1]
fmul v27.2d, v1.2d, v10.2d[1] fmul v27.2d, v1.2d, v10.d[1]
fmul v28.2d, v0.2d, v11.2d[0] fmul v28.2d, v0.2d, v11.d[0]
fmul v29.2d, v1.2d, v11.2d[0] fmul v29.2d, v1.2d, v11.d[0]
fmul v30.2d, v0.2d, v11.2d[1] fmul v30.2d, v0.2d, v11.d[1]
fmul v31.2d, v1.2d, v11.2d[1] fmul v31.2d, v1.2d, v11.d[1]
ld1 {v12.2d, v13.2d}, [pB] ld1 {v12.2d, v13.2d}, [pB]
add pB, pB, #32 add pB, pB, #32
@ -183,25 +183,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL4x8_M1 .macro KERNEL4x8_M1
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
fmla v18.2d, v0.2d, v8.2d[1] fmla v18.2d, v0.2d, v8.d[1]
fmla v19.2d, v1.2d, v8.2d[1] fmla v19.2d, v1.2d, v8.d[1]
fmla v20.2d, v0.2d, v9.2d[0] fmla v20.2d, v0.2d, v9.d[0]
fmla v21.2d, v1.2d, v9.2d[0] fmla v21.2d, v1.2d, v9.d[0]
fmla v22.2d, v0.2d, v9.2d[1] fmla v22.2d, v0.2d, v9.d[1]
fmla v23.2d, v1.2d, v9.2d[1] fmla v23.2d, v1.2d, v9.d[1]
fmla v24.2d, v0.2d, v10.2d[0] fmla v24.2d, v0.2d, v10.d[0]
fmla v25.2d, v1.2d, v10.2d[0] fmla v25.2d, v1.2d, v10.d[0]
fmla v26.2d, v0.2d, v10.2d[1] fmla v26.2d, v0.2d, v10.d[1]
fmla v27.2d, v1.2d, v10.2d[1] fmla v27.2d, v1.2d, v10.d[1]
fmla v28.2d, v0.2d, v11.2d[0] fmla v28.2d, v0.2d, v11.d[0]
fmla v29.2d, v1.2d, v11.2d[0] fmla v29.2d, v1.2d, v11.d[0]
fmla v30.2d, v0.2d, v11.2d[1] fmla v30.2d, v0.2d, v11.d[1]
fmla v31.2d, v1.2d, v11.2d[1] fmla v31.2d, v1.2d, v11.d[1]
ld1 {v12.2d, v13.2d}, [pB] // For next round ld1 {v12.2d, v13.2d}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
@ -214,25 +214,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL4x8_M2 .macro KERNEL4x8_M2
fmla v16.2d, v4.2d, v12.2d[0] fmla v16.2d, v4.2d, v12.d[0]
fmla v17.2d, v5.2d, v12.2d[0] fmla v17.2d, v5.2d, v12.d[0]
fmla v18.2d, v4.2d, v12.2d[1] fmla v18.2d, v4.2d, v12.d[1]
fmla v19.2d, v5.2d, v12.2d[1] fmla v19.2d, v5.2d, v12.d[1]
fmla v20.2d, v4.2d, v13.2d[0] fmla v20.2d, v4.2d, v13.d[0]
fmla v21.2d, v5.2d, v13.2d[0] fmla v21.2d, v5.2d, v13.d[0]
fmla v22.2d, v4.2d, v13.2d[1] fmla v22.2d, v4.2d, v13.d[1]
fmla v23.2d, v5.2d, v13.2d[1] fmla v23.2d, v5.2d, v13.d[1]
fmla v24.2d, v4.2d, v14.2d[0] fmla v24.2d, v4.2d, v14.d[0]
fmla v25.2d, v5.2d, v14.2d[0] fmla v25.2d, v5.2d, v14.d[0]
fmla v26.2d, v4.2d, v14.2d[1] fmla v26.2d, v4.2d, v14.d[1]
fmla v27.2d, v5.2d, v14.2d[1] fmla v27.2d, v5.2d, v14.d[1]
fmla v28.2d, v4.2d, v15.2d[0] fmla v28.2d, v4.2d, v15.d[0]
fmla v29.2d, v5.2d, v15.2d[0] fmla v29.2d, v5.2d, v15.d[0]
fmla v30.2d, v4.2d, v15.2d[1] fmla v30.2d, v4.2d, v15.d[1]
fmla v31.2d, v5.2d, v15.2d[1] fmla v31.2d, v5.2d, v15.d[1]
ld1 {v8.2d, v9.2d}, [pB] // For next round ld1 {v8.2d, v9.2d}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
@ -245,25 +245,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL4x8_E .macro KERNEL4x8_E
fmla v16.2d, v4.2d, v12.2d[0] fmla v16.2d, v4.2d, v12.d[0]
fmla v17.2d, v5.2d, v12.2d[0] fmla v17.2d, v5.2d, v12.d[0]
fmla v18.2d, v4.2d, v12.2d[1] fmla v18.2d, v4.2d, v12.d[1]
fmla v19.2d, v5.2d, v12.2d[1] fmla v19.2d, v5.2d, v12.d[1]
fmla v20.2d, v4.2d, v13.2d[0] fmla v20.2d, v4.2d, v13.d[0]
fmla v21.2d, v5.2d, v13.2d[0] fmla v21.2d, v5.2d, v13.d[0]
fmla v22.2d, v4.2d, v13.2d[1] fmla v22.2d, v4.2d, v13.d[1]
fmla v23.2d, v5.2d, v13.2d[1] fmla v23.2d, v5.2d, v13.d[1]
fmla v24.2d, v4.2d, v14.2d[0] fmla v24.2d, v4.2d, v14.d[0]
fmla v25.2d, v5.2d, v14.2d[0] fmla v25.2d, v5.2d, v14.d[0]
fmla v26.2d, v4.2d, v14.2d[1] fmla v26.2d, v4.2d, v14.d[1]
fmla v27.2d, v5.2d, v14.2d[1] fmla v27.2d, v5.2d, v14.d[1]
fmla v28.2d, v4.2d, v15.2d[0] fmla v28.2d, v4.2d, v15.d[0]
fmla v29.2d, v5.2d, v15.2d[0] fmla v29.2d, v5.2d, v15.d[0]
fmla v30.2d, v4.2d, v15.2d[1] fmla v30.2d, v4.2d, v15.d[1]
fmla v31.2d, v5.2d, v15.2d[1] fmla v31.2d, v5.2d, v15.d[1]
.endm .endm
.macro KERNEL4x8_SUB .macro KERNEL4x8_SUB
@ -274,25 +274,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v10.2d, v11.2d}, [pB] ld1 {v10.2d, v11.2d}, [pB]
add pB, pB, #32 add pB, pB, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
fmla v18.2d, v0.2d, v8.2d[1] fmla v18.2d, v0.2d, v8.d[1]
fmla v19.2d, v1.2d, v8.2d[1] fmla v19.2d, v1.2d, v8.d[1]
fmla v20.2d, v0.2d, v9.2d[0] fmla v20.2d, v0.2d, v9.d[0]
fmla v21.2d, v1.2d, v9.2d[0] fmla v21.2d, v1.2d, v9.d[0]
fmla v22.2d, v0.2d, v9.2d[1] fmla v22.2d, v0.2d, v9.d[1]
fmla v23.2d, v1.2d, v9.2d[1] fmla v23.2d, v1.2d, v9.d[1]
fmla v24.2d, v0.2d, v10.2d[0] fmla v24.2d, v0.2d, v10.d[0]
fmla v25.2d, v1.2d, v10.2d[0] fmla v25.2d, v1.2d, v10.d[0]
fmla v26.2d, v0.2d, v10.2d[1] fmla v26.2d, v0.2d, v10.d[1]
fmla v27.2d, v1.2d, v10.2d[1] fmla v27.2d, v1.2d, v10.d[1]
fmla v28.2d, v0.2d, v11.2d[0] fmla v28.2d, v0.2d, v11.d[0]
fmla v29.2d, v1.2d, v11.2d[0] fmla v29.2d, v1.2d, v11.d[0]
fmla v30.2d, v0.2d, v11.2d[1] fmla v30.2d, v0.2d, v11.d[1]
fmla v31.2d, v1.2d, v11.2d[1] fmla v31.2d, v1.2d, v11.d[1]
.endm .endm
.macro SAVE4x8 .macro SAVE4x8
@ -374,17 +374,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v10.2d, v11.2d}, [pB] ld1 {v10.2d, v11.2d}, [pB]
add pB, pB, #32 add pB, pB, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v18.2d, v0.2d, v8.2d[1] fmla v18.2d, v0.2d, v8.d[1]
fmla v20.2d, v0.2d, v9.2d[0] fmla v20.2d, v0.2d, v9.d[0]
fmla v22.2d, v0.2d, v9.2d[1] fmla v22.2d, v0.2d, v9.d[1]
fmla v24.2d, v0.2d, v10.2d[0] fmla v24.2d, v0.2d, v10.d[0]
fmla v26.2d, v0.2d, v10.2d[1] fmla v26.2d, v0.2d, v10.d[1]
fmla v28.2d, v0.2d, v11.2d[0] fmla v28.2d, v0.2d, v11.d[0]
fmla v30.2d, v0.2d, v11.2d[1] fmla v30.2d, v0.2d, v11.d[1]
.endm .endm
.macro SAVE2x8 .macro SAVE2x8
@ -520,17 +520,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d, v1.2d}, [pA] ld1 {v0.2d, v1.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
fmul v16.2d, v0.2d, v8.2d[0] fmul v16.2d, v0.2d, v8.d[0]
fmul v29.2d, v1.2d, v9.2d[1] fmul v29.2d, v1.2d, v9.d[1]
fmul v20.2d, v0.2d, v8.2d[1] fmul v20.2d, v0.2d, v8.d[1]
fmul v25.2d, v1.2d, v9.2d[0] fmul v25.2d, v1.2d, v9.d[0]
fmul v24.2d, v0.2d, v9.2d[0] fmul v24.2d, v0.2d, v9.d[0]
fmul v21.2d, v1.2d, v8.2d[1] fmul v21.2d, v1.2d, v8.d[1]
fmul v28.2d, v0.2d, v9.2d[1] fmul v28.2d, v0.2d, v9.d[1]
fmul v17.2d, v1.2d, v8.2d[0] fmul v17.2d, v1.2d, v8.d[0]
ld1 {v12.2d, v13.2d}, [pB] ld1 {v12.2d, v13.2d}, [pB]
add pB, pB, #32 add pB, pB, #32
@ -539,61 +539,61 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL4x4_M1 .macro KERNEL4x4_M1
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v29.2d, v1.2d, v9.2d[1] fmla v29.2d, v1.2d, v9.d[1]
ld1 {v12.2d, v13.2d}, [pB] // For next round ld1 {v12.2d, v13.2d}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
fmla v25.2d, v1.2d, v9.2d[0] fmla v25.2d, v1.2d, v9.d[0]
ld1 {v4.2d, v5.2d}, [pA] // For next round ld1 {v4.2d, v5.2d}, [pA] // For next round
add pA, pA, #32 add pA, pA, #32
fmla v24.2d, v0.2d, v9.2d[0] fmla v24.2d, v0.2d, v9.d[0]
fmla v21.2d, v1.2d, v8.2d[1] fmla v21.2d, v1.2d, v8.d[1]
prfm PLDL1KEEP, [pA, #512] prfm PLDL1KEEP, [pA, #512]
fmla v28.2d, v0.2d, v9.2d[1] fmla v28.2d, v0.2d, v9.d[1]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
.endm .endm
.macro KERNEL4x4_M2 .macro KERNEL4x4_M2
fmla v16.2d, v4.2d, v12.2d[0] fmla v16.2d, v4.2d, v12.d[0]
fmla v29.2d, v5.2d, v13.2d[1] fmla v29.2d, v5.2d, v13.d[1]
ld1 {v8.2d, v9.2d}, [pB] // For next round ld1 {v8.2d, v9.2d}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
fmla v20.2d, v4.2d, v12.2d[1] fmla v20.2d, v4.2d, v12.d[1]
fmla v25.2d, v5.2d, v13.2d[0] fmla v25.2d, v5.2d, v13.d[0]
ld1 {v0.2d, v1.2d}, [pA] // For next round ld1 {v0.2d, v1.2d}, [pA] // For next round
add pA, pA, #32 add pA, pA, #32
fmla v24.2d, v4.2d, v13.2d[0] fmla v24.2d, v4.2d, v13.d[0]
fmla v21.2d, v5.2d, v12.2d[1] fmla v21.2d, v5.2d, v12.d[1]
prfm PLDL1KEEP, [pB, #512] prfm PLDL1KEEP, [pB, #512]
fmla v28.2d, v4.2d, v13.2d[1] fmla v28.2d, v4.2d, v13.d[1]
fmla v17.2d, v5.2d, v12.2d[0] fmla v17.2d, v5.2d, v12.d[0]
.endm .endm
.macro KERNEL4x4_E .macro KERNEL4x4_E
fmla v16.2d, v4.2d, v12.2d[0] fmla v16.2d, v4.2d, v12.d[0]
fmla v29.2d, v5.2d, v13.2d[1] fmla v29.2d, v5.2d, v13.d[1]
fmla v20.2d, v4.2d, v12.2d[1] fmla v20.2d, v4.2d, v12.d[1]
fmla v25.2d, v5.2d, v13.2d[0] fmla v25.2d, v5.2d, v13.d[0]
fmla v24.2d, v4.2d, v13.2d[0] fmla v24.2d, v4.2d, v13.d[0]
fmla v21.2d, v5.2d, v12.2d[1] fmla v21.2d, v5.2d, v12.d[1]
fmla v28.2d, v4.2d, v13.2d[1] fmla v28.2d, v4.2d, v13.d[1]
fmla v17.2d, v5.2d, v12.2d[0] fmla v17.2d, v5.2d, v12.d[0]
.endm .endm
.macro KERNEL4x4_SUB .macro KERNEL4x4_SUB
@ -602,17 +602,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d, v1.2d}, [pA] ld1 {v0.2d, v1.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v29.2d, v1.2d, v9.2d[1] fmla v29.2d, v1.2d, v9.d[1]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
fmla v25.2d, v1.2d, v9.2d[0] fmla v25.2d, v1.2d, v9.d[0]
fmla v24.2d, v0.2d, v9.2d[0] fmla v24.2d, v0.2d, v9.d[0]
fmla v21.2d, v1.2d, v8.2d[1] fmla v21.2d, v1.2d, v8.d[1]
fmla v28.2d, v0.2d, v9.2d[1] fmla v28.2d, v0.2d, v9.d[1]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
.endm .endm
.macro SAVE4x4 .macro SAVE4x4
@ -660,10 +660,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d}, [pA] ld1 {v0.2d}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
fmla v24.2d, v0.2d, v9.2d[0] fmla v24.2d, v0.2d, v9.d[0]
fmla v28.2d, v0.2d, v9.2d[1] fmla v28.2d, v0.2d, v9.d[1]
.endm .endm
.macro SAVE2x4 .macro SAVE2x4
@ -746,10 +746,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d, v1.2d}, [pA] ld1 {v0.2d, v1.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
fmla v21.2d, v1.2d, v8.2d[1] fmla v21.2d, v1.2d, v8.d[1]
.endm .endm
.macro SAVE4x2 .macro SAVE4x2
@ -782,8 +782,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d}, [pA] ld1 {v0.2d}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
.endm .endm
.macro SAVE2x2 .macro SAVE2x2
@ -813,7 +813,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ldr d0 , [pA] ldr d0 , [pA]
add pA, pA, #8 add pA, pA, #8
fmla v16.2d, v8.2d, v0.2d[0] fmla v16.2d, v8.2d, v0.d[0]
.endm .endm
.macro SAVE1x2 .macro SAVE1x2
@ -842,8 +842,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d, v1.2d}, [pA] ld1 {v0.2d, v1.2d}, [pA]
add pA , pA, #32 add pA , pA, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
.endm .endm
.macro SAVE4x1 .macro SAVE4x1
@ -871,7 +871,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d}, [pA] ld1 {v0.2d}, [pA]
add pA , pA, #16 add pA , pA, #16
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
.endm .endm
.macro SAVE2x1 .macro SAVE2x1

228
kernel/arm64/dgemm_kernel_8x4.S Executable file → Normal file
View File

@ -151,141 +151,141 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ldp d8, d9, [pB], #16 ldp d8, d9, [pB], #16
fmul v16.2d, v0.2d, v8.2d[0] fmul v16.2d, v0.2d, v8.d[0]
fmul v20.2d, v0.2d, v9.2d[0] fmul v20.2d, v0.2d, v9.d[0]
ldp d10, d11, [pB], #16 ldp d10, d11, [pB], #16
fmul v17.2d, v1.2d, v8.2d[0] fmul v17.2d, v1.2d, v8.d[0]
fmul v21.2d, v1.2d, v9.2d[0] fmul v21.2d, v1.2d, v9.d[0]
ldp q2, q3, [pA], #32 ldp q2, q3, [pA], #32
fmul v24.2d, v0.2d, v10.2d[0] fmul v24.2d, v0.2d, v10.d[0]
fmul v28.2d, v0.2d, v11.2d[0] fmul v28.2d, v0.2d, v11.d[0]
ldp q4, q5, [pA], #32 ldp q4, q5, [pA], #32
fmul v25.2d, v1.2d, v10.2d[0] fmul v25.2d, v1.2d, v10.d[0]
fmul v29.2d, v1.2d, v11.2d[0] fmul v29.2d, v1.2d, v11.d[0]
ldp d12, d13, [pB], #16 ldp d12, d13, [pB], #16
fmul v18.2d, v2.2d, v8.2d[0] fmul v18.2d, v2.2d, v8.d[0]
fmul v22.2d, v2.2d, v9.2d[0] fmul v22.2d, v2.2d, v9.d[0]
ldp d14, d15, [pB], #16 ldp d14, d15, [pB], #16
fmul v26.2d, v2.2d, v10.2d[0] fmul v26.2d, v2.2d, v10.d[0]
fmul v30.2d, v2.2d, v11.2d[0] fmul v30.2d, v2.2d, v11.d[0]
ldp q6, q7, [pA], #32 ldp q6, q7, [pA], #32
fmul v19.2d, v3.2d, v8.2d[0] fmul v19.2d, v3.2d, v8.d[0]
fmul v27.2d, v3.2d, v10.2d[0] fmul v27.2d, v3.2d, v10.d[0]
prfm PLDL1KEEP, [pA, #A_PRE_SIZE] prfm PLDL1KEEP, [pA, #A_PRE_SIZE]
fmul v31.2d, v3.2d, v11.2d[0] fmul v31.2d, v3.2d, v11.d[0]
fmul v23.2d, v3.2d, v9.2d[0] fmul v23.2d, v3.2d, v9.d[0]
prfm PLDL1KEEP, [pA, #A_PRE_SIZE+64] prfm PLDL1KEEP, [pA, #A_PRE_SIZE+64]
.endm .endm
.macro KERNEL8x4_M1 .macro KERNEL8x4_M1
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v20.2d, v0.2d, v9.2d[0] fmla v20.2d, v0.2d, v9.d[0]
ldp q4, q5, [pA], #32 ldp q4, q5, [pA], #32
fmla v24.2d, v0.2d, v10.2d[0] fmla v24.2d, v0.2d, v10.d[0]
fmla v28.2d, v0.2d, v11.2d[0] fmla v28.2d, v0.2d, v11.d[0]
ldp d12, d13, [pB], #16 ldp d12, d13, [pB], #16
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
fmla v25.2d, v1.2d, v10.2d[0] fmla v25.2d, v1.2d, v10.d[0]
prfm PLDL1KEEP, [pA, #A_PRE_SIZE+64] prfm PLDL1KEEP, [pA, #A_PRE_SIZE+64]
fmla v21.2d, v1.2d, v9.2d[0] fmla v21.2d, v1.2d, v9.d[0]
fmla v29.2d, v1.2d, v11.2d[0] fmla v29.2d, v1.2d, v11.d[0]
ldp d14, d15, [pB], #16 ldp d14, d15, [pB], #16
fmla v18.2d, v2.2d, v8.2d[0] fmla v18.2d, v2.2d, v8.d[0]
fmla v22.2d, v2.2d, v9.2d[0] fmla v22.2d, v2.2d, v9.d[0]
prfm PLDL1KEEP, [pA, #A_PRE_SIZE] prfm PLDL1KEEP, [pA, #A_PRE_SIZE]
fmla v26.2d, v2.2d, v10.2d[0] fmla v26.2d, v2.2d, v10.d[0]
fmla v30.2d, v2.2d, v11.2d[0] fmla v30.2d, v2.2d, v11.d[0]
fmla v19.2d, v3.2d, v8.2d[0] fmla v19.2d, v3.2d, v8.d[0]
fmla v23.2d, v3.2d, v9.2d[0] fmla v23.2d, v3.2d, v9.d[0]
ldp q6, q7, [pA], #32 ldp q6, q7, [pA], #32
fmla v27.2d, v3.2d, v10.2d[0] fmla v27.2d, v3.2d, v10.d[0]
fmla v31.2d, v3.2d, v11.2d[0] fmla v31.2d, v3.2d, v11.d[0]
.endm .endm
.macro KERNEL8x4_M2 .macro KERNEL8x4_M2
fmla v16.2d, v4.2d, v12.2d[0] fmla v16.2d, v4.2d, v12.d[0]
fmla v20.2d, v4.2d, v13.2d[0] fmla v20.2d, v4.2d, v13.d[0]
fmla v24.2d, v4.2d, v14.2d[0] fmla v24.2d, v4.2d, v14.d[0]
fmla v28.2d, v4.2d, v15.2d[0] fmla v28.2d, v4.2d, v15.d[0]
ldp q0, q1, [pA], #32 ldp q0, q1, [pA], #32
fmla v17.2d, v5.2d, v12.2d[0] fmla v17.2d, v5.2d, v12.d[0]
fmla v25.2d, v5.2d, v14.2d[0] fmla v25.2d, v5.2d, v14.d[0]
ldp d8, d9, [pB], #16 ldp d8, d9, [pB], #16
fmla v21.2d, v5.2d, v13.2d[0] fmla v21.2d, v5.2d, v13.d[0]
fmla v29.2d, v5.2d, v15.2d[0] fmla v29.2d, v5.2d, v15.d[0]
ldp d10, d11, [pB], #16 ldp d10, d11, [pB], #16
fmla v18.2d, v6.2d, v12.2d[0] fmla v18.2d, v6.2d, v12.d[0]
fmla v22.2d, v6.2d, v13.2d[0] fmla v22.2d, v6.2d, v13.d[0]
prfm PLDL1KEEP, [pB, #B_PRE_SIZE] prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
fmla v26.2d, v6.2d, v14.2d[0] fmla v26.2d, v6.2d, v14.d[0]
fmla v30.2d, v6.2d, v15.2d[0] fmla v30.2d, v6.2d, v15.d[0]
fmla v19.2d, v7.2d, v12.2d[0] fmla v19.2d, v7.2d, v12.d[0]
fmla v23.2d, v7.2d, v13.2d[0] fmla v23.2d, v7.2d, v13.d[0]
ldp q2, q3, [pA], #32 ldp q2, q3, [pA], #32
fmla v27.2d, v7.2d, v14.2d[0] fmla v27.2d, v7.2d, v14.d[0]
fmla v31.2d, v7.2d, v15.2d[0] fmla v31.2d, v7.2d, v15.d[0]
.endm .endm
.macro KERNEL8x4_E .macro KERNEL8x4_E
fmla v16.2d, v4.2d, v12.2d[0] fmla v16.2d, v4.2d, v12.d[0]
fmla v20.2d, v4.2d, v13.2d[0] fmla v20.2d, v4.2d, v13.d[0]
fmla v24.2d, v4.2d, v14.2d[0] fmla v24.2d, v4.2d, v14.d[0]
fmla v28.2d, v4.2d, v15.2d[0] fmla v28.2d, v4.2d, v15.d[0]
fmla v17.2d, v5.2d, v12.2d[0] fmla v17.2d, v5.2d, v12.d[0]
fmla v25.2d, v5.2d, v14.2d[0] fmla v25.2d, v5.2d, v14.d[0]
fmla v21.2d, v5.2d, v13.2d[0] fmla v21.2d, v5.2d, v13.d[0]
fmla v29.2d, v5.2d, v15.2d[0] fmla v29.2d, v5.2d, v15.d[0]
prfm PLDL1KEEP, [pB, #B_PRE_SIZE] prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
fmla v18.2d, v6.2d, v12.2d[0] fmla v18.2d, v6.2d, v12.d[0]
fmla v22.2d, v6.2d, v13.2d[0] fmla v22.2d, v6.2d, v13.d[0]
fmla v26.2d, v6.2d, v14.2d[0] fmla v26.2d, v6.2d, v14.d[0]
fmla v30.2d, v6.2d, v15.2d[0] fmla v30.2d, v6.2d, v15.d[0]
fmla v19.2d, v7.2d, v12.2d[0] fmla v19.2d, v7.2d, v12.d[0]
fmla v23.2d, v7.2d, v13.2d[0] fmla v23.2d, v7.2d, v13.d[0]
fmla v27.2d, v7.2d, v14.2d[0] fmla v27.2d, v7.2d, v14.d[0]
fmla v31.2d, v7.2d, v15.2d[0] fmla v31.2d, v7.2d, v15.d[0]
.endm .endm
.macro KERNEL8x4_SUB .macro KERNEL8x4_SUB
@ -293,39 +293,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ldp d8, d9, [pB], #16 ldp d8, d9, [pB], #16
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v20.2d, v0.2d, v9.2d[0] fmla v20.2d, v0.2d, v9.d[0]
ldp d10, d11, [pB], #16 ldp d10, d11, [pB], #16
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
fmla v21.2d, v1.2d, v9.2d[0] fmla v21.2d, v1.2d, v9.d[0]
ldp q2, q3, [pA], #32 ldp q2, q3, [pA], #32
fmla v24.2d, v0.2d, v10.2d[0] fmla v24.2d, v0.2d, v10.d[0]
fmla v28.2d, v0.2d, v11.2d[0] fmla v28.2d, v0.2d, v11.d[0]
fmla v25.2d, v1.2d, v10.2d[0] fmla v25.2d, v1.2d, v10.d[0]
fmla v29.2d, v1.2d, v11.2d[0] fmla v29.2d, v1.2d, v11.d[0]
prfm PLDL1KEEP, [pA, #A_PRE_SIZE] prfm PLDL1KEEP, [pA, #A_PRE_SIZE]
fmla v18.2d, v2.2d, v8.2d[0] fmla v18.2d, v2.2d, v8.d[0]
fmla v22.2d, v2.2d, v9.2d[0] fmla v22.2d, v2.2d, v9.d[0]
prfm PLDL1KEEP, [pA, #A_PRE_SIZE+64] prfm PLDL1KEEP, [pA, #A_PRE_SIZE+64]
fmla v26.2d, v2.2d, v10.2d[0] fmla v26.2d, v2.2d, v10.d[0]
fmla v30.2d, v2.2d, v11.2d[0] fmla v30.2d, v2.2d, v11.d[0]
prfm PLDL1KEEP, [pB, #B_PRE_SIZE] prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
fmla v19.2d, v3.2d, v8.2d[0] fmla v19.2d, v3.2d, v8.d[0]
fmla v27.2d, v3.2d, v10.2d[0] fmla v27.2d, v3.2d, v10.d[0]
fmla v31.2d, v3.2d, v11.2d[0] fmla v31.2d, v3.2d, v11.d[0]
fmla v23.2d, v3.2d, v9.2d[0] fmla v23.2d, v3.2d, v9.d[0]
.endm .endm
.macro SAVE8x4 .macro SAVE8x4
@ -419,17 +419,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d, v1.2d}, [pA] ld1 {v0.2d, v1.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v29.2d, v1.2d, v9.2d[1] fmla v29.2d, v1.2d, v9.d[1]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
fmla v25.2d, v1.2d, v9.2d[0] fmla v25.2d, v1.2d, v9.d[0]
fmla v24.2d, v0.2d, v9.2d[0] fmla v24.2d, v0.2d, v9.d[0]
fmla v21.2d, v1.2d, v8.2d[1] fmla v21.2d, v1.2d, v8.d[1]
fmla v28.2d, v0.2d, v9.2d[1] fmla v28.2d, v0.2d, v9.d[1]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
.endm .endm
.macro SAVE4x4 .macro SAVE4x4
@ -479,10 +479,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d}, [pA] ld1 {v0.2d}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
fmla v24.2d, v0.2d, v9.2d[0] fmla v24.2d, v0.2d, v9.d[0]
fmla v28.2d, v0.2d, v9.2d[1] fmla v28.2d, v0.2d, v9.d[1]
.endm .endm
.macro SAVE2x4 .macro SAVE2x4
@ -573,15 +573,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v2.2d, v3.2d}, [pA] ld1 {v2.2d, v3.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
fmla v18.2d, v2.2d, v8.2d[0] fmla v18.2d, v2.2d, v8.d[0]
fmla v19.2d, v3.2d, v8.2d[0] fmla v19.2d, v3.2d, v8.d[0]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
fmla v21.2d, v1.2d, v8.2d[1] fmla v21.2d, v1.2d, v8.d[1]
fmla v22.2d, v2.2d, v8.2d[1] fmla v22.2d, v2.2d, v8.d[1]
fmla v23.2d, v3.2d, v8.2d[1] fmla v23.2d, v3.2d, v8.d[1]
.endm .endm
.macro SAVE8x2 .macro SAVE8x2
@ -620,10 +620,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d, v1.2d}, [pA] ld1 {v0.2d, v1.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
fmla v21.2d, v1.2d, v8.2d[1] fmla v21.2d, v1.2d, v8.d[1]
.endm .endm
.macro SAVE4x2 .macro SAVE4x2
@ -657,8 +657,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d}, [pA] ld1 {v0.2d}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
.endm .endm
.macro SAVE2x2 .macro SAVE2x2
@ -689,7 +689,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ldr d0 , [pA] ldr d0 , [pA]
add pA, pA, #8 add pA, pA, #8
fmla v16.2d, v8.2d, v0.2d[0] fmla v16.2d, v8.2d, v0.d[0]
.endm .endm
.macro SAVE1x2 .macro SAVE1x2
@ -724,10 +724,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v2.2d, v3.2d}, [pA] ld1 {v2.2d, v3.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
fmla v18.2d, v2.2d, v8.2d[0] fmla v18.2d, v2.2d, v8.d[0]
fmla v19.2d, v3.2d, v8.2d[0] fmla v19.2d, v3.2d, v8.d[0]
.endm .endm
.macro SAVE8x1 .macro SAVE8x1
@ -757,8 +757,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d, v1.2d}, [pA] ld1 {v0.2d, v1.2d}, [pA]
add pA , pA, #32 add pA , pA, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
.endm .endm
.macro SAVE4x1 .macro SAVE4x1
@ -785,7 +785,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d}, [pA] ld1 {v0.2d}, [pA]
add pA , pA, #16 add pA , pA, #16
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
.endm .endm
.macro SAVE2x1 .macro SAVE2x1

108
kernel/arm64/dtrmm_kernel_4x4.S
View File

@ -147,17 +147,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d, v1.2d}, [pA] ld1 {v0.2d, v1.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
fmul v16.2d, v0.2d, v8.2d[0] fmul v16.2d, v0.2d, v8.d[0]
fmul v29.2d, v1.2d, v9.2d[1] fmul v29.2d, v1.2d, v9.d[1]
fmul v20.2d, v0.2d, v8.2d[1] fmul v20.2d, v0.2d, v8.d[1]
fmul v25.2d, v1.2d, v9.2d[0] fmul v25.2d, v1.2d, v9.d[0]
fmul v24.2d, v0.2d, v9.2d[0] fmul v24.2d, v0.2d, v9.d[0]
fmul v21.2d, v1.2d, v8.2d[1] fmul v21.2d, v1.2d, v8.d[1]
fmul v28.2d, v0.2d, v9.2d[1] fmul v28.2d, v0.2d, v9.d[1]
fmul v17.2d, v1.2d, v8.2d[0] fmul v17.2d, v1.2d, v8.d[0]
ld1 {v12.2d, v13.2d}, [pB] ld1 {v12.2d, v13.2d}, [pB]
add pB, pB, #32 add pB, pB, #32
@ -166,61 +166,61 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL4x4_M1 .macro KERNEL4x4_M1
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v29.2d, v1.2d, v9.2d[1] fmla v29.2d, v1.2d, v9.d[1]
ld1 {v12.2d, v13.2d}, [pB] // For next round ld1 {v12.2d, v13.2d}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
fmla v25.2d, v1.2d, v9.2d[0] fmla v25.2d, v1.2d, v9.d[0]
ld1 {v4.2d, v5.2d}, [pA] // For next round ld1 {v4.2d, v5.2d}, [pA] // For next round
add pA, pA, #32 add pA, pA, #32
fmla v24.2d, v0.2d, v9.2d[0] fmla v24.2d, v0.2d, v9.d[0]
fmla v21.2d, v1.2d, v8.2d[1] fmla v21.2d, v1.2d, v8.d[1]
prfm PLDL1KEEP, [pA, #512] prfm PLDL1KEEP, [pA, #512]
fmla v28.2d, v0.2d, v9.2d[1] fmla v28.2d, v0.2d, v9.d[1]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
.endm .endm
.macro KERNEL4x4_M2 .macro KERNEL4x4_M2
fmla v16.2d, v4.2d, v12.2d[0] fmla v16.2d, v4.2d, v12.d[0]
fmla v29.2d, v5.2d, v13.2d[1] fmla v29.2d, v5.2d, v13.d[1]
ld1 {v8.2d, v9.2d}, [pB] // For next round ld1 {v8.2d, v9.2d}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
fmla v20.2d, v4.2d, v12.2d[1] fmla v20.2d, v4.2d, v12.d[1]
fmla v25.2d, v5.2d, v13.2d[0] fmla v25.2d, v5.2d, v13.d[0]
ld1 {v0.2d, v1.2d}, [pA] // For next round ld1 {v0.2d, v1.2d}, [pA] // For next round
add pA, pA, #32 add pA, pA, #32
fmla v24.2d, v4.2d, v13.2d[0] fmla v24.2d, v4.2d, v13.d[0]
fmla v21.2d, v5.2d, v12.2d[1] fmla v21.2d, v5.2d, v12.d[1]
prfm PLDL1KEEP, [pB, #512] prfm PLDL1KEEP, [pB, #512]
fmla v28.2d, v4.2d, v13.2d[1] fmla v28.2d, v4.2d, v13.d[1]
fmla v17.2d, v5.2d, v12.2d[0] fmla v17.2d, v5.2d, v12.d[0]
.endm .endm
.macro KERNEL4x4_E .macro KERNEL4x4_E
fmla v16.2d, v4.2d, v12.2d[0] fmla v16.2d, v4.2d, v12.d[0]
fmla v29.2d, v5.2d, v13.2d[1] fmla v29.2d, v5.2d, v13.d[1]
fmla v20.2d, v4.2d, v12.2d[1] fmla v20.2d, v4.2d, v12.d[1]
fmla v25.2d, v5.2d, v13.2d[0] fmla v25.2d, v5.2d, v13.d[0]
fmla v24.2d, v4.2d, v13.2d[0] fmla v24.2d, v4.2d, v13.d[0]
fmla v21.2d, v5.2d, v12.2d[1] fmla v21.2d, v5.2d, v12.d[1]
fmla v28.2d, v4.2d, v13.2d[1] fmla v28.2d, v4.2d, v13.d[1]
fmla v17.2d, v5.2d, v12.2d[0] fmla v17.2d, v5.2d, v12.d[0]
.endm .endm
.macro KERNEL4x4_SUB .macro KERNEL4x4_SUB
@ -229,17 +229,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d, v1.2d}, [pA] ld1 {v0.2d, v1.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v29.2d, v1.2d, v9.2d[1] fmla v29.2d, v1.2d, v9.d[1]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
fmla v25.2d, v1.2d, v9.2d[0] fmla v25.2d, v1.2d, v9.d[0]
fmla v24.2d, v0.2d, v9.2d[0] fmla v24.2d, v0.2d, v9.d[0]
fmla v21.2d, v1.2d, v8.2d[1] fmla v21.2d, v1.2d, v8.d[1]
fmla v28.2d, v0.2d, v9.2d[1] fmla v28.2d, v0.2d, v9.d[1]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
.endm .endm
.macro SAVE4x4 .macro SAVE4x4
@ -283,10 +283,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d}, [pA] ld1 {v0.2d}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
fmla v24.2d, v0.2d, v9.2d[0] fmla v24.2d, v0.2d, v9.d[0]
fmla v28.2d, v0.2d, v9.2d[1] fmla v28.2d, v0.2d, v9.d[1]
.endm .endm
.macro SAVE2x4 .macro SAVE2x4
@ -361,10 +361,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d, v1.2d}, [pA] ld1 {v0.2d, v1.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
fmla v21.2d, v1.2d, v8.2d[1] fmla v21.2d, v1.2d, v8.d[1]
.endm .endm
.macro SAVE4x2 .macro SAVE4x2
@ -395,8 +395,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d}, [pA] ld1 {v0.2d}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
.endm .endm
.macro SAVE2x2 .macro SAVE2x2
@ -424,7 +424,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ldr d0 , [pA] ldr d0 , [pA]
add pA, pA, #8 add pA, pA, #8
fmla v16.2d, v8.2d, v0.2d[0] fmla v16.2d, v8.2d, v0.d[0]
.endm .endm
.macro SAVE1x2 .macro SAVE1x2
@ -451,8 +451,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d, v1.2d}, [pA] ld1 {v0.2d, v1.2d}, [pA]
add pA , pA, #32 add pA , pA, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
.endm .endm
.macro SAVE4x1 .macro SAVE4x1
@ -479,7 +479,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d}, [pA] ld1 {v0.2d}, [pA]
add pA , pA, #16 add pA , pA, #16
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
.endm .endm
.macro SAVE2x1 .macro SAVE2x1

284
kernel/arm64/dtrmm_kernel_4x8.S Executable file → Normal file
View File

@ -157,25 +157,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v10.2d, v11.2d}, [pB] ld1 {v10.2d, v11.2d}, [pB]
add pB, pB, #32 add pB, pB, #32
fmul v16.2d, v0.2d, v8.2d[0] fmul v16.2d, v0.2d, v8.d[0]
fmul v17.2d, v1.2d, v8.2d[0] fmul v17.2d, v1.2d, v8.d[0]
fmul v18.2d, v0.2d, v8.2d[1] fmul v18.2d, v0.2d, v8.d[1]
fmul v19.2d, v1.2d, v8.2d[1] fmul v19.2d, v1.2d, v8.d[1]
fmul v20.2d, v0.2d, v9.2d[0] fmul v20.2d, v0.2d, v9.d[0]
fmul v21.2d, v1.2d, v9.2d[0] fmul v21.2d, v1.2d, v9.d[0]
fmul v22.2d, v0.2d, v9.2d[1] fmul v22.2d, v0.2d, v9.d[1]
fmul v23.2d, v1.2d, v9.2d[1] fmul v23.2d, v1.2d, v9.d[1]
fmul v24.2d, v0.2d, v10.2d[0] fmul v24.2d, v0.2d, v10.d[0]
fmul v25.2d, v1.2d, v10.2d[0] fmul v25.2d, v1.2d, v10.d[0]
fmul v26.2d, v0.2d, v10.2d[1] fmul v26.2d, v0.2d, v10.d[1]
fmul v27.2d, v1.2d, v10.2d[1] fmul v27.2d, v1.2d, v10.d[1]
fmul v28.2d, v0.2d, v11.2d[0] fmul v28.2d, v0.2d, v11.d[0]
fmul v29.2d, v1.2d, v11.2d[0] fmul v29.2d, v1.2d, v11.d[0]
fmul v30.2d, v0.2d, v11.2d[1] fmul v30.2d, v0.2d, v11.d[1]
fmul v31.2d, v1.2d, v11.2d[1] fmul v31.2d, v1.2d, v11.d[1]
ld1 {v12.2d, v13.2d}, [pB] ld1 {v12.2d, v13.2d}, [pB]
add pB, pB, #32 add pB, pB, #32
@ -186,25 +186,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL4x8_M1 .macro KERNEL4x8_M1
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
fmla v18.2d, v0.2d, v8.2d[1] fmla v18.2d, v0.2d, v8.d[1]
fmla v19.2d, v1.2d, v8.2d[1] fmla v19.2d, v1.2d, v8.d[1]
fmla v20.2d, v0.2d, v9.2d[0] fmla v20.2d, v0.2d, v9.d[0]
fmla v21.2d, v1.2d, v9.2d[0] fmla v21.2d, v1.2d, v9.d[0]
fmla v22.2d, v0.2d, v9.2d[1] fmla v22.2d, v0.2d, v9.d[1]
fmla v23.2d, v1.2d, v9.2d[1] fmla v23.2d, v1.2d, v9.d[1]
fmla v24.2d, v0.2d, v10.2d[0] fmla v24.2d, v0.2d, v10.d[0]
fmla v25.2d, v1.2d, v10.2d[0] fmla v25.2d, v1.2d, v10.d[0]
fmla v26.2d, v0.2d, v10.2d[1] fmla v26.2d, v0.2d, v10.d[1]
fmla v27.2d, v1.2d, v10.2d[1] fmla v27.2d, v1.2d, v10.d[1]
fmla v28.2d, v0.2d, v11.2d[0] fmla v28.2d, v0.2d, v11.d[0]
fmla v29.2d, v1.2d, v11.2d[0] fmla v29.2d, v1.2d, v11.d[0]
fmla v30.2d, v0.2d, v11.2d[1] fmla v30.2d, v0.2d, v11.d[1]
fmla v31.2d, v1.2d, v11.2d[1] fmla v31.2d, v1.2d, v11.d[1]
ld1 {v12.2d, v13.2d}, [pB] // For next round ld1 {v12.2d, v13.2d}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
@ -217,25 +217,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL4x8_M2 .macro KERNEL4x8_M2
fmla v16.2d, v4.2d, v12.2d[0] fmla v16.2d, v4.2d, v12.d[0]
fmla v17.2d, v5.2d, v12.2d[0] fmla v17.2d, v5.2d, v12.d[0]
fmla v18.2d, v4.2d, v12.2d[1] fmla v18.2d, v4.2d, v12.d[1]
fmla v19.2d, v5.2d, v12.2d[1] fmla v19.2d, v5.2d, v12.d[1]
fmla v20.2d, v4.2d, v13.2d[0] fmla v20.2d, v4.2d, v13.d[0]
fmla v21.2d, v5.2d, v13.2d[0] fmla v21.2d, v5.2d, v13.d[0]
fmla v22.2d, v4.2d, v13.2d[1] fmla v22.2d, v4.2d, v13.d[1]
fmla v23.2d, v5.2d, v13.2d[1] fmla v23.2d, v5.2d, v13.d[1]
fmla v24.2d, v4.2d, v14.2d[0] fmla v24.2d, v4.2d, v14.d[0]
fmla v25.2d, v5.2d, v14.2d[0] fmla v25.2d, v5.2d, v14.d[0]
fmla v26.2d, v4.2d, v14.2d[1] fmla v26.2d, v4.2d, v14.d[1]
fmla v27.2d, v5.2d, v14.2d[1] fmla v27.2d, v5.2d, v14.d[1]
fmla v28.2d, v4.2d, v15.2d[0] fmla v28.2d, v4.2d, v15.d[0]
fmla v29.2d, v5.2d, v15.2d[0] fmla v29.2d, v5.2d, v15.d[0]
fmla v30.2d, v4.2d, v15.2d[1] fmla v30.2d, v4.2d, v15.d[1]
fmla v31.2d, v5.2d, v15.2d[1] fmla v31.2d, v5.2d, v15.d[1]
ld1 {v8.2d, v9.2d}, [pB] // For next round ld1 {v8.2d, v9.2d}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
@ -248,25 +248,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL4x8_E .macro KERNEL4x8_E
fmla v16.2d, v4.2d, v12.2d[0] fmla v16.2d, v4.2d, v12.d[0]
fmla v17.2d, v5.2d, v12.2d[0] fmla v17.2d, v5.2d, v12.d[0]
fmla v18.2d, v4.2d, v12.2d[1] fmla v18.2d, v4.2d, v12.d[1]
fmla v19.2d, v5.2d, v12.2d[1] fmla v19.2d, v5.2d, v12.d[1]
fmla v20.2d, v4.2d, v13.2d[0] fmla v20.2d, v4.2d, v13.d[0]
fmla v21.2d, v5.2d, v13.2d[0] fmla v21.2d, v5.2d, v13.d[0]
fmla v22.2d, v4.2d, v13.2d[1] fmla v22.2d, v4.2d, v13.d[1]
fmla v23.2d, v5.2d, v13.2d[1] fmla v23.2d, v5.2d, v13.d[1]
fmla v24.2d, v4.2d, v14.2d[0] fmla v24.2d, v4.2d, v14.d[0]
fmla v25.2d, v5.2d, v14.2d[0] fmla v25.2d, v5.2d, v14.d[0]
fmla v26.2d, v4.2d, v14.2d[1] fmla v26.2d, v4.2d, v14.d[1]
fmla v27.2d, v5.2d, v14.2d[1] fmla v27.2d, v5.2d, v14.d[1]
fmla v28.2d, v4.2d, v15.2d[0] fmla v28.2d, v4.2d, v15.d[0]
fmla v29.2d, v5.2d, v15.2d[0] fmla v29.2d, v5.2d, v15.d[0]
fmla v30.2d, v4.2d, v15.2d[1] fmla v30.2d, v4.2d, v15.d[1]
fmla v31.2d, v5.2d, v15.2d[1] fmla v31.2d, v5.2d, v15.d[1]
.endm .endm
.macro KERNEL4x8_SUB .macro KERNEL4x8_SUB
@ -277,25 +277,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v10.2d, v11.2d}, [pB] ld1 {v10.2d, v11.2d}, [pB]
add pB, pB, #32 add pB, pB, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
fmla v18.2d, v0.2d, v8.2d[1] fmla v18.2d, v0.2d, v8.d[1]
fmla v19.2d, v1.2d, v8.2d[1] fmla v19.2d, v1.2d, v8.d[1]
fmla v20.2d, v0.2d, v9.2d[0] fmla v20.2d, v0.2d, v9.d[0]
fmla v21.2d, v1.2d, v9.2d[0] fmla v21.2d, v1.2d, v9.d[0]
fmla v22.2d, v0.2d, v9.2d[1] fmla v22.2d, v0.2d, v9.d[1]
fmla v23.2d, v1.2d, v9.2d[1] fmla v23.2d, v1.2d, v9.d[1]
fmla v24.2d, v0.2d, v10.2d[0] fmla v24.2d, v0.2d, v10.d[0]
fmla v25.2d, v1.2d, v10.2d[0] fmla v25.2d, v1.2d, v10.d[0]
fmla v26.2d, v0.2d, v10.2d[1] fmla v26.2d, v0.2d, v10.d[1]
fmla v27.2d, v1.2d, v10.2d[1] fmla v27.2d, v1.2d, v10.d[1]
fmla v28.2d, v0.2d, v11.2d[0] fmla v28.2d, v0.2d, v11.d[0]
fmla v29.2d, v1.2d, v11.2d[0] fmla v29.2d, v1.2d, v11.d[0]
fmla v30.2d, v0.2d, v11.2d[1] fmla v30.2d, v0.2d, v11.d[1]
fmla v31.2d, v1.2d, v11.2d[1] fmla v31.2d, v1.2d, v11.d[1]
.endm .endm
.macro SAVE4x8 .macro SAVE4x8
@ -369,17 +369,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v10.2d, v11.2d}, [pB] ld1 {v10.2d, v11.2d}, [pB]
add pB, pB, #32 add pB, pB, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v18.2d, v0.2d, v8.2d[1] fmla v18.2d, v0.2d, v8.d[1]
fmla v20.2d, v0.2d, v9.2d[0] fmla v20.2d, v0.2d, v9.d[0]
fmla v22.2d, v0.2d, v9.2d[1] fmla v22.2d, v0.2d, v9.d[1]
fmla v24.2d, v0.2d, v10.2d[0] fmla v24.2d, v0.2d, v10.d[0]
fmla v26.2d, v0.2d, v10.2d[1] fmla v26.2d, v0.2d, v10.d[1]
fmla v28.2d, v0.2d, v11.2d[0] fmla v28.2d, v0.2d, v11.d[0]
fmla v30.2d, v0.2d, v11.2d[1] fmla v30.2d, v0.2d, v11.d[1]
.endm .endm
.macro SAVE2x8 .macro SAVE2x8
@ -499,17 +499,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d, v1.2d}, [pA] ld1 {v0.2d, v1.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
fmul v16.2d, v0.2d, v8.2d[0] fmul v16.2d, v0.2d, v8.d[0]
fmul v29.2d, v1.2d, v9.2d[1] fmul v29.2d, v1.2d, v9.d[1]
fmul v20.2d, v0.2d, v8.2d[1] fmul v20.2d, v0.2d, v8.d[1]
fmul v25.2d, v1.2d, v9.2d[0] fmul v25.2d, v1.2d, v9.d[0]
fmul v24.2d, v0.2d, v9.2d[0] fmul v24.2d, v0.2d, v9.d[0]
fmul v21.2d, v1.2d, v8.2d[1] fmul v21.2d, v1.2d, v8.d[1]
fmul v28.2d, v0.2d, v9.2d[1] fmul v28.2d, v0.2d, v9.d[1]
fmul v17.2d, v1.2d, v8.2d[0] fmul v17.2d, v1.2d, v8.d[0]
ld1 {v12.2d, v13.2d}, [pB] ld1 {v12.2d, v13.2d}, [pB]
add pB, pB, #32 add pB, pB, #32
@ -518,61 +518,61 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL4x4_M1 .macro KERNEL4x4_M1
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v29.2d, v1.2d, v9.2d[1] fmla v29.2d, v1.2d, v9.d[1]
ld1 {v12.2d, v13.2d}, [pB] // For next round ld1 {v12.2d, v13.2d}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
fmla v25.2d, v1.2d, v9.2d[0] fmla v25.2d, v1.2d, v9.d[0]
ld1 {v4.2d, v5.2d}, [pA] // For next round ld1 {v4.2d, v5.2d}, [pA] // For next round
add pA, pA, #32 add pA, pA, #32
fmla v24.2d, v0.2d, v9.2d[0] fmla v24.2d, v0.2d, v9.d[0]
fmla v21.2d, v1.2d, v8.2d[1] fmla v21.2d, v1.2d, v8.d[1]
prfm PLDL1KEEP, [pA, #512] prfm PLDL1KEEP, [pA, #512]
fmla v28.2d, v0.2d, v9.2d[1] fmla v28.2d, v0.2d, v9.d[1]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
.endm .endm
.macro KERNEL4x4_M2 .macro KERNEL4x4_M2
fmla v16.2d, v4.2d, v12.2d[0] fmla v16.2d, v4.2d, v12.d[0]
fmla v29.2d, v5.2d, v13.2d[1] fmla v29.2d, v5.2d, v13.d[1]
ld1 {v8.2d, v9.2d}, [pB] // For next round ld1 {v8.2d, v9.2d}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
fmla v20.2d, v4.2d, v12.2d[1] fmla v20.2d, v4.2d, v12.d[1]
fmla v25.2d, v5.2d, v13.2d[0] fmla v25.2d, v5.2d, v13.d[0]
ld1 {v0.2d, v1.2d}, [pA] // For next round ld1 {v0.2d, v1.2d}, [pA] // For next round
add pA, pA, #32 add pA, pA, #32
fmla v24.2d, v4.2d, v13.2d[0] fmla v24.2d, v4.2d, v13.d[0]
fmla v21.2d, v5.2d, v12.2d[1] fmla v21.2d, v5.2d, v12.d[1]
prfm PLDL1KEEP, [pB, #512] prfm PLDL1KEEP, [pB, #512]
fmla v28.2d, v4.2d, v13.2d[1] fmla v28.2d, v4.2d, v13.d[1]
fmla v17.2d, v5.2d, v12.2d[0] fmla v17.2d, v5.2d, v12.d[0]
.endm .endm
.macro KERNEL4x4_E .macro KERNEL4x4_E
fmla v16.2d, v4.2d, v12.2d[0] fmla v16.2d, v4.2d, v12.d[0]
fmla v29.2d, v5.2d, v13.2d[1] fmla v29.2d, v5.2d, v13.d[1]
fmla v20.2d, v4.2d, v12.2d[1] fmla v20.2d, v4.2d, v12.d[1]
fmla v25.2d, v5.2d, v13.2d[0] fmla v25.2d, v5.2d, v13.d[0]
fmla v24.2d, v4.2d, v13.2d[0] fmla v24.2d, v4.2d, v13.d[0]
fmla v21.2d, v5.2d, v12.2d[1] fmla v21.2d, v5.2d, v12.d[1]
fmla v28.2d, v4.2d, v13.2d[1] fmla v28.2d, v4.2d, v13.d[1]
fmla v17.2d, v5.2d, v12.2d[0] fmla v17.2d, v5.2d, v12.d[0]
.endm .endm
.macro KERNEL4x4_SUB .macro KERNEL4x4_SUB
@ -581,17 +581,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d, v1.2d}, [pA] ld1 {v0.2d, v1.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v29.2d, v1.2d, v9.2d[1] fmla v29.2d, v1.2d, v9.d[1]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
fmla v25.2d, v1.2d, v9.2d[0] fmla v25.2d, v1.2d, v9.d[0]
fmla v24.2d, v0.2d, v9.2d[0] fmla v24.2d, v0.2d, v9.d[0]
fmla v21.2d, v1.2d, v8.2d[1] fmla v21.2d, v1.2d, v8.d[1]
fmla v28.2d, v0.2d, v9.2d[1] fmla v28.2d, v0.2d, v9.d[1]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
.endm .endm
.macro SAVE4x4 .macro SAVE4x4
@ -635,10 +635,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d}, [pA] ld1 {v0.2d}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
fmla v24.2d, v0.2d, v9.2d[0] fmla v24.2d, v0.2d, v9.d[0]
fmla v28.2d, v0.2d, v9.2d[1] fmla v28.2d, v0.2d, v9.d[1]
.endm .endm
.macro SAVE2x4 .macro SAVE2x4
@ -713,10 +713,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d, v1.2d}, [pA] ld1 {v0.2d, v1.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
fmla v21.2d, v1.2d, v8.2d[1] fmla v21.2d, v1.2d, v8.d[1]
.endm .endm
.macro SAVE4x2 .macro SAVE4x2
@ -747,8 +747,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d}, [pA] ld1 {v0.2d}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
.endm .endm
.macro SAVE2x2 .macro SAVE2x2
@ -776,7 +776,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ldr d0 , [pA] ldr d0 , [pA]
add pA, pA, #8 add pA, pA, #8
fmla v16.2d, v8.2d, v0.2d[0] fmla v16.2d, v8.2d, v0.d[0]
.endm .endm
.macro SAVE1x2 .macro SAVE1x2
@ -803,8 +803,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d, v1.2d}, [pA] ld1 {v0.2d, v1.2d}, [pA]
add pA , pA, #32 add pA , pA, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
.endm .endm
.macro SAVE4x1 .macro SAVE4x1
@ -831,7 +831,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d}, [pA] ld1 {v0.2d}, [pA]
add pA , pA, #16 add pA , pA, #16
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
.endm .endm
.macro SAVE2x1 .macro SAVE2x1

228
kernel/arm64/dtrmm_kernel_8x4.S Executable file → Normal file
View File

@ -157,25 +157,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v2.2d, v3.2d}, [pA] ld1 {v2.2d, v3.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
fmul v16.2d, v0.2d, v8.2d[0] fmul v16.2d, v0.2d, v8.d[0]
fmul v17.2d, v1.2d, v8.2d[0] fmul v17.2d, v1.2d, v8.d[0]
fmul v18.2d, v2.2d, v8.2d[0] fmul v18.2d, v2.2d, v8.d[0]
fmul v19.2d, v3.2d, v8.2d[0] fmul v19.2d, v3.2d, v8.d[0]
fmul v20.2d, v0.2d, v8.2d[1] fmul v20.2d, v0.2d, v8.d[1]
fmul v21.2d, v1.2d, v8.2d[1] fmul v21.2d, v1.2d, v8.d[1]
fmul v22.2d, v2.2d, v8.2d[1] fmul v22.2d, v2.2d, v8.d[1]
fmul v23.2d, v3.2d, v8.2d[1] fmul v23.2d, v3.2d, v8.d[1]
fmul v24.2d, v0.2d, v9.2d[0] fmul v24.2d, v0.2d, v9.d[0]
fmul v25.2d, v1.2d, v9.2d[0] fmul v25.2d, v1.2d, v9.d[0]
fmul v26.2d, v2.2d, v9.2d[0] fmul v26.2d, v2.2d, v9.d[0]
fmul v27.2d, v3.2d, v9.2d[0] fmul v27.2d, v3.2d, v9.d[0]
fmul v28.2d, v0.2d, v9.2d[1] fmul v28.2d, v0.2d, v9.d[1]
fmul v29.2d, v1.2d, v9.2d[1] fmul v29.2d, v1.2d, v9.d[1]
fmul v30.2d, v2.2d, v9.2d[1] fmul v30.2d, v2.2d, v9.d[1]
fmul v31.2d, v3.2d, v9.2d[1] fmul v31.2d, v3.2d, v9.d[1]
ld1 {v4.2d, v5.2d}, [pA] ld1 {v4.2d, v5.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
@ -186,25 +186,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x4_M1 .macro KERNEL8x4_M1
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
fmla v18.2d, v2.2d, v8.2d[0] fmla v18.2d, v2.2d, v8.d[0]
fmla v19.2d, v3.2d, v8.2d[0] fmla v19.2d, v3.2d, v8.d[0]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
fmla v21.2d, v1.2d, v8.2d[1] fmla v21.2d, v1.2d, v8.d[1]
fmla v22.2d, v2.2d, v8.2d[1] fmla v22.2d, v2.2d, v8.d[1]
fmla v23.2d, v3.2d, v8.2d[1] fmla v23.2d, v3.2d, v8.d[1]
fmla v24.2d, v0.2d, v9.2d[0] fmla v24.2d, v0.2d, v9.d[0]
fmla v25.2d, v1.2d, v9.2d[0] fmla v25.2d, v1.2d, v9.d[0]
fmla v26.2d, v2.2d, v9.2d[0] fmla v26.2d, v2.2d, v9.d[0]
fmla v27.2d, v3.2d, v9.2d[0] fmla v27.2d, v3.2d, v9.d[0]
fmla v28.2d, v0.2d, v9.2d[1] fmla v28.2d, v0.2d, v9.d[1]
fmla v29.2d, v1.2d, v9.2d[1] fmla v29.2d, v1.2d, v9.d[1]
fmla v30.2d, v2.2d, v9.2d[1] fmla v30.2d, v2.2d, v9.d[1]
fmla v31.2d, v3.2d, v9.2d[1] fmla v31.2d, v3.2d, v9.d[1]
ld1 {v4.2d, v5.2d}, [pA] ld1 {v4.2d, v5.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
@ -217,25 +217,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x4_M2 .macro KERNEL8x4_M2
fmla v16.2d, v4.2d, v12.2d[0] fmla v16.2d, v4.2d, v12.d[0]
fmla v17.2d, v5.2d, v12.2d[0] fmla v17.2d, v5.2d, v12.d[0]
fmla v18.2d, v6.2d, v12.2d[0] fmla v18.2d, v6.2d, v12.d[0]
fmla v19.2d, v7.2d, v12.2d[0] fmla v19.2d, v7.2d, v12.d[0]
fmla v20.2d, v4.2d, v12.2d[1] fmla v20.2d, v4.2d, v12.d[1]
fmla v21.2d, v5.2d, v12.2d[1] fmla v21.2d, v5.2d, v12.d[1]
fmla v22.2d, v6.2d, v12.2d[1] fmla v22.2d, v6.2d, v12.d[1]
fmla v23.2d, v7.2d, v12.2d[1] fmla v23.2d, v7.2d, v12.d[1]
fmla v24.2d, v4.2d, v13.2d[0] fmla v24.2d, v4.2d, v13.d[0]
fmla v25.2d, v5.2d, v13.2d[0] fmla v25.2d, v5.2d, v13.d[0]
fmla v26.2d, v6.2d, v13.2d[0] fmla v26.2d, v6.2d, v13.d[0]
fmla v27.2d, v7.2d, v13.2d[0] fmla v27.2d, v7.2d, v13.d[0]
fmla v28.2d, v4.2d, v13.2d[1] fmla v28.2d, v4.2d, v13.d[1]
fmla v29.2d, v5.2d, v13.2d[1] fmla v29.2d, v5.2d, v13.d[1]
fmla v30.2d, v6.2d, v13.2d[1] fmla v30.2d, v6.2d, v13.d[1]
fmla v31.2d, v7.2d, v13.2d[1] fmla v31.2d, v7.2d, v13.d[1]
ld1 {v0.2d, v1.2d}, [pA] ld1 {v0.2d, v1.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
@ -248,25 +248,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x4_E .macro KERNEL8x4_E
fmla v16.2d, v4.2d, v12.2d[0] fmla v16.2d, v4.2d, v12.d[0]
fmla v17.2d, v5.2d, v12.2d[0] fmla v17.2d, v5.2d, v12.d[0]
fmla v18.2d, v6.2d, v12.2d[0] fmla v18.2d, v6.2d, v12.d[0]
fmla v19.2d, v7.2d, v12.2d[0] fmla v19.2d, v7.2d, v12.d[0]
fmla v20.2d, v4.2d, v12.2d[1] fmla v20.2d, v4.2d, v12.d[1]
fmla v21.2d, v5.2d, v12.2d[1] fmla v21.2d, v5.2d, v12.d[1]
fmla v22.2d, v6.2d, v12.2d[1] fmla v22.2d, v6.2d, v12.d[1]
fmla v23.2d, v7.2d, v12.2d[1] fmla v23.2d, v7.2d, v12.d[1]
fmla v24.2d, v4.2d, v13.2d[0] fmla v24.2d, v4.2d, v13.d[0]
fmla v25.2d, v5.2d, v13.2d[0] fmla v25.2d, v5.2d, v13.d[0]
fmla v26.2d, v6.2d, v13.2d[0] fmla v26.2d, v6.2d, v13.d[0]
fmla v27.2d, v7.2d, v13.2d[0] fmla v27.2d, v7.2d, v13.d[0]
fmla v28.2d, v4.2d, v13.2d[1] fmla v28.2d, v4.2d, v13.d[1]
fmla v29.2d, v5.2d, v13.2d[1] fmla v29.2d, v5.2d, v13.d[1]
fmla v30.2d, v6.2d, v13.2d[1] fmla v30.2d, v6.2d, v13.d[1]
fmla v31.2d, v7.2d, v13.2d[1] fmla v31.2d, v7.2d, v13.d[1]
.endm .endm
.macro KERNEL8x4_SUB .macro KERNEL8x4_SUB
@ -277,25 +277,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v2.2d, v3.2d}, [pA] ld1 {v2.2d, v3.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
fmla v18.2d, v2.2d, v8.2d[0] fmla v18.2d, v2.2d, v8.d[0]
fmla v19.2d, v3.2d, v8.2d[0] fmla v19.2d, v3.2d, v8.d[0]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
fmla v21.2d, v1.2d, v8.2d[1] fmla v21.2d, v1.2d, v8.d[1]
fmla v22.2d, v2.2d, v8.2d[1] fmla v22.2d, v2.2d, v8.d[1]
fmla v23.2d, v3.2d, v8.2d[1] fmla v23.2d, v3.2d, v8.d[1]
fmla v24.2d, v0.2d, v9.2d[0] fmla v24.2d, v0.2d, v9.d[0]
fmla v25.2d, v1.2d, v9.2d[0] fmla v25.2d, v1.2d, v9.d[0]
fmla v26.2d, v2.2d, v9.2d[0] fmla v26.2d, v2.2d, v9.d[0]
fmla v27.2d, v3.2d, v9.2d[0] fmla v27.2d, v3.2d, v9.d[0]
fmla v28.2d, v0.2d, v9.2d[1] fmla v28.2d, v0.2d, v9.d[1]
fmla v29.2d, v1.2d, v9.2d[1] fmla v29.2d, v1.2d, v9.d[1]
fmla v30.2d, v2.2d, v9.2d[1] fmla v30.2d, v2.2d, v9.d[1]
fmla v31.2d, v3.2d, v9.2d[1] fmla v31.2d, v3.2d, v9.d[1]
.endm .endm
.macro SAVE8x4 .macro SAVE8x4
@ -351,17 +351,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d, v1.2d}, [pA] ld1 {v0.2d, v1.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v29.2d, v1.2d, v9.2d[1] fmla v29.2d, v1.2d, v9.d[1]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
fmla v25.2d, v1.2d, v9.2d[0] fmla v25.2d, v1.2d, v9.d[0]
fmla v24.2d, v0.2d, v9.2d[0] fmla v24.2d, v0.2d, v9.d[0]
fmla v21.2d, v1.2d, v8.2d[1] fmla v21.2d, v1.2d, v8.d[1]
fmla v28.2d, v0.2d, v9.2d[1] fmla v28.2d, v0.2d, v9.d[1]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
.endm .endm
.macro SAVE4x4 .macro SAVE4x4
@ -406,10 +406,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d}, [pA] ld1 {v0.2d}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
fmla v24.2d, v0.2d, v9.2d[0] fmla v24.2d, v0.2d, v9.d[0]
fmla v28.2d, v0.2d, v9.2d[1] fmla v28.2d, v0.2d, v9.d[1]
.endm .endm
.macro SAVE2x4 .macro SAVE2x4
@ -490,15 +490,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v2.2d, v3.2d}, [pA] ld1 {v2.2d, v3.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
fmla v18.2d, v2.2d, v8.2d[0] fmla v18.2d, v2.2d, v8.d[0]
fmla v19.2d, v3.2d, v8.2d[0] fmla v19.2d, v3.2d, v8.d[0]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
fmla v21.2d, v1.2d, v8.2d[1] fmla v21.2d, v1.2d, v8.d[1]
fmla v22.2d, v2.2d, v8.2d[1] fmla v22.2d, v2.2d, v8.d[1]
fmla v23.2d, v3.2d, v8.2d[1] fmla v23.2d, v3.2d, v8.d[1]
.endm .endm
.macro SAVE8x2 .macro SAVE8x2
@ -534,10 +534,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d, v1.2d}, [pA] ld1 {v0.2d, v1.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
fmla v21.2d, v1.2d, v8.2d[1] fmla v21.2d, v1.2d, v8.d[1]
.endm .endm
.macro SAVE4x2 .macro SAVE4x2
@ -568,8 +568,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d}, [pA] ld1 {v0.2d}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v20.2d, v0.2d, v8.2d[1] fmla v20.2d, v0.2d, v8.d[1]
.endm .endm
.macro SAVE2x2 .macro SAVE2x2
@ -597,7 +597,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ldr d0 , [pA] ldr d0 , [pA]
add pA, pA, #8 add pA, pA, #8
fmla v16.2d, v8.2d, v0.2d[0] fmla v16.2d, v8.2d, v0.d[0]
.endm .endm
.macro SAVE1x2 .macro SAVE1x2
@ -629,10 +629,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v2.2d, v3.2d}, [pA] ld1 {v2.2d, v3.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
fmla v18.2d, v2.2d, v8.2d[0] fmla v18.2d, v2.2d, v8.d[0]
fmla v19.2d, v3.2d, v8.2d[0] fmla v19.2d, v3.2d, v8.d[0]
.endm .endm
.macro SAVE8x1 .macro SAVE8x1
@ -660,8 +660,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d, v1.2d}, [pA] ld1 {v0.2d, v1.2d}, [pA]
add pA , pA, #32 add pA , pA, #32
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
fmla v17.2d, v1.2d, v8.2d[0] fmla v17.2d, v1.2d, v8.d[0]
.endm .endm
.macro SAVE4x1 .macro SAVE4x1
@ -686,7 +686,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2d}, [pA] ld1 {v0.2d}, [pA]
add pA , pA, #16 add pA , pA, #16
fmla v16.2d, v0.2d, v8.2d[0] fmla v16.2d, v0.2d, v8.d[0]
.endm .endm
.macro SAVE2x1 .macro SAVE2x1

384
kernel/arm64/sgemm_kernel_16x4.S
View File

@ -158,25 +158,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v3.4s}, [pA] ld1 {v3.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmul v16.4s, v0.4s, v8.2s[0] fmul v16.4s, v0.4s, v8.s[0]
fmul v17.4s, v1.4s, v8.2s[0] fmul v17.4s, v1.4s, v8.s[0]
fmul v18.4s, v2.4s, v8.2s[0] fmul v18.4s, v2.4s, v8.s[0]
fmul v19.4s, v3.4s, v8.2s[0] fmul v19.4s, v3.4s, v8.s[0]
fmul v20.4s, v0.4s, v8.2s[1] fmul v20.4s, v0.4s, v8.s[1]
fmul v21.4s, v1.4s, v8.2s[1] fmul v21.4s, v1.4s, v8.s[1]
fmul v22.4s, v2.4s, v8.2s[1] fmul v22.4s, v2.4s, v8.s[1]
fmul v23.4s, v3.4s, v8.2s[1] fmul v23.4s, v3.4s, v8.s[1]
fmul v24.4s, v0.4s, v9.2s[0] fmul v24.4s, v0.4s, v9.s[0]
fmul v25.4s, v1.4s, v9.2s[0] fmul v25.4s, v1.4s, v9.s[0]
fmul v26.4s, v2.4s, v9.2s[0] fmul v26.4s, v2.4s, v9.s[0]
fmul v27.4s, v3.4s, v9.2s[0] fmul v27.4s, v3.4s, v9.s[0]
fmul v28.4s, v0.4s, v9.2s[1] fmul v28.4s, v0.4s, v9.s[1]
fmul v29.4s, v1.4s, v9.2s[1] fmul v29.4s, v1.4s, v9.s[1]
fmul v30.4s, v2.4s, v9.2s[1] fmul v30.4s, v2.4s, v9.s[1]
fmul v31.4s, v3.4s, v9.2s[1] fmul v31.4s, v3.4s, v9.s[1]
ld1 {v12.2s, v13.2s}, [pB] ld1 {v12.2s, v13.2s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -191,25 +191,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL16x4_M1 .macro KERNEL16x4_M1
fmla v16.4s, v0.4s, v8.2s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v17.4s, v1.4s, v8.2s[0] fmla v17.4s, v1.4s, v8.s[0]
fmla v18.4s, v2.4s, v8.2s[0] fmla v18.4s, v2.4s, v8.s[0]
fmla v19.4s, v3.4s, v8.2s[0] fmla v19.4s, v3.4s, v8.s[0]
fmla v20.4s, v0.4s, v8.2s[1] fmla v20.4s, v0.4s, v8.s[1]
fmla v21.4s, v1.4s, v8.2s[1] fmla v21.4s, v1.4s, v8.s[1]
fmla v22.4s, v2.4s, v8.2s[1] fmla v22.4s, v2.4s, v8.s[1]
fmla v23.4s, v3.4s, v8.2s[1] fmla v23.4s, v3.4s, v8.s[1]
fmla v24.4s, v0.4s, v9.2s[0] fmla v24.4s, v0.4s, v9.s[0]
fmla v25.4s, v1.4s, v9.2s[0] fmla v25.4s, v1.4s, v9.s[0]
fmla v26.4s, v2.4s, v9.2s[0] fmla v26.4s, v2.4s, v9.s[0]
fmla v27.4s, v3.4s, v9.2s[0] fmla v27.4s, v3.4s, v9.s[0]
fmla v28.4s, v0.4s, v9.2s[1] fmla v28.4s, v0.4s, v9.s[1]
fmla v29.4s, v1.4s, v9.2s[1] fmla v29.4s, v1.4s, v9.s[1]
fmla v30.4s, v2.4s, v9.2s[1] fmla v30.4s, v2.4s, v9.s[1]
fmla v31.4s, v3.4s, v9.2s[1] fmla v31.4s, v3.4s, v9.s[1]
ld1 {v12.2s, v13.2s}, [pB] ld1 {v12.2s, v13.2s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -224,25 +224,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL16x4_M2 .macro KERNEL16x4_M2
fmla v16.4s, v4.4s, v12.2s[0] fmla v16.4s, v4.4s, v12.s[0]
fmla v17.4s, v5.4s, v12.2s[0] fmla v17.4s, v5.4s, v12.s[0]
fmla v18.4s, v6.4s, v12.2s[0] fmla v18.4s, v6.4s, v12.s[0]
fmla v19.4s, v7.4s, v12.2s[0] fmla v19.4s, v7.4s, v12.s[0]
fmla v20.4s, v4.4s, v12.2s[1] fmla v20.4s, v4.4s, v12.s[1]
fmla v21.4s, v5.4s, v12.2s[1] fmla v21.4s, v5.4s, v12.s[1]
fmla v22.4s, v6.4s, v12.2s[1] fmla v22.4s, v6.4s, v12.s[1]
fmla v23.4s, v7.4s, v12.2s[1] fmla v23.4s, v7.4s, v12.s[1]
fmla v24.4s, v4.4s, v13.2s[0] fmla v24.4s, v4.4s, v13.s[0]
fmla v25.4s, v5.4s, v13.2s[0] fmla v25.4s, v5.4s, v13.s[0]
fmla v26.4s, v6.4s, v13.2s[0] fmla v26.4s, v6.4s, v13.s[0]
fmla v27.4s, v7.4s, v13.2s[0] fmla v27.4s, v7.4s, v13.s[0]
fmla v28.4s, v4.4s, v13.2s[1] fmla v28.4s, v4.4s, v13.s[1]
fmla v29.4s, v5.4s, v13.2s[1] fmla v29.4s, v5.4s, v13.s[1]
fmla v30.4s, v6.4s, v13.2s[1] fmla v30.4s, v6.4s, v13.s[1]
fmla v31.4s, v7.4s, v13.2s[1] fmla v31.4s, v7.4s, v13.s[1]
ld1 {v8.2s, v9.2s}, [pB] ld1 {v8.2s, v9.2s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -257,25 +257,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL16x4_E .macro KERNEL16x4_E
fmla v16.4s, v4.4s, v12.2s[0] fmla v16.4s, v4.4s, v12.s[0]
fmla v17.4s, v5.4s, v12.2s[0] fmla v17.4s, v5.4s, v12.s[0]
fmla v18.4s, v6.4s, v12.2s[0] fmla v18.4s, v6.4s, v12.s[0]
fmla v19.4s, v7.4s, v12.2s[0] fmla v19.4s, v7.4s, v12.s[0]
fmla v20.4s, v4.4s, v12.2s[1] fmla v20.4s, v4.4s, v12.s[1]
fmla v21.4s, v5.4s, v12.2s[1] fmla v21.4s, v5.4s, v12.s[1]
fmla v22.4s, v6.4s, v12.2s[1] fmla v22.4s, v6.4s, v12.s[1]
fmla v23.4s, v7.4s, v12.2s[1] fmla v23.4s, v7.4s, v12.s[1]
fmla v24.4s, v4.4s, v13.2s[0] fmla v24.4s, v4.4s, v13.s[0]
fmla v25.4s, v5.4s, v13.2s[0] fmla v25.4s, v5.4s, v13.s[0]
fmla v26.4s, v6.4s, v13.2s[0] fmla v26.4s, v6.4s, v13.s[0]
fmla v27.4s, v7.4s, v13.2s[0] fmla v27.4s, v7.4s, v13.s[0]
fmla v28.4s, v4.4s, v13.2s[1] fmla v28.4s, v4.4s, v13.s[1]
fmla v29.4s, v5.4s, v13.2s[1] fmla v29.4s, v5.4s, v13.s[1]
fmla v30.4s, v6.4s, v13.2s[1] fmla v30.4s, v6.4s, v13.s[1]
fmla v31.4s, v7.4s, v13.2s[1] fmla v31.4s, v7.4s, v13.s[1]
.endm .endm
.macro KERNEL16x4_SUB .macro KERNEL16x4_SUB
@ -290,25 +290,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v3.4s}, [pA] ld1 {v3.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.4s, v0.4s, v8.2s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v17.4s, v1.4s, v8.2s[0] fmla v17.4s, v1.4s, v8.s[0]
fmla v18.4s, v2.4s, v8.2s[0] fmla v18.4s, v2.4s, v8.s[0]
fmla v19.4s, v3.4s, v8.2s[0] fmla v19.4s, v3.4s, v8.s[0]
fmla v20.4s, v0.4s, v8.2s[1] fmla v20.4s, v0.4s, v8.s[1]
fmla v21.4s, v1.4s, v8.2s[1] fmla v21.4s, v1.4s, v8.s[1]
fmla v22.4s, v2.4s, v8.2s[1] fmla v22.4s, v2.4s, v8.s[1]
fmla v23.4s, v3.4s, v8.2s[1] fmla v23.4s, v3.4s, v8.s[1]
fmla v24.4s, v0.4s, v9.2s[0] fmla v24.4s, v0.4s, v9.s[0]
fmla v25.4s, v1.4s, v9.2s[0] fmla v25.4s, v1.4s, v9.s[0]
fmla v26.4s, v2.4s, v9.2s[0] fmla v26.4s, v2.4s, v9.s[0]
fmla v27.4s, v3.4s, v9.2s[0] fmla v27.4s, v3.4s, v9.s[0]
fmla v28.4s, v0.4s, v9.2s[1] fmla v28.4s, v0.4s, v9.s[1]
fmla v29.4s, v1.4s, v9.2s[1] fmla v29.4s, v1.4s, v9.s[1]
fmla v30.4s, v2.4s, v9.2s[1] fmla v30.4s, v2.4s, v9.s[1]
fmla v31.4s, v3.4s, v9.2s[1] fmla v31.4s, v3.4s, v9.s[1]
.endm .endm
.macro SAVE16x4 .macro SAVE16x4
@ -370,14 +370,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v1.4s}, [pA] ld1 {v1.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmul v16.4s, v0.4s, v8.2s[0] fmul v16.4s, v0.4s, v8.s[0]
fmul v17.4s, v1.4s, v8.2s[0] fmul v17.4s, v1.4s, v8.s[0]
fmul v20.4s, v0.4s, v8.2s[1] fmul v20.4s, v0.4s, v8.s[1]
fmul v21.4s, v1.4s, v8.2s[1] fmul v21.4s, v1.4s, v8.s[1]
fmul v24.4s, v0.4s, v9.2s[0] fmul v24.4s, v0.4s, v9.s[0]
fmul v25.4s, v1.4s, v9.2s[0] fmul v25.4s, v1.4s, v9.s[0]
fmul v28.4s, v0.4s, v9.2s[1] fmul v28.4s, v0.4s, v9.s[1]
fmul v29.4s, v1.4s, v9.2s[1] fmul v29.4s, v1.4s, v9.s[1]
ld1 {v12.2s, v13.2s}, [pB] ld1 {v12.2s, v13.2s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -388,14 +388,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x4_M1 .macro KERNEL8x4_M1
fmla v16.4s, v0.4s, v8.2s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v17.4s, v1.4s, v8.2s[0] fmla v17.4s, v1.4s, v8.s[0]
fmla v20.4s, v0.4s, v8.2s[1] fmla v20.4s, v0.4s, v8.s[1]
fmla v21.4s, v1.4s, v8.2s[1] fmla v21.4s, v1.4s, v8.s[1]
fmla v24.4s, v0.4s, v9.2s[0] fmla v24.4s, v0.4s, v9.s[0]
fmla v25.4s, v1.4s, v9.2s[0] fmla v25.4s, v1.4s, v9.s[0]
fmla v28.4s, v0.4s, v9.2s[1] fmla v28.4s, v0.4s, v9.s[1]
fmla v29.4s, v1.4s, v9.2s[1] fmla v29.4s, v1.4s, v9.s[1]
ld1 {v12.2s, v13.2s}, [pB] ld1 {v12.2s, v13.2s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -406,14 +406,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x4_M2 .macro KERNEL8x4_M2
fmla v16.4s, v4.4s, v12.2s[0] fmla v16.4s, v4.4s, v12.s[0]
fmla v17.4s, v5.4s, v12.2s[0] fmla v17.4s, v5.4s, v12.s[0]
fmla v20.4s, v4.4s, v12.2s[1] fmla v20.4s, v4.4s, v12.s[1]
fmla v21.4s, v5.4s, v12.2s[1] fmla v21.4s, v5.4s, v12.s[1]
fmla v24.4s, v4.4s, v13.2s[0] fmla v24.4s, v4.4s, v13.s[0]
fmla v25.4s, v5.4s, v13.2s[0] fmla v25.4s, v5.4s, v13.s[0]
fmla v28.4s, v4.4s, v13.2s[1] fmla v28.4s, v4.4s, v13.s[1]
fmla v29.4s, v5.4s, v13.2s[1] fmla v29.4s, v5.4s, v13.s[1]
ld1 {v8.2s, v9.2s}, [pB] ld1 {v8.2s, v9.2s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -424,14 +424,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x4_E .macro KERNEL8x4_E
fmla v16.4s, v4.4s, v12.2s[0] fmla v16.4s, v4.4s, v12.s[0]
fmla v17.4s, v5.4s, v12.2s[0] fmla v17.4s, v5.4s, v12.s[0]
fmla v20.4s, v4.4s, v12.2s[1] fmla v20.4s, v4.4s, v12.s[1]
fmla v21.4s, v5.4s, v12.2s[1] fmla v21.4s, v5.4s, v12.s[1]
fmla v24.4s, v4.4s, v13.2s[0] fmla v24.4s, v4.4s, v13.s[0]
fmla v25.4s, v5.4s, v13.2s[0] fmla v25.4s, v5.4s, v13.s[0]
fmla v28.4s, v4.4s, v13.2s[1] fmla v28.4s, v4.4s, v13.s[1]
fmla v29.4s, v5.4s, v13.2s[1] fmla v29.4s, v5.4s, v13.s[1]
.endm .endm
.macro KERNEL8x4_SUB .macro KERNEL8x4_SUB
@ -442,14 +442,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v1.4s}, [pA] ld1 {v1.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.4s, v0.4s, v8.2s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v17.4s, v1.4s, v8.2s[0] fmla v17.4s, v1.4s, v8.s[0]
fmla v20.4s, v0.4s, v8.2s[1] fmla v20.4s, v0.4s, v8.s[1]
fmla v21.4s, v1.4s, v8.2s[1] fmla v21.4s, v1.4s, v8.s[1]
fmla v24.4s, v0.4s, v9.2s[0] fmla v24.4s, v0.4s, v9.s[0]
fmla v25.4s, v1.4s, v9.2s[0] fmla v25.4s, v1.4s, v9.s[0]
fmla v28.4s, v0.4s, v9.2s[1] fmla v28.4s, v0.4s, v9.s[1]
fmla v29.4s, v1.4s, v9.2s[1] fmla v29.4s, v1.4s, v9.s[1]
.endm .endm
.macro SAVE8x4 .macro SAVE8x4
@ -501,17 +501,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s, v1.2s}, [pA] ld1 {v0.2s, v1.2s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmul v16.2s, v0.2s, v8.2s[0] fmul v16.2s, v0.2s, v8.s[0]
fmul v29.2s, v1.2s, v9.2s[1] fmul v29.2s, v1.2s, v9.s[1]
fmul v20.2s, v0.2s, v8.2s[1] fmul v20.2s, v0.2s, v8.s[1]
fmul v25.2s, v1.2s, v9.2s[0] fmul v25.2s, v1.2s, v9.s[0]
fmul v24.2s, v0.2s, v9.2s[0] fmul v24.2s, v0.2s, v9.s[0]
fmul v21.2s, v1.2s, v8.2s[1] fmul v21.2s, v1.2s, v8.s[1]
fmul v28.2s, v0.2s, v9.2s[1] fmul v28.2s, v0.2s, v9.s[1]
fmul v17.2s, v1.2s, v8.2s[0] fmul v17.2s, v1.2s, v8.s[0]
ld1 {v12.2s, v13.2s}, [pB] ld1 {v12.2s, v13.2s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -520,61 +520,61 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL4x4_M1 .macro KERNEL4x4_M1
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v29.2s, v1.2s, v9.2s[1] fmla v29.2s, v1.2s, v9.s[1]
ld1 {v12.2s, v13.2s}, [pB] // For next round ld1 {v12.2s, v13.2s}, [pB] // For next round
add pB, pB, #16 add pB, pB, #16
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
fmla v25.2s, v1.2s, v9.2s[0] fmla v25.2s, v1.2s, v9.s[0]
ld1 {v4.2s, v5.2s}, [pA] // For next round ld1 {v4.2s, v5.2s}, [pA] // For next round
add pA, pA, #16 add pA, pA, #16
fmla v24.2s, v0.2s, v9.2s[0] fmla v24.2s, v0.2s, v9.s[0]
fmla v21.2s, v1.2s, v8.2s[1] fmla v21.2s, v1.2s, v8.s[1]
prfm PLDL1KEEP, [pB, #512] prfm PLDL1KEEP, [pB, #512]
fmla v28.2s, v0.2s, v9.2s[1] fmla v28.2s, v0.2s, v9.s[1]
fmla v17.2s, v1.2s, v8.2s[0] fmla v17.2s, v1.2s, v8.s[0]
.endm .endm
.macro KERNEL4x4_M2 .macro KERNEL4x4_M2
fmla v16.2s, v4.2s, v12.2s[0] fmla v16.2s, v4.2s, v12.s[0]
fmla v29.2s, v5.2s, v13.2s[1] fmla v29.2s, v5.2s, v13.s[1]
ld1 {v8.2s, v9.2s}, [pB] // For next round ld1 {v8.2s, v9.2s}, [pB] // For next round
add pB, pB, #16 add pB, pB, #16
fmla v20.2s, v4.2s, v12.2s[1] fmla v20.2s, v4.2s, v12.s[1]
fmla v25.2s, v5.2s, v13.2s[0] fmla v25.2s, v5.2s, v13.s[0]
ld1 {v0.2s, v1.2s}, [pA] // For next round ld1 {v0.2s, v1.2s}, [pA] // For next round
add pA, pA, #16 add pA, pA, #16
fmla v24.2s, v4.2s, v13.2s[0] fmla v24.2s, v4.2s, v13.s[0]
fmla v21.2s, v5.2s, v12.2s[1] fmla v21.2s, v5.2s, v12.s[1]
prfm PLDL1KEEP, [pA, #512] prfm PLDL1KEEP, [pA, #512]
fmla v28.2s, v4.2s, v13.2s[1] fmla v28.2s, v4.2s, v13.s[1]
fmla v17.2s, v5.2s, v12.2s[0] fmla v17.2s, v5.2s, v12.s[0]
.endm .endm
.macro KERNEL4x4_E .macro KERNEL4x4_E
fmla v16.2s, v4.2s, v12.2s[0] fmla v16.2s, v4.2s, v12.s[0]
fmla v29.2s, v5.2s, v13.2s[1] fmla v29.2s, v5.2s, v13.s[1]
fmla v20.2s, v4.2s, v12.2s[1] fmla v20.2s, v4.2s, v12.s[1]
fmla v25.2s, v5.2s, v13.2s[0] fmla v25.2s, v5.2s, v13.s[0]
fmla v24.2s, v4.2s, v13.2s[0] fmla v24.2s, v4.2s, v13.s[0]
fmla v21.2s, v5.2s, v12.2s[1] fmla v21.2s, v5.2s, v12.s[1]
fmla v28.2s, v4.2s, v13.2s[1] fmla v28.2s, v4.2s, v13.s[1]
fmla v17.2s, v5.2s, v12.2s[0] fmla v17.2s, v5.2s, v12.s[0]
.endm .endm
.macro KERNEL4x4_SUB .macro KERNEL4x4_SUB
@ -583,17 +583,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s, v1.2s}, [pA] ld1 {v0.2s, v1.2s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v29.2s, v1.2s, v9.2s[1] fmla v29.2s, v1.2s, v9.s[1]
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
fmla v25.2s, v1.2s, v9.2s[0] fmla v25.2s, v1.2s, v9.s[0]
fmla v24.2s, v0.2s, v9.2s[0] fmla v24.2s, v0.2s, v9.s[0]
fmla v21.2s, v1.2s, v8.2s[1] fmla v21.2s, v1.2s, v8.s[1]
fmla v28.2s, v0.2s, v9.2s[1] fmla v28.2s, v0.2s, v9.s[1]
fmla v17.2s, v1.2s, v8.2s[0] fmla v17.2s, v1.2s, v8.s[0]
.endm .endm
.macro SAVE4x4 .macro SAVE4x4
@ -638,10 +638,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s}, [pA] ld1 {v0.2s}, [pA]
add pA, pA, #8 add pA, pA, #8
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
fmla v24.2s, v0.2s, v9.2s[0] fmla v24.2s, v0.2s, v9.s[0]
fmla v28.2s, v0.2s, v9.2s[1] fmla v28.2s, v0.2s, v9.s[1]
.endm .endm
.macro SAVE2x4 .macro SAVE2x4
@ -729,15 +729,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v3.4s}, [pA] ld1 {v3.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.4s, v0.4s, v8.2s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v17.4s, v1.4s, v8.2s[0] fmla v17.4s, v1.4s, v8.s[0]
fmla v18.4s, v2.4s, v8.2s[0] fmla v18.4s, v2.4s, v8.s[0]
fmla v19.4s, v3.4s, v8.2s[0] fmla v19.4s, v3.4s, v8.s[0]
fmla v20.4s, v0.4s, v8.2s[1] fmla v20.4s, v0.4s, v8.s[1]
fmla v21.4s, v1.4s, v8.2s[1] fmla v21.4s, v1.4s, v8.s[1]
fmla v22.4s, v2.4s, v8.2s[1] fmla v22.4s, v2.4s, v8.s[1]
fmla v23.4s, v3.4s, v8.2s[1] fmla v23.4s, v3.4s, v8.s[1]
.endm .endm
.macro SAVE16x2 .macro SAVE16x2
@ -777,11 +777,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v1.4s}, [pA] ld1 {v1.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.4s, v0.4s, v8.2s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v17.4s, v1.4s, v8.2s[0] fmla v17.4s, v1.4s, v8.s[0]
fmla v20.4s, v0.4s, v8.2s[1] fmla v20.4s, v0.4s, v8.s[1]
fmla v21.4s, v1.4s, v8.2s[1] fmla v21.4s, v1.4s, v8.s[1]
.endm .endm
.macro SAVE8x2 .macro SAVE8x2
@ -817,10 +817,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s, v1.2s}, [pA] ld1 {v0.2s, v1.2s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v17.2s, v1.2s, v8.2s[0] fmla v17.2s, v1.2s, v8.s[0]
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
fmla v21.2s, v1.2s, v8.2s[1] fmla v21.2s, v1.2s, v8.s[1]
.endm .endm
.macro SAVE4x2 .macro SAVE4x2
@ -852,8 +852,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s}, [pA] ld1 {v0.2s}, [pA]
add pA, pA, #8 add pA, pA, #8
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
.endm .endm
.macro SAVE2x2 .macro SAVE2x2
@ -882,7 +882,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ldr s0 , [pA] ldr s0 , [pA]
add pA, pA, #4 add pA, pA, #4
fmla v16.2s, v8.2s, v0.2s[0] fmla v16.2s, v8.2s, v0.s[0]
.endm .endm
.macro SAVE1x2 .macro SAVE1x2
@ -918,10 +918,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v3.4s}, [pA] ld1 {v3.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.4s, v0.4s, v8.2s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v17.4s, v1.4s, v8.2s[0] fmla v17.4s, v1.4s, v8.s[0]
fmla v18.4s, v2.4s, v8.2s[0] fmla v18.4s, v2.4s, v8.s[0]
fmla v19.4s, v3.4s, v8.2s[0] fmla v19.4s, v3.4s, v8.s[0]
.endm .endm
.macro SAVE16x1 .macro SAVE16x1
@ -951,8 +951,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v1.4s}, [pA] ld1 {v1.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.4s, v0.4s, v8.2s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v17.4s, v1.4s, v8.2s[0] fmla v17.4s, v1.4s, v8.s[0]
.endm .endm
.macro SAVE8x1 .macro SAVE8x1
@ -978,8 +978,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s, v1.2s}, [pA] ld1 {v0.2s, v1.2s}, [pA]
add pA , pA, #16 add pA , pA, #16
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v17.2s, v1.2s, v8.2s[0] fmla v17.2s, v1.2s, v8.s[0]
.endm .endm
.macro SAVE4x1 .macro SAVE4x1
@ -1004,7 +1004,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s}, [pA] ld1 {v0.2s}, [pA]
add pA , pA, #8 add pA , pA, #8
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
.endm .endm
.macro SAVE2x1 .macro SAVE2x1

236
kernel/arm64/sgemm_kernel_4x4.S
View File

@ -192,164 +192,164 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.4s}, [pA_0] ld1 {v0.4s}, [pA_0]
add pA_0, pA_0, #16 add pA_0, pA_0, #16
fmul v16.4s, v0.4s, v8.4s[0] fmul v16.4s, v0.4s, v8.s[0]
fmul v20.4s, v0.4s, v8.4s[1] fmul v20.4s, v0.4s, v8.s[1]
ld1 {v2.4s}, [pA_1] ld1 {v2.4s}, [pA_1]
add pA_1, pA_1, #16 add pA_1, pA_1, #16
fmul v24.4s, v0.4s, v8.4s[2] fmul v24.4s, v0.4s, v8.s[2]
fmul v28.4s, v0.4s, v8.4s[3] fmul v28.4s, v0.4s, v8.s[3]
ld1 {v4.4s}, [pA_2] ld1 {v4.4s}, [pA_2]
add pA_2, pA_2, #16 add pA_2, pA_2, #16
fmul v17.4s, v2.4s, v8.4s[0] fmul v17.4s, v2.4s, v8.s[0]
fmul v21.4s, v2.4s, v8.4s[1] fmul v21.4s, v2.4s, v8.s[1]
ld1 {v6.4s}, [pA_3] ld1 {v6.4s}, [pA_3]
add pA_3, pA_3, #16 add pA_3, pA_3, #16
fmul v25.4s, v2.4s, v8.4s[2] fmul v25.4s, v2.4s, v8.s[2]
fmul v29.4s, v2.4s, v8.4s[3] fmul v29.4s, v2.4s, v8.s[3]
ld1 {v12.4s}, [pB] // for next round ld1 {v12.4s}, [pB] // for next round
add pB, pB, #16 add pB, pB, #16
fmul v18.4s, v4.4s, v8.4s[0] fmul v18.4s, v4.4s, v8.s[0]
fmul v19.4s, v6.4s, v8.4s[0] fmul v19.4s, v6.4s, v8.s[0]
ld1 {v1.4s}, [pA_0] // for next round ld1 {v1.4s}, [pA_0] // for next round
add pA_0, pA_0, #16 add pA_0, pA_0, #16
fmul v22.4s, v4.4s, v8.4s[1] fmul v22.4s, v4.4s, v8.s[1]
fmul v23.4s, v6.4s, v8.4s[1] fmul v23.4s, v6.4s, v8.s[1]
ld1 {v3.4s}, [pA_1] // for next round ld1 {v3.4s}, [pA_1] // for next round
add pA_1, pA_1, #16 add pA_1, pA_1, #16
fmul v26.4s, v4.4s, v8.4s[2] fmul v26.4s, v4.4s, v8.s[2]
fmul v27.4s, v6.4s, v8.4s[2] fmul v27.4s, v6.4s, v8.s[2]
ld1 {v5.4s}, [pA_2] // for next round ld1 {v5.4s}, [pA_2] // for next round
add pA_2, pA_2, #16 add pA_2, pA_2, #16
fmul v30.4s, v4.4s, v8.4s[3] fmul v30.4s, v4.4s, v8.s[3]
fmul v31.4s, v6.4s, v8.4s[3] fmul v31.4s, v6.4s, v8.s[3]
ld1 {v7.4s}, [pA_3] // for next round ld1 {v7.4s}, [pA_3] // for next round
add pA_3, pA_3, #16 add pA_3, pA_3, #16
.endm .endm
.macro KERNEL16x4_M2 .macro KERNEL16x4_M2
fmla v16.4s, v1.4s, v12.4s[0] fmla v16.4s, v1.4s, v12.s[0]
fmla v17.4s, v3.4s, v12.4s[0] fmla v17.4s, v3.4s, v12.s[0]
ld1 {v8.4s}, [pB] // for next round ld1 {v8.4s}, [pB] // for next round
add pB, pB, #16 add pB, pB, #16
fmla v18.4s, v5.4s, v12.4s[0] fmla v18.4s, v5.4s, v12.s[0]
fmla v19.4s, v7.4s, v12.4s[0] fmla v19.4s, v7.4s, v12.s[0]
ld1 {v0.4s}, [pA_0] // for next round ld1 {v0.4s}, [pA_0] // for next round
add pA_0, pA_0, #16 add pA_0, pA_0, #16
fmla v20.4s, v1.4s, v12.4s[1] fmla v20.4s, v1.4s, v12.s[1]
fmla v21.4s, v3.4s, v12.4s[1] fmla v21.4s, v3.4s, v12.s[1]
ld1 {v2.4s}, [pA_1] // for next round ld1 {v2.4s}, [pA_1] // for next round
add pA_1, pA_1, #16 add pA_1, pA_1, #16
fmla v22.4s, v5.4s, v12.4s[1] fmla v22.4s, v5.4s, v12.s[1]
fmla v23.4s, v7.4s, v12.4s[1] fmla v23.4s, v7.4s, v12.s[1]
ld1 {v4.4s}, [pA_2] // for next round ld1 {v4.4s}, [pA_2] // for next round
add pA_2, pA_2, #16 add pA_2, pA_2, #16
fmla v24.4s, v1.4s, v12.4s[2] fmla v24.4s, v1.4s, v12.s[2]
fmla v25.4s, v3.4s, v12.4s[2] fmla v25.4s, v3.4s, v12.s[2]
ld1 {v6.4s}, [pA_3] // for next round ld1 {v6.4s}, [pA_3] // for next round
add pA_3, pA_3, #16 add pA_3, pA_3, #16
fmla v26.4s, v5.4s, v12.4s[2] fmla v26.4s, v5.4s, v12.s[2]
fmla v27.4s, v7.4s, v12.4s[2] fmla v27.4s, v7.4s, v12.s[2]
prfm PLDL1KEEP, [pA_2, #512] prfm PLDL1KEEP, [pA_2, #512]
fmla v28.4s, v1.4s, v12.4s[3] fmla v28.4s, v1.4s, v12.s[3]
fmla v29.4s, v3.4s, v12.4s[3] fmla v29.4s, v3.4s, v12.s[3]
prfm PLDL1KEEP, [pA_3, #512] prfm PLDL1KEEP, [pA_3, #512]
fmla v30.4s, v5.4s, v12.4s[3] fmla v30.4s, v5.4s, v12.s[3]
fmla v31.4s, v7.4s, v12.4s[3] fmla v31.4s, v7.4s, v12.s[3]
prfm PLDL1KEEP, [pB, #512] prfm PLDL1KEEP, [pB, #512]
.endm .endm
.macro KERNEL16x4_M1 .macro KERNEL16x4_M1
fmla v16.4s, v0.4s, v8.4s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v17.4s, v2.4s, v8.4s[0] fmla v17.4s, v2.4s, v8.s[0]
ld1 {v12.4s}, [pB] // for next round ld1 {v12.4s}, [pB] // for next round
add pB, pB, #16 add pB, pB, #16
fmla v18.4s, v4.4s, v8.4s[0] fmla v18.4s, v4.4s, v8.s[0]
fmla v19.4s, v6.4s, v8.4s[0] fmla v19.4s, v6.4s, v8.s[0]
ld1 {v1.4s}, [pA_0] // for next round ld1 {v1.4s}, [pA_0] // for next round
add pA_0, pA_0, #16 add pA_0, pA_0, #16
fmla v20.4s, v0.4s, v8.4s[1] fmla v20.4s, v0.4s, v8.s[1]
fmla v21.4s, v2.4s, v8.4s[1] fmla v21.4s, v2.4s, v8.s[1]
ld1 {v3.4s}, [pA_1] // for next round ld1 {v3.4s}, [pA_1] // for next round
add pA_1, pA_1, #16 add pA_1, pA_1, #16
fmla v22.4s, v4.4s, v8.4s[1] fmla v22.4s, v4.4s, v8.s[1]
fmla v23.4s, v6.4s, v8.4s[1] fmla v23.4s, v6.4s, v8.s[1]
ld1 {v5.4s}, [pA_2] // for next round ld1 {v5.4s}, [pA_2] // for next round
add pA_2, pA_2, #16 add pA_2, pA_2, #16
fmla v24.4s, v0.4s, v8.4s[2] fmla v24.4s, v0.4s, v8.s[2]
fmla v25.4s, v2.4s, v8.4s[2] fmla v25.4s, v2.4s, v8.s[2]
ld1 {v7.4s}, [pA_3] // for next round ld1 {v7.4s}, [pA_3] // for next round
add pA_3, pA_3, #16 add pA_3, pA_3, #16
fmla v26.4s, v4.4s, v8.4s[2] fmla v26.4s, v4.4s, v8.s[2]
fmla v27.4s, v6.4s, v8.4s[2] fmla v27.4s, v6.4s, v8.s[2]
prfm PLDL1KEEP, [pA_0, #512] prfm PLDL1KEEP, [pA_0, #512]
fmla v28.4s, v0.4s, v8.4s[3] fmla v28.4s, v0.4s, v8.s[3]
fmla v29.4s, v2.4s, v8.4s[3] fmla v29.4s, v2.4s, v8.s[3]
prfm PLDL1KEEP, [pA_1, #512] prfm PLDL1KEEP, [pA_1, #512]
fmla v30.4s, v4.4s, v8.4s[3] fmla v30.4s, v4.4s, v8.s[3]
fmla v31.4s, v6.4s, v8.4s[3] fmla v31.4s, v6.4s, v8.s[3]
.endm .endm
.macro KERNEL16x4_E .macro KERNEL16x4_E
fmla v16.4s, v1.4s, v12.4s[0] fmla v16.4s, v1.4s, v12.s[0]
fmla v17.4s, v3.4s, v12.4s[0] fmla v17.4s, v3.4s, v12.s[0]
fmla v18.4s, v5.4s, v12.4s[0] fmla v18.4s, v5.4s, v12.s[0]
fmla v19.4s, v7.4s, v12.4s[0] fmla v19.4s, v7.4s, v12.s[0]
fmla v20.4s, v1.4s, v12.4s[1] fmla v20.4s, v1.4s, v12.s[1]
fmla v21.4s, v3.4s, v12.4s[1] fmla v21.4s, v3.4s, v12.s[1]
fmla v22.4s, v5.4s, v12.4s[1] fmla v22.4s, v5.4s, v12.s[1]
fmla v23.4s, v7.4s, v12.4s[1] fmla v23.4s, v7.4s, v12.s[1]
fmla v24.4s, v1.4s, v12.4s[2] fmla v24.4s, v1.4s, v12.s[2]
fmla v25.4s, v3.4s, v12.4s[2] fmla v25.4s, v3.4s, v12.s[2]
fmla v26.4s, v5.4s, v12.4s[2] fmla v26.4s, v5.4s, v12.s[2]
fmla v27.4s, v7.4s, v12.4s[2] fmla v27.4s, v7.4s, v12.s[2]
fmla v28.4s, v1.4s, v12.4s[3] fmla v28.4s, v1.4s, v12.s[3]
fmla v29.4s, v3.4s, v12.4s[3] fmla v29.4s, v3.4s, v12.s[3]
fmla v30.4s, v5.4s, v12.4s[3] fmla v30.4s, v5.4s, v12.s[3]
fmla v31.4s, v7.4s, v12.4s[3] fmla v31.4s, v7.4s, v12.s[3]
.endm .endm
.macro KERNEL16x4_SUB .macro KERNEL16x4_SUB
@ -359,34 +359,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.4s}, [pA_0] ld1 {v0.4s}, [pA_0]
add pA_0, pA_0, #16 add pA_0, pA_0, #16
fmla v16.4s, v0.4s, v8.4s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v20.4s, v0.4s, v8.4s[1] fmla v20.4s, v0.4s, v8.s[1]
fmla v24.4s, v0.4s, v8.4s[2] fmla v24.4s, v0.4s, v8.s[2]
fmla v28.4s, v0.4s, v8.4s[3] fmla v28.4s, v0.4s, v8.s[3]
ld1 {v2.4s}, [pA_1] ld1 {v2.4s}, [pA_1]
add pA_1, pA_1, #16 add pA_1, pA_1, #16
fmla v17.4s, v2.4s, v8.4s[0] fmla v17.4s, v2.4s, v8.s[0]
fmla v21.4s, v2.4s, v8.4s[1] fmla v21.4s, v2.4s, v8.s[1]
fmla v25.4s, v2.4s, v8.4s[2] fmla v25.4s, v2.4s, v8.s[2]
fmla v29.4s, v2.4s, v8.4s[3] fmla v29.4s, v2.4s, v8.s[3]
ld1 {v4.4s}, [pA_2] ld1 {v4.4s}, [pA_2]
add pA_2, pA_2, #16 add pA_2, pA_2, #16
fmla v18.4s, v4.4s, v8.4s[0] fmla v18.4s, v4.4s, v8.s[0]
fmla v22.4s, v4.4s, v8.4s[1] fmla v22.4s, v4.4s, v8.s[1]
fmla v26.4s, v4.4s, v8.4s[2] fmla v26.4s, v4.4s, v8.s[2]
fmla v30.4s, v4.4s, v8.4s[3] fmla v30.4s, v4.4s, v8.s[3]
ld1 {v6.4s}, [pA_3] ld1 {v6.4s}, [pA_3]
add pA_3, pA_3, #16 add pA_3, pA_3, #16
fmla v19.4s, v6.4s, v8.4s[0] fmla v19.4s, v6.4s, v8.s[0]
fmla v23.4s, v6.4s, v8.4s[1] fmla v23.4s, v6.4s, v8.s[1]
fmla v27.4s, v6.4s, v8.4s[2] fmla v27.4s, v6.4s, v8.s[2]
fmla v31.4s, v6.4s, v8.4s[3] fmla v31.4s, v6.4s, v8.s[3]
.endm .endm
.macro SAVE16x4 .macro SAVE16x4
@ -456,28 +456,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s, v1.2s}, [pA_0] ld1 {v0.2s, v1.2s}, [pA_0]
add pA_0, pA_0, #16 add pA_0, pA_0, #16
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v29.2s, v1.2s, v9.2s[1] fmla v29.2s, v1.2s, v9.s[1]
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
fmla v25.2s, v1.2s, v9.2s[0] fmla v25.2s, v1.2s, v9.s[0]
ld1 {v2.2s, v3.2s}, [pA_1] ld1 {v2.2s, v3.2s}, [pA_1]
add pA_1, pA_1, #16 add pA_1, pA_1, #16
fmla v24.2s, v0.2s, v9.2s[0] fmla v24.2s, v0.2s, v9.s[0]
fmla v21.2s, v1.2s, v8.2s[1] fmla v21.2s, v1.2s, v8.s[1]
fmla v28.2s, v0.2s, v9.2s[1] fmla v28.2s, v0.2s, v9.s[1]
fmla v17.2s, v1.2s, v8.2s[0] fmla v17.2s, v1.2s, v8.s[0]
fmla v18.2s, v2.2s, v8.2s[0] fmla v18.2s, v2.2s, v8.s[0]
fmla v31.2s, v3.2s, v9.2s[1] fmla v31.2s, v3.2s, v9.s[1]
fmla v22.2s, v2.2s, v8.2s[1] fmla v22.2s, v2.2s, v8.s[1]
fmla v27.2s, v3.2s, v9.2s[0] fmla v27.2s, v3.2s, v9.s[0]
fmla v26.2s, v2.2s, v9.2s[0] fmla v26.2s, v2.2s, v9.s[0]
fmla v23.2s, v3.2s, v8.2s[1] fmla v23.2s, v3.2s, v8.s[1]
fmla v30.2s, v2.2s, v9.2s[1] fmla v30.2s, v2.2s, v9.s[1]
fmla v19.2s, v3.2s, v8.2s[0] fmla v19.2s, v3.2s, v8.s[0]
.endm .endm
.macro SAVE8x4 .macro SAVE8x4
@ -556,17 +556,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s, v1.2s}, [pA_0] ld1 {v0.2s, v1.2s}, [pA_0]
add pA_0, pA_0, #16 add pA_0, pA_0, #16
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v29.2s, v1.2s, v9.2s[1] fmla v29.2s, v1.2s, v9.s[1]
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
fmla v25.2s, v1.2s, v9.2s[0] fmla v25.2s, v1.2s, v9.s[0]
fmla v24.2s, v0.2s, v9.2s[0] fmla v24.2s, v0.2s, v9.s[0]
fmla v21.2s, v1.2s, v8.2s[1] fmla v21.2s, v1.2s, v8.s[1]
fmla v28.2s, v0.2s, v9.2s[1] fmla v28.2s, v0.2s, v9.s[1]
fmla v17.2s, v1.2s, v8.2s[0] fmla v17.2s, v1.2s, v8.s[0]
.endm .endm
.macro SAVE4x4 .macro SAVE4x4
@ -614,10 +614,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s}, [pA_0] ld1 {v0.2s}, [pA_0]
add pA_0, pA_0, #8 add pA_0, pA_0, #8
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
fmla v24.2s, v0.2s, v9.2s[0] fmla v24.2s, v0.2s, v9.s[0]
fmla v28.2s, v0.2s, v9.2s[1] fmla v28.2s, v0.2s, v9.s[1]
.endm .endm
.macro SAVE2x4 .macro SAVE2x4
@ -700,10 +700,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s, v1.2s}, [pA_0] ld1 {v0.2s, v1.2s}, [pA_0]
add pA_0, pA_0, #16 add pA_0, pA_0, #16
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v17.2s, v1.2s, v8.2s[0] fmla v17.2s, v1.2s, v8.s[0]
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
fmla v21.2s, v1.2s, v8.2s[1] fmla v21.2s, v1.2s, v8.s[1]
.endm .endm
.macro SAVE4x2 .macro SAVE4x2
@ -736,8 +736,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s}, [pA_0] ld1 {v0.2s}, [pA_0]
add pA_0, pA_0, #8 add pA_0, pA_0, #8
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
.endm .endm
.macro SAVE2x2 .macro SAVE2x2
@ -767,7 +767,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ldr s0 , [pA_0] ldr s0 , [pA_0]
add pA_0, pA_0, #4 add pA_0, pA_0, #4
fmla v16.2s, v8.2s, v0.2s[0] fmla v16.2s, v8.2s, v0.s[0]
.endm .endm
.macro SAVE1x2 .macro SAVE1x2
@ -796,8 +796,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s, v1.2s}, [pA_0] ld1 {v0.2s, v1.2s}, [pA_0]
add pA_0 , pA_0, #16 add pA_0 , pA_0, #16
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v17.2s, v1.2s, v8.2s[0] fmla v17.2s, v1.2s, v8.s[0]
.endm .endm
.macro SAVE4x1 .macro SAVE4x1
@ -825,7 +825,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s}, [pA_0] ld1 {v0.2s}, [pA_0]
add pA_0 , pA_0, #8 add pA_0 , pA_0, #8
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
.endm .endm
.macro SAVE2x1 .macro SAVE2x1

472
kernel/arm64/sgemm_kernel_8x8.S
View File

@ -157,22 +157,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v1.4s}, [pA] ld1 {v1.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmul v16.4s, v0.4s, v4.4s[0] fmul v16.4s, v0.4s, v4.s[0]
fmul v17.4s, v1.4s, v4.4s[0] fmul v17.4s, v1.4s, v4.s[0]
fmul v18.4s, v0.4s, v4.4s[1] fmul v18.4s, v0.4s, v4.s[1]
fmul v19.4s, v1.4s, v4.4s[1] fmul v19.4s, v1.4s, v4.s[1]
fmul v20.4s, v0.4s, v4.4s[2] fmul v20.4s, v0.4s, v4.s[2]
fmul v21.4s, v1.4s, v4.4s[2] fmul v21.4s, v1.4s, v4.s[2]
fmul v22.4s, v0.4s, v4.4s[3] fmul v22.4s, v0.4s, v4.s[3]
fmul v23.4s, v1.4s, v4.4s[3] fmul v23.4s, v1.4s, v4.s[3]
fmul v24.4s, v0.4s, v5.4s[0] fmul v24.4s, v0.4s, v5.s[0]
fmul v25.4s, v1.4s, v5.4s[0] fmul v25.4s, v1.4s, v5.s[0]
fmul v26.4s, v0.4s, v5.4s[1] fmul v26.4s, v0.4s, v5.s[1]
fmul v27.4s, v1.4s, v5.4s[1] fmul v27.4s, v1.4s, v5.s[1]
fmul v28.4s, v0.4s, v5.4s[2] fmul v28.4s, v0.4s, v5.s[2]
fmul v29.4s, v1.4s, v5.4s[2] fmul v29.4s, v1.4s, v5.s[2]
fmul v30.4s, v0.4s, v5.4s[3] fmul v30.4s, v0.4s, v5.s[3]
fmul v31.4s, v1.4s, v5.4s[3] fmul v31.4s, v1.4s, v5.s[3]
ld1 {v6.4s}, [pB] ld1 {v6.4s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -185,22 +185,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x8_M1 .macro KERNEL8x8_M1
fmla v16.4s, v0.4s, v4.4s[0] fmla v16.4s, v0.4s, v4.s[0]
fmla v17.4s, v1.4s, v4.4s[0] fmla v17.4s, v1.4s, v4.s[0]
fmla v18.4s, v0.4s, v4.4s[1] fmla v18.4s, v0.4s, v4.s[1]
fmla v19.4s, v1.4s, v4.4s[1] fmla v19.4s, v1.4s, v4.s[1]
fmla v20.4s, v0.4s, v4.4s[2] fmla v20.4s, v0.4s, v4.s[2]
fmla v21.4s, v1.4s, v4.4s[2] fmla v21.4s, v1.4s, v4.s[2]
fmla v22.4s, v0.4s, v4.4s[3] fmla v22.4s, v0.4s, v4.s[3]
fmla v23.4s, v1.4s, v4.4s[3] fmla v23.4s, v1.4s, v4.s[3]
fmla v24.4s, v0.4s, v5.4s[0] fmla v24.4s, v0.4s, v5.s[0]
fmla v25.4s, v1.4s, v5.4s[0] fmla v25.4s, v1.4s, v5.s[0]
fmla v26.4s, v0.4s, v5.4s[1] fmla v26.4s, v0.4s, v5.s[1]
fmla v27.4s, v1.4s, v5.4s[1] fmla v27.4s, v1.4s, v5.s[1]
fmla v28.4s, v0.4s, v5.4s[2] fmla v28.4s, v0.4s, v5.s[2]
fmla v29.4s, v1.4s, v5.4s[2] fmla v29.4s, v1.4s, v5.s[2]
fmla v30.4s, v0.4s, v5.4s[3] fmla v30.4s, v0.4s, v5.s[3]
fmla v31.4s, v1.4s, v5.4s[3] fmla v31.4s, v1.4s, v5.s[3]
ld1 {v6.4s}, [pB] ld1 {v6.4s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -213,22 +213,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x8_M2 .macro KERNEL8x8_M2
fmla v16.4s, v2.4s, v6.4s[0] fmla v16.4s, v2.4s, v6.s[0]
fmla v17.4s, v3.4s, v6.4s[0] fmla v17.4s, v3.4s, v6.s[0]
fmla v18.4s, v2.4s, v6.4s[1] fmla v18.4s, v2.4s, v6.s[1]
fmla v19.4s, v3.4s, v6.4s[1] fmla v19.4s, v3.4s, v6.s[1]
fmla v20.4s, v2.4s, v6.4s[2] fmla v20.4s, v2.4s, v6.s[2]
fmla v21.4s, v3.4s, v6.4s[2] fmla v21.4s, v3.4s, v6.s[2]
fmla v22.4s, v2.4s, v6.4s[3] fmla v22.4s, v2.4s, v6.s[3]
fmla v23.4s, v3.4s, v6.4s[3] fmla v23.4s, v3.4s, v6.s[3]
fmla v24.4s, v2.4s, v7.4s[0] fmla v24.4s, v2.4s, v7.s[0]
fmla v25.4s, v3.4s, v7.4s[0] fmla v25.4s, v3.4s, v7.s[0]
fmla v26.4s, v2.4s, v7.4s[1] fmla v26.4s, v2.4s, v7.s[1]
fmla v27.4s, v3.4s, v7.4s[1] fmla v27.4s, v3.4s, v7.s[1]
fmla v28.4s, v2.4s, v7.4s[2] fmla v28.4s, v2.4s, v7.s[2]
fmla v29.4s, v3.4s, v7.4s[2] fmla v29.4s, v3.4s, v7.s[2]
fmla v30.4s, v2.4s, v7.4s[3] fmla v30.4s, v2.4s, v7.s[3]
fmla v31.4s, v3.4s, v7.4s[3] fmla v31.4s, v3.4s, v7.s[3]
ld1 {v4.4s}, [pB] ld1 {v4.4s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -241,22 +241,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x8_E .macro KERNEL8x8_E
fmla v16.4s, v2.4s, v6.4s[0] fmla v16.4s, v2.4s, v6.s[0]
fmla v17.4s, v3.4s, v6.4s[0] fmla v17.4s, v3.4s, v6.s[0]
fmla v18.4s, v2.4s, v6.4s[1] fmla v18.4s, v2.4s, v6.s[1]
fmla v19.4s, v3.4s, v6.4s[1] fmla v19.4s, v3.4s, v6.s[1]
fmla v20.4s, v2.4s, v6.4s[2] fmla v20.4s, v2.4s, v6.s[2]
fmla v21.4s, v3.4s, v6.4s[2] fmla v21.4s, v3.4s, v6.s[2]
fmla v22.4s, v2.4s, v6.4s[3] fmla v22.4s, v2.4s, v6.s[3]
fmla v23.4s, v3.4s, v6.4s[3] fmla v23.4s, v3.4s, v6.s[3]
fmla v24.4s, v2.4s, v7.4s[0] fmla v24.4s, v2.4s, v7.s[0]
fmla v25.4s, v3.4s, v7.4s[0] fmla v25.4s, v3.4s, v7.s[0]
fmla v26.4s, v2.4s, v7.4s[1] fmla v26.4s, v2.4s, v7.s[1]
fmla v27.4s, v3.4s, v7.4s[1] fmla v27.4s, v3.4s, v7.s[1]
fmla v28.4s, v2.4s, v7.4s[2] fmla v28.4s, v2.4s, v7.s[2]
fmla v29.4s, v3.4s, v7.4s[2] fmla v29.4s, v3.4s, v7.s[2]
fmla v30.4s, v2.4s, v7.4s[3] fmla v30.4s, v2.4s, v7.s[3]
fmla v31.4s, v3.4s, v7.4s[3] fmla v31.4s, v3.4s, v7.s[3]
.endm .endm
.macro KERNEL8x8_SUB .macro KERNEL8x8_SUB
@ -269,22 +269,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v1.4s}, [pA] ld1 {v1.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.4s, v0.4s, v4.4s[0] fmla v16.4s, v0.4s, v4.s[0]
fmla v17.4s, v1.4s, v4.4s[0] fmla v17.4s, v1.4s, v4.s[0]
fmla v18.4s, v0.4s, v4.4s[1] fmla v18.4s, v0.4s, v4.s[1]
fmla v19.4s, v1.4s, v4.4s[1] fmla v19.4s, v1.4s, v4.s[1]
fmla v20.4s, v0.4s, v4.4s[2] fmla v20.4s, v0.4s, v4.s[2]
fmla v21.4s, v1.4s, v4.4s[2] fmla v21.4s, v1.4s, v4.s[2]
fmla v22.4s, v0.4s, v4.4s[3] fmla v22.4s, v0.4s, v4.s[3]
fmla v23.4s, v1.4s, v4.4s[3] fmla v23.4s, v1.4s, v4.s[3]
fmla v24.4s, v0.4s, v5.4s[0] fmla v24.4s, v0.4s, v5.s[0]
fmla v25.4s, v1.4s, v5.4s[0] fmla v25.4s, v1.4s, v5.s[0]
fmla v26.4s, v0.4s, v5.4s[1] fmla v26.4s, v0.4s, v5.s[1]
fmla v27.4s, v1.4s, v5.4s[1] fmla v27.4s, v1.4s, v5.s[1]
fmla v28.4s, v0.4s, v5.4s[2] fmla v28.4s, v0.4s, v5.s[2]
fmla v29.4s, v1.4s, v5.4s[2] fmla v29.4s, v1.4s, v5.s[2]
fmla v30.4s, v0.4s, v5.4s[3] fmla v30.4s, v0.4s, v5.s[3]
fmla v31.4s, v1.4s, v5.4s[3] fmla v31.4s, v1.4s, v5.s[3]
.endm .endm
.macro SAVE8x8 .macro SAVE8x8
@ -367,14 +367,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.4s}, [pA] ld1 {v0.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmul v16.4s, v0.4s, v4.4s[0] fmul v16.4s, v0.4s, v4.s[0]
fmul v18.4s, v0.4s, v4.4s[1] fmul v18.4s, v0.4s, v4.s[1]
fmul v20.4s, v0.4s, v4.4s[2] fmul v20.4s, v0.4s, v4.s[2]
fmul v22.4s, v0.4s, v4.4s[3] fmul v22.4s, v0.4s, v4.s[3]
fmul v24.4s, v0.4s, v5.4s[0] fmul v24.4s, v0.4s, v5.s[0]
fmul v26.4s, v0.4s, v5.4s[1] fmul v26.4s, v0.4s, v5.s[1]
fmul v28.4s, v0.4s, v5.4s[2] fmul v28.4s, v0.4s, v5.s[2]
fmul v30.4s, v0.4s, v5.4s[3] fmul v30.4s, v0.4s, v5.s[3]
ld1 {v6.4s}, [pB] ld1 {v6.4s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -385,14 +385,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL4x8_M1 .macro KERNEL4x8_M1
fmla v16.4s, v0.4s, v4.4s[0] fmla v16.4s, v0.4s, v4.s[0]
fmla v18.4s, v0.4s, v4.4s[1] fmla v18.4s, v0.4s, v4.s[1]
fmla v20.4s, v0.4s, v4.4s[2] fmla v20.4s, v0.4s, v4.s[2]
fmla v22.4s, v0.4s, v4.4s[3] fmla v22.4s, v0.4s, v4.s[3]
fmla v24.4s, v0.4s, v5.4s[0] fmla v24.4s, v0.4s, v5.s[0]
fmla v26.4s, v0.4s, v5.4s[1] fmla v26.4s, v0.4s, v5.s[1]
fmla v28.4s, v0.4s, v5.4s[2] fmla v28.4s, v0.4s, v5.s[2]
fmla v30.4s, v0.4s, v5.4s[3] fmla v30.4s, v0.4s, v5.s[3]
ld1 {v6.4s}, [pB] ld1 {v6.4s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -403,14 +403,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL4x8_M2 .macro KERNEL4x8_M2
fmla v16.4s, v2.4s, v6.4s[0] fmla v16.4s, v2.4s, v6.s[0]
fmla v18.4s, v2.4s, v6.4s[1] fmla v18.4s, v2.4s, v6.s[1]
fmla v20.4s, v2.4s, v6.4s[2] fmla v20.4s, v2.4s, v6.s[2]
fmla v22.4s, v2.4s, v6.4s[3] fmla v22.4s, v2.4s, v6.s[3]
fmla v24.4s, v2.4s, v7.4s[0] fmla v24.4s, v2.4s, v7.s[0]
fmla v26.4s, v2.4s, v7.4s[1] fmla v26.4s, v2.4s, v7.s[1]
fmla v28.4s, v2.4s, v7.4s[2] fmla v28.4s, v2.4s, v7.s[2]
fmla v30.4s, v2.4s, v7.4s[3] fmla v30.4s, v2.4s, v7.s[3]
ld1 {v4.4s}, [pB] ld1 {v4.4s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -421,14 +421,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL4x8_E .macro KERNEL4x8_E
fmla v16.4s, v2.4s, v6.4s[0] fmla v16.4s, v2.4s, v6.s[0]
fmla v18.4s, v2.4s, v6.4s[1] fmla v18.4s, v2.4s, v6.s[1]
fmla v20.4s, v2.4s, v6.4s[2] fmla v20.4s, v2.4s, v6.s[2]
fmla v22.4s, v2.4s, v6.4s[3] fmla v22.4s, v2.4s, v6.s[3]
fmla v24.4s, v2.4s, v7.4s[0] fmla v24.4s, v2.4s, v7.s[0]
fmla v26.4s, v2.4s, v7.4s[1] fmla v26.4s, v2.4s, v7.s[1]
fmla v28.4s, v2.4s, v7.4s[2] fmla v28.4s, v2.4s, v7.s[2]
fmla v30.4s, v2.4s, v7.4s[3] fmla v30.4s, v2.4s, v7.s[3]
.endm .endm
.macro KERNEL4x8_SUB .macro KERNEL4x8_SUB
@ -439,14 +439,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.4s}, [pA] ld1 {v0.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.4s, v0.4s, v4.4s[0] fmla v16.4s, v0.4s, v4.s[0]
fmla v18.4s, v0.4s, v4.4s[1] fmla v18.4s, v0.4s, v4.s[1]
fmla v20.4s, v0.4s, v4.4s[2] fmla v20.4s, v0.4s, v4.s[2]
fmla v22.4s, v0.4s, v4.4s[3] fmla v22.4s, v0.4s, v4.s[3]
fmla v24.4s, v0.4s, v5.4s[0] fmla v24.4s, v0.4s, v5.s[0]
fmla v26.4s, v0.4s, v5.4s[1] fmla v26.4s, v0.4s, v5.s[1]
fmla v28.4s, v0.4s, v5.4s[2] fmla v28.4s, v0.4s, v5.s[2]
fmla v30.4s, v0.4s, v5.4s[3] fmla v30.4s, v0.4s, v5.s[3]
.endm .endm
.macro SAVE4x8 .macro SAVE4x8
@ -520,14 +520,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s}, [pA] ld1 {v0.2s}, [pA]
add pA, pA, #8 add pA, pA, #8
fmla v16.2s, v0.2s, v4.4s[0] fmla v16.2s, v0.2s, v4.s[0]
fmla v18.2s, v0.2s, v4.4s[1] fmla v18.2s, v0.2s, v4.s[1]
fmla v20.2s, v0.2s, v4.4s[2] fmla v20.2s, v0.2s, v4.s[2]
fmla v22.2s, v0.2s, v4.4s[3] fmla v22.2s, v0.2s, v4.s[3]
fmla v24.2s, v0.2s, v5.4s[0] fmla v24.2s, v0.2s, v5.s[0]
fmla v26.2s, v0.2s, v5.4s[1] fmla v26.2s, v0.2s, v5.s[1]
fmla v28.2s, v0.2s, v5.4s[2] fmla v28.2s, v0.2s, v5.s[2]
fmla v30.2s, v0.2s, v5.4s[3] fmla v30.2s, v0.2s, v5.s[3]
.endm .endm
.macro SAVE2x8 .macro SAVE2x8
@ -601,14 +601,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ldr s0, [pA] ldr s0, [pA]
add pA, pA, #4 add pA, pA, #4
fmla s16, s0, v4.4s[0] fmla s16, s0, v4.s[0]
fmla s18, s0, v4.4s[1] fmla s18, s0, v4.s[1]
fmla s20, s0, v4.4s[2] fmla s20, s0, v4.s[2]
fmla s22, s0, v4.4s[3] fmla s22, s0, v4.s[3]
fmla s24, s0, v5.4s[0] fmla s24, s0, v5.s[0]
fmla s26, s0, v5.4s[1] fmla s26, s0, v5.s[1]
fmla s28, s0, v5.4s[2] fmla s28, s0, v5.s[2]
fmla s30, s0, v5.4s[3] fmla s30, s0, v5.s[3]
.endm .endm
.macro SAVE1x8 .macro SAVE1x8
@ -682,14 +682,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v1.4s}, [pA] ld1 {v1.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmul v16.4s, v0.4s, v8.2s[0] fmul v16.4s, v0.4s, v8.s[0]
fmul v17.4s, v1.4s, v8.2s[0] fmul v17.4s, v1.4s, v8.s[0]
fmul v20.4s, v0.4s, v8.2s[1] fmul v20.4s, v0.4s, v8.s[1]
fmul v21.4s, v1.4s, v8.2s[1] fmul v21.4s, v1.4s, v8.s[1]
fmul v24.4s, v0.4s, v9.2s[0] fmul v24.4s, v0.4s, v9.s[0]
fmul v25.4s, v1.4s, v9.2s[0] fmul v25.4s, v1.4s, v9.s[0]
fmul v28.4s, v0.4s, v9.2s[1] fmul v28.4s, v0.4s, v9.s[1]
fmul v29.4s, v1.4s, v9.2s[1] fmul v29.4s, v1.4s, v9.s[1]
ld1 {v12.2s, v13.2s}, [pB] ld1 {v12.2s, v13.2s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -700,14 +700,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x4_M1 .macro KERNEL8x4_M1
fmla v16.4s, v0.4s, v8.2s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v17.4s, v1.4s, v8.2s[0] fmla v17.4s, v1.4s, v8.s[0]
fmla v20.4s, v0.4s, v8.2s[1] fmla v20.4s, v0.4s, v8.s[1]
fmla v21.4s, v1.4s, v8.2s[1] fmla v21.4s, v1.4s, v8.s[1]
fmla v24.4s, v0.4s, v9.2s[0] fmla v24.4s, v0.4s, v9.s[0]
fmla v25.4s, v1.4s, v9.2s[0] fmla v25.4s, v1.4s, v9.s[0]
fmla v28.4s, v0.4s, v9.2s[1] fmla v28.4s, v0.4s, v9.s[1]
fmla v29.4s, v1.4s, v9.2s[1] fmla v29.4s, v1.4s, v9.s[1]
ld1 {v12.2s, v13.2s}, [pB] ld1 {v12.2s, v13.2s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -718,14 +718,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x4_M2 .macro KERNEL8x4_M2
fmla v16.4s, v4.4s, v12.2s[0] fmla v16.4s, v4.4s, v12.s[0]
fmla v17.4s, v5.4s, v12.2s[0] fmla v17.4s, v5.4s, v12.s[0]
fmla v20.4s, v4.4s, v12.2s[1] fmla v20.4s, v4.4s, v12.s[1]
fmla v21.4s, v5.4s, v12.2s[1] fmla v21.4s, v5.4s, v12.s[1]
fmla v24.4s, v4.4s, v13.2s[0] fmla v24.4s, v4.4s, v13.s[0]
fmla v25.4s, v5.4s, v13.2s[0] fmla v25.4s, v5.4s, v13.s[0]
fmla v28.4s, v4.4s, v13.2s[1] fmla v28.4s, v4.4s, v13.s[1]
fmla v29.4s, v5.4s, v13.2s[1] fmla v29.4s, v5.4s, v13.s[1]
ld1 {v8.2s, v9.2s}, [pB] ld1 {v8.2s, v9.2s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -736,14 +736,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x4_E .macro KERNEL8x4_E
fmla v16.4s, v4.4s, v12.2s[0] fmla v16.4s, v4.4s, v12.s[0]
fmla v17.4s, v5.4s, v12.2s[0] fmla v17.4s, v5.4s, v12.s[0]
fmla v20.4s, v4.4s, v12.2s[1] fmla v20.4s, v4.4s, v12.s[1]
fmla v21.4s, v5.4s, v12.2s[1] fmla v21.4s, v5.4s, v12.s[1]
fmla v24.4s, v4.4s, v13.2s[0] fmla v24.4s, v4.4s, v13.s[0]
fmla v25.4s, v5.4s, v13.2s[0] fmla v25.4s, v5.4s, v13.s[0]
fmla v28.4s, v4.4s, v13.2s[1] fmla v28.4s, v4.4s, v13.s[1]
fmla v29.4s, v5.4s, v13.2s[1] fmla v29.4s, v5.4s, v13.s[1]
.endm .endm
.macro KERNEL8x4_SUB .macro KERNEL8x4_SUB
@ -754,14 +754,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v1.4s}, [pA] ld1 {v1.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.4s, v0.4s, v8.2s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v17.4s, v1.4s, v8.2s[0] fmla v17.4s, v1.4s, v8.s[0]
fmla v20.4s, v0.4s, v8.2s[1] fmla v20.4s, v0.4s, v8.s[1]
fmla v21.4s, v1.4s, v8.2s[1] fmla v21.4s, v1.4s, v8.s[1]
fmla v24.4s, v0.4s, v9.2s[0] fmla v24.4s, v0.4s, v9.s[0]
fmla v25.4s, v1.4s, v9.2s[0] fmla v25.4s, v1.4s, v9.s[0]
fmla v28.4s, v0.4s, v9.2s[1] fmla v28.4s, v0.4s, v9.s[1]
fmla v29.4s, v1.4s, v9.2s[1] fmla v29.4s, v1.4s, v9.s[1]
.endm .endm
.macro SAVE8x4 .macro SAVE8x4
@ -814,17 +814,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s, v1.2s}, [pA] ld1 {v0.2s, v1.2s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmul v16.2s, v0.2s, v8.2s[0] fmul v16.2s, v0.2s, v8.s[0]
fmul v29.2s, v1.2s, v9.2s[1] fmul v29.2s, v1.2s, v9.s[1]
fmul v20.2s, v0.2s, v8.2s[1] fmul v20.2s, v0.2s, v8.s[1]
fmul v25.2s, v1.2s, v9.2s[0] fmul v25.2s, v1.2s, v9.s[0]
fmul v24.2s, v0.2s, v9.2s[0] fmul v24.2s, v0.2s, v9.s[0]
fmul v21.2s, v1.2s, v8.2s[1] fmul v21.2s, v1.2s, v8.s[1]
fmul v28.2s, v0.2s, v9.2s[1] fmul v28.2s, v0.2s, v9.s[1]
fmul v17.2s, v1.2s, v8.2s[0] fmul v17.2s, v1.2s, v8.s[0]
ld1 {v12.2s, v13.2s}, [pB] ld1 {v12.2s, v13.2s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -833,61 +833,61 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL4x4_M1 .macro KERNEL4x4_M1
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v29.2s, v1.2s, v9.2s[1] fmla v29.2s, v1.2s, v9.s[1]
ld1 {v12.2s, v13.2s}, [pB] // For next round ld1 {v12.2s, v13.2s}, [pB] // For next round
add pB, pB, #16 add pB, pB, #16
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
fmla v25.2s, v1.2s, v9.2s[0] fmla v25.2s, v1.2s, v9.s[0]
ld1 {v4.2s, v5.2s}, [pA] // For next round ld1 {v4.2s, v5.2s}, [pA] // For next round
add pA, pA, #16 add pA, pA, #16
fmla v24.2s, v0.2s, v9.2s[0] fmla v24.2s, v0.2s, v9.s[0]
fmla v21.2s, v1.2s, v8.2s[1] fmla v21.2s, v1.2s, v8.s[1]
prfm PLDL1KEEP, [pB, #512] prfm PLDL1KEEP, [pB, #512]
fmla v28.2s, v0.2s, v9.2s[1] fmla v28.2s, v0.2s, v9.s[1]
fmla v17.2s, v1.2s, v8.2s[0] fmla v17.2s, v1.2s, v8.s[0]
.endm .endm
.macro KERNEL4x4_M2 .macro KERNEL4x4_M2
fmla v16.2s, v4.2s, v12.2s[0] fmla v16.2s, v4.2s, v12.s[0]
fmla v29.2s, v5.2s, v13.2s[1] fmla v29.2s, v5.2s, v13.s[1]
ld1 {v8.2s, v9.2s}, [pB] // For next round ld1 {v8.2s, v9.2s}, [pB] // For next round
add pB, pB, #16 add pB, pB, #16
fmla v20.2s, v4.2s, v12.2s[1] fmla v20.2s, v4.2s, v12.s[1]
fmla v25.2s, v5.2s, v13.2s[0] fmla v25.2s, v5.2s, v13.s[0]
ld1 {v0.2s, v1.2s}, [pA] // For next round ld1 {v0.2s, v1.2s}, [pA] // For next round
add pA, pA, #16 add pA, pA, #16
fmla v24.2s, v4.2s, v13.2s[0] fmla v24.2s, v4.2s, v13.s[0]
fmla v21.2s, v5.2s, v12.2s[1] fmla v21.2s, v5.2s, v12.s[1]
prfm PLDL1KEEP, [pA, #512] prfm PLDL1KEEP, [pA, #512]
fmla v28.2s, v4.2s, v13.2s[1] fmla v28.2s, v4.2s, v13.s[1]
fmla v17.2s, v5.2s, v12.2s[0] fmla v17.2s, v5.2s, v12.s[0]
.endm .endm
.macro KERNEL4x4_E .macro KERNEL4x4_E
fmla v16.2s, v4.2s, v12.2s[0] fmla v16.2s, v4.2s, v12.s[0]
fmla v29.2s, v5.2s, v13.2s[1] fmla v29.2s, v5.2s, v13.s[1]
fmla v20.2s, v4.2s, v12.2s[1] fmla v20.2s, v4.2s, v12.s[1]
fmla v25.2s, v5.2s, v13.2s[0] fmla v25.2s, v5.2s, v13.s[0]
fmla v24.2s, v4.2s, v13.2s[0] fmla v24.2s, v4.2s, v13.s[0]
fmla v21.2s, v5.2s, v12.2s[1] fmla v21.2s, v5.2s, v12.s[1]
fmla v28.2s, v4.2s, v13.2s[1] fmla v28.2s, v4.2s, v13.s[1]
fmla v17.2s, v5.2s, v12.2s[0] fmla v17.2s, v5.2s, v12.s[0]
.endm .endm
.macro KERNEL4x4_SUB .macro KERNEL4x4_SUB
@ -896,17 +896,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s, v1.2s}, [pA] ld1 {v0.2s, v1.2s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v29.2s, v1.2s, v9.2s[1] fmla v29.2s, v1.2s, v9.s[1]
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
fmla v25.2s, v1.2s, v9.2s[0] fmla v25.2s, v1.2s, v9.s[0]
fmla v24.2s, v0.2s, v9.2s[0] fmla v24.2s, v0.2s, v9.s[0]
fmla v21.2s, v1.2s, v8.2s[1] fmla v21.2s, v1.2s, v8.s[1]
fmla v28.2s, v0.2s, v9.2s[1] fmla v28.2s, v0.2s, v9.s[1]
fmla v17.2s, v1.2s, v8.2s[0] fmla v17.2s, v1.2s, v8.s[0]
.endm .endm
.macro SAVE4x4 .macro SAVE4x4
@ -951,10 +951,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s}, [pA] ld1 {v0.2s}, [pA]
add pA, pA, #8 add pA, pA, #8
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
fmla v24.2s, v0.2s, v9.2s[0] fmla v24.2s, v0.2s, v9.s[0]
fmla v28.2s, v0.2s, v9.2s[1] fmla v28.2s, v0.2s, v9.s[1]
.endm .endm
.macro SAVE2x4 .macro SAVE2x4
@ -1034,11 +1034,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v1.4s}, [pA] ld1 {v1.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.4s, v0.4s, v8.2s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v17.4s, v1.4s, v8.2s[0] fmla v17.4s, v1.4s, v8.s[0]
fmla v20.4s, v0.4s, v8.2s[1] fmla v20.4s, v0.4s, v8.s[1]
fmla v21.4s, v1.4s, v8.2s[1] fmla v21.4s, v1.4s, v8.s[1]
.endm .endm
.macro SAVE8x2 .macro SAVE8x2
@ -1074,10 +1074,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s, v1.2s}, [pA] ld1 {v0.2s, v1.2s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v17.2s, v1.2s, v8.2s[0] fmla v17.2s, v1.2s, v8.s[0]
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
fmla v21.2s, v1.2s, v8.2s[1] fmla v21.2s, v1.2s, v8.s[1]
.endm .endm
.macro SAVE4x2 .macro SAVE4x2
@ -1109,8 +1109,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s}, [pA] ld1 {v0.2s}, [pA]
add pA, pA, #8 add pA, pA, #8
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
.endm .endm
.macro SAVE2x2 .macro SAVE2x2
@ -1139,7 +1139,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ldr s0 , [pA] ldr s0 , [pA]
add pA, pA, #4 add pA, pA, #4
fmla v16.2s, v8.2s, v0.2s[0] fmla v16.2s, v8.2s, v0.s[0]
.endm .endm
.macro SAVE1x2 .macro SAVE1x2
@ -1169,8 +1169,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v1.4s}, [pA] ld1 {v1.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.4s, v0.4s, v8.2s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v17.4s, v1.4s, v8.2s[0] fmla v17.4s, v1.4s, v8.s[0]
.endm .endm
.macro SAVE8x1 .macro SAVE8x1
@ -1196,8 +1196,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s, v1.2s}, [pA] ld1 {v0.2s, v1.2s}, [pA]
add pA , pA, #16 add pA , pA, #16
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v17.2s, v1.2s, v8.2s[0] fmla v17.2s, v1.2s, v8.s[0]
.endm .endm
.macro SAVE4x1 .macro SAVE4x1
@ -1222,7 +1222,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s}, [pA] ld1 {v0.2s}, [pA]
add pA , pA, #8 add pA , pA, #8
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
.endm .endm
.macro SAVE2x1 .macro SAVE2x1

384
kernel/arm64/strmm_kernel_16x4.S Executable file → Normal file
View File

@ -161,25 +161,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v3.4s}, [pA] ld1 {v3.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmul v16.4s, v0.4s, v8.2s[0] fmul v16.4s, v0.4s, v8.s[0]
fmul v17.4s, v1.4s, v8.2s[0] fmul v17.4s, v1.4s, v8.s[0]
fmul v18.4s, v2.4s, v8.2s[0] fmul v18.4s, v2.4s, v8.s[0]
fmul v19.4s, v3.4s, v8.2s[0] fmul v19.4s, v3.4s, v8.s[0]
fmul v20.4s, v0.4s, v8.2s[1] fmul v20.4s, v0.4s, v8.s[1]
fmul v21.4s, v1.4s, v8.2s[1] fmul v21.4s, v1.4s, v8.s[1]
fmul v22.4s, v2.4s, v8.2s[1] fmul v22.4s, v2.4s, v8.s[1]
fmul v23.4s, v3.4s, v8.2s[1] fmul v23.4s, v3.4s, v8.s[1]
fmul v24.4s, v0.4s, v9.2s[0] fmul v24.4s, v0.4s, v9.s[0]
fmul v25.4s, v1.4s, v9.2s[0] fmul v25.4s, v1.4s, v9.s[0]
fmul v26.4s, v2.4s, v9.2s[0] fmul v26.4s, v2.4s, v9.s[0]
fmul v27.4s, v3.4s, v9.2s[0] fmul v27.4s, v3.4s, v9.s[0]
fmul v28.4s, v0.4s, v9.2s[1] fmul v28.4s, v0.4s, v9.s[1]
fmul v29.4s, v1.4s, v9.2s[1] fmul v29.4s, v1.4s, v9.s[1]
fmul v30.4s, v2.4s, v9.2s[1] fmul v30.4s, v2.4s, v9.s[1]
fmul v31.4s, v3.4s, v9.2s[1] fmul v31.4s, v3.4s, v9.s[1]
ld1 {v12.2s, v13.2s}, [pB] ld1 {v12.2s, v13.2s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -194,25 +194,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL16x4_M1 .macro KERNEL16x4_M1
fmla v16.4s, v0.4s, v8.2s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v17.4s, v1.4s, v8.2s[0] fmla v17.4s, v1.4s, v8.s[0]
fmla v18.4s, v2.4s, v8.2s[0] fmla v18.4s, v2.4s, v8.s[0]
fmla v19.4s, v3.4s, v8.2s[0] fmla v19.4s, v3.4s, v8.s[0]
fmla v20.4s, v0.4s, v8.2s[1] fmla v20.4s, v0.4s, v8.s[1]
fmla v21.4s, v1.4s, v8.2s[1] fmla v21.4s, v1.4s, v8.s[1]
fmla v22.4s, v2.4s, v8.2s[1] fmla v22.4s, v2.4s, v8.s[1]
fmla v23.4s, v3.4s, v8.2s[1] fmla v23.4s, v3.4s, v8.s[1]
fmla v24.4s, v0.4s, v9.2s[0] fmla v24.4s, v0.4s, v9.s[0]
fmla v25.4s, v1.4s, v9.2s[0] fmla v25.4s, v1.4s, v9.s[0]
fmla v26.4s, v2.4s, v9.2s[0] fmla v26.4s, v2.4s, v9.s[0]
fmla v27.4s, v3.4s, v9.2s[0] fmla v27.4s, v3.4s, v9.s[0]
fmla v28.4s, v0.4s, v9.2s[1] fmla v28.4s, v0.4s, v9.s[1]
fmla v29.4s, v1.4s, v9.2s[1] fmla v29.4s, v1.4s, v9.s[1]
fmla v30.4s, v2.4s, v9.2s[1] fmla v30.4s, v2.4s, v9.s[1]
fmla v31.4s, v3.4s, v9.2s[1] fmla v31.4s, v3.4s, v9.s[1]
ld1 {v12.2s, v13.2s}, [pB] ld1 {v12.2s, v13.2s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -227,25 +227,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL16x4_M2 .macro KERNEL16x4_M2
fmla v16.4s, v4.4s, v12.2s[0] fmla v16.4s, v4.4s, v12.s[0]
fmla v17.4s, v5.4s, v12.2s[0] fmla v17.4s, v5.4s, v12.s[0]
fmla v18.4s, v6.4s, v12.2s[0] fmla v18.4s, v6.4s, v12.s[0]
fmla v19.4s, v7.4s, v12.2s[0] fmla v19.4s, v7.4s, v12.s[0]
fmla v20.4s, v4.4s, v12.2s[1] fmla v20.4s, v4.4s, v12.s[1]
fmla v21.4s, v5.4s, v12.2s[1] fmla v21.4s, v5.4s, v12.s[1]
fmla v22.4s, v6.4s, v12.2s[1] fmla v22.4s, v6.4s, v12.s[1]
fmla v23.4s, v7.4s, v12.2s[1] fmla v23.4s, v7.4s, v12.s[1]
fmla v24.4s, v4.4s, v13.2s[0] fmla v24.4s, v4.4s, v13.s[0]
fmla v25.4s, v5.4s, v13.2s[0] fmla v25.4s, v5.4s, v13.s[0]
fmla v26.4s, v6.4s, v13.2s[0] fmla v26.4s, v6.4s, v13.s[0]
fmla v27.4s, v7.4s, v13.2s[0] fmla v27.4s, v7.4s, v13.s[0]
fmla v28.4s, v4.4s, v13.2s[1] fmla v28.4s, v4.4s, v13.s[1]
fmla v29.4s, v5.4s, v13.2s[1] fmla v29.4s, v5.4s, v13.s[1]
fmla v30.4s, v6.4s, v13.2s[1] fmla v30.4s, v6.4s, v13.s[1]
fmla v31.4s, v7.4s, v13.2s[1] fmla v31.4s, v7.4s, v13.s[1]
ld1 {v8.2s, v9.2s}, [pB] ld1 {v8.2s, v9.2s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -260,25 +260,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL16x4_E .macro KERNEL16x4_E
fmla v16.4s, v4.4s, v12.2s[0] fmla v16.4s, v4.4s, v12.s[0]
fmla v17.4s, v5.4s, v12.2s[0] fmla v17.4s, v5.4s, v12.s[0]
fmla v18.4s, v6.4s, v12.2s[0] fmla v18.4s, v6.4s, v12.s[0]
fmla v19.4s, v7.4s, v12.2s[0] fmla v19.4s, v7.4s, v12.s[0]
fmla v20.4s, v4.4s, v12.2s[1] fmla v20.4s, v4.4s, v12.s[1]
fmla v21.4s, v5.4s, v12.2s[1] fmla v21.4s, v5.4s, v12.s[1]
fmla v22.4s, v6.4s, v12.2s[1] fmla v22.4s, v6.4s, v12.s[1]
fmla v23.4s, v7.4s, v12.2s[1] fmla v23.4s, v7.4s, v12.s[1]
fmla v24.4s, v4.4s, v13.2s[0] fmla v24.4s, v4.4s, v13.s[0]
fmla v25.4s, v5.4s, v13.2s[0] fmla v25.4s, v5.4s, v13.s[0]
fmla v26.4s, v6.4s, v13.2s[0] fmla v26.4s, v6.4s, v13.s[0]
fmla v27.4s, v7.4s, v13.2s[0] fmla v27.4s, v7.4s, v13.s[0]
fmla v28.4s, v4.4s, v13.2s[1] fmla v28.4s, v4.4s, v13.s[1]
fmla v29.4s, v5.4s, v13.2s[1] fmla v29.4s, v5.4s, v13.s[1]
fmla v30.4s, v6.4s, v13.2s[1] fmla v30.4s, v6.4s, v13.s[1]
fmla v31.4s, v7.4s, v13.2s[1] fmla v31.4s, v7.4s, v13.s[1]
.endm .endm
.macro KERNEL16x4_SUB .macro KERNEL16x4_SUB
@ -293,25 +293,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v3.4s}, [pA] ld1 {v3.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.4s, v0.4s, v8.2s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v17.4s, v1.4s, v8.2s[0] fmla v17.4s, v1.4s, v8.s[0]
fmla v18.4s, v2.4s, v8.2s[0] fmla v18.4s, v2.4s, v8.s[0]
fmla v19.4s, v3.4s, v8.2s[0] fmla v19.4s, v3.4s, v8.s[0]
fmla v20.4s, v0.4s, v8.2s[1] fmla v20.4s, v0.4s, v8.s[1]
fmla v21.4s, v1.4s, v8.2s[1] fmla v21.4s, v1.4s, v8.s[1]
fmla v22.4s, v2.4s, v8.2s[1] fmla v22.4s, v2.4s, v8.s[1]
fmla v23.4s, v3.4s, v8.2s[1] fmla v23.4s, v3.4s, v8.s[1]
fmla v24.4s, v0.4s, v9.2s[0] fmla v24.4s, v0.4s, v9.s[0]
fmla v25.4s, v1.4s, v9.2s[0] fmla v25.4s, v1.4s, v9.s[0]
fmla v26.4s, v2.4s, v9.2s[0] fmla v26.4s, v2.4s, v9.s[0]
fmla v27.4s, v3.4s, v9.2s[0] fmla v27.4s, v3.4s, v9.s[0]
fmla v28.4s, v0.4s, v9.2s[1] fmla v28.4s, v0.4s, v9.s[1]
fmla v29.4s, v1.4s, v9.2s[1] fmla v29.4s, v1.4s, v9.s[1]
fmla v30.4s, v2.4s, v9.2s[1] fmla v30.4s, v2.4s, v9.s[1]
fmla v31.4s, v3.4s, v9.2s[1] fmla v31.4s, v3.4s, v9.s[1]
.endm .endm
.macro SAVE16x4 .macro SAVE16x4
@ -369,14 +369,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v1.4s}, [pA] ld1 {v1.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmul v16.4s, v0.4s, v8.2s[0] fmul v16.4s, v0.4s, v8.s[0]
fmul v17.4s, v1.4s, v8.2s[0] fmul v17.4s, v1.4s, v8.s[0]
fmul v20.4s, v0.4s, v8.2s[1] fmul v20.4s, v0.4s, v8.s[1]
fmul v21.4s, v1.4s, v8.2s[1] fmul v21.4s, v1.4s, v8.s[1]
fmul v24.4s, v0.4s, v9.2s[0] fmul v24.4s, v0.4s, v9.s[0]
fmul v25.4s, v1.4s, v9.2s[0] fmul v25.4s, v1.4s, v9.s[0]
fmul v28.4s, v0.4s, v9.2s[1] fmul v28.4s, v0.4s, v9.s[1]
fmul v29.4s, v1.4s, v9.2s[1] fmul v29.4s, v1.4s, v9.s[1]
ld1 {v12.2s, v13.2s}, [pB] ld1 {v12.2s, v13.2s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -387,14 +387,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x4_M1 .macro KERNEL8x4_M1
fmla v16.4s, v0.4s, v8.2s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v17.4s, v1.4s, v8.2s[0] fmla v17.4s, v1.4s, v8.s[0]
fmla v20.4s, v0.4s, v8.2s[1] fmla v20.4s, v0.4s, v8.s[1]
fmla v21.4s, v1.4s, v8.2s[1] fmla v21.4s, v1.4s, v8.s[1]
fmla v24.4s, v0.4s, v9.2s[0] fmla v24.4s, v0.4s, v9.s[0]
fmla v25.4s, v1.4s, v9.2s[0] fmla v25.4s, v1.4s, v9.s[0]
fmla v28.4s, v0.4s, v9.2s[1] fmla v28.4s, v0.4s, v9.s[1]
fmla v29.4s, v1.4s, v9.2s[1] fmla v29.4s, v1.4s, v9.s[1]
ld1 {v12.2s, v13.2s}, [pB] ld1 {v12.2s, v13.2s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -405,14 +405,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x4_M2 .macro KERNEL8x4_M2
fmla v16.4s, v4.4s, v12.2s[0] fmla v16.4s, v4.4s, v12.s[0]
fmla v17.4s, v5.4s, v12.2s[0] fmla v17.4s, v5.4s, v12.s[0]
fmla v20.4s, v4.4s, v12.2s[1] fmla v20.4s, v4.4s, v12.s[1]
fmla v21.4s, v5.4s, v12.2s[1] fmla v21.4s, v5.4s, v12.s[1]
fmla v24.4s, v4.4s, v13.2s[0] fmla v24.4s, v4.4s, v13.s[0]
fmla v25.4s, v5.4s, v13.2s[0] fmla v25.4s, v5.4s, v13.s[0]
fmla v28.4s, v4.4s, v13.2s[1] fmla v28.4s, v4.4s, v13.s[1]
fmla v29.4s, v5.4s, v13.2s[1] fmla v29.4s, v5.4s, v13.s[1]
ld1 {v8.2s, v9.2s}, [pB] ld1 {v8.2s, v9.2s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -423,14 +423,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x4_E .macro KERNEL8x4_E
fmla v16.4s, v4.4s, v12.2s[0] fmla v16.4s, v4.4s, v12.s[0]
fmla v17.4s, v5.4s, v12.2s[0] fmla v17.4s, v5.4s, v12.s[0]
fmla v20.4s, v4.4s, v12.2s[1] fmla v20.4s, v4.4s, v12.s[1]
fmla v21.4s, v5.4s, v12.2s[1] fmla v21.4s, v5.4s, v12.s[1]
fmla v24.4s, v4.4s, v13.2s[0] fmla v24.4s, v4.4s, v13.s[0]
fmla v25.4s, v5.4s, v13.2s[0] fmla v25.4s, v5.4s, v13.s[0]
fmla v28.4s, v4.4s, v13.2s[1] fmla v28.4s, v4.4s, v13.s[1]
fmla v29.4s, v5.4s, v13.2s[1] fmla v29.4s, v5.4s, v13.s[1]
.endm .endm
.macro KERNEL8x4_SUB .macro KERNEL8x4_SUB
@ -441,14 +441,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v1.4s}, [pA] ld1 {v1.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.4s, v0.4s, v8.2s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v17.4s, v1.4s, v8.2s[0] fmla v17.4s, v1.4s, v8.s[0]
fmla v20.4s, v0.4s, v8.2s[1] fmla v20.4s, v0.4s, v8.s[1]
fmla v21.4s, v1.4s, v8.2s[1] fmla v21.4s, v1.4s, v8.s[1]
fmla v24.4s, v0.4s, v9.2s[0] fmla v24.4s, v0.4s, v9.s[0]
fmla v25.4s, v1.4s, v9.2s[0] fmla v25.4s, v1.4s, v9.s[0]
fmla v28.4s, v0.4s, v9.2s[1] fmla v28.4s, v0.4s, v9.s[1]
fmla v29.4s, v1.4s, v9.2s[1] fmla v29.4s, v1.4s, v9.s[1]
.endm .endm
.macro SAVE8x4 .macro SAVE8x4
@ -496,17 +496,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s, v1.2s}, [pA] ld1 {v0.2s, v1.2s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmul v16.2s, v0.2s, v8.2s[0] fmul v16.2s, v0.2s, v8.s[0]
fmul v29.2s, v1.2s, v9.2s[1] fmul v29.2s, v1.2s, v9.s[1]
fmul v20.2s, v0.2s, v8.2s[1] fmul v20.2s, v0.2s, v8.s[1]
fmul v25.2s, v1.2s, v9.2s[0] fmul v25.2s, v1.2s, v9.s[0]
fmul v24.2s, v0.2s, v9.2s[0] fmul v24.2s, v0.2s, v9.s[0]
fmul v21.2s, v1.2s, v8.2s[1] fmul v21.2s, v1.2s, v8.s[1]
fmul v28.2s, v0.2s, v9.2s[1] fmul v28.2s, v0.2s, v9.s[1]
fmul v17.2s, v1.2s, v8.2s[0] fmul v17.2s, v1.2s, v8.s[0]
ld1 {v12.2s, v13.2s}, [pB] ld1 {v12.2s, v13.2s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -515,61 +515,61 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL4x4_M1 .macro KERNEL4x4_M1
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v29.2s, v1.2s, v9.2s[1] fmla v29.2s, v1.2s, v9.s[1]
ld1 {v12.2s, v13.2s}, [pB] // For next round ld1 {v12.2s, v13.2s}, [pB] // For next round
add pB, pB, #16 add pB, pB, #16
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
fmla v25.2s, v1.2s, v9.2s[0] fmla v25.2s, v1.2s, v9.s[0]
ld1 {v4.2s, v5.2s}, [pA] // For next round ld1 {v4.2s, v5.2s}, [pA] // For next round
add pA, pA, #16 add pA, pA, #16
fmla v24.2s, v0.2s, v9.2s[0] fmla v24.2s, v0.2s, v9.s[0]
fmla v21.2s, v1.2s, v8.2s[1] fmla v21.2s, v1.2s, v8.s[1]
prfm PLDL1KEEP, [pB, #512] prfm PLDL1KEEP, [pB, #512]
fmla v28.2s, v0.2s, v9.2s[1] fmla v28.2s, v0.2s, v9.s[1]
fmla v17.2s, v1.2s, v8.2s[0] fmla v17.2s, v1.2s, v8.s[0]
.endm .endm
.macro KERNEL4x4_M2 .macro KERNEL4x4_M2
fmla v16.2s, v4.2s, v12.2s[0] fmla v16.2s, v4.2s, v12.s[0]
fmla v29.2s, v5.2s, v13.2s[1] fmla v29.2s, v5.2s, v13.s[1]
ld1 {v8.2s, v9.2s}, [pB] // For next round ld1 {v8.2s, v9.2s}, [pB] // For next round
add pB, pB, #16 add pB, pB, #16
fmla v20.2s, v4.2s, v12.2s[1] fmla v20.2s, v4.2s, v12.s[1]
fmla v25.2s, v5.2s, v13.2s[0] fmla v25.2s, v5.2s, v13.s[0]
ld1 {v0.2s, v1.2s}, [pA] // For next round ld1 {v0.2s, v1.2s}, [pA] // For next round
add pA, pA, #16 add pA, pA, #16
fmla v24.2s, v4.2s, v13.2s[0] fmla v24.2s, v4.2s, v13.s[0]
fmla v21.2s, v5.2s, v12.2s[1] fmla v21.2s, v5.2s, v12.s[1]
prfm PLDL1KEEP, [pA, #512] prfm PLDL1KEEP, [pA, #512]
fmla v28.2s, v4.2s, v13.2s[1] fmla v28.2s, v4.2s, v13.s[1]
fmla v17.2s, v5.2s, v12.2s[0] fmla v17.2s, v5.2s, v12.s[0]
.endm .endm
.macro KERNEL4x4_E .macro KERNEL4x4_E
fmla v16.2s, v4.2s, v12.2s[0] fmla v16.2s, v4.2s, v12.s[0]
fmla v29.2s, v5.2s, v13.2s[1] fmla v29.2s, v5.2s, v13.s[1]
fmla v20.2s, v4.2s, v12.2s[1] fmla v20.2s, v4.2s, v12.s[1]
fmla v25.2s, v5.2s, v13.2s[0] fmla v25.2s, v5.2s, v13.s[0]
fmla v24.2s, v4.2s, v13.2s[0] fmla v24.2s, v4.2s, v13.s[0]
fmla v21.2s, v5.2s, v12.2s[1] fmla v21.2s, v5.2s, v12.s[1]
fmla v28.2s, v4.2s, v13.2s[1] fmla v28.2s, v4.2s, v13.s[1]
fmla v17.2s, v5.2s, v12.2s[0] fmla v17.2s, v5.2s, v12.s[0]
.endm .endm
.macro KERNEL4x4_SUB .macro KERNEL4x4_SUB
@ -578,17 +578,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s, v1.2s}, [pA] ld1 {v0.2s, v1.2s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v29.2s, v1.2s, v9.2s[1] fmla v29.2s, v1.2s, v9.s[1]
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
fmla v25.2s, v1.2s, v9.2s[0] fmla v25.2s, v1.2s, v9.s[0]
fmla v24.2s, v0.2s, v9.2s[0] fmla v24.2s, v0.2s, v9.s[0]
fmla v21.2s, v1.2s, v8.2s[1] fmla v21.2s, v1.2s, v8.s[1]
fmla v28.2s, v0.2s, v9.2s[1] fmla v28.2s, v0.2s, v9.s[1]
fmla v17.2s, v1.2s, v8.2s[0] fmla v17.2s, v1.2s, v8.s[0]
.endm .endm
.macro SAVE4x4 .macro SAVE4x4
@ -633,10 +633,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s}, [pA] ld1 {v0.2s}, [pA]
add pA, pA, #8 add pA, pA, #8
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
fmla v24.2s, v0.2s, v9.2s[0] fmla v24.2s, v0.2s, v9.s[0]
fmla v28.2s, v0.2s, v9.2s[1] fmla v28.2s, v0.2s, v9.s[1]
.endm .endm
.macro SAVE2x4 .macro SAVE2x4
@ -718,15 +718,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v3.4s}, [pA] ld1 {v3.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.4s, v0.4s, v8.2s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v17.4s, v1.4s, v8.2s[0] fmla v17.4s, v1.4s, v8.s[0]
fmla v18.4s, v2.4s, v8.2s[0] fmla v18.4s, v2.4s, v8.s[0]
fmla v19.4s, v3.4s, v8.2s[0] fmla v19.4s, v3.4s, v8.s[0]
fmla v20.4s, v0.4s, v8.2s[1] fmla v20.4s, v0.4s, v8.s[1]
fmla v21.4s, v1.4s, v8.2s[1] fmla v21.4s, v1.4s, v8.s[1]
fmla v22.4s, v2.4s, v8.2s[1] fmla v22.4s, v2.4s, v8.s[1]
fmla v23.4s, v3.4s, v8.2s[1] fmla v23.4s, v3.4s, v8.s[1]
.endm .endm
.macro SAVE16x2 .macro SAVE16x2
@ -764,11 +764,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v1.4s}, [pA] ld1 {v1.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.4s, v0.4s, v8.2s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v17.4s, v1.4s, v8.2s[0] fmla v17.4s, v1.4s, v8.s[0]
fmla v20.4s, v0.4s, v8.2s[1] fmla v20.4s, v0.4s, v8.s[1]
fmla v21.4s, v1.4s, v8.2s[1] fmla v21.4s, v1.4s, v8.s[1]
.endm .endm
.macro SAVE8x2 .macro SAVE8x2
@ -802,10 +802,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s, v1.2s}, [pA] ld1 {v0.2s, v1.2s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v17.2s, v1.2s, v8.2s[0] fmla v17.2s, v1.2s, v8.s[0]
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
fmla v21.2s, v1.2s, v8.2s[1] fmla v21.2s, v1.2s, v8.s[1]
.endm .endm
.macro SAVE4x2 .macro SAVE4x2
@ -837,8 +837,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s}, [pA] ld1 {v0.2s}, [pA]
add pA, pA, #8 add pA, pA, #8
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
.endm .endm
.macro SAVE2x2 .macro SAVE2x2
@ -866,7 +866,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ldr s0 , [pA] ldr s0 , [pA]
add pA, pA, #4 add pA, pA, #4
fmla v16.2s, v8.2s, v0.2s[0] fmla v16.2s, v8.2s, v0.s[0]
.endm .endm
.macro SAVE1x2 .macro SAVE1x2
@ -901,10 +901,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v3.4s}, [pA] ld1 {v3.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.4s, v0.4s, v8.2s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v17.4s, v1.4s, v8.2s[0] fmla v17.4s, v1.4s, v8.s[0]
fmla v18.4s, v2.4s, v8.2s[0] fmla v18.4s, v2.4s, v8.s[0]
fmla v19.4s, v3.4s, v8.2s[0] fmla v19.4s, v3.4s, v8.s[0]
.endm .endm
.macro SAVE16x1 .macro SAVE16x1
@ -934,8 +934,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v1.4s}, [pA] ld1 {v1.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.4s, v0.4s, v8.2s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v17.4s, v1.4s, v8.2s[0] fmla v17.4s, v1.4s, v8.s[0]
.endm .endm
.macro SAVE8x1 .macro SAVE8x1
@ -961,8 +961,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s, v1.2s}, [pA] ld1 {v0.2s, v1.2s}, [pA]
add pA , pA, #16 add pA , pA, #16
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v17.2s, v1.2s, v8.2s[0] fmla v17.2s, v1.2s, v8.s[0]
.endm .endm
.macro SAVE4x1 .macro SAVE4x1
@ -987,7 +987,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s}, [pA] ld1 {v0.2s}, [pA]
add pA , pA, #8 add pA , pA, #8
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
.endm .endm
.macro SAVE2x1 .macro SAVE2x1

108
kernel/arm64/strmm_kernel_4x4.S
View File

@ -147,17 +147,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s, v1.2s}, [pA] ld1 {v0.2s, v1.2s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmul v16.2s, v0.2s, v8.2s[0] fmul v16.2s, v0.2s, v8.s[0]
fmul v29.2s, v1.2s, v9.2s[1] fmul v29.2s, v1.2s, v9.s[1]
fmul v20.2s, v0.2s, v8.2s[1] fmul v20.2s, v0.2s, v8.s[1]
fmul v25.2s, v1.2s, v9.2s[0] fmul v25.2s, v1.2s, v9.s[0]
fmul v24.2s, v0.2s, v9.2s[0] fmul v24.2s, v0.2s, v9.s[0]
fmul v21.2s, v1.2s, v8.2s[1] fmul v21.2s, v1.2s, v8.s[1]
fmul v28.2s, v0.2s, v9.2s[1] fmul v28.2s, v0.2s, v9.s[1]
fmul v17.2s, v1.2s, v8.2s[0] fmul v17.2s, v1.2s, v8.s[0]
ld1 {v12.2s, v13.2s}, [pB] ld1 {v12.2s, v13.2s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -166,61 +166,61 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL4x4_M1 .macro KERNEL4x4_M1
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v29.2s, v1.2s, v9.2s[1] fmla v29.2s, v1.2s, v9.s[1]
ld1 {v12.2s, v13.2s}, [pB] // For next round ld1 {v12.2s, v13.2s}, [pB] // For next round
add pB, pB, #16 add pB, pB, #16
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
fmla v25.2s, v1.2s, v9.2s[0] fmla v25.2s, v1.2s, v9.s[0]
ld1 {v4.2s, v5.2s}, [pA] // For next round ld1 {v4.2s, v5.2s}, [pA] // For next round
add pA, pA, #16 add pA, pA, #16
fmla v24.2s, v0.2s, v9.2s[0] fmla v24.2s, v0.2s, v9.s[0]
fmla v21.2s, v1.2s, v8.2s[1] fmla v21.2s, v1.2s, v8.s[1]
prfm PLDL1KEEP, [pB, #512] prfm PLDL1KEEP, [pB, #512]
fmla v28.2s, v0.2s, v9.2s[1] fmla v28.2s, v0.2s, v9.s[1]
fmla v17.2s, v1.2s, v8.2s[0] fmla v17.2s, v1.2s, v8.s[0]
.endm .endm
.macro KERNEL4x4_M2 .macro KERNEL4x4_M2
fmla v16.2s, v4.2s, v12.2s[0] fmla v16.2s, v4.2s, v12.s[0]
fmla v29.2s, v5.2s, v13.2s[1] fmla v29.2s, v5.2s, v13.s[1]
ld1 {v8.2s, v9.2s}, [pB] // For next round ld1 {v8.2s, v9.2s}, [pB] // For next round
add pB, pB, #16 add pB, pB, #16
fmla v20.2s, v4.2s, v12.2s[1] fmla v20.2s, v4.2s, v12.s[1]
fmla v25.2s, v5.2s, v13.2s[0] fmla v25.2s, v5.2s, v13.s[0]
ld1 {v0.2s, v1.2s}, [pA] // For next round ld1 {v0.2s, v1.2s}, [pA] // For next round
add pA, pA, #16 add pA, pA, #16
fmla v24.2s, v4.2s, v13.2s[0] fmla v24.2s, v4.2s, v13.s[0]
fmla v21.2s, v5.2s, v12.2s[1] fmla v21.2s, v5.2s, v12.s[1]
prfm PLDL1KEEP, [pA, #512] prfm PLDL1KEEP, [pA, #512]
fmla v28.2s, v4.2s, v13.2s[1] fmla v28.2s, v4.2s, v13.s[1]
fmla v17.2s, v5.2s, v12.2s[0] fmla v17.2s, v5.2s, v12.s[0]
.endm .endm
.macro KERNEL4x4_E .macro KERNEL4x4_E
fmla v16.2s, v4.2s, v12.2s[0] fmla v16.2s, v4.2s, v12.s[0]
fmla v29.2s, v5.2s, v13.2s[1] fmla v29.2s, v5.2s, v13.s[1]
fmla v20.2s, v4.2s, v12.2s[1] fmla v20.2s, v4.2s, v12.s[1]
fmla v25.2s, v5.2s, v13.2s[0] fmla v25.2s, v5.2s, v13.s[0]
fmla v24.2s, v4.2s, v13.2s[0] fmla v24.2s, v4.2s, v13.s[0]
fmla v21.2s, v5.2s, v12.2s[1] fmla v21.2s, v5.2s, v12.s[1]
fmla v28.2s, v4.2s, v13.2s[1] fmla v28.2s, v4.2s, v13.s[1]
fmla v17.2s, v5.2s, v12.2s[0] fmla v17.2s, v5.2s, v12.s[0]
.endm .endm
.macro KERNEL4x4_SUB .macro KERNEL4x4_SUB
@ -229,17 +229,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s, v1.2s}, [pA] ld1 {v0.2s, v1.2s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v29.2s, v1.2s, v9.2s[1] fmla v29.2s, v1.2s, v9.s[1]
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
fmla v25.2s, v1.2s, v9.2s[0] fmla v25.2s, v1.2s, v9.s[0]
fmla v24.2s, v0.2s, v9.2s[0] fmla v24.2s, v0.2s, v9.s[0]
fmla v21.2s, v1.2s, v8.2s[1] fmla v21.2s, v1.2s, v8.s[1]
fmla v28.2s, v0.2s, v9.2s[1] fmla v28.2s, v0.2s, v9.s[1]
fmla v17.2s, v1.2s, v8.2s[0] fmla v17.2s, v1.2s, v8.s[0]
.endm .endm
.macro SAVE4x4 .macro SAVE4x4
@ -280,10 +280,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s}, [pA] ld1 {v0.2s}, [pA]
add pA, pA, #8 add pA, pA, #8
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
fmla v24.2s, v0.2s, v9.2s[0] fmla v24.2s, v0.2s, v9.s[0]
fmla v28.2s, v0.2s, v9.2s[1] fmla v28.2s, v0.2s, v9.s[1]
.endm .endm
.macro SAVE2x4 .macro SAVE2x4
@ -353,10 +353,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s, v1.2s}, [pA] ld1 {v0.2s, v1.2s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v17.2s, v1.2s, v8.2s[0] fmla v17.2s, v1.2s, v8.s[0]
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
fmla v21.2s, v1.2s, v8.2s[1] fmla v21.2s, v1.2s, v8.s[1]
.endm .endm
.macro SAVE4x2 .macro SAVE4x2
@ -386,8 +386,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s}, [pA] ld1 {v0.2s}, [pA]
add pA, pA, #8 add pA, pA, #8
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
.endm .endm
.macro SAVE2x2 .macro SAVE2x2
@ -414,7 +414,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ldr s0 , [pA] ldr s0 , [pA]
add pA, pA, #4 add pA, pA, #4
fmla v16.2s, v8.2s, v0.2s[0] fmla v16.2s, v8.2s, v0.s[0]
.endm .endm
.macro SAVE1x2 .macro SAVE1x2
@ -440,8 +440,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s, v1.2s}, [pA] ld1 {v0.2s, v1.2s}, [pA]
add pA , pA, #16 add pA , pA, #16
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v17.2s, v1.2s, v8.2s[0] fmla v17.2s, v1.2s, v8.s[0]
.endm .endm
.macro SAVE4x1 .macro SAVE4x1
@ -468,7 +468,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s}, [pA] ld1 {v0.2s}, [pA]
add pA , pA, #8 add pA , pA, #8
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
.endm .endm
.macro SAVE2x1 .macro SAVE2x1

472
kernel/arm64/strmm_kernel_8x8.S Executable file → Normal file
View File

@ -159,22 +159,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v1.4s}, [pA] ld1 {v1.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmul v16.4s, v0.4s, v4.4s[0] fmul v16.4s, v0.4s, v4.s[0]
fmul v17.4s, v1.4s, v4.4s[0] fmul v17.4s, v1.4s, v4.s[0]
fmul v18.4s, v0.4s, v4.4s[1] fmul v18.4s, v0.4s, v4.s[1]
fmul v19.4s, v1.4s, v4.4s[1] fmul v19.4s, v1.4s, v4.s[1]
fmul v20.4s, v0.4s, v4.4s[2] fmul v20.4s, v0.4s, v4.s[2]
fmul v21.4s, v1.4s, v4.4s[2] fmul v21.4s, v1.4s, v4.s[2]
fmul v22.4s, v0.4s, v4.4s[3] fmul v22.4s, v0.4s, v4.s[3]
fmul v23.4s, v1.4s, v4.4s[3] fmul v23.4s, v1.4s, v4.s[3]
fmul v24.4s, v0.4s, v5.4s[0] fmul v24.4s, v0.4s, v5.s[0]
fmul v25.4s, v1.4s, v5.4s[0] fmul v25.4s, v1.4s, v5.s[0]
fmul v26.4s, v0.4s, v5.4s[1] fmul v26.4s, v0.4s, v5.s[1]
fmul v27.4s, v1.4s, v5.4s[1] fmul v27.4s, v1.4s, v5.s[1]
fmul v28.4s, v0.4s, v5.4s[2] fmul v28.4s, v0.4s, v5.s[2]
fmul v29.4s, v1.4s, v5.4s[2] fmul v29.4s, v1.4s, v5.s[2]
fmul v30.4s, v0.4s, v5.4s[3] fmul v30.4s, v0.4s, v5.s[3]
fmul v31.4s, v1.4s, v5.4s[3] fmul v31.4s, v1.4s, v5.s[3]
ld1 {v6.4s}, [pB] ld1 {v6.4s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -187,22 +187,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x8_M1 .macro KERNEL8x8_M1
fmla v16.4s, v0.4s, v4.4s[0] fmla v16.4s, v0.4s, v4.s[0]
fmla v17.4s, v1.4s, v4.4s[0] fmla v17.4s, v1.4s, v4.s[0]
fmla v18.4s, v0.4s, v4.4s[1] fmla v18.4s, v0.4s, v4.s[1]
fmla v19.4s, v1.4s, v4.4s[1] fmla v19.4s, v1.4s, v4.s[1]
fmla v20.4s, v0.4s, v4.4s[2] fmla v20.4s, v0.4s, v4.s[2]
fmla v21.4s, v1.4s, v4.4s[2] fmla v21.4s, v1.4s, v4.s[2]
fmla v22.4s, v0.4s, v4.4s[3] fmla v22.4s, v0.4s, v4.s[3]
fmla v23.4s, v1.4s, v4.4s[3] fmla v23.4s, v1.4s, v4.s[3]
fmla v24.4s, v0.4s, v5.4s[0] fmla v24.4s, v0.4s, v5.s[0]
fmla v25.4s, v1.4s, v5.4s[0] fmla v25.4s, v1.4s, v5.s[0]
fmla v26.4s, v0.4s, v5.4s[1] fmla v26.4s, v0.4s, v5.s[1]
fmla v27.4s, v1.4s, v5.4s[1] fmla v27.4s, v1.4s, v5.s[1]
fmla v28.4s, v0.4s, v5.4s[2] fmla v28.4s, v0.4s, v5.s[2]
fmla v29.4s, v1.4s, v5.4s[2] fmla v29.4s, v1.4s, v5.s[2]
fmla v30.4s, v0.4s, v5.4s[3] fmla v30.4s, v0.4s, v5.s[3]
fmla v31.4s, v1.4s, v5.4s[3] fmla v31.4s, v1.4s, v5.s[3]
ld1 {v6.4s}, [pB] ld1 {v6.4s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -215,22 +215,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x8_M2 .macro KERNEL8x8_M2
fmla v16.4s, v2.4s, v6.4s[0] fmla v16.4s, v2.4s, v6.s[0]
fmla v17.4s, v3.4s, v6.4s[0] fmla v17.4s, v3.4s, v6.s[0]
fmla v18.4s, v2.4s, v6.4s[1] fmla v18.4s, v2.4s, v6.s[1]
fmla v19.4s, v3.4s, v6.4s[1] fmla v19.4s, v3.4s, v6.s[1]
fmla v20.4s, v2.4s, v6.4s[2] fmla v20.4s, v2.4s, v6.s[2]
fmla v21.4s, v3.4s, v6.4s[2] fmla v21.4s, v3.4s, v6.s[2]
fmla v22.4s, v2.4s, v6.4s[3] fmla v22.4s, v2.4s, v6.s[3]
fmla v23.4s, v3.4s, v6.4s[3] fmla v23.4s, v3.4s, v6.s[3]
fmla v24.4s, v2.4s, v7.4s[0] fmla v24.4s, v2.4s, v7.s[0]
fmla v25.4s, v3.4s, v7.4s[0] fmla v25.4s, v3.4s, v7.s[0]
fmla v26.4s, v2.4s, v7.4s[1] fmla v26.4s, v2.4s, v7.s[1]
fmla v27.4s, v3.4s, v7.4s[1] fmla v27.4s, v3.4s, v7.s[1]
fmla v28.4s, v2.4s, v7.4s[2] fmla v28.4s, v2.4s, v7.s[2]
fmla v29.4s, v3.4s, v7.4s[2] fmla v29.4s, v3.4s, v7.s[2]
fmla v30.4s, v2.4s, v7.4s[3] fmla v30.4s, v2.4s, v7.s[3]
fmla v31.4s, v3.4s, v7.4s[3] fmla v31.4s, v3.4s, v7.s[3]
ld1 {v4.4s}, [pB] ld1 {v4.4s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -243,22 +243,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x8_E .macro KERNEL8x8_E
fmla v16.4s, v2.4s, v6.4s[0] fmla v16.4s, v2.4s, v6.s[0]
fmla v17.4s, v3.4s, v6.4s[0] fmla v17.4s, v3.4s, v6.s[0]
fmla v18.4s, v2.4s, v6.4s[1] fmla v18.4s, v2.4s, v6.s[1]
fmla v19.4s, v3.4s, v6.4s[1] fmla v19.4s, v3.4s, v6.s[1]
fmla v20.4s, v2.4s, v6.4s[2] fmla v20.4s, v2.4s, v6.s[2]
fmla v21.4s, v3.4s, v6.4s[2] fmla v21.4s, v3.4s, v6.s[2]
fmla v22.4s, v2.4s, v6.4s[3] fmla v22.4s, v2.4s, v6.s[3]
fmla v23.4s, v3.4s, v6.4s[3] fmla v23.4s, v3.4s, v6.s[3]
fmla v24.4s, v2.4s, v7.4s[0] fmla v24.4s, v2.4s, v7.s[0]
fmla v25.4s, v3.4s, v7.4s[0] fmla v25.4s, v3.4s, v7.s[0]
fmla v26.4s, v2.4s, v7.4s[1] fmla v26.4s, v2.4s, v7.s[1]
fmla v27.4s, v3.4s, v7.4s[1] fmla v27.4s, v3.4s, v7.s[1]
fmla v28.4s, v2.4s, v7.4s[2] fmla v28.4s, v2.4s, v7.s[2]
fmla v29.4s, v3.4s, v7.4s[2] fmla v29.4s, v3.4s, v7.s[2]
fmla v30.4s, v2.4s, v7.4s[3] fmla v30.4s, v2.4s, v7.s[3]
fmla v31.4s, v3.4s, v7.4s[3] fmla v31.4s, v3.4s, v7.s[3]
.endm .endm
.macro KERNEL8x8_SUB .macro KERNEL8x8_SUB
@ -271,22 +271,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v1.4s}, [pA] ld1 {v1.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.4s, v0.4s, v4.4s[0] fmla v16.4s, v0.4s, v4.s[0]
fmla v17.4s, v1.4s, v4.4s[0] fmla v17.4s, v1.4s, v4.s[0]
fmla v18.4s, v0.4s, v4.4s[1] fmla v18.4s, v0.4s, v4.s[1]
fmla v19.4s, v1.4s, v4.4s[1] fmla v19.4s, v1.4s, v4.s[1]
fmla v20.4s, v0.4s, v4.4s[2] fmla v20.4s, v0.4s, v4.s[2]
fmla v21.4s, v1.4s, v4.4s[2] fmla v21.4s, v1.4s, v4.s[2]
fmla v22.4s, v0.4s, v4.4s[3] fmla v22.4s, v0.4s, v4.s[3]
fmla v23.4s, v1.4s, v4.4s[3] fmla v23.4s, v1.4s, v4.s[3]
fmla v24.4s, v0.4s, v5.4s[0] fmla v24.4s, v0.4s, v5.s[0]
fmla v25.4s, v1.4s, v5.4s[0] fmla v25.4s, v1.4s, v5.s[0]
fmla v26.4s, v0.4s, v5.4s[1] fmla v26.4s, v0.4s, v5.s[1]
fmla v27.4s, v1.4s, v5.4s[1] fmla v27.4s, v1.4s, v5.s[1]
fmla v28.4s, v0.4s, v5.4s[2] fmla v28.4s, v0.4s, v5.s[2]
fmla v29.4s, v1.4s, v5.4s[2] fmla v29.4s, v1.4s, v5.s[2]
fmla v30.4s, v0.4s, v5.4s[3] fmla v30.4s, v0.4s, v5.s[3]
fmla v31.4s, v1.4s, v5.4s[3] fmla v31.4s, v1.4s, v5.s[3]
.endm .endm
.macro SAVE8x8 .macro SAVE8x8
@ -361,14 +361,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.4s}, [pA] ld1 {v0.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmul v16.4s, v0.4s, v4.4s[0] fmul v16.4s, v0.4s, v4.s[0]
fmul v18.4s, v0.4s, v4.4s[1] fmul v18.4s, v0.4s, v4.s[1]
fmul v20.4s, v0.4s, v4.4s[2] fmul v20.4s, v0.4s, v4.s[2]
fmul v22.4s, v0.4s, v4.4s[3] fmul v22.4s, v0.4s, v4.s[3]
fmul v24.4s, v0.4s, v5.4s[0] fmul v24.4s, v0.4s, v5.s[0]
fmul v26.4s, v0.4s, v5.4s[1] fmul v26.4s, v0.4s, v5.s[1]
fmul v28.4s, v0.4s, v5.4s[2] fmul v28.4s, v0.4s, v5.s[2]
fmul v30.4s, v0.4s, v5.4s[3] fmul v30.4s, v0.4s, v5.s[3]
ld1 {v6.4s}, [pB] ld1 {v6.4s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -379,14 +379,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL4x8_M1 .macro KERNEL4x8_M1
fmla v16.4s, v0.4s, v4.4s[0] fmla v16.4s, v0.4s, v4.s[0]
fmla v18.4s, v0.4s, v4.4s[1] fmla v18.4s, v0.4s, v4.s[1]
fmla v20.4s, v0.4s, v4.4s[2] fmla v20.4s, v0.4s, v4.s[2]
fmla v22.4s, v0.4s, v4.4s[3] fmla v22.4s, v0.4s, v4.s[3]
fmla v24.4s, v0.4s, v5.4s[0] fmla v24.4s, v0.4s, v5.s[0]
fmla v26.4s, v0.4s, v5.4s[1] fmla v26.4s, v0.4s, v5.s[1]
fmla v28.4s, v0.4s, v5.4s[2] fmla v28.4s, v0.4s, v5.s[2]
fmla v30.4s, v0.4s, v5.4s[3] fmla v30.4s, v0.4s, v5.s[3]
ld1 {v6.4s}, [pB] ld1 {v6.4s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -397,14 +397,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL4x8_M2 .macro KERNEL4x8_M2
fmla v16.4s, v2.4s, v6.4s[0] fmla v16.4s, v2.4s, v6.s[0]
fmla v18.4s, v2.4s, v6.4s[1] fmla v18.4s, v2.4s, v6.s[1]
fmla v20.4s, v2.4s, v6.4s[2] fmla v20.4s, v2.4s, v6.s[2]
fmla v22.4s, v2.4s, v6.4s[3] fmla v22.4s, v2.4s, v6.s[3]
fmla v24.4s, v2.4s, v7.4s[0] fmla v24.4s, v2.4s, v7.s[0]
fmla v26.4s, v2.4s, v7.4s[1] fmla v26.4s, v2.4s, v7.s[1]
fmla v28.4s, v2.4s, v7.4s[2] fmla v28.4s, v2.4s, v7.s[2]
fmla v30.4s, v2.4s, v7.4s[3] fmla v30.4s, v2.4s, v7.s[3]
ld1 {v4.4s}, [pB] ld1 {v4.4s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -415,14 +415,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL4x8_E .macro KERNEL4x8_E
fmla v16.4s, v2.4s, v6.4s[0] fmla v16.4s, v2.4s, v6.s[0]
fmla v18.4s, v2.4s, v6.4s[1] fmla v18.4s, v2.4s, v6.s[1]
fmla v20.4s, v2.4s, v6.4s[2] fmla v20.4s, v2.4s, v6.s[2]
fmla v22.4s, v2.4s, v6.4s[3] fmla v22.4s, v2.4s, v6.s[3]
fmla v24.4s, v2.4s, v7.4s[0] fmla v24.4s, v2.4s, v7.s[0]
fmla v26.4s, v2.4s, v7.4s[1] fmla v26.4s, v2.4s, v7.s[1]
fmla v28.4s, v2.4s, v7.4s[2] fmla v28.4s, v2.4s, v7.s[2]
fmla v30.4s, v2.4s, v7.4s[3] fmla v30.4s, v2.4s, v7.s[3]
.endm .endm
.macro KERNEL4x8_SUB .macro KERNEL4x8_SUB
@ -433,14 +433,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.4s}, [pA] ld1 {v0.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.4s, v0.4s, v4.4s[0] fmla v16.4s, v0.4s, v4.s[0]
fmla v18.4s, v0.4s, v4.4s[1] fmla v18.4s, v0.4s, v4.s[1]
fmla v20.4s, v0.4s, v4.4s[2] fmla v20.4s, v0.4s, v4.s[2]
fmla v22.4s, v0.4s, v4.4s[3] fmla v22.4s, v0.4s, v4.s[3]
fmla v24.4s, v0.4s, v5.4s[0] fmla v24.4s, v0.4s, v5.s[0]
fmla v26.4s, v0.4s, v5.4s[1] fmla v26.4s, v0.4s, v5.s[1]
fmla v28.4s, v0.4s, v5.4s[2] fmla v28.4s, v0.4s, v5.s[2]
fmla v30.4s, v0.4s, v5.4s[3] fmla v30.4s, v0.4s, v5.s[3]
.endm .endm
.macro SAVE4x8 .macro SAVE4x8
@ -514,14 +514,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s}, [pA] ld1 {v0.2s}, [pA]
add pA, pA, #8 add pA, pA, #8
fmla v16.2s, v0.2s, v4.4s[0] fmla v16.2s, v0.2s, v4.s[0]
fmla v18.2s, v0.2s, v4.4s[1] fmla v18.2s, v0.2s, v4.s[1]
fmla v20.2s, v0.2s, v4.4s[2] fmla v20.2s, v0.2s, v4.s[2]
fmla v22.2s, v0.2s, v4.4s[3] fmla v22.2s, v0.2s, v4.s[3]
fmla v24.2s, v0.2s, v5.4s[0] fmla v24.2s, v0.2s, v5.s[0]
fmla v26.2s, v0.2s, v5.4s[1] fmla v26.2s, v0.2s, v5.s[1]
fmla v28.2s, v0.2s, v5.4s[2] fmla v28.2s, v0.2s, v5.s[2]
fmla v30.2s, v0.2s, v5.4s[3] fmla v30.2s, v0.2s, v5.s[3]
.endm .endm
.macro SAVE2x8 .macro SAVE2x8
@ -595,14 +595,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ldr s0, [pA] ldr s0, [pA]
add pA, pA, #4 add pA, pA, #4
fmla s16, s0, v4.4s[0] fmla s16, s0, v4.s[0]
fmla s18, s0, v4.4s[1] fmla s18, s0, v4.s[1]
fmla s20, s0, v4.4s[2] fmla s20, s0, v4.s[2]
fmla s22, s0, v4.4s[3] fmla s22, s0, v4.s[3]
fmla s24, s0, v5.4s[0] fmla s24, s0, v5.s[0]
fmla s26, s0, v5.4s[1] fmla s26, s0, v5.s[1]
fmla s28, s0, v5.4s[2] fmla s28, s0, v5.s[2]
fmla s30, s0, v5.4s[3] fmla s30, s0, v5.s[3]
.endm .endm
.macro SAVE1x8 .macro SAVE1x8
@ -676,14 +676,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v1.4s}, [pA] ld1 {v1.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmul v16.4s, v0.4s, v8.2s[0] fmul v16.4s, v0.4s, v8.s[0]
fmul v17.4s, v1.4s, v8.2s[0] fmul v17.4s, v1.4s, v8.s[0]
fmul v20.4s, v0.4s, v8.2s[1] fmul v20.4s, v0.4s, v8.s[1]
fmul v21.4s, v1.4s, v8.2s[1] fmul v21.4s, v1.4s, v8.s[1]
fmul v24.4s, v0.4s, v9.2s[0] fmul v24.4s, v0.4s, v9.s[0]
fmul v25.4s, v1.4s, v9.2s[0] fmul v25.4s, v1.4s, v9.s[0]
fmul v28.4s, v0.4s, v9.2s[1] fmul v28.4s, v0.4s, v9.s[1]
fmul v29.4s, v1.4s, v9.2s[1] fmul v29.4s, v1.4s, v9.s[1]
ld1 {v12.2s, v13.2s}, [pB] ld1 {v12.2s, v13.2s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -694,14 +694,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x4_M1 .macro KERNEL8x4_M1
fmla v16.4s, v0.4s, v8.2s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v17.4s, v1.4s, v8.2s[0] fmla v17.4s, v1.4s, v8.s[0]
fmla v20.4s, v0.4s, v8.2s[1] fmla v20.4s, v0.4s, v8.s[1]
fmla v21.4s, v1.4s, v8.2s[1] fmla v21.4s, v1.4s, v8.s[1]
fmla v24.4s, v0.4s, v9.2s[0] fmla v24.4s, v0.4s, v9.s[0]
fmla v25.4s, v1.4s, v9.2s[0] fmla v25.4s, v1.4s, v9.s[0]
fmla v28.4s, v0.4s, v9.2s[1] fmla v28.4s, v0.4s, v9.s[1]
fmla v29.4s, v1.4s, v9.2s[1] fmla v29.4s, v1.4s, v9.s[1]
ld1 {v12.2s, v13.2s}, [pB] ld1 {v12.2s, v13.2s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -712,14 +712,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x4_M2 .macro KERNEL8x4_M2
fmla v16.4s, v4.4s, v12.2s[0] fmla v16.4s, v4.4s, v12.s[0]
fmla v17.4s, v5.4s, v12.2s[0] fmla v17.4s, v5.4s, v12.s[0]
fmla v20.4s, v4.4s, v12.2s[1] fmla v20.4s, v4.4s, v12.s[1]
fmla v21.4s, v5.4s, v12.2s[1] fmla v21.4s, v5.4s, v12.s[1]
fmla v24.4s, v4.4s, v13.2s[0] fmla v24.4s, v4.4s, v13.s[0]
fmla v25.4s, v5.4s, v13.2s[0] fmla v25.4s, v5.4s, v13.s[0]
fmla v28.4s, v4.4s, v13.2s[1] fmla v28.4s, v4.4s, v13.s[1]
fmla v29.4s, v5.4s, v13.2s[1] fmla v29.4s, v5.4s, v13.s[1]
ld1 {v8.2s, v9.2s}, [pB] ld1 {v8.2s, v9.2s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -730,14 +730,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL8x4_E .macro KERNEL8x4_E
fmla v16.4s, v4.4s, v12.2s[0] fmla v16.4s, v4.4s, v12.s[0]
fmla v17.4s, v5.4s, v12.2s[0] fmla v17.4s, v5.4s, v12.s[0]
fmla v20.4s, v4.4s, v12.2s[1] fmla v20.4s, v4.4s, v12.s[1]
fmla v21.4s, v5.4s, v12.2s[1] fmla v21.4s, v5.4s, v12.s[1]
fmla v24.4s, v4.4s, v13.2s[0] fmla v24.4s, v4.4s, v13.s[0]
fmla v25.4s, v5.4s, v13.2s[0] fmla v25.4s, v5.4s, v13.s[0]
fmla v28.4s, v4.4s, v13.2s[1] fmla v28.4s, v4.4s, v13.s[1]
fmla v29.4s, v5.4s, v13.2s[1] fmla v29.4s, v5.4s, v13.s[1]
.endm .endm
.macro KERNEL8x4_SUB .macro KERNEL8x4_SUB
@ -748,14 +748,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v1.4s}, [pA] ld1 {v1.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.4s, v0.4s, v8.2s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v17.4s, v1.4s, v8.2s[0] fmla v17.4s, v1.4s, v8.s[0]
fmla v20.4s, v0.4s, v8.2s[1] fmla v20.4s, v0.4s, v8.s[1]
fmla v21.4s, v1.4s, v8.2s[1] fmla v21.4s, v1.4s, v8.s[1]
fmla v24.4s, v0.4s, v9.2s[0] fmla v24.4s, v0.4s, v9.s[0]
fmla v25.4s, v1.4s, v9.2s[0] fmla v25.4s, v1.4s, v9.s[0]
fmla v28.4s, v0.4s, v9.2s[1] fmla v28.4s, v0.4s, v9.s[1]
fmla v29.4s, v1.4s, v9.2s[1] fmla v29.4s, v1.4s, v9.s[1]
.endm .endm
.macro SAVE8x4 .macro SAVE8x4
@ -808,17 +808,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s, v1.2s}, [pA] ld1 {v0.2s, v1.2s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmul v16.2s, v0.2s, v8.2s[0] fmul v16.2s, v0.2s, v8.s[0]
fmul v29.2s, v1.2s, v9.2s[1] fmul v29.2s, v1.2s, v9.s[1]
fmul v20.2s, v0.2s, v8.2s[1] fmul v20.2s, v0.2s, v8.s[1]
fmul v25.2s, v1.2s, v9.2s[0] fmul v25.2s, v1.2s, v9.s[0]
fmul v24.2s, v0.2s, v9.2s[0] fmul v24.2s, v0.2s, v9.s[0]
fmul v21.2s, v1.2s, v8.2s[1] fmul v21.2s, v1.2s, v8.s[1]
fmul v28.2s, v0.2s, v9.2s[1] fmul v28.2s, v0.2s, v9.s[1]
fmul v17.2s, v1.2s, v8.2s[0] fmul v17.2s, v1.2s, v8.s[0]
ld1 {v12.2s, v13.2s}, [pB] ld1 {v12.2s, v13.2s}, [pB]
add pB, pB, #16 add pB, pB, #16
@ -827,61 +827,61 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL4x4_M1 .macro KERNEL4x4_M1
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v29.2s, v1.2s, v9.2s[1] fmla v29.2s, v1.2s, v9.s[1]
ld1 {v12.2s, v13.2s}, [pB] // For next round ld1 {v12.2s, v13.2s}, [pB] // For next round
add pB, pB, #16 add pB, pB, #16
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
fmla v25.2s, v1.2s, v9.2s[0] fmla v25.2s, v1.2s, v9.s[0]
ld1 {v4.2s, v5.2s}, [pA] // For next round ld1 {v4.2s, v5.2s}, [pA] // For next round
add pA, pA, #16 add pA, pA, #16
fmla v24.2s, v0.2s, v9.2s[0] fmla v24.2s, v0.2s, v9.s[0]
fmla v21.2s, v1.2s, v8.2s[1] fmla v21.2s, v1.2s, v8.s[1]
prfm PLDL1KEEP, [pB, #512] prfm PLDL1KEEP, [pB, #512]
fmla v28.2s, v0.2s, v9.2s[1] fmla v28.2s, v0.2s, v9.s[1]
fmla v17.2s, v1.2s, v8.2s[0] fmla v17.2s, v1.2s, v8.s[0]
.endm .endm
.macro KERNEL4x4_M2 .macro KERNEL4x4_M2
fmla v16.2s, v4.2s, v12.2s[0] fmla v16.2s, v4.2s, v12.s[0]
fmla v29.2s, v5.2s, v13.2s[1] fmla v29.2s, v5.2s, v13.s[1]
ld1 {v8.2s, v9.2s}, [pB] // For next round ld1 {v8.2s, v9.2s}, [pB] // For next round
add pB, pB, #16 add pB, pB, #16
fmla v20.2s, v4.2s, v12.2s[1] fmla v20.2s, v4.2s, v12.s[1]
fmla v25.2s, v5.2s, v13.2s[0] fmla v25.2s, v5.2s, v13.s[0]
ld1 {v0.2s, v1.2s}, [pA] // For next round ld1 {v0.2s, v1.2s}, [pA] // For next round
add pA, pA, #16 add pA, pA, #16
fmla v24.2s, v4.2s, v13.2s[0] fmla v24.2s, v4.2s, v13.s[0]
fmla v21.2s, v5.2s, v12.2s[1] fmla v21.2s, v5.2s, v12.s[1]
prfm PLDL1KEEP, [pA, #512] prfm PLDL1KEEP, [pA, #512]
fmla v28.2s, v4.2s, v13.2s[1] fmla v28.2s, v4.2s, v13.s[1]
fmla v17.2s, v5.2s, v12.2s[0] fmla v17.2s, v5.2s, v12.s[0]
.endm .endm
.macro KERNEL4x4_E .macro KERNEL4x4_E
fmla v16.2s, v4.2s, v12.2s[0] fmla v16.2s, v4.2s, v12.s[0]
fmla v29.2s, v5.2s, v13.2s[1] fmla v29.2s, v5.2s, v13.s[1]
fmla v20.2s, v4.2s, v12.2s[1] fmla v20.2s, v4.2s, v12.s[1]
fmla v25.2s, v5.2s, v13.2s[0] fmla v25.2s, v5.2s, v13.s[0]
fmla v24.2s, v4.2s, v13.2s[0] fmla v24.2s, v4.2s, v13.s[0]
fmla v21.2s, v5.2s, v12.2s[1] fmla v21.2s, v5.2s, v12.s[1]
fmla v28.2s, v4.2s, v13.2s[1] fmla v28.2s, v4.2s, v13.s[1]
fmla v17.2s, v5.2s, v12.2s[0] fmla v17.2s, v5.2s, v12.s[0]
.endm .endm
.macro KERNEL4x4_SUB .macro KERNEL4x4_SUB
@ -890,17 +890,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s, v1.2s}, [pA] ld1 {v0.2s, v1.2s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v29.2s, v1.2s, v9.2s[1] fmla v29.2s, v1.2s, v9.s[1]
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
fmla v25.2s, v1.2s, v9.2s[0] fmla v25.2s, v1.2s, v9.s[0]
fmla v24.2s, v0.2s, v9.2s[0] fmla v24.2s, v0.2s, v9.s[0]
fmla v21.2s, v1.2s, v8.2s[1] fmla v21.2s, v1.2s, v8.s[1]
fmla v28.2s, v0.2s, v9.2s[1] fmla v28.2s, v0.2s, v9.s[1]
fmla v17.2s, v1.2s, v8.2s[0] fmla v17.2s, v1.2s, v8.s[0]
.endm .endm
.macro SAVE4x4 .macro SAVE4x4
@ -945,10 +945,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s}, [pA] ld1 {v0.2s}, [pA]
add pA, pA, #8 add pA, pA, #8
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
fmla v24.2s, v0.2s, v9.2s[0] fmla v24.2s, v0.2s, v9.s[0]
fmla v28.2s, v0.2s, v9.2s[1] fmla v28.2s, v0.2s, v9.s[1]
.endm .endm
.macro SAVE2x4 .macro SAVE2x4
@ -1028,11 +1028,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v1.4s}, [pA] ld1 {v1.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.4s, v0.4s, v8.2s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v17.4s, v1.4s, v8.2s[0] fmla v17.4s, v1.4s, v8.s[0]
fmla v20.4s, v0.4s, v8.2s[1] fmla v20.4s, v0.4s, v8.s[1]
fmla v21.4s, v1.4s, v8.2s[1] fmla v21.4s, v1.4s, v8.s[1]
.endm .endm
.macro SAVE8x2 .macro SAVE8x2
@ -1068,10 +1068,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s, v1.2s}, [pA] ld1 {v0.2s, v1.2s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v17.2s, v1.2s, v8.2s[0] fmla v17.2s, v1.2s, v8.s[0]
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
fmla v21.2s, v1.2s, v8.2s[1] fmla v21.2s, v1.2s, v8.s[1]
.endm .endm
.macro SAVE4x2 .macro SAVE4x2
@ -1103,8 +1103,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s}, [pA] ld1 {v0.2s}, [pA]
add pA, pA, #8 add pA, pA, #8
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v20.2s, v0.2s, v8.2s[1] fmla v20.2s, v0.2s, v8.s[1]
.endm .endm
.macro SAVE2x2 .macro SAVE2x2
@ -1133,7 +1133,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ldr s0 , [pA] ldr s0 , [pA]
add pA, pA, #4 add pA, pA, #4
fmla v16.2s, v8.2s, v0.2s[0] fmla v16.2s, v8.2s, v0.s[0]
.endm .endm
.macro SAVE1x2 .macro SAVE1x2
@ -1163,8 +1163,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v1.4s}, [pA] ld1 {v1.4s}, [pA]
add pA, pA, #16 add pA, pA, #16
fmla v16.4s, v0.4s, v8.2s[0] fmla v16.4s, v0.4s, v8.s[0]
fmla v17.4s, v1.4s, v8.2s[0] fmla v17.4s, v1.4s, v8.s[0]
.endm .endm
.macro SAVE8x1 .macro SAVE8x1
@ -1190,8 +1190,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s, v1.2s}, [pA] ld1 {v0.2s, v1.2s}, [pA]
add pA , pA, #16 add pA , pA, #16
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
fmla v17.2s, v1.2s, v8.2s[0] fmla v17.2s, v1.2s, v8.s[0]
.endm .endm
.macro SAVE4x1 .macro SAVE4x1
@ -1216,7 +1216,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld1 {v0.2s}, [pA] ld1 {v0.2s}, [pA]
add pA , pA, #8 add pA , pA, #8
fmla v16.2s, v0.2s, v8.2s[0] fmla v16.2s, v0.2s, v8.s[0]
.endm .endm
.macro SAVE2x1 .macro SAVE2x1

464
kernel/arm64/zgemm_kernel_4x4.S
View File

@ -182,93 +182,93 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v2.2d, v3.2d}, [pA] ld2 {v2.2d, v3.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
fmul v16.2d, v0.2d, v8.2d[0] fmul v16.2d, v0.2d, v8.d[0]
OP_ii v16.2d, v1.2d, v9.2d[0] OP_ii v16.2d, v1.2d, v9.d[0]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v17.16b, v17.16b, v17.16b eor v17.16b, v17.16b, v17.16b
fmls v17.2d, v0.2d, v9.2d[0] fmls v17.2d, v0.2d, v9.d[0]
#else #else
fmul v17.2d, v0.2d, v9.2d[0] fmul v17.2d, v0.2d, v9.d[0]
#endif #endif
OP_ir v17.2d, v1.2d, v8.2d[0] OP_ir v17.2d, v1.2d, v8.d[0]
fmul v18.2d, v2.2d, v8.2d[0] fmul v18.2d, v2.2d, v8.d[0]
OP_ii v18.2d, v3.2d, v9.2d[0] OP_ii v18.2d, v3.2d, v9.d[0]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v19.16b, v19.16b, v19.16b eor v19.16b, v19.16b, v19.16b
fmls v19.2d, v2.2d, v9.2d[0] fmls v19.2d, v2.2d, v9.d[0]
#else #else
fmul v19.2d, v2.2d, v9.2d[0] fmul v19.2d, v2.2d, v9.d[0]
#endif #endif
OP_ir v19.2d, v3.2d, v8.2d[0] OP_ir v19.2d, v3.2d, v8.d[0]
fmul v20.2d, v0.2d, v8.2d[1] fmul v20.2d, v0.2d, v8.d[1]
OP_ii v20.2d, v1.2d, v9.2d[1] OP_ii v20.2d, v1.2d, v9.d[1]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v21.16b, v21.16b, v21.16b eor v21.16b, v21.16b, v21.16b
fmls v21.2d, v0.2d, v9.2d[1] fmls v21.2d, v0.2d, v9.d[1]
#else #else
fmul v21.2d, v0.2d, v9.2d[1] fmul v21.2d, v0.2d, v9.d[1]
#endif #endif
OP_ir v21.2d, v1.2d, v8.2d[1] OP_ir v21.2d, v1.2d, v8.d[1]
fmul v22.2d, v2.2d, v8.2d[1] fmul v22.2d, v2.2d, v8.d[1]
OP_ii v22.2d, v3.2d, v9.2d[1] OP_ii v22.2d, v3.2d, v9.d[1]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v23.16b, v23.16b, v23.16b eor v23.16b, v23.16b, v23.16b
fmls v23.2d, v2.2d, v9.2d[1] fmls v23.2d, v2.2d, v9.d[1]
#else #else
fmul v23.2d, v2.2d, v9.2d[1] fmul v23.2d, v2.2d, v9.d[1]
#endif #endif
OP_ir v23.2d, v3.2d, v8.2d[1] OP_ir v23.2d, v3.2d, v8.d[1]
fmul v24.2d, v0.2d, v10.2d[0] fmul v24.2d, v0.2d, v10.d[0]
OP_ii v24.2d, v1.2d, v11.2d[0] OP_ii v24.2d, v1.2d, v11.d[0]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v25.16b, v25.16b, v25.16b eor v25.16b, v25.16b, v25.16b
fmls v25.2d, v0.2d, v11.2d[0] fmls v25.2d, v0.2d, v11.d[0]
#else #else
fmul v25.2d, v0.2d, v11.2d[0] fmul v25.2d, v0.2d, v11.d[0]
#endif #endif
OP_ir v25.2d, v1.2d, v10.2d[0] OP_ir v25.2d, v1.2d, v10.d[0]
fmul v26.2d, v2.2d, v10.2d[0] fmul v26.2d, v2.2d, v10.d[0]
OP_ii v26.2d, v3.2d, v11.2d[0] OP_ii v26.2d, v3.2d, v11.d[0]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v27.16b, v27.16b, v27.16b eor v27.16b, v27.16b, v27.16b
fmls v27.2d, v2.2d, v11.2d[0] fmls v27.2d, v2.2d, v11.d[0]
#else #else
fmul v27.2d, v2.2d, v11.2d[0] fmul v27.2d, v2.2d, v11.d[0]
#endif #endif
OP_ir v27.2d, v3.2d, v10.2d[0] OP_ir v27.2d, v3.2d, v10.d[0]
fmul v28.2d, v0.2d, v10.2d[1] fmul v28.2d, v0.2d, v10.d[1]
OP_ii v28.2d, v1.2d, v11.2d[1] OP_ii v28.2d, v1.2d, v11.d[1]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v29.16b, v29.16b, v29.16b eor v29.16b, v29.16b, v29.16b
fmls v29.2d, v0.2d, v11.2d[1] fmls v29.2d, v0.2d, v11.d[1]
#else #else
fmul v29.2d, v0.2d, v11.2d[1] fmul v29.2d, v0.2d, v11.d[1]
#endif #endif
OP_ir v29.2d, v1.2d, v10.2d[1] OP_ir v29.2d, v1.2d, v10.d[1]
fmul v30.2d, v2.2d, v10.2d[1] fmul v30.2d, v2.2d, v10.d[1]
OP_ii v30.2d, v3.2d, v11.2d[1] OP_ii v30.2d, v3.2d, v11.d[1]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v31.16b, v31.16b, v31.16b eor v31.16b, v31.16b, v31.16b
fmls v31.2d, v2.2d, v11.2d[1] fmls v31.2d, v2.2d, v11.d[1]
#else #else
fmul v31.2d, v2.2d, v11.2d[1] fmul v31.2d, v2.2d, v11.d[1]
#endif #endif
OP_ir v31.2d, v3.2d, v10.2d[1] OP_ir v31.2d, v3.2d, v10.d[1]
ld2 {v12.2d, v13.2d}, [pB] ld2 {v12.2d, v13.2d}, [pB]
add pB, pB, #32 add pB, pB, #32
@ -281,161 +281,161 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL4x4_M1 .macro KERNEL4x4_M1
OP_rr v16.2d, v0.2d, v8.2d[0] OP_rr v16.2d, v0.2d, v8.d[0]
OP_ii v16.2d, v1.2d, v9.2d[0] OP_ii v16.2d, v1.2d, v9.d[0]
OP_ri v17.2d, v0.2d, v9.2d[0] OP_ri v17.2d, v0.2d, v9.d[0]
OP_ir v17.2d, v1.2d, v8.2d[0] OP_ir v17.2d, v1.2d, v8.d[0]
ld2 {v12.2d, v13.2d}, [pB] // For next round ld2 {v12.2d, v13.2d}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
OP_rr v18.2d, v2.2d, v8.2d[0] OP_rr v18.2d, v2.2d, v8.d[0]
OP_ii v18.2d, v3.2d, v9.2d[0] OP_ii v18.2d, v3.2d, v9.d[0]
OP_ri v19.2d, v2.2d, v9.2d[0] OP_ri v19.2d, v2.2d, v9.d[0]
OP_ir v19.2d, v3.2d, v8.2d[0] OP_ir v19.2d, v3.2d, v8.d[0]
ld2 {v14.2d, v15.2d}, [pB] // For next round ld2 {v14.2d, v15.2d}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
OP_rr v20.2d, v0.2d, v8.2d[1] OP_rr v20.2d, v0.2d, v8.d[1]
OP_ii v20.2d, v1.2d, v9.2d[1] OP_ii v20.2d, v1.2d, v9.d[1]
OP_ri v21.2d, v0.2d, v9.2d[1] OP_ri v21.2d, v0.2d, v9.d[1]
OP_ir v21.2d, v1.2d, v8.2d[1] OP_ir v21.2d, v1.2d, v8.d[1]
ld2 {v4.2d, v5.2d} , [pA] // For next round ld2 {v4.2d, v5.2d} , [pA] // For next round
add pA, pA, #32 add pA, pA, #32
OP_rr v22.2d, v2.2d, v8.2d[1] OP_rr v22.2d, v2.2d, v8.d[1]
OP_ii v22.2d, v3.2d, v9.2d[1] OP_ii v22.2d, v3.2d, v9.d[1]
OP_ri v23.2d, v2.2d, v9.2d[1] OP_ri v23.2d, v2.2d, v9.d[1]
OP_ir v23.2d, v3.2d, v8.2d[1] OP_ir v23.2d, v3.2d, v8.d[1]
ld2 {v6.2d, v7.2d} , [pA] // For next round ld2 {v6.2d, v7.2d} , [pA] // For next round
add pA, pA, #32 add pA, pA, #32
OP_rr v24.2d, v0.2d, v10.2d[0] OP_rr v24.2d, v0.2d, v10.d[0]
OP_ii v24.2d, v1.2d, v11.2d[0] OP_ii v24.2d, v1.2d, v11.d[0]
OP_ri v25.2d, v0.2d, v11.2d[0] OP_ri v25.2d, v0.2d, v11.d[0]
OP_ir v25.2d, v1.2d, v10.2d[0] OP_ir v25.2d, v1.2d, v10.d[0]
prfm PLDL1KEEP, [pA, #512] prfm PLDL1KEEP, [pA, #512]
OP_rr v26.2d, v2.2d, v10.2d[0] OP_rr v26.2d, v2.2d, v10.d[0]
OP_ii v26.2d, v3.2d, v11.2d[0] OP_ii v26.2d, v3.2d, v11.d[0]
OP_ri v27.2d, v2.2d, v11.2d[0] OP_ri v27.2d, v2.2d, v11.d[0]
OP_ir v27.2d, v3.2d, v10.2d[0] OP_ir v27.2d, v3.2d, v10.d[0]
prfm PLDL1KEEP, [pB, #512] prfm PLDL1KEEP, [pB, #512]
OP_rr v28.2d, v0.2d, v10.2d[1] OP_rr v28.2d, v0.2d, v10.d[1]
OP_ii v28.2d, v1.2d, v11.2d[1] OP_ii v28.2d, v1.2d, v11.d[1]
OP_ri v29.2d, v0.2d, v11.2d[1] OP_ri v29.2d, v0.2d, v11.d[1]
OP_ir v29.2d, v1.2d, v10.2d[1] OP_ir v29.2d, v1.2d, v10.d[1]
OP_rr v30.2d, v2.2d, v10.2d[1] OP_rr v30.2d, v2.2d, v10.d[1]
OP_ii v30.2d, v3.2d, v11.2d[1] OP_ii v30.2d, v3.2d, v11.d[1]
OP_ri v31.2d, v2.2d, v11.2d[1] OP_ri v31.2d, v2.2d, v11.d[1]
OP_ir v31.2d, v3.2d, v10.2d[1] OP_ir v31.2d, v3.2d, v10.d[1]
.endm .endm
.macro KERNEL4x4_M2 .macro KERNEL4x4_M2
OP_rr v16.2d, v4.2d, v12.2d[0] OP_rr v16.2d, v4.2d, v12.d[0]
OP_ii v16.2d, v5.2d, v13.2d[0] OP_ii v16.2d, v5.2d, v13.d[0]
OP_ri v17.2d, v4.2d, v13.2d[0] OP_ri v17.2d, v4.2d, v13.d[0]
OP_ir v17.2d, v5.2d, v12.2d[0] OP_ir v17.2d, v5.2d, v12.d[0]
ld2 {v8.2d, v9.2d}, [pB] // For next round ld2 {v8.2d, v9.2d}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
OP_rr v18.2d, v6.2d, v12.2d[0] OP_rr v18.2d, v6.2d, v12.d[0]
OP_ii v18.2d, v7.2d, v13.2d[0] OP_ii v18.2d, v7.2d, v13.d[0]
OP_ri v19.2d, v6.2d, v13.2d[0] OP_ri v19.2d, v6.2d, v13.d[0]
OP_ir v19.2d, v7.2d, v12.2d[0] OP_ir v19.2d, v7.2d, v12.d[0]
ld2 {v10.2d, v11.2d}, [pB] // For next round ld2 {v10.2d, v11.2d}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
OP_rr v20.2d, v4.2d, v12.2d[1] OP_rr v20.2d, v4.2d, v12.d[1]
OP_ii v20.2d, v5.2d, v13.2d[1] OP_ii v20.2d, v5.2d, v13.d[1]
OP_ri v21.2d, v4.2d, v13.2d[1] OP_ri v21.2d, v4.2d, v13.d[1]
OP_ir v21.2d, v5.2d, v12.2d[1] OP_ir v21.2d, v5.2d, v12.d[1]
ld2 {v0.2d, v1.2d}, [pA] // For next round ld2 {v0.2d, v1.2d}, [pA] // For next round
add pA, pA, #32 add pA, pA, #32
OP_rr v22.2d, v6.2d, v12.2d[1] OP_rr v22.2d, v6.2d, v12.d[1]
OP_ii v22.2d, v7.2d, v13.2d[1] OP_ii v22.2d, v7.2d, v13.d[1]
OP_ri v23.2d, v6.2d, v13.2d[1] OP_ri v23.2d, v6.2d, v13.d[1]
OP_ir v23.2d, v7.2d, v12.2d[1] OP_ir v23.2d, v7.2d, v12.d[1]
ld2 {v2.2d, v3.2d}, [pA] // For next round ld2 {v2.2d, v3.2d}, [pA] // For next round
add pA, pA, #32 add pA, pA, #32
OP_rr v24.2d, v4.2d, v14.2d[0] OP_rr v24.2d, v4.2d, v14.d[0]
OP_ii v24.2d, v5.2d, v15.2d[0] OP_ii v24.2d, v5.2d, v15.d[0]
OP_ri v25.2d, v4.2d, v15.2d[0] OP_ri v25.2d, v4.2d, v15.d[0]
OP_ir v25.2d, v5.2d, v14.2d[0] OP_ir v25.2d, v5.2d, v14.d[0]
prfm PLDL1KEEP, [pA, #512] prfm PLDL1KEEP, [pA, #512]
OP_rr v26.2d, v6.2d, v14.2d[0] OP_rr v26.2d, v6.2d, v14.d[0]
OP_ii v26.2d, v7.2d, v15.2d[0] OP_ii v26.2d, v7.2d, v15.d[0]
OP_ri v27.2d, v6.2d, v15.2d[0] OP_ri v27.2d, v6.2d, v15.d[0]
OP_ir v27.2d, v7.2d, v14.2d[0] OP_ir v27.2d, v7.2d, v14.d[0]
prfm PLDL1KEEP, [pB, #512] prfm PLDL1KEEP, [pB, #512]
OP_rr v28.2d, v4.2d, v14.2d[1] OP_rr v28.2d, v4.2d, v14.d[1]
OP_ii v28.2d, v5.2d, v15.2d[1] OP_ii v28.2d, v5.2d, v15.d[1]
OP_ri v29.2d, v4.2d, v15.2d[1] OP_ri v29.2d, v4.2d, v15.d[1]
OP_ir v29.2d, v5.2d, v14.2d[1] OP_ir v29.2d, v5.2d, v14.d[1]
OP_rr v30.2d, v6.2d, v14.2d[1] OP_rr v30.2d, v6.2d, v14.d[1]
OP_ii v30.2d, v7.2d, v15.2d[1] OP_ii v30.2d, v7.2d, v15.d[1]
OP_ri v31.2d, v6.2d, v15.2d[1] OP_ri v31.2d, v6.2d, v15.d[1]
OP_ir v31.2d, v7.2d, v14.2d[1] OP_ir v31.2d, v7.2d, v14.d[1]
.endm .endm
.macro KERNEL4x4_E .macro KERNEL4x4_E
OP_rr v16.2d, v4.2d, v12.2d[0] OP_rr v16.2d, v4.2d, v12.d[0]
OP_ii v16.2d, v5.2d, v13.2d[0] OP_ii v16.2d, v5.2d, v13.d[0]
OP_ri v17.2d, v4.2d, v13.2d[0] OP_ri v17.2d, v4.2d, v13.d[0]
OP_ir v17.2d, v5.2d, v12.2d[0] OP_ir v17.2d, v5.2d, v12.d[0]
OP_rr v18.2d, v6.2d, v12.2d[0] OP_rr v18.2d, v6.2d, v12.d[0]
OP_ii v18.2d, v7.2d, v13.2d[0] OP_ii v18.2d, v7.2d, v13.d[0]
OP_ri v19.2d, v6.2d, v13.2d[0] OP_ri v19.2d, v6.2d, v13.d[0]
OP_ir v19.2d, v7.2d, v12.2d[0] OP_ir v19.2d, v7.2d, v12.d[0]
OP_rr v20.2d, v4.2d, v12.2d[1] OP_rr v20.2d, v4.2d, v12.d[1]
OP_ii v20.2d, v5.2d, v13.2d[1] OP_ii v20.2d, v5.2d, v13.d[1]
OP_ri v21.2d, v4.2d, v13.2d[1] OP_ri v21.2d, v4.2d, v13.d[1]
OP_ir v21.2d, v5.2d, v12.2d[1] OP_ir v21.2d, v5.2d, v12.d[1]
OP_rr v22.2d, v6.2d, v12.2d[1] OP_rr v22.2d, v6.2d, v12.d[1]
OP_ii v22.2d, v7.2d, v13.2d[1] OP_ii v22.2d, v7.2d, v13.d[1]
OP_ri v23.2d, v6.2d, v13.2d[1] OP_ri v23.2d, v6.2d, v13.d[1]
OP_ir v23.2d, v7.2d, v12.2d[1] OP_ir v23.2d, v7.2d, v12.d[1]
OP_rr v24.2d, v4.2d, v14.2d[0] OP_rr v24.2d, v4.2d, v14.d[0]
OP_ii v24.2d, v5.2d, v15.2d[0] OP_ii v24.2d, v5.2d, v15.d[0]
OP_ri v25.2d, v4.2d, v15.2d[0] OP_ri v25.2d, v4.2d, v15.d[0]
OP_ir v25.2d, v5.2d, v14.2d[0] OP_ir v25.2d, v5.2d, v14.d[0]
OP_rr v26.2d, v6.2d, v14.2d[0] OP_rr v26.2d, v6.2d, v14.d[0]
OP_ii v26.2d, v7.2d, v15.2d[0] OP_ii v26.2d, v7.2d, v15.d[0]
OP_ri v27.2d, v6.2d, v15.2d[0] OP_ri v27.2d, v6.2d, v15.d[0]
OP_ir v27.2d, v7.2d, v14.2d[0] OP_ir v27.2d, v7.2d, v14.d[0]
OP_rr v28.2d, v4.2d, v14.2d[1] OP_rr v28.2d, v4.2d, v14.d[1]
OP_ii v28.2d, v5.2d, v15.2d[1] OP_ii v28.2d, v5.2d, v15.d[1]
OP_ri v29.2d, v4.2d, v15.2d[1] OP_ri v29.2d, v4.2d, v15.d[1]
OP_ir v29.2d, v5.2d, v14.2d[1] OP_ir v29.2d, v5.2d, v14.d[1]
OP_rr v30.2d, v6.2d, v14.2d[1] OP_rr v30.2d, v6.2d, v14.d[1]
OP_ii v30.2d, v7.2d, v15.2d[1] OP_ii v30.2d, v7.2d, v15.d[1]
OP_ri v31.2d, v6.2d, v15.2d[1] OP_ri v31.2d, v6.2d, v15.d[1]
OP_ir v31.2d, v7.2d, v14.2d[1] OP_ir v31.2d, v7.2d, v14.d[1]
.endm .endm
.macro KERNEL4x4_SUB .macro KERNEL4x4_SUB
@ -448,45 +448,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v2.2d, v3.2d}, [pA] ld2 {v2.2d, v3.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
OP_rr v16.2d, v0.2d, v8.2d[0] OP_rr v16.2d, v0.2d, v8.d[0]
OP_ii v16.2d, v1.2d, v9.2d[0] OP_ii v16.2d, v1.2d, v9.d[0]
OP_ri v17.2d, v0.2d, v9.2d[0] OP_ri v17.2d, v0.2d, v9.d[0]
OP_ir v17.2d, v1.2d, v8.2d[0] OP_ir v17.2d, v1.2d, v8.d[0]
OP_rr v18.2d, v2.2d, v8.2d[0] OP_rr v18.2d, v2.2d, v8.d[0]
OP_ii v18.2d, v3.2d, v9.2d[0] OP_ii v18.2d, v3.2d, v9.d[0]
OP_ri v19.2d, v2.2d, v9.2d[0] OP_ri v19.2d, v2.2d, v9.d[0]
OP_ir v19.2d, v3.2d, v8.2d[0] OP_ir v19.2d, v3.2d, v8.d[0]
OP_rr v20.2d, v0.2d, v8.2d[1] OP_rr v20.2d, v0.2d, v8.d[1]
OP_ii v20.2d, v1.2d, v9.2d[1] OP_ii v20.2d, v1.2d, v9.d[1]
OP_ri v21.2d, v0.2d, v9.2d[1] OP_ri v21.2d, v0.2d, v9.d[1]
OP_ir v21.2d, v1.2d, v8.2d[1] OP_ir v21.2d, v1.2d, v8.d[1]
OP_rr v22.2d, v2.2d, v8.2d[1] OP_rr v22.2d, v2.2d, v8.d[1]
OP_ii v22.2d, v3.2d, v9.2d[1] OP_ii v22.2d, v3.2d, v9.d[1]
OP_ri v23.2d, v2.2d, v9.2d[1] OP_ri v23.2d, v2.2d, v9.d[1]
OP_ir v23.2d, v3.2d, v8.2d[1] OP_ir v23.2d, v3.2d, v8.d[1]
OP_rr v24.2d, v0.2d, v10.2d[0] OP_rr v24.2d, v0.2d, v10.d[0]
OP_ii v24.2d, v1.2d, v11.2d[0] OP_ii v24.2d, v1.2d, v11.d[0]
OP_ri v25.2d, v0.2d, v11.2d[0] OP_ri v25.2d, v0.2d, v11.d[0]
OP_ir v25.2d, v1.2d, v10.2d[0] OP_ir v25.2d, v1.2d, v10.d[0]
OP_rr v26.2d, v2.2d, v10.2d[0] OP_rr v26.2d, v2.2d, v10.d[0]
OP_ii v26.2d, v3.2d, v11.2d[0] OP_ii v26.2d, v3.2d, v11.d[0]
OP_ri v27.2d, v2.2d, v11.2d[0] OP_ri v27.2d, v2.2d, v11.d[0]
OP_ir v27.2d, v3.2d, v10.2d[0] OP_ir v27.2d, v3.2d, v10.d[0]
OP_rr v28.2d, v0.2d, v10.2d[1] OP_rr v28.2d, v0.2d, v10.d[1]
OP_ii v28.2d, v1.2d, v11.2d[1] OP_ii v28.2d, v1.2d, v11.d[1]
OP_ri v29.2d, v0.2d, v11.2d[1] OP_ri v29.2d, v0.2d, v11.d[1]
OP_ir v29.2d, v1.2d, v10.2d[1] OP_ir v29.2d, v1.2d, v10.d[1]
OP_rr v30.2d, v2.2d, v10.2d[1] OP_rr v30.2d, v2.2d, v10.d[1]
OP_ii v30.2d, v3.2d, v11.2d[1] OP_ii v30.2d, v3.2d, v11.d[1]
OP_ri v31.2d, v2.2d, v11.2d[1] OP_ri v31.2d, v2.2d, v11.d[1]
OP_ir v31.2d, v3.2d, v10.2d[1] OP_ir v31.2d, v3.2d, v10.d[1]
.endm .endm
.macro SAVE4x4 .macro SAVE4x4
@ -582,25 +582,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.2d, v1.2d}, [pA] ld2 {v0.2d, v1.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
OP_rr v16.2d, v0.2d, v8.2d[0] OP_rr v16.2d, v0.2d, v8.d[0]
OP_ii v16.2d, v1.2d, v9.2d[0] OP_ii v16.2d, v1.2d, v9.d[0]
OP_ri v17.2d, v0.2d, v9.2d[0] OP_ri v17.2d, v0.2d, v9.d[0]
OP_ir v17.2d, v1.2d, v8.2d[0] OP_ir v17.2d, v1.2d, v8.d[0]
OP_rr v20.2d, v0.2d, v8.2d[1] OP_rr v20.2d, v0.2d, v8.d[1]
OP_ii v20.2d, v1.2d, v9.2d[1] OP_ii v20.2d, v1.2d, v9.d[1]
OP_ri v21.2d, v0.2d, v9.2d[1] OP_ri v21.2d, v0.2d, v9.d[1]
OP_ir v21.2d, v1.2d, v8.2d[1] OP_ir v21.2d, v1.2d, v8.d[1]
OP_rr v24.2d, v0.2d, v10.2d[0] OP_rr v24.2d, v0.2d, v10.d[0]
OP_ii v24.2d, v1.2d, v11.2d[0] OP_ii v24.2d, v1.2d, v11.d[0]
OP_ri v25.2d, v0.2d, v11.2d[0] OP_ri v25.2d, v0.2d, v11.d[0]
OP_ir v25.2d, v1.2d, v10.2d[0] OP_ir v25.2d, v1.2d, v10.d[0]
OP_rr v28.2d, v0.2d, v10.2d[1] OP_rr v28.2d, v0.2d, v10.d[1]
OP_ii v28.2d, v1.2d, v11.2d[1] OP_ii v28.2d, v1.2d, v11.d[1]
OP_ri v29.2d, v0.2d, v11.2d[1] OP_ri v29.2d, v0.2d, v11.d[1]
OP_ir v29.2d, v1.2d, v10.2d[1] OP_ir v29.2d, v1.2d, v10.d[1]
.endm .endm
.macro SAVE2x4 .macro SAVE2x4
@ -669,25 +669,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.d, v1.d}[0], [pA] ld2 {v0.d, v1.d}[0], [pA]
add pA, pA, #16 add pA, pA, #16
OP_rr d16, d0, v8.2d[0] OP_rr d16, d0, v8.d[0]
OP_ii d16, d1, v9.2d[0] OP_ii d16, d1, v9.d[0]
OP_ri d17, d0, v9.2d[0] OP_ri d17, d0, v9.d[0]
OP_ir d17, d1, v8.2d[0] OP_ir d17, d1, v8.d[0]
OP_rr d20, d0, v8.2d[1] OP_rr d20, d0, v8.d[1]
OP_ii d20, d1, v9.2d[1] OP_ii d20, d1, v9.d[1]
OP_ri d21, d0, v9.2d[1] OP_ri d21, d0, v9.d[1]
OP_ir d21, d1, v8.2d[1] OP_ir d21, d1, v8.d[1]
OP_rr d24, d0, v10.2d[0] OP_rr d24, d0, v10.d[0]
OP_ii d24, d1, v11.2d[0] OP_ii d24, d1, v11.d[0]
OP_ri d25, d0, v11.2d[0] OP_ri d25, d0, v11.d[0]
OP_ir d25, d1, v10.2d[0] OP_ir d25, d1, v10.d[0]
OP_rr d28, d0, v10.2d[1] OP_rr d28, d0, v10.d[1]
OP_ii d28, d1, v11.2d[1] OP_ii d28, d1, v11.d[1]
OP_ri d29, d0, v11.2d[1] OP_ri d29, d0, v11.d[1]
OP_ir d29, d1, v10.2d[1] OP_ir d29, d1, v10.d[1]
.endm .endm
.macro SAVE1x4 .macro SAVE1x4
@ -756,25 +756,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v2.2d, v3.2d}, [pA] ld2 {v2.2d, v3.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
OP_rr v16.2d, v0.2d, v8.2d[0] OP_rr v16.2d, v0.2d, v8.d[0]
OP_ii v16.2d, v1.2d, v9.2d[0] OP_ii v16.2d, v1.2d, v9.d[0]
OP_ri v17.2d, v0.2d, v9.2d[0] OP_ri v17.2d, v0.2d, v9.d[0]
OP_ir v17.2d, v1.2d, v8.2d[0] OP_ir v17.2d, v1.2d, v8.d[0]
OP_rr v18.2d, v2.2d, v8.2d[0] OP_rr v18.2d, v2.2d, v8.d[0]
OP_ii v18.2d, v3.2d, v9.2d[0] OP_ii v18.2d, v3.2d, v9.d[0]
OP_ri v19.2d, v2.2d, v9.2d[0] OP_ri v19.2d, v2.2d, v9.d[0]
OP_ir v19.2d, v3.2d, v8.2d[0] OP_ir v19.2d, v3.2d, v8.d[0]
OP_rr v20.2d, v0.2d, v8.2d[1] OP_rr v20.2d, v0.2d, v8.d[1]
OP_ii v20.2d, v1.2d, v9.2d[1] OP_ii v20.2d, v1.2d, v9.d[1]
OP_ri v21.2d, v0.2d, v9.2d[1] OP_ri v21.2d, v0.2d, v9.d[1]
OP_ir v21.2d, v1.2d, v8.2d[1] OP_ir v21.2d, v1.2d, v8.d[1]
OP_rr v22.2d, v2.2d, v8.2d[1] OP_rr v22.2d, v2.2d, v8.d[1]
OP_ii v22.2d, v3.2d, v9.2d[1] OP_ii v22.2d, v3.2d, v9.d[1]
OP_ri v23.2d, v2.2d, v9.2d[1] OP_ri v23.2d, v2.2d, v9.d[1]
OP_ir v23.2d, v3.2d, v8.2d[1] OP_ir v23.2d, v3.2d, v8.d[1]
.endm .endm
.macro SAVE4x2 .macro SAVE4x2
@ -833,15 +833,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.2d, v1.2d}, [pA] ld2 {v0.2d, v1.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
OP_rr v16.2d, v0.2d, v8.2d[0] OP_rr v16.2d, v0.2d, v8.d[0]
OP_ii v16.2d, v1.2d, v9.2d[0] OP_ii v16.2d, v1.2d, v9.d[0]
OP_ri v17.2d, v0.2d, v9.2d[0] OP_ri v17.2d, v0.2d, v9.d[0]
OP_ir v17.2d, v1.2d, v8.2d[0] OP_ir v17.2d, v1.2d, v8.d[0]
OP_rr v20.2d, v0.2d, v8.2d[1] OP_rr v20.2d, v0.2d, v8.d[1]
OP_ii v20.2d, v1.2d, v9.2d[1] OP_ii v20.2d, v1.2d, v9.d[1]
OP_ri v21.2d, v0.2d, v9.2d[1] OP_ri v21.2d, v0.2d, v9.d[1]
OP_ir v21.2d, v1.2d, v8.2d[1] OP_ir v21.2d, v1.2d, v8.d[1]
.endm .endm
.macro SAVE2x2 .macro SAVE2x2
@ -886,15 +886,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.d, v1.d}[0], [pA] ld2 {v0.d, v1.d}[0], [pA]
add pA, pA, #16 add pA, pA, #16
OP_rr d16, d0, v8.2d[0] OP_rr d16, d0, v8.d[0]
OP_ii d16, d1, v9.2d[0] OP_ii d16, d1, v9.d[0]
OP_ri d17, d0, v9.2d[0] OP_ri d17, d0, v9.d[0]
OP_ir d17, d1, v8.2d[0] OP_ir d17, d1, v8.d[0]
OP_rr d20, d0, v8.2d[1] OP_rr d20, d0, v8.d[1]
OP_ii d20, d1, v9.2d[1] OP_ii d20, d1, v9.d[1]
OP_ri d21, d0, v9.2d[1] OP_ri d21, d0, v9.d[1]
OP_ir d21, d1, v8.2d[1] OP_ir d21, d1, v8.d[1]
.endm .endm
.macro SAVE1x2 .macro SAVE1x2

464
kernel/arm64/ztrmm_kernel_4x4.S
View File

@ -185,93 +185,93 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v2.2d, v3.2d}, [pA] ld2 {v2.2d, v3.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
fmul v16.2d, v0.2d, v8.2d[0] fmul v16.2d, v0.2d, v8.d[0]
OP_ii v16.2d, v1.2d, v9.2d[0] OP_ii v16.2d, v1.2d, v9.d[0]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v17.16b, v17.16b, v17.16b eor v17.16b, v17.16b, v17.16b
fmls v17.2d, v0.2d, v9.2d[0] fmls v17.2d, v0.2d, v9.d[0]
#else #else
fmul v17.2d, v0.2d, v9.2d[0] fmul v17.2d, v0.2d, v9.d[0]
#endif #endif
OP_ir v17.2d, v1.2d, v8.2d[0] OP_ir v17.2d, v1.2d, v8.d[0]
fmul v18.2d, v2.2d, v8.2d[0] fmul v18.2d, v2.2d, v8.d[0]
OP_ii v18.2d, v3.2d, v9.2d[0] OP_ii v18.2d, v3.2d, v9.d[0]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v19.16b, v19.16b, v19.16b eor v19.16b, v19.16b, v19.16b
fmls v19.2d, v2.2d, v9.2d[0] fmls v19.2d, v2.2d, v9.d[0]
#else #else
fmul v19.2d, v2.2d, v9.2d[0] fmul v19.2d, v2.2d, v9.d[0]
#endif #endif
OP_ir v19.2d, v3.2d, v8.2d[0] OP_ir v19.2d, v3.2d, v8.d[0]
fmul v20.2d, v0.2d, v8.2d[1] fmul v20.2d, v0.2d, v8.d[1]
OP_ii v20.2d, v1.2d, v9.2d[1] OP_ii v20.2d, v1.2d, v9.d[1]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v21.16b, v21.16b, v21.16b eor v21.16b, v21.16b, v21.16b
fmls v21.2d, v0.2d, v9.2d[1] fmls v21.2d, v0.2d, v9.d[1]
#else #else
fmul v21.2d, v0.2d, v9.2d[1] fmul v21.2d, v0.2d, v9.d[1]
#endif #endif
OP_ir v21.2d, v1.2d, v8.2d[1] OP_ir v21.2d, v1.2d, v8.d[1]
fmul v22.2d, v2.2d, v8.2d[1] fmul v22.2d, v2.2d, v8.d[1]
OP_ii v22.2d, v3.2d, v9.2d[1] OP_ii v22.2d, v3.2d, v9.d[1]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v23.16b, v23.16b, v23.16b eor v23.16b, v23.16b, v23.16b
fmls v23.2d, v2.2d, v9.2d[1] fmls v23.2d, v2.2d, v9.d[1]
#else #else
fmul v23.2d, v2.2d, v9.2d[1] fmul v23.2d, v2.2d, v9.d[1]
#endif #endif
OP_ir v23.2d, v3.2d, v8.2d[1] OP_ir v23.2d, v3.2d, v8.d[1]
fmul v24.2d, v0.2d, v10.2d[0] fmul v24.2d, v0.2d, v10.d[0]
OP_ii v24.2d, v1.2d, v11.2d[0] OP_ii v24.2d, v1.2d, v11.d[0]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v25.16b, v25.16b, v25.16b eor v25.16b, v25.16b, v25.16b
fmls v25.2d, v0.2d, v11.2d[0] fmls v25.2d, v0.2d, v11.d[0]
#else #else
fmul v25.2d, v0.2d, v11.2d[0] fmul v25.2d, v0.2d, v11.d[0]
#endif #endif
OP_ir v25.2d, v1.2d, v10.2d[0] OP_ir v25.2d, v1.2d, v10.d[0]
fmul v26.2d, v2.2d, v10.2d[0] fmul v26.2d, v2.2d, v10.d[0]
OP_ii v26.2d, v3.2d, v11.2d[0] OP_ii v26.2d, v3.2d, v11.d[0]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v27.16b, v27.16b, v27.16b eor v27.16b, v27.16b, v27.16b
fmls v27.2d, v2.2d, v11.2d[0] fmls v27.2d, v2.2d, v11.d[0]
#else #else
fmul v27.2d, v2.2d, v11.2d[0] fmul v27.2d, v2.2d, v11.d[0]
#endif #endif
OP_ir v27.2d, v3.2d, v10.2d[0] OP_ir v27.2d, v3.2d, v10.d[0]
fmul v28.2d, v0.2d, v10.2d[1] fmul v28.2d, v0.2d, v10.d[1]
OP_ii v28.2d, v1.2d, v11.2d[1] OP_ii v28.2d, v1.2d, v11.d[1]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v29.16b, v29.16b, v29.16b eor v29.16b, v29.16b, v29.16b
fmls v29.2d, v0.2d, v11.2d[1] fmls v29.2d, v0.2d, v11.d[1]
#else #else
fmul v29.2d, v0.2d, v11.2d[1] fmul v29.2d, v0.2d, v11.d[1]
#endif #endif
OP_ir v29.2d, v1.2d, v10.2d[1] OP_ir v29.2d, v1.2d, v10.d[1]
fmul v30.2d, v2.2d, v10.2d[1] fmul v30.2d, v2.2d, v10.d[1]
OP_ii v30.2d, v3.2d, v11.2d[1] OP_ii v30.2d, v3.2d, v11.d[1]
#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \ #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
defined(RR) || defined(RC) || defined(CR) || defined(CC) defined(RR) || defined(RC) || defined(CR) || defined(CC)
eor v31.16b, v31.16b, v31.16b eor v31.16b, v31.16b, v31.16b
fmls v31.2d, v2.2d, v11.2d[1] fmls v31.2d, v2.2d, v11.d[1]
#else #else
fmul v31.2d, v2.2d, v11.2d[1] fmul v31.2d, v2.2d, v11.d[1]
#endif #endif
OP_ir v31.2d, v3.2d, v10.2d[1] OP_ir v31.2d, v3.2d, v10.d[1]
ld2 {v12.2d, v13.2d}, [pB] ld2 {v12.2d, v13.2d}, [pB]
add pB, pB, #32 add pB, pB, #32
@ -284,161 +284,161 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.endm .endm
.macro KERNEL4x4_M1 .macro KERNEL4x4_M1
OP_rr v16.2d, v0.2d, v8.2d[0] OP_rr v16.2d, v0.2d, v8.d[0]
OP_ii v16.2d, v1.2d, v9.2d[0] OP_ii v16.2d, v1.2d, v9.d[0]
OP_ri v17.2d, v0.2d, v9.2d[0] OP_ri v17.2d, v0.2d, v9.d[0]
OP_ir v17.2d, v1.2d, v8.2d[0] OP_ir v17.2d, v1.2d, v8.d[0]
ld2 {v12.2d, v13.2d}, [pB] // For next round ld2 {v12.2d, v13.2d}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
OP_rr v18.2d, v2.2d, v8.2d[0] OP_rr v18.2d, v2.2d, v8.d[0]
OP_ii v18.2d, v3.2d, v9.2d[0] OP_ii v18.2d, v3.2d, v9.d[0]
OP_ri v19.2d, v2.2d, v9.2d[0] OP_ri v19.2d, v2.2d, v9.d[0]
OP_ir v19.2d, v3.2d, v8.2d[0] OP_ir v19.2d, v3.2d, v8.d[0]
ld2 {v14.2d, v15.2d}, [pB] // For next round ld2 {v14.2d, v15.2d}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
OP_rr v20.2d, v0.2d, v8.2d[1] OP_rr v20.2d, v0.2d, v8.d[1]
OP_ii v20.2d, v1.2d, v9.2d[1] OP_ii v20.2d, v1.2d, v9.d[1]
OP_ri v21.2d, v0.2d, v9.2d[1] OP_ri v21.2d, v0.2d, v9.d[1]
OP_ir v21.2d, v1.2d, v8.2d[1] OP_ir v21.2d, v1.2d, v8.d[1]
ld2 {v4.2d, v5.2d} , [pA] // For next round ld2 {v4.2d, v5.2d} , [pA] // For next round
add pA, pA, #32 add pA, pA, #32
OP_rr v22.2d, v2.2d, v8.2d[1] OP_rr v22.2d, v2.2d, v8.d[1]
OP_ii v22.2d, v3.2d, v9.2d[1] OP_ii v22.2d, v3.2d, v9.d[1]
OP_ri v23.2d, v2.2d, v9.2d[1] OP_ri v23.2d, v2.2d, v9.d[1]
OP_ir v23.2d, v3.2d, v8.2d[1] OP_ir v23.2d, v3.2d, v8.d[1]
ld2 {v6.2d, v7.2d} , [pA] // For next round ld2 {v6.2d, v7.2d} , [pA] // For next round
add pA, pA, #32 add pA, pA, #32
OP_rr v24.2d, v0.2d, v10.2d[0] OP_rr v24.2d, v0.2d, v10.d[0]
OP_ii v24.2d, v1.2d, v11.2d[0] OP_ii v24.2d, v1.2d, v11.d[0]
OP_ri v25.2d, v0.2d, v11.2d[0] OP_ri v25.2d, v0.2d, v11.d[0]
OP_ir v25.2d, v1.2d, v10.2d[0] OP_ir v25.2d, v1.2d, v10.d[0]
prfm PLDL1KEEP, [pA, #512] prfm PLDL1KEEP, [pA, #512]
OP_rr v26.2d, v2.2d, v10.2d[0] OP_rr v26.2d, v2.2d, v10.d[0]
OP_ii v26.2d, v3.2d, v11.2d[0] OP_ii v26.2d, v3.2d, v11.d[0]
OP_ri v27.2d, v2.2d, v11.2d[0] OP_ri v27.2d, v2.2d, v11.d[0]
OP_ir v27.2d, v3.2d, v10.2d[0] OP_ir v27.2d, v3.2d, v10.d[0]
prfm PLDL1KEEP, [pB, #512] prfm PLDL1KEEP, [pB, #512]
OP_rr v28.2d, v0.2d, v10.2d[1] OP_rr v28.2d, v0.2d, v10.d[1]
OP_ii v28.2d, v1.2d, v11.2d[1] OP_ii v28.2d, v1.2d, v11.d[1]
OP_ri v29.2d, v0.2d, v11.2d[1] OP_ri v29.2d, v0.2d, v11.d[1]
OP_ir v29.2d, v1.2d, v10.2d[1] OP_ir v29.2d, v1.2d, v10.d[1]
OP_rr v30.2d, v2.2d, v10.2d[1] OP_rr v30.2d, v2.2d, v10.d[1]
OP_ii v30.2d, v3.2d, v11.2d[1] OP_ii v30.2d, v3.2d, v11.d[1]
OP_ri v31.2d, v2.2d, v11.2d[1] OP_ri v31.2d, v2.2d, v11.d[1]
OP_ir v31.2d, v3.2d, v10.2d[1] OP_ir v31.2d, v3.2d, v10.d[1]
.endm .endm
.macro KERNEL4x4_M2 .macro KERNEL4x4_M2
OP_rr v16.2d, v4.2d, v12.2d[0] OP_rr v16.2d, v4.2d, v12.d[0]
OP_ii v16.2d, v5.2d, v13.2d[0] OP_ii v16.2d, v5.2d, v13.d[0]
OP_ri v17.2d, v4.2d, v13.2d[0] OP_ri v17.2d, v4.2d, v13.d[0]
OP_ir v17.2d, v5.2d, v12.2d[0] OP_ir v17.2d, v5.2d, v12.d[0]
ld2 {v8.2d, v9.2d}, [pB] // For next round ld2 {v8.2d, v9.2d}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
OP_rr v18.2d, v6.2d, v12.2d[0] OP_rr v18.2d, v6.2d, v12.d[0]
OP_ii v18.2d, v7.2d, v13.2d[0] OP_ii v18.2d, v7.2d, v13.d[0]
OP_ri v19.2d, v6.2d, v13.2d[0] OP_ri v19.2d, v6.2d, v13.d[0]
OP_ir v19.2d, v7.2d, v12.2d[0] OP_ir v19.2d, v7.2d, v12.d[0]
ld2 {v10.2d, v11.2d}, [pB] // For next round ld2 {v10.2d, v11.2d}, [pB] // For next round
add pB, pB, #32 add pB, pB, #32
OP_rr v20.2d, v4.2d, v12.2d[1] OP_rr v20.2d, v4.2d, v12.d[1]
OP_ii v20.2d, v5.2d, v13.2d[1] OP_ii v20.2d, v5.2d, v13.d[1]
OP_ri v21.2d, v4.2d, v13.2d[1] OP_ri v21.2d, v4.2d, v13.d[1]
OP_ir v21.2d, v5.2d, v12.2d[1] OP_ir v21.2d, v5.2d, v12.d[1]
ld2 {v0.2d, v1.2d}, [pA] // For next round ld2 {v0.2d, v1.2d}, [pA] // For next round
add pA, pA, #32 add pA, pA, #32
OP_rr v22.2d, v6.2d, v12.2d[1] OP_rr v22.2d, v6.2d, v12.d[1]
OP_ii v22.2d, v7.2d, v13.2d[1] OP_ii v22.2d, v7.2d, v13.d[1]
OP_ri v23.2d, v6.2d, v13.2d[1] OP_ri v23.2d, v6.2d, v13.d[1]
OP_ir v23.2d, v7.2d, v12.2d[1] OP_ir v23.2d, v7.2d, v12.d[1]
ld2 {v2.2d, v3.2d}, [pA] // For next round ld2 {v2.2d, v3.2d}, [pA] // For next round
add pA, pA, #32 add pA, pA, #32
OP_rr v24.2d, v4.2d, v14.2d[0] OP_rr v24.2d, v4.2d, v14.d[0]
OP_ii v24.2d, v5.2d, v15.2d[0] OP_ii v24.2d, v5.2d, v15.d[0]
OP_ri v25.2d, v4.2d, v15.2d[0] OP_ri v25.2d, v4.2d, v15.d[0]
OP_ir v25.2d, v5.2d, v14.2d[0] OP_ir v25.2d, v5.2d, v14.d[0]
prfm PLDL1KEEP, [pA, #512] prfm PLDL1KEEP, [pA, #512]
OP_rr v26.2d, v6.2d, v14.2d[0] OP_rr v26.2d, v6.2d, v14.d[0]
OP_ii v26.2d, v7.2d, v15.2d[0] OP_ii v26.2d, v7.2d, v15.d[0]
OP_ri v27.2d, v6.2d, v15.2d[0] OP_ri v27.2d, v6.2d, v15.d[0]
OP_ir v27.2d, v7.2d, v14.2d[0] OP_ir v27.2d, v7.2d, v14.d[0]
prfm PLDL1KEEP, [pB, #512] prfm PLDL1KEEP, [pB, #512]
OP_rr v28.2d, v4.2d, v14.2d[1] OP_rr v28.2d, v4.2d, v14.d[1]
OP_ii v28.2d, v5.2d, v15.2d[1] OP_ii v28.2d, v5.2d, v15.d[1]
OP_ri v29.2d, v4.2d, v15.2d[1] OP_ri v29.2d, v4.2d, v15.d[1]
OP_ir v29.2d, v5.2d, v14.2d[1] OP_ir v29.2d, v5.2d, v14.d[1]
OP_rr v30.2d, v6.2d, v14.2d[1] OP_rr v30.2d, v6.2d, v14.d[1]
OP_ii v30.2d, v7.2d, v15.2d[1] OP_ii v30.2d, v7.2d, v15.d[1]
OP_ri v31.2d, v6.2d, v15.2d[1] OP_ri v31.2d, v6.2d, v15.d[1]
OP_ir v31.2d, v7.2d, v14.2d[1] OP_ir v31.2d, v7.2d, v14.d[1]
.endm .endm
.macro KERNEL4x4_E .macro KERNEL4x4_E
OP_rr v16.2d, v4.2d, v12.2d[0] OP_rr v16.2d, v4.2d, v12.d[0]
OP_ii v16.2d, v5.2d, v13.2d[0] OP_ii v16.2d, v5.2d, v13.d[0]
OP_ri v17.2d, v4.2d, v13.2d[0] OP_ri v17.2d, v4.2d, v13.d[0]
OP_ir v17.2d, v5.2d, v12.2d[0] OP_ir v17.2d, v5.2d, v12.d[0]
OP_rr v18.2d, v6.2d, v12.2d[0] OP_rr v18.2d, v6.2d, v12.d[0]
OP_ii v18.2d, v7.2d, v13.2d[0] OP_ii v18.2d, v7.2d, v13.d[0]
OP_ri v19.2d, v6.2d, v13.2d[0] OP_ri v19.2d, v6.2d, v13.d[0]
OP_ir v19.2d, v7.2d, v12.2d[0] OP_ir v19.2d, v7.2d, v12.d[0]
OP_rr v20.2d, v4.2d, v12.2d[1] OP_rr v20.2d, v4.2d, v12.d[1]
OP_ii v20.2d, v5.2d, v13.2d[1] OP_ii v20.2d, v5.2d, v13.d[1]
OP_ri v21.2d, v4.2d, v13.2d[1] OP_ri v21.2d, v4.2d, v13.d[1]
OP_ir v21.2d, v5.2d, v12.2d[1] OP_ir v21.2d, v5.2d, v12.d[1]
OP_rr v22.2d, v6.2d, v12.2d[1] OP_rr v22.2d, v6.2d, v12.d[1]
OP_ii v22.2d, v7.2d, v13.2d[1] OP_ii v22.2d, v7.2d, v13.d[1]
OP_ri v23.2d, v6.2d, v13.2d[1] OP_ri v23.2d, v6.2d, v13.d[1]
OP_ir v23.2d, v7.2d, v12.2d[1] OP_ir v23.2d, v7.2d, v12.d[1]
OP_rr v24.2d, v4.2d, v14.2d[0] OP_rr v24.2d, v4.2d, v14.d[0]
OP_ii v24.2d, v5.2d, v15.2d[0] OP_ii v24.2d, v5.2d, v15.d[0]
OP_ri v25.2d, v4.2d, v15.2d[0] OP_ri v25.2d, v4.2d, v15.d[0]
OP_ir v25.2d, v5.2d, v14.2d[0] OP_ir v25.2d, v5.2d, v14.d[0]
OP_rr v26.2d, v6.2d, v14.2d[0] OP_rr v26.2d, v6.2d, v14.d[0]
OP_ii v26.2d, v7.2d, v15.2d[0] OP_ii v26.2d, v7.2d, v15.d[0]
OP_ri v27.2d, v6.2d, v15.2d[0] OP_ri v27.2d, v6.2d, v15.d[0]
OP_ir v27.2d, v7.2d, v14.2d[0] OP_ir v27.2d, v7.2d, v14.d[0]
OP_rr v28.2d, v4.2d, v14.2d[1] OP_rr v28.2d, v4.2d, v14.d[1]
OP_ii v28.2d, v5.2d, v15.2d[1] OP_ii v28.2d, v5.2d, v15.d[1]
OP_ri v29.2d, v4.2d, v15.2d[1] OP_ri v29.2d, v4.2d, v15.d[1]
OP_ir v29.2d, v5.2d, v14.2d[1] OP_ir v29.2d, v5.2d, v14.d[1]
OP_rr v30.2d, v6.2d, v14.2d[1] OP_rr v30.2d, v6.2d, v14.d[1]
OP_ii v30.2d, v7.2d, v15.2d[1] OP_ii v30.2d, v7.2d, v15.d[1]
OP_ri v31.2d, v6.2d, v15.2d[1] OP_ri v31.2d, v6.2d, v15.d[1]
OP_ir v31.2d, v7.2d, v14.2d[1] OP_ir v31.2d, v7.2d, v14.d[1]
.endm .endm
.macro KERNEL4x4_SUB .macro KERNEL4x4_SUB
@ -451,45 +451,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v2.2d, v3.2d}, [pA] ld2 {v2.2d, v3.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
OP_rr v16.2d, v0.2d, v8.2d[0] OP_rr v16.2d, v0.2d, v8.d[0]
OP_ii v16.2d, v1.2d, v9.2d[0] OP_ii v16.2d, v1.2d, v9.d[0]
OP_ri v17.2d, v0.2d, v9.2d[0] OP_ri v17.2d, v0.2d, v9.d[0]
OP_ir v17.2d, v1.2d, v8.2d[0] OP_ir v17.2d, v1.2d, v8.d[0]
OP_rr v18.2d, v2.2d, v8.2d[0] OP_rr v18.2d, v2.2d, v8.d[0]
OP_ii v18.2d, v3.2d, v9.2d[0] OP_ii v18.2d, v3.2d, v9.d[0]
OP_ri v19.2d, v2.2d, v9.2d[0] OP_ri v19.2d, v2.2d, v9.d[0]
OP_ir v19.2d, v3.2d, v8.2d[0] OP_ir v19.2d, v3.2d, v8.d[0]
OP_rr v20.2d, v0.2d, v8.2d[1] OP_rr v20.2d, v0.2d, v8.d[1]
OP_ii v20.2d, v1.2d, v9.2d[1] OP_ii v20.2d, v1.2d, v9.d[1]
OP_ri v21.2d, v0.2d, v9.2d[1] OP_ri v21.2d, v0.2d, v9.d[1]
OP_ir v21.2d, v1.2d, v8.2d[1] OP_ir v21.2d, v1.2d, v8.d[1]
OP_rr v22.2d, v2.2d, v8.2d[1] OP_rr v22.2d, v2.2d, v8.d[1]
OP_ii v22.2d, v3.2d, v9.2d[1] OP_ii v22.2d, v3.2d, v9.d[1]
OP_ri v23.2d, v2.2d, v9.2d[1] OP_ri v23.2d, v2.2d, v9.d[1]
OP_ir v23.2d, v3.2d, v8.2d[1] OP_ir v23.2d, v3.2d, v8.d[1]
OP_rr v24.2d, v0.2d, v10.2d[0] OP_rr v24.2d, v0.2d, v10.d[0]
OP_ii v24.2d, v1.2d, v11.2d[0] OP_ii v24.2d, v1.2d, v11.d[0]
OP_ri v25.2d, v0.2d, v11.2d[0] OP_ri v25.2d, v0.2d, v11.d[0]
OP_ir v25.2d, v1.2d, v10.2d[0] OP_ir v25.2d, v1.2d, v10.d[0]
OP_rr v26.2d, v2.2d, v10.2d[0] OP_rr v26.2d, v2.2d, v10.d[0]
OP_ii v26.2d, v3.2d, v11.2d[0] OP_ii v26.2d, v3.2d, v11.d[0]
OP_ri v27.2d, v2.2d, v11.2d[0] OP_ri v27.2d, v2.2d, v11.d[0]
OP_ir v27.2d, v3.2d, v10.2d[0] OP_ir v27.2d, v3.2d, v10.d[0]
OP_rr v28.2d, v0.2d, v10.2d[1] OP_rr v28.2d, v0.2d, v10.d[1]
OP_ii v28.2d, v1.2d, v11.2d[1] OP_ii v28.2d, v1.2d, v11.d[1]
OP_ri v29.2d, v0.2d, v11.2d[1] OP_ri v29.2d, v0.2d, v11.d[1]
OP_ir v29.2d, v1.2d, v10.2d[1] OP_ir v29.2d, v1.2d, v10.d[1]
OP_rr v30.2d, v2.2d, v10.2d[1] OP_rr v30.2d, v2.2d, v10.d[1]
OP_ii v30.2d, v3.2d, v11.2d[1] OP_ii v30.2d, v3.2d, v11.d[1]
OP_ri v31.2d, v2.2d, v11.2d[1] OP_ri v31.2d, v2.2d, v11.d[1]
OP_ir v31.2d, v3.2d, v10.2d[1] OP_ir v31.2d, v3.2d, v10.d[1]
.endm .endm
.macro SAVE4x4 .macro SAVE4x4
@ -577,25 +577,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.2d, v1.2d}, [pA] ld2 {v0.2d, v1.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
OP_rr v16.2d, v0.2d, v8.2d[0] OP_rr v16.2d, v0.2d, v8.d[0]
OP_ii v16.2d, v1.2d, v9.2d[0] OP_ii v16.2d, v1.2d, v9.d[0]
OP_ri v17.2d, v0.2d, v9.2d[0] OP_ri v17.2d, v0.2d, v9.d[0]
OP_ir v17.2d, v1.2d, v8.2d[0] OP_ir v17.2d, v1.2d, v8.d[0]
OP_rr v20.2d, v0.2d, v8.2d[1] OP_rr v20.2d, v0.2d, v8.d[1]
OP_ii v20.2d, v1.2d, v9.2d[1] OP_ii v20.2d, v1.2d, v9.d[1]
OP_ri v21.2d, v0.2d, v9.2d[1] OP_ri v21.2d, v0.2d, v9.d[1]
OP_ir v21.2d, v1.2d, v8.2d[1] OP_ir v21.2d, v1.2d, v8.d[1]
OP_rr v24.2d, v0.2d, v10.2d[0] OP_rr v24.2d, v0.2d, v10.d[0]
OP_ii v24.2d, v1.2d, v11.2d[0] OP_ii v24.2d, v1.2d, v11.d[0]
OP_ri v25.2d, v0.2d, v11.2d[0] OP_ri v25.2d, v0.2d, v11.d[0]
OP_ir v25.2d, v1.2d, v10.2d[0] OP_ir v25.2d, v1.2d, v10.d[0]
OP_rr v28.2d, v0.2d, v10.2d[1] OP_rr v28.2d, v0.2d, v10.d[1]
OP_ii v28.2d, v1.2d, v11.2d[1] OP_ii v28.2d, v1.2d, v11.d[1]
OP_ri v29.2d, v0.2d, v11.2d[1] OP_ri v29.2d, v0.2d, v11.d[1]
OP_ir v29.2d, v1.2d, v10.2d[1] OP_ir v29.2d, v1.2d, v10.d[1]
.endm .endm
.macro SAVE2x4 .macro SAVE2x4
@ -660,25 +660,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.d, v1.d}[0], [pA] ld2 {v0.d, v1.d}[0], [pA]
add pA, pA, #16 add pA, pA, #16
OP_rr d16, d0, v8.2d[0] OP_rr d16, d0, v8.d[0]
OP_ii d16, d1, v9.2d[0] OP_ii d16, d1, v9.d[0]
OP_ri d17, d0, v9.2d[0] OP_ri d17, d0, v9.d[0]
OP_ir d17, d1, v8.2d[0] OP_ir d17, d1, v8.d[0]
OP_rr d20, d0, v8.2d[1] OP_rr d20, d0, v8.d[1]
OP_ii d20, d1, v9.2d[1] OP_ii d20, d1, v9.d[1]
OP_ri d21, d0, v9.2d[1] OP_ri d21, d0, v9.d[1]
OP_ir d21, d1, v8.2d[1] OP_ir d21, d1, v8.d[1]
OP_rr d24, d0, v10.2d[0] OP_rr d24, d0, v10.d[0]
OP_ii d24, d1, v11.2d[0] OP_ii d24, d1, v11.d[0]
OP_ri d25, d0, v11.2d[0] OP_ri d25, d0, v11.d[0]
OP_ir d25, d1, v10.2d[0] OP_ir d25, d1, v10.d[0]
OP_rr d28, d0, v10.2d[1] OP_rr d28, d0, v10.d[1]
OP_ii d28, d1, v11.2d[1] OP_ii d28, d1, v11.d[1]
OP_ri d29, d0, v11.2d[1] OP_ri d29, d0, v11.d[1]
OP_ir d29, d1, v10.2d[1] OP_ir d29, d1, v10.d[1]
.endm .endm
.macro SAVE1x4 .macro SAVE1x4
@ -743,25 +743,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v2.2d, v3.2d}, [pA] ld2 {v2.2d, v3.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
OP_rr v16.2d, v0.2d, v8.2d[0] OP_rr v16.2d, v0.2d, v8.d[0]
OP_ii v16.2d, v1.2d, v9.2d[0] OP_ii v16.2d, v1.2d, v9.d[0]
OP_ri v17.2d, v0.2d, v9.2d[0] OP_ri v17.2d, v0.2d, v9.d[0]
OP_ir v17.2d, v1.2d, v8.2d[0] OP_ir v17.2d, v1.2d, v8.d[0]
OP_rr v18.2d, v2.2d, v8.2d[0] OP_rr v18.2d, v2.2d, v8.d[0]
OP_ii v18.2d, v3.2d, v9.2d[0] OP_ii v18.2d, v3.2d, v9.d[0]
OP_ri v19.2d, v2.2d, v9.2d[0] OP_ri v19.2d, v2.2d, v9.d[0]
OP_ir v19.2d, v3.2d, v8.2d[0] OP_ir v19.2d, v3.2d, v8.d[0]
OP_rr v20.2d, v0.2d, v8.2d[1] OP_rr v20.2d, v0.2d, v8.d[1]
OP_ii v20.2d, v1.2d, v9.2d[1] OP_ii v20.2d, v1.2d, v9.d[1]
OP_ri v21.2d, v0.2d, v9.2d[1] OP_ri v21.2d, v0.2d, v9.d[1]
OP_ir v21.2d, v1.2d, v8.2d[1] OP_ir v21.2d, v1.2d, v8.d[1]
OP_rr v22.2d, v2.2d, v8.2d[1] OP_rr v22.2d, v2.2d, v8.d[1]
OP_ii v22.2d, v3.2d, v9.2d[1] OP_ii v22.2d, v3.2d, v9.d[1]
OP_ri v23.2d, v2.2d, v9.2d[1] OP_ri v23.2d, v2.2d, v9.d[1]
OP_ir v23.2d, v3.2d, v8.2d[1] OP_ir v23.2d, v3.2d, v8.d[1]
.endm .endm
.macro SAVE4x2 .macro SAVE4x2
@ -816,15 +816,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.2d, v1.2d}, [pA] ld2 {v0.2d, v1.2d}, [pA]
add pA, pA, #32 add pA, pA, #32
OP_rr v16.2d, v0.2d, v8.2d[0] OP_rr v16.2d, v0.2d, v8.d[0]
OP_ii v16.2d, v1.2d, v9.2d[0] OP_ii v16.2d, v1.2d, v9.d[0]
OP_ri v17.2d, v0.2d, v9.2d[0] OP_ri v17.2d, v0.2d, v9.d[0]
OP_ir v17.2d, v1.2d, v8.2d[0] OP_ir v17.2d, v1.2d, v8.d[0]
OP_rr v20.2d, v0.2d, v8.2d[1] OP_rr v20.2d, v0.2d, v8.d[1]
OP_ii v20.2d, v1.2d, v9.2d[1] OP_ii v20.2d, v1.2d, v9.d[1]
OP_ri v21.2d, v0.2d, v9.2d[1] OP_ri v21.2d, v0.2d, v9.d[1]
OP_ir v21.2d, v1.2d, v8.2d[1] OP_ir v21.2d, v1.2d, v8.d[1]
.endm .endm
.macro SAVE2x2 .macro SAVE2x2
@ -867,15 +867,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ld2 {v0.d, v1.d}[0], [pA] ld2 {v0.d, v1.d}[0], [pA]
add pA, pA, #16 add pA, pA, #16
OP_rr d16, d0, v8.2d[0] OP_rr d16, d0, v8.d[0]
OP_ii d16, d1, v9.2d[0] OP_ii d16, d1, v9.d[0]
OP_ri d17, d0, v9.2d[0] OP_ri d17, d0, v9.d[0]
OP_ir d17, d1, v8.2d[0] OP_ir d17, d1, v8.d[0]
OP_rr d20, d0, v8.2d[1] OP_rr d20, d0, v8.d[1]
OP_ii d20, d1, v9.2d[1] OP_ii d20, d1, v9.d[1]
OP_ri d21, d0, v9.2d[1] OP_ri d21, d0, v9.d[1]
OP_ir d21, d1, v8.2d[1] OP_ir d21, d1, v8.d[1]
.endm .endm
.macro SAVE1x2 .macro SAVE1x2