floraachy
/
OpenBLAS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Convert aligned moves to unaligned 

should have no performance impact on reasonably modern cpus and fixes occasional crashes in actual user code.
This commit is contained in:
Martin Kroeker 2020-04-13 14:58:52 +02:00 committed by GitHub
parent 20d0cb2f65
commit 5b0093b5fe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 93 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

186
kernel/x86_64/copy_sse2.S
View File

@ -54,7 +54,7 @@
#ifdef OPTERON #ifdef OPTERON
#define LOAD(OFFSET, ADDR, REG) xorps REG, REG; addpd OFFSET(ADDR), REG #define LOAD(OFFSET, ADDR, REG) xorps REG, REG; addpd OFFSET(ADDR), REG
#else #else
#define LOAD(OFFSET, ADDR, REG) movaps OFFSET(ADDR), REG #define LOAD(OFFSET, ADDR, REG) movups OFFSET(ADDR), REG
#endif #endif
PROLOGUE PROLOGUE
@ -104,14 +104,14 @@
sarq $4, %rax sarq $4, %rax
jle .L13 jle .L13
movaps -16 * SIZE(X), %xmm0 movups -16 * SIZE(X), %xmm0
movaps -14 * SIZE(X), %xmm1 movups -14 * SIZE(X), %xmm1
movaps -12 * SIZE(X), %xmm2 movups -12 * SIZE(X), %xmm2
movaps -10 * SIZE(X), %xmm3 movups -10 * SIZE(X), %xmm3
movaps -8 * SIZE(X), %xmm4 movups -8 * SIZE(X), %xmm4
movaps -6 * SIZE(X), %xmm5 movups -6 * SIZE(X), %xmm5
movaps -4 * SIZE(X), %xmm6 movups -4 * SIZE(X), %xmm6
movaps -2 * SIZE(X), %xmm7 movups -2 * SIZE(X), %xmm7
decq %rax decq %rax
jle .L12 jle .L12
@ -122,36 +122,36 @@
PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y)
#endif #endif
movaps %xmm0, -16 * SIZE(Y) movups %xmm0, -16 * SIZE(Y)
LOAD( 0 * SIZE, X, %xmm0) LOAD( 0 * SIZE, X, %xmm0)
movaps %xmm1, -14 * SIZE(Y) movups %xmm1, -14 * SIZE(Y)
LOAD( 2 * SIZE, X, %xmm1) LOAD( 2 * SIZE, X, %xmm1)
#ifdef PREFETCH #ifdef PREFETCH
PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X)
#endif #endif
movaps %xmm2, -12 * SIZE(Y) movups %xmm2, -12 * SIZE(Y)
LOAD( 4 * SIZE, X, %xmm2) LOAD( 4 * SIZE, X, %xmm2)
movaps %xmm3, -10 * SIZE(Y) movups %xmm3, -10 * SIZE(Y)
LOAD( 6 * SIZE, X, %xmm3) LOAD( 6 * SIZE, X, %xmm3)
#if defined(PREFETCHW) && !defined(FETCH128) #if defined(PREFETCHW) && !defined(FETCH128)
PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(Y) PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(Y)
#endif #endif
movaps %xmm4, -8 * SIZE(Y) movups %xmm4, -8 * SIZE(Y)
LOAD( 8 * SIZE, X, %xmm4) LOAD( 8 * SIZE, X, %xmm4)
movaps %xmm5, -6 * SIZE(Y) movups %xmm5, -6 * SIZE(Y)
LOAD(10 * SIZE, X, %xmm5) LOAD(10 * SIZE, X, %xmm5)
#if defined(PREFETCH) && !defined(FETCH128) #if defined(PREFETCH) && !defined(FETCH128)
PREFETCH (PREFETCHSIZE + 64) - PREOFFSET(X) PREFETCH (PREFETCHSIZE + 64) - PREOFFSET(X)
#endif #endif
movaps %xmm6, -4 * SIZE(Y) movups %xmm6, -4 * SIZE(Y)
LOAD(12 * SIZE, X, %xmm6) LOAD(12 * SIZE, X, %xmm6)
movaps %xmm7, -2 * SIZE(Y) movups %xmm7, -2 * SIZE(Y)
LOAD(14 * SIZE, X, %xmm7) LOAD(14 * SIZE, X, %xmm7)
subq $-16 * SIZE, Y subq $-16 * SIZE, Y
@ -161,14 +161,14 @@
ALIGN_3 ALIGN_3
.L12: .L12:
movaps %xmm0, -16 * SIZE(Y) movups %xmm0, -16 * SIZE(Y)
movaps %xmm1, -14 * SIZE(Y) movups %xmm1, -14 * SIZE(Y)
movaps %xmm2, -12 * SIZE(Y) movups %xmm2, -12 * SIZE(Y)
movaps %xmm3, -10 * SIZE(Y) movups %xmm3, -10 * SIZE(Y)
movaps %xmm4, -8 * SIZE(Y) movups %xmm4, -8 * SIZE(Y)
movaps %xmm5, -6 * SIZE(Y) movups %xmm5, -6 * SIZE(Y)
movaps %xmm6, -4 * SIZE(Y) movups %xmm6, -4 * SIZE(Y)
movaps %xmm7, -2 * SIZE(Y) movups %xmm7, -2 * SIZE(Y)
subq $-16 * SIZE, Y subq $-16 * SIZE, Y
subq $-16 * SIZE, X subq $-16 * SIZE, X
@ -179,15 +179,15 @@
jle .L14 jle .L14
ALIGN_3 ALIGN_3
movaps -16 * SIZE(X), %xmm0 movups -16 * SIZE(X), %xmm0
movaps -14 * SIZE(X), %xmm1 movups -14 * SIZE(X), %xmm1
movaps -12 * SIZE(X), %xmm2 movups -12 * SIZE(X), %xmm2
movaps -10 * SIZE(X), %xmm3 movups -10 * SIZE(X), %xmm3
movaps %xmm0, -16 * SIZE(Y) movups %xmm0, -16 * SIZE(Y)
movaps %xmm1, -14 * SIZE(Y) movups %xmm1, -14 * SIZE(Y)
movaps %xmm2, -12 * SIZE(Y) movups %xmm2, -12 * SIZE(Y)
movaps %xmm3, -10 * SIZE(Y) movups %xmm3, -10 * SIZE(Y)
addq $8 * SIZE, X addq $8 * SIZE, X
addq $8 * SIZE, Y addq $8 * SIZE, Y
@ -198,11 +198,11 @@
jle .L15 jle .L15
ALIGN_3 ALIGN_3
movaps -16 * SIZE(X), %xmm0 movups -16 * SIZE(X), %xmm0
movaps -14 * SIZE(X), %xmm1 movups -14 * SIZE(X), %xmm1
movaps %xmm0, -16 * SIZE(Y) movups %xmm0, -16 * SIZE(Y)
movaps %xmm1, -14 * SIZE(Y) movups %xmm1, -14 * SIZE(Y)
addq $4 * SIZE, X addq $4 * SIZE, X
addq $4 * SIZE, Y addq $4 * SIZE, Y
@ -213,8 +213,8 @@
jle .L16 jle .L16
ALIGN_3 ALIGN_3
movaps -16 * SIZE(X), %xmm0 movups -16 * SIZE(X), %xmm0
movaps %xmm0, -16 * SIZE(Y) movups %xmm0, -16 * SIZE(Y)
addq $2 * SIZE, X addq $2 * SIZE, X
addq $2 * SIZE, Y addq $2 * SIZE, Y
@ -246,13 +246,13 @@
sarq $4, %rax sarq $4, %rax
jle .L23 jle .L23
movaps -15 * SIZE(X), %xmm1 movups -15 * SIZE(X), %xmm1
movaps -13 * SIZE(X), %xmm2 movups -13 * SIZE(X), %xmm2
movaps -11 * SIZE(X), %xmm3 movups -11 * SIZE(X), %xmm3
movaps -9 * SIZE(X), %xmm4 movups -9 * SIZE(X), %xmm4
movaps -7 * SIZE(X), %xmm5 movups -7 * SIZE(X), %xmm5
movaps -5 * SIZE(X), %xmm6 movups -5 * SIZE(X), %xmm6
movaps -3 * SIZE(X), %xmm7 movups -3 * SIZE(X), %xmm7
decq %rax decq %rax
jle .L22 jle .L22
@ -264,11 +264,11 @@
#endif #endif
SHUFPD_1 %xmm1, %xmm0 SHUFPD_1 %xmm1, %xmm0
movaps %xmm0, -16 * SIZE(Y) movups %xmm0, -16 * SIZE(Y)
LOAD(-1 * SIZE, X, %xmm0) LOAD(-1 * SIZE, X, %xmm0)
SHUFPD_1 %xmm2, %xmm1 SHUFPD_1 %xmm2, %xmm1
movaps %xmm1, -14 * SIZE(Y) movups %xmm1, -14 * SIZE(Y)
LOAD( 1 * SIZE, X, %xmm1) LOAD( 1 * SIZE, X, %xmm1)
#ifdef PREFETCH #ifdef PREFETCH
@ -276,11 +276,11 @@
#endif #endif
SHUFPD_1 %xmm3, %xmm2 SHUFPD_1 %xmm3, %xmm2
movaps %xmm2, -12 * SIZE(Y) movups %xmm2, -12 * SIZE(Y)
LOAD( 3 * SIZE, X, %xmm2) LOAD( 3 * SIZE, X, %xmm2)
SHUFPD_1 %xmm4, %xmm3 SHUFPD_1 %xmm4, %xmm3
movaps %xmm3, -10 * SIZE(Y) movups %xmm3, -10 * SIZE(Y)
LOAD( 5 * SIZE, X, %xmm3) LOAD( 5 * SIZE, X, %xmm3)
#if defined(PREFETCHW) && !defined(FETCH128) #if defined(PREFETCHW) && !defined(FETCH128)
@ -288,11 +288,11 @@
#endif #endif
SHUFPD_1 %xmm5, %xmm4 SHUFPD_1 %xmm5, %xmm4
movaps %xmm4, -8 * SIZE(Y) movups %xmm4, -8 * SIZE(Y)
LOAD( 7 * SIZE, X, %xmm4) LOAD( 7 * SIZE, X, %xmm4)
SHUFPD_1 %xmm6, %xmm5 SHUFPD_1 %xmm6, %xmm5
movaps %xmm5, -6 * SIZE(Y) movups %xmm5, -6 * SIZE(Y)
LOAD( 9 * SIZE, X, %xmm5) LOAD( 9 * SIZE, X, %xmm5)
#if defined(PREFETCH) && !defined(FETCH128) #if defined(PREFETCH) && !defined(FETCH128)
@ -300,11 +300,11 @@
#endif #endif
SHUFPD_1 %xmm7, %xmm6 SHUFPD_1 %xmm7, %xmm6
movaps %xmm6, -4 * SIZE(Y) movups %xmm6, -4 * SIZE(Y)
LOAD(11 * SIZE, X, %xmm6) LOAD(11 * SIZE, X, %xmm6)
SHUFPD_1 %xmm0, %xmm7 SHUFPD_1 %xmm0, %xmm7
movaps %xmm7, -2 * SIZE(Y) movups %xmm7, -2 * SIZE(Y)
LOAD(13 * SIZE, X, %xmm7) LOAD(13 * SIZE, X, %xmm7)
subq $-16 * SIZE, X subq $-16 * SIZE, X
@ -315,26 +315,26 @@
.L22: .L22:
SHUFPD_1 %xmm1, %xmm0 SHUFPD_1 %xmm1, %xmm0
movaps %xmm0, -16 * SIZE(Y) movups %xmm0, -16 * SIZE(Y)
LOAD(-1 * SIZE, X, %xmm0) LOAD(-1 * SIZE, X, %xmm0)
SHUFPD_1 %xmm2, %xmm1 SHUFPD_1 %xmm2, %xmm1
movaps %xmm1, -14 * SIZE(Y) movups %xmm1, -14 * SIZE(Y)
SHUFPD_1 %xmm3, %xmm2 SHUFPD_1 %xmm3, %xmm2
movaps %xmm2, -12 * SIZE(Y) movups %xmm2, -12 * SIZE(Y)
SHUFPD_1 %xmm4, %xmm3 SHUFPD_1 %xmm4, %xmm3
movaps %xmm3, -10 * SIZE(Y) movups %xmm3, -10 * SIZE(Y)
SHUFPD_1 %xmm5, %xmm4 SHUFPD_1 %xmm5, %xmm4
movaps %xmm4, -8 * SIZE(Y) movups %xmm4, -8 * SIZE(Y)
SHUFPD_1 %xmm6, %xmm5 SHUFPD_1 %xmm6, %xmm5
movaps %xmm5, -6 * SIZE(Y) movups %xmm5, -6 * SIZE(Y)
SHUFPD_1 %xmm7, %xmm6 SHUFPD_1 %xmm7, %xmm6
movaps %xmm6, -4 * SIZE(Y) movups %xmm6, -4 * SIZE(Y)
SHUFPD_1 %xmm0, %xmm7 SHUFPD_1 %xmm0, %xmm7
movaps %xmm7, -2 * SIZE(Y) movups %xmm7, -2 * SIZE(Y)
subq $-16 * SIZE, X subq $-16 * SIZE, X
subq $-16 * SIZE, Y subq $-16 * SIZE, Y
@ -345,24 +345,24 @@
jle .L24 jle .L24
ALIGN_3 ALIGN_3
movaps -15 * SIZE(X), %xmm1 movups -15 * SIZE(X), %xmm1
movaps -13 * SIZE(X), %xmm2 movups -13 * SIZE(X), %xmm2
movaps -11 * SIZE(X), %xmm3 movups -11 * SIZE(X), %xmm3
movaps -9 * SIZE(X), %xmm8 movups -9 * SIZE(X), %xmm8
SHUFPD_1 %xmm1, %xmm0 SHUFPD_1 %xmm1, %xmm0
movaps %xmm0, -16 * SIZE(Y) movups %xmm0, -16 * SIZE(Y)
SHUFPD_1 %xmm2, %xmm1 SHUFPD_1 %xmm2, %xmm1
movaps %xmm1, -14 * SIZE(Y) movups %xmm1, -14 * SIZE(Y)
SHUFPD_1 %xmm3, %xmm2 SHUFPD_1 %xmm3, %xmm2
movaps %xmm2, -12 * SIZE(Y) movups %xmm2, -12 * SIZE(Y)
SHUFPD_1 %xmm8, %xmm3 SHUFPD_1 %xmm8, %xmm3
movaps %xmm3, -10 * SIZE(Y) movups %xmm3, -10 * SIZE(Y)
movaps %xmm8, %xmm0 movups %xmm8, %xmm0
addq $8 * SIZE, X addq $8 * SIZE, X
addq $8 * SIZE, Y addq $8 * SIZE, Y
@ -373,15 +373,15 @@
jle .L25 jle .L25
ALIGN_3 ALIGN_3
movaps -15 * SIZE(X), %xmm1 movups -15 * SIZE(X), %xmm1
movaps -13 * SIZE(X), %xmm2 movups -13 * SIZE(X), %xmm2
SHUFPD_1 %xmm1, %xmm0 SHUFPD_1 %xmm1, %xmm0
SHUFPD_1 %xmm2, %xmm1 SHUFPD_1 %xmm2, %xmm1
movaps %xmm0, -16 * SIZE(Y) movups %xmm0, -16 * SIZE(Y)
movaps %xmm1, -14 * SIZE(Y) movups %xmm1, -14 * SIZE(Y)
movaps %xmm2, %xmm0 movups %xmm2, %xmm0
addq $4 * SIZE, X addq $4 * SIZE, X
addq $4 * SIZE, Y addq $4 * SIZE, Y
@ -392,10 +392,10 @@
jle .L26 jle .L26
ALIGN_3 ALIGN_3
movaps -15 * SIZE(X), %xmm1 movups -15 * SIZE(X), %xmm1
SHUFPD_1 %xmm1, %xmm0 SHUFPD_1 %xmm1, %xmm0
movaps %xmm0, -16 * SIZE(Y) movups %xmm0, -16 * SIZE(Y)
addq $2 * SIZE, X addq $2 * SIZE, X
addq $2 * SIZE, Y addq $2 * SIZE, Y
@ -424,14 +424,14 @@
sarq $4, %rax sarq $4, %rax
jle .L23 jle .L23
movaps -16 * SIZE(X), %xmm0 movups -16 * SIZE(X), %xmm0
movaps -14 * SIZE(X), %xmm1 movups -14 * SIZE(X), %xmm1
movaps -12 * SIZE(X), %xmm2 movups -12 * SIZE(X), %xmm2
movaps -10 * SIZE(X), %xmm3 movups -10 * SIZE(X), %xmm3
movaps -8 * SIZE(X), %xmm4 movups -8 * SIZE(X), %xmm4
movaps -6 * SIZE(X), %xmm5 movups -6 * SIZE(X), %xmm5
movaps -4 * SIZE(X), %xmm6 movups -4 * SIZE(X), %xmm6
movaps -2 * SIZE(X), %xmm7 movups -2 * SIZE(X), %xmm7
decq %rax decq %rax
jle .L22 jle .L22
@ -515,16 +515,16 @@
jle .L24 jle .L24
ALIGN_3 ALIGN_3
movaps -16 * SIZE(X), %xmm0 movups -16 * SIZE(X), %xmm0
movlps %xmm0, -16 * SIZE(Y) movlps %xmm0, -16 * SIZE(Y)
movhps %xmm0, -15 * SIZE(Y) movhps %xmm0, -15 * SIZE(Y)
movaps -14 * SIZE(X), %xmm1 movups -14 * SIZE(X), %xmm1
movlps %xmm1, -14 * SIZE(Y) movlps %xmm1, -14 * SIZE(Y)
movhps %xmm1, -13 * SIZE(Y) movhps %xmm1, -13 * SIZE(Y)
movaps -12 * SIZE(X), %xmm2 movups -12 * SIZE(X), %xmm2
movlps %xmm2, -12 * SIZE(Y) movlps %xmm2, -12 * SIZE(Y)
movhps %xmm2, -11 * SIZE(Y) movhps %xmm2, -11 * SIZE(Y)
movaps -10 * SIZE(X), %xmm3 movups -10 * SIZE(X), %xmm3
movlps %xmm3, -10 * SIZE(Y) movlps %xmm3, -10 * SIZE(Y)
movhps %xmm3, -9 * SIZE(Y) movhps %xmm3, -9 * SIZE(Y)
@ -537,10 +537,10 @@
jle .L25 jle .L25
ALIGN_3 ALIGN_3
movaps -16 * SIZE(X), %xmm0 movups -16 * SIZE(X), %xmm0
movlps %xmm0, -16 * SIZE(Y) movlps %xmm0, -16 * SIZE(Y)
movhps %xmm0, -15 * SIZE(Y) movhps %xmm0, -15 * SIZE(Y)
movaps -14 * SIZE(X), %xmm1 movups -14 * SIZE(X), %xmm1
movlps %xmm1, -14 * SIZE(Y) movlps %xmm1, -14 * SIZE(Y)
movhps %xmm1, -13 * SIZE(Y) movhps %xmm1, -13 * SIZE(Y)
@ -553,7 +553,7 @@
jle .L26 jle .L26
ALIGN_3 ALIGN_3
movaps -16 * SIZE(X), %xmm0 movups -16 * SIZE(X), %xmm0
movlps %xmm0, -16 * SIZE(Y) movlps %xmm0, -16 * SIZE(Y)
movhps %xmm0, -15 * SIZE(Y) movhps %xmm0, -15 * SIZE(Y)