floraachy
/
OpenBLAS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

POWER10: convert dgemv inline assembly 

This patch makes use of compiler builtins and matches with assembly
performance. Tested with clang14 and gcc12.
This commit is contained in:
Rajalakshmi Srinivasaraghavan 2022-06-09 10:42:57 -05:00
parent bfd9c1b58c
commit 8419d538ff
1 changed files with 65 additions and 320 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

385
kernel/power/dgemv_t_power10.c
View File

@ -35,327 +35,72 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#if defined(HAVE_KERNEL4x8_ASM) #if defined(HAVE_KERNEL4x8_ASM)
static void dgemv_kernel_4x8(BLASLONG n, BLASLONG lda, double *ap, double *x, double *y, double alpha) { typedef __vector unsigned char vec_t;
static void dgemv_kernel_4x8(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOAT *y, FLOAT alpha) {
BLASLONG i;
FLOAT *a0, *a1, *a2, *a3, *a4, *a5, *a6, *a7; FLOAT *a0, *a1, *a2, *a3, *a4, *a5, *a6, *a7;
BLASLONG off2; __vector_pair vx, vp;
BLASLONG tempR; vec_t res[2],res1[2];
__asm__( register __vector double temp0 = {0, 0};
register __vector double temp1 = {0, 0};
"sldi %[temp],%[off], 4 \n\t" // lda * sizeof (double) *2 register __vector double temp2 = {0, 0};
"sldi %[off], %[off], 3 \n\t" // lda * sizeof (double) register __vector double temp3 = {0, 0};
"xxlxor 34,34,34 \n\t" register __vector double temp4 = {0, 0};
"xxlxor 35,34,34 \n\t" register __vector double temp5 = {0, 0};
"add %[a2], %[a0], %[temp] \n\t" register __vector double temp6 = {0, 0};
"add %[a1], %[a0], %[off] \n\t" register __vector double temp7 = {0, 0};
"xxlxor 4,34,34 \n\t" a0 = ap;
"xxlxor 5,34,34 \n\t" a1 = ap + lda;
"xxlxor 6,34,34 \n\t" a2 = a1 + lda;
"xxlxor 7,34,34 \n\t" a3 = a2 + lda;
"add %[a3], %[a2], %[off] \n\t" a4 = a3 + lda;
"add %[a4], %[a2], %[temp] \n\t" a5 = a4 + lda;
a6 = a5 + lda;
"xxlxor 8,34,34 \n\t" a7 = a6 + lda;
"xxlxor 9,34,34 \n\t" for (i = 0; i < n/2; i += 2) {
"add %[a5], %[a3], %[temp] \n\t" vp = *((__vector_pair *)((void *)&a0[i*2]));
"li %[off],0 \n\t" vx = *((__vector_pair *)((void *)&x[i*2]));
"li %[off2],16 \n\t" __builtin_vsx_disassemble_pair (res, &vx);
__builtin_vsx_disassemble_pair (res1, &vp);
"add %[a6], %[a4], %[temp] \n\t" temp0 = vec_madd ((__vector double)res[0], (__vector double)res1[0], temp0);
"add %[a7], %[a5], %[temp] \n\t" temp0 = vec_madd ((__vector double)res[1], (__vector double)res1[1], temp0);
vp = *((__vector_pair *)((void *)&a1[i*2]));
__builtin_vsx_disassemble_pair (res1, &vp);
temp1 = vec_madd ((__vector double)res[0], (__vector double)res1[0], temp1);
temp1 = vec_madd ((__vector double)res[1], (__vector double)res1[1], temp1);
"lxvp 32, 0(%[x]) \n\t" vp = *((__vector_pair *)((void *)&a2[i*2]));
"lxvp 36, 0(%[a0]) \n\t" __builtin_vsx_disassemble_pair (res1, &vp);
"lxvp 38, 0(%[a1]) \n\t" temp2 = vec_madd ((__vector double)res[0], (__vector double)res1[0], temp2);
"lxvp 40, 0(%[a2]) \n\t" temp2 = vec_madd ((__vector double)res[1], (__vector double)res1[1], temp2);
"lxvp 42, 0(%[a3]) \n\t" vp = *((__vector_pair *)((void *)&a3[i*2]));
"lxvp 44, 0(%[a4]) \n\t" __builtin_vsx_disassemble_pair (res1, &vp);
"lxvp 46, 0(%[a5]) \n\t" temp3 = vec_madd ((__vector double)res[0], (__vector double)res1[0], temp3);
"lxvp 48, 0(%[a6]) \n\t" temp3 = vec_madd ((__vector double)res[1], (__vector double)res1[1], temp3);
"lxvp 50, 0(%[a7]) \n\t" vp = *((__vector_pair *)((void *)&a4[i*2]));
#if defined(PREFETCH) __builtin_vsx_disassemble_pair (res1, &vp);
"li %[temp],896 \n\t" temp4 = vec_madd ((__vector double)res[0], (__vector double)res1[0], temp4);
#endif temp4 = vec_madd ((__vector double)res[1], (__vector double)res1[1], temp4);
"addic. %[n],%[n],-4 \n\t" vp = *((__vector_pair *)((void *)&a5[i*2]));
__builtin_vsx_disassemble_pair (res1, &vp);
"li %[off],32 \n\t" temp5 = vec_madd ((__vector double)res[0], (__vector double)res1[0], temp5);
temp5 = vec_madd ((__vector double)res[1], (__vector double)res1[1], temp5);
vp = *((__vector_pair *)((void *)&a6[i*2]));
"ble- two%= \n\t" __builtin_vsx_disassemble_pair (res1, &vp);
temp6 = vec_madd ((__vector double)res[0], (__vector double)res1[0], temp6);
//-------------------------------------------------- temp6 = vec_madd ((__vector double)res[1], (__vector double)res1[1], temp6);
".align 5 \n\t" vp = *((__vector_pair *)((void *)&a7[i*2]));
"one%=: \n\t" __builtin_vsx_disassemble_pair (res1, &vp);
"xvmaddadp 34,36,32 \n\t" temp7 = vec_madd ((__vector double)res[0], (__vector double)res1[0], temp7);
"xvmaddadp 35,38,32 \n\t" temp7 = vec_madd ((__vector double)res[1], (__vector double)res1[1], temp7);
"addi %[off2], %[off2],32 \n\t" }
"xvmaddadp 4,40,32 \n\t" y[0] += alpha * (temp0[0] + temp0[1]);
"xvmaddadp 5,42,32 \n\t" y[1] += alpha * (temp1[0] + temp1[1]);
"xvmaddadp 6,44,32 \n\t" y[2] += alpha * (temp2[0] + temp2[1]);
"xvmaddadp 7,46,32 \n\t" y[3] += alpha * (temp3[0] + temp3[1]);
"xvmaddadp 8,48,32 \n\t" y[4] += alpha * (temp4[0] + temp4[1]);
"xvmaddadp 9,50,32 \n\t" y[5] += alpha * (temp5[0] + temp5[1]);
y[6] += alpha * (temp6[0] + temp6[1]);
"xvmaddadp 34,37,33 \n\t" y[7] += alpha * (temp7[0] + temp7[1]);
"xvmaddadp 35,39,33 \n\t"
"lxvp 36, 32(%[a0]) \n\t"
"lxvp 38, 32(%[a1]) \n\t"
"xvmaddadp 4,41,33 \n\t"
"xvmaddadp 5,43,33 \n\t"
"addi %[off], %[off],32 \n\t"
"lxvp 40, 32(%[a2]) \n\t"
"lxvp 42, 32(%[a3]) \n\t"
"xvmaddadp 6,45,33 \n\t"
"xvmaddadp 7,47,33 \n\t"
"lxvp 44, 32(%[a4]) \n\t"
"lxvp 46, 32(%[a5]) \n\t"
"xvmaddadp 8,49,33 \n\t"
"xvmaddadp 9,51,33 \n\t"
"addic. %[n],%[n],-4 \n\t"
"lxvp 48, 32(%[a6]) \n\t"
"lxvp 50, 32(%[a7]) \n\t"
"lxvp 32, 32(%[x]) \n\t"
"ble- two%= \n\t"
"xvmaddadp 34,36,32 \n\t"
"xvmaddadp 35,38,32 \n\t"
"addi %[off2], %[off2],32 \n\t"
"xvmaddadp 4,40,32 \n\t"
"xvmaddadp 5,42,32 \n\t"
"xvmaddadp 6,44,32 \n\t"
"xvmaddadp 7,46,32 \n\t"
"xvmaddadp 8,48,32 \n\t"
"xvmaddadp 9,50,32 \n\t"
"xvmaddadp 34,37,33 \n\t"
"xvmaddadp 35,39,33 \n\t"
"lxvp 36, 64(%[a0]) \n\t"
"lxvp 38, 64(%[a1]) \n\t"
"xvmaddadp 4,41,33 \n\t"
"xvmaddadp 5,43,33 \n\t"
"addi %[off], %[off],32 \n\t"
"lxvp 40, 64(%[a2]) \n\t"
"lxvp 42, 64(%[a3]) \n\t"
"xvmaddadp 6,45,33 \n\t"
"xvmaddadp 7,47,33 \n\t"
"lxvp 44, 64(%[a4]) \n\t"
"lxvp 46, 64(%[a5]) \n\t"
"xvmaddadp 8,49,33 \n\t"
"xvmaddadp 9,51,33 \n\t"
"addic. %[n],%[n],-4 \n\t"
"lxvp 48, 64(%[a6]) \n\t"
"lxvp 50, 64(%[a7]) \n\t"
"lxvp 32, 64(%[x]) \n\t"
"ble- two%= \n\t"
"xvmaddadp 34,36,32 \n\t"
"xvmaddadp 35,38,32 \n\t"
#if defined(PREFETCH)
"addi %[temp],%[temp],128 \n\t"
#endif
"addi %[off2], %[off2],32 \n\t"
"xvmaddadp 4,40,32 \n\t"
"xvmaddadp 5,42,32 \n\t"
"xvmaddadp 6,44,32 \n\t"
"xvmaddadp 7,46,32 \n\t"
"xvmaddadp 8,48,32 \n\t"
"xvmaddadp 9,50,32 \n\t"
#if defined(PREFETCH)
"dcbt %[temp],%[a0] \n\t"
#endif
"xvmaddadp 34,37,33 \n\t"
"xvmaddadp 35,39,33 \n\t"
"lxvp 36, 96(%[a0]) \n\t"
"lxvp 38, 96(%[a1]) \n\t"
"xvmaddadp 4,41,33 \n\t"
"xvmaddadp 5,43,33 \n\t"
#if defined(PREFETCH)
"dcbt %[temp],%[a1] \n\t"
#endif
"lxvp 40, 96(%[a2]) \n\t"
"lxvp 42, 96(%[a3]) \n\t"
"addi %[off], %[off],32 \n\t"
"xvmaddadp 6,45,33 \n\t"
"xvmaddadp 7,47,33 \n\t"
"lxvp 44, 96(%[a4]) \n\t"
"lxvp 46, 96(%[a5]) \n\t"
"xvmaddadp 8,49,33 \n\t"
"xvmaddadp 9,51,33 \n\t"
#if defined(PREFETCH)
"dcbt %[temp],%[a3] \n\t"
#endif
"lxvp 48, 96(%[a6]) \n\t"
"lxvp 50, 96(%[a7]) \n\t"
"lxvp 32, 96(%[x]) \n\t"
"addic. %[n],%[n],-4 \n\t"
"ble- two%= \n\t"
"addi %[off2], %[off2],32 \n\t"
#if defined(PREFETCH)
"dcbt %[temp],%[a2] \n\t"
#endif
"xvmaddadp 34,36,32 \n\t"
"xvmaddadp 35,38,32 \n\t"
"xvmaddadp 4,40,32 \n\t"
"xvmaddadp 5,42,32 \n\t"
#if defined(PREFETCH)
"dcbt %[temp],%[a4] \n\t"
#endif
"xvmaddadp 6,44,32 \n\t"
"xvmaddadp 7,46,32 \n\t"
"xvmaddadp 8,48,32 \n\t"
"xvmaddadp 9,50,32 \n\t"
#if defined(PREFETCH)
"dcbt %[temp],%[a5] \n\t"
#endif
"xvmaddadp 34,37,33 \n\t"
"xvmaddadp 35,39,33 \n\t"
"lxvp 36, 128(%[a0]) \n\t"
"lxvp 38, 128(%[a1]) \n\t"
"xvmaddadp 4,41,33 \n\t"
"xvmaddadp 5,43,33 \n\t"
"addi %[off], %[off],32 \n\t"
"lxvp 40, 128(%[a2]) \n\t"
"lxvp 42, 128(%[a3]) \n\t"
#if defined(PREFETCH)
"dcbt %[temp],%[a6] \n\t"
#endif
"xvmaddadp 6,45,33 \n\t"
"xvmaddadp 7,47,33 \n\t"
"lxvp 44, 128(%[a4]) \n\t"
"lxvp 46, 128(%[a5]) \n\t"
"xvmaddadp 8,49,33 \n\t"
"xvmaddadp 9,51,33 \n\t"
#if defined(PREFETCH)
"dcbt %[temp],%[a7] \n\t"
#endif
"addic. %[n],%[n],-4 \n\t"
"lxvp 48, 128(%[a6]) \n\t"
"lxvp 50, 128(%[a7]) \n\t"
"lxvp 32, 128(%[x]) \n\t"
#if defined(PREFETCH)
"dcbt %[temp],%[x] \n\t"
#endif
"addi %[a0], %[a0], 128 \n\t"
"addi %[a1], %[a1], 128 \n\t"
"addi %[a2], %[a2], 128 \n\t"
"addi %[a3], %[a3], 128 \n\t"
"addi %[a4], %[a4], 128 \n\t"
"addi %[a5], %[a5], 128 \n\t"
"addi %[a6], %[a6], 128 \n\t"
"addi %[a7], %[a7], 128 \n\t"
"addi %[x], %[x], 128 \n\t"
"bgt+ one%= \n\t"
".align 5 \n\t"
"two%=: \n\t"
//--------------------------------------------
"xvmaddadp 34,36,32 \n\t"
"xvmaddadp 35,38,32 \n\t"
"xvmaddadp 4,40,32 \n\t"
"xvmaddadp 5,42,32 \n\t"
"xvmaddadp 6,44,32 \n\t"
"xvmaddadp 7,46,32 \n\t"
"xvmaddadp 8,48,32 \n\t"
"xvmaddadp 9,50,32 \n\t"
XXSPLTD_S(36,%x[alpha],0)
"xvmaddadp 34,37,33 \n\t"
"xvmaddadp 35,39,33 \n\t"
"xvmaddadp 4,41,33 \n\t"
"xvmaddadp 5,43,33 \n\t"
"xvmaddadp 6,45,33 \n\t"
"xvmaddadp 7,47,33 \n\t"
"xvmaddadp 8,49,33 \n\t"
"xvmaddadp 9,51,33 \n\t"
"lxvp 38, 0(%[y]) \n\t"
"lxvp 40, 32(%[y]) \n\t"
#if (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
XXMRGHD_S(42,34,35)
XXMRGLD_S(43,34,35)
XXMRGHD_S(44,4,5)
XXMRGLD_S(45,4,5)
#else
XXMRGLD_S(42,35,34)
XXMRGHD_S(43,35,34)
XXMRGLD_S(44,5,4)
XXMRGHD_S(45,5,4)
#endif
"xvadddp 42,42,43 \n\t"
#if (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
XXMRGHD_S(46,6,7)
XXMRGLD_S(47,6,7)
#else
XXMRGLD_S(46,7,6)
XXMRGHD_S(47,7,6)
#endif
"xvadddp 44,44,45 \n\t"
#if (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
XXMRGHD_S(48,8,9)
XXMRGLD_S(49,8,9)
#else
XXMRGLD_S(48,9,8)
XXMRGHD_S(49,9,8)
#endif
"xvadddp 46,46,47 \n\t"
#if (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
"xvmaddadp 38,42,36 \n\t"
"xvmaddadp 39,44,36 \n\t"
#else
"xvmaddadp 39,42,36 \n\t"
"xvmaddadp 38,44,36 \n\t"
#endif
"xvadddp 48,48,49 \n\t"
#if (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
"xvmaddadp 41,48,36 \n\t"
#else
"xvmaddadp 41,46,36 \n\t"
#endif
"stxvp 38, 0(%[y]) \n\t"
#if (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
"xvmaddadp 40,46,36 \n\t"
#else
"xvmaddadp 40,48,36 \n\t"
#endif
"stxvp 40, 32(%[y]) \n\t"
: [memy] "+m" (*(double (*)[8])y),
[n] "+&r" (n),
[a0] "=b" (a0),
[a1] "=&b" (a1),
[a2] "=&b" (a2),
[a3] "=&b" (a3),
[a4] "=&b" (a4),
[a5] "=&b" (a5),
[a6] "=&b" (a6),
[a7] "=&b" (a7),
[off] "+&b" (lda),
[off2]"=&b" (off2),
[temp] "=&b" (tempR)
: [memx] "m" (*(const double (*)[n])x),
[mem_ap] "m" (*(const double (*)[n*8]) ap),
[alpha] "d" (alpha),
"[a0]" (ap),
[x] "b" (x),
[y] "b" (y)
: "cc","vs4","vs5","vs6","vs7","vs8","vs9" ,"vs32","vs33","vs34","vs35", "vs36", "vs37", "vs38", "vs39",
"vs40", "vs41", "vs42", "vs43", "vs44", "vs45", "vs46", "vs47", "vs48", "vs49", "vs50", "vs51"
);
return;
} }
#else #else
static void dgemv_kernel_4x8(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOAT *y, FLOAT alpha) { static void dgemv_kernel_4x8(BLASLONG n, BLASLONG lda, FLOAT *ap, FLOAT *x, FLOAT *y, FLOAT alpha) {