diff --git a/kernel/power/dgemm_kernel_power10.c b/kernel/power/dgemm_kernel_power10.c index a0bc1a777..b2a29140e 100644 --- a/kernel/power/dgemm_kernel_power10.c +++ b/kernel/power/dgemm_kernel_power10.c @@ -87,22 +87,6 @@ typedef FLOAT v2sf_t __attribute__ ((vector_size (8))); rowC[0] += result[1] * alpha; #endif -#define SET_ACC_ZERO4() \ - __builtin_mma_xxsetaccz (&acc0); \ - __builtin_mma_xxsetaccz (&acc1); \ - __builtin_mma_xxsetaccz (&acc2); \ - __builtin_mma_xxsetaccz (&acc3); - -#define SET_ACC_ZERO8() \ - __builtin_mma_xxsetaccz (&acc0); \ - __builtin_mma_xxsetaccz (&acc1); \ - __builtin_mma_xxsetaccz (&acc2); \ - __builtin_mma_xxsetaccz (&acc3); \ - __builtin_mma_xxsetaccz (&acc4); \ - __builtin_mma_xxsetaccz (&acc5); \ - __builtin_mma_xxsetaccz (&acc6); \ - __builtin_mma_xxsetaccz (&acc7); - #define PREFETCH1(x, y) asm volatile ("dcbt %0, %1" : : "r" (x), "b" (y) : "memory"); #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) @@ -210,12 +194,22 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B, PREFETCH1 (CO + ldc + ldc, 128); PREFETCH1 (CO + ldc + ldc + ldc, 128); __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7; - SET_ACC_ZERO8 (); - for (l = 0; l < temp; l++) + vec_t *rowA = (vec_t *) & AO[0]; + __vector_pair rowB; + vec_t *rb = (vec_t *) & BO[0]; + __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]); + __builtin_mma_xvf64ger (&acc0, rowB, rowA[0]); + __builtin_mma_xvf64ger (&acc1, rowB, rowA[1]); + __builtin_mma_xvf64ger (&acc2, rowB, rowA[2]); + __builtin_mma_xvf64ger (&acc3, rowB, rowA[3]); + __builtin_mma_xvf64ger (&acc4, rowB, rowA[4]); + __builtin_mma_xvf64ger (&acc5, rowB, rowA[5]); + __builtin_mma_xvf64ger (&acc6, rowB, rowA[6]); + __builtin_mma_xvf64ger (&acc7, rowB, rowA[7]); + for (l = 1; l < temp; l++) { - vec_t *rowA = (vec_t *) & AO[l << 4]; - __vector_pair rowB; - vec_t *rb = (vec_t *) & BO[l << 2]; + rowA = (vec_t *) & AO[l << 4]; + rb = (vec_t *) & BO[l << 2]; __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]); __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]); __builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]); @@ -254,13 +248,19 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B, v4sf_t *rowC; v4sf_t result[4]; __vector_quad acc0, acc1, acc2, acc3; - SET_ACC_ZERO4 (); BLASLONG l = 0; - for (l = 0; l < temp; l++) + vec_t *rowA = (vec_t *) & AO[0]; + __vector_pair rowB; + vec_t *rb = (vec_t *) & BO[0]; + __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]); + __builtin_mma_xvf64ger (&acc0, rowB, rowA[0]); + __builtin_mma_xvf64ger (&acc1, rowB, rowA[1]); + __builtin_mma_xvf64ger (&acc2, rowB, rowA[2]); + __builtin_mma_xvf64ger (&acc3, rowB, rowA[3]); + for (l = 1; l < temp; l++) { - vec_t *rowA = (vec_t *) & AO[l << 3]; - __vector_pair rowB; - vec_t *rb = (vec_t *) & BO[l << 2]; + rowA = (vec_t *) & AO[l << 3]; + rb = (vec_t *) & BO[l << 2]; __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]); __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]); __builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]); @@ -291,14 +291,17 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B, v4sf_t *rowC; v4sf_t result[4]; __vector_quad acc0, acc1; - __builtin_mma_xxsetaccz (&acc0); - __builtin_mma_xxsetaccz (&acc1); BLASLONG l = 0; - for (l = 0; l < temp; l++) + vec_t *rowA = (vec_t *) & AO[0]; + __vector_pair rowB; + vec_t *rb = (vec_t *) & BO[0]; + __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]); + __builtin_mma_xvf64ger (&acc0, rowB, rowA[0]); + __builtin_mma_xvf64ger (&acc1, rowB, rowA[1]); + for (l = 1; l < temp; l++) { - vec_t *rowA = (vec_t *) & AO[l << 2]; - __vector_pair rowB; - vec_t *rb = (vec_t *) & BO[l << 2]; + rowA = (vec_t *) & AO[l << 2]; + rb = (vec_t *) & BO[l << 2]; __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]); __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]); __builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]); @@ -325,13 +328,16 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B, v4sf_t *rowC; v4sf_t result[4]; __vector_quad acc0; - __builtin_mma_xxsetaccz (&acc0); BLASLONG l = 0; - for (l = 0; l < temp; l++) + vec_t *rowA = (vec_t *) & AO[0]; + __vector_pair rowB; + vec_t *rb = (vec_t *) & BO[0]; + __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]); + __builtin_mma_xvf64ger (&acc0, rowB, rowA[0]); + for (l = 1; l < temp; l++) { - vec_t *rowA = (vec_t *) & AO[l << 1]; - __vector_pair rowB; - vec_t *rb = (vec_t *) & BO[l << 2]; + rowA = (vec_t *) & AO[l << 1]; + rb = (vec_t *) & BO[l << 2]; __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]); __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]); } @@ -414,16 +420,27 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B, v4sf_t *rowC; v4sf_t result[4]; __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7; - SET_ACC_ZERO8 (); BLASLONG l = 0; - for (l = 0; l < temp; l++) + FLOAT t[4] = { 0, 0, 0, 0 }; + t[0] = BO[0], t[1] = BO[1]; + __vector_pair rowB; + vec_t *rb = (vec_t *) & t[0]; + __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]); + vec_t *rowA = (vec_t *) & AO[0]; + __builtin_mma_xvf64ger (&acc0, rowB, rowA[0]); + __builtin_mma_xvf64ger (&acc1, rowB, rowA[1]); + __builtin_mma_xvf64ger (&acc2, rowB, rowA[2]); + __builtin_mma_xvf64ger (&acc3, rowB, rowA[3]); + __builtin_mma_xvf64ger (&acc4, rowB, rowA[4]); + __builtin_mma_xvf64ger (&acc5, rowB, rowA[5]); + __builtin_mma_xvf64ger (&acc6, rowB, rowA[6]); + __builtin_mma_xvf64ger (&acc7, rowB, rowA[7]); + for (l = 1; l < temp; l++) { - FLOAT t[4] = { 0, 0, 0, 0 }; t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1]; - __vector_pair rowB; - vec_t *rb = (vec_t *) & t[0]; + rb = (vec_t *) & t[0]; __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]); - vec_t *rowA = (vec_t *) & AO[l << 4]; + rowA = (vec_t *) & AO[l << 4]; __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]); __builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]); __builtin_mma_xvf64gerpp (&acc2, rowB, rowA[2]); @@ -461,16 +478,23 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B, v4sf_t *rowC; v4sf_t result[4]; __vector_quad acc0, acc1, acc2, acc3; - SET_ACC_ZERO4 (); BLASLONG l = 0; - for (l = 0; l < temp; l++) + FLOAT t[4] = { 0, 0, 0, 0 }; + t[0] = BO[0], t[1] = BO[1]; + __vector_pair rowB; + vec_t *rb = (vec_t *) & t[0]; + __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]); + vec_t *rowA = (vec_t *) & AO[0]; + __builtin_mma_xvf64ger (&acc0, rowB, rowA[0]); + __builtin_mma_xvf64ger (&acc1, rowB, rowA[1]); + __builtin_mma_xvf64ger (&acc2, rowB, rowA[2]); + __builtin_mma_xvf64ger (&acc3, rowB, rowA[3]); + for (l = 1; l < temp; l++) { - FLOAT t[4] = { 0, 0, 0, 0 }; t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1]; - __vector_pair rowB; - vec_t *rb = (vec_t *) & t[0]; + rb = (vec_t *) & t[0]; __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]); - vec_t *rowA = (vec_t *) & AO[l << 3]; + rowA = (vec_t *) & AO[l << 3]; __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]); __builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]); __builtin_mma_xvf64gerpp (&acc2, rowB, rowA[2]); @@ -500,17 +524,21 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B, v4sf_t *rowC; v4sf_t result[4]; __vector_quad acc0, acc1; - __builtin_mma_xxsetaccz (&acc0); - __builtin_mma_xxsetaccz (&acc1); BLASLONG l = 0; - for (l = 0; l < temp; l++) + FLOAT t[4] = { 0, 0, 0, 0 }; + t[0] = BO[0], t[1] = BO[1]; + __vector_pair rowB; + vec_t *rb = (vec_t *) & t[0]; + __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]); + vec_t *rowA = (vec_t *) & AO[0]; + __builtin_mma_xvf64ger (&acc0, rowB, rowA[0]); + __builtin_mma_xvf64ger (&acc1, rowB, rowA[1]); + for (l = 1; l < temp; l++) { - FLOAT t[4] = { 0, 0, 0, 0 }; t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1]; - __vector_pair rowB; - vec_t *rb = (vec_t *) & t[0]; + rb = (vec_t *) & t[0]; __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]); - vec_t *rowA = (vec_t *) & AO[l << 2]; + rowA = (vec_t *) & AO[l << 2]; __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]); __builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]); } @@ -536,16 +564,20 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B, v4sf_t *rowC; v4sf_t result[4]; __vector_quad acc0; - __builtin_mma_xxsetaccz (&acc0); BLASLONG l = 0; - for (l = 0; l < temp; l++) + FLOAT t[4] = { 0, 0, 0, 0 }; + t[0] = BO[0], t[1] = BO[1]; + __vector_pair rowB; + vec_t *rb = (vec_t *) & t[0]; + __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]); + vec_t *rowA = (vec_t *) & AO[0]; + __builtin_mma_xvf64ger (&acc0, rowB, rowA[0]); + for (l = 1; l < temp; l++) { - FLOAT t[4] = { 0, 0, 0, 0 }; t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1]; - __vector_pair rowB; - vec_t *rb = (vec_t *) & t[0]; + rb = (vec_t *) & t[0]; __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]); - vec_t *rowA = (vec_t *) & AO[l << 1]; + rowA = (vec_t *) & AO[l << 1]; __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]); } SAVE2x4_ACC (&acc0, 0); diff --git a/kernel/power/sgemm_kernel_power10.c b/kernel/power/sgemm_kernel_power10.c index 81a5ec76b..9fbf84695 100644 --- a/kernel/power/sgemm_kernel_power10.c +++ b/kernel/power/sgemm_kernel_power10.c @@ -134,21 +134,6 @@ typedef FLOAT v2sf_t __attribute__ ((vector_size (8))); __builtin_mma_xvf32gerpp (&acc5, rowB[i+1], rowA[j+2]); \ __builtin_mma_xvf32gerpp (&acc6, rowB[i], rowA[j+3]); \ __builtin_mma_xvf32gerpp (&acc7, rowB[i+1], rowA[j+3]); -#define SET_ACC_ZERO4() \ - __builtin_mma_xxsetaccz (&acc0); \ - __builtin_mma_xxsetaccz (&acc1); \ - __builtin_mma_xxsetaccz (&acc2); \ - __builtin_mma_xxsetaccz (&acc3); - -#define SET_ACC_ZERO8() \ - __builtin_mma_xxsetaccz (&acc0); \ - __builtin_mma_xxsetaccz (&acc1); \ - __builtin_mma_xxsetaccz (&acc2); \ - __builtin_mma_xxsetaccz (&acc3); \ - __builtin_mma_xxsetaccz (&acc4); \ - __builtin_mma_xxsetaccz (&acc5); \ - __builtin_mma_xxsetaccz (&acc6); \ - __builtin_mma_xxsetaccz (&acc7); #define PREFETCH1(x, y) asm volatile ("dcbt %0, %1" : : "r" (x), "b" (y) : "memory"); @@ -249,8 +234,20 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B, v4sf_t *rowC; v4sf_t result[4]; __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7; - SET_ACC_ZERO8 (); BLASLONG l = 0; + vec_t *rowA1 = (vec_t *) & AO[0]; + vec_t *rowB1 = (vec_t *) & BO[0]; + __builtin_mma_xvf32ger (&acc0, rowB1[0], rowA1[0]); + __builtin_mma_xvf32ger (&acc1, rowB1[1], rowA1[0]); + __builtin_mma_xvf32ger (&acc2, rowB1[0], rowA1[1]); + __builtin_mma_xvf32ger (&acc3, rowB1[1], rowA1[1]); + __builtin_mma_xvf32ger (&acc4, rowB1[0], rowA1[2]); + __builtin_mma_xvf32ger (&acc5, rowB1[1], rowA1[2]); + __builtin_mma_xvf32ger (&acc6, rowB1[0], rowA1[3]); + __builtin_mma_xvf32ger (&acc7, rowB1[1], rowA1[3]); + AO += 16; + BO += 8; + temp--; BLASLONG K = temp / 64; for (l = 0; l < K; l++) { @@ -454,12 +451,17 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B, v4sf_t *rowC; v4sf_t result[4]; __vector_quad acc0, acc1, acc2, acc3; - SET_ACC_ZERO4 (); BLASLONG l = 0; - for (l = 0; l < temp; l++) + vec_t *rowA = (vec_t *) & AO[0]; + vec_t *rowB = (vec_t *) & BO[0]; + __builtin_mma_xvf32ger (&acc0, rowB[0], rowA[0]); + __builtin_mma_xvf32ger (&acc1, rowB[1], rowA[0]); + __builtin_mma_xvf32ger (&acc2, rowB[0], rowA[1]); + __builtin_mma_xvf32ger (&acc3, rowB[1], rowA[1]); + for (l = 1; l < temp; l++) { - vec_t *rowA = (vec_t *) & AO[l << 3]; - vec_t *rowB = (vec_t *) & BO[l << 3]; + rowA = (vec_t *) & AO[l << 3]; + rowB = (vec_t *) & BO[l << 3]; __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]); __builtin_mma_xvf32gerpp (&acc1, rowB[1], rowA[0]); __builtin_mma_xvf32gerpp (&acc2, rowB[0], rowA[1]); @@ -489,13 +491,15 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B, v4sf_t *rowC; v4sf_t result[4]; __vector_quad acc0, acc1; - __builtin_mma_xxsetaccz (&acc0); - __builtin_mma_xxsetaccz (&acc1); BLASLONG l = 0; - for (l = 0; l < temp; l++) + vec_t *rowA = (vec_t *) & AO[0]; + vec_t *rowB = (vec_t *) & BO[0]; + __builtin_mma_xvf32ger (&acc0, rowB[0], rowA[0]); + __builtin_mma_xvf32ger (&acc1, rowB[1], rowA[0]); + for (l = 1; l < temp; l++) { - vec_t *rowA = (vec_t *) & AO[l << 2]; - vec_t *rowB = (vec_t *) & BO[l << 3]; + rowA = (vec_t *) & AO[l << 2]; + rowB = (vec_t *) & BO[l << 3]; __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]); __builtin_mma_xvf32gerpp (&acc1, rowB[1], rowA[0]); } @@ -522,15 +526,18 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B, v2sf_t *rowC; v2sf_t result[8]; __vector_quad acc0, acc1; - __builtin_mma_xxsetaccz (&acc0); - __builtin_mma_xxsetaccz (&acc1); BLASLONG l = 0; - for (l = 0; l < temp; l++) + FLOAT t[4] = { 0 }; + t[0] = AO[0], t[1] = AO[1]; + vec_t *rowA = (vec_t *) & t[0]; + vec_t *rowB = (vec_t *) & BO[0]; + __builtin_mma_xvf32ger (&acc0, rowB[0], rowA[0]); + __builtin_mma_xvf32ger (&acc1, rowB[1], rowA[0]); + for (l = 1; l < temp; l++) { - FLOAT t[4] = { 0 }; t[0] = AO[l << 1], t[1] = AO[(l << 1) + 1]; - vec_t *rowA = (vec_t *) & t[0]; - vec_t *rowB = (vec_t *) & BO[l << 3]; + rowA = (vec_t *) & t[0]; + rowB = (vec_t *) & BO[l << 3]; __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]); __builtin_mma_xvf32gerpp (&acc1, rowB[1], rowA[0]); } @@ -625,13 +632,23 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B, FLOAT *A1; A1 = AO + (16 * k); __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7; - SET_ACC_ZERO8 (); BLASLONG l = 0; - for (l = 0; l < k; l++) + vec_t *rowA = (vec_t *) & AO[0]; + vec_t *rowA1 = (vec_t *) & A1[0]; + vec_t *rowB = (vec_t *) & BO[0]; + __builtin_mma_xvf32ger (&acc0, rowB[0], rowA[0]); + __builtin_mma_xvf32ger (&acc1, rowB[0], rowA[1]); + __builtin_mma_xvf32ger (&acc2, rowB[0], rowA[2]); + __builtin_mma_xvf32ger (&acc3, rowB[0], rowA[3]); + __builtin_mma_xvf32ger (&acc4, rowB[0], rowA1[0]); + __builtin_mma_xvf32ger (&acc5, rowB[0], rowA1[1]); + __builtin_mma_xvf32ger (&acc6, rowB[0], rowA1[2]); + __builtin_mma_xvf32ger (&acc7, rowB[0], rowA1[3]); + for (l = 1; l < k; l++) { - vec_t *rowA = (vec_t *) & AO[l << 4]; - vec_t *rowA1 = (vec_t *) & A1[l << 4]; - vec_t *rowB = (vec_t *) & BO[l << 2]; + rowA = (vec_t *) & AO[l << 4]; + rowA1 = (vec_t *) & A1[l << 4]; + rowB = (vec_t *) & BO[l << 2]; __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]); __builtin_mma_xvf32gerpp (&acc1, rowB[0], rowA[1]); __builtin_mma_xvf32gerpp (&acc2, rowB[0], rowA[2]); @@ -673,12 +690,17 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B, v4sf_t *rowC; v4sf_t result[4]; __vector_quad acc0, acc1, acc2, acc3; - SET_ACC_ZERO4 (); BLASLONG l = 0; - for (l = 0; l < temp; l++) + vec_t *rowA = (vec_t *) & AO[0]; + vec_t *rowB = (vec_t *) & BO[0]; + __builtin_mma_xvf32ger (&acc0, rowB[0], rowA[0]); + __builtin_mma_xvf32ger (&acc1, rowB[0], rowA[1]); + __builtin_mma_xvf32ger (&acc2, rowB[0], rowA[2]); + __builtin_mma_xvf32ger (&acc3, rowB[0], rowA[3]); + for (l = 1; l < temp; l++) { - vec_t *rowA = (vec_t *) & AO[l << 4]; - vec_t *rowB = (vec_t *) & BO[l << 2]; + rowA = (vec_t *) & AO[l << 4]; + rowB = (vec_t *) & BO[l << 2]; __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]); __builtin_mma_xvf32gerpp (&acc1, rowB[0], rowA[1]); __builtin_mma_xvf32gerpp (&acc2, rowB[0], rowA[2]); @@ -710,13 +732,15 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B, v4sf_t *rowC; v4sf_t result[4]; __vector_quad acc0, acc1; - __builtin_mma_xxsetaccz (&acc0); - __builtin_mma_xxsetaccz (&acc1); BLASLONG l = 0; - for (l = 0; l < temp; l++) + vec_t *rowA = (vec_t *) & AO[0]; + vec_t *rowB = (vec_t *) & BO[0]; + __builtin_mma_xvf32ger (&acc0, rowB[0], rowA[0]); + __builtin_mma_xvf32ger (&acc1, rowB[0], rowA[1]); + for (l = 1; l < temp; l++) { - vec_t *rowA = (vec_t *) & AO[l << 3]; - vec_t *rowB = (vec_t *) & BO[l << 2]; + rowA = (vec_t *) & AO[l << 3]; + rowB = (vec_t *) & BO[l << 2]; __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]); __builtin_mma_xvf32gerpp (&acc1, rowB[0], rowA[1]); } @@ -742,12 +766,14 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B, v4sf_t *rowC; __vector_quad acc0; v4sf_t result[4]; - __builtin_mma_xxsetaccz (&acc0); BLASLONG l = 0; - for (l = 0; l < temp; l++) + vec_t *rowA = (vec_t *) & AO[0]; + vec_t *rowB = (vec_t *) & BO[0]; + __builtin_mma_xvf32ger (&acc0, rowB[0], rowA[0]); + for (l = 1; l < temp; l++) { - vec_t *rowA = (vec_t *) & AO[l << 2]; - vec_t *rowB = (vec_t *) & BO[l << 2]; + rowA = (vec_t *) & AO[l << 2]; + rowB = (vec_t *) & BO[l << 2]; __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]); } SAVE_ACC (&acc0, 0); @@ -771,14 +797,17 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B, v2sf_t *rowC; v2sf_t result[8]; __vector_quad acc0; - __builtin_mma_xxsetaccz (&acc0); BLASLONG l = 0; - for (l = 0; l < temp; l++) + FLOAT t[4] = { 0 }; + t[0] = AO[0], t[1] = AO[1]; + vec_t *rowA = (vec_t *) & t[0]; + vec_t *rowB = (vec_t *) & BO[0]; + __builtin_mma_xvf32ger (&acc0, rowB[0], rowA[0]); + for (l = 1; l < temp; l++) { - FLOAT t[4] = { 0 }; t[0] = AO[l << 1], t[1] = AO[(l << 1) + 1]; - vec_t *rowA = (vec_t *) & t[0]; - vec_t *rowB = (vec_t *) & BO[l << 2]; + rowA = (vec_t *) & t[0]; + rowB = (vec_t *) & BO[l << 2]; __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]); } SAVE4x2_ACC (&acc0, 0); @@ -856,15 +885,26 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B, FLOAT *A1; A1 = AO + (16 * k); __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7; - SET_ACC_ZERO8 (); BLASLONG l = 0; - for (l = 0; l < k; l++) + FLOAT t[4] = { 0 }; + t[0] = BO[0], t[1] = BO[1]; + vec_t *rowB = (vec_t *) & t[0]; + vec_t *rowA = (vec_t *) & AO[0]; + vec_t *rowA1 = (vec_t *) & A1[0]; + __builtin_mma_xvf32ger (&acc0, rowB[0], rowA[0]); + __builtin_mma_xvf32ger (&acc1, rowB[0], rowA[1]); + __builtin_mma_xvf32ger (&acc2, rowB[0], rowA[2]); + __builtin_mma_xvf32ger (&acc3, rowB[0], rowA[3]); + __builtin_mma_xvf32ger (&acc4, rowB[0], rowA1[0]); + __builtin_mma_xvf32ger (&acc5, rowB[0], rowA1[1]); + __builtin_mma_xvf32ger (&acc6, rowB[0], rowA1[2]); + __builtin_mma_xvf32ger (&acc7, rowB[0], rowA1[3]); + for (l = 1; l < k; l++) { - FLOAT t[4] = { 0 }; t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1]; - vec_t *rowB = (vec_t *) & t[0]; - vec_t *rowA = (vec_t *) & AO[l << 4]; - vec_t *rowA1 = (vec_t *) & A1[l << 4]; + rowB = (vec_t *) & t[0]; + rowA = (vec_t *) & AO[l << 4]; + rowA1 = (vec_t *) & A1[l << 4]; __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]); __builtin_mma_xvf32gerpp (&acc1, rowB[0], rowA[1]); __builtin_mma_xvf32gerpp (&acc2, rowB[0], rowA[2]); @@ -897,7 +937,6 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B, v4sf_t *rowC; v4sf_t result[4]; __vector_quad acc0, acc1, acc2, acc3; - SET_ACC_ZERO4 (); BLASLONG l = 0; #if defined(TRMMKERNEL) REFRESH_POINTERS (16, 2) @@ -905,12 +944,19 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B, BO = B; temp = k; #endif - for (l = 0; l < temp; l++) + FLOAT t[4] = { 0 }; + t[0] = BO[0], t[1] = BO[1]; + vec_t *rowB = (vec_t *) & t[0]; + vec_t *rowA = (vec_t *) & AO[0]; + __builtin_mma_xvf32ger (&acc0, rowB[0], rowA[0]); + __builtin_mma_xvf32ger (&acc1, rowB[0], rowA[1]); + __builtin_mma_xvf32ger (&acc2, rowB[0], rowA[2]); + __builtin_mma_xvf32ger (&acc3, rowB[0], rowA[3]); + for (l = 1; l < temp; l++) { - FLOAT t[4] = { 0 }; t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1]; - vec_t *rowB = (vec_t *) & t[0]; - vec_t *rowA = (vec_t *) & AO[l << 4]; + rowB = (vec_t *) & t[0]; + rowA = (vec_t *) & AO[l << 4]; __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]); __builtin_mma_xvf32gerpp (&acc1, rowB[0], rowA[1]); __builtin_mma_xvf32gerpp (&acc2, rowB[0], rowA[2]); @@ -934,8 +980,6 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B, v4sf_t *rowC; v4sf_t result[4]; __vector_quad acc0, acc1; - __builtin_mma_xxsetaccz (&acc0); - __builtin_mma_xxsetaccz (&acc1); #if defined(TRMMKERNEL) REFRESH_POINTERS (8, 2) #else @@ -943,12 +987,17 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B, temp = k; #endif BLASLONG l = 0; - for (l = 0; l < temp; l++) + FLOAT t[4] = { 0 }; + t[0] = BO[0], t[1] = BO[1]; + vec_t *rowB = (vec_t *) & t[0]; + vec_t *rowA = (vec_t *) & AO[0]; + __builtin_mma_xvf32ger (&acc0, rowB[0], rowA[0]); + __builtin_mma_xvf32ger (&acc1, rowB[0], rowA[1]); + for (l = 1; l < temp; l++) { - FLOAT t[4] = { 0 }; t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1]; - vec_t *rowB = (vec_t *) & t[0]; - vec_t *rowA = (vec_t *) & AO[l << 3]; + rowB = (vec_t *) & t[0]; + rowA = (vec_t *) & AO[l << 3]; __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]); __builtin_mma_xvf32gerpp (&acc1, rowB[0], rowA[1]); } @@ -968,7 +1017,6 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B, v4sf_t *rowC; v4sf_t result[4]; __vector_quad acc0; - __builtin_mma_xxsetaccz (&acc0); #if defined(TRMMKERNEL) REFRESH_POINTERS (4, 2) #else @@ -976,12 +1024,16 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B, temp = k; #endif BLASLONG l = 0; - for (l = 0; l < temp; l++) + FLOAT t[4] = { 0 }; + t[0] = BO[0], t[1] = BO[1]; + vec_t *rowB = (vec_t *) & t[0]; + vec_t *rowA = (vec_t *) & AO[0]; + __builtin_mma_xvf32ger (&acc0, rowB[0], rowA[0]); + for (l = 1; l < temp; l++) { - FLOAT t[4] = { 0 }; t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1]; - vec_t *rowB = (vec_t *) & t[0]; - vec_t *rowA = (vec_t *) & AO[l << 2]; + rowB = (vec_t *) & t[0]; + rowA = (vec_t *) & AO[l << 2]; __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]); } SAVE2x4_ACC (&acc0, 0);