diff --git a/kernel/mips/cgemv_n_msa.c b/kernel/mips/cgemv_n_msa.c index f1879ba00..a9db04aaf 100644 --- a/kernel/mips/cgemv_n_msa.c +++ b/kernel/mips/cgemv_n_msa.c @@ -376,128 +376,139 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. *((int *)(y + 2 * inc_y2 + 1)) = __msa_copy_s_w((v4i32) y0i, 2); \ *((int *)(y + 3 * inc_y2 + 1)) = __msa_copy_s_w((v4i32) y0i, 3); \ -#define CGEMV_N_MSA() \ - for (j = (n >> 2); j--;) \ - { \ - CLOAD_X4_SCALE(); \ - \ - k = 0; \ - y = y_org; \ - \ - for (i = (m >> 3); i--;) \ - { \ - CLOAD_Y8() \ - CGEMV_N_8x4(); \ - CSTORE_Y8(); \ - \ - k += 2 * 8; \ - y += inc_y2 * 8; \ - } \ - \ - if (m & 4) \ - { \ - CLOAD_Y4(); \ - CGEMV_N_4x4(); \ - CSTORE_Y4(); \ - \ - k += 2 * 4; \ - y += inc_y2 * 4; \ - } \ - \ - if (m & 3) \ - { \ - temp0_r = tp4r[0]; \ - temp1_r = tp4r[1]; \ - temp2_r = tp4r[2]; \ - temp3_r = tp4r[3]; \ - \ - temp0_i = tp4i[0]; \ - temp1_i = tp4i[1]; \ - temp2_i = tp4i[2]; \ - temp3_i = tp4i[3]; \ - \ - for (i = (m & 3); i--;) \ - { \ - CGEMV_N_1x4(); \ - \ - k += 2; \ - y += inc_y2; \ - } \ - } \ - \ - pa0 += 4 * lda2; \ - pa1 += 4 * lda2; \ - pa2 += 4 * lda2; \ - pa3 += 4 * lda2; \ - \ - x += 4 * inc_x2; \ - } \ - \ - if (n & 2) \ - { \ - CLOAD_X2_SCALE(); \ - \ - k = 0; \ - y = y_org; \ - \ - for (i = (m >> 3); i--;) \ - { \ - CLOAD_Y8(); \ - CGEMV_N_8x2(); \ - CSTORE_Y8(); \ - \ - k += 2 * 8; \ - y += inc_y2 * 8; \ - } \ - \ - if (m & 4) \ - { \ - CLOAD_Y4(); \ - CGEMV_N_4x2(); \ - CSTORE_Y4(); \ - \ - k += 2 * 4; \ - y += inc_y2 * 4; \ - } \ - \ - for (i = (m & 3); i--;) \ - { \ - CGEMV_N_1x2(); \ - \ - k += 2; \ - y += inc_y2; \ - } \ - \ - pa0 += 2 * lda2; \ - pa1 += 2 * lda2; \ - \ - x += 2 * inc_x2; \ - } \ - \ - if (n & 1) \ - { \ - CLOAD_X1_SCALE(); \ - \ - k = 0; \ - y = y_org; \ - \ - for (i = m; i--;) \ - { \ - CGEMV_N_1x1(); \ - \ - k += 2; \ - y += inc_y2; \ - } \ - \ - pa0 += lda2; \ - x += inc_x2; \ - } \ +#define CGEMV_N_MSA() \ + for (j = (n >> 2); j--;) \ + { \ + CLOAD_X4_SCALE(); \ + \ + k = 0; \ + k_pref = pref_offset; \ + y = y_org; \ + \ + for (i = (m >> 3); i--;) \ + { \ + PREFETCH(pa0 + k_pref + 16 + 0); \ + PREFETCH(pa0 + k_pref + 16 + 8); \ + PREFETCH(pa1 + k_pref + 16 + 0); \ + PREFETCH(pa1 + k_pref + 16 + 8); \ + PREFETCH(pa2 + k_pref + 16 + 0); \ + PREFETCH(pa2 + k_pref + 16 + 8); \ + PREFETCH(pa3 + k_pref + 16 + 0); \ + PREFETCH(pa3 + k_pref + 16 + 8); \ + \ + CLOAD_Y8() \ + CGEMV_N_8x4(); \ + CSTORE_Y8(); \ + \ + k += 2 * 8; \ + k_pref += 2 * 8; \ + y += inc_y2 * 8; \ + } \ + \ + if (m & 4) \ + { \ + CLOAD_Y4(); \ + CGEMV_N_4x4(); \ + CSTORE_Y4(); \ + \ + k += 2 * 4; \ + y += inc_y2 * 4; \ + } \ + \ + if (m & 3) \ + { \ + temp0_r = tp4r[0]; \ + temp1_r = tp4r[1]; \ + temp2_r = tp4r[2]; \ + temp3_r = tp4r[3]; \ + \ + temp0_i = tp4i[0]; \ + temp1_i = tp4i[1]; \ + temp2_i = tp4i[2]; \ + temp3_i = tp4i[3]; \ + \ + for (i = (m & 3); i--;) \ + { \ + CGEMV_N_1x4(); \ + \ + k += 2; \ + y += inc_y2; \ + } \ + } \ + \ + pa0 += 4 * lda2; \ + pa1 += 4 * lda2; \ + pa2 += 4 * lda2; \ + pa3 += 4 * lda2; \ + \ + x += 4 * inc_x2; \ + } \ + \ + if (n & 2) \ + { \ + CLOAD_X2_SCALE(); \ + \ + k = 0; \ + y = y_org; \ + \ + for (i = (m >> 3); i--;) \ + { \ + CLOAD_Y8(); \ + CGEMV_N_8x2(); \ + CSTORE_Y8(); \ + \ + k += 2 * 8; \ + y += inc_y2 * 8; \ + } \ + \ + if (m & 4) \ + { \ + CLOAD_Y4(); \ + CGEMV_N_4x2(); \ + CSTORE_Y4(); \ + \ + k += 2 * 4; \ + y += inc_y2 * 4; \ + } \ + \ + for (i = (m & 3); i--;) \ + { \ + CGEMV_N_1x2(); \ + \ + k += 2; \ + y += inc_y2; \ + } \ + \ + pa0 += 2 * lda2; \ + pa1 += 2 * lda2; \ + \ + x += 2 * inc_x2; \ + } \ + \ + if (n & 1) \ + { \ + CLOAD_X1_SCALE(); \ + \ + k = 0; \ + y = y_org; \ + \ + for (i = m; i--;) \ + { \ + CGEMV_N_1x1(); \ + \ + k += 2; \ + y += inc_y2; \ + } \ + \ + pa0 += lda2; \ + x += inc_x2; \ + } \ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r, FLOAT alpha_i, FLOAT *A, BLASLONG lda2, FLOAT *x, BLASLONG inc_x2, FLOAT *y, BLASLONG inc_y2, FLOAT *buffer) { - BLASLONG i, j, k; + BLASLONG i, j, k, k_pref, pref_offset; FLOAT *y_org = y; FLOAT *pa0, *pa1, *pa2, *pa3; FLOAT temp_r, temp_i, res0, res1, temp0_r; @@ -513,6 +524,10 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r, FLOAT alpha_i, inc_x2 = 2 * inc_x2; inc_y2 = 2 * inc_y2; + pref_offset = (uintptr_t)A & (L1_DATA_LINESIZE - 1); + pref_offset = L1_DATA_LINESIZE - pref_offset; + pref_offset = pref_offset / sizeof(FLOAT); + pa0 = A; pa1 = A + lda2; pa2 = A + 2 * lda2; diff --git a/kernel/mips/cgemv_t_msa.c b/kernel/mips/cgemv_t_msa.c index b9620bfb9..584e3de75 100644 --- a/kernel/mips/cgemv_t_msa.c +++ b/kernel/mips/cgemv_t_msa.c @@ -364,14 +364,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. tp0i = tp1i = tp2i = tp3i = zero; \ \ k = 0; \ + k_pref = pref_offset; \ x = srcx_org; \ \ for (i = (m >> 3); i--;) \ { \ + PREFETCH(pa0 + k_pref + 16 + 0); \ + PREFETCH(pa0 + k_pref + 16 + 8); \ + PREFETCH(pa1 + k_pref + 16 + 0); \ + PREFETCH(pa1 + k_pref + 16 + 8); \ + PREFETCH(pa2 + k_pref + 16 + 0); \ + PREFETCH(pa2 + k_pref + 16 + 8); \ + PREFETCH(pa3 + k_pref + 16 + 0); \ + PREFETCH(pa3 + k_pref + 16 + 8); \ + \ CLOAD_X8() \ CGEMV_T_8x4(); \ \ k += 2 * 8; \ + k_pref += 2 * 8; \ x += inc_x2 * 8; \ } \ \ @@ -531,7 +542,7 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alphar, FLOAT alphai, FLOAT *A, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer) { - BLASLONG i, j, k; + BLASLONG i, j, k, k_pref, pref_offset; FLOAT *pa0, *pa1, *pa2, *pa3; FLOAT *srcx_org = x; FLOAT temp0r, temp0i, temp2r, temp2i, temp1r, temp1i, temp3r, temp3i; @@ -546,6 +557,10 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alphar, FLOAT alphai, lda2 = 2 * lda; + pref_offset = (uintptr_t)A & (L1_DATA_LINESIZE - 1); + pref_offset = L1_DATA_LINESIZE - pref_offset; + pref_offset = pref_offset / sizeof(FLOAT); + pa0 = A; pa1 = A + lda2; pa2 = A + 2 * lda2; diff --git a/kernel/mips/zgemv_n_msa.c b/kernel/mips/zgemv_n_msa.c index aadc610aa..669c25758 100644 --- a/kernel/mips/zgemv_n_msa.c +++ b/kernel/mips/zgemv_n_msa.c @@ -312,14 +312,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SPLATI_D2_DP(tp4i, tp0i, tp1i); \ #define ZLOAD_X4_SCALE_GP() \ - x0r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *)(x + 0 * inc_x2))); \ - x0r = (v2f64) __msa_insert_d((v2i64) x0r, 1, *((long long *)(x + 1 * inc_x2))); \ - x1r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *)(x + 2 * inc_x2))); \ - x1r = (v2f64) __msa_insert_d((v2i64) x1r, 1, *((long long *)(x + 3 * inc_x2))); \ - x0i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *)(x + 0 * inc_x2 + 1))); \ - x0i = (v2f64) __msa_insert_d((v2i64) x0i, 1, *((long long *)(x + 1 * inc_x2 + 1))); \ - x1i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *)(x + 2 * inc_x2 + 1))); \ - x1i = (v2f64) __msa_insert_d((v2i64) x1i, 1, *((long long *)(x + 3 * inc_x2 + 1))); \ + x0r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((BLASLONG *)(x + 0 * inc_x2))); \ + x0r = (v2f64) __msa_insert_d((v2i64) x0r, 1, *((BLASLONG *)(x + 1 * inc_x2))); \ + x1r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((BLASLONG *)(x + 2 * inc_x2))); \ + x1r = (v2f64) __msa_insert_d((v2i64) x1r, 1, *((BLASLONG *)(x + 3 * inc_x2))); \ + x0i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((BLASLONG *)(x + 0 * inc_x2 + 1))); \ + x0i = (v2f64) __msa_insert_d((v2i64) x0i, 1, *((BLASLONG *)(x + 1 * inc_x2 + 1))); \ + x1i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((BLASLONG *)(x + 2 * inc_x2 + 1))); \ + x1i = (v2f64) __msa_insert_d((v2i64) x1i, 1, *((BLASLONG *)(x + 3 * inc_x2 + 1))); \ \ tp4r = alphar * x0r; \ tp4r OP3 alphai * x0i; \ @@ -337,10 +337,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SPLATI_D2_DP(tp5i, tp2i, tp3i); \ #define ZLOAD_X2_SCALE_GP() \ - x0r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *)(x + 0 * inc_x2))); \ - x0r = (v2f64) __msa_insert_d((v2i64) x0r, 1, *((long long *)(x + 1 * inc_x2))); \ - x0i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *)(x + 0 * inc_x2 + 1))); \ - x0i = (v2f64) __msa_insert_d((v2i64) x0i, 1, *((long long *)(x + 1 * inc_x2 + 1))); \ + x0r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((BLASLONG *)(x + 0 * inc_x2))); \ + x0r = (v2f64) __msa_insert_d((v2i64) x0r, 1, *((BLASLONG *)(x + 1 * inc_x2))); \ + x0i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((BLASLONG *)(x + 0 * inc_x2 + 1))); \ + x0i = (v2f64) __msa_insert_d((v2i64) x0i, 1, *((BLASLONG *)(x + 1 * inc_x2 + 1))); \ \ tp4r = alphar * x0r; \ tp4r OP3 alphai * x0i; \ @@ -377,182 +377,193 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ILVRL_D2_DP(y0i, y0r, y0, y1); \ ST_DP2(y0, y1, y, 2); \ -#define ZLOAD_Y4_GP() \ - y0r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *)(y + 0 * inc_y2))); \ - y0r = (v2f64) __msa_insert_d((v2i64) y0r, 1, *((long long *)(y + 1 * inc_y2))); \ - y1r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *)(y + 2 * inc_y2))); \ - y1r = (v2f64) __msa_insert_d((v2i64) y1r, 1, *((long long *)(y + 3 * inc_y2))); \ - y0i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *)(y + 0 * inc_y2 + 1))); \ - y0i = (v2f64) __msa_insert_d((v2i64) y0i, 1, *((long long *)(y + 1 * inc_y2 + 1))); \ - y1i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *)(y + 2 * inc_y2 + 1))); \ - y1i = (v2f64) __msa_insert_d((v2i64) y1i, 1, *((long long *)(y + 3 * inc_y2 + 1))); \ +#define ZLOAD_Y4_GP() \ + y0r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((BLASLONG *)(y + 0 * inc_y2))); \ + y0r = (v2f64) __msa_insert_d((v2i64) y0r, 1, *((BLASLONG *)(y + 1 * inc_y2))); \ + y1r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((BLASLONG *)(y + 2 * inc_y2))); \ + y1r = (v2f64) __msa_insert_d((v2i64) y1r, 1, *((BLASLONG *)(y + 3 * inc_y2))); \ + y0i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((BLASLONG *)(y + 0 * inc_y2 + 1))); \ + y0i = (v2f64) __msa_insert_d((v2i64) y0i, 1, *((BLASLONG *)(y + 1 * inc_y2 + 1))); \ + y1i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((BLASLONG *)(y + 2 * inc_y2 + 1))); \ + y1i = (v2f64) __msa_insert_d((v2i64) y1i, 1, *((BLASLONG *)(y + 3 * inc_y2 + 1))); \ -#define ZLOAD_Y2_GP() \ - y0r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *)(y + 0 * inc_y2))); \ - y0r = (v2f64) __msa_insert_d((v2i64) y0r, 1, *((long long *)(y + 1 * inc_y2))); \ - y0i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *)(y + 0 * inc_y2 + 1))); \ - y0i = (v2f64) __msa_insert_d((v2i64) y0i, 1, *((long long *)(y + 1 * inc_y2 + 1))); \ +#define ZLOAD_Y2_GP() \ + y0r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((BLASLONG *)(y + 0 * inc_y2))); \ + y0r = (v2f64) __msa_insert_d((v2i64) y0r, 1, *((BLASLONG *)(y + 1 * inc_y2))); \ + y0i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((BLASLONG *)(y + 0 * inc_y2 + 1))); \ + y0i = (v2f64) __msa_insert_d((v2i64) y0i, 1, *((BLASLONG *)(y + 1 * inc_y2 + 1))); \ -#define ZSTORE_Y4_GP() \ - *((long long *)(y + 0 * inc_y2)) = __msa_copy_s_d((v2i64) y0r, 0); \ - *((long long *)(y + 1 * inc_y2)) = __msa_copy_s_d((v2i64) y0r, 1); \ - *((long long *)(y + 2 * inc_y2)) = __msa_copy_s_d((v2i64) y1r, 0); \ - *((long long *)(y + 3 * inc_y2)) = __msa_copy_s_d((v2i64) y1r, 1); \ - *((long long *)(y + 0 * inc_y2 + 1)) = __msa_copy_s_d((v2i64) y0i, 0); \ - *((long long *)(y + 1 * inc_y2 + 1)) = __msa_copy_s_d((v2i64) y0i, 1); \ - *((long long *)(y + 2 * inc_y2 + 1)) = __msa_copy_s_d((v2i64) y1i, 0); \ - *((long long *)(y + 3 * inc_y2 + 1)) = __msa_copy_s_d((v2i64) y1i, 1); \ +#define ZSTORE_Y4_GP() \ + *((BLASLONG *)(y + 0 * inc_y2)) = __msa_copy_s_d((v2i64) y0r, 0); \ + *((BLASLONG *)(y + 1 * inc_y2)) = __msa_copy_s_d((v2i64) y0r, 1); \ + *((BLASLONG *)(y + 2 * inc_y2)) = __msa_copy_s_d((v2i64) y1r, 0); \ + *((BLASLONG *)(y + 3 * inc_y2)) = __msa_copy_s_d((v2i64) y1r, 1); \ + *((BLASLONG *)(y + 0 * inc_y2 + 1)) = __msa_copy_s_d((v2i64) y0i, 0); \ + *((BLASLONG *)(y + 1 * inc_y2 + 1)) = __msa_copy_s_d((v2i64) y0i, 1); \ + *((BLASLONG *)(y + 2 * inc_y2 + 1)) = __msa_copy_s_d((v2i64) y1i, 0); \ + *((BLASLONG *)(y + 3 * inc_y2 + 1)) = __msa_copy_s_d((v2i64) y1i, 1); \ -#define ZSTORE_Y2_GP() \ - *((long long *)(y + 0 * inc_y2)) = __msa_copy_s_d((v2i64) y0r, 0); \ - *((long long *)(y + 1 * inc_y2)) = __msa_copy_s_d((v2i64) y0r, 1); \ - *((long long *)(y + 0 * inc_y2 + 1)) = __msa_copy_s_d((v2i64) y0i, 0); \ - *((long long *)(y + 1 * inc_y2 + 1)) = __msa_copy_s_d((v2i64) y0i, 1); \ +#define ZSTORE_Y2_GP() \ + *((BLASLONG *)(y + 0 * inc_y2)) = __msa_copy_s_d((v2i64) y0r, 0); \ + *((BLASLONG *)(y + 1 * inc_y2)) = __msa_copy_s_d((v2i64) y0r, 1); \ + *((BLASLONG *)(y + 0 * inc_y2 + 1)) = __msa_copy_s_d((v2i64) y0i, 0); \ + *((BLASLONG *)(y + 1 * inc_y2 + 1)) = __msa_copy_s_d((v2i64) y0i, 1); \ -#define ZGEMV_N_MSA() \ - for (j = (n >> 2); j--;) \ - { \ - ZLOAD_X4_SCALE() \ - \ - k = 0; \ - y = y_org; \ - \ - for (i = (m >> 2); i--;) \ - { \ - ZLOAD_Y4() \ - ZGEMV_N_4x4() \ - ZSTORE_Y4() \ - \ - k += 2 * 4; \ - y += inc_y2 * 4; \ - } \ - \ - if (m & 2) \ - { \ - ZLOAD_Y2() \ - ZGEMV_N_2x4() \ - ZSTORE_Y2() \ - \ - k += 2 * 2; \ - y += inc_y2 * 2; \ - } \ - \ - if (m & 1) \ - { \ - temp0_r = tp4r[0]; \ - temp1_r = tp4r[1]; \ - temp2_r = tp5r[0]; \ - temp3_r = tp5r[1]; \ - \ - temp0_i = tp4i[0]; \ - temp1_i = tp4i[1]; \ - temp2_i = tp5i[0]; \ - temp3_i = tp5i[1]; \ - \ - ZGEMV_N_1x4() \ - k += 2; \ - y += inc_y2; \ - } \ - \ - pa0 += 4 * lda2; \ - pa1 += 4 * lda2; \ - pa2 += 4 * lda2; \ - pa3 += 4 * lda2; \ - \ - x += 4 * inc_x2; \ - } \ - \ - if (n & 2) \ - { \ - ZLOAD_X2_SCALE() \ - \ - k = 0; \ - y = y_org; \ - \ - for (i = (m >> 2); i--;) \ - { \ - ZLOAD_Y4() \ - ZGEMV_N_4x2() \ - ZSTORE_Y4() \ - \ - k += 2 * 4; \ - y += inc_y2 * 4; \ - } \ - \ - if (m & 2) \ - { \ - ZLOAD_Y2() \ - ZGEMV_N_2x2() \ - ZSTORE_Y2() \ - \ - k += 2 * 2; \ - y += inc_y2 * 2; \ - } \ - \ - if (m & 1) \ - { \ - temp0_r = tp4r[0]; \ - temp1_r = tp4r[1]; \ - \ - temp0_i = tp4i[0]; \ - temp1_i = tp4i[1]; \ - \ - ZGEMV_N_1x2() \ - \ - k += 2; \ - y += inc_y2; \ - } \ - \ - pa0 += 2 * lda2; \ - pa1 += 2 * lda2; \ - \ - x += 2 * inc_x2; \ - } \ - \ - if (n & 1) \ - { \ - ZLOAD_X1_SCALE() \ - \ - k = 0; \ - y = y_org; \ - \ - for (i = (m >> 2); i--;) \ - { \ - ZLOAD_Y4() \ - ZGEMV_N_4x1() \ - ZSTORE_Y4() \ - \ - k += 2 * 4; \ - y += inc_y2 * 4; \ - } \ - \ - if (m & 2) \ - { \ - ZLOAD_Y2() \ - ZGEMV_N_2x1() \ - ZSTORE_Y2() \ - \ - k += 2 * 2; \ - y += inc_y2 * 2; \ - } \ - \ - if (m & 1) \ - { \ - ZGEMV_N_1x1() \ - \ - k += 2; \ - y += inc_y2; \ - } \ - \ - pa0 += lda2; \ - x += inc_x2; \ - } \ +#define ZGEMV_N_MSA() \ + for (j = (n >> 2); j--;) \ + { \ + ZLOAD_X4_SCALE() \ + \ + k = 0; \ + k_pref = pref_offset; \ + y = y_org; \ + \ + for (i = (m >> 2); i--;) \ + { \ + PREFETCH(pa0 + k_pref + 8 + 0); \ + PREFETCH(pa0 + k_pref + 8 + 4); \ + PREFETCH(pa1 + k_pref + 8 + 0); \ + PREFETCH(pa1 + k_pref + 8 + 4); \ + PREFETCH(pa2 + k_pref + 8 + 0); \ + PREFETCH(pa2 + k_pref + 8 + 4); \ + PREFETCH(pa3 + k_pref + 8 + 0); \ + PREFETCH(pa3 + k_pref + 8 + 4); \ + \ + ZLOAD_Y4() \ + ZGEMV_N_4x4() \ + ZSTORE_Y4() \ + \ + k += 2 * 4; \ + k_pref += 2 * 4; \ + y += inc_y2 * 4; \ + } \ + \ + if (m & 2) \ + { \ + ZLOAD_Y2() \ + ZGEMV_N_2x4() \ + ZSTORE_Y2() \ + \ + k += 2 * 2; \ + y += inc_y2 * 2; \ + } \ + \ + if (m & 1) \ + { \ + temp0_r = tp4r[0]; \ + temp1_r = tp4r[1]; \ + temp2_r = tp5r[0]; \ + temp3_r = tp5r[1]; \ + \ + temp0_i = tp4i[0]; \ + temp1_i = tp4i[1]; \ + temp2_i = tp5i[0]; \ + temp3_i = tp5i[1]; \ + \ + ZGEMV_N_1x4() \ + k += 2; \ + y += inc_y2; \ + } \ + \ + pa0 += 4 * lda2; \ + pa1 += 4 * lda2; \ + pa2 += 4 * lda2; \ + pa3 += 4 * lda2; \ + \ + x += 4 * inc_x2; \ + } \ + \ + if (n & 2) \ + { \ + ZLOAD_X2_SCALE() \ + \ + k = 0; \ + y = y_org; \ + \ + for (i = (m >> 2); i--;) \ + { \ + ZLOAD_Y4() \ + ZGEMV_N_4x2() \ + ZSTORE_Y4() \ + \ + k += 2 * 4; \ + y += inc_y2 * 4; \ + } \ + \ + if (m & 2) \ + { \ + ZLOAD_Y2() \ + ZGEMV_N_2x2() \ + ZSTORE_Y2() \ + \ + k += 2 * 2; \ + y += inc_y2 * 2; \ + } \ + \ + if (m & 1) \ + { \ + temp0_r = tp4r[0]; \ + temp1_r = tp4r[1]; \ + \ + temp0_i = tp4i[0]; \ + temp1_i = tp4i[1]; \ + \ + ZGEMV_N_1x2() \ + \ + k += 2; \ + y += inc_y2; \ + } \ + \ + pa0 += 2 * lda2; \ + pa1 += 2 * lda2; \ + \ + x += 2 * inc_x2; \ + } \ + \ + if (n & 1) \ + { \ + ZLOAD_X1_SCALE() \ + \ + k = 0; \ + y = y_org; \ + \ + for (i = (m >> 2); i--;) \ + { \ + ZLOAD_Y4() \ + ZGEMV_N_4x1() \ + ZSTORE_Y4() \ + \ + k += 2 * 4; \ + y += inc_y2 * 4; \ + } \ + \ + if (m & 2) \ + { \ + ZLOAD_Y2() \ + ZGEMV_N_2x1() \ + ZSTORE_Y2() \ + \ + k += 2 * 2; \ + y += inc_y2 * 2; \ + } \ + \ + if (m & 1) \ + { \ + ZGEMV_N_1x1() \ + \ + k += 2; \ + y += inc_y2; \ + } \ + \ + pa0 += lda2; \ + x += inc_x2; \ + } \ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r, FLOAT alpha_i, FLOAT *A, BLASLONG lda2, FLOAT *x, BLASLONG inc_x2, FLOAT *y, BLASLONG inc_y2, FLOAT *buffer) { - BLASLONG i, j, k; + BLASLONG i, j, k, k_pref, pref_offset; FLOAT *y_org = y; FLOAT *pa0, *pa1, *pa2, *pa3; FLOAT temp0_r, temp1_r, temp2_r, temp3_r, temp0_i, temp1_i, temp2_i; @@ -569,6 +580,10 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r, FLOAT alpha_i, inc_x2 = 2 * inc_x2; inc_y2 = 2 * inc_y2; + pref_offset = (uintptr_t)A & (L1_DATA_LINESIZE - 1); + pref_offset = L1_DATA_LINESIZE - pref_offset; + pref_offset = pref_offset / sizeof(FLOAT); + pa0 = A; pa1 = A + lda2; pa2 = A + 2 * lda2; diff --git a/kernel/mips/zgemv_t_msa.c b/kernel/mips/zgemv_t_msa.c index b2147b045..e6febb577 100644 --- a/kernel/mips/zgemv_t_msa.c +++ b/kernel/mips/zgemv_t_msa.c @@ -44,64 +44,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define OP2 -= #endif -#define ZGEMV_T_4x4() \ - LD_DP4(pa0 + k, 2, t0, t1, t2, t3); \ - LD_DP4(pa1 + k, 2, t4, t5, t6, t7); \ - LD_DP4(pa2 + k, 2, t8, t9, t10, t11); \ - LD_DP4(pa3 + k, 2, t12, t13, t14, t15); \ - \ - PCKEVOD_D2_DP(t1, t0, src0r, src0i); \ - PCKEVOD_D2_DP(t3, t2, src1r, src1i); \ - PCKEVOD_D2_DP(t5, t4, src2r, src2i); \ - PCKEVOD_D2_DP(t7, t6, src3r, src3i); \ - PCKEVOD_D2_DP(t9, t8, src4r, src4i); \ - PCKEVOD_D2_DP(t11, t10, src5r, src5i); \ - PCKEVOD_D2_DP(t13, t12, src6r, src6i); \ - PCKEVOD_D2_DP(t15, t14, src7r, src7i); \ - \ - tp0r += src0r * x0r; \ - tp0r += src1r * x1r; \ - tp0r OP0 src0i * x0i; \ - tp0r OP0 src1i * x1i; \ - \ - tp1r += src2r * x0r; \ - tp1r += src3r * x1r; \ - tp1r OP0 src2i * x0i; \ - tp1r OP0 src3i * x1i; \ - \ - tp2r += src4r * x0r; \ - tp2r += src5r * x1r; \ - tp2r OP0 src4i * x0i; \ - tp2r OP0 src5i * x1i; \ - \ - tp3r += src6r * x0r; \ - tp3r += src7r * x1r; \ - tp3r OP0 src6i * x0i; \ - tp3r OP0 src7i * x1i; \ - \ - tp0i OP1 src0r * x0i; \ - tp0i OP1 src1r * x1i; \ - tp0i OP2 src0i * x0r; \ - tp0i OP2 src1i * x1r; \ - \ - tp1i OP1 src2r * x0i; \ - tp1i OP1 src3r * x1i; \ - tp1i OP2 src2i * x0r; \ - tp1i OP2 src3i * x1r; \ - \ - tp2i OP1 src4r * x0i; \ - tp2i OP1 src5r * x1i; \ - tp2i OP2 src4i * x0r; \ - tp2i OP2 src5i * x1r; \ - \ - tp3i OP1 src6r * x0i; \ - tp3i OP1 src7r * x1i; \ - tp3i OP2 src6i * x0r; \ - tp3i OP2 src7i * x1r; \ - -#define ZGEMV_T_4x2() \ - LD_DP4(pa0 + k, 2, t0, t1, t2, t3); \ - LD_DP4(pa1 + k, 2, t4, t5, t6, t7); \ +#define ZGEMV_T_8x1() \ + LD_DP4(pa0, 2, t0, t1, t2, t3); \ + LD_DP4(pa0 + 8, 2, t4, t5, t6, t7); \ \ PCKEVOD_D2_DP(t1, t0, src0r, src0i); \ PCKEVOD_D2_DP(t3, t2, src1r, src1i); \ @@ -109,27 +54,27 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PCKEVOD_D2_DP(t7, t6, src3r, src3i); \ \ tp0r += src0r * x0r; \ - tp0r += src1r * x1r; \ - tp0r OP0 src0i * x0i; \ - tp0r OP0 src1i * x1i; \ - \ - tp1r += src2r * x0r; \ - tp1r += src3r * x1r; \ - tp1r OP0 src2i * x0i; \ - tp1r OP0 src3i * x1i; \ - \ tp0i OP1 src0r * x0i; \ - tp0i OP1 src1r * x1i; \ + tp0r OP0 src0i * x0i; \ tp0i OP2 src0i * x0r; \ + \ + tp0r += src2r * x2r; \ + tp0i OP1 src2r * x2i; \ + tp0r OP0 src2i * x2i; \ + tp0i OP2 src2i * x2r; \ + \ + tp0r += src1r * x1r; \ + tp0i OP1 src1r * x1i; \ + tp0r OP0 src1i * x1i; \ tp0i OP2 src1i * x1r; \ \ - tp1i OP1 src2r * x0i; \ - tp1i OP1 src3r * x1i; \ - tp1i OP2 src2i * x0r; \ - tp1i OP2 src3i * x1r; \ + tp0r += src3r * x3r; \ + tp0i OP1 src3r * x3i; \ + tp0r OP0 src3i * x3i; \ + tp0i OP2 src3i * x3r; \ #define ZGEMV_T_4x1() \ - LD_DP4(pa0 + k, 2, t0, t1, t2, t3); \ + LD_DP4(pa0, 2, t0, t1, t2, t3); \ \ PCKEVOD_D2_DP(t1, t0, src0r, src0i); \ PCKEVOD_D2_DP(t3, t2, src1r, src1i); \ @@ -144,62 +89,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. tp0i OP2 src0i * x0r; \ tp0i OP2 src1i * x1r; \ -#define ZGEMV_T_2x4() \ - LD_DP2(pa0 + k, 2, t0, t1); \ - LD_DP2(pa1 + k, 2, t4, t5); \ - LD_DP2(pa2 + k, 2, t8, t9); \ - LD_DP2(pa3 + k, 2, t12, t13); \ - \ - PCKEVOD_D2_DP(t1, t0, src0r, src0i); \ - PCKEVOD_D2_DP(t5, t4, src2r, src2i); \ - PCKEVOD_D2_DP(t9, t8, src4r, src4i); \ - PCKEVOD_D2_DP(t13, t12, src6r, src6i); \ - \ - tp0r += src0r * x0r; \ - tp0r OP0 src0i * x0i; \ - \ - tp1r += src2r * x0r; \ - tp1r OP0 src2i * x0i; \ - \ - tp2r += src4r * x0r; \ - tp2r OP0 src4i * x0i; \ - \ - tp3r += src6r * x0r; \ - tp3r OP0 src6i * x0i; \ - \ - tp0i OP1 src0r * x0i; \ - tp0i OP2 src0i * x0r; \ - \ - tp1i OP1 src2r * x0i; \ - tp1i OP2 src2i * x0r; \ - \ - tp2i OP1 src4r * x0i; \ - tp2i OP2 src4i * x0r; \ - \ - tp3i OP1 src6r * x0i; \ - tp3i OP2 src6i * x0r; \ - -#define ZGEMV_T_2x2() \ - LD_DP2(pa0 + k, 2, t0, t1); \ - LD_DP2(pa1 + k, 2, t4, t5); \ - \ - PCKEVOD_D2_DP(t1, t0, src0r, src0i); \ - PCKEVOD_D2_DP(t5, t4, src2r, src2i); \ - \ - tp0r += src0r * x0r; \ - tp0r OP0 src0i * x0i; \ - \ - tp1r += src2r * x0r; \ - tp1r OP0 src2i * x0i; \ - \ - tp0i OP1 src0r * x0i; \ - tp0i OP2 src0i * x0r; \ - \ - tp1i OP1 src2r * x0i; \ - tp1i OP2 src2i * x0r; \ - #define ZGEMV_T_2x1() \ - LD_DP2(pa0 + k, 2, t0, t1); \ + LD_DP2(pa0, 2, t0, t1); \ \ PCKEVOD_D2_DP(t1, t0, src0r, src0i); \ \ @@ -209,104 +100,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. tp0i OP1 src0r * x0i; \ tp0i OP2 src0i * x0r; \ -#define ZGEMV_T_1x4() \ - temp0r += pa0[k + 0] * x[0 * inc_x2]; \ - temp0r OP0 pa0[k + 1] * x[0 * inc_x2 + 1]; \ - temp1r += pa1[k + 0] * x[0 * inc_x2]; \ - temp1r OP0 pa1[k + 1] * x[0 * inc_x2 + 1]; \ - temp2r += pa2[k + 0] * x[0 * inc_x2]; \ - temp2r OP0 pa2[k + 1] * x[0 * inc_x2 + 1]; \ - temp3r += pa3[k + 0] * x[0 * inc_x2]; \ - temp3r OP0 pa3[k + 1] * x[0 * inc_x2 + 1]; \ - \ - temp0i OP1 pa0[k + 0] * x[0 * inc_x2 + 1]; \ - temp0i OP2 pa0[k + 1] * x[0 * inc_x2]; \ - temp1i OP1 pa1[k + 0] * x[0 * inc_x2 + 1]; \ - temp1i OP2 pa1[k + 1] * x[0 * inc_x2]; \ - temp2i OP1 pa2[k + 0] * x[0 * inc_x2 + 1]; \ - temp2i OP2 pa2[k + 1] * x[0 * inc_x2]; \ - temp3i OP1 pa3[k + 0] * x[0 * inc_x2 + 1]; \ - temp3i OP2 pa3[k + 1] * x[0 * inc_x2]; \ - -#define ZGEMV_T_1x2() \ - temp0r += pa0[k + 0] * x[0 * inc_x2]; \ - temp0r OP0 pa0[k + 1] * x[0 * inc_x2 + 1]; \ - temp1r += pa1[k + 0] * x[0 * inc_x2]; \ - temp1r OP0 pa1[k + 1] * x[0 * inc_x2 + 1]; \ - \ - temp0i OP1 pa0[k + 0] * x[0 * inc_x2 + 1]; \ - temp0i OP2 pa0[k + 1] * x[0 * inc_x2]; \ - temp1i OP1 pa1[k + 0] * x[0 * inc_x2 + 1]; \ - temp1i OP2 pa1[k + 1] * x[0 * inc_x2]; \ - -#define ZGEMV_T_1x1() \ - temp0r += pa0[k + 0] * x[0 * inc_x2]; \ - temp0r OP0 pa0[k + 1] * x[0 * inc_x2 + 1]; \ - \ - temp0i OP1 pa0[k + 0] * x[0 * inc_x2 + 1]; \ - temp0i OP2 pa0[k + 1] * x[0 * inc_x2]; \ - -#define ZSCALE_STORE_Y4_GP() \ - res0r = y[0 * inc_y2]; \ - res1r = y[1 * inc_y2]; \ - res2r = y[2 * inc_y2]; \ - res3r = y[3 * inc_y2]; \ - \ - res0i = y[0 * inc_y2 + 1]; \ - res1i = y[1 * inc_y2 + 1]; \ - res2i = y[2 * inc_y2 + 1]; \ - res3i = y[3 * inc_y2 + 1]; \ - \ - res0r += alphar * temp0r; \ - res0r OP0 alphai * temp0i; \ - res1r += alphar * temp1r; \ - res1r OP0 alphai * temp1i; \ - res2r += alphar * temp2r; \ - res2r OP0 alphai * temp2i; \ - res3r += alphar * temp3r; \ - res3r OP0 alphai * temp3i; \ - \ - res0i OP1 alphar * temp0i; \ - res0i OP2 alphai * temp0r; \ - res1i OP1 alphar * temp1i; \ - res1i OP2 alphai * temp1r; \ - res2i OP1 alphar * temp2i; \ - res2i OP2 alphai * temp2r; \ - res3i OP1 alphar * temp3i; \ - res3i OP2 alphai * temp3r; \ - \ - y[0 * inc_y2] = res0r; \ - y[1 * inc_y2] = res1r; \ - y[2 * inc_y2] = res2r; \ - y[3 * inc_y2] = res3r; \ - \ - y[0 * inc_y2 + 1] = res0i; \ - y[1 * inc_y2 + 1] = res1i; \ - y[2 * inc_y2 + 1] = res2i; \ - y[3 * inc_y2 + 1] = res3i; \ - -#define ZSCALE_STORE_Y2_GP() \ - res0r = y[0 * inc_y2]; \ - res1r = y[1 * inc_y2]; \ - \ - res0i = y[0 * inc_y2 + 1]; \ - res1i = y[1 * inc_y2 + 1]; \ - \ - res0r += alphar * temp0r; \ - res0r OP0 alphai * temp0i; \ - res1r += alphar * temp1r; \ - res1r OP0 alphai * temp1i; \ - \ - res0i OP1 alphar * temp0i; \ - res0i OP2 alphai * temp0r; \ - res1i OP1 alphar * temp1i; \ - res1i OP2 alphai * temp1r; \ - \ - y[0 * inc_y2] = res0r; \ - y[1 * inc_y2] = res1r; \ - \ - y[0 * inc_y2 + 1] = res0i; \ - y[1 * inc_y2 + 1] = res1i; \ +#define ZGEMV_T_1x1() \ + temp0r += pa0[0] * x[0 * inc_x2]; \ + temp0r OP0 pa0[1] * x[0 * inc_x2 + 1]; \ + \ + temp0i OP1 pa0[0] * x[0 * inc_x2 + 1]; \ + temp0i OP2 pa0[1] * x[0 * inc_x2]; \ #define ZSCALE_STORE_Y1_GP() \ res0r = y[0 * inc_y2]; \ @@ -321,6 +120,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. y[0 * inc_y2] = res0r; \ y[0 * inc_y2 + 1] = res0i; \ +#define ZLOAD_X8_VECTOR() \ + LD_DP4(x, 2, x0, x1, x2, x3); \ + LD_DP4(x + 8, 2, x4, x5, x6, x7); \ + \ + PCKEVOD_D2_DP(x1, x0, x0r, x0i); \ + PCKEVOD_D2_DP(x3, x2, x1r, x1i); \ + PCKEVOD_D2_DP(x5, x4, x2r, x2i); \ + PCKEVOD_D2_DP(x7, x6, x3r, x3i); \ + #define ZLOAD_X4_VECTOR() \ LD_DP4(x, 2, x0, x1, x2, x3); \ PCKEVOD_D2_DP(x1, x0, x0r, x0i); \ @@ -330,6 +138,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. LD_DP2(x, 2, x0, x1); \ PCKEVOD_D2_DP(x1, x0, x0r, x0i); \ +#define ZLOAD_X8_GP() \ + x0r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *) (x + 0 * inc_x2))); \ + x0r = (v2f64) __msa_insert_d((v2i64) x0r, 1, *((long long *) (x + 1 * inc_x2))); \ + x1r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *) (x + 2 * inc_x2))); \ + x1r = (v2f64) __msa_insert_d((v2i64) x1r, 1, *((long long *) (x + 3 * inc_x2))); \ + x2r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *) (x + 4 * inc_x2))); \ + x2r = (v2f64) __msa_insert_d((v2i64) x2r, 1, *((long long *) (x + 5 * inc_x2))); \ + x3r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *) (x + 6 * inc_x2))); \ + x3r = (v2f64) __msa_insert_d((v2i64) x3r, 1, *((long long *) (x + 7 * inc_x2))); \ + x0i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *) (x + 0 * inc_x2 + 1))); \ + x0i = (v2f64) __msa_insert_d((v2i64) x0i, 1, *((long long *) (x + 1 * inc_x2 + 1))); \ + x1i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *) (x + 2 * inc_x2 + 1))); \ + x1i = (v2f64) __msa_insert_d((v2i64) x1i, 1, *((long long *) (x + 3 * inc_x2 + 1))); \ + x2i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *) (x + 4 * inc_x2 + 1))); \ + x2i = (v2f64) __msa_insert_d((v2i64) x2i, 1, *((long long *) (x + 5 * inc_x2 + 1))); \ + x3i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *) (x + 6 * inc_x2 + 1))); \ + x3i = (v2f64) __msa_insert_d((v2i64) x3i, 1, *((long long *) (x + 7 * inc_x2 + 1))); \ + #define ZLOAD_X4_GP() \ x0r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *) (x + 0 * inc_x2))); \ x0r = (v2f64) __msa_insert_d((v2i64) x0r, 1, *((long long *) (x + 1 * inc_x2))); \ @@ -346,196 +172,405 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. x0i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *) (x + 0 * inc_x2 + 1))); \ x0i = (v2f64) __msa_insert_d((v2i64) x0i, 1, *((long long *) (x + 1 * inc_x2 + 1))); \ -#define ZGEMV_T_MSA() \ - for (j = (n >> 2); j--;) \ - { \ - tp0r = tp1r = tp2r = tp3r = zero; \ - tp0i = tp1i = tp2i = tp3i = zero; \ - \ - k = 0; \ - x = srcx_org; \ - \ - for (i = (m >> 2); i--;) \ - { \ - ZLOAD_X4(); \ - ZGEMV_T_4x4(); \ - \ - k += 2 * 4; \ - x += inc_x2 * 4; \ - } \ - \ - if (m & 2) \ - { \ - ZLOAD_X2(); \ - ZGEMV_T_2x4(); \ - \ - k += 2 * 2; \ - x += inc_x2 * 2; \ - } \ - \ - temp0r = tp0r[0] + tp0r[1]; \ - temp1r = tp1r[0] + tp1r[1]; \ - temp2r = tp2r[0] + tp2r[1]; \ - temp3r = tp3r[0] + tp3r[1]; \ - temp0i = tp0i[0] + tp0i[1]; \ - temp1i = tp1i[0] + tp1i[1]; \ - temp2i = tp2i[0] + tp2i[1]; \ - temp3i = tp3i[0] + tp3i[1]; \ - \ - if (m & 1) \ - { \ - ZGEMV_T_1x4(); \ - \ - k += 2; \ - x += inc_x2; \ - } \ - \ - ZSCALE_STORE_Y4_GP(); \ - \ - pa0 += 4 * lda2; \ - pa1 += 4 * lda2; \ - pa2 += 4 * lda2; \ - pa3 += 4 * lda2; \ - y += 4 * inc_y2; \ - } \ - \ - if (n & 2) \ - { \ - tp0r = tp1r = zero; \ - tp0i = tp1i = zero; \ - \ - k = 0; \ - x = srcx_org; \ - \ - for (i = (m >> 2); i--;) \ - { \ - ZLOAD_X4(); \ - ZGEMV_T_4x2(); \ - \ - k += 2 * 4; \ - x += inc_x2 * 4; \ - } \ - \ - if (m & 2) \ - { \ - ZLOAD_X2(); \ - ZGEMV_T_2x2(); \ - \ - k += 2 * 2; \ - x += inc_x2 * 2; \ - } \ - \ - temp0r = tp0r[0] + tp0r[1]; \ - temp1r = tp1r[0] + tp1r[1]; \ - temp0i = tp0i[0] + tp0i[1]; \ - temp1i = tp1i[0] + tp1i[1]; \ - \ - if (m & 1) \ - { \ - ZGEMV_T_1x2(); \ - \ - k += 2; \ - x += inc_x2; \ - } \ - \ - ZSCALE_STORE_Y2_GP(); \ - \ - pa0 += 2 * lda2; \ - pa1 += 2 * lda2; \ - y += 2 * inc_y2; \ - } \ - \ - if (n & 1) \ - { \ - tp0r = zero; \ - tp0i = zero; \ - \ - k = 0; \ - x = srcx_org; \ - \ - for (i = (m >> 2); i--;) \ - { \ - ZLOAD_X4(); \ - ZGEMV_T_4x1(); \ - \ - k += 2 * 4; \ - x += inc_x2 * 4; \ - } \ - \ - if (m & 2) \ - { \ - ZLOAD_X2(); \ - ZGEMV_T_2x1(); \ - \ - k += 2 * 2; \ - x += inc_x2 * 2; \ - } \ - \ - temp0r = tp0r[0] + tp0r[1]; \ - temp0i = tp0i[0] + tp0i[1]; \ - \ - if (m & 1) \ - { \ - ZGEMV_T_1x1(); \ - \ - k += 2; \ - x += inc_x2; \ - } \ - \ - ZSCALE_STORE_Y1_GP(); \ - \ - pa0 += lda2; \ - y += inc_y2; \ - } \ +#define ZGEMV_T_MSA() \ + for (j = n; j--;) \ + { \ + tp0r = zero; \ + tp0i = zero; \ + tp1r = zero; \ + tp1i = zero; \ + tp2r = zero; \ + tp2i = zero; \ + tp3r = zero; \ + tp3i = zero; \ + \ + pa0 = A; \ + x = srcx_org; \ + \ + if (m >> 4) \ + { \ + x0 = LD_DP(x); \ + x1 = LD_DP(x + 1 * inc_x2); \ + t0 = LD_DP(pa0); \ + t1 = LD_DP(pa0 + 2); \ + \ + x4 = LD_DP(x + 4 * inc_x2); \ + x5 = LD_DP(x + 5 * inc_x2); \ + t4 = LD_DP(pa0 + 8); \ + t5 = LD_DP(pa0 + 10); \ + \ + for (i = (m >> 4) - 1; i--;) \ + { \ + pa0_pref = pa0 + pref_offset; \ + \ + PREFETCH(pa0_pref + 36); \ + PREFETCH(pa0_pref + 44); \ + PREFETCH(pa0_pref + 48); \ + PREFETCH(pa0_pref + 52); \ + PREFETCH(pa0_pref + 56); \ + PREFETCH(pa0_pref + 60); \ + PREFETCH(pa0_pref + 64); \ + PREFETCH(pa0_pref + 72); \ + \ + x0r = (v2f64) __msa_pckev_d((v2i64) x1, (v2i64) x0); \ + x0i = (v2f64) __msa_pckod_d((v2i64) x1, (v2i64) x0); \ + src0r = (v2f64) __msa_pckev_d((v2i64) t1, (v2i64) t0); \ + src0i = (v2f64) __msa_pckod_d((v2i64) t1, (v2i64) t0); \ + \ + tp0r += src0r * x0r; \ + x2 = LD_DP(x + 2 * inc_x2); \ + x2r = (v2f64) __msa_pckev_d((v2i64) x5, (v2i64) x4); \ + \ + tp0i OP1 src0r * x0i; \ + x3 = LD_DP(x + 3 * inc_x2); \ + x2i = (v2f64) __msa_pckod_d((v2i64) x5, (v2i64) x4); \ + \ + tp1r OP0 src0i * x0i; \ + t2 = LD_DP(pa0 + 4); \ + src2r = (v2f64) __msa_pckev_d((v2i64) t5, (v2i64) t4); \ + \ + tp1i OP2 src0i * x0r; \ + t3 = LD_DP(pa0 + 6); \ + src2i = (v2f64) __msa_pckod_d((v2i64) t5, (v2i64) t4); \ + \ + tp2r += src2r * x2r; \ + x6 = LD_DP(x + 6 * inc_x2); \ + \ + tp2i OP1 src2r * x2i; \ + x7 = LD_DP(x + 7 * inc_x2); \ + \ + tp3r OP0 src2i * x2i; \ + t6 = LD_DP(pa0 + 12); \ + \ + tp3i OP2 src2i * x2r; \ + t7 = LD_DP(pa0 + 14); \ + \ + x1r = (v2f64) __msa_pckev_d((v2i64) x3, (v2i64) x2); \ + x1i = (v2f64) __msa_pckod_d((v2i64) x3, (v2i64) x2); \ + src1r = (v2f64) __msa_pckev_d((v2i64) t3, (v2i64) t2); \ + src1i = (v2f64) __msa_pckod_d((v2i64) t3, (v2i64) t2); \ + \ + tp0r += src1r * x1r; \ + x0 = LD_DP(x + 8 * inc_x2); \ + x3r = (v2f64) __msa_pckev_d((v2i64) x7, (v2i64) x6); \ + \ + tp0i OP1 src1r * x1i; \ + x1 = LD_DP(x + 9 * inc_x2); \ + x3i = (v2f64) __msa_pckod_d((v2i64) x7, (v2i64) x6); \ + \ + tp1r OP0 src1i * x1i; \ + t0 = LD_DP(pa0 + 16); \ + src3r = (v2f64) __msa_pckev_d((v2i64) t7, (v2i64) t6); \ + \ + tp1i OP2 src1i * x1r; \ + t1 = LD_DP(pa0 + 18); \ + src3i = (v2f64) __msa_pckod_d((v2i64) t7, (v2i64) t6); \ + \ + tp2r += src3r * x3r; \ + x4 = LD_DP(x + 12 * inc_x2); \ + \ + tp2i OP1 src3r * x3i; \ + x5 = LD_DP(x + 13 * inc_x2); \ + \ + tp3r OP0 src3i * x3i; \ + t4 = LD_DP(pa0 + 24); \ + \ + tp3i OP2 src3i * x3r; \ + t5 = LD_DP(pa0 + 26); \ + \ + x0r = (v2f64) __msa_pckev_d((v2i64) x1, (v2i64) x0); \ + x0i = (v2f64) __msa_pckod_d((v2i64) x1, (v2i64) x0); \ + src0r = (v2f64) __msa_pckev_d((v2i64) t1, (v2i64) t0); \ + src0i = (v2f64) __msa_pckod_d((v2i64) t1, (v2i64) t0); \ + \ + tp0r += src0r * x0r; \ + x2 = LD_DP(x + 10 * inc_x2); \ + x2r = (v2f64) __msa_pckev_d((v2i64) x5, (v2i64) x4); \ + \ + tp0i OP1 src0r * x0i; \ + x3 = LD_DP(x + 11 * inc_x2); \ + x2i = (v2f64) __msa_pckod_d((v2i64) x5, (v2i64) x4); \ + \ + tp1r OP0 src0i * x0i; \ + t2 = LD_DP(pa0 + 20); \ + src2r = (v2f64) __msa_pckev_d((v2i64) t5, (v2i64) t4); \ + \ + tp1i OP2 src0i * x0r; \ + t3 = LD_DP(pa0 + 22); \ + src2i = (v2f64) __msa_pckod_d((v2i64) t5, (v2i64) t4); \ + \ + tp2r += src2r * x2r; \ + x6 = LD_DP(x + 14 * inc_x2); \ + \ + tp2i OP1 src2r * x2i; \ + x7 = LD_DP(x + 15 * inc_x2); \ + \ + tp3r OP0 src2i * x2i; \ + t6 = LD_DP(pa0 + 28); \ + \ + tp3i OP2 src2i * x2r; \ + t7 = LD_DP(pa0 + 30); \ + \ + x1r = (v2f64) __msa_pckev_d((v2i64) x3, (v2i64) x2); \ + x1i = (v2f64) __msa_pckod_d((v2i64) x3, (v2i64) x2); \ + src1r = (v2f64) __msa_pckev_d((v2i64) t3, (v2i64) t2); \ + src1i = (v2f64) __msa_pckod_d((v2i64) t3, (v2i64) t2); \ + \ + tp0r += src1r * x1r; \ + x0 = LD_DP(x + inc_x2 * 16); \ + x3r = (v2f64) __msa_pckev_d((v2i64) x7, (v2i64) x6); \ + \ + tp0i OP1 src1r * x1i; \ + x1 = LD_DP(x + inc_x2 * 16 + 1 * inc_x2); \ + x3i = (v2f64) __msa_pckod_d((v2i64) x7, (v2i64) x6); \ + \ + tp1r OP0 src1i * x1i; \ + t0 = LD_DP(pa0 + 2 * 16); \ + src3r = (v2f64) __msa_pckev_d((v2i64) t7, (v2i64) t6); \ + \ + tp1i OP2 src1i * x1r; \ + t1 = LD_DP(pa0 + 2 * 16 + 2); \ + src3i = (v2f64) __msa_pckod_d((v2i64) t7, (v2i64) t6); \ + \ + tp2r += src3r * x3r; \ + x4 = LD_DP(x + inc_x2 * 16 + 4 * inc_x2); \ + \ + tp2i OP1 src3r * x3i; \ + x5 = LD_DP(x + inc_x2 * 16 + 5 * inc_x2); \ + \ + tp3r OP0 src3i * x3i; \ + t4 = LD_DP(pa0 + 2 * 16 + 8); \ + \ + tp3i OP2 src3i * x3r; \ + t5 = LD_DP(pa0 + 2 * 16 + 10); \ + \ + pa0 += 2 * 16; \ + x += inc_x2 * 16; \ + } \ + \ + x0r = (v2f64) __msa_pckev_d((v2i64) x1, (v2i64) x0); \ + x0i = (v2f64) __msa_pckod_d((v2i64) x1, (v2i64) x0); \ + src0r = (v2f64) __msa_pckev_d((v2i64) t1, (v2i64) t0); \ + src0i = (v2f64) __msa_pckod_d((v2i64) t1, (v2i64) t0); \ + \ + tp0r += src0r * x0r; \ + x2 = LD_DP(x + 2 * inc_x2); \ + x2r = (v2f64) __msa_pckev_d((v2i64) x5, (v2i64) x4); \ + \ + tp0i OP1 src0r * x0i; \ + x3 = LD_DP(x + 3 * inc_x2); \ + x2i = (v2f64) __msa_pckod_d((v2i64) x5, (v2i64) x4); \ + \ + tp1r OP0 src0i * x0i; \ + t2 = LD_DP(pa0 + 4); \ + src2r = (v2f64) __msa_pckev_d((v2i64) t5, (v2i64) t4); \ + \ + tp1i OP2 src0i * x0r; \ + t3 = LD_DP(pa0 + 6); \ + src2i = (v2f64) __msa_pckod_d((v2i64) t5, (v2i64) t4); \ + \ + tp2r += src2r * x2r; \ + x6 = LD_DP(x + 6 * inc_x2); \ + \ + tp2i OP1 src2r * x2i; \ + x7 = LD_DP(x + 7 * inc_x2); \ + \ + tp3r OP0 src2i * x2i; \ + t6 = LD_DP(pa0 + 12); \ + \ + tp3i OP2 src2i * x2r; \ + t7 = LD_DP(pa0 + 14); \ + \ + x1r = (v2f64) __msa_pckev_d((v2i64) x3, (v2i64) x2); \ + x1i = (v2f64) __msa_pckod_d((v2i64) x3, (v2i64) x2); \ + src1r = (v2f64) __msa_pckev_d((v2i64) t3, (v2i64) t2); \ + src1i = (v2f64) __msa_pckod_d((v2i64) t3, (v2i64) t2); \ + \ + tp0r += src1r * x1r; \ + x0 = LD_DP(x + 8 * inc_x2); \ + x3r = (v2f64) __msa_pckev_d((v2i64) x7, (v2i64) x6); \ + \ + tp0i OP1 src1r * x1i; \ + x1 = LD_DP(x + 9 * inc_x2); \ + x3i = (v2f64) __msa_pckod_d((v2i64) x7, (v2i64) x6); \ + \ + tp1r OP0 src1i * x1i; \ + t0 = LD_DP(pa0 + 16); \ + src3r = (v2f64) __msa_pckev_d((v2i64) t7, (v2i64) t6); \ + \ + tp1i OP2 src1i * x1r; \ + t1 = LD_DP(pa0 + 18); \ + src3i = (v2f64) __msa_pckod_d((v2i64) t7, (v2i64) t6); \ + \ + tp2r += src3r * x3r; \ + x4 = LD_DP(x + 12 * inc_x2); \ + \ + tp2i OP1 src3r * x3i; \ + x5 = LD_DP(x + 13 * inc_x2); \ + \ + tp3r OP0 src3i * x3i; \ + t4 = LD_DP(pa0 + 24); \ + \ + tp3i OP2 src3i * x3r; \ + t5 = LD_DP(pa0 + 26); \ + \ + x0r = (v2f64) __msa_pckev_d((v2i64) x1, (v2i64) x0); \ + x0i = (v2f64) __msa_pckod_d((v2i64) x1, (v2i64) x0); \ + src0r = (v2f64) __msa_pckev_d((v2i64) t1, (v2i64) t0); \ + src0i = (v2f64) __msa_pckod_d((v2i64) t1, (v2i64) t0); \ + \ + tp0r += src0r * x0r; \ + x2 = LD_DP(x + 10 * inc_x2); \ + x2r = (v2f64) __msa_pckev_d((v2i64) x5, (v2i64) x4); \ + \ + tp0i OP1 src0r * x0i; \ + x3 = LD_DP(x + 11 * inc_x2); \ + x2i = (v2f64) __msa_pckod_d((v2i64) x5, (v2i64) x4); \ + \ + tp1r OP0 src0i * x0i; \ + t2 = LD_DP(pa0 + 20); \ + src2r = (v2f64) __msa_pckev_d((v2i64) t5, (v2i64) t4); \ + \ + tp1i OP2 src0i * x0r; \ + t3 = LD_DP(pa0 + 22); \ + src2i = (v2f64) __msa_pckod_d((v2i64) t5, (v2i64) t4); \ + \ + tp2r += src2r * x2r; \ + x6 = LD_DP(x + 14 * inc_x2); \ + \ + tp2i OP1 src2r * x2i; \ + x7 = LD_DP(x + 15 * inc_x2); \ + \ + tp3r OP0 src2i * x2i; \ + t6 = LD_DP(pa0 + 28); \ + \ + tp3i OP2 src2i * x2r; \ + t7 = LD_DP(pa0 + 30); \ + \ + x1r = (v2f64) __msa_pckev_d((v2i64) x3, (v2i64) x2); \ + x1i = (v2f64) __msa_pckod_d((v2i64) x3, (v2i64) x2); \ + src1r = (v2f64) __msa_pckev_d((v2i64) t3, (v2i64) t2); \ + src1i = (v2f64) __msa_pckod_d((v2i64) t3, (v2i64) t2); \ + \ + tp0r += src1r * x1r; \ + x3r = (v2f64) __msa_pckev_d((v2i64) x7, (v2i64) x6); \ + \ + tp0i OP1 src1r * x1i; \ + x3i = (v2f64) __msa_pckod_d((v2i64) x7, (v2i64) x6); \ + \ + tp1r OP0 src1i * x1i; \ + src3r = (v2f64) __msa_pckev_d((v2i64) t7, (v2i64) t6); \ + \ + tp1i OP2 src1i * x1r; \ + src3i = (v2f64) __msa_pckod_d((v2i64) t7, (v2i64) t6); \ + \ + tp2r += src3r * x3r; \ + tp2i OP1 src3r * x3i; \ + tp3r OP0 src3i * x3i; \ + tp3i OP2 src3i * x3r; \ + \ + pa0 += 2 * 16; \ + x += inc_x2 * 16; \ + \ + tp0r += tp1r + tp2r + tp3r; \ + tp0i += tp1i + tp2i + tp3i; \ + } \ + \ + if (m & 8) \ + { \ + ZLOAD_X8(); \ + ZGEMV_T_8x1(); \ + \ + pa0 += 2 * 8; \ + x += inc_x2 * 8; \ + } \ + \ + if (m & 4) \ + { \ + ZLOAD_X4(); \ + ZGEMV_T_4x1(); \ + \ + pa0 += 2 * 4; \ + x += inc_x2 * 4; \ + } \ + \ + if (m & 2) \ + { \ + ZLOAD_X2(); \ + ZGEMV_T_2x1(); \ + \ + pa0 += 2 * 2; \ + x += inc_x2 * 2; \ + } \ + \ + temp0r = tp0r[0] + tp0r[1]; \ + temp0i = tp0i[0] + tp0i[1]; \ + \ + if (m & 1) \ + { \ + ZGEMV_T_1x1(); \ + \ + pa0 += 2; \ + x += inc_x2; \ + } \ + \ + ZSCALE_STORE_Y1_GP(); \ + \ + A += lda2; \ + y += inc_y2; \ + } \ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alphar, FLOAT alphai, FLOAT *A, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer) { - BLASLONG i, j, k; + BLASLONG i, j, pref_offset; BLASLONG inc_x2, inc_y2, lda2; - FLOAT *pa0, *pa1, *pa2, *pa3; + FLOAT *pa0, *pa0_pref; FLOAT *srcx_org = x; - FLOAT temp0r, temp0i, temp2r, temp2i, temp1r, temp1i, temp3r, temp3i; - FLOAT res0r, res0i, res2r, res2i, res1r, res1i, res3r, res3i; + FLOAT temp0r, temp0i; + FLOAT res0r, res0i; v2f64 zero = {0}; v2f64 x0, x1, x2, x3, x0r, x1r, x0i, x1i; - v2f64 t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t13, t14, t15; - v2f64 src0r, src1r, src2r, src3r, src4r, src5r, src6r, src7r; - v2f64 src0i, src1i, src2i, src3i, src4i, src5i, src6i, src7i; + v2f64 x4, x5, x6, x7, x2r, x3r, x2i, x3i; + v2f64 t0, t1, t2, t3, t4, t5, t6, t7; + v2f64 src0r, src1r, src2r, src3r; + v2f64 src0i, src1i, src2i, src3i; v2f64 tp0r, tp1r, tp2r, tp3r, tp0i, tp1i, tp2i, tp3i; lda2 = 2 * lda; - pa0 = A; - pa1 = A + lda2; - pa2 = A + 2 * lda2; - pa3 = A + 3 * lda2; - inc_x2 = 2 * inc_x; inc_y2 = 2 * inc_y; + pref_offset = (uintptr_t)A & L1_DATA_LINESIZE; + pref_offset = L1_DATA_LINESIZE - pref_offset; + pref_offset = pref_offset / sizeof(FLOAT); + if (2 == inc_x2) { + #define ZLOAD_X8 ZLOAD_X8_VECTOR #define ZLOAD_X4 ZLOAD_X4_VECTOR #define ZLOAD_X2 ZLOAD_X2_VECTOR ZGEMV_T_MSA(); + #undef ZLOAD_X8 #undef ZLOAD_X4 #undef ZLOAD_X2 } else { + #define ZLOAD_X8 ZLOAD_X8_GP #define ZLOAD_X4 ZLOAD_X4_GP #define ZLOAD_X2 ZLOAD_X2_GP ZGEMV_T_MSA(); + #undef ZLOAD_X8 #undef ZLOAD_X4 #undef ZLOAD_X2 } - return(0); }