floraachy
/
OpenBLAS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added prefetch to CGEMV and ZGEMV. 

Signed-off-by: Shivraj Patil <shivraj.patil@imgtec.com>
This commit is contained in:
Shivraj Patil 2016-12-27 11:33:51 +05:30
parent 8f9975e013
commit a9bf8a781a
4 changed files with 757 additions and 677 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

249
kernel/mips/cgemv_n_msa.c
View File

@ -376,128 +376,139 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*((int *)(y + 2 * inc_y2 + 1)) = __msa_copy_s_w((v4i32) y0i, 2); \
*((int *)(y + 3 * inc_y2 + 1)) = __msa_copy_s_w((v4i32) y0i, 3); \
#define CGEMV_N_MSA() \
for (j = (n >> 2); j--;) \
{ \
CLOAD_X4_SCALE(); \
\
k = 0; \
y = y_org; \
\
for (i = (m >> 3); i--;) \
{ \
CLOAD_Y8() \
CGEMV_N_8x4(); \
CSTORE_Y8(); \
\
k += 2 * 8; \
y += inc_y2 * 8; \
} \
\
if (m & 4) \
{ \
CLOAD_Y4(); \
CGEMV_N_4x4(); \
CSTORE_Y4(); \
\
k += 2 * 4; \
y += inc_y2 * 4; \
} \
\
if (m & 3) \
{ \
temp0_r = tp4r[0]; \
temp1_r = tp4r[1]; \
temp2_r = tp4r[2]; \
temp3_r = tp4r[3]; \
\
temp0_i = tp4i[0]; \
temp1_i = tp4i[1]; \
temp2_i = tp4i[2]; \
temp3_i = tp4i[3]; \
\
for (i = (m & 3); i--;) \
{ \
CGEMV_N_1x4(); \
\
k += 2; \
y += inc_y2; \
} \
} \
\
pa0 += 4 * lda2; \
pa1 += 4 * lda2; \
pa2 += 4 * lda2; \
pa3 += 4 * lda2; \
\
x += 4 * inc_x2; \
} \
\
if (n & 2) \
{ \
CLOAD_X2_SCALE(); \
\
k = 0; \
y = y_org; \
\
for (i = (m >> 3); i--;) \
{ \
CLOAD_Y8(); \
CGEMV_N_8x2(); \
CSTORE_Y8(); \
\
k += 2 * 8; \
y += inc_y2 * 8; \
} \
\
if (m & 4) \
{ \
CLOAD_Y4(); \
CGEMV_N_4x2(); \
CSTORE_Y4(); \
\
k += 2 * 4; \
y += inc_y2 * 4; \
} \
\
for (i = (m & 3); i--;) \
{ \
CGEMV_N_1x2(); \
\
k += 2; \
y += inc_y2; \
} \
\
pa0 += 2 * lda2; \
pa1 += 2 * lda2; \
\
x += 2 * inc_x2; \
} \
\
if (n & 1) \
{ \
CLOAD_X1_SCALE(); \
\
k = 0; \
y = y_org; \
\
for (i = m; i--;) \
{ \
CGEMV_N_1x1(); \
\
k += 2; \
y += inc_y2; \
} \
\
pa0 += lda2; \
x += inc_x2; \
} \
#define CGEMV_N_MSA() \
for (j = (n >> 2); j--;) \
{ \
CLOAD_X4_SCALE(); \
\
k = 0; \
k_pref = pref_offset; \
y = y_org; \
\
for (i = (m >> 3); i--;) \
{ \
PREFETCH(pa0 + k_pref + 16 + 0); \
PREFETCH(pa0 + k_pref + 16 + 8); \
PREFETCH(pa1 + k_pref + 16 + 0); \
PREFETCH(pa1 + k_pref + 16 + 8); \
PREFETCH(pa2 + k_pref + 16 + 0); \
PREFETCH(pa2 + k_pref + 16 + 8); \
PREFETCH(pa3 + k_pref + 16 + 0); \
PREFETCH(pa3 + k_pref + 16 + 8); \
\
CLOAD_Y8() \
CGEMV_N_8x4(); \
CSTORE_Y8(); \
\
k += 2 * 8; \
k_pref += 2 * 8; \
y += inc_y2 * 8; \
} \
\
if (m & 4) \
{ \
CLOAD_Y4(); \
CGEMV_N_4x4(); \
CSTORE_Y4(); \
\
k += 2 * 4; \
y += inc_y2 * 4; \
} \
\
if (m & 3) \
{ \
temp0_r = tp4r[0]; \
temp1_r = tp4r[1]; \
temp2_r = tp4r[2]; \
temp3_r = tp4r[3]; \
\
temp0_i = tp4i[0]; \
temp1_i = tp4i[1]; \
temp2_i = tp4i[2]; \
temp3_i = tp4i[3]; \
\
for (i = (m & 3); i--;) \
{ \
CGEMV_N_1x4(); \
\
k += 2; \
y += inc_y2; \
} \
} \
\
pa0 += 4 * lda2; \
pa1 += 4 * lda2; \
pa2 += 4 * lda2; \
pa3 += 4 * lda2; \
\
x += 4 * inc_x2; \
} \
\
if (n & 2) \
{ \
CLOAD_X2_SCALE(); \
\
k = 0; \
y = y_org; \
\
for (i = (m >> 3); i--;) \
{ \
CLOAD_Y8(); \
CGEMV_N_8x2(); \
CSTORE_Y8(); \
\
k += 2 * 8; \
y += inc_y2 * 8; \
} \
\
if (m & 4) \
{ \
CLOAD_Y4(); \
CGEMV_N_4x2(); \
CSTORE_Y4(); \
\
k += 2 * 4; \
y += inc_y2 * 4; \
} \
\
for (i = (m & 3); i--;) \
{ \
CGEMV_N_1x2(); \
\
k += 2; \
y += inc_y2; \
} \
\
pa0 += 2 * lda2; \
pa1 += 2 * lda2; \
\
x += 2 * inc_x2; \
} \
\
if (n & 1) \
{ \
CLOAD_X1_SCALE(); \
\
k = 0; \
y = y_org; \
\
for (i = m; i--;) \
{ \
CGEMV_N_1x1(); \
\
k += 2; \
y += inc_y2; \
} \
\
pa0 += lda2; \
x += inc_x2; \
} \
int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r, FLOAT alpha_i,
FLOAT *A, BLASLONG lda2, FLOAT *x, BLASLONG inc_x2, FLOAT *y,
BLASLONG inc_y2, FLOAT *buffer)
{
BLASLONG i, j, k;
BLASLONG i, j, k, k_pref, pref_offset;
FLOAT *y_org = y;
FLOAT *pa0, *pa1, *pa2, *pa3;
FLOAT temp_r, temp_i, res0, res1, temp0_r;
@ -513,6 +524,10 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r, FLOAT alpha_i,
inc_x2 = 2 * inc_x2;
inc_y2 = 2 * inc_y2;
pref_offset = (uintptr_t)A & (L1_DATA_LINESIZE - 1);
pref_offset = L1_DATA_LINESIZE - pref_offset;
pref_offset = pref_offset / sizeof(FLOAT);
pa0 = A;
pa1 = A + lda2;
pa2 = A + 2 * lda2;

17
kernel/mips/cgemv_t_msa.c
View File

@ -364,14 +364,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
tp0i = tp1i = tp2i = tp3i = zero; \
\
k = 0; \
k_pref = pref_offset; \
x = srcx_org; \
\
for (i = (m >> 3); i--;) \
{ \
PREFETCH(pa0 + k_pref + 16 + 0); \
PREFETCH(pa0 + k_pref + 16 + 8); \
PREFETCH(pa1 + k_pref + 16 + 0); \
PREFETCH(pa1 + k_pref + 16 + 8); \
PREFETCH(pa2 + k_pref + 16 + 0); \
PREFETCH(pa2 + k_pref + 16 + 8); \
PREFETCH(pa3 + k_pref + 16 + 0); \
PREFETCH(pa3 + k_pref + 16 + 8); \
\
CLOAD_X8() \
CGEMV_T_8x4(); \
\
k += 2 * 8; \
k_pref += 2 * 8; \
x += inc_x2 * 8; \
} \
\
@ -531,7 +542,7 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alphar, FLOAT alphai,
FLOAT *A, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y,
BLASLONG inc_y, FLOAT *buffer)
{
BLASLONG i, j, k;
BLASLONG i, j, k, k_pref, pref_offset;
FLOAT *pa0, *pa1, *pa2, *pa3;
FLOAT *srcx_org = x;
FLOAT temp0r, temp0i, temp2r, temp2i, temp1r, temp1i, temp3r, temp3i;
@ -546,6 +557,10 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alphar, FLOAT alphai,
lda2 = 2 * lda;
pref_offset = (uintptr_t)A & (L1_DATA_LINESIZE - 1);
pref_offset = L1_DATA_LINESIZE - pref_offset;
pref_offset = pref_offset / sizeof(FLOAT);
pa0 = A;
pa1 = A + lda2;
pa2 = A + 2 * lda2;

373
kernel/mips/zgemv_n_msa.c
View File

@ -312,14 +312,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
SPLATI_D2_DP(tp4i, tp0i, tp1i); \
#define ZLOAD_X4_SCALE_GP() \
x0r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *)(x + 0 * inc_x2))); \
x0r = (v2f64) __msa_insert_d((v2i64) x0r, 1, *((long long *)(x + 1 * inc_x2))); \
x1r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *)(x + 2 * inc_x2))); \
x1r = (v2f64) __msa_insert_d((v2i64) x1r, 1, *((long long *)(x + 3 * inc_x2))); \
x0i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *)(x + 0 * inc_x2 + 1))); \
x0i = (v2f64) __msa_insert_d((v2i64) x0i, 1, *((long long *)(x + 1 * inc_x2 + 1))); \
x1i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *)(x + 2 * inc_x2 + 1))); \
x1i = (v2f64) __msa_insert_d((v2i64) x1i, 1, *((long long *)(x + 3 * inc_x2 + 1))); \
x0r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((BLASLONG *)(x + 0 * inc_x2))); \
x0r = (v2f64) __msa_insert_d((v2i64) x0r, 1, *((BLASLONG *)(x + 1 * inc_x2))); \
x1r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((BLASLONG *)(x + 2 * inc_x2))); \
x1r = (v2f64) __msa_insert_d((v2i64) x1r, 1, *((BLASLONG *)(x + 3 * inc_x2))); \
x0i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((BLASLONG *)(x + 0 * inc_x2 + 1))); \
x0i = (v2f64) __msa_insert_d((v2i64) x0i, 1, *((BLASLONG *)(x + 1 * inc_x2 + 1))); \
x1i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((BLASLONG *)(x + 2 * inc_x2 + 1))); \
x1i = (v2f64) __msa_insert_d((v2i64) x1i, 1, *((BLASLONG *)(x + 3 * inc_x2 + 1))); \
\
tp4r = alphar * x0r; \
tp4r OP3 alphai * x0i; \
@ -337,10 +337,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
SPLATI_D2_DP(tp5i, tp2i, tp3i); \
#define ZLOAD_X2_SCALE_GP() \
x0r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *)(x + 0 * inc_x2))); \
x0r = (v2f64) __msa_insert_d((v2i64) x0r, 1, *((long long *)(x + 1 * inc_x2))); \
x0i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *)(x + 0 * inc_x2 + 1))); \
x0i = (v2f64) __msa_insert_d((v2i64) x0i, 1, *((long long *)(x + 1 * inc_x2 + 1))); \
x0r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((BLASLONG *)(x + 0 * inc_x2))); \
x0r = (v2f64) __msa_insert_d((v2i64) x0r, 1, *((BLASLONG *)(x + 1 * inc_x2))); \
x0i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((BLASLONG *)(x + 0 * inc_x2 + 1))); \
x0i = (v2f64) __msa_insert_d((v2i64) x0i, 1, *((BLASLONG *)(x + 1 * inc_x2 + 1))); \
\
tp4r = alphar * x0r; \
tp4r OP3 alphai * x0i; \
@ -377,182 +377,193 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ILVRL_D2_DP(y0i, y0r, y0, y1); \
ST_DP2(y0, y1, y, 2); \
#define ZLOAD_Y4_GP() \
y0r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *)(y + 0 * inc_y2))); \
y0r = (v2f64) __msa_insert_d((v2i64) y0r, 1, *((long long *)(y + 1 * inc_y2))); \
y1r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *)(y + 2 * inc_y2))); \
y1r = (v2f64) __msa_insert_d((v2i64) y1r, 1, *((long long *)(y + 3 * inc_y2))); \
y0i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *)(y + 0 * inc_y2 + 1))); \
y0i = (v2f64) __msa_insert_d((v2i64) y0i, 1, *((long long *)(y + 1 * inc_y2 + 1))); \
y1i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *)(y + 2 * inc_y2 + 1))); \
y1i = (v2f64) __msa_insert_d((v2i64) y1i, 1, *((long long *)(y + 3 * inc_y2 + 1))); \
#define ZLOAD_Y4_GP() \
y0r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((BLASLONG *)(y + 0 * inc_y2))); \
y0r = (v2f64) __msa_insert_d((v2i64) y0r, 1, *((BLASLONG *)(y + 1 * inc_y2))); \
y1r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((BLASLONG *)(y + 2 * inc_y2))); \
y1r = (v2f64) __msa_insert_d((v2i64) y1r, 1, *((BLASLONG *)(y + 3 * inc_y2))); \
y0i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((BLASLONG *)(y + 0 * inc_y2 + 1))); \
y0i = (v2f64) __msa_insert_d((v2i64) y0i, 1, *((BLASLONG *)(y + 1 * inc_y2 + 1))); \
y1i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((BLASLONG *)(y + 2 * inc_y2 + 1))); \
y1i = (v2f64) __msa_insert_d((v2i64) y1i, 1, *((BLASLONG *)(y + 3 * inc_y2 + 1))); \
#define ZLOAD_Y2_GP() \
y0r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *)(y + 0 * inc_y2))); \
y0r = (v2f64) __msa_insert_d((v2i64) y0r, 1, *((long long *)(y + 1 * inc_y2))); \
y0i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *)(y + 0 * inc_y2 + 1))); \
y0i = (v2f64) __msa_insert_d((v2i64) y0i, 1, *((long long *)(y + 1 * inc_y2 + 1))); \
#define ZLOAD_Y2_GP() \
y0r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((BLASLONG *)(y + 0 * inc_y2))); \
y0r = (v2f64) __msa_insert_d((v2i64) y0r, 1, *((BLASLONG *)(y + 1 * inc_y2))); \
y0i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((BLASLONG *)(y + 0 * inc_y2 + 1))); \
y0i = (v2f64) __msa_insert_d((v2i64) y0i, 1, *((BLASLONG *)(y + 1 * inc_y2 + 1))); \
#define ZSTORE_Y4_GP() \
*((long long *)(y + 0 * inc_y2)) = __msa_copy_s_d((v2i64) y0r, 0); \
*((long long *)(y + 1 * inc_y2)) = __msa_copy_s_d((v2i64) y0r, 1); \
*((long long *)(y + 2 * inc_y2)) = __msa_copy_s_d((v2i64) y1r, 0); \
*((long long *)(y + 3 * inc_y2)) = __msa_copy_s_d((v2i64) y1r, 1); \
*((long long *)(y + 0 * inc_y2 + 1)) = __msa_copy_s_d((v2i64) y0i, 0); \
*((long long *)(y + 1 * inc_y2 + 1)) = __msa_copy_s_d((v2i64) y0i, 1); \
*((long long *)(y + 2 * inc_y2 + 1)) = __msa_copy_s_d((v2i64) y1i, 0); \
*((long long *)(y + 3 * inc_y2 + 1)) = __msa_copy_s_d((v2i64) y1i, 1); \
#define ZSTORE_Y4_GP() \
*((BLASLONG *)(y + 0 * inc_y2)) = __msa_copy_s_d((v2i64) y0r, 0); \
*((BLASLONG *)(y + 1 * inc_y2)) = __msa_copy_s_d((v2i64) y0r, 1); \
*((BLASLONG *)(y + 2 * inc_y2)) = __msa_copy_s_d((v2i64) y1r, 0); \
*((BLASLONG *)(y + 3 * inc_y2)) = __msa_copy_s_d((v2i64) y1r, 1); \
*((BLASLONG *)(y + 0 * inc_y2 + 1)) = __msa_copy_s_d((v2i64) y0i, 0); \
*((BLASLONG *)(y + 1 * inc_y2 + 1)) = __msa_copy_s_d((v2i64) y0i, 1); \
*((BLASLONG *)(y + 2 * inc_y2 + 1)) = __msa_copy_s_d((v2i64) y1i, 0); \
*((BLASLONG *)(y + 3 * inc_y2 + 1)) = __msa_copy_s_d((v2i64) y1i, 1); \
#define ZSTORE_Y2_GP() \
*((long long *)(y + 0 * inc_y2)) = __msa_copy_s_d((v2i64) y0r, 0); \
*((long long *)(y + 1 * inc_y2)) = __msa_copy_s_d((v2i64) y0r, 1); \
*((long long *)(y + 0 * inc_y2 + 1)) = __msa_copy_s_d((v2i64) y0i, 0); \
*((long long *)(y + 1 * inc_y2 + 1)) = __msa_copy_s_d((v2i64) y0i, 1); \
#define ZSTORE_Y2_GP() \
*((BLASLONG *)(y + 0 * inc_y2)) = __msa_copy_s_d((v2i64) y0r, 0); \
*((BLASLONG *)(y + 1 * inc_y2)) = __msa_copy_s_d((v2i64) y0r, 1); \
*((BLASLONG *)(y + 0 * inc_y2 + 1)) = __msa_copy_s_d((v2i64) y0i, 0); \
*((BLASLONG *)(y + 1 * inc_y2 + 1)) = __msa_copy_s_d((v2i64) y0i, 1); \
#define ZGEMV_N_MSA() \
for (j = (n >> 2); j--;) \
{ \
ZLOAD_X4_SCALE() \
\
k = 0; \
y = y_org; \
\
for (i = (m >> 2); i--;) \
{ \
ZLOAD_Y4() \
ZGEMV_N_4x4() \
ZSTORE_Y4() \
\
k += 2 * 4; \
y += inc_y2 * 4; \
} \
\
if (m & 2) \
{ \
ZLOAD_Y2() \
ZGEMV_N_2x4() \
ZSTORE_Y2() \
\
k += 2 * 2; \
y += inc_y2 * 2; \
} \
\
if (m & 1) \
{ \
temp0_r = tp4r[0]; \
temp1_r = tp4r[1]; \
temp2_r = tp5r[0]; \
temp3_r = tp5r[1]; \
\
temp0_i = tp4i[0]; \
temp1_i = tp4i[1]; \
temp2_i = tp5i[0]; \
temp3_i = tp5i[1]; \
\
ZGEMV_N_1x4() \
k += 2; \
y += inc_y2; \
} \
\
pa0 += 4 * lda2; \
pa1 += 4 * lda2; \
pa2 += 4 * lda2; \
pa3 += 4 * lda2; \
\
x += 4 * inc_x2; \
} \
\
if (n & 2) \
{ \
ZLOAD_X2_SCALE() \
\
k = 0; \
y = y_org; \
\
for (i = (m >> 2); i--;) \
{ \
ZLOAD_Y4() \
ZGEMV_N_4x2() \
ZSTORE_Y4() \
\
k += 2 * 4; \
y += inc_y2 * 4; \
} \
\
if (m & 2) \
{ \
ZLOAD_Y2() \
ZGEMV_N_2x2() \
ZSTORE_Y2() \
\
k += 2 * 2; \
y += inc_y2 * 2; \
} \
\
if (m & 1) \
{ \
temp0_r = tp4r[0]; \
temp1_r = tp4r[1]; \
\
temp0_i = tp4i[0]; \
temp1_i = tp4i[1]; \
\
ZGEMV_N_1x2() \
\
k += 2; \
y += inc_y2; \
} \
\
pa0 += 2 * lda2; \
pa1 += 2 * lda2; \
\
x += 2 * inc_x2; \
} \
\
if (n & 1) \
{ \
ZLOAD_X1_SCALE() \
\
k = 0; \
y = y_org; \
\
for (i = (m >> 2); i--;) \
{ \
ZLOAD_Y4() \
ZGEMV_N_4x1() \
ZSTORE_Y4() \
\
k += 2 * 4; \
y += inc_y2 * 4; \
} \
\
if (m & 2) \
{ \
ZLOAD_Y2() \
ZGEMV_N_2x1() \
ZSTORE_Y2() \
\
k += 2 * 2; \
y += inc_y2 * 2; \
} \
\
if (m & 1) \
{ \
ZGEMV_N_1x1() \
\
k += 2; \
y += inc_y2; \
} \
\
pa0 += lda2; \
x += inc_x2; \
} \
#define ZGEMV_N_MSA() \
for (j = (n >> 2); j--;) \
{ \
ZLOAD_X4_SCALE() \
\
k = 0; \
k_pref = pref_offset; \
y = y_org; \
\
for (i = (m >> 2); i--;) \
{ \
PREFETCH(pa0 + k_pref + 8 + 0); \
PREFETCH(pa0 + k_pref + 8 + 4); \
PREFETCH(pa1 + k_pref + 8 + 0); \
PREFETCH(pa1 + k_pref + 8 + 4); \
PREFETCH(pa2 + k_pref + 8 + 0); \
PREFETCH(pa2 + k_pref + 8 + 4); \
PREFETCH(pa3 + k_pref + 8 + 0); \
PREFETCH(pa3 + k_pref + 8 + 4); \
\
ZLOAD_Y4() \
ZGEMV_N_4x4() \
ZSTORE_Y4() \
\
k += 2 * 4; \
k_pref += 2 * 4; \
y += inc_y2 * 4; \
} \
\
if (m & 2) \
{ \
ZLOAD_Y2() \
ZGEMV_N_2x4() \
ZSTORE_Y2() \
\
k += 2 * 2; \
y += inc_y2 * 2; \
} \
\
if (m & 1) \
{ \
temp0_r = tp4r[0]; \
temp1_r = tp4r[1]; \
temp2_r = tp5r[0]; \
temp3_r = tp5r[1]; \
\
temp0_i = tp4i[0]; \
temp1_i = tp4i[1]; \
temp2_i = tp5i[0]; \
temp3_i = tp5i[1]; \
\
ZGEMV_N_1x4() \
k += 2; \
y += inc_y2; \
} \
\
pa0 += 4 * lda2; \
pa1 += 4 * lda2; \
pa2 += 4 * lda2; \
pa3 += 4 * lda2; \
\
x += 4 * inc_x2; \
} \
\
if (n & 2) \
{ \
ZLOAD_X2_SCALE() \
\
k = 0; \
y = y_org; \
\
for (i = (m >> 2); i--;) \
{ \
ZLOAD_Y4() \
ZGEMV_N_4x2() \
ZSTORE_Y4() \
\
k += 2 * 4; \
y += inc_y2 * 4; \
} \
\
if (m & 2) \
{ \
ZLOAD_Y2() \
ZGEMV_N_2x2() \
ZSTORE_Y2() \
\
k += 2 * 2; \
y += inc_y2 * 2; \
} \
\
if (m & 1) \
{ \
temp0_r = tp4r[0]; \
temp1_r = tp4r[1]; \
\
temp0_i = tp4i[0]; \
temp1_i = tp4i[1]; \
\
ZGEMV_N_1x2() \
\
k += 2; \
y += inc_y2; \
} \
\
pa0 += 2 * lda2; \
pa1 += 2 * lda2; \
\
x += 2 * inc_x2; \
} \
\
if (n & 1) \
{ \
ZLOAD_X1_SCALE() \
\
k = 0; \
y = y_org; \
\
for (i = (m >> 2); i--;) \
{ \
ZLOAD_Y4() \
ZGEMV_N_4x1() \
ZSTORE_Y4() \
\
k += 2 * 4; \
y += inc_y2 * 4; \
} \
\
if (m & 2) \
{ \
ZLOAD_Y2() \
ZGEMV_N_2x1() \
ZSTORE_Y2() \
\
k += 2 * 2; \
y += inc_y2 * 2; \
} \
\
if (m & 1) \
{ \
ZGEMV_N_1x1() \
\
k += 2; \
y += inc_y2; \
} \
\
pa0 += lda2; \
x += inc_x2; \
} \
int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r, FLOAT alpha_i,
FLOAT *A, BLASLONG lda2, FLOAT *x, BLASLONG inc_x2, FLOAT *y,
BLASLONG inc_y2, FLOAT *buffer)
{
BLASLONG i, j, k;
BLASLONG i, j, k, k_pref, pref_offset;
FLOAT *y_org = y;
FLOAT *pa0, *pa1, *pa2, *pa3;
FLOAT temp0_r, temp1_r, temp2_r, temp3_r, temp0_i, temp1_i, temp2_i;
@ -569,6 +580,10 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r, FLOAT alpha_i,
inc_x2 = 2 * inc_x2;
inc_y2 = 2 * inc_y2;
pref_offset = (uintptr_t)A & (L1_DATA_LINESIZE - 1);
pref_offset = L1_DATA_LINESIZE - pref_offset;
pref_offset = pref_offset / sizeof(FLOAT);
pa0 = A;
pa1 = A + lda2;
pa2 = A + 2 * lda2;

795
kernel/mips/zgemv_t_msa.c
View File

@ -44,64 +44,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define OP2 -=
#endif
#define ZGEMV_T_4x4() \
LD_DP4(pa0 + k, 2, t0, t1, t2, t3); \
LD_DP4(pa1 + k, 2, t4, t5, t6, t7); \
LD_DP4(pa2 + k, 2, t8, t9, t10, t11); \
LD_DP4(pa3 + k, 2, t12, t13, t14, t15); \
\
PCKEVOD_D2_DP(t1, t0, src0r, src0i); \
PCKEVOD_D2_DP(t3, t2, src1r, src1i); \
PCKEVOD_D2_DP(t5, t4, src2r, src2i); \
PCKEVOD_D2_DP(t7, t6, src3r, src3i); \
PCKEVOD_D2_DP(t9, t8, src4r, src4i); \
PCKEVOD_D2_DP(t11, t10, src5r, src5i); \
PCKEVOD_D2_DP(t13, t12, src6r, src6i); \
PCKEVOD_D2_DP(t15, t14, src7r, src7i); \
\
tp0r += src0r * x0r; \
tp0r += src1r * x1r; \
tp0r OP0 src0i * x0i; \
tp0r OP0 src1i * x1i; \
\
tp1r += src2r * x0r; \
tp1r += src3r * x1r; \
tp1r OP0 src2i * x0i; \
tp1r OP0 src3i * x1i; \
\
tp2r += src4r * x0r; \
tp2r += src5r * x1r; \
tp2r OP0 src4i * x0i; \
tp2r OP0 src5i * x1i; \
\
tp3r += src6r * x0r; \
tp3r += src7r * x1r; \
tp3r OP0 src6i * x0i; \
tp3r OP0 src7i * x1i; \
\
tp0i OP1 src0r * x0i; \
tp0i OP1 src1r * x1i; \
tp0i OP2 src0i * x0r; \
tp0i OP2 src1i * x1r; \
\
tp1i OP1 src2r * x0i; \
tp1i OP1 src3r * x1i; \
tp1i OP2 src2i * x0r; \
tp1i OP2 src3i * x1r; \
\
tp2i OP1 src4r * x0i; \
tp2i OP1 src5r * x1i; \
tp2i OP2 src4i * x0r; \
tp2i OP2 src5i * x1r; \
\
tp3i OP1 src6r * x0i; \
tp3i OP1 src7r * x1i; \
tp3i OP2 src6i * x0r; \
tp3i OP2 src7i * x1r; \
#define ZGEMV_T_4x2() \
LD_DP4(pa0 + k, 2, t0, t1, t2, t3); \
LD_DP4(pa1 + k, 2, t4, t5, t6, t7); \
#define ZGEMV_T_8x1() \
LD_DP4(pa0, 2, t0, t1, t2, t3); \
LD_DP4(pa0 + 8, 2, t4, t5, t6, t7); \
\
PCKEVOD_D2_DP(t1, t0, src0r, src0i); \
PCKEVOD_D2_DP(t3, t2, src1r, src1i); \
@ -109,27 +54,27 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
PCKEVOD_D2_DP(t7, t6, src3r, src3i); \
\
tp0r += src0r * x0r; \
tp0r += src1r * x1r; \
tp0r OP0 src0i * x0i; \
tp0r OP0 src1i * x1i; \
\
tp1r += src2r * x0r; \
tp1r += src3r * x1r; \
tp1r OP0 src2i * x0i; \
tp1r OP0 src3i * x1i; \
\
tp0i OP1 src0r * x0i; \
tp0i OP1 src1r * x1i; \
tp0r OP0 src0i * x0i; \
tp0i OP2 src0i * x0r; \
\
tp0r += src2r * x2r; \
tp0i OP1 src2r * x2i; \
tp0r OP0 src2i * x2i; \
tp0i OP2 src2i * x2r; \
\
tp0r += src1r * x1r; \
tp0i OP1 src1r * x1i; \
tp0r OP0 src1i * x1i; \
tp0i OP2 src1i * x1r; \
\
tp1i OP1 src2r * x0i; \
tp1i OP1 src3r * x1i; \
tp1i OP2 src2i * x0r; \
tp1i OP2 src3i * x1r; \
tp0r += src3r * x3r; \
tp0i OP1 src3r * x3i; \
tp0r OP0 src3i * x3i; \
tp0i OP2 src3i * x3r; \
#define ZGEMV_T_4x1() \
LD_DP4(pa0 + k, 2, t0, t1, t2, t3); \
LD_DP4(pa0, 2, t0, t1, t2, t3); \
\
PCKEVOD_D2_DP(t1, t0, src0r, src0i); \
PCKEVOD_D2_DP(t3, t2, src1r, src1i); \
@ -144,62 +89,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
tp0i OP2 src0i * x0r; \
tp0i OP2 src1i * x1r; \
#define ZGEMV_T_2x4() \
LD_DP2(pa0 + k, 2, t0, t1); \
LD_DP2(pa1 + k, 2, t4, t5); \
LD_DP2(pa2 + k, 2, t8, t9); \
LD_DP2(pa3 + k, 2, t12, t13); \
\
PCKEVOD_D2_DP(t1, t0, src0r, src0i); \
PCKEVOD_D2_DP(t5, t4, src2r, src2i); \
PCKEVOD_D2_DP(t9, t8, src4r, src4i); \
PCKEVOD_D2_DP(t13, t12, src6r, src6i); \
\
tp0r += src0r * x0r; \
tp0r OP0 src0i * x0i; \
\
tp1r += src2r * x0r; \
tp1r OP0 src2i * x0i; \
\
tp2r += src4r * x0r; \
tp2r OP0 src4i * x0i; \
\
tp3r += src6r * x0r; \
tp3r OP0 src6i * x0i; \
\
tp0i OP1 src0r * x0i; \
tp0i OP2 src0i * x0r; \
\
tp1i OP1 src2r * x0i; \
tp1i OP2 src2i * x0r; \
\
tp2i OP1 src4r * x0i; \
tp2i OP2 src4i * x0r; \
\
tp3i OP1 src6r * x0i; \
tp3i OP2 src6i * x0r; \
#define ZGEMV_T_2x2() \
LD_DP2(pa0 + k, 2, t0, t1); \
LD_DP2(pa1 + k, 2, t4, t5); \
\
PCKEVOD_D2_DP(t1, t0, src0r, src0i); \
PCKEVOD_D2_DP(t5, t4, src2r, src2i); \
\
tp0r += src0r * x0r; \
tp0r OP0 src0i * x0i; \
\
tp1r += src2r * x0r; \
tp1r OP0 src2i * x0i; \
\
tp0i OP1 src0r * x0i; \
tp0i OP2 src0i * x0r; \
\
tp1i OP1 src2r * x0i; \
tp1i OP2 src2i * x0r; \
#define ZGEMV_T_2x1() \
LD_DP2(pa0 + k, 2, t0, t1); \
LD_DP2(pa0, 2, t0, t1); \
\
PCKEVOD_D2_DP(t1, t0, src0r, src0i); \
\
@ -209,104 +100,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
tp0i OP1 src0r * x0i; \
tp0i OP2 src0i * x0r; \
#define ZGEMV_T_1x4() \
temp0r += pa0[k + 0] * x[0 * inc_x2]; \
temp0r OP0 pa0[k + 1] * x[0 * inc_x2 + 1]; \
temp1r += pa1[k + 0] * x[0 * inc_x2]; \
temp1r OP0 pa1[k + 1] * x[0 * inc_x2 + 1]; \
temp2r += pa2[k + 0] * x[0 * inc_x2]; \
temp2r OP0 pa2[k + 1] * x[0 * inc_x2 + 1]; \
temp3r += pa3[k + 0] * x[0 * inc_x2]; \
temp3r OP0 pa3[k + 1] * x[0 * inc_x2 + 1]; \
\
temp0i OP1 pa0[k + 0] * x[0 * inc_x2 + 1]; \
temp0i OP2 pa0[k + 1] * x[0 * inc_x2]; \
temp1i OP1 pa1[k + 0] * x[0 * inc_x2 + 1]; \
temp1i OP2 pa1[k + 1] * x[0 * inc_x2]; \
temp2i OP1 pa2[k + 0] * x[0 * inc_x2 + 1]; \
temp2i OP2 pa2[k + 1] * x[0 * inc_x2]; \
temp3i OP1 pa3[k + 0] * x[0 * inc_x2 + 1]; \
temp3i OP2 pa3[k + 1] * x[0 * inc_x2]; \
#define ZGEMV_T_1x2() \
temp0r += pa0[k + 0] * x[0 * inc_x2]; \
temp0r OP0 pa0[k + 1] * x[0 * inc_x2 + 1]; \
temp1r += pa1[k + 0] * x[0 * inc_x2]; \
temp1r OP0 pa1[k + 1] * x[0 * inc_x2 + 1]; \
\
temp0i OP1 pa0[k + 0] * x[0 * inc_x2 + 1]; \
temp0i OP2 pa0[k + 1] * x[0 * inc_x2]; \
temp1i OP1 pa1[k + 0] * x[0 * inc_x2 + 1]; \
temp1i OP2 pa1[k + 1] * x[0 * inc_x2]; \
#define ZGEMV_T_1x1() \
temp0r += pa0[k + 0] * x[0 * inc_x2]; \
temp0r OP0 pa0[k + 1] * x[0 * inc_x2 + 1]; \
\
temp0i OP1 pa0[k + 0] * x[0 * inc_x2 + 1]; \
temp0i OP2 pa0[k + 1] * x[0 * inc_x2]; \
#define ZSCALE_STORE_Y4_GP() \
res0r = y[0 * inc_y2]; \
res1r = y[1 * inc_y2]; \
res2r = y[2 * inc_y2]; \
res3r = y[3 * inc_y2]; \
\
res0i = y[0 * inc_y2 + 1]; \
res1i = y[1 * inc_y2 + 1]; \
res2i = y[2 * inc_y2 + 1]; \
res3i = y[3 * inc_y2 + 1]; \
\
res0r += alphar * temp0r; \
res0r OP0 alphai * temp0i; \
res1r += alphar * temp1r; \
res1r OP0 alphai * temp1i; \
res2r += alphar * temp2r; \
res2r OP0 alphai * temp2i; \
res3r += alphar * temp3r; \
res3r OP0 alphai * temp3i; \
\
res0i OP1 alphar * temp0i; \
res0i OP2 alphai * temp0r; \
res1i OP1 alphar * temp1i; \
res1i OP2 alphai * temp1r; \
res2i OP1 alphar * temp2i; \
res2i OP2 alphai * temp2r; \
res3i OP1 alphar * temp3i; \
res3i OP2 alphai * temp3r; \
\
y[0 * inc_y2] = res0r; \
y[1 * inc_y2] = res1r; \
y[2 * inc_y2] = res2r; \
y[3 * inc_y2] = res3r; \
\
y[0 * inc_y2 + 1] = res0i; \
y[1 * inc_y2 + 1] = res1i; \
y[2 * inc_y2 + 1] = res2i; \
y[3 * inc_y2 + 1] = res3i; \
#define ZSCALE_STORE_Y2_GP() \
res0r = y[0 * inc_y2]; \
res1r = y[1 * inc_y2]; \
\
res0i = y[0 * inc_y2 + 1]; \
res1i = y[1 * inc_y2 + 1]; \
\
res0r += alphar * temp0r; \
res0r OP0 alphai * temp0i; \
res1r += alphar * temp1r; \
res1r OP0 alphai * temp1i; \
\
res0i OP1 alphar * temp0i; \
res0i OP2 alphai * temp0r; \
res1i OP1 alphar * temp1i; \
res1i OP2 alphai * temp1r; \
\
y[0 * inc_y2] = res0r; \
y[1 * inc_y2] = res1r; \
\
y[0 * inc_y2 + 1] = res0i; \
y[1 * inc_y2 + 1] = res1i; \
#define ZGEMV_T_1x1() \
temp0r += pa0[0] * x[0 * inc_x2]; \
temp0r OP0 pa0[1] * x[0 * inc_x2 + 1]; \
\
temp0i OP1 pa0[0] * x[0 * inc_x2 + 1]; \
temp0i OP2 pa0[1] * x[0 * inc_x2]; \
#define ZSCALE_STORE_Y1_GP() \
res0r = y[0 * inc_y2]; \
@ -321,6 +120,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
y[0 * inc_y2] = res0r; \
y[0 * inc_y2 + 1] = res0i; \
#define ZLOAD_X8_VECTOR() \
LD_DP4(x, 2, x0, x1, x2, x3); \
LD_DP4(x + 8, 2, x4, x5, x6, x7); \
\
PCKEVOD_D2_DP(x1, x0, x0r, x0i); \
PCKEVOD_D2_DP(x3, x2, x1r, x1i); \
PCKEVOD_D2_DP(x5, x4, x2r, x2i); \
PCKEVOD_D2_DP(x7, x6, x3r, x3i); \
#define ZLOAD_X4_VECTOR() \
LD_DP4(x, 2, x0, x1, x2, x3); \
PCKEVOD_D2_DP(x1, x0, x0r, x0i); \
@ -330,6 +138,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
LD_DP2(x, 2, x0, x1); \
PCKEVOD_D2_DP(x1, x0, x0r, x0i); \
#define ZLOAD_X8_GP() \
x0r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *) (x + 0 * inc_x2))); \
x0r = (v2f64) __msa_insert_d((v2i64) x0r, 1, *((long long *) (x + 1 * inc_x2))); \
x1r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *) (x + 2 * inc_x2))); \
x1r = (v2f64) __msa_insert_d((v2i64) x1r, 1, *((long long *) (x + 3 * inc_x2))); \
x2r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *) (x + 4 * inc_x2))); \
x2r = (v2f64) __msa_insert_d((v2i64) x2r, 1, *((long long *) (x + 5 * inc_x2))); \
x3r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *) (x + 6 * inc_x2))); \
x3r = (v2f64) __msa_insert_d((v2i64) x3r, 1, *((long long *) (x + 7 * inc_x2))); \
x0i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *) (x + 0 * inc_x2 + 1))); \
x0i = (v2f64) __msa_insert_d((v2i64) x0i, 1, *((long long *) (x + 1 * inc_x2 + 1))); \
x1i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *) (x + 2 * inc_x2 + 1))); \
x1i = (v2f64) __msa_insert_d((v2i64) x1i, 1, *((long long *) (x + 3 * inc_x2 + 1))); \
x2i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *) (x + 4 * inc_x2 + 1))); \
x2i = (v2f64) __msa_insert_d((v2i64) x2i, 1, *((long long *) (x + 5 * inc_x2 + 1))); \
x3i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *) (x + 6 * inc_x2 + 1))); \
x3i = (v2f64) __msa_insert_d((v2i64) x3i, 1, *((long long *) (x + 7 * inc_x2 + 1))); \
#define ZLOAD_X4_GP() \
x0r = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *) (x + 0 * inc_x2))); \
x0r = (v2f64) __msa_insert_d((v2i64) x0r, 1, *((long long *) (x + 1 * inc_x2))); \
@ -346,196 +172,405 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
x0i = (v2f64) __msa_insert_d((v2i64) tp0r, 0, *((long long *) (x + 0 * inc_x2 + 1))); \
x0i = (v2f64) __msa_insert_d((v2i64) x0i, 1, *((long long *) (x + 1 * inc_x2 + 1))); \
#define ZGEMV_T_MSA() \
for (j = (n >> 2); j--;) \
{ \
tp0r = tp1r = tp2r = tp3r = zero; \
tp0i = tp1i = tp2i = tp3i = zero; \
\
k = 0; \
x = srcx_org; \
\
for (i = (m >> 2); i--;) \
{ \
ZLOAD_X4(); \
ZGEMV_T_4x4(); \
\
k += 2 * 4; \
x += inc_x2 * 4; \
} \
\
if (m & 2) \
{ \
ZLOAD_X2(); \
ZGEMV_T_2x4(); \
\
k += 2 * 2; \
x += inc_x2 * 2; \
} \
\
temp0r = tp0r[0] + tp0r[1]; \
temp1r = tp1r[0] + tp1r[1]; \
temp2r = tp2r[0] + tp2r[1]; \
temp3r = tp3r[0] + tp3r[1]; \
temp0i = tp0i[0] + tp0i[1]; \
temp1i = tp1i[0] + tp1i[1]; \
temp2i = tp2i[0] + tp2i[1]; \
temp3i = tp3i[0] + tp3i[1]; \
\
if (m & 1) \
{ \
ZGEMV_T_1x4(); \
\
k += 2; \
x += inc_x2; \
} \
\
ZSCALE_STORE_Y4_GP(); \
\
pa0 += 4 * lda2; \
pa1 += 4 * lda2; \
pa2 += 4 * lda2; \
pa3 += 4 * lda2; \
y += 4 * inc_y2; \
} \
\
if (n & 2) \
{ \
tp0r = tp1r = zero; \
tp0i = tp1i = zero; \
\
k = 0; \
x = srcx_org; \
\
for (i = (m >> 2); i--;) \
{ \
ZLOAD_X4(); \
ZGEMV_T_4x2(); \
\
k += 2 * 4; \
x += inc_x2 * 4; \
} \
\
if (m & 2) \
{ \
ZLOAD_X2(); \
ZGEMV_T_2x2(); \
\
k += 2 * 2; \
x += inc_x2 * 2; \
} \
\
temp0r = tp0r[0] + tp0r[1]; \
temp1r = tp1r[0] + tp1r[1]; \
temp0i = tp0i[0] + tp0i[1]; \
temp1i = tp1i[0] + tp1i[1]; \
\
if (m & 1) \
{ \
ZGEMV_T_1x2(); \
\
k += 2; \
x += inc_x2; \
} \
\
ZSCALE_STORE_Y2_GP(); \
\
pa0 += 2 * lda2; \
pa1 += 2 * lda2; \
y += 2 * inc_y2; \
} \
\
if (n & 1) \
{ \
tp0r = zero; \
tp0i = zero; \
\
k = 0; \
x = srcx_org; \
\
for (i = (m >> 2); i--;) \
{ \
ZLOAD_X4(); \
ZGEMV_T_4x1(); \
\
k += 2 * 4; \
x += inc_x2 * 4; \
} \
\
if (m & 2) \
{ \
ZLOAD_X2(); \
ZGEMV_T_2x1(); \
\
k += 2 * 2; \
x += inc_x2 * 2; \
} \
\
temp0r = tp0r[0] + tp0r[1]; \
temp0i = tp0i[0] + tp0i[1]; \
\
if (m & 1) \
{ \
ZGEMV_T_1x1(); \
\
k += 2; \
x += inc_x2; \
} \
\
ZSCALE_STORE_Y1_GP(); \
\
pa0 += lda2; \
y += inc_y2; \
} \
#define ZGEMV_T_MSA() \
for (j = n; j--;) \
{ \
tp0r = zero; \
tp0i = zero; \
tp1r = zero; \
tp1i = zero; \
tp2r = zero; \
tp2i = zero; \
tp3r = zero; \
tp3i = zero; \
\
pa0 = A; \
x = srcx_org; \
\
if (m >> 4) \
{ \
x0 = LD_DP(x); \
x1 = LD_DP(x + 1 * inc_x2); \
t0 = LD_DP(pa0); \
t1 = LD_DP(pa0 + 2); \
\
x4 = LD_DP(x + 4 * inc_x2); \
x5 = LD_DP(x + 5 * inc_x2); \
t4 = LD_DP(pa0 + 8); \
t5 = LD_DP(pa0 + 10); \
\
for (i = (m >> 4) - 1; i--;) \
{ \
pa0_pref = pa0 + pref_offset; \
\
PREFETCH(pa0_pref + 36); \
PREFETCH(pa0_pref + 44); \
PREFETCH(pa0_pref + 48); \
PREFETCH(pa0_pref + 52); \
PREFETCH(pa0_pref + 56); \
PREFETCH(pa0_pref + 60); \
PREFETCH(pa0_pref + 64); \
PREFETCH(pa0_pref + 72); \
\
x0r = (v2f64) __msa_pckev_d((v2i64) x1, (v2i64) x0); \
x0i = (v2f64) __msa_pckod_d((v2i64) x1, (v2i64) x0); \
src0r = (v2f64) __msa_pckev_d((v2i64) t1, (v2i64) t0); \
src0i = (v2f64) __msa_pckod_d((v2i64) t1, (v2i64) t0); \
\
tp0r += src0r * x0r; \
x2 = LD_DP(x + 2 * inc_x2); \
x2r = (v2f64) __msa_pckev_d((v2i64) x5, (v2i64) x4); \
\
tp0i OP1 src0r * x0i; \
x3 = LD_DP(x + 3 * inc_x2); \
x2i = (v2f64) __msa_pckod_d((v2i64) x5, (v2i64) x4); \
\
tp1r OP0 src0i * x0i; \
t2 = LD_DP(pa0 + 4); \
src2r = (v2f64) __msa_pckev_d((v2i64) t5, (v2i64) t4); \
\
tp1i OP2 src0i * x0r; \
t3 = LD_DP(pa0 + 6); \
src2i = (v2f64) __msa_pckod_d((v2i64) t5, (v2i64) t4); \
\
tp2r += src2r * x2r; \
x6 = LD_DP(x + 6 * inc_x2); \
\
tp2i OP1 src2r * x2i; \
x7 = LD_DP(x + 7 * inc_x2); \
\
tp3r OP0 src2i * x2i; \
t6 = LD_DP(pa0 + 12); \
\
tp3i OP2 src2i * x2r; \
t7 = LD_DP(pa0 + 14); \
\
x1r = (v2f64) __msa_pckev_d((v2i64) x3, (v2i64) x2); \
x1i = (v2f64) __msa_pckod_d((v2i64) x3, (v2i64) x2); \
src1r = (v2f64) __msa_pckev_d((v2i64) t3, (v2i64) t2); \
src1i = (v2f64) __msa_pckod_d((v2i64) t3, (v2i64) t2); \
\
tp0r += src1r * x1r; \
x0 = LD_DP(x + 8 * inc_x2); \
x3r = (v2f64) __msa_pckev_d((v2i64) x7, (v2i64) x6); \
\
tp0i OP1 src1r * x1i; \
x1 = LD_DP(x + 9 * inc_x2); \
x3i = (v2f64) __msa_pckod_d((v2i64) x7, (v2i64) x6); \
\
tp1r OP0 src1i * x1i; \
t0 = LD_DP(pa0 + 16); \
src3r = (v2f64) __msa_pckev_d((v2i64) t7, (v2i64) t6); \
\
tp1i OP2 src1i * x1r; \
t1 = LD_DP(pa0 + 18); \
src3i = (v2f64) __msa_pckod_d((v2i64) t7, (v2i64) t6); \
\
tp2r += src3r * x3r; \
x4 = LD_DP(x + 12 * inc_x2); \
\
tp2i OP1 src3r * x3i; \
x5 = LD_DP(x + 13 * inc_x2); \
\
tp3r OP0 src3i * x3i; \
t4 = LD_DP(pa0 + 24); \
\
tp3i OP2 src3i * x3r; \
t5 = LD_DP(pa0 + 26); \
\
x0r = (v2f64) __msa_pckev_d((v2i64) x1, (v2i64) x0); \
x0i = (v2f64) __msa_pckod_d((v2i64) x1, (v2i64) x0); \
src0r = (v2f64) __msa_pckev_d((v2i64) t1, (v2i64) t0); \
src0i = (v2f64) __msa_pckod_d((v2i64) t1, (v2i64) t0); \
\
tp0r += src0r * x0r; \
x2 = LD_DP(x + 10 * inc_x2); \
x2r = (v2f64) __msa_pckev_d((v2i64) x5, (v2i64) x4); \
\
tp0i OP1 src0r * x0i; \
x3 = LD_DP(x + 11 * inc_x2); \
x2i = (v2f64) __msa_pckod_d((v2i64) x5, (v2i64) x4); \
\
tp1r OP0 src0i * x0i; \
t2 = LD_DP(pa0 + 20); \
src2r = (v2f64) __msa_pckev_d((v2i64) t5, (v2i64) t4); \
\
tp1i OP2 src0i * x0r; \
t3 = LD_DP(pa0 + 22); \
src2i = (v2f64) __msa_pckod_d((v2i64) t5, (v2i64) t4); \
\
tp2r += src2r * x2r; \
x6 = LD_DP(x + 14 * inc_x2); \
\
tp2i OP1 src2r * x2i; \
x7 = LD_DP(x + 15 * inc_x2); \
\
tp3r OP0 src2i * x2i; \
t6 = LD_DP(pa0 + 28); \
\
tp3i OP2 src2i * x2r; \
t7 = LD_DP(pa0 + 30); \
\
x1r = (v2f64) __msa_pckev_d((v2i64) x3, (v2i64) x2); \
x1i = (v2f64) __msa_pckod_d((v2i64) x3, (v2i64) x2); \
src1r = (v2f64) __msa_pckev_d((v2i64) t3, (v2i64) t2); \
src1i = (v2f64) __msa_pckod_d((v2i64) t3, (v2i64) t2); \
\
tp0r += src1r * x1r; \
x0 = LD_DP(x + inc_x2 * 16); \
x3r = (v2f64) __msa_pckev_d((v2i64) x7, (v2i64) x6); \
\
tp0i OP1 src1r * x1i; \
x1 = LD_DP(x + inc_x2 * 16 + 1 * inc_x2); \
x3i = (v2f64) __msa_pckod_d((v2i64) x7, (v2i64) x6); \
\
tp1r OP0 src1i * x1i; \
t0 = LD_DP(pa0 + 2 * 16); \
src3r = (v2f64) __msa_pckev_d((v2i64) t7, (v2i64) t6); \
\
tp1i OP2 src1i * x1r; \
t1 = LD_DP(pa0 + 2 * 16 + 2); \
src3i = (v2f64) __msa_pckod_d((v2i64) t7, (v2i64) t6); \
\
tp2r += src3r * x3r; \
x4 = LD_DP(x + inc_x2 * 16 + 4 * inc_x2); \
\
tp2i OP1 src3r * x3i; \
x5 = LD_DP(x + inc_x2 * 16 + 5 * inc_x2); \
\
tp3r OP0 src3i * x3i; \
t4 = LD_DP(pa0 + 2 * 16 + 8); \
\
tp3i OP2 src3i * x3r; \
t5 = LD_DP(pa0 + 2 * 16 + 10); \
\
pa0 += 2 * 16; \
x += inc_x2 * 16; \
} \
\
x0r = (v2f64) __msa_pckev_d((v2i64) x1, (v2i64) x0); \
x0i = (v2f64) __msa_pckod_d((v2i64) x1, (v2i64) x0); \
src0r = (v2f64) __msa_pckev_d((v2i64) t1, (v2i64) t0); \
src0i = (v2f64) __msa_pckod_d((v2i64) t1, (v2i64) t0); \
\
tp0r += src0r * x0r; \
x2 = LD_DP(x + 2 * inc_x2); \
x2r = (v2f64) __msa_pckev_d((v2i64) x5, (v2i64) x4); \
\
tp0i OP1 src0r * x0i; \
x3 = LD_DP(x + 3 * inc_x2); \
x2i = (v2f64) __msa_pckod_d((v2i64) x5, (v2i64) x4); \
\
tp1r OP0 src0i * x0i; \
t2 = LD_DP(pa0 + 4); \
src2r = (v2f64) __msa_pckev_d((v2i64) t5, (v2i64) t4); \
\
tp1i OP2 src0i * x0r; \
t3 = LD_DP(pa0 + 6); \
src2i = (v2f64) __msa_pckod_d((v2i64) t5, (v2i64) t4); \
\
tp2r += src2r * x2r; \
x6 = LD_DP(x + 6 * inc_x2); \
\
tp2i OP1 src2r * x2i; \
x7 = LD_DP(x + 7 * inc_x2); \
\
tp3r OP0 src2i * x2i; \
t6 = LD_DP(pa0 + 12); \
\
tp3i OP2 src2i * x2r; \
t7 = LD_DP(pa0 + 14); \
\
x1r = (v2f64) __msa_pckev_d((v2i64) x3, (v2i64) x2); \
x1i = (v2f64) __msa_pckod_d((v2i64) x3, (v2i64) x2); \
src1r = (v2f64) __msa_pckev_d((v2i64) t3, (v2i64) t2); \
src1i = (v2f64) __msa_pckod_d((v2i64) t3, (v2i64) t2); \
\
tp0r += src1r * x1r; \
x0 = LD_DP(x + 8 * inc_x2); \
x3r = (v2f64) __msa_pckev_d((v2i64) x7, (v2i64) x6); \
\
tp0i OP1 src1r * x1i; \
x1 = LD_DP(x + 9 * inc_x2); \
x3i = (v2f64) __msa_pckod_d((v2i64) x7, (v2i64) x6); \
\
tp1r OP0 src1i * x1i; \
t0 = LD_DP(pa0 + 16); \
src3r = (v2f64) __msa_pckev_d((v2i64) t7, (v2i64) t6); \
\
tp1i OP2 src1i * x1r; \
t1 = LD_DP(pa0 + 18); \
src3i = (v2f64) __msa_pckod_d((v2i64) t7, (v2i64) t6); \
\
tp2r += src3r * x3r; \
x4 = LD_DP(x + 12 * inc_x2); \
\
tp2i OP1 src3r * x3i; \
x5 = LD_DP(x + 13 * inc_x2); \
\
tp3r OP0 src3i * x3i; \
t4 = LD_DP(pa0 + 24); \
\
tp3i OP2 src3i * x3r; \
t5 = LD_DP(pa0 + 26); \
\
x0r = (v2f64) __msa_pckev_d((v2i64) x1, (v2i64) x0); \
x0i = (v2f64) __msa_pckod_d((v2i64) x1, (v2i64) x0); \
src0r = (v2f64) __msa_pckev_d((v2i64) t1, (v2i64) t0); \
src0i = (v2f64) __msa_pckod_d((v2i64) t1, (v2i64) t0); \
\
tp0r += src0r * x0r; \
x2 = LD_DP(x + 10 * inc_x2); \
x2r = (v2f64) __msa_pckev_d((v2i64) x5, (v2i64) x4); \
\
tp0i OP1 src0r * x0i; \
x3 = LD_DP(x + 11 * inc_x2); \
x2i = (v2f64) __msa_pckod_d((v2i64) x5, (v2i64) x4); \
\
tp1r OP0 src0i * x0i; \
t2 = LD_DP(pa0 + 20); \
src2r = (v2f64) __msa_pckev_d((v2i64) t5, (v2i64) t4); \
\
tp1i OP2 src0i * x0r; \
t3 = LD_DP(pa0 + 22); \
src2i = (v2f64) __msa_pckod_d((v2i64) t5, (v2i64) t4); \
\
tp2r += src2r * x2r; \
x6 = LD_DP(x + 14 * inc_x2); \
\
tp2i OP1 src2r * x2i; \
x7 = LD_DP(x + 15 * inc_x2); \
\
tp3r OP0 src2i * x2i; \
t6 = LD_DP(pa0 + 28); \
\
tp3i OP2 src2i * x2r; \
t7 = LD_DP(pa0 + 30); \
\
x1r = (v2f64) __msa_pckev_d((v2i64) x3, (v2i64) x2); \
x1i = (v2f64) __msa_pckod_d((v2i64) x3, (v2i64) x2); \
src1r = (v2f64) __msa_pckev_d((v2i64) t3, (v2i64) t2); \
src1i = (v2f64) __msa_pckod_d((v2i64) t3, (v2i64) t2); \
\
tp0r += src1r * x1r; \
x3r = (v2f64) __msa_pckev_d((v2i64) x7, (v2i64) x6); \
\
tp0i OP1 src1r * x1i; \
x3i = (v2f64) __msa_pckod_d((v2i64) x7, (v2i64) x6); \
\
tp1r OP0 src1i * x1i; \
src3r = (v2f64) __msa_pckev_d((v2i64) t7, (v2i64) t6); \
\
tp1i OP2 src1i * x1r; \
src3i = (v2f64) __msa_pckod_d((v2i64) t7, (v2i64) t6); \
\
tp2r += src3r * x3r; \
tp2i OP1 src3r * x3i; \
tp3r OP0 src3i * x3i; \
tp3i OP2 src3i * x3r; \
\
pa0 += 2 * 16; \
x += inc_x2 * 16; \
\
tp0r += tp1r + tp2r + tp3r; \
tp0i += tp1i + tp2i + tp3i; \
} \
\
if (m & 8) \
{ \
ZLOAD_X8(); \
ZGEMV_T_8x1(); \
\
pa0 += 2 * 8; \
x += inc_x2 * 8; \
} \
\
if (m & 4) \
{ \
ZLOAD_X4(); \
ZGEMV_T_4x1(); \
\
pa0 += 2 * 4; \
x += inc_x2 * 4; \
} \
\
if (m & 2) \
{ \
ZLOAD_X2(); \
ZGEMV_T_2x1(); \
\
pa0 += 2 * 2; \
x += inc_x2 * 2; \
} \
\
temp0r = tp0r[0] + tp0r[1]; \
temp0i = tp0i[0] + tp0i[1]; \
\
if (m & 1) \
{ \
ZGEMV_T_1x1(); \
\
pa0 += 2; \
x += inc_x2; \
} \
\
ZSCALE_STORE_Y1_GP(); \
\
A += lda2; \
y += inc_y2; \
} \
int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alphar, FLOAT alphai,
FLOAT *A, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y,
BLASLONG inc_y, FLOAT *buffer)
{
BLASLONG i, j, k;
BLASLONG i, j, pref_offset;
BLASLONG inc_x2, inc_y2, lda2;
FLOAT *pa0, *pa1, *pa2, *pa3;
FLOAT *pa0, *pa0_pref;
FLOAT *srcx_org = x;
FLOAT temp0r, temp0i, temp2r, temp2i, temp1r, temp1i, temp3r, temp3i;
FLOAT res0r, res0i, res2r, res2i, res1r, res1i, res3r, res3i;
FLOAT temp0r, temp0i;
FLOAT res0r, res0i;
v2f64 zero = {0};
v2f64 x0, x1, x2, x3, x0r, x1r, x0i, x1i;
v2f64 t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t13, t14, t15;
v2f64 src0r, src1r, src2r, src3r, src4r, src5r, src6r, src7r;
v2f64 src0i, src1i, src2i, src3i, src4i, src5i, src6i, src7i;
v2f64 x4, x5, x6, x7, x2r, x3r, x2i, x3i;
v2f64 t0, t1, t2, t3, t4, t5, t6, t7;
v2f64 src0r, src1r, src2r, src3r;
v2f64 src0i, src1i, src2i, src3i;
v2f64 tp0r, tp1r, tp2r, tp3r, tp0i, tp1i, tp2i, tp3i;
lda2 = 2 * lda;
pa0 = A;
pa1 = A + lda2;
pa2 = A + 2 * lda2;
pa3 = A + 3 * lda2;
inc_x2 = 2 * inc_x;
inc_y2 = 2 * inc_y;
pref_offset = (uintptr_t)A & L1_DATA_LINESIZE;
pref_offset = L1_DATA_LINESIZE - pref_offset;
pref_offset = pref_offset / sizeof(FLOAT);
if (2 == inc_x2)
{
#define ZLOAD_X8 ZLOAD_X8_VECTOR
#define ZLOAD_X4 ZLOAD_X4_VECTOR
#define ZLOAD_X2 ZLOAD_X2_VECTOR
ZGEMV_T_MSA();
#undef ZLOAD_X8
#undef ZLOAD_X4
#undef ZLOAD_X2
}
else
{
#define ZLOAD_X8 ZLOAD_X8_GP
#define ZLOAD_X4 ZLOAD_X4_GP
#define ZLOAD_X2 ZLOAD_X2_GP
ZGEMV_T_MSA();
#undef ZLOAD_X8
#undef ZLOAD_X4
#undef ZLOAD_X2
}
return(0);
}