floraachy/OpenBLAS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

DTRSM bug fix for MIPS P5600 and I6400

Browse Source 
Signed-off-by: Kaustubh Raste <kaustubh.raste@imgtec.com>
This commit is contained in:
Kaustubh Raste
2016-05-17 15:48:02 +05:30
parent 8bf71e9e06
commit d7cbc7ac13
5 changed files with 610 additions and 587 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kernel/mips/dtrsm_kernel_RT_8x4_msa.c
+253 -104
View File
@@ -200,7 +200,7 @@ static void dsolve_8x4_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLO
ST_DP4(src_c0, src_c1, src_c2, src_c3, a, 2);
}
static void dsolve_8x2_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, int bk)
static void dsolve_8x2_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG bk)
{
v2f64 src_c0, src_c1, src_c2, src_c3, src_c4, src_c5, src_c6, src_c7;
v2f64 src_b0, src_b2, src_b3;
@@ -210,10 +210,40 @@ static void dsolve_8x2_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, int bk
if (bk > 0)
{
v2f64 src_a0, src_a1, src_a2, src_a3, src_b, src_b0;
BLASLONG i;
FLOAT *pba = a, *pbb = b;
v2f64 src_b, src_b1, src_a0, src_a1, src_a2, src_a3;
v2f64 src_a4, src_a5, src_a6, src_a7;
LD_DP4(a + 16, 2, src_a0, src_a1, src_a2, src_a3);
src_b0 = LD_DP(b + 4);
LD_DP4(pba, 2, src_a0, src_a1, src_a2, src_a3);
src_b0 = LD_DP(pbb);
for (i = bk - 1; i--;)
{
pba += 8;
pbb += 2;
LD_DP4(pba, 2, src_a4, src_a5, src_a6, src_a7);
src_b1 = LD_DP(pbb);
src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b;
src_c1 -= src_a1 * src_b;
src_c2 -= src_a2 * src_b;
src_c3 -= src_a3 * src_b;
src_b = (v2f64) __msa_ilvl_d((v2i64) src_b0, (v2i64) src_b0);
src_c4 -= src_a0 * src_b;
src_c5 -= src_a1 * src_b;
src_c6 -= src_a2 * src_b;
src_c7 -= src_a3 * src_b;
src_a0 = src_a4;
src_a1 = src_a5;
src_a2 = src_a6;
src_a3 = src_a7;
src_b0 = src_b1;
}
src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b;
@@ -228,6 +258,9 @@ static void dsolve_8x2_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, int bk
src_c7 -= src_a3 * src_b;
}
a -= 16;
b -= 4;
src_b0 = __msa_cast_to_vector_double(*(b + 0));
src_b0 = (v2f64) __msa_splati_d((v2i64) src_b0, 0);
src_b2 = LD_DP(b + 2);
@@ -256,13 +289,57 @@ static void dsolve_8x2_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, int bk
ST_DP4(src_c4, src_c5, src_c6, src_c7, a + 8, 2);
}
static void dsolve_8x1_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c)
static void dsolve_8x1_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG bk)
{
v2f64 src_c0, src_c1, src_c2, src_c3;
v2f64 src_b0;
LD_DP4(c, 2, src_c0, src_c1, src_c2, src_c3);
if (bk > 0)
{
BLASLONG i;
FLOAT *aa = a, *bb = b;
v2f64 src_a0, src_a1, src_a2, src_a3, src_a4, src_a5, src_a6, src_a7;
v2f64 src_b1;
LD_DP4(aa, 2, src_a0, src_a1, src_a2, src_a3);
src_b0 = LD_DP(bb);
aa += 8;
bb += 1;
for (i = (bk - 1); i--;)
{
LD_DP4(aa, 2, src_a4, src_a5, src_a6, src_a7);
src_b1 = LD_DP(bb);
src_b0 = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b0;
src_c1 -= src_a1 * src_b0;
src_c2 -= src_a2 * src_b0;
src_c3 -= src_a3 * src_b0;
src_a0 = src_a4;
src_a1 = src_a5;
src_a2 = src_a6;
src_a3 = src_a7;
src_b0 = src_b1;
aa += 8;
bb += 1;
}
src_b0 = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b0;
src_c1 -= src_a1 * src_b0;
src_c2 -= src_a2 * src_b0;
src_c3 -= src_a3 * src_b0;
}
a -= 8;
b -= 1;
src_b0 = __msa_cast_to_vector_double(*b);
src_b0 = (v2f64) __msa_splati_d((v2i64) src_b0, 0);
@@ -289,7 +366,7 @@ static void dsolve_4x4_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLO
if (bk > 0)
{
BLASLONG i;
FLOAT *aa = a + 16, *bb = b + 16;
FLOAT *aa = a, *bb = b;
v2f64 src_a0, src_a1, src_b, src_b0, src_b1;
for (i = bk; i--;)
@@ -318,6 +395,9 @@ static void dsolve_4x4_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLO
}
}
a -= 16;
b -= 16;
src_b12 = LD_DP(b + 12);
src_b13 = (v2f64) __msa_splati_d((v2i64) src_b12, 1);
src_b12 = (v2f64) __msa_splati_d((v2i64) src_b12, 0);
@@ -376,7 +456,7 @@ static void dsolve_4x4_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLO
ST_DP4(src_c0, src_c1, src_c2, src_c3, a, 2);
}
static void dsolve_4x2_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, int bk)
static void dsolve_4x2_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG bk)
{
v2f64 src_c0, src_c1, src_c2, src_c3, src_b0, src_b2, src_b3;
@@ -385,20 +465,31 @@ static void dsolve_4x2_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, int bk
if (bk > 0)
{
BLASLONG i;
FLOAT *aa = a, *bb = b;
v2f64 src_a0, src_a1, src_b, src_b0;
LD_DP2(a + 8, 2, src_a0, src_a1);
src_b0 = LD_DP(b + 4);
for (i = bk; i--;)
{
LD_DP2(aa, 2, src_a0, src_a1);
src_b0 = LD_DP(bb);
src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b;
src_c1 -= src_a1 * src_b;
src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b;
src_c1 -= src_a1 * src_b;
src_b = (v2f64) __msa_ilvl_d((v2i64) src_b0, (v2i64) src_b0);
src_c2 -= src_a0 * src_b;
src_c3 -= src_a1 * src_b;
src_b = (v2f64) __msa_ilvl_d((v2i64) src_b0, (v2i64) src_b0);
src_c2 -= src_a0 * src_b;
src_c3 -= src_a1 * src_b;
aa += 4;
bb += 2;
}
}
a -= 8;
b -= 4;
src_b0 = __msa_cast_to_vector_double(*(b + 0));
src_b0 = (v2f64) __msa_splati_d((v2i64) src_b0, 0);
src_b2 = LD_DP(b + 2);
@@ -420,17 +511,36 @@ static void dsolve_4x2_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, int bk
ST_DP4(src_c0, src_c1, src_c2, src_c3, a, 2);
}
static void dsolve_4x1_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c)
static void dsolve_4x1_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG bk)
{
FLOAT b0, c0, c1, c2, c3;
b0 = *(b + 0);
c0 = *(c + 0);
c1 = *(c + 1);
c2 = *(c + 2);
c3 = *(c + 3);
if (bk > 0)
{
BLASLONG i;
FLOAT *aa = a, *bb = b;
for (i = bk; i--;)
{
c0 -= aa[0] * bb[0];
c1 -= aa[1] * bb[0];
c2 -= aa[2] * bb[0];
c3 -= aa[3] * bb[0];
aa += 4;
bb += 1;
}
}
a -= 4;
b0 = *(b - 1);
c0 *= b0;
c1 *= b0;
c2 *= b0;
@@ -464,35 +574,27 @@ static void dsolve_2x4_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLO
if (bk > 0)
{
BLASLONG i;
FLOAT *aa = a + 8, *bb = b + 16;
FLOAT a0, a1, b0, b1, b2, b3;
FLOAT *aa = a, *bb = b;
for (i = bk; i--;)
{
a0 = aa[0];
a1 = aa[1];
b0 = bb[0];
c0 -= a0 * b0;
c1 -= a1 * b0;
b1 = bb[1];
c0_nxt1 -= a0 * b1;
c1_nxt1 -= a1 * b1;
b2 = bb[2];
c0_nxt2 -= a0 * b2;
c1_nxt2 -= a1 * b2;
b3 = bb[3];
c0_nxt3 -= a0 * b3;
c1_nxt3 -= a1 * b3;
c0 -= aa[0] * bb[0];
c1 -= aa[1] * bb[0];
c0_nxt1 -= aa[0] * bb[1];
c1_nxt1 -= aa[1] * bb[1];
c0_nxt2 -= aa[0] * bb[2];
c1_nxt2 -= aa[1] * bb[2];
c0_nxt3 -= aa[0] * bb[3];
c1_nxt3 -= aa[1] * bb[3];
aa += 2;
bb += 4;
}
}
a -= 8;
b -= 16;
b0 = *b;
b4 = *(b + 4);
b5 = *(b + 5);
@@ -539,44 +641,44 @@ static void dsolve_2x4_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLO
*(c + 0) = c0;
*(c + 1) = c1;
*(c + 0 + 1 * ldc) = c0_nxt1;
*(c + 1 + 1 * ldc) = c1_nxt1;
*(c + 0 + 2 * ldc) = c0_nxt2;
*(c + 1 + 2 * ldc) = c1_nxt2;
*(c + 0 + 3 * ldc) = c0_nxt3;
*(c + 1 + 3 * ldc) = c1_nxt3;
}
static void dsolve_2x2_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, int bk)
static void dsolve_2x2_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG bk)
{
FLOAT b0, b2, b3;
FLOAT c0, c1, c0_nxt, c1_nxt;
FLOAT b0, b2, b3, c0, c1, c0_nxt, c1_nxt;
c0 = *(c + 0);
c1 = *(c + 1);
c0_nxt = *(c + 0 + ldc);
c1_nxt = *(c + 1 + ldc);
if (bk > 0)
{
FLOAT a0, a1, b0, b1;
BLASLONG i;
FLOAT *aa = a, *bb = b;
a0 = a[4];
a1 = a[5];
for (i = bk; i--;)
{
c0 -= aa[0] * bb[0];
c1 -= aa[1] * bb[0];
b0 = b[4];
c0 -= a0 * b0;
c1 -= a1 * b0;
c0_nxt -= aa[0] * bb[1];
c1_nxt -= aa[1] * bb[1];
b1 = b[5];
c0_nxt -= a0 * b1;
c1_nxt -= a1 * b1;
aa += 2;
bb += 2;
}
}
a -= 4;
b -= 4;
b3 = *(b + 3);
b2 = *(b + 2);
b0 = *b;
@@ -601,20 +703,35 @@ static void dsolve_2x2_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, int bk
*(c + 1 + ldc) = c1_nxt;
}
static void dsolve_2x1_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c)
static void dsolve_2x1_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG bk)
{
FLOAT b0, c0, c1;
c0 = *(c + 0);
c1 = *(c + 1);
b0 = *b;
if (bk > 0)
{
BLASLONG i;
FLOAT *aa = a, *bb = b;
for (i = bk; i--;)
{
c0 -= aa[0] * bb[0];
c1 -= aa[1] * bb[0];
aa += 2;
bb += 1;
}
}
b0 = *(b - 1);
c0 *= b0;
c1 *= b0;
*(a + 0) = c0;
*(a + 1) = c1;
*(a - 2) = c0;
*(a - 1) = c1;
*(c + 0) = c0;
*(c + 1) = c1;
@@ -622,31 +739,33 @@ static void dsolve_2x1_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c)
static void dsolve_1x4_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG bk)
{
FLOAT b0, b4, b5, b8, b9, b10, b12, b13, b14, b15;
FLOAT c0, c0_nxt1, c0_nxt2, c0_nxt3;
FLOAT b0, b4, b5, b8, b9, b10, b12, b13, b14, b15, c0, c1, c2, c3;
c0 = *(c + 0);
c0_nxt1 = *(c + 1 * ldc);
c0_nxt2 = *(c + 2 * ldc);
c0_nxt3 = *(c + 3 * ldc);
c1 = *(c + 1 * ldc);
c2 = *(c + 2 * ldc);
c3 = *(c + 3 * ldc);
if (bk > 0)
{
BLASLONG i;
FLOAT *aa = a + 4, *bb = b + 16;
FLOAT *aa = a, *bb = b;
for (i = bk; i--;)
{
c0 -= aa[0] * bb[0];
c0_nxt1 -= aa[0] * bb[1];
c0_nxt2 -= aa[0] * bb[2];
c0_nxt3 -= aa[0] * bb[3];
c1 -= aa[0] * bb[1];
c2 -= aa[0] * bb[2];
c3 -= aa[0] * bb[3];
aa += 1;
bb += 4;
}
}
a -= 4;
b -= 16;
b0 = *b;
b4 = *(b + 4);
b5 = *(b + 5);
@@ -658,58 +777,86 @@ static void dsolve_1x4_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLO
b14 = *(b + 14);
b15 = *(b + 15);
c0_nxt3 *= b15;
c3 *= b15;
c0_nxt2 -= c0_nxt3 * b14;
c0_nxt2 *= b10;
c2 -= c3 * b14;
c2 *= b10;
c0_nxt1 -= c0_nxt3 * b13;
c0_nxt1 -= c0_nxt2 * b9;
c0_nxt1 *= b5;
c1 -= c3 * b13;
c1 -= c2 * b9;
c1 *= b5;
c0 -= c0_nxt3 * b12;
c0 -= c0_nxt2 * b8;
c0 -= c0_nxt1 * b4;
c0 -= c3 * b12;
c0 -= c2 * b8;
c0 -= c1 * b4;
c0 *= b0;
*(a + 0) = c0;
*(a + 1) = c0_nxt1;
*(a + 2) = c0_nxt2;
*(a + 3) = c0_nxt3;
*(a + 1) = c1;
*(a + 2) = c2;
*(a + 3) = c3;
*(c) = c0;
*(c + 1 * ldc) = c0_nxt1;
*(c + 2 * ldc) = c0_nxt2;
*(c + 3 * ldc) = c0_nxt3;
*(c + 0 * ldc) = c0;
*(c + 1 * ldc) = c1;
*(c + 2 * ldc) = c2;
*(c + 3 * ldc) = c3;
}
static void dsolve_1x2_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG bk)
{
FLOAT b0, b2, b3, c0, c0_nxt;
FLOAT b0, b2, b3, c0, c1;
c0 = *(c + 0);
c0_nxt = *(c + ldc);
c1 = *(c + ldc);
if (bk > 0)
{
c0 -= a[2] * b[4];
c0_nxt -= a[2] * b[5];
BLASLONG i;
FLOAT *aa = a, *bb = b;
for (i = bk; i--;)
{
c0 -= *aa * bb[0];
c1 -= *aa * bb[1];
aa += 1;
bb += 2;
}
}
a -= 2;
b -= 4;
b3 = *(b + 3);
b2 = *(b + 2);
b0 = *b;
c0_nxt *= b3;
c1 *= b3;
c0 -= c0_nxt * b2;
c0 -= c1 * b2;
c0 *= b0;
*(a + 0) = c0;
*(a + 1) = c0_nxt;
*(a + 1) = c1;
*(c + 0) = c0;
*(c + ldc) = c0_nxt;
*(c + ldc) = c1;
}
static void dsolve_1x1_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG bk)
{
if (bk > 0)
{
BLASLONG i;
for (i = 0; i < bk; i++)
{
*c -= a[i] * b[i];
}
}
*c *= *(b - 1);
*(a - 1) = *c;
}
int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, FLOAT *a, FLOAT *b,
@@ -729,12 +876,12 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, FLOAT *a, FLOAT *b,
aa = a;
c -= ldc;
b -= k;
bb = b + (kk - 1);
bb = b + kk;
cc = c;
for (i = (m >> 3); i--;)
{
dsolve_8x1_rt_msa(aa + 8 * kk - 8, bb, cc);
dsolve_8x1_rt_msa(aa + 8 * kk, bb, cc, k - kk);
aa += 8 * k;
cc += 8;
@@ -744,7 +891,7 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, FLOAT *a, FLOAT *b,
{
if (m & 4)
{
dsolve_4x1_rt_msa(aa + 4 * kk - 4, bb, cc);
dsolve_4x1_rt_msa(aa + 4 * kk, bb, cc, k - kk);
aa += 4 * k;
cc += 4;
@@ -752,7 +899,7 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, FLOAT *a, FLOAT *b,
if (m & 2)
{
dsolve_2x1_rt_msa(aa + 2 * kk - 2, bb, cc);
dsolve_2x1_rt_msa(aa + 2 * kk, bb, cc, k - kk);
aa += 2 * k;
cc += 2;
@@ -760,8 +907,7 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, FLOAT *a, FLOAT *b,
if (m & 1)
{
*cc *= *bb;
*(aa + kk - 1) = *cc;
dsolve_1x1_rt_msa(aa + kk, bb, cc, k - kk);
aa += k;
cc += 1;
@@ -782,7 +928,7 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, FLOAT *a, FLOAT *b,
for (i = (m >> 3); i--;)
{
dsolve_8x2_rt_msa(aa + 8 * kk - 16, bb - 4, cc, ldc, k - kk);
dsolve_8x2_rt_msa(aa + 8 * kk, bb, cc, ldc, k - kk);
aa += 8 * k;
cc += 8;
@@ -792,7 +938,7 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, FLOAT *a, FLOAT *b,
{
if (m & 4)
{
dsolve_4x2_rt_msa(aa + 4 * kk - 8, bb - 4, cc, ldc, k - kk);
dsolve_4x2_rt_msa(aa + 4 * kk, bb, cc, ldc, k - kk);
aa += 4 * k;
cc += 4;
@@ -800,7 +946,7 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, FLOAT *a, FLOAT *b,
if (m & 2)
{
dsolve_2x2_rt_msa(aa + 2 * kk - 4, bb - 4, cc, ldc, k - kk);
dsolve_2x2_rt_msa(aa + 2 * kk, bb, cc, ldc, k - kk);
aa += 2 * k;
cc += 2;
@@ -808,7 +954,10 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, FLOAT *a, FLOAT *b,
if (m & 1)
{
dsolve_1x2_rt_msa(aa + kk - 2, bb - 4, cc, ldc, k - kk);
dsolve_1x2_rt_msa(aa + kk, bb, cc, ldc, k - kk);
aa += k;
cc += 1;
}
}
@@ -836,7 +985,7 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, FLOAT *a, FLOAT *b,
{
if (m & 4)
{
dsolve_4x4_rt_msa(aa + kk * 4 - 16, bb - 16, cc, ldc, k - kk);
dsolve_4x4_rt_msa(aa + kk * 4, bb, cc, ldc, k - kk);
aa += 4 * k;
cc += 4;
@@ -844,7 +993,7 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, FLOAT *a, FLOAT *b,
if (m & 2)
{
dsolve_2x4_rt_msa(aa + kk * 2 - 8, bb - 16, cc, ldc, k - kk);
dsolve_2x4_rt_msa(aa + kk * 2, bb, cc, ldc, k - kk);
aa += 2 * k;
cc += 2;
@@ -852,7 +1001,7 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, FLOAT *a, FLOAT *b,
if (m & 1)
{
dsolve_1x4_rt_msa(aa + kk - 4, bb - 16, cc, ldc, k - kk);
dsolve_1x4_rt_msa(aa + kk, bb, cc, ldc, k - kk);
aa += k;
cc += 1;