x86: Fixed numpy CI failure when the target is ZEN.

This commit is contained in:
gxw 2024-07-10 15:11:12 +08:00
parent e1eef56e05
commit f3cebb3ca3
3 changed files with 279 additions and 296 deletions

View File

@ -85,7 +85,7 @@ void CNAME(blasint n, FLOAT alpha, FLOAT *x, blasint incx){
if (nthreads == 1) { if (nthreads == 1) {
#endif #endif
SCAL_K(n, 0, 0, alpha, x, incx, NULL, 0, NULL, 0); SCAL_K(n, 0, 0, alpha, x, incx, NULL, 0, NULL, 1);
#ifdef SMP #ifdef SMP
} else { } else {
@ -102,7 +102,7 @@ void CNAME(blasint n, FLOAT alpha, FLOAT *x, blasint incx){
#else #else
&alpha, &alpha,
#endif #endif
x, incx, NULL, 0, NULL, 0, (int (*)(void))SCAL_K, nthreads); x, incx, NULL, 0, NULL, 1, (int (*)(void))SCAL_K, nthreads);
} }
#endif #endif

View File

@ -43,21 +43,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
static void dscal_kernel_8( BLASLONG n, FLOAT *da , FLOAT *x ) static void dscal_kernel_8( BLASLONG n, FLOAT *da , FLOAT *x )
{ {
BLASLONG i; BLASLONG i;
FLOAT alpha = *da; FLOAT alpha = *da;
for( i=0; i<n; i+=8 ) for( i=0; i<n; i+=8 )
{ {
x[0] *= alpha; x[0] *= alpha;
x[1] *= alpha; x[1] *= alpha;
x[2] *= alpha; x[2] *= alpha;
x[3] *= alpha; x[3] *= alpha;
x[4] *= alpha; x[4] *= alpha;
x[5] *= alpha; x[5] *= alpha;
x[6] *= alpha; x[6] *= alpha;
x[7] *= alpha; x[7] *= alpha;
x+=8; x+=8;
} }
} }
@ -65,19 +65,19 @@ static void dscal_kernel_8( BLASLONG n, FLOAT *da , FLOAT *x )
static void dscal_kernel_8_zero( BLASLONG n, FLOAT *alpha , FLOAT *x ) static void dscal_kernel_8_zero( BLASLONG n, FLOAT *alpha , FLOAT *x )
{ {
BLASLONG i; BLASLONG i;
for( i=0; i<n; i+=8 ) for( i=0; i<n; i+=8 )
{ {
x[0] = 0.0; x[0] = 0.0;
x[1] = 0.0; x[1] = 0.0;
x[2] = 0.0; x[2] = 0.0;
x[3] = 0.0; x[3] = 0.0;
x[4] = 0.0; x[4] = 0.0;
x[5] = 0.0; x[5] = 0.0;
x[6] = 0.0; x[6] = 0.0;
x[7] = 0.0; x[7] = 0.0;
x+=8; x+=8;
} }
} }
@ -89,51 +89,51 @@ static void dscal_kernel_inc_8(BLASLONG n, FLOAT *alpha, FLOAT *x, BLASLONG inc_
static void dscal_kernel_inc_8(BLASLONG n, FLOAT *alpha, FLOAT *x, BLASLONG inc_x) static void dscal_kernel_inc_8(BLASLONG n, FLOAT *alpha, FLOAT *x, BLASLONG inc_x)
{ {
FLOAT *x1=NULL; FLOAT *x1=NULL;
BLASLONG inc_x3; BLASLONG inc_x3;
inc_x <<= 3; inc_x <<= 3;
inc_x3 = (inc_x << 1) + inc_x; inc_x3 = (inc_x << 1) + inc_x;
__asm__ __volatile__ __asm__ __volatile__
( (
"movddup (%3), %%xmm0 \n\t" // alpha "movddup (%3), %%xmm0 \n\t" // alpha
"leaq (%1,%4,4), %2 \n\t" "leaq (%1,%4,4), %2 \n\t"
".p2align 4 \n\t" ".p2align 4 \n\t"
"1: \n\t" "1: \n\t"
"movsd (%1) , %%xmm4 \n\t" "movsd (%1) , %%xmm4 \n\t"
"movhpd (%1,%4,1), %%xmm4 \n\t" "movhpd (%1,%4,1), %%xmm4 \n\t"
"movsd (%1,%4,2), %%xmm5 \n\t" "movsd (%1,%4,2), %%xmm5 \n\t"
"movhpd (%1,%5,1), %%xmm5 \n\t" "movhpd (%1,%5,1), %%xmm5 \n\t"
"movsd (%2) , %%xmm6 \n\t" "movsd (%2) , %%xmm6 \n\t"
"movhpd (%2,%4,1), %%xmm6 \n\t" "movhpd (%2,%4,1), %%xmm6 \n\t"
"movsd (%2,%4,2), %%xmm7 \n\t" "movsd (%2,%4,2), %%xmm7 \n\t"
"movhpd (%2,%5,1), %%xmm7 \n\t" "movhpd (%2,%5,1), %%xmm7 \n\t"
"mulpd %%xmm0, %%xmm4 \n\t" "mulpd %%xmm0, %%xmm4 \n\t"
"mulpd %%xmm0, %%xmm5 \n\t" "mulpd %%xmm0, %%xmm5 \n\t"
"mulpd %%xmm0, %%xmm6 \n\t" "mulpd %%xmm0, %%xmm6 \n\t"
"mulpd %%xmm0, %%xmm7 \n\t" "mulpd %%xmm0, %%xmm7 \n\t"
"movsd %%xmm4 , (%1) \n\t" "movsd %%xmm4 , (%1) \n\t"
"movhpd %%xmm4 , (%1,%4,1) \n\t" "movhpd %%xmm4 , (%1,%4,1) \n\t"
"movsd %%xmm5 , (%1,%4,2) \n\t" "movsd %%xmm5 , (%1,%4,2) \n\t"
"movhpd %%xmm5 , (%1,%5,1) \n\t" "movhpd %%xmm5 , (%1,%5,1) \n\t"
"movsd %%xmm6 , (%2) \n\t" "movsd %%xmm6 , (%2) \n\t"
"movhpd %%xmm6 , (%2,%4,1) \n\t" "movhpd %%xmm6 , (%2,%4,1) \n\t"
"movsd %%xmm7 , (%2,%4,2) \n\t" "movsd %%xmm7 , (%2,%4,2) \n\t"
"movhpd %%xmm7 , (%2,%5,1) \n\t" "movhpd %%xmm7 , (%2,%5,1) \n\t"
"leaq (%1,%4,8), %1 \n\t" "leaq (%1,%4,8), %1 \n\t"
"leaq (%2,%4,8), %2 \n\t" "leaq (%2,%4,8), %2 \n\t"
"subq $8, %0 \n\t" "subq $8, %0 \n\t"
"jnz 1b \n\t" "jnz 1b \n\t"
: :
"+r" (n), // 0 "+r" (n), // 0
@ -150,106 +150,96 @@ static void dscal_kernel_inc_8(BLASLONG n, FLOAT *alpha, FLOAT *x, BLASLONG inc_
"%xmm12", "%xmm13", "%xmm14", "%xmm15", "%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory" "memory"
); );
} }
int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *dummy, BLASLONG dummy2) int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *dummy, BLASLONG dummy2)
{ {
BLASLONG i=0,j=0; BLASLONG i = 0, j = 0;
if ( inc_x != 1 ) // Resolved issue 4728 when the caller is dscal
{ if (dummy2 == 1 && da == 0.0)
{
if ( da == 0.0 ) if ( inc_x != 1 )
{ {
BLASLONG n1 = n & -8;
BLASLONG n1 = n & -2; if ( n1 > 0 )
{
while(j < n1) dscal_kernel_inc_8(n1, &da, x, inc_x);
{ i = n1 * inc_x;
j = n1;
if (isinf(x[i])||isnan(x[i])) }
x[i]=NAN; while(j < n)
else x[i]=0.0; {
if (isinf(x[i+inc_x])||isnan(x[i+inc_x])) x[i] *= da;
x[i+inc_x]=NAN; i += inc_x ;
else x[i+inc_x]=0.0; j++;
i += 2*inc_x ; }
j+=2; }
else
} {
BLASLONG n1 = n & -8;
while(j < n) if ( n1 > 0)
{ dscal_kernel_8(n1 , &da , x);
for ( i = n1 ; i < n; i++ )
if (isinf(x[i])||isnan(x[i])) x[i] *= da;
x[i]=NAN; }
else x[i]=0.0; }
i += inc_x ; else
j++; {
if ( inc_x != 1 )
} {
} if( da == 0.0)
else {
{ BLASLONG n1 = n & -2;
while(j < n1)
BLASLONG n1 = n & -8; {
if ( n1 > 0 ) x[i] = 0.0;
{ x[i+inc_x] = 0.0;
dscal_kernel_inc_8(n1, &da, x, inc_x); i += 2 * inc_x ;
i = n1 * inc_x; j += 2;
j = n1; }
} while(j < n)
{
while(j < n) x[i] = 0.0;
{ i += inc_x ;
j++;
x[i] *= da; }
i += inc_x ; }
j++; else
{
} BLASLONG n1 = n & -8;
if ( n1 > 0 )
} {
dscal_kernel_inc_8(n1, &da, x, inc_x);
return(0); i = n1 * inc_x;
} j = n1;
}
BLASLONG n1 = n & -8; while(j < n)
if ( n1 > 0 ) {
{ x[i] *= da;
// if ( da == 0.0 ) i += inc_x ;
// dscal_kernel_8_zero(n1 , &da , x); j++;
// else }
dscal_kernel_8(n1 , &da , x); }
} }
else
if ( da == 0.0 ) {
{ if ( da == 0.0 )
for ( i=n1 ; i<n; i++ ) {
{ BLASLONG n1 = n & -8;
if(isinf(x[i])||isnan(x[i])) if ( n1 > 0)
x[i]=NAN; dscal_kernel_8_zero(n1, &da, x);
else x[i] = 0.0; for ( i = n1 ; i < n; i++ )
} x[i] = 0.0;
} }
else if (isinf(da)){ else
for ( i=n1 ; i<n; i++) {
if (x[i]==0.) x[i]=NAN; BLASLONG n1 = n & -8;
else x[i] *=da; if ( n1 > 0)
} dscal_kernel_8(n1 , &da , x);
else for ( i = n1 ; i < n; i++ )
{ x[i] *= da;
}
for ( i=n1 ; i<n; i++ ) }
{ }
if(isinf(x[i]))
x[i]=NAN;
else x[i] *= da;
}
}
return(0);
} }

View File

@ -39,21 +39,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
static void sscal_kernel_16( BLASLONG n, FLOAT *da , FLOAT *x ) static void sscal_kernel_16( BLASLONG n, FLOAT *da , FLOAT *x )
{ {
BLASLONG i; BLASLONG i;
FLOAT alpha = *da; FLOAT alpha = *da;
for( i=0; i<n; i+=8 ) for( i=0; i<n; i+=8 )
{ {
x[0] *= alpha; x[0] *= alpha;
x[1] *= alpha; x[1] *= alpha;
x[2] *= alpha; x[2] *= alpha;
x[3] *= alpha; x[3] *= alpha;
x[4] *= alpha; x[4] *= alpha;
x[5] *= alpha; x[5] *= alpha;
x[6] *= alpha; x[6] *= alpha;
x[7] *= alpha; x[7] *= alpha;
x+=8; x+=8;
} }
} }
@ -61,19 +61,19 @@ static void sscal_kernel_16( BLASLONG n, FLOAT *da , FLOAT *x )
static void sscal_kernel_16_zero( BLASLONG n, FLOAT *alpha , FLOAT *x ) static void sscal_kernel_16_zero( BLASLONG n, FLOAT *alpha , FLOAT *x )
{ {
BLASLONG i; BLASLONG i;
for( i=0; i<n; i+=8 ) for( i=0; i<n; i+=8 )
{ {
x[0] = 0.0; x[0] = 0.0;
x[1] = 0.0; x[1] = 0.0;
x[2] = 0.0; x[2] = 0.0;
x[3] = 0.0; x[3] = 0.0;
x[4] = 0.0; x[4] = 0.0;
x[5] = 0.0; x[5] = 0.0;
x[6] = 0.0; x[6] = 0.0;
x[7] = 0.0; x[7] = 0.0;
x+=8; x+=8;
} }
} }
@ -85,126 +85,119 @@ static void sscal_kernel_inc_8(BLASLONG n, FLOAT *alpha, FLOAT *x, BLASLONG inc_
static void sscal_kernel_inc_8(BLASLONG n, FLOAT *alpha, FLOAT *x, BLASLONG inc_x) static void sscal_kernel_inc_8(BLASLONG n, FLOAT *alpha, FLOAT *x, BLASLONG inc_x)
{ {
BLASLONG i; BLASLONG i;
BLASLONG inc_x2 = 2 * inc_x; BLASLONG inc_x2 = 2 * inc_x;
BLASLONG inc_x3 = inc_x2 + inc_x; BLASLONG inc_x3 = inc_x2 + inc_x;
FLOAT t0,t1,t2,t3; FLOAT t0,t1,t2,t3;
FLOAT da = alpha[0]; FLOAT da = alpha[0];
for ( i=0; i<n; i+=4 ) for ( i=0; i<n; i+=4 )
{ {
t0 = da * x[0]; t0 = da * x[0];
t1 = da * x[inc_x]; t1 = da * x[inc_x];
t2 = da * x[inc_x2]; t2 = da * x[inc_x2];
t3 = da * x[inc_x3]; t3 = da * x[inc_x3];
x[0] = t0; x[0] = t0;
x[inc_x] = t1; x[inc_x] = t1;
x[inc_x2] = t2; x[inc_x2] = t2;
x[inc_x3] = t3; x[inc_x3] = t3;
x+=4*inc_x; x+=4*inc_x;
} }
} }
int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *dummy, BLASLONG dummy2) int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *dummy, BLASLONG dummy2)
{ {
BLASLONG i=0,j=0; BLASLONG i = 0, j = 0;
if ( inc_x != 1 ) // Resolved issue 4728 when the caller is sscal
{ if (dummy2 == 1 && da == 0.0)
{
if ( da == 0.0 ) if ( inc_x != 1 )
{ {
BLASLONG n1 = n & -2; BLASLONG n1 = n & -8;
if ( n1 > 0 )
while(j < n1) {
{ sscal_kernel_inc_8(n1, &da, x, inc_x);
if (isinf(x[i])||isnan(x[i])) i = n1 * inc_x;
x[i]=NAN; j = n1;
else x[i]=0.0; }
if (isinf(x[i+inc_x])||isnan(x[i+inc_x])) while(j < n)
x[i+inc_x]=NAN; {
else x[i+inc_x]=0.0; x[i] *= da;
i += 2*inc_x ; i += inc_x ;
j+=2; j++;
}
} }
else
while(j < n) {
{ BLASLONG n1 = n & -16;
if (isinf(x[i])||isnan(x[i])) if ( n1 > 0)
x[i]=NAN; sscal_kernel_16(n1 , &da , x);
else x[i]=0.0; for ( i = n1 ; i < n; i++ )
i += inc_x ; x[i] *= da;
j++; }
}
} else
} {
else if ( inc_x != 1 )
{ {
#if 1 if( da == 0.0)
BLASLONG n1 = n & -8; {
if ( n1 > 0 ) BLASLONG n1 = n & -2;
{ while(j < n1)
sscal_kernel_inc_8(n1, &da, x, inc_x); {
i = n1 * inc_x; x[i] = 0.0;
j = n1; x[i+inc_x] = 0.0;
} i += 2 * inc_x ;
#endif j += 2;
while(j < n) }
{ while(j < n)
x[i] *= da; {
i += inc_x ; x[i] = 0.0;
j++; i += inc_x ;
j++;
} }
}
} else
return(0); {
} BLASLONG n1 = n & -8;
if ( n1 > 0 )
BLASLONG n1 = n & -16; {
if ( n1 > 0 ) sscal_kernel_inc_8(n1, &da, x, inc_x);
{ i = n1 * inc_x;
//if ( da == 0.0 ) j = n1;
// sscal_kernel_16_zero(n1 , &da , x); }
//else while(j < n)
sscal_kernel_16(n1 , &da , x); {
} x[i] *= da;
i += inc_x ;
if ( da == 0.0 ) j++;
{ }
for ( i=n1 ; i<n; i++ ) }
{ }
if (isinf(x[i])||isnan(x[i])) else
x[i]=NAN; {
else x[i]=0.0; if ( da == 0.0 )
} {
} BLASLONG n1 = n & -16;
else if ( isinf(da) ) if ( n1 > 0)
{ sscal_kernel_16_zero(n1, &da, x);
for ( i=n1 ; i<n; i++ ) for ( i = n1 ; i < n; i++ )
{ x[i] = 0.0;
if (x[i] == 0.0) }
x[i]=NAN; else
else x[i] *= da; {
} BLASLONG n1 = n & -16;
} if ( n1 > 0)
else sscal_kernel_16(n1 , &da , x);
{ for ( i = n1 ; i < n; i++ )
x[i] *= da;
for ( i=n1 ; i<n; i++ ) }
{ }
if (isinf(x[i])) }
x[i]=NAN;
else x[i] *= da;
}
}
return(0);
} }