removed obsolete files
This commit is contained in:
wernsaar
parent
658939faaa
commit
cd34e9701b
|
|
@ -1,218 +0,0 @@
|
||||||
/***************************************************************************
|
|
||||||
Copyright (c) 2014, The OpenBLAS Project
|
|
||||||
All rights reserved.
|
|
||||||
Redistribution and use in source and binary forms, with or without
|
|
||||||
modification, are permitted provided that the following conditions are
|
|
||||||
met:
|
|
||||||
1. Redistributions of source code must retain the above copyright
|
|
||||||
notice, this list of conditions and the following disclaimer.
|
|
||||||
2. Redistributions in binary form must reproduce the above copyright
|
|
||||||
notice, this list of conditions and the following disclaimer in
|
|
||||||
the documentation and/or other materials provided with the
|
|
||||||
distribution.
|
|
||||||
3. Neither the name of the OpenBLAS project nor the names of
|
|
||||||
its contributors may be used to endorse or promote products
|
|
||||||
derived from this software without specific prior written permission.
|
|
||||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
||||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
||||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
||||||
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
|
|
||||||
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
||||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
||||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
||||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
||||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
|
|
||||||
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
||||||
*****************************************************************************/
|
|
||||||
|
|
||||||
|
|
||||||
#include "common.h"
|
|
||||||
|
|
||||||
#if defined(BULLDOZER) || defined(PILEDRIVER)
|
|
||||||
#include "sgemv_n_microk_bulldozer.c"
|
|
||||||
#elif defined(HASWELL)
|
|
||||||
#include "sgemv_n_microk_haswell.c"
|
|
||||||
#else
|
|
||||||
#include "sgemv_n_microk_sandy.c"
|
|
||||||
#endif
|
|
||||||
|
|
||||||
static void copy_x(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_src)
|
|
||||||
{
|
|
||||||
BLASLONG i;
|
|
||||||
for ( i=0; i<n; i++ )
|
|
||||||
{
|
|
||||||
*dest = *src;
|
|
||||||
dest++;
|
|
||||||
src += inc_src;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static void add_y(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_dest)
|
|
||||||
{
|
|
||||||
BLASLONG i;
|
|
||||||
for ( i=0; i<n; i++ )
|
|
||||||
{
|
|
||||||
*dest += *src;
|
|
||||||
src++;
|
|
||||||
dest += inc_dest;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer)
|
|
||||||
{
|
|
||||||
BLASLONG i;
|
|
||||||
BLASLONG j;
|
|
||||||
FLOAT *a_ptr;
|
|
||||||
FLOAT *x_ptr;
|
|
||||||
FLOAT *y_ptr;
|
|
||||||
BLASLONG n1;
|
|
||||||
BLASLONG m1;
|
|
||||||
BLASLONG register m2;
|
|
||||||
BLASLONG register n2;
|
|
||||||
FLOAT *xbuffer,*ybuffer;
|
|
||||||
xbuffer = buffer;
|
|
||||||
ybuffer = xbuffer + 2048 + 256;
|
|
||||||
|
|
||||||
n1 = n / 512 ;
|
|
||||||
n2 = n % 512 ;
|
|
||||||
|
|
||||||
m1 = m / 64;
|
|
||||||
m2 = m % 64;
|
|
||||||
|
|
||||||
y_ptr = y;
|
|
||||||
x_ptr = x;
|
|
||||||
|
|
||||||
for (j=0; j<n1; j++)
|
|
||||||
{
|
|
||||||
|
|
||||||
if ( inc_x == 1 )
|
|
||||||
xbuffer = x_ptr;
|
|
||||||
else
|
|
||||||
copy_x(512,x_ptr,xbuffer,inc_x);
|
|
||||||
|
|
||||||
a_ptr = a + j * 512 * lda;
|
|
||||||
y_ptr = y;
|
|
||||||
|
|
||||||
for(i = 0; i<m1; i++ )
|
|
||||||
{
|
|
||||||
sgemv_kernel_64(512,alpha,a_ptr,lda,xbuffer,ybuffer);
|
|
||||||
add_y(64,ybuffer,y_ptr,inc_y);
|
|
||||||
y_ptr += 64 * inc_y;
|
|
||||||
a_ptr += 64;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
if ( m2 & 32 )
|
|
||||||
{
|
|
||||||
sgemv_kernel_32(512,alpha,a_ptr,lda,xbuffer,ybuffer);
|
|
||||||
add_y(32,ybuffer,y_ptr,inc_y);
|
|
||||||
y_ptr += 32 * inc_y;
|
|
||||||
a_ptr += 32;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
if ( m2 & 16 )
|
|
||||||
{
|
|
||||||
sgemv_kernel_16(512,alpha,a_ptr,lda,xbuffer,ybuffer);
|
|
||||||
add_y(16,ybuffer,y_ptr,inc_y);
|
|
||||||
y_ptr += 16 * inc_y;
|
|
||||||
a_ptr += 16;
|
|
||||||
}
|
|
||||||
if ( m2 & 8 )
|
|
||||||
{
|
|
||||||
sgemv_kernel_8(512,alpha,a_ptr,lda,xbuffer,ybuffer);
|
|
||||||
add_y(8,ybuffer,y_ptr,inc_y);
|
|
||||||
y_ptr += 8 * inc_y;
|
|
||||||
a_ptr += 8;
|
|
||||||
}
|
|
||||||
if ( m2 & 4 )
|
|
||||||
{
|
|
||||||
sgemv_kernel_4(512,alpha,a_ptr,lda,xbuffer,ybuffer);
|
|
||||||
add_y(4,ybuffer,y_ptr,inc_y);
|
|
||||||
y_ptr += 4 * inc_y;
|
|
||||||
a_ptr += 4;
|
|
||||||
}
|
|
||||||
if ( m2 & 2 )
|
|
||||||
{
|
|
||||||
sgemv_kernel_2(512,alpha,a_ptr,lda,xbuffer,ybuffer);
|
|
||||||
add_y(2,ybuffer,y_ptr,inc_y);
|
|
||||||
y_ptr += 2 * inc_y;
|
|
||||||
a_ptr += 2;
|
|
||||||
}
|
|
||||||
if ( m2 & 1 )
|
|
||||||
{
|
|
||||||
sgemv_kernel_1(512,alpha,a_ptr,lda,xbuffer,ybuffer);
|
|
||||||
add_y(1,ybuffer,y_ptr,inc_y);
|
|
||||||
}
|
|
||||||
x_ptr += 512 * inc_x;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
if ( n2 > 0 )
|
|
||||||
{
|
|
||||||
|
|
||||||
if ( inc_x == 1 )
|
|
||||||
xbuffer = x_ptr;
|
|
||||||
else
|
|
||||||
copy_x(n2,x_ptr,xbuffer,inc_x);
|
|
||||||
|
|
||||||
a_ptr = a + n1 * 512 * lda;
|
|
||||||
y_ptr = y;
|
|
||||||
|
|
||||||
for(i = 0; i<m1; i++ )
|
|
||||||
{
|
|
||||||
sgemv_kernel_64(n2,alpha,a_ptr,lda,xbuffer,ybuffer);
|
|
||||||
add_y(64,ybuffer,y_ptr,inc_y);
|
|
||||||
y_ptr += 64 * inc_y;
|
|
||||||
a_ptr += 64;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
if ( m2 & 32 )
|
|
||||||
{
|
|
||||||
sgemv_kernel_32(n2,alpha,a_ptr,lda,xbuffer,ybuffer);
|
|
||||||
add_y(32,ybuffer,y_ptr,inc_y);
|
|
||||||
y_ptr += 32 * inc_y;
|
|
||||||
a_ptr += 32;
|
|
||||||
|
|
||||||
}
|
|
||||||
if ( m2 & 16 )
|
|
||||||
{
|
|
||||||
sgemv_kernel_16(n2,alpha,a_ptr,lda,xbuffer,ybuffer);
|
|
||||||
add_y(16,ybuffer,y_ptr,inc_y);
|
|
||||||
y_ptr += 16 * inc_y;
|
|
||||||
a_ptr += 16;
|
|
||||||
}
|
|
||||||
if ( m2 & 8 )
|
|
||||||
{
|
|
||||||
sgemv_kernel_8(n2,alpha,a_ptr,lda,xbuffer,ybuffer);
|
|
||||||
add_y(8,ybuffer,y_ptr,inc_y);
|
|
||||||
y_ptr += 8 * inc_y;
|
|
||||||
a_ptr += 8;
|
|
||||||
}
|
|
||||||
if ( m2 & 4 )
|
|
||||||
{
|
|
||||||
sgemv_kernel_4(n2,alpha,a_ptr,lda,xbuffer,ybuffer);
|
|
||||||
add_y(4,ybuffer,y_ptr,inc_y);
|
|
||||||
y_ptr += 4 * inc_y;
|
|
||||||
a_ptr += 4;
|
|
||||||
}
|
|
||||||
if ( m2 & 2 )
|
|
||||||
{
|
|
||||||
sgemv_kernel_2(n2,alpha,a_ptr,lda,xbuffer,ybuffer);
|
|
||||||
add_y(2,ybuffer,y_ptr,inc_y);
|
|
||||||
y_ptr += 2 * inc_y;
|
|
||||||
a_ptr += 2;
|
|
||||||
}
|
|
||||||
if ( m2 & 1 )
|
|
||||||
{
|
|
||||||
sgemv_kernel_1(n2,alpha,a_ptr,lda,xbuffer,ybuffer);
|
|
||||||
add_y(1,ybuffer,y_ptr,inc_y);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
return(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1,451 +0,0 @@
|
||||||
/***************************************************************************
|
|
||||||
Copyright (c) 2014, The OpenBLAS Project
|
|
||||||
All rights reserved.
|
|
||||||
Redistribution and use in source and binary forms, with or without
|
|
||||||
modification, are permitted provided that the following conditions are
|
|
||||||
met:
|
|
||||||
1. Redistributions of source code must retain the above copyright
|
|
||||||
notice, this list of conditions and the following disclaimer.
|
|
||||||
2. Redistributions in binary form must reproduce the above copyright
|
|
||||||
notice, this list of conditions and the following disclaimer in
|
|
||||||
the documentation and/or other materials provided with the
|
|
||||||
distribution.
|
|
||||||
3. Neither the name of the OpenBLAS project nor the names of
|
|
||||||
its contributors may be used to endorse or promote products
|
|
||||||
derived from this software without specific prior written permission.
|
|
||||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
||||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
||||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
||||||
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
|
|
||||||
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
||||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
||||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
||||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
||||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
|
|
||||||
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
||||||
*****************************************************************************/
|
|
||||||
|
|
||||||
static void sgemv_kernel_64( long n, float alpha, float *a, long lda, float *x, float *y)
|
|
||||||
{
|
|
||||||
|
|
||||||
|
|
||||||
float *pre = a + lda*3;
|
|
||||||
|
|
||||||
__asm__ __volatile__
|
|
||||||
(
|
|
||||||
"movq %0, %%rax\n\t" // n -> rax
|
|
||||||
"vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1
|
|
||||||
"movq %2, %%rsi\n\t" // adress of a -> rsi
|
|
||||||
"movq %3, %%rcx\n\t" // value of lda > rcx
|
|
||||||
"movq %4, %%rdi\n\t" // adress of x -> rdi
|
|
||||||
"movq %5, %%rdx\n\t" // adress of y -> rdx
|
|
||||||
"movq %6, %%r8\n\t" // address for prefetch
|
|
||||||
"prefetcht0 (%%r8)\n\t" // Prefetch
|
|
||||||
"prefetcht0 64(%%r8)\n\t" // Prefetch
|
|
||||||
|
|
||||||
"vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero
|
|
||||||
"vxorps %%ymm9 , %%ymm9 , %%ymm9 \n\t" // set to zero
|
|
||||||
"vxorps %%ymm10, %%ymm10, %%ymm10\n\t" // set to zero
|
|
||||||
"vxorps %%ymm11, %%ymm11, %%ymm11\n\t" // set to zero
|
|
||||||
"vxorps %%ymm12, %%ymm12, %%ymm12\n\t" // set to zero
|
|
||||||
"vxorps %%ymm13, %%ymm13, %%ymm13\n\t" // set to zero
|
|
||||||
"vxorps %%ymm14, %%ymm14, %%ymm14\n\t" // set to zero
|
|
||||||
"vxorps %%ymm15, %%ymm15, %%ymm15\n\t" // set to zero
|
|
||||||
".align 16 \n\t"
|
|
||||||
".L01LOOP%=: \n\t"
|
|
||||||
"vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c
|
|
||||||
"nop \n\t"
|
|
||||||
"leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch
|
|
||||||
|
|
||||||
"prefetcht0 (%%r8)\n\t" // Prefetch
|
|
||||||
"vfmaddps %%ymm8 , 0*4(%%rsi), %%ymm0, %%ymm8 \n\t" // multiply a and c and add to temp
|
|
||||||
"prefetcht0 64(%%r8)\n\t" // Prefetch
|
|
||||||
"vfmaddps %%ymm9 , 8*4(%%rsi), %%ymm0, %%ymm9 \n\t" // multiply a and c and add to temp
|
|
||||||
"prefetcht0 128(%%r8)\n\t" // Prefetch
|
|
||||||
"vfmaddps %%ymm10, 16*4(%%rsi), %%ymm0, %%ymm10\n\t" // multiply a and c and add to temp
|
|
||||||
"vfmaddps %%ymm11, 24*4(%%rsi), %%ymm0, %%ymm11\n\t" // multiply a and c and add to temp
|
|
||||||
"prefetcht0 192(%%r8)\n\t" // Prefetch
|
|
||||||
"vfmaddps %%ymm12, 32*4(%%rsi), %%ymm0, %%ymm12\n\t" // multiply a and c and add to temp
|
|
||||||
"vfmaddps %%ymm13, 40*4(%%rsi), %%ymm0, %%ymm13\n\t" // multiply a and c and add to temp
|
|
||||||
"vfmaddps %%ymm14, 48*4(%%rsi), %%ymm0, %%ymm14\n\t" // multiply a and c and add to temp
|
|
||||||
"vfmaddps %%ymm15, 56*4(%%rsi), %%ymm0, %%ymm15\n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"addq $4 , %%rdi \n\t" // increment pointer of c
|
|
||||||
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
|
|
||||||
|
|
||||||
"dec %%rax \n\t" // n = n -1
|
|
||||||
"jnz .L01LOOP%= \n\t"
|
|
||||||
|
|
||||||
"vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm9 , %%ymm1, %%ymm9 \n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm10, %%ymm1, %%ymm10\n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm11, %%ymm1, %%ymm11\n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm12, %%ymm1, %%ymm12\n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm13, %%ymm1, %%ymm13\n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm14, %%ymm1, %%ymm14\n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm15, %%ymm1, %%ymm15\n\t" // scale by alpha
|
|
||||||
|
|
||||||
"vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm9 , 8*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm10, 16*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm11, 24*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm12, 32*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm13, 40*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm14, 48*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm15, 56*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
|
|
||||||
:
|
|
||||||
:
|
|
||||||
"m" (n), // 0
|
|
||||||
"m" (alpha), // 1
|
|
||||||
"m" (a), // 2
|
|
||||||
"m" (lda), // 3
|
|
||||||
"m" (x), // 4
|
|
||||||
"m" (y), // 5
|
|
||||||
"m" (pre) // 6
|
|
||||||
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
|
|
||||||
"%xmm0", "%xmm1",
|
|
||||||
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
|
|
||||||
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
|
|
||||||
"memory"
|
|
||||||
);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
static void sgemv_kernel_32( long n, float alpha, float *a, long lda, float *x, float *y)
|
|
||||||
{
|
|
||||||
|
|
||||||
|
|
||||||
float *pre = a + lda*3;
|
|
||||||
|
|
||||||
__asm__ __volatile__
|
|
||||||
(
|
|
||||||
"movq %0, %%rax\n\t" // n -> rax
|
|
||||||
"vbroadcastss %1, %%xmm1\n\t" // alpha -> xmm1
|
|
||||||
"movq %2, %%rsi\n\t" // adress of a -> rsi
|
|
||||||
"movq %3, %%rcx\n\t" // value of lda > rcx
|
|
||||||
"movq %4, %%rdi\n\t" // adress of x -> rdi
|
|
||||||
"movq %5, %%rdx\n\t" // adress of y -> rdx
|
|
||||||
"movq %6, %%r8\n\t" // address for prefetch
|
|
||||||
"prefetcht0 (%%r8)\n\t" // Prefetch
|
|
||||||
"prefetcht0 64(%%r8)\n\t" // Prefetch
|
|
||||||
|
|
||||||
"vxorps %%xmm8 , %%xmm8 , %%xmm8 \n\t" // set to zero
|
|
||||||
"vxorps %%xmm9 , %%xmm9 , %%xmm9 \n\t" // set to zero
|
|
||||||
"vxorps %%xmm10, %%xmm10, %%xmm10\n\t" // set to zero
|
|
||||||
"vxorps %%xmm11, %%xmm11, %%xmm11\n\t" // set to zero
|
|
||||||
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
|
|
||||||
"vxorps %%xmm13, %%xmm13, %%xmm13\n\t" // set to zero
|
|
||||||
"vxorps %%xmm14, %%xmm14, %%xmm14\n\t" // set to zero
|
|
||||||
"vxorps %%xmm15, %%xmm15, %%xmm15\n\t" // set to zero
|
|
||||||
".align 16 \n\t"
|
|
||||||
".L01LOOP%=: \n\t"
|
|
||||||
"vbroadcastss (%%rdi), %%xmm0 \n\t" // load values of c
|
|
||||||
"nop \n\t"
|
|
||||||
"leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch
|
|
||||||
|
|
||||||
"prefetcht0 (%%r8)\n\t" // Prefetch
|
|
||||||
"vfmaddps %%xmm8 , 0*4(%%rsi), %%xmm0, %%xmm8 \n\t" // multiply a and c and add to temp
|
|
||||||
"prefetcht0 64(%%r8)\n\t" // Prefetch
|
|
||||||
"vfmaddps %%xmm9 , 4*4(%%rsi), %%xmm0, %%xmm9 \n\t" // multiply a and c and add to temp
|
|
||||||
"vfmaddps %%xmm10, 8*4(%%rsi), %%xmm0, %%xmm10\n\t" // multiply a and c and add to temp
|
|
||||||
"vfmaddps %%xmm11, 12*4(%%rsi), %%xmm0, %%xmm11\n\t" // multiply a and c and add to temp
|
|
||||||
"vfmaddps %%xmm12, 16*4(%%rsi), %%xmm0, %%xmm12\n\t" // multiply a and c and add to temp
|
|
||||||
"vfmaddps %%xmm13, 20*4(%%rsi), %%xmm0, %%xmm13\n\t" // multiply a and c and add to temp
|
|
||||||
"vfmaddps %%xmm14, 24*4(%%rsi), %%xmm0, %%xmm14\n\t" // multiply a and c and add to temp
|
|
||||||
"vfmaddps %%xmm15, 28*4(%%rsi), %%xmm0, %%xmm15\n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"addq $4 , %%rdi \n\t" // increment pointer of c
|
|
||||||
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
|
|
||||||
|
|
||||||
"dec %%rax \n\t" // n = n -1
|
|
||||||
"jnz .L01LOOP%= \n\t"
|
|
||||||
|
|
||||||
"vmulps %%xmm8 , %%xmm1, %%xmm8 \n\t" // scale by alpha
|
|
||||||
"vmulps %%xmm9 , %%xmm1, %%xmm9 \n\t" // scale by alpha
|
|
||||||
"vmulps %%xmm10, %%xmm1, %%xmm10\n\t" // scale by alpha
|
|
||||||
"vmulps %%xmm11, %%xmm1, %%xmm11\n\t" // scale by alpha
|
|
||||||
"vmulps %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha
|
|
||||||
"vmulps %%xmm13, %%xmm1, %%xmm13\n\t" // scale by alpha
|
|
||||||
"vmulps %%xmm14, %%xmm1, %%xmm14\n\t" // scale by alpha
|
|
||||||
"vmulps %%xmm15, %%xmm1, %%xmm15\n\t" // scale by alpha
|
|
||||||
|
|
||||||
"vmovups %%xmm8 , (%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%xmm9 , 4*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%xmm10, 8*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%xmm11, 12*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%xmm12, 16*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%xmm13, 20*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%xmm14, 24*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%xmm15, 28*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
|
|
||||||
:
|
|
||||||
:
|
|
||||||
"m" (n), // 0
|
|
||||||
"m" (alpha), // 1
|
|
||||||
"m" (a), // 2
|
|
||||||
"m" (lda), // 3
|
|
||||||
"m" (x), // 4
|
|
||||||
"m" (y), // 5
|
|
||||||
"m" (pre) // 6
|
|
||||||
);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
static void sgemv_kernel_16( long n, float alpha, float *a, long lda, float *x, float *y)
|
|
||||||
{
|
|
||||||
|
|
||||||
float *pre = a + lda*3;
|
|
||||||
|
|
||||||
__asm__ __volatile__
|
|
||||||
(
|
|
||||||
"movq %0, %%rax\n\t" // n -> rax
|
|
||||||
"vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1
|
|
||||||
"movq %2, %%rsi\n\t" // adress of a -> rsi
|
|
||||||
"movq %3, %%rcx\n\t" // value of lda > rcx
|
|
||||||
"movq %4, %%rdi\n\t" // adress of x -> rdi
|
|
||||||
"movq %5, %%rdx\n\t" // adress of y -> rdx
|
|
||||||
"movq %6, %%r8\n\t" // address for prefetch
|
|
||||||
"prefetcht0 (%%r8)\n\t" // Prefetch
|
|
||||||
|
|
||||||
"vxorps %%ymm12, %%ymm12, %%ymm12\n\t" // set to zero
|
|
||||||
"vxorps %%ymm13, %%ymm13, %%ymm13\n\t" // set to zero
|
|
||||||
|
|
||||||
".L01LOOP%=: \n\t"
|
|
||||||
"vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c
|
|
||||||
"addq $4 , %%rdi \n\t" // increment pointer of c
|
|
||||||
|
|
||||||
"leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch
|
|
||||||
"prefetcht0 (%%r8)\n\t" // Prefetch
|
|
||||||
|
|
||||||
"vfmaddps %%ymm12, 0*4(%%rsi), %%ymm0, %%ymm12\n\t" // multiply a and c and add to temp
|
|
||||||
"vfmaddps %%ymm13, 8*4(%%rsi), %%ymm0, %%ymm13\n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
|
|
||||||
|
|
||||||
"dec %%rax \n\t" // n = n -1
|
|
||||||
"jnz .L01LOOP%= \n\t"
|
|
||||||
|
|
||||||
"vmulps %%ymm12, %%ymm1, %%ymm12\n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm13, %%ymm1, %%ymm13\n\t" // scale by alpha
|
|
||||||
|
|
||||||
"vmovups %%ymm12, (%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm13, 8*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
|
|
||||||
:
|
|
||||||
:
|
|
||||||
"m" (n), // 0
|
|
||||||
"m" (alpha), // 1
|
|
||||||
"m" (a), // 2
|
|
||||||
"m" (lda), // 3
|
|
||||||
"m" (x), // 4
|
|
||||||
"m" (y), // 5
|
|
||||||
"m" (pre) // 6
|
|
||||||
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
|
|
||||||
"%xmm0", "%xmm1",
|
|
||||||
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
|
|
||||||
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
|
|
||||||
"memory"
|
|
||||||
);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
static void sgemv_kernel_8( long n, float alpha, float *a, long lda, float *x, float *y)
|
|
||||||
{
|
|
||||||
|
|
||||||
|
|
||||||
__asm__ __volatile__
|
|
||||||
(
|
|
||||||
"movq %0, %%rax\n\t" // n -> rax
|
|
||||||
"vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1
|
|
||||||
"movq %2, %%rsi\n\t" // adress of a -> rsi
|
|
||||||
"movq %3, %%rcx\n\t" // value of lda > rcx
|
|
||||||
"movq %4, %%rdi\n\t" // adress of x -> rdi
|
|
||||||
"movq %5, %%rdx\n\t" // adress of y -> rdx
|
|
||||||
|
|
||||||
"vxorps %%ymm12, %%ymm12, %%ymm12\n\t" // set to zero
|
|
||||||
|
|
||||||
".L01LOOP%=: \n\t"
|
|
||||||
"vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c
|
|
||||||
"addq $4 , %%rdi \n\t" // increment pointer of c
|
|
||||||
|
|
||||||
"vfmaddps %%ymm12, 0*4(%%rsi), %%ymm0, %%ymm12\n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
|
|
||||||
|
|
||||||
"dec %%rax \n\t" // n = n -1
|
|
||||||
"jnz .L01LOOP%= \n\t"
|
|
||||||
|
|
||||||
"vmulps %%ymm12, %%ymm1, %%ymm12\n\t" // scale by alpha
|
|
||||||
|
|
||||||
"vmovups %%ymm12, (%%rdx) \n\t" // store temp -> y
|
|
||||||
|
|
||||||
:
|
|
||||||
:
|
|
||||||
"m" (n), // 0
|
|
||||||
"m" (alpha), // 1
|
|
||||||
"m" (a), // 2
|
|
||||||
"m" (lda), // 3
|
|
||||||
"m" (x), // 4
|
|
||||||
"m" (y) // 5
|
|
||||||
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
|
|
||||||
"%xmm0", "%xmm1",
|
|
||||||
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
|
|
||||||
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
|
|
||||||
"memory"
|
|
||||||
);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
static void sgemv_kernel_4( long n, float alpha, float *a, long lda, float *x, float *y)
|
|
||||||
{
|
|
||||||
|
|
||||||
|
|
||||||
__asm__ __volatile__
|
|
||||||
(
|
|
||||||
"movq %0, %%rax\n\t" // n -> rax
|
|
||||||
"vbroadcastss %1, %%xmm1\n\t" // alpha -> xmm1
|
|
||||||
"movq %2, %%rsi\n\t" // adress of a -> rsi
|
|
||||||
"movq %3, %%rcx\n\t" // value of lda > rcx
|
|
||||||
"movq %4, %%rdi\n\t" // adress of x -> rdi
|
|
||||||
"movq %5, %%rdx\n\t" // adress of y -> rdx
|
|
||||||
|
|
||||||
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
|
|
||||||
|
|
||||||
".L01LOOP%=: \n\t"
|
|
||||||
"vbroadcastss (%%rdi), %%xmm0 \n\t" // load values of c
|
|
||||||
"addq $4 , %%rdi \n\t" // increment pointer of c
|
|
||||||
|
|
||||||
"vfmaddps %%xmm12, 0*4(%%rsi), %%xmm0, %%xmm12\n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
|
|
||||||
|
|
||||||
"dec %%rax \n\t" // n = n -1
|
|
||||||
"jnz .L01LOOP%= \n\t"
|
|
||||||
|
|
||||||
"vmulps %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha
|
|
||||||
|
|
||||||
"vmovups %%xmm12, (%%rdx) \n\t" // store temp -> y
|
|
||||||
|
|
||||||
:
|
|
||||||
:
|
|
||||||
"m" (n), // 0
|
|
||||||
"m" (alpha), // 1
|
|
||||||
"m" (a), // 2
|
|
||||||
"m" (lda), // 3
|
|
||||||
"m" (x), // 4
|
|
||||||
"m" (y) // 5
|
|
||||||
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
|
|
||||||
"%xmm0", "%xmm1",
|
|
||||||
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
|
|
||||||
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
|
|
||||||
"memory"
|
|
||||||
);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
static void sgemv_kernel_2( long n, float alpha, float *a, long lda, float *x, float *y)
|
|
||||||
{
|
|
||||||
|
|
||||||
|
|
||||||
__asm__ __volatile__
|
|
||||||
(
|
|
||||||
"movq %0, %%rax\n\t" // n -> rax
|
|
||||||
"vmovss %1, %%xmm1\n\t" // alpha -> xmm1
|
|
||||||
"movq %2, %%rsi\n\t" // adress of a -> rsi
|
|
||||||
"movq %3, %%rcx\n\t" // value of lda > rcx
|
|
||||||
"movq %4, %%rdi\n\t" // adress of x -> rdi
|
|
||||||
"movq %5, %%rdx\n\t" // adress of y -> rdx
|
|
||||||
|
|
||||||
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
|
|
||||||
"vxorps %%xmm13, %%xmm13, %%xmm13\n\t" // set to zero
|
|
||||||
|
|
||||||
".L01LOOP%=: \n\t"
|
|
||||||
"vmovss (%%rdi), %%xmm0 \n\t" // load values of c
|
|
||||||
"addq $4 , %%rdi \n\t" // increment pointer of c
|
|
||||||
|
|
||||||
"vfmaddss %%xmm12, 0*4(%%rsi), %%xmm0, %%xmm12\n\t" // multiply a and c and add to temp
|
|
||||||
"vfmaddss %%xmm13, 1*4(%%rsi), %%xmm0, %%xmm13\n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
|
|
||||||
|
|
||||||
"dec %%rax \n\t" // n = n -1
|
|
||||||
"jnz .L01LOOP%= \n\t"
|
|
||||||
|
|
||||||
"vmulss %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha
|
|
||||||
"vmulss %%xmm13, %%xmm1, %%xmm13\n\t" // scale by alpha
|
|
||||||
|
|
||||||
"vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovss %%xmm13, 4(%%rdx) \n\t" // store temp -> y
|
|
||||||
|
|
||||||
:
|
|
||||||
:
|
|
||||||
"m" (n), // 0
|
|
||||||
"m" (alpha), // 1
|
|
||||||
"m" (a), // 2
|
|
||||||
"m" (lda), // 3
|
|
||||||
"m" (x), // 4
|
|
||||||
"m" (y) // 5
|
|
||||||
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
|
|
||||||
"%xmm0", "%xmm1",
|
|
||||||
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
|
|
||||||
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
|
|
||||||
"memory"
|
|
||||||
);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
static void sgemv_kernel_1( long n, float alpha, float *a, long lda, float *x, float *y)
|
|
||||||
{
|
|
||||||
|
|
||||||
|
|
||||||
__asm__ __volatile__
|
|
||||||
(
|
|
||||||
"movq %0, %%rax\n\t" // n -> rax
|
|
||||||
"vmovss %1, %%xmm1\n\t" // alpha -> xmm1
|
|
||||||
"movq %2, %%rsi\n\t" // adress of a -> rsi
|
|
||||||
"movq %3, %%rcx\n\t" // value of lda > rcx
|
|
||||||
"movq %4, %%rdi\n\t" // adress of x -> rdi
|
|
||||||
"movq %5, %%rdx\n\t" // adress of y -> rdx
|
|
||||||
|
|
||||||
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
|
|
||||||
|
|
||||||
".L01LOOP%=: \n\t"
|
|
||||||
"vmovss (%%rdi), %%xmm0 \n\t" // load values of c
|
|
||||||
"addq $4 , %%rdi \n\t" // increment pointer of c
|
|
||||||
|
|
||||||
"vfmaddss %%xmm12, 0*4(%%rsi), %%xmm0, %%xmm12\n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
|
|
||||||
|
|
||||||
"dec %%rax \n\t" // n = n -1
|
|
||||||
"jnz .L01LOOP%= \n\t"
|
|
||||||
|
|
||||||
"vmulss %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha
|
|
||||||
|
|
||||||
"vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y
|
|
||||||
|
|
||||||
:
|
|
||||||
:
|
|
||||||
"m" (n), // 0
|
|
||||||
"m" (alpha), // 1
|
|
||||||
"m" (a), // 2
|
|
||||||
"m" (lda), // 3
|
|
||||||
"m" (x), // 4
|
|
||||||
"m" (y) // 5
|
|
||||||
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
|
|
||||||
"%xmm0", "%xmm1",
|
|
||||||
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
|
|
||||||
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
|
|
||||||
"memory"
|
|
||||||
);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1,461 +0,0 @@
|
||||||
/***************************************************************************
|
|
||||||
Copyright (c) 2014, The OpenBLAS Project
|
|
||||||
All rights reserved.
|
|
||||||
Redistribution and use in source and binary forms, with or without
|
|
||||||
modification, are permitted provided that the following conditions are
|
|
||||||
met:
|
|
||||||
1. Redistributions of source code must retain the above copyright
|
|
||||||
notice, this list of conditions and the following disclaimer.
|
|
||||||
2. Redistributions in binary form must reproduce the above copyright
|
|
||||||
notice, this list of conditions and the following disclaimer in
|
|
||||||
the documentation and/or other materials provided with the
|
|
||||||
distribution.
|
|
||||||
3. Neither the name of the OpenBLAS project nor the names of
|
|
||||||
its contributors may be used to endorse or promote products
|
|
||||||
derived from this software without specific prior written permission.
|
|
||||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
||||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
||||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
||||||
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
|
|
||||||
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
||||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
||||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
||||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
||||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
|
|
||||||
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
||||||
*****************************************************************************/
|
|
||||||
|
|
||||||
static void sgemv_kernel_64( long n, float alpha, float *a, long lda, float *x, float *y)
|
|
||||||
{
|
|
||||||
|
|
||||||
|
|
||||||
float *pre = a + lda*2;
|
|
||||||
|
|
||||||
__asm__ __volatile__
|
|
||||||
(
|
|
||||||
"movq %0, %%rax\n\t" // n -> rax
|
|
||||||
"vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1
|
|
||||||
"movq %2, %%rsi\n\t" // adress of a -> rsi
|
|
||||||
"movq %3, %%rcx\n\t" // value of lda > rcx
|
|
||||||
"movq %4, %%rdi\n\t" // adress of x -> rdi
|
|
||||||
"movq %5, %%rdx\n\t" // adress of y -> rdx
|
|
||||||
"movq %6, %%r8\n\t" // address for prefetch
|
|
||||||
"prefetcht0 (%%r8)\n\t" // Prefetch
|
|
||||||
"prefetcht0 64(%%r8)\n\t" // Prefetch
|
|
||||||
|
|
||||||
"vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero
|
|
||||||
"vxorps %%ymm9 , %%ymm9 , %%ymm9 \n\t" // set to zero
|
|
||||||
"vxorps %%ymm10, %%ymm10, %%ymm10\n\t" // set to zero
|
|
||||||
"vxorps %%ymm11, %%ymm11, %%ymm11\n\t" // set to zero
|
|
||||||
"vxorps %%ymm12, %%ymm12, %%ymm12\n\t" // set to zero
|
|
||||||
"vxorps %%ymm13, %%ymm13, %%ymm13\n\t" // set to zero
|
|
||||||
"vxorps %%ymm14, %%ymm14, %%ymm14\n\t" // set to zero
|
|
||||||
"vxorps %%ymm15, %%ymm15, %%ymm15\n\t" // set to zero
|
|
||||||
".align 16 \n\t"
|
|
||||||
".L01LOOP%=: \n\t"
|
|
||||||
"vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c
|
|
||||||
"leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch
|
|
||||||
|
|
||||||
"prefetcht0 (%%r8)\n\t" // Prefetch
|
|
||||||
"vfmadd231ps 0*4(%%rsi), %%ymm0, %%ymm8 \n\t" // multiply a and c and add to temp
|
|
||||||
"vfmadd231ps 8*4(%%rsi), %%ymm0, %%ymm9 \n\t" // multiply a and c and add to temp
|
|
||||||
"prefetcht0 64(%%r8)\n\t" // Prefetch
|
|
||||||
"vfmadd231ps 16*4(%%rsi), %%ymm0, %%ymm10\n\t" // multiply a and c and add to temp
|
|
||||||
"vfmadd231ps 24*4(%%rsi), %%ymm0, %%ymm11\n\t" // multiply a and c and add to temp
|
|
||||||
"prefetcht0 128(%%r8)\n\t" // Prefetch
|
|
||||||
"vfmadd231ps 32*4(%%rsi), %%ymm0, %%ymm12\n\t" // multiply a and c and add to temp
|
|
||||||
"vfmadd231ps 40*4(%%rsi), %%ymm0, %%ymm13\n\t" // multiply a and c and add to temp
|
|
||||||
"prefetcht0 192(%%r8)\n\t" // Prefetch
|
|
||||||
"vfmadd231ps 48*4(%%rsi), %%ymm0, %%ymm14\n\t" // multiply a and c and add to temp
|
|
||||||
"vfmadd231ps 56*4(%%rsi), %%ymm0, %%ymm15\n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"addq $4 , %%rdi \n\t" // increment pointer of c
|
|
||||||
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
|
|
||||||
|
|
||||||
"dec %%rax \n\t" // n = n -1
|
|
||||||
"jnz .L01LOOP%= \n\t"
|
|
||||||
|
|
||||||
"vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm9 , %%ymm1, %%ymm9 \n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm10, %%ymm1, %%ymm10\n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm11, %%ymm1, %%ymm11\n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm12, %%ymm1, %%ymm12\n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm13, %%ymm1, %%ymm13\n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm14, %%ymm1, %%ymm14\n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm15, %%ymm1, %%ymm15\n\t" // scale by alpha
|
|
||||||
|
|
||||||
"vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm9 , 8*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm10, 16*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm11, 24*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm12, 32*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm13, 40*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm14, 48*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm15, 56*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
|
|
||||||
:
|
|
||||||
:
|
|
||||||
"m" (n), // 0
|
|
||||||
"m" (alpha), // 1
|
|
||||||
"m" (a), // 2
|
|
||||||
"m" (lda), // 3
|
|
||||||
"m" (x), // 4
|
|
||||||
"m" (y), // 5
|
|
||||||
"m" (pre) // 6
|
|
||||||
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc",
|
|
||||||
"%xmm0", "%xmm1",
|
|
||||||
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
|
|
||||||
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
|
|
||||||
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
|
|
||||||
"memory"
|
|
||||||
);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
static void sgemv_kernel_32( long n, float alpha, float *a, long lda, float *x, float *y)
|
|
||||||
{
|
|
||||||
|
|
||||||
|
|
||||||
float *pre = a + lda*3;
|
|
||||||
|
|
||||||
__asm__ __volatile__
|
|
||||||
(
|
|
||||||
"movq %0, %%rax\n\t" // n -> rax
|
|
||||||
"vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1
|
|
||||||
"movq %2, %%rsi\n\t" // adress of a -> rsi
|
|
||||||
"movq %3, %%rcx\n\t" // value of lda > rcx
|
|
||||||
"movq %4, %%rdi\n\t" // adress of x -> rdi
|
|
||||||
"movq %5, %%rdx\n\t" // adress of y -> rdx
|
|
||||||
"movq %6, %%r8\n\t" // address for prefetch
|
|
||||||
"prefetcht0 (%%r8)\n\t" // Prefetch
|
|
||||||
"prefetcht0 64(%%r8)\n\t" // Prefetch
|
|
||||||
|
|
||||||
"vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero
|
|
||||||
"vxorps %%ymm9 , %%ymm9 , %%ymm9 \n\t" // set to zero
|
|
||||||
"vxorps %%ymm10, %%ymm10, %%ymm10\n\t" // set to zero
|
|
||||||
"vxorps %%ymm11, %%ymm11, %%ymm11\n\t" // set to zero
|
|
||||||
".align 16 \n\t"
|
|
||||||
".L01LOOP%=: \n\t"
|
|
||||||
"vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c
|
|
||||||
"nop \n\t"
|
|
||||||
"leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch
|
|
||||||
|
|
||||||
"prefetcht0 (%%r8)\n\t" // Prefetch
|
|
||||||
"prefetcht0 64(%%r8)\n\t" // Prefetch
|
|
||||||
|
|
||||||
"vmulps 0*4(%%rsi), %%ymm0, %%ymm4 \n\t" // multiply a and c and add to temp
|
|
||||||
"vmulps 8*4(%%rsi), %%ymm0, %%ymm5 \n\t" // multiply a and c and add to temp
|
|
||||||
"vmulps 16*4(%%rsi), %%ymm0, %%ymm6 \n\t" // multiply a and c and add to temp
|
|
||||||
"vmulps 24*4(%%rsi), %%ymm0, %%ymm7 \n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"vaddps %%ymm8 , %%ymm4, %%ymm8 \n\t" // multiply a and c and add to temp
|
|
||||||
"vaddps %%ymm9 , %%ymm5, %%ymm9 \n\t" // multiply a and c and add to temp
|
|
||||||
"vaddps %%ymm10, %%ymm6, %%ymm10\n\t" // multiply a and c and add to temp
|
|
||||||
"vaddps %%ymm11, %%ymm7, %%ymm11\n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
"addq $4 , %%rdi \n\t" // increment pointer of c
|
|
||||||
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
|
|
||||||
|
|
||||||
"dec %%rax \n\t" // n = n -1
|
|
||||||
"jnz .L01LOOP%= \n\t"
|
|
||||||
|
|
||||||
"vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm9 , %%ymm1, %%ymm9 \n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm10, %%ymm1, %%ymm10\n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm11, %%ymm1, %%ymm11\n\t" // scale by alpha
|
|
||||||
|
|
||||||
"vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm9 , 8*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm10, 16*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm11, 24*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
|
|
||||||
:
|
|
||||||
:
|
|
||||||
"m" (n), // 0
|
|
||||||
"m" (alpha), // 1
|
|
||||||
"m" (a), // 2
|
|
||||||
"m" (lda), // 3
|
|
||||||
"m" (x), // 4
|
|
||||||
"m" (y), // 5
|
|
||||||
"m" (pre) // 6
|
|
||||||
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc",
|
|
||||||
"%xmm0", "%xmm1",
|
|
||||||
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
|
|
||||||
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
|
|
||||||
"memory"
|
|
||||||
);
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
static void sgemv_kernel_16( long n, float alpha, float *a, long lda, float *x, float *y)
|
|
||||||
{
|
|
||||||
|
|
||||||
float *pre = a + lda*3;
|
|
||||||
|
|
||||||
__asm__ __volatile__
|
|
||||||
(
|
|
||||||
"movq %0, %%rax\n\t" // n -> rax
|
|
||||||
"vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1
|
|
||||||
"movq %2, %%rsi\n\t" // adress of a -> rsi
|
|
||||||
"movq %3, %%rcx\n\t" // value of lda > rcx
|
|
||||||
"movq %4, %%rdi\n\t" // adress of x -> rdi
|
|
||||||
"movq %5, %%rdx\n\t" // adress of y -> rdx
|
|
||||||
"movq %6, %%r8\n\t" // address for prefetch
|
|
||||||
"prefetcht0 (%%r8)\n\t" // Prefetch
|
|
||||||
"prefetcht0 64(%%r8)\n\t" // Prefetch
|
|
||||||
|
|
||||||
"vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero
|
|
||||||
"vxorps %%ymm9 , %%ymm9 , %%ymm9 \n\t" // set to zero
|
|
||||||
".align 16 \n\t"
|
|
||||||
".L01LOOP%=: \n\t"
|
|
||||||
"vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c
|
|
||||||
"nop \n\t"
|
|
||||||
"leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch
|
|
||||||
|
|
||||||
"prefetcht0 (%%r8)\n\t" // Prefetch
|
|
||||||
|
|
||||||
"vmulps 0*4(%%rsi), %%ymm0, %%ymm4 \n\t" // multiply a and c and add to temp
|
|
||||||
"vmulps 8*4(%%rsi), %%ymm0, %%ymm5 \n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"vaddps %%ymm8 , %%ymm4, %%ymm8 \n\t" // multiply a and c and add to temp
|
|
||||||
"vaddps %%ymm9 , %%ymm5, %%ymm9 \n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"addq $4 , %%rdi \n\t" // increment pointer of c
|
|
||||||
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
|
|
||||||
|
|
||||||
"dec %%rax \n\t" // n = n -1
|
|
||||||
"jnz .L01LOOP%= \n\t"
|
|
||||||
|
|
||||||
"vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm9 , %%ymm1, %%ymm9 \n\t" // scale by alpha
|
|
||||||
|
|
||||||
"vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm9 , 8*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
|
|
||||||
:
|
|
||||||
:
|
|
||||||
"m" (n), // 0
|
|
||||||
"m" (alpha), // 1
|
|
||||||
"m" (a), // 2
|
|
||||||
"m" (lda), // 3
|
|
||||||
"m" (x), // 4
|
|
||||||
"m" (y), // 5
|
|
||||||
"m" (pre) // 6
|
|
||||||
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc",
|
|
||||||
"%xmm0", "%xmm1",
|
|
||||||
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
|
|
||||||
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
|
|
||||||
"memory"
|
|
||||||
);
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
static void sgemv_kernel_8( long n, float alpha, float *a, long lda, float *x, float *y)
|
|
||||||
{
|
|
||||||
|
|
||||||
__asm__ __volatile__
|
|
||||||
(
|
|
||||||
"movq %0, %%rax\n\t" // n -> rax
|
|
||||||
"vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1
|
|
||||||
"movq %2, %%rsi\n\t" // adress of a -> rsi
|
|
||||||
"movq %3, %%rcx\n\t" // value of lda > rcx
|
|
||||||
"movq %4, %%rdi\n\t" // adress of x -> rdi
|
|
||||||
"movq %5, %%rdx\n\t" // adress of y -> rdx
|
|
||||||
|
|
||||||
"vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero
|
|
||||||
".align 16 \n\t"
|
|
||||||
".L01LOOP%=: \n\t"
|
|
||||||
"vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c
|
|
||||||
|
|
||||||
"vmulps 0*4(%%rsi), %%ymm0, %%ymm4 \n\t" // multiply a and c and add to temp
|
|
||||||
"vaddps %%ymm8 , %%ymm4, %%ymm8 \n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"addq $4 , %%rdi \n\t" // increment pointer of c
|
|
||||||
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
|
|
||||||
|
|
||||||
"dec %%rax \n\t" // n = n -1
|
|
||||||
"jnz .L01LOOP%= \n\t"
|
|
||||||
|
|
||||||
"vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha
|
|
||||||
"vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y
|
|
||||||
|
|
||||||
:
|
|
||||||
:
|
|
||||||
"m" (n), // 0
|
|
||||||
"m" (alpha), // 1
|
|
||||||
"m" (a), // 2
|
|
||||||
"m" (lda), // 3
|
|
||||||
"m" (x), // 4
|
|
||||||
"m" (y) // 5
|
|
||||||
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc",
|
|
||||||
"%xmm0", "%xmm1",
|
|
||||||
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
|
|
||||||
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
|
|
||||||
"memory"
|
|
||||||
);
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
static void sgemv_kernel_4( long n, float alpha, float *a, long lda, float *x, float *y)
|
|
||||||
{
|
|
||||||
|
|
||||||
|
|
||||||
__asm__ __volatile__
|
|
||||||
(
|
|
||||||
"movq %0, %%rax\n\t" // n -> rax
|
|
||||||
"vbroadcastss %1, %%xmm1\n\t" // alpha -> xmm1
|
|
||||||
"movq %2, %%rsi\n\t" // adress of a -> rsi
|
|
||||||
"movq %3, %%rcx\n\t" // value of lda > rcx
|
|
||||||
"movq %4, %%rdi\n\t" // adress of x -> rdi
|
|
||||||
"movq %5, %%rdx\n\t" // adress of y -> rdx
|
|
||||||
|
|
||||||
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
|
|
||||||
|
|
||||||
".L01LOOP%=: \n\t"
|
|
||||||
"vbroadcastss (%%rdi), %%xmm0 \n\t" // load values of c
|
|
||||||
|
|
||||||
"vmulps 0*4(%%rsi), %%xmm0, %%xmm4 \n\t" // multiply a and c and add to temp
|
|
||||||
"vaddps %%xmm12, %%xmm4, %%xmm12 \n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"addq $4 , %%rdi \n\t" // increment pointer of c
|
|
||||||
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
|
|
||||||
|
|
||||||
"dec %%rax \n\t" // n = n -1
|
|
||||||
"jnz .L01LOOP%= \n\t"
|
|
||||||
|
|
||||||
"vmulps %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha
|
|
||||||
|
|
||||||
"vmovups %%xmm12, (%%rdx) \n\t" // store temp -> y
|
|
||||||
|
|
||||||
:
|
|
||||||
:
|
|
||||||
"m" (n), // 0
|
|
||||||
"m" (alpha), // 1
|
|
||||||
"m" (a), // 2
|
|
||||||
"m" (lda), // 3
|
|
||||||
"m" (x), // 4
|
|
||||||
"m" (y) // 5
|
|
||||||
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
|
|
||||||
"%xmm0", "%xmm1",
|
|
||||||
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
|
|
||||||
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
|
|
||||||
"memory"
|
|
||||||
);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
static void sgemv_kernel_2( long n, float alpha, float *a, long lda, float *x, float *y)
|
|
||||||
{
|
|
||||||
|
|
||||||
|
|
||||||
__asm__ __volatile__
|
|
||||||
(
|
|
||||||
"movq %0, %%rax\n\t" // n -> rax
|
|
||||||
"vmovss %1, %%xmm1\n\t" // alpha -> xmm1
|
|
||||||
"movq %2, %%rsi\n\t" // adress of a -> rsi
|
|
||||||
"movq %3, %%rcx\n\t" // value of lda > rcx
|
|
||||||
"movq %4, %%rdi\n\t" // adress of x -> rdi
|
|
||||||
"movq %5, %%rdx\n\t" // adress of y -> rdx
|
|
||||||
|
|
||||||
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
|
|
||||||
"vxorps %%xmm13, %%xmm13, %%xmm13\n\t" // set to zero
|
|
||||||
|
|
||||||
".L01LOOP%=: \n\t"
|
|
||||||
"vmovss (%%rdi), %%xmm0 \n\t" // load values of c
|
|
||||||
|
|
||||||
"vmulps 0*4(%%rsi), %%xmm0, %%xmm4 \n\t" // multiply a and c and add to temp
|
|
||||||
"vmulps 1*4(%%rsi), %%xmm0, %%xmm5 \n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"vaddps %%xmm12, %%xmm4, %%xmm12 \n\t" // multiply a and c and add to temp
|
|
||||||
"vaddps %%xmm13, %%xmm5, %%xmm13 \n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"addq $4 , %%rdi \n\t" // increment pointer of c
|
|
||||||
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
|
|
||||||
|
|
||||||
"dec %%rax \n\t" // n = n -1
|
|
||||||
"jnz .L01LOOP%= \n\t"
|
|
||||||
|
|
||||||
"vmulss %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha
|
|
||||||
"vmulss %%xmm13, %%xmm1, %%xmm13\n\t" // scale by alpha
|
|
||||||
|
|
||||||
"vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovss %%xmm13, 4(%%rdx) \n\t" // store temp -> y
|
|
||||||
|
|
||||||
:
|
|
||||||
:
|
|
||||||
"m" (n), // 0
|
|
||||||
"m" (alpha), // 1
|
|
||||||
"m" (a), // 2
|
|
||||||
"m" (lda), // 3
|
|
||||||
"m" (x), // 4
|
|
||||||
"m" (y) // 5
|
|
||||||
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
|
|
||||||
"%xmm0", "%xmm1",
|
|
||||||
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
|
|
||||||
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
|
|
||||||
"memory"
|
|
||||||
);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
static void sgemv_kernel_1( long n, float alpha, float *a, long lda, float *x, float *y)
|
|
||||||
{
|
|
||||||
|
|
||||||
|
|
||||||
__asm__ __volatile__
|
|
||||||
(
|
|
||||||
"movq %0, %%rax\n\t" // n -> rax
|
|
||||||
"vmovss %1, %%xmm1\n\t" // alpha -> xmm1
|
|
||||||
"movq %2, %%rsi\n\t" // adress of a -> rsi
|
|
||||||
"movq %3, %%rcx\n\t" // value of lda > rcx
|
|
||||||
"movq %4, %%rdi\n\t" // adress of x -> rdi
|
|
||||||
"movq %5, %%rdx\n\t" // adress of y -> rdx
|
|
||||||
|
|
||||||
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
|
|
||||||
|
|
||||||
".L01LOOP%=: \n\t"
|
|
||||||
"vmovss (%%rdi), %%xmm0 \n\t" // load values of c
|
|
||||||
"addq $4 , %%rdi \n\t" // increment pointer of c
|
|
||||||
|
|
||||||
"vmulss 0*4(%%rsi), %%xmm0, %%xmm4 \n\t" // multiply a and c and add to temp
|
|
||||||
"vaddss %%xmm12, %%xmm4, %%xmm12 \n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
|
|
||||||
|
|
||||||
"dec %%rax \n\t" // n = n -1
|
|
||||||
"jnz .L01LOOP%= \n\t"
|
|
||||||
|
|
||||||
"vmulss %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha
|
|
||||||
|
|
||||||
"vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y
|
|
||||||
|
|
||||||
:
|
|
||||||
:
|
|
||||||
"m" (n), // 0
|
|
||||||
"m" (alpha), // 1
|
|
||||||
"m" (a), // 2
|
|
||||||
"m" (lda), // 3
|
|
||||||
"m" (x), // 4
|
|
||||||
"m" (y) // 5
|
|
||||||
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
|
|
||||||
"%xmm0", "%xmm1",
|
|
||||||
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
|
|
||||||
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
|
|
||||||
"memory"
|
|
||||||
);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1,473 +0,0 @@
|
||||||
/***************************************************************************
|
|
||||||
Copyright (c) 2014, The OpenBLAS Project
|
|
||||||
All rights reserved.
|
|
||||||
Redistribution and use in source and binary forms, with or without
|
|
||||||
modification, are permitted provided that the following conditions are
|
|
||||||
met:
|
|
||||||
1. Redistributions of source code must retain the above copyright
|
|
||||||
notice, this list of conditions and the following disclaimer.
|
|
||||||
2. Redistributions in binary form must reproduce the above copyright
|
|
||||||
notice, this list of conditions and the following disclaimer in
|
|
||||||
the documentation and/or other materials provided with the
|
|
||||||
distribution.
|
|
||||||
3. Neither the name of the OpenBLAS project nor the names of
|
|
||||||
its contributors may be used to endorse or promote products
|
|
||||||
derived from this software without specific prior written permission.
|
|
||||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
||||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
||||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
||||||
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
|
|
||||||
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
||||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
||||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
||||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
||||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
|
|
||||||
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
||||||
*****************************************************************************/
|
|
||||||
|
|
||||||
static void sgemv_kernel_64( long n, float alpha, float *a, long lda, float *x, float *y)
|
|
||||||
{
|
|
||||||
|
|
||||||
|
|
||||||
float *pre = a + lda*2;
|
|
||||||
|
|
||||||
__asm__ __volatile__
|
|
||||||
(
|
|
||||||
"movq %0, %%rax\n\t" // n -> rax
|
|
||||||
"vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1
|
|
||||||
"movq %2, %%rsi\n\t" // adress of a -> rsi
|
|
||||||
"movq %3, %%rcx\n\t" // value of lda > rcx
|
|
||||||
"movq %4, %%rdi\n\t" // adress of x -> rdi
|
|
||||||
"movq %5, %%rdx\n\t" // adress of y -> rdx
|
|
||||||
"movq %6, %%r8\n\t" // address for prefetch
|
|
||||||
"prefetcht0 (%%r8)\n\t" // Prefetch
|
|
||||||
"prefetcht0 64(%%r8)\n\t" // Prefetch
|
|
||||||
|
|
||||||
"vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero
|
|
||||||
"vxorps %%ymm9 , %%ymm9 , %%ymm9 \n\t" // set to zero
|
|
||||||
"vxorps %%ymm10, %%ymm10, %%ymm10\n\t" // set to zero
|
|
||||||
"vxorps %%ymm11, %%ymm11, %%ymm11\n\t" // set to zero
|
|
||||||
"vxorps %%ymm12, %%ymm12, %%ymm12\n\t" // set to zero
|
|
||||||
"vxorps %%ymm13, %%ymm13, %%ymm13\n\t" // set to zero
|
|
||||||
"vxorps %%ymm14, %%ymm14, %%ymm14\n\t" // set to zero
|
|
||||||
"vxorps %%ymm15, %%ymm15, %%ymm15\n\t" // set to zero
|
|
||||||
".align 16 \n\t"
|
|
||||||
".L01LOOP%=: \n\t"
|
|
||||||
"vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c
|
|
||||||
"nop \n\t"
|
|
||||||
"leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch
|
|
||||||
|
|
||||||
"prefetcht0 (%%r8)\n\t" // Prefetch
|
|
||||||
"vmulps 0*4(%%rsi), %%ymm0, %%ymm4 \n\t" // multiply a and c and add to temp
|
|
||||||
"vmulps 8*4(%%rsi), %%ymm0, %%ymm5 \n\t" // multiply a and c and add to temp
|
|
||||||
"prefetcht0 64(%%r8)\n\t" // Prefetch
|
|
||||||
"vmulps 16*4(%%rsi), %%ymm0, %%ymm6 \n\t" // multiply a and c and add to temp
|
|
||||||
"vmulps 24*4(%%rsi), %%ymm0, %%ymm7 \n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"vaddps %%ymm8 , %%ymm4, %%ymm8 \n\t" // multiply a and c and add to temp
|
|
||||||
"vaddps %%ymm9 , %%ymm5, %%ymm9 \n\t" // multiply a and c and add to temp
|
|
||||||
"prefetcht0 128(%%r8)\n\t" // Prefetch
|
|
||||||
"vaddps %%ymm10, %%ymm6, %%ymm10\n\t" // multiply a and c and add to temp
|
|
||||||
"vaddps %%ymm11, %%ymm7, %%ymm11\n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"prefetcht0 192(%%r8)\n\t" // Prefetch
|
|
||||||
"vmulps 32*4(%%rsi), %%ymm0, %%ymm4 \n\t" // multiply a and c and add to temp
|
|
||||||
"vmulps 40*4(%%rsi), %%ymm0, %%ymm5 \n\t" // multiply a and c and add to temp
|
|
||||||
"vmulps 48*4(%%rsi), %%ymm0, %%ymm6 \n\t" // multiply a and c and add to temp
|
|
||||||
"vmulps 56*4(%%rsi), %%ymm0, %%ymm7 \n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"vaddps %%ymm12, %%ymm4, %%ymm12\n\t" // multiply a and c and add to temp
|
|
||||||
"vaddps %%ymm13, %%ymm5, %%ymm13\n\t" // multiply a and c and add to temp
|
|
||||||
"vaddps %%ymm14, %%ymm6, %%ymm14\n\t" // multiply a and c and add to temp
|
|
||||||
"vaddps %%ymm15, %%ymm7, %%ymm15\n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"addq $4 , %%rdi \n\t" // increment pointer of c
|
|
||||||
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
|
|
||||||
|
|
||||||
"dec %%rax \n\t" // n = n -1
|
|
||||||
"jnz .L01LOOP%= \n\t"
|
|
||||||
|
|
||||||
"vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm9 , %%ymm1, %%ymm9 \n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm10, %%ymm1, %%ymm10\n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm11, %%ymm1, %%ymm11\n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm12, %%ymm1, %%ymm12\n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm13, %%ymm1, %%ymm13\n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm14, %%ymm1, %%ymm14\n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm15, %%ymm1, %%ymm15\n\t" // scale by alpha
|
|
||||||
|
|
||||||
"vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm9 , 8*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm10, 16*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm11, 24*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm12, 32*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm13, 40*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm14, 48*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm15, 56*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
|
|
||||||
:
|
|
||||||
:
|
|
||||||
"m" (n), // 0
|
|
||||||
"m" (alpha), // 1
|
|
||||||
"m" (a), // 2
|
|
||||||
"m" (lda), // 3
|
|
||||||
"m" (x), // 4
|
|
||||||
"m" (y), // 5
|
|
||||||
"m" (pre) // 6
|
|
||||||
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc",
|
|
||||||
"%xmm0", "%xmm1",
|
|
||||||
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
|
|
||||||
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
|
|
||||||
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
|
|
||||||
"memory"
|
|
||||||
);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
static void sgemv_kernel_32( long n, float alpha, float *a, long lda, float *x, float *y)
|
|
||||||
{
|
|
||||||
|
|
||||||
|
|
||||||
float *pre = a + lda*3;
|
|
||||||
|
|
||||||
__asm__ __volatile__
|
|
||||||
(
|
|
||||||
"movq %0, %%rax\n\t" // n -> rax
|
|
||||||
"vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1
|
|
||||||
"movq %2, %%rsi\n\t" // adress of a -> rsi
|
|
||||||
"movq %3, %%rcx\n\t" // value of lda > rcx
|
|
||||||
"movq %4, %%rdi\n\t" // adress of x -> rdi
|
|
||||||
"movq %5, %%rdx\n\t" // adress of y -> rdx
|
|
||||||
"movq %6, %%r8\n\t" // address for prefetch
|
|
||||||
"prefetcht0 (%%r8)\n\t" // Prefetch
|
|
||||||
"prefetcht0 64(%%r8)\n\t" // Prefetch
|
|
||||||
|
|
||||||
"vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero
|
|
||||||
"vxorps %%ymm9 , %%ymm9 , %%ymm9 \n\t" // set to zero
|
|
||||||
"vxorps %%ymm10, %%ymm10, %%ymm10\n\t" // set to zero
|
|
||||||
"vxorps %%ymm11, %%ymm11, %%ymm11\n\t" // set to zero
|
|
||||||
".align 16 \n\t"
|
|
||||||
".L01LOOP%=: \n\t"
|
|
||||||
"vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c
|
|
||||||
"nop \n\t"
|
|
||||||
"leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch
|
|
||||||
|
|
||||||
"prefetcht0 (%%r8)\n\t" // Prefetch
|
|
||||||
"prefetcht0 64(%%r8)\n\t" // Prefetch
|
|
||||||
|
|
||||||
"vmulps 0*4(%%rsi), %%ymm0, %%ymm4 \n\t" // multiply a and c and add to temp
|
|
||||||
"vmulps 8*4(%%rsi), %%ymm0, %%ymm5 \n\t" // multiply a and c and add to temp
|
|
||||||
"vmulps 16*4(%%rsi), %%ymm0, %%ymm6 \n\t" // multiply a and c and add to temp
|
|
||||||
"vmulps 24*4(%%rsi), %%ymm0, %%ymm7 \n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"vaddps %%ymm8 , %%ymm4, %%ymm8 \n\t" // multiply a and c and add to temp
|
|
||||||
"vaddps %%ymm9 , %%ymm5, %%ymm9 \n\t" // multiply a and c and add to temp
|
|
||||||
"vaddps %%ymm10, %%ymm6, %%ymm10\n\t" // multiply a and c and add to temp
|
|
||||||
"vaddps %%ymm11, %%ymm7, %%ymm11\n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
"addq $4 , %%rdi \n\t" // increment pointer of c
|
|
||||||
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
|
|
||||||
|
|
||||||
"dec %%rax \n\t" // n = n -1
|
|
||||||
"jnz .L01LOOP%= \n\t"
|
|
||||||
|
|
||||||
"vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm9 , %%ymm1, %%ymm9 \n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm10, %%ymm1, %%ymm10\n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm11, %%ymm1, %%ymm11\n\t" // scale by alpha
|
|
||||||
|
|
||||||
"vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm9 , 8*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm10, 16*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm11, 24*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
|
|
||||||
:
|
|
||||||
:
|
|
||||||
"m" (n), // 0
|
|
||||||
"m" (alpha), // 1
|
|
||||||
"m" (a), // 2
|
|
||||||
"m" (lda), // 3
|
|
||||||
"m" (x), // 4
|
|
||||||
"m" (y), // 5
|
|
||||||
"m" (pre) // 6
|
|
||||||
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc",
|
|
||||||
"%xmm0", "%xmm1",
|
|
||||||
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
|
|
||||||
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
|
|
||||||
"memory"
|
|
||||||
);
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
static void sgemv_kernel_16( long n, float alpha, float *a, long lda, float *x, float *y)
|
|
||||||
{
|
|
||||||
|
|
||||||
float *pre = a + lda*3;
|
|
||||||
|
|
||||||
__asm__ __volatile__
|
|
||||||
(
|
|
||||||
"movq %0, %%rax\n\t" // n -> rax
|
|
||||||
"vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1
|
|
||||||
"movq %2, %%rsi\n\t" // adress of a -> rsi
|
|
||||||
"movq %3, %%rcx\n\t" // value of lda > rcx
|
|
||||||
"movq %4, %%rdi\n\t" // adress of x -> rdi
|
|
||||||
"movq %5, %%rdx\n\t" // adress of y -> rdx
|
|
||||||
"movq %6, %%r8\n\t" // address for prefetch
|
|
||||||
"prefetcht0 (%%r8)\n\t" // Prefetch
|
|
||||||
"prefetcht0 64(%%r8)\n\t" // Prefetch
|
|
||||||
|
|
||||||
"vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero
|
|
||||||
"vxorps %%ymm9 , %%ymm9 , %%ymm9 \n\t" // set to zero
|
|
||||||
".align 16 \n\t"
|
|
||||||
".L01LOOP%=: \n\t"
|
|
||||||
"vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c
|
|
||||||
"nop \n\t"
|
|
||||||
"leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch
|
|
||||||
|
|
||||||
"prefetcht0 (%%r8)\n\t" // Prefetch
|
|
||||||
|
|
||||||
"vmulps 0*4(%%rsi), %%ymm0, %%ymm4 \n\t" // multiply a and c and add to temp
|
|
||||||
"vmulps 8*4(%%rsi), %%ymm0, %%ymm5 \n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"vaddps %%ymm8 , %%ymm4, %%ymm8 \n\t" // multiply a and c and add to temp
|
|
||||||
"vaddps %%ymm9 , %%ymm5, %%ymm9 \n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"addq $4 , %%rdi \n\t" // increment pointer of c
|
|
||||||
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
|
|
||||||
|
|
||||||
"dec %%rax \n\t" // n = n -1
|
|
||||||
"jnz .L01LOOP%= \n\t"
|
|
||||||
|
|
||||||
"vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha
|
|
||||||
"vmulps %%ymm9 , %%ymm1, %%ymm9 \n\t" // scale by alpha
|
|
||||||
|
|
||||||
"vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovups %%ymm9 , 8*4(%%rdx) \n\t" // store temp -> y
|
|
||||||
|
|
||||||
:
|
|
||||||
:
|
|
||||||
"m" (n), // 0
|
|
||||||
"m" (alpha), // 1
|
|
||||||
"m" (a), // 2
|
|
||||||
"m" (lda), // 3
|
|
||||||
"m" (x), // 4
|
|
||||||
"m" (y), // 5
|
|
||||||
"m" (pre) // 6
|
|
||||||
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc",
|
|
||||||
"%xmm0", "%xmm1",
|
|
||||||
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
|
|
||||||
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
|
|
||||||
"memory"
|
|
||||||
);
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
static void sgemv_kernel_8( long n, float alpha, float *a, long lda, float *x, float *y)
|
|
||||||
{
|
|
||||||
|
|
||||||
__asm__ __volatile__
|
|
||||||
(
|
|
||||||
"movq %0, %%rax\n\t" // n -> rax
|
|
||||||
"vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1
|
|
||||||
"movq %2, %%rsi\n\t" // adress of a -> rsi
|
|
||||||
"movq %3, %%rcx\n\t" // value of lda > rcx
|
|
||||||
"movq %4, %%rdi\n\t" // adress of x -> rdi
|
|
||||||
"movq %5, %%rdx\n\t" // adress of y -> rdx
|
|
||||||
|
|
||||||
"vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero
|
|
||||||
".align 16 \n\t"
|
|
||||||
".L01LOOP%=: \n\t"
|
|
||||||
"vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c
|
|
||||||
|
|
||||||
"vmulps 0*4(%%rsi), %%ymm0, %%ymm4 \n\t" // multiply a and c and add to temp
|
|
||||||
"vaddps %%ymm8 , %%ymm4, %%ymm8 \n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"addq $4 , %%rdi \n\t" // increment pointer of c
|
|
||||||
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
|
|
||||||
|
|
||||||
"dec %%rax \n\t" // n = n -1
|
|
||||||
"jnz .L01LOOP%= \n\t"
|
|
||||||
|
|
||||||
"vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha
|
|
||||||
"vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y
|
|
||||||
|
|
||||||
:
|
|
||||||
:
|
|
||||||
"m" (n), // 0
|
|
||||||
"m" (alpha), // 1
|
|
||||||
"m" (a), // 2
|
|
||||||
"m" (lda), // 3
|
|
||||||
"m" (x), // 4
|
|
||||||
"m" (y) // 5
|
|
||||||
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc",
|
|
||||||
"%xmm0", "%xmm1",
|
|
||||||
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
|
|
||||||
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
|
|
||||||
"memory"
|
|
||||||
);
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
static void sgemv_kernel_4( long n, float alpha, float *a, long lda, float *x, float *y)
|
|
||||||
{
|
|
||||||
|
|
||||||
|
|
||||||
__asm__ __volatile__
|
|
||||||
(
|
|
||||||
"movq %0, %%rax\n\t" // n -> rax
|
|
||||||
"vbroadcastss %1, %%xmm1\n\t" // alpha -> xmm1
|
|
||||||
"movq %2, %%rsi\n\t" // adress of a -> rsi
|
|
||||||
"movq %3, %%rcx\n\t" // value of lda > rcx
|
|
||||||
"movq %4, %%rdi\n\t" // adress of x -> rdi
|
|
||||||
"movq %5, %%rdx\n\t" // adress of y -> rdx
|
|
||||||
|
|
||||||
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
|
|
||||||
|
|
||||||
".L01LOOP%=: \n\t"
|
|
||||||
"vbroadcastss (%%rdi), %%xmm0 \n\t" // load values of c
|
|
||||||
|
|
||||||
"vmulps 0*4(%%rsi), %%xmm0, %%xmm4 \n\t" // multiply a and c and add to temp
|
|
||||||
"vaddps %%xmm12, %%xmm4, %%xmm12 \n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"addq $4 , %%rdi \n\t" // increment pointer of c
|
|
||||||
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
|
|
||||||
|
|
||||||
"dec %%rax \n\t" // n = n -1
|
|
||||||
"jnz .L01LOOP%= \n\t"
|
|
||||||
|
|
||||||
"vmulps %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha
|
|
||||||
|
|
||||||
"vmovups %%xmm12, (%%rdx) \n\t" // store temp -> y
|
|
||||||
|
|
||||||
:
|
|
||||||
:
|
|
||||||
"m" (n), // 0
|
|
||||||
"m" (alpha), // 1
|
|
||||||
"m" (a), // 2
|
|
||||||
"m" (lda), // 3
|
|
||||||
"m" (x), // 4
|
|
||||||
"m" (y) // 5
|
|
||||||
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
|
|
||||||
"%xmm0", "%xmm1",
|
|
||||||
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
|
|
||||||
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
|
|
||||||
"memory"
|
|
||||||
);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
static void sgemv_kernel_2( long n, float alpha, float *a, long lda, float *x, float *y)
|
|
||||||
{
|
|
||||||
|
|
||||||
|
|
||||||
__asm__ __volatile__
|
|
||||||
(
|
|
||||||
"movq %0, %%rax\n\t" // n -> rax
|
|
||||||
"vmovss %1, %%xmm1\n\t" // alpha -> xmm1
|
|
||||||
"movq %2, %%rsi\n\t" // adress of a -> rsi
|
|
||||||
"movq %3, %%rcx\n\t" // value of lda > rcx
|
|
||||||
"movq %4, %%rdi\n\t" // adress of x -> rdi
|
|
||||||
"movq %5, %%rdx\n\t" // adress of y -> rdx
|
|
||||||
|
|
||||||
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
|
|
||||||
"vxorps %%xmm13, %%xmm13, %%xmm13\n\t" // set to zero
|
|
||||||
|
|
||||||
".L01LOOP%=: \n\t"
|
|
||||||
"vmovss (%%rdi), %%xmm0 \n\t" // load values of c
|
|
||||||
|
|
||||||
"vmulps 0*4(%%rsi), %%xmm0, %%xmm4 \n\t" // multiply a and c and add to temp
|
|
||||||
"vmulps 1*4(%%rsi), %%xmm0, %%xmm5 \n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"vaddps %%xmm12, %%xmm4, %%xmm12 \n\t" // multiply a and c and add to temp
|
|
||||||
"vaddps %%xmm13, %%xmm5, %%xmm13 \n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"addq $4 , %%rdi \n\t" // increment pointer of c
|
|
||||||
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
|
|
||||||
|
|
||||||
"dec %%rax \n\t" // n = n -1
|
|
||||||
"jnz .L01LOOP%= \n\t"
|
|
||||||
|
|
||||||
"vmulss %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha
|
|
||||||
"vmulss %%xmm13, %%xmm1, %%xmm13\n\t" // scale by alpha
|
|
||||||
|
|
||||||
"vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y
|
|
||||||
"vmovss %%xmm13, 4(%%rdx) \n\t" // store temp -> y
|
|
||||||
|
|
||||||
:
|
|
||||||
:
|
|
||||||
"m" (n), // 0
|
|
||||||
"m" (alpha), // 1
|
|
||||||
"m" (a), // 2
|
|
||||||
"m" (lda), // 3
|
|
||||||
"m" (x), // 4
|
|
||||||
"m" (y) // 5
|
|
||||||
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
|
|
||||||
"%xmm0", "%xmm1",
|
|
||||||
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
|
|
||||||
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
|
|
||||||
"memory"
|
|
||||||
);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
static void sgemv_kernel_1( long n, float alpha, float *a, long lda, float *x, float *y)
|
|
||||||
{
|
|
||||||
|
|
||||||
|
|
||||||
__asm__ __volatile__
|
|
||||||
(
|
|
||||||
"movq %0, %%rax\n\t" // n -> rax
|
|
||||||
"vmovss %1, %%xmm1\n\t" // alpha -> xmm1
|
|
||||||
"movq %2, %%rsi\n\t" // adress of a -> rsi
|
|
||||||
"movq %3, %%rcx\n\t" // value of lda > rcx
|
|
||||||
"movq %4, %%rdi\n\t" // adress of x -> rdi
|
|
||||||
"movq %5, %%rdx\n\t" // adress of y -> rdx
|
|
||||||
|
|
||||||
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
|
|
||||||
|
|
||||||
".L01LOOP%=: \n\t"
|
|
||||||
"vmovss (%%rdi), %%xmm0 \n\t" // load values of c
|
|
||||||
"addq $4 , %%rdi \n\t" // increment pointer of c
|
|
||||||
|
|
||||||
"vmulss 0*4(%%rsi), %%xmm0, %%xmm4 \n\t" // multiply a and c and add to temp
|
|
||||||
"vaddss %%xmm12, %%xmm4, %%xmm12 \n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
|
|
||||||
|
|
||||||
"dec %%rax \n\t" // n = n -1
|
|
||||||
"jnz .L01LOOP%= \n\t"
|
|
||||||
|
|
||||||
"vmulss %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha
|
|
||||||
|
|
||||||
"vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y
|
|
||||||
|
|
||||||
:
|
|
||||||
:
|
|
||||||
"m" (n), // 0
|
|
||||||
"m" (alpha), // 1
|
|
||||||
"m" (a), // 2
|
|
||||||
"m" (lda), // 3
|
|
||||||
"m" (x), // 4
|
|
||||||
"m" (y) // 5
|
|
||||||
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
|
|
||||||
"%xmm0", "%xmm1",
|
|
||||||
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
|
|
||||||
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
|
|
||||||
"memory"
|
|
||||||
);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1,232 +0,0 @@
|
||||||
/***************************************************************************
|
|
||||||
Copyright (c) 2014, The OpenBLAS Project
|
|
||||||
All rights reserved.
|
|
||||||
Redistribution and use in source and binary forms, with or without
|
|
||||||
modification, are permitted provided that the following conditions are
|
|
||||||
met:
|
|
||||||
1. Redistributions of source code must retain the above copyright
|
|
||||||
notice, this list of conditions and the following disclaimer.
|
|
||||||
2. Redistributions in binary form must reproduce the above copyright
|
|
||||||
notice, this list of conditions and the following disclaimer in
|
|
||||||
the documentation and/or other materials provided with the
|
|
||||||
distribution.
|
|
||||||
3. Neither the name of the OpenBLAS project nor the names of
|
|
||||||
its contributors may be used to endorse or promote products
|
|
||||||
derived from this software without specific prior written permission.
|
|
||||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
||||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
||||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
||||||
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
|
|
||||||
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
||||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
||||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
||||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
||||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
|
|
||||||
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
||||||
*****************************************************************************/
|
|
||||||
|
|
||||||
|
|
||||||
#include "common.h"
|
|
||||||
|
|
||||||
#if defined(BULLDOZER) || defined(PILEDRIVER)
|
|
||||||
#include "sgemv_t_microk_bulldozer.c"
|
|
||||||
#elif defined(HASWELL)
|
|
||||||
#include "sgemv_t_microk_haswell.c"
|
|
||||||
#else
|
|
||||||
#include "sgemv_t_microk_sandy.c"
|
|
||||||
#endif
|
|
||||||
|
|
||||||
static void copy_x(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_src)
|
|
||||||
{
|
|
||||||
BLASLONG i;
|
|
||||||
for ( i=0; i<n; i++ )
|
|
||||||
{
|
|
||||||
*dest = *src;
|
|
||||||
dest++;
|
|
||||||
src += inc_src;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static void sgemv_kernel_1( BLASLONG n, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *x, FLOAT *y)
|
|
||||||
{
|
|
||||||
|
|
||||||
FLOAT register temp0 = 0.0;
|
|
||||||
BLASLONG i;
|
|
||||||
for ( i=0; i<n ; i++)
|
|
||||||
{
|
|
||||||
temp0 += a[i] * x[i];
|
|
||||||
}
|
|
||||||
temp0 *= alpha ;
|
|
||||||
*y += temp0;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer)
|
|
||||||
{
|
|
||||||
BLASLONG i;
|
|
||||||
BLASLONG j;
|
|
||||||
FLOAT *a_ptr;
|
|
||||||
FLOAT *x_ptr;
|
|
||||||
FLOAT *y_ptr;
|
|
||||||
FLOAT *a_ptrl;
|
|
||||||
BLASLONG m1;
|
|
||||||
BLASLONG register m2;
|
|
||||||
FLOAT *xbuffer;
|
|
||||||
xbuffer = buffer;
|
|
||||||
BLASLONG register Mblock;
|
|
||||||
|
|
||||||
m1 = m / 1024 ;
|
|
||||||
m2 = m % 1024 ;
|
|
||||||
|
|
||||||
x_ptr = x;
|
|
||||||
a_ptr = a;
|
|
||||||
|
|
||||||
for (j=0; j<m1; j++)
|
|
||||||
{
|
|
||||||
|
|
||||||
if ( inc_x == 1 )
|
|
||||||
xbuffer = x_ptr;
|
|
||||||
else
|
|
||||||
copy_x(1024,x_ptr,xbuffer,inc_x);
|
|
||||||
|
|
||||||
y_ptr = y;
|
|
||||||
a_ptrl = a_ptr;
|
|
||||||
|
|
||||||
for(i = 0; i<n; i++ )
|
|
||||||
{
|
|
||||||
sgemv_kernel_16(1024,alpha,a_ptrl,lda,xbuffer,y_ptr);
|
|
||||||
y_ptr += inc_y;
|
|
||||||
a_ptrl += lda;
|
|
||||||
}
|
|
||||||
a_ptr += 1024;
|
|
||||||
x_ptr += 1024 * inc_x;
|
|
||||||
}
|
|
||||||
|
|
||||||
if ( m2 == 0 ) return(0);
|
|
||||||
|
|
||||||
Mblock = 512;
|
|
||||||
while ( Mblock >= 16 )
|
|
||||||
{
|
|
||||||
if ( m2 & Mblock)
|
|
||||||
{
|
|
||||||
|
|
||||||
if ( inc_x == 1 )
|
|
||||||
xbuffer = x_ptr;
|
|
||||||
else
|
|
||||||
copy_x(Mblock,x_ptr,xbuffer,inc_x);
|
|
||||||
|
|
||||||
y_ptr = y;
|
|
||||||
a_ptrl = a_ptr;
|
|
||||||
|
|
||||||
for(i = 0; i<n; i++ )
|
|
||||||
{
|
|
||||||
sgemv_kernel_16(Mblock,alpha,a_ptrl,lda,xbuffer,y_ptr);
|
|
||||||
y_ptr += inc_y;
|
|
||||||
a_ptrl += lda;
|
|
||||||
}
|
|
||||||
a_ptr += Mblock;
|
|
||||||
x_ptr += Mblock * inc_x;
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
Mblock /= 2;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
if ( m2 & Mblock)
|
|
||||||
{
|
|
||||||
|
|
||||||
if ( inc_x == 1 )
|
|
||||||
xbuffer = x_ptr;
|
|
||||||
else
|
|
||||||
copy_x(Mblock,x_ptr,xbuffer,inc_x);
|
|
||||||
|
|
||||||
y_ptr = y;
|
|
||||||
a_ptrl = a_ptr;
|
|
||||||
|
|
||||||
for(i = 0; i<n; i++ )
|
|
||||||
{
|
|
||||||
sgemv_kernel_1(Mblock,alpha,a_ptrl,lda,xbuffer,y_ptr);
|
|
||||||
y_ptr += inc_y;
|
|
||||||
a_ptrl += lda;
|
|
||||||
}
|
|
||||||
a_ptr += Mblock;
|
|
||||||
x_ptr += Mblock * inc_x;
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
Mblock /= 2;
|
|
||||||
|
|
||||||
|
|
||||||
if ( m2 & Mblock)
|
|
||||||
{
|
|
||||||
|
|
||||||
if ( inc_x == 1 )
|
|
||||||
xbuffer = x_ptr;
|
|
||||||
else
|
|
||||||
copy_x(Mblock,x_ptr,xbuffer,inc_x);
|
|
||||||
|
|
||||||
y_ptr = y;
|
|
||||||
a_ptrl = a_ptr;
|
|
||||||
|
|
||||||
for(i = 0; i<n; i++ )
|
|
||||||
{
|
|
||||||
sgemv_kernel_1(Mblock,alpha,a_ptrl,lda,xbuffer,y_ptr);
|
|
||||||
y_ptr += inc_y;
|
|
||||||
a_ptrl += lda;
|
|
||||||
}
|
|
||||||
a_ptr += Mblock;
|
|
||||||
x_ptr += Mblock * inc_x;
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
Mblock /= 2;
|
|
||||||
|
|
||||||
if ( m2 & Mblock)
|
|
||||||
{
|
|
||||||
|
|
||||||
if ( inc_x == 1 )
|
|
||||||
xbuffer = x_ptr;
|
|
||||||
else
|
|
||||||
copy_x(Mblock,x_ptr,xbuffer,inc_x);
|
|
||||||
|
|
||||||
y_ptr = y;
|
|
||||||
a_ptrl = a_ptr;
|
|
||||||
|
|
||||||
for(i = 0; i<n; i++ )
|
|
||||||
{
|
|
||||||
sgemv_kernel_1(Mblock,alpha,a_ptrl,lda,xbuffer,y_ptr);
|
|
||||||
y_ptr += inc_y;
|
|
||||||
a_ptrl += lda;
|
|
||||||
}
|
|
||||||
a_ptr += Mblock;
|
|
||||||
x_ptr += Mblock * inc_x;
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
Mblock /= 2;
|
|
||||||
|
|
||||||
if ( m2 & Mblock)
|
|
||||||
{
|
|
||||||
|
|
||||||
xbuffer = x_ptr;
|
|
||||||
|
|
||||||
y_ptr = y;
|
|
||||||
a_ptrl = a_ptr;
|
|
||||||
|
|
||||||
for(i = 0; i<n; i++ )
|
|
||||||
{
|
|
||||||
sgemv_kernel_1(Mblock,alpha,a_ptrl,lda,xbuffer,y_ptr);
|
|
||||||
y_ptr += inc_y;
|
|
||||||
a_ptrl += lda;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
return(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1,99 +0,0 @@
|
||||||
/***************************************************************************
|
|
||||||
Copyright (c) 2014, The OpenBLAS Project
|
|
||||||
All rights reserved.
|
|
||||||
Redistribution and use in source and binary forms, with or without
|
|
||||||
modification, are permitted provided that the following conditions are
|
|
||||||
met:
|
|
||||||
1. Redistributions of source code must retain the above copyright
|
|
||||||
notice, this list of conditions and the following disclaimer.
|
|
||||||
2. Redistributions in binary form must reproduce the above copyright
|
|
||||||
notice, this list of conditions and the following disclaimer in
|
|
||||||
the documentation and/or other materials provided with the
|
|
||||||
distribution.
|
|
||||||
3. Neither the name of the OpenBLAS project nor the names of
|
|
||||||
its contributors may be used to endorse or promote products
|
|
||||||
derived from this software without specific prior written permission.
|
|
||||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
||||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
||||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
||||||
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
|
|
||||||
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
||||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
||||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
||||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
||||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
|
|
||||||
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
||||||
*****************************************************************************/
|
|
||||||
|
|
||||||
static void sgemv_kernel_16( long n, float alpha, float *a, long lda, float *x, float *y)
|
|
||||||
{
|
|
||||||
|
|
||||||
//n = n / 16;
|
|
||||||
|
|
||||||
__asm__ __volatile__
|
|
||||||
(
|
|
||||||
"movq %0, %%rax\n\t" // n -> rax
|
|
||||||
"vmovss %1, %%xmm1\n\t" // alpha -> xmm1
|
|
||||||
"movq %2, %%rsi\n\t" // adress of a -> rsi
|
|
||||||
"movq %3, %%rcx\n\t" // value of lda > rcx
|
|
||||||
"movq %4, %%rdi\n\t" // adress of x -> rdi
|
|
||||||
"movq %5, %%rdx\n\t" // adress of y -> rdx
|
|
||||||
|
|
||||||
"leaq (, %%rcx,4), %%rcx \n\t" // scale lda by size of float
|
|
||||||
"leaq (%%rsi,%%rcx,1), %%r8 \n\t" // pointer to next line
|
|
||||||
|
|
||||||
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
|
|
||||||
"vxorps %%xmm13, %%xmm13, %%xmm13\n\t" // set to zero
|
|
||||||
"vxorps %%xmm14, %%xmm14, %%xmm14\n\t" // set to zero
|
|
||||||
"vxorps %%xmm15, %%xmm15, %%xmm15\n\t" // set to zero
|
|
||||||
|
|
||||||
"sarq $4, %%rax \n\t" // n = n / 16
|
|
||||||
|
|
||||||
".align 16 \n\t"
|
|
||||||
".L01LOOP%=: \n\t"
|
|
||||||
// "prefetcht0 512(%%rsi) \n\t"
|
|
||||||
"prefetcht0 (%%r8) \n\t" //prefetch next line of a
|
|
||||||
"vmovups (%%rsi), %%xmm4 \n\t"
|
|
||||||
"vmovups 4*4(%%rsi), %%xmm5 \n\t"
|
|
||||||
"vmovups 8*4(%%rsi), %%xmm6 \n\t"
|
|
||||||
"vmovups 12*4(%%rsi), %%xmm7 \n\t"
|
|
||||||
|
|
||||||
"vfmaddps %%xmm12, 0*4(%%rdi), %%xmm4, %%xmm12\n\t" // multiply a and c and add to temp
|
|
||||||
"vfmaddps %%xmm13, 4*4(%%rdi), %%xmm5, %%xmm13\n\t" // multiply a and c and add to temp
|
|
||||||
"vfmaddps %%xmm14, 8*4(%%rdi), %%xmm6, %%xmm14\n\t" // multiply a and c and add to temp
|
|
||||||
"vfmaddps %%xmm15, 12*4(%%rdi), %%xmm7, %%xmm15\n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"addq $16*4 , %%r8 \n\t" // increment prefetch pointer
|
|
||||||
"addq $16*4 , %%rsi \n\t" // increment pointer of a
|
|
||||||
"addq $16*4 , %%rdi \n\t" // increment pointer of c
|
|
||||||
"dec %%rax \n\t" // n = n -1
|
|
||||||
"jnz .L01LOOP%= \n\t"
|
|
||||||
|
|
||||||
"vaddps %%xmm12, %%xmm14, %%xmm12\n\t"
|
|
||||||
"vaddps %%xmm13, %%xmm15, %%xmm13\n\t"
|
|
||||||
"vaddps %%xmm12, %%xmm13, %%xmm12\n\t"
|
|
||||||
"vhaddps %%xmm12, %%xmm12, %%xmm12\n\t"
|
|
||||||
"vhaddps %%xmm12, %%xmm12, %%xmm12\n\t"
|
|
||||||
|
|
||||||
"vfmaddss (%%rdx), %%xmm12, %%xmm1, %%xmm12\n\t"
|
|
||||||
"vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y
|
|
||||||
|
|
||||||
:
|
|
||||||
:
|
|
||||||
"m" (n), // 0
|
|
||||||
"m" (alpha), // 1
|
|
||||||
"m" (a), // 2
|
|
||||||
"m" (lda), // 3
|
|
||||||
"m" (x), // 4
|
|
||||||
"m" (y) // 5
|
|
||||||
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
|
|
||||||
"%xmm0", "%xmm1",
|
|
||||||
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
|
|
||||||
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
|
|
||||||
"memory"
|
|
||||||
);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1,100 +0,0 @@
|
||||||
/***************************************************************************
|
|
||||||
Copyright (c) 2014, The OpenBLAS Project
|
|
||||||
All rights reserved.
|
|
||||||
Redistribution and use in source and binary forms, with or without
|
|
||||||
modification, are permitted provided that the following conditions are
|
|
||||||
met:
|
|
||||||
1. Redistributions of source code must retain the above copyright
|
|
||||||
notice, this list of conditions and the following disclaimer.
|
|
||||||
2. Redistributions in binary form must reproduce the above copyright
|
|
||||||
notice, this list of conditions and the following disclaimer in
|
|
||||||
the documentation and/or other materials provided with the
|
|
||||||
distribution.
|
|
||||||
3. Neither the name of the OpenBLAS project nor the names of
|
|
||||||
its contributors may be used to endorse or promote products
|
|
||||||
derived from this software without specific prior written permission.
|
|
||||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
||||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
||||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
||||||
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
|
|
||||||
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
||||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
||||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
||||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
||||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
|
|
||||||
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
||||||
*****************************************************************************/
|
|
||||||
|
|
||||||
static void sgemv_kernel_16( long n, float alpha, float *a, long lda, float *x, float *y)
|
|
||||||
{
|
|
||||||
|
|
||||||
//n = n / 16;
|
|
||||||
|
|
||||||
__asm__ __volatile__
|
|
||||||
(
|
|
||||||
"movq %0, %%rax\n\t" // n -> rax
|
|
||||||
"vmovss %1, %%xmm1\n\t" // alpha -> xmm1
|
|
||||||
"movq %2, %%rsi\n\t" // adress of a -> rsi
|
|
||||||
"movq %3, %%rcx\n\t" // value of lda > rcx
|
|
||||||
"movq %4, %%rdi\n\t" // adress of x -> rdi
|
|
||||||
"movq %5, %%rdx\n\t" // adress of y -> rdx
|
|
||||||
|
|
||||||
"leaq (, %%rcx,4), %%rcx \n\t" // scale lda by size of float
|
|
||||||
"leaq (%%rsi,%%rcx,1), %%r8 \n\t" // pointer to next line
|
|
||||||
|
|
||||||
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
|
|
||||||
"vxorps %%xmm13, %%xmm13, %%xmm13\n\t" // set to zero
|
|
||||||
"vxorps %%xmm14, %%xmm14, %%xmm14\n\t" // set to zero
|
|
||||||
"vxorps %%xmm15, %%xmm15, %%xmm15\n\t" // set to zero
|
|
||||||
|
|
||||||
"sarq $4, %%rax \n\t" // n = n / 16
|
|
||||||
|
|
||||||
".align 16 \n\t"
|
|
||||||
".L01LOOP%=: \n\t"
|
|
||||||
// "prefetcht0 512(%%rsi) \n\t"
|
|
||||||
"prefetcht0 (%%r8) \n\t" //prefetch next line of a
|
|
||||||
"vmovups (%%rsi), %%xmm4 \n\t"
|
|
||||||
"vmovups 4*4(%%rsi), %%xmm5 \n\t"
|
|
||||||
"vmovups 8*4(%%rsi), %%xmm6 \n\t"
|
|
||||||
"vmovups 12*4(%%rsi), %%xmm7 \n\t"
|
|
||||||
|
|
||||||
"vfmadd231ps 0*4(%%rdi), %%xmm4, %%xmm12\n\t" // multiply a and c and add to temp
|
|
||||||
"vfmadd231ps 4*4(%%rdi), %%xmm5, %%xmm13\n\t" // multiply a and c and add to temp
|
|
||||||
"vfmadd231ps 8*4(%%rdi), %%xmm6, %%xmm14\n\t" // multiply a and c and add to temp
|
|
||||||
"vfmadd231ps 12*4(%%rdi), %%xmm7, %%xmm15\n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"addq $16*4 , %%r8 \n\t" // increment prefetch pointer
|
|
||||||
"addq $16*4 , %%rsi \n\t" // increment pointer of a
|
|
||||||
"addq $16*4 , %%rdi \n\t" // increment pointer of c
|
|
||||||
"dec %%rax \n\t" // n = n -1
|
|
||||||
"jnz .L01LOOP%= \n\t"
|
|
||||||
|
|
||||||
"vaddps %%xmm12, %%xmm14, %%xmm12\n\t"
|
|
||||||
"vaddps %%xmm13, %%xmm15, %%xmm13\n\t"
|
|
||||||
"vaddps %%xmm12, %%xmm13, %%xmm12\n\t"
|
|
||||||
"vhaddps %%xmm12, %%xmm12, %%xmm12\n\t"
|
|
||||||
"vhaddps %%xmm12, %%xmm12, %%xmm12\n\t"
|
|
||||||
|
|
||||||
"vmulss %%xmm12, %%xmm1, %%xmm12\n\t"
|
|
||||||
"vaddss (%%rdx), %%xmm12,%%xmm12\n\t"
|
|
||||||
"vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y
|
|
||||||
|
|
||||||
:
|
|
||||||
:
|
|
||||||
"m" (n), // 0
|
|
||||||
"m" (alpha), // 1
|
|
||||||
"m" (a), // 2
|
|
||||||
"m" (lda), // 3
|
|
||||||
"m" (x), // 4
|
|
||||||
"m" (y) // 5
|
|
||||||
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
|
|
||||||
"%xmm0", "%xmm1",
|
|
||||||
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
|
|
||||||
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
|
|
||||||
"memory"
|
|
||||||
);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1,106 +0,0 @@
|
||||||
/***************************************************************************
|
|
||||||
Copyright (c) 2014, The OpenBLAS Project
|
|
||||||
All rights reserved.
|
|
||||||
Redistribution and use in source and binary forms, with or without
|
|
||||||
modification, are permitted provided that the following conditions are
|
|
||||||
met:
|
|
||||||
1. Redistributions of source code must retain the above copyright
|
|
||||||
notice, this list of conditions and the following disclaimer.
|
|
||||||
2. Redistributions in binary form must reproduce the above copyright
|
|
||||||
notice, this list of conditions and the following disclaimer in
|
|
||||||
the documentation and/or other materials provided with the
|
|
||||||
distribution.
|
|
||||||
3. Neither the name of the OpenBLAS project nor the names of
|
|
||||||
its contributors may be used to endorse or promote products
|
|
||||||
derived from this software without specific prior written permission.
|
|
||||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
||||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
||||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
||||||
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
|
|
||||||
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
||||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
||||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
||||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
||||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
|
|
||||||
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
||||||
*****************************************************************************/
|
|
||||||
|
|
||||||
static void sgemv_kernel_16( long n, float alpha, float *a, long lda, float *x, float *y)
|
|
||||||
{
|
|
||||||
|
|
||||||
//n = n / 16;
|
|
||||||
|
|
||||||
__asm__ __volatile__
|
|
||||||
(
|
|
||||||
"movq %0, %%rax\n\t" // n -> rax
|
|
||||||
"vmovss %1, %%xmm1\n\t" // alpha -> xmm1
|
|
||||||
"movq %2, %%rsi\n\t" // adress of a -> rsi
|
|
||||||
"movq %3, %%rcx\n\t" // value of lda > rcx
|
|
||||||
"movq %4, %%rdi\n\t" // adress of x -> rdi
|
|
||||||
"movq %5, %%rdx\n\t" // adress of y -> rdx
|
|
||||||
|
|
||||||
"leaq (, %%rcx,4), %%rcx \n\t" // scale lda by size of float
|
|
||||||
"leaq (%%rsi,%%rcx,1), %%r8 \n\t" // pointer to next line
|
|
||||||
|
|
||||||
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
|
|
||||||
"vxorps %%xmm13, %%xmm13, %%xmm13\n\t" // set to zero
|
|
||||||
"vxorps %%xmm14, %%xmm14, %%xmm14\n\t" // set to zero
|
|
||||||
"vxorps %%xmm15, %%xmm15, %%xmm15\n\t" // set to zero
|
|
||||||
|
|
||||||
"sarq $4, %%rax \n\t" // n = n / 16
|
|
||||||
|
|
||||||
".align 16 \n\t"
|
|
||||||
".L01LOOP%=: \n\t"
|
|
||||||
// "prefetcht0 512(%%rsi) \n\t"
|
|
||||||
"prefetcht0 (%%r8) \n\t" //prefetch next line of a
|
|
||||||
"vmovups (%%rsi), %%xmm4 \n\t"
|
|
||||||
"vmovups 4*4(%%rsi), %%xmm5 \n\t"
|
|
||||||
"vmovups 8*4(%%rsi), %%xmm6 \n\t"
|
|
||||||
"vmovups 12*4(%%rsi), %%xmm7 \n\t"
|
|
||||||
|
|
||||||
"vmulps 0*4(%%rdi), %%xmm4, %%xmm8 \n\t" // multiply a and c and add to temp
|
|
||||||
"vmulps 4*4(%%rdi), %%xmm5, %%xmm9 \n\t" // multiply a and c and add to temp
|
|
||||||
"vmulps 8*4(%%rdi), %%xmm6, %%xmm10\n\t" // multiply a and c and add to temp
|
|
||||||
"vmulps 12*4(%%rdi), %%xmm7, %%xmm11\n\t" // multiply a and c and add to temp
|
|
||||||
|
|
||||||
"vaddps %%xmm12, %%xmm8 , %%xmm12\n\t"
|
|
||||||
"vaddps %%xmm13, %%xmm9 , %%xmm13\n\t"
|
|
||||||
"vaddps %%xmm14, %%xmm10, %%xmm14\n\t"
|
|
||||||
"vaddps %%xmm15, %%xmm11, %%xmm15\n\t"
|
|
||||||
|
|
||||||
"addq $16*4 , %%r8 \n\t" // increment prefetch pointer
|
|
||||||
"addq $16*4 , %%rsi \n\t" // increment pointer of a
|
|
||||||
"addq $16*4 , %%rdi \n\t" // increment pointer of c
|
|
||||||
"dec %%rax \n\t" // n = n -1
|
|
||||||
"jnz .L01LOOP%= \n\t"
|
|
||||||
|
|
||||||
"vaddps %%xmm12, %%xmm14, %%xmm12\n\t"
|
|
||||||
"vaddps %%xmm13, %%xmm15, %%xmm13\n\t"
|
|
||||||
"vaddps %%xmm12, %%xmm13, %%xmm12\n\t"
|
|
||||||
"vhaddps %%xmm12, %%xmm12, %%xmm12\n\t"
|
|
||||||
"vhaddps %%xmm12, %%xmm12, %%xmm12\n\t"
|
|
||||||
|
|
||||||
"vmulss %%xmm12, %%xmm1, %%xmm12 \n\t"
|
|
||||||
"vaddss (%%rdx), %%xmm12, %%xmm12\n\t"
|
|
||||||
"vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y
|
|
||||||
|
|
||||||
:
|
|
||||||
:
|
|
||||||
"m" (n), // 0
|
|
||||||
"m" (alpha), // 1
|
|
||||||
"m" (a), // 2
|
|
||||||
"m" (lda), // 3
|
|
||||||
"m" (x), // 4
|
|
||||||
"m" (y) // 5
|
|
||||||
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc",
|
|
||||||
"%xmm0", "%xmm1",
|
|
||||||
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
|
|
||||||
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
|
|
||||||
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
|
|
||||||
"memory"
|
|
||||||
);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Loading…
Reference in New Issue