diff --git a/kernel/x86_64/sgemv_n_avx.c b/kernel/x86_64/sgemv_n_avx.c deleted file mode 100644 index 57aaad4b4..000000000 --- a/kernel/x86_64/sgemv_n_avx.c +++ /dev/null @@ -1,218 +0,0 @@ -/*************************************************************************** -Copyright (c) 2014, The OpenBLAS Project -All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: -1. Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in -the documentation and/or other materials provided with the -distribution. -3. Neither the name of the OpenBLAS project nor the names of -its contributors may be used to endorse or promote products -derived from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*****************************************************************************/ - - -#include "common.h" - -#if defined(BULLDOZER) || defined(PILEDRIVER) -#include "sgemv_n_microk_bulldozer.c" -#elif defined(HASWELL) -#include "sgemv_n_microk_haswell.c" -#else -#include "sgemv_n_microk_sandy.c" -#endif - -static void copy_x(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_src) -{ - BLASLONG i; - for ( i=0; i 0 ) - { - - if ( inc_x == 1 ) - xbuffer = x_ptr; - else - copy_x(n2,x_ptr,xbuffer,inc_x); - - a_ptr = a + n1 * 512 * lda; - y_ptr = y; - - for(i = 0; i rax - "vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1 - "movq %2, %%rsi\n\t" // adress of a -> rsi - "movq %3, %%rcx\n\t" // value of lda > rcx - "movq %4, %%rdi\n\t" // adress of x -> rdi - "movq %5, %%rdx\n\t" // adress of y -> rdx - "movq %6, %%r8\n\t" // address for prefetch - "prefetcht0 (%%r8)\n\t" // Prefetch - "prefetcht0 64(%%r8)\n\t" // Prefetch - - "vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero - "vxorps %%ymm9 , %%ymm9 , %%ymm9 \n\t" // set to zero - "vxorps %%ymm10, %%ymm10, %%ymm10\n\t" // set to zero - "vxorps %%ymm11, %%ymm11, %%ymm11\n\t" // set to zero - "vxorps %%ymm12, %%ymm12, %%ymm12\n\t" // set to zero - "vxorps %%ymm13, %%ymm13, %%ymm13\n\t" // set to zero - "vxorps %%ymm14, %%ymm14, %%ymm14\n\t" // set to zero - "vxorps %%ymm15, %%ymm15, %%ymm15\n\t" // set to zero - ".align 16 \n\t" - ".L01LOOP%=: \n\t" - "vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c - "nop \n\t" - "leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch - - "prefetcht0 (%%r8)\n\t" // Prefetch - "vfmaddps %%ymm8 , 0*4(%%rsi), %%ymm0, %%ymm8 \n\t" // multiply a and c and add to temp - "prefetcht0 64(%%r8)\n\t" // Prefetch - "vfmaddps %%ymm9 , 8*4(%%rsi), %%ymm0, %%ymm9 \n\t" // multiply a and c and add to temp - "prefetcht0 128(%%r8)\n\t" // Prefetch - "vfmaddps %%ymm10, 16*4(%%rsi), %%ymm0, %%ymm10\n\t" // multiply a and c and add to temp - "vfmaddps %%ymm11, 24*4(%%rsi), %%ymm0, %%ymm11\n\t" // multiply a and c and add to temp - "prefetcht0 192(%%r8)\n\t" // Prefetch - "vfmaddps %%ymm12, 32*4(%%rsi), %%ymm0, %%ymm12\n\t" // multiply a and c and add to temp - "vfmaddps %%ymm13, 40*4(%%rsi), %%ymm0, %%ymm13\n\t" // multiply a and c and add to temp - "vfmaddps %%ymm14, 48*4(%%rsi), %%ymm0, %%ymm14\n\t" // multiply a and c and add to temp - "vfmaddps %%ymm15, 56*4(%%rsi), %%ymm0, %%ymm15\n\t" // multiply a and c and add to temp - - "addq $4 , %%rdi \n\t" // increment pointer of c - "leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a - - "dec %%rax \n\t" // n = n -1 - "jnz .L01LOOP%= \n\t" - - "vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha - "vmulps %%ymm9 , %%ymm1, %%ymm9 \n\t" // scale by alpha - "vmulps %%ymm10, %%ymm1, %%ymm10\n\t" // scale by alpha - "vmulps %%ymm11, %%ymm1, %%ymm11\n\t" // scale by alpha - "vmulps %%ymm12, %%ymm1, %%ymm12\n\t" // scale by alpha - "vmulps %%ymm13, %%ymm1, %%ymm13\n\t" // scale by alpha - "vmulps %%ymm14, %%ymm1, %%ymm14\n\t" // scale by alpha - "vmulps %%ymm15, %%ymm1, %%ymm15\n\t" // scale by alpha - - "vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y - "vmovups %%ymm9 , 8*4(%%rdx) \n\t" // store temp -> y - "vmovups %%ymm10, 16*4(%%rdx) \n\t" // store temp -> y - "vmovups %%ymm11, 24*4(%%rdx) \n\t" // store temp -> y - "vmovups %%ymm12, 32*4(%%rdx) \n\t" // store temp -> y - "vmovups %%ymm13, 40*4(%%rdx) \n\t" // store temp -> y - "vmovups %%ymm14, 48*4(%%rdx) \n\t" // store temp -> y - "vmovups %%ymm15, 56*4(%%rdx) \n\t" // store temp -> y - - : - : - "m" (n), // 0 - "m" (alpha), // 1 - "m" (a), // 2 - "m" (lda), // 3 - "m" (x), // 4 - "m" (y), // 5 - "m" (pre) // 6 - : "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", - "%xmm0", "%xmm1", - "%xmm8", "%xmm9", "%xmm10", "%xmm11", - "%xmm12", "%xmm13", "%xmm14", "%xmm15", - "memory" - ); - -} - - - -static void sgemv_kernel_32( long n, float alpha, float *a, long lda, float *x, float *y) -{ - - - float *pre = a + lda*3; - - __asm__ __volatile__ - ( - "movq %0, %%rax\n\t" // n -> rax - "vbroadcastss %1, %%xmm1\n\t" // alpha -> xmm1 - "movq %2, %%rsi\n\t" // adress of a -> rsi - "movq %3, %%rcx\n\t" // value of lda > rcx - "movq %4, %%rdi\n\t" // adress of x -> rdi - "movq %5, %%rdx\n\t" // adress of y -> rdx - "movq %6, %%r8\n\t" // address for prefetch - "prefetcht0 (%%r8)\n\t" // Prefetch - "prefetcht0 64(%%r8)\n\t" // Prefetch - - "vxorps %%xmm8 , %%xmm8 , %%xmm8 \n\t" // set to zero - "vxorps %%xmm9 , %%xmm9 , %%xmm9 \n\t" // set to zero - "vxorps %%xmm10, %%xmm10, %%xmm10\n\t" // set to zero - "vxorps %%xmm11, %%xmm11, %%xmm11\n\t" // set to zero - "vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero - "vxorps %%xmm13, %%xmm13, %%xmm13\n\t" // set to zero - "vxorps %%xmm14, %%xmm14, %%xmm14\n\t" // set to zero - "vxorps %%xmm15, %%xmm15, %%xmm15\n\t" // set to zero - ".align 16 \n\t" - ".L01LOOP%=: \n\t" - "vbroadcastss (%%rdi), %%xmm0 \n\t" // load values of c - "nop \n\t" - "leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch - - "prefetcht0 (%%r8)\n\t" // Prefetch - "vfmaddps %%xmm8 , 0*4(%%rsi), %%xmm0, %%xmm8 \n\t" // multiply a and c and add to temp - "prefetcht0 64(%%r8)\n\t" // Prefetch - "vfmaddps %%xmm9 , 4*4(%%rsi), %%xmm0, %%xmm9 \n\t" // multiply a and c and add to temp - "vfmaddps %%xmm10, 8*4(%%rsi), %%xmm0, %%xmm10\n\t" // multiply a and c and add to temp - "vfmaddps %%xmm11, 12*4(%%rsi), %%xmm0, %%xmm11\n\t" // multiply a and c and add to temp - "vfmaddps %%xmm12, 16*4(%%rsi), %%xmm0, %%xmm12\n\t" // multiply a and c and add to temp - "vfmaddps %%xmm13, 20*4(%%rsi), %%xmm0, %%xmm13\n\t" // multiply a and c and add to temp - "vfmaddps %%xmm14, 24*4(%%rsi), %%xmm0, %%xmm14\n\t" // multiply a and c and add to temp - "vfmaddps %%xmm15, 28*4(%%rsi), %%xmm0, %%xmm15\n\t" // multiply a and c and add to temp - - "addq $4 , %%rdi \n\t" // increment pointer of c - "leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a - - "dec %%rax \n\t" // n = n -1 - "jnz .L01LOOP%= \n\t" - - "vmulps %%xmm8 , %%xmm1, %%xmm8 \n\t" // scale by alpha - "vmulps %%xmm9 , %%xmm1, %%xmm9 \n\t" // scale by alpha - "vmulps %%xmm10, %%xmm1, %%xmm10\n\t" // scale by alpha - "vmulps %%xmm11, %%xmm1, %%xmm11\n\t" // scale by alpha - "vmulps %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha - "vmulps %%xmm13, %%xmm1, %%xmm13\n\t" // scale by alpha - "vmulps %%xmm14, %%xmm1, %%xmm14\n\t" // scale by alpha - "vmulps %%xmm15, %%xmm1, %%xmm15\n\t" // scale by alpha - - "vmovups %%xmm8 , (%%rdx) \n\t" // store temp -> y - "vmovups %%xmm9 , 4*4(%%rdx) \n\t" // store temp -> y - "vmovups %%xmm10, 8*4(%%rdx) \n\t" // store temp -> y - "vmovups %%xmm11, 12*4(%%rdx) \n\t" // store temp -> y - "vmovups %%xmm12, 16*4(%%rdx) \n\t" // store temp -> y - "vmovups %%xmm13, 20*4(%%rdx) \n\t" // store temp -> y - "vmovups %%xmm14, 24*4(%%rdx) \n\t" // store temp -> y - "vmovups %%xmm15, 28*4(%%rdx) \n\t" // store temp -> y - - : - : - "m" (n), // 0 - "m" (alpha), // 1 - "m" (a), // 2 - "m" (lda), // 3 - "m" (x), // 4 - "m" (y), // 5 - "m" (pre) // 6 - ); - -} - -static void sgemv_kernel_16( long n, float alpha, float *a, long lda, float *x, float *y) -{ - - float *pre = a + lda*3; - - __asm__ __volatile__ - ( - "movq %0, %%rax\n\t" // n -> rax - "vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1 - "movq %2, %%rsi\n\t" // adress of a -> rsi - "movq %3, %%rcx\n\t" // value of lda > rcx - "movq %4, %%rdi\n\t" // adress of x -> rdi - "movq %5, %%rdx\n\t" // adress of y -> rdx - "movq %6, %%r8\n\t" // address for prefetch - "prefetcht0 (%%r8)\n\t" // Prefetch - - "vxorps %%ymm12, %%ymm12, %%ymm12\n\t" // set to zero - "vxorps %%ymm13, %%ymm13, %%ymm13\n\t" // set to zero - - ".L01LOOP%=: \n\t" - "vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c - "addq $4 , %%rdi \n\t" // increment pointer of c - - "leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch - "prefetcht0 (%%r8)\n\t" // Prefetch - - "vfmaddps %%ymm12, 0*4(%%rsi), %%ymm0, %%ymm12\n\t" // multiply a and c and add to temp - "vfmaddps %%ymm13, 8*4(%%rsi), %%ymm0, %%ymm13\n\t" // multiply a and c and add to temp - - "leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a - - "dec %%rax \n\t" // n = n -1 - "jnz .L01LOOP%= \n\t" - - "vmulps %%ymm12, %%ymm1, %%ymm12\n\t" // scale by alpha - "vmulps %%ymm13, %%ymm1, %%ymm13\n\t" // scale by alpha - - "vmovups %%ymm12, (%%rdx) \n\t" // store temp -> y - "vmovups %%ymm13, 8*4(%%rdx) \n\t" // store temp -> y - - : - : - "m" (n), // 0 - "m" (alpha), // 1 - "m" (a), // 2 - "m" (lda), // 3 - "m" (x), // 4 - "m" (y), // 5 - "m" (pre) // 6 - : "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", - "%xmm0", "%xmm1", - "%xmm8", "%xmm9", "%xmm10", "%xmm11", - "%xmm12", "%xmm13", "%xmm14", "%xmm15", - "memory" - ); - -} - - -static void sgemv_kernel_8( long n, float alpha, float *a, long lda, float *x, float *y) -{ - - - __asm__ __volatile__ - ( - "movq %0, %%rax\n\t" // n -> rax - "vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1 - "movq %2, %%rsi\n\t" // adress of a -> rsi - "movq %3, %%rcx\n\t" // value of lda > rcx - "movq %4, %%rdi\n\t" // adress of x -> rdi - "movq %5, %%rdx\n\t" // adress of y -> rdx - - "vxorps %%ymm12, %%ymm12, %%ymm12\n\t" // set to zero - - ".L01LOOP%=: \n\t" - "vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c - "addq $4 , %%rdi \n\t" // increment pointer of c - - "vfmaddps %%ymm12, 0*4(%%rsi), %%ymm0, %%ymm12\n\t" // multiply a and c and add to temp - - "leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a - - "dec %%rax \n\t" // n = n -1 - "jnz .L01LOOP%= \n\t" - - "vmulps %%ymm12, %%ymm1, %%ymm12\n\t" // scale by alpha - - "vmovups %%ymm12, (%%rdx) \n\t" // store temp -> y - - : - : - "m" (n), // 0 - "m" (alpha), // 1 - "m" (a), // 2 - "m" (lda), // 3 - "m" (x), // 4 - "m" (y) // 5 - : "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", - "%xmm0", "%xmm1", - "%xmm8", "%xmm9", "%xmm10", "%xmm11", - "%xmm12", "%xmm13", "%xmm14", "%xmm15", - "memory" - ); - -} - - -static void sgemv_kernel_4( long n, float alpha, float *a, long lda, float *x, float *y) -{ - - - __asm__ __volatile__ - ( - "movq %0, %%rax\n\t" // n -> rax - "vbroadcastss %1, %%xmm1\n\t" // alpha -> xmm1 - "movq %2, %%rsi\n\t" // adress of a -> rsi - "movq %3, %%rcx\n\t" // value of lda > rcx - "movq %4, %%rdi\n\t" // adress of x -> rdi - "movq %5, %%rdx\n\t" // adress of y -> rdx - - "vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero - - ".L01LOOP%=: \n\t" - "vbroadcastss (%%rdi), %%xmm0 \n\t" // load values of c - "addq $4 , %%rdi \n\t" // increment pointer of c - - "vfmaddps %%xmm12, 0*4(%%rsi), %%xmm0, %%xmm12\n\t" // multiply a and c and add to temp - - "leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a - - "dec %%rax \n\t" // n = n -1 - "jnz .L01LOOP%= \n\t" - - "vmulps %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha - - "vmovups %%xmm12, (%%rdx) \n\t" // store temp -> y - - : - : - "m" (n), // 0 - "m" (alpha), // 1 - "m" (a), // 2 - "m" (lda), // 3 - "m" (x), // 4 - "m" (y) // 5 - : "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", - "%xmm0", "%xmm1", - "%xmm8", "%xmm9", "%xmm10", "%xmm11", - "%xmm12", "%xmm13", "%xmm14", "%xmm15", - "memory" - ); - -} - -static void sgemv_kernel_2( long n, float alpha, float *a, long lda, float *x, float *y) -{ - - - __asm__ __volatile__ - ( - "movq %0, %%rax\n\t" // n -> rax - "vmovss %1, %%xmm1\n\t" // alpha -> xmm1 - "movq %2, %%rsi\n\t" // adress of a -> rsi - "movq %3, %%rcx\n\t" // value of lda > rcx - "movq %4, %%rdi\n\t" // adress of x -> rdi - "movq %5, %%rdx\n\t" // adress of y -> rdx - - "vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero - "vxorps %%xmm13, %%xmm13, %%xmm13\n\t" // set to zero - - ".L01LOOP%=: \n\t" - "vmovss (%%rdi), %%xmm0 \n\t" // load values of c - "addq $4 , %%rdi \n\t" // increment pointer of c - - "vfmaddss %%xmm12, 0*4(%%rsi), %%xmm0, %%xmm12\n\t" // multiply a and c and add to temp - "vfmaddss %%xmm13, 1*4(%%rsi), %%xmm0, %%xmm13\n\t" // multiply a and c and add to temp - - "leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a - - "dec %%rax \n\t" // n = n -1 - "jnz .L01LOOP%= \n\t" - - "vmulss %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha - "vmulss %%xmm13, %%xmm1, %%xmm13\n\t" // scale by alpha - - "vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y - "vmovss %%xmm13, 4(%%rdx) \n\t" // store temp -> y - - : - : - "m" (n), // 0 - "m" (alpha), // 1 - "m" (a), // 2 - "m" (lda), // 3 - "m" (x), // 4 - "m" (y) // 5 - : "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", - "%xmm0", "%xmm1", - "%xmm8", "%xmm9", "%xmm10", "%xmm11", - "%xmm12", "%xmm13", "%xmm14", "%xmm15", - "memory" - ); - -} - - - -static void sgemv_kernel_1( long n, float alpha, float *a, long lda, float *x, float *y) -{ - - - __asm__ __volatile__ - ( - "movq %0, %%rax\n\t" // n -> rax - "vmovss %1, %%xmm1\n\t" // alpha -> xmm1 - "movq %2, %%rsi\n\t" // adress of a -> rsi - "movq %3, %%rcx\n\t" // value of lda > rcx - "movq %4, %%rdi\n\t" // adress of x -> rdi - "movq %5, %%rdx\n\t" // adress of y -> rdx - - "vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero - - ".L01LOOP%=: \n\t" - "vmovss (%%rdi), %%xmm0 \n\t" // load values of c - "addq $4 , %%rdi \n\t" // increment pointer of c - - "vfmaddss %%xmm12, 0*4(%%rsi), %%xmm0, %%xmm12\n\t" // multiply a and c and add to temp - - "leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a - - "dec %%rax \n\t" // n = n -1 - "jnz .L01LOOP%= \n\t" - - "vmulss %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha - - "vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y - - : - : - "m" (n), // 0 - "m" (alpha), // 1 - "m" (a), // 2 - "m" (lda), // 3 - "m" (x), // 4 - "m" (y) // 5 - : "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", - "%xmm0", "%xmm1", - "%xmm8", "%xmm9", "%xmm10", "%xmm11", - "%xmm12", "%xmm13", "%xmm14", "%xmm15", - "memory" - ); - -} - - diff --git a/kernel/x86_64/sgemv_n_microk_haswell.c b/kernel/x86_64/sgemv_n_microk_haswell.c deleted file mode 100644 index 9db3869d2..000000000 --- a/kernel/x86_64/sgemv_n_microk_haswell.c +++ /dev/null @@ -1,461 +0,0 @@ -/*************************************************************************** -Copyright (c) 2014, The OpenBLAS Project -All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: -1. Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in -the documentation and/or other materials provided with the -distribution. -3. Neither the name of the OpenBLAS project nor the names of -its contributors may be used to endorse or promote products -derived from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*****************************************************************************/ - -static void sgemv_kernel_64( long n, float alpha, float *a, long lda, float *x, float *y) -{ - - - float *pre = a + lda*2; - - __asm__ __volatile__ - ( - "movq %0, %%rax\n\t" // n -> rax - "vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1 - "movq %2, %%rsi\n\t" // adress of a -> rsi - "movq %3, %%rcx\n\t" // value of lda > rcx - "movq %4, %%rdi\n\t" // adress of x -> rdi - "movq %5, %%rdx\n\t" // adress of y -> rdx - "movq %6, %%r8\n\t" // address for prefetch - "prefetcht0 (%%r8)\n\t" // Prefetch - "prefetcht0 64(%%r8)\n\t" // Prefetch - - "vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero - "vxorps %%ymm9 , %%ymm9 , %%ymm9 \n\t" // set to zero - "vxorps %%ymm10, %%ymm10, %%ymm10\n\t" // set to zero - "vxorps %%ymm11, %%ymm11, %%ymm11\n\t" // set to zero - "vxorps %%ymm12, %%ymm12, %%ymm12\n\t" // set to zero - "vxorps %%ymm13, %%ymm13, %%ymm13\n\t" // set to zero - "vxorps %%ymm14, %%ymm14, %%ymm14\n\t" // set to zero - "vxorps %%ymm15, %%ymm15, %%ymm15\n\t" // set to zero - ".align 16 \n\t" - ".L01LOOP%=: \n\t" - "vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c - "leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch - - "prefetcht0 (%%r8)\n\t" // Prefetch - "vfmadd231ps 0*4(%%rsi), %%ymm0, %%ymm8 \n\t" // multiply a and c and add to temp - "vfmadd231ps 8*4(%%rsi), %%ymm0, %%ymm9 \n\t" // multiply a and c and add to temp - "prefetcht0 64(%%r8)\n\t" // Prefetch - "vfmadd231ps 16*4(%%rsi), %%ymm0, %%ymm10\n\t" // multiply a and c and add to temp - "vfmadd231ps 24*4(%%rsi), %%ymm0, %%ymm11\n\t" // multiply a and c and add to temp - "prefetcht0 128(%%r8)\n\t" // Prefetch - "vfmadd231ps 32*4(%%rsi), %%ymm0, %%ymm12\n\t" // multiply a and c and add to temp - "vfmadd231ps 40*4(%%rsi), %%ymm0, %%ymm13\n\t" // multiply a and c and add to temp - "prefetcht0 192(%%r8)\n\t" // Prefetch - "vfmadd231ps 48*4(%%rsi), %%ymm0, %%ymm14\n\t" // multiply a and c and add to temp - "vfmadd231ps 56*4(%%rsi), %%ymm0, %%ymm15\n\t" // multiply a and c and add to temp - - "addq $4 , %%rdi \n\t" // increment pointer of c - "leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a - - "dec %%rax \n\t" // n = n -1 - "jnz .L01LOOP%= \n\t" - - "vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha - "vmulps %%ymm9 , %%ymm1, %%ymm9 \n\t" // scale by alpha - "vmulps %%ymm10, %%ymm1, %%ymm10\n\t" // scale by alpha - "vmulps %%ymm11, %%ymm1, %%ymm11\n\t" // scale by alpha - "vmulps %%ymm12, %%ymm1, %%ymm12\n\t" // scale by alpha - "vmulps %%ymm13, %%ymm1, %%ymm13\n\t" // scale by alpha - "vmulps %%ymm14, %%ymm1, %%ymm14\n\t" // scale by alpha - "vmulps %%ymm15, %%ymm1, %%ymm15\n\t" // scale by alpha - - "vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y - "vmovups %%ymm9 , 8*4(%%rdx) \n\t" // store temp -> y - "vmovups %%ymm10, 16*4(%%rdx) \n\t" // store temp -> y - "vmovups %%ymm11, 24*4(%%rdx) \n\t" // store temp -> y - "vmovups %%ymm12, 32*4(%%rdx) \n\t" // store temp -> y - "vmovups %%ymm13, 40*4(%%rdx) \n\t" // store temp -> y - "vmovups %%ymm14, 48*4(%%rdx) \n\t" // store temp -> y - "vmovups %%ymm15, 56*4(%%rdx) \n\t" // store temp -> y - - : - : - "m" (n), // 0 - "m" (alpha), // 1 - "m" (a), // 2 - "m" (lda), // 3 - "m" (x), // 4 - "m" (y), // 5 - "m" (pre) // 6 - : "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc", - "%xmm0", "%xmm1", - "%xmm4", "%xmm5", "%xmm6", "%xmm7", - "%xmm8", "%xmm9", "%xmm10", "%xmm11", - "%xmm12", "%xmm13", "%xmm14", "%xmm15", - "memory" - ); - -} - - - -static void sgemv_kernel_32( long n, float alpha, float *a, long lda, float *x, float *y) -{ - - - float *pre = a + lda*3; - - __asm__ __volatile__ - ( - "movq %0, %%rax\n\t" // n -> rax - "vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1 - "movq %2, %%rsi\n\t" // adress of a -> rsi - "movq %3, %%rcx\n\t" // value of lda > rcx - "movq %4, %%rdi\n\t" // adress of x -> rdi - "movq %5, %%rdx\n\t" // adress of y -> rdx - "movq %6, %%r8\n\t" // address for prefetch - "prefetcht0 (%%r8)\n\t" // Prefetch - "prefetcht0 64(%%r8)\n\t" // Prefetch - - "vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero - "vxorps %%ymm9 , %%ymm9 , %%ymm9 \n\t" // set to zero - "vxorps %%ymm10, %%ymm10, %%ymm10\n\t" // set to zero - "vxorps %%ymm11, %%ymm11, %%ymm11\n\t" // set to zero - ".align 16 \n\t" - ".L01LOOP%=: \n\t" - "vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c - "nop \n\t" - "leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch - - "prefetcht0 (%%r8)\n\t" // Prefetch - "prefetcht0 64(%%r8)\n\t" // Prefetch - - "vmulps 0*4(%%rsi), %%ymm0, %%ymm4 \n\t" // multiply a and c and add to temp - "vmulps 8*4(%%rsi), %%ymm0, %%ymm5 \n\t" // multiply a and c and add to temp - "vmulps 16*4(%%rsi), %%ymm0, %%ymm6 \n\t" // multiply a and c and add to temp - "vmulps 24*4(%%rsi), %%ymm0, %%ymm7 \n\t" // multiply a and c and add to temp - - "vaddps %%ymm8 , %%ymm4, %%ymm8 \n\t" // multiply a and c and add to temp - "vaddps %%ymm9 , %%ymm5, %%ymm9 \n\t" // multiply a and c and add to temp - "vaddps %%ymm10, %%ymm6, %%ymm10\n\t" // multiply a and c and add to temp - "vaddps %%ymm11, %%ymm7, %%ymm11\n\t" // multiply a and c and add to temp - - - - "addq $4 , %%rdi \n\t" // increment pointer of c - "leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a - - "dec %%rax \n\t" // n = n -1 - "jnz .L01LOOP%= \n\t" - - "vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha - "vmulps %%ymm9 , %%ymm1, %%ymm9 \n\t" // scale by alpha - "vmulps %%ymm10, %%ymm1, %%ymm10\n\t" // scale by alpha - "vmulps %%ymm11, %%ymm1, %%ymm11\n\t" // scale by alpha - - "vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y - "vmovups %%ymm9 , 8*4(%%rdx) \n\t" // store temp -> y - "vmovups %%ymm10, 16*4(%%rdx) \n\t" // store temp -> y - "vmovups %%ymm11, 24*4(%%rdx) \n\t" // store temp -> y - - : - : - "m" (n), // 0 - "m" (alpha), // 1 - "m" (a), // 2 - "m" (lda), // 3 - "m" (x), // 4 - "m" (y), // 5 - "m" (pre) // 6 - : "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc", - "%xmm0", "%xmm1", - "%xmm4", "%xmm5", "%xmm6", "%xmm7", - "%xmm8", "%xmm9", "%xmm10", "%xmm11", - "memory" - ); - - - -} - -static void sgemv_kernel_16( long n, float alpha, float *a, long lda, float *x, float *y) -{ - - float *pre = a + lda*3; - - __asm__ __volatile__ - ( - "movq %0, %%rax\n\t" // n -> rax - "vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1 - "movq %2, %%rsi\n\t" // adress of a -> rsi - "movq %3, %%rcx\n\t" // value of lda > rcx - "movq %4, %%rdi\n\t" // adress of x -> rdi - "movq %5, %%rdx\n\t" // adress of y -> rdx - "movq %6, %%r8\n\t" // address for prefetch - "prefetcht0 (%%r8)\n\t" // Prefetch - "prefetcht0 64(%%r8)\n\t" // Prefetch - - "vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero - "vxorps %%ymm9 , %%ymm9 , %%ymm9 \n\t" // set to zero - ".align 16 \n\t" - ".L01LOOP%=: \n\t" - "vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c - "nop \n\t" - "leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch - - "prefetcht0 (%%r8)\n\t" // Prefetch - - "vmulps 0*4(%%rsi), %%ymm0, %%ymm4 \n\t" // multiply a and c and add to temp - "vmulps 8*4(%%rsi), %%ymm0, %%ymm5 \n\t" // multiply a and c and add to temp - - "vaddps %%ymm8 , %%ymm4, %%ymm8 \n\t" // multiply a and c and add to temp - "vaddps %%ymm9 , %%ymm5, %%ymm9 \n\t" // multiply a and c and add to temp - - "addq $4 , %%rdi \n\t" // increment pointer of c - "leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a - - "dec %%rax \n\t" // n = n -1 - "jnz .L01LOOP%= \n\t" - - "vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha - "vmulps %%ymm9 , %%ymm1, %%ymm9 \n\t" // scale by alpha - - "vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y - "vmovups %%ymm9 , 8*4(%%rdx) \n\t" // store temp -> y - - : - : - "m" (n), // 0 - "m" (alpha), // 1 - "m" (a), // 2 - "m" (lda), // 3 - "m" (x), // 4 - "m" (y), // 5 - "m" (pre) // 6 - : "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc", - "%xmm0", "%xmm1", - "%xmm4", "%xmm5", "%xmm6", "%xmm7", - "%xmm8", "%xmm9", "%xmm10", "%xmm11", - "memory" - ); - - -} - - -static void sgemv_kernel_8( long n, float alpha, float *a, long lda, float *x, float *y) -{ - - __asm__ __volatile__ - ( - "movq %0, %%rax\n\t" // n -> rax - "vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1 - "movq %2, %%rsi\n\t" // adress of a -> rsi - "movq %3, %%rcx\n\t" // value of lda > rcx - "movq %4, %%rdi\n\t" // adress of x -> rdi - "movq %5, %%rdx\n\t" // adress of y -> rdx - - "vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero - ".align 16 \n\t" - ".L01LOOP%=: \n\t" - "vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c - - "vmulps 0*4(%%rsi), %%ymm0, %%ymm4 \n\t" // multiply a and c and add to temp - "vaddps %%ymm8 , %%ymm4, %%ymm8 \n\t" // multiply a and c and add to temp - - "addq $4 , %%rdi \n\t" // increment pointer of c - "leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a - - "dec %%rax \n\t" // n = n -1 - "jnz .L01LOOP%= \n\t" - - "vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha - "vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y - - : - : - "m" (n), // 0 - "m" (alpha), // 1 - "m" (a), // 2 - "m" (lda), // 3 - "m" (x), // 4 - "m" (y) // 5 - : "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc", - "%xmm0", "%xmm1", - "%xmm4", "%xmm5", "%xmm6", "%xmm7", - "%xmm8", "%xmm9", "%xmm10", "%xmm11", - "memory" - ); - - -} - - -static void sgemv_kernel_4( long n, float alpha, float *a, long lda, float *x, float *y) -{ - - - __asm__ __volatile__ - ( - "movq %0, %%rax\n\t" // n -> rax - "vbroadcastss %1, %%xmm1\n\t" // alpha -> xmm1 - "movq %2, %%rsi\n\t" // adress of a -> rsi - "movq %3, %%rcx\n\t" // value of lda > rcx - "movq %4, %%rdi\n\t" // adress of x -> rdi - "movq %5, %%rdx\n\t" // adress of y -> rdx - - "vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero - - ".L01LOOP%=: \n\t" - "vbroadcastss (%%rdi), %%xmm0 \n\t" // load values of c - - "vmulps 0*4(%%rsi), %%xmm0, %%xmm4 \n\t" // multiply a and c and add to temp - "vaddps %%xmm12, %%xmm4, %%xmm12 \n\t" // multiply a and c and add to temp - - "addq $4 , %%rdi \n\t" // increment pointer of c - "leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a - - "dec %%rax \n\t" // n = n -1 - "jnz .L01LOOP%= \n\t" - - "vmulps %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha - - "vmovups %%xmm12, (%%rdx) \n\t" // store temp -> y - - : - : - "m" (n), // 0 - "m" (alpha), // 1 - "m" (a), // 2 - "m" (lda), // 3 - "m" (x), // 4 - "m" (y) // 5 - : "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", - "%xmm0", "%xmm1", - "%xmm8", "%xmm9", "%xmm10", "%xmm11", - "%xmm12", "%xmm13", "%xmm14", "%xmm15", - "memory" - ); - -} - -static void sgemv_kernel_2( long n, float alpha, float *a, long lda, float *x, float *y) -{ - - - __asm__ __volatile__ - ( - "movq %0, %%rax\n\t" // n -> rax - "vmovss %1, %%xmm1\n\t" // alpha -> xmm1 - "movq %2, %%rsi\n\t" // adress of a -> rsi - "movq %3, %%rcx\n\t" // value of lda > rcx - "movq %4, %%rdi\n\t" // adress of x -> rdi - "movq %5, %%rdx\n\t" // adress of y -> rdx - - "vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero - "vxorps %%xmm13, %%xmm13, %%xmm13\n\t" // set to zero - - ".L01LOOP%=: \n\t" - "vmovss (%%rdi), %%xmm0 \n\t" // load values of c - - "vmulps 0*4(%%rsi), %%xmm0, %%xmm4 \n\t" // multiply a and c and add to temp - "vmulps 1*4(%%rsi), %%xmm0, %%xmm5 \n\t" // multiply a and c and add to temp - - "vaddps %%xmm12, %%xmm4, %%xmm12 \n\t" // multiply a and c and add to temp - "vaddps %%xmm13, %%xmm5, %%xmm13 \n\t" // multiply a and c and add to temp - - "addq $4 , %%rdi \n\t" // increment pointer of c - "leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a - - "dec %%rax \n\t" // n = n -1 - "jnz .L01LOOP%= \n\t" - - "vmulss %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha - "vmulss %%xmm13, %%xmm1, %%xmm13\n\t" // scale by alpha - - "vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y - "vmovss %%xmm13, 4(%%rdx) \n\t" // store temp -> y - - : - : - "m" (n), // 0 - "m" (alpha), // 1 - "m" (a), // 2 - "m" (lda), // 3 - "m" (x), // 4 - "m" (y) // 5 - : "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", - "%xmm0", "%xmm1", - "%xmm8", "%xmm9", "%xmm10", "%xmm11", - "%xmm12", "%xmm13", "%xmm14", "%xmm15", - "memory" - ); - -} - - - -static void sgemv_kernel_1( long n, float alpha, float *a, long lda, float *x, float *y) -{ - - - __asm__ __volatile__ - ( - "movq %0, %%rax\n\t" // n -> rax - "vmovss %1, %%xmm1\n\t" // alpha -> xmm1 - "movq %2, %%rsi\n\t" // adress of a -> rsi - "movq %3, %%rcx\n\t" // value of lda > rcx - "movq %4, %%rdi\n\t" // adress of x -> rdi - "movq %5, %%rdx\n\t" // adress of y -> rdx - - "vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero - - ".L01LOOP%=: \n\t" - "vmovss (%%rdi), %%xmm0 \n\t" // load values of c - "addq $4 , %%rdi \n\t" // increment pointer of c - - "vmulss 0*4(%%rsi), %%xmm0, %%xmm4 \n\t" // multiply a and c and add to temp - "vaddss %%xmm12, %%xmm4, %%xmm12 \n\t" // multiply a and c and add to temp - - "leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a - - "dec %%rax \n\t" // n = n -1 - "jnz .L01LOOP%= \n\t" - - "vmulss %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha - - "vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y - - : - : - "m" (n), // 0 - "m" (alpha), // 1 - "m" (a), // 2 - "m" (lda), // 3 - "m" (x), // 4 - "m" (y) // 5 - : "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", - "%xmm0", "%xmm1", - "%xmm8", "%xmm9", "%xmm10", "%xmm11", - "%xmm12", "%xmm13", "%xmm14", "%xmm15", - "memory" - ); - -} - - diff --git a/kernel/x86_64/sgemv_n_microk_sandy.c b/kernel/x86_64/sgemv_n_microk_sandy.c deleted file mode 100644 index 9bdb06600..000000000 --- a/kernel/x86_64/sgemv_n_microk_sandy.c +++ /dev/null @@ -1,473 +0,0 @@ -/*************************************************************************** -Copyright (c) 2014, The OpenBLAS Project -All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: -1. Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in -the documentation and/or other materials provided with the -distribution. -3. Neither the name of the OpenBLAS project nor the names of -its contributors may be used to endorse or promote products -derived from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*****************************************************************************/ - -static void sgemv_kernel_64( long n, float alpha, float *a, long lda, float *x, float *y) -{ - - - float *pre = a + lda*2; - - __asm__ __volatile__ - ( - "movq %0, %%rax\n\t" // n -> rax - "vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1 - "movq %2, %%rsi\n\t" // adress of a -> rsi - "movq %3, %%rcx\n\t" // value of lda > rcx - "movq %4, %%rdi\n\t" // adress of x -> rdi - "movq %5, %%rdx\n\t" // adress of y -> rdx - "movq %6, %%r8\n\t" // address for prefetch - "prefetcht0 (%%r8)\n\t" // Prefetch - "prefetcht0 64(%%r8)\n\t" // Prefetch - - "vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero - "vxorps %%ymm9 , %%ymm9 , %%ymm9 \n\t" // set to zero - "vxorps %%ymm10, %%ymm10, %%ymm10\n\t" // set to zero - "vxorps %%ymm11, %%ymm11, %%ymm11\n\t" // set to zero - "vxorps %%ymm12, %%ymm12, %%ymm12\n\t" // set to zero - "vxorps %%ymm13, %%ymm13, %%ymm13\n\t" // set to zero - "vxorps %%ymm14, %%ymm14, %%ymm14\n\t" // set to zero - "vxorps %%ymm15, %%ymm15, %%ymm15\n\t" // set to zero - ".align 16 \n\t" - ".L01LOOP%=: \n\t" - "vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c - "nop \n\t" - "leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch - - "prefetcht0 (%%r8)\n\t" // Prefetch - "vmulps 0*4(%%rsi), %%ymm0, %%ymm4 \n\t" // multiply a and c and add to temp - "vmulps 8*4(%%rsi), %%ymm0, %%ymm5 \n\t" // multiply a and c and add to temp - "prefetcht0 64(%%r8)\n\t" // Prefetch - "vmulps 16*4(%%rsi), %%ymm0, %%ymm6 \n\t" // multiply a and c and add to temp - "vmulps 24*4(%%rsi), %%ymm0, %%ymm7 \n\t" // multiply a and c and add to temp - - "vaddps %%ymm8 , %%ymm4, %%ymm8 \n\t" // multiply a and c and add to temp - "vaddps %%ymm9 , %%ymm5, %%ymm9 \n\t" // multiply a and c and add to temp - "prefetcht0 128(%%r8)\n\t" // Prefetch - "vaddps %%ymm10, %%ymm6, %%ymm10\n\t" // multiply a and c and add to temp - "vaddps %%ymm11, %%ymm7, %%ymm11\n\t" // multiply a and c and add to temp - - "prefetcht0 192(%%r8)\n\t" // Prefetch - "vmulps 32*4(%%rsi), %%ymm0, %%ymm4 \n\t" // multiply a and c and add to temp - "vmulps 40*4(%%rsi), %%ymm0, %%ymm5 \n\t" // multiply a and c and add to temp - "vmulps 48*4(%%rsi), %%ymm0, %%ymm6 \n\t" // multiply a and c and add to temp - "vmulps 56*4(%%rsi), %%ymm0, %%ymm7 \n\t" // multiply a and c and add to temp - - "vaddps %%ymm12, %%ymm4, %%ymm12\n\t" // multiply a and c and add to temp - "vaddps %%ymm13, %%ymm5, %%ymm13\n\t" // multiply a and c and add to temp - "vaddps %%ymm14, %%ymm6, %%ymm14\n\t" // multiply a and c and add to temp - "vaddps %%ymm15, %%ymm7, %%ymm15\n\t" // multiply a and c and add to temp - - "addq $4 , %%rdi \n\t" // increment pointer of c - "leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a - - "dec %%rax \n\t" // n = n -1 - "jnz .L01LOOP%= \n\t" - - "vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha - "vmulps %%ymm9 , %%ymm1, %%ymm9 \n\t" // scale by alpha - "vmulps %%ymm10, %%ymm1, %%ymm10\n\t" // scale by alpha - "vmulps %%ymm11, %%ymm1, %%ymm11\n\t" // scale by alpha - "vmulps %%ymm12, %%ymm1, %%ymm12\n\t" // scale by alpha - "vmulps %%ymm13, %%ymm1, %%ymm13\n\t" // scale by alpha - "vmulps %%ymm14, %%ymm1, %%ymm14\n\t" // scale by alpha - "vmulps %%ymm15, %%ymm1, %%ymm15\n\t" // scale by alpha - - "vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y - "vmovups %%ymm9 , 8*4(%%rdx) \n\t" // store temp -> y - "vmovups %%ymm10, 16*4(%%rdx) \n\t" // store temp -> y - "vmovups %%ymm11, 24*4(%%rdx) \n\t" // store temp -> y - "vmovups %%ymm12, 32*4(%%rdx) \n\t" // store temp -> y - "vmovups %%ymm13, 40*4(%%rdx) \n\t" // store temp -> y - "vmovups %%ymm14, 48*4(%%rdx) \n\t" // store temp -> y - "vmovups %%ymm15, 56*4(%%rdx) \n\t" // store temp -> y - - : - : - "m" (n), // 0 - "m" (alpha), // 1 - "m" (a), // 2 - "m" (lda), // 3 - "m" (x), // 4 - "m" (y), // 5 - "m" (pre) // 6 - : "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc", - "%xmm0", "%xmm1", - "%xmm4", "%xmm5", "%xmm6", "%xmm7", - "%xmm8", "%xmm9", "%xmm10", "%xmm11", - "%xmm12", "%xmm13", "%xmm14", "%xmm15", - "memory" - ); - -} - - - -static void sgemv_kernel_32( long n, float alpha, float *a, long lda, float *x, float *y) -{ - - - float *pre = a + lda*3; - - __asm__ __volatile__ - ( - "movq %0, %%rax\n\t" // n -> rax - "vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1 - "movq %2, %%rsi\n\t" // adress of a -> rsi - "movq %3, %%rcx\n\t" // value of lda > rcx - "movq %4, %%rdi\n\t" // adress of x -> rdi - "movq %5, %%rdx\n\t" // adress of y -> rdx - "movq %6, %%r8\n\t" // address for prefetch - "prefetcht0 (%%r8)\n\t" // Prefetch - "prefetcht0 64(%%r8)\n\t" // Prefetch - - "vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero - "vxorps %%ymm9 , %%ymm9 , %%ymm9 \n\t" // set to zero - "vxorps %%ymm10, %%ymm10, %%ymm10\n\t" // set to zero - "vxorps %%ymm11, %%ymm11, %%ymm11\n\t" // set to zero - ".align 16 \n\t" - ".L01LOOP%=: \n\t" - "vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c - "nop \n\t" - "leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch - - "prefetcht0 (%%r8)\n\t" // Prefetch - "prefetcht0 64(%%r8)\n\t" // Prefetch - - "vmulps 0*4(%%rsi), %%ymm0, %%ymm4 \n\t" // multiply a and c and add to temp - "vmulps 8*4(%%rsi), %%ymm0, %%ymm5 \n\t" // multiply a and c and add to temp - "vmulps 16*4(%%rsi), %%ymm0, %%ymm6 \n\t" // multiply a and c and add to temp - "vmulps 24*4(%%rsi), %%ymm0, %%ymm7 \n\t" // multiply a and c and add to temp - - "vaddps %%ymm8 , %%ymm4, %%ymm8 \n\t" // multiply a and c and add to temp - "vaddps %%ymm9 , %%ymm5, %%ymm9 \n\t" // multiply a and c and add to temp - "vaddps %%ymm10, %%ymm6, %%ymm10\n\t" // multiply a and c and add to temp - "vaddps %%ymm11, %%ymm7, %%ymm11\n\t" // multiply a and c and add to temp - - - - "addq $4 , %%rdi \n\t" // increment pointer of c - "leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a - - "dec %%rax \n\t" // n = n -1 - "jnz .L01LOOP%= \n\t" - - "vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha - "vmulps %%ymm9 , %%ymm1, %%ymm9 \n\t" // scale by alpha - "vmulps %%ymm10, %%ymm1, %%ymm10\n\t" // scale by alpha - "vmulps %%ymm11, %%ymm1, %%ymm11\n\t" // scale by alpha - - "vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y - "vmovups %%ymm9 , 8*4(%%rdx) \n\t" // store temp -> y - "vmovups %%ymm10, 16*4(%%rdx) \n\t" // store temp -> y - "vmovups %%ymm11, 24*4(%%rdx) \n\t" // store temp -> y - - : - : - "m" (n), // 0 - "m" (alpha), // 1 - "m" (a), // 2 - "m" (lda), // 3 - "m" (x), // 4 - "m" (y), // 5 - "m" (pre) // 6 - : "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc", - "%xmm0", "%xmm1", - "%xmm4", "%xmm5", "%xmm6", "%xmm7", - "%xmm8", "%xmm9", "%xmm10", "%xmm11", - "memory" - ); - - - -} - -static void sgemv_kernel_16( long n, float alpha, float *a, long lda, float *x, float *y) -{ - - float *pre = a + lda*3; - - __asm__ __volatile__ - ( - "movq %0, %%rax\n\t" // n -> rax - "vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1 - "movq %2, %%rsi\n\t" // adress of a -> rsi - "movq %3, %%rcx\n\t" // value of lda > rcx - "movq %4, %%rdi\n\t" // adress of x -> rdi - "movq %5, %%rdx\n\t" // adress of y -> rdx - "movq %6, %%r8\n\t" // address for prefetch - "prefetcht0 (%%r8)\n\t" // Prefetch - "prefetcht0 64(%%r8)\n\t" // Prefetch - - "vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero - "vxorps %%ymm9 , %%ymm9 , %%ymm9 \n\t" // set to zero - ".align 16 \n\t" - ".L01LOOP%=: \n\t" - "vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c - "nop \n\t" - "leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch - - "prefetcht0 (%%r8)\n\t" // Prefetch - - "vmulps 0*4(%%rsi), %%ymm0, %%ymm4 \n\t" // multiply a and c and add to temp - "vmulps 8*4(%%rsi), %%ymm0, %%ymm5 \n\t" // multiply a and c and add to temp - - "vaddps %%ymm8 , %%ymm4, %%ymm8 \n\t" // multiply a and c and add to temp - "vaddps %%ymm9 , %%ymm5, %%ymm9 \n\t" // multiply a and c and add to temp - - "addq $4 , %%rdi \n\t" // increment pointer of c - "leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a - - "dec %%rax \n\t" // n = n -1 - "jnz .L01LOOP%= \n\t" - - "vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha - "vmulps %%ymm9 , %%ymm1, %%ymm9 \n\t" // scale by alpha - - "vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y - "vmovups %%ymm9 , 8*4(%%rdx) \n\t" // store temp -> y - - : - : - "m" (n), // 0 - "m" (alpha), // 1 - "m" (a), // 2 - "m" (lda), // 3 - "m" (x), // 4 - "m" (y), // 5 - "m" (pre) // 6 - : "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc", - "%xmm0", "%xmm1", - "%xmm4", "%xmm5", "%xmm6", "%xmm7", - "%xmm8", "%xmm9", "%xmm10", "%xmm11", - "memory" - ); - - -} - - -static void sgemv_kernel_8( long n, float alpha, float *a, long lda, float *x, float *y) -{ - - __asm__ __volatile__ - ( - "movq %0, %%rax\n\t" // n -> rax - "vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1 - "movq %2, %%rsi\n\t" // adress of a -> rsi - "movq %3, %%rcx\n\t" // value of lda > rcx - "movq %4, %%rdi\n\t" // adress of x -> rdi - "movq %5, %%rdx\n\t" // adress of y -> rdx - - "vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero - ".align 16 \n\t" - ".L01LOOP%=: \n\t" - "vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c - - "vmulps 0*4(%%rsi), %%ymm0, %%ymm4 \n\t" // multiply a and c and add to temp - "vaddps %%ymm8 , %%ymm4, %%ymm8 \n\t" // multiply a and c and add to temp - - "addq $4 , %%rdi \n\t" // increment pointer of c - "leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a - - "dec %%rax \n\t" // n = n -1 - "jnz .L01LOOP%= \n\t" - - "vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha - "vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y - - : - : - "m" (n), // 0 - "m" (alpha), // 1 - "m" (a), // 2 - "m" (lda), // 3 - "m" (x), // 4 - "m" (y) // 5 - : "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc", - "%xmm0", "%xmm1", - "%xmm4", "%xmm5", "%xmm6", "%xmm7", - "%xmm8", "%xmm9", "%xmm10", "%xmm11", - "memory" - ); - - -} - - -static void sgemv_kernel_4( long n, float alpha, float *a, long lda, float *x, float *y) -{ - - - __asm__ __volatile__ - ( - "movq %0, %%rax\n\t" // n -> rax - "vbroadcastss %1, %%xmm1\n\t" // alpha -> xmm1 - "movq %2, %%rsi\n\t" // adress of a -> rsi - "movq %3, %%rcx\n\t" // value of lda > rcx - "movq %4, %%rdi\n\t" // adress of x -> rdi - "movq %5, %%rdx\n\t" // adress of y -> rdx - - "vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero - - ".L01LOOP%=: \n\t" - "vbroadcastss (%%rdi), %%xmm0 \n\t" // load values of c - - "vmulps 0*4(%%rsi), %%xmm0, %%xmm4 \n\t" // multiply a and c and add to temp - "vaddps %%xmm12, %%xmm4, %%xmm12 \n\t" // multiply a and c and add to temp - - "addq $4 , %%rdi \n\t" // increment pointer of c - "leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a - - "dec %%rax \n\t" // n = n -1 - "jnz .L01LOOP%= \n\t" - - "vmulps %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha - - "vmovups %%xmm12, (%%rdx) \n\t" // store temp -> y - - : - : - "m" (n), // 0 - "m" (alpha), // 1 - "m" (a), // 2 - "m" (lda), // 3 - "m" (x), // 4 - "m" (y) // 5 - : "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", - "%xmm0", "%xmm1", - "%xmm8", "%xmm9", "%xmm10", "%xmm11", - "%xmm12", "%xmm13", "%xmm14", "%xmm15", - "memory" - ); - -} - -static void sgemv_kernel_2( long n, float alpha, float *a, long lda, float *x, float *y) -{ - - - __asm__ __volatile__ - ( - "movq %0, %%rax\n\t" // n -> rax - "vmovss %1, %%xmm1\n\t" // alpha -> xmm1 - "movq %2, %%rsi\n\t" // adress of a -> rsi - "movq %3, %%rcx\n\t" // value of lda > rcx - "movq %4, %%rdi\n\t" // adress of x -> rdi - "movq %5, %%rdx\n\t" // adress of y -> rdx - - "vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero - "vxorps %%xmm13, %%xmm13, %%xmm13\n\t" // set to zero - - ".L01LOOP%=: \n\t" - "vmovss (%%rdi), %%xmm0 \n\t" // load values of c - - "vmulps 0*4(%%rsi), %%xmm0, %%xmm4 \n\t" // multiply a and c and add to temp - "vmulps 1*4(%%rsi), %%xmm0, %%xmm5 \n\t" // multiply a and c and add to temp - - "vaddps %%xmm12, %%xmm4, %%xmm12 \n\t" // multiply a and c and add to temp - "vaddps %%xmm13, %%xmm5, %%xmm13 \n\t" // multiply a and c and add to temp - - "addq $4 , %%rdi \n\t" // increment pointer of c - "leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a - - "dec %%rax \n\t" // n = n -1 - "jnz .L01LOOP%= \n\t" - - "vmulss %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha - "vmulss %%xmm13, %%xmm1, %%xmm13\n\t" // scale by alpha - - "vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y - "vmovss %%xmm13, 4(%%rdx) \n\t" // store temp -> y - - : - : - "m" (n), // 0 - "m" (alpha), // 1 - "m" (a), // 2 - "m" (lda), // 3 - "m" (x), // 4 - "m" (y) // 5 - : "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", - "%xmm0", "%xmm1", - "%xmm8", "%xmm9", "%xmm10", "%xmm11", - "%xmm12", "%xmm13", "%xmm14", "%xmm15", - "memory" - ); - -} - - - -static void sgemv_kernel_1( long n, float alpha, float *a, long lda, float *x, float *y) -{ - - - __asm__ __volatile__ - ( - "movq %0, %%rax\n\t" // n -> rax - "vmovss %1, %%xmm1\n\t" // alpha -> xmm1 - "movq %2, %%rsi\n\t" // adress of a -> rsi - "movq %3, %%rcx\n\t" // value of lda > rcx - "movq %4, %%rdi\n\t" // adress of x -> rdi - "movq %5, %%rdx\n\t" // adress of y -> rdx - - "vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero - - ".L01LOOP%=: \n\t" - "vmovss (%%rdi), %%xmm0 \n\t" // load values of c - "addq $4 , %%rdi \n\t" // increment pointer of c - - "vmulss 0*4(%%rsi), %%xmm0, %%xmm4 \n\t" // multiply a and c and add to temp - "vaddss %%xmm12, %%xmm4, %%xmm12 \n\t" // multiply a and c and add to temp - - "leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a - - "dec %%rax \n\t" // n = n -1 - "jnz .L01LOOP%= \n\t" - - "vmulss %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha - - "vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y - - : - : - "m" (n), // 0 - "m" (alpha), // 1 - "m" (a), // 2 - "m" (lda), // 3 - "m" (x), // 4 - "m" (y) // 5 - : "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", - "%xmm0", "%xmm1", - "%xmm8", "%xmm9", "%xmm10", "%xmm11", - "%xmm12", "%xmm13", "%xmm14", "%xmm15", - "memory" - ); - -} - - diff --git a/kernel/x86_64/sgemv_t_avx.c b/kernel/x86_64/sgemv_t_avx.c deleted file mode 100644 index 55fb3d623..000000000 --- a/kernel/x86_64/sgemv_t_avx.c +++ /dev/null @@ -1,232 +0,0 @@ -/*************************************************************************** -Copyright (c) 2014, The OpenBLAS Project -All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: -1. Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in -the documentation and/or other materials provided with the -distribution. -3. Neither the name of the OpenBLAS project nor the names of -its contributors may be used to endorse or promote products -derived from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*****************************************************************************/ - - -#include "common.h" - -#if defined(BULLDOZER) || defined(PILEDRIVER) -#include "sgemv_t_microk_bulldozer.c" -#elif defined(HASWELL) -#include "sgemv_t_microk_haswell.c" -#else -#include "sgemv_t_microk_sandy.c" -#endif - -static void copy_x(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_src) -{ - BLASLONG i; - for ( i=0; i= 16 ) - { - if ( m2 & Mblock) - { - - if ( inc_x == 1 ) - xbuffer = x_ptr; - else - copy_x(Mblock,x_ptr,xbuffer,inc_x); - - y_ptr = y; - a_ptrl = a_ptr; - - for(i = 0; i rax - "vmovss %1, %%xmm1\n\t" // alpha -> xmm1 - "movq %2, %%rsi\n\t" // adress of a -> rsi - "movq %3, %%rcx\n\t" // value of lda > rcx - "movq %4, %%rdi\n\t" // adress of x -> rdi - "movq %5, %%rdx\n\t" // adress of y -> rdx - - "leaq (, %%rcx,4), %%rcx \n\t" // scale lda by size of float - "leaq (%%rsi,%%rcx,1), %%r8 \n\t" // pointer to next line - - "vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero - "vxorps %%xmm13, %%xmm13, %%xmm13\n\t" // set to zero - "vxorps %%xmm14, %%xmm14, %%xmm14\n\t" // set to zero - "vxorps %%xmm15, %%xmm15, %%xmm15\n\t" // set to zero - - "sarq $4, %%rax \n\t" // n = n / 16 - - ".align 16 \n\t" - ".L01LOOP%=: \n\t" - // "prefetcht0 512(%%rsi) \n\t" - "prefetcht0 (%%r8) \n\t" //prefetch next line of a - "vmovups (%%rsi), %%xmm4 \n\t" - "vmovups 4*4(%%rsi), %%xmm5 \n\t" - "vmovups 8*4(%%rsi), %%xmm6 \n\t" - "vmovups 12*4(%%rsi), %%xmm7 \n\t" - - "vfmaddps %%xmm12, 0*4(%%rdi), %%xmm4, %%xmm12\n\t" // multiply a and c and add to temp - "vfmaddps %%xmm13, 4*4(%%rdi), %%xmm5, %%xmm13\n\t" // multiply a and c and add to temp - "vfmaddps %%xmm14, 8*4(%%rdi), %%xmm6, %%xmm14\n\t" // multiply a and c and add to temp - "vfmaddps %%xmm15, 12*4(%%rdi), %%xmm7, %%xmm15\n\t" // multiply a and c and add to temp - - "addq $16*4 , %%r8 \n\t" // increment prefetch pointer - "addq $16*4 , %%rsi \n\t" // increment pointer of a - "addq $16*4 , %%rdi \n\t" // increment pointer of c - "dec %%rax \n\t" // n = n -1 - "jnz .L01LOOP%= \n\t" - - "vaddps %%xmm12, %%xmm14, %%xmm12\n\t" - "vaddps %%xmm13, %%xmm15, %%xmm13\n\t" - "vaddps %%xmm12, %%xmm13, %%xmm12\n\t" - "vhaddps %%xmm12, %%xmm12, %%xmm12\n\t" - "vhaddps %%xmm12, %%xmm12, %%xmm12\n\t" - - "vfmaddss (%%rdx), %%xmm12, %%xmm1, %%xmm12\n\t" - "vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y - - : - : - "m" (n), // 0 - "m" (alpha), // 1 - "m" (a), // 2 - "m" (lda), // 3 - "m" (x), // 4 - "m" (y) // 5 - : "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", - "%xmm0", "%xmm1", - "%xmm4", "%xmm5", "%xmm6", "%xmm7", - "%xmm12", "%xmm13", "%xmm14", "%xmm15", - "memory" - ); - -} - - - diff --git a/kernel/x86_64/sgemv_t_microk_haswell.c b/kernel/x86_64/sgemv_t_microk_haswell.c deleted file mode 100644 index ecb9845bb..000000000 --- a/kernel/x86_64/sgemv_t_microk_haswell.c +++ /dev/null @@ -1,100 +0,0 @@ -/*************************************************************************** -Copyright (c) 2014, The OpenBLAS Project -All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: -1. Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in -the documentation and/or other materials provided with the -distribution. -3. Neither the name of the OpenBLAS project nor the names of -its contributors may be used to endorse or promote products -derived from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*****************************************************************************/ - -static void sgemv_kernel_16( long n, float alpha, float *a, long lda, float *x, float *y) -{ - - //n = n / 16; - - __asm__ __volatile__ - ( - "movq %0, %%rax\n\t" // n -> rax - "vmovss %1, %%xmm1\n\t" // alpha -> xmm1 - "movq %2, %%rsi\n\t" // adress of a -> rsi - "movq %3, %%rcx\n\t" // value of lda > rcx - "movq %4, %%rdi\n\t" // adress of x -> rdi - "movq %5, %%rdx\n\t" // adress of y -> rdx - - "leaq (, %%rcx,4), %%rcx \n\t" // scale lda by size of float - "leaq (%%rsi,%%rcx,1), %%r8 \n\t" // pointer to next line - - "vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero - "vxorps %%xmm13, %%xmm13, %%xmm13\n\t" // set to zero - "vxorps %%xmm14, %%xmm14, %%xmm14\n\t" // set to zero - "vxorps %%xmm15, %%xmm15, %%xmm15\n\t" // set to zero - - "sarq $4, %%rax \n\t" // n = n / 16 - - ".align 16 \n\t" - ".L01LOOP%=: \n\t" - // "prefetcht0 512(%%rsi) \n\t" - "prefetcht0 (%%r8) \n\t" //prefetch next line of a - "vmovups (%%rsi), %%xmm4 \n\t" - "vmovups 4*4(%%rsi), %%xmm5 \n\t" - "vmovups 8*4(%%rsi), %%xmm6 \n\t" - "vmovups 12*4(%%rsi), %%xmm7 \n\t" - - "vfmadd231ps 0*4(%%rdi), %%xmm4, %%xmm12\n\t" // multiply a and c and add to temp - "vfmadd231ps 4*4(%%rdi), %%xmm5, %%xmm13\n\t" // multiply a and c and add to temp - "vfmadd231ps 8*4(%%rdi), %%xmm6, %%xmm14\n\t" // multiply a and c and add to temp - "vfmadd231ps 12*4(%%rdi), %%xmm7, %%xmm15\n\t" // multiply a and c and add to temp - - "addq $16*4 , %%r8 \n\t" // increment prefetch pointer - "addq $16*4 , %%rsi \n\t" // increment pointer of a - "addq $16*4 , %%rdi \n\t" // increment pointer of c - "dec %%rax \n\t" // n = n -1 - "jnz .L01LOOP%= \n\t" - - "vaddps %%xmm12, %%xmm14, %%xmm12\n\t" - "vaddps %%xmm13, %%xmm15, %%xmm13\n\t" - "vaddps %%xmm12, %%xmm13, %%xmm12\n\t" - "vhaddps %%xmm12, %%xmm12, %%xmm12\n\t" - "vhaddps %%xmm12, %%xmm12, %%xmm12\n\t" - - "vmulss %%xmm12, %%xmm1, %%xmm12\n\t" - "vaddss (%%rdx), %%xmm12,%%xmm12\n\t" - "vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y - - : - : - "m" (n), // 0 - "m" (alpha), // 1 - "m" (a), // 2 - "m" (lda), // 3 - "m" (x), // 4 - "m" (y) // 5 - : "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", - "%xmm0", "%xmm1", - "%xmm4", "%xmm5", "%xmm6", "%xmm7", - "%xmm12", "%xmm13", "%xmm14", "%xmm15", - "memory" - ); - -} - - - diff --git a/kernel/x86_64/sgemv_t_microk_sandy.c b/kernel/x86_64/sgemv_t_microk_sandy.c deleted file mode 100644 index 4ecd6d3d0..000000000 --- a/kernel/x86_64/sgemv_t_microk_sandy.c +++ /dev/null @@ -1,106 +0,0 @@ -/*************************************************************************** -Copyright (c) 2014, The OpenBLAS Project -All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: -1. Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in -the documentation and/or other materials provided with the -distribution. -3. Neither the name of the OpenBLAS project nor the names of -its contributors may be used to endorse or promote products -derived from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*****************************************************************************/ - -static void sgemv_kernel_16( long n, float alpha, float *a, long lda, float *x, float *y) -{ - - //n = n / 16; - - __asm__ __volatile__ - ( - "movq %0, %%rax\n\t" // n -> rax - "vmovss %1, %%xmm1\n\t" // alpha -> xmm1 - "movq %2, %%rsi\n\t" // adress of a -> rsi - "movq %3, %%rcx\n\t" // value of lda > rcx - "movq %4, %%rdi\n\t" // adress of x -> rdi - "movq %5, %%rdx\n\t" // adress of y -> rdx - - "leaq (, %%rcx,4), %%rcx \n\t" // scale lda by size of float - "leaq (%%rsi,%%rcx,1), %%r8 \n\t" // pointer to next line - - "vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero - "vxorps %%xmm13, %%xmm13, %%xmm13\n\t" // set to zero - "vxorps %%xmm14, %%xmm14, %%xmm14\n\t" // set to zero - "vxorps %%xmm15, %%xmm15, %%xmm15\n\t" // set to zero - - "sarq $4, %%rax \n\t" // n = n / 16 - - ".align 16 \n\t" - ".L01LOOP%=: \n\t" - // "prefetcht0 512(%%rsi) \n\t" - "prefetcht0 (%%r8) \n\t" //prefetch next line of a - "vmovups (%%rsi), %%xmm4 \n\t" - "vmovups 4*4(%%rsi), %%xmm5 \n\t" - "vmovups 8*4(%%rsi), %%xmm6 \n\t" - "vmovups 12*4(%%rsi), %%xmm7 \n\t" - - "vmulps 0*4(%%rdi), %%xmm4, %%xmm8 \n\t" // multiply a and c and add to temp - "vmulps 4*4(%%rdi), %%xmm5, %%xmm9 \n\t" // multiply a and c and add to temp - "vmulps 8*4(%%rdi), %%xmm6, %%xmm10\n\t" // multiply a and c and add to temp - "vmulps 12*4(%%rdi), %%xmm7, %%xmm11\n\t" // multiply a and c and add to temp - - "vaddps %%xmm12, %%xmm8 , %%xmm12\n\t" - "vaddps %%xmm13, %%xmm9 , %%xmm13\n\t" - "vaddps %%xmm14, %%xmm10, %%xmm14\n\t" - "vaddps %%xmm15, %%xmm11, %%xmm15\n\t" - - "addq $16*4 , %%r8 \n\t" // increment prefetch pointer - "addq $16*4 , %%rsi \n\t" // increment pointer of a - "addq $16*4 , %%rdi \n\t" // increment pointer of c - "dec %%rax \n\t" // n = n -1 - "jnz .L01LOOP%= \n\t" - - "vaddps %%xmm12, %%xmm14, %%xmm12\n\t" - "vaddps %%xmm13, %%xmm15, %%xmm13\n\t" - "vaddps %%xmm12, %%xmm13, %%xmm12\n\t" - "vhaddps %%xmm12, %%xmm12, %%xmm12\n\t" - "vhaddps %%xmm12, %%xmm12, %%xmm12\n\t" - - "vmulss %%xmm12, %%xmm1, %%xmm12 \n\t" - "vaddss (%%rdx), %%xmm12, %%xmm12\n\t" - "vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y - - : - : - "m" (n), // 0 - "m" (alpha), // 1 - "m" (a), // 2 - "m" (lda), // 3 - "m" (x), // 4 - "m" (y) // 5 - : "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc", - "%xmm0", "%xmm1", - "%xmm4", "%xmm5", "%xmm6", "%xmm7", - "%xmm8", "%xmm9", "%xmm10", "%xmm11", - "%xmm12", "%xmm13", "%xmm14", "%xmm15", - "memory" - ); - -} - - -