floraachy/OpenBLAS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into betterPowerGEMVTail

Browse Source
This commit is contained in:
Chip Kerchner
2024-08-01 14:59:12 -05:00
parent e2334d0218 9afd0c8afd
commit 1a7b8c650d
75 changed files with 7473 additions and 1323 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
kernel/arm/scal.c
View File

@@ -43,9 +43,22 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS
if ( (n <= 0) || (inc_x <= 0))
return(0);
if (dummy2 == 0) {
while(j < n)
{
while(j < n)
{
if ( da == 0.0 )
x[i]=0.0;
else
x[i] = da * x[i] ;
i += inc_x ;
j++;
}
} else {
while(j < n)
{
if ( da == 0.0 )
if (!isnan(x[i]) && !isinf(x[i])) {
@@ -59,6 +72,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS
i += inc_x ;
j++;
}
}
return 0;

5
kernel/arm64/KERNEL.A64FX
View File

@@ -1 +1,6 @@
include $(KERNELDIR)/KERNEL.ARMV8SVE
SGEMVNKERNEL = gemv_n_sve.c
DGEMVNKERNEL = gemv_n_sve.c
SGEMVTKERNEL = gemv_t_sve.c
DGEMVTKERNEL = gemv_t_sve.c

20
kernel/arm64/KERNEL.ARMV8SVE
View File

@@ -131,6 +131,16 @@ SGEMMITCOPYOBJ = sgemm_itcopy$(TSUFFIX).$(SUFFIX)
SGEMMONCOPYOBJ = sgemm_oncopy$(TSUFFIX).$(SUFFIX)
SGEMMOTCOPYOBJ = sgemm_otcopy$(TSUFFIX).$(SUFFIX)
SGEMM_SMALL_M_PERMIT = gemm_small_kernel_permit_sve.c
SGEMM_SMALL_K_NT = sgemm_small_kernel_nt_sve.c
SGEMM_SMALL_K_B0_NT = sgemm_small_kernel_nt_sve.c
SGEMM_SMALL_K_NN = sgemm_small_kernel_nn_sve.c
SGEMM_SMALL_K_B0_NN = sgemm_small_kernel_nn_sve.c
SGEMM_SMALL_K_TT = sgemm_small_kernel_tt_sve.c
SGEMM_SMALL_K_B0_TT = sgemm_small_kernel_tt_sve.c
SGEMM_SMALL_K_TN = sgemm_small_kernel_tn_sve.c
SGEMM_SMALL_K_B0_TN = sgemm_small_kernel_tn_sve.c
STRMMUNCOPY_M = trmm_uncopy_sve_v1.c
STRMMLNCOPY_M = trmm_lncopy_sve_v1.c
STRMMUTCOPY_M = trmm_utcopy_sve_v1.c
@@ -152,6 +162,16 @@ DGEMMITCOPYOBJ = dgemm_itcopy$(TSUFFIX).$(SUFFIX)
DGEMMONCOPYOBJ = dgemm_oncopy$(TSUFFIX).$(SUFFIX)
DGEMMOTCOPYOBJ = dgemm_otcopy$(TSUFFIX).$(SUFFIX)
DGEMM_SMALL_M_PERMIT = gemm_small_kernel_permit_sve.c
DGEMM_SMALL_K_NT = dgemm_small_kernel_nt_sve.c
DGEMM_SMALL_K_B0_NT = dgemm_small_kernel_nt_sve.c
DGEMM_SMALL_K_NN = dgemm_small_kernel_nn_sve.c
DGEMM_SMALL_K_B0_NN = dgemm_small_kernel_nn_sve.c
DGEMM_SMALL_K_TT = dgemm_small_kernel_tt_sve.c
DGEMM_SMALL_K_B0_TT = dgemm_small_kernel_tt_sve.c
DGEMM_SMALL_K_TN = dgemm_small_kernel_tn_sve.c
DGEMM_SMALL_K_B0_TN = dgemm_small_kernel_tn_sve.c
DTRMMUNCOPY_M = trmm_uncopy_sve_v1.c
DTRMMLNCOPY_M = trmm_lncopy_sve_v1.c
DTRMMUTCOPY_M = trmm_utcopy_sve_v1.c

3
kernel/arm64/KERNEL.NEOVERSEV1
View File

@@ -1 +1,4 @@
include $(KERNELDIR)/KERNEL.ARMV8SVE
SGEMVTKERNEL = gemv_t_sve.c
DGEMVTKERNEL = gemv_t_sve.c

742
kernel/arm64/dgemm_small_kernel_nn_sve.c Normal file
View File

@@ -0,0 +1,742 @@
/***************************************************************************
Copyright (c) 2024, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#include "common.h"
#include <arm_neon.h>
#include <arm_sve.h>
#if defined(__ARM_NEON_SVE_BRIDGE) && defined(__has_include) && \
__has_include(<arm_neon_sve_bridge.h>)
#include <arm_neon_sve_bridge.h>
#else
#define svdup_neonq_f32(fixed_reg) \
({ \
svfloat32_t scalable_reg; \
asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \
scalable_reg; \
})
#define svdup_neonq_f64(fixed_reg) \
({ \
svfloat64_t scalable_reg; \
asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \
scalable_reg; \
})
#endif
#define RESET_A_POINTER() a_offset = A;
#define CREATE_A_POINTER(m, scale) FLOAT* a_offset##m = a_offset + scale;
#define UPDATE_A_POINTER(scale) a_offset = a_offset + scale;
#define A_ELEMENT_K(m, offset_k) *(a_offset##m + (k + offset_k) * lda)
#define A_ELEMENT(m) A_ELEMENT_K(m, 0)
#define RESET_B_POINTER() b_offset = B;
#define CREATE_B_POINTER(n, scale) FLOAT* b_offset##n = b_offset + scale * ldb;
#define UPDATE_B_POINTER(scale) b_offset = b_offset + scale * ldb;
#define B_ELEMENT_K(n, offset_k) *(b_offset##n + (k + offset_k))
#define B_ELEMENT(n) B_ELEMENT_K(n, 0)
#define CREATE_C_POINTER(n, scale) FLOAT* c_offset##n = c_offset + scale * ldc;
#define INCR_C_POINTER(m, incr) // c_offset ## m += incr;
#define UPDATE_C_POINTER(scale) c_offset = c_offset + scale * ldc;
#define C_ELEMENT(m, n) *(c_offset##n + ((m * v_size) + i))
// #undef C_ELEMENT
// #define C_ELEMENT(m, n) C[(i+(m))+(j+(n))*ldc]
#define PACK_ELEMENT_K(n, offset_k) packed_b[(k + offset_k) * 4 + n]
#define PACK_ELEMENT(n) PACK_ELEMENT_K(n, 0)
// ASIMD
#define DECLARE_RESULT_VECTOR2(m, n) \
float64x2_t result##m##n = vdupq_n_f64(0.0);
#define DECLARE_RESULT(m, n) float64_t result##m##n = 0.0;
#define BROADCAST_LOAD_A2(m, offset_k) \
float64x2_t a##m##_k##offset_k = vld1q_dup_f64(&A_ELEMENT_K(m, offset_k));
#define LOAD_A1(m, offset_k) \
float64_t a##m##_k##offset_k = A_ELEMENT_K(m, offset_k);
#define VECTOR_LOAD_B_K2(n, offset_k) \
float64x2_t b##k##n##_k##offset_k = vld1q_f64(&B_ELEMENT_K(n, offset_k));
#define TRANSPOSE_B2_K2(n0, n1, offset_k0, offset_k1) \
float64x2_t b##n0##_k##offset_k0 = \
vzip1q_f64(b##k##n0##_k##offset_k0, b##k##n1##_k##offset_k0); \
float64x2_t b##n0##_k##offset_k1 = \
vzip2q_f64(b##k##n0##_k##offset_k0, b##k##n1##_k##offset_k0);
#define SCALE_B2_K2(n0, offset_k0, offset_k1) \
svfloat64_t b##s##n0##_k##offset_k0 = svdup_neonq_f64(b##n0##_k##offset_k0); \
svfloat64_t b##s##n0##_k##offset_k1 = svdup_neonq_f64(b##n0##_k##offset_k1);
#define GATHER_LOAD_B2(n, offset_k) \
float64x2_t b##n##_k##offset_k = vdupq_n_f64(B_ELEMENT_K(n, offset_k)); \
b##n##_k##offset_k = \
vsetq_lane_f64(B_ELEMENT_K(n + 1, offset_k), b##n##_k##offset_k, 1);
#define VECTOR_UNPACK_B2(n, offset_k) \
float64x2_t b##n##_k##offset_k = vld1q_f64(&PACK_ELEMENT_K(n, offset_k));
#define VECTOR_PACK_B2(n, offset_k) \
vst1q_f64(&PACK_ELEMENT_K(n, offset_k), b##n##_k##offset_k);
#define PACK_B0(n, offset_k) \
PACK_ELEMENT_K(n, offset_k) = vget_lane_f64(b##n##_k##offset_k, 0);
#define UPDATE_RESULT_VECTOR2(m, n, offset_k) \
result##m##n = \
vfmaq_f64(result##m##n, a##m##_k##offset_k, b##n##_k##offset_k);
#define UPDATE_RESULT(m, n, offset_k) \
result##m##n = result##m##n + a##m##_k##offset_k * b##n##_k##offset_k;
#ifdef B0
#define SCATTER_STORE2(m, n) \
result##m##n = vmulq_f64(result##m##n, vdupq_n_f64(alpha)); \
C_ELEMENT(m, n + 0) = vgetq_lane_f64(result##m##n, 0); \
C_ELEMENT(m, n + 1) = vgetq_lane_f64(result##m##n, 1);
#else
#define SCATTER_STORE2(m, n) \
result##m##n = vmulq_f64(result##m##n, vdupq_n_f64(alpha)); \
C_ELEMENT(m, n + 0) = \
C_ELEMENT(m, n + 0) * beta + vgetq_lane_f64(result##m##n, 0); \
C_ELEMENT(m, n + 1) = \
C_ELEMENT(m, n + 1) * beta + vgetq_lane_f64(result##m##n, 1);
#endif
// SVE
#define DECLARE_RESULT_VECTOR(m, n) svfloat64_t result##m##n = svdup_f64(0.0);
#define BROADCAST_LOAD_A(m, offset_k) \
svfloat64_t a##s##m##_k##offset_k = svdup_f64(A_ELEMENT_K(m, offset_k));
#define BROADCAST_LOAD_B(n, offset_k) \
svfloat64_t b##s##n##_k##offset_k = svdup_f64(B_ELEMENT_K(n, offset_k));
#define VECTOR_LOAD_A(pg, m, offset_k) \
svfloat64_t a##s##m##_k##offset_k = svld1(pg, &A_ELEMENT_K(m, offset_k));
#define QUADWORD_LOAD_B(n, offset_k) \
svfloat64_t b##s##n##_k##offset_k = \
svld1rq(pg_true, &B_ELEMENT_K(n, offset_k));
#define PACK_B(n, offset_k) \
svst1(pg_first, &PACK_ELEMENT_K(n, offset_k), b##s##n##_k##offset_k);
#define VECTOR_PACK_B(n, offset_k) \
svst1(pg_true, &PACK_ELEMENT_K(n* v_size, offset_k), b##s##n##_k##offset_k);
#define QUADWORD_PACK_B(n, offset_k) \
svst1(pg_quad, &PACK_ELEMENT_K(n, offset_k), b##s##n##_k##offset_k);
#define UNPACK_VECTOR_B(n, offset_k) \
svfloat64_t b##s##n##_k##offset_k = \
svld1(pg_true, &PACK_ELEMENT_K(n * v_size, offset_k));
#define UNPACK_BROADCAST_B(n, offset_k) \
svfloat64_t b##s##n##_k##offset_k = svdup_f64(PACK_ELEMENT_K(n, offset_k));
#define UNPACK_QUADWORD_B(n, offset_k) \
svfloat64_t b##s##n##_k##offset_k = \
svld1rq(pg_true, &PACK_ELEMENT_K(n, offset_k));
#define UPDATE_RESULT_VECTOR(pg, m, n, offset_k) \
result##m##n = \
svmla_m(pg, result##m##n, a##s##m##_k##offset_k, b##s##n##_k##offset_k);
#define UPDATE_RESULT_VECTOR_QUADWORD(m, n, outer, lane, offset_k) \
result##m##n = svmla_lane( \
result##m##n, a##s##m##_k##offset_k, b##s##outer##_k##offset_k, lane);
#ifdef B0
#define VECTOR_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
svst1(pg, &C_ELEMENT(m, n), result##m##n);
#define SCATTER_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
#else
#define VECTOR_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
result##m##n = \
svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(m, n)), beta_vec); \
svst1(pg, &C_ELEMENT(m, n), result##m##n);
#define SCATTER_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
result##m##n = svmla_m(pg, \
result##m##n, \
svld1_gather_index(pg, &C_ELEMENT(m, n), ldc_vec), \
beta_vec); \
svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
#endif
#ifndef LIKELY
#ifdef __GNUC__
#define LIKELY(x) __builtin_expect(!!(x), 1)
#else
#define LIKELY(x) (x)
#endif
#endif
#ifdef B0
int
CNAME(BLASLONG M,
BLASLONG N,
BLASLONG K,
IFLOAT* A,
BLASLONG lda,
FLOAT alpha,
IFLOAT* B,
BLASLONG ldb,
FLOAT* C,
BLASLONG ldc)
#else
int
CNAME(BLASLONG M,
BLASLONG N,
BLASLONG K,
IFLOAT* A,
BLASLONG lda,
FLOAT alpha,
IFLOAT* B,
BLASLONG ldb,
FLOAT beta,
FLOAT* C,
BLASLONG ldc)
#endif
{
const uint64_t v_size = svcntd();
const uint64_t v_size2 = v_size * 2;
const svbool_t pg_true = svptrue_b64();
const svbool_t pg_quad = svwhilelt_b64(0, 2);
const svbool_t pg_first = svwhilelt_b64(0, 1);
const svfloat64_t alpha_vec = svdup_f64(alpha);
#ifndef B0
const svfloat64_t beta_vec = svdup_f64(beta);
#endif
const BLASLONG n4 = N & -4;
const BLASLONG n2 = N & -2;
const BLASLONG v_m2 = M & -v_size2;
const BLASLONG v_m1 = M & -v_size;
const BLASLONG k2 = K & -2;
const int pack_b = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0;
FLOAT* packed_b =
(pack_b) ? packed_b = (FLOAT*)malloc(K * 4 * sizeof(FLOAT)) : NULL;
FLOAT* b_offset = B;
FLOAT* a_offset = A;
FLOAT* c_offset = C;
BLASLONG j = 0;
for (; j < n4; j += 4) {
CREATE_C_POINTER(0, 0);
CREATE_C_POINTER(1, 1);
CREATE_C_POINTER(2, 2);
CREATE_C_POINTER(3, 3);
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
CREATE_B_POINTER(2, 2);
CREATE_B_POINTER(3, 3);
BLASLONG i = 0;
for (; i < v_m2; i += v_size2) {
CREATE_A_POINTER(0, 0);
CREATE_A_POINTER(1, v_size);
UPDATE_A_POINTER(v_size2);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
DECLARE_RESULT_VECTOR(1, 0);
DECLARE_RESULT_VECTOR(1, 1);
DECLARE_RESULT_VECTOR(1, 2);
DECLARE_RESULT_VECTOR(1, 3);
if (LIKELY(packed_b != NULL)) {
if (i == 0) {
for (; k < k2; k += 2) {
VECTOR_LOAD_B_K2(0, 0);
VECTOR_LOAD_B_K2(1, 0);
TRANSPOSE_B2_K2(0, 1, 0, 1);
SCALE_B2_K2(0, 0, 1);
VECTOR_PACK_B2(0, 0);
VECTOR_PACK_B2(0, 1);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
VECTOR_LOAD_A(pg_true, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1);
VECTOR_LOAD_B_K2(2, 0);
VECTOR_LOAD_B_K2(3, 0);
TRANSPOSE_B2_K2(2, 3, 0, 1);
SCALE_B2_K2(2, 0, 1);
VECTOR_PACK_B2(2, 0);
VECTOR_PACK_B2(2, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 1);
VECTOR_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 0);
VECTOR_LOAD_A(pg_true, 1, 1);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 1);
}
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
PACK_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
PACK_B(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
VECTOR_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0);
BROADCAST_LOAD_B(2, 0);
PACK_B(2, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0);
BROADCAST_LOAD_B(3, 0);
PACK_B(3, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0);
}
} else {
for (; k < K; k++) {
UNPACK_QUADWORD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UNPACK_QUADWORD_B(2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0);
VECTOR_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 0);
}
}
} else {
for (; k < k2; k += 2) {
VECTOR_LOAD_B_K2(0, 0);
VECTOR_LOAD_B_K2(1, 0);
TRANSPOSE_B2_K2(0, 1, 0, 1);
SCALE_B2_K2(0, 0, 1);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
VECTOR_LOAD_A(pg_true, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1);
VECTOR_LOAD_B_K2(2, 0);
VECTOR_LOAD_B_K2(3, 0);
TRANSPOSE_B2_K2(2, 3, 0, 1);
SCALE_B2_K2(2, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 1);
VECTOR_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 0);
VECTOR_LOAD_A(pg_true, 1, 1);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 1);
}
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
VECTOR_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0);
BROADCAST_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0);
BROADCAST_LOAD_B(3, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0);
}
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 0, 2);
VECTOR_STORE(pg_true, 0, 3);
VECTOR_STORE(pg_true, 1, 0);
VECTOR_STORE(pg_true, 1, 1);
VECTOR_STORE(pg_true, 1, 2);
VECTOR_STORE(pg_true, 1, 3);
INCR_C_POINTER(0, v_size2);
INCR_C_POINTER(1, v_size2);
INCR_C_POINTER(2, v_size2);
INCR_C_POINTER(3, v_size2);
}
for (; i < v_m1; i += v_size) {
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(v_size);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
if (LIKELY(packed_b != NULL)) {
for (; k < K; k++) {
UNPACK_QUADWORD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UNPACK_QUADWORD_B(2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0);
}
} else {
for (; k < k2; k += 2) {
VECTOR_LOAD_B_K2(0, 0);
VECTOR_LOAD_B_K2(1, 0);
TRANSPOSE_B2_K2(0, 1, 0, 1);
SCALE_B2_K2(0, 0, 1);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
VECTOR_LOAD_A(pg_true, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1);
VECTOR_LOAD_B_K2(2, 0);
VECTOR_LOAD_B_K2(3, 0);
TRANSPOSE_B2_K2(2, 3, 0, 1);
SCALE_B2_K2(2, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 1);
}
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
BROADCAST_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0);
BROADCAST_LOAD_B(3, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0);
}
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 0, 2);
VECTOR_STORE(pg_true, 0, 3);
INCR_C_POINTER(0, v_size);
INCR_C_POINTER(1, v_size);
INCR_C_POINTER(2, v_size);
INCR_C_POINTER(3, v_size);
}
for (; i < M; i += v_size) {
const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M));
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(0);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
if (LIKELY(packed_b != NULL)) {
for (; k < K; k++) {
UNPACK_QUADWORD_B(0, 0);
VECTOR_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UNPACK_QUADWORD_B(2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0);
}
} else {
for (; k < k2; k += 2) {
VECTOR_LOAD_B_K2(0, 0);
VECTOR_LOAD_B_K2(1, 0);
TRANSPOSE_B2_K2(0, 1, 0, 1);
SCALE_B2_K2(0, 0, 1);
VECTOR_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
VECTOR_LOAD_A(pg_tail, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1);
VECTOR_LOAD_B_K2(2, 0);
VECTOR_LOAD_B_K2(3, 0);
TRANSPOSE_B2_K2(2, 3, 0, 1);
SCALE_B2_K2(2, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 1);
}
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 1, 0);
BROADCAST_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 2, 0);
BROADCAST_LOAD_B(3, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 3, 0);
}
}
VECTOR_STORE(pg_tail, 0, 0);
VECTOR_STORE(pg_tail, 0, 1);
VECTOR_STORE(pg_tail, 0, 2);
VECTOR_STORE(pg_tail, 0, 3);
INCR_C_POINTER(0, 0);
INCR_C_POINTER(1, 0);
INCR_C_POINTER(2, 0);
INCR_C_POINTER(3, 0);
}
UPDATE_B_POINTER(4);
RESET_A_POINTER();
UPDATE_C_POINTER(4);
}
for (; j < n2; j += 2) {
CREATE_C_POINTER(0, 0);
CREATE_C_POINTER(1, 1);
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
BLASLONG i = 0;
for (; i < v_m2; i += v_size2) {
CREATE_A_POINTER(0, 0);
CREATE_A_POINTER(1, v_size);
UPDATE_A_POINTER(v_size2);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(1, 0);
DECLARE_RESULT_VECTOR(1, 1);
for (; k < k2; k += 2) {
VECTOR_LOAD_B_K2(0, 0);
VECTOR_LOAD_B_K2(1, 0);
TRANSPOSE_B2_K2(0, 1, 0, 1);
SCALE_B2_K2(0, 0, 1);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
VECTOR_LOAD_A(pg_true, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1);
VECTOR_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
VECTOR_LOAD_A(pg_true, 1, 1);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1);
}
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
VECTOR_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0);
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 1, 0);
VECTOR_STORE(pg_true, 1, 1);
INCR_C_POINTER(0, v_size2);
INCR_C_POINTER(1, v_size2);
}
for (; i < v_m1; i += v_size) {
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(v_size);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
for (; k < k2; k += 2) {
VECTOR_LOAD_B_K2(0, 0);
VECTOR_LOAD_B_K2(1, 0);
TRANSPOSE_B2_K2(0, 1, 0, 1);
SCALE_B2_K2(0, 0, 1);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
VECTOR_LOAD_A(pg_true, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1);
}
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
INCR_C_POINTER(0, v_size);
INCR_C_POINTER(1, v_size);
}
for (; i < M; i += v_size) {
const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M));
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(0);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
for (; k < k2; k += 2) {
VECTOR_LOAD_B_K2(0, 0);
VECTOR_LOAD_B_K2(1, 0);
TRANSPOSE_B2_K2(0, 1, 0, 1);
SCALE_B2_K2(0, 0, 1);
VECTOR_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
VECTOR_LOAD_A(pg_tail, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1);
}
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 1, 0);
}
VECTOR_STORE(pg_tail, 0, 0);
VECTOR_STORE(pg_tail, 0, 1);
INCR_C_POINTER(0, 0);
INCR_C_POINTER(1, 0);
}
UPDATE_B_POINTER(2);
RESET_A_POINTER();
UPDATE_C_POINTER(2);
}
for (; j < N; j++) {
CREATE_C_POINTER(0, 0);
CREATE_B_POINTER(0, 0);
BLASLONG i = 0;
for (; i < v_m2; i += v_size2) {
CREATE_A_POINTER(0, 0);
CREATE_A_POINTER(1, v_size);
UPDATE_A_POINTER(v_size2);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(1, 0);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
VECTOR_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 1, 0);
INCR_C_POINTER(0, v_size2);
}
for (; i < v_m1; i += v_size) {
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(v_size);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
}
VECTOR_STORE(pg_true, 0, 0);
INCR_C_POINTER(0, v_size);
}
for (; i < M; i += v_size) {
const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M));
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(0);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0);
}
VECTOR_STORE(pg_tail, 0, 0);
INCR_C_POINTER(0, 0);
}
UPDATE_B_POINTER(1);
RESET_A_POINTER();
UPDATE_C_POINTER(1);
}
if (pack_b)
free(packed_b);
return 0;
}

474
kernel/arm64/dgemm_small_kernel_nt_sve.c Normal file
View File

@@ -0,0 +1,474 @@
/***************************************************************************
Copyright (c) 2024, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#include "common.h"
#include <arm_neon.h>
#include <arm_sve.h>
#if defined(__ARM_NEON_SVE_BRIDGE) && defined(__has_include) && \
__has_include(<arm_neon_sve_bridge.h>)
#include <arm_neon_sve_bridge.h>
#else
#define svdup_neonq_f32(fixed_reg) \
({ \
svfloat32_t scalable_reg; \
asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \
scalable_reg; \
})
#define svdup_neonq_f64(fixed_reg) \
({ \
svfloat64_t scalable_reg; \
asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \
scalable_reg; \
})
#endif
#define RESET_A_POINTER() a_offset = A;
#define CREATE_A_POINTER(m, scale) FLOAT* a_offset##m = a_offset + scale;
#define UPDATE_A_POINTER(scale) a_offset = a_offset + scale;
#define A_ELEMENT_K(m, offset_k) *(a_offset##m + (k + offset_k) * lda)
#define A_ELEMENT(m) A_ELEMENT_K(m, 0)
#define RESET_B_POINTER() b_offset = B;
#define CREATE_B_POINTER(n, scale) FLOAT* b_offset##n = b_offset + scale;
#define UPDATE_B_POINTER(scale) b_offset = b_offset + scale;
#define B_ELEMENT_K(n, offset_k) *(b_offset##n + (k + offset_k) * ldb)
#define B_ELEMENT(n) B_ELEMENT_K(n, 0)
#define CREATE_C_POINTER(n, scale) FLOAT* c_offset##n = c_offset + scale * ldc;
#define INCR_C_POINTER(m, incr) // c_offset ## m += incr;
#define UPDATE_C_POINTER(scale) c_offset = c_offset + scale * ldc;
#define C_ELEMENT(m, n) *(c_offset##n + ((m * v_size) + i))
// #undef C_ELEMENT
// #define C_ELEMENT(m, n) C[(i+(m))+(j+(n))*ldc]
#define PACK_ELEMENT_K(n, offset_k) packed_b[(k + offset_k) * 4 + n]
#define PACK_ELEMENT(n) PACK_ELEMENT_K(n, 0)
// ASIMD
#define DECLARE_RESULT_VECTOR2(m, n) \
float64x2_t result##m##n = vdupq_n_f64(0.0);
#define DECLARE_RESULT(m, n) float64_t result##m##n = 0.0;
#define BROADCAST_LOAD_A2(m, offset_k) \
float64x2_t a##m##_k##offset_k = vld1q_dup_f64(&A_ELEMENT_K(m, offset_k));
#define LOAD_A1(m, offset_k) \
float64_t a##m##_k##offset_k = A_ELEMENT_K(m, offset_k);
#define VECTOR_LOAD_B2(n, offset_k) \
float64x2_t b##n##_k##offset_k = vld1q_f64(&B_ELEMENT_K(n, offset_k));
#define GATHER_LOAD_B2(n, offset_k) \
float64x2_t b##n##_k##offset_k = vdupq_n_f64(B_ELEMENT_K(n, offset_k)); \
b##n##_k##offset_k = \
vsetq_lane_f64(B_ELEMENT_K(n + 1, offset_k), b##n##_k##offset_k, 1);
#define UPDATE_RESULT_VECTOR2(m, n, offset_k) \
result##m##n = \
vfmaq_f64(result##m##n, a##m##_k##offset_k, b##n##_k##offset_k);
#define UPDATE_RESULT(m, n, offset_k) \
result##m##n = result##m##n + a##m##_k##offset_k * b##n##_k##offset_k;
#ifdef B0
#define SCATTER_STORE2(m, n) \
result##m##n = vmulq_f64(result##m##n, vdupq_n_f64(alpha)); \
C_ELEMENT(m, n + 0) = vgetq_lane_f64(result##m##n, 0); \
C_ELEMENT(m, n + 1) = vgetq_lane_f64(result##m##n, 1);
#else
#define SCATTER_STORE2(m, n) \
result##m##n = vmulq_f64(result##m##n, vdupq_n_f64(alpha)); \
C_ELEMENT(m, n + 0) = \
C_ELEMENT(m, n + 0) * beta + vgetq_lane_f64(result##m##n, 0); \
C_ELEMENT(m, n + 1) = \
C_ELEMENT(m, n + 1) * beta + vgetq_lane_f64(result##m##n, 1);
#endif
// SVE
#define DECLARE_RESULT_VECTOR(m, n) svfloat64_t result##m##n = svdup_f64(0.0);
#define BROADCAST_LOAD_A(m, offset_k) \
svfloat64_t a##s##m##_k##offset_k = svdup_f64(A_ELEMENT_K(m, offset_k));
#define BROADCAST_LOAD_B(n, offset_k) \
svfloat64_t b##s##n##_k##offset_k = svdup_f64(B_ELEMENT_K(n, offset_k));
#define VECTOR_LOAD_A(pg, m, offset_k) \
svfloat64_t a##s##m##_k##offset_k = svld1(pg, &A_ELEMENT_K(m, offset_k));
#define QUADWORD_LOAD_B(n, offset_k) \
svfloat64_t b##s##n##_k##offset_k = \
svld1rq(pg_true, &B_ELEMENT_K(n, offset_k));
#define UPDATE_RESULT_VECTOR(pg, m, n, offset_k) \
result##m##n = \
svmla_m(pg, result##m##n, a##s##m##_k##offset_k, b##s##n##_k##offset_k);
#define UPDATE_RESULT_VECTOR_QUADWORD(m, n, outer, lane, offset_k) \
result##m##n = svmla_lane( \
result##m##n, a##s##m##_k##offset_k, b##s##outer##_k##offset_k, lane);
#ifdef B0
#define VECTOR_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
svst1(pg, &C_ELEMENT(m, n), result##m##n);
#define SCATTER_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
#else
#define VECTOR_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
result##m##n = \
svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(m, n)), beta_vec); \
svst1(pg, &C_ELEMENT(m, n), result##m##n);
#define SCATTER_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
result##m##n = svmla_m(pg, \
result##m##n, \
svld1_gather_index(pg, &C_ELEMENT(m, n), ldc_vec), \
beta_vec); \
svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
#endif
#ifndef LIKELY
#ifdef __GNUC__
#define LIKELY(x) __builtin_expect(!!(x), 1)
#else
#define LIKELY(x) (x)
#endif
#endif
#ifdef B0
int
CNAME(BLASLONG M,
BLASLONG N,
BLASLONG K,
IFLOAT* A,
BLASLONG lda,
FLOAT alpha,
IFLOAT* B,
BLASLONG ldb,
FLOAT* C,
BLASLONG ldc)
#else
int
CNAME(BLASLONG M,
BLASLONG N,
BLASLONG K,
IFLOAT* A,
BLASLONG lda,
FLOAT alpha,
IFLOAT* B,
BLASLONG ldb,
FLOAT beta,
FLOAT* C,
BLASLONG ldc)
#endif
{
const uint64_t v_size = svcntd();
const uint64_t v_size2 = v_size * 2;
const svbool_t pg_true = svptrue_b64();
const svbool_t pg_quad = svwhilelt_b64(0, 2);
const svfloat64_t alpha_vec = svdup_f64(alpha);
#ifndef B0
const svfloat64_t beta_vec = svdup_f64(beta);
#endif
const BLASLONG n4 = N & -4;
const BLASLONG n2 = N & -2;
const BLASLONG v_m2 = M & -v_size2;
const BLASLONG v_m1 = M & -v_size;
FLOAT* b_offset = B;
FLOAT* a_offset = A;
FLOAT* c_offset = C;
BLASLONG j = 0;
for (; j < n4; j += 4) {
CREATE_C_POINTER(0, 0);
CREATE_C_POINTER(1, 1);
CREATE_C_POINTER(2, 2);
CREATE_C_POINTER(3, 3);
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
CREATE_B_POINTER(2, 2);
CREATE_B_POINTER(3, 3);
BLASLONG i = 0;
for (; i < v_m2; i += v_size2) {
CREATE_A_POINTER(0, 0);
CREATE_A_POINTER(1, v_size);
UPDATE_A_POINTER(v_size2);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
DECLARE_RESULT_VECTOR(1, 0);
DECLARE_RESULT_VECTOR(1, 1);
DECLARE_RESULT_VECTOR(1, 2);
DECLARE_RESULT_VECTOR(1, 3);
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
QUADWORD_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0);
VECTOR_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 0);
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 0, 2);
VECTOR_STORE(pg_true, 0, 3);
VECTOR_STORE(pg_true, 1, 0);
VECTOR_STORE(pg_true, 1, 1);
VECTOR_STORE(pg_true, 1, 2);
VECTOR_STORE(pg_true, 1, 3);
INCR_C_POINTER(0, v_size2);
INCR_C_POINTER(1, v_size2);
INCR_C_POINTER(2, v_size2);
INCR_C_POINTER(3, v_size2);
}
for (; i < v_m1; i += v_size) {
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(v_size);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
QUADWORD_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0);
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 0, 2);
VECTOR_STORE(pg_true, 0, 3);
INCR_C_POINTER(0, v_size);
INCR_C_POINTER(1, v_size);
INCR_C_POINTER(2, v_size);
INCR_C_POINTER(3, v_size);
}
for (; i < M; i += v_size) {
const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M));
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(0);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
QUADWORD_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0);
}
VECTOR_STORE(pg_tail, 0, 0);
VECTOR_STORE(pg_tail, 0, 1);
VECTOR_STORE(pg_tail, 0, 2);
VECTOR_STORE(pg_tail, 0, 3);
INCR_C_POINTER(0, 0);
INCR_C_POINTER(1, 0);
INCR_C_POINTER(2, 0);
INCR_C_POINTER(3, 0);
}
UPDATE_B_POINTER(4);
RESET_A_POINTER();
UPDATE_C_POINTER(4);
}
for (; j < n2; j += 2) {
CREATE_C_POINTER(0, 0);
CREATE_C_POINTER(1, 1);
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
BLASLONG i = 0;
for (; i < v_m2; i += v_size2) {
CREATE_A_POINTER(0, 0);
CREATE_A_POINTER(1, v_size);
UPDATE_A_POINTER(v_size2);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(1, 0);
DECLARE_RESULT_VECTOR(1, 1);
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
VECTOR_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 1, 0);
VECTOR_STORE(pg_true, 1, 1);
INCR_C_POINTER(0, v_size2);
INCR_C_POINTER(1, v_size2);
}
for (; i < v_m1; i += v_size) {
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(v_size);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
INCR_C_POINTER(0, v_size);
INCR_C_POINTER(1, v_size);
}
for (; i < M; i += v_size) {
const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M));
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(0);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
}
VECTOR_STORE(pg_tail, 0, 0);
VECTOR_STORE(pg_tail, 0, 1);
INCR_C_POINTER(0, 0);
INCR_C_POINTER(1, 0);
}
UPDATE_B_POINTER(2);
RESET_A_POINTER();
UPDATE_C_POINTER(2);
}
for (; j < N; j++) {
CREATE_C_POINTER(0, 0);
CREATE_B_POINTER(0, 0);
BLASLONG i = 0;
for (; i < v_m2; i += v_size2) {
CREATE_A_POINTER(0, 0);
CREATE_A_POINTER(1, v_size);
UPDATE_A_POINTER(v_size2);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(1, 0);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
VECTOR_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 1, 0);
INCR_C_POINTER(0, v_size2);
}
for (; i < v_m1; i += v_size) {
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(v_size);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
}
VECTOR_STORE(pg_true, 0, 0);
INCR_C_POINTER(0, v_size);
}
for (; i < M; i += v_size) {
const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M));
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(0);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0);
}
VECTOR_STORE(pg_tail, 0, 0);
INCR_C_POINTER(0, 0);
}
UPDATE_B_POINTER(1);
RESET_A_POINTER();
UPDATE_C_POINTER(1);
}
return 0;
}

571
kernel/arm64/dgemm_small_kernel_tn_sve.c Normal file
View File

@@ -0,0 +1,571 @@
/***************************************************************************
Copyright (c) 2024, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#include "common.h"
#include <arm_neon.h>
#include <arm_sve.h>
#if defined(__ARM_NEON_SVE_BRIDGE) && defined(__has_include) && \
__has_include(<arm_neon_sve_bridge.h>)
#include <arm_neon_sve_bridge.h>
#else
#define svdup_neonq_f32(fixed_reg) \
({ \
svfloat32_t scalable_reg; \
asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \
scalable_reg; \
})
#define svdup_neonq_f64(fixed_reg) \
({ \
svfloat64_t scalable_reg; \
asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \
scalable_reg; \
})
#endif
#define RESET_A_POINTER() a_offset = A;
#define CREATE_A_POINTER(m, scale) FLOAT* a_offset##m = a_offset + scale * lda;
#define UPDATE_A_POINTER(scale) a_offset = a_offset + scale * lda;
#define A_ELEMENT_K(m, offset_k) *(a_offset##m + (k + offset_k))
#define A_ELEMENT(m) A_ELEMENT_K(m, 0)
#define RESET_B_POINTER() b_offset = B;
#define CREATE_B_POINTER(n, scale) FLOAT* b_offset##n = b_offset + scale * ldb;
#define UPDATE_B_POINTER(scale) b_offset = b_offset + scale * ldb;
#define B_ELEMENT_K(n, offset_k) *(b_offset##n + (k + offset_k))
#define B_ELEMENT(n) B_ELEMENT_K(n, 0)
#define CREATE_C_POINTER(m, scale) FLOAT* c_offset##m = c_offset + scale;
#define INCR_C_POINTER(m, incr) // c_offset ## m += incr * ldc;
#define UPDATE_C_POINTER(scale) c_offset += scale;
#define C_ELEMENT(m, n) \
*(c_offset##m + ((j + n) * ldc)) // C[(i+(m))+(j+(n))*ldc]
// #undef C_ELEMENT
// #define C_ELEMENT(m, n) C[(i+(m))+(j+(n))*ldc]
#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * v_size2 + m]
#define PACK_ELEMENT(m) PACK_ELEMENT_K(m, 0)
// ASIMD
#define DECLARE_RESULT_VECTOR2(m, n) \
float64x2_t result##m##n = vdupq_n_f64(0.0);
#define DECLARE_RESULT(m, n) float64_t result##m##n = 0.0;
#define BROADCAST_LOAD_A2(m, offset_k) \
float64x2_t a##m##_k##offset_k = vld1q_dup_f64(&A_ELEMENT_K(m, offset_k));
#define LOAD_A1(m, offset_k) \
float64_t a##m##_k##offset_k = A_ELEMENT_K(m, offset_k);
#define GATHER_LOAD_B2(n, offset_k) \
float64x2_t b##n##_k##offset_k = vdupq_n_f64(B_ELEMENT_K(n, offset_k)); \
b##n##_k##offset_k = \
vsetq_lane_f64(B_ELEMENT_K(n + 1, offset_k), b##n##_k##offset_k, 1);
#define VECTOR_UNPACK_B2(n, offset_k) \
float64x2_t b##n##_k##offset_k = vld1q_f64(&PACK_ELEMENT_K(n, offset_k));
#define PACK_B0(n, offset_k) \
PACK_ELEMENT_K(n, offset_k) = vget_lane_f64(b##n##_k##offset_k, 0);
#define UPDATE_RESULT_VECTOR2(m, n, offset_k) \
result##m##n = \
vfmaq_f64(result##m##n, a##m##_k##offset_k, b##n##_k##offset_k);
#define UPDATE_RESULT(m, n, offset_k) \
result##m##n = result##m##n + a##m##_k##offset_k * b##n##_k##offset_k;
#ifdef B0
#define SCATTER_STORE2(m, n) \
result##m##n = vmulq_f64(result##m##n, vdupq_n_f64(alpha)); \
C_ELEMENT(m, n + 0) = vgetq_lane_f64(result##m##n, 0); \
C_ELEMENT(m, n + 1) = vgetq_lane_f64(result##m##n, 1);
#else
#define SCATTER_STORE2(m, n) \
result##m##n = vmulq_f64(result##m##n, vdupq_n_f64(alpha)); \
C_ELEMENT(m, n + 0) = \
C_ELEMENT(m, n + 0) * beta + vgetq_lane_f64(result##m##n, 0); \
C_ELEMENT(m, n + 1) = \
C_ELEMENT(m, n + 1) * beta + vgetq_lane_f64(result##m##n, 1);
#endif
// SVE
#define DECLARE_RESULT_VECTOR(m, n) svfloat64_t result##m##n = svdup_f64(0.0);
#define BROADCAST_LOAD_A(m, offset_k) \
svfloat64_t a##s##m##_k##offset_k = svdup_f64(A_ELEMENT_K(m, offset_k));
#define BROADCAST_LOAD_B(n, offset_k) \
svfloat64_t b##s##n##_k##offset_k = svdup_f64(B_ELEMENT_K(n, offset_k));
#define VECTOR_LOAD_A(pg, m, offset_k) \
svfloat64_t a##s##m##_k##offset_k = svld1(pg, &A_ELEMENT_K(m, offset_k));
#define GATHER_LOAD_A(pg, m, offset_k) \
svfloat64_t a##s##m##_k##offset_k = \
svld1_gather_index(pg, &A_ELEMENT_K(m, offset_k), lda_vec);
#define PACK_A(m, offset_k) \
svst1(pg_first, &PACK_ELEMENT_K(m, offset_k), a##s##m##_k##offset_k);
#define VECTOR_PACK_A(m, offset_k) \
svst1(pg_true, &PACK_ELEMENT_K(m* v_size, offset_k), a##s##m##_k##offset_k);
#define QUADWORD_PACK_A(m, offset_k) \
svst1(pg_quad, &PACK_ELEMENT_K(m, offset_k), a##s##m##_k##offset_k);
#define UNPACK_VECTOR_A(m, offset_k) \
svfloat64_t a##s##m##_k##offset_k = \
svld1(pg_true, &PACK_ELEMENT_K(m * v_size, offset_k));
#define UNPACK_BROADCAST_A(m, offset_k) \
svfloat64_t a##s##m##_k##offset_k = svdup_f64(PACK_ELEMENT_K(m, offset_k));
#define UNPACK_QUADWORD_A(m, offset_k) \
svfloat64_t a##s##m##_k##offset_k = \
svld1rq(pg_true, &PACK_ELEMENT_K(m, offset_k));
#define UPDATE_RESULT_VECTOR(pg, m, n, offset_k) \
result##m##n = \
svmla_m(pg, result##m##n, a##s##m##_k##offset_k, b##s##n##_k##offset_k);
#define UPDATE_RESULT_VECTOR_QUADWORD(m, n, outer, lane, offset_k) \
result##m##n = svmla_lane( \
result##m##n, a##s##m##_k##offset_k, b##s##outer##_k##offset_k, lane);
#ifdef B0
#define VECTOR_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
svst1(pg, &C_ELEMENT(m, n), result##m##n);
#define SCATTER_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
#else
#define VECTOR_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
result##m##n = \
svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(m, n)), beta_vec); \
svst1(pg, &C_ELEMENT(m, n), result##m##n);
#define SCATTER_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
result##m##n = svmla_m(pg, \
result##m##n, \
svld1_gather_index(pg, &C_ELEMENT(m, n), ldc_vec), \
beta_vec); \
svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
#endif
#ifndef LIKELY
#ifdef __GNUC__
#define LIKELY(x) __builtin_expect(!!(x), 1)
#else
#define LIKELY(x) (x)
#endif
#endif
#ifdef B0
int
CNAME(BLASLONG M,
BLASLONG N,
BLASLONG K,
IFLOAT* A,
BLASLONG lda,
FLOAT alpha,
IFLOAT* B,
BLASLONG ldb,
FLOAT* C,
BLASLONG ldc)
#else
int
CNAME(BLASLONG M,
BLASLONG N,
BLASLONG K,
IFLOAT* A,
BLASLONG lda,
FLOAT alpha,
IFLOAT* B,
BLASLONG ldb,
FLOAT beta,
FLOAT* C,
BLASLONG ldc)
#endif
{
const uint64_t v_size = svcntd();
const uint64_t v_size2 = v_size * 2;
const svbool_t pg_true = svptrue_b64();
const svbool_t pg_quad = svwhilelt_b64(0, 2);
const svbool_t pg_first = svwhilelt_b64(0, 1);
const svfloat64_t alpha_vec = svdup_f64(alpha);
#ifndef B0
const svfloat64_t beta_vec = svdup_f64(beta);
#endif
const svuint64_t lda_vec = svindex_u64(0LL, lda);
const BLASLONG v_m2 = M & -v_size2;
const BLASLONG v_m1 = M & -v_size;
const BLASLONG n4 = N & -4;
const BLASLONG n2 = N & -2;
const int pack_a = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0;
FLOAT* packed_a =
(pack_a) ? packed_a = (FLOAT*)malloc(K * v_size2 * sizeof(FLOAT)) : NULL;
FLOAT* a_offset = A;
FLOAT* b_offset = B;
FLOAT* c_offset = C;
BLASLONG i = 0;
for (; i < v_m2; i += v_size2) {
CREATE_C_POINTER(0, 0);
CREATE_C_POINTER(1, v_size);
CREATE_A_POINTER(0, 0);
CREATE_A_POINTER(1, v_size);
BLASLONG j = 0;
for (; j < n4; j += 4) {
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
CREATE_B_POINTER(2, 2);
CREATE_B_POINTER(3, 3);
UPDATE_B_POINTER(4);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
DECLARE_RESULT_VECTOR(1, 0);
DECLARE_RESULT_VECTOR(1, 1);
DECLARE_RESULT_VECTOR(1, 2);
DECLARE_RESULT_VECTOR(1, 3);
if (LIKELY(packed_a != NULL)) {
if (j == 0) {
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
VECTOR_PACK_A(0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
GATHER_LOAD_A(pg_true, 1, 0);
VECTOR_PACK_A(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0);
BROADCAST_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0);
BROADCAST_LOAD_B(3, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0);
}
} else {
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
UNPACK_VECTOR_A(0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
UNPACK_VECTOR_A(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0);
BROADCAST_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0);
BROADCAST_LOAD_B(3, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0);
}
}
} else {
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
GATHER_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0);
BROADCAST_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0);
BROADCAST_LOAD_B(3, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0);
}
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 0, 2);
VECTOR_STORE(pg_true, 0, 3);
VECTOR_STORE(pg_true, 1, 0);
VECTOR_STORE(pg_true, 1, 1);
VECTOR_STORE(pg_true, 1, 2);
VECTOR_STORE(pg_true, 1, 3);
INCR_C_POINTER(0, 4);
INCR_C_POINTER(1, 4);
}
for (; j < n2; j += 2) {
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
UPDATE_B_POINTER(2);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(1, 0);
DECLARE_RESULT_VECTOR(1, 1);
if (LIKELY(packed_a != NULL)) {
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
UNPACK_VECTOR_A(0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
UNPACK_VECTOR_A(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0);
}
} else {
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
GATHER_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0);
}
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 1, 0);
VECTOR_STORE(pg_true, 1, 1);
INCR_C_POINTER(0, 2);
INCR_C_POINTER(1, 2);
}
for (; j < N; j++) {
CREATE_B_POINTER(0, 0);
UPDATE_B_POINTER(1);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(1, 0);
if (LIKELY(packed_a != NULL)) {
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
UNPACK_VECTOR_A(0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
UNPACK_VECTOR_A(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
}
} else {
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
GATHER_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
}
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 1, 0);
INCR_C_POINTER(0, 1);
INCR_C_POINTER(1, 1);
}
UPDATE_A_POINTER(v_size2);
RESET_B_POINTER();
UPDATE_C_POINTER(v_size2);
}
for (; i < v_m1; i += v_size) {
CREATE_C_POINTER(0, 0);
CREATE_A_POINTER(0, 0);
BLASLONG j = 0;
for (; j < n4; j += 4) {
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
CREATE_B_POINTER(2, 2);
CREATE_B_POINTER(3, 3);
UPDATE_B_POINTER(4);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
BROADCAST_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0);
BROADCAST_LOAD_B(3, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0);
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 0, 2);
VECTOR_STORE(pg_true, 0, 3);
INCR_C_POINTER(0, 4);
}
for (; j < n2; j += 2) {
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
UPDATE_B_POINTER(2);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
INCR_C_POINTER(0, 2);
}
for (; j < N; j++) {
CREATE_B_POINTER(0, 0);
UPDATE_B_POINTER(1);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
}
VECTOR_STORE(pg_true, 0, 0);
INCR_C_POINTER(0, 1);
}
UPDATE_A_POINTER(v_size);
RESET_B_POINTER();
UPDATE_C_POINTER(v_size);
}
for (; i < M; i += v_size) {
const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M));
CREATE_C_POINTER(0, 0);
CREATE_A_POINTER(0, 0);
BLASLONG j = 0;
for (; j < n4; j += 4) {
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
CREATE_B_POINTER(2, 2);
CREATE_B_POINTER(3, 3);
UPDATE_B_POINTER(4);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 1, 0);
BROADCAST_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 2, 0);
BROADCAST_LOAD_B(3, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 3, 0);
}
VECTOR_STORE(pg_tail, 0, 0);
VECTOR_STORE(pg_tail, 0, 1);
VECTOR_STORE(pg_tail, 0, 2);
VECTOR_STORE(pg_tail, 0, 3);
INCR_C_POINTER(0, 4);
}
for (; j < n2; j += 2) {
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
UPDATE_B_POINTER(2);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 1, 0);
}
VECTOR_STORE(pg_tail, 0, 0);
VECTOR_STORE(pg_tail, 0, 1);
INCR_C_POINTER(0, 2);
}
for (; j < N; j++) {
CREATE_B_POINTER(0, 0);
UPDATE_B_POINTER(1);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0);
}
VECTOR_STORE(pg_tail, 0, 0);
INCR_C_POINTER(0, 1);
}
UPDATE_A_POINTER(0);
RESET_B_POINTER();
UPDATE_C_POINTER(0);
}
if (pack_a)
free(packed_a);
return 0;
}

564
kernel/arm64/dgemm_small_kernel_tt_sve.c Normal file
View File

@@ -0,0 +1,564 @@
/***************************************************************************
Copyright (c) 2024, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#include "common.h"
#include <arm_neon.h>
#include <arm_sve.h>
#if defined(__ARM_NEON_SVE_BRIDGE) && defined(__has_include) && \
__has_include(<arm_neon_sve_bridge.h>)
#include <arm_neon_sve_bridge.h>
#else
#define svdup_neonq_f32(fixed_reg) \
({ \
svfloat32_t scalable_reg; \
asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \
scalable_reg; \
})
#define svdup_neonq_f64(fixed_reg) \
({ \
svfloat64_t scalable_reg; \
asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \
scalable_reg; \
})
#endif
#define RESET_A_POINTER() a_offset = A;
#define CREATE_A_POINTER(m, scale) FLOAT* a_offset##m = a_offset + scale * lda;
#define UPDATE_A_POINTER(scale) a_offset = a_offset + scale * lda;
#define A_ELEMENT_K(m, offset_k) *(a_offset##m + (k + offset_k))
#define A_ELEMENT(m) A_ELEMENT_K(m, 0)
#define RESET_B_POINTER() b_offset = B;
#define CREATE_B_POINTER(n, scale) FLOAT* b_offset##n = b_offset + scale;
#define UPDATE_B_POINTER(scale) b_offset = b_offset + scale;
#define B_ELEMENT_K(n, offset_k) *(b_offset##n + (k + offset_k) * ldb)
#define B_ELEMENT(n) B_ELEMENT_K(n, 0)
#define CREATE_C_POINTER(m, scale) FLOAT* c_offset##m = c_offset + scale;
#define INCR_C_POINTER(m, incr) // c_offset ## m += incr * ldc;
#define UPDATE_C_POINTER(scale) c_offset += scale;
#define C_ELEMENT(m, n) \
*(c_offset##m + ((j + n) * ldc)) // C[(i+(m))+(j+(n))*ldc]
// #undef C_ELEMENT
// #define C_ELEMENT(m, n) C[(i+(m))+(j+(n))*ldc]
#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * v_size2 + m]
#define PACK_ELEMENT(m) PACK_ELEMENT_K(m, 0)
// ASIMD
#define DECLARE_RESULT_VECTOR2(m, n) \
float64x2_t result##m##n = vdupq_n_f64(0.0);
#define DECLARE_RESULT(m, n) float64_t result##m##n = 0.0;
#define BROADCAST_LOAD_A2(m, offset_k) \
float64x2_t a##m##_k##offset_k = vld1q_dup_f64(&A_ELEMENT_K(m, offset_k));
#define LOAD_A1(m, offset_k) \
float64_t a##m##_k##offset_k = A_ELEMENT_K(m, offset_k);
#define VECTOR_LOAD_B2(n, offset_k) \
float64x2_t b##n##_k##offset_k = vld1q_f64(&B_ELEMENT_K(n, offset_k));
#define GATHER_LOAD_B2(n, offset_k) \
float64x2_t b##n##_k##offset_k = vdupq_n_f64(B_ELEMENT_K(n, offset_k)); \
b##n##_k##offset_k = \
vsetq_lane_f64(B_ELEMENT_K(n + 1, offset_k), b##n##_k##offset_k, 1);
#define VECTOR_UNPACK_B2(n, offset_k) \
float64x2_t b##n##_k##offset_k = vld1q_f64(&PACK_ELEMENT_K(n, offset_k));
#define VECTOR_PACK_B2(n, offset_k) \
vst1q_f64(&PACK_ELEMENT_K(n, offset_k), b##n##_k##offset_k);
#define PACK_B0(n, offset_k) \
PACK_ELEMENT_K(n, offset_k) = vget_lane_f64(b##n##_k##offset_k, 0);
#define UPDATE_RESULT_VECTOR2(m, n, offset_k) \
result##m##n = \
vfmaq_f64(result##m##n, a##m##_k##offset_k, b##n##_k##offset_k);
#define UPDATE_RESULT(m, n, offset_k) \
result##m##n = result##m##n + a##m##_k##offset_k * b##n##_k##offset_k;
#ifdef B0
#define VECTOR_STORE2(m, n) \
vst1q_f64(&C_ELEMENT(m, n), vmulq_f64(result##m##n, vdupq_n_f64(alpha)));
#define STORE(m, n) C_ELEMENT(m, n) = alpha * result##m##n;
#else
#define VECTOR_STORE2(m, n) \
result##m##n = vmulq_f64(result##m##n, vdupq_n_f64(alpha)); \
result##m##n = \
vfmaq_f64(result##m##n, vld1q_f64(&C_ELEMENT(m, n)), vdupq_n_f64(beta)); \
vst1q_f64(&C_ELEMENT(m, n), result##m##n);
#define STORE(m, n) \
C_ELEMENT(m, n) = C_ELEMENT(m, n) * beta + alpha * result##m##n;
#endif
// SVE
#define DECLARE_RESULT_VECTOR(m, n) svfloat64_t result##m##n = svdup_f64(0.0);
#define BROADCAST_LOAD_A(m, offset_k) \
svfloat64_t a##s##m##_k##offset_k = svdup_f64(A_ELEMENT_K(m, offset_k));
#define BROADCAST_LOAD_B(n, offset_k) \
svfloat64_t b##s##n##_k##offset_k = svdup_f64(B_ELEMENT_K(n, offset_k));
#define VECTOR_LOAD_A(pg, m, offset_k) \
svfloat64_t a##s##m##_k##offset_k = svld1(pg, &A_ELEMENT_K(m, offset_k));
#define QUADWORD_LOAD_B(n, offset_k) \
svfloat64_t b##s##n##_k##offset_k = \
svld1rq(pg_true, &B_ELEMENT_K(n, offset_k));
#define GATHER_LOAD_A(pg, m, offset_k) \
svfloat64_t a##s##m##_k##offset_k = \
svld1_gather_index(pg, &A_ELEMENT_K(m, offset_k), lda_vec);
#define PACK_A(m, offset_k) \
svst1(pg_first, &PACK_ELEMENT_K(m, offset_k), a##s##m##_k##offset_k);
#define VECTOR_PACK_A(m, offset_k) \
svst1(pg_true, &PACK_ELEMENT_K(m* v_size, offset_k), a##s##m##_k##offset_k);
#define QUADWORD_PACK_A(m, offset_k) \
svst1(pg_quad, &PACK_ELEMENT_K(m, offset_k), a##s##m##_k##offset_k);
#define UNPACK_VECTOR_A(m, offset_k) \
svfloat64_t a##s##m##_k##offset_k = \
svld1(pg_true, &PACK_ELEMENT_K(m * v_size, offset_k));
#define UNPACK_BROADCAST_A(m, offset_k) \
svfloat64_t a##s##m##_k##offset_k = svdup_f64(PACK_ELEMENT_K(m, offset_k));
#define UNPACK_QUADWORD_A(m, offset_k) \
svfloat64_t a##s##m##_k##offset_k = \
svld1rq(pg_true, &PACK_ELEMENT_K(m, offset_k));
#define UPDATE_RESULT_VECTOR(pg, m, n, offset_k) \
result##m##n = \
svmla_m(pg, result##m##n, a##s##m##_k##offset_k, b##s##n##_k##offset_k);
#define UPDATE_RESULT_VECTOR_QUADWORD(m, n, outer, lane, offset_k) \
result##m##n = svmla_lane( \
result##m##n, a##s##m##_k##offset_k, b##s##outer##_k##offset_k, lane);
#ifdef B0
#define VECTOR_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
svst1(pg, &C_ELEMENT(m, n), result##m##n);
#define SCATTER_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
#else
#define VECTOR_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
result##m##n = \
svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(m, n)), beta_vec); \
svst1(pg, &C_ELEMENT(m, n), result##m##n);
#define SCATTER_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
result##m##n = svmla_m(pg, \
result##m##n, \
svld1_gather_index(pg, &C_ELEMENT(m, n), ldc_vec), \
beta_vec); \
svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
#endif
#ifndef LIKELY
#ifdef __GNUC__
#define LIKELY(x) __builtin_expect(!!(x), 1)
#else
#define LIKELY(x) (x)
#endif
#endif
#ifdef B0
int
CNAME(BLASLONG M,
BLASLONG N,
BLASLONG K,
IFLOAT* A,
BLASLONG lda,
FLOAT alpha,
IFLOAT* B,
BLASLONG ldb,
FLOAT* C,
BLASLONG ldc)
#else
int
CNAME(BLASLONG M,
BLASLONG N,
BLASLONG K,
IFLOAT* A,
BLASLONG lda,
FLOAT alpha,
IFLOAT* B,
BLASLONG ldb,
FLOAT beta,
FLOAT* C,
BLASLONG ldc)
#endif
{
const uint64_t v_size = svcntd();
const uint64_t v_size2 = v_size * 2;
const svbool_t pg_true = svptrue_b64();
const svbool_t pg_quad = svwhilelt_b64(0, 2);
const svbool_t pg_first = svwhilelt_b64(0, 1);
const svfloat64_t alpha_vec = svdup_f64(alpha);
#ifndef B0
const svfloat64_t beta_vec = svdup_f64(beta);
#endif
const svuint64_t lda_vec = svindex_u64(0LL, lda);
const BLASLONG v_m2 = M & -v_size2;
const BLASLONG v_m1 = M & -v_size;
const BLASLONG n4 = N & -4;
const BLASLONG n2 = N & -2;
const int pack_a = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0;
FLOAT* packed_a =
(pack_a) ? packed_a = (FLOAT*)malloc(K * v_size2 * sizeof(FLOAT)) : NULL;
FLOAT* a_offset = A;
FLOAT* b_offset = B;
FLOAT* c_offset = C;
BLASLONG i = 0;
for (; i < v_m2; i += v_size2) {
CREATE_C_POINTER(0, 0);
CREATE_C_POINTER(1, v_size);
CREATE_A_POINTER(0, 0);
CREATE_A_POINTER(1, v_size);
BLASLONG j = 0;
for (; j < n4; j += 4) {
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
CREATE_B_POINTER(2, 2);
CREATE_B_POINTER(3, 3);
UPDATE_B_POINTER(4);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
DECLARE_RESULT_VECTOR(1, 0);
DECLARE_RESULT_VECTOR(1, 1);
DECLARE_RESULT_VECTOR(1, 2);
DECLARE_RESULT_VECTOR(1, 3);
if (LIKELY(packed_a != NULL)) {
if (j == 0) {
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
VECTOR_PACK_A(0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
QUADWORD_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0);
GATHER_LOAD_A(pg_true, 1, 0);
VECTOR_PACK_A(1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 0);
}
} else {
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
UNPACK_VECTOR_A(0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
QUADWORD_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0);
UNPACK_VECTOR_A(1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 0);
}
}
} else {
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
QUADWORD_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0);
GATHER_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 0);
}
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 0, 2);
VECTOR_STORE(pg_true, 0, 3);
VECTOR_STORE(pg_true, 1, 0);
VECTOR_STORE(pg_true, 1, 1);
VECTOR_STORE(pg_true, 1, 2);
VECTOR_STORE(pg_true, 1, 3);
INCR_C_POINTER(0, 4);
INCR_C_POINTER(1, 4);
}
for (; j < n2; j += 2) {
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
UPDATE_B_POINTER(2);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(1, 0);
DECLARE_RESULT_VECTOR(1, 1);
if (LIKELY(packed_a != NULL)) {
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
UNPACK_VECTOR_A(0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UNPACK_VECTOR_A(1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
}
} else {
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
GATHER_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
}
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 1, 0);
VECTOR_STORE(pg_true, 1, 1);
INCR_C_POINTER(0, 2);
INCR_C_POINTER(1, 2);
}
for (; j < N; j++) {
CREATE_B_POINTER(0, 0);
UPDATE_B_POINTER(1);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(1, 0);
if (LIKELY(packed_a != NULL)) {
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
UNPACK_VECTOR_A(0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
UNPACK_VECTOR_A(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
}
} else {
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
GATHER_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
}
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 1, 0);
INCR_C_POINTER(0, 1);
INCR_C_POINTER(1, 1);
}
UPDATE_A_POINTER(v_size2);
RESET_B_POINTER();
UPDATE_C_POINTER(v_size2);
}
for (; i < v_m1; i += v_size) {
CREATE_C_POINTER(0, 0);
CREATE_A_POINTER(0, 0);
BLASLONG j = 0;
for (; j < n4; j += 4) {
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
CREATE_B_POINTER(2, 2);
CREATE_B_POINTER(3, 3);
UPDATE_B_POINTER(4);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
QUADWORD_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0);
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 0, 2);
VECTOR_STORE(pg_true, 0, 3);
INCR_C_POINTER(0, 4);
}
for (; j < n2; j += 2) {
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
UPDATE_B_POINTER(2);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
INCR_C_POINTER(0, 2);
}
for (; j < N; j++) {
CREATE_B_POINTER(0, 0);
UPDATE_B_POINTER(1);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
}
VECTOR_STORE(pg_true, 0, 0);
INCR_C_POINTER(0, 1);
}
UPDATE_A_POINTER(v_size);
RESET_B_POINTER();
UPDATE_C_POINTER(v_size);
}
for (; i < M; i += v_size) {
const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M));
CREATE_C_POINTER(0, 0);
CREATE_A_POINTER(0, 0);
BLASLONG j = 0;
for (; j < n4; j += 4) {
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
CREATE_B_POINTER(2, 2);
CREATE_B_POINTER(3, 3);
UPDATE_B_POINTER(4);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
GATHER_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
QUADWORD_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0);
}
VECTOR_STORE(pg_tail, 0, 0);
VECTOR_STORE(pg_tail, 0, 1);
VECTOR_STORE(pg_tail, 0, 2);
VECTOR_STORE(pg_tail, 0, 3);
INCR_C_POINTER(0, 4);
}
for (; j < n2; j += 2) {
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
UPDATE_B_POINTER(2);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
GATHER_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
}
VECTOR_STORE(pg_tail, 0, 0);
VECTOR_STORE(pg_tail, 0, 1);
INCR_C_POINTER(0, 2);
}
for (; j < N; j++) {
CREATE_B_POINTER(0, 0);
UPDATE_B_POINTER(1);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0);
}
VECTOR_STORE(pg_tail, 0, 0);
INCR_C_POINTER(0, 1);
}
UPDATE_A_POINTER(0);
RESET_B_POINTER();
UPDATE_C_POINTER(0);
}
if (pack_a)
free(packed_a);
return 0;
}

43
kernel/arm64/gemm_small_kernel_permit_sve.c Normal file
View File

@@ -0,0 +1,43 @@
/***************************************************************************
Copyright (c) 2024, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#include "common.h"
int CNAME(int transa, int transb, BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, FLOAT beta)
{
BLASLONG MNK = M * N * K;
#if defined(DOUBLE) // dgemm
if (MNK <= 64*64*64)
return 1;
#else // sgemm
if (MNK <= 64*64*64)
return 1;
#endif
return 0;
}

92
kernel/arm64/gemv_n_sve.c Normal file
View File

@@ -0,0 +1,92 @@
/***************************************************************************
Copyright (c) 2024, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written
permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#include <arm_sve.h>
#include "common.h"
#ifdef DOUBLE
#define SV_COUNT svcntd
#define SV_TYPE svfloat64_t
#define SV_TRUE svptrue_b64
#define SV_WHILE svwhilelt_b64_s64
#define SV_DUP svdup_f64
#else
#define SV_COUNT svcntw
#define SV_TYPE svfloat32_t
#define SV_TRUE svptrue_b32
#define SV_WHILE svwhilelt_b32_s64
#define SV_DUP svdup_f32
#endif
int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer)
{
BLASLONG i;
BLASLONG ix,iy;
BLASLONG j;
FLOAT *a_ptr;
FLOAT temp;
ix = 0;
a_ptr = a;
if (inc_y == 1) {
uint64_t sve_size = SV_COUNT();
for (j = 0; j < n; j++) {
SV_TYPE temp_vec = SV_DUP(alpha * x[ix]);
i = 0;
svbool_t pg = SV_WHILE(i, m);
while (svptest_any(SV_TRUE(), pg)) {
SV_TYPE a_vec = svld1(pg, a_ptr + i);
SV_TYPE y_vec = svld1(pg, y + i);
y_vec = svmla_x(pg, y_vec, temp_vec, a_vec);
svst1(pg, y + i, y_vec);
i += sve_size;
pg = SV_WHILE(i, m);
}
a_ptr += lda;
ix += inc_x;
}
return(0);
}
for (j = 0; j < n; j++) {
temp = alpha * x[ix];
iy = 0;
for (i = 0; i < m; i++) {
y[iy] += temp * a_ptr[i];
iy += inc_y;
}
a_ptr += lda;
ix += inc_x;
}
return (0);
}

67
kernel/arm64/gemv_t.S
View File

@@ -1,5 +1,5 @@
/*******************************************************************************
Copyright (c) 2015, The OpenBLAS Project
Copyright (c) 2015, 2024 The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
@@ -170,39 +170,48 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.macro KERNEL_F32_FINALIZE
#if !defined(DOUBLE)
fadd v1.4s, v1.4s, v2.4s
// F8 only has 2 accumulators
// so add into those pairs
fadd v1.4s, v1.4s, v3.4s
fadd v1.4s, v1.4s, v4.4s
#else
fadd v1.2d, v1.2d, v2.2d
fadd v1.2d, v1.2d, v3.2d
fadd v1.2d, v1.2d, v4.2d
fadd v2.4s, v2.4s, v4.4s
#endif
.endm
.macro KERNEL_F4
.macro KERNEL_F8
#if !defined(DOUBLE)
ld1 {v2.4s}, [A_PTR], #16
ld1 {v3.4s}, [X_PTR], #16
fmla v1.4s, v2.4s, v3.4s
#else
ld1 {v2.2d}, [A_PTR], #16
ld1 {v3.2d}, [X_PTR], #16
fmla v1.2d, v2.2d, v3.2d
ld1 {v4.2d}, [A_PTR], #16
ld1 {v5.2d}, [X_PTR], #16
fmla v1.2d, v4.2d, v5.2d
ld1 {v13.4s, v14.4s}, [A_PTR], #32
ld1 {v17.4s, v18.4s}, [X_PTR], #32
fmla v1.4s, v13.4s, v17.4s
fmla v2.4s, v14.4s, v18.4s
#else
ld1 {v13.2d, v14.2d, v15.2d, v16.2d}, [A_PTR], #64
ld1 {v17.2d, v18.2d, v19.2d, v20.2d}, [X_PTR], #64
fmla v1.2d, v13.2d, v17.2d
fmla v2.2d, v14.2d, v18.2d
fmla v3.2d, v15.2d, v19.2d
fmla v4.2d, v16.2d, v20.2d
#endif
.endm
.macro KERNEL_F4_FINALIZE
.macro KERNEL_F8_FINALIZE
#if !defined(DOUBLE)
ext v2.16b, v1.16b, v1.16b, #8
// Take the top two elements of v1 and
// put them into the first two lanes of v3
ext v3.16b, v1.16b, v1.16b, #8
fadd v1.2s, v1.2s, v3.2s
ext v4.16b, v2.16b, v2.16b, #8
fadd v2.2s, v2.2s, v4.2s
// Final pair
fadd v1.2s, v1.2s, v2.2s
faddp TEMP, v1.2s
#else
faddp TEMP, v1.2d
faddp TEMP1, v2.2d
faddp TEMP2, v3.2d
faddp TEMP3, v4.2d
fadd TEMP, TEMP, TEMP1
fadd TEMP2, TEMP2, TEMP3
fadd TEMP, TEMP, TEMP2
#endif
.endm
@@ -258,7 +267,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
asr I, M, #5
cmp I, xzr
beq .Lgemv_t_kernel_F4
beq .Lgemv_t_kernel_F8
.Lgemv_t_kernel_F320:
@@ -269,24 +278,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
KERNEL_F32_FINALIZE
.Lgemv_t_kernel_F4:
.Lgemv_t_kernel_F8:
ands I, M, #31
asr I, I, #2
asr I, I, #3
cmp I, xzr
beq .Lgemv_t_kernel_F1
.Lgemv_t_kernel_F40:
.Lgemv_t_kernel_F80:
KERNEL_F4
KERNEL_F8
subs I, I, #1
bne .Lgemv_t_kernel_F40
bne .Lgemv_t_kernel_F80
.Lgemv_t_kernel_F1:
KERNEL_F4_FINALIZE
KERNEL_F8_FINALIZE
ands I, M, #3
ands I, M, #7
ble .Lgemv_t_kernel_F_END
.Lgemv_t_kernel_F10:

120
kernel/arm64/gemv_t_sve.c Normal file
View File

@@ -0,0 +1,120 @@
/***************************************************************************
Copyright (c) 2024, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written
permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#include <arm_sve.h>
#include "common.h"
#ifdef DOUBLE
#define SV_COUNT svcntd
#define SV_TYPE svfloat64_t
#define SV_TRUE svptrue_b64
#define SV_WHILE svwhilelt_b64_s64
#define SV_DUP svdup_f64
#else
#define SV_COUNT svcntw
#define SV_TYPE svfloat32_t
#define SV_TRUE svptrue_b32
#define SV_WHILE svwhilelt_b32_s64
#define SV_DUP svdup_f32
#endif
int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer)
{
BLASLONG i;
BLASLONG ix,iy;
BLASLONG j;
FLOAT *a_ptr;
FLOAT temp;
iy = 0;
a_ptr = a;
if (inc_x == 1) {
svbool_t pg_true = SV_TRUE();
uint64_t sve_size = SV_COUNT();
uint64_t sve_size2 = sve_size * 2;
BLASLONG m1 = m & -sve_size;
BLASLONG m2 = m & -sve_size2;
for (j = 0; j < n; j++) {
BLASLONG i = 0;
SV_TYPE temp_vec_v2_0 = SV_DUP(0.0);
SV_TYPE temp_vec_v2_1 = SV_DUP(0.0);
for (; i < m2; i += sve_size2) {
SV_TYPE a_vec0 = svld1(pg_true, a_ptr + i);
SV_TYPE x_vec0 = svld1(pg_true, x + i);
SV_TYPE a_vec1 = svld1(pg_true, a_ptr + i + sve_size);
SV_TYPE x_vec1 = svld1(pg_true, x + i + sve_size);
temp_vec_v2_0 = svmla_m(pg_true, temp_vec_v2_0, a_vec0, x_vec0);
temp_vec_v2_1 = svmla_m(pg_true, temp_vec_v2_1, a_vec1, x_vec1);
}
SV_TYPE temp_vec_v1 = SV_DUP(0.0);
for (; i < m1; i += sve_size) {
SV_TYPE a_vec0 = svld1(pg_true, a_ptr + i);
SV_TYPE x_vec0 = svld1(pg_true, x + i);
temp_vec_v1 = svmla_m(pg_true, temp_vec_v1, a_vec0, x_vec0);
}
SV_TYPE temp_vec = SV_DUP(0.0);
for (; i < m; i += sve_size) {
svbool_t pg = SV_WHILE(i, m);
SV_TYPE a_vec = svld1(pg, a_ptr + i);
SV_TYPE x_vec = svld1(pg, x + i);
temp_vec = svmla_m(pg, temp_vec, a_vec, x_vec);
}
y[iy] += alpha * (
(svaddv(SV_TRUE(), temp_vec_v2_0) + svaddv(SV_TRUE(), temp_vec)) +
(svaddv(SV_TRUE(), temp_vec_v2_1) + svaddv(SV_TRUE(), temp_vec_v1))
);
iy += inc_y;
a_ptr += lda;
}
return(0);
}
for (j = 0; j < n; j++) {
temp = 0.0;
ix = 0;
for (i = 0; i < m; i++) {
temp += a_ptr[i] * x[ix];
ix += inc_x;
}
y[iy] += alpha * temp;
iy += inc_y;
a_ptr += lda;
}
return (0);
}

11
kernel/arm64/scal.S
View File

@@ -33,7 +33,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define X_COPY x5 /* X vector address */
#define INC_X x4 /* X stride */
#define I x1 /* loop variable */
#define FLAG x9
/*******************************************************************************
* Macro definitions
*******************************************************************************/
@@ -168,9 +168,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
cmp N, xzr
ble .Lscal_kernel_L999
//fcmp DA, #0.0
//beq .Lscal_kernel_zero
ldr FLAG, [sp]
cmp FLAG, #1
beq .Lscal_kernel_nansafe
fcmp DA, #0.0
beq .Lscal_kernel_zero
.Lscal_kernel_nansafe:
cmp INC_X, #1
bne .Lscal_kernel_S_BEGIN

687
kernel/arm64/sgemm_small_kernel_nn_sve.c Normal file
View File

@@ -0,0 +1,687 @@
/***************************************************************************
Copyright (c) 2024, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#include "common.h"
#include <arm_neon.h>
#include <arm_sve.h>
#if defined(__ARM_NEON_SVE_BRIDGE) && defined(__has_include) && \
__has_include(<arm_neon_sve_bridge.h>)
#include <arm_neon_sve_bridge.h>
#else
#define svdup_neonq_f32(fixed_reg) \
({ \
svfloat32_t scalable_reg; \
asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \
scalable_reg; \
})
#define svdup_neonq_f64(fixed_reg) \
({ \
svfloat64_t scalable_reg; \
asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \
scalable_reg; \
})
#endif
#define RESET_A_POINTER() a_offset = A;
#define CREATE_A_POINTER(m, scale) FLOAT* a_offset##m = a_offset + scale;
#define UPDATE_A_POINTER(scale) a_offset = a_offset + scale;
#define A_ELEMENT_K(m, offset_k) *(a_offset##m + (k + offset_k) * lda)
#define A_ELEMENT(m) A_ELEMENT_K(m, 0)
#define RESET_B_POINTER() b_offset = B;
#define CREATE_B_POINTER(n, scale) FLOAT* b_offset##n = b_offset + scale * ldb;
#define UPDATE_B_POINTER(scale) b_offset = b_offset + scale * ldb;
#define B_ELEMENT_K(n, offset_k) *(b_offset##n + (k + offset_k))
#define B_ELEMENT(n) B_ELEMENT_K(n, 0)
#define CREATE_C_POINTER(n, scale) FLOAT* c_offset##n = c_offset + scale * ldc;
#define INCR_C_POINTER(m, incr) // c_offset ## m += incr;
#define UPDATE_C_POINTER(scale) c_offset = c_offset + scale * ldc;
#define C_ELEMENT(m, n) *(c_offset##n + ((m * v_size) + i))
// #undef C_ELEMENT
// #define C_ELEMENT(m, n) C[(i+(m))+(j+(n))*ldc]
#define PACK_ELEMENT_K(n, offset_k) packed_b[(k + offset_k) * 4 + n]
#define PACK_ELEMENT(n) PACK_ELEMENT_K(n, 0)
// ASIMD
#define DECLARE_RESULT_VECTOR4(m, n) \
float32x4_t result##m##n = vdupq_n_f32(0.0);
#define DECLARE_RESULT(m, n) float32_t result##m##n = 0.0;
#define BROADCAST_LOAD_A4(m, offset_k) \
float32x4_t a##m##_k##offset_k = vld1q_dup_f32(&A_ELEMENT_K(m, offset_k));
#define LOAD_A1(m, offset_k) \
float32_t a##m##_k##offset_k = A_ELEMENT_K(m, offset_k);
#define VECTOR_LOAD_B_K4(n, offset_k) \
float32x4_t b##k##n##_k##offset_k = vld1q_f32(&B_ELEMENT_K(n, offset_k));
#define TRANSPOSE_B4_K4( \
n0, n1, n2, n3, offset_k0, offset_k1, offset_k2, offset_k3) \
float32x4_t b##t##n0##_k##offset_k0 = \
vzip1q_f32(b##k##n0##_k##offset_k0, b##k##n1##_k##offset_k0); \
float32x4_t b##t##n0##_k##offset_k1 = \
vzip2q_f32(b##k##n0##_k##offset_k0, b##k##n1##_k##offset_k0); \
float32x4_t b##t##n0##_k##offset_k2 = \
vzip1q_f32(b##k##n2##_k##offset_k0, b##k##n3##_k##offset_k0); \
float32x4_t b##t##n0##_k##offset_k3 = \
vzip2q_f32(b##k##n2##_k##offset_k0, b##k##n3##_k##offset_k0); \
float32x4_t b##n0##_k##offset_k0 = vreinterpretq_f32_f64( \
vzip1q_f64(vreinterpretq_f64_f32(b##t##n0##_k##offset_k0), \
vreinterpretq_f64_f32(b##t##n0##_k##offset_k2))); \
float32x4_t b##n0##_k##offset_k1 = vreinterpretq_f32_f64( \
vzip2q_f64(vreinterpretq_f64_f32(b##t##n0##_k##offset_k0), \
vreinterpretq_f64_f32(b##t##n0##_k##offset_k2))); \
float32x4_t b##n0##_k##offset_k2 = vreinterpretq_f32_f64( \
vzip1q_f64(vreinterpretq_f64_f32(b##t##n0##_k##offset_k1), \
vreinterpretq_f64_f32(b##t##n0##_k##offset_k3))); \
float32x4_t b##n0##_k##offset_k3 = vreinterpretq_f32_f64( \
vzip2q_f64(vreinterpretq_f64_f32(b##t##n0##_k##offset_k1), \
vreinterpretq_f64_f32(b##t##n0##_k##offset_k3)));
#define SCALE_B4_K4(n0, offset_k0, offset_k1, offset_k2, offset_k3) \
svfloat32_t b##s##n0##_k##offset_k0 = svdup_neonq_f32(b##n0##_k##offset_k0); \
svfloat32_t b##s##n0##_k##offset_k1 = svdup_neonq_f32(b##n0##_k##offset_k1); \
svfloat32_t b##s##n0##_k##offset_k2 = svdup_neonq_f32(b##n0##_k##offset_k2); \
svfloat32_t b##s##n0##_k##offset_k3 = svdup_neonq_f32(b##n0##_k##offset_k3);
#define GATHER_LOAD_B4(n, offset_k) \
float32x4_t b##n##_k##offset_k = vdupq_n_f32(B_ELEMENT_K(n, offset_k)); \
b##n##_k##offset_k = \
vsetq_lane_f32(B_ELEMENT_K(n + 1, offset_k), b##n##_k##offset_k, 1); \
b##n##_k##offset_k = \
vsetq_lane_f32(B_ELEMENT_K(n + 2, offset_k), b##n##_k##offset_k, 2); \
b##n##_k##offset_k = \
vsetq_lane_f32(B_ELEMENT_K(n + 3, offset_k), b##n##_k##offset_k, 3);
#define VECTOR_UNPACK_B4(n, offset_k) \
float32x4_t b##n##_k##offset_k = vld1q_f32(&PACK_ELEMENT_K(n, offset_k));
#define VECTOR_PACK_B4(n, offset_k) \
vst1q_f32(&PACK_ELEMENT_K(n, offset_k), b##n##_k##offset_k);
#define PACK_B0(n, offset_k) \
PACK_ELEMENT_K(n, offset_k) = vget_lane_f32(b##n##_k##offset_k, 0);
#define UPDATE_RESULT_VECTOR4(m, n, offset_k) \
result##m##n = \
vfmaq_f32(result##m##n, a##m##_k##offset_k, b##n##_k##offset_k);
#define UPDATE_RESULT(m, n, offset_k) \
result##m##n = result##m##n + a##m##_k##offset_k * b##n##_k##offset_k;
#ifdef B0
#define SCATTER_STORE4(m, n) \
result##m##n = vmulq_f32(result##m##n, vdupq_n_f32(alpha)); \
C_ELEMENT(m, n + 0) = vgetq_lane_f32(result##m##n, 0); \
C_ELEMENT(m, n + 1) = vgetq_lane_f32(result##m##n, 1); \
C_ELEMENT(m, n + 2) = vgetq_lane_f32(result##m##n, 2); \
C_ELEMENT(m, n + 3) = vgetq_lane_f32(result##m##n, 3);
#else
#define SCATTER_STORE4(m, n) \
result##m##n = vmulq_f32(result##m##n, vdupq_n_f32(alpha)); \
C_ELEMENT(m, n + 0) = \
C_ELEMENT(m, n + 0) * beta + vgetq_lane_f32(result##m##n, 0); \
C_ELEMENT(m, n + 1) = \
C_ELEMENT(m, n + 1) * beta + vgetq_lane_f32(result##m##n, 1); \
C_ELEMENT(m, n + 2) = \
C_ELEMENT(m, n + 2) * beta + vgetq_lane_f32(result##m##n, 2); \
C_ELEMENT(m, n + 3) = \
C_ELEMENT(m, n + 3) * beta + vgetq_lane_f32(result##m##n, 3);
#endif
// SVE
#define DECLARE_RESULT_VECTOR(m, n) svfloat32_t result##m##n = svdup_f32(0.0);
#define BROADCAST_LOAD_A(m, offset_k) \
svfloat32_t a##s##m##_k##offset_k = svdup_f32(A_ELEMENT_K(m, offset_k));
#define BROADCAST_LOAD_B(n, offset_k) \
svfloat32_t b##s##n##_k##offset_k = svdup_f32(B_ELEMENT_K(n, offset_k));
#define VECTOR_LOAD_A(pg, m, offset_k) \
svfloat32_t a##s##m##_k##offset_k = svld1(pg, &A_ELEMENT_K(m, offset_k));
#define QUADWORD_LOAD_B(n, offset_k) \
svfloat32_t b##s##n##_k##offset_k = \
svld1rq(pg_true, &B_ELEMENT_K(n, offset_k));
#define PACK_B(n, offset_k) \
svst1(pg_first, &PACK_ELEMENT_K(n, offset_k), b##s##n##_k##offset_k);
#define VECTOR_PACK_B(n, offset_k) \
svst1(pg_true, &PACK_ELEMENT_K(n* v_size, offset_k), b##s##n##_k##offset_k);
#define QUADWORD_PACK_B(n, offset_k) \
svst1(pg_quad, &PACK_ELEMENT_K(n, offset_k), b##s##n##_k##offset_k);
#define UNPACK_VECTOR_B(n, offset_k) \
svfloat32_t b##s##n##_k##offset_k = \
svld1(pg_true, &PACK_ELEMENT_K(n * v_size, offset_k));
#define UNPACK_BROADCAST_B(n, offset_k) \
svfloat32_t b##s##n##_k##offset_k = svdup_f32(PACK_ELEMENT_K(n, offset_k));
#define UNPACK_QUADWORD_B(n, offset_k) \
svfloat32_t b##s##n##_k##offset_k = \
svld1rq(pg_true, &PACK_ELEMENT_K(n, offset_k));
#define UPDATE_RESULT_VECTOR(pg, m, n, offset_k) \
result##m##n = \
svmla_m(pg, result##m##n, a##s##m##_k##offset_k, b##s##n##_k##offset_k);
#define UPDATE_RESULT_VECTOR_QUADWORD(m, n, outer, lane, offset_k) \
result##m##n = svmla_lane( \
result##m##n, a##s##m##_k##offset_k, b##s##outer##_k##offset_k, lane);
#ifdef B0
#define VECTOR_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
svst1(pg, &C_ELEMENT(m, n), result##m##n);
#define SCATTER_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
#else
#define VECTOR_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
result##m##n = \
svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(m, n)), beta_vec); \
svst1(pg, &C_ELEMENT(m, n), result##m##n);
#define SCATTER_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
result##m##n = svmla_m(pg, \
result##m##n, \
svld1_gather_index(pg, &C_ELEMENT(m, n), ldc_vec), \
beta_vec); \
svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
#endif
#ifndef LIKELY
#ifdef __GNUC__
#define LIKELY(x) __builtin_expect(!!(x), 1)
#else
#define LIKELY(x) (x)
#endif
#endif
#ifdef B0
int
CNAME(BLASLONG M,
BLASLONG N,
BLASLONG K,
IFLOAT* A,
BLASLONG lda,
FLOAT alpha,
IFLOAT* B,
BLASLONG ldb,
FLOAT* C,
BLASLONG ldc)
#else
int
CNAME(BLASLONG M,
BLASLONG N,
BLASLONG K,
IFLOAT* A,
BLASLONG lda,
FLOAT alpha,
IFLOAT* B,
BLASLONG ldb,
FLOAT beta,
FLOAT* C,
BLASLONG ldc)
#endif
{
const uint64_t v_size = svcntw();
const uint64_t v_size2 = v_size * 2;
const svbool_t pg_true = svptrue_b32();
const svbool_t pg_quad = svwhilelt_b32(0, 4);
const svbool_t pg_first = svwhilelt_b32(0, 1);
const svfloat32_t alpha_vec = svdup_f32(alpha);
#ifndef B0
const svfloat32_t beta_vec = svdup_f32(beta);
#endif
const BLASLONG n4 = N & -4;
const BLASLONG v_m2 = M & -v_size2;
const BLASLONG v_m1 = M & -v_size;
const BLASLONG k4 = K & -4;
const int pack_b = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0;
FLOAT* packed_b =
(pack_b) ? packed_b = (FLOAT*)malloc(K * 4 * sizeof(FLOAT)) : NULL;
FLOAT* b_offset = B;
FLOAT* a_offset = A;
FLOAT* c_offset = C;
BLASLONG j = 0;
for (; j < n4; j += 4) {
CREATE_C_POINTER(0, 0);
CREATE_C_POINTER(1, 1);
CREATE_C_POINTER(2, 2);
CREATE_C_POINTER(3, 3);
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
CREATE_B_POINTER(2, 2);
CREATE_B_POINTER(3, 3);
BLASLONG i = 0;
for (; i < v_m2; i += v_size2) {
CREATE_A_POINTER(0, 0);
CREATE_A_POINTER(1, v_size);
UPDATE_A_POINTER(v_size2);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
DECLARE_RESULT_VECTOR(1, 0);
DECLARE_RESULT_VECTOR(1, 1);
DECLARE_RESULT_VECTOR(1, 2);
DECLARE_RESULT_VECTOR(1, 3);
if (LIKELY(packed_b != NULL)) {
if (i == 0) {
for (; k < k4; k += 4) {
VECTOR_LOAD_B_K4(0, 0);
VECTOR_LOAD_B_K4(1, 0);
VECTOR_LOAD_B_K4(2, 0);
VECTOR_LOAD_B_K4(3, 0);
TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3);
SCALE_B4_K4(0, 0, 1, 2, 3);
VECTOR_PACK_B4(0, 0);
VECTOR_PACK_B4(0, 1);
VECTOR_PACK_B4(0, 2);
VECTOR_PACK_B4(0, 3);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
VECTOR_LOAD_A(pg_true, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1);
VECTOR_LOAD_A(pg_true, 0, 2);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2);
VECTOR_LOAD_A(pg_true, 0, 3);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3);
VECTOR_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0);
VECTOR_LOAD_A(pg_true, 1, 1);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 1);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 1);
VECTOR_LOAD_A(pg_true, 1, 2);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 2);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 2);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 2);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 2);
VECTOR_LOAD_A(pg_true, 1, 3);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 3);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 3);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 3);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 3);
}
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
PACK_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
PACK_B(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
VECTOR_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0);
BROADCAST_LOAD_B(2, 0);
PACK_B(2, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0);
BROADCAST_LOAD_B(3, 0);
PACK_B(3, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0);
}
} else {
for (; k < K; k++) {
UNPACK_QUADWORD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
VECTOR_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0);
}
}
} else {
for (; k < k4; k += 4) {
VECTOR_LOAD_B_K4(0, 0);
VECTOR_LOAD_B_K4(1, 0);
VECTOR_LOAD_B_K4(2, 0);
VECTOR_LOAD_B_K4(3, 0);
TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3);
SCALE_B4_K4(0, 0, 1, 2, 3);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
VECTOR_LOAD_A(pg_true, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1);
VECTOR_LOAD_A(pg_true, 0, 2);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2);
VECTOR_LOAD_A(pg_true, 0, 3);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3);
VECTOR_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0);
VECTOR_LOAD_A(pg_true, 1, 1);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 1);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 1);
VECTOR_LOAD_A(pg_true, 1, 2);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 2);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 2);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 2);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 2);
VECTOR_LOAD_A(pg_true, 1, 3);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 3);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 3);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 3);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 3);
}
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
VECTOR_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0);
BROADCAST_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0);
BROADCAST_LOAD_B(3, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0);
}
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 0, 2);
VECTOR_STORE(pg_true, 0, 3);
VECTOR_STORE(pg_true, 1, 0);
VECTOR_STORE(pg_true, 1, 1);
VECTOR_STORE(pg_true, 1, 2);
VECTOR_STORE(pg_true, 1, 3);
INCR_C_POINTER(0, v_size2);
INCR_C_POINTER(1, v_size2);
INCR_C_POINTER(2, v_size2);
INCR_C_POINTER(3, v_size2);
}
for (; i < v_m1; i += v_size) {
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(v_size);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
if (LIKELY(packed_b != NULL)) {
for (; k < K; k++) {
UNPACK_QUADWORD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
}
} else {
for (; k < k4; k += 4) {
VECTOR_LOAD_B_K4(0, 0);
VECTOR_LOAD_B_K4(1, 0);
VECTOR_LOAD_B_K4(2, 0);
VECTOR_LOAD_B_K4(3, 0);
TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3);
SCALE_B4_K4(0, 0, 1, 2, 3);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
VECTOR_LOAD_A(pg_true, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1);
VECTOR_LOAD_A(pg_true, 0, 2);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2);
VECTOR_LOAD_A(pg_true, 0, 3);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3);
}
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
BROADCAST_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0);
BROADCAST_LOAD_B(3, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0);
}
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 0, 2);
VECTOR_STORE(pg_true, 0, 3);
INCR_C_POINTER(0, v_size);
INCR_C_POINTER(1, v_size);
INCR_C_POINTER(2, v_size);
INCR_C_POINTER(3, v_size);
}
for (; i < M; i += v_size) {
const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M));
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(0);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
if (LIKELY(packed_b != NULL)) {
for (; k < K; k++) {
UNPACK_QUADWORD_B(0, 0);
VECTOR_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
}
} else {
for (; k < k4; k += 4) {
VECTOR_LOAD_B_K4(0, 0);
VECTOR_LOAD_B_K4(1, 0);
VECTOR_LOAD_B_K4(2, 0);
VECTOR_LOAD_B_K4(3, 0);
TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3);
SCALE_B4_K4(0, 0, 1, 2, 3);
VECTOR_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
VECTOR_LOAD_A(pg_tail, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1);
VECTOR_LOAD_A(pg_tail, 0, 2);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2);
VECTOR_LOAD_A(pg_tail, 0, 3);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3);
}
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 1, 0);
BROADCAST_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 2, 0);
BROADCAST_LOAD_B(3, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 3, 0);
}
}
VECTOR_STORE(pg_tail, 0, 0);
VECTOR_STORE(pg_tail, 0, 1);
VECTOR_STORE(pg_tail, 0, 2);
VECTOR_STORE(pg_tail, 0, 3);
INCR_C_POINTER(0, 0);
INCR_C_POINTER(1, 0);
INCR_C_POINTER(2, 0);
INCR_C_POINTER(3, 0);
}
UPDATE_B_POINTER(4);
RESET_A_POINTER();
UPDATE_C_POINTER(4);
}
for (; j < N; j++) {
CREATE_C_POINTER(0, 0);
CREATE_B_POINTER(0, 0);
BLASLONG i = 0;
for (; i < v_m2; i += v_size2) {
CREATE_A_POINTER(0, 0);
CREATE_A_POINTER(1, v_size);
UPDATE_A_POINTER(v_size2);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(1, 0);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
VECTOR_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 1, 0);
INCR_C_POINTER(0, v_size2);
}
for (; i < v_m1; i += v_size) {
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(v_size);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
}
VECTOR_STORE(pg_true, 0, 0);
INCR_C_POINTER(0, v_size);
}
for (; i < M; i += v_size) {
const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M));
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(0);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0);
}
VECTOR_STORE(pg_tail, 0, 0);
INCR_C_POINTER(0, 0);
}
UPDATE_B_POINTER(1);
RESET_A_POINTER();
UPDATE_C_POINTER(1);
}
if (pack_b)
free(packed_b);
return 0;
}

483
kernel/arm64/sgemm_small_kernel_nt_sve.c Normal file
View File

@@ -0,0 +1,483 @@
/***************************************************************************
Copyright (c) 2024, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#include "common.h"
#include <arm_neon.h>
#include <arm_sve.h>
#if defined(__ARM_NEON_SVE_BRIDGE) && defined(__has_include) && \
__has_include(<arm_neon_sve_bridge.h>)
#include <arm_neon_sve_bridge.h>
#else
#define svdup_neonq_f32(fixed_reg) \
({ \
svfloat32_t scalable_reg; \
asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \
scalable_reg; \
})
#define svdup_neonq_f64(fixed_reg) \
({ \
svfloat64_t scalable_reg; \
asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \
scalable_reg; \
})
#endif
#define RESET_A_POINTER() a_offset = A;
#define CREATE_A_POINTER(m, scale) FLOAT* a_offset##m = a_offset + scale;
#define UPDATE_A_POINTER(scale) a_offset = a_offset + scale;
#define A_ELEMENT_K(m, offset_k) *(a_offset##m + (k + offset_k) * lda)
#define A_ELEMENT(m) A_ELEMENT_K(m, 0)
#define RESET_B_POINTER() b_offset = B;
#define CREATE_B_POINTER(n, scale) FLOAT* b_offset##n = b_offset + scale;
#define UPDATE_B_POINTER(scale) b_offset = b_offset + scale;
#define B_ELEMENT_K(n, offset_k) *(b_offset##n + (k + offset_k) * ldb)
#define B_ELEMENT(n) B_ELEMENT_K(n, 0)
#define CREATE_C_POINTER(n, scale) FLOAT* c_offset##n = c_offset + scale * ldc;
#define INCR_C_POINTER(m, incr) // c_offset ## m += incr;
#define UPDATE_C_POINTER(scale) c_offset = c_offset + scale * ldc;
#define C_ELEMENT(m, n) *(c_offset##n + ((m * v_size) + i))
// #undef C_ELEMENT
// #define C_ELEMENT(m, n) C[(i+(m))+(j+(n))*ldc]
#define PACK_ELEMENT_K(n, offset_k) packed_b[(k + offset_k) * 4 + n]
#define PACK_ELEMENT(n) PACK_ELEMENT_K(n, 0)
// ASIMD
#define DECLARE_RESULT_VECTOR4(m, n) \
float32x4_t result##m##n = vdupq_n_f32(0.0);
#define DECLARE_RESULT(m, n) float32_t result##m##n = 0.0;
#define BROADCAST_LOAD_A4(m, offset_k) \
float32x4_t a##m##_k##offset_k = vld1q_dup_f32(&A_ELEMENT_K(m, offset_k));
#define LOAD_A1(m, offset_k) \
float32_t a##m##_k##offset_k = A_ELEMENT_K(m, offset_k);
#define VECTOR_LOAD_B4(n, offset_k) \
float32x4_t b##n##_k##offset_k = vld1q_f32(&B_ELEMENT_K(n, offset_k));
#define GATHER_LOAD_B4(n, offset_k) \
float32x4_t b##n##_k##offset_k = vdupq_n_f32(B_ELEMENT_K(n, offset_k)); \
b##n##_k##offset_k = \
vsetq_lane_f32(B_ELEMENT_K(n + 1, offset_k), b##n##_k##offset_k, 1); \
b##n##_k##offset_k = \
vsetq_lane_f32(B_ELEMENT_K(n + 2, offset_k), b##n##_k##offset_k, 2); \
b##n##_k##offset_k = \
vsetq_lane_f32(B_ELEMENT_K(n + 3, offset_k), b##n##_k##offset_k, 3);
#define VECTOR_UNPACK_B4(n, offset_k) \
float32x4_t b##n##_k##offset_k = vld1q_f32(&PACK_ELEMENT_K(n, offset_k));
#define VECTOR_PACK_B4(n, offset_k) \
vst1q_f32(&PACK_ELEMENT_K(n, offset_k), b##n##_k##offset_k);
#define PACK_B0(n, offset_k) \
PACK_ELEMENT_K(n, offset_k) = vget_lane_f32(b##n##_k##offset_k, 0);
#define UPDATE_RESULT_VECTOR4(m, n, offset_k) \
result##m##n = \
vfmaq_f32(result##m##n, a##m##_k##offset_k, b##n##_k##offset_k);
#define UPDATE_RESULT(m, n, offset_k) \
result##m##n = result##m##n + a##m##_k##offset_k * b##n##_k##offset_k;
#ifdef B0
#define SCATTER_STORE4(m, n) \
result##m##n = vmulq_f32(result##m##n, vdupq_n_f32(alpha)); \
C_ELEMENT(m, n + 0) = vgetq_lane_f32(result##m##n, 0); \
C_ELEMENT(m, n + 1) = vgetq_lane_f32(result##m##n, 1); \
C_ELEMENT(m, n + 2) = vgetq_lane_f32(result##m##n, 2); \
C_ELEMENT(m, n + 3) = vgetq_lane_f32(result##m##n, 3);
#else
#define SCATTER_STORE4(m, n) \
result##m##n = vmulq_f32(result##m##n, vdupq_n_f32(alpha)); \
C_ELEMENT(m, n + 0) = \
C_ELEMENT(m, n + 0) * beta + vgetq_lane_f32(result##m##n, 0); \
C_ELEMENT(m, n + 1) = \
C_ELEMENT(m, n + 1) * beta + vgetq_lane_f32(result##m##n, 1); \
C_ELEMENT(m, n + 2) = \
C_ELEMENT(m, n + 2) * beta + vgetq_lane_f32(result##m##n, 2); \
C_ELEMENT(m, n + 3) = \
C_ELEMENT(m, n + 3) * beta + vgetq_lane_f32(result##m##n, 3);
#endif
// SVE
#define DECLARE_RESULT_VECTOR(m, n) svfloat32_t result##m##n = svdup_f32(0.0);
#define BROADCAST_LOAD_A(m, offset_k) \
svfloat32_t a##s##m##_k##offset_k = svdup_f32(A_ELEMENT_K(m, offset_k));
#define BROADCAST_LOAD_B(n, offset_k) \
svfloat32_t b##s##n##_k##offset_k = svdup_f32(B_ELEMENT_K(n, offset_k));
#define VECTOR_LOAD_A(pg, m, offset_k) \
svfloat32_t a##s##m##_k##offset_k = svld1(pg, &A_ELEMENT_K(m, offset_k));
#define QUADWORD_LOAD_B(n, offset_k) \
svfloat32_t b##s##n##_k##offset_k = \
svld1rq(pg_true, &B_ELEMENT_K(n, offset_k));
#define PACK_B(n, offset_k) \
svst1(pg_first, &PACK_ELEMENT_K(n, offset_k), b##s##n##_k##offset_k);
#define VECTOR_PACK_B(n, offset_k) \
svst1(pg_true, &PACK_ELEMENT_K(n* v_size, offset_k), b##s##n##_k##offset_k);
#define QUADWORD_PACK_B(n, offset_k) \
svst1(pg_quad, &PACK_ELEMENT_K(n, offset_k), b##s##n##_k##offset_k);
#define UNPACK_VECTOR_B(n, offset_k) \
svfloat32_t b##s##n##_k##offset_k = \
svld1(pg_true, &PACK_ELEMENT_K(n * v_size, offset_k));
#define UNPACK_BROADCAST_B(n, offset_k) \
svfloat32_t b##s##n##_k##offset_k = svdup_f32(PACK_ELEMENT_K(n, offset_k));
#define UNPACK_QUADWORD_B(n, offset_k) \
svfloat32_t b##s##n##_k##offset_k = \
svld1rq(pg_true, &PACK_ELEMENT_K(n, offset_k));
#define UPDATE_RESULT_VECTOR(pg, m, n, offset_k) \
result##m##n = \
svmla_m(pg, result##m##n, a##s##m##_k##offset_k, b##s##n##_k##offset_k);
#define UPDATE_RESULT_VECTOR_QUADWORD(m, n, outer, lane, offset_k) \
result##m##n = svmla_lane( \
result##m##n, a##s##m##_k##offset_k, b##s##outer##_k##offset_k, lane);
#ifdef B0
#define VECTOR_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
svst1(pg, &C_ELEMENT(m, n), result##m##n);
#define SCATTER_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
#else
#define VECTOR_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
result##m##n = \
svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(m, n)), beta_vec); \
svst1(pg, &C_ELEMENT(m, n), result##m##n);
#define SCATTER_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
result##m##n = svmla_m(pg, \
result##m##n, \
svld1_gather_index(pg, &C_ELEMENT(m, n), ldc_vec), \
beta_vec); \
svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
#endif
#ifndef LIKELY
#ifdef __GNUC__
#define LIKELY(x) __builtin_expect(!!(x), 1)
#else
#define LIKELY(x) (x)
#endif
#endif
#ifdef B0
int
CNAME(BLASLONG M,
BLASLONG N,
BLASLONG K,
IFLOAT* A,
BLASLONG lda,
FLOAT alpha,
IFLOAT* B,
BLASLONG ldb,
FLOAT* C,
BLASLONG ldc)
#else
int
CNAME(BLASLONG M,
BLASLONG N,
BLASLONG K,
IFLOAT* A,
BLASLONG lda,
FLOAT alpha,
IFLOAT* B,
BLASLONG ldb,
FLOAT beta,
FLOAT* C,
BLASLONG ldc)
#endif
{
const uint64_t v_size = svcntw();
const uint64_t v_size2 = v_size * 2;
const svbool_t pg_true = svptrue_b32();
const svbool_t pg_quad = svwhilelt_b32(0, 4);
const svbool_t pg_first = svwhilelt_b32(0, 1);
const svfloat32_t alpha_vec = svdup_f32(alpha);
#ifndef B0
const svfloat32_t beta_vec = svdup_f32(beta);
#endif
const BLASLONG n4 = N & -4;
const BLASLONG v_m2 = M & -v_size2;
const BLASLONG v_m1 = M & -v_size;
const int pack_b = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0;
FLOAT* packed_b =
(pack_b) ? packed_b = (FLOAT*)malloc(K * 4 * sizeof(FLOAT)) : NULL;
FLOAT* b_offset = B;
FLOAT* a_offset = A;
FLOAT* c_offset = C;
BLASLONG j = 0;
for (; j < n4; j += 4) {
CREATE_C_POINTER(0, 0);
CREATE_C_POINTER(1, 1);
CREATE_C_POINTER(2, 2);
CREATE_C_POINTER(3, 3);
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
CREATE_B_POINTER(2, 2);
CREATE_B_POINTER(3, 3);
BLASLONG i = 0;
for (; i < v_m2; i += v_size2) {
CREATE_A_POINTER(0, 0);
CREATE_A_POINTER(1, v_size);
UPDATE_A_POINTER(v_size2);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
DECLARE_RESULT_VECTOR(1, 0);
DECLARE_RESULT_VECTOR(1, 1);
DECLARE_RESULT_VECTOR(1, 2);
DECLARE_RESULT_VECTOR(1, 3);
if (LIKELY(packed_b != NULL)) {
if (i == 0) {
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
QUADWORD_PACK_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
VECTOR_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0);
}
} else {
for (; k < K; k++) {
UNPACK_QUADWORD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
VECTOR_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0);
}
}
} else {
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
VECTOR_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0);
}
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 0, 2);
VECTOR_STORE(pg_true, 0, 3);
VECTOR_STORE(pg_true, 1, 0);
VECTOR_STORE(pg_true, 1, 1);
VECTOR_STORE(pg_true, 1, 2);
VECTOR_STORE(pg_true, 1, 3);
INCR_C_POINTER(0, v_size2);
INCR_C_POINTER(1, v_size2);
INCR_C_POINTER(2, v_size2);
INCR_C_POINTER(3, v_size2);
}
for (; i < v_m1; i += v_size) {
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(v_size);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
if (LIKELY(packed_b != NULL)) {
for (; k < K; k++) {
UNPACK_QUADWORD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
}
} else {
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
}
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 0, 2);
VECTOR_STORE(pg_true, 0, 3);
INCR_C_POINTER(0, v_size);
INCR_C_POINTER(1, v_size);
INCR_C_POINTER(2, v_size);
INCR_C_POINTER(3, v_size);
}
for (; i < M; i += v_size) {
const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M));
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(0);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
if (LIKELY(packed_b != NULL)) {
for (; k < K; k++) {
UNPACK_QUADWORD_B(0, 0);
VECTOR_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
}
} else {
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
}
}
VECTOR_STORE(pg_tail, 0, 0);
VECTOR_STORE(pg_tail, 0, 1);
VECTOR_STORE(pg_tail, 0, 2);
VECTOR_STORE(pg_tail, 0, 3);
INCR_C_POINTER(0, 0);
INCR_C_POINTER(1, 0);
INCR_C_POINTER(2, 0);
INCR_C_POINTER(3, 0);
}
UPDATE_B_POINTER(4);
RESET_A_POINTER();
UPDATE_C_POINTER(4);
}
for (; j < N; j++) {
CREATE_C_POINTER(0, 0);
CREATE_B_POINTER(0, 0);
BLASLONG i = 0;
for (; i < v_m2; i += v_size2) {
CREATE_A_POINTER(0, 0);
CREATE_A_POINTER(1, v_size);
UPDATE_A_POINTER(v_size2);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(1, 0);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
VECTOR_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 1, 0);
INCR_C_POINTER(0, v_size2);
}
for (; i < v_m1; i += v_size) {
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(v_size);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
}
VECTOR_STORE(pg_true, 0, 0);
INCR_C_POINTER(0, v_size);
}
for (; i < M; i += v_size) {
const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M));
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(0);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
VECTOR_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0);
}
VECTOR_STORE(pg_tail, 0, 0);
INCR_C_POINTER(0, 0);
}
UPDATE_B_POINTER(1);
RESET_A_POINTER();
UPDATE_C_POINTER(1);
}
if (pack_b)
free(packed_b);
return 0;
}

719
kernel/arm64/sgemm_small_kernel_tn_sve.c Normal file
View File

@@ -0,0 +1,719 @@
/***************************************************************************
Copyright (c) 2024, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#include "common.h"
#include <arm_neon.h>
#include <arm_sve.h>
#if defined(__ARM_NEON_SVE_BRIDGE) && defined(__has_include) && \
__has_include(<arm_neon_sve_bridge.h>)
#include <arm_neon_sve_bridge.h>
#else
#define svdup_neonq_f32(fixed_reg) \
({ \
svfloat32_t scalable_reg; \
asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \
scalable_reg; \
})
#define svdup_neonq_f64(fixed_reg) \
({ \
svfloat64_t scalable_reg; \
asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \
scalable_reg; \
})
#endif
#define RESET_A_POINTER() a_offset = A;
#define CREATE_A_POINTER(m, scale) FLOAT* a_offset##m = a_offset + scale * lda;
#define UPDATE_A_POINTER(scale) a_offset = a_offset + scale * lda;
#define A_ELEMENT_K(m, offset_k) *(a_offset##m + (k + offset_k))
#define A_ELEMENT(m) A_ELEMENT_K(m, 0)
#define RESET_B_POINTER() b_offset = B;
#define CREATE_B_POINTER(n, scale) FLOAT* b_offset##n = b_offset + scale * ldb;
#define UPDATE_B_POINTER(scale) b_offset = b_offset + scale * ldb;
#define B_ELEMENT_K(n, offset_k) *(b_offset##n + (k + offset_k))
#define B_ELEMENT(n) B_ELEMENT_K(n, 0)
#define CREATE_C_POINTER(m, scale) FLOAT* c_offset##m = c_offset + scale;
#define INCR_C_POINTER(m, incr) // c_offset ## m += incr * ldc;
#define UPDATE_C_POINTER(scale) c_offset += scale;
#define C_ELEMENT(m, n) \
*(c_offset##m + ((j + n) * ldc)) // C[(i+(m))+(j+(n))*ldc]
// #undef C_ELEMENT
// #define C_ELEMENT(m, n) C[(i+(m))+(j+(n))*ldc]
#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * v_size2 + m]
#define PACK_ELEMENT(m) PACK_ELEMENT_K(m, 0)
// ASIMD
#define DECLARE_RESULT_VECTOR4(m, n) \
float32x4_t result##m##n = vdupq_n_f32(0.0);
#define DECLARE_RESULT(m, n) float32_t result##m##n = 0.0;
#define BROADCAST_LOAD_A4(m, offset_k) \
float32x4_t a##m##_k##offset_k = vld1q_dup_f32(&A_ELEMENT_K(m, offset_k));
#define LOAD_A1(m, offset_k) \
float32_t a##m##_k##offset_k = A_ELEMENT_K(m, offset_k);
#define GATHER_LOAD_B4(n, offset_k) \
float32x4_t b##n##_k##offset_k = vdupq_n_f32(B_ELEMENT_K(n, offset_k)); \
b##n##_k##offset_k = \
vsetq_lane_f32(B_ELEMENT_K(n + 1, offset_k), b##n##_k##offset_k, 1); \
b##n##_k##offset_k = \
vsetq_lane_f32(B_ELEMENT_K(n + 2, offset_k), b##n##_k##offset_k, 2); \
b##n##_k##offset_k = \
vsetq_lane_f32(B_ELEMENT_K(n + 3, offset_k), b##n##_k##offset_k, 3);
#define VECTOR_UNPACK_B4(n, offset_k) \
float32x4_t b##n##_k##offset_k = vld1q_f32(&PACK_ELEMENT_K(n, offset_k));
#define PACK_B0(n, offset_k) \
PACK_ELEMENT_K(n, offset_k) = vget_lane_f32(b##n##_k##offset_k, 0);
#define UPDATE_RESULT_VECTOR4(m, n, offset_k) \
result##m##n = \
vfmaq_f32(result##m##n, a##m##_k##offset_k, b##n##_k##offset_k);
#define UPDATE_RESULT(m, n, offset_k) \
result##m##n = result##m##n + a##m##_k##offset_k * b##n##_k##offset_k;
#ifdef B0
#define SCATTER_STORE4(m, n) \
result##m##n = vmulq_f32(result##m##n, vdupq_n_f32(alpha)); \
C_ELEMENT(m, n + 0) = vgetq_lane_f32(result##m##n, 0); \
C_ELEMENT(m, n + 1) = vgetq_lane_f32(result##m##n, 1); \
C_ELEMENT(m, n + 2) = vgetq_lane_f32(result##m##n, 2); \
C_ELEMENT(m, n + 3) = vgetq_lane_f32(result##m##n, 3);
#else
#define SCATTER_STORE4(m, n) \
result##m##n = vmulq_f32(result##m##n, vdupq_n_f32(alpha)); \
C_ELEMENT(m, n + 0) = \
C_ELEMENT(m, n + 0) * beta + vgetq_lane_f32(result##m##n, 0); \
C_ELEMENT(m, n + 1) = \
C_ELEMENT(m, n + 1) * beta + vgetq_lane_f32(result##m##n, 1); \
C_ELEMENT(m, n + 2) = \
C_ELEMENT(m, n + 2) * beta + vgetq_lane_f32(result##m##n, 2); \
C_ELEMENT(m, n + 3) = \
C_ELEMENT(m, n + 3) * beta + vgetq_lane_f32(result##m##n, 3);
#endif
// SVE
#define DECLARE_RESULT_VECTOR(m, n) svfloat32_t result##m##n = svdup_f32(0.0);
#define BROADCAST_LOAD_A(m, offset_k) \
svfloat32_t a##s##m##_k##offset_k = svdup_f32(A_ELEMENT_K(m, offset_k));
#define BROADCAST_LOAD_B(n, offset_k) \
svfloat32_t b##s##n##_k##offset_k = svdup_f32(B_ELEMENT_K(n, offset_k));
#define VECTOR_LOAD_A(pg, m, offset_k) \
svfloat32_t a##s##m##_k##offset_k = svld1(pg, &A_ELEMENT_K(m, offset_k));
#define GATHER_LOAD_A(pg, m, offset_k) \
svfloat32_t a##s##m##_k##offset_k = \
svld1_gather_index(pg, &A_ELEMENT_K(m, offset_k), lda_vec);
#define PACK_A(m, offset_k) \
svst1(pg_first, &PACK_ELEMENT_K(m, offset_k), a##s##m##_k##offset_k);
#define VECTOR_PACK_A(m, offset_k) \
svst1(pg_true, &PACK_ELEMENT_K(m* v_size, offset_k), a##s##m##_k##offset_k);
#define QUADWORD_PACK_A(m, offset_k) \
svst1(pg_quad, &PACK_ELEMENT_K(m, offset_k), a##s##m##_k##offset_k);
#define UNPACK_VECTOR_A(m, offset_k) \
svfloat32_t a##s##m##_k##offset_k = \
svld1(pg_true, &PACK_ELEMENT_K(m * v_size, offset_k));
#define UNPACK_BROADCAST_A(m, offset_k) \
svfloat32_t a##s##m##_k##offset_k = svdup_f32(PACK_ELEMENT_K(m, offset_k));
#define UNPACK_QUADWORD_A(m, offset_k) \
svfloat32_t a##s##m##_k##offset_k = \
svld1rq(pg_true, &PACK_ELEMENT_K(m, offset_k));
#define UPDATE_RESULT_VECTOR(pg, m, n, offset_k) \
result##m##n = \
svmla_m(pg, result##m##n, a##s##m##_k##offset_k, b##s##n##_k##offset_k);
#define UPDATE_RESULT_VECTOR_QUADWORD(m, n, outer, lane, offset_k) \
result##m##n = svmla_lane( \
result##m##n, a##s##m##_k##offset_k, b##s##outer##_k##offset_k, lane);
#ifdef B0
#define VECTOR_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
svst1(pg, &C_ELEMENT(m, n), result##m##n);
#define SCATTER_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
#else
#define VECTOR_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
result##m##n = \
svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(m, n)), beta_vec); \
svst1(pg, &C_ELEMENT(m, n), result##m##n);
#define SCATTER_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
result##m##n = svmla_m(pg, \
result##m##n, \
svld1_gather_index(pg, &C_ELEMENT(m, n), ldc_vec), \
beta_vec); \
svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
#endif
#ifndef LIKELY
#ifdef __GNUC__
#define LIKELY(x) __builtin_expect(!!(x), 1)
#else
#define LIKELY(x) (x)
#endif
#endif
#ifdef B0
int
CNAME(BLASLONG M,
BLASLONG N,
BLASLONG K,
IFLOAT* A,
BLASLONG lda,
FLOAT alpha,
IFLOAT* B,
BLASLONG ldb,
FLOAT* C,
BLASLONG ldc)
#else
int
CNAME(BLASLONG M,
BLASLONG N,
BLASLONG K,
IFLOAT* A,
BLASLONG lda,
FLOAT alpha,
IFLOAT* B,
BLASLONG ldb,
FLOAT beta,
FLOAT* C,
BLASLONG ldc)
#endif
{
const uint64_t v_size = svcntw();
const uint64_t v_size2 = v_size * 2;
const svbool_t pg_true = svptrue_b32();
const svbool_t pg_quad = svwhilelt_b32(0, 4);
const svbool_t pg_first = svwhilelt_b32(0, 1);
const svfloat32_t alpha_vec = svdup_f32(alpha);
#ifndef B0
const svfloat32_t beta_vec = svdup_f32(beta);
#endif
const svuint32_t lda_vec = svindex_u32(0LL, lda);
const BLASLONG v_m2 = M & -v_size2;
const BLASLONG v_m1 = M & -v_size;
const BLASLONG n8 = N & -8;
const BLASLONG n4 = N & -4;
const int pack_a = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0;
FLOAT* packed_a =
(pack_a) ? packed_a = (FLOAT*)malloc(K * v_size2 * sizeof(FLOAT)) : NULL;
FLOAT* a_offset = A;
FLOAT* b_offset = B;
FLOAT* c_offset = C;
BLASLONG i = 0;
for (; i < v_m2; i += v_size2) {
CREATE_C_POINTER(0, 0);
CREATE_C_POINTER(1, v_size);
CREATE_A_POINTER(0, 0);
CREATE_A_POINTER(1, v_size);
BLASLONG j = 0;
for (; j < n8; j += 8) {
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
CREATE_B_POINTER(2, 2);
CREATE_B_POINTER(3, 3);
CREATE_B_POINTER(4, 4);
CREATE_B_POINTER(5, 5);
CREATE_B_POINTER(6, 6);
CREATE_B_POINTER(7, 7);
UPDATE_B_POINTER(8);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
DECLARE_RESULT_VECTOR(0, 4);
DECLARE_RESULT_VECTOR(0, 5);
DECLARE_RESULT_VECTOR(0, 6);
DECLARE_RESULT_VECTOR(0, 7);
DECLARE_RESULT_VECTOR(1, 0);
DECLARE_RESULT_VECTOR(1, 1);
DECLARE_RESULT_VECTOR(1, 2);
DECLARE_RESULT_VECTOR(1, 3);
DECLARE_RESULT_VECTOR(1, 4);
DECLARE_RESULT_VECTOR(1, 5);
DECLARE_RESULT_VECTOR(1, 6);
DECLARE_RESULT_VECTOR(1, 7);
if (LIKELY(packed_a != NULL)) {
if (j == 0) {
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
VECTOR_PACK_A(0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
GATHER_LOAD_A(pg_true, 1, 0);
VECTOR_PACK_A(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0);
BROADCAST_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0);
BROADCAST_LOAD_B(3, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0);
BROADCAST_LOAD_B(4, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 4, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 4, 0);
BROADCAST_LOAD_B(5, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 5, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 5, 0);
BROADCAST_LOAD_B(6, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 6, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 6, 0);
BROADCAST_LOAD_B(7, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 7, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 7, 0);
}
} else {
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
UNPACK_VECTOR_A(0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
UNPACK_VECTOR_A(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0);
BROADCAST_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0);
BROADCAST_LOAD_B(3, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0);
BROADCAST_LOAD_B(4, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 4, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 4, 0);
BROADCAST_LOAD_B(5, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 5, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 5, 0);
BROADCAST_LOAD_B(6, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 6, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 6, 0);
BROADCAST_LOAD_B(7, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 7, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 7, 0);
}
}
} else {
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
GATHER_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0);
BROADCAST_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0);
BROADCAST_LOAD_B(3, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0);
BROADCAST_LOAD_B(4, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 4, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 4, 0);
BROADCAST_LOAD_B(5, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 5, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 5, 0);
BROADCAST_LOAD_B(6, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 6, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 6, 0);
BROADCAST_LOAD_B(7, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 7, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 7, 0);
}
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 0, 2);
VECTOR_STORE(pg_true, 0, 3);
VECTOR_STORE(pg_true, 0, 4);
VECTOR_STORE(pg_true, 0, 5);
VECTOR_STORE(pg_true, 0, 6);
VECTOR_STORE(pg_true, 0, 7);
VECTOR_STORE(pg_true, 1, 0);
VECTOR_STORE(pg_true, 1, 1);
VECTOR_STORE(pg_true, 1, 2);
VECTOR_STORE(pg_true, 1, 3);
VECTOR_STORE(pg_true, 1, 4);
VECTOR_STORE(pg_true, 1, 5);
VECTOR_STORE(pg_true, 1, 6);
VECTOR_STORE(pg_true, 1, 7);
INCR_C_POINTER(0, 8);
INCR_C_POINTER(1, 8);
}
for (; j < n4; j += 4) {
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
CREATE_B_POINTER(2, 2);
CREATE_B_POINTER(3, 3);
UPDATE_B_POINTER(4);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
DECLARE_RESULT_VECTOR(1, 0);
DECLARE_RESULT_VECTOR(1, 1);
DECLARE_RESULT_VECTOR(1, 2);
DECLARE_RESULT_VECTOR(1, 3);
if (LIKELY(packed_a != NULL)) {
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
UNPACK_VECTOR_A(0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
UNPACK_VECTOR_A(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0);
BROADCAST_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0);
BROADCAST_LOAD_B(3, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0);
}
} else {
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
GATHER_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0);
BROADCAST_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0);
BROADCAST_LOAD_B(3, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0);
}
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 0, 2);
VECTOR_STORE(pg_true, 0, 3);
VECTOR_STORE(pg_true, 1, 0);
VECTOR_STORE(pg_true, 1, 1);
VECTOR_STORE(pg_true, 1, 2);
VECTOR_STORE(pg_true, 1, 3);
INCR_C_POINTER(0, 4);
INCR_C_POINTER(1, 4);
}
for (; j < N; j++) {
CREATE_B_POINTER(0, 0);
UPDATE_B_POINTER(1);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(1, 0);
if (LIKELY(packed_a != NULL)) {
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
UNPACK_VECTOR_A(0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
UNPACK_VECTOR_A(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
}
} else {
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
GATHER_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
}
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 1, 0);
INCR_C_POINTER(0, 1);
INCR_C_POINTER(1, 1);
}
UPDATE_A_POINTER(v_size2);
RESET_B_POINTER();
UPDATE_C_POINTER(v_size2);
}
for (; i < v_m1; i += v_size) {
CREATE_C_POINTER(0, 0);
CREATE_A_POINTER(0, 0);
BLASLONG j = 0;
for (; j < n8; j += 8) {
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
CREATE_B_POINTER(2, 2);
CREATE_B_POINTER(3, 3);
CREATE_B_POINTER(4, 4);
CREATE_B_POINTER(5, 5);
CREATE_B_POINTER(6, 6);
CREATE_B_POINTER(7, 7);
UPDATE_B_POINTER(8);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
DECLARE_RESULT_VECTOR(0, 4);
DECLARE_RESULT_VECTOR(0, 5);
DECLARE_RESULT_VECTOR(0, 6);
DECLARE_RESULT_VECTOR(0, 7);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
BROADCAST_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0);
BROADCAST_LOAD_B(3, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0);
BROADCAST_LOAD_B(4, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 4, 0);
BROADCAST_LOAD_B(5, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 5, 0);
BROADCAST_LOAD_B(6, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 6, 0);
BROADCAST_LOAD_B(7, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 7, 0);
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 0, 2);
VECTOR_STORE(pg_true, 0, 3);
VECTOR_STORE(pg_true, 0, 4);
VECTOR_STORE(pg_true, 0, 5);
VECTOR_STORE(pg_true, 0, 6);
VECTOR_STORE(pg_true, 0, 7);
INCR_C_POINTER(0, 8);
}
for (; j < n4; j += 4) {
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
CREATE_B_POINTER(2, 2);
CREATE_B_POINTER(3, 3);
UPDATE_B_POINTER(4);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
BROADCAST_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0);
BROADCAST_LOAD_B(3, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0);
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 0, 2);
VECTOR_STORE(pg_true, 0, 3);
INCR_C_POINTER(0, 4);
}
for (; j < N; j++) {
CREATE_B_POINTER(0, 0);
UPDATE_B_POINTER(1);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
}
VECTOR_STORE(pg_true, 0, 0);
INCR_C_POINTER(0, 1);
}
UPDATE_A_POINTER(v_size);
RESET_B_POINTER();
UPDATE_C_POINTER(v_size);
}
for (; i < M; i += v_size) {
const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M));
CREATE_C_POINTER(0, 0);
CREATE_A_POINTER(0, 0);
BLASLONG j = 0;
for (; j < n8; j += 8) {
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
CREATE_B_POINTER(2, 2);
CREATE_B_POINTER(3, 3);
CREATE_B_POINTER(4, 4);
CREATE_B_POINTER(5, 5);
CREATE_B_POINTER(6, 6);
CREATE_B_POINTER(7, 7);
UPDATE_B_POINTER(8);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
DECLARE_RESULT_VECTOR(0, 4);
DECLARE_RESULT_VECTOR(0, 5);
DECLARE_RESULT_VECTOR(0, 6);
DECLARE_RESULT_VECTOR(0, 7);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 1, 0);
BROADCAST_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 2, 0);
BROADCAST_LOAD_B(3, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 3, 0);
BROADCAST_LOAD_B(4, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 4, 0);
BROADCAST_LOAD_B(5, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 5, 0);
BROADCAST_LOAD_B(6, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 6, 0);
BROADCAST_LOAD_B(7, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 7, 0);
}
VECTOR_STORE(pg_tail, 0, 0);
VECTOR_STORE(pg_tail, 0, 1);
VECTOR_STORE(pg_tail, 0, 2);
VECTOR_STORE(pg_tail, 0, 3);
VECTOR_STORE(pg_tail, 0, 4);
VECTOR_STORE(pg_tail, 0, 5);
VECTOR_STORE(pg_tail, 0, 6);
VECTOR_STORE(pg_tail, 0, 7);
INCR_C_POINTER(0, 8);
}
for (; j < n4; j += 4) {
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
CREATE_B_POINTER(2, 2);
CREATE_B_POINTER(3, 3);
UPDATE_B_POINTER(4);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0);
BROADCAST_LOAD_B(1, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 1, 0);
BROADCAST_LOAD_B(2, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 2, 0);
BROADCAST_LOAD_B(3, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 3, 0);
}
VECTOR_STORE(pg_tail, 0, 0);
VECTOR_STORE(pg_tail, 0, 1);
VECTOR_STORE(pg_tail, 0, 2);
VECTOR_STORE(pg_tail, 0, 3);
INCR_C_POINTER(0, 4);
}
for (; j < N; j++) {
CREATE_B_POINTER(0, 0);
UPDATE_B_POINTER(1);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0);
}
VECTOR_STORE(pg_tail, 0, 0);
INCR_C_POINTER(0, 1);
}
UPDATE_A_POINTER(0);
RESET_B_POINTER();
UPDATE_C_POINTER(0);
}
if (pack_a)
free(packed_a);
return 0;
}

678
kernel/arm64/sgemm_small_kernel_tt_sve.c Normal file
View File

@@ -0,0 +1,678 @@
/***************************************************************************
Copyright (c) 2024, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#include "common.h"
#include <arm_neon.h>
#include <arm_sve.h>
#if defined(__ARM_NEON_SVE_BRIDGE) && defined(__has_include) && \
__has_include(<arm_neon_sve_bridge.h>)
#include <arm_neon_sve_bridge.h>
#else
#define svdup_neonq_f32(fixed_reg) \
({ \
svfloat32_t scalable_reg; \
asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \
scalable_reg; \
})
#define svdup_neonq_f64(fixed_reg) \
({ \
svfloat64_t scalable_reg; \
asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \
scalable_reg; \
})
#endif
#define RESET_A_POINTER() a_offset = A;
#define CREATE_A_POINTER(m, scale) FLOAT* a_offset##m = a_offset + scale * lda;
#define UPDATE_A_POINTER(scale) a_offset = a_offset + scale * lda;
#define A_ELEMENT_K(m, offset_k) *(a_offset##m + (k + offset_k))
#define A_ELEMENT(m) A_ELEMENT_K(m, 0)
#define RESET_B_POINTER() b_offset = B;
#define CREATE_B_POINTER(n, scale) FLOAT* b_offset##n = b_offset + scale;
#define UPDATE_B_POINTER(scale) b_offset = b_offset + scale;
#define B_ELEMENT_K(n, offset_k) *(b_offset##n + (k + offset_k) * ldb)
#define B_ELEMENT(n) B_ELEMENT_K(n, 0)
#define CREATE_C_POINTER(m, scale) FLOAT* c_offset##m = c_offset + scale;
#define INCR_C_POINTER(m, incr) // c_offset ## m += incr * ldc;
#define UPDATE_C_POINTER(scale) c_offset += scale;
#define C_ELEMENT(m, n) \
*(c_offset##m + ((j + n) * ldc)) // C[(i+(m))+(j+(n))*ldc]
// #undef C_ELEMENT
// #define C_ELEMENT(m, n) C[(i+(m))+(j+(n))*ldc]
#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * v_size2 + m]
#define PACK_ELEMENT(m) PACK_ELEMENT_K(m, 0)
// ASIMD
#define DECLARE_RESULT_VECTOR4(m, n) \
float32x4_t result##m##n = vdupq_n_f32(0.0);
#define DECLARE_RESULT(m, n) float32_t result##m##n = 0.0;
#define BROADCAST_LOAD_A4(m, offset_k) \
float32x4_t a##m##_k##offset_k = vld1q_dup_f32(&A_ELEMENT_K(m, offset_k));
#define LOAD_A1(m, offset_k) \
float32_t a##m##_k##offset_k = A_ELEMENT_K(m, offset_k);
#define VECTOR_LOAD_B4(n, offset_k) \
float32x4_t b##n##_k##offset_k = vld1q_f32(&B_ELEMENT_K(n, offset_k));
#define GATHER_LOAD_B4(n, offset_k) \
float32x4_t b##n##_k##offset_k = vdupq_n_f32(B_ELEMENT_K(n, offset_k)); \
b##n##_k##offset_k = \
vsetq_lane_f32(B_ELEMENT_K(n + 1, offset_k), b##n##_k##offset_k, 1); \
b##n##_k##offset_k = \
vsetq_lane_f32(B_ELEMENT_K(n + 2, offset_k), b##n##_k##offset_k, 2); \
b##n##_k##offset_k = \
vsetq_lane_f32(B_ELEMENT_K(n + 3, offset_k), b##n##_k##offset_k, 3);
#define VECTOR_UNPACK_B4(n, offset_k) \
float32x4_t b##n##_k##offset_k = vld1q_f32(&PACK_ELEMENT_K(n, offset_k));
#define VECTOR_PACK_B4(n, offset_k) \
vst1q_f32(&PACK_ELEMENT_K(n, offset_k), b##n##_k##offset_k);
#define PACK_B0(n, offset_k) \
PACK_ELEMENT_K(n, offset_k) = vget_lane_f32(b##n##_k##offset_k, 0);
#define UPDATE_RESULT_VECTOR4(m, n, offset_k) \
result##m##n = \
vfmaq_f32(result##m##n, a##m##_k##offset_k, b##n##_k##offset_k);
#define UPDATE_RESULT(m, n, offset_k) \
result##m##n = result##m##n + a##m##_k##offset_k * b##n##_k##offset_k;
#ifdef B0
#define VECTOR_STORE4(m, n) \
vst1q_f32(&C_ELEMENT(m, n), vmulq_f32(result##m##n, vdupq_n_f32(alpha)));
#define STORE(m, n) C_ELEMENT(m, n) = alpha * result##m##n;
#else
#define VECTOR_STORE4(m, n) \
result##m##n = vmulq_f32(result##m##n, vdupq_n_f32(alpha)); \
result##m##n = \
vfmaq_f32(result##m##n, vld1q_f32(&C_ELEMENT(m, n)), vdupq_n_f32(beta)); \
vst1q_f32(&C_ELEMENT(m, n), result##m##n);
#define STORE(m, n) \
C_ELEMENT(m, n) = C_ELEMENT(m, n) * beta + alpha * result##m##n;
#endif
// SVE
#define DECLARE_RESULT_VECTOR(m, n) svfloat32_t result##m##n = svdup_f32(0.0);
#define BROADCAST_LOAD_A(m, offset_k) \
svfloat32_t a##s##m##_k##offset_k = svdup_f32(A_ELEMENT_K(m, offset_k));
#define BROADCAST_LOAD_B(n, offset_k) \
svfloat32_t b##s##n##_k##offset_k = svdup_f32(B_ELEMENT_K(n, offset_k));
#define VECTOR_LOAD_A(pg, m, offset_k) \
svfloat32_t a##s##m##_k##offset_k = svld1(pg, &A_ELEMENT_K(m, offset_k));
#define QUADWORD_LOAD_B(n, offset_k) \
svfloat32_t b##s##n##_k##offset_k = \
svld1rq(pg_true, &B_ELEMENT_K(n, offset_k));
#define GATHER_LOAD_A(pg, m, offset_k) \
svfloat32_t a##s##m##_k##offset_k = \
svld1_gather_index(pg, &A_ELEMENT_K(m, offset_k), lda_vec);
#define PACK_A(m, offset_k) \
svst1(pg_first, &PACK_ELEMENT_K(m, offset_k), a##s##m##_k##offset_k);
#define VECTOR_PACK_A(m, offset_k) \
svst1(pg_true, &PACK_ELEMENT_K(m* v_size, offset_k), a##s##m##_k##offset_k);
#define QUADWORD_PACK_A(m, offset_k) \
svst1(pg_quad, &PACK_ELEMENT_K(m, offset_k), a##s##m##_k##offset_k);
#define UNPACK_VECTOR_A(m, offset_k) \
svfloat32_t a##s##m##_k##offset_k = \
svld1(pg_true, &PACK_ELEMENT_K(m * v_size, offset_k));
#define UNPACK_BROADCAST_A(m, offset_k) \
svfloat32_t a##s##m##_k##offset_k = svdup_f32(PACK_ELEMENT_K(m, offset_k));
#define UNPACK_QUADWORD_A(m, offset_k) \
svfloat32_t a##s##m##_k##offset_k = \
svld1rq(pg_true, &PACK_ELEMENT_K(m, offset_k));
#define UPDATE_RESULT_VECTOR(pg, m, n, offset_k) \
result##m##n = \
svmla_m(pg, result##m##n, a##s##m##_k##offset_k, b##s##n##_k##offset_k);
#define UPDATE_RESULT_VECTOR_QUADWORD(m, n, outer, lane, offset_k) \
result##m##n = svmla_lane( \
result##m##n, a##s##m##_k##offset_k, b##s##outer##_k##offset_k, lane);
#ifdef B0
#define VECTOR_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
svst1(pg, &C_ELEMENT(m, n), result##m##n);
#define SCATTER_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
#else
#define VECTOR_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
result##m##n = \
svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(m, n)), beta_vec); \
svst1(pg, &C_ELEMENT(m, n), result##m##n);
#define SCATTER_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
result##m##n = svmla_m(pg, \
result##m##n, \
svld1_gather_index(pg, &C_ELEMENT(m, n), ldc_vec), \
beta_vec); \
svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
#endif
#ifndef LIKELY
#ifdef __GNUC__
#define LIKELY(x) __builtin_expect(!!(x), 1)
#else
#define LIKELY(x) (x)
#endif
#endif
#ifdef B0
int
CNAME(BLASLONG M,
BLASLONG N,
BLASLONG K,
IFLOAT* A,
BLASLONG lda,
FLOAT alpha,
IFLOAT* B,
BLASLONG ldb,
FLOAT* C,
BLASLONG ldc)
#else
int
CNAME(BLASLONG M,
BLASLONG N,
BLASLONG K,
IFLOAT* A,
BLASLONG lda,
FLOAT alpha,
IFLOAT* B,
BLASLONG ldb,
FLOAT beta,
FLOAT* C,
BLASLONG ldc)
#endif
{
const uint64_t v_size = svcntw();
const uint64_t v_size2 = v_size * 2;
const svbool_t pg_true = svptrue_b32();
const svbool_t pg_quad = svwhilelt_b32(0, 4);
const svbool_t pg_first = svwhilelt_b32(0, 1);
const svfloat32_t alpha_vec = svdup_f32(alpha);
#ifndef B0
const svfloat32_t beta_vec = svdup_f32(beta);
#endif
const svuint32_t lda_vec = svindex_u32(0LL, lda);
const BLASLONG v_m2 = M & -v_size2;
const BLASLONG v_m1 = M & -v_size;
const BLASLONG n8 = N & -8;
const BLASLONG n4 = N & -4;
const int pack_a = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0;
FLOAT* packed_a =
(pack_a) ? packed_a = (FLOAT*)malloc(K * v_size2 * sizeof(FLOAT)) : NULL;
FLOAT* a_offset = A;
FLOAT* b_offset = B;
FLOAT* c_offset = C;
BLASLONG i = 0;
for (; i < v_m2; i += v_size2) {
CREATE_C_POINTER(0, 0);
CREATE_C_POINTER(1, v_size);
CREATE_A_POINTER(0, 0);
CREATE_A_POINTER(1, v_size);
BLASLONG j = 0;
for (; j < n8; j += 8) {
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
CREATE_B_POINTER(2, 2);
CREATE_B_POINTER(3, 3);
CREATE_B_POINTER(4, 4);
CREATE_B_POINTER(5, 5);
CREATE_B_POINTER(6, 6);
CREATE_B_POINTER(7, 7);
UPDATE_B_POINTER(8);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
DECLARE_RESULT_VECTOR(0, 4);
DECLARE_RESULT_VECTOR(0, 5);
DECLARE_RESULT_VECTOR(0, 6);
DECLARE_RESULT_VECTOR(0, 7);
DECLARE_RESULT_VECTOR(1, 0);
DECLARE_RESULT_VECTOR(1, 1);
DECLARE_RESULT_VECTOR(1, 2);
DECLARE_RESULT_VECTOR(1, 3);
DECLARE_RESULT_VECTOR(1, 4);
DECLARE_RESULT_VECTOR(1, 5);
DECLARE_RESULT_VECTOR(1, 6);
DECLARE_RESULT_VECTOR(1, 7);
if (LIKELY(packed_a != NULL)) {
if (j == 0) {
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
VECTOR_PACK_A(0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
QUADWORD_LOAD_B(4, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0);
GATHER_LOAD_A(pg_true, 1, 0);
VECTOR_PACK_A(1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0);
}
} else {
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
UNPACK_VECTOR_A(0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
QUADWORD_LOAD_B(4, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0);
UNPACK_VECTOR_A(1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0);
}
}
} else {
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
QUADWORD_LOAD_B(4, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0);
GATHER_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0);
}
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 0, 2);
VECTOR_STORE(pg_true, 0, 3);
VECTOR_STORE(pg_true, 0, 4);
VECTOR_STORE(pg_true, 0, 5);
VECTOR_STORE(pg_true, 0, 6);
VECTOR_STORE(pg_true, 0, 7);
VECTOR_STORE(pg_true, 1, 0);
VECTOR_STORE(pg_true, 1, 1);
VECTOR_STORE(pg_true, 1, 2);
VECTOR_STORE(pg_true, 1, 3);
VECTOR_STORE(pg_true, 1, 4);
VECTOR_STORE(pg_true, 1, 5);
VECTOR_STORE(pg_true, 1, 6);
VECTOR_STORE(pg_true, 1, 7);
INCR_C_POINTER(0, 8);
INCR_C_POINTER(1, 8);
}
for (; j < n4; j += 4) {
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
CREATE_B_POINTER(2, 2);
CREATE_B_POINTER(3, 3);
UPDATE_B_POINTER(4);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
DECLARE_RESULT_VECTOR(1, 0);
DECLARE_RESULT_VECTOR(1, 1);
DECLARE_RESULT_VECTOR(1, 2);
DECLARE_RESULT_VECTOR(1, 3);
if (LIKELY(packed_a != NULL)) {
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
UNPACK_VECTOR_A(0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
UNPACK_VECTOR_A(1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0);
}
} else {
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
GATHER_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0);
}
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 0, 2);
VECTOR_STORE(pg_true, 0, 3);
VECTOR_STORE(pg_true, 1, 0);
VECTOR_STORE(pg_true, 1, 1);
VECTOR_STORE(pg_true, 1, 2);
VECTOR_STORE(pg_true, 1, 3);
INCR_C_POINTER(0, 4);
INCR_C_POINTER(1, 4);
}
for (; j < N; j++) {
CREATE_B_POINTER(0, 0);
UPDATE_B_POINTER(1);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(1, 0);
if (LIKELY(packed_a != NULL)) {
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
UNPACK_VECTOR_A(0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
UNPACK_VECTOR_A(1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
}
} else {
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
GATHER_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
}
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 1, 0);
INCR_C_POINTER(0, 1);
INCR_C_POINTER(1, 1);
}
UPDATE_A_POINTER(v_size2);
RESET_B_POINTER();
UPDATE_C_POINTER(v_size2);
}
for (; i < v_m1; i += v_size) {
CREATE_C_POINTER(0, 0);
CREATE_A_POINTER(0, 0);
BLASLONG j = 0;
for (; j < n8; j += 8) {
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
CREATE_B_POINTER(2, 2);
CREATE_B_POINTER(3, 3);
CREATE_B_POINTER(4, 4);
CREATE_B_POINTER(5, 5);
CREATE_B_POINTER(6, 6);
CREATE_B_POINTER(7, 7);
UPDATE_B_POINTER(8);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
DECLARE_RESULT_VECTOR(0, 4);
DECLARE_RESULT_VECTOR(0, 5);
DECLARE_RESULT_VECTOR(0, 6);
DECLARE_RESULT_VECTOR(0, 7);
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
QUADWORD_LOAD_B(4, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0);
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 0, 2);
VECTOR_STORE(pg_true, 0, 3);
VECTOR_STORE(pg_true, 0, 4);
VECTOR_STORE(pg_true, 0, 5);
VECTOR_STORE(pg_true, 0, 6);
VECTOR_STORE(pg_true, 0, 7);
INCR_C_POINTER(0, 8);
}
for (; j < n4; j += 4) {
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
CREATE_B_POINTER(2, 2);
CREATE_B_POINTER(3, 3);
UPDATE_B_POINTER(4);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 0, 2);
VECTOR_STORE(pg_true, 0, 3);
INCR_C_POINTER(0, 4);
}
for (; j < N; j++) {
CREATE_B_POINTER(0, 0);
UPDATE_B_POINTER(1);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_true, 0, 0);
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
}
VECTOR_STORE(pg_true, 0, 0);
INCR_C_POINTER(0, 1);
}
UPDATE_A_POINTER(v_size);
RESET_B_POINTER();
UPDATE_C_POINTER(v_size);
}
for (; i < M; i += v_size) {
const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M));
CREATE_C_POINTER(0, 0);
CREATE_A_POINTER(0, 0);
BLASLONG j = 0;
for (; j < n8; j += 8) {
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
CREATE_B_POINTER(2, 2);
CREATE_B_POINTER(3, 3);
CREATE_B_POINTER(4, 4);
CREATE_B_POINTER(5, 5);
CREATE_B_POINTER(6, 6);
CREATE_B_POINTER(7, 7);
UPDATE_B_POINTER(8);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
DECLARE_RESULT_VECTOR(0, 4);
DECLARE_RESULT_VECTOR(0, 5);
DECLARE_RESULT_VECTOR(0, 6);
DECLARE_RESULT_VECTOR(0, 7);
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
GATHER_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
QUADWORD_LOAD_B(4, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0);
}
VECTOR_STORE(pg_tail, 0, 0);
VECTOR_STORE(pg_tail, 0, 1);
VECTOR_STORE(pg_tail, 0, 2);
VECTOR_STORE(pg_tail, 0, 3);
VECTOR_STORE(pg_tail, 0, 4);
VECTOR_STORE(pg_tail, 0, 5);
VECTOR_STORE(pg_tail, 0, 6);
VECTOR_STORE(pg_tail, 0, 7);
INCR_C_POINTER(0, 8);
}
for (; j < n4; j += 4) {
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
CREATE_B_POINTER(2, 2);
CREATE_B_POINTER(3, 3);
UPDATE_B_POINTER(4);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(0, 2);
DECLARE_RESULT_VECTOR(0, 3);
for (; k < K; k++) {
QUADWORD_LOAD_B(0, 0);
GATHER_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
}
VECTOR_STORE(pg_tail, 0, 0);
VECTOR_STORE(pg_tail, 0, 1);
VECTOR_STORE(pg_tail, 0, 2);
VECTOR_STORE(pg_tail, 0, 3);
INCR_C_POINTER(0, 4);
}
for (; j < N; j++) {
CREATE_B_POINTER(0, 0);
UPDATE_B_POINTER(1);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
for (; k < K; k++) {
BROADCAST_LOAD_B(0, 0);
GATHER_LOAD_A(pg_tail, 0, 0);
UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0);
}
VECTOR_STORE(pg_tail, 0, 0);
INCR_C_POINTER(0, 1);
}
UPDATE_A_POINTER(0);
RESET_B_POINTER();
UPDATE_C_POINTER(0);
}
if (pack_a)
free(packed_a);
return 0;
}

77
kernel/loongarch64/scal.S
View File

@@ -53,9 +53,86 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
PROLOGUE
li.d TEMP, SIZE
ld.d XX, $sp, 0 // Load dummy2
slli.d XX, XX, BASE_SHIFT
MTC a1, $r0
slli.d INCX, INCX, BASE_SHIFT
bge $r0, N, .L999
CMPEQ $fcc0, ALPHA, a1
bceqz $fcc0, .L50
beq XX, TEMP, .L50 // if dummp2 == 1, do not directly copy 0
srai.d I, N, 3
bne INCX, TEMP, .L20
bge $r0, I, .L15
.align 3
.L12:
ST a1, X, 0 * SIZE
ST a1, X, 1 * SIZE
ST a1, X, 2 * SIZE
ST a1, X, 3 * SIZE
ST a1, X, 4 * SIZE
ST a1, X, 5 * SIZE
ST a1, X, 6 * SIZE
ST a1, X, 7 * SIZE
addi.w I, I, -1
addi.d X, X, 8 * SIZE
blt $r0, I, .L12
.align 3
.L15:
andi I, N, 7
bge $r0, I, .L999
.align 3
.L16:
ST a1, X, 0 * SIZE
addi.d I, I, -1
addi.d X, X, SIZE
blt $r0, I, .L16
move $r4, $r17
fmov.d $f0, $f22
jirl $r0, $r1, 0x0
.align 3
.L20:
srai.d I, N, 3
bge $r0, I, .L25
.align 3
.L22:
ST a1, X, 0 * SIZE
add.d X, X, INCX
ST a1, X, 0 * SIZE
add.d X, X, INCX
ST a1, X, 0 * SIZE
add.d X, X, INCX
ST a1, X, 0 * SIZE
add.d X, X, INCX
ST a1, X, 0 * SIZE
add.d X, X, INCX
ST a1, X, 0 * SIZE
add.d X, X, INCX
ST a1, X, 0 * SIZE
add.d X, X, INCX
ST a1, X, 0 * SIZE
addi.d I, I, -1
add.d X, X, INCX
blt $r0, I, .L22
.align 3
.L25:
andi I, N, 7
bge $r0, I, .L999
.align 3
.L26:
addi.d I, I, -1
ST a1, X, 0 * SIZE
add.d X, X, INCX
blt $r0, I, .L26
move $r4, $r17
fmov.d $f0, $f22
jirl $r0, $r1, 0x0
.align 3
.L50:
srai.d I, N, 3

79
kernel/loongarch64/scal_lasx.S
View File

@@ -52,17 +52,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
bge $r0, N, .L999
bge $r0, INCX, .L999
li.d TEMP, 1
ld.d t1, $sp, 0 // Load dummp2
movgr2fr.d a1, $r0
FFINT a1, a1
movgr2fr.d a2, TEMP
FFINT a2, a2
slli.d TEMP, TEMP, BASE_SHIFT
slli.d INCX, INCX, BASE_SHIFT
slli.d t1, t1, BASE_SHIFT
CMPEQ $fcc0, ALPHA, a1
bcnez $fcc0, .L20 //ALPHA==0
CMPEQ $fcc0, ALPHA, a2
bcnez $fcc0, .L999 //ALPHA==1 return
.L1:
srai.d I, N, 3
beq INCX, TEMP, .L30 //ALPHA!=1 and INCX==1
beq INCX, TEMP, .L30 //ALPHA !=0|1 and INCX==1
MTG TEMP, ALPHA
#ifdef DOUBLE
xvreplgr2vr.d VALPHA, TEMP
@@ -72,7 +76,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
move XX, X
.align 3
.L10: //ALPHA!=1 and INCX!=1
.L10: //ALPHA !=0|1 and INCX!=1
bge $r0, I, .L32
.align 3
.L11:
@@ -165,6 +169,75 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
blt $r0, I, .L11
b .L32
.align 3
.L20:
beq t1, TEMP, .L1 // if dummp2 == 1, do not directly copy 0
srai.d I, N, 3
beq INCX, TEMP, .L24
bge $r0, I, .L22
.align 3
.L21:
ST a1, X, 0
add.d X, X, INCX
ST a1, X, 0
add.d X, X, INCX
ST a1, X, 0
add.d X, X, INCX
ST a1, X, 0
add.d X, X, INCX
ST a1, X, 0
add.d X, X, INCX
ST a1, X, 0
add.d X, X, INCX
ST a1, X, 0
add.d X, X, INCX
ST a1, X, 0
add.d X, X, INCX
addi.d I, I, -1
blt $r0, I, .L21
.align 3
.L22:
andi I, N, 7
bge $r0, I, .L999
.align 3
.L23:
ST a1, X, 0 * SIZE
addi.d I, I, -1
add.d X, X, INCX
blt $r0, I, .L23
jirl $r0, $r1, 0
.align 3
.L24:
bge $r0, I, .L26 /*N<8 INCX==1*/
.align 3
.L25:
xvxor.v VX0, VX0, VX0
xvst VX0, X, 0 * SIZE
#ifdef DOUBLE
xvst VX0, X, 4 * SIZE
#endif
addi.d I, I, -1
addi.d X, X, 8 * SIZE
blt $r0, I, .L25
.align 3
.L26:
andi I, N, 7
bge $r0, I, .L999
.align 3
.L27:
ST a1, X, 0 * SIZE
addi.d I, I, -1
addi.d X, X, SIZE
blt $r0, I, .L27
jirl $r0, $r1, 0
.align 3
.L30:
bge $r0, I, .L32/*N<8 INCX==1*/
MTG TEMP, ALPHA

82
kernel/loongarch64/scal_lsx.S
View File

@@ -51,6 +51,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
bge $r0, N, .L999
bge $r0, INCX, .L999
ld.d t1, $sp, 0 // Load dummy2
li.d TEMP, 1
movgr2fr.d a1, $r0
FFINT a1, a1
@@ -58,10 +59,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
FFINT a2, a2
slli.d TEMP, TEMP, BASE_SHIFT
slli.d INCX, INCX, BASE_SHIFT
slli.d t1, t1, BASE_SHIFT
CMPEQ $fcc0, ALPHA, a1
bcnez $fcc0, .L20 //ALPHA==0
CMPEQ $fcc0, ALPHA, a2
bcnez $fcc0, .L999 //ALPHA==1 return
.L1:
srai.d I, N, 3
beq INCX, TEMP, .L30 //ALPHA!=1 and INCX==1
beq INCX, TEMP, .L30 //ALPHA !=0|1 and INCX==1
MTG TEMP, ALPHA
#ifdef DOUBLE
vreplgr2vr.d VALPHA, TEMP
@@ -71,7 +76,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
move XX, X
.align 3
.L10: //ALPHA!=1 and INCX!=1
.L10: //ALPHA !=0|1 and INCX!=1
bge $r0, I, .L32
.align 3
@@ -169,6 +174,79 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
b .L32
.align 3
.L20:
beq t1, TEMP, .L1 // if dummp2 == 1, do not directly copy 0
srai.d I, N, 3
beq INCX, TEMP, .L24
bge $r0, I, .L22
.align 3
.L21:
ST a1, X, 0
add.d X, X, INCX
ST a1, X, 0
add.d X, X, INCX
ST a1, X, 0
add.d X, X, INCX
ST a1, X, 0
add.d X, X, INCX
ST a1, X, 0
add.d X, X, INCX
ST a1, X, 0
add.d X, X, INCX
ST a1, X, 0
add.d X, X, INCX
ST a1, X, 0
add.d X, X, INCX
addi.d I, I, -1
blt $r0, I, .L21
.align 3
.L22:
andi I, N, 7
bge $r0, I, .L999
.align 3
.L23:
ST a1, X, 0 * SIZE
addi.d I, I, -1
add.d X, X, INCX
blt $r0, I, .L23
jirl $r0, $r1, 0
.align 3
.L24:
bge $r0, I, .L26 /*N<8 INCX==1*/
.align 3
.L25:
vxor.v VX0, VX0, VX0
vst VX0, X, 0 * SIZE
#ifdef DOUBLE
vst VX0, X, 2 * SIZE
vst VX0, X, 4 * SIZE
vst VX0, X, 6 * SIZE
#else
vst VX0, X, 4 * SIZE
#endif
addi.d I, I, -1
addi.d X, X, 8 * SIZE
blt $r0, I, .L25
.align 3
.L26:
andi I, N, 7
bge $r0, I, .L999
.align 3
.L27:
ST a1, X, 0 * SIZE
addi.d I, I, -1
addi.d X, X, SIZE
blt $r0, I, .L27
jirl $r0, $r1, 0
.align 3
.L30:
bge $r0, I, .L32/*N<8 INCX==1*/
MTG TEMP, ALPHA

12
kernel/mips/dscal_msa.c
View File

@@ -42,7 +42,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x,
if (1 == inc_x)
{
if (0) //if (0.0 == da )
if (0.0 == da && !dummy2)
{
v2f64 zero_v = {0.0, 0.0};
@@ -240,14 +240,12 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x,
}
else
{
if (da == 0.0)
if (da == 0.0 && !dummy2)
{
for (i = n; i--;)
{
if (isfinite(*x))
*x = 0.0;
else
*x = NAN;
{
*x = 0.0;
x += inc_x;
}
}

41
kernel/mips/scal.c
View File

@@ -29,27 +29,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *dummy, BLASLONG dummy2)
{
BLASLONG i=0,j=0;
BLASLONG i = 0, j = 0;
while(j < n)
{
// Resolved issue 4728 when the caller is {s/d}scal
if (da == 0.0 && dummy2 == 1)
{
while(j < n)
{
x[i] = da * x[i] ;
if ( da == 0.0 )
if (isnan(x[i])||isinf(x[i]))
x[i]=NAN;
else
x[i]=0.0;
else if (isnan(da))
x[i]=NAN;
else
x[i] = da * x[i] ;
i += inc_x ;
j++;
}
}
else
{
while(j < n)
{
i += inc_x ;
j++;
}
return 0;
if ( da == 0.0 )
x[i] = 0.0;
else
x[i] = da * x[i] ;
i += inc_x ;
j++;
}
}
return 0;
}

9
kernel/mips/sscal_msa.c
View File

@@ -42,7 +42,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x,
if (1 == inc_x)
{
if (0) // if (0.0 == da)
if (0.0 == da && !dummy2)
{
v4f32 zero_v = {0.0, 0.0, 0.0, 0.0};
@@ -255,14 +255,11 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x,
}
else
{
if (0.0 == da)
if (0.0 == da && !dummy2)
{
for (i = n; i--;)
{
if (isfinite(*x))
*x = 0;
else
*x = NAN;
*x = 0;
x += inc_x;
}
}

9
kernel/mips64/scal.S
View File

@@ -48,6 +48,7 @@
#define TEMP $3
#define XX $5
#define DUMMY2 $6
#define ALPHA $f15
@@ -73,13 +74,13 @@
blez N, .L999
dsll INCX, INCX, BASE_SHIFT
CMPEQ $fcc0, ALPHA, a1
NOP
CMPEQ $fcc0, ALPHA, a1
LDARG DUMMY2, 8($sp)
bc1f $fcc0, .L50
NOP
dsll DUMMY2, DUMMY2, BASE_SHIFT
bc1t $fcc0, .L50
beq DUMMY2, TEMP, .L50 // If dummy2 == 1, do not directly copy 0
NOP
bne INCX, TEMP, .L20

34
kernel/power/dscal.c
View File

@@ -73,6 +73,15 @@ static void dscal_kernel_8_zero (BLASLONG n, FLOAT *x)
for( i=0; i<n; i+=8 )
{
x[0] = alpha;
x[1] = alpha;
x[2] = alpha;
x[3] = alpha;
x[4] = alpha;
x[5] = alpha;
x[6] = alpha;
x[7] = alpha;
#if 0
if(isfinite(x[0]))
x[0] = alpha;
else
@@ -106,7 +115,8 @@ static void dscal_kernel_8_zero (BLASLONG n, FLOAT *x)
else
x[7] = NAN;
x+=8;
}
#endif
}
}
@@ -130,6 +140,11 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS
if ( n >= 16 )
{
BLASLONG align = ((32 - ((uintptr_t)x & (uintptr_t)0x1F)) >> 3) & 0x3;
if (dummy2 == 0)
for (j = 0; j < align; j++) {
x [j] = 0.0;
}
else
for (j = 0; j < align; j++) {
if (isfinite(x[j]))
x[j] = 0.0;
@@ -151,7 +166,13 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS
j=n1;
}
#endif
if (dummy2 == 0)
while(j < n)
{
x[j]=0.0;
j++;
}
else
while(j < n)
{
if (!isfinite(x[j]))
@@ -202,7 +223,14 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS
if ( da == 0.0 )
{
if (dummy2 == 0)
while(j < n)
{
x[i]=0.0;
i += inc_x;
j++;
}
else
while(j < n)
{
if (!isfinite(x[i]))

138
kernel/power/gemm_kernel_power6.S
View File

@@ -864,15 +864,15 @@ LL(22):
LFD f22, 10 * SIZE(BO)
LFD f23, 11 * SIZE(BO)
FMADD f2, f18, f24, f2
FMADD f3, f19, f24, f3
FMADD f6, f18, f25, f6
FMADD f7, f19, f25, f7
FMADD f0, f18, f24, f0
FMADD f1, f19, f24, f1
FMADD f4, f18, f25, f4
FMADD f5, f19, f25, f5
FMADD f10, f18, f26, f10
FMADD f11, f19, f26, f11
FMADD f14, f18, f27, f14
FMADD f15, f19, f27, f15
FMADD f8, f18, f26, f8
FMADD f9, f19, f26, f9
FMADD f12, f18, f27, f12
FMADD f13, f19, f27, f13
LFD f16, 4 * SIZE(AO)
LFD f17, 5 * SIZE(AO)
@@ -899,15 +899,15 @@ LL(22):
LFD f22, 18 * SIZE(BO)
LFD f23, 19 * SIZE(BO)
FMADD f2, f18, f24, f2
FMADD f3, f19, f24, f3
FMADD f6, f18, f25, f6
FMADD f7, f19, f25, f7
FMADD f0, f18, f24, f0
FMADD f1, f19, f24, f1
FMADD f4, f18, f25, f4
FMADD f5, f19, f25, f5
FMADD f10, f18, f26, f10
FMADD f11, f19, f26, f11
FMADD f14, f18, f27, f14
FMADD f15, f19, f27, f15
FMADD f8, f18, f26, f8
FMADD f9, f19, f26, f9
FMADD f12, f18, f27, f12
FMADD f13, f19, f27, f13
LFD f16, 8 * SIZE(AO)
LFD f17, 9 * SIZE(AO)
@@ -923,14 +923,6 @@ LL(22):
addi BO, BO, 16 * SIZE
bdnz LL(22)
fadd f0, f2, f0
fadd f1, f3, f1
fadd f4, f6, f4
fadd f5, f7, f5
fadd f8, f10, f8
fadd f9, f11, f9
fadd f12, f14, f12
fadd f13, f15, f13
.align 4
LL(25):
@@ -1161,10 +1153,10 @@ LL(32):
LFD f22, 10 * SIZE(BO)
LFD f23, 11 * SIZE(BO)
FMADD f1, f17, f24, f1
FMADD f5, f17, f25, f5
FMADD f9, f17, f26, f9
FMADD f13, f17, f27, f13
FMADD f0, f17, f24, f0
FMADD f4, f17, f25, f4
FMADD f8, f17, f26, f8
FMADD f12, f17, f27, f12
LFD f24, 12 * SIZE(BO)
LFD f25, 13 * SIZE(BO)
@@ -1181,10 +1173,10 @@ LL(32):
LFD f22, 18 * SIZE(BO)
LFD f23, 19 * SIZE(BO)
FMADD f1, f19, f24, f1
FMADD f5, f19, f25, f5
FMADD f9, f19, f26, f9
FMADD f13, f19, f27, f13
FMADD f0, f19, f24, f0
FMADD f4, f19, f25, f4
FMADD f8, f19, f26, f8
FMADD f12, f19, f27, f12
LFD f16, 4 * SIZE(AO)
LFD f17, 5 * SIZE(AO)
@@ -1200,10 +1192,6 @@ LL(32):
addi BO, BO, 16 * SIZE
bdnz LL(32)
fadd f0, f1, f0
fadd f4, f5, f4
fadd f8, f9, f8
fadd f12, f13, f12
.align 4
LL(35):
@@ -1691,10 +1679,10 @@ LL(52):
FMADD f2, f16, f21, f2
FMADD f3, f17, f21, f3
FMADD f4, f18, f22, f4
FMADD f5, f19, f22, f5
FMADD f6, f18, f23, f6
FMADD f7, f19, f23, f7
FMADD f0, f18, f22, f0
FMADD f1, f19, f22, f1
FMADD f2, f18, f23, f2
FMADD f3, f19, f23, f3
LFD f16, 4 * SIZE(AO)
LFD f17, 5 * SIZE(AO)
@@ -1711,10 +1699,10 @@ LL(52):
FMADD f2, f16, f25, f2
FMADD f3, f17, f25, f3
FMADD f4, f18, f26, f4
FMADD f5, f19, f26, f5
FMADD f6, f18, f27, f6
FMADD f7, f19, f27, f7
FMADD f0, f18, f26, f0
FMADD f1, f19, f26, f1
FMADD f2, f18, f27, f2
FMADD f3, f19, f27, f3
LFD f16, 8 * SIZE(AO)
LFD f17, 9 * SIZE(AO)
@@ -1775,21 +1763,11 @@ LL(58):
LFD f18, 0 * SIZE(CO2)
LFD f19, 1 * SIZE(CO2)
FADD f0, f4, f0
FADD f1, f5, f1
FADD f2, f6, f2
FADD f3, f7, f3
FMADD f0, f0, f30, f16
FMADD f1, f1, f30, f17
FMADD f2, f2, f30, f18
FMADD f3, f3, f30, f19
#else
FADD f0, f4, f0
FADD f1, f5, f1
FADD f2, f6, f2
FADD f3, f7, f3
FMUL f0, f0, f30
FMUL f1, f1, f30
FMUL f2, f2, f30
@@ -1916,8 +1894,8 @@ LL(60):
LL(62):
FMADD f0, f16, f20, f0
FMADD f1, f16, f21, f1
FMADD f2, f17, f22, f2
FMADD f3, f17, f23, f3
FMADD f0, f17, f22, f0
FMADD f1, f17, f23, f1
LFD f20, 8 * SIZE(BO)
LFD f21, 9 * SIZE(BO)
@@ -1926,8 +1904,8 @@ LL(62):
FMADD f0, f18, f24, f0
FMADD f1, f18, f25, f1
FMADD f2, f19, f26, f2
FMADD f3, f19, f27, f3
FMADD f0, f19, f26, f0
FMADD f1, f19, f27, f1
LFD f16, 4 * SIZE(AO)
LFD f17, 5 * SIZE(AO)
@@ -1986,15 +1964,9 @@ LL(68):
LFD f16, 0 * SIZE(CO1)
LFD f18, 0 * SIZE(CO2)
FADD f0, f2, f0
FADD f1, f3, f1
FMADD f0, f0, f30, f16
FMADD f1, f1, f30, f18
#else
FADD f0, f2, f0
FADD f1, f3, f1
FMUL f0, f0, f30
FMUL f1, f1, f30
#endif
@@ -2007,7 +1979,6 @@ LL(68):
fmr f4, f0
fmr f5, f0
#ifdef TRMMKERNEL
#if ( defined(LEFT) && defined(TRANSA)) || \
(!defined(LEFT) && !defined(TRANSA))
@@ -2332,8 +2303,8 @@ LL(80):
LL(82):
FMADD f0, f16, f20, f0
FMADD f1, f17, f20, f1
FMADD f2, f18, f21, f2
FMADD f3, f19, f21, f3
FMADD f0, f18, f21, f0
FMADD f1, f19, f21, f1
LFD f16, 4 * SIZE(AO)
LFD f17, 5 * SIZE(AO)
@@ -2342,8 +2313,8 @@ LL(82):
FMADD f0, f16, f22, f0
FMADD f1, f17, f22, f1
FMADD f2, f18, f23, f2
FMADD f3, f19, f23, f3
FMADD f0, f18, f23, f0
FMADD f1, f19, f23, f1
LFD f16, 8 * SIZE(AO)
LFD f17, 9 * SIZE(AO)
@@ -2401,15 +2372,9 @@ LL(88):
LFD f16, 0 * SIZE(CO1)
LFD f17, 1 * SIZE(CO1)
FADD f0, f2, f0
FADD f1, f3, f1
FMADD f0, f0, f30, f16
FMADD f1, f1, f30, f17
#else
FADD f0, f2, f0
FADD f1, f3, f1
FMUL f0, f0, f30
FMUL f1, f1, f30
#endif
@@ -2418,9 +2383,6 @@ LL(88):
STFD f1, 1 * SIZE(CO1)
lfs f0, FZERO
fmr f1, f0
fmr f2, f0
fmr f3, f0
addi CO1, CO1, 2 * SIZE
@@ -2512,9 +2474,9 @@ LL(90):
LL(92):
FMADD f0, f16, f20, f0
FMADD f1, f17, f21, f1
FMADD f2, f18, f22, f2
FMADD f3, f19, f23, f3
FMADD f0, f17, f21, f0
FMADD f0, f18, f22, f0
FMADD f0, f19, f23, f0
LFD f16, 4 * SIZE(AO)
LFD f17, 5 * SIZE(AO)
@@ -2527,9 +2489,9 @@ LL(92):
LFD f23, 7 * SIZE(BO)
FMADD f0, f16, f20, f0
FMADD f1, f17, f21, f1
FMADD f2, f18, f22, f2
FMADD f3, f19, f23, f3
FMADD f0, f17, f21, f0
FMADD f0, f18, f22, f0
FMADD f0, f19, f23, f0
LFD f16, 8 * SIZE(AO)
LFD f17, 9 * SIZE(AO)
@@ -2583,16 +2545,8 @@ LL(98):
#ifndef TRMMKERNEL
LFD f16, 0 * SIZE(CO1)
FADD f0, f1, f0
FADD f2, f3, f2
FADD f0, f2, f0
FMADD f0, f0, f30, f16
#else
FADD f0, f1, f0
FADD f2, f3, f2
FADD f0, f2, f0
FMUL f0, f0, f30
#endif

821
kernel/power/gemv_t.S
View File

File diff suppressed because it is too large Load Diff

13
kernel/power/scal.S
View File

@@ -47,9 +47,11 @@
#ifndef __64BIT__
#define X r6
#define INCX r7
#define FLAG r11
#else
#define X r7
#define INCX r8
#define FLAG r12
#endif
#endif
@@ -57,9 +59,11 @@
#if !defined(__64BIT__) && defined(DOUBLE)
#define X r8
#define INCX r9
#define FLAG r13
#else
#define X r7
#define INCX r8
#define FLAG r12
#endif
#endif
@@ -84,9 +88,12 @@
cmpwi cr0, N, 0
blelr- cr0
// fcmpu cr0, FZERO, ALPHA
// bne- cr0, LL(A1I1)
b LL(A1I1)
fcmpu cr0, FZERO, ALPHA
bne- cr0, LL(A1I1)
ld FLAG, 48+64+8(SP)
cmpwi cr0, FLAG, 1
beq- cr0, LL(A1I1)
cmpwi cr0, INCX, SIZE
bne- cr0, LL(A0IN)

44
kernel/power/sscal.c
View File

@@ -74,7 +74,24 @@ static void sscal_kernel_16_zero( BLASLONG n, FLOAT *x )
for( i=0; i<n; i+=8 )
{
if (isfinite(x[0]))
x[0] = alpha;
x[1] = alpha;
x[2] = alpha;
x[3] = alpha;
x[4] = alpha;
x[5] = alpha;
x[6] = alpha;
x[7] = alpha;
x[8] = alpha;
x[9] = alpha;
x[10] = alpha;
x[11] = alpha;
x[12] = alpha;
x[13] = alpha;
x[14] = alpha;
x[15] = alpha;
#if 0
if (isfinite(x[0]))
x[0] = alpha;
else
x[0] = NAN;
@@ -107,7 +124,8 @@ static void sscal_kernel_16_zero( BLASLONG n, FLOAT *x )
else
x[7] = NAN;
x+=8;
}
#endif
}
}
@@ -132,6 +150,11 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS
if ( n >= 32 )
{
BLASLONG align = ((32 - ((uintptr_t)x & (uintptr_t)0x1F)) >> 2) & 0x7;
if (dummy2 == 0)
for (j = 0; j < align; j++){
x[j] = 0.0;
}
else
for (j = 0; j < align; j++) {
if (isfinite(x[j]))
x[j] = 0.0;
@@ -153,9 +176,15 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS
j=n1;
}
#endif
if (dummy2 == 0)
while(j < n)
{
x[j] = 0.0;
j++;
}
else
while(j < n)
{
if (isfinite(x[j]))
x[j]=0.0;
else
@@ -204,7 +233,14 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS
if ( da == 0.0 )
{
if (dummy2 == 0)
while(j < n)
{
x[i]=0.0;
i += inc_x;
j++;
}
else
while(j < n)
{
if (isfinite(x[i]))

551
kernel/power/zgemm_kernel_power6.S
View File

@@ -1159,9 +1159,9 @@ LL(20):
LL(22):
FMA1 f0, f16, f20, f0
FMA4 f3, f17, f20, f3
FMA2 f1, f16, f21, f1
FMA3 f2, f17, f21, f2
FMA4 f1, f17, f20, f1
FMA3 f0, f17, f21, f0
LFD f28, 4 * SIZE(AO)
LFD f29, 5 * SIZE(AO)
@@ -1169,9 +1169,9 @@ LL(22):
LFD f31, 7 * SIZE(AO)
FMA1 f4, f16, f22, f4
FMA4 f7, f17, f22, f7
FMA2 f5, f16, f23, f5
FMA3 f6, f17, f23, f6
FMA4 f5, f17, f22, f5
FMA3 f4, f17, f23, f4
LFD f20, 8 * SIZE(BO)
LFD f21, 9 * SIZE(BO)
@@ -1179,14 +1179,14 @@ LL(22):
LFD f23, 11 * SIZE(BO)
FMA1 f8, f16, f24, f8
FMA4 f11, f17, f24, f11
FMA2 f9, f16, f25, f9
FMA3 f10, f17, f25, f10
FMA4 f9, f17, f24, f9
FMA3 f8, f17, f25, f8
FMA1 f12, f16, f26, f12
FMA4 f15, f17, f26, f15
FMA2 f13, f16, f27, f13
FMA3 f14, f17, f27, f14
FMA4 f13, f17, f26, f13
FMA3 f12, f17, f27, f12
LFD f24, 12 * SIZE(BO)
LFD f25, 13 * SIZE(BO)
@@ -1194,14 +1194,14 @@ LL(22):
LFD f27, 15 * SIZE(BO)
FMA1 f0, f18, f20, f0
FMA4 f3, f19, f20, f3
FMA2 f1, f18, f21, f1
FMA3 f2, f19, f21, f2
FMA4 f1, f19, f20, f1
FMA3 f0, f19, f21, f0
FMA1 f4, f18, f22, f4
FMA4 f7, f19, f22, f7
FMA2 f5, f18, f23, f5
FMA3 f6, f19, f23, f6
FMA4 f5, f19, f22, f5
FMA3 f4, f19, f23, f4
LFD f20, 16 * SIZE(BO)
LFD f21, 17 * SIZE(BO)
@@ -1209,14 +1209,14 @@ LL(22):
LFD f23, 19 * SIZE(BO)
FMA1 f8, f18, f24, f8
FMA4 f11, f19, f24, f11
FMA2 f9, f18, f25, f9
FMA3 f10, f19, f25, f10
FMA4 f9, f19, f24, f9
FMA3 f8, f19, f25, f8
FMA1 f12, f18, f26, f12
FMA4 f15, f19, f26, f15
FMA2 f13, f18, f27, f13
FMA3 f14, f19, f27, f14
FMA4 f13, f19, f26, f13
FMA3 f12, f19, f27, f12
LFD f24, 20 * SIZE(BO)
LFD f25, 21 * SIZE(BO)
@@ -1224,9 +1224,9 @@ LL(22):
LFD f27, 23 * SIZE(BO)
FMA1 f0, f28, f20, f0
FMA4 f3, f29, f20, f3
FMA2 f1, f28, f21, f1
FMA3 f2, f29, f21, f2
FMA4 f1, f29, f20, f1
FMA3 f0, f29, f21, f0
LFD f16, 8 * SIZE(AO)
LFD f17, 9 * SIZE(AO)
@@ -1234,9 +1234,9 @@ LL(22):
LFD f19, 11 * SIZE(AO)
FMA1 f4, f28, f22, f4
FMA4 f7, f29, f22, f7
FMA2 f5, f28, f23, f5
FMA3 f6, f29, f23, f6
FMA4 f5, f29, f22, f5
FMA3 f4, f29, f23, f4
LFD f20, 24 * SIZE(BO)
LFD f21, 25 * SIZE(BO)
@@ -1244,14 +1244,14 @@ LL(22):
LFD f23, 27 * SIZE(BO)
FMA1 f8, f28, f24, f8
FMA4 f11, f29, f24, f11
FMA2 f9, f28, f25, f9
FMA3 f10, f29, f25, f10
FMA4 f9, f29, f24, f9
FMA3 f8, f29, f25, f8
FMA1 f12, f28, f26, f12
FMA4 f15, f29, f26, f15
FMA2 f13, f28, f27, f13
FMA3 f14, f29, f27, f14
FMA4 f13, f29, f26, f13
FMA3 f12, f29, f27, f12
LFD f24, 28 * SIZE(BO)
LFD f25, 29 * SIZE(BO)
@@ -1259,14 +1259,14 @@ LL(22):
LFD f27, 31 * SIZE(BO)
FMA1 f0, f30, f20, f0
FMA4 f3, f31, f20, f3
FMA2 f1, f30, f21, f1
FMA3 f2, f31, f21, f2
FMA4 f1, f31, f20, f1
FMA3 f0, f31, f21, f0
FMA1 f4, f30, f22, f4
FMA4 f7, f31, f22, f7
FMA2 f5, f30, f23, f5
FMA3 f6, f31, f23, f6
FMA4 f5, f31, f22, f5
FMA3 f4, f31, f23, f4
LFD f20, 32 * SIZE(BO)
LFD f21, 33 * SIZE(BO)
@@ -1274,14 +1274,14 @@ LL(22):
LFD f23, 35 * SIZE(BO)
FMA1 f8, f30, f24, f8
FMA4 f11, f31, f24, f11
FMA2 f9, f30, f25, f9
FMA3 f10, f31, f25, f10
FMA4 f9, f31, f24, f9
FMA3 f8, f31, f25, f8
FMA1 f12, f30, f26, f12
FMA4 f15, f31, f26, f15
FMA2 f13, f30, f27, f13
FMA3 f14, f31, f27, f14
FMA4 f13, f31, f26, f13
FMA3 f12, f31, f27, f12
LFD f24, 36 * SIZE(BO)
LFD f25, 37 * SIZE(BO)
@@ -1318,14 +1318,14 @@ LL(25):
LL(26):
FMA1 f0, f16, f20, f0
FMA4 f3, f17, f20, f3
FMA2 f1, f16, f21, f1
FMA3 f2, f17, f21, f2
FMA4 f1, f17, f20, f1
FMA3 f0, f17, f21, f0
FMA1 f4, f16, f22, f4
FMA4 f7, f17, f22, f7
FMA2 f5, f16, f23, f5
FMA3 f6, f17, f23, f6
FMA4 f5, f17, f22, f5
FMA3 f4, f17, f23, f4
LFD f20, 8 * SIZE(BO)
LFD f21, 9 * SIZE(BO)
@@ -1333,14 +1333,14 @@ LL(26):
LFD f23, 11 * SIZE(BO)
FMA1 f8, f16, f24, f8
FMA4 f11, f17, f24, f11
FMA2 f9, f16, f25, f9
FMA3 f10, f17, f25, f10
FMA4 f9, f17, f24, f9
FMA3 f8, f17, f25, f8
FMA1 f12, f16, f26, f12
FMA4 f15, f17, f26, f15
FMA2 f13, f16, f27, f13
FMA3 f14, f17, f27, f14
FMA4 f13, f17, f26, f13
FMA3 f12, f17, f27, f12
LFD f16, 2 * SIZE(AO)
LFD f17, 3 * SIZE(AO)
@@ -1363,47 +1363,42 @@ LL(28):
LFD f18, 0 * SIZE(CO2)
LFD f19, 1 * SIZE(CO2)
FADD f0, f0, f2
FADD f1, f1, f3
FADD f4, f4, f6
FADD f5, f5, f7
LFD f20, 0 * SIZE(CO3)
LFD f21, 1 * SIZE(CO3)
LFD f22, 0 * SIZE(CO4)
LFD f23, 1 * SIZE(CO4)
FADD f8, f8, f10
FADD f9, f9, f11
FADD f12, f12, f14
FADD f13, f13, f15
fmr f2, f0
fmr f3, f1
fmr f6, f4
fmr f7, f5
FNMSUB f24, f31, f1, f16
FMADD f25, f31, f0, f17
FNMSUB f26, f31, f5, f18
FMADD f27, f31, f4, f19
FMADD f24, f30, f0, f16
FMADD f25, f30, f1, f17
FMADD f26, f30, f4, f18
FMADD f27, f30, f5, f19
FMADD f0, f30, f0, f24
FMADD f1, f30, f1, f25
FMADD f4, f30, f4, f26
FMADD f5, f30, f5, f27
FNMSUB f0, f31, f3, f24
FMADD f1, f31, f2, f25
FNMSUB f4, f31, f7, f26
FMADD f5, f31, f6, f27
FNMSUB f24, f31, f9, f20
FMADD f25, f31, f8, f21
FNMSUB f26, f31, f13, f22
FMADD f27, f31, f12, f23
fmr f10, f8
fmr f11, f9
fmr f14, f12
fmr f15, f13
FMADD f8, f30, f8, f24
FMADD f9, f30, f9, f25
FMADD f12, f30, f12, f26
FMADD f13, f30, f13, f27
FMADD f24, f30, f8, f20
FMADD f25, f30, f9, f21
FMADD f26, f30, f12, f22
FMADD f27, f30, f13, f23
FNMSUB f8, f31, f11, f24
FMADD f9, f31, f10, f25
FNMSUB f12, f31, f15, f26
FMADD f13, f31, f14, f27
#else
FADD f0, f0, f2
FADD f1, f1, f3
FADD f4, f4, f6
FADD f5, f5, f7
FMUL f16, f31, f1
FMUL f17, f31, f0
FMUL f18, f31, f5
@@ -1414,11 +1409,6 @@ LL(28):
FMSUB f4, f30, f4, f18
FMADD f5, f30, f5, f19
FADD f8, f8, f10
FADD f9, f9, f11
FADD f12, f12, f14
FADD f13, f13, f15
FMUL f20, f31, f9
FMUL f21, f31, f8
FMUL f22, f31, f13
@@ -1616,15 +1606,15 @@ LL(32):
FMA2 f5, f16, f23, f5
FMA2 f7, f18, f23, f7
FMA4 f9, f17, f20, f9
FMA4 f11, f19, f20, f11
FMA3 f8, f17, f21, f8
FMA3 f10, f19, f21, f10
FMA4 f1, f17, f20, f1
FMA4 f3, f19, f20, f3
FMA3 f0, f17, f21, f0
FMA3 f2, f19, f21, f2
FMA4 f13, f17, f22, f13
FMA4 f15, f19, f22, f15
FMA3 f12, f17, f23, f12
FMA3 f14, f19, f23, f14
FMA4 f5, f17, f22, f5
FMA4 f7, f19, f22, f7
FMA3 f4, f17, f23, f4
FMA3 f6, f19, f23, f6
LFD f20, 8 * SIZE(BO)
LFD f21, 9 * SIZE(BO)
@@ -1646,15 +1636,15 @@ LL(32):
FMA2 f5, f28, f27, f5
FMA2 f7, f30, f27, f7
FMA4 f9, f29, f24, f9
FMA4 f11, f31, f24, f11
FMA3 f8, f29, f25, f8
FMA3 f10, f31, f25, f10
FMA4 f1, f29, f24, f1
FMA4 f3, f31, f24, f3
FMA3 f0, f29, f25, f0
FMA3 f2, f31, f25, f2
FMA4 f13, f29, f26, f13
FMA4 f15, f31, f26, f15
FMA3 f12, f29, f27, f12
FMA3 f14, f31, f27, f14
FMA4 f5, f29, f26, f5
FMA4 f7, f31, f26, f7
FMA3 f4, f29, f27, f4
FMA3 f6, f31, f27, f6
LFD f24, 12 * SIZE(BO)
LFD f25, 13 * SIZE(BO)
@@ -1676,15 +1666,15 @@ LL(32):
FMA2 f5, f16, f23, f5
FMA2 f7, f18, f23, f7
FMA4 f9, f17, f20, f9
FMA4 f11, f19, f20, f11
FMA3 f8, f17, f21, f8
FMA3 f10, f19, f21, f10
FMA4 f1, f17, f20, f1
FMA4 f3, f19, f20, f3
FMA3 f0, f17, f21, f0
FMA3 f2, f19, f21, f2
FMA4 f13, f17, f22, f13
FMA4 f15, f19, f22, f15
FMA3 f12, f17, f23, f12
FMA3 f14, f19, f23, f14
FMA4 f5, f17, f22, f5
FMA4 f7, f19, f22, f7
FMA3 f4, f17, f23, f4
FMA3 f6, f19, f23, f6
LFD f20, 16 * SIZE(BO)
LFD f21, 17 * SIZE(BO)
@@ -1706,15 +1696,15 @@ LL(32):
FMA2 f5, f28, f27, f5
FMA2 f7, f30, f27, f7
FMA4 f9, f29, f24, f9
FMA4 f11, f31, f24, f11
FMA3 f8, f29, f25, f8
FMA3 f10, f31, f25, f10
FMA4 f1, f29, f24, f1
FMA4 f3, f31, f24, f3
FMA3 f0, f29, f25, f0
FMA3 f2, f31, f25, f2
FMA4 f13, f29, f26, f13
FMA4 f15, f31, f26, f15
FMA3 f12, f29, f27, f12
FMA3 f14, f31, f27, f14
FMA4 f5, f29, f26, f5
FMA4 f7, f31, f26, f7
FMA3 f4, f29, f27, f4
FMA3 f6, f31, f27, f6
LFD f24, 20 * SIZE(BO)
LFD f25, 21 * SIZE(BO)
@@ -1736,15 +1726,15 @@ LL(32):
FMA2 f5, f16, f23, f5
FMA2 f7, f18, f23, f7
FMA4 f9, f17, f20, f9
FMA4 f11, f19, f20, f11
FMA3 f8, f17, f21, f8
FMA3 f10, f19, f21, f10
FMA4 f1, f17, f20, f1
FMA4 f3, f19, f20, f3
FMA3 f0, f17, f21, f0
FMA3 f2, f19, f21, f2
FMA4 f13, f17, f22, f13
FMA4 f15, f19, f22, f15
FMA3 f12, f17, f23, f12
FMA3 f14, f19, f23, f14
FMA4 f5, f17, f22, f5
FMA4 f7, f19, f22, f7
FMA3 f4, f17, f23, f4
FMA3 f6, f19, f23, f6
LFD f20, 24 * SIZE(BO)
LFD f21, 25 * SIZE(BO)
@@ -1766,15 +1756,15 @@ LL(32):
FMA2 f5, f28, f27, f5
FMA2 f7, f30, f27, f7
FMA4 f9, f29, f24, f9
FMA4 f11, f31, f24, f11
FMA3 f8, f29, f25, f8
FMA3 f10, f31, f25, f10
FMA4 f1, f29, f24, f1
FMA4 f3, f31, f24, f3
FMA3 f0, f29, f25, f0
FMA3 f2, f31, f25, f2
FMA4 f13, f29, f26, f13
FMA4 f15, f31, f26, f15
FMA3 f12, f29, f27, f12
FMA3 f14, f31, f27, f14
FMA4 f5, f29, f26, f5
FMA4 f7, f31, f26, f7
FMA3 f4, f29, f27, f4
FMA3 f6, f31, f27, f6
LFD f24, 28 * SIZE(BO)
LFD f25, 29 * SIZE(BO)
@@ -1796,15 +1786,15 @@ LL(32):
FMA2 f5, f16, f23, f5
FMA2 f7, f18, f23, f7
FMA4 f9, f17, f20, f9
FMA4 f11, f19, f20, f11
FMA3 f8, f17, f21, f8
FMA3 f10, f19, f21, f10
FMA4 f1, f17, f20, f1
FMA4 f3, f19, f20, f3
FMA3 f0, f17, f21, f0
FMA3 f2, f19, f21, f2
FMA4 f13, f17, f22, f13
FMA4 f15, f19, f22, f15
FMA3 f12, f17, f23, f12
FMA3 f14, f19, f23, f14
FMA4 f5, f17, f22, f5
FMA4 f7, f19, f22, f7
FMA3 f4, f17, f23, f4
FMA3 f6, f19, f23, f6
LFD f20, 32 * SIZE(BO)
LFD f21, 33 * SIZE(BO)
@@ -1826,15 +1816,15 @@ LL(32):
FMA2 f5, f28, f27, f5
FMA2 f7, f30, f27, f7
FMA4 f9, f29, f24, f9
FMA4 f11, f31, f24, f11
FMA3 f8, f29, f25, f8
FMA3 f10, f31, f25, f10
FMA4 f1, f29, f24, f1
FMA4 f3, f31, f24, f3
FMA3 f0, f29, f25, f0
FMA3 f2, f31, f25, f2
FMA4 f13, f29, f26, f13
FMA4 f15, f31, f26, f15
FMA3 f12, f29, f27, f12
FMA3 f14, f31, f27, f14
FMA4 f5, f29, f26, f5
FMA4 f7, f31, f26, f7
FMA3 f4, f29, f27, f4
FMA3 f6, f31, f27, f6
LFD f24, 36 * SIZE(BO)
LFD f25, 37 * SIZE(BO)
@@ -1883,20 +1873,20 @@ LL(36):
FMA2 f5, f16, f23, f5
FMA2 f7, f18, f23, f7
FMA4 f9, f17, f20, f9
FMA4 f11, f19, f20, f11
FMA3 f8, f17, f21, f8
FMA3 f10, f19, f21, f10
FMA4 f1, f17, f20, f1
FMA4 f3, f19, f20, f3
FMA3 f0, f17, f21, f0
FMA3 f2, f19, f21, f2
LFD f16, 4 * SIZE(AO)
LFD f18, 6 * SIZE(AO)
LFD f20, 4 * SIZE(BO)
LFD f21, 5 * SIZE(BO)
FMA4 f13, f17, f22, f13
FMA4 f15, f19, f22, f15
FMA3 f12, f17, f23, f12
FMA3 f14, f19, f23, f14
FMA4 f5, f17, f22, f5
FMA4 f7, f19, f22, f7
FMA3 f4, f17, f23, f4
FMA3 f6, f19, f23, f6
LFD f17, 5 * SIZE(AO)
LFD f19, 7 * SIZE(AO)
@@ -1916,52 +1906,42 @@ LL(38):
LFD f18, 2 * SIZE(CO1)
LFD f19, 3 * SIZE(CO1)
FADD f0, f0, f8
FADD f1, f1, f9
FADD f2, f2, f10
FADD f3, f3, f11
LFD f20, 0 * SIZE(CO2)
LFD f21, 1 * SIZE(CO2)
LFD f22, 2 * SIZE(CO2)
LFD f23, 3 * SIZE(CO2)
FADD f4, f4, f12
FADD f5, f5, f13
FADD f6, f6, f14
FADD f7, f7, f15
fmr f8, f0
fmr f9, f1
fmr f10, f2
fmr f11, f3
FNMSUB f24, f31, f1, f16
FMADD f25, f31, f0, f17
FNMSUB f26, f31, f3, f18
FMADD f27, f31, f2, f19
FMADD f24, f30, f0, f16
FMADD f25, f30, f1, f17
FMADD f26, f30, f2, f18
FMADD f27, f30, f3, f19
FMADD f0, f30, f0, f24
FMADD f1, f30, f1, f25
FMADD f2, f30, f2, f26
FMADD f3, f30, f3, f27
FNMSUB f0, f31, f9, f24
FMADD f1, f31, f8, f25
FNMSUB f2, f31, f11, f26
FMADD f3, f31, f10, f27
FNMSUB f24, f31, f5, f20
FMADD f25, f31, f4, f21
FNMSUB f26, f31, f7, f22
FMADD f27, f31, f6, f23
fmr f12, f4
fmr f13, f5
fmr f14, f6
fmr f15, f7
FMADD f4, f30, f4, f24
FMADD f5, f30, f5, f25
FMADD f6, f30, f6, f26
FMADD f7, f30, f7, f27
FMADD f24, f30, f4, f20
FMADD f25, f30, f5, f21
FMADD f26, f30, f6, f22
FMADD f27, f30, f7, f23
FNMSUB f4, f31, f13, f24
FMADD f5, f31, f12, f25
FNMSUB f6, f31, f15, f26
FMADD f7, f31, f14, f27
#else
FADD f0, f0, f8
FADD f1, f1, f9
FADD f2, f2, f10
FADD f3, f3, f11
FADD f4, f4, f12
FADD f5, f5, f13
FADD f6, f6, f14
FADD f7, f7, f15
FMUL f16, f31, f1
FMUL f17, f31, f0
FMUL f18, f31, f3
@@ -2101,14 +2081,14 @@ LL(40):
LL(42):
FMA1 f0, f16, f20, f0
FMA4 f3, f17, f20, f3
FMA2 f1, f16, f21, f1
FMA3 f2, f17, f21, f2
FMA4 f1, f17, f20, f1
FMA3 f0, f17, f21, f0
FMA1 f4, f16, f22, f4
FMA4 f7, f17, f22, f7
FMA2 f5, f16, f23, f5
FMA3 f6, f17, f23, f6
FMA4 f5, f17, f22, f5
FMA3 f4, f17, f23, f4
LFD f16, 2 * SIZE(AO)
LFD f17, 3 * SIZE(AO)
@@ -2119,14 +2099,14 @@ LL(42):
LFD f23, 7 * SIZE(BO)
FMA1 f0, f16, f20, f0
FMA4 f3, f17, f20, f3
FMA2 f1, f16, f21, f1
FMA3 f2, f17, f21, f2
FMA4 f1, f17, f20, f1
FMA3 f0, f17, f21, f0
FMA1 f4, f16, f22, f4
FMA4 f7, f17, f22, f7
FMA2 f5, f16, f23, f5
FMA3 f6, f17, f23, f6
FMA4 f5, f17, f22, f5
FMA3 f4, f17, f23, f4
LFD f16, 4 * SIZE(AO)
LFD f17, 5 * SIZE(AO)
@@ -2137,14 +2117,14 @@ LL(42):
LFD f23, 11 * SIZE(BO)
FMA1 f0, f16, f20, f0
FMA4 f3, f17, f20, f3
FMA2 f1, f16, f21, f1
FMA3 f2, f17, f21, f2
FMA4 f1, f17, f20, f1
FMA3 f0, f17, f21, f0
FMA1 f4, f16, f22, f4
FMA4 f7, f17, f22, f7
FMA2 f5, f16, f23, f5
FMA3 f6, f17, f23, f6
FMA4 f5, f17, f22, f5
FMA3 f4, f17, f23, f4
LFD f16, 6 * SIZE(AO)
LFD f17, 7 * SIZE(AO)
@@ -2155,14 +2135,14 @@ LL(42):
LFD f23, 15 * SIZE(BO)
FMA1 f0, f16, f20, f0
FMA4 f3, f17, f20, f3
FMA2 f1, f16, f21, f1
FMA3 f2, f17, f21, f2
FMA4 f1, f17, f20, f1
FMA3 f0, f17, f21, f0
FMA1 f4, f16, f22, f4
FMA4 f7, f17, f22, f7
FMA2 f5, f16, f23, f5
FMA3 f6, f17, f23, f6
FMA4 f5, f17, f22, f5
FMA3 f4, f17, f23, f4
LFD f16, 8 * SIZE(AO)
LFD f17, 9 * SIZE(AO)
@@ -2202,14 +2182,14 @@ LL(45):
LL(46):
FMA1 f0, f16, f20, f0
FMA4 f3, f17, f20, f3
FMA2 f1, f16, f21, f1
FMA3 f2, f17, f21, f2
FMA4 f1, f17, f20, f1
FMA3 f0, f17, f21, f0
FMA1 f4, f16, f22, f4
FMA4 f7, f17, f22, f7
FMA2 f5, f16, f23, f5
FMA3 f6, f17, f23, f6
FMA4 f5, f17, f22, f5
FMA3 f4, f17, f23, f4
LFD f16, 2 * SIZE(AO)
LFD f17, 3 * SIZE(AO)
@@ -2231,27 +2211,22 @@ LL(48):
LFD f20, 0 * SIZE(CO2)
LFD f21, 1 * SIZE(CO2)
FADD f0, f0, f2
FADD f1, f1, f3
FADD f4, f4, f6
FADD f5, f5, f7
fmr f2, f0
fmr f3, f1
fmr f6, f4
fmr f7, f5
FNMSUB f24, f31, f1, f16
FMADD f25, f31, f0, f17
FNMSUB f26, f31, f5, f20
FMADD f27, f31, f4, f21
FMADD f24, f30, f0, f16
FMADD f25, f30, f1, f17
FMADD f26, f30, f4, f20
FMADD f27, f30, f5, f21
FMADD f0, f30, f0, f24
FMADD f1, f30, f1, f25
FMADD f4, f30, f4, f26
FMADD f5, f30, f5, f27
FNMSUB f0, f31, f3, f24
FMADD f1, f31, f2, f25
FNMSUB f4, f31, f7, f26
FMADD f5, f31, f6, f27
#else
FADD f0, f0, f2
FADD f1, f1, f3
FADD f4, f4, f6
FADD f5, f5, f7
FMUL f16, f31, f1
FMUL f17, f31, f0
FMUL f18, f31, f5
@@ -2401,10 +2376,10 @@ LL(52):
FMA2 f1, f16, f21, f1
FMA2 f3, f18, f21, f3
FMA4 f9, f17, f20, f9
FMA4 f11, f19, f20, f11
FMA3 f8, f17, f21, f8
FMA3 f10, f19, f21, f10
FMA4 f1, f17, f20, f1
FMA4 f3, f19, f20, f3
FMA3 f0, f17, f21, f0
FMA3 f2, f19, f21, f2
LFD f16, 4 * SIZE(AO)
LFD f17, 5 * SIZE(AO)
@@ -2416,10 +2391,10 @@ LL(52):
FMA2 f1, f16, f23, f1
FMA2 f3, f18, f23, f3
FMA4 f9, f17, f22, f9
FMA4 f11, f19, f22, f11
FMA3 f8, f17, f23, f8
FMA3 f10, f19, f23, f10
FMA4 f1, f17, f22, f1
FMA4 f3, f19, f22, f3
FMA3 f0, f17, f23, f0
FMA3 f2, f19, f23, f2
LFD f16, 8 * SIZE(AO)
LFD f17, 9 * SIZE(AO)
@@ -2436,10 +2411,10 @@ LL(52):
FMA2 f1, f16, f21, f1
FMA2 f3, f18, f21, f3
FMA4 f9, f17, f20, f9
FMA4 f11, f19, f20, f11
FMA3 f8, f17, f21, f8
FMA3 f10, f19, f21, f10
FMA4 f1, f17, f20, f1
FMA4 f3, f19, f20, f3
FMA3 f0, f17, f21, f0
FMA3 f2, f19, f21, f2
LFD f16, 12 * SIZE(AO)
LFD f17, 13 * SIZE(AO)
@@ -2451,10 +2426,10 @@ LL(52):
FMA2 f1, f16, f23, f1
FMA2 f3, f18, f23, f3
FMA4 f9, f17, f22, f9
FMA4 f11, f19, f22, f11
FMA3 f8, f17, f23, f8
FMA3 f10, f19, f23, f10
FMA4 f1, f17, f22, f1
FMA4 f3, f19, f22, f3
FMA3 f0, f17, f23, f0
FMA3 f2, f19, f23, f2
LFD f16, 16 * SIZE(AO)
LFD f17, 17 * SIZE(AO)
@@ -2471,10 +2446,10 @@ LL(52):
FMA2 f1, f16, f21, f1
FMA2 f3, f18, f21, f3
FMA4 f9, f17, f20, f9
FMA4 f11, f19, f20, f11
FMA3 f8, f17, f21, f8
FMA3 f10, f19, f21, f10
FMA4 f1, f17, f20, f1
FMA4 f3, f19, f20, f3
FMA3 f0, f17, f21, f0
FMA3 f2, f19, f21, f2
LFD f16, 20 * SIZE(AO)
LFD f17, 21 * SIZE(AO)
@@ -2486,10 +2461,10 @@ LL(52):
FMA2 f1, f16, f23, f1
FMA2 f3, f18, f23, f3
FMA4 f9, f17, f22, f9
FMA4 f11, f19, f22, f11
FMA3 f8, f17, f23, f8
FMA3 f10, f19, f23, f10
FMA4 f1, f17, f22, f1
FMA4 f3, f19, f22, f3
FMA3 f0, f17, f23, f0
FMA3 f2, f19, f23, f2
LFD f16, 24 * SIZE(AO)
LFD f17, 25 * SIZE(AO)
@@ -2506,10 +2481,10 @@ LL(52):
FMA2 f1, f16, f21, f1
FMA2 f3, f18, f21, f3
FMA4 f9, f17, f20, f9
FMA4 f11, f19, f20, f11
FMA3 f8, f17, f21, f8
FMA3 f10, f19, f21, f10
FMA4 f1, f17, f20, f1
FMA4 f3, f19, f20, f3
FMA3 f0, f17, f21, f0
FMA3 f2, f19, f21, f2
LFD f16, 28 * SIZE(AO)
LFD f17, 29 * SIZE(AO)
@@ -2521,10 +2496,10 @@ LL(52):
FMA2 f1, f16, f23, f1
FMA2 f3, f18, f23, f3
FMA4 f9, f17, f22, f9
FMA4 f11, f19, f22, f11
FMA3 f8, f17, f23, f8
FMA3 f10, f19, f23, f10
FMA4 f1, f17, f22, f1
FMA4 f3, f19, f22, f3
FMA3 f0, f17, f23, f0
FMA3 f2, f19, f23, f2
LFD f16, 32 * SIZE(AO)
LFD f17, 33 * SIZE(AO)
@@ -2573,10 +2548,10 @@ LL(56):
LFD f16, 4 * SIZE(AO)
LFD f18, 6 * SIZE(AO)
FMA4 f9, f17, f20, f9
FMA4 f11, f19, f20, f11
FMA3 f8, f17, f21, f8
FMA3 f10, f19, f21, f10
FMA4 f1, f17, f20, f1
FMA4 f3, f19, f20, f3
FMA3 f0, f17, f21, f0
FMA3 f2, f19, f21, f2
LFD f17, 5 * SIZE(AO)
LFD f19, 7 * SIZE(AO)
@@ -2595,27 +2570,22 @@ LL(58):
LFD f18, 2 * SIZE(CO1)
LFD f19, 3 * SIZE(CO1)
FADD f0, f0, f8
FADD f1, f1, f9
FADD f2, f2, f10
FADD f3, f3, f11
fmr f8, f0
fmr f9, f1
fmr f10, f2
fmr f11, f3
FNMSUB f24, f31, f1, f16
FMADD f25, f31, f0, f17
FNMSUB f26, f31, f3, f18
FMADD f27, f31, f2, f19
FMADD f24, f30, f0, f16
FMADD f25, f30, f1, f17
FMADD f26, f30, f2, f18
FMADD f27, f30, f3, f19
FMADD f0, f30, f0, f24
FMADD f1, f30, f1, f25
FMADD f2, f30, f2, f26
FMADD f3, f30, f3, f27
FNMSUB f0, f31, f9, f24
FMADD f1, f31, f8, f25
FNMSUB f2, f31, f11, f26
FMADD f3, f31, f10, f27
#else
FADD f0, f0, f8
FADD f1, f1, f9
FADD f2, f2, f10
FADD f3, f3, f11
FMUL f16, f31, f1
FMUL f17, f31, f0
FMUL f18, f31, f3
@@ -2735,9 +2705,9 @@ LL(60):
LL(62):
FMA1 f0, f16, f20, f0
FMA4 f3, f17, f20, f3
FMA2 f1, f16, f21, f1
FMA3 f2, f17, f21, f2
FMA4 f1, f17, f20, f1
FMA3 f0, f17, f21, f0
LFD f16, 4 * SIZE(AO)
LFD f17, 5 * SIZE(AO)
@@ -2745,9 +2715,9 @@ LL(62):
LFD f21, 5 * SIZE(BO)
FMA1 f0, f18, f22, f0
FMA4 f3, f19, f22, f3
FMA2 f1, f18, f23, f1
FMA3 f2, f19, f23, f2
FMA4 f1, f19, f22, f1
FMA3 f0, f19, f23, f0
LFD f18, 6 * SIZE(AO)
LFD f19, 7 * SIZE(AO)
@@ -2755,9 +2725,9 @@ LL(62):
LFD f23, 7 * SIZE(BO)
FMA1 f0, f16, f20, f0
FMA4 f3, f17, f20, f3
FMA2 f1, f16, f21, f1
FMA3 f2, f17, f21, f2
FMA4 f1, f17, f20, f1
FMA3 f0, f17, f21, f0
LFD f16, 8 * SIZE(AO)
LFD f17, 9 * SIZE(AO)
@@ -2765,9 +2735,9 @@ LL(62):
LFD f21, 9 * SIZE(BO)
FMA1 f0, f18, f22, f0
FMA4 f3, f19, f22, f3
FMA2 f1, f18, f23, f1
FMA3 f2, f19, f23, f2
FMA4 f1, f19, f22, f1
FMA3 f0, f19, f23, f0
LFD f18, 10 * SIZE(AO)
LFD f19, 11 * SIZE(AO)
@@ -2803,11 +2773,11 @@ LL(65):
LL(66):
FMA1 f0, f16, f20, f0
FMA4 f3, f17, f20, f3
LFD f20, 2 * SIZE(BO)
FMA2 f1, f16, f21, f1
LFD f16, 2 * SIZE(AO)
FMA3 f2, f17, f21, f2
FMA4 f1, f17, f20, f1
LFD f20, 2 * SIZE(BO)
FMA3 f0, f17, f21, f0
LFD f17, 3 * SIZE(AO)
LFD f21, 3 * SIZE(BO)
@@ -2821,20 +2791,17 @@ LL(68):
LFD f16, 0 * SIZE(CO1)
LFD f17, 1 * SIZE(CO1)
FADD f0, f0, f2
FADD f1, f1, f3
fmr f2, f0
fmr f3, f1
FNMSUB f24, f31, f1, f16
FMADD f25, f31, f0, f17
FMADD f24, f30, f0, f16
FMADD f25, f30, f1, f17
FMADD f0, f30, f0, f24
FMADD f1, f30, f1, f25
FNMSUB f0, f31, f3, f24
FMADD f1, f31, f2, f25
#else
FADD f0, f0, f2
FADD f1, f1, f3
FMUL f16, f31, f1
FMUL f17, f31, f0

16
kernel/riscv64/KERNEL.RISCV64_GENERIC
View File

@@ -99,26 +99,26 @@ ZTRMMKERNEL = ../generic/ztrmmkernel_2x2.c
SGEMMKERNEL = ../generic/gemmkernel_2x2.c
SGEMMONCOPY = ../generic/gemm_ncopy_2.c
SGEMMOTCOPY = ../generic/gemm_tcopy_2.c
SGEMMONCOPYOBJ = sgemm_oncopy.o
SGEMMOTCOPYOBJ = sgemm_otcopy.o
SGEMMONCOPYOBJ = sgemm_oncopy$(TSUFFIX).$(SUFFIX)
SGEMMOTCOPYOBJ = sgemm_otcopy$(TSUFFIX).$(SUFFIX)
DGEMMKERNEL = ../generic/gemmkernel_2x2.c
DGEMMONCOPY = ../generic/gemm_ncopy_2.c
DGEMMOTCOPY = ../generic/gemm_tcopy_2.c
DGEMMONCOPYOBJ = dgemm_oncopy.o
DGEMMOTCOPYOBJ = dgemm_otcopy.o
DGEMMONCOPYOBJ = dgemm_oncopy$(TSUFFIX).$(SUFFIX)
DGEMMOTCOPYOBJ = dgemm_otcopy$(TSUFFIX).$(SUFFIX)
CGEMMKERNEL = ../generic/zgemmkernel_2x2.c
CGEMMONCOPY = ../generic/zgemm_ncopy_2.c
CGEMMOTCOPY = ../generic/zgemm_tcopy_2.c
CGEMMONCOPYOBJ = cgemm_oncopy.o
CGEMMOTCOPYOBJ = cgemm_otcopy.o
CGEMMONCOPYOBJ = cgemm_oncopy$(TSUFFIX).$(SUFFIX)
CGEMMOTCOPYOBJ = cgemm_otcopy$(TSUFFIX).$(SUFFIX)
ZGEMMKERNEL = ../generic/zgemmkernel_2x2.c
ZGEMMONCOPY = ../generic/zgemm_ncopy_2.c
ZGEMMOTCOPY = ../generic/zgemm_tcopy_2.c
ZGEMMONCOPYOBJ = zgemm_oncopy.o
ZGEMMOTCOPYOBJ = zgemm_otcopy.o
ZGEMMONCOPYOBJ = zgemm_oncopy$(TSUFFIX).$(SUFFIX)
ZGEMMOTCOPYOBJ = zgemm_otcopy$(TSUFFIX).$(SUFFIX)
STRSMKERNEL_LN = ../generic/trsm_kernel_LN.c
STRSMKERNEL_LT = ../generic/trsm_kernel_LT.c

18
kernel/riscv64/scal.c
View File

@@ -43,9 +43,9 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS
if ( (n <= 0) || (inc_x <= 0))
return(0);
while(j < n)
{
if (dummy2 == 1) {
while(j < n)
{
if ( da == 0.0 )
if (isfinite(x[i]))
@@ -57,7 +57,19 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS
i += inc_x ;
j++;
}
} else {
while(j < n)
{
if ( da == 0.0 )
x[i]=0.0;
else
x[i] = da * x[i] ;
i += inc_x ;
j++;
}
}
return 0;

4
kernel/riscv64/scal_rvv.c
View File

@@ -56,7 +56,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS
FLOAT_V_T v0;
if(inc_x == 1) {
if(da == 0.0) {
if(dummy2 == 0 && da == 0.0) {
int gvl = VSETVL_MAX;
v0 = VFMVVF_FLOAT(0.0, gvl);
for (size_t vl; n > 0; n -= vl, x += vl) {
@@ -75,7 +75,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS
} else {
BLASLONG stride_x = inc_x * sizeof(FLOAT);
if(da == 0.0) {
if(dummy2 == 0 && da == 0.0) {
int gvl = VSETVL_MAX;
v0 = VFMVVF_FLOAT(0.0, gvl);
for (size_t vl; n > 0; n -= vl, x += vl*inc_x) {

4
kernel/riscv64/scal_vector.c
View File

@@ -71,7 +71,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS
FLOAT_V_T v0, v1;
unsigned int gvl = 0;
if(inc_x == 1){
if (0){ //if(da == 0.0){
if(dummy2 == 0 && da == 0.0){
memset(&x[0], 0, n * sizeof(FLOAT));
}else{
gvl = VSETVL(n);
@@ -96,7 +96,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS
}
}
}else{
if (0) { //if(da == 0.0){
if(dummy2 == 0 && da == 0.0){
BLASLONG stride_x = inc_x * sizeof(FLOAT);
BLASLONG ix = 0;
gvl = VSETVL(n);

31
kernel/setparam-ref.c
View File

@@ -1244,6 +1244,36 @@ static void init_parameter(void) {
}
#else //ZARCH
#if (ARCH_RISCV64)
static void init_parameter(void) {
#ifdef BUILD_BFLOAT16
TABLE_NAME.sbgemm_p = SBGEMM_DEFAULT_P;
#endif
TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;
TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;
TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P;
TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P;
#ifdef BUILD_BFLOAT16
TABLE_NAME.sbgemm_r = SBGEMM_DEFAULT_R;
#endif
TABLE_NAME.sgemm_r = SGEMM_DEFAULT_R;
TABLE_NAME.dgemm_r = DGEMM_DEFAULT_R;
TABLE_NAME.cgemm_r = CGEMM_DEFAULT_R;
TABLE_NAME.zgemm_r = ZGEMM_DEFAULT_R;
#ifdef BUILD_BFLOAT16
TABLE_NAME.sbgemm_q = SBGEMM_DEFAULT_Q;
#endif
TABLE_NAME.sgemm_q = SGEMM_DEFAULT_Q;
TABLE_NAME.dgemm_q = DGEMM_DEFAULT_Q;
TABLE_NAME.cgemm_q = CGEMM_DEFAULT_Q;
TABLE_NAME.zgemm_q = ZGEMM_DEFAULT_Q;
}
#else //RISCV64
#ifdef ARCH_X86
static int get_l2_size_old(void){
int i, eax, ebx, ecx, edx, cpuid_level;
@@ -2046,6 +2076,7 @@ static void init_parameter(void) {
}
#endif //RISCV64
#endif //POWER
#endif //ZARCH
#endif //(ARCH_LOONGARCH64)

9
kernel/x86/scal.S
View File

@@ -57,19 +57,24 @@
#ifdef XDOUBLE
movl 44(%esp),%edi
movl 48(%esp),%esi
movl 64(%esp),%ecx
#elif defined(DOUBLE)
movl 36(%esp),%edi
movl 40(%esp),%esi
movl 56(%esp),%ecx
#else
movl 32(%esp),%edi
movl 36(%esp),%esi
movl 52(%esp),%ecx
#endif
ftst
fnstsw %ax
andb $68, %ah
// je .L300 # Alpha != ZERO
jmp .L300
je .L300 # Alpha != ZERO
cmpl $1,%ecx # dummy2 flag
je .L300
/* Alpha == ZERO */
cmpl $1,%esi

302
kernel/x86_64/dscal.c
View File

@@ -43,21 +43,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
static void dscal_kernel_8( BLASLONG n, FLOAT *da , FLOAT *x )
{
BLASLONG i;
FLOAT alpha = *da;
BLASLONG i;
FLOAT alpha = *da;
for( i=0; i<n; i+=8 )
{
x[0] *= alpha;
x[1] *= alpha;
x[2] *= alpha;
x[3] *= alpha;
x[4] *= alpha;
x[5] *= alpha;
x[6] *= alpha;
x[7] *= alpha;
x+=8;
}
for( i=0; i<n; i+=8 )
{
x[0] *= alpha;
x[1] *= alpha;
x[2] *= alpha;
x[3] *= alpha;
x[4] *= alpha;
x[5] *= alpha;
x[6] *= alpha;
x[7] *= alpha;
x+=8;
}
}
@@ -65,19 +65,19 @@ static void dscal_kernel_8( BLASLONG n, FLOAT *da , FLOAT *x )
static void dscal_kernel_8_zero( BLASLONG n, FLOAT *alpha , FLOAT *x )
{
BLASLONG i;
for( i=0; i<n; i+=8 )
{
x[0] = 0.0;
x[1] = 0.0;
x[2] = 0.0;
x[3] = 0.0;
x[4] = 0.0;
x[5] = 0.0;
x[6] = 0.0;
x[7] = 0.0;
x+=8;
}
BLASLONG i;
for( i=0; i<n; i+=8 )
{
x[0] = 0.0;
x[1] = 0.0;
x[2] = 0.0;
x[3] = 0.0;
x[4] = 0.0;
x[5] = 0.0;
x[6] = 0.0;
x[7] = 0.0;
x+=8;
}
}
@@ -89,51 +89,51 @@ static void dscal_kernel_inc_8(BLASLONG n, FLOAT *alpha, FLOAT *x, BLASLONG inc_
static void dscal_kernel_inc_8(BLASLONG n, FLOAT *alpha, FLOAT *x, BLASLONG inc_x)
{
FLOAT *x1=NULL;
BLASLONG inc_x3;
FLOAT *x1=NULL;
BLASLONG inc_x3;
inc_x <<= 3;
inc_x3 = (inc_x << 1) + inc_x;
inc_x <<= 3;
inc_x3 = (inc_x << 1) + inc_x;
__asm__ __volatile__
(
"movddup (%3), %%xmm0 \n\t" // alpha
"movddup (%3), %%xmm0 \n\t" // alpha
"leaq (%1,%4,4), %2 \n\t"
"leaq (%1,%4,4), %2 \n\t"
".p2align 4 \n\t"
".p2align 4 \n\t"
"1: \n\t"
"movsd (%1) , %%xmm4 \n\t"
"movhpd (%1,%4,1), %%xmm4 \n\t"
"movsd (%1,%4,2), %%xmm5 \n\t"
"movhpd (%1,%5,1), %%xmm5 \n\t"
"1: \n\t"
"movsd (%1) , %%xmm4 \n\t"
"movhpd (%1,%4,1), %%xmm4 \n\t"
"movsd (%1,%4,2), %%xmm5 \n\t"
"movhpd (%1,%5,1), %%xmm5 \n\t"
"movsd (%2) , %%xmm6 \n\t"
"movhpd (%2,%4,1), %%xmm6 \n\t"
"movsd (%2,%4,2), %%xmm7 \n\t"
"movhpd (%2,%5,1), %%xmm7 \n\t"
"movsd (%2) , %%xmm6 \n\t"
"movhpd (%2,%4,1), %%xmm6 \n\t"
"movsd (%2,%4,2), %%xmm7 \n\t"
"movhpd (%2,%5,1), %%xmm7 \n\t"
"mulpd %%xmm0, %%xmm4 \n\t"
"mulpd %%xmm0, %%xmm5 \n\t"
"mulpd %%xmm0, %%xmm6 \n\t"
"mulpd %%xmm0, %%xmm7 \n\t"
"mulpd %%xmm0, %%xmm4 \n\t"
"mulpd %%xmm0, %%xmm5 \n\t"
"mulpd %%xmm0, %%xmm6 \n\t"
"mulpd %%xmm0, %%xmm7 \n\t"
"movsd %%xmm4 , (%1) \n\t"
"movhpd %%xmm4 , (%1,%4,1) \n\t"
"movsd %%xmm5 , (%1,%4,2) \n\t"
"movhpd %%xmm5 , (%1,%5,1) \n\t"
"movsd %%xmm4 , (%1) \n\t"
"movhpd %%xmm4 , (%1,%4,1) \n\t"
"movsd %%xmm5 , (%1,%4,2) \n\t"
"movhpd %%xmm5 , (%1,%5,1) \n\t"
"movsd %%xmm6 , (%2) \n\t"
"movhpd %%xmm6 , (%2,%4,1) \n\t"
"movsd %%xmm7 , (%2,%4,2) \n\t"
"movhpd %%xmm7 , (%2,%5,1) \n\t"
"movsd %%xmm6 , (%2) \n\t"
"movhpd %%xmm6 , (%2,%4,1) \n\t"
"movsd %%xmm7 , (%2,%4,2) \n\t"
"movhpd %%xmm7 , (%2,%5,1) \n\t"
"leaq (%1,%4,8), %1 \n\t"
"leaq (%2,%4,8), %2 \n\t"
"leaq (%1,%4,8), %1 \n\t"
"leaq (%2,%4,8), %2 \n\t"
"subq $8, %0 \n\t"
"jnz 1b \n\t"
"subq $8, %0 \n\t"
"jnz 1b \n\t"
:
"+r" (n), // 0
@@ -150,106 +150,96 @@ static void dscal_kernel_inc_8(BLASLONG n, FLOAT *alpha, FLOAT *x, BLASLONG inc_
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}
int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *dummy, BLASLONG dummy2)
{
BLASLONG i=0,j=0;
BLASLONG i = 0, j = 0;
if ( inc_x != 1 )
{
if ( da == 0.0 )
{
BLASLONG n1 = n & -2;
while(j < n1)
{
if (isinf(x[i])||isnan(x[i]))
x[i]=NAN;
else x[i]=0.0;
if (isinf(x[i+inc_x])||isnan(x[i+inc_x]))
x[i+inc_x]=NAN;
else x[i+inc_x]=0.0;
i += 2*inc_x ;
j+=2;
}
while(j < n)
{
if (isinf(x[i])||isnan(x[i]))
x[i]=NAN;
else x[i]=0.0;
i += inc_x ;
j++;
}
}
else
{
BLASLONG n1 = n & -8;
if ( n1 > 0 )
{
dscal_kernel_inc_8(n1, &da, x, inc_x);
i = n1 * inc_x;
j = n1;
}
while(j < n)
{
x[i] *= da;
i += inc_x ;
j++;
}
}
return(0);
}
BLASLONG n1 = n & -8;
if ( n1 > 0 )
{
// if ( da == 0.0 )
// dscal_kernel_8_zero(n1 , &da , x);
// else
dscal_kernel_8(n1 , &da , x);
}
if ( da == 0.0 )
{
for ( i=n1 ; i<n; i++ )
{
if(isinf(x[i])||isnan(x[i]))
x[i]=NAN;
else x[i] = 0.0;
}
}
else if (isinf(da)){
for ( i=n1 ; i<n; i++)
if (x[i]==0.) x[i]=NAN;
else x[i] *=da;
}
else
{
for ( i=n1 ; i<n; i++ )
{
if(isinf(x[i]))
x[i]=NAN;
else x[i] *= da;
}
}
return(0);
// Resolved issue 4728 when the caller is dscal
if (dummy2 == 1 && da == 0.0)
{
if ( inc_x != 1 )
{
BLASLONG n1 = n & -8;
if ( n1 > 0 )
{
dscal_kernel_inc_8(n1, &da, x, inc_x);
i = n1 * inc_x;
j = n1;
}
while(j < n)
{
x[i] *= da;
i += inc_x ;
j++;
}
}
else
{
BLASLONG n1 = n & -8;
if ( n1 > 0)
dscal_kernel_8(n1 , &da , x);
for ( i = n1 ; i < n; i++ )
x[i] *= da;
}
}
else
{
if ( inc_x != 1 )
{
if( da == 0.0)
{
BLASLONG n1 = n & -2;
while(j < n1)
{
x[i] = 0.0;
x[i+inc_x] = 0.0;
i += 2 * inc_x ;
j += 2;
}
while(j < n)
{
x[i] = 0.0;
i += inc_x ;
j++;
}
}
else
{
BLASLONG n1 = n & -8;
if ( n1 > 0 )
{
dscal_kernel_inc_8(n1, &da, x, inc_x);
i = n1 * inc_x;
j = n1;
}
while(j < n)
{
x[i] *= da;
i += inc_x ;
j++;
}
}
}
else
{
if ( da == 0.0 )
{
BLASLONG n1 = n & -8;
if ( n1 > 0)
dscal_kernel_8_zero(n1, &da, x);
for ( i = n1 ; i < n; i++ )
x[i] = 0.0;
}
else
{
BLASLONG n1 = n & -8;
if ( n1 > 0)
dscal_kernel_8(n1 , &da , x);
for ( i = n1 ; i < n; i++ )
x[i] *= da;
}
}
}
}

8
kernel/x86_64/scal_atom.S
View File

@@ -60,8 +60,10 @@
#ifdef WINDOWS_ABI
movq 40(%rsp), X
movq 48(%rsp), INCX
movq 64(%rsp), %r9
movaps %xmm3, %xmm0
#else
movq 24(%rsp), %r9
#endif
SAVEREGISTERS
@@ -73,6 +75,10 @@
lea (, INCX, SIZE), INCX
comisd %xmm0, %xmm1
jne .L100
jp .L100
cmpq $1, %r9
je .L100
/* Alpha == ZERO */
cmpq $SIZE, INCX

6
kernel/x86_64/scal_sse.S
View File

@@ -60,8 +60,10 @@
#ifdef WINDOWS_ABI
movq 40(%rsp), X
movq 48(%rsp), INCX
movq 64(%rsp), %r9
movaps %xmm3, %xmm0
#else
movq 24(%rsp), %r9
#endif
SAVEREGISTERS
@@ -76,6 +78,8 @@
shufps $0, %xmm0, %xmm0
jne .L100 # Alpha != ZERO
cmpq $1, %r9
je .L100
/* Alpha == ZERO */
cmpq $SIZE, INCX

7
kernel/x86_64/scal_sse2.S
View File

@@ -48,6 +48,7 @@
#define X ARG2
#define INCX ARG3
#endif
#define FLAG %r9
#define XX %r10
#define I %rax
@@ -60,8 +61,10 @@
#ifdef WINDOWS_ABI
movq 40(%rsp), X
movq 48(%rsp), INCX
movq 64(%rsp), FLAG
movaps %xmm3, %xmm0
#else
movq 24(%rsp), FLAG
#endif
SAVEREGISTERS
@@ -75,6 +78,8 @@
comisd %xmm0, %xmm1
jne .L100 # Alpha != ZERO
jp .L100 # For Alpha = NaN
cmpq $1, FLAG
je .L100 # disable the Alpha=zero path as it does not handle x=inf or nan
/* Alpha == ZERO */
cmpq $SIZE, INCX

269
kernel/x86_64/sscal.c
View File

@@ -39,21 +39,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
static void sscal_kernel_16( BLASLONG n, FLOAT *da , FLOAT *x )
{
BLASLONG i;
FLOAT alpha = *da;
BLASLONG i;
FLOAT alpha = *da;
for( i=0; i<n; i+=8 )
{
x[0] *= alpha;
x[1] *= alpha;
x[2] *= alpha;
x[3] *= alpha;
x[4] *= alpha;
x[5] *= alpha;
x[6] *= alpha;
x[7] *= alpha;
x+=8;
}
for( i=0; i<n; i+=8 )
{
x[0] *= alpha;
x[1] *= alpha;
x[2] *= alpha;
x[3] *= alpha;
x[4] *= alpha;
x[5] *= alpha;
x[6] *= alpha;
x[7] *= alpha;
x+=8;
}
}
@@ -61,19 +61,19 @@ static void sscal_kernel_16( BLASLONG n, FLOAT *da , FLOAT *x )
static void sscal_kernel_16_zero( BLASLONG n, FLOAT *alpha , FLOAT *x )
{
BLASLONG i;
for( i=0; i<n; i+=8 )
{
x[0] = 0.0;
x[1] = 0.0;
x[2] = 0.0;
x[3] = 0.0;
x[4] = 0.0;
x[5] = 0.0;
x[6] = 0.0;
x[7] = 0.0;
x+=8;
}
BLASLONG i;
for( i=0; i<n; i+=8 )
{
x[0] = 0.0;
x[1] = 0.0;
x[2] = 0.0;
x[3] = 0.0;
x[4] = 0.0;
x[5] = 0.0;
x[6] = 0.0;
x[7] = 0.0;
x+=8;
}
}
@@ -85,126 +85,119 @@ static void sscal_kernel_inc_8(BLASLONG n, FLOAT *alpha, FLOAT *x, BLASLONG inc_
static void sscal_kernel_inc_8(BLASLONG n, FLOAT *alpha, FLOAT *x, BLASLONG inc_x)
{
BLASLONG i;
BLASLONG inc_x2 = 2 * inc_x;
BLASLONG inc_x3 = inc_x2 + inc_x;
FLOAT t0,t1,t2,t3;
FLOAT da = alpha[0];
BLASLONG i;
BLASLONG inc_x2 = 2 * inc_x;
BLASLONG inc_x3 = inc_x2 + inc_x;
FLOAT t0,t1,t2,t3;
FLOAT da = alpha[0];
for ( i=0; i<n; i+=4 )
{
t0 = da * x[0];
t1 = da * x[inc_x];
t2 = da * x[inc_x2];
t3 = da * x[inc_x3];
for ( i=0; i<n; i+=4 )
{
t0 = da * x[0];
t1 = da * x[inc_x];
t2 = da * x[inc_x2];
t3 = da * x[inc_x3];
x[0] = t0;
x[inc_x] = t1;
x[inc_x2] = t2;
x[inc_x3] = t3;
x[0] = t0;
x[inc_x] = t1;
x[inc_x2] = t2;
x[inc_x3] = t3;
x+=4*inc_x;
x+=4*inc_x;
}
}
}
int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *dummy, BLASLONG dummy2)
{
BLASLONG i=0,j=0;
BLASLONG i = 0, j = 0;
if ( inc_x != 1 )
{
if ( da == 0.0 )
{
BLASLONG n1 = n & -2;
while(j < n1)
{
if (isinf(x[i])||isnan(x[i]))
x[i]=NAN;
else x[i]=0.0;
if (isinf(x[i+inc_x])||isnan(x[i+inc_x]))
x[i+inc_x]=NAN;
else x[i+inc_x]=0.0;
i += 2*inc_x ;
j+=2;
}
while(j < n)
{
if (isinf(x[i])||isnan(x[i]))
x[i]=NAN;
else x[i]=0.0;
i += inc_x ;
j++;
}
}
else
{
#if 1
BLASLONG n1 = n & -8;
if ( n1 > 0 )
{
sscal_kernel_inc_8(n1, &da, x, inc_x);
i = n1 * inc_x;
j = n1;
}
#endif
while(j < n)
{
x[i] *= da;
i += inc_x ;
j++;
}
}
return(0);
}
BLASLONG n1 = n & -16;
if ( n1 > 0 )
{
//if ( da == 0.0 )
// sscal_kernel_16_zero(n1 , &da , x);
//else
sscal_kernel_16(n1 , &da , x);
}
if ( da == 0.0 )
{
for ( i=n1 ; i<n; i++ )
{
if (isinf(x[i])||isnan(x[i]))
x[i]=NAN;
else x[i]=0.0;
}
}
else if ( isinf(da) )
{
for ( i=n1 ; i<n; i++ )
{
if (x[i] == 0.0)
x[i]=NAN;
else x[i] *= da;
}
}
else
{
for ( i=n1 ; i<n; i++ )
{
if (isinf(x[i]))
x[i]=NAN;
else x[i] *= da;
}
}
return(0);
// Resolved issue 4728 when the caller is sscal
if (dummy2 == 1 && da == 0.0)
{
if ( inc_x != 1 )
{
BLASLONG n1 = n & -8;
if ( n1 > 0 )
{
sscal_kernel_inc_8(n1, &da, x, inc_x);
i = n1 * inc_x;
j = n1;
}
while(j < n)
{
x[i] *= da;
i += inc_x ;
j++;
}
}
else
{
BLASLONG n1 = n & -16;
if ( n1 > 0)
sscal_kernel_16(n1 , &da , x);
for ( i = n1 ; i < n; i++ )
x[i] *= da;
}
}
else
{
if ( inc_x != 1 )
{
if( da == 0.0)
{
BLASLONG n1 = n & -2;
while(j < n1)
{
x[i] = 0.0;
x[i+inc_x] = 0.0;
i += 2 * inc_x ;
j += 2;
}
while(j < n)
{
x[i] = 0.0;
i += inc_x ;
j++;
}
}
else
{
BLASLONG n1 = n & -8;
if ( n1 > 0 )
{
sscal_kernel_inc_8(n1, &da, x, inc_x);
i = n1 * inc_x;
j = n1;
}
while(j < n)
{
x[i] *= da;
i += inc_x ;
j++;
}
}
}
else
{
if ( da == 0.0 )
{
BLASLONG n1 = n & -16;
if ( n1 > 0)
sscal_kernel_16_zero(n1, &da, x);
for ( i = n1 ; i < n; i++ )
x[i] = 0.0;
}
else
{
BLASLONG n1 = n & -16;
if ( n1 > 0)
sscal_kernel_16(n1 , &da , x);
for ( i = n1 ; i < n; i++ )
x[i] *= da;
}
}
}
}

2
kernel/x86_64/zscal_atom.S
View File

@@ -74,7 +74,7 @@
pxor %xmm15, %xmm15
comisd %xmm0, %xmm15
jne .L30 # Alpha_r != ZERO
jp .L30
comisd %xmm1, %xmm15
jne .L30 # Alpha_i != ZERO

2
kernel/x86_64/zscal_sse.S
View File

@@ -76,7 +76,7 @@
pxor %xmm15, %xmm15
comiss %xmm0, %xmm15
jne .L100 # Alpha_r != ZERO
jp .L100 # Alpha_r == NAN
comiss %xmm1, %xmm15
jne .L100 # Alpha_i != ZERO

60
kernel/zarch/cscal.c
View File

@@ -234,12 +234,23 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r, FLOAT da_i,
} else {
while (j < n1) {
temp0 = -da_i * x[i + 1];
x[i + 1] = da_i * x[i];
if (isnan(x[i]) || isinf(x[i]))
temp0 = NAN;
else
temp0 = -da_i * x[i + 1];
if (!isinf(x[i + 1]))
x[i + 1] = da_i * x[i];
else
x[i + 1] = NAN;
x[i] = temp0;
temp1 = -da_i * x[i + 1 + inc_x];
x[i + 1 + inc_x] = da_i * x[i + inc_x];
if (isnan(x[i+inc_x]) || isinf(x[i+inc_x]))
temp1 = NAN;
else
temp1 = -da_i * x[i + 1 + inc_x];
if (!isinf(x[i + 1 + inc_x]))
x[i + 1 + inc_x] = da_i * x[i + inc_x];
else
x[i + 1 + inc_x] = NAN;
x[i + inc_x] = temp1;
i += 2 * inc_x;
j += 2;
@@ -247,9 +258,14 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r, FLOAT da_i,
}
while (j < n) {
temp0 = -da_i * x[i + 1];
x[i + 1] = da_i * x[i];
if (isnan(x[i]) || isinf(x[i]))
temp0 = NAN;
else
temp0 = -da_i * x[i + 1];
if (isinf(x[i + 1]))
x[i + 1] = NAN;
else
x[i + 1] = da_i * x[i];
x[i] = temp0;
i += inc_x;
j++;
@@ -332,26 +348,42 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r, FLOAT da_i,
j = n1;
}
if (da_r == 0.0) {
if (da_r == 0.0 || isnan(da_r)) {
if (da_i == 0.0) {
float res = 0.0;
if (isnan(da_r)) res = da_r;
while (j < n) {
x[i] = 0.0;
x[i + 1] = 0.0;
x[i] = res;
x[i + 1] = res;
i += 2;
j++;
}
} else if (isinf(da_r)) {
while(j < n)
{
x[i]= NAN;
x[i+1] = da_r;
i += 2 ;
j++;
}
} else {
while (j < n) {
temp0 = -da_i * x[i + 1];
x[i + 1] = da_i * x[i];
x[i] = temp0;
if (isinf(x[i])) temp0 = NAN;
if (!isinf(x[i + 1]))
x[i + 1] = da_i * x[i];
else
x[i + 1] = NAN;
if (x[i] == x[i])
x[i] = temp0;
i += 2;
j++;

44
kernel/zarch/dscal.c
View File

@@ -96,20 +96,28 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x,
if (inc_x == 1) {
if (da == 0.0) {
if (dummy2 == 0) {
BLASLONG n1 = n & -16;
if (n1 > 0) {
dscal_kernel_16_zero(n1, x);
j = n1;
}
BLASLONG n1 = n & -16;
if (n1 > 0) {
dscal_kernel_16_zero(n1, x);
j = n1;
while (j < n) {
x[j] = 0.0;
j++;
}
} else {
while (j < n) {
if (isfinite(x[j]))
x[j] = 0.0;
else
x[j] = NAN;
j++;
}
}
while (j < n) {
x[j] = 0.0;
j++;
}
} else {
BLASLONG n1 = n & -16;
@@ -127,11 +135,9 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x,
} else {
if (da == 0.0) {
if (dummy2 == 0) {
BLASLONG n1 = n & -4;
while (j < n1) {
x[i] = 0.0;
x[i + inc_x] = 0.0;
x[i + 2 * inc_x] = 0.0;
@@ -139,11 +145,13 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x,
i += inc_x * 4;
j += 4;
}
}
while (j < n) {
x[i] = 0.0;
if (dummy2==0 || isfinite(x[i]))
x[i] = 0.0;
else
x[i] = NAN;
i += inc_x;
j++;
}

75
kernel/zarch/sscal.c
View File

@@ -95,21 +95,31 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x,
if (inc_x == 1) {
if (da == 0.0) {
if (da == 0.0 || !isfinite(da)) {
if (dummy2 == 0) {
BLASLONG n1 = n & -32;
if (n1 > 0) {
BLASLONG n1 = n & -32;
if (n1 > 0) {
sscal_kernel_32_zero(n1, x);
j = n1;
}
sscal_kernel_32_zero(n1, x);
j = n1;
while (j < n) {
x[j] = 0.0;
j++;
}
} else {
float res = 0.0;
if (!isfinite(da)) res = NAN;
while (j < n) {
if (isfinite(x[i]))
x[j] = res;
else
x[j] = NAN;
j++;
}
}
while (j < n) {
x[j] = 0.0;
j++;
}
} else {
BLASLONG n1 = n & -32;
@@ -126,26 +136,37 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x,
} else {
if (da == 0.0) {
if (da == 0.0 || !isfinite(da)) {
if (dummy2 == 0) {
BLASLONG n1 = n & -2;
BLASLONG n1 = n & -2;
while (j < n1) {
while (j < n1) {
x[i] = 0.0;
x[i + inc_x] = 0.0;
x[i] = 0.0;
x[i + inc_x] = 0.0;
i += inc_x * 2;
j += 2;
i += inc_x * 2;
j += 2;
}
while (j < n) {
x[i] = 0.0;
i += inc_x;
j++;
}
}
while (j < n) {
x[i] = 0.0;
i += inc_x;
j++;
}
} else {
while (j < n) {
float res = 0.0;
if (!isfinite(da)) res = NAN;
if (isfinite(x[i]))
x[i] = res;
else
x[i] = NAN;
i += inc_x;
j++;
}
}
} else {
BLASLONG n1 = n & -2;

43
kernel/zarch/zscal.c
View File

@@ -237,13 +237,19 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r, FLOAT da_i,
temp0 = NAN;
else
temp0 = -da_i * x[i + 1];
x[i + 1] = da_i * x[i];
if (!isinf(x[i + 1]))
x[i + 1] = da_i * x[i];
else
x[i + 1] = NAN;
x[i] = temp0;
if (isnan(x[i + inc_x]) || isinf(x[i + inc_x]))
temp1 = NAN;
else
temp1 = -da_i * x[i + 1 + inc_x];
x[i + 1 + inc_x] = da_i * x[i + inc_x];
if (!isinf(x[i + 1 + inc_x]))
x[i + 1 + inc_x] = da_i * x[i + inc_x];
else
x[i + 1 + inc_x] = NAN;
x[i + inc_x] = temp1;
i += 2 * inc_x;
j += 2;
@@ -256,7 +262,10 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r, FLOAT da_i,
temp0 = NAN;
else
temp0 = -da_i * x[i + 1];
x[i + 1] = da_i * x[i];
if (!isinf(x[i +1]))
x[i + 1] = da_i * x[i];
else
x[i + 1] = NAN;
x[i] = temp0;
i += inc_x;
j++;
@@ -330,7 +339,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r, FLOAT da_i,
zscal_kernel_8_zero(n1, x);
else
zscal_kernel_8(n1, da_r, da_i, x);
else if (da_i == 0)
else if (da_i == 0 && da_r == da_r)
zscal_kernel_8_zero_i(n1, alpha, x);
else
zscal_kernel_8(n1, da_r, da_i, x);
@@ -339,29 +348,41 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r, FLOAT da_i,
j = n1;
}
if (da_r == 0.0) {
if (da_r == 0.0 || isnan(da_r)) {
if (da_i == 0.0) {
double res= 0.0;
if (isnan(da_r)) res = da_r;
while (j < n) {
x[i] = 0.0;
x[i + 1] = 0.0;
x[i] = res;
x[i + 1] = res;
i += 2;
j++;
}
} else if (isinf(da_r)) {
while (j < n) {
x[i] = NAN;
x[i + 1] = da_r;
i += 2;
j++;
}
} else {
while (j < n) {
if (isnan(x[i]) || isinf(x[i]))
if (isinf(x[i]))
temp0 = NAN;
else
temp0 = -da_i * x[i + 1];
x[i + 1] = da_i * x[i];
x[i] = temp0;
if (!isinf(x[i + 1]))
x[i + 1] = da_i * x[i];
else
x[i + 1] = NAN;
if (x[i]==x[i])
x[i] = temp0;
i += 2;
j++;