floraachy/OpenBLAS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Further tweak small GEMM for AArch64

Browse Source
This commit is contained in:
Chris Sidebottom
2024-06-24 10:47:47 +01:00
parent 7a6fa699f2
commit 8c472ef7e3
9 changed files with 772 additions and 2286 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kernel/arm64/dgemm_small_kernel_nt_sve.c
+114 -55
View File
@@ -46,13 +46,27 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
})
#endif
#define A_ELEMENT_K(m, offset_k) A[(i + (m)) + (k + offset_k) * lda]
#define RESET_A_POINTER() a_offset = A;
#define CREATE_A_POINTER(m, scale) FLOAT* a_offset##m = a_offset + scale;
#define UPDATE_A_POINTER(scale) a_offset = a_offset + scale;
#define A_ELEMENT_K(m, offset_k) *(a_offset##m + (k + offset_k) * lda)
#define A_ELEMENT(m) A_ELEMENT_K(m, 0)
#define B_ELEMENT_K(n, offset_k) B[(k + offset_k) * ldb + (j + (n))]
#define RESET_B_POINTER() b_offset = B;
#define CREATE_B_POINTER(n, scale) FLOAT* b_offset##n = b_offset + scale;
#define UPDATE_B_POINTER(scale) b_offset = b_offset + scale;
#define B_ELEMENT_K(n, offset_k) *(b_offset##n + (k + offset_k) * ldb)
#define B_ELEMENT(n) B_ELEMENT_K(n, 0)
#define C_ELEMENT(m, n) C[(i + (m)) + (j + (n)) * ldc]
#define CREATE_C_POINTER(n, scale) FLOAT* c_offset##n = c_offset + scale * ldc;
#define INCR_C_POINTER(m, incr) // c_offset ## m += incr;
#define UPDATE_C_POINTER(scale) c_offset = c_offset + scale * ldc;
#define C_ELEMENT(m, n) *(c_offset##n + ((m * v_size) + i))
// #undef C_ELEMENT
// #define C_ELEMENT(m, n) C[(i+(m))+(j+(n))*ldc]
#define PACK_ELEMENT_K(n, offset_k) packed_b[(k + offset_k) * 4 + n]
#define PACK_ELEMENT(n) PACK_ELEMENT_K(n, 0)
@@ -97,8 +111,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define BROADCAST_LOAD_B(n, offset_k) \
svfloat64_t b##s##n##_k##offset_k = svdup_f64(B_ELEMENT_K(n, offset_k));
#define VECTOR_LOAD_A(pg, m, offset_k) \
svfloat64_t a##s##m##_k##offset_k = \
svld1(pg, &A_ELEMENT_K(v_size * m, offset_k));
svfloat64_t a##s##m##_k##offset_k = svld1(pg, &A_ELEMENT_K(m, offset_k));
#define QUADWORD_LOAD_B(n, offset_k) \
svfloat64_t b##s##n##_k##offset_k = \
svld1rq(pg_true, &B_ELEMENT_K(n, offset_k));
@@ -111,26 +124,23 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifdef B0
#define VECTOR_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n);
svst1(pg, &C_ELEMENT(m, n), result##m##n);
#define SCATTER_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
svst1_scatter_index( \
pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n);
svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
#else
#define VECTOR_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
result##m##n = \
svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(v_size * m, n)), beta_vec); \
svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n);
svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(m, n)), beta_vec); \
svst1(pg, &C_ELEMENT(m, n), result##m##n);
#define SCATTER_STORE(pg, m, n) \
result##m##n = svmul_m(pg, result##m##n, alpha_vec); \
result##m##n = svmla_m( \
pg, \
result##m##n, \
svld1_gather_index(pg, &C_ELEMENT(v_size * m, n), svindex_u64(0LL, ldc)), \
beta_vec); \
svst1_scatter_index( \
pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n);
result##m##n = svmla_m(pg, \
result##m##n, \
svld1_gather_index(pg, &C_ELEMENT(m, n), ldc_vec), \
beta_vec); \
svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
#endif
#ifndef LIKELY
@@ -140,13 +150,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define LIKELY(x) (x)
#endif
#endif
#ifndef UNLIKELY
#ifdef __GNUC__
#define UNLIKELY(x) __builtin_expect(!!(x), 0)
#else
#define UNLIKELY(x) (x)
#endif
#endif
#ifdef B0
int
@@ -176,8 +179,7 @@ CNAME(BLASLONG M,
#endif
{
const uint64_t v_size = svcntd();
const uint64_t v_size32 = v_size * 32;
const uint64_t v_size3 = v_size * 3;
const uint64_t v_size2 = v_size * 2;
const svbool_t pg_true = svptrue_b64();
const svbool_t pg_quad = svwhilelt_b64(0, 2);
const svfloat64_t alpha_vec = svdup_f64(alpha);
@@ -186,14 +188,31 @@ CNAME(BLASLONG M,
#endif
const BLASLONG n4 = N & -4;
const BLASLONG n2 = N & -2;
const BLASLONG v_m3 = M - (M % v_size3);
const BLASLONG v_m2 = M & -v_size2;
const BLASLONG v_m1 = M & -v_size;
FLOAT* b_offset = B;
FLOAT* a_offset = A;
FLOAT* c_offset = C;
BLASLONG j = 0;
for (; j < n4; j += 4) {
CREATE_C_POINTER(0, 0);
CREATE_C_POINTER(1, 1);
CREATE_C_POINTER(2, 2);
CREATE_C_POINTER(3, 3);
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
CREATE_B_POINTER(2, 2);
CREATE_B_POINTER(3, 3);
BLASLONG i = 0;
for (; i < v_m3; i += v_size3) {
for (; i < v_m2; i += v_size2) {
CREATE_A_POINTER(0, 0);
CREATE_A_POINTER(1, v_size);
UPDATE_A_POINTER(v_size2);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
@@ -204,10 +223,6 @@ CNAME(BLASLONG M,
DECLARE_RESULT_VECTOR(1, 1);
DECLARE_RESULT_VECTOR(1, 2);
DECLARE_RESULT_VECTOR(1, 3);
DECLARE_RESULT_VECTOR(2, 0);
DECLARE_RESULT_VECTOR(2, 1);
DECLARE_RESULT_VECTOR(2, 2);
DECLARE_RESULT_VECTOR(2, 3);
for (; k < K; k++) {
@@ -223,11 +238,6 @@ CNAME(BLASLONG M,
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 0);
VECTOR_LOAD_A(pg_true, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(2, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(2, 1, 0, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(2, 2, 2, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(2, 3, 2, 1, 0);
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
@@ -237,13 +247,16 @@ CNAME(BLASLONG M,
VECTOR_STORE(pg_true, 1, 1);
VECTOR_STORE(pg_true, 1, 2);
VECTOR_STORE(pg_true, 1, 3);
VECTOR_STORE(pg_true, 2, 0);
VECTOR_STORE(pg_true, 2, 1);
VECTOR_STORE(pg_true, 2, 2);
VECTOR_STORE(pg_true, 2, 3);
INCR_C_POINTER(0, v_size2);
INCR_C_POINTER(1, v_size2);
INCR_C_POINTER(2, v_size2);
INCR_C_POINTER(3, v_size2);
}
for (; i < v_m1; i += v_size) {
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(v_size);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
@@ -264,9 +277,15 @@ CNAME(BLASLONG M,
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 0, 2);
VECTOR_STORE(pg_true, 0, 3);
INCR_C_POINTER(0, v_size);
INCR_C_POINTER(1, v_size);
INCR_C_POINTER(2, v_size);
INCR_C_POINTER(3, v_size);
}
for (; i < M; i += v_size) {
const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M));
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(0);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
@@ -288,20 +307,35 @@ CNAME(BLASLONG M,
VECTOR_STORE(pg_tail, 0, 1);
VECTOR_STORE(pg_tail, 0, 2);
VECTOR_STORE(pg_tail, 0, 3);
INCR_C_POINTER(0, 0);
INCR_C_POINTER(1, 0);
INCR_C_POINTER(2, 0);
INCR_C_POINTER(3, 0);
}
UPDATE_B_POINTER(4);
RESET_A_POINTER();
UPDATE_C_POINTER(4);
}
for (; j < n2; j += 2) {
CREATE_C_POINTER(0, 0);
CREATE_C_POINTER(1, 1);
CREATE_B_POINTER(0, 0);
CREATE_B_POINTER(1, 1);
BLASLONG i = 0;
for (; i < v_m3; i += v_size3) {
for (; i < v_m2; i += v_size2) {
CREATE_A_POINTER(0, 0);
CREATE_A_POINTER(1, v_size);
UPDATE_A_POINTER(v_size2);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
DECLARE_RESULT_VECTOR(1, 0);
DECLARE_RESULT_VECTOR(1, 1);
DECLARE_RESULT_VECTOR(2, 0);
DECLARE_RESULT_VECTOR(2, 1);
for (; k < K; k++) {
@@ -312,19 +346,19 @@ CNAME(BLASLONG M,
VECTOR_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
VECTOR_LOAD_A(pg_true, 2, 0);
UPDATE_RESULT_VECTOR_QUADWORD(2, 0, 0, 0, 0);
UPDATE_RESULT_VECTOR_QUADWORD(2, 1, 0, 1, 0);
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
VECTOR_STORE(pg_true, 1, 0);
VECTOR_STORE(pg_true, 1, 1);
VECTOR_STORE(pg_true, 2, 0);
VECTOR_STORE(pg_true, 2, 1);
INCR_C_POINTER(0, v_size2);
INCR_C_POINTER(1, v_size2);
}
for (; i < v_m1; i += v_size) {
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(v_size);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(0, 1);
@@ -338,9 +372,13 @@ CNAME(BLASLONG M,
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 0, 1);
INCR_C_POINTER(0, v_size);
INCR_C_POINTER(1, v_size);
}
for (; i < M; i += v_size) {
const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M));
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(0);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
@@ -355,17 +393,29 @@ CNAME(BLASLONG M,
}
VECTOR_STORE(pg_tail, 0, 0);
VECTOR_STORE(pg_tail, 0, 1);
INCR_C_POINTER(0, 0);
INCR_C_POINTER(1, 0);
}
UPDATE_B_POINTER(2);
RESET_A_POINTER();
UPDATE_C_POINTER(2);
}
for (; j < N; j++) {
CREATE_C_POINTER(0, 0);
CREATE_B_POINTER(0, 0);
BLASLONG i = 0;
for (; i < v_m3; i += v_size3) {
for (; i < v_m2; i += v_size2) {
CREATE_A_POINTER(0, 0);
CREATE_A_POINTER(1, v_size);
UPDATE_A_POINTER(v_size2);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
DECLARE_RESULT_VECTOR(1, 0);
DECLARE_RESULT_VECTOR(2, 0);
for (; k < K; k++) {
@@ -374,15 +424,16 @@ CNAME(BLASLONG M,
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
VECTOR_LOAD_A(pg_true, 1, 0);
UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
VECTOR_LOAD_A(pg_true, 2, 0);
UPDATE_RESULT_VECTOR(pg_true, 2, 0, 0);
}
VECTOR_STORE(pg_true, 0, 0);
VECTOR_STORE(pg_true, 1, 0);
VECTOR_STORE(pg_true, 2, 0);
INCR_C_POINTER(0, v_size2);
}
for (; i < v_m1; i += v_size) {
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(v_size);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
@@ -393,9 +444,12 @@ CNAME(BLASLONG M,
UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
}
VECTOR_STORE(pg_true, 0, 0);
INCR_C_POINTER(0, v_size);
}
for (; i < M; i += v_size) {
const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M));
CREATE_A_POINTER(0, 0);
UPDATE_A_POINTER(0);
BLASLONG k = 0;
DECLARE_RESULT_VECTOR(0, 0);
@@ -407,8 +461,13 @@ CNAME(BLASLONG M,
UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0);
}
VECTOR_STORE(pg_tail, 0, 0);
INCR_C_POINTER(0, 0);
}
UPDATE_B_POINTER(1);
RESET_A_POINTER();
UPDATE_C_POINTER(1);
}
return 0;
}
}