sbgemm: cooperlake: enable SBGEMM by small matrix path
This commit is contained in:
Wangyang Guo
parent
1d83ca4bca
commit
7d27b182fc
|
|
@ -1 +1,11 @@
|
|||
include $(KERNELDIR)/KERNEL.SKYLAKEX
|
||||
|
||||
SBGEMM_SMALL_M_PERMIT = sbgemm_small_kernel_permit_cooperlake.c
|
||||
SBGEMM_SMALL_K_NN = sbgemm_small_kernel_nn_cooperlake.c
|
||||
SBGEMM_SMALL_K_B0_NN = sbgemm_small_kernel_b0_nn_cooperlake.c
|
||||
SBGEMM_SMALL_K_NT = sbgemm_small_kernel_nt_cooperlake.c
|
||||
SBGEMM_SMALL_K_B0_NT = sbgemm_small_kernel_b0_nt_cooperlake.c
|
||||
SBGEMM_SMALL_K_TN = sbgemm_small_kernel_tn_cooperlake.c
|
||||
SBGEMM_SMALL_K_B0_TN = sbgemm_small_kernel_b0_tn_cooperlake.c
|
||||
SBGEMM_SMALL_K_TT = sbgemm_small_kernel_tt_cooperlake.c
|
||||
SBGEMM_SMALL_K_B0_TT = sbgemm_small_kernel_b0_tt_cooperlake.c
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
//#include "sbgemm.h"
|
||||
|
||||
#include <immintrin.h>
|
||||
|
||||
// Walk around those intrinsics that missed by compiler
|
||||
#define MM256_LOADU_EPI16(addr) \
|
||||
_mm256_maskz_loadu_epi16(~0, (addr))
|
||||
|
|
@ -1747,7 +1746,7 @@ void COL_MAJOR_OTCOPY_KERNEL_Kx8m(BLASLONG k, BLASLONG n, bfloat16 * B, BLASLONG
|
|||
}
|
||||
|
||||
// Scale matrix C when beta is not ZERO or ONE
|
||||
void sbgemm_scal_operation(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST float beta, float *C, OPENBLAS_CONST blasint ldc)
|
||||
void sbgemm_scal_operation(BLASLONG M, BLASLONG N, float beta, float *C, BLASLONG ldc)
|
||||
{
|
||||
float * C_addr0 = C;
|
||||
float * C_addr1 = C + ldc;
|
||||
|
|
@ -1759,12 +1758,6 @@ void sbgemm_scal_operation(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST
|
|||
__m512 array_512_0, array_512_1, array_512_2, array_512_3;
|
||||
__m512 BETAVECTOR = _mm512_set1_ps(beta);
|
||||
|
||||
if (Order == CblasRowMajor) {
|
||||
blasint tmp = M;
|
||||
M = N;
|
||||
N = tmp;
|
||||
}
|
||||
|
||||
BLASLONG tag_n_Nx = N & (~3);
|
||||
BLASLONG tag_n_Mx = M & (~15);
|
||||
unsigned short tail_mask = (((unsigned short)0xffff) >> (16-M+tag_n_Mx));
|
||||
|
|
@ -1828,7 +1821,7 @@ void sbgemm_scal_operation(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST
|
|||
}
|
||||
|
||||
// Zero C matrix when Beta is 0
|
||||
void sbgemm_zero_operation(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, float *C, OPENBLAS_CONST blasint ldc)
|
||||
void sbgemm_zero_operation(BLASLONG M, BLASLONG N, float *C, BLASLONG ldc)
|
||||
{
|
||||
float * C_addr0 = C;
|
||||
float * C_addr1 = C + ldc;
|
||||
|
|
@ -1839,12 +1832,6 @@ void sbgemm_zero_operation(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST
|
|||
|
||||
__m512 ZEROVECTOR = _mm512_setzero_ps();
|
||||
|
||||
if (Order == CblasRowMajor) {
|
||||
blasint tmp = M;
|
||||
M = N;
|
||||
N = tmp;
|
||||
}
|
||||
|
||||
BLASLONG tag_n_Nx = N & (~3);
|
||||
BLASLONG tag_n_Mx = M & (~15);
|
||||
unsigned short tail_mask = (((unsigned short)0xffff) >> (16-M+tag_n_Mx));
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
#include "sbgemm.h"
|
||||
#include "bf16_common_macros.h"
|
||||
#include <immintrin.h>
|
||||
|
||||
/* These macros are needed and should be placed at the right place
|
||||
#define BF16_BLOCK_STEP_N 8
|
||||
#define BF16_BLOCK_THRES_K 1024
|
||||
#define BF16_BLOCK_THRES_M 32
|
||||
|
|
@ -14,7 +12,6 @@
|
|||
|
||||
#define ONE 1.e0f
|
||||
#define ZERO 0.e0f
|
||||
*/
|
||||
|
||||
#undef STORE16_COMPLETE_RESULT
|
||||
#undef STORE16_MASK_COMPLETE_RESULT
|
||||
|
|
@ -1798,6 +1795,7 @@ void sbgemm_blocking_kernel_tt_one(blasint M, blasint N, blasint K, float alpha,
|
|||
}
|
||||
/* ----------------------------------------- End of TT kernels --------------------------------------- */
|
||||
|
||||
/*
|
||||
#ifndef ONE_ALPHA // ALPHA is not ONE
|
||||
void sbgemm_internal_kernel_alpha(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST blasint K,
|
||||
OPENBLAS_CONST float alpha, OPENBLAS_CONST bfloat16 *A, OPENBLAS_CONST blasint lda, OPENBLAS_CONST bfloat16 *B, OPENBLAS_CONST blasint ldb, float *C, OPENBLAS_CONST blasint ldc)
|
||||
|
|
@ -1836,3 +1834,4 @@ void sbgemm_internal_kernel_one(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_
|
|||
}
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
#define B0 1
|
||||
#include "./sbgemm_small_kernel_nn_cooperlake.c"
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
#define B0 1
|
||||
#include "./sbgemm_small_kernel_nt_cooperlake.c"
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
#define B0 1
|
||||
#include "./sbgemm_small_kernel_tn_cooperlake.c"
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
#define B0 1
|
||||
#include "./sbgemm_small_kernel_tt_cooperlake.c"
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
#define TRANS_NN
|
||||
#include "sbgemm_small_kernel_template_cooperlake.c"
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
#define TRANS_NT
|
||||
#include "sbgemm_small_kernel_template_cooperlake.c"
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
/***************************************************************************
|
||||
Copyright (c) 2021, The OpenBLAS Project
|
||||
All rights reserved.
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
1. Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in
|
||||
the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
3. Neither the name of the OpenBLAS project nor the names of
|
||||
its contributors may be used to endorse or promote products
|
||||
derived from this software without specific prior written permission.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
|
||||
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
|
||||
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*****************************************************************************/
|
||||
|
||||
#include "common.h"
|
||||
|
||||
#include "sbgemm_block_microk_cooperlake.c"
|
||||
// Define micro kernels for ALPHA not ONE scenarios
|
||||
#undef ONE_ALPHA
|
||||
#include "sbgemm_microk_cooperlake_template.c"
|
||||
|
||||
// Define micro kernels for ALPHA as ONE scenarios
|
||||
#define ONE_ALPHA 1
|
||||
#include "sbgemm_microk_cooperlake_template.c"
|
||||
|
||||
int CNAME(int transa, int transb, BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, FLOAT beta)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
/***************************************************************************
|
||||
Copyright (c) 2021, The OpenBLAS Project
|
||||
All rights reserved.
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
1. Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in
|
||||
the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
3. Neither the name of the OpenBLAS project nor the names of
|
||||
its contributors may be used to endorse or promote products
|
||||
derived from this software without specific prior written permission.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
|
||||
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
|
||||
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*****************************************************************************/
|
||||
|
||||
#include "common.h"
|
||||
#include <memory.h>
|
||||
|
||||
extern void sbgemm_scal_operation(BLASLONG M, BLASLONG N, float beta, float *C, BLASLONG ldc);
|
||||
extern void sbgemm_zero_operation(BLASLONG M, BLASLONG N, float *C, BLASLONG ldc);
|
||||
|
||||
extern void sbgemm_blocking_kernel_nn_alpha(blasint M, blasint N, blasint K, float alpha, bfloat16 *A, blasint lda, bfloat16 *B, blasint ldb, float *C, blasint ldc, bfloat16 * block_A, bfloat16 * block_B);
|
||||
extern void sbgemm_blocking_kernel_nn_one(blasint M, blasint N, blasint K, float alpha, bfloat16 *A, blasint lda, bfloat16 *B, blasint ldb, float *C, blasint ldc, bfloat16 * block_A, bfloat16 * block_B);
|
||||
extern void sbgemm_blocking_kernel_nt_alpha(blasint M, blasint N, blasint K, float alpha, bfloat16 *A, blasint lda, bfloat16 *B, blasint ldb, float *C, blasint ldc, bfloat16 * block_A, bfloat16 * block_B);
|
||||
extern void sbgemm_blocking_kernel_nt_one(blasint M, blasint N, blasint K, float alpha, bfloat16 *A, blasint lda, bfloat16 *B, blasint ldb, float *C, blasint ldc, bfloat16 * block_A, bfloat16 * block_B);
|
||||
extern void sbgemm_blocking_kernel_tn_alpha(blasint M, blasint N, blasint K, float alpha, bfloat16 *A, blasint lda, bfloat16 *B, blasint ldb, float *C, blasint ldc, bfloat16 * block_A, bfloat16 * block_B);
|
||||
extern void sbgemm_blocking_kernel_tn_one(blasint M, blasint N, blasint K, float alpha, bfloat16 *A, blasint lda, bfloat16 *B, blasint ldb, float *C, blasint ldc, bfloat16 * block_A, bfloat16 * block_B);
|
||||
extern void sbgemm_blocking_kernel_tt_alpha(blasint M, blasint N, blasint K, float alpha, bfloat16 *A, blasint lda, bfloat16 *B, blasint ldb, float *C, blasint ldc, bfloat16 * block_A, bfloat16 * block_B);
|
||||
extern void sbgemm_blocking_kernel_tt_one(blasint M, blasint N, blasint K, float alpha, bfloat16 *A, blasint lda, bfloat16 *B, blasint ldb, float *C, blasint ldc, bfloat16 * block_A, bfloat16 * block_B);
|
||||
|
||||
#if defined(TRANS_NN)
|
||||
#define SBGEMM_BLOCKING_KERNEL_ONE sbgemm_blocking_kernel_nn_one
|
||||
#define SBGEMM_BLOCKING_KERNEL_ALPHA sbgemm_blocking_kernel_nn_alpha
|
||||
#elif defined(TRANS_NT)
|
||||
#define SBGEMM_BLOCKING_KERNEL_ONE sbgemm_blocking_kernel_nt_one
|
||||
#define SBGEMM_BLOCKING_KERNEL_ALPHA sbgemm_blocking_kernel_nt_alpha
|
||||
#elif defined(TRANS_TN)
|
||||
#define SBGEMM_BLOCKING_KERNEL_ONE sbgemm_blocking_kernel_tn_one
|
||||
#define SBGEMM_BLOCKING_KERNEL_ALPHA sbgemm_blocking_kernel_tn_alpha
|
||||
#elif defined(TRANS_TT)
|
||||
#define SBGEMM_BLOCKING_KERNEL_ONE sbgemm_blocking_kernel_tt_one
|
||||
#define SBGEMM_BLOCKING_KERNEL_ALPHA sbgemm_blocking_kernel_tt_alpha
|
||||
#endif
|
||||
|
||||
#define BF16_BLOCK_THRES_K 1024
|
||||
// If we want to adjust this to be bigger, need to change COL_MAJOR_INCOPY_KERNEL_Kx32 kernel to be bigger also
|
||||
#define BF16_BLOCK_THRES_M 32
|
||||
#define BF16_BLOCK_THRES_N 1024
|
||||
|
||||
|
||||
#if defined(B0)
|
||||
int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, IFLOAT * A, BLASLONG lda, FLOAT alpha, IFLOAT * B, BLASLONG ldb, FLOAT * C, BLASLONG ldc)
|
||||
#else
|
||||
int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, IFLOAT * A, BLASLONG lda, FLOAT alpha, IFLOAT * B, BLASLONG ldb, FLOAT beta, FLOAT * C, BLASLONG ldc)
|
||||
#endif
|
||||
{
|
||||
bfloat16 * block_A;
|
||||
bfloat16 * block_B;
|
||||
|
||||
block_A = (bfloat16 *) malloc(sizeof(bfloat16) * BF16_BLOCK_THRES_K * BF16_BLOCK_THRES_M);
|
||||
block_B = (bfloat16 *) malloc(sizeof(bfloat16) * BF16_BLOCK_THRES_N * BF16_BLOCK_THRES_K);
|
||||
|
||||
#if defined(B0)
|
||||
sbgemm_zero_operation(M, N, C, ldc);
|
||||
#else
|
||||
sbgemm_scal_operation(M, N, beta, C, ldc);
|
||||
#endif
|
||||
|
||||
if (alpha == ONE) {
|
||||
SBGEMM_BLOCKING_KERNEL_ONE(M, N, K, alpha, A, lda, B, ldb, C, ldc, block_A, block_B);
|
||||
} else {
|
||||
SBGEMM_BLOCKING_KERNEL_ALPHA(M, N, K, alpha, A, lda, B, ldb, C, ldc, block_A, block_B);
|
||||
}
|
||||
free(block_A);
|
||||
free(block_B);
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
#define TRANS_TN
|
||||
#include "sbgemm_small_kernel_template_cooperlake.c"
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
#define TRANS_TT
|
||||
#include "sbgemm_small_kernel_template_cooperlake.c"
|
||||
Loading…
Reference in New Issue