Add bfloat16 based dot and conversion with single/double
1. Added bfloat16 based dot as new API: shdot
2. Implemented generic kernel and cooperlake-specific (AVX512-BF16) kernel for shdot
3. Added 4 conversion APIs for bfloat16 data type <=> single/double: shstobf16 shdtobf16 sbf16tos dbf16tod
shstobf16 -- convert single float array to bfloat16 array
shdtobf16 -- convert double float array to bfloat16 array
sbf16tos -- convert bfloat16 array to single float array
dbf16tod -- convert bfloat16 array to double float array
4. Implemented generic kernels for all 4 conversion APIs, and cooperlake-specific kernel for shstobf16 and shdtobf16
5. Update level1 thread facilitate functions and macros to support multi-threading for these new APIs
6. Fix Cooperlake platform detection/specify issue when under dynamic-arch building
7. Change the typedef of bfloat16 from unsigned short to more strict uint16_t
Signed-off-by: Chen, Guobing <guobing.chen@intel.com>
This commit is contained in:
Chen, Guobing
@@ -47,7 +47,9 @@ SBLAS3OBJS = \
|
||||
sgeadd.$(SUFFIX)
|
||||
|
||||
ifeq ($(BUILD_HALF),1)
|
||||
SHBLAS1OBJS = shdot.$(SUFFIX)
|
||||
SHBLAS3OBJS = shgemm.$(SUFFIX)
|
||||
SHEXTOBJS = shstobf16.$(SUFFIX) shdtobf16.$(SUFFIX) sbf16tos.$(SUFFIX) dbf16tod.$(SUFFIX)
|
||||
endif
|
||||
|
||||
DBLAS1OBJS = \
|
||||
@@ -281,7 +283,9 @@ CSBLAS3OBJS = \
|
||||
cblas_sgeadd.$(SUFFIX)
|
||||
|
||||
ifeq ($(BUILD_HALF),1)
|
||||
CSHBLAS1OBJS = cblas_shdot.$(SUFFIX)
|
||||
CSHBLAS3OBJS = cblas_shgemm.$(SUFFIX)
|
||||
CSHEXTOBJS = cblas_shstobf16.$(SUFFIX) cblas_shdtobf16.$(SUFFIX) cblas_sbf16tos.$(SUFFIX) cblas_dbf16tod.$(SUFFIX)
|
||||
endif
|
||||
|
||||
CDBLAS1OBJS = \
|
||||
@@ -374,6 +378,7 @@ override CFLAGS += -I.
|
||||
SBLAS1OBJS += $(CSBLAS1OBJS)
|
||||
SBLAS2OBJS += $(CSBLAS2OBJS)
|
||||
SBLAS3OBJS += $(CSBLAS3OBJS)
|
||||
SHBLAS1OBJS += $(CSHBLAS1OBJS)
|
||||
SHBLAS3OBJS += $(CSHBLAS3OBJS)
|
||||
DBLAS1OBJS += $(CDBLAS1OBJS)
|
||||
DBLAS2OBJS += $(CDBLAS2OBJS)
|
||||
@@ -385,10 +390,11 @@ ZBLAS1OBJS += $(CZBLAS1OBJS)
|
||||
ZBLAS2OBJS += $(CZBLAS2OBJS)
|
||||
ZBLAS3OBJS += $(CZBLAS3OBJS)
|
||||
|
||||
SHEXTOBJS += $(CSHEXTOBJS)
|
||||
endif
|
||||
|
||||
SBLASOBJS = $(SBLAS1OBJS) $(SBLAS2OBJS) $(SBLAS3OBJS)
|
||||
SHBLASOBJS = $(SHBLAS3OBJS)
|
||||
SHBLASOBJS = $(SHBLAS1OBJS) $(SHBLAS3OBJS)
|
||||
DBLASOBJS = $(DBLAS1OBJS) $(DBLAS2OBJS) $(DBLAS3OBJS)
|
||||
QBLASOBJS = $(QBLAS1OBJS) $(QBLAS2OBJS) $(QBLAS3OBJS)
|
||||
CBLASOBJS = $(CBLAS1OBJS) $(CBLAS2OBJS) $(CBLAS3OBJS)
|
||||
@@ -463,7 +469,7 @@ ZBLASOBJS += $(ZLAPACKOBJS)
|
||||
|
||||
endif
|
||||
|
||||
FUNCOBJS = $(SHBLASOBJS) $(SBLASOBJS) $(DBLASOBJS) $(CBLASOBJS) $(ZBLASOBJS)
|
||||
FUNCOBJS = $(SHEXTOBJS) $(SHBLASOBJS) $(SBLASOBJS) $(DBLASOBJS) $(CBLASOBJS) $(ZBLASOBJS)
|
||||
|
||||
ifdef EXPRECISION
|
||||
FUNCOBJS += $(QBLASOBJS) $(XBLASOBJS)
|
||||
@@ -491,7 +497,7 @@ endif
|
||||
clean ::
|
||||
@rm -f functable.h
|
||||
|
||||
level1 : $(SBLAS1OBJS) $(DBLAS1OBJS) $(QBLAS1OBJS) $(CBLAS1OBJS) $(ZBLAS1OBJS) $(XBLAS1OBJS)
|
||||
level1 : $(BEXTOBJS) $(SHBLAS1OBJS) $(SBLAS1OBJS) $(DBLAS1OBJS) $(QBLAS1OBJS) $(CBLAS1OBJS) $(ZBLAS1OBJS) $(XBLAS1OBJS)
|
||||
$(AR) $(ARFLAGS) -ru $(TOPDIR)/$(LIBNAME) $^
|
||||
|
||||
level2 : $(SBLAS2OBJS) $(DBLAS2OBJS) $(QBLAS2OBJS) $(CBLAS2OBJS) $(ZBLAS2OBJS) $(XBLAS2OBJS)
|
||||
@@ -725,6 +731,19 @@ sdsdot.$(SUFFIX) sdsdot.$(PSUFFIX) : sdsdot.c
|
||||
dsdot.$(SUFFIX) dsdot.$(PSUFFIX) : dsdot.c
|
||||
$(CC) $(CFLAGS) -c $< -o $(@F)
|
||||
|
||||
ifeq ($(BUILD_HALF),1)
|
||||
shdot.$(SUFFIX) shdot.$(PSUFFIX) : bf16dot.c
|
||||
$(CC) $(CFLAGS) -c $< -o $(@F)
|
||||
shstobf16.$(SUFFIX) shstobf16.$(PSUFFIX) : tobf16.c
|
||||
$(CC) $(CFLAGS) -DSINGLE_PREC -UDOUBLE_PREC -c $< -o $(@F)
|
||||
shdtobf16.$(SUFFIX) shdtobf16.$(PSUFFIX) : tobf16.c
|
||||
$(CC) $(CFLAGS) -USINGLE_PREC -DDOUBLE_PREC -c $< -o $(@F)
|
||||
sbf16tos.$(SUFFIX) sbf16tos.$(PSUFFIX) : bf16to.c
|
||||
$(CC) $(CFLAGS) -DSINGLE_PREC -UDOUBLE_PREC -c $< -o $(@F)
|
||||
dbf16tod.$(SUFFIX) dbf16tod.$(PSUFFIX) : bf16to.c
|
||||
$(CC) $(CFLAGS) -USINGLE_PREC -DDOUBLE_PREC -c $< -o $(@F)
|
||||
endif
|
||||
|
||||
sdot.$(SUFFIX) sdot.$(PSUFFIX) : dot.c
|
||||
$(CC) $(CFLAGS) -c $< -o $(@F)
|
||||
|
||||
@@ -1463,6 +1482,19 @@ cblas_sdsdot.$(SUFFIX) cblas_sdsdot.$(PSUFFIX) : sdsdot.c
|
||||
cblas_dsdot.$(SUFFIX) cblas_dsdot.$(PSUFFIX) : dsdot.c
|
||||
$(CC) $(CFLAGS) -DCBLAS -c $< -o $(@F)
|
||||
|
||||
ifeq ($(BUILD_HALF),1)
|
||||
cblas_shdot.$(SUFFIX) cblas_shdot.$(PSUFFIX) : bf16dot.c
|
||||
$(CC) $(CFLAGS) -DCBLAS -c $< -o $(@F)
|
||||
cblas_shstobf16.$(SUFFIX) cblas_shstobf16.$(PSUFFIX) : tobf16.c
|
||||
$(CC) $(CFLAGS) -DCBLAS -DSINGLE_PREC -UDOUBLE_PREC -c $< -o $(@F)
|
||||
cblas_shdtobf16.$(SUFFIX) cblas_shdtobf16.$(PSUFFIX) : tobf16.c
|
||||
$(CC) $(CFLAGS) -DCBLAS -USINGLE_PREC -DDOUBLE_PREC -c $< -o $(@F)
|
||||
cblas_sbf16tos.$(SUFFIX) cblas_sbf16tos.$(PSUFFIX) : bf16to.c
|
||||
$(CC) $(CFLAGS) -DCBLAS -DSINGLE_PREC -UDOUBLE_PREC -c $< -o $(@F)
|
||||
cblas_dbf16tod.$(SUFFIX) cblas_dbf16tod.$(PSUFFIX) : bf16to.c
|
||||
$(CC) $(CFLAGS) -DCBLAS -USINGLE_PREC -DDOUBLE_PREC -c $< -o $(@F)
|
||||
endif
|
||||
|
||||
cblas_sdot.$(SUFFIX) cblas_sdot.$(PSUFFIX) : dot.c
|
||||
$(CC) $(CFLAGS) -DCBLAS -c $< -o $(@F)
|
||||
|
||||
|
||||
52
interface/bf16dot.c
Normal file
52
interface/bf16dot.c
Normal file
@@ -0,0 +1,52 @@
|
||||
#include <stdio.h>
|
||||
#include "common.h"
|
||||
#ifdef FUNCTION_PROFILE
|
||||
#include "functable.h"
|
||||
#endif
|
||||
|
||||
#ifndef CBLAS
|
||||
float NAME(blasint *N, bfloat16 *x, blasint *INCX, bfloat16 *y, blasint *INCY){
|
||||
BLASLONG n = *N;
|
||||
BLASLONG incx = *INCX;
|
||||
BLASLONG incy = *INCY;
|
||||
float ret;
|
||||
PRINT_DEBUG_NAME;
|
||||
|
||||
if (n <= 0) return 0.;
|
||||
|
||||
IDEBUG_START;
|
||||
FUNCTION_PROFILE_START();
|
||||
|
||||
if (incx < 0) x -= (n - 1) * incx;
|
||||
if (incy < 0) y -= (n - 1) * incy;
|
||||
ret = BF16_DOT_K(n, x, incx, y, incy);
|
||||
|
||||
FUNCTION_PROFILE_END(1, 2 * n, 2 * n);
|
||||
IDEBUG_END;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
float CNAME(blasint n, bfloat16 *x, blasint incx, bfloat16 *y, blasint incy){
|
||||
|
||||
float ret;
|
||||
PRINT_DEBUG_CNAME;
|
||||
|
||||
if (n <= 0) return 0.;
|
||||
|
||||
IDEBUG_START;
|
||||
FUNCTION_PROFILE_START();
|
||||
|
||||
if (incx < 0) x -= (n - 1) * incx;
|
||||
if (incy < 0) y -= (n - 1) * incy;
|
||||
ret = BF16_DOT_K(n, x, incx, y, incy);
|
||||
|
||||
FUNCTION_PROFILE_END(1, 2 * n, 2 * n);
|
||||
IDEBUG_END;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
#endif
|
||||
62
interface/bf16to.c
Normal file
62
interface/bf16to.c
Normal file
@@ -0,0 +1,62 @@
|
||||
#include <stdio.h>
|
||||
#include "common.h"
|
||||
#ifdef FUNCTION_PROFILE
|
||||
#include "functable.h"
|
||||
#endif
|
||||
|
||||
#if defined(DOUBLE_PREC)
|
||||
#define FLOAT_TYPE double
|
||||
#elif defined(SINGLE_PREC)
|
||||
#define FLOAT_TYPE float
|
||||
#else
|
||||
#endif
|
||||
|
||||
#ifndef CBLAS
|
||||
void NAME(blasint *N, bfloat16 *in, blasint *INC_IN, FLOAT_TYPE *out, blasint *INC_OUT){
|
||||
BLASLONG n = *N;
|
||||
BLASLONG inc_in = *INC_IN;
|
||||
BLASLONG inc_out = *INC_OUT;
|
||||
|
||||
PRINT_DEBUG_NAME;
|
||||
|
||||
if (n <= 0) return;
|
||||
|
||||
IDEBUG_START;
|
||||
FUNCTION_PROFILE_START();
|
||||
|
||||
if (inc_in < 0) in -= (n - 1) * inc_in;
|
||||
if (inc_out < 0) out -= (n - 1) * inc_out;
|
||||
|
||||
#if defined(DOUBLE_PREC)
|
||||
D_BF16_TO_K(n, in, inc_in, out, inc_out);
|
||||
#elif defined(SINGLE_PREC)
|
||||
S_BF16_TO_K(n, in, inc_in, out, inc_out);
|
||||
#else
|
||||
#endif
|
||||
|
||||
FUNCTION_PROFILE_END(1, 2 * n, 2 * n);
|
||||
IDEBUG_END;
|
||||
}
|
||||
#else
|
||||
void CNAME(blasint n, bfloat16 * in, blasint inc_in, FLOAT_TYPE * out, blasint inc_out){
|
||||
PRINT_DEBUG_CNAME;
|
||||
|
||||
if (n <= 0) return;
|
||||
|
||||
IDEBUG_START;
|
||||
FUNCTION_PROFILE_START();
|
||||
|
||||
if (inc_in < 0) in -= (n - 1) * inc_in;
|
||||
if (inc_out < 0) out -= (n - 1) * inc_out;
|
||||
|
||||
#if defined(DOUBLE_PREC)
|
||||
D_BF16_TO_K(n, in, inc_in, out, inc_out);
|
||||
#elif defined(SINGLE_PREC)
|
||||
S_BF16_TO_K(n, in, inc_in, out, inc_out);
|
||||
#else
|
||||
#endif
|
||||
|
||||
FUNCTION_PROFILE_END(1, 2 * n, 2 * n);
|
||||
IDEBUG_END;
|
||||
}
|
||||
#endif
|
||||
61
interface/tobf16.c
Normal file
61
interface/tobf16.c
Normal file
@@ -0,0 +1,61 @@
|
||||
#include <stdio.h>
|
||||
#include "common.h"
|
||||
#ifdef FUNCTION_PROFILE
|
||||
#include "functable.h"
|
||||
#endif
|
||||
|
||||
#if defined(DOUBLE_PREC)
|
||||
#define FLOAT_TYPE double
|
||||
#elif defined(SINGLE_PREC)
|
||||
#define FLOAT_TYPE float
|
||||
#else
|
||||
#endif
|
||||
|
||||
#ifndef CBLAS
|
||||
void NAME(blasint *N, FLOAT_TYPE *in, blasint *INC_IN, bfloat16 *out, blasint *INC_OUT){
|
||||
BLASLONG n = *N;
|
||||
BLASLONG inc_in = *INC_IN;
|
||||
BLASLONG inc_out = *INC_OUT;
|
||||
|
||||
PRINT_DEBUG_NAME;
|
||||
|
||||
if (n <= 0) return;
|
||||
|
||||
IDEBUG_START;
|
||||
FUNCTION_PROFILE_START();
|
||||
|
||||
if (inc_in < 0) in -= (n - 1) * inc_in;
|
||||
if (inc_out < 0) out -= (n - 1) * inc_out;
|
||||
|
||||
#if defined(DOUBLE_PREC)
|
||||
D_TO_BF16_K(n, in, inc_in, out, inc_out);
|
||||
#elif defined(SINGLE_PREC)
|
||||
S_TO_BF16_K(n, in, inc_in, out, inc_out);
|
||||
#else
|
||||
#endif
|
||||
|
||||
FUNCTION_PROFILE_END(1, 2 * n, 2 * n);
|
||||
IDEBUG_END;
|
||||
}
|
||||
#else
|
||||
void CNAME(blasint n, FLOAT_TYPE *in, blasint inc_in, bfloat16 *out, blasint inc_out){
|
||||
PRINT_DEBUG_CNAME;
|
||||
|
||||
if (n <= 0) return;
|
||||
|
||||
IDEBUG_START;
|
||||
FUNCTION_PROFILE_START();
|
||||
|
||||
if (inc_in < 0) in -= (n - 1) * inc_in;
|
||||
if (inc_out < 0) out -= (n - 1) * inc_out;
|
||||
|
||||
#if defined(DOUBLE_PREC)
|
||||
D_TO_BF16_K(n, in, inc_in, out, inc_out);
|
||||
#elif defined(SINGLE_PREC)
|
||||
S_TO_BF16_K(n, in, inc_in, out, inc_out);
|
||||
#endif
|
||||
|
||||
FUNCTION_PROFILE_END(1, 2 * n, 2 * n);
|
||||
IDEBUG_END;
|
||||
}
|
||||
#endif
|
||||
Reference in New Issue
Block a user