846 lines
27 KiB
C
846 lines
27 KiB
C
/*********************************************************************/
|
|
/* Copyright 2009, 2010 The University of Texas at Austin. */
|
|
/* All rights reserved. */
|
|
/* */
|
|
/* Redistribution and use in source and binary forms, with or */
|
|
/* without modification, are permitted provided that the following */
|
|
/* conditions are met: */
|
|
/* */
|
|
/* 1. Redistributions of source code must retain the above */
|
|
/* copyright notice, this list of conditions and the following */
|
|
/* disclaimer. */
|
|
/* */
|
|
/* 2. Redistributions in binary form must reproduce the above */
|
|
/* copyright notice, this list of conditions and the following */
|
|
/* disclaimer in the documentation and/or other materials */
|
|
/* provided with the distribution. */
|
|
/* */
|
|
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
|
|
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
|
|
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
|
|
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
|
|
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
|
|
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
|
|
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
|
|
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
|
|
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
|
|
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
|
|
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
|
|
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
|
|
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
|
|
/* POSSIBILITY OF SUCH DAMAGE. */
|
|
/* */
|
|
/* The views and conclusions contained in the software and */
|
|
/* documentation are those of the authors and should not be */
|
|
/* interpreted as representing official policies, either expressed */
|
|
/* or implied, of The University of Texas at Austin. */
|
|
/*********************************************************************/
|
|
|
|
#include <stdio.h>
|
|
#include <string.h>
|
|
#include "common.h"
|
|
|
|
#ifdef BUILD_KERNEL
|
|
#include "kernelTS.h"
|
|
#endif
|
|
|
|
#undef DEBUG
|
|
|
|
static void init_parameter(void);
|
|
|
|
gotoblas_t TABLE_NAME = {
|
|
DTB_DEFAULT_ENTRIES ,
|
|
|
|
GEMM_DEFAULT_OFFSET_A, GEMM_DEFAULT_OFFSET_B, GEMM_DEFAULT_ALIGN,
|
|
|
|
0, 0, 0,
|
|
SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N, MAX(SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N),
|
|
#ifdef HAVE_EXCLUSIVE_CACHE
|
|
1,
|
|
#else
|
|
0,
|
|
#endif
|
|
|
|
samax_kTS, samin_kTS, smax_kTS, smin_kTS,
|
|
isamax_kTS, isamin_kTS, ismax_kTS, ismin_kTS,
|
|
snrm2_kTS, sasum_kTS, scopy_kTS, sdot_kTS,
|
|
dsdot_kTS,
|
|
srot_kTS, saxpy_kTS, sscal_kTS, sswap_kTS,
|
|
sgemv_nTS, sgemv_tTS, sger_kTS,
|
|
ssymv_LTS, ssymv_UTS,
|
|
|
|
sgemm_kernelTS, sgemm_betaTS,
|
|
#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N
|
|
sgemm_incopyTS, sgemm_itcopyTS,
|
|
#else
|
|
sgemm_oncopyTS, sgemm_otcopyTS,
|
|
#endif
|
|
sgemm_oncopyTS, sgemm_otcopyTS,
|
|
strsm_kernel_LNTS, strsm_kernel_LTTS, strsm_kernel_RNTS, strsm_kernel_RTTS,
|
|
#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N
|
|
strsm_iunucopyTS, strsm_iunncopyTS, strsm_iutucopyTS, strsm_iutncopyTS,
|
|
strsm_ilnucopyTS, strsm_ilnncopyTS, strsm_iltucopyTS, strsm_iltncopyTS,
|
|
#else
|
|
strsm_ounucopyTS, strsm_ounncopyTS, strsm_outucopyTS, strsm_outncopyTS,
|
|
strsm_olnucopyTS, strsm_olnncopyTS, strsm_oltucopyTS, strsm_oltncopyTS,
|
|
#endif
|
|
strsm_ounucopyTS, strsm_ounncopyTS, strsm_outucopyTS, strsm_outncopyTS,
|
|
strsm_olnucopyTS, strsm_olnncopyTS, strsm_oltucopyTS, strsm_oltncopyTS,
|
|
strmm_kernel_RNTS, strmm_kernel_RTTS, strmm_kernel_LNTS, strmm_kernel_LTTS,
|
|
#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N
|
|
strmm_iunucopyTS, strmm_iunncopyTS, strmm_iutucopyTS, strmm_iutncopyTS,
|
|
strmm_ilnucopyTS, strmm_ilnncopyTS, strmm_iltucopyTS, strmm_iltncopyTS,
|
|
#else
|
|
strmm_ounucopyTS, strmm_ounncopyTS, strmm_outucopyTS, strmm_outncopyTS,
|
|
strmm_olnucopyTS, strmm_olnncopyTS, strmm_oltucopyTS, strmm_oltncopyTS,
|
|
#endif
|
|
strmm_ounucopyTS, strmm_ounncopyTS, strmm_outucopyTS, strmm_outncopyTS,
|
|
strmm_olnucopyTS, strmm_olnncopyTS, strmm_oltucopyTS, strmm_oltncopyTS,
|
|
#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N
|
|
ssymm_iutcopyTS, ssymm_iltcopyTS,
|
|
#else
|
|
ssymm_outcopyTS, ssymm_oltcopyTS,
|
|
#endif
|
|
ssymm_outcopyTS, ssymm_oltcopyTS,
|
|
|
|
#ifndef NO_LAPACK
|
|
sneg_tcopyTS, slaswp_ncopyTS,
|
|
#else
|
|
NULL,NULL,
|
|
#endif
|
|
|
|
0, 0, 0,
|
|
DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N, MAX(DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N),
|
|
|
|
damax_kTS, damin_kTS, dmax_kTS, dmin_kTS,
|
|
idamax_kTS, idamin_kTS, idmax_kTS, idmin_kTS,
|
|
dnrm2_kTS, dasum_kTS, dcopy_kTS, ddot_kTS,
|
|
drot_kTS, daxpy_kTS, dscal_kTS, dswap_kTS,
|
|
dgemv_nTS, dgemv_tTS, dger_kTS,
|
|
dsymv_LTS, dsymv_UTS,
|
|
|
|
dgemm_kernelTS, dgemm_betaTS,
|
|
#if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N
|
|
dgemm_incopyTS, dgemm_itcopyTS,
|
|
#else
|
|
dgemm_oncopyTS, dgemm_otcopyTS,
|
|
#endif
|
|
dgemm_oncopyTS, dgemm_otcopyTS,
|
|
dtrsm_kernel_LNTS, dtrsm_kernel_LTTS, dtrsm_kernel_RNTS, dtrsm_kernel_RTTS,
|
|
#if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N
|
|
dtrsm_iunucopyTS, dtrsm_iunncopyTS, dtrsm_iutucopyTS, dtrsm_iutncopyTS,
|
|
dtrsm_ilnucopyTS, dtrsm_ilnncopyTS, dtrsm_iltucopyTS, dtrsm_iltncopyTS,
|
|
#else
|
|
dtrsm_ounucopyTS, dtrsm_ounncopyTS, dtrsm_outucopyTS, dtrsm_outncopyTS,
|
|
dtrsm_olnucopyTS, dtrsm_olnncopyTS, dtrsm_oltucopyTS, dtrsm_oltncopyTS,
|
|
#endif
|
|
dtrsm_ounucopyTS, dtrsm_ounncopyTS, dtrsm_outucopyTS, dtrsm_outncopyTS,
|
|
dtrsm_olnucopyTS, dtrsm_olnncopyTS, dtrsm_oltucopyTS, dtrsm_oltncopyTS,
|
|
dtrmm_kernel_RNTS, dtrmm_kernel_RTTS, dtrmm_kernel_LNTS, dtrmm_kernel_LTTS,
|
|
#if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N
|
|
dtrmm_iunucopyTS, dtrmm_iunncopyTS, dtrmm_iutucopyTS, dtrmm_iutncopyTS,
|
|
dtrmm_ilnucopyTS, dtrmm_ilnncopyTS, dtrmm_iltucopyTS, dtrmm_iltncopyTS,
|
|
#else
|
|
dtrmm_ounucopyTS, dtrmm_ounncopyTS, dtrmm_outucopyTS, dtrmm_outncopyTS,
|
|
dtrmm_olnucopyTS, dtrmm_olnncopyTS, dtrmm_oltucopyTS, dtrmm_oltncopyTS,
|
|
#endif
|
|
dtrmm_ounucopyTS, dtrmm_ounncopyTS, dtrmm_outucopyTS, dtrmm_outncopyTS,
|
|
dtrmm_olnucopyTS, dtrmm_olnncopyTS, dtrmm_oltucopyTS, dtrmm_oltncopyTS,
|
|
#if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N
|
|
dsymm_iutcopyTS, dsymm_iltcopyTS,
|
|
#else
|
|
dsymm_outcopyTS, dsymm_oltcopyTS,
|
|
#endif
|
|
dsymm_outcopyTS, dsymm_oltcopyTS,
|
|
|
|
#ifndef NO_LAPACK
|
|
dneg_tcopyTS, dlaswp_ncopyTS,
|
|
#else
|
|
NULL, NULL,
|
|
#endif
|
|
|
|
#ifdef EXPRECISION
|
|
|
|
0, 0, 0,
|
|
QGEMM_DEFAULT_UNROLL_M, QGEMM_DEFAULT_UNROLL_N, MAX(QGEMM_DEFAULT_UNROLL_M, QGEMM_DEFAULT_UNROLL_N),
|
|
|
|
qamax_kTS, qamin_kTS, qmax_kTS, qmin_kTS,
|
|
iqamax_kTS, iqamin_kTS, iqmax_kTS, iqmin_kTS,
|
|
qnrm2_kTS, qasum_kTS, qcopy_kTS, qdot_kTS,
|
|
qrot_kTS, qaxpy_kTS, qscal_kTS, qswap_kTS,
|
|
qgemv_nTS, qgemv_tTS, qger_kTS,
|
|
qsymv_LTS, qsymv_UTS,
|
|
|
|
qgemm_kernelTS, qgemm_betaTS,
|
|
#if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N
|
|
qgemm_incopyTS, qgemm_itcopyTS,
|
|
#else
|
|
qgemm_oncopyTS, qgemm_otcopyTS,
|
|
#endif
|
|
qgemm_oncopyTS, qgemm_otcopyTS,
|
|
qtrsm_kernel_LNTS, qtrsm_kernel_LTTS, qtrsm_kernel_RNTS, qtrsm_kernel_RTTS,
|
|
#if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N
|
|
qtrsm_iunucopyTS, qtrsm_iunncopyTS, qtrsm_iutucopyTS, qtrsm_iutncopyTS,
|
|
qtrsm_ilnucopyTS, qtrsm_ilnncopyTS, qtrsm_iltucopyTS, qtrsm_iltncopyTS,
|
|
#else
|
|
qtrsm_ounucopyTS, qtrsm_ounncopyTS, qtrsm_outucopyTS, qtrsm_outncopyTS,
|
|
qtrsm_olnucopyTS, qtrsm_olnncopyTS, qtrsm_oltucopyTS, qtrsm_oltncopyTS,
|
|
#endif
|
|
qtrsm_ounucopyTS, qtrsm_ounncopyTS, qtrsm_outucopyTS, qtrsm_outncopyTS,
|
|
qtrsm_olnucopyTS, qtrsm_olnncopyTS, qtrsm_oltucopyTS, qtrsm_oltncopyTS,
|
|
qtrmm_kernel_RNTS, qtrmm_kernel_RTTS, qtrmm_kernel_LNTS, qtrmm_kernel_LTTS,
|
|
#if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N
|
|
qtrmm_iunucopyTS, qtrmm_iunncopyTS, qtrmm_iutucopyTS, qtrmm_iutncopyTS,
|
|
qtrmm_ilnucopyTS, qtrmm_ilnncopyTS, qtrmm_iltucopyTS, qtrmm_iltncopyTS,
|
|
#else
|
|
qtrmm_ounucopyTS, qtrmm_ounncopyTS, qtrmm_outucopyTS, qtrmm_outncopyTS,
|
|
qtrmm_olnucopyTS, qtrmm_olnncopyTS, qtrmm_oltucopyTS, qtrmm_oltncopyTS,
|
|
#endif
|
|
qtrmm_ounucopyTS, qtrmm_ounncopyTS, qtrmm_outucopyTS, qtrmm_outncopyTS,
|
|
qtrmm_olnucopyTS, qtrmm_olnncopyTS, qtrmm_oltucopyTS, qtrmm_oltncopyTS,
|
|
#if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N
|
|
qsymm_iutcopyTS, qsymm_iltcopyTS,
|
|
#else
|
|
qsymm_outcopyTS, qsymm_oltcopyTS,
|
|
#endif
|
|
qsymm_outcopyTS, qsymm_oltcopyTS,
|
|
|
|
#ifndef NO_LAPACK
|
|
qneg_tcopyTS, qlaswp_ncopyTS,
|
|
#else
|
|
NULL, NULL,
|
|
#endif
|
|
|
|
#endif
|
|
|
|
0, 0, 0,
|
|
CGEMM_DEFAULT_UNROLL_M, CGEMM_DEFAULT_UNROLL_N, MAX(CGEMM_DEFAULT_UNROLL_M, CGEMM_DEFAULT_UNROLL_N),
|
|
|
|
camax_kTS, camin_kTS, icamax_kTS, icamin_kTS,
|
|
cnrm2_kTS, casum_kTS, ccopy_kTS,
|
|
cdotu_kTS, cdotc_kTS, csrot_kTS,
|
|
caxpy_kTS, caxpyc_kTS, cscal_kTS, cswap_kTS,
|
|
|
|
cgemv_nTS, cgemv_tTS, cgemv_rTS, cgemv_cTS,
|
|
cgemv_oTS, cgemv_uTS, cgemv_sTS, cgemv_dTS,
|
|
cgeru_kTS, cgerc_kTS, cgerv_kTS, cgerd_kTS,
|
|
csymv_LTS, csymv_UTS,
|
|
chemv_LTS, chemv_UTS, chemv_MTS, chemv_VTS,
|
|
|
|
cgemm_kernel_nTS, cgemm_kernel_lTS, cgemm_kernel_rTS, cgemm_kernel_bTS,
|
|
cgemm_betaTS,
|
|
|
|
#if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N
|
|
cgemm_incopyTS, cgemm_itcopyTS,
|
|
#else
|
|
cgemm_oncopyTS, cgemm_otcopyTS,
|
|
#endif
|
|
cgemm_oncopyTS, cgemm_otcopyTS,
|
|
|
|
ctrsm_kernel_LNTS, ctrsm_kernel_LTTS, ctrsm_kernel_LRTS, ctrsm_kernel_LCTS,
|
|
ctrsm_kernel_RNTS, ctrsm_kernel_RTTS, ctrsm_kernel_RRTS, ctrsm_kernel_RCTS,
|
|
|
|
#if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N
|
|
ctrsm_iunucopyTS, ctrsm_iunncopyTS, ctrsm_iutucopyTS, ctrsm_iutncopyTS,
|
|
ctrsm_ilnucopyTS, ctrsm_ilnncopyTS, ctrsm_iltucopyTS, ctrsm_iltncopyTS,
|
|
#else
|
|
ctrsm_ounucopyTS, ctrsm_ounncopyTS, ctrsm_outucopyTS, ctrsm_outncopyTS,
|
|
ctrsm_olnucopyTS, ctrsm_olnncopyTS, ctrsm_oltucopyTS, ctrsm_oltncopyTS,
|
|
#endif
|
|
ctrsm_ounucopyTS, ctrsm_ounncopyTS, ctrsm_outucopyTS, ctrsm_outncopyTS,
|
|
ctrsm_olnucopyTS, ctrsm_olnncopyTS, ctrsm_oltucopyTS, ctrsm_oltncopyTS,
|
|
|
|
ctrmm_kernel_RNTS, ctrmm_kernel_RTTS, ctrmm_kernel_RRTS, ctrmm_kernel_RCTS,
|
|
ctrmm_kernel_LNTS, ctrmm_kernel_LTTS, ctrmm_kernel_LRTS, ctrmm_kernel_LCTS,
|
|
|
|
#if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N
|
|
ctrmm_iunucopyTS, ctrmm_iunncopyTS, ctrmm_iutucopyTS, ctrmm_iutncopyTS,
|
|
ctrmm_ilnucopyTS, ctrmm_ilnncopyTS, ctrmm_iltucopyTS, ctrmm_iltncopyTS,
|
|
#else
|
|
ctrmm_ounucopyTS, ctrmm_ounncopyTS, ctrmm_outucopyTS, ctrmm_outncopyTS,
|
|
ctrmm_olnucopyTS, ctrmm_olnncopyTS, ctrmm_oltucopyTS, ctrmm_oltncopyTS,
|
|
#endif
|
|
ctrmm_ounucopyTS, ctrmm_ounncopyTS, ctrmm_outucopyTS, ctrmm_outncopyTS,
|
|
ctrmm_olnucopyTS, ctrmm_olnncopyTS, ctrmm_oltucopyTS, ctrmm_oltncopyTS,
|
|
|
|
#if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N
|
|
csymm_iutcopyTS, csymm_iltcopyTS,
|
|
#else
|
|
csymm_outcopyTS, csymm_oltcopyTS,
|
|
#endif
|
|
csymm_outcopyTS, csymm_oltcopyTS,
|
|
#if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N
|
|
chemm_iutcopyTS, chemm_iltcopyTS,
|
|
#else
|
|
chemm_outcopyTS, chemm_oltcopyTS,
|
|
#endif
|
|
chemm_outcopyTS, chemm_oltcopyTS,
|
|
|
|
cgemm3m_kernelTS,
|
|
|
|
cgemm3m_incopybTS, cgemm3m_incopyrTS,
|
|
cgemm3m_incopyiTS, cgemm3m_itcopybTS,
|
|
cgemm3m_itcopyrTS, cgemm3m_itcopyiTS,
|
|
cgemm3m_oncopybTS, cgemm3m_oncopyrTS,
|
|
cgemm3m_oncopyiTS, cgemm3m_otcopybTS,
|
|
cgemm3m_otcopyrTS, cgemm3m_otcopyiTS,
|
|
|
|
csymm3m_iucopybTS, csymm3m_ilcopybTS,
|
|
csymm3m_iucopyrTS, csymm3m_ilcopyrTS,
|
|
csymm3m_iucopyiTS, csymm3m_ilcopyiTS,
|
|
csymm3m_oucopybTS, csymm3m_olcopybTS,
|
|
csymm3m_oucopyrTS, csymm3m_olcopyrTS,
|
|
csymm3m_oucopyiTS, csymm3m_olcopyiTS,
|
|
|
|
chemm3m_iucopybTS, chemm3m_ilcopybTS,
|
|
chemm3m_iucopyrTS, chemm3m_ilcopyrTS,
|
|
chemm3m_iucopyiTS, chemm3m_ilcopyiTS,
|
|
|
|
chemm3m_oucopybTS, chemm3m_olcopybTS,
|
|
chemm3m_oucopyrTS, chemm3m_olcopyrTS,
|
|
chemm3m_oucopyiTS, chemm3m_olcopyiTS,
|
|
|
|
#ifndef NO_LAPACK
|
|
cneg_tcopyTS, claswp_ncopyTS,
|
|
#else
|
|
NULL, NULL,
|
|
#endif
|
|
|
|
0, 0, 0,
|
|
ZGEMM_DEFAULT_UNROLL_M, ZGEMM_DEFAULT_UNROLL_N, MAX(ZGEMM_DEFAULT_UNROLL_M, ZGEMM_DEFAULT_UNROLL_N),
|
|
|
|
zamax_kTS, zamin_kTS, izamax_kTS, izamin_kTS,
|
|
znrm2_kTS, zasum_kTS, zcopy_kTS,
|
|
zdotu_kTS, zdotc_kTS, zdrot_kTS,
|
|
zaxpy_kTS, zaxpyc_kTS, zscal_kTS, zswap_kTS,
|
|
|
|
zgemv_nTS, zgemv_tTS, zgemv_rTS, zgemv_cTS,
|
|
zgemv_oTS, zgemv_uTS, zgemv_sTS, zgemv_dTS,
|
|
zgeru_kTS, zgerc_kTS, zgerv_kTS, zgerd_kTS,
|
|
zsymv_LTS, zsymv_UTS,
|
|
zhemv_LTS, zhemv_UTS, zhemv_MTS, zhemv_VTS,
|
|
|
|
zgemm_kernel_nTS, zgemm_kernel_lTS, zgemm_kernel_rTS, zgemm_kernel_bTS,
|
|
zgemm_betaTS,
|
|
|
|
#if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N
|
|
zgemm_incopyTS, zgemm_itcopyTS,
|
|
#else
|
|
zgemm_oncopyTS, zgemm_otcopyTS,
|
|
#endif
|
|
zgemm_oncopyTS, zgemm_otcopyTS,
|
|
|
|
ztrsm_kernel_LNTS, ztrsm_kernel_LTTS, ztrsm_kernel_LRTS, ztrsm_kernel_LCTS,
|
|
ztrsm_kernel_RNTS, ztrsm_kernel_RTTS, ztrsm_kernel_RRTS, ztrsm_kernel_RCTS,
|
|
|
|
#if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N
|
|
ztrsm_iunucopyTS, ztrsm_iunncopyTS, ztrsm_iutucopyTS, ztrsm_iutncopyTS,
|
|
ztrsm_ilnucopyTS, ztrsm_ilnncopyTS, ztrsm_iltucopyTS, ztrsm_iltncopyTS,
|
|
#else
|
|
ztrsm_ounucopyTS, ztrsm_ounncopyTS, ztrsm_outucopyTS, ztrsm_outncopyTS,
|
|
ztrsm_olnucopyTS, ztrsm_olnncopyTS, ztrsm_oltucopyTS, ztrsm_oltncopyTS,
|
|
#endif
|
|
ztrsm_ounucopyTS, ztrsm_ounncopyTS, ztrsm_outucopyTS, ztrsm_outncopyTS,
|
|
ztrsm_olnucopyTS, ztrsm_olnncopyTS, ztrsm_oltucopyTS, ztrsm_oltncopyTS,
|
|
|
|
ztrmm_kernel_RNTS, ztrmm_kernel_RTTS, ztrmm_kernel_RRTS, ztrmm_kernel_RCTS,
|
|
ztrmm_kernel_LNTS, ztrmm_kernel_LTTS, ztrmm_kernel_LRTS, ztrmm_kernel_LCTS,
|
|
|
|
#if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N
|
|
ztrmm_iunucopyTS, ztrmm_iunncopyTS, ztrmm_iutucopyTS, ztrmm_iutncopyTS,
|
|
ztrmm_ilnucopyTS, ztrmm_ilnncopyTS, ztrmm_iltucopyTS, ztrmm_iltncopyTS,
|
|
#else
|
|
ztrmm_ounucopyTS, ztrmm_ounncopyTS, ztrmm_outucopyTS, ztrmm_outncopyTS,
|
|
ztrmm_olnucopyTS, ztrmm_olnncopyTS, ztrmm_oltucopyTS, ztrmm_oltncopyTS,
|
|
#endif
|
|
ztrmm_ounucopyTS, ztrmm_ounncopyTS, ztrmm_outucopyTS, ztrmm_outncopyTS,
|
|
ztrmm_olnucopyTS, ztrmm_olnncopyTS, ztrmm_oltucopyTS, ztrmm_oltncopyTS,
|
|
|
|
#if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N
|
|
zsymm_iutcopyTS, zsymm_iltcopyTS,
|
|
#else
|
|
zsymm_outcopyTS, zsymm_oltcopyTS,
|
|
#endif
|
|
zsymm_outcopyTS, zsymm_oltcopyTS,
|
|
#if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N
|
|
zhemm_iutcopyTS, zhemm_iltcopyTS,
|
|
#else
|
|
zhemm_outcopyTS, zhemm_oltcopyTS,
|
|
#endif
|
|
zhemm_outcopyTS, zhemm_oltcopyTS,
|
|
|
|
zgemm3m_kernelTS,
|
|
|
|
zgemm3m_incopybTS, zgemm3m_incopyrTS,
|
|
zgemm3m_incopyiTS, zgemm3m_itcopybTS,
|
|
zgemm3m_itcopyrTS, zgemm3m_itcopyiTS,
|
|
zgemm3m_oncopybTS, zgemm3m_oncopyrTS,
|
|
zgemm3m_oncopyiTS, zgemm3m_otcopybTS,
|
|
zgemm3m_otcopyrTS, zgemm3m_otcopyiTS,
|
|
|
|
zsymm3m_iucopybTS, zsymm3m_ilcopybTS,
|
|
zsymm3m_iucopyrTS, zsymm3m_ilcopyrTS,
|
|
zsymm3m_iucopyiTS, zsymm3m_ilcopyiTS,
|
|
zsymm3m_oucopybTS, zsymm3m_olcopybTS,
|
|
zsymm3m_oucopyrTS, zsymm3m_olcopyrTS,
|
|
zsymm3m_oucopyiTS, zsymm3m_olcopyiTS,
|
|
|
|
zhemm3m_iucopybTS, zhemm3m_ilcopybTS,
|
|
zhemm3m_iucopyrTS, zhemm3m_ilcopyrTS,
|
|
zhemm3m_iucopyiTS, zhemm3m_ilcopyiTS,
|
|
|
|
zhemm3m_oucopybTS, zhemm3m_olcopybTS,
|
|
zhemm3m_oucopyrTS, zhemm3m_olcopyrTS,
|
|
zhemm3m_oucopyiTS, zhemm3m_olcopyiTS,
|
|
|
|
#ifndef NO_LAPACK
|
|
zneg_tcopyTS, zlaswp_ncopyTS,
|
|
#else
|
|
NULL, NULL,
|
|
#endif
|
|
|
|
#ifdef EXPRECISION
|
|
|
|
0, 0, 0,
|
|
XGEMM_DEFAULT_UNROLL_M, XGEMM_DEFAULT_UNROLL_N, MAX(XGEMM_DEFAULT_UNROLL_M, XGEMM_DEFAULT_UNROLL_N),
|
|
|
|
xamax_kTS, xamin_kTS, ixamax_kTS, ixamin_kTS,
|
|
xnrm2_kTS, xasum_kTS, xcopy_kTS,
|
|
xdotu_kTS, xdotc_kTS, xqrot_kTS,
|
|
xaxpy_kTS, xaxpyc_kTS, xscal_kTS, xswap_kTS,
|
|
|
|
xgemv_nTS, xgemv_tTS, xgemv_rTS, xgemv_cTS,
|
|
xgemv_oTS, xgemv_uTS, xgemv_sTS, xgemv_dTS,
|
|
xgeru_kTS, xgerc_kTS, xgerv_kTS, xgerd_kTS,
|
|
xsymv_LTS, xsymv_UTS,
|
|
xhemv_LTS, xhemv_UTS, xhemv_MTS, xhemv_VTS,
|
|
|
|
xgemm_kernel_nTS, xgemm_kernel_lTS, xgemm_kernel_rTS, xgemm_kernel_bTS,
|
|
xgemm_betaTS,
|
|
|
|
#if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N
|
|
xgemm_incopyTS, xgemm_itcopyTS,
|
|
#else
|
|
xgemm_oncopyTS, xgemm_otcopyTS,
|
|
#endif
|
|
xgemm_oncopyTS, xgemm_otcopyTS,
|
|
|
|
xtrsm_kernel_LNTS, xtrsm_kernel_LTTS, xtrsm_kernel_LRTS, xtrsm_kernel_LCTS,
|
|
xtrsm_kernel_RNTS, xtrsm_kernel_RTTS, xtrsm_kernel_RRTS, xtrsm_kernel_RCTS,
|
|
|
|
#if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N
|
|
xtrsm_iunucopyTS, xtrsm_iunncopyTS, xtrsm_iutucopyTS, xtrsm_iutncopyTS,
|
|
xtrsm_ilnucopyTS, xtrsm_ilnncopyTS, xtrsm_iltucopyTS, xtrsm_iltncopyTS,
|
|
#else
|
|
xtrsm_ounucopyTS, xtrsm_ounncopyTS, xtrsm_outucopyTS, xtrsm_outncopyTS,
|
|
xtrsm_olnucopyTS, xtrsm_olnncopyTS, xtrsm_oltucopyTS, xtrsm_oltncopyTS,
|
|
#endif
|
|
xtrsm_ounucopyTS, xtrsm_ounncopyTS, xtrsm_outucopyTS, xtrsm_outncopyTS,
|
|
xtrsm_olnucopyTS, xtrsm_olnncopyTS, xtrsm_oltucopyTS, xtrsm_oltncopyTS,
|
|
|
|
xtrmm_kernel_RNTS, xtrmm_kernel_RTTS, xtrmm_kernel_RRTS, xtrmm_kernel_RCTS,
|
|
xtrmm_kernel_LNTS, xtrmm_kernel_LTTS, xtrmm_kernel_LRTS, xtrmm_kernel_LCTS,
|
|
|
|
#if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N
|
|
xtrmm_iunucopyTS, xtrmm_iunncopyTS, xtrmm_iutucopyTS, xtrmm_iutncopyTS,
|
|
xtrmm_ilnucopyTS, xtrmm_ilnncopyTS, xtrmm_iltucopyTS, xtrmm_iltncopyTS,
|
|
#else
|
|
xtrmm_ounucopyTS, xtrmm_ounncopyTS, xtrmm_outucopyTS, xtrmm_outncopyTS,
|
|
xtrmm_olnucopyTS, xtrmm_olnncopyTS, xtrmm_oltucopyTS, xtrmm_oltncopyTS,
|
|
#endif
|
|
xtrmm_ounucopyTS, xtrmm_ounncopyTS, xtrmm_outucopyTS, xtrmm_outncopyTS,
|
|
xtrmm_olnucopyTS, xtrmm_olnncopyTS, xtrmm_oltucopyTS, xtrmm_oltncopyTS,
|
|
|
|
#if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N
|
|
xsymm_iutcopyTS, xsymm_iltcopyTS,
|
|
#else
|
|
xsymm_outcopyTS, xsymm_oltcopyTS,
|
|
#endif
|
|
xsymm_outcopyTS, xsymm_oltcopyTS,
|
|
#if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N
|
|
xhemm_iutcopyTS, xhemm_iltcopyTS,
|
|
#else
|
|
xhemm_outcopyTS, xhemm_oltcopyTS,
|
|
#endif
|
|
xhemm_outcopyTS, xhemm_oltcopyTS,
|
|
|
|
xgemm3m_kernelTS,
|
|
|
|
xgemm3m_incopybTS, xgemm3m_incopyrTS,
|
|
xgemm3m_incopyiTS, xgemm3m_itcopybTS,
|
|
xgemm3m_itcopyrTS, xgemm3m_itcopyiTS,
|
|
xgemm3m_oncopybTS, xgemm3m_oncopyrTS,
|
|
xgemm3m_oncopyiTS, xgemm3m_otcopybTS,
|
|
xgemm3m_otcopyrTS, xgemm3m_otcopyiTS,
|
|
|
|
xsymm3m_iucopybTS, xsymm3m_ilcopybTS,
|
|
xsymm3m_iucopyrTS, xsymm3m_ilcopyrTS,
|
|
xsymm3m_iucopyiTS, xsymm3m_ilcopyiTS,
|
|
xsymm3m_oucopybTS, xsymm3m_olcopybTS,
|
|
xsymm3m_oucopyrTS, xsymm3m_olcopyrTS,
|
|
xsymm3m_oucopyiTS, xsymm3m_olcopyiTS,
|
|
|
|
xhemm3m_iucopybTS, xhemm3m_ilcopybTS,
|
|
xhemm3m_iucopyrTS, xhemm3m_ilcopyrTS,
|
|
xhemm3m_iucopyiTS, xhemm3m_ilcopyiTS,
|
|
|
|
xhemm3m_oucopybTS, xhemm3m_olcopybTS,
|
|
xhemm3m_oucopyrTS, xhemm3m_olcopyrTS,
|
|
xhemm3m_oucopyiTS, xhemm3m_olcopyiTS,
|
|
|
|
#ifndef NO_LAPACK
|
|
xneg_tcopyTS, xlaswp_ncopyTS,
|
|
#else
|
|
NULL, NULL,
|
|
#endif
|
|
|
|
#endif
|
|
|
|
init_parameter,
|
|
|
|
SNUMOPT, DNUMOPT, QNUMOPT,
|
|
|
|
};
|
|
|
|
#ifdef ARCH_X86
|
|
static int get_l2_size_old(void){
|
|
int i, eax, ebx, ecx, edx, cpuid_level;
|
|
int info[15];
|
|
|
|
cpuid(2, &eax, &ebx, &ecx, &edx);
|
|
|
|
info[ 0] = BITMASK(eax, 8, 0xff);
|
|
info[ 1] = BITMASK(eax, 16, 0xff);
|
|
info[ 2] = BITMASK(eax, 24, 0xff);
|
|
|
|
info[ 3] = BITMASK(ebx, 0, 0xff);
|
|
info[ 4] = BITMASK(ebx, 8, 0xff);
|
|
info[ 5] = BITMASK(ebx, 16, 0xff);
|
|
info[ 6] = BITMASK(ebx, 24, 0xff);
|
|
|
|
info[ 7] = BITMASK(ecx, 0, 0xff);
|
|
info[ 8] = BITMASK(ecx, 8, 0xff);
|
|
info[ 9] = BITMASK(ecx, 16, 0xff);
|
|
info[10] = BITMASK(ecx, 24, 0xff);
|
|
|
|
info[11] = BITMASK(edx, 0, 0xff);
|
|
info[12] = BITMASK(edx, 8, 0xff);
|
|
info[13] = BITMASK(edx, 16, 0xff);
|
|
info[14] = BITMASK(edx, 24, 0xff);
|
|
|
|
for (i = 0; i < 15; i++){
|
|
|
|
switch (info[i]){
|
|
|
|
/* This table is from http://www.sandpile.org/ia32/cpuid.htm */
|
|
|
|
case 0x1a :
|
|
return 96;
|
|
|
|
case 0x39 :
|
|
case 0x3b :
|
|
case 0x41 :
|
|
case 0x79 :
|
|
case 0x81 :
|
|
return 128;
|
|
|
|
case 0x3a :
|
|
return 192;
|
|
|
|
case 0x21 :
|
|
case 0x3c :
|
|
case 0x42 :
|
|
case 0x7a :
|
|
case 0x7e :
|
|
case 0x82 :
|
|
return 256;
|
|
|
|
case 0x3d :
|
|
return 384;
|
|
|
|
case 0x3e :
|
|
case 0x43 :
|
|
case 0x7b :
|
|
case 0x7f :
|
|
case 0x83 :
|
|
case 0x86 :
|
|
return 512;
|
|
|
|
case 0x44 :
|
|
case 0x78 :
|
|
case 0x7c :
|
|
case 0x84 :
|
|
case 0x87 :
|
|
return 1024;
|
|
|
|
case 0x45 :
|
|
case 0x7d :
|
|
case 0x85 :
|
|
return 2048;
|
|
|
|
case 0x48 :
|
|
return 3184;
|
|
|
|
case 0x49 :
|
|
return 4096;
|
|
|
|
case 0x4e :
|
|
return 6144;
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
#endif
|
|
|
|
static __inline__ int get_l2_size(void){
|
|
|
|
int eax, ebx, ecx, edx, l2;
|
|
|
|
cpuid(0x80000006, &eax, &ebx, &ecx, &edx);
|
|
|
|
l2 = BITMASK(ecx, 16, 0xffff);
|
|
|
|
#ifndef ARCH_X86
|
|
return l2;
|
|
|
|
#else
|
|
|
|
if (l2 > 0) return l2;
|
|
|
|
return get_l2_size_old();
|
|
#endif
|
|
}
|
|
|
|
static __inline__ int get_l3_size(void){
|
|
|
|
int eax, ebx, ecx, edx;
|
|
|
|
cpuid(0x80000006, &eax, &ebx, &ecx, &edx);
|
|
|
|
return BITMASK(edx, 18, 0x3fff) * 512;
|
|
}
|
|
|
|
|
|
static void init_parameter(void) {
|
|
|
|
int l2 = get_l2_size();
|
|
|
|
TABLE_NAME.sgemm_q = SGEMM_DEFAULT_Q;
|
|
TABLE_NAME.dgemm_q = DGEMM_DEFAULT_Q;
|
|
TABLE_NAME.cgemm_q = CGEMM_DEFAULT_Q;
|
|
TABLE_NAME.zgemm_q = ZGEMM_DEFAULT_Q;
|
|
#ifdef EXPRECISION
|
|
TABLE_NAME.qgemm_q = QGEMM_DEFAULT_Q;
|
|
TABLE_NAME.xgemm_q = XGEMM_DEFAULT_Q;
|
|
#endif
|
|
|
|
#if defined(CORE_KATMAI) || defined(CORE_COPPERMINE) || defined(CORE_BANIAS) || defined(CORE_YONAH)
|
|
|
|
#ifdef DEBUG
|
|
fprintf(stderr, "Katmai, Coppermine, Banias\n");
|
|
#endif
|
|
|
|
TABLE_NAME.sgemm_p = 64 * (l2 >> 7);
|
|
TABLE_NAME.dgemm_p = 32 * (l2 >> 7);
|
|
TABLE_NAME.cgemm_p = 32 * (l2 >> 7);
|
|
TABLE_NAME.zgemm_p = 16 * (l2 >> 7);
|
|
#ifdef EXPRECISION
|
|
TABLE_NAME.qgemm_p = 16 * (l2 >> 7);
|
|
TABLE_NAME.xgemm_p = 8 * (l2 >> 7);
|
|
#endif
|
|
#endif
|
|
|
|
#ifdef CORE_NORTHWOOD
|
|
|
|
#ifdef DEBUG
|
|
fprintf(stderr, "Northwood\n");
|
|
#endif
|
|
|
|
TABLE_NAME.sgemm_p = 96 * (l2 >> 7);
|
|
TABLE_NAME.dgemm_p = 48 * (l2 >> 7);
|
|
TABLE_NAME.cgemm_p = 48 * (l2 >> 7);
|
|
TABLE_NAME.zgemm_p = 24 * (l2 >> 7);
|
|
#ifdef EXPRECISION
|
|
TABLE_NAME.qgemm_p = 24 * (l2 >> 7);
|
|
TABLE_NAME.xgemm_p = 12 * (l2 >> 7);
|
|
#endif
|
|
#endif
|
|
|
|
#ifdef ATOM
|
|
|
|
#ifdef DEBUG
|
|
fprintf(stderr, "Atom\n");
|
|
#endif
|
|
|
|
TABLE_NAME.sgemm_p = 256;
|
|
TABLE_NAME.dgemm_p = 128;
|
|
TABLE_NAME.cgemm_p = 128;
|
|
TABLE_NAME.zgemm_p = 64;
|
|
#ifdef EXPRECISION
|
|
TABLE_NAME.qgemm_p = 64;
|
|
TABLE_NAME.xgemm_p = 32;
|
|
#endif
|
|
#endif
|
|
|
|
#ifdef CORE_PRESCOTT
|
|
|
|
#ifdef DEBUG
|
|
fprintf(stderr, "Prescott\n");
|
|
#endif
|
|
|
|
TABLE_NAME.sgemm_p = 56 * (l2 >> 7);
|
|
TABLE_NAME.dgemm_p = 28 * (l2 >> 7);
|
|
TABLE_NAME.cgemm_p = 28 * (l2 >> 7);
|
|
TABLE_NAME.zgemm_p = 14 * (l2 >> 7);
|
|
#ifdef EXPRECISION
|
|
TABLE_NAME.qgemm_p = 14 * (l2 >> 7);
|
|
TABLE_NAME.xgemm_p = 7 * (l2 >> 7);
|
|
#endif
|
|
#endif
|
|
|
|
#ifdef CORE2
|
|
|
|
#ifdef DEBUG
|
|
fprintf(stderr, "Core2\n");
|
|
#endif
|
|
|
|
TABLE_NAME.sgemm_p = 92 * (l2 >> 9);
|
|
TABLE_NAME.dgemm_p = 46 * (l2 >> 9);
|
|
TABLE_NAME.cgemm_p = 46 * (l2 >> 9);
|
|
TABLE_NAME.zgemm_p = 23 * (l2 >> 9);
|
|
#ifdef EXPRECISION
|
|
TABLE_NAME.qgemm_p = 92 * (l2 >> 9);
|
|
TABLE_NAME.xgemm_p = 46 * (l2 >> 9);
|
|
#endif
|
|
#endif
|
|
|
|
#ifdef PENRYN
|
|
|
|
#ifdef DEBUG
|
|
fprintf(stderr, "Penryn\n");
|
|
#endif
|
|
|
|
TABLE_NAME.sgemm_p = 42 * (l2 >> 9) + 8;
|
|
TABLE_NAME.dgemm_p = 42 * (l2 >> 9) + 8;
|
|
TABLE_NAME.cgemm_p = 21 * (l2 >> 9) + 4;
|
|
TABLE_NAME.zgemm_p = 21 * (l2 >> 9) + 4;
|
|
#ifdef EXPRECISION
|
|
TABLE_NAME.qgemm_p = 42 * (l2 >> 9) + 8;
|
|
TABLE_NAME.xgemm_p = 21 * (l2 >> 9) + 4;
|
|
#endif
|
|
#endif
|
|
|
|
#ifdef NEHALEM
|
|
|
|
#ifdef DEBUG
|
|
fprintf(stderr, "Nehalem\n");
|
|
#endif
|
|
|
|
TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;
|
|
TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;
|
|
TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P;
|
|
TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P;
|
|
#ifdef EXPRECISION
|
|
TABLE_NAME.qgemm_p = QGEMM_DEFAULT_P;
|
|
TABLE_NAME.xgemm_p = XGEMM_DEFAULT_P;
|
|
#endif
|
|
#endif
|
|
|
|
#ifdef OPTERON
|
|
|
|
#ifdef DEBUG
|
|
fprintf(stderr, "Opteron\n");
|
|
#endif
|
|
|
|
TABLE_NAME.sgemm_p = 224 + 56 * (l2 >> 7);
|
|
TABLE_NAME.dgemm_p = 112 + 28 * (l2 >> 7);
|
|
TABLE_NAME.cgemm_p = 112 + 28 * (l2 >> 7);
|
|
TABLE_NAME.zgemm_p = 56 + 14 * (l2 >> 7);
|
|
#ifdef EXPRECISION
|
|
TABLE_NAME.qgemm_p = 56 + 14 * (l2 >> 7);
|
|
TABLE_NAME.xgemm_p = 28 + 7 * (l2 >> 7);
|
|
#endif
|
|
#endif
|
|
|
|
#ifdef BARCELONA
|
|
|
|
#ifdef DEBUG
|
|
fprintf(stderr, "Barcelona\n");
|
|
#endif
|
|
|
|
TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;
|
|
TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;
|
|
TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P;
|
|
TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P;
|
|
#ifdef EXPRECISION
|
|
TABLE_NAME.qgemm_p = QGEMM_DEFAULT_P;
|
|
TABLE_NAME.xgemm_p = XGEMM_DEFAULT_P;
|
|
#endif
|
|
#endif
|
|
|
|
#ifdef NANO
|
|
|
|
#ifdef DEBUG
|
|
fprintf(stderr, "NANO\n");
|
|
#endif
|
|
|
|
TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;
|
|
TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;
|
|
TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P;
|
|
TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P;
|
|
#ifdef EXPRECISION
|
|
TABLE_NAME.qgemm_p = QGEMM_DEFAULT_P;
|
|
TABLE_NAME.xgemm_p = XGEMM_DEFAULT_P;
|
|
#endif
|
|
#endif
|
|
|
|
|
|
TABLE_NAME.sgemm_p = (TABLE_NAME.sgemm_p + SGEMM_DEFAULT_UNROLL_M - 1) & ~(SGEMM_DEFAULT_UNROLL_M - 1);
|
|
TABLE_NAME.dgemm_p = (TABLE_NAME.dgemm_p + DGEMM_DEFAULT_UNROLL_M - 1) & ~(DGEMM_DEFAULT_UNROLL_M - 1);
|
|
TABLE_NAME.cgemm_p = (TABLE_NAME.cgemm_p + CGEMM_DEFAULT_UNROLL_M - 1) & ~(CGEMM_DEFAULT_UNROLL_M - 1);
|
|
TABLE_NAME.zgemm_p = (TABLE_NAME.zgemm_p + ZGEMM_DEFAULT_UNROLL_M - 1) & ~(ZGEMM_DEFAULT_UNROLL_M - 1);
|
|
#ifdef QUAD_PRECISION
|
|
TABLE_NAME.qgemm_p = (TABLE_NAME.qgemm_p + QGEMM_DEFAULT_UNROLL_M - 1) & ~(QGEMM_DEFAULT_UNROLL_M - 1);
|
|
TABLE_NAME.xgemm_p = (TABLE_NAME.xgemm_p + XGEMM_DEFAULT_UNROLL_M - 1) & ~(XGEMM_DEFAULT_UNROLL_M - 1);
|
|
#endif
|
|
|
|
#ifdef DEBUG
|
|
fprintf(stderr, "L2 = %8d DGEMM_P .. %d\n", l2, TABLE_NAME.dgemm_p);
|
|
#endif
|
|
|
|
TABLE_NAME.sgemm_r = (((BUFFER_SIZE -
|
|
((TABLE_NAME.sgemm_p * TABLE_NAME.sgemm_q * 4 + TABLE_NAME.offsetA
|
|
+ TABLE_NAME.align) & ~TABLE_NAME.align)
|
|
) / (TABLE_NAME.sgemm_q * 4) - 15) & ~15);
|
|
|
|
TABLE_NAME.dgemm_r = (((BUFFER_SIZE -
|
|
((TABLE_NAME.dgemm_p * TABLE_NAME.dgemm_q * 8 + TABLE_NAME.offsetA
|
|
+ TABLE_NAME.align) & ~TABLE_NAME.align)
|
|
) / (TABLE_NAME.dgemm_q * 8) - 15) & ~15);
|
|
|
|
#ifdef EXPRECISION
|
|
TABLE_NAME.qgemm_r = (((BUFFER_SIZE -
|
|
((TABLE_NAME.qgemm_p * TABLE_NAME.qgemm_q * 16 + TABLE_NAME.offsetA
|
|
+ TABLE_NAME.align) & ~TABLE_NAME.align)
|
|
) / (TABLE_NAME.qgemm_q * 16) - 15) & ~15);
|
|
#endif
|
|
|
|
TABLE_NAME.cgemm_r = (((BUFFER_SIZE -
|
|
((TABLE_NAME.cgemm_p * TABLE_NAME.cgemm_q * 8 + TABLE_NAME.offsetA
|
|
+ TABLE_NAME.align) & ~TABLE_NAME.align)
|
|
) / (TABLE_NAME.cgemm_q * 8) - 15) & ~15);
|
|
|
|
TABLE_NAME.zgemm_r = (((BUFFER_SIZE -
|
|
((TABLE_NAME.zgemm_p * TABLE_NAME.zgemm_q * 16 + TABLE_NAME.offsetA
|
|
+ TABLE_NAME.align) & ~TABLE_NAME.align)
|
|
) / (TABLE_NAME.zgemm_q * 16) - 15) & ~15);
|
|
|
|
#ifdef EXPRECISION
|
|
TABLE_NAME.xgemm_r = (((BUFFER_SIZE -
|
|
((TABLE_NAME.xgemm_p * TABLE_NAME.xgemm_q * 32 + TABLE_NAME.offsetA
|
|
+ TABLE_NAME.align) & ~TABLE_NAME.align)
|
|
) / (TABLE_NAME.xgemm_q * 32) - 15) & ~15);
|
|
#endif
|
|
|
|
}
|