Merge pull request #4009 from Mousius/sve-gemm
Use SVE kernel for SGEMM/DGEMM on Arm(R) Neoverse(TM) V1
This commit is contained in:
Martin Kroeker
GitHub
commit
72caceb324
|
|
@ -1,5 +1,5 @@
|
||||||
/***************************************************************************
|
/***************************************************************************
|
||||||
Copyright (c) 2014, The OpenBLAS Project
|
Copyright (c) 2014, 2023 The OpenBLAS Project
|
||||||
All rights reserved.
|
All rights reserved.
|
||||||
Redistribution and use in source and binary forms, with or without
|
Redistribution and use in source and binary forms, with or without
|
||||||
modification, are permitted provided that the following conditions are
|
modification, are permitted provided that the following conditions are
|
||||||
|
|
@ -67,7 +67,7 @@ int main(int argc, char *argv[]){
|
||||||
int step = 1;
|
int step = 1;
|
||||||
int loops = 1;
|
int loops = 1;
|
||||||
|
|
||||||
if ((p = getenv("OPENBLAS_LOOPS"))) loops=*p;
|
if ((p = getenv("OPENBLAS_LOOPS"))) loops=atoi(p);
|
||||||
|
|
||||||
double time1,timeg;
|
double time1,timeg;
|
||||||
|
|
||||||
|
|
@ -77,7 +77,7 @@ int main(int argc, char *argv[]){
|
||||||
if (argc > 0) { to = MAX(atol(*argv), from); argc--; argv++;}
|
if (argc > 0) { to = MAX(atol(*argv), from); argc--; argv++;}
|
||||||
if (argc > 0) { step = atol(*argv); argc--; argv++;}
|
if (argc > 0) { step = atol(*argv); argc--; argv++;}
|
||||||
|
|
||||||
fprintf(stderr, "From : %3d To : %3d Step = %3d Uplo = %c Trans = %c\n", from, to, step,uplo,trans);
|
fprintf(stderr, "From : %3d To : %3d Step = %3d Uplo = %c Trans = %c Loops = %d\n", from, to, step,uplo,trans,loops);
|
||||||
|
|
||||||
|
|
||||||
if (( a = (FLOAT *)malloc(sizeof(FLOAT) * to * to * COMPSIZE)) == NULL){
|
if (( a = (FLOAT *)malloc(sizeof(FLOAT) * to * to * COMPSIZE)) == NULL){
|
||||||
|
|
|
||||||
|
|
@ -128,10 +128,10 @@ SGEMM_BETA = sgemm_beta.S
|
||||||
SGEMMKERNEL = sgemm_kernel_sve_v2x$(SGEMM_UNROLL_N).S
|
SGEMMKERNEL = sgemm_kernel_sve_v2x$(SGEMM_UNROLL_N).S
|
||||||
STRMMKERNEL = strmm_kernel_sve_v1x$(SGEMM_UNROLL_N).S
|
STRMMKERNEL = strmm_kernel_sve_v1x$(SGEMM_UNROLL_N).S
|
||||||
|
|
||||||
SGEMMINCOPY = sgemm_ncopy_sve_v1.c
|
SGEMMINCOPY = gemm_ncopy_sve_v1x$(SGEMM_UNROLL_N).c
|
||||||
SGEMMITCOPY = sgemm_tcopy_sve_v1.c
|
SGEMMITCOPY = gemm_tcopy_sve_v1x$(SGEMM_UNROLL_N).c
|
||||||
SGEMMONCOPY = sgemm_ncopy_$(DGEMM_UNROLL_N).S
|
SGEMMONCOPY = sgemm_ncopy_$(SGEMM_UNROLL_N).S
|
||||||
SGEMMOTCOPY = sgemm_tcopy_$(DGEMM_UNROLL_N).S
|
SGEMMOTCOPY = sgemm_tcopy_$(SGEMM_UNROLL_N).S
|
||||||
|
|
||||||
SGEMMINCOPYOBJ = sgemm_incopy$(TSUFFIX).$(SUFFIX)
|
SGEMMINCOPYOBJ = sgemm_incopy$(TSUFFIX).$(SUFFIX)
|
||||||
SGEMMITCOPYOBJ = sgemm_itcopy$(TSUFFIX).$(SUFFIX)
|
SGEMMITCOPYOBJ = sgemm_itcopy$(TSUFFIX).$(SUFFIX)
|
||||||
|
|
@ -149,8 +149,8 @@ SSYMMLCOPY_M = symm_lcopy_sve.c
|
||||||
DGEMMKERNEL = dgemm_kernel_sve_v2x$(DGEMM_UNROLL_N).S
|
DGEMMKERNEL = dgemm_kernel_sve_v2x$(DGEMM_UNROLL_N).S
|
||||||
DTRMMKERNEL = dtrmm_kernel_sve_v1x$(DGEMM_UNROLL_N).S
|
DTRMMKERNEL = dtrmm_kernel_sve_v1x$(DGEMM_UNROLL_N).S
|
||||||
|
|
||||||
DGEMMINCOPY = dgemm_ncopy_sve_v1.c
|
DGEMMINCOPY = gemm_ncopy_sve_v1x$(DGEMM_UNROLL_N).c
|
||||||
DGEMMITCOPY = dgemm_tcopy_sve_v1.c
|
DGEMMITCOPY = gemm_tcopy_sve_v1x$(DGEMM_UNROLL_N).c
|
||||||
DGEMMONCOPY = dgemm_ncopy_$(DGEMM_UNROLL_N).S
|
DGEMMONCOPY = dgemm_ncopy_$(DGEMM_UNROLL_N).S
|
||||||
DGEMMOTCOPY = dgemm_tcopy_$(DGEMM_UNROLL_N).S
|
DGEMMOTCOPY = dgemm_tcopy_$(DGEMM_UNROLL_N).S
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,75 +1,6 @@
|
||||||
SAMINKERNEL = ../arm/amin.c
|
include $(KERNELDIR)/KERNEL.ARMV8SVE
|
||||||
DAMINKERNEL = ../arm/amin.c
|
|
||||||
CAMINKERNEL = ../arm/zamin.c
|
|
||||||
ZAMINKERNEL = ../arm/zamin.c
|
|
||||||
|
|
||||||
SMAXKERNEL = ../arm/max.c
|
|
||||||
DMAXKERNEL = ../arm/max.c
|
|
||||||
|
|
||||||
SMINKERNEL = ../arm/min.c
|
|
||||||
DMINKERNEL = ../arm/min.c
|
|
||||||
|
|
||||||
ISAMINKERNEL = ../arm/iamin.c
|
|
||||||
IDAMINKERNEL = ../arm/iamin.c
|
|
||||||
ICAMINKERNEL = ../arm/izamin.c
|
|
||||||
IZAMINKERNEL = ../arm/izamin.c
|
|
||||||
|
|
||||||
ISMAXKERNEL = ../arm/imax.c
|
|
||||||
IDMAXKERNEL = ../arm/imax.c
|
|
||||||
|
|
||||||
ISMINKERNEL = ../arm/imin.c
|
|
||||||
IDMINKERNEL = ../arm/imin.c
|
|
||||||
|
|
||||||
STRSMKERNEL_LN = ../generic/trsm_kernel_LN.c
|
|
||||||
STRSMKERNEL_LT = ../generic/trsm_kernel_LT.c
|
|
||||||
STRSMKERNEL_RN = ../generic/trsm_kernel_RN.c
|
|
||||||
STRSMKERNEL_RT = ../generic/trsm_kernel_RT.c
|
|
||||||
|
|
||||||
DTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c
|
|
||||||
DTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c
|
|
||||||
DTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c
|
|
||||||
DTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c
|
|
||||||
|
|
||||||
CTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c
|
|
||||||
CTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c
|
|
||||||
CTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c
|
|
||||||
CTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c
|
|
||||||
|
|
||||||
ZTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c
|
|
||||||
ZTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c
|
|
||||||
ZTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c
|
|
||||||
ZTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c
|
|
||||||
|
|
||||||
SAMAXKERNEL = amax.S
|
|
||||||
DAMAXKERNEL = amax.S
|
|
||||||
CAMAXKERNEL = zamax.S
|
|
||||||
ZAMAXKERNEL = zamax.S
|
|
||||||
|
|
||||||
SAXPYKERNEL = axpy.S
|
|
||||||
DAXPYKERNEL = daxpy_thunderx2t99.S
|
DAXPYKERNEL = daxpy_thunderx2t99.S
|
||||||
CAXPYKERNEL = zaxpy.S
|
|
||||||
ZAXPYKERNEL = zaxpy.S
|
|
||||||
|
|
||||||
SROTKERNEL = rot.S
|
|
||||||
DROTKERNEL = rot.S
|
|
||||||
CROTKERNEL = zrot.S
|
|
||||||
ZROTKERNEL = zrot.S
|
|
||||||
|
|
||||||
SSCALKERNEL = scal.S
|
|
||||||
DSCALKERNEL = scal.S
|
|
||||||
CSCALKERNEL = zscal.S
|
|
||||||
ZSCALKERNEL = zscal.S
|
|
||||||
|
|
||||||
SGEMVNKERNEL = gemv_n.S
|
|
||||||
DGEMVNKERNEL = gemv_n.S
|
|
||||||
CGEMVNKERNEL = zgemv_n.S
|
|
||||||
ZGEMVNKERNEL = zgemv_n.S
|
|
||||||
|
|
||||||
SGEMVTKERNEL = gemv_t.S
|
|
||||||
DGEMVTKERNEL = gemv_t.S
|
|
||||||
CGEMVTKERNEL = zgemv_t.S
|
|
||||||
ZGEMVTKERNEL = zgemv_t.S
|
|
||||||
|
|
||||||
|
|
||||||
SASUMKERNEL = sasum_thunderx2t99.c
|
SASUMKERNEL = sasum_thunderx2t99.c
|
||||||
DASUMKERNEL = dasum_thunderx2t99.c
|
DASUMKERNEL = dasum_thunderx2t99.c
|
||||||
|
|
@ -100,67 +31,20 @@ DDOTKERNEL = dot.c
|
||||||
SDOTKERNEL = dot.c
|
SDOTKERNEL = dot.c
|
||||||
CDOTKERNEL = zdot_thunderx2t99.c
|
CDOTKERNEL = zdot_thunderx2t99.c
|
||||||
ZDOTKERNEL = zdot_thunderx2t99.c
|
ZDOTKERNEL = zdot_thunderx2t99.c
|
||||||
DSDOTKERNEL = dot.S
|
|
||||||
|
|
||||||
DGEMM_BETA = dgemm_beta.S
|
CTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c
|
||||||
SGEMM_BETA = sgemm_beta.S
|
CTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c
|
||||||
|
CTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c
|
||||||
|
CTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c
|
||||||
|
|
||||||
SGEMMKERNEL = sgemm_kernel_$(SGEMM_UNROLL_M)x$(SGEMM_UNROLL_N).S
|
CTRMMUNCOPY_M =
|
||||||
STRMMKERNEL = strmm_kernel_$(SGEMM_UNROLL_M)x$(SGEMM_UNROLL_N).S
|
CTRMMLNCOPY_M =
|
||||||
ifneq ($(SGEMM_UNROLL_M), $(SGEMM_UNROLL_N))
|
CTRMMUTCOPY_M =
|
||||||
ifeq ($(SGEMM_UNROLL_M), 16)
|
CTRMMLTCOPY_M =
|
||||||
SGEMMITCOPY = sgemm_tcopy_$(SGEMM_UNROLL_M).S
|
CHEMMLTCOPY_M =
|
||||||
else
|
CHEMMUTCOPY_M =
|
||||||
SGEMMITCOPY = ../generic/gemm_tcopy_$(SGEMM_UNROLL_M).c
|
CSYMMUCOPY_M =
|
||||||
endif
|
CSYMMLCOPY_M =
|
||||||
ifeq ($(SGEMM_UNROLL_M), 4)
|
|
||||||
SGEMMINCOPY = sgemm_ncopy_$(SGEMM_UNROLL_M).S
|
|
||||||
else
|
|
||||||
SGEMMINCOPY = ../generic/gemm_ncopy_$(SGEMM_UNROLL_M).c
|
|
||||||
endif
|
|
||||||
SGEMMINCOPYOBJ = sgemm_incopy$(TSUFFIX).$(SUFFIX)
|
|
||||||
SGEMMITCOPYOBJ = sgemm_itcopy$(TSUFFIX).$(SUFFIX)
|
|
||||||
endif
|
|
||||||
ifeq ($(SGEMM_UNROLL_N), 16)
|
|
||||||
SGEMMOTCOPY = sgemm_tcopy_$(SGEMM_UNROLL_N).S
|
|
||||||
else
|
|
||||||
SGEMMOTCOPY = ../generic/gemm_tcopy_$(SGEMM_UNROLL_N).c
|
|
||||||
endif
|
|
||||||
ifeq ($(SGEMM_UNROLL_N), 4)
|
|
||||||
SGEMMONCOPY = sgemm_ncopy_$(SGEMM_UNROLL_N).S
|
|
||||||
else
|
|
||||||
SGEMMONCOPY = ../generic/gemm_ncopy_$(SGEMM_UNROLL_N).c
|
|
||||||
endif
|
|
||||||
SGEMMONCOPYOBJ = sgemm_oncopy$(TSUFFIX).$(SUFFIX)
|
|
||||||
SGEMMOTCOPYOBJ = sgemm_otcopy$(TSUFFIX).$(SUFFIX)
|
|
||||||
|
|
||||||
DGEMMKERNEL = dgemm_kernel_$(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N).S
|
|
||||||
DTRMMKERNEL = dtrmm_kernel_$(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N).S
|
|
||||||
|
|
||||||
ifneq ($(DGEMM_UNROLL_M), $(DGEMM_UNROLL_N))
|
|
||||||
|
|
||||||
ifeq ($(DGEMM_UNROLL_M), 8)
|
|
||||||
DGEMMINCOPY = dgemm_ncopy_$(DGEMM_UNROLL_M).S
|
|
||||||
DGEMMITCOPY = dgemm_tcopy_$(DGEMM_UNROLL_M).S
|
|
||||||
else
|
|
||||||
DGEMMINCOPY = ../generic/gemm_ncopy_$(DGEMM_UNROLL_M).c
|
|
||||||
DGEMMITCOPY = ../generic/gemm_tcopy_$(DGEMM_UNROLL_M).c
|
|
||||||
endif
|
|
||||||
|
|
||||||
DGEMMINCOPYOBJ = dgemm_incopy$(TSUFFIX).$(SUFFIX)
|
|
||||||
DGEMMITCOPYOBJ = dgemm_itcopy$(TSUFFIX).$(SUFFIX)
|
|
||||||
endif
|
|
||||||
|
|
||||||
ifeq ($(DGEMM_UNROLL_N), 4)
|
|
||||||
DGEMMONCOPY = dgemm_ncopy_$(DGEMM_UNROLL_N).S
|
|
||||||
DGEMMOTCOPY = dgemm_tcopy_$(DGEMM_UNROLL_N).S
|
|
||||||
else
|
|
||||||
DGEMMONCOPY = ../generic/gemm_ncopy_$(DGEMM_UNROLL_N).c
|
|
||||||
DGEMMOTCOPY = ../generic/gemm_tcopy_$(DGEMM_UNROLL_N).c
|
|
||||||
endif
|
|
||||||
|
|
||||||
DGEMMONCOPYOBJ = dgemm_oncopy$(TSUFFIX).$(SUFFIX)
|
|
||||||
DGEMMOTCOPYOBJ = dgemm_otcopy$(TSUFFIX).$(SUFFIX)
|
|
||||||
|
|
||||||
CGEMMKERNEL = cgemm_kernel_$(CGEMM_UNROLL_M)x$(CGEMM_UNROLL_N).S
|
CGEMMKERNEL = cgemm_kernel_$(CGEMM_UNROLL_M)x$(CGEMM_UNROLL_N).S
|
||||||
CTRMMKERNEL = ctrmm_kernel_$(CGEMM_UNROLL_M)x$(CGEMM_UNROLL_N).S
|
CTRMMKERNEL = ctrmm_kernel_$(CGEMM_UNROLL_M)x$(CGEMM_UNROLL_N).S
|
||||||
|
|
@ -169,12 +53,34 @@ CGEMMINCOPY = ../generic/zgemm_ncopy_$(CGEMM_UNROLL_M).c
|
||||||
CGEMMITCOPY = ../generic/zgemm_tcopy_$(CGEMM_UNROLL_M).c
|
CGEMMITCOPY = ../generic/zgemm_tcopy_$(CGEMM_UNROLL_M).c
|
||||||
CGEMMINCOPYOBJ = cgemm_incopy$(TSUFFIX).$(SUFFIX)
|
CGEMMINCOPYOBJ = cgemm_incopy$(TSUFFIX).$(SUFFIX)
|
||||||
CGEMMITCOPYOBJ = cgemm_itcopy$(TSUFFIX).$(SUFFIX)
|
CGEMMITCOPYOBJ = cgemm_itcopy$(TSUFFIX).$(SUFFIX)
|
||||||
|
else
|
||||||
|
CGEMMINCOPYOBJ =
|
||||||
|
CGEMMITCOPYOBJ =
|
||||||
endif
|
endif
|
||||||
CGEMMONCOPY = ../generic/zgemm_ncopy_$(CGEMM_UNROLL_N).c
|
CGEMMONCOPY = ../generic/zgemm_ncopy_$(CGEMM_UNROLL_N).c
|
||||||
CGEMMOTCOPY = ../generic/zgemm_tcopy_$(CGEMM_UNROLL_N).c
|
CGEMMOTCOPY = ../generic/zgemm_tcopy_$(CGEMM_UNROLL_N).c
|
||||||
CGEMMONCOPYOBJ = cgemm_oncopy$(TSUFFIX).$(SUFFIX)
|
CGEMMONCOPYOBJ = cgemm_oncopy$(TSUFFIX).$(SUFFIX)
|
||||||
CGEMMOTCOPYOBJ = cgemm_otcopy$(TSUFFIX).$(SUFFIX)
|
CGEMMOTCOPYOBJ = cgemm_otcopy$(TSUFFIX).$(SUFFIX)
|
||||||
|
|
||||||
|
ZTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c
|
||||||
|
ZTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c
|
||||||
|
ZTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c
|
||||||
|
ZTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c
|
||||||
|
|
||||||
|
ZTRSMCOPYLN_M =
|
||||||
|
ZTRSMCOPYLT_M =
|
||||||
|
ZTRSMCOPYUN_M =
|
||||||
|
ZTRSMCOPYUT_M =
|
||||||
|
|
||||||
|
ZTRMMUNCOPY_M =
|
||||||
|
ZTRMMLNCOPY_M =
|
||||||
|
ZTRMMUTCOPY_M =
|
||||||
|
ZTRMMLTCOPY_M =
|
||||||
|
ZHEMMLTCOPY_M =
|
||||||
|
ZHEMMUTCOPY_M =
|
||||||
|
ZSYMMUCOPY_M =
|
||||||
|
ZSYMMLCOPY_M =
|
||||||
|
|
||||||
ZGEMMKERNEL = zgemm_kernel_$(ZGEMM_UNROLL_M)x$(ZGEMM_UNROLL_N).S
|
ZGEMMKERNEL = zgemm_kernel_$(ZGEMM_UNROLL_M)x$(ZGEMM_UNROLL_N).S
|
||||||
ZTRMMKERNEL = ztrmm_kernel_$(ZGEMM_UNROLL_M)x$(ZGEMM_UNROLL_N).S
|
ZTRMMKERNEL = ztrmm_kernel_$(ZGEMM_UNROLL_M)x$(ZGEMM_UNROLL_N).S
|
||||||
ifneq ($(ZGEMM_UNROLL_M), $(ZGEMM_UNROLL_N))
|
ifneq ($(ZGEMM_UNROLL_M), $(ZGEMM_UNROLL_N))
|
||||||
|
|
@ -182,6 +88,9 @@ ZGEMMINCOPY = ../generic/zgemm_ncopy_$(ZGEMM_UNROLL_M).c
|
||||||
ZGEMMITCOPY = ../generic/zgemm_tcopy_$(ZGEMM_UNROLL_M).c
|
ZGEMMITCOPY = ../generic/zgemm_tcopy_$(ZGEMM_UNROLL_M).c
|
||||||
ZGEMMINCOPYOBJ = zgemm_incopy$(TSUFFIX).$(SUFFIX)
|
ZGEMMINCOPYOBJ = zgemm_incopy$(TSUFFIX).$(SUFFIX)
|
||||||
ZGEMMITCOPYOBJ = zgemm_itcopy$(TSUFFIX).$(SUFFIX)
|
ZGEMMITCOPYOBJ = zgemm_itcopy$(TSUFFIX).$(SUFFIX)
|
||||||
|
else
|
||||||
|
ZGEMMINCOPYOBJ =
|
||||||
|
ZGEMMITCOPYOBJ =
|
||||||
endif
|
endif
|
||||||
ZGEMMONCOPY = ../generic/zgemm_ncopy_$(ZGEMM_UNROLL_N).c
|
ZGEMMONCOPY = ../generic/zgemm_ncopy_$(ZGEMM_UNROLL_N).c
|
||||||
ZGEMMOTCOPY = ../generic/zgemm_tcopy_$(ZGEMM_UNROLL_N).c
|
ZGEMMOTCOPY = ../generic/zgemm_tcopy_$(ZGEMM_UNROLL_N).c
|
||||||
|
|
|
||||||
|
|
@ -1,79 +0,0 @@
|
||||||
/*********************************************************************/
|
|
||||||
/* Copyright 2009, 2010 The University of Texas at Austin. */
|
|
||||||
/* All rights reserved. */
|
|
||||||
/* */
|
|
||||||
/* Redistribution and use in source and binary forms, with or */
|
|
||||||
/* without modification, are permitted provided that the following */
|
|
||||||
/* conditions are met: */
|
|
||||||
/* */
|
|
||||||
/* 1. Redistributions of source code must retain the above */
|
|
||||||
/* copyright notice, this list of conditions and the following */
|
|
||||||
/* disclaimer. */
|
|
||||||
/* */
|
|
||||||
/* 2. Redistributions in binary form must reproduce the above */
|
|
||||||
/* copyright notice, this list of conditions and the following */
|
|
||||||
/* disclaimer in the documentation and/or other materials */
|
|
||||||
/* provided with the distribution. */
|
|
||||||
/* */
|
|
||||||
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
|
|
||||||
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
|
|
||||||
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
|
|
||||||
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
|
|
||||||
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
|
|
||||||
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
|
|
||||||
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
|
|
||||||
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
|
|
||||||
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
|
|
||||||
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
|
|
||||||
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
|
|
||||||
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
|
|
||||||
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
|
|
||||||
/* POSSIBILITY OF SUCH DAMAGE. */
|
|
||||||
/* */
|
|
||||||
/* The views and conclusions contained in the software and */
|
|
||||||
/* documentation are those of the authors and should not be */
|
|
||||||
/* interpreted as representing official policies, either expressed */
|
|
||||||
/* or implied, of The University of Texas at Austin. */
|
|
||||||
/*********************************************************************/
|
|
||||||
|
|
||||||
#include <stdio.h>
|
|
||||||
#include "common.h"
|
|
||||||
#include <arm_sve.h>
|
|
||||||
|
|
||||||
// TODO: write in assembly with proper unrolling of inner loop
|
|
||||||
int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){
|
|
||||||
|
|
||||||
BLASLONG j;
|
|
||||||
IFLOAT *aoffset, *aoffset1, *boffset;
|
|
||||||
|
|
||||||
svint64_t lda_vec = svindex_s64(0LL, lda);
|
|
||||||
uint64_t sve_size = svcntd();
|
|
||||||
|
|
||||||
aoffset = a;
|
|
||||||
boffset = b;
|
|
||||||
|
|
||||||
j = 0;
|
|
||||||
svbool_t pg = svwhilelt_b64(j, n);
|
|
||||||
uint64_t active = svcntp_b64(svptrue_b64(), pg);
|
|
||||||
do {
|
|
||||||
|
|
||||||
aoffset1 = aoffset;
|
|
||||||
|
|
||||||
uint64_t i_cnt = m;
|
|
||||||
while (i_cnt--) {
|
|
||||||
svfloat64_t a_vec = svld1_gather_index(pg, (double *) aoffset1, lda_vec);
|
|
||||||
svst1_f64(pg, (double *) boffset, a_vec);
|
|
||||||
aoffset1++;
|
|
||||||
boffset += active;
|
|
||||||
}
|
|
||||||
aoffset += sve_size * lda;
|
|
||||||
|
|
||||||
j += svcntd();
|
|
||||||
pg = svwhilelt_b64(j, n);
|
|
||||||
active = svcntp_b64(svptrue_b64(), pg);
|
|
||||||
|
|
||||||
|
|
||||||
} while (svptest_any(svptrue_b64(), pg));
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
@ -1,77 +0,0 @@
|
||||||
/*********************************************************************/
|
|
||||||
/* Copyright 2009, 2010 The University of Texas at Austin. */
|
|
||||||
/* All rights reserved. */
|
|
||||||
/* */
|
|
||||||
/* Redistribution and use in source and binary forms, with or */
|
|
||||||
/* without modification, are permitted provided that the following */
|
|
||||||
/* conditions are met: */
|
|
||||||
/* */
|
|
||||||
/* 1. Redistributions of source code must retain the above */
|
|
||||||
/* copyright notice, this list of conditions and the following */
|
|
||||||
/* disclaimer. */
|
|
||||||
/* */
|
|
||||||
/* 2. Redistributions in binary form must reproduce the above */
|
|
||||||
/* copyright notice, this list of conditions and the following */
|
|
||||||
/* disclaimer in the documentation and/or other materials */
|
|
||||||
/* provided with the distribution. */
|
|
||||||
/* */
|
|
||||||
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
|
|
||||||
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
|
|
||||||
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
|
|
||||||
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
|
|
||||||
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
|
|
||||||
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
|
|
||||||
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
|
|
||||||
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
|
|
||||||
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
|
|
||||||
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
|
|
||||||
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
|
|
||||||
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
|
|
||||||
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
|
|
||||||
/* POSSIBILITY OF SUCH DAMAGE. */
|
|
||||||
/* */
|
|
||||||
/* The views and conclusions contained in the software and */
|
|
||||||
/* documentation are those of the authors and should not be */
|
|
||||||
/* interpreted as representing official policies, either expressed */
|
|
||||||
/* or implied, of The University of Texas at Austin. */
|
|
||||||
/*********************************************************************/
|
|
||||||
|
|
||||||
#include <stdio.h>
|
|
||||||
#include "common.h"
|
|
||||||
#include <arm_sve.h>
|
|
||||||
|
|
||||||
// TODO: write in assembly with proper unrolling of inner loop
|
|
||||||
int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){
|
|
||||||
|
|
||||||
BLASLONG j;
|
|
||||||
IFLOAT *aoffset, *aoffset1, *boffset;
|
|
||||||
|
|
||||||
uint64_t sve_size = svcntd();
|
|
||||||
|
|
||||||
aoffset = a;
|
|
||||||
boffset = b;
|
|
||||||
|
|
||||||
j = 0;
|
|
||||||
svbool_t pg = svwhilelt_b64(j, n);
|
|
||||||
uint64_t active = svcntp_b64(svptrue_b64(), pg);
|
|
||||||
do {
|
|
||||||
|
|
||||||
aoffset1 = aoffset;
|
|
||||||
|
|
||||||
uint64_t i_cnt = m;
|
|
||||||
while (i_cnt--) {
|
|
||||||
svfloat64_t a_vec = svld1(pg, (double *)aoffset1);
|
|
||||||
svst1_f64(pg, (double *) boffset, a_vec);
|
|
||||||
aoffset1 += lda;
|
|
||||||
boffset += active;
|
|
||||||
}
|
|
||||||
aoffset += sve_size;
|
|
||||||
|
|
||||||
j += svcntd();
|
|
||||||
pg = svwhilelt_b64(j, n);
|
|
||||||
active = svcntp_b64(svptrue_b64(), pg);
|
|
||||||
|
|
||||||
} while (svptest_any(svptrue_b64(), pg));
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
@ -0,0 +1,131 @@
|
||||||
|
/***************************************************************************
|
||||||
|
Copyright (c) 2023, The OpenBLAS Project
|
||||||
|
All rights reserved.
|
||||||
|
Redistribution and use in source and binary forms, with or without
|
||||||
|
modification, are permitted provided that the following conditions are
|
||||||
|
met:
|
||||||
|
1. Redistributions of source code must retain the above copyright
|
||||||
|
notice, this list of conditions and the following disclaimer.
|
||||||
|
2. Redistributions in binary form must reproduce the above copyright
|
||||||
|
notice, this list of conditions and the following disclaimer in
|
||||||
|
the documentation and/or other materials provided with the
|
||||||
|
distribution.
|
||||||
|
3. Neither the name of the OpenBLAS project nor the names of
|
||||||
|
its contributors may be used to endorse or promote products
|
||||||
|
derived from this software without specific prior written permission.
|
||||||
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||||
|
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||||
|
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A00 PARTICULAR PURPOSE
|
||||||
|
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
|
||||||
|
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||||
|
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||||
|
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||||
|
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||||
|
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
|
||||||
|
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
*****************************************************************************/
|
||||||
|
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <arm_sve.h>
|
||||||
|
|
||||||
|
#include "common.h"
|
||||||
|
|
||||||
|
#ifdef DOUBLE
|
||||||
|
#define COUNT "cntd"
|
||||||
|
#define SV_TYPE svfloat64_t
|
||||||
|
#define SV_INDEX svuint64_t
|
||||||
|
#define SV_INDEXER svindex_u64
|
||||||
|
#define SV_TRUE svptrue_b64
|
||||||
|
#define SV_WHILE svwhilelt_b64
|
||||||
|
#define SV_PREFETCH svprfd_gather_index
|
||||||
|
#else
|
||||||
|
#define COUNT "cntw"
|
||||||
|
#define SV_TYPE svfloat32_t
|
||||||
|
#define SV_INDEX svuint32_t
|
||||||
|
#define SV_INDEXER svindex_u32
|
||||||
|
#define SV_TRUE svptrue_b32
|
||||||
|
#define SV_WHILE svwhilelt_b32
|
||||||
|
#define SV_PREFETCH svprfw_gather_index
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#define INNER_COPY(pg, a_offset_inner, b_offset, lda, active) \
|
||||||
|
a_vec = svld1_gather_index(pg, a_offset_inner, lda_vec); \
|
||||||
|
svst1(pg, b_offset, a_vec); \
|
||||||
|
a_offset_inner++; \
|
||||||
|
b_offset += active;
|
||||||
|
|
||||||
|
int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b) {
|
||||||
|
uint64_t sve_size;
|
||||||
|
asm(COUNT" %[SIZE_]" : [SIZE_] "=r" (sve_size) : : );
|
||||||
|
|
||||||
|
IFLOAT *a_offset, *a_offset_inner, *b_offset;
|
||||||
|
a_offset = a;
|
||||||
|
b_offset = b;
|
||||||
|
|
||||||
|
SV_INDEX lda_vec = SV_INDEXER(0LL, lda);
|
||||||
|
SV_TYPE a_vec;
|
||||||
|
svbool_t pg_true = SV_TRUE();
|
||||||
|
|
||||||
|
BLASLONG single_vectors_n = n & -sve_size;
|
||||||
|
for (BLASLONG j = 0; j < single_vectors_n; j += sve_size) {
|
||||||
|
a_offset_inner = a_offset;
|
||||||
|
|
||||||
|
svbool_t pg = pg_true;
|
||||||
|
uint64_t active = sve_size;
|
||||||
|
uint64_t i_cnt = m >> 3;
|
||||||
|
while (i_cnt--) {
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (m & 4) {
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (m & 2) {
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (m & 1) {
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
}
|
||||||
|
|
||||||
|
a_offset += sve_size * lda;
|
||||||
|
}
|
||||||
|
|
||||||
|
BLASLONG remaining_n = n - single_vectors_n;
|
||||||
|
if (remaining_n) {
|
||||||
|
a_offset_inner = a_offset;
|
||||||
|
svbool_t pg = SV_WHILE(0L, remaining_n);
|
||||||
|
uint64_t active = remaining_n;
|
||||||
|
uint64_t i_cnt = m >> 2;
|
||||||
|
while (i_cnt--) {
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (m & 2) {
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (m & 1) {
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,125 @@
|
||||||
|
/***************************************************************************
|
||||||
|
Copyright (c) 2023, The OpenBLAS Project
|
||||||
|
All rights reserved.
|
||||||
|
Redistribution and use in source and binary forms, with or without
|
||||||
|
modification, are permitted provided that the following conditions are
|
||||||
|
met:
|
||||||
|
1. Redistributions of source code must retain the above copyright
|
||||||
|
notice, this list of conditions and the following disclaimer.
|
||||||
|
2. Redistributions in binary form must reproduce the above copyright
|
||||||
|
notice, this list of conditions and the following disclaimer in
|
||||||
|
the documentation and/or other materials provided with the
|
||||||
|
distribution.
|
||||||
|
3. Neither the name of the OpenBLAS project nor the names of
|
||||||
|
its contributors may be used to endorse or promote products
|
||||||
|
derived from this software without specific prior written permission.
|
||||||
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||||
|
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||||
|
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A00 PARTICULAR PURPOSE
|
||||||
|
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
|
||||||
|
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||||
|
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||||
|
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||||
|
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||||
|
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
|
||||||
|
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
*****************************************************************************/
|
||||||
|
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <arm_sve.h>
|
||||||
|
|
||||||
|
#include "common.h"
|
||||||
|
|
||||||
|
#ifdef DOUBLE
|
||||||
|
#define COUNT "cntd"
|
||||||
|
#define SV_TYPE svfloat64_t
|
||||||
|
#define SV_TRUE svptrue_b64
|
||||||
|
#define SV_WHILE svwhilelt_b64
|
||||||
|
#else
|
||||||
|
#define COUNT "cntw"
|
||||||
|
#define SV_TYPE svfloat32_t
|
||||||
|
#define SV_TRUE svptrue_b32
|
||||||
|
#define SV_WHILE svwhilelt_b32
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#define INNER_COPY(pg, a_offset_inner, b_offset, lda, active) \
|
||||||
|
a_vec = svld1(pg, a_offset_inner); \
|
||||||
|
svst1(pg, b_offset, a_vec); \
|
||||||
|
a_offset_inner += lda; \
|
||||||
|
b_offset += active;
|
||||||
|
|
||||||
|
int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){
|
||||||
|
uint64_t sve_size = svcntw();
|
||||||
|
asm(COUNT" %[SIZE_]" : [SIZE_] "=r" (sve_size) : : );
|
||||||
|
|
||||||
|
IFLOAT *a_offset, *a_offset_inner, *b_offset;
|
||||||
|
a_offset = a;
|
||||||
|
b_offset = b;
|
||||||
|
|
||||||
|
SV_TYPE a_vec;
|
||||||
|
svbool_t pg_true = SV_TRUE();
|
||||||
|
|
||||||
|
BLASLONG single_vectors_n = n & -sve_size;
|
||||||
|
for (BLASLONG j = 0; j < single_vectors_n; j += sve_size) {
|
||||||
|
a_offset_inner = a_offset;
|
||||||
|
|
||||||
|
svbool_t pg = pg_true;
|
||||||
|
uint64_t active = sve_size;
|
||||||
|
uint64_t i_cnt = m >> 3;
|
||||||
|
while (i_cnt--) {
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (m & 4) {
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (m & 2) {
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (m & 1) {
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
}
|
||||||
|
|
||||||
|
a_offset += sve_size;
|
||||||
|
}
|
||||||
|
|
||||||
|
BLASLONG remaining_n = n - single_vectors_n;
|
||||||
|
if (remaining_n) {
|
||||||
|
a_offset_inner = a_offset;
|
||||||
|
svbool_t pg = SV_WHILE(0L, remaining_n);
|
||||||
|
uint64_t active = remaining_n;
|
||||||
|
uint64_t i_cnt = m >> 2;
|
||||||
|
while (i_cnt--) {
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (m & 2) {
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (m & 1) {
|
||||||
|
INNER_COPY(pg, a_offset_inner, b_offset, lda, active);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
@ -1,78 +0,0 @@
|
||||||
/*********************************************************************/
|
|
||||||
/* Copyright 2009, 2010 The University of Texas at Austin. */
|
|
||||||
/* All rights reserved. */
|
|
||||||
/* */
|
|
||||||
/* Redistribution and use in source and binary forms, with or */
|
|
||||||
/* without modification, are permitted provided that the following */
|
|
||||||
/* conditions are met: */
|
|
||||||
/* */
|
|
||||||
/* 1. Redistributions of source code must retain the above */
|
|
||||||
/* copyright notice, this list of conditions and the following */
|
|
||||||
/* disclaimer. */
|
|
||||||
/* */
|
|
||||||
/* 2. Redistributions in binary form must reproduce the above */
|
|
||||||
/* copyright notice, this list of conditions and the following */
|
|
||||||
/* disclaimer in the documentation and/or other materials */
|
|
||||||
/* provided with the distribution. */
|
|
||||||
/* */
|
|
||||||
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
|
|
||||||
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
|
|
||||||
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
|
|
||||||
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
|
|
||||||
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
|
|
||||||
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
|
|
||||||
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
|
|
||||||
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
|
|
||||||
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
|
|
||||||
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
|
|
||||||
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
|
|
||||||
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
|
|
||||||
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
|
|
||||||
/* POSSIBILITY OF SUCH DAMAGE. */
|
|
||||||
/* */
|
|
||||||
/* The views and conclusions contained in the software and */
|
|
||||||
/* documentation are those of the authors and should not be */
|
|
||||||
/* interpreted as representing official policies, either expressed */
|
|
||||||
/* or implied, of The University of Texas at Austin. */
|
|
||||||
/*********************************************************************/
|
|
||||||
|
|
||||||
#include <stdio.h>
|
|
||||||
#include "common.h"
|
|
||||||
#include <arm_sve.h>
|
|
||||||
|
|
||||||
// TODO: write in assembly with proper unrolling of inner loop
|
|
||||||
int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){
|
|
||||||
|
|
||||||
BLASLONG j;
|
|
||||||
IFLOAT *aoffset, *aoffset1, *boffset;
|
|
||||||
|
|
||||||
svint32_t lda_vec = svindex_s32(0LL, lda);
|
|
||||||
uint32_t sve_size = svcntw();
|
|
||||||
|
|
||||||
aoffset = a;
|
|
||||||
boffset = b;
|
|
||||||
|
|
||||||
j = 0;
|
|
||||||
svbool_t pg = svwhilelt_b32(j, n);
|
|
||||||
uint32_t active = svcntp_b32(svptrue_b32(), pg);
|
|
||||||
do {
|
|
||||||
|
|
||||||
aoffset1 = aoffset;
|
|
||||||
|
|
||||||
uint32_t i_cnt = m;
|
|
||||||
while (i_cnt--) {
|
|
||||||
svfloat32_t a_vec = svld1_gather_index(pg, (float *) aoffset1, lda_vec);
|
|
||||||
svst1_f32(pg, (float *) boffset, a_vec);
|
|
||||||
aoffset1++;
|
|
||||||
boffset += active;
|
|
||||||
}
|
|
||||||
aoffset += sve_size * lda;
|
|
||||||
|
|
||||||
j += svcntw();
|
|
||||||
pg = svwhilelt_b32(j, n);
|
|
||||||
active = svcntp_b32(svptrue_b32(), pg);
|
|
||||||
|
|
||||||
} while (svptest_any(svptrue_b32(), pg));
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
@ -1,77 +0,0 @@
|
||||||
/*********************************************************************/
|
|
||||||
/* Copyright 2009, 2010 The University of Texas at Austin. */
|
|
||||||
/* All rights reserved. */
|
|
||||||
/* */
|
|
||||||
/* Redistribution and use in source and binary forms, with or */
|
|
||||||
/* without modification, are permitted provided that the following */
|
|
||||||
/* conditions are met: */
|
|
||||||
/* */
|
|
||||||
/* 1. Redistributions of source code must retain the above */
|
|
||||||
/* copyright notice, this list of conditions and the following */
|
|
||||||
/* disclaimer. */
|
|
||||||
/* */
|
|
||||||
/* 2. Redistributions in binary form must reproduce the above */
|
|
||||||
/* copyright notice, this list of conditions and the following */
|
|
||||||
/* disclaimer in the documentation and/or other materials */
|
|
||||||
/* provided with the distribution. */
|
|
||||||
/* */
|
|
||||||
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
|
|
||||||
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
|
|
||||||
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
|
|
||||||
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
|
|
||||||
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
|
|
||||||
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
|
|
||||||
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
|
|
||||||
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
|
|
||||||
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
|
|
||||||
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
|
|
||||||
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
|
|
||||||
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
|
|
||||||
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
|
|
||||||
/* POSSIBILITY OF SUCH DAMAGE. */
|
|
||||||
/* */
|
|
||||||
/* The views and conclusions contained in the software and */
|
|
||||||
/* documentation are those of the authors and should not be */
|
|
||||||
/* interpreted as representing official policies, either expressed */
|
|
||||||
/* or implied, of The University of Texas at Austin. */
|
|
||||||
/*********************************************************************/
|
|
||||||
|
|
||||||
#include <stdio.h>
|
|
||||||
#include "common.h"
|
|
||||||
#include <arm_sve.h>
|
|
||||||
|
|
||||||
// TODO: write in assembly with proper unrolling of inner loop
|
|
||||||
int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){
|
|
||||||
|
|
||||||
BLASLONG j;
|
|
||||||
IFLOAT *aoffset, *aoffset1, *boffset;
|
|
||||||
|
|
||||||
uint32_t sve_size = svcntw();
|
|
||||||
|
|
||||||
aoffset = a;
|
|
||||||
boffset = b;
|
|
||||||
|
|
||||||
j = 0;
|
|
||||||
svbool_t pg = svwhilelt_b32(j, n);
|
|
||||||
uint32_t active = svcntp_b32(svptrue_b32(), pg);
|
|
||||||
do {
|
|
||||||
|
|
||||||
aoffset1 = aoffset;
|
|
||||||
|
|
||||||
uint32_t i_cnt = m;
|
|
||||||
while (i_cnt--) {
|
|
||||||
svfloat32_t a_vec = svld1(pg, (float *) aoffset1);
|
|
||||||
svst1_f32(pg, (float *) boffset, a_vec);
|
|
||||||
aoffset1 += lda;
|
|
||||||
boffset += active;
|
|
||||||
}
|
|
||||||
aoffset += sve_size;
|
|
||||||
|
|
||||||
j += svcntw();
|
|
||||||
pg = svwhilelt_b32(j, n);
|
|
||||||
active = svcntp_b32(svptrue_b32(), pg);
|
|
||||||
|
|
||||||
} while (svptest_any(svptrue_b32(), pg));
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
6
param.h
6
param.h
|
|
@ -3380,10 +3380,10 @@ is a big desktop or server with abundant cache rather than a phone or embedded d
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#define SGEMM_DEFAULT_UNROLL_M 16
|
#define SGEMM_DEFAULT_UNROLL_M 16
|
||||||
#define SGEMM_DEFAULT_UNROLL_N 4
|
#define SGEMM_DEFAULT_UNROLL_N 8
|
||||||
|
|
||||||
#define DGEMM_DEFAULT_UNROLL_M 8
|
#define DGEMM_DEFAULT_UNROLL_M 4 // Actually 2VL (8) but kept separate to keep copies separate
|
||||||
#define DGEMM_DEFAULT_UNROLL_N 4
|
#define DGEMM_DEFAULT_UNROLL_N 8
|
||||||
|
|
||||||
#define CGEMM_DEFAULT_UNROLL_M 8
|
#define CGEMM_DEFAULT_UNROLL_M 8
|
||||||
#define CGEMM_DEFAULT_UNROLL_N 4
|
#define CGEMM_DEFAULT_UNROLL_N 4
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue