floraachy
/
OpenBLAS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop'

This commit is contained in:
Zhang Xianyi 2014-08-18 11:16:14 +08:00
parent f20c0f9819 a69dd3fbc5
commit a7126c2ce4
78 changed files with 19008 additions and 2465 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CONTRIBUTORS.md
View File

@ -114,5 +114,8 @@ In chronological order:
* carlkl <https://github.com/carlkl>
* [2013-12-13] Fixed LAPACKE building bug on Windows
* Isaac Dunham <https://github.com/idunham>
* [2014-08-03] Fixed link error on Linux/musl
* [Your name or handle] <[email or website]>
* [Date] [Brief summary of your changes]

15
Changelog.txt
View File

@ -1,4 +1,19 @@
OpenBLAS ChangeLog
====================================================================
Version 0.2.10
18-Aug-2014
common:
* Added some benchmark codes.
* Fix link error on Linux/musl.(Thanks Isaac Dunham)
x86/x86-64:
* Improved s/c/zgemm performance for Intel Haswell.
* Improved s/d/c/zgemv performance.
* Support the big numa machine.(EXPERIMENT)
ARM:
* Fix detection when cpuinfo uses "Processor". (Thanks Isaiah)
====================================================================
Version 0.2.10
16-Jul-2014

2
Makefile
View File

@ -247,7 +247,7 @@ ifndef NOFORTRAN
-@echo "SUFFIX = $(SUFFIX)" >> $(NETLIB_LAPACK_DIR)/make.inc
-@echo "PSUFFIX = $(PSUFFIX)" >> $(NETLIB_LAPACK_DIR)/make.inc
-@echo "CEXTRALIB = $(EXTRALIB)" >> $(NETLIB_LAPACK_DIR)/make.inc
ifeq ($(FC), GFORTRAN)
ifeq ($(FC), gfortran)
-@echo "TIMER = INT_ETIME" >> $(NETLIB_LAPACK_DIR)/make.inc
ifdef SMP
-@echo "LOADER = $(FC) -pthread" >> $(NETLIB_LAPACK_DIR)/make.inc

3
Makefile.install
View File

@ -79,7 +79,8 @@ endif
ifeq ($(OSNAME), Darwin)
@-cp $(LIBDYNNAME) $(DESTDIR)$(OPENBLAS_LIBRARY_DIR)
@-install_name_tool -id $(DESTDIR)$(OPENBLAS_LIBRARY_DIR)/$(LIBDYNNAME) $(DESTDIR)$(OPENBLAS_LIBRARY_DIR)/$(LIBDYNNAME)
@-ln -fs $(DESTDIR)$(OPENBLAS_LIBRARY_DIR)/$(LIBDYNNAME) $(DESTDIR)$(OPENBLAS_LIBRARY_DIR)/$(LIBPREFIX).dylib
@cd $(DESTDIR)$(OPENBLAS_LIBRARY_DIR) ; \
ln -fs $(LIBDYNNAME) $(LIBPREFIX).dylib
endif
ifeq ($(OSNAME), WINNT)
@-cp $(LIBDLLNAME) $(OPENBLAS_BINARY_DIR)

5
Makefile.rule
View File

@ -3,7 +3,7 @@
#
# This library's version
VERSION = 0.2.10
VERSION = 0.2.11
# If you set the suffix, the library name will be libopenblas_$(LIBNAMESUFFIX).a
# and libopenblas_$(LIBNAMESUFFIX).so. Meanwhile, the soname in shared library
@ -95,6 +95,9 @@ NO_WARMUP = 1
# If you want to disable CPU/Memory affinity on Linux.
NO_AFFINITY = 1
# if you are compiling for Linux and you have more than 16 numa nodes or more than 256 cpus
# BIGNUMA = 1
# Don't use AVX kernel on Sandy Bridge. It is compatible with old compilers
# and OS. However, the performance is low.
# NO_AVX = 1

4
Makefile.system
View File

@ -803,6 +803,10 @@ ifeq ($(USE_OPENMP), 1)
CCOMMON_OPT += -DUSE_OPENMP
endif
ifeq ($(BIGNUMA), 1)
CCOMMON_OPT += -DBIGNUMA
endif
endif
ifeq ($(NO_WARMUP), 1)

373
benchmark/Makefile
View File

@ -2,12 +2,12 @@ TOPDIR = ..
include $(TOPDIR)/Makefile.system
# ACML standard
ACML=/opt/acml5.3.1/gfortran64_mp/lib
LIBACML = -fopenmp $(ACML)/libacml_mp.a -lgfortran -lm
#ACML=/opt/acml5.3.1/gfortran64_mp/lib
#LIBACML = -fopenmp $(ACML)/libacml_mp.a -lgfortran -lm
# ACML custom
#ACML=/opt/pb/acml-5-3-1-gfortran-64bit/gfortran64_fma4_mp/lib
#LIBACML = -fopenmp $(ACML)/libacml_mp.a -lgfortran -lm
ACML=/opt/pb/acml-5-3-1-gfortran-64bit/gfortran64_fma4_mp/lib
LIBACML = -fopenmp $(ACML)/libacml_mp.a -lgfortran -lm
# Atlas Ubuntu
#ATLAS=/usr/lib/atlas-base
@ -18,12 +18,12 @@ ATLAS=/usr/lib64/atlas
LIBATLAS = -fopenmp $(ATLAS)/liblapack.a $(ATLAS)/libptcblas.a $(ATLAS)/libptf77blas.a $(ATLAS)/libatlas.a -lgfortran -lm
# Intel standard
MKL=/opt/intel/mkl/lib/intel64
LIBMKL = -L$(MKL) -lmkl_intel_lp64 -lmkl_gnu_thread -lmkl_core -lgomp -lpthread -lm
# MKL=/opt/intel/mkl/lib/intel64
# LIBMKL = -L$(MKL) -lmkl_intel_lp64 -lmkl_gnu_thread -lmkl_core -lgomp -lpthread -lm
# Intel custom
#MKL=/home/saar/intel_mkl
#LIBMKL = -L$(MKL) -lmkl_intel_lp64 -lmkl_gnu_thread -lmkl_core -lgomp -lpthread -lm
MKL=/home/saar/intel_mkl
LIBMKL = -L$(MKL) -lmkl_intel_lp64 -lmkl_gnu_thread -lmkl_core -lgomp -lpthread -lm
@ -34,9 +34,15 @@ goto :: slinpack.goto dlinpack.goto clinpack.goto zlinpack.goto \
strsm.goto dtrsm.goto ctrsm.goto ztrsm.goto \
ssyrk.goto dsyrk.goto csyrk.goto zsyrk.goto \
ssyr2k.goto dsyr2k.goto csyr2k.goto zsyr2k.goto \
sger.goto dger.goto \
ssymv.goto dsymv.goto \
chemm.goto zhemm.goto \
cherk.goto zherk.goto \
cher2k.goto zher2k.goto \
sgemv.goto dgemv.goto cgemv.goto zgemv.goto \
sgeev.goto dgeev.goto cgeev.goto zgeev.goto \
sgetri.goto dgetri.goto cgetri.goto zgetri.goto \
spotrf.goto dpotrf.goto cpotrf.goto zpotrf.goto \
ssymm.goto dsymm.goto csymm.goto zsymm.goto
acml :: slinpack.acml dlinpack.acml clinpack.acml zlinpack.acml \
@ -46,9 +52,15 @@ acml :: slinpack.acml dlinpack.acml clinpack.acml zlinpack.acml \
strsm.acml dtrsm.acml ctrsm.acml ztrsm.acml \
ssyrk.acml dsyrk.acml csyrk.acml zsyrk.acml \
ssyr2k.acml dsyr2k.acml csyr2k.acml zsyr2k.acml \
sger.acml dger.acml \
ssymv.acml dsymv.acml \
chemm.acml zhemm.acml \
cherk.acml zherk.acml \
cher2k.acml zher2k.acml \
sgemv.acml dgemv.acml cgemv.acml zgemv.acml \
sgeev.acml dgeev.acml cgeev.acml zgeev.acml \
sgetri.acml dgetri.acml cgetri.acml zgetri.acml \
spotrf.acml dpotrf.acml cpotrf.acml zpotrf.acml \
ssymm.acml dsymm.acml csymm.acml zsymm.acml
atlas :: slinpack.atlas dlinpack.atlas clinpack.atlas zlinpack.atlas \
@ -58,9 +70,16 @@ atlas :: slinpack.atlas dlinpack.atlas clinpack.atlas zlinpack.atlas \
strsm.atlas dtrsm.atlas ctrsm.atlas ztrsm.atlas \
ssyrk.atlas dsyrk.atlas csyrk.atlas zsyrk.atlas \
ssyr2k.atlas dsyr2k.atlas csyr2k.atlas zsyr2k.atlas \
sger.atlas dger.atlas \
ssymv.atlas dsymv.atlas \
chemm.acml zhemm.acml \
chemm.atlas zhemm.atlas \
cherk.atlas zherk.atlas \
cher2k.atlas zher2k.atlas \
sgemv.atlas dgemv.atlas cgemv.atlas zgemv.atlas \
sgeev.atlas dgeev.atlas cgeev.atlas zgeev.atlas \
sgetri.atlas dgetri.atlas cgetri.atlas zgetri.atlas \
spotrf.atlas dpotrf.atlas cpotrf.atlas zpotrf.atlas \
ssymm.atlas dsymm.atlas csymm.atlas zsymm.atlas
mkl :: slinpack.mkl dlinpack.mkl clinpack.mkl zlinpack.mkl \
@ -70,9 +89,15 @@ mkl :: slinpack.mkl dlinpack.mkl clinpack.mkl zlinpack.mkl \
strsm.mkl dtrsm.mkl ctrsm.mkl ztrsm.mkl \
ssyrk.mkl dsyrk.mkl csyrk.mkl zsyrk.mkl \
ssyr2k.mkl dsyr2k.mkl csyr2k.mkl zsyr2k.mkl \
sger.mkl dger.mkl \
ssymv.mkl dsymv.mkl \
chemm.mkl zhemm.mkl \
cherk.mkl zherk.mkl \
cher2k.mkl zher2k.mkl \
sgemv.mkl dgemv.mkl cgemv.mkl zgemv.mkl \
sgeev.mkl dgeev.mkl cgeev.mkl zgeev.mkl \
sgetri.mkl dgetri.mkl cgetri.mkl zgetri.mkl \
spotrf.mkl dpotrf.mkl cpotrf.mkl zpotrf.mkl \
ssymm.mkl dsymm.mkl csymm.mkl zsymm.mkl
all :: goto atlas acml mkl
@ -601,6 +626,277 @@ zher2k.atlas : zher2k.$(SUFFIX)
zher2k.mkl : zher2k.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBMKL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
##################################### Sgemv ####################################################
sgemv.goto : sgemv.$(SUFFIX) ../$(LIBNAME)
$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) -lm
sgemv.acml : sgemv.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBACML) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
sgemv.atlas : sgemv.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBATLAS) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
sgemv.mkl : sgemv.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBMKL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
##################################### Dgemv ####################################################
dgemv.goto : dgemv.$(SUFFIX) ../$(LIBNAME)
$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) -lm
dgemv.acml : dgemv.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBACML) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
dgemv.atlas : dgemv.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBATLAS) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
dgemv.mkl : dgemv.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBMKL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
##################################### Cgemv ####################################################
cgemv.goto : cgemv.$(SUFFIX) ../$(LIBNAME)
$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) -lm
cgemv.acml : cgemv.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBACML) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
cgemv.atlas : cgemv.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBATLAS) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
cgemv.mkl : cgemv.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBMKL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
##################################### Zgemv ####################################################
zgemv.goto : zgemv.$(SUFFIX) ../$(LIBNAME)
$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) -lm
zgemv.acml : zgemv.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBACML) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
zgemv.atlas : zgemv.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBATLAS) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
zgemv.mkl : zgemv.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBMKL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
##################################### Sger ####################################################
sger.goto : sger.$(SUFFIX) ../$(LIBNAME)
$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) -lm
sger.acml : sger.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBACML) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
sger.atlas : sger.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBATLAS) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
sger.mkl : sger.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBMKL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
##################################### Dger ####################################################
dger.goto : dger.$(SUFFIX) ../$(LIBNAME)
$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) -lm
dger.acml : dger.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBACML) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
dger.atlas : dger.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBATLAS) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
dger.mkl : dger.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBMKL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
##################################### Ssymv ####################################################
ssymv.goto : ssymv.$(SUFFIX) ../$(LIBNAME)
$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) -lm
ssymv.acml : ssymv.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBACML) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
ssymv.atlas : ssymv.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBATLAS) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
ssymv.mkl : ssymv.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBMKL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
##################################### Dsymv ####################################################
dsymv.goto : dsymv.$(SUFFIX) ../$(LIBNAME)
$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) -lm
dsymv.acml : dsymv.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBACML) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
dsymv.atlas : dsymv.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBATLAS) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
dsymv.mkl : dsymv.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBMKL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
##################################### Sgeev ####################################################
sgeev.goto : sgeev.$(SUFFIX) ../$(LIBNAME)
$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) -lm
sgeev.acml : sgeev.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBACML) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
sgeev.atlas : sgeev.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBATLAS) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
sgeev.mkl : sgeev.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBMKL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
##################################### Dgeev ####################################################
dgeev.goto : dgeev.$(SUFFIX) ../$(LIBNAME)
$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) -lm
dgeev.acml : dgeev.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBACML) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
dgeev.atlas : dgeev.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBATLAS) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
dgeev.mkl : dgeev.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBMKL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
##################################### Cgeev ####################################################
cgeev.goto : cgeev.$(SUFFIX) ../$(LIBNAME)
$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) -lm
cgeev.acml : cgeev.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBACML) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
cgeev.atlas : cgeev.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBATLAS) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
cgeev.mkl : cgeev.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBMKL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
##################################### Zgeev ####################################################
zgeev.goto : zgeev.$(SUFFIX) ../$(LIBNAME)
$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) -lm
zgeev.acml : zgeev.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBACML) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
zgeev.atlas : zgeev.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBATLAS) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
zgeev.mkl : zgeev.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBMKL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
##################################### Sgetri ####################################################
sgetri.goto : sgetri.$(SUFFIX) ../$(LIBNAME)
$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) -lm
sgetri.acml : sgetri.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBACML) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
sgetri.atlas : sgetri.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBATLAS) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
sgetri.mkl : sgetri.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBMKL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
##################################### Dgetri ####################################################
dgetri.goto : dgetri.$(SUFFIX) ../$(LIBNAME)
$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) -lm
dgetri.acml : dgetri.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBACML) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
dgetri.atlas : dgetri.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBATLAS) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
dgetri.mkl : dgetri.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBMKL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
##################################### Cgetri ####################################################
cgetri.goto : cgetri.$(SUFFIX) ../$(LIBNAME)
$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) -lm
cgetri.acml : cgetri.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBACML) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
cgetri.atlas : cgetri.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBATLAS) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
cgetri.mkl : cgetri.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBMKL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
##################################### Zgetri ####################################################
zgetri.goto : zgetri.$(SUFFIX) ../$(LIBNAME)
$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) -lm
zgetri.acml : zgetri.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBACML) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
zgetri.atlas : zgetri.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBATLAS) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
zgetri.mkl : zgetri.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBMKL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
##################################### Spotrf ####################################################
spotrf.goto : spotrf.$(SUFFIX) ../$(LIBNAME)
$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) -lm
spotrf.acml : spotrf.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBACML) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
spotrf.atlas : spotrf.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBATLAS) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
spotrf.mkl : spotrf.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBMKL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
##################################### Dpotrf ####################################################
dpotrf.goto : dpotrf.$(SUFFIX) ../$(LIBNAME)
$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) -lm
dpotrf.acml : dpotrf.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBACML) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
dpotrf.atlas : dpotrf.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBATLAS) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
dpotrf.mkl : dpotrf.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBMKL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
##################################### Cpotrf ####################################################
cpotrf.goto : cpotrf.$(SUFFIX) ../$(LIBNAME)
$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) -lm
cpotrf.acml : cpotrf.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBACML) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
cpotrf.atlas : cpotrf.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBATLAS) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
cpotrf.mkl : cpotrf.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBMKL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
##################################### Zpotrf ####################################################
zpotrf.goto : zpotrf.$(SUFFIX) ../$(LIBNAME)
$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) -lm
zpotrf.acml : zpotrf.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBACML) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
zpotrf.atlas : zpotrf.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBATLAS) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
zpotrf.mkl : zpotrf.$(SUFFIX)
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBMKL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
###################################################################################################
slinpack.$(SUFFIX) : linpack.c
@ -717,6 +1013,67 @@ cher2k.$(SUFFIX) : her2k.c
zher2k.$(SUFFIX) : her2k.c
$(CC) $(CFLAGS) -c -DCOMPLEX -DDOUBLE -o $(@F) $^
sgemv.$(SUFFIX) : gemv.c
$(CC) $(CFLAGS) -c -UCOMPLEX -UDOUBLE -o $(@F) $^
dgemv.$(SUFFIX) : gemv.c
$(CC) $(CFLAGS) -c -UCOMPLEX -DDOUBLE -o $(@F) $^
cgemv.$(SUFFIX) : gemv.c
$(CC) $(CFLAGS) -c -DCOMPLEX -UDOUBLE -o $(@F) $^
zgemv.$(SUFFIX) : gemv.c
$(CC) $(CFLAGS) -c -DCOMPLEX -DDOUBLE -o $(@F) $^
sger.$(SUFFIX) : ger.c
$(CC) $(CFLAGS) -c -UCOMPLEX -UDOUBLE -o $(@F) $^
dger.$(SUFFIX) : ger.c
$(CC) $(CFLAGS) -c -UCOMPLEX -DDOUBLE -o $(@F) $^
ssymv.$(SUFFIX) : symv.c
$(CC) $(CFLAGS) -c -UCOMPLEX -UDOUBLE -o $(@F) $^
dsymv.$(SUFFIX) : symv.c
$(CC) $(CFLAGS) -c -UCOMPLEX -DDOUBLE -o $(@F) $^
sgeev.$(SUFFIX) : geev.c
$(CC) $(CFLAGS) -c -UCOMPLEX -UDOUBLE -o $(@F) $^
dgeev.$(SUFFIX) : geev.c
$(CC) $(CFLAGS) -c -UCOMPLEX -DDOUBLE -o $(@F) $^
cgeev.$(SUFFIX) : geev.c
$(CC) $(CFLAGS) -c -DCOMPLEX -UDOUBLE -o $(@F) $^
zgeev.$(SUFFIX) : geev.c
$(CC) $(CFLAGS) -c -DCOMPLEX -DDOUBLE -o $(@F) $^
sgetri.$(SUFFIX) : getri.c
$(CC) $(CFLAGS) -c -UCOMPLEX -UDOUBLE -o $(@F) $^
dgetri.$(SUFFIX) : getri.c
$(CC) $(CFLAGS) -c -UCOMPLEX -DDOUBLE -o $(@F) $^
cgetri.$(SUFFIX) : getri.c
$(CC) $(CFLAGS) -c -DCOMPLEX -UDOUBLE -o $(@F) $^
zgetri.$(SUFFIX) : getri.c
$(CC) $(CFLAGS) -c -DCOMPLEX -DDOUBLE -o $(@F) $^
spotrf.$(SUFFIX) : potrf.c
$(CC) $(CFLAGS) -c -UCOMPLEX -UDOUBLE -o $(@F) $^
dpotrf.$(SUFFIX) : potrf.c
$(CC) $(CFLAGS) -c -UCOMPLEX -DDOUBLE -o $(@F) $^
cpotrf.$(SUFFIX) : potrf.c
$(CC) $(CFLAGS) -c -DCOMPLEX -UDOUBLE -o $(@F) $^
zpotrf.$(SUFFIX) : potrf.c
$(CC) $(CFLAGS) -c -DCOMPLEX -DDOUBLE -o $(@F) $^

4
benchmark/cholesky.c
View File

@ -119,7 +119,11 @@ static __inline double getmflops(int ratio, int m, double secs){
int MAIN__(int argc, char *argv[]){
#ifndef COMPLEX
char *trans[] = {"T", "N"};
#else
char *trans[] = {"C", "N"};
#endif
char *uplo[] = {"U", "L"};
FLOAT alpha[] = {1.0, 0.0};
FLOAT beta [] = {0.0, 0.0};

260
benchmark/geev.c Normal file
View File

@ -0,0 +1,260 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/
#include <stdio.h>
#include <stdlib.h>
#ifdef __CYGWIN32__
#include <sys/time.h>
#endif
#include "common.h"
#undef GEEV
#ifndef COMPLEX
#ifdef XDOUBLE
#define GEEV BLASFUNC(qgeev)
#elif defined(DOUBLE)
#define GEEV BLASFUNC(dgeev)
#else
#define GEEV BLASFUNC(sgeev)
#endif
#else
#ifdef XDOUBLE
#define GEEV BLASFUNC(xgeev)
#elif defined(DOUBLE)
#define GEEV BLASFUNC(zgeev)
#else
#define GEEV BLASFUNC(cgeev)
#endif
#endif
#ifndef COMPLEX
extern void GEEV( char* jobvl, char* jobvr, blasint* n, FLOAT* a,
blasint* lda, FLOAT* wr, FLOAT* wi, FLOAT* vl, blasint* ldvl,
FLOAT* vr, blasint* ldvr, FLOAT* work, blasint* lwork, blasint* info );
#else
extern void GEEV( char* jobvl, char* jobvr, blasint* n, FLOAT* a,
blasint* lda, FLOAT* wr, FLOAT* vl, blasint* ldvl,
FLOAT* vr, blasint* ldvr, FLOAT* work, blasint* lwork, FLOAT *rwork, blasint* info );
#endif
#if defined(__WIN32__) || defined(__WIN64__)
#ifndef DELTA_EPOCH_IN_MICROSECS
#define DELTA_EPOCH_IN_MICROSECS 11644473600000000ULL
#endif
int gettimeofday(struct timeval *tv, void *tz){
FILETIME ft;
unsigned __int64 tmpres = 0;
static int tzflag;
if (NULL != tv)
{
GetSystemTimeAsFileTime(&ft);
tmpres |= ft.dwHighDateTime;
tmpres <<= 32;
tmpres |= ft.dwLowDateTime;
/*converting file time to unix epoch*/
tmpres /= 10; /*convert into microseconds*/
tmpres -= DELTA_EPOCH_IN_MICROSECS;
tv->tv_sec = (long)(tmpres / 1000000UL);
tv->tv_usec = (long)(tmpres % 1000000UL);
}
return 0;
}
#endif
#if !defined(__WIN32__) && !defined(__WIN64__) && !defined(__CYGWIN32__) && 0
static void *huge_malloc(BLASLONG size){
int shmid;
void *address;
#ifndef SHM_HUGETLB
#define SHM_HUGETLB 04000
#endif
if ((shmid =shmget(IPC_PRIVATE,
(size + HUGE_PAGESIZE) & ~(HUGE_PAGESIZE - 1),
SHM_HUGETLB | IPC_CREAT |0600)) < 0) {
printf( "Memory allocation failed(shmget).\n");
exit(1);
}
address = shmat(shmid, NULL, SHM_RND);
if ((BLASLONG)address == -1){
printf( "Memory allocation failed(shmat).\n");
exit(1);
}
shmctl(shmid, IPC_RMID, 0);
return address;
}
#define malloc huge_malloc
#endif
int MAIN__(int argc, char *argv[]){
FLOAT *a,*vl,*vr,*wi,*wr,*work,*rwork;
FLOAT wkopt[4];
char job='V';
char *p;
blasint m, i, j, info,lwork;
double factor = 26.33;
int from = 1;
int to = 200;
int step = 1;
struct timeval start, stop;
double time1;
argc--;argv++;
if (argc > 0) { from = atol(*argv); argc--; argv++;}
if (argc > 0) { to = MAX(atol(*argv), from); argc--; argv++;}
if (argc > 0) { step = atol(*argv); argc--; argv++;}
if ((p = getenv("OPENBLAS_JOB"))) job=*p;
if ( job == 'N' ) factor = 10.0;
fprintf(stderr, "From : %3d To : %3d Step = %3d Job=%c\n", from, to, step,job);
if (( a = (FLOAT *)malloc(sizeof(FLOAT) * to * to * COMPSIZE)) == NULL){
fprintf(stderr,"Out of Memory!!\n");exit(1);
}
if (( vl = (FLOAT *)malloc(sizeof(FLOAT) * to * to * COMPSIZE)) == NULL){
fprintf(stderr,"Out of Memory!!\n");exit(1);
}
if (( vr = (FLOAT *)malloc(sizeof(FLOAT) * to * to * COMPSIZE)) == NULL){
fprintf(stderr,"Out of Memory!!\n");exit(1);
}
if (( wr = (FLOAT *)malloc(sizeof(FLOAT) * to * COMPSIZE)) == NULL){
fprintf(stderr,"Out of Memory!!\n");exit(1);
}
if (( wi = (FLOAT *)malloc(sizeof(FLOAT) * to * COMPSIZE)) == NULL){
fprintf(stderr,"Out of Memory!!\n");exit(1);
}
if (( rwork = (FLOAT *)malloc(sizeof(FLOAT) * to * COMPSIZE)) == NULL){
fprintf(stderr,"Out of Memory!!\n");exit(1);
}
for(j = 0; j < to; j++){
for(i = 0; i < to * COMPSIZE; i++){
a[i + j * to * COMPSIZE] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
}
}
lwork = -1;
m=to;
#ifndef COMPLEX
GEEV (&job, &job, &m, a, &m, wr, wi, vl, &m, vr, &m, wkopt, &lwork, &info);
#else
GEEV (&job, &job, &m, a, &m, wr, vl, &m, vr, &m, wkopt, &lwork,rwork, &info);
#endif
lwork = (blasint)wkopt[0];
if (( work = (FLOAT *)malloc(sizeof(FLOAT) * lwork * COMPSIZE)) == NULL){
fprintf(stderr,"Out of Memory!!\n");exit(1);
}
#ifdef linux
srandom(getpid());
#endif
fprintf(stderr, " SIZE FLops Time Lwork\n");
for(m = from; m <= to; m += step){
fprintf(stderr, " %6d : ", (int)m);
gettimeofday( &start, (struct timezone *)0);
lwork = -1;
#ifndef COMPLEX
GEEV (&job, &job, &m, a, &m, wr, wi, vl, &m, vr, &m, wkopt, &lwork, &info);
#else
GEEV (&job, &job, &m, a, &m, wr, vl, &m, vr, &m, wkopt, &lwork,rwork, &info);
#endif
lwork = (blasint)wkopt[0];
#ifndef COMPLEX
GEEV (&job, &job, &m, a, &m, wr, wi, vl, &m, vr, &m, work, &lwork, &info);
#else
GEEV (&job, &job, &m, a, &m, wr, vl, &m, vr, &m, work, &lwork,rwork, &info);
#endif
gettimeofday( &stop, (struct timezone *)0);
if (info) {
fprintf(stderr, "failed to compute eigenvalues .. %d\n", info);
exit(1);
}
time1 = (double)(stop.tv_sec - start.tv_sec) + (double)((stop.tv_usec - start.tv_usec)) * 1.e-6;
fprintf(stderr,
" %10.2f MFlops : %10.2f Sec : %d\n",
COMPSIZE * COMPSIZE * factor * (double)m * (double)m * (double)m / time1 * 1.e-6,time1,lwork);
}
return 0;
}
void main(int argc, char *argv[]) __attribute__((weak, alias("MAIN__")));

229
benchmark/gemv.c Normal file
View File

@ -0,0 +1,229 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#include <stdio.h>
#include <stdlib.h>
#ifdef __CYGWIN32__
#include <sys/time.h>
#endif
#include "common.h"
#undef GEMV
#ifndef COMPLEX
#ifdef DOUBLE
#define GEMV BLASFUNC(dgemv)
#else
#define GEMV BLASFUNC(sgemv)
#endif
#else
#ifdef DOUBLE
#define GEMV BLASFUNC(zgemv)
#else
#define GEMV BLASFUNC(cgemv)
#endif
#endif
#if defined(__WIN32__) || defined(__WIN64__)
#ifndef DELTA_EPOCH_IN_MICROSECS
#define DELTA_EPOCH_IN_MICROSECS 11644473600000000ULL
#endif
int gettimeofday(struct timeval *tv, void *tz){
FILETIME ft;
unsigned __int64 tmpres = 0;
static int tzflag;
if (NULL != tv)
{
GetSystemTimeAsFileTime(&ft);
tmpres |= ft.dwHighDateTime;
tmpres <<= 32;
tmpres |= ft.dwLowDateTime;
/*converting file time to unix epoch*/
tmpres /= 10; /*convert into microseconds*/
tmpres -= DELTA_EPOCH_IN_MICROSECS;
tv->tv_sec = (long)(tmpres / 1000000UL);
tv->tv_usec = (long)(tmpres % 1000000UL);
}
return 0;
}
#endif
#if !defined(__WIN32__) && !defined(__WIN64__) && !defined(__CYGWIN32__) && 0
static void *huge_malloc(BLASLONG size){
int shmid;
void *address;
#ifndef SHM_HUGETLB
#define SHM_HUGETLB 04000
#endif
if ((shmid =shmget(IPC_PRIVATE,
(size + HUGE_PAGESIZE) & ~(HUGE_PAGESIZE - 1),
SHM_HUGETLB | IPC_CREAT |0600)) < 0) {
printf( "Memory allocation failed(shmget).\n");
exit(1);
}
address = shmat(shmid, NULL, SHM_RND);
if ((BLASLONG)address == -1){
printf( "Memory allocation failed(shmat).\n");
exit(1);
}
shmctl(shmid, IPC_RMID, 0);
return address;
}
#define malloc huge_malloc
#endif
int MAIN__(int argc, char *argv[]){
FLOAT *a, *x, *y;
FLOAT alpha[] = {1.0, 1.0};
FLOAT beta [] = {1.0, 1.0};
char trans='N';
blasint m, i, j;
blasint inc_x=1,inc_y=1;
blasint n=0;
int has_param_n = 0;
int loops = 1;
int l;
char *p;
int from = 1;
int to = 200;
int step = 1;
struct timeval start, stop;
double time1,timeg;
argc--;argv++;
if (argc > 0) { from = atol(*argv); argc--; argv++;}
if (argc > 0) { to = MAX(atol(*argv), from); argc--; argv++;}
if (argc > 0) { step = atol(*argv); argc--; argv++;}
if ((p = getenv("OPENBLAS_LOOPS"))) loops = atoi(p);
if ((p = getenv("OPENBLAS_INCX"))) inc_x = atoi(p);
if ((p = getenv("OPENBLAS_INCY"))) inc_y = atoi(p);
if ((p = getenv("OPENBLAS_TRANS"))) trans=*p;
if ((p = getenv("OPENBLAS_PARAM_N"))) {
n = atoi(p);
if ((n>0) && (n<=to)) has_param_n = 1;
}
if ( has_param_n == 1 )
fprintf(stderr, "From : %3d To : %3d Step = %3d Trans = '%c' N = %d Inc_x = %d Inc_y = %d Loops = %d\n", from, to, step,trans,n,inc_x,inc_y,loops);
else
fprintf(stderr, "From : %3d To : %3d Step = %3d Trans = '%c' Inc_x = %d Inc_y = %d Loops = %d\n", from, to, step,trans,inc_x,inc_y,loops);
if (( a = (FLOAT *)malloc(sizeof(FLOAT) * to * to * COMPSIZE)) == NULL){
fprintf(stderr,"Out of Memory!!\n");exit(1);
}
if (( x = (FLOAT *)malloc(sizeof(FLOAT) * to * abs(inc_x) * COMPSIZE)) == NULL){
fprintf(stderr,"Out of Memory!!\n");exit(1);
}
if (( y = (FLOAT *)malloc(sizeof(FLOAT) * to * abs(inc_y) * COMPSIZE)) == NULL){
fprintf(stderr,"Out of Memory!!\n");exit(1);
}
#ifdef linux
srandom(getpid());
#endif
fprintf(stderr, " SIZE Flops\n");
for(m = from; m <= to; m += step)
{
timeg=0;
if ( has_param_n == 0 ) n = m;
fprintf(stderr, " %6dx%d : ", (int)m,(int)n);
for(j = 0; j < m; j++){
for(i = 0; i < n * COMPSIZE; i++){
a[i + j * m * COMPSIZE] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
}
}
for (l=0; l<loops; l++)
{
for(i = 0; i < n * COMPSIZE * abs(inc_x); i++){
x[i] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
}
for(i = 0; i < n * COMPSIZE * abs(inc_y); i++){
y[i] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
}
gettimeofday( &start, (struct timezone *)0);
GEMV (&trans, &m, &n, alpha, a, &m, x, &inc_x, beta, y, &inc_y );
gettimeofday( &stop, (struct timezone *)0);
time1 = (double)(stop.tv_sec - start.tv_sec) + (double)((stop.tv_usec - start.tv_usec)) * 1.e-6;
timeg += time1;
}
timeg /= loops;
fprintf(stderr,
" %10.2f MFlops\n",
COMPSIZE * COMPSIZE * 2. * (double)m * (double)n / timeg * 1.e-6);
}
return 0;
}
void main(int argc, char *argv[]) __attribute__((weak, alias("MAIN__")));

218
benchmark/ger.c Normal file
View File

@ -0,0 +1,218 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#include <stdio.h>
#include <stdlib.h>
#ifdef __CYGWIN32__
#include <sys/time.h>
#endif
#include "common.h"
#undef GER
#ifdef DOUBLE
#define GER BLASFUNC(dger)
#else
#define GER BLASFUNC(sger)
#endif
#if defined(__WIN32__) || defined(__WIN64__)
#ifndef DELTA_EPOCH_IN_MICROSECS
#define DELTA_EPOCH_IN_MICROSECS 11644473600000000ULL
#endif
int gettimeofday(struct timeval *tv, void *tz){
FILETIME ft;
unsigned __int64 tmpres = 0;
static int tzflag;
if (NULL != tv)
{
GetSystemTimeAsFileTime(&ft);
tmpres |= ft.dwHighDateTime;
tmpres <<= 32;
tmpres |= ft.dwLowDateTime;
/*converting file time to unix epoch*/
tmpres /= 10; /*convert into microseconds*/
tmpres -= DELTA_EPOCH_IN_MICROSECS;
tv->tv_sec = (long)(tmpres / 1000000UL);
tv->tv_usec = (long)(tmpres % 1000000UL);
}
return 0;
}
#endif
#if !defined(__WIN32__) && !defined(__WIN64__) && !defined(__CYGWIN32__) && 0
static void *huge_malloc(BLASLONG size){
int shmid;
void *address;
#ifndef SHM_HUGETLB
#define SHM_HUGETLB 04000
#endif
if ((shmid =shmget(IPC_PRIVATE,
(size + HUGE_PAGESIZE) & ~(HUGE_PAGESIZE - 1),
SHM_HUGETLB | IPC_CREAT |0600)) < 0) {
printf( "Memory allocation failed(shmget).\n");
exit(1);
}
address = shmat(shmid, NULL, SHM_RND);
if ((BLASLONG)address == -1){
printf( "Memory allocation failed(shmat).\n");
exit(1);
}
shmctl(shmid, IPC_RMID, 0);
return address;
}
#define malloc huge_malloc
#endif
int MAIN__(int argc, char *argv[]){
FLOAT *a, *x, *y;
FLOAT alpha[] = {1.0, 1.0};
blasint m, i, j;
blasint inc_x=1,inc_y=1;
blasint n=0;
int has_param_n = 0;
int loops = 1;
int l;
char *p;
int from = 1;
int to = 200;
int step = 1;
struct timeval start, stop;
double time1,timeg;
argc--;argv++;
if (argc > 0) { from = atol(*argv); argc--; argv++;}
if (argc > 0) { to = MAX(atol(*argv), from); argc--; argv++;}
if (argc > 0) { step = atol(*argv); argc--; argv++;}
if ((p = getenv("OPENBLAS_LOOPS"))) loops = atoi(p);
if ((p = getenv("OPENBLAS_INCX"))) inc_x = atoi(p);
if ((p = getenv("OPENBLAS_INCY"))) inc_y = atoi(p);
if ((p = getenv("OPENBLAS_PARAM_N"))) {
n = atoi(p);
if ((n>0) && (n<=to)) has_param_n = 1;
}
if ( has_param_n == 1 )
fprintf(stderr, "From : %3d To : %3d Step = %3d N = %d Inc_x = %d Inc_y = %d Loops = %d\n", from, to, step,n,inc_x,inc_y,loops);
else
fprintf(stderr, "From : %3d To : %3d Step = %3d Inc_x = %d Inc_y = %d Loops = %d\n", from, to, step,inc_x,inc_y,loops);
if (( a = (FLOAT *)malloc(sizeof(FLOAT) * to * to * COMPSIZE)) == NULL){
fprintf(stderr,"Out of Memory!!\n");exit(1);
}
if (( x = (FLOAT *)malloc(sizeof(FLOAT) * to * abs(inc_x) * COMPSIZE)) == NULL){
fprintf(stderr,"Out of Memory!!\n");exit(1);
}
if (( y = (FLOAT *)malloc(sizeof(FLOAT) * to * abs(inc_y) * COMPSIZE)) == NULL){
fprintf(stderr,"Out of Memory!!\n");exit(1);
}
#ifdef linux
srandom(getpid());
#endif
fprintf(stderr, " SIZE Flops\n");
for(m = from; m <= to; m += step)
{
timeg=0;
if ( has_param_n == 0 ) n = m;
fprintf(stderr, " %6dx%d : ", (int)m,(int)n);
for(j = 0; j < m; j++){
for(i = 0; i < n * COMPSIZE; i++){
a[i + j * m * COMPSIZE] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
}
}
for(i = 0; i < m * COMPSIZE * abs(inc_x); i++){
x[i] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
}
for(i = 0; i < n * COMPSIZE * abs(inc_y); i++){
y[i] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
}
for (l=0; l<loops; l++)
{
gettimeofday( &start, (struct timezone *)0);
GER (&m, &n, alpha, x, &inc_x, y, &inc_y, a , &m);
gettimeofday( &stop, (struct timezone *)0);
time1 = (double)(stop.tv_sec - start.tv_sec) + (double)((stop.tv_usec - start.tv_usec)) * 1.e-6;
timeg += time1;
}
timeg /= loops;
fprintf(stderr,
" %10.2f MFlops\n",
COMPSIZE * COMPSIZE * 2. * (double)m * (double)n / timeg * 1.e-6);
}
return 0;
}
void main(int argc, char *argv[]) __attribute__((weak, alias("MAIN__")));

234
benchmark/getri.c Normal file
View File

@ -0,0 +1,234 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/
#include <stdio.h>
#include <stdlib.h>
#ifdef __CYGWIN32__
#include <sys/time.h>
#endif
#include "common.h"
#undef GETRF
#undef GETRI
#ifndef COMPLEX
#ifdef XDOUBLE
#define GETRF BLASFUNC(qgetrf)
#define GETRI BLASFUNC(qgetri)
#elif defined(DOUBLE)
#define GETRF BLASFUNC(dgetrf)
#define GETRI BLASFUNC(dgetri)
#else
#define GETRF BLASFUNC(sgetrf)
#define GETRI BLASFUNC(sgetri)
#endif
#else
#ifdef XDOUBLE
#define GETRF BLASFUNC(xgetrf)
#define GETRI BLASFUNC(xgetri)
#elif defined(DOUBLE)
#define GETRF BLASFUNC(zgetrf)
#define GETRI BLASFUNC(zgetri)
#else
#define GETRF BLASFUNC(cgetrf)
#define GETRI BLASFUNC(cgetri)
#endif
#endif
extern void GETRI(blasint *m, FLOAT *a, blasint *lda, blasint *ipiv, FLOAT *work, blasint *lwork, blasint *info);
#if defined(__WIN32__) || defined(__WIN64__)
#ifndef DELTA_EPOCH_IN_MICROSECS
#define DELTA_EPOCH_IN_MICROSECS 11644473600000000ULL
#endif
int gettimeofday(struct timeval *tv, void *tz){
FILETIME ft;
unsigned __int64 tmpres = 0;
static int tzflag;
if (NULL != tv)
{
GetSystemTimeAsFileTime(&ft);
tmpres |= ft.dwHighDateTime;
tmpres <<= 32;
tmpres |= ft.dwLowDateTime;
/*converting file time to unix epoch*/
tmpres /= 10; /*convert into microseconds*/
tmpres -= DELTA_EPOCH_IN_MICROSECS;
tv->tv_sec = (long)(tmpres / 1000000UL);
tv->tv_usec = (long)(tmpres % 1000000UL);
}
return 0;
}
#endif
#if !defined(__WIN32__) && !defined(__WIN64__) && !defined(__CYGWIN32__) && 0
static void *huge_malloc(BLASLONG size){
int shmid;
void *address;
#ifndef SHM_HUGETLB
#define SHM_HUGETLB 04000
#endif
if ((shmid =shmget(IPC_PRIVATE,
(size + HUGE_PAGESIZE) & ~(HUGE_PAGESIZE - 1),
SHM_HUGETLB | IPC_CREAT |0600)) < 0) {
printf( "Memory allocation failed(shmget).\n");
exit(1);
}
address = shmat(shmid, NULL, SHM_RND);
if ((BLASLONG)address == -1){
printf( "Memory allocation failed(shmat).\n");
exit(1);
}
shmctl(shmid, IPC_RMID, 0);
return address;
}
#define malloc huge_malloc
#endif
int MAIN__(int argc, char *argv[]){
FLOAT *a,*work;
FLOAT wkopt[4];
blasint *ipiv;
blasint m, i, j, info,lwork;
int from = 1;
int to = 200;
int step = 1;
struct timeval start, stop;
double time1;
argc--;argv++;
if (argc > 0) { from = atol(*argv); argc--; argv++;}
if (argc > 0) { to = MAX(atol(*argv), from); argc--; argv++;}
if (argc > 0) { step = atol(*argv); argc--; argv++;}
fprintf(stderr, "From : %3d To : %3d Step = %3d\n", from, to, step);
if (( a = (FLOAT *)malloc(sizeof(FLOAT) * to * to * COMPSIZE)) == NULL){
fprintf(stderr,"Out of Memory!!\n");exit(1);
}
if (( ipiv = (blasint *)malloc(sizeof(blasint) * to * COMPSIZE)) == NULL){
fprintf(stderr,"Out of Memory!!\n");exit(1);
}
for(j = 0; j < to; j++){
for(i = 0; i < to * COMPSIZE; i++){
a[i + j * to * COMPSIZE] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
}
}
lwork = -1;
m=to;
GETRI(&m, a, &m, ipiv, wkopt, &lwork, &info);
lwork = (blasint)wkopt[0];
if (( work = (FLOAT *)malloc(sizeof(FLOAT) * lwork * COMPSIZE)) == NULL){
fprintf(stderr,"Out of Memory!!\n");exit(1);
}
#ifdef linux
srandom(getpid());
#endif
fprintf(stderr, " SIZE FLops Time Lwork\n");
for(m = from; m <= to; m += step){
fprintf(stderr, " %6d : ", (int)m);
GETRF (&m, &m, a, &m, ipiv, &info);
if (info) {
fprintf(stderr, "Matrix is not singular .. %d\n", info);
exit(1);
}
gettimeofday( &start, (struct timezone *)0);
lwork = -1;
GETRI(&m, a, &m, ipiv, wkopt, &lwork, &info);
lwork = (blasint)wkopt[0];
GETRI(&m, a, &m, ipiv, work, &lwork, &info);
gettimeofday( &stop, (struct timezone *)0);
if (info) {
fprintf(stderr, "failed compute inverse matrix .. %d\n", info);
exit(1);
}
time1 = (double)(stop.tv_sec - start.tv_sec) + (double)((stop.tv_usec - start.tv_usec)) * 1.e-6;
fprintf(stderr,
" %10.2f MFlops : %10.2f Sec : %d\n",
COMPSIZE * COMPSIZE * (4.0/3.0 * (double)m * (double)m *(double)m - (double)m *(double)m + 5.0/3.0* (double)m) / time1 * 1.e-6,time1,lwork);
}
return 0;
}
void main(int argc, char *argv[]) __attribute__((weak, alias("MAIN__")));

60
benchmark/plot-filter.sh Executable file
View File

@ -0,0 +1,60 @@
#!/bin/sh
# **********************************************************************************
# Copyright (c) 2014, The OpenBLAS Project
# All rights reserved.
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# 3. Neither the name of the OpenBLAS project nor the names of
# its contributors may be used to endorse or promote products
# derived from this software without specific prior written permission.
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# **********************************************************************************
# ************************************************************************
# sample filter for data output from benchmark programs
#
# usage example:
# ./dgemm.goto 2>&1|./plotfilter.sh >OpenBLAS
# ************************************************************************
if [ $# -eq 1 ]
then
arg1=$1
else
arg1=0
fi
case $arg1 in
L)
# Linpack Benchmark
awk '/MFlops/ { print $1,int($8) }'|tail --lines=+2
;;
C)
# Cholesky Benchmark
awk '/MFlops/ { print $3,int($9) }'|tail --lines=+2
;;
*)
awk '/MFlops/ { print $1,int($3) }'|tail --lines=+2
;;
esac

42
benchmark/plot-header Normal file
View File

@ -0,0 +1,42 @@
# **********************************************************************************
# Copyright (c) 2014, The OpenBLAS Project
# All rights reserved.
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# 3. Neither the name of the OpenBLAS project nor the names of
# its contributors may be used to endorse or promote products
# derived from this software without specific prior written permission.
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# **********************************************************************************
set term x11 font sans;
set ylabel "MFlops";
set xlabel "Size";
set grid xtics;
set grid ytics;
set key left;
set timestamp "generated on %Y-%m-%d by `whoami`"
set title "Dtrsm\nUPLO=U TRANS=N SIDE=L\nBulldozer 1 Thread"
plot 'OpenBLAS' smooth bezier, 'ACML' smooth bezier, 'MKL' smooth bezier;
set output "print.png";
show title;
show plot;
show output;

282
benchmark/potrf.c Normal file
View File

@ -0,0 +1,282 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/
#include <stdio.h>
#include <stdlib.h>
#ifdef __CYGWIN32__
#include <sys/time.h>
#endif
#include "common.h"
double fabs(double);
#undef POTRF
#ifndef COMPLEX
#ifdef XDOUBLE
#define POTRF BLASFUNC(qpotrf)
#define POTRS BLASFUNC(qpotrs)
#define POTRI BLASFUNC(qpotri)
#define SYRK BLASFUNC(qsyrk)
#elif defined(DOUBLE)
#define POTRF BLASFUNC(dpotrf)
#define POTRS BLASFUNC(dpotrs)
#define POTRI BLASFUNC(dpotri)
#define SYRK BLASFUNC(dsyrk)
#else
#define POTRF BLASFUNC(spotrf)
#define POTRS BLASFUNC(spotrs)
#define POTRI BLASFUNC(spotri)
#define SYRK BLASFUNC(ssyrk)
#endif
#else
#ifdef XDOUBLE
#define POTRF BLASFUNC(xpotrf)
#define POTRS BLASFUNC(xpotrs)
#define POTRI BLASFUNC(xpotri)
#define SYRK BLASFUNC(xherk)
#elif defined(DOUBLE)
#define POTRF BLASFUNC(zpotrf)
#define POTRS BLASFUNC(zpotrs)
#define POTRI BLASFUNC(zpotri)
#define SYRK BLASFUNC(zherk)
#else
#define POTRF BLASFUNC(cpotrf)
#define POTRS BLASFUNC(cpotrs)
#define POTRI BLASFUNC(cpotri)
#define SYRK BLASFUNC(cherk)
#endif
#endif
// extern void POTRI(char *uplo, blasint *m, FLOAT *a, blasint *lda, blasint *info);
// extern void POTRS(char *uplo, blasint *m, blasint *n, FLOAT *a, blasint *lda, FLOAT *b, blasint *ldb, blasint *info);
#if defined(__WIN32__) || defined(__WIN64__)
int gettimeofday(struct timeval *tv, void *tz){
FILETIME ft;
unsigned __int64 tmpres = 0;
static int tzflag;
if (NULL != tv)
{
GetSystemTimeAsFileTime(&ft);
tmpres |= ft.dwHighDateTime;
tmpres <<= 32;
tmpres |= ft.dwLowDateTime;
/*converting file time to unix epoch*/
tmpres /= 10; /*convert into microseconds*/
tmpres -= DELTA_EPOCH_IN_MICROSECS;
tv->tv_sec = (long)(tmpres / 1000000UL);
tv->tv_usec = (long)(tmpres % 1000000UL);
}
return 0;
}
#endif
int MAIN__(int argc, char *argv[]){
#ifndef COMPLEX
char *trans[] = {"T", "N"};
#else
char *trans[] = {"C", "N"};
#endif
char *uplo[] = {"U", "L"};
FLOAT alpha[] = {1.0, 0.0};
FLOAT beta [] = {0.0, 0.0};
FLOAT *a, *b;
char *p;
char btest = 'F';
blasint m, i, j, info, uplos=0;
double flops;
int from = 1;
int to = 200;
int step = 1;
struct timeval start, stop;
double time1;
argc--;argv++;
if (argc > 0) { from = atol(*argv); argc--; argv++;}
if (argc > 0) { to = MAX(atol(*argv), from); argc--; argv++;}
if (argc > 0) { step = atol(*argv); argc--; argv++;}
if ((p = getenv("OPENBLAS_UPLO")))
if (*p == 'L') uplos=1;
if ((p = getenv("OPENBLAS_TEST"))) btest=*p;
fprintf(stderr, "From : %3d To : %3d Step = %3d Uplo = %c\n", from, to, step,*uplo[uplos]);
if (( a = (FLOAT *)malloc(sizeof(FLOAT) * to * to * COMPSIZE)) == NULL){
fprintf(stderr,"Out of Memory!!\n");exit(1);
}
if (( b = (FLOAT *)malloc(sizeof(FLOAT) * to * to * COMPSIZE)) == NULL){
fprintf(stderr,"Out of Memory!!\n");exit(1);
}
for(m = from; m <= to; m += step){
#ifndef COMPLEX
if (uplos & 1) {
for (j = 0; j < m; j++) {
for(i = 0; i < j; i++) a[i + j * m] = 0.;
a[j + j * m] = ((double) rand() / (double) RAND_MAX) + 8.;
for(i = j + 1; i < m; i++) a[i + j * m] = ((double) rand() / (double) RAND_MAX) - 0.5;
}
} else {
for (j = 0; j < m; j++) {
for(i = 0; i < j; i++) a[i + j * m] = ((double) rand() / (double) RAND_MAX) - 0.5;
a[j + j * m] = ((double) rand() / (double) RAND_MAX) + 8.;
for(i = j + 1; i < m; i++) a[i + j * m] = 0.;
}
}
#else
if (uplos & 1) {
for (j = 0; j < m; j++) {
for(i = 0; i < j; i++) {
a[(i + j * m) * 2 + 0] = 0.;
a[(i + j * m) * 2 + 1] = 0.;
}
a[(j + j * m) * 2 + 0] = ((double) rand() / (double) RAND_MAX) + 8.;
a[(j + j * m) * 2 + 1] = 0.;
for(i = j + 1; i < m; i++) {
a[(i + j * m) * 2 + 0] = ((double) rand() / (double) RAND_MAX) - 0.5;
a[(i + j * m) * 2 + 1] = ((double) rand() / (double) RAND_MAX) - 0.5;
}
}
} else {
for (j = 0; j < m; j++) {
for(i = 0; i < j; i++) {
a[(i + j * m) * 2 + 0] = ((double) rand() / (double) RAND_MAX) - 0.5;
a[(i + j * m) * 2 + 1] = ((double) rand() / (double) RAND_MAX) - 0.5;
}
a[(j + j * m) * 2 + 0] = ((double) rand() / (double) RAND_MAX) + 8.;
a[(j + j * m) * 2 + 1] = 0.;
for(i = j + 1; i < m; i++) {
a[(i + j * m) * 2 + 0] = 0.;
a[(i + j * m) * 2 + 1] = 0.;
}
}
}
#endif
SYRK(uplo[uplos], trans[uplos], &m, &m, alpha, a, &m, beta, b, &m);
gettimeofday( &start, (struct timezone *)0);
POTRF(uplo[uplos], &m, b, &m, &info);
gettimeofday( &stop, (struct timezone *)0);
if (info != 0) {
fprintf(stderr, "Potrf info = %d\n", info);
exit(1);
}
time1 = (double)(stop.tv_sec - start.tv_sec) + (double)((stop.tv_usec - start.tv_usec)) * 1.e-6;
flops = COMPSIZE * COMPSIZE * (1.0/3.0 * (double)m * (double)m *(double)m +1.0/2.0* (double)m *(double)m + 1.0/6.0* (double)m) / time1 * 1.e-6;
if ( btest == 'S' )
{
for(j = 0; j < to; j++){
for(i = 0; i < to * COMPSIZE; i++){
a[i + j * to * COMPSIZE] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
}
}
gettimeofday( &start, (struct timezone *)0);
POTRS(uplo[uplos], &m, &m, b, &m, a, &m, &info);
gettimeofday( &stop, (struct timezone *)0);
if (info != 0) {
fprintf(stderr, "Potrs info = %d\n", info);
exit(1);
}
time1 = (double)(stop.tv_sec - start.tv_sec) + (double)((stop.tv_usec - start.tv_usec)) * 1.e-6;
flops = COMPSIZE * COMPSIZE * (2.0 * (double)m * (double)m *(double)m ) / time1 * 1.e-6;
}
if ( btest == 'I' )
{
gettimeofday( &start, (struct timezone *)0);
POTRI(uplo[uplos], &m, b, &m, &info);
gettimeofday( &stop, (struct timezone *)0);
if (info != 0) {
fprintf(stderr, "Potri info = %d\n", info);
exit(1);
}
time1 = (double)(stop.tv_sec - start.tv_sec) + (double)((stop.tv_usec - start.tv_usec)) * 1.e-6;
flops = COMPSIZE * COMPSIZE * (2.0/3.0 * (double)m * (double)m *(double)m +1.0/2.0* (double)m *(double)m + 5.0/6.0* (double)m) / time1 * 1.e-6;
}
fprintf(stderr, "%8d : %10.2f MFlops : %10.3f Sec : Test=%c\n",m,flops ,time1,btest);
}
return 0;
}
void main(int argc, char *argv[]) __attribute__((weak, alias("MAIN__")));

218
benchmark/symv.c Normal file
View File

@ -0,0 +1,218 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#include <stdio.h>
#include <stdlib.h>
#ifdef __CYGWIN32__
#include <sys/time.h>
#endif
#include "common.h"
#undef SYMV
#ifndef COMPLEX
#ifdef DOUBLE
#define SYMV BLASFUNC(dsymv)
#else
#define SYMV BLASFUNC(ssymv)
#endif
#else
#ifdef DOUBLE
#define SYMV BLASFUNC(zsymv)
#else
#define SYMV BLASFUNC(csymv)
#endif
#endif
#if defined(__WIN32__) || defined(__WIN64__)
#ifndef DELTA_EPOCH_IN_MICROSECS
#define DELTA_EPOCH_IN_MICROSECS 11644473600000000ULL
#endif
int gettimeofday(struct timeval *tv, void *tz){
FILETIME ft;
unsigned __int64 tmpres = 0;
static int tzflag;
if (NULL != tv)
{
GetSystemTimeAsFileTime(&ft);
tmpres |= ft.dwHighDateTime;
tmpres <<= 32;
tmpres |= ft.dwLowDateTime;
/*converting file time to unix epoch*/
tmpres /= 10; /*convert into microseconds*/
tmpres -= DELTA_EPOCH_IN_MICROSECS;
tv->tv_sec = (long)(tmpres / 1000000UL);
tv->tv_usec = (long)(tmpres % 1000000UL);
}
return 0;
}
#endif
#if !defined(__WIN32__) && !defined(__WIN64__) && !defined(__CYGWIN32__) && 0
static void *huge_malloc(BLASLONG size){
int shmid;
void *address;
#ifndef SHM_HUGETLB
#define SHM_HUGETLB 04000
#endif
if ((shmid =shmget(IPC_PRIVATE,
(size + HUGE_PAGESIZE) & ~(HUGE_PAGESIZE - 1),
SHM_HUGETLB | IPC_CREAT |0600)) < 0) {
printf( "Memory allocation failed(shmget).\n");
exit(1);
}
address = shmat(shmid, NULL, SHM_RND);
if ((BLASLONG)address == -1){
printf( "Memory allocation failed(shmat).\n");
exit(1);
}
shmctl(shmid, IPC_RMID, 0);
return address;
}
#define malloc huge_malloc
#endif
int MAIN__(int argc, char *argv[]){
FLOAT *a, *x, *y;
FLOAT alpha[] = {1.0, 1.0};
FLOAT beta [] = {1.0, 1.0};
char uplo='L';
blasint m, i, j;
blasint inc_x=1,inc_y=1;
int loops = 1;
int l;
char *p;
int from = 1;
int to = 200;
int step = 1;
struct timeval start, stop;
double time1,timeg;
argc--;argv++;
if (argc > 0) { from = atol(*argv); argc--; argv++;}
if (argc > 0) { to = MAX(atol(*argv), from); argc--; argv++;}
if (argc > 0) { step = atol(*argv); argc--; argv++;}
if ((p = getenv("OPENBLAS_LOOPS"))) loops = atoi(p);
if ((p = getenv("OPENBLAS_INCX"))) inc_x = atoi(p);
if ((p = getenv("OPENBLAS_INCY"))) inc_y = atoi(p);
if ((p = getenv("OPENBLAS_UPLO"))) uplo=*p;
fprintf(stderr, "From : %3d To : %3d Step = %3d Uplo = '%c' Inc_x = %d Inc_y = %d Loops = %d\n", from, to, step,uplo,inc_x,inc_y,loops);
if (( a = (FLOAT *)malloc(sizeof(FLOAT) * to * to * COMPSIZE)) == NULL){
fprintf(stderr,"Out of Memory!!\n");exit(1);
}
if (( x = (FLOAT *)malloc(sizeof(FLOAT) * to * abs(inc_x) * COMPSIZE)) == NULL){
fprintf(stderr,"Out of Memory!!\n");exit(1);
}
if (( y = (FLOAT *)malloc(sizeof(FLOAT) * to * abs(inc_y) * COMPSIZE)) == NULL){
fprintf(stderr,"Out of Memory!!\n");exit(1);
}
#ifdef linux
srandom(getpid());
#endif
fprintf(stderr, " SIZE Flops\n");
for(m = from; m <= to; m += step)
{
timeg=0;
fprintf(stderr, " %6dx%d : ", (int)m,(int)m);
for(j = 0; j < m; j++){
for(i = 0; i < m * COMPSIZE; i++){
a[i + j * m * COMPSIZE] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
}
}
for (l=0; l<loops; l++)
{
for(i = 0; i < m * COMPSIZE * abs(inc_x); i++){
x[i] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
}
for(i = 0; i < m * COMPSIZE * abs(inc_y); i++){
y[i] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
}
gettimeofday( &start, (struct timezone *)0);
SYMV (&uplo, &m, alpha, a, &m, x, &inc_x, beta, y, &inc_y );
gettimeofday( &stop, (struct timezone *)0);
time1 = (double)(stop.tv_sec - start.tv_sec) + (double)((stop.tv_usec - start.tv_usec)) * 1.e-6;
timeg += time1;
}
timeg /= loops;
fprintf(stderr,
" %10.2f MFlops\n",
COMPSIZE * COMPSIZE * 2. * (double)m * (double)m / timeg * 1.e-6);
}
return 0;
}
void main(int argc, char *argv[]) __attribute__((weak, alias("MAIN__")));

11
common.h
View File

@ -524,10 +524,21 @@ static __inline void blas_unlock(volatile BLASULONG *address){
*address = 0;
}
#ifdef OS_WINDOWS
static __inline int readenv_atoi(char *env) {
env_var_t p;
return readenv(p,env) ? 0 : atoi(p);
}
#else
static __inline int readenv_atoi(char *env) {
char *p;
if (( p = getenv(env) ))
return (atoi(p));
else
return(0);
}
#endif
#if !defined(XDOUBLE) || !defined(QUAD_PRECISION)

20
common_interface.h
View File

@ -679,13 +679,6 @@ int BLASFUNC(cgesv)(blasint *, blasint *, float *, blasint *, blasint *, float
int BLASFUNC(zgesv)(blasint *, blasint *, double *, blasint *, blasint *, double*, blasint *, blasint *);
int BLASFUNC(xgesv)(blasint *, blasint *, xdouble *, blasint *, blasint *, xdouble*, blasint *, blasint *);
int BLASFUNC(sgesvd)(char *, char *, blasint *, blasint *, float *, blasint *, float *, float *, blasint *, float *, blasint *, float *, blasint *, blasint *);
int BLASFUNC(dgesvd)(char *, char *, blasint *, blasint *, double *, blasint *, double *, double *, blasint *, double *, blasint *, double *, blasint *, blasint *);
int BLASFUNC(qgesvd)(char *, char *, blasint *, blasint *, xdouble *, blasint *, xdouble *, xdouble *, blasint *, xdouble *, blasint *, xdouble *, blasint *, blasint *);
int BLASFUNC(cgesvd)(char *, char *, blasint *, blasint *, float *, blasint *, float *, float *, blasint *, float *, blasint *, float *, blasint *, blasint *);
int BLASFUNC(zgesvd)(char *, char *, blasint *, blasint *, double *, blasint *, double *, double *, blasint *, double *, blasint *, double *, blasint *, blasint *);
int BLASFUNC(xgesvd)(char *, char *, blasint *, blasint *, xdouble *, blasint *, xdouble *, xdouble *, blasint *, xdouble *, blasint *, xdouble *, blasint *, blasint *);
int BLASFUNC(spotf2)(char *, blasint *, float *, blasint *, blasint *);
int BLASFUNC(dpotf2)(char *, blasint *, double *, blasint *, blasint *);
int BLASFUNC(qpotf2)(char *, blasint *, xdouble *, blasint *, blasint *);
@ -735,19 +728,6 @@ int BLASFUNC(ctrtri)(char *, char *, blasint *, float *, blasint *, blasint *);
int BLASFUNC(ztrtri)(char *, char *, blasint *, double *, blasint *, blasint *);
int BLASFUNC(xtrtri)(char *, char *, blasint *, xdouble *, blasint *, blasint *);
int BLASFUNC(spotri)(char *, blasint *, float *, blasint *, blasint *);
int BLASFUNC(dpotri)(char *, blasint *, double *, blasint *, blasint *);
int BLASFUNC(qpotri)(char *, blasint *, xdouble *, blasint *, blasint *);
int BLASFUNC(cpotri)(char *, blasint *, float *, blasint *, blasint *);
int BLASFUNC(zpotri)(char *, blasint *, double *, blasint *, blasint *);
int BLASFUNC(xpotri)(char *, blasint *, xdouble *, blasint *, blasint *);
int BLASFUNC(slarf)(char *, blasint *, blasint *, float *, blasint *, float *, float *, blasint *, float *);
int BLASFUNC(dlarf)(char *, blasint *, blasint *, double *, blasint *, double *, double *, blasint *, double *);
int BLASFUNC(qlarf)(char *, blasint *, blasint *, xdouble *, blasint *, xdouble *, xdouble *, blasint *, xdouble *);
int BLASFUNC(clarf)(char *, blasint *, blasint *, float *, blasint *, float *, float *, blasint *, float *);
int BLASFUNC(zlarf)(char *, blasint *, blasint *, double *, blasint *, double *, double *, blasint *, double *);
int BLASFUNC(xlarf)(char *, blasint *, blasint *, xdouble *, blasint *, xdouble *, xdouble *, blasint *, xdouble *);
FLOATRET BLASFUNC(slamch)(char *);
double BLASFUNC(dlamch)(char *);

16
common_param.h
View File

@ -919,14 +919,22 @@ extern gotoblas_t *gotoblas;
#define SGEMM_R SGEMM_DEFAULT_R
#define SGEMM_UNROLL_M SGEMM_DEFAULT_UNROLL_M
#define SGEMM_UNROLL_N SGEMM_DEFAULT_UNROLL_N
#ifdef SGEMM_DEFAULT_UNROLL_MN
#define SGEMM_UNROLL_MN SGEMM_DEFAULT_UNROLL_MN
#else
#define SGEMM_UNROLL_MN MAX((SGEMM_UNROLL_M), (SGEMM_UNROLL_N))
#endif
#define DGEMM_P DGEMM_DEFAULT_P
#define DGEMM_Q DGEMM_DEFAULT_Q
#define DGEMM_R DGEMM_DEFAULT_R
#define DGEMM_UNROLL_M DGEMM_DEFAULT_UNROLL_M
#define DGEMM_UNROLL_N DGEMM_DEFAULT_UNROLL_N
#ifdef DGEMM_DEFAULT_UNROLL_MN
#define DGEMM_UNROLL_MN DGEMM_DEFAULT_UNROLL_MN
#else
#define DGEMM_UNROLL_MN MAX((DGEMM_UNROLL_M), (DGEMM_UNROLL_N))
#endif
#define QGEMM_P QGEMM_DEFAULT_P
#define QGEMM_Q QGEMM_DEFAULT_Q
@ -940,14 +948,22 @@ extern gotoblas_t *gotoblas;
#define CGEMM_R CGEMM_DEFAULT_R
#define CGEMM_UNROLL_M CGEMM_DEFAULT_UNROLL_M
#define CGEMM_UNROLL_N CGEMM_DEFAULT_UNROLL_N
#ifdef CGEMM_DEFAULT_UNROLL_MN
#define CGEMM_UNROLL_MN CGEMM_DEFAULT_UNROLL_MN
#else
#define CGEMM_UNROLL_MN MAX((CGEMM_UNROLL_M), (CGEMM_UNROLL_N))
#endif
#define ZGEMM_P ZGEMM_DEFAULT_P
#define ZGEMM_Q ZGEMM_DEFAULT_Q
#define ZGEMM_R ZGEMM_DEFAULT_R
#define ZGEMM_UNROLL_M ZGEMM_DEFAULT_UNROLL_M
#define ZGEMM_UNROLL_N ZGEMM_DEFAULT_UNROLL_N
#ifdef ZGEMM_DEFAULT_UNROLL_MN
#define ZGEMM_UNROLL_MN ZGEMM_DEFAULT_UNROLL_MN
#else
#define ZGEMM_UNROLL_MN MAX((ZGEMM_UNROLL_M), (ZGEMM_UNROLL_N))
#endif
#define XGEMM_P XGEMM_DEFAULT_P
#define XGEMM_Q XGEMM_DEFAULT_Q

2
cpuid_arm.c
View File

@ -90,7 +90,7 @@ int detect(void)
while (fgets(buffer, sizeof(buffer), infile))
{
if (!strncmp("model name", buffer, 10))
if ((!strncmp("model name", buffer, 10)) || (!strncmp("Processor", buffer, 9)))
{
p = strchr(buffer, ':') + 2;
break;

12
driver/level3/level3.c
View File

@ -333,16 +333,10 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,
for(jjs = js; jjs < js + min_j; jjs += min_jj){
min_jj = min_j + js - jjs;
#if ( defined(BULLDOZER) || defined(PILEDRIVER) || defined(HASWELL) ) && defined(ARCH_X86_64) && !defined(XDOUBLE) && !defined(COMPLEX)
if (min_jj >= 6*GEMM_UNROLL_N) min_jj = 6*GEMM_UNROLL_N;
else
if (min_jj >= 3*GEMM_UNROLL_N) min_jj = 3*GEMM_UNROLL_N;
else
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
#else
if (min_jj >= 3*GEMM_UNROLL_N) min_jj = 3*GEMM_UNROLL_N;
else
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
#endif
START_RPCC();

8
driver/level3/level3_syrk_threaded.c
View File

@ -538,10 +538,10 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO
mask = MAX(QGEMM_UNROLL_M, QGEMM_UNROLL_N) - 1;
#elif defined(DOUBLE)
mode = BLAS_DOUBLE | BLAS_REAL;
mask = MAX(DGEMM_UNROLL_M, DGEMM_UNROLL_N) - 1;
mask = DGEMM_UNROLL_MN - 1;
#else
mode = BLAS_SINGLE | BLAS_REAL;
mask = MAX(SGEMM_UNROLL_M, SGEMM_UNROLL_N) - 1;
mask = SGEMM_UNROLL_MN - 1;
#endif
#else
#ifdef XDOUBLE
@ -549,10 +549,10 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO
mask = MAX(XGEMM_UNROLL_M, XGEMM_UNROLL_N) - 1;
#elif defined(DOUBLE)
mode = BLAS_DOUBLE | BLAS_COMPLEX;
mask = MAX(ZGEMM_UNROLL_M, ZGEMM_UNROLL_N) - 1;
mask = ZGEMM_UNROLL_MN - 1;
#else
mode = BLAS_SINGLE | BLAS_COMPLEX;
mask = MAX(CGEMM_UNROLL_M, CGEMM_UNROLL_N) - 1;
mask = CGEMM_UNROLL_MN - 1;
#endif
#endif

12
driver/level3/level3_thread.c
View File

@ -367,16 +367,10 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,
for(jjs = xxx; jjs < MIN(n_to, xxx + div_n); jjs += min_jj){
min_jj = MIN(n_to, xxx + div_n) - jjs;
#if ( defined(BULLDOZER) || defined(PILEDRIVER) || defined(HASWELL) ) && defined(ARCH_X86_64) && !defined(XDOUBLE) && !defined(COMPLEX)
if (min_jj >= 6*GEMM_UNROLL_N) min_jj = 6*GEMM_UNROLL_N;
else
if (min_jj >= 3*GEMM_UNROLL_N) min_jj = 3*GEMM_UNROLL_N;
else
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
#else
if (min_jj >= 3*GEMM_UNROLL_N) min_jj = 3*GEMM_UNROLL_N;
else
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
#endif
START_RPCC();

8
driver/level3/syrk_thread.c
View File

@ -57,10 +57,10 @@ int CNAME(int mode, blas_arg_t *arg, BLASLONG *range_m, BLASLONG *range_n, int (
switch (mode & BLAS_PREC) {
case BLAS_SINGLE:
mask = MAX(SGEMM_UNROLL_M, SGEMM_UNROLL_N) - 1;
mask = SGEMM_UNROLL_MN - 1;
break;
case BLAS_DOUBLE:
mask = MAX(DGEMM_UNROLL_M, DGEMM_UNROLL_N) - 1;
mask = DGEMM_UNROLL_MN - 1;
break;
#ifdef EXPRECISION
case BLAS_XDOUBLE:
@ -71,10 +71,10 @@ int CNAME(int mode, blas_arg_t *arg, BLASLONG *range_m, BLASLONG *range_n, int (
} else {
switch (mode & BLAS_PREC) {
case BLAS_SINGLE:
mask = MAX(CGEMM_UNROLL_M, CGEMM_UNROLL_N) - 1;
mask = CGEMM_UNROLL_MN - 1;
break;
case BLAS_DOUBLE:
mask = MAX(ZGEMM_UNROLL_M, ZGEMM_UNROLL_N) - 1;
mask = ZGEMM_UNROLL_MN - 1;
break;
#ifdef EXPRECISION
case BLAS_XDOUBLE:

16
driver/level3/trmm_L.c
View File

@ -135,7 +135,9 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO
for(jjs = js; jjs < js + min_j; jjs += min_jj){
min_jj = min_j + js - jjs;
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
else
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
START_RPCC();
@ -199,7 +201,9 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO
for(jjs = js; jjs < js + min_j; jjs += min_jj){
min_jj = min_j + js - jjs;
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
else
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
START_RPCC();
@ -288,7 +292,9 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO
for(jjs = js; jjs < js + min_j; jjs += min_jj){
min_jj = min_j + js - jjs;
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
else
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
START_RPCC();
@ -352,7 +358,9 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO
for(jjs = js; jjs < js + min_j; jjs += min_jj){
min_jj = min_j + js - jjs;
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
else
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
START_RPCC();

24
driver/level3/trmm_R.c
View File

@ -119,7 +119,9 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO
for(jjs = 0; jjs < ls - js; jjs += min_jj){
min_jj = ls - js - jjs;
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
else
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
#ifndef TRANSA
GEMM_ONCOPY(min_l, min_jj, a + (ls + (js + jjs) * lda) * COMPSIZE, lda, sb + min_l * jjs * COMPSIZE);
@ -137,7 +139,9 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO
for(jjs = 0; jjs < min_l; jjs += min_jj){
min_jj = min_l - jjs;
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
else
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
#ifndef TRANSA
TRMM_OLNCOPY(min_l, min_jj, a, lda, ls, ls + jjs, sb + min_l * (ls - js + jjs) * COMPSIZE);
@ -188,7 +192,9 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO
for(jjs = js; jjs < js + min_j; jjs += min_jj){
min_jj = min_j + js - jjs;
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
else
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
#ifndef TRANSA
GEMM_ONCOPY(min_l, min_jj, a + (ls + jjs * lda) * COMPSIZE, lda, sb + min_l * (jjs - js) * COMPSIZE);
@ -239,7 +245,9 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO
for(jjs = 0; jjs < min_l; jjs += min_jj){
min_jj = min_l - jjs;
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
else
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
#ifndef TRANSA
TRMM_OUNCOPY(min_l, min_jj, a, lda, ls, ls + jjs, sb + min_l * jjs * COMPSIZE);
@ -258,7 +266,9 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO
for(jjs = 0; jjs < js - ls - min_l; jjs += min_jj){
min_jj = js - ls - min_l - jjs;
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
else
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
#ifndef TRANSA
GEMM_ONCOPY(min_l, min_jj, a + (ls + (ls + min_l + jjs) * lda) * COMPSIZE, lda,
@ -313,7 +323,9 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO
for(jjs = js; jjs < js + min_j; jjs += min_jj){
min_jj = min_j + js - jjs;
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
else
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
#ifndef TRANSA
GEMM_ONCOPY(min_l, min_jj, a + (ls + (jjs - min_j) * lda) * COMPSIZE, lda, sb + min_l * (jjs - js) * COMPSIZE);

8
driver/level3/trsm_L.c
View File

@ -128,7 +128,9 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO
for(jjs = js; jjs < js + min_j; jjs += min_jj){
min_jj = min_j + js - jjs;
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
else
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
GEMM_ONCOPY(min_l, min_jj, b + (ls + jjs * ldb) * COMPSIZE, ldb, sb + min_l * (jjs - js) * COMPSIZE);
@ -194,7 +196,9 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO
for(jjs = js; jjs < js + min_j; jjs += min_jj){
min_jj = min_j + js - jjs;
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
else
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
GEMM_ONCOPY(min_l, min_jj, b + (ls - min_l + jjs * ldb) * COMPSIZE, ldb, sb + min_l * (jjs - js) * COMPSIZE);

16
driver/level3/trsm_R.c
View File

@ -123,7 +123,9 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO
for(jjs = js; jjs < js + min_j; jjs += min_jj){
min_jj = min_j + js - jjs;
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
else
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
#ifndef TRANSA
GEMM_ONCOPY(min_l, min_jj, a + (ls + jjs * lda) * COMPSIZE, lda, sb + min_l * (jjs - js) * COMPSIZE);
@ -177,7 +179,9 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO
for(jjs = 0; jjs < min_j - min_l - ls + js; jjs += min_jj){
min_jj = min_j - min_l - ls + js - jjs;
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
else
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
#ifndef TRANSA
GEMM_ONCOPY (min_l, min_jj, a + (ls + (ls + min_l + jjs) * lda) * COMPSIZE, lda,
@ -238,7 +242,9 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO
for(jjs = js; jjs < js + min_j; jjs += min_jj){
min_jj = min_j + js - jjs;
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
else
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
#ifndef TRANSA
GEMM_ONCOPY(min_l, min_jj, a + (ls + (jjs - min_j) * lda) * COMPSIZE, lda, sb + min_l * (jjs - js) * COMPSIZE);
@ -297,7 +303,9 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO
for(jjs = 0; jjs < min_j - js + ls; jjs += min_jj){
min_jj = min_j - js + ls - jjs;
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
else
if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
#ifndef TRANSA
GEMM_ONCOPY (min_l, min_jj, a + (ls + (js - min_j + jjs) * lda) * COMPSIZE, lda,

20
driver/others/init.c
View File

@ -85,8 +85,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <unistd.h>
#include <string.h>
#if defined(BIGNUMA)
// max number of nodes as defined in numa.h
// max cpus as defined in sched.h
#define MAX_NODES 128
#define MAX_CPUS CPU_SETSIZE
#else
#define MAX_NODES 16
#define MAX_CPUS 256
#endif
#define NCPUBITS (8*sizeof(unsigned long))
#define MAX_BITMASK_LEN (MAX_CPUS/NCPUBITS)
#define CPUELT(cpu) ((cpu) / NCPUBITS)
@ -544,16 +552,26 @@ static inline int is_dead(int id) {
return shmctl(id, IPC_STAT, &ds);
}
static void open_shmem(void) {
int try = 0;
do {
#if defined(BIGNUMA)
// raised to 32768, enough for 128 nodes and 1024 cups
shmid = shmget(SH_MAGIC, 32768, 0666);
#else
shmid = shmget(SH_MAGIC, 4096, 0666);
#endif
if (shmid == -1) {
#if defined(BIGNUMA)
shmid = shmget(SH_MAGIC, 32768, IPC_CREAT | 0666);
#else
shmid = shmget(SH_MAGIC, 4096, IPC_CREAT | 0666);
#endif
}
try ++;
@ -847,7 +865,7 @@ void gotoblas_set_affinity2(int threads) {};
void gotoblas_affinity_reschedule(void) {};
int get_num_procs(void) { return get_nprocs(); }
int get_num_procs(void) { return sysconf(_SC_NPROCESSORS_ONLN); }
int get_num_nodes(void) { return 1; }

2
driver/others/memory.c
View File

@ -162,7 +162,7 @@ int get_num_procs(void);
#else
int get_num_procs(void) {
static int nums = 0;
if (!nums) nums = get_nprocs();
if (!nums) nums = sysconf(_SC_NPROCESSORS_ONLN);
return nums;
}
#endif

4
interface/gemm.c
View File

@ -405,11 +405,11 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, enum CBLAS_TRANS
#ifndef COMPLEX
double MNK = (double) args.m * (double) args.n * (double) args.k;
if ( MNK <= (1024.0 * (double) GEMM_MULTITHREAD_THRESHOLD) )
if ( MNK <= (16.0 * 1024.0 * (double) GEMM_MULTITHREAD_THRESHOLD) )
nthreads_max = 1;
else
{
if ( MNK <= (65536.0 * (double) GEMM_MULTITHREAD_THRESHOLD) )
if ( MNK <= (2.0 * 65536.0 * (double) GEMM_MULTITHREAD_THRESHOLD) )
{
nthreads_max = 4;
if ( args.m < 16 * GEMM_MULTITHREAD_THRESHOLD )

13
interface/gemv.c
View File

@ -211,7 +211,18 @@ void CNAME(enum CBLAS_ORDER order,
buffer = (FLOAT *)blas_memory_alloc(1);
#ifdef SMP
nthreads = num_cpu_avail(2);
int nthreads_max = num_cpu_avail(2);
int nthreads_avail = nthreads_max;
double MNK = (double) m * (double) n;
if ( MNK <= (500.0 * 100.0 * (double) GEMM_MULTITHREAD_THRESHOLD) )
nthreads_max = 1;
if ( nthreads_max > nthreads_avail )
nthreads = nthreads_avail;
else
nthreads = nthreads_max;
if (nthreads == 1) {
#endif

14
interface/zgemv.c
View File

@ -233,7 +233,19 @@ void CNAME(enum CBLAS_ORDER order,
buffer = (FLOAT *)blas_memory_alloc(1);
#ifdef SMP
nthreads = num_cpu_avail(2);
int nthreads_max = num_cpu_avail(2);
int nthreads_avail = nthreads_max;
double MNK = (double) m * (double) n;
if ( MNK <= (80.0 * 20.0 * (double) GEMM_MULTITHREAD_THRESHOLD) )
nthreads_max = 1;
if ( nthreads_max > nthreads_avail )
nthreads = nthreads_avail;
else
nthreads = nthreads_max;
if (nthreads == 1) {
#endif

30
kernel/setparam-ref.c
View File

@ -54,7 +54,14 @@ gotoblas_t TABLE_NAME = {
GEMM_DEFAULT_OFFSET_A, GEMM_DEFAULT_OFFSET_B, GEMM_DEFAULT_ALIGN,
0, 0, 0,
SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N, MAX(SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N),
SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N,
#ifdef SGEMM_DEFAULT_UNROLL_MN
SGEMM_DEFAULT_UNROLL_MN,
#else
MAX(SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N),
#endif
#ifdef HAVE_EXCLUSIVE_CACHE
1,
#else
@ -110,7 +117,12 @@ gotoblas_t TABLE_NAME = {
#endif
0, 0, 0,
DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N, MAX(DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N),
DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N,
#ifdef DGEMM_DEFAULT_UNROLL_MN
DGEMM_DEFAULT_UNROLL_MN,
#else
MAX(DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N),
#endif
damax_kTS, damin_kTS, dmax_kTS, dmin_kTS,
idamax_kTS, idamin_kTS, idmax_kTS, idmin_kTS,
@ -214,7 +226,12 @@ gotoblas_t TABLE_NAME = {
#endif
0, 0, 0,
CGEMM_DEFAULT_UNROLL_M, CGEMM_DEFAULT_UNROLL_N, MAX(CGEMM_DEFAULT_UNROLL_M, CGEMM_DEFAULT_UNROLL_N),
CGEMM_DEFAULT_UNROLL_M, CGEMM_DEFAULT_UNROLL_N,
#ifdef CGEMM_DEFAULT_UNROLL_MN
CGEMM_DEFAULT_UNROLL_MN,
#else
MAX(CGEMM_DEFAULT_UNROLL_M, CGEMM_DEFAULT_UNROLL_N),
#endif
camax_kTS, camin_kTS, icamax_kTS, icamin_kTS,
cnrm2_kTS, casum_kTS, ccopy_kTS,
@ -307,7 +324,12 @@ gotoblas_t TABLE_NAME = {
#endif
0, 0, 0,
ZGEMM_DEFAULT_UNROLL_M, ZGEMM_DEFAULT_UNROLL_N, MAX(ZGEMM_DEFAULT_UNROLL_M, ZGEMM_DEFAULT_UNROLL_N),
ZGEMM_DEFAULT_UNROLL_M, ZGEMM_DEFAULT_UNROLL_N,
#ifdef ZGEMM_DEFAULT_UNROLL_MN
ZGEMM_DEFAULT_UNROLL_MN,
#else
MAX(ZGEMM_DEFAULT_UNROLL_M, ZGEMM_DEFAULT_UNROLL_N),
#endif
zamax_kTS, zamin_kTS, izamax_kTS, izamin_kTS,
znrm2_kTS, zasum_kTS, zcopy_kTS,

4
kernel/x86_64/KERNEL
View File

@ -373,11 +373,11 @@ endif
GEMVDEP = ../l2param.h
ifndef SGEMVNKERNEL
SGEMVNKERNEL = ../arm/gemv_n.c
SGEMVNKERNEL = sgemv_n.c
endif
ifndef SGEMVTKERNEL
SGEMVTKERNEL = ../arm/gemv_t.c
SGEMVTKERNEL = sgemv_t.c
endif
ifndef DGEMVNKERNEL

4
kernel/x86_64/KERNEL.BULLDOZER
View File

@ -1,6 +1,8 @@
SGEMVNKERNEL = sgemv_n.c
SGEMVTKERNEL = sgemv_t.c
ZGEMVNKERNEL = zgemv_n_dup.S
ZGEMVTKERNEL = zgemv_t.S
ZGEMVTKERNEL = zgemv_t.c
DGEMVNKERNEL = dgemv_n_bulldozer.S
DGEMVTKERNEL = dgemv_t_bulldozer.S

11
kernel/x86_64/KERNEL.HASWELL
View File

@ -1,3 +1,14 @@
SGEMVNKERNEL = sgemv_n.c
SGEMVTKERNEL = sgemv_t.c
DGEMVNKERNEL = dgemv_n.c
DGEMVTKERNEL = dgemv_t.c
ZGEMVNKERNEL = zgemv_n.c
ZGEMVTKERNEL = zgemv_t.c
CGEMVNKERNEL = cgemv_n.c
CGEMVTKERNEL = cgemv_t.c
SGEMMKERNEL = sgemm_kernel_16x4_haswell.S
SGEMMINCOPY = ../generic/gemm_ncopy_16.c

2
kernel/x86_64/KERNEL.NEHALEM
View File

@ -1,3 +1,5 @@
SGEMVNKERNEL = sgemv_n.c
SGEMVTKERNEL = sgemv_t.c
SGEMMKERNEL = gemm_kernel_4x8_nehalem.S
SGEMMINCOPY = gemm_ncopy_4.S

2
kernel/x86_64/KERNEL.PILEDRIVER
View File

@ -1,3 +1,5 @@
SGEMVNKERNEL = sgemv_n.c
SGEMVTKERNEL = sgemv_t.c
ZGEMVNKERNEL = zgemv_n_dup.S
ZGEMVTKERNEL = zgemv_t.S

5
kernel/x86_64/KERNEL.SANDYBRIDGE
View File

@ -1,3 +1,8 @@
SGEMVNKERNEL = sgemv_n.c
SGEMVTKERNEL = sgemv_t.c
ZGEMVNKERNEL = zgemv_n.c
SGEMMKERNEL = sgemm_kernel_16x4_sandy.S
SGEMMINCOPY = ../generic/gemm_ncopy_16.c

7212
kernel/x86_64/cgemm_kernel_8x2_haswell.S
View File

File diff suppressed because it is too large Load Diff

174
kernel/x86_64/cgemm_kernel_8x2_sandy.S
View File

@ -25,6 +25,32 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
**********************************************************************************/
/*********************************************************************
* 2014/07/29 Saar
* BLASTEST : OK
* CTEST : OK
* TEST : OK
*
* 2013/10/28 Saar
* Parameter:
* CGEMM_DEFAULT_UNROLL_N 2
* CGEMM_DEFAULT_UNROLL_M 8
* CGEMM_DEFAULT_P 768
* CGEMM_DEFAULT_Q 512
* A_PR1 512
* B_PR1 512
*
* 2014/07/29 Saar
* Performance at 6192x6192x6192:
* 1 thread: 49 GFLOPS (MKL: 52)
* 2 threads: 99 GFLOPS (MKL: 102)
* 3 threads: 148 GFLOPS (MKL: 150)
* 4 threads: 195 GFLOPS (MKL: 194)
* 8 threads: 354 GFLOPS (MKL: 317)
*
*
*********************************************************************/
#define ASSEMBLER
#include "common.h"
@ -192,22 +218,108 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
/***************************************************************************************************************************/
.macro KERNEL8x2_SUB
.macro KERNEL8x2_1
vmovups -16 * SIZE(AO, %rax, SIZE), %ymm0
vbroadcastss -8 * SIZE(BO, BI, SIZE), %ymm4
VFMADDPS_YR( %ymm8,%ymm4,%ymm0 )
vmovups -8 * SIZE(AO, %rax, SIZE), %ymm1
VFMADDPS_YR( %ymm12,%ymm4,%ymm1 )
vbroadcastss -7 * SIZE(BO, BI, SIZE), %ymm5
VFMADDPS_YI( %ymm9,%ymm5,%ymm0 )
VFMADDPS_YI( %ymm13,%ymm5,%ymm1 )
prefetcht0 A_PR1(AO, %rax, SIZE)
VFMADDPS_YR( %ymm8,%ymm4,%ymm0 )
vbroadcastss -6 * SIZE(BO, BI, SIZE), %ymm6
VFMADDPS_YR( %ymm10,%ymm6,%ymm0 )
VFMADDPS_YR( %ymm14,%ymm6,%ymm1 )
VFMADDPS_YI( %ymm9,%ymm5,%ymm0 )
vbroadcastss -5 * SIZE(BO, BI, SIZE), %ymm7
VFMADDPS_YR( %ymm12,%ymm4,%ymm1 )
VFMADDPS_YI( %ymm13,%ymm5,%ymm1 )
VFMADDPS_YR( %ymm10,%ymm6,%ymm0 )
vbroadcastss -4 * SIZE(BO, BI, SIZE), %ymm4
VFMADDPS_YI( %ymm11,%ymm7,%ymm0 )
vbroadcastss -3 * SIZE(BO, BI, SIZE), %ymm5
VFMADDPS_YR( %ymm14,%ymm6,%ymm1 )
vmovups 0 * SIZE(AO, %rax, SIZE), %ymm0
VFMADDPS_YI( %ymm15,%ymm7,%ymm1 )
vmovups 8 * SIZE(AO, %rax, SIZE), %ymm1
prefetcht0 A_PR1+64(AO, %rax, SIZE)
VFMADDPS_YR( %ymm8,%ymm4,%ymm0 )
vbroadcastss -2 * SIZE(BO, BI, SIZE), %ymm6
VFMADDPS_YI( %ymm9,%ymm5,%ymm0 )
vbroadcastss -1 * SIZE(BO, BI, SIZE), %ymm7
VFMADDPS_YR( %ymm12,%ymm4,%ymm1 )
VFMADDPS_YI( %ymm13,%ymm5,%ymm1 )
VFMADDPS_YR( %ymm10,%ymm6,%ymm0 )
vbroadcastss 0 * SIZE(BO, BI, SIZE), %ymm4
VFMADDPS_YI( %ymm11,%ymm7,%ymm0 )
vbroadcastss 1 * SIZE(BO, BI, SIZE), %ymm5
VFMADDPS_YR( %ymm14,%ymm6,%ymm1 )
vmovups 16 * SIZE(AO, %rax, SIZE), %ymm0
VFMADDPS_YI( %ymm15,%ymm7,%ymm1 )
vmovups 24 * SIZE(AO, %rax, SIZE), %ymm1
prefetcht0 A_PR1+128(AO, %rax, SIZE)
VFMADDPS_YR( %ymm8,%ymm4,%ymm0 )
vbroadcastss 2 * SIZE(BO, BI, SIZE), %ymm6
VFMADDPS_YI( %ymm9,%ymm5,%ymm0 )
vbroadcastss 3 * SIZE(BO, BI, SIZE), %ymm7
VFMADDPS_YR( %ymm12,%ymm4,%ymm1 )
VFMADDPS_YI( %ymm13,%ymm5,%ymm1 )
VFMADDPS_YR( %ymm10,%ymm6,%ymm0 )
vbroadcastss 4 * SIZE(BO, BI, SIZE), %ymm4
VFMADDPS_YI( %ymm11,%ymm7,%ymm0 )
vbroadcastss 5 * SIZE(BO, BI, SIZE), %ymm5
VFMADDPS_YR( %ymm14,%ymm6,%ymm1 )
vmovups 32 * SIZE(AO, %rax, SIZE), %ymm0
VFMADDPS_YI( %ymm15,%ymm7,%ymm1 )
vmovups 40 * SIZE(AO, %rax, SIZE), %ymm1
prefetcht0 A_PR1+192(AO, %rax, SIZE)
VFMADDPS_YR( %ymm8,%ymm4,%ymm0 )
vbroadcastss 6 * SIZE(BO, BI, SIZE), %ymm6
VFMADDPS_YI( %ymm9,%ymm5,%ymm0 )
vbroadcastss 7 * SIZE(BO, BI, SIZE), %ymm7
VFMADDPS_YR( %ymm12,%ymm4,%ymm1 )
VFMADDPS_YI( %ymm13,%ymm5,%ymm1 )
VFMADDPS_YR( %ymm10,%ymm6,%ymm0 )
VFMADDPS_YI( %ymm11,%ymm7,%ymm0 )
addq $ 16, BI
VFMADDPS_YR( %ymm14,%ymm6,%ymm1 )
VFMADDPS_YI( %ymm15,%ymm7,%ymm1 )
addq $ 64, %rax
.endm
.macro KERNEL8x2_SUB
vmovups -16 * SIZE(AO, %rax, SIZE), %ymm0
vmovups -8 * SIZE(AO, %rax, SIZE), %ymm1
vbroadcastss -8 * SIZE(BO, BI, SIZE), %ymm4
vbroadcastss -7 * SIZE(BO, BI, SIZE), %ymm5
VFMADDPS_YR( %ymm8,%ymm4,%ymm0 )
vbroadcastss -6 * SIZE(BO, BI, SIZE), %ymm6
VFMADDPS_YI( %ymm9,%ymm5,%ymm0 )
vbroadcastss -5 * SIZE(BO, BI, SIZE), %ymm7
VFMADDPS_YR( %ymm12,%ymm4,%ymm1 )
VFMADDPS_YI( %ymm13,%ymm5,%ymm1 )
VFMADDPS_YR( %ymm10,%ymm6,%ymm0 )
VFMADDPS_YI( %ymm11,%ymm7,%ymm0 )
VFMADDPS_YR( %ymm14,%ymm6,%ymm1 )
VFMADDPS_YI( %ymm15,%ymm7,%ymm1 )
addq $ 4 , BI
addq $ 16, %rax
.endm
@ -984,47 +1096,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.L2_8_12:
prefetcht0 A_PR1(AO,%rax,SIZE)
prefetcht0 B_PR1(BO,BI,SIZE)
KERNEL8x2_SUB
prefetcht0 A_PR1(AO,%rax,SIZE)
KERNEL8x2_SUB
prefetcht0 A_PR1(AO,%rax,SIZE)
KERNEL8x2_SUB
prefetcht0 A_PR1(AO,%rax,SIZE)
KERNEL8x2_SUB
KERNEL8x2_1
prefetcht0 A_PR1(AO,%rax,SIZE)
prefetcht0 B_PR1(BO,BI,SIZE)
KERNEL8x2_SUB
prefetcht0 A_PR1(AO,%rax,SIZE)
KERNEL8x2_SUB
prefetcht0 A_PR1(AO,%rax,SIZE)
KERNEL8x2_SUB
prefetcht0 A_PR1(AO,%rax,SIZE)
KERNEL8x2_SUB
KERNEL8x2_1
je .L2_8_16
prefetcht0 A_PR1(AO,%rax,SIZE)
prefetcht0 B_PR1(BO,BI,SIZE)
KERNEL8x2_SUB
prefetcht0 A_PR1(AO,%rax,SIZE)
KERNEL8x2_SUB
prefetcht0 A_PR1(AO,%rax,SIZE)
KERNEL8x2_SUB
prefetcht0 A_PR1(AO,%rax,SIZE)
KERNEL8x2_SUB
KERNEL8x2_1
prefetcht0 A_PR1(AO,%rax,SIZE)
prefetcht0 B_PR1(BO,BI,SIZE)
KERNEL8x2_SUB
prefetcht0 A_PR1(AO,%rax,SIZE)
KERNEL8x2_SUB
prefetcht0 A_PR1(AO,%rax,SIZE)
KERNEL8x2_SUB
prefetcht0 A_PR1(AO,%rax,SIZE)
KERNEL8x2_SUB
KERNEL8x2_1
je .L2_8_16
@ -1152,7 +1236,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.L2_4_12:
prefetcht0 A_PR1(AO,%rax,SIZE)
prefetcht0 B_PR1(BO,BI,SIZE)
KERNEL4x2_SUB
KERNEL4x2_SUB
prefetcht0 A_PR1(AO,%rax,SIZE)
@ -1160,7 +1243,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
KERNEL4x2_SUB
prefetcht0 A_PR1(AO,%rax,SIZE)
prefetcht0 B_PR1(BO,BI,SIZE)
KERNEL4x2_SUB
KERNEL4x2_SUB
prefetcht0 A_PR1(AO,%rax,SIZE)
@ -1170,7 +1252,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
je .L2_4_16
prefetcht0 A_PR1(AO,%rax,SIZE)
prefetcht0 B_PR1(BO,BI,SIZE)
KERNEL4x2_SUB
KERNEL4x2_SUB
prefetcht0 A_PR1(AO,%rax,SIZE)
@ -1178,7 +1259,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
KERNEL4x2_SUB
prefetcht0 A_PR1(AO,%rax,SIZE)
prefetcht0 B_PR1(BO,BI,SIZE)
KERNEL4x2_SUB
KERNEL4x2_SUB
prefetcht0 A_PR1(AO,%rax,SIZE)
@ -1305,14 +1385,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.L2_4_22:
prefetcht0 A_PR1(AO,%rax,SIZE)
prefetcht0 B_PR1(BO,BI,SIZE)
KERNEL2x2_SUB
KERNEL2x2_SUB
KERNEL2x2_SUB
KERNEL2x2_SUB
prefetcht0 A_PR1(AO,%rax,SIZE)
prefetcht0 B_PR1(BO,BI,SIZE)
KERNEL2x2_SUB
KERNEL2x2_SUB
KERNEL2x2_SUB
@ -1321,14 +1399,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
je .L2_4_26
prefetcht0 A_PR1(AO,%rax,SIZE)
prefetcht0 B_PR1(BO,BI,SIZE)
KERNEL2x2_SUB
KERNEL2x2_SUB
KERNEL2x2_SUB
KERNEL2x2_SUB
prefetcht0 A_PR1(AO,%rax,SIZE)
prefetcht0 B_PR1(BO,BI,SIZE)
KERNEL2x2_SUB
KERNEL2x2_SUB
KERNEL2x2_SUB
@ -1507,13 +1583,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.L2_4_42:
prefetcht0 A_PR1(AO,%rax,SIZE)
prefetcht0 B_PR1(BO,BI,SIZE)
KERNEL1x2_SUB
KERNEL1x2_SUB
KERNEL1x2_SUB
KERNEL1x2_SUB
prefetcht0 B_PR1(BO,BI,SIZE)
KERNEL1x2_SUB
KERNEL1x2_SUB
KERNEL1x2_SUB
@ -1522,13 +1596,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
je .L2_4_46
prefetcht0 A_PR1(AO,%rax,SIZE)
prefetcht0 B_PR1(BO,BI,SIZE)
KERNEL1x2_SUB
KERNEL1x2_SUB
KERNEL1x2_SUB
KERNEL1x2_SUB
prefetcht0 B_PR1(BO,BI,SIZE)
KERNEL1x2_SUB
KERNEL1x2_SUB
KERNEL1x2_SUB

255
kernel/x86_64/cgemv_n.c Normal file
View File

@ -0,0 +1,255 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#include <stdlib.h>
#include <stdio.h>
#include "common.h"
#if defined(HASWELL)
#include "cgemv_n_microk_haswell-2.c"
#endif
#define NBMAX 2048
#ifndef HAVE_KERNEL_16x4
static void cgemv_kernel_16x4(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)
{
BLASLONG i;
FLOAT *a0,*a1,*a2,*a3;
a0 = ap[0];
a1 = ap[1];
a2 = ap[2];
a3 = ap[3];
for ( i=0; i< 2*n; i+=2 )
{
#if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )
y[i] += a0[i]*x[0] - a0[i+1] * x[1];
y[i+1] += a0[i]*x[1] + a0[i+1] * x[0];
y[i] += a1[i]*x[2] - a1[i+1] * x[3];
y[i+1] += a1[i]*x[3] + a1[i+1] * x[2];
y[i] += a2[i]*x[4] - a2[i+1] * x[5];
y[i+1] += a2[i]*x[5] + a2[i+1] * x[4];
y[i] += a3[i]*x[6] - a3[i+1] * x[7];
y[i+1] += a3[i]*x[7] + a3[i+1] * x[6];
#else
y[i] += a0[i]*x[0] + a0[i+1] * x[1];
y[i+1] += a0[i]*x[1] - a0[i+1] * x[0];
y[i] += a1[i]*x[2] + a1[i+1] * x[3];
y[i+1] += a1[i]*x[3] - a1[i+1] * x[2];
y[i] += a2[i]*x[4] + a2[i+1] * x[5];
y[i+1] += a2[i]*x[5] - a2[i+1] * x[4];
y[i] += a3[i]*x[6] + a3[i+1] * x[7];
y[i+1] += a3[i]*x[7] - a3[i+1] * x[6];
#endif
}
}
#endif
static void cgemv_kernel_16x1(BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y)
{
BLASLONG i;
FLOAT *a0;
a0 = ap;
for ( i=0; i< 2*n; i+=2 )
{
#if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )
y[i] += a0[i]*x[0] - a0[i+1] * x[1];
y[i+1] += a0[i]*x[1] + a0[i+1] * x[0];
#else
y[i] += a0[i]*x[0] + a0[i+1] * x[1];
y[i+1] += a0[i]*x[1] - a0[i+1] * x[0];
#endif
}
}
static void zero_y(BLASLONG n, FLOAT *dest)
{
BLASLONG i;
for ( i=0; i<2*n; i++ )
{
*dest = 0.0;
dest++;
}
}
static void add_y(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_dest,FLOAT alpha_r, FLOAT alpha_i)
{
BLASLONG i;
FLOAT temp_r;
FLOAT temp_i;
for ( i=0; i<n; i++ )
{
#if !defined(XCONJ)
temp_r = alpha_r * src[0] - alpha_i * src[1];
temp_i = alpha_r * src[1] + alpha_i * src[0];
#else
temp_r = alpha_r * src[0] + alpha_i * src[1];
temp_i = -alpha_r * src[1] + alpha_i * src[0];
#endif
*dest += temp_r;
*(dest+1) += temp_i;
src+=2;
dest += inc_dest;
}
}
int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r,FLOAT alpha_i, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer)
{
BLASLONG i;
BLASLONG j;
FLOAT *a_ptr;
FLOAT *x_ptr;
FLOAT *y_ptr;
FLOAT *ap[4];
BLASLONG n1;
BLASLONG m1;
BLASLONG m2;
BLASLONG n2;
FLOAT xbuffer[8],*ybuffer;
#if 0
printf("%s %d %d %.16f %.16f %d %d %d\n","zgemv_n",m,n,alpha_r,alpha_i,lda,inc_x,inc_y);
#endif
if ( m < 1 ) return(0);
if ( n < 1 ) return(0);
ybuffer = buffer;
inc_x *= 2;
inc_y *= 2;
lda *= 2;
n1 = n / 4 ;
n2 = n % 4 ;
m1 = m - ( m % 16 );
m2 = (m % NBMAX) - (m % 16) ;
y_ptr = y;
BLASLONG NB = NBMAX;
while ( NB == NBMAX )
{
m1 -= NB;
if ( m1 < 0)
{
if ( m2 == 0 ) break;
NB = m2;
}
a_ptr = a;
x_ptr = x;
zero_y(NB,ybuffer);
for( i = 0; i < n1 ; i++)
{
xbuffer[0] = x_ptr[0];
xbuffer[1] = x_ptr[1];
x_ptr += inc_x;
xbuffer[2] = x_ptr[0];
xbuffer[3] = x_ptr[1];
x_ptr += inc_x;
xbuffer[4] = x_ptr[0];
xbuffer[5] = x_ptr[1];
x_ptr += inc_x;
xbuffer[6] = x_ptr[0];
xbuffer[7] = x_ptr[1];
x_ptr += inc_x;
ap[0] = a_ptr;
ap[1] = a_ptr + lda;
ap[2] = ap[1] + lda;
ap[3] = ap[2] + lda;
cgemv_kernel_16x4(NB,ap,xbuffer,ybuffer);
a_ptr += 4 * lda;
}
for( i = 0; i < n2 ; i++)
{
xbuffer[0] = x_ptr[0];
xbuffer[1] = x_ptr[1];
x_ptr += inc_x;
cgemv_kernel_16x1(NB,a_ptr,xbuffer,ybuffer);
a_ptr += 1 * lda;
}
add_y(NB,ybuffer,y_ptr,inc_y,alpha_r,alpha_i);
a += 2 * NB;
y_ptr += NB * inc_y;
}
j=0;
while ( j < (m % 16))
{
a_ptr = a;
x_ptr = x;
FLOAT temp_r = 0.0;
FLOAT temp_i = 0.0;
for( i = 0; i < n; i++ )
{
#if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )
temp_r += a_ptr[0] * x_ptr[0] - a_ptr[1] * x_ptr[1];
temp_i += a_ptr[0] * x_ptr[1] + a_ptr[1] * x_ptr[0];
#else
temp_r += a_ptr[0] * x_ptr[0] + a_ptr[1] * x_ptr[1];
temp_i += a_ptr[0] * x_ptr[1] - a_ptr[1] * x_ptr[0];
#endif
a_ptr += lda;
x_ptr += inc_x;
}
#if !defined(XCONJ)
y_ptr[0] += alpha_r * temp_r - alpha_i * temp_i;
y_ptr[1] += alpha_r * temp_i + alpha_i * temp_r;
#else
y_ptr[0] += alpha_r * temp_r + alpha_i * temp_i;
y_ptr[1] -= alpha_r * temp_i - alpha_i * temp_r;
#endif
y_ptr += inc_y;
a+=2;
j++;
}
return(0);
}

137
kernel/x86_64/cgemv_n_microk_haswell-2.c Normal file
View File

@ -0,0 +1,137 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#define HAVE_KERNEL_16x4 1
static void cgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) __attribute__ ((noinline));
static void cgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)
{
BLASLONG register i = 0;
__asm__ __volatile__
(
"vzeroupper \n\t"
"vbroadcastss (%2), %%ymm0 \n\t" // real part x0
"vbroadcastss 4(%2), %%ymm1 \n\t" // imag part x0
"vbroadcastss 8(%2), %%ymm2 \n\t" // real part x1
"vbroadcastss 12(%2), %%ymm3 \n\t" // imag part x1
"vbroadcastss 16(%2), %%ymm4 \n\t" // real part x2
"vbroadcastss 20(%2), %%ymm5 \n\t" // imag part x2
"vbroadcastss 24(%2), %%ymm6 \n\t" // real part x3
"vbroadcastss 28(%2), %%ymm7 \n\t" // imag part x3
".align 16 \n\t"
".L01LOOP%=: \n\t"
"prefetcht0 320(%4,%0,4) \n\t"
"vmovups (%4,%0,4), %%ymm8 \n\t" // 4 complex values form a0
"vmovups 32(%4,%0,4), %%ymm9 \n\t" // 4 complex values form a0
"prefetcht0 320(%5,%0,4) \n\t"
"vmovups (%5,%0,4), %%ymm10 \n\t" // 4 complex values form a1
"vmovups 32(%5,%0,4), %%ymm11 \n\t" // 4 complex values form a1
"vmulps %%ymm8 , %%ymm0, %%ymm12 \n\t" // a_r[0] * x_r , a_i[0] * x_r, a_r[1] * x_r, a_i[1] * x_r
"vmulps %%ymm8 , %%ymm1, %%ymm13 \n\t" // a_r[0] * x_i , a_i[0] * x_i, a_r[1] * x_i, a_i[1] * x_i
"vmulps %%ymm9 , %%ymm0, %%ymm14 \n\t" // a_r[2] * x_r , a_i[2] * x_r, a_r[3] * x_r, a_i[3] * x_r
"vmulps %%ymm9 , %%ymm1, %%ymm15 \n\t" // a_r[2] * x_i , a_i[2] * x_i, a_r[3] * x_i, a_i[3] * x_i
"prefetcht0 320(%6,%0,4) \n\t"
"vmovups (%6,%0,4), %%ymm8 \n\t" // 4 complex values form a2
"vmovups 32(%6,%0,4), %%ymm9 \n\t" // 4 complex values form a2
"vfmadd231ps %%ymm10, %%ymm2, %%ymm12 \n\t" // a_r[0] * x_r , a_i[0] * x_r, a_r[1] * x_r, a_i[1] * x_r
"vfmadd231ps %%ymm10, %%ymm3, %%ymm13 \n\t" // a_r[0] * x_i , a_i[0] * x_i, a_r[1] * x_i, a_i[1] * x_i
"vfmadd231ps %%ymm11, %%ymm2, %%ymm14 \n\t" // a_r[2] * x_r , a_i[2] * x_r, a_r[3] * x_r, a_i[3] * x_r
"vfmadd231ps %%ymm11, %%ymm3, %%ymm15 \n\t" // a_r[2] * x_i , a_i[2] * x_i, a_r[3] * x_i, a_i[3] * x_i
"prefetcht0 320(%7,%0,4) \n\t"
"vmovups (%7,%0,4), %%ymm10 \n\t" // 4 complex values form a3
"vmovups 32(%7,%0,4), %%ymm11 \n\t" // 4 complex values form a3
"vfmadd231ps %%ymm8 , %%ymm4, %%ymm12 \n\t" // a_r[0] * x_r , a_i[0] * x_r, a_r[1] * x_r, a_i[1] * x_r
"vfmadd231ps %%ymm8 , %%ymm5, %%ymm13 \n\t" // a_r[0] * x_i , a_i[0] * x_i, a_r[1] * x_i, a_i[1] * x_i
"vfmadd231ps %%ymm9 , %%ymm4, %%ymm14 \n\t" // a_r[2] * x_r , a_i[2] * x_r, a_r[3] * x_r, a_i[3] * x_r
"vfmadd231ps %%ymm9 , %%ymm5, %%ymm15 \n\t" // a_r[2] * x_i , a_i[2] * x_i, a_r[3] * x_i, a_i[3] * x_i
"vfmadd231ps %%ymm10, %%ymm6, %%ymm12 \n\t" // a_r[0] * x_r , a_i[0] * x_r, a_r[1] * x_r, a_i[1] * x_r
"vfmadd231ps %%ymm10, %%ymm7, %%ymm13 \n\t" // a_r[0] * x_i , a_i[0] * x_i, a_r[1] * x_i, a_i[1] * x_i
"vfmadd231ps %%ymm11, %%ymm6, %%ymm14 \n\t" // a_r[2] * x_r , a_i[2] * x_r, a_r[3] * x_r, a_i[3] * x_r
"vfmadd231ps %%ymm11, %%ymm7, %%ymm15 \n\t" // a_r[2] * x_i , a_i[2] * x_i, a_r[3] * x_i, a_i[3] * x_i
"prefetcht0 320(%3,%0,4) \n\t"
"vmovups (%3,%0,4), %%ymm10 \n\t"
"vmovups 32(%3,%0,4), %%ymm11 \n\t"
#if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )
"vpermilps $0xb1 , %%ymm13, %%ymm13 \n\t"
"vpermilps $0xb1 , %%ymm15, %%ymm15 \n\t"
"vaddsubps %%ymm13, %%ymm12, %%ymm8 \n\t"
"vaddsubps %%ymm15, %%ymm14, %%ymm9 \n\t"
#else
"vpermilps $0xb1 , %%ymm12, %%ymm12 \n\t"
"vpermilps $0xb1 , %%ymm14, %%ymm14 \n\t"
"vaddsubps %%ymm12, %%ymm13, %%ymm8 \n\t"
"vaddsubps %%ymm14, %%ymm15, %%ymm9 \n\t"
"vpermilps $0xb1 , %%ymm8 , %%ymm8 \n\t"
"vpermilps $0xb1 , %%ymm9 , %%ymm9 \n\t"
#endif
"vaddps %%ymm8, %%ymm10, %%ymm12 \n\t"
"vaddps %%ymm9, %%ymm11, %%ymm13 \n\t"
"vmovups %%ymm12, (%3,%0,4) \n\t" // 4 complex values to y
"vmovups %%ymm13, 32(%3,%0,4) \n\t"
"addq $16, %0 \n\t"
"subq $8 , %1 \n\t"
"jnz .L01LOOP%= \n\t"
"vzeroupper \n\t"
:
:
"r" (i), // 0
"r" (n), // 1
"r" (x), // 2
"r" (y), // 3
"r" (ap[0]), // 4
"r" (ap[1]), // 5
"r" (ap[2]), // 6
"r" (ap[3]) // 7
: "cc",
"%xmm0", "%xmm1", "%xmm2", "%xmm3",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}

265
kernel/x86_64/cgemv_t.c Normal file
View File

@ -0,0 +1,265 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#include "common.h"
#if defined(HASWELL)
#include "cgemv_t_microk_haswell-2.c"
#endif
#define NBMAX 2048
#ifndef HAVE_KERNEL_16x4
static void cgemv_kernel_16x4(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)
{
BLASLONG i;
FLOAT *a0,*a1,*a2,*a3;
a0 = ap[0];
a1 = ap[1];
a2 = ap[2];
a3 = ap[3];
FLOAT temp_r0 = 0.0;
FLOAT temp_r1 = 0.0;
FLOAT temp_r2 = 0.0;
FLOAT temp_r3 = 0.0;
FLOAT temp_i0 = 0.0;
FLOAT temp_i1 = 0.0;
FLOAT temp_i2 = 0.0;
FLOAT temp_i3 = 0.0;
for ( i=0; i< 2*n; i+=2 )
{
#if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )
temp_r0 += a0[i]*x[i] - a0[i+1]*x[i+1];
temp_i0 += a0[i]*x[i+1] + a0[i+1]*x[i];
temp_r1 += a1[i]*x[i] - a1[i+1]*x[i+1];
temp_i1 += a1[i]*x[i+1] + a1[i+1]*x[i];
temp_r2 += a2[i]*x[i] - a2[i+1]*x[i+1];
temp_i2 += a2[i]*x[i+1] + a2[i+1]*x[i];
temp_r3 += a3[i]*x[i] - a3[i+1]*x[i+1];
temp_i3 += a3[i]*x[i+1] + a3[i+1]*x[i];
#else
temp_r0 += a0[i]*x[i] + a0[i+1]*x[i+1];
temp_i0 += a0[i]*x[i+1] - a0[i+1]*x[i];
temp_r1 += a1[i]*x[i] + a1[i+1]*x[i+1];
temp_i1 += a1[i]*x[i+1] - a1[i+1]*x[i];
temp_r2 += a2[i]*x[i] + a2[i+1]*x[i+1];
temp_i2 += a2[i]*x[i+1] - a2[i+1]*x[i];
temp_r3 += a3[i]*x[i] + a3[i+1]*x[i+1];
temp_i3 += a3[i]*x[i+1] - a3[i+1]*x[i];
#endif
}
y[0] = temp_r0;
y[1] = temp_i0;
y[2] = temp_r1;
y[3] = temp_i1;
y[4] = temp_r2;
y[5] = temp_i2;
y[6] = temp_r3;
y[7] = temp_i3;
}
#endif
static void cgemv_kernel_16x1(BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y)
{
BLASLONG i;
FLOAT *a0;
a0 = ap;
FLOAT temp_r = 0.0;
FLOAT temp_i = 0.0;
for ( i=0; i< 2*n; i+=2 )
{
#if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )
temp_r += a0[i]*x[i] - a0[i+1]*x[i+1];
temp_i += a0[i]*x[i+1] + a0[i+1]*x[i];
#else
temp_r += a0[i]*x[i] + a0[i+1]*x[i+1];
temp_i += a0[i]*x[i+1] - a0[i+1]*x[i];
#endif
}
*y = temp_r;
*(y+1) = temp_i;
}
static void copy_x(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_src)
{
BLASLONG i;
for ( i=0; i<n; i++ )
{
*dest = *src;
*(dest+1) = *(src+1);
dest+=2;
src += inc_src;
}
}
int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r, FLOAT alpha_i, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer)
{
BLASLONG i;
BLASLONG j;
FLOAT *a_ptr;
FLOAT *x_ptr;
FLOAT *y_ptr;
FLOAT *ap[8];
BLASLONG n1;
BLASLONG m1;
BLASLONG m2;
BLASLONG n2;
FLOAT ybuffer[8],*xbuffer;
inc_x *= 2;
inc_y *= 2;
lda *= 2;
xbuffer = buffer;
n1 = n / 4 ;
n2 = n % 4 ;
m1 = m - ( m % 16 );
m2 = (m % NBMAX) - (m % 16) ;
BLASLONG NB = NBMAX;
while ( NB == NBMAX )
{
m1 -= NB;
if ( m1 < 0)
{
if ( m2 == 0 ) break;
NB = m2;
}
y_ptr = y;
a_ptr = a;
x_ptr = x;
copy_x(NB,x_ptr,xbuffer,inc_x);
for( i = 0; i < n1 ; i++)
{
ap[0] = a_ptr;
ap[1] = a_ptr + lda;
ap[2] = ap[1] + lda;
ap[3] = ap[2] + lda;
cgemv_kernel_16x4(NB,ap,xbuffer,ybuffer);
a_ptr += 4 * lda;
#if !defined(XCONJ)
y_ptr[0] += alpha_r * ybuffer[0] - alpha_i * ybuffer[1];
y_ptr[1] += alpha_r * ybuffer[1] + alpha_i * ybuffer[0];
y_ptr += inc_y;
y_ptr[0] += alpha_r * ybuffer[2] - alpha_i * ybuffer[3];
y_ptr[1] += alpha_r * ybuffer[3] + alpha_i * ybuffer[2];
y_ptr += inc_y;
y_ptr[0] += alpha_r * ybuffer[4] - alpha_i * ybuffer[5];
y_ptr[1] += alpha_r * ybuffer[5] + alpha_i * ybuffer[4];
y_ptr += inc_y;
y_ptr[0] += alpha_r * ybuffer[6] - alpha_i * ybuffer[7];
y_ptr[1] += alpha_r * ybuffer[7] + alpha_i * ybuffer[6];
y_ptr += inc_y;
#else
y_ptr[0] += alpha_r * ybuffer[0] + alpha_i * ybuffer[1];
y_ptr[1] -= alpha_r * ybuffer[1] - alpha_i * ybuffer[0];
y_ptr += inc_y;
y_ptr[0] += alpha_r * ybuffer[2] + alpha_i * ybuffer[3];
y_ptr[1] -= alpha_r * ybuffer[3] - alpha_i * ybuffer[2];
y_ptr += inc_y;
y_ptr[0] += alpha_r * ybuffer[4] + alpha_i * ybuffer[5];
y_ptr[1] -= alpha_r * ybuffer[5] - alpha_i * ybuffer[4];
y_ptr += inc_y;
y_ptr[0] += alpha_r * ybuffer[6] + alpha_i * ybuffer[7];
y_ptr[1] -= alpha_r * ybuffer[7] - alpha_i * ybuffer[6];
y_ptr += inc_y;
#endif
}
for( i = 0; i < n2 ; i++)
{
cgemv_kernel_16x1(NB,a_ptr,xbuffer,ybuffer);
a_ptr += 1 * lda;
#if !defined(XCONJ)
y_ptr[0] += alpha_r * ybuffer[0] - alpha_i * ybuffer[1];
y_ptr[1] += alpha_r * ybuffer[1] + alpha_i * ybuffer[0];
y_ptr += inc_y;
#else
y_ptr[0] += alpha_r * ybuffer[0] + alpha_i * ybuffer[1];
y_ptr[1] -= alpha_r * ybuffer[1] - alpha_i * ybuffer[0];
y_ptr += inc_y;
#endif
}
a += 2* NB;
x += NB * inc_x;
}
BLASLONG m3 = m % 16;
if ( m3 == 0 ) return(0);
x_ptr = x;
copy_x(m3,x_ptr,xbuffer,inc_x);
j=0;
a_ptr = a;
y_ptr = y;
while ( j < n)
{
FLOAT temp_r = 0.0;
FLOAT temp_i = 0.0;
for( i = 0; i < m3*2; i+=2 )
{
#if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )
temp_r += a_ptr[i] * xbuffer[i] - a_ptr[i+1] * xbuffer[i+1];
temp_i += a_ptr[i] * xbuffer[i+1] + a_ptr[i+1] * xbuffer[i];
#else
temp_r += a_ptr[i] * xbuffer[i] + a_ptr[i+1] * xbuffer[i+1];
temp_i += a_ptr[i] * xbuffer[i+1] - a_ptr[i+1] * xbuffer[i];
#endif
}
a_ptr += lda;
#if !defined(XCONJ)
y_ptr[0] += alpha_r * temp_r - alpha_i * temp_i;
y_ptr[1] += alpha_r * temp_i + alpha_i * temp_r;
#else
y_ptr[0] += alpha_r * temp_r + alpha_i * temp_i;
y_ptr[1] -= alpha_r * temp_i - alpha_i * temp_r;
#endif
y_ptr += inc_y;
j++;
}
return(0);
}

171
kernel/x86_64/cgemv_t_microk_haswell-2.c Normal file
View File

@ -0,0 +1,171 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary froms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary from must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#define HAVE_KERNEL_16x4 1
static void cgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) __attribute__ ((noinline));
static void cgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)
{
BLASLONG register i = 0;
__asm__ __volatile__
(
"vzeroupper \n\t"
"vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // temp
"vxorps %%ymm9 , %%ymm9 , %%ymm9 \n\t" // temp
"vxorps %%ymm10, %%ymm10, %%ymm10 \n\t" // temp
"vxorps %%ymm11, %%ymm11, %%ymm11 \n\t" // temp
"vxorps %%ymm12, %%ymm12, %%ymm12 \n\t" // temp
"vxorps %%ymm13, %%ymm13, %%ymm13 \n\t"
"vxorps %%ymm14, %%ymm14, %%ymm14 \n\t"
"vxorps %%ymm15, %%ymm15, %%ymm15 \n\t"
".align 16 \n\t"
".L01LOOP%=: \n\t"
"prefetcht0 192(%4,%0,4) \n\t"
"vmovups (%4,%0,4), %%ymm4 \n\t" // 4 complex values from a0
"prefetcht0 192(%5,%0,4) \n\t"
"vmovups (%5,%0,4), %%ymm5 \n\t" // 4 complex values from a1
"prefetcht0 192(%2,%0,4) \n\t"
"vmovups (%2,%0,4) , %%ymm6 \n\t" // 4 complex values from x
"vpermilps $0xb1, %%ymm6, %%ymm7 \n\t" // exchange real and imap parts
"vblendps $0x55, %%ymm6, %%ymm7, %%ymm0 \n\t" // only the real parts
"vblendps $0x55, %%ymm7, %%ymm6, %%ymm1 \n\t" // only the imag parts
"prefetcht0 192(%6,%0,4) \n\t"
"vmovups (%6,%0,4), %%ymm6 \n\t" // 4 complex values from a2
"prefetcht0 192(%7,%0,4) \n\t"
"vmovups (%7,%0,4), %%ymm7 \n\t" // 4 complex values from a3
"vfmadd231ps %%ymm4 , %%ymm0, %%ymm8 \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1
"vfmadd231ps %%ymm4 , %%ymm1, %%ymm9 \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1
"vfmadd231ps %%ymm5 , %%ymm0, %%ymm10 \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1
"vfmadd231ps %%ymm5 , %%ymm1, %%ymm11 \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1
"vfmadd231ps %%ymm6 , %%ymm0, %%ymm12 \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1
"vfmadd231ps %%ymm6 , %%ymm1, %%ymm13 \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1
"vfmadd231ps %%ymm7 , %%ymm0, %%ymm14 \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1
"vfmadd231ps %%ymm7 , %%ymm1, %%ymm15 \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1
"vmovups 32(%4,%0,4), %%ymm4 \n\t" // 2 complex values from a0
"vmovups 32(%5,%0,4), %%ymm5 \n\t" // 2 complex values from a1
"vmovups 32(%2,%0,4) , %%ymm6 \n\t" // 4 complex values from x
"vpermilps $0xb1, %%ymm6, %%ymm7 \n\t" // exchange real and imap parts
"vblendps $0x55, %%ymm6, %%ymm7, %%ymm0 \n\t" // only the real parts
"vblendps $0x55, %%ymm7, %%ymm6, %%ymm1 \n\t" // only the imag parts
"vmovups 32(%6,%0,4), %%ymm6 \n\t" // 2 complex values from a2
"vmovups 32(%7,%0,4), %%ymm7 \n\t" // 2 complex values from a3
"vfmadd231ps %%ymm4 , %%ymm0, %%ymm8 \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1
"vfmadd231ps %%ymm4 , %%ymm1, %%ymm9 \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1
"vfmadd231ps %%ymm5 , %%ymm0, %%ymm10 \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1
"vfmadd231ps %%ymm5 , %%ymm1, %%ymm11 \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1
"vfmadd231ps %%ymm6 , %%ymm0, %%ymm12 \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1
"vfmadd231ps %%ymm6 , %%ymm1, %%ymm13 \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1
"vfmadd231ps %%ymm7 , %%ymm0, %%ymm14 \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1
"vfmadd231ps %%ymm7 , %%ymm1, %%ymm15 \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1
"addq $16 , %0 \n\t"
"subq $8 , %1 \n\t"
"jnz .L01LOOP%= \n\t"
#if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )
"vpermilps $0xb1 , %%ymm9 , %%ymm9 \n\t"
"vpermilps $0xb1 , %%ymm11, %%ymm11 \n\t"
"vpermilps $0xb1 , %%ymm13, %%ymm13 \n\t"
"vpermilps $0xb1 , %%ymm15, %%ymm15 \n\t"
"vaddsubps %%ymm9 , %%ymm8, %%ymm8 \n\t"
"vaddsubps %%ymm11, %%ymm10, %%ymm10 \n\t"
"vaddsubps %%ymm13, %%ymm12, %%ymm12 \n\t"
"vaddsubps %%ymm15, %%ymm14, %%ymm14 \n\t"
#else
"vpermilps $0xb1 , %%ymm8 , %%ymm8 \n\t"
"vpermilps $0xb1 , %%ymm10, %%ymm10 \n\t"
"vpermilps $0xb1 , %%ymm12, %%ymm12 \n\t"
"vpermilps $0xb1 , %%ymm14, %%ymm14 \n\t"
"vaddsubps %%ymm8 , %%ymm9 , %%ymm8 \n\t"
"vaddsubps %%ymm10, %%ymm11, %%ymm10 \n\t"
"vaddsubps %%ymm12, %%ymm13, %%ymm12 \n\t"
"vaddsubps %%ymm14, %%ymm15, %%ymm14 \n\t"
"vpermilps $0xb1 , %%ymm8 , %%ymm8 \n\t"
"vpermilps $0xb1 , %%ymm10, %%ymm10 \n\t"
"vpermilps $0xb1 , %%ymm12, %%ymm12 \n\t"
"vpermilps $0xb1 , %%ymm14, %%ymm14 \n\t"
#endif
"vextractf128 $1, %%ymm8 , %%xmm9 \n\t"
"vextractf128 $1, %%ymm10, %%xmm11 \n\t"
"vextractf128 $1, %%ymm12, %%xmm13 \n\t"
"vextractf128 $1, %%ymm14, %%xmm15 \n\t"
"vaddps %%xmm8 , %%xmm9 , %%xmm8 \n\t"
"vaddps %%xmm10, %%xmm11, %%xmm10 \n\t"
"vaddps %%xmm12, %%xmm13, %%xmm12 \n\t"
"vaddps %%xmm14, %%xmm15, %%xmm14 \n\t"
"vshufpd $0x1, %%xmm8 , %%xmm8 , %%xmm9 \n\t"
"vshufpd $0x1, %%xmm10, %%xmm10, %%xmm11 \n\t"
"vshufpd $0x1, %%xmm12, %%xmm12, %%xmm13 \n\t"
"vshufpd $0x1, %%xmm14, %%xmm14, %%xmm15 \n\t"
"vaddps %%xmm8 , %%xmm9 , %%xmm8 \n\t"
"vaddps %%xmm10, %%xmm11, %%xmm10 \n\t"
"vaddps %%xmm12, %%xmm13, %%xmm12 \n\t"
"vaddps %%xmm14, %%xmm15, %%xmm14 \n\t"
"vmovsd %%xmm8 , (%3) \n\t"
"vmovsd %%xmm10, 8(%3) \n\t"
"vmovsd %%xmm12, 16(%3) \n\t"
"vmovsd %%xmm14, 24(%3) \n\t"
"vzeroupper \n\t"
:
:
"r" (i), // 0
"r" (n), // 1
"r" (x), // 2
"r" (y), // 3
"r" (ap[0]), // 4
"r" (ap[1]), // 5
"r" (ap[2]), // 6
"r" (ap[3]) // 7
: "cc",
"%xmm0", "%xmm1", "%xmm2", "%xmm3",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}

206
kernel/x86_64/dgemv_n.c Normal file
View File

@ -0,0 +1,206 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#include "common.h"
#if defined(HASWELL)
#include "dgemv_n_microk_haswell-2.c"
#endif
#define NBMAX 2048
#ifndef HAVE_KERNEL_16x4
static void dgemv_kernel_16x4(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)
{
BLASLONG i;
FLOAT *a0,*a1,*a2,*a3;
a0 = ap[0];
a1 = ap[1];
a2 = ap[2];
a3 = ap[3];
for ( i=0; i< n; i+=4 )
{
y[i] += a0[i]*x[0] + a1[i]*x[1] + a2[i]*x[2] + a3[i]*x[3];
y[i+1] += a0[i+1]*x[0] + a1[i+1]*x[1] + a2[i+1]*x[2] + a3[i+1]*x[3];
y[i+2] += a0[i+2]*x[0] + a1[i+2]*x[1] + a2[i+2]*x[2] + a3[i+2]*x[3];
y[i+3] += a0[i+3]*x[0] + a1[i+3]*x[1] + a2[i+3]*x[2] + a3[i+3]*x[3];
}
}
#endif
static void dgemv_kernel_16x1(BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y)
{
BLASLONG i;
FLOAT *a0;
a0 = ap;
for ( i=0; i< n; i+=4 )
{
y[i] += a0[i]*x[0];
y[i+1] += a0[i+1]*x[0];
y[i+2] += a0[i+2]*x[0];
y[i+3] += a0[i+3]*x[0];
}
}
static void zero_y(BLASLONG n, FLOAT *dest)
{
BLASLONG i;
for ( i=0; i<n; i++ )
{
*dest = 0.0;
dest++;
}
}
static void add_y(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_dest)
{
BLASLONG i;
if ( inc_dest == 1 )
{
for ( i=0; i<n; i+=4 )
{
dest[i] += src[i];
dest[i+1] += src[i+1];
dest[i+2] += src[i+2];
dest[i+3] += src[i+3];
}
}
else
{
for ( i=0; i<n; i++ )
{
*dest += *src;
src++;
dest += inc_dest;
}
}
}
int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer)
{
BLASLONG i;
BLASLONG j;
FLOAT *a_ptr;
FLOAT *x_ptr;
FLOAT *y_ptr;
FLOAT *ap[4];
BLASLONG n1;
BLASLONG m1;
BLASLONG m2;
BLASLONG n2;
FLOAT xbuffer[4],*ybuffer;
if ( m < 1 ) return(0);
if ( n < 1 ) return(0);
ybuffer = buffer;
n1 = n / 4 ;
n2 = n % 4 ;
m1 = m - ( m % 16 );
m2 = (m % NBMAX) - (m % 16) ;
y_ptr = y;
BLASLONG NB = NBMAX;
while ( NB == NBMAX )
{
m1 -= NB;
if ( m1 < 0)
{
if ( m2 == 0 ) break;
NB = m2;
}
a_ptr = a;
x_ptr = x;
zero_y(NB,ybuffer);
for( i = 0; i < n1 ; i++)
{
xbuffer[0] = alpha * x_ptr[0];
x_ptr += inc_x;
xbuffer[1] = alpha * x_ptr[0];
x_ptr += inc_x;
xbuffer[2] = alpha * x_ptr[0];
x_ptr += inc_x;
xbuffer[3] = alpha * x_ptr[0];
x_ptr += inc_x;
ap[0] = a_ptr;
ap[1] = a_ptr + lda;
ap[2] = ap[1] + lda;
ap[3] = ap[2] + lda;
dgemv_kernel_16x4(NB,ap,xbuffer,ybuffer);
a_ptr += 4 * lda;
}
for( i = 0; i < n2 ; i++)
{
xbuffer[0] = alpha * x_ptr[0];
x_ptr += inc_x;
dgemv_kernel_16x1(NB,a_ptr,xbuffer,ybuffer);
a_ptr += 1 * lda;
}
add_y(NB,ybuffer,y_ptr,inc_y);
a += NB;
y_ptr += NB * inc_y;
}
j=0;
while ( j < (m % 16))
{
a_ptr = a;
x_ptr = x;
FLOAT temp = 0.0;
for( i = 0; i < n; i++ )
{
temp += a_ptr[0] * x_ptr[0];
a_ptr += lda;
x_ptr += inc_x;
}
y_ptr[0] += alpha * temp;
y_ptr += inc_y;
a++;
j++;
}
return(0);
}

89
kernel/x86_64/dgemv_n_microk_haswell-2.c Normal file
View File

@ -0,0 +1,89 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#define HAVE_KERNEL_16x4 1
static void dgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) __attribute__ ((noinline));
static void dgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)
{
BLASLONG register i = 0;
__asm__ __volatile__
(
"vzeroupper \n\t"
"vbroadcastsd (%2), %%ymm12 \n\t" // x0
"vbroadcastsd 8(%2), %%ymm13 \n\t" // x1
"vbroadcastsd 16(%2), %%ymm14 \n\t" // x2
"vbroadcastsd 24(%2), %%ymm15 \n\t" // x3
".align 16 \n\t"
".L01LOOP%=: \n\t"
"prefetcht0 192(%3,%0,8) \n\t"
"vmovups (%3,%0,8), %%ymm4 \n\t" // 4 * y
"vmovups 32(%3,%0,8), %%ymm5 \n\t" // 4 * y
"prefetcht0 192(%4,%0,8) \n\t"
"vfmadd231pd (%4,%0,8), %%ymm12, %%ymm4 \n\t"
"vfmadd231pd 32(%4,%0,8), %%ymm12, %%ymm5 \n\t"
"prefetcht0 192(%5,%0,8) \n\t"
"vfmadd231pd (%5,%0,8), %%ymm13, %%ymm4 \n\t"
"vfmadd231pd 32(%5,%0,8), %%ymm13, %%ymm5 \n\t"
"prefetcht0 192(%6,%0,8) \n\t"
"vfmadd231pd (%6,%0,8), %%ymm14, %%ymm4 \n\t"
"vfmadd231pd 32(%6,%0,8), %%ymm14, %%ymm5 \n\t"
"prefetcht0 192(%7,%0,8) \n\t"
"vfmadd231pd (%7,%0,8), %%ymm15, %%ymm4 \n\t"
"vfmadd231pd 32(%7,%0,8), %%ymm15, %%ymm5 \n\t"
"vmovups %%ymm4, (%3,%0,8) \n\t" // 4 * y
"vmovups %%ymm5, 32(%3,%0,8) \n\t" // 4 * y
"addq $8 , %0 \n\t"
"subq $8 , %1 \n\t"
"jnz .L01LOOP%= \n\t"
"vzeroupper \n\t"
:
:
"r" (i), // 0
"r" (n), // 1
"r" (x), // 2
"r" (y), // 3
"r" (ap[0]), // 4
"r" (ap[1]), // 5
"r" (ap[2]), // 6
"r" (ap[3]) // 7
: "cc",
"%xmm4", "%xmm5",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}

191
kernel/x86_64/dgemv_t.c Normal file
View File

@ -0,0 +1,191 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#include "common.h"
#if defined(HASWELL)
#include "dgemv_t_microk_haswell-2.c"
#endif
#define NBMAX 2048
#ifndef HAVE_KERNEL_16x4
static void dgemv_kernel_16x4(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)
{
BLASLONG i;
FLOAT *a0,*a1,*a2,*a3;
a0 = ap[0];
a1 = ap[1];
a2 = ap[2];
a3 = ap[3];
FLOAT temp0 = 0.0;
FLOAT temp1 = 0.0;
FLOAT temp2 = 0.0;
FLOAT temp3 = 0.0;
for ( i=0; i< n; i+=4 )
{
temp0 += a0[i]*x[i] + a0[i+1]*x[i+1] + a0[i+2]*x[i+2] + a0[i+3]*x[i+3];
temp1 += a1[i]*x[i] + a1[i+1]*x[i+1] + a1[i+2]*x[i+2] + a1[i+3]*x[i+3];
temp2 += a2[i]*x[i] + a2[i+1]*x[i+1] + a2[i+2]*x[i+2] + a2[i+3]*x[i+3];
temp3 += a3[i]*x[i] + a3[i+1]*x[i+1] + a3[i+2]*x[i+2] + a3[i+3]*x[i+3];
}
y[0] = temp0;
y[1] = temp1;
y[2] = temp2;
y[3] = temp3;
}
#endif
static void dgemv_kernel_16x1(BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y)
{
BLASLONG i;
FLOAT *a0;
a0 = ap;
FLOAT temp = 0.0;
for ( i=0; i< n; i+=4 )
{
temp += a0[i]*x[i] + a0[i+1]*x[i+1] + a0[i+2]*x[i+2] + a0[i+3]*x[i+3];
}
*y = temp;
}
static void copy_x(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_src)
{
BLASLONG i;
for ( i=0; i<n; i++ )
{
*dest = *src;
dest++;
src += inc_src;
}
}
int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer)
{
BLASLONG i;
BLASLONG j;
FLOAT *a_ptr;
FLOAT *x_ptr;
FLOAT *y_ptr;
FLOAT *ap[4];
BLASLONG n1;
BLASLONG m1;
BLASLONG m2;
BLASLONG n2;
FLOAT ybuffer[4],*xbuffer;
if ( m < 1 ) return(0);
if ( n < 1 ) return(0);
xbuffer = buffer;
n1 = n / 4 ;
n2 = n % 4 ;
m1 = m - ( m % 16 );
m2 = (m % NBMAX) - (m % 16) ;
BLASLONG NB = NBMAX;
while ( NB == NBMAX )
{
m1 -= NB;
if ( m1 < 0)
{
if ( m2 == 0 ) break;
NB = m2;
}
y_ptr = y;
a_ptr = a;
x_ptr = x;
copy_x(NB,x_ptr,xbuffer,inc_x);
for( i = 0; i < n1 ; i++)
{
ap[0] = a_ptr;
ap[1] = a_ptr + lda;
ap[2] = ap[1] + lda;
ap[3] = ap[2] + lda;
dgemv_kernel_16x4(NB,ap,xbuffer,ybuffer);
a_ptr += 4 * lda;
*y_ptr += ybuffer[0]*alpha;
y_ptr += inc_y;
*y_ptr += ybuffer[1]*alpha;
y_ptr += inc_y;
*y_ptr += ybuffer[2]*alpha;
y_ptr += inc_y;
*y_ptr += ybuffer[3]*alpha;
y_ptr += inc_y;
}
for( i = 0; i < n2 ; i++)
{
dgemv_kernel_16x1(NB,a_ptr,xbuffer,ybuffer);
a_ptr += 1 * lda;
*y_ptr += ybuffer[0]*alpha;
y_ptr += inc_y;
}
a += NB;
x += NB * inc_x;
}
BLASLONG m3 = m % 16;
if ( m3 == 0 ) return(0);
x_ptr = x;
for ( i=0; i< m3; i++ )
{
xbuffer[i] = *x_ptr;
x_ptr += inc_x;
}
j=0;
a_ptr = a;
y_ptr = y;
while ( j < n)
{
FLOAT temp = 0.0;
for( i = 0; i < m3; i++ )
{
temp += a_ptr[i] * xbuffer[i];
}
a_ptr += lda;
y_ptr[0] += alpha * temp;
y_ptr += inc_y;
j++;
}
return(0);
}

107
kernel/x86_64/dgemv_t_microk_haswell-2.c Normal file
View File

@ -0,0 +1,107 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#define HAVE_KERNEL_16x4 1
static void dgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) __attribute__ ((noinline));
static void dgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)
{
BLASLONG register i = 0;
__asm__ __volatile__
(
"vzeroupper \n\t"
"vxorpd %%ymm4 , %%ymm4, %%ymm4 \n\t"
"vxorpd %%ymm5 , %%ymm5, %%ymm5 \n\t"
"vxorpd %%ymm6 , %%ymm6, %%ymm6 \n\t"
"vxorpd %%ymm7 , %%ymm7, %%ymm7 \n\t"
".align 16 \n\t"
".L01LOOP%=: \n\t"
"prefetcht0 384(%2,%0,8) \n\t"
"vmovups (%2,%0,8), %%ymm12 \n\t" // 4 * x
"vmovups 32(%2,%0,8), %%ymm13 \n\t" // 4 * x
"prefetcht0 384(%4,%0,8) \n\t"
"vfmadd231pd (%4,%0,8), %%ymm12, %%ymm4 \n\t"
"vfmadd231pd (%5,%0,8), %%ymm12, %%ymm5 \n\t"
"prefetcht0 384(%5,%0,8) \n\t"
"vfmadd231pd 32(%4,%0,8), %%ymm13, %%ymm4 \n\t"
"vfmadd231pd 32(%5,%0,8), %%ymm13, %%ymm5 \n\t"
"prefetcht0 384(%6,%0,8) \n\t"
"vfmadd231pd (%6,%0,8), %%ymm12, %%ymm6 \n\t"
"vfmadd231pd (%7,%0,8), %%ymm12, %%ymm7 \n\t"
"prefetcht0 384(%7,%0,8) \n\t"
"vfmadd231pd 32(%6,%0,8), %%ymm13, %%ymm6 \n\t"
"vfmadd231pd 32(%7,%0,8), %%ymm13, %%ymm7 \n\t"
"addq $8 , %0 \n\t"
"subq $8 , %1 \n\t"
"jnz .L01LOOP%= \n\t"
"vextractf128 $1 , %%ymm4, %%xmm12 \n\t"
"vextractf128 $1 , %%ymm5, %%xmm13 \n\t"
"vextractf128 $1 , %%ymm6, %%xmm14 \n\t"
"vextractf128 $1 , %%ymm7, %%xmm15 \n\t"
"vaddpd %%xmm4, %%xmm12, %%xmm4 \n\t"
"vaddpd %%xmm5, %%xmm13, %%xmm5 \n\t"
"vaddpd %%xmm6, %%xmm14, %%xmm6 \n\t"
"vaddpd %%xmm7, %%xmm15, %%xmm7 \n\t"
"vhaddpd %%xmm4, %%xmm4, %%xmm4 \n\t"
"vhaddpd %%xmm5, %%xmm5, %%xmm5 \n\t"
"vhaddpd %%xmm6, %%xmm6, %%xmm6 \n\t"
"vhaddpd %%xmm7, %%xmm7, %%xmm7 \n\t"
"vmovsd %%xmm4, (%3) \n\t"
"vmovsd %%xmm5, 8(%3) \n\t"
"vmovsd %%xmm6, 16(%3) \n\t"
"vmovsd %%xmm7, 24(%3) \n\t"
"vzeroupper \n\t"
:
:
"r" (i), // 0
"r" (n), // 1
"r" (x), // 2
"r" (y), // 3
"r" (ap[0]), // 4
"r" (ap[1]), // 5
"r" (ap[2]), // 6
"r" (ap[3]) // 7
: "cc",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}

3707
kernel/x86_64/sgemm_kernel_16x4_haswell.S
View File

File diff suppressed because it is too large Load Diff

212
kernel/x86_64/sgemv_n.c Normal file
View File

@ -0,0 +1,212 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#include "common.h"
#if defined(BULLDOZER) || defined(PILEDRIVER)
#include "sgemv_n_microk_bulldozer-2.c"
#elif defined(HASWELL)
#include "sgemv_n_microk_haswell-2.c"
#elif defined(SANDYBRIDGE)
#include "sgemv_n_microk_sandy-2.c"
#elif defined(NEHALEM)
#include "sgemv_n_microk_nehalem-2.c"
#endif
#define NBMAX 4096
#ifndef HAVE_KERNEL_16x4
static void sgemv_kernel_16x4(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)
{
BLASLONG i;
FLOAT *a0,*a1,*a2,*a3;
a0 = ap[0];
a1 = ap[1];
a2 = ap[2];
a3 = ap[3];
for ( i=0; i< n; i+=4 )
{
y[i] += a0[i]*x[0] + a1[i]*x[1] + a2[i]*x[2] + a3[i]*x[3];
y[i+1] += a0[i+1]*x[0] + a1[i+1]*x[1] + a2[i+1]*x[2] + a3[i+1]*x[3];
y[i+2] += a0[i+2]*x[0] + a1[i+2]*x[1] + a2[i+2]*x[2] + a3[i+2]*x[3];
y[i+3] += a0[i+3]*x[0] + a1[i+3]*x[1] + a2[i+3]*x[2] + a3[i+3]*x[3];
}
}
#endif
static void sgemv_kernel_16x1(BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y)
{
BLASLONG i;
FLOAT *a0;
a0 = ap;
for ( i=0; i< n; i+=4 )
{
y[i] += a0[i]*x[0];
y[i+1] += a0[i+1]*x[0];
y[i+2] += a0[i+2]*x[0];
y[i+3] += a0[i+3]*x[0];
}
}
static void zero_y(BLASLONG n, FLOAT *dest)
{
BLASLONG i;
for ( i=0; i<n; i++ )
{
*dest = 0.0;
dest++;
}
}
static void add_y(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_dest)
{
BLASLONG i;
if ( inc_dest == 1 )
{
for ( i=0; i<n; i+=4 )
{
dest[i] += src[i];
dest[i+1] += src[i+1];
dest[i+2] += src[i+2];
dest[i+3] += src[i+3];
}
}
else
{
for ( i=0; i<n; i++ )
{
*dest += *src;
src++;
dest += inc_dest;
}
}
}
int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer)
{
BLASLONG i;
BLASLONG j;
FLOAT *a_ptr;
FLOAT *x_ptr;
FLOAT *y_ptr;
FLOAT *ap[4];
BLASLONG n1;
BLASLONG m1;
BLASLONG m2;
BLASLONG n2;
FLOAT xbuffer[4],*ybuffer;
if ( m < 1 ) return(0);
if ( n < 1 ) return(0);
ybuffer = buffer;
n1 = n / 4 ;
n2 = n % 4 ;
m1 = m - ( m % 16 );
m2 = (m % NBMAX) - (m % 16) ;
y_ptr = y;
BLASLONG NB = NBMAX;
while ( NB == NBMAX )
{
m1 -= NB;
if ( m1 < 0)
{
if ( m2 == 0 ) break;
NB = m2;
}
a_ptr = a;
x_ptr = x;
zero_y(NB,ybuffer);
for( i = 0; i < n1 ; i++)
{
xbuffer[0] = alpha * x_ptr[0];
x_ptr += inc_x;
xbuffer[1] = alpha * x_ptr[0];
x_ptr += inc_x;
xbuffer[2] = alpha * x_ptr[0];
x_ptr += inc_x;
xbuffer[3] = alpha * x_ptr[0];
x_ptr += inc_x;
ap[0] = a_ptr;
ap[1] = a_ptr + lda;
ap[2] = ap[1] + lda;
ap[3] = ap[2] + lda;
sgemv_kernel_16x4(NB,ap,xbuffer,ybuffer);
a_ptr += 4 * lda;
}
for( i = 0; i < n2 ; i++)
{
xbuffer[0] = alpha * x_ptr[0];
x_ptr += inc_x;
sgemv_kernel_16x1(NB,a_ptr,xbuffer,ybuffer);
a_ptr += 1 * lda;
}
add_y(NB,ybuffer,y_ptr,inc_y);
a += NB;
y_ptr += NB * inc_y;
}
j=0;
while ( j < (m % 16))
{
a_ptr = a;
x_ptr = x;
FLOAT temp = 0.0;
for( i = 0; i < n; i++ )
{
temp += a_ptr[0] * x_ptr[0];
a_ptr += lda;
x_ptr += inc_x;
}
y_ptr[0] += alpha * temp;
y_ptr += inc_y;
a++;
j++;
}
return(0);
}

218
kernel/x86_64/sgemv_n_avx.c Normal file
View File

@ -0,0 +1,218 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#include "common.h"
#if defined(BULLDOZER) || defined(PILEDRIVER)
#include "sgemv_n_microk_bulldozer.c"
#elif defined(HASWELL)
#include "sgemv_n_microk_haswell.c"
#else
#include "sgemv_n_microk_sandy.c"
#endif
static void copy_x(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_src)
{
BLASLONG i;
for ( i=0; i<n; i++ )
{
*dest = *src;
dest++;
src += inc_src;
}
}
static void add_y(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_dest)
{
BLASLONG i;
for ( i=0; i<n; i++ )
{
*dest += *src;
src++;
dest += inc_dest;
}
}
int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer)
{
BLASLONG i;
BLASLONG j;
FLOAT *a_ptr;
FLOAT *x_ptr;
FLOAT *y_ptr;
BLASLONG n1;
BLASLONG m1;
BLASLONG register m2;
BLASLONG register n2;
FLOAT *xbuffer,*ybuffer;
xbuffer = buffer;
ybuffer = xbuffer + 2048 + 256;
n1 = n / 512 ;
n2 = n % 512 ;
m1 = m / 64;
m2 = m % 64;
y_ptr = y;
x_ptr = x;
for (j=0; j<n1; j++)
{
if ( inc_x == 1 )
xbuffer = x_ptr;
else
copy_x(512,x_ptr,xbuffer,inc_x);
a_ptr = a + j * 512 * lda;
y_ptr = y;
for(i = 0; i<m1; i++ )
{
sgemv_kernel_64(512,alpha,a_ptr,lda,xbuffer,ybuffer);
add_y(64,ybuffer,y_ptr,inc_y);
y_ptr += 64 * inc_y;
a_ptr += 64;
}
if ( m2 & 32 )
{
sgemv_kernel_32(512,alpha,a_ptr,lda,xbuffer,ybuffer);
add_y(32,ybuffer,y_ptr,inc_y);
y_ptr += 32 * inc_y;
a_ptr += 32;
}
if ( m2 & 16 )
{
sgemv_kernel_16(512,alpha,a_ptr,lda,xbuffer,ybuffer);
add_y(16,ybuffer,y_ptr,inc_y);
y_ptr += 16 * inc_y;
a_ptr += 16;
}
if ( m2 & 8 )
{
sgemv_kernel_8(512,alpha,a_ptr,lda,xbuffer,ybuffer);
add_y(8,ybuffer,y_ptr,inc_y);
y_ptr += 8 * inc_y;
a_ptr += 8;
}
if ( m2 & 4 )
{
sgemv_kernel_4(512,alpha,a_ptr,lda,xbuffer,ybuffer);
add_y(4,ybuffer,y_ptr,inc_y);
y_ptr += 4 * inc_y;
a_ptr += 4;
}
if ( m2 & 2 )
{
sgemv_kernel_2(512,alpha,a_ptr,lda,xbuffer,ybuffer);
add_y(2,ybuffer,y_ptr,inc_y);
y_ptr += 2 * inc_y;
a_ptr += 2;
}
if ( m2 & 1 )
{
sgemv_kernel_1(512,alpha,a_ptr,lda,xbuffer,ybuffer);
add_y(1,ybuffer,y_ptr,inc_y);
}
x_ptr += 512 * inc_x;
}
if ( n2 > 0 )
{
if ( inc_x == 1 )
xbuffer = x_ptr;
else
copy_x(n2,x_ptr,xbuffer,inc_x);
a_ptr = a + n1 * 512 * lda;
y_ptr = y;
for(i = 0; i<m1; i++ )
{
sgemv_kernel_64(n2,alpha,a_ptr,lda,xbuffer,ybuffer);
add_y(64,ybuffer,y_ptr,inc_y);
y_ptr += 64 * inc_y;
a_ptr += 64;
}
if ( m2 & 32 )
{
sgemv_kernel_32(n2,alpha,a_ptr,lda,xbuffer,ybuffer);
add_y(32,ybuffer,y_ptr,inc_y);
y_ptr += 32 * inc_y;
a_ptr += 32;
}
if ( m2 & 16 )
{
sgemv_kernel_16(n2,alpha,a_ptr,lda,xbuffer,ybuffer);
add_y(16,ybuffer,y_ptr,inc_y);
y_ptr += 16 * inc_y;
a_ptr += 16;
}
if ( m2 & 8 )
{
sgemv_kernel_8(n2,alpha,a_ptr,lda,xbuffer,ybuffer);
add_y(8,ybuffer,y_ptr,inc_y);
y_ptr += 8 * inc_y;
a_ptr += 8;
}
if ( m2 & 4 )
{
sgemv_kernel_4(n2,alpha,a_ptr,lda,xbuffer,ybuffer);
add_y(4,ybuffer,y_ptr,inc_y);
y_ptr += 4 * inc_y;
a_ptr += 4;
}
if ( m2 & 2 )
{
sgemv_kernel_2(n2,alpha,a_ptr,lda,xbuffer,ybuffer);
add_y(2,ybuffer,y_ptr,inc_y);
y_ptr += 2 * inc_y;
a_ptr += 2;
}
if ( m2 & 1 )
{
sgemv_kernel_1(n2,alpha,a_ptr,lda,xbuffer,ybuffer);
add_y(1,ybuffer,y_ptr,inc_y);
}
}
return(0);
}

99
kernel/x86_64/sgemv_n_microk_bulldozer-2.c Normal file
View File

@ -0,0 +1,99 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#define HAVE_KERNEL_16x4 1
static void sgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) __attribute__ ((noinline));
static void sgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)
{
BLASLONG register i = 0;
__asm__ __volatile__
(
"vbroadcastss (%2), %%xmm12 \n\t" // x0
"vbroadcastss 4(%2), %%xmm13 \n\t" // x1
"vbroadcastss 8(%2), %%xmm14 \n\t" // x2
"vbroadcastss 12(%2), %%xmm15 \n\t" // x3
".align 16 \n\t"
".L01LOOP%=: \n\t"
"vmovups (%3,%0,4), %%xmm4 \n\t" // 4 * y
"vmovups 16(%3,%0,4), %%xmm5 \n\t" // 4 * y
"vmovups 32(%3,%0,4), %%xmm6 \n\t" // 4 * y
"vmovups 48(%3,%0,4), %%xmm7 \n\t" // 4 * y
"prefetcht0 192(%4,%0,4) \n\t"
"vfmaddps %%xmm4, (%4,%0,4), %%xmm12, %%xmm4 \n\t"
"vfmaddps %%xmm5, 16(%4,%0,4), %%xmm12, %%xmm5 \n\t"
"vfmaddps %%xmm6, 32(%4,%0,4), %%xmm12, %%xmm6 \n\t"
"vfmaddps %%xmm7, 48(%4,%0,4), %%xmm12, %%xmm7 \n\t"
"prefetcht0 192(%5,%0,4) \n\t"
"vfmaddps %%xmm4, (%5,%0,4), %%xmm13, %%xmm4 \n\t"
"vfmaddps %%xmm5, 16(%5,%0,4), %%xmm13, %%xmm5 \n\t"
"vfmaddps %%xmm6, 32(%5,%0,4), %%xmm13, %%xmm6 \n\t"
"vfmaddps %%xmm7, 48(%5,%0,4), %%xmm13, %%xmm7 \n\t"
"prefetcht0 192(%6,%0,4) \n\t"
"vfmaddps %%xmm4, (%6,%0,4), %%xmm14, %%xmm4 \n\t"
"vfmaddps %%xmm5, 16(%6,%0,4), %%xmm14, %%xmm5 \n\t"
"vfmaddps %%xmm6, 32(%6,%0,4), %%xmm14, %%xmm6 \n\t"
"vfmaddps %%xmm7, 48(%6,%0,4), %%xmm14, %%xmm7 \n\t"
"prefetcht0 192(%7,%0,4) \n\t"
"vfmaddps %%xmm4, (%7,%0,4), %%xmm15, %%xmm4 \n\t"
"vfmaddps %%xmm5, 16(%7,%0,4), %%xmm15, %%xmm5 \n\t"
"vfmaddps %%xmm6, 32(%7,%0,4), %%xmm15, %%xmm6 \n\t"
"vfmaddps %%xmm7, 48(%7,%0,4), %%xmm15, %%xmm7 \n\t"
"vmovups %%xmm4, (%3,%0,4) \n\t" // 4 * y
"vmovups %%xmm5, 16(%3,%0,4) \n\t" // 4 * y
"vmovups %%xmm6, 32(%3,%0,4) \n\t" // 4 * y
"vmovups %%xmm7, 48(%3,%0,4) \n\t" // 4 * y
"addq $16, %0 \n\t"
"subq $16, %1 \n\t"
"jnz .L01LOOP%= \n\t"
:
:
"r" (i), // 0
"r" (n), // 1
"r" (x), // 2
"r" (y), // 3
"r" (ap[0]), // 4
"r" (ap[1]), // 5
"r" (ap[2]), // 6
"r" (ap[3]) // 7
: "cc",
"%xmm4", "%xmm5",
"%xmm6", "%xmm7",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}

451
kernel/x86_64/sgemv_n_microk_bulldozer.c Normal file
View File

@ -0,0 +1,451 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
static void sgemv_kernel_64( long n, float alpha, float *a, long lda, float *x, float *y)
{
float *pre = a + lda*3;
__asm__ __volatile__
(
"movq %0, %%rax\n\t" // n -> rax
"vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1
"movq %2, %%rsi\n\t" // adress of a -> rsi
"movq %3, %%rcx\n\t" // value of lda > rcx
"movq %4, %%rdi\n\t" // adress of x -> rdi
"movq %5, %%rdx\n\t" // adress of y -> rdx
"movq %6, %%r8\n\t" // address for prefetch
"prefetcht0 (%%r8)\n\t" // Prefetch
"prefetcht0 64(%%r8)\n\t" // Prefetch
"vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero
"vxorps %%ymm9 , %%ymm9 , %%ymm9 \n\t" // set to zero
"vxorps %%ymm10, %%ymm10, %%ymm10\n\t" // set to zero
"vxorps %%ymm11, %%ymm11, %%ymm11\n\t" // set to zero
"vxorps %%ymm12, %%ymm12, %%ymm12\n\t" // set to zero
"vxorps %%ymm13, %%ymm13, %%ymm13\n\t" // set to zero
"vxorps %%ymm14, %%ymm14, %%ymm14\n\t" // set to zero
"vxorps %%ymm15, %%ymm15, %%ymm15\n\t" // set to zero
".align 16 \n\t"
".L01LOOP%=: \n\t"
"vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c
"nop \n\t"
"leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch
"prefetcht0 (%%r8)\n\t" // Prefetch
"vfmaddps %%ymm8 , 0*4(%%rsi), %%ymm0, %%ymm8 \n\t" // multiply a and c and add to temp
"prefetcht0 64(%%r8)\n\t" // Prefetch
"vfmaddps %%ymm9 , 8*4(%%rsi), %%ymm0, %%ymm9 \n\t" // multiply a and c and add to temp
"prefetcht0 128(%%r8)\n\t" // Prefetch
"vfmaddps %%ymm10, 16*4(%%rsi), %%ymm0, %%ymm10\n\t" // multiply a and c and add to temp
"vfmaddps %%ymm11, 24*4(%%rsi), %%ymm0, %%ymm11\n\t" // multiply a and c and add to temp
"prefetcht0 192(%%r8)\n\t" // Prefetch
"vfmaddps %%ymm12, 32*4(%%rsi), %%ymm0, %%ymm12\n\t" // multiply a and c and add to temp
"vfmaddps %%ymm13, 40*4(%%rsi), %%ymm0, %%ymm13\n\t" // multiply a and c and add to temp
"vfmaddps %%ymm14, 48*4(%%rsi), %%ymm0, %%ymm14\n\t" // multiply a and c and add to temp
"vfmaddps %%ymm15, 56*4(%%rsi), %%ymm0, %%ymm15\n\t" // multiply a and c and add to temp
"addq $4 , %%rdi \n\t" // increment pointer of c
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
"dec %%rax \n\t" // n = n -1
"jnz .L01LOOP%= \n\t"
"vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha
"vmulps %%ymm9 , %%ymm1, %%ymm9 \n\t" // scale by alpha
"vmulps %%ymm10, %%ymm1, %%ymm10\n\t" // scale by alpha
"vmulps %%ymm11, %%ymm1, %%ymm11\n\t" // scale by alpha
"vmulps %%ymm12, %%ymm1, %%ymm12\n\t" // scale by alpha
"vmulps %%ymm13, %%ymm1, %%ymm13\n\t" // scale by alpha
"vmulps %%ymm14, %%ymm1, %%ymm14\n\t" // scale by alpha
"vmulps %%ymm15, %%ymm1, %%ymm15\n\t" // scale by alpha
"vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y
"vmovups %%ymm9 , 8*4(%%rdx) \n\t" // store temp -> y
"vmovups %%ymm10, 16*4(%%rdx) \n\t" // store temp -> y
"vmovups %%ymm11, 24*4(%%rdx) \n\t" // store temp -> y
"vmovups %%ymm12, 32*4(%%rdx) \n\t" // store temp -> y
"vmovups %%ymm13, 40*4(%%rdx) \n\t" // store temp -> y
"vmovups %%ymm14, 48*4(%%rdx) \n\t" // store temp -> y
"vmovups %%ymm15, 56*4(%%rdx) \n\t" // store temp -> y
:
:
"m" (n), // 0
"m" (alpha), // 1
"m" (a), // 2
"m" (lda), // 3
"m" (x), // 4
"m" (y), // 5
"m" (pre) // 6
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
"%xmm0", "%xmm1",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}
static void sgemv_kernel_32( long n, float alpha, float *a, long lda, float *x, float *y)
{
float *pre = a + lda*3;
__asm__ __volatile__
(
"movq %0, %%rax\n\t" // n -> rax
"vbroadcastss %1, %%xmm1\n\t" // alpha -> xmm1
"movq %2, %%rsi\n\t" // adress of a -> rsi
"movq %3, %%rcx\n\t" // value of lda > rcx
"movq %4, %%rdi\n\t" // adress of x -> rdi
"movq %5, %%rdx\n\t" // adress of y -> rdx
"movq %6, %%r8\n\t" // address for prefetch
"prefetcht0 (%%r8)\n\t" // Prefetch
"prefetcht0 64(%%r8)\n\t" // Prefetch
"vxorps %%xmm8 , %%xmm8 , %%xmm8 \n\t" // set to zero
"vxorps %%xmm9 , %%xmm9 , %%xmm9 \n\t" // set to zero
"vxorps %%xmm10, %%xmm10, %%xmm10\n\t" // set to zero
"vxorps %%xmm11, %%xmm11, %%xmm11\n\t" // set to zero
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
"vxorps %%xmm13, %%xmm13, %%xmm13\n\t" // set to zero
"vxorps %%xmm14, %%xmm14, %%xmm14\n\t" // set to zero
"vxorps %%xmm15, %%xmm15, %%xmm15\n\t" // set to zero
".align 16 \n\t"
".L01LOOP%=: \n\t"
"vbroadcastss (%%rdi), %%xmm0 \n\t" // load values of c
"nop \n\t"
"leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch
"prefetcht0 (%%r8)\n\t" // Prefetch
"vfmaddps %%xmm8 , 0*4(%%rsi), %%xmm0, %%xmm8 \n\t" // multiply a and c and add to temp
"prefetcht0 64(%%r8)\n\t" // Prefetch
"vfmaddps %%xmm9 , 4*4(%%rsi), %%xmm0, %%xmm9 \n\t" // multiply a and c and add to temp
"vfmaddps %%xmm10, 8*4(%%rsi), %%xmm0, %%xmm10\n\t" // multiply a and c and add to temp
"vfmaddps %%xmm11, 12*4(%%rsi), %%xmm0, %%xmm11\n\t" // multiply a and c and add to temp
"vfmaddps %%xmm12, 16*4(%%rsi), %%xmm0, %%xmm12\n\t" // multiply a and c and add to temp
"vfmaddps %%xmm13, 20*4(%%rsi), %%xmm0, %%xmm13\n\t" // multiply a and c and add to temp
"vfmaddps %%xmm14, 24*4(%%rsi), %%xmm0, %%xmm14\n\t" // multiply a and c and add to temp
"vfmaddps %%xmm15, 28*4(%%rsi), %%xmm0, %%xmm15\n\t" // multiply a and c and add to temp
"addq $4 , %%rdi \n\t" // increment pointer of c
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
"dec %%rax \n\t" // n = n -1
"jnz .L01LOOP%= \n\t"
"vmulps %%xmm8 , %%xmm1, %%xmm8 \n\t" // scale by alpha
"vmulps %%xmm9 , %%xmm1, %%xmm9 \n\t" // scale by alpha
"vmulps %%xmm10, %%xmm1, %%xmm10\n\t" // scale by alpha
"vmulps %%xmm11, %%xmm1, %%xmm11\n\t" // scale by alpha
"vmulps %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha
"vmulps %%xmm13, %%xmm1, %%xmm13\n\t" // scale by alpha
"vmulps %%xmm14, %%xmm1, %%xmm14\n\t" // scale by alpha
"vmulps %%xmm15, %%xmm1, %%xmm15\n\t" // scale by alpha
"vmovups %%xmm8 , (%%rdx) \n\t" // store temp -> y
"vmovups %%xmm9 , 4*4(%%rdx) \n\t" // store temp -> y
"vmovups %%xmm10, 8*4(%%rdx) \n\t" // store temp -> y
"vmovups %%xmm11, 12*4(%%rdx) \n\t" // store temp -> y
"vmovups %%xmm12, 16*4(%%rdx) \n\t" // store temp -> y
"vmovups %%xmm13, 20*4(%%rdx) \n\t" // store temp -> y
"vmovups %%xmm14, 24*4(%%rdx) \n\t" // store temp -> y
"vmovups %%xmm15, 28*4(%%rdx) \n\t" // store temp -> y
:
:
"m" (n), // 0
"m" (alpha), // 1
"m" (a), // 2
"m" (lda), // 3
"m" (x), // 4
"m" (y), // 5
"m" (pre) // 6
);
}
static void sgemv_kernel_16( long n, float alpha, float *a, long lda, float *x, float *y)
{
float *pre = a + lda*3;
__asm__ __volatile__
(
"movq %0, %%rax\n\t" // n -> rax
"vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1
"movq %2, %%rsi\n\t" // adress of a -> rsi
"movq %3, %%rcx\n\t" // value of lda > rcx
"movq %4, %%rdi\n\t" // adress of x -> rdi
"movq %5, %%rdx\n\t" // adress of y -> rdx
"movq %6, %%r8\n\t" // address for prefetch
"prefetcht0 (%%r8)\n\t" // Prefetch
"vxorps %%ymm12, %%ymm12, %%ymm12\n\t" // set to zero
"vxorps %%ymm13, %%ymm13, %%ymm13\n\t" // set to zero
".L01LOOP%=: \n\t"
"vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c
"addq $4 , %%rdi \n\t" // increment pointer of c
"leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch
"prefetcht0 (%%r8)\n\t" // Prefetch
"vfmaddps %%ymm12, 0*4(%%rsi), %%ymm0, %%ymm12\n\t" // multiply a and c and add to temp
"vfmaddps %%ymm13, 8*4(%%rsi), %%ymm0, %%ymm13\n\t" // multiply a and c and add to temp
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
"dec %%rax \n\t" // n = n -1
"jnz .L01LOOP%= \n\t"
"vmulps %%ymm12, %%ymm1, %%ymm12\n\t" // scale by alpha
"vmulps %%ymm13, %%ymm1, %%ymm13\n\t" // scale by alpha
"vmovups %%ymm12, (%%rdx) \n\t" // store temp -> y
"vmovups %%ymm13, 8*4(%%rdx) \n\t" // store temp -> y
:
:
"m" (n), // 0
"m" (alpha), // 1
"m" (a), // 2
"m" (lda), // 3
"m" (x), // 4
"m" (y), // 5
"m" (pre) // 6
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
"%xmm0", "%xmm1",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}
static void sgemv_kernel_8( long n, float alpha, float *a, long lda, float *x, float *y)
{
__asm__ __volatile__
(
"movq %0, %%rax\n\t" // n -> rax
"vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1
"movq %2, %%rsi\n\t" // adress of a -> rsi
"movq %3, %%rcx\n\t" // value of lda > rcx
"movq %4, %%rdi\n\t" // adress of x -> rdi
"movq %5, %%rdx\n\t" // adress of y -> rdx
"vxorps %%ymm12, %%ymm12, %%ymm12\n\t" // set to zero
".L01LOOP%=: \n\t"
"vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c
"addq $4 , %%rdi \n\t" // increment pointer of c
"vfmaddps %%ymm12, 0*4(%%rsi), %%ymm0, %%ymm12\n\t" // multiply a and c and add to temp
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
"dec %%rax \n\t" // n = n -1
"jnz .L01LOOP%= \n\t"
"vmulps %%ymm12, %%ymm1, %%ymm12\n\t" // scale by alpha
"vmovups %%ymm12, (%%rdx) \n\t" // store temp -> y
:
:
"m" (n), // 0
"m" (alpha), // 1
"m" (a), // 2
"m" (lda), // 3
"m" (x), // 4
"m" (y) // 5
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
"%xmm0", "%xmm1",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}
static void sgemv_kernel_4( long n, float alpha, float *a, long lda, float *x, float *y)
{
__asm__ __volatile__
(
"movq %0, %%rax\n\t" // n -> rax
"vbroadcastss %1, %%xmm1\n\t" // alpha -> xmm1
"movq %2, %%rsi\n\t" // adress of a -> rsi
"movq %3, %%rcx\n\t" // value of lda > rcx
"movq %4, %%rdi\n\t" // adress of x -> rdi
"movq %5, %%rdx\n\t" // adress of y -> rdx
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
".L01LOOP%=: \n\t"
"vbroadcastss (%%rdi), %%xmm0 \n\t" // load values of c
"addq $4 , %%rdi \n\t" // increment pointer of c
"vfmaddps %%xmm12, 0*4(%%rsi), %%xmm0, %%xmm12\n\t" // multiply a and c and add to temp
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
"dec %%rax \n\t" // n = n -1
"jnz .L01LOOP%= \n\t"
"vmulps %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha
"vmovups %%xmm12, (%%rdx) \n\t" // store temp -> y
:
:
"m" (n), // 0
"m" (alpha), // 1
"m" (a), // 2
"m" (lda), // 3
"m" (x), // 4
"m" (y) // 5
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
"%xmm0", "%xmm1",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}
static void sgemv_kernel_2( long n, float alpha, float *a, long lda, float *x, float *y)
{
__asm__ __volatile__
(
"movq %0, %%rax\n\t" // n -> rax
"vmovss %1, %%xmm1\n\t" // alpha -> xmm1
"movq %2, %%rsi\n\t" // adress of a -> rsi
"movq %3, %%rcx\n\t" // value of lda > rcx
"movq %4, %%rdi\n\t" // adress of x -> rdi
"movq %5, %%rdx\n\t" // adress of y -> rdx
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
"vxorps %%xmm13, %%xmm13, %%xmm13\n\t" // set to zero
".L01LOOP%=: \n\t"
"vmovss (%%rdi), %%xmm0 \n\t" // load values of c
"addq $4 , %%rdi \n\t" // increment pointer of c
"vfmaddss %%xmm12, 0*4(%%rsi), %%xmm0, %%xmm12\n\t" // multiply a and c and add to temp
"vfmaddss %%xmm13, 1*4(%%rsi), %%xmm0, %%xmm13\n\t" // multiply a and c and add to temp
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
"dec %%rax \n\t" // n = n -1
"jnz .L01LOOP%= \n\t"
"vmulss %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha
"vmulss %%xmm13, %%xmm1, %%xmm13\n\t" // scale by alpha
"vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y
"vmovss %%xmm13, 4(%%rdx) \n\t" // store temp -> y
:
:
"m" (n), // 0
"m" (alpha), // 1
"m" (a), // 2
"m" (lda), // 3
"m" (x), // 4
"m" (y) // 5
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
"%xmm0", "%xmm1",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}
static void sgemv_kernel_1( long n, float alpha, float *a, long lda, float *x, float *y)
{
__asm__ __volatile__
(
"movq %0, %%rax\n\t" // n -> rax
"vmovss %1, %%xmm1\n\t" // alpha -> xmm1
"movq %2, %%rsi\n\t" // adress of a -> rsi
"movq %3, %%rcx\n\t" // value of lda > rcx
"movq %4, %%rdi\n\t" // adress of x -> rdi
"movq %5, %%rdx\n\t" // adress of y -> rdx
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
".L01LOOP%=: \n\t"
"vmovss (%%rdi), %%xmm0 \n\t" // load values of c
"addq $4 , %%rdi \n\t" // increment pointer of c
"vfmaddss %%xmm12, 0*4(%%rsi), %%xmm0, %%xmm12\n\t" // multiply a and c and add to temp
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
"dec %%rax \n\t" // n = n -1
"jnz .L01LOOP%= \n\t"
"vmulss %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha
"vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y
:
:
"m" (n), // 0
"m" (alpha), // 1
"m" (a), // 2
"m" (lda), // 3
"m" (x), // 4
"m" (y) // 5
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
"%xmm0", "%xmm1",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}

88
kernel/x86_64/sgemv_n_microk_haswell-2.c Normal file
View File

@ -0,0 +1,88 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#define HAVE_KERNEL_16x4 1
static void sgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) __attribute__ ((noinline));
static void sgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)
{
BLASLONG register i = 0;
__asm__ __volatile__
(
"vzeroupper \n\t"
"vbroadcastss (%2), %%ymm12 \n\t" // x0
"vbroadcastss 4(%2), %%ymm13 \n\t" // x1
"vbroadcastss 8(%2), %%ymm14 \n\t" // x2
"vbroadcastss 12(%2), %%ymm15 \n\t" // x3
".align 16 \n\t"
".L01LOOP%=: \n\t"
"vmovups (%3,%0,4), %%ymm4 \n\t" // 8 * y
"vmovups 32(%3,%0,4), %%ymm5 \n\t" // 8 * y
"prefetcht0 192(%4,%0,4) \n\t"
"vfmadd231ps (%4,%0,4), %%ymm12, %%ymm4 \n\t"
"vfmadd231ps 32(%4,%0,4), %%ymm12, %%ymm5 \n\t"
"prefetcht0 192(%5,%0,4) \n\t"
"vfmadd231ps (%5,%0,4), %%ymm13, %%ymm4 \n\t"
"vfmadd231ps 32(%5,%0,4), %%ymm13, %%ymm5 \n\t"
"prefetcht0 192(%6,%0,4) \n\t"
"vfmadd231ps (%6,%0,4), %%ymm14, %%ymm4 \n\t"
"vfmadd231ps 32(%6,%0,4), %%ymm14, %%ymm5 \n\t"
"prefetcht0 192(%7,%0,4) \n\t"
"vfmadd231ps (%7,%0,4), %%ymm15, %%ymm4 \n\t"
"vfmadd231ps 32(%7,%0,4), %%ymm15, %%ymm5 \n\t"
"vmovups %%ymm4, (%3,%0,4) \n\t" // 8 * y
"vmovups %%ymm5, 32(%3,%0,4) \n\t" // 8 * y
"addq $16, %0 \n\t"
"subq $16, %1 \n\t"
"jnz .L01LOOP%= \n\t"
"vzeroupper \n\t"
:
:
"r" (i), // 0
"r" (n), // 1
"r" (x), // 2
"r" (y), // 3
"r" (ap[0]), // 4
"r" (ap[1]), // 5
"r" (ap[2]), // 6
"r" (ap[3]) // 7
: "cc",
"%xmm4", "%xmm5",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}

461
kernel/x86_64/sgemv_n_microk_haswell.c Normal file
View File

@ -0,0 +1,461 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
static void sgemv_kernel_64( long n, float alpha, float *a, long lda, float *x, float *y)
{
float *pre = a + lda*2;
__asm__ __volatile__
(
"movq %0, %%rax\n\t" // n -> rax
"vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1
"movq %2, %%rsi\n\t" // adress of a -> rsi
"movq %3, %%rcx\n\t" // value of lda > rcx
"movq %4, %%rdi\n\t" // adress of x -> rdi
"movq %5, %%rdx\n\t" // adress of y -> rdx
"movq %6, %%r8\n\t" // address for prefetch
"prefetcht0 (%%r8)\n\t" // Prefetch
"prefetcht0 64(%%r8)\n\t" // Prefetch
"vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero
"vxorps %%ymm9 , %%ymm9 , %%ymm9 \n\t" // set to zero
"vxorps %%ymm10, %%ymm10, %%ymm10\n\t" // set to zero
"vxorps %%ymm11, %%ymm11, %%ymm11\n\t" // set to zero
"vxorps %%ymm12, %%ymm12, %%ymm12\n\t" // set to zero
"vxorps %%ymm13, %%ymm13, %%ymm13\n\t" // set to zero
"vxorps %%ymm14, %%ymm14, %%ymm14\n\t" // set to zero
"vxorps %%ymm15, %%ymm15, %%ymm15\n\t" // set to zero
".align 16 \n\t"
".L01LOOP%=: \n\t"
"vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c
"leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch
"prefetcht0 (%%r8)\n\t" // Prefetch
"vfmadd231ps 0*4(%%rsi), %%ymm0, %%ymm8 \n\t" // multiply a and c and add to temp
"vfmadd231ps 8*4(%%rsi), %%ymm0, %%ymm9 \n\t" // multiply a and c and add to temp
"prefetcht0 64(%%r8)\n\t" // Prefetch
"vfmadd231ps 16*4(%%rsi), %%ymm0, %%ymm10\n\t" // multiply a and c and add to temp
"vfmadd231ps 24*4(%%rsi), %%ymm0, %%ymm11\n\t" // multiply a and c and add to temp
"prefetcht0 128(%%r8)\n\t" // Prefetch
"vfmadd231ps 32*4(%%rsi), %%ymm0, %%ymm12\n\t" // multiply a and c and add to temp
"vfmadd231ps 40*4(%%rsi), %%ymm0, %%ymm13\n\t" // multiply a and c and add to temp
"prefetcht0 192(%%r8)\n\t" // Prefetch
"vfmadd231ps 48*4(%%rsi), %%ymm0, %%ymm14\n\t" // multiply a and c and add to temp
"vfmadd231ps 56*4(%%rsi), %%ymm0, %%ymm15\n\t" // multiply a and c and add to temp
"addq $4 , %%rdi \n\t" // increment pointer of c
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
"dec %%rax \n\t" // n = n -1
"jnz .L01LOOP%= \n\t"
"vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha
"vmulps %%ymm9 , %%ymm1, %%ymm9 \n\t" // scale by alpha
"vmulps %%ymm10, %%ymm1, %%ymm10\n\t" // scale by alpha
"vmulps %%ymm11, %%ymm1, %%ymm11\n\t" // scale by alpha
"vmulps %%ymm12, %%ymm1, %%ymm12\n\t" // scale by alpha
"vmulps %%ymm13, %%ymm1, %%ymm13\n\t" // scale by alpha
"vmulps %%ymm14, %%ymm1, %%ymm14\n\t" // scale by alpha
"vmulps %%ymm15, %%ymm1, %%ymm15\n\t" // scale by alpha
"vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y
"vmovups %%ymm9 , 8*4(%%rdx) \n\t" // store temp -> y
"vmovups %%ymm10, 16*4(%%rdx) \n\t" // store temp -> y
"vmovups %%ymm11, 24*4(%%rdx) \n\t" // store temp -> y
"vmovups %%ymm12, 32*4(%%rdx) \n\t" // store temp -> y
"vmovups %%ymm13, 40*4(%%rdx) \n\t" // store temp -> y
"vmovups %%ymm14, 48*4(%%rdx) \n\t" // store temp -> y
"vmovups %%ymm15, 56*4(%%rdx) \n\t" // store temp -> y
:
:
"m" (n), // 0
"m" (alpha), // 1
"m" (a), // 2
"m" (lda), // 3
"m" (x), // 4
"m" (y), // 5
"m" (pre) // 6
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc",
"%xmm0", "%xmm1",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}
static void sgemv_kernel_32( long n, float alpha, float *a, long lda, float *x, float *y)
{
float *pre = a + lda*3;
__asm__ __volatile__
(
"movq %0, %%rax\n\t" // n -> rax
"vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1
"movq %2, %%rsi\n\t" // adress of a -> rsi
"movq %3, %%rcx\n\t" // value of lda > rcx
"movq %4, %%rdi\n\t" // adress of x -> rdi
"movq %5, %%rdx\n\t" // adress of y -> rdx
"movq %6, %%r8\n\t" // address for prefetch
"prefetcht0 (%%r8)\n\t" // Prefetch
"prefetcht0 64(%%r8)\n\t" // Prefetch
"vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero
"vxorps %%ymm9 , %%ymm9 , %%ymm9 \n\t" // set to zero
"vxorps %%ymm10, %%ymm10, %%ymm10\n\t" // set to zero
"vxorps %%ymm11, %%ymm11, %%ymm11\n\t" // set to zero
".align 16 \n\t"
".L01LOOP%=: \n\t"
"vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c
"nop \n\t"
"leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch
"prefetcht0 (%%r8)\n\t" // Prefetch
"prefetcht0 64(%%r8)\n\t" // Prefetch
"vmulps 0*4(%%rsi), %%ymm0, %%ymm4 \n\t" // multiply a and c and add to temp
"vmulps 8*4(%%rsi), %%ymm0, %%ymm5 \n\t" // multiply a and c and add to temp
"vmulps 16*4(%%rsi), %%ymm0, %%ymm6 \n\t" // multiply a and c and add to temp
"vmulps 24*4(%%rsi), %%ymm0, %%ymm7 \n\t" // multiply a and c and add to temp
"vaddps %%ymm8 , %%ymm4, %%ymm8 \n\t" // multiply a and c and add to temp
"vaddps %%ymm9 , %%ymm5, %%ymm9 \n\t" // multiply a and c and add to temp
"vaddps %%ymm10, %%ymm6, %%ymm10\n\t" // multiply a and c and add to temp
"vaddps %%ymm11, %%ymm7, %%ymm11\n\t" // multiply a and c and add to temp
"addq $4 , %%rdi \n\t" // increment pointer of c
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
"dec %%rax \n\t" // n = n -1
"jnz .L01LOOP%= \n\t"
"vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha
"vmulps %%ymm9 , %%ymm1, %%ymm9 \n\t" // scale by alpha
"vmulps %%ymm10, %%ymm1, %%ymm10\n\t" // scale by alpha
"vmulps %%ymm11, %%ymm1, %%ymm11\n\t" // scale by alpha
"vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y
"vmovups %%ymm9 , 8*4(%%rdx) \n\t" // store temp -> y
"vmovups %%ymm10, 16*4(%%rdx) \n\t" // store temp -> y
"vmovups %%ymm11, 24*4(%%rdx) \n\t" // store temp -> y
:
:
"m" (n), // 0
"m" (alpha), // 1
"m" (a), // 2
"m" (lda), // 3
"m" (x), // 4
"m" (y), // 5
"m" (pre) // 6
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc",
"%xmm0", "%xmm1",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"memory"
);
}
static void sgemv_kernel_16( long n, float alpha, float *a, long lda, float *x, float *y)
{
float *pre = a + lda*3;
__asm__ __volatile__
(
"movq %0, %%rax\n\t" // n -> rax
"vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1
"movq %2, %%rsi\n\t" // adress of a -> rsi
"movq %3, %%rcx\n\t" // value of lda > rcx
"movq %4, %%rdi\n\t" // adress of x -> rdi
"movq %5, %%rdx\n\t" // adress of y -> rdx
"movq %6, %%r8\n\t" // address for prefetch
"prefetcht0 (%%r8)\n\t" // Prefetch
"prefetcht0 64(%%r8)\n\t" // Prefetch
"vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero
"vxorps %%ymm9 , %%ymm9 , %%ymm9 \n\t" // set to zero
".align 16 \n\t"
".L01LOOP%=: \n\t"
"vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c
"nop \n\t"
"leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch
"prefetcht0 (%%r8)\n\t" // Prefetch
"vmulps 0*4(%%rsi), %%ymm0, %%ymm4 \n\t" // multiply a and c and add to temp
"vmulps 8*4(%%rsi), %%ymm0, %%ymm5 \n\t" // multiply a and c and add to temp
"vaddps %%ymm8 , %%ymm4, %%ymm8 \n\t" // multiply a and c and add to temp
"vaddps %%ymm9 , %%ymm5, %%ymm9 \n\t" // multiply a and c and add to temp
"addq $4 , %%rdi \n\t" // increment pointer of c
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
"dec %%rax \n\t" // n = n -1
"jnz .L01LOOP%= \n\t"
"vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha
"vmulps %%ymm9 , %%ymm1, %%ymm9 \n\t" // scale by alpha
"vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y
"vmovups %%ymm9 , 8*4(%%rdx) \n\t" // store temp -> y
:
:
"m" (n), // 0
"m" (alpha), // 1
"m" (a), // 2
"m" (lda), // 3
"m" (x), // 4
"m" (y), // 5
"m" (pre) // 6
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc",
"%xmm0", "%xmm1",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"memory"
);
}
static void sgemv_kernel_8( long n, float alpha, float *a, long lda, float *x, float *y)
{
__asm__ __volatile__
(
"movq %0, %%rax\n\t" // n -> rax
"vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1
"movq %2, %%rsi\n\t" // adress of a -> rsi
"movq %3, %%rcx\n\t" // value of lda > rcx
"movq %4, %%rdi\n\t" // adress of x -> rdi
"movq %5, %%rdx\n\t" // adress of y -> rdx
"vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero
".align 16 \n\t"
".L01LOOP%=: \n\t"
"vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c
"vmulps 0*4(%%rsi), %%ymm0, %%ymm4 \n\t" // multiply a and c and add to temp
"vaddps %%ymm8 , %%ymm4, %%ymm8 \n\t" // multiply a and c and add to temp
"addq $4 , %%rdi \n\t" // increment pointer of c
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
"dec %%rax \n\t" // n = n -1
"jnz .L01LOOP%= \n\t"
"vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha
"vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y
:
:
"m" (n), // 0
"m" (alpha), // 1
"m" (a), // 2
"m" (lda), // 3
"m" (x), // 4
"m" (y) // 5
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc",
"%xmm0", "%xmm1",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"memory"
);
}
static void sgemv_kernel_4( long n, float alpha, float *a, long lda, float *x, float *y)
{
__asm__ __volatile__
(
"movq %0, %%rax\n\t" // n -> rax
"vbroadcastss %1, %%xmm1\n\t" // alpha -> xmm1
"movq %2, %%rsi\n\t" // adress of a -> rsi
"movq %3, %%rcx\n\t" // value of lda > rcx
"movq %4, %%rdi\n\t" // adress of x -> rdi
"movq %5, %%rdx\n\t" // adress of y -> rdx
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
".L01LOOP%=: \n\t"
"vbroadcastss (%%rdi), %%xmm0 \n\t" // load values of c
"vmulps 0*4(%%rsi), %%xmm0, %%xmm4 \n\t" // multiply a and c and add to temp
"vaddps %%xmm12, %%xmm4, %%xmm12 \n\t" // multiply a and c and add to temp
"addq $4 , %%rdi \n\t" // increment pointer of c
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
"dec %%rax \n\t" // n = n -1
"jnz .L01LOOP%= \n\t"
"vmulps %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha
"vmovups %%xmm12, (%%rdx) \n\t" // store temp -> y
:
:
"m" (n), // 0
"m" (alpha), // 1
"m" (a), // 2
"m" (lda), // 3
"m" (x), // 4
"m" (y) // 5
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
"%xmm0", "%xmm1",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}
static void sgemv_kernel_2( long n, float alpha, float *a, long lda, float *x, float *y)
{
__asm__ __volatile__
(
"movq %0, %%rax\n\t" // n -> rax
"vmovss %1, %%xmm1\n\t" // alpha -> xmm1
"movq %2, %%rsi\n\t" // adress of a -> rsi
"movq %3, %%rcx\n\t" // value of lda > rcx
"movq %4, %%rdi\n\t" // adress of x -> rdi
"movq %5, %%rdx\n\t" // adress of y -> rdx
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
"vxorps %%xmm13, %%xmm13, %%xmm13\n\t" // set to zero
".L01LOOP%=: \n\t"
"vmovss (%%rdi), %%xmm0 \n\t" // load values of c
"vmulps 0*4(%%rsi), %%xmm0, %%xmm4 \n\t" // multiply a and c and add to temp
"vmulps 1*4(%%rsi), %%xmm0, %%xmm5 \n\t" // multiply a and c and add to temp
"vaddps %%xmm12, %%xmm4, %%xmm12 \n\t" // multiply a and c and add to temp
"vaddps %%xmm13, %%xmm5, %%xmm13 \n\t" // multiply a and c and add to temp
"addq $4 , %%rdi \n\t" // increment pointer of c
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
"dec %%rax \n\t" // n = n -1
"jnz .L01LOOP%= \n\t"
"vmulss %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha
"vmulss %%xmm13, %%xmm1, %%xmm13\n\t" // scale by alpha
"vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y
"vmovss %%xmm13, 4(%%rdx) \n\t" // store temp -> y
:
:
"m" (n), // 0
"m" (alpha), // 1
"m" (a), // 2
"m" (lda), // 3
"m" (x), // 4
"m" (y) // 5
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
"%xmm0", "%xmm1",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}
static void sgemv_kernel_1( long n, float alpha, float *a, long lda, float *x, float *y)
{
__asm__ __volatile__
(
"movq %0, %%rax\n\t" // n -> rax
"vmovss %1, %%xmm1\n\t" // alpha -> xmm1
"movq %2, %%rsi\n\t" // adress of a -> rsi
"movq %3, %%rcx\n\t" // value of lda > rcx
"movq %4, %%rdi\n\t" // adress of x -> rdi
"movq %5, %%rdx\n\t" // adress of y -> rdx
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
".L01LOOP%=: \n\t"
"vmovss (%%rdi), %%xmm0 \n\t" // load values of c
"addq $4 , %%rdi \n\t" // increment pointer of c
"vmulss 0*4(%%rsi), %%xmm0, %%xmm4 \n\t" // multiply a and c and add to temp
"vaddss %%xmm12, %%xmm4, %%xmm12 \n\t" // multiply a and c and add to temp
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
"dec %%rax \n\t" // n = n -1
"jnz .L01LOOP%= \n\t"
"vmulss %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha
"vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y
:
:
"m" (n), // 0
"m" (alpha), // 1
"m" (a), // 2
"m" (lda), // 3
"m" (x), // 4
"m" (y) // 5
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
"%xmm0", "%xmm1",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}

144
kernel/x86_64/sgemv_n_microk_nehalem-2.c Normal file
View File

@ -0,0 +1,144 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#define HAVE_KERNEL_16x4 1
static void sgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) __attribute__ ((noinline));
static void sgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)
{
BLASLONG register i = 0;
__asm__ __volatile__
(
"movss (%2), %%xmm12 \n\t" // x0
"movss 4(%2), %%xmm13 \n\t" // x1
"movss 8(%2), %%xmm14 \n\t" // x2
"movss 12(%2), %%xmm15 \n\t" // x3
"shufps $0, %%xmm12, %%xmm12\n\t"
"shufps $0, %%xmm13, %%xmm13\n\t"
"shufps $0, %%xmm14, %%xmm14\n\t"
"shufps $0, %%xmm15, %%xmm15\n\t"
".align 16 \n\t"
".L01LOOP%=: \n\t"
"movups (%3,%0,4), %%xmm4 \n\t" // 4 * y
"movups 16(%3,%0,4), %%xmm5 \n\t" // 4 * y
"movups 32(%3,%0,4), %%xmm6 \n\t" // 4 * y
"movups 48(%3,%0,4), %%xmm7 \n\t" // 4 * y
"prefetcht0 192(%4,%0,4) \n\t"
"movups (%4,%0,4), %%xmm8 \n\t"
"movups 16(%4,%0,4), %%xmm9 \n\t"
"movups 32(%4,%0,4), %%xmm10 \n\t"
"movups 48(%4,%0,4), %%xmm11 \n\t"
"mulps %%xmm12, %%xmm8 \n\t"
"addps %%xmm8 , %%xmm4 \n\t"
"mulps %%xmm12, %%xmm9 \n\t"
"addps %%xmm9 , %%xmm5 \n\t"
"mulps %%xmm12, %%xmm10 \n\t"
"addps %%xmm10, %%xmm6 \n\t"
"mulps %%xmm12, %%xmm11 \n\t"
"addps %%xmm11, %%xmm7 \n\t"
"prefetcht0 192(%5,%0,4) \n\t"
"movups (%5,%0,4), %%xmm8 \n\t"
"movups 16(%5,%0,4), %%xmm9 \n\t"
"movups 32(%5,%0,4), %%xmm10 \n\t"
"movups 48(%5,%0,4), %%xmm11 \n\t"
"mulps %%xmm13, %%xmm8 \n\t"
"addps %%xmm8 , %%xmm4 \n\t"
"mulps %%xmm13, %%xmm9 \n\t"
"addps %%xmm9 , %%xmm5 \n\t"
"mulps %%xmm13, %%xmm10 \n\t"
"addps %%xmm10, %%xmm6 \n\t"
"mulps %%xmm13, %%xmm11 \n\t"
"addps %%xmm11, %%xmm7 \n\t"
"prefetcht0 192(%6,%0,4) \n\t"
"movups (%6,%0,4), %%xmm8 \n\t"
"movups 16(%6,%0,4), %%xmm9 \n\t"
"movups 32(%6,%0,4), %%xmm10 \n\t"
"movups 48(%6,%0,4), %%xmm11 \n\t"
"mulps %%xmm14, %%xmm8 \n\t"
"addps %%xmm8 , %%xmm4 \n\t"
"mulps %%xmm14, %%xmm9 \n\t"
"addps %%xmm9 , %%xmm5 \n\t"
"mulps %%xmm14, %%xmm10 \n\t"
"addps %%xmm10, %%xmm6 \n\t"
"mulps %%xmm14, %%xmm11 \n\t"
"addps %%xmm11, %%xmm7 \n\t"
"prefetcht0 192(%7,%0,4) \n\t"
"movups (%7,%0,4), %%xmm8 \n\t"
"movups 16(%7,%0,4), %%xmm9 \n\t"
"movups 32(%7,%0,4), %%xmm10 \n\t"
"movups 48(%7,%0,4), %%xmm11 \n\t"
"mulps %%xmm15, %%xmm8 \n\t"
"addps %%xmm8 , %%xmm4 \n\t"
"mulps %%xmm15, %%xmm9 \n\t"
"addps %%xmm9 , %%xmm5 \n\t"
"mulps %%xmm15, %%xmm10 \n\t"
"addps %%xmm10, %%xmm6 \n\t"
"mulps %%xmm15, %%xmm11 \n\t"
"addps %%xmm11, %%xmm7 \n\t"
"movups %%xmm4, (%3,%0,4) \n\t" // 4 * y
"movups %%xmm5, 16(%3,%0,4) \n\t" // 4 * y
"movups %%xmm6, 32(%3,%0,4) \n\t" // 4 * y
"movups %%xmm7, 48(%3,%0,4) \n\t" // 4 * y
"addq $16, %0 \n\t"
"subq $16, %1 \n\t"
"jnz .L01LOOP%= \n\t"
:
:
"r" (i), // 0
"r" (n), // 1
"r" (x), // 2
"r" (y), // 3
"r" (ap[0]), // 4
"r" (ap[1]), // 5
"r" (ap[2]), // 6
"r" (ap[3]) // 7
: "cc",
"%xmm4", "%xmm5",
"%xmm6", "%xmm7",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}

97
kernel/x86_64/sgemv_n_microk_sandy-2.c Normal file
View File

@ -0,0 +1,97 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#define HAVE_KERNEL_16x4 1
static void sgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) __attribute__ ((noinline));
static void sgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)
{
BLASLONG register i = 0;
__asm__ __volatile__
(
"vzeroupper \n\t"
"vbroadcastss (%2), %%ymm12 \n\t" // x0
"vbroadcastss 4(%2), %%ymm13 \n\t" // x1
"vbroadcastss 8(%2), %%ymm14 \n\t" // x2
"vbroadcastss 12(%2), %%ymm15 \n\t" // x3
".align 16 \n\t"
".L01LOOP%=: \n\t"
"vmovups (%3,%0,4), %%ymm4 \n\t" // 8 * y
"vmovups 32(%3,%0,4), %%ymm5 \n\t" // 8 * y
"prefetcht0 192(%4,%0,4) \n\t"
"vmulps (%4,%0,4), %%ymm12, %%ymm8 \n\t"
"vaddps %%ymm4, %%ymm8 , %%ymm4 \n\t"
"vmulps 32(%4,%0,4), %%ymm12, %%ymm9 \n\t"
"vaddps %%ymm5, %%ymm9 , %%ymm5 \n\t"
"prefetcht0 192(%5,%0,4) \n\t"
"vmulps (%5,%0,4), %%ymm13, %%ymm10 \n\t"
"vaddps %%ymm4, %%ymm10, %%ymm4 \n\t"
"vmulps 32(%5,%0,4), %%ymm13, %%ymm11 \n\t"
"vaddps %%ymm5, %%ymm11, %%ymm5 \n\t"
"prefetcht0 192(%6,%0,4) \n\t"
"vmulps (%6,%0,4), %%ymm14, %%ymm8 \n\t"
"vaddps %%ymm4, %%ymm8 , %%ymm4 \n\t"
"vmulps 32(%6,%0,4), %%ymm14, %%ymm9 \n\t"
"vaddps %%ymm5, %%ymm9 , %%ymm5 \n\t"
"prefetcht0 192(%7,%0,4) \n\t"
"vmulps (%7,%0,4), %%ymm15, %%ymm10 \n\t"
"vaddps %%ymm4, %%ymm10, %%ymm4 \n\t"
"vmulps 32(%7,%0,4), %%ymm15, %%ymm11 \n\t"
"vaddps %%ymm5, %%ymm11, %%ymm5 \n\t"
"vmovups %%ymm4, (%3,%0,4) \n\t" // 8 * y
"vmovups %%ymm5, 32(%3,%0,4) \n\t" // 8 * y
"addq $16, %0 \n\t"
"subq $16, %1 \n\t"
"jnz .L01LOOP%= \n\t"
"vzeroupper \n\t"
:
:
"r" (i), // 0
"r" (n), // 1
"r" (x), // 2
"r" (y), // 3
"r" (ap[0]), // 4
"r" (ap[1]), // 5
"r" (ap[2]), // 6
"r" (ap[3]) // 7
: "cc",
"%xmm4", "%xmm5",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}

473
kernel/x86_64/sgemv_n_microk_sandy.c Normal file
View File

@ -0,0 +1,473 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
static void sgemv_kernel_64( long n, float alpha, float *a, long lda, float *x, float *y)
{
float *pre = a + lda*2;
__asm__ __volatile__
(
"movq %0, %%rax\n\t" // n -> rax
"vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1
"movq %2, %%rsi\n\t" // adress of a -> rsi
"movq %3, %%rcx\n\t" // value of lda > rcx
"movq %4, %%rdi\n\t" // adress of x -> rdi
"movq %5, %%rdx\n\t" // adress of y -> rdx
"movq %6, %%r8\n\t" // address for prefetch
"prefetcht0 (%%r8)\n\t" // Prefetch
"prefetcht0 64(%%r8)\n\t" // Prefetch
"vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero
"vxorps %%ymm9 , %%ymm9 , %%ymm9 \n\t" // set to zero
"vxorps %%ymm10, %%ymm10, %%ymm10\n\t" // set to zero
"vxorps %%ymm11, %%ymm11, %%ymm11\n\t" // set to zero
"vxorps %%ymm12, %%ymm12, %%ymm12\n\t" // set to zero
"vxorps %%ymm13, %%ymm13, %%ymm13\n\t" // set to zero
"vxorps %%ymm14, %%ymm14, %%ymm14\n\t" // set to zero
"vxorps %%ymm15, %%ymm15, %%ymm15\n\t" // set to zero
".align 16 \n\t"
".L01LOOP%=: \n\t"
"vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c
"nop \n\t"
"leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch
"prefetcht0 (%%r8)\n\t" // Prefetch
"vmulps 0*4(%%rsi), %%ymm0, %%ymm4 \n\t" // multiply a and c and add to temp
"vmulps 8*4(%%rsi), %%ymm0, %%ymm5 \n\t" // multiply a and c and add to temp
"prefetcht0 64(%%r8)\n\t" // Prefetch
"vmulps 16*4(%%rsi), %%ymm0, %%ymm6 \n\t" // multiply a and c and add to temp
"vmulps 24*4(%%rsi), %%ymm0, %%ymm7 \n\t" // multiply a and c and add to temp
"vaddps %%ymm8 , %%ymm4, %%ymm8 \n\t" // multiply a and c and add to temp
"vaddps %%ymm9 , %%ymm5, %%ymm9 \n\t" // multiply a and c and add to temp
"prefetcht0 128(%%r8)\n\t" // Prefetch
"vaddps %%ymm10, %%ymm6, %%ymm10\n\t" // multiply a and c and add to temp
"vaddps %%ymm11, %%ymm7, %%ymm11\n\t" // multiply a and c and add to temp
"prefetcht0 192(%%r8)\n\t" // Prefetch
"vmulps 32*4(%%rsi), %%ymm0, %%ymm4 \n\t" // multiply a and c and add to temp
"vmulps 40*4(%%rsi), %%ymm0, %%ymm5 \n\t" // multiply a and c and add to temp
"vmulps 48*4(%%rsi), %%ymm0, %%ymm6 \n\t" // multiply a and c and add to temp
"vmulps 56*4(%%rsi), %%ymm0, %%ymm7 \n\t" // multiply a and c and add to temp
"vaddps %%ymm12, %%ymm4, %%ymm12\n\t" // multiply a and c and add to temp
"vaddps %%ymm13, %%ymm5, %%ymm13\n\t" // multiply a and c and add to temp
"vaddps %%ymm14, %%ymm6, %%ymm14\n\t" // multiply a and c and add to temp
"vaddps %%ymm15, %%ymm7, %%ymm15\n\t" // multiply a and c and add to temp
"addq $4 , %%rdi \n\t" // increment pointer of c
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
"dec %%rax \n\t" // n = n -1
"jnz .L01LOOP%= \n\t"
"vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha
"vmulps %%ymm9 , %%ymm1, %%ymm9 \n\t" // scale by alpha
"vmulps %%ymm10, %%ymm1, %%ymm10\n\t" // scale by alpha
"vmulps %%ymm11, %%ymm1, %%ymm11\n\t" // scale by alpha
"vmulps %%ymm12, %%ymm1, %%ymm12\n\t" // scale by alpha
"vmulps %%ymm13, %%ymm1, %%ymm13\n\t" // scale by alpha
"vmulps %%ymm14, %%ymm1, %%ymm14\n\t" // scale by alpha
"vmulps %%ymm15, %%ymm1, %%ymm15\n\t" // scale by alpha
"vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y
"vmovups %%ymm9 , 8*4(%%rdx) \n\t" // store temp -> y
"vmovups %%ymm10, 16*4(%%rdx) \n\t" // store temp -> y
"vmovups %%ymm11, 24*4(%%rdx) \n\t" // store temp -> y
"vmovups %%ymm12, 32*4(%%rdx) \n\t" // store temp -> y
"vmovups %%ymm13, 40*4(%%rdx) \n\t" // store temp -> y
"vmovups %%ymm14, 48*4(%%rdx) \n\t" // store temp -> y
"vmovups %%ymm15, 56*4(%%rdx) \n\t" // store temp -> y
:
:
"m" (n), // 0
"m" (alpha), // 1
"m" (a), // 2
"m" (lda), // 3
"m" (x), // 4
"m" (y), // 5
"m" (pre) // 6
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc",
"%xmm0", "%xmm1",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}
static void sgemv_kernel_32( long n, float alpha, float *a, long lda, float *x, float *y)
{
float *pre = a + lda*3;
__asm__ __volatile__
(
"movq %0, %%rax\n\t" // n -> rax
"vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1
"movq %2, %%rsi\n\t" // adress of a -> rsi
"movq %3, %%rcx\n\t" // value of lda > rcx
"movq %4, %%rdi\n\t" // adress of x -> rdi
"movq %5, %%rdx\n\t" // adress of y -> rdx
"movq %6, %%r8\n\t" // address for prefetch
"prefetcht0 (%%r8)\n\t" // Prefetch
"prefetcht0 64(%%r8)\n\t" // Prefetch
"vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero
"vxorps %%ymm9 , %%ymm9 , %%ymm9 \n\t" // set to zero
"vxorps %%ymm10, %%ymm10, %%ymm10\n\t" // set to zero
"vxorps %%ymm11, %%ymm11, %%ymm11\n\t" // set to zero
".align 16 \n\t"
".L01LOOP%=: \n\t"
"vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c
"nop \n\t"
"leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch
"prefetcht0 (%%r8)\n\t" // Prefetch
"prefetcht0 64(%%r8)\n\t" // Prefetch
"vmulps 0*4(%%rsi), %%ymm0, %%ymm4 \n\t" // multiply a and c and add to temp
"vmulps 8*4(%%rsi), %%ymm0, %%ymm5 \n\t" // multiply a and c and add to temp
"vmulps 16*4(%%rsi), %%ymm0, %%ymm6 \n\t" // multiply a and c and add to temp
"vmulps 24*4(%%rsi), %%ymm0, %%ymm7 \n\t" // multiply a and c and add to temp
"vaddps %%ymm8 , %%ymm4, %%ymm8 \n\t" // multiply a and c and add to temp
"vaddps %%ymm9 , %%ymm5, %%ymm9 \n\t" // multiply a and c and add to temp
"vaddps %%ymm10, %%ymm6, %%ymm10\n\t" // multiply a and c and add to temp
"vaddps %%ymm11, %%ymm7, %%ymm11\n\t" // multiply a and c and add to temp
"addq $4 , %%rdi \n\t" // increment pointer of c
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
"dec %%rax \n\t" // n = n -1
"jnz .L01LOOP%= \n\t"
"vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha
"vmulps %%ymm9 , %%ymm1, %%ymm9 \n\t" // scale by alpha
"vmulps %%ymm10, %%ymm1, %%ymm10\n\t" // scale by alpha
"vmulps %%ymm11, %%ymm1, %%ymm11\n\t" // scale by alpha
"vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y
"vmovups %%ymm9 , 8*4(%%rdx) \n\t" // store temp -> y
"vmovups %%ymm10, 16*4(%%rdx) \n\t" // store temp -> y
"vmovups %%ymm11, 24*4(%%rdx) \n\t" // store temp -> y
:
:
"m" (n), // 0
"m" (alpha), // 1
"m" (a), // 2
"m" (lda), // 3
"m" (x), // 4
"m" (y), // 5
"m" (pre) // 6
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc",
"%xmm0", "%xmm1",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"memory"
);
}
static void sgemv_kernel_16( long n, float alpha, float *a, long lda, float *x, float *y)
{
float *pre = a + lda*3;
__asm__ __volatile__
(
"movq %0, %%rax\n\t" // n -> rax
"vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1
"movq %2, %%rsi\n\t" // adress of a -> rsi
"movq %3, %%rcx\n\t" // value of lda > rcx
"movq %4, %%rdi\n\t" // adress of x -> rdi
"movq %5, %%rdx\n\t" // adress of y -> rdx
"movq %6, %%r8\n\t" // address for prefetch
"prefetcht0 (%%r8)\n\t" // Prefetch
"prefetcht0 64(%%r8)\n\t" // Prefetch
"vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero
"vxorps %%ymm9 , %%ymm9 , %%ymm9 \n\t" // set to zero
".align 16 \n\t"
".L01LOOP%=: \n\t"
"vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c
"nop \n\t"
"leaq (%%r8 , %%rcx, 4), %%r8 \n\t" // add lda to pointer for prefetch
"prefetcht0 (%%r8)\n\t" // Prefetch
"vmulps 0*4(%%rsi), %%ymm0, %%ymm4 \n\t" // multiply a and c and add to temp
"vmulps 8*4(%%rsi), %%ymm0, %%ymm5 \n\t" // multiply a and c and add to temp
"vaddps %%ymm8 , %%ymm4, %%ymm8 \n\t" // multiply a and c and add to temp
"vaddps %%ymm9 , %%ymm5, %%ymm9 \n\t" // multiply a and c and add to temp
"addq $4 , %%rdi \n\t" // increment pointer of c
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
"dec %%rax \n\t" // n = n -1
"jnz .L01LOOP%= \n\t"
"vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha
"vmulps %%ymm9 , %%ymm1, %%ymm9 \n\t" // scale by alpha
"vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y
"vmovups %%ymm9 , 8*4(%%rdx) \n\t" // store temp -> y
:
:
"m" (n), // 0
"m" (alpha), // 1
"m" (a), // 2
"m" (lda), // 3
"m" (x), // 4
"m" (y), // 5
"m" (pre) // 6
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc",
"%xmm0", "%xmm1",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"memory"
);
}
static void sgemv_kernel_8( long n, float alpha, float *a, long lda, float *x, float *y)
{
__asm__ __volatile__
(
"movq %0, %%rax\n\t" // n -> rax
"vbroadcastss %1, %%ymm1\n\t" // alpha -> ymm1
"movq %2, %%rsi\n\t" // adress of a -> rsi
"movq %3, %%rcx\n\t" // value of lda > rcx
"movq %4, %%rdi\n\t" // adress of x -> rdi
"movq %5, %%rdx\n\t" // adress of y -> rdx
"vxorps %%ymm8 , %%ymm8 , %%ymm8 \n\t" // set to zero
".align 16 \n\t"
".L01LOOP%=: \n\t"
"vbroadcastss (%%rdi), %%ymm0 \n\t" // load values of c
"vmulps 0*4(%%rsi), %%ymm0, %%ymm4 \n\t" // multiply a and c and add to temp
"vaddps %%ymm8 , %%ymm4, %%ymm8 \n\t" // multiply a and c and add to temp
"addq $4 , %%rdi \n\t" // increment pointer of c
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
"dec %%rax \n\t" // n = n -1
"jnz .L01LOOP%= \n\t"
"vmulps %%ymm8 , %%ymm1, %%ymm8 \n\t" // scale by alpha
"vmovups %%ymm8 , (%%rdx) \n\t" // store temp -> y
:
:
"m" (n), // 0
"m" (alpha), // 1
"m" (a), // 2
"m" (lda), // 3
"m" (x), // 4
"m" (y) // 5
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc",
"%xmm0", "%xmm1",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"memory"
);
}
static void sgemv_kernel_4( long n, float alpha, float *a, long lda, float *x, float *y)
{
__asm__ __volatile__
(
"movq %0, %%rax\n\t" // n -> rax
"vbroadcastss %1, %%xmm1\n\t" // alpha -> xmm1
"movq %2, %%rsi\n\t" // adress of a -> rsi
"movq %3, %%rcx\n\t" // value of lda > rcx
"movq %4, %%rdi\n\t" // adress of x -> rdi
"movq %5, %%rdx\n\t" // adress of y -> rdx
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
".L01LOOP%=: \n\t"
"vbroadcastss (%%rdi), %%xmm0 \n\t" // load values of c
"vmulps 0*4(%%rsi), %%xmm0, %%xmm4 \n\t" // multiply a and c and add to temp
"vaddps %%xmm12, %%xmm4, %%xmm12 \n\t" // multiply a and c and add to temp
"addq $4 , %%rdi \n\t" // increment pointer of c
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
"dec %%rax \n\t" // n = n -1
"jnz .L01LOOP%= \n\t"
"vmulps %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha
"vmovups %%xmm12, (%%rdx) \n\t" // store temp -> y
:
:
"m" (n), // 0
"m" (alpha), // 1
"m" (a), // 2
"m" (lda), // 3
"m" (x), // 4
"m" (y) // 5
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
"%xmm0", "%xmm1",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}
static void sgemv_kernel_2( long n, float alpha, float *a, long lda, float *x, float *y)
{
__asm__ __volatile__
(
"movq %0, %%rax\n\t" // n -> rax
"vmovss %1, %%xmm1\n\t" // alpha -> xmm1
"movq %2, %%rsi\n\t" // adress of a -> rsi
"movq %3, %%rcx\n\t" // value of lda > rcx
"movq %4, %%rdi\n\t" // adress of x -> rdi
"movq %5, %%rdx\n\t" // adress of y -> rdx
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
"vxorps %%xmm13, %%xmm13, %%xmm13\n\t" // set to zero
".L01LOOP%=: \n\t"
"vmovss (%%rdi), %%xmm0 \n\t" // load values of c
"vmulps 0*4(%%rsi), %%xmm0, %%xmm4 \n\t" // multiply a and c and add to temp
"vmulps 1*4(%%rsi), %%xmm0, %%xmm5 \n\t" // multiply a and c and add to temp
"vaddps %%xmm12, %%xmm4, %%xmm12 \n\t" // multiply a and c and add to temp
"vaddps %%xmm13, %%xmm5, %%xmm13 \n\t" // multiply a and c and add to temp
"addq $4 , %%rdi \n\t" // increment pointer of c
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
"dec %%rax \n\t" // n = n -1
"jnz .L01LOOP%= \n\t"
"vmulss %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha
"vmulss %%xmm13, %%xmm1, %%xmm13\n\t" // scale by alpha
"vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y
"vmovss %%xmm13, 4(%%rdx) \n\t" // store temp -> y
:
:
"m" (n), // 0
"m" (alpha), // 1
"m" (a), // 2
"m" (lda), // 3
"m" (x), // 4
"m" (y) // 5
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
"%xmm0", "%xmm1",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}
static void sgemv_kernel_1( long n, float alpha, float *a, long lda, float *x, float *y)
{
__asm__ __volatile__
(
"movq %0, %%rax\n\t" // n -> rax
"vmovss %1, %%xmm1\n\t" // alpha -> xmm1
"movq %2, %%rsi\n\t" // adress of a -> rsi
"movq %3, %%rcx\n\t" // value of lda > rcx
"movq %4, %%rdi\n\t" // adress of x -> rdi
"movq %5, %%rdx\n\t" // adress of y -> rdx
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
".L01LOOP%=: \n\t"
"vmovss (%%rdi), %%xmm0 \n\t" // load values of c
"addq $4 , %%rdi \n\t" // increment pointer of c
"vmulss 0*4(%%rsi), %%xmm0, %%xmm4 \n\t" // multiply a and c and add to temp
"vaddss %%xmm12, %%xmm4, %%xmm12 \n\t" // multiply a and c and add to temp
"leaq (%%rsi, %%rcx, 4), %%rsi \n\t" // add lda to pointer of a
"dec %%rax \n\t" // n = n -1
"jnz .L01LOOP%= \n\t"
"vmulss %%xmm12, %%xmm1, %%xmm12\n\t" // scale by alpha
"vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y
:
:
"m" (n), // 0
"m" (alpha), // 1
"m" (a), // 2
"m" (lda), // 3
"m" (x), // 4
"m" (y) // 5
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
"%xmm0", "%xmm1",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}

197
kernel/x86_64/sgemv_t.c Normal file
View File

@ -0,0 +1,197 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#include "common.h"
#if defined(BULLDOZER) || defined(PILEDRIVER)
#include "sgemv_t_microk_bulldozer-2.c"
#elif defined(HASWELL)
#include "sgemv_t_microk_haswell-2.c"
#elif defined(SANDYBRIDGE)
#include "sgemv_t_microk_sandy-2.c"
#elif defined(NEHALEM)
#include "sgemv_t_microk_nehalem-2.c"
#endif
#define NBMAX 4096
#ifndef HAVE_KERNEL_16x4
static void sgemv_kernel_16x4(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)
{
BLASLONG i;
FLOAT *a0,*a1,*a2,*a3;
a0 = ap[0];
a1 = ap[1];
a2 = ap[2];
a3 = ap[3];
FLOAT temp0 = 0.0;
FLOAT temp1 = 0.0;
FLOAT temp2 = 0.0;
FLOAT temp3 = 0.0;
for ( i=0; i< n; i+=4 )
{
temp0 += a0[i]*x[i] + a0[i+1]*x[i+1] + a0[i+2]*x[i+2] + a0[i+3]*x[i+3];
temp1 += a1[i]*x[i] + a1[i+1]*x[i+1] + a1[i+2]*x[i+2] + a1[i+3]*x[i+3];
temp2 += a2[i]*x[i] + a2[i+1]*x[i+1] + a2[i+2]*x[i+2] + a2[i+3]*x[i+3];
temp3 += a3[i]*x[i] + a3[i+1]*x[i+1] + a3[i+2]*x[i+2] + a3[i+3]*x[i+3];
}
y[0] = temp0;
y[1] = temp1;
y[2] = temp2;
y[3] = temp3;
}
#endif
static void sgemv_kernel_16x1(BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y)
{
BLASLONG i;
FLOAT *a0;
a0 = ap;
FLOAT temp = 0.0;
for ( i=0; i< n; i+=4 )
{
temp += a0[i]*x[i] + a0[i+1]*x[i+1] + a0[i+2]*x[i+2] + a0[i+3]*x[i+3];
}
*y = temp;
}
static void copy_x(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_src)
{
BLASLONG i;
for ( i=0; i<n; i++ )
{
*dest = *src;
dest++;
src += inc_src;
}
}
int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer)
{
BLASLONG i;
BLASLONG j;
FLOAT *a_ptr;
FLOAT *x_ptr;
FLOAT *y_ptr;
FLOAT *ap[4];
BLASLONG n1;
BLASLONG m1;
BLASLONG m2;
BLASLONG n2;
FLOAT ybuffer[4],*xbuffer;
if ( m < 1 ) return(0);
if ( n < 1 ) return(0);
xbuffer = buffer;
n1 = n / 4 ;
n2 = n % 4 ;
m1 = m - ( m % 16 );
m2 = (m % NBMAX) - (m % 16) ;
BLASLONG NB = NBMAX;
while ( NB == NBMAX )
{
m1 -= NB;
if ( m1 < 0)
{
if ( m2 == 0 ) break;
NB = m2;
}
y_ptr = y;
a_ptr = a;
x_ptr = x;
copy_x(NB,x_ptr,xbuffer,inc_x);
for( i = 0; i < n1 ; i++)
{
ap[0] = a_ptr;
ap[1] = a_ptr + lda;
ap[2] = ap[1] + lda;
ap[3] = ap[2] + lda;
sgemv_kernel_16x4(NB,ap,xbuffer,ybuffer);
a_ptr += 4 * lda;
*y_ptr += ybuffer[0]*alpha;
y_ptr += inc_y;
*y_ptr += ybuffer[1]*alpha;
y_ptr += inc_y;
*y_ptr += ybuffer[2]*alpha;
y_ptr += inc_y;
*y_ptr += ybuffer[3]*alpha;
y_ptr += inc_y;
}
for( i = 0; i < n2 ; i++)
{
sgemv_kernel_16x1(NB,a_ptr,xbuffer,ybuffer);
a_ptr += 1 * lda;
*y_ptr += ybuffer[0]*alpha;
y_ptr += inc_y;
}
a += NB;
x += NB * inc_x;
}
BLASLONG m3 = m % 16;
if ( m3 == 0 ) return(0);
x_ptr = x;
for ( i=0; i< m3; i++ )
{
xbuffer[i] = *x_ptr;
x_ptr += inc_x;
}
j=0;
a_ptr = a;
y_ptr = y;
while ( j < n)
{
FLOAT temp = 0.0;
for( i = 0; i < m3; i++ )
{
temp += a_ptr[i] * xbuffer[i];
}
a_ptr += lda;
y_ptr[0] += alpha * temp;
y_ptr += inc_y;
j++;
}
return(0);
}

232
kernel/x86_64/sgemv_t_avx.c Normal file
View File

@ -0,0 +1,232 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#include "common.h"
#if defined(BULLDOZER) || defined(PILEDRIVER)
#include "sgemv_t_microk_bulldozer.c"
#elif defined(HASWELL)
#include "sgemv_t_microk_haswell.c"
#else
#include "sgemv_t_microk_sandy.c"
#endif
static void copy_x(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_src)
{
BLASLONG i;
for ( i=0; i<n; i++ )
{
*dest = *src;
dest++;
src += inc_src;
}
}
static void sgemv_kernel_1( BLASLONG n, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *x, FLOAT *y)
{
FLOAT register temp0 = 0.0;
BLASLONG i;
for ( i=0; i<n ; i++)
{
temp0 += a[i] * x[i];
}
temp0 *= alpha ;
*y += temp0;
}
int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer)
{
BLASLONG i;
BLASLONG j;
FLOAT *a_ptr;
FLOAT *x_ptr;
FLOAT *y_ptr;
FLOAT *a_ptrl;
BLASLONG m1;
BLASLONG register m2;
FLOAT *xbuffer;
xbuffer = buffer;
BLASLONG register Mblock;
m1 = m / 1024 ;
m2 = m % 1024 ;
x_ptr = x;
a_ptr = a;
for (j=0; j<m1; j++)
{
if ( inc_x == 1 )
xbuffer = x_ptr;
else
copy_x(1024,x_ptr,xbuffer,inc_x);
y_ptr = y;
a_ptrl = a_ptr;
for(i = 0; i<n; i++ )
{
sgemv_kernel_16(1024,alpha,a_ptrl,lda,xbuffer,y_ptr);
y_ptr += inc_y;
a_ptrl += lda;
}
a_ptr += 1024;
x_ptr += 1024 * inc_x;
}
if ( m2 == 0 ) return(0);
Mblock = 512;
while ( Mblock >= 16 )
{
if ( m2 & Mblock)
{
if ( inc_x == 1 )
xbuffer = x_ptr;
else
copy_x(Mblock,x_ptr,xbuffer,inc_x);
y_ptr = y;
a_ptrl = a_ptr;
for(i = 0; i<n; i++ )
{
sgemv_kernel_16(Mblock,alpha,a_ptrl,lda,xbuffer,y_ptr);
y_ptr += inc_y;
a_ptrl += lda;
}
a_ptr += Mblock;
x_ptr += Mblock * inc_x;
}
Mblock /= 2;
}
if ( m2 & Mblock)
{
if ( inc_x == 1 )
xbuffer = x_ptr;
else
copy_x(Mblock,x_ptr,xbuffer,inc_x);
y_ptr = y;
a_ptrl = a_ptr;
for(i = 0; i<n; i++ )
{
sgemv_kernel_1(Mblock,alpha,a_ptrl,lda,xbuffer,y_ptr);
y_ptr += inc_y;
a_ptrl += lda;
}
a_ptr += Mblock;
x_ptr += Mblock * inc_x;
}
Mblock /= 2;
if ( m2 & Mblock)
{
if ( inc_x == 1 )
xbuffer = x_ptr;
else
copy_x(Mblock,x_ptr,xbuffer,inc_x);
y_ptr = y;
a_ptrl = a_ptr;
for(i = 0; i<n; i++ )
{
sgemv_kernel_1(Mblock,alpha,a_ptrl,lda,xbuffer,y_ptr);
y_ptr += inc_y;
a_ptrl += lda;
}
a_ptr += Mblock;
x_ptr += Mblock * inc_x;
}
Mblock /= 2;
if ( m2 & Mblock)
{
if ( inc_x == 1 )
xbuffer = x_ptr;
else
copy_x(Mblock,x_ptr,xbuffer,inc_x);
y_ptr = y;
a_ptrl = a_ptr;
for(i = 0; i<n; i++ )
{
sgemv_kernel_1(Mblock,alpha,a_ptrl,lda,xbuffer,y_ptr);
y_ptr += inc_y;
a_ptrl += lda;
}
a_ptr += Mblock;
x_ptr += Mblock * inc_x;
}
Mblock /= 2;
if ( m2 & Mblock)
{
xbuffer = x_ptr;
y_ptr = y;
a_ptrl = a_ptr;
for(i = 0; i<n; i++ )
{
sgemv_kernel_1(Mblock,alpha,a_ptrl,lda,xbuffer,y_ptr);
y_ptr += inc_y;
a_ptrl += lda;
}
}
return(0);
}

109
kernel/x86_64/sgemv_t_microk_bulldozer-2.c Normal file
View File

@ -0,0 +1,109 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#define HAVE_KERNEL_16x4 1
static void sgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) __attribute__ ((noinline));
static void sgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)
{
BLASLONG register i = 0;
__asm__ __volatile__
(
"vxorps %%xmm4, %%xmm4, %%xmm4 \n\t"
"vxorps %%xmm5, %%xmm5, %%xmm5 \n\t"
"vxorps %%xmm6, %%xmm6, %%xmm6 \n\t"
"vxorps %%xmm7, %%xmm7, %%xmm7 \n\t"
".align 16 \n\t"
".L01LOOP%=: \n\t"
"vmovups (%2,%0,4), %%xmm12 \n\t" // 4 * x
"prefetcht0 384(%4,%0,4) \n\t"
"vfmaddps %%xmm4, (%4,%0,4), %%xmm12, %%xmm4 \n\t"
"vfmaddps %%xmm5, (%5,%0,4), %%xmm12, %%xmm5 \n\t"
"vmovups 16(%2,%0,4), %%xmm13 \n\t" // 4 * x
"vfmaddps %%xmm6, (%6,%0,4), %%xmm12, %%xmm6 \n\t"
"vfmaddps %%xmm7, (%7,%0,4), %%xmm12, %%xmm7 \n\t"
"prefetcht0 384(%5,%0,4) \n\t"
"vfmaddps %%xmm4, 16(%4,%0,4), %%xmm13, %%xmm4 \n\t"
"vfmaddps %%xmm5, 16(%5,%0,4), %%xmm13, %%xmm5 \n\t"
"vmovups 32(%2,%0,4), %%xmm14 \n\t" // 4 * x
"vfmaddps %%xmm6, 16(%6,%0,4), %%xmm13, %%xmm6 \n\t"
"vfmaddps %%xmm7, 16(%7,%0,4), %%xmm13, %%xmm7 \n\t"
"prefetcht0 384(%6,%0,4) \n\t"
"vfmaddps %%xmm4, 32(%4,%0,4), %%xmm14, %%xmm4 \n\t"
"vfmaddps %%xmm5, 32(%5,%0,4), %%xmm14, %%xmm5 \n\t"
"vmovups 48(%2,%0,4), %%xmm15 \n\t" // 4 * x
"vfmaddps %%xmm6, 32(%6,%0,4), %%xmm14, %%xmm6 \n\t"
"vfmaddps %%xmm7, 32(%7,%0,4), %%xmm14, %%xmm7 \n\t"
"prefetcht0 384(%7,%0,4) \n\t"
"vfmaddps %%xmm4, 48(%4,%0,4), %%xmm15, %%xmm4 \n\t"
"vfmaddps %%xmm5, 48(%5,%0,4), %%xmm15, %%xmm5 \n\t"
"vfmaddps %%xmm6, 48(%6,%0,4), %%xmm15, %%xmm6 \n\t"
"vfmaddps %%xmm7, 48(%7,%0,4), %%xmm15, %%xmm7 \n\t"
"addq $16, %0 \n\t"
"subq $16, %1 \n\t"
"jnz .L01LOOP%= \n\t"
"vhaddps %%xmm4, %%xmm4, %%xmm4 \n\t"
"vhaddps %%xmm5, %%xmm5, %%xmm5 \n\t"
"vhaddps %%xmm6, %%xmm6, %%xmm6 \n\t"
"vhaddps %%xmm7, %%xmm7, %%xmm7 \n\t"
"vhaddps %%xmm4, %%xmm4, %%xmm4 \n\t"
"vhaddps %%xmm5, %%xmm5, %%xmm5 \n\t"
"vhaddps %%xmm6, %%xmm6, %%xmm6 \n\t"
"vhaddps %%xmm7, %%xmm7, %%xmm7 \n\t"
"vmovss %%xmm4, (%3) \n\t"
"vmovss %%xmm5, 4(%3) \n\t"
"vmovss %%xmm6, 8(%3) \n\t"
"vmovss %%xmm7, 12(%3) \n\t"
:
:
"r" (i), // 0
"r" (n), // 1
"r" (x), // 2
"r" (y), // 3
"r" (ap[0]), // 4
"r" (ap[1]), // 5
"r" (ap[2]), // 6
"r" (ap[3]) // 7
: "cc",
"%xmm4", "%xmm5",
"%xmm6", "%xmm7",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}

99
kernel/x86_64/sgemv_t_microk_bulldozer.c Normal file
View File

@ -0,0 +1,99 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
static void sgemv_kernel_16( long n, float alpha, float *a, long lda, float *x, float *y)
{
//n = n / 16;
__asm__ __volatile__
(
"movq %0, %%rax\n\t" // n -> rax
"vmovss %1, %%xmm1\n\t" // alpha -> xmm1
"movq %2, %%rsi\n\t" // adress of a -> rsi
"movq %3, %%rcx\n\t" // value of lda > rcx
"movq %4, %%rdi\n\t" // adress of x -> rdi
"movq %5, %%rdx\n\t" // adress of y -> rdx
"leaq (, %%rcx,4), %%rcx \n\t" // scale lda by size of float
"leaq (%%rsi,%%rcx,1), %%r8 \n\t" // pointer to next line
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
"vxorps %%xmm13, %%xmm13, %%xmm13\n\t" // set to zero
"vxorps %%xmm14, %%xmm14, %%xmm14\n\t" // set to zero
"vxorps %%xmm15, %%xmm15, %%xmm15\n\t" // set to zero
"sarq $4, %%rax \n\t" // n = n / 16
".align 16 \n\t"
".L01LOOP%=: \n\t"
// "prefetcht0 512(%%rsi) \n\t"
"prefetcht0 (%%r8) \n\t" //prefetch next line of a
"vmovups (%%rsi), %%xmm4 \n\t"
"vmovups 4*4(%%rsi), %%xmm5 \n\t"
"vmovups 8*4(%%rsi), %%xmm6 \n\t"
"vmovups 12*4(%%rsi), %%xmm7 \n\t"
"vfmaddps %%xmm12, 0*4(%%rdi), %%xmm4, %%xmm12\n\t" // multiply a and c and add to temp
"vfmaddps %%xmm13, 4*4(%%rdi), %%xmm5, %%xmm13\n\t" // multiply a and c and add to temp
"vfmaddps %%xmm14, 8*4(%%rdi), %%xmm6, %%xmm14\n\t" // multiply a and c and add to temp
"vfmaddps %%xmm15, 12*4(%%rdi), %%xmm7, %%xmm15\n\t" // multiply a and c and add to temp
"addq $16*4 , %%r8 \n\t" // increment prefetch pointer
"addq $16*4 , %%rsi \n\t" // increment pointer of a
"addq $16*4 , %%rdi \n\t" // increment pointer of c
"dec %%rax \n\t" // n = n -1
"jnz .L01LOOP%= \n\t"
"vaddps %%xmm12, %%xmm14, %%xmm12\n\t"
"vaddps %%xmm13, %%xmm15, %%xmm13\n\t"
"vaddps %%xmm12, %%xmm13, %%xmm12\n\t"
"vhaddps %%xmm12, %%xmm12, %%xmm12\n\t"
"vhaddps %%xmm12, %%xmm12, %%xmm12\n\t"
"vfmaddss (%%rdx), %%xmm12, %%xmm1, %%xmm12\n\t"
"vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y
:
:
"m" (n), // 0
"m" (alpha), // 1
"m" (a), // 2
"m" (lda), // 3
"m" (x), // 4
"m" (y) // 5
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
"%xmm0", "%xmm1",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}

112
kernel/x86_64/sgemv_t_microk_haswell-2.c Normal file
View File

@ -0,0 +1,112 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#define HAVE_KERNEL_16x4 1
static void sgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) __attribute__ ((noinline));
static void sgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)
{
BLASLONG register i = 0;
__asm__ __volatile__
(
"vzeroupper \n\t"
"vxorps %%ymm4 , %%ymm4, %%ymm4 \n\t"
"vxorps %%ymm5 , %%ymm5, %%ymm5 \n\t"
"vxorps %%ymm6 , %%ymm6, %%ymm6 \n\t"
"vxorps %%ymm7 , %%ymm7, %%ymm7 \n\t"
".align 16 \n\t"
".L01LOOP%=: \n\t"
"prefetcht0 384(%2,%0,4) \n\t"
"vmovups (%2,%0,4), %%ymm12 \n\t" // 8 * x
"vmovups 32(%2,%0,4), %%ymm13 \n\t" // 8 * x
"prefetcht0 384(%4,%0,4) \n\t"
"vfmadd231ps (%4,%0,4), %%ymm12, %%ymm4 \n\t"
"vfmadd231ps (%5,%0,4), %%ymm12, %%ymm5 \n\t"
"prefetcht0 384(%5,%0,4) \n\t"
"vfmadd231ps 32(%4,%0,4), %%ymm13, %%ymm4 \n\t"
"vfmadd231ps 32(%5,%0,4), %%ymm13, %%ymm5 \n\t"
"prefetcht0 384(%6,%0,4) \n\t"
"vfmadd231ps (%6,%0,4), %%ymm12, %%ymm6 \n\t"
"vfmadd231ps (%7,%0,4), %%ymm12, %%ymm7 \n\t"
"prefetcht0 384(%7,%0,4) \n\t"
"vfmadd231ps 32(%6,%0,4), %%ymm13, %%ymm6 \n\t"
"vfmadd231ps 32(%7,%0,4), %%ymm13, %%ymm7 \n\t"
"addq $16, %0 \n\t"
"subq $16, %1 \n\t"
"jnz .L01LOOP%= \n\t"
"vextractf128 $1 , %%ymm4, %%xmm12 \n\t"
"vextractf128 $1 , %%ymm5, %%xmm13 \n\t"
"vextractf128 $1 , %%ymm6, %%xmm14 \n\t"
"vextractf128 $1 , %%ymm7, %%xmm15 \n\t"
"vaddps %%xmm4, %%xmm12, %%xmm4 \n\t"
"vaddps %%xmm5, %%xmm13, %%xmm5 \n\t"
"vaddps %%xmm6, %%xmm14, %%xmm6 \n\t"
"vaddps %%xmm7, %%xmm15, %%xmm7 \n\t"
"vhaddps %%xmm4, %%xmm4, %%xmm4 \n\t"
"vhaddps %%xmm5, %%xmm5, %%xmm5 \n\t"
"vhaddps %%xmm6, %%xmm6, %%xmm6 \n\t"
"vhaddps %%xmm7, %%xmm7, %%xmm7 \n\t"
"vhaddps %%xmm4, %%xmm4, %%xmm4 \n\t"
"vhaddps %%xmm5, %%xmm5, %%xmm5 \n\t"
"vhaddps %%xmm6, %%xmm6, %%xmm6 \n\t"
"vhaddps %%xmm7, %%xmm7, %%xmm7 \n\t"
"vmovss %%xmm4, (%3) \n\t"
"vmovss %%xmm5, 4(%3) \n\t"
"vmovss %%xmm6, 8(%3) \n\t"
"vmovss %%xmm7, 12(%3) \n\t"
"vzeroupper \n\t"
:
:
"r" (i), // 0
"r" (n), // 1
"r" (x), // 2
"r" (y), // 3
"r" (ap[0]), // 4
"r" (ap[1]), // 5
"r" (ap[2]), // 6
"r" (ap[3]) // 7
: "cc",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}

100
kernel/x86_64/sgemv_t_microk_haswell.c Normal file
View File

@ -0,0 +1,100 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
static void sgemv_kernel_16( long n, float alpha, float *a, long lda, float *x, float *y)
{
//n = n / 16;
__asm__ __volatile__
(
"movq %0, %%rax\n\t" // n -> rax
"vmovss %1, %%xmm1\n\t" // alpha -> xmm1
"movq %2, %%rsi\n\t" // adress of a -> rsi
"movq %3, %%rcx\n\t" // value of lda > rcx
"movq %4, %%rdi\n\t" // adress of x -> rdi
"movq %5, %%rdx\n\t" // adress of y -> rdx
"leaq (, %%rcx,4), %%rcx \n\t" // scale lda by size of float
"leaq (%%rsi,%%rcx,1), %%r8 \n\t" // pointer to next line
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
"vxorps %%xmm13, %%xmm13, %%xmm13\n\t" // set to zero
"vxorps %%xmm14, %%xmm14, %%xmm14\n\t" // set to zero
"vxorps %%xmm15, %%xmm15, %%xmm15\n\t" // set to zero
"sarq $4, %%rax \n\t" // n = n / 16
".align 16 \n\t"
".L01LOOP%=: \n\t"
// "prefetcht0 512(%%rsi) \n\t"
"prefetcht0 (%%r8) \n\t" //prefetch next line of a
"vmovups (%%rsi), %%xmm4 \n\t"
"vmovups 4*4(%%rsi), %%xmm5 \n\t"
"vmovups 8*4(%%rsi), %%xmm6 \n\t"
"vmovups 12*4(%%rsi), %%xmm7 \n\t"
"vfmadd231ps 0*4(%%rdi), %%xmm4, %%xmm12\n\t" // multiply a and c and add to temp
"vfmadd231ps 4*4(%%rdi), %%xmm5, %%xmm13\n\t" // multiply a and c and add to temp
"vfmadd231ps 8*4(%%rdi), %%xmm6, %%xmm14\n\t" // multiply a and c and add to temp
"vfmadd231ps 12*4(%%rdi), %%xmm7, %%xmm15\n\t" // multiply a and c and add to temp
"addq $16*4 , %%r8 \n\t" // increment prefetch pointer
"addq $16*4 , %%rsi \n\t" // increment pointer of a
"addq $16*4 , %%rdi \n\t" // increment pointer of c
"dec %%rax \n\t" // n = n -1
"jnz .L01LOOP%= \n\t"
"vaddps %%xmm12, %%xmm14, %%xmm12\n\t"
"vaddps %%xmm13, %%xmm15, %%xmm13\n\t"
"vaddps %%xmm12, %%xmm13, %%xmm12\n\t"
"vhaddps %%xmm12, %%xmm12, %%xmm12\n\t"
"vhaddps %%xmm12, %%xmm12, %%xmm12\n\t"
"vmulss %%xmm12, %%xmm1, %%xmm12\n\t"
"vaddss (%%rdx), %%xmm12,%%xmm12\n\t"
"vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y
:
:
"m" (n), // 0
"m" (alpha), // 1
"m" (a), // 2
"m" (lda), // 3
"m" (x), // 4
"m" (y) // 5
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
"%xmm0", "%xmm1",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}

159
kernel/x86_64/sgemv_t_microk_nehalem-2.c Normal file
View File

@ -0,0 +1,159 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#define HAVE_KERNEL_16x4 1
static void sgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) __attribute__ ((noinline));
static void sgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)
{
BLASLONG register i = 0;
__asm__ __volatile__
(
"xorps %%xmm0 , %%xmm0 \n\t"
"xorps %%xmm1 , %%xmm1 \n\t"
"xorps %%xmm2 , %%xmm2 \n\t"
"xorps %%xmm3 , %%xmm3 \n\t"
"xorps %%xmm4 , %%xmm4 \n\t"
"xorps %%xmm5 , %%xmm5 \n\t"
"xorps %%xmm6 , %%xmm6 \n\t"
"xorps %%xmm7 , %%xmm7 \n\t"
".align 16 \n\t"
".L01LOOP%=: \n\t"
"prefetcht0 384(%2,%0,4) \n\t"
"movups (%2,%0,4), %%xmm12 \n\t" // 4 * x
"movups 16(%2,%0,4), %%xmm13 \n\t" // 4 * x
"movups (%4,%0,4), %%xmm8 \n\t"
"movups 32(%2,%0,4), %%xmm14 \n\t" // 4 * x
"movups 48(%2,%0,4), %%xmm15 \n\t" // 4 * x
"prefetcht0 384(%4,%0,4) \n\t"
"movups 16(%4,%0,4), %%xmm9 \n\t"
"movups 32(%4,%0,4), %%xmm10 \n\t"
"movups 48(%4,%0,4), %%xmm11 \n\t"
"mulps %%xmm12, %%xmm8 \n\t"
"addps %%xmm8 , %%xmm0 \n\t"
"mulps %%xmm13, %%xmm9 \n\t"
"addps %%xmm9 , %%xmm4 \n\t"
"movups (%5,%0,4), %%xmm8 \n\t"
"mulps %%xmm14, %%xmm10 \n\t"
"addps %%xmm10, %%xmm0 \n\t"
"mulps %%xmm15, %%xmm11 \n\t"
"addps %%xmm11, %%xmm4 \n\t"
"prefetcht0 384(%5,%0,4) \n\t"
"movups 16(%5,%0,4), %%xmm9 \n\t"
"movups 32(%5,%0,4), %%xmm10 \n\t"
"movups 48(%5,%0,4), %%xmm11 \n\t"
"mulps %%xmm12, %%xmm8 \n\t"
"addps %%xmm8 , %%xmm1 \n\t"
"mulps %%xmm13, %%xmm9 \n\t"
"addps %%xmm9 , %%xmm5 \n\t"
"movups (%6,%0,4), %%xmm8 \n\t"
"mulps %%xmm14, %%xmm10 \n\t"
"addps %%xmm10, %%xmm1 \n\t"
"mulps %%xmm15, %%xmm11 \n\t"
"addps %%xmm11, %%xmm5 \n\t"
"prefetcht0 384(%6,%0,4) \n\t"
"movups 16(%6,%0,4), %%xmm9 \n\t"
"movups 32(%6,%0,4), %%xmm10 \n\t"
"movups 48(%6,%0,4), %%xmm11 \n\t"
"mulps %%xmm12, %%xmm8 \n\t"
"addps %%xmm8 , %%xmm2 \n\t"
"mulps %%xmm13, %%xmm9 \n\t"
"addps %%xmm9 , %%xmm6 \n\t"
"movups (%7,%0,4), %%xmm8 \n\t"
"mulps %%xmm14, %%xmm10 \n\t"
"addps %%xmm10, %%xmm2 \n\t"
"mulps %%xmm15, %%xmm11 \n\t"
"addps %%xmm11, %%xmm6 \n\t"
"prefetcht0 384(%7,%0,4) \n\t"
"movups 16(%7,%0,4), %%xmm9 \n\t"
"movups 32(%7,%0,4), %%xmm10 \n\t"
"movups 48(%7,%0,4), %%xmm11 \n\t"
"mulps %%xmm12, %%xmm8 \n\t"
"addps %%xmm8 , %%xmm3 \n\t"
"mulps %%xmm13, %%xmm9 \n\t"
"addps %%xmm9 , %%xmm7 \n\t"
"mulps %%xmm14, %%xmm10 \n\t"
"addps %%xmm10, %%xmm3 \n\t"
"mulps %%xmm15, %%xmm11 \n\t"
"addps %%xmm11, %%xmm7 \n\t"
"addq $16, %0 \n\t"
"subq $16, %1 \n\t"
"jnz .L01LOOP%= \n\t"
"addps %%xmm0, %%xmm4 \n\t"
"addps %%xmm1, %%xmm5 \n\t"
"addps %%xmm2, %%xmm6 \n\t"
"addps %%xmm3, %%xmm7 \n\t"
"haddps %%xmm4, %%xmm4 \n\t"
"haddps %%xmm5, %%xmm5 \n\t"
"haddps %%xmm6, %%xmm6 \n\t"
"haddps %%xmm7, %%xmm7 \n\t"
"haddps %%xmm4, %%xmm4 \n\t"
"haddps %%xmm5, %%xmm5 \n\t"
"haddps %%xmm6, %%xmm6 \n\t"
"haddps %%xmm7, %%xmm7 \n\t"
"movss %%xmm4, (%3) \n\t"
"movss %%xmm5, 4(%3) \n\t"
"movss %%xmm6, 8(%3) \n\t"
"movss %%xmm7, 12(%3) \n\t"
:
:
"r" (i), // 0
"r" (n), // 1
"r" (x), // 2
"r" (y), // 3
"r" (ap[0]), // 4
"r" (ap[1]), // 5
"r" (ap[2]), // 6
"r" (ap[3]) // 7
: "cc",
"%xmm0", "%xmm1", "%xmm2", "%xmm3",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}

132
kernel/x86_64/sgemv_t_microk_sandy-2.c Normal file
View File

@ -0,0 +1,132 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#define HAVE_KERNEL_16x4 1
static void sgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) __attribute__ ((noinline));
static void sgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)
{
BLASLONG register i = 0;
__asm__ __volatile__
(
"vzeroupper \n\t"
"vxorps %%ymm0 , %%ymm0, %%ymm0 \n\t"
"vxorps %%ymm1 , %%ymm1, %%ymm1 \n\t"
"vxorps %%ymm2 , %%ymm2, %%ymm2 \n\t"
"vxorps %%ymm3 , %%ymm3, %%ymm3 \n\t"
"vxorps %%ymm4 , %%ymm4, %%ymm4 \n\t"
"vxorps %%ymm5 , %%ymm5, %%ymm5 \n\t"
"vxorps %%ymm6 , %%ymm6, %%ymm6 \n\t"
"vxorps %%ymm7 , %%ymm7, %%ymm7 \n\t"
".align 16 \n\t"
".L01LOOP%=: \n\t"
"prefetcht0 384(%2,%0,4) \n\t"
"vmovups (%2,%0,4), %%ymm12 \n\t" // 8 * x
"vmovups 32(%2,%0,4), %%ymm13 \n\t" // 8 * x
"prefetcht0 384(%4,%0,4) \n\t"
"vmulps (%4,%0,4), %%ymm12, %%ymm8 \n\t"
"vaddps %%ymm4, %%ymm8 , %%ymm4 \n\t"
"vmulps 32(%4,%0,4), %%ymm13, %%ymm9 \n\t"
"vaddps %%ymm0, %%ymm9 , %%ymm0 \n\t"
"prefetcht0 384(%5,%0,4) \n\t"
"vmulps (%5,%0,4), %%ymm12, %%ymm10 \n\t"
"vaddps %%ymm1, %%ymm10, %%ymm1 \n\t"
"vmulps 32(%5,%0,4), %%ymm13, %%ymm11 \n\t"
"vaddps %%ymm5, %%ymm11, %%ymm5 \n\t"
"prefetcht0 384(%6,%0,4) \n\t"
"vmulps (%6,%0,4), %%ymm12, %%ymm8 \n\t"
"vaddps %%ymm6, %%ymm8 , %%ymm6 \n\t"
"vmulps 32(%6,%0,4), %%ymm13, %%ymm9 \n\t"
"vaddps %%ymm2, %%ymm9 , %%ymm2 \n\t"
"prefetcht0 384(%7,%0,4) \n\t"
"vmulps (%7,%0,4), %%ymm12, %%ymm10 \n\t"
"vaddps %%ymm7, %%ymm10, %%ymm7 \n\t"
"vmulps 32(%7,%0,4), %%ymm13, %%ymm11 \n\t"
"vaddps %%ymm3, %%ymm11, %%ymm3 \n\t"
"addq $16, %0 \n\t"
"subq $16, %1 \n\t"
"jnz .L01LOOP%= \n\t"
"vaddps %%ymm4, %%ymm0, %%ymm4 \n\t"
"vaddps %%ymm5, %%ymm1, %%ymm5 \n\t"
"vaddps %%ymm6, %%ymm2, %%ymm6 \n\t"
"vaddps %%ymm7, %%ymm3, %%ymm7 \n\t"
"vextractf128 $1 , %%ymm4, %%xmm12 \n\t"
"vextractf128 $1 , %%ymm5, %%xmm13 \n\t"
"vextractf128 $1 , %%ymm6, %%xmm14 \n\t"
"vextractf128 $1 , %%ymm7, %%xmm15 \n\t"
"vaddps %%xmm4, %%xmm12, %%xmm4 \n\t"
"vaddps %%xmm5, %%xmm13, %%xmm5 \n\t"
"vaddps %%xmm6, %%xmm14, %%xmm6 \n\t"
"vaddps %%xmm7, %%xmm15, %%xmm7 \n\t"
"vhaddps %%xmm4, %%xmm4, %%xmm4 \n\t"
"vhaddps %%xmm5, %%xmm5, %%xmm5 \n\t"
"vhaddps %%xmm6, %%xmm6, %%xmm6 \n\t"
"vhaddps %%xmm7, %%xmm7, %%xmm7 \n\t"
"vhaddps %%xmm4, %%xmm4, %%xmm4 \n\t"
"vhaddps %%xmm5, %%xmm5, %%xmm5 \n\t"
"vhaddps %%xmm6, %%xmm6, %%xmm6 \n\t"
"vhaddps %%xmm7, %%xmm7, %%xmm7 \n\t"
"vmovss %%xmm4, (%3) \n\t"
"vmovss %%xmm5, 4(%3) \n\t"
"vmovss %%xmm6, 8(%3) \n\t"
"vmovss %%xmm7, 12(%3) \n\t"
"vzeroupper \n\t"
:
:
"r" (i), // 0
"r" (n), // 1
"r" (x), // 2
"r" (y), // 3
"r" (ap[0]), // 4
"r" (ap[1]), // 5
"r" (ap[2]), // 6
"r" (ap[3]) // 7
: "cc",
"%xmm0", "%xmm1", "%xmm2", "%xmm3",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}

106
kernel/x86_64/sgemv_t_microk_sandy.c Normal file
View File

@ -0,0 +1,106 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
static void sgemv_kernel_16( long n, float alpha, float *a, long lda, float *x, float *y)
{
//n = n / 16;
__asm__ __volatile__
(
"movq %0, %%rax\n\t" // n -> rax
"vmovss %1, %%xmm1\n\t" // alpha -> xmm1
"movq %2, %%rsi\n\t" // adress of a -> rsi
"movq %3, %%rcx\n\t" // value of lda > rcx
"movq %4, %%rdi\n\t" // adress of x -> rdi
"movq %5, %%rdx\n\t" // adress of y -> rdx
"leaq (, %%rcx,4), %%rcx \n\t" // scale lda by size of float
"leaq (%%rsi,%%rcx,1), %%r8 \n\t" // pointer to next line
"vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
"vxorps %%xmm13, %%xmm13, %%xmm13\n\t" // set to zero
"vxorps %%xmm14, %%xmm14, %%xmm14\n\t" // set to zero
"vxorps %%xmm15, %%xmm15, %%xmm15\n\t" // set to zero
"sarq $4, %%rax \n\t" // n = n / 16
".align 16 \n\t"
".L01LOOP%=: \n\t"
// "prefetcht0 512(%%rsi) \n\t"
"prefetcht0 (%%r8) \n\t" //prefetch next line of a
"vmovups (%%rsi), %%xmm4 \n\t"
"vmovups 4*4(%%rsi), %%xmm5 \n\t"
"vmovups 8*4(%%rsi), %%xmm6 \n\t"
"vmovups 12*4(%%rsi), %%xmm7 \n\t"
"vmulps 0*4(%%rdi), %%xmm4, %%xmm8 \n\t" // multiply a and c and add to temp
"vmulps 4*4(%%rdi), %%xmm5, %%xmm9 \n\t" // multiply a and c and add to temp
"vmulps 8*4(%%rdi), %%xmm6, %%xmm10\n\t" // multiply a and c and add to temp
"vmulps 12*4(%%rdi), %%xmm7, %%xmm11\n\t" // multiply a and c and add to temp
"vaddps %%xmm12, %%xmm8 , %%xmm12\n\t"
"vaddps %%xmm13, %%xmm9 , %%xmm13\n\t"
"vaddps %%xmm14, %%xmm10, %%xmm14\n\t"
"vaddps %%xmm15, %%xmm11, %%xmm15\n\t"
"addq $16*4 , %%r8 \n\t" // increment prefetch pointer
"addq $16*4 , %%rsi \n\t" // increment pointer of a
"addq $16*4 , %%rdi \n\t" // increment pointer of c
"dec %%rax \n\t" // n = n -1
"jnz .L01LOOP%= \n\t"
"vaddps %%xmm12, %%xmm14, %%xmm12\n\t"
"vaddps %%xmm13, %%xmm15, %%xmm13\n\t"
"vaddps %%xmm12, %%xmm13, %%xmm12\n\t"
"vhaddps %%xmm12, %%xmm12, %%xmm12\n\t"
"vhaddps %%xmm12, %%xmm12, %%xmm12\n\t"
"vmulss %%xmm12, %%xmm1, %%xmm12 \n\t"
"vaddss (%%rdx), %%xmm12, %%xmm12\n\t"
"vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y
:
:
"m" (n), // 0
"m" (alpha), // 1
"m" (a), // 2
"m" (lda), // 3
"m" (x), // 4
"m" (y) // 5
: "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8", "cc",
"%xmm0", "%xmm1",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}

2080
kernel/x86_64/zgemm_kernel_4x2_haswell.S
View File

File diff suppressed because it is too large Load Diff

258
kernel/x86_64/zgemv_n.c Normal file
View File

@ -0,0 +1,258 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#include <stdlib.h>
#include <stdio.h>
#include "common.h"
#if defined(HASWELL)
#include "zgemv_n_microk_haswell-2.c"
#elif defined(SANDYBRIDGE)
#include "zgemv_n_microk_sandy-2.c"
#endif
#define NBMAX 1024
#ifndef HAVE_KERNEL_16x4
static void zgemv_kernel_16x4(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)
{
BLASLONG i;
FLOAT *a0,*a1,*a2,*a3;
a0 = ap[0];
a1 = ap[1];
a2 = ap[2];
a3 = ap[3];
for ( i=0; i< 2*n; i+=2 )
{
#if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )
y[i] += a0[i]*x[0] - a0[i+1] * x[1];
y[i+1] += a0[i]*x[1] + a0[i+1] * x[0];
y[i] += a1[i]*x[2] - a1[i+1] * x[3];
y[i+1] += a1[i]*x[3] + a1[i+1] * x[2];
y[i] += a2[i]*x[4] - a2[i+1] * x[5];
y[i+1] += a2[i]*x[5] + a2[i+1] * x[4];
y[i] += a3[i]*x[6] - a3[i+1] * x[7];
y[i+1] += a3[i]*x[7] + a3[i+1] * x[6];
#else
y[i] += a0[i]*x[0] + a0[i+1] * x[1];
y[i+1] += a0[i]*x[1] - a0[i+1] * x[0];
y[i] += a1[i]*x[2] + a1[i+1] * x[3];
y[i+1] += a1[i]*x[3] - a1[i+1] * x[2];
y[i] += a2[i]*x[4] + a2[i+1] * x[5];
y[i+1] += a2[i]*x[5] - a2[i+1] * x[4];
y[i] += a3[i]*x[6] + a3[i+1] * x[7];
y[i+1] += a3[i]*x[7] - a3[i+1] * x[6];
#endif
}
}
#endif
static void zgemv_kernel_16x1(BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y)
{
BLASLONG i;
FLOAT *a0;
a0 = ap;
for ( i=0; i< 2*n; i+=2 )
{
#if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )
y[i] += a0[i]*x[0] - a0[i+1] * x[1];
y[i+1] += a0[i]*x[1] + a0[i+1] * x[0];
#else
y[i] += a0[i]*x[0] + a0[i+1] * x[1];
y[i+1] += a0[i]*x[1] - a0[i+1] * x[0];
#endif
}
}
static void zero_y(BLASLONG n, FLOAT *dest)
{
BLASLONG i;
for ( i=0; i<2*n; i++ )
{
*dest = 0.0;
dest++;
}
}
static void add_y(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_dest,FLOAT alpha_r, FLOAT alpha_i)
{
BLASLONG i;
FLOAT temp_r;
FLOAT temp_i;
for ( i=0; i<n; i++ )
{
#if !defined(XCONJ)
temp_r = alpha_r * src[0] - alpha_i * src[1];
temp_i = alpha_r * src[1] + alpha_i * src[0];
#else
temp_r = alpha_r * src[0] + alpha_i * src[1];
temp_i = -alpha_r * src[1] + alpha_i * src[0];
#endif
*dest += temp_r;
*(dest+1) += temp_i;
src+=2;
dest += inc_dest;
}
}
int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r,FLOAT alpha_i, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer)
{
BLASLONG i;
BLASLONG j;
FLOAT *a_ptr;
FLOAT *x_ptr;
FLOAT *y_ptr;
FLOAT *ap[4];
BLASLONG n1;
BLASLONG m1;
BLASLONG m2;
BLASLONG n2;
FLOAT xbuffer[8],*ybuffer;
#if 0
printf("%s %d %d %.16f %.16f %d %d %d\n","zgemv_n",m,n,alpha_r,alpha_i,lda,inc_x,inc_y);
#endif
if ( m < 1 ) return(0);
if ( n < 1 ) return(0);
ybuffer = buffer;
inc_x *= 2;
inc_y *= 2;
lda *= 2;
n1 = n / 4 ;
n2 = n % 4 ;
m1 = m - ( m % 16 );
m2 = (m % NBMAX) - (m % 16) ;
y_ptr = y;
BLASLONG NB = NBMAX;
while ( NB == NBMAX )
{
m1 -= NB;
if ( m1 < 0)
{
if ( m2 == 0 ) break;
NB = m2;
}
a_ptr = a;
x_ptr = x;
zero_y(NB,ybuffer);
for( i = 0; i < n1 ; i++)
{
xbuffer[0] = x_ptr[0];
xbuffer[1] = x_ptr[1];
x_ptr += inc_x;
xbuffer[2] = x_ptr[0];
xbuffer[3] = x_ptr[1];
x_ptr += inc_x;
xbuffer[4] = x_ptr[0];
xbuffer[5] = x_ptr[1];
x_ptr += inc_x;
xbuffer[6] = x_ptr[0];
xbuffer[7] = x_ptr[1];
x_ptr += inc_x;
ap[0] = a_ptr;
ap[1] = a_ptr + lda;
ap[2] = ap[1] + lda;
ap[3] = ap[2] + lda;
zgemv_kernel_16x4(NB,ap,xbuffer,ybuffer);
a_ptr += 4 * lda;
}
for( i = 0; i < n2 ; i++)
{
xbuffer[0] = x_ptr[0];
xbuffer[1] = x_ptr[1];
x_ptr += inc_x;
zgemv_kernel_16x1(NB,a_ptr,xbuffer,ybuffer);
a_ptr += 1 * lda;
}
add_y(NB,ybuffer,y_ptr,inc_y,alpha_r,alpha_i);
a += 2 * NB;
y_ptr += NB * inc_y;
}
j=0;
while ( j < (m % 16))
{
a_ptr = a;
x_ptr = x;
FLOAT temp_r = 0.0;
FLOAT temp_i = 0.0;
for( i = 0; i < n; i++ )
{
#if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )
temp_r += a_ptr[0] * x_ptr[0] - a_ptr[1] * x_ptr[1];
temp_i += a_ptr[0] * x_ptr[1] + a_ptr[1] * x_ptr[0];
#else
temp_r += a_ptr[0] * x_ptr[0] + a_ptr[1] * x_ptr[1];
temp_i += a_ptr[0] * x_ptr[1] - a_ptr[1] * x_ptr[0];
#endif
a_ptr += lda;
x_ptr += inc_x;
}
#if !defined(XCONJ)
y_ptr[0] += alpha_r * temp_r - alpha_i * temp_i;
y_ptr[1] += alpha_r * temp_i + alpha_i * temp_r;
#else
y_ptr[0] += alpha_r * temp_r + alpha_i * temp_i;
y_ptr[1] -= alpha_r * temp_i - alpha_i * temp_r;
#endif
y_ptr += inc_y;
a+=2;
j++;
}
return(0);
}

137
kernel/x86_64/zgemv_n_microk_haswell-2.c Normal file
View File

@ -0,0 +1,137 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#define HAVE_KERNEL_16x4 1
static void zgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) __attribute__ ((noinline));
static void zgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)
{
BLASLONG register i = 0;
__asm__ __volatile__
(
"vzeroupper \n\t"
"vbroadcastsd (%2), %%ymm0 \n\t" // real part x0
"vbroadcastsd 8(%2), %%ymm1 \n\t" // imag part x0
"vbroadcastsd 16(%2), %%ymm2 \n\t" // real part x1
"vbroadcastsd 24(%2), %%ymm3 \n\t" // imag part x1
"vbroadcastsd 32(%2), %%ymm4 \n\t" // real part x2
"vbroadcastsd 40(%2), %%ymm5 \n\t" // imag part x2
"vbroadcastsd 48(%2), %%ymm6 \n\t" // real part x3
"vbroadcastsd 56(%2), %%ymm7 \n\t" // imag part x3
".align 16 \n\t"
".L01LOOP%=: \n\t"
"prefetcht0 192(%4,%0,8) \n\t"
"vmovups (%4,%0,8), %%ymm8 \n\t" // 2 complex values form a0
"vmovups 32(%4,%0,8), %%ymm9 \n\t" // 2 complex values form a0
"prefetcht0 192(%5,%0,8) \n\t"
"vmovups (%5,%0,8), %%ymm10 \n\t" // 2 complex values form a1
"vmovups 32(%5,%0,8), %%ymm11 \n\t" // 2 complex values form a1
"vmulpd %%ymm8 , %%ymm0, %%ymm12 \n\t" // a_r[0] * x_r , a_i[0] * x_r, a_r[1] * x_r, a_i[1] * x_r
"vmulpd %%ymm8 , %%ymm1, %%ymm13 \n\t" // a_r[0] * x_i , a_i[0] * x_i, a_r[1] * x_i, a_i[1] * x_i
"vmulpd %%ymm9 , %%ymm0, %%ymm14 \n\t" // a_r[2] * x_r , a_i[2] * x_r, a_r[3] * x_r, a_i[3] * x_r
"vmulpd %%ymm9 , %%ymm1, %%ymm15 \n\t" // a_r[2] * x_i , a_i[2] * x_i, a_r[3] * x_i, a_i[3] * x_i
"prefetcht0 192(%6,%0,8) \n\t"
"vmovups (%6,%0,8), %%ymm8 \n\t" // 2 complex values form a2
"vmovups 32(%6,%0,8), %%ymm9 \n\t" // 2 complex values form a2
"vfmadd231pd %%ymm10, %%ymm2, %%ymm12 \n\t" // a_r[0] * x_r , a_i[0] * x_r, a_r[1] * x_r, a_i[1] * x_r
"vfmadd231pd %%ymm10, %%ymm3, %%ymm13 \n\t" // a_r[0] * x_i , a_i[0] * x_i, a_r[1] * x_i, a_i[1] * x_i
"vfmadd231pd %%ymm11, %%ymm2, %%ymm14 \n\t" // a_r[2] * x_r , a_i[2] * x_r, a_r[3] * x_r, a_i[3] * x_r
"vfmadd231pd %%ymm11, %%ymm3, %%ymm15 \n\t" // a_r[2] * x_i , a_i[2] * x_i, a_r[3] * x_i, a_i[3] * x_i
"prefetcht0 192(%7,%0,8) \n\t"
"vmovups (%7,%0,8), %%ymm10 \n\t" // 2 complex values form a3
"vmovups 32(%7,%0,8), %%ymm11 \n\t" // 2 complex values form a3
"vfmadd231pd %%ymm8 , %%ymm4, %%ymm12 \n\t" // a_r[0] * x_r , a_i[0] * x_r, a_r[1] * x_r, a_i[1] * x_r
"vfmadd231pd %%ymm8 , %%ymm5, %%ymm13 \n\t" // a_r[0] * x_i , a_i[0] * x_i, a_r[1] * x_i, a_i[1] * x_i
"vfmadd231pd %%ymm9 , %%ymm4, %%ymm14 \n\t" // a_r[2] * x_r , a_i[2] * x_r, a_r[3] * x_r, a_i[3] * x_r
"vfmadd231pd %%ymm9 , %%ymm5, %%ymm15 \n\t" // a_r[2] * x_i , a_i[2] * x_i, a_r[3] * x_i, a_i[3] * x_i
"vfmadd231pd %%ymm10, %%ymm6, %%ymm12 \n\t" // a_r[0] * x_r , a_i[0] * x_r, a_r[1] * x_r, a_i[1] * x_r
"vfmadd231pd %%ymm10, %%ymm7, %%ymm13 \n\t" // a_r[0] * x_i , a_i[0] * x_i, a_r[1] * x_i, a_i[1] * x_i
"vfmadd231pd %%ymm11, %%ymm6, %%ymm14 \n\t" // a_r[2] * x_r , a_i[2] * x_r, a_r[3] * x_r, a_i[3] * x_r
"vfmadd231pd %%ymm11, %%ymm7, %%ymm15 \n\t" // a_r[2] * x_i , a_i[2] * x_i, a_r[3] * x_i, a_i[3] * x_i
"prefetcht0 192(%3,%0,8) \n\t"
"vmovups (%3,%0,8), %%ymm10 \n\t"
"vmovups 32(%3,%0,8), %%ymm11 \n\t"
#if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )
"vpermilpd $0x5 , %%ymm13, %%ymm13 \n\t"
"vpermilpd $0x5 , %%ymm15, %%ymm15 \n\t"
"vaddsubpd %%ymm13, %%ymm12, %%ymm8 \n\t"
"vaddsubpd %%ymm15, %%ymm14, %%ymm9 \n\t"
#else
"vpermilpd $0x5 , %%ymm12, %%ymm12 \n\t"
"vpermilpd $0x5 , %%ymm14, %%ymm14 \n\t"
"vaddsubpd %%ymm12, %%ymm13, %%ymm8 \n\t"
"vaddsubpd %%ymm14, %%ymm15, %%ymm9 \n\t"
"vpermilpd $0x5 , %%ymm8 , %%ymm8 \n\t"
"vpermilpd $0x5 , %%ymm9 , %%ymm9 \n\t"
#endif
"vaddpd %%ymm8, %%ymm10, %%ymm12 \n\t"
"vaddpd %%ymm9, %%ymm11, %%ymm13 \n\t"
"vmovups %%ymm12, (%3,%0,8) \n\t" // 2 complex values to y
"vmovups %%ymm13, 32(%3,%0,8) \n\t"
"addq $8 , %0 \n\t"
"subq $4 , %1 \n\t"
"jnz .L01LOOP%= \n\t"
"vzeroupper \n\t"
:
:
"r" (i), // 0
"r" (n), // 1
"r" (x), // 2
"r" (y), // 3
"r" (ap[0]), // 4
"r" (ap[1]), // 5
"r" (ap[2]), // 6
"r" (ap[3]) // 7
: "cc",
"%xmm0", "%xmm1", "%xmm2", "%xmm3",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}

149
kernel/x86_64/zgemv_n_microk_sandy-2.c Normal file
View File

@ -0,0 +1,149 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#define HAVE_KERNEL_16x4 1
static void zgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) __attribute__ ((noinline));
static void zgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)
{
BLASLONG register i = 0;
__asm__ __volatile__
(
"vzeroupper \n\t"
"vbroadcastsd (%2), %%ymm0 \n\t" // real part x0
"vbroadcastsd 8(%2), %%ymm1 \n\t" // imag part x0
"vbroadcastsd 16(%2), %%ymm2 \n\t" // real part x1
"vbroadcastsd 24(%2), %%ymm3 \n\t" // imag part x1
"vbroadcastsd 32(%2), %%ymm4 \n\t" // real part x2
"vbroadcastsd 40(%2), %%ymm5 \n\t" // imag part x2
"vbroadcastsd 48(%2), %%ymm6 \n\t" // real part x3
"vbroadcastsd 56(%2), %%ymm7 \n\t" // imag part x3
".align 16 \n\t"
".L01LOOP%=: \n\t"
"prefetcht0 256(%4,%0,8) \n\t"
"vmovups (%4,%0,8), %%ymm8 \n\t" // 2 complex values form a0
"vmovups 32(%4,%0,8), %%ymm9 \n\t" // 2 complex values form a0
"vmulpd %%ymm8 , %%ymm0 , %%ymm12 \n\t"
"vmulpd %%ymm8 , %%ymm1 , %%ymm13 \n\t"
"prefetcht0 256(%5,%0,8) \n\t"
"vmulpd %%ymm9 , %%ymm0 , %%ymm14 \n\t"
"vmovups (%5,%0,8), %%ymm8 \n\t" // 2 complex values form a0
"vmulpd %%ymm9 , %%ymm1 , %%ymm15 \n\t"
"vmovups 32(%5,%0,8), %%ymm9 \n\t" // 2 complex values form a0
"vmulpd %%ymm8 , %%ymm2 , %%ymm10 \n\t"
"vaddpd %%ymm12, %%ymm10, %%ymm12 \n\t"
"vmulpd %%ymm8 , %%ymm3 , %%ymm11 \n\t"
"vaddpd %%ymm13, %%ymm11, %%ymm13 \n\t"
"prefetcht0 256(%6,%0,8) \n\t"
"vmulpd %%ymm9 , %%ymm2 , %%ymm10 \n\t"
"vaddpd %%ymm14, %%ymm10, %%ymm14 \n\t"
"vmovups (%6,%0,8), %%ymm8 \n\t" // 2 complex values form a0
"vmulpd %%ymm9 , %%ymm3 , %%ymm11 \n\t"
"vaddpd %%ymm15, %%ymm11, %%ymm15 \n\t"
"vmovups 32(%6,%0,8), %%ymm9 \n\t" // 2 complex values form a0
"vmulpd %%ymm8 , %%ymm4 , %%ymm10 \n\t"
"vaddpd %%ymm12, %%ymm10, %%ymm12 \n\t"
"vmulpd %%ymm8 , %%ymm5 , %%ymm11 \n\t"
"vaddpd %%ymm13, %%ymm11, %%ymm13 \n\t"
"prefetcht0 256(%7,%0,8) \n\t"
"vmulpd %%ymm9 , %%ymm4 , %%ymm10 \n\t"
"vaddpd %%ymm14, %%ymm10, %%ymm14 \n\t"
"vmovups (%7,%0,8), %%ymm8 \n\t" // 2 complex values form a0
"vmulpd %%ymm9 , %%ymm5 , %%ymm11 \n\t"
"vaddpd %%ymm15, %%ymm11, %%ymm15 \n\t"
"vmovups 32(%7,%0,8), %%ymm9 \n\t" // 2 complex values form a0
"vmulpd %%ymm8 , %%ymm6 , %%ymm10 \n\t"
"vaddpd %%ymm12, %%ymm10, %%ymm12 \n\t"
"vmulpd %%ymm8 , %%ymm7 , %%ymm11 \n\t"
"vaddpd %%ymm13, %%ymm11, %%ymm13 \n\t"
"vmulpd %%ymm9 , %%ymm6 , %%ymm10 \n\t"
"vaddpd %%ymm14, %%ymm10, %%ymm14 \n\t"
"vmulpd %%ymm9 , %%ymm7 , %%ymm11 \n\t"
"vaddpd %%ymm15, %%ymm11, %%ymm15 \n\t"
"prefetcht0 256(%3,%0,8) \n\t"
"vmovups (%3,%0,8), %%ymm10 \n\t"
"vmovups 32(%3,%0,8), %%ymm11 \n\t"
#if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )
"vpermilpd $0x5 , %%ymm13, %%ymm13 \n\t"
"vpermilpd $0x5 , %%ymm15, %%ymm15 \n\t"
"vaddsubpd %%ymm13, %%ymm12, %%ymm8 \n\t"
"vaddsubpd %%ymm15, %%ymm14, %%ymm9 \n\t"
#else
"vpermilpd $0x5 , %%ymm12, %%ymm12 \n\t"
"vpermilpd $0x5 , %%ymm14, %%ymm14 \n\t"
"vaddsubpd %%ymm12, %%ymm13, %%ymm8 \n\t"
"vaddsubpd %%ymm14, %%ymm15, %%ymm9 \n\t"
"vpermilpd $0x5 , %%ymm8 , %%ymm8 \n\t"
"vpermilpd $0x5 , %%ymm9 , %%ymm9 \n\t"
#endif
"vaddpd %%ymm8, %%ymm10, %%ymm12 \n\t"
"vaddpd %%ymm9, %%ymm11, %%ymm13 \n\t"
"vmovups %%ymm12, (%3,%0,8) \n\t" // 2 complex values to y
"vmovups %%ymm13, 32(%3,%0,8) \n\t"
"addq $8 , %0 \n\t"
"subq $4 , %1 \n\t"
"jnz .L01LOOP%= \n\t"
"vzeroupper \n\t"
:
:
"r" (i), // 0
"r" (n), // 1
"r" (x), // 2
"r" (y), // 3
"r" (ap[0]), // 4
"r" (ap[1]), // 5
"r" (ap[2]), // 6
"r" (ap[3]) // 7
: "cc",
"%xmm0", "%xmm1", "%xmm2", "%xmm3",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}

272
kernel/x86_64/zgemv_t.c Normal file
View File

@ -0,0 +1,272 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#include "common.h"
#if defined(BULLDOZER) || defined(PILEDRIVER)
#include "zgemv_t_microk_bulldozer-2.c"
#elif defined(HASWELL)
#include "zgemv_t_microk_haswell-2.c"
#endif
#define NBMAX 1028
#ifndef HAVE_KERNEL_16x4
static void zgemv_kernel_16x4(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)
{
BLASLONG i;
FLOAT *a0,*a1,*a2,*a3;
a0 = ap[0];
a1 = ap[1];
a2 = ap[2];
a3 = ap[3];
FLOAT temp_r0 = 0.0;
FLOAT temp_r1 = 0.0;
FLOAT temp_r2 = 0.0;
FLOAT temp_r3 = 0.0;
FLOAT temp_i0 = 0.0;
FLOAT temp_i1 = 0.0;
FLOAT temp_i2 = 0.0;
FLOAT temp_i3 = 0.0;
for ( i=0; i< 2*n; i+=2 )
{
#if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )
temp_r0 += a0[i]*x[i] - a0[i+1]*x[i+1];
temp_i0 += a0[i]*x[i+1] + a0[i+1]*x[i];
temp_r1 += a1[i]*x[i] - a1[i+1]*x[i+1];
temp_i1 += a1[i]*x[i+1] + a1[i+1]*x[i];
temp_r2 += a2[i]*x[i] - a2[i+1]*x[i+1];
temp_i2 += a2[i]*x[i+1] + a2[i+1]*x[i];
temp_r3 += a3[i]*x[i] - a3[i+1]*x[i+1];
temp_i3 += a3[i]*x[i+1] + a3[i+1]*x[i];
#else
temp_r0 += a0[i]*x[i] + a0[i+1]*x[i+1];
temp_i0 += a0[i]*x[i+1] - a0[i+1]*x[i];
temp_r1 += a1[i]*x[i] + a1[i+1]*x[i+1];
temp_i1 += a1[i]*x[i+1] - a1[i+1]*x[i];
temp_r2 += a2[i]*x[i] + a2[i+1]*x[i+1];
temp_i2 += a2[i]*x[i+1] - a2[i+1]*x[i];
temp_r3 += a3[i]*x[i] + a3[i+1]*x[i+1];
temp_i3 += a3[i]*x[i+1] - a3[i+1]*x[i];
#endif
}
y[0] = temp_r0;
y[1] = temp_i0;
y[2] = temp_r1;
y[3] = temp_i1;
y[4] = temp_r2;
y[5] = temp_i2;
y[6] = temp_r3;
y[7] = temp_i3;
}
#endif
static void zgemv_kernel_16x1(BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y)
{
BLASLONG i;
FLOAT *a0;
a0 = ap;
FLOAT temp_r = 0.0;
FLOAT temp_i = 0.0;
for ( i=0; i< 2*n; i+=2 )
{
#if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )
temp_r += a0[i]*x[i] - a0[i+1]*x[i+1];
temp_i += a0[i]*x[i+1] + a0[i+1]*x[i];
#else
temp_r += a0[i]*x[i] + a0[i+1]*x[i+1];
temp_i += a0[i]*x[i+1] - a0[i+1]*x[i];
#endif
}
*y = temp_r;
*(y+1) = temp_i;
}
static void copy_x(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_src)
{
BLASLONG i;
for ( i=0; i<n; i++ )
{
*dest = *src;
*(dest+1) = *(src+1);
dest+=2;
src += inc_src;
}
}
int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r, FLOAT alpha_i, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer)
{
BLASLONG i;
BLASLONG j;
FLOAT *a_ptr;
FLOAT *x_ptr;
FLOAT *y_ptr;
FLOAT *ap[8];
BLASLONG n1;
BLASLONG m1;
BLASLONG m2;
BLASLONG n2;
FLOAT ybuffer[8],*xbuffer;
if ( m < 1 ) return(0);
if ( n < 1 ) return(0);
inc_x *= 2;
inc_y *= 2;
lda *= 2;
xbuffer = buffer;
n1 = n / 4 ;
n2 = n % 4 ;
m1 = m - ( m % 16 );
m2 = (m % NBMAX) - (m % 16) ;
BLASLONG NB = NBMAX;
while ( NB == NBMAX )
{
m1 -= NB;
if ( m1 < 0)
{
if ( m2 == 0 ) break;
NB = m2;
}
y_ptr = y;
a_ptr = a;
x_ptr = x;
copy_x(NB,x_ptr,xbuffer,inc_x);
for( i = 0; i < n1 ; i++)
{
ap[0] = a_ptr;
ap[1] = a_ptr + lda;
ap[2] = ap[1] + lda;
ap[3] = ap[2] + lda;
zgemv_kernel_16x4(NB,ap,xbuffer,ybuffer);
a_ptr += 4 * lda;
#if !defined(XCONJ)
y_ptr[0] += alpha_r * ybuffer[0] - alpha_i * ybuffer[1];
y_ptr[1] += alpha_r * ybuffer[1] + alpha_i * ybuffer[0];
y_ptr += inc_y;
y_ptr[0] += alpha_r * ybuffer[2] - alpha_i * ybuffer[3];
y_ptr[1] += alpha_r * ybuffer[3] + alpha_i * ybuffer[2];
y_ptr += inc_y;
y_ptr[0] += alpha_r * ybuffer[4] - alpha_i * ybuffer[5];
y_ptr[1] += alpha_r * ybuffer[5] + alpha_i * ybuffer[4];
y_ptr += inc_y;
y_ptr[0] += alpha_r * ybuffer[6] - alpha_i * ybuffer[7];
y_ptr[1] += alpha_r * ybuffer[7] + alpha_i * ybuffer[6];
y_ptr += inc_y;
#else
y_ptr[0] += alpha_r * ybuffer[0] + alpha_i * ybuffer[1];
y_ptr[1] -= alpha_r * ybuffer[1] - alpha_i * ybuffer[0];
y_ptr += inc_y;
y_ptr[0] += alpha_r * ybuffer[2] + alpha_i * ybuffer[3];
y_ptr[1] -= alpha_r * ybuffer[3] - alpha_i * ybuffer[2];
y_ptr += inc_y;
y_ptr[0] += alpha_r * ybuffer[4] + alpha_i * ybuffer[5];
y_ptr[1] -= alpha_r * ybuffer[5] - alpha_i * ybuffer[4];
y_ptr += inc_y;
y_ptr[0] += alpha_r * ybuffer[6] + alpha_i * ybuffer[7];
y_ptr[1] -= alpha_r * ybuffer[7] - alpha_i * ybuffer[6];
y_ptr += inc_y;
#endif
}
for( i = 0; i < n2 ; i++)
{
zgemv_kernel_16x1(NB,a_ptr,xbuffer,ybuffer);
a_ptr += 1 * lda;
#if !defined(XCONJ)
y_ptr[0] += alpha_r * ybuffer[0] - alpha_i * ybuffer[1];
y_ptr[1] += alpha_r * ybuffer[1] + alpha_i * ybuffer[0];
y_ptr += inc_y;
#else
y_ptr[0] += alpha_r * ybuffer[0] + alpha_i * ybuffer[1];
y_ptr[1] -= alpha_r * ybuffer[1] - alpha_i * ybuffer[0];
y_ptr += inc_y;
#endif
}
a += 2* NB;
x += NB * inc_x;
}
BLASLONG m3 = m % 16;
if ( m3 == 0 ) return(0);
x_ptr = x;
copy_x(m3,x_ptr,xbuffer,inc_x);
j=0;
a_ptr = a;
y_ptr = y;
while ( j < n)
{
FLOAT temp_r = 0.0;
FLOAT temp_i = 0.0;
for( i = 0; i < m3*2; i+=2 )
{
#if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )
temp_r += a_ptr[i] * xbuffer[i] - a_ptr[i+1] * xbuffer[i+1];
temp_i += a_ptr[i] * xbuffer[i+1] + a_ptr[i+1] * xbuffer[i];
#else
temp_r += a_ptr[i] * xbuffer[i] + a_ptr[i+1] * xbuffer[i+1];
temp_i += a_ptr[i] * xbuffer[i+1] - a_ptr[i+1] * xbuffer[i];
#endif
}
a_ptr += lda;
#if !defined(XCONJ)
y_ptr[0] += alpha_r * temp_r - alpha_i * temp_i;
y_ptr[1] += alpha_r * temp_i + alpha_i * temp_r;
#else
y_ptr[0] += alpha_r * temp_r + alpha_i * temp_i;
y_ptr[1] -= alpha_r * temp_i - alpha_i * temp_r;
#endif
y_ptr += inc_y;
j++;
}
return(0);
}

180
kernel/x86_64/zgemv_t_microk_bulldozer-2.c Normal file
View File

@ -0,0 +1,180 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary froms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary from must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#define HAVE_KERNEL_16x4 1
static void zgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) __attribute__ ((noinline));
static void zgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)
{
BLASLONG register i = 0;
__asm__ __volatile__
(
"vzeroupper \n\t"
"vxorpd %%xmm8 , %%xmm8 , %%xmm8 \n\t" // temp
"vxorpd %%xmm9 , %%xmm9 , %%xmm9 \n\t" // temp
"vxorpd %%xmm10, %%xmm10, %%xmm10 \n\t" // temp
"vxorpd %%xmm11, %%xmm11, %%xmm11 \n\t" // temp
"vxorpd %%xmm12, %%xmm12, %%xmm12 \n\t" // temp
"vxorpd %%xmm13, %%xmm13, %%xmm13 \n\t"
"vxorpd %%xmm14, %%xmm14, %%xmm14 \n\t"
"vxorpd %%xmm15, %%xmm15, %%xmm15 \n\t"
".align 16 \n\t"
".L01LOOP%=: \n\t"
"vmovddup (%2,%0,8), %%xmm0 \n\t" // real value from x0
"vmovddup 8(%2,%0,8), %%xmm1 \n\t" // imag value from x0
"prefetcht0 192(%4,%0,8) \n\t"
"vmovups (%4,%0,8), %%xmm4 \n\t" // 1 complex values from a0
"prefetcht0 192(%5,%0,8) \n\t"
"vmovups (%5,%0,8), %%xmm5 \n\t" // 1 complex values from a1
"prefetcht0 192(%6,%0,8) \n\t"
"vmovups (%6,%0,8), %%xmm6 \n\t" // 1 complex values from a2
"prefetcht0 192(%7,%0,8) \n\t"
"vmovups (%7,%0,8), %%xmm7 \n\t" // 1 complex values from a3
"vfmaddpd %%xmm8 , %%xmm4 , %%xmm0, %%xmm8 \n\t" // ar0*xr0,al0*xr0
"vfmaddpd %%xmm9 , %%xmm4 , %%xmm1, %%xmm9 \n\t" // ar0*xl0,al0*xl0
"vfmaddpd %%xmm10, %%xmm5 , %%xmm0, %%xmm10 \n\t" // ar0*xr0,al0*xr0
"vfmaddpd %%xmm11, %%xmm5 , %%xmm1, %%xmm11 \n\t" // ar0*xl0,al0*xl0
"vfmaddpd %%xmm12, %%xmm6 , %%xmm0, %%xmm12 \n\t" // ar0*xr0,al0*xr0
"vfmaddpd %%xmm13, %%xmm6 , %%xmm1, %%xmm13 \n\t" // ar0*xl0,al0*xl0
"vfmaddpd %%xmm14, %%xmm7 , %%xmm0, %%xmm14 \n\t" // ar0*xr0,al0*xr0
"vfmaddpd %%xmm15, %%xmm7 , %%xmm1, %%xmm15 \n\t" // ar0*xl0,al0*xl0
"vmovddup 16(%2,%0,8), %%xmm0 \n\t" // real value from x0
"vmovddup 24(%2,%0,8), %%xmm1 \n\t" // imag value from x0
"vmovups 16(%4,%0,8), %%xmm4 \n\t" // 1 complex values from a0
"vmovups 16(%5,%0,8), %%xmm5 \n\t" // 1 complex values from a1
"vmovups 16(%6,%0,8), %%xmm6 \n\t" // 1 complex values from a2
"vmovups 16(%7,%0,8), %%xmm7 \n\t" // 1 complex values from a3
"vfmaddpd %%xmm8 , %%xmm4 , %%xmm0, %%xmm8 \n\t" // ar0*xr0,al0*xr0
"vfmaddpd %%xmm9 , %%xmm4 , %%xmm1, %%xmm9 \n\t" // ar0*xl0,al0*xl0
"vfmaddpd %%xmm10, %%xmm5 , %%xmm0, %%xmm10 \n\t" // ar0*xr0,al0*xr0
"vfmaddpd %%xmm11, %%xmm5 , %%xmm1, %%xmm11 \n\t" // ar0*xl0,al0*xl0
"vfmaddpd %%xmm12, %%xmm6 , %%xmm0, %%xmm12 \n\t" // ar0*xr0,al0*xr0
"vfmaddpd %%xmm13, %%xmm6 , %%xmm1, %%xmm13 \n\t" // ar0*xl0,al0*xl0
"vfmaddpd %%xmm14, %%xmm7 , %%xmm0, %%xmm14 \n\t" // ar0*xr0,al0*xr0
"vfmaddpd %%xmm15, %%xmm7 , %%xmm1, %%xmm15 \n\t" // ar0*xl0,al0*xl0
"vmovddup 32(%2,%0,8), %%xmm0 \n\t" // real value from x0
"vmovddup 40(%2,%0,8), %%xmm1 \n\t" // imag value from x0
"vmovups 32(%4,%0,8), %%xmm4 \n\t" // 1 complex values from a0
"vmovups 32(%5,%0,8), %%xmm5 \n\t" // 1 complex values from a1
"vmovups 32(%6,%0,8), %%xmm6 \n\t" // 1 complex values from a2
"vmovups 32(%7,%0,8), %%xmm7 \n\t" // 1 complex values from a3
"vfmaddpd %%xmm8 , %%xmm4 , %%xmm0, %%xmm8 \n\t" // ar0*xr0,al0*xr0
"vfmaddpd %%xmm9 , %%xmm4 , %%xmm1, %%xmm9 \n\t" // ar0*xl0,al0*xl0
"vfmaddpd %%xmm10, %%xmm5 , %%xmm0, %%xmm10 \n\t" // ar0*xr0,al0*xr0
"vfmaddpd %%xmm11, %%xmm5 , %%xmm1, %%xmm11 \n\t" // ar0*xl0,al0*xl0
"vfmaddpd %%xmm12, %%xmm6 , %%xmm0, %%xmm12 \n\t" // ar0*xr0,al0*xr0
"vfmaddpd %%xmm13, %%xmm6 , %%xmm1, %%xmm13 \n\t" // ar0*xl0,al0*xl0
"vfmaddpd %%xmm14, %%xmm7 , %%xmm0, %%xmm14 \n\t" // ar0*xr0,al0*xr0
"vfmaddpd %%xmm15, %%xmm7 , %%xmm1, %%xmm15 \n\t" // ar0*xl0,al0*xl0
"vmovddup 48(%2,%0,8), %%xmm0 \n\t" // real value from x0
"vmovddup 56(%2,%0,8), %%xmm1 \n\t" // imag value from x0
"vmovups 48(%4,%0,8), %%xmm4 \n\t" // 1 complex values from a0
"vmovups 48(%5,%0,8), %%xmm5 \n\t" // 1 complex values from a1
"vmovups 48(%6,%0,8), %%xmm6 \n\t" // 1 complex values from a2
"vmovups 48(%7,%0,8), %%xmm7 \n\t" // 1 complex values from a3
"vfmaddpd %%xmm8 , %%xmm4 , %%xmm0, %%xmm8 \n\t" // ar0*xr0,al0*xr0
"vfmaddpd %%xmm9 , %%xmm4 , %%xmm1, %%xmm9 \n\t" // ar0*xl0,al0*xl0
"vfmaddpd %%xmm10, %%xmm5 , %%xmm0, %%xmm10 \n\t" // ar0*xr0,al0*xr0
"vfmaddpd %%xmm11, %%xmm5 , %%xmm1, %%xmm11 \n\t" // ar0*xl0,al0*xl0
"vfmaddpd %%xmm12, %%xmm6 , %%xmm0, %%xmm12 \n\t" // ar0*xr0,al0*xr0
"vfmaddpd %%xmm13, %%xmm6 , %%xmm1, %%xmm13 \n\t" // ar0*xl0,al0*xl0
"vfmaddpd %%xmm14, %%xmm7 , %%xmm0, %%xmm14 \n\t" // ar0*xr0,al0*xr0
"vfmaddpd %%xmm15, %%xmm7 , %%xmm1, %%xmm15 \n\t" // ar0*xl0,al0*xl0
"addq $8 , %0 \n\t"
"subq $4 , %1 \n\t"
"jnz .L01LOOP%= \n\t"
#if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )
"vpermilpd $0x1 , %%xmm9 , %%xmm9 \n\t"
"vpermilpd $0x1 , %%xmm11, %%xmm11 \n\t"
"vpermilpd $0x1 , %%xmm13, %%xmm13 \n\t"
"vpermilpd $0x1 , %%xmm15, %%xmm15 \n\t"
"vaddsubpd %%xmm9 , %%xmm8, %%xmm8 \n\t"
"vaddsubpd %%xmm11, %%xmm10, %%xmm10 \n\t"
"vaddsubpd %%xmm13, %%xmm12, %%xmm12 \n\t"
"vaddsubpd %%xmm15, %%xmm14, %%xmm14 \n\t"
#else
"vpermilpd $0x1 , %%xmm8 , %%xmm8 \n\t"
"vpermilpd $0x1 , %%xmm10, %%xmm10 \n\t"
"vpermilpd $0x1 , %%xmm12, %%xmm12 \n\t"
"vpermilpd $0x1 , %%xmm14, %%xmm14 \n\t"
"vaddsubpd %%xmm8 , %%xmm9 , %%xmm8 \n\t"
"vaddsubpd %%xmm10, %%xmm11, %%xmm10 \n\t"
"vaddsubpd %%xmm12, %%xmm13, %%xmm12 \n\t"
"vaddsubpd %%xmm14, %%xmm15, %%xmm14 \n\t"
"vpermilpd $0x1 , %%xmm8 , %%xmm8 \n\t"
"vpermilpd $0x1 , %%xmm10, %%xmm10 \n\t"
"vpermilpd $0x1 , %%xmm12, %%xmm12 \n\t"
"vpermilpd $0x1 , %%xmm14, %%xmm14 \n\t"
#endif
"vmovups %%xmm8 , (%3) \n\t"
"vmovups %%xmm10, 16(%3) \n\t"
"vmovups %%xmm12, 32(%3) \n\t"
"vmovups %%xmm14, 48(%3) \n\t"
"vzeroupper \n\t"
:
:
"r" (i), // 0
"r" (n), // 1
"r" (x), // 2
"r" (y), // 3
"r" (ap[0]), // 4
"r" (ap[1]), // 5
"r" (ap[2]), // 6
"r" (ap[3]) // 7
: "cc",
"%xmm0", "%xmm1", "%xmm2", "%xmm3",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}

162
kernel/x86_64/zgemv_t_microk_haswell-2.c Normal file
View File

@ -0,0 +1,162 @@
/***************************************************************************
Copyright (c) 2014, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary froms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary from must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
#define HAVE_KERNEL_16x4 1
static void zgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) __attribute__ ((noinline));
static void zgemv_kernel_16x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)
{
BLASLONG register i = 0;
__asm__ __volatile__
(
"vzeroupper \n\t"
"vxorpd %%ymm8 , %%ymm8 , %%ymm8 \n\t" // temp
"vxorpd %%ymm9 , %%ymm9 , %%ymm9 \n\t" // temp
"vxorpd %%ymm10, %%ymm10, %%ymm10 \n\t" // temp
"vxorpd %%ymm11, %%ymm11, %%ymm11 \n\t" // temp
"vxorpd %%ymm12, %%ymm12, %%ymm12 \n\t" // temp
"vxorpd %%ymm13, %%ymm13, %%ymm13 \n\t"
"vxorpd %%ymm14, %%ymm14, %%ymm14 \n\t"
"vxorpd %%ymm15, %%ymm15, %%ymm15 \n\t"
".align 16 \n\t"
".L01LOOP%=: \n\t"
"prefetcht0 192(%2,%0,8) \n\t"
"vmovddup (%2,%0,8), %%xmm0 \n\t" // real value from x0
"prefetcht0 192(%4,%0,8) \n\t"
"vmovups (%5,%0,8), %%ymm5 \n\t" // 2 complex values from a1
"vmovddup 8(%2,%0,8), %%xmm1 \n\t" // imag value from x0
"vmovups (%4,%0,8), %%ymm4 \n\t" // 2 complex values from a0
"prefetcht0 192(%5,%0,8) \n\t"
"vmovddup 16(%2,%0,8), %%xmm2 \n\t" // real value from x1
"prefetcht0 192(%6,%0,8) \n\t"
"vmovups (%6,%0,8), %%ymm6 \n\t" // 2 complex values from a2
"vmovddup 24(%2,%0,8), %%xmm3 \n\t" // imag value from x1
"prefetcht0 192(%7,%0,8) \n\t"
"vmovups (%7,%0,8), %%ymm7 \n\t" // 2 complex values from a3
"vinsertf128 $1, %%xmm2, %%ymm0 , %%ymm0 \n\t" // real values from x0 and x1
"vinsertf128 $1, %%xmm3, %%ymm1 , %%ymm1 \n\t" // imag values from x0 and x1
"vfmadd231pd %%ymm4 , %%ymm0, %%ymm8 \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1
"vfmadd231pd %%ymm4 , %%ymm1, %%ymm9 \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1
"vfmadd231pd %%ymm5 , %%ymm0, %%ymm10 \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1
"vfmadd231pd %%ymm5 , %%ymm1, %%ymm11 \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1
"vfmadd231pd %%ymm6 , %%ymm0, %%ymm12 \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1
"vfmadd231pd %%ymm6 , %%ymm1, %%ymm13 \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1
"vfmadd231pd %%ymm7 , %%ymm0, %%ymm14 \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1
"vfmadd231pd %%ymm7 , %%ymm1, %%ymm15 \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1
"vmovups 32(%4,%0,8), %%ymm4 \n\t" // 2 complex values from a0
"vmovups 32(%5,%0,8), %%ymm5 \n\t" // 2 complex values from a1
"vmovddup 32(%2,%0,8), %%xmm0 \n\t" // real value from x0
"vmovddup 40(%2,%0,8), %%xmm1 \n\t" // imag value from x0
"vmovddup 48(%2,%0,8), %%xmm2 \n\t" // real value from x1
"vmovddup 56(%2,%0,8), %%xmm3 \n\t" // imag value from x1
"vmovups 32(%6,%0,8), %%ymm6 \n\t" // 2 complex values from a2
"vmovups 32(%7,%0,8), %%ymm7 \n\t" // 2 complex values from a3
"vinsertf128 $1, %%xmm2, %%ymm0 , %%ymm0 \n\t" // real values from x0 and x1
"vinsertf128 $1, %%xmm3, %%ymm1 , %%ymm1 \n\t" // imag values from x0 and x1
"vfmadd231pd %%ymm4 , %%ymm0, %%ymm8 \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1
"vfmadd231pd %%ymm4 , %%ymm1, %%ymm9 \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1
"vfmadd231pd %%ymm5 , %%ymm0, %%ymm10 \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1
"vfmadd231pd %%ymm5 , %%ymm1, %%ymm11 \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1
"vfmadd231pd %%ymm6 , %%ymm0, %%ymm12 \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1
"vfmadd231pd %%ymm6 , %%ymm1, %%ymm13 \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1
"vfmadd231pd %%ymm7 , %%ymm0, %%ymm14 \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1
"vfmadd231pd %%ymm7 , %%ymm1, %%ymm15 \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1
"addq $8 , %0 \n\t"
"subq $4 , %1 \n\t"
"jnz .L01LOOP%= \n\t"
#if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )
"vpermilpd $0x5 , %%ymm9 , %%ymm9 \n\t"
"vpermilpd $0x5 , %%ymm11, %%ymm11 \n\t"
"vpermilpd $0x5 , %%ymm13, %%ymm13 \n\t"
"vpermilpd $0x5 , %%ymm15, %%ymm15 \n\t"
"vaddsubpd %%ymm9 , %%ymm8, %%ymm8 \n\t"
"vaddsubpd %%ymm11, %%ymm10, %%ymm10 \n\t"
"vaddsubpd %%ymm13, %%ymm12, %%ymm12 \n\t"
"vaddsubpd %%ymm15, %%ymm14, %%ymm14 \n\t"
#else
"vpermilpd $0x5 , %%ymm8 , %%ymm8 \n\t"
"vpermilpd $0x5 , %%ymm10, %%ymm10 \n\t"
"vpermilpd $0x5 , %%ymm12, %%ymm12 \n\t"
"vpermilpd $0x5 , %%ymm14, %%ymm14 \n\t"
"vaddsubpd %%ymm8 , %%ymm9 , %%ymm8 \n\t"
"vaddsubpd %%ymm10, %%ymm11, %%ymm10 \n\t"
"vaddsubpd %%ymm12, %%ymm13, %%ymm12 \n\t"
"vaddsubpd %%ymm14, %%ymm15, %%ymm14 \n\t"
"vpermilpd $0x5 , %%ymm8 , %%ymm8 \n\t"
"vpermilpd $0x5 , %%ymm10, %%ymm10 \n\t"
"vpermilpd $0x5 , %%ymm12, %%ymm12 \n\t"
"vpermilpd $0x5 , %%ymm14, %%ymm14 \n\t"
#endif
"vextractf128 $1, %%ymm8 , %%xmm9 \n\t"
"vextractf128 $1, %%ymm10, %%xmm11 \n\t"
"vextractf128 $1, %%ymm12, %%xmm13 \n\t"
"vextractf128 $1, %%ymm14, %%xmm15 \n\t"
"vaddpd %%xmm8 , %%xmm9 , %%xmm8 \n\t"
"vaddpd %%xmm10, %%xmm11, %%xmm10 \n\t"
"vaddpd %%xmm12, %%xmm13, %%xmm12 \n\t"
"vaddpd %%xmm14, %%xmm15, %%xmm14 \n\t"
"vmovups %%xmm8 , (%3) \n\t"
"vmovups %%xmm10, 16(%3) \n\t"
"vmovups %%xmm12, 32(%3) \n\t"
"vmovups %%xmm14, 48(%3) \n\t"
"vzeroupper \n\t"
:
:
"r" (i), // 0
"r" (n), // 1
"r" (x), // 2
"r" (y), // 3
"r" (ap[0]), // 4
"r" (ap[1]), // 5
"r" (ap[2]), // 6
"r" (ap[3]) // 7
: "cc",
"%xmm0", "%xmm1", "%xmm2", "%xmm3",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%xmm8", "%xmm9", "%xmm10", "%xmm11",
"%xmm12", "%xmm13", "%xmm14", "%xmm15",
"memory"
);
}

4
lapack-netlib/TESTING/dstest.in
View File

@ -1,8 +1,8 @@
Data file for testing DSGESV/DSPOSV LAPACK routines
12 Number of values of M
0 1 2 13 17 45 78 91 101 119 120 132 values of M (row dimension)
4 Number of values of NRHS
1 2 14 16 Values of NRHS (number of right hand sides)
6 Number of values of NRHS
1 2 14 15 16 13 Values of NRHS (number of right hand sides)
30.0 Threshold value of test ratio
T Put T to test the driver routine
T Put T to test the error exits

12
param.h
View File

@ -1134,9 +1134,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define QGEMM_DEFAULT_P 504
#define QGEMM_DEFAULT_R qgemm_r
#define CGEMM_DEFAULT_P 384
//#define CGEMM_DEFAULT_R cgemm_r
#define CGEMM_DEFAULT_R 1024
#define CGEMM_DEFAULT_P 768
#define CGEMM_DEFAULT_R cgemm_r
//#define CGEMM_DEFAULT_R 1024
#define ZGEMM_DEFAULT_P 512
#define ZGEMM_DEFAULT_R zgemm_r
@ -1148,7 +1148,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define SGEMM_DEFAULT_Q 384
#define DGEMM_DEFAULT_Q 256
#define QGEMM_DEFAULT_Q 128
#define CGEMM_DEFAULT_Q 192
#define CGEMM_DEFAULT_Q 512
#define ZGEMM_DEFAULT_Q 192
#define XGEMM_DEFAULT_Q 128
@ -1206,6 +1206,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define ZGEMM_DEFAULT_UNROLL_N 2
#define XGEMM_DEFAULT_UNROLL_N 1
#define DGEMM_DEFAULT_UNROLL_MN 16
#endif
#ifdef ARCH_X86
@ -1236,10 +1237,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define CGEMM_DEFAULT_P 384
#define ZGEMM_DEFAULT_P 256
#define SGEMM_DEFAULT_Q 384
#ifdef WINDOWS_ABI
#define SGEMM_DEFAULT_Q 320
#define DGEMM_DEFAULT_Q 128
#else
#define SGEMM_DEFAULT_Q 384
#define DGEMM_DEFAULT_Q 256
#endif
#define CGEMM_DEFAULT_Q 192