Merge pull request #4801 from markdryan/markdryan/riscv-dynamic-arch

Add autodetection for riscv64

.github/workflows/riscv64_vector.yml

@@ -28,6 +28,9 @@ jobs:
           - target: RISCV64_ZVL256B
             opts: TARGET=RISCV64_ZVL256B BINARY=64 ARCH=riscv64
             qemu_cpu: rv64,g=true,c=true,v=true,vext_spec=v1.0,vlen=256,elen=64
+          - target: DYNAMIC_ARCH=1
+            opts: TARGET=RISCV64_GENERIC BINARY=64 ARCH=riscv64 DYNAMIC_ARCH=1
+            qemu_cpu: rv64,g=true,c=true,v=true,vext_spec=v1.0,vlen=256,elen=64
 
     steps:
       - name: Checkout repository

Makefile.system

@@ -715,6 +715,17 @@ ifeq ($(ARCH), loongarch64)
 DYNAMIC_CORE = LOONGSON3R5 LOONGSON2K1000 LOONGSONGENERIC
 endif
 
+ifeq ($(ARCH), riscv64)
+DYNAMIC_CORE = RISCV64_GENERIC
+DYNAMIC_CORE += RISCV64_ZVL128B
+DYNAMIC_CORE += RISCV64_ZVL256B
+ifdef DYNAMIC_LIST
+override DYNAMIC_CORE = RISCV64_GENERIC $(DYNAMIC_LIST)
+XCCOMMON_OPT = -DDYNAMIC_LIST -DDYN_RISCV64_GENERIC
+XCCOMMON_OPT += $(foreach dcore,$(DYNAMIC_LIST),-DDYN_$(dcore))
+endif
+endif
+
 ifeq ($(ARCH), zarch)
 DYNAMIC_CORE = ZARCH_GENERIC
 

README.md

@@ -234,6 +234,8 @@ For **POWER**, the list encompasses POWER6, POWER8 and POWER9. POWER10 is additi
 
 on **ZARCH** it comprises Z13 and Z14 as well as generic zarch support.
 
+On **riscv64**, DYNAMIC_ARCH enables support for riscv64_zvl128b and riscv64_zvl256b in addition to generic riscv64 support.  A compiler that supports RVV 1.0 is required to build OpenBLAS for riscv64 when DYNAMIC_ARCH is enabled.
+
 The `TARGET` option can be used in conjunction with `DYNAMIC_ARCH=1` to specify which cpu model should be assumed for all the
 common code in the library, usually you will want to set this to the oldest model you expect to encounter.
 Please note that it is not possible to combine support for different architectures, so no combined 32 and 64 bit or x86_64 and arm64 in the same library.

driver/others/Makefile

@@ -30,12 +30,16 @@ else
 ifeq ($(ARCH),loongarch64)
 COMMONOBJS += dynamic_loongarch64.$(SUFFIX)
 else
+ifeq ($(ARCH),riscv64)
+COMMONOBJS += dynamic_riscv64.$(SUFFIX) detect_riscv64.$(SUFFIX)
+else
 COMMONOBJS	+=  dynamic.$(SUFFIX)
 endif
 endif
 endif
 endif
 endif
+endif
 else
 COMMONOBJS	+=  parameter.$(SUFFIX)
 endif
@@ -106,12 +110,16 @@ else
 ifeq ($(ARCH),loongarch64)
 HPLOBJS = memory.$(SUFFIX) xerbla.$(SUFFIX) dynamic_loongarch64.$(SUFFIX)
 else
+ifeq ($(ARCH),riscv64)
+HPLOBJS = memory.$(SUFFIX) xerbla.$(SUFFIX) dynamic_riscv64.$(SUFFIX) detect_riscv64.$(SUFFIX)
+else
 HPLOBJS = memory.$(SUFFIX) xerbla.$(SUFFIX) dynamic.$(SUFFIX)
 endif
 endif
 endif
 endif
 endif
+endif
 else
 HPLOBJS = memory.$(SUFFIX) xerbla.$(SUFFIX) parameter.$(SUFFIX)
 endif
@@ -209,6 +217,9 @@ addx.$(SUFFIX) : $(ARCH)/addx.c
 mulx.$(SUFFIX) : $(ARCH)/mulx.c
 	$(CC) $(CFLAGS) -c -DXDOUBLE -UCOMPLEX $< -o $(@F)
 
+detect_riscv64.$(SUFFIX): detect_riscv64.c
+	$(CC) $(CFLAGS) -c -march=rv64imafdcv $< -o $(@F)
+
 xerbla.$(PSUFFIX) : xerbla.c
 	$(CC) $(PFLAGS) -c $< -o $(@F)
 

driver/others/detect_riscv64.c

@@ -0,0 +1,75 @@
+/*****************************************************************************
+Copyright (c) 2024, The OpenBLAS Project
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   1. Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the
+      distribution.
+   3. Neither the name of the OpenBLAS project nor the names of
+      its contributors may be used to endorse or promote products
+      derived from this software without specific prior written
+      permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+**********************************************************************************/
+
+#include <stdint.h>
+
+#ifdef __riscv_v_intrinsic
+#include <riscv_vector.h>
+#endif
+
+unsigned detect_riscv64_get_vlenb(void) {
+#ifdef __riscv_v_intrinsic
+	return __riscv_vlenb();
+#else
+	return 0;
+#endif
+}
+
+/*
+ * Based on the approach taken here:
+ * https://code.videolan.org/videolan/dav1d/-/merge_requests/1629
+ *
+ * Only to be called after we've determined we have some sort of
+ * RVV support.
+ */
+
+uint64_t detect_riscv64_rvv100(void)
+{
+	uint64_t rvv10_supported;
+
+	/*
+	 * After the vsetvli statement vtype will either be a value > 0 if the
+	 * vsetvli succeeded or less than 0 if it failed.  If 0 < vtype
+	 * we're good and the function will return 1, otherwise there's no
+	 * RVV 1.0 and we return 0.
+	 */
+
+	asm volatile("vsetvli x0, x0, e8, m1, ta, ma\n\t"
+		     "csrr %0, vtype\n\t"
+		     "slt %0, x0, %0\n"
+		     : "=r" (rvv10_supported)
+		     :
+		     :);
+
+	return rvv10_supported;
+}
+

driver/others/dynamic_riscv64.c

@@ -0,0 +1,269 @@
+/*****************************************************************************
+Copyright (c) 2024, The OpenBLAS Project
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   1. Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the
+      distribution.
+   3. Neither the name of the OpenBLAS project nor the names of
+      its contributors may be used to endorse or promote products
+      derived from this software without specific prior written
+      permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+**********************************************************************************/
+
+#include <stdbool.h>
+
+#include "common.h"
+
+/*
+ * OpenBLAS contains some kernels that are optimised for RVV 1.0.  Before we
+ * can use these kernels we need to determine whether the device supports
+ * RVV 1.0 and what the device's VLEN is.  Our strategy will be as follows.
+ *
+ * First we'll invoke the hwprobe syscall to detect RVV 1.0.  In an ideal world,
+ * this is all we should need to do.  If the syscall is not implemented we
+ * should be able to deduce that RVV 1.0 is not supported (as it was added to
+ * Linux after hwprobe) and if the syscall is implemented we can use it to
+ * determine whether RVV 1.0 is supported.  However, there are some riscv64
+ * boards out there that implement RVV 1.0 but ship with a Linux kernel that
+ * predates RVV vector support and hwprobe support.  These kernels contain
+ * the backported RVV patches but not the hwprobe patches and so they
+ * advertise support for RVV via hwcap.  To cater for these boards we need
+ * to fall back to hwcap if hwprobe is not supported. Unfortunately, some
+ * boards indicate support for RVV via hwcap even though they only support
+ * RVV 0.7.1, which is incompatible with RVV 1.0.  So an additional check is
+ * required to test if the devices advertising support for RVV via hwcap really
+ * support RVV 1.0.  This test works by executing a vsetvli instruction that
+ * sets the tail agnostic and mask agnostic bits in the vtype register.
+ * These bits are not supported prior to RVV 0.9 so will cause the VIL bit to
+ * be set on the VTYPE register in CPUs supporting 0.7.1.  If this bit is set
+ * we can determine that RVV 1.0 is not supported.
+ *
+ * This approach is borrowed from
+ * VideoLan dav1d:
+ *   (https://code.videolan.org/videolan/dav1d/-/merge_requests/1629).
+ *
+ * We assume that if a kernel reports the presence of RVV via hwcap that
+ * the device supports the vsetvli instruction.
+ *
+ * For now we're just going to invoke the hwprobe syscall directly, rather than
+ * invoking it through glibc.  Support for hwprobe has been added to glibc but
+ * at the time of writing this support has not yet been included in a glibc
+ * release.  Once it has, it will be better to invoke hwprobe via glibc as doing
+ * so should take advantage of the vdso entry and be more efficient.
+ */
+
+/*
+ * This should work on Android as well but I have no way of testing.
+ */
+
+#if defined(OS_LINUX)
+#include <unistd.h>
+#include <sys/syscall.h>
+#include <stdint.h>
+#include <sys/auxv.h>
+
+#define DETECT_RISCV64_HWCAP_ISA_V (1 << ('V' - 'A'))
+
+struct riscv_hwprobe {
+	int64_t key;
+	uint64_t value;
+};
+
+/* The constants below are copied from
+ * /usr/include/riscv64-linux-gnu/asm/hwprobe.h. We duplicate the
+ *  constants as the header file from which they are copied will only
+ *  be present if we're building on a device with Linux 6.5 or greater.
+ */
+
+#define RISCV_HWPROBE_KEY_IMA_EXT_0	4
+#define		RISCV_HWPROBE_IMA_V		(1 << 2)
+
+#ifndef NR_riscv_hwprobe
+#ifndef NR_arch_specific_syscall
+#define NR_arch_specific_syscall 244
+#endif
+#define NR_riscv_hwprobe (NR_arch_specific_syscall + 14)
+#endif
+#endif // defined(OS_LINUX)
+
+unsigned detect_riscv64_get_vlenb(void);
+uint64_t detect_riscv64_rvv100(void);
+
+extern gotoblas_t gotoblas_RISCV64_GENERIC;
+#if !defined(DYNAMIC_LIST) || defined(DYN_RISCV64_ZVL256B)
+extern gotoblas_t gotoblas_RISCV64_ZVL256B;
+#endif
+#if !defined(DYNAMIC_LIST) || defined(DYN_RISCV64_ZVL128B)
+extern gotoblas_t gotoblas_RISCV64_ZVL128B;
+#endif
+
+#define CPU_GENERIC         0
+#define CPU_RISCV64_ZVL256B 1
+#define CPU_RISCV64_ZVL128B 2
+
+static char *cpuname[] = {
+	"riscv64_generic",
+	"riscv64_zvl256b",
+	"riscv64_zvl128b"
+};
+#define NUM_CORETYPES (sizeof(cpuname)/sizeof(char*))
+
+extern int openblas_verbose(void);
+extern void openblas_warning(int verbose, const char* msg);
+
+char* gotoblas_corename(void) {
+#if !defined(DYNAMIC_LIST) || defined(DYN_RISCV64_ZVL256B)
+	if (gotoblas == &gotoblas_RISCV64_ZVL256B)
+		return cpuname[CPU_RISCV64_ZVL256B];
+#endif
+#if !defined(DYNAMIC_LIST) || defined(DYN_RISCV64_ZVL128B)
+	if (gotoblas == &gotoblas_RISCV64_ZVL128B)
+		return cpuname[CPU_RISCV64_ZVL128B];
+#endif
+	if (gotoblas == &gotoblas_RISCV64_GENERIC)
+		return cpuname[CPU_GENERIC];
+
+	return "unknown";
+}
+
+static gotoblas_t* get_coretype(void) {
+	unsigned vlenb = 0;
+
+#if !defined(OS_LINUX)
+	return NULL;
+#else
+
+	/*
+	 * See the hwprobe documentation
+	 *
+	 * ( https://docs.kernel.org/arch/riscv/hwprobe.html )
+	 * for more details.
+	 */
+
+	struct riscv_hwprobe pairs[] = {
+		{ .key = RISCV_HWPROBE_KEY_IMA_EXT_0, },
+	};
+	int ret = syscall(NR_riscv_hwprobe, pairs, 1, 0, NULL, 0);
+	if (ret == 0) {
+		if (!(pairs[0].value & RISCV_HWPROBE_IMA_V))
+			return NULL;
+	} else {
+		if (!(getauxval(AT_HWCAP) & DETECT_RISCV64_HWCAP_ISA_V))
+			return NULL;
+
+		if (!detect_riscv64_rvv100())
+			return NULL;
+	}
+
+	/*
+	 * RVV 1.0 is supported.  We now just need to determine the coretype
+	 * based on the VLEN.
+	 */
+
+	vlenb = detect_riscv64_get_vlenb();
+
+	if (vlenb < 16)
+		return NULL;
+#if !defined(DYNAMIC_LIST) || defined(DYN_RISCV64_ZVL256B)
+	if (vlenb >= 32)
+		return &gotoblas_RISCV64_ZVL256B;
+#endif
+
+#if !defined(DYNAMIC_LIST) || defined(DYN_RISCV64_ZVL128B)
+	return &gotoblas_RISCV64_ZVL128B;
+#else
+	return NULL;
+#endif
+
+#endif  // !defined(OS_LINUX)
+}
+
+static gotoblas_t* force_coretype(char* coretype) {
+	size_t i;
+	char message[128];
+
+	for (i = 0; i < NUM_CORETYPES && strcasecmp(coretype, cpuname[i]); i++);
+
+	if (i == CPU_GENERIC)
+		return &gotoblas_RISCV64_GENERIC;
+
+	if (i == CPU_RISCV64_ZVL256B) {
+#if !defined(DYNAMIC_LIST) || defined(DYN_RISCV64_ZVL256B)
+		return &gotoblas_RISCV64_ZVL256B;
+#else
+		openblas_warning(1,
+				 "riscv64_zvl256b support not compiled in\n");
+		return NULL;
+#endif
+	}
+
+	if (i == CPU_RISCV64_ZVL128B) {
+#if !defined(DYNAMIC_LIST) || defined(DYN_RISCV64_ZVL128B)
+		return &gotoblas_RISCV64_ZVL128B;
+#else
+		openblas_warning(1,
+				 "riscv64_zvl128b support not compiled in\n");
+		return NULL;
+#endif
+	}
+
+	snprintf(message, sizeof(message), "Core not found: %s\n", coretype);
+	openblas_warning(1, message);
+
+	return NULL;
+}
+
+void gotoblas_dynamic_init(void) {
+
+	char coremsg[128];
+	char* p;
+
+	if (gotoblas) return;
+
+	p = getenv("OPENBLAS_CORETYPE");
+	if (p)
+		gotoblas = force_coretype(p);
+	else
+		gotoblas = get_coretype();
+
+	if (!gotoblas) {
+		snprintf(coremsg, sizeof(coremsg), "Falling back to generic riscv64 core\n");
+		openblas_warning(1, coremsg);
+		gotoblas = &gotoblas_RISCV64_GENERIC;
+	}
+
+	if (gotoblas->init) {
+		snprintf(coremsg, sizeof(coremsg), "Core: %s\n",
+			 gotoblas_corename());
+		openblas_warning(2, coremsg);
+		gotoblas->init();
+		return;
+	}
+
+	openblas_warning(0, "OpenBLAS : Architecture Initialization failed. No initialization function found.\n");
+	exit(1);
+}
+
+void gotoblas_dynamic_quit(void) {
+	gotoblas = NULL;
+}

kernel/riscv64/KERNEL.RISCV64_GENERIC

@@ -99,26 +99,26 @@ ZTRMMKERNEL	= ../generic/ztrmmkernel_2x2.c
 SGEMMKERNEL    =  ../generic/gemmkernel_2x2.c
 SGEMMONCOPY    =  ../generic/gemm_ncopy_2.c
 SGEMMOTCOPY    =  ../generic/gemm_tcopy_2.c
-SGEMMONCOPYOBJ =  sgemm_oncopy.o
+SGEMMONCOPYOBJ =  sgemm_oncopy$(TSUFFIX).$(SUFFIX)
-SGEMMOTCOPYOBJ =  sgemm_otcopy.o
+SGEMMOTCOPYOBJ =  sgemm_otcopy$(TSUFFIX).$(SUFFIX)
 
 DGEMMKERNEL    =  ../generic/gemmkernel_2x2.c
 DGEMMONCOPY    = ../generic/gemm_ncopy_2.c
 DGEMMOTCOPY    = ../generic/gemm_tcopy_2.c
-DGEMMONCOPYOBJ = dgemm_oncopy.o
+DGEMMONCOPYOBJ = dgemm_oncopy$(TSUFFIX).$(SUFFIX)
-DGEMMOTCOPYOBJ = dgemm_otcopy.o
+DGEMMOTCOPYOBJ = dgemm_otcopy$(TSUFFIX).$(SUFFIX)
 
 CGEMMKERNEL    = ../generic/zgemmkernel_2x2.c
 CGEMMONCOPY    = ../generic/zgemm_ncopy_2.c
 CGEMMOTCOPY    = ../generic/zgemm_tcopy_2.c
-CGEMMONCOPYOBJ =  cgemm_oncopy.o
+CGEMMONCOPYOBJ =  cgemm_oncopy$(TSUFFIX).$(SUFFIX)
-CGEMMOTCOPYOBJ =  cgemm_otcopy.o
+CGEMMOTCOPYOBJ =  cgemm_otcopy$(TSUFFIX).$(SUFFIX)
 
 ZGEMMKERNEL    = ../generic/zgemmkernel_2x2.c
 ZGEMMONCOPY    = ../generic/zgemm_ncopy_2.c
 ZGEMMOTCOPY    = ../generic/zgemm_tcopy_2.c
-ZGEMMONCOPYOBJ =  zgemm_oncopy.o
+ZGEMMONCOPYOBJ =  zgemm_oncopy$(TSUFFIX).$(SUFFIX)
-ZGEMMOTCOPYOBJ =  zgemm_otcopy.o
+ZGEMMOTCOPYOBJ =  zgemm_otcopy$(TSUFFIX).$(SUFFIX)
 
 STRSMKERNEL_LN	=  ../generic/trsm_kernel_LN.c
 STRSMKERNEL_LT	=  ../generic/trsm_kernel_LT.c

kernel/setparam-ref.c

@@ -1244,6 +1244,36 @@ static void init_parameter(void) {
 }
 #else //ZARCH
 
+#if (ARCH_RISCV64)
+static void init_parameter(void) {
+
+#ifdef BUILD_BFLOAT16
+  TABLE_NAME.sbgemm_p = SBGEMM_DEFAULT_P;
+#endif
+  TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;
+  TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;
+  TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P;
+  TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P;
+
+#ifdef BUILD_BFLOAT16
+  TABLE_NAME.sbgemm_r = SBGEMM_DEFAULT_R;
+#endif
+  TABLE_NAME.sgemm_r = SGEMM_DEFAULT_R;
+  TABLE_NAME.dgemm_r = DGEMM_DEFAULT_R;
+  TABLE_NAME.cgemm_r = CGEMM_DEFAULT_R;
+  TABLE_NAME.zgemm_r = ZGEMM_DEFAULT_R;
+
+
+#ifdef BUILD_BFLOAT16
+  TABLE_NAME.sbgemm_q = SBGEMM_DEFAULT_Q;
+#endif
+  TABLE_NAME.sgemm_q = SGEMM_DEFAULT_Q;
+  TABLE_NAME.dgemm_q = DGEMM_DEFAULT_Q;
+  TABLE_NAME.cgemm_q = CGEMM_DEFAULT_Q;
+  TABLE_NAME.zgemm_q = ZGEMM_DEFAULT_Q;
+}
+#else //RISCV64
+
 #ifdef ARCH_X86
 static int get_l2_size_old(void){
   int i, eax, ebx, ecx, edx, cpuid_level;
@@ -2046,6 +2076,7 @@ static void init_parameter(void) {
 
 
 }
+#endif //RISCV64
 #endif //POWER
 #endif //ZARCH
 #endif //(ARCH_LOONGARCH64)

lapack/laswp/riscv64/Makefile

@@ -1,6 +1,11 @@
 TOPDIR	= ../../..
 include ../../../Makefile.system
 
+ifeq ($(DYNAMIC_ARCH), 1)
+LASWP	= ../generic/laswp_k_4.c
+ZLASWP	= ../generic/zlaswp_k_4.c
+endif
+
 ifndef LASWP
 LASWP	= ../generic/laswp_k.c
 endif