* update intrinsics to match latest spec at https://github.com/riscv-non-isa/rvv-intrinsic-doc (in particular, __riscv_ prefixes for rvv intrinsics)
* fix multiple numerical stability and corner case issues * add a script to generate arbitrary gemm kernel shapes * add a generic zvl256b target to demonstrate large gemm kernel unrolls
This commit is contained in:
Sergei Lewis
+133
-246
@@ -26,264 +26,151 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*****************************************************************************/
|
||||
|
||||
#include "common.h"
|
||||
#if !defined(DOUBLE)
|
||||
#define VSETVL(n) vsetvl_e32m4(n)
|
||||
#define VSETVL_MAX vsetvlmax_e32m1()
|
||||
#define FLOAT_V_T vfloat32m4_t
|
||||
#define FLOAT_V_T_M1 vfloat32m1_t
|
||||
#define VFMVFS_FLOAT vfmv_f_s_f32m1_f32
|
||||
#define VLEV_FLOAT vle32_v_f32m4
|
||||
#define VLSEV_FLOAT vlse32_v_f32m4
|
||||
#define VFREDSUM_FLOAT vfredusum_vs_f32m4_f32m1
|
||||
#define VFMACCVV_FLOAT vfmacc_vv_f32m4
|
||||
#define VFMVVF_FLOAT vfmv_v_f_f32m4
|
||||
#define VFMVVF_FLOAT_M1 vfmv_v_f_f32m1
|
||||
#define VFDOTVV_FLOAT vfdot_vv_f32m4
|
||||
#define ABS fabsf
|
||||
#define MASK_T vbool8_t
|
||||
#define VFRSUBVF_MASK_FLOAT vfrsub_vf_f32m4_m
|
||||
#define VMFGTVF_FLOAT vmfgt_vf_f32m4_b8
|
||||
#define VMFIRSTM vmfirst_m_b8
|
||||
#define VFDIVVF_FLOAT vfdiv_vf_f32m4
|
||||
#define VMFLTVF_FLOAT vmflt_vf_f32m4_b8
|
||||
#define VFREDMAXVS_FLOAT vfredmax_vs_f32m4_f32m1
|
||||
|
||||
#ifdef RISCV64_ZVL256B
|
||||
# define LMUL m1
|
||||
# if defined(DOUBLE)
|
||||
# define ELEN 64
|
||||
# define MLEN 64
|
||||
# else
|
||||
# define ELEN 32
|
||||
# define MLEN 32
|
||||
# endif
|
||||
#else
|
||||
#define VSETVL(n) vsetvl_e64m4(n)
|
||||
#define VSETVL_MAX vsetvlmax_e64m1()
|
||||
#define FLOAT_V_T vfloat64m4_t
|
||||
#define FLOAT_V_T_M1 vfloat64m1_t
|
||||
#define VFMVFS_FLOAT vfmv_f_s_f64m1_f64
|
||||
#define VLEV_FLOAT vle64_v_f64m4
|
||||
#define VLSEV_FLOAT vlse64_v_f64m4
|
||||
#define VFREDSUM_FLOAT vfredusum_vs_f64m4_f64m1
|
||||
#define VFMACCVV_FLOAT vfmacc_vv_f64m4
|
||||
#define VFMVVF_FLOAT vfmv_v_f_f64m4
|
||||
#define VFMVVF_FLOAT_M1 vfmv_v_f_f64m1
|
||||
#define VFDOTVV_FLOAT vfdot_vv_f64m4
|
||||
#define ABS fabs
|
||||
#define MASK_T vbool16_t
|
||||
#define VFRSUBVF_MASK_FLOAT vfrsub_vf_f64m4_m
|
||||
#define VMFGTVF_FLOAT vmfgt_vf_f64m4_b16
|
||||
#define VMFIRSTM vmfirst_m_b16
|
||||
#define VFDIVVF_FLOAT vfdiv_vf_f64m4
|
||||
#define VMFLTVF_FLOAT vmflt_vf_f64m4_b16
|
||||
#define VFREDMAXVS_FLOAT vfredmax_vs_f64m4_f64m1
|
||||
# define LMUL m4
|
||||
# if defined(DOUBLE)
|
||||
# define ELEN 64
|
||||
# define MLEN 16
|
||||
# else
|
||||
# define ELEN 32
|
||||
# define MLEN 8
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#define _
|
||||
#define JOIN2_X(x, y) x ## y
|
||||
#define JOIN2(x, y) JOIN2_X(x, y)
|
||||
#define JOIN(v, w, x, y, z) JOIN2( JOIN2( JOIN2( JOIN2( v, w ), x), y), z)
|
||||
|
||||
#define VSETVL JOIN(__riscv_vsetvl, _e, ELEN, LMUL, _)
|
||||
#define FLOAT_V_T JOIN(vfloat, ELEN, LMUL, _t, _)
|
||||
#define FLOAT_V_T_M1 JOIN(vfloat, ELEN, m1, _t, _)
|
||||
#define VLEV_FLOAT JOIN(__riscv_vle, ELEN, _v_f, ELEN, LMUL)
|
||||
#define VLSEV_FLOAT JOIN(__riscv_vlse, ELEN, _v_f, ELEN, LMUL)
|
||||
#define VFMVVF_FLOAT JOIN(__riscv_vfmv, _v_f_f, ELEN, LMUL, _)
|
||||
#define VFMVVF_FLOAT_M1 JOIN(__riscv_vfmv, _v_f_f, ELEN, m1, _)
|
||||
#define MASK_T JOIN(vbool, MLEN, _t, _, _)
|
||||
#define VFABS JOIN(__riscv_vfabs, _v_f, ELEN, LMUL, _)
|
||||
#define VMFNE JOIN(__riscv_vmfne_vf_f,ELEN, LMUL, _b, MLEN)
|
||||
#define VMFGT JOIN(__riscv_vmfgt_vv_f,ELEN, LMUL, _b, MLEN)
|
||||
#define VMFEQ JOIN(__riscv_vmfeq_vv_f,ELEN, LMUL, _b, MLEN)
|
||||
#define VCPOP JOIN(__riscv_vcpop, _m_b, MLEN, _, _)
|
||||
#define VFREDMAX JOIN(__riscv_vfredmax_vs_f,ELEN,LMUL, JOIN2(_f, ELEN), m1)
|
||||
#define VFIRST JOIN(__riscv_vfirst, _m_b, MLEN, _, _)
|
||||
#define VRGATHER JOIN(__riscv_vrgather, _vx_f, ELEN, LMUL, _)
|
||||
#define VFDIV JOIN(__riscv_vfdiv, _vf_f, ELEN, LMUL, _)
|
||||
#define VFDIV_M JOIN(__riscv_vfdiv, _vv_f, ELEN, LMUL, _mu)
|
||||
#define VFMUL JOIN(__riscv_vfmul, _vv_f, ELEN, LMUL, _)
|
||||
#define VFMACC JOIN(__riscv_vfmacc, _vv_f, ELEN, LMUL, _)
|
||||
#define VFMACC_M JOIN(__riscv_vfmacc, _vv_f, ELEN, LMUL, _mu)
|
||||
#define VMSOF JOIN(__riscv_vmsof, _m_b, MLEN, _, _)
|
||||
#define VMANDN JOIN(__riscv_vmandn, _mm_b, MLEN, _, _)
|
||||
#define VFREDUSUM JOIN(__riscv_vfredusum_vs_f,ELEN,LMUL, JOIN2(_f, ELEN), m1)
|
||||
#if defined(DOUBLE)
|
||||
#define ABS fabs
|
||||
#else
|
||||
#define ABS fabsf
|
||||
#endif
|
||||
|
||||
#define EXTRACT_FLOAT0_V(v) JOIN(__riscv_vfmv_f_s_f, ELEN, LMUL, _f, ELEN)(v)
|
||||
|
||||
|
||||
FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x)
|
||||
{
|
||||
BLASLONG i=0, j=0;
|
||||
BLASLONG i=0;
|
||||
|
||||
if ( n < 0 ) return(0.0);
|
||||
// if(n == 1) return (ABS(x[0]));
|
||||
if(n < 0) return(0.0);
|
||||
|
||||
FLOAT_V_T vr, v0, v_zero;
|
||||
FLOAT_V_T v_ssq, v_scale, v0, v1, v_zero;
|
||||
unsigned int gvl = 0;
|
||||
FLOAT_V_T_M1 v_res, v_z0;
|
||||
gvl = VSETVL_MAX;
|
||||
v_res = VFMVVF_FLOAT_M1(0, gvl);
|
||||
v_z0 = VFMVVF_FLOAT_M1(0, gvl);
|
||||
|
||||
FLOAT scale = 0.0, ssq = 0.0;
|
||||
MASK_T mask;
|
||||
BLASLONG index = 0;
|
||||
if(inc_x == 1){
|
||||
BLASLONG n2 = n * 2;
|
||||
gvl = VSETVL(n2);
|
||||
vr = VFMVVF_FLOAT(0, gvl);
|
||||
v_zero = VFMVVF_FLOAT(0, gvl);
|
||||
for(i=0,j=0; i<n2/gvl; i++){
|
||||
v0 = VLEV_FLOAT(&x[j], gvl);
|
||||
//fabs(vector)
|
||||
mask = VMFLTVF_FLOAT(v0, 0, gvl);
|
||||
v0 = VFRSUBVF_MASK_FLOAT(mask, v0, v0, 0, gvl);
|
||||
//if scale change
|
||||
mask = VMFGTVF_FLOAT(v0, scale, gvl);
|
||||
index = VMFIRSTM(mask, gvl);
|
||||
if(index == -1){//no elements greater than scale
|
||||
if(scale != 0.0){
|
||||
v0 = VFDIVVF_FLOAT(v0, scale, gvl);
|
||||
vr = VFMACCVV_FLOAT(vr, v0, v0, gvl);
|
||||
}
|
||||
}else{//found greater element
|
||||
//ssq in vector vr: vr[0]
|
||||
v_res = VFREDSUM_FLOAT(v_res, vr, v_z0, gvl);
|
||||
//total ssq before current vector
|
||||
ssq += VFMVFS_FLOAT(v_res);
|
||||
//find max
|
||||
v_res = VFREDMAXVS_FLOAT(v_res, v0, v_z0, gvl);
|
||||
//update ssq before max_index
|
||||
ssq = ssq * (scale/VFMVFS_FLOAT(v_res))*(scale/VFMVFS_FLOAT(v_res));
|
||||
//update scale
|
||||
scale = VFMVFS_FLOAT(v_res);
|
||||
//ssq in vector vr
|
||||
v0 = VFDIVVF_FLOAT(v0, scale, gvl);
|
||||
vr = VFMACCVV_FLOAT(v_zero, v0, v0, gvl);
|
||||
}
|
||||
j += gvl;
|
||||
v_res = VFMVVF_FLOAT_M1(0, 1);
|
||||
v_z0 = VFMVVF_FLOAT_M1(0, 1);
|
||||
|
||||
gvl = VSETVL(n);
|
||||
v_ssq = VFMVVF_FLOAT(0, gvl);
|
||||
v_scale = VFMVVF_FLOAT(0, gvl);
|
||||
v_zero = VFMVVF_FLOAT(0, gvl);
|
||||
|
||||
unsigned int stride_x = inc_x * sizeof(FLOAT) * 2;
|
||||
int idx = 0;
|
||||
|
||||
for(i=0; i<n/gvl; i++){
|
||||
v0 = VLSEV_FLOAT( &x[idx], stride_x, gvl );
|
||||
v1 = VLSEV_FLOAT( &x[idx+1], stride_x, gvl );
|
||||
v0 = VFABS( v0, gvl );
|
||||
v1 = VFABS( v1, gvl );
|
||||
|
||||
MASK_T scale_mask0 = VMFGT( v0, v_scale, gvl );
|
||||
MASK_T scale_mask1 = VMFGT( v1, v_scale, gvl );
|
||||
if( VCPOP( scale_mask0, gvl ) + VCPOP( scale_mask1, gvl ) > 0 ){ // scale change?
|
||||
// find largest element in v0 and v1
|
||||
v_res = VFREDMAX( v0, v_z0, gvl );
|
||||
v_res = VFREDMAX( v1, v_res, gvl );
|
||||
FLOAT const largest_elt = EXTRACT_FLOAT( v_res );
|
||||
|
||||
v_scale = VFDIV( v_scale, largest_elt, gvl ); // scale/largest_elt
|
||||
v_scale = VFMUL( v_scale, v_scale, gvl ); // (scale/largest_elt)*(scale/largest_elt)
|
||||
v_ssq = VFMUL( v_scale, v_ssq, gvl ); // ssq*(scale/largest_elt)*(scale/largest_elt)
|
||||
|
||||
v_scale = VFMVVF_FLOAT( largest_elt, gvl ); // splated largest_elt becomes new scale
|
||||
}
|
||||
//ssq in vector vr: vr[0]
|
||||
v_res = VFREDSUM_FLOAT(v_res, vr, v_z0, gvl);
|
||||
//total ssq now
|
||||
ssq += VFMVFS_FLOAT(v_res);
|
||||
|
||||
//tail
|
||||
if(j < n2){
|
||||
gvl = VSETVL(n2-j);
|
||||
v0 = VLEV_FLOAT(&x[j], gvl);
|
||||
//fabs(vector)
|
||||
mask = VMFLTVF_FLOAT(v0, 0, gvl);
|
||||
v0 = VFRSUBVF_MASK_FLOAT(mask, v0, v0, 0, gvl);
|
||||
//if scale change
|
||||
mask = VMFGTVF_FLOAT(v0, scale, gvl);
|
||||
index = VMFIRSTM(mask, gvl);
|
||||
if(index == -1){//no elements greater than scale
|
||||
if(scale != 0.0)
|
||||
v0 = VFDIVVF_FLOAT(v0, scale, gvl);
|
||||
}else{//found greater element
|
||||
//find max
|
||||
v_res = VFREDMAXVS_FLOAT(v_res, v0, v_z0, gvl);
|
||||
//update ssq before max_index
|
||||
ssq = ssq * (scale/VFMVFS_FLOAT(v_res))*(scale/VFMVFS_FLOAT(v_res));
|
||||
//update scale
|
||||
scale = VFMVFS_FLOAT(v_res);
|
||||
v0 = VFDIVVF_FLOAT(v0, scale, gvl);
|
||||
}
|
||||
vr = VFMACCVV_FLOAT(v_zero, v0, v0, gvl);
|
||||
//ssq in vector vr: vr[0]
|
||||
v_res = VFREDSUM_FLOAT(v_res, vr, v_z0, gvl);
|
||||
//total ssq now
|
||||
ssq += VFMVFS_FLOAT(v_res);
|
||||
}
|
||||
}else{
|
||||
gvl = VSETVL(n);
|
||||
vr = VFMVVF_FLOAT(0, gvl);
|
||||
v_zero = VFMVVF_FLOAT(0, gvl);
|
||||
unsigned int stride_x = inc_x * sizeof(FLOAT) * 2;
|
||||
int idx = 0, inc_v = inc_x * gvl * 2;
|
||||
for(i=0,j=0; i<n/gvl; i++){
|
||||
v0 = VLSEV_FLOAT(&x[idx], stride_x, gvl);
|
||||
//fabs(vector)
|
||||
mask = VMFLTVF_FLOAT(v0, 0, gvl);
|
||||
v0 = VFRSUBVF_MASK_FLOAT(mask, v0, v0, 0, gvl);
|
||||
//if scale change
|
||||
mask = VMFGTVF_FLOAT(v0, scale, gvl);
|
||||
index = VMFIRSTM(mask, gvl);
|
||||
if(index == -1){//no elements greater than scale
|
||||
if(scale != 0.0){
|
||||
v0 = VFDIVVF_FLOAT(v0, scale, gvl);
|
||||
vr = VFMACCVV_FLOAT(vr, v0, v0, gvl);
|
||||
}
|
||||
}else{//found greater element
|
||||
//ssq in vector vr: vr[0]
|
||||
v_res = VFREDSUM_FLOAT(v_res, vr, v_z0, gvl);
|
||||
//total ssq before current vector
|
||||
ssq += VFMVFS_FLOAT(v_res);
|
||||
//find max
|
||||
v_res = VFREDMAXVS_FLOAT(v_res, v0, v_z0, gvl);
|
||||
//update ssq before max_index
|
||||
ssq = ssq * (scale/VFMVFS_FLOAT(v_res))*(scale/VFMVFS_FLOAT(v_res));
|
||||
//update scale
|
||||
scale = VFMVFS_FLOAT(v_res);
|
||||
//ssq in vector vr
|
||||
v0 = VFDIVVF_FLOAT(v0, scale, gvl);
|
||||
vr = VFMACCVV_FLOAT(v_zero, v0, v0, gvl);
|
||||
}
|
||||
MASK_T nonzero_mask0 = VMFNE( v0, 0, gvl );
|
||||
MASK_T nonzero_mask1 = VMFNE( v1, 0, gvl );
|
||||
v0 = VFDIV_M( nonzero_mask0, v_zero, v0, v_scale, gvl );
|
||||
v1 = VFDIV_M( nonzero_mask1, v_zero, v1, v_scale, gvl );
|
||||
v_ssq = VFMACC_M( nonzero_mask0, v_ssq, v0, v0, gvl );
|
||||
v_ssq = VFMACC_M( nonzero_mask1, v_ssq, v1, v1, gvl );
|
||||
|
||||
v0 = VLSEV_FLOAT(&x[idx+1], stride_x, gvl);
|
||||
//fabs(vector)
|
||||
mask = VMFLTVF_FLOAT(v0, 0, gvl);
|
||||
v0 = VFRSUBVF_MASK_FLOAT(mask, v0, v0, 0, gvl);
|
||||
//if scale change
|
||||
mask = VMFGTVF_FLOAT(v0, scale, gvl);
|
||||
index = VMFIRSTM(mask, gvl);
|
||||
if(index == -1){//no elements greater than scale
|
||||
if(scale != 0.0){
|
||||
v0 = VFDIVVF_FLOAT(v0, scale, gvl);
|
||||
vr = VFMACCVV_FLOAT(vr, v0, v0, gvl);
|
||||
}
|
||||
}else{//found greater element
|
||||
//ssq in vector vr: vr[0]
|
||||
v_res = VFREDSUM_FLOAT(v_res, vr, v_z0, gvl);
|
||||
//total ssq before current vector
|
||||
ssq += VFMVFS_FLOAT(v_res);
|
||||
//find max
|
||||
v_res = VFREDMAXVS_FLOAT(v_res, v0, v_z0, gvl);
|
||||
//update ssq before max_index
|
||||
ssq = ssq * (scale/VFMVFS_FLOAT(v_res))*(scale/VFMVFS_FLOAT(v_res));
|
||||
//update scale
|
||||
scale = VFMVFS_FLOAT(v_res);
|
||||
//ssq in vector vr
|
||||
v0 = VFDIVVF_FLOAT(v0, scale, gvl);
|
||||
vr = VFMACCVV_FLOAT(v_zero, v0, v0, gvl);
|
||||
}
|
||||
j += gvl;
|
||||
idx += inc_v;
|
||||
}
|
||||
//ssq in vector vr: vr[0]
|
||||
v_res = VFREDSUM_FLOAT(v_res, vr, v_z0, gvl);
|
||||
//total ssq now
|
||||
ssq += VFMVFS_FLOAT(v_res);
|
||||
|
||||
//tail
|
||||
if(j < n){
|
||||
gvl = VSETVL(n-j);
|
||||
v0 = VLSEV_FLOAT(&x[idx], stride_x, gvl);
|
||||
//fabs(vector)
|
||||
mask = VMFLTVF_FLOAT(v0, 0, gvl);
|
||||
v0 = VFRSUBVF_MASK_FLOAT(mask, v0, v0, 0, gvl);
|
||||
//if scale change
|
||||
mask = VMFGTVF_FLOAT(v0, scale, gvl);
|
||||
index = VMFIRSTM(mask, gvl);
|
||||
if(index == -1){//no elements greater than scale
|
||||
if(scale != 0.0){
|
||||
v0 = VFDIVVF_FLOAT(v0, scale, gvl);
|
||||
vr = VFMACCVV_FLOAT(v_zero, v0, v0, gvl);
|
||||
}
|
||||
}else{//found greater element
|
||||
//find max
|
||||
v_res = VFREDMAXVS_FLOAT(v_res, v0, v_z0, gvl);
|
||||
//update ssq before max_index
|
||||
ssq = ssq * (scale/VFMVFS_FLOAT(v_res))*(scale/VFMVFS_FLOAT(v_res));
|
||||
//update scale
|
||||
scale = VFMVFS_FLOAT(v_res);
|
||||
v0 = VFDIVVF_FLOAT(v0, scale, gvl);
|
||||
vr = VFMACCVV_FLOAT(v_zero, v0, v0, gvl);
|
||||
}
|
||||
|
||||
v0 = VLSEV_FLOAT(&x[idx+1], stride_x, gvl);
|
||||
//fabs(vector)
|
||||
mask = VMFLTVF_FLOAT(v0, 0, gvl);
|
||||
v0 = VFRSUBVF_MASK_FLOAT(mask, v0, v0, 0, gvl);
|
||||
//if scale change
|
||||
mask = VMFGTVF_FLOAT(v0, scale, gvl);
|
||||
index = VMFIRSTM(mask, gvl);
|
||||
if(index == -1){//no elements greater than scale
|
||||
if(scale != 0.0){
|
||||
v0 = VFDIVVF_FLOAT(v0, scale, gvl);
|
||||
vr = VFMACCVV_FLOAT(vr, v0, v0, gvl);
|
||||
}
|
||||
}else{//found greater element
|
||||
//ssq in vector vr: vr[0]
|
||||
v_res = VFREDSUM_FLOAT(v_res, vr, v_z0, gvl);
|
||||
//total ssq before current vector
|
||||
ssq += VFMVFS_FLOAT(v_res);
|
||||
//find max
|
||||
v_res = VFREDMAXVS_FLOAT(v_res, v0, v_z0, gvl);
|
||||
//update ssq before max_index
|
||||
ssq = ssq * (scale/VFMVFS_FLOAT(v_res))*(scale/VFMVFS_FLOAT(v_res));
|
||||
//update scale
|
||||
scale = VFMVFS_FLOAT(v_res);
|
||||
v0 = VFDIVVF_FLOAT(v0, scale, gvl);
|
||||
vr = VFMACCVV_FLOAT(v_zero, v0, v0, gvl);
|
||||
}
|
||||
//ssq in vector vr: vr[0]
|
||||
v_res = VFREDSUM_FLOAT(v_res, vr, v_z0, gvl);
|
||||
//total ssq now
|
||||
ssq += VFMVFS_FLOAT(v_res);
|
||||
}
|
||||
idx += inc_x * gvl * 2;
|
||||
}
|
||||
return(scale * sqrt(ssq));
|
||||
|
||||
v_res = VFREDUSUM(v_ssq, v_z0, gvl);
|
||||
FLOAT ssq = EXTRACT_FLOAT(v_res);
|
||||
FLOAT scale = EXTRACT_FLOAT0_V(v_scale);
|
||||
|
||||
//finish any tail using scalar ops
|
||||
i*=gvl;
|
||||
if(i<n){
|
||||
i *= inc_x*2;
|
||||
n *= inc_x*2;
|
||||
FLOAT temp;
|
||||
do{
|
||||
if ( x[i] != 0.0 ){
|
||||
temp = ABS( x[i] );
|
||||
if ( scale < temp ){
|
||||
ssq = 1 + ssq * ( scale / temp ) * ( scale / temp );
|
||||
scale = temp ;
|
||||
}else{
|
||||
ssq += ( temp / scale ) * ( temp / scale );
|
||||
}
|
||||
}
|
||||
|
||||
if ( x[i+1] != 0.0 ){
|
||||
temp = ABS( x[i+1] );
|
||||
if ( scale < temp ){
|
||||
ssq = 1 + ssq * ( scale / temp ) * ( scale / temp );
|
||||
scale = temp ;
|
||||
}else{
|
||||
ssq += ( temp / scale ) * ( temp / scale );
|
||||
}
|
||||
}
|
||||
|
||||
i += inc_x*2;
|
||||
}while(i<n);
|
||||
}
|
||||
|
||||
return(scale * sqrt(ssq));
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user