diff --git a/kernel/power/sbgemv_common_power10.c b/kernel/power/sbgemv_common_power10.c index 0510088b2..b0e611cb6 100644 --- a/kernel/power/sbgemv_common_power10.c +++ b/kernel/power/sbgemv_common_power10.c @@ -525,6 +525,42 @@ FORCEINLINE void vec_store4_pair(vec_f32 *v_y, vec_f32 *vy0) vec_store_pair(v_y + 6, vy0 + 6); } +FORCEINLINE void vec_setzero_2(__vector_quad *temp0) +{ + __builtin_mma_xxsetaccz(&temp0[0]); + __builtin_mma_xxsetaccz(&temp0[1]); +} + +FORCEINLINE void vec_setzero_4(__vector_quad *temp0) +{ + vec_setzero_2(temp0 + 0); + vec_setzero_2(temp0 + 2); +} + +FORCEINLINE void vec_setzero_8(__vector_quad *temp0) +{ + vec_setzero_4(temp0 + 0); + vec_setzero_4(temp0 + 4); +} + +FORCEINLINE void vec_reduce_2(vec_f32 *temp00, __vector_quad *temp0) +{ + __builtin_mma_disassemble_acc((void*)(temp00 + 0), &temp0[0]); + __builtin_mma_disassemble_acc((void*)(temp00 + 4), &temp0[1]); +} + +FORCEINLINE void vec_reduce_4(vec_f32 *temp00, __vector_quad *temp0) +{ + vec_reduce_2(temp00 + 0, temp0 + 0); + vec_reduce_2(temp00 + 8, temp0 + 2); +} + +FORCEINLINE void vec_reduce_8(vec_f32 *temp00, __vector_quad *temp0) +{ + vec_reduce_4(temp00 + 0, temp0 + 0); + vec_reduce_4(temp00 + 16, temp0 + 4); +} + #ifdef USE_MERGE_MMA FORCEINLINE void vec_load8_pair(vec_f32 *vy0, vec_f32 *v_y) { diff --git a/kernel/power/sbgemv_t_power10.c b/kernel/power/sbgemv_t_power10.c index 9a5c54f12..d2f6087f0 100644 --- a/kernel/power/sbgemv_t_power10.c +++ b/kernel/power/sbgemv_t_power10.c @@ -101,8 +101,7 @@ static void BF16GEMV_T_MMA_2(BLASLONG n, BLASLONG lda, IFLOAT *ap, IFLOAT *x, FL vec_f32 temp00[4*2]; vec_bf16 inp[4]; - __builtin_mma_xxsetaccz(&temp0[0]); - __builtin_mma_xxsetaccz(&temp0[1]); + vec_setzero_2(&temp0[0]); a0 = ap; a1 = ap + lda; @@ -141,8 +140,7 @@ static void BF16GEMV_T_MMA_2(BLASLONG n, BLASLONG lda, IFLOAT *ap, IFLOAT *x, FL vec_loadN_mult12a_mma(&temp0[0], &va0[i], &va1[i], inp[0], n); } - __builtin_mma_disassemble_acc((void*)(temp00 + 0), &temp0[0]); - __builtin_mma_disassemble_acc((void*)(temp00 + 4), &temp0[1]); + vec_reduce_2(temp00, &temp0[0]); y[0] = (alpha * (temp00[0][0] + temp00[1][1] + temp00[2][2] + temp00[3][3])) + (beta * y[0]); y[1] = (alpha * (temp00[4][0] + temp00[5][1] + temp00[6][2] + temp00[7][3])) + (beta * y[1]); @@ -156,10 +154,7 @@ static void BF16GEMV_T_MMA_4(BLASLONG n, BLASLONG lda, IFLOAT *ap, IFLOAT *x, FL vec_f32 temp00[4*4]; vec_bf16 inp[4]; - __builtin_mma_xxsetaccz(&temp0[0]); - __builtin_mma_xxsetaccz(&temp0[1]); - __builtin_mma_xxsetaccz(&temp0[2]); - __builtin_mma_xxsetaccz(&temp0[3]); + vec_setzero_4(&temp0[0]); a0 = ap; a1 = ap + lda; @@ -202,10 +197,7 @@ static void BF16GEMV_T_MMA_4(BLASLONG n, BLASLONG lda, IFLOAT *ap, IFLOAT *x, FL vec_loadN_mult14_mma(&temp0[0], &va0[i], &va1[i], &va2[i], &va3[i], inp[0], n); } - __builtin_mma_disassemble_acc((void*)(temp00 + 0), &temp0[0]); - __builtin_mma_disassemble_acc((void*)(temp00 + 4), &temp0[1]); - __builtin_mma_disassemble_acc((void*)(temp00 + 8), &temp0[2]); - __builtin_mma_disassemble_acc((void*)(temp00 + 12), &temp0[3]); + vec_reduce_4(temp00, &temp0[0]); vec_f32 t0, t1, t2, t3, t4, t5, t6, t7; vec_f32 a = { alpha, alpha, alpha, alpha }; @@ -239,23 +231,17 @@ static void BF16GEMV_T_MMA_8(BLASLONG n, BLASLONG lda, IFLOAT *ap, IFLOAT *x, FL vec_f32 temp00[4*8]; vec_bf16 inp[4]; - __builtin_mma_xxsetaccz(&temp0[0]); - __builtin_mma_xxsetaccz(&temp0[1]); - __builtin_mma_xxsetaccz(&temp0[2]); - __builtin_mma_xxsetaccz(&temp0[3]); - __builtin_mma_xxsetaccz(&temp0[4]); - __builtin_mma_xxsetaccz(&temp0[5]); - __builtin_mma_xxsetaccz(&temp0[6]); - __builtin_mma_xxsetaccz(&temp0[7]); + vec_setzero_8(&temp0[0]); + BLASLONG lda4 = lda << 2; a0 = ap; a1 = ap + lda; a2 = a1 + lda; a3 = a2 + lda; - a4 = a3 + lda; - a5 = a4 + lda; - a6 = a5 + lda; - a7 = a6 + lda; + a4 = a0 + lda4; + a5 = a1 + lda4; + a6 = a2 + lda4; + a7 = a3 + lda4; va0 = (vec_bf16 *)a0; va1 = (vec_bf16 *)a1; va2 = (vec_bf16 *)a2; @@ -301,14 +287,7 @@ static void BF16GEMV_T_MMA_8(BLASLONG n, BLASLONG lda, IFLOAT *ap, IFLOAT *x, FL vec_loadN_mult14_mma(&temp0[4], &va4[i], &va5[i], &va6[i], &va7[i], inp[0], n); } - __builtin_mma_disassemble_acc((void*)(temp00 + 0), &temp0[0]); - __builtin_mma_disassemble_acc((void*)(temp00 + 4), &temp0[1]); - __builtin_mma_disassemble_acc((void*)(temp00 + 8), &temp0[2]); - __builtin_mma_disassemble_acc((void*)(temp00 + 12), &temp0[3]); - __builtin_mma_disassemble_acc((void*)(temp00 + 16), &temp0[4]); - __builtin_mma_disassemble_acc((void*)(temp00 + 20), &temp0[5]); - __builtin_mma_disassemble_acc((void*)(temp00 + 24), &temp0[6]); - __builtin_mma_disassemble_acc((void*)(temp00 + 28), &temp0[7]); + vec_reduce_8(temp00, &temp0[0]); vec_f32 t0, t1, t2, t3, t4, t5, t6, t7, t10, t11, t12, t13, t14, t15, t16, t17; vec_f32 a = { alpha, alpha, alpha, alpha }; diff --git a/kernel/power/sbgemv_t_vsx.c b/kernel/power/sbgemv_t_vsx.c index 399989bb5..075040503 100644 --- a/kernel/power/sbgemv_t_vsx.c +++ b/kernel/power/sbgemv_t_vsx.c @@ -198,14 +198,15 @@ static void BF16GEMV_T_VSX_8(BLASLONG n, BLASLONG lda, IFLOAT *ap, IFLOAT *x, FL vec_bf16 zero = { 0, 0, 0, 0, 0, 0, 0, 0 }; vec_f32 inp[2]; + BLASLONG lda4 = lda << 2; a0 = ap; a1 = ap + lda; a2 = a1 + lda; a3 = a2 + lda; - a4 = a3 + lda; - a5 = a4 + lda; - a6 = a5 + lda; - a7 = a6 + lda; + a4 = a0 + lda4; + a5 = a1 + lda4; + a6 = a2 + lda4; + a7 = a3 + lda4; va0 = (vec_bf16 *)a0; va1 = (vec_bf16 *)a1; va2 = (vec_bf16 *)a2;