From 346b384d1c6f0cb00170c1bf8ad3e8302f142237 Mon Sep 17 00:00:00 2001
From: yancheng <yancheng@loongson.cn>
Date: Thu, 7 Dec 2023 11:30:02 +0800
Subject: [PATCH] loongarch64: Add optimization for max.

---
 kernel/loongarch64/KERNEL.LOONGSON2K1000 |   3 +
 kernel/loongarch64/KERNEL.LOONGSON3R5    |   3 +
 kernel/loongarch64/dmax_lasx.S           | 175 +++++++++++++++++++
 kernel/loongarch64/dmax_lsx.S            | 141 ++++++++++++++++
 kernel/loongarch64/smax_lasx.S           | 205 +++++++++++++++++++++++
 kernel/loongarch64/smax_lsx.S            | 171 +++++++++++++++++++
 6 files changed, 698 insertions(+)
 create mode 100644 kernel/loongarch64/dmax_lasx.S
 create mode 100644 kernel/loongarch64/dmax_lsx.S
 create mode 100644 kernel/loongarch64/smax_lasx.S
 create mode 100644 kernel/loongarch64/smax_lsx.S

diff --git a/kernel/loongarch64/KERNEL.LOONGSON2K1000 b/kernel/loongarch64/KERNEL.LOONGSON2K1000
index 279ff6a9c..e00893b72 100644
--- a/kernel/loongarch64/KERNEL.LOONGSON2K1000
+++ b/kernel/loongarch64/KERNEL.LOONGSON2K1000
@@ -13,4 +13,7 @@ DAMAXKERNEL =  damax_lsx.S
 SAMINKERNEL =  samin_lsx.S
 DAMINKERNEL =  damin_lsx.S
 
+SMAXKERNEL  =  smax_lsx.S
+DMAXKERNEL  =  dmax_lsx.S
+
 endif
diff --git a/kernel/loongarch64/KERNEL.LOONGSON3R5 b/kernel/loongarch64/KERNEL.LOONGSON3R5
index 83db79050..f238436f5 100644
--- a/kernel/loongarch64/KERNEL.LOONGSON3R5
+++ b/kernel/loongarch64/KERNEL.LOONGSON3R5
@@ -13,6 +13,9 @@ DAMAXKERNEL =  damax_lasx.S
 SAMINKERNEL =  samin_lasx.S
 DAMINKERNEL =  damin_lasx.S
 
+SMAXKERNEL  =  smax_lasx.S
+DMAXKERNEL =   dmax_lasx.S
+
 DGEMMKERNEL    = dgemm_kernel_16x4.S
 DGEMMINCOPY    = dgemm_ncopy_16.S
 DGEMMITCOPY    = dgemm_tcopy_16.S
diff --git a/kernel/loongarch64/dmax_lasx.S b/kernel/loongarch64/dmax_lasx.S
new file mode 100644
index 000000000..46366d2ec
--- /dev/null
+++ b/kernel/loongarch64/dmax_lasx.S
@@ -0,0 +1,175 @@
+#define ASSEMBLER
+
+#include "common.h"
+
+#define N $r4
+#define X $r5
+#define INCX $r6
+#define I $r12
+#define J $r13
+#define t1 $r14
+#define t2 $r18
+#define t3 $r15
+#define t4 $r17
+#define TEMP $r16
+#define m0 $xr8
+#define x1 $xr9
+#define x2 $xr10
+#define x3 $xr11
+#define x4 $xr12
+#define VX0 $xr20
+#define VX1 $xr21
+#define VM0 $xr22
+#define VM1 $xr23
+#define VM2 $xr19
+
+    PROLOGUE
+
+    bge $r0, N, .L999
+    bge $r0, INCX, .L999
+    li.d TEMP, 1
+    slli.d TEMP, TEMP, BASE_SHIFT
+    slli.d INCX, INCX, BASE_SHIFT
+    bne INCX, TEMP, .L20
+    xvld VM0, X, 0
+    srai.d I, N, 3
+    bge $r0, I, .L12
+    .align 3
+
+.L10:
+    xvld VX0, X, 0 * SIZE
+    xvld VX1, X, 4 * SIZE
+    addi.d I, I, -1
+    xvfmax.d VM1, VX1, VX0
+    addi.d X, X, 8 * SIZE
+    xvfmax.d VM0, VM0, VM1
+    blt $r0, I, .L10
+    .align 3
+
+.L11:
+    xvpickve.d x1, VM0, 0
+    xvpickve.d x2, VM0, 1
+    xvpickve.d x3, VM0, 2
+    xvpickve.d x4, VM0, 3
+    xvfmax.d VM1, x1, x2
+    xvfmax.d VM2, x3, x4
+    xvfmax.d VM0, VM1, VM2
+    .align 3
+
+.L12: //INCX==1 and N<8
+    andi I, N, 7
+    li.d J, 4
+    bge J, I, .L13 // 4<N<8
+    xvld VX0, X, 0
+    slli.d J, J, 1 // 8
+    sub.d I, J, I
+    slli.d I, I, BASE_SHIFT
+    xvldx VX1, X, I
+    xvfmax.d m0, VX0, VX1 //patial repeat read
+    xvpickve.d x1, m0, 0
+    xvpickve.d x2, m0, 1
+    xvpickve.d x3, m0, 2
+    xvpickve.d x4, m0, 3
+    xvfmax.d VM1, x1, x2
+    xvfmax.d m0, x3, x4
+    xvfmax.d m0, m0, VM1
+    xvfmax.d VM0, m0, VM0
+    fmov.d $f0, $f22
+    jirl $r0, $r1, 0x0
+    .align 3
+
+.L13: //INCX==1 and 0<=N<=4
+    bge $r0, I, .L15
+    .align 3
+
+.L14:
+    xvld x1, X, 0
+    addi.d I, I, -1
+    xvfmax.d VM0, VM0, x1
+    addi.d  X, X, SIZE
+    blt $r0, I, .L14
+    .align 3
+
+.L15:
+    fmov.d $f0, $f22
+    jirl $r0, $r1, 0x0
+    .align 3
+
+.L20: // INCX!=1
+    move TEMP, X // initialize the max value
+    ld.d t1, TEMP, 0 * SIZE
+    add.d TEMP, TEMP, INCX
+    xvinsgr2vr.d VM0, t1, 0
+    srai.d I, N, 3
+    bge $r0, I, .L23
+    ld.d t2, TEMP, 0 * SIZE
+    add.d TEMP, TEMP, INCX
+    ld.d t3, TEMP, 0 * SIZE
+    add.d TEMP, TEMP, INCX
+    ld.d t4, TEMP, 0 * SIZE
+    add.d TEMP, TEMP, INCX
+    xvinsgr2vr.d VM0, t2, 1
+    xvinsgr2vr.d VM0, t3, 2
+    xvinsgr2vr.d VM0, t4, 3
+    .align 3
+
+.L21:
+    ld.d t1, X, 0 * SIZE
+    add.d X, X, INCX
+    ld.d t2, X, 0 * SIZE
+    add.d X, X, INCX
+    ld.d t3, X, 0 * SIZE
+    add.d X, X, INCX
+    ld.d t4, X, 0 * SIZE
+    add.d X, X, INCX
+    xvinsgr2vr.d VX0, t1, 0
+    xvinsgr2vr.d VX0, t2, 1
+    xvinsgr2vr.d VX0, t3, 2
+    xvinsgr2vr.d VX0, t4, 3
+    ld.d t1, X, 0 * SIZE
+    add.d X, X, INCX
+    ld.d t2, X, 0 * SIZE
+    add.d X, X, INCX
+    ld.d t3, X, 0 * SIZE
+    add.d X, X, INCX
+    ld.d t4, X, 0 * SIZE
+    add.d X, X, INCX
+    xvinsgr2vr.d VX1, t1, 0
+    xvinsgr2vr.d VX1, t2, 1
+    xvinsgr2vr.d VX1, t3, 2
+    xvinsgr2vr.d VX1, t4, 3
+    addi.d I, I, -1
+    xvfmax.d VM1, VX1, VX0
+    xvfmax.d VM0, VM1, VM0
+    blt $r0, I, .L21
+    .align 3
+
+.L22:
+    xvpickve.d x1, VM0, 0
+    xvpickve.d x2, VM0, 1
+    xvpickve.d x3, VM0, 2
+    xvpickve.d x4, VM0, 3
+    xvfmax.d VM1, x1, x2
+    xvfmax.d VM2, x3, x4
+    xvfmax.d VM0, VM1, VM2
+    .align 3
+
+.L23: //INCX!=1 and N<8
+    andi I, N, 7
+    bge $r0, I, .L999
+    .align 3
+
+.L24:
+    xvld x1, X, 0
+    addi.d I, I, -1
+    xvfmax.d VM0, VM0, x1
+    add.d  X, X, INCX
+    blt $r0, I, .L24
+    .align 3
+
+.L999:
+    fmov.d $f0, $f22
+    jirl $r0, $r1, 0x0
+    .align 3
+
+    EPILOGUE
\ No newline at end of file
diff --git a/kernel/loongarch64/dmax_lsx.S b/kernel/loongarch64/dmax_lsx.S
new file mode 100644
index 000000000..99e9b2f30
--- /dev/null
+++ b/kernel/loongarch64/dmax_lsx.S
@@ -0,0 +1,141 @@
+#define ASSEMBLER
+
+#include "common.h"
+
+#define N $r4
+#define X $r5
+#define INCX $r6
+#define I $r12
+#define t1 $r14
+#define t2 $r18
+#define t3 $r15
+#define t4 $r17
+#define TEMP $r16
+#define x1 $vr9
+#define x2 $vr10
+#define VX0 $vr20
+#define VX1 $vr21
+#define VM0 $vr22
+#define VM1 $vr23
+#define VM2 $vr19
+#define VM3 $vr18
+
+    PROLOGUE
+
+    bge $r0, N, .L999
+    bge $r0, INCX, .L999
+    li.d TEMP, 1
+    slli.d TEMP, TEMP, BASE_SHIFT
+    slli.d INCX, INCX, BASE_SHIFT
+    bne INCX, TEMP, .L20
+    vld VM0, X, 0
+    srai.d I, N, 3
+    bge $r0, I, .L12
+    .align 3
+
+.L10:
+    vld VX0, X, 0 * SIZE
+    vld VX1, X, 2 * SIZE
+    addi.d I, I, -1
+    vfmax.d VM1, VX1, VX0
+    vld VX0, X, 4 * SIZE
+    vld VX1, X, 6 * SIZE
+    vfmax.d VM2, VX1, VX0
+    vfmax.d VM3, VM1, VM2
+    addi.d X, X, 8 * SIZE
+    vfmax.d VM0, VM0, VM3
+    blt $r0, I, .L10
+    .align 3
+
+.L11:
+    vreplvei.d x2, VM0, 1
+    vfmax.d VM0, VM0, x2
+    .align 3
+
+.L12: //INCX==1 and N<8
+    andi I, N, 7
+    bge $r0, I, .L14
+    .align 3
+
+.L13:
+    vld x1, X, 0
+    addi.d I, I, -1
+    vfmax.d VM0, VM0, x1
+    addi.d  X, X, SIZE
+    blt $r0, I, .L13
+    .align 3
+
+.L14:
+    fmov.d $f0, $f22
+    jirl $r0, $r1, 0x0
+    .align 3
+
+.L20: // INCX!=1
+    move TEMP, X // initialize the max value
+    ld.d t1, TEMP, 0 * SIZE
+    add.d TEMP, TEMP, INCX
+    vinsgr2vr.d VM0, t1, 0
+    srai.d I, N, 3
+    bge $r0, I, .L23
+    ld.d t2, TEMP, 0 * SIZE
+    add.d TEMP, TEMP, INCX
+    vinsgr2vr.d VM0, t2, 1
+    .align 3
+
+.L21:
+    ld.d t1, X, 0 * SIZE
+    add.d X, X, INCX
+    ld.d t2, X, 0 * SIZE
+    add.d X, X, INCX
+    vinsgr2vr.d VX0, t1, 0
+    vinsgr2vr.d VX0, t2, 1
+    ld.d t3, X, 0 * SIZE
+    add.d X, X, INCX
+    ld.d t4, X, 0 * SIZE
+    add.d X, X, INCX
+    vinsgr2vr.d VX1, t3, 0
+    vinsgr2vr.d VX1, t4, 1
+    vfmax.d VM1, VX0, VX1
+    ld.d t1, X, 0 * SIZE
+    add.d X, X, INCX
+    ld.d t2, X, 0 * SIZE
+    add.d X, X, INCX
+    vinsgr2vr.d VX0, t1, 0
+    vinsgr2vr.d VX0, t2, 1
+    ld.d t3, X, 0 * SIZE
+    add.d X, X, INCX
+    ld.d t4, X, 0 * SIZE
+    add.d X, X, INCX
+    vinsgr2vr.d VX1, t3, 0
+    vinsgr2vr.d VX1, t4, 1
+    addi.d I, I, -1
+    vfmax.d VM2, VX0, VX1
+    vfmax.d VM3, VM1, VM2
+    vfmax.d VM0, VM0, VM3
+    blt $r0, I, .L21
+    .align 3
+
+.L22:
+    vreplvei.d x2, VM0, 1
+    vfmax.d VM0, VM0, x2
+    .align 3
+
+.L23: //INCX!=1 and N<8
+    andi I, N, 7
+    bge $r0, I, .L999
+    .align 3
+
+.L24:
+    vld x1, X, 0
+    addi.d I, I, -1
+    vfmax.d VM0, VM0, x1
+    add.d  X, X, INCX
+    blt $r0, I, .L24
+    .align 3
+
+.L999:
+    fmov.d $f0, $f22
+    jirl $r0, $r1, 0x0
+    .align 3
+
+    EPILOGUE
diff --git a/kernel/loongarch64/smax_lasx.S b/kernel/loongarch64/smax_lasx.S
new file mode 100644
index 000000000..8cceb9928
--- /dev/null
+++ b/kernel/loongarch64/smax_lasx.S
@@ -0,0 +1,205 @@
+#define ASSEMBLER
+
+#include "common.h"
+
+#define N $r4
+#define X $r5
+#define INCX $r6
+#define I $r12
+#define J $r13
+#define t1 $r14
+#define t2 $r18
+#define t3 $r15
+#define t4 $r17
+#define TEMP $r16
+#define m0 $xr8
+#define x1 $xr9
+#define x2 $xr10
+#define x3 $xr11
+#define x4 $xr12
+#define x5 $xr13
+#define x6 $xr14
+#define x7 $xr15
+#define x8 $xr16
+#define VX0 $xr20
+#define VX1 $xr21
+#define VM0 $xr22
+#define VM1 $xr23
+#define VM2 $xr18
+#define VM3 $xr17
+
+    PROLOGUE
+
+    bge $r0, N, .L999
+    bge $r0, INCX, .L999
+    li.d TEMP, 1
+    slli.d TEMP, TEMP, BASE_SHIFT
+    slli.d INCX, INCX, BASE_SHIFT
+    bne INCX, TEMP, .L20
+    xvld VM0, X, 0
+    srai.d I, N, 3
+    bge $r0, I, .L12
+    .align 3
+
+.L10:
+    xvld VX0, X, 0 * SIZE
+    addi.d I, I, -1
+    xvfmax.s VM0, VM0, VX0
+    addi.d X, X, 8 * SIZE
+    blt $r0, I, .L10
+    .align 3
+
+.L11:
+    xvpickve.w x1, VM0, 0
+    xvpickve.w x2, VM0, 1
+    xvpickve.w x3, VM0, 2
+    xvpickve.w x4, VM0, 3
+    xvpickve.w x5, VM0, 4
+    xvpickve.w x6, VM0, 5
+    xvpickve.w x7, VM0, 6
+    xvpickve.w x8, VM0, 7
+    xvfmax.s VM3, x1, x2
+    xvfmax.s VM2, x3, x4
+    xvfmax.s VM1, x5, x6
+    xvfmax.s VM0, x7, x8
+    xvfmax.s VM2, VM2, VM3
+    xvfmax.s VM0, VM0, VM1
+    xvfmax.s VM0, VM0, VM2
+    .align 3
+
+.L12: //INCX==1 and N<8
+    andi I, N, 7
+    li.d J, 4
+    bge J, I, .L13 // 4<N<8
+    xvld VX0, X, 0
+    slli.d J, J, 1 // 8
+    sub.d I, J, I
+    slli.d I, I, BASE_SHIFT
+    xvldx VX1, X, I
+    xvfmax.s m0, VX0, VX1 //patial repeat read
+    xvpickve.w x1, m0, 0
+    xvpickve.w x2, m0, 1
+    xvpickve.w x3, m0, 2
+    xvpickve.w x4, m0, 3
+    xvfmax.s m0, x1, x2
+    xvfmax.s VM1, x3, x4
+    xvfmax.s m0, m0, VM1
+    xvfmax.s VM0, m0, VM0
+    fmov.s $f0, $f22
+    jirl $r0, $r1, 0x0
+    .align 3
+
+.L13: //INCX==1 and 0<=N<=4
+    bge $r0, I, .L15
+    .align 3
+
+.L14:
+    xvld x1, X, 0
+    addi.d I, I, -1
+    xvfmax.s VM0, VM0, x1
+    addi.d  X, X, SIZE
+    blt $r0, I, .L14
+    .align 3
+
+.L15:
+    fmov.s $f0, $f22
+    jirl $r0, $r1, 0x0
+    .align 3
+
+.L20: // INCX!=1
+    move TEMP, X // initialize the max value
+    ld.w t1, TEMP, 0 * SIZE
+    add.d TEMP, TEMP, INCX
+    xvinsgr2vr.w VM0, t1, 0
+    srai.d I, N, 3
+    bge $r0, I, .L23
+    ld.w t2, TEMP, 0 * SIZE
+    add.d TEMP, TEMP, INCX
+    ld.w t3, TEMP, 0 * SIZE
+    add.d TEMP, TEMP, INCX
+    ld.w t4, TEMP, 0 * SIZE
+    add.d TEMP, TEMP, INCX
+    xvinsgr2vr.w VM0, t2, 1
+    xvinsgr2vr.w VM0, t3, 2
+    xvinsgr2vr.w VM0, t4, 3
+    ld.w t1, TEMP, 0 * SIZE
+    add.d TEMP, TEMP, INCX
+    ld.w t2, TEMP, 0 * SIZE
+    add.d TEMP, TEMP, INCX
+    ld.w t3, TEMP, 0 * SIZE
+    add.d TEMP, TEMP, INCX
+    ld.w t4, TEMP, 0 * SIZE
+    add.d TEMP, TEMP, INCX
+    xvinsgr2vr.w VM0, t1, 4
+    xvinsgr2vr.w VM0, t2, 5
+    xvinsgr2vr.w VM0, t3, 6
+    xvinsgr2vr.w VM0, t4, 7
+    .align 3
+
+.L21:
+    ld.w t1, X, 0 * SIZE
+    add.d X, X, INCX
+    ld.w t2, X, 0 * SIZE
+    add.d X, X, INCX
+    ld.w t3, X, 0 * SIZE
+    add.d X, X, INCX
+    ld.w t4, X, 0 * SIZE
+    add.d X, X, INCX
+    xvinsgr2vr.w VX0, t1, 0
+    xvinsgr2vr.w VX0, t2, 1
+    xvinsgr2vr.w VX0, t3, 2
+    xvinsgr2vr.w VX0, t4, 3
+    ld.w t1, X, 0 * SIZE
+    add.d X, X, INCX
+    ld.w t2, X, 0 * SIZE
+    add.d X, X, INCX
+    ld.w t3, X, 0 * SIZE
+    add.d X, X, INCX
+    ld.w t4, X, 0 * SIZE
+    add.d X, X, INCX
+    xvinsgr2vr.w VX0, t1, 4
+    xvinsgr2vr.w VX0, t2, 5
+    xvinsgr2vr.w VX0, t3, 6
+    xvinsgr2vr.w VX0, t4, 7
+    addi.d I, I, -1
+    xvfmax.s VM0, VM0, VX0
+    blt $r0, I, .L21
+    .align 3
+
+.L22:
+    xvpickve.w x1, VM0, 0
+    xvpickve.w x2, VM0, 1
+    xvpickve.w x3, VM0, 2
+    xvpickve.w x4, VM0, 3
+    xvpickve.w x5, VM0, 4
+    xvpickve.w x6, VM0, 5
+    xvpickve.w x7, VM0, 6
+    xvpickve.w x8, VM0, 7
+    xvfmax.s VM3, x1, x2
+    xvfmax.s VM2, x3, x4
+    xvfmax.s VM1, x5, x6
+    xvfmax.s VM0, x7, x8
+    xvfmax.s VM2, VM2, VM3
+    xvfmax.s VM0, VM0, VM1
+    xvfmax.s VM0, VM0, VM2
+    .align 3
+
+.L23: //INCX!=1 and N<8
+    andi I, N, 7
+    bge $r0, I, .L999
+    .align 3
+
+.L24:
+    xvld x1, X, 0
+    addi.d I, I, -1
+    xvfmax.s VM0, VM0, x1
+    add.d  X, X, INCX
+    blt $r0, I, .L24
+    .align 3
+
+.L999:
+    fmov.s $f0, $f22
+    jirl $r0, $r1, 0x0
+    .align 3
+
+    EPILOGUE
diff --git a/kernel/loongarch64/smax_lsx.S b/kernel/loongarch64/smax_lsx.S
new file mode 100644
index 000000000..71e6863b8
--- /dev/null
+++ b/kernel/loongarch64/smax_lsx.S
@@ -0,0 +1,171 @@
+#define ASSEMBLER
+
+#include "common.h"
+
+#define N     $r4
+#define X     $r5
+#define INCX  $r6
+#define I     $r12
+#define J     $r13
+#define t1    $r14
+#define t2    $r18
+#define t3    $r15
+#define t4    $r17
+#define TEMP  $r16
+#define m0    $vr8
+#define x1    $vr9
+#define x2    $vr10
+#define x3    $vr11
+#define x4    $vr12
+#define VX0   $vr20
+#define VX1   $vr21
+#define VM0   $vr22
+#define VM1   $vr23
+
+    PROLOGUE
+
+    bge $r0, N, .L999
+    bge $r0, INCX, .L999
+    li.d TEMP, 1
+    slli.d TEMP, TEMP, BASE_SHIFT
+    slli.d INCX, INCX, BASE_SHIFT
+    bne INCX, TEMP, .L20
+    vld VM0, X, 0
+    srai.d I, N, 3
+    bge $r0, I, .L12
+    .align 3
+
+.L10:
+    vld VX0, X, 0 * SIZE
+    vld VX1, X, 4 * SIZE
+    addi.d I, I, -1
+    vfmax.s VM1, VX0, VX1
+    addi.d X, X, 8 * SIZE
+    vfmax.s VM0, VM0, VM1
+    blt $r0, I, .L10
+    .align 3
+
+.L11:
+    vreplvei.w x2, VM0, 1
+    vreplvei.w x3, VM0, 2
+    vreplvei.w x4, VM0, 3
+    vfmax.s VM1, x2, x3
+    vfmax.s VM0, x4, VM0
+    vfmax.s VM0, VM0, VM1
+    .align 3
+
+.L12: //INCX==1 and N<8
+    andi I, N, 7
+    li.d J, 4
+    bge J, I, .L13 // 4<N<8
+    vld VX0, X, 0
+    slli.d J, J, 1 // 8
+    sub.d I, J, I
+    slli.d I, I, BASE_SHIFT
+    vldx VX1, X, I
+    vfmax.s m0, VX0, VX1 //patial repeat read
+    vreplvei.w x2, m0, 1
+    vreplvei.w x3, m0, 2
+    vreplvei.w x4, m0, 3
+    vfmax.s x1, x2, x3
+    vfmax.s VM1, x4, m0
+    vfmax.s m0, x1, VM1
+    vfmax.s VM0, m0, VM0
+    fmov.s $f0, $f22
+    jirl $r0, $r1, 0x0
+    .align 3
+
+.L13: //INCX==1 and 0<=N<=4
+    bge $r0, I, .L15
+    .align 3
+
+.L14:
+    vld x1, X, 0
+    addi.d I, I, -1
+    vfmax.s VM0, VM0, x1
+    addi.d  X, X, SIZE
+    blt $r0, I, .L14
+    .align 3
+
+.L15:
+    fmov.s $f0, $f22
+    jirl $r0, $r1, 0x0
+    .align 3
+
+.L20: // INCX!=1
+    move TEMP, X
+    ld.w t1, TEMP, 0 * SIZE
+    add.d TEMP, TEMP, INCX
+    vinsgr2vr.w VM0, t1, 0
+    srai.d I, N, 3
+    bge $r0, I, .L23
+    ld.w t2, TEMP, 0 * SIZE
+    add.d TEMP, TEMP, INCX
+    ld.w t3, TEMP, 0 * SIZE
+    add.d TEMP, TEMP, INCX
+    ld.w t4, TEMP, 0 * SIZE
+    add.d TEMP, TEMP, INCX
+    vinsgr2vr.w VM0, t2, 1
+    vinsgr2vr.w VM0, t3, 2
+    vinsgr2vr.w VM0, t4, 3
+    .align 3
+
+.L21:
+    ld.w t1, X, 0 * SIZE
+    add.d X, X, INCX
+    ld.w t2, X, 0 * SIZE
+    add.d X, X, INCX
+    ld.w t3, X, 0 * SIZE
+    add.d X, X, INCX
+    ld.w t4, X, 0 * SIZE
+    add.d X, X, INCX
+    vinsgr2vr.w VX0, t1, 0
+    vinsgr2vr.w VX0, t2, 1
+    vinsgr2vr.w VX0, t3, 2
+    vinsgr2vr.w VX0, t4, 3
+    ld.w t1, X, 0 * SIZE
+    add.d X, X, INCX
+    ld.w t2, X, 0 * SIZE
+    add.d X, X, INCX
+    ld.w t3, X, 0 * SIZE
+    add.d X, X, INCX
+    ld.w t4, X, 0 * SIZE
+    add.d X, X, INCX
+    vinsgr2vr.w VX1, t1, 0
+    vinsgr2vr.w VX1, t2, 1
+    vinsgr2vr.w VX1, t3, 2
+    vinsgr2vr.w VX1, t4, 3
+    addi.d I, I, -1
+    vfmax.s VM1, VX0, VX1
+    vfmax.s VM0, VM0, VM1
+    blt $r0, I, .L21
+    .align 3
+
+.L22:
+    vreplvei.w x2, VM0, 1
+    vreplvei.w x3, VM0, 2
+    vreplvei.w x4, VM0, 3
+    vfmax.s VM1, x2, x3
+    vfmax.s VM0, x4, VM0
+    vfmax.s VM0, VM0, VM1
+    .align 3
+
+.L23: //INCX!=1 and N<8
+    andi I, N, 7
+    bge $r0, I, .L999
+    .align 3
+
+.L24:
+    vld x1, X, 0
+    addi.d I, I, -1
+    vfmax.s VM0, VM0, x1
+    add.d  X, X, INCX
+    blt $r0, I, .L24
+    .align 3
+
+.L999:
+    fmov.s $f0, $f22
+    jirl $r0, $r1, 0x0
+    .align 3
+
+    EPILOGUE