floraachy
/
OpenBLAS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add files via upload

This commit is contained in:
wjc404 2019-12-30 17:18:59 +08:00 committed by GitHub
parent bb2729c855
commit 700fe5b5ee
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 25 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
kernel/x86_64/zgemm3m_kernel_4x4_haswell.c
View File

@ -18,13 +18,26 @@
#define KERNEL_h_k1m4n4 \ #define KERNEL_h_k1m4n4 \
KERNEL_h_k1m4n2 "vbroadcastf128 16(%1),%%ymm3; vfmadd231pd %%ymm1,%%ymm3,%%ymm6; vfmadd231pd %%ymm2,%%ymm3,%%ymm7;" KERNEL_h_k1m4n2 "vbroadcastf128 16(%1),%%ymm3; vfmadd231pd %%ymm1,%%ymm3,%%ymm6; vfmadd231pd %%ymm2,%%ymm3,%%ymm7;"
#define KERNEL_k1m4n4 KERNEL_h_k1m4n4 "addq $32,%1;" #define KERNEL_k1m4n4 KERNEL_h_k1m4n4 "addq $32,%1;"
#define unit_kernel_k1m4n4(c1,c2,c3,c4,...) \ #define unit_kernel_k1m4n4(c1,c2,c3,c4,off1,off2,...) \
"vbroadcastf128 ("#__VA_ARGS__"),%%ymm3; vfmadd231pd %%ymm1,%%ymm3,"#c1"; vfmadd231pd %%ymm2,%%ymm3,"#c2";"\ "vbroadcastf128 "#off1"("#__VA_ARGS__"),%%ymm3; vfmadd231pd %%ymm1,%%ymm3,"#c1"; vfmadd231pd %%ymm2,%%ymm3,"#c2";"\
"vbroadcastf128 16("#__VA_ARGS__"),%%ymm3; vfmadd231pd %%ymm1,%%ymm3,"#c3"; vfmadd231pd %%ymm2,%%ymm3,"#c4";" "vbroadcastf128 "#off2"("#__VA_ARGS__"),%%ymm3; vfmadd231pd %%ymm1,%%ymm3,"#c3"; vfmadd231pd %%ymm2,%%ymm3,"#c4";"
#define KERNEL_h_k1m4n8 KERNEL_h_k1m4n4 unit_kernel_k1m4n4(%%ymm8,%%ymm9,%%ymm10,%%ymm11,%1,%%r12,1) #define KERNEL_h_k1m4n8 KERNEL_h_k1m4n4 unit_kernel_k1m4n4(%%ymm8,%%ymm9,%%ymm10,%%ymm11,0,16,%1,%%r12,1)
#define KERNEL_k1m4n8 KERNEL_h_k1m4n8 "addq $32,%1;" #define KERNEL_k1m4n8 KERNEL_h_k1m4n8 "addq $32,%1;"
#define KERNEL_h_k1m4n12 KERNEL_h_k1m4n8 unit_kernel_k1m4n4(%%ymm12,%%ymm13,%%ymm14,%%ymm15,%1,%%r12,2) #define KERNEL_h_k1m4n12 KERNEL_h_k1m4n8 unit_kernel_k1m4n4(%%ymm12,%%ymm13,%%ymm14,%%ymm15,0,16,%1,%%r12,2)
#define KERNEL_k1m4n12 KERNEL_h_k1m4n12 "addq $32,%1;" #define KERNEL_k1m4n12 KERNEL_h_k1m4n12 "addq $32,%1;"
#define KERNEL_k2m4n1 KERNEL_k1m4n1 KERNEL_k1m4n1
#define KERNEL_k2m4n2 KERNEL_k1m4n2 KERNEL_k1m4n2
#define KERNEL_k2m4n4 KERNEL_k1m4n4 KERNEL_k1m4n4
#define KERNEL_k2m4n8 KERNEL_k1m4n8 KERNEL_k1m4n8
#define KERNEL_k2m4n12 \
"vmovddup (%0),%%ymm1; vmovddup 8(%0),%%ymm2;"\
unit_kernel_k1m4n4(%%ymm4,%%ymm5,%%ymm6,%%ymm7,0,16,%1)\
unit_kernel_k1m4n4(%%ymm8,%%ymm9,%%ymm10,%%ymm11,0,16,%1,%%r12,1)\
unit_kernel_k1m4n4(%%ymm12,%%ymm13,%%ymm14,%%ymm15,0,16,%1,%%r12,2)\
"vmovddup 32(%0),%%ymm1; vmovddup 40(%0),%%ymm2; prefetcht0 512(%0); addq $64,%0;"\
unit_kernel_k1m4n4(%%ymm4,%%ymm5,%%ymm6,%%ymm7,32,48,%1)\
unit_kernel_k1m4n4(%%ymm8,%%ymm9,%%ymm10,%%ymm11,32,48,%1,%%r12,1)\
unit_kernel_k1m4n4(%%ymm12,%%ymm13,%%ymm14,%%ymm15,32,48,%1,%%r12,2) "addq $64,%1;"
#define INIT_m4n1 "vpxor %%ymm4,%%ymm4,%%ymm4;" #define INIT_m4n1 "vpxor %%ymm4,%%ymm4,%%ymm4;"
#define INIT_m4n2 INIT_m4n1 "vpxor %%ymm5,%%ymm5,%%ymm5;" #define INIT_m4n2 INIT_m4n1 "vpxor %%ymm5,%%ymm5,%%ymm5;"
#define INIT_m4n4 INIT_m4n2 "vpxor %%ymm6,%%ymm6,%%ymm6;vpxor %%ymm7,%%ymm7,%%ymm7;" #define INIT_m4n4 INIT_m4n2 "vpxor %%ymm6,%%ymm6,%%ymm6;vpxor %%ymm7,%%ymm7,%%ymm7;"
@ -53,18 +66,17 @@
"cmpq $24,%4; jb "#ndim"004042f;"\ "cmpq $24,%4; jb "#ndim"004042f;"\
#ndim"004041:\n\t"\ #ndim"004041:\n\t"\
"cmpq $126,%%r15; movq $126,%%r15; cmoveq %3,%%r15;"\ "cmpq $126,%%r15; movq $126,%%r15; cmoveq %3,%%r15;"\
"prefetcht0 512(%0);" KERNEL_k1m4n##ndim KERNEL_k1m4n##ndim\ KERNEL_k2m4n##ndim KERNEL_k2m4n##ndim\
"prefetcht0 512(%0);" KERNEL_k1m4n##ndim KERNEL_k1m4n##ndim\ "prefetcht1 (%5); subq $63,%5;"\
"prefetcht1 (%5); leaq -63(%5,%%r15,1),%5;"\ KERNEL_k2m4n##ndim KERNEL_k2m4n##ndim\
"prefetcht0 512(%0);" KERNEL_k1m4n##ndim KERNEL_k1m4n##ndim\ "addq %%r15,%5; prefetcht1 (%8); addq $32,%8;"\
"prefetcht0 512(%0);" KERNEL_k1m4n##ndim KERNEL_k1m4n##ndim\ "subq $8,%4; cmpq $16,%4; jnb "#ndim"004041b;"\
"prefetcht1 (%8); addq $32,%8;"\
"subq $8,%4; cmpq $24,%4; jnb "#ndim"004041b;"\
"movq %2,%5;"\ "movq %2,%5;"\
#ndim"004042:\n\t"\ #ndim"004042:\n\t"\
"testq %4,%4; jz "#ndim"004043f;"\ "testq %4,%4; jz "#ndim"004043f;"\
"prefetcht0 (%5); prefetcht0 63(%5); addq %3,%5;"\ "prefetcht0 (%5); prefetcht0 63(%5);"\
KERNEL_k1m4n##ndim\ KERNEL_k1m4n##ndim\
"prefetcht0 (%5,%3,4); prefetcht0 63(%5,%3,4); addq %3,%5;"\
"decq %4; jmp "#ndim"004042b;"\ "decq %4; jmp "#ndim"004042b;"\
#ndim"004043:\n\t"\ #ndim"004043:\n\t"\
"prefetcht0 (%%r14); prefetcht0 64(%%r14);"\ "prefetcht0 (%%r14); prefetcht0 64(%%r14);"\