diff --git a/Makefile.system b/Makefile.system
index 20d8d2f2a..20db80d07 100644
--- a/Makefile.system
+++ b/Makefile.system
@@ -245,6 +245,9 @@ ONLY_CBLAS = 0
 endif
 
 #For small matrix optimization
+ifeq ($(ARCH), x86_64)
+SMALL_MATRIX_OPT = 1
+endif
 ifeq ($(SMALL_MATRIX_OPT), 1)
 CCOMMON_OPT += -DSMALL_MATRIX_OPT
 endif
diff --git a/cmake/system.cmake b/cmake/system.cmake
index e51dc1fdc..7d2672998 100644
--- a/cmake/system.cmake
+++ b/cmake/system.cmake
@@ -258,6 +258,9 @@ if (NEED_PIC)
   endif()
 endif ()
 
+if (X86_64)
+  set(SMALL_MATRIX_OPT TRUE)
+endif ()
 if (SMALL_MATRIX_OPT)
   set(CCOMMON_OPT "${CCOMMON_OPT} -DSMALL_MATRIX_OPT")
 endif ()