floraachy/OpenBLAS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'shivam-develop' into shivam-Locks

Browse Source
This commit is contained in:
shivammonaka
2024-02-29 11:58:14 +05:30
parent bc191015e3 577d480c62
commit d49ebc54e1
1821 changed files with 269629 additions and 52620 deletions
Download Patch File Download Diff File Expand all files Collapse all files
driver/others/blas_l1_thread.c
+2 -2
View File
@@ -43,7 +43,7 @@
int blas_level1_thread(int mode, BLASLONG m, BLASLONG n, BLASLONG k, void *alpha,
void *a, BLASLONG lda,
void *b, BLASLONG ldb,
void *c, BLASLONG ldc, int (*function)(), int nthreads){
void *c, BLASLONG ldc, int (*function)(void), int nthreads){
blas_queue_t queue[MAX_CPU_NUMBER];
blas_arg_t args [MAX_CPU_NUMBER];
@@ -141,7 +141,7 @@ int blas_level1_thread(int mode, BLASLONG m, BLASLONG n, BLASLONG k, void *alpha
int blas_level1_thread_with_return_value(int mode, BLASLONG m, BLASLONG n, BLASLONG k, void *alpha,
void *a, BLASLONG lda,
void *b, BLASLONG ldb,
void *c, BLASLONG ldc, int (*function)(), int nthreads){
void *c, BLASLONG ldc, int (*function)(void), int nthreads){
blas_queue_t queue[MAX_CPU_NUMBER];
blas_arg_t args [MAX_CPU_NUMBER];
driver/others/blas_server.c
+4 -2
View File
@@ -93,7 +93,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#endif
#endif
extern unsigned int openblas_thread_timeout();
extern unsigned int openblas_thread_timeout(void);
#ifdef SMP_SERVER
@@ -113,6 +113,8 @@ extern unsigned int openblas_thread_timeout();
/* We need this global for checking if initialization is finished. */
int blas_server_avail __attribute__((aligned(ATTRIBUTE_SIZE))) = 0;
int blas_omp_threads_local = 1;
/* Local Variables */
#if defined(USE_PTHREAD_LOCK)
static pthread_mutex_t server_lock = PTHREAD_MUTEX_INITIALIZER;
@@ -973,7 +975,7 @@ void goto_set_num_threads(int num_threads) {
increased_threads = 1;
for(i = blas_num_threads - 1; i < num_threads - 1; i++){
for(i = (blas_num_threads > 0) ? blas_num_threads - 1 : 0; i < num_threads - 1; i++){
atomic_store_queue(&thread_status[i].queue, (blas_queue_t *)0);
thread_status[i].status = THREAD_STATUS_WAKEUP;
driver/others/blas_server_omp.c
+15 -4
View File
@@ -68,8 +68,10 @@
#endif
int blas_server_avail = 0;
int blas_omp_number_max = 0;
int blas_omp_threads_local = 1;
extern int openblas_omp_adaptive_env();
extern int openblas_omp_adaptive_env(void);
static void * blas_thread_buffer[MAX_PARALLEL_NUMBER][MAX_CPU_NUMBER];
#ifdef HAVE_C11
@@ -78,7 +80,7 @@ static atomic_bool blas_buffer_inuse[MAX_PARALLEL_NUMBER];
static _Bool blas_buffer_inuse[MAX_PARALLEL_NUMBER];
#endif
static void adjust_thread_buffers() {
static void adjust_thread_buffers(void) {
int i=0, j=0;
@@ -100,8 +102,6 @@ static void adjust_thread_buffers() {
void goto_set_num_threads(int num_threads) {
blas_num_threads_set = 1;
if (num_threads < 0) blas_num_threads_set = 0;
if (num_threads < 1) num_threads = blas_num_threads;
if (num_threads > MAX_CPU_NUMBER) num_threads = MAX_CPU_NUMBER;
@@ -126,6 +126,17 @@ void openblas_set_num_threads(int num_threads) {
int blas_thread_init(void){
#if defined(__FreeBSD__) && defined(__clang__)
extern int openblas_omp_num_threads_env(void);
if(blas_omp_number_max <= 0)
blas_omp_number_max= openblas_omp_num_threads_env();
if (blas_omp_number_max <= 0)
blas_omp_number_max=MAX_CPU_NUMBER;
#else
blas_omp_number_max = omp_get_max_threads();
#endif
blas_get_cpu_number();
adjust_thread_buffers();
driver/others/blas_server_win32.c
+187 -164
View File
@@ -48,34 +48,38 @@
#endif
#endif
#ifdef SMP_DEBUG
# define MT_TRACE(...) fprintf(stderr, __VA_ARGS__)
#else
# define MT_TRACE(...)
#endif
/* This is a thread implementation for Win32 lazy implementation */
/* Thread server common information */
typedef struct{
CRITICAL_SECTION lock;
HANDLE filled;
HANDLE killed;
blas_queue_t *queue; /* Parameter Pointer */
int shutdown; /* server shutdown flag */
} blas_pool_t;
static blas_queue_t *work_queue = NULL;
static HANDLE kickoff_event = NULL;
static CRITICAL_SECTION queue_lock;
/* We need this global for checking if initialization is finished. */
int blas_server_avail = 0;
int blas_omp_threads_local = 1;
/* Local Variables */
static BLASULONG server_lock = 0;
static blas_pool_t pool;
static HANDLE blas_threads [MAX_CPU_NUMBER];
static DWORD blas_threads_id[MAX_CPU_NUMBER];
static volatile int thread_target; // target num of live threads, volatile for cross-thread reads
//
// Legacy code path
//
static void legacy_exec(void *func, int mode, blas_arg_t *args, void *sb) {
static void legacy_exec(void *func, int mode, blas_arg_t *args, void *sb){
if (!(mode & BLAS_COMPLEX)){
if (!(mode & BLAS_COMPLEX)) {
#ifdef EXPRECISION
if ((mode & BLAS_PREC) == BLAS_XDOUBLE){
/* REAL / Extended Double */
@@ -90,7 +94,7 @@ static void legacy_exec(void *func, int mode, blas_arg_t *args, void *sb){
args -> c, args -> ldc, sb);
} else
#endif
if ((mode & BLAS_PREC) == BLAS_DOUBLE){
if ((mode & BLAS_PREC) == BLAS_DOUBLE) {
/* REAL / Double */
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, double,
double *, BLASLONG, double *, BLASLONG,
@@ -101,7 +105,7 @@ static void legacy_exec(void *func, int mode, blas_arg_t *args, void *sb){
args -> a, args -> lda,
args -> b, args -> ldb,
args -> c, args -> ldc, sb);
} else if ((mode & BLAS_PREC) == BLAS_SINGLE){
} else if ((mode & BLAS_PREC) == BLAS_SINGLE) {
/* REAL / Single */
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, float,
float *, BLASLONG, float *, BLASLONG,
@@ -113,7 +117,7 @@ static void legacy_exec(void *func, int mode, blas_arg_t *args, void *sb){
args -> b, args -> ldb,
args -> c, args -> ldc, sb);
#ifdef BUILD_BFLOAT16
} else if ((mode & BLAS_PREC) == BLAS_BFLOAT16){
} else if ((mode & BLAS_PREC) == BLAS_BFLOAT16) {
/* REAL / BFLOAT16 */
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, bfloat16,
bfloat16 *, BLASLONG, bfloat16 *, BLASLONG,
@@ -124,7 +128,7 @@ static void legacy_exec(void *func, int mode, blas_arg_t *args, void *sb){
args -> a, args -> lda,
args -> b, args -> ldb,
args -> c, args -> ldc, sb);
} else if ((mode & BLAS_PREC) == BLAS_STOBF16){
} else if ((mode & BLAS_PREC) == BLAS_STOBF16) {
/* REAL / BLAS_STOBF16 */
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, float,
float *, BLASLONG, bfloat16 *, BLASLONG,
@@ -135,7 +139,7 @@ static void legacy_exec(void *func, int mode, blas_arg_t *args, void *sb){
args -> a, args -> lda,
args -> b, args -> ldb,
args -> c, args -> ldc, sb);
} else if ((mode & BLAS_PREC) == BLAS_DTOBF16){
} else if ((mode & BLAS_PREC) == BLAS_DTOBF16) {
/* REAL / BLAS_DTOBF16 */
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, double,
double *, BLASLONG, bfloat16 *, BLASLONG,
@@ -152,7 +156,7 @@ static void legacy_exec(void *func, int mode, blas_arg_t *args, void *sb){
}
} else {
#ifdef EXPRECISION
if ((mode & BLAS_PREC) == BLAS_XDOUBLE){
if ((mode & BLAS_PREC) == BLAS_XDOUBLE) {
/* COMPLEX / Extended Double */
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble,
xdouble *, BLASLONG, xdouble *, BLASLONG,
@@ -166,7 +170,7 @@ static void legacy_exec(void *func, int mode, blas_arg_t *args, void *sb){
args -> c, args -> ldc, sb);
} else
#endif
if ((mode & BLAS_PREC) == BLAS_DOUBLE){
if ((mode & BLAS_PREC) == BLAS_DOUBLE) {
/* COMPLEX / Double */
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, double, double,
double *, BLASLONG, double *, BLASLONG,
@@ -196,88 +200,78 @@ static void legacy_exec(void *func, int mode, blas_arg_t *args, void *sb){
}
}
/* This is a main routine of threads. Each thread waits until job is */
/* queued. */
static DWORD WINAPI blas_thread_server(void *arg){
//
// This is a main routine of threads. Each thread waits until job is queued.
//
static DWORD WINAPI blas_thread_server(void *arg) {
/* Thread identifier */
#ifdef SMP_DEBUG
BLASLONG cpu = (BLASLONG)arg;
#endif
void *buffer, *sa, *sb;
blas_queue_t *queue;
DWORD action;
HANDLE handles[] = {pool.filled, pool.killed};
/* Each server needs each buffer */
buffer = blas_memory_alloc(2);
#ifdef SMP_DEBUG
fprintf(STDERR, "Server[%2ld] Thread is started!\n", cpu);
#endif
MT_TRACE("Server[%2ld] Thread is started!\n", cpu);
while (1){
while (1) {
/* Waiting for Queue */
#ifdef SMP_DEBUG
fprintf(STDERR, "Server[%2ld] Waiting for Queue.\n", cpu);
#endif
MT_TRACE("Server[%2ld] Waiting for Queue.\n", cpu);
do {
action = WaitForMultipleObjects(2, handles, FALSE, INFINITE);
} while ((action != WAIT_OBJECT_0) && (action != WAIT_OBJECT_0 + 1));
// event raised when work is added to the queue
WaitForSingleObject(kickoff_event, INFINITE);
if (action == WAIT_OBJECT_0 + 1) break;
if (cpu > thread_target - 2) {
//MT_TRACE("thread [%d] exiting.\n", cpu);
break; // excess thread, so worker thread exits
}
#ifdef SMP_DEBUG
fprintf(STDERR, "Server[%2ld] Got it.\n", cpu);
#endif
MT_TRACE("Server[%2ld] Got it.\n", cpu);
EnterCriticalSection(&pool.lock);
EnterCriticalSection(&queue_lock);
queue = pool.queue;
if (queue) pool.queue = queue->next;
queue = work_queue;
if (queue)
work_queue = work_queue->next;
LeaveCriticalSection(&pool.lock);
LeaveCriticalSection(&queue_lock);
if (queue) {
if (queue) {
int (*routine)(blas_arg_t *, void *, void *, void *, void *, BLASLONG) = queue -> routine;
if (pool.queue) SetEvent(pool.filled);
sa = queue -> sa;
sb = queue -> sb;
#ifdef CONSISTENT_FPCSR
__asm__ __volatile__ ("ldmxcsr %0" : : "m" (queue -> sse_mode));
__asm__ __volatile__ ("fldcw %0" : : "m" (queue -> x87_mode));
#endif
#ifdef CONSISTENT_FPCSR
__asm__ __volatile__ ("ldmxcsr %0" : : "m" (queue -> sse_mode));
__asm__ __volatile__ ("fldcw %0" : : "m" (queue -> x87_mode));
#endif
#ifdef SMP_DEBUG
fprintf(STDERR, "Server[%2ld] Started. Mode = 0x%03x M = %3ld N=%3ld K=%3ld\n",
MT_TRACE("Server[%2ld] Started. Mode = 0x%03x M = %3ld N=%3ld K=%3ld\n",
cpu, queue->mode, queue-> args ->m, queue->args->n, queue->args->k);
#endif
// fprintf(stderr, "queue start[%ld]!!!\n", cpu);
#ifdef MONITOR
main_status[cpu] = MAIN_RUNNING1;
#endif
#ifdef MONITOR
main_status[cpu] = MAIN_RUNNING1;
#endif
if (sa == NULL) sa = (void *)((BLASLONG)buffer + GEMM_OFFSET_A);
if (sa == NULL)
sa = (void *)((BLASLONG)buffer + GEMM_OFFSET_A);
if (sb == NULL) {
if (!(queue -> mode & BLAS_COMPLEX)){
if (!(queue -> mode & BLAS_COMPLEX)) {
#ifdef EXPRECISION
if ((queue -> mode & BLAS_PREC) == BLAS_XDOUBLE){
if ((queue -> mode & BLAS_PREC) == BLAS_XDOUBLE) {
sb = (void *)(((BLASLONG)sa + ((XGEMM_P * XGEMM_Q * sizeof(xdouble)
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
} else
#endif
if ((queue -> mode & BLAS_PREC) == BLAS_DOUBLE){
if ((queue -> mode & BLAS_PREC) == BLAS_DOUBLE) {
#ifdef BUILD_DOUBLE
sb = (void *)(((BLASLONG)sa + ((DGEMM_P * DGEMM_Q * sizeof(double)
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
@@ -311,70 +305,59 @@ static DWORD WINAPI blas_thread_server(void *arg){
/* Other types in future */
}
}
queue->sb=sb;
queue->sb=sb;
}
#ifdef MONITOR
main_status[cpu] = MAIN_RUNNING2;
#endif
#ifdef MONITOR
main_status[cpu] = MAIN_RUNNING2;
#endif
if (!(queue -> mode & BLAS_LEGACY)) {
(routine)(queue -> args, queue -> range_m, queue -> range_n, sa, sb, queue -> position);
(routine)(queue -> args, queue -> range_m, queue -> range_n, sa, sb, queue -> position);
} else {
legacy_exec(routine, queue -> mode, queue -> args, sb);
legacy_exec(routine, queue -> mode, queue -> args, sb);
}
}else{
continue; //if queue == NULL
}
} else {
continue; //if queue == NULL
}
#ifdef SMP_DEBUG
fprintf(STDERR, "Server[%2ld] Finished!\n", cpu);
#endif
EnterCriticalSection(&queue->lock);
queue -> status = BLAS_STATUS_FINISHED;
LeaveCriticalSection(&queue->lock);
SetEvent(queue->finish);
MT_TRACE("Server[%2ld] Finished!\n", cpu);
queue->finished = 1;
}
/* Shutdown procedure */
#ifdef SMP_DEBUG
fprintf(STDERR, "Server[%2ld] Shutdown!\n", cpu);
#endif
MT_TRACE("Server[%2ld] Shutdown!\n", cpu);
blas_memory_free(buffer);
return 0;
}
}
/* Initializing routine */
int blas_thread_init(void){
//
// Initializing routine
//
int blas_thread_init(void) {
BLASLONG i;
if (blas_server_avail || (blas_cpu_number <= 1)) return 0;
LOCK_COMMAND(&server_lock);
#ifdef SMP_DEBUG
fprintf(STDERR, "Initializing Thread(Num. threads = %d)\n",
blas_cpu_number);
#endif
MT_TRACE("Initializing Thread(Num. threads = %d)\n", blas_cpu_number);
if (!blas_server_avail){
if (!blas_server_avail) {
// create the kickoff Event
kickoff_event = CreateEvent(NULL, TRUE, FALSE, NULL);
InitializeCriticalSection(&pool.lock);
pool.filled = CreateEvent(NULL, FALSE, FALSE, NULL);
pool.killed = CreateEvent(NULL, TRUE, FALSE, NULL);
thread_target = blas_cpu_number;
pool.shutdown = 0;
pool.queue = NULL;
InitializeCriticalSection(&queue_lock);
for(i = 0; i < blas_cpu_number - 1; i++) {
//MT_TRACE("thread_init: creating thread [%d]\n", i);
for(i = 0; i < blas_cpu_number - 1; i++){
blas_threads[i] = CreateThread(NULL, 0,
blas_thread_server, (void *)i,
0, &blas_threads_id[i]);
@@ -388,15 +371,12 @@ int blas_thread_init(void){
return 0;
}
/*
User can call one of two routines.
exec_blas_async ... immediately returns after jobs are queued.
exec_blas ... returns after jobs are finished.
*/
int exec_blas_async(BLASLONG pos, blas_queue_t *queue){
//
// User can call one of two routines.
// exec_blas_async ... immediately returns after jobs are queued.
// exec_blas ... returns after jobs are finished.
//
int exec_blas_async(BLASLONG pos, blas_queue_t *queue) {
#if defined(SMP_SERVER)
// Handle lazy re-init of the thread-pool after a POSIX fork
@@ -409,8 +389,6 @@ int exec_blas_async(BLASLONG pos, blas_queue_t *queue){
current = queue;
while (current) {
InitializeCriticalSection(&current -> lock);
current -> finish = CreateEvent(NULL, FALSE, FALSE, NULL);
current -> position = pos;
#ifdef CONSISTENT_FPCSR
@@ -418,56 +396,71 @@ int exec_blas_async(BLASLONG pos, blas_queue_t *queue){
__asm__ __volatile__ ("stmxcsr %0" : "=m" (current -> sse_mode));
#endif
current->finished = 0;
current = current -> next;
pos ++;
}
EnterCriticalSection(&pool.lock);
EnterCriticalSection(&queue_lock);
if (pool.queue) {
current = pool.queue;
while (current -> next) current = current -> next;
current -> next = queue;
} else {
pool.queue = queue;
if (!work_queue)
{
work_queue = queue;
}
else
{
blas_queue_t *next_item = work_queue;
// find the end of the work queue
while (next_item)
next_item = next_item->next;
// add new work to the end
next_item = queue;
}
LeaveCriticalSection(&pool.lock);
LeaveCriticalSection(&queue_lock);
SetEvent(pool.filled);
SetEvent(kickoff_event);
return 0;
}
int exec_blas_async_wait(BLASLONG num, blas_queue_t *queue){
//
// Join. Wait for all queued tasks to complete
//
int exec_blas_async_wait(BLASLONG num, blas_queue_t *queue) {
#ifdef SMP_DEBUG
fprintf(STDERR, "Synchronization Waiting.\n");
#endif
MT_TRACE("Synchronization Waiting.\n");
while (num){
#ifdef SMP_DEBUG
fprintf(STDERR, "Waiting Queue ..\n");
#endif
while (num) {
MT_TRACE("Waiting Queue ..\n");
WaitForSingleObject(queue->finish, INFINITE);
while (!queue->finished)
YIELDING;
CloseHandle(queue->finish);
DeleteCriticalSection(&queue -> lock);
queue = queue->next;
num--;
}
queue = queue -> next;
num --;
}
MT_TRACE("Completely Done.\n\n");
#ifdef SMP_DEBUG
fprintf(STDERR, "Completely Done.\n\n");
#endif
// if work was added to the queue after this batch we can't sleep the worker threads
// by resetting the event
EnterCriticalSection(&queue_lock);
return 0;
if (work_queue == NULL)
ResetEvent(kickoff_event);
LeaveCriticalSection(&queue_lock);
return 0;
}
/* Execute Threads */
int exec_blas(BLASLONG num, blas_queue_t *queue){
//
// Execute Threads
//
int exec_blas(BLASLONG num, blas_queue_t *queue) {
#if defined(SMP_SERVER) && defined(OS_CYGWIN_NT)
// Handle lazy re-init of the thread-pool after a POSIX fork
@@ -480,29 +473,33 @@ int exec_blas(BLASLONG num, blas_queue_t *queue){
if ((num <= 0) || (queue == NULL)) return 0;
if ((num > 1) && queue -> next) exec_blas_async(1, queue -> next);
if ((num > 1) && queue -> next)
exec_blas_async(1, queue -> next);
routine = queue -> routine;
if (queue -> mode & BLAS_LEGACY) {
legacy_exec(routine, queue -> mode, queue -> args, queue -> sb);
} else
} else {
if (queue -> mode & BLAS_PTHREAD) {
void (*pthreadcompat)(void *) = queue -> routine;
(pthreadcompat)(queue -> args);
} else
(routine)(queue -> args, queue -> range_m, queue -> range_n,
queue -> sa, queue -> sb, 0);
queue -> sa, queue -> sb, 0);
}
if ((num > 1) && queue -> next) exec_blas_async_wait(num - 1, queue -> next);
if ((num > 1) && queue -> next)
exec_blas_async_wait(num - 1, queue -> next);
return 0;
}
/* Shutdown procedure, but user don't have to call this routine. The */
/* kernel automatically kill threads. */
int BLASFUNC(blas_thread_shutdown)(void){
//
// Shutdown procedure, but user don't have to call this routine. The
// kernel automatically kill threads.
//
int BLASFUNC(blas_thread_shutdown)(void) {
int i;
@@ -510,11 +507,9 @@ int BLASFUNC(blas_thread_shutdown)(void){
LOCK_COMMAND(&server_lock);
if (blas_server_avail){
if (blas_server_avail) {
SetEvent(pool.killed);
for(i = 0; i < blas_num_threads - 1; i++){
for (i = 0; i < blas_num_threads - 1; i++) {
// Could also just use WaitForMultipleObjects
DWORD wait_thread_value = WaitForSingleObject(blas_threads[i], 50);
@@ -528,9 +523,6 @@ int BLASFUNC(blas_thread_shutdown)(void){
CloseHandle(blas_threads[i]);
}
CloseHandle(pool.filled);
CloseHandle(pool.killed);
blas_server_avail = 0;
}
@@ -539,6 +531,9 @@ int BLASFUNC(blas_thread_shutdown)(void){
return 0;
}
//
// Legacy function to set numbef of threads
//
void goto_set_num_threads(int num_threads)
{
long i;
@@ -552,23 +547,48 @@ void goto_set_num_threads(int num_threads)
if (num_threads > MAX_CPU_NUMBER) num_threads = MAX_CPU_NUMBER;
if (blas_server_avail && num_threads < blas_num_threads) {
LOCK_COMMAND(&server_lock);
thread_target = num_threads;
SetEvent(kickoff_event);
for (i = num_threads - 1; i < blas_num_threads - 1; i++) {
//MT_TRACE("set_num_threads: waiting on thread [%d] to quit.\n", i);
WaitForSingleObject(blas_threads[i], INFINITE);
//MT_TRACE("set_num_threads: thread [%d] has quit.\n", i);
CloseHandle(blas_threads[i]);
}
blas_num_threads = num_threads;
ResetEvent(kickoff_event);
UNLOCK_COMMAND(&server_lock);
}
if (num_threads > blas_num_threads) {
LOCK_COMMAND(&server_lock);
//increased_threads = 1;
if (!blas_server_avail){
thread_target = num_threads;
InitializeCriticalSection(&pool.lock);
pool.filled = CreateEvent(NULL, FALSE, FALSE, NULL);
pool.killed = CreateEvent(NULL, TRUE, FALSE, NULL);
//increased_threads = 1;
if (!blas_server_avail) {
// create the kickoff Event
kickoff_event = CreateEvent(NULL, TRUE, FALSE, NULL);
InitializeCriticalSection(&queue_lock);
pool.shutdown = 0;
pool.queue = NULL;
blas_server_avail = 1;
}
for(i = blas_num_threads - 1; i < num_threads - 1; i++){
for (i = (blas_num_threads > 0) ? blas_num_threads - 1 : 0; i < num_threads - 1; i++) {
//MT_TRACE("set_num_threads: creating thread [%d]\n", i);
blas_threads[i] = CreateThread(NULL, 0,
blas_thread_server, (void *)i,
@@ -583,6 +603,9 @@ void goto_set_num_threads(int num_threads)
blas_cpu_number = num_threads;
}
//
// Openblas function to set thread count
//
void openblas_set_num_threads(int num)
{
goto_set_num_threads(num);
driver/others/dynamic.c
+49 -2
View File
@@ -220,6 +220,19 @@ extern gotoblas_t gotoblas_COOPERLAKE;
#else
#define gotoblas_COOPERLAKE gotoblas_PRESCOTT
#endif
#ifdef DYN_SAPPHIRERAPIDS
extern gotoblas_t gotoblas_SAPPHIRERAPIDS;
#elif defined(DYN_SKYLAKEX)
#define gotoblas_SAPPHIRERAPIDS gotoblas_SKYLAKEX
#elif defined(DYN_HASWELL)
#define gotoblas_SAPPHIRERAPIDS gotoblas_HASWELL
#elif defined(DYN_SANDYBRIDGE)
#define gotoblas_SAPPHIRERAPIDS gotoblas_SANDYBRIDGE
#elif defined(DYN_NEHALEM)
#define gotoblas_SAPPHIRERAPIDS gotoblas_NEHALEM
#else
#define gotoblas_SAPPHIRERAPIDS gotoblas_PRESCOTT
#endif
#else // not DYNAMIC_LIST
@@ -262,15 +275,18 @@ extern gotoblas_t gotoblas_EXCAVATOR;
#define gotoblas_SKYLAKEX gotoblas_SANDYBRIDGE
#define gotoblas_COOPERLAKE gotoblas_SANDYBRIDGE
#define gotoblas_ZEN gotoblas_SANDYBRIDGE
#define gotoblas_SAPPHIRERAPIDS gotoblas_SANDYBRIDGE
#else
extern gotoblas_t gotoblas_HASWELL;
extern gotoblas_t gotoblas_ZEN;
#ifndef NO_AVX512
extern gotoblas_t gotoblas_SKYLAKEX;
extern gotoblas_t gotoblas_COOPERLAKE;
extern gotoblas_t gotoblas_SAPPHIRERAPIDS;
#else
#define gotoblas_SKYLAKEX gotoblas_HASWELL
#define gotoblas_COOPERLAKE gotoblas_HASWELL
#define gotoblas_SAPPHIRERAPIDS gotoblas_HASWELL
#endif
#endif
#else
@@ -279,6 +295,7 @@ extern gotoblas_t gotoblas_COOPERLAKE;
#define gotoblas_HASWELL gotoblas_NEHALEM
#define gotoblas_SKYLAKEX gotoblas_NEHALEM
#define gotoblas_COOPERLAKE gotoblas_NEHALEM
#define gotoblas_SAPPHIRERAPIDS gotoblas_NEHALEM
#define gotoblas_BULLDOZER gotoblas_BARCELONA
#define gotoblas_PILEDRIVER gotoblas_BARCELONA
#define gotoblas_STEAMROLLER gotoblas_BARCELONA
@@ -378,6 +395,31 @@ int support_avx512_bf16(){
#endif
}
#define BIT_AMX_TILE 0x01000000
#define BIT_AMX_BF16 0x00400000
#define BIT_AMX_ENBD 0x00060000
int support_amx_bf16() {
#if !defined(NO_AVX) && !defined(NO_AVX512)
int eax, ebx, ecx, edx;
int ret=0;
if (!support_avx512())
return 0;
// CPUID.7.0:EDX indicates AMX support
cpuid_count(7, 0, &eax, &ebx, &ecx, &edx);
if ((edx & BIT_AMX_TILE) && (edx & BIT_AMX_BF16)) {
// CPUID.D.0:EAX[17:18] indicates AMX enabled
cpuid_count(0xd, 0, &eax, &ebx, &ecx, &edx);
if ((eax & BIT_AMX_ENBD) == BIT_AMX_ENBD)
ret = 1;
}
return ret;
#else
return 0;
#endif
}
extern void openblas_warning(int verbose, const char * msg);
#define FALLBACK_VERBOSE 1
#define NEHALEM_FALLBACK "OpenBLAS : Your OS does not support AVX instructions. OpenBLAS is using Nehalem kernels as a fallback, which may give poorer performance.\n"
@@ -689,6 +731,8 @@ static gotoblas_t *get_coretype(void){
}
}
if (model == 15){ // Sapphire Rapids
if(support_amx_bf16())
return &gotoblas_SAPPHIRERAPIDS;
if(support_avx512_bf16())
return &gotoblas_COOPERLAKE;
if (support_avx512())
@@ -762,7 +806,8 @@ static gotoblas_t *get_coretype(void){
}
return NULL;
}
case 0xf:
break;
case 0xf:
if (model <= 0x2) return &gotoblas_NORTHWOOD;
return &gotoblas_PRESCOTT;
}
@@ -941,7 +986,8 @@ static char *corename[] = {
"Excavator",
"Zen",
"SkylakeX",
"Cooperlake"
"Cooperlake",
"SapphireRapids"
};
char *gotoblas_corename(void) {
@@ -1006,6 +1052,7 @@ char *gotoblas_corename(void) {
if (gotoblas == &gotoblas_ZEN) return corename[23];
if (gotoblas == &gotoblas_SKYLAKEX) return corename[24];
if (gotoblas == &gotoblas_COOPERLAKE) return corename[25];
if (gotoblas == &gotoblas_SAPPHIRERAPIDS) return corename[26];
return corename[0];
}
driver/others/dynamic_arm64.c
+61 -13
View File
@@ -1,5 +1,6 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* Copyright 2023-2024 The OpenBLAS Project */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
@@ -109,6 +110,11 @@ extern gotoblas_t gotoblas_NEOVERSEN2;
#else
#define gotoblas_NEOVERSEN2 gotoblas_ARMV8
#endif
#ifdef DYN_ARMV8SVE
extern gotoblas_t gotoblas_ARMV8SVE;
#else
#define gotoblas_ARMV8SVE gotoblas_ARMV8
#endif
#ifdef DYN_CORTEX_A55
extern gotoblas_t gotoblas_CORTEXA55;
#else
@@ -116,10 +122,11 @@ extern gotoblas_t gotoblas_CORTEXA55;
#endif
#else
extern gotoblas_t gotoblas_CORTEXA53;
#define gotoblas_CORTEXA55 gotoblas_CORTEXA53
extern gotoblas_t gotoblas_CORTEXA57;
extern gotoblas_t gotoblas_CORTEXA72;
extern gotoblas_t gotoblas_CORTEXA73;
extern gotoblas_t gotoblas_FALKOR;
#define gotoblas_CORTEXA72 gotoblas_CORTEXA57
#define gotoblas_CORTEXA73 gotoblas_CORTEXA57
#define gotoblas_FALKOR gotoblas_CORTEXA57
extern gotoblas_t gotoblas_THUNDERX;
extern gotoblas_t gotoblas_THUNDERX2T99;
extern gotoblas_t gotoblas_TSV110;
@@ -128,17 +135,21 @@ extern gotoblas_t gotoblas_NEOVERSEN1;
#ifndef NO_SVE
extern gotoblas_t gotoblas_NEOVERSEV1;
extern gotoblas_t gotoblas_NEOVERSEN2;
extern gotoblas_t gotoblas_ARMV8SVE;
#else
#define gotoblas_NEOVERSEV1 gotoblas_ARMV8
#define gotoblas_NEOVERSEN2 gotoblas_ARMV8
#define gotoblas_ARMV8SVE gotoblas_ARMV8
#endif
extern gotoblas_t gotoblas_THUNDERX3T110;
extern gotoblas_t gotoblas_CORTEXA55;
#endif
#define gotoblas_NEOVERSEV2 gotoblas_NEOVERSEV1
extern void openblas_warning(int verbose, const char * msg);
#define FALLBACK_VERBOSE 1
#define NEOVERSEN1_FALLBACK "OpenBLAS : Your OS does not support SVE instructions. OpenBLAS is using Neoverse N1 kernels as a fallback, which may give poorer performance.\n"
#define NUM_CORETYPES 13
#define NUM_CORETYPES 17
/*
* In case asm/hwcap.h is outdated on the build system, make sure
@@ -147,6 +158,9 @@ extern void openblas_warning(int verbose, const char * msg);
#ifndef HWCAP_CPUID
#define HWCAP_CPUID (1 << 11)
#endif
#ifndef HWCAP_SVE
#define HWCAP_SVE (1 << 22)
#endif
#define get_cpu_ftr(id, var) ({ \
__asm__ __volatile__ ("mrs %0, "#id : "=r" (var)); \
@@ -165,9 +179,11 @@ static char *corename[] = {
"emag8180",
"neoversen1",
"neoversev1",
"neoversev2",
"neoversen2",
"thunderx3t110",
"cortexa55",
"armv8sve",
"unknown"
};
@@ -184,9 +200,11 @@ char *gotoblas_corename(void) {
if (gotoblas == &gotoblas_EMAG8180) return corename[ 9];
if (gotoblas == &gotoblas_NEOVERSEN1) return corename[10];
if (gotoblas == &gotoblas_NEOVERSEV1) return corename[11];
if (gotoblas == &gotoblas_NEOVERSEN2) return corename[12];
if (gotoblas == &gotoblas_THUNDERX3T110) return corename[13];
if (gotoblas == &gotoblas_CORTEXA55) return corename[14];
if (gotoblas == &gotoblas_NEOVERSEV2) return corename[12];
if (gotoblas == &gotoblas_NEOVERSEN2) return corename[13];
if (gotoblas == &gotoblas_THUNDERX3T110) return corename[14];
if (gotoblas == &gotoblas_CORTEXA55) return corename[15];
if (gotoblas == &gotoblas_ARMV8SVE) return corename[16];
return corename[NUM_CORETYPES];
}
@@ -218,9 +236,11 @@ static gotoblas_t *force_coretype(char *coretype) {
case 9: return (&gotoblas_EMAG8180);
case 10: return (&gotoblas_NEOVERSEN1);
case 11: return (&gotoblas_NEOVERSEV1);
case 12: return (&gotoblas_NEOVERSEN2);
case 13: return (&gotoblas_THUNDERX3T110);
case 14: return (&gotoblas_CORTEXA55);
case 12: return (&gotoblas_NEOVERSEV2);
case 13: return (&gotoblas_NEOVERSEN2);
case 14: return (&gotoblas_THUNDERX3T110);
case 15: return (&gotoblas_CORTEXA55);
case 16: return (&gotoblas_ARMV8SVE);
}
snprintf(message, 128, "Core not found: %s\n", coretype);
openblas_warning(1, message);
@@ -231,6 +251,10 @@ static gotoblas_t *get_coretype(void) {
int implementer, variant, part, arch, revision, midr_el1;
char coremsg[128];
#if defined (OS_DARWIN)
return &gotoblas_NEOVERSEN1;
#endif
#if (!defined OS_LINUX && !defined OS_ANDROID)
return NULL;
#else
@@ -281,9 +305,24 @@ static gotoblas_t *get_coretype(void) {
return &gotoblas_NEOVERSEN1;
#ifndef NO_SVE
case 0xd49:
return &gotoblas_NEOVERSEN2;
if (!(getauxval(AT_HWCAP) & HWCAP_SVE)) {
openblas_warning(FALLBACK_VERBOSE, NEOVERSEN1_FALLBACK);
return &gotoblas_NEOVERSEN1;
} else
return &gotoblas_NEOVERSEN2;
case 0xd40:
return &gotoblas_NEOVERSEV1;
if (!(getauxval(AT_HWCAP) & HWCAP_SVE)) {
openblas_warning(FALLBACK_VERBOSE, NEOVERSEN1_FALLBACK);
return &gotoblas_NEOVERSEN1;
}else
return &gotoblas_NEOVERSEV1;
case 0xd4f:
if (!(getauxval(AT_HWCAP) & HWCAP_SVE)) {
openblas_warning(FALLBACK_VERBOSE, NEOVERSEN1_FALLBACK);
return &gotoblas_NEOVERSEN1;
} else {
return &gotoblas_NEOVERSEV2;
}
#endif
case 0xd05: // Cortex A55
return &gotoblas_CORTEXA55;
@@ -328,10 +367,19 @@ static gotoblas_t *get_coretype(void) {
return &gotoblas_FALKOR;
}
break;
case 0x61: // Apple
return &gotoblas_NEOVERSEN1;
break;
default:
snprintf(coremsg, 128, "Unknown CPU model - implementer %x part %x\n",implementer,part);
openblas_warning(1, coremsg);
}
#ifndef NO_SVE
if ((getauxval(AT_HWCAP) & HWCAP_SVE)) {
return &gotoblas_ARMV8SVE;
}
#endif
return NULL;
#endif
}
driver/others/dynamic_loongarch64.c
+6 -11
View File
@@ -25,6 +25,7 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*******************************************************************************/
#include <sys/auxv.h>
#include "common.h"
extern gotoblas_t gotoblas_LOONGSON3R5;
@@ -74,21 +75,15 @@ static gotoblas_t *force_coretype(char *coretype) {
return NULL;
}
#define LASX_MASK 1<<7
#define LSX_MASK 1<<6
#define LOONGARCH_CFG2 0x02
#define LA_HWCAP_LSX (1U << 4)
#define LA_HWCAP_LASX (1U << 5)
static gotoblas_t *get_coretype(void) {
int ret = 0;
__asm__ volatile (
"cpucfg %0, %1 \n\t"
: "+&r"(ret)
: "r"(LOONGARCH_CFG2)
);
int hwcap = (int)getauxval(AT_HWCAP);
if (ret & LASX_MASK)
if (hwcap & LA_HWCAP_LASX)
return &gotoblas_LOONGSON3R5;
else if (ret & LSX_MASK)
else if (hwcap & LA_HWCAP_LSX)
return &gotoblas_LOONGSON2K1000;
else
return &gotoblas_LOONGSONGENERIC;
driver/others/dynamic_power.c
+97 -29
View File
@@ -3,7 +3,7 @@
extern gotoblas_t gotoblas_POWER6;
extern gotoblas_t gotoblas_POWER8;
#if (!defined __GNUC__) || ( __GNUC__ >= 6)
#if ((!defined __GNUC__) || ( __GNUC__ >= 6)) || defined(__clang__)
extern gotoblas_t gotoblas_POWER9;
#endif
#ifdef HAVE_P10_SUPPORT
@@ -20,14 +20,14 @@ static char *corename[] = {
"POWER10"
};
#define NUM_CORETYPES 4
#define NUM_CORETYPES 5
char *gotoblas_corename(void) {
#ifndef C_PGI
if (gotoblas == &gotoblas_POWER6) return corename[1];
#endif
if (gotoblas == &gotoblas_POWER8) return corename[2];
#if (!defined __GNUC__) || ( __GNUC__ >= 6)
#if ((!defined __GNUC__) || ( __GNUC__ >= 6)) || defined(__clang__)
if (gotoblas == &gotoblas_POWER9) return corename[3];
#endif
#ifdef HAVE_P10_SUPPORT
@@ -36,14 +36,44 @@ char *gotoblas_corename(void) {
return corename[0];
}
#if defined(__clang__)
static int __builtin_cpu_supports(char* arg)
{
return 0;
}
#define CPU_UNKNOWN 0
#define CPU_POWER5 5
#define CPU_POWER6 6
#define CPU_POWER8 8
#define CPU_POWER9 9
#define CPU_POWER10 10
#ifndef POWER_9
#define POWER_9 0x20000 /* 9 class CPU */
#endif
#ifndef POWER_10
#define POWER_10 0x40000 /* 10 class CPU */
#endif
#if defined(C_PGI) || defined(__clang__)
#ifdef _AIX
#include <sys/systemcfg.h>
static int cpuid(void)
{
int arch = _system_configuration.implementation;
#ifdef POWER_6
if (arch == POWER_6) return CPU_POWER6;
#endif
#ifdef POWER_7
else if (arch == POWER_7) return CPU_POWER6;
#endif
#ifdef POWER_8
else if (arch == POWER_8) return CPU_POWER8;
#endif
#ifdef POWER_9
else if (arch == POWER_9) return CPU_POWER9;
#endif
#ifdef POWER_10
else if (arch >= POWER_10) return CPU_POWER10;
#endif
return CPU_UNKNOWN;
}
#elif defined(C_PGI) || defined(__clang__)
/*
* NV HPC compilers do not yet implement __builtin_cpu_is().
* Fake a version here for use in the CPU detection code below.
@@ -53,21 +83,12 @@ static int __builtin_cpu_supports(char* arg)
* what was requested.
*/
#include <string.h>
/*
* Define POWER processor version table.
*
* NOTE NV HPC SDK compilers only support POWER8 and POWER9 at this time
*/
#define CPU_UNKNOWN 0
#define CPU_POWER5 5
#define CPU_POWER6 6
#define CPU_POWER8 8
#define CPU_POWER9 9
#define CPU_POWER10 10
static struct {
uint32_t pvr_mask;
uint32_t pvr_value;
@@ -160,7 +181,8 @@ static struct {
},
};
static int __builtin_cpu_is(const char *cpu) {
static int cpuid(void)
{
int i;
uint32_t pvr;
uint32_t cpu_type;
@@ -178,15 +200,54 @@ static int __builtin_cpu_is(const char *cpu) {
pvrPOWER[i].cpu_name, pvrPOWER[i].cpu_type);
#endif
cpu_type = pvrPOWER[i].cpu_type;
if (!strcmp(cpu, "power8"))
return cpu_type == CPU_POWER8;
if (!strcmp(cpu, "power9"))
return cpu_type == CPU_POWER9;
return 0;
return (int)(cpu_type);
}
#elif !defined(__BUILTIN_CPU_SUPPORTS__)
static int cpuid(void)
{
return CPU_UNKNOWN;
}
#endif /* _AIX */
#endif /* C_PGI */
#ifndef __BUILTIN_CPU_SUPPORTS__
#include <string.h>
#ifndef __has_builtin
#define __has_builtin(x) 0
#endif
#if defined(_AIX) || !__has_builtin(__builtin_cpu_is)
static int __builtin_cpu_is(const char *arg)
{
static int ipinfo = -1;
if (ipinfo < 0) {
ipinfo = cpuid();
}
#ifdef HAVE_P10_SUPPORT
if (ipinfo == CPU_POWER10) {
if (!strcmp(arg, "power10")) return 1;
}
#endif
if (ipinfo == CPU_POWER9) {
if (!strcmp(arg, "power9")) return 1;
} else if (ipinfo == CPU_POWER8) {
if (!strcmp(arg, "power8")) return 1;
#ifndef C_PGI
} else if (ipinfo == CPU_POWER6) {
if (!strcmp(arg, "power6")) return 1;
#endif
}
return 0;
}
#endif
#if defined(_AIX) || !__has_builtin(__builtin_cpu_supports)
static int __builtin_cpu_supports(const char *arg)
{
return 0;
}
#endif
#endif
static gotoblas_t *get_coretype(void) {
@@ -196,19 +257,23 @@ static gotoblas_t *get_coretype(void) {
#endif
if (__builtin_cpu_is("power8"))
return &gotoblas_POWER8;
#if (!defined __GNUC__) || ( __GNUC__ >= 6)
#if ((!defined __GNUC__) || ( __GNUC__ >= 6)) || defined(__clang__)
if (__builtin_cpu_is("power9"))
return &gotoblas_POWER9;
#endif
#ifdef HAVE_P10_SUPPORT
#if defined(_AIX) || defined(__clang__)
if (__builtin_cpu_is("power10"))
#else
if (__builtin_cpu_supports ("arch_3_1") && __builtin_cpu_supports ("mma"))
#endif
return &gotoblas_POWER10;
#endif
/* Fall back to the POWER9 implementation if the toolchain is too old or the MMA feature is not set */
#if (!defined __GNUC__) || ( __GNUC__ >= 11) || (__GNUC__ == 10 && __GNUC_MINOR__ >= 2)
if (__builtin_cpu_is("power10"))
return &gotoblas_POWER9;
#endif
#endif
return NULL;
}
@@ -233,7 +298,7 @@ static gotoblas_t *force_coretype(char * coretype) {
case 1: return (&gotoblas_POWER6);
#endif
case 2: return (&gotoblas_POWER8);
#if (!defined __GNUC__) || ( __GNUC__ >= 6)
#if ((!defined __GNUC__) || ( __GNUC__ >= 6)) || defined(__clang__)
case 3: return (&gotoblas_POWER9);
#endif
#ifdef HAVE_P10_SUPPORT
@@ -274,6 +339,9 @@ void gotoblas_dynamic_init(void) {
if (gotoblas && gotoblas -> init) {
strncpy(coren,gotoblas_corename(),20);
sprintf(coremsg, "Core: %s\n",coren);
if (getenv("GET_OPENBLAS_CORETYPE")) {
fprintf(stderr, "%s", coremsg);
}
openblas_warning(2, coremsg);
gotoblas -> init();
} else {
driver/others/dynamic_zarch.c
+1 -1
View File
@@ -13,7 +13,7 @@ extern gotoblas_t gotoblas_Z14;
#define NUM_CORETYPES 4
extern int openblas_verbose();
extern int openblas_verbose(void);
extern void openblas_warning(int verbose, const char* msg);
char* gotoblas_corename(void) {
driver/others/memory.c
+30 -26
View File
@@ -73,6 +73,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "common.h"
#define NEW_BUFFERS 512
#ifndef likely
#ifdef __GNUC__
#define likely(x) __builtin_expect(!!(x), 1)
@@ -422,15 +423,13 @@ This value is equal or large than blas_cpu_number. This means some threads are s
*/
int blas_num_threads = 0;
int blas_num_threads_set = 0;
int goto_get_num_procs (void) {
return blas_cpu_number;
}
static void blas_memory_init();
static void blas_memory_init(void);
void openblas_fork_handler()
void openblas_fork_handler(void)
{
// This handler shuts down the OpenBLAS-managed PTHREAD pool when OpenBLAS is
// built with "make USE_OPENMP=0".
@@ -447,9 +446,9 @@ void openblas_fork_handler()
#endif
}
extern int openblas_num_threads_env();
extern int openblas_goto_num_threads_env();
extern int openblas_omp_num_threads_env();
extern int openblas_num_threads_env(void);
extern int openblas_goto_num_threads_env(void);
extern int openblas_omp_num_threads_env(void);
int blas_get_cpu_number(void){
#if defined(OS_LINUX) || defined(OS_WINDOWS) || defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) || defined(OS_DRAGONFLY) || defined(OS_DARWIN) || defined(OS_ANDROID) || defined(OS_HAIKU)
@@ -593,7 +592,7 @@ static BLASULONG key_lock = 0UL;
#endif
/* Returns a pointer to the start of the per-thread memory allocation data */
static __inline struct alloc_t ** get_memory_table() {
static __inline struct alloc_t ** get_memory_table(void) {
#if defined(SMP)
LOCK_COMMAND(&key_lock);
lsk=local_storage_key;
@@ -1146,7 +1145,7 @@ static void blas_memory_cleanup(void* ptr){
}
}
static void blas_memory_init(){
static void blas_memory_init(void){
#if defined(SMP)
# if defined(OS_WINDOWS)
local_storage_key = TlsAlloc();
@@ -1503,7 +1502,7 @@ static void gotoblas_memory_init(void) {
/* Initialization for all function; this function should be called before main */
static int gotoblas_initialized = 0;
extern void openblas_read_env();
extern void openblas_read_env(void);
void CONSTRUCTOR gotoblas_init(void) {
@@ -1996,13 +1995,11 @@ This value is equal or large than blas_cpu_number. This means some threads are s
*/
int blas_num_threads = 0;
int blas_num_threads_set = 0;
int goto_get_num_procs (void) {
return blas_cpu_number;
}
void openblas_fork_handler()
void openblas_fork_handler(void)
{
// This handler shuts down the OpenBLAS-managed PTHREAD pool when OpenBLAS is
// built with "make USE_OPENMP=0".
@@ -2019,9 +2016,9 @@ void openblas_fork_handler()
#endif
}
extern int openblas_num_threads_env();
extern int openblas_goto_num_threads_env();
extern int openblas_omp_num_threads_env();
extern int openblas_num_threads_env(void);
extern int openblas_goto_num_threads_env(void);
extern int openblas_omp_num_threads_env(void);
int blas_get_cpu_number(void){
#if defined(OS_LINUX) || defined(OS_WINDOWS) || defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) || defined(OS_DRAGONFLY) || defined(OS_DARWIN) || defined(OS_ANDROID) || defined(OS_HAIKU)
@@ -2901,7 +2898,7 @@ void *blas_memory_alloc(int procpos){
#endif
position ++;
} while (position < 512+NUM_BUFFERS);
} while (position < NEW_BUFFERS + NUM_BUFFERS);
}
#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
UNLOCK_COMMAND(&alloc_lock);
@@ -3015,10 +3012,13 @@ void *blas_memory_alloc(int procpos){
#endif
if (memory_overflowed) goto terminate;
fprintf(stderr,"OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n");
fprintf(stderr,"To avoid this warning, please rebuild your copy of OpenBLAS with a larger NUM_THREADS setting\n");
fprintf(stderr,"or set the environment variable OPENBLAS_NUM_THREADS to %d or lower\n", MAX_CPU_NUMBER);
memory_overflowed=1;
new_release_info = (struct release_t*) malloc(512*sizeof(struct release_t));
newmemory = (struct newmemstruct*) malloc(512*sizeof(struct newmemstruct));
for (i = 0; i < 512; i++) {
MB;
new_release_info = (struct release_t*) malloc(NEW_BUFFERS * sizeof(struct release_t));
newmemory = (struct newmemstruct*) malloc(NEW_BUFFERS * sizeof(struct newmemstruct));
for (i = 0; i < NEW_BUFFERS; i++) {
newmemory[i].addr = (void *)0;
#if defined(WHEREAMI) && !defined(USE_OPENMP)
newmemory[i].pos = -1;
@@ -3131,12 +3131,12 @@ void blas_memory_free(void *free_area){
printf(" Position : %d\n", position);
#endif
if (unlikely(memory_overflowed && position >= NUM_BUFFERS)) {
while ((position < NUM_BUFFERS+512) && (newmemory[position-NUM_BUFFERS].addr != free_area))
while ((position < NUM_BUFFERS+NEW_BUFFERS) && (newmemory[position-NUM_BUFFERS].addr != free_area))
position++;
// arm: ensure all writes are finished before other thread takes this memory
WMB;
newmemory[position].used = 0;
if (position - NUM_BUFFERS >= NEW_BUFFERS) goto error;
newmemory[position-NUM_BUFFERS].used = 0;
#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
UNLOCK_COMMAND(&alloc_lock);
#endif
@@ -3214,14 +3214,18 @@ void blas_shutdown(void){
#endif
memory[pos].lock = 0;
}
if (memory_overflowed)
for (pos = 0; pos < 512; pos ++){
if (memory_overflowed) {
for (pos = 0; pos < NEW_BUFFERS; pos ++){
newmemory[pos].addr = (void *)0;
newmemory[pos].used = 0;
#if defined(WHEREAMI) && !defined(USE_OPENMP)
newmemory[pos].pos = -1;
#endif
newmemory[pos].lock = 0;
}
free(newmemory);
newmemory = NULL;
memory_overflowed = 0;
}
UNLOCK_COMMAND(&alloc_lock);
@@ -3339,7 +3343,7 @@ static void gotoblas_memory_init(void) {
/* Initialization for all function; this function should be called before main */
static int gotoblas_initialized = 0;
extern void openblas_read_env();
extern void openblas_read_env(void);
void CONSTRUCTOR gotoblas_init(void) {
driver/others/memory_qalloc.c
+4 -5
View File
@@ -283,13 +283,12 @@ The numbers of threads in the thread pool.
This value is equal or large than blas_cpu_number. This means some threads are sleep.
*/
int blas_num_threads = 0;
int blas_num_threads_set = 0;
int goto_get_num_procs (void) {
return blas_cpu_number;
}
void openblas_fork_handler()
void openblas_fork_handler(void)
{
// This handler shuts down the OpenBLAS-managed PTHREAD pool when OpenBLAS is
// built with "make USE_OPENMP=0".
@@ -306,9 +305,9 @@ void openblas_fork_handler()
#endif
}
extern int openblas_num_threads_env();
extern int openblas_goto_num_threads_env();
extern int openblas_omp_num_threads_env();
extern int openblas_num_threads_env(void);
extern int openblas_goto_num_threads_env(void);
extern int openblas_omp_num_threads_env(void);
int blas_get_cpu_number(void){
#if defined(OS_LINUX) || defined(OS_WINDOWS) || defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) || defined(OS_DRAGONFLY) || defined(OS_DARWIN) || defined(OS_ANDROID)
driver/others/openblas_env.c
+8 -8
View File
@@ -41,15 +41,15 @@ static int openblas_env_goto_num_threads=0;
static int openblas_env_omp_num_threads=0;
static int openblas_env_omp_adaptive=0;
int openblas_verbose() { return openblas_env_verbose;}
unsigned int openblas_thread_timeout() { return openblas_env_thread_timeout;}
int openblas_block_factor() { return openblas_env_block_factor;}
int openblas_num_threads_env() { return openblas_env_openblas_num_threads;}
int openblas_goto_num_threads_env() { return openblas_env_goto_num_threads;}
int openblas_omp_num_threads_env() { return openblas_env_omp_num_threads;}
int openblas_omp_adaptive_env() { return openblas_env_omp_adaptive;}
int openblas_verbose(void) { return openblas_env_verbose;}
unsigned int openblas_thread_timeout(void) { return openblas_env_thread_timeout;}
int openblas_block_factor(void) { return openblas_env_block_factor;}
int openblas_num_threads_env(void) { return openblas_env_openblas_num_threads;}
int openblas_goto_num_threads_env(void) { return openblas_env_goto_num_threads;}
int openblas_omp_num_threads_env(void) { return openblas_env_omp_num_threads;}
int openblas_omp_adaptive_env(void) { return openblas_env_omp_adaptive;}
void openblas_read_env() {
void openblas_read_env(void) {
int ret=0;
env_var_t p;
if (readenv(p,"OPENBLAS_VERBOSE")) ret = atoi(p);
driver/others/openblas_error_handle.c
+1 -1
View File
@@ -33,7 +33,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "common.h"
extern int openblas_verbose();
extern int openblas_verbose(void);
void openblas_warning(int verbose, const char * msg) {
int current_verbose;
driver/others/openblas_get_config.c
+4 -4
View File
@@ -69,13 +69,13 @@ static char* openblas_config_str=""
;
#ifdef DYNAMIC_ARCH
char *gotoblas_corename();
char *gotoblas_corename(void);
#endif
static char tmp_config_str[256];
int openblas_get_parallel();
int openblas_get_parallel(void);
char* CNAME() {
char* CNAME(void) {
char tmpstr[20];
strcpy(tmp_config_str, openblas_config_str);
#ifdef DYNAMIC_ARCH
@@ -90,7 +90,7 @@ char tmpstr[20];
}
char* openblas_get_corename() {
char* openblas_get_corename(void) {
#ifndef DYNAMIC_ARCH
return CHAR_CORENAME;
#else
driver/others/openblas_get_parallel.c
+3 -3
View File
@@ -42,17 +42,17 @@ static int parallel = 0;
#ifdef NEEDBUNDERSCORE
int CNAME() {
int CNAME(void) {
return parallel;
}
int NAME() {
int NAME(void) {
return parallel;
}
#else
//The CNAME and NAME are the same.
int NAME() {
int NAME(void) {
return parallel;
}
#endif
driver/others/openblas_set_num_threads.c
+13
View File
@@ -36,11 +36,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifdef SMP_SERVER
extern void openblas_set_num_threads(int num_threads) ;
extern int openblas_get_num_threads(void) ;
void openblas_set_num_threads_(int* num_threads){
openblas_set_num_threads(*num_threads);
}
int openblas_set_num_threads_local(int num_threads){
int ret = openblas_get_num_threads();
openblas_set_num_threads(num_threads);
blas_omp_threads_local=num_threads;
return ret;
}
#else
//Single thread
@@ -50,4 +59,8 @@ void openblas_set_num_threads(int num_threads) {
void openblas_set_num_threads_(int* num_threads){
}
int openblas_set_num_threads_local(int num_threads){
return 1;
}
#endif
driver/others/parameter.c
+1 -1
View File
@@ -40,7 +40,7 @@
#include <string.h>
#include "common.h"
extern int openblas_block_factor();
extern int openblas_block_factor(void);
int get_L2_size(void);
#define DEFAULT_GEMM_P 128