floraachy/OpenBLAS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Import GotoBLAS2 1.13 BSD version codes.

Browse Source
This commit is contained in:
Xianyi Zhang
2011-01-24 14:54:24 +00:00
commit 342bbc3871
1685 changed files with 1382682 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
driver/others/Makefile
+218
View File
@@ -0,0 +1,218 @@
TOPDIR = ../..
include ../../Makefile.system
COMMONOBJS = memory.$(SUFFIX) xerbla.$(SUFFIX) c_abs.$(SUFFIX) z_abs.$(SUFFIX)
COMMONOBJS += slamch.$(SUFFIX) slamc3.$(SUFFIX) dlamch.$(SUFFIX) dlamc3.$(SUFFIX)
ifdef SMP
COMMONOBJS += blas_server.$(SUFFIX) divtable.$(SUFFIX) blasL1thread.$(SUFFIX)
ifndef NO_AFFINITY
COMMONOBJS += init.$(SUFFIX)
endif
endif
# COMMONOBJS += info.$(SUFFIX)
ifdef DYNAMIC_ARCH
COMMONOBJS += dynamic.$(SUFFIX)
else
COMMONOBJS += parameter.$(SUFFIX)
endif
ifdef EXPRECISION
COMMONOBJS += x_abs.$(SUFFIX) qlamch.$(SUFFIX) qlamc3.$(SUFFIX)
endif
ifdef QUAD_PRECISION
COMMONOBJS += addx.$(SUFFIX) mulx.$(SUFFIX)
endif
ifeq ($(OSNAME), CYGWIN_NT)
ifeq ($(C_COMPILER), PGI)
# COMMONOBJS += __builtin_stinit.$(SUFFIX)
endif
endif
ifdef USE_CUDA
COMMONOBJS += cuda_init.$(SUFFIX)
endif
ifdef FUNCTION_PROFILE
COMMONOBJS += profile.$(SUFFIX)
endif
LIBOTHERS = libothers.$(LIBSUFFIX)
ifeq ($(CORE), PPC440)
MEMORY = memory_qalloc.c
endif
ifndef MEMORY
MEMORY = memory.c
endif
ifeq ($(USE_OPENMP), 1)
BLAS_SERVER = blas_server_omp.c
else
ifeq ($(OSNAME), WINNT)
BLAS_SERVER = blas_server_win32.c
endif
ifeq ($(OSNAME), CYGWIN_NT)
BLAS_SERVER = blas_server_win32.c
endif
ifeq ($(OSNAME), Interix)
BLAS_SERVER = blas_server_win32.c
endif
endif
ifndef BLAS_SERVER
BLAS_SERVER = blas_server.c
endif
ifdef DYNAMIC_ARCH
HPLOBJS = memory.$(SUFFIX) xerbla.$(SUFFIX) dynamic.$(SUFFIX)
else
HPLOBJS = memory.$(SUFFIX) xerbla.$(SUFFIX) parameter.$(SUFFIX)
endif
xerbla.$(SUFFIX) : xerbla.c
$(CC) $(CFLAGS) -c $< -o $(@F)
dynamic.$(SUFFIX) : dynamic.c
$(CC) $(CFLAGS) -c $< -o $(@F)
dynamic.$(PSUFFIX) : dynamic.c
$(CC) $(PFLAGS) -c $< -o $(@F)
parameter.$(SUFFIX) : parameter.c ../../param.h
$(CC) $(CFLAGS) -c $< -o $(@F)
init.$(SUFFIX) : init.c
$(CC) $(CFLAGS) -c $< -o $(@F)
profile.$(SUFFIX) : profile.c
$(CC) $(CFLAGS) -c $< -o $(@F)
memory.$(SUFFIX) : $(MEMORY) ../../common.h ../../param.h
$(CC) $(CFLAGS) -c $< -o $(@F)
blas_server.$(SUFFIX) : $(BLAS_SERVER) ../../common.h ../../common_thread.h ../../param.h
$(CC) $(CFLAGS) -c $< -o $(@F)
blasL1thread.$(SUFFIX) : blas_l1_thread.c ../../common.h ../../common_thread.h
$(CC) $(CFLAGS) -c $< -o $(@F)
cuda_init.$(SUFFIX) : cuda_init.c
$(CUCC) $(COMMON_OPT) -I$(TOPDIR) $(CUFLAGS) -DCNAME=$(*F) -c $< -o $(@F)
c_abs.$(SUFFIX) : abs.c
$(CC) $(CFLAGS) -c -UDOUBLE $< -o $(@F)
z_abs.$(SUFFIX) : abs.c
$(CC) $(CFLAGS) -c -DDOUBLE $< -o $(@F)
x_abs.$(SUFFIX) : abs.c
$(CC) $(CFLAGS) -c -DXDOUBLE $< -o $(@F)
slamch.$(SUFFIX) : lamch.c
$(CC) $(CFLAGS) -c -UDOUBLE $< -o $(@F)
dlamch.$(SUFFIX) : lamch.c
$(CC) $(CFLAGS) -c -DDOUBLE $< -o $(@F)
qlamch.$(SUFFIX) : lamch.c
$(CC) $(CFLAGS) -c -DXDOUBLE $< -o $(@F)
slamc3.$(SUFFIX) : lamc3.c
$(CC) $(CFLAGS) -c -UDOUBLE $< -o $(@F)
dlamc3.$(SUFFIX) : lamc3.c
$(CC) $(CFLAGS) -c -DDOUBLE $< -o $(@F)
qlamc3.$(SUFFIX) : lamc3.c
$(CC) $(CFLAGS) -c -DXDOUBLE $< -o $(@F)
divtable.$(SUFFIX) : divtable.c
$(CC) $(CFLAGS) -c -UDOUBLE $< -o $(@F)
__builtin_stinit.$(SUFFIX) : $(ARCH)/builtin_stinit.S
$(CC) $(CFLAGS) -c -UDOUBLE $< -o $(@F)
addx.$(SUFFIX) : $(ARCH)/addx.c
$(CC) $(CFLAGS) -c -DXDOUBLE -UCOMPLEX $< -o $(@F)
mulx.$(SUFFIX) : $(ARCH)/mulx.c
$(CC) $(CFLAGS) -c -DXDOUBLE -UCOMPLEX $< -o $(@F)
xerbla.$(PSUFFIX) : xerbla.c
$(CC) $(PFLAGS) -c $< -o $(@F)
parameter.$(PSUFFIX) : parameter.c ../../param.h
$(CC) $(PFLAGS) -c $< -o $(@F)
init.$(PSUFFIX) : init.c
$(CC) $(PFLAGS) -c $< -o $(@F)
profile.$(PSUFFIX) : profile.c
$(CC) $(PFLAGS) -c $< -o $(@F)
memory.$(PSUFFIX) : $(MEMORY) ../../common.h ../../param.h
$(CC) $(PFLAGS) -c $< -o $(@F)
blas_server.$(PSUFFIX) : $(BLAS_SERVER) ../../common.h ../../common_thread.h
$(CC) $(PFLAGS) -c $< -o $(@F)
blasL1thread.$(PSUFFIX) : blas_l1_thread.c ../../common.h ../../common_thread.h
$(CC) $(PFLAGS) -c $< -o $(@F)
cuda_init.$(PSUFFIX) : cuda_init.c
$(CUCC) $(COMMON_OPT) -I$(TOPDIR) $(CUFLAGS) -DCNAME=$(*F) -c $< -o $(@F)
c_abs.$(PSUFFIX) : abs.c
$(CC) $(PFLAGS) -c -UDOUBLE $< -o $(@F)
z_abs.$(PSUFFIX) : abs.c
$(CC) $(PFLAGS) -c -DDOUBLE $< -o $(@F)
x_abs.$(PSUFFIX) : abs.c
$(CC) $(PFLAGS) -c -DXDOUBLE $< -o $(@F)
slamch.$(PUFFIX) : lamch.c
$(CC) $(PFLAGS) -c -UDOUBLE $< -o $(@F)
dlamch.$(PUFFIX) : lamch.c
$(CC) $(PFLAGS) -c -DDOUBLE $< -o $(@F)
qlamch.$(PUFFIX) : lamch.c
$(CC) $(PFLAGS) -c -DXDOUBLE $< -o $(@F)
slamc3.$(PUFFIX) : lamc3.c
$(CC) $(PFLAGS) -c -UDOUBLE $< -o $(@F)
dlamc3.$(PUFFIX) : lamc3.c
$(CC) $(PFLAGS) -c -DDOUBLE $< -o $(@F)
qlamc3.$(PUFFIX) : lamc3.c
$(CC) $(PFLAGS) -c -DXDOUBLE $< -o $(@F)
divtable.$(PSUFFIX) : divtable.c
$(CC) $(PFLAGS) -c -UDOUBLE $< -o $(@F)
__builtin_stinit.$(PPSUFFIX) : $(ARCH)/builtin_stinit.S
$(CC) $(PFLAGS) -c -UDOUBLE $< -o $(@F)
addx.$(PSUFFIX) : $(ARCH)/addx.c
$(CC) $(PFLAGS) -c -DXDOUBLE -UCOMPLEX $< -o $(@F)
mulx.$(PSUFFIX) : $(ARCH)/mulx.c
$(CC) $(PFLAGS) -c -DXDOUBLE -UCOMPLEX $< -o $(@F)
info.$(SUFFIX) : info.c info.h ../../common.h ../../param.h
$(CC) $(CFLAGS) -c $< -o $(@F)
hpl : CFLAGS += -DHPL
hpl_p : CFLAGS += -DHPL
include $(TOPDIR)/Makefile.tail
driver/others/abs.c
+71
View File
@@ -0,0 +1,71 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/
#include "common.h"
double fabs(double);
double sqrt(double);
#ifdef NEED_F2CCONV
double
#else
FLOAT
#endif
CNAME(FLOAT *z){
FLOAT real = z[0];
FLOAT imag = z[1];
double temp;
real = fabs(real);
imag = fabs(imag);
if(imag > real){
temp = real;
real = imag;
imag = temp;
}
if (imag == 0.) return real;
temp = imag/real;
temp = real * sqrt(1.0 + temp*temp);
return temp;
}
driver/others/blas_l1_thread.c
+112
View File
@@ -0,0 +1,112 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/
#include <stdio.h>
#include <stdlib.h>
#include "common.h"
int blas_level1_thread(int mode, BLASLONG m, BLASLONG n, BLASLONG k, void *alpha,
void *a, BLASLONG lda,
void *b, BLASLONG ldb,
void *c, BLASLONG ldc, int (*function)(), int nthreads){
blas_queue_t queue[MAX_CPU_NUMBER];
blas_arg_t args [MAX_CPU_NUMBER];
BLASLONG i, width, astride, bstride;
int num_cpu, calc_type;
calc_type = (mode & BLAS_PREC) + ((mode & BLAS_COMPLEX) != 0) + 2;
mode |= BLAS_LEGACY;
for (i = 0; i < nthreads; i++) blas_queue_init(&queue[i]);
num_cpu = 0;
i = m;
while (i > 0){
/* Adjust Parameters */
width = blas_quickdivide(i + nthreads - num_cpu - 1,
nthreads - num_cpu);
i -= width;
if (i < 0) width = width + i;
astride = width * lda;
if (!(mode & BLAS_TRANSB_T)) {
bstride = width * ldb;
} else {
bstride = width;
}
astride <<= calc_type;
bstride <<= calc_type;
args[num_cpu].m = width;
args[num_cpu].n = n;
args[num_cpu].k = k;
args[num_cpu].a = (void *)a;
args[num_cpu].b = (void *)b;
args[num_cpu].c = (void *)c;
args[num_cpu].lda = lda;
args[num_cpu].ldb = ldb;
args[num_cpu].ldc = ldc;
args[num_cpu].alpha = alpha;
queue[num_cpu].mode = mode;
queue[num_cpu].routine = function;
queue[num_cpu].args = &args[num_cpu];
queue[num_cpu].next = &queue[num_cpu + 1];
a = (void *)((BLASULONG)a + astride);
b = (void *)((BLASULONG)b + bstride);
num_cpu ++;
}
if (num_cpu) {
queue[num_cpu - 1].next = NULL;
exec_blas(num_cpu, queue);
}
return 0;
}
driver/others/blas_server.c
+848
View File
@@ -0,0 +1,848 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/
#include "common.h"
#ifdef OS_LINUX
#include <dlfcn.h>
#include <sys/resource.h>
#endif
#ifdef SMP_SERVER
#undef MONITOR
#undef TIMING
#undef TIMING_DEBUG
#undef NEED_STACKATTR
#define ATTRIBUTE_SIZE 128
/* This is a thread server model implementation. The threads are */
/* spawned at first access to blas library, and still remains until */
/* destruction routine is called. The number of threads are */
/* equal to "OMP_NUM_THREADS - 1" and thread only wakes up when */
/* jobs is queued. */
/* We need this grobal for cheking if initialization is finished. */
int blas_server_avail __attribute__((aligned(ATTRIBUTE_SIZE))) = 0;
/* Local Variables */
#if defined(USE_PTHREAD_LOCK)
static pthread_mutex_t server_lock = PTHREAD_MUTEX_INITIALIZER;
#elif defined(USE_PTHREAD_SPINLOCK)
static pthread_spinlock_t server_lock = 0;
#else
static unsigned long server_lock = 0;
#endif
#define THREAD_STATUS_SLEEP 2
#define THREAD_STATUS_WAKEUP 4
static pthread_t blas_threads [MAX_CPU_NUMBER];
typedef struct {
blas_queue_t * volatile queue __attribute__((aligned(ATTRIBUTE_SIZE)));
#if defined(OS_LINUX) && !defined(NO_AFFINITY)
int node;
#endif
volatile long status;
pthread_mutex_t lock;
pthread_cond_t wakeup;
} thread_status_t;
static thread_status_t thread_status[MAX_CPU_NUMBER] __attribute__((aligned(ATTRIBUTE_SIZE)));
#ifndef THREAD_TIMEOUT
#define THREAD_TIMEOUT 28
#endif
static unsigned int thread_timeout = (1U << (THREAD_TIMEOUT));
#ifdef MONITOR
/* Monitor is a function to see thread's status for every seconds. */
/* Usually it turns off and it's for debugging. */
static pthread_t monitor_thread;
static int main_status[MAX_CPU_NUMBER];
#define MAIN_ENTER 0x01
#define MAIN_EXIT 0x02
#define MAIN_TRYLOCK 0x03
#define MAIN_LOCKSUCCESS 0x04
#define MAIN_QUEUING 0x05
#define MAIN_RECEIVING 0x06
#define MAIN_RUNNING1 0x07
#define MAIN_RUNNING2 0x08
#define MAIN_RUNNING3 0x09
#define MAIN_WAITING 0x0a
#define MAIN_SLEEPING 0x0b
#define MAIN_FINISH 0x0c
#define MAIN_DONE 0x0d
#endif
#define BLAS_QUEUE_FINISHED 3
#define BLAS_QUEUE_RUNNING 4
#ifdef TIMING
BLASLONG exit_time[MAX_CPU_NUMBER];
#endif
static void legacy_exec(void *func, int mode, blas_arg_t *args, void *sb){
if (!(mode & BLAS_COMPLEX)){
#ifdef EXPRECISION
if (mode & BLAS_XDOUBLE){
/* REAL / Extended Double */
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, xdouble,
xdouble *, BLASLONG, xdouble *, BLASLONG,
xdouble *, BLASLONG, void *) = func;
afunc(args -> m, args -> n, args -> k,
((xdouble *)args -> alpha)[0],
args -> a, args -> lda,
args -> b, args -> ldb,
args -> c, args -> ldc, sb);
} else
#endif
if (mode & BLAS_DOUBLE){
/* REAL / Double */
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, double,
double *, BLASLONG, double *, BLASLONG,
double *, BLASLONG, void *) = func;
afunc(args -> m, args -> n, args -> k,
((double *)args -> alpha)[0],
args -> a, args -> lda,
args -> b, args -> ldb,
args -> c, args -> ldc, sb);
} else {
/* REAL / Single */
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, float,
float *, BLASLONG, float *, BLASLONG,
float *, BLASLONG, void *) = func;
afunc(args -> m, args -> n, args -> k,
((float *)args -> alpha)[0],
args -> a, args -> lda,
args -> b, args -> ldb,
args -> c, args -> ldc, sb);
}
} else {
#ifdef EXPRECISION
if (mode & BLAS_XDOUBLE){
/* COMPLEX / Extended Double */
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble,
xdouble *, BLASLONG, xdouble *, BLASLONG,
xdouble *, BLASLONG, void *) = func;
afunc(args -> m, args -> n, args -> k,
((xdouble *)args -> alpha)[0],
((xdouble *)args -> alpha)[1],
args -> a, args -> lda,
args -> b, args -> ldb,
args -> c, args -> ldc, sb);
} else
#endif
if (mode & BLAS_DOUBLE){
/* COMPLEX / Double */
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, double, double,
double *, BLASLONG, double *, BLASLONG,
double *, BLASLONG, void *) = func;
afunc(args -> m, args -> n, args -> k,
((double *)args -> alpha)[0],
((double *)args -> alpha)[1],
args -> a, args -> lda,
args -> b, args -> ldb,
args -> c, args -> ldc, sb);
} else {
/* COMPLEX / Single */
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, float, float,
float *, BLASLONG, float *, BLASLONG,
float *, BLASLONG, void *) = func;
afunc(args -> m, args -> n, args -> k,
((float *)args -> alpha)[0],
((float *)args -> alpha)[1],
args -> a, args -> lda,
args -> b, args -> ldb,
args -> c, args -> ldc, sb);
}
}
}
#if defined(OS_LINUX) && !defined(NO_AFFINITY)
int gotoblas_set_affinity(int);
int gotoblas_set_affinity2(int);
int get_node(void);
#endif
static int increased_threads = 0;
static int blas_thread_server(void *arg){
/* Thread identifier */
BLASLONG cpu = (BLASLONG)arg;
unsigned int last_tick;
void *buffer, *sa, *sb;
blas_queue_t *queue;
#ifdef TIMING_DEBUG
unsigned long start, stop;
#endif
#if defined(OS_LINUX) && !defined(NO_AFFINITY)
if (!increased_threads)
thread_status[cpu].node = gotoblas_set_affinity(cpu + 1);
else
thread_status[cpu].node = gotoblas_set_affinity(-1);
#endif
#ifdef MONITOR
main_status[cpu] = MAIN_ENTER;
#endif
buffer = blas_memory_alloc(2);
#ifdef SMP_DEBUG
fprintf(STDERR, "Server[%2ld] Thread has just been spawned!\n", cpu);
#endif
while (1){
#ifdef MONITOR
main_status[cpu] = MAIN_QUEUING;
#endif
#ifdef TIMING
exit_time[cpu] = rpcc();
#endif
last_tick = (unsigned int)rpcc();
while (!thread_status[cpu].queue) {
YIELDING;
if ((unsigned int)rpcc() - last_tick > thread_timeout) {
pthread_mutex_lock (&thread_status[cpu].lock);
if (!thread_status[cpu].queue) {
thread_status[cpu].status = THREAD_STATUS_SLEEP;
while (thread_status[cpu].status == THREAD_STATUS_SLEEP) {
#ifdef MONITOR
main_status[cpu] = MAIN_SLEEPING;
#endif
pthread_cond_wait(&thread_status[cpu].wakeup, &thread_status[cpu].lock);
}
}
pthread_mutex_unlock(&thread_status[cpu].lock);
last_tick = (unsigned int)rpcc();
}
}
queue = thread_status[cpu].queue;
if ((long)queue == -1) break;
#ifdef MONITOR
main_status[cpu] = MAIN_RECEIVING;
#endif
#ifdef TIMING_DEBUG
start = rpcc();
#endif
if (queue) {
int (*routine)(blas_arg_t *, void *, void *, void *, void *, BLASLONG) = queue -> routine;
thread_status[cpu].queue = (blas_queue_t *)1;
sa = queue -> sa;
sb = queue -> sb;
#ifdef SMP_DEBUG
if (queue -> args) {
fprintf(STDERR, "Server[%2ld] Calculation started. Mode = 0x%03x M = %3ld N=%3ld K=%3ld\n",
cpu, queue->mode, queue-> args ->m, queue->args->n, queue->args->k);
}
#endif
#ifdef CONSISTENT_FPCSR
__asm__ __volatile__ ("ldmxcsr %0" : : "m" (queue -> sse_mode));
__asm__ __volatile__ ("fldcw %0" : : "m" (queue -> x87_mode));
#endif
#ifdef MONITOR
main_status[cpu] = MAIN_RUNNING1;
#endif
if (sa == NULL) sa = (void *)((BLASLONG)buffer + GEMM_OFFSET_A);
if (sb == NULL) {
if (!(queue -> mode & BLAS_COMPLEX)){
#ifdef EXPRECISION
if (queue -> mode & BLAS_XDOUBLE){
sb = (void *)(((BLASLONG)sa + ((QGEMM_P * QGEMM_Q * sizeof(xdouble)
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
} else
#endif
if (queue -> mode & BLAS_DOUBLE){
sb = (void *)(((BLASLONG)sa + ((DGEMM_P * DGEMM_Q * sizeof(double)
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
} else {
sb = (void *)(((BLASLONG)sa + ((SGEMM_P * SGEMM_Q * sizeof(float)
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
}
} else {
#ifdef EXPRECISION
if (queue -> mode & BLAS_XDOUBLE){
sb = (void *)(((BLASLONG)sa + ((XGEMM_P * XGEMM_Q * 2 * sizeof(xdouble)
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
} else
#endif
if (queue -> mode & BLAS_DOUBLE){
sb = (void *)(((BLASLONG)sa + ((ZGEMM_P * ZGEMM_Q * 2 * sizeof(double)
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
} else {
sb = (void *)(((BLASLONG)sa + ((CGEMM_P * CGEMM_Q * 2 * sizeof(float)
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
}
}
}
#ifdef MONITOR
main_status[cpu] = MAIN_RUNNING2;
#endif
if (queue -> mode & BLAS_LEGACY) {
legacy_exec(routine, queue -> mode, queue -> args, sb);
} else
if (queue -> mode & BLAS_PTHREAD) {
void (*pthreadcompat)(void *) = queue -> routine;
(pthreadcompat)(queue -> args);
} else
(routine)(queue -> args, queue -> range_m, queue -> range_n, sa, sb, queue -> position);
#ifdef SMP_DEBUG
fprintf(STDERR, "Server[%2ld] Calculation finished!\n", cpu);
#endif
#ifdef MONITOR
main_status[cpu] = MAIN_FINISH;
#endif
thread_status[cpu].queue = (blas_queue_t * volatile) ((long)thread_status[cpu].queue & 0); /* Need a trick */
WMB;
}
#ifdef MONITOR
main_status[cpu] = MAIN_DONE;
#endif
#ifdef TIMING_DEBUG
stop = rpcc();
fprintf(STDERR, "Thread[%ld] : %16lu %16lu (%8lu cycles)\n", cpu + 1,
start, stop,
stop - start);
#endif
}
/* Shutdown procedure */
#ifdef SMP_DEBUG
fprintf(STDERR, "Server[%2ld] Shutdown!\n", cpu);
#endif
blas_memory_free(buffer);
pthread_exit(NULL);
return 0;
}
#ifdef MONITOR
static BLASLONG num_suspend = 0;
static int blas_monitor(void *arg){
int i;
while(1){
for (i = 0; i < blas_num_threads - 1; i++){
switch (main_status[i]) {
case MAIN_ENTER :
fprintf(STDERR, "THREAD[%2d] : Entering.\n", i);
break;
case MAIN_EXIT :
fprintf(STDERR, "THREAD[%2d] : Exiting.\n", i);
break;
case MAIN_TRYLOCK :
fprintf(STDERR, "THREAD[%2d] : Trying lock operation.\n", i);
break;
case MAIN_QUEUING :
fprintf(STDERR, "THREAD[%2d] : Queuing.\n", i);
break;
case MAIN_RECEIVING :
fprintf(STDERR, "THREAD[%2d] : Receiving.\n", i);
break;
case MAIN_RUNNING1 :
fprintf(STDERR, "THREAD[%2d] : Running1.\n", i);
break;
case MAIN_RUNNING2 :
fprintf(STDERR, "THREAD[%2d] : Running2.\n", i);
break;
case MAIN_RUNNING3 :
fprintf(STDERR, "THREAD[%2d] : Running3.\n", i);
break;
case MAIN_WAITING :
fprintf(STDERR, "THREAD[%2d] : Waiting.\n", i);
break;
case MAIN_SLEEPING :
fprintf(STDERR, "THREAD[%2d] : Sleeping.\n", i);
break;
case MAIN_FINISH :
fprintf(STDERR, "THREAD[%2d] : Finishing.\n", i);
break;
case MAIN_DONE :
fprintf(STDERR, "THREAD[%2d] : Job is done.\n", i);
break;
}
fprintf(stderr, "Total number of suspended ... %ld\n", num_suspend);
}
sleep(1);
}
return 0;
}
#endif
/* Initializing routine */
int blas_thread_init(void){
BLASLONG i;
#ifdef NEED_STACKATTR
pthread_attr_t attr;
#endif
if (blas_server_avail) return 0;
#ifdef NEED_STACKATTR
pthread_attr_init(&attr);
pthread_attr_setguardsize(&attr, 0x1000U);
pthread_attr_setstacksize( &attr, 0x1000U);
#endif
LOCK_COMMAND(&server_lock);
if (!blas_server_avail){
char *p;
p = getenv("GOTO_THREAD_TIMEOUT");
if (p) {
thread_timeout = atoi(p);
if (thread_timeout < 4) thread_timeout = 4;
if (thread_timeout > 30) thread_timeout = 30;
thread_timeout = (1 << thread_timeout);
}
for(i = 0; i < blas_num_threads - 1; i++){
thread_status[i].queue = (blas_queue_t *)NULL;
thread_status[i].status = THREAD_STATUS_WAKEUP;
pthread_mutex_init(&thread_status[i].lock, NULL);
pthread_cond_init (&thread_status[i].wakeup, NULL);
#ifdef NEED_STACKATTR
pthread_create(&blas_threads[i], &attr,
(void *)&blas_thread_server, (void *)i);
#else
pthread_create(&blas_threads[i], NULL,
(void *)&blas_thread_server, (void *)i);
#endif
}
#ifdef MONITOR
pthread_create(&monitor_thread, NULL,
(void *)&blas_monitor, (void *)NULL);
#endif
blas_server_avail = 1;
}
UNLOCK_COMMAND(&server_lock);
return 0;
}
/*
User can call one of two routines.
exec_blas_async ... immediately returns after jobs are queued.
exec_blas ... returns after jobs are finished.
*/
static BLASULONG exec_queue_lock = 0;
int exec_blas_async(BLASLONG pos, blas_queue_t *queue){
BLASLONG i = 0;
blas_queue_t *current = queue;
#if defined(OS_LINUX) && !defined(NO_AFFINITY) && !defined(PARAMTEST)
int node = get_node();
int nodes = get_num_nodes();
#endif
#ifdef SMP_DEBUG
int exec_count = 0;
fprintf(STDERR, "Exec_blas_async is called. Position = %d\n", pos);
#endif
blas_lock(&exec_queue_lock);
while (queue) {
queue -> position = pos;
#ifdef CONSISTENT_FPCSR
__asm__ __volatile__ ("fnstcw %0" : "=m" (queue -> x87_mode));
__asm__ __volatile__ ("stmxcsr %0" : "=m" (queue -> sse_mode));
#endif
#if defined(OS_LINUX) && !defined(NO_AFFINITY) && !defined(PARAMTEST)
/* Node Mapping Mode */
if (queue -> mode & BLAS_NODE) {
do {
while((thread_status[i].node != node || thread_status[i].queue) && (i < blas_num_threads - 1)) i ++;
if (i < blas_num_threads - 1) break;
i ++;
if (i >= blas_num_threads - 1) {
i = 0;
node ++;
if (node >= nodes) node = 0;
}
} while (1);
} else {
while(thread_status[i].queue) {
i ++;
if (i >= blas_num_threads - 1) i = 0;
}
}
#else
while(thread_status[i].queue) {
i ++;
if (i >= blas_num_threads - 1) i = 0;
}
#endif
queue -> assigned = i;
WMB;
thread_status[i].queue = queue;
WMB;
queue = queue -> next;
pos ++;
#ifdef SMP_DEBUG
exec_count ++;
#endif
}
blas_unlock(&exec_queue_lock);
#ifdef SMP_DEBUG
fprintf(STDERR, "Done(Number of threads = %2ld).\n", exec_count);
#endif
while (current) {
pos = current -> assigned;
if ((BLASULONG)thread_status[pos].queue > 1) {
if (thread_status[pos].status == THREAD_STATUS_SLEEP) {
pthread_mutex_lock (&thread_status[pos].lock);
#ifdef MONITOR
num_suspend ++;
#endif
if (thread_status[pos].status == THREAD_STATUS_SLEEP) {
thread_status[pos].status = THREAD_STATUS_WAKEUP;
pthread_cond_signal(&thread_status[pos].wakeup);
}
pthread_mutex_unlock(&thread_status[pos].lock);
}
}
current = current -> next;
}
return 0;
}
int exec_blas_async_wait(BLASLONG num, blas_queue_t *queue){
while ((num > 0) && queue) {
while(thread_status[queue -> assigned].queue) {
YIELDING;
};
queue = queue -> next;
num --;
}
#ifdef SMP_DEBUG
fprintf(STDERR, "Done.\n\n");
#endif
return 0;
}
/* Execute Threads */
int exec_blas(BLASLONG num, blas_queue_t *queue){
int (*routine)(blas_arg_t *, void *, void *, double *, double *, BLASLONG);
#ifdef TIMING_DEBUG
BLASULONG start, stop;
#endif
if ((num <= 0) || (queue == NULL)) return 0;
#ifdef SMP_DEBUG
fprintf(STDERR, "Exec_blas is called. Number of executing threads : %ld\n", num);
#endif
#ifdef __ELF__
if (omp_in_parallel && (num > 1)) {
if (omp_in_parallel() > 0) {
fprintf(stderr,
"GotoBLAS Warning : Detect OpenMP Loop and this application may hang. "
"Please rebuild the library with USE_OPENMP=1 option.\n");
}
}
#endif
if ((num > 1) && queue -> next) exec_blas_async(1, queue -> next);
#ifdef TIMING_DEBUG
start = rpcc();
fprintf(STDERR, "\n");
#endif
routine = queue -> routine;
if (queue -> mode & BLAS_LEGACY) {
legacy_exec(routine, queue -> mode, queue -> args, queue -> sb);
} else
if (queue -> mode & BLAS_PTHREAD) {
void (*pthreadcompat)(void *) = queue -> routine;
(pthreadcompat)(queue -> args);
} else
(routine)(queue -> args, queue -> range_m, queue -> range_n,
queue -> sa, queue -> sb, 0);
#ifdef TIMING_DEBUG
stop = rpcc();
#endif
if ((num > 1) && queue -> next) exec_blas_async_wait(num - 1, queue -> next);
#ifdef TIMING_DEBUG
fprintf(STDERR, "Thread[0] : %16lu %16lu (%8lu cycles)\n",
start, stop,
stop - start);
#endif
return 0;
}
void goto_set_num_threads(int num_threads) {
long i;
if (num_threads < 1) num_threads = blas_num_threads;
if (num_threads > MAX_CPU_NUMBER) num_threads = MAX_CPU_NUMBER;
if (num_threads > blas_num_threads) {
LOCK_COMMAND(&server_lock);
increased_threads = 1;
for(i = blas_num_threads - 1; i < num_threads - 1; i++){
thread_status[i].queue = (blas_queue_t *)NULL;
thread_status[i].status = THREAD_STATUS_WAKEUP;
pthread_mutex_init(&thread_status[i].lock, NULL);
pthread_cond_init (&thread_status[i].wakeup, NULL);
#ifdef NEED_STACKATTR
pthread_create(&blas_threads[i], &attr,
(void *)&blas_thread_server, (void *)i);
#else
pthread_create(&blas_threads[i], NULL,
(void *)&blas_thread_server, (void *)i);
#endif
}
blas_num_threads = num_threads;
UNLOCK_COMMAND(&server_lock);
}
blas_cpu_number = num_threads;
}
/* Compatible function with pthread_create / join */
int gotoblas_pthread(int numthreads, void *function, void *args, int stride) {
blas_queue_t queue[MAX_CPU_NUMBER];
int i;
if (numthreads <= 0) return 0;
#ifdef SMP
if (blas_cpu_number == 0) blas_get_cpu_number();
#ifdef SMP_SERVER
if (blas_server_avail == 0) blas_thread_init();
#endif
#endif
for (i = 0; i < numthreads; i ++) {
queue[i].mode = BLAS_PTHREAD;
queue[i].routine = function;
queue[i].args = args;
queue[i].range_m = NULL;
queue[i].range_n = NULL;
queue[i].sa = args;
queue[i].sb = args;
queue[i].next = &queue[i + 1];
args += stride;
}
queue[numthreads - 1].next = NULL;
exec_blas(numthreads, queue);
return 0;
}
/* Shutdown procedure, but user don't have to call this routine. The */
/* kernel automatically kill threads. */
int BLASFUNC(blas_thread_shutdown)(void){
int i;
if (!blas_server_avail) return 0;
LOCK_COMMAND(&server_lock);
for (i = 0; i < blas_num_threads - 1; i++) {
blas_lock(&exec_queue_lock);
thread_status[i].queue = (blas_queue_t *)-1;
blas_unlock(&exec_queue_lock);
pthread_mutex_lock (&thread_status[i].lock);
thread_status[i].status = THREAD_STATUS_WAKEUP;
pthread_cond_signal (&thread_status[i].wakeup);
pthread_mutex_unlock(&thread_status[i].lock);
}
for(i = 0; i < blas_num_threads - 1; i++){
pthread_join(blas_threads[i], NULL);
}
for(i = 0; i < blas_num_threads - 1; i++){
pthread_mutex_destroy(&thread_status[i].lock);
pthread_cond_destroy (&thread_status[i].wakeup);
}
#ifdef NEED_STACKATTR
pthread_attr_destory(&attr);
#endif
blas_server_avail = 0;
UNLOCK_COMMAND(&server_lock);
return 0;
}
#endif
driver/others/blas_server_omp.c
+249
View File
@@ -0,0 +1,249 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/
#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>
#include "common.h"
#ifndef USE_OPENMP
#include "blas_server.c"
#else
int blas_server_avail = 0;
int blas_thread_init(void){
blas_get_cpu_number();
blas_server_avail = 1;
return 0;
}
int BLASFUNC(blas_thread_shutdown)(void){
blas_server_avail = 0;
return 0;
}
static void legacy_exec(void *func, int mode, blas_arg_t *args, void *sb){
if (!(mode & BLAS_COMPLEX)){
#ifdef EXPRECISION
if (mode & BLAS_XDOUBLE){
/* REAL / Extended Double */
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, xdouble,
xdouble *, BLASLONG, xdouble *, BLASLONG,
xdouble *, BLASLONG, void *) = func;
afunc(args -> m, args -> n, args -> k,
((xdouble *)args -> alpha)[0],
args -> a, args -> lda,
args -> b, args -> ldb,
args -> c, args -> ldc, sb);
} else
#endif
if (mode & BLAS_DOUBLE){
/* REAL / Double */
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, double,
double *, BLASLONG, double *, BLASLONG,
double *, BLASLONG, void *) = func;
afunc(args -> m, args -> n, args -> k,
((double *)args -> alpha)[0],
args -> a, args -> lda,
args -> b, args -> ldb,
args -> c, args -> ldc, sb);
} else {
/* REAL / Single */
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, float,
float *, BLASLONG, float *, BLASLONG,
float *, BLASLONG, void *) = func;
afunc(args -> m, args -> n, args -> k,
((float *)args -> alpha)[0],
args -> a, args -> lda,
args -> b, args -> ldb,
args -> c, args -> ldc, sb);
}
} else {
#ifdef EXPRECISION
if (mode & BLAS_XDOUBLE){
/* COMPLEX / Extended Double */
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble,
xdouble *, BLASLONG, xdouble *, BLASLONG,
xdouble *, BLASLONG, void *) = func;
afunc(args -> m, args -> n, args -> k,
((xdouble *)args -> alpha)[0],
((xdouble *)args -> alpha)[1],
args -> a, args -> lda,
args -> b, args -> ldb,
args -> c, args -> ldc, sb);
} else
#endif
if (mode & BLAS_DOUBLE){
/* COMPLEX / Double */
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, double, double,
double *, BLASLONG, double *, BLASLONG,
double *, BLASLONG, void *) = func;
afunc(args -> m, args -> n, args -> k,
((double *)args -> alpha)[0],
((double *)args -> alpha)[1],
args -> a, args -> lda,
args -> b, args -> ldb,
args -> c, args -> ldc, sb);
} else {
/* COMPLEX / Single */
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, float, float,
float *, BLASLONG, float *, BLASLONG,
float *, BLASLONG, void *) = func;
afunc(args -> m, args -> n, args -> k,
((float *)args -> alpha)[0],
((float *)args -> alpha)[1],
args -> a, args -> lda,
args -> b, args -> ldb,
args -> c, args -> ldc, sb);
}
}
}
static void exec_threads(blas_queue_t *queue){
void *buffer, *sa, *sb;
buffer = NULL;
sa = queue -> sa;
sb = queue -> sb;
#ifdef CONSISTENT_FPCSR
__asm__ __volatile__ ("ldmxcsr %0" : : "m" (queue -> sse_mode));
__asm__ __volatile__ ("fldcw %0" : : "m" (queue -> x87_mode));
#endif
if ((sa == NULL) && (sb == NULL) && ((queue -> mode & BLAS_PTHREAD) == 0)) {
buffer = blas_memory_alloc(2);
if (sa == NULL) sa = (void *)((BLASLONG)buffer + GEMM_OFFSET_A);
if (sb == NULL) {
if (!(queue -> mode & BLAS_COMPLEX)){
#ifdef EXPRECISION
if (queue -> mode & BLAS_XDOUBLE){
sb = (void *)(((BLASLONG)sa + ((QGEMM_P * QGEMM_Q * sizeof(xdouble)
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
} else
#endif
if (queue -> mode & BLAS_DOUBLE){
sb = (void *)(((BLASLONG)sa + ((DGEMM_P * DGEMM_Q * sizeof(double)
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
} else {
sb = (void *)(((BLASLONG)sa + ((SGEMM_P * SGEMM_Q * sizeof(float)
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
}
} else {
#ifdef EXPRECISION
if (queue -> mode & BLAS_XDOUBLE){
sb = (void *)(((BLASLONG)sa + ((XGEMM_P * XGEMM_Q * 2 * sizeof(xdouble)
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
} else
#endif
if (queue -> mode & BLAS_DOUBLE){
sb = (void *)(((BLASLONG)sa + ((ZGEMM_P * ZGEMM_Q * 2 * sizeof(double)
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
} else {
sb = (void *)(((BLASLONG)sa + ((CGEMM_P * CGEMM_Q * 2 * sizeof(float)
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
}
}
}
}
if (queue -> mode & BLAS_LEGACY) {
legacy_exec(queue -> routine, queue -> mode, queue -> args, sb);
} else
if (queue -> mode & BLAS_PTHREAD) {
void (*pthreadcompat)(void *) = queue -> routine;
(pthreadcompat)(queue -> args);
} else {
int (*routine)(blas_arg_t *, void *, void *, void *, void *, BLASLONG) = queue -> routine;
(routine)(queue -> args, queue -> range_m, queue -> range_n, sa, sb, queue -> position);
}
if (buffer != NULL) blas_memory_free(buffer);
}
int exec_blas(BLASLONG num, blas_queue_t *queue){
BLASLONG i;
if ((num <= 0) || (queue == NULL)) return 0;
#ifdef CONSISTENT_FPCSR
for (i = 0; i < num; i ++) {
__asm__ __volatile__ ("fnstcw %0" : "=m" (queue[i].x87_mode));
__asm__ __volatile__ ("stmxcsr %0" : "=m" (queue[i].sse_mode));
}
#endif
#pragma omp parallel for schedule(static)
for (i = 0; i < num; i ++) {
#ifndef USE_SIMPLE_THREADED_LEVEL3
queue[i].position = i;
#endif
exec_threads(&queue[i]);
}
return 0;
}
#endif
driver/others/blas_server_win32.c
+450
View File
@@ -0,0 +1,450 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/
#include <stdio.h>
#include <stdlib.h>
#include "common.h"
/* This is a thread implementation for Win32 lazy implementation */
/* Thread server common infomation */
typedef struct{
CRITICAL_SECTION lock;
HANDLE filled;
HANDLE killed;
blas_queue_t *queue; /* Parameter Pointer */
int shutdown; /* server shutdown flag */
} blas_pool_t;
/* We need this grobal for cheking if initialization is finished. */
int blas_server_avail = 0;
/* Local Variables */
static BLASULONG server_lock = 0;
static blas_pool_t pool;
static HANDLE blas_threads [MAX_CPU_NUMBER];
static DWORD blas_threads_id[MAX_CPU_NUMBER];
static void legacy_exec(void *func, int mode, blas_arg_t *args, void *sb){
if (!(mode & BLAS_COMPLEX)){
#ifdef EXPRECISION
if (mode & BLAS_XDOUBLE){
/* REAL / Extended Double */
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, xdouble,
xdouble *, BLASLONG, xdouble *, BLASLONG,
xdouble *, BLASLONG, void *) = func;
afunc(args -> m, args -> n, args -> k,
((xdouble *)args -> alpha)[0],
args -> a, args -> lda,
args -> b, args -> ldb,
args -> c, args -> ldc, sb);
} else
#endif
if (mode & BLAS_DOUBLE){
/* REAL / Double */
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, double,
double *, BLASLONG, double *, BLASLONG,
double *, BLASLONG, void *) = func;
afunc(args -> m, args -> n, args -> k,
((double *)args -> alpha)[0],
args -> a, args -> lda,
args -> b, args -> ldb,
args -> c, args -> ldc, sb);
} else {
/* REAL / Single */
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, float,
float *, BLASLONG, float *, BLASLONG,
float *, BLASLONG, void *) = func;
afunc(args -> m, args -> n, args -> k,
((float *)args -> alpha)[0],
args -> a, args -> lda,
args -> b, args -> ldb,
args -> c, args -> ldc, sb);
}
} else {
#ifdef EXPRECISION
if (mode & BLAS_XDOUBLE){
/* COMPLEX / Extended Double */
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble,
xdouble *, BLASLONG, xdouble *, BLASLONG,
xdouble *, BLASLONG, void *) = func;
afunc(args -> m, args -> n, args -> k,
((xdouble *)args -> alpha)[0],
((xdouble *)args -> alpha)[1],
args -> a, args -> lda,
args -> b, args -> ldb,
args -> c, args -> ldc, sb);
} else
#endif
if (mode & BLAS_DOUBLE){
/* COMPLEX / Double */
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, double, double,
double *, BLASLONG, double *, BLASLONG,
double *, BLASLONG, void *) = func;
afunc(args -> m, args -> n, args -> k,
((double *)args -> alpha)[0],
((double *)args -> alpha)[1],
args -> a, args -> lda,
args -> b, args -> ldb,
args -> c, args -> ldc, sb);
} else {
/* COMPLEX / Single */
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, float, float,
float *, BLASLONG, float *, BLASLONG,
float *, BLASLONG, void *) = func;
afunc(args -> m, args -> n, args -> k,
((float *)args -> alpha)[0],
((float *)args -> alpha)[1],
args -> a, args -> lda,
args -> b, args -> ldb,
args -> c, args -> ldc, sb);
}
}
}
/* This is a main routine of threads. Each thread waits until job is */
/* queued. */
static DWORD WINAPI blas_thread_server(void *arg){
/* Thread identifier */
#ifdef SMP_DEBUG
BLASLONG cpu = (BLASLONG)arg;
#endif
void *buffer, *sa, *sb;
blas_queue_t *queue;
DWORD action;
HANDLE handles[] = {pool.filled, pool.killed};
/* Each server needs each buffer */
buffer = blas_memory_alloc(2);
#ifdef SMP_DEBUG
fprintf(STDERR, "Server[%2ld] Thread is started!\n", cpu);
#endif
while (1){
/* Waiting for Queue */
#ifdef SMP_DEBUG
fprintf(STDERR, "Server[%2ld] Waiting for Queue.\n", cpu);
#endif
do {
action = WaitForMultipleObjects(2, handles, FALSE, INFINITE);
} while ((action != WAIT_OBJECT_0) && (action == WAIT_OBJECT_0 + 1));
if (action == WAIT_OBJECT_0 + 1) break;
#ifdef SMP_DEBUG
fprintf(STDERR, "Server[%2ld] Got it.\n", cpu);
#endif
EnterCriticalSection(&pool.lock);
queue = pool.queue;
if (queue) pool.queue = queue->next;
LeaveCriticalSection(&pool.lock);
if (queue) {
int (*routine)(blas_arg_t *, void *, void *, void *, void *, BLASLONG) = queue -> routine;
if (pool.queue) SetEvent(pool.filled);
sa = queue -> sa;
sb = queue -> sb;
#ifdef CONSISTENT_FPCSR
__asm__ __volatile__ ("ldmxcsr %0" : : "m" (queue -> sse_mode));
__asm__ __volatile__ ("fldcw %0" : : "m" (queue -> x87_mode));
#endif
#ifdef SMP_DEBUG
fprintf(STDERR, "Server[%2ld] Started. Mode = 0x%03x M = %3ld N=%3ld K=%3ld\n",
cpu, queue->mode, queue-> args ->m, queue->args->n, queue->args->k);
#endif
// fprintf(stderr, "queue start[%ld]!!!\n", cpu);
#ifdef MONITOR
main_status[cpu] = MAIN_RUNNING1;
#endif
if (sa == NULL) sa = (void *)((BLASLONG)buffer + GEMM_OFFSET_A);
if (sb == NULL) {
if (!(queue -> mode & BLAS_COMPLEX)){
#ifdef EXPRECISION
if (queue -> mode & BLAS_XDOUBLE){
sb = (void *)(((BLASLONG)sa + ((XGEMM_P * XGEMM_Q * sizeof(xdouble)
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
} else
#endif
if (queue -> mode & BLAS_DOUBLE){
sb = (void *)(((BLASLONG)sa + ((DGEMM_P * DGEMM_Q * sizeof(double)
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
} else {
sb = (void *)(((BLASLONG)sa + ((SGEMM_P * SGEMM_Q * sizeof(float)
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
}
} else {
#ifdef EXPRECISION
if (queue -> mode & BLAS_XDOUBLE){
sb = (void *)(((BLASLONG)sa + ((XGEMM_P * XGEMM_Q * 2 * sizeof(xdouble)
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
} else
#endif
if (queue -> mode & BLAS_DOUBLE){
sb = (void *)(((BLASLONG)sa + ((ZGEMM_P * ZGEMM_Q * 2 * sizeof(double)
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
} else {
sb = (void *)(((BLASLONG)sa + ((CGEMM_P * CGEMM_Q * 2 * sizeof(float)
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
}
}
}
#ifdef MONITOR
main_status[cpu] = MAIN_RUNNING2;
#endif
if (!(queue -> mode & BLAS_LEGACY)) {
(routine)(queue -> args, queue -> range_m, queue -> range_n, sa, sb, queue -> position);
} else {
legacy_exec(routine, queue -> mode, queue -> args, sb);
}
}
#ifdef SMP_DEBUG
fprintf(STDERR, "Server[%2ld] Finished!\n", cpu);
#endif
EnterCriticalSection(&queue->lock);
queue -> status = BLAS_STATUS_FINISHED;
LeaveCriticalSection(&queue->lock);
SetEvent(queue->finish);
}
/* Shutdown procedure */
#ifdef SMP_DEBUG
fprintf(STDERR, "Server[%2ld] Shutdown!\n", cpu);
#endif
blas_memory_free(buffer);
return 0;
}
/* Initializing routine */
int blas_thread_init(void){
BLASLONG i;
if (blas_server_avail || (blas_cpu_number <= 1)) return 0;
LOCK_COMMAND(&server_lock);
#ifdef SMP_DEBUG
fprintf(STDERR, "Initializing Thread(Num. threads = %d)\n",
blas_cpu_number);
#endif
if (!blas_server_avail){
InitializeCriticalSection(&pool.lock);
pool.filled = CreateEvent(NULL, FALSE, FALSE, NULL);
pool.killed = CreateEvent(NULL, TRUE, FALSE, NULL);
pool.shutdown = 0;
pool.queue = NULL;
for(i = 0; i < blas_cpu_number - 1; i++){
blas_threads[i] = CreateThread(NULL, 0,
blas_thread_server, (void *)i,
0, &blas_threads_id[i]);
}
blas_server_avail = 1;
}
UNLOCK_COMMAND(&server_lock);
return 0;
}
/*
User can call one of two routines.
exec_blas_async ... immediately returns after jobs are queued.
exec_blas ... returns after jobs are finished.
*/
int exec_blas_async(BLASLONG pos, blas_queue_t *queue){
blas_queue_t *current;
current = queue;
while (current) {
InitializeCriticalSection(&current -> lock);
current -> finish = CreateEvent(NULL, FALSE, FALSE, NULL);
current -> position = pos;
#ifdef CONSISTENT_FPCSR
__asm__ __volatile__ ("fnstcw %0" : "=m" (current -> x87_mode));
__asm__ __volatile__ ("stmxcsr %0" : "=m" (current -> sse_mode));
#endif
current = current -> next;
pos ++;
}
EnterCriticalSection(&pool.lock);
if (pool.queue) {
current = pool.queue;
while (current -> next) current = current -> next;
current -> next = queue;
} else {
pool.queue = queue;
}
LeaveCriticalSection(&pool.lock);
SetEvent(pool.filled);
return 0;
}
int exec_blas_async_wait(BLASLONG num, blas_queue_t *queue){
#ifdef SMP_DEBUG
fprintf(STDERR, "Synchronization Waiting.\n");
#endif
while (num){
#ifdef SMP_DEBUG
fprintf(STDERR, "Waiting Queue ..\n");
#endif
WaitForSingleObject(queue->finish, INFINITE);
CloseHandle(queue->finish);
DeleteCriticalSection(&queue -> lock);
queue = queue -> next;
num --;
}
#ifdef SMP_DEBUG
fprintf(STDERR, "Completely Done.\n\n");
#endif
return 0;
}
/* Execute Threads */
int exec_blas(BLASLONG num, blas_queue_t *queue){
#ifndef ALL_THREADED
int (*routine)(blas_arg_t *, void *, void *, double *, double *, BLASLONG);
#endif
if ((num <= 0) || (queue == NULL)) return 0;
if ((num > 1) && queue -> next) exec_blas_async(1, queue -> next);
routine = queue -> routine;
if (!(queue -> mode & BLAS_LEGACY)) {
(routine)(queue -> args, queue -> range_m, queue -> range_n,
queue -> sa, queue -> sb, 0);
} else {
legacy_exec(routine, queue -> mode, queue -> args, queue -> sb);
}
if ((num > 1) && queue -> next) exec_blas_async_wait(num - 1, queue -> next);
return 0;
}
/* Shutdown procedure, but user don't have to call this routine. The */
/* kernel automatically kill threads. */
int blas_thread_shutdown_(void){
int i;
if (!blas_server_avail) return 0;
LOCK_COMMAND(&server_lock);
if (blas_server_avail){
SetEvent(pool.killed);
for(i = 0; i < blas_cpu_number - 1; i++){
WaitForSingleObject(blas_threads[i], INFINITE);
}
blas_server_avail = 0;
}
UNLOCK_COMMAND(&server_lock);
return 0;
}
driver/others/divtable.c
+83
View File
@@ -0,0 +1,83 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/
#include "common.h"
#ifdef SMP
#ifndef USE64BITINT
unsigned int blas_quick_divide_table[] = {
0x00000000, 0x00000001, 0x80000001, 0x55555556,
0x40000001, 0x33333334, 0x2aaaaaab, 0x24924925,
0x20000001, 0x1c71c71d, 0x1999999a, 0x1745d175,
0x15555556, 0x13b13b14, 0x12492493, 0x11111112,
0x10000001, 0x0f0f0f10, 0x0e38e38f, 0x0d79435f,
0x0ccccccd, 0x0c30c30d, 0x0ba2e8bb, 0x0b21642d,
0x0aaaaaab, 0x0a3d70a4, 0x09d89d8a, 0x097b425f,
0x0924924a, 0x08d3dcb1, 0x08888889, 0x08421085,
0x08000001, 0x07c1f07d, 0x07878788, 0x07507508,
0x071c71c8, 0x06eb3e46, 0x06bca1b0, 0x06906907,
0x06666667, 0x063e7064, 0x06186187, 0x05f417d1,
0x05d1745e, 0x05b05b06, 0x0590b217, 0x0572620b,
0x05555556, 0x0539782a, 0x051eb852, 0x05050506,
0x04ec4ec5, 0x04d4873f, 0x04bda130, 0x04a7904b,
0x04924925, 0x047dc120, 0x0469ee59, 0x0456c798,
0x04444445, 0x04325c54, 0x04210843, 0x04104105,
0x04000001,
};
#else
BLASULONG blas_quick_divide_table[] = {
0x0000000000000000, 0x0000000000000001, 0x8000000000000001, 0x5555555555555557,
0x4000000000000001, 0x3333333333333335, 0x2aaaaaaaaaaaaaac, 0x2492492492492494,
0x2000000000000001, 0x1c71c71c71c71c73, 0x199999999999999b, 0x1745d1745d1745d3,
0x1555555555555557, 0x13b13b13b13b13b3, 0x124924924924924b, 0x1111111111111113,
0x1000000000000001, 0x0f0f0f0f0f0f0f11, 0x0e38e38e38e38e3a, 0x0d79435e50d79437,
0x0cccccccccccccce, 0x0c30c30c30c30c32, 0x0ba2e8ba2e8ba2ea, 0x0b21642c8590b218,
0x0aaaaaaaaaaaaaac, 0x0a3d70a3d70a3d72, 0x09d89d89d89d89da, 0x097b425ed097b427,
0x0924924924924926, 0x08d3dcb08d3dcb0a, 0x088888888888888a, 0x0842108421084212,
0x0800000000000001, 0x07c1f07c1f07c1f2, 0x0787878787878789, 0x0750750750750752,
0x071c71c71c71c71e, 0x06eb3e45306eb3e6, 0x06bca1af286bca1c, 0x0690690690690692,
0x0666666666666668, 0x063e7063e7063e72, 0x061861861861861a, 0x05f417d05f417d07,
0x05d1745d1745d176, 0x05b05b05b05b05b2, 0x0590b21642c8590d, 0x0572620ae4c415cb,
0x0555555555555557, 0x05397829cbc14e60, 0x051eb851eb851eba, 0x0505050505050507,
0x04ec4ec4ec4ec4ee, 0x04d4873ecade304f, 0x04bda12f684bda14, 0x04a7904a7904a792,
0x0492492492492494, 0x047dc11f7047dc13, 0x0469ee58469ee586, 0x0456c797dd49c343,
0x0444444444444446, 0x04325c53ef368eb2, 0x042108421084210a, 0x0410410410410412,
0x0400000000000001,
};
#endif
#endif
driver/others/dynamic.c
+219
View File
@@ -0,0 +1,219 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/
#include "common.h"
#ifdef ARCH_X86
#define EXTERN extern
#else
#define EXTERN
#endif
EXTERN gotoblas_t gotoblas_KATMAI;
EXTERN gotoblas_t gotoblas_COPPERMINE;
EXTERN gotoblas_t gotoblas_NORTHWOOD;
EXTERN gotoblas_t gotoblas_BANIAS;
EXTERN gotoblas_t gotoblas_ATHLON;
extern gotoblas_t gotoblas_PRESCOTT;
extern gotoblas_t gotoblas_ATOM;
extern gotoblas_t gotoblas_NANO;
extern gotoblas_t gotoblas_CORE2;
extern gotoblas_t gotoblas_PENRYN;
extern gotoblas_t gotoblas_DUNNINGTON;
extern gotoblas_t gotoblas_NEHALEM;
extern gotoblas_t gotoblas_OPTERON;
extern gotoblas_t gotoblas_OPTERON_SSE3;
extern gotoblas_t gotoblas_BARCELONA;
#define VENDOR_INTEL 1
#define VENDOR_AMD 2
#define VENDOR_CENTAUR 3
#define VENDOR_UNKNOWN 99
#define BITMASK(a, b, c) ((((a) >> (b)) & (c)))
static int get_vendor(void){
int eax, ebx, ecx, edx;
char vendor[13];
cpuid(0, &eax, &ebx, &ecx, &edx);
*(int *)(&vendor[0]) = ebx;
*(int *)(&vendor[4]) = edx;
*(int *)(&vendor[8]) = ecx;
vendor[12] = (char)0;
if (!strcmp(vendor, "GenuineIntel")) return VENDOR_INTEL;
if (!strcmp(vendor, "AuthenticAMD")) return VENDOR_AMD;
if (!strcmp(vendor, "CentaurHauls")) return VENDOR_CENTAUR;
if ((eax == 0) || ((eax & 0x500) != 0)) return VENDOR_INTEL;
return VENDOR_UNKNOWN;
}
static gotoblas_t *get_coretype(void){
int eax, ebx, ecx, edx;
int family, exfamily, model, vendor, exmodel;
cpuid(1, &eax, &ebx, &ecx, &edx);
family = BITMASK(eax, 8, 0x0f);
exfamily = BITMASK(eax, 20, 0xff);
model = BITMASK(eax, 4, 0x0f);
exmodel = BITMASK(eax, 16, 0x0f);
vendor = get_vendor();
if (vendor == VENDOR_INTEL){
switch (family) {
case 0x6:
switch (exmodel) {
case 0:
if (model <= 0x7) return &gotoblas_KATMAI;
if ((model == 0x8) || (model == 0xa) || (model == 0xb)) return &gotoblas_COPPERMINE;
if ((model == 0x9) || (model == 0xd)) return &gotoblas_BANIAS;
if (model == 14) return &gotoblas_BANIAS;
if (model == 15) return &gotoblas_CORE2;
return NULL;
case 1:
if (model == 6) return &gotoblas_CORE2;
if (model == 7) return &gotoblas_PENRYN;
if (model == 13) return &gotoblas_DUNNINGTON;
if ((model == 10) || (model == 11) || (model == 14) || (model == 15)) return &gotoblas_NEHALEM;
if (model == 12) return &gotoblas_ATOM;
return NULL;
}
case 0xf:
if (model <= 0x2) return &gotoblas_NORTHWOOD;
return &gotoblas_PRESCOTT;
}
}
if (vendor == VENDOR_AMD){
if (family <= 0xe) return &gotoblas_ATHLON;
if (family == 0xf){
if ((exfamily == 0) || (exfamily == 2)) {
if (ecx & (1 << 0)) return &gotoblas_OPTERON_SSE3;
else return &gotoblas_OPTERON;
} else {
return &gotoblas_BARCELONA;
}
}
}
if (vendor == VENDOR_CENTAUR) {
switch (family) {
case 0x6:
return &gotoblas_NANO;
break;
}
}
return NULL;
}
static char *corename[] = {
"Unknown",
"Katmai",
"Coppermine",
"Northwood",
"Prescott",
"Banias",
"Atom",
"Core2",
"Penryn",
"Dunnington",
"Nehalem",
"Athlon",
"Opteron",
"Opteron(SSE3)",
"Barcelona",
"Nano",
};
char *gotoblas_corename(void) {
if (gotoblas == &gotoblas_KATMAI) return corename[ 1];
if (gotoblas == &gotoblas_COPPERMINE) return corename[ 2];
if (gotoblas == &gotoblas_NORTHWOOD) return corename[ 3];
if (gotoblas == &gotoblas_PRESCOTT) return corename[ 4];
if (gotoblas == &gotoblas_BANIAS) return corename[ 5];
if (gotoblas == &gotoblas_ATOM) return corename[ 6];
if (gotoblas == &gotoblas_CORE2) return corename[ 7];
if (gotoblas == &gotoblas_PENRYN) return corename[ 8];
if (gotoblas == &gotoblas_DUNNINGTON) return corename[ 9];
if (gotoblas == &gotoblas_NEHALEM) return corename[10];
if (gotoblas == &gotoblas_ATHLON) return corename[11];
if (gotoblas == &gotoblas_OPTERON_SSE3) return corename[12];
if (gotoblas == &gotoblas_OPTERON) return corename[13];
if (gotoblas == &gotoblas_BARCELONA) return corename[14];
if (gotoblas == &gotoblas_NANO) return corename[15];
return corename[0];
}
void gotoblas_dynamic_init(void) {
if (gotoblas) return;
gotoblas = get_coretype();
#ifdef ARCH_X86
if (gotoblas == NULL) gotoblas = gotoblas_KATMAI;
#else
if (gotoblas == NULL) gotoblas = gotoblas_PRESCOTT;
#endif
if (gotoblas && gotoblas -> init) {
gotoblas -> init();
} else {
fprintf(stderr, "GotoBLAS : Architecture Initialization failed. No initialization function found.\n");
exit(1);
}
}
void gotoblas_dynamic_quit(void) {
gotoblas = NULL;
}
driver/others/init.c
+697
View File
@@ -0,0 +1,697 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/
#include "common.h"
#if defined(OS_LINUX) && defined(SMP)
#define _GNU_SOURCE
#include <sys/sysinfo.h>
#include <sys/syscall.h>
#include <sys/shm.h>
#include <fcntl.h>
#include <sched.h>
#include <dirent.h>
#include <dlfcn.h>
#define MAX_NODES 16
#define MAX_CPUS 256
#define SH_MAGIC 0x510510
#define CPUMAP_NAME "/sys/devices/system/node/node%d/cpumap"
#define SHARE_NAME "/sys/devices/system/cpu/cpu%d/cache/index%d/shared_cpu_map"
#define NODE_DIR "/sys/devices/system/node"
#undef DEBUG
/* Private variables */
typedef struct {
unsigned long lock;
unsigned int magic;
unsigned int shmid;
int num_nodes;
int num_procs;
int final_num_procs;
unsigned long avail;
unsigned long cpu_info [MAX_CPUS];
unsigned long node_info [MAX_NODES];
int cpu_use[MAX_CPUS];
} shm_t;
static cpu_set_t cpu_orig_mask[4];
static int cpu_mapping[MAX_CPUS];
static int node_mapping[MAX_CPUS * 4];
static int cpu_sub_mapping[MAX_CPUS];
static int disable_mapping;
/* Number of cores per nodes */
static int node_cpu[MAX_NODES];
static int node_equal = 0;
static shm_t *common = (void *)-1;
static int shmid, pshmid;
static void *paddr;
static unsigned long lprocmask, lnodemask;
static int numprocs = 1;
static int numnodes = 1;
#if 1
#define READ_CPU(x) ( (x) & 0xff)
#define READ_NODE(x) (((x) >> 8) & 0xff)
#define READ_CORE(x) (((x) >> 16) & 0xff)
#define WRITE_CPU(x) (x)
#define WRITE_NODE(x) ((x) << 8)
#define WRITE_CORE(x) ((x) << 16)
#else
#define READ_CPU(x) ( (x) & 0xff)
#define READ_CORE(x) (((x) >> 8) & 0xff)
#define READ_NODE(x) (((x) >> 16) & 0xff)
#define WRITE_CPU(x) (x)
#define WRITE_CORE(x) ((x) << 8)
#define WRITE_NODE(x) ((x) << 16)
#endif
static inline int popcount(unsigned long number) {
int count = 0;
while (number > 0) {
if (number & 1) count ++;
number >>= 1;
}
return count;
}
static inline int rcount(unsigned long number) {
int count = -1;
while ((number > 0) && ((number & 0)) == 0) {
count ++;
number >>= 1;
}
return count;
}
static inline unsigned long get_cpumap(int node) {
int infile;
unsigned long affinity;
char name[160];
char *p, *dummy;
sprintf(name, CPUMAP_NAME, node);
infile = open(name, O_RDONLY);
affinity = 0;
if (infile != -1) {
read(infile, name, sizeof(name));
p = name;
while ((*p == '0') || (*p == ',')) p++;
affinity = strtol(p, &dummy, 16);
close(infile);
}
return affinity;
}
static inline unsigned long get_share(int cpu, int level) {
int infile;
unsigned long affinity;
char name[160];
char *p;
sprintf(name, SHARE_NAME, cpu, level);
infile = open(name, O_RDONLY);
affinity = (1UL << cpu);
if (infile != -1) {
read(infile, name, sizeof(name));
p = name;
while ((*p == '0') || (*p == ',')) p++;
affinity = strtol(p, &p, 16);
close(infile);
}
return affinity;
}
static int numa_check(void) {
DIR *dp;
struct dirent *dir;
int node;
common -> num_nodes = 0;
dp = opendir(NODE_DIR);
if (dp == NULL) {
common -> num_nodes = 1;
return 0;
}
for (node = 0; node < MAX_NODES; node ++) common -> node_info[node] = 0;
while ((dir = readdir(dp)) != NULL) {
if (*(unsigned int *) dir -> d_name == 0x065646f6eU) {
node = atoi(&dir -> d_name[4]);
if (node > MAX_NODES) {
fprintf(stderr, "\nGotoBLAS Warining : MAX_NODES (NUMA) is too small. Terminated.\n");
exit(1);
}
common -> num_nodes ++;
common -> node_info[node] = get_cpumap(node);
}
}
closedir(dp);
if (common -> num_nodes == 1) return 1;
#ifdef DEBUG
fprintf(stderr, "Numa found : number of Nodes = %2d\n", common -> num_nodes);
for (node = 0; node < common -> num_nodes; node ++)
fprintf(stderr, "MASK (%2d) : %08lx\n", node, common -> node_info[node]);
#endif
return common -> num_nodes;
}
static void numa_mapping(void) {
int node, cpu, core;
int i, j, h;
unsigned long work, bit;
int count = 0;
for (node = 0; node < common -> num_nodes; node ++) {
core = 0;
for (cpu = 0; cpu < common -> num_procs; cpu ++) {
if (common -> node_info[node] & common -> avail & (1UL << cpu)) {
common -> cpu_info[count] = WRITE_CORE(core) | WRITE_NODE(node) | WRITE_CPU(cpu);
count ++;
core ++;
}
}
}
#ifdef DEBUG
fprintf(stderr, "\nFrom /sys ...\n\n");
for (cpu = 0; cpu < count; cpu++)
fprintf(stderr, "CPU (%2d) : %08lx\n", cpu, common -> cpu_info[cpu]);
#endif
h = 1;
while (h < count) h = 2 * h + 1;
while (h > 1) {
h /= 2;
for (i = h; i < count; i++) {
work = common -> cpu_info[i];
bit = CPU_ISSET(i, &cpu_orig_mask[0]);
j = i - h;
while (work < common -> cpu_info[j]) {
common -> cpu_info[j + h] = common -> cpu_info[j];
if (CPU_ISSET(j, &cpu_orig_mask[0])) {
CPU_SET(j + h, &cpu_orig_mask[0]);
} else {
CPU_CLR(j + h, &cpu_orig_mask[0]);
}
j -= h;
if (j < 0) break;
}
common -> cpu_info[j + h] = work;
if (bit) {
CPU_SET(j + h, &cpu_orig_mask[0]);
} else {
CPU_CLR(j + h, &cpu_orig_mask[0]);
}
}
}
#ifdef DEBUG
fprintf(stderr, "\nSorting ...\n\n");
for (cpu = 0; cpu < count; cpu++)
fprintf(stderr, "CPU (%2d) : %08lx\n", cpu, common -> cpu_info[cpu]);
#endif
}
static void disable_hyperthread(void) {
unsigned long share;
int cpu;
common -> avail = (1UL << common -> num_procs) - 1;
#ifdef DEBUG
fprintf(stderr, "\nAvail CPUs : %04lx.\n", common -> avail);
#endif
for (cpu = 0; cpu < common -> num_procs; cpu ++) {
share = (get_share(cpu, 1) & common -> avail);
if (popcount(share) > 1) {
#ifdef DEBUG
fprintf(stderr, "Detected Hyper Threading on CPU %4x; disabled CPU %04lx.\n",
cpu, share & ~(1UL << cpu));
#endif
common -> avail &= ~((share & ~(1UL << cpu)));
}
}
}
static void disable_affinity(void) {
#ifdef DEBUG
fprintf(stderr, "Final all available CPUs : %04lx.\n\n", common -> avail);
fprintf(stderr, "CPU mask : %04lx.\n\n", *(unsigned long *)&cpu_orig_mask[0]);
#endif
lprocmask = (1UL << common -> final_num_procs) - 1;
#ifndef USE_OPENMP
lprocmask &= *(unsigned long *)&cpu_orig_mask[0];
#endif
#ifdef DEBUG
fprintf(stderr, "I choose these CPUs : %04lx.\n\n", lprocmask);
#endif
}
static void setup_mempolicy(void) {
int cpu, mynode, maxcpu;
for (cpu = 0; cpu < MAX_NODES; cpu ++) node_cpu[cpu] = 0;
maxcpu = 0;
for (cpu = 0; cpu < numprocs; cpu ++) {
mynode = READ_NODE(common -> cpu_info[cpu_sub_mapping[cpu]]);
lnodemask |= (1UL << mynode);
node_cpu[mynode] ++;
if (maxcpu < node_cpu[mynode]) maxcpu = node_cpu[mynode];
}
node_equal = 1;
for (cpu = 0; cpu < MAX_NODES; cpu ++) if ((node_cpu[cpu] != 0) && (node_cpu[cpu] != maxcpu)) node_equal = 0;
if (lnodemask) {
#ifdef DEBUG
fprintf(stderr, "Node mask = %lx\n", lnodemask);
#endif
my_set_mempolicy(MPOL_INTERLEAVE, &lnodemask, sizeof(lnodemask) * 8);
numnodes = popcount(lnodemask);
}
}
static inline int is_dead(int id) {
struct shmid_ds ds;
return shmctl(id, IPC_STAT, &ds);
}
static void open_shmem(void) {
int try = 0;
do {
shmid = shmget(SH_MAGIC, 4096, 0666);
if (shmid == -1) {
shmid = shmget(SH_MAGIC, 4096, IPC_CREAT | 0666);
}
try ++;
} while ((try < 10) && (shmid == -1));
if (shmid == -1) {
fprintf(stderr, "GotoBLAS : Can't open shared memory. Terminated.\n");
exit(1);
}
if (shmid != -1) common = (shm_t *)shmat(shmid, NULL, 0);
#ifdef DEBUG
fprintf(stderr, "Shared Memory id = %x Address = %p\n", shmid, common);
#endif
}
static void create_pshmem(void) {
pshmid = shmget(IPC_PRIVATE, 4096, IPC_CREAT | 0666);
paddr = shmat(pshmid, NULL, 0);
shmctl(pshmid, IPC_RMID, 0);
#ifdef DEBUG
fprintf(stderr, "Private Shared Memory id = %x Address = %p\n", pshmid, paddr);
#endif
}
static void local_cpu_map(void) {
int cpu, id, mapping;
cpu = 0;
mapping = 0;
do {
id = common -> cpu_use[cpu];
if (id > 0) {
if (is_dead(id)) common -> cpu_use[cpu] = 0;
}
if ((common -> cpu_use[cpu] == 0) && (lprocmask & (1UL << cpu))) {
common -> cpu_use[cpu] = pshmid;
cpu_mapping[mapping] = READ_CPU(common -> cpu_info[cpu]);
cpu_sub_mapping[mapping] = cpu;
mapping ++;
}
cpu ++;
} while ((mapping < numprocs) && (cpu < common -> final_num_procs));
disable_mapping = 0;
if ((mapping < numprocs) || (numprocs == 1)) {
for (cpu = 0; cpu < common -> final_num_procs; cpu ++) {
if (common -> cpu_use[cpu] == pshmid) common -> cpu_use[cpu] = 0;
}
disable_mapping = 1;
}
#ifdef DEBUG
for (cpu = 0; cpu < numprocs; cpu ++) {
fprintf(stderr, "Local Mapping : %2d --> %2d (%2d)\n", cpu, cpu_mapping[cpu], cpu_sub_mapping[cpu]);
}
#endif
}
/* Public Functions */
int get_num_procs(void) { return numprocs; }
int get_num_nodes(void) { return numnodes; }
int get_node_equal(void) {
return (((blas_cpu_number % numnodes) == 0) && node_equal);
}
int gotoblas_set_affinity(int pos) {
cpu_set_t cpu_mask;
int mynode = 1;
/* if number of threads is larger than inital condition */
if (pos < 0) {
sched_setaffinity(0, sizeof(cpu_orig_mask), &cpu_orig_mask[0]);
return 0;
}
if (!disable_mapping) {
mynode = READ_NODE(common -> cpu_info[cpu_sub_mapping[pos]]);
#ifdef DEBUG
fprintf(stderr, "Giving Affinity[%4d %3d] --> %3d My node = %3d\n", getpid(), pos, cpu_mapping[pos], mynode);
#endif
CPU_ZERO(&cpu_mask);
CPU_SET (cpu_mapping[pos], &cpu_mask);
sched_setaffinity(0, sizeof(cpu_mask), &cpu_mask);
node_mapping[WhereAmI()] = mynode;
}
return mynode;
}
int get_node(void) {
if (!disable_mapping) return node_mapping[WhereAmI()];
return 1;
}
static int initialized = 0;
void gotoblas_affinity_init(void) {
int cpu, num_avail;
#ifndef USE_OPENMP
cpu_set_t cpu_mask;
#endif
if (initialized) return;
initialized = 1;
sched_getaffinity(0, sizeof(cpu_orig_mask), &cpu_orig_mask[0]);
#ifdef USE_OPENMP
numprocs = 0;
#else
numprocs = readenv("GOTO_NUM_THREADS");
#endif
if (numprocs == 0) numprocs = readenv("OMP_NUM_THREADS");
numnodes = 1;
if (numprocs == 1) {
disable_mapping = 1;
return;
}
create_pshmem();
open_shmem();
while ((common -> lock) && (common -> magic != SH_MAGIC)) {
if (is_dead(common -> shmid)) {
common -> lock = 0;
common -> shmid = 0;
common -> magic = 0;
} else {
sched_yield();
}
}
blas_lock(&common -> lock);
if ((common -> shmid) && is_dead(common -> shmid)) common -> magic = 0;
common -> shmid = pshmid;
if (common -> magic != SH_MAGIC) {
#ifdef DEBUG
fprintf(stderr, "Shared Memory Initialization.\n");
#endif
common -> num_procs = get_nprocs();
for (cpu = 0; cpu < common -> num_procs; cpu++) common -> cpu_info[cpu] = cpu;
numa_check();
disable_hyperthread();
if (common -> num_nodes > 1) numa_mapping();
common -> final_num_procs = popcount(common -> avail);
for (cpu = 0; cpu < common -> final_num_procs; cpu ++) common -> cpu_use[cpu] = 0;
common -> magic = SH_MAGIC;
}
disable_affinity();
num_avail = popcount(lprocmask);
if ((numprocs <= 0) || (numprocs > num_avail)) numprocs = num_avail;
#ifdef DEBUG
fprintf(stderr, "Number of threads = %d\n", numprocs);
#endif
local_cpu_map();
blas_unlock(&common -> lock);
#ifndef USE_OPENMP
if (!disable_mapping) {
#ifdef DEBUG
fprintf(stderr, "Giving Affinity[%3d] --> %3d\n", 0, cpu_mapping[0]);
#endif
CPU_ZERO(&cpu_mask);
CPU_SET (cpu_mapping[0], &cpu_mask);
sched_setaffinity(0, sizeof(cpu_mask), &cpu_mask);
node_mapping[WhereAmI()] = READ_NODE(common -> cpu_info[cpu_sub_mapping[0]]);
setup_mempolicy();
if (readenv("GOTOBLAS_MAIN_FREE")) {
sched_setaffinity(0, sizeof(cpu_orig_mask), &cpu_orig_mask[0]);
}
}
#endif
#ifdef DEBUG
fprintf(stderr, "Initialization is done.\n");
#endif
}
void gotoblas_affinity_quit(void) {
int i;
struct shmid_ds ds;
#ifdef DEBUG
fprintf(stderr, "Terminating ..\n");
#endif
if ((numprocs == 1) || (initialized == 0)) return;
if (!disable_mapping) {
blas_lock(&common -> lock);
for (i = 0; i < numprocs; i ++) common -> cpu_use[cpu_mapping[i]] = -1;
blas_unlock(&common -> lock);
}
shmctl(shmid, IPC_STAT, &ds);
if (ds.shm_nattch == 1) shmctl(shmid, IPC_RMID, 0);
shmdt(common);
shmdt(paddr);
initialized = 0;
}
#else
void gotoblas_affinity_init(void) {};
void gotoblas_set_affinity(int threads) {};
void gotoblas_set_affinity2(int threads) {};
void gotoblas_affinity_reschedule(void) {};
int get_num_procs(void) { return get_nprocs(); }
int get_num_nodes(void) { return 1; }
int get_node(void) { return 1;}
#endif
driver/others/lamc3.c
+50
View File
@@ -0,0 +1,50 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/
#include "common.h"
#ifdef NEED_F2CCONV
double
#else
FLOAT
#endif
NAME(FLOAT *a, FLOAT *b){
return *a + *b;
}
driver/others/lamch.c
+200
View File
@@ -0,0 +1,200 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/
#include "common.h"
#if 0
static FLOAT hdata[] __attribute__((aligned(128))) = {
#ifdef XDOUBLE
+0x1.0000000000000000P-00064L,
+0x1.0000000000000000P-16382L,
+0x1.0000000000000000P+00001L,
+0x1.0000000000000000P-00063L,
+0x1.0000000000000000P+00006L,
+0x1.0000000000000000P+00000L,
-0x1.ffe8000000000000P+00013L,
+0x1.0000000000000000P-16382L,
+0x1.0004000000000000P+00014L,
+0x1.fffffffffffffffeP+16383L,
#elif defined DOUBLE
+0x1.0000000000000P-0053,
+0x1.0000000000000P-1022,
+0x1.0000000000000P+0001,
+0x1.0000000000000P-0052,
+0x1.a800000000000P+0005,
+0x1.0000000000000P+0000,
-0x1.fe80000000000P+0009,
+0x1.0000000000000P-1022,
+0x1.0000000000000P+0010,
+0x1.fffffffffffffP+1023,
#else
+0x1.000000P-024f,
+0x1.000000P-126f,
+0x1.000000P+001f,
+0x1.000000P-023f,
+0x1.800000P+004f,
+0x1.000000P+000f,
-0x1.f40000P+006f,
+0x1.000000P-126f,
+0x1.000000P+007f,
+0x1.fffffeP+127f,
#endif
};
#endif
static unsigned int idata[] __attribute__((aligned(128))) = {
#if defined XDOUBLE
#ifndef __BIG_ENDIAN__
0x00000000, 0x80000000, 0x00003fbf, 0x00000000,
0x00000000, 0x80000000, 0x00000001, 0x00000000,
0x00000000, 0x80000000, 0x00004000, 0x00000000,
0x00000000, 0x80000000, 0x00003fc0, 0x00000000,
0x00000000, 0x80000000, 0x00004005, 0x00000000,
0x00000000, 0x80000000, 0x00003fff, 0x00000000,
0x00000000, 0xff400000, 0x0000c00c, 0x00000000,
0x00000000, 0x80000000, 0x00000001, 0x00000000,
0x00000000, 0x80200000, 0x0000400d, 0x00000000,
0xffffffff, 0xffffffff, 0x00007ffe, 0x00000000,
#else
0x00000000, 0x00003fbf, 0x80000000, 0x00000000,
0x00000000, 0x00000001, 0x80000000, 0x00000000,
0x00000000, 0x00004000, 0x80000000, 0x00000000,
0x00000000, 0x00003fc0, 0x80000000, 0x00000000,
0x00000000, 0x00004005, 0x80000000, 0x00000000,
0x00000000, 0x00003fff, 0x80000000, 0x00000000,
0x00000000, 0x0000c00c, 0xff400000, 0x00000000,
0x00000000, 0x00000001, 0x80000000, 0x00000000,
0x00000000, 0x0000400d, 0x80200000, 0x00000000,
0x00000000, 0x00007ffe, 0xffffffff, 0xffffffff,
#endif
#elif defined DOUBLE
#ifndef __BIG_ENDIAN__
0x00000000, 0x3ca00000,
0x00000000, 0x00100000,
0x00000000, 0x40000000,
0x00000000, 0x3cb00000,
0x00000000, 0x404a8000,
0x00000000, 0x3ff00000,
0x00000000, 0xc08fe800,
0x00000000, 0x00100000,
0x00000000, 0x40900000,
0xffffffff, 0x7fefffff,
#else
0x3ca00000, 0x00000000,
0x00100000, 0x00000000,
0x40000000, 0x00000000,
0x3cb00000, 0x00000000,
0x404a8000, 0x00000000,
0x3ff00000, 0x00000000,
0xc08fe800, 0x00000000,
0x00100000, 0x00000000,
0x40900000, 0x00000000,
0x7fefffff, 0xffffffff,
#endif
#else
0x33800000,
0x00800000,
0x40000000,
0x34000000,
0x41c00000,
0x3f800000,
0xc2fa0000,
0x00800000,
0x43000000,
0x7f7fffff,
#endif
};
#ifdef NEED_F2CCONV
double
#else
FLOAT
#endif
NAME(char *P){
char p = *P;
int pos;
FLOAT *hdata = (FLOAT *)idata;
TOUPPER(p);
switch (p) {
case 'E':
pos = 0;
break;
case 'S':
pos = 1;
break;
case 'B':
pos = 2;
break;
case 'P':
pos = 3;
break;
case 'N':
pos = 4;
break;
case 'R':
pos = 5;
break;
case 'M':
pos = 6;
break;
case 'U':
pos = 7;
break;
case 'L':
pos = 8;
break;
case 'O':
pos = 9;
break;
default:
pos = 0;
break;
}
return hdata[pos];
}
driver/others/lsame.c
+50
View File
@@ -0,0 +1,50 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/
#include <ctype.h>
int NAME(char *A, char *B){
char a = *A;
char b = *B;
if (a > 96) a -= 32;
if (b > 96) b -= 32;
return (a == b);
}
driver/others/memory.c
+1257
View File
File diff suppressed because it is too large Load Diff
driver/others/memory_qalloc.c
+77
View File
@@ -0,0 +1,77 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/
#include <stdio.h>
#include "common.h"
#ifndef SMP
#define blas_cpu_number 1
#else
int blas_cpu_number = 1;
int blas_get_cpu_number(void){
return blas_cpu_number;
}
#endif
#define FIXED_PAGESIZE 4096
void *sa = NULL;
void *sb = NULL;
static double static_buffer[BUFFER_SIZE/sizeof(double)];
void *blas_memory_alloc(int numproc){
if (sa == NULL){
#if 1
sa = (void *)qalloc(QFAST, BUFFER_SIZE);
#else
sa = (void *)malloc(BUFFER_SIZE);
#endif
sb = (void *)&static_buffer[0];
}
return sa;
}
void blas_memory_free(void *free_area){
return;
}
driver/others/parameter.c
+668
View File
@@ -0,0 +1,668 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/
#include <stdio.h>
#include <string.h>
#include "common.h"
int get_L2_size(void);
#define DEFAULT_GEMM_P 128
#define DEFAULT_GEMM_Q 128
#define DEFAULT_GEMM_R 128
/* Global Parameter */
#if SGEMM_P == sgemm_p
BLASLONG sgemm_p = DEFAULT_GEMM_P;
#else
BLASLONG sgemm_p = SGEMM_P;
#endif
#if DGEMM_P == dgemm_p
BLASLONG dgemm_p = DEFAULT_GEMM_P;
#else
BLASLONG dgemm_p = DGEMM_P;
#endif
#if CGEMM_P == cgemm_p
BLASLONG cgemm_p = DEFAULT_GEMM_P;
#else
BLASLONG cgemm_p = CGEMM_P;
#endif
#if ZGEMM_P == zgemm_p
BLASLONG zgemm_p = DEFAULT_GEMM_P;
#else
BLASLONG zgemm_p = ZGEMM_P;
#endif
#if SGEMM_Q == sgemm_q
BLASLONG sgemm_q = DEFAULT_GEMM_Q;
#else
BLASLONG sgemm_q = SGEMM_Q;
#endif
#if DGEMM_Q == dgemm_q
BLASLONG dgemm_q = DEFAULT_GEMM_Q;
#else
BLASLONG dgemm_q = DGEMM_Q;
#endif
#if CGEMM_Q == cgemm_q
BLASLONG cgemm_q = DEFAULT_GEMM_Q;
#else
BLASLONG cgemm_q = CGEMM_Q;
#endif
#if ZGEMM_Q == zgemm_q
BLASLONG zgemm_q = DEFAULT_GEMM_Q;
#else
BLASLONG zgemm_q = ZGEMM_Q;
#endif
#if SGEMM_R == sgemm_r
BLASLONG sgemm_r = DEFAULT_GEMM_R;
#else
BLASLONG sgemm_r = SGEMM_R;
#endif
#if DGEMM_R == dgemm_r
BLASLONG dgemm_r = DEFAULT_GEMM_R;
#else
BLASLONG dgemm_r = DGEMM_R;
#endif
#if CGEMM_R == cgemm_r
BLASLONG cgemm_r = DEFAULT_GEMM_R;
#else
BLASLONG cgemm_r = CGEMM_R;
#endif
#if ZGEMM_R == zgemm_r
BLASLONG zgemm_r = DEFAULT_GEMM_R;
#else
BLASLONG zgemm_r = ZGEMM_R;
#endif
#if defined(EXPRECISION) || defined(QUAD_PRECISION)
#if QGEMM_P == qgemm_p
BLASLONG qgemm_p = DEFAULT_GEMM_P;
#else
BLASLONG qgemm_p = QGEMM_P;
#endif
#if XGEMM_P == xgemm_p
BLASLONG xgemm_p = DEFAULT_GEMM_P;
#else
BLASLONG xgemm_p = XGEMM_P;
#endif
#if QGEMM_Q == qgemm_q
BLASLONG qgemm_q = DEFAULT_GEMM_Q;
#else
BLASLONG qgemm_q = QGEMM_Q;
#endif
#if XGEMM_Q == xgemm_q
BLASLONG xgemm_q = DEFAULT_GEMM_Q;
#else
BLASLONG xgemm_q = XGEMM_Q;
#endif
#if QGEMM_R == qgemm_r
BLASLONG qgemm_r = DEFAULT_GEMM_R;
#else
BLASLONG qgemm_r = QGEMM_R;
#endif
#if XGEMM_R == xgemm_r
BLASLONG xgemm_r = DEFAULT_GEMM_R;
#else
BLASLONG xgemm_r = XGEMM_R;
#endif
#endif
#if defined(ARCH_X86) || defined(ARCH_X86_64)
int get_L2_size(void){
int eax, ebx, ecx, edx;
#if defined(ATHLON) || defined(OPTERON) || defined(BARCELONA) || \
defined(CORE_PRESCOTT) || defined(CORE_CORE2) || defined(PENRYN) || defined(DUNNINGTON) || \
defined(CORE_NEHALEM) || defined(ATOM) || defined(GENERIC)
cpuid(0x80000006, &eax, &ebx, &ecx, &edx);
return BITMASK(ecx, 16, 0xffff);
#else
int info[15];
int i;
cpuid(2, &eax, &ebx, &ecx, &edx);
info[ 0] = BITMASK(eax, 8, 0xff);
info[ 1] = BITMASK(eax, 16, 0xff);
info[ 2] = BITMASK(eax, 24, 0xff);
info[ 3] = BITMASK(ebx, 0, 0xff);
info[ 4] = BITMASK(ebx, 8, 0xff);
info[ 5] = BITMASK(ebx, 16, 0xff);
info[ 6] = BITMASK(ebx, 24, 0xff);
info[ 7] = BITMASK(ecx, 0, 0xff);
info[ 8] = BITMASK(ecx, 8, 0xff);
info[ 9] = BITMASK(ecx, 16, 0xff);
info[10] = BITMASK(ecx, 24, 0xff);
info[11] = BITMASK(edx, 0, 0xff);
info[12] = BITMASK(edx, 8, 0xff);
info[13] = BITMASK(edx, 16, 0xff);
info[14] = BITMASK(edx, 24, 0xff);
for (i = 0; i < 15; i++){
switch (info[i]){
case 0x3b :
case 0x41 :
case 0x79 :
return 128;
break;
case 0x3c :
case 0x42 :
case 0x7a :
case 0x7e :
case 0x82 :
return 256;
break;
case 0x43 :
case 0x7b :
case 0x7f :
case 0x83 :
case 0x86 :
return 512;
break;
case 0x44 :
case 0x78 :
case 0x7c :
case 0x84 :
case 0x87 :
return 1024;
break;
case 0x45 :
case 0x7d :
case 0x85 :
return 2048;
case 0x49 :
return 4096;
break;
}
}
/* Never reached */
return 0;
#endif
}
void blas_set_parameter(void){
char *p;
int factor;
int size = get_L2_size();
#if defined(CORE_KATMAI) || defined(CORE_COPPERMINE) || defined(CORE_BANIAS)
size >>= 7;
#if defined(CORE_BANIAS) && (HAVE_HIT > 1)
sgemm_p = 64 / HAVE_HIT * size;
dgemm_p = 32 / HAVE_HIT * size;
cgemm_p = 32 / HAVE_HIT * size;
zgemm_p = 16 / HAVE_HIT * size;
#ifdef EXPRECISION
qgemm_p = 16 / HAVE_HIT * size;
xgemm_p = 8 / HAVE_HIT * size;
#endif
#ifdef QUAD_PRECISION
qgemm_p = 8 / HAVE_HIT * size;
xgemm_p = 4 / HAVE_HIT * size;
#endif
#else
sgemm_p = 64 * size;
dgemm_p = 32 * size;
cgemm_p = 32 * size;
zgemm_p = 16 * size;
#ifdef EXPRECISION
qgemm_p = 16 * size;
xgemm_p = 8 * size;
#endif
#ifdef QUAD_PRECISION
qgemm_p = 8 * size;
xgemm_p = 4 * size;
#endif
#endif
#endif
#if defined(CORE_NORTHWOOD)
size >>= 7;
#ifdef ALLOC_HUGETLB
sgemm_p = 128 * size;
dgemm_p = 64 * size;
cgemm_p = 64 * size;
zgemm_p = 32 * size;
#ifdef EXPRECISION
qgemm_p = 32 * size;
xgemm_p = 16 * size;
#endif
#ifdef QUAD_PRECISION
qgemm_p = 16 * size;
xgemm_p = 8 * size;
#endif
#else
sgemm_p = 96 * size;
dgemm_p = 48 * size;
cgemm_p = 48 * size;
zgemm_p = 24 * size;
#ifdef EXPRECISION
qgemm_p = 24 * size;
xgemm_p = 12 * size;
#endif
#ifdef QUAD_PRECISION
qgemm_p = 12 * size;
xgemm_p = 6 * size;
#endif
#endif
#endif
#if defined(CORE_CORE2)
size >>= 9;
sgemm_p = 92 * size;
dgemm_p = 46 * size;
cgemm_p = 46 * size;
zgemm_p = 23 * size;
#ifdef EXPRECISION
qgemm_p = 23 * size;
xgemm_p = 11 * size;
#endif
#ifdef QUAD_PRECISION
qgemm_p = 11 * size;
xgemm_p = 5 * size;
#endif
#endif
#if defined(PENRYN)
size >>= 9;
sgemm_p = 1024;
dgemm_p = 512;
cgemm_p = 512;
zgemm_p = 256;
#ifdef EXPRECISION
qgemm_p = 256;
xgemm_p = 128;
#endif
#ifdef QUAD_PRECISION
qgemm_p = 21 * size + 4;
xgemm_p = 10 * size + 2;
#endif
#endif
#if defined(DUNNINGTON)
size >>= 9;
sgemm_p = 384;
dgemm_p = 384;
cgemm_p = 384;
zgemm_p = 384;
#ifdef EXPRECISION
qgemm_p = 384;
xgemm_p = 384;
#endif
#ifdef QUAD_PRECISION
qgemm_p = 21 * size + 4;
xgemm_p = 10 * size + 2;
#endif
#endif
#if defined(NEHALEM)
sgemm_p = 1024;
dgemm_p = 512;
cgemm_p = 512;
zgemm_p = 256;
#ifdef EXPRECISION
qgemm_p = 256;
xgemm_p = 128;
#endif
#endif
#if defined(CORE_PRESCOTT) || defined(GENERIC)
size >>= 6;
if (size > 16) size = 16;
sgemm_p = 56 * size;
dgemm_p = 28 * size;
cgemm_p = 28 * size;
zgemm_p = 14 * size;
#ifdef EXPRECISION
qgemm_p = 14 * size;
xgemm_p = 7 * size;
#endif
#ifdef QUAD_PRECISION
qgemm_p = 7 * size;
xgemm_p = 3 * size;
#endif
#endif
#if defined(CORE_OPTERON)
sgemm_p = 224 + 14 * (size >> 5);
dgemm_p = 112 + 14 * (size >> 6);
cgemm_p = 116 + 14 * (size >> 6);
zgemm_p = 58 + 14 * (size >> 7);
#ifdef EXPRECISION
qgemm_p = 58 + 14 * (size >> 7);
xgemm_p = 29 + 14 * (size >> 8);
#endif
#ifdef QUAD_PRECISION
qgemm_p = 29 + 14 * (size >> 8);
xgemm_p = 15 + 14 * (size >> 9);
#endif
#endif
#if defined(ATOM)
size >>= 8;
sgemm_p = 256;
dgemm_p = 128;
cgemm_p = 128;
zgemm_p = 64;
#ifdef EXPRECISION
qgemm_p = 64;
xgemm_p = 32;
#endif
#ifdef QUAD_PRECISION
qgemm_p = 32;
xgemm_p = 16;
#endif
#endif
#if defined(CORE_BARCELONA)
size >>= 8;
sgemm_p = 232 * size;
dgemm_p = 116 * size;
cgemm_p = 116 * size;
zgemm_p = 58 * size;
#ifdef EXPRECISION
qgemm_p = 58 * size;
xgemm_p = 26 * size;
#endif
#ifdef QUAD_PRECISION
qgemm_p = 26 * size;
xgemm_p = 13 * size;
#endif
#endif
p = getenv("GOTO_BLOCK_FACTOR");
if (p) {
factor = atoi(p);
if (factor < 10) factor = 10;
if (factor > 200) factor = 200;
sgemm_p = ((long)((double)sgemm_p * (double)factor * 1.e-2)) & ~7L;
dgemm_p = ((long)((double)dgemm_p * (double)factor * 1.e-2)) & ~7L;
cgemm_p = ((long)((double)cgemm_p * (double)factor * 1.e-2)) & ~7L;
zgemm_p = ((long)((double)zgemm_p * (double)factor * 1.e-2)) & ~7L;
#ifdef EXPRECISION
qgemm_p = ((long)((double)qgemm_p * (double)factor * 1.e-2)) & ~7L;
xgemm_p = ((long)((double)xgemm_p * (double)factor * 1.e-2)) & ~7L;
#endif
}
if (sgemm_p == 0) sgemm_p = 64;
if (dgemm_p == 0) dgemm_p = 64;
if (cgemm_p == 0) cgemm_p = 64;
if (zgemm_p == 0) zgemm_p = 64;
#ifdef EXPRECISION
if (qgemm_p == 0) qgemm_p = 64;
if (xgemm_p == 0) xgemm_p = 64;
#endif
#ifdef QUAD_PRECISION
if (qgemm_p == 0) qgemm_p = 64;
if (xgemm_p == 0) xgemm_p = 64;
#endif
sgemm_p = (sgemm_p + SGEMM_UNROLL_M - 1) & ~(SGEMM_UNROLL_M - 1);
dgemm_p = (dgemm_p + DGEMM_UNROLL_M - 1) & ~(DGEMM_UNROLL_M - 1);
cgemm_p = (cgemm_p + CGEMM_UNROLL_M - 1) & ~(CGEMM_UNROLL_M - 1);
zgemm_p = (zgemm_p + ZGEMM_UNROLL_M - 1) & ~(ZGEMM_UNROLL_M - 1);
#ifdef QUAD_PRECISION
qgemm_p = (qgemm_p + QGEMM_UNROLL_M - 1) & ~(QGEMM_UNROLL_M - 1);
xgemm_p = (xgemm_p + XGEMM_UNROLL_M - 1) & ~(XGEMM_UNROLL_M - 1);
#endif
sgemm_r = (((BUFFER_SIZE - ((SGEMM_P * SGEMM_Q * 4 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (SGEMM_Q * 4)) - 15) & ~15;
dgemm_r = (((BUFFER_SIZE - ((DGEMM_P * DGEMM_Q * 8 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (DGEMM_Q * 8)) - 15) & ~15;
cgemm_r = (((BUFFER_SIZE - ((CGEMM_P * CGEMM_Q * 8 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (CGEMM_Q * 8)) - 15) & ~15;
zgemm_r = (((BUFFER_SIZE - ((ZGEMM_P * ZGEMM_Q * 16 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (ZGEMM_Q * 16)) - 15) & ~15;
#if defined(EXPRECISION) || defined(QUAD_PRECISION)
qgemm_r = (((BUFFER_SIZE - ((QGEMM_P * QGEMM_Q * 16 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (QGEMM_Q * 16)) - 15) & ~15;
xgemm_r = (((BUFFER_SIZE - ((XGEMM_P * XGEMM_Q * 32 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (XGEMM_Q * 32)) - 15) & ~15;
#endif
#if 0
fprintf(stderr, "SGEMM ... %3d, %3d, %3d\n", SGEMM_P, SGEMM_Q, SGEMM_R);
fprintf(stderr, "DGEMM ... %3d, %3d, %3d\n", DGEMM_P, DGEMM_Q, DGEMM_R);
fprintf(stderr, "CGEMM ... %3d, %3d, %3d\n", CGEMM_P, CGEMM_Q, CGEMM_R);
fprintf(stderr, "ZGEMM ... %3d, %3d, %3d\n", ZGEMM_P, ZGEMM_Q, ZGEMM_R);
#endif
return;
}
#if 0
int get_current_cpu_info(void){
int nlprocs, ncores, cmplegacy;
int htt = 0;
int apicid = 0;
#if defined(CORE_PRESCOTT) || defined(CORE_OPTERON)
int eax, ebx, ecx, edx;
cpuid(1, &eax, &ebx, &ecx, &edx);
nlprocs = BITMASK(ebx, 16, 0xff);
apicid = BITMASK(ebx, 24, 0xff);
htt = BITMASK(edx, 28, 0x01);
#endif
#if defined(CORE_PRESCOTT)
cpuid(4, &eax, &ebx, &ecx, &edx);
ncores = BITMASK(eax, 26, 0x3f);
if (htt == 0) nlprocs = 0;
#endif
#if defined(CORE_OPTERON)
cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
ncores = BITMASK(ecx, 0, 0xff);
cpuid(0x80000001, &eax, &ebx, &ecx, &edx);
cmplegacy = BITMASK(ecx, 1, 0x01);
if (htt == 0) {
nlprocs = 0;
ncores = 0;
cmplegacy = 0;
}
#endif
ncores ++;
fprintf(stderr, "APICID = %d Number of core = %d\n", apicid, ncores);
return 0;
}
#endif
#endif
#if defined(ARCH_IA64)
static inline BLASULONG cpuid(BLASULONG regnum){
BLASULONG value;
#ifndef __ECC
asm ("mov %0=cpuid[%r1]" : "=r"(value) : "rO"(regnum));
#else
value = __getIndReg(_IA64_REG_INDR_CPUID, regnum);
#endif
return value;
}
#if 1
void blas_set_parameter(void){
BLASULONG cpuid3, size;
cpuid3 = cpuid(3);
size = BITMASK(cpuid3, 16, 0xff);
sgemm_p = 192 * (size + 1);
dgemm_p = 96 * (size + 1);
cgemm_p = 96 * (size + 1);
zgemm_p = 48 * (size + 1);
#ifdef EXPRECISION
qgemm_p = 64 * (size + 1);
xgemm_p = 32 * (size + 1);
#endif
#ifdef QUAD_PRECISION
qgemm_p = 32 * (size + 1);
xgemm_p = 16 * (size + 1);
#endif
sgemm_r = (((BUFFER_SIZE - ((SGEMM_P * SGEMM_Q * 4 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (SGEMM_Q * 4)) - 15) & ~15;
dgemm_r = (((BUFFER_SIZE - ((DGEMM_P * DGEMM_Q * 8 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (DGEMM_Q * 8)) - 15) & ~15;
cgemm_r = (((BUFFER_SIZE - ((CGEMM_P * CGEMM_Q * 8 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (CGEMM_Q * 8)) - 15) & ~15;
zgemm_r = (((BUFFER_SIZE - ((ZGEMM_P * ZGEMM_Q * 16 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (ZGEMM_Q * 16)) - 15) & ~15;
#if defined(EXPRECISION) || defined(QUAD_PRECISION)
qgemm_r = (((BUFFER_SIZE - ((QGEMM_P * QGEMM_Q * 16 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (QGEMM_Q * 16)) - 15) & ~15;
xgemm_r = (((BUFFER_SIZE - ((XGEMM_P * XGEMM_Q * 32 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (XGEMM_Q * 32)) - 15) & ~15;
#endif
return;
}
#else
#define IA64_SYS_NAME "/sys/devices/system/cpu/cpu0/cache/index3/size"
#define IA64_PROC_NAME "/proc/pal/cpu0/cache_info"
void blas_set_parameter(void){
BLASULONG cpuid3;
int size = 0;
#if 1
char buffer[128];
FILE *infile;
if ((infile = fopen(IA64_SYS_NAME, "r")) != NULL) {
fgets(buffer, sizeof(buffer), infile);
fclose(infile);
size = atoi(buffer) / 1536;
}
if (size <= 0) {
if ((infile = fopen(IA64_PROC_NAME, "r")) != NULL) {
while(fgets(buffer, sizeof(buffer), infile) != NULL) {
if ((!strncmp("Data/Instruction Cache level 3", buffer, 30))) break;
}
fgets(buffer, sizeof(buffer), infile);
fclose(infile);
*strstr(buffer, "bytes") = (char)NULL;
size = atoi(strchr(buffer, ':') + 1) / 1572864;
}
}
#endif
/* The last resort */
if (size <= 0) {
cpuid3 = cpuid(3);
size = BITMASK(cpuid3, 16, 0xff) + 1;
}
sgemm_p = 320 * size;
dgemm_p = 160 * size;
cgemm_p = 160 * size;
zgemm_p = 80 * size;
#ifdef EXPRECISION
qgemm_p = 80 * size;
xgemm_p = 40 * size;
#endif
sgemm_r = (((BUFFER_SIZE - ((SGEMM_P * SGEMM_Q * 4 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (SGEMM_Q * 4)) - 15) & ~15;
dgemm_r = (((BUFFER_SIZE - ((DGEMM_P * DGEMM_Q * 8 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (DGEMM_Q * 8)) - 15) & ~15;
cgemm_r = (((BUFFER_SIZE - ((CGEMM_P * CGEMM_Q * 8 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (CGEMM_Q * 8)) - 15) & ~15;
zgemm_r = (((BUFFER_SIZE - ((ZGEMM_P * ZGEMM_Q * 16 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (ZGEMM_Q * 16)) - 15) & ~15;
#ifdef EXPRECISION
qgemm_r = (((BUFFER_SIZE - ((QGEMM_P * QGEMM_Q * 16 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (QGEMM_Q * 16)) - 15) & ~15;
xgemm_r = (((BUFFER_SIZE - ((XGEMM_P * XGEMM_Q * 32 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (XGEMM_Q * 32)) - 15) & ~15;
#endif
return;
}
#endif
#endif
driver/others/profile.c
+139
View File
@@ -0,0 +1,139 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/
#include "common.h"
#include <signal.h>
#include <strings.h>
#define USE_FUNCTABLE
#include "../../interface/functable.h"
func_profile_t function_profile_table[MAX_PROF_TABLE];
int gotoblas_profile = 1;
static struct sigaction sa, ig;
void gotoblas_profile_quit(void) {
int i;
unsigned long long calls, fops, cycles, tcycles, area;
sigaction(SIGPROF, &ig, NULL);
calls = 0;
fops = 0;
cycles = 0;
tcycles = 0;
area = 0;
for (i = 0; i < MAX_PROF_TABLE; i ++) {
if (function_profile_table[i].calls) {
calls += function_profile_table[i].calls;
cycles += function_profile_table[i].cycles;
tcycles += function_profile_table[i].tcycles;
area += function_profile_table[i].area;
fops += function_profile_table[i].fops;
}
}
if (cycles > 0) {
fprintf(stderr, "\n\t====== BLAS Profiling Result =======\n\n");
fprintf(stderr, " Function No. of Calls Time Consumption Efficiency Bytes/cycle\n");
for (i = 0; i < MAX_PROF_TABLE; i ++) {
if (function_profile_table[i].calls) {
#ifndef OS_WINDOWS
fprintf(stderr, "%-12s : %10Ld %8.2f%% %10.3f%% %8.2f\n",
#else
fprintf(stderr, "%-12s : %10lld %8.2f%% %10.3f%% %8.2f\n",
#endif
func_table[i],
function_profile_table[i].calls,
(double)function_profile_table[i].cycles / (double)cycles * 100.,
(double)function_profile_table[i].fops / (double)function_profile_table[i].tcycles * 100.,
(double)function_profile_table[i].area / (double)function_profile_table[i].cycles
);
}
}
fprintf(stderr, " --------------------------------------------------------------------\n");
#ifndef OS_WINDOWS
fprintf(stderr, "%-12s : %10Ld %10.3f%% %8.2f\n",
#else
fprintf(stderr, "%-12s : %10lld %10.3f%% %8.2f\n",
#endif
"Total",
calls,
(double)fops / (double)tcycles * 100.,
(double)area / (double)cycles);
}
sigaction(SIGPROF, &sa, NULL);
}
void gotoblas_profile_clear(void) {
int i;
for (i = 0; i < MAX_PROF_TABLE; i ++) {
function_profile_table[i].calls = 0;
function_profile_table[i].cycles = 0;
function_profile_table[i].tcycles = 0;
function_profile_table[i].area = 0;
function_profile_table[i].fops = 0;
}
}
void gotoblas_profile_init(void) {
gotoblas_profile_clear();
bzero(&sa, sizeof(struct sigaction));
sa.sa_handler = (void *)gotoblas_profile_quit;
sa.sa_flags = SA_NODEFER | SA_RESETHAND;
bzero(&ig, sizeof(struct sigaction));
ig.sa_handler = SIG_IGN;
ig.sa_flags |= SA_NODEFER | SA_RESETHAND;
sigaction(SIGPROF, &sa, NULL);
}
driver/others/xerbla.c
+70
View File
@@ -0,0 +1,70 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/
#include <stdio.h>
#include <stdlib.h>
#include "common.h"
#if defined(OS_WINDOWS) && (defined(__MINGW32__) || defined(__MINGW64__))
#include <conio.h>
#undef printf
#define printf _cprintf
#endif
#ifdef __ELF__
int __xerbla(char *message, blasint *info, blasint length){
printf(" ** On entry to %6s parameter number %2d had an illegal value\n",
message, *info);
return 0;
}
int BLASFUNC(xerbla)(char *, blasint *, blasint) __attribute__ ((weak, alias ("__xerbla")));
#else
int BLASFUNC(xerbla)(char *message, blasint *info, blasint length){
printf(" ** On entry to %6s parameter number %2d had an illegal value\n",
message, *info);
return 0;
}
#endif