From dcfbbe8a58504258dc482ba2eb5717d8bad105ab Mon Sep 17 00:00:00 2001 From: localvar Date: Thu, 14 Nov 2019 08:20:07 +0000 Subject: [PATCH 1/6] implement memory leak detection --- CMakeLists.txt | 6 +- src/inc/tutil.h | 35 ++- src/os/linux/src/tsystem.c | 5 +- src/system/detail/src/dnodeService.c | 8 + src/util/src/tmem.c | 309 ++++++++++++++++++++++++--- 5 files changed, 330 insertions(+), 33 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c89195a89a..47598d7cc8 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -125,7 +125,11 @@ IF (NOT DEFINED TD_CLUSTER) # debug flag # # ADD_DEFINITIONS(-D_CHECK_HEADER_FILE_) - # ADD_DEFINITIONS(-D_TAOS_MEM_TEST_) + + # TAOS_MEM_CHECK + # 1 to test memory allocation failure + # 2 to check memory leak + ADD_DEFINITIONS(-DTAOS_MEM_CHECK=2) IF (TD_CLUSTER) ADD_DEFINITIONS(-DCLUSTER) diff --git a/src/inc/tutil.h b/src/inc/tutil.h index 7725bd2d1b..4066e5a756 100644 --- a/src/inc/tutil.h +++ b/src/inc/tutil.h @@ -187,18 +187,43 @@ static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, unsigned int inLen, cha char *taosIpStr(uint32_t ipInt); -#ifdef _TAOS_MEM_TEST_ +extern void taos_dump_memory_leak_at_exit(const char* path); + +#if TAOS_MEM_CHECK == 1 + // Use during test to simulate the success and failure scenarios of memory allocation -extern void* taos_malloc(unsigned int size, char* _func); -extern void* taos_calloc(unsigned int num, unsigned int size, char* _func); -extern void* taos_realloc(void* ptr, unsigned int size, char* _func); +extern void* taos_malloc(size_t size, const char* func); +extern void* taos_calloc(size_t num, size_t size, const char* func); +extern void* taos_realloc(void* ptr, size_t size, const char* func); extern void taos_free(void* ptr); +extern char* taos_strdup(const char* str, const char* func); +extern char* taos_strndup(const char* str, size_t size, const char* func); #define malloc(size) taos_malloc(size, __FUNCTION__) #define calloc(num, size) taos_calloc(num, size, __FUNCTION__) #define realloc(ptr, size) taos_realloc(ptr, size, __FUNCTION__) #define free(ptr) taos_free(ptr) -#endif +#define strdup(str) taos_strdup(str, __FUNCTION__) +#define strndup(str, size) taos_strndup(str, size, __FUNCTION__) +#elif TAOS_MEM_CHECK == 2 + +extern void* taos_malloc(size_t size, const char* file, uint32_t line); +extern void* taos_calloc(size_t num, size_t size, const char* file, uint32_t line); +extern void* taos_realloc(void* ptr, size_t size, const char* file, uint32_t line); +extern void taos_free(void* ptr, const char* file, uint32_t line); +extern char* taos_strdup(const char* str, const char* file, uint32_t line); +extern char* taos_strndup(const char* str, size_t size, const char* file, uint32_t line); +extern ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line); + +#define malloc(size) taos_malloc(size, __FILE__, __LINE__) +#define calloc(num, size) taos_calloc(num, size, __FILE__, __LINE__) +#define realloc(ptr, size) taos_realloc(ptr, size, __FILE__, __LINE__) +#define free(ptr) taos_free(ptr, __FILE__, __LINE__) +#define strdup(str) taos_strdup(str, __FILE__, __LINE__) +#define strndup(str, size) taos_strndup(str, size, __FILE__, __LINE__) +#define getline(lineptr, n, stream) taos_getline(lineptr, n, stream, __FILE__, __LINE__) + +#endif #ifdef __cplusplus } diff --git a/src/os/linux/src/tsystem.c b/src/os/linux/src/tsystem.c index fa41e8e6d0..cae96db2cb 100644 --- a/src/os/linux/src/tsystem.c +++ b/src/os/linux/src/tsystem.c @@ -81,6 +81,7 @@ bool taosGetProcMemory(float *memoryUsedMB) { char * line = NULL; while (!feof(fp)) { tfree(line); + len = 0; getline(&line, &len, fp); if (line == NULL) { break; @@ -137,7 +138,7 @@ bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { return false; } - size_t len; + size_t len = 0; char * line = NULL; getline(&line, &len, fp); if (line == NULL) { @@ -409,6 +410,7 @@ bool taosGetCardInfo(int64_t *bytes) { while (!feof(fp)) { tfree(line); + len = 0; getline(&line, &len, fp); if (line == NULL) { break; @@ -480,6 +482,7 @@ bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) { while (!feof(fp)) { tfree(line); + len = 0; getline(&line, &len, fp); if (line == NULL) { break; diff --git a/src/system/detail/src/dnodeService.c b/src/system/detail/src/dnodeService.c index a14ec1fda6..55d5873695 100644 --- a/src/system/detail/src/dnodeService.c +++ b/src/system/detail/src/dnodeService.c @@ -61,6 +61,14 @@ int main(int argc, char *argv[]) { return 0; } else if (strcmp(argv[i], "-k") == 0) { dnodeParseParameterK(); +#if TAOS_MEM_CHECK == 2 + } else if (strcmp(argv[i], "--check-mem-leak") == 0) { + if ((i < argc - 1) && (argv[i+1][0] != '-')) { + taos_dump_memory_leak_at_exit(argv[++i]); + } else { + taos_dump_memory_leak_at_exit(NULL); + } +#endif } } diff --git a/src/util/src/tmem.c b/src/util/src/tmem.c index 462da884b9..3ca6cedf29 100644 --- a/src/util/src/tmem.c +++ b/src/util/src/tmem.c @@ -15,12 +15,15 @@ #include "os.h" #include "tlog.h" +#include "os.h" + +#if TAOS_MEM_CHECK == 1 extern int32_t taosGetTimestampSec(); -static int32_t startTime = 0; -static int64_t m_curLimit = 100*1024; +static int32_t startTime = 0; +static int64_t m_curLimit = 100 * 1024; -bool isMallocMem(unsigned int size, char* _func) { +static bool isMallocMem(size_t size, const char* func) { if (0 == startTime) { startTime = taosGetTimestampSec(); return true; @@ -29,9 +32,9 @@ bool isMallocMem(unsigned int size, char* _func) { if (currentTime - startTime < 10) return true; } - if (size > m_curLimit) { + if (size > m_curLimit) { if (3 == rand() % 20) { - pTrace("====no alloc mem in func: %s, size:%d", _func, size); + pTrace("====no alloc mem in func: %s, size:%d", func, size); return false; } } @@ -39,40 +42,294 @@ bool isMallocMem(unsigned int size, char* _func) { return true; } -void* taos_malloc(unsigned int size, char* _func) { - - if (false == isMallocMem(size, _func)) { +void* taos_malloc(size_t size, const char* func) { + if (false == isMallocMem(size, func)) { return NULL; } - - void *p = NULL; - p = malloc(size); + return malloc(size); +} + +void* taos_calloc(size_t num, size_t size, const char* func) { + if (false == isMallocMem(size, func)) { + return NULL; + } + return calloc(num, size); +} + +void* taos_realloc(void* ptr, size_t size, const char* func) { + if (false == isMallocMem(size, func)) { + return NULL; + } + return realloc(ptr, size); +} + +void taos_free(void* ptr) { free(ptr); } + +char* taos_strdup(const char* str, const char* func) { + size_t len = strlen(str); + return isMallocMem(len + 1, func) ? strdup(str) : NULL; +} + +char* taos_strndup(const char* str, size_t size, const char* func) { + size_t len = strlen(str); + if (len > size) { + len = size; + } + return isMallocMem(len + 1, func) ? strndup(str, len) : NULL; +} + +#elif TAOS_MEM_CHECK == 2 + +#define MAGIC 0x55AA + +typedef struct SMemBlock { + const char* file; + uint16_t line; + uint16_t magic; + uint32_t size; + struct SMemBlock* prev; + struct SMemBlock* next; + // TODO: need pading in 32bit platform + char data[0]; +} SMemBlock; + +static SMemBlock *blocks = NULL; +static uintptr_t lock = 0; +static FILE* fpMemLeak = NULL; + +static void add_mem_block(SMemBlock* blk) { + blk->prev = NULL; + while (atomic_val_compare_exchange_ptr(&lock, 0, 1) != 0); + blk->next = blocks; + if (blocks != NULL) { + blocks->prev = blk; + } + blocks = blk; + atomic_store_ptr(&lock, 0); +} + +static void remove_mem_block(SMemBlock* blk) { + while (atomic_val_compare_exchange_ptr(&lock, 0, 1) != 0); + + if (blocks == blk) { + blocks = blk->next; + } + if (blk->prev != NULL) { + blk->prev->next = blk->next; + } + if (blk->next != NULL) { + blk->next->prev = blk->prev; + } + + atomic_store_ptr(&lock, 0); + + blk->prev = NULL; + blk->next = NULL; +} + +void* taos_malloc(size_t size, const char* file, uint32_t line) { + if (size == 0) { + return NULL; + } + + SMemBlock *blk = (SMemBlock*)malloc(size + sizeof(SMemBlock)); + if (blk == NULL) { + return NULL; + } + + if (line > UINT16_MAX && fpMemLeak != NULL) { + fprintf(fpMemLeak, "%s:%d: line number too large.\n", file, line); + } + + if (size > UINT32_MAX && fpMemLeak != NULL) { + fprintf(fpMemLeak, "%s:%d: size too large: %lld.\n", file, line, size); + } + + blk->file = file; + blk->line = (uint16_t)line; + blk->magic = MAGIC; + blk->size = size; + add_mem_block(blk); + + return blk->data; +} + +void* taos_calloc(size_t num, size_t size, const char* file, uint32_t line) { + size *= num; + void* p = taos_malloc(size, file, line); + if (p != NULL) { + memset(p, 0, size); + } return p; } -void* taos_calloc(unsigned int num, unsigned int size, char* _func) { - - if (false == isMallocMem(size, _func)) { +void* taos_realloc(void* ptr, size_t size, const char* file, uint32_t line) { + if (size == 0) { + taos_free(ptr, file, line); return NULL; } - - void *p = NULL; - p = calloc(num, size); + + if (ptr == NULL) { + return taos_malloc(size, file, line); + } + + SMemBlock* blk = ((char*)ptr) - sizeof(SMemBlock); + if (blk->magic != MAGIC) { + if (fpMemLeak != NULL) { + fprintf(fpMemLeak, "%s:%d: memory not allocated by 'taos_malloc'.\n", file, line); + } + return realloc(ptr, size); + } + + remove_mem_block(blk); + + void* p = realloc(blk, size + sizeof(SMemBlock)); + if (p == NULL) { + add_mem_block(blk); + return NULL; + } + + if (size > UINT32_MAX && fpMemLeak != NULL) { + fprintf(fpMemLeak, "%s:%d: size too large: %lld.\n", file, line, size); + } + + blk = (SMemBlock*)p; + blk->size = size; + add_mem_block(blk); + return blk->data; +} + +void taos_free(void* ptr, const char* file, uint32_t line) { + if (ptr == NULL) { + return; + } + + SMemBlock* blk = ((char*)ptr) - sizeof(SMemBlock); + if (blk->magic != MAGIC) { + if (fpMemLeak != NULL) { + fprintf(fpMemLeak, "%s:%d: memory not allocated by 'taos_malloc'.\n", file, line); + } + free(ptr); + return; + } + + remove_mem_block(blk); + free(blk); +} + +char* taos_strdup(const char* str, const char* file, uint32_t line) { + size_t len = strlen(str); + char *p = taos_malloc(len + 1, file, line); + if (p != NULL) { + memcpy(p, str, len); + p[len] = 0; + } return p; } -void* taos_realloc(void* ptr, unsigned int size, char* _func) { - - if (false == isMallocMem(size, _func)) { - return NULL; +char* taos_strndup(const char* str, size_t size, const char* file, uint32_t line) { + size_t len = strlen(str); + if (len > size) { + len = size; + } + char *p = taos_malloc(len + 1, file, line); + if (p != NULL) { + memcpy(p, str, len); + p[len] = 0; } - - void *p = NULL; - p = realloc(ptr, size); return p; } -void taos_free(void* ptr) { - free(ptr); +ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { + char* buf = NULL; + size_t bufSize = 0; + ssize_t size = getline(&buf, &bufSize, stream); + if (size != -1) { + if (*n < size + 1) { + void* p = taos_realloc(*lineptr, size + 1, file, line); + if (p == NULL) { + free(buf); + return -1; + } + *lineptr = (char*)p; + *n = size + 1; + } + memcpy(*lineptr, buf, size + 1); + } + + free(buf); + return size; } +static void dump_memory_leak() { + const char* hex = "0123456789ABCDEF"; + const char* fmt = ":%d: addr=0x%p, size=%d, content(first 16 bytes)='"; + size_t numOfBlk = 0, totalSize = 0; + + fputs("memory blocks allocated but not freed before exit:\n\n", fpMemLeak); + + while (atomic_val_compare_exchange_ptr(&lock, 0, 1) != 0); + + for (SMemBlock* blk = blocks; blk != NULL; blk = blk->next) { + ++numOfBlk; + totalSize += blk->size; + + fputs(blk->file, fpMemLeak); + fprintf(fpMemLeak, fmt, blk->line, blk->data, blk->size); + + uint8_t c = (uint8_t)(blk->data[0]); + fputc(hex[c >> 4], fpMemLeak); + fputc(hex[c & 0x0f], fpMemLeak); + + size_t size = blk->size > 16 ? 16 : blk->size; + for (size_t i = 1; i < size; ++i) { + c = (uint8_t)(blk->data[i]); + fputc(' ', fpMemLeak); + fputc(hex[c >> 4], fpMemLeak); + fputc(hex[c & 0x0f], fpMemLeak); + } + + fputs("'\n", fpMemLeak); + } + + atomic_store_ptr(&lock, 0); + + fprintf("\nnumber of blocks: %lld, total bytes: %lld\n", numOfBlk, totalSize); + if (fpMemLeak != stdout) { + fclose(fpMemLeak); + fpMemLeak = NULL; + } +} + +static void dump_memory_leak_at_sig(int sig) { + fprintf(fpMemLeak, "signal %d received, exiting...\n", sig); + dump_memory_leak(); + struct sigaction act = {0}; + act.sa_handler = SIG_DFL; + sigaction(sig, &act, NULL); +} + +void taos_dump_memory_leak_at_exit(const char* path) { + if (path == NULL || path[0] == 0) { + fpMemLeak = stdout; + } else if ((fpMemLeak = fopen(path, "w")) == NULL) { + printf("failed to open memory leak dump file '%s', errno=%d\n", path, errno); + return; + } + + atexit(dump_memory_leak); + + struct sigaction act = {0}; + act.sa_handler = dump_memory_leak_at_sig; + sigaction(SIGFPE, &act, NULL); + sigaction(SIGSEGV, &act, NULL); + sigaction(SIGILL, &act, NULL); +} + +#endif + +#if TAOS_MEM_CHECK != 2 +void taos_dump_memory_leak_at_exit(const char* path) { + printf("memory leak detection not enabled!") +} +#endif \ No newline at end of file From a5bfb0d670e109fd1514ff117e1d598a068b05d5 Mon Sep 17 00:00:00 2001 From: localvar Date: Thu, 14 Nov 2019 08:46:53 +0000 Subject: [PATCH 2/6] update jni for memory leak detection --- src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h | 8 ++++++++ src/client/src/TSDBJNIConnector.c | 12 ++++++++++++ src/util/src/tmem.c | 5 +++++ 3 files changed, 25 insertions(+) diff --git a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h index 9f9632cadc..d99fca5ce1 100644 --- a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h +++ b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h @@ -9,6 +9,14 @@ extern "C" { #endif #undef com_taosdata_jdbc_TSDBJNIConnector_INVALID_CONNECTION_POINTER_VALUE #define com_taosdata_jdbc_TSDBJNIConnector_INVALID_CONNECTION_POINTER_VALUE 0LL +/* + * Class: com_taosdata_jdbc_TSDBJNIConnector + * Method: + * Signature: (Ljava/lang/String;)V + */ +JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_detectMemoryLeakImp + (JNIEnv *, jclass, jstring); + /* * Class: com_taosdata_jdbc_TSDBJNIConnector * Method: initImp diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index d958679544..4123c2aa12 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -111,6 +111,18 @@ void jniGetGlobalMethod(JNIEnv *env) { jniTrace("native method register finished"); } +JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_detectMemoryLeakImp(JNIEnv *env, jobject jobj, jstring jPath) { + if (jPath != NULL) { + const char *path = (*env)->GetStringUTFChars(env, jPath, NULL); + taos_dump_memory_leak_at_exit(path); + (*env)->ReleaseStringUTFChars(env, jPath, path); + } else { + taos_dump_memory_leak_at_exit(NULL); + } + + jniGetGlobalMethod(env); +} + JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_initImp(JNIEnv *env, jobject jobj, jstring jconfigDir) { if (jconfigDir != NULL) { const char *confDir = (*env)->GetStringUTFChars(env, jconfigDir, NULL); diff --git a/src/util/src/tmem.c b/src/util/src/tmem.c index 3ca6cedf29..81dff55049 100644 --- a/src/util/src/tmem.c +++ b/src/util/src/tmem.c @@ -310,6 +310,11 @@ static void dump_memory_leak_at_sig(int sig) { } void taos_dump_memory_leak_at_exit(const char* path) { + if (fpMemLeak != NULL) { + printf("memory leak detection already enabled.\n"); + return; + } + if (path == NULL || path[0] == 0) { fpMemLeak = stdout; } else if ((fpMemLeak = fopen(path, "w")) == NULL) { From 32abecfbc09269344f09fddc14036d90ef721d48 Mon Sep 17 00:00:00 2001 From: localvar Date: Thu, 14 Nov 2019 09:51:45 +0000 Subject: [PATCH 3/6] make 'taos_dump_memory_leak' public for java --- .../jni/com_taosdata_jdbc_TSDBJNIConnector.h | 8 ++++++++ src/client/src/TSDBJNIConnector.c | 9 +++++++-- src/inc/tutil.h | 3 ++- src/system/detail/src/dnodeService.c | 4 ++-- src/util/src/tmem.c | 20 +++++++++++++------ 5 files changed, 33 insertions(+), 11 deletions(-) diff --git a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h index d99fca5ce1..d11a030188 100644 --- a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h +++ b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h @@ -17,6 +17,14 @@ extern "C" { JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_detectMemoryLeakImp (JNIEnv *, jclass, jstring); +/* + * Class: com_taosdata_jdbc_TSDBJNIConnector + * Method: + * Signature: ()Ljava/lang/String; + */ +JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_dumpMemoryLeakImp + (JNIEnv *, jclass); + /* * Class: com_taosdata_jdbc_TSDBJNIConnector * Method: initImp diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index 4123c2aa12..57f312f573 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -114,15 +114,20 @@ void jniGetGlobalMethod(JNIEnv *env) { JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_detectMemoryLeakImp(JNIEnv *env, jobject jobj, jstring jPath) { if (jPath != NULL) { const char *path = (*env)->GetStringUTFChars(env, jPath, NULL); - taos_dump_memory_leak_at_exit(path); + taos_detect_memory_leak(path); (*env)->ReleaseStringUTFChars(env, jPath, path); } else { - taos_dump_memory_leak_at_exit(NULL); + taos_detect_memory_leak(NULL); } jniGetGlobalMethod(env); } +JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_dumpMemoryLeakImp(JNIEnv *env, jobject jobj) { + taos_dump_memory_leak(); + jniGetGlobalMethod(env); +} + JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_initImp(JNIEnv *env, jobject jobj, jstring jconfigDir) { if (jconfigDir != NULL) { const char *confDir = (*env)->GetStringUTFChars(env, jconfigDir, NULL); diff --git a/src/inc/tutil.h b/src/inc/tutil.h index 4066e5a756..0d454faf23 100644 --- a/src/inc/tutil.h +++ b/src/inc/tutil.h @@ -187,7 +187,8 @@ static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, unsigned int inLen, cha char *taosIpStr(uint32_t ipInt); -extern void taos_dump_memory_leak_at_exit(const char* path); +extern void taos_detect_memory_leak(const char* path); +extern void taos_dump_memory_leak(); #if TAOS_MEM_CHECK == 1 diff --git a/src/system/detail/src/dnodeService.c b/src/system/detail/src/dnodeService.c index 55d5873695..1d0c6f4893 100644 --- a/src/system/detail/src/dnodeService.c +++ b/src/system/detail/src/dnodeService.c @@ -64,9 +64,9 @@ int main(int argc, char *argv[]) { #if TAOS_MEM_CHECK == 2 } else if (strcmp(argv[i], "--check-mem-leak") == 0) { if ((i < argc - 1) && (argv[i+1][0] != '-')) { - taos_dump_memory_leak_at_exit(argv[++i]); + taos_detect_memory_leak(argv[++i]); } else { - taos_dump_memory_leak_at_exit(NULL); + taos_detect_memory_leak(NULL); } #endif } diff --git a/src/util/src/tmem.c b/src/util/src/tmem.c index 81dff55049..87a43a697d 100644 --- a/src/util/src/tmem.c +++ b/src/util/src/tmem.c @@ -261,11 +261,15 @@ ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char* file, return size; } -static void dump_memory_leak() { +void taos_dump_memory_leak() { const char* hex = "0123456789ABCDEF"; const char* fmt = ":%d: addr=0x%p, size=%d, content(first 16 bytes)='"; size_t numOfBlk = 0, totalSize = 0; + if (fpMemLeak == NULL) { + return; + } + fputs("memory blocks allocated but not freed before exit:\n\n", fpMemLeak); while (atomic_val_compare_exchange_ptr(&lock, 0, 1) != 0); @@ -303,13 +307,13 @@ static void dump_memory_leak() { static void dump_memory_leak_at_sig(int sig) { fprintf(fpMemLeak, "signal %d received, exiting...\n", sig); - dump_memory_leak(); + taos_dump_memory_leak(); struct sigaction act = {0}; act.sa_handler = SIG_DFL; sigaction(sig, &act, NULL); } -void taos_dump_memory_leak_at_exit(const char* path) { +void taos_detect_memory_leak(const char* path) { if (fpMemLeak != NULL) { printf("memory leak detection already enabled.\n"); return; @@ -322,7 +326,7 @@ void taos_dump_memory_leak_at_exit(const char* path) { return; } - atexit(dump_memory_leak); + atexit(taos_dump_memory_leak); struct sigaction act = {0}; act.sa_handler = dump_memory_leak_at_sig; @@ -334,7 +338,11 @@ void taos_dump_memory_leak_at_exit(const char* path) { #endif #if TAOS_MEM_CHECK != 2 -void taos_dump_memory_leak_at_exit(const char* path) { - printf("memory leak detection not enabled!") +void taos_dump_memory_leak() { + // do nothing +} + +void taos_detect_memory_leak(const char* path) { + printf("memory leak detection not enabled, please set 'TAOS_MEM_CHECK' to 2."); } #endif \ No newline at end of file From fc6737ab04c4a6828f3bf60f86a1b0dd45efe6a6 Mon Sep 17 00:00:00 2001 From: localvar Date: Thu, 14 Nov 2019 10:45:00 +0000 Subject: [PATCH 4/6] add option 'autoDump' for memory leak detection. --- .../jni/com_taosdata_jdbc_TSDBJNIConnector.h | 2 +- src/client/src/TSDBJNIConnector.c | 6 ++--- src/inc/tutil.h | 2 +- src/system/detail/src/dnodeService.c | 4 ++-- src/util/src/tmem.c | 23 +++++++++++-------- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h index d11a030188..2b99dde7af 100644 --- a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h +++ b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h @@ -15,7 +15,7 @@ extern "C" { * Signature: (Ljava/lang/String;)V */ JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_detectMemoryLeakImp - (JNIEnv *, jclass, jstring); + (JNIEnv *, jclass, jstring, jboolean); /* * Class: com_taosdata_jdbc_TSDBJNIConnector diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index 57f312f573..b76aaaf640 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -111,13 +111,13 @@ void jniGetGlobalMethod(JNIEnv *env) { jniTrace("native method register finished"); } -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_detectMemoryLeakImp(JNIEnv *env, jobject jobj, jstring jPath) { +JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_detectMemoryLeakImp(JNIEnv *env, jobject jobj, jstring jPath, jboolean jAutoDump) { if (jPath != NULL) { const char *path = (*env)->GetStringUTFChars(env, jPath, NULL); - taos_detect_memory_leak(path); + taos_detect_memory_leak(path, !!jAutoDump); (*env)->ReleaseStringUTFChars(env, jPath, path); } else { - taos_detect_memory_leak(NULL); + taos_detect_memory_leak(NULL, !!jAutoDump); } jniGetGlobalMethod(env); diff --git a/src/inc/tutil.h b/src/inc/tutil.h index 0d454faf23..383a62be2f 100644 --- a/src/inc/tutil.h +++ b/src/inc/tutil.h @@ -187,7 +187,7 @@ static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, unsigned int inLen, cha char *taosIpStr(uint32_t ipInt); -extern void taos_detect_memory_leak(const char* path); +extern void taos_detect_memory_leak(const char* path, bool autoDump); extern void taos_dump_memory_leak(); #if TAOS_MEM_CHECK == 1 diff --git a/src/system/detail/src/dnodeService.c b/src/system/detail/src/dnodeService.c index 1d0c6f4893..2d81d49740 100644 --- a/src/system/detail/src/dnodeService.c +++ b/src/system/detail/src/dnodeService.c @@ -64,9 +64,9 @@ int main(int argc, char *argv[]) { #if TAOS_MEM_CHECK == 2 } else if (strcmp(argv[i], "--check-mem-leak") == 0) { if ((i < argc - 1) && (argv[i+1][0] != '-')) { - taos_detect_memory_leak(argv[++i]); + taos_detect_memory_leak(argv[++i], true); } else { - taos_detect_memory_leak(NULL); + taos_detect_memory_leak(NULL, true); } #endif } diff --git a/src/util/src/tmem.c b/src/util/src/tmem.c index 87a43a697d..b97d099644 100644 --- a/src/util/src/tmem.c +++ b/src/util/src/tmem.c @@ -307,13 +307,15 @@ void taos_dump_memory_leak() { static void dump_memory_leak_at_sig(int sig) { fprintf(fpMemLeak, "signal %d received, exiting...\n", sig); - taos_dump_memory_leak(); + struct sigaction act = {0}; act.sa_handler = SIG_DFL; sigaction(sig, &act, NULL); + + taos_dump_memory_leak(); } -void taos_detect_memory_leak(const char* path) { +void taos_detect_memory_leak(const char* path, bool autoDump) { if (fpMemLeak != NULL) { printf("memory leak detection already enabled.\n"); return; @@ -326,13 +328,14 @@ void taos_detect_memory_leak(const char* path) { return; } - atexit(taos_dump_memory_leak); - - struct sigaction act = {0}; - act.sa_handler = dump_memory_leak_at_sig; - sigaction(SIGFPE, &act, NULL); - sigaction(SIGSEGV, &act, NULL); - sigaction(SIGILL, &act, NULL); + if (autoDump) { + atexit(taos_dump_memory_leak); + struct sigaction act = {0}; + act.sa_handler = dump_memory_leak_at_sig; + sigaction(SIGFPE, &act, NULL); + sigaction(SIGSEGV, &act, NULL); + sigaction(SIGILL, &act, NULL); + } } #endif @@ -342,7 +345,7 @@ void taos_dump_memory_leak() { // do nothing } -void taos_detect_memory_leak(const char* path) { +void taos_detect_memory_leak(const char* path, bool autoDump) { printf("memory leak detection not enabled, please set 'TAOS_MEM_CHECK' to 2."); } #endif \ No newline at end of file From e7bdf0c25e334a52d9808a756ff7054150b867f6 Mon Sep 17 00:00:00 2001 From: localvar Date: Wed, 20 Nov 2019 01:49:02 +0000 Subject: [PATCH 5/6] support other alloc mode --- CMakeLists.txt | 8 +- .../jni/com_taosdata_jdbc_TSDBJNIConnector.h | 4 +- src/client/src/TSDBJNIConnector.c | 11 +- src/inc/tutil.h | 55 ++- src/system/detail/src/dnodeService.c | 14 +- src/util/src/tmem.c | 364 ++++++++++++------ 6 files changed, 287 insertions(+), 169 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 47598d7cc8..a1b2c16f46 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -125,11 +125,9 @@ IF (NOT DEFINED TD_CLUSTER) # debug flag # # ADD_DEFINITIONS(-D_CHECK_HEADER_FILE_) - - # TAOS_MEM_CHECK - # 1 to test memory allocation failure - # 2 to check memory leak - ADD_DEFINITIONS(-DTAOS_MEM_CHECK=2) + IF (${MEM_CHECK} MATCHES "true") + ADD_DEFINITIONS(-DTAOS_MEM_CHECK) + ENDIF () IF (TD_CLUSTER) ADD_DEFINITIONS(-DCLUSTER) diff --git a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h index 2b99dde7af..958252b4de 100644 --- a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h +++ b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h @@ -14,8 +14,8 @@ extern "C" { * Method: * Signature: (Ljava/lang/String;)V */ -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_detectMemoryLeakImp - (JNIEnv *, jclass, jstring, jboolean); +JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setAllocModeImp + (JNIEnv *, jclass, jint, jstring, jboolean); /* * Class: com_taosdata_jdbc_TSDBJNIConnector diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index b76aaaf640..71f983dadb 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -111,21 +111,18 @@ void jniGetGlobalMethod(JNIEnv *env) { jniTrace("native method register finished"); } -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_detectMemoryLeakImp(JNIEnv *env, jobject jobj, jstring jPath, jboolean jAutoDump) { +JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setAllocModeImp(JNIEnv *env, jobject jobj, jint jMode, jstring jPath, jboolean jAutoDump) { if (jPath != NULL) { const char *path = (*env)->GetStringUTFChars(env, jPath, NULL); - taos_detect_memory_leak(path, !!jAutoDump); + taosSetAllocMode(jMode, path, !!jAutoDump); (*env)->ReleaseStringUTFChars(env, jPath, path); } else { - taos_detect_memory_leak(NULL, !!jAutoDump); + taosSetAllocMode(jMode, NULL, !!jAutoDump); } - - jniGetGlobalMethod(env); } JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_dumpMemoryLeakImp(JNIEnv *env, jobject jobj) { - taos_dump_memory_leak(); - jniGetGlobalMethod(env); + taosDumpMemoryLeak(); } JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_initImp(JNIEnv *env, jobject jobj, jstring jconfigDir) { diff --git a/src/inc/tutil.h b/src/inc/tutil.h index 383a62be2f..683927c816 100644 --- a/src/inc/tutil.h +++ b/src/inc/tutil.h @@ -187,44 +187,35 @@ static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, unsigned int inLen, cha char *taosIpStr(uint32_t ipInt); -extern void taos_detect_memory_leak(const char* path, bool autoDump); -extern void taos_dump_memory_leak(); +#define TAOS_ALLOC_MODE_DEFAULT 0 +#define TAOS_ALLOC_MODE_RANDOM_FAIL 1 +#define TAOS_ALLOC_MODE_DETECT_LEAK 2 +void taosSetAllocMode(int mode, const char* path, bool autoDump); +void taosDumpMemoryLeak(); -#if TAOS_MEM_CHECK == 1 +#ifdef TAOS_MEM_CHECK -// Use during test to simulate the success and failure scenarios of memory allocation -extern void* taos_malloc(size_t size, const char* func); -extern void* taos_calloc(size_t num, size_t size, const char* func); -extern void* taos_realloc(void* ptr, size_t size, const char* func); -extern void taos_free(void* ptr); -extern char* taos_strdup(const char* str, const char* func); -extern char* taos_strndup(const char* str, size_t size, const char* func); -#define malloc(size) taos_malloc(size, __FUNCTION__) -#define calloc(num, size) taos_calloc(num, size, __FUNCTION__) -#define realloc(ptr, size) taos_realloc(ptr, size, __FUNCTION__) -#define free(ptr) taos_free(ptr) -#define strdup(str) taos_strdup(str, __FUNCTION__) -#define strndup(str, size) taos_strndup(str, size, __FUNCTION__) +void * taos_malloc(size_t size, const char *file, uint32_t line); +void * taos_calloc(size_t num, size_t size, const char *file, uint32_t line); +void * taos_realloc(void *ptr, size_t size, const char *file, uint32_t line); +void taos_free(void *ptr, const char *file, uint32_t line); +char * taos_strdup(const char *str, const char *file, uint32_t line); +char * taos_strndup(const char *str, size_t size, const char *file, uint32_t line); +ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char *file, uint32_t line); -#elif TAOS_MEM_CHECK == 2 +#ifndef TAOS_MEM_CHECK_IMPL -extern void* taos_malloc(size_t size, const char* file, uint32_t line); -extern void* taos_calloc(size_t num, size_t size, const char* file, uint32_t line); -extern void* taos_realloc(void* ptr, size_t size, const char* file, uint32_t line); -extern void taos_free(void* ptr, const char* file, uint32_t line); -extern char* taos_strdup(const char* str, const char* file, uint32_t line); -extern char* taos_strndup(const char* str, size_t size, const char* file, uint32_t line); -extern ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line); - -#define malloc(size) taos_malloc(size, __FILE__, __LINE__) -#define calloc(num, size) taos_calloc(num, size, __FILE__, __LINE__) -#define realloc(ptr, size) taos_realloc(ptr, size, __FILE__, __LINE__) -#define free(ptr) taos_free(ptr, __FILE__, __LINE__) -#define strdup(str) taos_strdup(str, __FILE__, __LINE__) -#define strndup(str, size) taos_strndup(str, size, __FILE__, __LINE__) +#define malloc(size) taos_malloc(size, __FILE__, __LINE__) +#define calloc(num, size) taos_calloc(num, size, __FILE__, __LINE__) +#define realloc(ptr, size) taos_realloc(ptr, size, __FILE__, __LINE__) +#define free(ptr) taos_free(ptr, __FILE__, __LINE__) +#define strdup(str) taos_strdup(str, __FILE__, __LINE__) +#define strndup(str, size) taos_strndup(str, size, __FILE__, __LINE__) #define getline(lineptr, n, stream) taos_getline(lineptr, n, stream, __FILE__, __LINE__) -#endif +#endif // TAOS_MEM_CHECK_IMPL + +#endif // TAOS_MEM_CHECK #ifdef __cplusplus } diff --git a/src/system/detail/src/dnodeService.c b/src/system/detail/src/dnodeService.c index 2d81d49740..9764afc593 100644 --- a/src/system/detail/src/dnodeService.c +++ b/src/system/detail/src/dnodeService.c @@ -61,12 +61,18 @@ int main(int argc, char *argv[]) { return 0; } else if (strcmp(argv[i], "-k") == 0) { dnodeParseParameterK(); -#if TAOS_MEM_CHECK == 2 - } else if (strcmp(argv[i], "--check-mem-leak") == 0) { +#ifdef TAOS_MEM_CHECK + } else if (strcmp(argv[i], "--alloc-random-fail") == 0) { if ((i < argc - 1) && (argv[i+1][0] != '-')) { - taos_detect_memory_leak(argv[++i], true); + taosSetAllocMode(TAOS_ALLOC_MODE_RANDOM_FAIL, argv[++i], true); } else { - taos_detect_memory_leak(NULL, true); + taosSetAllocMode(TAOS_ALLOC_MODE_RANDOM_FAIL, NULL, true); + } + } else if (strcmp(argv[i], "--detect-mem-leak") == 0) { + if ((i < argc - 1) && (argv[i+1][0] != '-')) { + taosSetAllocMode(TAOS_ALLOC_MODE_DETECT_LEAK, argv[++i], true); + } else { + taosSetAllocMode(TAOS_ALLOC_MODE_DETECT_LEAK, NULL, true); } #endif } diff --git a/src/util/src/tmem.c b/src/util/src/tmem.c index b97d099644..5a42933ebe 100644 --- a/src/util/src/tmem.c +++ b/src/util/src/tmem.c @@ -15,72 +15,75 @@ #include "os.h" #include "tlog.h" -#include "os.h" -#if TAOS_MEM_CHECK == 1 +#define TAOS_MEM_CHECK_IMPL +#include "tutil.h" + + +#ifdef TAOS_MEM_CHECK + +static int allocMode = TAOS_ALLOC_MODE_DEFAULT; +static FILE* fpAllocLog = NULL; + +//////////////////////////////////////////////////////////////////////////////// +// memory allocator which fails randomly extern int32_t taosGetTimestampSec(); -static int32_t startTime = 0; -static int64_t m_curLimit = 100 * 1024; +static int32_t startTime = INT32_MAX;; -static bool isMallocMem(size_t size, const char* func) { - if (0 == startTime) { - startTime = taosGetTimestampSec(); - return true; - } else { - int32_t currentTime = taosGetTimestampSec(); - if (currentTime - startTime < 10) return true; +static bool random_alloc_fail(size_t size, const char* file, uint32_t line) { + if (taosGetTimestampSec() < startTime) { + return false; } - if (size > m_curLimit) { - if (3 == rand() % 20) { - pTrace("====no alloc mem in func: %s, size:%d", func, size); - return false; - } + if (size < 100 * (size_t)1024) { + return false; + } + + if (rand() % 20 != 0) { + return false; + } + + if (fpAllocLog != NULL) { + fprintf(fpAllocLog, "memory allocation(%zu bytes) at line %d of '%s' will fail.\n", size, line, file); } return true; } -void* taos_malloc(size_t size, const char* func) { - if (false == isMallocMem(size, func)) { - return NULL; - } - return malloc(size); +static void* malloc_random(size_t size, const char* file, uint32_t line) { + return random_alloc_fail(size, file, line) ? NULL : malloc(size); } -void* taos_calloc(size_t num, size_t size, const char* func) { - if (false == isMallocMem(size, func)) { - return NULL; - } - return calloc(num, size); +static void* calloc_random(size_t num, size_t size, const char* file, uint32_t line) { + return random_alloc_fail(num * size, file, line) ? NULL : calloc(num, size); } -void* taos_realloc(void* ptr, size_t size, const char* func) { - if (false == isMallocMem(size, func)) { - return NULL; - } - return realloc(ptr, size); +static void* realloc_random(void* ptr, size_t size, const char* file, uint32_t line) { + return random_alloc_fail(size, file, line) ? NULL : realloc(ptr, size); } -void taos_free(void* ptr) { free(ptr); } - -char* taos_strdup(const char* str, const char* func) { +static char* strdup_random(const char* str, const char* file, uint32_t line) { size_t len = strlen(str); - return isMallocMem(len + 1, func) ? strdup(str) : NULL; + return random_alloc_fail(len + 1, file, line) ? NULL : strdup(str); } -char* taos_strndup(const char* str, size_t size, const char* func) { +static char* strndup_random(const char* str, size_t size, const char* file, uint32_t line) { size_t len = strlen(str); if (len > size) { len = size; } - return isMallocMem(len + 1, func) ? strndup(str, len) : NULL; + return random_alloc_fail(len + 1, file, line) ? NULL : strndup(str, len); } -#elif TAOS_MEM_CHECK == 2 +static ssize_t getline_random(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { + return random_alloc_fail(*n, file, line) ? -1 : getline(lineptr, n, stream); +} -#define MAGIC 0x55AA +//////////////////////////////////////////////////////////////////////////////// +// memory allocator with leak detection + +#define MEMBLK_MAGIC 0x55AA typedef struct SMemBlock { const char* file; @@ -95,7 +98,6 @@ typedef struct SMemBlock { static SMemBlock *blocks = NULL; static uintptr_t lock = 0; -static FILE* fpMemLeak = NULL; static void add_mem_block(SMemBlock* blk) { blk->prev = NULL; @@ -127,7 +129,25 @@ static void remove_mem_block(SMemBlock* blk) { blk->next = NULL; } -void* taos_malloc(size_t size, const char* file, uint32_t line) { +static void free_detect_leak(void* ptr, const char* file, uint32_t line) { + if (ptr == NULL) { + return; + } + + SMemBlock* blk = (SMemBlock*)(((char*)ptr) - sizeof(SMemBlock)); + if (blk->magic != MEMBLK_MAGIC) { + if (fpAllocLog != NULL) { + fprintf(fpAllocLog, "%s:%d: memory not allocated by 'taos_malloc'.\n", file, line); + } + free(ptr); + return; + } + + remove_mem_block(blk); + free(blk); +} + +static void* malloc_detect_leak(size_t size, const char* file, uint32_t line) { if (size == 0) { return NULL; } @@ -137,46 +157,46 @@ void* taos_malloc(size_t size, const char* file, uint32_t line) { return NULL; } - if (line > UINT16_MAX && fpMemLeak != NULL) { - fprintf(fpMemLeak, "%s:%d: line number too large.\n", file, line); + if (line > UINT16_MAX && fpAllocLog != NULL) { + fprintf(fpAllocLog, "%s:%d: line number too large.\n", file, line); } - if (size > UINT32_MAX && fpMemLeak != NULL) { - fprintf(fpMemLeak, "%s:%d: size too large: %lld.\n", file, line, size); + if (size > UINT32_MAX && fpAllocLog != NULL) { + fprintf(fpAllocLog, "%s:%d: size too large: %zu.\n", file, line, size); } blk->file = file; blk->line = (uint16_t)line; - blk->magic = MAGIC; + blk->magic = MEMBLK_MAGIC; blk->size = size; add_mem_block(blk); return blk->data; } -void* taos_calloc(size_t num, size_t size, const char* file, uint32_t line) { +static void* calloc_detect_leak(size_t num, size_t size, const char* file, uint32_t line) { size *= num; - void* p = taos_malloc(size, file, line); + void* p = malloc_detect_leak(size, file, line); if (p != NULL) { memset(p, 0, size); } return p; } -void* taos_realloc(void* ptr, size_t size, const char* file, uint32_t line) { +static void* realloc_detect_leak(void* ptr, size_t size, const char* file, uint32_t line) { if (size == 0) { - taos_free(ptr, file, line); + free_detect_leak(ptr, file, line); return NULL; } if (ptr == NULL) { - return taos_malloc(size, file, line); + return malloc_detect_leak(size, file, line); } SMemBlock* blk = ((char*)ptr) - sizeof(SMemBlock); - if (blk->magic != MAGIC) { - if (fpMemLeak != NULL) { - fprintf(fpMemLeak, "%s:%d: memory not allocated by 'taos_malloc'.\n", file, line); + if (blk->magic != MEMBLK_MAGIC) { + if (fpAllocLog != NULL) { + fprintf(fpAllocLog, "%s:%d: memory not allocated by 'taos_malloc'.\n", file, line); } return realloc(ptr, size); } @@ -189,8 +209,8 @@ void* taos_realloc(void* ptr, size_t size, const char* file, uint32_t line) { return NULL; } - if (size > UINT32_MAX && fpMemLeak != NULL) { - fprintf(fpMemLeak, "%s:%d: size too large: %lld.\n", file, line, size); + if (size > UINT32_MAX && fpAllocLog != NULL) { + fprintf(fpAllocLog, "%s:%d: size too large: %zu.\n", file, line, size); } blk = (SMemBlock*)p; @@ -199,27 +219,9 @@ void* taos_realloc(void* ptr, size_t size, const char* file, uint32_t line) { return blk->data; } -void taos_free(void* ptr, const char* file, uint32_t line) { - if (ptr == NULL) { - return; - } - - SMemBlock* blk = ((char*)ptr) - sizeof(SMemBlock); - if (blk->magic != MAGIC) { - if (fpMemLeak != NULL) { - fprintf(fpMemLeak, "%s:%d: memory not allocated by 'taos_malloc'.\n", file, line); - } - free(ptr); - return; - } - - remove_mem_block(blk); - free(blk); -} - -char* taos_strdup(const char* str, const char* file, uint32_t line) { +static char* strdup_detect_leak(const char* str, const char* file, uint32_t line) { size_t len = strlen(str); - char *p = taos_malloc(len + 1, file, line); + char *p = malloc_detect_leak(len + 1, file, line); if (p != NULL) { memcpy(p, str, len); p[len] = 0; @@ -227,12 +229,12 @@ char* taos_strdup(const char* str, const char* file, uint32_t line) { return p; } -char* taos_strndup(const char* str, size_t size, const char* file, uint32_t line) { +static char* strndup_detect_leak(const char* str, size_t size, const char* file, uint32_t line) { size_t len = strlen(str); if (len > size) { len = size; } - char *p = taos_malloc(len + 1, file, line); + char *p = malloc_detect_leak(len + 1, file, line); if (p != NULL) { memcpy(p, str, len); p[len] = 0; @@ -240,13 +242,13 @@ char* taos_strndup(const char* str, size_t size, const char* file, uint32_t line return p; } -ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { +static ssize_t getline_detect_leak(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { char* buf = NULL; size_t bufSize = 0; ssize_t size = getline(&buf, &bufSize, stream); if (size != -1) { if (*n < size + 1) { - void* p = taos_realloc(*lineptr, size + 1, file, line); + void* p = realloc_detect_leak(*lineptr, size + 1, file, line); if (p == NULL) { free(buf); return -1; @@ -261,16 +263,16 @@ ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char* file, return size; } -void taos_dump_memory_leak() { +static void dump_memory_leak() { const char* hex = "0123456789ABCDEF"; - const char* fmt = ":%d: addr=0x%p, size=%d, content(first 16 bytes)='"; + const char* fmt = ":%d: addr=0x%p, size=%d, content(first 16 bytes)="; size_t numOfBlk = 0, totalSize = 0; - if (fpMemLeak == NULL) { + if (fpAllocLog == NULL) { return; } - fputs("memory blocks allocated but not freed before exit:\n\n", fpMemLeak); + fputs("memory blocks allocated but not freed before exit:\n", fpAllocLog); while (atomic_val_compare_exchange_ptr(&lock, 0, 1) != 0); @@ -278,74 +280,198 @@ void taos_dump_memory_leak() { ++numOfBlk; totalSize += blk->size; - fputs(blk->file, fpMemLeak); - fprintf(fpMemLeak, fmt, blk->line, blk->data, blk->size); - - uint8_t c = (uint8_t)(blk->data[0]); - fputc(hex[c >> 4], fpMemLeak); - fputc(hex[c & 0x0f], fpMemLeak); + fputs(blk->file, fpAllocLog); + fprintf(fpAllocLog, fmt, blk->line, blk->data, blk->size); + char sep = '\''; size_t size = blk->size > 16 ? 16 : blk->size; - for (size_t i = 1; i < size; ++i) { - c = (uint8_t)(blk->data[i]); - fputc(' ', fpMemLeak); - fputc(hex[c >> 4], fpMemLeak); - fputc(hex[c & 0x0f], fpMemLeak); + for (size_t i = 0; i < size; ++i) { + uint8_t c = (uint8_t)(blk->data[i]); + fputc(sep, fpAllocLog); + sep = ' '; + fputc(hex[c >> 4], fpAllocLog); + fputc(hex[c & 0x0f], fpAllocLog); } - fputs("'\n", fpMemLeak); + fputs("'\n", fpAllocLog); } atomic_store_ptr(&lock, 0); - fprintf("\nnumber of blocks: %lld, total bytes: %lld\n", numOfBlk, totalSize); - if (fpMemLeak != stdout) { - fclose(fpMemLeak); - fpMemLeak = NULL; - } + fprintf(fpAllocLog, "\nnumber of blocks: %zu, total bytes: %zu\n", numOfBlk, totalSize); + fflush(fpAllocLog); } -static void dump_memory_leak_at_sig(int sig) { - fprintf(fpMemLeak, "signal %d received, exiting...\n", sig); +static void dump_memory_leak_on_sig(int sig) { + fprintf(fpAllocLog, "signal %d received.\n", sig); + // restore default signal handler struct sigaction act = {0}; act.sa_handler = SIG_DFL; sigaction(sig, &act, NULL); - taos_dump_memory_leak(); + dump_memory_leak(); } -void taos_detect_memory_leak(const char* path, bool autoDump) { - if (fpMemLeak != NULL) { - printf("memory leak detection already enabled.\n"); +//////////////////////////////////////////////////////////////////////////////// +// interface functions + +void* taos_malloc(size_t size, const char* file, uint32_t line) { + switch (allocMode) { + case TAOS_ALLOC_MODE_DEFAULT: + return malloc(size); + + case TAOS_ALLOC_MODE_RANDOM_FAIL: + return malloc_random(size, file, line); + + case TAOS_ALLOC_MODE_DETECT_LEAK: + return malloc_detect_leak(size, file, line); + } + return malloc(size); +} + +void* taos_calloc(size_t num, size_t size, const char* file, uint32_t line) { + switch (allocMode) { + case TAOS_ALLOC_MODE_DEFAULT: + return calloc(num, size); + + case TAOS_ALLOC_MODE_RANDOM_FAIL: + return calloc_random(num, size, file, line); + + case TAOS_ALLOC_MODE_DETECT_LEAK: + return calloc_detect_leak(num, size, file, line); + } + return calloc(num, size); +} + +void* taos_realloc(void* ptr, size_t size, const char* file, uint32_t line) { + switch (allocMode) { + case TAOS_ALLOC_MODE_DEFAULT: + return realloc(ptr, size); + + case TAOS_ALLOC_MODE_RANDOM_FAIL: + return realloc_random(ptr, size, file, line); + + case TAOS_ALLOC_MODE_DETECT_LEAK: + return realloc_detect_leak(ptr, size, file, line); + } + return realloc(ptr, size); +} + +void taos_free(void* ptr, const char* file, uint32_t line) { + switch (allocMode) { + case TAOS_ALLOC_MODE_DEFAULT: + return free(ptr); + + case TAOS_ALLOC_MODE_RANDOM_FAIL: + return free(ptr); + + case TAOS_ALLOC_MODE_DETECT_LEAK: + return free_detect_leak(ptr, file, line); + } + return free(ptr); +} + +char* taos_strdup(const char* str, const char* file, uint32_t line) { + switch (allocMode) { + case TAOS_ALLOC_MODE_DEFAULT: + return strdup(str); + + case TAOS_ALLOC_MODE_RANDOM_FAIL: + return strdup_random(str, file, line); + + case TAOS_ALLOC_MODE_DETECT_LEAK: + return strdup_detect_leak(str, file, line); + } + return strdup(str); +} + +char* taos_strndup(const char* str, size_t size, const char* file, uint32_t line) { + switch (allocMode) { + case TAOS_ALLOC_MODE_DEFAULT: + return strndup(str, size); + + case TAOS_ALLOC_MODE_RANDOM_FAIL: + return strndup_random(str, size, file, line); + + case TAOS_ALLOC_MODE_DETECT_LEAK: + return strndup_detect_leak(str, size, file, line); + } + return strndup(str, size); +} + +ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { + switch (allocMode) { + case TAOS_ALLOC_MODE_DEFAULT: + return getline(lineptr, n, stream); + + case TAOS_ALLOC_MODE_RANDOM_FAIL: + return getline_random(lineptr, n, stream, file, line); + + case TAOS_ALLOC_MODE_DETECT_LEAK: + return getline_detect_leak(lineptr, n, stream, file, line); + } + return getline(lineptr, n, stream); +} + +static void close_alloc_log() { + if (fpAllocLog != NULL) { + if (fpAllocLog != stdout) { + fclose(fpAllocLog); + } + fpAllocLog = NULL; + } +} + +void taosSetAllocMode(int mode, const char* path, bool autoDump) { + assert(mode >= TAOS_ALLOC_MODE_DEFAULT); + assert(mode <= TAOS_ALLOC_MODE_DETECT_LEAK); + + if (fpAllocLog != NULL || allocMode != TAOS_ALLOC_MODE_DEFAULT) { + printf("memory allocation mode can only be set once.\n"); return; } if (path == NULL || path[0] == 0) { - fpMemLeak = stdout; - } else if ((fpMemLeak = fopen(path, "w")) == NULL) { - printf("failed to open memory leak dump file '%s', errno=%d\n", path, errno); + fpAllocLog = stdout; + } else if ((fpAllocLog = fopen(path, "w")) != NULL) { + atexit(close_alloc_log); + } else { + printf("failed to open memory allocation log file '%s', errno=%d\n", path, errno); return; } - if (autoDump) { - atexit(taos_dump_memory_leak); + allocMode = mode; + + if (mode == TAOS_ALLOC_MODE_RANDOM_FAIL) { + startTime = taosGetTimestampSec() + 10; + return; + } + + if (autoDump && mode == TAOS_ALLOC_MODE_DETECT_LEAK) { + atexit(dump_memory_leak); + struct sigaction act = {0}; - act.sa_handler = dump_memory_leak_at_sig; + act.sa_handler = dump_memory_leak_on_sig; sigaction(SIGFPE, &act, NULL); sigaction(SIGSEGV, &act, NULL); sigaction(SIGILL, &act, NULL); } } -#endif +void taosDumpMemoryLeak() { + dump_memory_leak(); + close_alloc_log(); +} -#if TAOS_MEM_CHECK != 2 -void taos_dump_memory_leak() { +#else // 'TAOS_MEM_CHECK' not defined + +void taosSetAllocMode(int mode, const char* path, bool autoDump) { // do nothing } -void taos_detect_memory_leak(const char* path, bool autoDump) { - printf("memory leak detection not enabled, please set 'TAOS_MEM_CHECK' to 2."); +void taosDumpMemoryLeak() { + // do nothing } -#endif \ No newline at end of file + +#endif // TAOS_MEM_CHECK From 91f5600c2e4ca9ca43beff4d8952a0f1b5dbda44 Mon Sep 17 00:00:00 2001 From: localvar Date: Thu, 21 Nov 2019 08:22:59 +0000 Subject: [PATCH 6/6] add memory check to GO connector also update log format --- src/connector/go/src/taosSql/utils.go | 24 +++++++++++++++++++++++- src/util/src/tmem.c | 8 ++++---- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/src/connector/go/src/taosSql/utils.go b/src/connector/go/src/taosSql/utils.go index a5a90059b5..a104322fcc 100755 --- a/src/connector/go/src/taosSql/utils.go +++ b/src/connector/go/src/taosSql/utils.go @@ -12,15 +12,25 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - package taosSql +/* +#cgo CFLAGS : -I/usr/include +#include +#cgo LDFLAGS: -L/usr/lib -ltaos +void taosSetAllocMode(int mode, const char* path, _Bool autoDump); +void taosDumpMemoryLeak(); +*/ +import "C" + + import ( "database/sql/driver" "errors" "fmt" "sync/atomic" "time" + "unsafe" ) // Returns the bool value of the input. @@ -398,3 +408,15 @@ func namedValueToValue(named []driver.NamedValue) ([]driver.Value, error) { } +/****************************************************************************** +* Utils for C memory issues debugging * +******************************************************************************/ +func SetAllocMode(mode int32, path string) { + cpath := C.CString(path) + defer C.free(unsafe.Pointer(cpath)) + C.taosSetAllocMode(C.int(mode), cpath, false) +} + +func DumpMemoryLeak() { + C.taosDumpMemoryLeak() +} diff --git a/src/util/src/tmem.c b/src/util/src/tmem.c index 5a42933ebe..2625e4e5e6 100644 --- a/src/util/src/tmem.c +++ b/src/util/src/tmem.c @@ -45,7 +45,7 @@ static bool random_alloc_fail(size_t size, const char* file, uint32_t line) { } if (fpAllocLog != NULL) { - fprintf(fpAllocLog, "memory allocation(%zu bytes) at line %d of '%s' will fail.\n", size, line, file); + fprintf(fpAllocLog, "%s:%d: memory allocation of %zu bytes will fail.\n", file, line, size); } return true; @@ -137,7 +137,7 @@ static void free_detect_leak(void* ptr, const char* file, uint32_t line) { SMemBlock* blk = (SMemBlock*)(((char*)ptr) - sizeof(SMemBlock)); if (blk->magic != MEMBLK_MAGIC) { if (fpAllocLog != NULL) { - fprintf(fpAllocLog, "%s:%d: memory not allocated by 'taos_malloc'.\n", file, line); + fprintf(fpAllocLog, "%s:%d: memory is allocated by default allocator.\n", file, line); } free(ptr); return; @@ -196,7 +196,7 @@ static void* realloc_detect_leak(void* ptr, size_t size, const char* file, uint3 SMemBlock* blk = ((char*)ptr) - sizeof(SMemBlock); if (blk->magic != MEMBLK_MAGIC) { if (fpAllocLog != NULL) { - fprintf(fpAllocLog, "%s:%d: memory not allocated by 'taos_malloc'.\n", file, line); + fprintf(fpAllocLog, "%s:%d: memory is allocated by default allocator.\n", file, line); } return realloc(ptr, size); } @@ -265,7 +265,7 @@ static ssize_t getline_detect_leak(char **lineptr, size_t *n, FILE *stream, cons static void dump_memory_leak() { const char* hex = "0123456789ABCDEF"; - const char* fmt = ":%d: addr=0x%p, size=%d, content(first 16 bytes)="; + const char* fmt = ":%d: addr=%p, size=%d, content(first 16 bytes)="; size_t numOfBlk = 0, totalSize = 0; if (fpAllocLog == NULL) {