Merge pull request #20034 from taosdata/feat/TD-22341

feat: add memory debug option

include/os/osMemory.h

@@ -33,6 +33,8 @@ extern "C" {
 
 #endif // if !defined(WINDOWS)
 
+int32_t taosMemoryDbgInit();
+int32_t taosMemoryDbgInitRestore();
 void   *taosMemoryMalloc(int64_t size);
 void   *taosMemoryCalloc(int64_t num, int64_t size);
 void   *taosMemoryRealloc(void *ptr, int64_t size);

source/client/inc/clientInt.h

@@ -151,6 +151,10 @@ typedef struct STscObj {
   SHashObj*     pRequests;
 } STscObj;
 
+typedef struct STscDbg {
+  bool memEnable;
+} STscDbg;
+
 typedef struct SResultColumn {
   union {
     char*    nullbitmap;  // bitmap, one bit for each item in the list

source/client/src/clientEnv.c

@@ -33,6 +33,7 @@
 #define TSC_VAR_NOT_RELEASE 1
 #define TSC_VAR_RELEASED    0
 
+STscDbg  tscDbg = {0};
 SAppInfo appInfo;
 int64_t  lastClusterId = 0;
 int32_t  clientReqRefPool = -1;
@@ -515,6 +516,18 @@ void tscWriteCrashInfo(int signum, void *sigInfo, void *context) {
 }
 
 void taos_init_imp(void) {
+#if defined(LINUX)
+  if (tscDbg.memEnable) {
+    int32_t code = taosMemoryDbgInit();
+    if (code) {
+      printf("failed to init memory dbg, error:%s\n", tstrerror(code));
+    } else {
+      tsAsyncLog = false;    
+      printf("memory dbg enabled\n");
+    }
+  }
+#endif
+
   // In the APIs of other program language, taos_cleanup is not available yet.
   // So, to make sure taos_cleanup will be invoked to clean up the allocated resource to suppress the valgrind warning.
   atexit(taos_cleanup);

source/client/src/clientImpl.c

@@ -2346,6 +2346,8 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
     return NULL;
   }
 
+  tscDebug("taos_query start with sql:%s", sql);
+
   SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
   tsem_init(&param->sem, 0, 0);
 
@@ -2360,6 +2362,8 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
     taosMemoryFree(param);
   }
 
+  tscDebug("taos_query end with sql:%s", sql);
+
   return pRequest;
 }
 

source/client/src/clientMain.c

@@ -884,7 +884,9 @@ void continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest)
 
 void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) {
   int64_t connId = *(int64_t *)taos;
+  tscDebug("taos_query_a start with sql:%s", sql);
   taosAsyncQueryImpl(connId, sql, fp, param, false);
+  tscDebug("taos_query_a end with sql:%s", sql);
 }
 
 void taos_query_a_with_reqid(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param, int64_t reqid) {

source/dnode/mgmt/exe/dmMain.c

@@ -29,6 +29,7 @@
 #define DM_MACHINE_CODE  "Get machine code."
 #define DM_VERSION       "Print program version."
 #define DM_EMAIL         "<support@taosdata.com>"
+#define DM_MEM_DBG       "Enable memory debug"
 // clang-format on
 static struct {
 #ifdef WINDOWS
@@ -37,6 +38,7 @@ static struct {
   bool         dumpConfig;
   bool         dumpSdb;
   bool         generateGrant;
+  bool         memDbg;
   bool         printAuth;
   bool         printVersion;
   bool         printHelp;
@@ -166,8 +168,10 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
     } else if (strcmp(argv[i], "-e") == 0) {
       global.envCmd[cmdEnvIndex] = argv[++i];
       cmdEnvIndex++;
+    } else if (strcmp(argv[i], "-dm") == 0) {
+      global.memDbg = true;
     } else if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 ||
-               strcmp(argv[i], "-?")) {
+               strcmp(argv[i], "-?") == 0) {
       global.printHelp = true;
     } else {
     }
@@ -212,6 +216,7 @@ static void dmPrintHelp() {
   printf("%s%s%s%s\n", indent, "-e,", indent, DM_ENV_CMD);
   printf("%s%s%s%s\n", indent, "-E,", indent, DM_ENV_FILE);
   printf("%s%s%s%s\n", indent, "-k,", indent, DM_MACHINE_CODE);
+  printf("%s%s%s%s\n", indent, "-dm,", indent, DM_MEM_DBG);
   printf("%s%s%s%s\n", indent, "-V,", indent, DM_VERSION);
 
   printf("\n\nReport bugs to %s.\n", DM_EMAIL);
@@ -272,6 +277,18 @@ int mainWindows(int argc, char **argv) {
     return 0;
   }
 
+#if defined(LINUX)
+  if (global.memDbg) {
+    int32_t code = taosMemoryDbgInit();
+    if (code) {
+      printf("failed to init memory dbg, error:%s\n", tstrerror(code));
+      return code;
+    }
+    tsAsyncLog = false;    
+    printf("memory dbg enabled\n");
+  }
+#endif
+
   if (dmInitLog() != 0) {
     printf("failed to start since init log error\n");
     taosCleanupArgs();

source/libs/command/inc/commandInt.h

@@ -101,6 +101,8 @@ extern "C" {
 #define COMMAND_SCHEDULE_POLICY "schedulePolicy"
 #define COMMAND_ENABLE_RESCHEDULE "enableReSchedule"
 #define COMMAND_CATALOG_DEBUG "catalogDebug"
+#define COMMAND_ENABLE_MEM_DEBUG "enableMemDebug"
+#define COMMAND_DISABLE_MEM_DEBUG "disableMemDebug"
 
 typedef struct SExplainGroup {
   int32_t   nodeNum;

source/libs/command/src/command.c

@@ -687,6 +687,21 @@ static int32_t execAlterCmd(char* cmd, char* value, bool* processed) {
     code = schedulerEnableReSchedule(atoi(value));
   } else if (0 == strcasecmp(cmd, COMMAND_CATALOG_DEBUG)) {
     code = ctgdHandleDbgCommand(value);
+  } else if (0 == strcasecmp(cmd, COMMAND_ENABLE_MEM_DEBUG)) {
+    code = taosMemoryDbgInit();
+    if (code) {
+      qError("failed to init memory dbg, error:%s", tstrerror(code));
+      return code;
+    }
+    tsAsyncLog = false;
+    qInfo("memory dbg enabled");
+  } else if (0 == strcasecmp(cmd, COMMAND_DISABLE_MEM_DEBUG)) {
+    code = taosMemoryDbgInitRestore();
+    if (code) {
+      qError("failed to restore from memory dbg, error:%s", tstrerror(code));
+      return code;
+    }
+    qInfo("memory dbg disabled");
   } else {
     goto _return;
   }

source/os/src/osMemory.c

@@ -228,6 +228,32 @@ void taosPrintBackTrace() {
 void taosPrintBackTrace() { return; }
 #endif
 
+int32_t taosMemoryDbgInit() {
+#if defined(LINUX)
+  int ret = mallopt(M_MMAP_THRESHOLD, 0);
+  if (0 == ret) {
+    return TAOS_SYSTEM_ERROR(errno);
+  }  
+
+  return 0;
+#else 
+  return TSDB_CODE_FAILED;
+#endif
+}
+
+int32_t taosMemoryDbgInitRestore() {
+#if defined(LINUX)
+  int ret = mallopt(M_MMAP_THRESHOLD, 128 * 1024);
+  if (0 == ret) {
+    return TAOS_SYSTEM_ERROR(errno);
+  }  
+
+  return 0;
+#else 
+  return TSDB_CODE_FAILED;
+#endif
+}
+
 void *taosMemoryMalloc(int64_t size) {
 #ifdef USE_TD_MEMORY
   void *tmp = malloc(size + sizeof(TdMemoryInfo));