enh: add rand error test

cmake/cmake.define

@@ -16,6 +16,12 @@ if (NOT DEFINED TD_GRANT)
   SET(TD_GRANT FALSE)
 endif()
 
+IF (NOT DEFINED BUILD_WITH_RAND_ERR)
+  SET(BUILD_WITH_RAND_ERR FALSE)
+ELSE ()
+  SET(BUILD_WITH_RAND_ERR TRUE)
+endif()
+
 IF ("${WEBSOCKET}" MATCHES "true")
     SET(TD_WEBSOCKET TRUE)
     MESSAGE("Enable websocket")

include/os/os.h

@@ -124,6 +124,9 @@ extern "C" {
 #include "taoserror.h"
 #include "tlog.h"
 
+extern int32_t tsRandErrChance;
+extern threadlocal bool tsEnableRandErr;
+
 #ifdef __cplusplus
 }
 #endif

source/common/src/tglobal.c

@@ -319,6 +319,7 @@ int32_t tsMaxTsmaNum = 3;
 int32_t tsMaxTsmaCalcDelay = 600;
 int64_t tsmaDataDeleteMark = 1000 * 60 * 60 * 24;  // in ms, default to 1d
 
+
 #define TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, pName) \
   if ((pItem = cfgGetItem(pCfg, pName)) == NULL) {  \
     TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND);           \
@@ -732,6 +733,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER));
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER));
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER));
+  TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "randErrorChance", tsRandErrChance, 0, 10000, CFG_SCOPE_BOTH, CFG_DYN_NONE));
 
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER, CFG_DYN_NONE));
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER));
@@ -1408,6 +1410,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "mqRebalanceInterval");
   tsMqRebalanceInterval = pItem->i32;
 
+  TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "randErrorChance");
+  tsRandErrChance = pItem->i32;
+
   TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "ttlUnit");
   tsTtlUnit = pItem->i32;
 

source/libs/qworker/src/qwUtil.c

@@ -276,7 +276,9 @@ void qwFreeTaskHandle(qTaskInfo_t *taskHandle) {
   // Note: free/kill may in RC
   qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle);
   if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) {
+    tsEnableRandErr = true;
     qDestroyTask(otaskHandle);
+    tsEnableRandErr = false;
     qDebug("task handle destroyed");
   }
 }

source/libs/qworker/src/qworker.c

@@ -161,7 +161,9 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) {
     if (taskHandle) {
       qwDbgSimulateSleep();
 
+      tsEnableRandErr = true;
       code = qExecTaskOpt(taskHandle, pResList, &useconds, &hasMore, &localFetch);
+      tsEnableRandErr = false;
       if (code) {
         if (code != TSDB_CODE_OPS_NOT_SUPPORT) {
           QW_TASK_ELOG("qExecTask failed, code:%x - %s", code, tstrerror(code));
@@ -768,8 +770,11 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) {
     QW_ERR_JRET(code);
   }
 
+  tsEnableRandErr = true;
   code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, qwMsg->msgInfo.compressMsg, sql,
                          OPTR_EXEC_MODEL_BATCH);
+  tsEnableRandErr = false;
+  
   sql = NULL;
   if (code) {
     QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code));
@@ -1266,7 +1271,10 @@ int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SDeleteRes *pRes) {
     QW_ERR_JRET(code);
   }
 
+  tsEnableRandErr = true;
   code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, 0, NULL, OPTR_EXEC_MODEL_BATCH);
+  tsEnableRandErr = false;
+  
   if (code) {
     QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code));
     QW_ERR_JRET(code);

source/os/CMakeLists.txt

@@ -21,6 +21,9 @@ endif ()
 if(USE_TD_MEMORY)
     add_definitions(-DUSE_TD_MEMORY) 
 endif ()
+if(BUILD_WITH_RAND_ERR)
+    add_definitions(-DBUILD_WITH_RAND_ERR) 
+endif ()
 if(BUILD_ADDR2LINE)
     if(NOT TD_WINDOWS)
         target_include_directories(

source/os/src/osMemory.c

@@ -21,6 +21,10 @@
 #endif
 #include "os.h"
 
+int32_t tsRandErrChance = 1;
+threadlocal bool tsEnableRandErr = 0;
+
+
 #if defined(USE_TD_MEMORY) || defined(USE_ADDR2LINE)
 
 #define TD_MEMORY_SYMBOL ('T' << 24 | 'A' << 16 | 'O' << 8 | 'S')
@@ -266,6 +270,16 @@ void *taosMemoryMalloc(int64_t size) {
 
   return (char *)tmp + sizeof(TdMemoryInfo);
 #else
+
+#ifdef BUILD_WITH_RAND_ERR
+  if (tsEnableRandErr) {
+    uint32_t r = taosRand() % 10001;
+    if ((r + 1) <= tsRandErrChance) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return NULL;
+    }
+  }
+#endif
   void *p = malloc(size);
   if (NULL == p) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -287,6 +301,16 @@ void *taosMemoryCalloc(int64_t num, int64_t size) {
 
   return (char *)tmp + sizeof(TdMemoryInfo);
 #else
+#ifdef BUILD_WITH_RAND_ERR
+  if (tsEnableRandErr) {
+    uint32_t r = taosRand() % 10001;
+    if ((r + 1) <= tsRandErrChance) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return NULL;
+    }
+  }
+#endif
+
   void *p = calloc(num, size);
   if (NULL == p) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -317,6 +341,16 @@ void *taosMemoryRealloc(void *ptr, int64_t size) {
 
   return (char *)tmp + sizeof(TdMemoryInfo);
 #else
+#ifdef BUILD_WITH_RAND_ERR
+  if (tsEnableRandErr) {
+    uint32_t r = taosRand() % 10001;
+    if ((r + 1) <= tsRandErrChance) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return NULL;
+    }
+  }
+#endif
+
   void *p = realloc(ptr, size);
   if (size > 0 && NULL == p) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -342,6 +376,16 @@ char *taosStrdup(const char *ptr) {
 
   return (char *)tmp + sizeof(TdMemoryInfo);
 #else
+#ifdef BUILD_WITH_RAND_ERR
+  if (tsEnableRandErr) {
+    uint32_t r = taosRand() % 10001;
+    if ((r + 1) <= tsRandErrChance) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return NULL;
+    }
+  }
+#endif
+
   return tstrdup(ptr);  
 #endif
 }
@@ -398,6 +442,16 @@ void *taosMemoryMallocAlign(uint32_t alignment, int64_t size) {
   ASSERT(0);
 #else
 #if defined(LINUX)
+#ifdef BUILD_WITH_RAND_ERR
+  if (tsEnableRandErr) {
+    uint32_t r = taosRand() % 10001;
+    if ((r + 1) <= tsRandErrChance) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return NULL;
+    }
+  }
+#endif
+
   void *p = memalign(alignment, size);
   if (NULL == p) {
     if (ENOMEM == errno) {