diff --git a/cmake/cmake.define b/cmake/cmake.define index f1a5cef67e..eb78b54cae 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -16,6 +16,12 @@ if (NOT DEFINED TD_GRANT) SET(TD_GRANT FALSE) endif() +IF (NOT DEFINED BUILD_WITH_RAND_ERR) + SET(BUILD_WITH_RAND_ERR FALSE) +ELSE () + SET(BUILD_WITH_RAND_ERR TRUE) +endif() + IF ("${WEBSOCKET}" MATCHES "true") SET(TD_WEBSOCKET TRUE) MESSAGE("Enable websocket") diff --git a/include/os/os.h b/include/os/os.h index 1749687d97..e71c76bdb3 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -124,6 +124,9 @@ extern "C" { #include "taoserror.h" #include "tlog.h" +extern int32_t tsRandErrChance; +extern threadlocal bool tsEnableRandErr; + #ifdef __cplusplus } #endif diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 46219fe34c..c3101d574a 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -319,6 +319,7 @@ int32_t tsMaxTsmaNum = 3; int32_t tsMaxTsmaCalcDelay = 600; int64_t tsmaDataDeleteMark = 1000 * 60 * 60 * 24; // in ms, default to 1d + #define TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, pName) \ if ((pItem = cfgGetItem(pCfg, pName)) == NULL) { \ TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); \ @@ -732,6 +733,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "randErrorChance", tsRandErrChance, 0, 10000, CFG_SCOPE_BOTH, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); @@ -1408,6 +1410,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "mqRebalanceInterval"); tsMqRebalanceInterval = pItem->i32; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "randErrorChance"); + tsRandErrChance = pItem->i32; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "ttlUnit"); tsTtlUnit = pItem->i32; diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c index 0b404ec176..441714313c 100644 --- a/source/libs/qworker/src/qwUtil.c +++ b/source/libs/qworker/src/qwUtil.c @@ -276,7 +276,9 @@ void qwFreeTaskHandle(qTaskInfo_t *taskHandle) { // Note: free/kill may in RC qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle); if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) { + tsEnableRandErr = true; qDestroyTask(otaskHandle); + tsEnableRandErr = false; qDebug("task handle destroyed"); } } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 6e684545f7..1311179e92 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -161,7 +161,9 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { if (taskHandle) { qwDbgSimulateSleep(); + tsEnableRandErr = true; code = qExecTaskOpt(taskHandle, pResList, &useconds, &hasMore, &localFetch); + tsEnableRandErr = false; if (code) { if (code != TSDB_CODE_OPS_NOT_SUPPORT) { QW_TASK_ELOG("qExecTask failed, code:%x - %s", code, tstrerror(code)); @@ -768,8 +770,11 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { QW_ERR_JRET(code); } + tsEnableRandErr = true; code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, qwMsg->msgInfo.compressMsg, sql, OPTR_EXEC_MODEL_BATCH); + tsEnableRandErr = false; + sql = NULL; if (code) { QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code)); @@ -1266,7 +1271,10 @@ int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SDeleteRes *pRes) { QW_ERR_JRET(code); } + tsEnableRandErr = true; code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, 0, NULL, OPTR_EXEC_MODEL_BATCH); + tsEnableRandErr = false; + if (code) { QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code)); QW_ERR_JRET(code); diff --git a/source/os/CMakeLists.txt b/source/os/CMakeLists.txt index 6438ce7ed0..32609301a9 100644 --- a/source/os/CMakeLists.txt +++ b/source/os/CMakeLists.txt @@ -21,6 +21,9 @@ endif () if(USE_TD_MEMORY) add_definitions(-DUSE_TD_MEMORY) endif () +if(BUILD_WITH_RAND_ERR) + add_definitions(-DBUILD_WITH_RAND_ERR) +endif () if(BUILD_ADDR2LINE) if(NOT TD_WINDOWS) target_include_directories( diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 761815aa2c..6d29fdeccd 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -21,6 +21,10 @@ #endif #include "os.h" +int32_t tsRandErrChance = 1; +threadlocal bool tsEnableRandErr = 0; + + #if defined(USE_TD_MEMORY) || defined(USE_ADDR2LINE) #define TD_MEMORY_SYMBOL ('T' << 24 | 'A' << 16 | 'O' << 8 | 'S') @@ -266,6 +270,16 @@ void *taosMemoryMalloc(int64_t size) { return (char *)tmp + sizeof(TdMemoryInfo); #else + +#ifdef BUILD_WITH_RAND_ERR + if (tsEnableRandErr) { + uint32_t r = taosRand() % 10001; + if ((r + 1) <= tsRandErrChance) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + } +#endif void *p = malloc(size); if (NULL == p) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -287,6 +301,16 @@ void *taosMemoryCalloc(int64_t num, int64_t size) { return (char *)tmp + sizeof(TdMemoryInfo); #else +#ifdef BUILD_WITH_RAND_ERR + if (tsEnableRandErr) { + uint32_t r = taosRand() % 10001; + if ((r + 1) <= tsRandErrChance) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + } +#endif + void *p = calloc(num, size); if (NULL == p) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -317,6 +341,16 @@ void *taosMemoryRealloc(void *ptr, int64_t size) { return (char *)tmp + sizeof(TdMemoryInfo); #else +#ifdef BUILD_WITH_RAND_ERR + if (tsEnableRandErr) { + uint32_t r = taosRand() % 10001; + if ((r + 1) <= tsRandErrChance) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + } +#endif + void *p = realloc(ptr, size); if (size > 0 && NULL == p) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -342,6 +376,16 @@ char *taosStrdup(const char *ptr) { return (char *)tmp + sizeof(TdMemoryInfo); #else +#ifdef BUILD_WITH_RAND_ERR + if (tsEnableRandErr) { + uint32_t r = taosRand() % 10001; + if ((r + 1) <= tsRandErrChance) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + } +#endif + return tstrdup(ptr); #endif } @@ -398,6 +442,16 @@ void *taosMemoryMallocAlign(uint32_t alignment, int64_t size) { ASSERT(0); #else #if defined(LINUX) +#ifdef BUILD_WITH_RAND_ERR + if (tsEnableRandErr) { + uint32_t r = taosRand() % 10001; + if ((r + 1) <= tsRandErrChance) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + } +#endif + void *p = memalign(alignment, size); if (NULL == p) { if (ENOMEM == errno) {