enh: support config randErrorScope dynamically

include/os/os.h

@@ -126,6 +126,7 @@ extern "C" {
 
 extern int32_t          tsRandErrChance;
 extern int64_t          tsRandErrDivisor;
+extern int64_t          tsRandErrScope;
 extern threadlocal bool tsEnableRandErr;
 
 #ifdef __cplusplus

include/util/tdef.h

@@ -568,6 +568,13 @@ enum {
   SND_WORKER_TYPE__UNIQUE,
 };
 
+enum {
+  RAND_ERR_MEMORY = 1,
+  RAND_ERR_FILE = 2,
+  // RAND_ERR_SCOPE_XXX... = 4,
+  // ...
+};
+
 #define DEFAULT_HANDLE 0
 #define MNODE_HANDLE   1
 #define QNODE_HANDLE   -1

source/common/src/tglobal.c

@@ -592,6 +592,9 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
                                 CFG_SCOPE_CLIENT, CFG_DYN_NONE));
   TAOS_CHECK_RETURN(
       cfgAddInt32(pCfg, "metaCacheMaxSize", tsMetaCacheMaxSize, -1, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
+  TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "randErrorChance", tsRandErrChance, 0, 10000, CFG_SCOPE_BOTH, CFG_DYN_NONE));
+  TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorDivisor", tsRandErrDivisor, 1, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH));
+  TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorScope", tsRandErrScope, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH));
 
   tsNumOfRpcThreads = tsNumOfCores / 2;
   tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS);
@@ -774,8 +777,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER));
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER));
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER));
-  TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "randErrorChance", tsRandErrChance, 0, 10000, CFG_SCOPE_BOTH, CFG_DYN_NONE));
-  TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorDivisor", tsRandErrDivisor, 1, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_SERVER));
 
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER, CFG_DYN_NONE));
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER));
@@ -1210,6 +1211,15 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
   TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "metaCacheMaxSize");
   tsMetaCacheMaxSize = pItem->i32;
 
+  TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "randErrorChance");
+  tsRandErrChance = pItem->i32;
+
+  TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "randErrorDivisor");
+  tsRandErrDivisor = pItem->i64;
+
+  TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "randErrorScope");
+  tsRandErrScope = pItem->i64;
+
   TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "countAlwaysReturnValue");
   tsCountAlwaysReturnValue = pItem->i32;
 
@@ -1466,12 +1476,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "mqRebalanceInterval");
   tsMqRebalanceInterval = pItem->i32;
 
-  TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "randErrorChance");
-  tsRandErrChance = pItem->i32;
-
-  TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "randErrorDivisor");
-  tsRandErrDivisor = pItem->i64;
-
   TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "ttlUnit");
   tsTtlUnit = pItem->i32;
 
@@ -1927,6 +1931,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) {
                                          {"mndSdbWriteDelta", &tsMndSdbWriteDelta},
                                          {"minDiskFreeSize", &tsMinDiskFreeSize},
                                          {"randErrorDivisor", &tsRandErrDivisor},
+                                         {"randErrorScope", &tsRandErrScope},
 
                                          {"cacheLazyLoadThreshold", &tsCacheLazyLoadThreshold},
                                          {"checkpointInterval", &tsStreamCheckpointInterval},
@@ -2205,6 +2210,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) {
                                          {"queryPlannerTrace", &tsQueryPlannerTrace},
                                          {"queryNodeChunkSize", &tsQueryNodeChunkSize},
                                          {"queryUseNodeAllocator", &tsQueryUseNodeAllocator},
+                                         {"randErrorDivisor", &tsRandErrDivisor},
+                                         {"randErrorScope", &tsRandErrScope},
                                          {"smlDot2Underline", &tsSmlDot2Underline},
                                          {"shellActivityTimer", &tsShellActivityTimer},
                                          {"useAdapter", &tsUseAdapter},

source/os/src/osFile.c

@@ -15,6 +15,7 @@
 #define ALLOW_FORBID_FUNC
 #include "os.h"
 #include "osSemaphore.h"
+#include "tdef.h"
 #include "zlib.h"
 
 #ifdef WINDOWS
@@ -66,7 +67,7 @@ typedef struct TdFile {
 
 #ifdef BUILD_WITH_RAND_ERR
 #define STUB_RAND_IO_ERR(ret)                                \
-  if (tsEnableRandErr) {                        \
+  if (tsEnableRandErr && (tsRandErrScope & RAND_ERR_FILE)) { \
     uint32_t r = taosRand() % tsRandErrDivisor;              \
     if ((r + 1) <= tsRandErrChance) {                        \
       errno = EIO;                                           \

source/os/src/osMemory.c

@@ -20,9 +20,11 @@
 #include <malloc.h>
 #endif
 #include "os.h"
+#include "tdef.h"
 
 int32_t          tsRandErrChance = 1;
 int64_t          tsRandErrDivisor = 10001;
+int64_t          tsRandErrScope = (RAND_ERR_MEMORY | RAND_ERR_FILE);
 threadlocal bool tsEnableRandErr = 0;
 
 #if defined(USE_TD_MEMORY) || defined(USE_ADDR2LINE)
@@ -272,7 +274,7 @@ void *taosMemoryMalloc(int64_t size) {
 #else
 
 #ifdef BUILD_WITH_RAND_ERR
-  if (tsEnableRandErr) {
+  if (tsEnableRandErr && (tsRandErrScope & RAND_ERR_MEMORY)) {
     uint32_t r = taosRand() % tsRandErrDivisor;
     if ((r + 1) <= tsRandErrChance) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -302,7 +304,7 @@ void *taosMemoryCalloc(int64_t num, int64_t size) {
   return (char *)tmp + sizeof(TdMemoryInfo);
 #else
 #ifdef BUILD_WITH_RAND_ERR
-  if (tsEnableRandErr) {
+  if (tsEnableRandErr && (tsRandErrScope & RAND_ERR_MEMORY)) {
     uint32_t r = taosRand() % tsRandErrDivisor;
     if ((r + 1) <= tsRandErrChance) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -342,7 +344,7 @@ void *taosMemoryRealloc(void *ptr, int64_t size) {
   return (char *)tmp + sizeof(TdMemoryInfo);
 #else
 #ifdef BUILD_WITH_RAND_ERR
-  if (tsEnableRandErr) {
+  if (tsEnableRandErr && (tsRandErrScope & RAND_ERR_MEMORY)) {
     uint32_t r = taosRand() % tsRandErrDivisor;
     if ((r + 1) <= tsRandErrChance) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -377,7 +379,7 @@ char *taosStrdup(const char *ptr) {
   return (char *)tmp + sizeof(TdMemoryInfo);
 #else
 #ifdef BUILD_WITH_RAND_ERR
-  if (tsEnableRandErr) {
+  if (tsEnableRandErr && (tsRandErrScope & RAND_ERR_MEMORY)) {
     uint32_t r = taosRand() % tsRandErrDivisor;
     if ((r + 1) <= tsRandErrChance) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -443,7 +445,7 @@ void *taosMemoryMallocAlign(uint32_t alignment, int64_t size) {
 #else
 #if defined(LINUX)
 #ifdef BUILD_WITH_RAND_ERR
-  if (tsEnableRandErr) {
+  if (tsEnableRandErr && (tsRandErrScope & RAND_ERR_MEMORY)) {
     uint32_t r = taosRand() % tsRandErrDivisor;
     if ((r + 1) <= tsRandErrChance) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;