Merge pull request #27155 from taosdata/enh/TD-31375-3.0

enh: simulate random i/o error

include/os/os.h

@@ -125,6 +125,7 @@ extern "C" {
 #include "tlog.h"
 
 extern int32_t          tsRandErrChance;
+extern int64_t          tsRandErrDivisor;
 extern threadlocal bool tsEnableRandErr;
 
 #ifdef __cplusplus

source/common/src/tglobal.c

@@ -775,6 +775,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER));
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER));
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "randErrorChance", tsRandErrChance, 0, 10000, CFG_SCOPE_BOTH, CFG_DYN_NONE));
+  TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorDivisor", tsRandErrDivisor, 1, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_SERVER));
 
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER, CFG_DYN_NONE));
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER));
@@ -1468,6 +1469,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "randErrorChance");
   tsRandErrChance = pItem->i32;
 
+  TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "randErrorDivisor");
+  tsRandErrDivisor = pItem->i64;
+
   TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "ttlUnit");
   tsTtlUnit = pItem->i32;
 
@@ -1922,6 +1926,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) {
 
                                          {"mndSdbWriteDelta", &tsMndSdbWriteDelta},
                                          {"minDiskFreeSize", &tsMinDiskFreeSize},
+                                         {"randErrorDivisor", &tsRandErrDivisor},
 
                                          {"cacheLazyLoadThreshold", &tsCacheLazyLoadThreshold},
                                          {"checkpointInterval", &tsStreamCheckpointInterval},

source/os/src/osFile.c

@@ -64,6 +64,20 @@ typedef struct TdFile {
 
 #define FILE_WITH_LOCK 1
 
+#ifdef BUILD_WITH_RAND_ERR
+#define STUB_RAND_IO_ERR(ret)                   \
+  if (tsEnableRandErr) {                        \
+    uint32_t r = taosRand() % tsRandErrDivisor; \
+    if ((r + 1) <= tsRandErrChance) {           \
+      errno = EIO;                              \
+      terrno = TAOS_SYSTEM_ERROR(errno);        \
+      return (ret);                             \
+    }                                           \
+  }
+#else
+#define STUB_RAND_IO_ERR(ret)
+#endif
+
 void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath) {
 #ifdef WINDOWS
 
@@ -335,7 +349,7 @@ FILE *taosOpenFileForStream(const char *path, int32_t tdFileOptions) {
     terrno = TSDB_CODE_INVALID_PARA;
     return NULL;
   }
-  FILE* f = fopen(path, mode);
+  FILE *f = fopen(path, mode);
   if (NULL == f) {
     terrno = TAOS_SYSTEM_ERROR(errno);
   }
@@ -694,6 +708,7 @@ int taosOpenFileNotStream(const char *path, int32_t tdFileOptions) {
 }
 
 int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) {
+  STUB_RAND_IO_ERR(terrno)
 #if FILE_WITH_LOCK
   (void)taosThreadRwlockRdlock(&(pFile->rwlock));
 #endif
@@ -747,6 +762,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) {
 }
 
 int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) {
+  STUB_RAND_IO_ERR(terrno)
   if (pFile == NULL) {
     terrno = TSDB_CODE_INVALID_PARA;
     return 0;
@@ -792,6 +808,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) {
 }
 
 int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t offset) {
+  STUB_RAND_IO_ERR(terrno)
   if (pFile == NULL) {
     terrno = TSDB_CODE_INVALID_PARA;
     return 0;
@@ -1131,6 +1148,7 @@ bool lastErrorIsFileNotExist() { return terrno == TAOS_SYSTEM_ERROR(ENOENT); }
 #endif  // WINDOWS
 
 TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) {
+  STUB_RAND_IO_ERR(NULL)
   FILE *fp = NULL;
 #ifdef WINDOWS
   HANDLE hFile = NULL;
@@ -1228,11 +1246,11 @@ int32_t taosCloseFile(TdFilePtr *ppFile) {
 #endif
   taosMemoryFree(*ppFile);
   *ppFile = NULL;
-  
   return code;
 }
 
 int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) {
+  STUB_RAND_IO_ERR(terrno)
   if (pFile == NULL) {
     terrno = TSDB_CODE_INVALID_PARA;
     return terrno;
@@ -1583,7 +1601,8 @@ int32_t taosLinkFile(char *src, char *dst) {
 }
 
 FILE *taosOpenCFile(const char *filename, const char *mode) {
-  FILE* f = fopen(filename, mode); 
+  STUB_RAND_IO_ERR(NULL)
+  FILE *f = fopen(filename, mode);
   if (NULL == f) {
     terrno = TAOS_SYSTEM_ERROR(errno);
   }
@@ -1603,10 +1622,12 @@ int taosSeekCFile(FILE *file, int64_t offset, int whence) {
 }
 
 size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream) {
+  STUB_RAND_IO_ERR(terrno)
   return fread(buffer, size, count, stream);
 }
 
 size_t taosWriteToCFile(const void *ptr, size_t size, size_t nitems, FILE *stream) {
+  STUB_RAND_IO_ERR(terrno)
   return fwrite(ptr, size, nitems, stream);
 }
 

source/os/src/osMemory.c

@@ -22,9 +22,9 @@
 #include "os.h"
 
 int32_t          tsRandErrChance = 1;
+int64_t          tsRandErrDivisor = 10001;
 threadlocal bool tsEnableRandErr = 0;
 
-
 #if defined(USE_TD_MEMORY) || defined(USE_ADDR2LINE)
 
 #define TD_MEMORY_SYMBOL ('T' << 24 | 'A' << 16 | 'O' << 8 | 'S')
@@ -273,7 +273,7 @@ void *taosMemoryMalloc(int64_t size) {
 
 #ifdef BUILD_WITH_RAND_ERR
   if (tsEnableRandErr) {
-    uint32_t r = taosRand() % 10001;
+    uint32_t r = taosRand() % tsRandErrDivisor;
     if ((r + 1) <= tsRandErrChance) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return NULL;
@@ -303,7 +303,7 @@ void *taosMemoryCalloc(int64_t num, int64_t size) {
 #else
 #ifdef BUILD_WITH_RAND_ERR
   if (tsEnableRandErr) {
-    uint32_t r = taosRand() % 10001;
+    uint32_t r = taosRand() % tsRandErrDivisor;
     if ((r + 1) <= tsRandErrChance) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return NULL;
@@ -343,7 +343,7 @@ void *taosMemoryRealloc(void *ptr, int64_t size) {
 #else
 #ifdef BUILD_WITH_RAND_ERR
   if (tsEnableRandErr) {
-    uint32_t r = taosRand() % 10001;
+    uint32_t r = taosRand() % tsRandErrDivisor;
     if ((r + 1) <= tsRandErrChance) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return NULL;
@@ -378,7 +378,7 @@ char *taosStrdup(const char *ptr) {
 #else
 #ifdef BUILD_WITH_RAND_ERR
   if (tsEnableRandErr) {
-    uint32_t r = taosRand() % 10001;
+    uint32_t r = taosRand() % tsRandErrDivisor;
     if ((r + 1) <= tsRandErrChance) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return NULL;
@@ -444,7 +444,7 @@ void *taosMemoryMallocAlign(uint32_t alignment, int64_t size) {
 #if defined(LINUX)
 #ifdef BUILD_WITH_RAND_ERR
   if (tsEnableRandErr) {
-    uint32_t r = taosRand() % 10001;
+    uint32_t r = taosRand() % tsRandErrDivisor;
     if ((r + 1) <= tsRandErrChance) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return NULL;