From 7e1bbd7459c5cd43260accb68ceb460a4eee2b20 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 12 Aug 2024 12:13:26 +0800 Subject: [PATCH 01/10] enh: simulate random i/o error --- source/os/src/osFile.c | 98 +++++++++++++++++++++++++++++++++++------- 1 file changed, 82 insertions(+), 16 deletions(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 821488c90c..ef57182154 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -64,6 +64,16 @@ typedef struct TdFile { #define FILE_WITH_LOCK 1 +#define STUB_RAND_IO_ERR(ret) \ + if (tsEnableRandErr) { \ + uint32_t r = taosRand() % 10001; \ + if ((r + 1) <= tsRandErrChance) { \ + errno = EIO; \ + terrno = TAOS_SYSTEM_ERROR(errno); \ + return (ret); \ + } \ + } + void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath) { #ifdef WINDOWS @@ -201,7 +211,10 @@ TdFilePtr taosCreateFile(const char *path, int32_t tdFileOptions) { return fp; } -int32_t taosRemoveFile(const char *path) { +int32_t taosRemoveFile(const char *path) { +#ifdef BUILD_WITH_RAND_ERR + STUB_RAND_IO_ERR(terrno) +#endif int32_t code = remove(path); if (-1 == code) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -211,6 +224,9 @@ int32_t taosRemoveFile(const char *path) { } int32_t taosRenameFile(const char *oldName, const char *newName) { +#ifdef BUILD_WITH_RAND_ERR + STUB_RAND_IO_ERR(terrno) +#endif #ifdef WINDOWS bool finished = false; @@ -694,6 +710,9 @@ int taosOpenFileNotStream(const char *path, int32_t tdFileOptions) { } int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { +#ifdef BUILD_WITH_RAND_ERR + STUB_RAND_IO_ERR(terrno) +#endif #if FILE_WITH_LOCK (void)taosThreadRwlockRdlock(&(pFile->rwlock)); #endif @@ -747,6 +766,9 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { } int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { +#ifdef BUILD_WITH_RAND_ERR + STUB_RAND_IO_ERR(terrno) +#endif if (pFile == NULL) { terrno = TSDB_CODE_INVALID_PARA; return 0; @@ -792,6 +814,9 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { } int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t offset) { +#ifdef BUILD_WITH_RAND_ERR + STUB_RAND_IO_ERR(terrno) +#endif if (pFile == NULL) { terrno = TSDB_CODE_INVALID_PARA; return 0; @@ -873,6 +898,9 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { } int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { +#ifdef BUILD_WITH_RAND_ERR + STUB_RAND_IO_ERR(terrno) +#endif if (pFile == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; @@ -888,7 +916,7 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { int32_t code = _fstat64(pFile->fd, &fileStat); #else struct stat fileStat; - int32_t code = fstat(pFile->fd, &fileStat); + int32_t code = fstat(pFile->fd, &fileStat); #endif if (-1 == code) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -944,7 +972,7 @@ int32_t taosUnLockFile(TdFilePtr pFile) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } - + #ifdef WINDOWS BOOL fSuccess = FALSE; OVERLAPPED overlapped = {0}; @@ -966,11 +994,13 @@ int32_t taosUnLockFile(TdFilePtr pFile) { } int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { +#ifdef BUILD_WITH_RAND_ERR + STUB_RAND_IO_ERR(terrno) +#endif if (NULL == pFile || pFile->fd < 0) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } - #ifdef WINDOWS HANDLE h = (HANDLE)_get_osfhandle(pFile->fd); @@ -1026,13 +1056,16 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { } int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size) { +#ifdef BUILD_WITH_RAND_ERR + STUB_RAND_IO_ERR(terrno) +#endif if (pFileOut == NULL || pFileIn == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } - if (pFileIn->fd < 0 || pFileOut->fd < 0) { - terrno = TSDB_CODE_INVALID_PARA; - return terrno; + if (pFileIn->fd < 0 || pFileOut->fd < 0) { + terrno = TSDB_CODE_INVALID_PARA; + return terrno; } #ifdef WINDOWS @@ -1131,6 +1164,9 @@ bool lastErrorIsFileNotExist() { return terrno == TAOS_SYSTEM_ERROR(ENOENT); } #endif // WINDOWS TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { +#ifdef BUILD_WITH_RAND_ERR + STUB_RAND_IO_ERR(NULL) +#endif FILE *fp = NULL; #ifdef WINDOWS HANDLE hFile = NULL; @@ -1191,6 +1227,9 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { } int32_t taosCloseFile(TdFilePtr *ppFile) { +#ifdef BUILD_WITH_RAND_ERR + STUB_RAND_IO_ERR(terrno) +#endif int32_t code = 0; if (ppFile == NULL || *ppFile == NULL) { return 0; @@ -1233,6 +1272,9 @@ int32_t taosCloseFile(TdFilePtr *ppFile) { } int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) { +#ifdef BUILD_WITH_RAND_ERR + STUB_RAND_IO_ERR(terrno) +#endif if (pFile == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; @@ -1295,6 +1337,9 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) } int32_t taosFsyncFile(TdFilePtr pFile) { +#ifdef BUILD_WITH_RAND_ERR + STUB_RAND_IO_ERR(terrno) +#endif if (pFile == NULL) { return 0; } @@ -1308,10 +1353,10 @@ int32_t taosFsyncFile(TdFilePtr pFile) { terrno = TAOS_SYSTEM_ERROR(errno); return terrno; } - + return 0; } - + #ifdef WINDOWS if (pFile->hFile != NULL) { if (pFile->tdFileOptions & TD_FILE_WRITE_THROUGH) { @@ -1359,11 +1404,15 @@ int32_t taosUmaskFile(int32_t maskVal) { } int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { +#ifdef BUILD_WITH_RAND_ERR + STUB_RAND_IO_ERR(terrno) +#endif int64_t ret = -1; int32_t code = 0; - + #if FILE_WITH_LOCK - (void)taosThreadRwlockRdlock(&(pFile->rwlock)); + (void) + taosThreadRwlockRdlock(&(pFile->rwlock)); #endif if (pFile == NULL || ptrBuf == NULL) { terrno = TSDB_CODE_INVALID_PARA; @@ -1433,16 +1482,19 @@ END: } int64_t taosGetsFile(TdFilePtr pFile, int32_t maxSize, char *__restrict buf) { +#ifdef BUILD_WITH_RAND_ERR + STUB_RAND_IO_ERR(terrno) +#endif if (pFile == NULL || buf == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } - + if (pFile->fp == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } - + if (fgets(buf, maxSize, pFile->fp) == NULL) { if (feof(pFile->fp)) { return 0; @@ -1451,7 +1503,7 @@ int64_t taosGetsFile(TdFilePtr pFile, int32_t maxSize, char *__restrict buf) { return terrno; } } - + return strlen(buf); } @@ -1582,7 +1634,10 @@ int32_t taosLinkFile(char *src, char *dst) { return 0; } -FILE *taosOpenCFile(const char *filename, const char *mode) { +FILE *taosOpenCFile(const char *filename, const char *mode) { +#ifdef BUILD_WITH_RAND_ERR + STUB_RAND_IO_ERR(NULL) +#endif FILE* f = fopen(filename, mode); if (NULL == f) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -1603,14 +1658,25 @@ int taosSeekCFile(FILE *file, int64_t offset, int whence) { } size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream) { +#ifdef BUILD_WITH_RAND_ERR + STUB_RAND_IO_ERR(terrno) +#endif return fread(buffer, size, count, stream); } size_t taosWriteToCFile(const void *ptr, size_t size, size_t nitems, FILE *stream) { +#ifdef BUILD_WITH_RAND_ERR + STUB_RAND_IO_ERR(terrno) +#endif return fwrite(ptr, size, nitems, stream); } -int taosCloseCFile(FILE *f) { return fclose(f); } +int taosCloseCFile(FILE *f) { +#ifdef BUILD_WITH_RAND_ERR + STUB_RAND_IO_ERR(terrno) +#endif + return fclose(f); +} int taosSetAutoDelFile(char *path) { #ifdef WINDOWS From 410008ac2f4bc3cd866dfe997799d88103a4ffa6 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 12 Aug 2024 12:21:07 +0800 Subject: [PATCH 02/10] enh: simulate random i/o error --- source/os/src/osFile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index ef57182154..fc1c6b4b09 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -1001,6 +1001,7 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } + #ifdef WINDOWS HANDLE h = (HANDLE)_get_osfhandle(pFile->fd); @@ -1411,8 +1412,7 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { int32_t code = 0; #if FILE_WITH_LOCK - (void) - taosThreadRwlockRdlock(&(pFile->rwlock)); + (void)taosThreadRwlockRdlock(&(pFile->rwlock)); #endif if (pFile == NULL || ptrBuf == NULL) { terrno = TSDB_CODE_INVALID_PARA; From 787715a51944f6aad2dc318d9206ba398c30f956 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 12 Aug 2024 12:23:46 +0800 Subject: [PATCH 03/10] enh: simulate random i/o error --- source/os/src/osFile.c | 85 +++++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index fc1c6b4b09..338bf27dc5 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -64,6 +64,7 @@ typedef struct TdFile { #define FILE_WITH_LOCK 1 +#define BUILD_WITH_RAND_IO_ERR BUILD_WITH_RAND_ERR #define STUB_RAND_IO_ERR(ret) \ if (tsEnableRandErr) { \ uint32_t r = taosRand() % 10001; \ @@ -155,7 +156,7 @@ int64_t taosCopyFile(const char *from, const char *to) { code = terrno; goto _err; } - + if (bytes == 0) break; size += bytes; @@ -176,7 +177,7 @@ int64_t taosCopyFile(const char *from, const char *to) { terrno = code; return -1; } - + return size; _err: @@ -207,15 +208,15 @@ TdFilePtr taosCreateFile(const char *path, int32_t tdFileOptions) { } } } - + return fp; } int32_t taosRemoveFile(const char *path) { -#ifdef BUILD_WITH_RAND_ERR +#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) #endif - int32_t code = remove(path); + int32_t code = remove(path); if (-1 == code) { terrno = TAOS_SYSTEM_ERROR(errno); return terrno; @@ -224,7 +225,7 @@ int32_t taosRemoveFile(const char *path) { } int32_t taosRenameFile(const char *oldName, const char *newName) { -#ifdef BUILD_WITH_RAND_ERR +#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) #endif #ifdef WINDOWS @@ -316,7 +317,7 @@ int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } - + struct stat fileStat; int32_t code = fstat(pFile->fd, &fileStat); if (-1 == code) { @@ -351,7 +352,7 @@ FILE *taosOpenFileForStream(const char *path, int32_t tdFileOptions) { terrno = TSDB_CODE_INVALID_PARA; return NULL; } - FILE* f = fopen(path, mode); + FILE *f = fopen(path, mode); if (NULL == f) { terrno = TAOS_SYSTEM_ERROR(errno); } @@ -710,7 +711,7 @@ int taosOpenFileNotStream(const char *path, int32_t tdFileOptions) { } int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { -#ifdef BUILD_WITH_RAND_ERR +#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) #endif #if FILE_WITH_LOCK @@ -724,7 +725,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } - + int64_t leftbytes = count; int64_t readbytes; char *tbuf = (char *)buf; @@ -766,7 +767,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { } int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { -#ifdef BUILD_WITH_RAND_ERR +#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) #endif if (pFile == NULL) { @@ -814,7 +815,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { } int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t offset) { -#ifdef BUILD_WITH_RAND_ERR +#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) #endif if (pFile == NULL) { @@ -875,7 +876,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { int32_t code = 0; ASSERT(pFile->fd >= 0); // Please check if you have closed the file. - + #ifdef WINDOWS int64_t ret = _lseeki64(pFile->fd, offset, whence); #else @@ -898,7 +899,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { } int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { -#ifdef BUILD_WITH_RAND_ERR +#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) #endif if (pFile == NULL) { @@ -994,7 +995,7 @@ int32_t taosUnLockFile(TdFilePtr pFile) { } int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { -#ifdef BUILD_WITH_RAND_ERR +#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) #endif if (NULL == pFile || pFile->fd < 0) { @@ -1057,7 +1058,7 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { } int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size) { -#ifdef BUILD_WITH_RAND_ERR +#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) #endif if (pFileOut == NULL || pFileIn == NULL) { @@ -1165,7 +1166,7 @@ bool lastErrorIsFileNotExist() { return terrno == TAOS_SYSTEM_ERROR(ENOENT); } #endif // WINDOWS TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { -#ifdef BUILD_WITH_RAND_ERR +#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(NULL) #endif FILE *fp = NULL; @@ -1228,7 +1229,7 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { } int32_t taosCloseFile(TdFilePtr *ppFile) { -#ifdef BUILD_WITH_RAND_ERR +#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) #endif int32_t code = 0; @@ -1268,19 +1269,19 @@ int32_t taosCloseFile(TdFilePtr *ppFile) { #endif taosMemoryFree(*ppFile); *ppFile = NULL; - + return code; } int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) { -#ifdef BUILD_WITH_RAND_ERR +#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) #endif if (pFile == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } - + int32_t code = 0; #ifdef WINDOWS @@ -1292,11 +1293,11 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) #if FILE_WITH_LOCK (void)taosThreadRwlockUnlock(&(pFile->rwlock)); #endif - + terrno = TSDB_CODE_INVALID_PARA; return terrno; } - + DWORD ret = 0; OVERLAPPED ol = {0}; ol.OffsetHigh = (uint32_t)((offset & 0xFFFFFFFF00000000LL) >> 0x20); @@ -1333,12 +1334,12 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) terrno = code; return code; } - + return ret; } int32_t taosFsyncFile(TdFilePtr pFile) { -#ifdef BUILD_WITH_RAND_ERR +#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) #endif if (pFile == NULL) { @@ -1405,7 +1406,7 @@ int32_t taosUmaskFile(int32_t maskVal) { } int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { -#ifdef BUILD_WITH_RAND_ERR +#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) #endif int64_t ret = -1; @@ -1426,7 +1427,7 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { terrno = TSDB_CODE_INVALID_PARA; goto END; } - + #ifdef WINDOWS size_t bufferSize = 512; *ptrBuf = taosMemoryMalloc(bufferSize); @@ -1482,7 +1483,7 @@ END: } int64_t taosGetsFile(TdFilePtr pFile, int32_t maxSize, char *__restrict buf) { -#ifdef BUILD_WITH_RAND_ERR +#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) #endif if (pFile == NULL || buf == NULL) { @@ -1545,17 +1546,17 @@ bool taosCheckAccessFile(const char *pathname, int32_t tdFileAccessOptions) { bool taosCheckExistFile(const char *pathname) { return taosCheckAccessFile(pathname, TD_FILE_ACCESS_EXIST_OK); }; int32_t taosCompressFile(char *srcFileName, char *destFileName) { - int32_t compressSize = 163840; - int32_t ret = 0; - int32_t len = 0; - gzFile dstFp = NULL; + int32_t compressSize = 163840; + int32_t ret = 0; + int32_t len = 0; + gzFile dstFp = NULL; TdFilePtr pSrcFile = NULL; - char *data = taosMemoryMalloc(compressSize); + char *data = taosMemoryMalloc(compressSize); if (NULL == data) { return terrno; } - + pSrcFile = taosOpenFile(srcFileName, TD_FILE_READ | TD_FILE_STREAM); if (pSrcFile == NULL) { ret = terrno; @@ -1635,14 +1636,14 @@ int32_t taosLinkFile(char *src, char *dst) { } FILE *taosOpenCFile(const char *filename, const char *mode) { -#ifdef BUILD_WITH_RAND_ERR +#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(NULL) #endif - FILE* f = fopen(filename, mode); + FILE *f = fopen(filename, mode); if (NULL == f) { terrno = TAOS_SYSTEM_ERROR(errno); } - return f; + return f; } int taosSeekCFile(FILE *file, int64_t offset, int whence) { @@ -1658,21 +1659,21 @@ int taosSeekCFile(FILE *file, int64_t offset, int whence) { } size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream) { -#ifdef BUILD_WITH_RAND_ERR +#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) #endif return fread(buffer, size, count, stream); } size_t taosWriteToCFile(const void *ptr, size_t size, size_t nitems, FILE *stream) { -#ifdef BUILD_WITH_RAND_ERR +#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) #endif return fwrite(ptr, size, nitems, stream); } int taosCloseCFile(FILE *f) { -#ifdef BUILD_WITH_RAND_ERR +#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) #endif return fclose(f); @@ -1686,6 +1687,6 @@ int taosSetAutoDelFile(char *path) { terrno = TAOS_SYSTEM_ERROR(errno); return terrno; } - return 0; -#endif + return 0; +#endif } From 563920eea85b2c15b5b26c0f0723fb3ebe6d3c36 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 12 Aug 2024 13:29:19 +0800 Subject: [PATCH 04/10] enh: simulate random i/o error --- source/os/src/osFile.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 338bf27dc5..a85e135e2e 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -64,7 +64,9 @@ typedef struct TdFile { #define FILE_WITH_LOCK 1 -#define BUILD_WITH_RAND_IO_ERR BUILD_WITH_RAND_ERR +#ifdef BUILD_WITH_RAND_ERR +#define BUILD_WITH_RAND_IO_ERR +#endif #define STUB_RAND_IO_ERR(ret) \ if (tsEnableRandErr) { \ uint32_t r = taosRand() % 10001; \ From c14363625d718bd3a263a4c88961e6ebc55600c3 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 12 Aug 2024 13:31:46 +0800 Subject: [PATCH 05/10] enh: simulate random i/o error --- source/os/src/osFile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index a85e135e2e..db63198a9d 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -66,7 +66,6 @@ typedef struct TdFile { #ifdef BUILD_WITH_RAND_ERR #define BUILD_WITH_RAND_IO_ERR -#endif #define STUB_RAND_IO_ERR(ret) \ if (tsEnableRandErr) { \ uint32_t r = taosRand() % 10001; \ @@ -76,6 +75,7 @@ typedef struct TdFile { return (ret); \ } \ } +#endif void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath) { #ifdef WINDOWS From 89b4c7a3edb1e819524cf041e0e89071e4c58489 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 12 Aug 2024 13:37:11 +0800 Subject: [PATCH 06/10] enh: simulate random i/o error --- source/os/src/osFile.c | 39 ++------------------------------------- 1 file changed, 2 insertions(+), 37 deletions(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index db63198a9d..9052669881 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -65,7 +65,6 @@ typedef struct TdFile { #define FILE_WITH_LOCK 1 #ifdef BUILD_WITH_RAND_ERR -#define BUILD_WITH_RAND_IO_ERR #define STUB_RAND_IO_ERR(ret) \ if (tsEnableRandErr) { \ uint32_t r = taosRand() % 10001; \ @@ -75,6 +74,8 @@ typedef struct TdFile { return (ret); \ } \ } +#else +#define STUB_RAND_IO_ERR(ret) #endif void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath) { @@ -215,9 +216,7 @@ TdFilePtr taosCreateFile(const char *path, int32_t tdFileOptions) { } int32_t taosRemoveFile(const char *path) { -#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) -#endif int32_t code = remove(path); if (-1 == code) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -227,9 +226,7 @@ int32_t taosRemoveFile(const char *path) { } int32_t taosRenameFile(const char *oldName, const char *newName) { -#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) -#endif #ifdef WINDOWS bool finished = false; @@ -713,9 +710,7 @@ int taosOpenFileNotStream(const char *path, int32_t tdFileOptions) { } int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { -#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) -#endif #if FILE_WITH_LOCK (void)taosThreadRwlockRdlock(&(pFile->rwlock)); #endif @@ -769,9 +764,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { } int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { -#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) -#endif if (pFile == NULL) { terrno = TSDB_CODE_INVALID_PARA; return 0; @@ -817,9 +810,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { } int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t offset) { -#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) -#endif if (pFile == NULL) { terrno = TSDB_CODE_INVALID_PARA; return 0; @@ -901,9 +892,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { } int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { -#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) -#endif if (pFile == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; @@ -997,9 +986,7 @@ int32_t taosUnLockFile(TdFilePtr pFile) { } int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { -#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) -#endif if (NULL == pFile || pFile->fd < 0) { terrno = TSDB_CODE_INVALID_PARA; return terrno; @@ -1060,9 +1047,7 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { } int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size) { -#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) -#endif if (pFileOut == NULL || pFileIn == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; @@ -1168,9 +1153,7 @@ bool lastErrorIsFileNotExist() { return terrno == TAOS_SYSTEM_ERROR(ENOENT); } #endif // WINDOWS TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { -#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(NULL) -#endif FILE *fp = NULL; #ifdef WINDOWS HANDLE hFile = NULL; @@ -1231,9 +1214,7 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { } int32_t taosCloseFile(TdFilePtr *ppFile) { -#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) -#endif int32_t code = 0; if (ppFile == NULL || *ppFile == NULL) { return 0; @@ -1276,9 +1257,7 @@ int32_t taosCloseFile(TdFilePtr *ppFile) { } int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) { -#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) -#endif if (pFile == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; @@ -1341,9 +1320,7 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) } int32_t taosFsyncFile(TdFilePtr pFile) { -#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) -#endif if (pFile == NULL) { return 0; } @@ -1408,9 +1385,7 @@ int32_t taosUmaskFile(int32_t maskVal) { } int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { -#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) -#endif int64_t ret = -1; int32_t code = 0; @@ -1485,9 +1460,7 @@ END: } int64_t taosGetsFile(TdFilePtr pFile, int32_t maxSize, char *__restrict buf) { -#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) -#endif if (pFile == NULL || buf == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; @@ -1638,9 +1611,7 @@ int32_t taosLinkFile(char *src, char *dst) { } FILE *taosOpenCFile(const char *filename, const char *mode) { -#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(NULL) -#endif FILE *f = fopen(filename, mode); if (NULL == f) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -1661,23 +1632,17 @@ int taosSeekCFile(FILE *file, int64_t offset, int whence) { } size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream) { -#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) -#endif return fread(buffer, size, count, stream); } size_t taosWriteToCFile(const void *ptr, size_t size, size_t nitems, FILE *stream) { -#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) -#endif return fwrite(ptr, size, nitems, stream); } int taosCloseCFile(FILE *f) { -#ifdef BUILD_WITH_RAND_IO_ERR STUB_RAND_IO_ERR(terrno) -#endif return fclose(f); } From eda233dd8d37e84fe0d1e444d5a382e5bdf67e03 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 12 Aug 2024 13:56:14 +0800 Subject: [PATCH 07/10] enh: simulate random i/o error --- source/os/src/osFile.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 9052669881..6c4b8aa768 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -892,7 +892,6 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { } int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { - STUB_RAND_IO_ERR(terrno) if (pFile == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; @@ -923,6 +922,8 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { *mtime = fileStat.st_mtime; } + STUB_RAND_IO_ERR(terrno) + return 0; } @@ -1214,7 +1215,6 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { } int32_t taosCloseFile(TdFilePtr *ppFile) { - STUB_RAND_IO_ERR(terrno) int32_t code = 0; if (ppFile == NULL || *ppFile == NULL) { return 0; @@ -1253,6 +1253,11 @@ int32_t taosCloseFile(TdFilePtr *ppFile) { taosMemoryFree(*ppFile); *ppFile = NULL; +#if BUILD_WITH_RAND_ERR + if (terrno == 0) { + STUB_RAND_IO_ERR(terrno) + } +#endif return code; } @@ -1642,8 +1647,13 @@ size_t taosWriteToCFile(const void *ptr, size_t size, size_t nitems, FILE *strea } int taosCloseCFile(FILE *f) { - STUB_RAND_IO_ERR(terrno) - return fclose(f); + int32_t code = fclose(f); +#if BUILD_WITH_RAND_ERR + if (code == 0) { + STUB_RAND_IO_ERR(terrno) + } +#endif + return code == 0 ? code : TAOS_SYSTEM_ERROR(errno); } int taosSetAutoDelFile(char *path) { From ee571773f8fadeb831ad770a21ba28ed962a705c Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 12 Aug 2024 18:45:50 +0800 Subject: [PATCH 08/10] fix: decrease the dimension when mocking i/o error --- source/os/src/osFile.c | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 6c4b8aa768..8c1e8cfaf1 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -216,7 +216,6 @@ TdFilePtr taosCreateFile(const char *path, int32_t tdFileOptions) { } int32_t taosRemoveFile(const char *path) { - STUB_RAND_IO_ERR(terrno) int32_t code = remove(path); if (-1 == code) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -226,7 +225,6 @@ int32_t taosRemoveFile(const char *path) { } int32_t taosRenameFile(const char *oldName, const char *newName) { - STUB_RAND_IO_ERR(terrno) #ifdef WINDOWS bool finished = false; @@ -922,8 +920,6 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { *mtime = fileStat.st_mtime; } - STUB_RAND_IO_ERR(terrno) - return 0; } @@ -987,7 +983,6 @@ int32_t taosUnLockFile(TdFilePtr pFile) { } int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { - STUB_RAND_IO_ERR(terrno) if (NULL == pFile || pFile->fd < 0) { terrno = TSDB_CODE_INVALID_PARA; return terrno; @@ -1048,7 +1043,6 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { } int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size) { - STUB_RAND_IO_ERR(terrno) if (pFileOut == NULL || pFileIn == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; @@ -1154,7 +1148,6 @@ bool lastErrorIsFileNotExist() { return terrno == TAOS_SYSTEM_ERROR(ENOENT); } #endif // WINDOWS TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { - STUB_RAND_IO_ERR(NULL) FILE *fp = NULL; #ifdef WINDOWS HANDLE hFile = NULL; @@ -1252,12 +1245,6 @@ int32_t taosCloseFile(TdFilePtr *ppFile) { #endif taosMemoryFree(*ppFile); *ppFile = NULL; - -#if BUILD_WITH_RAND_ERR - if (terrno == 0) { - STUB_RAND_IO_ERR(terrno) - } -#endif return code; } @@ -1325,7 +1312,6 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) } int32_t taosFsyncFile(TdFilePtr pFile) { - STUB_RAND_IO_ERR(terrno) if (pFile == NULL) { return 0; } @@ -1390,7 +1376,6 @@ int32_t taosUmaskFile(int32_t maskVal) { } int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { - STUB_RAND_IO_ERR(terrno) int64_t ret = -1; int32_t code = 0; @@ -1465,7 +1450,6 @@ END: } int64_t taosGetsFile(TdFilePtr pFile, int32_t maxSize, char *__restrict buf) { - STUB_RAND_IO_ERR(terrno) if (pFile == NULL || buf == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; @@ -1616,7 +1600,6 @@ int32_t taosLinkFile(char *src, char *dst) { } FILE *taosOpenCFile(const char *filename, const char *mode) { - STUB_RAND_IO_ERR(NULL) FILE *f = fopen(filename, mode); if (NULL == f) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -1646,15 +1629,7 @@ size_t taosWriteToCFile(const void *ptr, size_t size, size_t nitems, FILE *strea return fwrite(ptr, size, nitems, stream); } -int taosCloseCFile(FILE *f) { - int32_t code = fclose(f); -#if BUILD_WITH_RAND_ERR - if (code == 0) { - STUB_RAND_IO_ERR(terrno) - } -#endif - return code == 0 ? code : TAOS_SYSTEM_ERROR(errno); -} +int taosCloseCFile(FILE *f) { return fclose(f); } int taosSetAutoDelFile(char *path) { #ifdef WINDOWS From abe579a50391eb4044666f24cddbd3b1a68215d7 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 13 Aug 2024 00:27:16 +0800 Subject: [PATCH 09/10] Update osFile.c --- source/os/src/osFile.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 8c1e8cfaf1..625dcc1f45 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -1148,6 +1148,7 @@ bool lastErrorIsFileNotExist() { return terrno == TAOS_SYSTEM_ERROR(ENOENT); } #endif // WINDOWS TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { + STUB_RAND_IO_ERR(NULL) FILE *fp = NULL; #ifdef WINDOWS HANDLE hFile = NULL; @@ -1600,6 +1601,7 @@ int32_t taosLinkFile(char *src, char *dst) { } FILE *taosOpenCFile(const char *filename, const char *mode) { + STUB_RAND_IO_ERR(NULL) FILE *f = fopen(filename, mode); if (NULL == f) { terrno = TAOS_SYSTEM_ERROR(errno); From ee05a536ef7291e1adb5c6199455f141d88e18aa Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 13 Aug 2024 14:19:28 +0800 Subject: [PATCH 10/10] enh: support config randErrorDivisor dynamically --- include/os/os.h | 3 ++- source/common/src/tglobal.c | 5 +++++ source/os/src/osFile.c | 16 ++++++++-------- source/os/src/osMemory.c | 14 +++++++------- 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/include/os/os.h b/include/os/os.h index e71c76bdb3..08b68f36d4 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -124,7 +124,8 @@ extern "C" { #include "taoserror.h" #include "tlog.h" -extern int32_t tsRandErrChance; +extern int32_t tsRandErrChance; +extern int64_t tsRandErrDivisor; extern threadlocal bool tsEnableRandErr; #ifdef __cplusplus diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index cd0ca95d3b..31a8c704ca 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -734,6 +734,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "randErrorChance", tsRandErrChance, 0, 10000, CFG_SCOPE_BOTH, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorDivisor", tsRandErrDivisor, 1, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); @@ -1427,6 +1428,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "randErrorChance"); tsRandErrChance = pItem->i32; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "randErrorDivisor"); + tsRandErrDivisor = pItem->i64; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "ttlUnit"); tsTtlUnit = pItem->i32; @@ -1881,6 +1885,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"mndSdbWriteDelta", &tsMndSdbWriteDelta}, {"minDiskFreeSize", &tsMinDiskFreeSize}, + {"randErrorDivisor", &tsRandErrDivisor}, {"cacheLazyLoadThreshold", &tsCacheLazyLoadThreshold}, {"checkpointInterval", &tsStreamCheckpointInterval}, diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 625dcc1f45..b8160a14b7 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -65,14 +65,14 @@ typedef struct TdFile { #define FILE_WITH_LOCK 1 #ifdef BUILD_WITH_RAND_ERR -#define STUB_RAND_IO_ERR(ret) \ - if (tsEnableRandErr) { \ - uint32_t r = taosRand() % 10001; \ - if ((r + 1) <= tsRandErrChance) { \ - errno = EIO; \ - terrno = TAOS_SYSTEM_ERROR(errno); \ - return (ret); \ - } \ +#define STUB_RAND_IO_ERR(ret) \ + if (tsEnableRandErr) { \ + uint32_t r = taosRand() % tsRandErrDivisor; \ + if ((r + 1) <= tsRandErrChance) { \ + errno = EIO; \ + terrno = TAOS_SYSTEM_ERROR(errno); \ + return (ret); \ + } \ } #else #define STUB_RAND_IO_ERR(ret) diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 6d29fdeccd..297b17b957 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -21,10 +21,10 @@ #endif #include "os.h" -int32_t tsRandErrChance = 1; +int32_t tsRandErrChance = 1; +int64_t tsRandErrDivisor = 10001; threadlocal bool tsEnableRandErr = 0; - #if defined(USE_TD_MEMORY) || defined(USE_ADDR2LINE) #define TD_MEMORY_SYMBOL ('T' << 24 | 'A' << 16 | 'O' << 8 | 'S') @@ -273,7 +273,7 @@ void *taosMemoryMalloc(int64_t size) { #ifdef BUILD_WITH_RAND_ERR if (tsEnableRandErr) { - uint32_t r = taosRand() % 10001; + uint32_t r = taosRand() % tsRandErrDivisor; if ((r + 1) <= tsRandErrChance) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -303,7 +303,7 @@ void *taosMemoryCalloc(int64_t num, int64_t size) { #else #ifdef BUILD_WITH_RAND_ERR if (tsEnableRandErr) { - uint32_t r = taosRand() % 10001; + uint32_t r = taosRand() % tsRandErrDivisor; if ((r + 1) <= tsRandErrChance) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -343,7 +343,7 @@ void *taosMemoryRealloc(void *ptr, int64_t size) { #else #ifdef BUILD_WITH_RAND_ERR if (tsEnableRandErr) { - uint32_t r = taosRand() % 10001; + uint32_t r = taosRand() % tsRandErrDivisor; if ((r + 1) <= tsRandErrChance) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -378,7 +378,7 @@ char *taosStrdup(const char *ptr) { #else #ifdef BUILD_WITH_RAND_ERR if (tsEnableRandErr) { - uint32_t r = taosRand() % 10001; + uint32_t r = taosRand() % tsRandErrDivisor; if ((r + 1) <= tsRandErrChance) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -444,7 +444,7 @@ void *taosMemoryMallocAlign(uint32_t alignment, int64_t size) { #if defined(LINUX) #ifdef BUILD_WITH_RAND_ERR if (tsEnableRandErr) { - uint32_t r = taosRand() % 10001; + uint32_t r = taosRand() % tsRandErrDivisor; if ((r + 1) <= tsRandErrChance) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL;