xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #27155 from taosdata/enh/TD-31375-3.0 

enh: simulate random i/o error
This commit is contained in:
Hongze Cheng 2024-08-13 18:31:35 +08:00 committed by GitHub
parent 6f014d1ca1 ee05a536ef
commit b5951a9efa
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 73 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/os/os.h
View File

@ -124,7 +124,8 @@ extern "C" {
#include "taoserror.h" #include "taoserror.h"
#include "tlog.h" #include "tlog.h"
extern int32_t tsRandErrChance; extern int32_t tsRandErrChance;
extern int64_t tsRandErrDivisor;
extern threadlocal bool tsEnableRandErr; extern threadlocal bool tsEnableRandErr;
#ifdef __cplusplus #ifdef __cplusplus

5
source/common/src/tglobal.c
View File

@ -775,6 +775,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "randErrorChance", tsRandErrChance, 0, 10000, CFG_SCOPE_BOTH, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "randErrorChance", tsRandErrChance, 0, 10000, CFG_SCOPE_BOTH, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorDivisor", tsRandErrDivisor, 1, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER));
@ -1468,6 +1469,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "randErrorChance"); TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "randErrorChance");
tsRandErrChance = pItem->i32; tsRandErrChance = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "randErrorDivisor");
tsRandErrDivisor = pItem->i64;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "ttlUnit"); TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "ttlUnit");
tsTtlUnit = pItem->i32; tsTtlUnit = pItem->i32;
@ -1922,6 +1926,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) {
{"mndSdbWriteDelta", &tsMndSdbWriteDelta}, {"mndSdbWriteDelta", &tsMndSdbWriteDelta},
{"minDiskFreeSize", &tsMinDiskFreeSize}, {"minDiskFreeSize", &tsMinDiskFreeSize},
{"randErrorDivisor", &tsRandErrDivisor},
{"cacheLazyLoadThreshold", &tsCacheLazyLoadThreshold}, {"cacheLazyLoadThreshold", &tsCacheLazyLoadThreshold},
{"checkpointInterval", &tsStreamCheckpointInterval}, {"checkpointInterval", &tsStreamCheckpointInterval},

97
source/os/src/osFile.c
View File

@ -64,6 +64,20 @@ typedef struct TdFile {
#define FILE_WITH_LOCK 1 #define FILE_WITH_LOCK 1
#ifdef BUILD_WITH_RAND_ERR
#define STUB_RAND_IO_ERR(ret) \
if (tsEnableRandErr) { \
uint32_t r = taosRand() % tsRandErrDivisor; \
if ((r + 1) <= tsRandErrChance) { \
errno = EIO; \
terrno = TAOS_SYSTEM_ERROR(errno); \
return (ret); \
} \
}
#else
#define STUB_RAND_IO_ERR(ret)
#endif
void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath) { void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath) {
#ifdef WINDOWS #ifdef WINDOWS
@ -145,7 +159,7 @@ int64_t taosCopyFile(const char *from, const char *to) {
code = terrno; code = terrno;
goto _err; goto _err;
} }
if (bytes == 0) break; if (bytes == 0) break;
size += bytes; size += bytes;
@ -166,7 +180,7 @@ int64_t taosCopyFile(const char *from, const char *to) {
terrno = code; terrno = code;
return -1; return -1;
} }
return size; return size;
_err: _err:
@ -197,12 +211,12 @@ TdFilePtr taosCreateFile(const char *path, int32_t tdFileOptions) {
} }
} }
} }
return fp; return fp;
} }
int32_t taosRemoveFile(const char *path) { int32_t taosRemoveFile(const char *path) {
int32_t code = remove(path); int32_t code = remove(path);
if (-1 == code) { if (-1 == code) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return terrno; return terrno;
@ -300,7 +314,7 @@ int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
return terrno; return terrno;
} }
struct stat fileStat; struct stat fileStat;
int32_t code = fstat(pFile->fd, &fileStat); int32_t code = fstat(pFile->fd, &fileStat);
if (-1 == code) { if (-1 == code) {
@ -335,7 +349,7 @@ FILE *taosOpenFileForStream(const char *path, int32_t tdFileOptions) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
return NULL; return NULL;
} }
FILE* f = fopen(path, mode); FILE *f = fopen(path, mode);
if (NULL == f) { if (NULL == f) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
} }
@ -694,6 +708,7 @@ int taosOpenFileNotStream(const char *path, int32_t tdFileOptions) {
} }
int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) {
STUB_RAND_IO_ERR(terrno)
#if FILE_WITH_LOCK #if FILE_WITH_LOCK
(void)taosThreadRwlockRdlock(&(pFile->rwlock)); (void)taosThreadRwlockRdlock(&(pFile->rwlock));
#endif #endif
@ -705,7 +720,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
return terrno; return terrno;
} }
int64_t leftbytes = count; int64_t leftbytes = count;
int64_t readbytes; int64_t readbytes;
char *tbuf = (char *)buf; char *tbuf = (char *)buf;
@ -747,6 +762,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) {
} }
int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) {
STUB_RAND_IO_ERR(terrno)
if (pFile == NULL) { if (pFile == NULL) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
return 0; return 0;
@ -792,6 +808,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) {
} }
int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t offset) { int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t offset) {
STUB_RAND_IO_ERR(terrno)
if (pFile == NULL) { if (pFile == NULL) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
return 0; return 0;
@ -850,7 +867,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) {
int32_t code = 0; int32_t code = 0;
ASSERT(pFile->fd >= 0); // Please check if you have closed the file. ASSERT(pFile->fd >= 0); // Please check if you have closed the file.
#ifdef WINDOWS #ifdef WINDOWS
int64_t ret = _lseeki64(pFile->fd, offset, whence); int64_t ret = _lseeki64(pFile->fd, offset, whence);
#else #else
@ -888,7 +905,7 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) {
int32_t code = _fstat64(pFile->fd, &fileStat); int32_t code = _fstat64(pFile->fd, &fileStat);
#else #else
struct stat fileStat; struct stat fileStat;
int32_t code = fstat(pFile->fd, &fileStat); int32_t code = fstat(pFile->fd, &fileStat);
#endif #endif
if (-1 == code) { if (-1 == code) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
@ -944,7 +961,7 @@ int32_t taosUnLockFile(TdFilePtr pFile) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
return terrno; return terrno;
} }
#ifdef WINDOWS #ifdef WINDOWS
BOOL fSuccess = FALSE; BOOL fSuccess = FALSE;
OVERLAPPED overlapped = {0}; OVERLAPPED overlapped = {0};
@ -970,7 +987,7 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
return terrno; return terrno;
} }
#ifdef WINDOWS #ifdef WINDOWS
HANDLE h = (HANDLE)_get_osfhandle(pFile->fd); HANDLE h = (HANDLE)_get_osfhandle(pFile->fd);
@ -1030,9 +1047,9 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
return terrno; return terrno;
} }
if (pFileIn->fd < 0 || pFileOut->fd < 0) { if (pFileIn->fd < 0 || pFileOut->fd < 0) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
return terrno; return terrno;
} }
#ifdef WINDOWS #ifdef WINDOWS
@ -1131,6 +1148,7 @@ bool lastErrorIsFileNotExist() { return terrno == TAOS_SYSTEM_ERROR(ENOENT); }
#endif // WINDOWS #endif // WINDOWS
TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) {
STUB_RAND_IO_ERR(NULL)
FILE *fp = NULL; FILE *fp = NULL;
#ifdef WINDOWS #ifdef WINDOWS
HANDLE hFile = NULL; HANDLE hFile = NULL;
@ -1228,16 +1246,16 @@ int32_t taosCloseFile(TdFilePtr *ppFile) {
#endif #endif
taosMemoryFree(*ppFile); taosMemoryFree(*ppFile);
*ppFile = NULL; *ppFile = NULL;
return code; return code;
} }
int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) { int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) {
STUB_RAND_IO_ERR(terrno)
if (pFile == NULL) { if (pFile == NULL) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
return terrno; return terrno;
} }
int32_t code = 0; int32_t code = 0;
#ifdef WINDOWS #ifdef WINDOWS
@ -1249,11 +1267,11 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset)
#if FILE_WITH_LOCK #if FILE_WITH_LOCK
(void)taosThreadRwlockUnlock(&(pFile->rwlock)); (void)taosThreadRwlockUnlock(&(pFile->rwlock));
#endif #endif
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
return terrno; return terrno;
} }
DWORD ret = 0; DWORD ret = 0;
OVERLAPPED ol = {0}; OVERLAPPED ol = {0};
ol.OffsetHigh = (uint32_t)((offset & 0xFFFFFFFF00000000LL) >> 0x20); ol.OffsetHigh = (uint32_t)((offset & 0xFFFFFFFF00000000LL) >> 0x20);
@ -1290,7 +1308,7 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset)
terrno = code; terrno = code;
return code; return code;
} }
return ret; return ret;
} }
@ -1308,10 +1326,10 @@ int32_t taosFsyncFile(TdFilePtr pFile) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return terrno; return terrno;
} }
return 0; return 0;
} }
#ifdef WINDOWS #ifdef WINDOWS
if (pFile->hFile != NULL) { if (pFile->hFile != NULL) {
if (pFile->tdFileOptions & TD_FILE_WRITE_THROUGH) { if (pFile->tdFileOptions & TD_FILE_WRITE_THROUGH) {
@ -1361,7 +1379,7 @@ int32_t taosUmaskFile(int32_t maskVal) {
int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) {
int64_t ret = -1; int64_t ret = -1;
int32_t code = 0; int32_t code = 0;
#if FILE_WITH_LOCK #if FILE_WITH_LOCK
(void)taosThreadRwlockRdlock(&(pFile->rwlock)); (void)taosThreadRwlockRdlock(&(pFile->rwlock));
#endif #endif
@ -1377,7 +1395,7 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
goto END; goto END;
} }
#ifdef WINDOWS #ifdef WINDOWS
size_t bufferSize = 512; size_t bufferSize = 512;
*ptrBuf = taosMemoryMalloc(bufferSize); *ptrBuf = taosMemoryMalloc(bufferSize);
@ -1437,12 +1455,12 @@ int64_t taosGetsFile(TdFilePtr pFile, int32_t maxSize, char *__restrict buf) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
return terrno; return terrno;
} }
if (pFile->fp == NULL) { if (pFile->fp == NULL) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
return terrno; return terrno;
} }
if (fgets(buf, maxSize, pFile->fp) == NULL) { if (fgets(buf, maxSize, pFile->fp) == NULL) {
if (feof(pFile->fp)) { if (feof(pFile->fp)) {
return 0; return 0;
@ -1451,7 +1469,7 @@ int64_t taosGetsFile(TdFilePtr pFile, int32_t maxSize, char *__restrict buf) {
return terrno; return terrno;
} }
} }
return strlen(buf); return strlen(buf);
} }
@ -1493,17 +1511,17 @@ bool taosCheckAccessFile(const char *pathname, int32_t tdFileAccessOptions) {
bool taosCheckExistFile(const char *pathname) { return taosCheckAccessFile(pathname, TD_FILE_ACCESS_EXIST_OK); }; bool taosCheckExistFile(const char *pathname) { return taosCheckAccessFile(pathname, TD_FILE_ACCESS_EXIST_OK); };
int32_t taosCompressFile(char *srcFileName, char *destFileName) { int32_t taosCompressFile(char *srcFileName, char *destFileName) {
int32_t compressSize = 163840; int32_t compressSize = 163840;
int32_t ret = 0; int32_t ret = 0;
int32_t len = 0; int32_t len = 0;
gzFile dstFp = NULL; gzFile dstFp = NULL;
TdFilePtr pSrcFile = NULL; TdFilePtr pSrcFile = NULL;
char *data = taosMemoryMalloc(compressSize); char *data = taosMemoryMalloc(compressSize);
if (NULL == data) { if (NULL == data) {
return terrno; return terrno;
} }
pSrcFile = taosOpenFile(srcFileName, TD_FILE_READ | TD_FILE_STREAM); pSrcFile = taosOpenFile(srcFileName, TD_FILE_READ | TD_FILE_STREAM);
if (pSrcFile == NULL) { if (pSrcFile == NULL) {
ret = terrno; ret = terrno;
@ -1582,12 +1600,13 @@ int32_t taosLinkFile(char *src, char *dst) {
return 0; return 0;
} }
FILE *taosOpenCFile(const char *filename, const char *mode) { FILE *taosOpenCFile(const char *filename, const char *mode) {
FILE* f = fopen(filename, mode); STUB_RAND_IO_ERR(NULL)
FILE *f = fopen(filename, mode);
if (NULL == f) { if (NULL == f) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
} }
return f; return f;
} }
int taosSeekCFile(FILE *file, int64_t offset, int whence) { int taosSeekCFile(FILE *file, int64_t offset, int whence) {
@ -1603,10 +1622,12 @@ int taosSeekCFile(FILE *file, int64_t offset, int whence) {
} }
size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream) { size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream) {
STUB_RAND_IO_ERR(terrno)
return fread(buffer, size, count, stream); return fread(buffer, size, count, stream);
} }
size_t taosWriteToCFile(const void *ptr, size_t size, size_t nitems, FILE *stream) { size_t taosWriteToCFile(const void *ptr, size_t size, size_t nitems, FILE *stream) {
STUB_RAND_IO_ERR(terrno)
return fwrite(ptr, size, nitems, stream); return fwrite(ptr, size, nitems, stream);
} }
@ -1620,6 +1641,6 @@ int taosSetAutoDelFile(char *path) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return terrno; return terrno;
} }
return 0; return 0;
#endif #endif
} }

14
source/os/src/osMemory.c
View File

@ -21,10 +21,10 @@
#endif #endif
#include "os.h" #include "os.h"
int32_t tsRandErrChance = 1; int32_t tsRandErrChance = 1;
int64_t tsRandErrDivisor = 10001;
threadlocal bool tsEnableRandErr = 0; threadlocal bool tsEnableRandErr = 0;
#if defined(USE_TD_MEMORY) || defined(USE_ADDR2LINE) #if defined(USE_TD_MEMORY) || defined(USE_ADDR2LINE)
#define TD_MEMORY_SYMBOL ('T' << 24 | 'A' << 16 | 'O' << 8 | 'S') #define TD_MEMORY_SYMBOL ('T' << 24 | 'A' << 16 | 'O' << 8 | 'S')
@ -273,7 +273,7 @@ void *taosMemoryMalloc(int64_t size) {
#ifdef BUILD_WITH_RAND_ERR #ifdef BUILD_WITH_RAND_ERR
if (tsEnableRandErr) { if (tsEnableRandErr) {
uint32_t r = taosRand() % 10001; uint32_t r = taosRand() % tsRandErrDivisor;
if ((r + 1) <= tsRandErrChance) { if ((r + 1) <= tsRandErrChance) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
@ -303,7 +303,7 @@ void *taosMemoryCalloc(int64_t num, int64_t size) {
#else #else
#ifdef BUILD_WITH_RAND_ERR #ifdef BUILD_WITH_RAND_ERR
if (tsEnableRandErr) { if (tsEnableRandErr) {
uint32_t r = taosRand() % 10001; uint32_t r = taosRand() % tsRandErrDivisor;
if ((r + 1) <= tsRandErrChance) { if ((r + 1) <= tsRandErrChance) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
@ -343,7 +343,7 @@ void *taosMemoryRealloc(void *ptr, int64_t size) {
#else #else
#ifdef BUILD_WITH_RAND_ERR #ifdef BUILD_WITH_RAND_ERR
if (tsEnableRandErr) { if (tsEnableRandErr) {
uint32_t r = taosRand() % 10001; uint32_t r = taosRand() % tsRandErrDivisor;
if ((r + 1) <= tsRandErrChance) { if ((r + 1) <= tsRandErrChance) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
@ -378,7 +378,7 @@ char *taosStrdup(const char *ptr) {
#else #else
#ifdef BUILD_WITH_RAND_ERR #ifdef BUILD_WITH_RAND_ERR
if (tsEnableRandErr) { if (tsEnableRandErr) {
uint32_t r = taosRand() % 10001; uint32_t r = taosRand() % tsRandErrDivisor;
if ((r + 1) <= tsRandErrChance) { if ((r + 1) <= tsRandErrChance) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
@ -444,7 +444,7 @@ void *taosMemoryMallocAlign(uint32_t alignment, int64_t size) {
#if defined(LINUX) #if defined(LINUX)
#ifdef BUILD_WITH_RAND_ERR #ifdef BUILD_WITH_RAND_ERR
if (tsEnableRandErr) { if (tsEnableRandErr) {
uint32_t r = taosRand() % 10001; uint32_t r = taosRand() % tsRandErrDivisor;
if ((r + 1) <= tsRandErrChance) { if ((r + 1) <= tsRandErrChance) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;