From 5b982354e1ecf0cb2510e752f291f27035d684f7 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Fri, 12 Jul 2024 15:30:03 +0800 Subject: [PATCH 01/48] adj bloomfilter --- include/libs/executor/storageapi.h | 2 +- include/libs/stream/tstreamUpdate.h | 2 +- include/util/tbloomfilter.h | 20 ++--- include/util/tscalablebf.h | 2 +- include/util/tutil.h | 1 + source/libs/stream/src/streamUpdate.c | 29 +++++-- source/util/src/tbloomfilter.c | 69 ++++++++++++---- source/util/src/tscalablebf.c | 108 +++++++++++++++++++------- source/util/test/bloomFilterTest.cpp | 22 ++++-- 9 files changed, 186 insertions(+), 69 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 45f9f73fb1..45424cfb66 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -389,7 +389,7 @@ typedef struct SStateStore { void (*updateInfoDestroy)(SUpdateInfo* pInfo); void (*windowSBfDelete)(SUpdateInfo* pInfo, uint64_t count); - void (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count); + int32_t (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count); SUpdateInfo* (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo); diff --git a/include/libs/stream/tstreamUpdate.h b/include/libs/stream/tstreamUpdate.h index 632a82743f..726f8443f1 100644 --- a/include/libs/stream/tstreamUpdate.h +++ b/include/libs/stream/tstreamUpdate.h @@ -36,7 +36,7 @@ void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo); int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo); int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo); void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count); -void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count); +int32_t windowSBfAdd(SUpdateInfo *pInfo, uint64_t count); bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); #ifdef __cplusplus diff --git a/include/util/tbloomfilter.h b/include/util/tbloomfilter.h index 23bf5aaafb..2a3da90dad 100644 --- a/include/util/tbloomfilter.h +++ b/include/util/tbloomfilter.h @@ -19,6 +19,8 @@ #include "os.h" #include "tencode.h" #include "thash.h" +#include "tlog.h" +#include "tutil.h" #ifdef __cplusplus extern "C" { @@ -39,15 +41,15 @@ typedef struct SBloomFilter { double errorRate; } SBloomFilter; -SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate); -int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2); -int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len); -int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t h1, uint64_t h2); -void tBloomFilterDestroy(SBloomFilter *pBF); -void tBloomFilterDump(const SBloomFilter *pBF); -bool tBloomFilterIsFull(const SBloomFilter *pBF); -int32_t tBloomFilterEncode(const SBloomFilter *pBF, SEncoder *pEncoder); -SBloomFilter *tBloomFilterDecode(SDecoder *pDecoder); +int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilter **ppBF); +int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2); +int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len); +int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t h1, uint64_t h2); +void tBloomFilterDestroy(SBloomFilter *pBF); +void tBloomFilterDump(const SBloomFilter *pBF); +bool tBloomFilterIsFull(const SBloomFilter *pBF); +int32_t tBloomFilterEncode(const SBloomFilter *pBF, SEncoder *pEncoder); +int32_t tBloomFilterDecode(SDecoder *pDecoder, SBloomFilter **ppBF); #ifdef __cplusplus } diff --git a/include/util/tscalablebf.h b/include/util/tscalablebf.h index d3ce2eb23b..5e1efb6b25 100644 --- a/include/util/tscalablebf.h +++ b/include/util/tscalablebf.h @@ -32,7 +32,7 @@ typedef struct SScalableBf { _hash_fn_t hashFn2; } SScalableBf; -SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate); +int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf **ppSBf); int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len); int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len); int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len); diff --git a/include/util/tutil.h b/include/util/tutil.h index d1a18dc3e8..33a53f79b5 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -21,6 +21,7 @@ #include "tdef.h" #include "thash.h" #include "tmd5.h" +#include "tutil.h" #ifdef __cplusplus extern "C" { diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 0adb7050c6..e41c5cfffc 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -66,15 +66,29 @@ int32_t getValueBuff(TSKEY ts, char* pVal, int32_t len, char* buff) { return sizeof(TSKEY) + len; } -void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) { +int32_t windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (pInfo->numSBFs < count) { count = pInfo->numSBFs; } for (uint64_t i = 0; i < count; ++i) { int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND); - SScalableBf *tsSBF = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE); - taosArrayPush(pInfo->pTsSBFs, &tsSBF); + SScalableBf *tsSBF = NULL; + code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &tsSBF); + TSDB_CHECK_CODE(code, lino, _error); + void *res = taosArrayPush(pInfo->pTsSBFs, &tsSBF); + if (!res) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } } + +_error: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void clearItemHelper(void *p) { @@ -183,6 +197,7 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma } static SScalableBf *getSBf(SUpdateInfo *pInfo, TSKEY ts) { + int32_t code = 0; if (ts <= 0) { return NULL; } @@ -202,7 +217,7 @@ static SScalableBf *getSBf(SUpdateInfo *pInfo, TSKEY ts) { SScalableBf *res = taosArrayGetP(pInfo->pTsSBFs, index); if (res == NULL) { int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND); - res = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE); + code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &res); taosArrayPush(pInfo->pTsSBFs, &res); } return res; @@ -334,7 +349,11 @@ void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo) { return; } int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND); - pInfo->pCloseWinSBF = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE); + int32_t code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &pInfo->pCloseWinSBF); + if (code != TSDB_CODE_SUCCESS) { + pInfo->pCloseWinSBF = NULL; + uError("%s failed to add close window SBF since %s", __func__, tstrerror(code)); + } } void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo) { diff --git a/source/util/src/tbloomfilter.c b/source/util/src/tbloomfilter.c index 150faab571..557e12a5b2 100644 --- a/source/util/src/tbloomfilter.c +++ b/source/util/src/tbloomfilter.c @@ -35,13 +35,17 @@ static FORCE_INLINE bool getBit(uint64_t *buf, uint64_t index) { return buf[unitIndex] & mask; } -SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate) { +int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilter **ppBF) { + int32_t code = 0; + int32_t lino = 0; if (expectedEntries < 1 || errorRate <= 0 || errorRate >= 1.0) { - return NULL; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } SBloomFilter *pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); if (pBF == NULL) { - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } pBF->expectedEntries = expectedEntries; pBF->errorRate = errorRate; @@ -61,9 +65,16 @@ SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate) { pBF->buffer = taosMemoryCalloc(pBF->numUnits, sizeof(uint64_t)); if (pBF->buffer == NULL) { tBloomFilterDestroy(pBF); - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } - return pBF; + (*ppBF) = pBF; + +_error: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2) { @@ -133,27 +144,55 @@ int32_t tBloomFilterEncode(const SBloomFilter *pBF, SEncoder *pEncoder) { return 0; } -SBloomFilter *tBloomFilterDecode(SDecoder *pDecoder) { +int32_t tBloomFilterDecode(SDecoder *pDecoder, SBloomFilter **ppBF) { + int32_t code = 0; + int32_t lino = 0; SBloomFilter *pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); + if (!pBF) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } pBF->buffer = NULL; - if (tDecodeU32(pDecoder, &pBF->hashFunctions) < 0) goto _error; - if (tDecodeU64(pDecoder, &pBF->expectedEntries) < 0) goto _error; - if (tDecodeU64(pDecoder, &pBF->numUnits) < 0) goto _error; - if (tDecodeU64(pDecoder, &pBF->numBits) < 0) goto _error; - if (tDecodeU64(pDecoder, &pBF->size) < 0) goto _error; + if (tDecodeU32(pDecoder, &pBF->hashFunctions) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeU64(pDecoder, &pBF->expectedEntries) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeU64(pDecoder, &pBF->numUnits) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeU64(pDecoder, &pBF->numBits) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeU64(pDecoder, &pBF->size) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } pBF->buffer = taosMemoryCalloc(pBF->numUnits, sizeof(uint64_t)); for (int32_t i = 0; i < pBF->numUnits; i++) { uint64_t *pUnits = (uint64_t *)pBF->buffer; - if (tDecodeU64(pDecoder, pUnits + i) < 0) goto _error; + if (tDecodeU64(pDecoder, pUnits + i) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + } + if (tDecodeDouble(pDecoder, &pBF->errorRate) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } - if (tDecodeDouble(pDecoder, &pBF->errorRate) < 0) goto _error; pBF->hashFn1 = HASH_FUNCTION_1; pBF->hashFn2 = HASH_FUNCTION_2; - return pBF; + (*ppBF) = pBF; _error: tBloomFilterDestroy(pBF); - return NULL; + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + return TSDB_CODE_FAILED; } bool tBloomFilterIsFull(const SBloomFilter *pBF) { return pBF->size >= pBF->expectedEntries; } diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index 7af794546b..b996cf21bb 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -17,6 +17,7 @@ #include "tscalablebf.h" #include "taoserror.h" +#include "tutil.h" #define DEFAULT_GROWTH 2 #define DEFAULT_TIGHTENING_RATIO 0.5 @@ -24,52 +25,82 @@ #define SBF_INVALID -1 #define SBF_VALID 0 -static SBloomFilter *tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate); +static int32_t tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate, SBloomFilter **ppNormalBf); -SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate) { +int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf **ppSBf) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; const uint32_t defaultSize = 8; if (expectedEntries < 1 || errorRate <= 0 || errorRate >= 1.0) { - return NULL; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); if (pSBf == NULL) { - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } pSBf->maxBloomFilters = DEFAULT_MAX_BLOOMFILTERS; pSBf->status = SBF_VALID; pSBf->numBits = 0; pSBf->bfArray = taosArrayInit(defaultSize, sizeof(void *)); - if (tScalableBfAddFilter(pSBf, expectedEntries, errorRate * DEFAULT_TIGHTENING_RATIO) == NULL) { + if (!pSBf->bfArray) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } + + SBloomFilter *pNormalBf = NULL; + code = tScalableBfAddFilter(pSBf, expectedEntries, errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf); + if (code != TSDB_CODE_SUCCESS) { tScalableBfDestroy(pSBf); - return NULL; + TSDB_CHECK_CODE(code, lino, _error); } pSBf->growth = DEFAULT_GROWTH; pSBf->hashFn1 = HASH_FUNCTION_1; pSBf->hashFn2 = HASH_FUNCTION_2; - return pSBf; + (*ppSBf) = pSBf; + return TSDB_CODE_SUCCESS; + +_error: + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + return code; } int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (pSBf->status == SBF_INVALID) { - return TSDB_CODE_FAILED; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } - int32_t size = taosArrayGetSize(pSBf->bfArray); + int32_t size = taosArrayGetSize(pSBf->bfArray); SBloomFilter *pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); - ASSERT(pNormalBf); + if (!pNormalBf) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } if (tBloomFilterIsFull(pNormalBf)) { - pNormalBf = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, - pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO); - if (pNormalBf == NULL) { + code = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, + pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf); + if (code != TSDB_CODE_SUCCESS) { pSBf->status = SBF_INVALID; - return TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } } return tBloomFilterPut(pNormalBf, keyBuf, len); + +_error: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (pSBf->status == SBF_INVALID) { - return TSDB_CODE_FAILED; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len); uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len); @@ -83,14 +114,20 @@ int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { SBloomFilter *pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); ASSERT(pNormalBf); if (tBloomFilterIsFull(pNormalBf)) { - pNormalBf = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, - pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO); - if (pNormalBf == NULL) { + code = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, + pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf); + if (code != TSDB_CODE_SUCCESS) { pSBf->status = SBF_INVALID; - return TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } } return tBloomFilterPutHash(pNormalBf, h1, h2); + +_error: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len) { @@ -108,21 +145,31 @@ int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32 return TSDB_CODE_SUCCESS; } -static SBloomFilter *tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate) { +static int32_t tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate, SBloomFilter **ppNormalBf) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (taosArrayGetSize(pSBf->bfArray) >= pSBf->maxBloomFilters) { - return NULL; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } - SBloomFilter *pNormalBf = tBloomFilterInit(expectedEntries, errorRate); - if (pNormalBf == NULL) { - return NULL; - } + SBloomFilter *pNormalBf = NULL; + code = tBloomFilterInit(expectedEntries, errorRate, &pNormalBf); + TSDB_CHECK_CODE(code, lino, _error); + if (taosArrayPush(pSBf->bfArray, &pNormalBf) == NULL) { tBloomFilterDestroy(pNormalBf); - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } pSBf->numBits += pNormalBf->numBits; - return pNormalBf; + (*ppNormalBf) = pNormalBf; + +_error: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void tScalableBfDestroy(SScalableBf *pSBf) { @@ -154,6 +201,8 @@ int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) { } SScalableBf *tScalableBfDecode(SDecoder *pDecoder) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); pSBf->hashFn1 = HASH_FUNCTION_1; pSBf->hashFn2 = HASH_FUNCTION_2; @@ -166,8 +215,9 @@ SScalableBf *tScalableBfDecode(SDecoder *pDecoder) { } pSBf->bfArray = taosArrayInit(size * 2, sizeof(void *)); for (int32_t i = 0; i < size; i++) { - SBloomFilter *pBF = tBloomFilterDecode(pDecoder); - if (!pBF) goto _error; + SBloomFilter *pBF = NULL; + code = tBloomFilterDecode(pDecoder, &pBF); + TSDB_CHECK_CODE(code, lino, _error); taosArrayPush(pSBf->bfArray, &pBF); } if (tDecodeU32(pDecoder, &pSBf->growth) < 0) goto _error; diff --git a/source/util/test/bloomFilterTest.cpp b/source/util/test/bloomFilterTest.cpp index c51de3c8a4..7832527bf1 100644 --- a/source/util/test/bloomFilterTest.cpp +++ b/source/util/test/bloomFilterTest.cpp @@ -8,12 +8,15 @@ using namespace std; TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { int64_t ts1 = 1650803518000; - GTEST_ASSERT_EQ(NULL, tBloomFilterInit(100, 0)); - GTEST_ASSERT_EQ(NULL, tBloomFilterInit(100, 1)); - GTEST_ASSERT_EQ(NULL, tBloomFilterInit(100, -0.1)); - GTEST_ASSERT_EQ(NULL, tBloomFilterInit(0, 0.01)); + SBloomFilter *pBFTmp = NULL; + GTEST_ASSERT_EQ(0, tBloomFilterInit(100, 0, &pBFTmp)); + GTEST_ASSERT_EQ(0, tBloomFilterInit(100, 1, &pBFTmp)); + GTEST_ASSERT_EQ(0, tBloomFilterInit(100, -0.1, &pBFTmp)); + GTEST_ASSERT_EQ(0, tBloomFilterInit(0, 0.01, &pBFTmp)); - SBloomFilter *pBF1 = tBloomFilterInit(100, 0.005); + SBloomFilter *pBF1 = NULL, + int32_t code = tBloomFilterInit(100, 0.005, &pBF1); + GTEST_ASSERT_EQ(0,code); GTEST_ASSERT_EQ(pBF1->numBits, 1152); GTEST_ASSERT_EQ(pBF1->numUnits, 1152 / 64); int64_t count = 0; @@ -25,16 +28,19 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { } ASSERT_TRUE(tBloomFilterIsFull(pBF1)); - SBloomFilter *pBF2 = tBloomFilterInit(1000 * 10000, 0.1); + SBloomFilter *pBF2 = NULL; + GTEST_ASSERT_EQ(0, tBloomFilterInit(1000 * 10000, 0.1, &pBF2)); GTEST_ASSERT_EQ(pBF2->numBits, 47925312); GTEST_ASSERT_EQ(pBF2->numUnits, 47925312 / 64); - SBloomFilter *pBF3 = tBloomFilterInit(10000 * 10000, 0.001); + SBloomFilter *pBF3 = NULL; + GTEST_ASSERT_EQ(0, tBloomFilterInit(10000 * 10000, 0.001)); GTEST_ASSERT_EQ(pBF3->numBits, 1437758784); GTEST_ASSERT_EQ(pBF3->numUnits, 1437758784 / 64); int64_t size = 10000; - SBloomFilter *pBF4 = tBloomFilterInit(size, 0.001); + SBloomFilter *pBF4 = NULL; + GTEST_ASSERT_EQ(0, tBloomFilterInit(size, 0.001)); for (int64_t i = 0; i < 1000; i++) { int64_t ts = i + ts1; GTEST_ASSERT_EQ(tBloomFilterPut(pBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS); From 3b914248a52a7a0b2c862a0f2464374ef7b9b789 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Fri, 12 Jul 2024 15:52:54 +0800 Subject: [PATCH 02/48] adj bloomfilter --- include/util/tscalablebf.h | 14 ++--- source/libs/stream/src/streamUpdate.c | 19 +++++-- source/util/src/tscalablebf.c | 75 +++++++++++++++++++-------- 3 files changed, 74 insertions(+), 34 deletions(-) diff --git a/include/util/tscalablebf.h b/include/util/tscalablebf.h index 5e1efb6b25..584e023bb0 100644 --- a/include/util/tscalablebf.h +++ b/include/util/tscalablebf.h @@ -32,13 +32,13 @@ typedef struct SScalableBf { _hash_fn_t hashFn2; } SScalableBf; -int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf **ppSBf); -int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len); -int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len); -int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len); -void tScalableBfDestroy(SScalableBf *pSBf); -int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder); -SScalableBf *tScalableBfDecode(SDecoder *pDecoder); +int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf **ppSBf); +int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len); +int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len); +int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len); +void tScalableBfDestroy(SScalableBf *pSBf); +int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder); +int32_t tScalableBfDecode(SDecoder *pDecoder, SScalableBf **ppSBf); #ifdef __cplusplus } diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index e41c5cfffc..2da20cadae 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -420,6 +420,8 @@ int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo) } int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; ASSERT(pInfo); SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -440,8 +442,10 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { if (tDecodeI32(&decoder, &sBfSize) < 0) return -1; pInfo->pTsSBFs = taosArrayInit(sBfSize, sizeof(void *)); for (int32_t i = 0; i < sBfSize; i++) { - SScalableBf *pSBf = tScalableBfDecode(&decoder); - if (!pSBf) return -1; + SScalableBf *pSBf = NULL; + code = tScalableBfDecode(&decoder, &pSBf); + TSDB_CHECK_CODE(code, lino, _error); + taosArrayPush(pInfo->pTsSBFs, &pSBf); } @@ -449,7 +453,9 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { if (tDecodeI64(&decoder, &pInfo->interval) < 0) return -1; if (tDecodeI64(&decoder, &pInfo->watermark) < 0) return -1; if (tDecodeI64(&decoder, &pInfo->minTS) < 0) return -1; - pInfo->pCloseWinSBF = tScalableBfDecode(&decoder); + pInfo->pCloseWinSBF = NULL; + code = tScalableBfDecode(&decoder, &pInfo->pCloseWinSBF); + TSDB_CHECK_CODE(code, lino, _error); int32_t mapSize = 0; if (tDecodeI32(&decoder, &mapSize) < 0) return -1; @@ -482,7 +488,12 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { tEndDecode(&decoder); tDecoderClear(&decoder); - return 0; + +_error: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) { diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index b996cf21bb..1ee99b07f7 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -25,11 +25,12 @@ #define SBF_INVALID -1 #define SBF_VALID 0 -static int32_t tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate, SBloomFilter **ppNormalBf); +static int32_t tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate, + SBloomFilter **ppNormalBf); int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf **ppSBf) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t lino = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; const uint32_t defaultSize = 8; if (expectedEntries < 1 || errorRate <= 0 || errorRate >= 1.0) { code = TSDB_CODE_FAILED; @@ -47,7 +48,7 @@ int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _error); } - + SBloomFilter *pNormalBf = NULL; code = tScalableBfAddFilter(pSBf, expectedEntries, errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf); if (code != TSDB_CODE_SUCCESS) { @@ -104,7 +105,7 @@ int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { } uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len); uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len); - int32_t size = taosArrayGetSize(pSBf->bfArray); + int32_t size = taosArrayGetSize(pSBf->bfArray); for (int32_t i = size - 2; i >= 0; --i) { if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i), h1, h2) != TSDB_CODE_SUCCESS) { return TSDB_CODE_FAILED; @@ -115,7 +116,7 @@ int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { ASSERT(pNormalBf); if (tBloomFilterIsFull(pNormalBf)) { code = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, - pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf); + pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf); if (code != TSDB_CODE_SUCCESS) { pSBf->status = SBF_INVALID; TSDB_CHECK_CODE(code, lino, _error); @@ -136,7 +137,7 @@ int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32 } uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len); uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len); - int32_t size = taosArrayGetSize(pSBf->bfArray); + int32_t size = taosArrayGetSize(pSBf->bfArray); for (int32_t i = size - 1; i >= 0; --i) { if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i), h1, h2) != TSDB_CODE_SUCCESS) { return TSDB_CODE_FAILED; @@ -145,7 +146,8 @@ int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32 return TSDB_CODE_SUCCESS; } -static int32_t tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate, SBloomFilter **ppNormalBf) { +static int32_t tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate, + SBloomFilter **ppNormalBf) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; if (taosArrayGetSize(pSBf->bfArray) >= pSBf->maxBloomFilters) { @@ -163,7 +165,7 @@ static int32_t tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, TSDB_CHECK_CODE(code, lino, _error); } pSBf->numBits += pNormalBf->numBits; - (*ppNormalBf) = pNormalBf; + (*ppNormalBf) = pNormalBf; _error: if (code != TSDB_CODE_SUCCESS) { @@ -200,33 +202,60 @@ int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) { return 0; } -SScalableBf *tScalableBfDecode(SDecoder *pDecoder) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t lino = 0; +int32_t tScalableBfDecode(SDecoder *pDecoder, SScalableBf **ppSBf) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); + if (!pSBf) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } pSBf->hashFn1 = HASH_FUNCTION_1; pSBf->hashFn2 = HASH_FUNCTION_2; pSBf->bfArray = NULL; int32_t size = 0; - if (tDecodeI32(pDecoder, &size) < 0) goto _error; - if (size == 0) { - tScalableBfDestroy(pSBf); - return NULL; + if (tDecodeI32(pDecoder, &size) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } - pSBf->bfArray = taosArrayInit(size * 2, sizeof(void *)); + if (size == 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + pSBf->bfArray = taosArrayInit(size * 2, POINTER_BYTES); + if (!pSBf->bfArray) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } + for (int32_t i = 0; i < size; i++) { SBloomFilter *pBF = NULL; code = tBloomFilterDecode(pDecoder, &pBF); TSDB_CHECK_CODE(code, lino, _error); taosArrayPush(pSBf->bfArray, &pBF); } - if (tDecodeU32(pDecoder, &pSBf->growth) < 0) goto _error; - if (tDecodeU64(pDecoder, &pSBf->numBits) < 0) goto _error; - if (tDecodeU32(pDecoder, &pSBf->maxBloomFilters) < 0) goto _error; - if (tDecodeI8(pDecoder, &pSBf->status) < 0) goto _error; - return pSBf; + if (tDecodeU32(pDecoder, &pSBf->growth) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeU64(pDecoder, &pSBf->numBits) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeU32(pDecoder, &pSBf->maxBloomFilters) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeI8(pDecoder, &pSBf->status) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + (*ppSBf) = pSBf; _error: tScalableBfDestroy(pSBf); - return NULL; + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } From 89a62c526e682db508dba53313fdb221af4e4f17 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 15 Jul 2024 09:50:12 +0800 Subject: [PATCH 03/48] adj return --- source/util/src/tscalablebf.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index 1ee99b07f7..13e9fe79e8 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -38,7 +38,8 @@ int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf } SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); if (pSBf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } pSBf->maxBloomFilters = DEFAULT_MAX_BLOOMFILTERS; pSBf->status = SBF_VALID; From 60b268bbbe1901797a21eb1b5666b84860f9220c Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 15 Jul 2024 09:54:41 +0800 Subject: [PATCH 04/48] adj test --- source/util/test/bloomFilterTest.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/source/util/test/bloomFilterTest.cpp b/source/util/test/bloomFilterTest.cpp index 7832527bf1..2cd4edd519 100644 --- a/source/util/test/bloomFilterTest.cpp +++ b/source/util/test/bloomFilterTest.cpp @@ -70,12 +70,14 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) { int64_t ts1 = 1650803518000; - GTEST_ASSERT_EQ(NULL, tScalableBfInit(100, 0)); - GTEST_ASSERT_EQ(NULL, tScalableBfInit(100, 1)); - GTEST_ASSERT_EQ(NULL, tScalableBfInit(100, -0.1)); - GTEST_ASSERT_EQ(NULL, tScalableBfInit(0, 0.01)); + SScalableBf *tsSBF = NULL; + GTEST_ASSERT_NE(0, tScalableBfInit(100, 0, &tsSBF)); + GTEST_ASSERT_NE(0, tScalableBfInit(100, 1, &tsSBF)); + GTEST_ASSERT_NE(0, tScalableBfInit(100, -0.1, &tsSBF)); + GTEST_ASSERT_NE(0, tScalableBfInit(0, 0.01, &tsSBF)); - SScalableBf *pSBF1 = tScalableBfInit(100, 0.01); + SScalableBf *pSBF1 = NULL; + GTEST_ASSERT_EQ(0, tScalableBfInit(100, 0.01, &pSBF1)); GTEST_ASSERT_EQ(pSBF1->numBits, 1152); int64_t count = 0; int64_t index = 0; From 9b067d075f47588cd82302cfca4ab497270bcb38 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 15 Jul 2024 13:38:34 +0800 Subject: [PATCH 05/48] adj stream function return --- include/libs/executor/storageapi.h | 4 +- include/libs/stream/tstreamUpdate.h | 28 ++-- include/util/tbloomfilter.h | 20 +-- include/util/tscalablebf.h | 24 ++-- source/libs/executor/src/groupoperator.c | 2 +- .../executor/src/streamtimewindowoperator.c | 11 +- source/libs/stream/src/streamUpdate.c | 136 +++++++++++------- source/util/src/tbloomfilter.c | 28 ++-- source/util/src/tscalablebf.c | 40 +++--- source/util/test/bloomFilterTest.cpp | 36 ++--- 10 files changed, 181 insertions(+), 148 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 45424cfb66..168760c61c 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -381,7 +381,7 @@ typedef struct SStateStore { void** ppVal, int32_t* pVLen); int32_t (*streamStateCountWinAdd)(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen); - SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); + int32_t (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid); @@ -391,7 +391,7 @@ typedef struct SStateStore { void (*windowSBfDelete)(SUpdateInfo* pInfo, uint64_t count); int32_t (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count); - SUpdateInfo* (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); + int32_t (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo); void (*updateInfoDestoryColseWinSBF)(SUpdateInfo* pInfo); int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo); diff --git a/include/libs/stream/tstreamUpdate.h b/include/libs/stream/tstreamUpdate.h index 726f8443f1..9de0d9ab68 100644 --- a/include/libs/stream/tstreamUpdate.h +++ b/include/libs/stream/tstreamUpdate.h @@ -15,29 +15,29 @@ #ifndef _TSTREAMUPDATE_H_ #define _TSTREAMUPDATE_H_ +#include "storageapi.h" #include "taosdef.h" #include "tarray.h" #include "tcommon.h" #include "tmsg.h" -#include "storageapi.h" #ifdef __cplusplus extern "C" { #endif -SUpdateInfo *updateInfoInitP(SInterval *pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); -SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); -TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); -bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); -bool updateInfoIsTableInserted(SUpdateInfo *pInfo, int64_t tbUid); -void updateInfoDestroy(SUpdateInfo *pInfo); -void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo); -void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo); -int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo); -int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo); -void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count); -int32_t windowSBfAdd(SUpdateInfo *pInfo, uint64_t count); -bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); +int32_t updateInfoInitP(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); +int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); +TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); +bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); +bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid); +void updateInfoDestroy(SUpdateInfo* pInfo); +void updateInfoAddCloseWindowSBF(SUpdateInfo* pInfo); +void updateInfoDestoryColseWinSBF(SUpdateInfo* pInfo); +int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo); +int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo); +void windowSBfDelete(SUpdateInfo* pInfo, uint64_t count); +int32_t windowSBfAdd(SUpdateInfo* pInfo, uint64_t count); +bool isIncrementalTimeStamp(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); #ifdef __cplusplus } diff --git a/include/util/tbloomfilter.h b/include/util/tbloomfilter.h index 2a3da90dad..f0450cb862 100644 --- a/include/util/tbloomfilter.h +++ b/include/util/tbloomfilter.h @@ -37,19 +37,19 @@ typedef struct SBloomFilter { uint64_t size; _hash_fn_t hashFn1; _hash_fn_t hashFn2; - void *buffer; + void* buffer; double errorRate; } SBloomFilter; -int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilter **ppBF); -int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2); -int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len); -int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t h1, uint64_t h2); -void tBloomFilterDestroy(SBloomFilter *pBF); -void tBloomFilterDump(const SBloomFilter *pBF); -bool tBloomFilterIsFull(const SBloomFilter *pBF); -int32_t tBloomFilterEncode(const SBloomFilter *pBF, SEncoder *pEncoder); -int32_t tBloomFilterDecode(SDecoder *pDecoder, SBloomFilter **ppBF); +int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilter** ppBF); +int32_t tBloomFilterPutHash(SBloomFilter* pBF, uint64_t hash1, uint64_t hash2); +int32_t tBloomFilterPut(SBloomFilter* pBF, const void* keyBuf, uint32_t len); +int32_t tBloomFilterNoContain(const SBloomFilter* pBF, uint64_t h1, uint64_t h2); +void tBloomFilterDestroy(SBloomFilter* pBF); +void tBloomFilterDump(const SBloomFilter* pBF); +bool tBloomFilterIsFull(const SBloomFilter* pBF); +int32_t tBloomFilterEncode(const SBloomFilter* pBF, SEncoder* pEncoder); +int32_t tBloomFilterDecode(SDecoder* pDecoder, SBloomFilter** ppBF); #ifdef __cplusplus } diff --git a/include/util/tscalablebf.h b/include/util/tscalablebf.h index 584e023bb0..f82493aca2 100644 --- a/include/util/tscalablebf.h +++ b/include/util/tscalablebf.h @@ -23,22 +23,22 @@ extern "C" { #endif typedef struct SScalableBf { - SArray *bfArray; // array of bloom filters - uint32_t growth; - uint64_t numBits; - uint32_t maxBloomFilters; - int8_t status; + SArray* bfArray; // array of bloom filters + uint32_t growth; + uint64_t numBits; + uint32_t maxBloomFilters; + int8_t status; _hash_fn_t hashFn1; _hash_fn_t hashFn2; } SScalableBf; -int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf **ppSBf); -int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len); -int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len); -int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len); -void tScalableBfDestroy(SScalableBf *pSBf); -int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder); -int32_t tScalableBfDecode(SDecoder *pDecoder, SScalableBf **ppSBf); +int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf** ppSBf); +int32_t tScalableBfPutNoCheck(SScalableBf* pSBf, const void* keyBuf, uint32_t len); +int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len); +int32_t tScalableBfNoContain(const SScalableBf* pSBf, const void* keyBuf, uint32_t len); +void tScalableBfDestroy(SScalableBf* pSBf); +int32_t tScalableBfEncode(const SScalableBf* pSBf, SEncoder* pEncoder); +int32_t tScalableBfDecode(SDecoder* pDecoder, SScalableBf** ppSBf); #ifdef __cplusplus } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index f0fddc28fc..ccb8e702bd 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -1369,7 +1369,7 @@ void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup pScanInfo->pPartScalarSup = pExpr; pScanInfo->pPartTbnameSup = pTbnameExpr; if (!pScanInfo->pUpdateInfo) { - pScanInfo->pUpdateInfo = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen); + pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo); } } diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index d68108b9a5..d20df9d8a5 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -487,8 +487,11 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamInt pScanInfo->windowSup.parentType = type; pScanInfo->windowSup.pIntervalAggSup = &pInfo->aggSup; if (!pScanInfo->pUpdateInfo) { - pScanInfo->pUpdateInfo = - pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen); + int32_t code = pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate, + pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at since %s", __func__, __LINE__, tstrerror(code)); + } } pScanInfo->interval = pInfo->interval; @@ -1769,8 +1772,8 @@ void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uin pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type}; pScanInfo->pState = pAggSup->pState; if (!pScanInfo->pUpdateInfo) { - pScanInfo->pUpdateInfo = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark, - pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen); + pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark, + pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo); } pScanInfo->twAggSup = *pTwSup; pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo; diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 2da20cadae..7122b8f10f 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -18,6 +18,7 @@ #include "tencode.h" #include "tstreamUpdate.h" #include "ttime.h" +#include "tutil.h" #define DEFAULT_FALSE_POSITIVE 0.01 #define DEFAULT_BUCKET_SIZE 131072 @@ -34,7 +35,8 @@ static int64_t adjustExpEntries(int64_t entries) { return TMIN(DEFAULT_EXPECTED_ENTRIES, entries); } int compareKeyTs(void* pTs1, void* pTs2, void* pPkVal, __compar_fn_t cmpPkFn) { - return compareInt64Val(pTs1, pTs2);; + return compareInt64Val(pTs1, pTs2); + ; } int compareKeyTsAndPk(void* pValue1, void* pTs, void* pPkVal, __compar_fn_t cmpPkFn) { @@ -49,7 +51,7 @@ int compareKeyTsAndPk(void* pValue1, void* pTs, void* pPkVal, __compar_fn_t cmpP int32_t getKeyBuff(TSKEY ts, int64_t tbUid, void* pVal, int32_t len, char* buff) { *(TSKEY*)buff = ts; - memcpy(buff+ sizeof(TSKEY), &tbUid, sizeof(int64_t)); + memcpy(buff + sizeof(TSKEY), &tbUid, sizeof(int64_t)); if (len == 0) { return sizeof(TSKEY) + sizeof(int64_t); } @@ -66,7 +68,7 @@ int32_t getValueBuff(TSKEY ts, char* pVal, int32_t len, char* buff) { return sizeof(TSKEY) + len; } -int32_t windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) { +int32_t windowSBfAdd(SUpdateInfo* pInfo, uint64_t count) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; if (pInfo->numSBFs < count) { @@ -74,10 +76,10 @@ int32_t windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) { } for (uint64_t i = 0; i < count; ++i) { int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND); - SScalableBf *tsSBF = NULL; + SScalableBf* tsSBF = NULL; code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &tsSBF); TSDB_CHECK_CODE(code, lino, _error); - void *res = taosArrayPush(pInfo->pTsSBFs, &tsSBF); + void* res = taosArrayPush(pInfo->pTsSBFs, &tsSBF); if (!res) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _error); @@ -91,15 +93,15 @@ _error: return code; } -static void clearItemHelper(void *p) { - SScalableBf **pBf = p; +static void clearItemHelper(void* p) { + SScalableBf** pBf = p; tScalableBfDestroy(*pBf); } -void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count) { +void windowSBfDelete(SUpdateInfo* pInfo, uint64_t count) { if (count < pInfo->numSBFs) { for (uint64_t i = 0; i < count; ++i) { - SScalableBf *pTsSBFs = taosArrayGetP(pInfo->pTsSBFs, 0); + SScalableBf* pTsSBFs = taosArrayGetP(pInfo->pTsSBFs, 0); tScalableBfDestroy(pTsSBFs); taosArrayRemove(pInfo->pTsSBFs, 0); } @@ -138,14 +140,19 @@ static int64_t adjustWatermark(int64_t adjInterval, int64_t originInt, int64_t w return watermark; } -SUpdateInfo *updateInfoInitP(SInterval *pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen) { - return updateInfoInit(pInterval->interval, pInterval->precision, watermark, igUp, pkType, pkLen); +int32_t updateInfoInitP(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, + SUpdateInfo** ppInfo) { + return updateInfoInit(pInterval->interval, pInterval->precision, watermark, igUp, pkType, pkLen, ppInfo); } -SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen) { - SUpdateInfo *pInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo)); +int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, + SUpdateInfo** ppInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SUpdateInfo* pInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo)); if (pInfo == NULL) { - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); } pInfo->pTsBuckets = NULL; pInfo->pTsSBFs = NULL; @@ -159,17 +166,19 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma bfSize = (uint64_t)(pInfo->watermark / pInfo->interval); pInfo->numSBFs = bfSize; - pInfo->pTsSBFs = taosArrayInit(bfSize, sizeof(void *)); + pInfo->pTsSBFs = taosArrayInit(bfSize, sizeof(void*)); if (pInfo->pTsSBFs == NULL) { updateInfoDestroy(pInfo); - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); } windowSBfAdd(pInfo, bfSize); pInfo->pTsBuckets = taosArrayInit(DEFAULT_BUCKET_SIZE, sizeof(TSKEY)); if (pInfo->pTsBuckets == NULL) { updateInfoDestroy(pInfo); - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); } TSKEY dumy = 0; @@ -181,22 +190,41 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma } _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT); pInfo->pMap = taosHashInit(DEFAULT_MAP_CAPACITY, hashFn, true, HASH_NO_LOCK); + if (!pInfo->pMap) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } pInfo->maxDataVersion = 0; pInfo->pkColLen = pkLen; pInfo->pkColType = pkType; pInfo->pKeyBuff = taosMemoryCalloc(1, sizeof(TSKEY) + sizeof(int64_t) + pkLen); + if (!pInfo->pKeyBuff) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } pInfo->pValueBuff = taosMemoryCalloc(1, sizeof(TSKEY) + pkLen); + if (!pInfo->pValueBuff) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } if (pkLen != 0) { pInfo->comparePkRowFn = compareKeyTsAndPk; - pInfo->comparePkCol = getKeyComparFunc(pkType, TSDB_ORDER_ASC);; + pInfo->comparePkCol = getKeyComparFunc(pkType, TSDB_ORDER_ASC); + ; } else { pInfo->comparePkRowFn = compareKeyTs; pInfo->comparePkCol = NULL; } - return pInfo; + (*ppInfo) = pInfo; + +_end: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -static SScalableBf *getSBf(SUpdateInfo *pInfo, TSKEY ts) { +static SScalableBf* getSBf(SUpdateInfo* pInfo, TSKEY ts) { int32_t code = 0; if (ts <= 0) { return NULL; @@ -214,7 +242,7 @@ static SScalableBf *getSBf(SUpdateInfo *pInfo, TSKEY ts) { windowSBfAdd(pInfo, count); index = pInfo->numSBFs - 1; } - SScalableBf *res = taosArrayGetP(pInfo->pTsSBFs, index); + SScalableBf* res = taosArrayGetP(pInfo->pTsSBFs, index); if (res == NULL) { int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND); code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &res); @@ -223,13 +251,13 @@ static SScalableBf *getSBf(SUpdateInfo *pInfo, TSKEY ts) { return res; } -bool updateInfoIsTableInserted(SUpdateInfo *pInfo, int64_t tbUid) { - void *pVal = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t)); +bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid) { + void* pVal = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t)); if (pVal || taosHashGetSize(pInfo->pMap) >= DEFAULT_MAP_SIZE) return true; return false; } -TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol, int32_t primaryKeyCol) { +TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol) { if (pBlock == NULL || pBlock->info.rows == 0) return INT64_MIN; TSKEY maxTs = INT64_MIN; void* pPkVal = NULL; @@ -238,14 +266,14 @@ TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t p int32_t len = 0; int64_t tbUid = pBlock->info.id.uid; - SColumnInfoData *pColDataInfo = taosArrayGet(pBlock->pDataBlock, primaryTsCol); - SColumnInfoData *pPkDataInfo = NULL; + SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, primaryTsCol); + SColumnInfoData* pPkDataInfo = NULL; if (primaryKeyCol >= 0) { pPkDataInfo = taosArrayGet(pBlock->pDataBlock, primaryKeyCol); } for (int32_t i = 0; i < pBlock->info.rows; i++) { - TSKEY ts = ((TSKEY *)pColDataInfo->pData)[i]; + TSKEY ts = ((TSKEY*)pColDataInfo->pData)[i]; if (maxTs < ts) { maxTs = ts; if (primaryKeyCol >= 0) { @@ -253,7 +281,7 @@ TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t p maxLen = colDataGetRowLength(pPkDataInfo, i); } } - SScalableBf *pSBf = getSBf(pInfo, ts); + SScalableBf* pSBf = getSBf(pInfo, ts); if (pSBf) { if (primaryKeyCol >= 0) { pPkVal = colDataGetData(pPkDataInfo, i); @@ -263,7 +291,7 @@ TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t p tScalableBfPut(pSBf, pInfo->pKeyBuff, buffLen); } } - void *pMaxTs = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t)); + void* pMaxTs = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t)); if (pMaxTs == NULL || pInfo->comparePkRowFn(pMaxTs, &maxTs, pMaxPkVal, pInfo->comparePkCol) == -1) { int32_t valueLen = getValueBuff(maxTs, pMaxPkVal, maxLen, pInfo->pValueBuff); taosHashPut(pInfo->pMap, &tbUid, sizeof(int64_t), pInfo->pValueBuff, valueLen); @@ -271,14 +299,14 @@ TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t p return maxTs; } -bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) { +bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) { int32_t res = TSDB_CODE_FAILED; int32_t buffLen = 0; buffLen = getKeyBuff(ts, tableId, pPkVal, len, pInfo->pKeyBuff); - void* *pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); + void** pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); uint64_t index = ((uint64_t)tableId) % pInfo->numBuckets; - TSKEY maxTs = *(TSKEY *)taosArrayGet(pInfo->pTsBuckets, index); + TSKEY maxTs = *(TSKEY*)taosArrayGet(pInfo->pTsBuckets, index); if (ts < maxTs - pInfo->watermark) { // this window has been closed. if (pInfo->pCloseWinSBF) { @@ -292,10 +320,11 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* p return true; } - SScalableBf *pSBf = getSBf(pInfo, ts); + SScalableBf* pSBf = getSBf(pInfo, ts); int32_t size = taosHashGetSize(pInfo->pMap); - if ((!pMapMaxTs && size < DEFAULT_MAP_SIZE) || (pMapMaxTs && pInfo->comparePkRowFn(pMapMaxTs, &ts, pPkVal, pInfo->comparePkCol) == -1 )) { + if ((!pMapMaxTs && size < DEFAULT_MAP_SIZE) || + (pMapMaxTs && pInfo->comparePkRowFn(pMapMaxTs, &ts, pPkVal, pInfo->comparePkCol) == -1)) { int32_t valueLen = getValueBuff(ts, pPkVal, len, pInfo->pValueBuff); taosHashPut(pInfo->pMap, &tableId, sizeof(uint64_t), pInfo->pValueBuff, valueLen); // pSBf may be a null pointer @@ -324,7 +353,7 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* p return true; } -void updateInfoDestroy(SUpdateInfo *pInfo) { +void updateInfoDestroy(SUpdateInfo* pInfo) { if (pInfo == NULL) { return; } @@ -332,7 +361,7 @@ void updateInfoDestroy(SUpdateInfo *pInfo) { uint64_t size = taosArrayGetSize(pInfo->pTsSBFs); for (uint64_t i = 0; i < size; i++) { - SScalableBf *pSBF = taosArrayGetP(pInfo->pTsSBFs, i); + SScalableBf* pSBF = taosArrayGetP(pInfo->pTsSBFs, i); tScalableBfDestroy(pSBF); } @@ -344,7 +373,7 @@ void updateInfoDestroy(SUpdateInfo *pInfo) { taosMemoryFree(pInfo); } -void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo) { +void updateInfoAddCloseWindowSBF(SUpdateInfo* pInfo) { if (pInfo->pCloseWinSBF) { return; } @@ -356,7 +385,7 @@ void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo) { } } -void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo) { +void updateInfoDestoryColseWinSBF(SUpdateInfo* pInfo) { if (!pInfo || !pInfo->pCloseWinSBF) { return; } @@ -364,7 +393,7 @@ void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo) { pInfo->pCloseWinSBF = NULL; } -int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo) { +int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo) { if (!pInfo) { return 0; } @@ -376,7 +405,7 @@ int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo) int32_t size = taosArrayGetSize(pInfo->pTsBuckets); if (tEncodeI32(&encoder, size) < 0) return -1; for (int32_t i = 0; i < size; i++) { - TSKEY *pTs = (TSKEY *)taosArrayGet(pInfo->pTsBuckets, i); + TSKEY* pTs = (TSKEY*)taosArrayGet(pInfo->pTsBuckets, i); if (tEncodeI64(&encoder, *pTs) < 0) return -1; } @@ -385,7 +414,7 @@ int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo) int32_t sBfSize = taosArrayGetSize(pInfo->pTsSBFs); if (tEncodeI32(&encoder, sBfSize) < 0) return -1; for (int32_t i = 0; i < sBfSize; i++) { - SScalableBf *pSBf = taosArrayGetP(pInfo->pTsSBFs, i); + SScalableBf* pSBf = taosArrayGetP(pInfo->pTsSBFs, i); if (tScalableBfEncode(pSBf, &encoder) < 0) return -1; } @@ -398,13 +427,13 @@ int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo) int32_t mapSize = taosHashGetSize(pInfo->pMap); if (tEncodeI32(&encoder, mapSize) < 0) return -1; - void *pIte = NULL; + void* pIte = NULL; size_t keyLen = 0; while ((pIte = taosHashIterate(pInfo->pMap, pIte)) != NULL) { - void *key = taosHashGetKey(pIte, &keyLen); - if (tEncodeU64(&encoder, *(uint64_t *)key) < 0) return -1; + void* key = taosHashGetKey(pIte, &keyLen); + if (tEncodeU64(&encoder, *(uint64_t*)key) < 0) return -1; int32_t valueSize = taosHashGetValueSize(pIte); - if (tEncodeBinary(&encoder, (const uint8_t *)pIte, valueSize) < 0) return -1; + if (tEncodeBinary(&encoder, (const uint8_t*)pIte, valueSize) < 0) return -1; } if (tEncodeU64(&encoder, pInfo->maxDataVersion) < 0) return -1; @@ -419,7 +448,7 @@ int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo) return tlen; } -int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { +int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; ASSERT(pInfo); @@ -440,9 +469,9 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { int32_t sBfSize = 0; if (tDecodeI32(&decoder, &sBfSize) < 0) return -1; - pInfo->pTsSBFs = taosArrayInit(sBfSize, sizeof(void *)); + pInfo->pTsSBFs = taosArrayInit(sBfSize, sizeof(void*)); for (int32_t i = 0; i < sBfSize; i++) { - SScalableBf *pSBf = NULL; + SScalableBf* pSBf = NULL; code = tScalableBfDecode(&decoder, &pSBf); TSDB_CHECK_CODE(code, lino, _error); @@ -462,8 +491,8 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT); pInfo->pMap = taosHashInit(mapSize, hashFn, true, HASH_NO_LOCK); uint64_t uid = 0; - void* pVal = NULL; - int32_t valSize = 0; + void* pVal = NULL; + int32_t valSize = 0; for (int32_t i = 0; i < mapSize; i++) { if (tDecodeU64(&decoder, &uid) < 0) return -1; if (tDecodeBinary(&decoder, (uint8_t**)&pVal, &valSize) < 0) return -1; @@ -479,7 +508,8 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { pInfo->pValueBuff = taosMemoryCalloc(1, sizeof(TSKEY) + pInfo->pkColLen); if (pInfo->pkColLen != 0) { pInfo->comparePkRowFn = compareKeyTsAndPk; - pInfo->comparePkCol = getKeyComparFunc(pInfo->pkColType, TSDB_ORDER_ASC);; + pInfo->comparePkCol = getKeyComparFunc(pInfo->pkColType, TSDB_ORDER_ASC); + ; } else { pInfo->comparePkRowFn = compareKeyTs; pInfo->comparePkCol = NULL; @@ -496,8 +526,8 @@ _error: return code; } -bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) { - TSKEY *pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); +bool isIncrementalTimeStamp(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) { + TSKEY* pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); bool res = true; if (pMapMaxTs && pInfo->comparePkRowFn(pMapMaxTs, &ts, pPkVal, pInfo->comparePkCol) == 1) { res = false; diff --git a/source/util/src/tbloomfilter.c b/source/util/src/tbloomfilter.c index 557e12a5b2..9093fb9d91 100644 --- a/source/util/src/tbloomfilter.c +++ b/source/util/src/tbloomfilter.c @@ -22,27 +22,27 @@ #define UNIT_NUM_BITS 64ULL #define UNIT_ADDR_NUM_BITS 6ULL -static FORCE_INLINE bool setBit(uint64_t *buf, uint64_t index) { +static FORCE_INLINE bool setBit(uint64_t* buf, uint64_t index) { uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS; uint64_t old = buf[unitIndex]; buf[unitIndex] |= (1ULL << (index % UNIT_NUM_BITS)); return buf[unitIndex] != old; } -static FORCE_INLINE bool getBit(uint64_t *buf, uint64_t index) { +static FORCE_INLINE bool getBit(uint64_t* buf, uint64_t index) { uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS; uint64_t mask = 1ULL << (index % UNIT_NUM_BITS); return buf[unitIndex] & mask; } -int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilter **ppBF) { +int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilter** ppBF) { int32_t code = 0; int32_t lino = 0; if (expectedEntries < 1 || errorRate <= 0 || errorRate >= 1.0) { code = TSDB_CODE_FAILED; TSDB_CHECK_CODE(code, lino, _error); } - SBloomFilter *pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); + SBloomFilter* pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); if (pBF == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _error); @@ -77,7 +77,7 @@ _error: return code; } -int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2) { +int32_t tBloomFilterPutHash(SBloomFilter* pBF, uint64_t hash1, uint64_t hash2) { ASSERT(!tBloomFilterIsFull(pBF)); bool hasChange = false; const register uint64_t size = pBF->numBits; @@ -93,7 +93,7 @@ int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2) { return TSDB_CODE_FAILED; } -int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len) { +int32_t tBloomFilterPut(SBloomFilter* pBF, const void* keyBuf, uint32_t len) { uint64_t h1 = (uint64_t)pBF->hashFn1(keyBuf, len); uint64_t h2 = (uint64_t)pBF->hashFn2(keyBuf, len); bool hasChange = false; @@ -110,7 +110,7 @@ int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len) { return TSDB_CODE_FAILED; } -int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t hash1, uint64_t hash2) { +int32_t tBloomFilterNoContain(const SBloomFilter* pBF, uint64_t hash1, uint64_t hash2) { const register uint64_t size = pBF->numBits; uint64_t cbHash = hash1; for (uint32_t i = 0; i < pBF->hashFunctions; ++i) { @@ -122,7 +122,7 @@ int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t hash1, uint64_t return TSDB_CODE_FAILED; } -void tBloomFilterDestroy(SBloomFilter *pBF) { +void tBloomFilterDestroy(SBloomFilter* pBF) { if (pBF == NULL) { return; } @@ -130,24 +130,24 @@ void tBloomFilterDestroy(SBloomFilter *pBF) { taosMemoryFree(pBF); } -int32_t tBloomFilterEncode(const SBloomFilter *pBF, SEncoder *pEncoder) { +int32_t tBloomFilterEncode(const SBloomFilter* pBF, SEncoder* pEncoder) { if (tEncodeU32(pEncoder, pBF->hashFunctions) < 0) return -1; if (tEncodeU64(pEncoder, pBF->expectedEntries) < 0) return -1; if (tEncodeU64(pEncoder, pBF->numUnits) < 0) return -1; if (tEncodeU64(pEncoder, pBF->numBits) < 0) return -1; if (tEncodeU64(pEncoder, pBF->size) < 0) return -1; for (uint64_t i = 0; i < pBF->numUnits; i++) { - uint64_t *pUnits = (uint64_t *)pBF->buffer; + uint64_t* pUnits = (uint64_t*)pBF->buffer; if (tEncodeU64(pEncoder, pUnits[i]) < 0) return -1; } if (tEncodeDouble(pEncoder, pBF->errorRate) < 0) return -1; return 0; } -int32_t tBloomFilterDecode(SDecoder *pDecoder, SBloomFilter **ppBF) { +int32_t tBloomFilterDecode(SDecoder* pDecoder, SBloomFilter** ppBF) { int32_t code = 0; int32_t lino = 0; - SBloomFilter *pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); + SBloomFilter* pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); if (!pBF) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _error); @@ -175,7 +175,7 @@ int32_t tBloomFilterDecode(SDecoder *pDecoder, SBloomFilter **ppBF) { } pBF->buffer = taosMemoryCalloc(pBF->numUnits, sizeof(uint64_t)); for (int32_t i = 0; i < pBF->numUnits; i++) { - uint64_t *pUnits = (uint64_t *)pBF->buffer; + uint64_t* pUnits = (uint64_t*)pBF->buffer; if (tDecodeU64(pDecoder, pUnits + i) < 0) { code = TSDB_CODE_FAILED; TSDB_CHECK_CODE(code, lino, _error); @@ -195,4 +195,4 @@ _error: return TSDB_CODE_FAILED; } -bool tBloomFilterIsFull(const SBloomFilter *pBF) { return pBF->size >= pBF->expectedEntries; } +bool tBloomFilterIsFull(const SBloomFilter* pBF) { return pBF->size >= pBF->expectedEntries; } diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index 13e9fe79e8..0979000981 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -25,10 +25,10 @@ #define SBF_INVALID -1 #define SBF_VALID 0 -static int32_t tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate, - SBloomFilter **ppNormalBf); +static int32_t tScalableBfAddFilter(SScalableBf* pSBf, uint64_t expectedEntries, double errorRate, + SBloomFilter** ppNormalBf); -int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf **ppSBf) { +int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf** ppSBf) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; const uint32_t defaultSize = 8; @@ -36,7 +36,7 @@ int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf code = TSDB_CODE_FAILED; TSDB_CHECK_CODE(code, lino, _error); } - SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); + SScalableBf* pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); if (pSBf == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _error); @@ -44,13 +44,13 @@ int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf pSBf->maxBloomFilters = DEFAULT_MAX_BLOOMFILTERS; pSBf->status = SBF_VALID; pSBf->numBits = 0; - pSBf->bfArray = taosArrayInit(defaultSize, sizeof(void *)); + pSBf->bfArray = taosArrayInit(defaultSize, sizeof(void*)); if (!pSBf->bfArray) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _error); } - SBloomFilter *pNormalBf = NULL; + SBloomFilter* pNormalBf = NULL; code = tScalableBfAddFilter(pSBf, expectedEntries, errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf); if (code != TSDB_CODE_SUCCESS) { tScalableBfDestroy(pSBf); @@ -67,7 +67,7 @@ _error: return code; } -int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { +int32_t tScalableBfPutNoCheck(SScalableBf* pSBf, const void* keyBuf, uint32_t len) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; if (pSBf->status == SBF_INVALID) { @@ -75,7 +75,7 @@ int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t le TSDB_CHECK_CODE(code, lino, _error); } int32_t size = taosArrayGetSize(pSBf->bfArray); - SBloomFilter *pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); + SBloomFilter* pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); if (!pNormalBf) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _error); @@ -97,7 +97,7 @@ _error: return code; } -int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { +int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; if (pSBf->status == SBF_INVALID) { @@ -113,7 +113,7 @@ int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { } } - SBloomFilter *pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); + SBloomFilter* pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); ASSERT(pNormalBf); if (tBloomFilterIsFull(pNormalBf)) { code = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, @@ -132,7 +132,7 @@ _error: return code; } -int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len) { +int32_t tScalableBfNoContain(const SScalableBf* pSBf, const void* keyBuf, uint32_t len) { if (pSBf->status == SBF_INVALID) { return TSDB_CODE_FAILED; } @@ -147,8 +147,8 @@ int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32 return TSDB_CODE_SUCCESS; } -static int32_t tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate, - SBloomFilter **ppNormalBf) { +static int32_t tScalableBfAddFilter(SScalableBf* pSBf, uint64_t expectedEntries, double errorRate, + SBloomFilter** ppNormalBf) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; if (taosArrayGetSize(pSBf->bfArray) >= pSBf->maxBloomFilters) { @@ -156,7 +156,7 @@ static int32_t tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, TSDB_CHECK_CODE(code, lino, _error); } - SBloomFilter *pNormalBf = NULL; + SBloomFilter* pNormalBf = NULL; code = tBloomFilterInit(expectedEntries, errorRate, &pNormalBf); TSDB_CHECK_CODE(code, lino, _error); @@ -175,7 +175,7 @@ _error: return code; } -void tScalableBfDestroy(SScalableBf *pSBf) { +void tScalableBfDestroy(SScalableBf* pSBf) { if (pSBf == NULL) { return; } @@ -185,7 +185,7 @@ void tScalableBfDestroy(SScalableBf *pSBf) { taosMemoryFree(pSBf); } -int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) { +int32_t tScalableBfEncode(const SScalableBf* pSBf, SEncoder* pEncoder) { if (!pSBf) { if (tEncodeI32(pEncoder, 0) < 0) return -1; return 0; @@ -193,7 +193,7 @@ int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) { int32_t size = taosArrayGetSize(pSBf->bfArray); if (tEncodeI32(pEncoder, size) < 0) return -1; for (int32_t i = 0; i < size; i++) { - SBloomFilter *pBF = taosArrayGetP(pSBf->bfArray, i); + SBloomFilter* pBF = taosArrayGetP(pSBf->bfArray, i); if (tBloomFilterEncode(pBF, pEncoder) < 0) return -1; } if (tEncodeU32(pEncoder, pSBf->growth) < 0) return -1; @@ -203,10 +203,10 @@ int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) { return 0; } -int32_t tScalableBfDecode(SDecoder *pDecoder, SScalableBf **ppSBf) { +int32_t tScalableBfDecode(SDecoder* pDecoder, SScalableBf** ppSBf) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; - SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); + SScalableBf* pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); if (!pSBf) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _error); @@ -230,7 +230,7 @@ int32_t tScalableBfDecode(SDecoder *pDecoder, SScalableBf **ppSBf) { } for (int32_t i = 0; i < size; i++) { - SBloomFilter *pBF = NULL; + SBloomFilter* pBF = NULL; code = tBloomFilterDecode(pDecoder, &pBF); TSDB_CHECK_CODE(code, lino, _error); taosArrayPush(pSBf->bfArray, &pBF); diff --git a/source/util/test/bloomFilterTest.cpp b/source/util/test/bloomFilterTest.cpp index 2cd4edd519..9d913f8815 100644 --- a/source/util/test/bloomFilterTest.cpp +++ b/source/util/test/bloomFilterTest.cpp @@ -8,15 +8,14 @@ using namespace std; TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { int64_t ts1 = 1650803518000; - SBloomFilter *pBFTmp = NULL; + SBloomFilter* pBFTmp = NULL; GTEST_ASSERT_EQ(0, tBloomFilterInit(100, 0, &pBFTmp)); GTEST_ASSERT_EQ(0, tBloomFilterInit(100, 1, &pBFTmp)); GTEST_ASSERT_EQ(0, tBloomFilterInit(100, -0.1, &pBFTmp)); GTEST_ASSERT_EQ(0, tBloomFilterInit(0, 0.01, &pBFTmp)); - SBloomFilter *pBF1 = NULL, - int32_t code = tBloomFilterInit(100, 0.005, &pBF1); - GTEST_ASSERT_EQ(0,code); + SBloomFilter *pBF1 = NULL, int32_t code = tBloomFilterInit(100, 0.005, &pBF1); + GTEST_ASSERT_EQ(0, code); GTEST_ASSERT_EQ(pBF1->numBits, 1152); GTEST_ASSERT_EQ(pBF1->numUnits, 1152 / 64); int64_t count = 0; @@ -28,18 +27,18 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { } ASSERT_TRUE(tBloomFilterIsFull(pBF1)); - SBloomFilter *pBF2 = NULL; + SBloomFilter* pBF2 = NULL; GTEST_ASSERT_EQ(0, tBloomFilterInit(1000 * 10000, 0.1, &pBF2)); GTEST_ASSERT_EQ(pBF2->numBits, 47925312); GTEST_ASSERT_EQ(pBF2->numUnits, 47925312 / 64); - SBloomFilter *pBF3 = NULL; + SBloomFilter* pBF3 = NULL; GTEST_ASSERT_EQ(0, tBloomFilterInit(10000 * 10000, 0.001)); GTEST_ASSERT_EQ(pBF3->numBits, 1437758784); GTEST_ASSERT_EQ(pBF3->numUnits, 1437758784 / 64); int64_t size = 10000; - SBloomFilter *pBF4 = NULL; + SBloomFilter* pBF4 = NULL; GTEST_ASSERT_EQ(0, tBloomFilterInit(size, 0.001)); for (int64_t i = 0; i < 1000; i++) { int64_t ts = i + ts1; @@ -48,17 +47,17 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { ASSERT_TRUE(!tBloomFilterIsFull(pBF4)); for (int64_t i = 0; i < 1000; i++) { - int64_t ts = i + ts1; - uint64_t h1 = (uint64_t) pBF4->hashFn1((const char*)&ts, sizeof(int64_t)); - uint64_t h2 = (uint64_t) pBF4->hashFn2((const char*)&ts, sizeof(int64_t)); + int64_t ts = i + ts1; + uint64_t h1 = (uint64_t)pBF4->hashFn1((const char*)&ts, sizeof(int64_t)); + uint64_t h2 = (uint64_t)pBF4->hashFn2((const char*)&ts, sizeof(int64_t)); GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, h1, h2), TSDB_CODE_FAILED); } for (int64_t i = 2000; i < 3000; i++) { - int64_t ts = i + ts1; - uint64_t h1 = (uint64_t) pBF4->hashFn1((const char*)&ts, sizeof(int64_t)); - uint64_t h2 = (uint64_t) pBF4->hashFn2((const char*)&ts, sizeof(int64_t)); - GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, h1, h2), TSDB_CODE_SUCCESS); + int64_t ts = i + ts1; + uint64_t h1 = (uint64_t)pBF4->hashFn1((const char*)&ts, sizeof(int64_t)); + uint64_t h2 = (uint64_t)pBF4->hashFn2((const char*)&ts, sizeof(int64_t)); + GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, h1, h2), TSDB_CODE_SUCCESS); } tBloomFilterDestroy(pBF1); @@ -70,13 +69,13 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) { int64_t ts1 = 1650803518000; - SScalableBf *tsSBF = NULL; + SScalableBf* tsSBF = NULL; GTEST_ASSERT_NE(0, tScalableBfInit(100, 0, &tsSBF)); GTEST_ASSERT_NE(0, tScalableBfInit(100, 1, &tsSBF)); GTEST_ASSERT_NE(0, tScalableBfInit(100, -0.1, &tsSBF)); GTEST_ASSERT_NE(0, tScalableBfInit(0, 0.01, &tsSBF)); - SScalableBf *pSBF1 = NULL; + SScalableBf* pSBF1 = NULL; GTEST_ASSERT_EQ(0, tScalableBfInit(100, 0.01, &pSBF1)); GTEST_ASSERT_EQ(pSBF1->numBits, 1152); int64_t count = 0; @@ -116,7 +115,7 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) { int32_t aSize = taosArrayGetSize(pSBF1->bfArray); int64_t totalBits = 0; for (int64_t i = 0; i < aSize; i++) { - SBloomFilter *pBF = (SBloomFilter *)taosArrayGetP(pSBF1->bfArray, i); + SBloomFilter* pBF = (SBloomFilter*)taosArrayGetP(pSBF1->bfArray, i); ASSERT_TRUE(tBloomFilterIsFull(pBF)); totalBits += pBF->numBits; } @@ -128,7 +127,8 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) { } int64_t size = 10000; - SScalableBf *pSBF4 = tScalableBfInit(size, 0.001); + SScalableBf* pSBF4 = NULL; + GTEST_ASSERT_EQ(0, tScalableBfInit(size, 0.001, pSBF4)); for (int64_t i = 0; i < 1000; i++) { int64_t ts = i + ts1; GTEST_ASSERT_EQ(tScalableBfPut(pSBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS); From bb25622a3055d348a907beb674721a67c8ccf5b5 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 15 Jul 2024 13:47:02 +0800 Subject: [PATCH 06/48] adj ci --- source/util/test/bloomFilterTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/test/bloomFilterTest.cpp b/source/util/test/bloomFilterTest.cpp index 9d913f8815..d14aa2d453 100644 --- a/source/util/test/bloomFilterTest.cpp +++ b/source/util/test/bloomFilterTest.cpp @@ -128,7 +128,7 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) { int64_t size = 10000; SScalableBf* pSBF4 = NULL; - GTEST_ASSERT_EQ(0, tScalableBfInit(size, 0.001, pSBF4)); + GTEST_ASSERT_EQ(0, tScalableBfInit(size, 0.001, &pSBF4)); for (int64_t i = 0; i < 1000; i++) { int64_t ts = i + ts1; GTEST_ASSERT_EQ(tScalableBfPut(pSBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS); From 2062e6f9e93ab4b6efd44ec7f2c992d367deb3fb Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 15 Jul 2024 14:11:02 +0800 Subject: [PATCH 07/48] adj ut --- source/util/test/bloomFilterTest.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/source/util/test/bloomFilterTest.cpp b/source/util/test/bloomFilterTest.cpp index d14aa2d453..9d83cbf086 100644 --- a/source/util/test/bloomFilterTest.cpp +++ b/source/util/test/bloomFilterTest.cpp @@ -9,12 +9,13 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { int64_t ts1 = 1650803518000; SBloomFilter* pBFTmp = NULL; - GTEST_ASSERT_EQ(0, tBloomFilterInit(100, 0, &pBFTmp)); - GTEST_ASSERT_EQ(0, tBloomFilterInit(100, 1, &pBFTmp)); - GTEST_ASSERT_EQ(0, tBloomFilterInit(100, -0.1, &pBFTmp)); - GTEST_ASSERT_EQ(0, tBloomFilterInit(0, 0.01, &pBFTmp)); + GTEST_ASSERT_NE(0, tBloomFilterInit(100, 0, &pBFTmp)); + GTEST_ASSERT_NE(0, tBloomFilterInit(100, 1, &pBFTmp)); + GTEST_ASSERT_NE(0, tBloomFilterInit(100, -0.1, &pBFTmp)); + GTEST_ASSERT_NE(0, tBloomFilterInit(0, 0.01, &pBFTmp)); - SBloomFilter *pBF1 = NULL, int32_t code = tBloomFilterInit(100, 0.005, &pBF1); + SBloomFilter *pBF1 = NULL; + int32_t code = tBloomFilterInit(100, 0.005, &pBF1); GTEST_ASSERT_EQ(0, code); GTEST_ASSERT_EQ(pBF1->numBits, 1152); GTEST_ASSERT_EQ(pBF1->numUnits, 1152 / 64); @@ -33,13 +34,13 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { GTEST_ASSERT_EQ(pBF2->numUnits, 47925312 / 64); SBloomFilter* pBF3 = NULL; - GTEST_ASSERT_EQ(0, tBloomFilterInit(10000 * 10000, 0.001)); + GTEST_ASSERT_EQ(0, tBloomFilterInit(10000 * 10000, 0.001, &pBF3)); GTEST_ASSERT_EQ(pBF3->numBits, 1437758784); GTEST_ASSERT_EQ(pBF3->numUnits, 1437758784 / 64); int64_t size = 10000; SBloomFilter* pBF4 = NULL; - GTEST_ASSERT_EQ(0, tBloomFilterInit(size, 0.001)); + GTEST_ASSERT_EQ(0, tBloomFilterInit(size, 0.001, &pBF4)); for (int64_t i = 0; i < 1000; i++) { int64_t ts = i + ts1; GTEST_ASSERT_EQ(tBloomFilterPut(pBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS); From 503822a9c5bd00cbaa259bff918d8ccf4b8a74bb Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 15 Jul 2024 14:24:09 +0800 Subject: [PATCH 08/48] adj ut --- source/libs/executor/src/streamtimewindowoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index d20df9d8a5..97cb20a7ea 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -490,7 +490,7 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamInt int32_t code = pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo); if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at since %s", __func__, __LINE__, tstrerror(code)); + qError("%s failed at %d since %s", __func__, __LINE__, tstrerror(code)); } } From 4efd774ce8abe18f683929cce18bf4d80cdeb079 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 15 Jul 2024 14:32:31 +0800 Subject: [PATCH 09/48] adj stream function return --- source/libs/stream/src/streamUpdate.c | 34 +++++++++++++++++++++------ 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 7122b8f10f..8727ce0174 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -224,17 +224,20 @@ _end: return code; } -static SScalableBf* getSBf(SUpdateInfo* pInfo, TSKEY ts) { - int32_t code = 0; +static int32_t getSBf(SUpdateInfo* pInfo, TSKEY ts, SScalableBf** ppSBf) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (ts <= 0) { - return NULL; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->minTS < 0) { pInfo->minTS = (TSKEY)(ts / pInfo->interval * pInfo->interval); } int64_t index = (int64_t)((ts - pInfo->minTS) / pInfo->interval); if (index < 0) { - return NULL; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } if (index >= pInfo->numSBFs) { uint64_t count = index + 1 - pInfo->numSBFs; @@ -246,9 +249,17 @@ static SScalableBf* getSBf(SUpdateInfo* pInfo, TSKEY ts) { if (res == NULL) { int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND); code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &res); + TSDB_CHECK_CODE(code, lino, _end); + taosArrayPush(pInfo->pTsSBFs, &res); } - return res; + (*ppSBf) = res; + +_end: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid) { @@ -258,6 +269,7 @@ bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid) { } TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol) { + int32_t code = TSDB_CODE_SUCCESS; if (pBlock == NULL || pBlock->info.rows == 0) return INT64_MIN; TSKEY maxTs = INT64_MIN; void* pPkVal = NULL; @@ -281,7 +293,11 @@ TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t p maxLen = colDataGetRowLength(pPkDataInfo, i); } } - SScalableBf* pSBf = getSBf(pInfo, ts); + SScalableBf* pSBf = NULL; + int32_t code = getSBf(pInfo, ts, &pSBf); + if (code != TSDB_CODE_SUCCESS) { + uWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } if (pSBf) { if (primaryKeyCol >= 0) { pPkVal = colDataGetData(pPkDataInfo, i); @@ -320,7 +336,11 @@ bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* p return true; } - SScalableBf* pSBf = getSBf(pInfo, ts); + SScalableBf* pSBf = NULL; + int32_t code = getSBf(pInfo, ts, &pSBf); + if (code != TSDB_CODE_SUCCESS) { + uWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } int32_t size = taosHashGetSize(pInfo->pMap); if ((!pMapMaxTs && size < DEFAULT_MAP_SIZE) || From 57dc3f4562ee30b3e6fa141083cd213cba21b27e Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Mon, 15 Jul 2024 15:15:17 +0800 Subject: [PATCH 10/48] fix: disable dynamic modification of 'tempdir' --- source/common/src/tglobal.c | 2 +- tests/army/cmdline/fullopt.py | 21 ++++++++++----------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 119438fbd6..7538b504e3 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -504,7 +504,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "secondEp", "", CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddString(pCfg, "fqdn", defaultFqdn, CFG_SCOPE_SERVER, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_CLIENT) != 0) return -1; - if (cfgAddDir(pCfg, "tempDir", tsTempDir, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddDir(pCfg, "tempDir", tsTempDir, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; if (cfgAddFloat(pCfg, "minimalTmpDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; diff --git a/tests/army/cmdline/fullopt.py b/tests/army/cmdline/fullopt.py index 77a22b9256..6fc8e858a3 100644 --- a/tests/army/cmdline/fullopt.py +++ b/tests/army/cmdline/fullopt.py @@ -28,13 +28,13 @@ from frame import * class TDTestCase(TBase): updatecfgDict = { - 'queryMaxConcurrentTables': '2K', - 'streamMax': '1M', + 'queryMaxConcurrentTables': '2K', + 'streamMax': '1M', 'totalMemoryKB': '1G', 'streamMax': '1P', 'streamBufferSize':'1T', 'slowLogScope':"query" - } + } def insertData(self): tdLog.info(f"insert data.") @@ -53,7 +53,7 @@ class TDTestCase(TBase): def doTaos(self): tdLog.info(f"check taos command options...") - + # local command options = [ "DebugFlag 143", @@ -71,7 +71,6 @@ class TDTestCase(TBase): "smlTsDefaultName tsdef", "serverPort 6030", "timezone tz", - "tempDir /var/tmp" ] # exec for option in options: @@ -83,7 +82,7 @@ class TDTestCase(TBase): # help rets = etool.runBinFile("taos", "--help") self.checkListNotEmpty(rets) - # b r w s + # b r w s sql = f"select * from {self.db}.{self.stb} limit 10" rets = etool.runBinFile("taos", f'-B -r -w 100 -s "{sql}" ') self.checkListNotEmpty(rets) @@ -124,16 +123,16 @@ class TDTestCase(TBase): tdLog.info(f"check taosd command options...") idx = 1 # dnode1 cfg = sc.dnodeCfgPath(idx) - + # -s sdb = "./sdb.json" eos.delFile(sdb) etool.exeBinFile("taosd", f"-s -c {cfg}") - + # -C etool.exeBinFile("taosd", "-C") - # -k + # -k etool.exeBinFile("taosd", "-k", False) # -V rets = etool.runBinFile("taosd", "-V") @@ -153,7 +152,7 @@ class TDTestCase(TBase): sc.dnodeStop(idx) etool.exeBinFile("taosd", "-a http://192.168.1.10") - #exe + #exe etool.exeBinFile("taosd", f"-E abc -c {cfg}", False) sc.dnodeStop(idx) etool.exeBinFile("taosd", f"-e def -c {cfg}", False) @@ -182,7 +181,7 @@ class TDTestCase(TBase): # do action self.doTaosd() - + tdLog.success(f"{__file__} successfully executed") From 019a1eb76b36995d2eeb9dfba05065f7ba068afc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 15 Jul 2024 15:28:32 +0800 Subject: [PATCH 11/48] fix(tsdb): check return value. --- source/common/src/tdatablock.c | 4 ++++ source/dnode/vnode/src/tsdb/tsdbRead2.c | 19 ++++++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 809721d606..8f5d382cde 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -615,6 +615,10 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex) int32_t index = (tsColumnIndex == -1) ? 0 : tsColumnIndex; SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, index); + if (pColInfoData == NULL) { + return 0; + } + if (pColInfoData->info.type != TSDB_DATA_TYPE_TIMESTAMP) { return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 272c0762d8..ec7d90184c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -123,7 +123,7 @@ static void tColRowGetPriamyKeyDeepCopy(SBlockData* pBlock, int32_t irow, int32_ pKey->pks[0].val = cv.value.val; } else { pKey->pks[0].nData = cv.value.nData; - memcpy(pKey->pks[0].pData, cv.value.pData, cv.value.nData); + /*void* p = */memcpy(pKey->pks[0].pData, cv.value.pData, cv.value.nData); } } @@ -185,6 +185,10 @@ static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pC if (IS_VAR_DATA_TYPE(pCols[i].type)) { pSupInfo->buildBuf[i] = taosMemoryMalloc(pCols[i].bytes); + if (pSupInfo->buildBuf[i] == NULL) { + tsdbError("failed to prepare memory for set columnId slot list, size:%d, code:out of memory", pCols[i].bytes); + return TSDB_CODE_OUT_OF_MEMORY; + } } else { pSupInfo->buildBuf[i] = NULL; } @@ -530,7 +534,10 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void // allocate buffer in order to load data blocks from file SBlockLoadSuppInfo* pSup = &pReader->suppInfo; pSup->tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID; - setColumnIdSlotList(pSup, pCond->colList, pCond->pSlotList, pCond->numOfCols); + code = setColumnIdSlotList(pSup, pCond->colList, pCond->pSlotList, pCond->numOfCols); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } code = initResBlockInfo(&pReader->resBlockInfo, capacity, pResBlock, pCond, pSup); if (code != TSDB_CODE_SUCCESS) { @@ -2496,8 +2503,8 @@ void updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInf pResBlock->info.dataLoad = 1; pResBlock->info.version = pReader->info.verRange.maxVer; - blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotId[0]); - blockDataUpdatePkRange(pResBlock, pReader->suppInfo.pkDstSlot, ASCENDING_TRAVERSE(pReader->info.order)); + int32_t code = blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotId[0]); + code = blockDataUpdatePkRange(pResBlock, pReader->suppInfo.pkDstSlot, ASCENDING_TRAVERSE(pReader->info.order)); setComposedBlockFlag(pReader, true); pReader->cost.composedBlocks += 1; @@ -3205,7 +3212,6 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader, int64_t en // set the correct start position in case of the first/last file block, according to the query time window static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) { - int64_t lastKey = ASCENDING_TRAVERSE(pReader->info.order) ? INT64_MIN : INT64_MAX; SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); SReaderStatus* pStatus = &pReader->status; SFileBlockDumpInfo* pDumpInfo = &pStatus->fBlockDumpInfo; @@ -5476,6 +5482,9 @@ void tsdbUntakeReadSnap2(STsdbReader* pReader, STsdbReadSnap* pSnap, bool proact void tsdbReaderSetId2(STsdbReader* pReader, const char* idstr) { taosMemoryFreeClear(pReader->idStr); pReader->idStr = taosStrdup(idstr); + if (pReader->idStr == NULL) { + // no need to do anything + } pReader->status.fileIter.pSttBlockReader->mergeTree.idStr = pReader->idStr; } From 3cdb957f224768dfde2b5016800709d449f136db Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 15 Jul 2024 15:32:11 +0800 Subject: [PATCH 12/48] adj function of stream operator result --- include/libs/executor/storageapi.h | 2 +- include/libs/stream/tstreamUpdate.h | 2 +- source/libs/executor/inc/executorInt.h | 2 +- source/libs/executor/inc/streamexecutorInt.h | 1 + source/libs/executor/src/groupoperator.c | 18 +- source/libs/executor/src/scanoperator.c | 69 +++++-- .../executor/src/streamcountwindowoperator.c | 90 ++++----- .../executor/src/streameventwindowoperator.c | 112 ++++++----- .../executor/src/streamtimewindowoperator.c | 185 ++++++++++-------- source/libs/stream/src/streamUpdate.c | 101 ++++++++-- 10 files changed, 368 insertions(+), 214 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 168760c61c..49d86f8c37 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -394,7 +394,7 @@ typedef struct SStateStore { int32_t (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo); void (*updateInfoDestoryColseWinSBF)(SUpdateInfo* pInfo); - int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo); + int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen); int32_t (*updateInfoDeserialize)(void* buf, int32_t bufLen, SUpdateInfo* pInfo); SStreamStateCur* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key); diff --git a/include/libs/stream/tstreamUpdate.h b/include/libs/stream/tstreamUpdate.h index 9de0d9ab68..e1b33de429 100644 --- a/include/libs/stream/tstreamUpdate.h +++ b/include/libs/stream/tstreamUpdate.h @@ -33,7 +33,7 @@ bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid); void updateInfoDestroy(SUpdateInfo* pInfo); void updateInfoAddCloseWindowSBF(SUpdateInfo* pInfo); void updateInfoDestoryColseWinSBF(SUpdateInfo* pInfo); -int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo); +int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen); int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo); void windowSBfDelete(SUpdateInfo* pInfo, uint64_t count); int32_t windowSBfAdd(SUpdateInfo* pInfo, uint64_t count); diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 9547fa12f3..6db3c780e2 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -929,7 +929,7 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, i SStreamState* pState, int32_t keySize, int16_t keyType, SStateStore* pStore, SReadHandle* pHandle, STimeWindowAggSupp* pTwAggSup, const char* taskIdStr, SStorageAPI* pApi, int32_t tsIndex); -void initDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, +int32_t initDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic); void getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins); void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); diff --git a/source/libs/executor/inc/streamexecutorInt.h b/source/libs/executor/inc/streamexecutorInt.h index 8b61b93601..c6053cc96e 100644 --- a/source/libs/executor/inc/streamexecutorInt.h +++ b/source/libs/executor/inc/streamexecutorInt.h @@ -20,6 +20,7 @@ extern "C" { #endif #include "executorInt.h" +#include "tutil.h" void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type); bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo); diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index ccb8e702bd..c7236a88e8 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -17,6 +17,7 @@ #include "function.h" #include "os.h" #include "tname.h" +#include "tutil.h" #include "tdatablock.h" #include "tmsg.h" @@ -1357,11 +1358,13 @@ static void destroyStreamPartitionOperatorInfo(void* param) { taosMemoryFreeClear(param); } -void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr, SExprSupp* pTbnameExpr) { +int32_t initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr, SExprSupp* pTbnameExpr) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStorageAPI* pAPI = &downstream->pTaskInfo->storageAPI; if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - return; + return code; } SStreamScanInfo* pScanInfo = downstream->info; @@ -1369,8 +1372,9 @@ void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup pScanInfo->pPartScalarSup = pExpr; pScanInfo->pPartTbnameSup = pTbnameExpr; if (!pScanInfo->pUpdateInfo) { - pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo); + code = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo); } + return code; } SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag) { @@ -1421,6 +1425,7 @@ void freePartItem(void* ptr) { SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamPartitionOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -1511,14 +1516,19 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr destroyStreamPartitionOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState); - initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup, &pInfo->tbnameCalSup); + code = initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup, &pInfo->tbnameCalSup); + TSDB_CHECK_CODE(code, lino, _error); + code = appendDownstream(pOperator, &downstream, 1); + TSDB_CHECK_CODE(code, lino, _error); + return pOperator; _error: pTaskInfo->code = code; destroyStreamPartitionOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return NULL; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ce4915ca4d..1e5e116d6b 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2477,27 +2477,54 @@ static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey, SSDataBlock* pBl } } -int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff) { - int32_t len = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo); - len += encodeSTimeWindowAggSupp(NULL, &pInfo->twAggSup); - *pBuff = taosMemoryCalloc(1, len); - void* buf = *pBuff; - encodeSTimeWindowAggSupp(&buf, &pInfo->twAggSup); - pInfo->stateStore.updateInfoSerialize(buf, len, pInfo->pUpdateInfo); - return len; +int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff, int32_t* pLen) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t len = 0; + code = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo, &len); + TSDB_CHECK_CODE(code, lino, _end); + + len += encodeSTimeWindowAggSupp(NULL, &pInfo->twAggSup); + *pBuff = taosMemoryCalloc(1, len); + if (!(*pBuff)) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + void* buf = *pBuff; + encodeSTimeWindowAggSupp(&buf, &pInfo->twAggSup); + int32_t tmp = 0; + code = pInfo->stateStore.updateInfoSerialize(buf, len, pInfo->pUpdateInfo, &tmp); + TSDB_CHECK_CODE(code, lino, _end); + + *pLen = len; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void streamScanOperatorSaveCheckpoint(SStreamScanInfo* pInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (!pInfo->pState) { return; } if (needSaveStreamOperatorInfo(&pInfo->basic)) { void* pBuf = NULL; - int32_t len = streamScanOperatorEncode(pInfo, &pBuf); + int32_t len = 0; + code = streamScanOperatorEncode(pInfo, &pBuf, &len); + TSDB_CHECK_CODE(code, lino, _end); pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_CHECKPOINT_NAME, strlen(STREAM_SCAN_OP_CHECKPOINT_NAME), pBuf, len); taosMemoryFree(pBuf); saveStreamOperatorStateComplete(&pInfo->basic); } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } // other properties are recovered from the execution plan @@ -3094,17 +3121,35 @@ static void destroyStreamScanOperatorInfo(void* param) { } void streamScanReleaseState(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamScanInfo* pInfo = pOperator->info; + void* pBuff = NULL; if (!pInfo->pState) { return; } if (!pInfo->pUpdateInfo) { return; } - int32_t len = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo); - void* pBuff = taosMemoryCalloc(1, len); - pInfo->stateStore.updateInfoSerialize(pBuff, len, pInfo->pUpdateInfo); + int32_t len = 0; + code = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo, &len); + TSDB_CHECK_CODE(code, lino, _end); + + pBuff = taosMemoryCalloc(1, len); + if (!pBuff) { + code = TSDB_CODE_OUT_OF_MEMORY; +TSDB_CHECK_CODE(code, lino, _end); + } + + int32_t tmp = 0; + code = pInfo->stateStore.updateInfoSerialize(pBuff, len, pInfo->pUpdateInfo, &tmp); + TSDB_CHECK_CODE(code, lino, _end); + pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_STATE_NAME, strlen(STREAM_SCAN_OP_STATE_NAME), pBuff, len); +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } taosMemoryFree(pBuff); } diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index 857d048457..14f2dfbada 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -68,9 +68,7 @@ void destroyStreamCountAggOperatorInfo(void* param) { taosMemoryFreeClear(param); } -bool isSlidingCountWindow(SStreamAggSupporter* pAggSup) { - return pAggSup->windowCount != pAggSup->windowSliding; -} +bool isSlidingCountWindow(SStreamAggSupporter* pAggSup) { return pAggSup->windowCount != pAggSup->windowSliding; } void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, SCountWindowInfo* pCurWin, SBuffInfo* pBuffInfo) { @@ -89,13 +87,14 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, ASSERT(pBuffInfo->pCur); pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pBuffInfo->pCur); code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin, - (void**)&pCurWin->winInfo.pStatePos, &size); + (void**)&pCurWin->winInfo.pStatePos, &size); if (code == TSDB_CODE_FAILED) { pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, (void**)&pCurWin->winInfo.pStatePos, &size); } } else { - pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount); + pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin, + pAggSup->windowCount); code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin, (void**)&pCurWin->winInfo.pStatePos, &size); if (code == TSDB_CODE_FAILED) { @@ -107,15 +106,16 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, pBuffInfo->rebuildWindow = true; } } else { - code = pAggSup->stateStore.streamStateCountWinAddIfNotExist( - pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos, &size); + code = pAggSup->stateStore.streamStateCountWinAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, + pAggSup->windowCount, + (void**)&pCurWin->winInfo.pStatePos, &size); } if (code == TSDB_CODE_SUCCESS) { pCurWin->winInfo.isOutput = true; } - pCurWin->pWindowCount= - (COUNT_TYPE*) ((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - sizeof(COUNT_TYPE))); + pCurWin->pWindowCount = + (COUNT_TYPE*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - sizeof(COUNT_TYPE))); if (*pCurWin->pWindowCount == pAggSup->windowCount) { pBuffInfo->rebuildWindow = true; @@ -133,7 +133,7 @@ static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowI int32_t start, int32_t rows, int32_t maxRows, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) { SSessionKey sWinKey = pWinInfo->winInfo.sessionWin; - int32_t num = 0; + int32_t num = 0; for (int32_t i = start; i < rows; i++) { if (pTs[i] < pWinInfo->winInfo.sessionWin.win.ekey) { num++; @@ -181,7 +181,7 @@ void getCountWinRange(SStreamAggSupporter* pAggSup, const SSessionKey* pKey, ESt pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pAggSup->pState, pKey); } SSessionKey tmpKey = {.groupId = pKey->groupId, .win.ekey = INT64_MIN, .win.skey = INT64_MIN}; - int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0); + int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0); if (code != TSDB_CODE_SUCCESS) { pAggSup->stateStore.streamStateFreeCur(pCur); return; @@ -239,7 +239,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl TSKEY* startTsCols = (int64_t*)pStartTsCol->pData; blockDataEnsureCapacity(pAggSup->pScanBlock, rows * 2); SStreamStateCur* pCur = NULL; - COUNT_TYPE slidingRows = 0; + COUNT_TYPE slidingRows = 0; for (int32_t i = 0; i < rows;) { if (pInfo->ignoreExpiredData && @@ -269,7 +269,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } getCountWinRange(pAggSup, &curWin.winInfo.sessionWin, STREAM_DELETE_DATA, &range); range.win.skey = TMIN(startTsCols[i], range.win.skey); - range.win.ekey = TMAX(startTsCols[rows-1], range.win.ekey); + range.win.ekey = TMAX(startTsCols[rows - 1], range.win.ekey); uint64_t uid = 0; appendDataToSpecialBlock(pAggSup->pScanBlock, &range.win.skey, &range.win.ekey, &uid, &range.groupId, NULL); break; @@ -289,8 +289,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) { code = saveResult(curWin.winInfo, pStUpdated); if (code != TSDB_CODE_SUCCESS) { - qError("%s do stream count aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo), - tstrerror(code)); + qError("%s do stream count aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); break; } } @@ -397,13 +396,14 @@ void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato int32_t mapSize = 0; buf = taosDecodeFixedI32(buf, &mapSize); for (int32_t i = 0; i < mapSize; i++) { - SSessionKey key = {0}; + SSessionKey key = {0}; SCountWindowInfo curWin = {0}; buf = decodeSSessionKey(buf, &key); SBuffInfo buffInfo = {.rebuildWindow = false, .winBuffOp = NONE_WINDOW, .pCur = NULL}; setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo); buf = decodeSResultWindowInfo(buf, &curWin.winInfo, pInfo->streamAggSup.resultRowSize); - tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); + tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, + sizeof(SResultWindowInfo)); } // 2.twAggSup @@ -441,9 +441,9 @@ void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) { uint64_t* gpDatas = (uint64_t*)pGroupCol->pData; SRowBuffPos* pPos = NULL; - int32_t size = 0; + int32_t size = 0; for (int32_t i = 0; i < pBlock->info.rows; i++) { - SSessionKey key = {.groupId = gpDatas[i], .win.skey = startDatas[i], .win.ekey = endDatas[i]}; + SSessionKey key = {.groupId = gpDatas[i], .win.skey = startDatas[i], .win.ekey = endDatas[i]}; SStreamStateCur* pCur = NULL; if (isSlidingCountWindow(pAggSup)) { pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &key, pAggSup->windowCount); @@ -452,7 +452,7 @@ void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) { } while (1) { SSessionKey tmpKey = {.groupId = gpDatas[i], .win.skey = INT64_MIN, .win.ekey = INT64_MIN}; - int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, (void **)&pPos, &size); + int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, (void**)&pPos, &size); if (code != TSDB_CODE_SUCCESS || tmpKey.win.skey > endDatas[i]) { pAggSup->stateStore.streamStateFreeCur(pCur); break; @@ -482,7 +482,7 @@ void doDeleteCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SAr SSessionKey key = {.win.skey = startDatas[i], .win.ekey = endDatas[i], .groupId = gpDatas[i]}; while (1) { SSessionKey curWin = {0}; - int32_t code = pAggSup->stateStore.streamStateCountGetKeyByRange(pAggSup->pState, &key, &curWin); + int32_t code = pAggSup->stateStore.streamStateCountGetKeyByRange(pAggSup->pState, &key, &curWin); if (code == TSDB_CODE_FAILED) { break; } @@ -511,11 +511,11 @@ void deleteCountWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHa } static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) { - SExprSupp* pSup = &pOperator->exprSupp; - SStreamCountAggOperatorInfo* pInfo = pOperator->info; - SOptrBasicInfo* pBInfo = &pInfo->binfo; - SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExprSupp* pSup = &pOperator->exprSupp; + SStreamCountAggOperatorInfo* pInfo = pOperator->info; + SOptrBasicInfo* pBInfo = &pInfo->binfo; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; qDebug("stask:%s %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status); if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -635,7 +635,7 @@ void streamCountReloadState(SOperatorInfo* pOperator) { int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_COUNT_OP_STATE_NAME, strlen(STREAM_COUNT_OP_STATE_NAME), &pBuf, &size); - TSKEY ts = *(TSKEY*)pBuf; + TSKEY ts = *(TSKEY*)pBuf; pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts); taosMemoryFree(pBuf); @@ -650,11 +650,13 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys SExecTaskInfo* pTaskInfo, SReadHandle* pHandle) { SCountWinodwPhysiNode* pCountNode = (SCountWinodwPhysiNode*)pPhyNode; int32_t numOfCols = 0; - int32_t code = TSDB_CODE_OUT_OF_MEMORY; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamCountAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamCountAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - goto _error; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } pOperator->pTaskInfo = pTaskInfo; @@ -664,18 +666,14 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pCountNode->window.pExprs, NULL, &numOfScalar); code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); } SExprSupp* pExpSup = &pOperator->exprSupp; SExprInfo* pExprInfo = createExprInfo(pCountNode->window.pFuncs, NULL, &numOfCols); SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); pInfo->twAggSup = (STimeWindowAggSupp){ .waterMark = pCountNode->window.watermark, @@ -686,12 +684,11 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys }; pInfo->primaryTsIndex = ((SColumnNode*)pCountNode->window.pTspk)->slotId; - code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, - pTaskInfo->streamInfo.pState, sizeof(COUNT_TYPE), 0, &pTaskInfo->storageAPI.stateStore, pHandle, - &pInfo->twAggSup, GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState, + sizeof(COUNT_TYPE), 0, &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup, + GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex); + TSDB_CHECK_CODE(code, lino, _error); + pInfo->streamAggSup.windowCount = pCountNode->windowCount; pInfo->streamAggSup.windowSliding = pCountNode->windowSliding; @@ -709,7 +706,8 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->dataVersion = 0; pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey)); if (!pInfo->historyWins) { - goto _error; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT); @@ -735,8 +733,12 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys setOperatorStreamStateFn(pOperator, streamCountReleaseState, streamCountReloadState); if (downstream) { - initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic); + code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, + &pInfo->twAggSup, &pInfo->basic); + TSDB_CHECK_CODE(code, lino, _error); + code = appendDownstream(pOperator, &downstream, 1); + TSDB_CHECK_CODE(code, lino, _error); } return pOperator; @@ -747,6 +749,6 @@ _error: taosMemoryFreeClear(pOperator); pTaskInfo->code = code; + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return NULL; } - diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 1809d0cc67..e6ff4fe242 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -28,9 +28,9 @@ #include "tlog.h" #include "ttime.h" -#define IS_NORMAL_EVENT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT) -#define STREAM_EVENT_OP_STATE_NAME "StreamEventHistoryState" -#define STREAM_EVENT_OP_CHECKPOINT_NAME "StreamEventOperator_Checkpoint" +#define IS_NORMAL_EVENT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT) +#define STREAM_EVENT_OP_STATE_NAME "StreamEventHistoryState" +#define STREAM_EVENT_OP_CHECKPOINT_NAME "StreamEventOperator_Checkpoint" typedef struct SEventWinfowFlag { bool startFlag; @@ -86,10 +86,11 @@ void destroyStreamEventOperatorInfo(void* param) { void setEventWindowFlag(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo) { char* pFlagInfo = (char*)pWinInfo->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize); - pWinInfo->pWinFlag = (SEventWinfowFlag*) pFlagInfo; + pWinInfo->pWinFlag = (SEventWinfowFlag*)pFlagInfo; } -void setEventWindowInfo(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SRowBuffPos* pPos, SEventWindowInfo* pWinInfo) { +void setEventWindowInfo(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SRowBuffPos* pPos, + SEventWindowInfo* pWinInfo) { pWinInfo->winInfo.sessionWin = *pKey; pWinInfo->winInfo.pStatePos = pPos; setEventWindowFlag(pAggSup, pWinInfo); @@ -122,22 +123,23 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI pCurWin->winInfo.sessionWin.groupId = groupId; pCurWin->winInfo.sessionWin.win.skey = ts; pCurWin->winInfo.sessionWin.win.ekey = ts; - SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin); - SSessionKey leftWinKey = {.groupId = groupId}; - void* pVal = NULL; - int32_t len = 0; + SStreamStateCur* pCur = + pAggSup->stateStore.streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin); + SSessionKey leftWinKey = {.groupId = groupId}; + void* pVal = NULL; + int32_t len = 0; code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &leftWinKey, &pVal, &len); - if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &leftWinKey.win) ) { + if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &leftWinKey.win)) { bool inWin = isInTimeWindow(&leftWinKey.win, ts, 0); setEventWindowInfo(pAggSup, &leftWinKey, pVal, pCurWin); - if(inWin || (pCurWin->pWinFlag->startFlag && !pCurWin->pWinFlag->endFlag) ) { + if (inWin || (pCurWin->pWinFlag->startFlag && !pCurWin->pWinFlag->endFlag)) { pCurWin->winInfo.isOutput = !isWindowIncomplete(pCurWin); goto _end; } } pAggSup->stateStore.streamStateFreeCur(pCur); pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin); - SSessionKey rightWinKey = {.groupId = groupId}; + SSessionKey rightWinKey = {.groupId = groupId}; code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &rightWinKey, &pVal, &len); bool inWin = isInTimeWindow(&rightWinKey.win, ts, 0); if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &rightWinKey.win) && (inWin || (start && !end))) { @@ -149,7 +151,7 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI } } - SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId}; + SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId}; pAggSup->stateStore.streamStateSessionAllocWinBuffByNextPosition(pAggSup->pState, pCur, &winKey, &pVal, &len); setEventWindowInfo(pAggSup, &winKey, pVal, pCurWin); pCurWin->pWinFlag->startFlag = start; @@ -164,7 +166,7 @@ _end: if (code != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_KEY_INVALID(pNextWinKey); } - if(pCurWin->winInfo.pStatePos->needFree) { + if (pCurWin->winInfo.pStatePos->needFree) { pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->winInfo.sessionWin); } pAggSup->stateStore.streamStateFreeCur(pCur); @@ -173,14 +175,14 @@ _end: } int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo, SSessionKey* pNextWinKey, - TSKEY* pTsData, bool* starts, bool* ends, int32_t rows, int32_t start, SSHashObj* pResultRows, - SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) { + TSKEY* pTsData, bool* starts, bool* ends, int32_t rows, int32_t start, + SSHashObj* pResultRows, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) { *pRebuild = false; - if (!pWinInfo->pWinFlag->startFlag && !(starts[start]) ) { + if (!pWinInfo->pWinFlag->startFlag && !(starts[start])) { return 1; } - TSKEY maxTs = INT64_MAX; + TSKEY maxTs = INT64_MAX; STimeWindow* pWin = &pWinInfo->winInfo.sessionWin.win; if (pWinInfo->pWinFlag->endFlag) { maxTs = pWin->ekey + 1; @@ -219,7 +221,7 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW memcpy(pWinInfo->winInfo.pStatePos->pKey, &pWinInfo->winInfo.sessionWin, sizeof(SSessionKey)); if (ends[i]) { - if (pWinInfo->pWinFlag->endFlag && pWin->skey <= pTsData[i] && pTsData[i] < pWin->ekey ) { + if (pWinInfo->pWinFlag->endFlag && pWin->skey <= pTsData[i] && pTsData[i] < pWin->ekey) { *pRebuild = true; } return i + 1 - start; @@ -230,26 +232,28 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW static int32_t compactEventWindow(SOperatorInfo* pOperator, SEventWindowInfo* pCurWin, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool addGap) { - SExprSupp* pSup = &pOperator->exprSupp; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - int32_t winNum = 0; - SStreamEventAggOperatorInfo* pInfo = pOperator->info; - SResultRow* pCurResult = NULL; - int32_t numOfOutput = pOperator->exprSupp.numOfExprs; - SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + SExprSupp* pSup = &pOperator->exprSupp; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + int32_t winNum = 0; + SStreamEventAggOperatorInfo* pInfo = pOperator->info; + SResultRow* pCurResult = NULL; + int32_t numOfOutput = pOperator->exprSupp.numOfExprs; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; while (1) { if (!pCurWin->pWinFlag->startFlag || pCurWin->pWinFlag->endFlag) { break; } SEventWindowInfo nextWinInfo = {0}; getNextSessionWinInfo(pAggSup, pStUpdated, &pCurWin->winInfo, &nextWinInfo.winInfo); - if (!IS_VALID_SESSION_WIN(nextWinInfo.winInfo) || !inWinRange(&pAggSup->winRange, &nextWinInfo.winInfo.sessionWin.win)) { + if (!IS_VALID_SESSION_WIN(nextWinInfo.winInfo) || + !inWinRange(&pAggSup->winRange, &nextWinInfo.winInfo.sessionWin.win)) { releaseOutputBuf(pAggSup->pState, nextWinInfo.winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore); break; } setEventWindowFlag(pAggSup, &nextWinInfo); - compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, &pCurWin->winInfo, &nextWinInfo.winInfo, pStUpdated, pStDeleted, false); + compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, &pCurWin->winInfo, &nextWinInfo.winInfo, pStUpdated, + pStDeleted, false); pCurWin->pWinFlag->endFlag = nextWinInfo.pWinFlag->endFlag; winNum++; } @@ -275,7 +279,7 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl int32_t winRows = 0; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; SColumnInfoData* pColStart = NULL; - SColumnInfoData* pColEnd = NULL; + SColumnInfoData* pColEnd = NULL; pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version); pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow; @@ -290,7 +294,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl return; } - SFilterColumnParam paramStart = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), .pDataBlock = pSDataBlock->pDataBlock}; + SFilterColumnParam paramStart = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), + .pDataBlock = pSDataBlock->pDataBlock}; code = filterSetDataFromSlotId(pInfo->pStartCondInfo, ¶mStart); if (code != TSDB_CODE_SUCCESS) { qError("set data from start slotId error."); @@ -299,7 +304,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl int32_t statusStart = 0; filterExecute(pInfo->pStartCondInfo, pSDataBlock, &pColStart, NULL, paramStart.numOfCols, &statusStart); - SFilterColumnParam paramEnd = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), .pDataBlock = pSDataBlock->pDataBlock}; + SFilterColumnParam paramEnd = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), + .pDataBlock = pSDataBlock->pDataBlock}; code = filterSetDataFromSlotId(pInfo->pEndCondInfo, ¶mEnd); if (code != TSDB_CODE_SUCCESS) { qError("set data from end slotId error."); @@ -320,20 +326,23 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl int32_t winIndex = 0; bool allEqual = true; SEventWindowInfo curWin = {0}; - SSessionKey nextWinKey = {0}; - setEventOutputBuf(pAggSup, tsCols, groupId, (bool*)pColStart->pData, (bool*)pColEnd->pData, i, rows, &curWin, &nextWinKey); + SSessionKey nextWinKey = {0}; + setEventOutputBuf(pAggSup, tsCols, groupId, (bool*)pColStart->pData, (bool*)pColEnd->pData, i, rows, &curWin, + &nextWinKey); setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo); bool rebuild = false; - winRows = updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData, rows, i, - pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild); + winRows = + updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData, + rows, i, pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild); ASSERT(winRows >= 1); if (rebuild) { uint64_t uid = 0; appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey, - &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); + &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); doDeleteEventWindow(pAggSup, pSeUpdated, &curWin.winInfo.sessionWin); - if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator) && !isWindowIncomplete(&curWin)) { + if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator) && + !isWindowIncomplete(&curWin)) { saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin); } releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore); @@ -603,7 +612,8 @@ void streamEventReleaseState(SOperatorInfo* pOperator) { char* pBuff = taosMemoryCalloc(1, resSize); memcpy(pBuff, pInfo->historyWins->pData, winSize); memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY)); - qDebug("===stream=== event window operator relase state. save result count:%d", (int32_t)taosArrayGetSize(pInfo->historyWins)); + qDebug("===stream=== event window operator relase state. save result count:%d", + (int32_t)taosArrayGetSize(pInfo->historyWins)); pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_STATE_NAME, strlen(STREAM_EVENT_OP_STATE_NAME), pBuff, resSize); pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState); @@ -647,7 +657,7 @@ void streamEventReloadState(SOperatorInfo* pOperator) { qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey, pSeKeyBuf[i].groupId, i); getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo.winInfo); - //event window has been deleted + // event window has been deleted if (!IS_VALID_SESSION_WIN(curInfo.winInfo)) { continue; } @@ -659,7 +669,7 @@ void streamEventReloadState(SOperatorInfo* pOperator) { compactEventWindow(pOperator, &curInfo, pInfo->pSeUpdated, pInfo->pSeDeleted, false); qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey, - curInfo.winInfo.sessionWin.groupId); + curInfo.winInfo.sessionWin.groupId); if (IS_VALID_SESSION_WIN(curInfo.winInfo)) { saveSessionOutputBuf(pAggSup, &curInfo.winInfo); } @@ -693,6 +703,7 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys SStreamEventWinodwPhysiNode* pEventNode = (SStreamEventWinodwPhysiNode*)pPhyNode; int32_t tsSlotId = ((SColumnNode*)pEventNode->window.pTspk)->slotId; int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamEventAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamEventAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -783,20 +794,18 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAgg, NULL, destroyStreamEventOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamEventReleaseState, streamEventReloadState); - initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic); + code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, + &pInfo->twAggSup, &pInfo->basic); + TSDB_CHECK_CODE(code, lino, _error); + code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); code = filterInitFromNode((SNode*)pEventNode->pStartCond, &pInfo->pStartCondInfo, 0); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); + code = filterInitFromNode((SNode*)pEventNode->pEndCond, &pInfo->pEndCondInfo, 0); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); return pOperator; @@ -804,5 +813,6 @@ _error: destroyStreamEventOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); pTaskInfo->code = code; + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return NULL; } diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 97cb20a7ea..1650518f84 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -28,14 +28,18 @@ #include "tlog.h" #include "ttime.h" -#define IS_FINAL_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) -#define IS_MID_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) -#define IS_NORMAL_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) +#define IS_FINAL_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) +#define IS_MID_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) +#define IS_NORMAL_INTERVAL_OP(op) \ + ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || \ + (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) -#define IS_FINAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) -#define IS_NORMAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) +#define IS_FINAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) +#define IS_NORMAL_SESSION_OP(op) \ + ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || \ + (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) -#define IS_NORMAL_STATE_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) +#define IS_NORMAL_STATE_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) #define DEAULT_DELETE_MARK INT64_MAX #define STREAM_INTERVAL_OP_STATE_NAME "StreamIntervalHistoryState" @@ -263,14 +267,16 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa if (chIds) { int32_t childId = getChildIndex(pBlock); if (pInvalidWins) { - qDebug("===stream===save invalid delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d", winRes.ts, winRes.groupId, childId); + qDebug("===stream===save invalid delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d", winRes.ts, + winRes.groupId, childId); taosHashPut(pInvalidWins, &winRes, sizeof(SWinKey), NULL, 0); } SArray* chArray = *(void**)chIds; int32_t index = taosArraySearchIdx(chArray, &childId, compareInt32Val, TD_EQ); if (index != -1) { - qDebug("===stream===try push delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d ,continue", win.skey, winGpId, childId); + qDebug("===stream===try push delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d ,continue", win.skey, + winGpId, childId); getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); continue; } @@ -390,7 +396,7 @@ static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWin } void destroyFlusedPos(void* pRes) { - SRowBuffPos* pPos = (SRowBuffPos*) pRes; + SRowBuffPos* pPos = (SRowBuffPos*)pRes; if (!pPos->needFree && !pPos->pRowBuff) { taosMemoryFreeClear(pPos->pKey); taosMemoryFree(pPos); @@ -626,17 +632,17 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB blockDataUpdateTsWindow(pBlock, 0); } -static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval, SArray* pPullWins, - int32_t numOfCh, SOperatorInfo* pOperator) { +static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval, + SArray* pPullWins, int32_t numOfCh, SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; - SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); - TSKEY* tsData = (TSKEY*)pStartCol->pData; - SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); - TSKEY* tsEndData = (TSKEY*)pEndCol->pData; - SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); - uint64_t* groupIdData = (uint64_t*)pGroupCol->pData; - int32_t chId = getChildIndex(pBlock); - bool res = false; + SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); + TSKEY* tsData = (TSKEY*)pStartCol->pData; + SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); + TSKEY* tsEndData = (TSKEY*)pEndCol->pData; + SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); + uint64_t* groupIdData = (uint64_t*)pGroupCol->pData; + int32_t chId = getChildIndex(pBlock); + bool res = false; for (int32_t i = 0; i < pBlock->info.rows; i++) { TSKEY winTs = tsData[i]; while (winTs <= tsEndData[i]) { @@ -652,7 +658,7 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina // pull data is over taosArrayDestroy(chArray); taosHashRemove(pMap, &winRes, sizeof(SWinKey)); - res =true; + res = true; qDebug("===stream===retrive pull data over.window %" PRId64, winRes.ts); void* pFinalCh = taosHashGet(pFinalMap, &winRes, sizeof(SWinKey)); @@ -665,7 +671,8 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey}; // add pull data request - qDebug("===stream===prepare final retrive for delete window:%" PRId64 ",groupId:%" PRId64 ", size:%d", winRes.ts, winRes.groupId, numOfCh); + qDebug("===stream===prepare final retrive for delete window:%" PRId64 ",groupId:%" PRId64 ", size:%d", + winRes.ts, winRes.groupId, numOfCh); if (IS_MID_INTERVAL_OP(pOperator)) { SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info; taosArrayPush(pInfo->pMidPullDatas, &winRes); @@ -673,7 +680,7 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina taosArrayPush(pInfo->pDelWins, &winRes); addPullWindow(pMap, &winRes, numOfCh); if (pInfo->destHasPrimaryKey) { - tSimpleHashPut(pInfo->pDeletedMap,&winRes, sizeof(SWinKey), NULL, 0); + tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0); } qDebug("===stream===prepare final retrive for delete %" PRId64 ", size:%d", winRes.ts, numOfCh); } @@ -692,7 +699,7 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, i for (int32_t i = 0; i < size; i++) { SWinKey* winKey = taosArrayGet(wins, i); STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval); - void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey)); + void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey)); if (!chIds) { SPullWindowInfo pull = { .window = nextWin, .groupId = winKey->groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey}; @@ -700,7 +707,7 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, i if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) { addPullWindow(pInfo->pPullDataMap, winKey, pInfo->numOfChild); if (pInfo->destHasPrimaryKey) { - tSimpleHashPut(pInfo->pDeletedMap,winKey, sizeof(SWinKey), NULL, 0); + tSimpleHashPut(pInfo->pDeletedMap, winKey, sizeof(SWinKey), NULL, 0); } qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, pInfo->numOfChild); } @@ -753,7 +760,8 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDat if (pBlock->info.id.groupId == 0) { pBlock->info.id.groupId = groupId; void* tbname = NULL; - if (pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname, false) < 0) { + if (pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname, + false) < 0) { pBlock->info.parTbName[0] = 0; } else { memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); @@ -831,9 +839,7 @@ static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pN return startPos; } -bool hasSrcPrimaryKeyCol(SSteamOpBasicInfo* pInfo) { - return pInfo->primaryPkIndex != -1; -} +bool hasSrcPrimaryKeyCol(SSteamOpBasicInfo* pInfo) { return pInfo->primaryPkIndex != -1; } static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, uint64_t groupId, SSHashObj* pUpdatedMap, SSHashObj* pDeletedMap) { @@ -854,23 +860,22 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); tsCols = (int64_t*)pColDataInfo->pData; - void* pPkVal = NULL; - int32_t pkLen = 0; + void* pPkVal = NULL; + int32_t pkLen = 0; SColumnInfoData* pPkColDataInfo = NULL; if (hasSrcPrimaryKeyCol(&pInfo->basic)) { pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex); } if (pSDataBlock->info.window.skey != tsCols[0] || pSDataBlock->info.window.ekey != tsCols[endRowId]) { - qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 - ",maxKey %" PRId64, - pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); + qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 ",maxKey %" PRId64, + pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); blockDataUpdateTsWindow(pSDataBlock, pInfo->primaryTsIndex); // timestamp of the data is incorrect if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) { qError("table uid %" PRIu64 " data block timestamp is out of range! minKey %" PRId64 ",maxKey %" PRId64, - pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); + pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); } } @@ -918,7 +923,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) { addPullWindow(pInfo->pPullDataMap, &winRes, pInfo->numOfChild); if (pInfo->destHasPrimaryKey) { - tSimpleHashPut(pInfo->pDeletedMap,&winRes, sizeof(SWinKey), NULL, 0); + tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0); } } } else { @@ -964,7 +969,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat }; if (pInfo->destHasPrimaryKey && code == TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator)) { - tSimpleHashPut(pDeletedMap,&key, sizeof(SWinKey), NULL, 0); + tSimpleHashPut(pDeletedMap, &key, sizeof(SWinKey), NULL, 0); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { @@ -1367,7 +1372,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { pInfo->binfo.pRes->info.type = pBlock->info.type; } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || pBlock->info.type == STREAM_CLEAR) { - SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); + SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); SHashObj* finalMap = IS_FINAL_INTERVAL_OP(pOperator) ? pInfo->pFinalPullDataMap : NULL; doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, finalMap); if (IS_FINAL_INTERVAL_OP(pOperator)) { @@ -1401,7 +1406,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_RETRIEVE) { - if(!IS_FINAL_INTERVAL_OP(pOperator)) { + if (!IS_FINAL_INTERVAL_OP(pOperator)) { pInfo->recvRetrive = true; copyDataBlock(pInfo->pMidRetriveRes, pBlock); pInfo->pMidRetriveRes->info.type = STREAM_MID_RETRIEVE; @@ -1527,8 +1532,8 @@ void streamIntervalReloadState(SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL && pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) { - int32_t size = 0; - void* pBuf = NULL; + int32_t size = 0; + void* pBuf = NULL; int32_t code = pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME, strlen(STREAM_INTERVAL_OP_STATE_NAME), &pBuf, &size); if (code == 0) { @@ -1602,7 +1607,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsIndex); code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, - pInfo->pState, &pTaskInfo->storageAPI.functionStore); + pInfo->pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1757,27 +1762,37 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num } } -void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, - STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic) { +int32_t initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, + STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) { SStreamPartitionOperatorInfo* pScanInfo = downstream->info; pScanInfo->tsColIndex = tsColIndex; } if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup, pBasic); - return; + code = initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup, pBasic); + return code; } SStreamScanInfo* pScanInfo = downstream->info; pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type}; pScanInfo->pState = pAggSup->pState; if (!pScanInfo->pUpdateInfo) { - pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark, - pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo); + code = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark, + pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, + &pScanInfo->pUpdateInfo); + TSDB_CHECK_CODE(code, lino, _end); } pScanInfo->twAggSup = *pTwSup; pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo; pBasic->primaryPkIndex = pScanInfo->primaryKeyIndex; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static TSKEY sesionTs(void* pKey) { @@ -2150,8 +2165,8 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData TSKEY* endTsCols = (int64_t*)pEndTsCol->pData; - void* pPkVal = NULL; - int32_t pkLen = 0; + void* pPkVal = NULL; + int32_t pkLen = 0; SColumnInfoData* pPkColDataInfo = NULL; if (hasSrcPrimaryKeyCol(&pInfo->basic)) { pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex); @@ -2456,8 +2471,8 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa pBlock->info.id.groupId = pKey->groupId; void* tbname = NULL; - if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, - &tbname, false) < 0) { + if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname, + false) < 0) { pBlock->info.parTbName[0] = 0; } else { memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); @@ -2937,7 +2952,7 @@ void streamSessionSemiReloadState(SOperatorInfo* pOperator) { int32_t size = 0; void* pBuf = NULL; int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, - strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); + strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); @@ -2970,7 +2985,7 @@ void streamSessionReloadState(SOperatorInfo* pOperator) { int32_t size = 0; void* pBuf = NULL; int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, - strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); + strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); @@ -3020,6 +3035,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; int32_t numOfCols = 0; int32_t code = TSDB_CODE_OUT_OF_MEMORY; + int32_t lino = 0; SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -3114,8 +3130,12 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState); if (downstream) { - initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic); + code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, + &pInfo->twAggSup, &pInfo->basic); + TSDB_CHECK_CODE(code, lino, _error); + code = appendDownstream(pOperator, &downstream, 1); + TSDB_CHECK_CODE(code, lino, _error); } return pOperator; @@ -3126,6 +3146,7 @@ _error: taosMemoryFreeClear(pOperator); pTaskInfo->code = code; + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return NULL; } @@ -3576,7 +3597,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl if (!allEqual) { uint64_t uid = 0; appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey, - &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); + &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin); releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore); @@ -3683,9 +3704,9 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato SSessionKey key = {0}; SResultWindowInfo winfo = {0}; buf = decodeSSessionKey(buf, &key); - pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &winfo.sessionWin, NULL, - pAggSup->stateKeySize, compareStateKey, - (void**)&winfo.pStatePos, &pAggSup->resultRowSize); + pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &winfo.sessionWin, NULL, pAggSup->stateKeySize, + compareStateKey, (void**)&winfo.pStatePos, + &pAggSup->resultRowSize); buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); } @@ -3879,7 +3900,7 @@ void streamStateReloadState(SOperatorInfo* pOperator) { int32_t size = 0; void* pBuf = NULL; int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_STATE_OP_STATE_NAME, - strlen(STREAM_STATE_OP_STATE_NAME), &pBuf, &size); + strlen(STREAM_STATE_OP_STATE_NAME), &pBuf, &size); int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); qDebug("===stream=== reload state. get result count:%d", num); SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; @@ -3943,12 +3964,13 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; SColumnNode* pColNode = (SColumnNode*)(pStateNode->pStateKey); int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _error; + TSDB_CHECK_CODE(code, lino, _error); } pInfo->stateCol = extractColumnFromColumnNode(pColNode); @@ -3957,9 +3979,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar); code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); } pInfo->twAggSup = (STimeWindowAggSupp){ @@ -3986,9 +4006,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState, keySize, type, &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup, GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pInfo->pSeDeleted = tSimpleHashInit(64, hashFn); pInfo->pDelIterator = NULL; @@ -4001,7 +4020,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->dataVersion = 0; pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey)); if (!pInfo->historyWins) { - goto _error; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } if (pHandle) { pInfo->isHistoryOp = pHandle->fillHistory; @@ -4028,17 +4048,20 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState); - initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic); + code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, + &pInfo->twAggSup, &pInfo->basic); + TSDB_CHECK_CODE(code, lino, _error); + code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); + return pOperator; _error: destroyStreamStateOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); pTaskInfo->code = code; + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return NULL; } @@ -4205,11 +4228,12 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision, }; - pInfo->twAggSup = (STimeWindowAggSupp){.waterMark = pIntervalPhyNode->window.watermark, - .calTrigger = pIntervalPhyNode->window.triggerType, - .maxTs = INT64_MIN, - .minTs = INT64_MAX, - .deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval)}; + pInfo->twAggSup = + (STimeWindowAggSupp){.waterMark = pIntervalPhyNode->window.watermark, + .calTrigger = pIntervalPhyNode->window.triggerType, + .maxTs = INT64_MIN, + .minTs = INT64_MAX, + .deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval)}; ASSERTS(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay"); @@ -4337,7 +4361,7 @@ static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS .ts = nextWin.skey, .groupId = groupId, }; - void* chIds = taosHashGet(pInfo->pPullDataMap, &key, sizeof(SWinKey)); + void* chIds = taosHashGet(pInfo->pPullDataMap, &key, sizeof(SWinKey)); int32_t index = -1; SArray* chArray = NULL; int32_t chId = 0; @@ -4409,8 +4433,8 @@ static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS static void addMidRetriveWindow(SArray* wins, SHashObj* pMidPullMap, int32_t numOfChild) { int32_t size = taosArrayGetSize(wins); for (int32_t i = 0; i < size; i++) { - SWinKey* winKey = taosArrayGet(wins, i); - void* chIds = taosHashGet(pMidPullMap, winKey, sizeof(SWinKey)); + SWinKey* winKey = taosArrayGet(wins, i); + void* chIds = taosHashGet(pMidPullMap, winKey, sizeof(SWinKey)); if (!chIds) { addPullWindow(pMidPullMap, winKey, numOfChild); qDebug("===stream===prepare mid operator retrive for delete %" PRId64 ", size:%d", winKey->ts, numOfChild); @@ -4613,5 +4637,6 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) { void setStreamOperatorCompleted(SOperatorInfo* pOperator) { setOperatorCompleted(pOperator); - qDebug("stask:%s %s status: %d. set completed", GET_TASKID(pOperator->pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status); + qDebug("stask:%s %s status: %d. set completed", GET_TASKID(pOperator->pTaskInfo), + getStreamOpName(pOperator->operatorType), pOperator->status); } diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 8727ce0174..0dc1eafb03 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -304,6 +304,7 @@ TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t p len = colDataGetRowLength(pPkDataInfo, i); } int32_t buffLen = getKeyBuff(ts, tbUid, pPkVal, len, pInfo->pKeyBuff); + // we don't care whether the data is updated or not tScalableBfPut(pSBf, pInfo->pKeyBuff, buffLen); } } @@ -413,59 +414,119 @@ void updateInfoDestoryColseWinSBF(SUpdateInfo* pInfo) { pInfo->pCloseWinSBF = NULL; } -int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo) { +int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (!pInfo) { - return 0; + return TSDB_CODE_SUCCESS; } SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); - if (tStartEncode(&encoder) < 0) return -1; + if (tStartEncode(&encoder) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } int32_t size = taosArrayGetSize(pInfo->pTsBuckets); - if (tEncodeI32(&encoder, size) < 0) return -1; + if (tEncodeI32(&encoder, size) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + for (int32_t i = 0; i < size; i++) { TSKEY* pTs = (TSKEY*)taosArrayGet(pInfo->pTsBuckets, i); - if (tEncodeI64(&encoder, *pTs) < 0) return -1; + if (tEncodeI64(&encoder, *pTs) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } } - if (tEncodeU64(&encoder, pInfo->numBuckets) < 0) return -1; + if (tEncodeU64(&encoder, pInfo->numBuckets) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } int32_t sBfSize = taosArrayGetSize(pInfo->pTsSBFs); - if (tEncodeI32(&encoder, sBfSize) < 0) return -1; + if (tEncodeI32(&encoder, sBfSize) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } for (int32_t i = 0; i < sBfSize; i++) { SScalableBf* pSBf = taosArrayGetP(pInfo->pTsSBFs, i); - if (tScalableBfEncode(pSBf, &encoder) < 0) return -1; + if (tScalableBfEncode(pSBf, &encoder) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } } - if (tEncodeU64(&encoder, pInfo->numSBFs) < 0) return -1; - if (tEncodeI64(&encoder, pInfo->interval) < 0) return -1; - if (tEncodeI64(&encoder, pInfo->watermark) < 0) return -1; - if (tEncodeI64(&encoder, pInfo->minTS) < 0) return -1; + if (tEncodeU64(&encoder, pInfo->numSBFs) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + if (tEncodeI64(&encoder, pInfo->interval) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + if (tEncodeI64(&encoder, pInfo->watermark) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + if (tEncodeI64(&encoder, pInfo->minTS) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } - if (tScalableBfEncode(pInfo->pCloseWinSBF, &encoder) < 0) return -1; + if (tScalableBfEncode(pInfo->pCloseWinSBF, &encoder) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } int32_t mapSize = taosHashGetSize(pInfo->pMap); - if (tEncodeI32(&encoder, mapSize) < 0) return -1; + if (tEncodeI32(&encoder, mapSize) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } void* pIte = NULL; size_t keyLen = 0; while ((pIte = taosHashIterate(pInfo->pMap, pIte)) != NULL) { void* key = taosHashGetKey(pIte, &keyLen); - if (tEncodeU64(&encoder, *(uint64_t*)key) < 0) return -1; + if (tEncodeU64(&encoder, *(uint64_t*)key) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } int32_t valueSize = taosHashGetValueSize(pIte); - if (tEncodeBinary(&encoder, (const uint8_t*)pIte, valueSize) < 0) return -1; + if (tEncodeBinary(&encoder, (const uint8_t*)pIte, valueSize) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } } - if (tEncodeU64(&encoder, pInfo->maxDataVersion) < 0) return -1; + if (tEncodeU64(&encoder, pInfo->maxDataVersion) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } - if (tEncodeI32(&encoder, pInfo->pkColLen) < 0) return -1; - if (tEncodeI8(&encoder, pInfo->pkColType) < 0) return -1; + if (tEncodeI32(&encoder, pInfo->pkColLen) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + if (tEncodeI8(&encoder, pInfo->pkColType) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } tEndEncode(&encoder); int32_t tlen = encoder.pos; tEncoderClear(&encoder); - return tlen; + *pLen = tlen; + +_end: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) { From 51000ffe7fdeeb7e6132229429cb74992ce0ef50 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Thu, 11 Jul 2024 19:03:33 +0800 Subject: [PATCH 13/48] fix: parseOneStbRow support for jsontag --- source/libs/parser/src/parTranslater.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 9b1c8096aa..104ffcc945 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -12815,6 +12815,7 @@ static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt) int32_t numOfTags = getNumOfTags(pParFileCxt->pStbMeta); uint8_t precision = getTableInfo(pParFileCxt->pStbMeta).precision; SSchema* pSchemas = getTableTagSchema(pParFileCxt->pStbMeta); + bool isJson = false; for (int i = 0; i < sz; i++) { const char* pSql = pParFileCxt->pSql; @@ -12832,6 +12833,7 @@ static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt) // parse tag const SSchema* pTagSchema = &pSchemas[index]; + isJson = (pTagSchema->type == TSDB_DATA_TYPE_JSON); code = checkAndTrimValue(&token, pParFileCxt->tmpTokenBuf, pMsgBuf, pTagSchema->type); if (TSDB_CODE_SUCCESS == code && TK_NK_VARIABLE == token.type) { code = buildInvalidOperationMsg(pMsgBuf, "not expected row value"); @@ -12859,7 +12861,9 @@ static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt) if (TSDB_CODE_SUCCESS == code) { pParFileCxt->tagNameFilled = true; - code = tTagNew(pParFileCxt->aTagVals, 1, false, &pParFileCxt->pTag); + if (!isJson) { + code = tTagNew(pParFileCxt->aTagVals, 1, false, &pParFileCxt->pTag); + } } return code; From abe49ff61a8839d933171d47db47dc333961e51c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 15 Jul 2024 17:11:36 +0800 Subject: [PATCH 14/48] enh: add scheduler return code processing --- include/libs/scheduler/scheduler.h | 4 ---- source/libs/scheduler/src/schDbg.c | 12 ------------ source/libs/scheduler/src/schFlowCtrl.c | 13 +++++++++++++ source/libs/scheduler/src/schJob.c | 1 + source/libs/scheduler/src/scheduler.c | 24 ++++++++++++++++++------ 5 files changed, 32 insertions(+), 22 deletions(-) diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 1e2ccf8705..8ab3b898ca 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -23,8 +23,6 @@ extern "C" { #include "catalog.h" #include "planner.h" -extern tsem_t schdRspSem; - typedef struct SQueryProfileSummary { int64_t startTs; // Object created and added into the message queue int64_t endTs; // the timestamp when the task is completed @@ -101,8 +99,6 @@ void schedulerFreeJob(int64_t* job, int32_t errCode); void schedulerDestroy(void); -void schdExecCallback(SExecResult* pResult, void* param, int32_t code); - #ifdef __cplusplus } #endif diff --git a/source/libs/scheduler/src/schDbg.c b/source/libs/scheduler/src/schDbg.c index d6c2b638b8..deaaaec7c1 100644 --- a/source/libs/scheduler/src/schDbg.c +++ b/source/libs/scheduler/src/schDbg.c @@ -16,17 +16,5 @@ #include "query.h" #include "schInt.h" -tsem_t schdRspSem; SSchDebug gSCHDebug = {0}; -void schdExecCallback(SExecResult* pResult, void* param, int32_t code) { - if (code) { - pResult->code = code; - } - - *(SExecResult*)param = *pResult; - - taosMemoryFree(pResult); - - tsem_post(&schdRspSem); -} diff --git a/source/libs/scheduler/src/schFlowCtrl.c b/source/libs/scheduler/src/schFlowCtrl.c index 8c2b65e125..1f6e7fbd63 100644 --- a/source/libs/scheduler/src/schFlowCtrl.c +++ b/source/libs/scheduler/src/schFlowCtrl.c @@ -50,6 +50,10 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) { int32_t taskNum = taosArrayGetSize(pJob->dataSrcTasks); for (int32_t i = 0; i < taskNum; ++i) { SSchTask *pTask = *(SSchTask **)taosArrayGet(pJob->dataSrcTasks, i); + if (NULL == pTask) { + SCH_JOB_DLOG("fail to get the %dth task", i); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } sum += pTask->plan->execNodeStat.tableNum; } @@ -214,6 +218,10 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) { for (int32_t i = 0; i < taskNum; ++i) { pTask = *(SSchTask **)taosArrayGet(ctrl->taskList, i); + if (NULL == pTask) { + SCH_JOB_ELOG("fail to get the %dth task", i); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode); if (pTask->plan->execNodeStat.tableNum > remainNum && ctrl->execTaskNum > 0) { @@ -243,6 +251,11 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) { if (i < (taskNum - 1)) { SSchTask *pLastTask = *(SSchTask **)taosArrayGetLast(ctrl->taskList); + if (NULL == pLastTask) { + SCH_JOB_ELOG("fail to get the last task, num:%d", taosArrayGetSize(ctrl->taskList)); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + if (remainNum < pLastTask->plan->execNodeStat.tableNum) { SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d, smallestInList:%d", ep->fqdn, ep->port, ctrl->tableNumSum, ctrl->execTaskNum, pLastTask->plan->execNodeStat.tableNum); diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 278768981a..9ad1c1ff30 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -47,6 +47,7 @@ void schUpdateJobErrCode(SSchJob *pJob, int32_t errCode) { return; _return: + SCH_JOB_DLOG("job errCode updated to %s", tstrerror(errCode)); } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index fc92be8214..53ffc3bc6e 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -108,14 +108,26 @@ int32_t schedulerGetTasksStatus(int64_t jobId, SArray *pSub) { for (int32_t i = pJob->levelNum - 1; i >= 0; --i) { SSchLevel *pLevel = taosArrayGet(pJob->levels, i); + if (NULL == pLevel) { + qError("failed to get level %d", i); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } for (int32_t m = 0; m < pLevel->taskNum; ++m) { SSchTask *pTask = taosArrayGet(pLevel->subTasks, m); + if (NULL == pTask) { + qError("failed to get task %d, total: %d", m, pLevel->taskNum); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SQuerySubDesc subDesc = {0}; subDesc.tid = pTask->taskId; strcpy(subDesc.status, jobTaskStatusStr(pTask->status)); - taosArrayPush(pSub, &subDesc); + if (NULL == taosArrayPush(pSub, &subDesc)) { + qError("taosArrayPush task %d failed, error: %x, ", m, terrno); + SCH_ERR_JRET(terrno); + } } } @@ -141,7 +153,7 @@ int32_t schedulerUpdatePolicy(int32_t policy) { qDebug("schedule policy updated to %d", schMgmt.cfg.schPolicy); break; default: - return TSDB_CODE_TSC_INVALID_INPUT; + SCH_RET(TSDB_CODE_TSC_INVALID_INPUT); } return TSDB_CODE_SUCCESS; @@ -159,14 +171,14 @@ void schedulerFreeJob(int64_t *jobId, int32_t errCode) { SSchJob *pJob = schAcquireJob(*jobId); if (NULL == pJob) { - qDebug("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId); + qWarn("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId); return; } SCH_JOB_DLOG("start to free job 0x%" PRIx64 ", code:%s", *jobId, tstrerror(errCode)); - schHandleJobDrop(pJob, errCode); + (void)schHandleJobDrop(pJob, errCode); // ignore any error - schReleaseJob(*jobId); + (void)schReleaseJob(*jobId); // ignore error *jobId = 0; } @@ -182,7 +194,7 @@ void schedulerDestroy(void) { if (refId == 0) { break; } - taosRemoveRef(schMgmt.jobRef, pJob->refId); + (void)taosRemoveRef(schMgmt.jobRef, pJob->refId); // ignore error pJob = taosIterateRef(schMgmt.jobRef, refId); } From 5b17b3226c8bc508bbaaba778ab169852eee888f Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 15 Jul 2024 17:31:52 +0800 Subject: [PATCH 15/48] fix: old input password buffer size only 20 bytes --- tools/shell/src/shellArguments.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 8368837cc4..1eb61d2394 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -311,7 +311,7 @@ static void shellInitArgs(int argc, char *argv[]) { if (strlen(argv[i]) == 2) { printf("Enter password: "); taosSetConsoleEcho(false); - if (scanf("%20s", shell.args.password) > 1) { + if (scanf("%128s", shell.args.password) > 1) { fprintf(stderr, "password reading error\n"); } taosSetConsoleEcho(true); From 513bbc50b603ab283a5fbe801901ba54b0b36d8d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 15 Jul 2024 17:35:00 +0800 Subject: [PATCH 16/48] enh: refact encode and decode function --- include/util/tencode.h | 157 +++++++++++++++++++++----------------- source/util/src/tencode.c | 14 +++- 2 files changed, 95 insertions(+), 76 deletions(-) diff --git a/include/util/tencode.h b/include/util/tencode.h index b8da040689..a50a9ea8e4 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -18,6 +18,7 @@ #include "tcoding.h" #include "tlist.h" +#include "tutil.h" #ifdef __cplusplus extern "C" { @@ -49,30 +50,6 @@ typedef struct { #define tPut(TYPE, BUF, VAL) ((TYPE*)(BUF))[0] = (VAL) #define tGet(TYPE, BUF, VAL) (VAL) = ((TYPE*)(BUF))[0] -#define tRPut16(PDEST, PSRC) \ - ((uint8_t*)(PDEST))[0] = ((uint8_t*)(PSRC))[1]; \ - ((uint8_t*)(PDEST))[1] = ((uint8_t*)(PSRC))[0]; - -#define tRPut32(PDEST, PSRC) \ - ((uint8_t*)(PDEST))[0] = ((uint8_t*)(PSRC))[3]; \ - ((uint8_t*)(PDEST))[1] = ((uint8_t*)(PSRC))[2]; \ - ((uint8_t*)(PDEST))[2] = ((uint8_t*)(PSRC))[1]; \ - ((uint8_t*)(PDEST))[3] = ((uint8_t*)(PSRC))[0]; - -#define tRPut64(PDEST, PSRC) \ - ((uint8_t*)(PDEST))[0] = ((uint8_t*)(PSRC))[7]; \ - ((uint8_t*)(PDEST))[1] = ((uint8_t*)(PSRC))[6]; \ - ((uint8_t*)(PDEST))[2] = ((uint8_t*)(PSRC))[5]; \ - ((uint8_t*)(PDEST))[3] = ((uint8_t*)(PSRC))[4]; \ - ((uint8_t*)(PDEST))[4] = ((uint8_t*)(PSRC))[3]; \ - ((uint8_t*)(PDEST))[5] = ((uint8_t*)(PSRC))[2]; \ - ((uint8_t*)(PDEST))[6] = ((uint8_t*)(PSRC))[1]; \ - ((uint8_t*)(PDEST))[7] = ((uint8_t*)(PSRC))[0]; - -#define tRGet16 tRPut16 -#define tRGet32 tRPut32 -#define tRGet64 tRPut64 - #define TD_CODER_POS(CODER) ((CODER)->pos) #define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos) #define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE)) @@ -149,18 +126,22 @@ static int32_t tDecodeCStr(SDecoder* pCoder, char** val); static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val); /* ------------------------ IMPL ------------------------ */ -#define TD_ENCODE_MACRO(CODER, VAL, TYPE, BITS) \ - if ((CODER)->data) { \ - if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, sizeof(VAL))) return -1; \ - tPut(TYPE, TD_CODER_CURRENT(CODER), (VAL)); \ - } \ - TD_CODER_MOVE_POS(CODER, sizeof(VAL)); \ +#define TD_ENCODE_MACRO(CODER, VAL, TYPE, BITS) \ + if ((CODER)->data) { \ + if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, sizeof(VAL))) { \ + return TSDB_CODE_OUT_OF_RANGE; \ + } \ + tPut(TYPE, TD_CODER_CURRENT(CODER), (VAL)); \ + } \ + TD_CODER_MOVE_POS(CODER, sizeof(VAL)); \ return 0; #define TD_ENCODE_VARIANT_MACRO(CODER, VAL) \ while ((VAL) >= ENCODE_LIMIT) { \ if ((CODER)->data) { \ - if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \ + if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) { \ + return TSDB_CODE_OUT_OF_RANGE; \ + } \ TD_CODER_CURRENT(CODER)[0] = ((VAL) | ENCODE_LIMIT) & 0xff; \ } \ \ @@ -169,23 +150,29 @@ static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val); } \ \ if ((CODER)->data) { \ - if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \ + if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) { \ + return TSDB_CODE_OUT_OF_RANGE; \ + } \ TD_CODER_CURRENT(CODER)[0] = (uint8_t)(VAL); \ } \ TD_CODER_MOVE_POS(CODER, 1); \ return 0; -#define TD_DECODE_MACRO(CODER, PVAL, TYPE, BITS) \ - if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, sizeof(*(PVAL)))) return -1; \ - tGet(TYPE, TD_CODER_CURRENT(CODER), *(PVAL)); \ - TD_CODER_MOVE_POS(CODER, sizeof(*(PVAL))); \ +#define TD_DECODE_MACRO(CODER, PVAL, TYPE, BITS) \ + if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, sizeof(*(PVAL)))) { \ + return TSDB_CODE_OUT_OF_RANGE; \ + } \ + tGet(TYPE, TD_CODER_CURRENT(CODER), *(PVAL)); \ + TD_CODER_MOVE_POS(CODER, sizeof(*(PVAL))); \ return 0; #define TD_DECODE_VARIANT_MACRO(CODER, PVAL, TYPE) \ int32_t i = 0; \ *(PVAL) = 0; \ for (;;) { \ - if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \ + if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) { \ + return TSDB_CODE_OUT_OF_RANGE; \ + } \ TYPE tval = TD_CODER_CURRENT(CODER)[0]; \ if (tval < ENCODE_LIMIT) { \ *(PVAL) |= (tval << (7 * i)); \ @@ -203,7 +190,9 @@ static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val); // 8 static FORCE_INLINE int32_t tEncodeU8(SEncoder* pCoder, uint8_t val) { if (pCoder->data) { - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(val))) return -1; + if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(val))) { + return TSDB_CODE_OUT_OF_RANGE; + } tPut(uint8_t, TD_CODER_CURRENT(pCoder), val); } TD_CODER_MOVE_POS(pCoder, sizeof(val)); @@ -212,7 +201,9 @@ static FORCE_INLINE int32_t tEncodeU8(SEncoder* pCoder, uint8_t val) { static FORCE_INLINE int32_t tEncodeI8(SEncoder* pCoder, int8_t val) { if (pCoder->data) { - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(val))) return -1; + if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(val))) { + return TSDB_CODE_OUT_OF_RANGE; + } tPut(int8_t, TD_CODER_CURRENT(pCoder), val); } TD_CODER_MOVE_POS(pCoder, sizeof(val)); @@ -265,10 +256,12 @@ static FORCE_INLINE int32_t tEncodeDouble(SEncoder* pCoder, double val) { } static FORCE_INLINE int32_t tEncodeBinary(SEncoder* pCoder, const uint8_t* val, uint32_t len) { - if (tEncodeU32v(pCoder, len) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeU32v(pCoder, len)); if (len) { if (pCoder->data) { - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, len)) return -1; + if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, len)) { + return TSDB_CODE_OUT_OF_RANGE; + } memcpy(TD_CODER_CURRENT(pCoder), val, len); } @@ -288,14 +281,18 @@ static FORCE_INLINE int32_t tEncodeCStr(SEncoder* pCoder, const char* val) { /* ------------------------ FOR DECODER ------------------------ */ // 8 static FORCE_INLINE int32_t tDecodeU8(SDecoder* pCoder, uint8_t* val) { - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(*val))) return -1; + if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(*val))) { + return TSDB_CODE_OUT_OF_RANGE; + } tGet(uint8_t, TD_CODER_CURRENT(pCoder), *val); TD_CODER_MOVE_POS(pCoder, sizeof(*val)); return 0; } static FORCE_INLINE int32_t tDecodeI8(SDecoder* pCoder, int8_t* val) { - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(*val))) return -1; + if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(*val))) { + return TSDB_CODE_OUT_OF_RANGE; + } tGet(int8_t, TD_CODER_CURRENT(pCoder), *val); TD_CODER_MOVE_POS(pCoder, sizeof(*val)); return 0; @@ -318,10 +315,10 @@ static FORCE_INLINE int32_t tDecodeU16v(SDecoder* pCoder, uint16_t* val) { static FORCE_INLINE int32_t tDecodeI16v(SDecoder* pCoder, int16_t* val) { uint16_t tval; - if (tDecodeU16v(pCoder, &tval) < 0) { - return -1; + TAOS_CHECK_RETURN(tDecodeU16v(pCoder, &tval)); + if (val) { + *val = ZIGZAGD(int16_t, tval); } - if (val) *val = ZIGZAGD(int16_t, tval); return 0; } @@ -332,10 +329,10 @@ static FORCE_INLINE int32_t tDecodeU32v(SDecoder* pCoder, uint32_t* val) { static FORCE_INLINE int32_t tDecodeI32v(SDecoder* pCoder, int32_t* val) { uint32_t tval; - if (tDecodeU32v(pCoder, &tval) < 0) { - return -1; + TAOS_CHECK_RETURN(tDecodeU32v(pCoder, &tval)); + if (val) { + *val = ZIGZAGD(int32_t, tval); } - if (val) *val = ZIGZAGD(int32_t, tval); return 0; } @@ -346,10 +343,10 @@ static FORCE_INLINE int32_t tDecodeU64v(SDecoder* pCoder, uint64_t* val) { static FORCE_INLINE int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val) { uint64_t tval; - if (tDecodeU64v(pCoder, &tval) < 0) { - return -1; + TAOS_CHECK_RETURN(tDecodeU64v(pCoder, &tval)); + if (val) { + *val = ZIGZAGD(int64_t, tval); } - if (val) *val = ZIGZAGD(int64_t, tval); return 0; } @@ -359,11 +356,11 @@ static FORCE_INLINE int32_t tDecodeFloat(SDecoder* pCoder, float* val) { float f; } v; - if (tDecodeU32(pCoder, &(v.ui)) < 0) { - return -1; - } + TAOS_CHECK_RETURN(tDecodeU32(pCoder, &(v.ui))); - *val = v.f; + if (val) { + *val = v.f; + } return 0; } @@ -373,20 +370,26 @@ static FORCE_INLINE int32_t tDecodeDouble(SDecoder* pCoder, double* val) { double d; } v; - if (tDecodeU64(pCoder, &(v.ui)) < 0) { - return -1; - } + TAOS_CHECK_RETURN(tDecodeU64(pCoder, &(v.ui))); - *val = v.d; + if (val) { + *val = v.d; + } return 0; } static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint32_t* len) { uint32_t length = 0; - if (tDecodeU32v(pCoder, &length) < 0) return -1; - if (len) *len = length; - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) return -1; + TAOS_CHECK_RETURN(tDecodeU32v(pCoder, &length)); + if (len) { + *len = length; + } + + if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) { + return TSDB_CODE_OUT_OF_RANGE; + } + if (val) { *val = (uint8_t*)TD_CODER_CURRENT(pCoder); } @@ -396,7 +399,7 @@ static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint3 } static FORCE_INLINE int32_t tDecodeCStrAndLen(SDecoder* pCoder, char** val, uint32_t* len) { - if (tDecodeBinary(pCoder, (uint8_t**)val, len) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeBinary(pCoder, (uint8_t**)val, len)); (*len) -= 1; return 0; } @@ -409,7 +412,7 @@ static FORCE_INLINE int32_t tDecodeCStr(SDecoder* pCoder, char** val) { static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) { char* pStr; uint32_t len; - if (tDecodeCStrAndLen(pCoder, &pStr, &len) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeCStrAndLen(pCoder, &pStr, &len)); memcpy(val, pStr, len + 1); return 0; @@ -417,13 +420,19 @@ static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) { static FORCE_INLINE int32_t tDecodeBinaryAlloc(SDecoder* pCoder, void** val, uint64_t* len) { uint64_t length = 0; - if (tDecodeU64v(pCoder, &length) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeU64v(pCoder, &length)); if (length) { if (len) *len = length; - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) return -1; + if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) { + return TSDB_CODE_OUT_OF_RANGE; + } + *val = taosMemoryMalloc(length); - if (*val == NULL) return -1; + if (*val == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(*val, TD_CODER_CURRENT(pCoder), length); TD_CODER_MOVE_POS(pCoder, length); @@ -435,13 +444,17 @@ static FORCE_INLINE int32_t tDecodeBinaryAlloc(SDecoder* pCoder, void** val, uin static FORCE_INLINE int32_t tDecodeBinaryAlloc32(SDecoder* pCoder, void** val, uint32_t* len) { uint32_t length = 0; - if (tDecodeU32v(pCoder, &length) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeU32v(pCoder, &length)); if (length) { if (len) *len = length; - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) return -1; + if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) { + return TSDB_CODE_OUT_OF_RANGE; + } *val = taosMemoryMalloc(length); - if (*val == NULL) return -1; + if (*val == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } memcpy(*val, TD_CODER_CURRENT(pCoder), length); TD_CODER_MOVE_POS(pCoder, length); @@ -452,7 +465,7 @@ static FORCE_INLINE int32_t tDecodeBinaryAlloc32(SDecoder* pCoder, void** val, u } static FORCE_INLINE int32_t tDecodeCStrAndLenAlloc(SDecoder* pCoder, char** val, uint64_t* len) { - if (tDecodeBinaryAlloc(pCoder, (void**)val, len) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeBinaryAlloc(pCoder, (void**)val, len)); (*len) -= 1; return 0; } diff --git a/source/util/src/tencode.c b/source/util/src/tencode.c index 7f8a0edfdd..d99732fedf 100644 --- a/source/util/src/tencode.c +++ b/source/util/src/tencode.c @@ -72,10 +72,14 @@ int32_t tStartEncode(SEncoder* pCoder) { SEncoderNode* pNode; if (pCoder->data) { - if (pCoder->size - pCoder->pos < sizeof(int32_t)) return -1; + if (pCoder->size - pCoder->pos < sizeof(int32_t)) { + return TSDB_CODE_OUT_OF_RANGE; + } pNode = tEncoderMalloc(pCoder, sizeof(*pNode)); - if (pNode == NULL) return -1; + if (pNode == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } pNode->data = pCoder->data; pNode->pos = pCoder->pos; @@ -119,10 +123,12 @@ int32_t tStartDecode(SDecoder* pCoder) { SDecoderNode* pNode; int32_t len; - if (tDecodeI32(pCoder, &len) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeI32(pCoder, &len)); pNode = tDecoderMalloc(pCoder, sizeof(*pNode)); - if (pNode == NULL) return -1; + if (pNode == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } pNode->data = pCoder->data; pNode->pos = pCoder->pos; From 581f21496f4509c8f075046de522e12ea0b9e119 Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Mon, 15 Jul 2024 10:03:41 +0000 Subject: [PATCH 17/48] refacto return code --- include/common/tmisce.h | 2 +- source/common/src/tmisce.c | 161 ++++++++++++++++++++++++++----------- 2 files changed, 114 insertions(+), 49 deletions(-) diff --git a/include/common/tmisce.h b/include/common/tmisce.h index 267ca814d4..39274e807c 100644 --- a/include/common/tmisce.h +++ b/include/common/tmisce.h @@ -31,7 +31,7 @@ typedef struct SCorEpSet { int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t len); int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp); -void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port); +int32_t addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port); bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2); void epsetAssign(SEpSet* dst, const SEpSet* pSrc); diff --git a/source/common/src/tmisce.c b/source/common/src/tmisce.c index d2b52b201e..7009cf26d0 100644 --- a/source/common/src/tmisce.c +++ b/source/common/src/tmisce.c @@ -34,18 +34,26 @@ int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) { pEp->port = tsServerPort; } + if (pEp->port <= 0) { + return TSDB_CODE_INVALID_PARA; + } + return 0; } -void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port) { +int32_t addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port) { if (pEpSet == NULL || fqdn == NULL || strlen(fqdn) == 0) { - return; + return TSDB_CODE_INVALID_PARA; } int32_t index = pEpSet->numOfEps; + if (index >= sizeof(pEpSet->eps) / sizeof(pEpSet->eps[0])) { + return TSDB_CODE_OUT_OF_RANGE; + } tstrncpy(pEpSet->eps[index].fqdn, fqdn, tListLen(pEpSet->eps[index].fqdn)); pEpSet->eps[index].port = port; pEpSet->numOfEps += 1; + return 0; } bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2) { @@ -131,75 +139,96 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet) { return ep; } -int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t bufLen) { - int len = snprintf(pBuf, bufLen, "epset:{"); - if (len < 0) { - return -1; - } +int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t cap) { + int32_t ret = 0; + int32_t nwrite = 0; - for (int _i = 0; (_i < pEpSet->numOfEps) && (bufLen > len); _i++) { + nwrite = snprintf(pBuf + nwrite, cap, "epset:{"); + if (nwrite <= 0 || nwrite >= cap) { + return TSDB_CODE_OUT_OF_BUFFER; + } + cap -= nwrite; + + for (int _i = 0; (_i < pEpSet->numOfEps) && (cap > 0); _i++) { int32_t ret = 0; if (_i == pEpSet->numOfEps - 1) { - ret = snprintf(pBuf + len, bufLen - len, "%d. %s:%d", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port); + ret = snprintf(pBuf + nwrite, cap, "%d. %s:%d", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port); } else { - ret = snprintf(pBuf + len, bufLen - len, "%d. %s:%d, ", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port); + ret = snprintf(pBuf + nwrite, cap, "%d. %s:%d, ", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port); } - if (ret < 0) { - return -1; + if (ret <= 0 || ret >= cap) { + return TSDB_CODE_OUT_OF_BUFFER; } - len += ret; + nwrite += ret; + cap -= ret; } - if (len < bufLen) { - /*len += */snprintf(pBuf + len, bufLen - len, "}, inUse:%d", pEpSet->inUse); + if (cap <= 0) { + return TSDB_CODE_OUT_OF_BUFFER; } + ret = snprintf(pBuf + nwrite, cap, "}, inUse:%d", pEpSet->inUse); + if (ret <= 0 || ret >= cap) { + return TSDB_CODE_OUT_OF_BUFFER; + } return TSDB_CODE_SUCCESS; } int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t startTime) { + int32_t code = 0; SJson* pJson = tjsonCreateObject(); - if (pJson == NULL) return -1; + if (pJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; + char tmp[4096] = {0}; - tjsonAddDoubleToObject(pJson, "reportVersion", 1); + TAOS_CHECK_GOTO(code = tjsonAddDoubleToObject(pJson, "reportVersion", 1), NULL, _exit); - tjsonAddIntegerToObject(pJson, "clusterId", clusterId); - tjsonAddIntegerToObject(pJson, "startTime", startTime); + TAOS_CHECK_GOTO(code = tjsonAddIntegerToObject(pJson, "clusterId", clusterId), NULL, _exit); + TAOS_CHECK_GOTO(code = tjsonAddIntegerToObject(pJson, "startTime", startTime), NULL, _exit); // Do NOT invoke the taosGetFqdn here. // this function may be invoked when memory exception occurs,so we should assume that it is running in a memory locked // environment. The lock operation by taosGetFqdn may cause this program deadlock. - tjsonAddStringToObject(pJson, "fqdn", tsLocalFqdn); + TAOS_CHECK_GOTO(code = tjsonAddStringToObject(pJson, "fqdn", tsLocalFqdn), NULL, _exit); - tjsonAddIntegerToObject(pJson, "pid", taosGetPId()); + TAOS_CHECK_GOTO(code = tjsonAddIntegerToObject(pJson, "pid", taosGetPId()), NULL, _exit); - taosGetAppName(tmp, NULL); - tjsonAddStringToObject(pJson, "appName", tmp); + code = taosGetAppName(tmp, NULL); + if (code != 0) { + code = TAOS_SYSTEM_ERROR(errno); + TAOS_CHECK_GOTO(code, NULL, _exit); + } + TAOS_CHECK_GOTO(code = tjsonAddStringToObject(pJson, "appName", tmp), NULL, _exit); if (taosGetOsReleaseName(tmp, NULL, NULL, sizeof(tmp)) == 0) { - tjsonAddStringToObject(pJson, "os", tmp); + TAOS_CHECK_GOTO(code = tjsonAddStringToObject(pJson, "os", tmp), NULL, _exit); + } else { + // do nothing } float numOfCores = 0; if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores) == 0) { - tjsonAddStringToObject(pJson, "cpuModel", tmp); - tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores); + TAOS_CHECK_GOTO(code = tjsonAddStringToObject(pJson, "cpuModel", tmp), NULL, _exit); + TAOS_CHECK_GOTO(code = tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores), NULL, _exit); } else { - tjsonAddDoubleToObject(pJson, "numOfCpu", tsNumOfCores); + TAOS_CHECK_GOTO(code = tjsonAddDoubleToObject(pJson, "numOfCpu", tsNumOfCores), NULL, _exit); } - snprintf(tmp, sizeof(tmp), "%" PRId64 " kB", tsTotalMemoryKB); - tjsonAddStringToObject(pJson, "memory", tmp); + int32_t nBytes = snprintf(tmp, sizeof(tmp), "%" PRId64 " kB", tsTotalMemoryKB); + if (nBytes <= 9 || nBytes >= sizeof(tmp)) { + TAOS_CHECK_GOTO(code = TSDB_CODE_OUT_OF_RANGE, NULL, _exit); + } + TAOS_CHECK_GOTO(code = tjsonAddStringToObject(pJson, "memory", tmp), NULL, _exit); - tjsonAddStringToObject(pJson, "version", version); - tjsonAddStringToObject(pJson, "buildInfo", buildinfo); - tjsonAddStringToObject(pJson, "gitInfo", gitinfo); + TAOS_CHECK_GOTO(code = tjsonAddStringToObject(pJson, "version", version), NULL, _exit); + TAOS_CHECK_GOTO(code = tjsonAddStringToObject(pJson, "buildInfo", buildinfo), NULL, _exit); - tjsonAddIntegerToObject(pJson, "crashSig", signum); - tjsonAddIntegerToObject(pJson, "crashTs", taosGetTimestampUs()); + TAOS_CHECK_GOTO(code = tjsonAddStringToObject(pJson, "gitInfo", gitinfo), NULL, _exit); + + TAOS_CHECK_GOTO(code = tjsonAddIntegerToObject(pJson, "crashSig", signum), NULL, _exit); + TAOS_CHECK_GOTO(code = tjsonAddIntegerToObject(pJson, "crashTs", taosGetTimestampUs()), NULL, _exit); #ifdef _TD_DARWIN_64 taosLogTraceToBuf(tmp, sizeof(tmp), 4); @@ -209,26 +238,47 @@ int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t taosLogTraceToBuf(tmp, sizeof(tmp), 8); #endif - tjsonAddStringToObject(pJson, "stackInfo", tmp); + TAOS_CHECK_GOTO(code = tjsonAddStringToObject(pJson, "stackInfo", tmp), NULL, _exit); char* pCont = tjsonToString(pJson); + if (pCont == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_CHECK_GOTO(code, NULL, _exit); + goto _exit; + } + tjsonDelete(pJson); - *pMsg = pCont; - - return TSDB_CODE_SUCCESS; + pJson = NULL; +_exit: + tjsonDelete(pJson); + TAOS_RETURN(code); } int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) { + int32_t code = 0; SConfig* pConf = taosGetCfg(); + if (pConf == NULL) { + return TSDB_CODE_INVALID_CFG; + } + int32_t numOfRows = 0; int32_t col = startCol; SConfigItem* pItem = NULL; + SConfigIter* pIter = NULL; - blockDataEnsureCapacity(pBlock, cfgGetSize(pConf)); - SConfigIter* pIter = cfgCreateIter(pConf); + int8_t locked = 0; + + TAOS_CHECK_GOTO(code = blockDataEnsureCapacity(pBlock, cfgGetSize(pConf)), NULL, _exit); + + pIter = cfgCreateIter(pConf); + if (pIter == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_CHECK_GOTO(code, NULL, _exit); + } cfgLock(pConf); + locked = 1; while ((pItem = cfgNextIter(pIter)) != NULL) { col = startCol; @@ -236,29 +286,44 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) { // GRANT_CFG_SKIP; char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE); + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, col++); - colDataSetVal(pColInfo, numOfRows, name, false); + if (pColInfo == NULL) { + code = TSDB_CODE_OUT_OF_RANGE; + TAOS_CHECK_GOTO(code, NULL, _exit); + } + + TAOS_CHECK_GOTO(code = colDataSetVal(pColInfo, numOfRows, name, false), NULL, _exit); char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; int32_t valueLen = 0; cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen); varDataSetLen(value, valueLen); + pColInfo = taosArrayGet(pBlock->pDataBlock, col++); - colDataSetVal(pColInfo, numOfRows, value, false); + if (pColInfo == NULL) { + code = TSDB_CODE_OUT_OF_RANGE; + TAOS_CHECK_GOTO(code, NULL, _exit); + } + + TAOS_CHECK_GOTO(code = colDataSetVal(pColInfo, numOfRows, value, false), NULL, _exit); char scope[TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE] = {0}; cfgDumpItemScope(pItem, &scope[VARSTR_HEADER_SIZE], TSDB_CONFIG_SCOPE_LEN, &valueLen); varDataSetLen(scope, valueLen); + pColInfo = taosArrayGet(pBlock->pDataBlock, col++); - colDataSetVal(pColInfo, numOfRows, scope, false); + if (pColInfo == NULL) { + code = TSDB_CODE_OUT_OF_RANGE; + TAOS_CHECK_GOTO(code, NULL, _exit); + } + TAOS_CHECK_GOTO(code = colDataSetVal(pColInfo, numOfRows, scope, false), NULL, _exit); numOfRows++; } - - cfgUnLock(pConf); - pBlock->info.rows = numOfRows; - +_exit: + if (locked) cfgUnLock(pConf); cfgDestroyIter(pIter); - return TSDB_CODE_SUCCESS; + TAOS_RETURN(code); } From d2b05bfbbd54e6566493cd45e1fc2ce713a91c85 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 15 Jul 2024 18:06:38 +0800 Subject: [PATCH 18/48] enh: refact more encode and decode function --- include/util/tencode.h | 290 ++++++++++++++++++++------------------ source/util/src/tencode.c | 10 +- 2 files changed, 158 insertions(+), 142 deletions(-) diff --git a/include/util/tencode.h b/include/util/tencode.h index a50a9ea8e4..419be59a92 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -47,9 +47,6 @@ typedef struct { SDecoderNode* dStack; } SDecoder; -#define tPut(TYPE, BUF, VAL) ((TYPE*)(BUF))[0] = (VAL) -#define tGet(TYPE, BUF, VAL) (VAL) = ((TYPE*)(BUF))[0] - #define TD_CODER_POS(CODER) ((CODER)->pos) #define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos) #define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE)) @@ -77,6 +74,7 @@ void tEncoderInit(SEncoder* pCoder, uint8_t* data, uint32_t size); void tEncoderClear(SEncoder* pCoder); int32_t tStartEncode(SEncoder* pCoder); void tEndEncode(SEncoder* pCoder); +static int32_t tEncodeFixed(SEncoder* pCoder, const void* val, uint32_t size); static int32_t tEncodeU8(SEncoder* pCoder, uint8_t val); static int32_t tEncodeI8(SEncoder* pCoder, int8_t val); static int32_t tEncodeU16(SEncoder* pCoder, uint16_t val); @@ -104,6 +102,7 @@ void tDecoderClear(SDecoder* SDecoder); int32_t tStartDecode(SDecoder* pCoder); void tEndDecode(SDecoder* pCoder); static bool tDecodeIsEnd(SDecoder* pCoder); +static int32_t tDecodeFixed(SDecoder* pCoder, void* val, uint32_t size); static int32_t tDecodeU8(SDecoder* pCoder, uint8_t* val); static int32_t tDecodeI8(SDecoder* pCoder, int8_t* val); static int32_t tDecodeU16(SDecoder* pCoder, uint16_t* val); @@ -126,111 +125,65 @@ static int32_t tDecodeCStr(SDecoder* pCoder, char** val); static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val); /* ------------------------ IMPL ------------------------ */ -#define TD_ENCODE_MACRO(CODER, VAL, TYPE, BITS) \ - if ((CODER)->data) { \ - if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, sizeof(VAL))) { \ - return TSDB_CODE_OUT_OF_RANGE; \ - } \ - tPut(TYPE, TD_CODER_CURRENT(CODER), (VAL)); \ - } \ - TD_CODER_MOVE_POS(CODER, sizeof(VAL)); \ - return 0; - -#define TD_ENCODE_VARIANT_MACRO(CODER, VAL) \ - while ((VAL) >= ENCODE_LIMIT) { \ - if ((CODER)->data) { \ - if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) { \ - return TSDB_CODE_OUT_OF_RANGE; \ - } \ - TD_CODER_CURRENT(CODER)[0] = ((VAL) | ENCODE_LIMIT) & 0xff; \ - } \ - \ - (VAL) >>= 7; \ - TD_CODER_MOVE_POS(CODER, 1); \ - } \ - \ - if ((CODER)->data) { \ - if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) { \ - return TSDB_CODE_OUT_OF_RANGE; \ - } \ - TD_CODER_CURRENT(CODER)[0] = (uint8_t)(VAL); \ - } \ - TD_CODER_MOVE_POS(CODER, 1); \ - return 0; - -#define TD_DECODE_MACRO(CODER, PVAL, TYPE, BITS) \ - if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, sizeof(*(PVAL)))) { \ - return TSDB_CODE_OUT_OF_RANGE; \ - } \ - tGet(TYPE, TD_CODER_CURRENT(CODER), *(PVAL)); \ - TD_CODER_MOVE_POS(CODER, sizeof(*(PVAL))); \ - return 0; - -#define TD_DECODE_VARIANT_MACRO(CODER, PVAL, TYPE) \ - int32_t i = 0; \ - *(PVAL) = 0; \ - for (;;) { \ - if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) { \ - return TSDB_CODE_OUT_OF_RANGE; \ - } \ - TYPE tval = TD_CODER_CURRENT(CODER)[0]; \ - if (tval < ENCODE_LIMIT) { \ - *(PVAL) |= (tval << (7 * i)); \ - TD_CODER_MOVE_POS(pCoder, 1); \ - break; \ - } else { \ - *(PVAL) |= (((tval) & (ENCODE_LIMIT - 1)) << (7 * i)); \ - i++; \ - TD_CODER_MOVE_POS(pCoder, 1); \ - } \ - } \ - \ - return 0; - -// 8 -static FORCE_INLINE int32_t tEncodeU8(SEncoder* pCoder, uint8_t val) { +static FORCE_INLINE int32_t tEncodeFixed(SEncoder* pCoder, const void* val, uint32_t size) { if (pCoder->data) { - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(val))) { - return TSDB_CODE_OUT_OF_RANGE; + if (pCoder->pos + size > pCoder->size) { + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } - tPut(uint8_t, TD_CODER_CURRENT(pCoder), val); + memcpy(pCoder->data + pCoder->pos, val, size); } - TD_CODER_MOVE_POS(pCoder, sizeof(val)); - return 0; + + pCoder->pos += size; + TAOS_RETURN(0); } -static FORCE_INLINE int32_t tEncodeI8(SEncoder* pCoder, int8_t val) { - if (pCoder->data) { - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(val))) { - return TSDB_CODE_OUT_OF_RANGE; - } - tPut(int8_t, TD_CODER_CURRENT(pCoder), val); - } - TD_CODER_MOVE_POS(pCoder, sizeof(val)); - return 0; +static FORCE_INLINE int32_t tEncodeU8(SEncoder* pCoder, uint8_t val) { return tEncodeFixed(pCoder, &val, sizeof(val)); } +static FORCE_INLINE int32_t tEncodeI8(SEncoder* pCoder, int8_t val) { return tEncodeFixed(pCoder, &val, sizeof(val)); } +static FORCE_INLINE int32_t tEncodeU16(SEncoder* pCoder, uint16_t val) { + return tEncodeFixed(pCoder, &val, sizeof(val)); +} +static FORCE_INLINE int32_t tEncodeI16(SEncoder* pCoder, int16_t val) { + return tEncodeFixed(pCoder, &val, sizeof(val)); +} +static FORCE_INLINE int32_t tEncodeU32(SEncoder* pCoder, uint32_t val) { + return tEncodeFixed(pCoder, &val, sizeof(val)); +} +static FORCE_INLINE int32_t tEncodeI32(SEncoder* pCoder, int32_t val) { + return tEncodeFixed(pCoder, &val, sizeof(val)); +} +static FORCE_INLINE int32_t tEncodeU64(SEncoder* pCoder, uint64_t val) { + return tEncodeFixed(pCoder, &val, sizeof(val)); +} +static FORCE_INLINE int32_t tEncodeI64(SEncoder* pCoder, int64_t val) { + return tEncodeFixed(pCoder, &val, sizeof(val)); +} +static FORCE_INLINE int32_t tEncodeU16v(SEncoder* pCoder, uint16_t val) { + while (val >= ENCODE_LIMIT) { + TAOS_CHECK_RETURN(tEncodeU8(pCoder, (val | ENCODE_LIMIT) & 0xff)); + val >>= 7; + } + TAOS_RETURN(tEncodeU8(pCoder, val)); } - -// 16 -static FORCE_INLINE int32_t tEncodeU16(SEncoder* pCoder, uint16_t val) { TD_ENCODE_MACRO(pCoder, val, uint16_t, 16); } -static FORCE_INLINE int32_t tEncodeI16(SEncoder* pCoder, int16_t val) { TD_ENCODE_MACRO(pCoder, val, int16_t, 16); } -// 32 -static FORCE_INLINE int32_t tEncodeU32(SEncoder* pCoder, uint32_t val) { TD_ENCODE_MACRO(pCoder, val, uint32_t, 32); } -static FORCE_INLINE int32_t tEncodeI32(SEncoder* pCoder, int32_t val) { TD_ENCODE_MACRO(pCoder, val, int32_t, 32); } -// 64 -static FORCE_INLINE int32_t tEncodeU64(SEncoder* pCoder, uint64_t val) { TD_ENCODE_MACRO(pCoder, val, uint64_t, 64); } -static FORCE_INLINE int32_t tEncodeI64(SEncoder* pCoder, int64_t val) { TD_ENCODE_MACRO(pCoder, val, int64_t, 64); } -// 16v -static FORCE_INLINE int32_t tEncodeU16v(SEncoder* pCoder, uint16_t val) { TD_ENCODE_VARIANT_MACRO(pCoder, val); } static FORCE_INLINE int32_t tEncodeI16v(SEncoder* pCoder, int16_t val) { return tEncodeU16v(pCoder, ZIGZAGE(int16_t, val)); } -// 32v -static FORCE_INLINE int32_t tEncodeU32v(SEncoder* pCoder, uint32_t val) { TD_ENCODE_VARIANT_MACRO(pCoder, val); } +static FORCE_INLINE int32_t tEncodeU32v(SEncoder* pCoder, uint32_t val) { + while (val >= ENCODE_LIMIT) { + TAOS_CHECK_RETURN(tEncodeU8(pCoder, (val | ENCODE_LIMIT) & 0xff)); + val >>= 7; + } + TAOS_RETURN(tEncodeU8(pCoder, val)); +} static FORCE_INLINE int32_t tEncodeI32v(SEncoder* pCoder, int32_t val) { return tEncodeU32v(pCoder, ZIGZAGE(int32_t, val)); } -// 64v -static FORCE_INLINE int32_t tEncodeU64v(SEncoder* pCoder, uint64_t val) { TD_ENCODE_VARIANT_MACRO(pCoder, val); } +static FORCE_INLINE int32_t tEncodeU64v(SEncoder* pCoder, uint64_t val) { + while (val >= ENCODE_LIMIT) { + TAOS_CHECK_RETURN(tEncodeU8(pCoder, (val | ENCODE_LIMIT) & 0xff)); + val >>= 7; + } + TAOS_RETURN(tEncodeU8(pCoder, val)); +} static FORCE_INLINE int32_t tEncodeI64v(SEncoder* pCoder, int64_t val) { return tEncodeU64v(pCoder, ZIGZAGE(int64_t, val)); } @@ -260,14 +213,14 @@ static FORCE_INLINE int32_t tEncodeBinary(SEncoder* pCoder, const uint8_t* val, if (len) { if (pCoder->data) { if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, len)) { - return TSDB_CODE_OUT_OF_RANGE; + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } memcpy(TD_CODER_CURRENT(pCoder), val, len); } TD_CODER_MOVE_POS(pCoder, len); } - return 0; + TAOS_RETURN(0); } static FORCE_INLINE int32_t tEncodeCStrWithLen(SEncoder* pCoder, const char* val, uint32_t len) { @@ -279,75 +232,138 @@ static FORCE_INLINE int32_t tEncodeCStr(SEncoder* pCoder, const char* val) { } /* ------------------------ FOR DECODER ------------------------ */ -// 8 -static FORCE_INLINE int32_t tDecodeU8(SDecoder* pCoder, uint8_t* val) { - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(*val))) { - return TSDB_CODE_OUT_OF_RANGE; +static int32_t tDecodeFixed(SDecoder* pCoder, void* val, uint32_t size) { + if (pCoder->pos + size > pCoder->size) { + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + } else if (val) { + memcpy(val, pCoder->data + pCoder->pos, size); } - tGet(uint8_t, TD_CODER_CURRENT(pCoder), *val); - TD_CODER_MOVE_POS(pCoder, sizeof(*val)); - return 0; + pCoder->pos += size; + TAOS_RETURN(0); } -static FORCE_INLINE int32_t tDecodeI8(SDecoder* pCoder, int8_t* val) { - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(*val))) { - return TSDB_CODE_OUT_OF_RANGE; - } - tGet(int8_t, TD_CODER_CURRENT(pCoder), *val); - TD_CODER_MOVE_POS(pCoder, sizeof(*val)); - return 0; +static FORCE_INLINE int32_t tDecodeU8(SDecoder* pCoder, uint8_t* val) { + return tDecodeFixed(pCoder, val, sizeof(*val)); } +static FORCE_INLINE int32_t tDecodeI8(SDecoder* pCoder, int8_t* val) { return tDecodeFixed(pCoder, val, sizeof(*val)); } + // 16 -static FORCE_INLINE int32_t tDecodeU16(SDecoder* pCoder, uint16_t* val) { TD_DECODE_MACRO(pCoder, val, uint16_t, 16); } -static FORCE_INLINE int32_t tDecodeI16(SDecoder* pCoder, int16_t* val) { TD_DECODE_MACRO(pCoder, val, int16_t, 16); } +static FORCE_INLINE int32_t tDecodeU16(SDecoder* pCoder, uint16_t* val) { + return tDecodeFixed(pCoder, val, sizeof(*val)); +} +static FORCE_INLINE int32_t tDecodeI16(SDecoder* pCoder, int16_t* val) { + return tDecodeFixed(pCoder, val, sizeof(*val)); +} // 32 -static FORCE_INLINE int32_t tDecodeU32(SDecoder* pCoder, uint32_t* val) { TD_DECODE_MACRO(pCoder, val, uint32_t, 32); } -static FORCE_INLINE int32_t tDecodeI32(SDecoder* pCoder, int32_t* val) { TD_DECODE_MACRO(pCoder, val, int32_t, 32); } +static FORCE_INLINE int32_t tDecodeU32(SDecoder* pCoder, uint32_t* val) { + return tDecodeFixed(pCoder, val, sizeof(*val)); +} +static FORCE_INLINE int32_t tDecodeI32(SDecoder* pCoder, int32_t* val) { + return tDecodeFixed(pCoder, val, sizeof(*val)); +} // 64 -static FORCE_INLINE int32_t tDecodeU64(SDecoder* pCoder, uint64_t* val) { TD_DECODE_MACRO(pCoder, val, uint64_t, 64); } -static FORCE_INLINE int32_t tDecodeI64(SDecoder* pCoder, int64_t* val) { TD_DECODE_MACRO(pCoder, val, int64_t, 64); } +static FORCE_INLINE int32_t tDecodeU64(SDecoder* pCoder, uint64_t* val) { + return tDecodeFixed(pCoder, val, sizeof(*val)); +} +static FORCE_INLINE int32_t tDecodeI64(SDecoder* pCoder, int64_t* val) { + return tDecodeFixed(pCoder, val, sizeof(*val)); +} // 16v static FORCE_INLINE int32_t tDecodeU16v(SDecoder* pCoder, uint16_t* val) { - TD_DECODE_VARIANT_MACRO(pCoder, val, uint16_t); + uint8_t byte; + uint16_t tval = 0; + for (int32_t i = 0;; i++) { + TAOS_CHECK_RETURN(tDecodeU8(pCoder, &byte)); + if (byte < ENCODE_LIMIT) { + tval |= (byte << (7 * i)); + break; + } else { + tval |= ((byte & (ENCODE_LIMIT - 1)) << (7 * i)); + } + } + + if (val) { + *val = tval; + } + + TAOS_RETURN(0); } static FORCE_INLINE int32_t tDecodeI16v(SDecoder* pCoder, int16_t* val) { uint16_t tval; TAOS_CHECK_RETURN(tDecodeU16v(pCoder, &tval)); + if (val) { *val = ZIGZAGD(int16_t, tval); } - return 0; + + TAOS_RETURN(0); } // 32v static FORCE_INLINE int32_t tDecodeU32v(SDecoder* pCoder, uint32_t* val) { - TD_DECODE_VARIANT_MACRO(pCoder, val, uint32_t); + uint8_t byte; + uint32_t tval = 0; + for (int32_t i = 0;; i++) { + TAOS_CHECK_RETURN(tDecodeU8(pCoder, &byte)); + if (byte < ENCODE_LIMIT) { + tval |= (byte << (7 * i)); + break; + } else { + tval |= ((byte & (ENCODE_LIMIT - 1)) << (7 * i)); + } + } + + if (val) { + *val = tval; + } + + TAOS_RETURN(0); } static FORCE_INLINE int32_t tDecodeI32v(SDecoder* pCoder, int32_t* val) { uint32_t tval; TAOS_CHECK_RETURN(tDecodeU32v(pCoder, &tval)); + if (val) { *val = ZIGZAGD(int32_t, tval); } - return 0; + + TAOS_RETURN(0); } // 64v static FORCE_INLINE int32_t tDecodeU64v(SDecoder* pCoder, uint64_t* val) { - TD_DECODE_VARIANT_MACRO(pCoder, val, uint64_t); + uint8_t byte; + uint64_t tval = 0; + for (int32_t i = 0;; i++) { + TAOS_CHECK_RETURN(tDecodeU8(pCoder, &byte)); + if (byte < ENCODE_LIMIT) { + tval |= (byte << (7 * i)); + break; + } else { + tval |= ((byte & (ENCODE_LIMIT - 1)) << (7 * i)); + } + } + + if (val) { + *val = tval; + } + + TAOS_RETURN(0); } static FORCE_INLINE int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val) { uint64_t tval; TAOS_CHECK_RETURN(tDecodeU64v(pCoder, &tval)); + if (val) { *val = ZIGZAGD(int64_t, tval); } - return 0; + + TAOS_RETURN(0); } static FORCE_INLINE int32_t tDecodeFloat(SDecoder* pCoder, float* val) { @@ -361,7 +377,7 @@ static FORCE_INLINE int32_t tDecodeFloat(SDecoder* pCoder, float* val) { if (val) { *val = v.f; } - return 0; + TAOS_RETURN(0); } static FORCE_INLINE int32_t tDecodeDouble(SDecoder* pCoder, double* val) { @@ -375,7 +391,7 @@ static FORCE_INLINE int32_t tDecodeDouble(SDecoder* pCoder, double* val) { if (val) { *val = v.d; } - return 0; + TAOS_RETURN(0); } static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint32_t* len) { @@ -387,7 +403,7 @@ static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint3 } if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) { - return TSDB_CODE_OUT_OF_RANGE; + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } if (val) { @@ -395,13 +411,13 @@ static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint3 } TD_CODER_MOVE_POS(pCoder, length); - return 0; + TAOS_RETURN(0); } static FORCE_INLINE int32_t tDecodeCStrAndLen(SDecoder* pCoder, char** val, uint32_t* len) { TAOS_CHECK_RETURN(tDecodeBinary(pCoder, (uint8_t**)val, len)); (*len) -= 1; - return 0; + TAOS_RETURN(0); } static FORCE_INLINE int32_t tDecodeCStr(SDecoder* pCoder, char** val) { @@ -415,7 +431,7 @@ static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) { TAOS_CHECK_RETURN(tDecodeCStrAndLen(pCoder, &pStr, &len)); memcpy(val, pStr, len + 1); - return 0; + TAOS_RETURN(0); } static FORCE_INLINE int32_t tDecodeBinaryAlloc(SDecoder* pCoder, void** val, uint64_t* len) { @@ -425,12 +441,12 @@ static FORCE_INLINE int32_t tDecodeBinaryAlloc(SDecoder* pCoder, void** val, uin if (len) *len = length; if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) { - return TSDB_CODE_OUT_OF_RANGE; + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } *val = taosMemoryMalloc(length); if (*val == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } memcpy(*val, TD_CODER_CURRENT(pCoder), length); @@ -439,7 +455,7 @@ static FORCE_INLINE int32_t tDecodeBinaryAlloc(SDecoder* pCoder, void** val, uin } else { *val = NULL; } - return 0; + TAOS_RETURN(0); } static FORCE_INLINE int32_t tDecodeBinaryAlloc32(SDecoder* pCoder, void** val, uint32_t* len) { @@ -449,11 +465,11 @@ static FORCE_INLINE int32_t tDecodeBinaryAlloc32(SDecoder* pCoder, void** val, u if (len) *len = length; if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) { - return TSDB_CODE_OUT_OF_RANGE; + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } *val = taosMemoryMalloc(length); if (*val == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } memcpy(*val, TD_CODER_CURRENT(pCoder), length); @@ -461,13 +477,13 @@ static FORCE_INLINE int32_t tDecodeBinaryAlloc32(SDecoder* pCoder, void** val, u } else { *val = NULL; } - return 0; + TAOS_RETURN(0); } static FORCE_INLINE int32_t tDecodeCStrAndLenAlloc(SDecoder* pCoder, char** val, uint64_t* len) { TAOS_CHECK_RETURN(tDecodeBinaryAlloc(pCoder, (void**)val, len)); (*len) -= 1; - return 0; + TAOS_RETURN(0); } static FORCE_INLINE int32_t tDecodeCStrAlloc(SDecoder* pCoder, char** val) { diff --git a/source/util/src/tencode.c b/source/util/src/tencode.c index d99732fedf..0c7275d481 100644 --- a/source/util/src/tencode.c +++ b/source/util/src/tencode.c @@ -73,12 +73,12 @@ int32_t tStartEncode(SEncoder* pCoder) { if (pCoder->data) { if (pCoder->size - pCoder->pos < sizeof(int32_t)) { - return TSDB_CODE_OUT_OF_RANGE; + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } pNode = tEncoderMalloc(pCoder, sizeof(*pNode)); if (pNode == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } pNode->data = pCoder->data; @@ -95,7 +95,7 @@ int32_t tStartEncode(SEncoder* pCoder) { pCoder->pos += sizeof(int32_t); } - return 0; + TAOS_RETURN(0); } void tEndEncode(SEncoder* pCoder) { @@ -127,7 +127,7 @@ int32_t tStartDecode(SDecoder* pCoder) { pNode = tDecoderMalloc(pCoder, sizeof(*pNode)); if (pNode == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } pNode->data = pCoder->data; @@ -141,7 +141,7 @@ int32_t tStartDecode(SDecoder* pCoder) { pNode->pNext = pCoder->dStack; pCoder->dStack = pNode; - return 0; + TAOS_RETURN(0); } void tEndDecode(SDecoder* pCoder) { From b5ccc3db92ef7ae268c6d9f22baee895aae8916b Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 15 Jul 2024 18:19:05 +0800 Subject: [PATCH 19/48] adj stream function result --- source/libs/stream/src/streamUpdate.c | 7 +++++-- source/util/src/tbloomfilter.c | 3 ++- source/util/src/tscalablebf.c | 3 ++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 0dc1eafb03..61b7e02a4c 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -563,9 +563,12 @@ int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) { if (tDecodeI64(&decoder, &pInfo->interval) < 0) return -1; if (tDecodeI64(&decoder, &pInfo->watermark) < 0) return -1; if (tDecodeI64(&decoder, &pInfo->minTS) < 0) return -1; - pInfo->pCloseWinSBF = NULL; + code = tScalableBfDecode(&decoder, &pInfo->pCloseWinSBF); - TSDB_CHECK_CODE(code, lino, _error); + if (code != TSDB_CODE_SUCCESS) { + pInfo->pCloseWinSBF = NULL; + code = TSDB_CODE_SUCCESS; + } int32_t mapSize = 0; if (tDecodeI32(&decoder, &mapSize) < 0) return -1; diff --git a/source/util/src/tbloomfilter.c b/source/util/src/tbloomfilter.c index 9093fb9d91..1cf4ecf72e 100644 --- a/source/util/src/tbloomfilter.c +++ b/source/util/src/tbloomfilter.c @@ -188,11 +188,12 @@ int32_t tBloomFilterDecode(SDecoder* pDecoder, SBloomFilter** ppBF) { pBF->hashFn1 = HASH_FUNCTION_1; pBF->hashFn2 = HASH_FUNCTION_2; (*ppBF) = pBF; + return TSDB_CODE_SUCCESS; _error: tBloomFilterDestroy(pBF); uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); - return TSDB_CODE_FAILED; + return code; } bool tBloomFilterIsFull(const SBloomFilter* pBF) { return pBF->size >= pBF->expectedEntries; } diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index 0979000981..4a4dfd2653 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -254,8 +254,9 @@ int32_t tScalableBfDecode(SDecoder* pDecoder, SScalableBf** ppSBf) { (*ppSBf) = pSBf; _error: - tScalableBfDestroy(pSBf); + if (code != TSDB_CODE_SUCCESS) { + tScalableBfDestroy(pSBf); uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } return code; From aed8a75e357fea02a40ac3f17944930c7aa6eef1 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 15 Jul 2024 19:07:20 +0800 Subject: [PATCH 20/48] feat:[TD-30975]process return value in schemaless --- include/common/tname.h | 2 +- include/libs/parser/parser.h | 4 +- source/client/inc/clientSml.h | 24 +- source/client/src/clientRawBlockWrite.c | 23 +- source/client/src/clientSml.c | 633 ++++++++++++++++-------- source/client/src/clientSmlJson.c | 105 ++-- source/client/src/clientSmlLine.c | 66 ++- source/client/src/clientSmlTelnet.c | 30 +- source/client/test/smlTest.cpp | 114 +++-- source/common/src/tdatablock.c | 5 +- source/common/src/tname.c | 13 +- source/libs/parser/src/parInsertSml.c | 101 +++- 12 files changed, 744 insertions(+), 376 deletions(-) diff --git a/include/common/tname.h b/include/common/tname.h index 6a89d2a6be..f0a68f28c4 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -81,7 +81,7 @@ typedef struct { // uint64_t uid; // child table uid, may be useful } RandTableName; -void buildChildTableName(RandTableName* rName); +int32_t buildChildTableName(RandTableName* rName); #ifdef __cplusplus } diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 3ac357055e..4ad24fe91d 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -149,10 +149,10 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* void qDestroyBoundColInfo(void* pInfo); -SQuery* smlInitHandle(); +int32_t smlInitHandle(SQuery** query); int32_t smlBuildRow(STableDataCxt* pTableCxt); int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* kv, int32_t index); -STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta); +int32_t smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta, STableDataCxt** cxt); void clearColValArraySml(SArray* pCols); int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 122914fd34..d369e0597c 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -82,7 +82,7 @@ extern "C" { #define IS_SLASH_LETTER_IN_MEASUREMENT(sql) \ (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == SLASH)) -#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) +#define MOVE_FORWARD_ONE(sql, len) ((void)memmove((void *)((sql)-1), (sql), len)) #define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen) \ for (int i = 1; i < keyLen; ++i) { \ @@ -211,19 +211,19 @@ extern int64_t smlFactorS[]; typedef int32_t (*_equal_fn_sml)(const void *, const void *); -SSmlHandle *smlBuildSmlInfo(TAOS *taos); +int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle); void smlDestroyInfo(SSmlHandle *info); int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset); int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset); bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg); -int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); -bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); +void smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); +int32_t smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision); -SSmlTableInfo* smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen); -SSmlSTableMeta* smlBuildSTableMeta(bool isDataFormat); +int32_t smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen, SSmlTableInfo** tInfo); +int32_t smlBuildSTableMeta(bool isDataFormat, SSmlSTableMeta** sMeta); int32_t smlSetCTableName(SSmlTableInfo *oneTable); -void getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo); -STableMeta* smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen); +int32_t getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo); +int32_t smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen, STableMeta **pTableMeta); int32_t is_same_child_table_telnet(const void *a, const void *b); int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len); int32_t smlClearForRerun(SSmlHandle *info); @@ -236,7 +236,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseJSON(SSmlHandle *info, char *payload); -SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement); +int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSmlSTableMeta** sMeta); bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv); bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv); int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements); @@ -246,9 +246,9 @@ void smlBuildTsKv(SSmlKv *kv, int64_t ts); int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv); int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs); -static inline bool smlDoubleToInt64OverFlow(double num) { - if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true; - return false; +static inline int32_t smlDoubleToInt64OverFlow(double num) { + if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return 0; + return -1; } static inline void smlStrReplace(char* src, int32_t len){ diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index ce404092b0..fe12850e14 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1572,9 +1572,8 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pDat } // uError("td23101 0vgId:%d, vgId:%d, name:%s, uid:%"PRIu64, vgData.vgId, pTableMeta->vgId, tbname, pTableMeta->uid); - pQuery = smlInitHandle(); - if (pQuery == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = smlInitHandle(&pQuery); + if (code != TSDB_CODE_SUCCESS) { goto end; } pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); @@ -1656,9 +1655,9 @@ int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const cha if (code != TSDB_CODE_SUCCESS) { goto end; } - pQuery = smlInitHandle(); - if (pQuery == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + + code = smlInitHandle(&pQuery); + if (code != TSDB_CODE_SUCCESS) { goto end; } pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); @@ -1755,12 +1754,12 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { conn.requestObjRefId = pRequest->self; conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); - pQuery = smlInitHandle(); - if (pQuery == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = smlInitHandle(&pQuery); + if (code != TSDB_CODE_SUCCESS) { SET_ERROR_MSG("init sml handle failed"); goto end; } + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); while (++rspObj.common.resIter < rspObj.rsp.common.blockNum) { void* pRetrieve = taosArrayGetP(rspObj.rsp.common.blockData, rspObj.common.resIter); @@ -1903,12 +1902,12 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) conn.requestObjRefId = pRequest->self; conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); - pQuery = smlInitHandle(); - if (pQuery == NULL) { + code = smlInitHandle(&pQuery); + if (code != TSDB_CODE_SUCCESS) { SET_ERROR_MSG("init sml handle failed"); - code = TSDB_CODE_OUT_OF_MEMORY; goto end; } + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.common.blockNum); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 7e63dcd36a..6e2f4ba367 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -110,11 +110,13 @@ int64_t smlFactorS[] = {1000LL, 1000000LL, 1000000000LL}; static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo *conn, const char *pTabName, AUTH_TYPE type) { SUserAuthInfo pAuth = {0}; - snprintf(pAuth.user, sizeof(pAuth.user), "%s", info->taos->user); + (void)snprintf(pAuth.user, sizeof(pAuth.user), "%s", info->taos->user); if (NULL == pTabName) { - tNameSetDbName(&pAuth.tbName, info->taos->acctId, info->pRequest->pDb, strlen(info->pRequest->pDb)); + if (tNameSetDbName(&pAuth.tbName, info->taos->acctId, info->pRequest->pDb, strlen(info->pRequest->pDb)) != 0){ + return TSDB_CODE_SML_INVALID_DATA; + } } else { - toName(info->taos->acctId, info->pRequest->pDb, pTabName, &pAuth.tbName); + (void)toName(info->taos->acctId, info->pRequest->pDb, pTabName, &pAuth.tbName); //ignore } pAuth.type = type; @@ -129,17 +131,19 @@ static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo *conn, const char : code; } -int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) { - if (pBuf->buf) { - memset(pBuf->buf, 0, pBuf->len); - if (msg1) strncat(pBuf->buf, msg1, pBuf->len); - int32_t left = pBuf->len - strlen(pBuf->buf); - if (left > 2 && msg2) { - strncat(pBuf->buf, ":", left - 1); - strncat(pBuf->buf, msg2, left - 2); - } +void smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) { + if (pBuf->buf == NULL) { + return; + } + (void)memset(pBuf->buf, 0, pBuf->len); + if (msg1) { + (void)strncat(pBuf->buf, msg1, pBuf->len); + } + int32_t left = pBuf->len - strlen(pBuf->buf); + if (left > 2 && msg2) { + (void)strncat(pBuf->buf, ":", left - 1); + (void)strncat(pBuf->buf, msg2, left - 2); } - return TSDB_CODE_SML_INVALID_DATA; } int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision) { @@ -161,10 +165,10 @@ int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, u return convertTimePrecision(tsInt64, fromPrecision, toPrecision); } -SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen) { +int32_t smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen, SSmlTableInfo** tInfo) { SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); if (!tag) { - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } tag->sTableName = measure; @@ -173,14 +177,12 @@ SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measu tag->cols = taosArrayInit(numRows, POINTER_BYTES); if (tag->cols == NULL) { uError("SML:smlBuildTableInfo failed to allocate memory"); - goto cleanup; + taosMemoryFree(tag); + return TSDB_CODE_OUT_OF_MEMORY; } - return tag; - -cleanup: - taosMemoryFree(tag); - return NULL; + *tInfo = tag; + return TSDB_CODE_SUCCESS; } void smlBuildTsKv(SSmlKv *kv, int64_t ts) { @@ -191,34 +193,50 @@ void smlBuildTsKv(SSmlKv *kv, int64_t ts) { kv->length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; } -SSmlSTableMeta *smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement) { - SSmlSTableMeta *sMeta = NULL; +static void smlDestroySTableMeta(void *para) { + SSmlSTableMeta *meta = *(SSmlSTableMeta **)para; + taosHashCleanup(meta->tagHash); + taosHashCleanup(meta->colHash); + (void)taosArrayDestroy(meta->tags); + (void)taosArrayDestroy(meta->cols); + taosMemoryFreeClear(meta->tableMeta); + taosMemoryFree(meta); +} + +int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSmlSTableMeta** sMeta) { + int32_t code = TSDB_CODE_SUCCESS; char *measure = currElement->measure; int measureLen = currElement->measureLen; if (currElement->measureEscaped) { measure = (char *)taosMemoryMalloc(measureLen); - memcpy(measure, currElement->measure, measureLen); + if (measure == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(measure, currElement->measure, measureLen); PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); smlStrReplace(measure, measureLen); } - STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen); + STableMeta *pTableMeta = NULL; + code = smlGetMeta(info, measure, measureLen, &pTableMeta); if (currElement->measureEscaped) { taosMemoryFree(measure); } - if (pTableMeta == NULL) { + if (code != TSDB_CODE_SUCCESS) { info->dataFormat = false; info->reRun = true; - terrno = TSDB_CODE_SUCCESS; - return sMeta; + return code; } - sMeta = smlBuildSTableMeta(info->dataFormat); - if (sMeta == NULL) { + code = smlBuildSTableMeta(info->dataFormat, sMeta); + if (code != TSDB_CODE_SUCCESS) { taosMemoryFreeClear(pTableMeta); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return sMeta; + return code; + } + (*sMeta)->tableMeta = pTableMeta; + code = taosHashPut(info->superTables, currElement->measure, currElement->measureLen, sMeta, POINTER_BYTES); + if (code != TSDB_CODE_SUCCESS){ + smlDestroySTableMeta(*sMeta); + return code; } - sMeta->tableMeta = pTableMeta; - taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES); for (int i = 1; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) { SSchema *col = pTableMeta->schema + i; SSmlKv kv = {.key = col->name, .keyLen = strlen(col->name), .type = col->type}; @@ -232,40 +250,38 @@ SSmlSTableMeta *smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currEleme } if (i < pTableMeta->tableInfo.numOfColumns) { - taosArrayPush(sMeta->cols, &kv); + if(taosArrayPush((*sMeta)->cols, &kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } } else { - taosArrayPush(sMeta->tags, &kv); + if(taosArrayPush((*sMeta)->tags, &kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } } } - return sMeta; + return TSDB_CODE_SUCCESS; } bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { // cnt begin 0, add ts so + 2 if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) { - info->dataFormat = false; - info->reRun = true; - return false; + goto END; } // bind data int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, cnt + 1); if (unlikely(ret != TSDB_CODE_SUCCESS)) { uDebug("smlBuildCol error, retry"); - info->dataFormat = false; - info->reRun = true; - return false; + goto END; } if (cnt >= taosArrayGetSize(info->maxColKVs)) { - info->dataFormat = false; - info->reRun = true; - return false; + goto END; } SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxColKVs, cnt); - + if (maxKV == NULL){ + goto END; + } if (unlikely(!IS_SAME_KEY)) { - info->dataFormat = false; - info->reRun = true; - return false; + goto END; } if (unlikely(IS_VAR_DATA_TYPE(kv->type) && kv->length > maxKV->length)) { @@ -273,6 +289,11 @@ bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { info->needModifySchema = true; } return true; + +END: + info->dataFormat = false; + info->reRun = true; + return false; } bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { @@ -285,6 +306,9 @@ bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { } SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt); + if (maxKV == NULL) { + goto END; + } if (unlikely(!IS_SAME_KEY)) { goto END; } @@ -306,8 +330,8 @@ int32_t smlJoinMeasureTag(SSmlLineInfo *elements) { if (elements->measureTag == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - memcpy(elements->measureTag, elements->measure, elements->measureLen); - memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen); + (void)memcpy(elements->measureTag, elements->measure, elements->measureLen); + (void)memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen); elements->measureTagsLen = elements->measureLen + elements->tagsLen; return TSDB_CODE_SUCCESS; } @@ -325,14 +349,14 @@ static bool smlIsPKTable(STableMeta *pTableMeta){ int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements) { bool isSameMeasure = IS_SAME_SUPER_TABLE; if (isSameMeasure) { - return 0; + return TSDB_CODE_SUCCESS; } SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); SSmlSTableMeta *sMeta = NULL; if (unlikely(tmp == NULL)) { - sMeta = smlBuildSuperTableInfo(info, elements); - if (sMeta == NULL) return -1; + int32_t code = smlBuildSuperTableInfo(info, elements, &sMeta); + if (code != 0) return code; } else { sMeta = *tmp; } @@ -342,39 +366,59 @@ int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements) { info->maxColKVs = sMeta->cols; if(smlIsPKTable(sMeta->tableMeta)){ - terrno = TSDB_CODE_SML_NOT_SUPPORT_PK; - return -1; + return TSDB_CODE_SML_NOT_SUPPORT_PK; } return 0; } int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements) { + int32_t code = TSDB_CODE_SUCCESS; SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureTagsLen); SSmlTableInfo *tinfo = NULL; if (unlikely(oneTable == NULL)) { - tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); - if (unlikely(!tinfo)) { - return TSDB_CODE_OUT_OF_MEMORY; + code = smlBuildTableInfo(1, elements->measure, elements->measureLen, &tinfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + code = taosHashPut(info->childTables, elements->measureTag, elements->measureTagsLen, &tinfo, POINTER_BYTES); + if(code != 0){ + smlDestroyTableInfo(&tinfo); + return code; } - taosHashPut(info->childTables, elements->measureTag, elements->measureTagsLen, &tinfo, POINTER_BYTES); tinfo->tags = taosArrayDup(info->preLineTagKV, NULL); + if(tinfo->tags == NULL){ + smlDestroyTableInfo(&tinfo); + return TSDB_CODE_OUT_OF_MEMORY; + } for (size_t i = 0; i < taosArrayGetSize(info->preLineTagKV); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(info->preLineTagKV, i); + if(kv == NULL){ + smlDestroyTableInfo(&tinfo); + return TSDB_CODE_SML_INVALID_DATA; + } if (kv->keyEscaped) kv->key = NULL; if (kv->valueEscaped) kv->value = NULL; } - smlSetCTableName(tinfo); - getTableUid(info, elements, tinfo); + code = smlSetCTableName(tinfo); + if (code != TSDB_CODE_SUCCESS){ + smlDestroyTableInfo(&tinfo); + return code; + } + code = getTableUid(info, elements, tinfo); + if (code != TSDB_CODE_SUCCESS){ + smlDestroyTableInfo(&tinfo); + return code; + } if (info->dataFormat) { info->currSTableMeta->uid = tinfo->uid; - tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); - if (tinfo->tableDataCtx == NULL) { + code = smlInitTableDataCtx(info->pQuery, info->currSTableMeta, &tinfo->tableDataCtx); + if (code != TSDB_CODE_SUCCESS) { smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); smlDestroyTableInfo(&tinfo); - return TSDB_CODE_SML_INVALID_DATA; + return code; } } } else { @@ -404,9 +448,16 @@ int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv * uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format false, ts:%" PRId64, info->id, kvTs->i); if (elements->colArray == NULL) { elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); + if (elements->colArray == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + if (taosArrayPush(elements->colArray, kvTs) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + if (taosArrayPush(elements->colArray, kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; } - taosArrayPush(elements->colArray, kvTs); - taosArrayPush(elements->colArray, kv); } info->preLine = *elements; @@ -442,6 +493,9 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) { size_t totalNameLen = delimiter * (taosArrayGetSize(tags) - 1); for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); + if(tag == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } totalNameLen += tag->length; } if (totalNameLen < TSDB_TABLE_NAME_LEN) { @@ -449,12 +503,15 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) { } } if (autoChildName) { - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); + (void)memset(childTableName, 0, TSDB_TABLE_NAME_LEN); for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); - strncat(childTableName, tag->value, tag->length); + if(tag == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } + (void)strncat(childTableName, tag->value, tag->length); if (i != taosArrayGetSize(tags) - 1) { - strcat(childTableName, tsSmlAutoChildTableNameDelimiter); + (void)strcat(childTableName, tsSmlAutoChildTableNameDelimiter); } } if (tsSmlDot2Underline) { @@ -466,10 +523,13 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) { for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); + if(tag == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } // handle child table name if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) { - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); + (void)memset(childTableName, 0, TSDB_TABLE_NAME_LEN); + (void)strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); if (tsSmlDot2Underline) { smlStrReplace(childTableName, strlen(childTableName)); } @@ -483,60 +543,60 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) { } int32_t smlSetCTableName(SSmlTableInfo *oneTable) { - smlParseTableName(oneTable->tags, oneTable->childTableName); + int32_t code = smlParseTableName(oneTable->tags, oneTable->childTableName); + if(code != TSDB_CODE_SUCCESS){ + return code; + } if (strlen(oneTable->childTableName) == 0) { SArray *dst = taosArrayDup(oneTable->tags, NULL); + if (dst == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } ASSERT(oneTable->sTableNameLen < TSDB_TABLE_NAME_LEN); char superName[TSDB_TABLE_NAME_LEN] = {0}; RandTableName rName = {dst, NULL, (uint8_t)oneTable->sTableNameLen, oneTable->childTableName}; if (tsSmlDot2Underline) { - memcpy(superName, oneTable->sTableName, oneTable->sTableNameLen); + (void)memcpy(superName, oneTable->sTableName, oneTable->sTableNameLen); smlStrReplace(superName, oneTable->sTableNameLen); rName.stbFullName = superName; } else { rName.stbFullName = oneTable->sTableName; } - buildChildTableName(&rName); - taosArrayDestroy(dst); + code = buildChildTableName(&rName); + if (code != TSDB_CODE_SUCCESS){ + return code; + } + (void)taosArrayDestroy(dst); } return TSDB_CODE_SUCCESS; } -void getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo) { +int32_t getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo) { char key[TSDB_TABLE_NAME_LEN * 2 + 1] = {0}; size_t nLen = strlen(tinfo->childTableName); - memcpy(key, currElement->measure, currElement->measureLen); + (void)memcpy(key, currElement->measure, currElement->measureLen); if (tsSmlDot2Underline) { smlStrReplace(key, currElement->measureLen); } - memcpy(key + currElement->measureLen + 1, tinfo->childTableName, nLen); + (void)memcpy(key + currElement->measureLen + 1, tinfo->childTableName, nLen); void *uid = taosHashGet(info->tableUids, key, currElement->measureLen + 1 + nLen); // use \0 as separator for stable name and child table name if (uid == NULL) { tinfo->uid = info->uid++; - taosHashPut(info->tableUids, key, currElement->measureLen + 1 + nLen, &tinfo->uid, sizeof(uint64_t)); + return taosHashPut(info->tableUids, key, currElement->measureLen + 1 + nLen, &tinfo->uid, sizeof(uint64_t)); } else { tinfo->uid = *(uint64_t *)uid; } + return TSDB_CODE_SUCCESS; } -static void smlDestroySTableMeta(void *para) { - SSmlSTableMeta *meta = *(SSmlSTableMeta **)para; - taosHashCleanup(meta->tagHash); - taosHashCleanup(meta->colHash); - taosArrayDestroy(meta->tags); - taosArrayDestroy(meta->cols); - taosMemoryFreeClear(meta->tableMeta); - taosMemoryFree(meta); -} - -SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) { +int32_t smlBuildSTableMeta(bool isDataFormat, SSmlSTableMeta **sMeta) { SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); if (!meta) { - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } if (unlikely(!isDataFormat)) { @@ -564,14 +624,15 @@ SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) { uError("SML:smlBuildSTableMeta failed to allocate memory"); goto cleanup; } - return meta; + *sMeta = meta; + return TSDB_CODE_SUCCESS; cleanup: smlDestroySTableMeta(meta); - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } -bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { +int32_t smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->length; char *endptr = NULL; @@ -667,7 +728,7 @@ bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg) { kvVal->type = TSDB_DATA_TYPE_FLOAT; kvVal->f = (float)result; } else if ((left == 1 && *endptr == 'i') || (left == 3 && strncasecmp(endptr, "i64", left) == 0)) { - if (smlDoubleToInt64OverFlow(result)) { + if (smlDoubleToInt64OverFlow(result) == 0) { errno = 0; int64_t tmp = taosStr2Int64(pVal, &endptr, 10); if (errno == ERANGE) { @@ -743,8 +804,8 @@ bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg) { return true; } -STableMeta *smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen) { - STableMeta *pTableMeta = NULL; +int32_t smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen, STableMeta **pTableMeta) { + *pTableMeta = NULL; SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); @@ -754,14 +815,14 @@ STableMeta *smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen conn.requestId = info->pRequest->requestId; conn.requestObjRefId = info->pRequest->self; conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - memcpy(pName.tname, measure, measureLen); + (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + (void)memcpy(pName.tname, measure, measureLen); - int32_t code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); + int32_t code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, pTableMeta); if (code != TSDB_CODE_SUCCESS) { - return NULL; + return code; } - return pTableMeta; + return TSDB_CODE_SUCCESS; } static int64_t smlGenId() { @@ -803,7 +864,7 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm *action = SCHEMA_ACTION_ADD_COLUMN; } } - return 0; + return TSDB_CODE_SUCCESS; } #define BOUNDARY 1024 @@ -838,6 +899,9 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH for (int j = 0; j < taosArrayGetSize(cols); ++j) { if (j == 0 && !isTag) continue; SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); + if (kv == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info); if (code != TSDB_CODE_SUCCESS) { return code; @@ -846,6 +910,9 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH for (int j = 0; j < taosArrayGetSize(checkDumplicateCols); ++j) { SSmlKv *kv = (SSmlKv *)taosArrayGet(checkDumplicateCols, j); + if (kv == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } if(taosHashGet(schemaHash, kv->key, kv->keyLen) != NULL){ return TSDB_CODE_PAR_DUPLICATED_COLUMN; } @@ -854,10 +921,18 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH } static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) { + int32_t code = TSDB_CODE_SUCCESS; SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (hashTmp == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto END; + } int32_t i = 0; for (; i < length; i++) { - taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); + code = taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); + if (code != 0){ + goto END; + } } if (isTag) { @@ -867,13 +942,19 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool } for (; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + if (kv == NULL){ + code = TSDB_CODE_SML_INVALID_DATA; + goto END; + } if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { - taosHashCleanup(hashTmp); - return TSDB_CODE_SML_INVALID_DATA; + code = TSDB_CODE_SML_INVALID_DATA; + goto END; } } + +END: taosHashCleanup(hashTmp); - return 0; + return code; } static int32_t getBytes(uint8_t type, int32_t length) { @@ -889,6 +970,9 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO SArray *results, int32_t numOfCols, bool isTag) { for (int j = 0; j < taosArrayGetSize(cols); ++j) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); + if (kv == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } ESchemaAction action = SCHEMA_ACTION_NULL; int code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info); if (code != 0) { @@ -898,8 +982,10 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO SField field = {0}; field.type = kv->type; field.bytes = getBytes(kv->type, kv->length); - memcpy(field.name, kv->key, kv->keyLen); - taosArrayPush(results, &field); + (void)memcpy(field.name, kv->key, kv->keyLen); + if (taosArrayPush(results, &field) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen); if (index == NULL) { @@ -909,6 +995,9 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO uint16_t newIndex = *index; if (isTag) newIndex -= numOfCols; SField *field = (SField *)taosArrayGet(results, newIndex); + if (field == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } field->bytes = getBytes(kv->type, kv->length); } } @@ -917,6 +1006,9 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO int32_t len = 0; for (int j = 0; j < taosArrayGetSize(results); ++j) { SField *field = taosArrayGet(results, j); + if (field == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } len += field->bytes; } if (len > maxLen) { @@ -933,21 +1025,33 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, int32_t code = TSDB_CODE_SUCCESS; SCmdMsgInfo pCmdMsg = {0}; char *pSql = NULL; + char *msg = NULL; // put front for free pReq.numOfColumns = taosArrayGetSize(pColumns); + pReq.pTags = pTags; + pReq.numOfTags = taosArrayGetSize(pTags); + pReq.pColumns = taosArrayInit(pReq.numOfColumns, sizeof(SFieldWithOptions)); + if (pReq.pColumns == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } for (int32_t i = 0; i < pReq.numOfColumns; ++i) { - SField *pField = taosArrayGet(pColumns, i); + SField *pField = taosArrayGet(pColumns, i); + if (pField == NULL){ + code = TSDB_CODE_SML_INVALID_DATA; + goto end; + } SFieldWithOptions fieldWithOption = {0}; setFieldWithOptions(&fieldWithOption, pField); setDefaultOptionsForField(&fieldWithOption); - taosArrayPush(pReq.pColumns, &fieldWithOption); + if (taosArrayPush(pReq.pColumns, &fieldWithOption) == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } } - pReq.numOfTags = taosArrayGetSize(pTags); - pReq.pTags = pTags; - if (action == SCHEMA_ACTION_CREATE_STABLE) { pReq.colVer = 1; pReq.tagVer = 1; @@ -968,6 +1072,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, pSql = (action == SCHEMA_ACTION_ADD_COLUMN) ? "sml_add_column" : "sml_modify_column_size"; } else { uError("SML:0x%" PRIx64 " invalid action:%d", info->id, action); + code = TSDB_CODE_SML_INVALID_DATA; goto end; } @@ -987,40 +1092,55 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SField field = {0}; field.type = TSDB_DATA_TYPE_NCHAR; field.bytes = TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; - strcpy(field.name, tsSmlTagName); - taosArrayPush(pReq.pTags, &field); + tstrncpy(field.name, tsSmlTagName, sizeof(field.name)); + if (taosArrayPush(pReq.pTags, &field) == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } } pReq.commentLen = -1; pReq.igExists = true; - tNameExtractFullName(pName, pReq.name); + (void)tNameExtractFullName(pName, pReq.name); pCmdMsg.epSet = getEpSet_s(&info->taos->pAppInfo->mgmtEp); pCmdMsg.msgType = TDMT_MND_CREATE_STB; pCmdMsg.msgLen = tSerializeSMCreateStbReq(NULL, 0, &pReq); - pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); - if (NULL == pCmdMsg.pMsg) { + if (pCmdMsg.msgLen < 0){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + msg = taosMemoryMalloc(pCmdMsg.msgLen); + if (NULL == msg) { code = TSDB_CODE_OUT_OF_MEMORY; goto end; } - tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); - SQuery pQuery; - memset(&pQuery, 0, sizeof(pQuery)); + pCmdMsg.pMsg = msg; + if (tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) < 0){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + + SQuery pQuery = {0}; + (void)memset(&pQuery, 0, sizeof(pQuery)); pQuery.execMode = QUERY_EXEC_MODE_RPC; pQuery.pCmdMsg = &pCmdMsg; pQuery.msgType = pQuery.pCmdMsg->msgType; pQuery.stableQuery = true; - launchQueryImpl(pRequest, &pQuery, true, NULL); + (void)launchQueryImpl(pRequest, &pQuery, true, NULL); // no need to check return value if (pRequest->code == TSDB_CODE_SUCCESS) { - catalogRemoveTableMeta(info->pCatalog, pName); + code = catalogRemoveTableMeta(info->pCatalog, pName); + if (code != TSDB_CODE_SUCCESS){ + goto end; + } } code = pRequest->code; - taosMemoryFree(pCmdMsg.pMsg); end: + taosMemoryFree(msg); destroyRequest(pRequest); tFreeSMCreateStbReq(&pReq); return code; @@ -1053,11 +1173,15 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { size_t superTableLen = 0; void *superTable = taosHashGetKey(tmp, &superTableLen); char *measure = taosMemoryMalloc(superTableLen); - memcpy(measure, superTable, superTableLen); + if (measure == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + (void)memcpy(measure, superTable, superTableLen); PROCESS_SLASH_IN_MEASUREMENT(measure, superTableLen); smlStrReplace(measure, superTableLen); - memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - memcpy(pName.tname, measure, superTableLen); + (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + (void)memcpy(pName.tname, measure, superTableLen); taosMemoryFree(measure); code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); @@ -1069,23 +1193,32 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { } uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas create table:%s", info->id, pName.tname); SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); + if (pColumns == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); + if (pTags == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + (void)taosArrayDestroy(pColumns); + goto end; + } code = smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBuildFieldsList tag1 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } code = smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBuildFieldsList col1 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } code = smlSendMetaMsg(info, &pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE); - taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pColumns); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); goto end; @@ -1107,9 +1240,16 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (hashTmp == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } for (uint16_t i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { - taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + code = taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + if (code != 0){ + goto end; + } } ESchemaAction action = SCHEMA_ACTION_NULL; @@ -1126,46 +1266,64 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { action); SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); + if (pColumns == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - + if (pTags == NULL){ + (void)taosArrayDestroy(pColumns); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { SField field = {0}; field.type = pTableMeta->schema[i].type; field.bytes = pTableMeta->schema[i].bytes; - strcpy(field.name, pTableMeta->schema[i].name); + tstrncpy(field.name, pTableMeta->schema[i].name, sizeof(field.name)); if (i < pTableMeta->tableInfo.numOfColumns) { - taosArrayPush(pColumns, &field); + if (taosArrayPush(pColumns, &field) == NULL){ + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } } else { - taosArrayPush(pTags, &field); + if (taosArrayPush(pTags, &field) == NULL){ + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } } } code = smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags, pTableMeta->tableInfo.numOfColumns, true); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBuildFieldsList tag2 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } if (taosArrayGetSize(pTags) + pTableMeta->tableInfo.numOfColumns > TSDB_MAX_COLUMNS) { uError("SML:0x%" PRIx64 " too many columns than 4096", info->id); code = TSDB_CODE_PAR_TOO_MANY_COLUMNS; - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } if (taosArrayGetSize(pTags) > TSDB_MAX_TAGS) { uError("SML:0x%" PRIx64 " too many tags than 128", info->id); code = TSDB_CODE_PAR_INVALID_TAGS_NUM; - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); - taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pColumns); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); goto end; @@ -1185,7 +1343,10 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { taosHashClear(hashTmp); for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + code = taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } } action = SCHEMA_ACTION_NULL; code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, sTableData->tags, &action, false); @@ -1201,18 +1362,36 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { action); SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); + if (pColumns == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - + if (pTags == NULL){ + (void)taosArrayDestroy(pColumns); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { SField field = {0}; field.type = pTableMeta->schema[i].type; field.bytes = pTableMeta->schema[i].bytes; - strcpy(field.name, pTableMeta->schema[i].name); + tstrncpy(field.name, pTableMeta->schema[i].name, sizeof(field.name)); if (i < pTableMeta->tableInfo.numOfColumns) { - taosArrayPush(pColumns, &field); + if (taosArrayPush(pColumns, &field) == NULL){ + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } } else { - taosArrayPush(pTags, &field); + if (taosArrayPush(pTags, &field) == NULL){ + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } } } @@ -1220,21 +1399,21 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { pTableMeta->tableInfo.numOfColumns, false); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBuildFieldsList col2 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } if (taosArrayGetSize(pColumns) + pTableMeta->tableInfo.numOfTags > TSDB_MAX_COLUMNS) { uError("SML:0x%" PRIx64 " too many columns than 4096", info->id); code = TSDB_CODE_PAR_TOO_MANY_COLUMNS; - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); - taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pColumns); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); goto end; @@ -1284,13 +1463,13 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas end success, format:%d, needModifySchema:%d", info->id, info->dataFormat, info->needModifySchema); - return 0; + return TSDB_CODE_SUCCESS; end: taosHashCancelIterate(info->superTables, tmp); taosHashCleanup(hashTmp); taosMemoryFreeClear(pTableMeta); - catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); + code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); uError("SML:0x%" PRIx64 " smlModifyDBSchemas end failed:%d:%s, format:%d, needModifySchema:%d", info->id, code, tstrerror(code), info->dataFormat, info->needModifySchema); @@ -1301,9 +1480,14 @@ static int32_t smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols terrno = 0; for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + if (kv == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); if (ret == 0) { - taosArrayPush(metaArray, kv); + if (taosArrayPush(metaArray, kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } if(taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { return TSDB_CODE_PAR_DUPLICATED_COLUMN; } @@ -1317,10 +1501,16 @@ static int32_t smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg, SHashObj* checkDuplicate) { for (int i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); - + if (kv == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } int16_t *index = (int16_t *)taosHashGet(metaHash, kv->key, kv->keyLen); if (index) { SSmlKv *value = (SSmlKv *)taosArrayGet(metaArray, *index); + if (value == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } + if (isTag) { if (kv->length > value->length) { value->length = kv->length; @@ -1345,7 +1535,9 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols int16_t size = tmp; int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); if (ret == 0) { - taosArrayPush(metaArray, kv); + if(taosArrayPush(metaArray, kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } if(taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { return TSDB_CODE_PAR_DUPLICATED_COLUMN; } @@ -1365,7 +1557,7 @@ void smlDestroyTableInfo(void *para) { taosHashCleanup(kvHash); } - taosArrayDestroy(tag->cols); + (void)taosArrayDestroy(tag->cols); taosArrayDestroyEx(tag->tags, freeSSmlKv); taosMemoryFree(tag); } @@ -1391,13 +1583,13 @@ void smlDestroyInfo(SSmlHandle *info) { cJSON *tags = (cJSON *)taosArrayGetP(info->tagJsonArray, i); cJSON_Delete(tags); } - taosArrayDestroy(info->tagJsonArray); + (void)taosArrayDestroy(info->tagJsonArray); for (int i = 0; i < taosArrayGetSize(info->valueJsonArray); i++) { cJSON *value = (cJSON *)taosArrayGetP(info->valueJsonArray, i); cJSON_Delete(value); } - taosArrayDestroy(info->valueJsonArray); + (void)taosArrayDestroy(info->valueJsonArray); taosArrayDestroyEx(info->preLineTagKV, freeSSmlKv); @@ -1418,21 +1610,22 @@ void smlDestroyInfo(SSmlHandle *info) { taosMemoryFreeClear(info); } -SSmlHandle *smlBuildSmlInfo(TAOS *taos) { +int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { int32_t code = TSDB_CODE_SUCCESS; SSmlHandle *info = (SSmlHandle *)taosMemoryCalloc(1, sizeof(SSmlHandle)); if (NULL == info) { - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } if (taos != NULL) { info->taos = acquireTscObj(*(int64_t *)taos); if (info->taos == NULL) { - goto cleanup; + code = TSDB_CODE_TSC_DISCONNECTED; + goto ERROR; } code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code); - goto cleanup; + goto ERROR; } } @@ -1440,27 +1633,37 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) { info->childTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); info->tableUids = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); info->superTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (info->pVgHash == NULL || info->childTables == NULL || info->superTables == NULL || info->tableUids == NULL) { + uError("create SSmlHandle hash obj failed"); + code = TSDB_CODE_OUT_OF_MEMORY; + goto ERROR; + } taosHashSetFreeFp(info->superTables, smlDestroySTableMeta); taosHashSetFreeFp(info->childTables, smlDestroyTableInfo); info->id = smlGenId(); - info->pQuery = smlInitHandle(); + code = smlInitHandle(&info->pQuery); + if (code != TSDB_CODE_SUCCESS){ + goto ERROR; + } info->dataFormat = true; info->tagJsonArray = taosArrayInit(8, POINTER_BYTES); info->valueJsonArray = taosArrayInit(8, POINTER_BYTES); info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv)); - if (NULL == info->pVgHash || NULL == info->childTables || NULL == info->superTables || NULL == info->tableUids) { - uError("create SSmlHandle failed"); - goto cleanup; + if (info->tagJsonArray == NULL || info->valueJsonArray == NULL || info->preLineTagKV == NULL) { + uError("SML:0x%" PRIx64 " failed to allocate memory", info->id); + code = TSDB_CODE_OUT_OF_MEMORY; + goto ERROR; } - return info; + *handle = info; + return code; -cleanup: +ERROR: smlDestroyInfo(info); - return NULL; + return code; } static int32_t smlPushCols(SArray *colsArray, SArray *cols) { @@ -1472,14 +1675,21 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { for (size_t i = 0; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); terrno = 0; - taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); + int32_t code = taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); if (terrno == TSDB_CODE_DUP_KEY) { taosHashCleanup(kvHash); return TSDB_CODE_PAR_DUPLICATED_COLUMN; } + if (code != TSDB_CODE_SUCCESS){ + taosHashCleanup(kvHash); + return code; + } } - taosArrayPush(colsArray, &kvHash); + if (taosArrayPush(colsArray, &kvHash) == NULL) { + taosHashCleanup(kvHash); + return TSDB_CODE_OUT_OF_MEMORY; + } return TSDB_CODE_SUCCESS; } @@ -1542,9 +1752,10 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { } else { uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); - SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); - if (meta == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + SSmlSTableMeta *meta = NULL; + ret = smlBuildSTableMeta(info->dataFormat, &meta); + if (ret != TSDB_CODE_SUCCESS) { + return ret; } ret = taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); if (ret != TSDB_CODE_SUCCESS) { @@ -1572,11 +1783,17 @@ static int32_t smlInsertData(SSmlHandle *info) { if (info->pRequest->dbList == NULL) { info->pRequest->dbList = taosArrayInit(1, TSDB_DB_FNAME_LEN); + if (info->pRequest->dbList == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } } char *data = (char *)taosArrayReserve(info->pRequest->dbList, 1); + if (data == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); - tNameGetFullDbName(&pName, data); + (void)tNameGetFullDbName(&pName, data); //ignore SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); while (oneTable) { @@ -1584,18 +1801,26 @@ static int32_t smlInsertData(SSmlHandle *info) { int measureLen = tableData->sTableNameLen; char *measure = (char *)taosMemoryMalloc(tableData->sTableNameLen); - memcpy(measure, tableData->sTableName, tableData->sTableNameLen); + if (measure == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(measure, tableData->sTableName, tableData->sTableNameLen); PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); smlStrReplace(measure, measureLen); - memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - memcpy(pName.tname, measure, measureLen); + (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + (void)memcpy(pName.tname, measure, measureLen); if (info->pRequest->tableList == NULL) { info->pRequest->tableList = taosArrayInit(1, sizeof(SName)); + if (info->pRequest->tableList == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + } + if (taosArrayPush(info->pRequest->tableList, &pName) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; } - taosArrayPush(info->pRequest->tableList, &pName); - strcpy(pName.tname, tableData->childTableName); + tstrncpy(pName.tname, tableData->childTableName, sizeof(pName.tname)); SRequestConnInfo conn = {0}; conn.pTrans = info->taos->pAppInfo->pTransporter; @@ -1618,7 +1843,13 @@ static int32_t smlInsertData(SSmlHandle *info) { taosHashCancelIterate(info->childTables, oneTable); return code; } - taosHashPut(info->pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg)); + code = taosHashPut(info->pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg)); + if (code != TSDB_CODE_SUCCESS){ + uError("SML:0x%" PRIx64 " taosHashPut failed. table name: %s", info->id, tableData->childTableName); + taosMemoryFree(measure); + taosHashCancelIterate(info->childTables, oneTable); + return code; + } SSmlSTableMeta **pMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen); @@ -1657,7 +1888,8 @@ static int32_t smlInsertData(SSmlHandle *info) { SAppClusterSummary *pActivity = &info->taos->pAppInfo->summary; atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); - launchQueryImpl(info->pRequest, info->pQuery, true, NULL); + (void)launchQueryImpl(info->pRequest, info->pQuery, true, NULL); // no need to check return code + uDebug("SML:0x%" PRIx64 " smlInsertData end, format:%d, code:%d,%s", info->id, info->dataFormat, info->pRequest->code, tstrerror(info->pRequest->code)); @@ -1777,7 +2009,11 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char if (code != TSDB_CODE_SUCCESS) { if (rawLine != NULL) { char *print = taosMemoryCalloc(len + 1, 1); - memcpy(print, tmp, len); + if (print == NULL) { + uError("SML:0x%" PRIx64 " smlParseLine failed. out of memory", info->id); + return code; + } + (void)memcpy(print, tmp, len); uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, print); taosMemoryFree(print); } else { @@ -1892,7 +2128,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, int protocol, int precision, int32_t ttl, int64_t reqid) { int32_t code = TSDB_CODE_SUCCESS; if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; + uError("SML:taos_schemaless_insert error taos is null"); return NULL; } SRequestObj *request = NULL; @@ -1905,11 +2141,13 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, return NULL; } - info = smlBuildSmlInfo(taos); - if (info == NULL) { - request->code = TSDB_CODE_OUT_OF_MEMORY; - uError("SML:taos_schemaless_insert error SSmlHandle is null"); - return (TAOS_RES *)request; + SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; + code = smlBuildSmlInfo(taos, &info); + if (code != TSDB_CODE_SUCCESS) { + request->code = code; + smlBuildInvalidDataMsg(&msg, "init SSmlHandle failed", NULL); + uError("SML:taos_schemaless_insert error SSmlHandle is null, err msg:%s", tstrerror(code)); + goto end; } info->pRequest = request; info->pRequest->pQuery = info->pQuery; @@ -1922,7 +2160,6 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, smlSetReqSQL(request, lines, rawLine, rawLineEnd); - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; if (request->pDb == NULL) { request->code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; smlBuildInvalidDataMsg(&msg, "Database not specified", NULL); @@ -1961,7 +2198,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, break; } taosMsleep(100); - refreshMeta(request->pTscObj, request); + (void)refreshMeta(request->pTscObj, request); //ignore return code,try again uInfo("SML:%" PRIx64 " retry:%d/10,ver is old retry or object is creating code:%d, msg:%s", info->id, cnt, code, tstrerror(code)); smlDestroyInfo(info); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 845884d5ac..519e55d0d1 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -29,29 +29,29 @@ (start)++; \ } -static char *smlJsonGetObj(char *payload) { +static int32_t smlJsonGetObj(char **payload) { int leftBracketCnt = 0; bool isInQuote = false; - while (*payload) { - if (*payload == '"' && *(payload - 1) != '\\') { + while (**payload) { + if (**payload == '"' && *((*payload) - 1) != '\\') { isInQuote = !isInQuote; - } else if (!isInQuote && unlikely(*payload == '{')) { + } else if (!isInQuote && unlikely(**payload == '{')) { leftBracketCnt++; - payload++; + (*payload)++; continue; - } else if (!isInQuote && unlikely(*payload == '}')) { + } else if (!isInQuote && unlikely(**payload == '}')) { leftBracketCnt--; - payload++; + (*payload)++; if (leftBracketCnt == 0) { - return payload; + return 0; } else if (leftBracketCnt < 0) { - return NULL; + return -1; } continue; } - payload++; + (*payload)++; } - return NULL; + return -1; } int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { @@ -99,8 +99,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { offset[index++] = *start - sTmp; element->timestamp = (*start); if (*(*start) == '{') { - char *tmp = smlJsonGetObj((*start)); - if (tmp) { + char *tmp = *start; + int32_t code = smlJsonGetObj(&tmp); + if (code == 0) { element->timestampLen = tmp - (*start); *start = tmp; } @@ -127,8 +128,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { offset[index++] = *start - sTmp; element->cols = (*start); if (*(*start) == '{') { - char *tmp = smlJsonGetObj((*start)); - if (tmp) { + char *tmp = *start; + int32_t code = smlJsonGetObj(&tmp); + if (code == 0) { element->colsLen = tmp - (*start); *start = tmp; } @@ -153,8 +155,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { JUMP_JSON_SPACE((*start)) offset[index++] = *start - sTmp; element->tags = (*start); - char *tmp = smlJsonGetObj((*start)); - if (tmp) { + char *tmp = *start; + int32_t code = smlJsonGetObj(&tmp); + if (code == 0) { element->tagsLen = tmp - (*start); *start = tmp; } @@ -209,8 +212,9 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { (*start) += offset[index++]; element->timestamp = *start; if (*(*start) == '{') { - char *tmp = smlJsonGetObj((*start)); - if (tmp) { + char *tmp = *start; + int32_t code = smlJsonGetObj(&tmp); + if (code == 0) { element->timestampLen = tmp - (*start); *start = tmp; } @@ -227,8 +231,9 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { (*start) += offset[index++]; element->cols = *start; if (*(*start) == '{') { - char *tmp = smlJsonGetObj((*start)); - if (tmp) { + char *tmp = *start; + int32_t code = smlJsonGetObj(&tmp); + if (code == 0) { element->colsLen = tmp - (*start); *start = tmp; } @@ -244,8 +249,9 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { } else if ((*start)[1] == 't' && (*start)[2] == 'a') { (*start) += offset[index++]; element->tags = (*start); - char *tmp = smlJsonGetObj((*start)); - if (tmp) { + char *tmp = *start; + int32_t code = smlJsonGetObj(&tmp); + if (code == 0) { element->tagsLen = tmp - (*start); *start = tmp; } @@ -261,7 +267,7 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { uError("elements != %d", OTD_JSON_FIELDS_NUM); return TSDB_CODE_TSC_INVALID_JSON; } - return 0; + return TSDB_CODE_SUCCESS; } static inline int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *metric, SSmlLineInfo *elements) { @@ -492,20 +498,17 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ int32_t tagNum = cJSON_GetArraySize(tags); if (unlikely(tagNum == 0)) { uError("SML:Tag should not be empty"); - terrno = TSDB_CODE_TSC_INVALID_JSON; - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } for (int32_t i = 0; i < tagNum; ++i) { cJSON *tag = cJSON_GetArrayItem(tags, i); if (unlikely(tag == NULL)) { - terrno = TSDB_CODE_TSC_INVALID_JSON; - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } size_t keyLen = strlen(tag->string); if (unlikely(IS_INVALID_COL_LEN(keyLen))) { uError("OTD:Tag key length is 0 or too large than 64"); - terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - return -1; + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } // add kv to SSmlKv @@ -516,19 +519,17 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ // value int32_t ret = smlParseValueFromJSON(tag, &kv); if (unlikely(ret != TSDB_CODE_SUCCESS)) { - terrno = ret; - return -1; + return ret; } taosArrayPush(preLineKV, &kv); if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { - terrno = TSDB_CODE_SUCCESS; - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } cnt++; } - return 0; + return TSDB_CODE_SUCCESS; } static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) { @@ -536,12 +537,18 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo if(info->dataFormat){ ret = smlProcessSuperTable(info, elements); if(ret != 0){ - return terrno; + if(info->reRun){ + return TSDB_CODE_SUCCESS; + } + return ret; } } ret = smlProcessTagJson(info, tags); if(ret != 0){ - return terrno; + if(info->reRun){ + return TSDB_CODE_SUCCESS; + } + return ret; } ret = smlJoinMeasureTag(elements); if(ret != 0){ @@ -703,6 +710,9 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo // Parse tags bool needFree = info->dataFormat; elements->tags = cJSON_PrintUnformatted(tagsJson); + if (elements->tags == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } elements->tagsLen = strlen(elements->tags); if (is_same_child_table_telnet(elements, &info->preLine) != 0) { ret = smlParseTagsFromJSON(info, tagsJson, elements); @@ -823,7 +833,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * } if (ret != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_VALUE; + return ret; } if (unlikely(**start == '\0' && elements->measure == NULL)) return TSDB_CODE_SUCCESS; @@ -844,18 +854,23 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * cJSON *valueJson = cJSON_Parse(elements->cols); if (unlikely(valueJson == NULL)) { uError("SML:0x%" PRIx64 " parse json cols failed:%s", info->id, elements->cols); + elements->cols[elements->colsLen] = tmp; return TSDB_CODE_TSC_INVALID_JSON; } - taosArrayPush(info->tagJsonArray, &valueJson); + if (taosArrayPush(info->tagJsonArray, &valueJson) == NULL){ + cJSON_Delete(valueJson); + elements->cols[elements->colsLen] = tmp; + return TSDB_CODE_OUT_OF_MEMORY; + } ret = smlParseValueFromJSONObj(valueJson, &kv); if (ret != TSDB_CODE_SUCCESS) { - uError("SML:Failed to parse value from JSON Obj:%s", elements->cols); + uError("SML:0x%" PRIx64 " Failed to parse value from JSON Obj:%s", info->id, elements->cols); elements->cols[elements->colsLen] = tmp; return TSDB_CODE_TSC_INVALID_VALUE; } elements->cols[elements->colsLen] = tmp; } else if (smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) { - uError("SML:cols invalidate:%s", elements->cols); + uError("SML:0x%" PRIx64 " cols invalidate:%s", info->id, elements->cols); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -870,7 +885,11 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * return TSDB_CODE_TSC_INVALID_JSON; } - taosArrayPush(info->tagJsonArray, &tagsJson); + if (taosArrayPush(info->tagJsonArray, &tagsJson) == NULL){ + cJSON_Delete(tagsJson); + uError("SML:0x%" PRIx64 " taosArrayPush failed", info->id); + return TSDB_CODE_OUT_OF_MEMORY; + } ret = smlParseTagsFromJSON(info, tagsJson, elements); if (unlikely(ret)) { uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); @@ -937,7 +956,7 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) { return ret; } info->lines = (SSmlLineInfo *)tmp; - memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo)); + (void)memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo)); } ret = smlParseJSONString(info, &dataPointStart, info->lines + cnt); if ((info->lines + cnt)->measure == NULL) break; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 7535cbfd0c..f2443080a6 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -103,7 +103,10 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { return code; } char* tmp = taosMemoryCalloc(pVal->length, 1); - memcpy(tmp, pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN); + if (tmp == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(tmp, pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN); code = doGeomFromText(tmp, (unsigned char **)&pVal->value, &pVal->length); taosMemoryFree(tmp); if (code != TSDB_CODE_SUCCESS) { @@ -212,8 +215,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ while (*sql < sqlEnd) { if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); - terrno = TSDB_CODE_SML_INVALID_DATA; - return -1; + return TSDB_CODE_SML_INVALID_DATA; } if (unlikely(IS_EQUAL(*sql,escapeChar))) { keyLen = *sql - key; @@ -230,8 +232,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); - terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - return -1; + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } // parse value @@ -245,8 +246,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ break; } else if (unlikely(IS_EQUAL(*sql,escapeChar))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); - terrno = TSDB_CODE_SML_INVALID_DATA; - return -1; + return TSDB_CODE_SML_INVALID_DATA; } if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) { @@ -261,24 +261,28 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ if (unlikely(valueLen == 0)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); - terrno = TSDB_CODE_SML_INVALID_DATA; - return -1; + return TSDB_CODE_SML_INVALID_DATA; } if (unlikely(valueLen - valueLenEscaped > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { - terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - return -1; + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; } if (keyEscaped) { char *tmp = (char *)taosMemoryMalloc(keyLen); - memcpy(tmp, key, keyLen); + if (tmp == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(tmp, key, keyLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, keyLen); key = tmp; } if (valueEscaped) { char *tmp = (char *)taosMemoryMalloc(valueLen); - memcpy(tmp, value, valueLen); + if (tmp == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(tmp, value, valueLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, valueLen); value = tmp; } @@ -289,11 +293,12 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ .length = valueLen, .keyEscaped = keyEscaped, .valueEscaped = valueEscaped}; - taosArrayPush(preLineKV, &kv); + if(taosArrayPush(preLineKV, &kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { - terrno = TSDB_CODE_SUCCESS; - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } cnt++; @@ -302,7 +307,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ } (*sql)++; } - return 0; + return TSDB_CODE_SUCCESS; } static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *elements) { @@ -315,13 +320,19 @@ static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if(info->dataFormat){ ret = smlProcessSuperTable(info, elements); if(ret != 0){ - return terrno; + if(info->reRun){ + return TSDB_CODE_SUCCESS; + } + return ret; } } ret = smlProcessTagLine(info, sql, sqlEnd); if(ret != 0){ - return terrno; + if (info->reRun){ + return TSDB_CODE_SUCCESS; + } + return ret; } return smlProcessChildTable(info, elements); @@ -410,7 +421,10 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if (keyEscaped) { char *tmp = (char *)taosMemoryMalloc(kv.keyLen); - memcpy(tmp, key, kv.keyLen); + if (tmp == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(tmp, key, kv.keyLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, kv.keyLen); kv.key = tmp; kv.keyEscaped = keyEscaped; @@ -418,7 +432,10 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if (valueEscaped) { char *tmp = (char *)taosMemoryMalloc(kv.length); - memcpy(tmp, kv.value, kv.length); + if (tmp == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(tmp, kv.value, kv.length); PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length); ASSERT(kv.type != TSDB_DATA_TYPE_GEOMETRY); if(kv.type == TSDB_DATA_TYPE_VARBINARY){ @@ -437,8 +454,13 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL } else { if (currElement->colArray == NULL) { currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1); + if (currElement->colArray == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + if (taosArrayPush(currElement->colArray, &kv) == NULL){ // reserve for timestamp + return TSDB_CODE_OUT_OF_MEMORY; } - taosArrayPush(currElement->colArray, &kv); // reserve for timestamp } cnt++; diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index bc0e560178..ddd2cf5ad0 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -84,8 +84,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ while (sql < sqlEnd) { if (unlikely(*sql == SPACE)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); - terrno = TSDB_CODE_SML_INVALID_DATA; - return -1; + return TSDB_CODE_SML_INVALID_DATA; } if (unlikely(*sql == EQUAL)) { keyLen = sql - key; @@ -97,8 +96,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ if (unlikely(IS_INVALID_COL_LEN(keyLen))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); - terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - return -1; + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } // parse value @@ -111,8 +109,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ } if (unlikely(*sql == EQUAL)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); - terrno = TSDB_CODE_SML_INVALID_DATA; - return -1; + return TSDB_CODE_SML_INVALID_DATA; } sql++; } @@ -120,13 +117,11 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ if (unlikely(valueLen == 0)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); - terrno = TSDB_CODE_TSC_INVALID_VALUE; - return -1; + return TSDB_CODE_TSC_INVALID_VALUE; } if (unlikely(valueLen > (TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { - terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - return -1; + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; } SSmlKv kv = {.key = key, @@ -138,12 +133,11 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ .valueEscaped = false}; taosArrayPush(preLineKV, &kv); if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { - terrno = TSDB_CODE_SUCCESS; - return -1; + return TSDB_CODE_SML_INVALID_DATA; } cnt++; } - return 0; + return TSDB_CODE_SUCCESS; } static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements) { @@ -156,13 +150,19 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS if(info->dataFormat){ ret = smlProcessSuperTable(info, elements); if(ret != 0){ - return terrno; + if(info->reRun){ + return TSDB_CODE_SUCCESS; + } + return ret; } } ret = smlProcessTagTelnet(info, data, sqlEnd); if(ret != 0){ - return terrno; + if (info->reRun){ + return TSDB_CODE_SUCCESS; + } + return ret; } ret = smlJoinMeasureTag(elements); diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 054698fff0..77079ed863 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -40,13 +40,16 @@ TEST(testCase, smlParseInfluxString_Test) { msgBuf.len = 256; SSmlLineInfo elements = {0}; - SSmlHandle *info = smlBuildSmlInfo(NULL); + SSmlHandle *info = nullptr; + int32_t code = smlBuildSmlInfo(nullptr, &info); + ASSERT_EQ(code, 0); info->protocol = TSDB_SML_LINE_PROTOCOL; info->dataFormat = false; // case 1 char *tmp = "\\,st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3"; char *sql = (char *)taosMemoryCalloc(256, 1); - memcpy(sql, tmp, strlen(tmp) + 1); + ASSERT_NE(sql, nullptr); + (void)memcpy(sql, tmp, strlen(tmp) + 1); int ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); @@ -62,22 +65,22 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; // case 2 false tmp = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_NE(ret, 0); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; // case 4 tag is null tmp = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000"; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); @@ -92,13 +95,13 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; // case 5 tag is null tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 "; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql + 1); @@ -112,36 +115,36 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + 1 + elements.measureTagsLen + 3 + elements.colsLen + 2); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; // case 6 tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 "; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; - smlClearForRerun(info); + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; + ASSERT_EQ(smlClearForRerun(info), 0); // case 7 tmp = " st , "; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_NE(ret, 0); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; // case 8 false tmp = ", st , "; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_NE(ret, 0); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; taosMemoryFree(sql); smlDestroyInfo(info); @@ -228,7 +231,9 @@ TEST(testCase, smlParseCols_Error_Test) { "escape_test,tag1=\"tag1_value\",tag2=\"tag2_value\" col0=\"co\"l\"0_value\",col1=\"col1_value\" 1680918783010000000" }; - SSmlHandle *info = smlBuildSmlInfo(NULL); + SSmlHandle *info = nullptr; + int32_t code = smlBuildSmlInfo(nullptr, &info); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); info->protocol = TSDB_SML_LINE_PROTOCOL; info->dataFormat = false; for (int i = 0; i < sizeof(data) / sizeof(data[0]); i++) { @@ -238,13 +243,14 @@ TEST(testCase, smlParseCols_Error_Test) { msgBuf.len = 256; int32_t len = strlen(data[i]); char *sql = (char *)taosMemoryCalloc(256, 1); - memcpy(sql, data[i], len + 1); + ASSERT_NE(sql, nullptr); + (void)memcpy(sql, data[i], len + 1); SSmlLineInfo elements = {0}; int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements); // printf("i:%d\n", i); ASSERT_NE(ret, TSDB_CODE_SUCCESS); taosMemoryFree(sql); - taosArrayDestroy(elements.colArray); + (void)taosArrayDestroy(elements.colArray); } smlDestroyInfo(info); } @@ -254,7 +260,9 @@ TEST(testCase, smlParseCols_Test) { SSmlMsgBuf msgBuf; msgBuf.buf = msg; msgBuf.len = 256; - SSmlHandle *info = smlBuildSmlInfo(NULL); + SSmlHandle *info = nullptr; + int32_t code = smlBuildSmlInfo(nullptr, &info); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); info->protocol = TSDB_SML_LINE_PROTOCOL; info->dataFormat = false; SSmlLineInfo elements = {0}; @@ -267,7 +275,8 @@ TEST(testCase, smlParseCols_Test) { "cboolf=f,cnch_=l\"iuwq\" 1626006833639000000"; int32_t len = strlen(data); char *sql = (char *)taosMemoryCalloc(1024, 1); - memcpy(sql, data, len + 1); + ASSERT_NE(sql, nullptr); + (void)memcpy(sql, data, len + 1); int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements); ASSERT_EQ(ret, TSDB_CODE_SUCCESS); int32_t size = taosArrayGetSize(elements.colArray); @@ -275,6 +284,7 @@ TEST(testCase, smlParseCols_Test) { // binary SSmlKv *kv = (SSmlKv *)taosArrayGet(elements.colArray, 1); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cb=in", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY); @@ -285,6 +295,7 @@ TEST(testCase, smlParseCols_Test) { // nchar kv = (SSmlKv *)taosArrayGet(elements.colArray, 2); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); @@ -293,6 +304,7 @@ TEST(testCase, smlParseCols_Test) { // bool kv = (SSmlKv *)taosArrayGet(elements.colArray, 3); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cbool", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); @@ -301,6 +313,7 @@ TEST(testCase, smlParseCols_Test) { // double kv = (SSmlKv *)taosArrayGet(elements.colArray, 4); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); @@ -310,6 +323,7 @@ TEST(testCase, smlParseCols_Test) { // float kv = (SSmlKv *)taosArrayGet(elements.colArray, 5); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cf64_", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); @@ -319,6 +333,7 @@ TEST(testCase, smlParseCols_Test) { // float kv = (SSmlKv *)taosArrayGet(elements.colArray, 6); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cf32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); @@ -328,6 +343,7 @@ TEST(testCase, smlParseCols_Test) { // tiny int kv = (SSmlKv *)taosArrayGet(elements.colArray, 7); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "ci8", 3), 0); ASSERT_EQ(kv->keyLen, 3); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_TINYINT); @@ -336,6 +352,7 @@ TEST(testCase, smlParseCols_Test) { // unsigned tiny int kv = (SSmlKv *)taosArrayGet(elements.colArray, 8); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cu8", 3), 0); ASSERT_EQ(kv->keyLen, 3); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UTINYINT); @@ -344,6 +361,7 @@ TEST(testCase, smlParseCols_Test) { // small int kv = (SSmlKv *)taosArrayGet(elements.colArray, 9); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "ci16", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_SMALLINT); @@ -352,6 +370,7 @@ TEST(testCase, smlParseCols_Test) { // unsigned smallint kv = (SSmlKv *)taosArrayGet(elements.colArray, 10); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cu16", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_USMALLINT); @@ -360,6 +379,7 @@ TEST(testCase, smlParseCols_Test) { // int kv = (SSmlKv *)taosArrayGet(elements.colArray, 11); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "ci32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_INT); @@ -368,6 +388,7 @@ TEST(testCase, smlParseCols_Test) { // unsigned int kv = (SSmlKv *)taosArrayGet(elements.colArray, 12); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cu32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UINT); @@ -376,6 +397,7 @@ TEST(testCase, smlParseCols_Test) { // bigint kv = (SSmlKv *)taosArrayGet(elements.colArray, 13); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "ci64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); @@ -384,6 +406,7 @@ TEST(testCase, smlParseCols_Test) { // bigint kv = (SSmlKv *)taosArrayGet(elements.colArray, 14); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "ci", 2), 0); ASSERT_EQ(kv->keyLen, 2); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); @@ -392,6 +415,7 @@ TEST(testCase, smlParseCols_Test) { // unsigned bigint kv = (SSmlKv *)taosArrayGet(elements.colArray, 15); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cu64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UBIGINT); @@ -400,6 +424,7 @@ TEST(testCase, smlParseCols_Test) { // bool kv = (SSmlKv *)taosArrayGet(elements.colArray, 16); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cbooltrue", 9), 0); ASSERT_EQ(kv->keyLen, 9); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); @@ -408,6 +433,7 @@ TEST(testCase, smlParseCols_Test) { // bool kv = (SSmlKv *)taosArrayGet(elements.colArray, 17); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cboolt", 6), 0); ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); @@ -416,6 +442,7 @@ TEST(testCase, smlParseCols_Test) { // bool kv = (SSmlKv *)taosArrayGet(elements.colArray, 18); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cboolf", 6), 0); ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); @@ -424,13 +451,14 @@ TEST(testCase, smlParseCols_Test) { // nchar kv = (SSmlKv *)taosArrayGet(elements.colArray, 19); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cnch_", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); ASSERT_EQ(kv->length, 4); ASSERT_EQ(strncasecmp(kv->value, "iuwq", 4), 0); - taosArrayDestroy(elements.colArray); + (void)taosArrayDestroy(elements.colArray); taosMemoryFree(sql); smlDestroyInfo(info); } @@ -471,7 +499,9 @@ TEST(testCase, smlParseNumber_Test) { } TEST(testCase, smlParseTelnetLine_error_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL); + SSmlHandle *info = nullptr; + int32_t code = smlBuildSmlInfo(nullptr, &info); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); info->dataFormat = false; info->protocol = TSDB_SML_TELNET_PROTOCOL; ASSERT_NE(info, nullptr); @@ -509,7 +539,9 @@ TEST(testCase, smlParseTelnetLine_error_Test) { } TEST(testCase, smlParseTelnetLine_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL); + SSmlHandle *info = nullptr; + int32_t code = smlBuildSmlInfo(nullptr, &info); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); info->dataFormat = false; info->protocol = TSDB_SML_TELNET_PROTOCOL; ASSERT_NE(info, nullptr); @@ -528,7 +560,9 @@ TEST(testCase, smlParseTelnetLine_Test) { } //TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { -// SSmlHandle *info = smlBuildSmlInfo(NULL); +// SSmlHandle *info = nullptr; +// int32_t code = smlBuildSmlInfo(nullptr, &info); +// ASSERT_EQ(code, TSDB_CODE_SUCCESS); // info->protocol = TSDB_SML_JSON_PROTOCOL; // ASSERT_NE(info, nullptr); // @@ -568,7 +602,7 @@ TEST(testCase, smlParseNumber_performance_Test) { for (int j = 0; j < 10000000; ++j) { kv.value = str[i]; kv.length = strlen(str[i]); - smlParseNumber(&kv, &msgBuf); + (void)smlParseNumber(&kv, &msgBuf); //ignore result } printf("smlParseNumber:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t1); printf("\n"); @@ -576,7 +610,7 @@ TEST(testCase, smlParseNumber_performance_Test) { for (int j = 0; j < 10000000; ++j) { kv.value = str[i]; kv.length = strlen(str[i]); - smlParseNumberOld(&kv, &msgBuf); + (void)smlParseNumberOld(&kv, &msgBuf); //ignore result } printf("smlParseNumberOld:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t2); printf("\n\n"); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 809721d606..9b5ecb4693 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2486,7 +2486,10 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, cha RandTableName rname = { .tags = tags, .stbFullName = stbFullName, .stbFullNameLen = strlen(stbFullName), .ctbShortName = cname}; - buildChildTableName(&rname); + int32_t code = buildChildTableName(&rname); + if(code != TSDB_CODE_SUCCESS){ + return code; + } taosArrayDestroy(tags); if ((rname.ctbShortName && rname.ctbShortName[0]) == 0) { diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 4fe2beb6aa..e4604c6bab 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -293,17 +293,19 @@ static int compareKv(const void* p1, const void* p2) { /* * use stable name and tags to grearate child table name */ -void buildChildTableName(RandTableName* rName) { +int32_t buildChildTableName(RandTableName* rName) { SStringBuilder sb = {0}; taosStringBuilderAppendStringLen(&sb, rName->stbFullName, rName->stbFullNameLen); if (sb.buf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return; + return TSDB_CODE_OUT_OF_MEMORY; } taosArraySort(rName->tags, compareKv); for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) { taosStringBuilderAppendChar(&sb, ','); SSmlKv* tagKv = taosArrayGet(rName->tags, j); + if(tagKv == NULL) { + return TSDB_CODE_SML_INVALID_DATA; + } taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen); taosStringBuilderAppendChar(&sb, '='); if (IS_VAR_DATA_TYPE(tagKv->type)) { @@ -323,8 +325,9 @@ void buildChildTableName(RandTableName* rName) { rName->ctbShortName[0] = 't'; rName->ctbShortName[1] = '_'; for (int i = 0; i < 16; i++) { - sprintf(temp, "%02x", context.digest[i]); - strcat(rName->ctbShortName, temp); + (void)sprintf(temp, "%02x", context.digest[i]); + (void)strcat(rName->ctbShortName, temp); } taosStringBuilderDestroy(&sb); + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index db2d34b844..89eaf14f75 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -112,7 +112,11 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem for (int i = 0; i < tags->numOfBound; ++i) { SSchema* pTagSchema = &pSchema[tags->pColIndex[i]]; SSmlKv* kv = taosArrayGet(cols, i); - + if (kv == NULL){ + code = TSDB_CODE_SML_INVALID_DATA; + uError("SML smlBuildTagRow error kv is null"); + goto end; + } if (kv->keyLen != strlen(pTagSchema->name) || memcmp(kv->key, pTagSchema->name, kv->keyLen) != 0 || kv->type != pTagSchema->type) { code = TSDB_CODE_SML_INVALID_DATA; @@ -120,7 +124,11 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem goto end; } - taosArrayPush(*tagName, pTagSchema->name); + if (taosArrayPush(*tagName, pTagSchema->name) == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + uError("SML smlBuildTagRow error push tag name"); + goto end; + } STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; // strcpy(val.colName, pTagSchema->name); if (pTagSchema->type == TSDB_DATA_TYPE_BINARY || pTagSchema->type == TSDB_DATA_TYPE_VARBINARY || @@ -141,7 +149,7 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem goto end; } char buf[512] = {0}; - snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); + (void)snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); taosMemoryFree(p); code = buildSyntaxErrMsg(msg, buf, kv->value); goto end; @@ -149,9 +157,13 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem val.pData = p; val.nData = output; } else { - memcpy(&val.i64, &(kv->value), kv->length); + (void)memcpy(&val.i64, &(kv->value), kv->length); + } + if (taosArrayPush(pTagArray, &val) == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + uError("SML smlBuildTagRow error push tag array"); + goto end; } - taosArrayPush(pTagArray, &val); } code = tTagNew(pTagArray, 1, false, ppTag); @@ -166,20 +178,19 @@ end: return code; } -STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta) { - STableDataCxt* pTableCxt = NULL; +int32_t smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta, STableDataCxt** cxt) { SVCreateTbReq* pCreateTbReq = NULL; int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false, false); + sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, cxt, false, false); if (ret != TSDB_CODE_SUCCESS) { - return NULL; + return ret; } - ret = initTableColSubmitData(pTableCxt); + ret = initTableColSubmitData(*cxt); if (ret != TSDB_CODE_SUCCESS) { - return NULL; + return ret; } - return pTableCxt; + return TSDB_CODE_SUCCESS; } void clearColValArraySml(SArray* pCols) { @@ -197,6 +208,9 @@ void clearColValArraySml(SArray* pCols) { int32_t smlBuildRow(STableDataCxt* pTableCxt) { SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); + if (pRow == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } int ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); if (TSDB_CODE_SUCCESS != ret) { return ret; @@ -211,18 +225,22 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32 int ret = TSDB_CODE_SUCCESS; SSchema* pColSchema = schema + index; SColVal* pVal = taosArrayGet(pTableCxt->pValues, index); + if (pVal == NULL) { + return TSDB_CODE_SUCCESS; + } SSmlKv* kv = (SSmlKv*)data; if (kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 || kv->type != pColSchema->type) { ret = TSDB_CODE_SML_INVALID_DATA; char* tmp = taosMemoryCalloc(kv->keyLen + 1, 1); if (tmp) { - memcpy(tmp, kv->key, kv->keyLen); + (void)memcpy(tmp, kv->key, kv->keyLen); uInfo("SML data(name:%s type:%s) is not same like the db data(name:%s type:%s)", tmp, tDataTypes[kv->type].name, pColSchema->name, tDataTypes[pColSchema->type].name); taosMemoryFree(tmp); } else { uError("SML smlBuildCol out of memory"); + ret = TSDB_CODE_OUT_OF_MEMORY; } goto end; } @@ -256,9 +274,9 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32 } else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) { pVal->value.nData = kv->length; pVal->value.pData = taosMemoryMalloc(kv->length); - memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length); + (void)memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length); } else { - memcpy(&pVal->value.val, &(kv->value), kv->length); + (void)memcpy(&pVal->value.val, &(kv->value), kv->length); } pVal->flag = CV_FLAG_VALUE; @@ -276,8 +294,13 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc SVCreateTbReq* pCreateTblReq = NULL; SArray* tagName = NULL; - insInitBoundColsInfo(getNumOfTags(pTableMeta), &bindTags); - int ret = smlBoundColumnData(tags, &bindTags, pTagsSchema, true); + int ret = insInitBoundColsInfo(getNumOfTags(pTableMeta), &bindTags); + if (ret != TSDB_CODE_SUCCESS) { + buildInvalidOperationMsg(&pBuf, "init bound cols error"); + goto end; + } + + ret = smlBoundColumnData(tags, &bindTags, pTagsSchema, true); if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "bound tags error"); goto end; @@ -299,7 +322,11 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc ttl); pCreateTblReq->ctb.stbName = taosMemoryCalloc(1, sTableNameLen + 1); - memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen); + if (pCreateTblReq->ctb.stbName == NULL){ + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + (void)memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen); if (dataFormat) { STableDataCxt** pTableCxt = (STableDataCxt**)taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, @@ -345,11 +372,18 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc for (int32_t r = 0; r < rowNum; ++r) { void* rowData = taosArrayGetP(cols, r); - + if (rowData == NULL) { + ret = TSDB_CODE_SML_INVALID_DATA; + goto end; + } // 1. set the parsed value from sql string for (int c = 0; c < pTableCxt->boundColsInfo.numOfBound; ++c) { SSchema* pColSchema = &pSchema[pTableCxt->boundColsInfo.pColIndex[c]]; SColVal* pVal = taosArrayGet(pTableCxt->pValues, pTableCxt->boundColsInfo.pColIndex[c]); + if (pVal == NULL) { + ret = TSDB_CODE_SML_INVALID_DATA; + goto end; + } void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name)); if (p == NULL) { continue; @@ -388,14 +422,22 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc } else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) { pVal->value.nData = kv->length; pVal->value.pData = taosMemoryMalloc(kv->length); - memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length); + if (NULL == pVal->value.pData) { + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + (void)memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length); } else { - memcpy(&pVal->value.val, &(kv->value), kv->length); + (void)memcpy(&pVal->value.val, &(kv->value), kv->length); } pVal->flag = CV_FLAG_VALUE; } SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); + if (NULL == pRow) { + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); if (TSDB_CODE_SUCCESS != ret) { buildInvalidOperationMsg(&pBuf, "tRowBuild error"); @@ -415,11 +457,12 @@ end: return ret; } -SQuery* smlInitHandle() { +int32_t smlInitHandle(SQuery** query) { + *query = NULL; SQuery* pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); if (NULL == pQuery) { uError("create pQuery error"); - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->haveResultSet = false; @@ -428,14 +471,22 @@ SQuery* smlInitHandle() { if (NULL == stmt) { uError("create SVnodeModifyOpStmt error"); qDestroyQuery(pQuery); - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (stmt->pTableBlockHashObj == NULL){ + uError("create pTableBlockHashObj error"); + qDestroyQuery(pQuery); + nodesDestroyNode((SNode*)stmt); + return TSDB_CODE_OUT_OF_MEMORY; + } stmt->freeHashFunc = insDestroyTableDataCxtHashMap; stmt->freeArrayFunc = insDestroyVgroupDataCxtList; pQuery->pRoot = (SNode*)stmt; - return pQuery; + *query = pQuery; + + return TSDB_CODE_SUCCESS; } int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash) { From 82b517dd2ae3a5c550a4b1cdd6ad29b3352d6873 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 15 Jul 2024 19:28:04 +0800 Subject: [PATCH 21/48] enh: add scheduler return code processing --- source/libs/scheduler/inc/schInt.h | 2 +- source/libs/scheduler/src/schJob.c | 143 ++++++++++++++++++++++++---- source/libs/scheduler/src/schUtil.c | 10 +- 3 files changed, 129 insertions(+), 26 deletions(-) diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index c7f6c20b7d..92459e06f5 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -606,7 +606,7 @@ void schFreeTask(SSchJob *pJob, SSchTask *pTask); void schDropTaskInHashList(SSchJob *pJob, SHashObj *list); int32_t schNotifyTaskInHashList(SSchJob *pJob, SHashObj *list, ETaskNotifyType type, SSchTask *pTask); int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level); -int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask); +void schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask); int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel); int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask); void schDirectPostJobRes(SSchedulerReq *pReq, int32_t errCode); diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 9ad1c1ff30..8601c943d9 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -167,9 +167,18 @@ _return: int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { for (int32_t i = 0; i < pJob->levelNum; ++i) { SSchLevel *pLevel = taosArrayGet(pJob->levels, i); - + if (NULL == pLevel) { + SCH_JOB_ELOG("fail to get the %dth level, levelNum: %d", i, pJob->levelNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + for (int32_t m = 0; m < pLevel->taskNum; ++m) { SSchTask *pTask = taosArrayGet(pLevel->subTasks, m); + if (NULL == pTask) { + SCH_JOB_ELOG("fail to get the %dth task in level %d, taskNum: %d", m, pLevel->level, pLevel->taskNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SSubplan *pPlan = pTask->plan; int32_t childNum = pPlan->pChildren ? (int32_t)LIST_LENGTH(pPlan->pChildren) : 0; int32_t parentNum = pPlan->pParents ? (int32_t)LIST_LENGTH(pPlan->pParents) : 0; @@ -189,6 +198,11 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { for (int32_t n = 0; n < childNum; ++n) { SSubplan *child = (SSubplan *)nodesListGetNode(pPlan->pChildren, n); + if (NULL == child) { + SCH_JOB_ELOG("fail to get the %dth child subplan, childNum: %d", n, childNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SSchTask **childTask = taosHashGet(planToTask, &child, POINTER_BYTES); if (NULL == childTask || NULL == *childTask) { SCH_TASK_ELOG("subplan children relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n); @@ -223,6 +237,11 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { for (int32_t n = 0; n < parentNum; ++n) { SSubplan *parent = (SSubplan *)nodesListGetNode(pPlan->pParents, n); + if (NULL == parent) { + SCH_JOB_ELOG("fail to get the %dth parent subplan, parentNum: %d", n, parentNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SSchTask **parentTask = taosHashGet(planToTask, &parent, POINTER_BYTES); if (NULL == parentTask || NULL == *parentTask) { SCH_TASK_ELOG("subplan parent relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n); @@ -242,6 +261,11 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { } SSchLevel *pLevel = taosArrayGet(pJob->levels, 0); + if (NULL == pLevel) { + SCH_JOB_ELOG("fail to get level 0 level, levelNum:%d", taosArrayGetSize(pJob->levels)); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + if (SCH_IS_QUERY_JOB(pJob)) { if (pLevel->taskNum > 1) { SCH_JOB_ELOG("invalid query plan, level:0, taskNum:%d", pLevel->taskNum); @@ -249,6 +273,11 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { } SSchTask *pTask = taosArrayGet(pLevel->subTasks, 0); + if (NULL == pLevel) { + SCH_JOB_ELOG("fail to get the first task in level 0, taskNum:%d", taosArrayGetSize(pLevel->subTasks)); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + if (SUBPLAN_TYPE_MODIFY != pTask->plan->subplanType || EXPLAIN_MODE_DISABLE != pJob->attr.explainMode) { pJob->attr.needFetch = true; } @@ -262,7 +291,9 @@ int32_t schAppendJobDataSrc(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } - taosArrayPush(pJob->dataSrcTasks, &pTask); + if (NULL == taosArrayPush(pJob->dataSrcTasks, &pTask)) { + SCH_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -319,6 +350,11 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { } pLevel = taosArrayGet(pJob->levels, i); + if (NULL == pLevel) { + SCH_JOB_ELOG("fail to get the %dth level, levelNum: %d", i, levelNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + pLevel->level = i; plans = (SNodeListNode *)nodesListGetNode(pDag->pSubplans, i); @@ -343,6 +379,10 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { for (int32_t n = 0; n < taskNum; ++n) { SSubplan *plan = (SSubplan *)nodesListGetNode(plans->pNodeList, n); + if (NULL == plan) { + SCH_JOB_ELOG("fail to get the %dth subplan, taskNum: %d", n, taskNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } SCH_SET_JOB_TYPE(pJob, plan->subplanType); @@ -422,10 +462,13 @@ int32_t schDumpJobFetchRes(SSchJob *pJob, void **pData) { if (NULL == *pData) { SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp)); - if (rsp) { - rsp->completed = 1; + if (NULL == rsp) { + SCH_JOB_ELOG("malloc SRetrieveTableRsp %d failed, code:%x", sizeof(SRetrieveTableRsp), terrno); + SCH_ERR_JRET(terrno); } + rsp->completed = 1; + *pData = rsp; SCH_JOB_DLOG("empty res and set query complete, code:%x", code); } @@ -441,8 +484,14 @@ _return: int32_t schNotifyUserExecRes(SSchJob *pJob) { SExecResult *pRes = taosMemoryCalloc(1, sizeof(SExecResult)); - if (pRes) { - schDumpJobExecRes(pJob, pRes); + if (NULL == pRes) { + qError("malloc execResult %d failed, error: %x", sizeof(SExecResult), terrno); + SCH_RET(terrno); + } + + int32_t code = schDumpJobExecRes(pJob, pRes); + if (TSDB_CODE_SUCCESS != code && TSDB_CODE_SUCCESS == atomic_load_32(pJob->errCode)) { + atomic_store_32(&pJob->errCode, code); } SCH_JOB_DLOG("sch start to invoke exec cb, code: %s", tstrerror(pJob->errCode)); @@ -455,7 +504,10 @@ int32_t schNotifyUserExecRes(SSchJob *pJob) { int32_t schNotifyUserFetchRes(SSchJob *pJob) { void *pRes = NULL; - schDumpJobFetchRes(pJob, &pRes); + int32_t code = schDumpJobFetchRes(pJob, &pRes); + if (TSDB_CODE_SUCCESS != code && TSDB_CODE_SUCCESS == atomic_load_32(pJob->errCode)) { + atomic_store_32(&pJob->errCode, code); + } SCH_JOB_DLOG("sch start to invoke fetch cb, code: %s", tstrerror(pJob->errCode)); (*pJob->userRes.fetchFp)(pRes, pJob->userRes.cbParam, atomic_load_32(&pJob->errCode)); @@ -479,13 +531,13 @@ void schPostJobRes(SSchJob *pJob, SCH_OP_TYPE op) { if (SCH_JOB_IN_SYNC_OP(pJob)) { SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); - tsem_post(&pJob->rspSem); + (void)tsem_post(&pJob->rspSem); // ignore error } else if (SCH_JOB_IN_ASYNC_EXEC_OP(pJob)) { SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); - schNotifyUserExecRes(pJob); + (void)schNotifyUserExecRes(pJob); // ignore error } else if (SCH_JOB_IN_ASYNC_FETCH_OP(pJob)) { SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); - schNotifyUserFetchRes(pJob); + (void)schNotifyUserFetchRes(pJob); // ignore error } else { SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); SCH_JOB_ELOG("job not in any operation, status:%s", jobTaskStatusStr(pJob->status)); @@ -520,7 +572,8 @@ int32_t schHandleJobFailure(SSchJob *pJob, int32_t errCode) { return TSDB_CODE_SCH_IGNORE_ERROR; } - schSwitchJobStatus(pJob, JOB_TASK_STATUS_FAIL, &errCode); + (void)schSwitchJobStatus(pJob, JOB_TASK_STATUS_FAIL, &errCode); // ignore error + return TSDB_CODE_SCH_IGNORE_ERROR; } @@ -531,7 +584,8 @@ int32_t schHandleJobDrop(SSchJob *pJob, int32_t errCode) { return TSDB_CODE_SCH_IGNORE_ERROR; } - schSwitchJobStatus(pJob, JOB_TASK_STATUS_DROP, &errCode); + (void)schSwitchJobStatus(pJob, JOB_TASK_STATUS_DROP, &errCode); // ignore error + return TSDB_CODE_SCH_IGNORE_ERROR; } @@ -573,9 +627,18 @@ int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask) { atomic_sub_fetch_32(&pJob->levelIdx, 1); pLevel = taosArrayGet(pJob->levels, pJob->levelIdx); + if (NULL == pLevel) { + SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", pJob->levelIdx, taosArrayGetSize(pJob->levels)); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + for (int32_t i = 0; i < pLevel->taskNum; ++i) { SSchTask *pTask = taosArrayGet(pLevel->subTasks, i); - + if (NULL == pTask) { + SCH_JOB_ELOG("fail to get the %dth task in level %d, taskNum:%d", i, pLevel->level, pLevel->taskNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + if (pTask->children && taosArrayGetSize(pTask->children) > 0) { continue; } @@ -603,7 +666,11 @@ int32_t schSaveJobExecRes(SSchJob *pJob, SQueryTableRsp *rsp) { } } - taosArrayAddBatch((SArray *)pJob->execRes.res, taosArrayGet(rsp->tbVerInfo, 0), taosArrayGetSize(rsp->tbVerInfo)); + if (NULL == taosArrayAddBatch((SArray *)pJob->execRes.res, taosArrayGet(rsp->tbVerInfo, 0), taosArrayGetSize(rsp->tbVerInfo))) { + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + SCH_ERR_RET(terrno); + } + taosArrayDestroy(rsp->tbVerInfo); pJob->execRes.msgType = TDMT_SCH_QUERY; @@ -630,6 +697,11 @@ int32_t schLaunchJob(SSchJob *pJob) { SCH_ERR_RET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_PART_SUCC, NULL)); } else { SSchLevel *level = taosArrayGet(pJob->levels, pJob->levelIdx); + if (NULL == level) { + SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", pJob->levelIdx, taosArrayGetSize(pJob->levels)); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SCH_ERR_RET(schLaunchLevelTasks(pJob, level)); } @@ -662,10 +734,19 @@ void schFreeJobImpl(void *job) { int32_t numOfLevels = taosArrayGetSize(pJob->levels); for (int32_t i = 0; i < numOfLevels; ++i) { SSchLevel *pLevel = taosArrayGet(pJob->levels, i); + if (NULL == pLevel) { + SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", i, numOfLevels); + continue; + } int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks); for (int32_t j = 0; j < numOfTasks; ++j) { SSchTask *pTask = taosArrayGet(pLevel->subTasks, j); + if (NULL == pLevel) { + SCH_JOB_ELOG("fail to get the %dth task, taskNum:%d", j, numOfTasks); + continue; + } + schFreeTask(pJob, pTask); } @@ -688,12 +769,12 @@ void schFreeJobImpl(void *job) { destroyQueryExecRes(&pJob->execRes); qDestroyQueryPlan(pJob->pDag); - nodesReleaseAllocatorWeakRef(pJob->allocatorRefId); + (void)nodesReleaseAllocatorWeakRef(pJob->allocatorRefId); // ignore error taosMemoryFreeClear(pJob->userRes.execRes); taosMemoryFreeClear(pJob->fetchRes); taosMemoryFreeClear(pJob->sql); - tsem_destroy(&pJob->rspSem); + (void)tsem_destroy(&pJob->rspSem); // ignore error taosMemoryFree(pJob); int32_t jobNum = atomic_sub_fetch_32(&schMgmt.jobNum, 1); @@ -712,7 +793,7 @@ int32_t schJobFetchRows(SSchJob *pJob) { if (schChkCurrentOp(pJob, SCH_OP_FETCH, true)) { SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); - tsem_wait(&pJob->rspSem); + (void)tsem_wait(&pJob->rspSem); // ignore error SCH_RET(schDumpJobFetchRes(pJob, pJob->userRes.fetchRes)); } } else { @@ -740,9 +821,18 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { pJob->conn = *pReq->pConn; if (pReq->sql) { pJob->sql = taosStrdup(pReq->sql); + if (NULL == pJob->sql) { + qError("QID:0x%" PRIx64 " strdup sql %s failed", pReq->pDag->queryId, pReq->sql); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } } pJob->pDag = pReq->pDag; pJob->allocatorRefId = nodesMakeAllocatorWeakRef(pReq->allocatorRefId); + if (pJob->allocatorRefId <= 0) { + qError("QID:0x%" PRIx64 " nodesMakeAllocatorWeakRef failed", pReq->pDag->queryId); + SCH_ERR_JRET(terrno); + } + pJob->chkKillFp = pReq->chkKillFp; pJob->chkKillParam = pReq->chkKillParam; pJob->userRes.execFp = pReq->execFp; @@ -753,6 +843,10 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { qDebug("QID:0x%" PRIx64 " input exec nodeList is empty", pReq->pDag->queryId); } else { pJob->nodeList = taosArrayDup(pReq->pNodeList, NULL); + if (NULL == pJob->nodeList) { + qError("QID:0x%" PRIx64 " taosArrayDup failed, origNum:%d", pReq->pDag->queryId, taosArrayGetSize(pReq->pNodeList)); + SCH_ERR_JRET(terrno); + } } pJob->taskList = taosHashInit(pReq->pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, @@ -801,7 +895,7 @@ _return: } else if (pJob->refId < 0) { schFreeJobImpl(pJob); } else { - taosRemoveRef(schMgmt.jobRef, pJob->refId); + (void)taosRemoveRef(schMgmt.jobRef, pJob->refId); // ignore error } SCH_RET(code); @@ -815,7 +909,7 @@ int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq) { if (pReq->syncReq) { SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); - tsem_wait(&pJob->rspSem); + (void)tsem_wait(&pJob->rspSem); // ignore error } SCH_JOB_DLOG("job exec done, job status:%s, jobId:0x%" PRIx64, SCH_GET_JOB_STATUS_STR(pJob), pJob->refId); @@ -846,7 +940,7 @@ int32_t schChkResetJobRetry(SSchJob *pJob, int32_t rspCode) { } SCH_UNLOCK(SCH_WRITE, &pJob->resLock); - schUpdateJobStatus(pJob, JOB_TASK_STATUS_EXEC); + SCH_ERR_RET(schUpdateJobStatus(pJob, JOB_TASK_STATUS_EXEC)); } return TSDB_CODE_SUCCESS; @@ -867,6 +961,10 @@ int32_t schResetJobForRetry(SSchJob *pJob, int32_t rspCode, bool *inRetry) { int32_t numOfLevels = taosArrayGetSize(pJob->levels); for (int32_t i = 0; i < numOfLevels; ++i) { SSchLevel *pLevel = taosArrayGet(pJob->levels, i); + if (NULL == pLevel) { + SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", i, numOfLevels); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } pLevel->taskExecDoneNum = 0; pLevel->taskLaunchedNum = 0; @@ -874,6 +972,11 @@ int32_t schResetJobForRetry(SSchJob *pJob, int32_t rspCode, bool *inRetry) { int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks); for (int32_t j = 0; j < numOfTasks; ++j) { SSchTask *pTask = taosArrayGet(pLevel->subTasks, j); + if (NULL == pTask) { + SCH_JOB_ELOG("fail to get the %dth task in level %d, taskNum:%d", j, i, numOfTasks); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SCH_LOCK_TASK(pTask); code = schChkUpdateRedirectCtx(pJob, pTask, NULL, rspCode); if (TSDB_CODE_SUCCESS != code) { diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 82b2e021af..689c98d395 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -316,18 +316,18 @@ void schFreeRpcCtx(SRpcCtx *pCtx) { } } -int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask) { +void schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask) { + *pTask = NULL; + int32_t s = taosHashGetSize(pTaskList); if (s <= 0) { - return TSDB_CODE_SUCCESS; + return; } SSchTask **task = taosHashGet(pTaskList, &taskId, sizeof(taskId)); if (NULL == task || NULL == (*task)) { - return TSDB_CODE_SUCCESS; + return; } *pTask = *task; - - return TSDB_CODE_SUCCESS; } From 51e9723d6673360f48f1ee397ce6516d82b45b20 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 15 Jul 2024 19:37:52 +0800 Subject: [PATCH 22/48] refactor: do some refactor. --- include/util/tarray.h | 2 +- source/client/src/clientHb.c | 5 ++- source/common/src/tdatablock.c | 4 ++- source/common/src/tmsg.c | 8 +++-- source/dnode/mnode/impl/src/mndStreamUtil.c | 3 +- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tq/tqSink.c | 12 +++++-- source/dnode/vnode/src/tq/tqUtil.c | 12 ++++--- source/dnode/vnode/src/tsdb/tsdbCache.c | 3 +- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 3 +- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 5 ++- source/dnode/vnode/src/tsdb/tsdbRead2.c | 16 +++++++--- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 31 ++++++++++++------- source/libs/executor/src/executil.c | 6 ++-- source/libs/executor/src/streamfilloperator.c | 4 ++- .../executor/src/streamtimewindowoperator.c | 3 +- source/libs/executor/src/timewindowoperator.c | 5 +-- source/libs/stream/src/streamCheckStatus.c | 4 ++- source/libs/stream/src/streamMsg.c | 6 ++-- source/libs/stream/src/streamTask.c | 17 +++++++--- source/util/src/tarray.c | 4 +-- source/util/src/tconfig.c | 3 +- 22 files changed, 105 insertions(+), 53 deletions(-) diff --git a/include/util/tarray.h b/include/util/tarray.h index e494f78f48..cef09e40f8 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -205,7 +205,7 @@ void taosArrayClearEx(SArray* pArray, void (*fp)(void*)); void taosArrayClearP(SArray* pArray, void (*fp)(void*)); -void* taosArrayDestroy(SArray* pArray); +void taosArrayDestroy(SArray* pArray); void taosArrayDestroyP(SArray* pArray, FDelete fp); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 1b7b9263a5..d0ba4c6773 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -1398,7 +1398,10 @@ void hbMgrCleanUp() { taosThreadMutexLock(&clientHbMgr.lock); appHbMgrCleanup(); - clientHbMgr.appHbMgrs = taosArrayDestroy(clientHbMgr.appHbMgrs); + + taosArrayDestroy(clientHbMgr.appHbMgrs); + clientHbMgr.appHbMgrs = NULL; + taosThreadMutexUnlock(&clientHbMgr.lock); } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 8f5d382cde..142e75dbcb 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1516,7 +1516,9 @@ void blockDataFreeRes(SSDataBlock* pBlock) { colDataDestroy(pColInfoData); } - pBlock->pDataBlock = taosArrayDestroy(pBlock->pDataBlock); + taosArrayDestroy(pBlock->pDataBlock); + pBlock->pDataBlock = NULL; + taosMemoryFreeClear(pBlock->pBlockAgg); memset(&pBlock->info, 0, sizeof(SDataBlockInfo)); } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 7e89753241..cb097a8fe7 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -9505,7 +9505,9 @@ int32_t tDecodeMqDataRsp(SDecoder *pDecoder, void *pRsp) { static void tDeleteMqDataRspCommon(void *rsp) { SMqDataRspCommon *pRsp = rsp; - pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen); + taosArrayDestroy(pRsp->blockDataLen); + pRsp->blockDataLen = NULL; + taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree); pRsp->blockData = NULL; taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSchemaWrapper); @@ -9558,7 +9560,9 @@ void tDeleteSTaosxRsp(void *rsp) { tDeleteMqDataRspCommon(rsp); STaosxRsp *pRsp = (STaosxRsp *)rsp; - pRsp->createTableLen = taosArrayDestroy(pRsp->createTableLen); + taosArrayDestroy(pRsp->createTableLen); + pRsp->createTableLen = NULL; + taosArrayDestroyP(pRsp->createTableReq, (FDelete)taosMemoryFree); pRsp->createTableReq = NULL; } diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index c4adbd0fc3..e4e30bdf10 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -978,7 +978,8 @@ void mndAddConsensusTasks(SCheckpointConsensusInfo *pInfo, const SRestoreCheckpo } void mndClearConsensusRspEntry(SCheckpointConsensusInfo* pInfo) { - pInfo->pTaskList = taosArrayDestroy(pInfo->pTaskList); + taosArrayDestroy(pInfo->pTaskList); + pInfo->pTaskList = NULL; } int64_t mndClearConsensusCheckpointId(SHashObj* pHash, int64_t streamId) { diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 84e1996f2c..241a47aca9 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -911,7 +911,7 @@ void tMergeTreeClose(SMergeTree *pMTree); SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols); void * destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); -void * destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost); +void destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost); // tsdbCache ============================================================================================== typedef enum { diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 5f3e1e3d14..d3327b5d6b 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -249,7 +249,9 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S } tTagNew(tagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag); - tagArray = taosArrayDestroy(tagArray); + taosArrayDestroy(tagArray); + tagArray = NULL; + if (pCreateTbReq->ctb.pTag == NULL) { tdDestroySVCreateTbReq(pCreateTbReq); code = TSDB_CODE_OUT_OF_MEMORY; @@ -505,7 +507,9 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat pTableData->aRowP = taosArrayInit(numOfRows, sizeof(SRow*)); if (pTableData->aRowP == NULL || pVals == NULL) { - pTableData->aRowP = taosArrayDestroy(pTableData->aRowP); + taosArrayDestroy(pTableData->aRowP); + pTableData->aRowP = NULL; + taosArrayDestroy(pVals); code = TSDB_CODE_OUT_OF_MEMORY; tqError("s-task:%s failed to prepare write stream res blocks, code:%s", id, tstrerror(code)); @@ -530,7 +534,9 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat if (ts < earlyTs) { tqError("s-task:%s ts:%" PRId64 " of generated results out of valid time range %" PRId64 " , discarded", id, ts, earlyTs); - pTableData->aRowP = taosArrayDestroy(pTableData->aRowP); + taosArrayDestroy(pTableData->aRowP); + pTableData->aRowP = NULL; + taosArrayDestroy(pVals); return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 5290c39d42..eb1da51b45 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -54,19 +54,23 @@ static int32_t tqInitTaosxRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset) { if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL || pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) { if (pRsp->blockData != NULL) { - pRsp->blockData = taosArrayDestroy(pRsp->blockData); + taosArrayDestroy(pRsp->blockData); + pRsp->blockData = NULL; } if (pRsp->blockDataLen != NULL) { - pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen); + taosArrayDestroy(pRsp->blockDataLen); + pRsp->blockDataLen = NULL; } if (pRsp->blockTbName != NULL) { - pRsp->blockTbName = taosArrayDestroy(pRsp->blockTbName); + taosArrayDestroy(pRsp->blockTbName); + pRsp->blockTbName = NULL; } if (pRsp->blockSchema != NULL) { - pRsp->blockSchema = taosArrayDestroy(pRsp->blockSchema); + taosArrayDestroy(pRsp->blockSchema); + pRsp->blockSchema = NULL; } return -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 4597a44845..38dd945b28 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -3187,7 +3187,8 @@ static void clearLastFileSet(SFSNextRowIter *state) { int32_t iter = 0; while ((pe = tSimpleHashIterate(state->pr->pTableMap, pe, &iter)) != NULL) { STableLoadInfo *pInfo = *(STableLoadInfo **)pe; - pInfo->pTombData = taosArrayDestroy(pInfo->pTombData); + taosArrayDestroy(pInfo->pTombData); + pInfo->pTombData = NULL; } } } diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 4d6ebf721e..817468c6b6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -311,7 +311,8 @@ void* tsdbCacherowsReaderClose(void* pReader) { int32_t iter = 0; while ((pe = tSimpleHashIterate(p->pTableMap, pe, &iter)) != NULL) { STableLoadInfo* pInfo = *(STableLoadInfo**)pe; - pInfo->pTombData = taosArrayDestroy(pInfo->pTombData); + taosArrayDestroy(pInfo->pTombData); + pInfo->pTombData = NULL; } tSimpleHashCleanup(p->pTableMap); diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 57b8a99fb1..fbda31f731 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -103,9 +103,9 @@ void destroyLDataIter(SLDataIter *pIter) { taosMemoryFree(pIter); } -void *destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost) { +void destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost) { if (pLDataIterArray == NULL) { - return NULL; + return; } int32_t numOfLevel = taosArrayGetSize(pLDataIterArray); @@ -132,7 +132,6 @@ void *destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoa } taosArrayDestroy(pLDataIterArray); - return NULL; } // choose the unpinned slot to load next data block diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index ec7d90184c..dc80fcabfb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -246,6 +246,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, TFileSetArray* pFileSetA SBlockLoadSuppInfo* pInfo = &pReader->suppInfo; size_t numOfFileset = TARRAY2_SIZE(pFileSetArray); bool asc = ASCENDING_TRAVERSE(pReader->info.order); + int32_t code = TSDB_CODE_SUCCESS; pIter->index = asc ? -1 : numOfFileset; pIter->order = pReader->info.order; @@ -255,7 +256,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, TFileSetArray* pFileSetA if (pIter->pSttBlockReader == NULL) { pIter->pSttBlockReader = taosMemoryCalloc(1, sizeof(struct SSttBlockReader)); if (pIter->pSttBlockReader == NULL) { - int32_t code = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; tsdbError("failed to prepare the last block iterator, since:%s %s", tstrerror(code), pReader->idStr); return code; } @@ -269,10 +270,14 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, TFileSetArray* pFileSetA pSttReader->uid = 0; tMergeTreeClose(&pSttReader->mergeTree); - initRowKey(&pSttReader->currentKey, INT64_MIN, pInfo->numOfPks, pInfo->pk.type, pInfo->pk.bytes, asc); + code = initRowKey(&pSttReader->currentKey, INT64_MIN, pInfo->numOfPks, pInfo->pk.type, pInfo->pk.bytes, asc); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("failed init row key, %s", pReader->idStr); + } else { + tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, pReader->idStr); + } - tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, pReader->idStr); - return TSDB_CODE_SUCCESS; + return code; } static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bool* hasNext) { @@ -290,7 +295,8 @@ static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bo pIter->pSttBlockReader->uid = 0; tMergeTreeClose(&pIter->pSttBlockReader->mergeTree); - pReader->status.pLDataIterArray = destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost); + destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost); + pReader->status.pLDataIterArray = NULL; pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES); // check file the time range of coverage diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index 273ec6d797..332ed4d05c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -146,17 +146,17 @@ int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, in } case TSDB_DATA_TYPE_INT:{ int32_t min = INT32_MIN; - memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); + (void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); break; } case TSDB_DATA_TYPE_SMALLINT:{ int16_t min = INT16_MIN; - memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); + (void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); break; } case TSDB_DATA_TYPE_TINYINT:{ int8_t min = INT8_MIN; - memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); + (void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); break; } case TSDB_DATA_TYPE_UTINYINT: @@ -308,7 +308,9 @@ void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) { pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter); } - pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); + taosArrayDestroy(pInfo->delSkyline); + pInfo->delSkyline = NULL; + pInfo->lastProcKey.ts = ts; // todo check the nextProcKey info pInfo->sttKeyInfo.nextProcKey.ts = ts + step; @@ -329,11 +331,16 @@ void clearBlockScanInfo(STableBlockScanInfo* p) { p->iiter.iter = tsdbTbDataIterDestroy(p->iiter.iter); } - p->delSkyline = taosArrayDestroy(p->delSkyline); - p->pBlockList = taosArrayDestroy(p->pBlockList); - p->pBlockIdxList = taosArrayDestroy(p->pBlockIdxList); - p->pMemDelData = taosArrayDestroy(p->pMemDelData); - p->pFileDelData = taosArrayDestroy(p->pFileDelData); + taosArrayDestroy(p->delSkyline); + p->delSkyline = NULL; + taosArrayDestroy(p->pBlockList); + p->pBlockList = NULL; + taosArrayDestroy(p->pBlockIdxList); + p->pBlockIdxList = NULL; + taosArrayDestroy(p->pMemDelData); + p->pMemDelData = NULL; + taosArrayDestroy(p->pFileDelData); + p->pFileDelData = NULL; clearRowKey(&p->lastProcKey); clearRowKey(&p->sttRange.skey); @@ -579,7 +586,8 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3 } taosArrayAddAll(pBlockIter->blockList, pTableScanInfo->pBlockList); - pTableScanInfo->pBlockList = taosArrayDestroy(pTableScanInfo->pBlockList); + taosArrayDestroy(pTableScanInfo->pBlockList); + pTableScanInfo->pBlockList = NULL; int64_t et = taosGetTimestampUs(); tsdbDebug("%p create blocks info struct completed for one table, %d blocks not sorted, elapsed time:%.2f ms %s", @@ -624,7 +632,8 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3 for (int32_t i = 0; i < numOfTables; ++i) { STableBlockScanInfo* pTableScanInfo = taosArrayGetP(pTableList, i); - pTableScanInfo->pBlockList = taosArrayDestroy(pTableScanInfo->pBlockList); + taosArrayDestroy(pTableScanInfo->pBlockList); + pTableScanInfo->pBlockList = NULL; } int64_t et = taosGetTimestampUs(); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 173ef0b829..c623e94a12 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -96,7 +96,8 @@ void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) { pGroupResInfo->freeItem = false; pGroupResInfo->pRows = NULL; } else { - pGroupResInfo->pRows = taosArrayDestroy(pGroupResInfo->pRows); + taosArrayDestroy(pGroupResInfo->pRows); + pGroupResInfo->pRows = NULL; } pGroupResInfo->index = 0; } @@ -2102,7 +2103,8 @@ void* tableListDestroy(STableListInfo* pTableListInfo) { return NULL; } - pTableListInfo->pTableList = taosArrayDestroy(pTableListInfo->pTableList); + taosArrayDestroy(pTableListInfo->pTableList); + pTableListInfo->pTableList = NULL; taosMemoryFreeClear(pTableListInfo->groupOffset); taosHashCleanup(pTableListInfo->map); diff --git a/source/libs/executor/src/streamfilloperator.c b/source/libs/executor/src/streamfilloperator.c index c1a38b66ba..0002c94fa7 100644 --- a/source/libs/executor/src/streamfilloperator.c +++ b/source/libs/executor/src/streamfilloperator.c @@ -129,7 +129,9 @@ static void destroyStreamFillOperatorInfo(void* param) { pInfo->pRes = blockDataDestroy(pInfo->pRes); pInfo->pSrcBlock = blockDataDestroy(pInfo->pSrcBlock); pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); - pInfo->matchInfo.pList = taosArrayDestroy(pInfo->matchInfo.pList); + + taosArrayDestroy(pInfo->matchInfo.pList); + pInfo->matchInfo.pList = NULL; taosMemoryFree(pInfo); } diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index d68108b9a5..d3f90c3134 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -406,7 +406,8 @@ void clearGroupResInfo(SGroupResInfo* pGroupResInfo) { } pGroupResInfo->freeItem = false; } - pGroupResInfo->pRows = taosArrayDestroy(pGroupResInfo->pRows); + taosArrayDestroy(pGroupResInfo->pRows); + pGroupResInfo->pRows = NULL; pGroupResInfo->index = 0; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index bad15d1834..7455296f0f 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1136,9 +1136,10 @@ void destroyIntervalOperatorInfo(void* param) { tdListFree(pInfo->binfo.resultRowInfo.openWindow); - pInfo->pInterpCols = taosArrayDestroy(pInfo->pInterpCols); - taosArrayDestroyEx(pInfo->pPrevValues, freeItem); + taosArrayDestroy(pInfo->pInterpCols); + pInfo->pInterpCols = NULL; + taosArrayDestroyEx(pInfo->pPrevValues, freeItem); pInfo->pPrevValues = NULL; cleanupGroupResInfo(&pInfo->groupResInfo); diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 226a06be7e..3ea3e814ed 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -308,7 +308,9 @@ int32_t streamTaskStopMonitorCheckRsp(STaskCheckInfo* pInfo, const char* id) { void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) { ASSERT(pInfo->inCheckProcess == 0); - pInfo->pList = taosArrayDestroy(pInfo->pList); + taosArrayDestroy(pInfo->pList); + pInfo->pList = NULL; + if (pInfo->checkRspTmr != NULL) { /*bool ret = */ taosTmrStop(pInfo->checkRspTmr); pInfo->checkRspTmr = NULL; diff --git a/source/libs/stream/src/streamMsg.c b/source/libs/stream/src/streamMsg.c index 40582b5144..1cc48f02b6 100644 --- a/source/libs/stream/src/streamMsg.c +++ b/source/libs/stream/src/streamMsg.c @@ -438,11 +438,13 @@ void tCleanupStreamHbMsg(SStreamHbMsg* pMsg) { } if (pMsg->pUpdateNodes != NULL) { - pMsg->pUpdateNodes = taosArrayDestroy(pMsg->pUpdateNodes); + taosArrayDestroy(pMsg->pUpdateNodes); + pMsg->pUpdateNodes = NULL; } if (pMsg->pTaskStatus != NULL) { - pMsg->pTaskStatus = taosArrayDestroy(pMsg->pTaskStatus); + taosArrayDestroy(pMsg->pTaskStatus); + pMsg->pTaskStatus = NULL; } pMsg->msgId = -1; diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 9bcad87264..2fb0adbdf4 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -280,10 +280,12 @@ void tFreeStreamTask(SStreamTask* pTask) { taosMemoryFree(pTask->outputInfo.pTokenBucket); taosThreadMutexDestroy(&pTask->lock); - pTask->msgInfo.pSendInfo = taosArrayDestroy(pTask->msgInfo.pSendInfo); + taosArrayDestroy(pTask->msgInfo.pSendInfo); + pTask->msgInfo.pSendInfo = NULL; taosThreadMutexDestroy(&pTask->msgInfo.lock); - pTask->outputInfo.pNodeEpsetUpdateList = taosArrayDestroy(pTask->outputInfo.pNodeEpsetUpdateList); + taosArrayDestroy(pTask->outputInfo.pNodeEpsetUpdateList); + pTask->outputInfo.pNodeEpsetUpdateList = NULL; if ((pTask->status.removeBackendFiles) && (pTask->pMeta != NULL)) { char* path = taosMemoryCalloc(1, strlen(pTask->pMeta->path) + 128); @@ -1055,9 +1057,14 @@ void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo) { } taosThreadMutexDestroy(&pInfo->lock); - pInfo->pDispatchTriggerList = taosArrayDestroy(pInfo->pDispatchTriggerList); - pInfo->pReadyMsgList = taosArrayDestroy(pInfo->pReadyMsgList); - pInfo->pCheckpointReadyRecvList = taosArrayDestroy(pInfo->pCheckpointReadyRecvList); + taosArrayDestroy(pInfo->pDispatchTriggerList); + pInfo->pDispatchTriggerList = NULL; + + taosArrayDestroy(pInfo->pReadyMsgList); + pInfo->pReadyMsgList = NULL; + + taosArrayDestroy(pInfo->pCheckpointReadyRecvList); + pInfo->pCheckpointReadyRecvList = NULL; if (pInfo->pChkptTriggerTmr != NULL) { taosTmrStop(pInfo->pChkptTriggerTmr); diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index d9686d77f8..e5f9fe43e7 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -387,13 +387,11 @@ void taosArrayClearP(SArray* pArray, void (*fp)(void*)) { taosArrayClear(pArray); } -void* taosArrayDestroy(SArray* pArray) { +void taosArrayDestroy(SArray* pArray) { if (pArray) { taosMemoryFree(pArray->pData); taosMemoryFree(pArray); } - - return NULL; } void taosArrayDestroyP(SArray* pArray, FDelete fp) { diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index adde3a3331..3ab0f83238 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -96,7 +96,8 @@ void cfgItemFreeVal(SConfigItem *pItem) { } if (pItem->array) { - pItem->array = taosArrayDestroy(pItem->array); + taosArrayDestroy(pItem->array); + pItem->array = NULL; } } From dd590f36e27f7fd1356504672cc346b17ee9e4b2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 15 Jul 2024 23:53:12 +0800 Subject: [PATCH 23/48] refactor: do some refactor. --- include/common/tdataformat.h | 2 +- source/common/src/tdataformat.c | 76 +++++---- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead2.c | 169 ++++++++++++++------ source/dnode/vnode/src/tsdb/tsdbReadUtil.h | 8 +- 6 files changed, 161 insertions(+), 98 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 7aec39817a..04e13fbdb3 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -136,7 +136,7 @@ int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag); int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag); void tRowGetPrimaryKey(SRow *pRow, SRowKey *key); int32_t tRowKeyCompare(const SRowKey *key1, const SRowKey *key2); -int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc); +void tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc); // SRowIter ================================ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 61af2f6aae..27f865353e 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -305,11 +305,11 @@ static int32_t tRowBuildTupleRow(SArray *aColVal, const SRowBuildScanInfo *sinfo *(int32_t *)(fixed + schema->columns[i].offset) = varlen - fixed - sinfo->tupleFixedSize; varlen += tPutU32v(varlen, colValArray[colValIndex].value.nData); if (colValArray[colValIndex].value.nData) { - memcpy(varlen, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData); + (void)memcpy(varlen, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData); varlen += colValArray[colValIndex].value.nData; } } else { - memcpy(fixed + schema->columns[i].offset, &colValArray[colValIndex].value.val, + (void)memcpy(fixed + schema->columns[i].offset, &colValArray[colValIndex].value.val, tDataTypes[schema->columns[i].type].bytes); } } else if (COL_VAL_IS_NULL(&colValArray[colValIndex])) { // NULL @@ -384,12 +384,12 @@ static int32_t tRowBuildKVRow(SArray *aColVal, const SRowBuildScanInfo *sinfo, c payloadSize += tPutI16v(payload + payloadSize, colValArray[colValIndex].cid); payloadSize += tPutU32v(payload + payloadSize, colValArray[colValIndex].value.nData); if (colValArray[colValIndex].value.nData > 0) { - memcpy(payload + payloadSize, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData); + (void)memcpy(payload + payloadSize, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData); } payloadSize += colValArray[colValIndex].value.nData; } else { payloadSize += tPutI16v(payload + payloadSize, colValArray[colValIndex].cid); - memcpy(payload + payloadSize, &colValArray[colValIndex].value.val, + (void)memcpy(payload + payloadSize, &colValArray[colValIndex].value.val, tDataTypes[schema->columns[i].type].bytes); payloadSize += tDataTypes[schema->columns[i].type].bytes; } @@ -475,7 +475,7 @@ int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted, value.nData = infos[iInfo].bind->length[iRow]; value.pData = (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow; } else { - memcpy(&value.val, (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow, + (void)memcpy(&value.val, (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow, infos[iInfo].bind->buffer_length); } colVal = COL_VAL_VALUE(infos[iInfo].columnId, value); @@ -509,7 +509,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) pColVal->cid = pTColumn->colId; pColVal->value.type = pTColumn->type; pColVal->flag = CV_FLAG_VALUE; - memcpy(&pColVal->value.val, &pRow->ts, sizeof(TSKEY)); + (void)memcpy(&pColVal->value.val, &pRow->ts, sizeof(TSKEY)); return 0; } @@ -573,7 +573,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) pColVal->value.pData = NULL; } } else { - memcpy(&pColVal->value.val, pData, pTColumn->bytes); + (void)memcpy(&pColVal->value.val, pData, pTColumn->bytes); } } return 0; @@ -624,7 +624,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) pColVal->value.pData = varlen + *(int32_t *)(fixed + pTColumn->offset); pColVal->value.pData += tGetU32v(pColVal->value.pData, &pColVal->value.nData); } else { - memcpy(&pColVal->value.val, fixed + pTColumn->offset, TYPE_BYTES[pTColumn->type]); + (void)memcpy(&pColVal->value.val, fixed + pTColumn->offset, TYPE_BYTES[pTColumn->type]); } } @@ -861,7 +861,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { pIter->cv.cid = pTColumn->colId; pIter->cv.value.type = pTColumn->type; pIter->cv.flag = CV_FLAG_VALUE; - memcpy(&pIter->cv.value.val, &pIter->pRow->ts, sizeof(TSKEY)); + (void)memcpy(&pIter->cv.value.val, &pIter->pRow->ts, sizeof(TSKEY)); goto _exit; } @@ -906,7 +906,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { pIter->cv.value.pData = NULL; } } else { - memcpy(&pIter->cv.value.val, pData, pTColumn->bytes); + (void)memcpy(&pIter->cv.value.val, pData, pTColumn->bytes); } } @@ -965,7 +965,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { pIter->cv.value.pData = NULL; } } else { - memcpy(&pIter->cv.value.val, pIter->pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]); + (void)memcpy(&pIter->cv.value.val, pIter->pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]); } goto _exit; } @@ -1288,7 +1288,7 @@ void tRowGetPrimaryKey(SRow *row, SRowKey *key) { key->pks[i].pData = tdata; key->pks[i].pData += tGetU32v(key->pks[i].pData, &key->pks[i].nData); } else { - memcpy(&key->pks[i].val, tdata, tDataTypes[indices[i].type].bytes); + (void)memcpy(&key->pks[i].val, tdata, tDataTypes[indices[i].type].bytes); } } } @@ -1378,7 +1378,7 @@ FORCE_INLINE int32_t tRowKeyCompare(const SRowKey *key1, const SRowKey *key2) { return 0; } -int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc) { +void tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc) { pDst->ts = pSrc->ts; pDst->numOfPKs = pSrc->numOfPKs; @@ -1392,12 +1392,10 @@ int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc) { } else { pVal->nData = pSrc->pks[i].nData; ASSERT(pSrc->pks[i].pData != NULL); - memcpy(pVal->pData, pSrc->pks[i].pData, pVal->nData); + (void)memcpy(pVal->pData, pSrc->pks[i].pData, pVal->nData); } } } - - return TSDB_CODE_SUCCESS; } // STag ======================================== @@ -1528,7 +1526,7 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { } else { p = p ? p + n : p; n += tDataTypes[pTagVal->type].bytes; - if (p) memcpy(p, &(pTagVal->i64), tDataTypes[pTagVal->type].bytes); + if (p) (void)memcpy(p, &(pTagVal->i64), tDataTypes[pTagVal->type].bytes); } return n; @@ -1550,7 +1548,7 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { if (IS_VAR_DATA_TYPE(pTagVal->type)) { n += tGetBinary(p + n, &pTagVal->pData, &pTagVal->nData); } else { - memcpy(&(pTagVal->i64), p + n, tDataTypes[pTagVal->type].bytes); + (void)memcpy(&(pTagVal->i64), p + n, tDataTypes[pTagVal->type].bytes); n += tDataTypes[pTagVal->type].bytes; } @@ -1661,7 +1659,7 @@ char *tTagValToData(const STagVal *value, bool isJson) { } varDataLen(data + typeBytes) = value->nData; - memcpy(varDataVal(data + typeBytes), value->pData, value->nData); + (void)memcpy(varDataVal(data + typeBytes), value->pData, value->nData); } else { data = ((char *)&(value->i64)) - typeBytes; // json with type } @@ -1713,7 +1711,7 @@ bool tTagGet(const STag *pTag, STagVal *pTagVal) { } else if (c > 0) { lidx = midx + 1; } else { - memcpy(pTagVal, &tv, sizeof(tv)); + (void)memcpy(pTagVal, &tv, sizeof(tv)); return true; } } @@ -1892,7 +1890,7 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, uint8_t *pData, if (nData) { code = tRealloc(&pColData->pData, pColData->nData + nData); if (code) goto _exit; - memcpy(pColData->pData + pColData->nData, pData, nData); + (void)memcpy(pColData->pData + pColData->nData, pData, nData); pColData->nData += nData; } } else { @@ -1900,7 +1898,7 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, uint8_t *pData, code = tRealloc(&pColData->pData, pColData->nData + tDataTypes[pColData->type].bytes); if (code) goto _exit; if (pData) { - memcpy(pColData->pData + pColData->nData, pData, TYPE_BYTES[pColData->type]); + (void)memcpy(pColData->pData + pColData->nData, pData, TYPE_BYTES[pColData->type]); } else { memset(pColData->pData + pColData->nData, 0, TYPE_BYTES[pColData->type]); } @@ -2594,7 +2592,7 @@ static FORCE_INLINE void tColDataGetValue4(SColData *pColData, int32_t iVal, SCo } value.pData = pColData->pData + pColData->aOffset[iVal]; } else { - memcpy(&value.val, pColData->pData + tDataTypes[pColData->type].bytes * iVal, tDataTypes[pColData->type].bytes); + (void)memcpy(&value.val, pColData->pData + tDataTypes[pColData->type].bytes * iVal, tDataTypes[pColData->type].bytes); } *pColVal = COL_VAL_VALUE(pColData->cid, value); } @@ -2692,7 +2690,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT1_SIZE(pColData->nVal)); + (void)memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT1_SIZE(pColData->nVal)); break; case (HAS_VALUE | HAS_NULL | HAS_NONE): pColData->pBitMap = xMalloc(arg, BIT2_SIZE(pColData->nVal)); @@ -2700,7 +2698,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT2_SIZE(pColData->nVal)); + (void)memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT2_SIZE(pColData->nVal)); break; default: pColData->pBitMap = NULL; @@ -2714,7 +2712,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - memcpy(pColData->aOffset, pColDataFrom->aOffset, pColData->nVal << 2); + (void)memcpy(pColData->aOffset, pColDataFrom->aOffset, pColData->nVal << 2); } else { pColData->aOffset = NULL; } @@ -2727,7 +2725,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall goto _exit; } - memcpy(pColData->pData, pColDataFrom->pData, pColData->nData); + (void)memcpy(pColData->pData, pColDataFrom->pData, pColData->nData); } else { pColData->pData = NULL; } @@ -3119,10 +3117,10 @@ static int32_t tColDataCopyRowCell(SColData *pFromColData, int32_t iFromRow, SCo pToColData->aOffset[iToRow + 1] = pToColData->aOffset[iToRow] + nData; } - memcpy(pToColData->pData + pToColData->aOffset[iToRow], pFromColData->pData + pFromColData->aOffset[iFromRow], + (void)memcpy(pToColData->pData + pToColData->aOffset[iToRow], pFromColData->pData + pFromColData->aOffset[iFromRow], nData); } else { - memcpy(&pToColData->pData[TYPE_BYTES[pToColData->type] * iToRow], + (void)memcpy(&pToColData->pData[TYPE_BYTES[pToColData->type] * iToRow], &pFromColData->pData[TYPE_BYTES[pToColData->type] * iFromRow], TYPE_BYTES[pToColData->type]); } return code; @@ -3361,7 +3359,7 @@ static void tColDataMergeImpl(SColData *pColData, int32_t iStart, int32_t iEnd / ASSERT(0); } else { if (iv != iStart) { - memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * iStart], + (void)memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * iStart], &pColData->pData[TYPE_BYTES[pColData->type] * iv], TYPE_BYTES[pColData->type]); } memmove(&pColData->pData[TYPE_BYTES[pColData->type] * (iStart + 1)], @@ -3579,11 +3577,11 @@ static int32_t tPutColDataVersion0(uint8_t *pBuf, SColData *pColData) { case (HAS_NULL | HAS_NONE): case (HAS_VALUE | HAS_NONE): case (HAS_VALUE | HAS_NULL): - if (pBuf) memcpy(pBuf + n, pColData->pBitMap, BIT1_SIZE(pColData->nVal)); + if (pBuf) (void)memcpy(pBuf + n, pColData->pBitMap, BIT1_SIZE(pColData->nVal)); n += BIT1_SIZE(pColData->nVal); break; case (HAS_VALUE | HAS_NULL | HAS_NONE): - if (pBuf) memcpy(pBuf + n, pColData->pBitMap, BIT2_SIZE(pColData->nVal)); + if (pBuf) (void)memcpy(pBuf + n, pColData->pBitMap, BIT2_SIZE(pColData->nVal)); n += BIT2_SIZE(pColData->nVal); break; default: @@ -3593,14 +3591,14 @@ static int32_t tPutColDataVersion0(uint8_t *pBuf, SColData *pColData) { // value if (pColData->flag & HAS_VALUE) { if (IS_VAR_DATA_TYPE(pColData->type)) { - if (pBuf) memcpy(pBuf + n, pColData->aOffset, pColData->nVal << 2); + if (pBuf) (void)memcpy(pBuf + n, pColData->aOffset, pColData->nVal << 2); n += (pColData->nVal << 2); n += tPutI32v(pBuf ? pBuf + n : NULL, pColData->nData); - if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData); + if (pBuf) (void)memcpy(pBuf + n, pColData->pData, pColData->nData); n += pColData->nData; } else { - if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData); + if (pBuf) (void)memcpy(pBuf + n, pColData->pData, pColData->nData); n += pColData->nData; } } @@ -4348,7 +4346,7 @@ int32_t tCompressData(void *input, // input ASSERT(outputSize >= extraSizeNeeded); if (info->cmprAlg == NO_COMPRESSION) { - memcpy(output, input, info->originalSize); + (void)memcpy(output, input, info->originalSize); info->compressedSize = info->originalSize; } else if (info->cmprAlg == ONE_STAGE_COMP || info->cmprAlg == TWO_STAGE_COMP) { SBuffer local; @@ -4385,7 +4383,7 @@ int32_t tCompressData(void *input, // input } else { DEFINE_VAR(info->cmprAlg) if ((l1 == L1_UNKNOWN && l2 == L2_UNKNOWN) || (l1 == L1_DISABLED && l2 == L2_DISABLED)) { - memcpy(output, input, info->originalSize); + (void)memcpy(output, input, info->originalSize); info->compressedSize = info->originalSize; return 0; } @@ -4431,7 +4429,7 @@ int32_t tDecompressData(void *input, // input if (info->cmprAlg == NO_COMPRESSION) { ASSERT(info->compressedSize == info->originalSize); - memcpy(output, input, info->compressedSize); + (void)memcpy(output, input, info->compressedSize); } else if (info->cmprAlg == ONE_STAGE_COMP || info->cmprAlg == TWO_STAGE_COMP) { SBuffer local; @@ -4468,7 +4466,7 @@ int32_t tDecompressData(void *input, // input } else { DEFINE_VAR(info->cmprAlg); if (l1 == L1_DISABLED && l2 == L2_DISABLED) { - memcpy(output, input, info->compressedSize); + (void)memcpy(output, input, info->compressedSize); return 0; } SBuffer local; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 38dd945b28..ccdf4a14d6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -2579,7 +2579,7 @@ typedef struct { static int32_t lastIterOpen(SFSLastIter *iter, STFileSet *pFileSet, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, SCacheRowsReader *pr, int64_t lastTs, int16_t *aCols, int nCols) { int32_t code = 0; - pr->pLDataIterArray = destroySttBlockReader(pr->pLDataIterArray, NULL); + destroySttBlockReader(pr->pLDataIterArray, NULL); pr->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); SMergeTreeConf conf = { diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 817468c6b6..46c3ba4785 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -203,7 +203,7 @@ int32_t tsdbReuseCacherowsReader(void* reader, void* pTableIdList, int32_t numOf pReader->pTableList = pTableIdList; pReader->numOfTables = numOfTables; pReader->lastTs = INT64_MIN; - pReader->pLDataIterArray = destroySttBlockReader(pReader->pLDataIterArray, NULL); + destroySttBlockReader(pReader->pLDataIterArray, NULL); pReader->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); return TSDB_CODE_SUCCESS; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index dc80fcabfb..048e407c76 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -45,7 +45,7 @@ typedef struct { bool moreThanCapcity; } SDataBlockToLoadInfo; -static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter); +static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInfo** pInfo); static int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity, STsdbReader* pReader); static TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader); @@ -123,7 +123,7 @@ static void tColRowGetPriamyKeyDeepCopy(SBlockData* pBlock, int32_t irow, int32_ pKey->pks[0].val = cv.value.val; } else { pKey->pks[0].nData = cv.value.nData; - /*void* p = */memcpy(pKey->pks[0].pData, cv.value.pData, cv.value.nData); + (void)(void)memcpy(pKey->pks[0].pData, cv.value.pData, cv.value.nData); } } @@ -155,9 +155,9 @@ static void tRowGetPrimaryKeyDeepCopy(SRow* pRow, SRowKey* pKey) { if (IS_VAR_DATA_TYPE(indices[i].type)) { tdata += tGetU32v(tdata, &pKey->pks[i].nData); - memcpy(pKey->pks[i].pData, tdata, pKey->pks[i].nData); + (void)memcpy(pKey->pks[i].pData, tdata, pKey->pks[i].nData); } else { - memcpy(&pKey->pks[i].val, data + indices[i].offset, tDataTypes[pKey->pks[i].type].bytes); + (void)memcpy(&pKey->pks[i].val, data + indices[i].offset, tDataTypes[pKey->pks[i].type].bytes); } } } @@ -304,7 +304,7 @@ static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bo while (1) { if (pReader->pFileReader != NULL) { - tsdbDataFileReaderClose(&pReader->pFileReader); + (void) tsdbDataFileReaderClose(&pReader->pFileReader); } pReader->status.pCurrentFileset = pIter->pFilesetList->data[pIter->index]; @@ -377,15 +377,20 @@ _err: bool shouldFreePkBuf(SBlockLoadSuppInfo* pSupp) { return (pSupp->numOfPks > 0) && IS_VAR_DATA_TYPE(pSupp->pk.type); } -void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool needFree) { +int32_t resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool needFree) { pIter->order = order; pIter->index = -1; pIter->numOfBlocks = 0; if (pIter->blockList == NULL) { pIter->blockList = taosArrayInit(4, sizeof(SFileDataBlockInfo)); + if (pIter->blockList == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } else { clearDataBlockIterator(pIter, needFree); } + + return TSDB_CODE_SUCCESS; } static void initReaderStatus(SReaderStatus* pStatus) { @@ -818,7 +823,7 @@ static void updateLastKeyInfo(SRowKey* pKey, SFileDataBlockInfo* pBlockInfo, SDa } else { uint8_t* p = asc ? pBlockInfo->lastPk.pData : pBlockInfo->firstPk.pData; pKey->pks[0].nData = asc ? varDataLen(pBlockInfo->lastPk.pData) : varDataLen(pBlockInfo->firstPk.pData); - memcpy(pKey->pks[0].pData, p, pKey->pks[0].nData); + (void)memcpy(pKey->pks[0].pData, p, pKey->pks[0].nData); } } @@ -836,7 +841,7 @@ static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int } if (pColVal->value.nData > 0) { // pData may be null, if nData is 0 - memcpy(varDataVal(pSup->buildBuf[colIndex]), pColVal->value.pData, pColVal->value.nData); + (void)memcpy(varDataVal(pSup->buildBuf[colIndex]), pColVal->value.pData, pColVal->value.nData); } colDataSetVal(pColInfoData, rowIndex, pSup->buildBuf[colIndex], false); @@ -848,19 +853,21 @@ static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int return TSDB_CODE_SUCCESS; } -static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) { +static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInfo** pInfo) { + *pInfo = NULL; + if (pBlockIter->blockList == NULL) { - return NULL; + return TSDB_CODE_SUCCESS; } size_t num = TARRAY_SIZE(pBlockIter->blockList); if (num == 0) { ASSERT(pBlockIter->numOfBlocks == num); - return NULL; + return TSDB_CODE_SUCCESS; } - SFileDataBlockInfo* pBlockInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index); - return pBlockInfo; + *pInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index); + return (*pInfo) != NULL? TSDB_CODE_SUCCESS:TSDB_CODE_FAILED; } static int doBinarySearchKey(const TSKEY* keyList, int num, int pos, TSKEY key, int order) { @@ -974,10 +981,10 @@ static int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData static void copyPrimaryTsCol(const SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData, int32_t dumpedRows, bool asc) { if (asc) { - memcpy(pColData->pData, &pBlockData->aTSKEY[pDumpInfo->rowIndex], dumpedRows * sizeof(int64_t)); + (void)memcpy(pColData->pData, &pBlockData->aTSKEY[pDumpInfo->rowIndex], dumpedRows * sizeof(int64_t)); } else { int32_t startIndex = pDumpInfo->rowIndex - dumpedRows + 1; - memcpy(pColData->pData, &pBlockData->aTSKEY[startIndex], dumpedRows * sizeof(int64_t)); + (void)memcpy(pColData->pData, &pBlockData->aTSKEY[startIndex], dumpedRows * sizeof(int64_t)); // todo: opt perf by extract the loop // reverse the array list @@ -1008,7 +1015,7 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo // ASSERT((((uint64_t)pColData->pData) & (0x8 - 1)) == 0); // 1. copy data in a batch model - memcpy(pColData->pData, p, dumpedRows * tDataTypes[pData->type].bytes); + (void)memcpy(pColData->pData, p, dumpedRows * tDataTypes[pData->type].bytes); // 2. reverse the array list in case of descending order scan data block if (!asc) { @@ -1123,7 +1130,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBlockData* pBlockData = &pStatus->fileBlockData; - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + SFileDataBlockInfo* pBlockInfo = NULL; SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; int32_t numOfOutputCols = pSupInfo->numOfCols; int32_t code = TSDB_CODE_SUCCESS; @@ -1131,6 +1138,11 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro bool asc = ASCENDING_TRAVERSE(pReader->info.order); int32_t step = asc ? 1 : -1; + code = getCurrentBlockInfo(pBlockIter, &pBlockInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + SColVal cv = {0}; SBrinRecord tmp; blockInfoToRecord(&tmp, pBlockInfo, pSupInfo); @@ -1314,9 +1326,11 @@ static FORCE_INLINE STSchema* getTableSchemaImpl(STsdbReader* pReader, uint64_t static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockIter, SBlockData* pBlockData, uint64_t uid) { - int32_t code = 0; - STSchema* pSchema = pReader->info.pSchema; - int64_t st = taosGetTimestampUs(); + int32_t code = 0; + STSchema* pSchema = pReader->info.pSchema; + int64_t st = taosGetTimestampUs(); + SFileDataBlockInfo* pBlockInfo = NULL; + SBlockLoadSuppInfo* pSup = &pReader->suppInfo; tBlockDataReset(pBlockData); @@ -1328,8 +1342,11 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI } } - SBlockLoadSuppInfo* pSup = &pReader->suppInfo; - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + code = getCurrentBlockInfo(pBlockIter, &pBlockInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBrinRecord tmp; @@ -2524,11 +2541,12 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { int32_t step = asc ? 1 : -1; double el = 0; SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); + SFileDataBlockInfo* pBlockInfo = NULL; SSttBlockReader* pSttBlockReader = pReader->status.fileIter.pSttBlockReader; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - if (pBlockInfo == NULL) { + code = getCurrentBlockInfo(&pReader->status.blockIter, &pBlockInfo); + if (code != TSDB_CODE_SUCCESS) { return 0; } @@ -2572,7 +2590,10 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { pDumpInfo->rowIndex += step; if (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0) { - pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); // NOTE: get the new block info + code = getCurrentBlockInfo(&pReader->status.blockIter, &pBlockInfo); // NOTE: get the new block info + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } // continue check for the next file block if the last ts in the current block // is overlapped with the next neighbor block @@ -2797,6 +2818,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr return loadDataFileTombDataForAll(pReader); } +// pTableIter can be NULL, no need to handle the return value static void resetTableListIndex(SReaderStatus* pStatus) { STableUidList* pList = &pStatus->uidList; @@ -2838,7 +2860,7 @@ static bool moveToNextTableForPreFileSetMem(SReaderStatus* pStatus) { return (pStatus->pProcMemTableIter != NULL); } -static void buildCleanBlockFromSttFiles(STsdbReader* pReader, STableBlockScanInfo* pScanInfo) { +static int32_t buildCleanBlockFromSttFiles(STsdbReader* pReader, STableBlockScanInfo* pScanInfo) { SReaderStatus* pStatus = &pReader->status; SSttBlockReader* pSttBlockReader = pStatus->fileIter.pSttBlockReader; SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; @@ -2846,7 +2868,10 @@ static void buildCleanBlockFromSttFiles(STsdbReader* pReader, STableBlockScanInf bool asc = ASCENDING_TRAVERSE(pReader->info.order); SDataBlockInfo* pInfo = &pResBlock->info; - blockDataEnsureCapacity(pResBlock, pScanInfo->numOfRowsInStt); + int32_t code = blockDataEnsureCapacity(pResBlock, pScanInfo->numOfRowsInStt); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pInfo->rows = pScanInfo->numOfRowsInStt; pInfo->id.uid = pScanInfo->uid; @@ -2891,8 +2916,8 @@ static void buildCleanBlockFromDataFiles(STsdbReader* pReader, STableBlockScanIn pInfo->pks[0].val = pBlockInfo->firstPk.val; pInfo->pks[1].val = pBlockInfo->lastPk.val; } else { - memcpy(pInfo->pks[0].pData, varDataVal(pBlockInfo->firstPk.pData), varDataLen(pBlockInfo->firstPk.pData)); - memcpy(pInfo->pks[1].pData, varDataVal(pBlockInfo->lastPk.pData), varDataLen(pBlockInfo->lastPk.pData)); + (void)memcpy(pInfo->pks[0].pData, varDataVal(pBlockInfo->firstPk.pData), varDataLen(pBlockInfo->firstPk.pData)); + (void)memcpy(pInfo->pks[1].pData, varDataVal(pBlockInfo->lastPk.pData), varDataLen(pBlockInfo->lastPk.pData)); pInfo->pks[0].nData = varDataLen(pBlockInfo->firstPk.pData); pInfo->pks[1].nData = varDataLen(pBlockInfo->lastPk.pData); @@ -2963,8 +2988,8 @@ static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) { // if only require the total rows, no need to load data from stt file if it is clean stt blocks if (pReader->info.execMode == READER_EXEC_ROWS && pScanInfo->cleanSttBlocks) { - buildCleanBlockFromSttFiles(pReader, pScanInfo); - return TSDB_CODE_SUCCESS; + code = buildCleanBlockFromSttFiles(pReader, pScanInfo); + return code; } int64_t st = taosGetTimestampUs(); @@ -3019,11 +3044,16 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; STableBlockScanInfo* pScanInfo = NULL; - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + SFileDataBlockInfo* pBlockInfo = NULL; SSttBlockReader* pSttBlockReader = pReader->status.fileIter.pSttBlockReader; bool asc = ASCENDING_TRAVERSE(pReader->info.order); int32_t code = TSDB_CODE_SUCCESS; + code = getCurrentBlockInfo(pBlockIter, &pBlockInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) { setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->lastKey, pReader->info.order); return code; @@ -3080,8 +3110,8 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // return the stt file block ASSERT(pReader->info.execMode == READER_EXEC_ROWS && pSttBlockReader->mergeTree.pIter == NULL); - buildCleanBlockFromSttFiles(pReader, pScanInfo); - return TSDB_CODE_SUCCESS; + code = buildCleanBlockFromSttFiles(pReader, pScanInfo); + return code; } } else { SBlockData* pBData = &pReader->status.fileBlockData; @@ -3218,11 +3248,12 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader, int64_t en // set the correct start position in case of the first/last file block, according to the query time window static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) { - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + SFileDataBlockInfo* pBlockInfo = NULL; SReaderStatus* pStatus = &pReader->status; SFileBlockDumpInfo* pDumpInfo = &pStatus->fBlockDumpInfo; - if (pBlockInfo) { + int32_t code = getCurrentBlockInfo(pBlockIter, &pBlockInfo); + if (code != TSDB_CODE_SUCCESS) { pDumpInfo->totalRows = pBlockInfo->numRow; pDumpInfo->rowIndex = ASCENDING_TRAVERSE(pReader->info.order) ? 0 : pBlockInfo->numRow - 1; } else { @@ -3255,7 +3286,7 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl code = initBlockIterator(pReader, pBlockIter, num.numOfBlocks, pTableList); } else { // no block data, only last block exists tBlockDataReset(&pReader->status.fileBlockData); - resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); + code = resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); resetTableListIndex(&pReader->status); } @@ -3365,7 +3396,10 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { } tBlockDataReset(pBlockData); - resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); + code = resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); + if (code != TSDB_CODE_SUCCESS) { + return code; + } resetTableListIndex(&pReader->status); ERetrieveType type = doReadDataFromSttFiles(pReader); @@ -3742,7 +3776,12 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc while (1) { CHECK_FILEBLOCK_STATE st; - SFileDataBlockInfo* pFileBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); + SFileDataBlockInfo* pFileBlockInfo = NULL; + int32_t code = getCurrentBlockInfo(&pReader->status.blockIter, &pFileBlockInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (pFileBlockInfo == NULL) { st = CHECK_FILEBLOCK_QUIT; break; @@ -4224,6 +4263,7 @@ uint64_t tsdbGetReaderMaxVersion2(STsdbReader* pReader) { return pReader->info.v static int32_t doOpenReaderImpl(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; + int32_t code = TSDB_CODE_SUCCESS; if (pReader->bFilesetDelimited) { getMemTableTimeRange(pReader, &pReader->status.memTableMaxKey, &pReader->status.memTableMinKey); @@ -4231,9 +4271,11 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { } initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); - resetDataBlockIterator(&pStatus->blockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); + code = resetDataBlockIterator(&pStatus->blockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - int32_t code = TSDB_CODE_SUCCESS; if (pStatus->fileIter.numOfFiles == 0) { pStatus->loadFromFile = false; } else { @@ -4278,13 +4320,17 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi SSDataBlock* pResBlock, void** ppReader, const char* idstr, SHashObj** pIgnoreTables) { STimeWindow window = pCond->twindows; SVnodeCfg* pConf = &(((SVnode*)pVnode)->config); + int32_t code = 0; int32_t capacity = pConf->tsdbCfg.maxRows; if (pResBlock != NULL) { - blockDataEnsureCapacity(pResBlock, capacity); + code = blockDataEnsureCapacity(pResBlock, capacity); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } } - int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, capacity, pResBlock, idstr); + code = tsdbReaderCreate(pVnode, pCond, ppReader, capacity, pResBlock, idstr); if (code != TSDB_CODE_SUCCESS) { goto _err; } @@ -4498,7 +4544,7 @@ static int32_t doSuspendCurrentReader(STsdbReader* pCurrentReader) { tsdbDataFileReaderClose(&pCurrentReader->pFileReader); SReadCostSummary* pCost = &pCurrentReader->cost; - pStatus->pLDataIterArray = destroySttBlockReader(pStatus->pLDataIterArray, &pCost->sttCost); + destroySttBlockReader(pStatus->pLDataIterArray, &pCost->sttCost); pStatus->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); } @@ -4922,9 +4968,9 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ } int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave, bool* hasNullSMA) { - SColumnDataAgg** pBlockSMA = &pDataBlock->pBlockAgg; - - int32_t code = 0; + SColumnDataAgg** pBlockSMA = &pDataBlock->pBlockAgg; + SFileDataBlockInfo* pBlockInfo = NULL; + int32_t code = 0; *allHave = false; *pBlockSMA = NULL; @@ -4937,7 +4983,11 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, return TSDB_CODE_SUCCESS; } - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); + code = getCurrentBlockInfo(&pReader->status.blockIter, &pBlockInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + SBlockLoadSuppInfo* pSup = &pReader->suppInfo; SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; @@ -5020,7 +5070,12 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; int32_t code = TSDB_CODE_SUCCESS; - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter); + SFileDataBlockInfo* pBlockInfo = NULL; + + code = getCurrentBlockInfo(&pStatus->blockIter, &pBlockInfo); + if (code != TSDB_CODE_SUCCESS) { + return NULL; + } if (pReader->code != TSDB_CODE_SUCCESS) { return NULL; @@ -5115,7 +5170,12 @@ int32_t tsdbReaderReset2(STsdbReader* pReader, SQueryTableDataCond* pCond) { int32_t numOfTables = tSimpleHashGetSize(pStatus->pTableMap); initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); - resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); + code = resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); + if (code != TSDB_CODE_SUCCESS) { + tsdbReleaseReader(pReader); + return code; + } + resetTableListIndex(&pReader->status); bool asc = ASCENDING_TRAVERSE(pReader->info.order); @@ -5218,8 +5278,13 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT while (true) { if (hasNext) { - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); - int32_t numOfRows = pBlockInfo->numRow; + SFileDataBlockInfo* pBlockInfo = NULL; + code = getCurrentBlockInfo(pBlockIter, &pBlockInfo); + if (code != TSDB_CODE_SUCCESS) { + break; + } + + int32_t numOfRows = pBlockInfo->numRow; pTableBlockInfo->totalRows += numOfRows; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index 6c76f8c372..48ef00baae 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -358,10 +358,10 @@ int32_t pkCompEx(SRowKey* p1, SRowKey* p2); int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc); void clearRowKey(SRowKey* pKey); -bool shouldFreePkBuf(SBlockLoadSuppInfo *pSupp); -void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool hasPk); -void clearDataBlockIterator(SDataBlockIter* pIter, bool needFree); -void cleanupDataBlockIterator(SDataBlockIter* pIter, bool hasPk); +bool shouldFreePkBuf(SBlockLoadSuppInfo* pSupp); +int32_t resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool hasPk); +void clearDataBlockIterator(SDataBlockIter* pIter, bool needFree); +void cleanupDataBlockIterator(SDataBlockIter* pIter, bool hasPk); typedef struct { SArray* pTombData; From bdd6ba6b90f53035ec27149d9c4ea2ccf494eea3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 16 Jul 2024 09:18:54 +0800 Subject: [PATCH 24/48] adjust format --- source/common/src/tmisce.c | 52 +++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/source/common/src/tmisce.c b/source/common/src/tmisce.c index 7009cf26d0..7b349e91b0 100644 --- a/source/common/src/tmisce.c +++ b/source/common/src/tmisce.c @@ -15,9 +15,9 @@ #define _DEFAULT_SOURCE #include "tmisce.h" +#include "tdatablock.h" #include "tglobal.h" #include "tjson.h" -#include "tdatablock.h" int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) { pEp->port = 0; @@ -178,57 +178,57 @@ int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t cap) { int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t startTime) { int32_t code = 0; - SJson* pJson = tjsonCreateObject(); + SJson* pJson = tjsonCreateObject(); if (pJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; char tmp[4096] = {0}; - TAOS_CHECK_GOTO(code = tjsonAddDoubleToObject(pJson, "reportVersion", 1), NULL, _exit); + TAOS_CHECK_GOTO(tjsonAddDoubleToObject(pJson, "reportVersion", 1), NULL, _exit); - TAOS_CHECK_GOTO(code = tjsonAddIntegerToObject(pJson, "clusterId", clusterId), NULL, _exit); - TAOS_CHECK_GOTO(code = tjsonAddIntegerToObject(pJson, "startTime", startTime), NULL, _exit); + TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "clusterId", clusterId), NULL, _exit); + TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "startTime", startTime), NULL, _exit); // Do NOT invoke the taosGetFqdn here. // this function may be invoked when memory exception occurs,so we should assume that it is running in a memory locked // environment. The lock operation by taosGetFqdn may cause this program deadlock. - TAOS_CHECK_GOTO(code = tjsonAddStringToObject(pJson, "fqdn", tsLocalFqdn), NULL, _exit); + TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "fqdn", tsLocalFqdn), NULL, _exit); - TAOS_CHECK_GOTO(code = tjsonAddIntegerToObject(pJson, "pid", taosGetPId()), NULL, _exit); + TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "pid", taosGetPId()), NULL, _exit); code = taosGetAppName(tmp, NULL); if (code != 0) { code = TAOS_SYSTEM_ERROR(errno); TAOS_CHECK_GOTO(code, NULL, _exit); } - TAOS_CHECK_GOTO(code = tjsonAddStringToObject(pJson, "appName", tmp), NULL, _exit); + TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "appName", tmp), NULL, _exit); if (taosGetOsReleaseName(tmp, NULL, NULL, sizeof(tmp)) == 0) { - TAOS_CHECK_GOTO(code = tjsonAddStringToObject(pJson, "os", tmp), NULL, _exit); + TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "os", tmp), NULL, _exit); } else { // do nothing } float numOfCores = 0; if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores) == 0) { - TAOS_CHECK_GOTO(code = tjsonAddStringToObject(pJson, "cpuModel", tmp), NULL, _exit); - TAOS_CHECK_GOTO(code = tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores), NULL, _exit); + TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "cpuModel", tmp), NULL, _exit); + TAOS_CHECK_GOTO(tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores), NULL, _exit); } else { - TAOS_CHECK_GOTO(code = tjsonAddDoubleToObject(pJson, "numOfCpu", tsNumOfCores), NULL, _exit); + TAOS_CHECK_GOTO(tjsonAddDoubleToObject(pJson, "numOfCpu", tsNumOfCores), NULL, _exit); } int32_t nBytes = snprintf(tmp, sizeof(tmp), "%" PRId64 " kB", tsTotalMemoryKB); if (nBytes <= 9 || nBytes >= sizeof(tmp)) { - TAOS_CHECK_GOTO(code = TSDB_CODE_OUT_OF_RANGE, NULL, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_RANGE, NULL, _exit); } - TAOS_CHECK_GOTO(code = tjsonAddStringToObject(pJson, "memory", tmp), NULL, _exit); + TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "memory", tmp), NULL, _exit); - TAOS_CHECK_GOTO(code = tjsonAddStringToObject(pJson, "version", version), NULL, _exit); - TAOS_CHECK_GOTO(code = tjsonAddStringToObject(pJson, "buildInfo", buildinfo), NULL, _exit); + TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "version", version), NULL, _exit); + TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "buildInfo", buildinfo), NULL, _exit); - TAOS_CHECK_GOTO(code = tjsonAddStringToObject(pJson, "gitInfo", gitinfo), NULL, _exit); + TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "gitInfo", gitinfo), NULL, _exit); - TAOS_CHECK_GOTO(code = tjsonAddIntegerToObject(pJson, "crashSig", signum), NULL, _exit); - TAOS_CHECK_GOTO(code = tjsonAddIntegerToObject(pJson, "crashTs", taosGetTimestampUs()), NULL, _exit); + TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "crashSig", signum), NULL, _exit); + TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "crashTs", taosGetTimestampUs()), NULL, _exit); #ifdef _TD_DARWIN_64 taosLogTraceToBuf(tmp, sizeof(tmp), 4); @@ -238,7 +238,7 @@ int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t taosLogTraceToBuf(tmp, sizeof(tmp), 8); #endif - TAOS_CHECK_GOTO(code = tjsonAddStringToObject(pJson, "stackInfo", tmp), NULL, _exit); + TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "stackInfo", tmp), NULL, _exit); char* pCont = tjsonToString(pJson); if (pCont == NULL) { @@ -256,8 +256,8 @@ _exit: } int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) { - int32_t code = 0; - SConfig* pConf = taosGetCfg(); + int32_t code = 0; + SConfig* pConf = taosGetCfg(); if (pConf == NULL) { return TSDB_CODE_INVALID_CFG; } @@ -269,7 +269,7 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) { int8_t locked = 0; - TAOS_CHECK_GOTO(code = blockDataEnsureCapacity(pBlock, cfgGetSize(pConf)), NULL, _exit); + TAOS_CHECK_GOTO(blockDataEnsureCapacity(pBlock, cfgGetSize(pConf)), NULL, _exit); pIter = cfgCreateIter(pConf); if (pIter == NULL) { @@ -293,7 +293,7 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) { TAOS_CHECK_GOTO(code, NULL, _exit); } - TAOS_CHECK_GOTO(code = colDataSetVal(pColInfo, numOfRows, name, false), NULL, _exit); + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, name, false), NULL, _exit); char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; int32_t valueLen = 0; @@ -306,7 +306,7 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) { TAOS_CHECK_GOTO(code, NULL, _exit); } - TAOS_CHECK_GOTO(code = colDataSetVal(pColInfo, numOfRows, value, false), NULL, _exit); + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, value, false), NULL, _exit); char scope[TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE] = {0}; cfgDumpItemScope(pItem, &scope[VARSTR_HEADER_SIZE], TSDB_CONFIG_SCOPE_LEN, &valueLen); @@ -317,7 +317,7 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) { code = TSDB_CODE_OUT_OF_RANGE; TAOS_CHECK_GOTO(code, NULL, _exit); } - TAOS_CHECK_GOTO(code = colDataSetVal(pColInfo, numOfRows, scope, false), NULL, _exit); + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, scope, false), NULL, _exit); numOfRows++; } From 8dde576c6f725e8ca0130f1e11c4da3de5830bcb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 15 Jul 2024 18:51:58 +0800 Subject: [PATCH 25/48] fix(stream): add explicit create table into sink cache. --- source/dnode/vnode/src/tq/tqSink.c | 41 ++++++++++++++++++------------ 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 1c8f102725..9b1e8075da 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -46,6 +46,7 @@ static int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, static SArray* createDefaultTagColName(); static void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName, int64_t gid, bool newSubTableRule); +static int32_t doCreateSinkInfo(const char* pDstTableName, STableSinkInfo** pInfo); int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq, const char* pIdStr, bool newSubTableRule) { @@ -270,6 +271,14 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1); taosArrayPush(reqs.pArray, pCreateTbReq); + + STableSinkInfo* pInfo = NULL; + bool alreadyCached = tqGetTableInfo(pTask->outputInfo.tbSink.pTblInfo, gid, &pInfo); + if (!alreadyCached) { + code = doCreateSinkInfo(pCreateTbReq->name, &pInfo); + doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pInfo, gid, pTask->id.idStr); + } + tqDebug("s-task:%s build create table:%s msg complete", pTask->id.idStr, pCreateTbReq->name); } @@ -634,6 +643,18 @@ int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkI return TSDB_CODE_SUCCESS; } +int32_t doCreateSinkInfo(const char* pDstTableName, STableSinkInfo** pInfo) { + int32_t nameLen = strlen(pDstTableName); + (*pInfo) = taosMemoryCalloc(1, sizeof(STableSinkInfo) + nameLen + 1); + if (*pInfo == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + (*pInfo)->name.len = nameLen; + memcpy((*pInfo)->name.data, pDstTableName, nameLen); + return TSDB_CODE_SUCCESS; +} + int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDataBlock, char* stbFullName, SSubmitTbData* pTableData) { uint64_t groupId = pDataBlock->info.id.groupId; @@ -670,22 +691,15 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat if (pTask->subtableWithoutMd5 != 1 && !isAutoTableName(dstTableName) && !alreadyAddGroupId(dstTableName, groupId) && groupId != 0) { tqDebug("s-task:%s append groupId:%" PRId64 " for generated dstTable:%s", id, groupId, dstTableName); - if(pTask->ver == SSTREAM_TASK_SUBTABLE_CHANGED_VER){ + if (pTask->ver == SSTREAM_TASK_SUBTABLE_CHANGED_VER) { buildCtbNameAddGroupId(NULL, dstTableName, groupId); - }else if(pTask->ver > SSTREAM_TASK_SUBTABLE_CHANGED_VER && stbFullName) { + } else if (pTask->ver > SSTREAM_TASK_SUBTABLE_CHANGED_VER && stbFullName) { buildCtbNameAddGroupId(stbFullName, dstTableName, groupId); } } } - int32_t nameLen = strlen(dstTableName); - pTableSinkInfo = taosMemoryCalloc(1, sizeof(STableSinkInfo) + nameLen + 1); - if (pTableSinkInfo == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pTableSinkInfo->name.len = nameLen; - memcpy(pTableSinkInfo->name.data, dstTableName, nameLen); + int32_t code = doCreateSinkInfo(dstTableName, &pTableSinkInfo); tqDebug("s-task:%s build new sinkTableInfo to add cache, dstTable:%s", id, dstTableName); } @@ -693,7 +707,7 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat pTableData->uid = pTableSinkInfo->uid; if (pTableData->uid == 0) { - tqTrace("s-task:%s cached tableInfo uid is invalid, acquire it from meta", id); + tqTrace("s-task:%s cached tableInfo:%s uid is invalid, acquire it from meta", id, pTableSinkInfo->name.data); return doWaitForDstTableCreated(pVnode, pTask, pTableSinkInfo, dstTableName, &pTableData->uid); } else { tqTrace("s-task:%s set the dstTable uid from cache:%" PRId64, id, pTableData->uid); @@ -929,11 +943,6 @@ bool hasOnlySubmitData(const SArray* pBlocks, int32_t numOfBlocks) { } int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSinkInfo, uint64_t groupId, const char* id) { - if (tSimpleHashGetSize(pSinkTableMap) > MAX_CACHE_TABLE_INFO_NUM) { - taosMemoryFreeClear(pTableSinkInfo); // too many items, failed to cache it - return TSDB_CODE_FAILED; - } - int32_t code = tSimpleHashPut(pSinkTableMap, &groupId, sizeof(uint64_t), &pTableSinkInfo, POINTER_BYTES); if (code != TSDB_CODE_SUCCESS) { taosMemoryFreeClear(pTableSinkInfo); From 8d337e94ac304efd196a0d8771d1b41d38a9577c Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 16 Jul 2024 01:34:38 +0000 Subject: [PATCH 26/48] fix/TD-30989 --- include/libs/audit/audit.h | 2 +- include/util/taoserror.h | 5 +++++ source/util/src/terror.c | 4 ++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/include/libs/audit/audit.h b/include/libs/audit/audit.h index dd3df27866..4fa69f1b4f 100644 --- a/include/libs/audit/audit.h +++ b/include/libs/audit/audit.h @@ -50,7 +50,7 @@ typedef struct { int32_t auditInit(const SAuditCfg *pCfg); void auditCleanup(); -void auditSend(SJson *pJson); +int32_t auditSend(SJson *pJson); void auditRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2, char *detail, int32_t len); void auditAddRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2, diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 900d3eedad..053bb20e2a 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -942,6 +942,11 @@ int32_t taosGetErrSize(); // UTIL #define TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x6000) +// AUDIT +#define TSDB_CODE_AUDIT_NOT_FORMAT_TO_JSON TAOS_DEF_ERROR_CODE(0, 0x6100) +#define TSDB_CODE_AUDIT_FAIL_SEND_AUDIT_RECORD TAOS_DEF_ERROR_CODE(0, 0x6101) +#define TSDB_CODE_AUDIT_FAIL_GENERATE_JSON TAOS_DEF_ERROR_CODE(0, 0x6102) + #ifdef __cplusplus } #endif diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 347f0be4ff..ab50b83937 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -789,6 +789,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDLITE_IVLD_OPEN_DIR, "Invalid TDLite open TAOS_DEFINE_ERROR(TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY, "Queue out of memory") +//AUDIT +TAOS_DEFINE_ERROR(TSDB_CODE_AUDIT_NOT_FORMAT_TO_JSON, "can't format to json") +TAOS_DEFINE_ERROR(TSDB_CODE_AUDIT_FAIL_SEND_AUDIT_RECORD, "Failed to send out audit record") +TAOS_DEFINE_ERROR(TSDB_CODE_AUDIT_FAIL_GENERATE_JSON, "Failed to generate json") #ifdef TAOS_ERROR_C }; #endif From 101d1936e9187b0adf09c4e819070edd5d2d5353 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 16 Jul 2024 09:39:58 +0800 Subject: [PATCH 27/48] feat:[TD-30975]process return value in schemaless --- source/client/src/clientSml.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 6e2f4ba367..1c705f1f6f 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1025,7 +1025,6 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, int32_t code = TSDB_CODE_SUCCESS; SCmdMsgInfo pCmdMsg = {0}; char *pSql = NULL; - char *msg = NULL; // put front for free pReq.numOfColumns = taosArrayGetSize(pColumns); @@ -1110,15 +1109,15 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, code = TSDB_CODE_OUT_OF_MEMORY; goto end; } - msg = taosMemoryMalloc(pCmdMsg.msgLen); - if (NULL == msg) { + pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); + if (NULL == pCmdMsg.pMsg) { code = TSDB_CODE_OUT_OF_MEMORY; goto end; } - pCmdMsg.pMsg = msg; if (tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) < 0){ code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pCmdMsg.pMsg); goto end; } @@ -1140,7 +1139,6 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, code = pRequest->code; end: - taosMemoryFree(msg); destroyRequest(pRequest); tFreeSMCreateStbReq(&pReq); return code; @@ -1620,12 +1618,12 @@ int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { info->taos = acquireTscObj(*(int64_t *)taos); if (info->taos == NULL) { code = TSDB_CODE_TSC_DISCONNECTED; - goto ERROR; + goto FAILED; } code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code); - goto ERROR; + goto FAILED; } } @@ -1636,7 +1634,7 @@ int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { if (info->pVgHash == NULL || info->childTables == NULL || info->superTables == NULL || info->tableUids == NULL) { uError("create SSmlHandle hash obj failed"); code = TSDB_CODE_OUT_OF_MEMORY; - goto ERROR; + goto FAILED; } taosHashSetFreeFp(info->superTables, smlDestroySTableMeta); taosHashSetFreeFp(info->childTables, smlDestroyTableInfo); @@ -1644,7 +1642,7 @@ int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { info->id = smlGenId(); code = smlInitHandle(&info->pQuery); if (code != TSDB_CODE_SUCCESS){ - goto ERROR; + goto FAILED; } info->dataFormat = true; @@ -1655,13 +1653,13 @@ int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { if (info->tagJsonArray == NULL || info->valueJsonArray == NULL || info->preLineTagKV == NULL) { uError("SML:0x%" PRIx64 " failed to allocate memory", info->id); code = TSDB_CODE_OUT_OF_MEMORY; - goto ERROR; + goto FAILED; } *handle = info; return code; -ERROR: +FAILED: smlDestroyInfo(info); return code; } From 2d3d6e7b31f6052115d5ef9677a736d32741c819 Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 16 Jul 2024 02:10:15 +0000 Subject: [PATCH 28/48] fix/TD-30989 --- source/libs/audit/src/auditMain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/audit/src/auditMain.c b/source/libs/audit/src/auditMain.c index aa3b669c1b..7a8de49abe 100644 --- a/source/libs/audit/src/auditMain.c +++ b/source/libs/audit/src/auditMain.c @@ -32,8 +32,8 @@ char* tsAuditBatchUri = "/audit-batch"; int32_t auditInit(const SAuditCfg *pCfg) { tsAudit.cfg = *pCfg; tsAudit.records = taosArrayInit(0, sizeof(SAuditRecord *)); - taosThreadMutexInit(&tsAudit.lock, NULL); - return 0; + if(tsAudit.records == NULL) return TSDB_CODE_OUT_OF_MEMORY; + return taosThreadMutexInit(&tsAudit.lock, NULL); } static FORCE_INLINE void auditDeleteRecord(SAuditRecord * record) { From 88281f7f34f5deb4a642c65e1a910ecef0c977fd Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 16 Jul 2024 11:50:19 +0800 Subject: [PATCH 29/48] more fix --- include/util/tencode.h | 83 +++++++++++++++++++-------------------- source/util/src/tencode.c | 2 +- 2 files changed, 41 insertions(+), 44 deletions(-) diff --git a/include/util/tencode.h b/include/util/tencode.h index 419be59a92..9b9f8af1b6 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -47,11 +47,8 @@ typedef struct { SDecoderNode* dStack; } SDecoder; -#define TD_CODER_POS(CODER) ((CODER)->pos) -#define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos) -#define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE)) -#define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE)) -#define TD_CODER_REMAIN_CAPACITY(CODER) ((CODER)->size - (CODER)->pos) +#define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos) +#define TD_CODER_REMAIN_CAPACITY(CODER) ((CODER)->size - (CODER)->pos) #define tEncodeSize(E, S, SIZE, RET) \ do { \ @@ -134,7 +131,7 @@ static FORCE_INLINE int32_t tEncodeFixed(SEncoder* pCoder, const void* val, uint } pCoder->pos += size; - TAOS_RETURN(0); + return 0; } static FORCE_INLINE int32_t tEncodeU8(SEncoder* pCoder, uint8_t val) { return tEncodeFixed(pCoder, &val, sizeof(val)); } @@ -162,7 +159,7 @@ static FORCE_INLINE int32_t tEncodeU16v(SEncoder* pCoder, uint16_t val) { TAOS_CHECK_RETURN(tEncodeU8(pCoder, (val | ENCODE_LIMIT) & 0xff)); val >>= 7; } - TAOS_RETURN(tEncodeU8(pCoder, val)); + return tEncodeU8(pCoder, val); } static FORCE_INLINE int32_t tEncodeI16v(SEncoder* pCoder, int16_t val) { return tEncodeU16v(pCoder, ZIGZAGE(int16_t, val)); @@ -172,7 +169,7 @@ static FORCE_INLINE int32_t tEncodeU32v(SEncoder* pCoder, uint32_t val) { TAOS_CHECK_RETURN(tEncodeU8(pCoder, (val | ENCODE_LIMIT) & 0xff)); val >>= 7; } - TAOS_RETURN(tEncodeU8(pCoder, val)); + return tEncodeU8(pCoder, val); } static FORCE_INLINE int32_t tEncodeI32v(SEncoder* pCoder, int32_t val) { return tEncodeU32v(pCoder, ZIGZAGE(int32_t, val)); @@ -182,7 +179,7 @@ static FORCE_INLINE int32_t tEncodeU64v(SEncoder* pCoder, uint64_t val) { TAOS_CHECK_RETURN(tEncodeU8(pCoder, (val | ENCODE_LIMIT) & 0xff)); val >>= 7; } - TAOS_RETURN(tEncodeU8(pCoder, val)); + return tEncodeU8(pCoder, val); } static FORCE_INLINE int32_t tEncodeI64v(SEncoder* pCoder, int64_t val) { return tEncodeU64v(pCoder, ZIGZAGE(int64_t, val)); @@ -212,15 +209,15 @@ static FORCE_INLINE int32_t tEncodeBinary(SEncoder* pCoder, const uint8_t* val, TAOS_CHECK_RETURN(tEncodeU32v(pCoder, len)); if (len) { if (pCoder->data) { - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, len)) { + if (pCoder->pos + len > pCoder->size) { TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } - memcpy(TD_CODER_CURRENT(pCoder), val, len); + memcpy(pCoder->data + pCoder->pos, val, len); } - TD_CODER_MOVE_POS(pCoder, len); + pCoder->pos += len; } - TAOS_RETURN(0); + return 0; } static FORCE_INLINE int32_t tEncodeCStrWithLen(SEncoder* pCoder, const char* val, uint32_t len) { @@ -239,7 +236,7 @@ static int32_t tDecodeFixed(SDecoder* pCoder, void* val, uint32_t size) { memcpy(val, pCoder->data + pCoder->pos, size); } pCoder->pos += size; - TAOS_RETURN(0); + return 0; } static FORCE_INLINE int32_t tDecodeU8(SDecoder* pCoder, uint8_t* val) { @@ -277,10 +274,10 @@ static FORCE_INLINE int32_t tDecodeU16v(SDecoder* pCoder, uint16_t* val) { for (int32_t i = 0;; i++) { TAOS_CHECK_RETURN(tDecodeU8(pCoder, &byte)); if (byte < ENCODE_LIMIT) { - tval |= (byte << (7 * i)); + tval |= (((uint16_t)byte) << (7 * i)); break; } else { - tval |= ((byte & (ENCODE_LIMIT - 1)) << (7 * i)); + tval |= ((((uint16_t)byte) & (ENCODE_LIMIT - 1)) << (7 * i)); } } @@ -288,7 +285,7 @@ static FORCE_INLINE int32_t tDecodeU16v(SDecoder* pCoder, uint16_t* val) { *val = tval; } - TAOS_RETURN(0); + return 0; } static FORCE_INLINE int32_t tDecodeI16v(SDecoder* pCoder, int16_t* val) { @@ -299,7 +296,7 @@ static FORCE_INLINE int32_t tDecodeI16v(SDecoder* pCoder, int16_t* val) { *val = ZIGZAGD(int16_t, tval); } - TAOS_RETURN(0); + return 0; } // 32v @@ -309,10 +306,10 @@ static FORCE_INLINE int32_t tDecodeU32v(SDecoder* pCoder, uint32_t* val) { for (int32_t i = 0;; i++) { TAOS_CHECK_RETURN(tDecodeU8(pCoder, &byte)); if (byte < ENCODE_LIMIT) { - tval |= (byte << (7 * i)); + tval |= (((uint32_t)byte) << (7 * i)); break; } else { - tval |= ((byte & (ENCODE_LIMIT - 1)) << (7 * i)); + tval |= ((((uint32_t)byte) & (ENCODE_LIMIT - 1)) << (7 * i)); } } @@ -320,7 +317,7 @@ static FORCE_INLINE int32_t tDecodeU32v(SDecoder* pCoder, uint32_t* val) { *val = tval; } - TAOS_RETURN(0); + return 0; } static FORCE_INLINE int32_t tDecodeI32v(SDecoder* pCoder, int32_t* val) { @@ -331,7 +328,7 @@ static FORCE_INLINE int32_t tDecodeI32v(SDecoder* pCoder, int32_t* val) { *val = ZIGZAGD(int32_t, tval); } - TAOS_RETURN(0); + return 0; } // 64v @@ -341,10 +338,10 @@ static FORCE_INLINE int32_t tDecodeU64v(SDecoder* pCoder, uint64_t* val) { for (int32_t i = 0;; i++) { TAOS_CHECK_RETURN(tDecodeU8(pCoder, &byte)); if (byte < ENCODE_LIMIT) { - tval |= (byte << (7 * i)); + tval |= (((uint64_t)byte) << (7 * i)); break; } else { - tval |= ((byte & (ENCODE_LIMIT - 1)) << (7 * i)); + tval |= ((((uint64_t)byte) & (ENCODE_LIMIT - 1)) << (7 * i)); } } @@ -352,7 +349,7 @@ static FORCE_INLINE int32_t tDecodeU64v(SDecoder* pCoder, uint64_t* val) { *val = tval; } - TAOS_RETURN(0); + return 0; } static FORCE_INLINE int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val) { @@ -363,7 +360,7 @@ static FORCE_INLINE int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val) { *val = ZIGZAGD(int64_t, tval); } - TAOS_RETURN(0); + return 0; } static FORCE_INLINE int32_t tDecodeFloat(SDecoder* pCoder, float* val) { @@ -377,7 +374,7 @@ static FORCE_INLINE int32_t tDecodeFloat(SDecoder* pCoder, float* val) { if (val) { *val = v.f; } - TAOS_RETURN(0); + return 0; } static FORCE_INLINE int32_t tDecodeDouble(SDecoder* pCoder, double* val) { @@ -391,7 +388,7 @@ static FORCE_INLINE int32_t tDecodeDouble(SDecoder* pCoder, double* val) { if (val) { *val = v.d; } - TAOS_RETURN(0); + return 0; } static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint32_t* len) { @@ -402,22 +399,22 @@ static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint3 *len = length; } - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) { + if (pCoder->pos + length > pCoder->size) { TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } if (val) { - *val = (uint8_t*)TD_CODER_CURRENT(pCoder); + *val = pCoder->data + pCoder->pos; } - TD_CODER_MOVE_POS(pCoder, length); - TAOS_RETURN(0); + pCoder->pos += length; + return 0; } static FORCE_INLINE int32_t tDecodeCStrAndLen(SDecoder* pCoder, char** val, uint32_t* len) { TAOS_CHECK_RETURN(tDecodeBinary(pCoder, (uint8_t**)val, len)); (*len) -= 1; - TAOS_RETURN(0); + return 0; } static FORCE_INLINE int32_t tDecodeCStr(SDecoder* pCoder, char** val) { @@ -431,7 +428,7 @@ static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) { TAOS_CHECK_RETURN(tDecodeCStrAndLen(pCoder, &pStr, &len)); memcpy(val, pStr, len + 1); - TAOS_RETURN(0); + return 0; } static FORCE_INLINE int32_t tDecodeBinaryAlloc(SDecoder* pCoder, void** val, uint64_t* len) { @@ -440,7 +437,7 @@ static FORCE_INLINE int32_t tDecodeBinaryAlloc(SDecoder* pCoder, void** val, uin if (length) { if (len) *len = length; - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) { + if (pCoder->pos + length > pCoder->size) { TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } @@ -449,13 +446,13 @@ static FORCE_INLINE int32_t tDecodeBinaryAlloc(SDecoder* pCoder, void** val, uin TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(*val, TD_CODER_CURRENT(pCoder), length); + memcpy(*val, pCoder->data + pCoder->pos, length); - TD_CODER_MOVE_POS(pCoder, length); + pCoder->pos += length; } else { *val = NULL; } - TAOS_RETURN(0); + return 0; } static FORCE_INLINE int32_t tDecodeBinaryAlloc32(SDecoder* pCoder, void** val, uint32_t* len) { @@ -464,26 +461,26 @@ static FORCE_INLINE int32_t tDecodeBinaryAlloc32(SDecoder* pCoder, void** val, u if (length) { if (len) *len = length; - if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) { + if (pCoder->pos + length > pCoder->size) { TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } *val = taosMemoryMalloc(length); if (*val == NULL) { TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(*val, TD_CODER_CURRENT(pCoder), length); + memcpy(*val, pCoder->data + pCoder->pos, length); - TD_CODER_MOVE_POS(pCoder, length); + pCoder->pos += length; } else { *val = NULL; } - TAOS_RETURN(0); + return 0; } static FORCE_INLINE int32_t tDecodeCStrAndLenAlloc(SDecoder* pCoder, char** val, uint64_t* len) { TAOS_CHECK_RETURN(tDecodeBinaryAlloc(pCoder, (void**)val, len)); (*len) -= 1; - TAOS_RETURN(0); + return 0; } static FORCE_INLINE int32_t tDecodeCStrAlloc(SDecoder* pCoder, char** val) { diff --git a/source/util/src/tencode.c b/source/util/src/tencode.c index 0c7275d481..13b3fa4f5a 100644 --- a/source/util/src/tencode.c +++ b/source/util/src/tencode.c @@ -115,7 +115,7 @@ void tEndEncode(SEncoder* pCoder) { (void)tEncodeI32(pCoder, len); - TD_CODER_MOVE_POS(pCoder, len); + pCoder->pos += len; } } From e76e06cf2eb3fd085bfffc6a3715f806c79a33c3 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 16 Jul 2024 11:50:40 +0800 Subject: [PATCH 30/48] feat:[TD-30975]process return value in schemaless --- source/client/inc/clientSml.h | 6 +++--- source/client/src/clientSml.c | 4 ++-- source/client/src/clientSmlJson.c | 3 ++- utils/test/c/sml_test.c | 36 ++++++++++++++++++++----------- 4 files changed, 31 insertions(+), 18 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index d369e0597c..e901f86b41 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -246,9 +246,9 @@ void smlBuildTsKv(SSmlKv *kv, int64_t ts); int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv); int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs); -static inline int32_t smlDoubleToInt64OverFlow(double num) { - if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return 0; - return -1; +static inline bool smlDoubleToInt64OverFlow(double num) { + if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true; + return false; } static inline void smlStrReplace(char* src, int32_t len){ diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 1c705f1f6f..02fbe7aae5 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -728,7 +728,7 @@ bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg) { kvVal->type = TSDB_DATA_TYPE_FLOAT; kvVal->f = (float)result; } else if ((left == 1 && *endptr == 'i') || (left == 3 && strncasecmp(endptr, "i64", left) == 0)) { - if (smlDoubleToInt64OverFlow(result) == 0) { + if (smlDoubleToInt64OverFlow(result)) { errno = 0; int64_t tmp = taosStr2Int64(pVal, &endptr, 10); if (errno == ERANGE) { @@ -1467,7 +1467,7 @@ end: taosHashCancelIterate(info->superTables, tmp); taosHashCleanup(hashTmp); taosMemoryFreeClear(pTableMeta); - code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); + (void)catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); // ignore refresh meta code if there is an error uError("SML:0x%" PRIx64 " smlModifyDBSchemas end failed:%d:%s, format:%d, needModifySchema:%d", info->id, code, tstrerror(code), info->dataFormat, info->needModifySchema); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 519e55d0d1..b0348b4f91 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -739,7 +739,8 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo // notice!!! put ts back to tag to ensure get meta->precision int64_t ts = smlParseTSFromJSON(info, tsJson); if (unlikely(ts < 0)) { - uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); + char* tmp = cJSON_PrintUnformatted(tsJson); + uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %s %" PRId64, info->id, info->msgBuf.buf,tmp, ts); return TSDB_CODE_INVALID_TIMESTAMP; } SSmlKv kvTs = {0}; diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 2338674ae3..a3830d2518 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -121,7 +121,8 @@ int smlProcess_json1_Test() { char *sql1[1] = {0}; for (int i = 0; i < 1; i++) { sql1[i] = taosMemoryCalloc(1, 1024); - strncpy(sql1[i], sql[i], 1023); + ASSERT(sql1[i] != NULL); + (void)strncpy(sql1[i], sql[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, @@ -149,7 +150,8 @@ int smlProcess_json1_Test() { char *sql3[1] = {0}; for (int i = 0; i < 1; i++) { sql3[i] = taosMemoryCalloc(1, 1024); - strncpy(sql3[i], sql2[i], 1023); + ASSERT(sql3[i] != NULL); + (void)strncpy(sql3[i], sql2[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL, @@ -176,7 +178,8 @@ int smlProcess_json1_Test() { char *sql5[1] = {0}; for (int i = 0; i < 1; i++) { sql5[i] = taosMemoryCalloc(1, 1024); - strncpy(sql5[i], sql4[i], 1023); + ASSERT(sql5[i] != NULL); + (void)strncpy(sql5[i], sql4[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql5, sizeof(sql5) / sizeof(sql5[0]), TSDB_SML_JSON_PROTOCOL, @@ -215,7 +218,8 @@ int smlProcess_json2_Test() { char *sql1[1] = {0}; for (int i = 0; i < 1; i++) { sql1[i] = taosMemoryCalloc(1, 1024); - strncpy(sql1[i], sql[i], 1023); + ASSERT(sql1[i] != NULL); + (void)strncpy(sql1[i], sql[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, @@ -249,7 +253,8 @@ int smlProcess_json3_Test() { char *sql1[1] = {0}; for (int i = 0; i < 1; i++) { sql1[i] = taosMemoryCalloc(1, 1024); - strncpy(sql1[i], sql[i], 1023); + ASSERT(sql1[i] != NULL); + (void)strncpy(sql1[i], sql[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, @@ -286,7 +291,8 @@ int smlProcess_json_tag_not_same_Test() { char *sql1[1] = {0}; for (int i = 0; i < 1; i++) { sql1[i] = taosMemoryCalloc(1, 1024); - strncpy(sql1[i], sql[i], 1023); + ASSERT(sql1[i] != NULL); + (void)strncpy(sql1[i], sql[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, @@ -312,7 +318,8 @@ int smlProcess_json_tag_not_same_Test() { char *sql3[1] = {0}; for (int i = 0; i < 1; i++) { sql3[i] = taosMemoryCalloc(1, 1024); - strncpy(sql3[i], sql2[i], 1023); + ASSERT(sql3[i] != NULL); + (void)strncpy(sql3[i], sql2[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL, @@ -338,7 +345,8 @@ int smlProcess_json_tag_not_same_Test() { char *sql5[1] = {0}; for (int i = 0; i < 1; i++) { sql5[i] = taosMemoryCalloc(1, 1024); - strncpy(sql5[i], sql4[i], 1023); + ASSERT(sql5[i] != NULL); + (void)strncpy(sql5[i], sql4[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql5, sizeof(sql5) / sizeof(sql5[0]), TSDB_SML_JSON_PROTOCOL, @@ -1147,7 +1155,8 @@ int sml_19221_Test() { taos_free_result(pRes); char *tmp = (char *)taosMemoryCalloc(1024, 1); - memcpy(tmp, sql[0], strlen(sql[0])); + ASSERT(tmp != NULL); + (void)memcpy(tmp, sql[0], strlen(sql[0])); *(char *)(tmp + 44) = 0; int32_t totalRows = 0; pRes = taos_schemaless_insert_raw(taos, tmp, strlen(sql[0]), &totalRows, TSDB_SML_LINE_PROTOCOL, @@ -1218,7 +1227,8 @@ int sml_ts3116_Test() { taos_free_result(pRes); int32_t totalRows = 0; char *tmp = (char *)taosMemoryCalloc(1024, 1); - memcpy(tmp, sql, strlen(sql)); + ASSERT(tmp != NULL); + (void)memcpy(tmp, sql, strlen(sql)); totalRows = 0; pRes = taos_schemaless_insert_raw(taos, tmp, strlen(tmp), &totalRows, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); @@ -1235,7 +1245,8 @@ int sml_ts3116_Test() { taos_free_result(pRes); tmp = (char *)taosMemoryCalloc(1024, 1); - memcpy(tmp, sql1, strlen(sql1)); + ASSERT(tmp != NULL); + (void)memcpy(tmp, sql1, strlen(sql1)); totalRows = 0; pRes = taos_schemaless_insert_raw(taos, tmp, strlen(tmp), &totalRows, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); @@ -2049,7 +2060,8 @@ int sml_td29373_Test() { char *sql3[1] = {0}; for (int i = 0; i < 1; i++) { sql3[i] = taosMemoryCalloc(1, 1024); - strncpy(sql3[i], sql2[i], 1023); + ASSERT(sql3[i] != NULL); + (void)strncpy(sql3[i], sql2[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL, From cb48954ef33dcb9d7a83fbad0acf75e28de84077 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 16 Jul 2024 13:26:26 +0800 Subject: [PATCH 31/48] enh: add return code validation --- include/os/osMemory.h | 2 ++ include/os/osString.h | 2 ++ source/libs/qworker/inc/qwInt.h | 6 ++-- source/libs/qworker/src/qwDbg.c | 6 ++-- source/libs/qworker/src/qwMsg.c | 4 +-- source/libs/qworker/src/qwUtil.c | 6 ++-- source/libs/qworker/src/qworker.c | 20 +++++------ source/libs/qworker/test/qworkerTests.cpp | 4 +-- source/libs/scheduler/inc/schInt.h | 2 +- source/libs/scheduler/src/schFlowCtrl.c | 2 +- source/libs/scheduler/src/schJob.c | 41 ++++++++++------------- source/libs/scheduler/src/scheduler.c | 2 +- 12 files changed, 48 insertions(+), 49 deletions(-) diff --git a/include/os/osMemory.h b/include/os/osMemory.h index 683d10e926..6166f1dc07 100644 --- a/include/os/osMemory.h +++ b/include/os/osMemory.h @@ -53,6 +53,8 @@ void taosPrintBackTrace(); void taosMemoryTrim(int32_t size); void *taosMemoryMallocAlign(uint32_t alignment, int64_t size); +#define TAOS_MEMSET(_s, _c, _n) ((void)memset(_s, _c, _n)) + #define taosMemoryFreeClear(ptr) \ do { \ if (ptr) { \ diff --git a/include/os/osString.h b/include/os/osString.h index ac7dd7eda8..5fa5a25c6d 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -59,6 +59,8 @@ typedef enum { M2C = 0, C2M } ConvType; (dst)[(size)-1] = 0; \ } while (0) +#define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src)) + char *tstrdup(const char *src); int32_t taosUcs4len(TdUcs4 *ucs4); int64_t taosStr2int64(const char *str); diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index dfd00dc5a7..93bbc44a25 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -218,8 +218,8 @@ typedef struct SQWorkerMgmt { #define QW_IDS() sId, qId, tId, rId, eId #define QW_FPARAMS() mgmt, QW_IDS() -#define QW_STAT_INC(_item, _n) atomic_add_fetch_64(&(_item), _n) -#define QW_STAT_DEC(_item, _n) atomic_sub_fetch_64(&(_item), _n) +#define QW_STAT_INC(_item, _n) (void)atomic_add_fetch_64(&(_item), _n) +#define QW_STAT_DEC(_item, _n) (void)atomic_sub_fetch_64(&(_item), _n) #define QW_STAT_GET(_item) atomic_load_64(&(_item)) #define QW_GET_EVENT(ctx, event) atomic_load_8(&(ctx)->events[event]) @@ -250,7 +250,7 @@ typedef struct SQWorkerMgmt { } while (0) #define QW_SET_RSP_CODE(ctx, code) atomic_store_32(&(ctx)->rspCode, code) -#define QW_UPDATE_RSP_CODE(ctx, code) atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code) +#define QW_UPDATE_RSP_CODE(ctx, code) (void)atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code) #define QW_QUERY_RUNNING(ctx) (QW_GET_PHASE(ctx) == QW_PHASE_PRE_QUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_CQUERY) #define QW_FETCH_RUNNING(ctx) ((ctx)->inFetch) diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c index b7a4b718e2..d3b8d36b25 100644 --- a/source/libs/qworker/src/qwDbg.c +++ b/source/libs/qworker/src/qwDbg.c @@ -221,11 +221,11 @@ void qwDbgSimulateRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx, bool *rsped) { SEpSet epSet = {0}; epSet.inUse = 1; epSet.numOfEps = 3; - strcpy(epSet.eps[0].fqdn, "localhost"); + TAOS_STRCPY(epSet.eps[0].fqdn, "localhost"); epSet.eps[0].port = 7100; - strcpy(epSet.eps[1].fqdn, "localhost"); + TAOS_STRCPY(epSet.eps[1].fqdn, "localhost"); epSet.eps[1].port = 7200; - strcpy(epSet.eps[2].fqdn, "localhost"); + TAOS_STRCPY(epSet.eps[2].fqdn, "localhost"); epSet.eps[2].port = 7300; ctx->phase = QW_PHASE_POST_QUERY; diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 9f4a540f3c..f84e5e463e 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -21,7 +21,7 @@ int32_t qwMallocFetchRsp(int8_t rpcMalloc, int32_t length, SRetrieveTableRsp **r } if (NULL == *rsp) { - memset(pRsp, 0, sizeof(SRetrieveTableRsp)); + TAOS_MEMSET(pRsp, 0, sizeof(SRetrieveTableRsp)); } *rsp = pRsp; @@ -169,7 +169,7 @@ int32_t qwBuildAndSendFetchRsp(int32_t rspType, SRpcHandleInfo *pConn, SRetrieve if (NULL == pRsp) { QW_RET(terrno); } - memset(pRsp, 0, sizeof(SRetrieveTableRsp)); + TAOS_MEMSET(pRsp, 0, sizeof(SRetrieveTableRsp)); dataLength = 0; } diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c index 0451532cbe..0b404ec176 100644 --- a/source/libs/qworker/src/qwUtil.c +++ b/source/libs/qworker/src/qwUtil.c @@ -342,7 +342,7 @@ int32_t qwSendExplainResponse(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { if (NULL == pExec) { QW_ERR_JRET(terrno); } - memcpy(pExec, taosArrayGet(execInfoList, 0), localRsp.rsp.numOfPlans * sizeof(SExplainExecInfo)); + (void)memcpy(pExec, taosArrayGet(execInfoList, 0), localRsp.rsp.numOfPlans * sizeof(SExplainExecInfo)); localRsp.rsp.subplanInfo = pExec; localRsp.qId = qId; localRsp.tId = tId; @@ -537,7 +537,7 @@ int32_t qwSaveTbVersionInfo(qTaskInfo_t pTaskInfo, SQWTaskCtx *ctx) { } if (dbFName[0] && tbName[0]) { - sprintf(tbInfo.tbFName, "%s.%s", dbFName, tbName); + (void)sprintf(tbInfo.tbFName, "%s.%s", dbFName, tbName); } else { tbInfo.tbFName[0] = 0; } @@ -613,7 +613,7 @@ void qwDestroyImpl(void *pMgmt) { taosMemoryFree(mgmt); - atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); + (void)atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); qwCloseRef(); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 5840cc0245..6e684545f7 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -452,8 +452,8 @@ int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes pRes->skey = pDelRes->skey; pRes->ekey = pDelRes->ekey; pRes->affectedRows = pDelRes->affectedRows; - strcpy(pRes->tableFName, pDelRes->tableName); - strcpy(pRes->tsColName, pDelRes->tsColName); + TAOS_STRCPY(pRes->tableFName, pDelRes->tableName); + TAOS_STRCPY(pRes->tsColName, pDelRes->tsColName); taosMemoryFree(output.pData); return TSDB_CODE_SUCCESS; @@ -1126,8 +1126,8 @@ int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { sch->hbConnInfo.handle = NULL; } - memcpy(&sch->hbConnInfo, &qwMsg->connInfo, sizeof(qwMsg->connInfo)); - memcpy(&sch->hbEpId, &req->epId, sizeof(req->epId)); + (void)memcpy(&sch->hbConnInfo, &qwMsg->connInfo, sizeof(qwMsg->connInfo)); + (void)memcpy(&sch->hbEpId, &req->epId, sizeof(req->epId)); QW_UNLOCK(QW_WRITE, &sch->hbConnLock); @@ -1138,7 +1138,7 @@ int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { _return: - memcpy(&rsp.epId, &req->epId, sizeof(req->epId)); + (void)memcpy(&rsp.epId, &req->epId, sizeof(req->epId)); code = qwBuildAndSendHbRsp(&qwMsg->connInfo, &rsp, code); if (code) { @@ -1299,19 +1299,19 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const S int32_t qwNum = atomic_add_fetch_32(&gQwMgmt.qwNum, 1); if (1 == qwNum) { - memset(gQwMgmt.param, 0, sizeof(gQwMgmt.param)); + TAOS_MEMSET(gQwMgmt.param, 0, sizeof(gQwMgmt.param)); } int32_t code = qwOpenRef(); if (code) { - atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); + (void)atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); QW_RET(code); } SQWorker *mgmt = taosMemoryCalloc(1, sizeof(SQWorker)); if (NULL == mgmt) { qError("calloc %d failed", (int32_t)sizeof(SQWorker)); - atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); + (void)atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); QW_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -1345,7 +1345,7 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const S if (pMsgCb) { mgmt->msgCb = *pMsgCb; } else { - memset(&mgmt->msgCb, 0, sizeof(mgmt->msgCb)); + TAOS_MEMSET(&mgmt->msgCb, 0, sizeof(mgmt->msgCb)); } mgmt->refId = taosAddRef(gQwMgmt.qwRef, mgmt); @@ -1379,7 +1379,7 @@ _return: taosTmrCleanUp(mgmt->timer); taosMemoryFreeClear(mgmt); - atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); + (void)atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); } QW_RET(code); diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index d292b271c5..91e1dda2ac 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -170,7 +170,7 @@ int32_t qwtPutReqToFetchQueue(void *node, struct SRpcMsg *pMsg) { printf("malloc failed"); assert(0); } - memcpy(newMsg, pMsg, sizeof(struct SRpcMsg)); + (void)memcpy(newMsg, pMsg, sizeof(struct SRpcMsg)); qwtTestFetchQueue[qwtTestFetchQueueWIdx++] = newMsg; if (qwtTestFetchQueueWIdx >= qwtTestFetchQueueSize) { qwtTestFetchQueueWIdx = 0; @@ -199,7 +199,7 @@ int32_t qwtPutReqToQueue(void *node, EQueueType qtype, struct SRpcMsg *pMsg) { printf("malloc failed"); assert(0); } - memcpy(newMsg, pMsg, sizeof(struct SRpcMsg)); + (void)memcpy(newMsg, pMsg, sizeof(struct SRpcMsg)); qwtTestQueryQueue[qwtTestQueryQueueWIdx++] = newMsg; if (qwtTestQueryQueueWIdx >= qwtTestQueryQueueSize) { qwtTestQueryQueueWIdx = 0; diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 92459e06f5..17476072bb 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -583,7 +583,7 @@ char *schGetOpStr(SCH_OP_TYPE type); int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync); int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq); int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq); -int32_t schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes); +void schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes); int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet); int32_t schHandleTaskSetRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode); void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq, int32_t errCode); diff --git a/source/libs/scheduler/src/schFlowCtrl.c b/source/libs/scheduler/src/schFlowCtrl.c index 1f6e7fbd63..20e1600737 100644 --- a/source/libs/scheduler/src/schFlowCtrl.c +++ b/source/libs/scheduler/src/schFlowCtrl.c @@ -252,7 +252,7 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) { if (i < (taskNum - 1)) { SSchTask *pLastTask = *(SSchTask **)taosArrayGetLast(ctrl->taskList); if (NULL == pLastTask) { - SCH_JOB_ELOG("fail to get the last task, num:%d", taosArrayGetSize(ctrl->taskList)); + SCH_JOB_ELOG("fail to get the last task, num:%d", (int32_t)taosArrayGetSize(ctrl->taskList)); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); } diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 8601c943d9..3f771157d9 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -262,7 +262,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { SSchLevel *pLevel = taosArrayGet(pJob->levels, 0); if (NULL == pLevel) { - SCH_JOB_ELOG("fail to get level 0 level, levelNum:%d", taosArrayGetSize(pJob->levels)); + SCH_JOB_ELOG("fail to get level 0 level, levelNum:%d", (int32_t)taosArrayGetSize(pJob->levels)); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } @@ -274,7 +274,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { SSchTask *pTask = taosArrayGet(pLevel->subTasks, 0); if (NULL == pLevel) { - SCH_JOB_ELOG("fail to get the first task in level 0, taskNum:%d", taosArrayGetSize(pLevel->subTasks)); + SCH_JOB_ELOG("fail to get the first task in level 0, taskNum:%d", (int32_t)taosArrayGetSize(pLevel->subTasks)); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } @@ -424,7 +424,7 @@ _return: SCH_RET(code); } -int32_t schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes) { +void schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes) { pRes->code = atomic_load_32(&pJob->errCode); pRes->numOfRows = pJob->resNumOfRows; @@ -436,8 +436,6 @@ int32_t schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes) { SCH_UNLOCK(SCH_WRITE, &pJob->resLock); SCH_JOB_DLOG("execRes dumped, code: %s", tstrerror(pRes->code)); - - return TSDB_CODE_SUCCESS; } int32_t schDumpJobFetchRes(SSchJob *pJob, void **pData) { @@ -463,7 +461,7 @@ int32_t schDumpJobFetchRes(SSchJob *pJob, void **pData) { if (NULL == *pData) { SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp)); if (NULL == rsp) { - SCH_JOB_ELOG("malloc SRetrieveTableRsp %d failed, code:%x", sizeof(SRetrieveTableRsp), terrno); + SCH_JOB_ELOG("malloc SRetrieveTableRsp %d failed, code:%x", (int32_t)sizeof(SRetrieveTableRsp), terrno); SCH_ERR_JRET(terrno); } @@ -485,14 +483,11 @@ _return: int32_t schNotifyUserExecRes(SSchJob *pJob) { SExecResult *pRes = taosMemoryCalloc(1, sizeof(SExecResult)); if (NULL == pRes) { - qError("malloc execResult %d failed, error: %x", sizeof(SExecResult), terrno); + qError("malloc execResult %d failed, error: %x", (int32_t)sizeof(SExecResult), terrno); SCH_RET(terrno); } - int32_t code = schDumpJobExecRes(pJob, pRes); - if (TSDB_CODE_SUCCESS != code && TSDB_CODE_SUCCESS == atomic_load_32(pJob->errCode)) { - atomic_store_32(&pJob->errCode, code); - } + schDumpJobExecRes(pJob, pRes); SCH_JOB_DLOG("sch start to invoke exec cb, code: %s", tstrerror(pJob->errCode)); (*pJob->userRes.execFp)(pRes, pJob->userRes.cbParam, atomic_load_32(&pJob->errCode)); @@ -505,7 +500,7 @@ int32_t schNotifyUserFetchRes(SSchJob *pJob) { void *pRes = NULL; int32_t code = schDumpJobFetchRes(pJob, &pRes); - if (TSDB_CODE_SUCCESS != code && TSDB_CODE_SUCCESS == atomic_load_32(pJob->errCode)) { + if (TSDB_CODE_SUCCESS != code && TSDB_CODE_SUCCESS == atomic_load_32(&pJob->errCode)) { atomic_store_32(&pJob->errCode, code); } @@ -624,11 +619,11 @@ int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask) { SSchLevel *pLevel = pTask->level; int32_t doneNum = atomic_load_32(&pLevel->taskExecDoneNum); if (doneNum == pLevel->taskNum) { - atomic_sub_fetch_32(&pJob->levelIdx, 1); + (void)atomic_sub_fetch_32(&pJob->levelIdx, 1); pLevel = taosArrayGet(pJob->levels, pJob->levelIdx); if (NULL == pLevel) { - SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", pJob->levelIdx, taosArrayGetSize(pJob->levels)); + SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", pJob->levelIdx, (int32_t)taosArrayGetSize(pJob->levels)); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } @@ -698,7 +693,7 @@ int32_t schLaunchJob(SSchJob *pJob) { } else { SSchLevel *level = taosArrayGet(pJob->levels, pJob->levelIdx); if (NULL == level) { - SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", pJob->levelIdx, taosArrayGetSize(pJob->levels)); + SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", pJob->levelIdx, (int32_t)taosArrayGetSize(pJob->levels)); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } @@ -844,7 +839,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { } else { pJob->nodeList = taosArrayDup(pReq->pNodeList, NULL); if (NULL == pJob->nodeList) { - qError("QID:0x%" PRIx64 " taosArrayDup failed, origNum:%d", pReq->pDag->queryId, taosArrayGetSize(pReq->pNodeList)); + qError("QID:0x%" PRIx64 " taosArrayDup failed, origNum:%d", pReq->pDag->queryId, (int32_t)taosArrayGetSize(pReq->pNodeList)); SCH_ERR_JRET(terrno); } } @@ -880,7 +875,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { SCH_ERR_JRET(terrno); } - atomic_add_fetch_32(&schMgmt.jobNum, 1); + (void)atomic_add_fetch_32(&schMgmt.jobNum, 1); *pJobId = pJob->refId; @@ -963,7 +958,7 @@ int32_t schResetJobForRetry(SSchJob *pJob, int32_t rspCode, bool *inRetry) { SSchLevel *pLevel = taosArrayGet(pJob->levels, i); if (NULL == pLevel) { SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", i, numOfLevels); - SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } pLevel->taskExecDoneNum = 0; @@ -974,7 +969,7 @@ int32_t schResetJobForRetry(SSchJob *pJob, int32_t rspCode, bool *inRetry) { SSchTask *pTask = taosArrayGet(pLevel->subTasks, j); if (NULL == pTask) { SCH_JOB_ELOG("fail to get the %dth task in level %d, taskNum:%d", j, i, numOfTasks); - SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } SCH_LOCK_TASK(pTask); @@ -1077,7 +1072,7 @@ void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq, int } if (errCode) { - schHandleJobFailure(pJob, errCode); + (void)schHandleJobFailure(pJob, errCode); // handle internal } SCH_JOB_DLOG("job end %s operation with code %s", schGetOpStr(type), tstrerror(errCode)); @@ -1156,11 +1151,11 @@ void schProcessOnCbEnd(SSchJob *pJob, SSchTask *pTask, int32_t errCode) { } if (errCode) { - schHandleJobFailure(pJob, errCode); + (void)schHandleJobFailure(pJob, errCode); // ignore error } if (pJob) { - schReleaseJob(pJob->refId); + (void)schReleaseJob(pJob->refId); // ignore error } } @@ -1195,7 +1190,7 @@ _return: SCH_UNLOCK_TASK(pTask); } if (pJob) { - schReleaseJob(rId); + (void)schReleaseJob(rId); // ignore error } SCH_RET(code); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 53ffc3bc6e..8f85e066cd 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -122,7 +122,7 @@ int32_t schedulerGetTasksStatus(int64_t jobId, SArray *pSub) { SQuerySubDesc subDesc = {0}; subDesc.tid = pTask->taskId; - strcpy(subDesc.status, jobTaskStatusStr(pTask->status)); + TAOS_STRCPY(subDesc.status, jobTaskStatusStr(pTask->status)); if (NULL == taosArrayPush(pSub, &subDesc)) { qError("taosArrayPush task %d failed, error: %x, ", m, terrno); From 459a452db21c933c81c38fc0dc92b409b654d2b3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 16 Jul 2024 14:19:07 +0800 Subject: [PATCH 32/48] fix more --- source/util/src/tencode.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/util/src/tencode.c b/source/util/src/tencode.c index 13b3fa4f5a..aa9f157f3a 100644 --- a/source/util/src/tencode.c +++ b/source/util/src/tencode.c @@ -95,7 +95,7 @@ int32_t tStartEncode(SEncoder* pCoder) { pCoder->pos += sizeof(int32_t); } - TAOS_RETURN(0); + return 0; } void tEndEncode(SEncoder* pCoder) { @@ -141,7 +141,7 @@ int32_t tStartDecode(SDecoder* pCoder) { pNode->pNext = pCoder->dStack; pCoder->dStack = pNode; - TAOS_RETURN(0); + return 0; } void tEndDecode(SDecoder* pCoder) { From 9003371dffc0a557dd4251022a8e03fee04865eb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 16 Jul 2024 14:56:30 +0800 Subject: [PATCH 33/48] add return code --- include/util/talgo.h | 9 +++++---- include/util/tarray.h | 2 +- source/util/src/talgo.c | 37 ++++++++++++++++++++++++++++--------- source/util/src/tarray.c | 5 ++--- 4 files changed, 36 insertions(+), 17 deletions(-) diff --git a/include/util/talgo.h b/include/util/talgo.h index 29c51cc16b..dda213ae5a 100644 --- a/include/util/talgo.h +++ b/include/util/talgo.h @@ -52,7 +52,7 @@ typedef int32_t (*__ext_compar_fn_t)(const void *p1, const void *p2, const void * @param param * @param comparFn */ -void taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn); +int32_t taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn); /** * Non-recursive quick sort. @@ -98,8 +98,8 @@ void *taosbsearch(const void *key, const void *base, int32_t nmemb, int32_t size * @param maxroot: if heap is max root heap * @return */ -void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, - __ext_compar_fn_t compar, char *buf, bool maxroot); +int32_t taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, + __ext_compar_fn_t compar, char *buf, bool maxroot); /** * sort heap to make sure it is a max/min root heap @@ -114,7 +114,8 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const * @param maxroot: if heap is max root heap * @return */ -void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, bool maxroot); +int32_t taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, + bool maxroot); #ifdef __cplusplus } diff --git a/include/util/tarray.h b/include/util/tarray.h index cef09e40f8..0d01c9248f 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -251,7 +251,7 @@ int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t * @return */ -void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param); +int32_t taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param); int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode); void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz, int8_t sver); diff --git a/source/util/src/talgo.c b/source/util/src/talgo.c index 3e5a86588d..d7cd0348f4 100644 --- a/source/util/src/talgo.c +++ b/source/util/src/talgo.c @@ -147,10 +147,14 @@ static void tqsortImpl(void *src, int32_t start, int32_t end, int64_t size, cons } } -void taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn) { +int32_t taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn) { char *buf = taosMemoryCalloc(1, size); // prepare the swap buffer + if (NULL == buf) { + return TSDB_CODE_OUT_OF_MEMORY; + } tqsortImpl(src, 0, (int32_t)numOfElem - 1, (int32_t)size, param, comparFn, buf); taosMemoryFreeClear(buf); + return 0; } #define DOSWAP(a, b, size) \ @@ -324,14 +328,17 @@ void *taosbsearch(const void *key, const void *base, int32_t nmemb, int32_t size } } -void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, - __ext_compar_fn_t compar, char *buf, bool maxroot) { +int32_t taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, + __ext_compar_fn_t compar, char *buf, bool maxroot) { int32_t parent; int32_t child; char *tmp = NULL; if (buf == NULL) { tmp = taosMemoryMalloc(size); + if (NULL == tmp) { + return TSDB_CODE_OUT_OF_MEMORY; + } } else { tmp = buf; } @@ -378,24 +385,31 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const if (buf == NULL) { taosMemoryFree(tmp); } + + return 0; } -void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, - bool maxroot) { +int32_t taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, + bool maxroot) { int32_t i; char *buf = taosMemoryCalloc(1, size); if (buf == NULL) { - return; + return TSDB_CODE_OUT_OF_MEMORY; } if (base && size > 0) { for (i = len / 2 - 1; i >= 0; i--) { - taosheapadjust(base, size, i, len - 1, parcompar, compar, buf, maxroot); + int32_t code = taosheapadjust(base, size, i, len - 1, parcompar, compar, buf, maxroot); + if (code) { + taosMemoryFree(buf); + return code; + } } } taosMemoryFree(buf); + return 0; } static void taosMerge(void *src, int32_t start, int32_t leftend, int32_t end, int64_t size, const void *param, @@ -441,7 +455,10 @@ static int32_t taosMergeSortHelper(void *src, int64_t numOfElem, int64_t size, c // short array sort, instead of merge sort process const int32_t THRESHOLD_SIZE = 6; char *buf = taosMemoryCalloc(1, size); // prepare the swap buffer - if (buf == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (buf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + for (int32_t start = 0; start < numOfElem - 1; start += THRESHOLD_SIZE) { int32_t end = (start + THRESHOLD_SIZE - 1) <= numOfElem - 1 ? (start + THRESHOLD_SIZE - 1) : numOfElem - 1; tInsertSort(src, size, start, end, param, comparFn, buf); @@ -451,7 +468,9 @@ static int32_t taosMergeSortHelper(void *src, int64_t numOfElem, int64_t size, c if (numOfElem > THRESHOLD_SIZE) { int32_t currSize; void *tmp = taosMemoryMalloc(numOfElem * size); - if (tmp == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } for (currSize = THRESHOLD_SIZE; currSize <= numOfElem - 1; currSize = 2 * currSize) { int32_t leftStart; diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 4c056178c2..ec5e41f256 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -518,9 +518,8 @@ void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t // todo remove it // order array -void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param) { - taosqsort(pArray->pData, pArray->size, pArray->elemSize, param, fn); - // taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param); +int32_t taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param) { + return taosqsort(pArray->pData, pArray->size, pArray->elemSize, param, fn); } void taosArraySwap(SArray* a, SArray* b) { From d2cb53ab3d789130c5766a64e7700b2e6e853d57 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 16 Jul 2024 15:49:08 +0800 Subject: [PATCH 34/48] refactor: remove the invalid return value. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 100 ++++++++++++++------- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 90 +++++++++++++------ source/dnode/vnode/src/tsdb/tsdbReadUtil.h | 31 ++++--- 3 files changed, 145 insertions(+), 76 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 048e407c76..d1702a94fe 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -679,7 +679,12 @@ static int32_t loadFileBlockBrinInfo(STsdbReader* pReader, SArray* pIndexList, S cleanupInfoForNextFileset(pReader->status.pTableMap); initBrinRecordIter(&iter, pReader->pFileReader, pIndexList); - while (((pRecord = getNextBrinRecord(&iter)) != NULL)) { + while (1) { + int32_t code = getNextBrinRecord(&iter, &pRecord); + if (code != TSDB_CODE_SUCCESS) { + break; + } + if (pRecord->suid > pReader->info.suid) { break; } @@ -707,7 +712,11 @@ static int32_t loadFileBlockBrinInfo(STsdbReader* pReader, SArray* pIndexList, S ASSERT(pRecord->suid == pReader->info.suid && uid == pRecord->uid); - STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); + STableBlockScanInfo* pScanInfo = NULL; + code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } // here we should find the first timestamp that is greater than the lastProcKey // the window is an open interval NOW. @@ -1409,8 +1418,10 @@ static bool getNeighborBlockOfTable(SDataBlockIter* pBlockIter, SFileDataBlockIn static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t index, int32_t step) { + int32_t code = TSDB_CODE_SUCCESS; if (index < 0 || index >= pBlockIter->numOfBlocks) { - return -1; + code = -1; + return code; } SFileDataBlockInfo fblock = *(SFileDataBlockInfo*)taosArrayGet(pBlockIter->blockList, index); @@ -1421,8 +1432,12 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte for (int32_t i = index - 1; i >= pBlockIter->index; --i) { SFileDataBlockInfo* pBlockInfo = taosArrayGet(pBlockIter->blockList, i); - STableBlockScanInfo* pBlockScanInfo = - getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); + STableBlockScanInfo* pBlockScanInfo = NULL; + code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx); pTableDataBlockIdx->globalIndex = i + 1; @@ -1432,8 +1447,12 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte for (int32_t i = index + 1; i <= pBlockIter->index; ++i) { SFileDataBlockInfo* pBlockInfo = taosArrayGet(pBlockIter->blockList, i); - STableBlockScanInfo* pBlockScanInfo = - getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); + STableBlockScanInfo* pBlockScanInfo = NULL; + code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx); pTableDataBlockIdx->globalIndex = i - 1; @@ -1442,7 +1461,12 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte } taosArraySet(pBlockIter->blockList, pBlockIter->index, &fblock); - STableBlockScanInfo* pBlockScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, fblock.uid, pReader->idStr); + STableBlockScanInfo* pBlockScanInfo = NULL; + code = getTableBlockScanInfo(pReader->status.pTableMap, fblock.uid, &pBlockScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, fblock.tbBlockIdx); pTableDataBlockIdx->globalIndex = pBlockIter->index; } @@ -2555,9 +2579,9 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { return code; } - STableBlockScanInfo* pBlockScanInfo = - getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); - if (pBlockScanInfo == NULL) { + STableBlockScanInfo* pBlockScanInfo = NULL; + code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { goto _end; } @@ -2896,6 +2920,7 @@ static int32_t buildCleanBlockFromSttFiles(STsdbReader* pReader, STableBlockScan tsdbDebug("%p uid:%" PRId64 " return clean stt block as one, brange:%" PRId64 "-%" PRId64 " rows:%" PRId64 " %s", pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, pReader->idStr); + return code; } static void buildCleanBlockFromDataFiles(STsdbReader* pReader, STableBlockScanInfo* pScanInfo, @@ -3063,9 +3088,9 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { return pReader->code; } - pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); - if (pScanInfo == NULL) { - return terrno; + code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; } if (pScanInfo->sttKeyInfo.status == STT_FILE_READER_UNINIT) { @@ -4206,6 +4231,7 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e // TODO refactor: with createDataBlockScanInfo int32_t tsdbSetTableList2(STsdbReader* pReader, const void* pTableList, int32_t num) { + int32_t code = TSDB_CODE_SUCCESS; int32_t size = tSimpleHashGetSize(pReader->status.pTableMap); STableBlockScanInfo** p = NULL; @@ -4216,7 +4242,7 @@ int32_t tsdbSetTableList2(STsdbReader* pReader, const void* pTableList, int32_t } if (size < num) { - int32_t code = ensureBlockScanInfoBuf(&pReader->blockInfoBuf, num); + code = ensureBlockScanInfoBuf(&pReader->blockInfoBuf, num); if (code) { return code; } @@ -4237,8 +4263,16 @@ int32_t tsdbSetTableList2(STsdbReader* pReader, const void* pTableList, int32_t for (int32_t i = 0; i < num; ++i) { pUidList->tableUidList[i] = pList[i].uid; - STableBlockScanInfo* pInfo = getPosInBlockInfoBuf(&pReader->blockInfoBuf, i); - initTableBlockScanInfo(pInfo, pList[i].uid, pReader->status.pTableMap, pReader); + STableBlockScanInfo* pInfo = NULL; + code = getPosInBlockInfoBuf(&pReader->blockInfoBuf, i, &pInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = initTableBlockScanInfo(pInfo, pList[i].uid, pReader->status.pTableMap, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } return TDB_CODE_SUCCESS; @@ -4412,11 +4446,10 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi } STsdbReader* p = (pReader->innerReader[0] != NULL) ? pReader->innerReader[0] : pReader; - pReader->status.pTableMap = - createDataBlockScanInfo(p, &pReader->blockInfoBuf, pTableList, &pReader->status.uidList, numOfTables); - if (pReader->status.pTableMap == NULL) { + + code = createDataBlockScanInfo(p, &pReader->blockInfoBuf, pTableList, &pReader->status.uidList, numOfTables, &pReader->status.pTableMap); + if (code != TSDB_CODE_SUCCESS) { *ppReader = NULL; - code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -5067,40 +5100,40 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, return code; } -static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { +static int32_t doRetrieveDataBlock(STsdbReader* pReader, SSDataBlock** pBlock) { SReaderStatus* pStatus = &pReader->status; int32_t code = TSDB_CODE_SUCCESS; SFileDataBlockInfo* pBlockInfo = NULL; + *pBlock = NULL; code = getCurrentBlockInfo(&pStatus->blockIter, &pBlockInfo); if (code != TSDB_CODE_SUCCESS) { - return NULL; + return code; } if (pReader->code != TSDB_CODE_SUCCESS) { - return NULL; + return pReader->code; } - STableBlockScanInfo* pBlockScanInfo = getTableBlockScanInfo(pStatus->pTableMap, pBlockInfo->uid, pReader->idStr); - if (pBlockScanInfo == NULL) { - return NULL; + STableBlockScanInfo* pBlockScanInfo = NULL; + code = getTableBlockScanInfo(pStatus->pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; } code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData, pBlockScanInfo->uid); if (code != TSDB_CODE_SUCCESS) { tBlockDataReset(&pStatus->fileBlockData); - terrno = code; - return NULL; + return code; } code = copyBlockDataToSDataBlock(pReader, &pBlockScanInfo->lastProcKey); if (code != TSDB_CODE_SUCCESS) { tBlockDataReset(&pStatus->fileBlockData); - terrno = code; - return NULL; } - return pReader->resBlockInfo.pResBlock; + *pBlock = pReader->resBlockInfo.pResBlock; + return code; } SSDataBlock* tsdbRetrieveDataBlock2(STsdbReader* pReader, SArray* pIdList) { @@ -5121,7 +5154,8 @@ SSDataBlock* tsdbRetrieveDataBlock2(STsdbReader* pReader, SArray* pIdList) { return pTReader->resBlockInfo.pResBlock; } - SSDataBlock* ret = doRetrieveDataBlock(pTReader); + SSDataBlock* ret = NULL; + doRetrieveDataBlock(pTReader, &ret); qTrace("tsdb/read-retrieve: %p, unlock read mutex", pReader); tsdbReleaseReader(pReader); diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index e84800691f..b1cc545eca 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -112,22 +112,28 @@ void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf) { taosArrayDestroy(pBuf->pData); } -void* getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index) { +int32_t getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index, STableBlockScanInfo** pInfo) { + *pInfo = NULL; + int32_t bucketIndex = index / pBuf->numPerBucket; char** pBucket = taosArrayGet(pBuf->pData, bucketIndex); - return (*pBucket) + (index % pBuf->numPerBucket) * sizeof(STableBlockScanInfo); -} - -STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id) { - STableBlockScanInfo** p = tSimpleHashGet(pTableMap, &uid, sizeof(uid)); - if (p == NULL || *p == NULL) { - terrno = TSDB_CODE_INVALID_PARA; - int32_t size = tSimpleHashGetSize(pTableMap); - tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id); - return NULL; + if (pBucket == NULL) { + return TSDB_CODE_FAILED; } - return *p; + *pInfo = (STableBlockScanInfo*)((*pBucket) + (index % pBuf->numPerBucket) * sizeof(STableBlockScanInfo)); + return TSDB_CODE_SUCCESS; +} + +int32_t getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, STableBlockScanInfo** pInfo, const char* id) { + *pInfo = tSimpleHashGet(pTableMap, &uid, sizeof(uid)); + if (pInfo == NULL || *pInfo == NULL) { + int32_t size = tSimpleHashGetSize(pTableMap); + tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id); + return TSDB_CODE_INVALID_PARA; + } + + return TSDB_CODE_SUCCESS; } int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc) { @@ -245,20 +251,27 @@ int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSH initLastProcKey(pScanInfo, pReader); pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT; - tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES); + int32_t code = tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + tsdbTrace("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pReader, pScanInfo->uid, pScanInfo->lastProcKey.ts, pReader->idStr); - return TSDB_CODE_SUCCESS; + return code; } // NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model -SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, - STableUidList* pUidList, int32_t numOfTables) { +int32_t createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, + STableUidList* pUidList, int32_t numOfTables, SSHashObj** pHashObj) { + int32_t code = 0; + *pHashObj = NULL; + // allocate buffer in order to load data blocks from file // todo use simple hash instead, optimize the memory consumption SSHashObj* pTableMap = tSimpleHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); if (pTableMap == NULL) { - return NULL; + return terrno; } int64_t st = taosGetTimestampUs(); @@ -267,7 +280,7 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t)); if (pUidList->tableUidList == NULL) { tSimpleHashCleanup(pTableMap); - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } pUidList->currentIndex = 0; @@ -275,8 +288,16 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf for (int32_t j = 0; j < numOfTables; ++j) { pUidList->tableUidList[j] = idList[j].uid; - STableBlockScanInfo* pScanInfo = getPosInBlockInfoBuf(pBuf, j); - initTableBlockScanInfo(pScanInfo, idList[j].uid, pTableMap, pTsdbReader); + STableBlockScanInfo* pScanInfo = NULL; + code = getPosInBlockInfoBuf(pBuf, j, &pScanInfo); + if (code != TSDB_CODE_SUCCESS) { + break; + } + + code = initTableBlockScanInfo(pScanInfo, idList[j].uid, pTableMap, pTsdbReader); + if (code != TSDB_CODE_SUCCESS) { + break; + } } taosSort(pUidList->tableUidList, numOfTables, sizeof(uint64_t), uidComparFunc); @@ -286,7 +307,8 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf (sizeof(STableBlockScanInfo) * numOfTables) / 1024.0, pTsdbReader->cost.createScanInfoList, pTsdbReader->idStr); - return pTableMap; + *pHashObj = pTableMap; + return code; } void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) { @@ -391,11 +413,13 @@ void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray pIter->pBrinBlockList = pList; } -SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter) { +int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord) { + *pRecord = NULL; + if (pIter->blockIndex == -1 || (pIter->recordIndex + 1) >= pIter->block.numOfRecords) { pIter->blockIndex += 1; if (pIter->blockIndex >= taosArrayGetSize(pIter->pBrinBlockList)) { - return NULL; + return TSDB_CODE_FAILED; } pIter->pCurrentBlk = taosArrayGet(pIter->pBrinBlockList, pIter->blockIndex); @@ -404,7 +428,7 @@ SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter) { int32_t code = tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block); if (code != TSDB_CODE_SUCCESS) { tsdbError("failed to read brinBlock from file, code:%s", tstrerror(code)); - return NULL; + return TSDB_CODE_FAILED; } pIter->recordIndex = -1; @@ -412,7 +436,9 @@ SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter) { pIter->recordIndex += 1; tBrinBlockGet(&pIter->block, pIter->recordIndex, &pIter->record); - return &pIter->record; + *pRecord = &pIter->record; + + return TSDB_CODE_SUCCESS; } void clearBrinBlockIter(SBrinRecordIter* pIter) { tBrinBlockDestroy(&pIter->block); } @@ -670,7 +696,13 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_ STombRecord record = {0}; uint64_t uid = pReader->status.uidList.tableUidList[*j]; - STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); + STableBlockScanInfo* pScanInfo = NULL; + + code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (pScanInfo->pFileDelData == NULL) { pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData)); } @@ -702,7 +734,11 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_ } uid = pReader->status.uidList.tableUidList[*j]; - pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); + code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (pScanInfo->pFileDelData == NULL) { pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData)); } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index 48ef00baae..ba75cb77a8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -315,25 +315,24 @@ typedef struct SBrinRecordIter { } SBrinRecordIter; int32_t uidComparFunc(const void* p1, const void* p2); +int32_t getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, STableBlockScanInfo** pInfo, const char* id); -STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id); - -SSHashObj* createDataBlockScanInfo(STsdbReader* pReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, - STableUidList* pUidList, int32_t numOfTables); -int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSHashObj* pTableMap, - STsdbReader* pReader); -void clearBlockScanInfo(STableBlockScanInfo* p); -void destroyAllBlockScanInfo(SSHashObj* pTableMap); -void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step); -void cleanupInfoForNextFileset(SSHashObj* pTableMap); -int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables); -void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf); -void* getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index); +int32_t createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, + STableUidList* pUidList, int32_t numOfTables, SSHashObj** pHashObj); +int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSHashObj* pTableMap, + STsdbReader* pReader); +void clearBlockScanInfo(STableBlockScanInfo* p); +void destroyAllBlockScanInfo(SSHashObj* pTableMap); +void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step); +void cleanupInfoForNextFileset(SSHashObj* pTableMap); +int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables); +void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf); +int32_t getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index, STableBlockScanInfo** pRes); // brin records iterator -void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList); -SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter); -void clearBrinBlockIter(SBrinRecordIter* pIter); +void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList); +int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord); +void clearBrinBlockIter(SBrinRecordIter* pIter); // initialize block iterator API int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList); From 19ba5af5e78ebd96c2889643ffc31aa5f997de0b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 16 Jul 2024 15:59:16 +0800 Subject: [PATCH 35/48] refactor: remove the invalid return value. --- source/dnode/vnode/src/inc/tsdb.h | 6 ++--- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 30 +++++++++++---------- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 9 ++++--- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 241a47aca9..943ba099f6 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -909,9 +909,9 @@ void tMergeTreeUnpinSttBlock(SMergeTree *pMTree); bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree); void tMergeTreeClose(SMergeTree *pMTree); -SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols); -void * destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); -void destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost); +int32_t tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, SSttBlockLoadInfo **pInfo); +void destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); +void destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost); // tsdbCache ============================================================================================== typedef enum { diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index fbda31f731..2c03603d73 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -23,11 +23,12 @@ static void tLDataIterClose2(SLDataIter *pIter); // SLDataIter ================================================= -SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols) { +int32_t tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, SSttBlockLoadInfo **pInfo) { + *pInfo = NULL; + SSttBlockLoadInfo *pLoadInfo = taosMemoryCalloc(1, sizeof(SSttBlockLoadInfo)); if (pLoadInfo == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } pLoadInfo->blockData[0].sttBlockIndex = -1; @@ -37,26 +38,29 @@ SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t code = tBlockDataCreate(&pLoadInfo->blockData[0].data); if (code) { - terrno = code; + taosMemoryFreeClear(pLoadInfo); + return code; } code = tBlockDataCreate(&pLoadInfo->blockData[1].data); if (code) { - terrno = code; + taosMemoryFreeClear(pLoadInfo); + return code; } pLoadInfo->aSttBlk = taosArrayInit(4, sizeof(SSttBlk)); if (pLoadInfo->aSttBlk == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFreeClear(pLoadInfo); - return NULL; + return code; } pLoadInfo->pSchema = pSchema; pLoadInfo->colIds = colList; pLoadInfo->numOfCols = numOfCols; - return pLoadInfo; + *pInfo = pLoadInfo; + return code; } static void freeItem(void* pValue) { @@ -66,9 +70,9 @@ static void freeItem(void* pValue) { } } -void *destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { +void destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { if (pLoadInfo == NULL) { - return NULL; + return; } pLoadInfo->currentLoadBlockIndex = 1; @@ -94,7 +98,6 @@ void *destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { taosArrayDestroy(pLoadInfo->aSttBlk); taosMemoryFree(pLoadInfo); - return NULL; } void destroyLDataIter(SLDataIter *pIter) { @@ -913,9 +916,8 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoF } if (pLoadInfo == NULL) { - pLoadInfo = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols); - if (pLoadInfo == NULL) { - code = terrno; + code = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols, &pLoadInfo); + if (code != TSDB_CODE_SUCCESS) { goto _end; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index b1cc545eca..4bca7f73f8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -996,6 +996,7 @@ int32_t adjustSttDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet) int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArray, STsdb* pTsdb, SMergeTreeConf* pConf, const char* pstr) { int32_t numOfRows = 0; + int32_t code = 0; // no data exists, go to end int32_t numOfLevels = pFileSet->lvlArr->size; @@ -1020,7 +1021,7 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra conf.file[0] = *pSttLevel->fobjArr->data[i]->f; const char* pName = pSttLevel->fobjArr->data[i]->fname; - int32_t code = tsdbSttFileReaderOpen(pName, &conf, &pIter->pReader); + code = tsdbSttFileReaderOpen(pName, &conf, &pIter->pReader); if (code != TSDB_CODE_SUCCESS) { tsdbError("open stt file reader error. file:%s, code %s, %s", pName, tstrerror(code), pstr); continue; @@ -1028,8 +1029,8 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra } if (pIter->pBlockLoadInfo == NULL) { - pIter->pBlockLoadInfo = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols); - if (pIter->pBlockLoadInfo == NULL) { + code = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols, &pIter->pBlockLoadInfo); + if (code != TSDB_CODE_SUCCESS) { tsdbError("failed to create block load info, code: out of memory, %s", pstr); continue; } @@ -1037,7 +1038,7 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra // load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file TStatisBlkArray* pStatisBlkArray = NULL; - int32_t code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray**)&pStatisBlkArray); + code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray**)&pStatisBlkArray); if (code != TSDB_CODE_SUCCESS) { tsdbError("failed to load stt block statistics, code:%s, %s", tstrerror(code), pstr); continue; From 1ce180050bfe02d035923ca6154f6a3ffd1af0ef Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 16 Jul 2024 16:01:37 +0800 Subject: [PATCH 36/48] feat:[TD-30975]process return value in schemaless --- include/os/osString.h | 2 +- source/client/src/clientSml.c | 12 ++++++++---- source/client/src/clientSmlJson.c | 10 ++++++++-- source/client/src/clientSmlLine.c | 2 +- source/client/src/clientSmlTelnet.c | 4 +++- 5 files changed, 21 insertions(+), 9 deletions(-) diff --git a/include/os/osString.h b/include/os/osString.h index ac7dd7eda8..8672da616d 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -55,7 +55,7 @@ typedef enum { M2C = 0, C2M } ConvType; #define tstrncpy(dst, src, size) \ do { \ - strncpy((dst), (src), (size)); \ + (void)strncpy((dst), (src), (size)); \ (dst)[(size)-1] = 0; \ } while (0) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 02fbe7aae5..00445b4d12 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1672,6 +1672,10 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { } for (size_t i = 0; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + if (kv == NULL){ + taosHashCleanup(kvHash); + return TSDB_CODE_SML_INVALID_DATA; + } terrno = 0; int32_t code = taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); if (terrno == TSDB_CODE_DUP_KEY) { @@ -1884,7 +1888,7 @@ static int32_t smlInsertData(SSmlHandle *info) { info->cost.insertRpcTime = taosGetTimestampUs(); SAppClusterSummary *pActivity = &info->taos->pAppInfo->summary; - atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); + (void)atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); // no need to check return code (void)launchQueryImpl(info->pRequest, info->pQuery, true, NULL); // no need to check return code @@ -1921,7 +1925,7 @@ int32_t smlClearForRerun(SSmlHandle *info) { info->lines = (SSmlLineInfo *)taosMemoryCalloc(info->lineNum, sizeof(SSmlLineInfo)); } - memset(&info->preLine, 0, sizeof(SSmlLineInfo)); + (void)memset(&info->preLine, 0, sizeof(SSmlLineInfo)); info->currSTableMeta = NULL; info->currTableDataCtx = NULL; @@ -1951,7 +1955,7 @@ static bool getLine(SSmlHandle *info, char *lines[], char **rawLine, char *rawLi if (*rawLine != NULL && (uDebugFlag & DEBUG_DEBUG)) { char *print = taosMemoryCalloc(*len + 1, 1); - memcpy(print, *tmp, *len); + (void)memcpy(print, *tmp, *len); uDebug("SML:0x%" PRIx64 " smlParseLine is raw, numLines:%d, protocol:%d, len:%d, data:%s", info->id, numLines, info->protocol, *len, print); taosMemoryFree(print); @@ -2114,7 +2118,7 @@ void smlSetReqSQL(SRequestObj *request, char *lines[], char *rawLine, char *rawL uError("malloc %d for sml sql failed", rlen + 1); return; } - memcpy(sql, p, rlen); + (void)memcpy(sql, p, rlen); sql[rlen] = 0; request->sqlstr = sql; diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index b0348b4f91..14d6a95e22 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -472,7 +472,11 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { break; } case cJSON_String: { - smlConvertJSONString(kv, "binary", root); + int32_t ret = smlConvertJSONString(kv, "binary", root); + if (ret != TSDB_CODE_SUCCESS) { + uError("OTD:Failed to parse binary value from JSON Obj"); + return ret; + } break; } case cJSON_Object: { @@ -521,7 +525,9 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ if (unlikely(ret != TSDB_CODE_SUCCESS)) { return ret; } - taosArrayPush(preLineKV, &kv); + if (taosArrayPush(preLineKV, &kv) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { return TSDB_CODE_TSC_INVALID_JSON; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index f2443080a6..3456b7df17 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -152,7 +152,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { if(data == NULL){ return TSDB_CODE_OUT_OF_MEMORY; } - memcpy(data, pVal->value + (NCHAR_ADD_LEN - 1), pVal->length); + (void)memcpy(data, pVal->value + (NCHAR_ADD_LEN - 1), pVal->length); pVal->value = data; } diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index ddd2cf5ad0..a25ef4ec95 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -131,7 +131,9 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ .length = valueLen, .keyEscaped = false, .valueEscaped = false}; - taosArrayPush(preLineKV, &kv); + if (taosArrayPush(preLineKV, &kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { return TSDB_CODE_SML_INVALID_DATA; } From d93d5325e097572ad3e684247c5af7f32dac7db5 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 16 Jul 2024 16:03:15 +0800 Subject: [PATCH 37/48] terrno/cos: use new macro to return terrno --- include/common/cos_cp.h | 4 +- source/common/src/cos.c | 167 +++++++++++++++++------------------- source/common/src/cos_cp.c | 169 +++++++++++++++---------------------- 3 files changed, 148 insertions(+), 192 deletions(-) diff --git a/include/common/cos_cp.h b/include/common/cos_cp.h index fd778b1a1d..29532c0265 100644 --- a/include/common/cos_cp.h +++ b/include/common/cos_cp.h @@ -60,8 +60,8 @@ typedef struct { } SCheckpoint; int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint); -void cos_cp_close(TdFilePtr fd); -void cos_cp_remove(char const* filepath); +int32_t cos_cp_close(TdFilePtr fd); +int32_t cos_cp_remove(char const* filepath); int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint); int32_t cos_cp_dump(SCheckpoint* checkpoint); diff --git a/source/common/src/cos.c b/source/common/src/cos.c index fe6e20c47f..f4d42075f2 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -3,6 +3,7 @@ #include "cos.h" #include "cos_cp.h" #include "tdef.h" +#include "tutil.h" extern char tsS3Endpoint[]; extern char tsS3AccessKeyId[]; @@ -40,7 +41,7 @@ int32_t s3Begin() { if ((status = S3_initialize("s3", verifyPeerG | S3_INIT_ALL, hostname)) != S3StatusOK) { uError("Failed to initialize libs3: %s\n", S3_get_status_name(status)); - return -1; + TAOS_RETURN(TSDB_CODE_FAILED); } protocolG = !tsS3Https; @@ -48,12 +49,12 @@ int32_t s3Begin() { uriStyleG = S3UriStyleVirtualHost; } - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } void s3End() { S3_deinitialize(); } -int32_t s3Init() { return 0; /*s3Begin();*/ } +int32_t s3Init() { TAOS_RETURN(TSDB_CODE_SUCCESS); /*s3Begin();*/ } void s3CleanUp() { /*s3End();*/ } @@ -61,17 +62,17 @@ void s3CleanUp() { /*s3End();*/ static int32_t s3ListBucket(char const *bucketname); int32_t s3CheckCfg() { - int32_t code = 0; + int32_t code = 0, lino = 0; if (!tsS3Enabled) { fprintf(stderr, "s3 not configured.\n"); - goto _exit; + TAOS_CHECK_GOTO(code, &lino, _exit); } code = s3Begin(); if (code != 0) { fprintf(stderr, "failed to initialize s3.\n"); - goto _exit; + TAOS_CHECK_GOTO(code, &lino, _exit); } // test put @@ -91,20 +92,17 @@ int32_t s3CheckCfg() { TdFilePtr fp = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); if (!fp) { - code = TAOS_SYSTEM_ERROR(errno); fprintf(stderr, "failed to open test file: %s.\n", path); // uError("ERROR: %s Failed to open %s", __func__, path); - goto _exit; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); } if (taosWriteFile(fp, testdata, strlen(testdata)) < 0) { - code = TAOS_SYSTEM_ERROR(errno); fprintf(stderr, "failed to write test file: %s.\n", path); - goto _exit; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); } if (taosFsyncFile(fp) < 0) { - code = TAOS_SYSTEM_ERROR(errno); fprintf(stderr, "failed to fsync test file: %s.\n", path); - goto _exit; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); } taosCloseFile(&fp); @@ -112,7 +110,7 @@ int32_t s3CheckCfg() { code = s3PutObjectFromFileOffset(path, objectname[0], 0, 16); if (code != 0) { fprintf(stderr, "put object %s : failed.\n", objectname[0]); - goto _exit; + TAOS_CHECK_GOTO(code, &lino, _exit); } fprintf(stderr, "put object %s: success.\n\n", objectname[0]); @@ -121,7 +119,7 @@ int32_t s3CheckCfg() { code = s3ListBucket(tsS3BucketName); if (code != 0) { fprintf(stderr, "listing bucket %s : failed.\n", tsS3BucketName); - goto _exit; + TAOS_CHECK_GOTO(code, &lino, _exit); } fprintf(stderr, "listing bucket %s: success.\n\n", tsS3BucketName); @@ -134,7 +132,7 @@ int32_t s3CheckCfg() { code = s3GetObjectBlock(objectname[0], c_offset, c_len, true, &pBlock); if (code != 0) { fprintf(stderr, "get object %s : failed.\n", objectname[0]); - goto _exit; + TAOS_CHECK_GOTO(code, &lino, _exit); } char buf[7] = {0}; memcpy(buf, pBlock, c_len); @@ -147,14 +145,14 @@ int32_t s3CheckCfg() { code = s3DeleteObjects(objectname, 1); if (code != 0) { fprintf(stderr, "delete object %s : failed.\n", objectname[0]); - goto _exit; + TAOS_CHECK_GOTO(code, &lino, _exit); } fprintf(stderr, "delete object %s: success.\n\n", objectname[0]); s3End(); _exit: - return code; + TAOS_RETURN(code); } static int should_retry() { @@ -169,7 +167,7 @@ static int should_retry() { } */ - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } static void s3PrintError(const char *filename, int lineno, const char *funcname, S3Status status, @@ -202,7 +200,6 @@ static S3Status responsePropertiesCallbackNull(const S3ResponseProperties *prope } static S3Status responsePropertiesCallback(const S3ResponseProperties *properties, void *callbackData) { - //(void)callbackData; TS3SizeCBD *cbd = callbackData; if (properties->contentLength > 0) { cbd->content_length = properties->contentLength; @@ -249,7 +246,7 @@ static int32_t s3ListBucket(char const *bucketname) { SArray *objectArray = getListByPrefix("s3"); if (objectArray == NULL) { - return -1; + TAOS_RETURN(TSDB_CODE_FAILED); } const char **object_name = TARRAY_DATA(objectArray); @@ -262,7 +259,7 @@ static int32_t s3ListBucket(char const *bucketname) { taosArrayDestroyEx(objectArray, s3FreeObjectKey); - return code; + TAOS_RETURN(code); } typedef struct growbuffer { @@ -607,13 +604,13 @@ static int32_t s3PutObjectFromFileSimple(S3BucketContext *bucket_context, char c code = TAOS_SYSTEM_ERROR(EIO); } - return code; + TAOS_RETURN(code); } static int32_t s3PutObjectFromFileWithoutCp(S3BucketContext *bucket_context, char const *object_name, int64_t contentLength, S3PutProperties *put_prop, put_object_callback_data *data) { - int32_t code = 0; + int32_t code = 0, lino = 0; uint64_t totalContentLength = contentLength; uint64_t todoContentLength = contentLength; UploadManager manager = {0}; @@ -647,8 +644,7 @@ static int32_t s3PutObjectFromFileWithoutCp(S3BucketContext *bucket_context, cha if (manager.upload_id == 0 || manager.status != S3StatusOK) { s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg); - code = TAOS_SYSTEM_ERROR(EIO); - goto clean; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit); } upload: @@ -672,8 +668,7 @@ upload: } while (S3_status_is_retryable(partData.put_object_data.status) && should_retry()); if (partData.put_object_data.status != S3StatusOK) { s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg); - code = TAOS_SYSTEM_ERROR(EIO); - goto clean; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit); } contentLength -= chunk_size; todoContentLength -= chunk_size; @@ -686,8 +681,7 @@ upload: int n; for (i = 0; i < totalSeq; i++) { if (!manager.etags[i]) { - code = TAOS_SYSTEM_ERROR(EIO); - goto clean; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit); } n = snprintf(buf, sizeof(buf), "%d" @@ -704,11 +698,13 @@ upload: } while (S3_status_is_retryable(manager.status) && should_retry()); if (manager.status != S3StatusOK) { s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg); - code = TAOS_SYSTEM_ERROR(EIO); - goto clean; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit); } -clean: +_exit: + if (code) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } if (manager.upload_id) { taosMemoryFree(manager.upload_id); } @@ -718,13 +714,13 @@ clean: growbuffer_destroy(manager.gb); taosMemoryFree(manager.etags); - return code; + TAOS_RETURN(code); } static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const char *file, int32_t lmtime, char const *object_name, int64_t contentLength, S3PutProperties *put_prop, put_object_callback_data *data) { - int32_t code = 0; + int32_t code = 0, lino = 0; uint64_t totalContentLength = contentLength; // uint64_t todoContentLength = contentLength; @@ -750,7 +746,7 @@ static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const manager.upload_id = strdup(cp.upload_id); need_init_upload = false; } else { - cos_cp_remove(file_cp_path); + TAOS_CHECK_GOTO(cos_cp_remove(file_cp_path), &lino, _exit); } } @@ -764,15 +760,14 @@ static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const if (manager.upload_id == 0 || manager.status != S3StatusOK) { s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg); code = TAOS_SYSTEM_ERROR(EIO); - goto clean; + goto _exit; } cos_cp_build_upload(&cp, file, contentLength, lmtime, manager.upload_id, chunk_size); } if (cos_cp_open(file_cp_path, &cp)) { - code = TAOS_SYSTEM_ERROR(EIO); - goto clean; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit); } int part_num = 0; @@ -802,8 +797,7 @@ upload: if (i > 0 && cp.parts[i - 1].completed) { if (taosLSeekFile(data->infileFD, cp.parts[i].offset, SEEK_SET) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto clean; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); } } @@ -827,25 +821,23 @@ upload: } while (S3_status_is_retryable(partData.put_object_data.status) && should_retry()); if (partData.put_object_data.status != S3StatusOK) { s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg); - code = TAOS_SYSTEM_ERROR(EIO); + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit); //(void)cos_cp_dump(&cp); - goto clean; } if (!manager.etags[seq - 1]) { - code = TAOS_SYSTEM_ERROR(EIO); - goto clean; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit); } cos_cp_update(&cp, cp.parts[seq - 1].index, manager.etags[seq - 1], 0); - (void)cos_cp_dump(&cp); + TAOS_CHECK_GOTO(cos_cp_dump(&cp), &lino, _exit); contentLength -= chunk_size; // todoContentLength -= chunk_size; } - cos_cp_close(cp.thefile); + TAOS_CHECK_GOTO(cos_cp_close(cp.thefile), &lino, _exit); cp.thefile = 0; int size = 0; @@ -869,20 +861,23 @@ upload: } while (S3_status_is_retryable(manager.status) && should_retry()); if (manager.status != S3StatusOK) { s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg); - code = TAOS_SYSTEM_ERROR(EIO); - goto clean; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit); } - cos_cp_remove(file_cp_path); + TAOS_CHECK_GOTO(cos_cp_remove(file_cp_path), &lino, _exit); -clean: +_exit: /* if (parts) { taosMemoryFree(parts); } */ + if (code) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + if (cp.thefile) { - cos_cp_close(cp.thefile); + (void)cos_cp_close(cp.thefile); } if (cp.parts) { taosMemoryFree(cp.parts); @@ -899,7 +894,7 @@ clean: taosMemoryFree(manager.etags); growbuffer_destroy(manager.gb); - return code; + TAOS_RETURN(code); } int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t withcp) { @@ -917,15 +912,13 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t w put_object_callback_data data = {0}; if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(errno); uError("ERROR: %s Failed to stat file %s: ", __func__, file); - return code; + TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) { - code = TAOS_SYSTEM_ERROR(errno); uError("ERROR: %s Failed to open file %s: ", __func__, file); - return code; + TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength = @@ -956,7 +949,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t w growbuffer_destroy(data.gb); } - return code; + TAOS_RETURN(code); } int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) { @@ -974,22 +967,19 @@ int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int put_object_callback_data data = {0}; if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(errno); uError("ERROR: %s Failed to stat file %s: ", __func__, file); - return code; + TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } contentLength = size; if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) { - code = TAOS_SYSTEM_ERROR(errno); uError("ERROR: %s Failed to open file %s: ", __func__, file); - return code; + TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } if (taosLSeekFile(data.infileFD, offset, SEEK_SET) < 0) { taosCloseFile(&data.infileFD); - code = TAOS_SYSTEM_ERROR(errno); - return code; + TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength = @@ -1016,7 +1006,7 @@ int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int growbuffer_destroy(data.gb); } - return code; + TAOS_RETURN(code); } typedef struct list_bucket_callback_data { @@ -1130,11 +1120,11 @@ int32_t s3DeleteObjects(const char *object_name[], int nobject) { if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg); - code = -1; + code = TSDB_CODE_FAILED; } } - return code; + TAOS_RETURN(code); } void s3DeleteObjectsByPrefix(const char *prefix) { @@ -1187,17 +1177,19 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, if (cbd.status != S3StatusOK) { uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg); - return TAOS_SYSTEM_ERROR(EIO); + + TAOS_RETURN(TAOS_SYSTEM_ERROR(EIO)); } if (check && cbd.buf_pos != size) { uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg); - return TAOS_SYSTEM_ERROR(EIO); + + TAOS_RETURN(TAOS_SYSTEM_ERROR(EIO)); } *ppBlock = cbd.buf; - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } static S3Status getObjectCallback(int bufferSize, const char *buffer, void *callbackData) { @@ -1218,9 +1210,8 @@ int32_t s3GetObjectToFile(const char *object_name, const char *fileName) { TdFilePtr pFile = taosOpenFile(fileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - uError("[s3] open file error, errno:%d, fileName:%s", terrno, fileName); - return -1; + uError("[s3] open file error, errno:%d, fileName:%s", TAOS_SYSTEM_ERROR(errno), fileName); + TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } TS3GetData cbd = {0}; @@ -1232,16 +1223,17 @@ int32_t s3GetObjectToFile(const char *object_name, const char *fileName) { if (cbd.status != S3StatusOK) { uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); taosCloseFile(&pFile); - return TAOS_SYSTEM_ERROR(EIO); + TAOS_RETURN(TAOS_SYSTEM_ERROR(EIO)); } taosCloseFile(&pFile); - return 0; + + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) { SArray *objectArray = getListByPrefix(prefix); - if (objectArray == NULL) return -1; + if (objectArray == NULL) TAOS_RETURN(TSDB_CODE_FAILED); for (size_t i = 0; i < taosArrayGetSize(objectArray); i++) { char *object = taosArrayGetP(objectArray, i); @@ -1255,7 +1247,7 @@ int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) { } if (s3GetObjectToFile(object, fileName) != 0) { taosArrayDestroyEx(objectArray, s3FreeObjectKey); - return -1; + TAOS_RETURN(TSDB_CODE_FAILED); } } taosArrayDestroyEx(objectArray, s3FreeObjectKey); @@ -1279,7 +1271,7 @@ long s3Size(const char *object_name) { if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg); - return -1; + TAOS_RETURN(TSDB_CODE_FAILED); } size = cbd.content_length; @@ -1306,7 +1298,7 @@ int32_t s3Init() { // set log output, default stderr cos_log_set_output(NULL); - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } void s3CleanUp() { cos_http_io_deinitialize(); } @@ -1363,10 +1355,10 @@ int32_t s3PutObjectFromFile(const char *file_str, const char *object_str) { cos_pool_destroy(p); if (s->code != 200) { - return code = s->code; + TAOS_RETURN(s->code); } - return code; + TAOS_RETURN(code); } int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str, int8_t withcp) { @@ -1405,10 +1397,10 @@ int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str, int8_ cos_pool_destroy(p); if (s->code != 200) { - return code = s->code; + TAOS_RETURN(s->code); } - return code; + TAOS_RETURN(code); } void s3DeleteObjectsByPrefix(const char *prefix_str) { @@ -1465,7 +1457,7 @@ int32_t s3DeleteObjects(const char *object_name[], int nobject) { cos_warn_log("delete objects failed\n"); } - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } bool s3Exists(const char *object_name) { @@ -1588,10 +1580,9 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_ s = cos_get_object_to_buffer(options, &bucket, &object, headers, NULL, &download_buffer, &resp_headers); log_status(s); if (!cos_status_is_ok(s)) { - vError("s3: %d(%s)", s->code, s->error_msg); - vError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); - code = TAOS_SYSTEM_ERROR(EIO); - return code; + uError("s3: %d(%s)", s->code, s->error_msg); + uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + TAOS_RETURN(TAOS_SYSTEM_ERROR(EIO)); } // print_headers(resp_headers); @@ -1614,7 +1605,7 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_ *ppBlock = buf; - return code; + TAOS_RETURN(code); } typedef struct { diff --git a/source/common/src/cos_cp.c b/source/common/src/cos_cp.c index 509f0a3a1c..00d3de9500 100644 --- a/source/common/src/cos_cp.c +++ b/source/common/src/cos_cp.c @@ -9,42 +9,35 @@ int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint) { TdFilePtr fd = taosOpenFile(cp_path, TD_FILE_WRITE | TD_FILE_CREATE /* | TD_FILE_TRUNC*/ | TD_FILE_WRITE_THROUGH); if (!fd) { - code = TAOS_SYSTEM_ERROR(errno); - uError("ERROR: %s Failed to open %s", __func__, cp_path); - return code; + uError("%s Failed to open %s", __func__, cp_path); + TAOS_CHECK_RETURN(TAOS_SYSTEM_ERROR(errno)); } checkpoint->thefile = fd; - return code; + TAOS_RETURN(code); } -void cos_cp_close(TdFilePtr fd) { taosCloseFile(&fd); } -void cos_cp_remove(char const* filepath) { (void)taosRemoveFile(filepath); } +int32_t cos_cp_close(TdFilePtr fd) { return taosCloseFile(&fd); } +int32_t cos_cp_remove(char const* filepath) { return taosRemoveFile(filepath); } static int32_t cos_cp_parse_body(char* cp_body, SCheckpoint* cp) { - int32_t code = 0; + int32_t code = 0, lino = 0; cJSON const* item2 = NULL; cJSON* json = cJSON_Parse(cp_body); if (NULL == json) { - code = TSDB_CODE_FILE_CORRUPTED; - uError("ERROR: %s Failed to parse json", __func__); - goto _exit; + TAOS_CHECK_GOTO(TSDB_CODE_FILE_CORRUPTED, &lino, _exit); } cJSON const* item = cJSON_GetObjectItem(json, "ver"); if (!cJSON_IsNumber(item) || item->valuedouble != 1) { - code = TSDB_CODE_FILE_CORRUPTED; - uError("ERROR: %s Failed to parse json ver: %f", __func__, item->valuedouble); - goto _exit; + TAOS_CHECK_GOTO(TSDB_CODE_FILE_CORRUPTED, &lino, _exit); } item = cJSON_GetObjectItem(json, "type"); if (!cJSON_IsNumber(item)) { - code = TSDB_CODE_FILE_CORRUPTED; - uError("ERROR: %s Failed to parse json", __func__); - goto _exit; + TAOS_CHECK_GOTO(TSDB_CODE_FILE_CORRUPTED, &lino, _exit); } cp->cp_type = item->valuedouble; @@ -155,40 +148,35 @@ static int32_t cos_cp_parse_body(char* cp_body, SCheckpoint* cp) { } _exit: + if (code) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } if (json) cJSON_Delete(json); if (cp_body) taosMemoryFree(cp_body); - - return code; + TAOS_RETURN(code); } int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint) { - int32_t code = 0; + int32_t code = 0, lino = 0; TdFilePtr fd = taosOpenFile(filepath, TD_FILE_READ); if (!fd) { - code = TAOS_SYSTEM_ERROR(errno); - uError("ERROR: %s Failed to open %s", __func__, filepath); - goto _exit; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); } int64_t size = -1; - code = taosStatFile(filepath, &size, NULL, NULL); - if (code) { - uError("ERROR: %s Failed to stat %s", __func__, filepath); - goto _exit; - } + TAOS_CHECK_GOTO(taosStatFile(filepath, &size, NULL, NULL), &lino, _exit); char* cp_body = taosMemoryMalloc(size + 1); + if (!cp_body) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } int64_t n = taosReadFile(fd, cp_body, size); if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - uError("ERROR: %s Failed to read %s", __func__, filepath); - goto _exit; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); } else if (n != size) { - code = TSDB_CODE_FILE_CORRUPTED; - uError("ERROR: %s Failed to read %s %" PRId64 "/%" PRId64, __func__, filepath, n, size); - goto _exit; + TAOS_CHECK_GOTO(TSDB_CODE_FILE_CORRUPTED, &lino, _exit); } taosCloseFile(&fd); cp_body[size] = '\0'; @@ -196,184 +184,161 @@ int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint) { return cos_cp_parse_body(cp_body, checkpoint); _exit: + if (code) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } if (fd) { taosCloseFile(&fd); } if (cp_body) { taosMemoryFree(cp_body); } - - return code; + TAOS_RETURN(code); } static int32_t cos_cp_save_json(cJSON const* json, SCheckpoint* checkpoint) { - int32_t code = 0; + int32_t code = 0, lino = 0; char* data = cJSON_PrintUnformatted(json); - if (NULL == data) { - return TSDB_CODE_OUT_OF_MEMORY; + if (!data) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } TdFilePtr fp = checkpoint->thefile; if (taosFtruncateFile(fp, 0) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); } if (taosLSeekFile(fp, 0, SEEK_SET) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); } if (taosWriteFile(fp, data, strlen(data)) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); } if (taosFsyncFile(fp) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); } _exit: + if (code) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } taosMemoryFree(data); - return code; + TAOS_RETURN(code); } int32_t cos_cp_dump(SCheckpoint* cp) { - int32_t code = 0; - int32_t lino = 0; + int32_t code = 0, lino = 0; cJSON* ojson = NULL; cJSON* json = cJSON_CreateObject(); - if (!json) return TSDB_CODE_OUT_OF_MEMORY; + if (!json) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } if (NULL == cJSON_AddNumberToObject(json, "ver", 1)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddNumberToObject(json, "type", cp->cp_type)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddStringToObject(json, "md5", cp->md5)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddStringToObject(json, "upload_id", cp->upload_id)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (COS_CP_TYPE_UPLOAD == cp->cp_type) { ojson = cJSON_AddObjectToObject(json, "file"); if (!ojson) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->file_size)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddNumberToObject(ojson, "lastmodified", cp->file_last_modified)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddStringToObject(ojson, "path", cp->file_path)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddStringToObject(ojson, "file_md5", cp->file_md5)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } } else if (COS_CP_TYPE_DOWNLOAD == cp->cp_type) { ojson = cJSON_AddObjectToObject(json, "object"); if (!ojson) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddNumberToObject(ojson, "object_size", cp->object_size)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddStringToObject(ojson, "object_name", cp->object_name)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddStringToObject(ojson, "object_last_modified", cp->object_last_modified)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddStringToObject(ojson, "object_etag", cp->object_etag)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } } ojson = cJSON_AddObjectToObject(json, "cpparts"); if (!ojson) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddNumberToObject(ojson, "number", cp->part_num)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->part_size)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } cJSON* ajson = cJSON_AddArrayToObject(ojson, "parts"); if (!ajson) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } for (int i = 0; i < cp->part_num; ++i) { cJSON* item = cJSON_CreateObject(); if (!item) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } cJSON_AddItemToArray(ajson, item); if (NULL == cJSON_AddNumberToObject(item, "index", cp->parts[i].index)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddNumberToObject(item, "offset", cp->parts[i].offset)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddNumberToObject(item, "size", cp->parts[i].size)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddNumberToObject(item, "completed", cp->parts[i].completed)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddNumberToObject(item, "crc64", cp->parts[i].crc64)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } if (NULL == cJSON_AddStringToObject(item, "etag", cp->parts[i].etag)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } } code = cos_cp_save_json(json, cp); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(code, &lino, _exit); _exit: if (code) { uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } cJSON_Delete(json); - return code; + TAOS_RETURN(code); } void cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPart* parts, int64_t* consume_bytes) {} From cfa9435a203e8470a470b104f2a7f5449437e6a9 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 16 Jul 2024 16:26:01 +0800 Subject: [PATCH 38/48] fix: invalid ref id issue --- source/libs/scheduler/src/schJob.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 3f771157d9..a8e2e79aee 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -822,12 +822,13 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { } } pJob->pDag = pReq->pDag; - pJob->allocatorRefId = nodesMakeAllocatorWeakRef(pReq->allocatorRefId); - if (pJob->allocatorRefId <= 0) { - qError("QID:0x%" PRIx64 " nodesMakeAllocatorWeakRef failed", pReq->pDag->queryId); - SCH_ERR_JRET(terrno); + if (pReq->allocatorRefId > 0) { + pJob->allocatorRefId = nodesMakeAllocatorWeakRef(pReq->allocatorRefId); + if (pJob->allocatorRefId <= 0) { + qError("QID:0x%" PRIx64 " nodesMakeAllocatorWeakRef failed", pReq->pDag->queryId); + SCH_ERR_JRET(terrno); + } } - pJob->chkKillFp = pReq->chkKillFp; pJob->chkKillParam = pReq->chkKillParam; pJob->userRes.execFp = pReq->execFp; From 5fbc7e25880315dfc06a34b58aba43b141fa6bd6 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 16 Jul 2024 18:43:00 +0800 Subject: [PATCH 39/48] fix usings of TAOS_CHECK_{RETURN, GOTO} --- source/common/src/cos.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/common/src/cos.c b/source/common/src/cos.c index f4d42075f2..813f31c6ff 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -66,7 +66,7 @@ int32_t s3CheckCfg() { if (!tsS3Enabled) { fprintf(stderr, "s3 not configured.\n"); - TAOS_CHECK_GOTO(code, &lino, _exit); + goto _exit; } code = s3Begin(); @@ -759,8 +759,7 @@ static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const if (manager.upload_id == 0 || manager.status != S3StatusOK) { s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg); - code = TAOS_SYSTEM_ERROR(EIO); - goto _exit; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit); } cos_cp_build_upload(&cp, file, contentLength, lmtime, manager.upload_id, chunk_size); From 94f596c4ced8484f1b254669dec28b19a29c4a8b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 16 Jul 2024 18:52:01 +0800 Subject: [PATCH 40/48] refactor: remove the invalid return value. --- include/common/tmsgcb.h | 4 +- include/libs/executor/storageapi.h | 6 +- include/libs/sync/sync.h | 3 +- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/inc/vnodeInt.h | 2 - source/dnode/vnode/src/tsdb/tsdbCache.c | 5 +- source/dnode/vnode/src/tsdb/tsdbRead2.c | 466 ++++++++++++++------- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 7 +- source/dnode/vnode/src/tsdb/tsdbReadUtil.h | 2 +- source/libs/executor/src/scanoperator.c | 19 +- source/libs/sync/inc/syncInt.h | 14 +- source/libs/transport/inc/transComm.h | 2 +- 12 files changed, 344 insertions(+), 188 deletions(-) diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index 03bf8da707..0c61aa5a51 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -17,13 +17,13 @@ #define _TD_COMMON_MSG_CB_H_ #include "os.h" +#include "tmsg.h" #ifdef __cplusplus extern "C" { #endif typedef struct SRpcMsg SRpcMsg; -typedef struct SEpSet SEpSet; typedef struct SMgmtWrapper SMgmtWrapper; typedef struct SRpcHandleInfo SRpcHandleInfo; @@ -46,7 +46,7 @@ typedef int32_t (*PutToQueueFp)(void* pMgmt, EQueueType qtype, SRpcMsg* pMsg); typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype); typedef int32_t (*SendReqFp)(const SEpSet* pEpSet, SRpcMsg* pMsg); typedef void (*SendRspFp)(SRpcMsg* pMsg); -typedef void (*RegisterBrokenLinkArgFp)(SRpcMsg* pMsg); +typedef void (*RegisterBrokenLinkArgFp)(struct SRpcMsg* pMsg); typedef void (*ReleaseHandleFp)(SRpcHandleInfo* pHandle, int8_t type); typedef void (*ReportStartup)(const char* name, const char* desc); diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 45f9f73fb1..edb6ec9cbf 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -105,7 +105,7 @@ typedef struct SMTbCursor { } SMTbCursor; typedef struct SMCtbCursor { - SMeta* pMeta; + struct SMeta* pMeta; void* pCur; tb_uid_t suid; void* pKey; @@ -134,7 +134,7 @@ typedef struct SMetaTableInfo { } SMetaTableInfo; typedef struct SSnapContext { - SMeta* pMeta; + struct SMeta* pMeta; int64_t snapVersion; void* pCur; int64_t suid; @@ -178,7 +178,7 @@ typedef struct TsdReader { int32_t (*tsdNextDataBlock)(); int32_t (*tsdReaderRetrieveBlockSMAInfo)(); - SSDataBlock *(*tsdReaderRetrieveDataBlock)(); + int32_t (*tsdReaderRetrieveDataBlock)(); void (*tsdReaderReleaseDataBlock)(); diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 0be03b82f3..1fb077e3ca 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -70,7 +70,6 @@ typedef int64_t SyncIndex; typedef int64_t SyncTerm; typedef struct SSyncNode SSyncNode; -typedef struct SWal SWal; typedef struct SSyncRaftEntry SSyncRaftEntry; typedef enum { @@ -238,7 +237,7 @@ typedef struct SSyncInfo { int32_t batchSize; SSyncCfg syncCfg; char path[TSDB_FILENAME_LEN]; - SWal* pWal; + struct SWal* pWal; SSyncFSM* pFsm; SMsgCb* msgcb; int32_t pingMs; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index e35c152e9b..990d03f940 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -167,7 +167,7 @@ void tsdbReaderClose2(STsdbReader *pReader); int32_t tsdbNextDataBlock2(STsdbReader *pReader, bool *hasNext); int32_t tsdbRetrieveDatablockSMA2(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave, bool *hasNullSMA); void tsdbReleaseDataBlock2(STsdbReader *pReader); -SSDataBlock *tsdbRetrieveDataBlock2(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); +int32_t tsdbRetrieveDataBlock2(STsdbReader *pReader, SSDataBlock **pBlock, SArray *pIdList); int32_t tsdbReaderReset2(STsdbReader *pReader, SQueryTableDataCond *pCond); int32_t tsdbGetFileBlocksDistInfo2(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader *pHandle); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 4a47e08730..70aa55915a 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -54,7 +54,6 @@ extern "C" { #endif typedef struct SVnodeInfo SVnodeInfo; -typedef struct SMeta SMeta; typedef struct SSma SSma; typedef struct STsdb STsdb; typedef struct STQ STQ; @@ -153,7 +152,6 @@ int32_t vnodeBufPoolRegisterQuery(SVBufPool* pPool, SQueryNode* pQNode); void vnodeBufPoolDeregisterQuery(SVBufPool* pPool, SQueryNode* pQNode, bool proactive); // meta -typedef struct SMCtbCursor SMCtbCursor; typedef struct SMStbCursor SMStbCursor; typedef struct STbUidStore STbUidStore; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index ccdf4a14d6..ed09ca821b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -3212,7 +3212,10 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs pIter->pMemDelData = NULL; - loadMemTombData(&pIter->pMemDelData, pMem, pIMem, pr->info.verRange.maxVer); + code = loadMemTombData(&pIter->pMemDelData, pMem, pIMem, pr->info.verRange.maxVer); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } pIter->idx = (SBlockIdx){.suid = suid, .uid = uid}; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index d1702a94fe..7d1e65ba81 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -398,75 +398,85 @@ static void initReaderStatus(SReaderStatus* pStatus) { pStatus->loadFromFile = true; } -static SSDataBlock* createResBlock(SQueryTableDataCond* pCond, int32_t capacity) { - SSDataBlock* pResBlock = createDataBlock(); - if (pResBlock == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; +static int32_t createResBlock(SQueryTableDataCond* pCond, int32_t capacity, SSDataBlock** pResBlock) { + *pResBlock = createDataBlock(); + if (*pResBlock == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; } for (int32_t i = 0; i < pCond->numOfCols; ++i) { SColumnInfoData colInfo = {0}; colInfo.info = pCond->colList[i]; - blockDataAppendColInfo(pResBlock, &colInfo); + int32_t code = blockDataAppendColInfo(*pResBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(*pResBlock); + *pResBlock = NULL; + return code; + } } - int32_t code = blockDataEnsureCapacity(pResBlock, capacity); + int32_t code = blockDataEnsureCapacity(*pResBlock, capacity); if (code != TSDB_CODE_SUCCESS) { - terrno = code; - taosMemoryFree(pResBlock); - return NULL; + taosMemoryFree(*pResBlock); + *pResBlock = NULL; } - return pResBlock; + + return code; } static int32_t tsdbInitReaderLock(STsdbReader* pReader) { int32_t code = taosThreadMutexInit(&pReader->readerMutex, NULL); - qTrace("tsdb/read: %p, post-init read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + tsdbTrace("tsdb/read: %p, post-init read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); return code; } static int32_t tsdbUninitReaderLock(STsdbReader* pReader) { - int32_t code = -1; - qTrace("tsdb/read: %p, pre-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + int32_t code = TSDB_CODE_SUCCESS; + tsdbTrace("tsdb/read: %p, pre-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); code = taosThreadMutexDestroy(&pReader->readerMutex); - qTrace("tsdb/read: %p, post-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + tsdbTrace("tsdb/read: %p, post-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); return code; } static int32_t tsdbAcquireReader(STsdbReader* pReader) { int32_t code = -1; - qTrace("tsdb/read: %p, pre-take read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + tsdbTrace("tsdb/read: %p, pre-take read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); code = taosThreadMutexLock(&pReader->readerMutex); - qTrace("tsdb/read: %p, post-take read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + tsdbTrace("tsdb/read: %p, post-take read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); return code; } static int32_t tsdbTryAcquireReader(STsdbReader* pReader) { int32_t code = taosThreadMutexTryLock(&pReader->readerMutex); - qTrace("tsdb/read: %p, post-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); - + if (code != TSDB_CODE_SUCCESS) { + tsdbError("tsdb/read: %p, post-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + } else { + tsdbTrace("tsdb/read: %p, post-trytask read mutex: %p", pReader, &pReader->readerMutex); + } return code; } static int32_t tsdbReleaseReader(STsdbReader* pReader) { int32_t code = taosThreadMutexUnlock(&pReader->readerMutex); - qTrace("tsdb/read: %p, post-untake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); - + if (code != TSDB_CODE_SUCCESS) { + tsdbError("tsdb/read: %p post-untake read mutex:%p failed, code:%d", pReader, &pReader->readerMutex, code); + } else { + tsdbTrace("tsdb/read: %p, post-untake read mutex: %p", pReader, &pReader->readerMutex); + } return code; } void tsdbReleaseDataBlock2(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; if (!pStatus->composedDataBlock) { - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); } } @@ -474,11 +484,16 @@ static int32_t initResBlockInfo(SResultBlockInfo* pResBlockInfo, int64_t capacit SQueryTableDataCond* pCond, SBlockLoadSuppInfo* pSup) { pResBlockInfo->capacity = capacity; pResBlockInfo->pResBlock = pResBlock; - terrno = 0; + int32_t code = 0; if (pResBlockInfo->pResBlock == NULL) { pResBlockInfo->freeBlock = true; - pResBlockInfo->pResBlock = createResBlock(pCond, pResBlockInfo->capacity); + pResBlockInfo->pResBlock = NULL; + + code = createResBlock(pCond, pResBlockInfo->capacity, &pResBlockInfo->pResBlock); + if (code != TSDB_CODE_SUCCESS) { + return code; + } if (pSup->numOfPks > 0) { SSDataBlock* p = pResBlockInfo->pResBlock; @@ -505,7 +520,7 @@ static int32_t initResBlockInfo(SResultBlockInfo* pResBlockInfo, int64_t capacit pResBlockInfo->freeBlock = false; } - return terrno; + return code; } static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void** ppReader, int32_t capacity, @@ -576,8 +591,15 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void goto _end; } - tsdbInitReaderLock(pReader); - tsem_init(&pReader->resumeAfterSuspend, 0, 0); + code = tsdbInitReaderLock(pReader); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + + code = tsem_init(&pReader->resumeAfterSuspend, 0, 0); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } *ppReader = pReader; return code; @@ -648,7 +670,11 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFileReader* pFileRead break; } - taosArrayPush(pIndexList, pBrinBlk); + void* p1 = taosArrayPush(pIndexList, pBrinBlk); + if (p1 == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + i += 1; } @@ -788,13 +814,17 @@ static int32_t loadFileBlockBrinInfo(STsdbReader* pReader, SArray* pIndexList, S pBlockNum->numOfBlocks += 1; if (taosArrayGetSize(pTableScanInfoList) == 0) { - taosArrayPush(pTableScanInfoList, &pScanInfo); + p1 = taosArrayPush(pTableScanInfoList, &pScanInfo); } else { STableBlockScanInfo** p = taosArrayGetLast(pTableScanInfoList); if ((*p)->uid != uid) { - taosArrayPush(pTableScanInfoList, &pScanInfo); + p1 = taosArrayPush(pTableScanInfoList, &pScanInfo); } } + + if (p1 == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } clearBrinBlockIter(&iter); @@ -838,6 +868,8 @@ static void updateLastKeyInfo(SRowKey* pKey, SFileDataBlockInfo* pBlockInfo, SDa static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_t colIndex, SColVal* pColVal, SBlockLoadSuppInfo* pSup) { + int32_t code = 0; + if (IS_VAR_DATA_TYPE(pColVal->value.type)) { if (!COL_VAL_IS_VALUE(pColVal)) { colDataSetNULL(pColInfoData, rowIndex); @@ -853,13 +885,13 @@ static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int (void)memcpy(varDataVal(pSup->buildBuf[colIndex]), pColVal->value.pData, pColVal->value.nData); } - colDataSetVal(pColInfoData, rowIndex, pSup->buildBuf[colIndex], false); + code = colDataSetVal(pColInfoData, rowIndex, pSup->buildBuf[colIndex], false); } } else { - colDataSetVal(pColInfoData, rowIndex, (const char*)&pColVal->value.val, !COL_VAL_IS_VALUE(pColVal)); + code = colDataSetVal(pColInfoData, rowIndex, (const char*)&pColVal->value.val, !COL_VAL_IS_VALUE(pColVal)); } - return TSDB_CODE_SUCCESS; + return code; } static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInfo** pInfo) { @@ -1420,8 +1452,7 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte int32_t step) { int32_t code = TSDB_CODE_SUCCESS; if (index < 0 || index >= pBlockIter->numOfBlocks) { - code = -1; - return code; + return TSDB_CODE_FAILED; } SFileDataBlockInfo fblock = *(SFileDataBlockInfo*)taosArrayGet(pBlockIter->blockList, index); @@ -1813,7 +1844,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (ps == NULL) { return terrno; } - tsdbRowMergerInit(pMerger, ps); + int32_t code = tsdbRowMergerInit(pMerger, ps); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } SRowKey minKey = k; @@ -1846,7 +1880,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pfKey, pReader); + code = doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pfKey, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (pkCompEx(&minKey, pSttKey) == 0) { @@ -1855,7 +1892,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (code != TSDB_CODE_SUCCESS) { return code; } - doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); + code = doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (pkCompEx(&minKey, &k) == 0) { @@ -1897,7 +1937,10 @@ static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* if (ps == NULL) { return terrno; } - tsdbRowMergerInit(pMerger, ps); + code = tsdbRowMergerInit(pMerger, ps); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } bool dataInDataFile = hasDataInFileBlock(pBlockData, pDumpInfo); @@ -1944,7 +1987,10 @@ static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* return code; } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader); + code = doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); code = tsdbRowMergerAdd(pMerger, pRow1, NULL); @@ -1953,7 +1999,10 @@ static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* } // pSttKey will be changed when sttBlockReader iterates to the next row, so use pKey instead. - doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); + code = doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { @@ -2019,7 +2068,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (ps == NULL) { return terrno; } - tsdbRowMergerInit(pMerger, ps); + + code = tsdbRowMergerInit(pMerger, ps); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } SRowKey minKey = k; @@ -2059,7 +2112,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pfKey, pReader); + code = doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pfKey, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (pkCompEx(&minKey, pSttKey) == 0) { @@ -2069,7 +2125,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); + code = doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (pkCompEx(&minKey, &ik) == 0) { @@ -2160,10 +2219,9 @@ static void doForwardDataIter(SRowKey* pKey, SIterInfo* pIter, STableBlockScanIn } // handle the open interval issue. Find the first row key that is greater than the given one. -static int32_t forwardDataIter(SRowKey* pKey, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { +static void forwardDataIter(SRowKey* pKey, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { doForwardDataIter(pKey, &pBlockScanInfo->iter, pBlockScanInfo, pReader); doForwardDataIter(pKey, &pBlockScanInfo->iiter, pBlockScanInfo, pReader); - return TSDB_CODE_SUCCESS; } static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { @@ -2196,7 +2254,10 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea return code; } - loadMemTombData(&pBlockScanInfo->pMemDelData, d, di, pReader->info.verRange.maxVer); + code = loadMemTombData(&pBlockScanInfo->pMemDelData, d, di, pReader->info.verRange.maxVer); + if (code != TSDB_CODE_SUCCESS) { + return code; + } if (forward) { forwardDataIter(&startKey.key, pBlockScanInfo, pReader); @@ -2309,8 +2370,15 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan return false; } - initMemDataIterator(pScanInfo, pReader); - initDelSkylineIterator(pScanInfo, pReader->info.order, &pReader->cost); + code = initMemDataIterator(pScanInfo, pReader); + if (code != TSDB_CODE_SUCCESS) { + return false; + } + + code = initDelSkylineIterator(pScanInfo, pReader->info.order, &pReader->cost); + if (code != TSDB_CODE_SUCCESS) { + return code; + } if (conf.rspRows) { pScanInfo->cleanSttBlocks = isCleanSttBlock(info.pKeyRangeList, &pReader->info.window, pScanInfo, order); @@ -2383,7 +2451,11 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc if (ps == NULL) { return terrno; } - tsdbRowMergerInit(pMerger, ps); + + code = tsdbRowMergerInit(pMerger, ps); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } tRowKeyAssign(&pBlockScanInfo->lastProcKey, pKey); @@ -2399,7 +2471,11 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc return code; } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader); + code = doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2446,8 +2522,16 @@ int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanIn } TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); - tsdbRowMergerAdd(pMerger, pRow1, NULL); - doMergeRowsInSttBlock(pSttBlockReader, pScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); + code = tsdbRowMergerAdd(pMerger, pRow1, NULL); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doMergeRowsInSttBlock(pSttBlockReader, pScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2527,7 +2611,10 @@ static int32_t loadNeighborIfOverlap(SFileDataBlockInfo* pBlockInfo, STableBlock // 2. remove it from the scan block list int32_t neighborIndex = tableDataBlockIdx->globalIndex; - setFileBlockActiveInBlockIter(pReader, pBlockIter, neighborIndex, step); + code = setFileBlockActiveInBlockIter(pReader, pBlockIter, neighborIndex, step); + if (code != TSDB_CODE_SUCCESS) { + return code; + } // 3. load the neighbor block, and set it to be the currently accessed file data block code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pBlockInfo->uid); @@ -2599,7 +2686,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } SBlockData* pBlockData = &pReader->status.fileBlockData; - initSttBlockReader(pSttBlockReader, pBlockScanInfo, pReader); + (void) initSttBlockReader(pSttBlockReader, pBlockScanInfo, pReader); while (1) { bool hasBlockData = false; @@ -2696,7 +2783,10 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t orde if (pSource == NULL) { pSource = pBlockScanInfo->pMemDelData; } else { - taosArrayAddAll(pSource, pBlockScanInfo->pMemDelData); + void* p1 = taosArrayAddAll(pSource, pBlockScanInfo->pMemDelData); + if (p1 == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } code = tsdbBuildDeleteSkyline(pSource, 0, taosArrayGetSize(pSource) - 1, pBlockScanInfo->delSkyline); @@ -3094,7 +3184,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { } if (pScanInfo->sttKeyInfo.status == STT_FILE_READER_UNINIT) { - initSttBlockReader(pSttBlockReader, pScanInfo, pReader); + (void) initSttBlockReader(pSttBlockReader, pScanInfo, pReader); } TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader); @@ -3148,7 +3238,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { int64_t st = taosGetTimestampUs(); // let's load data from stt files, make sure clear the cleanStt block flag before load the data from stt files - initSttBlockReader(pSttBlockReader, pScanInfo, pReader); + (void) initSttBlockReader(pSttBlockReader, pScanInfo, pReader); // no data in stt block, no need to proceed. while (hasDataInSttBlock(pScanInfo)) { @@ -3209,10 +3299,17 @@ static int32_t buildBlockFromBufferSeqForPreFileset(STsdbReader* pReader, int64_ continue; } - initMemDataIterator(*pBlockScanInfo, pReader); - initDelSkylineIterator(*pBlockScanInfo, pReader->info.order, &pReader->cost); + int32_t code = initMemDataIterator(*pBlockScanInfo, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - int32_t code = buildDataBlockFromBuf(pReader, *pBlockScanInfo, endKey); + code = initDelSkylineIterator(*pBlockScanInfo, pReader->info.order, &pReader->cost); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = buildDataBlockFromBuf(pReader, *pBlockScanInfo, endKey); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3251,10 +3348,17 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader, int64_t en continue; } - initMemDataIterator(*pBlockScanInfo, pReader); - initDelSkylineIterator(*pBlockScanInfo, pReader->info.order, &pReader->cost); + int32_t code = initMemDataIterator(*pBlockScanInfo, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - int32_t code = buildDataBlockFromBuf(pReader, *pBlockScanInfo, endKey); + code = initDelSkylineIterator(*pBlockScanInfo, pReader->info.order, &pReader->cost); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = buildDataBlockFromBuf(pReader, *pBlockScanInfo, endKey); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3693,6 +3797,7 @@ FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, S int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey* pCurKey, SArray* pDelList, STsdbReader* pReader) { SRowMerger* pMerger = &pReader->status.merger; + int32_t code = 0; while (1) { pIter->hasVal = tsdbTbDataIterNext(pIter->iter); @@ -3727,10 +3832,13 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey* pCurKey, SArra } } - tsdbRowMergerAdd(pMerger, pRow, pTSchema); + code = tsdbRowMergerAdd(pMerger, pRow, pTSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } - return TSDB_CODE_SUCCESS; + return code; } static int32_t doMergeRowsInFileBlockImpl(SBlockData* pBlockData, int32_t rowIndex, SRowKey* pKey, SRowMerger* pMerger, @@ -3748,7 +3856,7 @@ static int32_t doMergeRowsInFileBlockImpl(SBlockData* pBlockData, int32_t rowInd } TSDBROW fRow = tsdbRowFromBlockData(pBlockData, rowIndex); - tsdbRowMergerAdd(pMerger, &fRow, NULL); + int32_t code = tsdbRowMergerAdd(pMerger, &fRow, NULL); rowIndex += step; } @@ -3790,6 +3898,7 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc bool asc = ASCENDING_TRAVERSE(pReader->info.order); int32_t step = asc ? 1 : -1; SVersionRange* pRange = &pReader->info.verRange; + int32_t code = 0; pDumpInfo->rowIndex += step; if ((pDumpInfo->rowIndex <= pBlockData->nRow - 1 && asc) || (pDumpInfo->rowIndex >= 0 && !asc)) { @@ -3802,7 +3911,7 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc CHECK_FILEBLOCK_STATE st; SFileDataBlockInfo* pFileBlockInfo = NULL; - int32_t code = getCurrentBlockInfo(&pReader->status.blockIter, &pFileBlockInfo); + code = getCurrentBlockInfo(&pReader->status.blockIter, &pFileBlockInfo); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3812,19 +3921,20 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc break; } - checkForNeighborFileBlock(pReader, pScanInfo, pFileBlockInfo, pMerger, pKey, &st); - if (st == CHECK_FILEBLOCK_QUIT) { + code = checkForNeighborFileBlock(pReader, pScanInfo, pFileBlockInfo, pMerger, pKey, &st); + if (st == CHECK_FILEBLOCK_QUIT || code != TSDB_CODE_SUCCESS) { break; } } } - return TSDB_CODE_SUCCESS; + return code; } int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, SRowMerger* pMerger, int32_t pkSrcSlot, SVersionRange* pVerRange, const char* idStr) { SRowKey* pRowKey = &pScanInfo->lastProcKey; + int32_t code = TSDB_CODE_SUCCESS; while (nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pkSrcSlot, pVerRange)) { SRowKey* pNextKey = getCurrentKeyInSttBlock(pSttBlockReader); @@ -3832,7 +3942,10 @@ int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanI int32_t ret = pkCompEx(pRowKey, pNextKey); if (ret == 0) { TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); - tsdbRowMergerAdd(pMerger, pRow1, NULL); + code = tsdbRowMergerAdd(pMerger, pRow1, NULL); + if (code != TSDB_CODE_SUCCESS) { + break; + } } else { tsdbTrace("uid:%" PRIu64 " last del index:%d, del range:%d, lastKeyInStt:%" PRId64 ", %s", pScanInfo->uid, pScanInfo->sttBlockDelIndex, (int32_t)taosArrayGetSize(pScanInfo->delSkyline), @@ -3841,7 +3954,7 @@ int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanI } } - return TSDB_CODE_SUCCESS; + return code; } int32_t doMergeMemTableMultiRows(TSDBROW* pRow, SRowKey* pKey, uint64_t uid, SIterInfo* pIter, SArray* pDelList, @@ -4100,7 +4213,11 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pT if (colId == pSchema->columns[j].colId) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); - tRowGet(pTSRow, pSchema, j, &colVal); + code = tRowGet(pTSRow, pSchema, j, &colVal); + if (code) { + return code; + } + code = doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo); if (code) { return code; @@ -4185,7 +4302,11 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e do { TSDBROW row = {.type = -1}; bool freeTSRow = false; - tsdbGetNextRowInMem(pBlockScanInfo, pReader, &row, endKey, &freeTSRow); + code = tsdbGetNextRowInMem(pBlockScanInfo, pReader, &row, endKey, &freeTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (row.type == -1) { break; } @@ -4278,20 +4399,6 @@ int32_t tsdbSetTableList2(STsdbReader* pReader, const void* pTableList, int32_t return TDB_CODE_SUCCESS; } -void* tsdbGetIdx2(SMeta* pMeta) { - if (pMeta == NULL) { - return NULL; - } - return metaGetIdx(pMeta); -} - -void* tsdbGetIvtIdx2(SMeta* pMeta) { - if (pMeta == NULL) { - return NULL; - } - return metaGetIvtIdx(pMeta); -} - uint64_t tsdbGetReaderMaxVersion2(STsdbReader* pReader) { return pReader->info.verRange.maxVer; } static int32_t doOpenReaderImpl(STsdbReader* pReader) { @@ -4304,7 +4411,11 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { pReader->status.bProcMemFirstFileset = true; } - initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); + code = initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + code = resetDataBlockIterator(&pStatus->blockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); if (code != TSDB_CODE_SUCCESS) { return code; @@ -4336,7 +4447,7 @@ static void clearSharedPtr(STsdbReader* p) { p->pSchemaMap = NULL; } -static void setSharedPtr(STsdbReader* pDst, const STsdbReader* pSrc) { +static int32_t setSharedPtr(STsdbReader* pDst, const STsdbReader* pSrc) { pDst->status.pTableMap = pSrc->status.pTableMap; pDst->status.uidList = pSrc->status.uidList; pDst->info.pSchema = pSrc->info.pSchema; @@ -4345,8 +4456,10 @@ static void setSharedPtr(STsdbReader* pDst, const STsdbReader* pSrc) { pDst->pReadSnap->pfSetArray = pSrc->pReadSnap->pfSetArray; if (pDst->info.pSchema) { - tsdbRowMergerInit(&pDst->status.merger, pDst->info.pSchema); + return tsdbRowMergerInit(&pDst->status.merger, pDst->info.pSchema); } + + return TSDB_CODE_SUCCESS; } // ====================================== EXPOSED APIs ====================================== @@ -4427,7 +4540,10 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi } if (pReader->info.pSchema != NULL) { - tsdbRowMergerInit(&pReader->status.merger, pReader->info.pSchema); + code = tsdbRowMergerInit(&pReader->status.merger, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } pReader->pSchemaMap = tSimpleHashInit(8, taosFastHash); @@ -4482,7 +4598,7 @@ void tsdbReaderClose2(STsdbReader* pReader) { return; } - tsdbAcquireReader(pReader); + (void)tsdbAcquireReader(pReader); { if (pReader->innerReader[0] != NULL || pReader->innerReader[1] != NULL) { @@ -4521,7 +4637,7 @@ void tsdbReaderClose2(STsdbReader* pReader) { clearBlockScanInfoBuf(&pReader->blockInfoBuf); if (pReader->pFileReader != NULL) { - tsdbDataFileReaderClose(&pReader->pFileReader); + (void) tsdbDataFileReaderClose(&pReader->pFileReader); } SReadCostSummary* pCost = &pReader->cost; @@ -4537,15 +4653,15 @@ void tsdbReaderClose2(STsdbReader* pReader) { destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost); taosMemoryFreeClear(pReader->status.uidList.tableUidList); - qTrace("tsdb/reader-close: %p, untake snapshot", pReader); + tsdbTrace("tsdb/reader-close: %p, untake snapshot", pReader); void* p = pReader->pReadSnap; if ((p == atomic_val_compare_exchange_ptr((void**)&pReader->pReadSnap, p, NULL)) && (p != NULL)) { tsdbUntakeReadSnap2(pReader, p, true); } - tsem_destroy(&pReader->resumeAfterSuspend); - tsdbReleaseReader(pReader); - tsdbUninitReaderLock(pReader); + (void) tsem_destroy(&pReader->resumeAfterSuspend); + (void) tsdbReleaseReader(pReader); + (void) tsdbUninitReaderLock(pReader); tsdbDebug( "%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 @@ -4574,11 +4690,14 @@ static int32_t doSuspendCurrentReader(STsdbReader* pCurrentReader) { SReaderStatus* pStatus = &pCurrentReader->status; if (pStatus->loadFromFile) { - tsdbDataFileReaderClose(&pCurrentReader->pFileReader); + (void) tsdbDataFileReaderClose(&pCurrentReader->pFileReader); SReadCostSummary* pCost = &pCurrentReader->cost; destroySttBlockReader(pStatus->pLDataIterArray, &pCost->sttCost); pStatus->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); + if (pStatus->pLDataIterArray == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } // resetDataBlockScanInfo excluding lastKey @@ -4606,14 +4725,14 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { if (pReader->step == EXTERNAL_ROWS_PREV) { - doSuspendCurrentReader(pReader->innerReader[0]); + code = doSuspendCurrentReader(pReader->innerReader[0]); } else if (pReader->step == EXTERNAL_ROWS_MAIN) { - doSuspendCurrentReader(pReader); + code = doSuspendCurrentReader(pReader); } else { - doSuspendCurrentReader(pReader->innerReader[1]); + code = doSuspendCurrentReader(pReader->innerReader[1]); } } else { - doSuspendCurrentReader(pReader); + code = doSuspendCurrentReader(pReader); } // make sure only release once @@ -4648,13 +4767,12 @@ static int32_t tsdbSetQueryReseek(void* pQHandle) { code = tsdbTryAcquireReader(pReader); if (code == 0) { if (pReader->flag == READER_STATUS_SUSPEND) { - tsdbReleaseReader(pReader); + code = tsdbReleaseReader(pReader); return code; } - tsdbReaderSuspend2(pReader); - tsdbReleaseReader(pReader); - + code = tsdbReaderSuspend2(pReader); + (void) tsdbReleaseReader(pReader); return code; } else if (code == EBUSY) { return TSDB_CODE_VND_QUERY_BUSY; @@ -4671,7 +4789,7 @@ int32_t tsdbReaderResume2(STsdbReader* pReader) { // restore reader's state, task snapshot int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); if (numOfTables > 0) { - qTrace("tsdb/reader: %p, take snapshot", pReader); + tsdbTrace("tsdb/reader: %p, take snapshot", pReader); code = tsdbTakeReadSnap2(pReader, tsdbSetQueryReseek, &pReader->pReadSnap); if (code != TSDB_CODE_SUCCESS) { goto _err; @@ -4692,10 +4810,16 @@ int32_t tsdbReaderResume2(STsdbReader* pReader) { // we need only one row pPrevReader->resBlockInfo.capacity = 1; - setSharedPtr(pPrevReader, pReader); + code = setSharedPtr(pPrevReader, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pNextReader->resBlockInfo.capacity = 1; - setSharedPtr(pNextReader, pReader); + code = setSharedPtr(pNextReader, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } if (pReader->step == 0 || pReader->step == EXTERNAL_ROWS_PREV) { code = doOpenReaderImpl(pPrevReader); @@ -4864,12 +4988,17 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { #endif code = tsdbAcquireReader(pReader); - qTrace("tsdb/read: %p, take read mutex, code: %d", pReader, code); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + tsdbTrace("tsdb/read: %p, take read mutex, code: %d", pReader, code); if (pReader->flag == READER_STATUS_SUSPEND) { code = tsdbReaderResume2(pReader); if (code != TSDB_CODE_SUCCESS) { - tsdbReleaseReader(pReader); + // release reader failure should be suppressed here, to avoid over-write the original error code + (void) tsdbReleaseReader(pReader); return code; } } @@ -4877,7 +5006,7 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { if (pReader->innerReader[0] != NULL && pReader->step == 0) { code = doTsdbNextDataBlock2(pReader->innerReader[0], hasNext); if (code) { - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return code; } @@ -4885,8 +5014,8 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { if (*hasNext) { pStatus = &pReader->innerReader[0]->status; if (pStatus->composedDataBlock) { - qTrace("tsdb/read: %p, unlock read mutex", pReader); - tsdbReleaseReader(pReader); + tsdbTrace("tsdb/read: %p, unlock read mutex", pReader); + code = tsdbReleaseReader(pReader); } return code; @@ -4908,14 +5037,14 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { code = doTsdbNextDataBlock2(pReader, hasNext); if (code != TSDB_CODE_SUCCESS) { - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return code; } if (*hasNext) { if (pStatus->composedDataBlock) { - qTrace("tsdb/read: %p, unlock read mutex", pReader); - tsdbReleaseReader(pReader); + tsdbTrace("tsdb/read: %p, unlock read mutex", pReader); + code = tsdbReleaseReader(pReader); } return code; } @@ -4931,7 +5060,7 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { code = doTsdbNextDataBlock2(pReader->innerReader[1], hasNext); if (code != TSDB_CODE_SUCCESS) { - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return code; } @@ -4939,17 +5068,16 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { if (*hasNext) { pStatus = &pReader->innerReader[1]->status; if (pStatus->composedDataBlock) { - qTrace("tsdb/read: %p, unlock read mutex", pReader); - tsdbReleaseReader(pReader); + tsdbTrace("tsdb/read: %p, unlock read mutex", pReader); + code = tsdbReleaseReader(pReader); } return code; } } - qTrace("tsdb/read: %p, unlock read mutex", pReader); - tsdbReleaseReader(pReader); - + tsdbTrace("tsdb/read: %p, unlock read mutex", pReader); + code = tsdbReleaseReader(pReader); return code; } @@ -5136,7 +5264,9 @@ static int32_t doRetrieveDataBlock(STsdbReader* pReader, SSDataBlock** pBlock) { return code; } -SSDataBlock* tsdbRetrieveDataBlock2(STsdbReader* pReader, SArray* pIdList) { +int32_t tsdbRetrieveDataBlock2(STsdbReader* pReader, SSDataBlock** pBlock, SArray* pIdList) { + *pBlock = NULL; + STsdbReader* pTReader = pReader; if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { if (pReader->step == EXTERNAL_ROWS_PREV) { @@ -5151,39 +5281,39 @@ SSDataBlock* tsdbRetrieveDataBlock2(STsdbReader* pReader, SArray* pIdList) { // tsdbReaderSuspend2(pReader); // tsdbReaderResume2(pReader); - return pTReader->resBlockInfo.pResBlock; + *pBlock = pTReader->resBlockInfo.pResBlock; } - SSDataBlock* ret = NULL; - doRetrieveDataBlock(pTReader, &ret); + int32_t code = doRetrieveDataBlock(pTReader, pBlock); - qTrace("tsdb/read-retrieve: %p, unlock read mutex", pReader); - tsdbReleaseReader(pReader); + tsdbTrace("tsdb/read-retrieve: %p, unlock read mutex", pReader); + (void) tsdbReleaseReader(pReader); // tsdbReaderSuspend2(pReader); // tsdbReaderResume2(pReader); - - return ret; + return code; } int32_t tsdbReaderReset2(STsdbReader* pReader, SQueryTableDataCond* pCond) { int32_t code = TSDB_CODE_SUCCESS; - qTrace("tsdb/reader-reset: %p, take read mutex", pReader); - tsdbAcquireReader(pReader); + tsdbTrace("tsdb/reader-reset: %p, take read mutex", pReader); + code = tsdbAcquireReader(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } if (pReader->flag == READER_STATUS_SUSPEND) { code = tsdbReaderResume2(pReader); if (code != TSDB_CODE_SUCCESS) { - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return code; } } if (isEmptyQueryTimeWindow(&pReader->info.window) || pReader->pReadSnap == NULL) { tsdbDebug("tsdb reader reset return %p, %s", pReader->pReadSnap, pReader->idStr); - tsdbReleaseReader(pReader); - return TSDB_CODE_SUCCESS; + return tsdbReleaseReader(pReader); } SReaderStatus* pStatus = &pReader->status; @@ -5199,14 +5329,18 @@ int32_t tsdbReaderReset2(STsdbReader* pReader, SQueryTableDataCond* pCond) { memset(&pReader->suppInfo.tsColAgg, 0, sizeof(SColumnDataAgg)); pReader->suppInfo.tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID; - tsdbDataFileReaderClose(&pReader->pFileReader); + (void) tsdbDataFileReaderClose(&pReader->pFileReader); int32_t numOfTables = tSimpleHashGetSize(pStatus->pTableMap); - initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); + code = initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + code = resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); if (code != TSDB_CODE_SUCCESS) { - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return code; } @@ -5232,7 +5366,7 @@ int32_t tsdbReaderReset2(STsdbReader* pReader, SQueryTableDataCond* pCond) { tsdbError("%p reset reader failed, numOfTables:%d, query range:%" PRId64 " - %" PRId64 " in query %s", pReader, numOfTables, pReader->info.window.skey, pReader->info.window.ekey, pReader->idStr); - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return code; } } @@ -5242,8 +5376,7 @@ int32_t tsdbReaderReset2(STsdbReader* pReader, SQueryTableDataCond* pCond) { pReader, pReader->info.suid, numOfTables, pCond->twindows.skey, pReader->info.window.skey, pReader->info.window.ekey, pReader->idStr); - tsdbReleaseReader(pReader); - + code = tsdbReleaseReader(pReader); return code; } @@ -5267,12 +5400,15 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT pTableBlockInfo->numOfVgroups = 1; // find the start data block in file - tsdbAcquireReader(pReader); + code = tsdbAcquireReader(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (pReader->flag == READER_STATUS_SUSPEND) { code = tsdbReaderResume2(pReader); if (code != TSDB_CODE_SUCCESS) { - tsdbReleaseReader(pReader); - return code; + return tsdbReleaseReader(pReader); } } @@ -5354,7 +5490,7 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT } // record the data in stt files - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return code; } @@ -5411,11 +5547,15 @@ int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader* pReader) { int64_t rows = 0; SReaderStatus* pStatus = &pReader->status; - tsdbAcquireReader(pReader); + code = tsdbAcquireReader(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (pReader->flag == READER_STATUS_SUSPEND) { code = tsdbReaderResume2(pReader); if (code != TSDB_CODE_SUCCESS) { - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return code; } } @@ -5446,7 +5586,7 @@ int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader* pReader) { pStatus->pTableIter = tSimpleHashIterate(pStatus->pTableMap, pStatus->pTableIter, &iter); } - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return rows; } @@ -5493,12 +5633,16 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs SVersionRange* pRange = &pReader->info.verRange; // lock - taosThreadMutexLock(&pTsdb->mutex); + code = taosThreadMutexLock(&pTsdb->mutex); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("failed to lock tsdb, code:%d", tstrerror(code)); + return code; + } // alloc STsdbReadSnap* pSnap = (STsdbReadSnap*)taosMemoryCalloc(1, sizeof(STsdbReadSnap)); if (pSnap == NULL) { - taosThreadMutexUnlock(&pTsdb->mutex); + (void) taosThreadMutexUnlock(&pTsdb->mutex); code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -5508,7 +5652,7 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs pSnap->pMem = pTsdb->mem; pSnap->pNode = taosMemoryMalloc(sizeof(*pSnap->pNode)); if (pSnap->pNode == NULL) { - taosThreadMutexUnlock(&pTsdb->mutex); + (void) taosThreadMutexUnlock(&pTsdb->mutex); code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -5516,14 +5660,14 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs pSnap->pNode->pQHandle = pReader; pSnap->pNode->reseek = reseek; - tsdbRefMemTable(pTsdb->mem, pSnap->pNode); + (void)tsdbRefMemTable(pTsdb->mem, pSnap->pNode); } if (pTsdb->imem && (pRange->minVer <= pTsdb->imem->maxVer && pRange->maxVer >= pTsdb->imem->minVer)) { pSnap->pIMem = pTsdb->imem; pSnap->pINode = taosMemoryMalloc(sizeof(*pSnap->pINode)); if (pSnap->pINode == NULL) { - taosThreadMutexUnlock(&pTsdb->mutex); + (void) taosThreadMutexUnlock(&pTsdb->mutex); code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -5531,14 +5675,14 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs pSnap->pINode->pQHandle = pReader; pSnap->pINode->reseek = reseek; - tsdbRefMemTable(pTsdb->imem, pSnap->pINode); + (void)tsdbRefMemTable(pTsdb->imem, pSnap->pINode); } // fs code = tsdbFSCreateRefSnapshotWithoutLock(pTsdb->pFS, &pSnap->pfSetArray); // unlock - taosThreadMutexUnlock(&pTsdb->mutex); + (void) taosThreadMutexUnlock(&pTsdb->mutex); if (code == TSDB_CODE_SUCCESS) { tsdbTrace("vgId:%d, take read snapshot", TD_VID(pTsdb->pVnode)); @@ -5566,17 +5710,17 @@ void tsdbUntakeReadSnap2(STsdbReader* pReader, STsdbReadSnap* pSnap, bool proact if (pSnap) { if (pSnap->pMem) { - tsdbUnrefMemTable(pSnap->pMem, pSnap->pNode, proactive); + (void) tsdbUnrefMemTable(pSnap->pMem, pSnap->pNode, proactive); } if (pSnap->pIMem) { - tsdbUnrefMemTable(pSnap->pIMem, pSnap->pINode, proactive); + (void) tsdbUnrefMemTable(pSnap->pIMem, pSnap->pINode, proactive); } if (pSnap->pNode) taosMemoryFree(pSnap->pNode); if (pSnap->pINode) taosMemoryFree(pSnap->pINode); - tsdbFSDestroyRefSnapshot(&pSnap->pfSetArray); + (void) tsdbFSDestroyRefSnapshot(&pSnap->pfSetArray); taosMemoryFree(pSnap); } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index 4bca7f73f8..1d330d6deb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -842,9 +842,12 @@ int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileRead return doLoadTombDataFromTombBlk(pBlkArray, pReader, pSttFileReader, false); } -void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver) { +int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver) { if (*ppMemDelData == NULL) { *ppMemDelData = taosArrayInit(4, sizeof(SDelData)); + if (*ppMemDelData == NULL) { + return TSDB_CODE_SUCCESS; + } } SArray* pMemDelData = *ppMemDelData; @@ -872,6 +875,8 @@ void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemT p = p->pNext; } } + + return TSDB_CODE_SUCCESS; } int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo, diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index ba75cb77a8..45a2384b65 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -339,7 +339,7 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3 bool blockIteratorNext(SDataBlockIter* pBlockIter, const char* idStr); // load tomb data API (stt/mem only for one table each, tomb data from data files are load for all tables at one time) -void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver); +int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver); int32_t loadDataFileTombDataForAll(STsdbReader* pReader); int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pLoadInfo); int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo, diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ce4915ca4d..f6c5118c88 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -388,9 +388,10 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pCost->totalCheckedRows += pBlock->info.rows; pCost->loadBlocks += 1; - SSDataBlock* p = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pTableScanInfo->dataReader, NULL); - if (p == NULL) { - return terrno; + SSDataBlock* p = NULL; + int32_t code = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pTableScanInfo->dataReader, &p, NULL); + if (p == NULL || code != TSDB_CODE_SUCCESS) { + return code; } ASSERT(p == pBlock); @@ -1351,7 +1352,12 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU } if (hasNext) { - /*SSDataBlock* p = */ pAPI->tsdReader.tsdReaderRetrieveDataBlock(pReader, NULL); + SSDataBlock* p = NULL; + code = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pReader, &p, NULL); + if (code != TSDB_CODE_SUCCESS) { + return NULL; + } + doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); pBlock->info.id.groupId = tableListGetTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid); } @@ -2929,8 +2935,9 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } - SSDataBlock* pBlock = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pInfo->dataReader, NULL); - if (pBlock == NULL) { + SSDataBlock* pBlock = NULL; + code = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pInfo->dataReader, &pBlock, NULL); + if (pBlock == NULL || code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, terrno); } diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 3d8588a55a..009854b45b 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -74,16 +74,16 @@ typedef struct SRaftStore { TdThreadMutex mutex; } SRaftStore; -typedef struct SSyncHbTimerData { +struct SSyncHbTimerData { int64_t syncNodeRid; SSyncTimer* pTimer; SRaftId destId; uint64_t logicClock; int64_t execTime; int64_t rid; -} SSyncHbTimerData; +}; -typedef struct SSyncTimer { +struct SSyncTimer { void* pTimer; TAOS_TMR_CALLBACK timerCb; uint64_t logicClock; @@ -92,7 +92,7 @@ typedef struct SSyncTimer { int64_t timeStamp; SRaftId destId; int64_t hbDataRid; -} SSyncTimer; +}; typedef struct SElectTimerParam { uint64_t logicClock; @@ -106,7 +106,7 @@ typedef struct SPeerState { int64_t lastSendTime; } SPeerState; -typedef struct SSyncNode { +struct SSyncNode { // init by SSyncInfo SyncGroupId vgId; SRaftCfg raftCfg; @@ -116,7 +116,7 @@ typedef struct SSyncNode { // sync io SSyncLogBuffer* pLogBuf; - SWal* pWal; + struct SWal* pWal; const SMsgCb* msgcb; int32_t (*syncSendMSg)(const SEpSet* pEpSet, SRpcMsg* pMsg); int32_t (*syncEqMsg)(const SMsgCb* msgcb, SRpcMsg* pMsg); @@ -234,7 +234,7 @@ typedef struct SSyncNode { bool isStart; -} SSyncNode; +}; // open/close -------------- SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo, int32_t vnodeVersion); diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index e62ca7d69a..acb9bd20f3 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -103,7 +103,7 @@ typedef void* queue[2]; #define TRANS_MAGIC_NUM 0x5f375a86 #define TRANS_NOVALID_PACKET(src) ((src) != TRANS_MAGIC_NUM ? 1 : 0) -typedef SRpcMsg STransMsg; +typedef struct SRpcMsg STransMsg; typedef SRpcCtx STransCtx; typedef SRpcCtxVal STransCtxVal; typedef SRpcInfo STrans; From 352abe0b5cc05973301077e8a5f1dd869ef695e4 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 16 Jul 2024 19:03:12 +0800 Subject: [PATCH 41/48] fix: scheduler return code --- include/os/osMath.h | 6 +- include/os/osMemory.h | 1 + source/libs/scheduler/inc/schInt.h | 17 ++- source/libs/scheduler/src/schJob.c | 4 +- source/libs/scheduler/src/schRemote.c | 91 +++++++++---- source/libs/scheduler/src/schStatus.c | 7 +- source/libs/scheduler/src/schTask.c | 124 ++++++++++++++---- source/libs/scheduler/src/schUtil.c | 47 ++++--- source/libs/scheduler/src/scheduler.c | 3 +- source/libs/scheduler/test/schedulerTests.cpp | 34 +++-- 10 files changed, 244 insertions(+), 90 deletions(-) diff --git a/include/os/osMath.h b/include/os/osMath.h index 3c05d15397..edbfa935ce 100644 --- a/include/os/osMath.h +++ b/include/os/osMath.h @@ -32,9 +32,9 @@ extern "C" { #define TSWAP(a, b) \ do { \ char *__tmp = (char*)alloca(sizeof(a)); \ - memcpy(__tmp, &(a), sizeof(a)); \ - memcpy(&(a), &(b), sizeof(a)); \ - memcpy(&(b), __tmp, sizeof(a)); \ + (void)memcpy(__tmp, &(a), sizeof(a)); \ + (void)memcpy(&(a), &(b), sizeof(a)); \ + (void)memcpy(&(b), __tmp, sizeof(a)); \ } while (0) #ifdef WINDOWS diff --git a/include/os/osMemory.h b/include/os/osMemory.h index 6166f1dc07..c6a5ce27c4 100644 --- a/include/os/osMemory.h +++ b/include/os/osMemory.h @@ -54,6 +54,7 @@ void taosMemoryTrim(int32_t size); void *taosMemoryMallocAlign(uint32_t alignment, int64_t size); #define TAOS_MEMSET(_s, _c, _n) ((void)memset(_s, _c, _n)) +#define TAOS_MEMCPY(_d, _s, _n) ((void)memcpy(_d, _s, _n)) #define taosMemoryFreeClear(ptr) \ do { \ diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 17476072bb..5ea79c6ae9 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -336,7 +336,7 @@ extern SSchedulerMgmt schMgmt; ((_job)->attr.localExec && SCH_IS_QUERY_JOB(_job) && (!SCH_IS_INSERT_JOB(_job)) && \ (!SCH_IS_DATA_BIND_QRY_TASK(_task))) -#define SCH_UPDATE_REDIRECT_CODE(job, _code) atomic_val_compare_exchange_32(&((job)->redirectCode), 0, _code) +#define SCH_UPDATE_REDIRECT_CODE(job, _code) (void)atomic_val_compare_exchange_32(&((job)->redirectCode), 0, _code) #define SCH_GET_REDIRECT_CODE(job, _code) \ (((!NO_RET_REDIRECT_ERROR(_code)) || (job)->redirectCode == 0) ? (_code) : (job)->redirectCode) @@ -413,7 +413,7 @@ extern SSchedulerMgmt schMgmt; #define SCH_LOG_TASK_START_TS(_task) \ do { \ int64_t us = taosGetTimestampUs(); \ - taosArrayPush((_task)->profile.execTime, &us); \ + (void)taosArrayPush((_task)->profile.execTime, &us); \ if (0 == (_task)->execId) { \ (_task)->profile.startTs = us; \ } \ @@ -422,7 +422,10 @@ extern SSchedulerMgmt schMgmt; #define SCH_LOG_TASK_WAIT_TS(_task) \ do { \ int64_t us = taosGetTimestampUs(); \ - (_task)->profile.waitTime += us - *(int64_t *)taosArrayGet((_task)->profile.execTime, (_task)->execId); \ + int64_t* startus = (int64_t*)taosArrayGet((_task)->profile.execTime, (_task)->execId); \ + if (NULL != startus) { \ + (_task)->profile.waitTime += us - *startus; \ + } \ } while (0) #define SCH_LOG_TASK_END_TS(_task) \ @@ -430,7 +433,9 @@ extern SSchedulerMgmt schMgmt; int64_t us = taosGetTimestampUs(); \ int32_t idx = (_task)->execId % (_task)->maxExecTimes; \ int64_t *startts = taosArrayGet((_task)->profile.execTime, (_task)->execId); \ - *startts = us - *startts; \ + if (NULL != startts) { \ + *startts = us - *startts; \ + } \ (_task)->profile.endTs = us; \ } while (0) @@ -538,7 +543,7 @@ void schCleanClusterHb(void *pTrans); int32_t schLaunchTask(SSchJob *job, SSchTask *task); int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask); int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType, void *param); -SSchJob *schAcquireJob(int64_t refId); +int32_t schAcquireJob(int64_t refId, SSchJob **ppJob); int32_t schReleaseJob(int64_t refId); void schFreeFlowCtrl(SSchJob *pJob); int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel); @@ -578,7 +583,7 @@ int32_t schJobFetchRows(SSchJob *pJob); int32_t schJobFetchRowsA(SSchJob *pJob); int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, void *handle, int32_t execId); int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList); -char *schDumpEpSet(SEpSet *pEpSet); +int32_t schDumpEpSet(SEpSet *pEpSet, char** ppRes); char *schGetOpStr(SCH_OP_TYPE type); int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync); int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq); diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index a8e2e79aee..58a0706223 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -1165,7 +1165,9 @@ int32_t schProcessOnCbBegin(SSchJob **job, SSchTask **task, uint64_t qId, int64_ int8_t status = 0; SSchTask *pTask = NULL; - SSchJob *pJob = schAcquireJob(rId); + SSchJob *pJob = NULL; + + (void)schAcquireJob(rId, &pJob); if (NULL == pJob) { qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, qId, tId, rId); SCH_ERR_RET(TSDB_CODE_QRY_JOB_NOT_EXIST); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index d0dc04d6b4..08a8f684f5 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -113,7 +113,7 @@ int32_t schProcessFetchRsp(SSchJob *pJob, SSchTask *pTask, char *msg, int32_t rs } atomic_store_ptr(&pJob->fetchRes, rsp); - atomic_add_fetch_64(&pJob->resNumOfRows, htobe64(rsp->numOfRows)); + (void)atomic_add_fetch_64(&pJob->resNumOfRows, htobe64(rsp->numOfRows)); if (rsp->completed) { SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCC); @@ -166,13 +166,27 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD SCH_LOCK(SCH_WRITE, &pJob->resLock); if (NULL == pJob->execRes.res) { pJob->execRes.res = (void*)taosArrayInit(batchRsp.nRsps, POINTER_BYTES); + if (NULL == pJob->execRes.res) { + code = terrno; + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + + tDecoderClear(&coder); + SCH_ERR_JRET(code); + } + pJob->execRes.msgType = TDMT_VND_CREATE_TABLE; } for (int32_t i = 0; i < batchRsp.nRsps; ++i) { SVCreateTbRsp *rsp = batchRsp.pRsps + i; if (rsp->pMeta) { - taosArrayPush((SArray*)pJob->execRes.res, &rsp->pMeta); + if (NULL == taosArrayPush((SArray*)pJob->execRes.res, &rsp->pMeta)) { + code = terrno; + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + + tDecoderClear(&coder); + SCH_ERR_JRET(code); + } } if (TSDB_CODE_SUCCESS != rsp->code) { @@ -264,7 +278,7 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD SCH_ERR_JRET(code); } - atomic_add_fetch_64(&pJob->resNumOfRows, rsp->affectedRows); + (void)atomic_add_fetch_64(&pJob->resNumOfRows, rsp->affectedRows); int32_t createTbRspNum = taosArrayGetSize(rsp->aCreateTbRsp); SCH_TASK_DLOG("submit succeed, affectedRows:%d, createTbRspNum:%d", rsp->affectedRows, createTbRspNum); @@ -275,7 +289,12 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD SSubmitRsp2 *sum = pJob->execRes.res; sum->affectedRows += rsp->affectedRows; if (sum->aCreateTbRsp) { - taosArrayAddAll(sum->aCreateTbRsp, rsp->aCreateTbRsp); + if (NULL == taosArrayAddAll(sum->aCreateTbRsp, rsp->aCreateTbRsp)) { + code = terrno; + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + SCH_ERR_JRET(code); + } + taosArrayDestroy(rsp->aCreateTbRsp); } else { TSWAP(sum->aCreateTbRsp, rsp->aCreateTbRsp); @@ -313,10 +332,14 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD SDecoder coder = {0}; SVDeleteRsp rsp = {0}; tDecoderInit(&coder, msg, msgSize); - tDecodeSVDeleteRsp(&coder, &rsp); + if (tDecodeSVDeleteRsp(&coder, &rsp) < 0) { + code = terrno; + tDecoderClear(&coder); + SCH_ERR_JRET(code); + } tDecoderClear(&coder); - atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows); + (void)atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows); SCH_TASK_DLOG("delete succeed, affectedRows:%" PRId64, rsp.affectedRows); } @@ -351,7 +374,7 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD SCH_ERR_JRET(schSaveJobExecRes(pJob, &rsp)); - atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows); + (void)atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows); taosMemoryFreeClear(msg); @@ -479,7 +502,7 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, pParam->taskId, code); // called if drop task rsp received code - rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); + (void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error if (pMsg) { taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); @@ -501,7 +524,7 @@ int32_t schHandleNotifyCallback(void *param, SDataBuf *pMsg, int32_t code) { int32_t schHandleLinkBrokenCallback(void *param, SDataBuf *pMsg, int32_t code) { SSchCallbackParamHeader *head = (SSchCallbackParamHeader *)param; - rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); + (void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error qDebug("handle %p is broken", pMsg->handle); @@ -531,7 +554,7 @@ int32_t schHandleHbCallback(void *param, SDataBuf *pMsg, int32_t code) { if (code) { qError("hb rsp error:%s", tstrerror(code)); - rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); + (void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error SCH_ERR_JRET(code); } @@ -585,9 +608,14 @@ int32_t schMakeCallbackParam(SSchJob *pJob, SSchTask *pTask, int32_t msgType, bo param->head.isHbParam = true; SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + taosMemoryFree(param); + SCH_TASK_ELOG("fail to get the %dth condidateAddr, totalNum: %d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } param->nodeEpId.nodeId = addr->nodeId; SEp *pEp = SCH_GET_CUR_EP(addr); - strcpy(param->nodeEpId.ep.fqdn, pEp->fqdn); + TAOS_STRCPY(param->nodeEpId.ep.fqdn, pEp->fqdn); param->nodeEpId.ep.port = pEp->port; param->pTrans = trans->pTrans; *pParam = param; @@ -712,7 +740,7 @@ int32_t schMakeHbCallbackParam(SSchJob *pJob, SSchTask *pTask, void **pParam) { int32_t schCloneHbRpcCtx(SRpcCtx *pSrc, SRpcCtx *pDst) { int32_t code = 0; - memcpy(pDst, pSrc, sizeof(SRpcCtx)); + TAOS_MEMCPY(pDst, pSrc, sizeof(SRpcCtx)); pDst->brokenVal.val = NULL; pDst->args = NULL; @@ -760,7 +788,7 @@ int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; - memcpy(&epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); + TAOS_MEMCPY(&epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); pCtx->args = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); if (NULL == pCtx->args) { @@ -877,7 +905,7 @@ int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHe SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(dst, pSrc, sizeof(*dst)); + TAOS_MEMCPY(dst, pSrc, sizeof(*dst)); *pDst = (SSchCallbackParamHeader *)dst; return TSDB_CODE_SUCCESS; @@ -889,7 +917,7 @@ int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHe SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(dst, pSrc, sizeof(*dst)); + TAOS_MEMCPY(dst, pSrc, sizeof(*dst)); *pDst = (SSchCallbackParamHeader *)dst; return TSDB_CODE_SUCCESS; @@ -904,7 +932,7 @@ int32_t schCloneSMsgSendInfo(void *src, void **dst) { SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(pDst, pSrc, sizeof(*pSrc)); + TAOS_MEMCPY(pDst, pSrc, sizeof(*pSrc)); pDst->param = NULL; SCH_ERR_JRET(schCloneCallbackParam(pSrc->param, (SSchCallbackParamHeader **)&pDst->param)); @@ -948,6 +976,10 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, SSchTrans *trans, SQuery if (isHb && persistHandle && trans->pHandle == 0) { trans->pHandle = rpcAllocHandle(); + if (NULL == trans->pHandle) { + SCH_TASK_ELOG("rpcAllocHandle failed, code:%x", terrno); + SCH_ERR_JRET(terrno); + } } if (pJob && pTask) { @@ -1000,7 +1032,7 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray *taskAction) { req.header.vgId = nodeEpId->nodeId; req.sId = schMgmt.sId; - memcpy(&req.epId, nodeEpId, sizeof(SQueryNodeEpId)); + TAOS_MEMCPY(&req.epId, nodeEpId, sizeof(SQueryNodeEpId)); SCH_LOCK(SCH_READ, &schMgmt.hbLock); SSchHbTrans *hb = taosHashGet(schMgmt.hbConnections, nodeEpId, sizeof(SQueryNodeEpId)); @@ -1013,7 +1045,7 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray *taskAction) { SCH_LOCK(SCH_WRITE, &hb->lock); code = schCloneHbRpcCtx(&hb->rpcCtx, &rpcCtx); - memcpy(&trans, &hb->trans, sizeof(trans)); + TAOS_MEMCPY(&trans, &hb->trans, sizeof(trans)); SCH_UNLOCK(SCH_WRITE, &hb->lock); SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); @@ -1039,7 +1071,7 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray *taskAction) { SQueryNodeAddr addr = {.nodeId = nodeEpId->nodeId}; addr.epSet.inUse = 0; addr.epSet.numOfEps = 1; - memcpy(&addr.epSet.eps[0], &nodeEpId->ep, sizeof(nodeEpId->ep)); + TAOS_MEMCPY(&addr.epSet.eps[0], &nodeEpId->ep, sizeof(nodeEpId->ep)); code = schAsyncSendMsg(NULL, NULL, &trans, &addr, msgType, msg, msgSize, true, &rpcCtx); msg = NULL; @@ -1064,6 +1096,11 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, if (NULL == addr) { addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + SCH_TASK_ELOG("fail to get condidateAddr, candidateIdx %d, totalNum: %d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_JRET(terrno); + } + isCandidateAddr = true; SCH_TASK_DLOG("target candidateIdx %d, epInUse %d/%d", pTask->candidateIdx, addr->epSet.inUse, addr->epSet.numOfEps); @@ -1082,7 +1119,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(msg, pTask->msg, msgSize); + TAOS_MEMCPY(msg, pTask->msg, msgSize); break; } @@ -1098,13 +1135,21 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, req.msg = pTask->msg; req.source = pJob->source; msgSize = tSerializeSVDeleteReq(NULL, 0, &req); + if (msgSize < 0) { + SCH_TASK_ELOG("tSerializeSVDeleteReq failed, code:%x", terrno); + SCH_ERR_JRET(terrno); + } msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - tSerializeSVDeleteReq(msg, msgSize, &req); + msgSize = tSerializeSVDeleteReq(msg, msgSize, &req); + if (msgSize < 0) { + SCH_TASK_ELOG("tSerializeSVDeleteReq second failed, code:%x", terrno); + SCH_ERR_JRET(terrno); + } break; } case TDMT_SCH_QUERY: @@ -1221,7 +1266,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, req.sId = schMgmt.sId; req.header.vgId = addr->nodeId; req.epId.nodeId = addr->nodeId; - memcpy(&req.epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); + TAOS_MEMCPY(&req.epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); msgSize = tSerializeSSchedulerHbReq(NULL, 0, &req); if (msgSize < 0) { diff --git a/source/libs/scheduler/src/schStatus.c b/source/libs/scheduler/src/schStatus.c index d37393137f..f24ee74101 100644 --- a/source/libs/scheduler/src/schStatus.c +++ b/source/libs/scheduler/src/schStatus.c @@ -43,7 +43,7 @@ int32_t schSwitchJobStatus(SSchJob* pJob, int32_t status, void* param) { SCH_RET(schProcessOnJobFailure(pJob, (param ? *(int32_t*)param : 0))); break; case JOB_TASK_STATUS_DROP: - schProcessOnJobDropped(pJob, *(int32_t*)param); + (void)schProcessOnJobDropped(pJob, *(int32_t*)param); // ignore error if (taosRemoveRef(schMgmt.jobRef, pJob->refId)) { SCH_JOB_ELOG("remove job from job list failed, refId:0x%" PRIx64, pJob->refId); @@ -65,7 +65,8 @@ _return: } int32_t schHandleOpBeginEvent(int64_t jobId, SSchJob** job, SCH_OP_TYPE type, SSchedulerReq* pReq) { - SSchJob* pJob = schAcquireJob(jobId); + SSchJob* pJob = NULL; + (void)schAcquireJob(jobId, &pJob); if (NULL == pJob) { qDebug("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, jobId); SCH_ERR_RET(TSDB_CODE_SCH_JOB_NOT_EXISTS); @@ -90,7 +91,7 @@ int32_t schHandleOpEndEvent(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq code = pJob->errCode; } - schReleaseJob(pJob->refId); + (void)schReleaseJob(pJob->refId); // ignore error return code; } diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index c317a63ce1..6dd6aa9aae 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -165,7 +165,7 @@ int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, v SCH_RET(schDropTaskExecNode(pJob, pTask, handle, execId)); } - schUpdateTaskExecNode(pJob, pTask, handle, execId); + SCH_ERR_RET(schUpdateTaskExecNode(pJob, pTask, handle, execId)); if ((execId != pTask->execId || execId <= pTask->failedExecId) || pTask->waitRetry) { // ignore it SCH_TASK_DLOG("handle not updated since execId %d is already not current execId %d, waitRetry %d", execId, @@ -297,7 +297,11 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { for (int32_t i = 0; i < parentNum; ++i) { SSchTask *parent = *(SSchTask **)taosArrayGet(pTask->parents, i); - + if (NULL == parent) { + SCH_TASK_ELOG("fail to get task %d parent, parentNum: %d", i, parentNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SCH_LOCK(SCH_WRITE, &parent->planLock); SDownstreamSourceNode source = { .type = QUERY_NODE_DOWNSTREAM_SOURCE, @@ -308,9 +312,14 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { .fetchMsgType = SCH_FETCH_TYPE(pTask), .localExec = SCH_IS_LOCAL_EXEC_TASK(pJob, pTask), }; - qSetSubplanExecutionNode(parent->plan, pTask->plan->id.groupId, &source); + code = qSetSubplanExecutionNode(parent->plan, pTask->plan->id.groupId, &source); + if (TSDB_CODE_SUCCESS != code) { + SCH_TASK_ELOG("qSetSubplanExecutionNode failed, groupId: %d", pTask->plan->id.groupId); + } SCH_UNLOCK(SCH_WRITE, &parent->planLock); + SCH_ERR_RET(code); + int32_t readyNum = atomic_add_fetch_32(&parent->childReady, 1); if (SCH_TASK_READY_FOR_LAUNCH(readyNum, parent)) { @@ -413,16 +422,16 @@ void schResetTaskForRetry(SSchJob *pJob, SSchTask *pTask) { schDropTaskOnExecNode(pJob, pTask); if (pTask->delayTimer) { - taosTmrStopA(&pTask->delayTimer); + (void)taosTmrStopA(&pTask->delayTimer); // ignore error } taosHashClear(pTask->execNodes); - schRemoveTaskFromExecList(pJob, pTask); + (void)schRemoveTaskFromExecList(pJob, pTask); // ignore error schDeregisterTaskHb(pJob, pTask); taosMemoryFreeClear(pTask->msg); pTask->msgLen = 0; pTask->lastMsgType = 0; pTask->childReady = 0; - memset(&pTask->succeedAddr, 0, sizeof(pTask->succeedAddr)); + TAOS_MEMSET(&pTask->succeedAddr, 0, sizeof(pTask->succeedAddr)); } int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode) { @@ -443,11 +452,21 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 SCH_ERR_JRET(schUpdateTaskCandidateAddr(pJob, pTask, pData->pEpSet)); } else if (SYNC_SELF_LEADER_REDIRECT_ERROR(rspCode)) { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + SCH_TASK_ELOG("fail to get the %dth condidateAddr, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SEp *pEp = &addr->epSet.eps[addr->epSet.inUse]; SCH_TASK_DLOG("task retry node %d current ep, idx:%d/%d,%s:%d, code:%s", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps, pEp->fqdn, pEp->port, tstrerror(rspCode)); } else { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + SCH_TASK_ELOG("fail to get the %dth condidateAddr, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SCH_SWITCH_EPSET(addr); SCH_TASK_DLOG("switch task target node %d epset to %d/%d", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps); } @@ -476,7 +495,7 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 for (int32_t i = 0; i < childrenNum; ++i) { SSchTask *pChild = taosArrayGetP(pTask->children, i); SCH_LOCK_TASK(pChild); - schDoTaskRedirect(pJob, pChild, NULL, rspCode); + (void)schDoTaskRedirect(pJob, pChild, NULL, rspCode); // error handled internal SCH_UNLOCK_TASK(pChild); } @@ -494,18 +513,23 @@ int32_t schResetTaskSetLevelInfo(SSchJob *pJob, SSchTask *pTask) { atomic_load_32(&pLevel->taskExecDoneNum), atomic_load_32(&pLevel->taskLaunchedNum)); if (SCH_GET_TASK_STATUS(pTask) >= JOB_TASK_STATUS_PART_SUCC) { - atomic_sub_fetch_32(&pLevel->taskExecDoneNum, 1); + (void)atomic_sub_fetch_32(&pLevel->taskExecDoneNum, 1); } - atomic_sub_fetch_32(&pLevel->taskLaunchedNum, 1); + (void)atomic_sub_fetch_32(&pLevel->taskLaunchedNum, 1); int32_t childrenNum = taosArrayGetSize(pTask->children); for (int32_t i = 0; i < childrenNum; ++i) { SSchTask *pChild = taosArrayGetP(pTask->children, i); + if (NULL == pChild) { + SCH_TASK_ELOG("fail to get the %dth child, childrenNum:%d", i, childrenNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SCH_LOCK_TASK(pChild); pLevel = pChild->level; - atomic_sub_fetch_32(&pLevel->taskExecDoneNum, 1); - atomic_sub_fetch_32(&pLevel->taskLaunchedNum, 1); + (void)atomic_sub_fetch_32(&pLevel->taskExecDoneNum, 1); + (void)atomic_sub_fetch_32(&pLevel->taskLaunchedNum, 1); SCH_UNLOCK_TASK(pChild); } @@ -711,9 +735,9 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo } int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { - atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1); + (void)atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1); - schRemoveTaskFromExecList(pJob, pTask); + (void)schRemoveTaskFromExecList(pJob, pTask); // ignore error SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT); if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) { @@ -724,6 +748,11 @@ int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { if (SCH_IS_DATA_BIND_TASK(pTask)) { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + SCH_TASK_ELOG("fail to the %dth condidateAddr, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SCH_SWITCH_EPSET(addr); } else { SCH_ERR_RET(schSwitchTaskCandidateAddr(pJob, pTask)); @@ -743,6 +772,11 @@ int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) { for (int32_t i = 0; i < nodeNum; ++i) { SQueryNodeLoad *nload = taosArrayGet(pJob->nodeList, i); + if (NULL == nload) { + SCH_TASK_ELOG("fail to get the %dth node in nodeList, nodeNum:%d", i, nodeNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SQueryNodeAddr *naddr = &nload->addr; if (NULL == taosArrayPush(pTask->candidateAddrs, naddr)) { @@ -810,6 +844,7 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { } int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet) { + int32_t code = TSDB_CODE_SUCCESS; if (NULL == pTask->candidateAddrs || 1 != taosArrayGetSize(pTask->candidateAddrs)) { SCH_TASK_ELOG("not able to update cndidate addr, addr num %d", (int32_t)(pTask->candidateAddrs ? taosArrayGetSize(pTask->candidateAddrs) : 0)); @@ -817,18 +852,27 @@ int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSe } SQueryNodeAddr *pAddr = taosArrayGet(pTask->candidateAddrs, 0); + if (NULL == pAddr) { + SCH_TASK_ELOG("fail to get task 0th condidataAddr, totalNum:%d", (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } - char *origEpset = schDumpEpSet(&pAddr->epSet); - char *newEpset = schDumpEpSet(pEpSet); + char *origEpset = NULL; + char *newEpset = NULL; + + SCH_ERR_RET(schDumpEpSet(&pAddr->epSet, &origEpset)); + SCH_ERR_JRET(schDumpEpSet(pEpSet, &newEpset)); SCH_TASK_DLOG("update task target node %d epset from %s to %s", pAddr->nodeId, origEpset, newEpset); + TAOS_MEMCPY(&pAddr->epSet, pEpSet, sizeof(pAddr->epSet)); + +_return: + taosMemoryFree(origEpset); taosMemoryFree(newEpset); - memcpy(&pAddr->epSet, pEpSet, sizeof(pAddr->epSet)); - - return TSDB_CODE_SUCCESS; + return code; } int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask) { @@ -877,7 +921,6 @@ void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask) { } int32_t size = (int32_t)taosHashGetSize(pTask->execNodes); - if (size <= 0) { SCH_TASK_DLOG("task has no execNodes, no need to drop it, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); return; @@ -889,7 +932,7 @@ void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask) { if (nodeInfo->handle) { SCH_SET_TASK_HANDLE(pTask, nodeInfo->handle); void *pExecId = taosHashGetKey(nodeInfo, NULL); - schBuildAndSendMsg(pJob, pTask, &nodeInfo->addr, TDMT_SCH_DROP_TASK, pExecId); + (void)schBuildAndSendMsg(pJob, pTask, &nodeInfo->addr, TDMT_SCH_DROP_TASK, pExecId); // ignore error and continue SCH_TASK_DLOG("start to drop task's %dth execNode", i); } else { @@ -939,6 +982,11 @@ int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList) { for (int32_t i = 0; i < taskNum; ++i) { STaskStatus *pStatus = taosArrayGet(pStatusList, i); + if (NULL == pStatus) { + qError("fail to get the %dth task status in hb rsp, taskNum:%d", i, taskNum); + continue; + } + int32_t code = 0; qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d task status in server: %s", pStatus->queryId, pStatus->taskId, @@ -983,10 +1031,15 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { for (int32_t i = 0; i < resNum; ++i) { SExplainLocalRsp *localRsp = taosArrayGet(pExplainRes, i); + if (NULL == localRsp) { + qError("fail to get the %dth LOCAL explain rsp msg, total:%d", i, resNum); + continue; + } qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", begin to handle LOCAL explain rsp msg", localRsp->qId, localRsp->tId); - pJob = schAcquireJob(localRsp->rId); + pJob = NULL; + (void)schAcquireJob(localRsp->rId, &pJob); if (NULL == pJob) { qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, localRsp->qId, localRsp->tId, localRsp->rId); @@ -996,7 +1049,7 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { int8_t status = 0; if (schJobNeedToStop(pJob, &status)) { SCH_TASK_DLOG("will not do further processing cause of job status %s", jobTaskStatusStr(status)); - schReleaseJob(pJob->refId); + (void)schReleaseJob(pJob->refId); SCH_ERR_JRET(TSDB_CODE_SCH_IGNORE_ERROR); } @@ -1006,7 +1059,7 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { code = schProcessExplainRsp(pJob, pTask, &localRsp->rsp); } - schReleaseJob(pJob->refId); + (void)schReleaseJob(pJob->refId); qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", end to handle LOCAL explain rsp msg, code:%x", localRsp->qId, localRsp->tId, code); @@ -1022,6 +1075,11 @@ _return: for (int32_t i = 0; i < resNum; ++i) { SExplainLocalRsp *localRsp = taosArrayGet(pExplainRes, i); + if (NULL == localRsp) { + qError("in _return fail to get the %dth LOCAL explain rsp msg, total:%d", i, resNum); + continue; + } + tFreeSExplainRsp(&localRsp->rsp); } @@ -1076,6 +1134,9 @@ int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { if (SCH_IS_EXPLAIN_JOB(pJob)) { explainRes = taosArrayInit(pJob->taskNum, sizeof(SExplainLocalRsp)); + if (NULL == explainRes) { + SCH_ERR_RET(terrno); + } } SCH_ERR_JRET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, @@ -1097,7 +1158,9 @@ _return: int32_t schLaunchTaskImpl(void *param) { SSchTaskCtx *pCtx = (SSchTaskCtx *)param; - SSchJob *pJob = schAcquireJob(pCtx->jobRid); + SSchJob *pJob = NULL; + + (void)schAcquireJob(pCtx->jobRid, &pJob); if (NULL == pJob) { qDebug("job refId 0x%" PRIx64 " already not exist", pCtx->jobRid); taosMemoryFree(param); @@ -1113,7 +1176,7 @@ int32_t schLaunchTaskImpl(void *param) { int8_t status = 0; int32_t code = 0; - atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1); + (void)atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1); pTask->execId++; pTask->retryTimes++; pTask->waitRetry = false; @@ -1160,7 +1223,7 @@ _return: SCH_UNLOCK_TASK(pTask); } - schReleaseJob(pJob->refId); + (void)schReleaseJob(pJob->refId); taosMemoryFree(param); @@ -1178,7 +1241,7 @@ int32_t schAsyncLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { if (pJob->taskNum >= SCH_MIN_AYSNC_EXEC_NUM) { param->asyncLaunch = true; - taosAsyncExec(schLaunchTaskImpl, param, NULL); + SCH_ERR_RET(taosAsyncExec(schLaunchTaskImpl, param, NULL)); } else { SCH_ERR_RET(schLaunchTaskImpl(param)); } @@ -1252,7 +1315,7 @@ int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } - taosTmrReset(schHandleTimerEvent, pTask->delayExecMs, (void *)param, schMgmt.timer, &pTask->delayTimer); + (void)taosTmrReset(schHandleTimerEvent, pTask->delayExecMs, (void *)param, schMgmt.timer, &pTask->delayTimer); return TSDB_CODE_SUCCESS; } @@ -1283,7 +1346,7 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { SCH_LOCK_TASK(pTask); if (pTask->delayTimer) { - taosTmrStopA(&pTask->delayTimer); + (void)taosTmrStopA(&pTask->delayTimer); } schDropTaskOnExecNode(pJob, pTask); SCH_UNLOCK_TASK(pTask); @@ -1327,6 +1390,9 @@ int32_t schExecLocalFetch(SSchJob *pJob, SSchTask *pTask) { if (SCH_IS_EXPLAIN_JOB(pJob)) { explainRes = taosArrayInit(pJob->taskNum, sizeof(SExplainLocalRsp)); + if (NULL == explainRes) { + SCH_ERR_RET(terrno); + } } SCH_ERR_JRET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 689c98d395..811890dde5 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -22,9 +22,14 @@ #include "tref.h" #include "trpc.h" -FORCE_INLINE SSchJob *schAcquireJob(int64_t refId) { +FORCE_INLINE int32_t schAcquireJob(int64_t refId, SSchJob** ppJob) { qDebug("sch acquire jobId:0x%" PRIx64, refId); - return (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); + *ppJob = (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); + if (NULL == *ppJob) { + return terrno; + } + + return TSDB_CODE_SUCCESS; } FORCE_INLINE int32_t schReleaseJob(int64_t refId) { @@ -36,15 +41,16 @@ FORCE_INLINE int32_t schReleaseJob(int64_t refId) { return taosReleaseRef(schMgmt.jobRef, refId); } -char *schDumpEpSet(SEpSet *pEpSet) { +int32_t schDumpEpSet(SEpSet *pEpSet, char** ppRes) { + *ppRes = NULL; if (NULL == pEpSet) { - return NULL; + return TSDB_CODE_SUCCESS; } int32_t maxSize = 1024; char *str = taosMemoryMalloc(maxSize); if (NULL == str) { - return NULL; + return terrno; } int32_t n = 0; @@ -54,7 +60,8 @@ char *schDumpEpSet(SEpSet *pEpSet) { n += snprintf(str + n, maxSize - n, "[%s:%d]", pEp->fqdn, pEp->port); } - return str; + *ppRes = str; + return TSDB_CODE_SUCCESS; } char *schGetOpStr(SCH_OP_TYPE type) { @@ -73,7 +80,7 @@ char *schGetOpStr(SCH_OP_TYPE type) { } void schFreeHbTrans(SSchHbTrans *pTrans) { - rpcReleaseHandle((void *)pTrans->trans.pHandleId, TAOS_CONN_CLIENT); + (void)rpcReleaseHandle((void *)pTrans->trans.pHandleId, TAOS_CONN_CLIENT); schFreeRpcCtx(&pTrans->rpcCtx); } @@ -86,7 +93,7 @@ void schCleanClusterHb(void *pTrans) { if (hb->trans.pTrans == pTrans) { SQueryNodeEpId *pEpId = taosHashGetKey(hb, NULL); schFreeHbTrans(hb); - taosHashRemove(schMgmt.hbConnections, pEpId, sizeof(SQueryNodeEpId)); + (void)taosHashRemove(schMgmt.hbConnections, pEpId, sizeof(SQueryNodeEpId)); } hb = taosHashIterate(schMgmt.hbConnections, hb); @@ -109,7 +116,7 @@ int32_t schRemoveHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *ep int64_t taskNum = atomic_load_64(&hb->taskNum); if (taskNum <= 0) { schFreeHbTrans(hb); - taosHashRemove(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); + (void)taosHashRemove(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); } SCH_UNLOCK(SCH_WRITE, &schMgmt.hbLock); @@ -165,7 +172,7 @@ int32_t schRegisterHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId * break; } - atomic_add_fetch_64(&hb->taskNum, 1); + (void)atomic_add_fetch_64(&hb->taskNum, 1); SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); @@ -178,12 +185,17 @@ void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask) { } SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + SCH_TASK_ELOG("fail to get the %dth condidateAddr in task, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + return; + } + SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; SEp *pEp = SCH_GET_CUR_EP(addr); - strcpy(epId.ep.fqdn, pEp->fqdn); + TAOS_STRCPY(epId.ep.fqdn, pEp->fqdn); epId.ep.port = pEp->port; SCH_LOCK(SCH_READ, &schMgmt.hbLock); @@ -197,7 +209,7 @@ void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask) { int64_t taskNum = atomic_sub_fetch_64(&hb->taskNum, 1); if (0 == taskNum) { SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); - schRemoveHbConnection(pJob, pTask, &epId); + (void)schRemoveHbConnection(pJob, pTask, &epId); } else { SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); } @@ -211,12 +223,17 @@ int32_t schEnsureHbConnection(SSchJob *pJob, SSchTask *pTask) { } SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + SCH_TASK_ELOG("fail to get the %dth condidateAddr in task, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + return TSDB_CODE_SCH_INTERNAL_ERROR; + } + SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; SEp *pEp = SCH_GET_CUR_EP(addr); - strcpy(epId.ep.fqdn, pEp->fqdn); + TAOS_STRCPY(epId.ep.fqdn, pEp->fqdn); epId.ep.port = pEp->port; SCH_ERR_RET(schRegisterHbConnection(pJob, pTask, &epId)); @@ -240,7 +257,7 @@ int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans) { } SCH_LOCK(SCH_WRITE, &hb->lock); - memcpy(&hb->trans, trans, sizeof(*trans)); + TAOS_MEMCPY(&hb->trans, trans, sizeof(*trans)); SCH_UNLOCK(SCH_WRITE, &hb->lock); SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); @@ -256,7 +273,7 @@ void schCloseJobRef(void) { } if (schMgmt.jobRef >= 0) { - taosCloseRef(schMgmt.jobRef); + (void)taosCloseRef(schMgmt.jobRef); schMgmt.jobRef = -1; } } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 8f85e066cd..45e0ecf738 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -169,7 +169,8 @@ void schedulerFreeJob(int64_t *jobId, int32_t errCode) { return; } - SSchJob *pJob = schAcquireJob(*jobId); + SSchJob *pJob = NULL; + (void)schAcquireJob(*jobId, &pJob); if (NULL == pJob) { qWarn("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId); return; diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 78e876f82c..7bae175ba9 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -431,8 +431,11 @@ void *schtSendRsp(void *param) { taosMsleep(1); } - pJob = schAcquireJob(job); - + code = schAcquireJob(job, &pJob); + if (code) { + return NULL; + } + void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; @@ -457,8 +460,13 @@ void *schtSendRsp(void *param) { void *schtCreateFetchRspThread(void *param) { int64_t job = *(int64_t *)param; - SSchJob *pJob = schAcquireJob(job); + SSchJob *pJob = NULL; + (void)schAcquireJob(job, &pJob); + if (NULL == pJob) { + return NULL; + } + taosSsleep(1); int32_t code = 0; @@ -572,7 +580,9 @@ void *schtRunJobThread(void *aa) { code = schedulerExecJob(&req, &queryJobRefId); assert(code == 0); - pJob = schAcquireJob(queryJobRefId); + pJob = NULL; + code = schAcquireJob(queryJobRefId, &pJob); + if (NULL == pJob) { taosArrayDestroy(qnodeList); schtFreeQueryDag(dag); @@ -728,7 +738,9 @@ TEST(queryTest, normalCase) { code = schedulerExecJob(&req, &job); ASSERT_EQ(code, 0); - SSchJob *pJob = schAcquireJob(job); + SSchJob *pJob = NULL; + code = schAcquireJob(job, &pJob); + ASSERT_EQ(code, 0); void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { @@ -839,8 +851,10 @@ TEST(queryTest, readyFirstCase) { code = schedulerExecJob(&req, &job); ASSERT_EQ(code, 0); - SSchJob *pJob = schAcquireJob(job); - + SSchJob *pJob = NULL; + code = schAcquireJob(job, &pJob); + ASSERT_EQ(code, 0); + void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; @@ -956,7 +970,9 @@ TEST(queryTest, flowCtrlCase) { code = schedulerExecJob(&req, &job); ASSERT_EQ(code, 0); - SSchJob *pJob = schAcquireJob(job); + SSchJob *pJob = NULL; + code = schAcquireJob(job, &pJob); + ASSERT_EQ(code, 0); while (!queryDone) { void *pIter = taosHashIterate(pJob->execTasks, NULL); @@ -1094,7 +1110,7 @@ TEST(otherTest, otherCase) { schReleaseJob(0); schFreeRpcCtx(NULL); - ASSERT_EQ(schDumpEpSet(NULL), (char *)NULL); + ASSERT_EQ(schDumpEpSet(NULL, NULL), TSDB_CODE_SUCCESS); ASSERT_EQ(strcmp(schGetOpStr(SCH_OP_NULL), "NULL"), 0); ASSERT_EQ(strcmp(schGetOpStr((SCH_OP_TYPE)100), "UNKNOWN"), 0); } From b29e1c2b0d7a250e39728cd3a48a5784dfc644e4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 16 Jul 2024 19:13:14 +0800 Subject: [PATCH 42/48] fix(stream):fix syntax error. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 7d1e65ba81..c9c61a990f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -5635,7 +5635,7 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs // lock code = taosThreadMutexLock(&pTsdb->mutex); if (code != TSDB_CODE_SUCCESS) { - tsdbError("failed to lock tsdb, code:%d", tstrerror(code)); + tsdbError("failed to lock tsdb, code:%s", tstrerror(code)); return code; } From 13d330f3c60506d554fac3b91537f67d729ce8a1 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 16 Jul 2024 19:14:21 +0800 Subject: [PATCH 43/48] mem alloc result checking --- source/common/src/cos.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/source/common/src/cos.c b/source/common/src/cos.c index 813f31c6ff..2d3eb3f92b 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -637,6 +637,9 @@ static int32_t s3PutObjectFromFileWithoutCp(S3BucketContext *bucket_context, cha {&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0}; manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *)); + if (!manager.etags) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } manager.next_etags_pos = 0; do { S3_initiate_multipart(bucket_context, object_name, 0, &handler, 0, timeoutMsG, &manager); @@ -740,6 +743,9 @@ static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const SCheckpoint cp = {0}; cp.parts = taosMemoryCalloc(max_part_num, sizeof(SCheckpointPart)); + if (!cp.parts) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } if (taosCheckExistFile(file_cp_path)) { if (!cos_cp_load(file_cp_path, &cp) && cos_cp_is_valid_upload(&cp, contentLength, lmtime)) { @@ -785,6 +791,10 @@ static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const {&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0}; manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *)); + if (!manager.etags) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } + manager.next_etags_pos = 0; upload: @@ -1538,7 +1548,7 @@ bool s3Get(const char *object_name, const char *path) { int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_size, bool check, uint8_t **ppBlock) { (void)check; - int32_t code = 0; + int32_t code = 0, lino = 0; cos_pool_t *p = NULL; int is_cname = 0; cos_status_t *s = NULL; @@ -1591,6 +1601,10 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_ cos_list_for_each_entry(cos_buf_t, content, &download_buffer, node) { len += cos_buf_size(content); } // char *buf = cos_pcalloc(p, (apr_size_t)(len + 1)); char *buf = taosMemoryCalloc(1, (apr_size_t)(len)); + if (!buf) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } + // buf[len] = '\0'; cos_list_for_each_entry(cos_buf_t, content, &download_buffer, node) { size = cos_buf_size(content); @@ -1599,6 +1613,7 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_ } // cos_warn_log("Download data=%s", buf); +_exit: //销毁内存池 cos_pool_destroy(p); From 6fbea461986d82fccd616b6d3d3dedcd35682ea0 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 16 Jul 2024 19:20:36 +0800 Subject: [PATCH 44/48] init cp_body early --- source/common/src/cos_cp.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/common/src/cos_cp.c b/source/common/src/cos_cp.c index 00d3de9500..d7be47ce45 100644 --- a/source/common/src/cos_cp.c +++ b/source/common/src/cos_cp.c @@ -158,6 +158,7 @@ _exit: int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint) { int32_t code = 0, lino = 0; + char* cp_body = NULL; TdFilePtr fd = taosOpenFile(filepath, TD_FILE_READ); if (!fd) { @@ -167,7 +168,7 @@ int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint) { int64_t size = -1; TAOS_CHECK_GOTO(taosStatFile(filepath, &size, NULL, NULL), &lino, _exit); - char* cp_body = taosMemoryMalloc(size + 1); + cp_body = taosMemoryMalloc(size + 1); if (!cp_body) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } From f3cfd297b86f6d3c6b6af34a9badcbc8ee06f512 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 16 Jul 2024 19:42:29 +0800 Subject: [PATCH 45/48] fix(stream): return after get value. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 2 +- tests/script/tsim/insert/update2.sim | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index c9c61a990f..da6e029091 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -5280,8 +5280,8 @@ int32_t tsdbRetrieveDataBlock2(STsdbReader* pReader, SSDataBlock** pBlock, SArra if (pStatus->composedDataBlock || pReader->info.execMode == READER_EXEC_ROWS) { // tsdbReaderSuspend2(pReader); // tsdbReaderResume2(pReader); - *pBlock = pTReader->resBlockInfo.pResBlock; + return TSDB_CODE_SUCCESS; } int32_t code = doRetrieveDataBlock(pTReader, pBlock); diff --git a/tests/script/tsim/insert/update2.sim b/tests/script/tsim/insert/update2.sim index d2b5b05267..8ea767da0e 100644 --- a/tests/script/tsim/insert/update2.sim +++ b/tests/script/tsim/insert/update2.sim @@ -120,7 +120,8 @@ endi if $data01 != 2 then return -1 endi -if $data02 != 1 then +if $data02 != 1 then + print expect 1 , actual $data02 return -1 endi if $data03 != 1 then From 749e7d6cf7707a9dec333538488f7af71a7cdf7f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 17 Jul 2024 00:49:53 +0800 Subject: [PATCH 46/48] fix(stream): return after get value. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 5 ++++- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index da6e029091..e885e2ed6e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -2884,6 +2884,9 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr size_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); SArray* pIndexList = taosArrayInit(numOfTables, sizeof(SBrinBlk)); + if (pIndexList == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } while (1) { // only check here, since the iterate data in memory is very fast. @@ -3382,7 +3385,7 @@ static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) SFileBlockDumpInfo* pDumpInfo = &pStatus->fBlockDumpInfo; int32_t code = getCurrentBlockInfo(pBlockIter, &pBlockInfo); - if (code != TSDB_CODE_SUCCESS) { + if (code == TSDB_CODE_SUCCESS) { pDumpInfo->totalRows = pBlockInfo->numRow; pDumpInfo->rowIndex = ASCENDING_TRAVERSE(pReader->info.order) ? 0 : pBlockInfo->numRow - 1; } else { diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index 1d330d6deb..84f235ab25 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -126,8 +126,8 @@ int32_t getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index, STableBlockScan } int32_t getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, STableBlockScanInfo** pInfo, const char* id) { - *pInfo = tSimpleHashGet(pTableMap, &uid, sizeof(uid)); - if (pInfo == NULL || *pInfo == NULL) { + *pInfo = *(STableBlockScanInfo**)tSimpleHashGet(pTableMap, &uid, sizeof(uid)); + if (pInfo == NULL) { int32_t size = tSimpleHashGetSize(pTableMap); tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id); return TSDB_CODE_INVALID_PARA; From 872cae1386715174946ceba11f0e58587902da15 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 17 Jul 2024 01:08:39 +0800 Subject: [PATCH 47/48] refactor: do some refactor. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index e885e2ed6e..cf2d23cfc8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -898,13 +898,13 @@ static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInf *pInfo = NULL; if (pBlockIter->blockList == NULL) { - return TSDB_CODE_SUCCESS; + return TSDB_CODE_FAILED; } size_t num = TARRAY_SIZE(pBlockIter->blockList); if (num == 0) { ASSERT(pBlockIter->numOfBlocks == num); - return TSDB_CODE_SUCCESS; + return TSDB_CODE_FAILED; } *pInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index); From a62c0fda83aca172d84673bb57966c3867ad7030 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 17 Jul 2024 09:12:42 +0800 Subject: [PATCH 48/48] fix: scheduler UT issues --- source/libs/scheduler/test/schedulerTests.cpp | 190 +++++++++++++++--- 1 file changed, 159 insertions(+), 31 deletions(-) diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 7bae175ba9..f906e3ec3e 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -75,9 +75,7 @@ int32_t schtStartFetch = 0; void schtInitLogFile() { const char *defaultLogFileNamePrefix = "taoslog"; const int32_t maxLogFileNum = 10; - rpcInit(); tsAsyncLog = 0; - rpcInit(); qDebugFlag = 159; strcpy(tsLogDir, TD_LOG_DIR_PATH); @@ -136,8 +134,13 @@ int32_t schtBuildSubmitRspMsg(uint32_t *msize, void **rspMsg) { tEncodeSize(tEncodeSSubmitRsp2, &submitRsp, msgSize, ret); void *msg = taosMemoryCalloc(1, msgSize); + if (NULL == msg) { + return terrno; + } tEncoderInit(&ec, (uint8_t *)msg, msgSize); - tEncodeSSubmitRsp2(&ec, &submitRsp); + if (tEncodeSSubmitRsp2(&ec, &submitRsp) < 0) { + return -1; + } tEncoderClear(&ec); *rspMsg = msg; @@ -152,11 +155,26 @@ void schtBuildQueryDag(SQueryPlan *dag) { dag->queryId = qId; dag->numOfSubplans = 2; dag->pSubplans = nodesMakeList(); + if (NULL == dag->pSubplans) { + return; + } SNodeListNode *scan = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + if (NULL == scan) { + return; + } SNodeListNode *merge = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + if (NULL == merge) { + return; + } SSubplan *scanPlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + if (NULL == scanPlan) { + return; + } SSubplan *mergePlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + if (NULL == mergePlan) { + return; + } scanPlan->id.queryId = qId; scanPlan->id.groupId = 0x0000000000000002; @@ -170,7 +188,13 @@ void schtBuildQueryDag(SQueryPlan *dag) { scanPlan->pChildren = NULL; scanPlan->level = 1; scanPlan->pParents = nodesMakeList(); + if (NULL == scanPlan->pParents) { + return; + } scanPlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); + if (NULL == scanPlan->pNode) { + return; + } scanPlan->msgType = TDMT_SCH_QUERY; mergePlan->id.queryId = qId; @@ -181,21 +205,33 @@ void schtBuildQueryDag(SQueryPlan *dag) { mergePlan->execNode.epSet.numOfEps = 0; mergePlan->pChildren = nodesMakeList(); + if (NULL == mergePlan->pChildren) { + return; + } mergePlan->pParents = NULL; mergePlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_MERGE); + if (NULL == mergePlan->pNode) { + return; + } mergePlan->msgType = TDMT_SCH_QUERY; merge->pNodeList = nodesMakeList(); + if (NULL == merge->pNodeList) { + return; + } scan->pNodeList = nodesMakeList(); + if (NULL == scan->pNodeList) { + return; + } - nodesListAppend(merge->pNodeList, (SNode *)mergePlan); - nodesListAppend(scan->pNodeList, (SNode *)scanPlan); + (void)nodesListAppend(merge->pNodeList, (SNode *)mergePlan); + (void)nodesListAppend(scan->pNodeList, (SNode *)scanPlan); - nodesListAppend(mergePlan->pChildren, (SNode *)scanPlan); - nodesListAppend(scanPlan->pParents, (SNode *)mergePlan); + (void)nodesListAppend(mergePlan->pChildren, (SNode *)scanPlan); + (void)nodesListAppend(scanPlan->pParents, (SNode *)mergePlan); - nodesListAppend(dag->pSubplans, (SNode *)merge); - nodesListAppend(dag->pSubplans, (SNode *)scan); + (void)nodesListAppend(dag->pSubplans, (SNode *)merge); + (void)nodesListAppend(dag->pSubplans, (SNode *)scan); } void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { @@ -205,18 +241,42 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { dag->queryId = qId; dag->numOfSubplans = 2; dag->pSubplans = nodesMakeList(); + if (NULL == dag->pSubplans) { + return; + } SNodeListNode *scan = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + if (NULL == scan) { + return; + } SNodeListNode *merge = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + if (NULL == merge) { + return; + } SSubplan *mergePlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + if (NULL == mergePlan) { + return; + } merge->pNodeList = nodesMakeList(); + if (NULL == merge->pNodeList) { + return; + } scan->pNodeList = nodesMakeList(); + if (NULL == scan->pNodeList) { + return; + } mergePlan->pChildren = nodesMakeList(); + if (NULL == mergePlan->pChildren) { + return; + } for (int32_t i = 0; i < scanPlanNum; ++i) { SSubplan *scanPlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + if (NULL == scanPlan) { + return; + } scanPlan->id.queryId = qId; scanPlan->id.groupId = 0x0000000000000002; scanPlan->id.subplanId = 0x0000000000000003 + i; @@ -233,13 +293,19 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { scanPlan->pChildren = NULL; scanPlan->level = 1; scanPlan->pParents = nodesMakeList(); + if (NULL == scanPlan->pParents) { + return; + } scanPlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); + if (NULL == scanPlan->pNode) { + return; + } scanPlan->msgType = TDMT_SCH_QUERY; - nodesListAppend(scanPlan->pParents, (SNode *)mergePlan); - nodesListAppend(mergePlan->pChildren, (SNode *)scanPlan); + (void)nodesListAppend(scanPlan->pParents, (SNode *)mergePlan); + (void)nodesListAppend(mergePlan->pChildren, (SNode *)scanPlan); - nodesListAppend(scan->pNodeList, (SNode *)scanPlan); + (void)nodesListAppend(scan->pNodeList, (SNode *)scanPlan); } mergePlan->id.queryId = qId; @@ -251,12 +317,15 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { mergePlan->pParents = NULL; mergePlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_MERGE); + if (NULL == mergePlan->pNode) { + return; + } mergePlan->msgType = TDMT_SCH_QUERY; - nodesListAppend(merge->pNodeList, (SNode *)mergePlan); + (void)nodesListAppend(merge->pNodeList, (SNode *)mergePlan); - nodesListAppend(dag->pSubplans, (SNode *)merge); - nodesListAppend(dag->pSubplans, (SNode *)scan); + (void)nodesListAppend(dag->pSubplans, (SNode *)merge); + (void)nodesListAppend(dag->pSubplans, (SNode *)scan); } void schtFreeQueryDag(SQueryPlan *dag) {} @@ -267,10 +336,22 @@ void schtBuildInsertDag(SQueryPlan *dag) { dag->queryId = qId; dag->numOfSubplans = 2; dag->pSubplans = nodesMakeList(); + if (NULL == dag->pSubplans) { + return; + } SNodeListNode *inserta = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + if (NULL == inserta) { + return; + } inserta->pNodeList = nodesMakeList(); + if (NULL == inserta->pNodeList) { + return; + } SSubplan *insertPlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + if (NULL == insertPlan) { + return; + } insertPlan->id.queryId = qId; insertPlan->id.groupId = 0x0000000000000003; @@ -286,13 +367,22 @@ void schtBuildInsertDag(SQueryPlan *dag) { insertPlan->pParents = NULL; insertPlan->pNode = NULL; insertPlan->pDataSink = (SDataSinkNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_INSERT); + if (NULL == insertPlan->pDataSink) { + return; + } ((SDataInserterNode *)insertPlan->pDataSink)->size = 1; ((SDataInserterNode *)insertPlan->pDataSink)->pData = taosMemoryCalloc(1, 1); + if (NULL == ((SDataInserterNode *)insertPlan->pDataSink)->pData) { + return; + } insertPlan->msgType = TDMT_VND_SUBMIT; - nodesListAppend(inserta->pNodeList, (SNode *)insertPlan); + (void)nodesListAppend(inserta->pNodeList, (SNode *)insertPlan); insertPlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + if (NULL == insertPlan) { + return; + } insertPlan->id.queryId = qId; insertPlan->id.groupId = 0x0000000000000003; @@ -308,22 +398,31 @@ void schtBuildInsertDag(SQueryPlan *dag) { insertPlan->pParents = NULL; insertPlan->pNode = NULL; insertPlan->pDataSink = (SDataSinkNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_INSERT); + if (NULL == insertPlan->pDataSink) { + return; + } ((SDataInserterNode *)insertPlan->pDataSink)->size = 1; ((SDataInserterNode *)insertPlan->pDataSink)->pData = taosMemoryCalloc(1, 1); + if (NULL == ((SDataInserterNode *)insertPlan->pDataSink)->pData) { + return; + } insertPlan->msgType = TDMT_VND_SUBMIT; - nodesListAppend(inserta->pNodeList, (SNode *)insertPlan); + (void)nodesListAppend(inserta->pNodeList, (SNode *)insertPlan); - nodesListAppend(dag->pSubplans, (SNode *)inserta); + (void)nodesListAppend(dag->pSubplans, (SNode *)inserta); } int32_t schtPlanToString(const SSubplan *subplan, char **str, int32_t *len) { *str = (char *)taosMemoryCalloc(1, 20); + if (NULL == *str) { + return -1; + } *len = 20; return 0; } -void schtExecNode(SSubplan *subplan, uint64_t groupId, SQueryNodeAddr *ep) {} +int32_t schtExecNode(SSubplan *subplan, uint64_t groupId, SQueryNodeAddr *ep) { return 0; } void schtRpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *pRid) {} @@ -442,16 +541,16 @@ void *schtSendRsp(void *param) { SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildSubmitRspMsg(&msg.len, &rmsg); + (void)schtBuildSubmitRspMsg(&msg.len, &rmsg); msg.msgType = TDMT_VND_SUBMIT_RSP; msg.pData = rmsg; - schHandleResponseMsg(pJob, task, task->execId, &msg, 0); + (void)schHandleResponseMsg(pJob, task, task->execId, &msg, 0); pIter = taosHashIterate(pJob->execTasks, pIter); } - schReleaseJob(job); + (void)schReleaseJob(job); schtJobDone = true; @@ -472,13 +571,13 @@ void *schtCreateFetchRspThread(void *param) { int32_t code = 0; SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildFetchRspMsg(&msg.len, &rmsg); + (void)schtBuildFetchRspMsg(&msg.len, &rmsg); msg.msgType = TDMT_SCH_MERGE_FETCH_RSP; msg.pData = rmsg; code = schHandleResponseMsg(pJob, pJob->fetchTask, pJob->fetchTask->execId, &msg, 0); - schReleaseJob(job); + (void)schReleaseJob(job); assert(code == 0); return NULL; @@ -496,12 +595,17 @@ void *schtFetchRspThread(void *aa) { taosUsleep(100); param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); - + if (NULL == param) { + return NULL; + } param->queryId = schtQueryId; param->taskId = schtFetchTaskId; int32_t code = 0; SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp)); + if (NULL == rsp) { + return NULL; + } rsp->completed = 1; rsp->numOfRows = 10; @@ -557,12 +661,17 @@ void *schtRunJobThread(void *aa) { schtBuildQueryDag(dag); SArray *qnodeList = taosArrayInit(1, sizeof(SQueryNodeLoad)); - + if (NULL == qnodeList) { + assert(0); + } + SQueryNodeLoad load = {0}; load.addr.epSet.numOfEps = 1; strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); load.addr.epSet.eps[0].port = 6031; - taosArrayPush(qnodeList, &load); + if (NULL == taosArrayPush(qnodeList, &load)) { + assert(0); + } queryDone = 0; @@ -590,16 +699,24 @@ void *schtRunJobThread(void *aa) { } execTasks = taosHashInit(5, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + if (NULL == execTasks) { + assert(0); + } void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; schtFetchTaskId = task->taskId - 1; - taosHashPut(execTasks, &task->taskId, sizeof(task->taskId), task, sizeof(*task)); + if (taosHashPut(execTasks, &task->taskId, sizeof(task->taskId), task, sizeof(*task))) { + assert(0); + } pIter = taosHashIterate(pJob->execTasks, pIter); } param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); + if (NULL == param) { + assert(0); + } param->refId = queryJobRefId; param->queryId = pJob->queryId; @@ -611,7 +728,9 @@ void *schtRunJobThread(void *aa) { SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildQueryRspMsg(&msg.len, &rmsg); + if (schtBuildQueryRspMsg(&msg.len, &rmsg)) { + assert(0); + } msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; @@ -622,6 +741,9 @@ void *schtRunJobThread(void *aa) { } param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); + if (NULL == param) { + assert(0); + } param->refId = queryJobRefId; param->queryId = pJob->queryId; @@ -632,7 +754,9 @@ void *schtRunJobThread(void *aa) { param->taskId = task->taskId - 1; SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildQueryRspMsg(&msg.len, &rmsg); + if (schtBuildQueryRspMsg(&msg.len, &rmsg)) { + assert(0); + } msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; @@ -1110,13 +1234,17 @@ TEST(otherTest, otherCase) { schReleaseJob(0); schFreeRpcCtx(NULL); - ASSERT_EQ(schDumpEpSet(NULL, NULL), TSDB_CODE_SUCCESS); + char* ep = NULL; + ASSERT_EQ(schDumpEpSet(NULL, &ep), TSDB_CODE_SUCCESS); ASSERT_EQ(strcmp(schGetOpStr(SCH_OP_NULL), "NULL"), 0); ASSERT_EQ(strcmp(schGetOpStr((SCH_OP_TYPE)100), "UNKNOWN"), 0); } int main(int argc, char **argv) { schtInitLogFile(); + if (rpcInit()) { + assert(0); + } taosSeedRand(taosGetTimestampSec()); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS();