From 5b982354e1ecf0cb2510e752f291f27035d684f7 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Fri, 12 Jul 2024 15:30:03 +0800 Subject: [PATCH 01/34] adj bloomfilter --- include/libs/executor/storageapi.h | 2 +- include/libs/stream/tstreamUpdate.h | 2 +- include/util/tbloomfilter.h | 20 ++--- include/util/tscalablebf.h | 2 +- include/util/tutil.h | 1 + source/libs/stream/src/streamUpdate.c | 29 +++++-- source/util/src/tbloomfilter.c | 69 ++++++++++++---- source/util/src/tscalablebf.c | 108 +++++++++++++++++++------- source/util/test/bloomFilterTest.cpp | 22 ++++-- 9 files changed, 186 insertions(+), 69 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 45f9f73fb1..45424cfb66 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -389,7 +389,7 @@ typedef struct SStateStore { void (*updateInfoDestroy)(SUpdateInfo* pInfo); void (*windowSBfDelete)(SUpdateInfo* pInfo, uint64_t count); - void (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count); + int32_t (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count); SUpdateInfo* (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo); diff --git a/include/libs/stream/tstreamUpdate.h b/include/libs/stream/tstreamUpdate.h index 632a82743f..726f8443f1 100644 --- a/include/libs/stream/tstreamUpdate.h +++ b/include/libs/stream/tstreamUpdate.h @@ -36,7 +36,7 @@ void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo); int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo); int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo); void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count); -void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count); +int32_t windowSBfAdd(SUpdateInfo *pInfo, uint64_t count); bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); #ifdef __cplusplus diff --git a/include/util/tbloomfilter.h b/include/util/tbloomfilter.h index 23bf5aaafb..2a3da90dad 100644 --- a/include/util/tbloomfilter.h +++ b/include/util/tbloomfilter.h @@ -19,6 +19,8 @@ #include "os.h" #include "tencode.h" #include "thash.h" +#include "tlog.h" +#include "tutil.h" #ifdef __cplusplus extern "C" { @@ -39,15 +41,15 @@ typedef struct SBloomFilter { double errorRate; } SBloomFilter; -SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate); -int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2); -int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len); -int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t h1, uint64_t h2); -void tBloomFilterDestroy(SBloomFilter *pBF); -void tBloomFilterDump(const SBloomFilter *pBF); -bool tBloomFilterIsFull(const SBloomFilter *pBF); -int32_t tBloomFilterEncode(const SBloomFilter *pBF, SEncoder *pEncoder); -SBloomFilter *tBloomFilterDecode(SDecoder *pDecoder); +int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilter **ppBF); +int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2); +int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len); +int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t h1, uint64_t h2); +void tBloomFilterDestroy(SBloomFilter *pBF); +void tBloomFilterDump(const SBloomFilter *pBF); +bool tBloomFilterIsFull(const SBloomFilter *pBF); +int32_t tBloomFilterEncode(const SBloomFilter *pBF, SEncoder *pEncoder); +int32_t tBloomFilterDecode(SDecoder *pDecoder, SBloomFilter **ppBF); #ifdef __cplusplus } diff --git a/include/util/tscalablebf.h b/include/util/tscalablebf.h index d3ce2eb23b..5e1efb6b25 100644 --- a/include/util/tscalablebf.h +++ b/include/util/tscalablebf.h @@ -32,7 +32,7 @@ typedef struct SScalableBf { _hash_fn_t hashFn2; } SScalableBf; -SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate); +int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf **ppSBf); int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len); int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len); int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len); diff --git a/include/util/tutil.h b/include/util/tutil.h index d1a18dc3e8..33a53f79b5 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -21,6 +21,7 @@ #include "tdef.h" #include "thash.h" #include "tmd5.h" +#include "tutil.h" #ifdef __cplusplus extern "C" { diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 0adb7050c6..e41c5cfffc 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -66,15 +66,29 @@ int32_t getValueBuff(TSKEY ts, char* pVal, int32_t len, char* buff) { return sizeof(TSKEY) + len; } -void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) { +int32_t windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (pInfo->numSBFs < count) { count = pInfo->numSBFs; } for (uint64_t i = 0; i < count; ++i) { int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND); - SScalableBf *tsSBF = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE); - taosArrayPush(pInfo->pTsSBFs, &tsSBF); + SScalableBf *tsSBF = NULL; + code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &tsSBF); + TSDB_CHECK_CODE(code, lino, _error); + void *res = taosArrayPush(pInfo->pTsSBFs, &tsSBF); + if (!res) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } } + +_error: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void clearItemHelper(void *p) { @@ -183,6 +197,7 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma } static SScalableBf *getSBf(SUpdateInfo *pInfo, TSKEY ts) { + int32_t code = 0; if (ts <= 0) { return NULL; } @@ -202,7 +217,7 @@ static SScalableBf *getSBf(SUpdateInfo *pInfo, TSKEY ts) { SScalableBf *res = taosArrayGetP(pInfo->pTsSBFs, index); if (res == NULL) { int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND); - res = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE); + code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &res); taosArrayPush(pInfo->pTsSBFs, &res); } return res; @@ -334,7 +349,11 @@ void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo) { return; } int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND); - pInfo->pCloseWinSBF = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE); + int32_t code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &pInfo->pCloseWinSBF); + if (code != TSDB_CODE_SUCCESS) { + pInfo->pCloseWinSBF = NULL; + uError("%s failed to add close window SBF since %s", __func__, tstrerror(code)); + } } void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo) { diff --git a/source/util/src/tbloomfilter.c b/source/util/src/tbloomfilter.c index 150faab571..557e12a5b2 100644 --- a/source/util/src/tbloomfilter.c +++ b/source/util/src/tbloomfilter.c @@ -35,13 +35,17 @@ static FORCE_INLINE bool getBit(uint64_t *buf, uint64_t index) { return buf[unitIndex] & mask; } -SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate) { +int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilter **ppBF) { + int32_t code = 0; + int32_t lino = 0; if (expectedEntries < 1 || errorRate <= 0 || errorRate >= 1.0) { - return NULL; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } SBloomFilter *pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); if (pBF == NULL) { - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } pBF->expectedEntries = expectedEntries; pBF->errorRate = errorRate; @@ -61,9 +65,16 @@ SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate) { pBF->buffer = taosMemoryCalloc(pBF->numUnits, sizeof(uint64_t)); if (pBF->buffer == NULL) { tBloomFilterDestroy(pBF); - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } - return pBF; + (*ppBF) = pBF; + +_error: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2) { @@ -133,27 +144,55 @@ int32_t tBloomFilterEncode(const SBloomFilter *pBF, SEncoder *pEncoder) { return 0; } -SBloomFilter *tBloomFilterDecode(SDecoder *pDecoder) { +int32_t tBloomFilterDecode(SDecoder *pDecoder, SBloomFilter **ppBF) { + int32_t code = 0; + int32_t lino = 0; SBloomFilter *pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); + if (!pBF) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } pBF->buffer = NULL; - if (tDecodeU32(pDecoder, &pBF->hashFunctions) < 0) goto _error; - if (tDecodeU64(pDecoder, &pBF->expectedEntries) < 0) goto _error; - if (tDecodeU64(pDecoder, &pBF->numUnits) < 0) goto _error; - if (tDecodeU64(pDecoder, &pBF->numBits) < 0) goto _error; - if (tDecodeU64(pDecoder, &pBF->size) < 0) goto _error; + if (tDecodeU32(pDecoder, &pBF->hashFunctions) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeU64(pDecoder, &pBF->expectedEntries) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeU64(pDecoder, &pBF->numUnits) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeU64(pDecoder, &pBF->numBits) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeU64(pDecoder, &pBF->size) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } pBF->buffer = taosMemoryCalloc(pBF->numUnits, sizeof(uint64_t)); for (int32_t i = 0; i < pBF->numUnits; i++) { uint64_t *pUnits = (uint64_t *)pBF->buffer; - if (tDecodeU64(pDecoder, pUnits + i) < 0) goto _error; + if (tDecodeU64(pDecoder, pUnits + i) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + } + if (tDecodeDouble(pDecoder, &pBF->errorRate) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } - if (tDecodeDouble(pDecoder, &pBF->errorRate) < 0) goto _error; pBF->hashFn1 = HASH_FUNCTION_1; pBF->hashFn2 = HASH_FUNCTION_2; - return pBF; + (*ppBF) = pBF; _error: tBloomFilterDestroy(pBF); - return NULL; + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + return TSDB_CODE_FAILED; } bool tBloomFilterIsFull(const SBloomFilter *pBF) { return pBF->size >= pBF->expectedEntries; } diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index 7af794546b..b996cf21bb 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -17,6 +17,7 @@ #include "tscalablebf.h" #include "taoserror.h" +#include "tutil.h" #define DEFAULT_GROWTH 2 #define DEFAULT_TIGHTENING_RATIO 0.5 @@ -24,52 +25,82 @@ #define SBF_INVALID -1 #define SBF_VALID 0 -static SBloomFilter *tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate); +static int32_t tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate, SBloomFilter **ppNormalBf); -SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate) { +int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf **ppSBf) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; const uint32_t defaultSize = 8; if (expectedEntries < 1 || errorRate <= 0 || errorRate >= 1.0) { - return NULL; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); if (pSBf == NULL) { - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } pSBf->maxBloomFilters = DEFAULT_MAX_BLOOMFILTERS; pSBf->status = SBF_VALID; pSBf->numBits = 0; pSBf->bfArray = taosArrayInit(defaultSize, sizeof(void *)); - if (tScalableBfAddFilter(pSBf, expectedEntries, errorRate * DEFAULT_TIGHTENING_RATIO) == NULL) { + if (!pSBf->bfArray) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } + + SBloomFilter *pNormalBf = NULL; + code = tScalableBfAddFilter(pSBf, expectedEntries, errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf); + if (code != TSDB_CODE_SUCCESS) { tScalableBfDestroy(pSBf); - return NULL; + TSDB_CHECK_CODE(code, lino, _error); } pSBf->growth = DEFAULT_GROWTH; pSBf->hashFn1 = HASH_FUNCTION_1; pSBf->hashFn2 = HASH_FUNCTION_2; - return pSBf; + (*ppSBf) = pSBf; + return TSDB_CODE_SUCCESS; + +_error: + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + return code; } int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (pSBf->status == SBF_INVALID) { - return TSDB_CODE_FAILED; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } - int32_t size = taosArrayGetSize(pSBf->bfArray); + int32_t size = taosArrayGetSize(pSBf->bfArray); SBloomFilter *pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); - ASSERT(pNormalBf); + if (!pNormalBf) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } if (tBloomFilterIsFull(pNormalBf)) { - pNormalBf = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, - pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO); - if (pNormalBf == NULL) { + code = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, + pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf); + if (code != TSDB_CODE_SUCCESS) { pSBf->status = SBF_INVALID; - return TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } } return tBloomFilterPut(pNormalBf, keyBuf, len); + +_error: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (pSBf->status == SBF_INVALID) { - return TSDB_CODE_FAILED; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len); uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len); @@ -83,14 +114,20 @@ int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { SBloomFilter *pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); ASSERT(pNormalBf); if (tBloomFilterIsFull(pNormalBf)) { - pNormalBf = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, - pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO); - if (pNormalBf == NULL) { + code = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, + pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf); + if (code != TSDB_CODE_SUCCESS) { pSBf->status = SBF_INVALID; - return TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } } return tBloomFilterPutHash(pNormalBf, h1, h2); + +_error: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len) { @@ -108,21 +145,31 @@ int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32 return TSDB_CODE_SUCCESS; } -static SBloomFilter *tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate) { +static int32_t tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate, SBloomFilter **ppNormalBf) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (taosArrayGetSize(pSBf->bfArray) >= pSBf->maxBloomFilters) { - return NULL; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } - SBloomFilter *pNormalBf = tBloomFilterInit(expectedEntries, errorRate); - if (pNormalBf == NULL) { - return NULL; - } + SBloomFilter *pNormalBf = NULL; + code = tBloomFilterInit(expectedEntries, errorRate, &pNormalBf); + TSDB_CHECK_CODE(code, lino, _error); + if (taosArrayPush(pSBf->bfArray, &pNormalBf) == NULL) { tBloomFilterDestroy(pNormalBf); - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } pSBf->numBits += pNormalBf->numBits; - return pNormalBf; + (*ppNormalBf) = pNormalBf; + +_error: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void tScalableBfDestroy(SScalableBf *pSBf) { @@ -154,6 +201,8 @@ int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) { } SScalableBf *tScalableBfDecode(SDecoder *pDecoder) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); pSBf->hashFn1 = HASH_FUNCTION_1; pSBf->hashFn2 = HASH_FUNCTION_2; @@ -166,8 +215,9 @@ SScalableBf *tScalableBfDecode(SDecoder *pDecoder) { } pSBf->bfArray = taosArrayInit(size * 2, sizeof(void *)); for (int32_t i = 0; i < size; i++) { - SBloomFilter *pBF = tBloomFilterDecode(pDecoder); - if (!pBF) goto _error; + SBloomFilter *pBF = NULL; + code = tBloomFilterDecode(pDecoder, &pBF); + TSDB_CHECK_CODE(code, lino, _error); taosArrayPush(pSBf->bfArray, &pBF); } if (tDecodeU32(pDecoder, &pSBf->growth) < 0) goto _error; diff --git a/source/util/test/bloomFilterTest.cpp b/source/util/test/bloomFilterTest.cpp index c51de3c8a4..7832527bf1 100644 --- a/source/util/test/bloomFilterTest.cpp +++ b/source/util/test/bloomFilterTest.cpp @@ -8,12 +8,15 @@ using namespace std; TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { int64_t ts1 = 1650803518000; - GTEST_ASSERT_EQ(NULL, tBloomFilterInit(100, 0)); - GTEST_ASSERT_EQ(NULL, tBloomFilterInit(100, 1)); - GTEST_ASSERT_EQ(NULL, tBloomFilterInit(100, -0.1)); - GTEST_ASSERT_EQ(NULL, tBloomFilterInit(0, 0.01)); + SBloomFilter *pBFTmp = NULL; + GTEST_ASSERT_EQ(0, tBloomFilterInit(100, 0, &pBFTmp)); + GTEST_ASSERT_EQ(0, tBloomFilterInit(100, 1, &pBFTmp)); + GTEST_ASSERT_EQ(0, tBloomFilterInit(100, -0.1, &pBFTmp)); + GTEST_ASSERT_EQ(0, tBloomFilterInit(0, 0.01, &pBFTmp)); - SBloomFilter *pBF1 = tBloomFilterInit(100, 0.005); + SBloomFilter *pBF1 = NULL, + int32_t code = tBloomFilterInit(100, 0.005, &pBF1); + GTEST_ASSERT_EQ(0,code); GTEST_ASSERT_EQ(pBF1->numBits, 1152); GTEST_ASSERT_EQ(pBF1->numUnits, 1152 / 64); int64_t count = 0; @@ -25,16 +28,19 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { } ASSERT_TRUE(tBloomFilterIsFull(pBF1)); - SBloomFilter *pBF2 = tBloomFilterInit(1000 * 10000, 0.1); + SBloomFilter *pBF2 = NULL; + GTEST_ASSERT_EQ(0, tBloomFilterInit(1000 * 10000, 0.1, &pBF2)); GTEST_ASSERT_EQ(pBF2->numBits, 47925312); GTEST_ASSERT_EQ(pBF2->numUnits, 47925312 / 64); - SBloomFilter *pBF3 = tBloomFilterInit(10000 * 10000, 0.001); + SBloomFilter *pBF3 = NULL; + GTEST_ASSERT_EQ(0, tBloomFilterInit(10000 * 10000, 0.001)); GTEST_ASSERT_EQ(pBF3->numBits, 1437758784); GTEST_ASSERT_EQ(pBF3->numUnits, 1437758784 / 64); int64_t size = 10000; - SBloomFilter *pBF4 = tBloomFilterInit(size, 0.001); + SBloomFilter *pBF4 = NULL; + GTEST_ASSERT_EQ(0, tBloomFilterInit(size, 0.001)); for (int64_t i = 0; i < 1000; i++) { int64_t ts = i + ts1; GTEST_ASSERT_EQ(tBloomFilterPut(pBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS); From 3b914248a52a7a0b2c862a0f2464374ef7b9b789 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Fri, 12 Jul 2024 15:52:54 +0800 Subject: [PATCH 02/34] adj bloomfilter --- include/util/tscalablebf.h | 14 ++--- source/libs/stream/src/streamUpdate.c | 19 +++++-- source/util/src/tscalablebf.c | 75 +++++++++++++++++++-------- 3 files changed, 74 insertions(+), 34 deletions(-) diff --git a/include/util/tscalablebf.h b/include/util/tscalablebf.h index 5e1efb6b25..584e023bb0 100644 --- a/include/util/tscalablebf.h +++ b/include/util/tscalablebf.h @@ -32,13 +32,13 @@ typedef struct SScalableBf { _hash_fn_t hashFn2; } SScalableBf; -int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf **ppSBf); -int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len); -int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len); -int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len); -void tScalableBfDestroy(SScalableBf *pSBf); -int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder); -SScalableBf *tScalableBfDecode(SDecoder *pDecoder); +int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf **ppSBf); +int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len); +int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len); +int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len); +void tScalableBfDestroy(SScalableBf *pSBf); +int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder); +int32_t tScalableBfDecode(SDecoder *pDecoder, SScalableBf **ppSBf); #ifdef __cplusplus } diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index e41c5cfffc..2da20cadae 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -420,6 +420,8 @@ int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo) } int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; ASSERT(pInfo); SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -440,8 +442,10 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { if (tDecodeI32(&decoder, &sBfSize) < 0) return -1; pInfo->pTsSBFs = taosArrayInit(sBfSize, sizeof(void *)); for (int32_t i = 0; i < sBfSize; i++) { - SScalableBf *pSBf = tScalableBfDecode(&decoder); - if (!pSBf) return -1; + SScalableBf *pSBf = NULL; + code = tScalableBfDecode(&decoder, &pSBf); + TSDB_CHECK_CODE(code, lino, _error); + taosArrayPush(pInfo->pTsSBFs, &pSBf); } @@ -449,7 +453,9 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { if (tDecodeI64(&decoder, &pInfo->interval) < 0) return -1; if (tDecodeI64(&decoder, &pInfo->watermark) < 0) return -1; if (tDecodeI64(&decoder, &pInfo->minTS) < 0) return -1; - pInfo->pCloseWinSBF = tScalableBfDecode(&decoder); + pInfo->pCloseWinSBF = NULL; + code = tScalableBfDecode(&decoder, &pInfo->pCloseWinSBF); + TSDB_CHECK_CODE(code, lino, _error); int32_t mapSize = 0; if (tDecodeI32(&decoder, &mapSize) < 0) return -1; @@ -482,7 +488,12 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { tEndDecode(&decoder); tDecoderClear(&decoder); - return 0; + +_error: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) { diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index b996cf21bb..1ee99b07f7 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -25,11 +25,12 @@ #define SBF_INVALID -1 #define SBF_VALID 0 -static int32_t tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate, SBloomFilter **ppNormalBf); +static int32_t tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate, + SBloomFilter **ppNormalBf); int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf **ppSBf) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t lino = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; const uint32_t defaultSize = 8; if (expectedEntries < 1 || errorRate <= 0 || errorRate >= 1.0) { code = TSDB_CODE_FAILED; @@ -47,7 +48,7 @@ int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _error); } - + SBloomFilter *pNormalBf = NULL; code = tScalableBfAddFilter(pSBf, expectedEntries, errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf); if (code != TSDB_CODE_SUCCESS) { @@ -104,7 +105,7 @@ int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { } uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len); uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len); - int32_t size = taosArrayGetSize(pSBf->bfArray); + int32_t size = taosArrayGetSize(pSBf->bfArray); for (int32_t i = size - 2; i >= 0; --i) { if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i), h1, h2) != TSDB_CODE_SUCCESS) { return TSDB_CODE_FAILED; @@ -115,7 +116,7 @@ int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { ASSERT(pNormalBf); if (tBloomFilterIsFull(pNormalBf)) { code = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, - pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf); + pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf); if (code != TSDB_CODE_SUCCESS) { pSBf->status = SBF_INVALID; TSDB_CHECK_CODE(code, lino, _error); @@ -136,7 +137,7 @@ int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32 } uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len); uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len); - int32_t size = taosArrayGetSize(pSBf->bfArray); + int32_t size = taosArrayGetSize(pSBf->bfArray); for (int32_t i = size - 1; i >= 0; --i) { if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i), h1, h2) != TSDB_CODE_SUCCESS) { return TSDB_CODE_FAILED; @@ -145,7 +146,8 @@ int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32 return TSDB_CODE_SUCCESS; } -static int32_t tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate, SBloomFilter **ppNormalBf) { +static int32_t tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate, + SBloomFilter **ppNormalBf) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; if (taosArrayGetSize(pSBf->bfArray) >= pSBf->maxBloomFilters) { @@ -163,7 +165,7 @@ static int32_t tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, TSDB_CHECK_CODE(code, lino, _error); } pSBf->numBits += pNormalBf->numBits; - (*ppNormalBf) = pNormalBf; + (*ppNormalBf) = pNormalBf; _error: if (code != TSDB_CODE_SUCCESS) { @@ -200,33 +202,60 @@ int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) { return 0; } -SScalableBf *tScalableBfDecode(SDecoder *pDecoder) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t lino = 0; +int32_t tScalableBfDecode(SDecoder *pDecoder, SScalableBf **ppSBf) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); + if (!pSBf) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } pSBf->hashFn1 = HASH_FUNCTION_1; pSBf->hashFn2 = HASH_FUNCTION_2; pSBf->bfArray = NULL; int32_t size = 0; - if (tDecodeI32(pDecoder, &size) < 0) goto _error; - if (size == 0) { - tScalableBfDestroy(pSBf); - return NULL; + if (tDecodeI32(pDecoder, &size) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } - pSBf->bfArray = taosArrayInit(size * 2, sizeof(void *)); + if (size == 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + pSBf->bfArray = taosArrayInit(size * 2, POINTER_BYTES); + if (!pSBf->bfArray) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } + for (int32_t i = 0; i < size; i++) { SBloomFilter *pBF = NULL; code = tBloomFilterDecode(pDecoder, &pBF); TSDB_CHECK_CODE(code, lino, _error); taosArrayPush(pSBf->bfArray, &pBF); } - if (tDecodeU32(pDecoder, &pSBf->growth) < 0) goto _error; - if (tDecodeU64(pDecoder, &pSBf->numBits) < 0) goto _error; - if (tDecodeU32(pDecoder, &pSBf->maxBloomFilters) < 0) goto _error; - if (tDecodeI8(pDecoder, &pSBf->status) < 0) goto _error; - return pSBf; + if (tDecodeU32(pDecoder, &pSBf->growth) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeU64(pDecoder, &pSBf->numBits) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeU32(pDecoder, &pSBf->maxBloomFilters) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + if (tDecodeI8(pDecoder, &pSBf->status) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); + } + (*ppSBf) = pSBf; _error: tScalableBfDestroy(pSBf); - return NULL; + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } From 89a62c526e682db508dba53313fdb221af4e4f17 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 15 Jul 2024 09:50:12 +0800 Subject: [PATCH 03/34] adj return --- source/util/src/tscalablebf.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index 1ee99b07f7..13e9fe79e8 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -38,7 +38,8 @@ int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf } SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); if (pSBf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } pSBf->maxBloomFilters = DEFAULT_MAX_BLOOMFILTERS; pSBf->status = SBF_VALID; From 60b268bbbe1901797a21eb1b5666b84860f9220c Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 15 Jul 2024 09:54:41 +0800 Subject: [PATCH 04/34] adj test --- source/util/test/bloomFilterTest.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/source/util/test/bloomFilterTest.cpp b/source/util/test/bloomFilterTest.cpp index 7832527bf1..2cd4edd519 100644 --- a/source/util/test/bloomFilterTest.cpp +++ b/source/util/test/bloomFilterTest.cpp @@ -70,12 +70,14 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) { int64_t ts1 = 1650803518000; - GTEST_ASSERT_EQ(NULL, tScalableBfInit(100, 0)); - GTEST_ASSERT_EQ(NULL, tScalableBfInit(100, 1)); - GTEST_ASSERT_EQ(NULL, tScalableBfInit(100, -0.1)); - GTEST_ASSERT_EQ(NULL, tScalableBfInit(0, 0.01)); + SScalableBf *tsSBF = NULL; + GTEST_ASSERT_NE(0, tScalableBfInit(100, 0, &tsSBF)); + GTEST_ASSERT_NE(0, tScalableBfInit(100, 1, &tsSBF)); + GTEST_ASSERT_NE(0, tScalableBfInit(100, -0.1, &tsSBF)); + GTEST_ASSERT_NE(0, tScalableBfInit(0, 0.01, &tsSBF)); - SScalableBf *pSBF1 = tScalableBfInit(100, 0.01); + SScalableBf *pSBF1 = NULL; + GTEST_ASSERT_EQ(0, tScalableBfInit(100, 0.01, &pSBF1)); GTEST_ASSERT_EQ(pSBF1->numBits, 1152); int64_t count = 0; int64_t index = 0; From 9b067d075f47588cd82302cfca4ab497270bcb38 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 15 Jul 2024 13:38:34 +0800 Subject: [PATCH 05/34] adj stream function return --- include/libs/executor/storageapi.h | 4 +- include/libs/stream/tstreamUpdate.h | 28 ++-- include/util/tbloomfilter.h | 20 +-- include/util/tscalablebf.h | 24 ++-- source/libs/executor/src/groupoperator.c | 2 +- .../executor/src/streamtimewindowoperator.c | 11 +- source/libs/stream/src/streamUpdate.c | 136 +++++++++++------- source/util/src/tbloomfilter.c | 28 ++-- source/util/src/tscalablebf.c | 40 +++--- source/util/test/bloomFilterTest.cpp | 36 ++--- 10 files changed, 181 insertions(+), 148 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 45424cfb66..168760c61c 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -381,7 +381,7 @@ typedef struct SStateStore { void** ppVal, int32_t* pVLen); int32_t (*streamStateCountWinAdd)(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen); - SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); + int32_t (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid); @@ -391,7 +391,7 @@ typedef struct SStateStore { void (*windowSBfDelete)(SUpdateInfo* pInfo, uint64_t count); int32_t (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count); - SUpdateInfo* (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); + int32_t (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo); void (*updateInfoDestoryColseWinSBF)(SUpdateInfo* pInfo); int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo); diff --git a/include/libs/stream/tstreamUpdate.h b/include/libs/stream/tstreamUpdate.h index 726f8443f1..9de0d9ab68 100644 --- a/include/libs/stream/tstreamUpdate.h +++ b/include/libs/stream/tstreamUpdate.h @@ -15,29 +15,29 @@ #ifndef _TSTREAMUPDATE_H_ #define _TSTREAMUPDATE_H_ +#include "storageapi.h" #include "taosdef.h" #include "tarray.h" #include "tcommon.h" #include "tmsg.h" -#include "storageapi.h" #ifdef __cplusplus extern "C" { #endif -SUpdateInfo *updateInfoInitP(SInterval *pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); -SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); -TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); -bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); -bool updateInfoIsTableInserted(SUpdateInfo *pInfo, int64_t tbUid); -void updateInfoDestroy(SUpdateInfo *pInfo); -void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo); -void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo); -int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo); -int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo); -void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count); -int32_t windowSBfAdd(SUpdateInfo *pInfo, uint64_t count); -bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); +int32_t updateInfoInitP(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); +int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); +TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); +bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); +bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid); +void updateInfoDestroy(SUpdateInfo* pInfo); +void updateInfoAddCloseWindowSBF(SUpdateInfo* pInfo); +void updateInfoDestoryColseWinSBF(SUpdateInfo* pInfo); +int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo); +int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo); +void windowSBfDelete(SUpdateInfo* pInfo, uint64_t count); +int32_t windowSBfAdd(SUpdateInfo* pInfo, uint64_t count); +bool isIncrementalTimeStamp(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); #ifdef __cplusplus } diff --git a/include/util/tbloomfilter.h b/include/util/tbloomfilter.h index 2a3da90dad..f0450cb862 100644 --- a/include/util/tbloomfilter.h +++ b/include/util/tbloomfilter.h @@ -37,19 +37,19 @@ typedef struct SBloomFilter { uint64_t size; _hash_fn_t hashFn1; _hash_fn_t hashFn2; - void *buffer; + void* buffer; double errorRate; } SBloomFilter; -int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilter **ppBF); -int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2); -int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len); -int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t h1, uint64_t h2); -void tBloomFilterDestroy(SBloomFilter *pBF); -void tBloomFilterDump(const SBloomFilter *pBF); -bool tBloomFilterIsFull(const SBloomFilter *pBF); -int32_t tBloomFilterEncode(const SBloomFilter *pBF, SEncoder *pEncoder); -int32_t tBloomFilterDecode(SDecoder *pDecoder, SBloomFilter **ppBF); +int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilter** ppBF); +int32_t tBloomFilterPutHash(SBloomFilter* pBF, uint64_t hash1, uint64_t hash2); +int32_t tBloomFilterPut(SBloomFilter* pBF, const void* keyBuf, uint32_t len); +int32_t tBloomFilterNoContain(const SBloomFilter* pBF, uint64_t h1, uint64_t h2); +void tBloomFilterDestroy(SBloomFilter* pBF); +void tBloomFilterDump(const SBloomFilter* pBF); +bool tBloomFilterIsFull(const SBloomFilter* pBF); +int32_t tBloomFilterEncode(const SBloomFilter* pBF, SEncoder* pEncoder); +int32_t tBloomFilterDecode(SDecoder* pDecoder, SBloomFilter** ppBF); #ifdef __cplusplus } diff --git a/include/util/tscalablebf.h b/include/util/tscalablebf.h index 584e023bb0..f82493aca2 100644 --- a/include/util/tscalablebf.h +++ b/include/util/tscalablebf.h @@ -23,22 +23,22 @@ extern "C" { #endif typedef struct SScalableBf { - SArray *bfArray; // array of bloom filters - uint32_t growth; - uint64_t numBits; - uint32_t maxBloomFilters; - int8_t status; + SArray* bfArray; // array of bloom filters + uint32_t growth; + uint64_t numBits; + uint32_t maxBloomFilters; + int8_t status; _hash_fn_t hashFn1; _hash_fn_t hashFn2; } SScalableBf; -int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf **ppSBf); -int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len); -int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len); -int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len); -void tScalableBfDestroy(SScalableBf *pSBf); -int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder); -int32_t tScalableBfDecode(SDecoder *pDecoder, SScalableBf **ppSBf); +int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf** ppSBf); +int32_t tScalableBfPutNoCheck(SScalableBf* pSBf, const void* keyBuf, uint32_t len); +int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len); +int32_t tScalableBfNoContain(const SScalableBf* pSBf, const void* keyBuf, uint32_t len); +void tScalableBfDestroy(SScalableBf* pSBf); +int32_t tScalableBfEncode(const SScalableBf* pSBf, SEncoder* pEncoder); +int32_t tScalableBfDecode(SDecoder* pDecoder, SScalableBf** ppSBf); #ifdef __cplusplus } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index f0fddc28fc..ccb8e702bd 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -1369,7 +1369,7 @@ void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup pScanInfo->pPartScalarSup = pExpr; pScanInfo->pPartTbnameSup = pTbnameExpr; if (!pScanInfo->pUpdateInfo) { - pScanInfo->pUpdateInfo = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen); + pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo); } } diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index d68108b9a5..d20df9d8a5 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -487,8 +487,11 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamInt pScanInfo->windowSup.parentType = type; pScanInfo->windowSup.pIntervalAggSup = &pInfo->aggSup; if (!pScanInfo->pUpdateInfo) { - pScanInfo->pUpdateInfo = - pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen); + int32_t code = pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate, + pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at since %s", __func__, __LINE__, tstrerror(code)); + } } pScanInfo->interval = pInfo->interval; @@ -1769,8 +1772,8 @@ void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uin pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type}; pScanInfo->pState = pAggSup->pState; if (!pScanInfo->pUpdateInfo) { - pScanInfo->pUpdateInfo = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark, - pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen); + pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark, + pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo); } pScanInfo->twAggSup = *pTwSup; pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo; diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 2da20cadae..7122b8f10f 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -18,6 +18,7 @@ #include "tencode.h" #include "tstreamUpdate.h" #include "ttime.h" +#include "tutil.h" #define DEFAULT_FALSE_POSITIVE 0.01 #define DEFAULT_BUCKET_SIZE 131072 @@ -34,7 +35,8 @@ static int64_t adjustExpEntries(int64_t entries) { return TMIN(DEFAULT_EXPECTED_ENTRIES, entries); } int compareKeyTs(void* pTs1, void* pTs2, void* pPkVal, __compar_fn_t cmpPkFn) { - return compareInt64Val(pTs1, pTs2);; + return compareInt64Val(pTs1, pTs2); + ; } int compareKeyTsAndPk(void* pValue1, void* pTs, void* pPkVal, __compar_fn_t cmpPkFn) { @@ -49,7 +51,7 @@ int compareKeyTsAndPk(void* pValue1, void* pTs, void* pPkVal, __compar_fn_t cmpP int32_t getKeyBuff(TSKEY ts, int64_t tbUid, void* pVal, int32_t len, char* buff) { *(TSKEY*)buff = ts; - memcpy(buff+ sizeof(TSKEY), &tbUid, sizeof(int64_t)); + memcpy(buff + sizeof(TSKEY), &tbUid, sizeof(int64_t)); if (len == 0) { return sizeof(TSKEY) + sizeof(int64_t); } @@ -66,7 +68,7 @@ int32_t getValueBuff(TSKEY ts, char* pVal, int32_t len, char* buff) { return sizeof(TSKEY) + len; } -int32_t windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) { +int32_t windowSBfAdd(SUpdateInfo* pInfo, uint64_t count) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; if (pInfo->numSBFs < count) { @@ -74,10 +76,10 @@ int32_t windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) { } for (uint64_t i = 0; i < count; ++i) { int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND); - SScalableBf *tsSBF = NULL; + SScalableBf* tsSBF = NULL; code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &tsSBF); TSDB_CHECK_CODE(code, lino, _error); - void *res = taosArrayPush(pInfo->pTsSBFs, &tsSBF); + void* res = taosArrayPush(pInfo->pTsSBFs, &tsSBF); if (!res) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _error); @@ -91,15 +93,15 @@ _error: return code; } -static void clearItemHelper(void *p) { - SScalableBf **pBf = p; +static void clearItemHelper(void* p) { + SScalableBf** pBf = p; tScalableBfDestroy(*pBf); } -void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count) { +void windowSBfDelete(SUpdateInfo* pInfo, uint64_t count) { if (count < pInfo->numSBFs) { for (uint64_t i = 0; i < count; ++i) { - SScalableBf *pTsSBFs = taosArrayGetP(pInfo->pTsSBFs, 0); + SScalableBf* pTsSBFs = taosArrayGetP(pInfo->pTsSBFs, 0); tScalableBfDestroy(pTsSBFs); taosArrayRemove(pInfo->pTsSBFs, 0); } @@ -138,14 +140,19 @@ static int64_t adjustWatermark(int64_t adjInterval, int64_t originInt, int64_t w return watermark; } -SUpdateInfo *updateInfoInitP(SInterval *pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen) { - return updateInfoInit(pInterval->interval, pInterval->precision, watermark, igUp, pkType, pkLen); +int32_t updateInfoInitP(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, + SUpdateInfo** ppInfo) { + return updateInfoInit(pInterval->interval, pInterval->precision, watermark, igUp, pkType, pkLen, ppInfo); } -SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen) { - SUpdateInfo *pInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo)); +int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, + SUpdateInfo** ppInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SUpdateInfo* pInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo)); if (pInfo == NULL) { - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); } pInfo->pTsBuckets = NULL; pInfo->pTsSBFs = NULL; @@ -159,17 +166,19 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma bfSize = (uint64_t)(pInfo->watermark / pInfo->interval); pInfo->numSBFs = bfSize; - pInfo->pTsSBFs = taosArrayInit(bfSize, sizeof(void *)); + pInfo->pTsSBFs = taosArrayInit(bfSize, sizeof(void*)); if (pInfo->pTsSBFs == NULL) { updateInfoDestroy(pInfo); - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); } windowSBfAdd(pInfo, bfSize); pInfo->pTsBuckets = taosArrayInit(DEFAULT_BUCKET_SIZE, sizeof(TSKEY)); if (pInfo->pTsBuckets == NULL) { updateInfoDestroy(pInfo); - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); } TSKEY dumy = 0; @@ -181,22 +190,41 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma } _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT); pInfo->pMap = taosHashInit(DEFAULT_MAP_CAPACITY, hashFn, true, HASH_NO_LOCK); + if (!pInfo->pMap) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } pInfo->maxDataVersion = 0; pInfo->pkColLen = pkLen; pInfo->pkColType = pkType; pInfo->pKeyBuff = taosMemoryCalloc(1, sizeof(TSKEY) + sizeof(int64_t) + pkLen); + if (!pInfo->pKeyBuff) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } pInfo->pValueBuff = taosMemoryCalloc(1, sizeof(TSKEY) + pkLen); + if (!pInfo->pValueBuff) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } if (pkLen != 0) { pInfo->comparePkRowFn = compareKeyTsAndPk; - pInfo->comparePkCol = getKeyComparFunc(pkType, TSDB_ORDER_ASC);; + pInfo->comparePkCol = getKeyComparFunc(pkType, TSDB_ORDER_ASC); + ; } else { pInfo->comparePkRowFn = compareKeyTs; pInfo->comparePkCol = NULL; } - return pInfo; + (*ppInfo) = pInfo; + +_end: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -static SScalableBf *getSBf(SUpdateInfo *pInfo, TSKEY ts) { +static SScalableBf* getSBf(SUpdateInfo* pInfo, TSKEY ts) { int32_t code = 0; if (ts <= 0) { return NULL; @@ -214,7 +242,7 @@ static SScalableBf *getSBf(SUpdateInfo *pInfo, TSKEY ts) { windowSBfAdd(pInfo, count); index = pInfo->numSBFs - 1; } - SScalableBf *res = taosArrayGetP(pInfo->pTsSBFs, index); + SScalableBf* res = taosArrayGetP(pInfo->pTsSBFs, index); if (res == NULL) { int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND); code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &res); @@ -223,13 +251,13 @@ static SScalableBf *getSBf(SUpdateInfo *pInfo, TSKEY ts) { return res; } -bool updateInfoIsTableInserted(SUpdateInfo *pInfo, int64_t tbUid) { - void *pVal = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t)); +bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid) { + void* pVal = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t)); if (pVal || taosHashGetSize(pInfo->pMap) >= DEFAULT_MAP_SIZE) return true; return false; } -TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol, int32_t primaryKeyCol) { +TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol) { if (pBlock == NULL || pBlock->info.rows == 0) return INT64_MIN; TSKEY maxTs = INT64_MIN; void* pPkVal = NULL; @@ -238,14 +266,14 @@ TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t p int32_t len = 0; int64_t tbUid = pBlock->info.id.uid; - SColumnInfoData *pColDataInfo = taosArrayGet(pBlock->pDataBlock, primaryTsCol); - SColumnInfoData *pPkDataInfo = NULL; + SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, primaryTsCol); + SColumnInfoData* pPkDataInfo = NULL; if (primaryKeyCol >= 0) { pPkDataInfo = taosArrayGet(pBlock->pDataBlock, primaryKeyCol); } for (int32_t i = 0; i < pBlock->info.rows; i++) { - TSKEY ts = ((TSKEY *)pColDataInfo->pData)[i]; + TSKEY ts = ((TSKEY*)pColDataInfo->pData)[i]; if (maxTs < ts) { maxTs = ts; if (primaryKeyCol >= 0) { @@ -253,7 +281,7 @@ TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t p maxLen = colDataGetRowLength(pPkDataInfo, i); } } - SScalableBf *pSBf = getSBf(pInfo, ts); + SScalableBf* pSBf = getSBf(pInfo, ts); if (pSBf) { if (primaryKeyCol >= 0) { pPkVal = colDataGetData(pPkDataInfo, i); @@ -263,7 +291,7 @@ TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t p tScalableBfPut(pSBf, pInfo->pKeyBuff, buffLen); } } - void *pMaxTs = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t)); + void* pMaxTs = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t)); if (pMaxTs == NULL || pInfo->comparePkRowFn(pMaxTs, &maxTs, pMaxPkVal, pInfo->comparePkCol) == -1) { int32_t valueLen = getValueBuff(maxTs, pMaxPkVal, maxLen, pInfo->pValueBuff); taosHashPut(pInfo->pMap, &tbUid, sizeof(int64_t), pInfo->pValueBuff, valueLen); @@ -271,14 +299,14 @@ TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t p return maxTs; } -bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) { +bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) { int32_t res = TSDB_CODE_FAILED; int32_t buffLen = 0; buffLen = getKeyBuff(ts, tableId, pPkVal, len, pInfo->pKeyBuff); - void* *pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); + void** pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); uint64_t index = ((uint64_t)tableId) % pInfo->numBuckets; - TSKEY maxTs = *(TSKEY *)taosArrayGet(pInfo->pTsBuckets, index); + TSKEY maxTs = *(TSKEY*)taosArrayGet(pInfo->pTsBuckets, index); if (ts < maxTs - pInfo->watermark) { // this window has been closed. if (pInfo->pCloseWinSBF) { @@ -292,10 +320,11 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* p return true; } - SScalableBf *pSBf = getSBf(pInfo, ts); + SScalableBf* pSBf = getSBf(pInfo, ts); int32_t size = taosHashGetSize(pInfo->pMap); - if ((!pMapMaxTs && size < DEFAULT_MAP_SIZE) || (pMapMaxTs && pInfo->comparePkRowFn(pMapMaxTs, &ts, pPkVal, pInfo->comparePkCol) == -1 )) { + if ((!pMapMaxTs && size < DEFAULT_MAP_SIZE) || + (pMapMaxTs && pInfo->comparePkRowFn(pMapMaxTs, &ts, pPkVal, pInfo->comparePkCol) == -1)) { int32_t valueLen = getValueBuff(ts, pPkVal, len, pInfo->pValueBuff); taosHashPut(pInfo->pMap, &tableId, sizeof(uint64_t), pInfo->pValueBuff, valueLen); // pSBf may be a null pointer @@ -324,7 +353,7 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* p return true; } -void updateInfoDestroy(SUpdateInfo *pInfo) { +void updateInfoDestroy(SUpdateInfo* pInfo) { if (pInfo == NULL) { return; } @@ -332,7 +361,7 @@ void updateInfoDestroy(SUpdateInfo *pInfo) { uint64_t size = taosArrayGetSize(pInfo->pTsSBFs); for (uint64_t i = 0; i < size; i++) { - SScalableBf *pSBF = taosArrayGetP(pInfo->pTsSBFs, i); + SScalableBf* pSBF = taosArrayGetP(pInfo->pTsSBFs, i); tScalableBfDestroy(pSBF); } @@ -344,7 +373,7 @@ void updateInfoDestroy(SUpdateInfo *pInfo) { taosMemoryFree(pInfo); } -void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo) { +void updateInfoAddCloseWindowSBF(SUpdateInfo* pInfo) { if (pInfo->pCloseWinSBF) { return; } @@ -356,7 +385,7 @@ void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo) { } } -void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo) { +void updateInfoDestoryColseWinSBF(SUpdateInfo* pInfo) { if (!pInfo || !pInfo->pCloseWinSBF) { return; } @@ -364,7 +393,7 @@ void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo) { pInfo->pCloseWinSBF = NULL; } -int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo) { +int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo) { if (!pInfo) { return 0; } @@ -376,7 +405,7 @@ int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo) int32_t size = taosArrayGetSize(pInfo->pTsBuckets); if (tEncodeI32(&encoder, size) < 0) return -1; for (int32_t i = 0; i < size; i++) { - TSKEY *pTs = (TSKEY *)taosArrayGet(pInfo->pTsBuckets, i); + TSKEY* pTs = (TSKEY*)taosArrayGet(pInfo->pTsBuckets, i); if (tEncodeI64(&encoder, *pTs) < 0) return -1; } @@ -385,7 +414,7 @@ int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo) int32_t sBfSize = taosArrayGetSize(pInfo->pTsSBFs); if (tEncodeI32(&encoder, sBfSize) < 0) return -1; for (int32_t i = 0; i < sBfSize; i++) { - SScalableBf *pSBf = taosArrayGetP(pInfo->pTsSBFs, i); + SScalableBf* pSBf = taosArrayGetP(pInfo->pTsSBFs, i); if (tScalableBfEncode(pSBf, &encoder) < 0) return -1; } @@ -398,13 +427,13 @@ int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo) int32_t mapSize = taosHashGetSize(pInfo->pMap); if (tEncodeI32(&encoder, mapSize) < 0) return -1; - void *pIte = NULL; + void* pIte = NULL; size_t keyLen = 0; while ((pIte = taosHashIterate(pInfo->pMap, pIte)) != NULL) { - void *key = taosHashGetKey(pIte, &keyLen); - if (tEncodeU64(&encoder, *(uint64_t *)key) < 0) return -1; + void* key = taosHashGetKey(pIte, &keyLen); + if (tEncodeU64(&encoder, *(uint64_t*)key) < 0) return -1; int32_t valueSize = taosHashGetValueSize(pIte); - if (tEncodeBinary(&encoder, (const uint8_t *)pIte, valueSize) < 0) return -1; + if (tEncodeBinary(&encoder, (const uint8_t*)pIte, valueSize) < 0) return -1; } if (tEncodeU64(&encoder, pInfo->maxDataVersion) < 0) return -1; @@ -419,7 +448,7 @@ int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo) return tlen; } -int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { +int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; ASSERT(pInfo); @@ -440,9 +469,9 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { int32_t sBfSize = 0; if (tDecodeI32(&decoder, &sBfSize) < 0) return -1; - pInfo->pTsSBFs = taosArrayInit(sBfSize, sizeof(void *)); + pInfo->pTsSBFs = taosArrayInit(sBfSize, sizeof(void*)); for (int32_t i = 0; i < sBfSize; i++) { - SScalableBf *pSBf = NULL; + SScalableBf* pSBf = NULL; code = tScalableBfDecode(&decoder, &pSBf); TSDB_CHECK_CODE(code, lino, _error); @@ -462,8 +491,8 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT); pInfo->pMap = taosHashInit(mapSize, hashFn, true, HASH_NO_LOCK); uint64_t uid = 0; - void* pVal = NULL; - int32_t valSize = 0; + void* pVal = NULL; + int32_t valSize = 0; for (int32_t i = 0; i < mapSize; i++) { if (tDecodeU64(&decoder, &uid) < 0) return -1; if (tDecodeBinary(&decoder, (uint8_t**)&pVal, &valSize) < 0) return -1; @@ -479,7 +508,8 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { pInfo->pValueBuff = taosMemoryCalloc(1, sizeof(TSKEY) + pInfo->pkColLen); if (pInfo->pkColLen != 0) { pInfo->comparePkRowFn = compareKeyTsAndPk; - pInfo->comparePkCol = getKeyComparFunc(pInfo->pkColType, TSDB_ORDER_ASC);; + pInfo->comparePkCol = getKeyComparFunc(pInfo->pkColType, TSDB_ORDER_ASC); + ; } else { pInfo->comparePkRowFn = compareKeyTs; pInfo->comparePkCol = NULL; @@ -496,8 +526,8 @@ _error: return code; } -bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) { - TSKEY *pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); +bool isIncrementalTimeStamp(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) { + TSKEY* pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); bool res = true; if (pMapMaxTs && pInfo->comparePkRowFn(pMapMaxTs, &ts, pPkVal, pInfo->comparePkCol) == 1) { res = false; diff --git a/source/util/src/tbloomfilter.c b/source/util/src/tbloomfilter.c index 557e12a5b2..9093fb9d91 100644 --- a/source/util/src/tbloomfilter.c +++ b/source/util/src/tbloomfilter.c @@ -22,27 +22,27 @@ #define UNIT_NUM_BITS 64ULL #define UNIT_ADDR_NUM_BITS 6ULL -static FORCE_INLINE bool setBit(uint64_t *buf, uint64_t index) { +static FORCE_INLINE bool setBit(uint64_t* buf, uint64_t index) { uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS; uint64_t old = buf[unitIndex]; buf[unitIndex] |= (1ULL << (index % UNIT_NUM_BITS)); return buf[unitIndex] != old; } -static FORCE_INLINE bool getBit(uint64_t *buf, uint64_t index) { +static FORCE_INLINE bool getBit(uint64_t* buf, uint64_t index) { uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS; uint64_t mask = 1ULL << (index % UNIT_NUM_BITS); return buf[unitIndex] & mask; } -int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilter **ppBF) { +int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilter** ppBF) { int32_t code = 0; int32_t lino = 0; if (expectedEntries < 1 || errorRate <= 0 || errorRate >= 1.0) { code = TSDB_CODE_FAILED; TSDB_CHECK_CODE(code, lino, _error); } - SBloomFilter *pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); + SBloomFilter* pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); if (pBF == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _error); @@ -77,7 +77,7 @@ _error: return code; } -int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2) { +int32_t tBloomFilterPutHash(SBloomFilter* pBF, uint64_t hash1, uint64_t hash2) { ASSERT(!tBloomFilterIsFull(pBF)); bool hasChange = false; const register uint64_t size = pBF->numBits; @@ -93,7 +93,7 @@ int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2) { return TSDB_CODE_FAILED; } -int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len) { +int32_t tBloomFilterPut(SBloomFilter* pBF, const void* keyBuf, uint32_t len) { uint64_t h1 = (uint64_t)pBF->hashFn1(keyBuf, len); uint64_t h2 = (uint64_t)pBF->hashFn2(keyBuf, len); bool hasChange = false; @@ -110,7 +110,7 @@ int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len) { return TSDB_CODE_FAILED; } -int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t hash1, uint64_t hash2) { +int32_t tBloomFilterNoContain(const SBloomFilter* pBF, uint64_t hash1, uint64_t hash2) { const register uint64_t size = pBF->numBits; uint64_t cbHash = hash1; for (uint32_t i = 0; i < pBF->hashFunctions; ++i) { @@ -122,7 +122,7 @@ int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t hash1, uint64_t return TSDB_CODE_FAILED; } -void tBloomFilterDestroy(SBloomFilter *pBF) { +void tBloomFilterDestroy(SBloomFilter* pBF) { if (pBF == NULL) { return; } @@ -130,24 +130,24 @@ void tBloomFilterDestroy(SBloomFilter *pBF) { taosMemoryFree(pBF); } -int32_t tBloomFilterEncode(const SBloomFilter *pBF, SEncoder *pEncoder) { +int32_t tBloomFilterEncode(const SBloomFilter* pBF, SEncoder* pEncoder) { if (tEncodeU32(pEncoder, pBF->hashFunctions) < 0) return -1; if (tEncodeU64(pEncoder, pBF->expectedEntries) < 0) return -1; if (tEncodeU64(pEncoder, pBF->numUnits) < 0) return -1; if (tEncodeU64(pEncoder, pBF->numBits) < 0) return -1; if (tEncodeU64(pEncoder, pBF->size) < 0) return -1; for (uint64_t i = 0; i < pBF->numUnits; i++) { - uint64_t *pUnits = (uint64_t *)pBF->buffer; + uint64_t* pUnits = (uint64_t*)pBF->buffer; if (tEncodeU64(pEncoder, pUnits[i]) < 0) return -1; } if (tEncodeDouble(pEncoder, pBF->errorRate) < 0) return -1; return 0; } -int32_t tBloomFilterDecode(SDecoder *pDecoder, SBloomFilter **ppBF) { +int32_t tBloomFilterDecode(SDecoder* pDecoder, SBloomFilter** ppBF) { int32_t code = 0; int32_t lino = 0; - SBloomFilter *pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); + SBloomFilter* pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); if (!pBF) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _error); @@ -175,7 +175,7 @@ int32_t tBloomFilterDecode(SDecoder *pDecoder, SBloomFilter **ppBF) { } pBF->buffer = taosMemoryCalloc(pBF->numUnits, sizeof(uint64_t)); for (int32_t i = 0; i < pBF->numUnits; i++) { - uint64_t *pUnits = (uint64_t *)pBF->buffer; + uint64_t* pUnits = (uint64_t*)pBF->buffer; if (tDecodeU64(pDecoder, pUnits + i) < 0) { code = TSDB_CODE_FAILED; TSDB_CHECK_CODE(code, lino, _error); @@ -195,4 +195,4 @@ _error: return TSDB_CODE_FAILED; } -bool tBloomFilterIsFull(const SBloomFilter *pBF) { return pBF->size >= pBF->expectedEntries; } +bool tBloomFilterIsFull(const SBloomFilter* pBF) { return pBF->size >= pBF->expectedEntries; } diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index 13e9fe79e8..0979000981 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -25,10 +25,10 @@ #define SBF_INVALID -1 #define SBF_VALID 0 -static int32_t tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate, - SBloomFilter **ppNormalBf); +static int32_t tScalableBfAddFilter(SScalableBf* pSBf, uint64_t expectedEntries, double errorRate, + SBloomFilter** ppNormalBf); -int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf **ppSBf) { +int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf** ppSBf) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; const uint32_t defaultSize = 8; @@ -36,7 +36,7 @@ int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf code = TSDB_CODE_FAILED; TSDB_CHECK_CODE(code, lino, _error); } - SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); + SScalableBf* pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); if (pSBf == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _error); @@ -44,13 +44,13 @@ int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf pSBf->maxBloomFilters = DEFAULT_MAX_BLOOMFILTERS; pSBf->status = SBF_VALID; pSBf->numBits = 0; - pSBf->bfArray = taosArrayInit(defaultSize, sizeof(void *)); + pSBf->bfArray = taosArrayInit(defaultSize, sizeof(void*)); if (!pSBf->bfArray) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _error); } - SBloomFilter *pNormalBf = NULL; + SBloomFilter* pNormalBf = NULL; code = tScalableBfAddFilter(pSBf, expectedEntries, errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf); if (code != TSDB_CODE_SUCCESS) { tScalableBfDestroy(pSBf); @@ -67,7 +67,7 @@ _error: return code; } -int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { +int32_t tScalableBfPutNoCheck(SScalableBf* pSBf, const void* keyBuf, uint32_t len) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; if (pSBf->status == SBF_INVALID) { @@ -75,7 +75,7 @@ int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t le TSDB_CHECK_CODE(code, lino, _error); } int32_t size = taosArrayGetSize(pSBf->bfArray); - SBloomFilter *pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); + SBloomFilter* pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); if (!pNormalBf) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _error); @@ -97,7 +97,7 @@ _error: return code; } -int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { +int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; if (pSBf->status == SBF_INVALID) { @@ -113,7 +113,7 @@ int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { } } - SBloomFilter *pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); + SBloomFilter* pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); ASSERT(pNormalBf); if (tBloomFilterIsFull(pNormalBf)) { code = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, @@ -132,7 +132,7 @@ _error: return code; } -int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len) { +int32_t tScalableBfNoContain(const SScalableBf* pSBf, const void* keyBuf, uint32_t len) { if (pSBf->status == SBF_INVALID) { return TSDB_CODE_FAILED; } @@ -147,8 +147,8 @@ int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32 return TSDB_CODE_SUCCESS; } -static int32_t tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate, - SBloomFilter **ppNormalBf) { +static int32_t tScalableBfAddFilter(SScalableBf* pSBf, uint64_t expectedEntries, double errorRate, + SBloomFilter** ppNormalBf) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; if (taosArrayGetSize(pSBf->bfArray) >= pSBf->maxBloomFilters) { @@ -156,7 +156,7 @@ static int32_t tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, TSDB_CHECK_CODE(code, lino, _error); } - SBloomFilter *pNormalBf = NULL; + SBloomFilter* pNormalBf = NULL; code = tBloomFilterInit(expectedEntries, errorRate, &pNormalBf); TSDB_CHECK_CODE(code, lino, _error); @@ -175,7 +175,7 @@ _error: return code; } -void tScalableBfDestroy(SScalableBf *pSBf) { +void tScalableBfDestroy(SScalableBf* pSBf) { if (pSBf == NULL) { return; } @@ -185,7 +185,7 @@ void tScalableBfDestroy(SScalableBf *pSBf) { taosMemoryFree(pSBf); } -int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) { +int32_t tScalableBfEncode(const SScalableBf* pSBf, SEncoder* pEncoder) { if (!pSBf) { if (tEncodeI32(pEncoder, 0) < 0) return -1; return 0; @@ -193,7 +193,7 @@ int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) { int32_t size = taosArrayGetSize(pSBf->bfArray); if (tEncodeI32(pEncoder, size) < 0) return -1; for (int32_t i = 0; i < size; i++) { - SBloomFilter *pBF = taosArrayGetP(pSBf->bfArray, i); + SBloomFilter* pBF = taosArrayGetP(pSBf->bfArray, i); if (tBloomFilterEncode(pBF, pEncoder) < 0) return -1; } if (tEncodeU32(pEncoder, pSBf->growth) < 0) return -1; @@ -203,10 +203,10 @@ int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) { return 0; } -int32_t tScalableBfDecode(SDecoder *pDecoder, SScalableBf **ppSBf) { +int32_t tScalableBfDecode(SDecoder* pDecoder, SScalableBf** ppSBf) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; - SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); + SScalableBf* pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); if (!pSBf) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _error); @@ -230,7 +230,7 @@ int32_t tScalableBfDecode(SDecoder *pDecoder, SScalableBf **ppSBf) { } for (int32_t i = 0; i < size; i++) { - SBloomFilter *pBF = NULL; + SBloomFilter* pBF = NULL; code = tBloomFilterDecode(pDecoder, &pBF); TSDB_CHECK_CODE(code, lino, _error); taosArrayPush(pSBf->bfArray, &pBF); diff --git a/source/util/test/bloomFilterTest.cpp b/source/util/test/bloomFilterTest.cpp index 2cd4edd519..9d913f8815 100644 --- a/source/util/test/bloomFilterTest.cpp +++ b/source/util/test/bloomFilterTest.cpp @@ -8,15 +8,14 @@ using namespace std; TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { int64_t ts1 = 1650803518000; - SBloomFilter *pBFTmp = NULL; + SBloomFilter* pBFTmp = NULL; GTEST_ASSERT_EQ(0, tBloomFilterInit(100, 0, &pBFTmp)); GTEST_ASSERT_EQ(0, tBloomFilterInit(100, 1, &pBFTmp)); GTEST_ASSERT_EQ(0, tBloomFilterInit(100, -0.1, &pBFTmp)); GTEST_ASSERT_EQ(0, tBloomFilterInit(0, 0.01, &pBFTmp)); - SBloomFilter *pBF1 = NULL, - int32_t code = tBloomFilterInit(100, 0.005, &pBF1); - GTEST_ASSERT_EQ(0,code); + SBloomFilter *pBF1 = NULL, int32_t code = tBloomFilterInit(100, 0.005, &pBF1); + GTEST_ASSERT_EQ(0, code); GTEST_ASSERT_EQ(pBF1->numBits, 1152); GTEST_ASSERT_EQ(pBF1->numUnits, 1152 / 64); int64_t count = 0; @@ -28,18 +27,18 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { } ASSERT_TRUE(tBloomFilterIsFull(pBF1)); - SBloomFilter *pBF2 = NULL; + SBloomFilter* pBF2 = NULL; GTEST_ASSERT_EQ(0, tBloomFilterInit(1000 * 10000, 0.1, &pBF2)); GTEST_ASSERT_EQ(pBF2->numBits, 47925312); GTEST_ASSERT_EQ(pBF2->numUnits, 47925312 / 64); - SBloomFilter *pBF3 = NULL; + SBloomFilter* pBF3 = NULL; GTEST_ASSERT_EQ(0, tBloomFilterInit(10000 * 10000, 0.001)); GTEST_ASSERT_EQ(pBF3->numBits, 1437758784); GTEST_ASSERT_EQ(pBF3->numUnits, 1437758784 / 64); int64_t size = 10000; - SBloomFilter *pBF4 = NULL; + SBloomFilter* pBF4 = NULL; GTEST_ASSERT_EQ(0, tBloomFilterInit(size, 0.001)); for (int64_t i = 0; i < 1000; i++) { int64_t ts = i + ts1; @@ -48,17 +47,17 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { ASSERT_TRUE(!tBloomFilterIsFull(pBF4)); for (int64_t i = 0; i < 1000; i++) { - int64_t ts = i + ts1; - uint64_t h1 = (uint64_t) pBF4->hashFn1((const char*)&ts, sizeof(int64_t)); - uint64_t h2 = (uint64_t) pBF4->hashFn2((const char*)&ts, sizeof(int64_t)); + int64_t ts = i + ts1; + uint64_t h1 = (uint64_t)pBF4->hashFn1((const char*)&ts, sizeof(int64_t)); + uint64_t h2 = (uint64_t)pBF4->hashFn2((const char*)&ts, sizeof(int64_t)); GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, h1, h2), TSDB_CODE_FAILED); } for (int64_t i = 2000; i < 3000; i++) { - int64_t ts = i + ts1; - uint64_t h1 = (uint64_t) pBF4->hashFn1((const char*)&ts, sizeof(int64_t)); - uint64_t h2 = (uint64_t) pBF4->hashFn2((const char*)&ts, sizeof(int64_t)); - GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, h1, h2), TSDB_CODE_SUCCESS); + int64_t ts = i + ts1; + uint64_t h1 = (uint64_t)pBF4->hashFn1((const char*)&ts, sizeof(int64_t)); + uint64_t h2 = (uint64_t)pBF4->hashFn2((const char*)&ts, sizeof(int64_t)); + GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, h1, h2), TSDB_CODE_SUCCESS); } tBloomFilterDestroy(pBF1); @@ -70,13 +69,13 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) { int64_t ts1 = 1650803518000; - SScalableBf *tsSBF = NULL; + SScalableBf* tsSBF = NULL; GTEST_ASSERT_NE(0, tScalableBfInit(100, 0, &tsSBF)); GTEST_ASSERT_NE(0, tScalableBfInit(100, 1, &tsSBF)); GTEST_ASSERT_NE(0, tScalableBfInit(100, -0.1, &tsSBF)); GTEST_ASSERT_NE(0, tScalableBfInit(0, 0.01, &tsSBF)); - SScalableBf *pSBF1 = NULL; + SScalableBf* pSBF1 = NULL; GTEST_ASSERT_EQ(0, tScalableBfInit(100, 0.01, &pSBF1)); GTEST_ASSERT_EQ(pSBF1->numBits, 1152); int64_t count = 0; @@ -116,7 +115,7 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) { int32_t aSize = taosArrayGetSize(pSBF1->bfArray); int64_t totalBits = 0; for (int64_t i = 0; i < aSize; i++) { - SBloomFilter *pBF = (SBloomFilter *)taosArrayGetP(pSBF1->bfArray, i); + SBloomFilter* pBF = (SBloomFilter*)taosArrayGetP(pSBF1->bfArray, i); ASSERT_TRUE(tBloomFilterIsFull(pBF)); totalBits += pBF->numBits; } @@ -128,7 +127,8 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) { } int64_t size = 10000; - SScalableBf *pSBF4 = tScalableBfInit(size, 0.001); + SScalableBf* pSBF4 = NULL; + GTEST_ASSERT_EQ(0, tScalableBfInit(size, 0.001, pSBF4)); for (int64_t i = 0; i < 1000; i++) { int64_t ts = i + ts1; GTEST_ASSERT_EQ(tScalableBfPut(pSBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS); From bb25622a3055d348a907beb674721a67c8ccf5b5 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 15 Jul 2024 13:47:02 +0800 Subject: [PATCH 06/34] adj ci --- source/util/test/bloomFilterTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/test/bloomFilterTest.cpp b/source/util/test/bloomFilterTest.cpp index 9d913f8815..d14aa2d453 100644 --- a/source/util/test/bloomFilterTest.cpp +++ b/source/util/test/bloomFilterTest.cpp @@ -128,7 +128,7 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) { int64_t size = 10000; SScalableBf* pSBF4 = NULL; - GTEST_ASSERT_EQ(0, tScalableBfInit(size, 0.001, pSBF4)); + GTEST_ASSERT_EQ(0, tScalableBfInit(size, 0.001, &pSBF4)); for (int64_t i = 0; i < 1000; i++) { int64_t ts = i + ts1; GTEST_ASSERT_EQ(tScalableBfPut(pSBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS); From 2062e6f9e93ab4b6efd44ec7f2c992d367deb3fb Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 15 Jul 2024 14:11:02 +0800 Subject: [PATCH 07/34] adj ut --- source/util/test/bloomFilterTest.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/source/util/test/bloomFilterTest.cpp b/source/util/test/bloomFilterTest.cpp index d14aa2d453..9d83cbf086 100644 --- a/source/util/test/bloomFilterTest.cpp +++ b/source/util/test/bloomFilterTest.cpp @@ -9,12 +9,13 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { int64_t ts1 = 1650803518000; SBloomFilter* pBFTmp = NULL; - GTEST_ASSERT_EQ(0, tBloomFilterInit(100, 0, &pBFTmp)); - GTEST_ASSERT_EQ(0, tBloomFilterInit(100, 1, &pBFTmp)); - GTEST_ASSERT_EQ(0, tBloomFilterInit(100, -0.1, &pBFTmp)); - GTEST_ASSERT_EQ(0, tBloomFilterInit(0, 0.01, &pBFTmp)); + GTEST_ASSERT_NE(0, tBloomFilterInit(100, 0, &pBFTmp)); + GTEST_ASSERT_NE(0, tBloomFilterInit(100, 1, &pBFTmp)); + GTEST_ASSERT_NE(0, tBloomFilterInit(100, -0.1, &pBFTmp)); + GTEST_ASSERT_NE(0, tBloomFilterInit(0, 0.01, &pBFTmp)); - SBloomFilter *pBF1 = NULL, int32_t code = tBloomFilterInit(100, 0.005, &pBF1); + SBloomFilter *pBF1 = NULL; + int32_t code = tBloomFilterInit(100, 0.005, &pBF1); GTEST_ASSERT_EQ(0, code); GTEST_ASSERT_EQ(pBF1->numBits, 1152); GTEST_ASSERT_EQ(pBF1->numUnits, 1152 / 64); @@ -33,13 +34,13 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { GTEST_ASSERT_EQ(pBF2->numUnits, 47925312 / 64); SBloomFilter* pBF3 = NULL; - GTEST_ASSERT_EQ(0, tBloomFilterInit(10000 * 10000, 0.001)); + GTEST_ASSERT_EQ(0, tBloomFilterInit(10000 * 10000, 0.001, &pBF3)); GTEST_ASSERT_EQ(pBF3->numBits, 1437758784); GTEST_ASSERT_EQ(pBF3->numUnits, 1437758784 / 64); int64_t size = 10000; SBloomFilter* pBF4 = NULL; - GTEST_ASSERT_EQ(0, tBloomFilterInit(size, 0.001)); + GTEST_ASSERT_EQ(0, tBloomFilterInit(size, 0.001, &pBF4)); for (int64_t i = 0; i < 1000; i++) { int64_t ts = i + ts1; GTEST_ASSERT_EQ(tBloomFilterPut(pBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS); From 503822a9c5bd00cbaa259bff918d8ccf4b8a74bb Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 15 Jul 2024 14:24:09 +0800 Subject: [PATCH 08/34] adj ut --- source/libs/executor/src/streamtimewindowoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index d20df9d8a5..97cb20a7ea 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -490,7 +490,7 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamInt int32_t code = pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo); if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at since %s", __func__, __LINE__, tstrerror(code)); + qError("%s failed at %d since %s", __func__, __LINE__, tstrerror(code)); } } From 4efd774ce8abe18f683929cce18bf4d80cdeb079 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 15 Jul 2024 14:32:31 +0800 Subject: [PATCH 09/34] adj stream function return --- source/libs/stream/src/streamUpdate.c | 34 +++++++++++++++++++++------ 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 7122b8f10f..8727ce0174 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -224,17 +224,20 @@ _end: return code; } -static SScalableBf* getSBf(SUpdateInfo* pInfo, TSKEY ts) { - int32_t code = 0; +static int32_t getSBf(SUpdateInfo* pInfo, TSKEY ts, SScalableBf** ppSBf) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (ts <= 0) { - return NULL; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->minTS < 0) { pInfo->minTS = (TSKEY)(ts / pInfo->interval * pInfo->interval); } int64_t index = (int64_t)((ts - pInfo->minTS) / pInfo->interval); if (index < 0) { - return NULL; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } if (index >= pInfo->numSBFs) { uint64_t count = index + 1 - pInfo->numSBFs; @@ -246,9 +249,17 @@ static SScalableBf* getSBf(SUpdateInfo* pInfo, TSKEY ts) { if (res == NULL) { int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND); code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &res); + TSDB_CHECK_CODE(code, lino, _end); + taosArrayPush(pInfo->pTsSBFs, &res); } - return res; + (*ppSBf) = res; + +_end: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid) { @@ -258,6 +269,7 @@ bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid) { } TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol) { + int32_t code = TSDB_CODE_SUCCESS; if (pBlock == NULL || pBlock->info.rows == 0) return INT64_MIN; TSKEY maxTs = INT64_MIN; void* pPkVal = NULL; @@ -281,7 +293,11 @@ TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t p maxLen = colDataGetRowLength(pPkDataInfo, i); } } - SScalableBf* pSBf = getSBf(pInfo, ts); + SScalableBf* pSBf = NULL; + int32_t code = getSBf(pInfo, ts, &pSBf); + if (code != TSDB_CODE_SUCCESS) { + uWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } if (pSBf) { if (primaryKeyCol >= 0) { pPkVal = colDataGetData(pPkDataInfo, i); @@ -320,7 +336,11 @@ bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* p return true; } - SScalableBf* pSBf = getSBf(pInfo, ts); + SScalableBf* pSBf = NULL; + int32_t code = getSBf(pInfo, ts, &pSBf); + if (code != TSDB_CODE_SUCCESS) { + uWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } int32_t size = taosHashGetSize(pInfo->pMap); if ((!pMapMaxTs && size < DEFAULT_MAP_SIZE) || From 57dc3f4562ee30b3e6fa141083cd213cba21b27e Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Mon, 15 Jul 2024 15:15:17 +0800 Subject: [PATCH 10/34] fix: disable dynamic modification of 'tempdir' --- source/common/src/tglobal.c | 2 +- tests/army/cmdline/fullopt.py | 21 ++++++++++----------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 119438fbd6..7538b504e3 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -504,7 +504,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "secondEp", "", CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddString(pCfg, "fqdn", defaultFqdn, CFG_SCOPE_SERVER, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_CLIENT) != 0) return -1; - if (cfgAddDir(pCfg, "tempDir", tsTempDir, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddDir(pCfg, "tempDir", tsTempDir, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; if (cfgAddFloat(pCfg, "minimalTmpDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; diff --git a/tests/army/cmdline/fullopt.py b/tests/army/cmdline/fullopt.py index 77a22b9256..6fc8e858a3 100644 --- a/tests/army/cmdline/fullopt.py +++ b/tests/army/cmdline/fullopt.py @@ -28,13 +28,13 @@ from frame import * class TDTestCase(TBase): updatecfgDict = { - 'queryMaxConcurrentTables': '2K', - 'streamMax': '1M', + 'queryMaxConcurrentTables': '2K', + 'streamMax': '1M', 'totalMemoryKB': '1G', 'streamMax': '1P', 'streamBufferSize':'1T', 'slowLogScope':"query" - } + } def insertData(self): tdLog.info(f"insert data.") @@ -53,7 +53,7 @@ class TDTestCase(TBase): def doTaos(self): tdLog.info(f"check taos command options...") - + # local command options = [ "DebugFlag 143", @@ -71,7 +71,6 @@ class TDTestCase(TBase): "smlTsDefaultName tsdef", "serverPort 6030", "timezone tz", - "tempDir /var/tmp" ] # exec for option in options: @@ -83,7 +82,7 @@ class TDTestCase(TBase): # help rets = etool.runBinFile("taos", "--help") self.checkListNotEmpty(rets) - # b r w s + # b r w s sql = f"select * from {self.db}.{self.stb} limit 10" rets = etool.runBinFile("taos", f'-B -r -w 100 -s "{sql}" ') self.checkListNotEmpty(rets) @@ -124,16 +123,16 @@ class TDTestCase(TBase): tdLog.info(f"check taosd command options...") idx = 1 # dnode1 cfg = sc.dnodeCfgPath(idx) - + # -s sdb = "./sdb.json" eos.delFile(sdb) etool.exeBinFile("taosd", f"-s -c {cfg}") - + # -C etool.exeBinFile("taosd", "-C") - # -k + # -k etool.exeBinFile("taosd", "-k", False) # -V rets = etool.runBinFile("taosd", "-V") @@ -153,7 +152,7 @@ class TDTestCase(TBase): sc.dnodeStop(idx) etool.exeBinFile("taosd", "-a http://192.168.1.10") - #exe + #exe etool.exeBinFile("taosd", f"-E abc -c {cfg}", False) sc.dnodeStop(idx) etool.exeBinFile("taosd", f"-e def -c {cfg}", False) @@ -182,7 +181,7 @@ class TDTestCase(TBase): # do action self.doTaosd() - + tdLog.success(f"{__file__} successfully executed") From 019a1eb76b36995d2eeb9dfba05065f7ba068afc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 15 Jul 2024 15:28:32 +0800 Subject: [PATCH 11/34] fix(tsdb): check return value. --- source/common/src/tdatablock.c | 4 ++++ source/dnode/vnode/src/tsdb/tsdbRead2.c | 19 ++++++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 809721d606..8f5d382cde 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -615,6 +615,10 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex) int32_t index = (tsColumnIndex == -1) ? 0 : tsColumnIndex; SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, index); + if (pColInfoData == NULL) { + return 0; + } + if (pColInfoData->info.type != TSDB_DATA_TYPE_TIMESTAMP) { return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 272c0762d8..ec7d90184c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -123,7 +123,7 @@ static void tColRowGetPriamyKeyDeepCopy(SBlockData* pBlock, int32_t irow, int32_ pKey->pks[0].val = cv.value.val; } else { pKey->pks[0].nData = cv.value.nData; - memcpy(pKey->pks[0].pData, cv.value.pData, cv.value.nData); + /*void* p = */memcpy(pKey->pks[0].pData, cv.value.pData, cv.value.nData); } } @@ -185,6 +185,10 @@ static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pC if (IS_VAR_DATA_TYPE(pCols[i].type)) { pSupInfo->buildBuf[i] = taosMemoryMalloc(pCols[i].bytes); + if (pSupInfo->buildBuf[i] == NULL) { + tsdbError("failed to prepare memory for set columnId slot list, size:%d, code:out of memory", pCols[i].bytes); + return TSDB_CODE_OUT_OF_MEMORY; + } } else { pSupInfo->buildBuf[i] = NULL; } @@ -530,7 +534,10 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void // allocate buffer in order to load data blocks from file SBlockLoadSuppInfo* pSup = &pReader->suppInfo; pSup->tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID; - setColumnIdSlotList(pSup, pCond->colList, pCond->pSlotList, pCond->numOfCols); + code = setColumnIdSlotList(pSup, pCond->colList, pCond->pSlotList, pCond->numOfCols); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } code = initResBlockInfo(&pReader->resBlockInfo, capacity, pResBlock, pCond, pSup); if (code != TSDB_CODE_SUCCESS) { @@ -2496,8 +2503,8 @@ void updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInf pResBlock->info.dataLoad = 1; pResBlock->info.version = pReader->info.verRange.maxVer; - blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotId[0]); - blockDataUpdatePkRange(pResBlock, pReader->suppInfo.pkDstSlot, ASCENDING_TRAVERSE(pReader->info.order)); + int32_t code = blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotId[0]); + code = blockDataUpdatePkRange(pResBlock, pReader->suppInfo.pkDstSlot, ASCENDING_TRAVERSE(pReader->info.order)); setComposedBlockFlag(pReader, true); pReader->cost.composedBlocks += 1; @@ -3205,7 +3212,6 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader, int64_t en // set the correct start position in case of the first/last file block, according to the query time window static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) { - int64_t lastKey = ASCENDING_TRAVERSE(pReader->info.order) ? INT64_MIN : INT64_MAX; SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); SReaderStatus* pStatus = &pReader->status; SFileBlockDumpInfo* pDumpInfo = &pStatus->fBlockDumpInfo; @@ -5476,6 +5482,9 @@ void tsdbUntakeReadSnap2(STsdbReader* pReader, STsdbReadSnap* pSnap, bool proact void tsdbReaderSetId2(STsdbReader* pReader, const char* idstr) { taosMemoryFreeClear(pReader->idStr); pReader->idStr = taosStrdup(idstr); + if (pReader->idStr == NULL) { + // no need to do anything + } pReader->status.fileIter.pSttBlockReader->mergeTree.idStr = pReader->idStr; } From 3cdb957f224768dfde2b5016800709d449f136db Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 15 Jul 2024 15:32:11 +0800 Subject: [PATCH 12/34] adj function of stream operator result --- include/libs/executor/storageapi.h | 2 +- include/libs/stream/tstreamUpdate.h | 2 +- source/libs/executor/inc/executorInt.h | 2 +- source/libs/executor/inc/streamexecutorInt.h | 1 + source/libs/executor/src/groupoperator.c | 18 +- source/libs/executor/src/scanoperator.c | 69 +++++-- .../executor/src/streamcountwindowoperator.c | 90 ++++----- .../executor/src/streameventwindowoperator.c | 112 ++++++----- .../executor/src/streamtimewindowoperator.c | 185 ++++++++++-------- source/libs/stream/src/streamUpdate.c | 101 ++++++++-- 10 files changed, 368 insertions(+), 214 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 168760c61c..49d86f8c37 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -394,7 +394,7 @@ typedef struct SStateStore { int32_t (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo); void (*updateInfoDestoryColseWinSBF)(SUpdateInfo* pInfo); - int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo); + int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen); int32_t (*updateInfoDeserialize)(void* buf, int32_t bufLen, SUpdateInfo* pInfo); SStreamStateCur* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key); diff --git a/include/libs/stream/tstreamUpdate.h b/include/libs/stream/tstreamUpdate.h index 9de0d9ab68..e1b33de429 100644 --- a/include/libs/stream/tstreamUpdate.h +++ b/include/libs/stream/tstreamUpdate.h @@ -33,7 +33,7 @@ bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid); void updateInfoDestroy(SUpdateInfo* pInfo); void updateInfoAddCloseWindowSBF(SUpdateInfo* pInfo); void updateInfoDestoryColseWinSBF(SUpdateInfo* pInfo); -int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo); +int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen); int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo); void windowSBfDelete(SUpdateInfo* pInfo, uint64_t count); int32_t windowSBfAdd(SUpdateInfo* pInfo, uint64_t count); diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 9547fa12f3..6db3c780e2 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -929,7 +929,7 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, i SStreamState* pState, int32_t keySize, int16_t keyType, SStateStore* pStore, SReadHandle* pHandle, STimeWindowAggSupp* pTwAggSup, const char* taskIdStr, SStorageAPI* pApi, int32_t tsIndex); -void initDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, +int32_t initDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic); void getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins); void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); diff --git a/source/libs/executor/inc/streamexecutorInt.h b/source/libs/executor/inc/streamexecutorInt.h index 8b61b93601..c6053cc96e 100644 --- a/source/libs/executor/inc/streamexecutorInt.h +++ b/source/libs/executor/inc/streamexecutorInt.h @@ -20,6 +20,7 @@ extern "C" { #endif #include "executorInt.h" +#include "tutil.h" void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type); bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo); diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index ccb8e702bd..c7236a88e8 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -17,6 +17,7 @@ #include "function.h" #include "os.h" #include "tname.h" +#include "tutil.h" #include "tdatablock.h" #include "tmsg.h" @@ -1357,11 +1358,13 @@ static void destroyStreamPartitionOperatorInfo(void* param) { taosMemoryFreeClear(param); } -void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr, SExprSupp* pTbnameExpr) { +int32_t initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr, SExprSupp* pTbnameExpr) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStorageAPI* pAPI = &downstream->pTaskInfo->storageAPI; if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - return; + return code; } SStreamScanInfo* pScanInfo = downstream->info; @@ -1369,8 +1372,9 @@ void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup pScanInfo->pPartScalarSup = pExpr; pScanInfo->pPartTbnameSup = pTbnameExpr; if (!pScanInfo->pUpdateInfo) { - pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo); + code = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo); } + return code; } SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag) { @@ -1421,6 +1425,7 @@ void freePartItem(void* ptr) { SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamPartitionOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -1511,14 +1516,19 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr destroyStreamPartitionOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState); - initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup, &pInfo->tbnameCalSup); + code = initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup, &pInfo->tbnameCalSup); + TSDB_CHECK_CODE(code, lino, _error); + code = appendDownstream(pOperator, &downstream, 1); + TSDB_CHECK_CODE(code, lino, _error); + return pOperator; _error: pTaskInfo->code = code; destroyStreamPartitionOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return NULL; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ce4915ca4d..1e5e116d6b 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2477,27 +2477,54 @@ static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey, SSDataBlock* pBl } } -int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff) { - int32_t len = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo); - len += encodeSTimeWindowAggSupp(NULL, &pInfo->twAggSup); - *pBuff = taosMemoryCalloc(1, len); - void* buf = *pBuff; - encodeSTimeWindowAggSupp(&buf, &pInfo->twAggSup); - pInfo->stateStore.updateInfoSerialize(buf, len, pInfo->pUpdateInfo); - return len; +int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff, int32_t* pLen) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t len = 0; + code = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo, &len); + TSDB_CHECK_CODE(code, lino, _end); + + len += encodeSTimeWindowAggSupp(NULL, &pInfo->twAggSup); + *pBuff = taosMemoryCalloc(1, len); + if (!(*pBuff)) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + void* buf = *pBuff; + encodeSTimeWindowAggSupp(&buf, &pInfo->twAggSup); + int32_t tmp = 0; + code = pInfo->stateStore.updateInfoSerialize(buf, len, pInfo->pUpdateInfo, &tmp); + TSDB_CHECK_CODE(code, lino, _end); + + *pLen = len; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void streamScanOperatorSaveCheckpoint(SStreamScanInfo* pInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (!pInfo->pState) { return; } if (needSaveStreamOperatorInfo(&pInfo->basic)) { void* pBuf = NULL; - int32_t len = streamScanOperatorEncode(pInfo, &pBuf); + int32_t len = 0; + code = streamScanOperatorEncode(pInfo, &pBuf, &len); + TSDB_CHECK_CODE(code, lino, _end); pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_CHECKPOINT_NAME, strlen(STREAM_SCAN_OP_CHECKPOINT_NAME), pBuf, len); taosMemoryFree(pBuf); saveStreamOperatorStateComplete(&pInfo->basic); } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } // other properties are recovered from the execution plan @@ -3094,17 +3121,35 @@ static void destroyStreamScanOperatorInfo(void* param) { } void streamScanReleaseState(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamScanInfo* pInfo = pOperator->info; + void* pBuff = NULL; if (!pInfo->pState) { return; } if (!pInfo->pUpdateInfo) { return; } - int32_t len = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo); - void* pBuff = taosMemoryCalloc(1, len); - pInfo->stateStore.updateInfoSerialize(pBuff, len, pInfo->pUpdateInfo); + int32_t len = 0; + code = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo, &len); + TSDB_CHECK_CODE(code, lino, _end); + + pBuff = taosMemoryCalloc(1, len); + if (!pBuff) { + code = TSDB_CODE_OUT_OF_MEMORY; +TSDB_CHECK_CODE(code, lino, _end); + } + + int32_t tmp = 0; + code = pInfo->stateStore.updateInfoSerialize(pBuff, len, pInfo->pUpdateInfo, &tmp); + TSDB_CHECK_CODE(code, lino, _end); + pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_STATE_NAME, strlen(STREAM_SCAN_OP_STATE_NAME), pBuff, len); +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } taosMemoryFree(pBuff); } diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index 857d048457..14f2dfbada 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -68,9 +68,7 @@ void destroyStreamCountAggOperatorInfo(void* param) { taosMemoryFreeClear(param); } -bool isSlidingCountWindow(SStreamAggSupporter* pAggSup) { - return pAggSup->windowCount != pAggSup->windowSliding; -} +bool isSlidingCountWindow(SStreamAggSupporter* pAggSup) { return pAggSup->windowCount != pAggSup->windowSliding; } void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, SCountWindowInfo* pCurWin, SBuffInfo* pBuffInfo) { @@ -89,13 +87,14 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, ASSERT(pBuffInfo->pCur); pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pBuffInfo->pCur); code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin, - (void**)&pCurWin->winInfo.pStatePos, &size); + (void**)&pCurWin->winInfo.pStatePos, &size); if (code == TSDB_CODE_FAILED) { pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, (void**)&pCurWin->winInfo.pStatePos, &size); } } else { - pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount); + pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin, + pAggSup->windowCount); code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin, (void**)&pCurWin->winInfo.pStatePos, &size); if (code == TSDB_CODE_FAILED) { @@ -107,15 +106,16 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, pBuffInfo->rebuildWindow = true; } } else { - code = pAggSup->stateStore.streamStateCountWinAddIfNotExist( - pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos, &size); + code = pAggSup->stateStore.streamStateCountWinAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, + pAggSup->windowCount, + (void**)&pCurWin->winInfo.pStatePos, &size); } if (code == TSDB_CODE_SUCCESS) { pCurWin->winInfo.isOutput = true; } - pCurWin->pWindowCount= - (COUNT_TYPE*) ((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - sizeof(COUNT_TYPE))); + pCurWin->pWindowCount = + (COUNT_TYPE*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - sizeof(COUNT_TYPE))); if (*pCurWin->pWindowCount == pAggSup->windowCount) { pBuffInfo->rebuildWindow = true; @@ -133,7 +133,7 @@ static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowI int32_t start, int32_t rows, int32_t maxRows, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) { SSessionKey sWinKey = pWinInfo->winInfo.sessionWin; - int32_t num = 0; + int32_t num = 0; for (int32_t i = start; i < rows; i++) { if (pTs[i] < pWinInfo->winInfo.sessionWin.win.ekey) { num++; @@ -181,7 +181,7 @@ void getCountWinRange(SStreamAggSupporter* pAggSup, const SSessionKey* pKey, ESt pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pAggSup->pState, pKey); } SSessionKey tmpKey = {.groupId = pKey->groupId, .win.ekey = INT64_MIN, .win.skey = INT64_MIN}; - int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0); + int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0); if (code != TSDB_CODE_SUCCESS) { pAggSup->stateStore.streamStateFreeCur(pCur); return; @@ -239,7 +239,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl TSKEY* startTsCols = (int64_t*)pStartTsCol->pData; blockDataEnsureCapacity(pAggSup->pScanBlock, rows * 2); SStreamStateCur* pCur = NULL; - COUNT_TYPE slidingRows = 0; + COUNT_TYPE slidingRows = 0; for (int32_t i = 0; i < rows;) { if (pInfo->ignoreExpiredData && @@ -269,7 +269,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } getCountWinRange(pAggSup, &curWin.winInfo.sessionWin, STREAM_DELETE_DATA, &range); range.win.skey = TMIN(startTsCols[i], range.win.skey); - range.win.ekey = TMAX(startTsCols[rows-1], range.win.ekey); + range.win.ekey = TMAX(startTsCols[rows - 1], range.win.ekey); uint64_t uid = 0; appendDataToSpecialBlock(pAggSup->pScanBlock, &range.win.skey, &range.win.ekey, &uid, &range.groupId, NULL); break; @@ -289,8 +289,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) { code = saveResult(curWin.winInfo, pStUpdated); if (code != TSDB_CODE_SUCCESS) { - qError("%s do stream count aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo), - tstrerror(code)); + qError("%s do stream count aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); break; } } @@ -397,13 +396,14 @@ void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato int32_t mapSize = 0; buf = taosDecodeFixedI32(buf, &mapSize); for (int32_t i = 0; i < mapSize; i++) { - SSessionKey key = {0}; + SSessionKey key = {0}; SCountWindowInfo curWin = {0}; buf = decodeSSessionKey(buf, &key); SBuffInfo buffInfo = {.rebuildWindow = false, .winBuffOp = NONE_WINDOW, .pCur = NULL}; setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo); buf = decodeSResultWindowInfo(buf, &curWin.winInfo, pInfo->streamAggSup.resultRowSize); - tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); + tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, + sizeof(SResultWindowInfo)); } // 2.twAggSup @@ -441,9 +441,9 @@ void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) { uint64_t* gpDatas = (uint64_t*)pGroupCol->pData; SRowBuffPos* pPos = NULL; - int32_t size = 0; + int32_t size = 0; for (int32_t i = 0; i < pBlock->info.rows; i++) { - SSessionKey key = {.groupId = gpDatas[i], .win.skey = startDatas[i], .win.ekey = endDatas[i]}; + SSessionKey key = {.groupId = gpDatas[i], .win.skey = startDatas[i], .win.ekey = endDatas[i]}; SStreamStateCur* pCur = NULL; if (isSlidingCountWindow(pAggSup)) { pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &key, pAggSup->windowCount); @@ -452,7 +452,7 @@ void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) { } while (1) { SSessionKey tmpKey = {.groupId = gpDatas[i], .win.skey = INT64_MIN, .win.ekey = INT64_MIN}; - int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, (void **)&pPos, &size); + int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, (void**)&pPos, &size); if (code != TSDB_CODE_SUCCESS || tmpKey.win.skey > endDatas[i]) { pAggSup->stateStore.streamStateFreeCur(pCur); break; @@ -482,7 +482,7 @@ void doDeleteCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SAr SSessionKey key = {.win.skey = startDatas[i], .win.ekey = endDatas[i], .groupId = gpDatas[i]}; while (1) { SSessionKey curWin = {0}; - int32_t code = pAggSup->stateStore.streamStateCountGetKeyByRange(pAggSup->pState, &key, &curWin); + int32_t code = pAggSup->stateStore.streamStateCountGetKeyByRange(pAggSup->pState, &key, &curWin); if (code == TSDB_CODE_FAILED) { break; } @@ -511,11 +511,11 @@ void deleteCountWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHa } static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) { - SExprSupp* pSup = &pOperator->exprSupp; - SStreamCountAggOperatorInfo* pInfo = pOperator->info; - SOptrBasicInfo* pBInfo = &pInfo->binfo; - SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExprSupp* pSup = &pOperator->exprSupp; + SStreamCountAggOperatorInfo* pInfo = pOperator->info; + SOptrBasicInfo* pBInfo = &pInfo->binfo; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; qDebug("stask:%s %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status); if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -635,7 +635,7 @@ void streamCountReloadState(SOperatorInfo* pOperator) { int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_COUNT_OP_STATE_NAME, strlen(STREAM_COUNT_OP_STATE_NAME), &pBuf, &size); - TSKEY ts = *(TSKEY*)pBuf; + TSKEY ts = *(TSKEY*)pBuf; pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts); taosMemoryFree(pBuf); @@ -650,11 +650,13 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys SExecTaskInfo* pTaskInfo, SReadHandle* pHandle) { SCountWinodwPhysiNode* pCountNode = (SCountWinodwPhysiNode*)pPhyNode; int32_t numOfCols = 0; - int32_t code = TSDB_CODE_OUT_OF_MEMORY; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamCountAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamCountAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - goto _error; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } pOperator->pTaskInfo = pTaskInfo; @@ -664,18 +666,14 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pCountNode->window.pExprs, NULL, &numOfScalar); code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); } SExprSupp* pExpSup = &pOperator->exprSupp; SExprInfo* pExprInfo = createExprInfo(pCountNode->window.pFuncs, NULL, &numOfCols); SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); pInfo->twAggSup = (STimeWindowAggSupp){ .waterMark = pCountNode->window.watermark, @@ -686,12 +684,11 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys }; pInfo->primaryTsIndex = ((SColumnNode*)pCountNode->window.pTspk)->slotId; - code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, - pTaskInfo->streamInfo.pState, sizeof(COUNT_TYPE), 0, &pTaskInfo->storageAPI.stateStore, pHandle, - &pInfo->twAggSup, GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState, + sizeof(COUNT_TYPE), 0, &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup, + GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex); + TSDB_CHECK_CODE(code, lino, _error); + pInfo->streamAggSup.windowCount = pCountNode->windowCount; pInfo->streamAggSup.windowSliding = pCountNode->windowSliding; @@ -709,7 +706,8 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->dataVersion = 0; pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey)); if (!pInfo->historyWins) { - goto _error; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT); @@ -735,8 +733,12 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys setOperatorStreamStateFn(pOperator, streamCountReleaseState, streamCountReloadState); if (downstream) { - initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic); + code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, + &pInfo->twAggSup, &pInfo->basic); + TSDB_CHECK_CODE(code, lino, _error); + code = appendDownstream(pOperator, &downstream, 1); + TSDB_CHECK_CODE(code, lino, _error); } return pOperator; @@ -747,6 +749,6 @@ _error: taosMemoryFreeClear(pOperator); pTaskInfo->code = code; + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return NULL; } - diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 1809d0cc67..e6ff4fe242 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -28,9 +28,9 @@ #include "tlog.h" #include "ttime.h" -#define IS_NORMAL_EVENT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT) -#define STREAM_EVENT_OP_STATE_NAME "StreamEventHistoryState" -#define STREAM_EVENT_OP_CHECKPOINT_NAME "StreamEventOperator_Checkpoint" +#define IS_NORMAL_EVENT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT) +#define STREAM_EVENT_OP_STATE_NAME "StreamEventHistoryState" +#define STREAM_EVENT_OP_CHECKPOINT_NAME "StreamEventOperator_Checkpoint" typedef struct SEventWinfowFlag { bool startFlag; @@ -86,10 +86,11 @@ void destroyStreamEventOperatorInfo(void* param) { void setEventWindowFlag(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo) { char* pFlagInfo = (char*)pWinInfo->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize); - pWinInfo->pWinFlag = (SEventWinfowFlag*) pFlagInfo; + pWinInfo->pWinFlag = (SEventWinfowFlag*)pFlagInfo; } -void setEventWindowInfo(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SRowBuffPos* pPos, SEventWindowInfo* pWinInfo) { +void setEventWindowInfo(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SRowBuffPos* pPos, + SEventWindowInfo* pWinInfo) { pWinInfo->winInfo.sessionWin = *pKey; pWinInfo->winInfo.pStatePos = pPos; setEventWindowFlag(pAggSup, pWinInfo); @@ -122,22 +123,23 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI pCurWin->winInfo.sessionWin.groupId = groupId; pCurWin->winInfo.sessionWin.win.skey = ts; pCurWin->winInfo.sessionWin.win.ekey = ts; - SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin); - SSessionKey leftWinKey = {.groupId = groupId}; - void* pVal = NULL; - int32_t len = 0; + SStreamStateCur* pCur = + pAggSup->stateStore.streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin); + SSessionKey leftWinKey = {.groupId = groupId}; + void* pVal = NULL; + int32_t len = 0; code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &leftWinKey, &pVal, &len); - if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &leftWinKey.win) ) { + if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &leftWinKey.win)) { bool inWin = isInTimeWindow(&leftWinKey.win, ts, 0); setEventWindowInfo(pAggSup, &leftWinKey, pVal, pCurWin); - if(inWin || (pCurWin->pWinFlag->startFlag && !pCurWin->pWinFlag->endFlag) ) { + if (inWin || (pCurWin->pWinFlag->startFlag && !pCurWin->pWinFlag->endFlag)) { pCurWin->winInfo.isOutput = !isWindowIncomplete(pCurWin); goto _end; } } pAggSup->stateStore.streamStateFreeCur(pCur); pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin); - SSessionKey rightWinKey = {.groupId = groupId}; + SSessionKey rightWinKey = {.groupId = groupId}; code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &rightWinKey, &pVal, &len); bool inWin = isInTimeWindow(&rightWinKey.win, ts, 0); if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &rightWinKey.win) && (inWin || (start && !end))) { @@ -149,7 +151,7 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI } } - SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId}; + SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId}; pAggSup->stateStore.streamStateSessionAllocWinBuffByNextPosition(pAggSup->pState, pCur, &winKey, &pVal, &len); setEventWindowInfo(pAggSup, &winKey, pVal, pCurWin); pCurWin->pWinFlag->startFlag = start; @@ -164,7 +166,7 @@ _end: if (code != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_KEY_INVALID(pNextWinKey); } - if(pCurWin->winInfo.pStatePos->needFree) { + if (pCurWin->winInfo.pStatePos->needFree) { pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->winInfo.sessionWin); } pAggSup->stateStore.streamStateFreeCur(pCur); @@ -173,14 +175,14 @@ _end: } int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo, SSessionKey* pNextWinKey, - TSKEY* pTsData, bool* starts, bool* ends, int32_t rows, int32_t start, SSHashObj* pResultRows, - SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) { + TSKEY* pTsData, bool* starts, bool* ends, int32_t rows, int32_t start, + SSHashObj* pResultRows, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) { *pRebuild = false; - if (!pWinInfo->pWinFlag->startFlag && !(starts[start]) ) { + if (!pWinInfo->pWinFlag->startFlag && !(starts[start])) { return 1; } - TSKEY maxTs = INT64_MAX; + TSKEY maxTs = INT64_MAX; STimeWindow* pWin = &pWinInfo->winInfo.sessionWin.win; if (pWinInfo->pWinFlag->endFlag) { maxTs = pWin->ekey + 1; @@ -219,7 +221,7 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW memcpy(pWinInfo->winInfo.pStatePos->pKey, &pWinInfo->winInfo.sessionWin, sizeof(SSessionKey)); if (ends[i]) { - if (pWinInfo->pWinFlag->endFlag && pWin->skey <= pTsData[i] && pTsData[i] < pWin->ekey ) { + if (pWinInfo->pWinFlag->endFlag && pWin->skey <= pTsData[i] && pTsData[i] < pWin->ekey) { *pRebuild = true; } return i + 1 - start; @@ -230,26 +232,28 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW static int32_t compactEventWindow(SOperatorInfo* pOperator, SEventWindowInfo* pCurWin, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool addGap) { - SExprSupp* pSup = &pOperator->exprSupp; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - int32_t winNum = 0; - SStreamEventAggOperatorInfo* pInfo = pOperator->info; - SResultRow* pCurResult = NULL; - int32_t numOfOutput = pOperator->exprSupp.numOfExprs; - SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + SExprSupp* pSup = &pOperator->exprSupp; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + int32_t winNum = 0; + SStreamEventAggOperatorInfo* pInfo = pOperator->info; + SResultRow* pCurResult = NULL; + int32_t numOfOutput = pOperator->exprSupp.numOfExprs; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; while (1) { if (!pCurWin->pWinFlag->startFlag || pCurWin->pWinFlag->endFlag) { break; } SEventWindowInfo nextWinInfo = {0}; getNextSessionWinInfo(pAggSup, pStUpdated, &pCurWin->winInfo, &nextWinInfo.winInfo); - if (!IS_VALID_SESSION_WIN(nextWinInfo.winInfo) || !inWinRange(&pAggSup->winRange, &nextWinInfo.winInfo.sessionWin.win)) { + if (!IS_VALID_SESSION_WIN(nextWinInfo.winInfo) || + !inWinRange(&pAggSup->winRange, &nextWinInfo.winInfo.sessionWin.win)) { releaseOutputBuf(pAggSup->pState, nextWinInfo.winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore); break; } setEventWindowFlag(pAggSup, &nextWinInfo); - compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, &pCurWin->winInfo, &nextWinInfo.winInfo, pStUpdated, pStDeleted, false); + compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, &pCurWin->winInfo, &nextWinInfo.winInfo, pStUpdated, + pStDeleted, false); pCurWin->pWinFlag->endFlag = nextWinInfo.pWinFlag->endFlag; winNum++; } @@ -275,7 +279,7 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl int32_t winRows = 0; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; SColumnInfoData* pColStart = NULL; - SColumnInfoData* pColEnd = NULL; + SColumnInfoData* pColEnd = NULL; pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version); pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow; @@ -290,7 +294,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl return; } - SFilterColumnParam paramStart = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), .pDataBlock = pSDataBlock->pDataBlock}; + SFilterColumnParam paramStart = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), + .pDataBlock = pSDataBlock->pDataBlock}; code = filterSetDataFromSlotId(pInfo->pStartCondInfo, ¶mStart); if (code != TSDB_CODE_SUCCESS) { qError("set data from start slotId error."); @@ -299,7 +304,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl int32_t statusStart = 0; filterExecute(pInfo->pStartCondInfo, pSDataBlock, &pColStart, NULL, paramStart.numOfCols, &statusStart); - SFilterColumnParam paramEnd = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), .pDataBlock = pSDataBlock->pDataBlock}; + SFilterColumnParam paramEnd = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), + .pDataBlock = pSDataBlock->pDataBlock}; code = filterSetDataFromSlotId(pInfo->pEndCondInfo, ¶mEnd); if (code != TSDB_CODE_SUCCESS) { qError("set data from end slotId error."); @@ -320,20 +326,23 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl int32_t winIndex = 0; bool allEqual = true; SEventWindowInfo curWin = {0}; - SSessionKey nextWinKey = {0}; - setEventOutputBuf(pAggSup, tsCols, groupId, (bool*)pColStart->pData, (bool*)pColEnd->pData, i, rows, &curWin, &nextWinKey); + SSessionKey nextWinKey = {0}; + setEventOutputBuf(pAggSup, tsCols, groupId, (bool*)pColStart->pData, (bool*)pColEnd->pData, i, rows, &curWin, + &nextWinKey); setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo); bool rebuild = false; - winRows = updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData, rows, i, - pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild); + winRows = + updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData, + rows, i, pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild); ASSERT(winRows >= 1); if (rebuild) { uint64_t uid = 0; appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey, - &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); + &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); doDeleteEventWindow(pAggSup, pSeUpdated, &curWin.winInfo.sessionWin); - if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator) && !isWindowIncomplete(&curWin)) { + if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator) && + !isWindowIncomplete(&curWin)) { saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin); } releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore); @@ -603,7 +612,8 @@ void streamEventReleaseState(SOperatorInfo* pOperator) { char* pBuff = taosMemoryCalloc(1, resSize); memcpy(pBuff, pInfo->historyWins->pData, winSize); memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY)); - qDebug("===stream=== event window operator relase state. save result count:%d", (int32_t)taosArrayGetSize(pInfo->historyWins)); + qDebug("===stream=== event window operator relase state. save result count:%d", + (int32_t)taosArrayGetSize(pInfo->historyWins)); pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_STATE_NAME, strlen(STREAM_EVENT_OP_STATE_NAME), pBuff, resSize); pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState); @@ -647,7 +657,7 @@ void streamEventReloadState(SOperatorInfo* pOperator) { qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey, pSeKeyBuf[i].groupId, i); getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo.winInfo); - //event window has been deleted + // event window has been deleted if (!IS_VALID_SESSION_WIN(curInfo.winInfo)) { continue; } @@ -659,7 +669,7 @@ void streamEventReloadState(SOperatorInfo* pOperator) { compactEventWindow(pOperator, &curInfo, pInfo->pSeUpdated, pInfo->pSeDeleted, false); qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey, - curInfo.winInfo.sessionWin.groupId); + curInfo.winInfo.sessionWin.groupId); if (IS_VALID_SESSION_WIN(curInfo.winInfo)) { saveSessionOutputBuf(pAggSup, &curInfo.winInfo); } @@ -693,6 +703,7 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys SStreamEventWinodwPhysiNode* pEventNode = (SStreamEventWinodwPhysiNode*)pPhyNode; int32_t tsSlotId = ((SColumnNode*)pEventNode->window.pTspk)->slotId; int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamEventAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamEventAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -783,20 +794,18 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAgg, NULL, destroyStreamEventOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamEventReleaseState, streamEventReloadState); - initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic); + code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, + &pInfo->twAggSup, &pInfo->basic); + TSDB_CHECK_CODE(code, lino, _error); + code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); code = filterInitFromNode((SNode*)pEventNode->pStartCond, &pInfo->pStartCondInfo, 0); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); + code = filterInitFromNode((SNode*)pEventNode->pEndCond, &pInfo->pEndCondInfo, 0); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); return pOperator; @@ -804,5 +813,6 @@ _error: destroyStreamEventOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); pTaskInfo->code = code; + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return NULL; } diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 97cb20a7ea..1650518f84 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -28,14 +28,18 @@ #include "tlog.h" #include "ttime.h" -#define IS_FINAL_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) -#define IS_MID_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) -#define IS_NORMAL_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) +#define IS_FINAL_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) +#define IS_MID_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) +#define IS_NORMAL_INTERVAL_OP(op) \ + ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || \ + (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) -#define IS_FINAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) -#define IS_NORMAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) +#define IS_FINAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) +#define IS_NORMAL_SESSION_OP(op) \ + ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || \ + (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) -#define IS_NORMAL_STATE_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) +#define IS_NORMAL_STATE_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) #define DEAULT_DELETE_MARK INT64_MAX #define STREAM_INTERVAL_OP_STATE_NAME "StreamIntervalHistoryState" @@ -263,14 +267,16 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa if (chIds) { int32_t childId = getChildIndex(pBlock); if (pInvalidWins) { - qDebug("===stream===save invalid delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d", winRes.ts, winRes.groupId, childId); + qDebug("===stream===save invalid delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d", winRes.ts, + winRes.groupId, childId); taosHashPut(pInvalidWins, &winRes, sizeof(SWinKey), NULL, 0); } SArray* chArray = *(void**)chIds; int32_t index = taosArraySearchIdx(chArray, &childId, compareInt32Val, TD_EQ); if (index != -1) { - qDebug("===stream===try push delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d ,continue", win.skey, winGpId, childId); + qDebug("===stream===try push delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d ,continue", win.skey, + winGpId, childId); getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); continue; } @@ -390,7 +396,7 @@ static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWin } void destroyFlusedPos(void* pRes) { - SRowBuffPos* pPos = (SRowBuffPos*) pRes; + SRowBuffPos* pPos = (SRowBuffPos*)pRes; if (!pPos->needFree && !pPos->pRowBuff) { taosMemoryFreeClear(pPos->pKey); taosMemoryFree(pPos); @@ -626,17 +632,17 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB blockDataUpdateTsWindow(pBlock, 0); } -static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval, SArray* pPullWins, - int32_t numOfCh, SOperatorInfo* pOperator) { +static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval, + SArray* pPullWins, int32_t numOfCh, SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; - SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); - TSKEY* tsData = (TSKEY*)pStartCol->pData; - SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); - TSKEY* tsEndData = (TSKEY*)pEndCol->pData; - SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); - uint64_t* groupIdData = (uint64_t*)pGroupCol->pData; - int32_t chId = getChildIndex(pBlock); - bool res = false; + SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); + TSKEY* tsData = (TSKEY*)pStartCol->pData; + SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); + TSKEY* tsEndData = (TSKEY*)pEndCol->pData; + SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); + uint64_t* groupIdData = (uint64_t*)pGroupCol->pData; + int32_t chId = getChildIndex(pBlock); + bool res = false; for (int32_t i = 0; i < pBlock->info.rows; i++) { TSKEY winTs = tsData[i]; while (winTs <= tsEndData[i]) { @@ -652,7 +658,7 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina // pull data is over taosArrayDestroy(chArray); taosHashRemove(pMap, &winRes, sizeof(SWinKey)); - res =true; + res = true; qDebug("===stream===retrive pull data over.window %" PRId64, winRes.ts); void* pFinalCh = taosHashGet(pFinalMap, &winRes, sizeof(SWinKey)); @@ -665,7 +671,8 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey}; // add pull data request - qDebug("===stream===prepare final retrive for delete window:%" PRId64 ",groupId:%" PRId64 ", size:%d", winRes.ts, winRes.groupId, numOfCh); + qDebug("===stream===prepare final retrive for delete window:%" PRId64 ",groupId:%" PRId64 ", size:%d", + winRes.ts, winRes.groupId, numOfCh); if (IS_MID_INTERVAL_OP(pOperator)) { SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info; taosArrayPush(pInfo->pMidPullDatas, &winRes); @@ -673,7 +680,7 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina taosArrayPush(pInfo->pDelWins, &winRes); addPullWindow(pMap, &winRes, numOfCh); if (pInfo->destHasPrimaryKey) { - tSimpleHashPut(pInfo->pDeletedMap,&winRes, sizeof(SWinKey), NULL, 0); + tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0); } qDebug("===stream===prepare final retrive for delete %" PRId64 ", size:%d", winRes.ts, numOfCh); } @@ -692,7 +699,7 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, i for (int32_t i = 0; i < size; i++) { SWinKey* winKey = taosArrayGet(wins, i); STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval); - void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey)); + void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey)); if (!chIds) { SPullWindowInfo pull = { .window = nextWin, .groupId = winKey->groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey}; @@ -700,7 +707,7 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, i if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) { addPullWindow(pInfo->pPullDataMap, winKey, pInfo->numOfChild); if (pInfo->destHasPrimaryKey) { - tSimpleHashPut(pInfo->pDeletedMap,winKey, sizeof(SWinKey), NULL, 0); + tSimpleHashPut(pInfo->pDeletedMap, winKey, sizeof(SWinKey), NULL, 0); } qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, pInfo->numOfChild); } @@ -753,7 +760,8 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDat if (pBlock->info.id.groupId == 0) { pBlock->info.id.groupId = groupId; void* tbname = NULL; - if (pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname, false) < 0) { + if (pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname, + false) < 0) { pBlock->info.parTbName[0] = 0; } else { memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); @@ -831,9 +839,7 @@ static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pN return startPos; } -bool hasSrcPrimaryKeyCol(SSteamOpBasicInfo* pInfo) { - return pInfo->primaryPkIndex != -1; -} +bool hasSrcPrimaryKeyCol(SSteamOpBasicInfo* pInfo) { return pInfo->primaryPkIndex != -1; } static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, uint64_t groupId, SSHashObj* pUpdatedMap, SSHashObj* pDeletedMap) { @@ -854,23 +860,22 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); tsCols = (int64_t*)pColDataInfo->pData; - void* pPkVal = NULL; - int32_t pkLen = 0; + void* pPkVal = NULL; + int32_t pkLen = 0; SColumnInfoData* pPkColDataInfo = NULL; if (hasSrcPrimaryKeyCol(&pInfo->basic)) { pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex); } if (pSDataBlock->info.window.skey != tsCols[0] || pSDataBlock->info.window.ekey != tsCols[endRowId]) { - qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 - ",maxKey %" PRId64, - pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); + qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 ",maxKey %" PRId64, + pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); blockDataUpdateTsWindow(pSDataBlock, pInfo->primaryTsIndex); // timestamp of the data is incorrect if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) { qError("table uid %" PRIu64 " data block timestamp is out of range! minKey %" PRId64 ",maxKey %" PRId64, - pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); + pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); } } @@ -918,7 +923,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) { addPullWindow(pInfo->pPullDataMap, &winRes, pInfo->numOfChild); if (pInfo->destHasPrimaryKey) { - tSimpleHashPut(pInfo->pDeletedMap,&winRes, sizeof(SWinKey), NULL, 0); + tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0); } } } else { @@ -964,7 +969,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat }; if (pInfo->destHasPrimaryKey && code == TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator)) { - tSimpleHashPut(pDeletedMap,&key, sizeof(SWinKey), NULL, 0); + tSimpleHashPut(pDeletedMap, &key, sizeof(SWinKey), NULL, 0); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { @@ -1367,7 +1372,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { pInfo->binfo.pRes->info.type = pBlock->info.type; } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || pBlock->info.type == STREAM_CLEAR) { - SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); + SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); SHashObj* finalMap = IS_FINAL_INTERVAL_OP(pOperator) ? pInfo->pFinalPullDataMap : NULL; doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, finalMap); if (IS_FINAL_INTERVAL_OP(pOperator)) { @@ -1401,7 +1406,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_RETRIEVE) { - if(!IS_FINAL_INTERVAL_OP(pOperator)) { + if (!IS_FINAL_INTERVAL_OP(pOperator)) { pInfo->recvRetrive = true; copyDataBlock(pInfo->pMidRetriveRes, pBlock); pInfo->pMidRetriveRes->info.type = STREAM_MID_RETRIEVE; @@ -1527,8 +1532,8 @@ void streamIntervalReloadState(SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL && pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) { - int32_t size = 0; - void* pBuf = NULL; + int32_t size = 0; + void* pBuf = NULL; int32_t code = pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME, strlen(STREAM_INTERVAL_OP_STATE_NAME), &pBuf, &size); if (code == 0) { @@ -1602,7 +1607,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsIndex); code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, - pInfo->pState, &pTaskInfo->storageAPI.functionStore); + pInfo->pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1757,27 +1762,37 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num } } -void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, - STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic) { +int32_t initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, + STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) { SStreamPartitionOperatorInfo* pScanInfo = downstream->info; pScanInfo->tsColIndex = tsColIndex; } if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup, pBasic); - return; + code = initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup, pBasic); + return code; } SStreamScanInfo* pScanInfo = downstream->info; pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type}; pScanInfo->pState = pAggSup->pState; if (!pScanInfo->pUpdateInfo) { - pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark, - pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo); + code = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark, + pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, + &pScanInfo->pUpdateInfo); + TSDB_CHECK_CODE(code, lino, _end); } pScanInfo->twAggSup = *pTwSup; pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo; pBasic->primaryPkIndex = pScanInfo->primaryKeyIndex; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static TSKEY sesionTs(void* pKey) { @@ -2150,8 +2165,8 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData TSKEY* endTsCols = (int64_t*)pEndTsCol->pData; - void* pPkVal = NULL; - int32_t pkLen = 0; + void* pPkVal = NULL; + int32_t pkLen = 0; SColumnInfoData* pPkColDataInfo = NULL; if (hasSrcPrimaryKeyCol(&pInfo->basic)) { pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex); @@ -2456,8 +2471,8 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa pBlock->info.id.groupId = pKey->groupId; void* tbname = NULL; - if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, - &tbname, false) < 0) { + if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname, + false) < 0) { pBlock->info.parTbName[0] = 0; } else { memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); @@ -2937,7 +2952,7 @@ void streamSessionSemiReloadState(SOperatorInfo* pOperator) { int32_t size = 0; void* pBuf = NULL; int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, - strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); + strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); @@ -2970,7 +2985,7 @@ void streamSessionReloadState(SOperatorInfo* pOperator) { int32_t size = 0; void* pBuf = NULL; int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, - strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); + strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); @@ -3020,6 +3035,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; int32_t numOfCols = 0; int32_t code = TSDB_CODE_OUT_OF_MEMORY; + int32_t lino = 0; SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -3114,8 +3130,12 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState); if (downstream) { - initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic); + code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, + &pInfo->twAggSup, &pInfo->basic); + TSDB_CHECK_CODE(code, lino, _error); + code = appendDownstream(pOperator, &downstream, 1); + TSDB_CHECK_CODE(code, lino, _error); } return pOperator; @@ -3126,6 +3146,7 @@ _error: taosMemoryFreeClear(pOperator); pTaskInfo->code = code; + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return NULL; } @@ -3576,7 +3597,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl if (!allEqual) { uint64_t uid = 0; appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey, - &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); + &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin); releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore); @@ -3683,9 +3704,9 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato SSessionKey key = {0}; SResultWindowInfo winfo = {0}; buf = decodeSSessionKey(buf, &key); - pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &winfo.sessionWin, NULL, - pAggSup->stateKeySize, compareStateKey, - (void**)&winfo.pStatePos, &pAggSup->resultRowSize); + pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &winfo.sessionWin, NULL, pAggSup->stateKeySize, + compareStateKey, (void**)&winfo.pStatePos, + &pAggSup->resultRowSize); buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); } @@ -3879,7 +3900,7 @@ void streamStateReloadState(SOperatorInfo* pOperator) { int32_t size = 0; void* pBuf = NULL; int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_STATE_OP_STATE_NAME, - strlen(STREAM_STATE_OP_STATE_NAME), &pBuf, &size); + strlen(STREAM_STATE_OP_STATE_NAME), &pBuf, &size); int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); qDebug("===stream=== reload state. get result count:%d", num); SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; @@ -3943,12 +3964,13 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; SColumnNode* pColNode = (SColumnNode*)(pStateNode->pStateKey); int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _error; + TSDB_CHECK_CODE(code, lino, _error); } pInfo->stateCol = extractColumnFromColumnNode(pColNode); @@ -3957,9 +3979,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar); code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); } pInfo->twAggSup = (STimeWindowAggSupp){ @@ -3986,9 +4006,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState, keySize, type, &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup, GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pInfo->pSeDeleted = tSimpleHashInit(64, hashFn); pInfo->pDelIterator = NULL; @@ -4001,7 +4020,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->dataVersion = 0; pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey)); if (!pInfo->historyWins) { - goto _error; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } if (pHandle) { pInfo->isHistoryOp = pHandle->fillHistory; @@ -4028,17 +4048,20 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState); - initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic); + code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, + &pInfo->twAggSup, &pInfo->basic); + TSDB_CHECK_CODE(code, lino, _error); + code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); + return pOperator; _error: destroyStreamStateOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); pTaskInfo->code = code; + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return NULL; } @@ -4205,11 +4228,12 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision, }; - pInfo->twAggSup = (STimeWindowAggSupp){.waterMark = pIntervalPhyNode->window.watermark, - .calTrigger = pIntervalPhyNode->window.triggerType, - .maxTs = INT64_MIN, - .minTs = INT64_MAX, - .deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval)}; + pInfo->twAggSup = + (STimeWindowAggSupp){.waterMark = pIntervalPhyNode->window.watermark, + .calTrigger = pIntervalPhyNode->window.triggerType, + .maxTs = INT64_MIN, + .minTs = INT64_MAX, + .deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval)}; ASSERTS(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay"); @@ -4337,7 +4361,7 @@ static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS .ts = nextWin.skey, .groupId = groupId, }; - void* chIds = taosHashGet(pInfo->pPullDataMap, &key, sizeof(SWinKey)); + void* chIds = taosHashGet(pInfo->pPullDataMap, &key, sizeof(SWinKey)); int32_t index = -1; SArray* chArray = NULL; int32_t chId = 0; @@ -4409,8 +4433,8 @@ static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS static void addMidRetriveWindow(SArray* wins, SHashObj* pMidPullMap, int32_t numOfChild) { int32_t size = taosArrayGetSize(wins); for (int32_t i = 0; i < size; i++) { - SWinKey* winKey = taosArrayGet(wins, i); - void* chIds = taosHashGet(pMidPullMap, winKey, sizeof(SWinKey)); + SWinKey* winKey = taosArrayGet(wins, i); + void* chIds = taosHashGet(pMidPullMap, winKey, sizeof(SWinKey)); if (!chIds) { addPullWindow(pMidPullMap, winKey, numOfChild); qDebug("===stream===prepare mid operator retrive for delete %" PRId64 ", size:%d", winKey->ts, numOfChild); @@ -4613,5 +4637,6 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) { void setStreamOperatorCompleted(SOperatorInfo* pOperator) { setOperatorCompleted(pOperator); - qDebug("stask:%s %s status: %d. set completed", GET_TASKID(pOperator->pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status); + qDebug("stask:%s %s status: %d. set completed", GET_TASKID(pOperator->pTaskInfo), + getStreamOpName(pOperator->operatorType), pOperator->status); } diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 8727ce0174..0dc1eafb03 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -304,6 +304,7 @@ TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t p len = colDataGetRowLength(pPkDataInfo, i); } int32_t buffLen = getKeyBuff(ts, tbUid, pPkVal, len, pInfo->pKeyBuff); + // we don't care whether the data is updated or not tScalableBfPut(pSBf, pInfo->pKeyBuff, buffLen); } } @@ -413,59 +414,119 @@ void updateInfoDestoryColseWinSBF(SUpdateInfo* pInfo) { pInfo->pCloseWinSBF = NULL; } -int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo) { +int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (!pInfo) { - return 0; + return TSDB_CODE_SUCCESS; } SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); - if (tStartEncode(&encoder) < 0) return -1; + if (tStartEncode(&encoder) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } int32_t size = taosArrayGetSize(pInfo->pTsBuckets); - if (tEncodeI32(&encoder, size) < 0) return -1; + if (tEncodeI32(&encoder, size) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + for (int32_t i = 0; i < size; i++) { TSKEY* pTs = (TSKEY*)taosArrayGet(pInfo->pTsBuckets, i); - if (tEncodeI64(&encoder, *pTs) < 0) return -1; + if (tEncodeI64(&encoder, *pTs) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } } - if (tEncodeU64(&encoder, pInfo->numBuckets) < 0) return -1; + if (tEncodeU64(&encoder, pInfo->numBuckets) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } int32_t sBfSize = taosArrayGetSize(pInfo->pTsSBFs); - if (tEncodeI32(&encoder, sBfSize) < 0) return -1; + if (tEncodeI32(&encoder, sBfSize) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } for (int32_t i = 0; i < sBfSize; i++) { SScalableBf* pSBf = taosArrayGetP(pInfo->pTsSBFs, i); - if (tScalableBfEncode(pSBf, &encoder) < 0) return -1; + if (tScalableBfEncode(pSBf, &encoder) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } } - if (tEncodeU64(&encoder, pInfo->numSBFs) < 0) return -1; - if (tEncodeI64(&encoder, pInfo->interval) < 0) return -1; - if (tEncodeI64(&encoder, pInfo->watermark) < 0) return -1; - if (tEncodeI64(&encoder, pInfo->minTS) < 0) return -1; + if (tEncodeU64(&encoder, pInfo->numSBFs) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + if (tEncodeI64(&encoder, pInfo->interval) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + if (tEncodeI64(&encoder, pInfo->watermark) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + if (tEncodeI64(&encoder, pInfo->minTS) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } - if (tScalableBfEncode(pInfo->pCloseWinSBF, &encoder) < 0) return -1; + if (tScalableBfEncode(pInfo->pCloseWinSBF, &encoder) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } int32_t mapSize = taosHashGetSize(pInfo->pMap); - if (tEncodeI32(&encoder, mapSize) < 0) return -1; + if (tEncodeI32(&encoder, mapSize) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } void* pIte = NULL; size_t keyLen = 0; while ((pIte = taosHashIterate(pInfo->pMap, pIte)) != NULL) { void* key = taosHashGetKey(pIte, &keyLen); - if (tEncodeU64(&encoder, *(uint64_t*)key) < 0) return -1; + if (tEncodeU64(&encoder, *(uint64_t*)key) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } int32_t valueSize = taosHashGetValueSize(pIte); - if (tEncodeBinary(&encoder, (const uint8_t*)pIte, valueSize) < 0) return -1; + if (tEncodeBinary(&encoder, (const uint8_t*)pIte, valueSize) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } } - if (tEncodeU64(&encoder, pInfo->maxDataVersion) < 0) return -1; + if (tEncodeU64(&encoder, pInfo->maxDataVersion) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } - if (tEncodeI32(&encoder, pInfo->pkColLen) < 0) return -1; - if (tEncodeI8(&encoder, pInfo->pkColType) < 0) return -1; + if (tEncodeI32(&encoder, pInfo->pkColLen) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + if (tEncodeI8(&encoder, pInfo->pkColType) < 0) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } tEndEncode(&encoder); int32_t tlen = encoder.pos; tEncoderClear(&encoder); - return tlen; + *pLen = tlen; + +_end: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) { From 51000ffe7fdeeb7e6132229429cb74992ce0ef50 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Thu, 11 Jul 2024 19:03:33 +0800 Subject: [PATCH 13/34] fix: parseOneStbRow support for jsontag --- source/libs/parser/src/parTranslater.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 9b1c8096aa..104ffcc945 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -12815,6 +12815,7 @@ static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt) int32_t numOfTags = getNumOfTags(pParFileCxt->pStbMeta); uint8_t precision = getTableInfo(pParFileCxt->pStbMeta).precision; SSchema* pSchemas = getTableTagSchema(pParFileCxt->pStbMeta); + bool isJson = false; for (int i = 0; i < sz; i++) { const char* pSql = pParFileCxt->pSql; @@ -12832,6 +12833,7 @@ static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt) // parse tag const SSchema* pTagSchema = &pSchemas[index]; + isJson = (pTagSchema->type == TSDB_DATA_TYPE_JSON); code = checkAndTrimValue(&token, pParFileCxt->tmpTokenBuf, pMsgBuf, pTagSchema->type); if (TSDB_CODE_SUCCESS == code && TK_NK_VARIABLE == token.type) { code = buildInvalidOperationMsg(pMsgBuf, "not expected row value"); @@ -12859,7 +12861,9 @@ static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt) if (TSDB_CODE_SUCCESS == code) { pParFileCxt->tagNameFilled = true; - code = tTagNew(pParFileCxt->aTagVals, 1, false, &pParFileCxt->pTag); + if (!isJson) { + code = tTagNew(pParFileCxt->aTagVals, 1, false, &pParFileCxt->pTag); + } } return code; From b5ccc3db92ef7ae268c6d9f22baee895aae8916b Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 15 Jul 2024 18:19:05 +0800 Subject: [PATCH 14/34] adj stream function result --- source/libs/stream/src/streamUpdate.c | 7 +++++-- source/util/src/tbloomfilter.c | 3 ++- source/util/src/tscalablebf.c | 3 ++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 0dc1eafb03..61b7e02a4c 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -563,9 +563,12 @@ int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) { if (tDecodeI64(&decoder, &pInfo->interval) < 0) return -1; if (tDecodeI64(&decoder, &pInfo->watermark) < 0) return -1; if (tDecodeI64(&decoder, &pInfo->minTS) < 0) return -1; - pInfo->pCloseWinSBF = NULL; + code = tScalableBfDecode(&decoder, &pInfo->pCloseWinSBF); - TSDB_CHECK_CODE(code, lino, _error); + if (code != TSDB_CODE_SUCCESS) { + pInfo->pCloseWinSBF = NULL; + code = TSDB_CODE_SUCCESS; + } int32_t mapSize = 0; if (tDecodeI32(&decoder, &mapSize) < 0) return -1; diff --git a/source/util/src/tbloomfilter.c b/source/util/src/tbloomfilter.c index 9093fb9d91..1cf4ecf72e 100644 --- a/source/util/src/tbloomfilter.c +++ b/source/util/src/tbloomfilter.c @@ -188,11 +188,12 @@ int32_t tBloomFilterDecode(SDecoder* pDecoder, SBloomFilter** ppBF) { pBF->hashFn1 = HASH_FUNCTION_1; pBF->hashFn2 = HASH_FUNCTION_2; (*ppBF) = pBF; + return TSDB_CODE_SUCCESS; _error: tBloomFilterDestroy(pBF); uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); - return TSDB_CODE_FAILED; + return code; } bool tBloomFilterIsFull(const SBloomFilter* pBF) { return pBF->size >= pBF->expectedEntries; } diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index 0979000981..4a4dfd2653 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -254,8 +254,9 @@ int32_t tScalableBfDecode(SDecoder* pDecoder, SScalableBf** ppSBf) { (*ppSBf) = pSBf; _error: - tScalableBfDestroy(pSBf); + if (code != TSDB_CODE_SUCCESS) { + tScalableBfDestroy(pSBf); uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } return code; From aed8a75e357fea02a40ac3f17944930c7aa6eef1 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 15 Jul 2024 19:07:20 +0800 Subject: [PATCH 15/34] feat:[TD-30975]process return value in schemaless --- include/common/tname.h | 2 +- include/libs/parser/parser.h | 4 +- source/client/inc/clientSml.h | 24 +- source/client/src/clientRawBlockWrite.c | 23 +- source/client/src/clientSml.c | 633 ++++++++++++++++-------- source/client/src/clientSmlJson.c | 105 ++-- source/client/src/clientSmlLine.c | 66 ++- source/client/src/clientSmlTelnet.c | 30 +- source/client/test/smlTest.cpp | 114 +++-- source/common/src/tdatablock.c | 5 +- source/common/src/tname.c | 13 +- source/libs/parser/src/parInsertSml.c | 101 +++- 12 files changed, 744 insertions(+), 376 deletions(-) diff --git a/include/common/tname.h b/include/common/tname.h index 6a89d2a6be..f0a68f28c4 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -81,7 +81,7 @@ typedef struct { // uint64_t uid; // child table uid, may be useful } RandTableName; -void buildChildTableName(RandTableName* rName); +int32_t buildChildTableName(RandTableName* rName); #ifdef __cplusplus } diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 3ac357055e..4ad24fe91d 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -149,10 +149,10 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* void qDestroyBoundColInfo(void* pInfo); -SQuery* smlInitHandle(); +int32_t smlInitHandle(SQuery** query); int32_t smlBuildRow(STableDataCxt* pTableCxt); int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* kv, int32_t index); -STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta); +int32_t smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta, STableDataCxt** cxt); void clearColValArraySml(SArray* pCols); int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 122914fd34..d369e0597c 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -82,7 +82,7 @@ extern "C" { #define IS_SLASH_LETTER_IN_MEASUREMENT(sql) \ (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == SLASH)) -#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) +#define MOVE_FORWARD_ONE(sql, len) ((void)memmove((void *)((sql)-1), (sql), len)) #define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen) \ for (int i = 1; i < keyLen; ++i) { \ @@ -211,19 +211,19 @@ extern int64_t smlFactorS[]; typedef int32_t (*_equal_fn_sml)(const void *, const void *); -SSmlHandle *smlBuildSmlInfo(TAOS *taos); +int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle); void smlDestroyInfo(SSmlHandle *info); int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset); int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset); bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg); -int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); -bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); +void smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); +int32_t smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision); -SSmlTableInfo* smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen); -SSmlSTableMeta* smlBuildSTableMeta(bool isDataFormat); +int32_t smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen, SSmlTableInfo** tInfo); +int32_t smlBuildSTableMeta(bool isDataFormat, SSmlSTableMeta** sMeta); int32_t smlSetCTableName(SSmlTableInfo *oneTable); -void getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo); -STableMeta* smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen); +int32_t getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo); +int32_t smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen, STableMeta **pTableMeta); int32_t is_same_child_table_telnet(const void *a, const void *b); int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len); int32_t smlClearForRerun(SSmlHandle *info); @@ -236,7 +236,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseJSON(SSmlHandle *info, char *payload); -SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement); +int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSmlSTableMeta** sMeta); bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv); bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv); int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements); @@ -246,9 +246,9 @@ void smlBuildTsKv(SSmlKv *kv, int64_t ts); int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv); int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs); -static inline bool smlDoubleToInt64OverFlow(double num) { - if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true; - return false; +static inline int32_t smlDoubleToInt64OverFlow(double num) { + if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return 0; + return -1; } static inline void smlStrReplace(char* src, int32_t len){ diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index ce404092b0..fe12850e14 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1572,9 +1572,8 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pDat } // uError("td23101 0vgId:%d, vgId:%d, name:%s, uid:%"PRIu64, vgData.vgId, pTableMeta->vgId, tbname, pTableMeta->uid); - pQuery = smlInitHandle(); - if (pQuery == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = smlInitHandle(&pQuery); + if (code != TSDB_CODE_SUCCESS) { goto end; } pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); @@ -1656,9 +1655,9 @@ int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const cha if (code != TSDB_CODE_SUCCESS) { goto end; } - pQuery = smlInitHandle(); - if (pQuery == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + + code = smlInitHandle(&pQuery); + if (code != TSDB_CODE_SUCCESS) { goto end; } pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); @@ -1755,12 +1754,12 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { conn.requestObjRefId = pRequest->self; conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); - pQuery = smlInitHandle(); - if (pQuery == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = smlInitHandle(&pQuery); + if (code != TSDB_CODE_SUCCESS) { SET_ERROR_MSG("init sml handle failed"); goto end; } + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); while (++rspObj.common.resIter < rspObj.rsp.common.blockNum) { void* pRetrieve = taosArrayGetP(rspObj.rsp.common.blockData, rspObj.common.resIter); @@ -1903,12 +1902,12 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) conn.requestObjRefId = pRequest->self; conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); - pQuery = smlInitHandle(); - if (pQuery == NULL) { + code = smlInitHandle(&pQuery); + if (code != TSDB_CODE_SUCCESS) { SET_ERROR_MSG("init sml handle failed"); - code = TSDB_CODE_OUT_OF_MEMORY; goto end; } + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.common.blockNum); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 7e63dcd36a..6e2f4ba367 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -110,11 +110,13 @@ int64_t smlFactorS[] = {1000LL, 1000000LL, 1000000000LL}; static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo *conn, const char *pTabName, AUTH_TYPE type) { SUserAuthInfo pAuth = {0}; - snprintf(pAuth.user, sizeof(pAuth.user), "%s", info->taos->user); + (void)snprintf(pAuth.user, sizeof(pAuth.user), "%s", info->taos->user); if (NULL == pTabName) { - tNameSetDbName(&pAuth.tbName, info->taos->acctId, info->pRequest->pDb, strlen(info->pRequest->pDb)); + if (tNameSetDbName(&pAuth.tbName, info->taos->acctId, info->pRequest->pDb, strlen(info->pRequest->pDb)) != 0){ + return TSDB_CODE_SML_INVALID_DATA; + } } else { - toName(info->taos->acctId, info->pRequest->pDb, pTabName, &pAuth.tbName); + (void)toName(info->taos->acctId, info->pRequest->pDb, pTabName, &pAuth.tbName); //ignore } pAuth.type = type; @@ -129,17 +131,19 @@ static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo *conn, const char : code; } -int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) { - if (pBuf->buf) { - memset(pBuf->buf, 0, pBuf->len); - if (msg1) strncat(pBuf->buf, msg1, pBuf->len); - int32_t left = pBuf->len - strlen(pBuf->buf); - if (left > 2 && msg2) { - strncat(pBuf->buf, ":", left - 1); - strncat(pBuf->buf, msg2, left - 2); - } +void smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) { + if (pBuf->buf == NULL) { + return; + } + (void)memset(pBuf->buf, 0, pBuf->len); + if (msg1) { + (void)strncat(pBuf->buf, msg1, pBuf->len); + } + int32_t left = pBuf->len - strlen(pBuf->buf); + if (left > 2 && msg2) { + (void)strncat(pBuf->buf, ":", left - 1); + (void)strncat(pBuf->buf, msg2, left - 2); } - return TSDB_CODE_SML_INVALID_DATA; } int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision) { @@ -161,10 +165,10 @@ int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, u return convertTimePrecision(tsInt64, fromPrecision, toPrecision); } -SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen) { +int32_t smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen, SSmlTableInfo** tInfo) { SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); if (!tag) { - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } tag->sTableName = measure; @@ -173,14 +177,12 @@ SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measu tag->cols = taosArrayInit(numRows, POINTER_BYTES); if (tag->cols == NULL) { uError("SML:smlBuildTableInfo failed to allocate memory"); - goto cleanup; + taosMemoryFree(tag); + return TSDB_CODE_OUT_OF_MEMORY; } - return tag; - -cleanup: - taosMemoryFree(tag); - return NULL; + *tInfo = tag; + return TSDB_CODE_SUCCESS; } void smlBuildTsKv(SSmlKv *kv, int64_t ts) { @@ -191,34 +193,50 @@ void smlBuildTsKv(SSmlKv *kv, int64_t ts) { kv->length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; } -SSmlSTableMeta *smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement) { - SSmlSTableMeta *sMeta = NULL; +static void smlDestroySTableMeta(void *para) { + SSmlSTableMeta *meta = *(SSmlSTableMeta **)para; + taosHashCleanup(meta->tagHash); + taosHashCleanup(meta->colHash); + (void)taosArrayDestroy(meta->tags); + (void)taosArrayDestroy(meta->cols); + taosMemoryFreeClear(meta->tableMeta); + taosMemoryFree(meta); +} + +int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSmlSTableMeta** sMeta) { + int32_t code = TSDB_CODE_SUCCESS; char *measure = currElement->measure; int measureLen = currElement->measureLen; if (currElement->measureEscaped) { measure = (char *)taosMemoryMalloc(measureLen); - memcpy(measure, currElement->measure, measureLen); + if (measure == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(measure, currElement->measure, measureLen); PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); smlStrReplace(measure, measureLen); } - STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen); + STableMeta *pTableMeta = NULL; + code = smlGetMeta(info, measure, measureLen, &pTableMeta); if (currElement->measureEscaped) { taosMemoryFree(measure); } - if (pTableMeta == NULL) { + if (code != TSDB_CODE_SUCCESS) { info->dataFormat = false; info->reRun = true; - terrno = TSDB_CODE_SUCCESS; - return sMeta; + return code; } - sMeta = smlBuildSTableMeta(info->dataFormat); - if (sMeta == NULL) { + code = smlBuildSTableMeta(info->dataFormat, sMeta); + if (code != TSDB_CODE_SUCCESS) { taosMemoryFreeClear(pTableMeta); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return sMeta; + return code; + } + (*sMeta)->tableMeta = pTableMeta; + code = taosHashPut(info->superTables, currElement->measure, currElement->measureLen, sMeta, POINTER_BYTES); + if (code != TSDB_CODE_SUCCESS){ + smlDestroySTableMeta(*sMeta); + return code; } - sMeta->tableMeta = pTableMeta; - taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES); for (int i = 1; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) { SSchema *col = pTableMeta->schema + i; SSmlKv kv = {.key = col->name, .keyLen = strlen(col->name), .type = col->type}; @@ -232,40 +250,38 @@ SSmlSTableMeta *smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currEleme } if (i < pTableMeta->tableInfo.numOfColumns) { - taosArrayPush(sMeta->cols, &kv); + if(taosArrayPush((*sMeta)->cols, &kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } } else { - taosArrayPush(sMeta->tags, &kv); + if(taosArrayPush((*sMeta)->tags, &kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } } } - return sMeta; + return TSDB_CODE_SUCCESS; } bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { // cnt begin 0, add ts so + 2 if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) { - info->dataFormat = false; - info->reRun = true; - return false; + goto END; } // bind data int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, cnt + 1); if (unlikely(ret != TSDB_CODE_SUCCESS)) { uDebug("smlBuildCol error, retry"); - info->dataFormat = false; - info->reRun = true; - return false; + goto END; } if (cnt >= taosArrayGetSize(info->maxColKVs)) { - info->dataFormat = false; - info->reRun = true; - return false; + goto END; } SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxColKVs, cnt); - + if (maxKV == NULL){ + goto END; + } if (unlikely(!IS_SAME_KEY)) { - info->dataFormat = false; - info->reRun = true; - return false; + goto END; } if (unlikely(IS_VAR_DATA_TYPE(kv->type) && kv->length > maxKV->length)) { @@ -273,6 +289,11 @@ bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { info->needModifySchema = true; } return true; + +END: + info->dataFormat = false; + info->reRun = true; + return false; } bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { @@ -285,6 +306,9 @@ bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { } SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt); + if (maxKV == NULL) { + goto END; + } if (unlikely(!IS_SAME_KEY)) { goto END; } @@ -306,8 +330,8 @@ int32_t smlJoinMeasureTag(SSmlLineInfo *elements) { if (elements->measureTag == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - memcpy(elements->measureTag, elements->measure, elements->measureLen); - memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen); + (void)memcpy(elements->measureTag, elements->measure, elements->measureLen); + (void)memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen); elements->measureTagsLen = elements->measureLen + elements->tagsLen; return TSDB_CODE_SUCCESS; } @@ -325,14 +349,14 @@ static bool smlIsPKTable(STableMeta *pTableMeta){ int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements) { bool isSameMeasure = IS_SAME_SUPER_TABLE; if (isSameMeasure) { - return 0; + return TSDB_CODE_SUCCESS; } SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); SSmlSTableMeta *sMeta = NULL; if (unlikely(tmp == NULL)) { - sMeta = smlBuildSuperTableInfo(info, elements); - if (sMeta == NULL) return -1; + int32_t code = smlBuildSuperTableInfo(info, elements, &sMeta); + if (code != 0) return code; } else { sMeta = *tmp; } @@ -342,39 +366,59 @@ int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements) { info->maxColKVs = sMeta->cols; if(smlIsPKTable(sMeta->tableMeta)){ - terrno = TSDB_CODE_SML_NOT_SUPPORT_PK; - return -1; + return TSDB_CODE_SML_NOT_SUPPORT_PK; } return 0; } int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements) { + int32_t code = TSDB_CODE_SUCCESS; SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureTagsLen); SSmlTableInfo *tinfo = NULL; if (unlikely(oneTable == NULL)) { - tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); - if (unlikely(!tinfo)) { - return TSDB_CODE_OUT_OF_MEMORY; + code = smlBuildTableInfo(1, elements->measure, elements->measureLen, &tinfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + code = taosHashPut(info->childTables, elements->measureTag, elements->measureTagsLen, &tinfo, POINTER_BYTES); + if(code != 0){ + smlDestroyTableInfo(&tinfo); + return code; } - taosHashPut(info->childTables, elements->measureTag, elements->measureTagsLen, &tinfo, POINTER_BYTES); tinfo->tags = taosArrayDup(info->preLineTagKV, NULL); + if(tinfo->tags == NULL){ + smlDestroyTableInfo(&tinfo); + return TSDB_CODE_OUT_OF_MEMORY; + } for (size_t i = 0; i < taosArrayGetSize(info->preLineTagKV); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(info->preLineTagKV, i); + if(kv == NULL){ + smlDestroyTableInfo(&tinfo); + return TSDB_CODE_SML_INVALID_DATA; + } if (kv->keyEscaped) kv->key = NULL; if (kv->valueEscaped) kv->value = NULL; } - smlSetCTableName(tinfo); - getTableUid(info, elements, tinfo); + code = smlSetCTableName(tinfo); + if (code != TSDB_CODE_SUCCESS){ + smlDestroyTableInfo(&tinfo); + return code; + } + code = getTableUid(info, elements, tinfo); + if (code != TSDB_CODE_SUCCESS){ + smlDestroyTableInfo(&tinfo); + return code; + } if (info->dataFormat) { info->currSTableMeta->uid = tinfo->uid; - tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); - if (tinfo->tableDataCtx == NULL) { + code = smlInitTableDataCtx(info->pQuery, info->currSTableMeta, &tinfo->tableDataCtx); + if (code != TSDB_CODE_SUCCESS) { smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); smlDestroyTableInfo(&tinfo); - return TSDB_CODE_SML_INVALID_DATA; + return code; } } } else { @@ -404,9 +448,16 @@ int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv * uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format false, ts:%" PRId64, info->id, kvTs->i); if (elements->colArray == NULL) { elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); + if (elements->colArray == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + if (taosArrayPush(elements->colArray, kvTs) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + if (taosArrayPush(elements->colArray, kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; } - taosArrayPush(elements->colArray, kvTs); - taosArrayPush(elements->colArray, kv); } info->preLine = *elements; @@ -442,6 +493,9 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) { size_t totalNameLen = delimiter * (taosArrayGetSize(tags) - 1); for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); + if(tag == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } totalNameLen += tag->length; } if (totalNameLen < TSDB_TABLE_NAME_LEN) { @@ -449,12 +503,15 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) { } } if (autoChildName) { - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); + (void)memset(childTableName, 0, TSDB_TABLE_NAME_LEN); for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); - strncat(childTableName, tag->value, tag->length); + if(tag == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } + (void)strncat(childTableName, tag->value, tag->length); if (i != taosArrayGetSize(tags) - 1) { - strcat(childTableName, tsSmlAutoChildTableNameDelimiter); + (void)strcat(childTableName, tsSmlAutoChildTableNameDelimiter); } } if (tsSmlDot2Underline) { @@ -466,10 +523,13 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) { for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); + if(tag == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } // handle child table name if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) { - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); + (void)memset(childTableName, 0, TSDB_TABLE_NAME_LEN); + (void)strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); if (tsSmlDot2Underline) { smlStrReplace(childTableName, strlen(childTableName)); } @@ -483,60 +543,60 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) { } int32_t smlSetCTableName(SSmlTableInfo *oneTable) { - smlParseTableName(oneTable->tags, oneTable->childTableName); + int32_t code = smlParseTableName(oneTable->tags, oneTable->childTableName); + if(code != TSDB_CODE_SUCCESS){ + return code; + } if (strlen(oneTable->childTableName) == 0) { SArray *dst = taosArrayDup(oneTable->tags, NULL); + if (dst == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } ASSERT(oneTable->sTableNameLen < TSDB_TABLE_NAME_LEN); char superName[TSDB_TABLE_NAME_LEN] = {0}; RandTableName rName = {dst, NULL, (uint8_t)oneTable->sTableNameLen, oneTable->childTableName}; if (tsSmlDot2Underline) { - memcpy(superName, oneTable->sTableName, oneTable->sTableNameLen); + (void)memcpy(superName, oneTable->sTableName, oneTable->sTableNameLen); smlStrReplace(superName, oneTable->sTableNameLen); rName.stbFullName = superName; } else { rName.stbFullName = oneTable->sTableName; } - buildChildTableName(&rName); - taosArrayDestroy(dst); + code = buildChildTableName(&rName); + if (code != TSDB_CODE_SUCCESS){ + return code; + } + (void)taosArrayDestroy(dst); } return TSDB_CODE_SUCCESS; } -void getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo) { +int32_t getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo) { char key[TSDB_TABLE_NAME_LEN * 2 + 1] = {0}; size_t nLen = strlen(tinfo->childTableName); - memcpy(key, currElement->measure, currElement->measureLen); + (void)memcpy(key, currElement->measure, currElement->measureLen); if (tsSmlDot2Underline) { smlStrReplace(key, currElement->measureLen); } - memcpy(key + currElement->measureLen + 1, tinfo->childTableName, nLen); + (void)memcpy(key + currElement->measureLen + 1, tinfo->childTableName, nLen); void *uid = taosHashGet(info->tableUids, key, currElement->measureLen + 1 + nLen); // use \0 as separator for stable name and child table name if (uid == NULL) { tinfo->uid = info->uid++; - taosHashPut(info->tableUids, key, currElement->measureLen + 1 + nLen, &tinfo->uid, sizeof(uint64_t)); + return taosHashPut(info->tableUids, key, currElement->measureLen + 1 + nLen, &tinfo->uid, sizeof(uint64_t)); } else { tinfo->uid = *(uint64_t *)uid; } + return TSDB_CODE_SUCCESS; } -static void smlDestroySTableMeta(void *para) { - SSmlSTableMeta *meta = *(SSmlSTableMeta **)para; - taosHashCleanup(meta->tagHash); - taosHashCleanup(meta->colHash); - taosArrayDestroy(meta->tags); - taosArrayDestroy(meta->cols); - taosMemoryFreeClear(meta->tableMeta); - taosMemoryFree(meta); -} - -SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) { +int32_t smlBuildSTableMeta(bool isDataFormat, SSmlSTableMeta **sMeta) { SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); if (!meta) { - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } if (unlikely(!isDataFormat)) { @@ -564,14 +624,15 @@ SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) { uError("SML:smlBuildSTableMeta failed to allocate memory"); goto cleanup; } - return meta; + *sMeta = meta; + return TSDB_CODE_SUCCESS; cleanup: smlDestroySTableMeta(meta); - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } -bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { +int32_t smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->length; char *endptr = NULL; @@ -667,7 +728,7 @@ bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg) { kvVal->type = TSDB_DATA_TYPE_FLOAT; kvVal->f = (float)result; } else if ((left == 1 && *endptr == 'i') || (left == 3 && strncasecmp(endptr, "i64", left) == 0)) { - if (smlDoubleToInt64OverFlow(result)) { + if (smlDoubleToInt64OverFlow(result) == 0) { errno = 0; int64_t tmp = taosStr2Int64(pVal, &endptr, 10); if (errno == ERANGE) { @@ -743,8 +804,8 @@ bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg) { return true; } -STableMeta *smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen) { - STableMeta *pTableMeta = NULL; +int32_t smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen, STableMeta **pTableMeta) { + *pTableMeta = NULL; SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); @@ -754,14 +815,14 @@ STableMeta *smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen conn.requestId = info->pRequest->requestId; conn.requestObjRefId = info->pRequest->self; conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - memcpy(pName.tname, measure, measureLen); + (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + (void)memcpy(pName.tname, measure, measureLen); - int32_t code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); + int32_t code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, pTableMeta); if (code != TSDB_CODE_SUCCESS) { - return NULL; + return code; } - return pTableMeta; + return TSDB_CODE_SUCCESS; } static int64_t smlGenId() { @@ -803,7 +864,7 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm *action = SCHEMA_ACTION_ADD_COLUMN; } } - return 0; + return TSDB_CODE_SUCCESS; } #define BOUNDARY 1024 @@ -838,6 +899,9 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH for (int j = 0; j < taosArrayGetSize(cols); ++j) { if (j == 0 && !isTag) continue; SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); + if (kv == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info); if (code != TSDB_CODE_SUCCESS) { return code; @@ -846,6 +910,9 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH for (int j = 0; j < taosArrayGetSize(checkDumplicateCols); ++j) { SSmlKv *kv = (SSmlKv *)taosArrayGet(checkDumplicateCols, j); + if (kv == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } if(taosHashGet(schemaHash, kv->key, kv->keyLen) != NULL){ return TSDB_CODE_PAR_DUPLICATED_COLUMN; } @@ -854,10 +921,18 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH } static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) { + int32_t code = TSDB_CODE_SUCCESS; SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (hashTmp == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto END; + } int32_t i = 0; for (; i < length; i++) { - taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); + code = taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); + if (code != 0){ + goto END; + } } if (isTag) { @@ -867,13 +942,19 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool } for (; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + if (kv == NULL){ + code = TSDB_CODE_SML_INVALID_DATA; + goto END; + } if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { - taosHashCleanup(hashTmp); - return TSDB_CODE_SML_INVALID_DATA; + code = TSDB_CODE_SML_INVALID_DATA; + goto END; } } + +END: taosHashCleanup(hashTmp); - return 0; + return code; } static int32_t getBytes(uint8_t type, int32_t length) { @@ -889,6 +970,9 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO SArray *results, int32_t numOfCols, bool isTag) { for (int j = 0; j < taosArrayGetSize(cols); ++j) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); + if (kv == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } ESchemaAction action = SCHEMA_ACTION_NULL; int code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info); if (code != 0) { @@ -898,8 +982,10 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO SField field = {0}; field.type = kv->type; field.bytes = getBytes(kv->type, kv->length); - memcpy(field.name, kv->key, kv->keyLen); - taosArrayPush(results, &field); + (void)memcpy(field.name, kv->key, kv->keyLen); + if (taosArrayPush(results, &field) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen); if (index == NULL) { @@ -909,6 +995,9 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO uint16_t newIndex = *index; if (isTag) newIndex -= numOfCols; SField *field = (SField *)taosArrayGet(results, newIndex); + if (field == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } field->bytes = getBytes(kv->type, kv->length); } } @@ -917,6 +1006,9 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO int32_t len = 0; for (int j = 0; j < taosArrayGetSize(results); ++j) { SField *field = taosArrayGet(results, j); + if (field == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } len += field->bytes; } if (len > maxLen) { @@ -933,21 +1025,33 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, int32_t code = TSDB_CODE_SUCCESS; SCmdMsgInfo pCmdMsg = {0}; char *pSql = NULL; + char *msg = NULL; // put front for free pReq.numOfColumns = taosArrayGetSize(pColumns); + pReq.pTags = pTags; + pReq.numOfTags = taosArrayGetSize(pTags); + pReq.pColumns = taosArrayInit(pReq.numOfColumns, sizeof(SFieldWithOptions)); + if (pReq.pColumns == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } for (int32_t i = 0; i < pReq.numOfColumns; ++i) { - SField *pField = taosArrayGet(pColumns, i); + SField *pField = taosArrayGet(pColumns, i); + if (pField == NULL){ + code = TSDB_CODE_SML_INVALID_DATA; + goto end; + } SFieldWithOptions fieldWithOption = {0}; setFieldWithOptions(&fieldWithOption, pField); setDefaultOptionsForField(&fieldWithOption); - taosArrayPush(pReq.pColumns, &fieldWithOption); + if (taosArrayPush(pReq.pColumns, &fieldWithOption) == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } } - pReq.numOfTags = taosArrayGetSize(pTags); - pReq.pTags = pTags; - if (action == SCHEMA_ACTION_CREATE_STABLE) { pReq.colVer = 1; pReq.tagVer = 1; @@ -968,6 +1072,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, pSql = (action == SCHEMA_ACTION_ADD_COLUMN) ? "sml_add_column" : "sml_modify_column_size"; } else { uError("SML:0x%" PRIx64 " invalid action:%d", info->id, action); + code = TSDB_CODE_SML_INVALID_DATA; goto end; } @@ -987,40 +1092,55 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SField field = {0}; field.type = TSDB_DATA_TYPE_NCHAR; field.bytes = TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; - strcpy(field.name, tsSmlTagName); - taosArrayPush(pReq.pTags, &field); + tstrncpy(field.name, tsSmlTagName, sizeof(field.name)); + if (taosArrayPush(pReq.pTags, &field) == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } } pReq.commentLen = -1; pReq.igExists = true; - tNameExtractFullName(pName, pReq.name); + (void)tNameExtractFullName(pName, pReq.name); pCmdMsg.epSet = getEpSet_s(&info->taos->pAppInfo->mgmtEp); pCmdMsg.msgType = TDMT_MND_CREATE_STB; pCmdMsg.msgLen = tSerializeSMCreateStbReq(NULL, 0, &pReq); - pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); - if (NULL == pCmdMsg.pMsg) { + if (pCmdMsg.msgLen < 0){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + msg = taosMemoryMalloc(pCmdMsg.msgLen); + if (NULL == msg) { code = TSDB_CODE_OUT_OF_MEMORY; goto end; } - tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); - SQuery pQuery; - memset(&pQuery, 0, sizeof(pQuery)); + pCmdMsg.pMsg = msg; + if (tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) < 0){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + + SQuery pQuery = {0}; + (void)memset(&pQuery, 0, sizeof(pQuery)); pQuery.execMode = QUERY_EXEC_MODE_RPC; pQuery.pCmdMsg = &pCmdMsg; pQuery.msgType = pQuery.pCmdMsg->msgType; pQuery.stableQuery = true; - launchQueryImpl(pRequest, &pQuery, true, NULL); + (void)launchQueryImpl(pRequest, &pQuery, true, NULL); // no need to check return value if (pRequest->code == TSDB_CODE_SUCCESS) { - catalogRemoveTableMeta(info->pCatalog, pName); + code = catalogRemoveTableMeta(info->pCatalog, pName); + if (code != TSDB_CODE_SUCCESS){ + goto end; + } } code = pRequest->code; - taosMemoryFree(pCmdMsg.pMsg); end: + taosMemoryFree(msg); destroyRequest(pRequest); tFreeSMCreateStbReq(&pReq); return code; @@ -1053,11 +1173,15 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { size_t superTableLen = 0; void *superTable = taosHashGetKey(tmp, &superTableLen); char *measure = taosMemoryMalloc(superTableLen); - memcpy(measure, superTable, superTableLen); + if (measure == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + (void)memcpy(measure, superTable, superTableLen); PROCESS_SLASH_IN_MEASUREMENT(measure, superTableLen); smlStrReplace(measure, superTableLen); - memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - memcpy(pName.tname, measure, superTableLen); + (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + (void)memcpy(pName.tname, measure, superTableLen); taosMemoryFree(measure); code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); @@ -1069,23 +1193,32 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { } uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas create table:%s", info->id, pName.tname); SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); + if (pColumns == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); + if (pTags == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + (void)taosArrayDestroy(pColumns); + goto end; + } code = smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBuildFieldsList tag1 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } code = smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBuildFieldsList col1 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } code = smlSendMetaMsg(info, &pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE); - taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pColumns); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); goto end; @@ -1107,9 +1240,16 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (hashTmp == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } for (uint16_t i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { - taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + code = taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + if (code != 0){ + goto end; + } } ESchemaAction action = SCHEMA_ACTION_NULL; @@ -1126,46 +1266,64 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { action); SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); + if (pColumns == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - + if (pTags == NULL){ + (void)taosArrayDestroy(pColumns); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { SField field = {0}; field.type = pTableMeta->schema[i].type; field.bytes = pTableMeta->schema[i].bytes; - strcpy(field.name, pTableMeta->schema[i].name); + tstrncpy(field.name, pTableMeta->schema[i].name, sizeof(field.name)); if (i < pTableMeta->tableInfo.numOfColumns) { - taosArrayPush(pColumns, &field); + if (taosArrayPush(pColumns, &field) == NULL){ + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } } else { - taosArrayPush(pTags, &field); + if (taosArrayPush(pTags, &field) == NULL){ + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } } } code = smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags, pTableMeta->tableInfo.numOfColumns, true); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBuildFieldsList tag2 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } if (taosArrayGetSize(pTags) + pTableMeta->tableInfo.numOfColumns > TSDB_MAX_COLUMNS) { uError("SML:0x%" PRIx64 " too many columns than 4096", info->id); code = TSDB_CODE_PAR_TOO_MANY_COLUMNS; - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } if (taosArrayGetSize(pTags) > TSDB_MAX_TAGS) { uError("SML:0x%" PRIx64 " too many tags than 128", info->id); code = TSDB_CODE_PAR_INVALID_TAGS_NUM; - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); - taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pColumns); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); goto end; @@ -1185,7 +1343,10 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { taosHashClear(hashTmp); for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + code = taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } } action = SCHEMA_ACTION_NULL; code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, sTableData->tags, &action, false); @@ -1201,18 +1362,36 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { action); SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); + if (pColumns == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - + if (pTags == NULL){ + (void)taosArrayDestroy(pColumns); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { SField field = {0}; field.type = pTableMeta->schema[i].type; field.bytes = pTableMeta->schema[i].bytes; - strcpy(field.name, pTableMeta->schema[i].name); + tstrncpy(field.name, pTableMeta->schema[i].name, sizeof(field.name)); if (i < pTableMeta->tableInfo.numOfColumns) { - taosArrayPush(pColumns, &field); + if (taosArrayPush(pColumns, &field) == NULL){ + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } } else { - taosArrayPush(pTags, &field); + if (taosArrayPush(pTags, &field) == NULL){ + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } } } @@ -1220,21 +1399,21 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { pTableMeta->tableInfo.numOfColumns, false); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBuildFieldsList col2 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } if (taosArrayGetSize(pColumns) + pTableMeta->tableInfo.numOfTags > TSDB_MAX_COLUMNS) { uError("SML:0x%" PRIx64 " too many columns than 4096", info->id); code = TSDB_CODE_PAR_TOO_MANY_COLUMNS; - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); + (void)taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pTags); goto end; } code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); - taosArrayDestroy(pColumns); + (void)taosArrayDestroy(pColumns); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); goto end; @@ -1284,13 +1463,13 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas end success, format:%d, needModifySchema:%d", info->id, info->dataFormat, info->needModifySchema); - return 0; + return TSDB_CODE_SUCCESS; end: taosHashCancelIterate(info->superTables, tmp); taosHashCleanup(hashTmp); taosMemoryFreeClear(pTableMeta); - catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); + code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); uError("SML:0x%" PRIx64 " smlModifyDBSchemas end failed:%d:%s, format:%d, needModifySchema:%d", info->id, code, tstrerror(code), info->dataFormat, info->needModifySchema); @@ -1301,9 +1480,14 @@ static int32_t smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols terrno = 0; for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + if (kv == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); if (ret == 0) { - taosArrayPush(metaArray, kv); + if (taosArrayPush(metaArray, kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } if(taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { return TSDB_CODE_PAR_DUPLICATED_COLUMN; } @@ -1317,10 +1501,16 @@ static int32_t smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg, SHashObj* checkDuplicate) { for (int i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); - + if (kv == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } int16_t *index = (int16_t *)taosHashGet(metaHash, kv->key, kv->keyLen); if (index) { SSmlKv *value = (SSmlKv *)taosArrayGet(metaArray, *index); + if (value == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } + if (isTag) { if (kv->length > value->length) { value->length = kv->length; @@ -1345,7 +1535,9 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols int16_t size = tmp; int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); if (ret == 0) { - taosArrayPush(metaArray, kv); + if(taosArrayPush(metaArray, kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } if(taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { return TSDB_CODE_PAR_DUPLICATED_COLUMN; } @@ -1365,7 +1557,7 @@ void smlDestroyTableInfo(void *para) { taosHashCleanup(kvHash); } - taosArrayDestroy(tag->cols); + (void)taosArrayDestroy(tag->cols); taosArrayDestroyEx(tag->tags, freeSSmlKv); taosMemoryFree(tag); } @@ -1391,13 +1583,13 @@ void smlDestroyInfo(SSmlHandle *info) { cJSON *tags = (cJSON *)taosArrayGetP(info->tagJsonArray, i); cJSON_Delete(tags); } - taosArrayDestroy(info->tagJsonArray); + (void)taosArrayDestroy(info->tagJsonArray); for (int i = 0; i < taosArrayGetSize(info->valueJsonArray); i++) { cJSON *value = (cJSON *)taosArrayGetP(info->valueJsonArray, i); cJSON_Delete(value); } - taosArrayDestroy(info->valueJsonArray); + (void)taosArrayDestroy(info->valueJsonArray); taosArrayDestroyEx(info->preLineTagKV, freeSSmlKv); @@ -1418,21 +1610,22 @@ void smlDestroyInfo(SSmlHandle *info) { taosMemoryFreeClear(info); } -SSmlHandle *smlBuildSmlInfo(TAOS *taos) { +int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { int32_t code = TSDB_CODE_SUCCESS; SSmlHandle *info = (SSmlHandle *)taosMemoryCalloc(1, sizeof(SSmlHandle)); if (NULL == info) { - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } if (taos != NULL) { info->taos = acquireTscObj(*(int64_t *)taos); if (info->taos == NULL) { - goto cleanup; + code = TSDB_CODE_TSC_DISCONNECTED; + goto ERROR; } code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code); - goto cleanup; + goto ERROR; } } @@ -1440,27 +1633,37 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) { info->childTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); info->tableUids = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); info->superTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (info->pVgHash == NULL || info->childTables == NULL || info->superTables == NULL || info->tableUids == NULL) { + uError("create SSmlHandle hash obj failed"); + code = TSDB_CODE_OUT_OF_MEMORY; + goto ERROR; + } taosHashSetFreeFp(info->superTables, smlDestroySTableMeta); taosHashSetFreeFp(info->childTables, smlDestroyTableInfo); info->id = smlGenId(); - info->pQuery = smlInitHandle(); + code = smlInitHandle(&info->pQuery); + if (code != TSDB_CODE_SUCCESS){ + goto ERROR; + } info->dataFormat = true; info->tagJsonArray = taosArrayInit(8, POINTER_BYTES); info->valueJsonArray = taosArrayInit(8, POINTER_BYTES); info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv)); - if (NULL == info->pVgHash || NULL == info->childTables || NULL == info->superTables || NULL == info->tableUids) { - uError("create SSmlHandle failed"); - goto cleanup; + if (info->tagJsonArray == NULL || info->valueJsonArray == NULL || info->preLineTagKV == NULL) { + uError("SML:0x%" PRIx64 " failed to allocate memory", info->id); + code = TSDB_CODE_OUT_OF_MEMORY; + goto ERROR; } - return info; + *handle = info; + return code; -cleanup: +ERROR: smlDestroyInfo(info); - return NULL; + return code; } static int32_t smlPushCols(SArray *colsArray, SArray *cols) { @@ -1472,14 +1675,21 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { for (size_t i = 0; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); terrno = 0; - taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); + int32_t code = taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); if (terrno == TSDB_CODE_DUP_KEY) { taosHashCleanup(kvHash); return TSDB_CODE_PAR_DUPLICATED_COLUMN; } + if (code != TSDB_CODE_SUCCESS){ + taosHashCleanup(kvHash); + return code; + } } - taosArrayPush(colsArray, &kvHash); + if (taosArrayPush(colsArray, &kvHash) == NULL) { + taosHashCleanup(kvHash); + return TSDB_CODE_OUT_OF_MEMORY; + } return TSDB_CODE_SUCCESS; } @@ -1542,9 +1752,10 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { } else { uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); - SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); - if (meta == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + SSmlSTableMeta *meta = NULL; + ret = smlBuildSTableMeta(info->dataFormat, &meta); + if (ret != TSDB_CODE_SUCCESS) { + return ret; } ret = taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); if (ret != TSDB_CODE_SUCCESS) { @@ -1572,11 +1783,17 @@ static int32_t smlInsertData(SSmlHandle *info) { if (info->pRequest->dbList == NULL) { info->pRequest->dbList = taosArrayInit(1, TSDB_DB_FNAME_LEN); + if (info->pRequest->dbList == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } } char *data = (char *)taosArrayReserve(info->pRequest->dbList, 1); + if (data == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); - tNameGetFullDbName(&pName, data); + (void)tNameGetFullDbName(&pName, data); //ignore SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); while (oneTable) { @@ -1584,18 +1801,26 @@ static int32_t smlInsertData(SSmlHandle *info) { int measureLen = tableData->sTableNameLen; char *measure = (char *)taosMemoryMalloc(tableData->sTableNameLen); - memcpy(measure, tableData->sTableName, tableData->sTableNameLen); + if (measure == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(measure, tableData->sTableName, tableData->sTableNameLen); PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); smlStrReplace(measure, measureLen); - memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - memcpy(pName.tname, measure, measureLen); + (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + (void)memcpy(pName.tname, measure, measureLen); if (info->pRequest->tableList == NULL) { info->pRequest->tableList = taosArrayInit(1, sizeof(SName)); + if (info->pRequest->tableList == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + } + if (taosArrayPush(info->pRequest->tableList, &pName) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; } - taosArrayPush(info->pRequest->tableList, &pName); - strcpy(pName.tname, tableData->childTableName); + tstrncpy(pName.tname, tableData->childTableName, sizeof(pName.tname)); SRequestConnInfo conn = {0}; conn.pTrans = info->taos->pAppInfo->pTransporter; @@ -1618,7 +1843,13 @@ static int32_t smlInsertData(SSmlHandle *info) { taosHashCancelIterate(info->childTables, oneTable); return code; } - taosHashPut(info->pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg)); + code = taosHashPut(info->pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg)); + if (code != TSDB_CODE_SUCCESS){ + uError("SML:0x%" PRIx64 " taosHashPut failed. table name: %s", info->id, tableData->childTableName); + taosMemoryFree(measure); + taosHashCancelIterate(info->childTables, oneTable); + return code; + } SSmlSTableMeta **pMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen); @@ -1657,7 +1888,8 @@ static int32_t smlInsertData(SSmlHandle *info) { SAppClusterSummary *pActivity = &info->taos->pAppInfo->summary; atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); - launchQueryImpl(info->pRequest, info->pQuery, true, NULL); + (void)launchQueryImpl(info->pRequest, info->pQuery, true, NULL); // no need to check return code + uDebug("SML:0x%" PRIx64 " smlInsertData end, format:%d, code:%d,%s", info->id, info->dataFormat, info->pRequest->code, tstrerror(info->pRequest->code)); @@ -1777,7 +2009,11 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char if (code != TSDB_CODE_SUCCESS) { if (rawLine != NULL) { char *print = taosMemoryCalloc(len + 1, 1); - memcpy(print, tmp, len); + if (print == NULL) { + uError("SML:0x%" PRIx64 " smlParseLine failed. out of memory", info->id); + return code; + } + (void)memcpy(print, tmp, len); uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, print); taosMemoryFree(print); } else { @@ -1892,7 +2128,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, int protocol, int precision, int32_t ttl, int64_t reqid) { int32_t code = TSDB_CODE_SUCCESS; if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; + uError("SML:taos_schemaless_insert error taos is null"); return NULL; } SRequestObj *request = NULL; @@ -1905,11 +2141,13 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, return NULL; } - info = smlBuildSmlInfo(taos); - if (info == NULL) { - request->code = TSDB_CODE_OUT_OF_MEMORY; - uError("SML:taos_schemaless_insert error SSmlHandle is null"); - return (TAOS_RES *)request; + SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; + code = smlBuildSmlInfo(taos, &info); + if (code != TSDB_CODE_SUCCESS) { + request->code = code; + smlBuildInvalidDataMsg(&msg, "init SSmlHandle failed", NULL); + uError("SML:taos_schemaless_insert error SSmlHandle is null, err msg:%s", tstrerror(code)); + goto end; } info->pRequest = request; info->pRequest->pQuery = info->pQuery; @@ -1922,7 +2160,6 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, smlSetReqSQL(request, lines, rawLine, rawLineEnd); - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; if (request->pDb == NULL) { request->code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; smlBuildInvalidDataMsg(&msg, "Database not specified", NULL); @@ -1961,7 +2198,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, break; } taosMsleep(100); - refreshMeta(request->pTscObj, request); + (void)refreshMeta(request->pTscObj, request); //ignore return code,try again uInfo("SML:%" PRIx64 " retry:%d/10,ver is old retry or object is creating code:%d, msg:%s", info->id, cnt, code, tstrerror(code)); smlDestroyInfo(info); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 845884d5ac..519e55d0d1 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -29,29 +29,29 @@ (start)++; \ } -static char *smlJsonGetObj(char *payload) { +static int32_t smlJsonGetObj(char **payload) { int leftBracketCnt = 0; bool isInQuote = false; - while (*payload) { - if (*payload == '"' && *(payload - 1) != '\\') { + while (**payload) { + if (**payload == '"' && *((*payload) - 1) != '\\') { isInQuote = !isInQuote; - } else if (!isInQuote && unlikely(*payload == '{')) { + } else if (!isInQuote && unlikely(**payload == '{')) { leftBracketCnt++; - payload++; + (*payload)++; continue; - } else if (!isInQuote && unlikely(*payload == '}')) { + } else if (!isInQuote && unlikely(**payload == '}')) { leftBracketCnt--; - payload++; + (*payload)++; if (leftBracketCnt == 0) { - return payload; + return 0; } else if (leftBracketCnt < 0) { - return NULL; + return -1; } continue; } - payload++; + (*payload)++; } - return NULL; + return -1; } int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { @@ -99,8 +99,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { offset[index++] = *start - sTmp; element->timestamp = (*start); if (*(*start) == '{') { - char *tmp = smlJsonGetObj((*start)); - if (tmp) { + char *tmp = *start; + int32_t code = smlJsonGetObj(&tmp); + if (code == 0) { element->timestampLen = tmp - (*start); *start = tmp; } @@ -127,8 +128,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { offset[index++] = *start - sTmp; element->cols = (*start); if (*(*start) == '{') { - char *tmp = smlJsonGetObj((*start)); - if (tmp) { + char *tmp = *start; + int32_t code = smlJsonGetObj(&tmp); + if (code == 0) { element->colsLen = tmp - (*start); *start = tmp; } @@ -153,8 +155,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { JUMP_JSON_SPACE((*start)) offset[index++] = *start - sTmp; element->tags = (*start); - char *tmp = smlJsonGetObj((*start)); - if (tmp) { + char *tmp = *start; + int32_t code = smlJsonGetObj(&tmp); + if (code == 0) { element->tagsLen = tmp - (*start); *start = tmp; } @@ -209,8 +212,9 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { (*start) += offset[index++]; element->timestamp = *start; if (*(*start) == '{') { - char *tmp = smlJsonGetObj((*start)); - if (tmp) { + char *tmp = *start; + int32_t code = smlJsonGetObj(&tmp); + if (code == 0) { element->timestampLen = tmp - (*start); *start = tmp; } @@ -227,8 +231,9 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { (*start) += offset[index++]; element->cols = *start; if (*(*start) == '{') { - char *tmp = smlJsonGetObj((*start)); - if (tmp) { + char *tmp = *start; + int32_t code = smlJsonGetObj(&tmp); + if (code == 0) { element->colsLen = tmp - (*start); *start = tmp; } @@ -244,8 +249,9 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { } else if ((*start)[1] == 't' && (*start)[2] == 'a') { (*start) += offset[index++]; element->tags = (*start); - char *tmp = smlJsonGetObj((*start)); - if (tmp) { + char *tmp = *start; + int32_t code = smlJsonGetObj(&tmp); + if (code == 0) { element->tagsLen = tmp - (*start); *start = tmp; } @@ -261,7 +267,7 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { uError("elements != %d", OTD_JSON_FIELDS_NUM); return TSDB_CODE_TSC_INVALID_JSON; } - return 0; + return TSDB_CODE_SUCCESS; } static inline int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *metric, SSmlLineInfo *elements) { @@ -492,20 +498,17 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ int32_t tagNum = cJSON_GetArraySize(tags); if (unlikely(tagNum == 0)) { uError("SML:Tag should not be empty"); - terrno = TSDB_CODE_TSC_INVALID_JSON; - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } for (int32_t i = 0; i < tagNum; ++i) { cJSON *tag = cJSON_GetArrayItem(tags, i); if (unlikely(tag == NULL)) { - terrno = TSDB_CODE_TSC_INVALID_JSON; - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } size_t keyLen = strlen(tag->string); if (unlikely(IS_INVALID_COL_LEN(keyLen))) { uError("OTD:Tag key length is 0 or too large than 64"); - terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - return -1; + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } // add kv to SSmlKv @@ -516,19 +519,17 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ // value int32_t ret = smlParseValueFromJSON(tag, &kv); if (unlikely(ret != TSDB_CODE_SUCCESS)) { - terrno = ret; - return -1; + return ret; } taosArrayPush(preLineKV, &kv); if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { - terrno = TSDB_CODE_SUCCESS; - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } cnt++; } - return 0; + return TSDB_CODE_SUCCESS; } static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) { @@ -536,12 +537,18 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo if(info->dataFormat){ ret = smlProcessSuperTable(info, elements); if(ret != 0){ - return terrno; + if(info->reRun){ + return TSDB_CODE_SUCCESS; + } + return ret; } } ret = smlProcessTagJson(info, tags); if(ret != 0){ - return terrno; + if(info->reRun){ + return TSDB_CODE_SUCCESS; + } + return ret; } ret = smlJoinMeasureTag(elements); if(ret != 0){ @@ -703,6 +710,9 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo // Parse tags bool needFree = info->dataFormat; elements->tags = cJSON_PrintUnformatted(tagsJson); + if (elements->tags == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } elements->tagsLen = strlen(elements->tags); if (is_same_child_table_telnet(elements, &info->preLine) != 0) { ret = smlParseTagsFromJSON(info, tagsJson, elements); @@ -823,7 +833,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * } if (ret != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_VALUE; + return ret; } if (unlikely(**start == '\0' && elements->measure == NULL)) return TSDB_CODE_SUCCESS; @@ -844,18 +854,23 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * cJSON *valueJson = cJSON_Parse(elements->cols); if (unlikely(valueJson == NULL)) { uError("SML:0x%" PRIx64 " parse json cols failed:%s", info->id, elements->cols); + elements->cols[elements->colsLen] = tmp; return TSDB_CODE_TSC_INVALID_JSON; } - taosArrayPush(info->tagJsonArray, &valueJson); + if (taosArrayPush(info->tagJsonArray, &valueJson) == NULL){ + cJSON_Delete(valueJson); + elements->cols[elements->colsLen] = tmp; + return TSDB_CODE_OUT_OF_MEMORY; + } ret = smlParseValueFromJSONObj(valueJson, &kv); if (ret != TSDB_CODE_SUCCESS) { - uError("SML:Failed to parse value from JSON Obj:%s", elements->cols); + uError("SML:0x%" PRIx64 " Failed to parse value from JSON Obj:%s", info->id, elements->cols); elements->cols[elements->colsLen] = tmp; return TSDB_CODE_TSC_INVALID_VALUE; } elements->cols[elements->colsLen] = tmp; } else if (smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) { - uError("SML:cols invalidate:%s", elements->cols); + uError("SML:0x%" PRIx64 " cols invalidate:%s", info->id, elements->cols); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -870,7 +885,11 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * return TSDB_CODE_TSC_INVALID_JSON; } - taosArrayPush(info->tagJsonArray, &tagsJson); + if (taosArrayPush(info->tagJsonArray, &tagsJson) == NULL){ + cJSON_Delete(tagsJson); + uError("SML:0x%" PRIx64 " taosArrayPush failed", info->id); + return TSDB_CODE_OUT_OF_MEMORY; + } ret = smlParseTagsFromJSON(info, tagsJson, elements); if (unlikely(ret)) { uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); @@ -937,7 +956,7 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) { return ret; } info->lines = (SSmlLineInfo *)tmp; - memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo)); + (void)memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo)); } ret = smlParseJSONString(info, &dataPointStart, info->lines + cnt); if ((info->lines + cnt)->measure == NULL) break; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 7535cbfd0c..f2443080a6 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -103,7 +103,10 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { return code; } char* tmp = taosMemoryCalloc(pVal->length, 1); - memcpy(tmp, pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN); + if (tmp == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(tmp, pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN); code = doGeomFromText(tmp, (unsigned char **)&pVal->value, &pVal->length); taosMemoryFree(tmp); if (code != TSDB_CODE_SUCCESS) { @@ -212,8 +215,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ while (*sql < sqlEnd) { if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); - terrno = TSDB_CODE_SML_INVALID_DATA; - return -1; + return TSDB_CODE_SML_INVALID_DATA; } if (unlikely(IS_EQUAL(*sql,escapeChar))) { keyLen = *sql - key; @@ -230,8 +232,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); - terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - return -1; + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } // parse value @@ -245,8 +246,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ break; } else if (unlikely(IS_EQUAL(*sql,escapeChar))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); - terrno = TSDB_CODE_SML_INVALID_DATA; - return -1; + return TSDB_CODE_SML_INVALID_DATA; } if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) { @@ -261,24 +261,28 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ if (unlikely(valueLen == 0)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); - terrno = TSDB_CODE_SML_INVALID_DATA; - return -1; + return TSDB_CODE_SML_INVALID_DATA; } if (unlikely(valueLen - valueLenEscaped > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { - terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - return -1; + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; } if (keyEscaped) { char *tmp = (char *)taosMemoryMalloc(keyLen); - memcpy(tmp, key, keyLen); + if (tmp == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(tmp, key, keyLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, keyLen); key = tmp; } if (valueEscaped) { char *tmp = (char *)taosMemoryMalloc(valueLen); - memcpy(tmp, value, valueLen); + if (tmp == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(tmp, value, valueLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, valueLen); value = tmp; } @@ -289,11 +293,12 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ .length = valueLen, .keyEscaped = keyEscaped, .valueEscaped = valueEscaped}; - taosArrayPush(preLineKV, &kv); + if(taosArrayPush(preLineKV, &kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { - terrno = TSDB_CODE_SUCCESS; - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } cnt++; @@ -302,7 +307,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ } (*sql)++; } - return 0; + return TSDB_CODE_SUCCESS; } static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *elements) { @@ -315,13 +320,19 @@ static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if(info->dataFormat){ ret = smlProcessSuperTable(info, elements); if(ret != 0){ - return terrno; + if(info->reRun){ + return TSDB_CODE_SUCCESS; + } + return ret; } } ret = smlProcessTagLine(info, sql, sqlEnd); if(ret != 0){ - return terrno; + if (info->reRun){ + return TSDB_CODE_SUCCESS; + } + return ret; } return smlProcessChildTable(info, elements); @@ -410,7 +421,10 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if (keyEscaped) { char *tmp = (char *)taosMemoryMalloc(kv.keyLen); - memcpy(tmp, key, kv.keyLen); + if (tmp == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(tmp, key, kv.keyLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, kv.keyLen); kv.key = tmp; kv.keyEscaped = keyEscaped; @@ -418,7 +432,10 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if (valueEscaped) { char *tmp = (char *)taosMemoryMalloc(kv.length); - memcpy(tmp, kv.value, kv.length); + if (tmp == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(tmp, kv.value, kv.length); PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length); ASSERT(kv.type != TSDB_DATA_TYPE_GEOMETRY); if(kv.type == TSDB_DATA_TYPE_VARBINARY){ @@ -437,8 +454,13 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL } else { if (currElement->colArray == NULL) { currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1); + if (currElement->colArray == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + if (taosArrayPush(currElement->colArray, &kv) == NULL){ // reserve for timestamp + return TSDB_CODE_OUT_OF_MEMORY; } - taosArrayPush(currElement->colArray, &kv); // reserve for timestamp } cnt++; diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index bc0e560178..ddd2cf5ad0 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -84,8 +84,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ while (sql < sqlEnd) { if (unlikely(*sql == SPACE)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); - terrno = TSDB_CODE_SML_INVALID_DATA; - return -1; + return TSDB_CODE_SML_INVALID_DATA; } if (unlikely(*sql == EQUAL)) { keyLen = sql - key; @@ -97,8 +96,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ if (unlikely(IS_INVALID_COL_LEN(keyLen))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); - terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - return -1; + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } // parse value @@ -111,8 +109,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ } if (unlikely(*sql == EQUAL)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); - terrno = TSDB_CODE_SML_INVALID_DATA; - return -1; + return TSDB_CODE_SML_INVALID_DATA; } sql++; } @@ -120,13 +117,11 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ if (unlikely(valueLen == 0)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); - terrno = TSDB_CODE_TSC_INVALID_VALUE; - return -1; + return TSDB_CODE_TSC_INVALID_VALUE; } if (unlikely(valueLen > (TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { - terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - return -1; + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; } SSmlKv kv = {.key = key, @@ -138,12 +133,11 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ .valueEscaped = false}; taosArrayPush(preLineKV, &kv); if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { - terrno = TSDB_CODE_SUCCESS; - return -1; + return TSDB_CODE_SML_INVALID_DATA; } cnt++; } - return 0; + return TSDB_CODE_SUCCESS; } static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements) { @@ -156,13 +150,19 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS if(info->dataFormat){ ret = smlProcessSuperTable(info, elements); if(ret != 0){ - return terrno; + if(info->reRun){ + return TSDB_CODE_SUCCESS; + } + return ret; } } ret = smlProcessTagTelnet(info, data, sqlEnd); if(ret != 0){ - return terrno; + if (info->reRun){ + return TSDB_CODE_SUCCESS; + } + return ret; } ret = smlJoinMeasureTag(elements); diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 054698fff0..77079ed863 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -40,13 +40,16 @@ TEST(testCase, smlParseInfluxString_Test) { msgBuf.len = 256; SSmlLineInfo elements = {0}; - SSmlHandle *info = smlBuildSmlInfo(NULL); + SSmlHandle *info = nullptr; + int32_t code = smlBuildSmlInfo(nullptr, &info); + ASSERT_EQ(code, 0); info->protocol = TSDB_SML_LINE_PROTOCOL; info->dataFormat = false; // case 1 char *tmp = "\\,st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3"; char *sql = (char *)taosMemoryCalloc(256, 1); - memcpy(sql, tmp, strlen(tmp) + 1); + ASSERT_NE(sql, nullptr); + (void)memcpy(sql, tmp, strlen(tmp) + 1); int ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); @@ -62,22 +65,22 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; // case 2 false tmp = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_NE(ret, 0); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; // case 4 tag is null tmp = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000"; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); @@ -92,13 +95,13 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; // case 5 tag is null tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 "; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql + 1); @@ -112,36 +115,36 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + 1 + elements.measureTagsLen + 3 + elements.colsLen + 2); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; // case 6 tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 "; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; - smlClearForRerun(info); + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; + ASSERT_EQ(smlClearForRerun(info), 0); // case 7 tmp = " st , "; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_NE(ret, 0); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; // case 8 false tmp = ", st , "; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_NE(ret, 0); - taosArrayDestroy(elements.colArray); - elements.colArray = NULL; + (void)taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; taosMemoryFree(sql); smlDestroyInfo(info); @@ -228,7 +231,9 @@ TEST(testCase, smlParseCols_Error_Test) { "escape_test,tag1=\"tag1_value\",tag2=\"tag2_value\" col0=\"co\"l\"0_value\",col1=\"col1_value\" 1680918783010000000" }; - SSmlHandle *info = smlBuildSmlInfo(NULL); + SSmlHandle *info = nullptr; + int32_t code = smlBuildSmlInfo(nullptr, &info); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); info->protocol = TSDB_SML_LINE_PROTOCOL; info->dataFormat = false; for (int i = 0; i < sizeof(data) / sizeof(data[0]); i++) { @@ -238,13 +243,14 @@ TEST(testCase, smlParseCols_Error_Test) { msgBuf.len = 256; int32_t len = strlen(data[i]); char *sql = (char *)taosMemoryCalloc(256, 1); - memcpy(sql, data[i], len + 1); + ASSERT_NE(sql, nullptr); + (void)memcpy(sql, data[i], len + 1); SSmlLineInfo elements = {0}; int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements); // printf("i:%d\n", i); ASSERT_NE(ret, TSDB_CODE_SUCCESS); taosMemoryFree(sql); - taosArrayDestroy(elements.colArray); + (void)taosArrayDestroy(elements.colArray); } smlDestroyInfo(info); } @@ -254,7 +260,9 @@ TEST(testCase, smlParseCols_Test) { SSmlMsgBuf msgBuf; msgBuf.buf = msg; msgBuf.len = 256; - SSmlHandle *info = smlBuildSmlInfo(NULL); + SSmlHandle *info = nullptr; + int32_t code = smlBuildSmlInfo(nullptr, &info); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); info->protocol = TSDB_SML_LINE_PROTOCOL; info->dataFormat = false; SSmlLineInfo elements = {0}; @@ -267,7 +275,8 @@ TEST(testCase, smlParseCols_Test) { "cboolf=f,cnch_=l\"iuwq\" 1626006833639000000"; int32_t len = strlen(data); char *sql = (char *)taosMemoryCalloc(1024, 1); - memcpy(sql, data, len + 1); + ASSERT_NE(sql, nullptr); + (void)memcpy(sql, data, len + 1); int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements); ASSERT_EQ(ret, TSDB_CODE_SUCCESS); int32_t size = taosArrayGetSize(elements.colArray); @@ -275,6 +284,7 @@ TEST(testCase, smlParseCols_Test) { // binary SSmlKv *kv = (SSmlKv *)taosArrayGet(elements.colArray, 1); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cb=in", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY); @@ -285,6 +295,7 @@ TEST(testCase, smlParseCols_Test) { // nchar kv = (SSmlKv *)taosArrayGet(elements.colArray, 2); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); @@ -293,6 +304,7 @@ TEST(testCase, smlParseCols_Test) { // bool kv = (SSmlKv *)taosArrayGet(elements.colArray, 3); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cbool", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); @@ -301,6 +313,7 @@ TEST(testCase, smlParseCols_Test) { // double kv = (SSmlKv *)taosArrayGet(elements.colArray, 4); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); @@ -310,6 +323,7 @@ TEST(testCase, smlParseCols_Test) { // float kv = (SSmlKv *)taosArrayGet(elements.colArray, 5); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cf64_", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); @@ -319,6 +333,7 @@ TEST(testCase, smlParseCols_Test) { // float kv = (SSmlKv *)taosArrayGet(elements.colArray, 6); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cf32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); @@ -328,6 +343,7 @@ TEST(testCase, smlParseCols_Test) { // tiny int kv = (SSmlKv *)taosArrayGet(elements.colArray, 7); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "ci8", 3), 0); ASSERT_EQ(kv->keyLen, 3); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_TINYINT); @@ -336,6 +352,7 @@ TEST(testCase, smlParseCols_Test) { // unsigned tiny int kv = (SSmlKv *)taosArrayGet(elements.colArray, 8); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cu8", 3), 0); ASSERT_EQ(kv->keyLen, 3); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UTINYINT); @@ -344,6 +361,7 @@ TEST(testCase, smlParseCols_Test) { // small int kv = (SSmlKv *)taosArrayGet(elements.colArray, 9); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "ci16", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_SMALLINT); @@ -352,6 +370,7 @@ TEST(testCase, smlParseCols_Test) { // unsigned smallint kv = (SSmlKv *)taosArrayGet(elements.colArray, 10); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cu16", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_USMALLINT); @@ -360,6 +379,7 @@ TEST(testCase, smlParseCols_Test) { // int kv = (SSmlKv *)taosArrayGet(elements.colArray, 11); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "ci32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_INT); @@ -368,6 +388,7 @@ TEST(testCase, smlParseCols_Test) { // unsigned int kv = (SSmlKv *)taosArrayGet(elements.colArray, 12); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cu32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UINT); @@ -376,6 +397,7 @@ TEST(testCase, smlParseCols_Test) { // bigint kv = (SSmlKv *)taosArrayGet(elements.colArray, 13); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "ci64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); @@ -384,6 +406,7 @@ TEST(testCase, smlParseCols_Test) { // bigint kv = (SSmlKv *)taosArrayGet(elements.colArray, 14); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "ci", 2), 0); ASSERT_EQ(kv->keyLen, 2); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); @@ -392,6 +415,7 @@ TEST(testCase, smlParseCols_Test) { // unsigned bigint kv = (SSmlKv *)taosArrayGet(elements.colArray, 15); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cu64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UBIGINT); @@ -400,6 +424,7 @@ TEST(testCase, smlParseCols_Test) { // bool kv = (SSmlKv *)taosArrayGet(elements.colArray, 16); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cbooltrue", 9), 0); ASSERT_EQ(kv->keyLen, 9); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); @@ -408,6 +433,7 @@ TEST(testCase, smlParseCols_Test) { // bool kv = (SSmlKv *)taosArrayGet(elements.colArray, 17); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cboolt", 6), 0); ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); @@ -416,6 +442,7 @@ TEST(testCase, smlParseCols_Test) { // bool kv = (SSmlKv *)taosArrayGet(elements.colArray, 18); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cboolf", 6), 0); ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); @@ -424,13 +451,14 @@ TEST(testCase, smlParseCols_Test) { // nchar kv = (SSmlKv *)taosArrayGet(elements.colArray, 19); + ASSERT_NE(kv, nullptr); ASSERT_EQ(strncasecmp(kv->key, "cnch_", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); ASSERT_EQ(kv->length, 4); ASSERT_EQ(strncasecmp(kv->value, "iuwq", 4), 0); - taosArrayDestroy(elements.colArray); + (void)taosArrayDestroy(elements.colArray); taosMemoryFree(sql); smlDestroyInfo(info); } @@ -471,7 +499,9 @@ TEST(testCase, smlParseNumber_Test) { } TEST(testCase, smlParseTelnetLine_error_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL); + SSmlHandle *info = nullptr; + int32_t code = smlBuildSmlInfo(nullptr, &info); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); info->dataFormat = false; info->protocol = TSDB_SML_TELNET_PROTOCOL; ASSERT_NE(info, nullptr); @@ -509,7 +539,9 @@ TEST(testCase, smlParseTelnetLine_error_Test) { } TEST(testCase, smlParseTelnetLine_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL); + SSmlHandle *info = nullptr; + int32_t code = smlBuildSmlInfo(nullptr, &info); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); info->dataFormat = false; info->protocol = TSDB_SML_TELNET_PROTOCOL; ASSERT_NE(info, nullptr); @@ -528,7 +560,9 @@ TEST(testCase, smlParseTelnetLine_Test) { } //TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { -// SSmlHandle *info = smlBuildSmlInfo(NULL); +// SSmlHandle *info = nullptr; +// int32_t code = smlBuildSmlInfo(nullptr, &info); +// ASSERT_EQ(code, TSDB_CODE_SUCCESS); // info->protocol = TSDB_SML_JSON_PROTOCOL; // ASSERT_NE(info, nullptr); // @@ -568,7 +602,7 @@ TEST(testCase, smlParseNumber_performance_Test) { for (int j = 0; j < 10000000; ++j) { kv.value = str[i]; kv.length = strlen(str[i]); - smlParseNumber(&kv, &msgBuf); + (void)smlParseNumber(&kv, &msgBuf); //ignore result } printf("smlParseNumber:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t1); printf("\n"); @@ -576,7 +610,7 @@ TEST(testCase, smlParseNumber_performance_Test) { for (int j = 0; j < 10000000; ++j) { kv.value = str[i]; kv.length = strlen(str[i]); - smlParseNumberOld(&kv, &msgBuf); + (void)smlParseNumberOld(&kv, &msgBuf); //ignore result } printf("smlParseNumberOld:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t2); printf("\n\n"); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 809721d606..9b5ecb4693 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2486,7 +2486,10 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, cha RandTableName rname = { .tags = tags, .stbFullName = stbFullName, .stbFullNameLen = strlen(stbFullName), .ctbShortName = cname}; - buildChildTableName(&rname); + int32_t code = buildChildTableName(&rname); + if(code != TSDB_CODE_SUCCESS){ + return code; + } taosArrayDestroy(tags); if ((rname.ctbShortName && rname.ctbShortName[0]) == 0) { diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 4fe2beb6aa..e4604c6bab 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -293,17 +293,19 @@ static int compareKv(const void* p1, const void* p2) { /* * use stable name and tags to grearate child table name */ -void buildChildTableName(RandTableName* rName) { +int32_t buildChildTableName(RandTableName* rName) { SStringBuilder sb = {0}; taosStringBuilderAppendStringLen(&sb, rName->stbFullName, rName->stbFullNameLen); if (sb.buf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return; + return TSDB_CODE_OUT_OF_MEMORY; } taosArraySort(rName->tags, compareKv); for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) { taosStringBuilderAppendChar(&sb, ','); SSmlKv* tagKv = taosArrayGet(rName->tags, j); + if(tagKv == NULL) { + return TSDB_CODE_SML_INVALID_DATA; + } taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen); taosStringBuilderAppendChar(&sb, '='); if (IS_VAR_DATA_TYPE(tagKv->type)) { @@ -323,8 +325,9 @@ void buildChildTableName(RandTableName* rName) { rName->ctbShortName[0] = 't'; rName->ctbShortName[1] = '_'; for (int i = 0; i < 16; i++) { - sprintf(temp, "%02x", context.digest[i]); - strcat(rName->ctbShortName, temp); + (void)sprintf(temp, "%02x", context.digest[i]); + (void)strcat(rName->ctbShortName, temp); } taosStringBuilderDestroy(&sb); + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index db2d34b844..89eaf14f75 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -112,7 +112,11 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem for (int i = 0; i < tags->numOfBound; ++i) { SSchema* pTagSchema = &pSchema[tags->pColIndex[i]]; SSmlKv* kv = taosArrayGet(cols, i); - + if (kv == NULL){ + code = TSDB_CODE_SML_INVALID_DATA; + uError("SML smlBuildTagRow error kv is null"); + goto end; + } if (kv->keyLen != strlen(pTagSchema->name) || memcmp(kv->key, pTagSchema->name, kv->keyLen) != 0 || kv->type != pTagSchema->type) { code = TSDB_CODE_SML_INVALID_DATA; @@ -120,7 +124,11 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem goto end; } - taosArrayPush(*tagName, pTagSchema->name); + if (taosArrayPush(*tagName, pTagSchema->name) == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + uError("SML smlBuildTagRow error push tag name"); + goto end; + } STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; // strcpy(val.colName, pTagSchema->name); if (pTagSchema->type == TSDB_DATA_TYPE_BINARY || pTagSchema->type == TSDB_DATA_TYPE_VARBINARY || @@ -141,7 +149,7 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem goto end; } char buf[512] = {0}; - snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); + (void)snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); taosMemoryFree(p); code = buildSyntaxErrMsg(msg, buf, kv->value); goto end; @@ -149,9 +157,13 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem val.pData = p; val.nData = output; } else { - memcpy(&val.i64, &(kv->value), kv->length); + (void)memcpy(&val.i64, &(kv->value), kv->length); + } + if (taosArrayPush(pTagArray, &val) == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + uError("SML smlBuildTagRow error push tag array"); + goto end; } - taosArrayPush(pTagArray, &val); } code = tTagNew(pTagArray, 1, false, ppTag); @@ -166,20 +178,19 @@ end: return code; } -STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta) { - STableDataCxt* pTableCxt = NULL; +int32_t smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta, STableDataCxt** cxt) { SVCreateTbReq* pCreateTbReq = NULL; int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false, false); + sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, cxt, false, false); if (ret != TSDB_CODE_SUCCESS) { - return NULL; + return ret; } - ret = initTableColSubmitData(pTableCxt); + ret = initTableColSubmitData(*cxt); if (ret != TSDB_CODE_SUCCESS) { - return NULL; + return ret; } - return pTableCxt; + return TSDB_CODE_SUCCESS; } void clearColValArraySml(SArray* pCols) { @@ -197,6 +208,9 @@ void clearColValArraySml(SArray* pCols) { int32_t smlBuildRow(STableDataCxt* pTableCxt) { SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); + if (pRow == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } int ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); if (TSDB_CODE_SUCCESS != ret) { return ret; @@ -211,18 +225,22 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32 int ret = TSDB_CODE_SUCCESS; SSchema* pColSchema = schema + index; SColVal* pVal = taosArrayGet(pTableCxt->pValues, index); + if (pVal == NULL) { + return TSDB_CODE_SUCCESS; + } SSmlKv* kv = (SSmlKv*)data; if (kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 || kv->type != pColSchema->type) { ret = TSDB_CODE_SML_INVALID_DATA; char* tmp = taosMemoryCalloc(kv->keyLen + 1, 1); if (tmp) { - memcpy(tmp, kv->key, kv->keyLen); + (void)memcpy(tmp, kv->key, kv->keyLen); uInfo("SML data(name:%s type:%s) is not same like the db data(name:%s type:%s)", tmp, tDataTypes[kv->type].name, pColSchema->name, tDataTypes[pColSchema->type].name); taosMemoryFree(tmp); } else { uError("SML smlBuildCol out of memory"); + ret = TSDB_CODE_OUT_OF_MEMORY; } goto end; } @@ -256,9 +274,9 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32 } else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) { pVal->value.nData = kv->length; pVal->value.pData = taosMemoryMalloc(kv->length); - memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length); + (void)memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length); } else { - memcpy(&pVal->value.val, &(kv->value), kv->length); + (void)memcpy(&pVal->value.val, &(kv->value), kv->length); } pVal->flag = CV_FLAG_VALUE; @@ -276,8 +294,13 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc SVCreateTbReq* pCreateTblReq = NULL; SArray* tagName = NULL; - insInitBoundColsInfo(getNumOfTags(pTableMeta), &bindTags); - int ret = smlBoundColumnData(tags, &bindTags, pTagsSchema, true); + int ret = insInitBoundColsInfo(getNumOfTags(pTableMeta), &bindTags); + if (ret != TSDB_CODE_SUCCESS) { + buildInvalidOperationMsg(&pBuf, "init bound cols error"); + goto end; + } + + ret = smlBoundColumnData(tags, &bindTags, pTagsSchema, true); if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "bound tags error"); goto end; @@ -299,7 +322,11 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc ttl); pCreateTblReq->ctb.stbName = taosMemoryCalloc(1, sTableNameLen + 1); - memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen); + if (pCreateTblReq->ctb.stbName == NULL){ + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + (void)memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen); if (dataFormat) { STableDataCxt** pTableCxt = (STableDataCxt**)taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, @@ -345,11 +372,18 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc for (int32_t r = 0; r < rowNum; ++r) { void* rowData = taosArrayGetP(cols, r); - + if (rowData == NULL) { + ret = TSDB_CODE_SML_INVALID_DATA; + goto end; + } // 1. set the parsed value from sql string for (int c = 0; c < pTableCxt->boundColsInfo.numOfBound; ++c) { SSchema* pColSchema = &pSchema[pTableCxt->boundColsInfo.pColIndex[c]]; SColVal* pVal = taosArrayGet(pTableCxt->pValues, pTableCxt->boundColsInfo.pColIndex[c]); + if (pVal == NULL) { + ret = TSDB_CODE_SML_INVALID_DATA; + goto end; + } void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name)); if (p == NULL) { continue; @@ -388,14 +422,22 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc } else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) { pVal->value.nData = kv->length; pVal->value.pData = taosMemoryMalloc(kv->length); - memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length); + if (NULL == pVal->value.pData) { + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + (void)memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length); } else { - memcpy(&pVal->value.val, &(kv->value), kv->length); + (void)memcpy(&pVal->value.val, &(kv->value), kv->length); } pVal->flag = CV_FLAG_VALUE; } SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); + if (NULL == pRow) { + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); if (TSDB_CODE_SUCCESS != ret) { buildInvalidOperationMsg(&pBuf, "tRowBuild error"); @@ -415,11 +457,12 @@ end: return ret; } -SQuery* smlInitHandle() { +int32_t smlInitHandle(SQuery** query) { + *query = NULL; SQuery* pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); if (NULL == pQuery) { uError("create pQuery error"); - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->haveResultSet = false; @@ -428,14 +471,22 @@ SQuery* smlInitHandle() { if (NULL == stmt) { uError("create SVnodeModifyOpStmt error"); qDestroyQuery(pQuery); - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (stmt->pTableBlockHashObj == NULL){ + uError("create pTableBlockHashObj error"); + qDestroyQuery(pQuery); + nodesDestroyNode((SNode*)stmt); + return TSDB_CODE_OUT_OF_MEMORY; + } stmt->freeHashFunc = insDestroyTableDataCxtHashMap; stmt->freeArrayFunc = insDestroyVgroupDataCxtList; pQuery->pRoot = (SNode*)stmt; - return pQuery; + *query = pQuery; + + return TSDB_CODE_SUCCESS; } int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash) { From 51e9723d6673360f48f1ee397ce6516d82b45b20 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 15 Jul 2024 19:37:52 +0800 Subject: [PATCH 16/34] refactor: do some refactor. --- include/util/tarray.h | 2 +- source/client/src/clientHb.c | 5 ++- source/common/src/tdatablock.c | 4 ++- source/common/src/tmsg.c | 8 +++-- source/dnode/mnode/impl/src/mndStreamUtil.c | 3 +- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tq/tqSink.c | 12 +++++-- source/dnode/vnode/src/tq/tqUtil.c | 12 ++++--- source/dnode/vnode/src/tsdb/tsdbCache.c | 3 +- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 3 +- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 5 ++- source/dnode/vnode/src/tsdb/tsdbRead2.c | 16 +++++++--- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 31 ++++++++++++------- source/libs/executor/src/executil.c | 6 ++-- source/libs/executor/src/streamfilloperator.c | 4 ++- .../executor/src/streamtimewindowoperator.c | 3 +- source/libs/executor/src/timewindowoperator.c | 5 +-- source/libs/stream/src/streamCheckStatus.c | 4 ++- source/libs/stream/src/streamMsg.c | 6 ++-- source/libs/stream/src/streamTask.c | 17 +++++++--- source/util/src/tarray.c | 4 +-- source/util/src/tconfig.c | 3 +- 22 files changed, 105 insertions(+), 53 deletions(-) diff --git a/include/util/tarray.h b/include/util/tarray.h index e494f78f48..cef09e40f8 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -205,7 +205,7 @@ void taosArrayClearEx(SArray* pArray, void (*fp)(void*)); void taosArrayClearP(SArray* pArray, void (*fp)(void*)); -void* taosArrayDestroy(SArray* pArray); +void taosArrayDestroy(SArray* pArray); void taosArrayDestroyP(SArray* pArray, FDelete fp); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 1b7b9263a5..d0ba4c6773 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -1398,7 +1398,10 @@ void hbMgrCleanUp() { taosThreadMutexLock(&clientHbMgr.lock); appHbMgrCleanup(); - clientHbMgr.appHbMgrs = taosArrayDestroy(clientHbMgr.appHbMgrs); + + taosArrayDestroy(clientHbMgr.appHbMgrs); + clientHbMgr.appHbMgrs = NULL; + taosThreadMutexUnlock(&clientHbMgr.lock); } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 8f5d382cde..142e75dbcb 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1516,7 +1516,9 @@ void blockDataFreeRes(SSDataBlock* pBlock) { colDataDestroy(pColInfoData); } - pBlock->pDataBlock = taosArrayDestroy(pBlock->pDataBlock); + taosArrayDestroy(pBlock->pDataBlock); + pBlock->pDataBlock = NULL; + taosMemoryFreeClear(pBlock->pBlockAgg); memset(&pBlock->info, 0, sizeof(SDataBlockInfo)); } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 7e89753241..cb097a8fe7 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -9505,7 +9505,9 @@ int32_t tDecodeMqDataRsp(SDecoder *pDecoder, void *pRsp) { static void tDeleteMqDataRspCommon(void *rsp) { SMqDataRspCommon *pRsp = rsp; - pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen); + taosArrayDestroy(pRsp->blockDataLen); + pRsp->blockDataLen = NULL; + taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree); pRsp->blockData = NULL; taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSchemaWrapper); @@ -9558,7 +9560,9 @@ void tDeleteSTaosxRsp(void *rsp) { tDeleteMqDataRspCommon(rsp); STaosxRsp *pRsp = (STaosxRsp *)rsp; - pRsp->createTableLen = taosArrayDestroy(pRsp->createTableLen); + taosArrayDestroy(pRsp->createTableLen); + pRsp->createTableLen = NULL; + taosArrayDestroyP(pRsp->createTableReq, (FDelete)taosMemoryFree); pRsp->createTableReq = NULL; } diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index c4adbd0fc3..e4e30bdf10 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -978,7 +978,8 @@ void mndAddConsensusTasks(SCheckpointConsensusInfo *pInfo, const SRestoreCheckpo } void mndClearConsensusRspEntry(SCheckpointConsensusInfo* pInfo) { - pInfo->pTaskList = taosArrayDestroy(pInfo->pTaskList); + taosArrayDestroy(pInfo->pTaskList); + pInfo->pTaskList = NULL; } int64_t mndClearConsensusCheckpointId(SHashObj* pHash, int64_t streamId) { diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 84e1996f2c..241a47aca9 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -911,7 +911,7 @@ void tMergeTreeClose(SMergeTree *pMTree); SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols); void * destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); -void * destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost); +void destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost); // tsdbCache ============================================================================================== typedef enum { diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 5f3e1e3d14..d3327b5d6b 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -249,7 +249,9 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S } tTagNew(tagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag); - tagArray = taosArrayDestroy(tagArray); + taosArrayDestroy(tagArray); + tagArray = NULL; + if (pCreateTbReq->ctb.pTag == NULL) { tdDestroySVCreateTbReq(pCreateTbReq); code = TSDB_CODE_OUT_OF_MEMORY; @@ -505,7 +507,9 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat pTableData->aRowP = taosArrayInit(numOfRows, sizeof(SRow*)); if (pTableData->aRowP == NULL || pVals == NULL) { - pTableData->aRowP = taosArrayDestroy(pTableData->aRowP); + taosArrayDestroy(pTableData->aRowP); + pTableData->aRowP = NULL; + taosArrayDestroy(pVals); code = TSDB_CODE_OUT_OF_MEMORY; tqError("s-task:%s failed to prepare write stream res blocks, code:%s", id, tstrerror(code)); @@ -530,7 +534,9 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat if (ts < earlyTs) { tqError("s-task:%s ts:%" PRId64 " of generated results out of valid time range %" PRId64 " , discarded", id, ts, earlyTs); - pTableData->aRowP = taosArrayDestroy(pTableData->aRowP); + taosArrayDestroy(pTableData->aRowP); + pTableData->aRowP = NULL; + taosArrayDestroy(pVals); return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 5290c39d42..eb1da51b45 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -54,19 +54,23 @@ static int32_t tqInitTaosxRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset) { if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL || pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) { if (pRsp->blockData != NULL) { - pRsp->blockData = taosArrayDestroy(pRsp->blockData); + taosArrayDestroy(pRsp->blockData); + pRsp->blockData = NULL; } if (pRsp->blockDataLen != NULL) { - pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen); + taosArrayDestroy(pRsp->blockDataLen); + pRsp->blockDataLen = NULL; } if (pRsp->blockTbName != NULL) { - pRsp->blockTbName = taosArrayDestroy(pRsp->blockTbName); + taosArrayDestroy(pRsp->blockTbName); + pRsp->blockTbName = NULL; } if (pRsp->blockSchema != NULL) { - pRsp->blockSchema = taosArrayDestroy(pRsp->blockSchema); + taosArrayDestroy(pRsp->blockSchema); + pRsp->blockSchema = NULL; } return -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 4597a44845..38dd945b28 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -3187,7 +3187,8 @@ static void clearLastFileSet(SFSNextRowIter *state) { int32_t iter = 0; while ((pe = tSimpleHashIterate(state->pr->pTableMap, pe, &iter)) != NULL) { STableLoadInfo *pInfo = *(STableLoadInfo **)pe; - pInfo->pTombData = taosArrayDestroy(pInfo->pTombData); + taosArrayDestroy(pInfo->pTombData); + pInfo->pTombData = NULL; } } } diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 4d6ebf721e..817468c6b6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -311,7 +311,8 @@ void* tsdbCacherowsReaderClose(void* pReader) { int32_t iter = 0; while ((pe = tSimpleHashIterate(p->pTableMap, pe, &iter)) != NULL) { STableLoadInfo* pInfo = *(STableLoadInfo**)pe; - pInfo->pTombData = taosArrayDestroy(pInfo->pTombData); + taosArrayDestroy(pInfo->pTombData); + pInfo->pTombData = NULL; } tSimpleHashCleanup(p->pTableMap); diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 57b8a99fb1..fbda31f731 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -103,9 +103,9 @@ void destroyLDataIter(SLDataIter *pIter) { taosMemoryFree(pIter); } -void *destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost) { +void destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost) { if (pLDataIterArray == NULL) { - return NULL; + return; } int32_t numOfLevel = taosArrayGetSize(pLDataIterArray); @@ -132,7 +132,6 @@ void *destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoa } taosArrayDestroy(pLDataIterArray); - return NULL; } // choose the unpinned slot to load next data block diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index ec7d90184c..dc80fcabfb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -246,6 +246,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, TFileSetArray* pFileSetA SBlockLoadSuppInfo* pInfo = &pReader->suppInfo; size_t numOfFileset = TARRAY2_SIZE(pFileSetArray); bool asc = ASCENDING_TRAVERSE(pReader->info.order); + int32_t code = TSDB_CODE_SUCCESS; pIter->index = asc ? -1 : numOfFileset; pIter->order = pReader->info.order; @@ -255,7 +256,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, TFileSetArray* pFileSetA if (pIter->pSttBlockReader == NULL) { pIter->pSttBlockReader = taosMemoryCalloc(1, sizeof(struct SSttBlockReader)); if (pIter->pSttBlockReader == NULL) { - int32_t code = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; tsdbError("failed to prepare the last block iterator, since:%s %s", tstrerror(code), pReader->idStr); return code; } @@ -269,10 +270,14 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, TFileSetArray* pFileSetA pSttReader->uid = 0; tMergeTreeClose(&pSttReader->mergeTree); - initRowKey(&pSttReader->currentKey, INT64_MIN, pInfo->numOfPks, pInfo->pk.type, pInfo->pk.bytes, asc); + code = initRowKey(&pSttReader->currentKey, INT64_MIN, pInfo->numOfPks, pInfo->pk.type, pInfo->pk.bytes, asc); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("failed init row key, %s", pReader->idStr); + } else { + tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, pReader->idStr); + } - tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, pReader->idStr); - return TSDB_CODE_SUCCESS; + return code; } static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bool* hasNext) { @@ -290,7 +295,8 @@ static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bo pIter->pSttBlockReader->uid = 0; tMergeTreeClose(&pIter->pSttBlockReader->mergeTree); - pReader->status.pLDataIterArray = destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost); + destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost); + pReader->status.pLDataIterArray = NULL; pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES); // check file the time range of coverage diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index 273ec6d797..332ed4d05c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -146,17 +146,17 @@ int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, in } case TSDB_DATA_TYPE_INT:{ int32_t min = INT32_MIN; - memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); + (void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); break; } case TSDB_DATA_TYPE_SMALLINT:{ int16_t min = INT16_MIN; - memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); + (void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); break; } case TSDB_DATA_TYPE_TINYINT:{ int8_t min = INT8_MIN; - memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); + (void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); break; } case TSDB_DATA_TYPE_UTINYINT: @@ -308,7 +308,9 @@ void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) { pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter); } - pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); + taosArrayDestroy(pInfo->delSkyline); + pInfo->delSkyline = NULL; + pInfo->lastProcKey.ts = ts; // todo check the nextProcKey info pInfo->sttKeyInfo.nextProcKey.ts = ts + step; @@ -329,11 +331,16 @@ void clearBlockScanInfo(STableBlockScanInfo* p) { p->iiter.iter = tsdbTbDataIterDestroy(p->iiter.iter); } - p->delSkyline = taosArrayDestroy(p->delSkyline); - p->pBlockList = taosArrayDestroy(p->pBlockList); - p->pBlockIdxList = taosArrayDestroy(p->pBlockIdxList); - p->pMemDelData = taosArrayDestroy(p->pMemDelData); - p->pFileDelData = taosArrayDestroy(p->pFileDelData); + taosArrayDestroy(p->delSkyline); + p->delSkyline = NULL; + taosArrayDestroy(p->pBlockList); + p->pBlockList = NULL; + taosArrayDestroy(p->pBlockIdxList); + p->pBlockIdxList = NULL; + taosArrayDestroy(p->pMemDelData); + p->pMemDelData = NULL; + taosArrayDestroy(p->pFileDelData); + p->pFileDelData = NULL; clearRowKey(&p->lastProcKey); clearRowKey(&p->sttRange.skey); @@ -579,7 +586,8 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3 } taosArrayAddAll(pBlockIter->blockList, pTableScanInfo->pBlockList); - pTableScanInfo->pBlockList = taosArrayDestroy(pTableScanInfo->pBlockList); + taosArrayDestroy(pTableScanInfo->pBlockList); + pTableScanInfo->pBlockList = NULL; int64_t et = taosGetTimestampUs(); tsdbDebug("%p create blocks info struct completed for one table, %d blocks not sorted, elapsed time:%.2f ms %s", @@ -624,7 +632,8 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3 for (int32_t i = 0; i < numOfTables; ++i) { STableBlockScanInfo* pTableScanInfo = taosArrayGetP(pTableList, i); - pTableScanInfo->pBlockList = taosArrayDestroy(pTableScanInfo->pBlockList); + taosArrayDestroy(pTableScanInfo->pBlockList); + pTableScanInfo->pBlockList = NULL; } int64_t et = taosGetTimestampUs(); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 173ef0b829..c623e94a12 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -96,7 +96,8 @@ void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) { pGroupResInfo->freeItem = false; pGroupResInfo->pRows = NULL; } else { - pGroupResInfo->pRows = taosArrayDestroy(pGroupResInfo->pRows); + taosArrayDestroy(pGroupResInfo->pRows); + pGroupResInfo->pRows = NULL; } pGroupResInfo->index = 0; } @@ -2102,7 +2103,8 @@ void* tableListDestroy(STableListInfo* pTableListInfo) { return NULL; } - pTableListInfo->pTableList = taosArrayDestroy(pTableListInfo->pTableList); + taosArrayDestroy(pTableListInfo->pTableList); + pTableListInfo->pTableList = NULL; taosMemoryFreeClear(pTableListInfo->groupOffset); taosHashCleanup(pTableListInfo->map); diff --git a/source/libs/executor/src/streamfilloperator.c b/source/libs/executor/src/streamfilloperator.c index c1a38b66ba..0002c94fa7 100644 --- a/source/libs/executor/src/streamfilloperator.c +++ b/source/libs/executor/src/streamfilloperator.c @@ -129,7 +129,9 @@ static void destroyStreamFillOperatorInfo(void* param) { pInfo->pRes = blockDataDestroy(pInfo->pRes); pInfo->pSrcBlock = blockDataDestroy(pInfo->pSrcBlock); pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); - pInfo->matchInfo.pList = taosArrayDestroy(pInfo->matchInfo.pList); + + taosArrayDestroy(pInfo->matchInfo.pList); + pInfo->matchInfo.pList = NULL; taosMemoryFree(pInfo); } diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index d68108b9a5..d3f90c3134 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -406,7 +406,8 @@ void clearGroupResInfo(SGroupResInfo* pGroupResInfo) { } pGroupResInfo->freeItem = false; } - pGroupResInfo->pRows = taosArrayDestroy(pGroupResInfo->pRows); + taosArrayDestroy(pGroupResInfo->pRows); + pGroupResInfo->pRows = NULL; pGroupResInfo->index = 0; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index bad15d1834..7455296f0f 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1136,9 +1136,10 @@ void destroyIntervalOperatorInfo(void* param) { tdListFree(pInfo->binfo.resultRowInfo.openWindow); - pInfo->pInterpCols = taosArrayDestroy(pInfo->pInterpCols); - taosArrayDestroyEx(pInfo->pPrevValues, freeItem); + taosArrayDestroy(pInfo->pInterpCols); + pInfo->pInterpCols = NULL; + taosArrayDestroyEx(pInfo->pPrevValues, freeItem); pInfo->pPrevValues = NULL; cleanupGroupResInfo(&pInfo->groupResInfo); diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 226a06be7e..3ea3e814ed 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -308,7 +308,9 @@ int32_t streamTaskStopMonitorCheckRsp(STaskCheckInfo* pInfo, const char* id) { void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) { ASSERT(pInfo->inCheckProcess == 0); - pInfo->pList = taosArrayDestroy(pInfo->pList); + taosArrayDestroy(pInfo->pList); + pInfo->pList = NULL; + if (pInfo->checkRspTmr != NULL) { /*bool ret = */ taosTmrStop(pInfo->checkRspTmr); pInfo->checkRspTmr = NULL; diff --git a/source/libs/stream/src/streamMsg.c b/source/libs/stream/src/streamMsg.c index 40582b5144..1cc48f02b6 100644 --- a/source/libs/stream/src/streamMsg.c +++ b/source/libs/stream/src/streamMsg.c @@ -438,11 +438,13 @@ void tCleanupStreamHbMsg(SStreamHbMsg* pMsg) { } if (pMsg->pUpdateNodes != NULL) { - pMsg->pUpdateNodes = taosArrayDestroy(pMsg->pUpdateNodes); + taosArrayDestroy(pMsg->pUpdateNodes); + pMsg->pUpdateNodes = NULL; } if (pMsg->pTaskStatus != NULL) { - pMsg->pTaskStatus = taosArrayDestroy(pMsg->pTaskStatus); + taosArrayDestroy(pMsg->pTaskStatus); + pMsg->pTaskStatus = NULL; } pMsg->msgId = -1; diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 9bcad87264..2fb0adbdf4 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -280,10 +280,12 @@ void tFreeStreamTask(SStreamTask* pTask) { taosMemoryFree(pTask->outputInfo.pTokenBucket); taosThreadMutexDestroy(&pTask->lock); - pTask->msgInfo.pSendInfo = taosArrayDestroy(pTask->msgInfo.pSendInfo); + taosArrayDestroy(pTask->msgInfo.pSendInfo); + pTask->msgInfo.pSendInfo = NULL; taosThreadMutexDestroy(&pTask->msgInfo.lock); - pTask->outputInfo.pNodeEpsetUpdateList = taosArrayDestroy(pTask->outputInfo.pNodeEpsetUpdateList); + taosArrayDestroy(pTask->outputInfo.pNodeEpsetUpdateList); + pTask->outputInfo.pNodeEpsetUpdateList = NULL; if ((pTask->status.removeBackendFiles) && (pTask->pMeta != NULL)) { char* path = taosMemoryCalloc(1, strlen(pTask->pMeta->path) + 128); @@ -1055,9 +1057,14 @@ void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo) { } taosThreadMutexDestroy(&pInfo->lock); - pInfo->pDispatchTriggerList = taosArrayDestroy(pInfo->pDispatchTriggerList); - pInfo->pReadyMsgList = taosArrayDestroy(pInfo->pReadyMsgList); - pInfo->pCheckpointReadyRecvList = taosArrayDestroy(pInfo->pCheckpointReadyRecvList); + taosArrayDestroy(pInfo->pDispatchTriggerList); + pInfo->pDispatchTriggerList = NULL; + + taosArrayDestroy(pInfo->pReadyMsgList); + pInfo->pReadyMsgList = NULL; + + taosArrayDestroy(pInfo->pCheckpointReadyRecvList); + pInfo->pCheckpointReadyRecvList = NULL; if (pInfo->pChkptTriggerTmr != NULL) { taosTmrStop(pInfo->pChkptTriggerTmr); diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index d9686d77f8..e5f9fe43e7 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -387,13 +387,11 @@ void taosArrayClearP(SArray* pArray, void (*fp)(void*)) { taosArrayClear(pArray); } -void* taosArrayDestroy(SArray* pArray) { +void taosArrayDestroy(SArray* pArray) { if (pArray) { taosMemoryFree(pArray->pData); taosMemoryFree(pArray); } - - return NULL; } void taosArrayDestroyP(SArray* pArray, FDelete fp) { diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index adde3a3331..3ab0f83238 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -96,7 +96,8 @@ void cfgItemFreeVal(SConfigItem *pItem) { } if (pItem->array) { - pItem->array = taosArrayDestroy(pItem->array); + taosArrayDestroy(pItem->array); + pItem->array = NULL; } } From dd590f36e27f7fd1356504672cc346b17ee9e4b2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 15 Jul 2024 23:53:12 +0800 Subject: [PATCH 17/34] refactor: do some refactor. --- include/common/tdataformat.h | 2 +- source/common/src/tdataformat.c | 76 +++++---- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead2.c | 169 ++++++++++++++------ source/dnode/vnode/src/tsdb/tsdbReadUtil.h | 8 +- 6 files changed, 161 insertions(+), 98 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 7aec39817a..04e13fbdb3 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -136,7 +136,7 @@ int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag); int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag); void tRowGetPrimaryKey(SRow *pRow, SRowKey *key); int32_t tRowKeyCompare(const SRowKey *key1, const SRowKey *key2); -int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc); +void tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc); // SRowIter ================================ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 61af2f6aae..27f865353e 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -305,11 +305,11 @@ static int32_t tRowBuildTupleRow(SArray *aColVal, const SRowBuildScanInfo *sinfo *(int32_t *)(fixed + schema->columns[i].offset) = varlen - fixed - sinfo->tupleFixedSize; varlen += tPutU32v(varlen, colValArray[colValIndex].value.nData); if (colValArray[colValIndex].value.nData) { - memcpy(varlen, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData); + (void)memcpy(varlen, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData); varlen += colValArray[colValIndex].value.nData; } } else { - memcpy(fixed + schema->columns[i].offset, &colValArray[colValIndex].value.val, + (void)memcpy(fixed + schema->columns[i].offset, &colValArray[colValIndex].value.val, tDataTypes[schema->columns[i].type].bytes); } } else if (COL_VAL_IS_NULL(&colValArray[colValIndex])) { // NULL @@ -384,12 +384,12 @@ static int32_t tRowBuildKVRow(SArray *aColVal, const SRowBuildScanInfo *sinfo, c payloadSize += tPutI16v(payload + payloadSize, colValArray[colValIndex].cid); payloadSize += tPutU32v(payload + payloadSize, colValArray[colValIndex].value.nData); if (colValArray[colValIndex].value.nData > 0) { - memcpy(payload + payloadSize, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData); + (void)memcpy(payload + payloadSize, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData); } payloadSize += colValArray[colValIndex].value.nData; } else { payloadSize += tPutI16v(payload + payloadSize, colValArray[colValIndex].cid); - memcpy(payload + payloadSize, &colValArray[colValIndex].value.val, + (void)memcpy(payload + payloadSize, &colValArray[colValIndex].value.val, tDataTypes[schema->columns[i].type].bytes); payloadSize += tDataTypes[schema->columns[i].type].bytes; } @@ -475,7 +475,7 @@ int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted, value.nData = infos[iInfo].bind->length[iRow]; value.pData = (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow; } else { - memcpy(&value.val, (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow, + (void)memcpy(&value.val, (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow, infos[iInfo].bind->buffer_length); } colVal = COL_VAL_VALUE(infos[iInfo].columnId, value); @@ -509,7 +509,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) pColVal->cid = pTColumn->colId; pColVal->value.type = pTColumn->type; pColVal->flag = CV_FLAG_VALUE; - memcpy(&pColVal->value.val, &pRow->ts, sizeof(TSKEY)); + (void)memcpy(&pColVal->value.val, &pRow->ts, sizeof(TSKEY)); return 0; } @@ -573,7 +573,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) pColVal->value.pData = NULL; } } else { - memcpy(&pColVal->value.val, pData, pTColumn->bytes); + (void)memcpy(&pColVal->value.val, pData, pTColumn->bytes); } } return 0; @@ -624,7 +624,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) pColVal->value.pData = varlen + *(int32_t *)(fixed + pTColumn->offset); pColVal->value.pData += tGetU32v(pColVal->value.pData, &pColVal->value.nData); } else { - memcpy(&pColVal->value.val, fixed + pTColumn->offset, TYPE_BYTES[pTColumn->type]); + (void)memcpy(&pColVal->value.val, fixed + pTColumn->offset, TYPE_BYTES[pTColumn->type]); } } @@ -861,7 +861,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { pIter->cv.cid = pTColumn->colId; pIter->cv.value.type = pTColumn->type; pIter->cv.flag = CV_FLAG_VALUE; - memcpy(&pIter->cv.value.val, &pIter->pRow->ts, sizeof(TSKEY)); + (void)memcpy(&pIter->cv.value.val, &pIter->pRow->ts, sizeof(TSKEY)); goto _exit; } @@ -906,7 +906,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { pIter->cv.value.pData = NULL; } } else { - memcpy(&pIter->cv.value.val, pData, pTColumn->bytes); + (void)memcpy(&pIter->cv.value.val, pData, pTColumn->bytes); } } @@ -965,7 +965,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { pIter->cv.value.pData = NULL; } } else { - memcpy(&pIter->cv.value.val, pIter->pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]); + (void)memcpy(&pIter->cv.value.val, pIter->pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]); } goto _exit; } @@ -1288,7 +1288,7 @@ void tRowGetPrimaryKey(SRow *row, SRowKey *key) { key->pks[i].pData = tdata; key->pks[i].pData += tGetU32v(key->pks[i].pData, &key->pks[i].nData); } else { - memcpy(&key->pks[i].val, tdata, tDataTypes[indices[i].type].bytes); + (void)memcpy(&key->pks[i].val, tdata, tDataTypes[indices[i].type].bytes); } } } @@ -1378,7 +1378,7 @@ FORCE_INLINE int32_t tRowKeyCompare(const SRowKey *key1, const SRowKey *key2) { return 0; } -int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc) { +void tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc) { pDst->ts = pSrc->ts; pDst->numOfPKs = pSrc->numOfPKs; @@ -1392,12 +1392,10 @@ int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc) { } else { pVal->nData = pSrc->pks[i].nData; ASSERT(pSrc->pks[i].pData != NULL); - memcpy(pVal->pData, pSrc->pks[i].pData, pVal->nData); + (void)memcpy(pVal->pData, pSrc->pks[i].pData, pVal->nData); } } } - - return TSDB_CODE_SUCCESS; } // STag ======================================== @@ -1528,7 +1526,7 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { } else { p = p ? p + n : p; n += tDataTypes[pTagVal->type].bytes; - if (p) memcpy(p, &(pTagVal->i64), tDataTypes[pTagVal->type].bytes); + if (p) (void)memcpy(p, &(pTagVal->i64), tDataTypes[pTagVal->type].bytes); } return n; @@ -1550,7 +1548,7 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { if (IS_VAR_DATA_TYPE(pTagVal->type)) { n += tGetBinary(p + n, &pTagVal->pData, &pTagVal->nData); } else { - memcpy(&(pTagVal->i64), p + n, tDataTypes[pTagVal->type].bytes); + (void)memcpy(&(pTagVal->i64), p + n, tDataTypes[pTagVal->type].bytes); n += tDataTypes[pTagVal->type].bytes; } @@ -1661,7 +1659,7 @@ char *tTagValToData(const STagVal *value, bool isJson) { } varDataLen(data + typeBytes) = value->nData; - memcpy(varDataVal(data + typeBytes), value->pData, value->nData); + (void)memcpy(varDataVal(data + typeBytes), value->pData, value->nData); } else { data = ((char *)&(value->i64)) - typeBytes; // json with type } @@ -1713,7 +1711,7 @@ bool tTagGet(const STag *pTag, STagVal *pTagVal) { } else if (c > 0) { lidx = midx + 1; } else { - memcpy(pTagVal, &tv, sizeof(tv)); + (void)memcpy(pTagVal, &tv, sizeof(tv)); return true; } } @@ -1892,7 +1890,7 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, uint8_t *pData, if (nData) { code = tRealloc(&pColData->pData, pColData->nData + nData); if (code) goto _exit; - memcpy(pColData->pData + pColData->nData, pData, nData); + (void)memcpy(pColData->pData + pColData->nData, pData, nData); pColData->nData += nData; } } else { @@ -1900,7 +1898,7 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, uint8_t *pData, code = tRealloc(&pColData->pData, pColData->nData + tDataTypes[pColData->type].bytes); if (code) goto _exit; if (pData) { - memcpy(pColData->pData + pColData->nData, pData, TYPE_BYTES[pColData->type]); + (void)memcpy(pColData->pData + pColData->nData, pData, TYPE_BYTES[pColData->type]); } else { memset(pColData->pData + pColData->nData, 0, TYPE_BYTES[pColData->type]); } @@ -2594,7 +2592,7 @@ static FORCE_INLINE void tColDataGetValue4(SColData *pColData, int32_t iVal, SCo } value.pData = pColData->pData + pColData->aOffset[iVal]; } else { - memcpy(&value.val, pColData->pData + tDataTypes[pColData->type].bytes * iVal, tDataTypes[pColData->type].bytes); + (void)memcpy(&value.val, pColData->pData + tDataTypes[pColData->type].bytes * iVal, tDataTypes[pColData->type].bytes); } *pColVal = COL_VAL_VALUE(pColData->cid, value); } @@ -2692,7 +2690,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT1_SIZE(pColData->nVal)); + (void)memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT1_SIZE(pColData->nVal)); break; case (HAS_VALUE | HAS_NULL | HAS_NONE): pColData->pBitMap = xMalloc(arg, BIT2_SIZE(pColData->nVal)); @@ -2700,7 +2698,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT2_SIZE(pColData->nVal)); + (void)memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT2_SIZE(pColData->nVal)); break; default: pColData->pBitMap = NULL; @@ -2714,7 +2712,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - memcpy(pColData->aOffset, pColDataFrom->aOffset, pColData->nVal << 2); + (void)memcpy(pColData->aOffset, pColDataFrom->aOffset, pColData->nVal << 2); } else { pColData->aOffset = NULL; } @@ -2727,7 +2725,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall goto _exit; } - memcpy(pColData->pData, pColDataFrom->pData, pColData->nData); + (void)memcpy(pColData->pData, pColDataFrom->pData, pColData->nData); } else { pColData->pData = NULL; } @@ -3119,10 +3117,10 @@ static int32_t tColDataCopyRowCell(SColData *pFromColData, int32_t iFromRow, SCo pToColData->aOffset[iToRow + 1] = pToColData->aOffset[iToRow] + nData; } - memcpy(pToColData->pData + pToColData->aOffset[iToRow], pFromColData->pData + pFromColData->aOffset[iFromRow], + (void)memcpy(pToColData->pData + pToColData->aOffset[iToRow], pFromColData->pData + pFromColData->aOffset[iFromRow], nData); } else { - memcpy(&pToColData->pData[TYPE_BYTES[pToColData->type] * iToRow], + (void)memcpy(&pToColData->pData[TYPE_BYTES[pToColData->type] * iToRow], &pFromColData->pData[TYPE_BYTES[pToColData->type] * iFromRow], TYPE_BYTES[pToColData->type]); } return code; @@ -3361,7 +3359,7 @@ static void tColDataMergeImpl(SColData *pColData, int32_t iStart, int32_t iEnd / ASSERT(0); } else { if (iv != iStart) { - memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * iStart], + (void)memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * iStart], &pColData->pData[TYPE_BYTES[pColData->type] * iv], TYPE_BYTES[pColData->type]); } memmove(&pColData->pData[TYPE_BYTES[pColData->type] * (iStart + 1)], @@ -3579,11 +3577,11 @@ static int32_t tPutColDataVersion0(uint8_t *pBuf, SColData *pColData) { case (HAS_NULL | HAS_NONE): case (HAS_VALUE | HAS_NONE): case (HAS_VALUE | HAS_NULL): - if (pBuf) memcpy(pBuf + n, pColData->pBitMap, BIT1_SIZE(pColData->nVal)); + if (pBuf) (void)memcpy(pBuf + n, pColData->pBitMap, BIT1_SIZE(pColData->nVal)); n += BIT1_SIZE(pColData->nVal); break; case (HAS_VALUE | HAS_NULL | HAS_NONE): - if (pBuf) memcpy(pBuf + n, pColData->pBitMap, BIT2_SIZE(pColData->nVal)); + if (pBuf) (void)memcpy(pBuf + n, pColData->pBitMap, BIT2_SIZE(pColData->nVal)); n += BIT2_SIZE(pColData->nVal); break; default: @@ -3593,14 +3591,14 @@ static int32_t tPutColDataVersion0(uint8_t *pBuf, SColData *pColData) { // value if (pColData->flag & HAS_VALUE) { if (IS_VAR_DATA_TYPE(pColData->type)) { - if (pBuf) memcpy(pBuf + n, pColData->aOffset, pColData->nVal << 2); + if (pBuf) (void)memcpy(pBuf + n, pColData->aOffset, pColData->nVal << 2); n += (pColData->nVal << 2); n += tPutI32v(pBuf ? pBuf + n : NULL, pColData->nData); - if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData); + if (pBuf) (void)memcpy(pBuf + n, pColData->pData, pColData->nData); n += pColData->nData; } else { - if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData); + if (pBuf) (void)memcpy(pBuf + n, pColData->pData, pColData->nData); n += pColData->nData; } } @@ -4348,7 +4346,7 @@ int32_t tCompressData(void *input, // input ASSERT(outputSize >= extraSizeNeeded); if (info->cmprAlg == NO_COMPRESSION) { - memcpy(output, input, info->originalSize); + (void)memcpy(output, input, info->originalSize); info->compressedSize = info->originalSize; } else if (info->cmprAlg == ONE_STAGE_COMP || info->cmprAlg == TWO_STAGE_COMP) { SBuffer local; @@ -4385,7 +4383,7 @@ int32_t tCompressData(void *input, // input } else { DEFINE_VAR(info->cmprAlg) if ((l1 == L1_UNKNOWN && l2 == L2_UNKNOWN) || (l1 == L1_DISABLED && l2 == L2_DISABLED)) { - memcpy(output, input, info->originalSize); + (void)memcpy(output, input, info->originalSize); info->compressedSize = info->originalSize; return 0; } @@ -4431,7 +4429,7 @@ int32_t tDecompressData(void *input, // input if (info->cmprAlg == NO_COMPRESSION) { ASSERT(info->compressedSize == info->originalSize); - memcpy(output, input, info->compressedSize); + (void)memcpy(output, input, info->compressedSize); } else if (info->cmprAlg == ONE_STAGE_COMP || info->cmprAlg == TWO_STAGE_COMP) { SBuffer local; @@ -4468,7 +4466,7 @@ int32_t tDecompressData(void *input, // input } else { DEFINE_VAR(info->cmprAlg); if (l1 == L1_DISABLED && l2 == L2_DISABLED) { - memcpy(output, input, info->compressedSize); + (void)memcpy(output, input, info->compressedSize); return 0; } SBuffer local; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 38dd945b28..ccdf4a14d6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -2579,7 +2579,7 @@ typedef struct { static int32_t lastIterOpen(SFSLastIter *iter, STFileSet *pFileSet, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, SCacheRowsReader *pr, int64_t lastTs, int16_t *aCols, int nCols) { int32_t code = 0; - pr->pLDataIterArray = destroySttBlockReader(pr->pLDataIterArray, NULL); + destroySttBlockReader(pr->pLDataIterArray, NULL); pr->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); SMergeTreeConf conf = { diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 817468c6b6..46c3ba4785 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -203,7 +203,7 @@ int32_t tsdbReuseCacherowsReader(void* reader, void* pTableIdList, int32_t numOf pReader->pTableList = pTableIdList; pReader->numOfTables = numOfTables; pReader->lastTs = INT64_MIN; - pReader->pLDataIterArray = destroySttBlockReader(pReader->pLDataIterArray, NULL); + destroySttBlockReader(pReader->pLDataIterArray, NULL); pReader->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); return TSDB_CODE_SUCCESS; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index dc80fcabfb..048e407c76 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -45,7 +45,7 @@ typedef struct { bool moreThanCapcity; } SDataBlockToLoadInfo; -static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter); +static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInfo** pInfo); static int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity, STsdbReader* pReader); static TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader); @@ -123,7 +123,7 @@ static void tColRowGetPriamyKeyDeepCopy(SBlockData* pBlock, int32_t irow, int32_ pKey->pks[0].val = cv.value.val; } else { pKey->pks[0].nData = cv.value.nData; - /*void* p = */memcpy(pKey->pks[0].pData, cv.value.pData, cv.value.nData); + (void)(void)memcpy(pKey->pks[0].pData, cv.value.pData, cv.value.nData); } } @@ -155,9 +155,9 @@ static void tRowGetPrimaryKeyDeepCopy(SRow* pRow, SRowKey* pKey) { if (IS_VAR_DATA_TYPE(indices[i].type)) { tdata += tGetU32v(tdata, &pKey->pks[i].nData); - memcpy(pKey->pks[i].pData, tdata, pKey->pks[i].nData); + (void)memcpy(pKey->pks[i].pData, tdata, pKey->pks[i].nData); } else { - memcpy(&pKey->pks[i].val, data + indices[i].offset, tDataTypes[pKey->pks[i].type].bytes); + (void)memcpy(&pKey->pks[i].val, data + indices[i].offset, tDataTypes[pKey->pks[i].type].bytes); } } } @@ -304,7 +304,7 @@ static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bo while (1) { if (pReader->pFileReader != NULL) { - tsdbDataFileReaderClose(&pReader->pFileReader); + (void) tsdbDataFileReaderClose(&pReader->pFileReader); } pReader->status.pCurrentFileset = pIter->pFilesetList->data[pIter->index]; @@ -377,15 +377,20 @@ _err: bool shouldFreePkBuf(SBlockLoadSuppInfo* pSupp) { return (pSupp->numOfPks > 0) && IS_VAR_DATA_TYPE(pSupp->pk.type); } -void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool needFree) { +int32_t resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool needFree) { pIter->order = order; pIter->index = -1; pIter->numOfBlocks = 0; if (pIter->blockList == NULL) { pIter->blockList = taosArrayInit(4, sizeof(SFileDataBlockInfo)); + if (pIter->blockList == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } else { clearDataBlockIterator(pIter, needFree); } + + return TSDB_CODE_SUCCESS; } static void initReaderStatus(SReaderStatus* pStatus) { @@ -818,7 +823,7 @@ static void updateLastKeyInfo(SRowKey* pKey, SFileDataBlockInfo* pBlockInfo, SDa } else { uint8_t* p = asc ? pBlockInfo->lastPk.pData : pBlockInfo->firstPk.pData; pKey->pks[0].nData = asc ? varDataLen(pBlockInfo->lastPk.pData) : varDataLen(pBlockInfo->firstPk.pData); - memcpy(pKey->pks[0].pData, p, pKey->pks[0].nData); + (void)memcpy(pKey->pks[0].pData, p, pKey->pks[0].nData); } } @@ -836,7 +841,7 @@ static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int } if (pColVal->value.nData > 0) { // pData may be null, if nData is 0 - memcpy(varDataVal(pSup->buildBuf[colIndex]), pColVal->value.pData, pColVal->value.nData); + (void)memcpy(varDataVal(pSup->buildBuf[colIndex]), pColVal->value.pData, pColVal->value.nData); } colDataSetVal(pColInfoData, rowIndex, pSup->buildBuf[colIndex], false); @@ -848,19 +853,21 @@ static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int return TSDB_CODE_SUCCESS; } -static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) { +static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInfo** pInfo) { + *pInfo = NULL; + if (pBlockIter->blockList == NULL) { - return NULL; + return TSDB_CODE_SUCCESS; } size_t num = TARRAY_SIZE(pBlockIter->blockList); if (num == 0) { ASSERT(pBlockIter->numOfBlocks == num); - return NULL; + return TSDB_CODE_SUCCESS; } - SFileDataBlockInfo* pBlockInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index); - return pBlockInfo; + *pInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index); + return (*pInfo) != NULL? TSDB_CODE_SUCCESS:TSDB_CODE_FAILED; } static int doBinarySearchKey(const TSKEY* keyList, int num, int pos, TSKEY key, int order) { @@ -974,10 +981,10 @@ static int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData static void copyPrimaryTsCol(const SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData, int32_t dumpedRows, bool asc) { if (asc) { - memcpy(pColData->pData, &pBlockData->aTSKEY[pDumpInfo->rowIndex], dumpedRows * sizeof(int64_t)); + (void)memcpy(pColData->pData, &pBlockData->aTSKEY[pDumpInfo->rowIndex], dumpedRows * sizeof(int64_t)); } else { int32_t startIndex = pDumpInfo->rowIndex - dumpedRows + 1; - memcpy(pColData->pData, &pBlockData->aTSKEY[startIndex], dumpedRows * sizeof(int64_t)); + (void)memcpy(pColData->pData, &pBlockData->aTSKEY[startIndex], dumpedRows * sizeof(int64_t)); // todo: opt perf by extract the loop // reverse the array list @@ -1008,7 +1015,7 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo // ASSERT((((uint64_t)pColData->pData) & (0x8 - 1)) == 0); // 1. copy data in a batch model - memcpy(pColData->pData, p, dumpedRows * tDataTypes[pData->type].bytes); + (void)memcpy(pColData->pData, p, dumpedRows * tDataTypes[pData->type].bytes); // 2. reverse the array list in case of descending order scan data block if (!asc) { @@ -1123,7 +1130,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBlockData* pBlockData = &pStatus->fileBlockData; - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + SFileDataBlockInfo* pBlockInfo = NULL; SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; int32_t numOfOutputCols = pSupInfo->numOfCols; int32_t code = TSDB_CODE_SUCCESS; @@ -1131,6 +1138,11 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro bool asc = ASCENDING_TRAVERSE(pReader->info.order); int32_t step = asc ? 1 : -1; + code = getCurrentBlockInfo(pBlockIter, &pBlockInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + SColVal cv = {0}; SBrinRecord tmp; blockInfoToRecord(&tmp, pBlockInfo, pSupInfo); @@ -1314,9 +1326,11 @@ static FORCE_INLINE STSchema* getTableSchemaImpl(STsdbReader* pReader, uint64_t static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockIter, SBlockData* pBlockData, uint64_t uid) { - int32_t code = 0; - STSchema* pSchema = pReader->info.pSchema; - int64_t st = taosGetTimestampUs(); + int32_t code = 0; + STSchema* pSchema = pReader->info.pSchema; + int64_t st = taosGetTimestampUs(); + SFileDataBlockInfo* pBlockInfo = NULL; + SBlockLoadSuppInfo* pSup = &pReader->suppInfo; tBlockDataReset(pBlockData); @@ -1328,8 +1342,11 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI } } - SBlockLoadSuppInfo* pSup = &pReader->suppInfo; - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + code = getCurrentBlockInfo(pBlockIter, &pBlockInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBrinRecord tmp; @@ -2524,11 +2541,12 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { int32_t step = asc ? 1 : -1; double el = 0; SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); + SFileDataBlockInfo* pBlockInfo = NULL; SSttBlockReader* pSttBlockReader = pReader->status.fileIter.pSttBlockReader; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - if (pBlockInfo == NULL) { + code = getCurrentBlockInfo(&pReader->status.blockIter, &pBlockInfo); + if (code != TSDB_CODE_SUCCESS) { return 0; } @@ -2572,7 +2590,10 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { pDumpInfo->rowIndex += step; if (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0) { - pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); // NOTE: get the new block info + code = getCurrentBlockInfo(&pReader->status.blockIter, &pBlockInfo); // NOTE: get the new block info + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } // continue check for the next file block if the last ts in the current block // is overlapped with the next neighbor block @@ -2797,6 +2818,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr return loadDataFileTombDataForAll(pReader); } +// pTableIter can be NULL, no need to handle the return value static void resetTableListIndex(SReaderStatus* pStatus) { STableUidList* pList = &pStatus->uidList; @@ -2838,7 +2860,7 @@ static bool moveToNextTableForPreFileSetMem(SReaderStatus* pStatus) { return (pStatus->pProcMemTableIter != NULL); } -static void buildCleanBlockFromSttFiles(STsdbReader* pReader, STableBlockScanInfo* pScanInfo) { +static int32_t buildCleanBlockFromSttFiles(STsdbReader* pReader, STableBlockScanInfo* pScanInfo) { SReaderStatus* pStatus = &pReader->status; SSttBlockReader* pSttBlockReader = pStatus->fileIter.pSttBlockReader; SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; @@ -2846,7 +2868,10 @@ static void buildCleanBlockFromSttFiles(STsdbReader* pReader, STableBlockScanInf bool asc = ASCENDING_TRAVERSE(pReader->info.order); SDataBlockInfo* pInfo = &pResBlock->info; - blockDataEnsureCapacity(pResBlock, pScanInfo->numOfRowsInStt); + int32_t code = blockDataEnsureCapacity(pResBlock, pScanInfo->numOfRowsInStt); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pInfo->rows = pScanInfo->numOfRowsInStt; pInfo->id.uid = pScanInfo->uid; @@ -2891,8 +2916,8 @@ static void buildCleanBlockFromDataFiles(STsdbReader* pReader, STableBlockScanIn pInfo->pks[0].val = pBlockInfo->firstPk.val; pInfo->pks[1].val = pBlockInfo->lastPk.val; } else { - memcpy(pInfo->pks[0].pData, varDataVal(pBlockInfo->firstPk.pData), varDataLen(pBlockInfo->firstPk.pData)); - memcpy(pInfo->pks[1].pData, varDataVal(pBlockInfo->lastPk.pData), varDataLen(pBlockInfo->lastPk.pData)); + (void)memcpy(pInfo->pks[0].pData, varDataVal(pBlockInfo->firstPk.pData), varDataLen(pBlockInfo->firstPk.pData)); + (void)memcpy(pInfo->pks[1].pData, varDataVal(pBlockInfo->lastPk.pData), varDataLen(pBlockInfo->lastPk.pData)); pInfo->pks[0].nData = varDataLen(pBlockInfo->firstPk.pData); pInfo->pks[1].nData = varDataLen(pBlockInfo->lastPk.pData); @@ -2963,8 +2988,8 @@ static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) { // if only require the total rows, no need to load data from stt file if it is clean stt blocks if (pReader->info.execMode == READER_EXEC_ROWS && pScanInfo->cleanSttBlocks) { - buildCleanBlockFromSttFiles(pReader, pScanInfo); - return TSDB_CODE_SUCCESS; + code = buildCleanBlockFromSttFiles(pReader, pScanInfo); + return code; } int64_t st = taosGetTimestampUs(); @@ -3019,11 +3044,16 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; STableBlockScanInfo* pScanInfo = NULL; - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + SFileDataBlockInfo* pBlockInfo = NULL; SSttBlockReader* pSttBlockReader = pReader->status.fileIter.pSttBlockReader; bool asc = ASCENDING_TRAVERSE(pReader->info.order); int32_t code = TSDB_CODE_SUCCESS; + code = getCurrentBlockInfo(pBlockIter, &pBlockInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) { setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->lastKey, pReader->info.order); return code; @@ -3080,8 +3110,8 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // return the stt file block ASSERT(pReader->info.execMode == READER_EXEC_ROWS && pSttBlockReader->mergeTree.pIter == NULL); - buildCleanBlockFromSttFiles(pReader, pScanInfo); - return TSDB_CODE_SUCCESS; + code = buildCleanBlockFromSttFiles(pReader, pScanInfo); + return code; } } else { SBlockData* pBData = &pReader->status.fileBlockData; @@ -3218,11 +3248,12 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader, int64_t en // set the correct start position in case of the first/last file block, according to the query time window static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) { - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + SFileDataBlockInfo* pBlockInfo = NULL; SReaderStatus* pStatus = &pReader->status; SFileBlockDumpInfo* pDumpInfo = &pStatus->fBlockDumpInfo; - if (pBlockInfo) { + int32_t code = getCurrentBlockInfo(pBlockIter, &pBlockInfo); + if (code != TSDB_CODE_SUCCESS) { pDumpInfo->totalRows = pBlockInfo->numRow; pDumpInfo->rowIndex = ASCENDING_TRAVERSE(pReader->info.order) ? 0 : pBlockInfo->numRow - 1; } else { @@ -3255,7 +3286,7 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl code = initBlockIterator(pReader, pBlockIter, num.numOfBlocks, pTableList); } else { // no block data, only last block exists tBlockDataReset(&pReader->status.fileBlockData); - resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); + code = resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); resetTableListIndex(&pReader->status); } @@ -3365,7 +3396,10 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { } tBlockDataReset(pBlockData); - resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); + code = resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); + if (code != TSDB_CODE_SUCCESS) { + return code; + } resetTableListIndex(&pReader->status); ERetrieveType type = doReadDataFromSttFiles(pReader); @@ -3742,7 +3776,12 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc while (1) { CHECK_FILEBLOCK_STATE st; - SFileDataBlockInfo* pFileBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); + SFileDataBlockInfo* pFileBlockInfo = NULL; + int32_t code = getCurrentBlockInfo(&pReader->status.blockIter, &pFileBlockInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (pFileBlockInfo == NULL) { st = CHECK_FILEBLOCK_QUIT; break; @@ -4224,6 +4263,7 @@ uint64_t tsdbGetReaderMaxVersion2(STsdbReader* pReader) { return pReader->info.v static int32_t doOpenReaderImpl(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; + int32_t code = TSDB_CODE_SUCCESS; if (pReader->bFilesetDelimited) { getMemTableTimeRange(pReader, &pReader->status.memTableMaxKey, &pReader->status.memTableMinKey); @@ -4231,9 +4271,11 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { } initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); - resetDataBlockIterator(&pStatus->blockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); + code = resetDataBlockIterator(&pStatus->blockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - int32_t code = TSDB_CODE_SUCCESS; if (pStatus->fileIter.numOfFiles == 0) { pStatus->loadFromFile = false; } else { @@ -4278,13 +4320,17 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi SSDataBlock* pResBlock, void** ppReader, const char* idstr, SHashObj** pIgnoreTables) { STimeWindow window = pCond->twindows; SVnodeCfg* pConf = &(((SVnode*)pVnode)->config); + int32_t code = 0; int32_t capacity = pConf->tsdbCfg.maxRows; if (pResBlock != NULL) { - blockDataEnsureCapacity(pResBlock, capacity); + code = blockDataEnsureCapacity(pResBlock, capacity); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } } - int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, capacity, pResBlock, idstr); + code = tsdbReaderCreate(pVnode, pCond, ppReader, capacity, pResBlock, idstr); if (code != TSDB_CODE_SUCCESS) { goto _err; } @@ -4498,7 +4544,7 @@ static int32_t doSuspendCurrentReader(STsdbReader* pCurrentReader) { tsdbDataFileReaderClose(&pCurrentReader->pFileReader); SReadCostSummary* pCost = &pCurrentReader->cost; - pStatus->pLDataIterArray = destroySttBlockReader(pStatus->pLDataIterArray, &pCost->sttCost); + destroySttBlockReader(pStatus->pLDataIterArray, &pCost->sttCost); pStatus->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); } @@ -4922,9 +4968,9 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ } int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave, bool* hasNullSMA) { - SColumnDataAgg** pBlockSMA = &pDataBlock->pBlockAgg; - - int32_t code = 0; + SColumnDataAgg** pBlockSMA = &pDataBlock->pBlockAgg; + SFileDataBlockInfo* pBlockInfo = NULL; + int32_t code = 0; *allHave = false; *pBlockSMA = NULL; @@ -4937,7 +4983,11 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, return TSDB_CODE_SUCCESS; } - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); + code = getCurrentBlockInfo(&pReader->status.blockIter, &pBlockInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + SBlockLoadSuppInfo* pSup = &pReader->suppInfo; SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; @@ -5020,7 +5070,12 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; int32_t code = TSDB_CODE_SUCCESS; - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter); + SFileDataBlockInfo* pBlockInfo = NULL; + + code = getCurrentBlockInfo(&pStatus->blockIter, &pBlockInfo); + if (code != TSDB_CODE_SUCCESS) { + return NULL; + } if (pReader->code != TSDB_CODE_SUCCESS) { return NULL; @@ -5115,7 +5170,12 @@ int32_t tsdbReaderReset2(STsdbReader* pReader, SQueryTableDataCond* pCond) { int32_t numOfTables = tSimpleHashGetSize(pStatus->pTableMap); initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); - resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); + code = resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); + if (code != TSDB_CODE_SUCCESS) { + tsdbReleaseReader(pReader); + return code; + } + resetTableListIndex(&pReader->status); bool asc = ASCENDING_TRAVERSE(pReader->info.order); @@ -5218,8 +5278,13 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT while (true) { if (hasNext) { - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); - int32_t numOfRows = pBlockInfo->numRow; + SFileDataBlockInfo* pBlockInfo = NULL; + code = getCurrentBlockInfo(pBlockIter, &pBlockInfo); + if (code != TSDB_CODE_SUCCESS) { + break; + } + + int32_t numOfRows = pBlockInfo->numRow; pTableBlockInfo->totalRows += numOfRows; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index 6c76f8c372..48ef00baae 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -358,10 +358,10 @@ int32_t pkCompEx(SRowKey* p1, SRowKey* p2); int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc); void clearRowKey(SRowKey* pKey); -bool shouldFreePkBuf(SBlockLoadSuppInfo *pSupp); -void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool hasPk); -void clearDataBlockIterator(SDataBlockIter* pIter, bool needFree); -void cleanupDataBlockIterator(SDataBlockIter* pIter, bool hasPk); +bool shouldFreePkBuf(SBlockLoadSuppInfo* pSupp); +int32_t resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool hasPk); +void clearDataBlockIterator(SDataBlockIter* pIter, bool needFree); +void cleanupDataBlockIterator(SDataBlockIter* pIter, bool hasPk); typedef struct { SArray* pTombData; From 101d1936e9187b0adf09c4e819070edd5d2d5353 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 16 Jul 2024 09:39:58 +0800 Subject: [PATCH 18/34] feat:[TD-30975]process return value in schemaless --- source/client/src/clientSml.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 6e2f4ba367..1c705f1f6f 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1025,7 +1025,6 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, int32_t code = TSDB_CODE_SUCCESS; SCmdMsgInfo pCmdMsg = {0}; char *pSql = NULL; - char *msg = NULL; // put front for free pReq.numOfColumns = taosArrayGetSize(pColumns); @@ -1110,15 +1109,15 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, code = TSDB_CODE_OUT_OF_MEMORY; goto end; } - msg = taosMemoryMalloc(pCmdMsg.msgLen); - if (NULL == msg) { + pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); + if (NULL == pCmdMsg.pMsg) { code = TSDB_CODE_OUT_OF_MEMORY; goto end; } - pCmdMsg.pMsg = msg; if (tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) < 0){ code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pCmdMsg.pMsg); goto end; } @@ -1140,7 +1139,6 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, code = pRequest->code; end: - taosMemoryFree(msg); destroyRequest(pRequest); tFreeSMCreateStbReq(&pReq); return code; @@ -1620,12 +1618,12 @@ int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { info->taos = acquireTscObj(*(int64_t *)taos); if (info->taos == NULL) { code = TSDB_CODE_TSC_DISCONNECTED; - goto ERROR; + goto FAILED; } code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code); - goto ERROR; + goto FAILED; } } @@ -1636,7 +1634,7 @@ int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { if (info->pVgHash == NULL || info->childTables == NULL || info->superTables == NULL || info->tableUids == NULL) { uError("create SSmlHandle hash obj failed"); code = TSDB_CODE_OUT_OF_MEMORY; - goto ERROR; + goto FAILED; } taosHashSetFreeFp(info->superTables, smlDestroySTableMeta); taosHashSetFreeFp(info->childTables, smlDestroyTableInfo); @@ -1644,7 +1642,7 @@ int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { info->id = smlGenId(); code = smlInitHandle(&info->pQuery); if (code != TSDB_CODE_SUCCESS){ - goto ERROR; + goto FAILED; } info->dataFormat = true; @@ -1655,13 +1653,13 @@ int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { if (info->tagJsonArray == NULL || info->valueJsonArray == NULL || info->preLineTagKV == NULL) { uError("SML:0x%" PRIx64 " failed to allocate memory", info->id); code = TSDB_CODE_OUT_OF_MEMORY; - goto ERROR; + goto FAILED; } *handle = info; return code; -ERROR: +FAILED: smlDestroyInfo(info); return code; } From e76e06cf2eb3fd085bfffc6a3715f806c79a33c3 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 16 Jul 2024 11:50:40 +0800 Subject: [PATCH 19/34] feat:[TD-30975]process return value in schemaless --- source/client/inc/clientSml.h | 6 +++--- source/client/src/clientSml.c | 4 ++-- source/client/src/clientSmlJson.c | 3 ++- utils/test/c/sml_test.c | 36 ++++++++++++++++++++----------- 4 files changed, 31 insertions(+), 18 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index d369e0597c..e901f86b41 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -246,9 +246,9 @@ void smlBuildTsKv(SSmlKv *kv, int64_t ts); int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv); int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs); -static inline int32_t smlDoubleToInt64OverFlow(double num) { - if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return 0; - return -1; +static inline bool smlDoubleToInt64OverFlow(double num) { + if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true; + return false; } static inline void smlStrReplace(char* src, int32_t len){ diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 1c705f1f6f..02fbe7aae5 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -728,7 +728,7 @@ bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg) { kvVal->type = TSDB_DATA_TYPE_FLOAT; kvVal->f = (float)result; } else if ((left == 1 && *endptr == 'i') || (left == 3 && strncasecmp(endptr, "i64", left) == 0)) { - if (smlDoubleToInt64OverFlow(result) == 0) { + if (smlDoubleToInt64OverFlow(result)) { errno = 0; int64_t tmp = taosStr2Int64(pVal, &endptr, 10); if (errno == ERANGE) { @@ -1467,7 +1467,7 @@ end: taosHashCancelIterate(info->superTables, tmp); taosHashCleanup(hashTmp); taosMemoryFreeClear(pTableMeta); - code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); + (void)catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); // ignore refresh meta code if there is an error uError("SML:0x%" PRIx64 " smlModifyDBSchemas end failed:%d:%s, format:%d, needModifySchema:%d", info->id, code, tstrerror(code), info->dataFormat, info->needModifySchema); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 519e55d0d1..b0348b4f91 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -739,7 +739,8 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo // notice!!! put ts back to tag to ensure get meta->precision int64_t ts = smlParseTSFromJSON(info, tsJson); if (unlikely(ts < 0)) { - uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); + char* tmp = cJSON_PrintUnformatted(tsJson); + uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %s %" PRId64, info->id, info->msgBuf.buf,tmp, ts); return TSDB_CODE_INVALID_TIMESTAMP; } SSmlKv kvTs = {0}; diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 2338674ae3..a3830d2518 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -121,7 +121,8 @@ int smlProcess_json1_Test() { char *sql1[1] = {0}; for (int i = 0; i < 1; i++) { sql1[i] = taosMemoryCalloc(1, 1024); - strncpy(sql1[i], sql[i], 1023); + ASSERT(sql1[i] != NULL); + (void)strncpy(sql1[i], sql[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, @@ -149,7 +150,8 @@ int smlProcess_json1_Test() { char *sql3[1] = {0}; for (int i = 0; i < 1; i++) { sql3[i] = taosMemoryCalloc(1, 1024); - strncpy(sql3[i], sql2[i], 1023); + ASSERT(sql3[i] != NULL); + (void)strncpy(sql3[i], sql2[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL, @@ -176,7 +178,8 @@ int smlProcess_json1_Test() { char *sql5[1] = {0}; for (int i = 0; i < 1; i++) { sql5[i] = taosMemoryCalloc(1, 1024); - strncpy(sql5[i], sql4[i], 1023); + ASSERT(sql5[i] != NULL); + (void)strncpy(sql5[i], sql4[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql5, sizeof(sql5) / sizeof(sql5[0]), TSDB_SML_JSON_PROTOCOL, @@ -215,7 +218,8 @@ int smlProcess_json2_Test() { char *sql1[1] = {0}; for (int i = 0; i < 1; i++) { sql1[i] = taosMemoryCalloc(1, 1024); - strncpy(sql1[i], sql[i], 1023); + ASSERT(sql1[i] != NULL); + (void)strncpy(sql1[i], sql[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, @@ -249,7 +253,8 @@ int smlProcess_json3_Test() { char *sql1[1] = {0}; for (int i = 0; i < 1; i++) { sql1[i] = taosMemoryCalloc(1, 1024); - strncpy(sql1[i], sql[i], 1023); + ASSERT(sql1[i] != NULL); + (void)strncpy(sql1[i], sql[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, @@ -286,7 +291,8 @@ int smlProcess_json_tag_not_same_Test() { char *sql1[1] = {0}; for (int i = 0; i < 1; i++) { sql1[i] = taosMemoryCalloc(1, 1024); - strncpy(sql1[i], sql[i], 1023); + ASSERT(sql1[i] != NULL); + (void)strncpy(sql1[i], sql[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, @@ -312,7 +318,8 @@ int smlProcess_json_tag_not_same_Test() { char *sql3[1] = {0}; for (int i = 0; i < 1; i++) { sql3[i] = taosMemoryCalloc(1, 1024); - strncpy(sql3[i], sql2[i], 1023); + ASSERT(sql3[i] != NULL); + (void)strncpy(sql3[i], sql2[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL, @@ -338,7 +345,8 @@ int smlProcess_json_tag_not_same_Test() { char *sql5[1] = {0}; for (int i = 0; i < 1; i++) { sql5[i] = taosMemoryCalloc(1, 1024); - strncpy(sql5[i], sql4[i], 1023); + ASSERT(sql5[i] != NULL); + (void)strncpy(sql5[i], sql4[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql5, sizeof(sql5) / sizeof(sql5[0]), TSDB_SML_JSON_PROTOCOL, @@ -1147,7 +1155,8 @@ int sml_19221_Test() { taos_free_result(pRes); char *tmp = (char *)taosMemoryCalloc(1024, 1); - memcpy(tmp, sql[0], strlen(sql[0])); + ASSERT(tmp != NULL); + (void)memcpy(tmp, sql[0], strlen(sql[0])); *(char *)(tmp + 44) = 0; int32_t totalRows = 0; pRes = taos_schemaless_insert_raw(taos, tmp, strlen(sql[0]), &totalRows, TSDB_SML_LINE_PROTOCOL, @@ -1218,7 +1227,8 @@ int sml_ts3116_Test() { taos_free_result(pRes); int32_t totalRows = 0; char *tmp = (char *)taosMemoryCalloc(1024, 1); - memcpy(tmp, sql, strlen(sql)); + ASSERT(tmp != NULL); + (void)memcpy(tmp, sql, strlen(sql)); totalRows = 0; pRes = taos_schemaless_insert_raw(taos, tmp, strlen(tmp), &totalRows, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); @@ -1235,7 +1245,8 @@ int sml_ts3116_Test() { taos_free_result(pRes); tmp = (char *)taosMemoryCalloc(1024, 1); - memcpy(tmp, sql1, strlen(sql1)); + ASSERT(tmp != NULL); + (void)memcpy(tmp, sql1, strlen(sql1)); totalRows = 0; pRes = taos_schemaless_insert_raw(taos, tmp, strlen(tmp), &totalRows, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); @@ -2049,7 +2060,8 @@ int sml_td29373_Test() { char *sql3[1] = {0}; for (int i = 0; i < 1; i++) { sql3[i] = taosMemoryCalloc(1, 1024); - strncpy(sql3[i], sql2[i], 1023); + ASSERT(sql3[i] != NULL); + (void)strncpy(sql3[i], sql2[i], 1023); } pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL, From 691c1dfa7dbd175a0a03659517d22de1dd8587b1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 16 Jul 2024 14:45:09 +0800 Subject: [PATCH 20/34] fix(stream): check dst stable schema before generating auto create table. --- include/libs/stream/tstream.h | 2 + source/dnode/vnode/src/tq/tqSink.c | 78 ++++++++++++++++++++--------- source/libs/stream/src/streamTask.c | 1 + 3 files changed, 57 insertions(+), 24 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 5ba0ce454c..fd9b5667d7 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -237,6 +237,8 @@ typedef struct { int64_t stbUid; char stbFullName[TSDB_TABLE_FNAME_LEN]; SSchemaWrapper* pSchemaWrapper; + SSchemaWrapper* pTagSchema; + bool autoCreateCtb; void* vnode; // not available to encoder and decoder FTbSink* tbSinkFunc; STSchema* pTSchema; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 9b1e8075da..6494eb1bd1 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -725,24 +725,33 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat if (metaGetTableEntryByName(&mr, dstTableName) < 0) { metaReaderClear(&mr); - tqDebug("s-task:%s stream write into table:%s, table auto created", id, dstTableName); + if (pTask->outputInfo.tbSink.autoCreateCtb) { + tqDebug("s-task:%s stream write into table:%s, table auto created", id, dstTableName); - SArray* pTagArray = taosArrayInit(pTSchema->numOfCols + 1, sizeof(STagVal)); + SArray* pTagArray = taosArrayInit(pTSchema->numOfCols + 1, sizeof(STagVal)); - pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE; - pTableData->pCreateTbReq = - buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock, pTagArray, - pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1); - taosArrayDestroy(pTagArray); + pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE; + pTableData->pCreateTbReq = + buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock, pTagArray, + pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1); + taosArrayDestroy(pTagArray); - if (pTableData->pCreateTbReq == NULL) { - tqError("s-task:%s failed to build auto create table req, code:%s", id, tstrerror(terrno)); - taosMemoryFree(pTableSinkInfo); - return terrno; + if (pTableData->pCreateTbReq == NULL) { + tqError("s-task:%s failed to build auto create dst-table req:%s, code:%s", id, dstTableName, + tstrerror(terrno)); + taosMemoryFree(pTableSinkInfo); + return terrno; + } + + pTableSinkInfo->uid = 0; + doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id); + } else { + metaReaderClear(&mr); + + tqError("s-task:%s vgId:%d dst-table:%s not auto-created, and not create in tsdb, discard data", id, + vgId, dstTableName); + return TSDB_CODE_TDB_TABLE_NOT_EXIST; } - - pTableSinkInfo->uid = 0; - doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id); } else { bool isValid = isValidDstChildTable(&mr, vgId, dstTableName, suid); if (!isValid) { @@ -788,16 +797,34 @@ int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_ } void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { - const SArray* pBlocks = (const SArray*)data; - SVnode* pVnode = (SVnode*)vnode; - int64_t suid = pTask->outputInfo.tbSink.stbUid; - char* stbFullName = pTask->outputInfo.tbSink.stbFullName; - STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema; - int32_t vgId = TD_VID(pVnode); - int32_t numOfBlocks = taosArrayGetSize(pBlocks); - int32_t code = TSDB_CODE_SUCCESS; - const char* id = pTask->id.idStr; - int64_t earlyTs = tsdbGetEarliestTs(pVnode->pTsdb); + const SArray* pBlocks = (const SArray*)data; + SVnode* pVnode = (SVnode*)vnode; + int64_t suid = pTask->outputInfo.tbSink.stbUid; + char* stbFullName = pTask->outputInfo.tbSink.stbFullName; + STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema; + int32_t vgId = TD_VID(pVnode); + int32_t numOfBlocks = taosArrayGetSize(pBlocks); + int32_t code = TSDB_CODE_SUCCESS; + const char* id = pTask->id.idStr; + int64_t earlyTs = tsdbGetEarliestTs(pVnode->pTsdb); + STaskOutputInfo* pOutputInfo = &pTask->outputInfo; + + if (pTask->outputInfo.tbSink.pTagSchema == NULL) { + SMetaReader mer1 = {0}; + metaReaderDoInit(&mer1, pVnode->pMeta, META_READER_LOCK); + + code = metaReaderGetTableEntryByUid(&mer1, pOutputInfo->tbSink.stbUid); + pOutputInfo->tbSink.pTagSchema = tCloneSSchemaWrapper(&mer1.me.stbEntry.schemaTag); + metaReaderClear(&mer1); + + SSchemaWrapper* pTagSchema = pOutputInfo->tbSink.pTagSchema; + SSchema* pCol1 = &pTagSchema->pSchema[0]; + if (pTagSchema->nCols == 1 && pCol1->type == TSDB_DATA_TYPE_UBIGINT && strcmp(pCol1->name, "group_id") == 0) { + pOutputInfo->tbSink.autoCreateCtb = true; + } else { + pOutputInfo->tbSink.autoCreateCtb = false; + } + } bool onlySubmitData = hasOnlySubmitData(pBlocks, numOfBlocks); if (!onlySubmitData) { @@ -829,6 +856,8 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = TD_REQ_FROM_APP}; code = setDstTableDataUid(pVnode, pTask, pDataBlock, stbFullName, &tbData); if (code != TSDB_CODE_SUCCESS) { + tqError("vgId:%d s-task:%s dst-table not exist, stb:%" PRId64 " discard stream results", vgId, id, + stbFullName); continue; } @@ -882,6 +911,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { if (index == NULL) { // no data yet, append it code = setDstTableDataUid(pVnode, pTask, pDataBlock, stbFullName, &tbData); if (code != TSDB_CODE_SUCCESS) { + tqError("vgId:%d dst-table gid:%" PRId64 " not exist, discard stream results", vgId, groupId); continue; } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 4cbe0cb136..a40359150d 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -263,6 +263,7 @@ void tFreeStreamTask(SStreamTask* pTask) { tDeleteSchemaWrapper(pTask->outputInfo.tbSink.pSchemaWrapper); taosMemoryFree(pTask->outputInfo.tbSink.pTSchema); tSimpleHashCleanup(pTask->outputInfo.tbSink.pTblInfo); + tDeleteSchemaWrapper(pTask->outputInfo.tbSink.pTagSchema); } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { taosArrayDestroy(pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos); } From d2cb53ab3d789130c5766a64e7700b2e6e853d57 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 16 Jul 2024 15:49:08 +0800 Subject: [PATCH 21/34] refactor: remove the invalid return value. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 100 ++++++++++++++------- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 90 +++++++++++++------ source/dnode/vnode/src/tsdb/tsdbReadUtil.h | 31 ++++--- 3 files changed, 145 insertions(+), 76 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 048e407c76..d1702a94fe 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -679,7 +679,12 @@ static int32_t loadFileBlockBrinInfo(STsdbReader* pReader, SArray* pIndexList, S cleanupInfoForNextFileset(pReader->status.pTableMap); initBrinRecordIter(&iter, pReader->pFileReader, pIndexList); - while (((pRecord = getNextBrinRecord(&iter)) != NULL)) { + while (1) { + int32_t code = getNextBrinRecord(&iter, &pRecord); + if (code != TSDB_CODE_SUCCESS) { + break; + } + if (pRecord->suid > pReader->info.suid) { break; } @@ -707,7 +712,11 @@ static int32_t loadFileBlockBrinInfo(STsdbReader* pReader, SArray* pIndexList, S ASSERT(pRecord->suid == pReader->info.suid && uid == pRecord->uid); - STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); + STableBlockScanInfo* pScanInfo = NULL; + code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } // here we should find the first timestamp that is greater than the lastProcKey // the window is an open interval NOW. @@ -1409,8 +1418,10 @@ static bool getNeighborBlockOfTable(SDataBlockIter* pBlockIter, SFileDataBlockIn static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t index, int32_t step) { + int32_t code = TSDB_CODE_SUCCESS; if (index < 0 || index >= pBlockIter->numOfBlocks) { - return -1; + code = -1; + return code; } SFileDataBlockInfo fblock = *(SFileDataBlockInfo*)taosArrayGet(pBlockIter->blockList, index); @@ -1421,8 +1432,12 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte for (int32_t i = index - 1; i >= pBlockIter->index; --i) { SFileDataBlockInfo* pBlockInfo = taosArrayGet(pBlockIter->blockList, i); - STableBlockScanInfo* pBlockScanInfo = - getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); + STableBlockScanInfo* pBlockScanInfo = NULL; + code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx); pTableDataBlockIdx->globalIndex = i + 1; @@ -1432,8 +1447,12 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte for (int32_t i = index + 1; i <= pBlockIter->index; ++i) { SFileDataBlockInfo* pBlockInfo = taosArrayGet(pBlockIter->blockList, i); - STableBlockScanInfo* pBlockScanInfo = - getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); + STableBlockScanInfo* pBlockScanInfo = NULL; + code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx); pTableDataBlockIdx->globalIndex = i - 1; @@ -1442,7 +1461,12 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte } taosArraySet(pBlockIter->blockList, pBlockIter->index, &fblock); - STableBlockScanInfo* pBlockScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, fblock.uid, pReader->idStr); + STableBlockScanInfo* pBlockScanInfo = NULL; + code = getTableBlockScanInfo(pReader->status.pTableMap, fblock.uid, &pBlockScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, fblock.tbBlockIdx); pTableDataBlockIdx->globalIndex = pBlockIter->index; } @@ -2555,9 +2579,9 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { return code; } - STableBlockScanInfo* pBlockScanInfo = - getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); - if (pBlockScanInfo == NULL) { + STableBlockScanInfo* pBlockScanInfo = NULL; + code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { goto _end; } @@ -2896,6 +2920,7 @@ static int32_t buildCleanBlockFromSttFiles(STsdbReader* pReader, STableBlockScan tsdbDebug("%p uid:%" PRId64 " return clean stt block as one, brange:%" PRId64 "-%" PRId64 " rows:%" PRId64 " %s", pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, pReader->idStr); + return code; } static void buildCleanBlockFromDataFiles(STsdbReader* pReader, STableBlockScanInfo* pScanInfo, @@ -3063,9 +3088,9 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { return pReader->code; } - pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); - if (pScanInfo == NULL) { - return terrno; + code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; } if (pScanInfo->sttKeyInfo.status == STT_FILE_READER_UNINIT) { @@ -4206,6 +4231,7 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e // TODO refactor: with createDataBlockScanInfo int32_t tsdbSetTableList2(STsdbReader* pReader, const void* pTableList, int32_t num) { + int32_t code = TSDB_CODE_SUCCESS; int32_t size = tSimpleHashGetSize(pReader->status.pTableMap); STableBlockScanInfo** p = NULL; @@ -4216,7 +4242,7 @@ int32_t tsdbSetTableList2(STsdbReader* pReader, const void* pTableList, int32_t } if (size < num) { - int32_t code = ensureBlockScanInfoBuf(&pReader->blockInfoBuf, num); + code = ensureBlockScanInfoBuf(&pReader->blockInfoBuf, num); if (code) { return code; } @@ -4237,8 +4263,16 @@ int32_t tsdbSetTableList2(STsdbReader* pReader, const void* pTableList, int32_t for (int32_t i = 0; i < num; ++i) { pUidList->tableUidList[i] = pList[i].uid; - STableBlockScanInfo* pInfo = getPosInBlockInfoBuf(&pReader->blockInfoBuf, i); - initTableBlockScanInfo(pInfo, pList[i].uid, pReader->status.pTableMap, pReader); + STableBlockScanInfo* pInfo = NULL; + code = getPosInBlockInfoBuf(&pReader->blockInfoBuf, i, &pInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = initTableBlockScanInfo(pInfo, pList[i].uid, pReader->status.pTableMap, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } return TDB_CODE_SUCCESS; @@ -4412,11 +4446,10 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi } STsdbReader* p = (pReader->innerReader[0] != NULL) ? pReader->innerReader[0] : pReader; - pReader->status.pTableMap = - createDataBlockScanInfo(p, &pReader->blockInfoBuf, pTableList, &pReader->status.uidList, numOfTables); - if (pReader->status.pTableMap == NULL) { + + code = createDataBlockScanInfo(p, &pReader->blockInfoBuf, pTableList, &pReader->status.uidList, numOfTables, &pReader->status.pTableMap); + if (code != TSDB_CODE_SUCCESS) { *ppReader = NULL; - code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -5067,40 +5100,40 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, return code; } -static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { +static int32_t doRetrieveDataBlock(STsdbReader* pReader, SSDataBlock** pBlock) { SReaderStatus* pStatus = &pReader->status; int32_t code = TSDB_CODE_SUCCESS; SFileDataBlockInfo* pBlockInfo = NULL; + *pBlock = NULL; code = getCurrentBlockInfo(&pStatus->blockIter, &pBlockInfo); if (code != TSDB_CODE_SUCCESS) { - return NULL; + return code; } if (pReader->code != TSDB_CODE_SUCCESS) { - return NULL; + return pReader->code; } - STableBlockScanInfo* pBlockScanInfo = getTableBlockScanInfo(pStatus->pTableMap, pBlockInfo->uid, pReader->idStr); - if (pBlockScanInfo == NULL) { - return NULL; + STableBlockScanInfo* pBlockScanInfo = NULL; + code = getTableBlockScanInfo(pStatus->pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; } code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData, pBlockScanInfo->uid); if (code != TSDB_CODE_SUCCESS) { tBlockDataReset(&pStatus->fileBlockData); - terrno = code; - return NULL; + return code; } code = copyBlockDataToSDataBlock(pReader, &pBlockScanInfo->lastProcKey); if (code != TSDB_CODE_SUCCESS) { tBlockDataReset(&pStatus->fileBlockData); - terrno = code; - return NULL; } - return pReader->resBlockInfo.pResBlock; + *pBlock = pReader->resBlockInfo.pResBlock; + return code; } SSDataBlock* tsdbRetrieveDataBlock2(STsdbReader* pReader, SArray* pIdList) { @@ -5121,7 +5154,8 @@ SSDataBlock* tsdbRetrieveDataBlock2(STsdbReader* pReader, SArray* pIdList) { return pTReader->resBlockInfo.pResBlock; } - SSDataBlock* ret = doRetrieveDataBlock(pTReader); + SSDataBlock* ret = NULL; + doRetrieveDataBlock(pTReader, &ret); qTrace("tsdb/read-retrieve: %p, unlock read mutex", pReader); tsdbReleaseReader(pReader); diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index e84800691f..b1cc545eca 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -112,22 +112,28 @@ void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf) { taosArrayDestroy(pBuf->pData); } -void* getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index) { +int32_t getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index, STableBlockScanInfo** pInfo) { + *pInfo = NULL; + int32_t bucketIndex = index / pBuf->numPerBucket; char** pBucket = taosArrayGet(pBuf->pData, bucketIndex); - return (*pBucket) + (index % pBuf->numPerBucket) * sizeof(STableBlockScanInfo); -} - -STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id) { - STableBlockScanInfo** p = tSimpleHashGet(pTableMap, &uid, sizeof(uid)); - if (p == NULL || *p == NULL) { - terrno = TSDB_CODE_INVALID_PARA; - int32_t size = tSimpleHashGetSize(pTableMap); - tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id); - return NULL; + if (pBucket == NULL) { + return TSDB_CODE_FAILED; } - return *p; + *pInfo = (STableBlockScanInfo*)((*pBucket) + (index % pBuf->numPerBucket) * sizeof(STableBlockScanInfo)); + return TSDB_CODE_SUCCESS; +} + +int32_t getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, STableBlockScanInfo** pInfo, const char* id) { + *pInfo = tSimpleHashGet(pTableMap, &uid, sizeof(uid)); + if (pInfo == NULL || *pInfo == NULL) { + int32_t size = tSimpleHashGetSize(pTableMap); + tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id); + return TSDB_CODE_INVALID_PARA; + } + + return TSDB_CODE_SUCCESS; } int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc) { @@ -245,20 +251,27 @@ int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSH initLastProcKey(pScanInfo, pReader); pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT; - tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES); + int32_t code = tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + tsdbTrace("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pReader, pScanInfo->uid, pScanInfo->lastProcKey.ts, pReader->idStr); - return TSDB_CODE_SUCCESS; + return code; } // NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model -SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, - STableUidList* pUidList, int32_t numOfTables) { +int32_t createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, + STableUidList* pUidList, int32_t numOfTables, SSHashObj** pHashObj) { + int32_t code = 0; + *pHashObj = NULL; + // allocate buffer in order to load data blocks from file // todo use simple hash instead, optimize the memory consumption SSHashObj* pTableMap = tSimpleHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); if (pTableMap == NULL) { - return NULL; + return terrno; } int64_t st = taosGetTimestampUs(); @@ -267,7 +280,7 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t)); if (pUidList->tableUidList == NULL) { tSimpleHashCleanup(pTableMap); - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } pUidList->currentIndex = 0; @@ -275,8 +288,16 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf for (int32_t j = 0; j < numOfTables; ++j) { pUidList->tableUidList[j] = idList[j].uid; - STableBlockScanInfo* pScanInfo = getPosInBlockInfoBuf(pBuf, j); - initTableBlockScanInfo(pScanInfo, idList[j].uid, pTableMap, pTsdbReader); + STableBlockScanInfo* pScanInfo = NULL; + code = getPosInBlockInfoBuf(pBuf, j, &pScanInfo); + if (code != TSDB_CODE_SUCCESS) { + break; + } + + code = initTableBlockScanInfo(pScanInfo, idList[j].uid, pTableMap, pTsdbReader); + if (code != TSDB_CODE_SUCCESS) { + break; + } } taosSort(pUidList->tableUidList, numOfTables, sizeof(uint64_t), uidComparFunc); @@ -286,7 +307,8 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf (sizeof(STableBlockScanInfo) * numOfTables) / 1024.0, pTsdbReader->cost.createScanInfoList, pTsdbReader->idStr); - return pTableMap; + *pHashObj = pTableMap; + return code; } void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) { @@ -391,11 +413,13 @@ void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray pIter->pBrinBlockList = pList; } -SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter) { +int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord) { + *pRecord = NULL; + if (pIter->blockIndex == -1 || (pIter->recordIndex + 1) >= pIter->block.numOfRecords) { pIter->blockIndex += 1; if (pIter->blockIndex >= taosArrayGetSize(pIter->pBrinBlockList)) { - return NULL; + return TSDB_CODE_FAILED; } pIter->pCurrentBlk = taosArrayGet(pIter->pBrinBlockList, pIter->blockIndex); @@ -404,7 +428,7 @@ SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter) { int32_t code = tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block); if (code != TSDB_CODE_SUCCESS) { tsdbError("failed to read brinBlock from file, code:%s", tstrerror(code)); - return NULL; + return TSDB_CODE_FAILED; } pIter->recordIndex = -1; @@ -412,7 +436,9 @@ SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter) { pIter->recordIndex += 1; tBrinBlockGet(&pIter->block, pIter->recordIndex, &pIter->record); - return &pIter->record; + *pRecord = &pIter->record; + + return TSDB_CODE_SUCCESS; } void clearBrinBlockIter(SBrinRecordIter* pIter) { tBrinBlockDestroy(&pIter->block); } @@ -670,7 +696,13 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_ STombRecord record = {0}; uint64_t uid = pReader->status.uidList.tableUidList[*j]; - STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); + STableBlockScanInfo* pScanInfo = NULL; + + code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (pScanInfo->pFileDelData == NULL) { pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData)); } @@ -702,7 +734,11 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_ } uid = pReader->status.uidList.tableUidList[*j]; - pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); + code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (pScanInfo->pFileDelData == NULL) { pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData)); } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index 48ef00baae..ba75cb77a8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -315,25 +315,24 @@ typedef struct SBrinRecordIter { } SBrinRecordIter; int32_t uidComparFunc(const void* p1, const void* p2); +int32_t getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, STableBlockScanInfo** pInfo, const char* id); -STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id); - -SSHashObj* createDataBlockScanInfo(STsdbReader* pReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, - STableUidList* pUidList, int32_t numOfTables); -int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSHashObj* pTableMap, - STsdbReader* pReader); -void clearBlockScanInfo(STableBlockScanInfo* p); -void destroyAllBlockScanInfo(SSHashObj* pTableMap); -void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step); -void cleanupInfoForNextFileset(SSHashObj* pTableMap); -int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables); -void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf); -void* getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index); +int32_t createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, + STableUidList* pUidList, int32_t numOfTables, SSHashObj** pHashObj); +int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSHashObj* pTableMap, + STsdbReader* pReader); +void clearBlockScanInfo(STableBlockScanInfo* p); +void destroyAllBlockScanInfo(SSHashObj* pTableMap); +void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step); +void cleanupInfoForNextFileset(SSHashObj* pTableMap); +int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables); +void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf); +int32_t getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index, STableBlockScanInfo** pRes); // brin records iterator -void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList); -SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter); -void clearBrinBlockIter(SBrinRecordIter* pIter); +void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList); +int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord); +void clearBrinBlockIter(SBrinRecordIter* pIter); // initialize block iterator API int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList); From 19ba5af5e78ebd96c2889643ffc31aa5f997de0b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 16 Jul 2024 15:59:16 +0800 Subject: [PATCH 22/34] refactor: remove the invalid return value. --- source/dnode/vnode/src/inc/tsdb.h | 6 ++--- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 30 +++++++++++---------- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 9 ++++--- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 241a47aca9..943ba099f6 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -909,9 +909,9 @@ void tMergeTreeUnpinSttBlock(SMergeTree *pMTree); bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree); void tMergeTreeClose(SMergeTree *pMTree); -SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols); -void * destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); -void destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost); +int32_t tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, SSttBlockLoadInfo **pInfo); +void destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); +void destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost); // tsdbCache ============================================================================================== typedef enum { diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index fbda31f731..2c03603d73 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -23,11 +23,12 @@ static void tLDataIterClose2(SLDataIter *pIter); // SLDataIter ================================================= -SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols) { +int32_t tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, SSttBlockLoadInfo **pInfo) { + *pInfo = NULL; + SSttBlockLoadInfo *pLoadInfo = taosMemoryCalloc(1, sizeof(SSttBlockLoadInfo)); if (pLoadInfo == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } pLoadInfo->blockData[0].sttBlockIndex = -1; @@ -37,26 +38,29 @@ SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t code = tBlockDataCreate(&pLoadInfo->blockData[0].data); if (code) { - terrno = code; + taosMemoryFreeClear(pLoadInfo); + return code; } code = tBlockDataCreate(&pLoadInfo->blockData[1].data); if (code) { - terrno = code; + taosMemoryFreeClear(pLoadInfo); + return code; } pLoadInfo->aSttBlk = taosArrayInit(4, sizeof(SSttBlk)); if (pLoadInfo->aSttBlk == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFreeClear(pLoadInfo); - return NULL; + return code; } pLoadInfo->pSchema = pSchema; pLoadInfo->colIds = colList; pLoadInfo->numOfCols = numOfCols; - return pLoadInfo; + *pInfo = pLoadInfo; + return code; } static void freeItem(void* pValue) { @@ -66,9 +70,9 @@ static void freeItem(void* pValue) { } } -void *destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { +void destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { if (pLoadInfo == NULL) { - return NULL; + return; } pLoadInfo->currentLoadBlockIndex = 1; @@ -94,7 +98,6 @@ void *destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { taosArrayDestroy(pLoadInfo->aSttBlk); taosMemoryFree(pLoadInfo); - return NULL; } void destroyLDataIter(SLDataIter *pIter) { @@ -913,9 +916,8 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoF } if (pLoadInfo == NULL) { - pLoadInfo = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols); - if (pLoadInfo == NULL) { - code = terrno; + code = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols, &pLoadInfo); + if (code != TSDB_CODE_SUCCESS) { goto _end; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index b1cc545eca..4bca7f73f8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -996,6 +996,7 @@ int32_t adjustSttDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet) int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArray, STsdb* pTsdb, SMergeTreeConf* pConf, const char* pstr) { int32_t numOfRows = 0; + int32_t code = 0; // no data exists, go to end int32_t numOfLevels = pFileSet->lvlArr->size; @@ -1020,7 +1021,7 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra conf.file[0] = *pSttLevel->fobjArr->data[i]->f; const char* pName = pSttLevel->fobjArr->data[i]->fname; - int32_t code = tsdbSttFileReaderOpen(pName, &conf, &pIter->pReader); + code = tsdbSttFileReaderOpen(pName, &conf, &pIter->pReader); if (code != TSDB_CODE_SUCCESS) { tsdbError("open stt file reader error. file:%s, code %s, %s", pName, tstrerror(code), pstr); continue; @@ -1028,8 +1029,8 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra } if (pIter->pBlockLoadInfo == NULL) { - pIter->pBlockLoadInfo = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols); - if (pIter->pBlockLoadInfo == NULL) { + code = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols, &pIter->pBlockLoadInfo); + if (code != TSDB_CODE_SUCCESS) { tsdbError("failed to create block load info, code: out of memory, %s", pstr); continue; } @@ -1037,7 +1038,7 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra // load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file TStatisBlkArray* pStatisBlkArray = NULL; - int32_t code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray**)&pStatisBlkArray); + code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray**)&pStatisBlkArray); if (code != TSDB_CODE_SUCCESS) { tsdbError("failed to load stt block statistics, code:%s, %s", tstrerror(code), pstr); continue; From 1ce180050bfe02d035923ca6154f6a3ffd1af0ef Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 16 Jul 2024 16:01:37 +0800 Subject: [PATCH 23/34] feat:[TD-30975]process return value in schemaless --- include/os/osString.h | 2 +- source/client/src/clientSml.c | 12 ++++++++---- source/client/src/clientSmlJson.c | 10 ++++++++-- source/client/src/clientSmlLine.c | 2 +- source/client/src/clientSmlTelnet.c | 4 +++- 5 files changed, 21 insertions(+), 9 deletions(-) diff --git a/include/os/osString.h b/include/os/osString.h index ac7dd7eda8..8672da616d 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -55,7 +55,7 @@ typedef enum { M2C = 0, C2M } ConvType; #define tstrncpy(dst, src, size) \ do { \ - strncpy((dst), (src), (size)); \ + (void)strncpy((dst), (src), (size)); \ (dst)[(size)-1] = 0; \ } while (0) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 02fbe7aae5..00445b4d12 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1672,6 +1672,10 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { } for (size_t i = 0; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + if (kv == NULL){ + taosHashCleanup(kvHash); + return TSDB_CODE_SML_INVALID_DATA; + } terrno = 0; int32_t code = taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); if (terrno == TSDB_CODE_DUP_KEY) { @@ -1884,7 +1888,7 @@ static int32_t smlInsertData(SSmlHandle *info) { info->cost.insertRpcTime = taosGetTimestampUs(); SAppClusterSummary *pActivity = &info->taos->pAppInfo->summary; - atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); + (void)atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); // no need to check return code (void)launchQueryImpl(info->pRequest, info->pQuery, true, NULL); // no need to check return code @@ -1921,7 +1925,7 @@ int32_t smlClearForRerun(SSmlHandle *info) { info->lines = (SSmlLineInfo *)taosMemoryCalloc(info->lineNum, sizeof(SSmlLineInfo)); } - memset(&info->preLine, 0, sizeof(SSmlLineInfo)); + (void)memset(&info->preLine, 0, sizeof(SSmlLineInfo)); info->currSTableMeta = NULL; info->currTableDataCtx = NULL; @@ -1951,7 +1955,7 @@ static bool getLine(SSmlHandle *info, char *lines[], char **rawLine, char *rawLi if (*rawLine != NULL && (uDebugFlag & DEBUG_DEBUG)) { char *print = taosMemoryCalloc(*len + 1, 1); - memcpy(print, *tmp, *len); + (void)memcpy(print, *tmp, *len); uDebug("SML:0x%" PRIx64 " smlParseLine is raw, numLines:%d, protocol:%d, len:%d, data:%s", info->id, numLines, info->protocol, *len, print); taosMemoryFree(print); @@ -2114,7 +2118,7 @@ void smlSetReqSQL(SRequestObj *request, char *lines[], char *rawLine, char *rawL uError("malloc %d for sml sql failed", rlen + 1); return; } - memcpy(sql, p, rlen); + (void)memcpy(sql, p, rlen); sql[rlen] = 0; request->sqlstr = sql; diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index b0348b4f91..14d6a95e22 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -472,7 +472,11 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { break; } case cJSON_String: { - smlConvertJSONString(kv, "binary", root); + int32_t ret = smlConvertJSONString(kv, "binary", root); + if (ret != TSDB_CODE_SUCCESS) { + uError("OTD:Failed to parse binary value from JSON Obj"); + return ret; + } break; } case cJSON_Object: { @@ -521,7 +525,9 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ if (unlikely(ret != TSDB_CODE_SUCCESS)) { return ret; } - taosArrayPush(preLineKV, &kv); + if (taosArrayPush(preLineKV, &kv) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { return TSDB_CODE_TSC_INVALID_JSON; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index f2443080a6..3456b7df17 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -152,7 +152,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { if(data == NULL){ return TSDB_CODE_OUT_OF_MEMORY; } - memcpy(data, pVal->value + (NCHAR_ADD_LEN - 1), pVal->length); + (void)memcpy(data, pVal->value + (NCHAR_ADD_LEN - 1), pVal->length); pVal->value = data; } diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index ddd2cf5ad0..a25ef4ec95 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -131,7 +131,9 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ .length = valueLen, .keyEscaped = false, .valueEscaped = false}; - taosArrayPush(preLineKV, &kv); + if (taosArrayPush(preLineKV, &kv) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { return TSDB_CODE_SML_INVALID_DATA; } From 38ee5dc4723c3597476d2a176b849f2a07e2cd91 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 16 Jul 2024 16:00:59 +0800 Subject: [PATCH 24/34] fix(stream):fix syntax error. --- source/dnode/vnode/src/tq/tqSink.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 6494eb1bd1..3efc653f64 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -856,8 +856,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = TD_REQ_FROM_APP}; code = setDstTableDataUid(pVnode, pTask, pDataBlock, stbFullName, &tbData); if (code != TSDB_CODE_SUCCESS) { - tqError("vgId:%d s-task:%s dst-table not exist, stb:%" PRId64 " discard stream results", vgId, id, - stbFullName); + tqError("vgId:%d s-task:%s dst-table not exist, stb:%s discard stream results", vgId, id, stbFullName); continue; } From 94f596c4ced8484f1b254669dec28b19a29c4a8b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 16 Jul 2024 18:52:01 +0800 Subject: [PATCH 25/34] refactor: remove the invalid return value. --- include/common/tmsgcb.h | 4 +- include/libs/executor/storageapi.h | 6 +- include/libs/sync/sync.h | 3 +- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/inc/vnodeInt.h | 2 - source/dnode/vnode/src/tsdb/tsdbCache.c | 5 +- source/dnode/vnode/src/tsdb/tsdbRead2.c | 466 ++++++++++++++------- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 7 +- source/dnode/vnode/src/tsdb/tsdbReadUtil.h | 2 +- source/libs/executor/src/scanoperator.c | 19 +- source/libs/sync/inc/syncInt.h | 14 +- source/libs/transport/inc/transComm.h | 2 +- 12 files changed, 344 insertions(+), 188 deletions(-) diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index 03bf8da707..0c61aa5a51 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -17,13 +17,13 @@ #define _TD_COMMON_MSG_CB_H_ #include "os.h" +#include "tmsg.h" #ifdef __cplusplus extern "C" { #endif typedef struct SRpcMsg SRpcMsg; -typedef struct SEpSet SEpSet; typedef struct SMgmtWrapper SMgmtWrapper; typedef struct SRpcHandleInfo SRpcHandleInfo; @@ -46,7 +46,7 @@ typedef int32_t (*PutToQueueFp)(void* pMgmt, EQueueType qtype, SRpcMsg* pMsg); typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype); typedef int32_t (*SendReqFp)(const SEpSet* pEpSet, SRpcMsg* pMsg); typedef void (*SendRspFp)(SRpcMsg* pMsg); -typedef void (*RegisterBrokenLinkArgFp)(SRpcMsg* pMsg); +typedef void (*RegisterBrokenLinkArgFp)(struct SRpcMsg* pMsg); typedef void (*ReleaseHandleFp)(SRpcHandleInfo* pHandle, int8_t type); typedef void (*ReportStartup)(const char* name, const char* desc); diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 45f9f73fb1..edb6ec9cbf 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -105,7 +105,7 @@ typedef struct SMTbCursor { } SMTbCursor; typedef struct SMCtbCursor { - SMeta* pMeta; + struct SMeta* pMeta; void* pCur; tb_uid_t suid; void* pKey; @@ -134,7 +134,7 @@ typedef struct SMetaTableInfo { } SMetaTableInfo; typedef struct SSnapContext { - SMeta* pMeta; + struct SMeta* pMeta; int64_t snapVersion; void* pCur; int64_t suid; @@ -178,7 +178,7 @@ typedef struct TsdReader { int32_t (*tsdNextDataBlock)(); int32_t (*tsdReaderRetrieveBlockSMAInfo)(); - SSDataBlock *(*tsdReaderRetrieveDataBlock)(); + int32_t (*tsdReaderRetrieveDataBlock)(); void (*tsdReaderReleaseDataBlock)(); diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 0be03b82f3..1fb077e3ca 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -70,7 +70,6 @@ typedef int64_t SyncIndex; typedef int64_t SyncTerm; typedef struct SSyncNode SSyncNode; -typedef struct SWal SWal; typedef struct SSyncRaftEntry SSyncRaftEntry; typedef enum { @@ -238,7 +237,7 @@ typedef struct SSyncInfo { int32_t batchSize; SSyncCfg syncCfg; char path[TSDB_FILENAME_LEN]; - SWal* pWal; + struct SWal* pWal; SSyncFSM* pFsm; SMsgCb* msgcb; int32_t pingMs; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index e35c152e9b..990d03f940 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -167,7 +167,7 @@ void tsdbReaderClose2(STsdbReader *pReader); int32_t tsdbNextDataBlock2(STsdbReader *pReader, bool *hasNext); int32_t tsdbRetrieveDatablockSMA2(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave, bool *hasNullSMA); void tsdbReleaseDataBlock2(STsdbReader *pReader); -SSDataBlock *tsdbRetrieveDataBlock2(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); +int32_t tsdbRetrieveDataBlock2(STsdbReader *pReader, SSDataBlock **pBlock, SArray *pIdList); int32_t tsdbReaderReset2(STsdbReader *pReader, SQueryTableDataCond *pCond); int32_t tsdbGetFileBlocksDistInfo2(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader *pHandle); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 4a47e08730..70aa55915a 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -54,7 +54,6 @@ extern "C" { #endif typedef struct SVnodeInfo SVnodeInfo; -typedef struct SMeta SMeta; typedef struct SSma SSma; typedef struct STsdb STsdb; typedef struct STQ STQ; @@ -153,7 +152,6 @@ int32_t vnodeBufPoolRegisterQuery(SVBufPool* pPool, SQueryNode* pQNode); void vnodeBufPoolDeregisterQuery(SVBufPool* pPool, SQueryNode* pQNode, bool proactive); // meta -typedef struct SMCtbCursor SMCtbCursor; typedef struct SMStbCursor SMStbCursor; typedef struct STbUidStore STbUidStore; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index ccdf4a14d6..ed09ca821b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -3212,7 +3212,10 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs pIter->pMemDelData = NULL; - loadMemTombData(&pIter->pMemDelData, pMem, pIMem, pr->info.verRange.maxVer); + code = loadMemTombData(&pIter->pMemDelData, pMem, pIMem, pr->info.verRange.maxVer); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } pIter->idx = (SBlockIdx){.suid = suid, .uid = uid}; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index d1702a94fe..7d1e65ba81 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -398,75 +398,85 @@ static void initReaderStatus(SReaderStatus* pStatus) { pStatus->loadFromFile = true; } -static SSDataBlock* createResBlock(SQueryTableDataCond* pCond, int32_t capacity) { - SSDataBlock* pResBlock = createDataBlock(); - if (pResBlock == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; +static int32_t createResBlock(SQueryTableDataCond* pCond, int32_t capacity, SSDataBlock** pResBlock) { + *pResBlock = createDataBlock(); + if (*pResBlock == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; } for (int32_t i = 0; i < pCond->numOfCols; ++i) { SColumnInfoData colInfo = {0}; colInfo.info = pCond->colList[i]; - blockDataAppendColInfo(pResBlock, &colInfo); + int32_t code = blockDataAppendColInfo(*pResBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(*pResBlock); + *pResBlock = NULL; + return code; + } } - int32_t code = blockDataEnsureCapacity(pResBlock, capacity); + int32_t code = blockDataEnsureCapacity(*pResBlock, capacity); if (code != TSDB_CODE_SUCCESS) { - terrno = code; - taosMemoryFree(pResBlock); - return NULL; + taosMemoryFree(*pResBlock); + *pResBlock = NULL; } - return pResBlock; + + return code; } static int32_t tsdbInitReaderLock(STsdbReader* pReader) { int32_t code = taosThreadMutexInit(&pReader->readerMutex, NULL); - qTrace("tsdb/read: %p, post-init read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + tsdbTrace("tsdb/read: %p, post-init read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); return code; } static int32_t tsdbUninitReaderLock(STsdbReader* pReader) { - int32_t code = -1; - qTrace("tsdb/read: %p, pre-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + int32_t code = TSDB_CODE_SUCCESS; + tsdbTrace("tsdb/read: %p, pre-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); code = taosThreadMutexDestroy(&pReader->readerMutex); - qTrace("tsdb/read: %p, post-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + tsdbTrace("tsdb/read: %p, post-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); return code; } static int32_t tsdbAcquireReader(STsdbReader* pReader) { int32_t code = -1; - qTrace("tsdb/read: %p, pre-take read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + tsdbTrace("tsdb/read: %p, pre-take read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); code = taosThreadMutexLock(&pReader->readerMutex); - qTrace("tsdb/read: %p, post-take read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + tsdbTrace("tsdb/read: %p, post-take read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); return code; } static int32_t tsdbTryAcquireReader(STsdbReader* pReader) { int32_t code = taosThreadMutexTryLock(&pReader->readerMutex); - qTrace("tsdb/read: %p, post-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); - + if (code != TSDB_CODE_SUCCESS) { + tsdbError("tsdb/read: %p, post-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + } else { + tsdbTrace("tsdb/read: %p, post-trytask read mutex: %p", pReader, &pReader->readerMutex); + } return code; } static int32_t tsdbReleaseReader(STsdbReader* pReader) { int32_t code = taosThreadMutexUnlock(&pReader->readerMutex); - qTrace("tsdb/read: %p, post-untake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); - + if (code != TSDB_CODE_SUCCESS) { + tsdbError("tsdb/read: %p post-untake read mutex:%p failed, code:%d", pReader, &pReader->readerMutex, code); + } else { + tsdbTrace("tsdb/read: %p, post-untake read mutex: %p", pReader, &pReader->readerMutex); + } return code; } void tsdbReleaseDataBlock2(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; if (!pStatus->composedDataBlock) { - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); } } @@ -474,11 +484,16 @@ static int32_t initResBlockInfo(SResultBlockInfo* pResBlockInfo, int64_t capacit SQueryTableDataCond* pCond, SBlockLoadSuppInfo* pSup) { pResBlockInfo->capacity = capacity; pResBlockInfo->pResBlock = pResBlock; - terrno = 0; + int32_t code = 0; if (pResBlockInfo->pResBlock == NULL) { pResBlockInfo->freeBlock = true; - pResBlockInfo->pResBlock = createResBlock(pCond, pResBlockInfo->capacity); + pResBlockInfo->pResBlock = NULL; + + code = createResBlock(pCond, pResBlockInfo->capacity, &pResBlockInfo->pResBlock); + if (code != TSDB_CODE_SUCCESS) { + return code; + } if (pSup->numOfPks > 0) { SSDataBlock* p = pResBlockInfo->pResBlock; @@ -505,7 +520,7 @@ static int32_t initResBlockInfo(SResultBlockInfo* pResBlockInfo, int64_t capacit pResBlockInfo->freeBlock = false; } - return terrno; + return code; } static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void** ppReader, int32_t capacity, @@ -576,8 +591,15 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void goto _end; } - tsdbInitReaderLock(pReader); - tsem_init(&pReader->resumeAfterSuspend, 0, 0); + code = tsdbInitReaderLock(pReader); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + + code = tsem_init(&pReader->resumeAfterSuspend, 0, 0); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } *ppReader = pReader; return code; @@ -648,7 +670,11 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFileReader* pFileRead break; } - taosArrayPush(pIndexList, pBrinBlk); + void* p1 = taosArrayPush(pIndexList, pBrinBlk); + if (p1 == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + i += 1; } @@ -788,13 +814,17 @@ static int32_t loadFileBlockBrinInfo(STsdbReader* pReader, SArray* pIndexList, S pBlockNum->numOfBlocks += 1; if (taosArrayGetSize(pTableScanInfoList) == 0) { - taosArrayPush(pTableScanInfoList, &pScanInfo); + p1 = taosArrayPush(pTableScanInfoList, &pScanInfo); } else { STableBlockScanInfo** p = taosArrayGetLast(pTableScanInfoList); if ((*p)->uid != uid) { - taosArrayPush(pTableScanInfoList, &pScanInfo); + p1 = taosArrayPush(pTableScanInfoList, &pScanInfo); } } + + if (p1 == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } clearBrinBlockIter(&iter); @@ -838,6 +868,8 @@ static void updateLastKeyInfo(SRowKey* pKey, SFileDataBlockInfo* pBlockInfo, SDa static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_t colIndex, SColVal* pColVal, SBlockLoadSuppInfo* pSup) { + int32_t code = 0; + if (IS_VAR_DATA_TYPE(pColVal->value.type)) { if (!COL_VAL_IS_VALUE(pColVal)) { colDataSetNULL(pColInfoData, rowIndex); @@ -853,13 +885,13 @@ static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int (void)memcpy(varDataVal(pSup->buildBuf[colIndex]), pColVal->value.pData, pColVal->value.nData); } - colDataSetVal(pColInfoData, rowIndex, pSup->buildBuf[colIndex], false); + code = colDataSetVal(pColInfoData, rowIndex, pSup->buildBuf[colIndex], false); } } else { - colDataSetVal(pColInfoData, rowIndex, (const char*)&pColVal->value.val, !COL_VAL_IS_VALUE(pColVal)); + code = colDataSetVal(pColInfoData, rowIndex, (const char*)&pColVal->value.val, !COL_VAL_IS_VALUE(pColVal)); } - return TSDB_CODE_SUCCESS; + return code; } static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInfo** pInfo) { @@ -1420,8 +1452,7 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte int32_t step) { int32_t code = TSDB_CODE_SUCCESS; if (index < 0 || index >= pBlockIter->numOfBlocks) { - code = -1; - return code; + return TSDB_CODE_FAILED; } SFileDataBlockInfo fblock = *(SFileDataBlockInfo*)taosArrayGet(pBlockIter->blockList, index); @@ -1813,7 +1844,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (ps == NULL) { return terrno; } - tsdbRowMergerInit(pMerger, ps); + int32_t code = tsdbRowMergerInit(pMerger, ps); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } SRowKey minKey = k; @@ -1846,7 +1880,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pfKey, pReader); + code = doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pfKey, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (pkCompEx(&minKey, pSttKey) == 0) { @@ -1855,7 +1892,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (code != TSDB_CODE_SUCCESS) { return code; } - doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); + code = doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (pkCompEx(&minKey, &k) == 0) { @@ -1897,7 +1937,10 @@ static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* if (ps == NULL) { return terrno; } - tsdbRowMergerInit(pMerger, ps); + code = tsdbRowMergerInit(pMerger, ps); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } bool dataInDataFile = hasDataInFileBlock(pBlockData, pDumpInfo); @@ -1944,7 +1987,10 @@ static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* return code; } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader); + code = doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); code = tsdbRowMergerAdd(pMerger, pRow1, NULL); @@ -1953,7 +1999,10 @@ static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* } // pSttKey will be changed when sttBlockReader iterates to the next row, so use pKey instead. - doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); + code = doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { @@ -2019,7 +2068,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (ps == NULL) { return terrno; } - tsdbRowMergerInit(pMerger, ps); + + code = tsdbRowMergerInit(pMerger, ps); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } SRowKey minKey = k; @@ -2059,7 +2112,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pfKey, pReader); + code = doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pfKey, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (pkCompEx(&minKey, pSttKey) == 0) { @@ -2069,7 +2125,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); + code = doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (pkCompEx(&minKey, &ik) == 0) { @@ -2160,10 +2219,9 @@ static void doForwardDataIter(SRowKey* pKey, SIterInfo* pIter, STableBlockScanIn } // handle the open interval issue. Find the first row key that is greater than the given one. -static int32_t forwardDataIter(SRowKey* pKey, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { +static void forwardDataIter(SRowKey* pKey, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { doForwardDataIter(pKey, &pBlockScanInfo->iter, pBlockScanInfo, pReader); doForwardDataIter(pKey, &pBlockScanInfo->iiter, pBlockScanInfo, pReader); - return TSDB_CODE_SUCCESS; } static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { @@ -2196,7 +2254,10 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea return code; } - loadMemTombData(&pBlockScanInfo->pMemDelData, d, di, pReader->info.verRange.maxVer); + code = loadMemTombData(&pBlockScanInfo->pMemDelData, d, di, pReader->info.verRange.maxVer); + if (code != TSDB_CODE_SUCCESS) { + return code; + } if (forward) { forwardDataIter(&startKey.key, pBlockScanInfo, pReader); @@ -2309,8 +2370,15 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan return false; } - initMemDataIterator(pScanInfo, pReader); - initDelSkylineIterator(pScanInfo, pReader->info.order, &pReader->cost); + code = initMemDataIterator(pScanInfo, pReader); + if (code != TSDB_CODE_SUCCESS) { + return false; + } + + code = initDelSkylineIterator(pScanInfo, pReader->info.order, &pReader->cost); + if (code != TSDB_CODE_SUCCESS) { + return code; + } if (conf.rspRows) { pScanInfo->cleanSttBlocks = isCleanSttBlock(info.pKeyRangeList, &pReader->info.window, pScanInfo, order); @@ -2383,7 +2451,11 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc if (ps == NULL) { return terrno; } - tsdbRowMergerInit(pMerger, ps); + + code = tsdbRowMergerInit(pMerger, ps); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } tRowKeyAssign(&pBlockScanInfo->lastProcKey, pKey); @@ -2399,7 +2471,11 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc return code; } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader); + code = doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2446,8 +2522,16 @@ int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanIn } TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); - tsdbRowMergerAdd(pMerger, pRow1, NULL); - doMergeRowsInSttBlock(pSttBlockReader, pScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); + code = tsdbRowMergerAdd(pMerger, pRow1, NULL); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doMergeRowsInSttBlock(pSttBlockReader, pScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2527,7 +2611,10 @@ static int32_t loadNeighborIfOverlap(SFileDataBlockInfo* pBlockInfo, STableBlock // 2. remove it from the scan block list int32_t neighborIndex = tableDataBlockIdx->globalIndex; - setFileBlockActiveInBlockIter(pReader, pBlockIter, neighborIndex, step); + code = setFileBlockActiveInBlockIter(pReader, pBlockIter, neighborIndex, step); + if (code != TSDB_CODE_SUCCESS) { + return code; + } // 3. load the neighbor block, and set it to be the currently accessed file data block code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pBlockInfo->uid); @@ -2599,7 +2686,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } SBlockData* pBlockData = &pReader->status.fileBlockData; - initSttBlockReader(pSttBlockReader, pBlockScanInfo, pReader); + (void) initSttBlockReader(pSttBlockReader, pBlockScanInfo, pReader); while (1) { bool hasBlockData = false; @@ -2696,7 +2783,10 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t orde if (pSource == NULL) { pSource = pBlockScanInfo->pMemDelData; } else { - taosArrayAddAll(pSource, pBlockScanInfo->pMemDelData); + void* p1 = taosArrayAddAll(pSource, pBlockScanInfo->pMemDelData); + if (p1 == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } code = tsdbBuildDeleteSkyline(pSource, 0, taosArrayGetSize(pSource) - 1, pBlockScanInfo->delSkyline); @@ -3094,7 +3184,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { } if (pScanInfo->sttKeyInfo.status == STT_FILE_READER_UNINIT) { - initSttBlockReader(pSttBlockReader, pScanInfo, pReader); + (void) initSttBlockReader(pSttBlockReader, pScanInfo, pReader); } TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader); @@ -3148,7 +3238,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { int64_t st = taosGetTimestampUs(); // let's load data from stt files, make sure clear the cleanStt block flag before load the data from stt files - initSttBlockReader(pSttBlockReader, pScanInfo, pReader); + (void) initSttBlockReader(pSttBlockReader, pScanInfo, pReader); // no data in stt block, no need to proceed. while (hasDataInSttBlock(pScanInfo)) { @@ -3209,10 +3299,17 @@ static int32_t buildBlockFromBufferSeqForPreFileset(STsdbReader* pReader, int64_ continue; } - initMemDataIterator(*pBlockScanInfo, pReader); - initDelSkylineIterator(*pBlockScanInfo, pReader->info.order, &pReader->cost); + int32_t code = initMemDataIterator(*pBlockScanInfo, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - int32_t code = buildDataBlockFromBuf(pReader, *pBlockScanInfo, endKey); + code = initDelSkylineIterator(*pBlockScanInfo, pReader->info.order, &pReader->cost); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = buildDataBlockFromBuf(pReader, *pBlockScanInfo, endKey); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3251,10 +3348,17 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader, int64_t en continue; } - initMemDataIterator(*pBlockScanInfo, pReader); - initDelSkylineIterator(*pBlockScanInfo, pReader->info.order, &pReader->cost); + int32_t code = initMemDataIterator(*pBlockScanInfo, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - int32_t code = buildDataBlockFromBuf(pReader, *pBlockScanInfo, endKey); + code = initDelSkylineIterator(*pBlockScanInfo, pReader->info.order, &pReader->cost); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = buildDataBlockFromBuf(pReader, *pBlockScanInfo, endKey); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3693,6 +3797,7 @@ FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, S int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey* pCurKey, SArray* pDelList, STsdbReader* pReader) { SRowMerger* pMerger = &pReader->status.merger; + int32_t code = 0; while (1) { pIter->hasVal = tsdbTbDataIterNext(pIter->iter); @@ -3727,10 +3832,13 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey* pCurKey, SArra } } - tsdbRowMergerAdd(pMerger, pRow, pTSchema); + code = tsdbRowMergerAdd(pMerger, pRow, pTSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } - return TSDB_CODE_SUCCESS; + return code; } static int32_t doMergeRowsInFileBlockImpl(SBlockData* pBlockData, int32_t rowIndex, SRowKey* pKey, SRowMerger* pMerger, @@ -3748,7 +3856,7 @@ static int32_t doMergeRowsInFileBlockImpl(SBlockData* pBlockData, int32_t rowInd } TSDBROW fRow = tsdbRowFromBlockData(pBlockData, rowIndex); - tsdbRowMergerAdd(pMerger, &fRow, NULL); + int32_t code = tsdbRowMergerAdd(pMerger, &fRow, NULL); rowIndex += step; } @@ -3790,6 +3898,7 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc bool asc = ASCENDING_TRAVERSE(pReader->info.order); int32_t step = asc ? 1 : -1; SVersionRange* pRange = &pReader->info.verRange; + int32_t code = 0; pDumpInfo->rowIndex += step; if ((pDumpInfo->rowIndex <= pBlockData->nRow - 1 && asc) || (pDumpInfo->rowIndex >= 0 && !asc)) { @@ -3802,7 +3911,7 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc CHECK_FILEBLOCK_STATE st; SFileDataBlockInfo* pFileBlockInfo = NULL; - int32_t code = getCurrentBlockInfo(&pReader->status.blockIter, &pFileBlockInfo); + code = getCurrentBlockInfo(&pReader->status.blockIter, &pFileBlockInfo); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3812,19 +3921,20 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc break; } - checkForNeighborFileBlock(pReader, pScanInfo, pFileBlockInfo, pMerger, pKey, &st); - if (st == CHECK_FILEBLOCK_QUIT) { + code = checkForNeighborFileBlock(pReader, pScanInfo, pFileBlockInfo, pMerger, pKey, &st); + if (st == CHECK_FILEBLOCK_QUIT || code != TSDB_CODE_SUCCESS) { break; } } } - return TSDB_CODE_SUCCESS; + return code; } int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, SRowMerger* pMerger, int32_t pkSrcSlot, SVersionRange* pVerRange, const char* idStr) { SRowKey* pRowKey = &pScanInfo->lastProcKey; + int32_t code = TSDB_CODE_SUCCESS; while (nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pkSrcSlot, pVerRange)) { SRowKey* pNextKey = getCurrentKeyInSttBlock(pSttBlockReader); @@ -3832,7 +3942,10 @@ int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanI int32_t ret = pkCompEx(pRowKey, pNextKey); if (ret == 0) { TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); - tsdbRowMergerAdd(pMerger, pRow1, NULL); + code = tsdbRowMergerAdd(pMerger, pRow1, NULL); + if (code != TSDB_CODE_SUCCESS) { + break; + } } else { tsdbTrace("uid:%" PRIu64 " last del index:%d, del range:%d, lastKeyInStt:%" PRId64 ", %s", pScanInfo->uid, pScanInfo->sttBlockDelIndex, (int32_t)taosArrayGetSize(pScanInfo->delSkyline), @@ -3841,7 +3954,7 @@ int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanI } } - return TSDB_CODE_SUCCESS; + return code; } int32_t doMergeMemTableMultiRows(TSDBROW* pRow, SRowKey* pKey, uint64_t uid, SIterInfo* pIter, SArray* pDelList, @@ -4100,7 +4213,11 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pT if (colId == pSchema->columns[j].colId) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); - tRowGet(pTSRow, pSchema, j, &colVal); + code = tRowGet(pTSRow, pSchema, j, &colVal); + if (code) { + return code; + } + code = doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo); if (code) { return code; @@ -4185,7 +4302,11 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e do { TSDBROW row = {.type = -1}; bool freeTSRow = false; - tsdbGetNextRowInMem(pBlockScanInfo, pReader, &row, endKey, &freeTSRow); + code = tsdbGetNextRowInMem(pBlockScanInfo, pReader, &row, endKey, &freeTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (row.type == -1) { break; } @@ -4278,20 +4399,6 @@ int32_t tsdbSetTableList2(STsdbReader* pReader, const void* pTableList, int32_t return TDB_CODE_SUCCESS; } -void* tsdbGetIdx2(SMeta* pMeta) { - if (pMeta == NULL) { - return NULL; - } - return metaGetIdx(pMeta); -} - -void* tsdbGetIvtIdx2(SMeta* pMeta) { - if (pMeta == NULL) { - return NULL; - } - return metaGetIvtIdx(pMeta); -} - uint64_t tsdbGetReaderMaxVersion2(STsdbReader* pReader) { return pReader->info.verRange.maxVer; } static int32_t doOpenReaderImpl(STsdbReader* pReader) { @@ -4304,7 +4411,11 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { pReader->status.bProcMemFirstFileset = true; } - initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); + code = initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + code = resetDataBlockIterator(&pStatus->blockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); if (code != TSDB_CODE_SUCCESS) { return code; @@ -4336,7 +4447,7 @@ static void clearSharedPtr(STsdbReader* p) { p->pSchemaMap = NULL; } -static void setSharedPtr(STsdbReader* pDst, const STsdbReader* pSrc) { +static int32_t setSharedPtr(STsdbReader* pDst, const STsdbReader* pSrc) { pDst->status.pTableMap = pSrc->status.pTableMap; pDst->status.uidList = pSrc->status.uidList; pDst->info.pSchema = pSrc->info.pSchema; @@ -4345,8 +4456,10 @@ static void setSharedPtr(STsdbReader* pDst, const STsdbReader* pSrc) { pDst->pReadSnap->pfSetArray = pSrc->pReadSnap->pfSetArray; if (pDst->info.pSchema) { - tsdbRowMergerInit(&pDst->status.merger, pDst->info.pSchema); + return tsdbRowMergerInit(&pDst->status.merger, pDst->info.pSchema); } + + return TSDB_CODE_SUCCESS; } // ====================================== EXPOSED APIs ====================================== @@ -4427,7 +4540,10 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi } if (pReader->info.pSchema != NULL) { - tsdbRowMergerInit(&pReader->status.merger, pReader->info.pSchema); + code = tsdbRowMergerInit(&pReader->status.merger, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } pReader->pSchemaMap = tSimpleHashInit(8, taosFastHash); @@ -4482,7 +4598,7 @@ void tsdbReaderClose2(STsdbReader* pReader) { return; } - tsdbAcquireReader(pReader); + (void)tsdbAcquireReader(pReader); { if (pReader->innerReader[0] != NULL || pReader->innerReader[1] != NULL) { @@ -4521,7 +4637,7 @@ void tsdbReaderClose2(STsdbReader* pReader) { clearBlockScanInfoBuf(&pReader->blockInfoBuf); if (pReader->pFileReader != NULL) { - tsdbDataFileReaderClose(&pReader->pFileReader); + (void) tsdbDataFileReaderClose(&pReader->pFileReader); } SReadCostSummary* pCost = &pReader->cost; @@ -4537,15 +4653,15 @@ void tsdbReaderClose2(STsdbReader* pReader) { destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost); taosMemoryFreeClear(pReader->status.uidList.tableUidList); - qTrace("tsdb/reader-close: %p, untake snapshot", pReader); + tsdbTrace("tsdb/reader-close: %p, untake snapshot", pReader); void* p = pReader->pReadSnap; if ((p == atomic_val_compare_exchange_ptr((void**)&pReader->pReadSnap, p, NULL)) && (p != NULL)) { tsdbUntakeReadSnap2(pReader, p, true); } - tsem_destroy(&pReader->resumeAfterSuspend); - tsdbReleaseReader(pReader); - tsdbUninitReaderLock(pReader); + (void) tsem_destroy(&pReader->resumeAfterSuspend); + (void) tsdbReleaseReader(pReader); + (void) tsdbUninitReaderLock(pReader); tsdbDebug( "%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 @@ -4574,11 +4690,14 @@ static int32_t doSuspendCurrentReader(STsdbReader* pCurrentReader) { SReaderStatus* pStatus = &pCurrentReader->status; if (pStatus->loadFromFile) { - tsdbDataFileReaderClose(&pCurrentReader->pFileReader); + (void) tsdbDataFileReaderClose(&pCurrentReader->pFileReader); SReadCostSummary* pCost = &pCurrentReader->cost; destroySttBlockReader(pStatus->pLDataIterArray, &pCost->sttCost); pStatus->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); + if (pStatus->pLDataIterArray == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } // resetDataBlockScanInfo excluding lastKey @@ -4606,14 +4725,14 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { if (pReader->step == EXTERNAL_ROWS_PREV) { - doSuspendCurrentReader(pReader->innerReader[0]); + code = doSuspendCurrentReader(pReader->innerReader[0]); } else if (pReader->step == EXTERNAL_ROWS_MAIN) { - doSuspendCurrentReader(pReader); + code = doSuspendCurrentReader(pReader); } else { - doSuspendCurrentReader(pReader->innerReader[1]); + code = doSuspendCurrentReader(pReader->innerReader[1]); } } else { - doSuspendCurrentReader(pReader); + code = doSuspendCurrentReader(pReader); } // make sure only release once @@ -4648,13 +4767,12 @@ static int32_t tsdbSetQueryReseek(void* pQHandle) { code = tsdbTryAcquireReader(pReader); if (code == 0) { if (pReader->flag == READER_STATUS_SUSPEND) { - tsdbReleaseReader(pReader); + code = tsdbReleaseReader(pReader); return code; } - tsdbReaderSuspend2(pReader); - tsdbReleaseReader(pReader); - + code = tsdbReaderSuspend2(pReader); + (void) tsdbReleaseReader(pReader); return code; } else if (code == EBUSY) { return TSDB_CODE_VND_QUERY_BUSY; @@ -4671,7 +4789,7 @@ int32_t tsdbReaderResume2(STsdbReader* pReader) { // restore reader's state, task snapshot int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); if (numOfTables > 0) { - qTrace("tsdb/reader: %p, take snapshot", pReader); + tsdbTrace("tsdb/reader: %p, take snapshot", pReader); code = tsdbTakeReadSnap2(pReader, tsdbSetQueryReseek, &pReader->pReadSnap); if (code != TSDB_CODE_SUCCESS) { goto _err; @@ -4692,10 +4810,16 @@ int32_t tsdbReaderResume2(STsdbReader* pReader) { // we need only one row pPrevReader->resBlockInfo.capacity = 1; - setSharedPtr(pPrevReader, pReader); + code = setSharedPtr(pPrevReader, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pNextReader->resBlockInfo.capacity = 1; - setSharedPtr(pNextReader, pReader); + code = setSharedPtr(pNextReader, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } if (pReader->step == 0 || pReader->step == EXTERNAL_ROWS_PREV) { code = doOpenReaderImpl(pPrevReader); @@ -4864,12 +4988,17 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { #endif code = tsdbAcquireReader(pReader); - qTrace("tsdb/read: %p, take read mutex, code: %d", pReader, code); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + tsdbTrace("tsdb/read: %p, take read mutex, code: %d", pReader, code); if (pReader->flag == READER_STATUS_SUSPEND) { code = tsdbReaderResume2(pReader); if (code != TSDB_CODE_SUCCESS) { - tsdbReleaseReader(pReader); + // release reader failure should be suppressed here, to avoid over-write the original error code + (void) tsdbReleaseReader(pReader); return code; } } @@ -4877,7 +5006,7 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { if (pReader->innerReader[0] != NULL && pReader->step == 0) { code = doTsdbNextDataBlock2(pReader->innerReader[0], hasNext); if (code) { - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return code; } @@ -4885,8 +5014,8 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { if (*hasNext) { pStatus = &pReader->innerReader[0]->status; if (pStatus->composedDataBlock) { - qTrace("tsdb/read: %p, unlock read mutex", pReader); - tsdbReleaseReader(pReader); + tsdbTrace("tsdb/read: %p, unlock read mutex", pReader); + code = tsdbReleaseReader(pReader); } return code; @@ -4908,14 +5037,14 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { code = doTsdbNextDataBlock2(pReader, hasNext); if (code != TSDB_CODE_SUCCESS) { - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return code; } if (*hasNext) { if (pStatus->composedDataBlock) { - qTrace("tsdb/read: %p, unlock read mutex", pReader); - tsdbReleaseReader(pReader); + tsdbTrace("tsdb/read: %p, unlock read mutex", pReader); + code = tsdbReleaseReader(pReader); } return code; } @@ -4931,7 +5060,7 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { code = doTsdbNextDataBlock2(pReader->innerReader[1], hasNext); if (code != TSDB_CODE_SUCCESS) { - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return code; } @@ -4939,17 +5068,16 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { if (*hasNext) { pStatus = &pReader->innerReader[1]->status; if (pStatus->composedDataBlock) { - qTrace("tsdb/read: %p, unlock read mutex", pReader); - tsdbReleaseReader(pReader); + tsdbTrace("tsdb/read: %p, unlock read mutex", pReader); + code = tsdbReleaseReader(pReader); } return code; } } - qTrace("tsdb/read: %p, unlock read mutex", pReader); - tsdbReleaseReader(pReader); - + tsdbTrace("tsdb/read: %p, unlock read mutex", pReader); + code = tsdbReleaseReader(pReader); return code; } @@ -5136,7 +5264,9 @@ static int32_t doRetrieveDataBlock(STsdbReader* pReader, SSDataBlock** pBlock) { return code; } -SSDataBlock* tsdbRetrieveDataBlock2(STsdbReader* pReader, SArray* pIdList) { +int32_t tsdbRetrieveDataBlock2(STsdbReader* pReader, SSDataBlock** pBlock, SArray* pIdList) { + *pBlock = NULL; + STsdbReader* pTReader = pReader; if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { if (pReader->step == EXTERNAL_ROWS_PREV) { @@ -5151,39 +5281,39 @@ SSDataBlock* tsdbRetrieveDataBlock2(STsdbReader* pReader, SArray* pIdList) { // tsdbReaderSuspend2(pReader); // tsdbReaderResume2(pReader); - return pTReader->resBlockInfo.pResBlock; + *pBlock = pTReader->resBlockInfo.pResBlock; } - SSDataBlock* ret = NULL; - doRetrieveDataBlock(pTReader, &ret); + int32_t code = doRetrieveDataBlock(pTReader, pBlock); - qTrace("tsdb/read-retrieve: %p, unlock read mutex", pReader); - tsdbReleaseReader(pReader); + tsdbTrace("tsdb/read-retrieve: %p, unlock read mutex", pReader); + (void) tsdbReleaseReader(pReader); // tsdbReaderSuspend2(pReader); // tsdbReaderResume2(pReader); - - return ret; + return code; } int32_t tsdbReaderReset2(STsdbReader* pReader, SQueryTableDataCond* pCond) { int32_t code = TSDB_CODE_SUCCESS; - qTrace("tsdb/reader-reset: %p, take read mutex", pReader); - tsdbAcquireReader(pReader); + tsdbTrace("tsdb/reader-reset: %p, take read mutex", pReader); + code = tsdbAcquireReader(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } if (pReader->flag == READER_STATUS_SUSPEND) { code = tsdbReaderResume2(pReader); if (code != TSDB_CODE_SUCCESS) { - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return code; } } if (isEmptyQueryTimeWindow(&pReader->info.window) || pReader->pReadSnap == NULL) { tsdbDebug("tsdb reader reset return %p, %s", pReader->pReadSnap, pReader->idStr); - tsdbReleaseReader(pReader); - return TSDB_CODE_SUCCESS; + return tsdbReleaseReader(pReader); } SReaderStatus* pStatus = &pReader->status; @@ -5199,14 +5329,18 @@ int32_t tsdbReaderReset2(STsdbReader* pReader, SQueryTableDataCond* pCond) { memset(&pReader->suppInfo.tsColAgg, 0, sizeof(SColumnDataAgg)); pReader->suppInfo.tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID; - tsdbDataFileReaderClose(&pReader->pFileReader); + (void) tsdbDataFileReaderClose(&pReader->pFileReader); int32_t numOfTables = tSimpleHashGetSize(pStatus->pTableMap); - initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); + code = initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + code = resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo)); if (code != TSDB_CODE_SUCCESS) { - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return code; } @@ -5232,7 +5366,7 @@ int32_t tsdbReaderReset2(STsdbReader* pReader, SQueryTableDataCond* pCond) { tsdbError("%p reset reader failed, numOfTables:%d, query range:%" PRId64 " - %" PRId64 " in query %s", pReader, numOfTables, pReader->info.window.skey, pReader->info.window.ekey, pReader->idStr); - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return code; } } @@ -5242,8 +5376,7 @@ int32_t tsdbReaderReset2(STsdbReader* pReader, SQueryTableDataCond* pCond) { pReader, pReader->info.suid, numOfTables, pCond->twindows.skey, pReader->info.window.skey, pReader->info.window.ekey, pReader->idStr); - tsdbReleaseReader(pReader); - + code = tsdbReleaseReader(pReader); return code; } @@ -5267,12 +5400,15 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT pTableBlockInfo->numOfVgroups = 1; // find the start data block in file - tsdbAcquireReader(pReader); + code = tsdbAcquireReader(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (pReader->flag == READER_STATUS_SUSPEND) { code = tsdbReaderResume2(pReader); if (code != TSDB_CODE_SUCCESS) { - tsdbReleaseReader(pReader); - return code; + return tsdbReleaseReader(pReader); } } @@ -5354,7 +5490,7 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT } // record the data in stt files - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return code; } @@ -5411,11 +5547,15 @@ int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader* pReader) { int64_t rows = 0; SReaderStatus* pStatus = &pReader->status; - tsdbAcquireReader(pReader); + code = tsdbAcquireReader(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (pReader->flag == READER_STATUS_SUSPEND) { code = tsdbReaderResume2(pReader); if (code != TSDB_CODE_SUCCESS) { - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return code; } } @@ -5446,7 +5586,7 @@ int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader* pReader) { pStatus->pTableIter = tSimpleHashIterate(pStatus->pTableMap, pStatus->pTableIter, &iter); } - tsdbReleaseReader(pReader); + (void) tsdbReleaseReader(pReader); return rows; } @@ -5493,12 +5633,16 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs SVersionRange* pRange = &pReader->info.verRange; // lock - taosThreadMutexLock(&pTsdb->mutex); + code = taosThreadMutexLock(&pTsdb->mutex); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("failed to lock tsdb, code:%d", tstrerror(code)); + return code; + } // alloc STsdbReadSnap* pSnap = (STsdbReadSnap*)taosMemoryCalloc(1, sizeof(STsdbReadSnap)); if (pSnap == NULL) { - taosThreadMutexUnlock(&pTsdb->mutex); + (void) taosThreadMutexUnlock(&pTsdb->mutex); code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -5508,7 +5652,7 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs pSnap->pMem = pTsdb->mem; pSnap->pNode = taosMemoryMalloc(sizeof(*pSnap->pNode)); if (pSnap->pNode == NULL) { - taosThreadMutexUnlock(&pTsdb->mutex); + (void) taosThreadMutexUnlock(&pTsdb->mutex); code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -5516,14 +5660,14 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs pSnap->pNode->pQHandle = pReader; pSnap->pNode->reseek = reseek; - tsdbRefMemTable(pTsdb->mem, pSnap->pNode); + (void)tsdbRefMemTable(pTsdb->mem, pSnap->pNode); } if (pTsdb->imem && (pRange->minVer <= pTsdb->imem->maxVer && pRange->maxVer >= pTsdb->imem->minVer)) { pSnap->pIMem = pTsdb->imem; pSnap->pINode = taosMemoryMalloc(sizeof(*pSnap->pINode)); if (pSnap->pINode == NULL) { - taosThreadMutexUnlock(&pTsdb->mutex); + (void) taosThreadMutexUnlock(&pTsdb->mutex); code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -5531,14 +5675,14 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs pSnap->pINode->pQHandle = pReader; pSnap->pINode->reseek = reseek; - tsdbRefMemTable(pTsdb->imem, pSnap->pINode); + (void)tsdbRefMemTable(pTsdb->imem, pSnap->pINode); } // fs code = tsdbFSCreateRefSnapshotWithoutLock(pTsdb->pFS, &pSnap->pfSetArray); // unlock - taosThreadMutexUnlock(&pTsdb->mutex); + (void) taosThreadMutexUnlock(&pTsdb->mutex); if (code == TSDB_CODE_SUCCESS) { tsdbTrace("vgId:%d, take read snapshot", TD_VID(pTsdb->pVnode)); @@ -5566,17 +5710,17 @@ void tsdbUntakeReadSnap2(STsdbReader* pReader, STsdbReadSnap* pSnap, bool proact if (pSnap) { if (pSnap->pMem) { - tsdbUnrefMemTable(pSnap->pMem, pSnap->pNode, proactive); + (void) tsdbUnrefMemTable(pSnap->pMem, pSnap->pNode, proactive); } if (pSnap->pIMem) { - tsdbUnrefMemTable(pSnap->pIMem, pSnap->pINode, proactive); + (void) tsdbUnrefMemTable(pSnap->pIMem, pSnap->pINode, proactive); } if (pSnap->pNode) taosMemoryFree(pSnap->pNode); if (pSnap->pINode) taosMemoryFree(pSnap->pINode); - tsdbFSDestroyRefSnapshot(&pSnap->pfSetArray); + (void) tsdbFSDestroyRefSnapshot(&pSnap->pfSetArray); taosMemoryFree(pSnap); } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index 4bca7f73f8..1d330d6deb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -842,9 +842,12 @@ int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileRead return doLoadTombDataFromTombBlk(pBlkArray, pReader, pSttFileReader, false); } -void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver) { +int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver) { if (*ppMemDelData == NULL) { *ppMemDelData = taosArrayInit(4, sizeof(SDelData)); + if (*ppMemDelData == NULL) { + return TSDB_CODE_SUCCESS; + } } SArray* pMemDelData = *ppMemDelData; @@ -872,6 +875,8 @@ void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemT p = p->pNext; } } + + return TSDB_CODE_SUCCESS; } int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo, diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index ba75cb77a8..45a2384b65 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -339,7 +339,7 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3 bool blockIteratorNext(SDataBlockIter* pBlockIter, const char* idStr); // load tomb data API (stt/mem only for one table each, tomb data from data files are load for all tables at one time) -void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver); +int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver); int32_t loadDataFileTombDataForAll(STsdbReader* pReader); int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pLoadInfo); int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo, diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ce4915ca4d..f6c5118c88 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -388,9 +388,10 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pCost->totalCheckedRows += pBlock->info.rows; pCost->loadBlocks += 1; - SSDataBlock* p = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pTableScanInfo->dataReader, NULL); - if (p == NULL) { - return terrno; + SSDataBlock* p = NULL; + int32_t code = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pTableScanInfo->dataReader, &p, NULL); + if (p == NULL || code != TSDB_CODE_SUCCESS) { + return code; } ASSERT(p == pBlock); @@ -1351,7 +1352,12 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU } if (hasNext) { - /*SSDataBlock* p = */ pAPI->tsdReader.tsdReaderRetrieveDataBlock(pReader, NULL); + SSDataBlock* p = NULL; + code = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pReader, &p, NULL); + if (code != TSDB_CODE_SUCCESS) { + return NULL; + } + doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); pBlock->info.id.groupId = tableListGetTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid); } @@ -2929,8 +2935,9 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } - SSDataBlock* pBlock = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pInfo->dataReader, NULL); - if (pBlock == NULL) { + SSDataBlock* pBlock = NULL; + code = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pInfo->dataReader, &pBlock, NULL); + if (pBlock == NULL || code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, terrno); } diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 3d8588a55a..009854b45b 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -74,16 +74,16 @@ typedef struct SRaftStore { TdThreadMutex mutex; } SRaftStore; -typedef struct SSyncHbTimerData { +struct SSyncHbTimerData { int64_t syncNodeRid; SSyncTimer* pTimer; SRaftId destId; uint64_t logicClock; int64_t execTime; int64_t rid; -} SSyncHbTimerData; +}; -typedef struct SSyncTimer { +struct SSyncTimer { void* pTimer; TAOS_TMR_CALLBACK timerCb; uint64_t logicClock; @@ -92,7 +92,7 @@ typedef struct SSyncTimer { int64_t timeStamp; SRaftId destId; int64_t hbDataRid; -} SSyncTimer; +}; typedef struct SElectTimerParam { uint64_t logicClock; @@ -106,7 +106,7 @@ typedef struct SPeerState { int64_t lastSendTime; } SPeerState; -typedef struct SSyncNode { +struct SSyncNode { // init by SSyncInfo SyncGroupId vgId; SRaftCfg raftCfg; @@ -116,7 +116,7 @@ typedef struct SSyncNode { // sync io SSyncLogBuffer* pLogBuf; - SWal* pWal; + struct SWal* pWal; const SMsgCb* msgcb; int32_t (*syncSendMSg)(const SEpSet* pEpSet, SRpcMsg* pMsg); int32_t (*syncEqMsg)(const SMsgCb* msgcb, SRpcMsg* pMsg); @@ -234,7 +234,7 @@ typedef struct SSyncNode { bool isStart; -} SSyncNode; +}; // open/close -------------- SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo, int32_t vnodeVersion); diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index e62ca7d69a..acb9bd20f3 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -103,7 +103,7 @@ typedef void* queue[2]; #define TRANS_MAGIC_NUM 0x5f375a86 #define TRANS_NOVALID_PACKET(src) ((src) != TRANS_MAGIC_NUM ? 1 : 0) -typedef SRpcMsg STransMsg; +typedef struct SRpcMsg STransMsg; typedef SRpcCtx STransCtx; typedef SRpcCtxVal STransCtxVal; typedef SRpcInfo STrans; From 352abe0b5cc05973301077e8a5f1dd869ef695e4 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 16 Jul 2024 19:03:12 +0800 Subject: [PATCH 26/34] fix: scheduler return code --- include/os/osMath.h | 6 +- include/os/osMemory.h | 1 + source/libs/scheduler/inc/schInt.h | 17 ++- source/libs/scheduler/src/schJob.c | 4 +- source/libs/scheduler/src/schRemote.c | 91 +++++++++---- source/libs/scheduler/src/schStatus.c | 7 +- source/libs/scheduler/src/schTask.c | 124 ++++++++++++++---- source/libs/scheduler/src/schUtil.c | 47 ++++--- source/libs/scheduler/src/scheduler.c | 3 +- source/libs/scheduler/test/schedulerTests.cpp | 34 +++-- 10 files changed, 244 insertions(+), 90 deletions(-) diff --git a/include/os/osMath.h b/include/os/osMath.h index 3c05d15397..edbfa935ce 100644 --- a/include/os/osMath.h +++ b/include/os/osMath.h @@ -32,9 +32,9 @@ extern "C" { #define TSWAP(a, b) \ do { \ char *__tmp = (char*)alloca(sizeof(a)); \ - memcpy(__tmp, &(a), sizeof(a)); \ - memcpy(&(a), &(b), sizeof(a)); \ - memcpy(&(b), __tmp, sizeof(a)); \ + (void)memcpy(__tmp, &(a), sizeof(a)); \ + (void)memcpy(&(a), &(b), sizeof(a)); \ + (void)memcpy(&(b), __tmp, sizeof(a)); \ } while (0) #ifdef WINDOWS diff --git a/include/os/osMemory.h b/include/os/osMemory.h index 6166f1dc07..c6a5ce27c4 100644 --- a/include/os/osMemory.h +++ b/include/os/osMemory.h @@ -54,6 +54,7 @@ void taosMemoryTrim(int32_t size); void *taosMemoryMallocAlign(uint32_t alignment, int64_t size); #define TAOS_MEMSET(_s, _c, _n) ((void)memset(_s, _c, _n)) +#define TAOS_MEMCPY(_d, _s, _n) ((void)memcpy(_d, _s, _n)) #define taosMemoryFreeClear(ptr) \ do { \ diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 17476072bb..5ea79c6ae9 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -336,7 +336,7 @@ extern SSchedulerMgmt schMgmt; ((_job)->attr.localExec && SCH_IS_QUERY_JOB(_job) && (!SCH_IS_INSERT_JOB(_job)) && \ (!SCH_IS_DATA_BIND_QRY_TASK(_task))) -#define SCH_UPDATE_REDIRECT_CODE(job, _code) atomic_val_compare_exchange_32(&((job)->redirectCode), 0, _code) +#define SCH_UPDATE_REDIRECT_CODE(job, _code) (void)atomic_val_compare_exchange_32(&((job)->redirectCode), 0, _code) #define SCH_GET_REDIRECT_CODE(job, _code) \ (((!NO_RET_REDIRECT_ERROR(_code)) || (job)->redirectCode == 0) ? (_code) : (job)->redirectCode) @@ -413,7 +413,7 @@ extern SSchedulerMgmt schMgmt; #define SCH_LOG_TASK_START_TS(_task) \ do { \ int64_t us = taosGetTimestampUs(); \ - taosArrayPush((_task)->profile.execTime, &us); \ + (void)taosArrayPush((_task)->profile.execTime, &us); \ if (0 == (_task)->execId) { \ (_task)->profile.startTs = us; \ } \ @@ -422,7 +422,10 @@ extern SSchedulerMgmt schMgmt; #define SCH_LOG_TASK_WAIT_TS(_task) \ do { \ int64_t us = taosGetTimestampUs(); \ - (_task)->profile.waitTime += us - *(int64_t *)taosArrayGet((_task)->profile.execTime, (_task)->execId); \ + int64_t* startus = (int64_t*)taosArrayGet((_task)->profile.execTime, (_task)->execId); \ + if (NULL != startus) { \ + (_task)->profile.waitTime += us - *startus; \ + } \ } while (0) #define SCH_LOG_TASK_END_TS(_task) \ @@ -430,7 +433,9 @@ extern SSchedulerMgmt schMgmt; int64_t us = taosGetTimestampUs(); \ int32_t idx = (_task)->execId % (_task)->maxExecTimes; \ int64_t *startts = taosArrayGet((_task)->profile.execTime, (_task)->execId); \ - *startts = us - *startts; \ + if (NULL != startts) { \ + *startts = us - *startts; \ + } \ (_task)->profile.endTs = us; \ } while (0) @@ -538,7 +543,7 @@ void schCleanClusterHb(void *pTrans); int32_t schLaunchTask(SSchJob *job, SSchTask *task); int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask); int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType, void *param); -SSchJob *schAcquireJob(int64_t refId); +int32_t schAcquireJob(int64_t refId, SSchJob **ppJob); int32_t schReleaseJob(int64_t refId); void schFreeFlowCtrl(SSchJob *pJob); int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel); @@ -578,7 +583,7 @@ int32_t schJobFetchRows(SSchJob *pJob); int32_t schJobFetchRowsA(SSchJob *pJob); int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, void *handle, int32_t execId); int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList); -char *schDumpEpSet(SEpSet *pEpSet); +int32_t schDumpEpSet(SEpSet *pEpSet, char** ppRes); char *schGetOpStr(SCH_OP_TYPE type); int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync); int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq); diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index a8e2e79aee..58a0706223 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -1165,7 +1165,9 @@ int32_t schProcessOnCbBegin(SSchJob **job, SSchTask **task, uint64_t qId, int64_ int8_t status = 0; SSchTask *pTask = NULL; - SSchJob *pJob = schAcquireJob(rId); + SSchJob *pJob = NULL; + + (void)schAcquireJob(rId, &pJob); if (NULL == pJob) { qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, qId, tId, rId); SCH_ERR_RET(TSDB_CODE_QRY_JOB_NOT_EXIST); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index d0dc04d6b4..08a8f684f5 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -113,7 +113,7 @@ int32_t schProcessFetchRsp(SSchJob *pJob, SSchTask *pTask, char *msg, int32_t rs } atomic_store_ptr(&pJob->fetchRes, rsp); - atomic_add_fetch_64(&pJob->resNumOfRows, htobe64(rsp->numOfRows)); + (void)atomic_add_fetch_64(&pJob->resNumOfRows, htobe64(rsp->numOfRows)); if (rsp->completed) { SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCC); @@ -166,13 +166,27 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD SCH_LOCK(SCH_WRITE, &pJob->resLock); if (NULL == pJob->execRes.res) { pJob->execRes.res = (void*)taosArrayInit(batchRsp.nRsps, POINTER_BYTES); + if (NULL == pJob->execRes.res) { + code = terrno; + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + + tDecoderClear(&coder); + SCH_ERR_JRET(code); + } + pJob->execRes.msgType = TDMT_VND_CREATE_TABLE; } for (int32_t i = 0; i < batchRsp.nRsps; ++i) { SVCreateTbRsp *rsp = batchRsp.pRsps + i; if (rsp->pMeta) { - taosArrayPush((SArray*)pJob->execRes.res, &rsp->pMeta); + if (NULL == taosArrayPush((SArray*)pJob->execRes.res, &rsp->pMeta)) { + code = terrno; + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + + tDecoderClear(&coder); + SCH_ERR_JRET(code); + } } if (TSDB_CODE_SUCCESS != rsp->code) { @@ -264,7 +278,7 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD SCH_ERR_JRET(code); } - atomic_add_fetch_64(&pJob->resNumOfRows, rsp->affectedRows); + (void)atomic_add_fetch_64(&pJob->resNumOfRows, rsp->affectedRows); int32_t createTbRspNum = taosArrayGetSize(rsp->aCreateTbRsp); SCH_TASK_DLOG("submit succeed, affectedRows:%d, createTbRspNum:%d", rsp->affectedRows, createTbRspNum); @@ -275,7 +289,12 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD SSubmitRsp2 *sum = pJob->execRes.res; sum->affectedRows += rsp->affectedRows; if (sum->aCreateTbRsp) { - taosArrayAddAll(sum->aCreateTbRsp, rsp->aCreateTbRsp); + if (NULL == taosArrayAddAll(sum->aCreateTbRsp, rsp->aCreateTbRsp)) { + code = terrno; + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + SCH_ERR_JRET(code); + } + taosArrayDestroy(rsp->aCreateTbRsp); } else { TSWAP(sum->aCreateTbRsp, rsp->aCreateTbRsp); @@ -313,10 +332,14 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD SDecoder coder = {0}; SVDeleteRsp rsp = {0}; tDecoderInit(&coder, msg, msgSize); - tDecodeSVDeleteRsp(&coder, &rsp); + if (tDecodeSVDeleteRsp(&coder, &rsp) < 0) { + code = terrno; + tDecoderClear(&coder); + SCH_ERR_JRET(code); + } tDecoderClear(&coder); - atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows); + (void)atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows); SCH_TASK_DLOG("delete succeed, affectedRows:%" PRId64, rsp.affectedRows); } @@ -351,7 +374,7 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD SCH_ERR_JRET(schSaveJobExecRes(pJob, &rsp)); - atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows); + (void)atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows); taosMemoryFreeClear(msg); @@ -479,7 +502,7 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, pParam->taskId, code); // called if drop task rsp received code - rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); + (void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error if (pMsg) { taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); @@ -501,7 +524,7 @@ int32_t schHandleNotifyCallback(void *param, SDataBuf *pMsg, int32_t code) { int32_t schHandleLinkBrokenCallback(void *param, SDataBuf *pMsg, int32_t code) { SSchCallbackParamHeader *head = (SSchCallbackParamHeader *)param; - rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); + (void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error qDebug("handle %p is broken", pMsg->handle); @@ -531,7 +554,7 @@ int32_t schHandleHbCallback(void *param, SDataBuf *pMsg, int32_t code) { if (code) { qError("hb rsp error:%s", tstrerror(code)); - rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); + (void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error SCH_ERR_JRET(code); } @@ -585,9 +608,14 @@ int32_t schMakeCallbackParam(SSchJob *pJob, SSchTask *pTask, int32_t msgType, bo param->head.isHbParam = true; SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + taosMemoryFree(param); + SCH_TASK_ELOG("fail to get the %dth condidateAddr, totalNum: %d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } param->nodeEpId.nodeId = addr->nodeId; SEp *pEp = SCH_GET_CUR_EP(addr); - strcpy(param->nodeEpId.ep.fqdn, pEp->fqdn); + TAOS_STRCPY(param->nodeEpId.ep.fqdn, pEp->fqdn); param->nodeEpId.ep.port = pEp->port; param->pTrans = trans->pTrans; *pParam = param; @@ -712,7 +740,7 @@ int32_t schMakeHbCallbackParam(SSchJob *pJob, SSchTask *pTask, void **pParam) { int32_t schCloneHbRpcCtx(SRpcCtx *pSrc, SRpcCtx *pDst) { int32_t code = 0; - memcpy(pDst, pSrc, sizeof(SRpcCtx)); + TAOS_MEMCPY(pDst, pSrc, sizeof(SRpcCtx)); pDst->brokenVal.val = NULL; pDst->args = NULL; @@ -760,7 +788,7 @@ int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; - memcpy(&epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); + TAOS_MEMCPY(&epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); pCtx->args = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); if (NULL == pCtx->args) { @@ -877,7 +905,7 @@ int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHe SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(dst, pSrc, sizeof(*dst)); + TAOS_MEMCPY(dst, pSrc, sizeof(*dst)); *pDst = (SSchCallbackParamHeader *)dst; return TSDB_CODE_SUCCESS; @@ -889,7 +917,7 @@ int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHe SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(dst, pSrc, sizeof(*dst)); + TAOS_MEMCPY(dst, pSrc, sizeof(*dst)); *pDst = (SSchCallbackParamHeader *)dst; return TSDB_CODE_SUCCESS; @@ -904,7 +932,7 @@ int32_t schCloneSMsgSendInfo(void *src, void **dst) { SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(pDst, pSrc, sizeof(*pSrc)); + TAOS_MEMCPY(pDst, pSrc, sizeof(*pSrc)); pDst->param = NULL; SCH_ERR_JRET(schCloneCallbackParam(pSrc->param, (SSchCallbackParamHeader **)&pDst->param)); @@ -948,6 +976,10 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, SSchTrans *trans, SQuery if (isHb && persistHandle && trans->pHandle == 0) { trans->pHandle = rpcAllocHandle(); + if (NULL == trans->pHandle) { + SCH_TASK_ELOG("rpcAllocHandle failed, code:%x", terrno); + SCH_ERR_JRET(terrno); + } } if (pJob && pTask) { @@ -1000,7 +1032,7 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray *taskAction) { req.header.vgId = nodeEpId->nodeId; req.sId = schMgmt.sId; - memcpy(&req.epId, nodeEpId, sizeof(SQueryNodeEpId)); + TAOS_MEMCPY(&req.epId, nodeEpId, sizeof(SQueryNodeEpId)); SCH_LOCK(SCH_READ, &schMgmt.hbLock); SSchHbTrans *hb = taosHashGet(schMgmt.hbConnections, nodeEpId, sizeof(SQueryNodeEpId)); @@ -1013,7 +1045,7 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray *taskAction) { SCH_LOCK(SCH_WRITE, &hb->lock); code = schCloneHbRpcCtx(&hb->rpcCtx, &rpcCtx); - memcpy(&trans, &hb->trans, sizeof(trans)); + TAOS_MEMCPY(&trans, &hb->trans, sizeof(trans)); SCH_UNLOCK(SCH_WRITE, &hb->lock); SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); @@ -1039,7 +1071,7 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray *taskAction) { SQueryNodeAddr addr = {.nodeId = nodeEpId->nodeId}; addr.epSet.inUse = 0; addr.epSet.numOfEps = 1; - memcpy(&addr.epSet.eps[0], &nodeEpId->ep, sizeof(nodeEpId->ep)); + TAOS_MEMCPY(&addr.epSet.eps[0], &nodeEpId->ep, sizeof(nodeEpId->ep)); code = schAsyncSendMsg(NULL, NULL, &trans, &addr, msgType, msg, msgSize, true, &rpcCtx); msg = NULL; @@ -1064,6 +1096,11 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, if (NULL == addr) { addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + SCH_TASK_ELOG("fail to get condidateAddr, candidateIdx %d, totalNum: %d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_JRET(terrno); + } + isCandidateAddr = true; SCH_TASK_DLOG("target candidateIdx %d, epInUse %d/%d", pTask->candidateIdx, addr->epSet.inUse, addr->epSet.numOfEps); @@ -1082,7 +1119,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(msg, pTask->msg, msgSize); + TAOS_MEMCPY(msg, pTask->msg, msgSize); break; } @@ -1098,13 +1135,21 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, req.msg = pTask->msg; req.source = pJob->source; msgSize = tSerializeSVDeleteReq(NULL, 0, &req); + if (msgSize < 0) { + SCH_TASK_ELOG("tSerializeSVDeleteReq failed, code:%x", terrno); + SCH_ERR_JRET(terrno); + } msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - tSerializeSVDeleteReq(msg, msgSize, &req); + msgSize = tSerializeSVDeleteReq(msg, msgSize, &req); + if (msgSize < 0) { + SCH_TASK_ELOG("tSerializeSVDeleteReq second failed, code:%x", terrno); + SCH_ERR_JRET(terrno); + } break; } case TDMT_SCH_QUERY: @@ -1221,7 +1266,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, req.sId = schMgmt.sId; req.header.vgId = addr->nodeId; req.epId.nodeId = addr->nodeId; - memcpy(&req.epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); + TAOS_MEMCPY(&req.epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); msgSize = tSerializeSSchedulerHbReq(NULL, 0, &req); if (msgSize < 0) { diff --git a/source/libs/scheduler/src/schStatus.c b/source/libs/scheduler/src/schStatus.c index d37393137f..f24ee74101 100644 --- a/source/libs/scheduler/src/schStatus.c +++ b/source/libs/scheduler/src/schStatus.c @@ -43,7 +43,7 @@ int32_t schSwitchJobStatus(SSchJob* pJob, int32_t status, void* param) { SCH_RET(schProcessOnJobFailure(pJob, (param ? *(int32_t*)param : 0))); break; case JOB_TASK_STATUS_DROP: - schProcessOnJobDropped(pJob, *(int32_t*)param); + (void)schProcessOnJobDropped(pJob, *(int32_t*)param); // ignore error if (taosRemoveRef(schMgmt.jobRef, pJob->refId)) { SCH_JOB_ELOG("remove job from job list failed, refId:0x%" PRIx64, pJob->refId); @@ -65,7 +65,8 @@ _return: } int32_t schHandleOpBeginEvent(int64_t jobId, SSchJob** job, SCH_OP_TYPE type, SSchedulerReq* pReq) { - SSchJob* pJob = schAcquireJob(jobId); + SSchJob* pJob = NULL; + (void)schAcquireJob(jobId, &pJob); if (NULL == pJob) { qDebug("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, jobId); SCH_ERR_RET(TSDB_CODE_SCH_JOB_NOT_EXISTS); @@ -90,7 +91,7 @@ int32_t schHandleOpEndEvent(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq code = pJob->errCode; } - schReleaseJob(pJob->refId); + (void)schReleaseJob(pJob->refId); // ignore error return code; } diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index c317a63ce1..6dd6aa9aae 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -165,7 +165,7 @@ int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, v SCH_RET(schDropTaskExecNode(pJob, pTask, handle, execId)); } - schUpdateTaskExecNode(pJob, pTask, handle, execId); + SCH_ERR_RET(schUpdateTaskExecNode(pJob, pTask, handle, execId)); if ((execId != pTask->execId || execId <= pTask->failedExecId) || pTask->waitRetry) { // ignore it SCH_TASK_DLOG("handle not updated since execId %d is already not current execId %d, waitRetry %d", execId, @@ -297,7 +297,11 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { for (int32_t i = 0; i < parentNum; ++i) { SSchTask *parent = *(SSchTask **)taosArrayGet(pTask->parents, i); - + if (NULL == parent) { + SCH_TASK_ELOG("fail to get task %d parent, parentNum: %d", i, parentNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SCH_LOCK(SCH_WRITE, &parent->planLock); SDownstreamSourceNode source = { .type = QUERY_NODE_DOWNSTREAM_SOURCE, @@ -308,9 +312,14 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { .fetchMsgType = SCH_FETCH_TYPE(pTask), .localExec = SCH_IS_LOCAL_EXEC_TASK(pJob, pTask), }; - qSetSubplanExecutionNode(parent->plan, pTask->plan->id.groupId, &source); + code = qSetSubplanExecutionNode(parent->plan, pTask->plan->id.groupId, &source); + if (TSDB_CODE_SUCCESS != code) { + SCH_TASK_ELOG("qSetSubplanExecutionNode failed, groupId: %d", pTask->plan->id.groupId); + } SCH_UNLOCK(SCH_WRITE, &parent->planLock); + SCH_ERR_RET(code); + int32_t readyNum = atomic_add_fetch_32(&parent->childReady, 1); if (SCH_TASK_READY_FOR_LAUNCH(readyNum, parent)) { @@ -413,16 +422,16 @@ void schResetTaskForRetry(SSchJob *pJob, SSchTask *pTask) { schDropTaskOnExecNode(pJob, pTask); if (pTask->delayTimer) { - taosTmrStopA(&pTask->delayTimer); + (void)taosTmrStopA(&pTask->delayTimer); // ignore error } taosHashClear(pTask->execNodes); - schRemoveTaskFromExecList(pJob, pTask); + (void)schRemoveTaskFromExecList(pJob, pTask); // ignore error schDeregisterTaskHb(pJob, pTask); taosMemoryFreeClear(pTask->msg); pTask->msgLen = 0; pTask->lastMsgType = 0; pTask->childReady = 0; - memset(&pTask->succeedAddr, 0, sizeof(pTask->succeedAddr)); + TAOS_MEMSET(&pTask->succeedAddr, 0, sizeof(pTask->succeedAddr)); } int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode) { @@ -443,11 +452,21 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 SCH_ERR_JRET(schUpdateTaskCandidateAddr(pJob, pTask, pData->pEpSet)); } else if (SYNC_SELF_LEADER_REDIRECT_ERROR(rspCode)) { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + SCH_TASK_ELOG("fail to get the %dth condidateAddr, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SEp *pEp = &addr->epSet.eps[addr->epSet.inUse]; SCH_TASK_DLOG("task retry node %d current ep, idx:%d/%d,%s:%d, code:%s", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps, pEp->fqdn, pEp->port, tstrerror(rspCode)); } else { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + SCH_TASK_ELOG("fail to get the %dth condidateAddr, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SCH_SWITCH_EPSET(addr); SCH_TASK_DLOG("switch task target node %d epset to %d/%d", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps); } @@ -476,7 +495,7 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 for (int32_t i = 0; i < childrenNum; ++i) { SSchTask *pChild = taosArrayGetP(pTask->children, i); SCH_LOCK_TASK(pChild); - schDoTaskRedirect(pJob, pChild, NULL, rspCode); + (void)schDoTaskRedirect(pJob, pChild, NULL, rspCode); // error handled internal SCH_UNLOCK_TASK(pChild); } @@ -494,18 +513,23 @@ int32_t schResetTaskSetLevelInfo(SSchJob *pJob, SSchTask *pTask) { atomic_load_32(&pLevel->taskExecDoneNum), atomic_load_32(&pLevel->taskLaunchedNum)); if (SCH_GET_TASK_STATUS(pTask) >= JOB_TASK_STATUS_PART_SUCC) { - atomic_sub_fetch_32(&pLevel->taskExecDoneNum, 1); + (void)atomic_sub_fetch_32(&pLevel->taskExecDoneNum, 1); } - atomic_sub_fetch_32(&pLevel->taskLaunchedNum, 1); + (void)atomic_sub_fetch_32(&pLevel->taskLaunchedNum, 1); int32_t childrenNum = taosArrayGetSize(pTask->children); for (int32_t i = 0; i < childrenNum; ++i) { SSchTask *pChild = taosArrayGetP(pTask->children, i); + if (NULL == pChild) { + SCH_TASK_ELOG("fail to get the %dth child, childrenNum:%d", i, childrenNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SCH_LOCK_TASK(pChild); pLevel = pChild->level; - atomic_sub_fetch_32(&pLevel->taskExecDoneNum, 1); - atomic_sub_fetch_32(&pLevel->taskLaunchedNum, 1); + (void)atomic_sub_fetch_32(&pLevel->taskExecDoneNum, 1); + (void)atomic_sub_fetch_32(&pLevel->taskLaunchedNum, 1); SCH_UNLOCK_TASK(pChild); } @@ -711,9 +735,9 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo } int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { - atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1); + (void)atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1); - schRemoveTaskFromExecList(pJob, pTask); + (void)schRemoveTaskFromExecList(pJob, pTask); // ignore error SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT); if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) { @@ -724,6 +748,11 @@ int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { if (SCH_IS_DATA_BIND_TASK(pTask)) { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + SCH_TASK_ELOG("fail to the %dth condidateAddr, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SCH_SWITCH_EPSET(addr); } else { SCH_ERR_RET(schSwitchTaskCandidateAddr(pJob, pTask)); @@ -743,6 +772,11 @@ int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) { for (int32_t i = 0; i < nodeNum; ++i) { SQueryNodeLoad *nload = taosArrayGet(pJob->nodeList, i); + if (NULL == nload) { + SCH_TASK_ELOG("fail to get the %dth node in nodeList, nodeNum:%d", i, nodeNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + SQueryNodeAddr *naddr = &nload->addr; if (NULL == taosArrayPush(pTask->candidateAddrs, naddr)) { @@ -810,6 +844,7 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { } int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet) { + int32_t code = TSDB_CODE_SUCCESS; if (NULL == pTask->candidateAddrs || 1 != taosArrayGetSize(pTask->candidateAddrs)) { SCH_TASK_ELOG("not able to update cndidate addr, addr num %d", (int32_t)(pTask->candidateAddrs ? taosArrayGetSize(pTask->candidateAddrs) : 0)); @@ -817,18 +852,27 @@ int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSe } SQueryNodeAddr *pAddr = taosArrayGet(pTask->candidateAddrs, 0); + if (NULL == pAddr) { + SCH_TASK_ELOG("fail to get task 0th condidataAddr, totalNum:%d", (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } - char *origEpset = schDumpEpSet(&pAddr->epSet); - char *newEpset = schDumpEpSet(pEpSet); + char *origEpset = NULL; + char *newEpset = NULL; + + SCH_ERR_RET(schDumpEpSet(&pAddr->epSet, &origEpset)); + SCH_ERR_JRET(schDumpEpSet(pEpSet, &newEpset)); SCH_TASK_DLOG("update task target node %d epset from %s to %s", pAddr->nodeId, origEpset, newEpset); + TAOS_MEMCPY(&pAddr->epSet, pEpSet, sizeof(pAddr->epSet)); + +_return: + taosMemoryFree(origEpset); taosMemoryFree(newEpset); - memcpy(&pAddr->epSet, pEpSet, sizeof(pAddr->epSet)); - - return TSDB_CODE_SUCCESS; + return code; } int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask) { @@ -877,7 +921,6 @@ void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask) { } int32_t size = (int32_t)taosHashGetSize(pTask->execNodes); - if (size <= 0) { SCH_TASK_DLOG("task has no execNodes, no need to drop it, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); return; @@ -889,7 +932,7 @@ void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask) { if (nodeInfo->handle) { SCH_SET_TASK_HANDLE(pTask, nodeInfo->handle); void *pExecId = taosHashGetKey(nodeInfo, NULL); - schBuildAndSendMsg(pJob, pTask, &nodeInfo->addr, TDMT_SCH_DROP_TASK, pExecId); + (void)schBuildAndSendMsg(pJob, pTask, &nodeInfo->addr, TDMT_SCH_DROP_TASK, pExecId); // ignore error and continue SCH_TASK_DLOG("start to drop task's %dth execNode", i); } else { @@ -939,6 +982,11 @@ int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList) { for (int32_t i = 0; i < taskNum; ++i) { STaskStatus *pStatus = taosArrayGet(pStatusList, i); + if (NULL == pStatus) { + qError("fail to get the %dth task status in hb rsp, taskNum:%d", i, taskNum); + continue; + } + int32_t code = 0; qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d task status in server: %s", pStatus->queryId, pStatus->taskId, @@ -983,10 +1031,15 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { for (int32_t i = 0; i < resNum; ++i) { SExplainLocalRsp *localRsp = taosArrayGet(pExplainRes, i); + if (NULL == localRsp) { + qError("fail to get the %dth LOCAL explain rsp msg, total:%d", i, resNum); + continue; + } qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", begin to handle LOCAL explain rsp msg", localRsp->qId, localRsp->tId); - pJob = schAcquireJob(localRsp->rId); + pJob = NULL; + (void)schAcquireJob(localRsp->rId, &pJob); if (NULL == pJob) { qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, localRsp->qId, localRsp->tId, localRsp->rId); @@ -996,7 +1049,7 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { int8_t status = 0; if (schJobNeedToStop(pJob, &status)) { SCH_TASK_DLOG("will not do further processing cause of job status %s", jobTaskStatusStr(status)); - schReleaseJob(pJob->refId); + (void)schReleaseJob(pJob->refId); SCH_ERR_JRET(TSDB_CODE_SCH_IGNORE_ERROR); } @@ -1006,7 +1059,7 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { code = schProcessExplainRsp(pJob, pTask, &localRsp->rsp); } - schReleaseJob(pJob->refId); + (void)schReleaseJob(pJob->refId); qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", end to handle LOCAL explain rsp msg, code:%x", localRsp->qId, localRsp->tId, code); @@ -1022,6 +1075,11 @@ _return: for (int32_t i = 0; i < resNum; ++i) { SExplainLocalRsp *localRsp = taosArrayGet(pExplainRes, i); + if (NULL == localRsp) { + qError("in _return fail to get the %dth LOCAL explain rsp msg, total:%d", i, resNum); + continue; + } + tFreeSExplainRsp(&localRsp->rsp); } @@ -1076,6 +1134,9 @@ int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { if (SCH_IS_EXPLAIN_JOB(pJob)) { explainRes = taosArrayInit(pJob->taskNum, sizeof(SExplainLocalRsp)); + if (NULL == explainRes) { + SCH_ERR_RET(terrno); + } } SCH_ERR_JRET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, @@ -1097,7 +1158,9 @@ _return: int32_t schLaunchTaskImpl(void *param) { SSchTaskCtx *pCtx = (SSchTaskCtx *)param; - SSchJob *pJob = schAcquireJob(pCtx->jobRid); + SSchJob *pJob = NULL; + + (void)schAcquireJob(pCtx->jobRid, &pJob); if (NULL == pJob) { qDebug("job refId 0x%" PRIx64 " already not exist", pCtx->jobRid); taosMemoryFree(param); @@ -1113,7 +1176,7 @@ int32_t schLaunchTaskImpl(void *param) { int8_t status = 0; int32_t code = 0; - atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1); + (void)atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1); pTask->execId++; pTask->retryTimes++; pTask->waitRetry = false; @@ -1160,7 +1223,7 @@ _return: SCH_UNLOCK_TASK(pTask); } - schReleaseJob(pJob->refId); + (void)schReleaseJob(pJob->refId); taosMemoryFree(param); @@ -1178,7 +1241,7 @@ int32_t schAsyncLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { if (pJob->taskNum >= SCH_MIN_AYSNC_EXEC_NUM) { param->asyncLaunch = true; - taosAsyncExec(schLaunchTaskImpl, param, NULL); + SCH_ERR_RET(taosAsyncExec(schLaunchTaskImpl, param, NULL)); } else { SCH_ERR_RET(schLaunchTaskImpl(param)); } @@ -1252,7 +1315,7 @@ int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } - taosTmrReset(schHandleTimerEvent, pTask->delayExecMs, (void *)param, schMgmt.timer, &pTask->delayTimer); + (void)taosTmrReset(schHandleTimerEvent, pTask->delayExecMs, (void *)param, schMgmt.timer, &pTask->delayTimer); return TSDB_CODE_SUCCESS; } @@ -1283,7 +1346,7 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { SCH_LOCK_TASK(pTask); if (pTask->delayTimer) { - taosTmrStopA(&pTask->delayTimer); + (void)taosTmrStopA(&pTask->delayTimer); } schDropTaskOnExecNode(pJob, pTask); SCH_UNLOCK_TASK(pTask); @@ -1327,6 +1390,9 @@ int32_t schExecLocalFetch(SSchJob *pJob, SSchTask *pTask) { if (SCH_IS_EXPLAIN_JOB(pJob)) { explainRes = taosArrayInit(pJob->taskNum, sizeof(SExplainLocalRsp)); + if (NULL == explainRes) { + SCH_ERR_RET(terrno); + } } SCH_ERR_JRET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 689c98d395..811890dde5 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -22,9 +22,14 @@ #include "tref.h" #include "trpc.h" -FORCE_INLINE SSchJob *schAcquireJob(int64_t refId) { +FORCE_INLINE int32_t schAcquireJob(int64_t refId, SSchJob** ppJob) { qDebug("sch acquire jobId:0x%" PRIx64, refId); - return (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); + *ppJob = (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); + if (NULL == *ppJob) { + return terrno; + } + + return TSDB_CODE_SUCCESS; } FORCE_INLINE int32_t schReleaseJob(int64_t refId) { @@ -36,15 +41,16 @@ FORCE_INLINE int32_t schReleaseJob(int64_t refId) { return taosReleaseRef(schMgmt.jobRef, refId); } -char *schDumpEpSet(SEpSet *pEpSet) { +int32_t schDumpEpSet(SEpSet *pEpSet, char** ppRes) { + *ppRes = NULL; if (NULL == pEpSet) { - return NULL; + return TSDB_CODE_SUCCESS; } int32_t maxSize = 1024; char *str = taosMemoryMalloc(maxSize); if (NULL == str) { - return NULL; + return terrno; } int32_t n = 0; @@ -54,7 +60,8 @@ char *schDumpEpSet(SEpSet *pEpSet) { n += snprintf(str + n, maxSize - n, "[%s:%d]", pEp->fqdn, pEp->port); } - return str; + *ppRes = str; + return TSDB_CODE_SUCCESS; } char *schGetOpStr(SCH_OP_TYPE type) { @@ -73,7 +80,7 @@ char *schGetOpStr(SCH_OP_TYPE type) { } void schFreeHbTrans(SSchHbTrans *pTrans) { - rpcReleaseHandle((void *)pTrans->trans.pHandleId, TAOS_CONN_CLIENT); + (void)rpcReleaseHandle((void *)pTrans->trans.pHandleId, TAOS_CONN_CLIENT); schFreeRpcCtx(&pTrans->rpcCtx); } @@ -86,7 +93,7 @@ void schCleanClusterHb(void *pTrans) { if (hb->trans.pTrans == pTrans) { SQueryNodeEpId *pEpId = taosHashGetKey(hb, NULL); schFreeHbTrans(hb); - taosHashRemove(schMgmt.hbConnections, pEpId, sizeof(SQueryNodeEpId)); + (void)taosHashRemove(schMgmt.hbConnections, pEpId, sizeof(SQueryNodeEpId)); } hb = taosHashIterate(schMgmt.hbConnections, hb); @@ -109,7 +116,7 @@ int32_t schRemoveHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *ep int64_t taskNum = atomic_load_64(&hb->taskNum); if (taskNum <= 0) { schFreeHbTrans(hb); - taosHashRemove(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); + (void)taosHashRemove(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); } SCH_UNLOCK(SCH_WRITE, &schMgmt.hbLock); @@ -165,7 +172,7 @@ int32_t schRegisterHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId * break; } - atomic_add_fetch_64(&hb->taskNum, 1); + (void)atomic_add_fetch_64(&hb->taskNum, 1); SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); @@ -178,12 +185,17 @@ void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask) { } SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + SCH_TASK_ELOG("fail to get the %dth condidateAddr in task, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + return; + } + SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; SEp *pEp = SCH_GET_CUR_EP(addr); - strcpy(epId.ep.fqdn, pEp->fqdn); + TAOS_STRCPY(epId.ep.fqdn, pEp->fqdn); epId.ep.port = pEp->port; SCH_LOCK(SCH_READ, &schMgmt.hbLock); @@ -197,7 +209,7 @@ void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask) { int64_t taskNum = atomic_sub_fetch_64(&hb->taskNum, 1); if (0 == taskNum) { SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); - schRemoveHbConnection(pJob, pTask, &epId); + (void)schRemoveHbConnection(pJob, pTask, &epId); } else { SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); } @@ -211,12 +223,17 @@ int32_t schEnsureHbConnection(SSchJob *pJob, SSchTask *pTask) { } SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + if (NULL == addr) { + SCH_TASK_ELOG("fail to get the %dth condidateAddr in task, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + return TSDB_CODE_SCH_INTERNAL_ERROR; + } + SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; SEp *pEp = SCH_GET_CUR_EP(addr); - strcpy(epId.ep.fqdn, pEp->fqdn); + TAOS_STRCPY(epId.ep.fqdn, pEp->fqdn); epId.ep.port = pEp->port; SCH_ERR_RET(schRegisterHbConnection(pJob, pTask, &epId)); @@ -240,7 +257,7 @@ int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans) { } SCH_LOCK(SCH_WRITE, &hb->lock); - memcpy(&hb->trans, trans, sizeof(*trans)); + TAOS_MEMCPY(&hb->trans, trans, sizeof(*trans)); SCH_UNLOCK(SCH_WRITE, &hb->lock); SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); @@ -256,7 +273,7 @@ void schCloseJobRef(void) { } if (schMgmt.jobRef >= 0) { - taosCloseRef(schMgmt.jobRef); + (void)taosCloseRef(schMgmt.jobRef); schMgmt.jobRef = -1; } } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 8f85e066cd..45e0ecf738 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -169,7 +169,8 @@ void schedulerFreeJob(int64_t *jobId, int32_t errCode) { return; } - SSchJob *pJob = schAcquireJob(*jobId); + SSchJob *pJob = NULL; + (void)schAcquireJob(*jobId, &pJob); if (NULL == pJob) { qWarn("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId); return; diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 78e876f82c..7bae175ba9 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -431,8 +431,11 @@ void *schtSendRsp(void *param) { taosMsleep(1); } - pJob = schAcquireJob(job); - + code = schAcquireJob(job, &pJob); + if (code) { + return NULL; + } + void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; @@ -457,8 +460,13 @@ void *schtSendRsp(void *param) { void *schtCreateFetchRspThread(void *param) { int64_t job = *(int64_t *)param; - SSchJob *pJob = schAcquireJob(job); + SSchJob *pJob = NULL; + (void)schAcquireJob(job, &pJob); + if (NULL == pJob) { + return NULL; + } + taosSsleep(1); int32_t code = 0; @@ -572,7 +580,9 @@ void *schtRunJobThread(void *aa) { code = schedulerExecJob(&req, &queryJobRefId); assert(code == 0); - pJob = schAcquireJob(queryJobRefId); + pJob = NULL; + code = schAcquireJob(queryJobRefId, &pJob); + if (NULL == pJob) { taosArrayDestroy(qnodeList); schtFreeQueryDag(dag); @@ -728,7 +738,9 @@ TEST(queryTest, normalCase) { code = schedulerExecJob(&req, &job); ASSERT_EQ(code, 0); - SSchJob *pJob = schAcquireJob(job); + SSchJob *pJob = NULL; + code = schAcquireJob(job, &pJob); + ASSERT_EQ(code, 0); void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { @@ -839,8 +851,10 @@ TEST(queryTest, readyFirstCase) { code = schedulerExecJob(&req, &job); ASSERT_EQ(code, 0); - SSchJob *pJob = schAcquireJob(job); - + SSchJob *pJob = NULL; + code = schAcquireJob(job, &pJob); + ASSERT_EQ(code, 0); + void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; @@ -956,7 +970,9 @@ TEST(queryTest, flowCtrlCase) { code = schedulerExecJob(&req, &job); ASSERT_EQ(code, 0); - SSchJob *pJob = schAcquireJob(job); + SSchJob *pJob = NULL; + code = schAcquireJob(job, &pJob); + ASSERT_EQ(code, 0); while (!queryDone) { void *pIter = taosHashIterate(pJob->execTasks, NULL); @@ -1094,7 +1110,7 @@ TEST(otherTest, otherCase) { schReleaseJob(0); schFreeRpcCtx(NULL); - ASSERT_EQ(schDumpEpSet(NULL), (char *)NULL); + ASSERT_EQ(schDumpEpSet(NULL, NULL), TSDB_CODE_SUCCESS); ASSERT_EQ(strcmp(schGetOpStr(SCH_OP_NULL), "NULL"), 0); ASSERT_EQ(strcmp(schGetOpStr((SCH_OP_TYPE)100), "UNKNOWN"), 0); } From b29e1c2b0d7a250e39728cd3a48a5784dfc644e4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 16 Jul 2024 19:13:14 +0800 Subject: [PATCH 27/34] fix(stream):fix syntax error. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 7d1e65ba81..c9c61a990f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -5635,7 +5635,7 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs // lock code = taosThreadMutexLock(&pTsdb->mutex); if (code != TSDB_CODE_SUCCESS) { - tsdbError("failed to lock tsdb, code:%d", tstrerror(code)); + tsdbError("failed to lock tsdb, code:%s", tstrerror(code)); return code; } From f3cfd297b86f6d3c6b6af34a9badcbc8ee06f512 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 16 Jul 2024 19:42:29 +0800 Subject: [PATCH 28/34] fix(stream): return after get value. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 2 +- tests/script/tsim/insert/update2.sim | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index c9c61a990f..da6e029091 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -5280,8 +5280,8 @@ int32_t tsdbRetrieveDataBlock2(STsdbReader* pReader, SSDataBlock** pBlock, SArra if (pStatus->composedDataBlock || pReader->info.execMode == READER_EXEC_ROWS) { // tsdbReaderSuspend2(pReader); // tsdbReaderResume2(pReader); - *pBlock = pTReader->resBlockInfo.pResBlock; + return TSDB_CODE_SUCCESS; } int32_t code = doRetrieveDataBlock(pTReader, pBlock); diff --git a/tests/script/tsim/insert/update2.sim b/tests/script/tsim/insert/update2.sim index d2b5b05267..8ea767da0e 100644 --- a/tests/script/tsim/insert/update2.sim +++ b/tests/script/tsim/insert/update2.sim @@ -120,7 +120,8 @@ endi if $data01 != 2 then return -1 endi -if $data02 != 1 then +if $data02 != 1 then + print expect 1 , actual $data02 return -1 endi if $data03 != 1 then From 749e7d6cf7707a9dec333538488f7af71a7cdf7f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 17 Jul 2024 00:49:53 +0800 Subject: [PATCH 29/34] fix(stream): return after get value. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 5 ++++- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index da6e029091..e885e2ed6e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -2884,6 +2884,9 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr size_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); SArray* pIndexList = taosArrayInit(numOfTables, sizeof(SBrinBlk)); + if (pIndexList == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } while (1) { // only check here, since the iterate data in memory is very fast. @@ -3382,7 +3385,7 @@ static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) SFileBlockDumpInfo* pDumpInfo = &pStatus->fBlockDumpInfo; int32_t code = getCurrentBlockInfo(pBlockIter, &pBlockInfo); - if (code != TSDB_CODE_SUCCESS) { + if (code == TSDB_CODE_SUCCESS) { pDumpInfo->totalRows = pBlockInfo->numRow; pDumpInfo->rowIndex = ASCENDING_TRAVERSE(pReader->info.order) ? 0 : pBlockInfo->numRow - 1; } else { diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index 1d330d6deb..84f235ab25 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -126,8 +126,8 @@ int32_t getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index, STableBlockScan } int32_t getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, STableBlockScanInfo** pInfo, const char* id) { - *pInfo = tSimpleHashGet(pTableMap, &uid, sizeof(uid)); - if (pInfo == NULL || *pInfo == NULL) { + *pInfo = *(STableBlockScanInfo**)tSimpleHashGet(pTableMap, &uid, sizeof(uid)); + if (pInfo == NULL) { int32_t size = tSimpleHashGetSize(pTableMap); tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id); return TSDB_CODE_INVALID_PARA; From 872cae1386715174946ceba11f0e58587902da15 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 17 Jul 2024 01:08:39 +0800 Subject: [PATCH 30/34] refactor: do some refactor. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index e885e2ed6e..cf2d23cfc8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -898,13 +898,13 @@ static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInf *pInfo = NULL; if (pBlockIter->blockList == NULL) { - return TSDB_CODE_SUCCESS; + return TSDB_CODE_FAILED; } size_t num = TARRAY_SIZE(pBlockIter->blockList); if (num == 0) { ASSERT(pBlockIter->numOfBlocks == num); - return TSDB_CODE_SUCCESS; + return TSDB_CODE_FAILED; } *pInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index); From a62c0fda83aca172d84673bb57966c3867ad7030 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 17 Jul 2024 09:12:42 +0800 Subject: [PATCH 31/34] fix: scheduler UT issues --- source/libs/scheduler/test/schedulerTests.cpp | 190 +++++++++++++++--- 1 file changed, 159 insertions(+), 31 deletions(-) diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 7bae175ba9..f906e3ec3e 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -75,9 +75,7 @@ int32_t schtStartFetch = 0; void schtInitLogFile() { const char *defaultLogFileNamePrefix = "taoslog"; const int32_t maxLogFileNum = 10; - rpcInit(); tsAsyncLog = 0; - rpcInit(); qDebugFlag = 159; strcpy(tsLogDir, TD_LOG_DIR_PATH); @@ -136,8 +134,13 @@ int32_t schtBuildSubmitRspMsg(uint32_t *msize, void **rspMsg) { tEncodeSize(tEncodeSSubmitRsp2, &submitRsp, msgSize, ret); void *msg = taosMemoryCalloc(1, msgSize); + if (NULL == msg) { + return terrno; + } tEncoderInit(&ec, (uint8_t *)msg, msgSize); - tEncodeSSubmitRsp2(&ec, &submitRsp); + if (tEncodeSSubmitRsp2(&ec, &submitRsp) < 0) { + return -1; + } tEncoderClear(&ec); *rspMsg = msg; @@ -152,11 +155,26 @@ void schtBuildQueryDag(SQueryPlan *dag) { dag->queryId = qId; dag->numOfSubplans = 2; dag->pSubplans = nodesMakeList(); + if (NULL == dag->pSubplans) { + return; + } SNodeListNode *scan = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + if (NULL == scan) { + return; + } SNodeListNode *merge = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + if (NULL == merge) { + return; + } SSubplan *scanPlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + if (NULL == scanPlan) { + return; + } SSubplan *mergePlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + if (NULL == mergePlan) { + return; + } scanPlan->id.queryId = qId; scanPlan->id.groupId = 0x0000000000000002; @@ -170,7 +188,13 @@ void schtBuildQueryDag(SQueryPlan *dag) { scanPlan->pChildren = NULL; scanPlan->level = 1; scanPlan->pParents = nodesMakeList(); + if (NULL == scanPlan->pParents) { + return; + } scanPlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); + if (NULL == scanPlan->pNode) { + return; + } scanPlan->msgType = TDMT_SCH_QUERY; mergePlan->id.queryId = qId; @@ -181,21 +205,33 @@ void schtBuildQueryDag(SQueryPlan *dag) { mergePlan->execNode.epSet.numOfEps = 0; mergePlan->pChildren = nodesMakeList(); + if (NULL == mergePlan->pChildren) { + return; + } mergePlan->pParents = NULL; mergePlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_MERGE); + if (NULL == mergePlan->pNode) { + return; + } mergePlan->msgType = TDMT_SCH_QUERY; merge->pNodeList = nodesMakeList(); + if (NULL == merge->pNodeList) { + return; + } scan->pNodeList = nodesMakeList(); + if (NULL == scan->pNodeList) { + return; + } - nodesListAppend(merge->pNodeList, (SNode *)mergePlan); - nodesListAppend(scan->pNodeList, (SNode *)scanPlan); + (void)nodesListAppend(merge->pNodeList, (SNode *)mergePlan); + (void)nodesListAppend(scan->pNodeList, (SNode *)scanPlan); - nodesListAppend(mergePlan->pChildren, (SNode *)scanPlan); - nodesListAppend(scanPlan->pParents, (SNode *)mergePlan); + (void)nodesListAppend(mergePlan->pChildren, (SNode *)scanPlan); + (void)nodesListAppend(scanPlan->pParents, (SNode *)mergePlan); - nodesListAppend(dag->pSubplans, (SNode *)merge); - nodesListAppend(dag->pSubplans, (SNode *)scan); + (void)nodesListAppend(dag->pSubplans, (SNode *)merge); + (void)nodesListAppend(dag->pSubplans, (SNode *)scan); } void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { @@ -205,18 +241,42 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { dag->queryId = qId; dag->numOfSubplans = 2; dag->pSubplans = nodesMakeList(); + if (NULL == dag->pSubplans) { + return; + } SNodeListNode *scan = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + if (NULL == scan) { + return; + } SNodeListNode *merge = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + if (NULL == merge) { + return; + } SSubplan *mergePlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + if (NULL == mergePlan) { + return; + } merge->pNodeList = nodesMakeList(); + if (NULL == merge->pNodeList) { + return; + } scan->pNodeList = nodesMakeList(); + if (NULL == scan->pNodeList) { + return; + } mergePlan->pChildren = nodesMakeList(); + if (NULL == mergePlan->pChildren) { + return; + } for (int32_t i = 0; i < scanPlanNum; ++i) { SSubplan *scanPlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + if (NULL == scanPlan) { + return; + } scanPlan->id.queryId = qId; scanPlan->id.groupId = 0x0000000000000002; scanPlan->id.subplanId = 0x0000000000000003 + i; @@ -233,13 +293,19 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { scanPlan->pChildren = NULL; scanPlan->level = 1; scanPlan->pParents = nodesMakeList(); + if (NULL == scanPlan->pParents) { + return; + } scanPlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); + if (NULL == scanPlan->pNode) { + return; + } scanPlan->msgType = TDMT_SCH_QUERY; - nodesListAppend(scanPlan->pParents, (SNode *)mergePlan); - nodesListAppend(mergePlan->pChildren, (SNode *)scanPlan); + (void)nodesListAppend(scanPlan->pParents, (SNode *)mergePlan); + (void)nodesListAppend(mergePlan->pChildren, (SNode *)scanPlan); - nodesListAppend(scan->pNodeList, (SNode *)scanPlan); + (void)nodesListAppend(scan->pNodeList, (SNode *)scanPlan); } mergePlan->id.queryId = qId; @@ -251,12 +317,15 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { mergePlan->pParents = NULL; mergePlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_MERGE); + if (NULL == mergePlan->pNode) { + return; + } mergePlan->msgType = TDMT_SCH_QUERY; - nodesListAppend(merge->pNodeList, (SNode *)mergePlan); + (void)nodesListAppend(merge->pNodeList, (SNode *)mergePlan); - nodesListAppend(dag->pSubplans, (SNode *)merge); - nodesListAppend(dag->pSubplans, (SNode *)scan); + (void)nodesListAppend(dag->pSubplans, (SNode *)merge); + (void)nodesListAppend(dag->pSubplans, (SNode *)scan); } void schtFreeQueryDag(SQueryPlan *dag) {} @@ -267,10 +336,22 @@ void schtBuildInsertDag(SQueryPlan *dag) { dag->queryId = qId; dag->numOfSubplans = 2; dag->pSubplans = nodesMakeList(); + if (NULL == dag->pSubplans) { + return; + } SNodeListNode *inserta = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + if (NULL == inserta) { + return; + } inserta->pNodeList = nodesMakeList(); + if (NULL == inserta->pNodeList) { + return; + } SSubplan *insertPlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + if (NULL == insertPlan) { + return; + } insertPlan->id.queryId = qId; insertPlan->id.groupId = 0x0000000000000003; @@ -286,13 +367,22 @@ void schtBuildInsertDag(SQueryPlan *dag) { insertPlan->pParents = NULL; insertPlan->pNode = NULL; insertPlan->pDataSink = (SDataSinkNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_INSERT); + if (NULL == insertPlan->pDataSink) { + return; + } ((SDataInserterNode *)insertPlan->pDataSink)->size = 1; ((SDataInserterNode *)insertPlan->pDataSink)->pData = taosMemoryCalloc(1, 1); + if (NULL == ((SDataInserterNode *)insertPlan->pDataSink)->pData) { + return; + } insertPlan->msgType = TDMT_VND_SUBMIT; - nodesListAppend(inserta->pNodeList, (SNode *)insertPlan); + (void)nodesListAppend(inserta->pNodeList, (SNode *)insertPlan); insertPlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); + if (NULL == insertPlan) { + return; + } insertPlan->id.queryId = qId; insertPlan->id.groupId = 0x0000000000000003; @@ -308,22 +398,31 @@ void schtBuildInsertDag(SQueryPlan *dag) { insertPlan->pParents = NULL; insertPlan->pNode = NULL; insertPlan->pDataSink = (SDataSinkNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_INSERT); + if (NULL == insertPlan->pDataSink) { + return; + } ((SDataInserterNode *)insertPlan->pDataSink)->size = 1; ((SDataInserterNode *)insertPlan->pDataSink)->pData = taosMemoryCalloc(1, 1); + if (NULL == ((SDataInserterNode *)insertPlan->pDataSink)->pData) { + return; + } insertPlan->msgType = TDMT_VND_SUBMIT; - nodesListAppend(inserta->pNodeList, (SNode *)insertPlan); + (void)nodesListAppend(inserta->pNodeList, (SNode *)insertPlan); - nodesListAppend(dag->pSubplans, (SNode *)inserta); + (void)nodesListAppend(dag->pSubplans, (SNode *)inserta); } int32_t schtPlanToString(const SSubplan *subplan, char **str, int32_t *len) { *str = (char *)taosMemoryCalloc(1, 20); + if (NULL == *str) { + return -1; + } *len = 20; return 0; } -void schtExecNode(SSubplan *subplan, uint64_t groupId, SQueryNodeAddr *ep) {} +int32_t schtExecNode(SSubplan *subplan, uint64_t groupId, SQueryNodeAddr *ep) { return 0; } void schtRpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *pRid) {} @@ -442,16 +541,16 @@ void *schtSendRsp(void *param) { SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildSubmitRspMsg(&msg.len, &rmsg); + (void)schtBuildSubmitRspMsg(&msg.len, &rmsg); msg.msgType = TDMT_VND_SUBMIT_RSP; msg.pData = rmsg; - schHandleResponseMsg(pJob, task, task->execId, &msg, 0); + (void)schHandleResponseMsg(pJob, task, task->execId, &msg, 0); pIter = taosHashIterate(pJob->execTasks, pIter); } - schReleaseJob(job); + (void)schReleaseJob(job); schtJobDone = true; @@ -472,13 +571,13 @@ void *schtCreateFetchRspThread(void *param) { int32_t code = 0; SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildFetchRspMsg(&msg.len, &rmsg); + (void)schtBuildFetchRspMsg(&msg.len, &rmsg); msg.msgType = TDMT_SCH_MERGE_FETCH_RSP; msg.pData = rmsg; code = schHandleResponseMsg(pJob, pJob->fetchTask, pJob->fetchTask->execId, &msg, 0); - schReleaseJob(job); + (void)schReleaseJob(job); assert(code == 0); return NULL; @@ -496,12 +595,17 @@ void *schtFetchRspThread(void *aa) { taosUsleep(100); param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); - + if (NULL == param) { + return NULL; + } param->queryId = schtQueryId; param->taskId = schtFetchTaskId; int32_t code = 0; SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp)); + if (NULL == rsp) { + return NULL; + } rsp->completed = 1; rsp->numOfRows = 10; @@ -557,12 +661,17 @@ void *schtRunJobThread(void *aa) { schtBuildQueryDag(dag); SArray *qnodeList = taosArrayInit(1, sizeof(SQueryNodeLoad)); - + if (NULL == qnodeList) { + assert(0); + } + SQueryNodeLoad load = {0}; load.addr.epSet.numOfEps = 1; strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); load.addr.epSet.eps[0].port = 6031; - taosArrayPush(qnodeList, &load); + if (NULL == taosArrayPush(qnodeList, &load)) { + assert(0); + } queryDone = 0; @@ -590,16 +699,24 @@ void *schtRunJobThread(void *aa) { } execTasks = taosHashInit(5, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + if (NULL == execTasks) { + assert(0); + } void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; schtFetchTaskId = task->taskId - 1; - taosHashPut(execTasks, &task->taskId, sizeof(task->taskId), task, sizeof(*task)); + if (taosHashPut(execTasks, &task->taskId, sizeof(task->taskId), task, sizeof(*task))) { + assert(0); + } pIter = taosHashIterate(pJob->execTasks, pIter); } param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); + if (NULL == param) { + assert(0); + } param->refId = queryJobRefId; param->queryId = pJob->queryId; @@ -611,7 +728,9 @@ void *schtRunJobThread(void *aa) { SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildQueryRspMsg(&msg.len, &rmsg); + if (schtBuildQueryRspMsg(&msg.len, &rmsg)) { + assert(0); + } msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; @@ -622,6 +741,9 @@ void *schtRunJobThread(void *aa) { } param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); + if (NULL == param) { + assert(0); + } param->refId = queryJobRefId; param->queryId = pJob->queryId; @@ -632,7 +754,9 @@ void *schtRunJobThread(void *aa) { param->taskId = task->taskId - 1; SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildQueryRspMsg(&msg.len, &rmsg); + if (schtBuildQueryRspMsg(&msg.len, &rmsg)) { + assert(0); + } msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; @@ -1110,13 +1234,17 @@ TEST(otherTest, otherCase) { schReleaseJob(0); schFreeRpcCtx(NULL); - ASSERT_EQ(schDumpEpSet(NULL, NULL), TSDB_CODE_SUCCESS); + char* ep = NULL; + ASSERT_EQ(schDumpEpSet(NULL, &ep), TSDB_CODE_SUCCESS); ASSERT_EQ(strcmp(schGetOpStr(SCH_OP_NULL), "NULL"), 0); ASSERT_EQ(strcmp(schGetOpStr((SCH_OP_TYPE)100), "UNKNOWN"), 0); } int main(int argc, char **argv) { schtInitLogFile(); + if (rpcInit()) { + assert(0); + } taosSeedRand(taosGetTimestampSec()); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); From 5057749790944407a2ef3f68d7ef5e7ca815df48 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 17 Jul 2024 10:18:58 +0800 Subject: [PATCH 32/34] fix(stream): fix race condition for dispatch msg. --- source/libs/stream/src/streamDispatch.c | 56 +++++++++++++++---------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 617adaa016..1959180a3e 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -1162,10 +1162,10 @@ void streamClearChkptReadyMsg(SActiveCheckpointInfo* pActiveInfo) { static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId, int32_t downstreamNodeId) { stDebug("s-task:%s destroy dispatch msg:%p", pTask->id.idStr, pTask->msgInfo.pData); - bool delayDispatch = (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER); - clearBufferedDispatchMsg(pTask); - int64_t el = taosGetTimestampMs() - pTask->msgInfo.startTs; + bool delayDispatch = (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER); + + clearBufferedDispatchMsg(pTask); // put data into inputQ of current task is also allowed if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) { @@ -1189,13 +1189,24 @@ static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId return 0; } -static int32_t setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int32_t code, int64_t now, const char* id) { +static bool setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int32_t code, int64_t now, int32_t* pNotRsp, const char* id) { int32_t numOfRsp = 0; bool alreadySet = false; bool updated = false; + bool allRsp = false; + *pNotRsp = 0; taosThreadMutexLock(&pMsgInfo->lock); - for (int32_t j = 0; j < taosArrayGetSize(pMsgInfo->pSendInfo); ++j) { + int32_t numOfDispatchBranch = taosArrayGetSize(pMsgInfo->pSendInfo); + + for(int32_t i = 0; i < numOfDispatchBranch; ++i) { + SDispatchEntry* pEntry = taosArrayGet(pMsgInfo->pSendInfo, i); + if (pEntry->rspTs != -1) { + numOfRsp += 1; + } + } + + for (int32_t j = 0; j < numOfDispatchBranch; ++j) { SDispatchEntry* pEntry = taosArrayGet(pMsgInfo->pSendInfo, j); if (pEntry->nodeId == vgId) { ASSERT(!alreadySet); @@ -1203,18 +1214,20 @@ static int32_t setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int3 pEntry->status = code; alreadySet = true; updated = true; - stDebug("s-task:%s record the rsp recv, ts:%" PRId64 " code:%d, idx:%d", id, now, code, j); - } - - if (pEntry->rspTs != -1) { numOfRsp += 1; + + stDebug("s-task:%s record the rsp recv, ts:%" PRId64 " code:%d, idx:%d, total recv:%d/%d", id, now, code, j, + numOfRsp, numOfDispatchBranch); } } - taosThreadMutexUnlock(&pMsgInfo->lock); - ASSERT(updated); + *pNotRsp = numOfDispatchBranch - numOfRsp; + allRsp = (numOfRsp == numOfDispatchBranch); - return numOfRsp; + taosThreadMutexUnlock(&pMsgInfo->lock); + + ASSERT(updated); + return allRsp; } bool isDispatchRspTimeout(SDispatchEntry* pEntry, int64_t now) { @@ -1240,7 +1253,8 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i int32_t vgId = pTask->pMeta->vgId; SDispatchMsgInfo* pMsgInfo = &pTask->msgInfo; int64_t now = taosGetTimestampMs(); - int32_t totalRsp = 0; + bool allRsp = false; + int32_t notRsp = 0; taosThreadMutexLock(&pMsgInfo->lock); int32_t msgId = pMsgInfo->msgId; @@ -1269,18 +1283,18 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i if (code == TSDB_CODE_STREAM_TASK_NOT_EXIST) { // destination task does not exist, not retry anymore stError("s-task:%s failed to dispatch msg to task:0x%x(vgId:%d), msgId:%d no retry, since task destroyed already", id, pRsp->downstreamTaskId, pRsp->downstreamNodeId, msgId); - totalRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, TSDB_CODE_SUCCESS, now, id); + allRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, TSDB_CODE_SUCCESS, now, ¬Rsp, id); } else { stError("s-task:%s failed to dispatch msgId:%d to task:0x%x(vgId:%d), code:%s, add to retry list", id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, tstrerror(code)); - totalRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, code, now, id); + allRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, code, now, ¬Rsp, id); } } else { // code == 0 if (pRsp->inputStatus == TASK_INPUT_STATUS__BLOCKED) { pTask->inputq.status = TASK_INPUT_STATUS__BLOCKED; // block the input of current task, to push pressure to upstream - totalRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, pRsp->inputStatus, now, id); + allRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, pRsp->inputStatus, now, ¬Rsp, id); stTrace("s-task:%s inputQ of downstream task:0x%x(vgId:%d) is full, wait for retry dispatch", id, pRsp->downstreamTaskId, pRsp->downstreamNodeId); } else { @@ -1292,7 +1306,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i id, pRsp->downstreamTaskId, pRsp->downstreamNodeId); } - totalRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, TSDB_CODE_SUCCESS, now, id); + allRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, TSDB_CODE_SUCCESS, now, ¬Rsp, id); { bool delayDispatch = (pMsgInfo->dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER); @@ -1317,13 +1331,11 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i } } - int32_t notRsp = taosArrayGetSize(pMsgInfo->pSendInfo) - totalRsp; if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { - if (notRsp > 0) { + if (!allRsp) { stDebug( "s-task:%s recv dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%s, " - "waiting " - "for %d rsp", + "waiting for %d rsp", id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, tstrerror(code), notRsp); } else { stDebug( @@ -1337,7 +1349,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i } // all msg rsp already, continue - if (notRsp == 0) { + if (allRsp) { ASSERT(pTask->outputq.status == TASK_OUTPUT_STATUS__WAIT); // we need to re-try send dispatch msg to downstream tasks From acaa85bc7402d345073485e8c6ab07dbafb1954e Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Wed, 17 Jul 2024 12:25:25 +0800 Subject: [PATCH 33/34] doc: refine document about compression --- docs/en/12-taos-sql/03-table.md | 1 + docs/en/12-taos-sql/04-stable.md | 20 ++++-- .../31-compress.md} | 0 docs/en/14-reference/12-config/index.md | 51 ++++++++++++++ docs/zh/12-taos-sql/03-table.md | 1 + docs/zh/12-taos-sql/04-stable.md | 19 +++-- .../31-compress.md} | 1 + docs/zh/14-reference/12-config/index.md | 54 ++++++++++++++- docs/zh/21-tdinternal/07-tsz.md | 69 ------------------- 9 files changed, 136 insertions(+), 80 deletions(-) rename docs/en/{21-tdinternal/08-compress.md => 12-taos-sql/31-compress.md} (100%) rename docs/zh/{21-tdinternal/08-compress.md => 12-taos-sql/31-compress.md} (98%) delete mode 100644 docs/zh/21-tdinternal/07-tsz.md diff --git a/docs/en/12-taos-sql/03-table.md b/docs/en/12-taos-sql/03-table.md index b45064ce08..332db09332 100644 --- a/docs/en/12-taos-sql/03-table.md +++ b/docs/en/12-taos-sql/03-table.md @@ -49,6 +49,7 @@ table_option: { 7. Escape character "\`" can be used to avoid the conflict between table names and reserved keywords, above rules will be bypassed when using escape character on table names, but the upper limit for the name length is still valid. The table names specified using escape character are case sensitive. For example \`aBc\` and \`abc\` are different table names but `abc` and `aBc` are same table names because they are both converted to `abc` internally. Only ASCII visible characters can be used with escape character. +8. For the details of using `ENCODE` and `COMPRESS`, please refer to [Encode and Compress for Column](./compress). **Parameter description** diff --git a/docs/en/12-taos-sql/04-stable.md b/docs/en/12-taos-sql/04-stable.md index ad179a5fe3..ea897904da 100644 --- a/docs/en/12-taos-sql/04-stable.md +++ b/docs/en/12-taos-sql/04-stable.md @@ -13,17 +13,29 @@ create_definition: col_name column_definition column_definition: - type_name + type_name [comment 'string_value'] [PRIMARY KEY] [ENCODE 'encode_type'] [COMPRESS 'compress_type'] [LEVEL 'level_type'] + +table_options: + table_option ... + +table_option: { + COMMENT 'string_value' + | SMA(col_name [, col_name] ...) + | TTL value +} + ``` **More explanations** -- Each supertable can have a maximum of 4096 columns, including tags. The minimum number of columns is 3: a timestamp column used as the key, one tag column, and one data column. -- The TAGS keyword defines the tag columns for the supertable. The following restrictions apply to tag columns: +1. Each supertable can have a maximum of 4096 columns, including tags. The minimum number of columns is 3: a timestamp column used as the key, one tag column, and one data column. +2. Since version 3.3.0.0, besides the timestamp, you can specify another column as primary key using `PRIMARY KEY` keyword, the column specified using `primary key` must be type of integer or varchar. +2. The TAGS keyword defines the tag columns for the supertable. The following restrictions apply to tag columns: - A tag column can use the TIMESTAMP data type, but the values in the column must be fixed numbers. Timestamps including formulae, such as "now + 10s", cannot be stored in a tag column. - The name of a tag column cannot be the same as the name of any other column. - The name of a tag column cannot be a reserved keyword. - Each supertable must contain between 1 and 128 tags. The total length of the TAGS keyword cannot exceed 16 KB. -- For more information about table parameters, see Create a Table. +3. Regarding how to use `ENCODE` and `COMPRESS`, please refer to [Encode and Compress for Column](./compress). +3. For more information about table parameters, see Create a Table. ## View a Supertable diff --git a/docs/en/21-tdinternal/08-compress.md b/docs/en/12-taos-sql/31-compress.md similarity index 100% rename from docs/en/21-tdinternal/08-compress.md rename to docs/en/12-taos-sql/31-compress.md diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index f50551b5de..5020296422 100755 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -729,6 +729,57 @@ The charset that takes effect is UTF-8. | Value Range | -1: none message is compressed; 0: all messages are compressed; N (N>0): messages exceeding N bytes are compressed | | Default | -1 | +### fPrecision + +| Attribute | Description | +| -------- | -------------------------------- | +| Application | Server Only | +| Meaning | Compression precision for float data type | +| Value Range | 0.1 ~ 0.00000001 | +| Default | 0.00000001 | +| Note | The floating value below this setting will be cut off | + +### dPrecision + +| Attribute | Description | +| -------- | -------------------------------- | +| Applicable | Server Only | +| Meaning | Compression precision for double data type | +| Value Range | 0.1 ~ 0.0000000000000001 | +| Default | 0.0000000000000001 | +| Note | The floating value below this setting will be cut off | + +### lossyColumn + +| Attribute | Description | +| -------- | -------------------------------- | +| Applicable | Server Only | +| Meaning | Enable TSZ lossy compression for float and/or double | +| Value Range | float, double | +| Default | none: disable TSZ lossy compression | + +**补充说明** +1. It's only available since 3.2.0.0 version, and can't downgrade to previous version once upgrading to 3.2.0.0 and enabling this parameter +2. TSZ compression algorithm compresses data based on data prediction technique, so it's more suitable for data with specific pattern +3. TSZ compression algorithm may take longer time but it has better compression ratio, so it's suitable when you have enough CPU resources and are more sensitive to disk occupation +4. Example: enable TSZ for both float and double +```shell +lossyColumns float|double +``` +5. After configuring, taosd service needs to restarted. After restarting, if you see the following output in taosd logfile, it means the function has been enabled +```sql + 02/22 10:49:27.607990 00002933 UTL lossyColumns float|double +``` + +### ifAdtFse + +| Attribute | Description | +| -------- | -------------------------------- | +| Applicable | Server Only | +| Meaning | Replace HUFFMAN with FSE in TSZ, FSE is faster when compressing but slower when uncompressing | +| Value Range | 0: Use HUFFMAN, 1: Use FSE | +| Default | 0: Use HUFFMAN | + ## Other Parameters diff --git a/docs/zh/12-taos-sql/03-table.md b/docs/zh/12-taos-sql/03-table.md index f5ff1b83ef..dfac06d6b1 100644 --- a/docs/zh/12-taos-sql/03-table.md +++ b/docs/zh/12-taos-sql/03-table.md @@ -49,6 +49,7 @@ table_option: { 6. 使用数据类型 BINARY/NCHAR/GEOMETRY，需指定其最长的字节数，如 BINARY(20)，表示 20 字节。 7. 为了兼容支持更多形式的表名，TDengine 引入新的转义符 "\`"，可以让表名与关键词不冲突，同时不受限于上述表名称合法性约束检查。但是同样具有长度限制要求。使用转义字符以后，不再对转义字符中的内容进行大小写统一， 例如：\`aBc\` 和 \`abc\` 是不同的表名，但是 abc 和 aBc 是相同的表名。 +8. 关于 `ENCODE` 和 `COMPRESS` 的使用，请参考[按列压缩](./compress) **参数说明** diff --git a/docs/zh/12-taos-sql/04-stable.md b/docs/zh/12-taos-sql/04-stable.md index 9c6de9e220..403ea74d1d 100644 --- a/docs/zh/12-taos-sql/04-stable.md +++ b/docs/zh/12-taos-sql/04-stable.md @@ -13,17 +13,28 @@ create_definition: col_name column_definition column_definition: - type_name + type_name [comment 'string_value'] [PRIMARY KEY] [ENCODE 'encode_type'] [COMPRESS 'compress_type'] [LEVEL 'level_type'] + +table_options: + table_option ... + +table_option: { + COMMENT 'string_value' + | SMA(col_name [, col_name] ...) + | TTL value +} ``` **使用说明** -- 超级表中列的最大个数为 4096，需要注意，这里的 4096 是包含 TAG 列在内的，最小个数为 3，包含一个时间戳主键、一个 TAG 列和一个数据列。 -- TAGS语法指定超级表的标签列，标签列需要遵循以下约定： +1. 超级表中列的最大个数为 4096，需要注意，这里的 4096 是包含 TAG 列在内的，最小个数为 3，包含一个时间戳主键、一个 TAG 列和一个数据列。 +2. 除时间戳主键列之外，还可以通过 PRIMARY KEY 关键字指定第二列为额外的主键列。被指定为主键列的第二列必须为整型或字符串类型（varchar） +3. TAGS语法指定超级表的标签列，标签列需要遵循以下约定： - TAGS 中的 TIMESTAMP 列写入数据时需要提供给定值，而暂不支持四则运算，例如 NOW + 10s 这类表达式。 - TAGS 列名不能与其他列名相同。 - TAGS 列名不能为预留关键字。 - TAGS 最多允许 128 个，至少 1 个，总长度不超过 16 KB。 -- 关于表参数的详细说明，参见 CREATE TABLE 中的介绍。 +4. 关于 `ENCODE` 和 `COMPRESS` 的使用，请参考 [按列压缩](./compress) +5. 关于 table_option 中的参数说明，请参考 [建表 SQL 说明](./table) ## 查看超级表 diff --git a/docs/zh/21-tdinternal/08-compress.md b/docs/zh/12-taos-sql/31-compress.md similarity index 98% rename from docs/zh/21-tdinternal/08-compress.md rename to docs/zh/12-taos-sql/31-compress.md index dfec63c212..0f2b260832 100644 --- a/docs/zh/21-tdinternal/08-compress.md +++ b/docs/zh/12-taos-sql/31-compress.md @@ -1,5 +1,6 @@ --- title: 可配置压缩算法 +sidebar_label: 可配置压缩 description: 可配置压缩算法 --- diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 814b543859..c3d23d624d 100755 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -784,6 +784,57 @@ charset 的有效值是 UTF-8。 | 取值范围 | -1: 所有消息都不压缩; 0: 所有消息都压缩; N (N>0): 只有大于 N 个字节的消息才压缩 | | 缺省值 | -1 | +### fPrecision +FLOAT 类型压缩精度控制： + +| 属性 | 说明 | +| -------- | -------------------------------- | +| 适用范围 | 服务器端 | +| 含义 | 设置 float 类型浮点数压缩精度 | +| 取值范围 | 0.1 ~ 0.00000001 | +| 缺省值 | 0.00000001 | +| 补充说明 | 小于此值的浮点数尾数部分将被截取 | + +### dPrecision +| 属性 | 说明 | +| -------- | -------------------------------- | +| 适用范围 | 服务器端 | +| 含义 | 设置 double 类型浮点数压缩精度 | +| 取值范围 | 0.1 ~ 0.0000000000000001 | +| 缺省值 | 0.0000000000000001 | +| 补充说明 | 小于此值的浮点数尾数部分将被截取 | + +### lossyColumn + +| 属性 | 说明 | +| -------- | -------------------------------- | +| 适用范围 | 服务器端 | +| 含义 | 对 float 和/或 double 类型启用 TSZ 有损压缩 | +| 取值范围 | float, double | +| 缺省值 | none：表示关闭无损压缩 | + +**补充说明** +1. 在 3.2.0.0 及以后版本生效，启用该参数后不能回退到升级前的版本 +2. TSZ 压缩算法是通过数据预测技术完成的压缩，所以更适合有规律变化的数据 +3. TSZ 压缩时间会更长一些，如果您的服务器 CPU 空闲多，存储空间小的情况下适合选用 +4. 示例：对 float 和 double 类型都启用有损压缩 +```shell +lossyColumns float|double +``` +5. 配置需重启服务生效，重启如果在 taosd 日志中看到以下内容，表明配置已生效： +```sql + 02/22 10:49:27.607990 00002933 UTL lossyColumns float|double +``` + +### ifAdtFse + +| 属性 | 说明 | +| -------- | -------------------------------- | +| 适用范围 | 服务器端 | +| 含义 | 在启用 TSZ 有损压缩时，使用 FSE 算法替换 HUFFMAN 算法， FSE 算法压缩速度更快，但解压稍慢，追求压缩速度可选用此算法 | +| 取值范围 | 0：关闭 1：打开 | +| 缺省值 | 0：关闭 | + ## 3.0 中有效的配置参数列表 | # | **参数** | **适用于 2.X ** | **适用于 3.0 ** | 3.0 版本的当前行为 | @@ -922,8 +973,5 @@ charset 的有效值是 UTF-8。 | 73 | probeSeconds | 是 | 否 | 3.0 行为未知 | | 74 | probeKillSeconds | 是 | 否 | 3.0 行为未知 | | 75 | probeInterval | 是 | 否 | 3.0 行为未知 | -| 76 | lossyColumns | 是 | 否 | 3.0 行为未知 | -| 77 | fPrecision | 是 | 否 | 3.0 行为未知 | -| 78 | dPrecision | 是 | 否 | 3.0 行为未知 | | 79 | maxRange | 是 | 否 | 3.0 行为未知 | | 80 | range | 是 | 否 | 3.0 行为未知 | diff --git a/docs/zh/21-tdinternal/07-tsz.md b/docs/zh/21-tdinternal/07-tsz.md deleted file mode 100644 index db1a340ab8..0000000000 --- a/docs/zh/21-tdinternal/07-tsz.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: TSZ 压缩算法 -description: TDengine 对浮点数进行高效压缩的算法 ---- - -TSZ 压缩算法是 TDengine 为浮点数据类型提供的可选压缩算法，可以实现浮点数有损至无损全状态压缩，相比默认压缩算法， TSZ 压缩算法压缩率更高，即使切至无损状态，压缩率也会比默认压缩高一倍。 - -## 适合场景 - -- TSZ 压缩算法是通过数据预测技术完成的压缩，所以更适合有规律变化的数据 -- TSZ 压缩时间会更长一些，如果您的服务器 CPU 空闲多，存储空间小的情况下适合选用 - -## 使用步骤 -- TDengine 支持版本为 3.2.0.0 或以上 -- 开启选项 - 在 taos.cfg 配置中增加以下内容，即可开启 TSZ 压缩算法，功能打开后，会替换默认算法。 - 以下表示字段类型是 float 及 double 类型都使用此压缩算法，也可以单独只配置一个 - -```sql - lossyColumns float|double -``` - -- 配置需重启服务生效 -- Taosd 日志输出以下内容，表明功能已生效： - -```sql - 02/22 10:49:27.607990 00002933 UTL lossyColumns float|double -``` - -## 配置参数 - -### fPrecision -FLOAT 类型精度控制： - -| 属性 | 说明 | -| -------- | -------------------------------- | -| 适用范围 | 服务器端 | -| 含义 | 设置 float 类型浮点数压缩精度 | -| 取值范围 | 0.1 ~ 0.00000001 | -| 缺省值 | 0.00000001 | -| 补充说明 | 小于此值的浮点数尾数部分将被截取 | - - - -### dPrecision -DOUBLE 类型精度控制： - -| 属性 | 说明 | -| -------- | -------------------------------- | -| 适用范围 | 服务器端 | -| 含义 | 设置 double 类型浮点数压缩精度 | -| 取值范围 | 0.1 ~ 0.0000000000000001 | -| 缺省值 | 0.0000000000000001 | -| 补充说明 | 小于此值的浮点数尾数部分将被截取 | - - -### ifAdtFse -TSZ 压缩中可选择的算法 FSE，默认为 HUFFMAN： - -| 属性 | 说明 | -| -------- | -------------------------------- | -| 适用范围 | 服务器端 | -| 含义 | 使用 FSE 算法替换 HUFFMAN 算法， FSE 算法压缩速度更快，但解压稍慢，追求压缩速度可选用此算法 | -| 取值范围 | 0：关闭 1：打开 | -| 缺省值 | 0：关闭 | - - -## 注意事项 -- 打开 TSZ 后生成的存储数据格式，回退至 3.2.0.0 之前的版本，数据将不能被识别 From 573e68fdc8af3fb0e8b90254c754bb3e3086d754 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Wed, 17 Jul 2024 12:32:19 +0800 Subject: [PATCH 34/34] docs: correct broken links --- docs/en/12-taos-sql/03-table.md | 2 +- docs/en/12-taos-sql/04-stable.md | 4 ++-- docs/zh/12-taos-sql/03-table.md | 2 +- docs/zh/12-taos-sql/04-stable.md | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/en/12-taos-sql/03-table.md b/docs/en/12-taos-sql/03-table.md index 332db09332..47a56d3175 100644 --- a/docs/en/12-taos-sql/03-table.md +++ b/docs/en/12-taos-sql/03-table.md @@ -49,7 +49,7 @@ table_option: { 7. Escape character "\`" can be used to avoid the conflict between table names and reserved keywords, above rules will be bypassed when using escape character on table names, but the upper limit for the name length is still valid. The table names specified using escape character are case sensitive. For example \`aBc\` and \`abc\` are different table names but `abc` and `aBc` are same table names because they are both converted to `abc` internally. Only ASCII visible characters can be used with escape character. -8. For the details of using `ENCODE` and `COMPRESS`, please refer to [Encode and Compress for Column](./compress). +8. For the details of using `ENCODE` and `COMPRESS`, please refer to [Encode and Compress for Column](../compress). **Parameter description** diff --git a/docs/en/12-taos-sql/04-stable.md b/docs/en/12-taos-sql/04-stable.md index ea897904da..e7101eaba6 100644 --- a/docs/en/12-taos-sql/04-stable.md +++ b/docs/en/12-taos-sql/04-stable.md @@ -34,8 +34,8 @@ table_option: { - The name of a tag column cannot be the same as the name of any other column. - The name of a tag column cannot be a reserved keyword. - Each supertable must contain between 1 and 128 tags. The total length of the TAGS keyword cannot exceed 16 KB. -3. Regarding how to use `ENCODE` and `COMPRESS`, please refer to [Encode and Compress for Column](./compress). -3. For more information about table parameters, see Create a Table. +3. Regarding how to use `ENCODE` and `COMPRESS`, please refer to [Encode and Compress for Column](../compress). +3. For more information about table parameters, see [Create a Table](../table). ## View a Supertable diff --git a/docs/zh/12-taos-sql/03-table.md b/docs/zh/12-taos-sql/03-table.md index dfac06d6b1..f665aa2277 100644 --- a/docs/zh/12-taos-sql/03-table.md +++ b/docs/zh/12-taos-sql/03-table.md @@ -49,7 +49,7 @@ table_option: { 6. 使用数据类型 BINARY/NCHAR/GEOMETRY，需指定其最长的字节数，如 BINARY(20)，表示 20 字节。 7. 为了兼容支持更多形式的表名，TDengine 引入新的转义符 "\`"，可以让表名与关键词不冲突，同时不受限于上述表名称合法性约束检查。但是同样具有长度限制要求。使用转义字符以后，不再对转义字符中的内容进行大小写统一， 例如：\`aBc\` 和 \`abc\` 是不同的表名，但是 abc 和 aBc 是相同的表名。 -8. 关于 `ENCODE` 和 `COMPRESS` 的使用，请参考[按列压缩](./compress) +8. 关于 `ENCODE` 和 `COMPRESS` 的使用，请参考[按列压缩](../compress) **参数说明** diff --git a/docs/zh/12-taos-sql/04-stable.md b/docs/zh/12-taos-sql/04-stable.md index 403ea74d1d..5ad2f4a19d 100644 --- a/docs/zh/12-taos-sql/04-stable.md +++ b/docs/zh/12-taos-sql/04-stable.md @@ -33,8 +33,8 @@ table_option: { - TAGS 列名不能与其他列名相同。 - TAGS 列名不能为预留关键字。 - TAGS 最多允许 128 个，至少 1 个，总长度不超过 16 KB。 -4. 关于 `ENCODE` 和 `COMPRESS` 的使用，请参考 [按列压缩](./compress) -5. 关于 table_option 中的参数说明，请参考 [建表 SQL 说明](./table) +4. 关于 `ENCODE` 和 `COMPRESS` 的使用，请参考 [按列压缩](../compress) +5. 关于 table_option 中的参数说明，请参考 [建表 SQL 说明](../table) ## 查看超级表