From ffe2973ade468cef70cab78350ae6a9eaf8d92a5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Jul 2024 13:19:01 +0800 Subject: [PATCH 01/43] refactor return code --- source/dnode/vnode/src/inc/tsdb.h | 3 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 8 +- source/dnode/vnode/src/tsdb/tsdbSnapInfo.c | 199 ++++++++++-------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 83 +++++--- source/libs/sync/src/syncUtil.c | 16 +- 5 files changed, 175 insertions(+), 134 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 3d26104722..d496853cd5 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -226,7 +226,7 @@ void tMapDataGetItemByIdx(SMapData *pMapData, int32_t idx, void *pItem, int32 int32_t tMapDataSearch(SMapData *pMapData, void *pSearchItem, int32_t (*tGetItemFn)(uint8_t *, void *), int32_t (*tItemCmprFn)(const void *, const void *), void *pItem); int32_t tPutMapData(uint8_t *p, SMapData *pMapData); -int32_t tGetMapData(uint8_t *p, SMapData *pMapData); +int32_t tGetMapData(uint8_t *p, SMapData *pMapData, int32_t *decodedSize); int32_t tMapDataToArray(SMapData *pMapData, int32_t itemSize, int32_t (*tGetItemFn)(uint8_t *, void *), SArray **ppArray); // other @@ -706,7 +706,6 @@ typedef TARRAY2(STsdbFSetPartition *) STsdbFSetPartList; STsdbFSetPartList *tsdbFSetPartListCreate(); void tsdbFSetPartListDestroy(STsdbFSetPartList **ppList); -int32_t tSerializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList); int32_t tDeserializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList); int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList *pList, TFileSetRangeArray **ppRanges); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 932bf2d92c..dfc7dd8c7c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -803,11 +803,9 @@ int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *m if (code) goto _err; // decode - int64_t n = tGetMapData(pReader->aBuf[0], mDataBlk); - if (n < 0) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } + int32_t n; + code = tGetMapData(pReader->aBuf[0], mDataBlk, &n); + if (code) goto _err; ASSERT(n == size); return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c index bc7bf5449e..f515cd5352 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c @@ -81,7 +81,7 @@ static int32_t tsdbFTypeToFRangeType(tsdb_ftype_t ftype) { static int32_t tsdbTFileSetToFSetPartition(STFileSet* fset, STsdbFSetPartition** ppSP) { STsdbFSetPartition* p = tsdbFSetPartitionCreate(); if (p == NULL) { - goto _err; + return terrno; } p->fid = fset->fid; @@ -134,10 +134,6 @@ static int32_t tsdbTFileSetToFSetPartition(STFileSet* fset, STsdbFSetPartition** } ppSP[0] = p; return 0; - -_err: - tsdbFSetPartitionClear(&p); - return -1; } // fset partition list @@ -160,9 +156,11 @@ void tsdbFSetPartListDestroy(STsdbFSetPartList** ppList) { } int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList* pList, TFileSetRangeArray** ppRanges) { + int32_t code = 0; + TFileSetRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TFileSetRangeArray)); if (pDiff == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } TARRAY2_INIT(pDiff); @@ -171,7 +169,7 @@ int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList* pList, TFileSetRangeArray TARRAY2_FOREACH(pList, part) { STFileSetRange* r = taosMemoryCalloc(1, sizeof(STFileSetRange)); if (r == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } int64_t maxVerValid = -1; @@ -202,7 +200,7 @@ _err: if (pDiff) { tsdbTFileSetRangeArrayDestroy(&pDiff); } - return -1; + return code; } // serialization @@ -235,96 +233,103 @@ int32_t tTsdbFSetPartListDataLenCalc(STsdbFSetPartList* pList) { return datLen; } -int32_t tSerializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartList* pList) { +static int32_t tSerializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartList* pList, int32_t* encodeSize) { SEncoder encoder = {0}; + int8_t reserved8 = 0; + int16_t reserved16 = 0; + int64_t reserved64 = 0; + int8_t msgVer = TSDB_SNAP_MSG_VER; + int32_t len = TARRAY2_SIZE(pList); + int32_t code = 0; + tEncoderInit(&encoder, buf, bufLen); - - int8_t reserved8 = 0; - int16_t reserved16 = 0; - int64_t reserved64 = 0; - - int8_t msgVer = TSDB_SNAP_MSG_VER; - int32_t len = TARRAY2_SIZE(pList); - - if (tStartEncode(&encoder) < 0) goto _err; - if (tEncodeI8(&encoder, msgVer) < 0) goto _err; - if (tEncodeI32(&encoder, len) < 0) goto _err; + if ((code = tStartEncode(&encoder))) goto _exit; + if ((code = tEncodeI8(&encoder, msgVer))) goto _exit; + if ((code = tEncodeI32(&encoder, len))) goto _exit; for (int32_t u = 0; u < len; u++) { STsdbFSetPartition* p = TARRAY2_GET(pList, u); - if (tEncodeI64(&encoder, p->fid) < 0) goto _err; - if (tEncodeI8(&encoder, p->stat) < 0) goto _err; - if (tEncodeI8(&encoder, reserved8) < 0) goto _err; - if (tEncodeI16(&encoder, reserved16) < 0) goto _err; + if ((code = tEncodeI64(&encoder, p->fid))) goto _exit; + if ((code = tEncodeI8(&encoder, p->stat))) goto _exit; + if ((code = tEncodeI8(&encoder, reserved8))) goto _exit; + if ((code = tEncodeI16(&encoder, reserved16))) goto _exit; int32_t typMax = TSDB_FSET_RANGE_TYP_MAX; - if (tEncodeI32(&encoder, typMax) < 0) goto _err; + if ((code = tEncodeI32(&encoder, typMax))) goto _exit; for (int32_t i = 0; i < typMax; i++) { SVerRangeList* iList = &p->verRanges[i]; int32_t iLen = TARRAY2_SIZE(iList); - if (tEncodeI32(&encoder, iLen) < 0) goto _err; + if ((code = tEncodeI32(&encoder, iLen))) goto _exit; for (int32_t j = 0; j < iLen; j++) { SVersionRange r = TARRAY2_GET(iList, j); - if (tEncodeI64(&encoder, r.minVer) < 0) goto _err; - if (tEncodeI64(&encoder, r.maxVer) < 0) goto _err; - if (tEncodeI64(&encoder, reserved64) < 0) goto _err; + if ((code = tEncodeI64(&encoder, r.minVer))) goto _exit; + if ((code = tEncodeI64(&encoder, r.maxVer))) goto _exit; + if ((code = tEncodeI64(&encoder, reserved64))) goto _exit; } } } tEndEncode(&encoder); - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -_err: + if (encodeSize) { + encodeSize[0] = encoder.pos; + } + +_exit: tEncoderClear(&encoder); - return -1; + return code; } int32_t tDeserializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartList* pList) { SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - - int8_t reserved8 = 0; - int16_t reserved16 = 0; - int64_t reserved64 = 0; + int8_t reserved8 = 0; + int16_t reserved16 = 0; + int64_t reserved64 = 0; + int32_t code = 0; STsdbFSetPartition* p = NULL; + tDecoderInit(&decoder, buf, bufLen); int8_t msgVer = 0; int32_t len = 0; - if (tStartDecode(&decoder) < 0) goto _err; - if (tDecodeI8(&decoder, &msgVer) < 0) goto _err; - if (msgVer != TSDB_SNAP_MSG_VER) goto _err; - if (tDecodeI32(&decoder, &len) < 0) goto _err; + if ((code = tStartDecode(&decoder))) goto _err; + if ((code = tDecodeI8(&decoder, &msgVer))) goto _err; + if (msgVer != TSDB_SNAP_MSG_VER) { + code = TSDB_CODE_INVALID_MSG; + goto _err; + } + if ((code = tDecodeI32(&decoder, &len))) goto _err; for (int32_t u = 0; u < len; u++) { p = tsdbFSetPartitionCreate(); - if (p == NULL) goto _err; - if (tDecodeI64(&decoder, &p->fid) < 0) goto _err; - if (tDecodeI8(&decoder, &p->stat) < 0) goto _err; - if (tDecodeI8(&decoder, &reserved8) < 0) goto _err; - if (tDecodeI16(&decoder, &reserved16) < 0) goto _err; + if (p == NULL) { + code = terrno; + goto _err; + } + + if ((code = tDecodeI64(&decoder, &p->fid))) goto _err; + if ((code = tDecodeI8(&decoder, &p->stat))) goto _err; + if ((code = tDecodeI8(&decoder, &reserved8))) goto _err; + if ((code = tDecodeI16(&decoder, &reserved16))) goto _err; int32_t typMax = 0; - if (tDecodeI32(&decoder, &typMax) < 0) goto _err; + if ((code = tDecodeI32(&decoder, &typMax))) goto _err; for (int32_t i = 0; i < typMax; i++) { SVerRangeList* iList = &p->verRanges[i]; int32_t iLen = 0; - if (tDecodeI32(&decoder, &iLen) < 0) goto _err; + if ((code = tDecodeI32(&decoder, &iLen))) goto _err; for (int32_t j = 0; j < iLen; j++) { SVersionRange r = {0}; - if (tDecodeI64(&decoder, &r.minVer) < 0) goto _err; - if (tDecodeI64(&decoder, &r.maxVer) < 0) goto _err; - if (tDecodeI64(&decoder, &reserved64) < 0) goto _err; - if (TARRAY2_APPEND(iList, r)) goto _err; + if ((code = tDecodeI64(&decoder, &r.minVer))) goto _err; + if ((code = tDecodeI64(&decoder, &r.maxVer))) goto _err; + if ((code = tDecodeI64(&decoder, &reserved64))) goto _err; + if ((code = TARRAY2_APPEND(iList, r))) goto _err; } } - if (TARRAY2_APPEND(pList, p)) goto _err; + if ((code = TARRAY2_APPEND(pList, p))) goto _err; p = NULL; } @@ -337,7 +342,7 @@ _err: tsdbFSetPartitionClear(&p); } tDecoderClear(&decoder); - return -1; + return code; } // fs state @@ -352,8 +357,9 @@ static STsdbFSetPartList* tsdbSnapGetFSetPartList(STFileSystem* fs) { STFileSet* fset; TARRAY2_FOREACH(fs->fSetArr, fset) { STsdbFSetPartition* pItem = NULL; - if (tsdbTFileSetToFSetPartition(fset, &pItem) < 0) { - code = -1; + code = tsdbTFileSetToFSetPartition(fset, &pItem); + if (code) { + terrno = code; break; } ASSERT(pItem != NULL); @@ -404,7 +410,9 @@ static int32_t tsdbPartitionInfoInit(SVnode* pVnode, STsdbPartitionInfo* pInfo) for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) { STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, j); pInfo->pLists[j] = tsdbSnapGetFSetPartList(pTsdb->pFS); - if (pInfo->pLists[j] == NULL) return -1; + if (pInfo->pLists[j] == NULL) { + return terrno; + } } return 0; } @@ -432,9 +440,10 @@ static int32_t tsdbPartitionInfoSerialize(STsdbPartitionInfo* pInfo, uint8_t* bu SSyncTLV* pSubHead = (void*)((char*)buf + offset); int32_t valOffset = offset + sizeof(*pSubHead); ASSERT(pSubHead->val == (char*)buf + valOffset); - if ((tlen = tSerializeTsdbFSetPartList(pSubHead->val, bufLen - valOffset, pInfo->pLists[j])) < 0) { + int32_t code = tSerializeTsdbFSetPartList(pSubHead->val, bufLen - valOffset, pInfo->pLists[j], &tlen); + if (code) { tsdbError("vgId:%d, failed to serialize fset partition list of tsdb %d since %s", pInfo->vgId, j, terrstr()); - return -1; + return code; } pSubHead->typ = pInfo->subTyps[j]; pSubHead->len = tlen; @@ -460,17 +469,18 @@ static int32_t tTsdbRepOptsDataLenCalc(STsdbRepOpts* pInfo) { } int32_t tSerializeTsdbRepOpts(void* buf, int32_t bufLen, STsdbRepOpts* pOpts) { + int32_t code = 0; SEncoder encoder = {0}; + int64_t reserved64 = 0; + int8_t msgVer = TSDB_SNAP_MSG_VER; + tEncoderInit(&encoder, buf, bufLen); - int64_t reserved64 = 0; - int8_t msgVer = TSDB_SNAP_MSG_VER; - - if (tStartEncode(&encoder) < 0) goto _err; - if (tEncodeI8(&encoder, msgVer) < 0) goto _err; + if ((code = tStartEncode(&encoder))) goto _err; + if ((code = tEncodeI8(&encoder, msgVer))) goto _err; int16_t format = pOpts->format; - if (tEncodeI16(&encoder, format) < 0) goto _err; - if (tEncodeI64(&encoder, reserved64) < 0) goto _err; + if ((code = tEncodeI16(&encoder, format))) goto _err; + if ((code = tEncodeI64(&encoder, reserved64))) goto _err; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -479,23 +489,24 @@ int32_t tSerializeTsdbRepOpts(void* buf, int32_t bufLen, STsdbRepOpts* pOpts) { _err: tEncoderClear(&encoder); - return -1; + return code; } int32_t tDeserializeTsdbRepOpts(void* buf, int32_t bufLen, STsdbRepOpts* pOpts) { + int32_t code; SDecoder decoder = {0}; + int64_t reserved64 = 0; + int8_t msgVer = 0; + tDecoderInit(&decoder, buf, bufLen); - int64_t reserved64 = 0; - int8_t msgVer = 0; - - if (tStartDecode(&decoder) < 0) goto _err; - if (tDecodeI8(&decoder, &msgVer) < 0) goto _err; + if ((code = tStartDecode(&decoder))) goto _err; + if ((code = tDecodeI8(&decoder, &msgVer))) goto _err; if (msgVer != TSDB_SNAP_MSG_VER) goto _err; int16_t format = 0; - if (tDecodeI16(&decoder, &format) < 0) goto _err; + if ((code = tDecodeI16(&decoder, &format))) goto _err; pOpts->format = format; - if (tDecodeI64(&decoder, &reserved64) < 0) goto _err; + if ((code = tDecodeI64(&decoder, &reserved64))) goto _err; tEndDecode(&decoder); tDecoderClear(&decoder); @@ -503,7 +514,7 @@ int32_t tDeserializeTsdbRepOpts(void* buf, int32_t bufLen, STsdbRepOpts* pOpts) _err: tDecoderClear(&decoder); - return -1; + return code; } static int32_t tsdbRepOptsEstSize(STsdbRepOpts* pOpts) { @@ -518,7 +529,7 @@ static int32_t tsdbRepOptsSerialize(STsdbRepOpts* pOpts, void* buf, int32_t bufL int32_t offset = 0; int32_t tlen = 0; if ((tlen = tSerializeTsdbRepOpts(pSubHead->val, bufLen, pOpts)) < 0) { - return -1; + return tlen; } pSubHead->typ = SNAP_DATA_RAW; pSubHead->len = tlen; @@ -528,8 +539,10 @@ static int32_t tsdbRepOptsSerialize(STsdbRepOpts* pOpts, void* buf, int32_t bufL // snap info static int32_t tsdbSnapPrepDealWithSnapInfo(SVnode* pVnode, SSnapshot* pSnap, STsdbRepOpts* pInfo) { - if (!pSnap->data) return 0; - int32_t code = -1; + if (!pSnap->data) { + return 0; + } + int32_t code = 0; SSyncTLV* pHead = (void*)pSnap->data; int32_t offset = 0; @@ -546,30 +559,30 @@ static int32_t tsdbSnapPrepDealWithSnapInfo(SVnode* pVnode, SSnapshot* pSnap, ST case SNAP_DATA_RSMA2: { } break; case SNAP_DATA_RAW: { - if (tDeserializeTsdbRepOpts(buf, bufLen, pInfo) < 0) { - terrno = TSDB_CODE_INVALID_DATA_FMT; + code = tDeserializeTsdbRepOpts(buf, bufLen, pInfo); + if (code < 0) { tsdbError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr()); - goto _out; + return code; } } break; default: + code = TSDB_CODE_INVALID_MSG; tsdbError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), pField->typ); - goto _out; + return code; } } - code = 0; -_out: return code; } int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) { ASSERT(pSnap->type == TDMT_SYNC_PREP_SNAPSHOT || pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY); STsdbPartitionInfo partitionInfo = {0}; - int code = -1; + int code = 0; STsdbPartitionInfo* pInfo = &partitionInfo; - if (tsdbPartitionInfoInit(pVnode, pInfo) != 0) { + code = tsdbPartitionInfoInit(pVnode, pInfo); + if (code) { goto _out; } @@ -577,7 +590,7 @@ int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) { STsdbRepOpts opts = {.format = TSDB_SNAP_REP_FMT_RAW}; if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) { STsdbRepOpts leaderOpts = {0}; - if (tsdbSnapPrepDealWithSnapInfo(pVnode, pSnap, &leaderOpts) < 0) { + if ((code = tsdbSnapPrepDealWithSnapInfo(pVnode, pSnap, &leaderOpts)) < 0) { tsdbError("vgId:%d, failed to deal with snap info for reply since %s", TD_VID(pVnode), terrstr()); goto _out; } @@ -589,7 +602,7 @@ int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) { int32_t bufLen = headLen; bufLen += tsdbPartitionInfoEstSize(pInfo); bufLen += tsdbRepOptsEstSize(&opts); - if (syncSnapInfoDataRealloc(pSnap, bufLen) != 0) { + if ((code = syncSnapInfoDataRealloc(pSnap, bufLen)) != 0) { tsdbError("vgId:%d, failed to realloc memory for data of snap info. bytes:%d", TD_VID(pVnode), bufLen); goto _out; } @@ -599,7 +612,8 @@ int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) { int32_t offset = headLen; int32_t tlen = 0; - if ((tlen = tsdbPartitionInfoSerialize(pInfo, buf + offset, bufLen - offset)) < 0) { + if ((tlen = tsdbPartitionInfoSerialize(pInfo, (uint8_t*)(buf + offset), bufLen - offset)) < 0) { + code = tlen; tsdbError("vgId:%d, failed to serialize tsdb partition info since %s", TD_VID(pVnode), terrstr()); goto _out; } @@ -607,6 +621,7 @@ int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) { ASSERT(offset <= bufLen); if ((tlen = tsdbRepOptsSerialize(&opts, buf + offset, bufLen - offset)) < 0) { + code = tlen; tsdbError("vgId:%d, failed to serialize tsdb rep opts since %s", TD_VID(pVnode), terrstr()); goto _out; } @@ -620,7 +635,7 @@ int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) { tsdbInfo("vgId:%d, tsdb snap info prepared. type:%s, val length:%d", TD_VID(pVnode), TMSG_INFO(pHead->typ), pHead->len); - code = 0; + _out: tsdbPartitionInfoClear(pInfo); return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 58075cf0ac..66db7baab9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -152,7 +152,7 @@ int32_t tPutMapData(uint8_t *p, SMapData *pMapData) { } #endif -int32_t tGetMapData(uint8_t *p, SMapData *pMapData) { +int32_t tGetMapData(uint8_t *p, SMapData *pMapData, int32_t *decodedSize) { int32_t n = 0; int32_t offset; @@ -160,7 +160,9 @@ int32_t tGetMapData(uint8_t *p, SMapData *pMapData) { n += tGetI32v(p + n, &pMapData->nItem); if (pMapData->nItem) { - if (tRealloc((uint8_t **)&pMapData->aOffset, sizeof(int32_t) * pMapData->nItem)) return -1; + if (tRealloc((uint8_t **)&pMapData->aOffset, sizeof(int32_t) * pMapData->nItem)) { + return TSDB_CODE_OUT_OF_MEMORY; + } int32_t lOffset = 0; for (int32_t iItem = 0; iItem < pMapData->nItem; iItem++) { @@ -170,12 +172,18 @@ int32_t tGetMapData(uint8_t *p, SMapData *pMapData) { } n += tGetI32v(p + n, &pMapData->nData); - if (tRealloc(&pMapData->pData, pMapData->nData)) return -1; + if (tRealloc(&pMapData->pData, pMapData->nData)) { + return TSDB_CODE_OUT_OF_MEMORY; + } memcpy(pMapData->pData, p + n, pMapData->nData); n += pMapData->nData; } - return n; + if (decodedSize) { + *decodedSize = n; + } + + return 0; } #ifdef BUILD_NO_CALL @@ -680,20 +688,17 @@ int32_t tsdbRowCompareWithoutVersion(const void *p1, const void *p2) { // STSDBRowIter ====================================================== int32_t tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { - int32_t code = 0; - pIter->pRow = pRow; if (pRow->type == TSDBROW_ROW_FMT) { - code = tRowIterOpen(pRow->pTSRow, pTSchema, &pIter->pIter); - if (code) goto _exit; + int32_t code = tRowIterOpen(pRow->pTSRow, pTSchema, &pIter->pIter); + if (code) return code; } else if (pRow->type == TSDBROW_COL_FMT) { pIter->iColData = 0; } else { ASSERT(0); } -_exit: - return code; + return 0; } void tsdbRowClose(STSDBRowIter *pIter) { @@ -760,7 +765,9 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) --iCol; continue; } else if (pTSchema->columns[jCol].colId > pTColumn->colId) { - taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type)); + if (taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type)) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } continue; } @@ -780,14 +787,15 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) } if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - return code; + return TSDB_CODE_OUT_OF_MEMORY; } } for (; iCol < pMerger->pTSchema->numOfCols; ++iCol) { pTColumn = &pMerger->pTSchema->columns[iCol]; - taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type)); + if (taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type)) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } pMerger->version = key.version; @@ -962,8 +970,7 @@ _exit: */ // delete skyline ====================================================== -static int32_t tsdbMergeSkyline(SArray *pSkyline1, SArray *pSkyline2, SArray *pSkyline) { - int32_t code = 0; +static void tsdbMergeSkyline(SArray *pSkyline1, SArray *pSkyline2, SArray *pSkyline) { int32_t i1 = 0; int32_t n1 = taosArrayGetSize(pSkyline1); int32_t i2 = 0; @@ -1017,7 +1024,6 @@ static int32_t tsdbMergeSkyline(SArray *pSkyline1, SArray *pSkyline2, SArray *pS } pSkyline->size = TARRAY_ELEM_IDX(pSkyline, pItem); - return code; } int32_t tsdbBuildDeleteSkylineImpl(SArray *aSkyline, int32_t sidx, int32_t eidx, SArray *pSkyline) { @@ -1029,8 +1035,13 @@ int32_t tsdbBuildDeleteSkylineImpl(SArray *aSkyline, int32_t sidx, int32_t eidx, if (sidx == eidx) { TSDBKEY *pItem1 = taosArrayGet(aSkyline, sidx * 2); TSDBKEY *pItem2 = taosArrayGet(aSkyline, sidx * 2 + 1); - taosArrayPush(pSkyline, &pItem1); - taosArrayPush(pSkyline, &pItem2); + if (taosArrayPush(pSkyline, &pItem1) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + if (taosArrayPush(pSkyline, &pItem2) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } else { SArray *pSkyline1 = NULL; SArray *pSkyline2 = NULL; @@ -1049,7 +1060,7 @@ int32_t tsdbBuildDeleteSkylineImpl(SArray *aSkyline, int32_t sidx, int32_t eidx, code = tsdbBuildDeleteSkylineImpl(aSkyline, midx + 1, eidx, pSkyline2); if (code) goto _clear; - code = tsdbMergeSkyline(pSkyline1, pSkyline2, pSkyline); + tsdbMergeSkyline(pSkyline1, pSkyline2, pSkyline); _clear: taosArrayDestroy(pSkyline1); @@ -1064,13 +1075,28 @@ int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SAr int32_t code = 0; int32_t dataNum = eidx - sidx + 1; SArray *aTmpSkyline = taosArrayInit(dataNum * 2, sizeof(TSDBKEY)); - SArray *pSkyline = taosArrayInit(dataNum * 2, POINTER_BYTES); + if (aTmpSkyline == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SArray *pSkyline = taosArrayInit(dataNum * 2, POINTER_BYTES); + if (pSkyline == NULL) { + taosArrayDestroy(aTmpSkyline); + return TSDB_CODE_OUT_OF_MEMORY; + } taosArrayClear(aSkyline); for (int32_t i = sidx; i <= eidx; ++i) { pDelData = (SDelData *)taosArrayGet(aDelData, i); - taosArrayPush(aTmpSkyline, &(TSDBKEY){.ts = pDelData->sKey, .version = pDelData->version}); - taosArrayPush(aTmpSkyline, &(TSDBKEY){.ts = pDelData->eKey, .version = 0}); + if (taosArrayPush(aTmpSkyline, &(TSDBKEY){.ts = pDelData->sKey, .version = pDelData->version}) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _clear; + } + + if (taosArrayPush(aTmpSkyline, &(TSDBKEY){.ts = pDelData->eKey, .version = 0}) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _clear; + } } code = tsdbBuildDeleteSkylineImpl(aTmpSkyline, sidx, eidx, pSkyline); @@ -1079,7 +1105,10 @@ int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SAr int32_t skylineNum = taosArrayGetSize(pSkyline); for (int32_t i = 0; i < skylineNum; ++i) { TSDBKEY *p = taosArrayGetP(pSkyline, i); - taosArrayPush(aSkyline, p); + if (taosArrayPush(aSkyline, p) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _clear; + } } _clear: @@ -1519,8 +1548,8 @@ int32_t tBlockDataDecompress(SBufferReader *br, SBlockData *blockData, SBuffer * SBlockCol blockCol; code = tGetBlockCol(&br2, &blockCol, hdr.fmtVer, hdr.cmprAlg); - if (blockCol.alg == 0) blockCol.alg = hdr.cmprAlg; TSDB_CHECK_CODE(code, lino, _exit); + if (blockCol.alg == 0) blockCol.alg = hdr.cmprAlg; code = tBlockDataDecompressColData(&hdr, &blockCol, br, blockData, assist); TSDB_CHECK_CODE(code, lino, _exit); } @@ -1812,7 +1841,9 @@ int32_t tsdbGetColCmprAlgFromSet(SHashObj *set, int16_t colId, uint32_t *alg) { if (set == NULL) return -1; uint32_t *ret = taosHashGet(set, &colId, sizeof(colId)); - if (ret == NULL) return -1; + if (ret == NULL) { + return TSDB_CODE_NOT_FOUND; + } *alg = *ret; return 0; diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 93a06ddf30..66b7938440 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -43,17 +43,16 @@ void syncUtilNodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet) { bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId) { uint32_t ipv4 = 0xFFFFFFFF; - sDebug("vgId:%d, start to resolve sync addr fqdn in %d seconds, " - "dnode:%d cluster:%" PRId64 " fqdn:%s port:%u ", - vgId, tsResolveFQDNRetryTime, - pInfo->nodeId, pInfo->clusterId, pInfo->nodeFqdn, pInfo->nodePort); - for(int i = 0; i < tsResolveFQDNRetryTime; i++){ + sDebug( + "vgId:%d, start to resolve sync addr fqdn in %d seconds, " + "dnode:%d cluster:%" PRId64 " fqdn:%s port:%u ", + vgId, tsResolveFQDNRetryTime, pInfo->nodeId, pInfo->clusterId, pInfo->nodeFqdn, pInfo->nodePort); + for (int i = 0; i < tsResolveFQDNRetryTime; i++) { ipv4 = taosGetIpv4FromFqdn(pInfo->nodeFqdn); if (ipv4 == 0xFFFFFFFF || ipv4 == 1) { sError("failed to resolve ipv4 addr, fqdn:%s, wait one second", pInfo->nodeFqdn); taosSsleep(1); - } - else{ + } else { break; } } @@ -501,8 +500,7 @@ void syncLogSendRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteRepl int32_t syncSnapInfoDataRealloc(SSnapshot* pSnap, int32_t size) { void* data = taosMemoryRealloc(pSnap->data, size); if (data == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + return terrno = TSDB_CODE_OUT_OF_MEMORY; } pSnap->data = data; return 0; From 5734fe8778f8e7070c1cba44ccccb0af4d5ceb26 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Jul 2024 16:07:09 +0800 Subject: [PATCH 02/43] fix compile error --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 16bacf90b3..28abed1ce7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -759,7 +759,7 @@ int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *m // decode int32_t n; code = tGetMapData(pReader->aBuf[0], mDataBlk, &n); - if (code) goto _err; + if (code) goto _exit; ASSERT(n == size); _exit: From 747ce8d3a76c061ea629e2a674052720827ea8eb Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 19 Jul 2024 22:27:49 +0800 Subject: [PATCH 03/43] test: add the more compilation cache directories in CI --- tests/parallel_test/container_build.sh | 29 +++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 3d3069f05f..a1a5ecbbda 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -63,12 +63,24 @@ docker run \ -v /root/.cos-local.1:/root/.cos-local.2 \ -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \ -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ + -v ${REP_REAL_PATH}/community/contrib/apr/:${REP_DIR}/community/contrib/apr \ + -v ${REP_REAL_PATH}/community/contrib/apr-util/:${REP_DIR}/community/contrib/apr-util \ -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ - -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \ -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ + -v ${REP_REAL_PATH}/community/contrib/curl/:${REP_DIR}/community/contrib/curl \ + -v ${REP_REAL_PATH}/community/contrib/curl2/:${REP_DIR}/community/contrib/curl2 \ + -v ${REP_REAL_PATH}/community/contrib/geo/:${REP_DIR}/community/contrib/geos \ + -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \ + -v ${REP_REAL_PATH}/community/contrib/libs3/:${REP_DIR}/community/contrib/libs3 \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ + -v ${REP_REAL_PATH}/community/contrib/lzma2/:${REP_DIR}/community/contrib/lzma2 \ + -v ${REP_REAL_PATH}/community/contrib/mxml/:${REP_DIR}/community/contrib/mxml \ + -v ${REP_REAL_PATH}/community/contrib/openssl/:${REP_DIR}/community/contrib/openssl \ + -v ${REP_REAL_PATH}/community/contrib/pcre2/:${REP_DIR}/community/contrib/pcre2 \ + -v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \ -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ + -v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \ --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1" # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ @@ -98,13 +110,24 @@ docker run \ -v /root/.cos-local.1:/root/.cos-local.2 \ -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \ -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ + -v ${REP_REAL_PATH}/community/contrib/apr/:${REP_DIR}/community/contrib/apr \ + -v ${REP_REAL_PATH}/community/contrib/apr-util/:${REP_DIR}/community/contrib/apr-util \ -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ - -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \ -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ + -v ${REP_REAL_PATH}/community/contrib/curl/:${REP_DIR}/community/contrib/curl \ + -v ${REP_REAL_PATH}/community/contrib/curl2/:${REP_DIR}/community/contrib/curl2 \ + -v ${REP_REAL_PATH}/community/contrib/geo/:${REP_DIR}/community/contrib/geos \ + -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \ + -v ${REP_REAL_PATH}/community/contrib/libs3/:${REP_DIR}/community/contrib/libs3 \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ + -v ${REP_REAL_PATH}/community/contrib/lzma2/:${REP_DIR}/community/contrib/lzma2 \ + -v ${REP_REAL_PATH}/community/contrib/mxml/:${REP_DIR}/community/contrib/mxml \ + -v ${REP_REAL_PATH}/community/contrib/openssl/:${REP_DIR}/community/contrib/openssl \ + -v ${REP_REAL_PATH}/community/contrib/pcre2/:${REP_DIR}/community/contrib/pcre2 \ + -v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \ -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ - -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ + -v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \ --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=false -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true $CMAKE_BUILD_TYPE -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 " mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan From d4e0a4389386a2bccfe939f0af4baebeaa23d437 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 19 Jul 2024 22:29:23 +0800 Subject: [PATCH 04/43] test: add the more compilation cache directories in CI --- tests/parallel_test/container_build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index a1a5ecbbda..5657f65743 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -69,7 +69,7 @@ docker run \ -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ -v ${REP_REAL_PATH}/community/contrib/curl/:${REP_DIR}/community/contrib/curl \ -v ${REP_REAL_PATH}/community/contrib/curl2/:${REP_DIR}/community/contrib/curl2 \ - -v ${REP_REAL_PATH}/community/contrib/geo/:${REP_DIR}/community/contrib/geos \ + -v ${REP_REAL_PATH}/community/contrib/geos/:${REP_DIR}/community/contrib/geos \ -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \ -v ${REP_REAL_PATH}/community/contrib/libs3/:${REP_DIR}/community/contrib/libs3 \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ @@ -116,7 +116,7 @@ docker run \ -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ -v ${REP_REAL_PATH}/community/contrib/curl/:${REP_DIR}/community/contrib/curl \ -v ${REP_REAL_PATH}/community/contrib/curl2/:${REP_DIR}/community/contrib/curl2 \ - -v ${REP_REAL_PATH}/community/contrib/geo/:${REP_DIR}/community/contrib/geos \ + -v ${REP_REAL_PATH}/community/contrib/geos/:${REP_DIR}/community/contrib/geos \ -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \ -v ${REP_REAL_PATH}/community/contrib/libs3/:${REP_DIR}/community/contrib/libs3 \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ From 33ee871acfbe883901ce9393c92a524931100926 Mon Sep 17 00:00:00 2001 From: dmchen Date: Sat, 20 Jul 2024 01:25:18 +0000 Subject: [PATCH 05/43] fix/TD-30989 --- source/dnode/mnode/impl/src/mndDb.c | 2 + source/dnode/mnode/impl/src/mndDef.c | 186 ++++++++++++----------- source/dnode/mnode/impl/src/mndDnode.c | 198 +++++++++++++------------ source/dnode/mnode/impl/src/mndDump.c | 10 +- source/dnode/mnode/impl/src/mndFunc.c | 152 ++++++++++++------- 5 files changed, 311 insertions(+), 237 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 8432855c6f..e1be5d6d79 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -910,7 +910,9 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { } } + mInfo("mndCheckDbPrivilege1, %d", code); if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DB, NULL) != 0) { + mInfo("mndCheckDbPrivilege2, %d", code); goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 9dc5f920ad..aacf391236 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -12,173 +12,179 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - #define _DEFAULT_SOURCE #include "mndDef.h" #include "mndConsumer.h" +#include "taoserror.h" static void *freeStreamTasks(SArray *pTaskLevel); int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { - if (tStartEncode(pEncoder) < 0) return -1; - if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1; + TAOS_CHECK_RETURN(tStartEncode(pEncoder)); + TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->name)); - if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->updateTime) < 0) return -1; - if (tEncodeI32(pEncoder, pObj->version) < 0) return -1; - if (tEncodeI32(pEncoder, pObj->totalLevel) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->smaId) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->createTime)); + TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->updateTime)); + TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pObj->version)); + TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pObj->totalLevel)); + TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->smaId)); - if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1; - if (tEncodeI8(pEncoder, pObj->status) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->uid)); + TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->status)); - if (tEncodeI8(pEncoder, pObj->conf.igExpired) < 0) return -1; - if (tEncodeI8(pEncoder, pObj->conf.trigger) < 0) return -1; - if (tEncodeI8(pEncoder, pObj->conf.fillHistory) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->conf.triggerParam) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->conf.watermark) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->conf.igExpired)); + TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->conf.trigger)); + TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->conf.fillHistory)); + TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->conf.triggerParam)); + TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->conf.watermark)); - if (tEncodeI64(pEncoder, pObj->sourceDbUid) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->targetDbUid) < 0) return -1; - if (tEncodeCStr(pEncoder, pObj->sourceDb) < 0) return -1; - if (tEncodeCStr(pEncoder, pObj->targetDb) < 0) return -1; - if (tEncodeCStr(pEncoder, pObj->targetSTbName) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->targetStbUid) < 0) return -1; - if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->sourceDbUid)); + TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->targetDbUid)); + TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->sourceDb)); + TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->targetDb)); + TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->targetSTbName)); + TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->targetStbUid)); + TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pObj->fixedSinkVgId)); if (pObj->sql != NULL) { - if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->sql)); } else { - if (tEncodeCStr(pEncoder, "") < 0) return -1; + TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, "")); } if (pObj->ast != NULL) { - if (tEncodeCStr(pEncoder, pObj->ast) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->ast)); } else { - if (tEncodeCStr(pEncoder, "") < 0) return -1; + TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, "")); } if (pObj->physicalPlan != NULL) { - if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->physicalPlan)); } else { - if (tEncodeCStr(pEncoder, "") < 0) return -1; + TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, "")); } int32_t sz = taosArrayGetSize(pObj->tasks); - if (tEncodeI32(pEncoder, sz) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeI32(pEncoder, sz)); for (int32_t i = 0; i < sz; i++) { SArray *pArray = taosArrayGetP(pObj->tasks, i); int32_t innerSz = taosArrayGetSize(pArray); - if (tEncodeI32(pEncoder, innerSz) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeI32(pEncoder, innerSz)); for (int32_t j = 0; j < innerSz; j++) { SStreamTask *pTask = taosArrayGetP(pArray, j); if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER){ pTask->ver = SSTREAM_TASK_VER; } - if (tEncodeStreamTask(pEncoder, pTask) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeStreamTask(pEncoder, pTask)); } } - if (tEncodeSSchemaWrapper(pEncoder, &pObj->outputSchema) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pEncoder, &pObj->outputSchema)); // 3.0.20 ver =2 - if (tEncodeI64(pEncoder, pObj->checkpointFreq) < 0) return -1; - if (tEncodeI8(pEncoder, pObj->igCheckUpdate) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->checkpointFreq)); + TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->igCheckUpdate)); // 3.0.50 ver = 3 - if (tEncodeI64(pEncoder, pObj->checkpointId) < 0) return -1; - if (tEncodeI8(pEncoder, pObj->subTableWithoutMd5) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->checkpointId)); + TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->subTableWithoutMd5)); - if (tEncodeCStrWithLen(pEncoder, pObj->reserve, sizeof(pObj->reserve) - 1) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeCStrWithLen(pEncoder, pObj->reserve, sizeof(pObj->reserve) - 1)); tEndEncode(pEncoder); return pEncoder->pos; } int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) { - if (tStartDecode(pDecoder) < 0) return -1; - if (tDecodeCStrTo(pDecoder, pObj->name) < 0) return -1; + int32_t code = 0; + TAOS_CHECK_RETURN(tStartDecode(pDecoder)); + TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pObj->name)); - if (tDecodeI64(pDecoder, &pObj->createTime) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->updateTime) < 0) return -1; - if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1; - if (tDecodeI32(pDecoder, &pObj->totalLevel) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->smaId) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->createTime)); + TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->updateTime)); + TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pObj->version)); + TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pObj->totalLevel)); + TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->smaId)); - if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1; - if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->uid)); + TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pObj->status)); - if (tDecodeI8(pDecoder, &pObj->conf.igExpired) < 0) return -1; - if (tDecodeI8(pDecoder, &pObj->conf.trigger) < 0) return -1; - if (tDecodeI8(pDecoder, &pObj->conf.fillHistory) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->conf.triggerParam) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->conf.watermark) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pObj->conf.igExpired)); + TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pObj->conf.trigger)); + TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pObj->conf.fillHistory)); + TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->conf.triggerParam)); + TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->conf.watermark)); - if (tDecodeI64(pDecoder, &pObj->sourceDbUid) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->targetDbUid) < 0) return -1; - if (tDecodeCStrTo(pDecoder, pObj->sourceDb) < 0) return -1; - if (tDecodeCStrTo(pDecoder, pObj->targetDb) < 0) return -1; - if (tDecodeCStrTo(pDecoder, pObj->targetSTbName) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->targetStbUid) < 0) return -1; - if (tDecodeI32(pDecoder, &pObj->fixedSinkVgId) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->sourceDbUid)); + TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->targetDbUid)); + TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pObj->sourceDb)); + TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pObj->targetDb)); + TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pObj->targetSTbName)); + TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->targetStbUid)); + TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pObj->fixedSinkVgId)); - if (tDecodeCStrAlloc(pDecoder, &pObj->sql) < 0) return -1; - if (tDecodeCStrAlloc(pDecoder, &pObj->ast) < 0) return -1; - if (tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeCStrAlloc(pDecoder, &pObj->sql)); + TAOS_CHECK_RETURN(tDecodeCStrAlloc(pDecoder, &pObj->ast)); + TAOS_CHECK_RETURN(tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan)); if (pObj->tasks != NULL) { pObj->tasks = freeStreamTasks(pObj->tasks); } int32_t sz; - if (tDecodeI32(pDecoder, &sz) < 0) { - return -1; - } + TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &sz)); if (sz != 0) { pObj->tasks = taosArrayInit(sz, sizeof(void *)); + if (pObj->tasks == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(code); + } for (int32_t i = 0; i < sz; i++) { int32_t innerSz; - if (tDecodeI32(pDecoder, &innerSz) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &innerSz)); SArray *pArray = taosArrayInit(innerSz, sizeof(void *)); - for (int32_t j = 0; j < innerSz; j++) { - SStreamTask *pTask = taosMemoryCalloc(1, sizeof(SStreamTask)); - if (pTask == NULL) { - taosArrayDestroy(pArray); - return -1; + if (pArray != NULL) { + for (int32_t j = 0; j < innerSz; j++) { + SStreamTask *pTask = taosMemoryCalloc(1, sizeof(SStreamTask)); + if (pTask == NULL) { + taosArrayDestroy(pArray); + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(code); + } + if ((code = tDecodeStreamTask(pDecoder, pTask)) < 0) { + taosMemoryFree(pTask); + taosArrayDestroy(pArray); + TAOS_RETURN(code); + } + taosArrayPush(pArray, &pTask); } - if (tDecodeStreamTask(pDecoder, pTask) < 0) { - taosMemoryFree(pTask); - taosArrayDestroy(pArray); - return -1; - } - taosArrayPush(pArray, &pTask); } taosArrayPush(pObj->tasks, &pArray); } } - if (tDecodeSSchemaWrapper(pDecoder, &pObj->outputSchema) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeSSchemaWrapper(pDecoder, &pObj->outputSchema)); // 3.0.20 if (sver >= 2) { - if (tDecodeI64(pDecoder, &pObj->checkpointFreq) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->checkpointFreq)); if (!tDecodeIsEnd(pDecoder)) { - if (tDecodeI8(pDecoder, &pObj->igCheckUpdate) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pObj->igCheckUpdate)); } } if (sver >= 3) { - if (tDecodeI64(pDecoder, &pObj->checkpointId) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->checkpointId)); } if (sver >= 5) { - if (tDecodeI8(pDecoder, &pObj->subTableWithoutMd5) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pObj->subTableWithoutMd5)); } - if (tDecodeCStrTo(pDecoder, pObj->reserve) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pObj->reserve)); tEndDecode(pDecoder); - return 0; + TAOS_RETURN(code); } void *freeStreamTasks(SArray *pTaskLevel) { @@ -220,7 +226,10 @@ void tFreeStreamObj(SStreamObj *pStream) { SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) { SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp)); - if (pVgEpNew == NULL) return NULL; + if (pVgEpNew == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } pVgEpNew->vgId = pVgEp->vgId; // pVgEpNew->qmsg = taosStrdup(pVgEp->qmsg); pVgEpNew->epSet = pVgEp->epSet; @@ -256,6 +265,7 @@ void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp, int8_t sver) { static void *topicNameDup(void *p) { return taosStrdup((char *)p); } SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType, char *topic, SCMSubscribeReq *subscribe) { + terrno = 0; SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj)); if (pConsumer == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -531,8 +541,10 @@ void *tDecodeSMqConsumerEp(const void *buf, SMqConsumerEp *pConsumerEp, int8_t s } SMqSubscribeObj *tNewSubscribeObj(const char *key) { + terrno = 0; SMqSubscribeObj *pSubObj = taosMemoryCalloc(1, sizeof(SMqSubscribeObj)); if (pSubObj == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -548,8 +560,12 @@ SMqSubscribeObj *tNewSubscribeObj(const char *key) { } SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) { + terrno = 0; SMqSubscribeObj *pSubNew = taosMemoryMalloc(sizeof(SMqSubscribeObj)); - if (pSubNew == NULL) return NULL; + if (pSubNew == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } memcpy(pSubNew->key, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); taosInitRWLatch(&pSubNew->lock); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 73addea6fe..65957dfd40 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -165,15 +165,24 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { } pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, NULL, "create-dnode"); - if (pTrans == NULL) goto _OVER; + if (pTrans == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } mInfo("trans:%d, used to create dnode:%s on first deploy", pTrans->id, dnodeObj.ep); pRaw = mndDnodeActionEncode(&dnodeObj); - if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; + if (pRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } + TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pRaw), NULL, _OVER); (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); pRaw = NULL; - if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER); code = 0; mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1); @@ -563,10 +572,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { SStatisReq statisReq = {0}; int32_t code = -1; - if (tDeserializeSStatisReq(pReq->pCont, pReq->contLen, &statisReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return code; - } + TAOS_RETURN(tDeserializeSStatisReq(pReq->pCont, pReq->contLen, &statisReq)); if (tsMonitorLogProtocol) { mInfo("process statis req,\n %s", statisReq.pCont); @@ -586,7 +592,8 @@ static int32_t mndUpdateDnodeObj(SMnode *pMnode, SDnodeObj *pDnode) { int32_t code = 0; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "update-dnode-obj"); if (pTrans == NULL) { - code = terrno; + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; goto _exit; } @@ -600,9 +607,8 @@ static int32_t mndUpdateDnodeObj(SMnode *pMnode, SDnodeObj *pDnode) { } (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); - if (mndTransPrepare(pMnode, pTrans) != 0) { + if ((code = mndTransPrepare(pMnode, pTrans)) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - code = terrno; goto _exit; } @@ -617,10 +623,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { SDnodeObj *pDnode = NULL; int32_t code = -1; - if (tDeserializeSStatusReq(pReq->pCont, pReq->contLen, &statusReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } + TAOS_CHECK_GOTO(tDeserializeSStatusReq(pReq->pCont, pReq->contLen, &statusReq), NULL, _OVER); int64_t clusterid = mndGetClusterId(pMnode); if (statusReq.clusterId != 0 && statusReq.clusterId != clusterid) { @@ -634,6 +637,8 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { pDnode = mndAcquireDnodeByEp(pMnode, statusReq.dnodeEp); if (pDnode == NULL) { mInfo("dnode:%s, not created yet", statusReq.dnodeEp); + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; goto _OVER; } } else { @@ -871,16 +876,25 @@ static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pC snprintf(dnodeObj.ep, TSDB_EP_LEN - 1, "%s:%u", pCreate->fqdn, pCreate->port); pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq, "create-dnode"); - if (pTrans == NULL) goto _OVER; + if (pTrans == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } mInfo("trans:%d, used to create dnode:%s", pTrans->id, dnodeObj.ep); - if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER); pRaw = mndDnodeActionEncode(&dnodeObj); - if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; + if (pRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } + TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pRaw), NULL, _OVER); (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); pRaw = NULL; - if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER); code = 0; mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1); @@ -901,7 +915,7 @@ static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) { rsp.dnodeList = taosArrayInit(5, sizeof(SEpSet)); if (NULL == rsp.dnodeList) { mError("failed to alloc epSet while process dnode list req"); - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } @@ -922,7 +936,7 @@ static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) { int32_t rspLen = tSerializeSDnodeListRsp(NULL, 0, &rsp); void *pRsp = rpcMallocCont(rspLen); if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } @@ -940,7 +954,7 @@ _OVER: tFreeSDnodeListRsp(&rsp); - return code; + TAOS_RETURN(code); } static void getSlowLogScopeString(int32_t scope, char* result){ @@ -980,7 +994,7 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { rsp.variables = taosArrayInit(16, sizeof(SVariablesInfo)); if (NULL == rsp.variables) { mError("failed to alloc SVariablesInfo array while process show variables req"); - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } @@ -1036,7 +1050,7 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { int32_t rspLen = tSerializeSShowVariablesRsp(NULL, 0, &rsp); void *pRsp = rpcMallocCont(rspLen); if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } @@ -1053,7 +1067,7 @@ _OVER: } tFreeSShowVariablesRsp(&rsp); - return code; + TAOS_RETURN(code); } static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { @@ -1062,23 +1076,17 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { SDnodeObj *pDnode = NULL; SCreateDnodeReq createReq = {0}; - if ((terrno = grantCheck(TSDB_GRANT_DNODE)) != 0 || (terrno = grantCheck(TSDB_GRANT_CPU_CORES)) != 0) { - code = terrno; + if ((code = grantCheck(TSDB_GRANT_DNODE)) != 0 || (code = grantCheck(TSDB_GRANT_CPU_CORES)) != 0) { goto _OVER; } - if (tDeserializeSCreateDnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } + TAOS_CHECK_GOTO(tDeserializeSCreateDnodeReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER); mInfo("dnode:%s:%d, start to create", createReq.fqdn, createReq.port); - if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DNODE) != 0) { - goto _OVER; - } + TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DNODE), NULL, _OVER); if (createReq.fqdn[0] == 0 || createReq.port <= 0 || createReq.port > UINT16_MAX) { - terrno = TSDB_CODE_MND_INVALID_DNODE_EP; + code = TSDB_CODE_MND_INVALID_DNODE_EP; goto _OVER; } @@ -1086,7 +1094,7 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { snprintf(ep, TSDB_EP_LEN, "%s:%d", createReq.fqdn, createReq.port); pDnode = mndAcquireDnodeByEp(pMnode, ep); if (pDnode != NULL) { - terrno = TSDB_CODE_MND_DNODE_ALREADY_EXIST; + code = TSDB_CODE_MND_DNODE_ALREADY_EXIST; goto _OVER; } @@ -1108,7 +1116,7 @@ _OVER: mndReleaseDnode(pMnode, pDnode); tFreeSCreateDnodeReq(&createReq); - return code; + TAOS_RETURN(code); } extern int32_t mndProcessRestoreDnodeReqImpl(SRpcMsg *pReq); @@ -1126,44 +1134,56 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM STrans *pTrans = NULL; pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "drop-dnode"); - if (pTrans == NULL) goto _OVER; + if (pTrans == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } mndTransSetSerial(pTrans); mInfo("trans:%d, used to drop dnode:%d, force:%d", pTrans->id, pDnode->id, force); - if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER); pRaw = mndDnodeActionEncode(pDnode); - if (pRaw == NULL) goto _OVER; - if (mndTransAppendRedolog(pTrans, pRaw) != 0) goto _OVER; + if (pRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } + TAOS_CHECK_GOTO(mndTransAppendRedolog(pTrans, pRaw), NULL, _OVER); (void)sdbSetRawStatus(pRaw, SDB_STATUS_DROPPING); pRaw = NULL; pRaw = mndDnodeActionEncode(pDnode); - if (pRaw == NULL) goto _OVER; - if (mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; + if (pRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } + TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pRaw), NULL, _OVER); (void)sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); pRaw = NULL; if (pMObj != NULL) { mInfo("trans:%d, mnode on dnode:%d will be dropped", pTrans->id, pDnode->id); - if (mndSetDropMnodeInfoToTrans(pMnode, pTrans, pMObj, force) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndSetDropMnodeInfoToTrans(pMnode, pTrans, pMObj, force), NULL, _OVER); } if (pQObj != NULL) { mInfo("trans:%d, qnode on dnode:%d will be dropped", pTrans->id, pDnode->id); - if (mndSetDropQnodeInfoToTrans(pMnode, pTrans, pQObj, force) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndSetDropQnodeInfoToTrans(pMnode, pTrans, pQObj, force), NULL, _OVER); } if (pSObj != NULL) { mInfo("trans:%d, snode on dnode:%d will be dropped", pTrans->id, pDnode->id); - if (mndSetDropSnodeInfoToTrans(pMnode, pTrans, pSObj, force) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndSetDropSnodeInfoToTrans(pMnode, pTrans, pSObj, force), NULL, _OVER); } if (numOfVnodes > 0) { mInfo("trans:%d, %d vnodes on dnode:%d will be dropped", pTrans->id, numOfVnodes, pDnode->id); - if (mndSetMoveVgroupsInfoToTrans(pMnode, pTrans, pDnode->id, force, unsafe) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndSetMoveVgroupsInfoToTrans(pMnode, pTrans, pDnode->id, force, unsafe), NULL, _OVER); } - if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER); mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, pDnode->fqdn, IP_WHITE_DROP, 1); code = 0; @@ -1171,7 +1191,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM _OVER: mndTransDrop(pTrans); sdbFreeRaw(pRaw); - return code; + TAOS_RETURN(code); } static bool mndIsEmptyDnode(SMnode *pMnode, int32_t dnodeId) { @@ -1209,16 +1229,11 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { SSnodeObj *pSObj = NULL; SDropDnodeReq dropReq = {0}; - if (tDeserializeSDropDnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } + TAOS_CHECK_GOTO(tDeserializeSDropDnodeReq(pReq->pCont, pReq->contLen, &dropReq), NULL, _OVER); mInfo("dnode:%d, start to drop, ep:%s:%d, force:%s, unsafe:%s", dropReq.dnodeId, dropReq.fqdn, dropReq.port, dropReq.force ? "true" : "false", dropReq.unsafe ? "true" : "false"); - if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { - goto _OVER; - } + TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE), NULL, _OVER); bool force = dropReq.force; if (dropReq.unsafe) { @@ -1232,7 +1247,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { snprintf(ep, sizeof(ep), dropReq.fqdn, dropReq.port); pDnode = mndAcquireDnodeByEp(pMnode, ep); if (pDnode == NULL) { - terrno = err; + code = err; goto _OVER; } } @@ -1242,11 +1257,11 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { pMObj = mndAcquireMnode(pMnode, dropReq.dnodeId); if (pMObj != NULL) { if (sdbGetSize(pMnode->pSdb, SDB_MNODE) <= 1) { - terrno = TSDB_CODE_MND_TOO_FEW_MNODES; + code = TSDB_CODE_MND_TOO_FEW_MNODES; goto _OVER; } if (pMnode->selfDnodeId == dropReq.dnodeId) { - terrno = TSDB_CODE_MND_CANT_DROP_LEADER; + code = TSDB_CODE_MND_CANT_DROP_LEADER; goto _OVER; } } @@ -1255,7 +1270,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { bool isonline = mndIsDnodeOnline(pDnode, taosGetTimestampMs()); if (isonline && force) { - terrno = TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE; + code = TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE; mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); goto _OVER; @@ -1263,7 +1278,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { bool isEmpty = mndIsEmptyDnode(pMnode, pDnode->id); if (!isonline && !force && !isEmpty) { - terrno = TSDB_CODE_DNODE_OFFLINE; + code = TSDB_CODE_DNODE_OFFLINE; mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); goto _OVER; @@ -1287,11 +1302,11 @@ _OVER: mndReleaseQnode(pMnode, pQObj); mndReleaseSnode(pMnode, pSObj); tFreeSDropDnodeReq(&dropReq); - return code; + TAOS_RETURN(code); } static int32_t mndMCfg2DCfg(SMCfgDnodeReq *pMCfgReq, SDCfgDnodeReq *pDCfgReq) { - terrno = 0; + int32_t code = 0; char *p = pMCfgReq->config; while (*p) { if (*p == ' ') { @@ -1314,12 +1329,12 @@ static int32_t mndMCfg2DCfg(SMCfgDnodeReq *pMCfgReq, SDCfgDnodeReq *pDCfgReq) { strcpy(pDCfgReq->value, pMCfgReq->value); } - return 0; + TAOS_RETURN(code); _err: mError("dnode:%d, failed to config since invalid conf:%s", pMCfgReq->dnodeId, pMCfgReq->config); - terrno = TSDB_CODE_INVALID_CFG; - return -1; + code = TSDB_CODE_INVALID_CFG; + TAOS_RETURN(code); } static int32_t mndSendCfgDnodeReq(SMnode *pMnode, int32_t dnodeId, SDCfgDnodeReq *pDcfgReq) { @@ -1349,23 +1364,21 @@ static int32_t mndSendCfgDnodeReq(SMnode *pMnode, int32_t dnodeId, SDCfgDnodeReq } if (code == -1) { - terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + code = TSDB_CODE_MND_DNODE_NOT_EXIST; } - return code; + TAOS_RETURN(code); } static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { + int32_t code = 0; SMnode *pMnode = pReq->info.node; SMCfgDnodeReq cfgReq = {0}; - if (tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } + TAOS_RETURN(tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq)); int8_t updateIpWhiteList = 0; mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value); - if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) { + if ((code = mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE)) != 0) { tFreeSMCfgDnodeReq(&cfgReq); - return -1; + TAOS_RETURN(code); } SDCfgDnodeReq dcfgReq = {0}; @@ -1381,9 +1394,9 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { if (flag > 1024 * 1024 || (flag > -1 && flag < 1024) || flag < -1) { mError("dnode:%d, failed to config s3blocksize since value:%d. Valid range: -1 or [1024, 1024 * 1024]", cfgReq.dnodeId, flag); - terrno = TSDB_CODE_INVALID_CFG; + code = TSDB_CODE_INVALID_CFG; tFreeSMCfgDnodeReq(&cfgReq); - return -1; + TAOS_RETURN(code); } strcpy(dcfgReq.config, "s3blocksize"); @@ -1393,14 +1406,14 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { if (mndMCfg2DCfg(&cfgReq, &dcfgReq)) goto _err_out; if (strlen(dcfgReq.config) > TSDB_DNODE_CONFIG_LEN) { mError("dnode:%d, failed to config since config is too long", cfgReq.dnodeId); - terrno = TSDB_CODE_INVALID_CFG; + code = TSDB_CODE_INVALID_CFG; goto _err_out; } if (strncasecmp(dcfgReq.config, "enableWhiteList", strlen("enableWhiteList")) == 0) { updateIpWhiteList = 1; } - if (cfgCheckRangeForDynUpdate(taosGetCfg(), dcfgReq.config, dcfgReq.value, true) != 0) goto _err_out; + TAOS_CHECK_GOTO(cfgCheckRangeForDynUpdate(taosGetCfg(), dcfgReq.config, dcfgReq.value, true), NULL, _err_out); } { // audit @@ -1412,15 +1425,15 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { tFreeSMCfgDnodeReq(&cfgReq); - int32_t code = mndSendCfgDnodeReq(pMnode, cfgReq.dnodeId, &dcfgReq); + code = mndSendCfgDnodeReq(pMnode, cfgReq.dnodeId, &dcfgReq); // dont care suss or succ; if (updateIpWhiteList) mndRefreshUserIpWhiteList(pMnode); - return code; + TAOS_RETURN(code); _err_out: tFreeSMCfgDnodeReq(&cfgReq); - return -1; + TAOS_RETURN(code); } static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp) { @@ -1503,17 +1516,16 @@ _exit: } static int32_t mndProcessCreateEncryptKeyReq(SRpcMsg *pReq) { + int32_t code = 0; + #ifdef TD_ENTERPRISE SMnode *pMnode = pReq->info.node; SMCfgDnodeReq cfgReq = {0}; - if (tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } + TAOS_RETURN(tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq)); - if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) { + if ((code = mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE)) != 0) { tFreeSMCfgDnodeReq(&cfgReq); - return -1; + TAOS_RETURN(code); } const STraceId *trace = &pReq->info.traceId; SDCfgDnodeReq dcfgReq = {0}; @@ -1523,13 +1535,13 @@ static int32_t mndProcessCreateEncryptKeyReq(SRpcMsg *pReq) { tFreeSMCfgDnodeReq(&cfgReq); return mndProcessCreateEncryptKeyReqImpl(pReq, cfgReq.dnodeId, &dcfgReq); } else { - terrno = TSDB_CODE_PAR_INTERNAL_ERROR; + code = TSDB_CODE_PAR_INTERNAL_ERROR; tFreeSMCfgDnodeReq(&cfgReq); - return -1; + TAOS_RETURN(code); } #else - return 0; + TAOS_RETURN(code); #endif } @@ -1710,7 +1722,7 @@ static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter) { // get int32_t value from 'SMCfgDnodeReq' static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pMCfgReq, int32_t optLen, int32_t *pOutValue) { - terrno = 0; + int32_t code = 0; if (' ' != pMCfgReq->config[optLen] && 0 != pMCfgReq->config[optLen]) { goto _err; } @@ -1725,12 +1737,12 @@ static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pMCfgReq, int32_t optLen, int32 *pOutValue = atoi(pMCfgReq->value); } - return 0; + TAOS_RETURN(code); _err: mError("dnode:%d, failed to config since invalid conf:%s", pMCfgReq->dnodeId, pMCfgReq->config); - terrno = TSDB_CODE_INVALID_CFG; - return -1; + code = TSDB_CODE_INVALID_CFG; + TAOS_RETURN(code); } SArray *mndGetAllDnodeFqdns(SMnode *pMnode) { diff --git a/source/dnode/mnode/impl/src/mndDump.c b/source/dnode/mnode/impl/src/mndDump.c index 5561eea405..22dbc8eb57 100644 --- a/source/dnode/mnode/impl/src/mndDump.c +++ b/source/dnode/mnode/impl/src/mndDump.c @@ -29,12 +29,14 @@ void reportStartup(const char *name, const char *desc) {} void sendRsp(SRpcMsg *pMsg) { rpcFreeCont(pMsg->pCont); } int32_t sendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { - terrno = TSDB_CODE_INVALID_PTR; - return -1; + int32_t code = 0; + code = TSDB_CODE_INVALID_PTR; + TAOS_RETURN(code); } int32_t sendSyncReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { - terrno = TSDB_CODE_INVALID_PTR; - return -1; + int32_t code = 0; + code = TSDB_CODE_INVALID_PTR; + TAOS_RETURN(code); } char *i642str(int64_t val) { diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 333fdf83bb..f5d9a7b8e2 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -223,6 +223,7 @@ static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOld, SFuncObj *pNew) { } static SFuncObj *mndAcquireFunc(SMnode *pMnode, char *funcName) { + terrno = 0; SSdb *pSdb = pMnode->pSdb; SFuncObj *pFunc = sdbAcquire(pSdb, SDB_FUNC, funcName); if (pFunc == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { @@ -240,7 +241,7 @@ static int32_t mndCreateFunc(SMnode *pMnode, SRpcMsg *pReq, SCreateFuncReq *pCre int32_t code = -1; STrans *pTrans = NULL; - if ((terrno = grantCheck(TSDB_GRANT_USER)) < 0) { + if ((code = grantCheck(TSDB_GRANT_USER)) < 0) { return code; } @@ -260,7 +261,7 @@ static int32_t mndCreateFunc(SMnode *pMnode, SRpcMsg *pReq, SCreateFuncReq *pCre func.codeSize = pCreate->codeLen; func.pCode = taosMemoryMalloc(func.codeSize); if (func.pCode == NULL || func.pCode == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } @@ -270,7 +271,11 @@ static int32_t mndCreateFunc(SMnode *pMnode, SRpcMsg *pReq, SCreateFuncReq *pCre memcpy(func.pCode, pCreate->pCode, func.codeSize); pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-func"); - if (pTrans == NULL) goto _OVER; + if (pTrans == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } mInfo("trans:%d, used to create func:%s", pTrans->id, pCreate->name); SFuncObj *oldFunc = mndAcquireFunc(pMnode, pCreate->name); @@ -279,31 +284,61 @@ static int32_t mndCreateFunc(SMnode *pMnode, SRpcMsg *pReq, SCreateFuncReq *pCre func.createdTime = oldFunc->createdTime; SSdbRaw *pRedoRaw = mndFuncActionEncode(oldFunc); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER; - if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) goto _OVER; + if (pRedoRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } + TAOS_CHECK_GOTO(mndTransAppendRedolog(pTrans, pRedoRaw), NULL, _OVER); + TAOS_CHECK_GOTO(sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY), NULL, _OVER); SSdbRaw *pUndoRaw = mndFuncActionEncode(oldFunc); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER; - if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) goto _OVER; + if (pUndoRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } + TAOS_CHECK_GOTO(mndTransAppendUndolog(pTrans, pUndoRaw), NULL, _OVER); + TAOS_CHECK_GOTO(sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY), NULL, _OVER); SSdbRaw *pCommitRaw = mndFuncActionEncode(&func); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER; - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) goto _OVER; + if (pCommitRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } + TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pCommitRaw), NULL, _OVER); + TAOS_CHECK_GOTO(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY), NULL, _OVER); } else { SSdbRaw *pRedoRaw = mndFuncActionEncode(&func); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER; - if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) goto _OVER; + if (pRedoRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } + TAOS_CHECK_GOTO(mndTransAppendRedolog(pTrans, pRedoRaw), NULL, _OVER); + TAOS_CHECK_GOTO(sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING), NULL, _OVER); SSdbRaw *pUndoRaw = mndFuncActionEncode(&func); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER; - if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) goto _OVER; + if (pUndoRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } + TAOS_CHECK_GOTO(mndTransAppendUndolog(pTrans, pUndoRaw), NULL, _OVER); + TAOS_CHECK_GOTO(sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED), NULL, _OVER); SSdbRaw *pCommitRaw = mndFuncActionEncode(&func); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER; - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) goto _OVER; + if (pCommitRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } + TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pCommitRaw), NULL, _OVER); + TAOS_CHECK_GOTO(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY), NULL, _OVER); } - if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER); code = 0; @@ -315,32 +350,48 @@ _OVER: taosMemoryFree(func.pCode); taosMemoryFree(func.pComment); mndTransDrop(pTrans); - return code; + TAOS_RETURN(code); } static int32_t mndDropFunc(SMnode *pMnode, SRpcMsg *pReq, SFuncObj *pFunc) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "drop-func"); - if (pTrans == NULL) goto _OVER; + if (pTrans == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } mInfo("trans:%d, used to drop user:%s", pTrans->id, pFunc->name); SSdbRaw *pRedoRaw = mndFuncActionEncode(pFunc); - if (pRedoRaw == NULL) goto _OVER; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER; + if (pRedoRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } + TAOS_CHECK_GOTO(mndTransAppendRedolog(pTrans, pRedoRaw), NULL, _OVER); (void)sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); SSdbRaw *pUndoRaw = mndFuncActionEncode(pFunc); - if (pUndoRaw == NULL) goto _OVER; - if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER; + if (pUndoRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } + TAOS_CHECK_GOTO(mndTransAppendUndolog(pTrans, pUndoRaw), NULL, _OVER); (void)sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); SSdbRaw *pCommitRaw = mndFuncActionEncode(pFunc); - if (pCommitRaw == NULL) goto _OVER; - if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER; + if (pCommitRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } + TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pCommitRaw), NULL, _OVER); (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); - if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER); code = 0; @@ -355,18 +406,14 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) { SFuncObj *pFunc = NULL; SCreateFuncReq createReq = {0}; - if (tDeserializeSCreateFuncReq(pReq->pCont, pReq->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } + TAOS_CHECK_GOTO(tDeserializeSCreateFuncReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER); + #ifdef WINDOWS - terrno = TSDB_CODE_MND_INVALID_PLATFORM; + code = TSDB_CODE_MND_INVALID_PLATFORM; goto _OVER; #endif mInfo("func:%s, start to create, size:%d", createReq.name, createReq.codeLen); - if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC) != 0) { - goto _OVER; - } + TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC), NULL, _OVER); pFunc = mndAcquireFunc(pMnode, createReq.name); if (pFunc != NULL) { @@ -378,7 +425,7 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) { mInfo("func:%s, replace function is set", createReq.name); code = 0; } else { - terrno = TSDB_CODE_MND_FUNC_ALREADY_EXIST; + code = TSDB_CODE_MND_FUNC_ALREADY_EXIST; goto _OVER; } } else if (terrno == TSDB_CODE_MND_FUNC_ALREADY_EXIST) { @@ -386,22 +433,22 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) { } if (createReq.name[0] == 0) { - terrno = TSDB_CODE_MND_INVALID_FUNC_NAME; + code = TSDB_CODE_MND_INVALID_FUNC_NAME; goto _OVER; } if (createReq.pCode == NULL) { - terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; + code = TSDB_CODE_MND_INVALID_FUNC_CODE; goto _OVER; } if (createReq.codeLen <= 1) { - terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; + code = TSDB_CODE_MND_INVALID_FUNC_CODE; goto _OVER; } if (createReq.bufSize < 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) { - terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE; + code = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE; goto _OVER; } @@ -415,7 +462,7 @@ _OVER: mndReleaseFunc(pMnode, pFunc); tFreeSCreateFuncReq(&createReq); - return code; + TAOS_RETURN(code); } static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) { @@ -424,18 +471,13 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) { SFuncObj *pFunc = NULL; SDropFuncReq dropReq = {0}; - if (tDeserializeSDropFuncReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } + TAOS_CHECK_GOTO(tDeserializeSDropFuncReq(pReq->pCont, pReq->contLen, &dropReq), NULL, _OVER); mInfo("func:%s, start to drop", dropReq.name); - if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC) != 0) { - goto _OVER; - } + TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC), NULL, _OVER); if (dropReq.name[0] == 0) { - terrno = TSDB_CODE_MND_INVALID_FUNC_NAME; + code = TSDB_CODE_MND_INVALID_FUNC_NAME; goto _OVER; } @@ -446,7 +488,7 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) { code = 0; goto _OVER; } else { - terrno = TSDB_CODE_MND_FUNC_NOT_EXIST; + code = TSDB_CODE_MND_FUNC_NOT_EXIST; goto _OVER; } } @@ -460,7 +502,7 @@ _OVER: } mndReleaseFunc(pMnode, pFunc); - return code; + TAOS_RETURN(code); } static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) { @@ -470,25 +512,25 @@ static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) { SRetrieveFuncRsp retrieveRsp = {0}; if (tDeserializeSRetrieveFuncReq(pReq->pCont, pReq->contLen, &retrieveReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_INVALID_MSG; goto RETRIEVE_FUNC_OVER; } if (retrieveReq.numOfFuncs <= 0 || retrieveReq.numOfFuncs > TSDB_FUNC_MAX_RETRIEVE) { - terrno = TSDB_CODE_MND_INVALID_FUNC_RETRIEVE; + code = TSDB_CODE_MND_INVALID_FUNC_RETRIEVE; goto RETRIEVE_FUNC_OVER; } retrieveRsp.numOfFuncs = retrieveReq.numOfFuncs; retrieveRsp.pFuncInfos = taosArrayInit(retrieveReq.numOfFuncs, sizeof(SFuncInfo)); if (retrieveRsp.pFuncInfos == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto RETRIEVE_FUNC_OVER; } retrieveRsp.pFuncExtraInfos = taosArrayInit(retrieveReq.numOfFuncs, sizeof(SFuncExtraInfo)); if (retrieveRsp.pFuncExtraInfos == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto RETRIEVE_FUNC_OVER; } @@ -541,7 +583,7 @@ static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) { int32_t contLen = tSerializeSRetrieveFuncRsp(NULL, 0, &retrieveRsp); void *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto RETRIEVE_FUNC_OVER; } @@ -556,7 +598,7 @@ RETRIEVE_FUNC_OVER: tFreeSRetrieveFuncReq(&retrieveReq); tFreeSRetrieveFuncRsp(&retrieveRsp); - return code; + TAOS_RETURN(code); } static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int32_t len) { From fc0f6b94be5dfa6d5f220a2c684a1e402fc4be01 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Sat, 20 Jul 2024 10:11:37 +0800 Subject: [PATCH 06/43] test: add the more compilation cache directories in CI --- tests/parallel_test/container_build.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 5657f65743..1bedbf101e 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -61,7 +61,10 @@ docker run \ -v /root/go/pkg/mod:/root/go/pkg/mod \ -v /root/.cache/go-build:/root/.cache/go-build \ -v /root/.cos-local.1:/root/.cos-local.2 \ - -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \ + -v ${REP_REAL_PATH}/enterprise/contrib/grant-lib:${REP_DIR}/enterprise/contrib/grant-lib \ + -v ${REP_REAL_PATH}/community/tools/taosadapter:${REP_DIR}/community/tools/taosadapter \ + -v ${REP_REAL_PATH}/community/tools/taos-tools:${REP_DIR}/community/tools/taos-tools \ + -v ${REP_REAL_PATH}/community/tools/taosws-rs:${REP_DIR}/community/tools/taosws-rs \ -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ -v ${REP_REAL_PATH}/community/contrib/apr/:${REP_DIR}/community/contrib/apr \ -v ${REP_REAL_PATH}/community/contrib/apr-util/:${REP_DIR}/community/contrib/apr-util \ @@ -108,7 +111,10 @@ docker run \ -v /root/go/pkg/mod:/root/go/pkg/mod \ -v /root/.cache/go-build:/root/.cache/go-build \ -v /root/.cos-local.1:/root/.cos-local.2 \ - -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \ + -v ${REP_REAL_PATH}/enterprise/contrib/grant-lib:${REP_DIR}/enterprise/contrib/grant-lib \ + -v ${REP_REAL_PATH}/community/tools/taosadapter:${REP_DIR}/community/tools/taosadapter \ + -v ${REP_REAL_PATH}/community/tools/taos-tools:${REP_DIR}/community/tools/taos-tools \ + -v ${REP_REAL_PATH}/community/tools/taosws-rs:${REP_DIR}/community/tools/taosws-rs \ -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ -v ${REP_REAL_PATH}/community/contrib/apr/:${REP_DIR}/community/contrib/apr \ -v ${REP_REAL_PATH}/community/contrib/apr-util/:${REP_DIR}/community/contrib/apr-util \ From 7002935f750ac9f44afe2bf3851e105cbee5364a Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 22 Jul 2024 11:35:38 +0800 Subject: [PATCH 07/43] test: add the more compilation cache directories in CI --- tests/parallel_test/container_build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 1bedbf101e..85e3d2ab73 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -65,7 +65,6 @@ docker run \ -v ${REP_REAL_PATH}/community/tools/taosadapter:${REP_DIR}/community/tools/taosadapter \ -v ${REP_REAL_PATH}/community/tools/taos-tools:${REP_DIR}/community/tools/taos-tools \ -v ${REP_REAL_PATH}/community/tools/taosws-rs:${REP_DIR}/community/tools/taosws-rs \ - -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ -v ${REP_REAL_PATH}/community/contrib/apr/:${REP_DIR}/community/contrib/apr \ -v ${REP_REAL_PATH}/community/contrib/apr-util/:${REP_DIR}/community/contrib/apr-util \ -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ From 3e2632b6d6ee1df476ffb4caf20694d6a6747753 Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 22 Jul 2024 05:12:33 +0000 Subject: [PATCH 08/43] remove useless --- source/dnode/mnode/impl/src/mndDb.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index e1be5d6d79..8432855c6f 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -910,9 +910,7 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { } } - mInfo("mndCheckDbPrivilege1, %d", code); if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DB, NULL) != 0) { - mInfo("mndCheckDbPrivilege2, %d", code); goto _OVER; } From 49dde394235006eee17ace5184d9cee3e52b7d36 Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 22 Jul 2024 05:24:39 +0000 Subject: [PATCH 09/43] error str --- source/dnode/mnode/impl/src/mndDnode.c | 23 ++++++++++++++--------- source/dnode/mnode/impl/src/mndFunc.c | 4 ++-- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 65957dfd40..2c3a50401b 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -600,15 +600,20 @@ static int32_t mndUpdateDnodeObj(SMnode *pMnode, SDnodeObj *pDnode) { pDnode->updateTime = taosGetTimestampMs(); SSdbRaw *pCommitRaw = mndDnodeActionEncode(pDnode); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + if (pCommitRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _exit; + } + if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, tstrerror(code)); code = terrno; goto _exit; } (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); if ((code = mndTransPrepare(pMnode, pTrans)) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mError("trans:%d, failed to prepare since %s", pTrans->id, tstrerror(code)); goto _exit; } @@ -949,7 +954,7 @@ static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) { _OVER: if (code != 0) { - mError("failed to get dnode list since %s", terrstr()); + mError("failed to get dnode list since %s", tstrerror(code)); } tFreeSDnodeListRsp(&rsp); @@ -1063,7 +1068,7 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { _OVER: if (code != 0) { - mError("failed to get show variables info since %s", terrstr()); + mError("failed to get show variables info since %s", tstrerror(code)); } tFreeSShowVariablesRsp(&rsp); @@ -1111,7 +1116,7 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("dnode:%s:%d, failed to create since %s", createReq.fqdn, createReq.port, terrstr()); + mError("dnode:%s:%d, failed to create since %s", createReq.fqdn, createReq.port, tstrerror(code)); } mndReleaseDnode(pMnode, pDnode); @@ -1271,7 +1276,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { if (isonline && force) { code = TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE; - mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), + mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, tstrerror(code), numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); goto _OVER; } @@ -1279,7 +1284,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { bool isEmpty = mndIsEmptyDnode(pMnode, pDnode->id); if (!isonline && !force && !isEmpty) { code = TSDB_CODE_DNODE_OFFLINE; - mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), + mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, tstrerror(code), numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); goto _OVER; } @@ -1294,7 +1299,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("dnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); + mError("dnode:%d, failed to drop since %s", dropReq.dnodeId, tstrerror(code)); } mndReleaseDnode(pMnode, pDnode); diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index f5d9a7b8e2..b84936aa6d 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -457,7 +457,7 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) { _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("func:%s, failed to create since %s", createReq.name, terrstr()); + mError("func:%s, failed to create since %s", createReq.name, tstrerror(code)); } mndReleaseFunc(pMnode, pFunc); @@ -498,7 +498,7 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) { _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("func:%s, failed to drop since %s", dropReq.name, terrstr()); + mError("func:%s, failed to drop since %s", dropReq.name, tstrerror(code)); } mndReleaseFunc(pMnode, pFunc); From 5ce3e84254edbe823ecceaa9a7acf9fb9a09b13a Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 22 Jul 2024 13:30:34 +0800 Subject: [PATCH 10/43] test: add the more compilation cache directories in CI --- .../container_build_newmachine.sh | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100755 tests/parallel_test/container_build_newmachine.sh diff --git a/tests/parallel_test/container_build_newmachine.sh b/tests/parallel_test/container_build_newmachine.sh new file mode 100755 index 0000000000..dcf7835820 --- /dev/null +++ b/tests/parallel_test/container_build_newmachine.sh @@ -0,0 +1,88 @@ +#!/bin/bash +set -e + +function usage() { + echo "$0" + echo -e "\t -w work dir" + echo -e "\t -e enterprise edition" + echo -e "\t -t make thread count" + echo -e "\t -h help" +} + +ent=0 +while getopts "w:t:eh" opt; do + case $opt in + w) + WORKDIR=$OPTARG + ;; + e) + ent=1 + ;; + t) + THREAD_COUNT=$OPTARG + ;; + h) + usage + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" + usage + exit 0 + ;; + esac +done + +if [ -z "$WORKDIR" ]; then + usage + exit 1 +fi +# if [ -z "$THREAD_COUNT" ]; then +# THREAD_COUNT=1 +# fi + +ulimit -c unlimited + +if [ $ent -eq 0 ]; then + REP_DIR=/home/TDengine + REP_REAL_PATH=$WORKDIR/TDengine + REP_MOUNT_PARAM=$REP_REAL_PATH:/home/TDengine +else + REP_DIR=/home/TDinternal + REP_REAL_PATH=$WORKDIR/TDinternal + REP_MOUNT_PARAM=$REP_REAL_PATH:/home/TDinternal + +fi +date +docker run \ + -v $REP_MOUNT_PARAM \ + -v /root/.cargo/registry:/root/.cargo/registry \ + -v /root/.cargo/git:/root/.cargo/git \ + -v /root/go/pkg/mod:/root/go/pkg/mod \ + -v /root/.cache/go-build:/root/.cache/go-build \ + -v /root/.cos-local.1:/root/.cos-local.2 \ + --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=false -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true $CMAKE_BUILD_TYPE -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 " + # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ + +if [[ -d ${WORKDIR}/debugNoSan ]] ;then + echo "delete ${WORKDIR}/debugNoSan" + rm -rf ${WORKDIR}/debugNoSan +fi +if [[ -d ${WORKDIR}/debugSan ]] ;then + echo "delete ${WORKDIR}/debugSan" + rm -rf ${WORKDIR}/debugSan +fi + +if [ "$(uname -m)" = "aarch64" ] ;then + CMAKE_BUILD_TYPE="-DCMAKE_BUILD_TYPE=Debug" +else + CMAKE_BUILD_TYPE="-DCMAKE_BUILD_TYPE=Release" +fi + + +mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan +date + +ret=$? +exit $ret + From 2288e23a6bf9cf391f07432df247b6fe593d6be1 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 22 Jul 2024 13:54:44 +0800 Subject: [PATCH 11/43] test: add the more compilation cache directories in CI --- tests/parallel_test/container_build_newmachine.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/container_build_newmachine.sh b/tests/parallel_test/container_build_newmachine.sh index dcf7835820..3c87cf156f 100755 --- a/tests/parallel_test/container_build_newmachine.sh +++ b/tests/parallel_test/container_build_newmachine.sh @@ -61,7 +61,7 @@ docker run \ -v /root/go/pkg/mod:/root/go/pkg/mod \ -v /root/.cache/go-build:/root/.cache/go-build \ -v /root/.cos-local.1:/root/.cos-local.2 \ - --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=false -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true $CMAKE_BUILD_TYPE -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 " + --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true $CMAKE_BUILD_TYPE -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 " # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ if [[ -d ${WORKDIR}/debugNoSan ]] ;then From d48bb62d28b703a44a4a37e02e28010d527a737e Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 22 Jul 2024 06:13:48 +0000 Subject: [PATCH 12/43] check return --- source/dnode/mnode/impl/src/mndDnode.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 2c3a50401b..ab312576d4 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -572,7 +572,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { SStatisReq statisReq = {0}; int32_t code = -1; - TAOS_RETURN(tDeserializeSStatisReq(pReq->pCont, pReq->contLen, &statisReq)); + TAOS_CHECK_RETURN(tDeserializeSStatisReq(pReq->pCont, pReq->contLen, &statisReq)); if (tsMonitorLogProtocol) { mInfo("process statis req,\n %s", statisReq.pCont); @@ -1378,7 +1378,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { int32_t code = 0; SMnode *pMnode = pReq->info.node; SMCfgDnodeReq cfgReq = {0}; - TAOS_RETURN(tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq)); + TAOS_CHECK_RETURN(tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq)); int8_t updateIpWhiteList = 0; mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value); if ((code = mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE)) != 0) { @@ -1526,7 +1526,7 @@ static int32_t mndProcessCreateEncryptKeyReq(SRpcMsg *pReq) { #ifdef TD_ENTERPRISE SMnode *pMnode = pReq->info.node; SMCfgDnodeReq cfgReq = {0}; - TAOS_RETURN(tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq)); + TAOS_CHECK_RETURN(tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq)); if ((code = mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE)) != 0) { tFreeSMCfgDnodeReq(&cfgReq); From 862a40afa01a5ed814e9c308e2c1a98c7d0bd274 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 22 Jul 2024 17:30:49 +0800 Subject: [PATCH 13/43] fix:[TD-31017]process return value in mnode for tmq --- include/common/tmsg.h | 15 +- include/util/tdef.h | 3 +- source/client/src/clientTmq.c | 18 +- source/common/src/tmsg.c | 14 +- source/dnode/mnode/impl/inc/mndConsumer.h | 22 +- source/dnode/mnode/impl/inc/mndDef.h | 7 +- source/dnode/mnode/impl/inc/mndSubscribe.h | 5 +- source/dnode/mnode/impl/inc/mndTopic.h | 5 +- source/dnode/mnode/impl/src/mndConsumer.c | 507 ++++++-------- source/dnode/mnode/impl/src/mndDb.c | 3 - source/dnode/mnode/impl/src/mndDef.c | 74 +- source/dnode/mnode/impl/src/mndScheduler.c | 49 +- source/dnode/mnode/impl/src/mndSubscribe.c | 760 ++++++++++----------- source/dnode/mnode/impl/src/mndTopic.c | 462 +++++-------- source/dnode/mnode/impl/src/mndUser.c | 18 +- source/dnode/mnode/impl/src/mndVgroup.c | 9 - 16 files changed, 892 insertions(+), 1079 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a5dea8a44e..e9cde9e99b 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2828,19 +2828,24 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc return tlen; } -static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq* pReq) { +static FORCE_INLINE int32_t tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq* pReq) { buf = taosDecodeFixedI64(buf, &pReq->consumerId); buf = taosDecodeStringTo(buf, pReq->cgroup); buf = taosDecodeStringTo(buf, pReq->clientId); - int32_t topicNum; + int32_t topicNum = 0; buf = taosDecodeFixedI32(buf, &topicNum); pReq->topicNames = taosArrayInit(topicNum, sizeof(void*)); + if (pReq->topicNames == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } for (int32_t i = 0; i < topicNum; i++) { - char* name; + char* name = NULL; buf = taosDecodeString(buf, &name); - taosArrayPush(pReq->topicNames, &name); + if (taosArrayPush(pReq->topicNames, &name) == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } } buf = taosDecodeFixedI8(buf, &pReq->withTbName); @@ -2849,7 +2854,7 @@ static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq buf = taosDecodeFixedI8(buf, &pReq->resetOffsetCfg); buf = taosDecodeFixedI8(buf, &pReq->enableReplay); buf = taosDecodeFixedI8(buf, &pReq->enableBatchMeta); - return buf; + return 0; } typedef struct { diff --git a/include/util/tdef.h b/include/util/tdef.h index 9c2858ed30..53bb8a493c 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -556,7 +556,8 @@ typedef struct { char name[TSDB_LOG_VAR_LEN]; } SLogVar; -#define TMQ_SEPARATOR ':' +#define TMQ_SEPARATOR ":" +#define TMQ_SEPARATOR_CHAR ':' enum { SND_WORKER_TYPE__SHARED = 1, diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index c363686343..09fbed4ba4 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -456,11 +456,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse pOffset.consumerId = tmq->consumerId; pOffset.offset.val = *offset; - int32_t groupLen = strlen(tmq->groupId); - memcpy(pOffset.offset.subKey, tmq->groupId, groupLen); - pOffset.offset.subKey[groupLen] = TMQ_SEPARATOR; - strcpy(pOffset.offset.subKey + groupLen + 1, pTopicName); - + (void)snprintf(pOffset.offset.subKey, TSDB_SUBSCRIBE_KEY_LEN, "%s%s%s", tmq->groupId, TMQ_SEPARATOR, pTopicName); int32_t len = 0; int32_t code = 0; tEncodeSize(tEncodeMqVgOffset, &pOffset, len, code); @@ -1667,11 +1663,7 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) } void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) { - int32_t groupLen = strlen(tmq->groupId); - memcpy(pReq->subKey, tmq->groupId, groupLen); - pReq->subKey[groupLen] = TMQ_SEPARATOR; - strcpy(pReq->subKey + groupLen + 1, pTopic->topicName); - + (void)snprintf(pReq->subKey, TSDB_SUBSCRIBE_KEY_LEN, "%s%s%s", tmq->groupId, TMQ_SEPARATOR, pTopic->topicName); pReq->withTbName = tmq->withTbName; pReq->consumerId = tmq->consumerId; pReq->timeout = timeout; @@ -2902,11 +2894,7 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep SMqVgOffset pOffset = {0}; pOffset.consumerId = tmq->consumerId; - - int32_t groupLen = strlen(tmq->groupId); - memcpy(pOffset.offset.subKey, tmq->groupId, groupLen); - pOffset.offset.subKey[groupLen] = TMQ_SEPARATOR; - strcpy(pOffset.offset.subKey + groupLen + 1, tname); + (void)snprintf(pOffset.offset.subKey, TSDB_SUBSCRIBE_KEY_LEN, "%s%s%s", tmq->groupId, TMQ_SEPARATOR, tname); int32_t len = 0; tEncodeSize(tEncodeMqVgOffset, &pOffset, len, code); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 9efff055bf..a15afa64db 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -9197,23 +9197,23 @@ int32_t tDecodeSTqOffsetVal(SDecoder *pDecoder, STqOffsetVal *pOffsetVal) { int32_t tFormatOffset(char *buf, int32_t maxLen, const STqOffsetVal *pVal) { if (pVal->type == TMQ_OFFSET__RESET_NONE) { - snprintf(buf, maxLen, "none"); + (void)snprintf(buf, maxLen, "none"); } else if (pVal->type == TMQ_OFFSET__RESET_EARLIEST) { - snprintf(buf, maxLen, "earliest"); + (void)snprintf(buf, maxLen, "earliest"); } else if (pVal->type == TMQ_OFFSET__RESET_LATEST) { - snprintf(buf, maxLen, "latest"); + (void)snprintf(buf, maxLen, "latest"); } else if (pVal->type == TMQ_OFFSET__LOG) { - snprintf(buf, maxLen, "wal:%" PRId64, pVal->version); + (void)snprintf(buf, maxLen, "wal:%" PRId64, pVal->version); } else if (pVal->type == TMQ_OFFSET__SNAPSHOT_DATA || pVal->type == TMQ_OFFSET__SNAPSHOT_META) { if (IS_VAR_DATA_TYPE(pVal->primaryKey.type)) { char *tmp = taosMemoryCalloc(1, pVal->primaryKey.nData + 1); if (tmp == NULL) return TSDB_CODE_OUT_OF_MEMORY; - memcpy(tmp, pVal->primaryKey.pData, pVal->primaryKey.nData); - snprintf(buf, maxLen, "tsdb:%" PRId64 "|%" PRId64 ",pk type:%d,val:%s", pVal->uid, pVal->ts, + (void)memcpy(tmp, pVal->primaryKey.pData, pVal->primaryKey.nData); + (void)snprintf(buf, maxLen, "tsdb:%" PRId64 "|%" PRId64 ",pk type:%d,val:%s", pVal->uid, pVal->ts, pVal->primaryKey.type, tmp); taosMemoryFree(tmp); } else { - snprintf(buf, maxLen, "tsdb:%" PRId64 "|%" PRId64 ",pk type:%d,val:%" PRId64, pVal->uid, pVal->ts, + (void)snprintf(buf, maxLen, "tsdb:%" PRId64 "|%" PRId64 ",pk type:%d,val:%" PRId64, pVal->uid, pVal->ts, pVal->primaryKey.type, pVal->primaryKey.val); } } else { diff --git a/source/dnode/mnode/impl/inc/mndConsumer.h b/source/dnode/mnode/impl/inc/mndConsumer.h index 5184ad0eca..e1a790867e 100644 --- a/source/dnode/mnode/impl/inc/mndConsumer.h +++ b/source/dnode/mnode/impl/inc/mndConsumer.h @@ -30,10 +30,10 @@ enum { int32_t mndInitConsumer(SMnode *pMnode); void mndCleanupConsumer(SMnode *pMnode); -void mndSendConsumerMsg(SMnode *pMnode, int64_t consumerId, uint16_t msgType, SRpcHandleInfo* info); +int32_t mndSendConsumerMsg(SMnode *pMnode, int64_t consumerId, uint16_t msgType, SRpcHandleInfo* info); -SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int64_t consumerId); -void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer); +int32_t mndAcquireConsumer(SMnode *pMnode, int64_t consumerId, SMqConsumerObj** pConsumer); +void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer); SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer); SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw); @@ -43,6 +43,22 @@ int32_t mndSetConsumerDropLogs(STrans *pTrans, SMqConsumerObj *pConsumer); const char *mndConsumerStatusName(int status); +#define MND_TMQ_NULL_CHECK(c) \ + do { \ + if (c == NULL) { \ + code = TSDB_CODE_OUT_OF_MEMORY; \ + goto END; \ + } \ + } while (0) + +#define MND_TMQ_RETURN_CHECK(c) \ + do { \ + code = c; \ + if (code != 0) { \ + goto END; \ + } \ + } while (0) + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 089c4a10b3..5319cc88eb 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -622,7 +622,8 @@ typedef struct { int32_t resetOffsetCfg; } SMqConsumerObj; -SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType, char *topic, SCMSubscribeReq *subscribe); +int32_t tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType, + char *topic, SCMSubscribeReq *subscribe, SMqConsumerObj** ppConsumer); void tClearSMqConsumerObj(SMqConsumerObj* pConsumer); void tDeleteSMqConsumerObj(SMqConsumerObj* pConsumer); int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer); @@ -665,8 +666,8 @@ typedef struct { char* qmsg; // SubPlanToString } SMqSubscribeObj; -SMqSubscribeObj* tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]); -SMqSubscribeObj* tCloneSubscribeObj(const SMqSubscribeObj* pSub); +int32_t tNewSubscribeObj(const char *key, SMqSubscribeObj **ppSub); +int32_t tCloneSubscribeObj(const SMqSubscribeObj* pSub, SMqSubscribeObj **ppSub); void tDeleteSubscribeObj(SMqSubscribeObj* pSub); int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeObj* pSub); void* tDecodeSubscribeObj(const void* buf, SMqSubscribeObj* pSub, int8_t sver); diff --git a/source/dnode/mnode/impl/inc/mndSubscribe.h b/source/dnode/mnode/impl/inc/mndSubscribe.h index eb9902a75c..614c14c7d2 100644 --- a/source/dnode/mnode/impl/inc/mndSubscribe.h +++ b/source/dnode/mnode/impl/inc/mndSubscribe.h @@ -26,12 +26,9 @@ int32_t mndInitSubscribe(SMnode *pMnode); void mndCleanupSubscribe(SMnode *pMnode); int32_t mndGetGroupNumByTopic(SMnode *pMnode, const char *topicName); -SMqSubscribeObj *mndAcquireSubscribe(SMnode *pMnode, const char *CGroup, const char *topicName); -SMqSubscribeObj *mndAcquireSubscribeByKey(SMnode *pMnode, const char *key); +int32_t mndAcquireSubscribeByKey(SMnode *pMnode, const char *key, SMqSubscribeObj** pSub); void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub); -void mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName); - int32_t mndDropSubByTopic(SMnode *pMnode, STrans *pTrans, const char *topic); int32_t mndSetDropSubCommitLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub); diff --git a/source/dnode/mnode/impl/inc/mndTopic.h b/source/dnode/mnode/impl/inc/mndTopic.h index b2b86775ab..dc6c137455 100644 --- a/source/dnode/mnode/impl/inc/mndTopic.h +++ b/source/dnode/mnode/impl/inc/mndTopic.h @@ -25,13 +25,12 @@ extern "C" { int32_t mndInitTopic(SMnode *pMnode); void mndCleanupTopic(SMnode *pMnode); -SMqTopicObj *mndAcquireTopic(SMnode *pMnode, const char *topicName); +int32_t mndAcquireTopic(SMnode *pMnode, const char *topicName, SMqTopicObj **pTopic); void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic); int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); bool mndTopicExistsForDb(SMnode *pMnode, SDbObj *pDb); -const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]); +void mndTopicGetShowName(const char* fullTopic, char* topic); -int32_t mndSetTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic); int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics); #ifdef __cplusplus diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 7788bf005a..532a224732 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -40,7 +40,6 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg); static int32_t mndProcessAskEpReq(SRpcMsg *pMsg); static int32_t mndProcessMqHbReq(SRpcMsg *pMsg); static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg); -static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg); int32_t mndInitConsumer(SMnode *pMnode) { SSdbTable table = { @@ -57,7 +56,6 @@ int32_t mndInitConsumer(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_TMQ_HB, mndProcessMqHbReq); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_ASK_EP, mndProcessAskEpReq); // mndSetMsgHandle(pMnode, TDMT_MND_TMQ_TIMER, mndProcessMqTimerMsg); - mndSetMsgHandle(pMnode, TDMT_MND_TMQ_CONSUMER_RECOVER, mndProcessConsumerRecoverMsg); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, mndProcessConsumerClearMsg); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONSUMERS, mndRetrieveConsumer); @@ -68,13 +66,10 @@ int32_t mndInitConsumer(SMnode *pMnode) { void mndCleanupConsumer(SMnode *pMnode) {} -void mndSendConsumerMsg(SMnode *pMnode, int64_t consumerId, uint16_t msgType, SRpcHandleInfo *info) { - void *msg = rpcMallocCont(sizeof(int64_t)); - if (msg == NULL) { - mError("consumer:0x%" PRIx64 " failed to clear consumer due to out of memory. alloc size:%d", consumerId, - (int32_t)sizeof(int64_t)); - return; - } +int32_t mndSendConsumerMsg(SMnode *pMnode, int64_t consumerId, uint16_t msgType, SRpcHandleInfo *info) { + int32_t code = 0; + void *msg = rpcMallocCont(sizeof(int64_t)); + MND_TMQ_NULL_CHECK(msg); *(int64_t*)msg = consumerId; SRpcMsg rpcMsg = { @@ -85,108 +80,51 @@ void mndSendConsumerMsg(SMnode *pMnode, int64_t consumerId, uint16_t msgType, SR }; mInfo("mndSendConsumerMsg type:%d consumer:0x%" PRIx64, msgType, consumerId); - int32_t code = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); - if (code != 0){ - mError("consumer:%"PRId64" send consumer msg:%d error:%d", consumerId, msgType, code); - } + MND_TMQ_RETURN_CHECK(tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg)); + return code; + +END: + taosMemoryFree(msg); + return code; } static int32_t validateTopics(const SArray *pTopicList, SMnode *pMnode, const char *pUser, bool enableReplay) { SMqTopicObj *pTopic = NULL; int32_t code = 0; - int32_t lino = 0; int32_t numOfTopics = taosArrayGetSize(pTopicList); for (int32_t i = 0; i < numOfTopics; i++) { char *pOneTopic = taosArrayGetP(pTopicList, i); - pTopic = mndAcquireTopic(pMnode, pOneTopic); - if (pTopic == NULL) { // terrno has been set by callee function - code = TSDB_CODE_MND_RETURN_VALUE_NULL; - if (terrno != 0) code = terrno; - goto FAILED; - } - - TAOS_CHECK_GOTO(mndCheckTopicPrivilege(pMnode, pUser, MND_OPER_SUBSCRIBE, pTopic), &lino, FAILED); - - TAOS_CHECK_GOTO(grantCheckExpire(TSDB_GRANT_SUBSCRIPTION), &lino, FAILED); + MND_TMQ_RETURN_CHECK(mndAcquireTopic(pMnode, pOneTopic, &pTopic)); + MND_TMQ_RETURN_CHECK(mndCheckTopicPrivilege(pMnode, pUser, MND_OPER_SUBSCRIBE, pTopic)); + MND_TMQ_RETURN_CHECK(grantCheckExpire(TSDB_GRANT_SUBSCRIPTION)); if (enableReplay) { if (pTopic->subType != TOPIC_SUB_TYPE__COLUMN) { code = TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT; - goto FAILED; + goto END; } else if (pTopic->ntbUid == 0 && pTopic->ctbStbUid == 0) { SDbObj *pDb = mndAcquireDb(pMnode, pTopic->db); if (pDb == NULL) { code = TSDB_CODE_MND_RETURN_VALUE_NULL; - if (terrno != 0) code = terrno; - goto FAILED; + goto END; } if (pDb->cfg.numOfVgroups != 1) { mndReleaseDb(pMnode, pDb); code = TSDB_CODE_TMQ_REPLAY_NEED_ONE_VGROUP; - goto FAILED; + goto END; } mndReleaseDb(pMnode, pDb); } } - mndReleaseTopic(pMnode, pTopic); } - return 0; -FAILED: - mndReleaseTopic(pMnode, pTopic); - TAOS_RETURN(code); -} -static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) { - int32_t code = 0; - int32_t lino = 0; - SMnode *pMnode = pMsg->info.node; - SMqConsumerRecoverMsg *pRecoverMsg = pMsg->pCont; - SMqConsumerObj *pConsumerNew = NULL; - STrans *pTrans = NULL; - SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pRecoverMsg->consumerId); - if (pConsumer == NULL) { - mError("cannot find consumer %" PRId64 " when processing consumer recover msg", pRecoverMsg->consumerId); - code = TSDB_CODE_MND_RETURN_VALUE_NULL; - if (terrno != 0) code = terrno; - goto END; - } - - mInfo("receive consumer recover msg, consumer:0x%" PRIx64 " status:%d(%s)", pRecoverMsg->consumerId, - pConsumer->status, mndConsumerStatusName(pConsumer->status)); - - if (pConsumer->status != MQ_CONSUMER_STATUS_LOST) { - code = TSDB_CODE_MND_CONSUMER_NOT_READY; - goto END; - } - - pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup, CONSUMER_UPDATE_REC, NULL, NULL); - if (pConsumerNew == NULL){ - code = TSDB_CODE_MND_RETURN_VALUE_NULL; - if (terrno != 0) code = terrno; - goto END; - } - - pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "recover-csm"); - if (pTrans == NULL) { - code = TSDB_CODE_MND_RETURN_VALUE_NULL; - if (terrno != 0) code = terrno; - goto END; - } - - TAOS_CHECK_GOTO(validateTopics(pConsumer->assignedTopics, pMnode, pMsg->info.conn.user, false), &lino, END); - - TAOS_CHECK_GOTO(mndSetConsumerCommitLogs(pTrans, pConsumerNew), &lino, END); - - code = mndTransPrepare(pMnode, pTrans); END: - mndReleaseConsumer(pMnode, pConsumer); - tDeleteSMqConsumerObj(pConsumerNew); - mndTransDrop(pTrans); - TAOS_RETURN(code); + mndReleaseTopic(pMnode, pTopic); + return code; } static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) { @@ -196,55 +134,39 @@ static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) { SMqConsumerClearMsg *pClearMsg = pMsg->pCont; SMqConsumerObj *pConsumerNew = NULL; STrans *pTrans = NULL; + SMqConsumerObj *pConsumer = NULL; - SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pClearMsg->consumerId); - if (pConsumer == NULL) { - mError("consumer:0x%" PRIx64 " failed to be found to clear it", pClearMsg->consumerId); - TAOS_RETURN(code); - } - + MND_TMQ_RETURN_CHECK(mndAcquireConsumer(pMnode, pClearMsg->consumerId, &pConsumer)); mInfo("consumer:0x%" PRIx64 " needs to be cleared, status %s", pClearMsg->consumerId, mndConsumerStatusName(pConsumer->status)); - pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup, -1, NULL, NULL); - if (pConsumerNew == NULL){ - code = TSDB_CODE_MND_RETURN_VALUE_NULL; - if (terrno != 0) code = terrno; - goto END; - } - + MND_TMQ_RETURN_CHECK(tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup, -1, NULL, NULL, &pConsumerNew)); pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pMsg, "clear-csm"); - if (pTrans == NULL) { - code = TSDB_CODE_MND_RETURN_VALUE_NULL; - if (terrno != 0) code = terrno; - goto END; - } - - // this is the drop action, not the update action - TAOS_CHECK_GOTO(mndSetConsumerDropLogs(pTrans, pConsumerNew), &lino, END); - + MND_TMQ_NULL_CHECK(pTrans); + MND_TMQ_RETURN_CHECK(mndSetConsumerDropLogs(pTrans, pConsumerNew)); code = mndTransPrepare(pMnode, pTrans); END: mndReleaseConsumer(pMnode, pConsumer); tDeleteSMqConsumerObj(pConsumerNew); mndTransDrop(pTrans); - TAOS_RETURN(code); + return code; } static int32_t checkPrivilege(SMnode *pMnode, SMqConsumerObj *pConsumer, SMqHbRsp *rsp, char *user) { + int32_t code = 0; rsp->topicPrivileges = taosArrayInit(taosArrayGetSize(pConsumer->currentTopics), sizeof(STopicPrivilege)); - if (rsp->topicPrivileges == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } + MND_TMQ_NULL_CHECK(rsp->topicPrivileges); for (int32_t i = 0; i < taosArrayGetSize(pConsumer->currentTopics); i++) { char *topic = taosArrayGetP(pConsumer->currentTopics, i); - SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); - if (pTopic == NULL) { // terrno has been set by callee function + SMqTopicObj *pTopic = NULL; + code = mndAcquireTopic(pMnode, topic, &pTopic); + if (code != TDB_CODE_SUCCESS) { continue; } STopicPrivilege *data = taosArrayReserve(rsp->topicPrivileges, 1); - strcpy(data->topic, topic); + MND_TMQ_NULL_CHECK(data); + (void)strcpy(data->topic, topic); if (mndCheckTopicPrivilege(pMnode, user, MND_OPER_SUBSCRIBE, pTopic) != 0 || grantCheckExpire(TSDB_GRANT_SUBSCRIPTION) < 0) { data->noPrivilege = 1; @@ -253,22 +175,30 @@ static int32_t checkPrivilege(SMnode *pMnode, SMqConsumerObj *pConsumer, SMqHbRs } mndReleaseTopic(pMnode, pTopic); } - return 0; +END: + return code; } static void storeOffsetRows(SMnode *pMnode, SMqHbReq *req, SMqConsumerObj *pConsumer){ for (int i = 0; i < taosArrayGetSize(req->topics); i++) { TopicOffsetRows *data = taosArrayGet(req->topics, i); + if (data == NULL){ + continue; + } mInfo("heartbeat report offset rows.%s:%s", pConsumer->cgroup, data->topicName); - SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, data->topicName); - if (pSub == NULL) { + SMqSubscribeObj *pSub = NULL; + char key[TSDB_SUBSCRIBE_KEY_LEN] = {0}; + (void)snprintf(key, TSDB_SUBSCRIBE_KEY_LEN, "%s%s%s", pConsumer->cgroup, TMQ_SEPARATOR, data->topicName); + int32_t code = mndAcquireSubscribeByKey(pMnode, key, &pSub); + if (code != 0) { + mError("failed to acquire subscribe by key:%s, code:%d", key, code); continue; } taosWLockLatch(&pSub->lock); SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &pConsumer->consumerId, sizeof(int64_t)); if (pConsumerEp) { - taosArrayDestroy(pConsumerEp->offsetRows); + (void)taosArrayDestroy(pConsumerEp->offsetRows); pConsumerEp->offsetRows = data->offsetRows; data->offsetRows = NULL; } @@ -303,36 +233,24 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { SMqHbReq req = {0}; SMqHbRsp rsp = {0}; SMqConsumerObj *pConsumer = NULL; - - TAOS_CHECK_GOTO(tDeserializeSMqHbReq(pMsg->pCont, pMsg->contLen, &req), NULL, end); - + MND_TMQ_RETURN_CHECK(tDeserializeSMqHbReq(pMsg->pCont, pMsg->contLen, &req)); int64_t consumerId = req.consumerId; - pConsumer = mndAcquireConsumer(pMnode, consumerId); - if (pConsumer == NULL) { - mError("consumer:0x%" PRIx64 " not exist", consumerId); - code = TSDB_CODE_MND_CONSUMER_NOT_EXIST; - goto end; - } - - TAOS_CHECK_GOTO(checkPrivilege(pMnode, pConsumer, &rsp, pMsg->info.conn.user), NULL, end); - + MND_TMQ_RETURN_CHECK(mndAcquireConsumer(pMnode, consumerId, &pConsumer)); + MND_TMQ_RETURN_CHECK(checkPrivilege(pMnode, pConsumer, &rsp, pMsg->info.conn.user)); atomic_store_32(&pConsumer->hbStatus, 0); - int32_t status = atomic_load_32(&pConsumer->status); - if (status == MQ_CONSUMER_STATUS_LOST) { mInfo("try to recover consumer:0x%" PRIx64, consumerId); - mndSendConsumerMsg(pMnode, pConsumer->consumerId, TDMT_MND_TMQ_CONSUMER_RECOVER, &pMsg->info); + MND_TMQ_RETURN_CHECK(mndSendConsumerMsg(pMnode, pConsumer->consumerId, TDMT_MND_TMQ_CONSUMER_RECOVER, &pMsg->info)); } - storeOffsetRows(pMnode, &req, pConsumer); code = buildMqHbRsp(pMsg, &rsp); -end: +END: tDestroySMqHbRsp(&rsp); mndReleaseConsumer(pMnode, pConsumer); tDestroySMqHbReq(&req); - TAOS_RETURN(code); + return code; } static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t epoch, SMqAskEpRsp *rsp){ @@ -349,19 +267,22 @@ static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t e // handle all topics subscribed by this consumer for (int32_t i = 0; i < numOfTopics; i++) { char *topic = taosArrayGetP(pConsumer->currentTopics, i); - SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic); - // txn guarantees pSub is created - if (pSub == NULL) { + SMqSubscribeObj *pSub = NULL; + char key[TSDB_SUBSCRIBE_KEY_LEN] = {0}; + (void)snprintf(key, TSDB_SUBSCRIBE_KEY_LEN, "%s%s%s", pConsumer->cgroup, TMQ_SEPARATOR, topic); + int32_t code = mndAcquireSubscribeByKey(pMnode, key, &pSub); + if (code != 0) { continue; } taosRLockLatch(&pSub->lock); SMqSubTopicEp topicEp = {0}; - strcpy(topicEp.topic, topic); + (void)strcpy(topicEp.topic, topic); // 2.1 fetch topic schema - SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); - if (pTopic == NULL) { + SMqTopicObj *pTopic = NULL; + code = mndAcquireTopic(pMnode, topic, &pTopic); + if (code != TDB_CODE_SUCCESS) { taosRUnLockLatch(&pSub->lock); mndReleaseSubscribe(pMnode, pSub); continue; @@ -371,16 +292,27 @@ static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t e topicEp.schema.nCols = pTopic->schema.nCols; if (topicEp.schema.nCols) { topicEp.schema.pSchema = taosMemoryCalloc(topicEp.schema.nCols, sizeof(SSchema)); - memcpy(topicEp.schema.pSchema, pTopic->schema.pSchema, topicEp.schema.nCols * sizeof(SSchema)); + if (topicEp.schema.pSchema == NULL) { + taosRUnLockLatch(&pTopic->lock); + taosRUnLockLatch(&pSub->lock); + mndReleaseSubscribe(pMnode, pSub); + mndReleaseTopic(pMnode, pTopic); + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(topicEp.schema.pSchema, pTopic->schema.pSchema, topicEp.schema.nCols * sizeof(SSchema)); } taosRUnLockLatch(&pTopic->lock); mndReleaseTopic(pMnode, pTopic); // 2.2 iterate all vg assigned to the consumer of that topic SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &pConsumer->consumerId, sizeof(int64_t)); - int32_t vgNum = taosArrayGetSize(pConsumerEp->vgs); - - // this customer assigned vgroups + if (pConsumerEp == NULL) { + taosRUnLockLatch(&pConsumer->lock); + taosRUnLockLatch(&pSub->lock); + mndReleaseSubscribe(pMnode, pSub); + return TSDB_CODE_OUT_OF_MEMORY; + } + int32_t vgNum = taosArrayGetSize(pConsumerEp->vgs); topicEp.vgs = taosArrayInit(vgNum, sizeof(SMqSubVgEp)); if (topicEp.vgs == NULL) { taosRUnLockLatch(&pConsumer->lock); @@ -391,9 +323,9 @@ static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t e for (int32_t j = 0; j < vgNum; j++) { SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j); - // char offsetKey[TSDB_PARTITION_KEY_LEN]; - // mndMakePartitionKey(offsetKey, pConsumer->cgroup, topic, pVgEp->vgId); - + if (pVgEp == NULL) { + continue; + } if (epoch == -1) { SVgObj *pVgroup = mndAcquireVgroup(pMnode, pVgEp->vgId); if (pVgroup) { @@ -401,17 +333,10 @@ static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t e mndReleaseVgroup(pMnode, pVgroup); } } - // 2.2.1 build vg ep - SMqSubVgEp vgEp = { - .epSet = pVgEp->epSet, - .vgId = pVgEp->vgId, - .offset = -1, - }; - - taosArrayPush(topicEp.vgs, &vgEp); + SMqSubVgEp vgEp = {.epSet = pVgEp->epSet, .vgId = pVgEp->vgId, .offset = -1}; + (void)taosArrayPush(topicEp.vgs, &vgEp); } - taosArrayPush(rsp->topics, &topicEp); - + (void)taosArrayPush(rsp->topics, &topicEp); taosRUnLockLatch(&pSub->lock); mndReleaseSubscribe(pMnode, pSub); } @@ -437,12 +362,16 @@ static int32_t buildAskEpRsp(SRpcMsg *pMsg, SMqAskEpRsp *rsp, int32_t serverEpoc pHead->walever = 0; void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - tEncodeSMqAskEpRsp(&abuf, rsp); + code = tEncodeSMqAskEpRsp(&abuf, rsp); + if (code != 0) { + rpcFreeCont(buf); + return code; + } // send rsp pMsg->info.rsp = buf; pMsg->info.rspLen = tlen; - TAOS_RETURN(code); + return code; } static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { @@ -450,33 +379,22 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { SMqAskEpReq req = {0}; SMqAskEpRsp rsp = {0}; int32_t code = 0; + SMqConsumerObj *pConsumer = NULL; - TAOS_CHECK_RETURN(tDeserializeSMqAskEpReq(pMsg->pCont, pMsg->contLen, &req)); - + MND_TMQ_RETURN_CHECK(tDeserializeSMqAskEpReq(pMsg->pCont, pMsg->contLen, &req)); int64_t consumerId = req.consumerId; - - SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId); - if (pConsumer == NULL) { - mError("consumer:0x%" PRIx64 " group:%s not exists in sdb", consumerId, req.cgroup); - return TSDB_CODE_MND_CONSUMER_NOT_EXIST; - } - + MND_TMQ_RETURN_CHECK(mndAcquireConsumer(pMnode, consumerId, &pConsumer)); if (strncmp(req.cgroup, pConsumer->cgroup, tListLen(pConsumer->cgroup)) != 0) { mError("consumer:0x%" PRIx64 " group:%s not consistent with data in sdb, saved cgroup:%s", consumerId, req.cgroup, pConsumer->cgroup); code = TSDB_CODE_MND_CONSUMER_NOT_EXIST; goto END; } - atomic_store_32(&pConsumer->hbStatus, 0); - - // 1. check consumer status int32_t status = atomic_load_32(&pConsumer->status); - if (status == MQ_CONSUMER_STATUS_LOST) { - mndSendConsumerMsg(pMnode, pConsumer->consumerId, TDMT_MND_TMQ_CONSUMER_RECOVER, &pMsg->info); + MND_TMQ_RETURN_CHECK(mndSendConsumerMsg(pMnode, pConsumer->consumerId, TDMT_MND_TMQ_CONSUMER_RECOVER, &pMsg->info)); } - if (status != MQ_CONSUMER_STATUS_READY) { mInfo("consumer:0x%" PRIx64 " not ready, status: %s", consumerId, mndConsumerStatusName(status)); code = TSDB_CODE_MND_CONSUMER_NOT_READY; @@ -490,10 +408,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { if (epoch != serverEpoch) { mInfo("process ask ep, consumer:0x%" PRIx64 "(epoch %d) update with server epoch %d", consumerId, epoch, serverEpoch); - code = addEpSetInfo(pMnode, pConsumer, epoch, &rsp); - if(code != 0){ - goto END; - } + MND_TMQ_RETURN_CHECK(addEpSetInfo(pMnode, pConsumer, epoch, &rsp)); } code = buildAskEpRsp(pMsg, &rsp, serverEpoch, consumerId); @@ -507,27 +422,29 @@ END: int32_t mndSetConsumerDropLogs(STrans *pTrans, SMqConsumerObj *pConsumer) { int32_t code = 0; SSdbRaw *pCommitRaw = mndConsumerActionEncode(pConsumer); - if (pCommitRaw == NULL) { - code = TSDB_CODE_MND_RETURN_VALUE_NULL; - if (terrno != 0) code = terrno; - TAOS_RETURN(code); + MND_TMQ_NULL_CHECK(pCommitRaw); + code = mndTransAppendCommitlog(pTrans, pCommitRaw); + if (code != 0) { + sdbFreeRaw(pCommitRaw); + goto END; } - TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw)); - TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED)); - TAOS_RETURN(code); + MND_TMQ_RETURN_CHECK(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED)); +END: + return code; } int32_t mndSetConsumerCommitLogs(STrans *pTrans, SMqConsumerObj *pConsumer) { int32_t code = 0; SSdbRaw *pCommitRaw = mndConsumerActionEncode(pConsumer); - if (pCommitRaw == NULL) { - code = TSDB_CODE_MND_RETURN_VALUE_NULL; - if (terrno != 0) code = terrno; - TAOS_RETURN(code); + MND_TMQ_NULL_CHECK(pCommitRaw); + code = mndTransAppendCommitlog(pTrans, pCommitRaw); + if (code != 0) { + sdbFreeRaw(pCommitRaw); + goto END; } - TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw)); - TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)); - TAOS_RETURN(code); + MND_TMQ_RETURN_CHECK(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)); +END: + return code; } static void freeItem(void *param) { @@ -538,29 +455,35 @@ static void freeItem(void *param) { } static int32_t getTopicAddDelete(SMqConsumerObj *pExistedConsumer, SMqConsumerObj *pConsumerNew){ + int32_t code = 0; pConsumerNew->rebNewTopics = taosArrayInit(0, sizeof(void *)); + MND_TMQ_NULL_CHECK(pConsumerNew->rebNewTopics); pConsumerNew->rebRemovedTopics = taosArrayInit(0, sizeof(void *)); - if(pConsumerNew->rebNewTopics == NULL || pConsumerNew->rebRemovedTopics == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; - } + MND_TMQ_NULL_CHECK(pConsumerNew->rebRemovedTopics); + int32_t newTopicNum = taosArrayGetSize(pConsumerNew->assignedTopics); int32_t oldTopicNum = taosArrayGetSize(pExistedConsumer->currentTopics); - int32_t i = 0, j = 0; while (i < oldTopicNum || j < newTopicNum) { if (i >= oldTopicNum) { - char *newTopicCopy = taosStrdup(taosArrayGetP(pConsumerNew->assignedTopics, j)); - taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy); + void* tmp = taosArrayGetP(pConsumerNew->assignedTopics, j); + MND_TMQ_NULL_CHECK(tmp); + char *newTopicCopy = taosStrdup(tmp); + MND_TMQ_NULL_CHECK(taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy)); j++; continue; } else if (j >= newTopicNum) { - char *oldTopicCopy = taosStrdup(taosArrayGetP(pExistedConsumer->currentTopics, i)); - taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy); + void* tmp = taosArrayGetP(pExistedConsumer->currentTopics, i); + MND_TMQ_NULL_CHECK(tmp); + char *oldTopicCopy = taosStrdup(tmp); + MND_TMQ_NULL_CHECK(taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy)); i++; continue; } else { char *oldTopic = taosArrayGetP(pExistedConsumer->currentTopics, i); + MND_TMQ_NULL_CHECK(oldTopic); char *newTopic = taosArrayGetP(pConsumerNew->assignedTopics, j); + MND_TMQ_NULL_CHECK(newTopic); int comp = strcmp(oldTopic, newTopic); if (comp == 0) { i++; @@ -568,22 +491,22 @@ static int32_t getTopicAddDelete(SMqConsumerObj *pExistedConsumer, SMqConsumerOb continue; } else if (comp < 0) { char *oldTopicCopy = taosStrdup(oldTopic); - taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy); + MND_TMQ_NULL_CHECK(taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy)); i++; continue; } else { char *newTopicCopy = taosStrdup(newTopic); - taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy); + MND_TMQ_NULL_CHECK(taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy)); j++; continue; } } } - return 0; +END: + return code; } static int32_t checkAndSortTopic(SMnode *pMnode, SArray *pTopicList){ - int32_t code = 0; taosArraySort(pTopicList, taosArrayCompareString); taosArrayRemoveDuplicate(pTopicList, taosArrayCompareString, freeItem); @@ -594,24 +517,21 @@ static int32_t checkAndSortTopic(SMnode *pMnode, SArray *pTopicList){ return TSDB_CODE_TMQ_GROUP_OUT_OF_RANGE; } } - TAOS_RETURN(code); + return 0; } -static SMqConsumerObj* buildSubConsumer(SMnode *pMnode, SCMSubscribeReq *subscribe){ +static int32_t buildSubConsumer(SMnode *pMnode, SCMSubscribeReq *subscribe, SMqConsumerObj** ppConsumer){ int64_t consumerId = subscribe->consumerId; char *cgroup = subscribe->cgroup; SMqConsumerObj *pConsumerNew = NULL; SMqConsumerObj *pExistedConsumer = NULL; - pExistedConsumer = mndAcquireConsumer(pMnode, consumerId); - if (pExistedConsumer == NULL) { + int32_t code = mndAcquireConsumer(pMnode, consumerId, &pExistedConsumer); + if (code != 0) { mInfo("receive subscribe request from new consumer:0x%" PRIx64 ",cgroup:%s, numOfTopics:%d", consumerId, subscribe->cgroup, (int32_t)taosArrayGetSize(subscribe->topicNames)); - pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup, CONSUMER_INSERT_SUB, NULL, subscribe); - if (pConsumerNew == NULL) { - goto _over; - } + MND_TMQ_RETURN_CHECK(tNewSMqConsumerObj(consumerId, cgroup, CONSUMER_INSERT_SUB, NULL, subscribe, &pConsumerNew)); } else { int32_t status = atomic_load_32(&pExistedConsumer->status); @@ -621,82 +541,53 @@ static SMqConsumerObj* buildSubConsumer(SMnode *pMnode, SCMSubscribeReq *subscri (int32_t)taosArrayGetSize(subscribe->topicNames)); if (status != MQ_CONSUMER_STATUS_READY) { - terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; - goto _over; - } - pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup, CONSUMER_UPDATE_SUB, NULL, subscribe); - if (pConsumerNew == NULL) { - goto _over; - } - - int32_t code = getTopicAddDelete(pExistedConsumer, pConsumerNew); - if (code != 0){ - terrno = code; - goto _over; + code = TSDB_CODE_MND_CONSUMER_NOT_READY; + goto END; } + MND_TMQ_RETURN_CHECK(tNewSMqConsumerObj(consumerId, cgroup, CONSUMER_UPDATE_SUB, NULL, subscribe, &pConsumerNew)); + MND_TMQ_RETURN_CHECK(getTopicAddDelete(pExistedConsumer, pConsumerNew)); } mndReleaseConsumer(pMnode, pExistedConsumer); - return pConsumerNew; + if (ppConsumer){ + *ppConsumer = pConsumerNew; + } + return code; -_over: +END: mndReleaseConsumer(pMnode, pExistedConsumer); tDeleteSMqConsumerObj(pConsumerNew); - return NULL; + return code; } int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; char *msgStr = pMsg->pCont; int32_t code = 0; - - SCMSubscribeReq subscribe = {0}; - tDeserializeSCMSubscribeReq(msgStr, &subscribe); - SMqConsumerObj *pConsumerNew = NULL; STrans *pTrans = NULL; + SCMSubscribeReq subscribe = {0}; + MND_TMQ_RETURN_CHECK(tDeserializeSCMSubscribeReq(msgStr, &subscribe)); if(taosArrayGetSize(subscribe.topicNames) == 0){ - SMqConsumerObj *pConsumerTmp = mndAcquireConsumer(pMnode, subscribe.consumerId); - if(pConsumerTmp == NULL){ - goto _over; - } + SMqConsumerObj *pConsumerTmp = NULL; + MND_TMQ_RETURN_CHECK(mndAcquireConsumer(pMnode, subscribe.consumerId, &pConsumerTmp)); mndReleaseConsumer(pMnode, pConsumerTmp); } - - code = checkAndSortTopic(pMnode, subscribe.topicNames); - if(code != TSDB_CODE_SUCCESS){ - goto _over; - } - + MND_TMQ_RETURN_CHECK(checkAndSortTopic(pMnode, subscribe.topicNames)); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "subscribe"); - if (pTrans == NULL) { - code = TSDB_CODE_MND_RETURN_VALUE_NULL; - if (terrno != 0) code = terrno; - goto _over; - } + MND_TMQ_NULL_CHECK(pTrans); - code = validateTopics(subscribe.topicNames, pMnode, pMsg->info.conn.user, subscribe.enableReplay); - if (code != TSDB_CODE_SUCCESS) { - goto _over; - } - - pConsumerNew = buildSubConsumer(pMnode, &subscribe); - if(pConsumerNew == NULL){ - code = TSDB_CODE_MND_RETURN_VALUE_NULL; - if (terrno != 0) code = terrno; - goto _over; - } - - TAOS_CHECK_GOTO(mndSetConsumerCommitLogs(pTrans, pConsumerNew), NULL, _over); - - TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _over); + MND_TMQ_RETURN_CHECK(validateTopics(subscribe.topicNames, pMnode, pMsg->info.conn.user, subscribe.enableReplay)); + MND_TMQ_RETURN_CHECK(buildSubConsumer(pMnode, &subscribe, &pConsumerNew)); + MND_TMQ_RETURN_CHECK(mndSetConsumerCommitLogs(pTrans, pConsumerNew)); + MND_TMQ_RETURN_CHECK(mndTransPrepare(pMnode, pTrans)); code = TSDB_CODE_ACTION_IN_PROGRESS; -_over: +END: mndTransDrop(pTrans); tDeleteSMqConsumerObj(pConsumerNew); taosArrayDestroyP(subscribe.topicNames, (FDelete)taosMemoryFree); - TAOS_RETURN(code); + return code; } SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { @@ -838,8 +729,7 @@ static bool existInCurrentTopicList(const SMqConsumerObj *pConsumer, const char int32_t size = taosArrayGetSize(pConsumer->currentTopics); for (int32_t i = 0; i < size; i++) { char *topic = taosArrayGetP(pConsumer->currentTopics, i); - - if (strcmp(topic, pTopic) == 0) { + if (topic && strcmp(topic, pTopic) == 0) { existing = true; break; } @@ -865,32 +755,45 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, } else if (pNewConsumer->updateType == CONSUMER_UPDATE_REC) { int32_t sz = taosArrayGetSize(pOldConsumer->assignedTopics); for (int32_t i = 0; i < sz; i++) { - char *topic = taosStrdup(taosArrayGetP(pOldConsumer->assignedTopics, i)); - taosArrayPush(pOldConsumer->rebNewTopics, &topic); + void * tmp = taosArrayGetP(pOldConsumer->assignedTopics, i); + if (tmp == NULL){ + return TSDB_CODE_TMQ_INVALID_MSG; + } + char *topic = taosStrdup(tmp); + if (taosArrayPush(pOldConsumer->rebNewTopics, &topic) == NULL) { + taosMemoryFree(topic); + return TSDB_CODE_TMQ_INVALID_MSG; + } } pOldConsumer->status = MQ_CONSUMER_STATUS_REBALANCE; mInfo("consumer:0x%" PRIx64 " recover update", pOldConsumer->consumerId); } else if (pNewConsumer->updateType == CONSUMER_UPDATE_REB) { - atomic_add_fetch_32(&pOldConsumer->epoch, 1); - + (void)atomic_add_fetch_32(&pOldConsumer->epoch, 1); pOldConsumer->rebalanceTime = taosGetTimestampMs(); mInfo("[rebalance] consumer:0x%" PRIx64 " rebalance update, only rebalance time", pOldConsumer->consumerId); } else if (pNewConsumer->updateType == CONSUMER_ADD_REB) { - char *pNewTopic = taosStrdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0)); + void *tmp = taosArrayGetP(pNewConsumer->rebNewTopics, 0); + if (tmp == NULL){ + return TSDB_CODE_TMQ_INVALID_MSG; + } + char *pNewTopic = taosStrdup(tmp); removeFromTopicList(pOldConsumer->rebNewTopics, pNewTopic, pOldConsumer->consumerId, "new"); bool existing = existInCurrentTopicList(pOldConsumer, pNewTopic); if (existing) { mError("[rebalance] consumer:0x%" PRIx64 " add new topic:%s should not in currentTopics", pOldConsumer->consumerId, pNewTopic); taosMemoryFree(pNewTopic); } else { - taosArrayPush(pOldConsumer->currentTopics, &pNewTopic); + if (taosArrayPush(pOldConsumer->currentTopics, &pNewTopic) == NULL) { + taosMemoryFree(pNewTopic); + return TSDB_CODE_TMQ_INVALID_MSG; + } taosArraySort(pOldConsumer->currentTopics, taosArrayCompareString); } int32_t status = pOldConsumer->status; updateConsumerStatus(pOldConsumer); pOldConsumer->rebalanceTime = taosGetTimestampMs(); - atomic_add_fetch_32(&pOldConsumer->epoch, 1); + (void)atomic_add_fetch_32(&pOldConsumer->epoch, 1); mInfo("[rebalance] consumer:0x%" PRIx64 " rebalance update add, state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64 ", current topics:%d, newTopics:%d, removeTopics:%d", @@ -901,13 +804,16 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, } else if (pNewConsumer->updateType == CONSUMER_REMOVE_REB) { char *topic = taosArrayGetP(pNewConsumer->rebRemovedTopics, 0); + if (topic == NULL){ + return TSDB_CODE_TMQ_INVALID_MSG; + } removeFromTopicList(pOldConsumer->rebRemovedTopics, topic, pOldConsumer->consumerId, "remove"); removeFromTopicList(pOldConsumer->currentTopics, topic, pOldConsumer->consumerId, "current"); int32_t status = pOldConsumer->status; updateConsumerStatus(pOldConsumer); pOldConsumer->rebalanceTime = taosGetTimestampMs(); - atomic_add_fetch_32(&pOldConsumer->epoch, 1); + (void)atomic_add_fetch_32(&pOldConsumer->epoch, 1); mInfo("[rebalance]consumer:0x%" PRIx64 " rebalance update remove, state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64 ", current topics:%d, newTopics:%d, removeTopics:%d", @@ -921,13 +827,13 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, return 0; } -SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int64_t consumerId) { +int32_t mndAcquireConsumer(SMnode *pMnode, int64_t consumerId, SMqConsumerObj** pConsumer) { SSdb *pSdb = pMnode->pSdb; - SMqConsumerObj *pConsumer = sdbAcquire(pSdb, SDB_CONSUMER, &consumerId); - if (pConsumer == NULL) { - terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; + *pConsumer = sdbAcquire(pSdb, SDB_CONSUMER, &consumerId); + if (*pConsumer == NULL) { + return TSDB_CODE_MND_CONSUMER_NOT_EXIST; } - return pConsumer; + return 0; } void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer) { @@ -940,6 +846,7 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SMqConsumerObj *pConsumer = NULL; + int32_t code = 0; while (numOfRows < rowsCapacity) { pShow->pIter = sdbFetch(pSdb, SDB_CONSUMER, pShow->pIter, (void **)&pConsumer); @@ -964,34 +871,37 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * } if (numOfRows + topicSz > rowsCapacity) { - blockDataEnsureCapacity(pBlock, numOfRows + topicSz); + MND_TMQ_RETURN_CHECK(blockDataEnsureCapacity(pBlock, numOfRows + topicSz)); } for (int32_t i = 0; i < topicSz; i++) { - SColumnInfoData *pColInfo; + SColumnInfoData *pColInfo = NULL; int32_t cols = 0; // consumer id char consumerIdHex[32] = {0}; - sprintf(varDataVal(consumerIdHex), "0x%" PRIx64, pConsumer->consumerId); + (void)sprintf(varDataVal(consumerIdHex), "0x%" PRIx64, pConsumer->consumerId); varDataSetLen(consumerIdHex, strlen(varDataVal(consumerIdHex))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)consumerIdHex, false); + MND_TMQ_NULL_CHECK(pColInfo); + MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)consumerIdHex, false)); // consumer group char cgroup[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(cgroup, pConsumer->cgroup); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)cgroup, false); + MND_TMQ_NULL_CHECK(pColInfo); + MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)cgroup, false)); // client id char clientId[256 + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(clientId, pConsumer->clientId); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)clientId, false); + MND_TMQ_NULL_CHECK(pColInfo); + MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)clientId, false)); // status char status[20 + VARSTR_HEADER_SIZE] = {0}; @@ -999,46 +909,48 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * STR_TO_VARSTR(status, pStatusName); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)status, false); + MND_TMQ_NULL_CHECK(pColInfo); + MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)status, false)); // one subscribed topic pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + MND_TMQ_NULL_CHECK(pColInfo); if (hasTopic) { char topic[TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - const char *topicName = mndTopicGetShowName(taosArrayGetP(pConsumer->assignedTopics, i)); - STR_TO_VARSTR(topic, topicName); - colDataSetVal(pColInfo, numOfRows, (const char *)topic, false); + mndTopicGetShowName(taosArrayGetP(pConsumer->assignedTopics, i), topic + VARSTR_HEADER_SIZE); + *(VarDataLenT *)(topic) = strlen(topic + VARSTR_HEADER_SIZE); + MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)topic, false)); } else { - colDataSetVal(pColInfo, numOfRows, NULL, true); + MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, NULL, true)); } - // end point - /*pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);*/ - /*colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->ep, true);*/ - // up time pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->createTime, false); + MND_TMQ_NULL_CHECK(pColInfo); + MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->createTime, false)); // subscribe time pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->subscribeTime, false); + MND_TMQ_NULL_CHECK(pColInfo); + MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->subscribeTime, false)); // rebalance time pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->rebalanceTime, pConsumer->rebalanceTime == 0); + MND_TMQ_NULL_CHECK(pColInfo); + MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->rebalanceTime, pConsumer->rebalanceTime == 0)); char buf[TSDB_OFFSET_LEN] = {0}; STqOffsetVal pVal = {.type = pConsumer->resetOffsetCfg}; - tFormatOffset(buf, TSDB_OFFSET_LEN, &pVal); + MND_TMQ_RETURN_CHECK(tFormatOffset(buf, TSDB_OFFSET_LEN, &pVal)); char parasStr[64 + TSDB_OFFSET_LEN + VARSTR_HEADER_SIZE] = {0}; - sprintf(varDataVal(parasStr), "tbname:%d,commit:%d,interval:%dms,reset:%s", pConsumer->withTbName, + (void)sprintf(varDataVal(parasStr), "tbname:%d,commit:%d,interval:%dms,reset:%s", pConsumer->withTbName, pConsumer->autoCommit, pConsumer->autoCommitInterval, buf); varDataSetLen(parasStr, strlen(varDataVal(parasStr))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)parasStr, false); + MND_TMQ_NULL_CHECK(pColInfo); + MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)parasStr, false)); numOfRows++; } @@ -1051,6 +963,9 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * pShow->numOfRows += numOfRows; return numOfRows; + +END: + return code; } static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) { diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 8432855c6f..3e94411e52 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1525,9 +1525,6 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) { if (mndSetDropDbPrepareLogs(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) goto _OVER; - /*if (mndDropOffsetByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ - /*if (mndDropSubByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ - /*if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ if (mndDropStreamByDb(pMnode, pTrans, pDb) != 0) goto _OVER; #ifdef TD_ENTERPRISE if (mndDropViewByDb(pMnode, pTrans, pDb) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 9dc5f920ad..3aa0b5a01d 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -255,15 +255,17 @@ void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp, int8_t sver) { static void *topicNameDup(void *p) { return taosStrdup((char *)p); } -SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType, char *topic, SCMSubscribeReq *subscribe) { +int32_t tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType, + char *topic, SCMSubscribeReq *subscribe, SMqConsumerObj** ppConsumer) { + int32_t code = 0; SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj)); if (pConsumer == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + goto END; } pConsumer->consumerId = consumerId; - memcpy(pConsumer->cgroup, cgroup, TSDB_CGROUP_LEN); + (void)memcpy(pConsumer->cgroup, cgroup, TSDB_CGROUP_LEN); pConsumer->epoch = 0; pConsumer->status = MQ_CONSUMER_STATUS_REBALANCE; @@ -276,20 +278,26 @@ SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t upda if (updateType == CONSUMER_ADD_REB){ pConsumer->rebNewTopics = taosArrayInit(0, sizeof(void *)); if(pConsumer->rebNewTopics == NULL){ - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto END; } char* topicTmp = taosStrdup(topic); - taosArrayPush(pConsumer->rebNewTopics, &topicTmp); + if (taosArrayPush(pConsumer->rebNewTopics, &topicTmp) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto END; + } }else if (updateType == CONSUMER_REMOVE_REB) { pConsumer->rebRemovedTopics = taosArrayInit(0, sizeof(void *)); if(pConsumer->rebRemovedTopics == NULL){ - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto END; } char* topicTmp = taosStrdup(topic); - taosArrayPush(pConsumer->rebRemovedTopics, &topicTmp); + if (taosArrayPush(pConsumer->rebRemovedTopics, &topicTmp) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto END; + } }else if (updateType == CONSUMER_INSERT_SUB){ tstrncpy(pConsumer->clientId, subscribe->clientId, tListLen(pConsumer->clientId)); pConsumer->withTbName = subscribe->withTbName; @@ -297,10 +305,9 @@ SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t upda pConsumer->autoCommitInterval = subscribe->autoCommitInterval; pConsumer->resetOffsetCfg = subscribe->resetOffsetCfg; - pConsumer->rebNewTopics = taosArrayDup(subscribe->topicNames, topicNameDup); if (pConsumer->rebNewTopics == NULL){ - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto END; } pConsumer->assignedTopics = subscribe->topicNames; @@ -310,11 +317,12 @@ SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t upda subscribe->topicNames = NULL; } - return pConsumer; + *ppConsumer = pConsumer; + return 0; END: tDeleteSMqConsumerObj(pConsumer); - return NULL; + return code; } void tClearSMqConsumerObj(SMqConsumerObj *pConsumer) { @@ -530,27 +538,36 @@ void *tDecodeSMqConsumerEp(const void *buf, SMqConsumerEp *pConsumerEp, int8_t s return (void *)buf; } -SMqSubscribeObj *tNewSubscribeObj(const char *key) { +int32_t tNewSubscribeObj(const char *key, SMqSubscribeObj **ppSub) { + int32_t code = 0; SMqSubscribeObj *pSubObj = taosMemoryCalloc(1, sizeof(SMqSubscribeObj)); - if (pSubObj == NULL) { - return NULL; - } + MND_TMQ_NULL_CHECK(pSubObj); - memcpy(pSubObj->key, key, TSDB_SUBSCRIBE_KEY_LEN); + (void)memcpy(pSubObj->key, key, TSDB_SUBSCRIBE_KEY_LEN); taosInitRWLatch(&pSubObj->lock); pSubObj->vgNum = 0; pSubObj->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); - - // TODO set hash free fp - /*taosHashSetFreeFp(pSubObj->consumerHash, tDeleteSMqConsumerEp);*/ + MND_TMQ_NULL_CHECK(pSubObj->consumerHash); pSubObj->unassignedVgs = taosArrayInit(0, POINTER_BYTES); - return pSubObj; + MND_TMQ_NULL_CHECK(pSubObj->unassignedVgs); + if (ppSub){ + *ppSub = pSubObj; + } + return code; + +END: + taosMemoryFree(pSubObj); + return code; } -SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) { +int32_t tCloneSubscribeObj(const SMqSubscribeObj *pSub, SMqSubscribeObj **ppSub) { + int32_t code = 0; SMqSubscribeObj *pSubNew = taosMemoryMalloc(sizeof(SMqSubscribeObj)); - if (pSubNew == NULL) return NULL; - memcpy(pSubNew->key, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); + if (pSubNew == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto END; + } + (void)memcpy(pSubNew->key, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); taosInitRWLatch(&pSubNew->lock); pSubNew->dbUid = pSub->dbUid; @@ -560,8 +577,7 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) { pSubNew->vgNum = pSub->vgNum; pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); - // TODO set hash free fp - /*taosHashSetFreeFp(pSubNew->consumerHash, tDeleteSMqConsumerEp);*/ + void *pIter = NULL; SMqConsumerEp *pConsumerEp = NULL; while (1) { @@ -576,9 +592,11 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) { } pSubNew->unassignedVgs = taosArrayDup(pSub->unassignedVgs, (__array_item_dup_fn_t)tCloneSMqVgEp); pSubNew->offsetRows = taosArrayDup(pSub->offsetRows, NULL); - memcpy(pSubNew->dbName, pSub->dbName, TSDB_DB_FNAME_LEN); + (void)memcpy(pSubNew->dbName, pSub->dbName, TSDB_DB_FNAME_LEN); pSubNew->qmsg = taosStrdup(pSub->qmsg); - return pSubNew; + +END: + return code; } void tDeleteSubscribeObj(SMqSubscribeObj *pSub) { diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index db8c08e2e3..484d7bba28 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -759,25 +759,27 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib SVgObj* pVgroup = NULL; SQueryPlan* pPlan = NULL; SSubplan* pSubplan = NULL; + int32_t code = 0; if (pTopic->subType == TOPIC_SUB_TYPE__COLUMN) { pPlan = qStringToQueryPlan(pTopic->physicalPlan); if (pPlan == NULL) { - terrno = TSDB_CODE_QRY_INVALID_INPUT; - return -1; + return TSDB_CODE_QRY_INVALID_INPUT; } } else if (pTopic->subType == TOPIC_SUB_TYPE__TABLE && pTopic->ast != NULL) { SNode* pAst = NULL; - if (nodesStringToNode(pTopic->ast, &pAst) != 0) { + code = nodesStringToNode(pTopic->ast, &pAst); + if (code != 0) { mError("topic:%s, failed to create since %s", pTopic->name, terrstr()); - return -1; + return code; } SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true}; - if (qCreateQueryPlan(&cxt, &pPlan, NULL) != 0) { + code = qCreateQueryPlan(&cxt, &pPlan, NULL); + if (code != 0) { mError("failed to create topic:%s since %s", pTopic->name, terrstr()); nodesDestroyNode(pAst); - return -1; + return code; } nodesDestroyNode(pAst); } @@ -785,18 +787,19 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib if (pPlan) { int32_t levelNum = LIST_LENGTH(pPlan->pSubplans); if (levelNum != 1) { - qDestroyQueryPlan(pPlan); - terrno = TSDB_CODE_MND_INVALID_TOPIC_QUERY; - return -1; + code = TSDB_CODE_MND_INVALID_TOPIC_QUERY; + goto END; } SNodeListNode* pNodeListNode = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0); - + if (pNodeListNode == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto END; + } int32_t opNum = LIST_LENGTH(pNodeListNode->pNodeList); if (opNum != 1) { - qDestroyQueryPlan(pPlan); - terrno = TSDB_CODE_MND_INVALID_TOPIC_QUERY; - return -1; + code = TSDB_CODE_MND_INVALID_TOPIC_QUERY; + goto END; } pSubplan = (SSubplan*)nodesListGetNode(pNodeListNode->pNodeList, 0); @@ -817,12 +820,18 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib pSub->vgNum++; SMqVgEp* pVgEp = taosMemoryMalloc(sizeof(SMqVgEp)); + if (pVgEp == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto END; + } pVgEp->epSet = mndGetVgroupEpset(pMnode, pVgroup); pVgEp->vgId = pVgroup->vgId; - taosArrayPush(pSub->unassignedVgs, &pVgEp); - + if (taosArrayPush(pSub->unassignedVgs, &pVgEp) == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pVgEp); + goto END; + } mInfo("init subscription %s for topic:%s assign vgId:%d", pSub->key, pTopic->name, pVgEp->vgId); - sdbRelease(pSdb, pVgroup); } @@ -830,14 +839,14 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib int32_t msgLen; if (qSubPlanToString(pSubplan, &pSub->qmsg, &msgLen) < 0) { - qDestroyQueryPlan(pPlan); - terrno = TSDB_CODE_QRY_INVALID_INPUT; - return -1; + code = TSDB_CODE_QRY_INVALID_INPUT; + goto END; } } else { pSub->qmsg = taosStrdup(""); } +END: qDestroyQueryPlan(pPlan); - return 0; + return code; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index e2bedc258a..731e25175f 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -41,14 +41,21 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg); static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg); static int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter); -static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj *hash); +static int32_t mndCheckConsumer(SRpcMsg *pMsg, SHashObj *hash); static int32_t mndSetSubCommitLogs(STrans *pTrans, SMqSubscribeObj *pSub) { + int32_t code = 0; SSdbRaw *pCommitRaw = mndSubActionEncode(pSub); - if (pCommitRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; - return 0; + MND_TMQ_NULL_CHECK(pCommitRaw); + code = mndTransAppendCommitlog(pTrans, pCommitRaw); + if (code != 0) { + sdbFreeRaw(pCommitRaw); + goto END; + } + code = sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + +END: + return code; } int32_t mndInitSubscribe(SMnode *pMnode) { @@ -74,41 +81,37 @@ int32_t mndInitSubscribe(SMnode *pMnode) { return sdbSetTable(pMnode->pSdb, table); } -static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *subKey) { - SMqSubscribeObj *pSub = tNewSubscribeObj(subKey); - if (pSub == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } +static int32_t mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *subKey, SMqSubscribeObj** pSub) { + int32_t code = 0; + MND_TMQ_RETURN_CHECK(tNewSubscribeObj(subKey, pSub)); + (*pSub)->dbUid = pTopic->dbUid; + (*pSub)->stbUid = pTopic->stbUid; + (*pSub)->subType = pTopic->subType; + (*pSub)->withMeta = pTopic->withMeta; - pSub->dbUid = pTopic->dbUid; - pSub->stbUid = pTopic->stbUid; - pSub->subType = pTopic->subType; - pSub->withMeta = pTopic->withMeta; + MND_TMQ_RETURN_CHECK(mndSchedInitSubEp(pMnode, pTopic, *pSub)); + return code; - if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) { - tDeleteSubscribeObj(pSub); - taosMemoryFree(pSub); - return NULL; - } - - return pSub; +END: + tDeleteSubscribeObj(*pSub); + taosMemoryFree(*pSub); + return code; } static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg, SSubplan *pPlan) { SMqRebVgReq req = {0}; + int32_t code = 0; + SEncoder encoder = {0}; + req.oldConsumerId = pRebVg->oldConsumerId; req.newConsumerId = pRebVg->newConsumerId; req.vgId = pRebVg->pVgEp->vgId; if (pPlan) { pPlan->execNode.epSet = pRebVg->pVgEp->epSet; pPlan->execNode.nodeId = pRebVg->pVgEp->vgId; - int32_t msgLen; - if (qSubPlanToString(pPlan, &req.qmsg, &msgLen) < 0) { - terrno = TSDB_CODE_QRY_INVALID_INPUT; - return -1; - } + int32_t msgLen = 0; + MND_TMQ_RETURN_CHECK(qSubPlanToString(pPlan, &req.qmsg, &msgLen)); } else { req.qmsg = taosStrdup(""); } @@ -118,62 +121,47 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj tstrncpy(req.subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); int32_t tlen = 0; - int32_t ret = 0; - tEncodeSize(tEncodeSMqRebVgReq, &req, tlen, ret); - if (ret < 0) { - taosMemoryFree(req.qmsg); - return -1; + tEncodeSize(tEncodeSMqRebVgReq, &req, tlen, code); + if (code < 0) { + goto END; } tlen += sizeof(SMsgHead); void *buf = taosMemoryMalloc(tlen); - if (buf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(req.qmsg); - return -1; - } - + MND_TMQ_NULL_CHECK(buf); SMsgHead *pMsgHead = (SMsgHead *)buf; - pMsgHead->contLen = htonl(tlen); pMsgHead->vgId = htonl(pRebVg->pVgEp->vgId); - SEncoder encoder = {0}; tEncoderInit(&encoder, POINTER_SHIFT(buf, sizeof(SMsgHead)), tlen); - if (tEncodeSMqRebVgReq(&encoder, &req) < 0) { - taosMemoryFreeClear(buf); - tEncoderClear(&encoder); - taosMemoryFree(req.qmsg); - return -1; - } - tEncoderClear(&encoder); + MND_TMQ_RETURN_CHECK(tEncodeSMqRebVgReq(&encoder, &req)); *pBuf = buf; *pLen = tlen; +END: + tEncoderClear(&encoder); taosMemoryFree(req.qmsg); - return 0; + return code; } static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg, SSubplan *pPlan) { + int32_t code = 0; + void *buf = NULL; + if (pRebVg->oldConsumerId == pRebVg->newConsumerId) { if (pRebVg->oldConsumerId == -1) return 0; // drop stream, no consumer, while split vnode,all consumerId is -1 - terrno = TSDB_CODE_MND_INVALID_SUB_OPTION; - return -1; - } - - void *buf; - int32_t tlen; - if (mndBuildSubChangeReq(&buf, &tlen, pSub, pRebVg, pPlan) < 0) { - return -1; + code = TSDB_CODE_MND_INVALID_SUB_OPTION; + goto END; } + int32_t tlen = 0; + MND_TMQ_RETURN_CHECK(mndBuildSubChangeReq(&buf, &tlen, pSub, pRebVg, pPlan)); int32_t vgId = pRebVg->pVgEp->vgId; SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); if (pVgObj == NULL) { - taosMemoryFree(buf); - terrno = TSDB_CODE_MND_VGROUP_NOT_EXIST; - return -1; + code = TSDB_CODE_MND_VGROUP_NOT_EXIST; + goto END; } STransAction action = {0}; @@ -183,71 +171,87 @@ static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, SMqSubsc action.msgType = TDMT_VND_TMQ_SUBSCRIBE; mndReleaseVgroup(pMnode, pVgObj); - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(buf); - return -1; - } - return 0; + MND_TMQ_RETURN_CHECK(mndTransAppendRedoAction(pTrans, &action)); + return code; + +END: + taosMemoryFree(buf); + return code; } -static int32_t mndSplitSubscribeKey(const char *key, char *topic, char *cgroup, bool fullName) { +static void mndSplitSubscribeKey(const char *key, char *topic, char *cgroup, bool fullName) { int32_t i = 0; - while (key[i] != TMQ_SEPARATOR) { + while (key[i] != TMQ_SEPARATOR_CHAR) { i++; } - memcpy(cgroup, key, i); + (void)memcpy(cgroup, key, i); cgroup[i] = 0; if (fullName) { - strcpy(topic, &key[i + 1]); + (void)strcpy(topic, &key[i + 1]); } else { while (key[i] != '.') { i++; } - strcpy(topic, &key[i + 1]); + (void)strcpy(topic, &key[i + 1]); } - return 0; } -static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { - SMqRebInfo *pRebInfo = taosHashGet(pHash, key, strlen(key) + 1); +static int32_t mndGetOrCreateRebSub(SHashObj *pHash, const char *key, SMqRebInfo **pReb) { + int32_t code = 0; + SMqRebInfo* pRebInfo = taosHashGet(pHash, key, strlen(key) + 1); if (pRebInfo == NULL) { pRebInfo = tNewSMqRebSubscribe(key); if (pRebInfo == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + goto END; + } + code = taosHashPut(pHash, key, strlen(key) + 1, pRebInfo, sizeof(SMqRebInfo)); + taosMemoryFreeClear(pRebInfo); + if (code != 0) { + goto END; } - taosHashPut(pHash, key, strlen(key) + 1, pRebInfo, sizeof(SMqRebInfo)); - taosMemoryFree(pRebInfo); pRebInfo = taosHashGet(pHash, key, strlen(key) + 1); + MND_TMQ_NULL_CHECK(pRebInfo); } - return pRebInfo; + if (pReb){ + *pReb = pRebInfo; + } + +END: + return code; } -static void pushVgDataToHash(SArray *vgs, SHashObj *pHash, int64_t consumerId, char *key) { - SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(vgs); - SMqRebOutputVg outputVg = {consumerId, -1, pVgEp}; - taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("[rebalance] sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64, key, pVgEp->vgId, consumerId); +static int32_t pushVgDataToHash(SArray *vgs, SHashObj *pHash, int64_t consumerId, char *key) { + int32_t code = 0; + SMqVgEp **pVgEp = (SMqVgEp **)taosArrayPop(vgs); + MND_TMQ_NULL_CHECK(pVgEp); + SMqRebOutputVg outputVg = {consumerId, -1, *pVgEp}; + MND_TMQ_RETURN_CHECK(taosHashPut(pHash, &(*pVgEp)->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg))); + mInfo("[rebalance] sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64, key, (*pVgEp)->vgId, consumerId); +END: + return code; } -static void processRemovedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, const SMqRebInputObj *pInput) { +static int32_t processRemovedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, const SMqRebInputObj *pInput) { + int32_t code = 0; int32_t numOfRemoved = taosArrayGetSize(pInput->pRebInfo->removedConsumers); int32_t actualRemoved = 0; for (int32_t i = 0; i < numOfRemoved; i++) { - uint64_t consumerId = *(uint64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i); - SMqConsumerEp *pConsumerEp = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); + int64_t* consumerId = (int64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i); + MND_TMQ_NULL_CHECK(consumerId); + SMqConsumerEp *pConsumerEp = taosHashGet(pOutput->pSub->consumerHash, consumerId, sizeof(int64_t)); if (pConsumerEp == NULL) { continue; } int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs); for (int32_t j = 0; j < consumerVgNum; j++) { - pushVgDataToHash(pConsumerEp->vgs, pHash, consumerId, pOutput->pSub->key); + MND_TMQ_RETURN_CHECK(pushVgDataToHash(pConsumerEp->vgs, pHash, *consumerId, pOutput->pSub->key)); } - taosArrayDestroy(pConsumerEp->vgs); - taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); - taosArrayPush(pOutput->removedConsumers, &consumerId); + (void)taosArrayDestroy(pConsumerEp->vgs); + MND_TMQ_RETURN_CHECK(taosHashRemove(pOutput->pSub->consumerHash, consumerId, sizeof(int64_t))); + MND_TMQ_NULL_CHECK(taosArrayPush(pOutput->removedConsumers, consumerId)); actualRemoved++; } @@ -257,45 +261,42 @@ static void processRemovedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, c } else { mInfo("[rebalance] sub:%s removed %d consumers", pOutput->pSub->key, numOfRemoved); } +END: + return code; } -static void processNewConsumers(SMqRebOutputObj *pOutput, const SMqRebInputObj *pInput) { +static int32_t processNewConsumers(SMqRebOutputObj *pOutput, const SMqRebInputObj *pInput) { + int32_t code = 0; int32_t numOfNewConsumers = taosArrayGetSize(pInput->pRebInfo->newConsumers); for (int32_t i = 0; i < numOfNewConsumers; i++) { - int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i); - + int64_t* consumerId = (int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i); + MND_TMQ_NULL_CHECK(consumerId); SMqConsumerEp newConsumerEp = {0}; - newConsumerEp.consumerId = consumerId; + newConsumerEp.consumerId = *consumerId; newConsumerEp.vgs = taosArrayInit(0, sizeof(void *)); - - taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp)); - taosArrayPush(pOutput->newConsumers, &consumerId); - mInfo("[rebalance] sub:%s mq rebalance add new consumer:0x%" PRIx64, pOutput->pSub->key, consumerId); + MND_TMQ_NULL_CHECK(newConsumerEp.vgs); + MND_TMQ_RETURN_CHECK(taosHashPut(pOutput->pSub->consumerHash, consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp))); + MND_TMQ_NULL_CHECK(taosArrayPush(pOutput->newConsumers, consumerId)); + mInfo("[rebalance] sub:%s mq rebalance add new consumer:0x%" PRIx64, pOutput->pSub->key, *consumerId); } +END: + return code; } -static void processUnassignedVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash) { +static int32_t processUnassignedVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash) { + int32_t code = 0; int32_t numOfVgroups = taosArrayGetSize(pOutput->pSub->unassignedVgs); for (int32_t i = 0; i < numOfVgroups; i++) { - pushVgDataToHash(pOutput->pSub->unassignedVgs, pHash, -1, pOutput->pSub->key); + MND_TMQ_RETURN_CHECK(pushVgDataToHash(pOutput->pSub->unassignedVgs, pHash, -1, pOutput->pSub->key)); } +END: + return code; } -// static void putNoTransferToOutput(SMqRebOutputObj *pOutput, SMqConsumerEp *pConsumerEp){ -// for(int i = 0; i < taosArrayGetSize(pConsumerEp->vgs); i++){ -// SMqVgEp *pVgEp = (SMqVgEp *)taosArrayGetP(pConsumerEp->vgs, i); -// SMqRebOutputVg outputVg = { -// .oldConsumerId = pConsumerEp->consumerId, -// .newConsumerId = pConsumerEp->consumerId, -// .pVgEp = pVgEp, -// }; -// taosArrayPush(pOutput->rebVgs, &outputVg); -// } -// } - -static void processModifiedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt, +static int32_t processModifiedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt, int32_t remainderVgCnt) { + int32_t code = 0; int32_t cnt = 0; void *pIter = NULL; @@ -308,28 +309,31 @@ static void processModifiedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs); - taosArrayPush(pOutput->modifyConsumers, &pConsumerEp->consumerId); + MND_TMQ_NULL_CHECK(taosArrayPush(pOutput->modifyConsumers, &pConsumerEp->consumerId)); if (consumerVgNum > minVgCnt) { if (cnt < remainderVgCnt) { while (taosArrayGetSize(pConsumerEp->vgs) > minVgCnt + 1) { // pop until equal minVg + 1 - pushVgDataToHash(pConsumerEp->vgs, pHash, pConsumerEp->consumerId, pOutput->pSub->key); + MND_TMQ_RETURN_CHECK(pushVgDataToHash(pConsumerEp->vgs, pHash, pConsumerEp->consumerId, pOutput->pSub->key)); } cnt++; } else { while (taosArrayGetSize(pConsumerEp->vgs) > minVgCnt) { - pushVgDataToHash(pConsumerEp->vgs, pHash, pConsumerEp->consumerId, pOutput->pSub->key); + MND_TMQ_RETURN_CHECK(pushVgDataToHash(pConsumerEp->vgs, pHash, pConsumerEp->consumerId, pOutput->pSub->key)); } } } - // putNoTransferToOutput(pOutput, pConsumerEp); } +END: + return code; } static int32_t processRemoveAddVgs(SMnode *pMnode, SMqRebOutputObj *pOutput) { + int32_t code = 0; int32_t totalVgNum = 0; SVgObj *pVgroup = NULL; void *pIter = NULL; SArray *newVgs = taosArrayInit(0, POINTER_BYTES); + MND_TMQ_NULL_CHECK(newVgs); while (1) { pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); if (pIter == NULL) { @@ -343,9 +347,10 @@ static int32_t processRemoveAddVgs(SMnode *pMnode, SMqRebOutputObj *pOutput) { totalVgNum++; SMqVgEp *pVgEp = taosMemoryMalloc(sizeof(SMqVgEp)); + MND_TMQ_NULL_CHECK(pVgEp); pVgEp->epSet = mndGetVgroupEpset(pMnode, pVgroup); pVgEp->vgId = pVgroup->vgId; - taosArrayPush(newVgs, &pVgEp); + MND_TMQ_NULL_CHECK(taosArrayPush(newVgs, &pVgEp)); sdbRelease(pMnode->pSdb, pVgroup); } @@ -354,13 +359,14 @@ static int32_t processRemoveAddVgs(SMnode *pMnode, SMqRebOutputObj *pOutput) { pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); if (pIter == NULL) break; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; - int32_t j = 0; while (j < taosArrayGetSize(pConsumerEp->vgs)) { SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j); + MND_TMQ_NULL_CHECK(pVgEp); bool find = false; for (int32_t k = 0; k < taosArrayGetSize(newVgs); k++) { SMqVgEp *pnewVgEp = taosArrayGetP(newVgs, k); + MND_TMQ_NULL_CHECK(pnewVgEp); if (pVgEp->vgId == pnewVgEp->vgId) { tDeleteSMqVgEp(pnewVgEp); taosArrayRemove(newVgs, k); @@ -379,23 +385,29 @@ static int32_t processRemoveAddVgs(SMnode *pMnode, SMqRebOutputObj *pOutput) { } if (taosArrayGetSize(pOutput->pSub->unassignedVgs) == 0 && taosArrayGetSize(newVgs) != 0) { - taosArrayAddAll(pOutput->pSub->unassignedVgs, newVgs); + MND_TMQ_NULL_CHECK(taosArrayAddAll(pOutput->pSub->unassignedVgs, newVgs)); mInfo("[rebalance] processRemoveAddVgs add new vg num:%d", (int)taosArrayGetSize(newVgs)); - taosArrayDestroy(newVgs); + (void)taosArrayDestroy(newVgs); } else { - taosArrayDestroyP(newVgs, (FDelete)tDeleteSMqVgEp); + (void)taosArrayDestroyP(newVgs, (FDelete)tDeleteSMqVgEp); } return totalVgNum; +END: + return code; } -static void processSubOffsetRows(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { - SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pInput->pRebInfo->key); // put all offset rows - if (pSub == NULL) { - return; - } +static int32_t processSubOffsetRows(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { + int32_t code = 0; + SMqSubscribeObj *pSub = NULL; + MND_TMQ_RETURN_CHECK(mndAcquireSubscribeByKey(pMnode, pInput->pRebInfo->key, &pSub)); // put all offset rows taosRLockLatch(&pSub->lock); if (pOutput->pSub->offsetRows == NULL) { pOutput->pSub->offsetRows = taosArrayInit(4, sizeof(OffsetRows)); + if(pOutput->pSub->offsetRows == NULL) { + taosRUnLockLatch(&pSub->lock); + code = TSDB_CODE_OUT_OF_MEMORY; + goto END; + } } void *pIter = NULL; while (1) { @@ -406,9 +418,11 @@ static void processSubOffsetRows(SMnode *pMnode, const SMqRebInputObj *pInput, S for (int j = 0; j < taosArrayGetSize(pConsumerEp->offsetRows); j++) { OffsetRows *d1 = taosArrayGet(pConsumerEp->offsetRows, j); + MND_TMQ_NULL_CHECK(d1); bool jump = false; for (int i = 0; pConsumerEpNew && i < taosArrayGetSize(pConsumerEpNew->vgs); i++) { SMqVgEp *pVgEp = taosArrayGetP(pConsumerEpNew->vgs, i); + MND_TMQ_NULL_CHECK(pVgEp); if (pVgEp->vgId == d1->vgId) { jump = true; mInfo("pSub->offsetRows jump, because consumer id:0x%" PRIx64 " and vgId:%d not change", @@ -420,6 +434,7 @@ static void processSubOffsetRows(SMnode *pMnode, const SMqRebInputObj *pInput, S bool find = false; for (int i = 0; i < taosArrayGetSize(pOutput->pSub->offsetRows); i++) { OffsetRows *d2 = taosArrayGet(pOutput->pSub->offsetRows, i); + MND_TMQ_NULL_CHECK(d2); if (d1->vgId == d2->vgId) { d2->rows += d1->rows; d2->offset = d1->offset; @@ -430,18 +445,22 @@ static void processSubOffsetRows(SMnode *pMnode, const SMqRebInputObj *pInput, S } } if (!find) { - taosArrayPush(pOutput->pSub->offsetRows, d1); + MND_TMQ_NULL_CHECK(taosArrayPush(pOutput->pSub->offsetRows, d1)); } } } taosRUnLockLatch(&pSub->lock); mndReleaseSubscribe(pMnode, pSub); + +END: + return code; } static void printRebalanceLog(SMqRebOutputObj *pOutput) { mInfo("sub:%s mq rebalance calculation completed, re-balanced vg", pOutput->pSub->key); for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) { SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i); + if (pOutputRebVg == NULL) continue; mInfo("sub:%s mq rebalance vgId:%d, moved from consumer:0x%" PRIx64 ", to consumer:0x%" PRIx64, pOutput->pSub->key, pOutputRebVg->pVgEp->vgId, pOutputRebVg->oldConsumerId, pOutputRebVg->newConsumerId); } @@ -456,6 +475,7 @@ static void printRebalanceLog(SMqRebOutputObj *pOutput) { pConsumerEp->consumerId, sz); for (int32_t i = 0; i < sz; i++) { SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i); + if (pVgEp == NULL) continue; mInfo("sub:%s mq rebalance final cfg: vg %d to consumer:0x%" PRIx64, pOutput->pSub->key, pVgEp->vgId, pConsumerEp->consumerId); } @@ -480,10 +500,11 @@ static void calcVgroupsCnt(const SMqRebInputObj *pInput, int32_t totalVgNum, con pSubKey, totalVgNum, pInput->oldConsumerNum, numOfAdded, numOfRemoved, *minVgCnt, *remainderVgCnt); } -static void assignVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt) { +static int32_t assignVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt) { SMqRebOutputVg *pRebVg = NULL; void *pAssignIter = NULL; void *pIter = NULL; + int32_t code = 0; while (1) { pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); @@ -500,7 +521,7 @@ static void assignVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t min pRebVg = (SMqRebOutputVg *)pAssignIter; pRebVg->newConsumerId = pConsumerEp->consumerId; - taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); + MND_TMQ_NULL_CHECK(taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp)); mInfo("[rebalance] mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " for average", pRebVg->pVgEp->vgId, pConsumerEp->consumerId); } @@ -521,7 +542,7 @@ static void assignVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t min pRebVg = (SMqRebOutputVg *)pAssignIter; pRebVg->newConsumerId = pConsumerEp->consumerId; - taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); + MND_TMQ_NULL_CHECK(taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp)); mInfo("[rebalance] mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " for average + 1", pRebVg->pVgEp->vgId, pConsumerEp->consumerId); } @@ -530,6 +551,8 @@ static void assignVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t min taosHashCancelIterate(pOutput->pSub->consumerHash, pIter); if (pAssignIter != NULL) { mError("[rebalance]sub:%s assign iter is not NULL, never should reach here", pOutput->pSub->key); + code = TSDB_CODE_PAR_INTERNAL_ERROR; + goto END; } while (1) { pAssignIter = taosHashIterate(pHash, pAssignIter); @@ -538,30 +561,39 @@ static void assignVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t min } SMqRebOutputVg *pRebOutput = (SMqRebOutputVg *)pAssignIter; - taosArrayPush(pOutput->rebVgs, pRebOutput); + MND_TMQ_NULL_CHECK(taosArrayPush(pOutput->rebVgs, pRebOutput)); if (taosHashGetSize(pOutput->pSub->consumerHash) == 0) { // if all consumer is removed - taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp); // put all vg into unassigned + MND_TMQ_NULL_CHECK(taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp)); // put all vg into unassigned } } + +END: + return code; } -static void mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { +static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { int32_t totalVgNum = processRemoveAddVgs(pMnode, pOutput); + if (totalVgNum < 0){ + return totalVgNum; + } const char *pSubKey = pOutput->pSub->key; int32_t minVgCnt = 0; int32_t remainderVgCnt = 0; - - SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); - - processRemovedConsumers(pOutput, pHash, pInput); - processUnassignedVgroups(pOutput, pHash); + int32_t code = 0; + SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + MND_TMQ_NULL_CHECK(pHash); + MND_TMQ_RETURN_CHECK(processRemovedConsumers(pOutput, pHash, pInput)); + MND_TMQ_RETURN_CHECK(processUnassignedVgroups(pOutput, pHash)); calcVgroupsCnt(pInput, totalVgNum, pSubKey, &minVgCnt, &remainderVgCnt); - processModifiedConsumers(pOutput, pHash, minVgCnt, remainderVgCnt); - processNewConsumers(pOutput, pInput); - assignVgroups(pOutput, pHash, minVgCnt); - processSubOffsetRows(pMnode, pInput, pOutput); + MND_TMQ_RETURN_CHECK(processModifiedConsumers(pOutput, pHash, minVgCnt, remainderVgCnt)); + MND_TMQ_RETURN_CHECK(processNewConsumers(pOutput, pInput)); + MND_TMQ_RETURN_CHECK(assignVgroups(pOutput, pHash, minVgCnt)); + MND_TMQ_RETURN_CHECK(processSubOffsetRows(pMnode, pInput, pOutput)); printRebalanceLog(pOutput); taosHashCleanup(pHash); + +END: + return code; } static int32_t presistConsumerByType(STrans *pTrans, SArray *consumers, int8_t type, char *cgroup, char *topic) { @@ -569,18 +601,10 @@ static int32_t presistConsumerByType(STrans *pTrans, SArray *consumers, int8_t t SMqConsumerObj *pConsumerNew = NULL; int32_t consumerNum = taosArrayGetSize(consumers); for (int32_t i = 0; i < consumerNum; i++) { - int64_t consumerId = *(int64_t *)taosArrayGet(consumers, i); - pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup, type, topic, NULL); - if (pConsumerNew == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto END; - } - - code = mndSetConsumerCommitLogs(pTrans, pConsumerNew); - if (code != 0) { - goto END; - } - + int64_t* consumerId = (int64_t *)taosArrayGet(consumers, i); + MND_TMQ_NULL_CHECK(consumerId); + MND_TMQ_RETURN_CHECK(tNewSMqConsumerObj(*consumerId, cgroup, type, topic, NULL, &pConsumerNew)); + MND_TMQ_RETURN_CHECK(mndSetConsumerCommitLogs(pTrans, pConsumerNew)); tDeleteSMqConsumerObj(pConsumerNew); } pConsumerNew = NULL; @@ -591,17 +615,12 @@ END: } static int32_t mndPresistConsumer(STrans *pTrans, const SMqRebOutputObj *pOutput, char *cgroup, char *topic) { - int32_t code = presistConsumerByType(pTrans, pOutput->modifyConsumers, CONSUMER_UPDATE_REB, cgroup, NULL); - if (code != 0) { - return code; - } - - code = presistConsumerByType(pTrans, pOutput->newConsumers, CONSUMER_ADD_REB, cgroup, topic); - if (code != 0) { - return code; - } - - return presistConsumerByType(pTrans, pOutput->removedConsumers, CONSUMER_REMOVE_REB, cgroup, topic); + int32_t code = 0; + MND_TMQ_RETURN_CHECK(presistConsumerByType(pTrans, pOutput->modifyConsumers, CONSUMER_UPDATE_REB, cgroup, NULL)); + MND_TMQ_RETURN_CHECK(presistConsumerByType(pTrans, pOutput->newConsumers, CONSUMER_ADD_REB, cgroup, topic)); + MND_TMQ_RETURN_CHECK(presistConsumerByType(pTrans, pOutput->removedConsumers, CONSUMER_REMOVE_REB, cgroup, topic)); +END: + return code; } static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOutputObj *pOutput) { @@ -610,11 +629,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu STrans *pTrans = NULL; if (strcmp(pOutput->pSub->qmsg, "") != 0) { - code = qStringToSubplan(pOutput->pSub->qmsg, &pPlan); - if (code != 0) { - terrno = code; - goto END; - } + MND_TMQ_RETURN_CHECK(qStringToSubplan(pOutput->pSub->qmsg, &pPlan)); } char topic[TSDB_TOPIC_FNAME_LEN] = {0}; @@ -628,39 +643,28 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu } mndTransSetDbName(pTrans, pOutput->pSub->dbName, cgroup); - code = mndTransCheckConflict(pMnode, pTrans); - if (code != 0) { - goto END; - } + MND_TMQ_RETURN_CHECK(mndTransCheckConflict(pMnode, pTrans)); // 1. redo action: action to all vg const SArray *rebVgs = pOutput->rebVgs; int32_t vgNum = taosArrayGetSize(rebVgs); for (int32_t i = 0; i < vgNum; i++) { SMqRebOutputVg *pRebVg = taosArrayGet(rebVgs, i); - code = mndPersistSubChangeVgReq(pMnode, pTrans, pOutput->pSub, pRebVg, pPlan); - if (code != 0) { - goto END; - } + MND_TMQ_NULL_CHECK(pRebVg); + MND_TMQ_RETURN_CHECK(mndPersistSubChangeVgReq(pMnode, pTrans, pOutput->pSub, pRebVg, pPlan)); } // 2. commit log: subscribe and vg assignment - code = mndSetSubCommitLogs(pTrans, pOutput->pSub); - if (code != 0) { - goto END; - } + MND_TMQ_RETURN_CHECK(mndSetSubCommitLogs(pTrans, pOutput->pSub)); // 3. commit log: consumer to update status and epoch - code = mndPresistConsumer(pTrans, pOutput, cgroup, topic); - if (code != 0) { - goto END; - } + MND_TMQ_RETURN_CHECK(mndPresistConsumer(pTrans, pOutput, cgroup, topic)); // 4. set cb mndTransSetCb(pTrans, TRANS_START_FUNC_MQ_REB, TRANS_STOP_FUNC_MQ_REB, NULL, 0); // 5. execution - code = mndTransPrepare(pMnode, pTrans); + MND_TMQ_RETURN_CHECK(mndTransPrepare(pMnode, pTrans)); END: nodesDestroyNode((SNode *)pPlan); @@ -670,46 +674,63 @@ END: static void freeRebalanceItem(void *param) { SMqRebInfo *pInfo = param; - taosArrayDestroy(pInfo->newConsumers); - taosArrayDestroy(pInfo->removedConsumers); + (void)taosArrayDestroy(pInfo->newConsumers); + (void)taosArrayDestroy(pInfo->removedConsumers); } // type = 0 remove type = 1 add -static void buildRebInfo(SHashObj *rebSubHash, SArray *topicList, int8_t type, char *group, int64_t consumerId) { +static int32_t buildRebInfo(SHashObj *rebSubHash, SArray *topicList, int8_t type, char *group, int64_t consumerId) { + int32_t code = 0; int32_t topicNum = taosArrayGetSize(topicList); for (int32_t i = 0; i < topicNum; i++) { - char key[TSDB_SUBSCRIBE_KEY_LEN]; char *removedTopic = taosArrayGetP(topicList, i); - mndMakeSubscribeKey(key, group, removedTopic); - SMqRebInfo *pRebSub = mndGetOrCreateRebSub(rebSubHash, key); + MND_TMQ_NULL_CHECK(removedTopic); + char key[TSDB_SUBSCRIBE_KEY_LEN] = {0}; + (void)snprintf(key, TSDB_SUBSCRIBE_KEY_LEN, "%s%s%s", group, TMQ_SEPARATOR, removedTopic); + SMqRebInfo *pRebSub = NULL; + MND_TMQ_RETURN_CHECK(mndGetOrCreateRebSub(rebSubHash, key, &pRebSub)); if (type == 0) - taosArrayPush(pRebSub->removedConsumers, &consumerId); + MND_TMQ_NULL_CHECK(taosArrayPush(pRebSub->removedConsumers, &consumerId)); else if (type == 1) - taosArrayPush(pRebSub->newConsumers, &consumerId); + MND_TMQ_NULL_CHECK(taosArrayPush(pRebSub->newConsumers, &consumerId)); } + +END: + return code; } static void checkForVgroupSplit(SMnode *pMnode, SMqConsumerObj *pConsumer, SHashObj *rebSubHash) { int32_t newTopicNum = taosArrayGetSize(pConsumer->currentTopics); for (int32_t i = 0; i < newTopicNum; i++) { char *topic = taosArrayGetP(pConsumer->currentTopics, i); - SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic); - if (pSub == NULL) { + if (topic == NULL){ + continue; + } + SMqSubscribeObj *pSub = NULL; + char key[TSDB_SUBSCRIBE_KEY_LEN] = {0}; + (void)snprintf(key, TSDB_SUBSCRIBE_KEY_LEN, "%s%s%s", pConsumer->cgroup, TMQ_SEPARATOR, topic); + int32_t code = mndAcquireSubscribeByKey(pMnode, key, &pSub); + if (code != 0) { continue; } taosRLockLatch(&pSub->lock); // iterate all vg assigned to the consumer of that topic SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &pConsumer->consumerId, sizeof(int64_t)); - int32_t vgNum = taosArrayGetSize(pConsumerEp->vgs); - + if (pConsumerEp == NULL){ + taosRUnLockLatch(&pSub->lock); + mndReleaseSubscribe(pMnode, pSub); + continue; + } + int32_t vgNum = taosArrayGetSize(pConsumerEp->vgs); for (int32_t j = 0; j < vgNum; j++) { SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j); + if (pVgEp == NULL) { + continue; + } SVgObj *pVgroup = mndAcquireVgroup(pMnode, pVgEp->vgId); if (!pVgroup) { - char key[TSDB_SUBSCRIBE_KEY_LEN]; - mndMakeSubscribeKey(key, pConsumer->cgroup, topic); - mndGetOrCreateRebSub(rebSubHash, key); + (void)mndGetOrCreateRebSub(rebSubHash, key, NULL); mInfo("vnode splitted, vgId:%d rebalance will be triggered", pVgEp->vgId); } mndReleaseVgroup(pMnode, pVgroup); @@ -719,11 +740,12 @@ static void checkForVgroupSplit(SMnode *pMnode, SMqConsumerObj *pConsumer, SHash } } -static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj *rebSubHash) { +static int32_t mndCheckConsumer(SRpcMsg *pMsg, SHashObj *rebSubHash) { SMnode *pMnode = pMsg->info.node; SSdb *pSdb = pMnode->pSdb; SMqConsumerObj *pConsumer = NULL; void *pIter = NULL; + int32_t code = 0; // iterate all consumers, find all modification while (1) { @@ -742,27 +764,29 @@ static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj *rebSubHash) { if (status == MQ_CONSUMER_STATUS_READY) { if (taosArrayGetSize(pConsumer->assignedTopics) == 0) { // unsubscribe or close - mndSendConsumerMsg(pMnode, pConsumer->consumerId, TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, &pMsg->info); + MND_TMQ_RETURN_CHECK(mndSendConsumerMsg(pMnode, pConsumer->consumerId, TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, &pMsg->info)); } else if (hbStatus > MND_CONSUMER_LOST_HB_CNT) { taosRLockLatch(&pConsumer->lock); - buildRebInfo(rebSubHash, pConsumer->currentTopics, 0, pConsumer->cgroup, pConsumer->consumerId); + MND_TMQ_RETURN_CHECK(buildRebInfo(rebSubHash, pConsumer->currentTopics, 0, pConsumer->cgroup, pConsumer->consumerId)); taosRUnLockLatch(&pConsumer->lock); } else { checkForVgroupSplit(pMnode, pConsumer, rebSubHash); } } else if (status == MQ_CONSUMER_STATUS_LOST) { if (hbStatus > MND_CONSUMER_LOST_CLEAR_THRESHOLD) { // clear consumer if lost a day - mndSendConsumerMsg(pMnode, pConsumer->consumerId, TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, &pMsg->info); + MND_TMQ_RETURN_CHECK(mndSendConsumerMsg(pMnode, pConsumer->consumerId, TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, &pMsg->info)); } } else { taosRLockLatch(&pConsumer->lock); - buildRebInfo(rebSubHash, pConsumer->rebNewTopics, 1, pConsumer->cgroup, pConsumer->consumerId); - buildRebInfo(rebSubHash, pConsumer->rebRemovedTopics, 0, pConsumer->cgroup, pConsumer->consumerId); + MND_TMQ_RETURN_CHECK(buildRebInfo(rebSubHash, pConsumer->rebNewTopics, 1, pConsumer->cgroup, pConsumer->consumerId)); + MND_TMQ_RETURN_CHECK(buildRebInfo(rebSubHash, pConsumer->rebRemovedTopics, 0, pConsumer->cgroup, pConsumer->consumerId)); taosRUnLockLatch(&pConsumer->lock); } mndReleaseConsumer(pMnode, pConsumer); } +END: + return code; } bool mndRebTryStart() { @@ -781,31 +805,35 @@ void mndRebCntDec() { } static void clearRebOutput(SMqRebOutputObj *rebOutput) { - taosArrayDestroy(rebOutput->newConsumers); - taosArrayDestroy(rebOutput->modifyConsumers); - taosArrayDestroy(rebOutput->removedConsumers); - taosArrayDestroy(rebOutput->rebVgs); + (void)taosArrayDestroy(rebOutput->newConsumers); + (void)taosArrayDestroy(rebOutput->modifyConsumers); + (void)taosArrayDestroy(rebOutput->removedConsumers); + (void)taosArrayDestroy(rebOutput->rebVgs); tDeleteSubscribeObj(rebOutput->pSub); taosMemoryFree(rebOutput->pSub); } static int32_t initRebOutput(SMqRebOutputObj *rebOutput) { + int32_t code = 0; rebOutput->newConsumers = taosArrayInit(0, sizeof(int64_t)); + MND_TMQ_NULL_CHECK(rebOutput->newConsumers); rebOutput->removedConsumers = taosArrayInit(0, sizeof(int64_t)); + MND_TMQ_NULL_CHECK(rebOutput->removedConsumers); rebOutput->modifyConsumers = taosArrayInit(0, sizeof(int64_t)); + MND_TMQ_NULL_CHECK(rebOutput->modifyConsumers); rebOutput->rebVgs = taosArrayInit(0, sizeof(SMqRebOutputVg)); + MND_TMQ_NULL_CHECK(rebOutput->rebVgs); + return code; - if (rebOutput->newConsumers == NULL || rebOutput->removedConsumers == NULL || rebOutput->modifyConsumers == NULL || - rebOutput->rebVgs == NULL) { - clearRebOutput(rebOutput); - return TSDB_CODE_OUT_OF_MEMORY; - } - return 0; +END: + clearRebOutput(rebOutput); + return code; } // This function only works when there are dirty consumers -static void checkConsumer(SMnode *pMnode, SMqSubscribeObj *pSub) { - void *pIter = NULL; +static int32_t checkConsumer(SMnode *pMnode, SMqSubscribeObj *pSub) { + int32_t code = 0; + void *pIter = NULL; while (1) { pIter = taosHashIterate(pSub->consumerHash, pIter); if (pIter == NULL) { @@ -813,83 +841,89 @@ static void checkConsumer(SMnode *pMnode, SMqSubscribeObj *pSub) { } SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; - SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pConsumerEp->consumerId); - if (pConsumer != NULL) { + SMqConsumerObj *pConsumer = NULL; + code = mndAcquireConsumer(pMnode, pConsumerEp->consumerId, &pConsumer); + if (code == 0) { mndReleaseConsumer(pMnode, pConsumer); continue; } mError("consumer:0x%" PRIx64 " not exists in sdb for exception", pConsumerEp->consumerId); - taosArrayAddAll(pSub->unassignedVgs, pConsumerEp->vgs); + MND_TMQ_NULL_CHECK(taosArrayAddAll(pSub->unassignedVgs, pConsumerEp->vgs)); - taosArrayDestroy(pConsumerEp->vgs); - taosHashRemove(pSub->consumerHash, &pConsumerEp->consumerId, sizeof(int64_t)); + (void)taosArrayDestroy(pConsumerEp->vgs); + MND_TMQ_RETURN_CHECK(taosHashRemove(pSub->consumerHash, &pConsumerEp->consumerId, sizeof(int64_t))); } +END: + return code; } static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOutputObj *rebOutput) { + int32_t code = 0; const char *key = rebInput->pRebInfo->key; - SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, key); + SMqSubscribeObj *pSub = NULL; + MND_TMQ_RETURN_CHECK(mndAcquireSubscribeByKey(pMnode, key, &pSub)); if (pSub == NULL) { // split sub key and extract topic - char topic[TSDB_TOPIC_FNAME_LEN]; - char cgroup[TSDB_CGROUP_LEN]; + char topic[TSDB_TOPIC_FNAME_LEN] = {0}; + char cgroup[TSDB_CGROUP_LEN] = {0}; mndSplitSubscribeKey(key, topic, cgroup, true); - - SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); - if (pTopic == NULL) { - mError("[rebalance] mq rebalance %s ignored since topic %s doesn't exist", key, topic); - return -1; - } - + SMqTopicObj *pTopic = NULL; + MND_TMQ_RETURN_CHECK(mndAcquireTopic(pMnode, topic, &pTopic)); taosRLockLatch(&pTopic->lock); rebInput->oldConsumerNum = 0; - rebOutput->pSub = mndCreateSubscription(pMnode, pTopic, key); - - if (rebOutput->pSub == NULL) { + code = mndCreateSubscription(pMnode, pTopic, key, &rebOutput->pSub); + if (code != 0) { mError("[rebalance] mq rebalance %s failed create sub since %s, ignore", key, terrstr()); taosRUnLockLatch(&pTopic->lock); mndReleaseTopic(pMnode, pTopic); - return -1; + return code; } - memcpy(rebOutput->pSub->dbName, pTopic->db, TSDB_DB_FNAME_LEN); + (void)memcpy(rebOutput->pSub->dbName, pTopic->db, TSDB_DB_FNAME_LEN); taosRUnLockLatch(&pTopic->lock); mndReleaseTopic(pMnode, pTopic); mInfo("[rebalance] sub topic:%s has no consumers sub yet", key); } else { taosRLockLatch(&pSub->lock); - rebOutput->pSub = tCloneSubscribeObj(pSub); - checkConsumer(pMnode, rebOutput->pSub); + code = tCloneSubscribeObj(pSub, &rebOutput->pSub); + if(code != 0){ + taosRUnLockLatch(&pSub->lock); + goto END; + } + code =checkConsumer(pMnode, rebOutput->pSub); + if(code != 0){ + taosRUnLockLatch(&pSub->lock); + goto END; + } rebInput->oldConsumerNum = taosHashGetSize(rebOutput->pSub->consumerHash); taosRUnLockLatch(&pSub->lock); mInfo("[rebalance] sub topic:%s has %d consumers sub till now", key, rebInput->oldConsumerNum); mndReleaseSubscribe(pMnode, pSub); } - return 0; + +END: + return code; } static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { int code = 0; void *pIter = NULL; SMnode *pMnode = pMsg->info.node; - mDebug("[rebalance] start to process mq timer") - - if (!mndRebTryStart()) { - mInfo("[rebalance] mq rebalance already in progress, do nothing") return code; + mDebug("[rebalance] start to process mq timer"); + if (!mndRebTryStart()) { + mInfo("[rebalance] mq rebalance already in progress, do nothing"); + return code; } SHashObj *rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); - if (rebSubHash == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto END; - } + MND_TMQ_NULL_CHECK(rebSubHash); taosHashSetFreeFp(rebSubHash, freeRebalanceItem); - mndCheckConsumer(pMsg, rebSubHash); + MND_TMQ_RETURN_CHECK(mndCheckConsumer(pMsg, rebSubHash)); if (taosHashGetSize(rebSubHash) > 0) { mInfo("[rebalance] mq rebalance start, total required re-balanced trans:%d", taosHashGetSize(rebSubHash)) } @@ -902,21 +936,25 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { SMqRebInputObj rebInput = {0}; SMqRebOutputObj rebOutput = {0}; - code = initRebOutput(&rebOutput); - if (code != 0) { - goto END; - } - + MND_TMQ_RETURN_CHECK(initRebOutput(&rebOutput)); rebInput.pRebInfo = (SMqRebInfo *)pIter; - - if (buildRebOutput(pMnode, &rebInput, &rebOutput) != 0) { - continue; + code = buildRebOutput(pMnode, &rebInput, &rebOutput); + if (code != 0) { + mError("mq rebalance buildRebOutput, msg:%s", tstrerror(code)) } - mndDoRebalance(pMnode, &rebInput, &rebOutput); + if (code == 0){ + code = mndDoRebalance(pMnode, &rebInput, &rebOutput); + if (code != 0) { + mError("mq rebalance do rebalance error, msg:%s", tstrerror(code)) + } + } - if (mndPersistRebResult(pMnode, pMsg, &rebOutput) != 0) { - mError("mq rebalance persist output error, possibly vnode splitted or dropped,msg:%s", terrstr()) + if (code == 0){ + code = mndPersistRebResult(pMnode, pMsg, &rebOutput); + if (code != 0) { + mError("mq rebalance persist output error, possibly vnode splitted or dropped,msg:%s", tstrerror(code)) + } } clearRebOutput(&rebOutput); @@ -937,7 +975,7 @@ END: static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STrans *pTrans) { void *pIter = NULL; SVgObj *pVgObj = NULL; - int32_t ret = 0; + int32_t code = 0; while (1) { pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void **)&pVgObj); if (pIter == NULL) { @@ -949,40 +987,33 @@ static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STran continue; } SMqVDeleteReq *pReq = taosMemoryCalloc(1, sizeof(SMqVDeleteReq)); - if (pReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - ret = -1; - goto END; - } + MND_TMQ_NULL_CHECK(pReq); pReq->head.vgId = htonl(pVgObj->vgId); pReq->vgId = pVgObj->vgId; pReq->consumerId = -1; - memcpy(pReq->subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); + (void)memcpy(pReq->subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgObj); - ; action.pCont = pReq; action.contLen = sizeof(SMqVDeleteReq); action.msgType = TDMT_VND_TMQ_DELETE_SUB; action.acceptableCode = TSDB_CODE_MND_VGROUP_NOT_EXIST; sdbRelease(pMnode->pSdb, pVgObj); - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - ret = -1; - goto END; - } + MND_TMQ_RETURN_CHECK(mndTransAppendRedoAction(pTrans, &action)); } + END: sdbRelease(pMnode->pSdb, pVgObj); sdbCancelFetch(pMnode->pSdb, pIter); - return ret; + return code; } static int32_t mndDropConsumerByGroup(SMnode *pMnode, STrans *pTrans, char *cgroup, char *topic) { void *pIter = NULL; SMqConsumerObj *pConsumer = NULL; - int ret = 0; + int code = 0; while (1) { pIter = sdbFetch(pMnode->pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); if (pIter == NULL) { @@ -997,12 +1028,8 @@ static int32_t mndDropConsumerByGroup(SMnode *pMnode, STrans *pTrans, char *cgro int32_t sz = taosArrayGetSize(pConsumer->assignedTopics); for (int32_t i = 0; i < sz; i++) { char *name = taosArrayGetP(pConsumer->assignedTopics, i); - if (strcmp(topic, name) == 0) { - int32_t code = mndSetConsumerDropLogs(pTrans, pConsumer); - if (code != 0) { - ret = code; - goto END; - } + if (name && strcmp(topic, name) == 0) { + MND_TMQ_RETURN_CHECK(mndSetConsumerDropLogs(pTrans, pConsumer)); } } @@ -1012,7 +1039,7 @@ static int32_t mndDropConsumerByGroup(SMnode *pMnode, STrans *pTrans, char *cgro END: sdbRelease(pMnode->pSdb, pConsumer); sdbCancelFetch(pMnode->pSdb, pIter); - return ret; + return code; } static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) { @@ -1020,68 +1047,41 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) { SMDropCgroupReq dropReq = {0}; STrans *pTrans = NULL; int32_t code = TSDB_CODE_ACTION_IN_PROGRESS; + SMqSubscribeObj *pSub = NULL; - if (tDeserializeSMDropCgroupReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, dropReq.cgroup, dropReq.topic); - if (pSub == NULL) { + MND_TMQ_RETURN_CHECK(tDeserializeSMDropCgroupReq(pMsg->pCont, pMsg->contLen, &dropReq)); + char key[TSDB_SUBSCRIBE_KEY_LEN] = {0}; + (void)snprintf(key, TSDB_SUBSCRIBE_KEY_LEN, "%s%s%s", dropReq.cgroup, TMQ_SEPARATOR, dropReq.topic); + code = mndAcquireSubscribeByKey(pMnode, key, &pSub); + if (code != 0) { if (dropReq.igNotExists) { mInfo("cgroup:%s on topic:%s, not exist, ignore not exist is set", dropReq.cgroup, dropReq.topic); return 0; } else { - terrno = TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST; + code = TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST; mError("topic:%s, cgroup:%s, failed to drop since %s", dropReq.topic, dropReq.cgroup, terrstr()); - return -1; + return code; } } taosWLockLatch(&pSub->lock); if (taosHashGetSize(pSub->consumerHash) != 0) { - terrno = TSDB_CODE_MND_CGROUP_USED; + code = TSDB_CODE_MND_CGROUP_USED; mError("cgroup:%s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr()); - code = -1; - goto end; + goto END; } pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "drop-cgroup"); - if (pTrans == NULL) { - mError("cgroup: %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr()); - code = -1; - goto end; - } - + MND_TMQ_NULL_CHECK(pTrans); mInfo("trans:%d, used to drop cgroup:%s on topic %s", pTrans->id, dropReq.cgroup, dropReq.topic); mndTransSetDbName(pTrans, pSub->dbName, dropReq.cgroup); - code = mndTransCheckConflict(pMnode, pTrans); - if (code != 0) { - goto end; - } + MND_TMQ_RETURN_CHECK(mndTransCheckConflict(pMnode, pTrans)); + MND_TMQ_RETURN_CHECK(mndDropConsumerByGroup(pMnode, pTrans, dropReq.cgroup, dropReq.topic)); + MND_TMQ_RETURN_CHECK(sendDeleteSubToVnode(pMnode, pSub, pTrans)); + MND_TMQ_RETURN_CHECK(mndSetDropSubCommitLogs(pMnode, pTrans, pSub)); + MND_TMQ_RETURN_CHECK(mndTransPrepare(pMnode, pTrans)); - code = mndDropConsumerByGroup(pMnode, pTrans, dropReq.cgroup, dropReq.topic); - if (code != 0) { - goto end; - } - - code = sendDeleteSubToVnode(pMnode, pSub, pTrans); - if (code != 0) { - goto end; - } - - code = mndSetDropSubCommitLogs(pMnode, pTrans, pSub); - if (code != 0) { - mError("cgroup %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr()); - goto end; - } - - code = mndTransPrepare(pMnode, pTrans); - if (code != 0) { - goto end; - } - -end: +END: taosWUnLockLatch(&pSub->lock); mndReleaseSubscribe(pMnode, pSub); mndTransDrop(pTrans); @@ -1229,31 +1229,13 @@ static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubsc return 0; } -void mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName) { - int32_t tlen = strlen(cgroup); - memcpy(key, cgroup, tlen); - key[tlen] = TMQ_SEPARATOR; - strcpy(key + tlen + 1, topicName); -} - -SMqSubscribeObj *mndAcquireSubscribe(SMnode *pMnode, const char *cgroup, const char *topicName) { - SSdb *pSdb = pMnode->pSdb; - char key[TSDB_SUBSCRIBE_KEY_LEN]; - mndMakeSubscribeKey(key, cgroup, topicName); - SMqSubscribeObj *pSub = sdbAcquire(pSdb, SDB_SUBSCRIBE, key); - if (pSub == NULL) { - terrno = TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST; - } - return pSub; -} - -SMqSubscribeObj *mndAcquireSubscribeByKey(SMnode *pMnode, const char *key) { +int32_t mndAcquireSubscribeByKey(SMnode *pMnode, const char *key, SMqSubscribeObj** pSub) { SSdb *pSdb = pMnode->pSdb; - SMqSubscribeObj *pSub = sdbAcquire(pSdb, SDB_SUBSCRIBE, key); - if (pSub == NULL) { - terrno = TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST; + *pSub = sdbAcquire(pSdb, SDB_SUBSCRIBE, key); + if (*pSub == NULL) { + return TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST; } - return pSub; + return 0; } int32_t mndGetGroupNumByTopic(SMnode *pMnode, const char *topicName) { @@ -1286,20 +1268,18 @@ void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub) { sdbRelease(pSdb, pSub); } -static int32_t mndSetDropSubRedoLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) { - SSdbRaw *pRedoRaw = mndSubActionEncode(pSub); - if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; - if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPED) != 0) return -1; - return 0; -} - int32_t mndSetDropSubCommitLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) { + int32_t code = 0; SSdbRaw *pCommitRaw = mndSubActionEncode(pSub); - if (pCommitRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; - return 0; + MND_TMQ_NULL_CHECK(pCommitRaw); + code = mndTransAppendCommitlog(pTrans, pCommitRaw); + if (code != 0){ + sdbFreeRaw(pCommitRaw); + goto END; + } + code = sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); +END: + return code; } int32_t mndDropSubByTopic(SMnode *pMnode, STrans *pTrans, const char *topicName) { @@ -1325,15 +1305,8 @@ int32_t mndDropSubByTopic(SMnode *pMnode, STrans *pTrans, const char *topicName) goto END; } - code = sendDeleteSubToVnode(pMnode, pSub, pTrans); - if (code != 0) { - goto END; - } - - code = mndSetDropSubCommitLogs(pMnode, pTrans, pSub); - if (code != 0) { - goto END; - } + MND_TMQ_RETURN_CHECK(sendDeleteSubToVnode(pMnode, pSub, pTrans)); + MND_TMQ_RETURN_CHECK(mndSetDropSubCommitLogs(pMnode, pTrans, pSub)); } END: @@ -1345,22 +1318,27 @@ END: static int32_t buildResult(SSDataBlock *pBlock, int32_t *numOfRows, int64_t consumerId, const char *topic, const char *cgroup, SArray *vgs, SArray *offsetRows) { + int32_t code = 0; int32_t sz = taosArrayGetSize(vgs); for (int32_t j = 0; j < sz; j++) { SMqVgEp *pVgEp = taosArrayGetP(vgs, j); + MND_TMQ_NULL_CHECK(pVgEp); - SColumnInfoData *pColInfo; + SColumnInfoData *pColInfo = NULL; int32_t cols = 0; pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, *numOfRows, (const char *)topic, false); + MND_TMQ_NULL_CHECK(pColInfo); + MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, *numOfRows, (const char *)topic, false)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, *numOfRows, (const char *)cgroup, false); + MND_TMQ_NULL_CHECK(pColInfo); + MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, *numOfRows, (const char *)cgroup, false)); // vg id pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, *numOfRows, (const char *)&pVgEp->vgId, false); + MND_TMQ_NULL_CHECK(pColInfo); + MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, *numOfRows, (const char *)&pVgEp->vgId, false)); // consumer id char consumerIdHex[32] = {0}; @@ -1368,7 +1346,8 @@ static int32_t buildResult(SSDataBlock *pBlock, int32_t *numOfRows, int64_t cons varDataSetLen(consumerIdHex, strlen(varDataVal(consumerIdHex))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, *numOfRows, (const char *)consumerIdHex, consumerId == -1); + MND_TMQ_NULL_CHECK(pColInfo); + MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, *numOfRows, (const char *)consumerIdHex, consumerId == -1)); mInfo("mnd show subscriptions: topic %s, consumer:0x%" PRIx64 " cgroup %s vgid %d", varDataVal(topic), consumerId, varDataVal(cgroup), pVgEp->vgId); @@ -1377,6 +1356,7 @@ static int32_t buildResult(SSDataBlock *pBlock, int32_t *numOfRows, int64_t cons OffsetRows *data = NULL; for (int i = 0; i < taosArrayGetSize(offsetRows); i++) { OffsetRows *tmp = taosArrayGet(offsetRows, i); + MND_TMQ_NULL_CHECK(tmp); if (tmp->vgId != pVgEp->vgId) { mError("mnd show subscriptions: do not find vgId:%d, %d in offsetRows", tmp->vgId, pVgEp->vgId); continue; @@ -1386,23 +1366,29 @@ static int32_t buildResult(SSDataBlock *pBlock, int32_t *numOfRows, int64_t cons if (data) { // vg id char buf[TSDB_OFFSET_LEN * 2 + VARSTR_HEADER_SIZE] = {0}; - tFormatOffset(varDataVal(buf), TSDB_OFFSET_LEN, &data->offset); - sprintf(varDataVal(buf) + strlen(varDataVal(buf)), "/%" PRId64, data->ever); + MND_TMQ_RETURN_CHECK(tFormatOffset(varDataVal(buf), TSDB_OFFSET_LEN, &data->offset)); + (void)sprintf(varDataVal(buf) + strlen(varDataVal(buf)), "/%" PRId64, data->ever); varDataSetLen(buf, strlen(varDataVal(buf))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, *numOfRows, (const char *)buf, false); + MND_TMQ_NULL_CHECK(pColInfo); + MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, *numOfRows, (const char *)buf, false)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, *numOfRows, (const char *)&data->rows, false); + MND_TMQ_NULL_CHECK(pColInfo); + MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, *numOfRows, (const char *)&data->rows, false)); } else { pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + MND_TMQ_NULL_CHECK(pColInfo); colDataSetNULL(pColInfo, *numOfRows); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + MND_TMQ_NULL_CHECK(pColInfo); colDataSetNULL(pColInfo, *numOfRows); mError("mnd show subscriptions: do not find vgId:%d in offsetRows", pVgEp->vgId); } (*numOfRows)++; } return 0; +END: + return code; } int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) { @@ -1410,6 +1396,7 @@ int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SMqSubscribeObj *pSub = NULL; + int32_t code = 0; mInfo("mnd show subscriptions begin"); @@ -1422,7 +1409,7 @@ int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock taosRLockLatch(&pSub->lock); if (numOfRows + pSub->vgNum > rowsCapacity) { - blockDataEnsureCapacity(pBlock, numOfRows + pSub->vgNum); + MND_TMQ_RETURN_CHECK(blockDataEnsureCapacity(pBlock, numOfRows + pSub->vgNum)) ; } // topic and cgroup @@ -1439,11 +1426,11 @@ int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock if (pIter == NULL) break; pConsumerEp = (SMqConsumerEp *)pIter; - buildResult(pBlock, &numOfRows, pConsumerEp->consumerId, topic, cgroup, pConsumerEp->vgs, - pConsumerEp->offsetRows); + MND_TMQ_RETURN_CHECK(buildResult(pBlock, &numOfRows, pConsumerEp->consumerId, topic, cgroup, pConsumerEp->vgs, + pConsumerEp->offsetRows)); } - buildResult(pBlock, &numOfRows, -1, topic, cgroup, pSub->unassignedVgs, pSub->offsetRows); + MND_TMQ_RETURN_CHECK(buildResult(pBlock, &numOfRows, -1, topic, cgroup, pSub->unassignedVgs, pSub->offsetRows)); pBlock->info.rows = numOfRows; @@ -1455,6 +1442,9 @@ int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock pShow->numOfRows += numOfRows; return numOfRows; + +END: + return code; } void mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter) { diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index bcb38a3902..959a3ef008 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -44,8 +44,6 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq); static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter); -static int32_t mndSetDropTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic); - int32_t mndInitTopic(SMnode *pMnode) { SSdbTable table = { .sdbType = SDB_TOPIC, @@ -70,9 +68,16 @@ int32_t mndInitTopic(SMnode *pMnode) { void mndCleanupTopic(SMnode *pMnode) {} -const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]) { - // - return strchr(topic, '.') + 1; +void mndTopicGetShowName(const char* fullTopic, char* topic) { + if (fullTopic == NULL) { + return; + } + char* tmp = strchr(fullTopic, '.'); + if (tmp == NULL) { + tstrncpy(topic, fullTopic, TSDB_TOPIC_FNAME_LEN); + }else { + tstrncpy(topic, tmp+1, TSDB_TOPIC_FNAME_LEN); + } } SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { @@ -289,25 +294,25 @@ static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic) { taosMemoryFreeClear(pTopic->ast); taosMemoryFreeClear(pTopic->physicalPlan); if (pTopic->schema.nCols) taosMemoryFreeClear(pTopic->schema.pSchema); - taosArrayDestroy(pTopic->ntbColIds); + (void)taosArrayDestroy(pTopic->ntbColIds); return 0; } static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pOldTopic, SMqTopicObj *pNewTopic) { mTrace("topic:%s perform update action", pOldTopic->name); - atomic_exchange_64(&pOldTopic->updateTime, pNewTopic->updateTime); - atomic_exchange_32(&pOldTopic->version, pNewTopic->version); + (void)atomic_exchange_64(&pOldTopic->updateTime, pNewTopic->updateTime); + (void)atomic_exchange_32(&pOldTopic->version, pNewTopic->version); return 0; } -SMqTopicObj *mndAcquireTopic(SMnode *pMnode, const char *topicName) { +int32_t mndAcquireTopic(SMnode *pMnode, const char *topicName, SMqTopicObj **pTopic) { SSdb *pSdb = pMnode->pSdb; - SMqTopicObj *pTopic = sdbAcquire(pSdb, SDB_TOPIC, topicName); - if (pTopic == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { - terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST; + *pTopic = sdbAcquire(pSdb, SDB_TOPIC, topicName); + if (*pTopic == NULL) { + return TSDB_CODE_MND_TOPIC_NOT_EXIST; } - return pTopic; + return TDB_CODE_SUCCESS; } void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic) { @@ -316,25 +321,23 @@ void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic) { } static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) { - terrno = TSDB_CODE_MND_INVALID_TOPIC; - - if (pCreate->sql == NULL) return -1; + if (pCreate->sql == NULL) return TSDB_CODE_MND_INVALID_TOPIC; if (pCreate->subType == TOPIC_SUB_TYPE__COLUMN) { - if (pCreate->ast == NULL || pCreate->ast[0] == 0) return -1; + if (pCreate->ast == NULL || pCreate->ast[0] == 0) return TSDB_CODE_MND_INVALID_TOPIC; } else if (pCreate->subType == TOPIC_SUB_TYPE__TABLE) { - if (pCreate->subStbName[0] == 0) return -1; + if (pCreate->subStbName[0] == 0) return TSDB_CODE_MND_INVALID_TOPIC; } else if (pCreate->subType == TOPIC_SUB_TYPE__DB) { - if (pCreate->subDbName[0] == 0) return -1; + if (pCreate->subDbName[0] == 0) return TSDB_CODE_MND_INVALID_TOPIC; } - terrno = TSDB_CODE_SUCCESS; return 0; } static int32_t extractTopicTbInfo(SNode *pAst, SMqTopicObj *pTopic) { SNodeList *pNodeList = NULL; - nodesCollectColumns((SSelectStmt *)pAst, SQL_CLAUSE_FROM, NULL, COLLECT_COL_TYPE_ALL, &pNodeList); + int32_t code = 0; + MND_TMQ_RETURN_CHECK(nodesCollectColumns((SSelectStmt *)pAst, SQL_CLAUSE_FROM, NULL, COLLECT_COL_TYPE_ALL, &pNodeList)); int64_t suid = ((SRealTableNode *)((SSelectStmt *)pAst)->pFromTable)->pMeta->suid; int8_t tableType = ((SRealTableNode *)((SSelectStmt *)pAst)->pFromTable)->pMeta->tableType; if (tableType == TSDB_CHILD_TABLE) { @@ -345,17 +348,19 @@ static int32_t extractTopicTbInfo(SNode *pAst, SMqTopicObj *pTopic) { SColumnNode *pCol = (SColumnNode *)pNode; if (pCol->tableType == TSDB_NORMAL_TABLE) { pTopic->ntbUid = pCol->tableId; - taosArrayPush(pTopic->ntbColIds, &pCol->colId); + MND_TMQ_NULL_CHECK(taosArrayPush(pTopic->ntbColIds, &pCol->colId)); } } } nodesDestroyList(pNodeList); - return 0; + +END: + return code; } static int32_t sendCheckInfoToVnode(STrans *pTrans, SMnode *pMnode, SMqTopicObj *topicObj){ - STqCheckInfo info; - memcpy(info.topic, topicObj->name, TSDB_TOPIC_FNAME_LEN); + STqCheckInfo info = {0}; + (void)memcpy(info.topic, topicObj->name, TSDB_TOPIC_FNAME_LEN); info.ntbUid = topicObj->ntbUid; info.colIdList = topicObj->ntbColIds; // broadcast forbid alter info @@ -363,7 +368,7 @@ static int32_t sendCheckInfoToVnode(STrans *pTrans, SMnode *pMnode, SMqTopicObj SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; int32_t code = 0; - void *buf = NULL; + void *buf = NULL; while (1) { // iterate vg @@ -378,16 +383,17 @@ static int32_t sendCheckInfoToVnode(STrans *pTrans, SMnode *pMnode, SMqTopicObj int32_t len; tEncodeSize(tEncodeSTqCheckInfo, &info, len, code); if (code != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto END; } buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); + MND_TMQ_NULL_CHECK(buf); void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - SEncoder encoder; + SEncoder encoder = {0}; tEncoderInit(&encoder, abuf, len); code = tEncodeSTqCheckInfo(&encoder, &info); if (code < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + tEncoderClear(&encoder); goto END; } tEncoderClear(&encoder); @@ -398,10 +404,7 @@ static int32_t sendCheckInfoToVnode(STrans *pTrans, SMnode *pMnode, SMqTopicObj action.pCont = buf; action.contLen = sizeof(SMsgHead) + len; action.msgType = TDMT_VND_TMQ_ADD_CHECKINFO; - code = mndTransAppendRedoAction(pTrans, &action); - if (code != 0) { - goto END; - } + MND_TMQ_RETURN_CHECK(mndTransAppendRedoAction(pTrans, &action)); sdbRelease(pSdb, pVgroup); buf = NULL; } @@ -417,33 +420,22 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * const char *userName) { mInfo("start to create topic:%s", pCreate->name); STrans *pTrans = NULL; - int32_t code = -1; + int32_t code = 0; SNode *pAst = NULL; SQueryPlan *pPlan = NULL; SMqTopicObj topicObj = {0}; pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq, "create-topic"); - if (pTrans == NULL) { - mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); - code = -1; - goto _OUT; - } - + MND_TMQ_NULL_CHECK(pTrans); mndTransSetDbName(pTrans, pDb->name, NULL); - code = mndTransCheckConflict(pMnode, pTrans); - if (code != 0) { - goto _OUT; - } + MND_TMQ_RETURN_CHECK(mndTransCheckConflict(pMnode, pTrans)); mInfo("trans:%d to create topic:%s", pTrans->id, pCreate->name); tstrncpy(topicObj.name, pCreate->name, TSDB_TOPIC_FNAME_LEN); tstrncpy(topicObj.db, pDb->name, TSDB_DB_FNAME_LEN); tstrncpy(topicObj.createUser, userName, TSDB_USER_LEN); - code = mndCheckTopicPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_TOPIC, &topicObj); - if (code != 0) { - goto _OUT; - } + MND_TMQ_RETURN_CHECK(mndCheckTopicPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_TOPIC, &topicObj)); topicObj.createTime = taosGetTimestampMs(); topicObj.updateTime = topicObj.createTime; @@ -458,56 +450,25 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * if (pCreate->subType == TOPIC_SUB_TYPE__COLUMN) { topicObj.ast = taosStrdup(pCreate->ast); topicObj.astLen = strlen(pCreate->ast) + 1; - qDebugL("topic:%s ast %s", topicObj.name, topicObj.ast); - - code = nodesStringToNode(pCreate->ast, &pAst); - if (code != 0) { - mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); - goto _OUT; - } - + MND_TMQ_RETURN_CHECK(nodesStringToNode(pCreate->ast, &pAst)); SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true}; - code = qCreateQueryPlan(&cxt, &pPlan, NULL); - if (code != 0) { - mError("failed to create topic:%s since %s", pCreate->name, terrstr()); - goto _OUT; - } + MND_TMQ_RETURN_CHECK(qCreateQueryPlan(&cxt, &pPlan, NULL)); topicObj.ntbColIds = taosArrayInit(0, sizeof(int16_t)); - if (topicObj.ntbColIds == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = terrno; - goto _OUT; - } - - extractTopicTbInfo(pAst, &topicObj); - + MND_TMQ_NULL_CHECK(topicObj.ntbColIds); + MND_TMQ_RETURN_CHECK(extractTopicTbInfo(pAst, &topicObj)); if (topicObj.ntbUid == 0) { - taosArrayDestroy(topicObj.ntbColIds); + (void)taosArrayDestroy(topicObj.ntbColIds); topicObj.ntbColIds = NULL; } - code = qExtractResultSchema(pAst, &topicObj.schema.nCols, &topicObj.schema.pSchema); - if (code != 0) { - mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); - goto _OUT; - } - - code = nodesNodeToString((SNode *)pPlan, false, &topicObj.physicalPlan, NULL); - if (code != 0) { - mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); - goto _OUT; - } + MND_TMQ_RETURN_CHECK(qExtractResultSchema(pAst, &topicObj.schema.nCols, &topicObj.schema.pSchema)); + MND_TMQ_RETURN_CHECK(nodesNodeToString((SNode *)pPlan, false, &topicObj.physicalPlan, NULL)); } else if (pCreate->subType == TOPIC_SUB_TYPE__TABLE) { SStbObj *pStb = mndAcquireStb(pMnode, pCreate->subStbName); - if (pStb == NULL) { - terrno = TSDB_CODE_MND_STB_NOT_EXIST; - code = terrno; - goto _OUT; - } - - strcpy(topicObj.stbName, pCreate->subStbName); + MND_TMQ_NULL_CHECK(pStb); + tstrncpy(topicObj.stbName, pCreate->subStbName, TSDB_TABLE_FNAME_LEN); topicObj.stbUid = pStb->uid; mndReleaseStb(pMnode, pStb); if(pCreate->ast != NULL){ @@ -518,34 +479,25 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * } SSdbRaw *pCommitRaw = mndTopicActionEncode(&topicObj); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - code = -1; - goto _OUT; + MND_TMQ_NULL_CHECK(pCommitRaw); + code = mndTransAppendCommitlog(pTrans, pCommitRaw); + if(code != 0) { + sdbFreeRaw(pCommitRaw); + goto END; } - (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); - + MND_TMQ_RETURN_CHECK(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)); if (topicObj.ntbUid != 0) { - code = sendCheckInfoToVnode(pTrans, pMnode, &topicObj); - if (code != 0){ - goto _OUT; - } + MND_TMQ_RETURN_CHECK(sendCheckInfoToVnode(pTrans, pMnode, &topicObj)); } - - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - code = -1; - goto _OUT; - } - + MND_TMQ_RETURN_CHECK(mndTransPrepare(pMnode, pTrans)); code = TSDB_CODE_ACTION_IN_PROGRESS; -_OUT: +END: taosMemoryFreeClear(topicObj.physicalPlan); taosMemoryFreeClear(topicObj.sql); taosMemoryFreeClear(topicObj.ast); - taosArrayDestroy(topicObj.ntbColIds); + (void)taosArrayDestroy(topicObj.ntbColIds); if (topicObj.schema.nCols) { taosMemoryFreeClear(topicObj.schema.pSchema); } @@ -557,75 +509,64 @@ _OUT: static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; - int32_t code = -1; + int32_t code = TDB_CODE_SUCCESS; SMqTopicObj *pTopic = NULL; SDbObj *pDb = NULL; SCMCreateTopicReq createTopicReq = {0}; if (tDeserializeSCMCreateTopicReq(pReq->pCont, pReq->contLen, &createTopicReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; + code = TSDB_CODE_INVALID_MSG; + goto END; } mInfo("topic:%s start to create, sql:%s", createTopicReq.name, createTopicReq.sql); - if (mndCheckCreateTopicReq(&createTopicReq) != 0) { - mError("topic:%s failed to create since %s", createTopicReq.name, terrstr()); - goto _OVER; - } + MND_TMQ_RETURN_CHECK(mndCheckCreateTopicReq(&createTopicReq)); - pTopic = mndAcquireTopic(pMnode, createTopicReq.name); - if (pTopic != NULL) { + code = mndAcquireTopic(pMnode, createTopicReq.name, &pTopic); + if (code == TDB_CODE_SUCCESS) { if (createTopicReq.igExists) { mInfo("topic:%s already exist, ignore exist is set", createTopicReq.name); - code = 0; - goto _OVER; + goto END; } else { - terrno = TSDB_CODE_MND_TOPIC_ALREADY_EXIST; - goto _OVER; + code = TSDB_CODE_MND_TOPIC_ALREADY_EXIST; + goto END; } - } else if (terrno != TSDB_CODE_MND_TOPIC_NOT_EXIST) { - goto _OVER; + } else if (code != TSDB_CODE_MND_TOPIC_NOT_EXIST) { + goto END; } pDb = mndAcquireDb(pMnode, createTopicReq.subDbName); - if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - goto _OVER; - } + MND_TMQ_NULL_CHECK(pDb); if (pDb->cfg.walRetentionPeriod == 0) { - terrno = TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO; + code = TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO; mError("db:%s, not allowed to create topic when WAL_RETENTION_PERIOD is zero", pDb->name); - goto _OVER; + goto END; } if (sdbGetSize(pMnode->pSdb, SDB_TOPIC) >= tmqMaxTopicNum){ - terrno = TSDB_CODE_TMQ_TOPIC_OUT_OF_RANGE; + code = TSDB_CODE_TMQ_TOPIC_OUT_OF_RANGE; mError("topic num out of range"); - goto _OVER; - } - - if ((terrno = grantCheck(TSDB_GRANT_SUBSCRIPTION)) < 0) { - goto _OVER; + goto END; } + MND_TMQ_RETURN_CHECK(grantCheck(TSDB_GRANT_SUBSCRIPTION)); code = mndCreateTopic(pMnode, pReq, &createTopicReq, pDb, pReq->info.conn.user); if (code == 0) { code = TSDB_CODE_ACTION_IN_PROGRESS; } - SName dbname = {0}; - tNameFromString(&dbname, createTopicReq.subDbName, T_NAME_ACCT | T_NAME_DB); + { + SName dbname = {0}; + (void)tNameFromString(&dbname, createTopicReq.subDbName, T_NAME_ACCT | T_NAME_DB); // ignore error + SName topicName = {0}; + (void)tNameFromString(&topicName, createTopicReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); // ignore error + auditRecord(pReq, pMnode->clusterId, "createTopic", dbname.dbname, topicName.dbname, + createTopicReq.sql, strlen(createTopicReq.sql)); + } - SName topicName = {0}; - tNameFromString(&topicName, createTopicReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - //reuse this function for topic - - auditRecord(pReq, pMnode->clusterId, "createTopic", dbname.dbname, topicName.dbname, - createTopicReq.sql, strlen(createTopicReq.sql)); - -_OVER: +END: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("failed to create topic:%s since %s", createTopicReq.name, terrstr()); } @@ -638,22 +579,20 @@ _OVER: } static int32_t mndDropTopic(SMnode *pMnode, STrans *pTrans, SRpcMsg *pReq, SMqTopicObj *pTopic) { - int32_t code = -1; - if (mndUserRemoveTopic(pMnode, pTrans, pTopic->name) != 0) { - goto _OVER; + int32_t code = 0; + SSdbRaw *pCommitRaw = NULL; + MND_TMQ_RETURN_CHECK(mndUserRemoveTopic(pMnode, pTrans, pTopic->name)); + pCommitRaw = mndTopicActionEncode(pTopic); + MND_TMQ_NULL_CHECK(pCommitRaw); + code = mndTransAppendCommitlog(pTrans, pCommitRaw); + if(code != 0) { + sdbFreeRaw(pCommitRaw); + goto END; } + MND_TMQ_RETURN_CHECK(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED)); + MND_TMQ_RETURN_CHECK(mndTransPrepare(pMnode, pTrans)); - SSdbRaw *pCommitRaw = mndTopicActionEncode(pTopic); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER; - (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); - - if (mndTransPrepare(pMnode, pTrans) != 0) { - goto _OVER; - } - - code = 0; - -_OVER: +END: return code; } @@ -661,7 +600,7 @@ static bool checkTopic(SArray *topics, char *topicName){ int32_t sz = taosArrayGetSize(topics); for (int32_t i = 0; i < sz; i++) { char *name = taosArrayGetP(topics, i); - if (strcmp(name, topicName) == 0) { + if (name && strcmp(name, topicName) == 0) { return true; } } @@ -682,29 +621,26 @@ static int32_t mndDropConsumerByTopic(SMnode *pMnode, STrans *pTrans, char *topi bool found = checkTopic(pConsumer->assignedTopics, topicName); if (found){ if (pConsumer->status == MQ_CONSUMER_STATUS_LOST) { - code = mndSetConsumerDropLogs(pTrans, pConsumer); - if (code != 0) { - goto end; - } + MND_TMQ_RETURN_CHECK(mndSetConsumerDropLogs(pTrans, pConsumer)); sdbRelease(pSdb, pConsumer); continue; } mError("topic:%s, failed to drop since subscribed by consumer:0x%" PRIx64 ", in consumer group %s", topicName, pConsumer->consumerId, pConsumer->cgroup); code = TSDB_CODE_MND_TOPIC_SUBSCRIBED; - goto end; + goto END; } if (checkTopic(pConsumer->rebNewTopics, topicName) || checkTopic(pConsumer->rebRemovedTopics, topicName)) { code = TSDB_CODE_MND_TOPIC_SUBSCRIBED; mError("topic:%s, failed to drop since subscribed by consumer:%" PRId64 ", in consumer group %s (reb new)", topicName, pConsumer->consumerId, pConsumer->cgroup); - goto end; + goto END; } sdbRelease(pSdb, pConsumer); } -end: +END: sdbRelease(pSdb, pConsumer); sdbCancelFetch(pSdb, pIter); return code; @@ -726,13 +662,10 @@ static int32_t mndDropCheckInfoByTopic(SMnode *pMnode, STrans *pTrans, SMqTopicO } buf = taosMemoryCalloc(1, sizeof(SMsgHead) + TSDB_TOPIC_FNAME_LEN); - if (buf == NULL){ - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } + MND_TMQ_NULL_CHECK(buf); void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); ((SMsgHead *)buf)->vgId = htonl(pVgroup->vgId); - memcpy(abuf, pTopic->name, TSDB_TOPIC_FNAME_LEN); + (void)memcpy(abuf, pTopic->name, TSDB_TOPIC_FNAME_LEN); STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgroup); @@ -742,12 +675,12 @@ static int32_t mndDropCheckInfoByTopic(SMnode *pMnode, STrans *pTrans, SMqTopicO code = mndTransAppendRedoAction(pTrans, &action); if (code != 0) { taosMemoryFree(buf); - goto end; + goto END; } sdbRelease(pSdb, pVgroup); } -end: +END: sdbRelease(pSdb, pVgroup); sdbCancelFetch(pSdb, pIter); return code; @@ -761,70 +694,40 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { STrans *pTrans = NULL; if (tDeserializeSMDropTopicReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; + return TSDB_CODE_INVALID_MSG; } - pTopic = mndAcquireTopic(pMnode, dropReq.name); - if (pTopic == NULL) { + code = mndAcquireTopic(pMnode, dropReq.name, &pTopic); + if (code != 0) { if (dropReq.igNotExists) { mInfo("topic:%s, not exist, ignore not exist is set", dropReq.name); tFreeSMDropTopicReq(&dropReq); return 0; } else { - terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST; mError("topic:%s, failed to drop since %s", dropReq.name, terrstr()); tFreeSMDropTopicReq(&dropReq); - return -1; + return code; } } pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "drop-topic"); - if (pTrans == NULL) { - mError("topic:%s, failed to drop since %s", pTopic->name, terrstr()); - code = -1; - goto end; - } + MND_TMQ_NULL_CHECK(pTrans); mndTransSetDbName(pTrans, pTopic->db, NULL); - code = mndTransCheckConflict(pMnode, pTrans); - if (code != 0) { - goto end; - } - + MND_TMQ_RETURN_CHECK(mndTransCheckConflict(pMnode, pTrans)); mInfo("trans:%d, used to drop topic:%s", pTrans->id, pTopic->name); - - code = mndCheckTopicPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_TOPIC, pTopic); - if (code != 0) { - goto end; - } - - code = mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pTopic->db); - if (code != 0) { - goto end; - } - - code = mndDropConsumerByTopic(pMnode, pTrans, dropReq.name); - if (code != 0) { - goto end; - } - - code = mndDropSubByTopic(pMnode, pTrans, dropReq.name); - if (code < 0) { - mError("topic:%s, failed to drop since %s", pTopic->name, terrstr()); - goto end; - } + MND_TMQ_RETURN_CHECK(mndCheckTopicPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_TOPIC, pTopic)); + MND_TMQ_RETURN_CHECK(mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pTopic->db)); + MND_TMQ_RETURN_CHECK(mndDropConsumerByTopic(pMnode, pTrans, dropReq.name)); + MND_TMQ_RETURN_CHECK(mndDropSubByTopic(pMnode, pTrans, dropReq.name)); if (pTopic->ntbUid != 0) { - code = mndDropCheckInfoByTopic(pMnode, pTrans, pTopic); - if (code != 0) { - goto end; - } + MND_TMQ_RETURN_CHECK(mndDropCheckInfoByTopic(pMnode, pTrans, pTopic)); } code = mndDropTopic(pMnode, pTrans, pReq, pTopic); -end: +END: mndReleaseTopic(pMnode, pTopic); mndTransDrop(pTrans); if (code != 0) { @@ -834,8 +737,7 @@ end: } SName name = {0}; - tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - + (void)tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); // ignore error auditRecord(pReq, pMnode->clusterId, "dropTopic", name.dbname, name.tname, dropReq.sql, dropReq.sqlLen); tFreeSMDropTopicReq(&dropReq); @@ -849,8 +751,7 @@ int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) { SSdb *pSdb = pMnode->pSdb; SDbObj *pDb = mndAcquireDb(pMnode, dbName); if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - return -1; + return TSDB_CODE_MND_DB_NOT_SELECTED; } int32_t numOfTopics = 0; @@ -875,18 +776,24 @@ int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) { } static void schemaToJson(SSchema *schema, int32_t nCols, char *schemaJson){ - char* string = NULL; + char* string = NULL; + int32_t code = 0; cJSON* columns = cJSON_CreateArray(); - if (columns == NULL) { - return; - } + MND_TMQ_NULL_CHECK(columns); for (int i = 0; i < nCols; i++) { cJSON* column = cJSON_CreateObject(); + MND_TMQ_NULL_CHECK(column); SSchema* s = schema + i; cJSON* cname = cJSON_CreateString(s->name); - cJSON_AddItemToObject(column, "name", cname); + MND_TMQ_NULL_CHECK(cname); + if (!cJSON_AddItemToObject(column, "name", cname)) { + return; + } cJSON* ctype = cJSON_CreateString(tDataTypes[s->type].name); - cJSON_AddItemToObject(column, "type", ctype); + MND_TMQ_NULL_CHECK(ctype); + if (!cJSON_AddItemToObject(column, "type", ctype)) { + return; + } int32_t length = 0; if (s->type == TSDB_DATA_TYPE_BINARY || s->type == TSDB_DATA_TYPE_VARBINARY || s->type == TSDB_DATA_TYPE_GEOMETRY) { @@ -897,8 +804,13 @@ static void schemaToJson(SSchema *schema, int32_t nCols, char *schemaJson){ length = s->bytes; } cJSON* cbytes = cJSON_CreateNumber(length); - cJSON_AddItemToObject(column, "length", cbytes); - cJSON_AddItemToArray(columns, column); + MND_TMQ_NULL_CHECK(cbytes); + if (!cJSON_AddItemToObject(column, "length", cbytes)){ + return; + } + if (!cJSON_AddItemToArray(columns, column)){ + return; + } } string = cJSON_PrintUnformatted(columns); cJSON_Delete(columns); @@ -910,6 +822,9 @@ static void schemaToJson(SSchema *schema, int32_t nCols, char *schemaJson){ mError("mndRetrieveTopic build schema error json:%p, json len:%zu", string, len); } taosMemoryFree(string); + +END: + return; } static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) { @@ -917,13 +832,16 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SMqTopicObj *pTopic = NULL; + int32_t code = 0; + char *sql = NULL; + char *schemaJson = NULL; while (numOfRows < rowsCapacity) { pShow->pIter = sdbFetch(pSdb, SDB_TOPIC, pShow->pIter, (void **)&pTopic); if (pShow->pIter == NULL) break; - SColumnInfoData *pColInfo; - SName n; + SColumnInfoData *pColInfo= NULL; + SName n = {0}; int32_t cols = 0; char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE + 5] = {0}; @@ -931,28 +849,34 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl STR_TO_VARSTR(topicName, pName); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)topicName, false); + MND_TMQ_NULL_CHECK(pColInfo); + MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)topicName, false)); char dbName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&n, pTopic->db, T_NAME_ACCT | T_NAME_DB); - tNameGetDbName(&n, varDataVal(dbName)); + MND_TMQ_RETURN_CHECK(tNameFromString(&n, pTopic->db, T_NAME_ACCT | T_NAME_DB)); + MND_TMQ_RETURN_CHECK(tNameGetDbName(&n, varDataVal(dbName))); varDataSetLen(dbName, strlen(varDataVal(dbName))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)dbName, false); + MND_TMQ_NULL_CHECK(pColInfo); + MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)dbName, false)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pTopic->createTime, false); + MND_TMQ_NULL_CHECK(pColInfo); + MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)&pTopic->createTime, false)); - char *sql = taosMemoryMalloc(strlen(pTopic->sql) + VARSTR_HEADER_SIZE); + sql = taosMemoryMalloc(strlen(pTopic->sql) + VARSTR_HEADER_SIZE); + MND_TMQ_NULL_CHECK(sql); STR_TO_VARSTR(sql, pTopic->sql); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)sql, false); + MND_TMQ_NULL_CHECK(pColInfo); + MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)sql, false)); - taosMemoryFree(sql); + taosMemoryFreeClear(sql); - char *schemaJson = taosMemoryMalloc(TSDB_SHOW_SCHEMA_JSON_LEN + VARSTR_HEADER_SIZE); + schemaJson = taosMemoryMalloc(TSDB_SHOW_SCHEMA_JSON_LEN + VARSTR_HEADER_SIZE); + MND_TMQ_NULL_CHECK(schemaJson); if(pTopic->subType == TOPIC_SUB_TYPE__COLUMN){ schemaToJson(pTopic->schema.pSchema, pTopic->schema.nCols, schemaJson); }else if(pTopic->subType == TOPIC_SUB_TYPE__TABLE){ @@ -969,8 +893,9 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl } pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)schemaJson, false); - taosMemoryFree(schemaJson); + MND_TMQ_NULL_CHECK(pColInfo); + MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)schemaJson, false)); + taosMemoryFreeClear(schemaJson); char mete[4 + VARSTR_HEADER_SIZE] = {0}; if(pTopic->withMeta){ @@ -980,7 +905,8 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl } pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)mete, false); + MND_TMQ_NULL_CHECK(pColInfo); + MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)mete, false)); char type[8 + VARSTR_HEADER_SIZE] = {0}; if(pTopic->subType == TOPIC_SUB_TYPE__COLUMN){ @@ -992,7 +918,8 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl } pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)type, false); + MND_TMQ_NULL_CHECK(pColInfo); + MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)type, false)); numOfRows++; sdbRelease(pSdb, pTopic); @@ -1000,24 +927,11 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl pShow->numOfRows += numOfRows; return numOfRows; -} -int32_t mndSetTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic) { - SSdbRaw *pCommitRaw = mndTopicActionEncode(pTopic); - if (pCommitRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; - - return 0; -} - -static int32_t mndSetDropTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic) { - SSdbRaw *pCommitRaw = mndTopicActionEncode(pTopic); - if (pCommitRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; - - return 0; +END: + taosMemoryFreeClear(schemaJson); + taosMemoryFreeClear(sql); + return code; } static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter) { @@ -1047,33 +961,3 @@ bool mndTopicExistsForDb(SMnode *pMnode, SDbObj *pDb) { return false; } - -#if 0 -int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { - int32_t code = 0; - SSdb *pSdb = pMnode->pSdb; - - void *pIter = NULL; - SMqTopicObj *pTopic = NULL; - while (1) { - pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic); - if (pIter == NULL) break; - - if (pTopic->dbUid != pDb->uid) { - sdbRelease(pSdb, pTopic); - continue; - } - - if (mndSetDropTopicCommitLogs(pMnode, pTrans, pTopic) < 0) { - sdbRelease(pSdb, pTopic); - sdbCancelFetch(pSdb, pIter); - code = -1; - break; - } - - sdbRelease(pSdb, pTopic); - } - - return code; -} -#endif diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index fd5267e471..cd0cd97026 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -1357,7 +1357,7 @@ SHashObj *mndDupUseDbHash(SHashObj *pOld) { } int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) { - memcpy(pNew, pUser, sizeof(SUserObj)); + (void)memcpy(pNew, pUser, sizeof(SUserObj)); pNew->authVersion++; pNew->updateTime = taosGetTimestampMs(); @@ -1377,7 +1377,7 @@ int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) { taosRUnLockLatch(&pUser->lock); if (pNew->readDbs == NULL || pNew->writeDbs == NULL || pNew->topics == NULL) { - return -1; + return TSDB_CODE_INVALID_PARA; } return 0; } @@ -1983,10 +1983,11 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode if (ALTER_USER_ADD_SUBSCRIBE_TOPIC_PRIV(pAlterReq->alterType, pAlterReq->privileges)) { int32_t len = strlen(pAlterReq->objname) + 1; - SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pAlterReq->objname); - if (pTopic == NULL) { + SMqTopicObj *pTopic = NULL; + int32_t code = mndAcquireTopic(pMnode, pAlterReq->objname, &pTopic); + if (code != 0) { mndReleaseTopic(pMnode, pTopic); - return -1; + return code; } taosHashPut(pNewUser->topics, pTopic->name, len, pTopic->name, TSDB_TOPIC_FNAME_LEN); mndReleaseTopic(pMnode, pTopic); @@ -1994,10 +1995,11 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode if (ALTER_USER_DEL_SUBSCRIBE_TOPIC_PRIV(pAlterReq->alterType, pAlterReq->privileges)) { int32_t len = strlen(pAlterReq->objname) + 1; - SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pAlterReq->objname); - if (pTopic == NULL) { + SMqTopicObj *pTopic = NULL; + int32_t code = mndAcquireTopic(pMnode, pAlterReq->objname, &pTopic); + if (code != 0) { mndReleaseTopic(pMnode, pTopic); - return -1; + return code; } taosHashRemove(pNewUser->topics, pAlterReq->objname, len); mndReleaseTopic(pMnode, pTopic); diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 0422bfabff..d100f84d5a 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -2725,15 +2725,6 @@ int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgro SDbObj dbObj = {0}; SArray *pArray = mndBuildDnodesArray(pMnode, 0); - // int32_t numOfTopics = 0; - // if (mndGetNumOfTopics(pMnode, pDb->name, &numOfTopics) != 0) { - // goto _OVER; - // } - // if (numOfTopics > 0) { - // terrno = TSDB_CODE_MND_TOPIC_MUST_BE_DELETED; - // goto _OVER; - // } - int32_t numOfStreams = 0; if (mndGetNumOfStreams(pMnode, pDb->name, &numOfStreams) != 0) { goto _OVER; From 7b39915027a3e653e48cf1bf54bdff51de373792 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 22 Jul 2024 17:43:26 +0800 Subject: [PATCH 14/43] fix:[TD-31017]process return value in mnode for tmq --- source/dnode/mnode/impl/src/mndConsumer.c | 4 +++- source/dnode/mnode/impl/src/mndSubscribe.c | 6 ++++-- source/dnode/mnode/impl/src/mndTopic.c | 6 ++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 532a224732..cea664f64d 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -604,7 +604,9 @@ SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { if (buf == NULL) goto CM_ENCODE_OVER; void *abuf = buf; - tEncodeSMqConsumerObj(&abuf, pConsumer); + if(tEncodeSMqConsumerObj(&abuf, pConsumer) < 0){ + goto CM_ENCODE_OVER; + } int32_t dataPos = 0; SDB_SET_INT32(pRaw, dataPos, tlen, CM_ENCODE_OVER); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 731e25175f..ec915eb9d8 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -1109,7 +1109,9 @@ static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { if (buf == NULL) goto SUB_ENCODE_OVER; void *abuf = buf; - tEncodeSubscribeObj(&abuf, pSub); + if (tEncodeSubscribeObj(&abuf, pSub) < 0){ + goto SUB_ENCODE_OVER; + } int32_t dataPos = 0; SDB_SET_INT32(pRaw, dataPos, tlen, SUB_ENCODE_OVER); @@ -1342,7 +1344,7 @@ static int32_t buildResult(SSDataBlock *pBlock, int32_t *numOfRows, int64_t cons // consumer id char consumerIdHex[32] = {0}; - sprintf(varDataVal(consumerIdHex), "0x%" PRIx64, consumerId); + (void)sprintf(varDataVal(consumerIdHex), "0x%" PRIx64, consumerId); varDataSetLen(consumerIdHex, strlen(varDataVal(consumerIdHex))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 959a3ef008..2fb0870ed6 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -134,7 +134,9 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { goto TOPIC_ENCODE_OVER; } void *aswBuf = swBuf; - taosEncodeSSchemaWrapper(&aswBuf, &pTopic->schema); + if(taosEncodeSSchemaWrapper(&aswBuf, &pTopic->schema) < 0){ + goto TOPIC_ENCODE_OVER; + } SDB_SET_BINARY(pRaw, dataPos, swBuf, schemaLen, TOPIC_ENCODE_OVER); } @@ -264,7 +266,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { } int16_t colId; SDB_GET_INT16(pRaw, dataPos, &colId, TOPIC_DECODE_OVER); - taosArrayPush(pTopic->ntbColIds, &colId); + (void)taosArrayPush(pTopic->ntbColIds, &colId); } SDB_GET_INT64(pRaw, dataPos, &pTopic->ctbStbUid, TOPIC_DECODE_OVER); From bab043dad3f23d9b394e418600ceca4f1c2fa06d Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 22 Jul 2024 09:56:46 +0000 Subject: [PATCH 15/43] fix case --- source/libs/stream/src/streamMsg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamMsg.c b/source/libs/stream/src/streamMsg.c index b08280f9ed..bc0faacb32 100644 --- a/source/libs/stream/src/streamMsg.c +++ b/source/libs/stream/src/streamMsg.c @@ -549,7 +549,7 @@ int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { if (tEncodeCStrWithLen(pEncoder, pTask->reserve, sizeof(pTask->reserve) - 1) < 0) return -1; tEndEncode(pEncoder); - return pEncoder->pos; + return 0; } int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { From 7718d5e042162b4c93f795cf3a600cff523fe666 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 22 Jul 2024 18:47:41 +0800 Subject: [PATCH 16/43] fix: os return code --- include/os/os.h | 1 + include/os/osTime.h | 8 ++++---- source/os/src/osDir.c | 6 +++--- source/os/src/osMemory.c | 39 +++++++++++++++++++++++++++++++-------- source/os/src/osRand.c | 2 +- source/os/src/osTime.c | 26 ++++++++++++++++++++------ 6 files changed, 60 insertions(+), 22 deletions(-) diff --git a/include/os/os.h b/include/os/os.h index ac1a750b78..1749687d97 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -97,6 +97,7 @@ extern "C" { #include #endif + #include "osThread.h" #include "osAtomic.h" diff --git a/include/os/osTime.h b/include/os/osTime.h index 87df3a2650..c367416175 100644 --- a/include/os/osTime.h +++ b/include/os/osTime.h @@ -65,28 +65,28 @@ int32_t taosGetTimestampSec(); //@return timestamp in millisecond static FORCE_INLINE int64_t taosGetTimestampMs() { struct timeval systemTime; - taosGetTimeOfDay(&systemTime); + (void)taosGetTimeOfDay(&systemTime); return (int64_t)systemTime.tv_sec * 1000LL + (int64_t)systemTime.tv_usec / 1000; } //@return timestamp in microsecond static FORCE_INLINE int64_t taosGetTimestampUs() { struct timeval systemTime; - taosGetTimeOfDay(&systemTime); + (void)taosGetTimeOfDay(&systemTime); return (int64_t)systemTime.tv_sec * 1000000LL + (int64_t)systemTime.tv_usec; } //@return timestamp in nanosecond static FORCE_INLINE int64_t taosGetTimestampNs() { struct timespec systemTime = {0}; - taosClockGetTime(CLOCK_REALTIME, &systemTime); + (void)taosClockGetTime(CLOCK_REALTIME, &systemTime); return (int64_t)systemTime.tv_sec * 1000000000LL + (int64_t)systemTime.tv_nsec; } //@return timestamp of monotonic clock in millisecond static FORCE_INLINE int64_t taosGetMonoTimestampMs() { struct timespec systemTime = {0}; - taosClockGetTime(CLOCK_MONOTONIC, &systemTime); + (void)taosClockGetTime(CLOCK_MONOTONIC, &systemTime); return (int64_t)systemTime.tv_sec * 1000LL + (int64_t)systemTime.tv_nsec / 1000000; } diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 667c3c146a..4348629467 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -44,7 +44,7 @@ int wordexp(char *words, wordexp_t *pwordexp, int flags) { pwordexp->we_wordc = 1; pwordexp->we_wordv[0] = pwordexp->wordPos; - memset(pwordexp->wordPos, 0, 1025); + (void)memset(pwordexp->wordPos, 0, 1025); if (_fullpath(pwordexp->wordPos, words, 1024) == NULL) { pwordexp->we_wordv[0] = words; printf("failed to parse relative path:%s to abs path\n", words); @@ -100,7 +100,7 @@ void taosRemoveDir(const char *dirname) { if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue; char filename[1024] = {0}; - snprintf(filename, sizeof(filename), "%s%s%s", dirname, TD_DIRSEP, taosGetDirEntryName(de)); + (void)snprintf(filename, sizeof(filename), "%s%s%s", dirname, TD_DIRSEP, taosGetDirEntryName(de)); if (taosDirEntryIsDir(de)) { taosRemoveDir(filename); } else { @@ -110,7 +110,7 @@ void taosRemoveDir(const char *dirname) { } taosCloseDir(&pDir); - rmdir(dirname); + (void)rmdir(dirname); // printf("dir:%s is removed\n", dirname); return; diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index d130ebe469..3e49808dad 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -186,7 +186,7 @@ static void print_line(Dwarf_Debug dbg, Dwarf_Line line, Dwarf_Addr pc) { dwarf_linesrc(line, &linesrc, NULL); dwarf_lineno(line, &lineno, NULL); } - printf("BackTrace %08" PRId64 " %s:%" DW_PR_DUu "\n", taosGetSelfPthreadId(), linesrc, lineno); + (void)printf("BackTrace %08" PRId64 " %s:%" DW_PR_DUu "\n", taosGetSelfPthreadId(), linesrc, lineno); if (line) dwarf_dealloc(dbg, linesrc, DW_DLA_STRING); } void taosPrintBackTrace() { @@ -266,7 +266,11 @@ void *taosMemoryMalloc(int64_t size) { return (char *)tmp + sizeof(TdMemoryInfo); #else - return malloc(size); + void* p = malloc(size); + if (NULL == p) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } + return p; #endif } @@ -283,7 +287,11 @@ void *taosMemoryCalloc(int64_t num, int64_t size) { return (char *)tmp + sizeof(TdMemoryInfo); #else - return calloc(num, size); + void* p = calloc(num, size); + if (NULL == p) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } + return p; #endif } @@ -309,7 +317,11 @@ void *taosMemoryRealloc(void *ptr, int64_t size) { return (char *)tmp + sizeof(TdMemoryInfo); #else - return realloc(ptr, size); + void* p = realloc(ptr, size); + if (size > 0 && NULL == p) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } + return p; #endif } @@ -330,7 +342,12 @@ char *taosStrdup(const char *ptr) { return (char *)tmp + sizeof(TdMemoryInfo); #else - return tstrdup(ptr); + char* p = tstrdup(ptr); + if (ptr != NULL && NULL == p) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } + return p; + #endif } @@ -357,8 +374,7 @@ int64_t taosMemorySize(void *ptr) { TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char *)ptr - sizeof(TdMemoryInfo)); ASSERT(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); if (pTdMemoryInfo->symbol != TD_MEMORY_SYMBOL) { - +return NULL; - + + return NULL; } return pTdMemoryInfo->memorySize; @@ -378,7 +394,7 @@ void taosMemoryTrim(int32_t size) { // do nothing return; #else - malloc_trim(size); + (void)malloc_trim(size); #endif } @@ -388,6 +404,13 @@ void *taosMemoryMallocAlign(uint32_t alignment, int64_t size) { #else #if defined(LINUX) void *p = memalign(alignment, size); + if (NULL == p) { + if (ENOMEM == errno) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } else { + terrno = TAOS_SYSTEM_ERROR(errno); + } + } return p; #else return taosMemoryMalloc(size); diff --git a/source/os/src/osRand.c b/source/os/src/osRand.c index 0e58b7a8ec..da4e8dfb9d 100644 --- a/source/os/src/osRand.c +++ b/source/os/src/osRand.c @@ -69,7 +69,7 @@ uint32_t taosSafeRand(void) { if (len < 0) { seed = (int)taosGetTimestampSec(); } - taosCloseFile(&pFile); + (void)taosCloseFile(&pFile); } return (uint32_t)seed; diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index 9cad1dd6ef..c650ac989d 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -345,6 +345,7 @@ char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm) { } int32_t taosGetTimeOfDay(struct timeval *tv) { + int32_t code = 0; #ifdef WINDOWS LARGE_INTEGER t; FILETIME f; @@ -359,11 +360,18 @@ int32_t taosGetTimeOfDay(struct timeval *tv) { tv->tv_usec = (t.QuadPart % 10000000) / 10; return 0; #else - return gettimeofday(tv, NULL); + code = gettimeofday(tv, NULL); + return (-1 == code) ? (terrno = TAOS_SYSTEM_ERROR(errno)) : 0; #endif } -time_t taosTime(time_t *t) { return time(t); } +time_t taosTime(time_t *t) { + time_t r = time(t); + if (r == (time_t)-1) { + terrno = TAOS_SYSTEM_ERROR(errno); + } + return r; +} /* * mktime64 - Converts date to seconds. @@ -458,7 +466,11 @@ time_t taosMktime(struct tm *timep) { timep->tm_sec, tz); #endif #else - return mktime(timep); + time_t r = mktime(timep); + if (r == (time_t)-1) { + terrno = TAOS_SYSTEM_ERROR(errno); + } + return r; #endif } @@ -470,7 +482,7 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf) { if (result == NULL) { res = localtime(timep); if (res == NULL && buf != NULL) { - sprintf(buf, "NaN"); + (void)sprintf(buf, "NaN"); } return res; } @@ -528,7 +540,7 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf) { #else res = localtime_r(timep, result); if (res == NULL && buf != NULL) { - sprintf(buf, "NaN"); + (void)sprintf(buf, "NaN"); } #endif return result; @@ -642,6 +654,7 @@ struct tm *taosLocalTimeNolock(struct tm *result, const time_t *timep, int dst) int32_t taosGetTimestampSec() { return (int32_t)time(NULL); } int32_t taosClockGetTime(int clock_id, struct timespec *pTS) { + int32_t code = 0; #ifdef WINDOWS LARGE_INTEGER t; FILETIME f; @@ -656,6 +669,7 @@ int32_t taosClockGetTime(int clock_id, struct timespec *pTS) { pTS->tv_nsec = (t.QuadPart % 10000000) * 100; return (0); #else - return clock_gettime(clock_id, pTS); + code = clock_gettime(clock_id, pTS); + return (-1 == code) ? (terrno = TAOS_SYSTEM_ERROR(errno)) : 0; #endif } From 268194eb12607894b420eb5a7ceb7974cee1a090 Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 23 Jul 2024 01:58:17 +0000 Subject: [PATCH 17/43] fix case --- source/dnode/mnode/impl/src/mndFunc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index b84936aa6d..5ad1c33015 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -539,6 +539,7 @@ static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) { SFuncObj *pFunc = mndAcquireFunc(pMnode, funcName); if (pFunc == NULL) { + if (terrno != 0) code = terrno; goto RETRIEVE_FUNC_OVER; } From ea0140068ca4390617a6d6fa7eb5152f3131de23 Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 23 Jul 2024 02:07:06 +0000 Subject: [PATCH 18/43] fix case --- source/dnode/mnode/impl/src/mndDnode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index ab312576d4..b7446dbd3c 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -1408,7 +1408,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); #endif } else { - if (mndMCfg2DCfg(&cfgReq, &dcfgReq)) goto _err_out; + TAOS_CHECK_GOTO (mndMCfg2DCfg(&cfgReq, &dcfgReq), NULL, _err_out); if (strlen(dcfgReq.config) > TSDB_DNODE_CONFIG_LEN) { mError("dnode:%d, failed to config since config is too long", cfgReq.dnodeId); code = TSDB_CODE_INVALID_CFG; From 10263b6b4e76a606fb67a79a04f6e657b2a4ccd3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 23 Jul 2024 11:01:22 +0800 Subject: [PATCH 19/43] enh: refactor return code --- source/dnode/vnode/CMakeLists.txt | 1 - source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/meta/metaCache.c | 139 ++++++------- source/dnode/vnode/src/meta/metaCommit.c | 48 ++++- source/dnode/vnode/src/meta/metaEntry.c | 145 +++++++------- source/dnode/vnode/src/meta/metaIdx.c | 118 ----------- source/dnode/vnode/src/meta/metaOpen.c | 223 ++++++++------------- source/dnode/vnode/src/meta/metaSnapshot.c | 102 +++++----- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 3 +- 9 files changed, 313 insertions(+), 468 deletions(-) delete mode 100644 source/dnode/vnode/src/meta/metaIdx.c diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 400725b041..f70a8844ba 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -22,7 +22,6 @@ set( # meta "src/meta/metaOpen.c" - "src/meta/metaIdx.c" "src/meta/metaTable.c" "src/meta/metaSma.c" "src/meta/metaQuery.c" diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 0dc53afc48..32e4fea336 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -318,7 +318,7 @@ void* tdUidStoreFree(STbUidStore* pStore); // SMetaSnapReader ======================================== int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader); -int32_t metaSnapReaderClose(SMetaSnapReader** ppReader); +void metaSnapReaderClose(SMetaSnapReader** ppReader); int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData); // SMetaSnapWriter ======================================== int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter); diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 64cee3513c..3d4757d3e4 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -117,91 +117,81 @@ static void freeCacheEntryFp(void* param) { } int32_t metaCacheOpen(SMeta* pMeta) { - int32_t code = 0; - SMetaCache* pCache = NULL; + int32_t code = 0; + int32_t lino; - pCache = (SMetaCache*)taosMemoryMalloc(sizeof(SMetaCache)); - if (pCache == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + pMeta->pCache = (SMetaCache*)taosMemoryCalloc(1, sizeof(SMetaCache)); + if (pMeta->pCache == NULL) { + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } // open entry cache - pCache->sEntryCache.nEntry = 0; - pCache->sEntryCache.nBucket = META_CACHE_BASE_BUCKET; - pCache->sEntryCache.aBucket = - (SMetaCacheEntry**)taosMemoryCalloc(pCache->sEntryCache.nBucket, sizeof(SMetaCacheEntry*)); - if (pCache->sEntryCache.aBucket == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + pMeta->pCache->sEntryCache.nEntry = 0; + pMeta->pCache->sEntryCache.nBucket = META_CACHE_BASE_BUCKET; + pMeta->pCache->sEntryCache.aBucket = + (SMetaCacheEntry**)taosMemoryCalloc(pMeta->pCache->sEntryCache.nBucket, sizeof(SMetaCacheEntry*)); + if (pMeta->pCache->sEntryCache.aBucket == NULL) { + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } // open stats cache - pCache->sStbStatsCache.nEntry = 0; - pCache->sStbStatsCache.nBucket = META_CACHE_STATS_BUCKET; - pCache->sStbStatsCache.aBucket = - (SMetaStbStatsEntry**)taosMemoryCalloc(pCache->sStbStatsCache.nBucket, sizeof(SMetaStbStatsEntry*)); - if (pCache->sStbStatsCache.aBucket == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err2; + pMeta->pCache->sStbStatsCache.nEntry = 0; + pMeta->pCache->sStbStatsCache.nBucket = META_CACHE_STATS_BUCKET; + pMeta->pCache->sStbStatsCache.aBucket = + (SMetaStbStatsEntry**)taosMemoryCalloc(pMeta->pCache->sStbStatsCache.nBucket, sizeof(SMetaStbStatsEntry*)); + if (pMeta->pCache->sStbStatsCache.aBucket == NULL) { + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } - pCache->sTagFilterResCache.pUidResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5); - if (pCache->sTagFilterResCache.pUidResCache == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err2; + pMeta->pCache->sTagFilterResCache.pUidResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5); + if (pMeta->pCache->sTagFilterResCache.pUidResCache == NULL) { + TSDB_CHECK_CODE(code = terrno, lino, _exit); } - pCache->sTagFilterResCache.accTimes = 0; - pCache->sTagFilterResCache.pTableEntry = + pMeta->pCache->sTagFilterResCache.accTimes = 0; + pMeta->pCache->sTagFilterResCache.pTableEntry = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); - if (pCache->sTagFilterResCache.pTableEntry == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err2; + if (pMeta->pCache->sTagFilterResCache.pTableEntry == NULL) { + TSDB_CHECK_CODE(code = terrno, lino, _exit); } - taosHashSetFreeFp(pCache->sTagFilterResCache.pTableEntry, freeCacheEntryFp); - taosThreadMutexInit(&pCache->sTagFilterResCache.lock, NULL); + taosHashSetFreeFp(pMeta->pCache->sTagFilterResCache.pTableEntry, freeCacheEntryFp); + taosThreadMutexInit(&pMeta->pCache->sTagFilterResCache.lock, NULL); - pCache->STbGroupResCache.pResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5); - if (pCache->STbGroupResCache.pResCache == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err2; + pMeta->pCache->STbGroupResCache.pResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5); + if (pMeta->pCache->STbGroupResCache.pResCache == NULL) { + TSDB_CHECK_CODE(code = terrno, lino, _exit); } - pCache->STbGroupResCache.accTimes = 0; - pCache->STbGroupResCache.pTableEntry = + pMeta->pCache->STbGroupResCache.accTimes = 0; + pMeta->pCache->STbGroupResCache.pTableEntry = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); - if (pCache->STbGroupResCache.pTableEntry == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err2; + if (pMeta->pCache->STbGroupResCache.pTableEntry == NULL) { + TSDB_CHECK_CODE(code = terrno, lino, _exit); } - taosHashSetFreeFp(pCache->STbGroupResCache.pTableEntry, freeCacheEntryFp); - taosThreadMutexInit(&pCache->STbGroupResCache.lock, NULL); + taosHashSetFreeFp(pMeta->pCache->STbGroupResCache.pTableEntry, freeCacheEntryFp); + taosThreadMutexInit(&pMeta->pCache->STbGroupResCache.lock, NULL); - pCache->STbFilterCache.pStb = taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); - if (pCache->STbFilterCache.pStb == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err2; + pMeta->pCache->STbFilterCache.pStb = + taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + if (pMeta->pCache->STbFilterCache.pStb == NULL) { + TSDB_CHECK_CODE(code = terrno, lino, _exit); } - pCache->STbFilterCache.pStbName = + pMeta->pCache->STbFilterCache.pStbName = taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); - if (pCache->STbFilterCache.pStbName == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err2; + if (pMeta->pCache->STbFilterCache.pStbName == NULL) { + TSDB_CHECK_CODE(code = terrno, lino, _exit); } - pMeta->pCache = pCache; - return code; - -_err2: - entryCacheClose(pMeta); - -_err: - taosMemoryFree(pCache); - metaError("vgId:%d, meta open cache failed since %s", TD_VID(pMeta->pVnode), tstrerror(code)); +_exit: + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, lino, tstrerror(code)); + metaCacheClose(pMeta); + } else { + metaDebug("vgId:%d, %s success", TD_VID(pMeta->pVnode), __func__); + } return code; } @@ -289,8 +279,7 @@ int32_t metaCacheUpsert(SMeta* pMeta, SMetaInfo* pInfo) { } } else { // insert if (pCache->sEntryCache.nEntry >= pCache->sEntryCache.nBucket) { - code = metaRehashCache(pCache, 1); - if (code) goto _exit; + TAOS_UNUSED(metaRehashCache(pCache, 1)); iBucket = TABS(pInfo->uid) % pCache->sEntryCache.nBucket; } @@ -328,8 +317,7 @@ int32_t metaCacheDrop(SMeta* pMeta, int64_t uid) { pCache->sEntryCache.nEntry--; if (pCache->sEntryCache.nEntry < pCache->sEntryCache.nBucket / 4 && pCache->sEntryCache.nBucket > META_CACHE_BASE_BUCKET) { - code = metaRehashCache(pCache, 0); - if (code) goto _exit; + TAOS_UNUSED(metaRehashCache(pCache, 0)); } } else { code = TSDB_CODE_NOT_FOUND; @@ -351,7 +339,9 @@ int32_t metaCacheGet(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo) { } if (pEntry) { - *pInfo = pEntry->info; + if (pInfo) { + *pInfo = pEntry->info; + } } else { code = TSDB_CODE_NOT_FOUND; } @@ -415,9 +405,7 @@ int32_t metaStatsCacheUpsert(SMeta* pMeta, SMetaStbStats* pInfo) { (*ppEntry)->info.ctbNum = pInfo->ctbNum; } else { // insert if (pCache->sStbStatsCache.nEntry >= pCache->sStbStatsCache.nBucket) { - code = metaRehashStatsCache(pCache, 1); - if (code) goto _exit; - + TAOS_UNUSED(metaRehashStatsCache(pCache, 1)); iBucket = TABS(pInfo->uid) % pCache->sStbStatsCache.nBucket; } @@ -454,8 +442,7 @@ int32_t metaStatsCacheDrop(SMeta* pMeta, int64_t uid) { pCache->sStbStatsCache.nEntry--; if (pCache->sStbStatsCache.nEntry < pCache->sStbStatsCache.nBucket / 4 && pCache->sStbStatsCache.nBucket > META_CACHE_STATS_BUCKET) { - code = metaRehashStatsCache(pCache, 0); - if (code) goto _exit; + TAOS_UNUSED(metaRehashStatsCache(pCache, 0)); } } else { code = TSDB_CODE_NOT_FOUND; @@ -477,7 +464,9 @@ int32_t metaStatsCacheGet(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo) { } if (pEntry) { - *pInfo = pEntry->info; + if (pInfo) { + *pInfo = pEntry->info; + } } else { code = TSDB_CODE_NOT_FOUND; } @@ -502,7 +491,9 @@ static int checkAllEntriesInCache(const STagFilterResEntry* pEntry, SArray* pInv // check whether it is existed in LRU cache, and remove it from linked list if not. LRUHandle* pRes = taosLRUCacheLookup(pCache, buf, len); if (pRes == NULL) { // remove the item in the linked list - taosArrayPush(pInvalidRes, &pNode); + if (taosArrayPush(pInvalidRes, &pNode) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } else { taosLRUCacheRelease(pCache, pRes, false); } @@ -626,7 +617,7 @@ static int32_t addNewEntry(SHashObj* pTableEntry, const void* pKey, int32_t keyL p->hitTimes = 0; tdListInit(&p->list, keyLen); - taosHashPut(pTableEntry, &suid, sizeof(uint64_t), &p, POINTER_BYTES); + TAOS_CHECK_RETURN(taosHashPut(pTableEntry, &suid, sizeof(uint64_t), &p, POINTER_BYTES)); tdListAppend(&p->list, pKey); return 0; } @@ -956,9 +947,7 @@ int32_t metaInitTbFilterCache(SMeta* pMeta) { } if (tbNum && pTbArr) { for (int32_t i = 0; i < tbNum; ++i) { - if (metaPutTbToFilterCache(pMeta, pTbArr[i], 1) != 0) { - return terrno ? terrno : -1; - } + TAOS_CHECK_RETURN(metaPutTbToFilterCache(pMeta, pTbArr[i], 1)); } } #else diff --git a/source/dnode/vnode/src/meta/metaCommit.c b/source/dnode/vnode/src/meta/metaCommit.c index f8b41e413b..bbf2217b31 100644 --- a/source/dnode/vnode/src/meta/metaCommit.c +++ b/source/dnode/vnode/src/meta/metaCommit.c @@ -21,7 +21,10 @@ static FORCE_INLINE void *metaMalloc(void *pPool, size_t size) { static FORCE_INLINE void metaFree(void *pPool, void *p) { vnodeBufPoolFree((SVBufPool *)pPool, p); } // begin a meta txn -int metaBegin(SMeta *pMeta, int8_t heap) { +int32_t metaBegin(SMeta *pMeta, int8_t heap) { + int32_t code = 0; + int32_t lino; + void *(*xMalloc)(void *, size_t) = NULL; void (*xFree)(void *, void *) = NULL; void *xArg = NULL; @@ -36,12 +39,19 @@ int metaBegin(SMeta *pMeta, int8_t heap) { xArg = pMeta->pVnode->inUse; } - if (tdbBegin(pMeta->pEnv, &pMeta->txn, xMalloc, xFree, xArg, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { - return -1; + code = tdbBegin(pMeta->pEnv, &pMeta->txn, xMalloc, xFree, xArg, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tdbCommit(pMeta->pEnv, pMeta->txn); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + metaError("vgId:%d %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, + tstrerror(terrno)); + } else { + metaDebug("vgId:%d %s success", TD_VID(pMeta->pVnode), __func__); } - - tdbCommit(pMeta->pEnv, pMeta->txn); - return 0; } @@ -49,20 +59,36 @@ int metaBegin(SMeta *pMeta, int8_t heap) { TXN *metaGetTxn(SMeta *pMeta) { return pMeta->txn; } int metaCommit(SMeta *pMeta, TXN *txn) { return tdbCommit(pMeta->pEnv, txn); } int metaFinishCommit(SMeta *pMeta, TXN *txn) { return tdbPostCommit(pMeta->pEnv, txn); } -int metaPrepareAsyncCommit(SMeta *pMeta) { - // return tdbPrepareAsyncCommit(pMeta->pEnv, pMeta->txn); - int code = 0; + +int metaPrepareAsyncCommit(SMeta *pMeta) { + // return tdbPrepareAsyncCommit(pMeta->pEnv, pMeta->txn); + int code = 0; + int32_t lino; + metaWLock(pMeta); - code = ttlMgrFlush(pMeta->pTtlMgr, pMeta->txn); + TAOS_UNUSED(ttlMgrFlush(pMeta->pTtlMgr, pMeta->txn)); metaULock(pMeta); + code = tdbCommit(pMeta->pEnv, pMeta->txn); + TSDB_CHECK_CODE(code, lino, _exit); pMeta->changed = false; + +_exit: + if (code) { + metaError("vgId:%d %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, + tstrerror(terrno)); + } else { + metaDebug("vgId:%d %s success", TD_VID(pMeta->pVnode), __func__); + } return code; } // abort the meta txn int metaAbort(SMeta *pMeta) { - if (!pMeta->txn) return 0; + if (!pMeta->txn) { + return 0; + } + int code = tdbAbort(pMeta->pEnv, pMeta->txn); if (code) { metaError("vgId:%d, failed to abort meta since %s", TD_VID(pMeta->pVnode), tstrerror(terrno)); diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index 9a111ae2d4..6e94cca390 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -17,159 +17,166 @@ int meteEncodeColCmprEntry(SEncoder *pCoder, const SMetaEntry *pME) { const SColCmprWrapper *pw = &pME->colCmpr; - if (tEncodeI32v(pCoder, pw->nCols) < 0) return -1; - if (tEncodeI32v(pCoder, pw->version) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pw->nCols)); + TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pw->version)); uDebug("encode cols:%d", pw->nCols); for (int32_t i = 0; i < pw->nCols; i++) { SColCmpr *p = &pw->pColCmpr[i]; - if (tEncodeI16v(pCoder, p->id) < 0) return -1; - if (tEncodeU32(pCoder, p->alg) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeI16v(pCoder, p->id)); + TAOS_CHECK_RETURN(tEncodeU32(pCoder, p->alg)); } return 0; } int meteDecodeColCmprEntry(SDecoder *pDecoder, SMetaEntry *pME) { SColCmprWrapper *pWrapper = &pME->colCmpr; - if (tDecodeI32v(pDecoder, &pWrapper->nCols) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeI32v(pDecoder, &pWrapper->nCols)); if (pWrapper->nCols == 0) { return 0; } - if (tDecodeI32v(pDecoder, &pWrapper->version) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeI32v(pDecoder, &pWrapper->version)); uDebug("dencode cols:%d", pWrapper->nCols); pWrapper->pColCmpr = (SColCmpr *)tDecoderMalloc(pDecoder, pWrapper->nCols * sizeof(SColCmpr)); - if (pWrapper->pColCmpr == NULL) return -1; + if (pWrapper->pColCmpr == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } for (int i = 0; i < pWrapper->nCols; i++) { SColCmpr *p = &pWrapper->pColCmpr[i]; - if (tDecodeI16v(pDecoder, &p->id) < 0) return -1; - if (tDecodeU32(pDecoder, &p->alg) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeI16v(pDecoder, &p->id)); + TAOS_CHECK_RETURN(tDecodeU32(pDecoder, &p->alg)); } return 0; } -static FORCE_INLINE void metatInitDefaultSColCmprWrapper(SDecoder *pDecoder, SColCmprWrapper *pCmpr, - SSchemaWrapper *pSchema) { +static FORCE_INLINE int32_t metatInitDefaultSColCmprWrapper(SDecoder *pDecoder, SColCmprWrapper *pCmpr, + SSchemaWrapper *pSchema) { pCmpr->nCols = pSchema->nCols; - pCmpr->pColCmpr = (SColCmpr *)tDecoderMalloc(pDecoder, pCmpr->nCols * sizeof(SColCmpr)); + if ((pCmpr->pColCmpr = (SColCmpr *)tDecoderMalloc(pDecoder, pCmpr->nCols * sizeof(SColCmpr))) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + for (int32_t i = 0; i < pCmpr->nCols; i++) { SColCmpr *pColCmpr = &pCmpr->pColCmpr[i]; SSchema *pColSchema = &pSchema->pSchema[i]; pColCmpr->id = pColSchema->colId; pColCmpr->alg = createDefaultColCmprByType(pColSchema->type); } + return 0; } int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) { - if (tStartEncode(pCoder) < 0) return -1; + TAOS_CHECK_RETURN(tStartEncode(pCoder)); + TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->version)); + TAOS_CHECK_RETURN(tEncodeI8(pCoder, pME->type)); + TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->uid)); - if (tEncodeI64(pCoder, pME->version) < 0) return -1; - if (tEncodeI8(pCoder, pME->type) < 0) return -1; - if (tEncodeI64(pCoder, pME->uid) < 0) return -1; - if (pME->name == NULL || tEncodeCStr(pCoder, pME->name) < 0) return -1; + if (pME->name == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pME->name)); if (pME->type == TSDB_SUPER_TABLE) { - if (tEncodeI8(pCoder, pME->flags) < 0) return -1; // TODO: need refactor? - if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaRow) < 0) return -1; - if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeI8(pCoder, pME->flags)); + TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaRow)); + TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag)); if (TABLE_IS_ROLLUP(pME->flags)) { - if (tEncodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam)); } } else if (pME->type == TSDB_CHILD_TABLE) { - if (tEncodeI64(pCoder, pME->ctbEntry.btime) < 0) return -1; - if (tEncodeI32(pCoder, pME->ctbEntry.ttlDays) < 0) return -1; - if (tEncodeI32v(pCoder, pME->ctbEntry.commentLen) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->ctbEntry.btime)); + TAOS_CHECK_RETURN(tEncodeI32(pCoder, pME->ctbEntry.ttlDays)); + TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->ctbEntry.commentLen)); if (pME->ctbEntry.commentLen > 0) { - if (tEncodeCStr(pCoder, pME->ctbEntry.comment) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pME->ctbEntry.comment)); } - if (tEncodeI64(pCoder, pME->ctbEntry.suid) < 0) return -1; - if (tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->ctbEntry.suid)); + TAOS_CHECK_RETURN(tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags)); } else if (pME->type == TSDB_NORMAL_TABLE) { - if (tEncodeI64(pCoder, pME->ntbEntry.btime) < 0) return -1; - if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1; - if (tEncodeI32v(pCoder, pME->ntbEntry.commentLen) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->ntbEntry.btime)); + TAOS_CHECK_RETURN(tEncodeI32(pCoder, pME->ntbEntry.ttlDays)); + TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->ntbEntry.commentLen)); if (pME->ntbEntry.commentLen > 0) { - if (tEncodeCStr(pCoder, pME->ntbEntry.comment) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pME->ntbEntry.comment)); } - if (tEncodeI32v(pCoder, pME->ntbEntry.ncid) < 0) return -1; - if (tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schemaRow) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->ntbEntry.ncid)); + TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schemaRow)); } else if (pME->type == TSDB_TSMA_TABLE) { - if (tEncodeTSma(pCoder, pME->smaEntry.tsma) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeTSma(pCoder, pME->smaEntry.tsma)); } else { metaError("meta/entry: invalide table type: %" PRId8 " encode failed.", pME->type); - - return -1; + return TSDB_CODE_INVALID_PARA; } - if (meteEncodeColCmprEntry(pCoder, pME) < 0) return -1; + TAOS_CHECK_RETURN(meteEncodeColCmprEntry(pCoder, pME)); tEndEncode(pCoder); return 0; } int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) { - if (tStartDecode(pCoder) < 0) return -1; - - if (tDecodeI64(pCoder, &pME->version) < 0) return -1; - if (tDecodeI8(pCoder, &pME->type) < 0) return -1; - if (tDecodeI64(pCoder, &pME->uid) < 0) return -1; - if (tDecodeCStr(pCoder, &pME->name) < 0) return -1; + TAOS_CHECK_RETURN(tStartDecode(pCoder)); + TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->version)); + TAOS_CHECK_RETURN(tDecodeI8(pCoder, &pME->type)); + TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->uid)); + TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pME->name)); if (pME->type == TSDB_SUPER_TABLE) { - if (tDecodeI8(pCoder, &pME->flags) < 0) return -1; // TODO: need refactor? - if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaRow) < 0) return -1; - if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaTag) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeI8(pCoder, &pME->flags)); + TAOS_CHECK_RETURN(tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaRow)); + TAOS_CHECK_RETURN(tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaTag)); if (TABLE_IS_ROLLUP(pME->flags)) { - if (tDecodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam)); } } else if (pME->type == TSDB_CHILD_TABLE) { - if (tDecodeI64(pCoder, &pME->ctbEntry.btime) < 0) return -1; - if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1; - if (tDecodeI32v(pCoder, &pME->ctbEntry.commentLen) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->ctbEntry.btime)); + TAOS_CHECK_RETURN(tDecodeI32(pCoder, &pME->ctbEntry.ttlDays)); + TAOS_CHECK_RETURN(tDecodeI32v(pCoder, &pME->ctbEntry.commentLen)); if (pME->ctbEntry.commentLen > 0) { - if (tDecodeCStr(pCoder, &pME->ctbEntry.comment) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pME->ctbEntry.comment)); } - if (tDecodeI64(pCoder, &pME->ctbEntry.suid) < 0) return -1; - if (tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags) < 0) return -1; // (TODO) + TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->ctbEntry.suid)); + TAOS_CHECK_RETURN(tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags)); } else if (pME->type == TSDB_NORMAL_TABLE) { - if (tDecodeI64(pCoder, &pME->ntbEntry.btime) < 0) return -1; - if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1; - if (tDecodeI32v(pCoder, &pME->ntbEntry.commentLen) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->ntbEntry.btime)); + TAOS_CHECK_RETURN(tDecodeI32(pCoder, &pME->ntbEntry.ttlDays)); + TAOS_CHECK_RETURN(tDecodeI32v(pCoder, &pME->ntbEntry.commentLen)); if (pME->ntbEntry.commentLen > 0) { - if (tDecodeCStr(pCoder, &pME->ntbEntry.comment) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pME->ntbEntry.comment)); } - if (tDecodeI32v(pCoder, &pME->ntbEntry.ncid) < 0) return -1; - if (tDecodeSSchemaWrapperEx(pCoder, &pME->ntbEntry.schemaRow) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeI32v(pCoder, &pME->ntbEntry.ncid)); + TAOS_CHECK_RETURN(tDecodeSSchemaWrapperEx(pCoder, &pME->ntbEntry.schemaRow)); } else if (pME->type == TSDB_TSMA_TABLE) { pME->smaEntry.tsma = tDecoderMalloc(pCoder, sizeof(STSma)); if (!pME->smaEntry.tsma) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } - if (tDecodeTSma(pCoder, pME->smaEntry.tsma, true) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeTSma(pCoder, pME->smaEntry.tsma, true)); } else { metaError("meta/entry: invalide table type: %" PRId8 " decode failed.", pME->type); - return -1; + return TSDB_CODE_INVALID_PARA; } if (pME->type == TSDB_SUPER_TABLE) { if (TABLE_IS_COL_COMPRESSED(pME->flags)) { - if (meteDecodeColCmprEntry(pCoder, pME) < 0) return -1; + TAOS_CHECK_RETURN(meteDecodeColCmprEntry(pCoder, pME)); if (pME->colCmpr.nCols == 0) { - metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow); + TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow)); } } else { - metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow); + TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow)); TABLE_SET_COL_COMPRESSED(pME->flags); } } else if (pME->type == TSDB_NORMAL_TABLE) { if (!tDecodeIsEnd(pCoder)) { uDebug("set type: %d, tableName:%s", pME->type, pME->name); - if (meteDecodeColCmprEntry(pCoder, pME) < 0) return -1; + TAOS_CHECK_RETURN(meteDecodeColCmprEntry(pCoder, pME)); if (pME->colCmpr.nCols == 0) { - metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow); + TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow)); } } else { uDebug("set default type: %d, tableName:%s", pME->type, pME->name); - metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow); + TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow)); } TABLE_SET_COL_COMPRESSED(pME->flags); } diff --git a/source/dnode/vnode/src/meta/metaIdx.c b/source/dnode/vnode/src/meta/metaIdx.c deleted file mode 100644 index dc62ab2b9f..0000000000 --- a/source/dnode/vnode/src/meta/metaIdx.c +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifdef USE_INVERTED_INDEX -#include "index.h" -#endif -#include "meta.h" - -struct SMetaIdx { -#ifdef USE_INVERTED_INDEX - SIndex *pIdx; -#endif - /* data */ -#ifdef WINDOWS - size_t avoidCompilationErrors; -#endif -}; - -int metaOpenIdx(SMeta *pMeta) { -#if 0 - char idxDir[128]; // TODO - char * err = NULL; - rocksdb_options_t *options = rocksdb_options_create(); - - // TODO - sprintf(idxDir, "%s/index", pMeta->path); - - if (pMeta->pCache) { - rocksdb_options_set_row_cache(options, pMeta->pCache); - } - rocksdb_options_set_create_if_missing(options, 1); - - pMeta->pIdx = rocksdb_open(options, idxDir, &err); - if (pMeta->pIdx == NULL) { - // TODO: handle error - rocksdb_options_destroy(options); - return -1; - } - - rocksdb_options_destroy(options); -#endif - -#ifdef USE_INVERTED_INDEX - // SIndexOpts opts; - // if (indexOpen(&opts, pMeta->path, &pMeta->pIdx->pIdx) != 0) { - // return -1; - //} - -#endif - return 0; -} - -#ifdef BUILD_NO_CALL -void metaCloseIdx(SMeta *pMeta) { /* TODO */ -#if 0 - if (pMeta->pIdx) { - rocksdb_close(pMeta->pIdx); - pMeta->pIdx = NULL; - } -#endif - -#ifdef USE_INVERTED_INDEX - // SIndexOpts opts; - // if (indexClose(pMeta->pIdx->pIdx) != 0) { - // return -1; - //} - // return 0; - -#endif -} - -int metaSaveTableToIdx(SMeta *pMeta, const STbCfg *pTbCfg) { -#ifdef USE_INVERTED_INDEX - // if (pTbCfgs->type == META_CHILD_TABLE) { - // char buf[8] = {0}; - // int16_t colId = (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId; - // sprintf(buf, "%d", colId); // colname - - // char *pTagVal = (char *)tdGetKVRowValOfCol(pTbCfg->ctbCfg.pTag, (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId); - - // tb_uid_t suid = pTbCfg->ctbCfg.suid; // super id - // tb_uid_t tuid = 0; // child table uid - // SIndexMultiTerm *terms = indexMultiTermCreate(); - // SIndexTerm *term = - // indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_BINARY, buf, strlen(buf), pTagVal, strlen(pTagVal), tuid); - // indexMultiTermAdd(terms, term); - - // int ret = indexPut(pMeta->pIdx->pIdx, terms); - // indexMultiTermDestroy(terms); - // return ret; - //} else { - // return DB_DONOTINDEX; - //} -#endif - // TODO - return 0; -} - -int metaRemoveTableFromIdx(SMeta *pMeta, tb_uid_t uid) { -#ifdef USE_INVERTED_INDEX - -#endif - // TODO - return 0; -} -#endif \ No newline at end of file diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index e3b3de6cd6..94eb461613 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -39,13 +39,14 @@ static int32_t metaDestroyLock(SMeta *pMeta) { return taosThreadRwlockDestroy(&p static void metaCleanup(SMeta **ppMeta); -int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { - SMeta *pMeta = NULL; - int ret; - int offset; - char path[TSDB_FILENAME_LEN] = {0}; - - *ppMeta = NULL; +int32_t metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { + SMeta *pMeta = NULL; + int32_t code = 0; + int32_t lino; + int32_t offset; + char path[TSDB_FILENAME_LEN] = {0}; + char indexFullPath[128] = {0}; + sprintf(indexFullPath, "%s/%s", pMeta->path, "invert"); // create handle vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, path, TSDB_FILENAME_LEN); @@ -53,8 +54,7 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { snprintf(path + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, VNODE_META_DIR); if ((pMeta = taosMemoryCalloc(1, sizeof(*pMeta) + strlen(path) + 1)) == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } metaInitLock(pMeta); @@ -69,163 +69,103 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { taosMkDir(pMeta->path); // open env - ret = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv, rollback, - pVnode->config.tdbEncryptAlgorithm, pVnode->config.tdbEncryptKey); - if (ret < 0) { - metaError("vgId:%d, failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv, rollback, + pVnode->config.tdbEncryptAlgorithm, pVnode->config.tdbEncryptKey); + TSDB_CHECK_CODE(code, lino, _exit); // open pTbDb - ret = tdbTbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb, 0); - if (ret < 0) { - metaError("vgId:%d, failed to open meta table db since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = tdbTbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb, 0); + TSDB_CHECK_CODE(code, lino, _exit); // open pSkmDb - ret = tdbTbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb, 0); - if (ret < 0) { - metaError("vgId:%d, failed to open meta schema db since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = tdbTbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb, 0); + TSDB_CHECK_CODE(code, lino, _exit); // open pUidIdx - ret = tdbTbOpen("uid.idx", sizeof(tb_uid_t), sizeof(SUidIdxVal), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx, 0); - if (ret < 0) { - metaError("vgId:%d, failed to open meta uid idx since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = tdbTbOpen("uid.idx", sizeof(tb_uid_t), sizeof(SUidIdxVal), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx, 0); + TSDB_CHECK_CODE(code, lino, _exit); // open pNameIdx - ret = tdbTbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx, 0); - if (ret < 0) { - metaError("vgId:%d, failed to open meta name index since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = tdbTbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx, 0); + TSDB_CHECK_CODE(code, lino, _exit); // open pCtbIdx - ret = tdbTbOpen("ctb.idx", sizeof(SCtbIdxKey), -1, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx, 0); - if (ret < 0) { - metaError("vgId:%d, failed to open meta child table index since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = tdbTbOpen("ctb.idx", sizeof(SCtbIdxKey), -1, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx, 0); + TSDB_CHECK_CODE(code, lino, _exit); // open pSuidIdx - ret = tdbTbOpen("suid.idx", sizeof(tb_uid_t), 0, uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pSuidIdx, 0); - if (ret < 0) { - metaError("vgId:%d, failed to open meta super table index since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = tdbTbOpen("suid.idx", sizeof(tb_uid_t), 0, uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pSuidIdx, 0); + TSDB_CHECK_CODE(code, lino, _exit); - char indexFullPath[128] = {0}; - sprintf(indexFullPath, "%s/%s", pMeta->path, "invert"); - taosMkDir(indexFullPath); + TAOS_UNUSED(taosMkDir(indexFullPath)); SIndexOpts opts = {.cacheSize = 8 * 1024 * 1024}; - ret = indexOpen(&opts, indexFullPath, (SIndex **)&pMeta->pTagIvtIdx); - if (ret < 0) { - metaError("vgId:%d, failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = indexOpen(&opts, indexFullPath, (SIndex **)&pMeta->pTagIvtIdx); + TSDB_CHECK_CODE(code, lino, _exit); - ret = tdbTbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx, 0); - if (ret < 0) { - metaError("vgId:%d, failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = tdbTbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx, 0); + TSDB_CHECK_CODE(code, lino, _exit); // open pTtlMgr ("ttlv1.idx") char logPrefix[128] = {0}; sprintf(logPrefix, "vgId:%d", TD_VID(pVnode)); - ret = ttlMgrOpen(&pMeta->pTtlMgr, pMeta->pEnv, 0, logPrefix, tsTtlFlushThreshold); - if (ret < 0) { - metaError("vgId:%d, failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = ttlMgrOpen(&pMeta->pTtlMgr, pMeta->pEnv, 0, logPrefix, tsTtlFlushThreshold); + TSDB_CHECK_CODE(code, lino, _exit); // open pSmaIdx - ret = tdbTbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx, 0); - if (ret < 0) { - metaError("vgId:%d, failed to open meta sma index since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = tdbTbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx, 0); + TSDB_CHECK_CODE(code, lino, _exit); // idx table create time - ret = tdbTbOpen("ctime.idx", sizeof(SBtimeIdxKey), 0, btimeIdxCmpr, pMeta->pEnv, &pMeta->pBtimeIdx, 0); - if (ret < 0) { - metaError("vgId:%d, failed to open meta ctime index since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = tdbTbOpen("ctime.idx", sizeof(SBtimeIdxKey), 0, btimeIdxCmpr, pMeta->pEnv, &pMeta->pBtimeIdx, 0); + TSDB_CHECK_CODE(code, lino, _exit); // idx num of col, normal table only - ret = tdbTbOpen("ncol.idx", sizeof(SNcolIdxKey), 0, ncolIdxCmpr, pMeta->pEnv, &pMeta->pNcolIdx, 0); - if (ret < 0) { - metaError("vgId:%d, failed to open meta ncol index since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = tdbTbOpen("ncol.idx", sizeof(SNcolIdxKey), 0, ncolIdxCmpr, pMeta->pEnv, &pMeta->pNcolIdx, 0); + TSDB_CHECK_CODE(code, lino, _exit); - ret = tdbTbOpen("stream.task.db", sizeof(int64_t), -1, taskIdxKeyCmpr, pMeta->pEnv, &pMeta->pStreamDb, 0); - if (ret < 0) { - metaError("vgId:%d, failed to open meta stream task index since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = tdbTbOpen("stream.task.db", sizeof(int64_t), -1, taskIdxKeyCmpr, pMeta->pEnv, &pMeta->pStreamDb, 0); + TSDB_CHECK_CODE(code, lino, _exit); - // open index - if (metaOpenIdx(pMeta) < 0) { - metaError("vgId:%d, failed to open meta index since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = metaCacheOpen(pMeta); + TSDB_CHECK_CODE(code, lino, _exit); - int32_t code = metaCacheOpen(pMeta); + code = metaInitTbFilterCache(pMeta); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: if (code) { - terrno = code; - metaError("vgId:%d, failed to open meta cache since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; + metaError("vgId:%d %s failed at %s:%d since %s", TD_VID(pVnode), __func__, __FILE__, __LINE__, tstrerror(code)); + metaCleanup(&pMeta); + *ppMeta = NULL; + } else { + metaDebug("vgId:%d %s success", TD_VID(pVnode), __func__); + *ppMeta = pMeta; } - - if (metaInitTbFilterCache(pMeta) != 0) { - goto _err; - } - - metaDebug("vgId:%d, meta is opened", TD_VID(pVnode)); - - *ppMeta = pMeta; - return 0; - -_err: - metaCleanup(&pMeta); - return -1; + return code; } -int metaUpgrade(SVnode *pVnode, SMeta **ppMeta) { - int code = TSDB_CODE_SUCCESS; - SMeta *pMeta = *ppMeta; +int32_t metaUpgrade(SVnode *pVnode, SMeta **ppMeta) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino; + SMeta *pMeta = *ppMeta; if (ttlMgrNeedUpgrade(pMeta->pEnv)) { code = metaBegin(pMeta, META_BEGIN_HEAP_OS); - if (code < 0) { - metaError("vgId:%d, failed to upgrade meta, meta begin failed since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + TSDB_CHECK_CODE(code, lino, _exit); code = ttlMgrUpgrade(pMeta->pTtlMgr, pMeta); - if (code < 0) { - metaError("vgId:%d, failed to upgrade meta ttl since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + TSDB_CHECK_CODE(code, lino, _exit); code = metaCommit(pMeta, pMeta->txn); - if (code < 0) { - metaError("vgId:%d, failed to upgrade meta ttl, meta commit failed since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + TSDB_CHECK_CODE(code, lino, _exit); } - return TSDB_CODE_SUCCESS; - -_err: - metaCleanup(ppMeta); +_exit: + if (code) { + metaError("vgId:%d %s failed at %s:%d since %s", TD_VID(pVnode), __func__, __FILE__, __LINE__, tstrerror(code)); + metaCleanup(ppMeta); + } return code; } @@ -235,33 +175,42 @@ int metaClose(SMeta **ppMeta) { } int metaAlterCache(SMeta *pMeta, int32_t nPage) { + int32_t code = 0; metaWLock(pMeta); - - if (tdbAlter(pMeta->pEnv, nPage) < 0) { - metaULock(pMeta); - return -1; - } - + code = tdbAlter(pMeta->pEnv, nPage); metaULock(pMeta); - return 0; + + if (code) { + metaError("vgId:%d %s failed since %s", TD_VID(pMeta->pVnode), __func__, tstrerror(code)); + } + return code; } int32_t metaRLock(SMeta *pMeta) { metaTrace("meta rlock %p", &pMeta->lock); - int32_t ret = taosThreadRwlockRdlock(&pMeta->lock); - return ret; + int32_t code = taosThreadRwlockRdlock(&pMeta->lock); + if (code) { + return TAOS_SYSTEM_ERROR(code); + } + return 0; } int32_t metaWLock(SMeta *pMeta) { metaTrace("meta wlock %p", &pMeta->lock); - int32_t ret = taosThreadRwlockWrlock(&pMeta->lock); - return ret; + int32_t code = taosThreadRwlockWrlock(&pMeta->lock); + if (code) { + return TAOS_SYSTEM_ERROR(code); + } + return 0; } int32_t metaULock(SMeta *pMeta) { metaTrace("meta ulock %p", &pMeta->lock); - int32_t ret = taosThreadRwlockUnlock(&pMeta->lock); - return ret; + int32_t code = taosThreadRwlockUnlock(&pMeta->lock); + if (code) { + return TAOS_SYSTEM_ERROR(code); + } + return 0; } static void metaCleanup(SMeta **ppMeta) { diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index 34d4f8e845..95c65e5e80 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -25,14 +25,14 @@ struct SMetaSnapReader { int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader) { int32_t code = 0; + int32_t lino; int32_t c = 0; SMetaSnapReader* pReader = NULL; // alloc pReader = (SMetaSnapReader*)taosMemoryCalloc(1, sizeof(*pReader)); if (pReader == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } pReader->pMeta = pMeta; pReader->sver = sver; @@ -40,36 +40,29 @@ int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapRe // impl code = tdbTbcOpen(pMeta->pTbDb, &pReader->pTbc, NULL); - if (code) { - taosMemoryFree(pReader); - goto _err; - } + TSDB_CHECK_CODE(code, lino, _exit); code = tdbTbcMoveTo(pReader->pTbc, &(STbDbKey){.version = sver, .uid = INT64_MIN}, sizeof(STbDbKey), &c); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: if (code) { - taosMemoryFree(pReader); - goto _err; + metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, lino, tstrerror(code)); + metaSnapReaderClose(&pReader); + *ppReader = NULL; + } else { + metaInfo("vgId:%d, %s success", TD_VID(pMeta->pVnode), __func__); + *ppReader = pReader; } - - metaInfo("vgId:%d, vnode snapshot meta reader opened", TD_VID(pMeta->pVnode)); - - *ppReader = pReader; - return code; - -_err: - metaError("vgId:%d, vnode snapshot meta reader open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code)); - *ppReader = NULL; return code; } -int32_t metaSnapReaderClose(SMetaSnapReader** ppReader) { - int32_t code = 0; - - tdbTbcClose((*ppReader)->pTbc); - taosMemoryFree(*ppReader); - *ppReader = NULL; - - return code; +void metaSnapReaderClose(SMetaSnapReader** ppReader) { + if (ppReader && *ppReader) { + tdbTbcClose((*ppReader)->pTbc); + taosMemoryFree(*ppReader); + *ppReader = NULL; + } } int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) { @@ -106,7 +99,7 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) { *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + nData); if (*ppData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + goto _exit; } SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData); @@ -122,10 +115,10 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) { } _exit: - return code; - -_err: - metaError("vgId:%d, vnode snapshot meta read data failed since %s", TD_VID(pReader->pMeta->pVnode), tstrerror(code)); + if (code) { + metaError("vgId:%d, vnode snapshot meta read data failed since %s", TD_VID(pReader->pMeta->pVnode), + tstrerror(code)); + } return code; } @@ -138,26 +131,30 @@ struct SMetaSnapWriter { int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter) { int32_t code = 0; + int32_t lino; SMetaSnapWriter* pWriter; // alloc pWriter = (SMetaSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter)); if (pWriter == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } pWriter->pMeta = pMeta; pWriter->sver = sver; pWriter->ever = ever; - metaBegin(pMeta, META_BEGIN_HEAP_NIL); + code = metaBegin(pMeta, META_BEGIN_HEAP_NIL); + TSDB_CHECK_CODE(code, lino, _exit); - *ppWriter = pWriter; - return code; - -_err: - metaError("vgId:%d, meta snapshot writer open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code)); - *ppWriter = NULL; +_exit: + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, lino, tstrerror(code)); + taosMemoryFree(pWriter); + *ppWriter = NULL; + } else { + metaDebug("vgId:%d, %s success", TD_VID(pMeta->pVnode), __func__); + *ppWriter = pWriter; + } return code; } @@ -189,25 +186,24 @@ _err: int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; - int32_t line = 0; + int32_t lino = 0; SMeta* pMeta = pWriter->pMeta; SMetaEntry metaEntry = {0}; SDecoder* pDecoder = &(SDecoder){0}; tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); code = metaDecodeEntry(pDecoder, &metaEntry); - VND_CHECK_CODE(code, line, _err); + TSDB_CHECK_CODE(code, lino, _exit); code = metaHandleEntry(pMeta, &metaEntry); - VND_CHECK_CODE(code, line, _err); + TSDB_CHECK_CODE(code, lino, _exit); +_exit: + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, lino, tstrerror(code)); + } tDecoderClear(pDecoder); return code; - -_err: - tDecoderClear(pDecoder); - metaError("vgId:%d, vnode snapshot meta write failed since %s at line:%d", TD_VID(pMeta->pVnode), terrstr(), line); - return code; } typedef struct STableInfoForChildTable { @@ -468,19 +464,17 @@ int32_t setForSnapShot(SSnapContext* ctx, int64_t uid) { return c; } -void taosXSetTablePrimaryKey(SSnapContext* ctx, int64_t uid){ - bool ret = false; - SSchemaWrapper *schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1); - if (schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY){ +void taosXSetTablePrimaryKey(SSnapContext* ctx, int64_t uid) { + bool ret = false; + SSchemaWrapper* schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1); + if (schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) { ret = true; } tDeleteSchemaWrapper(schema); ctx->hasPrimaryKey = ret; } -bool taosXGetTablePrimaryKey(SSnapContext* ctx){ - return ctx->hasPrimaryKey; -} +bool taosXGetTablePrimaryKey(SSnapContext* ctx) { return ctx->hasPrimaryKey; } int32_t getTableInfoFromSnapshot(SSnapContext* ctx, void** pBuf, int32_t* contLen, int16_t* type, int64_t* uid) { int32_t ret = 0; diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 75ba2be100..046f12f9e7 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -312,8 +312,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) goto _exit; } else { pReader->metaDone = 1; - code = metaSnapReaderClose(&pReader->pMetaReader); - TSDB_CHECK_CODE(code, lino, _exit); + metaSnapReaderClose(&pReader->pMetaReader); } } From eaf499ad1ab0dc260a55bf1707c684af109904de Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 23 Jul 2024 06:17:46 +0000 Subject: [PATCH 20/43] fix/TD-30989 --- source/dnode/mnode/impl/src/mndSubscribe.c | 113 ++++++++++++--------- source/dnode/mnode/impl/src/mndSync.c | 51 ++++++---- 2 files changed, 94 insertions(+), 70 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index e2bedc258a..c61949b316 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -44,11 +44,16 @@ static void mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter); static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj *hash); static int32_t mndSetSubCommitLogs(STrans *pTrans, SMqSubscribeObj *pSub) { + int32_t code = 0; SSdbRaw *pCommitRaw = mndSubActionEncode(pSub); - if (pCommitRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; - return 0; + if (pCommitRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw)); + TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)); + TAOS_RETURN(code); } int32_t mndInitSubscribe(SMnode *pMnode) { @@ -75,6 +80,7 @@ int32_t mndInitSubscribe(SMnode *pMnode) { } static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *subKey) { + int32_t code = 0; SMqSubscribeObj *pSub = tNewSubscribeObj(subKey); if (pSub == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -86,7 +92,7 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj pSub->subType = pTopic->subType; pSub->withMeta = pTopic->withMeta; - if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) { + if ((terrno = mndSchedInitSubEp(pMnode, pTopic, pSub)) < 0) { tDeleteSubscribeObj(pSub); taosMemoryFree(pSub); return NULL; @@ -97,6 +103,7 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg, SSubplan *pPlan) { + int32_t code = 0; SMqRebVgReq req = {0}; req.oldConsumerId = pRebVg->oldConsumerId; req.newConsumerId = pRebVg->newConsumerId; @@ -106,8 +113,8 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj pPlan->execNode.nodeId = pRebVg->pVgEp->vgId; int32_t msgLen; if (qSubPlanToString(pPlan, &req.qmsg, &msgLen) < 0) { - terrno = TSDB_CODE_QRY_INVALID_INPUT; - return -1; + code = TSDB_CODE_QRY_INVALID_INPUT; + TAOS_RETURN(code); } } else { req.qmsg = taosStrdup(""); @@ -122,7 +129,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj tEncodeSize(tEncodeSMqRebVgReq, &req, tlen, ret); if (ret < 0) { taosMemoryFree(req.qmsg); - return -1; + TAOS_RETURN(ret); } tlen += sizeof(SMsgHead); @@ -130,7 +137,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(req.qmsg); - return -1; + TAOS_RETURN(ret); } SMsgHead *pMsgHead = (SMsgHead *)buf; @@ -140,40 +147,41 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj SEncoder encoder = {0}; tEncoderInit(&encoder, POINTER_SHIFT(buf, sizeof(SMsgHead)), tlen); - if (tEncodeSMqRebVgReq(&encoder, &req) < 0) { + if ((code = tEncodeSMqRebVgReq(&encoder, &req)) < 0) { taosMemoryFreeClear(buf); tEncoderClear(&encoder); taosMemoryFree(req.qmsg); - return -1; + TAOS_RETURN(code); } tEncoderClear(&encoder); *pBuf = buf; *pLen = tlen; taosMemoryFree(req.qmsg); - return 0; + TAOS_RETURN(code); } static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg, SSubplan *pPlan) { + int32_t code = 0; if (pRebVg->oldConsumerId == pRebVg->newConsumerId) { if (pRebVg->oldConsumerId == -1) return 0; // drop stream, no consumer, while split vnode,all consumerId is -1 - terrno = TSDB_CODE_MND_INVALID_SUB_OPTION; - return -1; + code = TSDB_CODE_MND_INVALID_SUB_OPTION; + TAOS_RETURN(code); } void *buf; int32_t tlen; - if (mndBuildSubChangeReq(&buf, &tlen, pSub, pRebVg, pPlan) < 0) { - return -1; + if ((code = mndBuildSubChangeReq(&buf, &tlen, pSub, pRebVg, pPlan)) < 0) { + TAOS_RETURN(code); } int32_t vgId = pRebVg->pVgEp->vgId; SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); if (pVgObj == NULL) { taosMemoryFree(buf); - terrno = TSDB_CODE_MND_VGROUP_NOT_EXIST; - return -1; + code = TSDB_CODE_MND_VGROUP_NOT_EXIST; + TAOS_RETURN(code); } STransAction action = {0}; @@ -183,11 +191,11 @@ static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, SMqSubsc action.msgType = TDMT_VND_TMQ_SUBSCRIBE; mndReleaseVgroup(pMnode, pVgObj); - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(buf); - return -1; + TAOS_RETURN(code); } - return 0; + TAOS_RETURN(code); } static int32_t mndSplitSubscribeKey(const char *key, char *topic, char *cgroup, bool fullName) { @@ -209,6 +217,7 @@ static int32_t mndSplitSubscribeKey(const char *key, char *topic, char *cgroup, } static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { + terrno = 0; SMqRebInfo *pRebInfo = taosHashGet(pHash, key, strlen(key) + 1); if (pRebInfo == NULL) { pRebInfo = tNewSMqRebSubscribe(key); @@ -612,7 +621,6 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu if (strcmp(pOutput->pSub->qmsg, "") != 0) { code = qStringToSubplan(pOutput->pSub->qmsg, &pPlan); if (code != 0) { - terrno = code; goto END; } } @@ -623,7 +631,8 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "tmq-reb"); if (pTrans == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; goto END; } @@ -665,7 +674,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu END: nodesDestroyNode((SNode *)pPlan); mndTransDrop(pTrans); - return code; + TAOS_RETURN(code); } static void freeRebalanceItem(void *param) { @@ -827,6 +836,7 @@ static void checkConsumer(SMnode *pMnode, SMqSubscribeObj *pSub) { } static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOutputObj *rebOutput) { + int32_t code = 0; const char *key = rebInput->pRebInfo->key; SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, key); @@ -838,8 +848,10 @@ static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOu SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); if (pTopic == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; mError("[rebalance] mq rebalance %s ignored since topic %s doesn't exist", key, topic); - return -1; + TAOS_RETURN(code); } taosRLockLatch(&pTopic->lock); @@ -848,10 +860,12 @@ static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOu rebOutput->pSub = mndCreateSubscription(pMnode, pTopic, key); if (rebOutput->pSub == NULL) { - mError("[rebalance] mq rebalance %s failed create sub since %s, ignore", key, terrstr()); + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + mError("[rebalance] mq rebalance %s failed create sub since %s, ignore", key, tstrerror(code)); taosRUnLockLatch(&pTopic->lock); mndReleaseTopic(pMnode, pTopic); - return -1; + TAOS_RETURN(code); } memcpy(rebOutput->pSub->dbName, pTopic->db, TSDB_DB_FNAME_LEN); @@ -869,7 +883,7 @@ static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOu mInfo("[rebalance] sub topic:%s has %d consumers sub till now", key, rebInput->oldConsumerNum); mndReleaseSubscribe(pMnode, pSub); } - return 0; + TAOS_RETURN(code); } static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { @@ -885,6 +899,7 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { SHashObj *rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); if (rebSubHash == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; + if (terrno != 0) code = terrno; goto END; } taosHashSetFreeFp(rebSubHash, freeRebalanceItem); @@ -915,8 +930,8 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { mndDoRebalance(pMnode, &rebInput, &rebOutput); - if (mndPersistRebResult(pMnode, pMsg, &rebOutput) != 0) { - mError("mq rebalance persist output error, possibly vnode splitted or dropped,msg:%s", terrstr()) + if ((code = mndPersistRebResult(pMnode, pMsg, &rebOutput)) != 0) { + mError("mq rebalance persist output error, possibly vnode splitted or dropped,msg:%s", tstrerror(code)) } clearRebOutput(&rebOutput); @@ -931,7 +946,7 @@ END: taosHashCleanup(rebSubHash); mndRebCntDec(); - return code; + TAOS_RETURN(code); } static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STrans *pTrans) { @@ -950,8 +965,7 @@ static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STran } SMqVDeleteReq *pReq = taosMemoryCalloc(1, sizeof(SMqVDeleteReq)); if (pReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - ret = -1; + ret = TSDB_CODE_OUT_OF_MEMORY; goto END; } pReq->head.vgId = htonl(pVgObj->vgId); @@ -968,15 +982,14 @@ static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STran action.acceptableCode = TSDB_CODE_MND_VGROUP_NOT_EXIST; sdbRelease(pMnode->pSdb, pVgObj); - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - ret = -1; + if ((ret = mndTransAppendRedoAction(pTrans, &action)) != 0) { goto END; } } END: sdbRelease(pMnode->pSdb, pVgObj); sdbCancelFetch(pMnode->pSdb, pIter); - return ret; + TAOS_RETURN(ret); } static int32_t mndDropConsumerByGroup(SMnode *pMnode, STrans *pTrans, char *cgroup, char *topic) { @@ -1012,7 +1025,7 @@ static int32_t mndDropConsumerByGroup(SMnode *pMnode, STrans *pTrans, char *cgro END: sdbRelease(pMnode->pSdb, pConsumer); sdbCancelFetch(pMnode->pSdb, pIter); - return ret; + TAOS_RETURN(ret); } static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) { @@ -1022,8 +1035,8 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) { int32_t code = TSDB_CODE_ACTION_IN_PROGRESS; if (tDeserializeSMDropCgroupReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; + code = TSDB_CODE_INVALID_MSG; + TAOS_RETURN(code); } SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, dropReq.cgroup, dropReq.topic); @@ -1032,24 +1045,24 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) { mInfo("cgroup:%s on topic:%s, not exist, ignore not exist is set", dropReq.cgroup, dropReq.topic); return 0; } else { - terrno = TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST; - mError("topic:%s, cgroup:%s, failed to drop since %s", dropReq.topic, dropReq.cgroup, terrstr()); - return -1; + code = TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST; + mError("topic:%s, cgroup:%s, failed to drop since %s", dropReq.topic, dropReq.cgroup, tstrerror(code)); + TAOS_RETURN(code); } } taosWLockLatch(&pSub->lock); if (taosHashGetSize(pSub->consumerHash) != 0) { - terrno = TSDB_CODE_MND_CGROUP_USED; - mError("cgroup:%s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr()); - code = -1; + code = TSDB_CODE_MND_CGROUP_USED; + mError("cgroup:%s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, tstrerror(code)); goto end; } pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "drop-cgroup"); if (pTrans == NULL) { - mError("cgroup: %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr()); - code = -1; + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + mError("cgroup: %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, tstrerror(code)); goto end; } @@ -1088,9 +1101,9 @@ end: if (code != 0) { mError("cgroup %s on topic:%s, failed to drop", dropReq.cgroup, dropReq.topic); - return code; + TAOS_RETURN(code); } - return TSDB_CODE_ACTION_IN_PROGRESS; + TAOS_RETURN(TSDB_CODE_ACTION_IN_PROGRESS); } void mndCleanupSubscribe(SMnode *pMnode) {} @@ -1340,7 +1353,7 @@ END: sdbRelease(pSdb, pSub); sdbCancelFetch(pSdb, pIter); - return code; + TAOS_RETURN(code); } static int32_t buildResult(SSDataBlock *pBlock, int32_t *numOfRows, int64_t consumerId, const char *topic, diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 573b75ff5a..70d0b858f6 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -123,15 +123,16 @@ _OUT: } static int32_t mndTransValidateImp(SMnode *pMnode, STrans *pTrans) { + int32_t code = 0; if (pTrans->stage == TRN_STAGE_PREPARE) { - if (mndTransCheckConflict(pMnode, pTrans) < 0) { + if ((code = mndTransCheckConflict(pMnode, pTrans)) < 0) { mError("trans:%d, failed to validate trans conflicts.", pTrans->id); - return -1; + TAOS_RETURN(code); } return mndTransValidatePrepareStage(pMnode, pTrans); } - return 0; + TAOS_RETURN(code); } static int32_t mndTransValidate(SMnode *pMnode, SSdbRaw *pRaw) { @@ -139,10 +140,18 @@ static int32_t mndTransValidate(SMnode *pMnode, SSdbRaw *pRaw) { int32_t code = -1; SSdbRow *pRow = mndTransDecode(pRaw); - if (pRow == NULL) goto _OUT; + if (pRow == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OUT; + } pTrans = sdbGetRowObj(pRow); - if (pTrans == NULL) goto _OUT; + if (pTrans == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OUT; + } code = mndTransValidateImp(pMnode, pTrans); @@ -150,11 +159,10 @@ _OUT: if (pTrans) mndTransDropData(pTrans); if (pRow) taosMemoryFreeClear(pRow); if (code) terrno = (terrno ? terrno : TSDB_CODE_MND_TRANS_CONFLICT); - return code; + TAOS_RETURN(code); } int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { - terrno = TSDB_CODE_SUCCESS; SSyncMgmt *pMgmt = &pMnode->syncMgmt; SSdbRaw *pRaw = pMsg->pCont; STrans *pTrans = NULL; @@ -163,7 +171,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { if (transId <= 0) { mError("trans:%d, invalid commit msg, cache transId:%d seq:%" PRId64, transId, pMgmt->transId, pMgmt->transSeq); - terrno = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_INVALID_MSG; goto _OUT; } @@ -176,7 +184,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { if (code != 0) { mError("trans:%d, failed to validate requested trans since %s", transId, terrstr()); code = 0; - pMeta->code = terrno; + pMeta->code = code; goto _OUT; } @@ -184,13 +192,15 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { if (code != 0) { mError("trans:%d, failed to write to sdb since %s", transId, terrstr()); code = 0; - pMeta->code = terrno; + pMeta->code = code; goto _OUT; } pTrans = mndAcquireTrans(pMnode, transId); if (pTrans == NULL) { - mError("trans:%d, not found while execute in mnode since %s", transId, terrstr()); + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + mError("trans:%d, not found while execute in mnode since %s", transId, tstrerror(code)); goto _OUT; } @@ -207,7 +217,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { _OUT: if (pTrans) mndReleaseTrans(pMnode, pTrans); - return code; + TAOS_RETURN(code); } static int32_t mndPostMgmtCode(SMnode *pMnode, int32_t code) { @@ -258,7 +268,7 @@ _OUT: mndPostMgmtCode(pMnode, code ? code : pMeta->code); rpcFreeCont(pMsg->pCont); pMsg->pCont = NULL; - return code; + TAOS_RETURN(code); } SyncIndex mndSyncAppliedIndex(const SSyncFSM *pFSM) { @@ -466,16 +476,18 @@ int32_t mndInitSync(SMnode *pMnode) { pNode->clusterId); } + int32_t code = 0; tsem_init(&pMgmt->syncSem, 0, 0); pMgmt->sync = syncOpen(&syncInfo, true); if (pMgmt->sync <= 0) { - mError("failed to open sync since %s", terrstr()); - return -1; + if (terrno != 0) code = terrno; + mError("failed to open sync since %s", tstrerror(code)); + TAOS_RETURN(code); } pMnode->pSdb->sync = pMgmt->sync; mInfo("mnode-sync is opened, id:%" PRId64, pMgmt->sync); - return 0; + TAOS_RETURN(code); } void mndCleanupSync(SMnode *pMnode) { @@ -518,10 +530,10 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; SRpcMsg req = {.msgType = TDMT_MND_APPLY_MSG, .contLen = sdbGetRawTotalSize(pRaw)}; - if (req.contLen <= 0) return -1; + if (req.contLen <= 0) return TSDB_CODE_OUT_OF_MEMORY; req.pCont = rpcMallocCont(req.contLen); - if (req.pCont == NULL) return -1; + if (req.pCont == NULL) return TSDB_CODE_OUT_OF_MEMORY; memcpy(req.pCont, pRaw, req.contLen); taosThreadMutexLock(&pMgmt->lock); @@ -531,8 +543,7 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { mError("trans:%d, can't be proposed since trans:%d already waiting for confirm", transId, pMgmt->transId); taosThreadMutexUnlock(&pMgmt->lock); rpcFreeCont(req.pCont); - terrno = TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED; - return terrno; + TAOS_RETURN(TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED); } mInfo("trans:%d, will be proposed", transId); From 1ebec2c402e9c279de8a61396b42d4c2761aa76d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 23 Jul 2024 14:50:45 +0800 Subject: [PATCH 21/43] enh: refactor error code --- source/libs/tdb/CMakeLists.txt | 1 - source/libs/tdb/src/db/tdbBtree.c | 4 +- source/libs/tdb/src/db/tdbDb.c | 37 ++--- source/libs/tdb/src/db/tdbOs.c | 98 ------------ source/libs/tdb/src/db/tdbPCache.c | 57 ++++--- source/libs/tdb/src/db/tdbPage.c | 60 ++++--- source/libs/tdb/src/db/tdbPager.c | 244 ++++++++++++++--------------- source/libs/tdb/src/db/tdbTable.c | 10 +- source/libs/tdb/src/db/tdbTxn.c | 4 +- source/libs/tdb/src/db/tdbUtil.c | 7 +- source/libs/tdb/src/inc/tdbInt.h | 19 +-- source/os/src/osFile.c | 30 ++-- 12 files changed, 232 insertions(+), 339 deletions(-) delete mode 100644 source/libs/tdb/src/db/tdbOs.c diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index cc50919413..bb8d33366c 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -10,7 +10,6 @@ target_sources(tdb "src/db/tdbTable.c" "src/db/tdbTxn.c" "src/db/tdbPage.c" - "src/db/tdbOs.c" ) target_include_directories( diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 4cca63a966..4dd3d3298c 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -76,14 +76,14 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg if (keyLen == 0) { tdbError("tdb/btree-open: key len cannot be zero."); - return -1; + return TSDB_CODE_INVALID_PARA; } *ppBt = NULL; pBt = (SBTree *)tdbOsCalloc(1, sizeof(*pBt)); if (pBt == NULL) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } // pBt->keyLen diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 9a1d89eeac..b9ba9de51f 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -15,8 +15,8 @@ #include "tdbInt.h" -int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, int8_t rollback, int32_t encryptAlgorithm, - char *encryptKey) { +int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, int8_t rollback, + int32_t encryptAlgorithm, char *encryptKey) { TDB *pDb; int dsize; int zsize; @@ -31,7 +31,7 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i pPtr = (uint8_t *)tdbOsCalloc(1, zsize); if (pPtr == NULL) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } pDb = (TDB *)pPtr; @@ -51,38 +51,38 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i pDb->jfd = -1; pDb->encryptAlgorithm = encryptAlgorithm; - if(encryptKey != NULL){ - strncpy(pDb->encryptKey, encryptKey, ENCRYPT_KEY_LEN); + if (encryptKey != NULL) { + strncpy(pDb->encryptKey, encryptKey, ENCRYPT_KEY_LEN); } ret = tdbPCacheOpen(szPage, pages, &(pDb->pCache)); if (ret < 0) { - return -1; + return ret; } pDb->nPgrHash = 8; tsize = sizeof(SPager *) * pDb->nPgrHash; pDb->pgrHash = tdbOsMalloc(tsize); if (pDb->pgrHash == NULL) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } memset(pDb->pgrHash, 0, tsize); ret = taosMulModeMkDir(dbname, 0755, false); if (ret < 0) { - return -1; + return TAOS_SYSTEM_ERROR(ret); } #ifdef USE_MAINDB // open main db ret = tdbTbOpen(TDB_MAINDB_NAME, -1, sizeof(SBtInfo), NULL, pDb, &pDb->pMainDb, rollback); if (ret < 0) { - return -1; + return ret; } ret = tdbTbOpen(TDB_FREEDB_NAME, sizeof(SPgno), 0, NULL, pDb, &pDb->pFreeDb, rollback); if (ret < 0) { - return -1; + return ret; } #endif @@ -125,12 +125,13 @@ int32_t tdbBegin(TDB *pDb, TXN **ppTxn, void *(*xMalloc)(void *, size_t), void ( TXN *pTxn = tdbOsCalloc(1, sizeof(*pTxn)); if (!pTxn) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } - if (tdbTxnOpen(pTxn, txnId, xMalloc, xFree, xArg, flags) < 0) { + ret = tdbTxnOpen(pTxn, txnId, xMalloc, xFree, xArg, flags); + if (ret < 0) { tdbOsFree(pTxn); - return -1; + return ret; } for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { @@ -139,7 +140,7 @@ int32_t tdbBegin(TDB *pDb, TXN **ppTxn, void *(*xMalloc)(void *, size_t), void ( tdbError("failed to begin pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, pTxn->txnId); tdbTxnClose(pTxn); - return -1; + return ret; } } @@ -157,7 +158,7 @@ int32_t tdbCommit(TDB *pDb, TXN *pTxn) { if (ret < 0) { tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, pTxn->txnId); - return -1; + return ret; } } @@ -173,7 +174,7 @@ int32_t tdbPostCommit(TDB *pDb, TXN *pTxn) { if (ret < 0) { tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, pTxn->txnId); - return -1; + return ret; } } @@ -191,7 +192,7 @@ int32_t tdbPrepareAsyncCommit(TDB *pDb, TXN *pTxn) { if (ret < 0) { tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, pTxn->txnId); - return -1; + return ret; } } @@ -207,7 +208,7 @@ int32_t tdbAbort(TDB *pDb, TXN *pTxn) { if (ret < 0) { tdbError("failed to abort pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, pTxn->txnId); - return -1; + return ret; } } diff --git a/source/libs/tdb/src/db/tdbOs.c b/source/libs/tdb/src/db/tdbOs.c deleted file mode 100644 index a2fce8925b..0000000000 --- a/source/libs/tdb/src/db/tdbOs.c +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tdbOs.h" - -#ifndef TDB_FOR_TDENGINE - -// tdbOsRead -i64 tdbOsRead(tdb_fd_t fd, void *pData, i64 nBytes) { - i64 nRead = 0; - i64 iRead = 0; - u8 *pBuf = (u8 *)pData; - - while (nBytes > 0) { - iRead = read(fd, pBuf, nBytes); - if (iRead < 0) { - if (errno == EINTR) { - continue; - } else { - return -1; - } - } else if (iRead == 0) { - break; - } - - nRead += iRead; - pBuf += iRead; - nBytes -= iRead; - } - - return nRead; -} - -// tdbOsPRead -i64 tdbOsPRead(tdb_fd_t fd, void *pData, i64 nBytes, i64 offset) { - i64 nRead = 0; - i64 iRead = 0; - i64 iOffset = offset; - u8 *pBuf = (u8 *)pData; - - while (nBytes > 0) { - iRead = pread(fd, pBuf, nBytes, iOffset); - if (iRead < 0) { - if (errno == EINTR) { - continue; - } else { - return -1; - } - } else if (iRead == 0) { - break; - } - - nRead += iRead; - pBuf += iRead; - iOffset += iRead; - nBytes -= iRead; - } - - return nRead; -} - -// tdbOsWrite -i64 tdbOsWrite(tdb_fd_t fd, const void *pData, i64 nBytes) { - i64 nWrite = 0; - i64 iWrite = 0; - u8 *pBuf = (u8 *)pData; - - while (nBytes > 0) { - iWrite = write(fd, pBuf, nBytes); - if (iWrite < 0) { - if (errno == EINTR) { - continue; - } - - return -1; - } - - nWrite += iWrite; - pBuf += iWrite; - nBytes -= iWrite; - } - - return nWrite; -} - -#endif \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 455128e6ec..6cb5f89758 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -50,30 +50,36 @@ static void tdbPCacheLock(SPCache *pCache) { tdbMutexLock(&(pCache->mutex)); } static void tdbPCacheUnlock(SPCache *pCache) { tdbMutexUnlock(&(pCache->mutex)); } int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) { + int32_t code = 0; + int32_t lino; SPCache *pCache; void *pPtr; SPage *pPgHdr; pCache = (SPCache *)tdbOsCalloc(1, sizeof(*pCache) + sizeof(SPage *) * cacheSize); if (pCache == NULL) { - return -1; + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } pCache->szPage = pageSize; pCache->nPages = cacheSize; pCache->aPage = (SPage **)tdbOsCalloc(cacheSize, sizeof(SPage *)); if (pCache->aPage == NULL) { - tdbOsFree(pCache); - return -1; + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } - if (tdbPCacheOpenImpl(pCache) < 0) { - tdbOsFree(pCache); - return -1; - } + code = tdbPCacheOpenImpl(pCache); + TSDB_CHECK_CODE(code, lino, _exit); - *ppCache = pCache; - return 0; +_exit: + if (code) { + tdbError("%s failed at %s:%d since %s", __func__, __FILE__, __LINE__, tstrerror(code)); + tdbPCacheClose(pCache); + *ppCache = NULL; + } else { + *ppCache = pCache; + } + return code; } int tdbPCacheClose(SPCache *pCache) { @@ -99,14 +105,14 @@ static int tdbPCacheAlterImpl(SPCache *pCache, int32_t nPage) { } else if (pCache->nPages < nPage) { SPage **aPage = tdbOsCalloc(nPage, sizeof(SPage *)); if (aPage == NULL) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } for (int32_t iPage = pCache->nPages; iPage < nPage; iPage++) { - if (tdbPageCreate(pCache->szPage, &aPage[iPage], tdbDefaultMalloc, NULL) < 0) { - // TODO: handle error + int32_t code = tdbPageCreate(pCache->szPage, &aPage[iPage], tdbDefaultMalloc, NULL); + if (code) { tdbOsFree(aPage); - return -1; + return code; } // pPage->pgid = 0; @@ -156,15 +162,11 @@ static int tdbPCacheAlterImpl(SPCache *pCache, int32_t nPage) { } int tdbPCacheAlter(SPCache *pCache, int32_t nPage) { - int ret = 0; - + int code; tdbPCacheLock(pCache); - - ret = tdbPCacheAlterImpl(pCache, nPage); - + code = tdbPCacheAlterImpl(pCache, nPage); tdbPCacheUnlock(pCache); - - return ret; + return code; } SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) { @@ -180,9 +182,6 @@ SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) { tdbPCacheUnlock(pCache); - // printf("thread %" PRId64 " fetch page %d pgno %d pPage %p nRef %d\n", taosGetSelfPthreadId(), pPage->id, - // TDB_PAGE_PGNO(pPage), pPage, nRef); - if (pPage) { tdbTrace("pcache/fetch page %p/%d/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id, nRef); } else { @@ -285,6 +284,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) if (!pTxn) { tdbError("tdb/pcache: null ptr pTxn, fetch impl failed."); + terrno = TSDB_CODE_INVALID_PARA; return NULL; } @@ -327,7 +327,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) ret = tdbPageCreate(pCache->szPage, &pPage, pTxn->xMalloc, pTxn->xArg); if (ret < 0 || pPage == NULL) { tdbError("tdb/pcache: ret: %" PRId32 " pPage: %p, page create failed.", ret, pPage); - // TODO: recycle other backup pages + terrno = ret; return NULL; } @@ -475,10 +475,8 @@ static int tdbPCacheOpenImpl(SPCache *pCache) { pCache->nFree = 0; pCache->pFree = NULL; for (int i = 0; i < pCache->nPages; i++) { - if (tdbPageCreate(pCache->szPage, &pPage, tdbDefaultMalloc, NULL) < 0) { - // TODO: handle error - return -1; - } + ret = tdbPageCreate(pCache->szPage, &pPage, tdbDefaultMalloc, NULL); + if (ret) return ret; // pPage->pgid = 0; pPage->isAnchor = 0; @@ -504,8 +502,7 @@ static int tdbPCacheOpenImpl(SPCache *pCache) { pCache->nHash = pCache->nPages < 8 ? 8 : pCache->nPages; pCache->pgHash = (SPage **)tdbOsCalloc(pCache->nHash, sizeof(SPage *)); if (pCache->pgHash == NULL) { - // TODO - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } // Open LRU list diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c index c6310f4985..322b735163 100644 --- a/source/libs/tdb/src/db/tdbPage.c +++ b/source/libs/tdb/src/db/tdbPage.c @@ -45,12 +45,12 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) if (!xMalloc) { tdbError("tdb/page-create: null xMalloc."); - return -1; + return TSDB_CODE_INVALID_PARA; } if (!TDB_IS_PGSIZE_VLD(pageSize)) { tdbError("tdb/page-create: invalid pageSize: %d.", pageSize); - return -1; + return TSDB_CODE_INVALID_PARA; } *ppPage = NULL; @@ -58,7 +58,7 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) ptr = (u8 *)(xMalloc(arg, size)); if (ptr == NULL) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } memset(ptr, 0, size); @@ -86,12 +86,12 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) if (pPage->isDirty) { tdbError("tdb/page-destroy: dirty page: %" PRIu8 ".", pPage->isDirty); - return -1; + return TSDB_CODE_INVALID_PARA; } if (!xFree) { tdbError("tdb/page-destroy: null xFree."); - return -1; + return TSDB_CODE_INVALID_PARA; } for (int iOvfl = 0; iOvfl < pPage->nOverflow; iOvfl++) { @@ -129,7 +129,8 @@ void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell tdbTrace("page/init: %p %" PRIu8 " %p", pPage, szAmHdr, xCellSize); pPage->pPageHdr = pPage->pData + szAmHdr; if (TDB_PAGE_NCELLS(pPage) == 0) { - return tdbPageZero(pPage, szAmHdr, xCellSize); + tdbPageZero(pPage, szAmHdr, xCellSize); + return; } pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage); pPage->pFreeStart = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * TDB_PAGE_NCELLS(pPage); @@ -159,7 +160,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl if (szCell > TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)) { tdbError("tdb/page-insert-cell: invalid page, szCell: %d, max free: %lu", szCell, TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)); - return -1; + return TSDB_CODE_INVALID_DATA_FMT; } nFree = TDB_PAGE_NFREE(pPage); @@ -207,7 +208,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl if (pPage->pFreeStart != pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)) { tdbError("tdb/page-insert-cell: invalid page, pFreeStart: %p, pCellIdx: %p, nCells: %d", pPage->pFreeStart, pPage->pCellIdx, nCells); - return -1; + return TSDB_CODE_INVALID_DATA_FMT; } } @@ -234,7 +235,7 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) { if (idx < 0 || idx >= nCells + pPage->nOverflow) { tdbError("tdb/page-drop-cell: idx: %d out of range, nCells: %d, nOvfl: %d.", idx, nCells, pPage->nOverflow); - return -1; + return TSDB_CODE_INVALID_PARA; } iOvfl = 0; @@ -265,14 +266,14 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) { pPage->aiOvfl[iOvfl]--; if (pPage->aiOvfl[iOvfl] <= 0) { tdbError("tdb/page-drop-cell: invalid ai idx: %d", pPage->aiOvfl[iOvfl]); - return -1; + return TSDB_CODE_INVALID_DATA_FMT; } } return 0; } -void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) { +int32_t tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) { int delta, nFree; pToPage->pFreeStart = pToPage->pPageHdr + (pFromPage->pFreeStart - pFromPage->pPageHdr); @@ -280,7 +281,7 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) { if (pToPage->pFreeEnd < pToPage->pFreeStart) { tdbError("tdb/page-copy: invalid to page, pFreeStart: %p, pFreeEnd: %p", pToPage->pFreeStart, pToPage->pFreeEnd); - return; + return TSDB_CODE_INVALID_DATA_FMT; } memcpy(pToPage->pPageHdr, pFromPage->pPageHdr, pFromPage->pFreeStart - pFromPage->pPageHdr); @@ -289,7 +290,7 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) { if (TDB_PAGE_CCELLS(pToPage) != pToPage->pFreeEnd - pToPage->pData) { tdbError("tdb/page-copy: invalid to page, cell body: %d, range: %ld", TDB_PAGE_CCELLS(pToPage), pToPage->pFreeEnd - pToPage->pData); - return; + return TSDB_CODE_INVALID_DATA_FMT; } delta = (pToPage->pPageHdr - pToPage->pData) - (pFromPage->pPageHdr - pFromPage->pData); @@ -304,6 +305,10 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) { if (deepCopyOvfl) { int szCell = (*pFromPage->xCellSize)(pFromPage, pFromPage->apOvfl[iOvfl], 0, NULL, NULL); pNewCell = (SCell *)tdbOsMalloc(szCell); + if (pNewCell == NULL) { + tdbError("tdb/page-copy: out of memory, size: %d", szCell); + return TSDB_CODE_OUT_OF_MEMORY; + } memcpy(pNewCell, pFromPage->apOvfl[iOvfl], szCell); tdbTrace("tdbPage/copy/new ovfl cell: %p/%p/%p", pNewCell, pToPage, pFromPage); } @@ -312,6 +317,7 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) { pToPage->aiOvfl[iOvfl] = pFromPage->aiOvfl[iOvfl]; } pToPage->nOverflow = pFromPage->nOverflow; + return 0; } int tdbPageCapacity(int pageSize, int amHdrSize) { @@ -343,12 +349,12 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { if (nFree < szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { tdbError("tdb/page-allocate: invalid cell size, nFree: %d, szCell: %d, szOffset: %d", nFree, szCell, TDB_PAGE_OFFSET_SIZE(pPage)); - return -1; + return TSDB_CODE_INVALID_PARA; } if (TDB_PAGE_CCELLS(pPage) != pPage->pFreeEnd - pPage->pData) { tdbError("tdb/page-allocate: invalid page, cell body: %d, range: %ld", TDB_PAGE_CCELLS(pPage), pPage->pFreeEnd - pPage->pData); - return -1; + return TSDB_CODE_INVALID_DATA_FMT; } // 1. Try to allocate from the free space block area @@ -363,7 +369,7 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { cellFree = TDB_PAGE_FCELL(pPage); if (cellFree != 0 && cellFree < pPage->pFreeEnd - pPage->pData) { tdbError("tdb/page-allocate: cellFree: %d, pFreeEnd: %p, pData: %p.", cellFree, pPage->pFreeEnd, pPage->pData); - return -1; + return TSDB_CODE_INVALID_DATA_FMT; } if (cellFree && pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) { SCell *pPrevFreeCell = NULL; @@ -408,19 +414,19 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { } // 3. Try to dfragment and allocate again - tdbPageDefragment(pPage); + TAOS_CHECK_RETURN(tdbPageDefragment(pPage)); if (pPage->pFreeEnd - pPage->pFreeStart != nFree) { tdbError("tdb/page-allocate: nFree: %d, pFreeStart: %p, pFreeEnd: %p.", nFree, pPage->pFreeStart, pPage->pFreeEnd); - return -1; + return TSDB_CODE_INVALID_DATA_FMT; } if (TDB_PAGE_NFREE(pPage) != nFree) { tdbError("tdb/page-allocate: nFree: %d, page free: %d.", nFree, TDB_PAGE_NFREE(pPage)); - return -1; + return TSDB_CODE_INVALID_DATA_FMT; } if (pPage->pFreeEnd - pPage->pData != TDB_PAGE_CCELLS(pPage)) { tdbError("tdb/page-allocate: ccells: %d, pFreeStart: %p, pData: %p.", TDB_PAGE_CCELLS(pPage), pPage->pFreeStart, pPage->pData); - return -1; + return TSDB_CODE_INVALID_DATA_FMT; } pPage->pFreeEnd -= szCell; @@ -430,7 +436,7 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { _alloc_finish: if (NULL == pCell) { tdbError("tdb/page-allocate: null ptr pCell."); - return -1; + return TSDB_CODE_OUT_OF_BUFFER; } pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); @@ -447,15 +453,15 @@ static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) { if (pCell < pPage->pFreeEnd) { tdbError("tdb/page-free: invalid cell, cell: %p, free end: %p", pCell, pPage->pFreeEnd); - return -1; + return TSDB_CODE_INVALID_PARA; } if (pCell + szCell > (u8 *)(pPage->pPageFtr)) { tdbError("tdb/page-free: cell crosses page footer, cell: %p, size: %d footer: %p", pCell, szCell, pPage->pFreeEnd); - return -1; + return TSDB_CODE_INVALID_PARA; } if (pCell != TDB_PAGE_CELL_AT(pPage, idx)) { tdbError("tdb/page-free: cell pos incorrect, cell: %p, pos: %p", pCell, TDB_PAGE_CELL_AT(pPage, idx)); - return -1; + return TSDB_CODE_INVALID_PARA; } nFree = TDB_PAGE_NFREE(pPage); @@ -470,7 +476,7 @@ static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) { TDB_PAGE_FCELL_SET(pPage, pCell - pPage->pData); } else { tdbError("tdb/page-free: invalid cell size: %d", szCell); - return -1; + return TSDB_CODE_INVALID_PARA; } } @@ -502,7 +508,9 @@ static int tdbPageDefragment(SPage *pPage) { int32_t nCell = TDB_PAGE_NCELLS(pPage); SCellIdx *aCellIdx = (SCellIdx *)tdbOsMalloc(sizeof(SCellIdx) * nCell); - if (aCellIdx == NULL) return -1; + if (aCellIdx == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } for (int32_t iCell = 0; iCell < nCell; iCell++) { aCellIdx[iCell].iCell = iCell; aCellIdx[iCell].offset = TDB_PAGE_CELL_OFFSET_AT(pPage, iCell); diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 9dbac11b02..14165045ca 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "tdbInt.h" #include "crypt.h" +#include "tdbInt.h" #include "tglobal.h" /* #pragma pack(push, 1) @@ -41,9 +41,10 @@ struct hashset_st { static const unsigned int prime = 39; static const unsigned int prime2 = 5009; -hashset_t hashset_create(void) { +static hashset_t hashset_create(void) { hashset_t set = tdbOsCalloc(1, sizeof(struct hashset_st)); if (!set) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -52,6 +53,7 @@ hashset_t hashset_create(void) { set->items = tdbOsCalloc(set->capacity, sizeof(size_t)); if (!set->items) { tdbOsFree(set); + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } set->mask = set->capacity - 1; @@ -69,7 +71,7 @@ void hashset_destroy(hashset_t set) { } } -int hashset_add_member(hashset_t set, void *item) { +static int hashset_add_member(hashset_t set, void *item) { size_t value = (size_t)item; size_t h; @@ -88,7 +90,7 @@ int hashset_add_member(hashset_t set, void *item) { return 1; } -int hashset_add(hashset_t set, void *item) { +static int hashset_add(hashset_t set, void *item) { int ret = hashset_add_member(set, item); size_t old_capacity = set->capacity; @@ -113,7 +115,7 @@ int hashset_add(hashset_t set, void *item) { return ret; } -int hashset_remove(hashset_t set, void *item) { +static int hashset_remove(hashset_t set, void *item) { size_t value = (size_t)item; for (size_t h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) { @@ -127,7 +129,7 @@ int hashset_remove(hashset_t set, void *item) { return 0; } -int hashset_contains(hashset_t set, void *item) { +static int hashset_contains(hashset_t set, void *item) { size_t value = (size_t)item; for (size_t h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) { @@ -177,7 +179,7 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { + fsize + 8 + 1; /* jFileName */ pPtr = (uint8_t *)tdbOsCalloc(1, zsize); if (pPtr == NULL) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } pPager = (SPager *)pPtr; @@ -198,12 +200,12 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { pPager->fd = tdbOsOpen(pPager->dbFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); if (TDB_FD_INVALID(pPager->fd)) { // if (pPager->fd < 0) { - return -1; + return TAOS_SYSTEM_ERROR(errno); } ret = tdbGnrtFileID(pPager->fd, pPager->fid, false); if (ret < 0) { - return -1; + return TAOS_SYSTEM_ERROR(errno); } // pPager->jfd = -1; @@ -221,11 +223,6 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { int tdbPagerClose(SPager *pPager) { if (pPager) { - /* - if (pPager->inTran) { - tdbOsClose(pPager->jfd); - } - */ tdbOsClose(pPager->fd); tdbOsFree(pPager); } @@ -254,8 +251,8 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { !hashset_contains(pPager->pActiveTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))))) { ret = tdbPagerWritePageToJournal(pPager, pPage); if (ret < 0) { - tdbError("failed to write page to journal since %s", tstrerror(terrno)); - return -1; + tdbError("failed to write page to journal since %s", tstrerror(ret)); + return ret; } if (pPager->pActiveTxn->jPageSet) { @@ -278,11 +275,13 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) { pTxn->jfd = tdbOsOpen(jTxnFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); if (TDB_FD_INVALID(pTxn->jfd)) { tdbError("failed to open file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } pTxn->jPageSet = hashset_create(); + if (pTxn->jPageSet == NULL) { + return terrno; + } pPager->pActiveTxn = pTxn; @@ -319,8 +318,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { ret = tdbOsFSync(pTxn->jfd); if (ret < 0) { tdbError("failed to fsync: %s. jFileName:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } // loop to write the dirty pages to file @@ -331,13 +329,13 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { if (pPage->nOverflow != 0) { tdbError("tdb/pager-commit: %p, pPage: %p, ovfl: %d, commit page failed.", pPager, pPage, pPage->nOverflow); - return -1; + return TSDB_CODE_INVALID_DATA_FMT; } ret = tdbPagerPWritePageToDB(pPager, pPage); if (ret < 0) { tdbError("failed to write page to db since %s", tstrerror(terrno)); - return -1; + return ret; } } @@ -368,8 +366,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { // sync the db file if (tdbOsFSync(pPager->fd) < 0) { tdbError("failed to fsync fd due to %s. file:%s", strerror(errno), pPager->dbFileName); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } return 0; @@ -382,14 +379,12 @@ int tdbPagerPostCommit(SPager *pPager, TXN *pTxn) { // remove the journal file if (tdbOsClose(pTxn->jfd) < 0) { tdbError("failed to close jfd: %s. file:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) { tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } // pPager->inTran = 0; @@ -408,8 +403,7 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) { ret = tdbOsFSync(pTxn->jfd); if (ret < 0) { tdbError("failed to fsync jfd: %s. jfile:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } // loop to write the dirty pages to file @@ -426,7 +420,7 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) { ret = tdbPagerPWritePageToDB(pPager, pPage); if (ret < 0) { tdbError("failed to write page to db since %s", tstrerror(terrno)); - return -1; + return ret; } } @@ -445,36 +439,28 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) { tdbPCacheRelease(pPager->pCache, pPage, pTxn); } - /* - tdbTrace("reset dirty tree: %p", &pPager->rbt); - tRBTreeCreate(&pPager->rbt, pageCmpFn); - - // sync the db file - if (tdbOsFSync(pPager->fd) < 0) { - tdbError("failed to fsync fd due to %s. file:%s", strerror(errno), pPager->dbFileName); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - */ return 0; } -static char* tdbEncryptPage(SPager *pPager, char* pPageData, int32_t pageSize, const char* function, - int64_t offset){ +static char *tdbEncryptPage(SPager *pPager, char *pPageData, int32_t pageSize, const char *function, int64_t offset) { int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm; - char* encryptKey = pPager->pEnv->encryptKey; + char *encryptKey = pPager->pEnv->encryptKey; - char* buf = pPageData; + char *buf = pPageData; - if(encryptAlgorithm == DND_CA_SM4){ - //tdbInfo("CBC_Encrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__); - //ASSERT(strlen(encryptKey) > 0); + if (encryptAlgorithm == DND_CA_SM4) { + // tdbInfo("CBC_Encrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__); + // ASSERT(strlen(encryptKey) > 0); - //tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Encrypt", offset, pPage->pData[0]); + // tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Encrypt", offset, pPage->pData[0]); buf = taosMemoryMalloc(pageSize); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } - unsigned char packetData[128]; + unsigned char packetData[128]; int32_t count = 0; while (count < pageSize) { @@ -488,19 +474,19 @@ static char* tdbEncryptPage(SPager *pPager, char* pPageData, int32_t pageSize, c int32_t newLen = CBC_Encrypt(&opts); memcpy(buf + count, packetData, newLen); - count += newLen; + count += newLen; } - //tdbInfo("CBC tdb offset:%" PRId64 ", Encrypt count:%d %s", offset, count, function); + // tdbInfo("CBC tdb offset:%" PRId64 ", Encrypt count:%d %s", offset, count, function); - //tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d after Encrypt", offset, (uint8_t)buf[0]); + // tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d after Encrypt", offset, (uint8_t)buf[0]); } return buf; } -void tdbFreeEncryptBuf(SPager *pPager, char* buf){ +void tdbFreeEncryptBuf(SPager *pPager, char *buf) { int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm; - if(encryptAlgorithm == DND_CA_SM4) taosMemoryFreeClear(buf); + if (encryptAlgorithm == DND_CA_SM4) taosMemoryFreeClear(buf); } // recovery dirty pages int tdbPagerAbort(SPager *pPager, TXN *pTxn) { @@ -518,26 +504,24 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { ret = tdbOsFSync(pTxn->jfd); if (ret < 0) { tdbError("failed to fsync jfd: %s. jfile:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } tdb_fd_t jfd = pTxn->jfd; ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize); if (ret < 0) { - return -1; + return ret; } if (tdbOsLSeek(jfd, 0L, SEEK_SET) < 0) { tdbError("failed to lseek jfd due to %s. file:%s, offset:0", strerror(errno), pPager->dbFileName); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } u8 *pageBuf = tdbOsCalloc(1, pPager->pageSize); if (pageBuf == NULL) { - return -1; + return terrno = TSDB_CODE_OUT_OF_MEMORY; } tdbDebug("pager/abort: %p, %d/%d, txnId:%" PRId64, pPager, pPager->dbOrigSize, pPager->dbFileSize, pTxn->txnId); @@ -549,7 +533,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { int ret = tdbOsRead(jfd, &pgno, sizeof(pgno)); if (ret < 0) { tdbOsFree(pageBuf); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } tdbTrace("pager/abort: restore pgno:%d,", pgno); @@ -559,27 +543,28 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { ret = tdbOsRead(jfd, pageBuf, pPager->pageSize); if (ret < 0) { tdbOsFree(pageBuf); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } i64 offset = pPager->pageSize * (pgno - 1); if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { tdbError("failed to lseek fd due to %s. file:%s, offset:%" PRId64, strerror(errno), pPager->dbFileName, offset); - terrno = TAOS_SYSTEM_ERROR(errno); tdbOsFree(pageBuf); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } - char* buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset); + char *buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset); + if (buf == NULL) { + return terrno; + } ret = tdbOsWrite(pPager->fd, buf, pPager->pageSize); if (ret < 0) { tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName, pPager->pageSize); tdbFreeEncryptBuf(pPager, buf); - terrno = TAOS_SYSTEM_ERROR(errno); tdbOsFree(pageBuf); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } tdbFreeEncryptBuf(pPager, buf); @@ -587,9 +572,8 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { if (tdbOsFSync(pPager->fd) < 0) { tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName); - terrno = TAOS_SYSTEM_ERROR(errno); tdbOsFree(pageBuf); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } tdbOsFree(pageBuf); @@ -617,8 +601,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { // 4, remove the journal file if (tdbOsClose(pTxn->jfd) < 0) { tdbError("failed to close jfd: %s. file:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } char jTxnFileName[TDB_FILENAME_LEN]; @@ -626,8 +609,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) { tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } // pPager->inTran = 0; @@ -658,7 +640,7 @@ int tdbPagerFlushPage(SPager *pPager, TXN *pTxn) { ret = tdbPagerPWritePageToDB(pPager, pPage); if (ret < 0) { tdbError("failed to write page to db since %s", tstrerror(terrno)); - return -1; + return ret; } tdbTrace("tdb/flush:%p, pgno:%d, %d/%d/%d", pPager, pgno, pPager->dbOrigSize, pPager->dbFileSize, maxPgno); @@ -717,13 +699,13 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa ret = tdbPagerAllocPage(pPager, &pgno, pTxn); if (ret < 0) { tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno); - return -1; + return ret; } } if (pgno == 0) { tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno); - return -1; + return TSDB_CODE_INVALID_DATA_FMT; } // fetch a page container @@ -739,7 +721,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa ret = tdbPagerInitPage(pPager, pPage, initPage, arg, loadPage); if (ret < 0) { tdbError("tdb/pager: %p, pPage: %p, init page failed.", pPager, pPage); - return -1; + return ret; } } @@ -748,11 +730,11 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa if (!TDB_PAGE_INITIALIZED(pPage)) { tdbError("tdb/pager: %p, pPage: %p, fetch page uninited.", pPager, pPage); - return -1; + return TSDB_CODE_INVALID_DATA_FMT; } if (pPage->pPager != pPager) { tdbError("tdb/pager: %p/%p, fetch page failed.", pPager, pPage->pPager); - return -1; + return TSDB_CODE_INVALID_DATA_FMT; } *ppgno = pgno; @@ -771,12 +753,17 @@ int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn) { SPgno pgno = TDB_PAGE_PGNO(pPage); if (pPager->frps) { - taosArrayPush(pPager->frps, &pgno); + if (taosArrayPush(pPager->frps, &pgno) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } pPage->pPager = NULL; return code; } pPager->frps = taosArrayInit(8, sizeof(SPgno)); + if (pPager->frps == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } // memset(pPage->pData, 0, pPage->pageSize); tdbTrace("tdb/insert-free-page: tbc recycle page: %d.", pgno); // printf("tdb/insert-free-page: tbc recycle page: %d.\n", pgno); @@ -785,7 +772,7 @@ int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn) { tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code); taosArrayDestroy(pPager->frps); pPager->frps = NULL; - return -1; + return code; } while (TARRAY_SIZE(pPager->frps) > 0) { @@ -796,7 +783,7 @@ int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn) { tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code); taosArrayDestroy(pPager->frps); pPager->frps = NULL; - return -1; + return code; } } @@ -822,7 +809,7 @@ static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno, TXN *pTxn) { code = tdbTbcOpen(pPager->pEnv->pFreeDb, &pCur, pTxn); if (code < 0) { - return 0; + return code; } code = tdbTbcMoveToFirst(pCur); @@ -924,20 +911,19 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage } int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm; - char* encryptKey = pPager->pEnv->encryptKey; + char *encryptKey = pPager->pEnv->encryptKey; - if(encryptAlgorithm == DND_CA_SM4){ - //tdbInfo("CBC_Decrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__); - //ASSERT(strlen(encryptKey) > 0); + if (encryptAlgorithm == DND_CA_SM4) { + // tdbInfo("CBC_Decrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__); + // ASSERT(strlen(encryptKey) > 0); - //uint8_t flags = pPage->pData[0]; - //tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Decrypt", ((i64)pPage->pageSize) * (pgno - 1), flags); + // uint8_t flags = pPage->pData[0]; + // tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Decrypt", ((i64)pPage->pageSize) * (pgno - 1), flags); - unsigned char packetData[128]; + unsigned char packetData[128]; int32_t count = 0; - while(count < pPage->pageSize) - { + while (count < pPage->pageSize) { SCryptOpts opts = {0}; opts.len = 128; opts.source = pPage->pData + count; @@ -945,20 +931,23 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage opts.unitLen = 128; strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN); - int newLen = CBC_Decrypt(&opts); + int newLen = CBC_Decrypt(&opts); memcpy(pPage->pData + count, packetData, newLen); count += newLen; } - //tdbInfo("CBC tdb offset:%" PRId64 ", Decrypt count:%d %s", ((i64)pPage->pageSize) * (pgno - 1), count, __FUNCTION__); + // tdbInfo("CBC tdb offset:%" PRId64 ", Decrypt count:%d %s", ((i64)pPage->pageSize) * (pgno - 1), count, + // __FUNCTION__); - //tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d after Decrypt %s", ((i64)pPage->pageSize) * (pgno - 1), pPage->pData[0], __FUNCTION__); + // tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d after Decrypt %s", ((i64)pPage->pageSize) * (pgno - 1), + // pPage->pData[0], __FUNCTION__); } } else { init = 0; } - //tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d initPage %s", ((i64)pPage->pageSize) * (pgno - 1), pPage->pData[0], __FUNCTION__); + // tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d initPage %s", ((i64)pPage->pageSize) * (pgno - 1), pPage->pData[0], + // __FUNCTION__); ret = (*initPage)(pPage, arg, init); if (ret < 0) { @@ -1001,16 +990,14 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) { if (ret < 0) { tdbError("failed to write pgno due to %s. file:%s, pgno:%u, txnId:%" PRId64, strerror(errno), pPager->jFileName, pgno, pPager->pActiveTxn->txnId); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } ret = tdbOsWrite(pPager->pActiveTxn->jfd, pPage->pData, pPage->pageSize); if (ret < 0) { tdbError("failed to write page data due to %s. file:%s, pageSize:%d, txnId:%" PRId64, strerror(errno), pPager->jFileName, pPage->pageSize, pPager->pActiveTxn->txnId); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } return 0; @@ -1044,15 +1031,14 @@ static int tdbPagerPWritePageToDB(SPager *pPager, SPage *pPage) { offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1); - char* buf = tdbEncryptPage(pPager, pPage->pData, pPage->pageSize, __FUNCTION__, offset); + char *buf = tdbEncryptPage(pPager, pPage->pData, pPage->pageSize, __FUNCTION__, offset); ret = tdbOsPWrite(pPager->fd, buf, pPage->pageSize, offset); if (ret < 0) { tdbFreeEncryptBuf(pPager, buf); tdbError("failed to pwrite page data due to %s. file:%s, pageSize:%d", strerror(errno), pPager->dbFileName, pPage->pageSize); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } tdbFreeEncryptBuf(pPager, buf); @@ -1072,18 +1058,17 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) { ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize); if (ret < 0) { - return -1; + return TAOS_SYSTEM_ERROR(errno); } if (tdbOsLSeek(jfd, 0L, SEEK_SET) < 0) { tdbError("failed to lseek jfd due to %s. file:%s, offset:0", strerror(errno), pPager->dbFileName); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } pageBuf = tdbOsCalloc(1, pPager->pageSize); if (pageBuf == NULL) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } tdbDebug("pager/restore: %p, %d/%d, txnId:%s", pPager, pPager->dbOrigSize, pPager->dbFileSize, jFileName); @@ -1095,7 +1080,7 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) { int ret = tdbOsRead(jfd, &pgno, sizeof(pgno)); if (ret < 0) { tdbOsFree(pageBuf); - return -1; + return TAOS_SYSTEM_ERROR(errno); } tdbTrace("pager/restore: restore pgno:%d,", pgno); @@ -1103,27 +1088,28 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) { ret = tdbOsRead(jfd, pageBuf, pPager->pageSize); if (ret < 0) { tdbOsFree(pageBuf); - return -1; + return TAOS_SYSTEM_ERROR(errno); } i64 offset = pPager->pageSize * (pgno - 1); if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { tdbError("failed to lseek fd due to %s. file:%s, offset:%" PRId64, strerror(errno), pPager->dbFileName, offset); - terrno = TAOS_SYSTEM_ERROR(errno); tdbOsFree(pageBuf); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } - char* buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset); + char *buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset); + if (buf == NULL) { + return terrno; + } ret = tdbOsWrite(pPager->fd, buf, pPager->pageSize); if (ret < 0) { tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName, pPager->pageSize); tdbFreeEncryptBuf(pPager, buf); - terrno = TAOS_SYSTEM_ERROR(errno); tdbOsFree(pageBuf); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } tdbFreeEncryptBuf(pPager, buf); @@ -1131,9 +1117,8 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) { if (tdbOsFSync(pPager->fd) < 0) { tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName); - terrno = TAOS_SYSTEM_ERROR(errno); tdbOsFree(pageBuf); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } tdbOsFree(pageBuf); @@ -1160,21 +1145,27 @@ static int32_t txnIdCompareDesc(const void *pLeft, const void *pRight) { } int tdbPagerRestoreJournals(SPager *pPager) { + int32_t code = 0; tdbDirEntryPtr pDirEntry; tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName); if (pDir == NULL) { tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(errno)); - return -1; + return TAOS_SYSTEM_ERROR(errno); } SArray *pTxnList = taosArrayInit(16, sizeof(int64_t)); + if (pTxnList == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } while ((pDirEntry = tdbReadDir(pDir)) != NULL) { char *name = tdbDirEntryBaseName(tdbGetDirEntryName(pDirEntry)); if (strncmp(TDB_MAINDB_NAME "-journal", name, 16) == 0) { int64_t txnId = -1; sscanf(name, TDB_MAINDB_NAME "-journal.%" PRId64, &txnId); - taosArrayPush(pTxnList, &txnId); + if (taosArrayPush(pTxnList, &txnId) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } } taosArraySort(pTxnList, txnIdCompareDesc); @@ -1185,12 +1176,12 @@ int tdbPagerRestoreJournals(SPager *pPager) { memcpy(jname, pPager->pEnv->dbName, dirLen); jname[dirLen] = '/'; sprintf(jname + dirLen + 1, TDB_MAINDB_NAME "-journal.%" PRId64, *pTxnId); - if (tdbPagerRestore(pPager, jname) < 0) { + code = tdbPagerRestore(pPager, jname); + if (code) { taosArrayDestroy(pTxnList); tdbCloseDir(&pDir); - - tdbError("failed to restore file due to %s. jFileName:%s", strerror(errno), jname); - return -1; + tdbError("failed to restore file due to %s. jFileName:%s", strerror(code), jname); + return code; } } @@ -1205,7 +1196,7 @@ int tdbPagerRollback(SPager *pPager) { tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName); if (pDir == NULL) { tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(errno)); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } while ((pDirEntry = tdbReadDir(pDir)) != NULL) { @@ -1221,8 +1212,7 @@ int tdbPagerRollback(SPager *pPager) { tdbCloseDir(&pDir); tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), name); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } } } diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index 3bc6f75bef..365be222ef 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -41,7 +41,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF pTb = (TTB *)tdbOsCalloc(1, sizeof(*pTb)); if (pTb == NULL) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } // pTb->pEnv @@ -54,7 +54,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF pPager = tdbEnvGetPager(pEnv, fFullName); if (!pPager) { tdbOsFree(pTb); - return -1; + return terrno; } ret = tdbTbGet(pPager->pEnv->pMainDb, tbname, strlen(tbname) + 1, &pData, &nData); @@ -74,7 +74,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager); if (ret < 0) { tdbOsFree(pTb); - return -1; + return ret; } tdbEnvAddPager(pEnv, pPager); @@ -109,7 +109,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF ret = tdbPagerRestoreJournals(pPager); if (ret < 0) { tdbOsFree(pTb); - return -1; + return ret; } } else { tdbPagerRollback(pPager); @@ -119,7 +119,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF ret = tdbBtreeOpen(keyLen, valLen, pPager, tbname, pgno, keyCmprFn, pEnv, &(pTb->pBt)); if (ret < 0) { tdbOsFree(pTb); - return -1; + return ret; } *ppTb = pTb; diff --git a/source/libs/tdb/src/db/tdbTxn.c b/source/libs/tdb/src/db/tdbTxn.c index 0aeed3c140..24a70f62b2 100644 --- a/source/libs/tdb/src/db/tdbTxn.c +++ b/source/libs/tdb/src/db/tdbTxn.c @@ -20,7 +20,7 @@ int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void // not support read-committed version at the moment if (flags != 0 && flags != (TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)) { tdbError("tdb/txn: invalid txn flags: %" PRId32, flags); - return -1; + return TSDB_CODE_INVALID_PARA; } pTxn->flags = flags; @@ -39,7 +39,7 @@ int tdbTxnCloseImpl(TXN *pTxn) { } if (pTxn->jfd) { - tdbOsClose(pTxn->jfd); + TAOS_UNUSED(tdbOsClose(pTxn->jfd)); ASSERT(pTxn->jfd == NULL); } diff --git a/source/libs/tdb/src/db/tdbUtil.c b/source/libs/tdb/src/db/tdbUtil.c index 9021e08ffe..d6d33bc7d8 100644 --- a/source/libs/tdb/src/db/tdbUtil.c +++ b/source/libs/tdb/src/db/tdbUtil.c @@ -38,9 +38,8 @@ void tdbFree(void *p) { int tdbGnrtFileID(tdb_fd_t fd, uint8_t *fileid, bool unique) { int64_t stDev = 0, stIno = 0; - if (taosDevInoFile(fd, &stDev, &stIno) < 0) { - return -1; - } + int32_t code = taosDevInoFile(fd, &stDev, &stIno); + return code; memset(fileid, 0, TDB_FILE_ID_LEN); @@ -59,7 +58,7 @@ int tdbGetFileSize(tdb_fd_t fd, int szPage, SPgno *size) { ret = tdbOsFileSize(fd, &szBytes); if (ret < 0) { - return -1; + return TAOS_SYSTEM_ERROR(errno); } *size = szBytes / szPage; diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index b359f8a4cc..605fe6a1a4 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -17,6 +17,7 @@ #define _TD_TDB_INTERNAL_H_ #include "tdb.h" +#include "tutil.h" #include "tdef.h" #include "tlog.h" @@ -338,15 +339,15 @@ static inline int tdbTryLockPage(tdb_spinlock_t *pLock) { ((*(pPage)->xCellSize)(pPage, pCell, 0, NULL, NULL) + (pPage)->pPageMethods->szOffset) #define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) -int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); -int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); -void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt)); -void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt)); -int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl); -int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt); -int tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell, TXN *pTxn, SBTree *pBt); -void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int copyOvflCells); -int tdbPageCapacity(int pageSize, int amHdrSize); +int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); +int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); +void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt)); +void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt)); +int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl); +int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt); +int tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell, TXN *pTxn, SBTree *pBt); +int32_t tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl); +int tdbPageCapacity(int pageSize, int amHdrSize); static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { SCell *pCell; diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index f63339bcfc..79acec1211 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -67,7 +67,7 @@ typedef struct TdFile { void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath) { #ifdef WINDOWS - char tmpPath[PATH_MAX]; + char tmpPath[PATH_MAX]; int32_t len = (int32_t)strlen(inputTmpDir); memcpy(tmpPath, inputTmpDir, len); @@ -269,13 +269,13 @@ int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { #else if (pFile == NULL || pFile->fd < 0) { - return -1; + return TSDB_CODE_INVALID_PARA; } struct stat fileStat; int32_t code = fstat(pFile->fd, &fileStat); if (code < 0) { printf("taosFStatFile run fstat fail."); - return code; + return TAOS_SYSTEM_ERROR(errno); } if (stDev != NULL) { @@ -1239,7 +1239,7 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { #ifdef WINDOWS size_t bufferSize = 512; *ptrBuf = taosMemoryMalloc(bufferSize); - if (*ptrBuf == NULL) goto END; + if (*ptrBuf == NULL) goto END; size_t bytesRead = 0; size_t totalBytesRead = 0; @@ -1274,7 +1274,7 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { ret = getline(ptrBuf, &len, pFile->fp); #endif - END: +END: #if FILE_WITH_LOCK taosThreadRwlockUnlock(&(pFile->rwlock)); #endif @@ -1413,34 +1413,30 @@ int32_t taosLinkFile(char *src, char *dst) { return 0; } -FILE* taosOpenCFile(const char* filename, const char* mode) { - return fopen(filename, mode); -} +FILE *taosOpenCFile(const char *filename, const char *mode) { return fopen(filename, mode); } -int taosSeekCFile(FILE* file, int64_t offset, int whence) { +int taosSeekCFile(FILE *file, int64_t offset, int whence) { #ifdef WINDOWS return _fseeki64(file, offset, whence); #else return fseeko(file, offset, whence); -#endif +#endif } -size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream ) { +size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream) { return fread(buffer, size, count, stream); } -size_t taosWriteToCFile(const void* ptr, size_t size, size_t nitems, FILE* stream) { +size_t taosWriteToCFile(const void *ptr, size_t size, size_t nitems, FILE *stream) { return fwrite(ptr, size, nitems, stream); } -int taosCloseCFile(FILE *f) { - return fclose(f); -} +int taosCloseCFile(FILE *f) { return fclose(f); } -int taosSetAutoDelFile(char* path) { +int taosSetAutoDelFile(char *path) { #ifdef WINDOWS return SetFileAttributes(path, FILE_ATTRIBUTE_TEMPORARY); #else return unlink(path); -#endif +#endif } \ No newline at end of file From 32faa6fcccba410df143602fe0ee7350d9c40c8c Mon Sep 17 00:00:00 2001 From: sima Date: Mon, 22 Jul 2024 10:58:55 +0800 Subject: [PATCH 22/43] enh:[TD-31063] Handling return value in qnode.c --- include/dnode/qnode/qnode.h | 5 +- source/dnode/mgmt/mgmt_qnode/src/qmInt.c | 9 +- source/dnode/mgmt/mgmt_qnode/src/qmWorker.c | 4 +- source/dnode/qnode/src/qnode.c | 19 +-- source/dnode/vnode/src/meta/metaQuery.c | 9 +- source/dnode/vnode/src/vnd/vnodeQuery.c | 166 +++++++++++++++----- 6 files changed, 151 insertions(+), 61 deletions(-) diff --git a/include/dnode/qnode/qnode.h b/include/dnode/qnode/qnode.h index e7f9d00ff3..d6efa52ea7 100644 --- a/include/dnode/qnode/qnode.h +++ b/include/dnode/qnode/qnode.h @@ -34,9 +34,10 @@ typedef struct { * @brief Start one Qnode in Dnode. * * @param pOption Option of the qnode. - * @return SQnode* The qnode object. + * @param pQnode The qnode object. + * @return int32_t The error code. */ -SQnode *qndOpen(const SQnodeOpt *pOption); +int32_t qndOpen(const SQnodeOpt *pOption, SQnode **pQnode); /** * @brief Stop Qnode in Dnode. diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmInt.c b/source/dnode/mgmt/mgmt_qnode/src/qmInt.c index 553f39e5da..3138614189 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmInt.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmInt.c @@ -34,11 +34,8 @@ static void qmClose(SQnodeMgmt *pMgmt) { } static int32_t qndOpenWrapper(SQnodeOpt *pOption, SQnode **pQnode) { - *pQnode = qndOpen(pOption); - if (*pQnode == NULL) { - return terrno; - } - return 0; + int32_t code = qndOpen(pOption, pQnode); + return code; } static int32_t qmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { int32_t code = 0; @@ -62,7 +59,7 @@ static int32_t qmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { if (code != 0) { dError("failed to open qnode since %s", tstrerror(code)); qmClose(pMgmt); - return -1; + return code; } tmsgReportStartup("qnode-impl", "initialized"); diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c index 23a2aa36e8..5b380c2a0b 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c @@ -48,8 +48,8 @@ static int32_t qmPutNodeMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pMsg) { } int32_t qmPutNodeMsgToQueryQueue(SQnodeMgmt *pMgmt, SRpcMsg *pMsg) { - qndPreprocessQueryMsg(pMgmt->pQnode, pMsg); - + int32_t code = qndPreprocessQueryMsg(pMgmt->pQnode, pMsg); + if (code) return code; return qmPutNodeMsgToWorker(&pMgmt->queryWorker, pMsg); } diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index 8cd967a8a8..b0fcd3400d 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -19,21 +19,22 @@ #include "query.h" #include "qworker.h" -SQnode *qndOpen(const SQnodeOpt *pOption) { - SQnode *pQnode = taosMemoryCalloc(1, sizeof(SQnode)); - if (NULL == pQnode) { +int32_t qndOpen(const SQnodeOpt *pOption, SQnode **pQnode) { + *pQnode = taosMemoryCalloc(1, sizeof(SQnode)); + if (NULL == *pQnode) { qError("calloc SQnode failed"); - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } - pQnode->qndId = QNODE_HANDLE; + (*pQnode)->qndId = QNODE_HANDLE; - if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, (void **)&pQnode->pQuery, &pOption->msgCb)) { + int32_t code = qWorkerInit(NODE_TYPE_QNODE, (*pQnode)->qndId, (void **)&(*pQnode)->pQuery, &pOption->msgCb); + if (TSDB_CODE_SUCCESS != code) { taosMemoryFreeClear(pQnode); - return NULL; + return code; } - pQnode->msgCb = pOption->msgCb; - return pQnode; + (*pQnode)->msgCb = pOption->msgCb; + return TSDB_CODE_SUCCESS; } void qndClose(SQnode *pQnode) { diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 88a551e47c..062c019d48 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -1542,9 +1542,14 @@ int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables, int32_t // slow path: search TDB int64_t ctbNum = 0; int32_t colNum = 0; - vnodeGetCtbNum(pVnode, uid, &ctbNum); - vnodeGetStbColumnNum(pVnode, uid, &colNum); + code = vnodeGetCtbNum(pVnode, uid, &ctbNum); + if (TSDB_CODE_SUCCESS == code) { + code = vnodeGetStbColumnNum(pVnode, uid, &colNum); + } metaULock(pVnodeObj->pMeta); + if (TSDB_CODE_SUCCESS != code) { + goto _exit; + } if (numOfTables) *numOfTables = ctbNum; if (numOfCols) *numOfCols = colNum; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 5b17e0f1da..afaf2142d4 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -47,7 +47,7 @@ int32_t fillTableColCmpr(SMetaReader *reader, SSchemaExt *pExt, int32_t numOfCol return 0; } -int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { +int32_t vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { STableInfoReq infoReq = {0}; STableMetaRsp metaRsp = {0}; SMetaReader mer1 = {0}; @@ -67,10 +67,10 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { } metaRsp.dbId = pVnode->config.dbId; - strcpy(metaRsp.tbName, infoReq.tbName); - memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName)); + (void)strcpy(metaRsp.tbName, infoReq.tbName); + (void)memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName)); - sprintf(tableFName, "%s.%s", infoReq.dbFName, infoReq.tbName); + (void)sprintf(tableFName, "%s.%s", infoReq.dbFName, infoReq.tbName); code = vnodeValidateTableHash(pVnode, tableFName); if (code) { goto _exit4; @@ -89,7 +89,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { metaRsp.tuid = mer1.me.uid; if (mer1.me.type == TSDB_SUPER_TABLE) { - strcpy(metaRsp.stbName, mer1.me.name); + (void)strcpy(metaRsp.stbName, mer1.me.name); schema = mer1.me.stbEntry.schemaRow; schemaTag = mer1.me.stbEntry.schemaTag; metaRsp.suid = mer1.me.uid; @@ -97,7 +97,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_NOLOCK); if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit2; - strcpy(metaRsp.stbName, mer2.me.name); + (void)strcpy(metaRsp.stbName, mer2.me.name); metaRsp.suid = mer2.me.uid; schema = mer2.me.stbEntry.schemaRow; schemaTag = mer2.me.stbEntry.schemaTag; @@ -114,10 +114,13 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { metaRsp.tversion = schemaTag.version; metaRsp.pSchemas = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * (metaRsp.numOfColumns + metaRsp.numOfTags)); metaRsp.pSchemaExt = (SSchemaExt *)taosMemoryCalloc(metaRsp.numOfColumns, sizeof(SSchemaExt)); - - memcpy(metaRsp.pSchemas, schema.pSchema, sizeof(SSchema) * schema.nCols); + if (NULL == metaRsp.pSchemas || NULL == metaRsp.pSchemaExt) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + (void)memcpy(metaRsp.pSchemas, schema.pSchema, sizeof(SSchema) * schema.nCols); if (schemaTag.nCols) { - memcpy(metaRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols); + (void)memcpy(metaRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols); } if (metaRsp.pSchemaExt) { SMetaReader *pReader = mer1.me.type == TSDB_CHILD_TABLE ? &mer2 : &mer1; @@ -148,7 +151,12 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp); + + rspLen = tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp); + if (rspLen < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } _exit: taosMemoryFree(metaRsp.pSchemas); @@ -174,10 +182,10 @@ _exit4: *pMsg = rpcMsg; } - return TSDB_CODE_SUCCESS; + return code; } -int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { +int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { STableCfgReq cfgReq = {0}; STableCfgRsp cfgRsp = {0}; SMetaReader mer1 = {0}; @@ -196,10 +204,10 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { goto _exit; } - strcpy(cfgRsp.tbName, cfgReq.tbName); - memcpy(cfgRsp.dbFName, cfgReq.dbFName, sizeof(cfgRsp.dbFName)); + (void)strcpy(cfgRsp.tbName, cfgReq.tbName); + (void)memcpy(cfgRsp.dbFName, cfgReq.dbFName, sizeof(cfgRsp.dbFName)); - sprintf(tableFName, "%s.%s", cfgReq.dbFName, cfgReq.tbName); + (void)sprintf(tableFName, "%s.%s", cfgReq.dbFName, cfgReq.tbName); code = vnodeValidateTableHash(pVnode, tableFName); if (code) { goto _exit; @@ -222,24 +230,36 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_LOCK); if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; - strcpy(cfgRsp.stbName, mer2.me.name); + (void)strcpy(cfgRsp.stbName, mer2.me.name); schema = mer2.me.stbEntry.schemaRow; schemaTag = mer2.me.stbEntry.schemaTag; cfgRsp.ttl = mer1.me.ctbEntry.ttlDays; cfgRsp.commentLen = mer1.me.ctbEntry.commentLen; if (mer1.me.ctbEntry.commentLen > 0) { cfgRsp.pComment = taosStrdup(mer1.me.ctbEntry.comment); + if (NULL == cfgRsp.pComment) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } } STag *pTag = (STag *)mer1.me.ctbEntry.pTags; cfgRsp.tagsLen = pTag->len; cfgRsp.pTags = taosMemoryMalloc(cfgRsp.tagsLen); - memcpy(cfgRsp.pTags, pTag, cfgRsp.tagsLen); + if (NULL == cfgRsp.pTags) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + (void)memcpy(cfgRsp.pTags, pTag, cfgRsp.tagsLen); } else if (mer1.me.type == TSDB_NORMAL_TABLE) { schema = mer1.me.ntbEntry.schemaRow; cfgRsp.ttl = mer1.me.ntbEntry.ttlDays; cfgRsp.commentLen = mer1.me.ntbEntry.commentLen; if (mer1.me.ntbEntry.commentLen > 0) { cfgRsp.pComment = taosStrdup(mer1.me.ntbEntry.comment); + if (NULL == cfgRsp.pComment) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } } } else { ASSERT(0); @@ -250,9 +270,13 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { cfgRsp.pSchemas = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * (cfgRsp.numOfColumns + cfgRsp.numOfTags)); cfgRsp.pSchemaExt = (SSchemaExt *)taosMemoryMalloc(cfgRsp.numOfColumns * sizeof(SSchemaExt)); - memcpy(cfgRsp.pSchemas, schema.pSchema, sizeof(SSchema) * schema.nCols); + if (NULL == cfgRsp.pSchemas || NULL == cfgRsp.pSchemaExt) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + (void)memcpy(cfgRsp.pSchemas, schema.pSchema, sizeof(SSchema) * schema.nCols); if (schemaTag.nCols) { - memcpy(cfgRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols); + (void)memcpy(cfgRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols); } // if (useCompress(cfgRsp.tableType)) { @@ -285,7 +309,12 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - tSerializeSTableCfgRsp(pRsp, rspLen, &cfgRsp); + + rspLen = tSerializeSTableCfgRsp(pRsp, rspLen, &cfgRsp); + if (rspLen < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } _exit: rpcMsg.info = pMsg->info; @@ -307,7 +336,7 @@ _exit: tFreeSTableCfgRsp(&cfgRsp); metaReaderClear(&mer2); metaReaderClear(&mer1); - return TSDB_CODE_SUCCESS; + return code; } static FORCE_INLINE void vnodeFreeSBatchRspMsg(void *p) { @@ -352,6 +381,10 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { for (int32_t i = 0; i < msgNum; ++i) { req = taosArrayGet(batchReq.pMsgs, i); + if (req == NULL) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } reqMsg.msgType = req->msgType; reqMsg.pCont = req->msg; @@ -359,13 +392,16 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { switch (req->msgType) { case TDMT_VND_TABLE_META: - vnodeGetTableMeta(pVnode, &reqMsg, false); + // error code has been set into reqMsg, no need to handle it here. + (void)vnodeGetTableMeta(pVnode, &reqMsg, false); break; case TDMT_VND_TABLE_CFG: - vnodeGetTableCfg(pVnode, &reqMsg, false); + // error code has been set into reqMsg, no need to handle it here. + (void)vnodeGetTableCfg(pVnode, &reqMsg, false); break; case TDMT_VND_GET_STREAM_PROGRESS: - vnodeGetStreamProgress(pVnode, &reqMsg, false); + // error code has been set into reqMsg, no need to handle it here. + (void)vnodeGetStreamProgress(pVnode, &reqMsg, false); break; default: qError("invalid req msgType %d", req->msgType); @@ -381,7 +417,11 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { rsp.rspCode = reqMsg.code; rsp.msg = reqMsg.pCont; - taosArrayPush(batchRsp.pRsps, &rsp); + if (NULL == taosArrayPush(batchRsp.pRsps, &rsp)) { + qError("taosArrayPush failed"); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } } rspSize = tSerializeSBatchRsp(NULL, 0, &batchRsp); @@ -501,7 +541,12 @@ int32_t vnodeGetTableList(void *pVnode, int8_t type, SArray *pList) { } int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list) { + int32_t code = TSDB_CODE_SUCCESS; SMCtbCursor *pCur = metaOpenCtbCursor(pVnode, uid, 1); + if (NULL == pCur) { + qError("vnode get all table list failed"); + return TSDB_CODE_FAILED; + } while (1) { tb_uid_t id = metaCtbCursorNext(pCur); @@ -510,11 +555,15 @@ int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list) { } STableKeyInfo info = {uid = id}; - taosArrayPush(list, &info); + if (NULL == taosArrayPush(list, &info)) { + qError("taosArrayPush failed"); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } } - +_exit: metaCloseCtbCursor(pCur); - return TSDB_CODE_SUCCESS; + return code; } int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg) { @@ -522,8 +571,13 @@ int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bo } int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list) { + int32_t code = TSDB_CODE_SUCCESS; SVnode *pVnodeObj = pVnode; SMCtbCursor *pCur = metaOpenCtbCursor(pVnodeObj, suid, 1); + if (NULL == pCur) { + qError("vnode get all table list failed"); + return TSDB_CODE_FAILED; + } while (1) { tb_uid_t id = metaCtbCursorNext(pCur); @@ -531,14 +585,20 @@ int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list) { break; } - taosArrayPush(list, &id); + if (NULL == taosArrayPush(list, &id)) { + qError("taosArrayPush failed"); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } } +_exit: metaCloseCtbCursor(pCur); - return TSDB_CODE_SUCCESS; + return code; } int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list) { + int32_t code = TSDB_CODE_SUCCESS; SMStbCursor *pCur = metaOpenStbCursor(pVnode->pMeta, suid); if (!pCur) { return TSDB_CODE_FAILED; @@ -550,15 +610,21 @@ int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list) { break; } - taosArrayPush(list, &id); + if (NULL == taosArrayPush(list, &id)) { + qError("taosArrayPush failed"); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } } +_exit: metaCloseStbCursor(pCur); - return TSDB_CODE_SUCCESS; + return code; } int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg, void *arg1), void *arg) { + int32_t code = TSDB_CODE_SUCCESS; SMStbCursor *pCur = metaOpenStbCursor(pVnode->pMeta, suid); if (!pCur) { return TSDB_CODE_FAILED; @@ -574,11 +640,16 @@ int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bo continue; } - taosArrayPush(list, &id); + if (NULL == taosArrayPush(list, &id)) { + qError("taosArrayPush failed"); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } } +_exit: metaCloseStbCursor(pCur); - return TSDB_CODE_SUCCESS; + return code; } int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) { @@ -700,16 +771,20 @@ int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) { *num = 0; int64_t arrSize = taosArrayGetSize(suidList); + int32_t code = TSDB_CODE_SUCCESS; for (int64_t i = 0; i < arrSize; ++i) { tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i); int64_t ctbNum = 0; int32_t numOfCols = 0; - metaGetStbStats(pVnode, suid, &ctbNum, &numOfCols); - + code = metaGetStbStats(pVnode, suid, &ctbNum, &numOfCols); + if (TSDB_CODE_SUCCESS != code) { + goto _exit; + } *num += ctbNum * (numOfCols - 1); } +_exit: taosArrayDestroy(suidList); return TSDB_CODE_SUCCESS; } @@ -728,7 +803,11 @@ int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num) { } int64_t ctbNum = 0; - vnodeGetCtbNum(pVnode, id, &ctbNum); + int32_t code = vnodeGetCtbNum(pVnode, id, &ctbNum); + if (TSDB_CODE_SUCCESS != code) { + metaCloseStbCursor(pCur); + return code; + } *num += ctbNum; } @@ -771,14 +850,17 @@ int32_t vnodeGetStreamProgress(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { rsp.vgId = req.vgId; rsp.streamId = req.streamId; rspLen = tSerializeStreamProgressRsp(0, 0, &rsp); + if (rspLen < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _OVER; + } if (direct) { buf = rpcMallocCont(rspLen); } else { buf = taosMemoryCalloc(1, rspLen); } if (!buf) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = -1; + code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } } @@ -787,7 +869,11 @@ int32_t vnodeGetStreamProgress(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { code = tqGetStreamExecInfo(pVnode, req.streamId, &rsp.progressDelay, &rsp.fillHisFinished); } if (code == TSDB_CODE_SUCCESS) { - tSerializeStreamProgressRsp(buf, rspLen, &rsp); + rspLen = tSerializeStreamProgressRsp(buf, rspLen, &rsp); + if (rspLen < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _OVER; + } rpcMsg.pCont = buf; buf = NULL; rpcMsg.contLen = rspLen; From 9130ac613879056cb3a630f5f0eb6bb0cfa9889b Mon Sep 17 00:00:00 2001 From: sima Date: Mon, 22 Jul 2024 19:10:08 +0800 Subject: [PATCH 23/43] enh:[TD-31063] Handling return value of taoshash and taosarray --- source/libs/scalar/src/filter.c | 61 +++++++++++++++++++++++++++++++-- source/libs/scalar/src/scalar.c | 22 ++++++++++++ 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 105243608e..524245e4e1 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -997,9 +997,15 @@ int32_t filterDetachCnfGroups(SArray *group, SArray *left, SArray *right) { for (int32_t l = 0; l < leftSize; ++l) { SFilterGroup *gp1 = taosArrayGet(left, l); + if (NULL == gp1) { + FLT_ERR_RET(TSDB_CODE_OUT_OF_RANGE); + } for (int32_t r = 0; r < rightSize; ++r) { SFilterGroup *gp2 = taosArrayGet(right, r); + if (NULL == gp2) { + FLT_ERR_RET(TSDB_CODE_OUT_OF_RANGE); + } FLT_ERR_RET(filterDetachCnfGroup(gp1, gp2, group)); } @@ -2307,6 +2313,9 @@ int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx *gRes, uint32_t colI for (uint32_t i = 0; i < size; ++i) { SFilterUnit *u = taosArrayGetP(colArray, i); + if (NULL == u) { + FLT_ERR_JRET(TSDB_CODE_OUT_OF_RANGE); + } uint8_t optr = FILTER_UNIT_OPTR(u); FLT_ERR_RET(filterAddRangeOptr(ctx, optr, LOGIC_COND_TYPE_AND, empty, NULL)); @@ -2637,6 +2646,9 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx **gRes1, SFilter for (int32_t i = 0; i < ctxSize; ++i) { pctx = taosArrayGet(colCtxs, i); + if (NULL == pctx) { + FLT_ERR_JRET(TSDB_CODE_OUT_OF_RANGE); + } colInfo = &(*gRes1)->colInfo[pctx->colIdx]; filterFreeColInfo(colInfo); @@ -2763,6 +2775,9 @@ int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray *group) { for (size_t i = 0; i < groupSize; ++i) { SFilterGroup *pg = taosArrayGet(group, i); + if (NULL == pg) { + FLT_ERR_RET(TSDB_CODE_OUT_OF_RANGE); + } pg->unitFlags = taosMemoryCalloc(pg->unitNum, sizeof(*pg->unitFlags)); if (pg->unitFlags == NULL) { FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); @@ -2819,7 +2834,9 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t gResNum for (int32_t n = 0; n < usize; ++n) { SFilterUnit *u = (SFilterUnit *)taosArrayGetP((SArray *)colInfo->info, n); - + if (NULL == u) { + FLT_ERR_JRET(TSDB_CODE_OUT_OF_RANGE); + } FLT_ERR_JRET(filterAddUnitFromUnit(info, &oinfo, u, &uidx)); FLT_ERR_JRET(filterAddUnitToGroup(&ng, uidx)); } @@ -3814,6 +3831,9 @@ int32_t fltSclMergeSort(SArray *pts1, SArray *pts2, SArray *result) { while (i < len1 && j < len2) { SFltSclPoint *pt1 = taosArrayGet(pts1, i); SFltSclPoint *pt2 = taosArrayGet(pts2, j); + if (NULL == pt1 || NULL == pt2) { + FLT_ERR_RET(TSDB_CODE_OUT_OF_RANGE); + } bool less = fltSclLessPoint(pt1, pt2); if (less) { if (NULL == taosArrayPush(result, pt1)) { @@ -3830,6 +3850,9 @@ int32_t fltSclMergeSort(SArray *pts1, SArray *pts2, SArray *result) { if (i < len1) { for (; i < len1; ++i) { SFltSclPoint *pt1 = taosArrayGet(pts1, i); + if (NULL == pt1) { + FLT_ERR_RET(TSDB_CODE_OUT_OF_RANGE); + } if (NULL == taosArrayPush(result, pt1)) { FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -3838,6 +3861,9 @@ int32_t fltSclMergeSort(SArray *pts1, SArray *pts2, SArray *result) { if (j < len2) { for (; j < len2; ++j) { SFltSclPoint *pt2 = taosArrayGet(pts2, j); + if (NULL == pt2) { + FLT_ERR_RET(TSDB_CODE_OUT_OF_RANGE); + } if (NULL == taosArrayPush(result, pt2)) { FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -3859,6 +3885,9 @@ int32_t fltSclMerge(SArray *pts1, SArray *pts2, bool isUnion, SArray *merged) { int32_t count = 0; for (int32_t i = 0; i < taosArrayGetSize(all); ++i) { SFltSclPoint *pt = taosArrayGet(all, i); + if (NULL == pt) { + FLT_ERR_RET(TSDB_CODE_OUT_OF_RANGE); + } if (pt->start) { ++count; if (count == countRequired) { @@ -3893,6 +3922,9 @@ typedef struct { int32_t fltSclGetOrCreateColumnRange(SColumnNode *colNode, SArray *colRangeList, SFltSclColumnRange **colRange) { for (int32_t i = 0; i < taosArrayGetSize(colRangeList); ++i) { *colRange = taosArrayGet(colRangeList, i); + if (NULL == colRange) { + return TSDB_CODE_OUT_OF_RANGE; + } if (nodesEqualNode((SNode *)(*colRange)->colNode, (SNode *)colNode)) { return TSDB_CODE_SUCCESS; } @@ -4044,6 +4076,9 @@ int32_t filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32 SArray *colRanges = info->sclCtx.fltSclRange; for (int32_t i = 0; i < taosArrayGetSize(colRanges); ++i) { SFltSclColumnRange *colRange = taosArrayGet(colRanges, i); + if (NULL == colRange) { + FLT_ERR_RET(TSDB_CODE_OUT_OF_RANGE); + } bool foundCol = false; int32_t j = 0; for (; j < numOfCols; ++j) { @@ -4332,11 +4367,17 @@ int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict) { SArray *colRanges = info->sclCtx.fltSclRange; if (taosArrayGetSize(colRanges) == 1) { SFltSclColumnRange *colRange = taosArrayGet(colRanges, 0); + if (NULL == colRange) { + FLT_ERR_JRET(TSDB_CODE_OUT_OF_RANGE); + } SArray *points = colRange->points; if (taosArrayGetSize(points) == 2) { *win = TSWINDOW_DESC_INITIALIZER; SFltSclPoint *startPt = taosArrayGet(points, 0); SFltSclPoint *endPt = taosArrayGet(points, 1); + if (NULL == startPt || NULL == endPt) { + FLT_ERR_JRET(TSDB_CODE_OUT_OF_RANGE); + } SFltSclDatum start; SFltSclDatum end; FLT_ERR_JRET(fltSclGetTimeStampDatum(startPt, &start)); @@ -4816,6 +4857,9 @@ int32_t fltSclProcessCNF(SArray *sclOpListCNF, SArray *colRangeList) { size_t sz = taosArrayGetSize(sclOpListCNF); for (int32_t i = 0; i < sz; ++i) { SFltSclOperator *sclOper = taosArrayGet(sclOpListCNF, i); + if (NULL == sclOper) { + FLT_ERR_RET(TSDB_CODE_OUT_OF_RANGE); + } SFltSclColumnRange *colRange = NULL; FLT_ERR_RET(fltSclGetOrCreateColumnRange(sclOper->colNode, colRangeList, &colRange)); SArray *points = taosArrayInit(4, sizeof(SFltSclPoint)); @@ -4920,22 +4964,27 @@ static int32_t fltSclCollectOperators(SNode *pNode, SArray *sclOpList) { int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) { SArray *sclOpList = taosArrayInit(16, sizeof(SFltSclOperator)); + int32_t code = TSDB_CODE_SUCCESS; if (NULL == sclOpList) { FLT_ERR_RET(terrno); } - FLT_ERR_RET(fltSclCollectOperators(*pNode, sclOpList)); + FLT_ERR_JRET(fltSclCollectOperators(*pNode, sclOpList)); SArray *colRangeList = taosArrayInit(16, sizeof(SFltSclColumnRange)); if (NULL == colRangeList) { FLT_ERR_RET(terrno); } - FLT_ERR_RET(fltSclProcessCNF(sclOpList, colRangeList)); + FLT_ERR_JRET(fltSclProcessCNF(sclOpList, colRangeList)); pInfo->sclCtx.fltSclRange = colRangeList; for (int32_t i = 0; i < taosArrayGetSize(sclOpList); ++i) { SFltSclOperator *sclOp = taosArrayGet(sclOpList, i); + if (NULL == sclOp) { + FLT_ERR_JRET(TSDB_CODE_OUT_OF_RANGE); + } nodesDestroyNode((SNode *)sclOp->colNode); nodesDestroyNode((SNode *)sclOp->valNode); } +_return: taosArrayDestroy(sclOpList); return TSDB_CODE_SUCCESS; } @@ -4946,6 +4995,9 @@ int32_t fltGetDataFromColId(void *param, int32_t id, void **data) { for (int32_t j = 0; j < numOfCols; ++j) { SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, j); + if (NULL == pColInfo) { + FLT_ERR_RET(TSDB_CODE_OUT_OF_RANGE); + } if (id == pColInfo->info.colId) { *data = pColInfo; break; @@ -4965,6 +5017,9 @@ int32_t fltGetDataFromSlotId(void *param, int32_t id, void **data) { } SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, id); + if (NULL == pColInfo) { + return TSDB_CODE_OUT_OF_RANGE; + } *data = pColInfo; return TSDB_CODE_SUCCESS; diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 346bbb2811..0d4ebd8e72 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -90,6 +90,9 @@ int32_t sclConvertValueToSclParam(SValueNode *pValueNode, SScalarParam *out, int int32_t sclExtendResRows(SScalarParam *pDst, SScalarParam *pSrc, SArray *pBlockList) { SSDataBlock *pb = taosArrayGetP(pBlockList, 0); + if (NULL == pb) { + SCL_ERR_RET(TSDB_CODE_OUT_OF_RANGE); + } SScalarParam *pLeft = taosMemoryCalloc(1, sizeof(SScalarParam)); int32_t code = TSDB_CODE_SUCCESS; if (NULL == pLeft) { @@ -336,6 +339,9 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t switch (nodeType(node)) { case QUERY_NODE_LEFT_VALUE: { SSDataBlock *pb = taosArrayGetP(ctx->pBlockList, 0); + if (NULL == pb) { + SCL_ERR_RET(TSDB_CODE_OUT_OF_RANGE); + } param->numOfRows = pb->info.rows; break; } @@ -393,6 +399,9 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t int32_t index = -1; for (int32_t i = 0; i < taosArrayGetSize(ctx->pBlockList); ++i) { SSDataBlock *pb = taosArrayGetP(ctx->pBlockList, i); + if (NULL == pb) { + SCL_ERR_RET(TSDB_CODE_OUT_OF_RANGE); + } if (pb->info.id.blockId == ref->dataBlockId) { index = i; break; @@ -461,6 +470,9 @@ int32_t sclInitParamList(SScalarParam **pParams, SNodeList *pParamList, SScalarC if (NULL == pParamList) { if (ctx->pBlockList) { SSDataBlock *pBlock = taosArrayGetP(ctx->pBlockList, 0); + if (NULL == pBlock) { + SCL_ERR_RET(TSDB_CODE_OUT_OF_RANGE); + } *rowNum = pBlock->info.rows; } else { *rowNum = 1; @@ -919,6 +931,9 @@ int32_t sclExecCaseWhen(SCaseWhenNode *node, SScalarCtx *ctx, SScalarParam *outp if (ctx->pBlockList) { SSDataBlock *pb = taosArrayGetP(ctx->pBlockList, 0); + if (NULL == pb) { + SCL_ERR_RET(TSDB_CODE_OUT_OF_RANGE); + } rowNum = pb->info.rows; output->numOfRows = pb->info.rows; } @@ -1532,6 +1547,10 @@ EDealRes sclWalkTarget(SNode *pNode, SScalarCtx *ctx) { int32_t index = -1; for (int32_t i = 0; i < taosArrayGetSize(ctx->pBlockList); ++i) { SSDataBlock *pb = taosArrayGetP(ctx->pBlockList, i); + if (NULL == pb) { + ctx->code = TSDB_CODE_OUT_OF_RANGE; + return DEAL_RES_ERROR; + } if (pb->info.id.blockId == target->dataBlockId) { index = i; break; @@ -1784,6 +1803,9 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { } SSDataBlock *pb = taosArrayGetP(pBlockList, 0); + if (NULL == pb) { + SCL_ERR_JRET(TSDB_CODE_OUT_OF_RANGE); + } if (1 == res->numOfRows && pb->info.rows > 0) { SCL_ERR_JRET(sclExtendResRows(pDst, res, pBlockList)); } else { From 6255a7ba101901f2ed342ea4201660f1d6efab81 Mon Sep 17 00:00:00 2001 From: sima Date: Tue, 23 Jul 2024 14:34:43 +0800 Subject: [PATCH 24/43] enh:[TD-31063] use properer errcode. --- source/dnode/vnode/src/vnd/vnodeQuery.c | 26 ++++++++++---------- source/libs/scalar/src/filter.c | 12 +++++----- source/libs/scalar/src/scalar.c | 15 +++++------- source/libs/scalar/src/sclfunc.c | 32 ++++++++++++------------- source/libs/scalar/src/sclvector.c | 4 ++-- 5 files changed, 43 insertions(+), 46 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index afaf2142d4..f7b618d18b 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -62,7 +62,7 @@ int32_t vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { // decode req if (tDeserializeSTableInfoReq(pMsg->pCont, pMsg->contLen, &infoReq) != 0) { - code = TSDB_CODE_INVALID_MSG; + code = terrno; goto _exit4; } @@ -137,7 +137,7 @@ int32_t vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { // encode and send response rspLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp); if (rspLen < 0) { - code = TSDB_CODE_INVALID_MSG; + code = terrno; goto _exit; } @@ -154,7 +154,7 @@ int32_t vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { rspLen = tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp); if (rspLen < 0) { - code = TSDB_CODE_INVALID_MSG; + code = terrno; goto _exit; } @@ -200,7 +200,7 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { // decode req if (tDeserializeSTableCfgReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) { - code = TSDB_CODE_INVALID_MSG; + code = terrno; goto _exit; } @@ -295,7 +295,7 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { // encode and send response rspLen = tSerializeSTableCfgRsp(NULL, 0, &cfgRsp); if (rspLen < 0) { - code = TSDB_CODE_INVALID_MSG; + code = terrno; goto _exit; } @@ -312,7 +312,7 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { rspLen = tSerializeSTableCfgRsp(pRsp, rspLen, &cfgRsp); if (rspLen < 0) { - code = TSDB_CODE_INVALID_MSG; + code = terrno; goto _exit; } @@ -360,7 +360,7 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { void *pRsp = NULL; if (tDeserializeSBatchReq(pMsg->pCont, pMsg->contLen, &batchReq)) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; qError("tDeserializeSBatchReq failed"); goto _exit; } @@ -382,7 +382,7 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { for (int32_t i = 0; i < msgNum; ++i) { req = taosArrayGet(batchReq.pMsgs, i); if (req == NULL) { - code = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_OUT_OF_RANGE; goto _exit; } @@ -427,18 +427,18 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { rspSize = tSerializeSBatchRsp(NULL, 0, &batchRsp); if (rspSize < 0) { qError("tSerializeSBatchRsp failed"); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } pRsp = rpcMallocCont(rspSize); if (pRsp == NULL) { qError("rpcMallocCont %d failed", rspSize); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } if (tSerializeSBatchRsp(pRsp, rspSize, &batchRsp) < 0) { qError("tSerializeSBatchRsp %d failed", rspSize); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } @@ -851,7 +851,7 @@ int32_t vnodeGetStreamProgress(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { rsp.streamId = req.streamId; rspLen = tSerializeStreamProgressRsp(0, 0, &rsp); if (rspLen < 0) { - code = TSDB_CODE_INVALID_MSG; + code = terrno; goto _OVER; } if (direct) { @@ -871,7 +871,7 @@ int32_t vnodeGetStreamProgress(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { if (code == TSDB_CODE_SUCCESS) { rspLen = tSerializeStreamProgressRsp(buf, rspLen, &rsp); if (rspLen < 0) { - code = TSDB_CODE_INVALID_MSG; + code = terrno; goto _OVER; } rpcMsg.pCont = buf; diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 524245e4e1..19b4c521a4 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1218,7 +1218,7 @@ int32_t filterAddUnitImpl(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, FLT_PACKAGE_UNIT_HASH_KEY(&v, optr, optr2, left->idx, (right ? right->idx : -1), (right2 ? right2->idx : -1)); if (taosHashPut(info->pctx.unitHash, v, sizeof(v), uidx, sizeof(*uidx))) { fltError("taosHashPut to set failed"); - FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + FLT_ERR_RET(terrno); } } @@ -2163,7 +2163,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData)); if (len < 0) { qError("filterInitValFieldData taosUcs4ToMbs error 1"); - return TSDB_CODE_APP_ERROR; + return TSDB_CODE_SCALAR_CONVERT_ERROR; } varDataSetLen(newValData, len); (void)varDataCopy(fi->data, newValData); @@ -3442,7 +3442,7 @@ int32_t filterExecuteImplMisc(void *pinfo, int32_t numOfRows, SColumnInfoData *p if (len < 0) { qError("castConvert1 taosUcs4ToMbs error"); taosMemoryFreeClear(newColData); - FLT_ERR_RET(TSDB_CODE_APP_ERROR); + FLT_ERR_RET(TSDB_CODE_SCALAR_CONVERT_ERROR); } else { varDataSetLen(newColData, len); p[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, newColData, @@ -3517,7 +3517,7 @@ int32_t filterExecuteImpl(void *pinfo, int32_t numOfRows, SColumnInfoData *pRes, if (len < 0) { qError("castConvert1 taosUcs4ToMbs error"); taosMemoryFreeClear(newColData); - FLT_ERR_RET(TSDB_CODE_APP_ERROR); + FLT_ERR_RET(TSDB_CODE_SCALAR_CONVERT_ERROR); } else { varDataSetLen(newColData, len); p[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, cunit->valData); @@ -4439,7 +4439,7 @@ int32_t filterConverNcharColumns(SFilterInfo *info, int32_t rows, bool *gotNchar bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), (TdUcs4 *)varDataVal(dst), bufSize, &len); if (!ret) { qError("filterConverNcharColumns taosMbsToUcs4 error"); - return TSDB_CODE_FAILED; + return TSDB_CODE_SCALAR_CONVERT_ERROR; } varDataLen(dst) = len; } @@ -4473,7 +4473,7 @@ int32_t fltAddValueNodeToConverList(SFltTreeStat *stat, SValueNode *pNode) { if (NULL == stat->nodeList) { stat->nodeList = taosArrayInit(10, POINTER_BYTES); if (NULL == stat->nodeList) { - FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + FLT_ERR_RET(terrno); } } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 0d4ebd8e72..040e090661 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -119,7 +119,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false); if (NULL == pObj) { sclError("taosHashInit failed, size:%d", 256); - SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + SCL_ERR_RET(terrno); } taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(type)); @@ -179,7 +179,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { if (taosHashPut(pObj, buf, (size_t)len, NULL, 0)) { sclError("taosHashPut to set failed"); - SCL_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + SCL_ERR_JRET(terrno); } colInfoDataCleanup(out.columnData, out.numOfRows); @@ -353,10 +353,7 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } param->numOfRows = 1; - int32_t code = sclCreateColumnInfoData(&valueNode->node.resType, 1, param); - if (code != TSDB_CODE_SUCCESS) { - SCL_RET(TSDB_CODE_OUT_OF_MEMORY); - } + SCL_ERR_RET(sclCreateColumnInfoData(&valueNode->node.resType, 1, param)); if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type || valueNode->isNull) { colDataSetNULL(param->columnData, 0); } else { @@ -383,7 +380,7 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t taosHashCleanup(param->pHashFilter); param->pHashFilter = NULL; sclError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param)); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } param->colAlloced = false; break; @@ -1655,7 +1652,7 @@ int32_t sclCalcConstants(SNode *pNode, bool dual, SNode **pRes) { ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM); - SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + SCL_ERR_RET(terrno); } nodesRewriteExprPostOrder(&pNode, sclConstantsRewriter, (void *)&ctx); @@ -1789,7 +1786,7 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM); - SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + SCL_ERR_RET(terrno); } nodesWalkExprPostOrder(pNode, sclCalcWalker, (void *)&ctx); diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 5baaa57b1d..fd747ce78a 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -399,7 +399,7 @@ static int32_t concatCopyHelper(const char *input, char *output, bool hasNchar, bool ret = taosMbsToUcs4(varDataVal(input), len, newBuf, (varDataLen(input) + 1) * TSDB_NCHAR_SIZE, &len); if (!ret) { taosMemoryFree(newBuf); - return TSDB_CODE_FAILED; + return TSDB_CODE_SCALAR_CONVERT_ERROR; } memcpy(varDataVal(output) + *dataLen, newBuf, varDataLen(input) * TSDB_NCHAR_SIZE); *dataLen += varDataLen(input) * TSDB_NCHAR_SIZE; @@ -818,7 +818,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } else if (inputType == TSDB_DATA_TYPE_NCHAR) { int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); if (len < 0) { - code = TSDB_CODE_FAILED; + code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; } @@ -837,7 +837,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } else if (inputType == TSDB_DATA_TYPE_NCHAR) { int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); if (len < 0) { - code = TSDB_CODE_FAILED; + code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; } convBuf[len] = 0; @@ -855,7 +855,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } else if (inputType == TSDB_DATA_TYPE_NCHAR) { int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); if (len < 0) { - code = TSDB_CODE_FAILED; + code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; } @@ -874,7 +874,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } else if (inputType == TSDB_DATA_TYPE_NCHAR) { int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); if (len < 0) { - code = TSDB_CODE_FAILED; + code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; } convBuf[len] = 0; @@ -892,7 +892,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } else if (inputType == TSDB_DATA_TYPE_NCHAR) { int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); if (len < 0) { - code = TSDB_CODE_FAILED; + code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; } convBuf[len] = 0; @@ -910,7 +910,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } else if (inputType == TSDB_DATA_TYPE_NCHAR) { int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); if (len < 0) { - code = TSDB_CODE_FAILED; + code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; } convBuf[len] = 0; @@ -928,7 +928,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } else if (inputType == TSDB_DATA_TYPE_NCHAR) { int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); if (len < 0) { - code = TSDB_CODE_FAILED; + code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; } convBuf[len] = 0; @@ -946,7 +946,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } else if (inputType == TSDB_DATA_TYPE_NCHAR) { int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); if (len < 0) { - code = TSDB_CODE_FAILED; + code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; } @@ -965,7 +965,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } else if (inputType == TSDB_DATA_TYPE_NCHAR) { int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); if (len < 0) { - code = TSDB_CODE_FAILED; + code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; } convBuf[len] = 0; @@ -983,7 +983,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } else if (inputType == TSDB_DATA_TYPE_NCHAR) { int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); if (len < 0) { - code = TSDB_CODE_FAILED; + code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; } convBuf[len] = 0; @@ -1001,7 +1001,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } else if (inputType == TSDB_DATA_TYPE_NCHAR) { int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); if (len < 0) { - code = TSDB_CODE_FAILED; + code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; } convBuf[len] = 0; @@ -1041,7 +1041,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } else if (inputType == TSDB_DATA_TYPE_NCHAR) { int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); if (len < 0) { - code = TSDB_CODE_FAILED; + code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; } len = TMIN(len, outputLen - VARSTR_HEADER_SIZE); @@ -1075,7 +1075,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp len = sprintf(tmp, "%.*s", outputCharLen, *(int8_t *)input ? "true" : "false"); bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len); if (!ret) { - code = TSDB_CODE_FAILED; + code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; } @@ -1085,7 +1085,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp bool ret = taosMbsToUcs4(input + VARSTR_HEADER_SIZE, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len); if (!ret) { - code = TSDB_CODE_FAILED; + code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; } varDataSetLen(output, len); @@ -1099,7 +1099,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp len = outputCharLen > len ? len : outputCharLen; bool ret = taosMbsToUcs4(buf, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len); if (!ret) { - code = TSDB_CODE_FAILED; + code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; } varDataSetLen(output, len); diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index f841ec6618..cbc671a73a 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -105,7 +105,7 @@ int32_t convertNcharToDouble(const void *inData, void *outData) { int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp); if (len < 0) { sclError("castConvert taosUcs4ToMbs error 1"); - SCL_ERR_JRET(TSDB_CODE_FAILED); + SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR); } tmp[len] = 0; @@ -596,7 +596,7 @@ int32_t ncharTobinary(void *buf, void **out) { // todo need to remove , if tobi sclError("charset:%s to %s. val:%s convert ncharTobinary failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char *)varDataVal(buf)); taosMemoryFree(*out); - SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + SCL_ERR_RET(TSDB_CODE_SCALAR_CONVERT_ERROR); } varDataSetLen(*out, len); SCL_RET(TSDB_CODE_SUCCESS); From 9e8d94a69a10812ac870a19e96c2e9204b643d8e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 23 Jul 2024 15:03:24 +0800 Subject: [PATCH 25/43] fix: add function return code validation --- include/os/osEnv.h | 4 +- include/os/osTimezone.h | 2 +- source/client/src/clientMonitor.c | 3 +- source/common/src/tglobal.c | 1 - source/dnode/mgmt/node_mgmt/src/dmEnv.c | 2 - source/dnode/mgmt/node_mgmt/src/dmNodes.c | 1 - source/dnode/mgmt/node_util/src/dmFile.c | 1 - source/dnode/mnode/impl/src/mndMain.c | 1 - source/libs/tdb/src/db/tdbDb.c | 2 +- source/libs/wal/src/walMgmt.c | 2 +- source/os/src/osDir.c | 106 ++++++++++++++-------- source/os/src/osEnv.c | 39 ++++---- source/os/src/osFile.c | 70 +++++++++----- source/os/src/osSleep.c | 8 +- source/os/src/osTimer.c | 31 ++++--- source/os/src/osTimezone.c | 49 ++++++---- source/util/src/tconfig.c | 3 +- 17 files changed, 197 insertions(+), 128 deletions(-) diff --git a/include/os/osEnv.h b/include/os/osEnv.h index 4f4a58d4e8..5f7cdf44dc 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -54,7 +54,7 @@ extern SDiskSpace tsDataSpace; extern SDiskSpace tsLogSpace; extern SDiskSpace tsTempSpace; -void osDefaultInit(); +int32_t osDefaultInit(); void osUpdate(); void osCleanup(); @@ -66,7 +66,7 @@ bool osLogSpaceSufficient(); bool osDataSpaceSufficient(); bool osTempSpaceSufficient(); -void osSetTimezone(const char *timezone); +int32_t osSetTimezone(const char *timezone); void osSetSystemLocale(const char *inLocale, const char *inCharSet); void osSetProcPath(int32_t argc, char **argv); diff --git a/include/os/osTimezone.h b/include/os/osTimezone.h index ce573d320b..fab42e8e11 100644 --- a/include/os/osTimezone.h +++ b/include/os/osTimezone.h @@ -55,7 +55,7 @@ enum TdTimezone { }; void taosGetSystemTimezone(char *outTimezone, enum TdTimezone *tsTimezone); -void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight, enum TdTimezone *tsTimezone); +int32_t taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight, enum TdTimezone *tsTimezone); #ifdef __cplusplus } diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index b8b57160fe..e3b073dbc8 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -801,9 +801,8 @@ int32_t monitorInit() { } if (taosMulModeMkDir(tmpSlowLogPath, 0777, true) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); tscError("failed to create dir:%s since %s", tmpSlowLogPath, terrstr()); - return -1; + return terrno; } if (tsem2_init(&monitorSem, 0, 0) != 0) { diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 7538b504e3..c4905de898 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1306,7 +1306,6 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi taosSetAllDebugFlag(pCfg, cfgGetItem(pCfg, "debugFlag")->i32); if (taosMulModeMkDir(tsLogDir, 0777, true) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); printf("failed to create dir:%s since %s", tsLogDir, terrstr()); cfgCleanup(pCfg); return -1; diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index f636629b3a..98a8bccf0d 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -244,7 +244,6 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { pWrapper = &pDnode->wrappers[ntype]; if (taosMkDir(pWrapper->path) != 0) { dmReleaseWrapper(pWrapper); - terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); return -1; } @@ -314,7 +313,6 @@ static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) { pWrapper = &pDnode->wrappers[ntype]; if (taosMkDir(pWrapper->path) != 0) { taosThreadMutexUnlock(&pDnode->mutex); - terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); return -1; } diff --git a/source/dnode/mgmt/node_mgmt/src/dmNodes.c b/source/dnode/mgmt/node_mgmt/src/dmNodes.c index a8bf5be3e2..f25bc03a4d 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmNodes.c +++ b/source/dnode/mgmt/node_mgmt/src/dmNodes.c @@ -20,7 +20,6 @@ int32_t dmOpenNode(SMgmtWrapper *pWrapper) { SDnode *pDnode = pWrapper->pDnode; if (taosMkDir(pWrapper->path) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); dError("node:%s, failed to create dir:%s since %s", pWrapper->name, pWrapper->path, terrstr()); return -1; } diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c index 752abb83a2..d86c345ce3 100644 --- a/source/dnode/mgmt/node_util/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -328,7 +328,6 @@ int32_t dmUpdateEncryptKey(char *key, bool toLogFile) { terrno = 0; if (taosMkDir(folder) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); encryptError("failed to create dir:%s since %s", folder, terrstr()); goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 723e3701a1..680cc7ab28 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -454,7 +454,6 @@ static int32_t mndCreateDir(SMnode *pMnode, const char *path) { } if (taosMkDir(pMnode->path) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); return -1; } diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 9a1d89eeac..1ac6c727ca 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -70,7 +70,7 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i ret = taosMulModeMkDir(dbname, 0755, false); if (ret < 0) { - return -1; + return terrno; } #ifdef USE_MAINDB diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 3dbaed1bc7..88992ada2a 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -103,7 +103,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { tstrncpy(pWal->path, path, sizeof(pWal->path)); if (taosMkDir(pWal->path) != 0) { - wError("vgId:%d, path:%s, failed to create directory since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); + wError("vgId:%d, path:%s, failed to create directory since %s", pWal->cfg.vgId, pWal->path, tstrerror(terrno)); goto _err; } diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 4348629467..c04add2f42 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -109,7 +109,7 @@ void taosRemoveDir(const char *dirname) { } } - taosCloseDir(&pDir); + (void)taosCloseDir(&pDir); (void)rmdir(dirname); // printf("dir:%s is removed\n", dirname); @@ -127,8 +127,13 @@ int32_t taosMkDir(const char *dirname) { #else int32_t code = mkdir(dirname, 0755); #endif - if (code < 0 && errno == EEXIST) { - return 0; + if (-1 == code) { + if (errno == EEXIST) { + return 0; + } else { + terrno = TAOS_SYSTEM_ERROR(errno); + code = terrno; + } } return code; @@ -143,7 +148,7 @@ int32_t taosMulMkDir(const char *dirname) { taosRealPath(dirname, temp, sizeof(temp)); if (temp[1] == ':') pos += 3; #else - strcpy(temp, dirname); + (void)strcpy(temp, dirname); #endif if (taosDirExist(temp)) return code; @@ -165,7 +170,7 @@ int32_t taosMulMkDir(const char *dirname) { code = mkdir(temp, 0755); #endif if (code < 0 && errno != EEXIST) { - // terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); return code; } *pos = TD_DIRSEP[0]; @@ -181,7 +186,7 @@ int32_t taosMulMkDir(const char *dirname) { code = mkdir(temp, 0755); #endif if (code < 0 && errno != EEXIST) { - // terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); return code; } } @@ -194,7 +199,10 @@ int32_t taosMulMkDir(const char *dirname) { } int32_t taosMulModeMkDir(const char *dirname, int mode, bool checkAccess) { - if (dirname == NULL || strlen(dirname) >= TDDIRMAXLEN) return -1; + if (dirname == NULL || strlen(dirname) >= TDDIRMAXLEN) { + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } char temp[TDDIRMAXLEN]; char *pos = temp; int32_t code = 0; @@ -202,14 +210,18 @@ int32_t taosMulModeMkDir(const char *dirname, int mode, bool checkAccess) { taosRealPath(dirname, temp, sizeof(temp)); if (temp[1] == ':') pos += 3; #else - strcpy(temp, dirname); + (void)strcpy(temp, dirname); #endif if (taosDirExist(temp)) { if (checkAccess && taosCheckAccessFile(temp, TD_FILE_ACCESS_EXIST_OK | TD_FILE_ACCESS_READ_OK | TD_FILE_ACCESS_WRITE_OK)) { return 0; } - return chmod(temp, mode); + code = chmod(temp, mode); + if (-1 == code) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } } if (strncmp(temp, TD_DIRSEP, 1) == 0) { @@ -229,8 +241,8 @@ int32_t taosMulModeMkDir(const char *dirname, int mode, bool checkAccess) { code = mkdir(temp, mode); #endif if (code < 0 && errno != EEXIST) { - // terrno = TAOS_SYSTEM_ERROR(errno); - return code; + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; } *pos = TD_DIRSEP[0]; } @@ -245,8 +257,8 @@ int32_t taosMulModeMkDir(const char *dirname, int mode, bool checkAccess) { code = mkdir(temp, mode); #endif if (code < 0 && errno != EEXIST) { - // terrno = TAOS_SYSTEM_ERROR(errno); - return code; + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; } } @@ -254,10 +266,14 @@ int32_t taosMulModeMkDir(const char *dirname, int mode, bool checkAccess) { if (checkAccess && taosCheckAccessFile(temp, TD_FILE_ACCESS_EXIST_OK | TD_FILE_ACCESS_READ_OK | TD_FILE_ACCESS_WRITE_OK)) { return 0; } - return chmod(temp, mode); } - return chmod(temp, mode); + code = chmod(temp, mode); + if (-1 == code) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } + return code; } void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { @@ -271,7 +287,7 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue; char filename[1024]; - snprintf(filename, sizeof(filename), "%s/%s", dirname, taosGetDirEntryName(de)); + (void)snprintf(filename, sizeof(filename), "%s/%s", dirname, taosGetDirEntryName(de)); if (taosDirEntryIsDir(de)) { continue; } else { @@ -299,34 +315,25 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { } } - taosCloseDir(&pDir); - rmdir(dirname); + (void)taosCloseDir(&pDir); + (void)rmdir(dirname); } int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) { wordexp_t full_path; - switch (wordexp(dirname, &full_path, 0)) { + int32_t code = wordexp(dirname, &full_path, 0); + switch (code) { case 0: break; case WRDE_NOSPACE: wordfree(&full_path); - // printf("failed to expand path:%s since Out of memory\n", dirname); - return -1; - case WRDE_BADCHAR: - // printf("failed to expand path:%s since illegal occurrence of newline or one of |, &, ;, <, >, (, ), {, }\n", - // dirname); - return -1; - case WRDE_SYNTAX: - // printf("failed to expand path:%s since Shell syntax error, such as unbalanced parentheses or unmatched - // quotes\n", dirname); - return -1; + // FALL THROUGH default: - // printf("failed to expand path:%s since %s\n", dirname, strerror(errno)); - return -1; + return code; } if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) { - strncpy(outname, full_path.we_wordv[0], maxlen); + (void)strncpy(outname, full_path.we_wordv[0], maxlen); } wordfree(&full_path); @@ -343,21 +350,23 @@ int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen) { #endif if (strlen(tmp) < maxlen) { if (realPath == NULL) { - strncpy(dirname, tmp, maxlen); + (void)strncpy(dirname, tmp, maxlen); } else { - strncpy(realPath, tmp, maxlen); + (void)strncpy(realPath, tmp, maxlen); } return 0; } } - return -1; + terrno = TAOS_SYSTEM_ERROR(errno); + + return terrno; } bool taosIsDir(const char *dirname) { TdDirPtr pDir = taosOpenDir(dirname); if (pDir != NULL) { - taosCloseDir(&pDir); + (void)taosCloseDir(&pDir); return true; } return false; @@ -410,6 +419,7 @@ char *taosDirEntryBaseName(char *name) { TdDirPtr taosOpenDir(const char *dirname) { if (dirname == NULL) { + terrno = TSDB_CODE_INVALID_PARA; return NULL; } @@ -436,12 +446,17 @@ TdDirPtr taosOpenDir(const char *dirname) { dirPtr->pDir = pDir; return dirPtr; #else - return (TdDirPtr)opendir(dirname); + TdDirPtr ptr = (TdDirPtr)opendir(dirname); + if (NULL == ptr) { + terrno = TAOS_SYSTEM_ERROR(errno); + } + return ptr; #endif } TdDirEntryPtr taosReadDir(TdDirPtr pDir) { if (pDir == NULL) { + terrno = TSDB_CODE_INVALID_PARA; return NULL; } #ifdef WINDOWS @@ -456,7 +471,13 @@ TdDirEntryPtr taosReadDir(TdDirPtr pDir) { return NULL; } #else - return (TdDirEntryPtr)readdir((DIR *)pDir); + errno = 0; + terrno = 0; + TdDirEntryPtr p = (TdDirEntryPtr)readdir((DIR *)pDir); + if (NULL == p && errno) { + terrno = TAOS_SYSTEM_ERROR(errno); + } + return p; #endif } @@ -484,7 +505,8 @@ char *taosGetDirEntryName(TdDirEntryPtr pDirEntry) { int32_t taosCloseDir(TdDirPtr *ppDir) { if (ppDir == NULL || *ppDir == NULL) { - return -1; + terrno = TSDB_CODE_INVALID_PARA; + return terrno; } #ifdef WINDOWS FindClose((*ppDir)->hFind); @@ -497,8 +519,12 @@ int32_t taosCloseDir(TdDirPtr *ppDir) { *ppDir = NULL; return 0; #else - closedir((DIR *)*ppDir); + int32_t code = closedir((DIR *)*ppDir); *ppDir = NULL; + if (-1 == code) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } return 0; #endif } diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 948040ac76..72f0a41710 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -45,11 +45,16 @@ char tsAVX2Supported = 0; char tsFMASupported = 0; char tsAVX512Supported = 0; -void osDefaultInit() { +int32_t osDefaultInit() { + int32_t code = TSDB_CODE_SUCCESS; + taosSeedRand(taosSafeRand()); taosGetSystemLocale(tsLocale, tsCharset); taosGetSystemTimezone(tsTimezoneStr, &tsTimezone); - taosSetSystemTimezone(tsTimezoneStr, tsTimezoneStr, &tsDaylight, &tsTimezone); + code = taosSetSystemTimezone(tsTimezoneStr, tsTimezoneStr, &tsDaylight, &tsTimezone); + if (code) { + return code; + } taosGetSystemInfo(); // deadlock in query @@ -65,33 +70,35 @@ void osDefaultInit() { tmpDir = getenv("temp"); } if (tmpDir != NULL) { - strcpy(tsTempDir, tmpDir); + (void)strcpy(tsTempDir, tmpDir); } - strcpy(tsOsName, "Windows"); + (void)strcpy(tsOsName, "Windows"); #elif defined(_TD_DARWIN_64) - strcpy(tsOsName, "Darwin"); + (void)strcpy(tsOsName, "Darwin"); #else - strcpy(tsOsName, "Linux"); + (void)strcpy(tsOsName, "Linux"); #endif if (configDir[0] == 0) { - strcpy(configDir, TD_CFG_DIR_PATH); + (void)strcpy(configDir, TD_CFG_DIR_PATH); } - strcpy(tsDataDir, TD_DATA_DIR_PATH); - strcpy(tsLogDir, TD_LOG_DIR_PATH); + (void)strcpy(tsDataDir, TD_DATA_DIR_PATH); + (void)strcpy(tsLogDir, TD_LOG_DIR_PATH); if(strlen(tsTempDir) == 0){ - strcpy(tsTempDir, TD_TMP_DIR_PATH); + (void)strcpy(tsTempDir, TD_TMP_DIR_PATH); } + + return code; } void osUpdate() { if (tsLogDir[0] != 0) { - taosGetDiskSize(tsLogDir, &tsLogSpace.size); + (void)taosGetDiskSize(tsLogDir, &tsLogSpace.size); } if (tsDataDir[0] != 0) { - taosGetDiskSize(tsDataDir, &tsDataSpace.size); + (void)taosGetDiskSize(tsDataDir, &tsDataSpace.size); } if (tsTempDir[0] != 0) { - taosGetDiskSize(tsTempDir, &tsTempSpace.size); + (void)taosGetDiskSize(tsTempDir, &tsTempSpace.size); } } @@ -109,11 +116,11 @@ bool osDataSpaceSufficient() { return tsDataSpace.size.avail > tsDataSpace.reser bool osTempSpaceSufficient() { return tsTempSpace.size.avail > tsTempSpace.reserved; } -void osSetTimezone(const char *tz) { taosSetSystemTimezone(tz, tsTimezoneStr, &tsDaylight, &tsTimezone); } +int32_t osSetTimezone(const char *tz) { return taosSetSystemTimezone(tz, tsTimezoneStr, &tsDaylight, &tsTimezone); } void osSetSystemLocale(const char *inLocale, const char *inCharSet) { - memcpy(tsLocale, inLocale, strlen(inLocale) + 1); - memcpy(tsCharset, inCharSet, strlen(inCharSet) + 1); + (void)memcpy(tsLocale, inLocale, strlen(inLocale) + 1); + (void)memcpy(tsCharset, inCharSet, strlen(inCharSet) + 1); } void osSetProcPath(int32_t argc, char **argv) { tsProcPath = argv[0]; } diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index f63339bcfc..0581df573a 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -90,24 +90,24 @@ void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, cha char tmpPath[PATH_MAX]; int32_t len = strlen(inputTmpDir); - memcpy(tmpPath, inputTmpDir, len); + (void)memcpy(tmpPath, inputTmpDir, len); static uint64_t seqId = 0; if (tmpPath[len - 1] != '/') { tmpPath[len++] = '/'; } - strcpy(tmpPath + len, TD_TMP_FILE_PREFIX); + (void)strcpy(tmpPath + len, TD_TMP_FILE_PREFIX); if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) { - strcat(tmpPath, fileNamePrefix); - strcat(tmpPath, "-%d-%s"); + (void)strcat(tmpPath, fileNamePrefix); + (void)strcat(tmpPath, "-%d-%s"); } char rand[32] = {0}; - sprintf(rand, "%" PRIu64, atomic_add_fetch_64(&seqId, 1)); + (void)sprintf(rand, "%" PRIu64, atomic_add_fetch_64(&seqId, 1)); - snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); + (void)snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); #endif } @@ -123,6 +123,7 @@ int64_t taosCopyFile(const char *from, const char *to) { char buffer[4096]; int64_t size = 0; int64_t bytes; + int32_t code = TSDB_CODE_SUCCESS; // fidfrom = open(from, O_RDONLY); TdFilePtr pFileFrom = taosOpenFile(from, TD_FILE_READ); @@ -143,7 +144,7 @@ int64_t taosCopyFile(const char *from, const char *to) { if (bytes < sizeof(buffer)) break; } - int code = taosFsyncFile(pFileTo); + code = taosFsyncFile(pFileTo); taosCloseFile(&pFileFrom); taosCloseFile(&pFileTo); @@ -154,10 +155,12 @@ int64_t taosCopyFile(const char *from, const char *to) { return size; _err: + if (pFileFrom != NULL) taosCloseFile(&pFileFrom); if (pFileTo != NULL) taosCloseFile(&pFileTo); /* coverity[+retval] */ taosRemoveFile(to); + return -1; #endif } @@ -301,11 +304,15 @@ FILE *taosOpenFileForStream(const char *path, int32_t tdFileOptions) { } else { mode = (tdFileOptions & TD_FILE_TEXT) ? "rt+" : "rb+"; } - ASSERT(!(tdFileOptions & TD_FILE_EXCL)); if (tdFileOptions & TD_FILE_EXCL) { + terrno = TSDB_CODE_INVALID_PARA; return NULL; } - return fopen(path, mode); + FILE* f = fopen(path, mode); + if (NULL == f) { + terrno = TAOS_SYSTEM_ERROR(errno); + } + return f; } #ifdef WINDOWS @@ -653,6 +660,9 @@ int taosOpenFileNotStream(const char *path, int32_t tdFileOptions) { access |= (tdFileOptions & TD_FILE_CLOEXEC) ? O_CLOEXEC : 0; int fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO); + if (-1 == fd) { + terrno = TAOS_SYSTEM_ERROR(errno); + } return fd; } @@ -1060,14 +1070,14 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { #ifdef WINDOWS if (hFile != NULL) CloseHandle(hFile); #else - if (fd >= 0) close(fd); + if (fd >= 0) (void)close(fd); #endif - if (fp != NULL) fclose(fp); + if (fp != NULL) (void)fclose(fp); return NULL; } #if FILE_WITH_LOCK - taosThreadRwlockInit(&(pFile->rwlock), NULL); + (void)taosThreadRwlockInit(&(pFile->rwlock), NULL); #endif pFile->fp = fp; pFile->refId = 0; @@ -1082,9 +1092,15 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { // Remove it instantly, so when the program exits normally/abnormally, the file // will be automatically remove by OS. if (tdFileOptions & TD_FILE_AUTO_DEL) { - unlink(path); + if (-1 == unlink(path)) { + terrno = TAOS_SYSTEM_ERROR(errno); + (void)close(fd); + taosMemoryFree(pFile); + return NULL; + } } #endif + return pFile; } @@ -1094,11 +1110,11 @@ int32_t taosCloseFile(TdFilePtr *ppFile) { return 0; } #if FILE_WITH_LOCK - taosThreadRwlockWrlock(&((*ppFile)->rwlock)); + (void)taosThreadRwlockWrlock(&((*ppFile)->rwlock)); #endif if ((*ppFile)->fp != NULL) { - fflush((*ppFile)->fp); - fclose((*ppFile)->fp); + (void)fflush((*ppFile)->fp); + (void)fclose((*ppFile)->fp); (*ppFile)->fp = NULL; } #ifdef WINDOWS @@ -1113,16 +1129,20 @@ int32_t taosCloseFile(TdFilePtr *ppFile) { // warning: never fsync silently in base lib /*fsync((*ppFile)->fd);*/ code = close((*ppFile)->fd); + if (-1 == code) { + terrno = TAOS_SYSTEM_ERROR(errno); + } (*ppFile)->fd = -1; #endif } (*ppFile)->refId = 0; #if FILE_WITH_LOCK - taosThreadRwlockUnlock(&((*ppFile)->rwlock)); - taosThreadRwlockDestroy(&((*ppFile)->rwlock)); + (void)taosThreadRwlockUnlock(&((*ppFile)->rwlock)); + (void)taosThreadRwlockDestroy(&((*ppFile)->rwlock)); #endif taosMemoryFree(*ppFile); *ppFile = NULL; + return code; } @@ -1155,20 +1175,28 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) } #else #if FILE_WITH_LOCK - taosThreadRwlockRdlock(&(pFile->rwlock)); + (void)taosThreadRwlockRdlock(&(pFile->rwlock)); #endif ASSERT(pFile->fd >= 0); // Please check if you have closed the file. if (pFile->fd < 0) { #if FILE_WITH_LOCK - taosThreadRwlockUnlock(&(pFile->rwlock)); + (void)taosThreadRwlockUnlock(&(pFile->rwlock)); #endif + terrno = TSDB_CODE_INVALID_PARA; return -1; } + int32_t code = 0; int64_t ret = pread(pFile->fd, buf, count, offset); + if (-1 == ret) { + code = TAOS_SYSTEM_ERROR(errno); + } #endif #if FILE_WITH_LOCK - taosThreadRwlockUnlock(&(pFile->rwlock)); + (void)taosThreadRwlockUnlock(&(pFile->rwlock)); #endif + + terrno = code; + return ret; } diff --git a/source/os/src/osSleep.c b/source/os/src/osSleep.c index f6cc7d608d..f72805b15d 100644 --- a/source/os/src/osSleep.c +++ b/source/os/src/osSleep.c @@ -26,7 +26,9 @@ void taosSsleep(int32_t s) { #ifdef WINDOWS Sleep(1000 * s); #else - sleep(s); + while (s > 0) { + s = sleep(s); + } #endif } @@ -35,7 +37,7 @@ void taosMsleep(int32_t ms) { #ifdef WINDOWS Sleep(ms); #else - usleep(ms * 1000); + (void)usleep(ms * 1000); #endif } @@ -51,6 +53,6 @@ void taosUsleep(int32_t us) { WaitForSingleObject(timer, INFINITE); CloseHandle(timer); #else - usleep(us); + (void)usleep(us); #endif } diff --git a/source/os/src/osTimer.c b/source/os/src/osTimer.c index 36d364382a..6e5d9844a9 100644 --- a/source/os/src/osTimer.c +++ b/source/os/src/osTimer.c @@ -80,7 +80,7 @@ void taos_block_sigalrm(void) { static void taosDeleteTimer(void *tharg) { timer_t *pTimer = tharg; - timer_delete(*pTimer); + (void)timer_delete(*pTimer); } static TdThread timerThread; @@ -90,9 +90,9 @@ static volatile bool stopTimer = false; static void *taosProcessAlarmSignal(void *tharg) { // Block the signal sigset_t sigset; - sigemptyset(&sigset); - sigaddset(&sigset, SIGALRM); - sigprocmask(SIG_BLOCK, &sigset, NULL); + (void)sigemptyset(&sigset); + (void)sigaddset(&sigset, SIGALRM); + (void)sigprocmask(SIG_BLOCK, &sigset, NULL); void (*callback)(int) = tharg; struct sigevent sevent = {{0}}; @@ -110,7 +110,8 @@ static void *taosProcessAlarmSignal(void *tharg) { sevent.sigev_signo = SIGALRM; if (timer_create(CLOCK_REALTIME, &sevent, &timerId) == -1) { - // printf("Failed to create timer"); + terrno = TAOS_SYSTEM_ERROR(errno); + return NULL; } taosThreadCleanupPush(taosDeleteTimer, &timerId); @@ -122,15 +123,17 @@ static void *taosProcessAlarmSignal(void *tharg) { ts.it_interval.tv_sec = 0; ts.it_interval.tv_nsec = 1000000 * MSECONDS_PER_TICK; - if (timer_settime(timerId, 0, &ts, NULL)) { - // printf("Failed to init timer"); + if (-1 == timer_settime(timerId, 0, &ts, NULL)) { + terrno = TAOS_SYSTEM_ERROR(errno); break; } int signo; + int32_t code = 0; while (!stopTimer) { - if (sigwait(&sigset, &signo)) { - // printf("Failed to wait signal: number %d", signo); + code = sigwait(&sigset, &signo); + if (code) { + terrno = TAOS_SYSTEM_ERROR(code); continue; } /* //printf("Signal handling: number %d ......\n", signo); */ @@ -178,14 +181,12 @@ int taosInitTimer(void (*callback)(int), int ms) { #else stopTimer = false; TdThreadAttr tattr; - taosThreadAttrInit(&tattr); + (void)taosThreadAttrInit(&tattr); int code = taosThreadCreate(&timerThread, &tattr, taosProcessAlarmSignal, callback); - taosThreadAttrDestroy(&tattr); + (void)taosThreadAttrDestroy(&tattr); if (code != 0) { // printf("failed to create timer thread"); - return -1; - } else { - // printf("timer thread:0x%08" PRIx64 " is created", taosGetPthreadId(timerThread)); + return TAOS_SYSTEM_ERROR(code); } return 0; @@ -210,7 +211,7 @@ void taosUninitTimer() { stopTimer = true; // printf("join timer thread:0x%08" PRIx64, taosGetPthreadId(timerThread)); - taosThreadJoin(timerThread, NULL); + (void)taosThreadJoin(timerThread, NULL); #endif } diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index 72f7dda41c..8780fb2bc2 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -742,12 +742,20 @@ char *tz_win[554][2] = {{"Asia/Shanghai", "China Standard Time"}, static int isdst_now = 0; -void taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, int8_t *outDaylight, +int32_t taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, int8_t *outDaylight, enum TdTimezone *tsTimezone) { - if (inTimezoneStr == NULL || inTimezoneStr[0] == 0) return; + if (inTimezoneStr == NULL || inTimezoneStr[0] == 0) { + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + int32_t code = TSDB_CODE_SUCCESS; size_t len = strlen(inTimezoneStr); - char *buf = taosMemoryCalloc(len + 1, 1); + if (len >= TD_TIMEZONE_LEN) { + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + char buf[TD_TIMEZONE_LEN] = {0}; for (int32_t i = 0; i < len; i++) { if (inTimezoneStr[i] == ' ' || inTimezoneStr[i] == '(') { buf[i] = 0; @@ -813,17 +821,22 @@ void taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, int8 *outDaylight = isdst_now; #else - setenv("TZ", buf, 1); + code = setenv("TZ", buf, 1); + if (-1 == code) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } + tzset(); int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); *tsTimezone = tz; tz += isdst_now; - sprintf(outTimezoneStr, "%s (%s, %s%02d00)", buf, tzname[isdst_now], tz >= 0 ? "+" : "-", abs(tz)); + (void)sprintf(outTimezoneStr, "%s (%s, %s%02d00)", buf, tzname[isdst_now], tz >= 0 ? "+" : "-", abs(tz)); *outDaylight = isdst_now; #endif - taosMemoryFree(buf); + return code; } void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { @@ -914,7 +927,7 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { { int n = readlink("/etc/localtime", buf, sizeof(buf)-1); if (n < 0) { - printf("read /etc/localtime error, reason:%s", strerror(errno)); + (void)printf("read /etc/localtime error, reason:%s", strerror(errno)); if (taosCheckExistFile("/etc/timezone")) { /* @@ -924,7 +937,7 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { */ time_t tx1 = taosGetTimestampSec(); struct tm tm1; - taosLocalTime(&tx1, &tm1, NULL); + (void)taosLocalTime(&tx1, &tm1, NULL); /* load time zone string from /etc/timezone */ // FILE *f = fopen("/etc/timezone", "r"); errno = 0; @@ -933,12 +946,12 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { if (pFile != NULL) { int len = taosReadFile(pFile, buf, 64); if (len < 64 && taosGetErrorFile(pFile)) { - taosCloseFile(&pFile); - printf("read /etc/timezone error, reason:%s", strerror(errno)); + (void)taosCloseFile(&pFile); + (void)printf("read /etc/timezone error, reason:%s", strerror(errno)); return; } - taosCloseFile(&pFile); + (void)taosCloseFile(&pFile); buf[sizeof(buf) - 1] = 0; char *lineEnd = strstr(buf, "\n"); @@ -948,7 +961,7 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { // for CentOS system, /etc/timezone does not exist. Ignore the TZ environment variables if (strlen(buf) > 0) { - setenv("TZ", buf, 1); + (void)setenv("TZ", buf, 1); } } // get and set default timezone @@ -970,10 +983,10 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { * Asia/Shanghai (CST, +0800) * Europe/London (BST, +0100) */ - snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", + (void)snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); } else { - printf("There is not /etc/timezone.\n"); + (void)printf("There is not /etc/timezone.\n"); } return; } @@ -981,7 +994,7 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { char *zi = strstr(buf, "zoneinfo"); if (!zi) { - printf("parsing /etc/localtime failed"); + (void)printf("parsing /etc/localtime failed"); return; } tz = zi + strlen("zoneinfo") + 1; @@ -1000,7 +1013,7 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { // return; // } - setenv("TZ", tz, 1); + (void)setenv("TZ", tz, 1); tzset(); } @@ -1011,7 +1024,7 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { */ time_t tx1 = taosGetTimestampSec(); struct tm tm1; - taosLocalTime(&tx1, &tm1, NULL); + (void)taosLocalTime(&tx1, &tm1, NULL); isdst_now = tm1.tm_isdst; /* @@ -1020,7 +1033,7 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { * Asia/Shanghai (CST, +0800) * Europe/London (BST, +0100) */ - snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], + (void)snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], -timezone / 3600); #endif } diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 3ab0f83238..268b0b8497 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -136,8 +136,7 @@ static int32_t cfgCheckAndSetConf(SConfigItem *pItem, const char *conf) { static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { char fullDir[PATH_MAX] = {0}; if (taosExpandDir(inputDir, fullDir, PATH_MAX) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to expand dir:%s since %s", inputDir, terrstr()); + uError("failed to expand dir:%s", inputDir); return -1; } From 43d7532006986f63a410ab318bf6a24ba8e21582 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 23 Jul 2024 15:12:21 +0800 Subject: [PATCH 26/43] fix: a little bug --- source/dnode/vnode/src/meta/metaOpen.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 94eb461613..ec26e94c5a 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -46,7 +46,6 @@ int32_t metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { int32_t offset; char path[TSDB_FILENAME_LEN] = {0}; char indexFullPath[128] = {0}; - sprintf(indexFullPath, "%s/%s", pMeta->path, "invert"); // create handle vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, path, TSDB_FILENAME_LEN); @@ -97,6 +96,7 @@ int32_t metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { code = tdbTbOpen("suid.idx", sizeof(tb_uid_t), 0, uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pSuidIdx, 0); TSDB_CHECK_CODE(code, lino, _exit); + sprintf(indexFullPath, "%s/%s", pMeta->path, "invert"); TAOS_UNUSED(taosMkDir(indexFullPath)); SIndexOpts opts = {.cacheSize = 8 * 1024 * 1024}; From efa518f624c6953b155ddeb5905df8b292b323a0 Mon Sep 17 00:00:00 2001 From: sima Date: Fri, 19 Jul 2024 09:17:12 +0800 Subject: [PATCH 27/43] enh:[TD-31043] Handling return value in builtins.c --- include/util/taoserror.h | 3 + source/libs/function/src/builtins.c | 168 ++++++++++++++++------------ source/util/src/terror.c | 2 + 3 files changed, 101 insertions(+), 72 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 22da6d5390..7d1ce80b6c 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -860,6 +860,9 @@ int32_t taosGetErrSize(); #define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR TAOS_DEF_ERROR_CODE(0, 0x2807) #define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2808) #define TSDB_CODE_FUNC_TO_CHAR_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2809) +#define TSDB_CODE_FUNC_TIME_UNIT_INVALID TAOS_DEF_ERROR_CODE(0, 0x280A) +#define TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL TAOS_DEF_ERROR_CODE(0, 0x280B) + //udf #define TSDB_CODE_UDF_STOPPING TAOS_DEF_ERROR_CODE(0, 0x2901) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index f5c98933fd..9c57c682cf 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -25,7 +25,7 @@ static int32_t buildFuncErrMsg(char* pErrBuf, int32_t len, int32_t errCode, const char* pFormat, ...) { va_list vArgList; va_start(vArgList, pFormat); - vsnprintf(pErrBuf, len, pFormat, vArgList); + (void)vsnprintf(pErrBuf, len, pFormat, vArgList); va_end(vArgList); return errCode; } @@ -42,27 +42,24 @@ static int32_t invaildFuncParaValueErrMsg(char* pErrBuf, int32_t len, const char return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_PARA_VALUE, "Invalid parameter value : %s", pFuncName); } -#define TIME_UNIT_INVALID 1 -#define TIME_UNIT_TOO_SMALL 2 - static int32_t validateTimeUnitParam(uint8_t dbPrec, const SValueNode* pVal) { if (!IS_DURATION_VAL(pVal->flag)) { - return TIME_UNIT_INVALID; + return TSDB_CODE_FUNC_TIME_UNIT_INVALID; } if (TSDB_TIME_PRECISION_MILLI == dbPrec && (0 == strcasecmp(pVal->literal, "1u") || 0 == strcasecmp(pVal->literal, "1b"))) { - return TIME_UNIT_TOO_SMALL; + return TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL; } if (TSDB_TIME_PRECISION_MICRO == dbPrec && 0 == strcasecmp(pVal->literal, "1b")) { - return TIME_UNIT_TOO_SMALL; + return TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL; } if (pVal->literal[0] != '1' || (pVal->literal[1] != 'u' && pVal->literal[1] != 'a' && pVal->literal[1] != 's' && pVal->literal[1] != 'm' && pVal->literal[1] != 'h' && pVal->literal[1] != 'd' && pVal->literal[1] != 'w' && pVal->literal[1] != 'b')) { - return TIME_UNIT_INVALID; + return TSDB_CODE_FUNC_TIME_UNIT_INVALID; } return TSDB_CODE_SUCCESS; @@ -138,13 +135,13 @@ static bool validateTimezoneFormat(const SValueNode* pVal) { } if (i == 2) { - memcpy(buf, &tz[i - 1], 2); + (void)memcpy(buf, &tz[i - 1], 2); hour = taosStr2Int8(buf, NULL, 10); if (!validateHourRange(hour)) { return false; } } else if (i == 4) { - memcpy(buf, &tz[i - 1], 2); + (void)memcpy(buf, &tz[i - 1], 2); minute = taosStr2Int8(buf, NULL, 10); if (!validateMinuteRange(hour, minute, tz[0])) { return false; @@ -167,13 +164,13 @@ static bool validateTimezoneFormat(const SValueNode* pVal) { } if (i == 2) { - memcpy(buf, &tz[i - 1], 2); + (void)memcpy(buf, &tz[i - 1], 2); hour = taosStr2Int8(buf, NULL, 10); if (!validateHourRange(hour)) { return false; } } else if (i == 5) { - memcpy(buf, &tz[i - 1], 2); + (void)memcpy(buf, &tz[i - 1], 2); minute = taosStr2Int8(buf, NULL, 10); if (!validateMinuteRange(hour, minute, tz[0])) { return false; @@ -215,7 +212,7 @@ static int32_t addTimezoneParam(SNodeList* pList) { time_t t = taosTime(NULL); struct tm tmInfo; if (taosLocalTime(&t, &tmInfo, buf) != NULL) { - strftime(buf, sizeof(buf), "%z", &tmInfo); + (void)strftime(buf, sizeof(buf), "%z", &tmInfo); } int32_t len = (int32_t)strlen(buf); @@ -225,15 +222,27 @@ static int32_t addTimezoneParam(SNodeList* pList) { } pVal->literal = strndup(buf, len); + if (pVal->literal == NULL) { + nodesDestroyNode((SNode*)pVal); + return TSDB_CODE_OUT_OF_MEMORY; + } pVal->translate = true; pVal->node.resType.type = TSDB_DATA_TYPE_BINARY; pVal->node.resType.bytes = len + VARSTR_HEADER_SIZE; pVal->node.resType.precision = TSDB_TIME_PRECISION_MILLI; pVal->datum.p = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE + 1); + if (pVal->datum.p == NULL) { + nodesDestroyNode((SNode*)pVal); + return TSDB_CODE_OUT_OF_MEMORY; + } varDataSetLen(pVal->datum.p, len); - strncpy(varDataVal(pVal->datum.p), pVal->literal, len); + (void)strncpy(varDataVal(pVal->datum.p), pVal->literal, len); - nodesListAppend(pList, (SNode*)pVal); + int32_t code = nodesListAppend(pList, (SNode*)pVal); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode((SNode*)pVal); + return code; + } return TSDB_CODE_SUCCESS; } @@ -252,7 +261,11 @@ static int32_t addDbPrecisonParam(SNodeList** pList, uint8_t precision) { pVal->datum.i = (int64_t)precision; pVal->typeData = (int64_t)precision; - nodesListMakeAppend(pList, (SNode*)pVal); + int32_t code = nodesListMakeAppend(pList, (SNode*)pVal); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode((SNode*)pVal); + return code; + } return TSDB_CODE_SUCCESS; } @@ -936,13 +949,13 @@ static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len uint8_t dbPrec = pFunc->node.resType.precision; - int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1)); - if (ret == TIME_UNIT_TOO_SMALL) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + int32_t code = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1)); + if (code == TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL) { + return buildFuncErrMsg(pErrBuf, len, code, "ELAPSED function time unit parameter should be greater than db precision"); - } else if (ret == TIME_UNIT_INVALID) { + } else if (code == TSDB_CODE_FUNC_TIME_UNIT_INVALID) { return buildFuncErrMsg( - pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + pErrBuf, len, code, "ELAPSED function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]"); } } @@ -1062,7 +1075,7 @@ static int8_t validateHistogramBinType(char* binTypeStr) { return binType; } -static bool validateHistogramBinDesc(char* binDescStr, int8_t binType, char* errMsg, int32_t msgLen) { +static int32_t validateHistogramBinDesc(char* binDescStr, int8_t binType, char* errMsg, int32_t msgLen) { const char* msg1 = "HISTOGRAM function requires four parameters"; const char* msg3 = "HISTOGRAM function invalid format for binDesc parameter"; const char* msg4 = "HISTOGRAM function binDesc parameter \"count\" should be in range [1, 1000]"; @@ -1070,6 +1083,7 @@ static bool validateHistogramBinDesc(char* binDescStr, int8_t binType, char* err const char* msg6 = "HISTOGRAM function binDesc parameter \"width\" cannot be 0"; const char* msg7 = "HISTOGRAM function binDesc parameter \"start\" cannot be 0 with \"log_bin\" type"; const char* msg8 = "HISTOGRAM function binDesc parameter \"factor\" cannot be negative or equal to 0/1"; + const char* msg9 = "HISTOGRAM function out of memory"; cJSON* binDesc = cJSON_Parse(binDescStr); int32_t numOfBins; @@ -1078,9 +1092,9 @@ static bool validateHistogramBinDesc(char* binDescStr, int8_t binType, char* err int32_t numOfParams = cJSON_GetArraySize(binDesc); int32_t startIndex; if (numOfParams != 4) { - snprintf(errMsg, msgLen, "%s", msg1); + (void)snprintf(errMsg, msgLen, "%s", msg1); cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } cJSON* start = cJSON_GetObjectItem(binDesc, "start"); @@ -1090,22 +1104,22 @@ static bool validateHistogramBinDesc(char* binDescStr, int8_t binType, char* err cJSON* infinity = cJSON_GetObjectItem(binDesc, "infinity"); if (!cJSON_IsNumber(start) || !cJSON_IsNumber(count) || !cJSON_IsBool(infinity)) { - snprintf(errMsg, msgLen, "%s", msg3); + (void)snprintf(errMsg, msgLen, "%s", msg3); cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } if (count->valueint <= 0 || count->valueint > 1000) { // limit count to 1000 - snprintf(errMsg, msgLen, "%s", msg4); + (void)snprintf(errMsg, msgLen, "%s", msg4); cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } if (isinf(start->valuedouble) || (width != NULL && isinf(width->valuedouble)) || (factor != NULL && isinf(factor->valuedouble)) || (count != NULL && isinf(count->valuedouble))) { - snprintf(errMsg, msgLen, "%s", msg5); + (void)snprintf(errMsg, msgLen, "%s", msg5); cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } int32_t counter = (int32_t)count->valueint; @@ -1118,53 +1132,58 @@ static bool validateHistogramBinDesc(char* binDescStr, int8_t binType, char* err } intervals = taosMemoryCalloc(numOfBins, sizeof(double)); + if (intervals == NULL) { + (void)snprintf(errMsg, msgLen, "%s", msg9); + cJSON_Delete(binDesc); + return TSDB_CODE_FAILED; + } if (cJSON_IsNumber(width) && factor == NULL && binType == LINEAR_BIN) { // linear bin process if (width->valuedouble == 0) { - snprintf(errMsg, msgLen, "%s", msg6); + (void)snprintf(errMsg, msgLen, "%s", msg6); taosMemoryFree(intervals); cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } for (int i = 0; i < counter + 1; ++i) { intervals[startIndex] = start->valuedouble + i * width->valuedouble; if (isinf(intervals[startIndex])) { - snprintf(errMsg, msgLen, "%s", msg5); + (void)snprintf(errMsg, msgLen, "%s", msg5); taosMemoryFree(intervals); cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } startIndex++; } } else if (cJSON_IsNumber(factor) && width == NULL && binType == LOG_BIN) { // log bin process if (start->valuedouble == 0) { - snprintf(errMsg, msgLen, "%s", msg7); + (void)snprintf(errMsg, msgLen, "%s", msg7); taosMemoryFree(intervals); cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } if (factor->valuedouble < 0 || factor->valuedouble == 0 || factor->valuedouble == 1) { - snprintf(errMsg, msgLen, "%s", msg8); + (void)snprintf(errMsg, msgLen, "%s", msg8); taosMemoryFree(intervals); cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } for (int i = 0; i < counter + 1; ++i) { intervals[startIndex] = start->valuedouble * pow(factor->valuedouble, i * 1.0); if (isinf(intervals[startIndex])) { - snprintf(errMsg, msgLen, "%s", msg5); + (void)snprintf(errMsg, msgLen, "%s", msg5); taosMemoryFree(intervals); cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } startIndex++; } } else { - snprintf(errMsg, msgLen, "%s", msg3); + (void)snprintf(errMsg, msgLen, "%s", msg3); taosMemoryFree(intervals); cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } if (infinity->valueint == true) { @@ -1172,7 +1191,7 @@ static bool validateHistogramBinDesc(char* binDescStr, int8_t binType, char* err intervals[numOfBins - 1] = INFINITY; // in case of desc bin orders, -inf/inf should be swapped if (numOfBins < 4) { - return false; + return TSDB_CODE_FAILED; } if (intervals[1] > intervals[numOfBins - 2]) { @@ -1181,46 +1200,51 @@ static bool validateHistogramBinDesc(char* binDescStr, int8_t binType, char* err } } else if (cJSON_IsArray(binDesc)) { /* user input bins */ if (binType != USER_INPUT_BIN) { - snprintf(errMsg, msgLen, "%s", msg3); + (void)snprintf(errMsg, msgLen, "%s", msg3); cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } numOfBins = cJSON_GetArraySize(binDesc); intervals = taosMemoryCalloc(numOfBins, sizeof(double)); + if (intervals == NULL) { + (void)snprintf(errMsg, msgLen, "%s", msg9); + cJSON_Delete(binDesc); + return TSDB_CODE_OUT_OF_MEMORY; + } cJSON* bin = binDesc->child; if (bin == NULL) { - snprintf(errMsg, msgLen, "%s", msg3); + (void)snprintf(errMsg, msgLen, "%s", msg3); taosMemoryFree(intervals); cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } int i = 0; while (bin) { intervals[i] = bin->valuedouble; if (!cJSON_IsNumber(bin)) { - snprintf(errMsg, msgLen, "%s", msg3); + (void)snprintf(errMsg, msgLen, "%s", msg3); taosMemoryFree(intervals); cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } if (i != 0 && intervals[i] <= intervals[i - 1]) { - snprintf(errMsg, msgLen, "%s", msg3); + (void)snprintf(errMsg, msgLen, "%s", msg3); taosMemoryFree(intervals); cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } bin = bin->next; i++; } } else { - snprintf(errMsg, msgLen, "%s", msg3); + (void)snprintf(errMsg, msgLen, "%s", msg3); cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } cJSON_Delete(binDesc); taosMemoryFree(intervals); - return true; + return TSDB_CODE_SUCCESS; } static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { @@ -1265,7 +1289,7 @@ static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t l if (i == 2) { char errMsg[128] = {0}; binDesc = varDataVal(pValue->datum.p); - if (!validateHistogramBinDesc(binDesc, binType, errMsg, (int32_t)sizeof(errMsg))) { + if (TSDB_CODE_SUCCESS != validateHistogramBinDesc(binDesc, binType, errMsg, (int32_t)sizeof(errMsg))) { return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, errMsg); } } @@ -1323,7 +1347,7 @@ static int32_t translateHistogramImpl(SFunctionNode* pFunc, char* pErrBuf, int32 if (i == 2) { char errMsg[128] = {0}; binDesc = varDataVal(pValue->datum.p); - if (!validateHistogramBinDesc(binDesc, binType, errMsg, (int32_t)sizeof(errMsg))) { + if (TSDB_CODE_SUCCESS != validateHistogramBinDesc(binDesc, binType, errMsg, (int32_t)sizeof(errMsg))) { return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, errMsg); } } @@ -1507,12 +1531,12 @@ static int32_t translateStateDuration(SFunctionNode* pFunc, char* pErrBuf, int32 if (numOfParams == 4) { uint8_t dbPrec = pFunc->node.resType.precision; - int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 3)); - if (ret == TIME_UNIT_TOO_SMALL) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + int32_t code = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 3)); + if (code == TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL) { + return buildFuncErrMsg(pErrBuf, len, code, "STATEDURATION function time unit parameter should be greater than db precision"); - } else if (ret == TIME_UNIT_INVALID) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + } else if (code == TSDB_CODE_FUNC_TIME_UNIT_INVALID) { + return buildFuncErrMsg(pErrBuf, len, code, "STATEDURATION function time unit parameter should be one of the following: [1b, 1u, 1a, " "1s, 1m, 1h, 1d, 1w]"); } @@ -2268,13 +2292,13 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_ } uint8_t dbPrec = pFunc->node.resType.precision; - int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1)); - if (ret == TIME_UNIT_TOO_SMALL) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + int32_t code = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1)); + if (code == TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL) { + return buildFuncErrMsg(pErrBuf, len, code, "TIMETRUNCATE function time unit parameter should be greater than db precision"); - } else if (ret == TIME_UNIT_INVALID) { + } else if (code == TSDB_CODE_FUNC_TIME_UNIT_INVALID) { return buildFuncErrMsg( - pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + pErrBuf, len, code, "TIMETRUNCATE function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]"); } @@ -2291,7 +2315,7 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_ // add database precision as param - int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); + code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2330,13 +2354,13 @@ static int32_t translateTimeDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t le uint8_t dbPrec = pFunc->node.resType.precision; if (3 == numOfParams) { - int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 2)); - if (ret == TIME_UNIT_TOO_SMALL) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + int32_t code = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 2)); + if (code == TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL) { + return buildFuncErrMsg(pErrBuf, len, code, "TIMEDIFF function time unit parameter should be greater than db precision"); - } else if (ret == TIME_UNIT_INVALID) { + } else if (code == TSDB_CODE_FUNC_TIME_UNIT_INVALID) { return buildFuncErrMsg( - pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + pErrBuf, len, code, "TIMEDIFF function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]"); } } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 4db1475fa9..3bd1ae3ed5 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -707,6 +707,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_FORMAT_ERR, "Func to_timest TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR, "Func to_timestamp failed for wrong timestamp") TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED, "Func to_timestamp failed for unsupported timestamp format") TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_CHAR_NOT_SUPPORTED, "Func to_char failed for unsupported format") +TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TIME_UNIT_INVALID, "Invalid function time unit") +TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL, "Function time unit cannot be smaller than db precision") //udf TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING, "udf is stopping") From 6a9441528a366b99307e53c23bf4abfca904f716 Mon Sep 17 00:00:00 2001 From: sima Date: Fri, 19 Jul 2024 18:43:35 +0800 Subject: [PATCH 28/43] enh:[TD-31043] Handling return value in builtinsimpl.c --- include/libs/function/function.h | 2 +- include/libs/function/tudf.h | 2 +- include/util/taoserror.h | 2 + source/client/src/clientEnv.c | 6 +- source/libs/executor/src/executorInt.c | 4 +- source/libs/executor/src/projectoperator.c | 33 +- source/libs/executor/src/timewindowoperator.c | 13 +- source/libs/function/inc/builtinsimpl.h | 46 +- source/libs/function/inc/tfunctionInt.h | 2 +- source/libs/function/inc/tpercentile.h | 3 +- source/libs/function/src/builtinsimpl.c | 1102 ++++++++++------- .../libs/function/src/detail/tavgfunction.c | 15 +- source/libs/function/src/thistogram.c | 2 +- source/libs/function/src/tpercentile.c | 74 +- source/libs/function/src/tudf.c | 19 +- source/libs/parser/test/parTestMain.cpp | 3 +- source/libs/planner/test/planTestMain.cpp | 3 +- source/util/src/terror.c | 2 + 18 files changed, 802 insertions(+), 531 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 700cc5ba7f..24fa2898ea 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -36,7 +36,7 @@ typedef struct SFuncExecEnv { } SFuncExecEnv; typedef bool (*FExecGetEnv)(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv); -typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo); +typedef int32_t (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo); typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx); typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock); typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h index 6b15833917..7a8927ca90 100644 --- a/include/libs/function/tudf.h +++ b/include/libs/function/tudf.h @@ -77,7 +77,7 @@ void freeUdfInterBuf(SUdfInterBuf *buf); // high level APIs bool udfAggGetEnv(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv); -bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo); +int32_t udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo); int32_t udfAggProcess(struct SqlFunctionCtx *pCtx); int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 7d1ce80b6c..9b49c1908d 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -862,6 +862,8 @@ int32_t taosGetErrSize(); #define TSDB_CODE_FUNC_TO_CHAR_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2809) #define TSDB_CODE_FUNC_TIME_UNIT_INVALID TAOS_DEF_ERROR_CODE(0, 0x280A) #define TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL TAOS_DEF_ERROR_CODE(0, 0x280B) +#define TSDB_CODE_FUNC_INVALID_VALUE_RANGE TAOS_DEF_ERROR_CODE(0, 0x280C) +#define TSDB_CODE_FUNC_SETUP_ERROR TAOS_DEF_ERROR_CODE(0, 0x280D) //udf diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 724229af16..f640618897 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -897,7 +897,11 @@ void taos_init_imp(void) { tscError("failed to init task queue"); return; } - fmFuncMgtInit(); + if (fmFuncMgtInit() != TSDB_CODE_SUCCESS) { + tscInitRes = -1; + tscError("failed to init function manager"); + return; + } nodesInitAllocatorSet(); clientConnRefPool = taosOpenRef(200, destroyTscObj); diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index a3e3501114..4ee7030d9a 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -521,8 +521,8 @@ int32_t setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t n if (!pResInfo->initialized) { if (pCtx[i].functionId != -1) { - bool ini = pCtx[i].fpSet.init(&pCtx[i], pResInfo); - if (!ini && fmIsUserDefinedFunc(pCtx[i].functionId)) { + int32_t code = pCtx[i].fpSet.init(&pCtx[i], pResInfo); + if (code != TSDB_CODE_SUCCESS && fmIsUserDefinedFunc(pCtx[i].functionId)){ pResInfo->initialized = false; return TSDB_CODE_UDF_FUNC_EXEC_FAILURE; } diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 6167497c21..93901b6b33 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -44,8 +44,8 @@ static int32_t doGenerateSourceData(SOperatorInfo* pOperator); static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator); static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator); static SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols); -static void setFunctionResultOutput(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t stage, - int32_t numOfExprs); +static int32_t setFunctionResultOutput(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SAggSupporter* pSup, + int32_t stage, int32_t numOfExprs); static void destroyProjectOperatorInfo(void* param) { if (NULL == param) { @@ -142,7 +142,10 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys } initBasicInfo(&pInfo->binfo, pResBlock); - setFunctionResultOutput(pOperator, &pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, numOfCols); + code = setFunctionResultOutput(pOperator, &pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, numOfCols); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } code = filterInitFromNode((SNode*)pProjPhyNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); if (code != TSDB_CODE_SUCCESS) { @@ -447,7 +450,11 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy goto _error; } - setFunctionResultOutput(pOperator, &pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, numOfExpr); + code = setFunctionResultOutput(pOperator, &pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, numOfExpr); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + code = filterInitFromNode((SNode*)pPhyNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -589,7 +596,8 @@ SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) { return (rows > 0) ? pInfo->pRes : NULL; } -void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size) { +int32_t initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size) { + int32_t code = TSDB_CODE_SUCCESS; for (int32_t j = 0; j < size; ++j) { struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[j]); if (isRowEntryInitialized(pResInfo) || fmIsPseudoColumnFunc(pCtx[j].functionId) || pCtx[j].functionId == -1 || @@ -597,8 +605,12 @@ void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size) { continue; } - pCtx[j].fpSet.init(&pCtx[j], pCtx[j].resultInfo); + code = pCtx[j].fpSet.init(&pCtx[j], pCtx[j].resultInfo); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } + return code; } /* @@ -610,7 +622,7 @@ void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size) { * offset[0] offset[1] offset[2] */ // TODO refactor: some function move away -void setFunctionResultOutput(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t stage, +int32_t setFunctionResultOutput(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t stage, int32_t numOfExprs) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; @@ -632,7 +644,7 @@ void setFunctionResultOutput(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SA pCtx[i].scanFlag = stage; } - initCtxOutputBuffer(pCtx, numOfExprs); + return initCtxOutputBuffer(pCtx, numOfExprs); } SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols) { @@ -841,7 +853,10 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc // do nothing } else if (fmIsIndefiniteRowsFunc(pfCtx->functionId)) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pfCtx); - pfCtx->fpSet.init(pfCtx, pResInfo); + code = pfCtx->fpSet.init(pfCtx, pResInfo); + if (TSDB_CODE_SUCCESS != code) { + goto _exit; + } pfCtx->pOutput = taosArrayGet(pResult->pDataBlock, outputSlotId); pfCtx->offset = createNewColModel ? 0 : pResult->info.rows; // set the start offset diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index fb3176a2a8..5678400410 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1086,12 +1086,13 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { return (rows == 0) ? NULL : pBlock; } -static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprSupp* pSup, int32_t numOfOutput) { +static int32_t doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprSupp* pSup, int32_t numOfOutput) { SResultRow* pResult = getResultRowByPos(pResultBuf, p1, false); if (NULL == pResult) { - return; + return TSDB_CODE_SUCCESS; } + int32_t code = TSDB_CODE_SUCCESS; SqlFunctionCtx* pCtx = pSup->pCtx; for (int32_t i = 0; i < numOfOutput; ++i) { pCtx[i].resultInfo = getResultEntryInfo(pResult, i, pSup->rowEntryInfoOffset); @@ -1101,15 +1102,19 @@ static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, } pResInfo->initialized = false; if (pCtx[i].functionId != -1) { - pCtx[i].fpSet.init(&pCtx[i], pResInfo); + code = pCtx[i].fpSet.init(&pCtx[i], pResInfo); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } } SFilePage* bufPage = getBufPage(pResultBuf, p1->pageId); if (NULL == bufPage) { - return; + return TSDB_CODE_SUCCESS; } setBufPageDirty(bufPage, true); releaseBufPage(pResultBuf, bufPage); + return TSDB_CODE_SUCCESS; } static void destroyStateWindowOperatorInfo(void* param) { diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index b48a617b9c..274ef61feb 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -49,9 +49,9 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems) int32_t saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); -const char* loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos); +int32_t loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos, char** value); -bool functionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); +int32_t functionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, char* finalResult); int32_t combineFunction(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); @@ -74,7 +74,7 @@ int32_t sumInvertFunction(SqlFunctionCtx* pCtx); int32_t sumCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); -bool minmaxFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); +int32_t minmaxFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); bool getMinmaxFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t minFunction(SqlFunctionCtx* pCtx); int32_t maxFunction(SqlFunctionCtx* pCtx); @@ -83,7 +83,7 @@ int32_t minCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t maxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); bool getAvgFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); -bool avgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); +int32_t avgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t avgFunction(SqlFunctionCtx* pCtx); int32_t avgFunctionMerge(SqlFunctionCtx* pCtx); int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); @@ -97,7 +97,7 @@ int32_t avgCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t getAvgInfoSize(); bool getStddevFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); -bool stddevFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); +int32_t stddevFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t stddevFunction(SqlFunctionCtx* pCtx); int32_t stddevFunctionMerge(SqlFunctionCtx* pCtx); int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); @@ -111,18 +111,18 @@ int32_t stddevCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t getStddevInfoSize(); bool getLeastSQRFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); -bool leastSQRFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); +int32_t leastSQRFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t leastSQRFunction(SqlFunctionCtx* pCtx); int32_t leastSQRFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t leastSQRCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); bool getPercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); -bool percentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); +int32_t percentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t percentileFunction(SqlFunctionCtx* pCtx); int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); bool getApercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); -bool apercentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); +int32_t apercentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t apercentileFunction(SqlFunctionCtx* pCtx); int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx); int32_t apercentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); @@ -131,16 +131,16 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) int32_t getApercentileMaxSize(); bool getDiffFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); -bool diffFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo); +int32_t diffFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo); int32_t diffFunction(SqlFunctionCtx* pCtx); int32_t diffFunctionByRow(SArray* pCtx); bool getDerivativeFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); -bool derivativeFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo); +int32_t derivativeFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo); int32_t derivativeFunction(SqlFunctionCtx* pCtx); bool getIrateFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); -bool irateFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo); +int32_t irateFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo); int32_t irateFunction(SqlFunctionCtx* pCtx); int32_t irateFunctionMerge(SqlFunctionCtx* pCtx); int32_t irateFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); @@ -165,7 +165,7 @@ EFuncDataRequired lastDynDataReq(void* pRes, SDataBlockInfo* pBlockInfo); int32_t lastRowFunction(SqlFunctionCtx* pCtx); bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv); -bool topBotFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); +int32_t topBotFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t topFunction(SqlFunctionCtx* pCtx); int32_t bottomFunction(SqlFunctionCtx* pCtx); int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); @@ -174,7 +174,7 @@ int32_t bottomCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t getTopBotInfoSize(int64_t numOfItems); bool getSpreadFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); -bool spreadFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); +int32_t spreadFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t spreadFunction(SqlFunctionCtx* pCtx); int32_t spreadFunctionMerge(SqlFunctionCtx* pCtx); int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); @@ -183,7 +183,7 @@ int32_t getSpreadInfoSize(); int32_t spreadCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); bool getElapsedFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); -bool elapsedFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); +int32_t elapsedFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t elapsedFunction(SqlFunctionCtx* pCtx); int32_t elapsedFunctionMerge(SqlFunctionCtx* pCtx); int32_t elapsedFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); @@ -192,7 +192,7 @@ int32_t getElapsedInfoSize(); int32_t elapsedCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); bool getHistogramFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); -bool histogramFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); +int32_t histogramFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t histogramFunction(SqlFunctionCtx* pCtx); int32_t histogramFunctionPartial(SqlFunctionCtx* pCtx); int32_t histogramFunctionMerge(SqlFunctionCtx* pCtx); @@ -210,7 +210,7 @@ int32_t getHLLInfoSize(); int32_t hllCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); bool getStateFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); -bool stateFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); +int32_t stateFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t stateCountFunction(SqlFunctionCtx* pCtx); int32_t stateDurationFunction(SqlFunctionCtx* pCtx); @@ -218,35 +218,35 @@ bool getCsumFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t csumFunction(SqlFunctionCtx* pCtx); bool getMavgFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); -bool mavgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); +int32_t mavgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t mavgFunction(SqlFunctionCtx* pCtx); bool getSampleFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); -bool sampleFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); +int32_t sampleFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t sampleFunction(SqlFunctionCtx* pCtx); int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); bool getTailFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); -bool tailFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); +int32_t tailFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t tailFunction(SqlFunctionCtx* pCtx); bool getUniqueFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); -bool uniqueFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); +int32_t uniqueFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t uniqueFunction(SqlFunctionCtx* pCtx); bool getModeFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); -bool modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); +int32_t modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t modeFunction(SqlFunctionCtx* pCtx); int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); bool getTwaFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); -bool twaFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); +int32_t twaFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t twaFunction(SqlFunctionCtx* pCtx); int32_t twaFinalize(struct SqlFunctionCtx* pCtx, SSDataBlock* pBlock); bool getSelectivityFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); -bool blockDistSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); +int32_t blockDistSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t blockDistFunction(SqlFunctionCtx* pCtx); int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); diff --git a/source/libs/function/inc/tfunctionInt.h b/source/libs/function/inc/tfunctionInt.h index b4c48abf37..8d9e42576d 100644 --- a/source/libs/function/inc/tfunctionInt.h +++ b/source/libs/function/inc/tfunctionInt.h @@ -37,7 +37,7 @@ static FORCE_INLINE void initResultRowEntry(SResultRowEntryInfo *pResInfo, int32 pResInfo->complete = false; pResInfo->numOfRes = 0; - memset(GET_ROWCELL_INTERBUF(pResInfo), 0, bufLen); + (void)memset(GET_ROWCELL_INTERBUF(pResInfo), 0, bufLen); } #ifdef __cplusplus diff --git a/source/libs/function/inc/tpercentile.h b/source/libs/function/inc/tpercentile.h index 34815a34ad..90fb279259 100644 --- a/source/libs/function/inc/tpercentile.h +++ b/source/libs/function/inc/tpercentile.h @@ -67,7 +67,8 @@ typedef struct tMemBucket { SHashObj *groupPagesMap; // disk page map for different groups; } tMemBucket; -tMemBucket *tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval, bool hasWindowOrGroup); +int32_t tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval, bool hasWindowOrGroup, + tMemBucket **pBucket); void tMemBucketDestroy(tMemBucket *pBucket); diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 5f6565be1f..5c960e2580 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -429,7 +429,7 @@ typedef struct SGroupKeyInfo { (_p).val = (_v); \ } while (0) -int32_t funcInputUpdate(SqlFunctionCtx* pCtx) { +void funcInputUpdate(SqlFunctionCtx* pCtx) { SFuncInputRowIter* pIter = &pCtx->rowIter; if (!pCtx->bInputFinished) { @@ -448,11 +448,9 @@ int32_t funcInputUpdate(SqlFunctionCtx* pCtx) { } else { pIter->finalRow = true; } - - return TSDB_CODE_SUCCESS; } -bool funcInputGetNextRowDescPk(SFuncInputRowIter* pIter, SFuncInputRow* pRow) { +int32_t funcInputGetNextRowDescPk(SFuncInputRowIter* pIter, SFuncInputRow* pRow, bool *res) { if (pIter->finalRow) { if (pIter->hasPrev) { pRow->ts = pIter->prevBlockTsEnd; @@ -462,29 +460,41 @@ bool funcInputGetNextRowDescPk(SFuncInputRowIter* pIter, SFuncInputRow* pRow) { pRow->rowIndex = 0; pIter->hasPrev = false; - return true; + *res = true; + return TSDB_CODE_SUCCESS; } else { - return false; + *res = false; + return TSDB_CODE_SUCCESS; } } if (pIter->hasPrev) { if (pIter->prevBlockTsEnd == pIter->tsList[pIter->inputEndIndex]) { - blockDataDestroy(pIter->pPrevRowBlock); + (void)blockDataDestroy(pIter->pPrevRowBlock); pIter->pPrevRowBlock = blockDataExtractBlock(pIter->pSrcBlock, pIter->inputEndIndex, 1); pIter->prevIsDataNull = colDataIsNull_f(pIter->pDataCol->nullbitmap, pIter->inputEndIndex); pIter->pPrevData = taosMemoryMalloc(pIter->pDataCol->info.bytes); + if (NULL == pIter->pPrevData) { + qError("out of memory when function get input row."); + return TSDB_CODE_OUT_OF_MEMORY; + } char* srcData = colDataGetData(pIter->pDataCol, pIter->inputEndIndex); - memcpy(pIter->pPrevData, srcData, pIter->pDataCol->info.bytes); + (void)memcpy(pIter->pPrevData, srcData, pIter->pDataCol->info.bytes); pIter->pPrevPk = taosMemoryMalloc(pIter->pPkCol->info.bytes); + if (NULL == pIter->pPrevPk) { + qError("out of memory when function get input row."); + taosMemoryFree(pIter->pPrevData); + return TSDB_CODE_OUT_OF_MEMORY; + } char* pkData = colDataGetData(pIter->pPkCol, pIter->inputEndIndex); - memcpy(pIter->pPrevPk, pkData, pIter->pPkCol->info.bytes); + (void)memcpy(pIter->pPrevPk, pkData, pIter->pPkCol->info.bytes); pIter->pPrevRowBlock = blockDataExtractBlock(pIter->pSrcBlock, pIter->inputEndIndex, 1); pIter->hasPrev = true; - return false; + *res = false; + return TSDB_CODE_SUCCESS; } else { int32_t idx = pIter->rowIndex; while (pIter->tsList[idx] == pIter->prevBlockTsEnd) { @@ -506,7 +516,8 @@ bool funcInputGetNextRowDescPk(SFuncInputRowIter* pIter, SFuncInputRow* pRow) { } pIter->hasPrev = false; pIter->rowIndex = idx; - return true; + *res = true; + return TSDB_CODE_SUCCESS; } } else { TSKEY tsEnd = pIter->tsList[pIter->inputEndIndex]; @@ -522,18 +533,29 @@ bool funcInputGetNextRowDescPk(SFuncInputRowIter* pIter, SFuncInputRow* pRow) { pRow->block = pIter->pSrcBlock; pIter->rowIndex = idx + 1; - return true; + *res = true; + return TSDB_CODE_SUCCESS; } else { pIter->hasPrev = true; pIter->prevBlockTsEnd = tsEnd; pIter->prevIsDataNull = colDataIsNull_f(pIter->pDataCol->nullbitmap, pIter->inputEndIndex); pIter->pPrevData = taosMemoryMalloc(pIter->pDataCol->info.bytes); - memcpy(pIter->pPrevData, colDataGetData(pIter->pDataCol, pIter->inputEndIndex), pIter->pDataCol->info.bytes); + if (NULL == pIter->pPrevData) { + qError("out of memory when function get input row."); + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(pIter->pPrevData, colDataGetData(pIter->pDataCol, pIter->inputEndIndex), pIter->pDataCol->info.bytes); pIter->pPrevPk = taosMemoryMalloc(pIter->pPkCol->info.bytes); - memcpy(pIter->pPrevPk, colDataGetData(pIter->pPkCol, pIter->inputEndIndex), pIter->pPkCol->info.bytes); + if (NULL == pIter->pPrevPk) { + qError("out of memory when function get input row."); + taosMemoryFree(pIter->pPrevData); + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(pIter->pPrevPk, colDataGetData(pIter->pPkCol, pIter->inputEndIndex), pIter->pPkCol->info.bytes); pIter->pPrevRowBlock = blockDataExtractBlock(pIter->pSrcBlock, pIter->inputEndIndex, 1); - return false; + *res = false; + return TSDB_CODE_SUCCESS; } } } @@ -602,24 +624,27 @@ bool funcInputGetNextRowNoPk(SFuncInputRowIter *pIter, SFuncInputRow* pRow) { } } -bool funcInputGetNextRow(SqlFunctionCtx* pCtx, SFuncInputRow* pRow) { +int32_t funcInputGetNextRow(SqlFunctionCtx* pCtx, SFuncInputRow* pRow, bool *res) { SFuncInputRowIter* pIter = &pCtx->rowIter; if (pCtx->hasPrimaryKey) { if (pCtx->order == TSDB_ORDER_ASC) { - return funcInputGetNextRowAscPk(pIter, pRow); + *res = funcInputGetNextRowAscPk(pIter, pRow); + return TSDB_CODE_SUCCESS; } else { - return funcInputGetNextRowDescPk(pIter, pRow); + return funcInputGetNextRowDescPk(pIter, pRow, res); } } else { - return funcInputGetNextRowNoPk(pIter, pRow); + *res = funcInputGetNextRowNoPk(pIter, pRow); + return TSDB_CODE_SUCCESS; } + return TSDB_CODE_SUCCESS; } // This function append the selectivity to subsidiaries function context directly, without fetching data // from intermediate disk based buf page -void appendSelectivityCols(SqlFunctionCtx* pCtx, SSDataBlock* pSrcBlock, int32_t rowIndex, int32_t pos) { +int32_t appendSelectivityCols(SqlFunctionCtx* pCtx, SSDataBlock* pSrcBlock, int32_t rowIndex, int32_t pos) { if (pCtx->subsidiaries.num <= 0) { - return; + return TSDB_CODE_SUCCESS; } for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) { @@ -641,29 +666,34 @@ void appendSelectivityCols(SqlFunctionCtx* pCtx, SSDataBlock* pSrcBlock, int32_t if (colDataIsNull_s(pSrcCol, rowIndex) == true) { colDataSetNULL(pDstCol, pos); } else { - colDataSetVal(pDstCol, pos, pData, false); + int32_t code = colDataSetVal(pDstCol, pos, pData, false); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } } + return TSDB_CODE_SUCCESS; } bool funcInputGetNextRowIndex(SInputColumnInfoData* pInput, int32_t from, bool firstOccur, int32_t* pRowIndex, int32_t* nextFrom); -static int32_t firstLastTransferInfoImpl(SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst); +static bool firstLastTransferInfoImpl(SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst); -bool functionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { +int32_t functionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { if (pResultInfo->initialized) { - return false; + return TSDB_CODE_SUCCESS; // already initialized } if (pCtx->pOutput != NULL) { - memset(pCtx->pOutput, 0, (size_t)pCtx->resDataInfo.bytes); + (void)memset(pCtx->pOutput, 0, (size_t)pCtx->resDataInfo.bytes); } initResultRowEntry(pResultInfo, pCtx->resDataInfo.interBufSize); - return true; + return TSDB_CODE_SUCCESS; } int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + int32_t code = TSDB_CODE_SUCCESS; int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); @@ -671,9 +701,9 @@ int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; char* in = GET_ROWCELL_INTERBUF(pResInfo); - colDataSetVal(pCol, pBlock->info.rows, in, pResInfo->isNullRes); + code = colDataSetVal(pCol, pBlock->info.rows, in, pResInfo->isNullRes); - return pResInfo->numOfRes; + return code; } int32_t firstCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { @@ -684,9 +714,7 @@ int32_t firstCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); SFirstLastRes* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); - if (TSDB_CODE_SUCCESS == firstLastTransferInfoImpl(pSBuf, pDBuf, true)) { - pDBuf->hasResult = true; - } + pDBuf->hasResult = firstLastTransferInfoImpl(pSBuf, pDBuf, true); pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes); pDResInfo->isNullRes &= pSResInfo->isNullRes; @@ -701,9 +729,9 @@ int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; char* in = finalResult; - colDataSetVal(pCol, pBlock->info.rows, in, pResInfo->isNullRes); + int32_t code = colDataSetVal(pCol, pBlock->info.rows, in, pResInfo->isNullRes); - return pResInfo->numOfRes; + return code; } EFuncDataRequired countDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) { @@ -972,9 +1000,12 @@ EFuncDataRequired statisDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWin return FUNC_DATA_REQUIRED_SMA_LOAD; } -bool minmaxFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { - if (!functionSetup(pCtx, pResultInfo)) { - return false; // not initialized since it has been initialized +int32_t minmaxFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { + if (pResultInfo->initialized) { + return TSDB_CODE_SUCCESS; + } + if (TSDB_CODE_SUCCESS != functionSetup(pCtx, pResultInfo)) { + return TSDB_CODE_FUNC_SETUP_ERROR; // not initialized since it has been initialized } SMinmaxResInfo* buf = GET_ROWCELL_INTERBUF(pResultInfo); @@ -983,7 +1014,7 @@ bool minmaxFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) buf->nullTupleSaved = false; buf->nullTuplePos.pageId = -1; - return true; + return TSDB_CODE_SUCCESS; } bool getMinmaxFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { @@ -1097,12 +1128,12 @@ int32_t setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STu if ((pCtx->saveHandle.pBuf != NULL && pTuplePos->pageId != -1) || (pCtx->saveHandle.pState && pTuplePos->streamTupleKey.ts > 0)) { int32_t numOfCols = pCtx->subsidiaries.num; - const char* p = loadTupleData(pCtx, pTuplePos); - if (p == NULL) { - terrno = TSDB_CODE_NOT_FOUND; + char* p = NULL; + int32_t code = loadTupleData(pCtx, pTuplePos, &p); + if (p == NULL || TSDB_CODE_SUCCESS != code) { qError("Load tuple data failed since %s, groupId:%" PRIu64 ", ts:%" PRId64, terrstr(), pTuplePos->streamTupleKey.groupId, pTuplePos->streamTupleKey.ts); - return terrno; + return TSDB_CODE_NOT_FOUND; } bool* nullList = (bool*)p; @@ -1123,7 +1154,10 @@ int32_t setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STu if (nullList[j]) { colDataSetNULL(pDstCol, rowIndex); } else { - colDataSetVal(pDstCol, rowIndex, pStart, false); + code = colDataSetVal(pDstCol, rowIndex, pStart, false); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } pStart += pDstCol->info.bytes; } @@ -1134,11 +1168,12 @@ int32_t setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STu // This function append the selectivity to subsidiaries function context directly, without fetching data // from intermediate disk based buf page -void appendSelectivityValue(SqlFunctionCtx* pCtx, int32_t rowIndex, int32_t pos) { +int32_t appendSelectivityValue(SqlFunctionCtx* pCtx, int32_t rowIndex, int32_t pos) { if (pCtx->subsidiaries.num <= 0) { - return; + return TSDB_CODE_SUCCESS; } + int32_t code = TSDB_CODE_SUCCESS; for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) { SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j]; @@ -1158,9 +1193,13 @@ void appendSelectivityValue(SqlFunctionCtx* pCtx, int32_t rowIndex, int32_t pos) if (colDataIsNull_s(pSrcCol, rowIndex) == true) { colDataSetNULL(pDstCol, pos); } else { - colDataSetVal(pDstCol, pos, pData, false); + code = colDataSetVal(pDstCol, pos, pData, false); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } } + return code; } void replaceTupleData(STuplePos* pDestPos, STuplePos* pSourcePos) { *pDestPos = *pSourcePos; } @@ -1244,14 +1283,17 @@ bool getStddevFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { return true; } -bool stddevFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { - if (!functionSetup(pCtx, pResultInfo)) { - return false; +int32_t stddevFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { + if (pResultInfo->initialized) { + return TSDB_CODE_SUCCESS; + } + if (TSDB_CODE_SUCCESS != functionSetup(pCtx, pResultInfo)) { + return TSDB_CODE_FUNC_SETUP_ERROR; } SStddevRes* pRes = GET_ROWCELL_INTERBUF(pResultInfo); - memset(pRes, 0, sizeof(SStddevRes)); - return true; + (void)memset(pRes, 0, sizeof(SStddevRes)); + return TSDB_CODE_SUCCESS; } int32_t stddevFunction(SqlFunctionCtx* pCtx) { @@ -1588,16 +1630,19 @@ int32_t stddevPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t resultBytes = getStddevInfoSize(); char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); - memcpy(varDataVal(res), pInfo, resultBytes); + if (NULL == res) { + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(varDataVal(res), pInfo, resultBytes); varDataSetLen(res, resultBytes); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - colDataSetVal(pCol, pBlock->info.rows, res, false); + int32_t code = colDataSetVal(pCol, pBlock->info.rows, res, false); taosMemoryFree(res); - return pResInfo->numOfRes; + return code; } int32_t stddevCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { @@ -1620,16 +1665,19 @@ bool getLeastSQRFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { return true; } -bool leastSQRFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { - if (!functionSetup(pCtx, pResultInfo)) { - return false; +int32_t leastSQRFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { + if (pResultInfo->initialized) { + return TSDB_CODE_SUCCESS; + } + if (TSDB_CODE_SUCCESS != functionSetup(pCtx, pResultInfo)) { + return TSDB_CODE_FUNC_SETUP_ERROR; } SLeastSQRInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo); GET_TYPED_DATA(pInfo->startVal, double, pCtx->param[1].param.nType, &pCtx->param[1].param.i); GET_TYPED_DATA(pInfo->stepVal, double, pCtx->param[2].param.nType, &pCtx->param[2].param.i); - return true; + return TSDB_CODE_SUCCESS; } int32_t leastSQRFunction(SqlFunctionCtx* pCtx) { @@ -1802,7 +1850,7 @@ int32_t leastSQRFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { if (0 == pInfo->num) { colDataSetNULL(pCol, currentRow); - return 0; + return TSDB_CODE_SUCCESS; } double(*param)[3] = pInfo->matrix; @@ -1815,7 +1863,7 @@ int32_t leastSQRFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { if (0 == param00) { colDataSetNULL(pCol, currentRow); - return 0; + return TSDB_CODE_SUCCESS; } // param[0][1] = 0; @@ -1830,19 +1878,19 @@ int32_t leastSQRFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { char interceptBuf[64] = {0}; int n = snprintf(slopBuf, 64, "%.6lf", param02); if (n > LEASTSQUARES_DOUBLE_ITEM_LENGTH) { - snprintf(slopBuf, 64, "%." DOUBLE_PRECISION_DIGITS, param02); + (void)snprintf(slopBuf, 64, "%." DOUBLE_PRECISION_DIGITS, param02); } n = snprintf(interceptBuf, 64, "%.6lf", param12); if (n > LEASTSQUARES_DOUBLE_ITEM_LENGTH) { - snprintf(interceptBuf, 64, "%." DOUBLE_PRECISION_DIGITS, param12); + (void)snprintf(interceptBuf, 64, "%." DOUBLE_PRECISION_DIGITS, param12); } size_t len = snprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE, "{slop:%s, intercept:%s}", slopBuf, interceptBuf); varDataSetLen(buf, len); - colDataSetVal(pCol, currentRow, buf, pResInfo->isNullRes); + int32_t code = colDataSetVal(pCol, currentRow, buf, pResInfo->isNullRes); - return pResInfo->numOfRes; + return code; } int32_t leastSQRCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { @@ -1872,9 +1920,12 @@ bool getPercentileFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { return true; } -bool percentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { - if (!functionSetup(pCtx, pResultInfo)) { - return false; +int32_t percentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { + if (pResultInfo->initialized) { + return TSDB_CODE_SUCCESS; + } + if (TSDB_CODE_SUCCESS != functionSetup(pCtx, pResultInfo)) { + return TSDB_CODE_FUNC_SETUP_ERROR; } // in the first round, get the min-max value of all involved data @@ -1883,10 +1934,11 @@ bool percentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultI SET_DOUBLE_VAL(&pInfo->maxval, -DBL_MAX); pInfo->numOfElems = 0; - return true; + return TSDB_CODE_SUCCESS; } int32_t percentileFunction(SqlFunctionCtx* pCtx) { + int32_t code = TSDB_CODE_SUCCESS; int32_t numOfElems = 0; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); @@ -1905,7 +1957,10 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) { pResInfo->complete = true; return TSDB_CODE_SUCCESS; } else { - pInfo->pMemBucket = tMemBucketCreate(pCol->info.bytes, type, pInfo->minval, pInfo->maxval, pCtx->hasWindowOrGroup); + code = tMemBucketCreate(pCol->info.bytes, type, pInfo->minval, pInfo->maxval, pCtx->hasWindowOrGroup, &pInfo->pMemBucket); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } } @@ -1966,7 +2021,7 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pCol, i); numOfElems += 1; - int32_t code = tMemBucketPut(pInfo->pMemBucket, data, 1); + code = tMemBucketPut(pInfo->pMemBucket, data, 1); if (code != TSDB_CODE_SUCCESS) { tMemBucketDestroy(pInfo->pMemBucket); return code; @@ -2014,10 +2069,13 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); varDataSetLen(buf, len); - colDataSetVal(pCol, pBlock->info.rows, buf, false); + code = colDataSetVal(pCol, pBlock->info.rows, buf, false); + if (code != TSDB_CODE_SUCCESS) { + goto _fin_error; + } tMemBucketDestroy(pMemBucket); - return pResInfo->numOfRes; + return TSDB_CODE_SUCCESS; } else { SVariant* pVal = &pCtx->param[1].param; @@ -2076,9 +2134,12 @@ static void buildTDigestInfo(SAPercentileInfo* pInfo) { pInfo->pTDigest = (TDigest*)((char*)pInfo + sizeof(SAPercentileInfo)); } -bool apercentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { - if (!functionSetup(pCtx, pResultInfo)) { - return false; +int32_t apercentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { + if (pResultInfo->initialized) { + return TSDB_CODE_SUCCESS; + } + if (TSDB_CODE_SUCCESS != functionSetup(pCtx, pResultInfo)) { + return TSDB_CODE_FUNC_SETUP_ERROR; } SAPercentileInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo); @@ -2092,7 +2153,7 @@ bool apercentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResult } else if (pCtx->numOfParams == 3) { pInfo->algo = getApercentileAlgo(varDataVal(pCtx->param[2].param.pz)); if (pInfo->algo == APERCT_ALGO_UNKNOWN) { - return false; + return TSDB_CODE_FUNC_FUNTION_PARA_VALUE; } } @@ -2106,7 +2167,7 @@ bool apercentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResult pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries, pInfo->pHisto, pInfo->pHisto->elems); } - return true; + return TSDB_CODE_SUCCESS; } int32_t apercentileFunction(SqlFunctionCtx* pCtx) { @@ -2186,7 +2247,7 @@ static void apercentileTransferInfo(SAPercentileInfo* pInput, SAPercentileInfo* tdigestAutoFill(pTDigest, COMPRESSION); if (pTDigest->num_centroids <= 0 && pTDigest->num_buffered_pts == 0) { - memcpy(pTDigest, pInput->pTDigest, (size_t)TDIGEST_SIZE(COMPRESSION)); + (void)memcpy(pTDigest, pInput->pTDigest, (size_t)TDIGEST_SIZE(COMPRESSION)); tdigestAutoFill(pTDigest, COMPRESSION); } else { tdigestMerge(pTDigest, pInput->pTDigest); @@ -2205,7 +2266,7 @@ static void apercentileTransferInfo(SAPercentileInfo* pInput, SAPercentileInfo* SHistogramInfo* pHisto = pOutput->pHisto; if (pHisto->numOfElems <= 0) { - memcpy(pHisto, pInput->pHisto, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1)); + (void)memcpy(pHisto, pInput->pHisto, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1)); pHisto->elems = (SHistBin*)((char*)pHisto + sizeof(SHistogramInfo)); qDebug("%s merge histo, total:%" PRId64 ", entry:%d, %p", __FUNCTION__, pHisto->numOfElems, pHisto->numOfEntries, @@ -2216,7 +2277,7 @@ static void apercentileTransferInfo(SAPercentileInfo* pInput, SAPercentileInfo* pHisto->numOfEntries, pInput->pHisto); SHistogramInfo* pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN); - memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN); + (void)memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN); pHisto->elems = (SHistBin*)((char*)pHisto + sizeof(SHistogramInfo)); qDebug("%s merge histo, total:%" PRId64 ", entry:%d, %p", __FUNCTION__, pHisto->numOfElems, pHisto->numOfEntries, @@ -2300,22 +2361,25 @@ int32_t apercentilePartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t resultBytes = getApercentileMaxSize(); char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); + if (NULL == res) { + return TSDB_CODE_OUT_OF_MEMORY; + } if (pInfo->algo == APERCT_ALGO_TDIGEST) { - memcpy(varDataVal(res), pInfo, resultBytes); + (void)memcpy(varDataVal(res), pInfo, resultBytes); varDataSetLen(res, resultBytes); } else { - memcpy(varDataVal(res), pInfo, resultBytes); + (void)memcpy(varDataVal(res), pInfo, resultBytes); varDataSetLen(res, resultBytes); } int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - colDataSetVal(pCol, pBlock->info.rows, res, false); + int32_t code = colDataSetVal(pCol, pBlock->info.rows, res, false); taosMemoryFree(res); - return pResInfo->numOfRes; + return code; } int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { @@ -2447,7 +2511,7 @@ static FORCE_INLINE TSKEY getRowPTs(SColumnInfoData* pTsColInfo, int32_t rowInde return *(TSKEY*)colDataGetData(pTsColInfo, rowIndex); } -static void prepareBuf(SqlFunctionCtx* pCtx) { +static int32_t prepareBuf(SqlFunctionCtx* pCtx) { if (pCtx->subsidiaries.rowLen == 0) { int32_t rowLen = 0; for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) { @@ -2457,7 +2521,11 @@ static void prepareBuf(SqlFunctionCtx* pCtx) { pCtx->subsidiaries.rowLen = rowLen + pCtx->subsidiaries.num * sizeof(bool); pCtx->subsidiaries.buf = taosMemoryMalloc(pCtx->subsidiaries.rowLen); + if (NULL == pCtx->subsidiaries.buf) { + return TSDB_CODE_OUT_OF_MEMORY; + } } + return TSDB_CODE_SUCCESS; } static int32_t firstlastSaveTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SqlFunctionCtx* pCtx, @@ -2485,12 +2553,12 @@ static int32_t doSaveCurrentVal(SqlFunctionCtx* pCtx, int32_t rowIndex, int64_t pInfo->bytes = varDataTLen(pData); } - memcpy(pInfo->buf, pData, pInfo->bytes); + (void)memcpy(pInfo->buf, pData, pInfo->bytes); if (pkData != NULL) { if (IS_VAR_DATA_TYPE(pInfo->pkType)) { pInfo->pkBytes = varDataTLen(pkData); } - memcpy(pInfo->buf + pInfo->bytes, pkData, pInfo->pkBytes); + (void)memcpy(pInfo->buf + pInfo->bytes, pkData, pInfo->pkBytes); pInfo->pkData = pInfo->buf + pInfo->bytes; } @@ -2818,9 +2886,9 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } -static int32_t firstLastTransferInfoImpl(SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst) { +static bool firstLastTransferInfoImpl(SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst) { if (!pInput->hasResult) { - return TSDB_CODE_FAILED; + return false; } __compar_fn_t pkCompareFn = NULL; if (pInput->pkData) { @@ -2830,12 +2898,12 @@ static int32_t firstLastTransferInfoImpl(SFirstLastRes* pInput, SFirstLastRes* p if (isFirst) { if (pInput->ts > pOutput->ts || (pInput->ts == pOutput->ts && pkCompareFn && pkCompareFn(pInput->pkData, pOutput->pkData) > 0)) { - return TSDB_CODE_FAILED; + return false; } } else { if (pInput->ts < pOutput->ts || (pInput->ts == pOutput->ts && pkCompareFn && pkCompareFn(pInput->pkData, pOutput->pkData) > 0)) { - return TSDB_CODE_FAILED; + return false; } } } @@ -2845,25 +2913,24 @@ static int32_t firstLastTransferInfoImpl(SFirstLastRes* pInput, SFirstLastRes* p pOutput->bytes = pInput->bytes; pOutput->pkType = pInput->pkType; - memcpy(pOutput->buf, pInput->buf, pOutput->bytes); + (void)memcpy(pOutput->buf, pInput->buf, pOutput->bytes); if (pInput->pkData) { pOutput->pkBytes = pInput->pkBytes; - memcpy(pOutput->buf + pOutput->bytes, pInput->pkData, pOutput->pkBytes); + (void)memcpy(pOutput->buf + pOutput->bytes, pInput->pkData, pOutput->pkBytes); pOutput->pkData = pOutput->buf + pOutput->bytes; } - return TSDB_CODE_SUCCESS; + return true; } static int32_t firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst, int32_t rowIndex) { - if (TSDB_CODE_SUCCESS == firstLastTransferInfoImpl(pInput, pOutput, isFirst)) { + if (firstLastTransferInfoImpl(pInput, pOutput, isFirst)) { int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pOutput); - if (code != TSDB_CODE_SUCCESS) { + if (TSDB_CODE_SUCCESS != code) { return code; } pOutput->hasResult = true; } - return TSDB_CODE_SUCCESS; } @@ -2923,7 +2990,10 @@ int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pResInfo); - colDataSetVal(pCol, pBlock->info.rows, pRes->buf, pRes->isNull || pResInfo->isNullRes); + code = colDataSetVal(pCol, pBlock->info.rows, pRes->buf, pRes->isNull || pResInfo->isNullRes); + if (TSDB_CODE_SUCCESS != code) { + return code; + } // handle selectivity code = setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); @@ -2941,14 +3011,20 @@ int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { // todo check for failure char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); - memcpy(varDataVal(res), pRes, resultBytes); + if (NULL == res) { + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(varDataVal(res), pRes, resultBytes); varDataSetLen(res, resultBytes); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - colDataSetVal(pCol, pBlock->info.rows, res, false); + code = colDataSetVal(pCol, pBlock->info.rows, res, false); + if (TSDB_CODE_SUCCESS != code) { + return TSDB_CODE_OUT_OF_MEMORY; + } code = setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); taosMemoryFree(res); @@ -2963,10 +3039,7 @@ int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); SFirstLastRes* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); - if (TSDB_CODE_SUCCESS == firstLastTransferInfoImpl(pSBuf, pDBuf, false)) { - pDBuf->hasResult = true; - } - + pDBuf->hasResult = firstLastTransferInfoImpl(pSBuf, pDBuf, false); pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes); pDResInfo->isNullRes &= pSResInfo->isNullRes; return TSDB_CODE_SUCCESS; @@ -2987,7 +3060,7 @@ static int32_t doSaveLastrow(SqlFunctionCtx* pCtx, char* pData, int32_t rowIndex pInfo->bytes = varDataTLen(pData); } - memcpy(pInfo->buf, pData, pInfo->bytes); + (void)memcpy(pInfo->buf, pData, pInfo->bytes); } if (pCtx->hasPrimaryKey) { @@ -2995,7 +3068,7 @@ static int32_t doSaveLastrow(SqlFunctionCtx* pCtx, char* pData, int32_t rowIndex if (IS_VAR_DATA_TYPE(pInfo->pkType)) { pInfo->pkBytes = varDataTLen(pkData); } - memcpy(pInfo->buf + pInfo->bytes, pkData, pInfo->pkBytes); + (void)memcpy(pInfo->buf + pInfo->bytes, pkData, pInfo->pkBytes); pInfo->pkData = pInfo->buf + pInfo->bytes; } pInfo->ts = cts; @@ -3101,11 +3174,13 @@ bool getDiffFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { return true; } -bool diffFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { - if (!functionSetup(pCtx, pResInfo)) { - return false; +int32_t diffFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { + if (pResInfo->initialized) { + return TSDB_CODE_SUCCESS; + } + if (TSDB_CODE_SUCCESS != functionSetup(pCtx, pResInfo)) { + return TSDB_CODE_FUNC_SETUP_ERROR; } - SDiffInfo* pDiffInfo = GET_ROWCELL_INTERBUF(pResInfo); pDiffInfo->hasPrev = false; pDiffInfo->isFirstRow = true; @@ -3116,7 +3191,7 @@ bool diffFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { } else { pDiffInfo->ignoreOption = 0; } - return true; + return TSDB_CODE_SUCCESS; } static int32_t doSetPrevVal(SDiffInfo* pDiffInfo, int32_t type, const char* pv, int64_t ts) { @@ -3155,7 +3230,7 @@ static int32_t doSetPrevVal(SDiffInfo* pDiffInfo, int32_t type, const char* pv, return TSDB_CODE_SUCCESS; } -static int32_t diffIsNegtive(SDiffInfo* pDiffInfo, int32_t type, const char* pv) { +static bool diffIsNegtive(SDiffInfo* pDiffInfo, int32_t type, const char* pv) { switch (type) { case TSDB_DATA_TYPE_UINT: { int64_t v = *(uint32_t*)pv; @@ -3382,14 +3457,17 @@ int32_t setDoDiffResult(SqlFunctionCtx* pCtx, SFuncInputRow* pRow, int32_t pos) SColumnInfoData* pInputCol = pInput->pData[0]; int8_t inputType = pInputCol->info.type; SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; - + int32_t code = TSDB_CODE_SUCCESS; if (pRow->isDataNull) { colDataSetNull_f_s(pOutput, pos); pOutput->hasNull = true; // handle selectivity if (pCtx->subsidiaries.num > 0) { - appendSelectivityCols(pCtx, pRow->block, pRow->rowIndex, pos); + code = appendSelectivityCols(pCtx, pRow->block, pRow->rowIndex, pos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } return TSDB_CODE_SUCCESS; } @@ -3399,13 +3477,16 @@ int32_t setDoDiffResult(SqlFunctionCtx* pCtx, SFuncInputRow* pRow, int32_t pos) if (pRow->ts == pDiffInfo->prevTs) { return TSDB_CODE_FUNC_DUP_TIMESTAMP; } - int32_t code = doHandleDiff(pDiffInfo, inputType, pv, pOutput, pos, pRow->ts); + code = doHandleDiff(pDiffInfo, inputType, pv, pOutput, pos, pRow->ts); if (code != TSDB_CODE_SUCCESS) { return code; } // handle selectivity if (pCtx->subsidiaries.num > 0) { - appendSelectivityCols(pCtx, pRow->block, pRow->rowIndex, pos); + code = appendSelectivityCols(pCtx, pRow->block, pRow->rowIndex, pos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } return TSDB_CODE_SUCCESS; @@ -3424,6 +3505,9 @@ int32_t diffFunctionByRow(SArray* pCtxArray) { int32_t numOfElems = 0; SArray* pRows = taosArrayInit_s(sizeof(SFuncInputRow), diffColNum); + if (NULL == pRows) { + return TSDB_CODE_OUT_OF_MEMORY; + } bool keepNull = false; for (int i = 0; i < diffColNum; ++i) { @@ -3439,12 +3523,24 @@ int32_t diffFunctionByRow(SArray* pCtxArray) { SqlFunctionCtx* pCtx0 = *(SqlFunctionCtx**)taosArrayGet(pCtxArray, 0); SFuncInputRow* pRow0 = (SFuncInputRow*)taosArrayGet(pRows, 0); int32_t startOffset = pCtx0->offset; - while (funcInputGetNextRow(pCtx0, pRow0)) { + bool result = false; + while (1) { + code = funcInputGetNextRow(pCtx0, pRow0, &result); + if (TSDB_CODE_SUCCESS != code) { + goto _exit; + } + if (!result) { + break; + } bool hasNotNullValue = !diffResultIsNull(pCtx0, pRow0); for (int i = 1; i < diffColNum; ++i) { SqlFunctionCtx* pCtx = *(SqlFunctionCtx**)taosArrayGet(pCtxArray, i); SFuncInputRow* pRow = (SFuncInputRow*)taosArrayGet(pRows, i); - if(!funcInputGetNextRow(pCtx, pRow)) { + code = funcInputGetNextRow(pCtx, pRow, &result); + if (TSDB_CODE_SUCCESS != code) { + goto _exit; + } + if (!result) { // rows are not equal code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; goto _exit; @@ -3497,9 +3593,12 @@ bool getTopBotFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { return true; } -bool topBotFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { - if (!functionSetup(pCtx, pResInfo)) { - return false; +int32_t topBotFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { + if (pResInfo->initialized) { + return TSDB_CODE_SUCCESS; + } + if (TSDB_CODE_SUCCESS != functionSetup(pCtx, pResInfo)) { + return TSDB_CODE_FUNC_SETUP_ERROR; } STopBotRes* pRes = GET_ROWCELL_INTERBUF(pResInfo); @@ -3509,7 +3608,7 @@ bool topBotFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { pRes->nullTupleSaved = false; pRes->nullTuplePos.pageId = -1; - return true; + return TSDB_CODE_SUCCESS; } static STopBotRes* getTopBotOutputInfo(SqlFunctionCtx* pCtx) { @@ -3523,7 +3622,7 @@ static STopBotRes* getTopBotOutputInfo(SqlFunctionCtx* pCtx) { static int32_t doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type, uint64_t uid, SResultRowEntryInfo* pEntryInfo, bool isTopQuery); -static void addResult(SqlFunctionCtx* pCtx, STopBotResItem* pSourceItem, int16_t type, bool isTopQuery); +static int32_t addResult(SqlFunctionCtx* pCtx, STopBotResItem* pSourceItem, int16_t type, bool isTopQuery); int32_t topFunction(SqlFunctionCtx* pCtx) { int32_t numOfElems = 0; @@ -3630,8 +3729,10 @@ static int32_t topBotResComparFn(const void* p1, const void* p2, const void* par int32_t doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type, uint64_t uid, SResultRowEntryInfo* pEntryInfo, bool isTopQuery) { STopBotRes* pRes = getTopBotOutputInfo(pCtx); + int32_t code = TSDB_CODE_SUCCESS; SVariant val = {0}; + // TODO(smj) : this func need err code taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); STopBotResItem* pItems = pRes->pItems; @@ -3644,7 +3745,7 @@ int32_t doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSD // save the data of this tuple if (pCtx->subsidiaries.num > 0) { - int32_t code = saveTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos); + code = saveTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3655,8 +3756,11 @@ int32_t doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSD #endif // allocate the buffer and keep the data of this row into the new allocated buffer pEntryInfo->numOfRes++; - taosheapsort((void*)pItems, sizeof(STopBotResItem), pEntryInfo->numOfRes, (const void*)&type, topBotResComparFn, - !isTopQuery); + code = taosheapsort((void*)pItems, sizeof(STopBotResItem), pEntryInfo->numOfRes, (const void*)&type, + topBotResComparFn, !isTopQuery); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } else { // replace the minimum value in the result if ((isTopQuery && ((IS_SIGNED_NUMERIC_TYPE(type) && val.i > pItems[0].v.i) || (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u > pItems[0].v.u) || @@ -3673,7 +3777,7 @@ int32_t doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSD // save the data of this tuple by over writing the old data if (pCtx->subsidiaries.num > 0) { - int32_t code = updateTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos); + code = updateTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3681,8 +3785,11 @@ int32_t doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSD #ifdef BUF_PAGE_DEBUG qDebug("page_copyTuple pageId:%d, offset:%d", pItem->tuplePos.pageId, pItem->tuplePos.offset); #endif - taosheapadjust((void*)pItems, sizeof(STopBotResItem), 0, pEntryInfo->numOfRes - 1, (const void*)&type, + code = taosheapadjust((void*)pItems, sizeof(STopBotResItem), 0, pEntryInfo->numOfRes - 1, (const void*)&type, topBotResComparFn, NULL, !isTopQuery); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } @@ -3721,9 +3828,9 @@ void* serializeTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SSubsid char* p = colDataGetData(pCol, rowIndex); if (IS_VAR_DATA_TYPE(pCol->info.type)) { - memcpy(pStart + offset, p, (pCol->info.type == TSDB_DATA_TYPE_JSON) ? getJsonValueLen(p) : varDataTLen(p)); + (void)memcpy(pStart + offset, p, (pCol->info.type == TSDB_DATA_TYPE_JSON) ? getJsonValueLen(p) : varDataTLen(p)); } else { - memcpy(pStart + offset, p, pCol->info.bytes); + (void)memcpy(pStart + offset, p, pCol->info.bytes); } offset += pCol->info.bytes; @@ -3761,7 +3868,7 @@ static int32_t doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, } p = (STuplePos){.pageId = pHandle->currentPage, .offset = pPage->num}; - memcpy(pPage->data + pPage->num, pBuf, length); + (void)memcpy(pPage->data + pPage->num, pBuf, length); pPage->num += length; setBufPageDirty(pPage, true); @@ -3777,7 +3884,10 @@ static int32_t doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, } int32_t saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { - prepareBuf(pCtx); + int32_t code = prepareBuf(pCtx); + if (TSDB_CODE_SUCCESS != code) { + return code; + } SWinKey key = {0}; if (pCtx->saveHandle.pBuf == NULL) { @@ -3797,42 +3907,52 @@ static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf if (pPage == NULL) { return terrno; } - memcpy(pPage->data + pPos->offset, pBuf, length); + (void)memcpy(pPage->data + pPos->offset, pBuf, length); setBufPageDirty(pPage, true); releaseBufPage(pHandle->pBuf, pPage); } else { - pStore->streamStateFuncPut(pHandle->pState, &pPos->streamTupleKey, pBuf, length); + int32_t code = pStore->streamStateFuncPut(pHandle->pState, &pPos->streamTupleKey, pBuf, length); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } return TSDB_CODE_SUCCESS; } int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { - prepareBuf(pCtx); + int32_t code = prepareBuf(pCtx); + if (TSDB_CODE_SUCCESS != code) { + return code; + } char* buf = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf); return doUpdateTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, pPos, pCtx->pStore); } -static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPos, SFunctionStateStore* pStore) { +static int32_t doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPos, SFunctionStateStore* pStore, char** value) { if (pHandle->pBuf != NULL) { SFilePage* pPage = getBufPage(pHandle->pBuf, pPos->pageId); if (pPage == NULL) { - return NULL; + *value = NULL; + return terrno; } - char* p = pPage->data + pPos->offset; + *value = pPage->data + pPos->offset; releaseBufPage(pHandle->pBuf, pPage); - return p; + return TSDB_CODE_SUCCESS; } else { - void* value = NULL; + *value = NULL; int32_t vLen; - pStore->streamStateFuncGet(pHandle->pState, &pPos->streamTupleKey, &value, &vLen); - return (char*)value; + int32_t code = pStore->streamStateFuncGet(pHandle->pState, &pPos->streamTupleKey, (void **)(value), &vLen); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + return TSDB_CODE_SUCCESS; } } -const char* loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos) { - return doLoadTupleData(&pCtx->saveHandle, pPos, pCtx->pStore); +int32_t loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos, char** value) { + return doLoadTupleData(&pCtx->saveHandle, pPos, pCtx->pStore, value); } int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { @@ -3855,22 +3975,29 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { } for (int32_t i = 0; i < pEntryInfo->numOfRes; ++i) { STopBotResItem* pItem = &pRes->pItems[i]; - colDataSetVal(pCol, currentRow, (const char*)&pItem->v.i, false); + code = colDataSetVal(pCol, currentRow, (const char*)&pItem->v.i, false); + if (TSDB_CODE_SUCCESS != code) { + return code; + } #ifdef BUF_PAGE_DEBUG qDebug("page_finalize i:%d,item:%p,pageId:%d, offset:%d\n", i, pItem, pItem->tuplePos.pageId, pItem->tuplePos.offset); #endif code = setSelectivityValue(pCtx, pBlock, &pRes->pItems[i].tuplePos, currentRow); + if (TSDB_CODE_SUCCESS != code) { + return code; + } currentRow += 1; } return code; } -void addResult(SqlFunctionCtx* pCtx, STopBotResItem* pSourceItem, int16_t type, bool isTopQuery) { +int32_t addResult(SqlFunctionCtx* pCtx, STopBotResItem* pSourceItem, int16_t type, bool isTopQuery) { SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); STopBotRes* pRes = getTopBotOutputInfo(pCtx); STopBotResItem* pItems = pRes->pItems; + int32_t code = TSDB_CODE_SUCCESS; // not full yet if (pEntryInfo->numOfRes < pRes->maxSize) { @@ -3880,8 +4007,11 @@ void addResult(SqlFunctionCtx* pCtx, STopBotResItem* pSourceItem, int16_t type, pItem->tuplePos.pageId = -1; replaceTupleData(&pItem->tuplePos, &pSourceItem->tuplePos); pEntryInfo->numOfRes++; - taosheapsort((void*)pItems, sizeof(STopBotResItem), pEntryInfo->numOfRes, (const void*)&type, topBotResComparFn, - !isTopQuery); + code = taosheapsort((void*)pItems, sizeof(STopBotResItem), pEntryInfo->numOfRes, (const void*)&type, + topBotResComparFn, !isTopQuery); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } else { // replace the minimum value in the result if ((isTopQuery && ((IS_SIGNED_NUMERIC_TYPE(type) && pSourceItem->v.i > pItems[0].v.i) || (IS_UNSIGNED_NUMERIC_TYPE(type) && pSourceItem->v.u > pItems[0].v.u) || @@ -3898,18 +4028,26 @@ void addResult(SqlFunctionCtx* pCtx, STopBotResItem* pSourceItem, int16_t type, // save the data of this tuple by over writing the old data replaceTupleData(&pItem->tuplePos, &pSourceItem->tuplePos); - taosheapadjust((void*)pItems, sizeof(STopBotResItem), 0, pEntryInfo->numOfRes - 1, (const void*)&type, - topBotResComparFn, NULL, !isTopQuery); + code = taosheapadjust((void*)pItems, sizeof(STopBotResItem), 0, pEntryInfo->numOfRes - 1, (const void*)&type, + topBotResComparFn, NULL, !isTopQuery); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } } + return code; } int32_t topCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); STopBotRes* pSBuf = getTopBotOutputInfo(pSourceCtx); int16_t type = pSBuf->type; + int32_t code = TSDB_CODE_SUCCESS; for (int32_t i = 0; i < pSResInfo->numOfRes; i++) { - addResult(pDestCtx, pSBuf->pItems + i, type, true); + code = addResult(pDestCtx, pSBuf->pItems + i, type, true); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } return TSDB_CODE_SUCCESS; } @@ -3918,8 +4056,12 @@ int32_t bottomCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); STopBotRes* pSBuf = getTopBotOutputInfo(pSourceCtx); int16_t type = pSBuf->type; + int32_t code = TSDB_CODE_SUCCESS; for (int32_t i = 0; i < pSResInfo->numOfRes; i++) { - addResult(pDestCtx, pSBuf->pItems + i, type, false); + code = addResult(pDestCtx, pSBuf->pItems + i, type, false); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } return TSDB_CODE_SUCCESS; } @@ -3931,16 +4073,19 @@ bool getSpreadFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { return true; } -bool spreadFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { - if (!functionSetup(pCtx, pResultInfo)) { - return false; +int32_t spreadFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { + if (pResultInfo->initialized) { + return TSDB_CODE_SUCCESS; + } + if (TSDB_CODE_SUCCESS != functionSetup(pCtx, pResultInfo)) { + return TSDB_CODE_FUNC_SETUP_ERROR; } SSpreadInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo); SET_DOUBLE_VAL(&pInfo->min, DBL_MAX); SET_DOUBLE_VAL(&pInfo->max, -DBL_MAX); pInfo->hasResult = false; - return true; + return TSDB_CODE_SUCCESS; } int32_t spreadFunction(SqlFunctionCtx* pCtx) { @@ -4073,16 +4218,22 @@ int32_t spreadPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t resultBytes = getSpreadInfoSize(); char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); - memcpy(varDataVal(res), pInfo, resultBytes); + if (NULL == res) { + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(varDataVal(res), pInfo, resultBytes); varDataSetLen(res, resultBytes); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - colDataSetVal(pCol, pBlock->info.rows, res, false); + int32_t code = colDataSetVal(pCol, pBlock->info.rows, res, false); + if (TSDB_CODE_SUCCESS != code) { + return code; + } taosMemoryFree(res); - return pResInfo->numOfRes; + return TSDB_CODE_SUCCESS; } int32_t spreadCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { @@ -4104,9 +4255,12 @@ bool getElapsedFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { return true; } -bool elapsedFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { - if (!functionSetup(pCtx, pResultInfo)) { - return false; +int32_t elapsedFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { + if (pResultInfo->initialized) { + return TSDB_CODE_SUCCESS; + } + if (TSDB_CODE_SUCCESS != functionSetup(pCtx, pResultInfo)) { + return TSDB_CODE_FUNC_SETUP_ERROR; } SElapsedInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo); @@ -4120,7 +4274,7 @@ bool elapsedFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo pInfo->timeUnit = 1; } - return true; + return TSDB_CODE_SUCCESS; } int32_t elapsedFunction(SqlFunctionCtx* pCtx) { @@ -4248,16 +4402,21 @@ int32_t elapsedPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t resultBytes = getElapsedInfoSize(); char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); - memcpy(varDataVal(res), pInfo, resultBytes); + if (NULL == res) { + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(varDataVal(res), pInfo, resultBytes); varDataSetLen(res, resultBytes); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - colDataSetVal(pCol, pBlock->info.rows, res, false); - + int32_t code = colDataSetVal(pCol, pBlock->info.rows, res, false); + if (TSDB_CODE_SUCCESS != code) { + return code; + } taosMemoryFree(res); - return pResInfo->numOfRes; + return code; } int32_t elapsedCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { @@ -4297,7 +4456,7 @@ static int8_t getHistogramBinType(char* binTypeStr) { return binType; } -static bool getHistogramBinDesc(SHistoFuncInfo* pInfo, char* binDescStr, int8_t binType, bool normalized) { +static int32_t getHistogramBinDesc(SHistoFuncInfo* pInfo, char* binDescStr, int8_t binType, bool normalized) { cJSON* binDesc = cJSON_Parse(binDescStr); int32_t numOfBins; double* intervals; @@ -4306,7 +4465,7 @@ static bool getHistogramBinDesc(SHistoFuncInfo* pInfo, char* binDescStr, int8_t int32_t startIndex; if (numOfParams != 4) { cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } cJSON* start = cJSON_GetObjectItem(binDesc, "start"); @@ -4317,18 +4476,18 @@ static bool getHistogramBinDesc(SHistoFuncInfo* pInfo, char* binDescStr, int8_t if (!cJSON_IsNumber(start) || !cJSON_IsNumber(count) || !cJSON_IsBool(infinity)) { cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } if (count->valueint <= 0 || count->valueint > 1000) { // limit count to 1000 cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } if (isinf(start->valuedouble) || (width != NULL && isinf(width->valuedouble)) || (factor != NULL && isinf(factor->valuedouble)) || (count != NULL && isinf(count->valuedouble))) { cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } int32_t counter = (int32_t)count->valueint; @@ -4341,19 +4500,24 @@ static bool getHistogramBinDesc(SHistoFuncInfo* pInfo, char* binDescStr, int8_t } intervals = taosMemoryCalloc(numOfBins, sizeof(double)); + if (NULL == intervals) { + cJSON_Delete(binDesc); + qError("histogram function out of memory"); + return TSDB_CODE_OUT_OF_MEMORY; + } if (cJSON_IsNumber(width) && factor == NULL && binType == LINEAR_BIN) { // linear bin process if (width->valuedouble == 0) { taosMemoryFree(intervals); cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } for (int i = 0; i < counter + 1; ++i) { intervals[startIndex] = start->valuedouble + i * width->valuedouble; if (isinf(intervals[startIndex])) { taosMemoryFree(intervals); cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } startIndex++; } @@ -4362,26 +4526,26 @@ static bool getHistogramBinDesc(SHistoFuncInfo* pInfo, char* binDescStr, int8_t if (start->valuedouble == 0) { taosMemoryFree(intervals); cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } if (factor->valuedouble < 0 || factor->valuedouble == 0 || factor->valuedouble == 1) { taosMemoryFree(intervals); cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } for (int i = 0; i < counter + 1; ++i) { intervals[startIndex] = start->valuedouble * pow(factor->valuedouble, i * 1.0); if (isinf(intervals[startIndex])) { taosMemoryFree(intervals); cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } startIndex++; } } else { taosMemoryFree(intervals); cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } if (infinity->valueint == true) { @@ -4389,7 +4553,7 @@ static bool getHistogramBinDesc(SHistoFuncInfo* pInfo, char* binDescStr, int8_t intervals[numOfBins - 1] = INFINITY; // in case of desc bin orders, -inf/inf should be swapped if (numOfBins < 4) { - return false; + return TSDB_CODE_FAILED; } if (intervals[1] > intervals[numOfBins - 2]) { TSWAP(intervals[0], intervals[numOfBins - 1]); @@ -4398,15 +4562,20 @@ static bool getHistogramBinDesc(SHistoFuncInfo* pInfo, char* binDescStr, int8_t } else if (cJSON_IsArray(binDesc)) { /* user input bins */ if (binType != USER_INPUT_BIN) { cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } numOfBins = cJSON_GetArraySize(binDesc); intervals = taosMemoryCalloc(numOfBins, sizeof(double)); + if (NULL == intervals) { + cJSON_Delete(binDesc); + qError("histogram function out of memory"); + return TSDB_CODE_OUT_OF_MEMORY; + } cJSON* bin = binDesc->child; if (bin == NULL) { taosMemoryFree(intervals); cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } int i = 0; while (bin) { @@ -4414,19 +4583,19 @@ static bool getHistogramBinDesc(SHistoFuncInfo* pInfo, char* binDescStr, int8_t if (!cJSON_IsNumber(bin)) { taosMemoryFree(intervals); cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } if (i != 0 && intervals[i] <= intervals[i - 1]) { taosMemoryFree(intervals); cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } bin = bin->next; i++; } } else { cJSON_Delete(binDesc); - return false; + return TSDB_CODE_FAILED; } pInfo->numOfBins = numOfBins - 1; @@ -4440,12 +4609,15 @@ static bool getHistogramBinDesc(SHistoFuncInfo* pInfo, char* binDescStr, int8_t taosMemoryFree(intervals); cJSON_Delete(binDesc); - return true; + return TSDB_CODE_SUCCESS; } -bool histogramFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { - if (!functionSetup(pCtx, pResultInfo)) { - return false; +int32_t histogramFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { + if (pResultInfo->initialized) { + return TSDB_CODE_SUCCESS; + } + if (TSDB_CODE_SUCCESS != functionSetup(pCtx, pResultInfo)) { + return TSDB_CODE_FUNC_SETUP_ERROR; } SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo); @@ -4458,21 +4630,22 @@ bool histogramFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultIn taosMemoryFree(binTypeStr); if (binType == UNKNOWN_BIN) { - return false; + return TSDB_CODE_FUNC_FUNTION_PARA_VALUE; } char* binDesc = strndup(varDataVal(pCtx->param[2].param.pz), varDataLen(pCtx->param[2].param.pz)); int64_t normalized = pCtx->param[3].param.i; if (normalized != 0 && normalized != 1) { taosMemoryFree(binDesc); - return false; + return TSDB_CODE_FUNC_FUNTION_PARA_VALUE; } - if (!getHistogramBinDesc(pInfo, binDesc, binType, (bool)normalized)) { + int32_t code = getHistogramBinDesc(pInfo, binDesc, binType, (bool)normalized); + if (TSDB_CODE_SUCCESS != code) { taosMemoryFree(binDesc); - return false; + return code; } taosMemoryFree(binDesc); - return true; + return TSDB_CODE_SUCCESS; } static int32_t histogramFunctionImpl(SqlFunctionCtx* pCtx, bool isPartial) { @@ -4556,6 +4729,7 @@ int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + int32_t code = TSDB_CODE_SUCCESS; int32_t currentRow = pBlock->info.rows; @@ -4580,11 +4754,14 @@ int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { pInfo->bins[i].upper, pInfo->bins[i].percentage); } varDataSetLen(buf, len); - colDataSetVal(pCol, currentRow, buf, false); + code = colDataSetVal(pCol, currentRow, buf, false); + if (TSDB_CODE_SUCCESS != code) { + return code; + } currentRow++; } - return pResInfo->numOfRes; + return code; } int32_t histogramPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { @@ -4593,16 +4770,19 @@ int32_t histogramPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t resultBytes = getHistogramInfoSize(); char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); - memcpy(varDataVal(res), pInfo, resultBytes); + if (NULL == res) { + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(varDataVal(res), pInfo, resultBytes); varDataSetLen(res, resultBytes); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - colDataSetVal(pCol, pBlock->info.rows, res, false); + int32_t code = colDataSetVal(pCol, pBlock->info.rows, res, false); taosMemoryFree(res); - return pResInfo->numOfRes; + return code; } int32_t histogramCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { @@ -4818,16 +4998,19 @@ int32_t hllPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t resultBytes = getHLLInfoSize(); char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); - memcpy(varDataVal(res), pInfo, resultBytes); + if (NULL == res) { + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(varDataVal(res), pInfo, resultBytes); varDataSetLen(res, resultBytes); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - colDataSetVal(pCol, pBlock->info.rows, res, false); + int32_t code = colDataSetVal(pCol, pBlock->info.rows, res, false); taosMemoryFree(res); - return pResInfo->numOfRes; + return code; } int32_t hllCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { @@ -4930,6 +5113,7 @@ static bool checkStateOp(int8_t op, SColumnInfoData* pCol, int32_t index, SVaria } int32_t stateCountFunction(SqlFunctionCtx* pCtx) { + int32_t code = TSDB_CODE_SUCCESS; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SStateInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -4960,7 +5144,10 @@ int32_t stateCountFunction(SqlFunctionCtx* pCtx) { colDataSetNULL(pOutput, i); // handle selectivity if (pCtx->subsidiaries.num > 0) { - appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1); + code = appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } continue; } @@ -4973,11 +5160,17 @@ int32_t stateCountFunction(SqlFunctionCtx* pCtx) { } else { pInfo->count = 0; } - colDataSetVal(pOutput, pCtx->offset + numOfElems - 1, (char*)&output, false); + code = colDataSetVal(pOutput, pCtx->offset + numOfElems - 1, (char*)&output, false); + if (TSDB_CODE_SUCCESS != code) { + return code; + } // handle selectivity if (pCtx->subsidiaries.num > 0) { - appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1); + code = appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } } @@ -4986,6 +5179,7 @@ int32_t stateCountFunction(SqlFunctionCtx* pCtx) { } int32_t stateDurationFunction(SqlFunctionCtx* pCtx) { + int32_t code = TSDB_CODE_SUCCESS; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SStateInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -5022,7 +5216,10 @@ int32_t stateDurationFunction(SqlFunctionCtx* pCtx) { colDataSetNULL(pOutput, i); // handle selectivity if (pCtx->subsidiaries.num > 0) { - appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1); + code = appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } continue; } @@ -5039,11 +5236,17 @@ int32_t stateDurationFunction(SqlFunctionCtx* pCtx) { } else { pInfo->durationStart = 0; } - colDataSetVal(pOutput, pCtx->offset + numOfElems - 1, (char*)&output, false); + code = colDataSetVal(pOutput, pCtx->offset + numOfElems - 1, (char*)&output, false); + if (TSDB_CODE_SUCCESS != code) { + return code; + } // handle selectivity if (pCtx->subsidiaries.num > 0) { - appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1); + code = appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } } @@ -5057,6 +5260,7 @@ bool getCsumFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { } int32_t csumFunction(SqlFunctionCtx* pCtx) { + int32_t code = TSDB_CODE_SUCCESS; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SSumRes* pSumRes = GET_ROWCELL_INTERBUF(pResInfo); @@ -5088,12 +5292,18 @@ int32_t csumFunction(SqlFunctionCtx* pCtx) { int64_t v; GET_TYPED_DATA(v, int64_t, type, data); pSumRes->isum += v; - colDataSetVal(pOutput, pos, (char*)&pSumRes->isum, false); + code = colDataSetVal(pOutput, pos, (char*)&pSumRes->isum, false); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { uint64_t v; GET_TYPED_DATA(v, uint64_t, type, data); pSumRes->usum += v; - colDataSetVal(pOutput, pos, (char*)&pSumRes->usum, false); + code = colDataSetVal(pOutput, pos, (char*)&pSumRes->usum, false); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } else if (IS_FLOAT_TYPE(type)) { double v; GET_TYPED_DATA(v, double, type, data); @@ -5102,13 +5312,19 @@ int32_t csumFunction(SqlFunctionCtx* pCtx) { if (isinf(pSumRes->dsum) || isnan(pSumRes->dsum)) { colDataSetNULL(pOutput, pos); } else { - colDataSetVal(pOutput, pos, (char*)&pSumRes->dsum, false); + code = colDataSetVal(pOutput, pos, (char*)&pSumRes->dsum, false); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } } // handle selectivity if (pCtx->subsidiaries.num > 0) { - appendSelectivityValue(pCtx, i, pos); + code = appendSelectivityValue(pCtx, i, pos); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } numOfElems++; @@ -5123,9 +5339,12 @@ bool getMavgFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { return true; } -bool mavgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { - if (!functionSetup(pCtx, pResultInfo)) { - return false; +int32_t mavgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { + if (pResultInfo->initialized) { + return TSDB_CODE_SUCCESS; + } + if (TSDB_CODE_SUCCESS != functionSetup(pCtx, pResultInfo)) { + return TSDB_CODE_FUNC_SETUP_ERROR; } SMavgInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo); @@ -5135,14 +5354,15 @@ bool mavgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { pInfo->isPrevTsSet = false; pInfo->numOfPoints = pCtx->param[1].param.i; if (pInfo->numOfPoints < 1 || pInfo->numOfPoints > MAVG_MAX_POINTS_NUM) { - return false; + return TSDB_CODE_FUNC_FUNTION_PARA_VALUE; } pInfo->pointsMeet = false; - return true; + return TSDB_CODE_SUCCESS; } int32_t mavgFunction(SqlFunctionCtx* pCtx) { + int32_t code = TSDB_CODE_SUCCESS; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SMavgInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -5191,12 +5411,18 @@ int32_t mavgFunction(SqlFunctionCtx* pCtx) { if (isinf(result) || isnan(result)) { colDataSetNULL(pOutput, pos); } else { - colDataSetVal(pOutput, pos, (char*)&result, false); + code = colDataSetVal(pOutput, pos, (char*)&result, false); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } // handle selectivity if (pCtx->subsidiaries.num > 0) { - appendSelectivityValue(pCtx, i, pos); + code = appendSelectivityValue(pCtx, i, pos); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } numOfElems++; @@ -5230,9 +5456,12 @@ bool getSampleFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { return true; } -bool sampleFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { - if (!functionSetup(pCtx, pResultInfo)) { - return false; +int32_t sampleFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { + if (pResultInfo->initialized) { + return TSDB_CODE_SUCCESS; + } + if (TSDB_CODE_SUCCESS != functionSetup(pCtx, pResultInfo)) { + return TSDB_CODE_FUNC_SETUP_ERROR; } taosSeedRand(taosSafeRand()); @@ -5248,7 +5477,7 @@ bool sampleFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) pInfo->data = (char*)pInfo + sizeof(SSampleInfo); pInfo->tuplePos = (STuplePos*)((char*)pInfo + sizeof(SSampleInfo) + pInfo->samples * pInfo->colBytes); - return true; + return TSDB_CODE_SUCCESS; } static void sampleAssignResult(SSampleInfo* pInfo, char* data, int32_t index) { @@ -5330,8 +5559,14 @@ int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return code; } for (int32_t i = 0; i < pInfo->numSampled; ++i) { - colDataSetVal(pCol, currentRow + i, pInfo->data + i * pInfo->colBytes, false); + code = colDataSetVal(pCol, currentRow + i, pInfo->data + i * pInfo->colBytes, false); + if (TSDB_CODE_SUCCESS != code) { + return code; + } code = setSelectivityValue(pCtx, pBlock, &pInfo->tuplePos[i], currentRow + i); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } return code; @@ -5347,7 +5582,7 @@ bool getTailFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { return true; } -bool tailFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { +int32_t tailFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { #if 0 if (!functionSetup(pCtx, pResultInfo)) { return false; @@ -5378,7 +5613,7 @@ bool tailFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { } #endif - return true; + return TSDB_CODE_SUCCESS; } static void tailAssignResult(STailItem* pItem, char* data, int32_t colBytes, TSKEY ts, bool isNull) { @@ -5483,7 +5718,7 @@ bool getUniqueFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { return true; } -bool uniqueFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { +int32_t uniqueFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { #if 0 if (!functionSetup(pCtx, pResInfo)) { return false; @@ -5499,7 +5734,7 @@ bool uniqueFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { pInfo->pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); } #endif - return true; + return TSDB_CODE_SUCCESS; } #if 0 @@ -5580,9 +5815,12 @@ bool getModeFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { return true; } -bool modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { - if (!functionSetup(pCtx, pResInfo)) { - return false; +int32_t modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { + if (pResInfo->initialized) { + return TSDB_CODE_SUCCESS; + } + if (TSDB_CODE_SUCCESS != functionSetup(pCtx, pResInfo)) { + return TSDB_CODE_FUNC_SETUP_ERROR; } SModeInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -5592,13 +5830,19 @@ bool modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { taosHashClear(pInfo->pHash); } else { pInfo->pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (NULL == pInfo->pHash) { + return TSDB_CODE_OUT_OF_MEMORY; + } } pInfo->nullTupleSaved = false; pInfo->nullTuplePos.pageId = -1; pInfo->buf = taosMemoryMalloc(pInfo->colBytes); + if (NULL == pInfo->buf) { + return TSDB_CODE_OUT_OF_MEMORY; + } - return true; + return TSDB_CODE_SUCCESS; } static void modeFunctionCleanup(SModeInfo * pInfo) { @@ -5608,9 +5852,9 @@ static void modeFunctionCleanup(SModeInfo * pInfo) { static int32_t saveModeTupleData(SqlFunctionCtx* pCtx, char* data, SModeInfo *pInfo, STuplePos* pPos) { if (IS_VAR_DATA_TYPE(pInfo->colType)) { - memcpy(pInfo->buf, data, varDataTLen(data)); + (void)memcpy(pInfo->buf, data, varDataTLen(data)); } else { - memcpy(pInfo->buf, data, pInfo->colBytes); + (void)memcpy(pInfo->buf, data, pInfo->colBytes); } return doSaveTupleData(&pCtx->saveHandle, pInfo->buf, pInfo->colBytes, NULL, pPos, pCtx->pStore); @@ -5638,7 +5882,10 @@ static int32_t doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCt } } - taosHashPut(pInfo->pHash, data, hashKeyBytes, &item, sizeof(SModeItem)); + code = taosHashPut(pInfo->pHash, data, hashKeyBytes, &item, sizeof(SModeItem)); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } else { pHashItem->count += 1; if (pCtx->subsidiaries.num > 0) { @@ -5715,8 +5962,9 @@ int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { } if (maxCount != 0) { - const char* pData = loadTupleData(pCtx, &resDataPos); - if (pData == NULL) { + char* pData = NULL; + code = loadTupleData(pCtx, &resDataPos, &pData); + if (pData == NULL || TSDB_CODE_SUCCESS != code) { code = terrno = TSDB_CODE_NOT_FOUND; qError("Load tuple data failed since %s, groupId:%" PRIu64 ", ts:%" PRId64, terrstr(), resDataPos.streamTupleKey.groupId, resDataPos.streamTupleKey.ts); @@ -5724,7 +5972,10 @@ int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return code; } - colDataSetVal(pCol, currentRow, pData, false); + code = colDataSetVal(pCol, currentRow, pData, false); + if (TSDB_CODE_SUCCESS != code) { + return code; + } code = setSelectivityValue(pCtx, pBlock, &resTuplePos, currentRow); } else { colDataSetNULL(pCol, currentRow); @@ -5741,16 +5992,19 @@ bool getTwaFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv) { return true; } -bool twaFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { - if (!functionSetup(pCtx, pResultInfo)) { - return false; +int32_t twaFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { + if (pResultInfo->initialized) { + return TSDB_CODE_SUCCESS; + } + if (TSDB_CODE_SUCCESS != functionSetup(pCtx, pResultInfo)) { + return TSDB_CODE_FUNC_SETUP_ERROR; } STwaInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); pInfo->numOfElems = 0; pInfo->p.key = INT64_MIN; pInfo->win = TSWINDOW_INITIALIZER; - return true; + return TSDB_CODE_SUCCESS; } static double twa_get_area(SPoint1 s, SPoint1 e) { @@ -5768,6 +6022,7 @@ static double twa_get_area(SPoint1 s, SPoint1 e) { } int32_t twaFunction(SqlFunctionCtx* pCtx) { + int32_t code = TSDB_CODE_SUCCESS; SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; @@ -5782,8 +6037,16 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { funcInputUpdate(pCtx); SFuncInputRow row = {0}; + bool result = false; if (pCtx->start.key != INT64_MIN && last->key == INT64_MIN) { - while (funcInputGetNextRow(pCtx, &row)) { + while (1) { + code = funcInputGetNextRow(pCtx, &row, &result); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + if (!result) { + break; + } if (row.isDataNull) { continue; } @@ -5798,7 +6061,14 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { break; } } else if (pInfo->p.key == INT64_MIN) { - while (funcInputGetNextRow(pCtx, &row)) { + while (1) { + code = funcInputGetNextRow(pCtx, &row, &result); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + if (!result) { + break; + } if (row.isDataNull) { continue; } @@ -5816,181 +6086,69 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { SPoint1 st = {0}; // calculate the value of - switch (pInputCol->info.type) { - case TSDB_DATA_TYPE_TINYINT: { - while (funcInputGetNextRow(pCtx, &row)) { - if (row.isDataNull) { - continue; - } - pInfo->numOfElems++; - + while (1) { + code = funcInputGetNextRow(pCtx, &row, &result); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + if (!result) { + break; + } + if (row.isDataNull) { + continue; + } + pInfo->numOfElems++; + switch (pInputCol->info.type) { + case TSDB_DATA_TYPE_TINYINT: { INIT_INTP_POINT(st, row.ts, *(int8_t*)row.pData); - if (pInfo->p.key == st.key) { - return TSDB_CODE_FUNC_DUP_TIMESTAMP; - } - - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; + break; } - break; - } - - case TSDB_DATA_TYPE_SMALLINT: { - while (funcInputGetNextRow(pCtx, &row)) { - if (row.isDataNull) { - continue; - } - pInfo->numOfElems++; - + case TSDB_DATA_TYPE_SMALLINT: { INIT_INTP_POINT(st, row.ts, *(int16_t*)row.pData); - if (pInfo->p.key == st.key) { - return TSDB_CODE_FUNC_DUP_TIMESTAMP; - } - - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; + break; } - break; - } - case TSDB_DATA_TYPE_INT: { - while (funcInputGetNextRow(pCtx, &row)) { - if (row.isDataNull) { - continue; - } - pInfo->numOfElems++; - + case TSDB_DATA_TYPE_INT: { INIT_INTP_POINT(st, row.ts, *(int32_t*)row.pData); - if (pInfo->p.key == st.key) { - return TSDB_CODE_FUNC_DUP_TIMESTAMP; - } - - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; + break; } - break; - } - case TSDB_DATA_TYPE_BIGINT: { - while (funcInputGetNextRow(pCtx, &row)) { - if (row.isDataNull) { - continue; - } - pInfo->numOfElems++; - + case TSDB_DATA_TYPE_BIGINT: { INIT_INTP_POINT(st, row.ts, *(int64_t*)row.pData); - if (pInfo->p.key == st.key) { - return TSDB_CODE_FUNC_DUP_TIMESTAMP; - } - - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; + break; } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - while (funcInputGetNextRow(pCtx, &row)) { - if (row.isDataNull) { - continue; - } - pInfo->numOfElems++; - + case TSDB_DATA_TYPE_FLOAT: { INIT_INTP_POINT(st, row.ts, *(float_t*)row.pData); - if (pInfo->p.key == st.key) { - return TSDB_CODE_FUNC_DUP_TIMESTAMP; - } - - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; + break; } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - while (funcInputGetNextRow(pCtx, &row)) { - if (row.isDataNull) { - continue; - } - pInfo->numOfElems++; - + case TSDB_DATA_TYPE_DOUBLE: { INIT_INTP_POINT(st, row.ts, *(double*)row.pData); - if (pInfo->p.key == st.key) { - return TSDB_CODE_FUNC_DUP_TIMESTAMP; - } - - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; + break; } - break; - } - case TSDB_DATA_TYPE_UTINYINT: { - while (funcInputGetNextRow(pCtx, &row)) { - if (row.isDataNull) { - continue; - } - pInfo->numOfElems++; - + case TSDB_DATA_TYPE_UTINYINT: { INIT_INTP_POINT(st, row.ts, *(uint8_t*)row.pData); - if (pInfo->p.key == st.key) { - return TSDB_CODE_FUNC_DUP_TIMESTAMP; - } - - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; + break; } - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - while (funcInputGetNextRow(pCtx, &row)) { - if (row.isDataNull) { - continue; - } - pInfo->numOfElems++; - + case TSDB_DATA_TYPE_USMALLINT: { INIT_INTP_POINT(st, row.ts, *(uint16_t*)row.pData); - if (pInfo->p.key == st.key) { - return TSDB_CODE_FUNC_DUP_TIMESTAMP; - } - - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; + break; } - break; - } - case TSDB_DATA_TYPE_UINT: { - while (funcInputGetNextRow(pCtx, &row)) { - if (row.isDataNull) { - continue; - } - pInfo->numOfElems++; - + case TSDB_DATA_TYPE_UINT: { INIT_INTP_POINT(st, row.ts, *(uint32_t*)row.pData); - if (pInfo->p.key == st.key) { - return TSDB_CODE_FUNC_DUP_TIMESTAMP; - } - - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; + break; } - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - while (funcInputGetNextRow(pCtx, &row)) { - if (row.isDataNull) { - continue; - } - pInfo->numOfElems++; - + case TSDB_DATA_TYPE_UBIGINT: { INIT_INTP_POINT(st, row.ts, *(uint64_t*)row.pData); - if (pInfo->p.key == st.key) { - return TSDB_CODE_FUNC_DUP_TIMESTAMP; - } - - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; + break; } - break; + default: { + return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; + } + } + if (pInfo->p.key == st.key) { + return TSDB_CODE_FUNC_DUP_TIMESTAMP; } - default: - return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; } // the last interpolated time window value @@ -6040,14 +6198,17 @@ int32_t twaFinalize(struct SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return functionFinalize(pCtx, pBlock); } -bool blockDistSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { - if (!functionSetup(pCtx, pResultInfo)) { - return false; +int32_t blockDistSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { + if (pResultInfo->initialized) { + return TSDB_CODE_SUCCESS; + } + if (TSDB_CODE_SUCCESS != functionSetup(pCtx, pResultInfo)) { + return TSDB_CODE_FUNC_SETUP_ERROR; } STableBlockDistInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); pInfo->minRows = INT32_MAX; - return true; + return TSDB_CODE_SUCCESS; } int32_t blockDistFunction(SqlFunctionCtx* pCtx) { @@ -6059,7 +6220,7 @@ int32_t blockDistFunction(SqlFunctionCtx* pCtx) { STableBlockDistInfo* pDistInfo = GET_ROWCELL_INTERBUF(pResInfo); STableBlockDistInfo p1 = {0}; - tDeserializeBlockDistInfo(varDataVal(pInputCol->pData), varDataLen(pInputCol->pData), &p1); + (void)tDeserializeBlockDistInfo(varDataVal(pInputCol->pData), varDataLen(pInputCol->pData), &p1); pDistInfo->numOfBlocks += p1.numOfBlocks; pDistInfo->numOfTables += p1.numOfTables; @@ -6176,7 +6337,10 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { pData->numOfBlocks, pData->totalSize / 1024.0, averageSize / 1024.0, compRatio, '%'); varDataSetLen(st, len); - colDataSetVal(pColInfo, row++, st, false); + int32_t code = colDataSetVal(pColInfo, row++, st, false); + if (TSDB_CODE_SUCCESS != code) { + return code; + } int64_t avgRows = 0; if (pData->numOfBlocks > 0) { @@ -6186,22 +6350,34 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { len = sprintf(st + VARSTR_HEADER_SIZE, "Block_Rows=[%" PRId64 "] MinRows=[%d] MaxRows=[%d] AvgRows=[%" PRId64 "]", pData->totalRows, pData->minRows, pData->maxRows, avgRows); varDataSetLen(st, len); - colDataSetVal(pColInfo, row++, st, false); + code = colDataSetVal(pColInfo, row++, st, false); + if (TSDB_CODE_SUCCESS != code) { + return code; + } len = sprintf(st + VARSTR_HEADER_SIZE, "Inmem_Rows=[%d] Stt_Rows=[%d] ", pData->numOfInmemRows, pData->numOfSttRows); varDataSetLen(st, len); - colDataSetVal(pColInfo, row++, st, false); + code = colDataSetVal(pColInfo, row++, st, false); + if (TSDB_CODE_SUCCESS != code) { + return code; + } len = sprintf(st + VARSTR_HEADER_SIZE, "Total_Tables=[%d] Total_Filesets=[%d] Total_Vgroups=[%d]", pData->numOfTables, pData->numOfFiles, pData->numOfVgroups); varDataSetLen(st, len); - colDataSetVal(pColInfo, row++, st, false); + code = colDataSetVal(pColInfo, row++, st, false); + if (TSDB_CODE_SUCCESS != code) { + return code; + } len = sprintf(st + VARSTR_HEADER_SIZE, "--------------------------------------------------------------------------------"); varDataSetLen(st, len); - colDataSetVal(pColInfo, row++, st, false); + code = colDataSetVal(pColInfo, row++, st, false); + if (TSDB_CODE_SUCCESS != code) { + return code; + } int32_t maxVal = 0; int32_t minVal = INT32_MAX; @@ -6240,7 +6416,10 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { } varDataSetLen(st, len); - colDataSetVal(pColInfo, row++, st, false); + code = colDataSetVal(pColInfo, row++, st, false); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } return TSDB_CODE_SUCCESS; @@ -6251,9 +6430,12 @@ bool getDerivativeFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv) { return true; } -bool derivativeFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { - if (!functionSetup(pCtx, pResInfo)) { - return false; // not initialized since it has been initialized +int32_t derivativeFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { + if (pResInfo->initialized) { + return TSDB_CODE_SUCCESS; + } + if (TSDB_CODE_SUCCESS != functionSetup(pCtx, pResInfo)) { + return TSDB_CODE_FUNC_SETUP_ERROR; } SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -6262,7 +6444,7 @@ bool derivativeFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { pDerivInfo->prevTs = -1; pDerivInfo->tsWindow = pCtx->param[1].param.i; pDerivInfo->valueSet = false; - return true; + return TSDB_CODE_SUCCESS; } int32_t derivativeFunction(SqlFunctionCtx* pCtx) { @@ -6275,13 +6457,22 @@ int32_t derivativeFunction(SqlFunctionCtx* pCtx) { int32_t numOfElems = 0; SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; SColumnInfoData* pTsOutput = pCtx->pTsOutput; + int32_t code = TSDB_CODE_SUCCESS; funcInputUpdate(pCtx); double v = 0; if (pCtx->order == TSDB_ORDER_ASC) { SFuncInputRow row = {0}; - while (funcInputGetNextRow(pCtx, &row)) { + bool result = false; + while (1) { + code = funcInputGetNextRow(pCtx, &row, &result); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + if (!result) { + break; + } if (row.isDataNull) { continue; } @@ -6302,7 +6493,10 @@ int32_t derivativeFunction(SqlFunctionCtx* pCtx) { if (isinf(r) || isnan(r)) { colDataSetNULL(pOutput, pos); } else { - colDataSetVal(pOutput, pos, (const char*)&r, false); + code = colDataSetVal(pOutput, pos, (const char*)&r, false); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (pTsOutput != NULL) { @@ -6311,7 +6505,10 @@ int32_t derivativeFunction(SqlFunctionCtx* pCtx) { // handle selectivity if (pCtx->subsidiaries.num > 0) { - appendSelectivityCols(pCtx, row.block, row.rowIndex, pos); + code = appendSelectivityCols(pCtx, row.block, row.rowIndex, pos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } numOfElems++; @@ -6323,7 +6520,15 @@ int32_t derivativeFunction(SqlFunctionCtx* pCtx) { } } else { SFuncInputRow row = {0}; - while (funcInputGetNextRow(pCtx, &row)) { + bool result = false; + while (1) { + code = funcInputGetNextRow(pCtx, &row, &result); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + if (!result) { + break; + } if (row.isDataNull) { continue; } @@ -6344,7 +6549,10 @@ int32_t derivativeFunction(SqlFunctionCtx* pCtx) { if (isinf(r) || isnan(r)) { colDataSetNULL(pOutput, pos); } else { - colDataSetVal(pOutput, pos, (const char*)&r, false); + code = colDataSetVal(pOutput, pos, (const char*)&r, false); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (pTsOutput != NULL) { @@ -6353,7 +6561,10 @@ int32_t derivativeFunction(SqlFunctionCtx* pCtx) { // handle selectivity if (pCtx->subsidiaries.num > 0) { - appendSelectivityCols(pCtx, row.block, row.rowIndex, pos); + code = appendSelectivityCols(pCtx, row.block, row.rowIndex, pos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } numOfElems++; } @@ -6377,9 +6588,12 @@ bool getIrateFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv) { return true; } -bool irateFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { - if (!functionSetup(pCtx, pResInfo)) { - return false; // not initialized since it has been initialized +int32_t irateFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { + if (pResInfo->initialized) { + return TSDB_CODE_SUCCESS; + } + if (TSDB_CODE_SUCCESS != functionSetup(pCtx, pResInfo)) { + return TSDB_CODE_FUNC_SETUP_ERROR; } SRateInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -6390,7 +6604,7 @@ bool irateFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { pInfo->lastValue = (double)INT64_MIN; pInfo->hasResult = 0; - return true; + return TSDB_CODE_SUCCESS; } static void doSaveRateInfo(SRateInfo* pRateInfo, bool isFirst, int64_t ts, char* pk, double v) { @@ -6399,14 +6613,14 @@ static void doSaveRateInfo(SRateInfo* pRateInfo, bool isFirst, int64_t ts, char* pRateInfo->firstKey = ts; if (pRateInfo->firstPk) { int32_t pkBytes = IS_VAR_DATA_TYPE(pRateInfo->pkType) ? varDataTLen(pk) : pRateInfo->pkBytes; - memcpy(pRateInfo->firstPk, pk, pkBytes); + (void)memcpy(pRateInfo->firstPk, pk, pkBytes); } } else { pRateInfo->lastValue = v; pRateInfo->lastKey = ts; if (pRateInfo->lastPk) { int32_t pkBytes = IS_VAR_DATA_TYPE(pRateInfo->pkType) ? varDataTLen(pk) : pRateInfo->pkBytes; - memcpy(pRateInfo->lastPk, pk, pkBytes); + (void)memcpy(pRateInfo->lastPk, pk, pkBytes); } } } @@ -6429,6 +6643,7 @@ static void initializeRateInfo(SqlFunctionCtx* pCtx, SRateInfo* pRateInfo, bool } int32_t irateFunction(SqlFunctionCtx* pCtx) { + int32_t code = TSDB_CODE_SUCCESS; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SRateInfo* pRateInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -6444,7 +6659,15 @@ int32_t irateFunction(SqlFunctionCtx* pCtx) { int32_t numOfElems = 0; int32_t type = pInputCol->info.type; SFuncInputRow row = {0}; - while (funcInputGetNextRow(pCtx, &row)) { + bool result = false; + while (1) { + code = funcInputGetNextRow(pCtx, &row, &result); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + if (!result) { + break; + } if (row.isDataNull) { continue; } @@ -6589,16 +6812,19 @@ int32_t iratePartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t resultBytes = getIrateInfoSize(pInfo->pkBytes); char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); - memcpy(varDataVal(res), pInfo, resultBytes); + if (NULL == res) { + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(varDataVal(res), pInfo, resultBytes); varDataSetLen(res, resultBytes); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - colDataSetVal(pCol, pBlock->info.rows, res, false); + int32_t code = colDataSetVal(pCol, pBlock->info.rows, res, false); taosMemoryFree(res); - return pResInfo->numOfRes; + return code; } int32_t irateFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { @@ -6610,9 +6836,9 @@ int32_t irateFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SRateInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); double result = doCalcRate(pInfo, (double)TSDB_TICK_PER_SECOND(pCtx->param[1].param.i)); - colDataSetVal(pCol, pBlock->info.rows, (const char*)&result, pResInfo->isNullRes); + int32_t code = colDataSetVal(pCol, pBlock->info.rows, (const char*)&result, pResInfo->isNullRes); - return pResInfo->numOfRes; + return code; } int32_t groupConstValueFunction(SqlFunctionCtx* pCtx) { @@ -6637,10 +6863,10 @@ int32_t groupConstValueFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, startIndex); if (IS_VAR_DATA_TYPE(pInputCol->info.type)) { - memcpy(pInfo->data, data, - (pInputCol->info.type == TSDB_DATA_TYPE_JSON) ? getJsonValueLen(data) : varDataTLen(data)); + (void)memcpy(pInfo->data, data, + (pInputCol->info.type == TSDB_DATA_TYPE_JSON) ? getJsonValueLen(data) : varDataTLen(data)); } else { - memcpy(pInfo->data, data, pInputCol->info.bytes); + (void)memcpy(pInfo->data, data, pInputCol->info.bytes); } pInfo->hasResult = true; @@ -6656,6 +6882,7 @@ int32_t groupKeyFunction(SqlFunctionCtx* pCtx) { int32_t groupConstValueFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + int32_t code = TSDB_CODE_SUCCESS; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); @@ -6665,13 +6892,16 @@ int32_t groupConstValueFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { if (pInfo->hasResult) { int32_t currentRow = pBlock->info.rows; for (; currentRow < pBlock->info.rows + pResInfo->numOfRes; ++currentRow) { - colDataSetVal(pCol, currentRow, pInfo->data, pInfo->isNull ? true : false); + code = colDataSetVal(pCol, currentRow, pInfo->data, pInfo->isNull ? true : false); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } } else { pResInfo->numOfRes = 0; } - return pResInfo->numOfRes; + return code; } int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock){ @@ -6697,10 +6927,10 @@ int32_t groupKeyCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { } if (IS_VAR_DATA_TYPE(pSourceCtx->resDataInfo.type)) { - memcpy(pDBuf->data, pSBuf->data, + (void)memcpy(pDBuf->data, pSBuf->data, (pSourceCtx->resDataInfo.type == TSDB_DATA_TYPE_JSON) ? getJsonValueLen(pSBuf->data) : varDataTLen(pSBuf->data)); } else { - memcpy(pDBuf->data, pSBuf->data, pSourceCtx->resDataInfo.bytes); + (void)memcpy(pDBuf->data, pSBuf->data, pSourceCtx->resDataInfo.bytes); } pDBuf->hasResult = true; diff --git a/source/libs/function/src/detail/tavgfunction.c b/source/libs/function/src/detail/tavgfunction.c index 3d51f0cd16..3b8cf6e6b5 100644 --- a/source/libs/function/src/detail/tavgfunction.c +++ b/source/libs/function/src/detail/tavgfunction.c @@ -354,14 +354,17 @@ bool getAvgFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { return true; } -bool avgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { - if (!functionSetup(pCtx, pResultInfo)) { - return false; +int32_t avgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { + if (pResultInfo->initialized) { + return TSDB_CODE_SUCCESS; + } + if (TSDB_CODE_SUCCESS != functionSetup(pCtx, pResultInfo)) { + return TSDB_CODE_FUNC_SETUP_ERROR; } SAvgRes* pRes = GET_ROWCELL_INTERBUF(pResultInfo); - memset(pRes, 0, sizeof(SAvgRes)); - return true; + (void)memset(pRes, 0, sizeof(SAvgRes)); + return TSDB_CODE_SUCCESS; } static int32_t calculateAvgBySMAInfo(SAvgRes* pRes, int32_t numOfRows, int32_t type, const SColumnDataAgg* pAgg) { @@ -859,5 +862,5 @@ int32_t avgPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { colDataSetVal(pCol, pBlock->info.rows, res, false); taosMemoryFree(res); - return pResInfo->numOfRes; + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/function/src/thistogram.c b/source/libs/function/src/thistogram.c index b56691f35d..3f4ce3811b 100644 --- a/source/libs/function/src/thistogram.c +++ b/source/libs/function/src/thistogram.c @@ -50,7 +50,7 @@ SHistogramInfo* tHistogramCreate(int32_t numOfEntries) { } SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins) { - memset(pBuf, 0, sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfBins + 1)); + (void)memset(pBuf, 0, sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfBins + 1)); SHistogramInfo* pHisto = (SHistogramInfo*)pBuf; pHisto->elems = (SHistBin*)((char*)pBuf + sizeof(SHistogramInfo)); diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index a068186992..d7b4eb21c0 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -238,67 +238,71 @@ static void resetSlotInfo(tMemBucket *pBucket) { } } -tMemBucket *tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval, bool hasWindowOrGroup) { - tMemBucket *pBucket = (tMemBucket *)taosMemoryCalloc(1, sizeof(tMemBucket)); - if (pBucket == NULL) { - return NULL; +int32_t tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval, bool hasWindowOrGroup, + tMemBucket **pBucket) { + *pBucket = (tMemBucket *)taosMemoryCalloc(1, sizeof(tMemBucket)); + if (*pBucket == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; } if (hasWindowOrGroup) { // With window or group by, we need to shrink page size and reduce page num to save memory. - pBucket->numOfSlots = DEFAULT_NUM_OF_SLOT / 8 ; // 128 bucket - pBucket->bufPageSize = 4096; // 4k per page + (*pBucket)->numOfSlots = DEFAULT_NUM_OF_SLOT / 8 ; // 128 bucket + (*pBucket)->bufPageSize = 4096; // 4k per page } else { - pBucket->numOfSlots = DEFAULT_NUM_OF_SLOT; - pBucket->bufPageSize = 16384 * 4; // 16k per page + (*pBucket)->numOfSlots = DEFAULT_NUM_OF_SLOT; + (*pBucket)->bufPageSize = 16384 * 4; // 16k per page } - pBucket->type = dataType; - pBucket->bytes = nElemSize; - pBucket->total = 0; - pBucket->times = 1; + (*pBucket)->type = dataType; + (*pBucket)->bytes = nElemSize; + (*pBucket)->total = 0; + (*pBucket)->times = 1; - pBucket->maxCapacity = 200000; - pBucket->groupPagesMap = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); - if (setBoundingBox(&pBucket->range, pBucket->type, minval, maxval) != 0) { + (*pBucket)->maxCapacity = 200000; + (*pBucket)->groupPagesMap = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if ((*pBucket)->groupPagesMap == NULL) { + tMemBucketDestroy(*pBucket); + return TSDB_CODE_OUT_OF_MEMORY; + } + if (setBoundingBox(&(*pBucket)->range, (*pBucket)->type, minval, maxval) != 0) { // qError("MemBucket:%p, invalid value range: %f-%f", pBucket, minval, maxval); - taosMemoryFree(pBucket); - return NULL; + tMemBucketDestroy(*pBucket); + return TSDB_CODE_FUNC_INVALID_VALUE_RANGE; } - pBucket->elemPerPage = (pBucket->bufPageSize - sizeof(SFilePage)) / pBucket->bytes; - pBucket->comparFn = getKeyComparFunc(pBucket->type, TSDB_ORDER_ASC); + (*pBucket)->elemPerPage = ((*pBucket)->bufPageSize - sizeof(SFilePage)) / (*pBucket)->bytes; + (*pBucket)->comparFn = getKeyComparFunc((*pBucket)->type, TSDB_ORDER_ASC); - pBucket->hashFunc = getHashFunc(pBucket->type); - if (pBucket->hashFunc == NULL) { + (*pBucket)->hashFunc = getHashFunc((*pBucket)->type); + if ((*pBucket)->hashFunc == NULL) { // qError("MemBucket:%p, not support data type %d, failed", pBucket, pBucket->type); - taosMemoryFree(pBucket); - return NULL; + tMemBucketDestroy(*pBucket); + return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; } - pBucket->pSlots = (tMemBucketSlot *)taosMemoryCalloc(pBucket->numOfSlots, sizeof(tMemBucketSlot)); - if (pBucket->pSlots == NULL) { - taosMemoryFree(pBucket); - return NULL; + (*pBucket)->pSlots = (tMemBucketSlot *)taosMemoryCalloc((*pBucket)->numOfSlots, sizeof(tMemBucketSlot)); + if ((*pBucket)->pSlots == NULL) { + tMemBucketDestroy(*pBucket); + return TSDB_CODE_OUT_OF_MEMORY; } - resetSlotInfo(pBucket); + resetSlotInfo((*pBucket)); if (!osTempSpaceAvailable()) { - terrno = TSDB_CODE_NO_DISKSPACE; // qError("MemBucket create disk based Buf failed since %s", terrstr(terrno)); - tMemBucketDestroy(pBucket); - return NULL; + tMemBucketDestroy(*pBucket); + return TSDB_CODE_NO_DISKSPACE; } - int32_t ret = createDiskbasedBuf(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 1024, "1", tsTempDir); + int32_t ret = createDiskbasedBuf(&(*pBucket)->pBuffer, (*pBucket)->bufPageSize, (*pBucket)->bufPageSize * 1024, "1", tsTempDir); if (ret != 0) { - tMemBucketDestroy(pBucket); - return NULL; + tMemBucketDestroy(*pBucket); + return ret; } // qDebug("MemBucket:%p, elem size:%d", pBucket, pBucket->bytes); - return pBucket; + return TSDB_CODE_SUCCESS; } void tMemBucketDestroy(tMemBucket *pBucket) { diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 3c5e4014b3..c9b8a1e08b 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1011,7 +1011,7 @@ void releaseUdfFuncHandle(char *udfName, UdfcFuncHandle handle); int32_t cleanUpUdfs(); bool udfAggGetEnv(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv); -bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo); +int32_t udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo); int32_t udfAggProcess(struct SqlFunctionCtx *pCtx); int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock); @@ -1196,15 +1196,18 @@ bool udfAggGetEnv(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv) { return true; } -bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo) { - if (functionSetup(pCtx, pResultCellInfo) != true) { - return false; +int32_t udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo) { + if (pResultCellInfo->initialized) { + return TSDB_CODE_SUCCESS; + } + if (functionSetup(pCtx, pResultCellInfo) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_FUNC_SETUP_ERROR; } UdfcFuncHandle handle; int32_t udfCode = 0; if ((udfCode = acquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) { fnError("udfAggInit error. step doSetupUdf. udf code: %d", udfCode); - return false; + return TSDB_CODE_FUNC_SETUP_ERROR; } SUdfcUvSession *session = (SUdfcUvSession *)handle; SUdfAggRes *udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(pResultCellInfo); @@ -1218,7 +1221,7 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResult if ((udfCode = doCallUdfAggInit(handle, &buf)) != 0) { fnError("udfAggInit error. step doCallUdfAggInit. udf code: %d", udfCode); releaseUdfFuncHandle(pCtx->udfName, handle); - return false; + return TSDB_CODE_FUNC_SETUP_ERROR; } if (buf.bufLen <= session->bufSize) { memcpy(udfRes->interResBuf, buf.buf, buf.bufLen); @@ -1227,11 +1230,11 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResult } else { fnError("udfc inter buf size %d is greater than function bufSize %d", buf.bufLen, session->bufSize); releaseUdfFuncHandle(pCtx->udfName, handle); - return false; + return TSDB_CODE_FUNC_SETUP_ERROR; } releaseUdfFuncHandle(pCtx->udfName, handle); freeUdfInterBuf(&buf); - return true; + return TSDB_CODE_SUCCESS; } int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { diff --git a/source/libs/parser/test/parTestMain.cpp b/source/libs/parser/test/parTestMain.cpp index 8d13d7cf0e..c8925b3df7 100644 --- a/source/libs/parser/test/parTestMain.cpp +++ b/source/libs/parser/test/parTestMain.cpp @@ -35,7 +35,8 @@ namespace ParserTest { class ParserEnv : public testing::Environment { public: virtual void SetUp() { - fmFuncMgtInit(); + // TODO(smj) : How to handle return value of fmFuncMgtInit + (void)fmFuncMgtInit(); initMetaDataEnv(); generateMetaData(); initLog(TD_TMP_DIR_PATH "td"); diff --git a/source/libs/planner/test/planTestMain.cpp b/source/libs/planner/test/planTestMain.cpp index 4e013c44b8..e05fa27c1e 100644 --- a/source/libs/planner/test/planTestMain.cpp +++ b/source/libs/planner/test/planTestMain.cpp @@ -27,7 +27,8 @@ class PlannerEnv : public testing::Environment { public: virtual void SetUp() { - fmFuncMgtInit(); + // TODO(smj) : How to handle return value of fmFuncMgtInit + (void)fmFuncMgtInit(); initMetaDataEnv(); generateMetaData(); initLog(TD_TMP_DIR_PATH "td"); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 3bd1ae3ed5..09c7224c1e 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -709,6 +709,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED, "Func to_tim TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_CHAR_NOT_SUPPORTED, "Func to_char failed for unsupported format") TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TIME_UNIT_INVALID, "Invalid function time unit") TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL, "Function time unit cannot be smaller than db precision") +TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_INVALID_VALUE_RANGE, "Function got invalid value range") +TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_SETUP_ERROR, "Function set up failed") //udf TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING, "udf is stopping") From ed892e506195038c9c66457b88beac64eec61607 Mon Sep 17 00:00:00 2001 From: sima Date: Sat, 20 Jul 2024 18:11:47 +0800 Subject: [PATCH 29/43] enh:[TD-31043] Handling return value in thistogram.c --- source/libs/function/inc/thistogram.h | 7 +- source/libs/function/src/builtinsimpl.c | 30 +++++-- source/libs/function/src/thistogram.c | 111 ++++++++++++++---------- 3 files changed, 92 insertions(+), 56 deletions(-) diff --git a/source/libs/function/inc/thistogram.h b/source/libs/function/inc/thistogram.h index 20111086cd..5bc6a87c70 100644 --- a/source/libs/function/inc/thistogram.h +++ b/source/libs/function/inc/thistogram.h @@ -55,14 +55,15 @@ typedef struct SHistogramInfo { #endif } SHistogramInfo; -SHistogramInfo* tHistogramCreate(int32_t numOfBins); +int32_t tHistogramCreate(int32_t numOfEntries, SHistogramInfo** pHisto); SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins); int32_t tHistogramAdd(SHistogramInfo** pHisto, double val); int64_t tHistogramSum(SHistogramInfo* pHisto, double v); -double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num); -SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries); +int32_t tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num, double** pVal); +int32_t tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries, + SHistogramInfo** pResHistogram); void tHistogramDestroy(SHistogramInfo** pHisto); void tHistogramPrint(SHistogramInfo* pHisto); diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 5c960e2580..cdd2e05869 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2213,8 +2213,8 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) { double v = 0; GET_TYPED_DATA(v, double, type, data); int32_t code = tHistogramAdd(&pInfo->pHisto, v); - if (code != 0) { - return TSDB_CODE_FAILED; + if (code != TSDB_CODE_SUCCESS) { + return code; } } @@ -2227,7 +2227,7 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } -static void apercentileTransferInfo(SAPercentileInfo* pInput, SAPercentileInfo* pOutput, bool* hasRes) { +static int32_t apercentileTransferInfo(SAPercentileInfo* pInput, SAPercentileInfo* pOutput, bool* hasRes) { pOutput->percent = pInput->percent; pOutput->algo = pInput->algo; if (pOutput->algo == APERCT_ALGO_TDIGEST) { @@ -2276,7 +2276,12 @@ static void apercentileTransferInfo(SAPercentileInfo* pInput, SAPercentileInfo* qDebug("%s input histogram, elem:%" PRId64 ", entry:%d, %p", __FUNCTION__, pHisto->numOfElems, pHisto->numOfEntries, pInput->pHisto); - SHistogramInfo* pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN); + SHistogramInfo* pRes = NULL; + int32_t code = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN, &pRes); + if (TSDB_CODE_SUCCESS != pRes) { + tHistogramDestroy(&pRes); + return code; + } (void)memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN); pHisto->elems = (SHistBin*)((char*)pHisto + sizeof(SHistogramInfo)); @@ -2307,7 +2312,10 @@ int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pCol, i); SAPercentileInfo* pInputInfo = (SAPercentileInfo*)varDataVal(data); - apercentileTransferInfo(pInputInfo, pInfo, &hasRes); + int32_t code = apercentileTransferInfo(pInputInfo, pInfo, &hasRes); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } if (pInfo->algo != APERCT_ALGO_TDIGEST) { @@ -2340,7 +2348,12 @@ int32_t apercentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries, pInfo->pHisto, pInfo->pHisto->elems); double ratio[] = {pInfo->percent}; - double* res = tHistogramUniform(pInfo->pHisto, ratio, 1); + double* res = NULL; + int32_t code = tHistogramUniform(pInfo->pHisto, ratio, 1, &res); + if (TSDB_CODE_SUCCESS != code) { + taosMemoryFree(res); + return code; + } pInfo->result = *res; // memcpy(pCtx->pOutput, res, sizeof(double)); taosMemoryFree(res); @@ -2391,7 +2404,10 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) qDebug("%s start to combine apercentile, %p", __FUNCTION__, pDBuf->pHisto); - apercentileTransferInfo(pSBuf, pDBuf, NULL); + int32_t code = apercentileTransferInfo(pSBuf, pDBuf, NULL); + if (TSDB_CODE_SUCCESS != code) { + return code; + } pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes); pDResInfo->isNullRes &= pSResInfo->isNullRes; return TSDB_CODE_SUCCESS; diff --git a/source/libs/function/src/thistogram.c b/source/libs/function/src/thistogram.c index 3f4ce3811b..ed36921b73 100644 --- a/source/libs/function/src/thistogram.c +++ b/source/libs/function/src/thistogram.c @@ -32,9 +32,12 @@ */ static int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val); -SHistogramInfo* tHistogramCreate(int32_t numOfEntries) { +int32_t tHistogramCreate(int32_t numOfEntries, SHistogramInfo** pHisto) { /* need one redundant slot */ - SHistogramInfo* pHisto = taosMemoryMalloc(sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfEntries + 1)); + *pHisto = taosMemoryMalloc(sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfEntries + 1)); + if (NULL == *pHisto) { + return TSDB_CODE_OUT_OF_MEMORY; + } #if !defined(USE_ARRAYLIST) pHisto->pList = SSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); @@ -46,7 +49,8 @@ SHistogramInfo* tHistogramCreate(int32_t numOfEntries) { pss->pTree = pHisto->pLoserTree; #endif - return tHistogramCreateFrom(pHisto, numOfEntries); + *pHisto = tHistogramCreateFrom(pHisto, numOfEntries); + return TSDB_CODE_SUCCESS; } SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins) { @@ -67,15 +71,19 @@ SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins) { } int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { + int32_t code = TSDB_CODE_SUCCESS; if (*pHisto == NULL) { - *pHisto = tHistogramCreate(MAX_HISTOGRAM_BIN); + code = tHistogramCreate(MAX_HISTOGRAM_BIN, pHisto); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } #if defined(USE_ARRAYLIST) int32_t idx = histoBinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val); if (ASSERTS(idx >= 0 && idx <= (*pHisto)->maxEntries && (*pHisto)->elems != NULL, "tHistogramAdd Error, idx:%d, maxEntries:%d, elems:%p", idx, (*pHisto)->maxEntries, (*pHisto)->elems)) { - return -1; + return TSDB_CODE_FAILED; } if ((*pHisto)->elems[idx].val == val && idx >= 0) { @@ -89,23 +97,23 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { if (idx > 0) { if (ASSERTS((*pHisto)->elems[idx - 1].val <= val, "tHistogramAdd Error, elems[%d].val:%lf, val:%lf", idx - 1, (*pHisto)->elems[idx - 1].val, val)) { - return -1; + return TSDB_CODE_FAILED; } } else { if (ASSERTS((*pHisto)->elems[idx].val > val, "tHistogramAdd Error, elems[%d].val:%lf, val:%lf", idx, (*pHisto)->elems[idx].val, val)) { - return -1; + return TSDB_CODE_FAILED; } } } else if ((*pHisto)->numOfElems > 0) { if (ASSERTS((*pHisto)->elems[(*pHisto)->numOfEntries].val <= val, "tHistogramAdd Error, elems[%d].val:%lf, val:%lf", (*pHisto)->numOfEntries, (*pHisto)->elems[idx].val, val)) { - return -1; + return TSDB_CODE_FAILED; } } - int32_t code = histogramCreateBin(*pHisto, idx, val); - if (code != 0) { + code = histogramCreateBin(*pHisto, idx, val); + if (code != TSDB_CODE_SUCCESS) { return code; } } @@ -286,7 +294,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { } (*pHisto)->numOfElems += 1; - return 0; + return code; } int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val) { @@ -335,7 +343,7 @@ static void histogramMergeImpl(SHistBin* pHistBin, int32_t* size) { s1->val = newVal; s1->num = s1->num + s2->num; - memmove(&pHistBin[index + 1], &pHistBin[index + 2], (oldSize - index - 2) * sizeof(SHistBin)); + (void)memmove(&pHistBin[index + 1], &pHistBin[index + 2], (oldSize - index - 2) * sizeof(SHistBin)); (*size) -= 1; #endif } @@ -345,12 +353,12 @@ int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val) { #if defined(USE_ARRAYLIST) int32_t remain = pHisto->numOfEntries - index; if (remain > 0) { - memmove(&pHisto->elems[index + 1], &pHisto->elems[index], sizeof(SHistBin) * remain); + (void)memmove(&pHisto->elems[index + 1], &pHisto->elems[index], sizeof(SHistBin) * remain); } if (ASSERTS(index >= 0 && index <= pHisto->maxEntries, "histogramCreateBin Error, index:%d, maxEntries:%d", index, pHisto->maxEntries)) { - return -1; + return TSDB_CODE_FAILED; } pHisto->elems[index].num = 1; @@ -367,10 +375,10 @@ int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val) { #endif if (ASSERTS(pHisto->numOfEntries <= pHisto->maxEntries, "histogramCreateBin Error, numOfEntries:%d, maxEntries:%d", pHisto->numOfEntries, pHisto->maxEntries)) { - return -1; + return TSDB_CODE_FAILED; } - return 0; + return TSDB_CODE_SUCCESS; } void tHistogramDestroy(SHistogramInfo** pHisto) { @@ -383,17 +391,17 @@ void tHistogramDestroy(SHistogramInfo** pHisto) { } void tHistogramPrint(SHistogramInfo* pHisto) { - printf("total entries: %d, elements: %" PRId64 "\n", pHisto->numOfEntries, pHisto->numOfElems); + (void)printf("total entries: %d, elements: %" PRId64 "\n", pHisto->numOfEntries, pHisto->numOfElems); #if defined(USE_ARRAYLIST) for (int32_t i = 0; i < pHisto->numOfEntries; ++i) { - printf("%d: (%f, %" PRId64 ")\n", i + 1, pHisto->elems[i].val, pHisto->elems[i].num); + (void)printf("%d: (%f, %" PRId64 ")\n", i + 1, pHisto->elems[i].val, pHisto->elems[i].num); } #else tSkipListNode* pNode = pHisto->pList->pHead.pForward[0]; for (int32_t i = 0; i < pHisto->numOfEntries; ++i) { SHistBin* pEntry = (SHistBin*)pNode->pData; - printf("%d: (%f, %" PRId64 ")\n", i + 1, pEntry->val, pEntry->num); + (void)printf("%d: (%f, %" PRId64 ")\n", i + 1, pEntry->val, pEntry->num); pNode = pNode->pForward[0]; } #endif @@ -443,21 +451,24 @@ int64_t tHistogramSum(SHistogramInfo* pHisto, double v) { #endif } -double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) { +int32_t tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num, double** pVal) { #if defined(USE_ARRAYLIST) - double* pVal = taosMemoryMalloc(num * sizeof(double)); + *pVal = taosMemoryMalloc(num * sizeof(double)); + if (NULL == *pVal) { + return TSDB_CODE_OUT_OF_MEMORY; + } for (int32_t i = 0; i < num; ++i) { double numOfElem = (ratio[i] / 100) * pHisto->numOfElems; if (numOfElem == 0) { - pVal[i] = pHisto->min; + (*pVal)[i] = pHisto->min; continue; } else if (numOfElem <= pHisto->elems[0].num) { - pVal[i] = pHisto->elems[0].val; + (*pVal)[i] = pHisto->elems[0].val; continue; } else if (numOfElem == pHisto->numOfElems) { - pVal[i] = pHisto->max; + (*pVal)[i] = pHisto->max; continue; } @@ -479,37 +490,39 @@ double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) { double delta = numOfElem - total; if (fabs(delta) < FLT_EPSILON) { - pVal[i] = pHisto->elems[j].val; + (*pVal)[i] = pHisto->elems[j].val; } double start = (double)pHisto->elems[j].num; double range = pHisto->elems[j + 1].num - start; if (range == 0) { - pVal[i] = (pHisto->elems[j + 1].val - pHisto->elems[j].val) * delta / start + pHisto->elems[j].val; + (*pVal)[i] = (pHisto->elems[j + 1].val - pHisto->elems[j].val) * delta / start + pHisto->elems[j].val; } else { double factor = (-2 * start + sqrt(4 * start * start - 4 * range * (-2 * delta))) / (2 * range); - pVal[i] = pHisto->elems[j].val + (pHisto->elems[j + 1].val - pHisto->elems[j].val) * factor; + (*pVal)[i] = pHisto->elems[j].val + (pHisto->elems[j + 1].val - pHisto->elems[j].val) * factor; } } #else double* pVal = taosMemoryMalloc(num * sizeof(double)); - + if (NULL == *pVal) { + return TSDB_CODE_OUT_OF_MEMORY; + } for (int32_t i = 0; i < num; ++i) { double numOfElem = ratio[i] * pHisto->numOfElems; tSkipListNode* pFirst = pHisto->pList->pHead.pForward[0]; SHistBin* pEntry = (SHistBin*)pFirst->pData; if (numOfElem == 0) { - pVal[i] = pHisto->min; + (*pVal)[i] = pHisto->min; printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]); continue; } else if (numOfElem <= pEntry->num) { - pVal[i] = pEntry->val; + (*pVal)[i] = pEntry->val; printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]); continue; } else if (numOfElem == pHisto->numOfElems) { - pVal[i] = pHisto->max; + (*pVal)[i] = pHisto->max; printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]); continue; } @@ -540,34 +553,40 @@ double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) { if (fabs(delta) < FLT_EPSILON) { // printf("i/numofSlot: %f, v:%f, %f\n", // (double)i/numOfSlots, numOfElem, pHisto->elems[j].val); - pVal[i] = pPrev->val; + (*pVal)[i] = pPrev->val; } double start = pPrev->num; double range = pEntry->num - start; if (range == 0) { - pVal[i] = (pEntry->val - pPrev->val) * delta / start + pPrev->val; + (*pVal)[i] = (pEntry->val - pPrev->val) * delta / start + pPrev->val; } else { double factor = (-2 * start + sqrt(4 * start * start - 4 * range * (-2 * delta))) / (2 * range); - pVal[i] = pPrev->val + (pEntry->val - pPrev->val) * factor; + (*pVal)[i] = pPrev->val + (pEntry->val - pPrev->val) * factor; } // printf("i/numofSlot: %f, v:%f, %f\n", (double)i/numOfSlots, // numOfElem, val); } #endif - return pVal; + return TSDB_CODE_SUCCESS; } -SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries) { - SHistogramInfo* pResHistogram = tHistogramCreate(numOfEntries); +int32_t tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries, SHistogramInfo** pResHistogram) { + int32_t code = tHistogramCreate(numOfEntries, pResHistogram); + if (TSDB_CODE_SUCCESS != code) { + return code; + } // error in histogram info if (pHisto1->numOfEntries > MAX_HISTOGRAM_BIN || pHisto2->numOfEntries > MAX_HISTOGRAM_BIN) { - return pResHistogram; + return code; } SHistBin* pHistoBins = taosMemoryCalloc(1, sizeof(SHistBin) * (pHisto1->numOfEntries + pHisto2->numOfEntries)); + if (NULL == pHistoBins) { + return TSDB_CODE_OUT_OF_MEMORY; + } int32_t i = 0, j = 0, k = 0; while (i < pHisto1->numOfEntries && j < pHisto2->numOfEntries) { @@ -583,28 +602,28 @@ SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2 if (i < pHisto1->numOfEntries) { int32_t remain = pHisto1->numOfEntries - i; - memcpy(&pHistoBins[k], &pHisto1->elems[i], sizeof(SHistBin) * remain); + (void)memcpy(&pHistoBins[k], &pHisto1->elems[i], sizeof(SHistBin) * remain); k += remain; } if (j < pHisto2->numOfEntries) { int32_t remain = pHisto2->numOfEntries - j; - memcpy(&pHistoBins[k], &pHisto2->elems[j], sizeof(SHistBin) * remain); + (void)memcpy(&pHistoBins[k], &pHisto2->elems[j], sizeof(SHistBin) * remain); k += remain; } /* update other information */ - pResHistogram->numOfElems = pHisto1->numOfElems + pHisto2->numOfElems; - pResHistogram->min = (pHisto1->min < pHisto2->min) ? pHisto1->min : pHisto2->min; - pResHistogram->max = (pHisto1->max > pHisto2->max) ? pHisto1->max : pHisto2->max; + (*pResHistogram)->numOfElems = pHisto1->numOfElems + pHisto2->numOfElems; + (*pResHistogram)->min = (pHisto1->min < pHisto2->min) ? pHisto1->min : pHisto2->min; + (*pResHistogram)->max = (pHisto1->max > pHisto2->max) ? pHisto1->max : pHisto2->max; while (k > numOfEntries) { histogramMergeImpl(pHistoBins, &k); } - pResHistogram->numOfEntries = k; - memcpy(pResHistogram->elems, pHistoBins, sizeof(SHistBin) * k); + (*pResHistogram)->numOfEntries = k; + (void)memcpy((*pResHistogram)->elems, pHistoBins, sizeof(SHistBin) * k); taosMemoryFree(pHistoBins); - return pResHistogram; + return TSDB_CODE_SUCCESS; } From 2a6a1b620c59017fb3a6949dcb144a10e2d16aee Mon Sep 17 00:00:00 2001 From: sima Date: Sat, 20 Jul 2024 19:26:13 +0800 Subject: [PATCH 30/43] enh:[TD-31043] Handling return value in tpercentile.c --- source/libs/function/src/builtinsimpl.c | 5 +-- source/libs/function/src/tpercentile.c | 44 ++++++++++++++++--------- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index cdd2e05869..2e68763924 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2235,7 +2235,7 @@ static int32_t apercentileTransferInfo(SAPercentileInfo* pInput, SAPercentileInf tdigestAutoFill(pInput->pTDigest, COMPRESSION); if (pInput->pTDigest->num_centroids == 0 && pInput->pTDigest->num_buffered_pts == 0) { - return; + return TSDB_CODE_SUCCESS; } if (hasRes) { @@ -2255,7 +2255,7 @@ static int32_t apercentileTransferInfo(SAPercentileInfo* pInput, SAPercentileInf } else { buildHistogramInfo(pInput); if (pInput->pHisto->numOfElems <= 0) { - return; + return TSDB_CODE_SUCCESS; } if (hasRes) { @@ -2290,6 +2290,7 @@ static int32_t apercentileTransferInfo(SAPercentileInfo* pInput, SAPercentileInf tHistogramDestroy(&pRes); } } + return TSDB_CODE_SUCCESS; } int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx) { diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index d7b4eb21c0..905436bea1 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -28,9 +28,12 @@ int32_t getGroupId(int32_t numOfSlots, int32_t slotIndex, int32_t times) { return (times * numOfSlots) + slotIndex; } -static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) { - SFilePage *buffer = +static int32_t loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx, SFilePage ** buffer) { + *buffer = (SFilePage *)taosMemoryCalloc(1, pMemBucket->bytes * pMemBucket->pSlots[slotIdx].info.size + sizeof(SFilePage)); + if (NULL == *buffer) { + return TSDB_CODE_OUT_OF_MEMORY; + } int32_t groupId = getGroupId(pMemBucket->numOfSlots, slotIdx, pMemBucket->times); @@ -39,8 +42,8 @@ static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) if (p != NULL) { pIdList = *(SArray **)p; } else { - taosMemoryFree(buffer); - return NULL; + taosMemoryFree(*buffer); + return TSDB_CODE_OUT_OF_MEMORY; } int32_t offset = 0; @@ -49,16 +52,16 @@ static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) SFilePage *pg = getBufPage(pMemBucket->pBuffer, *pageId); if (pg == NULL) { - taosMemoryFree(buffer); - return NULL; + taosMemoryFree(*buffer); + return terrno; } - memcpy(buffer->data + offset, pg->data, (size_t)(pg->num * pMemBucket->bytes)); + (void)memcpy((*buffer)->data + offset, pg->data, (size_t)(pg->num * pMemBucket->bytes)); offset += (int32_t)(pg->num * pMemBucket->bytes); } - taosSort(buffer->data, pMemBucket->pSlots[slotIdx].info.size, pMemBucket->bytes, pMemBucket->comparFn); - return buffer; + taosSort((*buffer)->data, pMemBucket->pSlots[slotIdx].info.size, pMemBucket->bytes, pMemBucket->comparFn); + return TSDB_CODE_SUCCESS; } static void resetBoundingBox(MinMaxEntry *range, int32_t type) { @@ -398,7 +401,14 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { void *p = taosHashGet(pBucket->groupPagesMap, &groupId, sizeof(groupId)); if (p == NULL) { pPageIdList = taosArrayInit(4, sizeof(int32_t)); - taosHashPut(pBucket->groupPagesMap, &groupId, sizeof(groupId), &pPageIdList, POINTER_BYTES); + if (NULL == pPageIdList) { + return TSDB_CODE_OUT_OF_MEMORY; + } + int32_t code = taosHashPut(pBucket->groupPagesMap, &groupId, sizeof(groupId), &pPageIdList, POINTER_BYTES); + if (TSDB_CODE_SUCCESS != code) { + taosArrayDestroy(pPageIdList); + return code; + } } else { pPageIdList = *(SArray **)p; } @@ -408,10 +418,13 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { return terrno; } pSlot->info.pageId = pageId; - taosArrayPush(pPageIdList, &pageId); + if (taosArrayPush(pPageIdList, &pageId) == NULL) { + taosArrayDestroy(pPageIdList); + return TSDB_CODE_OUT_OF_MEMORY; + } } - memcpy(pSlot->info.data->data + pSlot->info.data->num * pBucket->bytes, d, pBucket->bytes); + (void)memcpy(pSlot->info.data->data + pSlot->info.data->num * pBucket->bytes, d, pBucket->bytes); pSlot->info.data->num += 1; pSlot->info.size += 1; @@ -497,9 +510,10 @@ int32_t getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction if (pSlot->info.size <= pMemBucket->maxCapacity) { // data in buffer and file are merged together to be processed. - SFilePage *buffer = loadDataFromFilePage(pMemBucket, i); - if (buffer == NULL) { - return terrno; + SFilePage *buffer = NULL; + int32_t code = loadDataFromFilePage(pMemBucket, i, &buffer); + if (TSDB_CODE_SUCCESS != code) { + return code; } int32_t currentIdx = count - num; From 8b50b282b4b090622768da320284be1680a7f282 Mon Sep 17 00:00:00 2001 From: sima Date: Sat, 20 Jul 2024 19:29:24 +0800 Subject: [PATCH 31/43] enh:[TD-31043] Handling return value in tavgfunction.c --- source/libs/function/src/detail/tavgfunction.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/source/libs/function/src/detail/tavgfunction.c b/source/libs/function/src/detail/tavgfunction.c index 3b8cf6e6b5..6667e84bed 100644 --- a/source/libs/function/src/detail/tavgfunction.c +++ b/source/libs/function/src/detail/tavgfunction.c @@ -853,14 +853,17 @@ int32_t avgPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t resultBytes = getAvgInfoSize(); char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); - memcpy(varDataVal(res), pInfo, resultBytes); + if (NULL == res) { + return TSDB_CODE_OUT_OF_MEMORY; + } + (void)memcpy(varDataVal(res), pInfo, resultBytes); varDataSetLen(res, resultBytes); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - colDataSetVal(pCol, pBlock->info.rows, res, false); + int32_t code = colDataSetVal(pCol, pBlock->info.rows, res, false); taosMemoryFree(res); - return TSDB_CODE_SUCCESS; + return code; } From 06431b124961c32ed79314a64f43c726afe6ae3f Mon Sep 17 00:00:00 2001 From: sima Date: Sat, 20 Jul 2024 19:42:51 +0800 Subject: [PATCH 32/43] enh:[TD-31043] Handling return value in tminmax.c --- source/libs/function/src/detail/tminmax.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/source/libs/function/src/detail/tminmax.c b/source/libs/function/src/detail/tminmax.c index 590a15c753..b316b7a512 100644 --- a/source/libs/function/src/detail/tminmax.c +++ b/source/libs/function/src/detail/tminmax.c @@ -20,6 +20,7 @@ #include "tglobal.h" #define __COMPARE_ACQUIRED_MAX(i, end, bm, _data, ctx, val, pos) \ + int32_t code = TSDB_CODE_SUCCESS; \ for (; i < (end); ++i) { \ if (colDataIsNull_f(bm, i)) { \ continue; \ @@ -28,12 +29,16 @@ if ((val) < (_data)[i]) { \ (val) = (_data)[i]; \ if ((ctx)->subsidiaries.num > 0) { \ - updateTupleData((ctx), i, (ctx)->pSrcBlock, pos); \ + code = updateTupleData((ctx), i, (ctx)->pSrcBlock, pos); \ + if (TSDB_CODE_SUCCESS != code) { \ + return code; \ + } \ } \ } \ } #define __COMPARE_ACQUIRED_MIN(i, end, bm, _data, ctx, val, pos) \ + int32_t code = TSDB_CODE_SUCCESS; \ for (; i < (end); ++i) { \ if (colDataIsNull_f(bm, i)) { \ continue; \ @@ -42,7 +47,10 @@ if ((val) > (_data)[i]) { \ (val) = (_data)[i]; \ if ((ctx)->subsidiaries.num > 0) { \ - updateTupleData((ctx), i, (ctx)->pSrcBlock, pos); \ + code = updateTupleData((ctx), i, (ctx)->pSrcBlock, pos); \ + if (TSDB_CODE_SUCCESS != code) { \ + return code; \ + } \ } \ } \ } @@ -571,7 +579,7 @@ static int32_t findRowIndex(int32_t start, int32_t num, SColumnInfoData* pCol, c return -1; } -static void doExtractVal(SColumnInfoData* pCol, int32_t i, int32_t end, SqlFunctionCtx* pCtx, SMinmaxResInfo* pBuf, +static int32_t doExtractVal(SColumnInfoData* pCol, int32_t i, int32_t end, SqlFunctionCtx* pCtx, SMinmaxResInfo* pBuf, bool isMinFunc) { if (isMinFunc) { switch (pCol->info.type) { @@ -700,6 +708,7 @@ static void doExtractVal(SColumnInfoData* pCol, int32_t i, int32_t end, SqlFunct } } } + return TSDB_CODE_SUCCESS; } static int32_t saveRelatedTupleTag(SqlFunctionCtx* pCtx, SInputColumnInfoData* pInput, void* tval) { @@ -840,7 +849,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems) break; } default: - memcpy(&pBuf->v, p, pCol->info.bytes); + (void)memcpy(&pBuf->v, p, pCol->info.bytes); break; } @@ -858,7 +867,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems) goto _over; } - doExtractVal(pCol, i, end, pCtx, pBuf, isMinFunc); + code = doExtractVal(pCol, i, end, pCtx, pBuf, isMinFunc); } else { numOfElems = numOfRows; From 1285100f68a9a261a1671a5dab85d3bc91196be6 Mon Sep 17 00:00:00 2001 From: sima Date: Sat, 20 Jul 2024 17:34:29 +0800 Subject: [PATCH 33/43] enh:[TD-31043] Handling return value in functionMgt.c --- include/common/tdatablock.h | 2 +- include/libs/function/functionMgt.h | 4 +- source/libs/executor/src/executil.c | 3 + source/libs/executor/src/scanoperator.c | 5 +- source/libs/function/src/builtinsimpl.c | 2 +- source/libs/function/src/functionMgt.c | 104 +++++++++++++----------- source/libs/function/src/thistogram.c | 2 +- source/libs/parser/src/parTranslater.c | 15 ++-- source/libs/planner/src/planOptimizer.c | 8 +- 9 files changed, 81 insertions(+), 64 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 89f6beaab4..c09e96d16d 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -49,7 +49,7 @@ typedef struct SBlockOrderInfo { #define colDataSetNull_f_s(c_, r_) \ do { \ colDataSetNull_f((c_)->nullbitmap, r_); \ - memset(((char*)(c_)->pData) + (c_)->info.bytes * (r_), 0, (c_)->info.bytes); \ + (void)memset(((char*)(c_)->pData) + (c_)->info.bytes * (r_), 0, (c_)->info.bytes); \ } while (0) #define colDataClearNull_f(bm_, r_) \ diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index d3fb953dc3..1da1cbf716 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -260,7 +260,7 @@ bool fmIsProcessByRowFunc(int32_t funcId); bool fmisSelectGroupConstValueFunc(int32_t funcId); void getLastCacheDataType(SDataType* pType, int32_t pkBytes); -SFunctionNode* createFunction(const char* pName, SNodeList* pParameterList); +int32_t createFunction(const char* pName, SNodeList* pParameterList, SFunctionNode** pFunc); int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMidFunc, SFunctionNode** pMergeFunc); @@ -273,7 +273,7 @@ typedef enum EFuncDataRequired { } EFuncDataRequired; EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow); -EFuncDataRequired fmFuncDynDataRequired(int32_t funcId, void* pRes, SDataBlockInfo* pBlockInfo); +int32_t fmFuncDynDataRequired(int32_t funcId, void* pRes, SDataBlockInfo* pBlockInfo, int32_t *reqStatus); int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet); int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 698ace3e4e..f4489a4f2a 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1684,14 +1684,17 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, if (fmIsAggFunc(pCtx->functionId) || fmIsIndefiniteRowsFunc(pCtx->functionId)) { bool isUdaf = fmIsUserDefinedFunc(pCtx->functionId); if (!isUdaf) { + // TODO(xxx) : need handle return value of fmGetFuncExecFuncs. fmGetFuncExecFuncs(pCtx->functionId, &pCtx->fpSet); } else { char* udfName = pExpr->pExpr->_function.pFunctNode->functionName; pCtx->udfName = taosStrdup(udfName); + // TODO(xxx) : need handle return value of fmGetUdafExecFuncs. fmGetUdafExecFuncs(pCtx->functionId, &pCtx->fpSet); } pCtx->fpSet.getEnv(pExpr->pExpr->_function.pFunctNode, &env); } else { + // TODO(xxx) : need handle return value of fmGetScalarFuncExecFuncs. fmGetScalarFuncExecFuncs(pCtx->functionId, &pCtx->sfp); if (pCtx->sfp.getEnv != NULL) { pCtx->sfp.getEnv(pExpr->pExpr->_function.pFunctNode, &env); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 520cba92a1..39883566d1 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -202,8 +202,9 @@ static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo* SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, i, pTableScanInfo->base.pdInfo.pExprSup->rowEntryInfoOffset); - int32_t reqStatus = fmFuncDynDataRequired(functionId, pEntry, pBlockInfo); - if (reqStatus != FUNC_DATA_REQUIRED_NOT_LOAD) { + int32_t reqStatus; + code = fmFuncDynDataRequired(functionId, pEntry, pBlockInfo, &reqStatus); + if (code != TSDB_CODE_SUCCESS || reqStatus != FUNC_DATA_REQUIRED_NOT_LOAD) { notLoadBlock = false; break; } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 2e68763924..69e33f7972 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2278,7 +2278,7 @@ static int32_t apercentileTransferInfo(SAPercentileInfo* pInput, SAPercentileInf SHistogramInfo* pRes = NULL; int32_t code = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN, &pRes); - if (TSDB_CODE_SUCCESS != pRes) { + if (TSDB_CODE_SUCCESS != code) { tHistogramDestroy(&pRes); return code; } diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 8255997836..9e06a36e11 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -61,7 +61,7 @@ static bool isSpecificClassifyFunc(int32_t funcId, uint64_t classification) { } int32_t fmFuncMgtInit() { - taosThreadOnce(&functionHashTableInit, doInitFunctionTable); + (void)taosThreadOnce(&functionHashTableInit, doInitFunctionTable); return initFunctionCode; } @@ -115,20 +115,24 @@ EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWin return funcMgtBuiltins[pFunc->funcId].dataRequiredFunc(pFunc, pTimeWindow); } -EFuncDataRequired fmFuncDynDataRequired(int32_t funcId, void* pRes, SDataBlockInfo* pBlockInfo) { +int32_t fmFuncDynDataRequired(int32_t funcId, void* pRes, SDataBlockInfo* pBlockInfo, int32_t *reqStatus) { if (fmIsUserDefinedFunc(funcId) || funcId < 0 || funcId >= funcMgtBuiltinsNum) { + *reqStatus = -1; return TSDB_CODE_FAILED; } const char* name = funcMgtBuiltins[funcId].name; if ((strcmp(name, "_group_key") == 0) || (strcmp(name, "_select_value") == 0)) { - return FUNC_DATA_REQUIRED_NOT_LOAD; + *reqStatus = FUNC_DATA_REQUIRED_NOT_LOAD; + return TSDB_CODE_SUCCESS;; } if (funcMgtBuiltins[funcId].dynDataRequiredFunc == NULL) { - return FUNC_DATA_REQUIRED_DATA_LOAD; + *reqStatus = FUNC_DATA_REQUIRED_DATA_LOAD; + return TSDB_CODE_SUCCESS; } else { - return funcMgtBuiltins[funcId].dynDataRequiredFunc(pRes, pBlockInfo); + *reqStatus = funcMgtBuiltins[funcId].dynDataRequiredFunc(pRes, pBlockInfo); + return TSDB_CODE_SUCCESS; } } @@ -378,29 +382,30 @@ static int32_t getFuncInfo(SFunctionNode* pFunc) { return fmGetFuncInfo(pFunc, msg, sizeof(msg)); } -SFunctionNode* createFunction(const char* pName, SNodeList* pParameterList) { - SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); - if (NULL == pFunc) { - return NULL; +int32_t createFunction(const char* pName, SNodeList* pParameterList, SFunctionNode** pFunc) { + *pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == *pFunc) { + return TSDB_CODE_OUT_OF_MEMORY; } - snprintf(pFunc->functionName, sizeof(pFunc->functionName), "%s", pName); - pFunc->pParameterList = pParameterList; - if (TSDB_CODE_SUCCESS != getFuncInfo(pFunc)) { - pFunc->pParameterList = NULL; - nodesDestroyNode((SNode*)pFunc); - return NULL; + (void)snprintf((*pFunc)->functionName, sizeof((*pFunc)->functionName), "%s", pName); + (*pFunc)->pParameterList = pParameterList; + int32_t code = getFuncInfo((*pFunc)); + if (TSDB_CODE_SUCCESS != code) { + (*pFunc)->pParameterList = NULL; + nodesDestroyNode((SNode*)*pFunc); + return code; } - return pFunc; + return code; } -static SNode* createColumnByFunc(const SFunctionNode* pFunc) { - SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); - if (NULL == pCol) { - return NULL; +static int32_t createColumnByFunc(const SFunctionNode* pFunc, SColumnNode** pCol) { + *pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == *pCol) { + return TSDB_CODE_OUT_OF_MEMORY; } - strcpy(pCol->colName, pFunc->node.aliasName); - pCol->node.resType = pFunc->node.resType; - return (SNode*)pCol; + (void)strcpy((*pCol)->colName, pFunc->node.aliasName); + (*pCol)->node.resType = pFunc->node.resType; + return TSDB_CODE_SUCCESS; } bool fmIsDistExecFunc(int32_t funcId) { @@ -418,17 +423,17 @@ static int32_t createPartialFunction(const SFunctionNode* pSrcFunc, SFunctionNod if (NULL == pParameterList) { return TSDB_CODE_OUT_OF_MEMORY; } - *pPartialFunc = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pPartialFunc, pParameterList); - if (NULL == *pPartialFunc) { + int32_t code = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pPartialFunc, pParameterList,pPartialFunc ); + if (TSDB_CODE_SUCCESS != code) { nodesDestroyList(pParameterList); - return TSDB_CODE_OUT_OF_MEMORY; + return code; } (*pPartialFunc)->hasOriginalFunc = true; (*pPartialFunc)->originalFuncId = pSrcFunc->hasOriginalFunc ? pSrcFunc->originalFuncId : pSrcFunc->funcId; char name[TSDB_FUNC_NAME_LEN + TSDB_NAME_DELIMITER_LEN + TSDB_POINTER_PRINT_BYTES + 1] = {0}; int32_t len = snprintf(name, sizeof(name) - 1, "%s.%p", (*pPartialFunc)->functionName, pSrcFunc); - taosCreateMD5Hash(name, len); - strncpy((*pPartialFunc)->node.aliasName, name, TSDB_COL_NAME_LEN - 1); + (void)taosCreateMD5Hash(name, len); + (void)strncpy((*pPartialFunc)->node.aliasName, name, TSDB_COL_NAME_LEN - 1); (*pPartialFunc)->hasPk = pSrcFunc->hasPk; (*pPartialFunc)->pkBytes = pSrcFunc->pkBytes; return TSDB_CODE_SUCCESS; @@ -436,7 +441,11 @@ static int32_t createPartialFunction(const SFunctionNode* pSrcFunc, SFunctionNod static int32_t createMergeFuncPara(const SFunctionNode* pSrcFunc, const SFunctionNode* pPartialFunc, SNodeList** pParameterList) { - SNode* pRes = createColumnByFunc(pPartialFunc); + SNode *pRes = NULL; + int32_t code = createColumnByFunc(pPartialFunc, (SColumnNode**)&pRes); + if (TSDB_CODE_SUCCESS != code) { + return code; + } if (NULL != funcMgtBuiltins[pSrcFunc->funcId].createMergeParaFuc) { return funcMgtBuiltins[pSrcFunc->funcId].createMergeParaFuc(pSrcFunc->pParameterList, pRes, pParameterList); } else { @@ -452,16 +461,13 @@ static int32_t createMidFunction(const SFunctionNode* pSrcFunc, const SFunctionN int32_t code = createMergeFuncPara(pSrcFunc, pPartialFunc, &pParameterList); if (TSDB_CODE_SUCCESS == code) { if(funcMgtBuiltins[pSrcFunc->funcId].pMiddleFunc != NULL){ - pFunc = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMiddleFunc, pParameterList); + code = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMiddleFunc, pParameterList, &pFunc); }else{ - pFunc = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMergeFunc, pParameterList); - } - if (NULL == pFunc) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMergeFunc, pParameterList, &pFunc); } } if (TSDB_CODE_SUCCESS == code) { - strcpy(pFunc->node.aliasName, pPartialFunc->node.aliasName); + (void)strcpy(pFunc->node.aliasName, pPartialFunc->node.aliasName); } if (TSDB_CODE_SUCCESS == code) { @@ -481,10 +487,7 @@ static int32_t createMergeFunction(const SFunctionNode* pSrcFunc, const SFunctio int32_t code = createMergeFuncPara(pSrcFunc, pPartialFunc, &pParameterList); if (TSDB_CODE_SUCCESS == code) { - pFunc = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMergeFunc, pParameterList); - if (NULL == pFunc) { - code = TSDB_CODE_OUT_OF_MEMORY; - } + code = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMergeFunc, pParameterList, &pFunc); } if (TSDB_CODE_SUCCESS == code) { pFunc->hasOriginalFunc = true; @@ -493,7 +496,7 @@ static int32_t createMergeFunction(const SFunctionNode* pSrcFunc, const SFunctio if (fmIsSameInOutType(pSrcFunc->funcId)) { pFunc->node.resType = pSrcFunc->node.resType; } - strcpy(pFunc->node.aliasName, pSrcFunc->node.aliasName); + (void)strcpy(pFunc->node.aliasName, pSrcFunc->node.aliasName); } if (TSDB_CODE_SUCCESS == code) { @@ -541,13 +544,13 @@ static int32_t fmCreateStateFunc(const SFunctionNode* pFunc, SFunctionNode** pSt if (funcMgtBuiltins[pFunc->funcId].pStateFunc) { SNodeList* pParams = nodesCloneList(pFunc->pParameterList); if (!pParams) return TSDB_CODE_OUT_OF_MEMORY; - *pStateFunc = createFunction(funcMgtBuiltins[pFunc->funcId].pStateFunc, pParams); - if (!*pStateFunc) { + int32_t code = createFunction(funcMgtBuiltins[pFunc->funcId].pStateFunc, pParams, pStateFunc); + if (TSDB_CODE_SUCCESS != code) { nodesDestroyList(pParams); return TSDB_CODE_FUNC_FUNTION_ERROR; } - strcpy((*pStateFunc)->node.aliasName, pFunc->node.aliasName); - strcpy((*pStateFunc)->node.userAlias, pFunc->node.userAlias); + (void)strcpy((*pStateFunc)->node.aliasName, pFunc->node.aliasName); + (void)strcpy((*pStateFunc)->node.userAlias, pFunc->node.userAlias); } return TSDB_CODE_SUCCESS; } @@ -587,13 +590,13 @@ static int32_t fmCreateStateMergeFunc(SFunctionNode* pFunc, SFunctionNode** pSta if (funcMgtBuiltins[pFunc->funcId].pMergeFunc) { SNodeList* pParams = nodesCloneList(pFunc->pParameterList); if (!pParams) return TSDB_CODE_OUT_OF_MEMORY; - *pStateMergeFunc = createFunction(funcMgtBuiltins[pFunc->funcId].pMergeFunc, pParams); - if (!*pStateMergeFunc) { + int32_t code = createFunction(funcMgtBuiltins[pFunc->funcId].pMergeFunc, pParams, pStateMergeFunc); + if (TSDB_CODE_SUCCESS != code) { nodesDestroyList(pParams); - return TSDB_CODE_FUNC_FUNTION_ERROR; + return code; } - strcpy((*pStateMergeFunc)->node.aliasName, pFunc->node.aliasName); - strcpy((*pStateMergeFunc)->node.userAlias, pFunc->node.userAlias); + (void)strcpy((*pStateMergeFunc)->node.aliasName, pFunc->node.aliasName); + (void)strcpy((*pStateMergeFunc)->node.userAlias, pFunc->node.userAlias); } return TSDB_CODE_SUCCESS; } @@ -646,6 +649,9 @@ bool fmIsMyStateFunc(int32_t funcId, int32_t stateFuncId) { } if (strcmp(pFunc->pStateFunc, pStateFunc->name) == 0) return true; int32_t stateMergeFuncId = fmGetFuncId(pFunc->pStateFunc); + if (stateMergeFuncId == -1) { + return false; + } const SBuiltinFuncDefinition* pStateMergeFunc = &funcMgtBuiltins[stateMergeFuncId]; return strcmp(pStateFunc->name, pStateMergeFunc->pMergeFunc) == 0; } diff --git a/source/libs/function/src/thistogram.c b/source/libs/function/src/thistogram.c index ed36921b73..f57f6aa118 100644 --- a/source/libs/function/src/thistogram.c +++ b/source/libs/function/src/thistogram.c @@ -49,7 +49,7 @@ int32_t tHistogramCreate(int32_t numOfEntries, SHistogramInfo** pHisto) { pss->pTree = pHisto->pLoserTree; #endif - *pHisto = tHistogramCreateFrom(pHisto, numOfEntries); + *pHisto = tHistogramCreateFrom(*pHisto, numOfEntries); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 844a608456..e67001889e 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -10420,8 +10420,9 @@ static int32_t createLastTsSelectStmt(char* pDb, const char* pTable, const char* return code; } - SNode* pFunc = (SNode*)createFunction("last", pParameterList); - if (NULL == pFunc) { + SNode* pFunc = NULL; + code = createFunction("last", pParameterList, (SFunctionNode**)&pFunc); + if (code) { nodesDestroyList(pParameterList); return TSDB_CODE_OUT_OF_MEMORY; } @@ -10438,8 +10439,9 @@ static int32_t createLastTsSelectStmt(char* pDb, const char* pTable, const char* return code; } - SFunctionNode* pFunc1 = createFunction("_vgid", NULL); - if (NULL == pFunc1) { + SFunctionNode* pFunc1 = NULL; + code = createFunction("_vgid", NULL, &pFunc1); + if (code) { nodesDestroyList(pProjectionList); return TSDB_CODE_OUT_OF_MEMORY; } @@ -10451,8 +10453,9 @@ static int32_t createLastTsSelectStmt(char* pDb, const char* pTable, const char* return code; } - SFunctionNode* pFunc2 = createFunction("_vgver", NULL); - if (NULL == pFunc2) { + SFunctionNode* pFunc2 = NULL; + code = createFunction("_vgver", NULL, &pFunc2); + if (code) { nodesDestroyList(pProjectionList); return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index b1db171875..b72cad012d 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -5336,9 +5336,13 @@ static bool stbJoinOptShouldBeOptimized(SLogicNode* pNode) { } int32_t stbJoinOptAddFuncToScanNode(char* funcName, SScanLogicNode* pScan) { - SFunctionNode* pUidFunc = createFunction(funcName, NULL); + SFunctionNode* pUidFunc = NULL; + int32_t code = createFunction(funcName, NULL, &pUidFunc); + if (TSDB_CODE_SUCCESS != code) { + return code; + } snprintf(pUidFunc->node.aliasName, sizeof(pUidFunc->node.aliasName), "%s.%p", pUidFunc->functionName, pUidFunc); - int32_t code = nodesListStrictAppend(pScan->pScanPseudoCols, (SNode*)pUidFunc); + code = nodesListStrictAppend(pScan->pScanPseudoCols, (SNode*)pUidFunc); if (TSDB_CODE_SUCCESS == code) { code = createColumnByRewriteExpr((SNode*)pUidFunc, &pScan->node.pTargets); } From de0fa610f8ec9b9a885195494525bc82358d7487 Mon Sep 17 00:00:00 2001 From: sima Date: Mon, 22 Jul 2024 18:59:41 +0800 Subject: [PATCH 34/43] enh:[TD-31043] Handling return value of taoshash and taosarray --- source/libs/function/src/builtinsimpl.c | 164 ++++++++++++++++-- .../libs/function/src/detail/tavgfunction.c | 9 +- source/libs/function/src/tpercentile.c | 10 ++ 3 files changed, 162 insertions(+), 21 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 69e33f7972..5d5b11fb46 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -655,6 +655,9 @@ int32_t appendSelectivityCols(SqlFunctionCtx* pCtx, SSDataBlock* pSrcBlock, int3 int32_t srcSlotId = pFuncParam->pCol->slotId; SColumnInfoData* pSrcCol = taosArrayGet(pSrcBlock->pDataBlock, srcSlotId); + if (NULL == pSrcCol) { + return TSDB_CODE_OUT_OF_RANGE; + } char* pData = colDataGetData(pSrcCol, rowIndex); @@ -662,7 +665,9 @@ int32_t appendSelectivityCols(SqlFunctionCtx* pCtx, SSDataBlock* pSrcBlock, int3 int32_t dstSlotId = pc->pExpr->base.resSchema.slotId; SColumnInfoData* pDstCol = taosArrayGet(pCtx->pDstBlock->pDataBlock, dstSlotId); - + if (NULL == pDstCol) { + return TSDB_CODE_OUT_OF_RANGE; + } if (colDataIsNull_s(pSrcCol, rowIndex) == true) { colDataSetNULL(pDstCol, pos); } else { @@ -696,7 +701,9 @@ int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t code = TSDB_CODE_SUCCESS; int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - + if (NULL == pCol) { + return TSDB_CODE_OUT_OF_RANGE; + } SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; @@ -724,7 +731,9 @@ int32_t firstCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, char* finalResult) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - + if (NULL == pCol) { + return TSDB_CODE_OUT_OF_RANGE; + } SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; @@ -1056,6 +1065,9 @@ int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t currentRow = pBlock->info.rows; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + if (NULL == pCol) { + return TSDB_CODE_OUT_OF_RANGE; + } pEntryInfo->isNullRes = (pEntryInfo->numOfRes == 0) ? 1 : 0; // NOTE: do nothing change it, for performance issue @@ -1151,6 +1163,9 @@ int32_t setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STu } SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId); + if (NULL == pDstCol) { + return TSDB_CODE_OUT_OF_RANGE; + } if (nullList[j]) { colDataSetNULL(pDstCol, rowIndex); } else { @@ -1182,6 +1197,9 @@ int32_t appendSelectivityValue(SqlFunctionCtx* pCtx, int32_t rowIndex, int32_t p int32_t srcSlotId = pFuncParam->pCol->slotId; SColumnInfoData* pSrcCol = taosArrayGet(pCtx->pSrcBlock->pDataBlock, srcSlotId); + if (NULL == pSrcCol) { + return TSDB_CODE_OUT_OF_RANGE; + } char* pData = colDataGetData(pSrcCol, rowIndex); @@ -1189,6 +1207,9 @@ int32_t appendSelectivityValue(SqlFunctionCtx* pCtx, int32_t rowIndex, int32_t p int32_t dstSlotId = pc->pExpr->base.resSchema.slotId; SColumnInfoData* pDstCol = taosArrayGet(pCtx->pDstBlock->pDataBlock, dstSlotId); + if (NULL == pDstCol) { + return TSDB_CODE_OUT_OF_RANGE; + } if (colDataIsNull_s(pSrcCol, rowIndex) == true) { colDataSetNULL(pDstCol, pos); @@ -1638,6 +1659,10 @@ int32_t stddevPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + if (NULL == pCol) { + taosMemoryFree(res); + return TSDB_CODE_OUT_OF_RANGE; + } int32_t code = colDataSetVal(pCol, pBlock->info.rows, res, false); @@ -1846,6 +1871,9 @@ int32_t leastSQRFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + if (NULL == pCol) { + return TSDB_CODE_OUT_OF_RANGE; + } int32_t currentRow = pBlock->info.rows; if (0 == pInfo->num) { @@ -2067,6 +2095,10 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + if (NULL == pCol) { + code = TSDB_CODE_OUT_OF_RANGE; + goto _fin_error; + } varDataSetLen(buf, len); code = colDataSetVal(pCol, pBlock->info.rows, buf, false); @@ -2389,6 +2421,10 @@ int32_t apercentilePartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + if (NULL == pCol) { + taosMemoryFree(res); + return TSDB_CODE_OUT_OF_RANGE; + } int32_t code = colDataSetVal(pCol, pBlock->info.rows, res, false); @@ -3002,6 +3038,9 @@ int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t code = TSDB_CODE_SUCCESS; int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + if (NULL == pCol) { + return TSDB_CODE_OUT_OF_RANGE; + } SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; @@ -3037,6 +3076,10 @@ int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + if (NULL == pCol) { + taosMemoryFree(res); + return TSDB_CODE_OUT_OF_RANGE; + } code = colDataSetVal(pCol, pBlock->info.rows, res, false); if (TSDB_CODE_SUCCESS != code) { @@ -3529,6 +3572,10 @@ int32_t diffFunctionByRow(SArray* pCtxArray) { bool keepNull = false; for (int i = 0; i < diffColNum; ++i) { SqlFunctionCtx* pCtx = *(SqlFunctionCtx**)taosArrayGet(pCtxArray, i); + if (NULL == pCtx) { + code = TSDB_CODE_OUT_OF_RANGE; + goto _exit; + } funcInputUpdate(pCtx); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SDiffInfo* pDiffInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -3539,6 +3586,10 @@ int32_t diffFunctionByRow(SArray* pCtxArray) { SqlFunctionCtx* pCtx0 = *(SqlFunctionCtx**)taosArrayGet(pCtxArray, 0); SFuncInputRow* pRow0 = (SFuncInputRow*)taosArrayGet(pRows, 0); + if (NULL == pCtx0 || NULL == pRow0) { + code = TSDB_CODE_OUT_OF_RANGE; + goto _exit; + } int32_t startOffset = pCtx0->offset; bool result = false; while (1) { @@ -3553,6 +3604,10 @@ int32_t diffFunctionByRow(SArray* pCtxArray) { for (int i = 1; i < diffColNum; ++i) { SqlFunctionCtx* pCtx = *(SqlFunctionCtx**)taosArrayGet(pCtxArray, i); SFuncInputRow* pRow = (SFuncInputRow*)taosArrayGet(pRows, i); + if (NULL == pCtx || NULL == pRow) { + code = TSDB_CODE_OUT_OF_RANGE; + goto _exit; + } code = funcInputGetNextRow(pCtx, pRow, &result); if (TSDB_CODE_SUCCESS != code) { goto _exit; @@ -3572,6 +3627,10 @@ int32_t diffFunctionByRow(SArray* pCtxArray) { for (int i = 0; i < diffColNum; ++i) { SqlFunctionCtx* pCtx = *(SqlFunctionCtx**)taosArrayGet(pCtxArray, i); SFuncInputRow* pRow = (SFuncInputRow*)taosArrayGet(pRows, i); + if (NULL == pCtx || NULL == pRow) { + code = TSDB_CODE_OUT_OF_RANGE; + goto _exit; + } if ((keepNull || hasNotNullValue) && !isFirstRow(pCtx, pRow)){ code = setDoDiffResult(pCtx, pRow, pos); if (code != TSDB_CODE_SUCCESS) { @@ -3590,6 +3649,10 @@ int32_t diffFunctionByRow(SArray* pCtxArray) { for (int i = 0; i < diffColNum; ++i) { SqlFunctionCtx* pCtx = *(SqlFunctionCtx**)taosArrayGet(pCtxArray, i); + if (NULL == pCtx) { + code = TSDB_CODE_OUT_OF_RANGE; + goto _exit; + } SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); pResInfo->numOfRes = numOfElems; } @@ -3819,8 +3882,8 @@ int32_t doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSD * |(n columns, one bit for each column)| src column #1| src column #2| * +------------------------------------+--------------+--------------+ */ -void* serializeTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SSubsidiaryResInfo* pSubsidiaryies, - char* buf) { +int32_t serializeTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SSubsidiaryResInfo* pSubsidiaryies, + char* buf, char** res) { char* nullList = buf; char* pStart = (char*)(nullList + sizeof(bool) * pSubsidiaryies->num); @@ -3838,6 +3901,9 @@ void* serializeTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SSubsid int32_t srcSlotId = pFuncParam->pCol->slotId; SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, srcSlotId); + if (NULL == pCol) { + return TSDB_CODE_OUT_OF_RANGE; + } if ((nullList[i] = colDataIsNull_s(pCol, rowIndex)) == true) { offset += pCol->info.bytes; continue; @@ -3853,7 +3919,8 @@ void* serializeTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SSubsid offset += pCol->info.bytes; } - return buf; + *res = buf; + return TSDB_CODE_SUCCESS; } static int32_t doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, SWinKey* key, @@ -3909,12 +3976,19 @@ int32_t saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* SWinKey key = {0}; if (pCtx->saveHandle.pBuf == NULL) { SColumnInfoData* pColInfo = taosArrayGet(pSrcBlock->pDataBlock, pCtx->saveHandle.pState->tsIndex); + if (NULL == pColInfo) { + return TSDB_CODE_OUT_OF_RANGE; + } ASSERT(pColInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP); key.groupId = pSrcBlock->info.id.groupId; key.ts = *(int64_t*)colDataGetData(pColInfo, rowIndex); } - char* buf = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf); + char* buf = NULL; + code = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf, &buf); + if (TSDB_CODE_SUCCESS != code) { + return code; + } return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, &key, pPos, pCtx->pStore); } @@ -3943,7 +4017,11 @@ int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBloc return code; } - char* buf = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf); + char* buf = NULL; + code = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf, &buf); + if (TSDB_CODE_SUCCESS != code) { + return code; + } return doUpdateTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, pPos, pCtx->pStore); } @@ -3982,6 +4060,9 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + if (NULL == pCol) { + return TSDB_CODE_OUT_OF_RANGE; + } // todo assign the tag value and the corresponding row data int32_t currentRow = pBlock->info.rows; @@ -4242,13 +4323,19 @@ int32_t spreadPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { varDataSetLen(res, resultBytes); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + int32_t code = TSDB_CODE_SUCCESS; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - - int32_t code = colDataSetVal(pCol, pBlock->info.rows, res, false); - if (TSDB_CODE_SUCCESS != code) { - return code; + if (NULL == pCol) { + code = TSDB_CODE_OUT_OF_RANGE; + goto _exit; } + code = colDataSetVal(pCol, pBlock->info.rows, res, false); + if (TSDB_CODE_SUCCESS != code) { + goto _exit; + } + +_exit: taosMemoryFree(res); return TSDB_CODE_SUCCESS; } @@ -4426,12 +4513,18 @@ int32_t elapsedPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { varDataSetLen(res, resultBytes); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + int32_t code = TSDB_CODE_SUCCESS; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - - int32_t code = colDataSetVal(pCol, pBlock->info.rows, res, false); - if (TSDB_CODE_SUCCESS != code) { - return code; + if (NULL == pCol) { + code = TSDB_CODE_OUT_OF_RANGE; + goto _exit; } + + code = colDataSetVal(pCol, pBlock->info.rows, res, false); + if (TSDB_CODE_SUCCESS != code) { + goto _exit; + } +_exit: taosMemoryFree(res); return code; } @@ -4749,6 +4842,9 @@ int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t code = TSDB_CODE_SUCCESS; int32_t currentRow = pBlock->info.rows; + if (NULL == pCol) { + return TSDB_CODE_OUT_OF_RANGE; + } if (pInfo->normalized) { for (int32_t k = 0; k < pResInfo->numOfRes; ++k) { @@ -4794,10 +4890,15 @@ int32_t histogramPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { varDataSetLen(res, resultBytes); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + int32_t code = TSDB_CODE_SUCCESS; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + if (NULL == pCol) { + code = TSDB_CODE_OUT_OF_RANGE; + goto _exit; + } + code = colDataSetVal(pCol, pBlock->info.rows, res, false); - int32_t code = colDataSetVal(pCol, pBlock->info.rows, res, false); - +_exit: taosMemoryFree(res); return code; } @@ -5022,10 +5123,16 @@ int32_t hllPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { varDataSetLen(res, resultBytes); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + int32_t code = TSDB_CODE_SUCCESS; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + if (NULL == pCol) { + code = TSDB_CODE_OUT_OF_RANGE; + goto _exit; + } - int32_t code = colDataSetVal(pCol, pBlock->info.rows, res, false); + code = colDataSetVal(pCol, pBlock->info.rows, res, false); +_exit: taosMemoryFree(res); return code; } @@ -5568,6 +5675,9 @@ int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + if (NULL == pCol) { + return TSDB_CODE_OUT_OF_RANGE; + } int32_t currentRow = pBlock->info.rows; if (pInfo->numSampled == 0) { @@ -5962,6 +6072,9 @@ int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); int32_t currentRow = pBlock->info.rows; + if (NULL == pCol) { + return TSDB_CODE_OUT_OF_RANGE; + } STuplePos resDataPos, resTuplePos; int32_t maxCount = 0; @@ -6332,6 +6445,9 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { STableBlockDistInfo* pData = GET_ROWCELL_INTERBUF(pResInfo); SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0); + if (NULL == pColInfo) { + return TSDB_CODE_OUT_OF_RANGE; + } if (pData->totalRows == 0) { pData->minRows = 0; @@ -6837,6 +6953,10 @@ int32_t iratePartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + if (NULL == pCol) { + taosMemoryFree(res); + return TSDB_CODE_OUT_OF_RANGE; + } int32_t code = colDataSetVal(pCol, pBlock->info.rows, res, false); @@ -6847,6 +6967,9 @@ int32_t iratePartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t irateFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + if (NULL == pCol) { + return TSDB_CODE_OUT_OF_RANGE; + } SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; @@ -6901,6 +7024,9 @@ int32_t groupConstValueFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; int32_t code = TSDB_CODE_SUCCESS; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + if (NULL == pCol) { + return TSDB_CODE_OUT_OF_RANGE; + } SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); diff --git a/source/libs/function/src/detail/tavgfunction.c b/source/libs/function/src/detail/tavgfunction.c index 6667e84bed..df9edb9948 100644 --- a/source/libs/function/src/detail/tavgfunction.c +++ b/source/libs/function/src/detail/tavgfunction.c @@ -852,7 +852,7 @@ int32_t avgPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SAvgRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); int32_t resultBytes = getAvgInfoSize(); char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); - + int32_t code = TSDB_CODE_SUCCESS; if (NULL == res) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -861,9 +861,14 @@ int32_t avgPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + if(NULL == pCol) { + code = TSDB_CODE_OUT_OF_RANGE; + goto _exit; + } - int32_t code = colDataSetVal(pCol, pBlock->info.rows, res, false); + code = colDataSetVal(pCol, pBlock->info.rows, res, false); +_exit: taosMemoryFree(res); return code; } diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 905436bea1..89959d32b6 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -49,6 +49,10 @@ static int32_t loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx, SFi int32_t offset = 0; for (int32_t i = 0; i < taosArrayGetSize(pIdList); ++i) { int32_t *pageId = taosArrayGet(pIdList, i); + if (pageId == NULL) { + taosMemoryFree(*buffer); + return TSDB_CODE_OUT_OF_RANGE; + } SFilePage *pg = getBufPage(pMemBucket->pBuffer, *pageId); if (pg == NULL) { @@ -119,6 +123,9 @@ int32_t findOnlyResult(tMemBucket *pMemBucket, double *result) { ASSERT(list->size == 1); int32_t *pageId = taosArrayGet(list, 0); + if (NULL == pageId) { + return TSDB_CODE_OUT_OF_RANGE; + } SFilePage *pPage = getBufPage(pMemBucket->pBuffer, *pageId); if (pPage == NULL) { return terrno; @@ -559,6 +566,9 @@ int32_t getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction for (int32_t f = 0; f < list->size; ++f) { int32_t *pageId = taosArrayGet(list, f); + if (NULL == pageId) { + return TSDB_CODE_OUT_OF_RANGE; + } SFilePage *pg = getBufPage(pMemBucket->pBuffer, *pageId); if (pg == NULL) { return terrno; From 0473225362d303c53aa6831230943a491ab2d0b0 Mon Sep 17 00:00:00 2001 From: sima Date: Tue, 23 Jul 2024 09:23:31 +0800 Subject: [PATCH 35/43] enh:[TD-31063] Handling return value of serialize/deserialize. --- source/libs/function/src/builtinsimpl.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 5d5b11fb46..945c1ac9cd 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -6350,7 +6350,10 @@ int32_t blockDistFunction(SqlFunctionCtx* pCtx) { STableBlockDistInfo* pDistInfo = GET_ROWCELL_INTERBUF(pResInfo); STableBlockDistInfo p1 = {0}; - (void)tDeserializeBlockDistInfo(varDataVal(pInputCol->pData), varDataLen(pInputCol->pData), &p1); + if (tDeserializeBlockDistInfo(varDataVal(pInputCol->pData), varDataLen(pInputCol->pData), &p1) < 0) { + qError("failed to deserialize block dist info"); + return TSDB_CODE_FAILED; + } pDistInfo->numOfBlocks += p1.numOfBlocks; pDistInfo->numOfTables += p1.numOfTables; From 70699ba61e71807c50beb7cab4a7ed82e5922afb Mon Sep 17 00:00:00 2001 From: sima Date: Tue, 23 Jul 2024 14:11:40 +0800 Subject: [PATCH 36/43] enh:[TD-31043] use check_rows_loop in stream_basic.py --- tests/system-test/8-stream/stream_basic.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/system-test/8-stream/stream_basic.py b/tests/system-test/8-stream/stream_basic.py index 2067c2395e..95f4f1addc 100644 --- a/tests/system-test/8-stream/stream_basic.py +++ b/tests/system-test/8-stream/stream_basic.py @@ -93,9 +93,8 @@ class TDTestCase: tdSql.execute("create stream stream2 fill_history 1 into stb subtable(concat('new-', tname)) AS SELECT " "_wstart, count(*), avg(i) FROM st PARTITION BY tbname tname INTERVAL(1m)", show=True) - time.sleep(2) - tdSql.query("select * from sta") - tdSql.checkRows(3) + sql= "select * from sta" + tdSql.check_rows_loop(3, sql, loopCount=100, waitTime=0.5) tdSql.query("select tbname from sta order by tbname") if not tdSql.getData(0, 0).startswith('nee_w-t1_sta_'): tdLog.exit("error1") @@ -106,8 +105,8 @@ class TDTestCase: if not tdSql.getData(2, 0).startswith('nee_w-t3_sta_'): tdLog.exit("error3") - tdSql.query("select * from stb") - tdSql.checkRows(3) + sql= "select * from stb" + tdSql.check_rows_loop(3, sql, loopCount=100, waitTime=0.5) tdSql.query("select tbname from stb order by tbname") if not tdSql.getData(0, 0).startswith('new-t1_stb_'): tdLog.exit("error4") From dc23446d6bbe0e478759afa11f6a8d2cf80aa463 Mon Sep 17 00:00:00 2001 From: sima Date: Tue, 23 Jul 2024 15:37:08 +0800 Subject: [PATCH 37/43] enh:[TD-31063] use properer errcode. --- source/libs/function/src/builtinsimpl.c | 4 ++-- source/libs/function/src/functionMgt.c | 4 ++-- source/libs/function/src/tpercentile.c | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 945c1ac9cd..1197891cab 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3566,7 +3566,7 @@ int32_t diffFunctionByRow(SArray* pCtxArray) { SArray* pRows = taosArrayInit_s(sizeof(SFuncInputRow), diffColNum); if (NULL == pRows) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } bool keepNull = false; @@ -5958,7 +5958,7 @@ int32_t modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { } else { pInfo->pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); if (NULL == pInfo->pHash) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } pInfo->nullTupleSaved = false; diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 9e06a36e11..0424b2f179 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -35,14 +35,14 @@ static void doInitFunctionTable() { gFunMgtService.pFuncNameHashTable = taosHashInit(funcMgtBuiltinsNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); if (NULL == gFunMgtService.pFuncNameHashTable) { - initFunctionCode = TSDB_CODE_FAILED; + initFunctionCode = terrno; return; } for (int32_t i = 0; i < funcMgtBuiltinsNum; ++i) { if (TSDB_CODE_SUCCESS != taosHashPut(gFunMgtService.pFuncNameHashTable, funcMgtBuiltins[i].name, strlen(funcMgtBuiltins[i].name), &i, sizeof(int32_t))) { - initFunctionCode = TSDB_CODE_FAILED; + initFunctionCode = terrno; return; } } diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 89959d32b6..40e0407a54 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -273,7 +273,7 @@ int32_t tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, dou (*pBucket)->groupPagesMap = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); if ((*pBucket)->groupPagesMap == NULL) { tMemBucketDestroy(*pBucket); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (setBoundingBox(&(*pBucket)->range, (*pBucket)->type, minval, maxval) != 0) { // qError("MemBucket:%p, invalid value range: %f-%f", pBucket, minval, maxval); @@ -409,7 +409,7 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { if (p == NULL) { pPageIdList = taosArrayInit(4, sizeof(int32_t)); if (NULL == pPageIdList) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } int32_t code = taosHashPut(pBucket->groupPagesMap, &groupId, sizeof(groupId), &pPageIdList, POINTER_BYTES); if (TSDB_CODE_SUCCESS != code) { From 02346072fce735ce9e2f6c40a4b9e4f9bf2c6e41 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 23 Jul 2024 16:02:07 +0800 Subject: [PATCH 38/43] fix:[TD-31017]process return value in mnode for tmq --- source/dnode/mnode/impl/src/mndConsumer.c | 5 ++--- source/dnode/mnode/impl/src/mndDef.c | 4 +++- source/dnode/mnode/impl/src/mndSubscribe.c | 13 +++++++------ 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index cea664f64d..1b77a0ad6a 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -362,10 +362,9 @@ static int32_t buildAskEpRsp(SRpcMsg *pMsg, SMqAskEpRsp *rsp, int32_t serverEpoc pHead->walever = 0; void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - code = tEncodeSMqAskEpRsp(&abuf, rsp); - if (code != 0) { + if (tEncodeSMqAskEpRsp(&abuf, rsp) < 0) { rpcFreeCont(buf); - return code; + return TSDB_CODE_TSC_INTERNAL_ERROR; } // send rsp diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 3aa0b5a01d..e90962ca5e 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -594,7 +594,9 @@ int32_t tCloneSubscribeObj(const SMqSubscribeObj *pSub, SMqSubscribeObj **ppSub) pSubNew->offsetRows = taosArrayDup(pSub->offsetRows, NULL); (void)memcpy(pSubNew->dbName, pSub->dbName, TSDB_DB_FNAME_LEN); pSubNew->qmsg = taosStrdup(pSub->qmsg); - + if (ppSub) { + *ppSub = pSubNew; + } END: return code; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index ec915eb9d8..a08e94e24f 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -397,9 +397,11 @@ END: } static int32_t processSubOffsetRows(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { - int32_t code = 0; SMqSubscribeObj *pSub = NULL; - MND_TMQ_RETURN_CHECK(mndAcquireSubscribeByKey(pMnode, pInput->pRebInfo->key, &pSub)); // put all offset rows + int32_t code = mndAcquireSubscribeByKey(pMnode, pInput->pRebInfo->key, &pSub); // put all offset rows + if( code != 0){ + return 0; + } taosRLockLatch(&pSub->lock); if (pOutput->pSub->offsetRows == NULL) { pOutput->pSub->offsetRows = taosArrayInit(4, sizeof(OffsetRows)); @@ -858,12 +860,11 @@ END: } static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOutputObj *rebOutput) { - int32_t code = 0; const char *key = rebInput->pRebInfo->key; SMqSubscribeObj *pSub = NULL; - MND_TMQ_RETURN_CHECK(mndAcquireSubscribeByKey(pMnode, key, &pSub)); + int32_t code = mndAcquireSubscribeByKey(pMnode, key, &pSub); - if (pSub == NULL) { + if (code != 0) { // split sub key and extract topic char topic[TSDB_TOPIC_FNAME_LEN] = {0}; char cgroup[TSDB_CGROUP_LEN] = {0}; @@ -893,7 +894,7 @@ static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOu taosRUnLockLatch(&pSub->lock); goto END; } - code =checkConsumer(pMnode, rebOutput->pSub); + code = checkConsumer(pMnode, rebOutput->pSub); if(code != 0){ taosRUnLockLatch(&pSub->lock); goto END; From 7a8190f5b34426c98656ed5347db664fce8fd1c3 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 23 Jul 2024 17:10:26 +0800 Subject: [PATCH 39/43] test: add deploy in CI --- tests/ci/ci_deploy_dependcy_file_list.txt | 8 + .../container_build_newmachine.sh | 0 tests/ci/deploy_ci_machines.sh | 69 ++++++++ tests/ci/remove_corefile.sh | 17 ++ tests/ci/restore.sh | 159 ++++++++++++++++++ tests/ci/start_http.sh | 16 ++ 6 files changed, 269 insertions(+) create mode 100644 tests/ci/ci_deploy_dependcy_file_list.txt rename tests/{parallel_test => ci}/container_build_newmachine.sh (100%) create mode 100644 tests/ci/deploy_ci_machines.sh create mode 100644 tests/ci/remove_corefile.sh create mode 100644 tests/ci/restore.sh create mode 100644 tests/ci/start_http.sh diff --git a/tests/ci/ci_deploy_dependcy_file_list.txt b/tests/ci/ci_deploy_dependcy_file_list.txt new file mode 100644 index 0000000000..1e7afdd7db --- /dev/null +++ b/tests/ci/ci_deploy_dependcy_file_list.txt @@ -0,0 +1,8 @@ +/home/m.json +/home/log_server.json +/var/lib/jenkins/workspace/restore.sh +/var/lib/jenkins/workspace/start_http.sh +/var/lib/jenkins/workspace/TDinternal +/var/lib/jenkins/workspace/remove_corefile.sh +/var/lib/jenkins/workspace/CI_disk_Monitor.py +/var/lib/jenkins/workspace/start_CI_Monitor.sh diff --git a/tests/parallel_test/container_build_newmachine.sh b/tests/ci/container_build_newmachine.sh similarity index 100% rename from tests/parallel_test/container_build_newmachine.sh rename to tests/ci/container_build_newmachine.sh diff --git a/tests/ci/deploy_ci_machines.sh b/tests/ci/deploy_ci_machines.sh new file mode 100644 index 0000000000..457e590e3e --- /dev/null +++ b/tests/ci/deploy_ci_machines.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +function scp_file_from_host { + # check at least three parameters + if [ "$#" -lt 3 ]; then + echo "Usage: $0 host passwd source_filename [dest_filename]" + exit 1 + fi + + host=$1 + passwd=$2 + source_filename=$3 + # If the fourth parameter is not provided, use the third parameter as the default value + dest_filename=${4:-$3} + + # use sshpass and scp for secure file transfer + sshpass -p "$passwd" scp -o StrictHostKeyChecking=no -r "$host":"$source_filename" "$dest_filename" +} + + +# install docker and sshpass +curl -fsSL https://mirrors.ustc.edu.cn/docker-ce/linux/ubuntu/gpg | sudo apt-key add - +sudo add-apt-repository "deb [arch=amd64] https://mirrors.ustc.edu.cn/docker-ce/linux/ubuntu $(lsb_release -cs) stable" +sudo apt-get update +sudo apt-get install -y docker-ce sshpass jq +sudo systemctl enable docker +sudo systemctl start docker + +# create a log directory +mkdir -p /var/lib/jenkins/workspace/log + +# Assuming you have a file called 'file_list.txt' with one filename per line +file_list="ci_deploy_dependcy_file_list.txt" +moniterip="192.168.1.59" +passwd_all="12345678" + +# Read the file list and call scp_file_from_host for each file +while IFS= read -r source_filename; do + scp_file_from_host "$moniterip" "$passwd_all" "$source_filename" +done < "$file_list" + +# modify the configuration file +ip=$(ifconfig |grep inet|grep 192 |awk '{print $2}') +sed -i "s/${moniterip}/$ip/" /home/log_server.json +sed -i "s/${moniterip}/$ip/" /home/m.json + +#mkdir corefile dir and configure the system to automatically set corefile dir at startup +mkdir -p /home/coredump/ && echo "echo '/home/coredump/core_%e-%p' | sudo tee /proc/sys/kernel/core_pattern " >> /root/.bashrc + + +# get image from 0.212 +image_ip="192.168.0.212" +scp_file_from_host $image_ip $passwd_all "/home/tang/work/image/taos_image.tar " "/home/taos_image.tar" +docker load -i /home/taos_image.tar + +#start http server +nohup /var/lib/jenkins/workspace/log/start_http.sh & + +# start CI monitor and remove corefile in crontable +(crontab -l;echo "0 1 * * * /usr/bin/bash /var/lib/jenkins/workspace/remove_corefile.sh") | crontab +(crontab -l;echo "@reboot /usr/bin/bash /var/lib/jenkins/workspace/start_CI_Monitor.sh") | crontab + + +# generate cache dir +cd /var/lib/jenkins/workspace/TDinternal/community/tests/parallel_test || exit +time ./container_build_newmachine.sh -w /var/lib/jenkins/workspace -e + +# test if the CI machine compilation is successful +time ./container_build.sh -w /var/lib/jenkins/workspace -e \ No newline at end of file diff --git a/tests/ci/remove_corefile.sh b/tests/ci/remove_corefile.sh new file mode 100644 index 0000000000..7374101547 --- /dev/null +++ b/tests/ci/remove_corefile.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +date_str=$(date -d "4 day ago" +%Y%m%d) +if [ ! -z "$1" ]; then + date_str="$1" +fi +script_dir=$(dirname $0) +cd "${script_dir}"/log || exit +# date >>core.list +# find . -name "core.*" | grep "$date_str" >>core.list +# find . -name "core.*" | grep "$date_str" | xargs rm -rf +# find . -name "build_*" | grep "$date_str" | xargs rm -rf +for file in *; do + if [[ $file == *"$date_str"* ]]; then + rm -rf "$file" + fi +done diff --git a/tests/ci/restore.sh b/tests/ci/restore.sh new file mode 100644 index 0000000000..4c9daf73fe --- /dev/null +++ b/tests/ci/restore.sh @@ -0,0 +1,159 @@ +#!/bin/bash + set -x +JENKINS_LOG=jenkins.log +# pr_num=14228 +# n=1 +function usage() { + echo "$0" + echo -e "\t -p PR number" + echo -e "\t -n build number" + echo -e "\t -c container name" + echo -e "\t -h help" +} +while getopts "p:n:c:h" opt; do + case $opt in + p) + pr_num=$OPTARG + ;; + n) + n=$OPTARG + ;; + c) + container_name=$OPTARG + ;; + h) + usage + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" + usage + exit 1 + ;; + esac +done +if [ -z "$container_name" ]; then + echo "container name not specified" + usage + exit 1 +fi +if [ -z "$pr_num" ]; then + echo "PR number not specified" + usage + exit 1 +fi +if [ -z "$n" ]; then + echo "build number not specified" + usage + exit 1 +fi +pr_num=`echo "$pr_num"|sed "s/PR-//"` +container_count=`docker ps -a -f name=$container_name|wc -l` +if [ $container_count -gt 1 ]; then + docker ps -a -f name=$container_name + echo "container $container_name exists" + exit 1 +fi +cd $(dirname $0) +info=`grep -n "^[0-9]\{8\}-[0-9]\{6\}" jenkins.log | grep -A 1 "PR-${pr_num}:${n}:"` +# 22131:20220625-113105 NewTest/PR-14228:PR-14228:1:3.0 +# 22270:20220625-121154 NewTest/PR-14221:PR-14221:2:3.0 +ci_hosts="\ +192.168.0.212 \ +192.168.0.215 \ +192.168.0.217 \ +192.168.0.219 \ +" +if [ -z "$info" ]; then + echo "PR-${pr_num}:${n} not found" + for host in $ci_hosts; do + ssh root@$host "sh -c \"grep -n \\\"^[0-9]\\\\\{8\\\\\}-[0-9]\\\\\{6\\\\\}\\\" /var/lib/jenkins/workspace/jenkins.log | grep \\\"PR-${pr_num}:${n}:\\\"\"" + if [ $? -eq 0 ]; then + echo "this PR is possibly on host $host" + break + fi + done + exit 1 +fi +line_num=`echo "$info"|wc -l` +curr=`echo "$info"|head -n1` +if [ "$line_num" == "2" ]; then + next=`echo "$info"|tail -n1` +fi + +# check if it is TDinternal CI +internal=0 +commit_prefix=community +echo "$curr"|grep -q TDinternalCI +if [ $? -eq 0 ]; then + internal=1 + commit_prefix=tdinternal +fi + +curr_line=`echo "$curr"|cut -d: -f1` +next_line='$' +if [ ! -z "$next" ]; then + next_line=`echo "$next"|cut -d: -f1` + next_line=$(( next_line - 1 )) +fi +# echo "$curr_line, $next_line" + +details=`sed -n "${curr_line},${next_line}p" $JENKINS_LOG` +merge_line=`echo "$details"|grep -A 10 "$commit_prefix log merged: "|grep "Merge .* into"|head -n1` +if [ -z "$merge_line" ]; then + echo "merge commit not found" + exit 1 +fi +echo "$merge_line" +branch=`echo "$merge_line"|awk '{print $2}'` +commit_id=`echo "$merge_line"|awk '{print $4}'` +# echo "$details" +community_id=`echo "$details"|grep "community log: commit"|awk '{print $NF}'` +internal_id=`echo "$details"|grep "tdinternal log: commit"|awk '{print $NF}'` +python_connector_id=`echo "$details"|grep "python connector log: commit"|awk '{print $NF}'` +# change_branch=`echo "$details"|grep "CHANGE_BRANCH"|sed "s/CHANGE_BRANCH://"` + +# if [ -z "${branch}" ]; then +# branch="$change_branch" +# fi + +PWD=`pwd` +log_dir=`ls log|grep "PR-${pr_num}_${n}_"` +if [ -z "$log_dir" ]; then + echo "no log dir found" +else + mount_dir="-v ${PWD}/log/$log_dir:/home/log" + build_dir=`ls log/$log_dir | grep "build_"` + if [ ! -z "$build_dir" ]; then + mount_dir="$mount_dir -v ${PWD}/log/$log_dir/$build_dir:/home/TDinternal/debug/build" + fi +fi + +docker run -d --privileged -it --name $container_name \ + $mount_dir \ + taos_test:v1.0 bash + +if [ $internal -eq 0 ]; then + docker exec $container_name /home/setup.sh -c $commit_id -m $branch -n + echo "TDinternal checkout: $internal_id" + docker exec $container_name sh -c "cd /home/TDinternal; git checkout $internal_id" +else + docker exec $container_name /home/setup.sh -e -c $commit_id -m $branch -n + echo "community checkout: $community_id" + docker exec $container_name sh -c "cd /home/TDinternal/community; git checkout $community_id" +fi +echo +echo "* run the following command to enter the container:" +echo " docker exec -it $container_name bash" +if [ -z "$log_dir" ]; then + echo "* no log dir found" +else + echo "* log and coredump files are located in /home/log" +fi +if [ -z "$build_dir" ]; then + echo "* no build dir found" +else + echo "* build files are located in /home/TDinternal/debug/build" +fi +echo "* source files are located in /home/TDinternal" + diff --git a/tests/ci/start_http.sh b/tests/ci/start_http.sh new file mode 100644 index 0000000000..f73ffc85c8 --- /dev/null +++ b/tests/ci/start_http.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +script_dir=$(dirname $0) +cd $script_dir || exit +script_name=$(basename $0) +ps -ef|grep -v grep|grep -v $$|grep -q "$script_name" +if [ $? -eq 0 ]; then + exit 0 +fi +while [ 1 ]; do + ps -ef|grep python|grep -q 8081 + if [ $? -ne 0 ]; then + python3 -m http.server 8081 + fi + sleep 60 +done \ No newline at end of file From 158466a52558434c2c7758852812057f1e0707c8 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 23 Jul 2024 17:12:56 +0800 Subject: [PATCH 40/43] test: add deploy in CI --- tests/ci/deploy_ci_machines.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/deploy_ci_machines.sh b/tests/ci/deploy_ci_machines.sh index 457e590e3e..1f1b9ebf27 100644 --- a/tests/ci/deploy_ci_machines.sh +++ b/tests/ci/deploy_ci_machines.sh @@ -32,7 +32,7 @@ mkdir -p /var/lib/jenkins/workspace/log # Assuming you have a file called 'file_list.txt' with one filename per line file_list="ci_deploy_dependcy_file_list.txt" moniterip="192.168.1.59" -passwd_all="12345678" +passwd_all="abcdefg" # Read the file list and call scp_file_from_host for each file while IFS= read -r source_filename; do From fbd17ca914328fe7a246f0217d35814e9327c6b1 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 23 Jul 2024 17:19:42 +0800 Subject: [PATCH 41/43] test: add deploy in CI --- ...ile_list.txt => ci_deploy_dependency_file_list.txt} | 0 tests/ci/deploy_ci_machines.sh | 10 +++++----- 2 files changed, 5 insertions(+), 5 deletions(-) rename tests/ci/{ci_deploy_dependcy_file_list.txt => ci_deploy_dependency_file_list.txt} (100%) diff --git a/tests/ci/ci_deploy_dependcy_file_list.txt b/tests/ci/ci_deploy_dependency_file_list.txt similarity index 100% rename from tests/ci/ci_deploy_dependcy_file_list.txt rename to tests/ci/ci_deploy_dependency_file_list.txt diff --git a/tests/ci/deploy_ci_machines.sh b/tests/ci/deploy_ci_machines.sh index 1f1b9ebf27..e34ec1d6e5 100644 --- a/tests/ci/deploy_ci_machines.sh +++ b/tests/ci/deploy_ci_machines.sh @@ -30,19 +30,19 @@ sudo systemctl start docker mkdir -p /var/lib/jenkins/workspace/log # Assuming you have a file called 'file_list.txt' with one filename per line -file_list="ci_deploy_dependcy_file_list.txt" -moniterip="192.168.1.59" +file_list="ci_deploy_dependency_file_list.txt" +monitorip="192.168.1.59" passwd_all="abcdefg" # Read the file list and call scp_file_from_host for each file while IFS= read -r source_filename; do - scp_file_from_host "$moniterip" "$passwd_all" "$source_filename" + scp_file_from_host "$monitorip" "$passwd_all" "$source_filename" done < "$file_list" # modify the configuration file ip=$(ifconfig |grep inet|grep 192 |awk '{print $2}') -sed -i "s/${moniterip}/$ip/" /home/log_server.json -sed -i "s/${moniterip}/$ip/" /home/m.json +sed -i "s/${monitorip}/$ip/" /home/log_server.json +sed -i "s/${monitorip}/$ip/" /home/m.json #mkdir corefile dir and configure the system to automatically set corefile dir at startup mkdir -p /home/coredump/ && echo "echo '/home/coredump/core_%e-%p' | sudo tee /proc/sys/kernel/core_pattern " >> /root/.bashrc From 9735c069c85f670d458966e53b6abe799dc6326d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 23 Jul 2024 18:47:20 +0800 Subject: [PATCH 42/43] fix: windows compile issue --- source/os/src/osFile.c | 100 +++++++++++++++++++++++++++++------------ 1 file changed, 72 insertions(+), 28 deletions(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 0581df573a..167c141569 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -127,40 +127,56 @@ int64_t taosCopyFile(const char *from, const char *to) { // fidfrom = open(from, O_RDONLY); TdFilePtr pFileFrom = taosOpenFile(from, TD_FILE_READ); - if (pFileFrom == NULL) goto _err; + if (pFileFrom == NULL) { + code = terrno; + goto _err; + } // fidto = open(to, O_WRONLY | O_CREAT | O_EXCL, 0755); TdFilePtr pFileTo = taosOpenFile(to, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_EXCL); - if (pFileTo == NULL) goto _err; + if (pFileTo == NULL) { + code = terrno; + goto _err; + } while (true) { bytes = taosReadFile(pFileFrom, buffer, sizeof(buffer)); - if (bytes < 0) goto _err; + if (bytes < 0) { + code = terrno; + goto _err; + } + if (bytes == 0) break; size += bytes; - if (taosWriteFile(pFileTo, (void *)buffer, bytes) < bytes) goto _err; + if (taosWriteFile(pFileTo, (void *)buffer, bytes) < bytes) { + code = terrno; + goto _err; + } if (bytes < sizeof(buffer)) break; } code = taosFsyncFile(pFileTo); - taosCloseFile(&pFileFrom); - taosCloseFile(&pFileTo); + (void)taosCloseFile(&pFileFrom); + (void)taosCloseFile(&pFileTo); if (code != 0) { + terrno = code; return -1; } + return size; _err: - if (pFileFrom != NULL) taosCloseFile(&pFileFrom); - if (pFileTo != NULL) taosCloseFile(&pFileTo); + if (pFileFrom != NULL) (void)taosCloseFile(&pFileFrom); + if (pFileTo != NULL) (void)taosCloseFile(&pFileTo); /* coverity[+retval] */ - taosRemoveFile(to); - + (void)taosRemoveFile(to); + + terrno = code; return -1; #endif } @@ -168,24 +184,31 @@ _err: TdFilePtr taosCreateFile(const char *path, int32_t tdFileOptions) { TdFilePtr fp = taosOpenFile(path, tdFileOptions); if (!fp) { - if (errno == ENOENT) { + if (terrno == TAOS_SYSTEM_ERROR(ENOENT)) { // Try to create directory recursively - char *s = taosStrdup(path); + char s[PATH_MAX]; + tstrncpy(s, path, sizeof(s)); if (taosMulMkDir(taosDirName(s)) != 0) { - taosMemoryFree(s); return NULL; } - taosMemoryFree(s); fp = taosOpenFile(path, tdFileOptions); if (!fp) { return NULL; } } } + return fp; } -int32_t taosRemoveFile(const char *path) { return remove(path); } +int32_t taosRemoveFile(const char *path) { + int32_t code = remove(path); + if (-1 == code) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } + return code; +} int32_t taosRenameFile(const char *oldName, const char *newName) { #ifdef WINDOWS @@ -217,8 +240,9 @@ int32_t taosRenameFile(const char *oldName, const char *newName) { return finished ? 0 : -1; #else int32_t code = rename(oldName, newName); - if (code < 0) { - printf("failed to rename file %s to %s, reason:%s\n", oldName, newName, strerror(errno)); + if (-1 == code) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; } return code; @@ -233,8 +257,9 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *a struct stat fileStat; int32_t code = stat(path, &fileStat); #endif - if (code < 0) { - return code; + if (-1 == code) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; } if (size != NULL) { @@ -272,13 +297,15 @@ int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { #else if (pFile == NULL || pFile->fd < 0) { - return -1; + terrno = TSDB_CODE_INVALID_PARA; + return terrno; } + struct stat fileStat; int32_t code = fstat(pFile->fd, &fileStat); - if (code < 0) { - printf("taosFStatFile run fstat fail."); - return code; + if (-1 == code) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; } if (stDev != NULL) { @@ -1150,6 +1177,8 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) if (pFile == NULL) { return 0; } + + int32_t code = 0; #ifdef WINDOWS #if FILE_WITH_LOCK @@ -1185,7 +1214,6 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) terrno = TSDB_CODE_INVALID_PARA; return -1; } - int32_t code = 0; int64_t ret = pread(pFile->fd, buf, count, offset); if (-1 == ret) { code = TAOS_SYSTEM_ERROR(errno); @@ -1205,20 +1233,36 @@ int32_t taosFsyncFile(TdFilePtr pFile) { return 0; } + int32_t code = 0; // this implementation is WRONG // fflush is not a replacement of fsync - if (pFile->fp != NULL) return fflush(pFile->fp); + if (pFile->fp != NULL) { + code = fflush(pFile->fp); + if (0 != code) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } + + return code; + } + #ifdef WINDOWS if (pFile->hFile != NULL) { if (pFile->tdFileOptions & TD_FILE_WRITE_THROUGH) { return 0; } return !FlushFileBuffers(pFile->hFile); + } #else if (pFile->fd >= 0) { - return fsync(pFile->fd); -#endif + code = fsync(pFile->fd); + if (-1 == code) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } } +#endif + return 0; } @@ -1471,4 +1515,4 @@ int taosSetAutoDelFile(char* path) { #else return unlink(path); #endif -} \ No newline at end of file +} From 04f51842e4164712d5f782ccfa2cbc061a3ca435 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 23 Jul 2024 23:56:35 +0800 Subject: [PATCH 43/43] fix:[TD-31017]process return value in mnd for tmq --- source/dnode/mnode/impl/src/mndSubscribe.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index dcf407009b..a110d442cf 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -100,7 +100,6 @@ END: static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg, SSubplan *pPlan) { - int32_t code = 0; SMqRebVgReq req = {0}; int32_t code = 0; SEncoder encoder = {0}; @@ -862,7 +861,6 @@ END: } static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOutputObj *rebOutput) { - int32_t code = 0; const char *key = rebInput->pRebInfo->key; SMqSubscribeObj *pSub = NULL; int32_t code = mndAcquireSubscribeByKey(pMnode, key, &pSub);