From 1ce919b5b78f41541c6de69123ed45ff8e07ff04 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 10 Oct 2024 17:17:21 +0800 Subject: [PATCH 01/13] fix(stream):check release function of stream --- source/common/src/tdatablock.c | 5 ++++- source/libs/executor/src/executor.c | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 7a67522231..8df4965e65 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2497,6 +2497,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { // for debug int32_t dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf, const char* taskIdStr) { + int32_t lino = 0; int32_t size = 2048 * 1024; int32_t code = 0; char* dumpBuf = NULL; @@ -2530,6 +2531,7 @@ int32_t dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); if (pColInfoData == NULL) { code = terrno; + lino = __LINE__; goto _exit; } @@ -2609,6 +2611,7 @@ int32_t dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf code = taosUcs4ToMbs((TdUcs4*)varDataVal(pData), dataSize, pBuf); if (code < 0) { uError("func %s failed to convert to ucs charset since %s", __func__, tstrerror(code)); + lino = __LINE__; goto _exit; } len += snprintf(dumpBuf + len, size - len, " %15s |", pBuf); @@ -2626,7 +2629,7 @@ _exit: *pDataBuf = dumpBuf; dumpBuf = NULL; } else { - uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); if (dumpBuf) { taosMemoryFree(dumpBuf); } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index cbf392f67e..362e3c2631 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1635,7 +1635,9 @@ int32_t getTableListInfo(const SExecTaskInfo* pTaskInfo, SArray** pList) { int32_t qStreamOperatorReleaseState(qTaskInfo_t tInfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tInfo; - pTaskInfo->pRoot->fpSet.releaseStreamStateFn(pTaskInfo->pRoot); + if (pTaskInfo->pRoot->fpSet.releaseStreamStateFn != NULL) { + pTaskInfo->pRoot->fpSet.releaseStreamStateFn(pTaskInfo->pRoot); + } return 0; } From 7c0c710465f9d42455d52e5f282874cb61ccef1c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 11 Oct 2024 09:07:42 +0800 Subject: [PATCH 02/13] fix: ctest tag not exist --- cmake/stub_CMakeLists.txt.in | 2 +- tests/parallel_test/cases.task | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cmake/stub_CMakeLists.txt.in b/cmake/stub_CMakeLists.txt.in index 6c54d33be7..4c7a2fb002 100644 --- a/cmake/stub_CMakeLists.txt.in +++ b/cmake/stub_CMakeLists.txt.in @@ -2,7 +2,7 @@ # stub ExternalProject_Add(stub GIT_REPOSITORY https://github.com/coolxv/cpp-stub.git - GIT_TAG 5e903b8e + GIT_TAG 3137465194014d66a8402941e80d2bccc6346f51 GIT_SUBMODULES "src" SOURCE_DIR "${TD_CONTRIB_DIR}/cpp-stub" BINARY_DIR "${TD_CONTRIB_DIR}/cpp-stub/src" diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index d5e7de3014..0d3ed1f8e6 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -6,8 +6,6 @@ ,,n,unit-test,bash test.sh -,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py - # # army-test # @@ -1543,6 +1541,7 @@ ,,n,develop-test,python3 ./test.py -f 2-query/ts-range.py ,,n,develop-test,python3 ./test.py -f 2-query/tag_scan.py ,,n,develop-test,python3 ./test.py -f 2-query/show_create_db.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/custom_col_tag.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/default_json.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/demo.py From ca3aee591f03b7768e301b5002ec6cb573a00a5c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 11 Oct 2024 09:15:09 +0800 Subject: [PATCH 03/13] fix mem leak --- .../executor/src/streamtimewindowoperator.c | 85 ++++++++++--------- 1 file changed, 45 insertions(+), 40 deletions(-) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 6fc50bb860..be27f277c0 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -49,7 +49,7 @@ #define STREAM_SESSION_OP_CHECKPOINT_NAME "StreamSessionOperator_Checkpoint" #define STREAM_STATE_OP_CHECKPOINT_NAME "StreamStateOperator_Checkpoint" -#define MAX_STREAM_HISTORY_RESULT 20000000 +#define MAX_STREAM_HISTORY_RESULT 20000000 typedef struct SStateWindowInfo { SResultWindowInfo winInfo; @@ -449,7 +449,7 @@ void destroyFlusedPos(void* pRes) { } void destroyFlusedppPos(void* ppRes) { - void *pRes = *(void **)ppRes; + void* pRes = *(void**)ppRes; destroyFlusedPos(pRes); } @@ -507,7 +507,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) { } taosArrayDestroy(pInfo->pMidPullDatas); - if (pInfo->pState !=NULL && pInfo->pState->dump == 1) { + if (pInfo->pState != NULL && pInfo->pState->dump == 1) { taosMemoryFreeClear(pInfo->pState->pTdbState->pOwner); taosMemoryFreeClear(pInfo->pState->pTdbState); } @@ -548,7 +548,8 @@ void reloadFromDownStream(SOperatorInfo* downstream, SStreamIntervalOperatorInfo bool hasSrcPrimaryKeyCol(SSteamOpBasicInfo* pInfo) { return pInfo->primaryPkIndex != -1; } -int32_t initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamIntervalOperatorInfo* pInfo, struct SSteamOpBasicInfo* pBasic) { +int32_t initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamIntervalOperatorInfo* pInfo, + struct SSteamOpBasicInfo* pBasic) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SStateStore* pAPI = &downstream->pTaskInfo->storageAPI.stateStore; @@ -1028,7 +1029,7 @@ static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pN } static int32_t doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, uint64_t groupId, - SSHashObj* pUpdatedMap, SSHashObj* pDeletedMap) { + SSHashObj* pUpdatedMap, SSHashObj* pDeletedMap) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info; @@ -1661,7 +1662,7 @@ static int32_t doStreamFinalIntervalAggNext(SOperatorInfo* pOperator, SSDataBloc pInfo->binfo.pRes->info.type = pBlock->info.type; } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || pBlock->info.type == STREAM_CLEAR) { - SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); + SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); QUERY_CHECK_NULL(delWins, code, lino, _end, terrno); SHashObj* finalMap = IS_FINAL_INTERVAL_OP(pOperator) ? pInfo->pFinalPullDataMap : NULL; code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, finalMap); @@ -1897,9 +1898,8 @@ _end: } } -int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, - SExecTaskInfo* pTaskInfo, int32_t numOfChild, - SReadHandle* pHandle, SOperatorInfo** pOptrInfo) { +int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, + int32_t numOfChild, SReadHandle* pHandle, SOperatorInfo** pOptrInfo) { QRY_PARAM_CHECK(pOptrInfo); int32_t code = TSDB_CODE_SUCCESS; @@ -1959,8 +1959,8 @@ int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiN pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsIndex); - int32_t numOfCols = 0; - SExprInfo* pExprInfo = NULL; + int32_t numOfCols = 0; + SExprInfo* pExprInfo = NULL; code = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &pExprInfo, &numOfCols); QUERY_CHECK_CODE(code, lino, _error); @@ -2042,11 +2042,13 @@ int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiN pOperator->info = pInfo; if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) { - pOperator->fpSet = createOperatorFpSet(NULL, doStreamMidIntervalAggNext, NULL, destroyStreamFinalIntervalOperatorInfo, - optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + pOperator->fpSet = + createOperatorFpSet(NULL, doStreamMidIntervalAggNext, NULL, destroyStreamFinalIntervalOperatorInfo, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); } else { - pOperator->fpSet = createOperatorFpSet(NULL, doStreamFinalIntervalAggNext, NULL, destroyStreamFinalIntervalOperatorInfo, - optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + pOperator->fpSet = + createOperatorFpSet(NULL, doStreamFinalIntervalAggNext, NULL, destroyStreamFinalIntervalOperatorInfo, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); } setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState); if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL || @@ -2220,10 +2222,10 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, in *(pSup->pState) = *pState; pSup->stateStore.streamStateSetNumber(pSup->pState, -1, tsIndex); int32_t funResSize = getMaxFunResSize(pExpSup, numOfOutput); - pSup->pState->pFileState = NULL; - code = pSup->stateStore.streamFileStateInit( - tsStreamBufferSize, sizeof(SSessionKey), pSup->resultRowSize, funResSize, sesionTs, pSup->pState, - pTwAggSup->deleteMark, taskIdStr, pHandle->checkpointId, STREAM_STATE_BUFF_SORT, &pSup->pState->pFileState); + pSup->pState->pFileState = NULL; + code = pSup->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SSessionKey), pSup->resultRowSize, funResSize, + sesionTs, pSup->pState, pTwAggSup->deleteMark, taskIdStr, + pHandle->checkpointId, STREAM_STATE_BUFF_SORT, &pSup->pState->pFileState); QUERY_CHECK_CODE(code, lino, _end); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); @@ -3309,8 +3311,8 @@ int32_t doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpe QUERY_CHECK_CONDITION((winCode == TSDB_CODE_SUCCESS), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); - code = - tSimpleHashPut(pInfo->streamAggSup.pResultRows, &winfo.sessionWin, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); + code = tSimpleHashPut(pInfo->streamAggSup.pResultRows, &winfo.sessionWin, sizeof(SSessionKey), &winfo, + sizeof(SResultWindowInfo)); QUERY_CHECK_CODE(code, lino, _end); } @@ -3772,8 +3774,8 @@ _end: } } -int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, - SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** pOptrInfo) { +int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, + SReadHandle* pHandle, SOperatorInfo** pOptrInfo) { QRY_PARAM_CHECK(pOptrInfo); SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; @@ -3802,12 +3804,12 @@ int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode } } SExprSupp* pExpSup = &pOperator->exprSupp; - + SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno); pInfo->binfo.pRes = pResBlock; - SExprInfo* pExprInfo = NULL; + SExprInfo* pExprInfo = NULL; code = createExprInfo(pSessionNode->window.pFuncs, NULL, &pExprInfo, &numOfCols); QUERY_CHECK_CODE(code, lino, _error); @@ -3886,8 +3888,9 @@ int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode QUERY_CHECK_CODE(code, lino, _error); } } - pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionAggNext, NULL, destroyStreamSessionAggOperatorInfo, - optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + pOperator->fpSet = + createOperatorFpSet(optrDummyOpenFn, doStreamSessionAggNext, NULL, destroyStreamSessionAggOperatorInfo, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState); if (downstream) { @@ -4102,8 +4105,8 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { } int32_t createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, - SExecTaskInfo* pTaskInfo, int32_t numOfChild, - SReadHandle* pHandle, SOperatorInfo** pOptrInfo) { + SExecTaskInfo* pTaskInfo, int32_t numOfChild, SReadHandle* pHandle, + SOperatorInfo** pOptrInfo) { QRY_PARAM_CHECK(pOptrInfo); int32_t code = TSDB_CODE_SUCCESS; @@ -4111,7 +4114,7 @@ int32_t createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhys SOperatorInfo* pOperator = NULL; code = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo, pHandle, &pOperator); if (pOperator == NULL || code != 0) { - downstream = NULL; + downstream = NULL; QUERY_CHECK_CODE(code, lino, _error); } @@ -4618,8 +4621,8 @@ int32_t doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera QUERY_CHECK_CODE(code, lino, _end); buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); - code = - tSimpleHashPut(pInfo->streamAggSup.pResultRows, &winfo.sessionWin, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); + code = tSimpleHashPut(pInfo->streamAggSup.pResultRows, &winfo.sessionWin, sizeof(SSessionKey), &winfo, + sizeof(SResultWindowInfo)); QUERY_CHECK_CODE(code, lino, _end); } @@ -4846,7 +4849,7 @@ void streamStateReleaseState(SOperatorInfo* pOperator) { int32_t resSize = winSize + sizeof(TSKEY); char* pBuff = taosMemoryCalloc(1, resSize); if (!pBuff) { - return ; + return; } memcpy(pBuff, pInfo->historyWins->pData, winSize); memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY)); @@ -4944,7 +4947,7 @@ void streamStateReloadState(SOperatorInfo* pOperator) { QUERY_CHECK_CODE(code, lino, _end); } } - taosMemoryFree(pBuf); + taosMemoryFreeClear(pBuf); SOperatorInfo* downstream = pOperator->pDownstream[0]; if (downstream->fpSet.reloadStreamStateFn) { @@ -4953,6 +4956,7 @@ void streamStateReloadState(SOperatorInfo* pOperator) { reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup); _end: + taosMemoryFreeClear(pBuf); if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo)); } @@ -5001,9 +5005,9 @@ int32_t createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno); pInfo->binfo.pRes = pResBlock; - SExprSupp* pExpSup = &pOperator->exprSupp; - int32_t numOfCols = 0; - SExprInfo* pExprInfo = NULL; + SExprSupp* pExpSup = &pOperator->exprSupp; + int32_t numOfCols = 0; + SExprInfo* pExprInfo = NULL; code = createExprInfo(pStateNode->window.pFuncs, NULL, &pExprInfo, &numOfCols); QUERY_CHECK_CODE(code, lino, _error); @@ -5335,7 +5339,7 @@ int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* *(pInfo->pState) = *(pTaskInfo->streamInfo.pState); pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsIndex); - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; SExprInfo* pExprInfo = NULL; code = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &pExprInfo, &numOfCols); QUERY_CHECK_CODE(code, lino, _error); @@ -5383,7 +5387,8 @@ int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pInfo->pState->pFileState = NULL; code = pTaskInfo->storageAPI.stateStore.streamFileStateInit( tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, compareTs, pInfo->pState, - pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo), pHandle->checkpointId, STREAM_STATE_BUFF_HASH, &pInfo->pState->pFileState); + pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo), pHandle->checkpointId, STREAM_STATE_BUFF_HASH, + &pInfo->pState->pFileState); QUERY_CHECK_CODE(code, lino, _error); pInfo->pOperator = pOperator; @@ -5397,7 +5402,7 @@ int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pInfo->recvGetAll = false; code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes); - QUERY_CHECK_CODE(code, lino, _error); + QUERY_CHECK_CODE(code, lino, _error); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pInfo->pDeletedMap = tSimpleHashInit(4096, hashFn); From 8028fa97101b91fac6182ab1432ea140a3f7360a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 11 Oct 2024 09:31:35 +0800 Subject: [PATCH 04/13] fix invalid free or mem leak --- source/dnode/vnode/src/meta/metaTable.c | 7 ++++++- source/dnode/vnode/src/tsdb/tsdbFSetRW.c | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 8814e87140..08ee422126 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -3133,7 +3133,12 @@ static void colCompressDebug(SHashObj *pColCmprObj) { int32_t metaGetColCmpr(SMeta *pMeta, tb_uid_t uid, SHashObj **ppColCmprObj) { int rc = 0; - SHashObj *pColCmprObj = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK); + SHashObj *pColCmprObj = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK); + if (pColCmprObj == NULL) { + pColCmprObj = NULL; + return TSDB_CODE_OUT_OF_MEMORY; + } + void *pData = NULL; int nData = 0; SMetaEntry e = {0}; diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRW.c b/source/dnode/vnode/src/tsdb/tsdbFSetRW.c index ea404142fe..60e42bd2b8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSetRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRW.c @@ -46,8 +46,11 @@ static int32_t tsdbFSetWriteTableDataBegin(SFSetWriter *writer, const TABLEID *t code = tsdbUpdateSkmTb(writer->config->tsdb, writer->ctx->tbid, writer->skmTb); TSDB_CHECK_CODE(code, lino, _exit); + if (writer->pColCmprObj != NULL) { + taosHashCleanup(writer->pColCmprObj); + writer->pColCmprObj = NULL; + } code = metaGetColCmpr(writer->config->tsdb->pVnode->pMeta, tbid->suid ? tbid->suid : tbid->uid, &writer->pColCmprObj); - // TODO: TSDB_CHECK_CODE(code, lino, _exit); writer->blockDataIdx = 0; for (int32_t i = 0; i < ARRAY_SIZE(writer->blockData); i++) { @@ -127,6 +130,8 @@ _exit: TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); } taosHashCleanup(writer->pColCmprObj); + writer->pColCmprObj = NULL; + return code; } From f537832664033b41be8b972fbb80d9e97187f1b3 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 11 Oct 2024 10:53:32 +0800 Subject: [PATCH 05/13] fix:[TD-32412] unsafe function --- include/client/taos.h | 1 + source/client/src/clientEnv.c | 12 ++--- source/client/src/clientHb.c | 2 +- source/client/src/clientImpl.c | 20 ++++---- source/client/src/clientJniConnector.c | 4 +- source/client/src/clientMain.c | 55 +++++++++++++--------- source/client/src/clientMonitor.c | 6 +-- source/client/src/clientRawBlockWrite.c | 16 +++---- source/client/src/clientSml.c | 8 ++-- source/client/src/clientStmt.c | 10 ++-- source/client/src/clientStmt2.c | 10 ++-- source/client/src/clientTmq.c | 10 ++-- source/dnode/mnode/impl/src/mndConsumer.c | 8 ++-- source/dnode/mnode/impl/src/mndSubscribe.c | 9 ++-- source/dnode/vnode/src/tq/tqSink.c | 4 +- 15 files changed, 94 insertions(+), 81 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 4bd0744c9e..80dbe27c47 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -251,6 +251,7 @@ DLL_EXPORT int64_t taos_affected_rows64(TAOS_RES *res); DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res); DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); +DLL_EXPORT int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); DLL_EXPORT void taos_stop_query(TAOS_RES *res); DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col); DLL_EXPORT int taos_is_null_by_column(TAOS_RES *res, int columnIndex, bool result[], int *rows); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 3b755c2921..4b4bd5f2c0 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -114,7 +114,7 @@ static void concatStrings(SArray *list, char *buf, int size) { db = dot + 1; } if (i != 0) { - (void)strcat(buf, ","); + (void)strncat(buf, ",", size - 1 - len); len += 1; } int ret = snprintf(buf + len, size - len, "%s", db); @@ -1132,27 +1132,27 @@ static setConfRet taos_set_config_imp(const char *config){ static bool setConfFlag = false; if (setConfFlag) { ret.retCode = SET_CONF_RET_ERR_ONLY_ONCE; - strcpy(ret.retMsg, "configuration can only set once"); + tstrncpy(ret.retMsg, "configuration can only set once", RET_MSG_LENGTH); return ret; } taosInitGlobalCfg(); cJSON *root = cJSON_Parse(config); if (root == NULL){ ret.retCode = SET_CONF_RET_ERR_JSON_PARSE; - strcpy(ret.retMsg, "parse json error"); + tstrncpy(ret.retMsg, "parse json error", RET_MSG_LENGTH); return ret; } int size = cJSON_GetArraySize(root); if(!cJSON_IsObject(root) || size == 0) { ret.retCode = SET_CONF_RET_ERR_JSON_INVALID; - strcpy(ret.retMsg, "json content is invalid, must be not empty object"); + tstrncpy(ret.retMsg, "json content is invalid, must be not empty object", RET_MSG_LENGTH); return ret; } if(size >= 1000) { ret.retCode = SET_CONF_RET_ERR_TOO_LONG; - strcpy(ret.retMsg, "json object size is too long"); + tstrncpy(ret.retMsg, "json object size is too long", RET_MSG_LENGTH); return ret; } @@ -1160,7 +1160,7 @@ static setConfRet taos_set_config_imp(const char *config){ cJSON *item = cJSON_GetArrayItem(root, i); if(!item) { ret.retCode = SET_CONF_RET_ERR_INNER; - strcpy(ret.retMsg, "inner error"); + tstrncpy(ret.retMsg, "inner error", RET_MSG_LENGTH); return ret; } if(!taosReadConfigOption(item->string, item->valuestring, NULL, NULL, TAOS_CFG_CSTATUS_OPTION, TSDB_CFG_CTYPE_B_CLIENT)){ diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 62d8d470ba..75716d0bd2 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -772,7 +772,7 @@ static int32_t hbGetUserAuthInfo(SClientHbKey *connKey, SHbParam *param, SClient SUserAuthVersion *qUserAuth = (SUserAuthVersion *)taosMemoryRealloc(pKv->value, (userNum + 1) * sizeof(SUserAuthVersion)); if (qUserAuth) { - (void)strncpy((qUserAuth + userNum)->user, pTscObj->user, TSDB_USER_LEN); + tstrncpy((qUserAuth + userNum)->user, pTscObj->user, TSDB_USER_LEN); (qUserAuth + userNum)->version = htonl(-1); // force get userAuthInfo pKv->value = qUserAuth; pKv->valueLen += sizeof(SUserAuthVersion); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 774cac750b..5dbd417938 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1928,10 +1928,10 @@ TAOS* taos_connect_l(const char* ip, int ipLen, const char* user, int userLen, c char userStr[TSDB_USER_LEN] = {0}; char passStr[TSDB_PASSWORD_LEN] = {0}; - (void)strncpy(ipStr, ip, TMIN(TSDB_EP_LEN - 1, ipLen)); - (void)strncpy(userStr, user, TMIN(TSDB_USER_LEN - 1, userLen)); - (void)strncpy(passStr, pass, TMIN(TSDB_PASSWORD_LEN - 1, passLen)); - (void)strncpy(dbStr, db, TMIN(TSDB_DB_NAME_LEN - 1, dbLen)); + tstrncpy(ipStr, ip, TMIN(TSDB_EP_LEN - 1, ipLen)); + tstrncpy(userStr, user, TMIN(TSDB_USER_LEN - 1, userLen)); + tstrncpy(passStr, pass, TMIN(TSDB_PASSWORD_LEN - 1, passLen)); + tstrncpy(dbStr, db, TMIN(TSDB_DB_NAME_LEN - 1, dbLen)); return taos_connect(ipStr, userStr, passStr, dbStr, port); } @@ -2275,7 +2275,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int char* jsonInnerData = data + CHAR_BYTES; char dst[TSDB_MAX_JSON_TAG_LEN] = {0}; if (jsonInnerType == TSDB_DATA_TYPE_NULL) { - (void)sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L); + (void)snprintf(varDataVal(dst), TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE, "%s", TSDB_DATA_NULL_STR_L); varDataSetLen(dst, strlen(varDataVal(dst))); } else if (tTagIsJson(data)) { char* jsonString = NULL; @@ -2298,10 +2298,10 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int *(char*)POINTER_SHIFT(varDataVal(dst), length + CHAR_BYTES) = '\"'; } else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) { double jsonVd = *(double*)(jsonInnerData); - (void)sprintf(varDataVal(dst), "%.9lf", jsonVd); + (void)snprintf(varDataVal(dst), TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE, "%.9lf", jsonVd); varDataSetLen(dst, strlen(varDataVal(dst))); } else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) { - (void)sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false"); + (void)snprintf(varDataVal(dst), TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE, "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false"); varDataSetLen(dst, strlen(varDataVal(dst))); } else { tscError("doConvertJson error: invalid type:%d", jsonInnerType); @@ -2658,8 +2658,8 @@ int32_t appendTbToReq(SHashObj* pHash, int32_t pos1, int32_t len1, int32_t pos2, return -1; } - char dbFName[TSDB_DB_FNAME_LEN]; - (void)sprintf(dbFName, "%d.%.*s", acctId, dbLen, dbName); + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + (void)snprintf(dbFName, TSDB_DB_FNAME_LEN, "%d.%.*s", acctId, dbLen, dbName); STablesReq* pDb = taosHashGet(pHash, dbFName, strlen(dbFName)); if (pDb) { @@ -2672,7 +2672,7 @@ int32_t appendTbToReq(SHashObj* pHash, int32_t pos1, int32_t len1, int32_t pos2, if (NULL == db.pTables) { return terrno; } - (void)strcpy(db.dbFName, dbFName); + tstrncpy(db.dbFName, dbFName, TSDB_DB_FNAME_LEN); if (NULL == taosArrayPush(db.pTables, &name)) { return terrno; } diff --git a/source/client/src/clientJniConnector.c b/source/client/src/clientJniConnector.c index d783c6d8e4..f87611ac00 100644 --- a/source/client/src/clientJniConnector.c +++ b/source/client/src/clientJniConnector.c @@ -1083,14 +1083,14 @@ JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_stmtErrorMsgIm TAOS *tscon = (TAOS *)con; if (tscon == NULL) { jniError("jobj:%p, connection already closed", jobj); - (void)sprintf(errMsg, "jobj:%p, connection already closed", jobj); + (void)snprintf(errMsg, sizeof(errMsg), "jobj:%p, connection already closed", jobj); return (*env)->NewStringUTF(env, errMsg); } TAOS_STMT *pStmt = (TAOS_STMT *)stmt; if (pStmt == NULL) { jniError("jobj:%p, conn:%p, invalid stmt", jobj, tscon); - (void)sprintf(errMsg, "jobj:%p, conn:%p, invalid stmt", jobj, tscon); + (void)snprintf(errMsg, sizeof(errMsg), "jobj:%p, conn:%p, invalid stmt", jobj, tscon); return (*env)->NewStringUTF(env, errMsg); } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 48bf8d129c..a35c7c7a4c 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -482,71 +482,75 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { } int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) { + return taos_print_row_with_size(str, INT32_MAX, row, fields, num_fields); +} +int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields){ int32_t len = 0; for (int i = 0; i < num_fields; ++i) { - if (i > 0) { + if (i > 0 && len < size - 1) { str[len++] = ' '; } if (row[i] == NULL) { - len += sprintf(str + len, "%s", TSDB_DATA_NULL_STR); + len += snprintf(str + len, size - len, "%s", TSDB_DATA_NULL_STR); continue; } switch (fields[i].type) { case TSDB_DATA_TYPE_TINYINT: - len += sprintf(str + len, "%d", *((int8_t *)row[i])); + len += snprintf(str + len, size - len, "%d", *((int8_t *)row[i])); break; case TSDB_DATA_TYPE_UTINYINT: - len += sprintf(str + len, "%u", *((uint8_t *)row[i])); + len += snprintf(str + len, size - len, "%u", *((uint8_t *)row[i])); break; case TSDB_DATA_TYPE_SMALLINT: - len += sprintf(str + len, "%d", *((int16_t *)row[i])); + len += snprintf(str + len, size - len, "%d", *((int16_t *)row[i])); break; case TSDB_DATA_TYPE_USMALLINT: - len += sprintf(str + len, "%u", *((uint16_t *)row[i])); + len += snprintf(str + len, size - len, "%u", *((uint16_t *)row[i])); break; case TSDB_DATA_TYPE_INT: - len += sprintf(str + len, "%d", *((int32_t *)row[i])); + len += snprintf(str + len, size - len, "%d", *((int32_t *)row[i])); break; case TSDB_DATA_TYPE_UINT: - len += sprintf(str + len, "%u", *((uint32_t *)row[i])); + len += snprintf(str + len, size - len, "%u", *((uint32_t *)row[i])); break; case TSDB_DATA_TYPE_BIGINT: - len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); + len += snprintf(str + len, size - len, "%" PRId64, *((int64_t *)row[i])); break; case TSDB_DATA_TYPE_UBIGINT: - len += sprintf(str + len, "%" PRIu64, *((uint64_t *)row[i])); + len += snprintf(str + len, size - len, "%" PRIu64, *((uint64_t *)row[i])); break; case TSDB_DATA_TYPE_FLOAT: { float fv = 0; fv = GET_FLOAT_VAL(row[i]); - len += sprintf(str + len, "%f", fv); + len += snprintf(str + len, size - len, "%f", fv); } break; case TSDB_DATA_TYPE_DOUBLE: { double dv = 0; dv = GET_DOUBLE_VAL(row[i]); - len += sprintf(str + len, "%lf", dv); + len += snprintf(str + len, size - len, "%lf", dv); } break; case TSDB_DATA_TYPE_VARBINARY: { void *data = NULL; - uint32_t size = 0; + uint32_t tmp = 0; int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE); - if (taosAscii2Hex(row[i], charLen, &data, &size) < 0) { + if (taosAscii2Hex(row[i], charLen, &data, &tmp) < 0) { break; } - (void)memcpy(str + len, data, size); - len += size; + uint32_t copyLen = TMIN(size - len - 1, tmp); + (void)memcpy(str + len, data, copyLen); + len += copyLen; taosMemoryFree(data); } break; case TSDB_DATA_TYPE_BINARY: @@ -566,21 +570,28 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) } } - (void)memcpy(str + len, row[i], charLen); - len += charLen; + uint32_t copyLen = TMIN(size - len - 1, charLen); + (void)memcpy(str + len, row[i], copyLen); + len += copyLen; } break; case TSDB_DATA_TYPE_TIMESTAMP: - len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); + len += snprintf(str + len, size - len, "%" PRId64, *((int64_t *)row[i])); break; case TSDB_DATA_TYPE_BOOL: - len += sprintf(str + len, "%d", *((int8_t *)row[i])); + len += snprintf(str + len, size - len, "%d", *((int8_t *)row[i])); default: break; } + + if (len >= size - 1) { + break; + } + } + if (len < size){ + str[len] = 0; } - str[len] = 0; return len; } @@ -945,7 +956,7 @@ int taos_get_current_db(TAOS *taos, char *database, int len, int *required) { if (required) *required = strlen(pTscObj->db) + 1; TSC_ERR_JRET(TSDB_CODE_INVALID_PARA); } else { - (void)strcpy(database, pTscObj->db); + tstrncpy(database, pTscObj->db, len); code = 0; } _return: diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index 6667c4c741..c800efb5d7 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -183,7 +183,7 @@ FAILED: static void generateClusterReport(taos_collector_registry_t* registry, void* pTransporter, SEpSet* epSet) { char ts[50] = {0}; - (void)sprintf(ts, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); + (void)snprintf(ts, sizeof(ts), "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); char* pCont = (char*)taos_collector_registry_bridge_new(registry, ts, "%" PRId64, NULL); if (NULL == pCont) { tscError("generateClusterReport failed, get null content."); @@ -401,7 +401,7 @@ static void monitorWriteSlowLog2File(MonitorSlowLogData* slowLogData, char* tmpP return; } pClient->lastCheckTime = taosGetMonoTimestampMs(); - (void)strcpy(pClient->path, path); + tstrncpy(pClient->path, path, PATH_MAX); pClient->offset = 0; pClient->pFile = pFile; if (taosHashPut(monitorSlowLogHash, &slowLogData->clusterId, LONG_BYTES, &pClient, POINTER_BYTES) != 0) { @@ -458,7 +458,7 @@ static char* readFile(TdFilePtr pFile, int64_t* offset, int64_t size) { return NULL; } char* buf = pCont; - (void)strcat(buf++, "["); + (void)strncat(buf++, "[", totalSize - 1); int64_t readSize = taosReadFile(pFile, buf, totalSize - 4); // 4 reserved for [] if (readSize <= 0) { if (readSize < 0) { diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index b55cc75340..acba8117c6 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -926,7 +926,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { for (int32_t i = 0; i < req.schemaRow.nCols; i++) { SSchema* pSchema = req.schemaRow.pSchema + i; SFieldWithOptions field = {.type = pSchema->type, .flags = pSchema->flags, .bytes = pSchema->bytes}; - (void)strcpy(field.name, pSchema->name); + tstrncpy(field.name, pSchema->name, TSDB_COL_NAME_LEN); if (createDefaultCompress) { field.compress = createDefaultColCmprByType(pSchema->type); @@ -941,7 +941,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { for (int32_t i = 0; i < req.schemaTag.nCols; i++) { SSchema* pSchema = req.schemaTag.pSchema + i; SField field = {.type = pSchema->type, .flags = pSchema->flags, .bytes = pSchema->bytes}; - (void)strcpy(field.name, pSchema->name); + tstrncpy(field.name, pSchema->name, TSDB_COL_NAME_LEN); RAW_NULL_CHECK(taosArrayPush(pReq.pTags, &field)); } @@ -1244,7 +1244,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { if (pTableBatch == NULL) { SVgroupCreateTableBatch tBatch = {0}; tBatch.info = pInfo; - (void)strcpy(tBatch.dbName, pRequest->pDb); + tstrncpy(tBatch.dbName, pRequest->pDb, TSDB_DB_NAME_LEN); tBatch.req.pArray = taosArrayInit(4, sizeof(struct SVCreateTbReq)); RAW_NULL_CHECK(tBatch.req.pArray); @@ -1769,8 +1769,8 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { RAW_NULL_CHECK(tbName); SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; - (void)strcpy(pName.dbname, pRequest->pDb); - (void)strcpy(pName.tname, tbName); + tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN); + tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN); RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta)); @@ -1928,15 +1928,15 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) uDebug(LOG_ID_TAG " write raw metadata block tbname:%s", LOG_ID_VALUE, tbName); SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; - (void)strcpy(pName.dbname, pRequest->pDb); - (void)strcpy(pName.tname, tbName); + tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN); + tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN); // find schema data info SVCreateTbReq* pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, tbName, strlen(tbName)); SVgroupInfo vg = {0}; RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg)); if (pCreateReqDst) { // change stable name to get meta - (void)strcpy(pName.tname, pCreateReqDst->ctb.stbName); + tstrncpy(pName.tname, pCreateReqDst->ctb.stbName, TSDB_TABLE_NAME_LEN); } RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta)); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index e4e5a54a0b..7749557055 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -137,7 +137,7 @@ void smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2 } (void)memset(pBuf->buf, 0, pBuf->len); if (msg1) { - (void)strncat(pBuf->buf, msg1, pBuf->len); + (void)strncat(pBuf->buf, msg1, pBuf->len - 1); } int32_t left = pBuf->len - strlen(pBuf->buf); if (left > 2 && msg2) { @@ -515,9 +515,9 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam if (tag == NULL) { return TSDB_CODE_SML_INVALID_DATA; } - (void)strncat(childTableName, tag->value, tag->length); + (void)strncat(childTableName, tag->value, TSDB_TABLE_NAME_LEN - 1 - strlen(childTableName)); if (i != taosArrayGetSize(tags) - 1) { - (void)strcat(childTableName, tsSmlAutoChildTableNameDelimiter); + (void)strncat(childTableName, tsSmlAutoChildTableNameDelimiter, TSDB_TABLE_NAME_LEN - 1 - strlen(childTableName)); } } if (tsSmlDot2Underline) { @@ -538,7 +538,7 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam // handle child table name if (childTableNameLen == tag->keyLen && strncmp(tag->key, tbnameKey, tag->keyLen) == 0) { (void)memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - (void)strncpy(childTableName, tag->value, + tstrncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); if (tsSmlDot2Underline) { smlStrReplace(childTableName, strlen(childTableName)); diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index f3d765af2f..e56d4cc4f6 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -236,7 +236,7 @@ int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, } (void)memcpy(&pStmt->bInfo.sname, tbName, sizeof(*tbName)); - (void)strncpy(pStmt->bInfo.tbFName, tbFName, sizeof(pStmt->bInfo.tbFName) - 1); + tstrncpy(pStmt->bInfo.tbFName, tbFName, TSDB_TABLE_FNAME_LEN); pStmt->bInfo.tbFName[sizeof(pStmt->bInfo.tbFName) - 1] = 0; pStmt->bInfo.tbUid = autoCreateTbl ? 0 : pTableMeta->uid; @@ -1018,13 +1018,13 @@ int stmtSetTbName(TAOS_STMT* stmt, const char* tbName) { STMT_ERR_RET(stmtGetFromCache(pStmt)); if (pStmt->bInfo.needParse) { - (void)strncpy(pStmt->bInfo.tbName, tbName, sizeof(pStmt->bInfo.tbName) - 1); + tstrncpy(pStmt->bInfo.tbName, tbName, sizeof(pStmt->bInfo.tbName)); pStmt->bInfo.tbName[sizeof(pStmt->bInfo.tbName) - 1] = 0; STMT_ERR_RET(stmtParseSql(pStmt)); } } else { - (void)strncpy(pStmt->bInfo.tbName, tbName, sizeof(pStmt->bInfo.tbName) - 1); + tstrncpy(pStmt->bInfo.tbName, tbName, sizeof(pStmt->bInfo.tbName)); pStmt->bInfo.tbName[sizeof(pStmt->bInfo.tbName) - 1] = 0; pStmt->exec.pRequest->requestId++; pStmt->bInfo.needParse = false; @@ -1172,7 +1172,7 @@ int32_t stmtAppendTablePostHandle(STscStmt* pStmt, SStmtQNode* param) { } if (0 == pStmt->sql.siInfo.firstName[0]) { - (void)strcpy(pStmt->sql.siInfo.firstName, pStmt->bInfo.tbName); + tstrncpy(pStmt->sql.siInfo.firstName, pStmt->bInfo.tbName, TSDB_TABLE_NAME_LEN); } param->tblData.getFromHash = pStmt->sql.siInfo.tbFromHash; @@ -1313,7 +1313,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { // param->tblData.aCol = taosArrayInit(20, POINTER_BYTES); param->restoreTbCols = false; - (void)strcpy(param->tblData.tbName, pStmt->bInfo.tbName); + tstrncpy(param->tblData.tbName, pStmt->bInfo.tbName, TSDB_TABLE_NAME_LEN); } int64_t startUs3 = taosGetTimestampUs(); diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index 841171bacf..0837154fce 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -187,7 +187,7 @@ static int32_t stmtUpdateBindInfo(TAOS_STMT2* stmt, STableMeta* pTableMeta, void } (void)memcpy(&pStmt->bInfo.sname, tbName, sizeof(*tbName)); - (void)strncpy(pStmt->bInfo.tbFName, tbFName, sizeof(pStmt->bInfo.tbFName) - 1); + tstrncpy(pStmt->bInfo.tbFName, tbFName, sizeof(pStmt->bInfo.tbFName)); pStmt->bInfo.tbFName[sizeof(pStmt->bInfo.tbFName) - 1] = 0; pStmt->bInfo.tbUid = autoCreateTbl ? 0 : pTableMeta->uid; @@ -961,13 +961,13 @@ int stmtSetTbName2(TAOS_STMT2* stmt, const char* tbName) { STMT_ERR_RET(stmtGetFromCache(pStmt)); if (pStmt->bInfo.needParse) { - (void)strncpy(pStmt->bInfo.tbName, tbName, sizeof(pStmt->bInfo.tbName) - 1); + tstrncpy(pStmt->bInfo.tbName, tbName, sizeof(pStmt->bInfo.tbName)); pStmt->bInfo.tbName[sizeof(pStmt->bInfo.tbName) - 1] = 0; STMT_ERR_RET(stmtParseSql(pStmt)); } } else { - (void)strncpy(pStmt->bInfo.tbName, tbName, sizeof(pStmt->bInfo.tbName) - 1); + tstrncpy(pStmt->bInfo.tbName, tbName, sizeof(pStmt->bInfo.tbName)); pStmt->bInfo.tbName[sizeof(pStmt->bInfo.tbName) - 1] = 0; pStmt->exec.pRequest->requestId++; pStmt->bInfo.needParse = false; @@ -1113,7 +1113,7 @@ static int32_t stmtAppendTablePostHandle(STscStmt2* pStmt, SStmtQNode* param) { } if (0 == pStmt->sql.siInfo.firstName[0]) { - (void)strcpy(pStmt->sql.siInfo.firstName, pStmt->bInfo.tbName); + tstrncpy(pStmt->sql.siInfo.firstName, pStmt->bInfo.tbName, TSDB_TABLE_NAME_LEN); } param->tblData.getFromHash = pStmt->sql.siInfo.tbFromHash; @@ -1367,7 +1367,7 @@ int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx) { // param->tblData.aCol = taosArrayInit(20, POINTER_BYTES); param->restoreTbCols = false; - (void)strcpy(param->tblData.tbName, pStmt->bInfo.tbName); + tstrncpy(param->tblData.tbName, pStmt->bInfo.tbName, TSDB_TABLE_NAME_LEN); } int64_t startUs3 = taosGetTimestampUs(); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 42de2925da..eecc48eee2 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -993,7 +993,7 @@ void tmqSendHbReq(void* param, void* tmrId) { if (data == NULL) { continue; } - (void)strcpy(data->topicName, pTopic->topicName); + tstrncpy(data->topicName, pTopic->topicName, TSDB_TOPIC_FNAME_LEN); data->offsetRows = taosArrayInit(numOfVgroups, sizeof(OffsetRows)); if (data->offsetRows == NULL) { continue; @@ -1126,7 +1126,7 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic if (pVgEp == NULL) { continue; } - (void)sprintf(vgKey, "%s:%d", pTopic->topicName, pVgEp->vgId); + (void)snprintf(vgKey, sizeof(vgKey), "%s:%d", pTopic->topicName, pVgEp->vgId); SVgroupSaveInfo* pInfo = taosHashGet(pVgOffsetHashMap, vgKey, strlen(vgKey)); STqOffsetVal offsetNew = {0}; @@ -1187,7 +1187,7 @@ static void buildNewTopicList(tmq_t* tmq, SArray* newTopics, const SMqAskEpRsp* continue; } char vgKey[TSDB_TOPIC_FNAME_LEN + 22] = {0}; - (void)sprintf(vgKey, "%s:%d", pTopicCur->topicName, pVgCur->vgId); + (void)snprintf(vgKey, sizeof(vgKey), "%s:%d", pTopicCur->topicName, pVgCur->vgId); char buf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(buf, TSDB_OFFSET_LEN, &pVgCur->offsetInfo.endOffset); @@ -1992,7 +1992,7 @@ END: if (pRspWrapper) { pRspWrapper->code = code; pRspWrapper->pollRsp.vgId = vgId; - (void)strcpy(pRspWrapper->pollRsp.topicName, pParam->topicName); + tstrncpy(pRspWrapper->pollRsp.topicName, pParam->topicName, TSDB_TOPIC_FNAME_LEN); code = taosWriteQitem(tmq->mqueue, pRspWrapper); if (code != 0) { tmqFreeRspWrapper(pRspWrapper); @@ -2156,7 +2156,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p } pParam->refId = pTmq->refId; - (void)strcpy(pParam->topicName, pTopic->topicName); + tstrncpy(pParam->topicName, pTopic->topicName, TSDB_TOPIC_FNAME_LEN); pParam->vgId = pVg->vgId; pParam->requestId = req.reqId; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index a96b8b22f5..9f7c163eec 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -167,7 +167,7 @@ static int32_t checkPrivilege(SMnode *pMnode, SMqConsumerObj *pConsumer, SMqHbRs } STopicPrivilege *data = taosArrayReserve(rsp->topicPrivileges, 1); MND_TMQ_NULL_CHECK(data); - (void)strcpy(data->topic, topic); + tstrncpy(data->topic, topic, TSDB_TOPIC_FNAME_LEN); if (mndCheckTopicPrivilege(pMnode, user, MND_OPER_SUBSCRIBE, pTopic) != 0 || grantCheckExpire(TSDB_GRANT_SUBSCRIPTION) < 0) { data->noPrivilege = 1; @@ -278,7 +278,7 @@ static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t e taosRLockLatch(&pSub->lock); SMqSubTopicEp topicEp = {0}; - (void)strcpy(topicEp.topic, topic); + tstrncpy(topicEp.topic, topic, TSDB_TOPIC_FNAME_LEN); // 2.1 fetch topic schema SMqTopicObj *pTopic = NULL; @@ -910,7 +910,7 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * // consumer id char consumerIdHex[TSDB_CONSUMER_ID_LEN + VARSTR_HEADER_SIZE] = {0}; - (void)sprintf(varDataVal(consumerIdHex), "0x%" PRIx64, pConsumer->consumerId); + (void)snprintf(varDataVal(consumerIdHex), TSDB_CONSUMER_ID_LEN, "0x%" PRIx64, pConsumer->consumerId); varDataSetLen(consumerIdHex, strlen(varDataVal(consumerIdHex))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -993,7 +993,7 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * parasStr = taosMemoryCalloc(1, pShow->pMeta->pSchemas[cols].bytes); MND_TMQ_NULL_CHECK(parasStr); - (void)sprintf(varDataVal(parasStr), "tbname:%d,commit:%d,interval:%dms,reset:%s", pConsumer->withTbName, + (void)snprintf(varDataVal(parasStr), pShow->pMeta->pSchemas[cols].bytes - VARSTR_HEADER_SIZE, "tbname:%d,commit:%d,interval:%dms,reset:%s", pConsumer->withTbName, pConsumer->autoCommit, pConsumer->autoCommitInterval, buf); varDataSetLen(parasStr, strlen(varDataVal(parasStr))); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 93d8b6dcde..bcca01a230 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -187,12 +187,12 @@ static void mndSplitSubscribeKey(const char *key, char *topic, char *cgroup, boo (void)memcpy(cgroup, key, i); cgroup[i] = 0; if (fullName) { - (void)strcpy(topic, &key[i + 1]); + tstrncpy(topic, &key[i + 1], TSDB_TOPIC_FNAME_LEN); } else { while (key[i] != '.') { i++; } - (void)strcpy(topic, &key[i + 1]); + tstrncpy(topic, &key[i + 1], TSDB_CGROUP_LEN); } } @@ -1361,7 +1361,7 @@ static int32_t buildResult(SSDataBlock *pBlock, int32_t *numOfRows, int64_t cons // consumer id char consumerIdHex[TSDB_CONSUMER_ID_LEN] = {0}; - (void)sprintf(varDataVal(consumerIdHex), "0x%" PRIx64, consumerId); + (void)snprintf(varDataVal(consumerIdHex), TSDB_CONSUMER_ID_LEN - VARSTR_HEADER_SIZE, "0x%" PRIx64, consumerId); varDataSetLen(consumerIdHex, strlen(varDataVal(consumerIdHex))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -1398,7 +1398,8 @@ static int32_t buildResult(SSDataBlock *pBlock, int32_t *numOfRows, int64_t cons // vg id char buf[TSDB_OFFSET_LEN * 2 + VARSTR_HEADER_SIZE] = {0}; (void)tFormatOffset(varDataVal(buf), TSDB_OFFSET_LEN, &data->offset); - (void)sprintf(varDataVal(buf) + strlen(varDataVal(buf)), "/%" PRId64, data->ever); + (void)snprintf(varDataVal(buf) + strlen(varDataVal(buf)), + sizeof(buf) - VARSTR_HEADER_SIZE - strlen(varDataVal(buf)), "/%" PRId64, data->ever); varDataSetLen(buf, strlen(varDataVal(buf))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); MND_TMQ_NULL_CHECK(pColInfo); diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index b18fa42cae..2a43780df2 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -107,7 +107,7 @@ int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* p groupId, name, skey, ekey); SSingleDeleteReq req = {.startTs = skey, .endTs = ekey}; - strncpy(req.tbname, name, TSDB_TABLE_NAME_LEN - 1); + tstrncpy(req.tbname, name, TSDB_TABLE_NAME_LEN); void* p = taosArrayPush(deleteReq->deleteReqs, &req); if (p == NULL) { return terrno; @@ -235,7 +235,7 @@ int32_t setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* return terrno; } - strcpy(pCreateTableReq->name, pDataBlock->info.parTbName); + tstrncpy(pCreateTableReq->name, pDataBlock->info.parTbName, TSDB_TABLE_NAME_LEN); buildCtbNameAddGroupId(stbFullName, pCreateTableReq->name, gid); // tqDebug("gen name from:%s", pDataBlock->info.parTbName); } else { From 7ba8a6a680443be7569a299bf4228806a9ceacb3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 11 Oct 2024 12:11:59 +0800 Subject: [PATCH 06/13] fix: manual testing errors --- .gitignore | 21 +++++++++++++++++++++ include/libs/nodes/cmdnodes.h | 2 +- include/util/taoserror.h | 7 ++++--- include/util/tjson.h | 1 + source/dnode/mnode/impl/src/mndAnode.c | 23 +++++++++++------------ source/libs/parser/src/parTranslater.c | 4 ++++ source/util/src/terror.c | 1 + source/util/src/tjson.c | 13 +++++++++++++ 8 files changed, 56 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 1798a920eb..d155512039 100644 --- a/.gitignore +++ b/.gitignore @@ -138,3 +138,24 @@ tags *CMakeCache* *CMakeFiles* .history/ +*.txt +*.tcl +*.pc +contrib/geos +contrib/libuv +contrib/pcre2 +contrib/zlib +deps_tmp_CMakeLists.txt.in +*.a +*.ctest +pcre2-config +pcre2_test.sh +pcre2_grep_test.sh +pcre2_chartables.c +geos-config +config.h +pcre2.h +zconf.h +version.h +geos_c.h + diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index ba1e21b897..bbf2889289 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -320,7 +320,7 @@ typedef struct SAlterDnodeStmt { typedef struct { ENodeType type; - char url[TSDB_ANAL_ANODE_URL_LEN]; + char url[TSDB_ANAL_ANODE_URL_LEN + 3]; } SCreateAnodeStmt; typedef struct { diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 603207d8c0..2620342b22 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -482,9 +482,10 @@ int32_t taosGetErrSize(); #define TSDB_CODE_MND_ANODE_TOO_LONG_URL TAOS_DEF_ERROR_CODE(0, 0x0432) #define TSDB_CODE_MND_ANODE_INVALID_PROTOCOL TAOS_DEF_ERROR_CODE(0, 0x0433) #define TSDB_CODE_MND_ANODE_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0434) -#define TSDB_CODE_MND_ANODE_TOO_MANY_ALGO TAOS_DEF_ERROR_CODE(0, 0x0435) -#define TSDB_CODE_MND_ANODE_TOO_LONG_ALGO_NAME TAOS_DEF_ERROR_CODE(0, 0x0436) -#define TSDB_CODE_MND_ANODE_TOO_MANY_ALGO_TYPE TAOS_DEF_ERROR_CODE(0, 0x0437) +#define TSDB_CODE_MND_ANODE_INVALID_ALGO_TYPE TAOS_DEF_ERROR_CODE(0, 0x0435) +#define TSDB_CODE_MND_ANODE_TOO_MANY_ALGO TAOS_DEF_ERROR_CODE(0, 0x0436) +#define TSDB_CODE_MND_ANODE_TOO_LONG_ALGO_NAME TAOS_DEF_ERROR_CODE(0, 0x0437) +#define TSDB_CODE_MND_ANODE_TOO_MANY_ALGO_TYPE TAOS_DEF_ERROR_CODE(0, 0x0438) // analysis #define TSDB_CODE_ANAL_URL_RSP_IS_NULL TAOS_DEF_ERROR_CODE(0, 0x0440) diff --git a/include/util/tjson.h b/include/util/tjson.h index 50d1a4d438..88c2a1efb7 100644 --- a/include/util/tjson.h +++ b/include/util/tjson.h @@ -71,6 +71,7 @@ int32_t tjsonGetObjectValueString(const SJson* pJson, char** pStringValue); void tjsonGetObjectValueBigInt(const SJson* pJson, int64_t* pVal); void tjsonGetObjectValueDouble(const SJson* pJson, double* pVal); int32_t tjsonGetStringValue(const SJson* pJson, const char* pName, char* pVal); +int32_t tjsonGetStringValue2(const SJson* pJson, const char* pName, char* pVal, int32_t maxLen); int32_t tjsonDupStringValue(const SJson* pJson, const char* pName, char** pVal); int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal); int32_t tjsonGetIntValue(const SJson* pJson, const char* pName, int32_t* pVal); diff --git a/source/dnode/mnode/impl/src/mndAnode.c b/source/dnode/mnode/impl/src/mndAnode.c index 7e02db0e90..4b0c288bf2 100644 --- a/source/dnode/mnode/impl/src/mndAnode.c +++ b/source/dnode/mnode/impl/src/mndAnode.c @@ -43,7 +43,7 @@ static void mndCancelGetNextAnode(SMnode *pMnode, void *pIter); static int32_t mndRetrieveAnodesFull(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextAnodeFull(SMnode *pMnode, void *pIter); static int32_t mndGetAnodeAlgoList(const char *url, SAnodeObj *pObj); -static int32_t mndGetAnodeStatus(SAnodeObj *pObj, char *status); +static int32_t mndGetAnodeStatus(SAnodeObj *pObj, char *status, int32_t statusLen); int32_t mndInitAnode(SMnode *pMnode) { SSdbTable table = { @@ -603,7 +603,7 @@ static int32_t mndRetrieveAnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB if (code != 0) goto _end; status[0] = 0; - if (mndGetAnodeStatus(pObj, status) == 0) { + if (mndGetAnodeStatus(pObj, status, 64) == 0) { STR_TO_VARSTR(buf, status); } else { STR_TO_VARSTR(buf, "offline"); @@ -698,7 +698,7 @@ static int32_t mndDecodeAlgoList(SJson *pJson, SAnodeObj *pObj) { code = tjsonGetDoubleValue(pJson, "protocol", &tmp); if (code < 0) return TSDB_CODE_INVALID_JSON_FORMAT; protocol = (int32_t)(tmp * 1000); - if (protocol != 100) return TSDB_CODE_MND_ANODE_INVALID_PROTOCOL; + if (protocol != 100 && protocol != 1000) return TSDB_CODE_MND_ANODE_INVALID_PROTOCOL; code = tjsonGetDoubleValue(pJson, "version", &tmp); pObj->version = (int32_t)(tmp * 1000); @@ -722,10 +722,10 @@ static int32_t mndDecodeAlgoList(SJson *pJson, SAnodeObj *pObj) { SJson *detail = tjsonGetArrayItem(details, d); if (detail == NULL) return TSDB_CODE_INVALID_JSON_FORMAT; - code = tjsonGetStringValue(detail, "type", buf); + code = tjsonGetStringValue2(detail, "type", buf, sizeof(buf)); if (code < 0) return TSDB_CODE_INVALID_JSON_FORMAT; EAnalAlgoType type = taosAnalAlgoInt(buf); - if (type < 0 || type >= ANAL_ALGO_TYPE_END) continue; + if (type < 0 || type >= ANAL_ALGO_TYPE_END) return TSDB_CODE_MND_ANODE_INVALID_ALGO_TYPE; SJson *algos = tjsonGetObjectItem(detail, "algo"); if (algos == NULL) return TSDB_CODE_INVALID_JSON_FORMAT; @@ -734,13 +734,12 @@ static int32_t mndDecodeAlgoList(SJson *pJson, SAnodeObj *pObj) { SJson *algo = tjsonGetArrayItem(algos, a); if (algo == NULL) return TSDB_CODE_INVALID_JSON_FORMAT; - code = tjsonGetStringValue(algo, "name", buf); - if (code < 0) return TSDB_CODE_INVALID_JSON_FORMAT; + code = tjsonGetStringValue2(algo, "name", buf, sizeof(buf)); + if (code < 0) return TSDB_CODE_MND_ANODE_TOO_LONG_ALGO_NAME; SAnodeAlgo algoObj = {0}; algoObj.nameLen = strlen(buf) + 1; - if (algoObj.nameLen > TSDB_ANAL_ALGO_NAME_LEN) return TSDB_CODE_MND_ANODE_TOO_LONG_ALGO_NAME; - if (algoObj.nameLen <= 1) return TSDB_CODE_OUT_OF_MEMORY; + if (algoObj.nameLen <= 1) return TSDB_CODE_INVALID_JSON_FORMAT; algoObj.name = taosMemoryCalloc(algoObj.nameLen, 1); tstrncpy(algoObj.name, buf, algoObj.nameLen); @@ -764,7 +763,7 @@ static int32_t mndGetAnodeAlgoList(const char *url, SAnodeObj *pObj) { TAOS_RETURN(code); } -static int32_t mndGetAnodeStatus(SAnodeObj *pObj, char *status) { +static int32_t mndGetAnodeStatus(SAnodeObj *pObj, char *status, int32_t statusLen) { int32_t code = 0; int32_t protocol = 0; double tmp = 0; @@ -780,12 +779,12 @@ static int32_t mndGetAnodeStatus(SAnodeObj *pObj, char *status) { goto _OVER; } protocol = (int32_t)(tmp * 1000); - if (protocol != 100) { + if (protocol != 100 && protocol != 1000) { code = TSDB_CODE_MND_ANODE_INVALID_PROTOCOL; goto _OVER; } - code = tjsonGetStringValue(pJson, "status", status); + code = tjsonGetStringValue2(pJson, "status", status, statusLen); if (code < 0) { code = TSDB_CODE_INVALID_JSON_FORMAT; goto _OVER; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index ab06008e6c..dab2126103 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -9581,6 +9581,10 @@ static int32_t translateDropUser(STranslateContext* pCxt, SDropUserStmt* pStmt) static int32_t translateCreateAnode(STranslateContext* pCxt, SCreateAnodeStmt* pStmt) { SMCreateAnodeReq createReq = {0}; createReq.urlLen = strlen(pStmt->url) + 1; + if (createReq.urlLen > TSDB_ANAL_ANODE_URL_LEN) { + return TSDB_CODE_MND_ANODE_TOO_LONG_URL; + } + createReq.url = taosMemoryCalloc(createReq.urlLen, 1); if (createReq.url == NULL) { return TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 3598262d5d..ad43426efa 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -349,6 +349,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_ANODE_ALREADY_EXIST, "Anode already exists" TAOS_DEFINE_ERROR(TSDB_CODE_MND_ANODE_NOT_EXIST, "Anode not there") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ANODE_TOO_LONG_URL, "Anode too long url") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ANODE_INVALID_PROTOCOL, "Anode invalid protocol") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_ANODE_INVALID_ALGO_TYPE, "Anode invalid algorithm type") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ANODE_INVALID_VERSION, "Anode invalid version") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ANODE_TOO_MANY_ALGO, "Anode too many algorithm") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ANODE_TOO_LONG_ALGO_NAME, "Anode too long algorithm name") diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 314f205057..752a3136ed 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -207,6 +207,19 @@ int32_t tjsonGetStringValue(const SJson* pJson, const char* pName, char* pVal) { return TSDB_CODE_SUCCESS; } +int32_t tjsonGetStringValue2(const SJson* pJson, const char* pName, char* pVal, int32_t maxLen) { + char* p = cJSON_GetStringValue(tjsonGetObjectItem((cJSON*)pJson, pName)); + if (NULL == p) { + return TSDB_CODE_SUCCESS; + } + int32_t len = strlen(p); + if (len >= maxLen-1) { + return TSDB_CODE_OUT_OF_MEMORY; + } + strcpy(pVal, p); + return TSDB_CODE_SUCCESS; +} + int32_t tjsonDupStringValue(const SJson* pJson, const char* pName, char** pVal) { char* p = cJSON_GetStringValue(tjsonGetObjectItem((cJSON*)pJson, pName)); if (NULL == p) { From 0199c8dddf5a155b8b28920b0e218281dccbe9df Mon Sep 17 00:00:00 2001 From: Jinqing Kuang Date: Tue, 8 Oct 2024 11:12:57 +0800 Subject: [PATCH 07/13] fix(query)[TD-32415]. Replace memory-unsafe functions - Replace memory unsafe functions with secure alternatives - Fix automatical generation of subtable names --- include/common/tdatablock.h | 2 +- source/common/src/tdatablock.c | 147 ++++++++++++++++++------ source/common/test/dataformatTest.cpp | 58 +++++++++- source/dnode/vnode/src/tq/tqSink.c | 23 +++- source/libs/stream/src/streamDispatch.c | 8 +- 5 files changed, 192 insertions(+), 46 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index cbfd1efb84..99cdb53103 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -277,7 +277,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** pReq, const SSDataBlock* pData bool alreadyAddGroupId(char* ctbName, int64_t groupId); bool isAutoTableName(char* ctbName); -void buildCtbNameAddGroupId(const char* stbName, char* ctbName, uint64_t groupId); +int32_t buildCtbNameAddGroupId(const char* stbName, char* ctbName, uint64_t groupId, size_t cap); int32_t buildCtbNameByGroupId(const char* stbName, uint64_t groupId, char** pName); int32_t buildCtbNameByGroupIdImpl(const char* stbName, uint64_t groupId, char* pBuf); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index d727be30ce..18f0df82be 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2446,9 +2446,11 @@ _error: return NULL; } -static char* formatTimestamp(char* buf, int32_t bufSize, int64_t val, int precision) { +static int32_t formatTimestamp(char* buf, size_t cap, int64_t val, int precision) { time_t tt; int32_t ms = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (precision == TSDB_TIME_PRECISION_NANO) { tt = (time_t)(val / 1000000000); ms = val % 1000000000; @@ -2460,14 +2462,6 @@ static char* formatTimestamp(char* buf, int32_t bufSize, int64_t val, int precis ms = val % 1000; } - /* comment out as it make testcases like select_with_tags.sim fail. - but in windows, this may cause the call to localtime crash if tt < 0, - need to find a better solution. - if (tt < 0) { - tt = 0; - } - */ - if (tt <= 0 && ms < 0) { tt--; if (precision == TSDB_TIME_PRECISION_NANO) { @@ -2479,20 +2473,35 @@ static char* formatTimestamp(char* buf, int32_t bufSize, int64_t val, int precis } } struct tm ptm = {0}; - if (taosLocalTime(&tt, &ptm, buf, bufSize) == NULL) { - return buf; + if (taosLocalTime(&tt, &ptm, buf, cap) == NULL) { + code = TSDB_CODE_INTERNAL_ERROR; + TSDB_CHECK_CODE(code, lino, _end); } - size_t pos = strftime(buf, bufSize, "%Y-%m-%d %H:%M:%S", &ptm); + size_t pos = strftime(buf, cap, "%Y-%m-%d %H:%M:%S", &ptm); + if (pos == 0) { + code = TSDB_CODE_OUT_OF_BUFFER; + TSDB_CHECK_CODE(code, lino, _end); + } + int32_t nwritten = 0; if (precision == TSDB_TIME_PRECISION_NANO) { - sprintf(buf + pos, ".%09d", ms); + nwritten = snprintf(buf + pos, cap - pos, ".%09d", ms); } else if (precision == TSDB_TIME_PRECISION_MICRO) { - sprintf(buf + pos, ".%06d", ms); + nwritten = snprintf(buf + pos, cap - pos, ".%06d", ms); } else { - sprintf(buf + pos, ".%03d", ms); + nwritten = snprintf(buf + pos, cap - pos, ".%03d", ms); } - return buf; + if (nwritten >= cap - pos) { + code = TSDB_CODE_OUT_OF_BUFFER; + TSDB_CHECK_CODE(code, lino, _end); + } + +_end: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } // for debug @@ -2543,7 +2552,10 @@ int32_t dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf switch (pColInfoData->info.type) { case TSDB_DATA_TYPE_TIMESTAMP: memset(pBuf, 0, sizeof(pBuf)); - (void)formatTimestamp(pBuf, sizeof(pBuf), *(uint64_t*)var, pColInfoData->info.precision); + code = formatTimestamp(pBuf, sizeof(pBuf), *(uint64_t*)var, pColInfoData->info.precision); + if (code != TSDB_CODE_SUCCESS) { + snprintf(pBuf, sizeof(pBuf), "NaN"); + } len += snprintf(dumpBuf + len, size - len, " %25s |", pBuf); if (len >= size - 1) goto _exit; break; @@ -2857,27 +2869,98 @@ _end: return code; } -void buildCtbNameAddGroupId(const char* stbName, char* ctbName, uint64_t groupId) { - char tmp[TSDB_TABLE_NAME_LEN] = {0}; - if (stbName == NULL){ - snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%"PRIu64, groupId); - }else{ +// Construct the child table name in the form of __ and store it in `ctbName`. +// If the name length exceeds TSDB_TABLE_NAME_LEN, first convert _ to an MD5 value and then +// concatenate. If the length is still too long, convert to an MD5 value as well. +int32_t buildCtbNameAddGroupId(const char* stbName, char* ctbName, uint64_t groupId, size_t cap) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + char tmp[TSDB_TABLE_NAME_LEN] = {0}; + char* suffix = tmp; + size_t suffixCap = sizeof(tmp); + size_t suffixLen = 0; + size_t prefixLen = 0; + T_MD5_CTX context; + + if (ctbName == NULL || cap < TSDB_TABLE_NAME_LEN) { + code = TSDB_CODE_INTERNAL_ERROR; + TSDB_CHECK_CODE(code, lino, _end); + } + + prefixLen = strlen(ctbName); + + if (stbName == NULL) { + suffixLen = snprintf(suffix, suffixCap, "%" PRIu64, groupId); + if (suffixLen >= suffixCap) { + code = TSDB_CODE_INTERNAL_ERROR; + TSDB_CHECK_CODE(code, lino, _end); + } + } else { int32_t i = strlen(stbName) - 1; - for(; i >= 0; i--){ - if (stbName[i] == '.'){ + for (; i >= 0; i--) { + if (stbName[i] == '.') { break; } } - snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%s_%"PRIu64, stbName + i + 1, groupId); - } - - ctbName[TSDB_TABLE_NAME_LEN - strlen(tmp) - 1] = 0; // put stbname + groupId to the end - (void)strcat(ctbName, tmp); - for(int i = 0; i < strlen(ctbName); i++){ - if(ctbName[i] == '.'){ - ctbName[i] = '_'; + suffixLen = snprintf(suffix, suffixCap, "%s_%" PRIu64, stbName + i + 1, groupId); + if (suffixLen >= suffixCap) { + suffixCap = suffixLen + 1; + suffix = taosMemoryMalloc(suffixCap); + TSDB_CHECK_NULL(suffix, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY); + suffixLen = snprintf(suffix, suffixCap, "%s_%" PRIu64, stbName + i + 1, groupId); + if (suffixLen >= suffixCap) { + code = TSDB_CODE_INTERNAL_ERROR; + TSDB_CHECK_CODE(code, lino, _end); + } } } + + if (prefixLen + suffixLen + 1 >= TSDB_TABLE_NAME_LEN) { + // If the name length exceeeds the limit, convert the suffix to MD5 value. + tMD5Init(&context); + tMD5Update(&context, (uint8_t*)suffix, suffixLen); + tMD5Final(&context); + suffixLen = snprintf(suffix, suffixCap, "%016" PRIx64 "%016" PRIx64, *(uint64_t*)context.digest, + *(uint64_t*)(context.digest + 8)); + if (suffixLen >= suffixCap) { + code = TSDB_CODE_INTERNAL_ERROR; + TSDB_CHECK_CODE(code, lino, _end); + } + } + + if (prefixLen + suffixLen + 1 >= TSDB_TABLE_NAME_LEN) { + // If the name is still too long, convert the ctbName to MD5 value. + tMD5Init(&context); + tMD5Update(&context, (uint8_t*)ctbName, prefixLen); + tMD5Final(&context); + prefixLen = snprintf(ctbName, cap, "t_%016" PRIx64 "%016" PRIx64, *(uint64_t*)context.digest, + *(uint64_t*)(context.digest + 8)); + if (prefixLen >= cap) { + code = TSDB_CODE_INTERNAL_ERROR; + TSDB_CHECK_CODE(code, lino, _end); + } + } + + if (prefixLen + suffixLen + 1 >= TSDB_TABLE_NAME_LEN) { + code = TSDB_CODE_INTERNAL_ERROR; + TSDB_CHECK_CODE(code, lino, _end); + } + + ctbName[prefixLen] = '_'; + tstrncpy(&ctbName[prefixLen + 1], suffix, cap - prefixLen - 1); + + for (char* p = ctbName; *p; ++p) { + if (*p == '.') *p = '_'; + } + +_end: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + if (suffix != tmp) { + taosMemoryFree(suffix); + } + return code; } // auto stream subtable name starts with 't_', followed by the first segment of MD5 digest for group vals. diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp index e8b7b132f2..ebf91025bb 100644 --- a/source/common/test/dataformatTest.cpp +++ b/source/common/test/dataformatTest.cpp @@ -480,7 +480,7 @@ TEST(testCase, StreamAllNormTest) { char ctbName[TSDB_TABLE_NAME_LEN] = {0}; uint64_t groupId = 12345; - buildCtbNameAddGroupId(NULL, ctbName, groupId); + buildCtbNameAddGroupId(NULL, ctbName, groupId, sizeof(ctbName)); ASSERT_STREQ("_12345", ctbName); } @@ -490,7 +490,7 @@ TEST(testCase, StreamWithStbName) { char ctbName[TSDB_TABLE_NAME_LEN] = {0}; uint64_t groupId = 12345; - buildCtbNameAddGroupId(stbName, ctbName, groupId); + buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName)); ASSERT_STREQ("_stb_12345", ctbName); } @@ -500,7 +500,7 @@ TEST(testCase, StreamWithoutDotInStbName) { char ctbName[TSDB_TABLE_NAME_LEN] = {0}; uint64_t groupId = 12345; - buildCtbNameAddGroupId(stbName, ctbName, groupId); + buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName)); ASSERT_STREQ("_table_12345", ctbName); } @@ -510,11 +510,59 @@ TEST(testCase, StreamWithoutDotInStbName2) { char ctbName[TSDB_TABLE_NAME_LEN] = {0}; uint64_t groupId = 12345; - buildCtbNameAddGroupId(stbName, ctbName, groupId); + buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName)); ASSERT_STREQ("__12345", ctbName); } +TEST(testCase, StreamWithLongStbName) { + char ctbName[TSDB_TABLE_NAME_LEN]; + char expectName[TSDB_TABLE_NAME_LEN]; + char *stbName = "a_simle_stb_name"; + uint64_t groupId = UINT64_MAX; + + // test basic function + strcpy(ctbName, "a_simple_ctb_name"); + EXPECT_EQ(buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName)), TSDB_CODE_SUCCESS); + EXPECT_STREQ(ctbName, "a_simple_ctb_name_a_simle_stb_name_18446744073709551615"); + + // test null stbName + strcpy(ctbName, "a_simple_ctb_name"); + stbName = NULL; + EXPECT_EQ(buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName)), TSDB_CODE_SUCCESS); + EXPECT_STREQ(ctbName, "a_simple_ctb_name_18446744073709551615"); + + // test buffer capcity check + EXPECT_EQ(buildCtbNameAddGroupId(stbName, NULL, groupId, sizeof(ctbName)), TSDB_CODE_INTERNAL_ERROR); + EXPECT_EQ(buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName) - 1), TSDB_CODE_INTERNAL_ERROR); + + // test md5 conversion of stbName with groupid + for (int32_t i = 0; i < 159; ++i) ctbName[i] = 'A'; + ctbName[159] = '\0'; + stbName = taosStrdup(ctbName); + snprintf(expectName, TSDB_TABLE_NAME_LEN, "%s_d85f0d87946d76eeedd7b7b78b7492a2", ctbName); + EXPECT_EQ(buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName)), TSDB_CODE_SUCCESS); + EXPECT_STREQ(ctbName, expectName); + + // test md5 conversion of all parts + for (int32_t i = 0; i < 190; ++i) ctbName[i] = 'A'; + ctbName[190] = '\0'; + tstrncpy(expectName, "t_d38a8b2df999bef0082ffc80a59a9cd7_d85f0d87946d76eeedd7b7b78b7492a2", TSDB_TABLE_NAME_LEN); + EXPECT_EQ(buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName)), TSDB_CODE_SUCCESS); + EXPECT_STREQ(ctbName, expectName); + + // test larger stbName + taosMemoryFree(stbName); + for (int32_t i = 0; i < 190; ++i) ctbName[i] = 'A'; + ctbName[190] = '\0'; + stbName = taosStrdup(ctbName); + tstrncpy(expectName, "t_d38a8b2df999bef0082ffc80a59a9cd7_9c99cc7c52073b63fb750af402d9b84b", TSDB_TABLE_NAME_LEN); + EXPECT_EQ(buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName)), TSDB_CODE_SUCCESS); + EXPECT_STREQ(ctbName, expectName); + + taosMemoryFree(stbName); +} + #if 1 TEST(testCase, NoneTest) { const static int nCols = 14; @@ -570,4 +618,4 @@ for (int r = 0; r < nRows; ++r) { taosArrayDestroy(pArray); taosMemoryFree(pTSchema); } -#endif \ No newline at end of file +#endif diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index b18fa42cae..d8f10e6b6e 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -73,14 +73,19 @@ int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* p } if (varTbName != NULL && varTbName != (void*)-1) { - name = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN); + size_t cap = TMAX(TSDB_TABLE_NAME_LEN, varDataLen(varTbName) + 1); + name = taosMemoryMalloc(cap); if (name == NULL) { return terrno; } memcpy(name, varDataVal(varTbName), varDataLen(varTbName)); + name[varDataLen(varTbName)] = '\0'; if (newSubTableRule && !isAutoTableName(name) && !alreadyAddGroupId(name, groupId) && groupId != 0 && stbFullName) { - buildCtbNameAddGroupId(stbFullName, name, groupId); + int32_t code = buildCtbNameAddGroupId(stbFullName, name, groupId, cap); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } else if (stbFullName) { int32_t code = buildCtbNameByGroupId(stbFullName, groupId, &name); @@ -236,7 +241,10 @@ int32_t setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* } strcpy(pCreateTableReq->name, pDataBlock->info.parTbName); - buildCtbNameAddGroupId(stbFullName, pCreateTableReq->name, gid); + int32_t code = buildCtbNameAddGroupId(stbFullName, pCreateTableReq->name, gid, TSDB_TABLE_NAME_LEN); + if (code != TSDB_CODE_SUCCESS) { + return code; + } // tqDebug("gen name from:%s", pDataBlock->info.parTbName); } else { pCreateTableReq->name = taosStrdup(pDataBlock->info.parTbName); @@ -852,9 +860,12 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat !alreadyAddGroupId(dstTableName, groupId) && groupId != 0) { tqDebug("s-task:%s append groupId:%" PRId64 " for generated dstTable:%s", id, groupId, dstTableName); if (pTask->ver == SSTREAM_TASK_SUBTABLE_CHANGED_VER) { - buildCtbNameAddGroupId(NULL, dstTableName, groupId); + code = buildCtbNameAddGroupId(NULL, dstTableName, groupId, sizeof(pDataBlock->info.parTbName)); } else if (pTask->ver > SSTREAM_TASK_SUBTABLE_CHANGED_VER && stbFullName) { - buildCtbNameAddGroupId(stbFullName, dstTableName, groupId); + code = buildCtbNameAddGroupId(stbFullName, dstTableName, groupId, sizeof(pDataBlock->info.parTbName)); + } + if (code != TSDB_CODE_SUCCESS) { + return code; } } } @@ -1250,4 +1261,4 @@ int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* } return TSDB_CODE_SUCCESS; -} \ No newline at end of file +} diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 78cbd844a0..133663ac28 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -706,9 +706,13 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S if (pTask->subtableWithoutMd5 != 1 && !isAutoTableName(pDataBlock->info.parTbName) && !alreadyAddGroupId(pDataBlock->info.parTbName, groupId) && groupId != 0) { if (pTask->ver == SSTREAM_TASK_SUBTABLE_CHANGED_VER) { - buildCtbNameAddGroupId(NULL, pDataBlock->info.parTbName, groupId); + code = buildCtbNameAddGroupId(NULL, pDataBlock->info.parTbName, groupId, sizeof(pDataBlock->info.parTbName)); } else if (pTask->ver > SSTREAM_TASK_SUBTABLE_CHANGED_VER) { - buildCtbNameAddGroupId(pTask->outputInfo.shuffleDispatcher.stbFullName, pDataBlock->info.parTbName, groupId); + code = buildCtbNameAddGroupId(pTask->outputInfo.shuffleDispatcher.stbFullName, pDataBlock->info.parTbName, + groupId, sizeof(pDataBlock->info.parTbName)); + } + if (code != TSDB_CODE_SUCCESS) { + return code; } } } else { From e32e58a8f69eef646f63459088e748d2126a1842 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 11 Oct 2024 14:54:03 +0800 Subject: [PATCH 08/13] fix:[TD-32412] unsafe function --- source/client/src/clientSml.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 7749557055..0864116e36 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -515,7 +515,7 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam if (tag == NULL) { return TSDB_CODE_SML_INVALID_DATA; } - (void)strncat(childTableName, tag->value, TSDB_TABLE_NAME_LEN - 1 - strlen(childTableName)); + (void)strncat(childTableName, tag->value, TMIN(tag->length, TSDB_TABLE_NAME_LEN - 1 - strlen(childTableName))); if (i != taosArrayGetSize(tags) - 1) { (void)strncat(childTableName, tsSmlAutoChildTableNameDelimiter, TSDB_TABLE_NAME_LEN - 1 - strlen(childTableName)); } @@ -538,8 +538,7 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam // handle child table name if (childTableNameLen == tag->keyLen && strncmp(tag->key, tbnameKey, tag->keyLen) == 0) { (void)memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - tstrncpy(childTableName, tag->value, - (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); + tstrncpy(childTableName, tag->value, TMIN(TSDB_TABLE_NAME_LEN, tag->length)); if (tsSmlDot2Underline) { smlStrReplace(childTableName, strlen(childTableName)); } From ef0782d3789765d860233ceb15ccea3267f8d07a Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Fri, 11 Oct 2024 15:27:29 +0800 Subject: [PATCH 09/13] fix(stream): check reload function --- source/libs/executor/src/executor.c | 4 +++- source/libs/parser/src/parTranslater.c | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 362e3c2631..27dd687f40 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1643,7 +1643,9 @@ int32_t qStreamOperatorReleaseState(qTaskInfo_t tInfo) { int32_t qStreamOperatorReloadState(qTaskInfo_t tInfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tInfo; - pTaskInfo->pRoot->fpSet.reloadStreamStateFn(pTaskInfo->pRoot); + if (pTaskInfo->pRoot->fpSet.reloadStreamStateFn != NULL) { + pTaskInfo->pRoot->fpSet.reloadStreamStateFn(pTaskInfo->pRoot); + } return 0; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index ab06008e6c..2fbf0cf2de 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -10813,7 +10813,7 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm } } - if (NULL != pSelect->pGroupByList) { + if (NULL != pSelect->pGroupByList || (pSelect->pWindow == NULL && pSelect->hasAggFuncs)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported Group by"); } From a2ef8c55a1a9219f66a6e90a09d34914061b706a Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Fri, 11 Oct 2024 15:46:38 +0800 Subject: [PATCH 10/13] fix(stream):check stream sql --- source/libs/parser/src/parTranslater.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 2fbf0cf2de..d18658a8c5 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -10813,10 +10813,15 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm } } - if (NULL != pSelect->pGroupByList || (pSelect->pWindow == NULL && pSelect->hasAggFuncs)) { + if (NULL != pSelect->pGroupByList) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported Group by"); } + if (NULL == pSelect->pWindow && pSelect->hasAggFuncs) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "Non window query only support scalar function, aggreate function is not allowed"); + } + return TSDB_CODE_SUCCESS; } From 96338d021fff4de1a26a207b7c5a96963d947947 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Fri, 11 Oct 2024 15:47:55 +0800 Subject: [PATCH 11/13] fix(stream):check stream sql --- source/libs/parser/src/parTranslater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d18658a8c5..75d942ad1f 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -10819,7 +10819,7 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm if (NULL == pSelect->pWindow && pSelect->hasAggFuncs) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "Non window query only support scalar function, aggreate function is not allowed"); + "Non window query only support scalar function, aggregate function is not allowed"); } return TSDB_CODE_SUCCESS; From 5a157ab0b10c0ab25eda2a9436dcf60d7309529d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 11 Oct 2024 17:46:34 +0800 Subject: [PATCH 12/13] fix:[TD-32412] unsafe function --- source/client/src/clientImpl.c | 26 +++++++++++++------------- source/client/src/clientSml.c | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 5dbd417938..2c67cafdf5 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1921,19 +1921,19 @@ TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, cons return NULL; } -TAOS* taos_connect_l(const char* ip, int ipLen, const char* user, int userLen, const char* pass, int passLen, - const char* db, int dbLen, uint16_t port) { - char ipStr[TSDB_EP_LEN] = {0}; - char dbStr[TSDB_DB_NAME_LEN] = {0}; - char userStr[TSDB_USER_LEN] = {0}; - char passStr[TSDB_PASSWORD_LEN] = {0}; - - tstrncpy(ipStr, ip, TMIN(TSDB_EP_LEN - 1, ipLen)); - tstrncpy(userStr, user, TMIN(TSDB_USER_LEN - 1, userLen)); - tstrncpy(passStr, pass, TMIN(TSDB_PASSWORD_LEN - 1, passLen)); - tstrncpy(dbStr, db, TMIN(TSDB_DB_NAME_LEN - 1, dbLen)); - return taos_connect(ipStr, userStr, passStr, dbStr, port); -} +//TAOS* taos_connect_l(const char* ip, int ipLen, const char* user, int userLen, const char* pass, int passLen, +// const char* db, int dbLen, uint16_t port) { +// char ipStr[TSDB_EP_LEN] = {0}; +// char dbStr[TSDB_DB_NAME_LEN] = {0}; +// char userStr[TSDB_USER_LEN] = {0}; +// char passStr[TSDB_PASSWORD_LEN] = {0}; +// +// tstrncpy(ipStr, ip, TMIN(TSDB_EP_LEN - 1, ipLen)); +// tstrncpy(userStr, user, TMIN(TSDB_USER_LEN - 1, userLen)); +// tstrncpy(passStr, pass, TMIN(TSDB_PASSWORD_LEN - 1, passLen)); +// tstrncpy(dbStr, db, TMIN(TSDB_DB_NAME_LEN - 1, dbLen)); +// return taos_connect(ipStr, userStr, passStr, dbStr, port); +//} void doSetOneRowPtr(SReqResultInfo* pResultInfo) { for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) { diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 0864116e36..d5cca55701 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -538,7 +538,7 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam // handle child table name if (childTableNameLen == tag->keyLen && strncmp(tag->key, tbnameKey, tag->keyLen) == 0) { (void)memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - tstrncpy(childTableName, tag->value, TMIN(TSDB_TABLE_NAME_LEN, tag->length)); + tstrncpy(childTableName, tag->value, TMIN(TSDB_TABLE_NAME_LEN, tag->length + 1)); if (tsSmlDot2Underline) { smlStrReplace(childTableName, strlen(childTableName)); } From 795c272ac977b64d2f7c48804b2af4ff816984d9 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 11 Oct 2024 18:34:08 +0800 Subject: [PATCH 13/13] fix(tsdb/scan): ignore lcn file only --- source/dnode/vnode/src/tsdb/tsdbFS2.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index d3b783847c..0c9d9e56cf 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -433,6 +433,21 @@ static int32_t tsdbFSCreateFileObjHash(STFileSystem *fs, STFileHash *hash) { if (fset->farr[i] != NULL) { code = tsdbFSAddEntryToFileObjHash(hash, fset->farr[i]->fname); TSDB_CHECK_CODE(code, lino, _exit); + + if (TSDB_FTYPE_DATA == i && fset->farr[i]->f->lcn > 0) { + STFileObj *fobj = fset->farr[i]; + int32_t lcn = fobj->f->lcn; + char lcn_name[TSDB_FILENAME_LEN]; + + snprintf(lcn_name, TSDB_FQDN_LEN, "%s", fobj->fname); + char *dot = strrchr(lcn_name, '.'); + if (dot) { + snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lcn_name), "%d.data", lcn); + + code = tsdbFSAddEntryToFileObjHash(hash, lcn_name); + TSDB_CHECK_CODE(code, lino, _exit); + } + } } } @@ -535,9 +550,7 @@ static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) { for (const STfsFile *file = NULL; (file = tfsReaddir(dir)) != NULL;) { if (taosIsDir(file->aname)) continue; - if (tsdbFSGetFileObjHashEntry(&fobjHash, file->aname) == NULL && - strncmp(file->aname + strlen(file->aname) - 3, ".cp", 3) && - strncmp(file->aname + strlen(file->aname) - 5, ".data", 5)) { + if (tsdbFSGetFileObjHashEntry(&fobjHash, file->aname) == NULL) { tsdbRemoveFile(file->aname); } }