From b1f710fb8537191f7633e919432ede715b3e80af Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 18 May 2022 00:23:27 +0800 Subject: [PATCH 01/29] enh: trace optimization --- source/dnode/vnode/src/tsdb/tsdbRead.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 254d452cb3..72c198f48c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -372,13 +372,13 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, STsdbReadHandle* pReadHandle, } if (level == TSDB_RETENTION_L0) { - tsdbDebug("%p rsma level %d is selected to query", pReadHandle, TSDB_RETENTION_L0); + tsdbDebug("vgId:%d readh:%p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle, TSDB_RETENTION_L0); return VND_RSMA0(pVnode); } else if (level == TSDB_RETENTION_L1) { - tsdbDebug("%p rsma level %d is selected to query", pReadHandle, TSDB_RETENTION_L1); + tsdbDebug("vgId:%d readh:%p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle, TSDB_RETENTION_L1); return VND_RSMA1(pVnode); } else { - tsdbDebug("%p rsma level %d is selected to query", pReadHandle, TSDB_RETENTION_L2); + tsdbDebug("vgId:%d readh:%p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle, TSDB_RETENTION_L2); return VND_RSMA2(pVnode); } } From 0c24651319d010929d2082e1954576e269d0439d Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Wed, 18 May 2022 14:46:30 +0800 Subject: [PATCH 02/29] enh: add windows build test to ci --- Jenkinsfile2 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index cbf663cdcf..bc2c9ff494 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -226,6 +226,13 @@ pipeline { stages { stage('run test') { parallel { + stage('windows test') { + agent{label " windows10_01 || windows10_02 || windows10_03 || windows10_04 "} + steps { + pre_test_win() + pre_test_build_win() + } + } stage('linux test') { agent{label " slave3_0 || slave15 || slave16 || slave17 "} options { skipDefaultCheckout() } From f4ef09bf6b8c164a23df529acf15f2c9f768e76c Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Wed, 18 May 2022 15:11:54 +0800 Subject: [PATCH 03/29] enh: print ip address before build --- Jenkinsfile2 | 52 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index bc2c9ff494..20a86dbcea 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -117,27 +117,29 @@ def pre_test(){ def pre_test_win(){ bat ''' hostname + ipconfig + set date /t time /t - taskkill /f /t /im python.exe - taskkill /f /t /im bash.exe rd /s /Q C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug - exit 0 ''' bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git reset --hard git fetch || git fetch + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git reset --hard git fetch || git fetch - git checkout -f ''' script { if (env.CHANGE_TARGET == 'master') { bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git checkout master + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git checkout master ''' @@ -145,6 +147,8 @@ def pre_test_win(){ bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git checkout 2.0 + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git checkout 2.0 ''' @@ -152,6 +156,8 @@ def pre_test_win(){ bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git checkout 3.0 + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git checkout 3.0 ''' @@ -159,6 +165,8 @@ def pre_test_win(){ bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git checkout develop + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git checkout develop ''' @@ -169,30 +177,50 @@ def pre_test_win(){ bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git pull - git log -5 + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git pull - git fetch origin +refs/pull/${CHANGE_ID}/merge + ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + git fetch origin +refs/pull/%CHANGE_ID%/merge + ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git checkout -qf FETCH_HEAD - git log -5 ''' } else if (env.CHANGE_URL =~ /\/TDinternal\//) { bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git pull - git fetch origin +refs/pull/${CHANGE_ID}/merge + ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + git fetch origin +refs/pull/%CHANGE_ID%/merge + ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git checkout -qf FETCH_HEAD - git log -5 + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git pull - git log -5 ''' } else { - sh ''' - echo "unmatched reposiotry ${CHANGE_URL}" + bat ''' + echo "unmatched reposiotry %CHANGE_URL%" ''' } } + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + git log -5 + ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + git log -5 + ''' bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git submodule update --init --recursive From 5fdf57a1ea5083493ecedd14cb0babad3c2f65a6 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Wed, 18 May 2022 15:33:15 +0800 Subject: [PATCH 04/29] enh: return error if cmake fails --- Jenkinsfile2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 20a86dbcea..29f4c45229 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -235,8 +235,8 @@ def pre_test_build_win() { cd debug call "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" x64 set CL=/MP8 - cmake .. -G "NMake Makefiles JOM" - jom -j 4 || exit 8 + cmake .. -G "NMake Makefiles JOM" || exit 7 + jom -j 6 || exit 8 time /t ''' return 1 From cd2f10542dcb4cf37d3388db359332bb425ee11e Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Wed, 18 May 2022 15:51:59 +0800 Subject: [PATCH 05/29] enh: add some log --- Jenkinsfile2 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 29f4c45229..4eb53c315f 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -215,10 +215,12 @@ def pre_test_win(){ } bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + git branch git log -5 ''' bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + git branch git log -5 ''' bat ''' @@ -233,9 +235,13 @@ def pre_test_build_win() { cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal mkdir debug cd debug + time /t call "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" x64 set CL=/MP8 + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> cmake" + time /t cmake .. -G "NMake Makefiles JOM" || exit 7 + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> jom -j 6" jom -j 6 || exit 8 time /t ''' From c7088f08126a7ccd45f914a3f885b6a60fb5f9df Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Wed, 18 May 2022 16:10:35 +0800 Subject: [PATCH 06/29] enh: print time during windows build --- Jenkinsfile2 | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 4eb53c315f..7a6f793399 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -242,6 +242,7 @@ def pre_test_build_win() { time /t cmake .. -G "NMake Makefiles JOM" || exit 7 echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> jom -j 6" + time /t jom -j 6 || exit 8 time /t ''' From 19d64d7c25b631ca6d9c21ed59084fb4d270f951 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 18 May 2022 16:30:50 +0800 Subject: [PATCH 07/29] feat: check ts in submit req msg --- source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/tsdb/tsdbWrite.c | 45 +++++++++++++++++++++---- source/dnode/vnode/src/vnd/vnodeSvr.c | 5 +++ 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 9727d9df9f..0327c4be13 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -104,6 +104,7 @@ int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeep int tsdbClose(STsdb** pTsdb); int tsdbBegin(STsdb* pTsdb); int tsdbCommit(STsdb* pTsdb); +int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); int tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkRsp* pRsp); tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 341ab94ca4..c4d18ef9e6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -15,7 +15,7 @@ #include "tsdb.h" -static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); +// static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp *pRsp) { SSubmitMsgIter msgIter = {0}; @@ -54,7 +54,38 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp * return 0; } -static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { +#if 0 +static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, STable *pTable, STSRow *row, TSKEY minKey, TSKEY maxKey, + TSKEY now) { + TSKEY rowKey = TD_ROW_KEY(row); + if (rowKey < minKey || rowKey > maxKey) { + tsdbError("vgId:%d table %s tid %d uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64 + " maxKey %" PRId64 " row key %" PRId64, + REPO_ID(pTsdb), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), now, minKey, maxKey, + rowKey); + terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; + return -1; + } + + return 0; +} +#endif + +static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, STSRow *row, TSKEY minKey, TSKEY maxKey, + TSKEY now) { + TSKEY rowKey = TD_ROW_KEY(row); + if (rowKey < minKey || rowKey > maxKey) { + tsdbError("vgId:%d table uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64 + " maxKey %" PRId64 " row key %" PRId64, + REPO_ID(pTsdb), uid, now, minKey, maxKey, rowKey); + terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; + return -1; + } + + return 0; +} + +int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { ASSERT(pMsg != NULL); // STsdbMeta * pMeta = pTsdb->tsdbMeta; SSubmitMsgIter msgIter = {0}; @@ -112,14 +143,14 @@ static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { return -1; } } - - tsdbInitSubmitBlkIter(pBlock, &blkIter); - while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) { - if (tsdbCheckRowRange(pTsdb, pTable, row, minKey, maxKey, now) < 0) { +#endif + tInitSubmitBlkIter(&msgIter, pBlock, &blkIter); + while ((row = tGetSubmitBlkNext(&blkIter)) != NULL) { + if (tsdbCheckRowRange(pTsdb, msgIter.uid, row, minKey, maxKey, now) < 0) { return -1; } } -#endif + } if (terrno != TSDB_CODE_SUCCESS) return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 464331319e..80157ec0ba 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -626,6 +626,11 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in vnodeDebugPrintSubmitMsg(pVnode, pReq, __func__); #endif + if (tsdbScanAndConvertSubmitMsg(pVnode->pTsdb, pSubmitReq) < 0) { + pRsp->code = terrno; + goto _exit; + } + // handle the request if (tInitSubmitMsgIter(pSubmitReq, &msgIter) < 0) { pRsp->code = TSDB_CODE_INVALID_MSG; From c85232e14c7168ecd56c61950481abaac9169461 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 18 May 2022 16:37:09 +0800 Subject: [PATCH 08/29] update table meta based on sversion --- include/libs/catalog/catalog.h | 5 +++ source/client/src/clientEnv.c | 10 ++++++ source/client/src/clientImpl.c | 57 ++++++++++++++++++++++++++++--- source/libs/catalog/src/catalog.c | 5 +++ 4 files changed, 73 insertions(+), 4 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 04a24c4f32..a29dc3a90a 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -59,6 +59,11 @@ typedef struct SMetaData { SArray *pQnodeList; // qnode list, SArray } SMetaData; +typedef struct STbSVersion { + char* tbFName; + int32_t sver; +} STbSVersion; + typedef struct SCatalogCfg { uint32_t maxTblCacheNum; uint32_t maxDBCacheNum; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index c82cce9af3..30997def74 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -83,6 +83,15 @@ void closeTransporter(STscObj *pTscObj) { rpcClose(pTscObj->pAppInfo->pTransporter); } +static bool clientRpcRfp(int32_t code) { + if (code == TSDB_CODE_RPC_REDIRECT) { + return true; + } else { + return false; + } +} + + // TODO refactor void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { SRpcInit rpcInit; @@ -91,6 +100,7 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.label = "TSC"; rpcInit.numOfThreads = numOfThread; rpcInit.cfp = processMsgFromServer; + rpcInit.rfp = clientRpcRfp; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)user; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 23787651b2..5d3a139aeb 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -310,9 +310,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList } } - if (pRes) { - *pRes = res.res; - } + *pRes = res.res; pRequest->code = res.code; terrno = res.code; @@ -324,7 +322,49 @@ int32_t getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList) return getPlan(pRequest, pQuery, &pRequest->body.pDag, *pNodeList); } +int32_t validateSversion(SRequestObj* pRequest, void* res) { + SArray* pArray = NULL; + int32_t code = 0; + + if (TDMT_VND_SUBMIT == pRequest->type) { + SSubmitRsp* pRsp = (SSubmitRsp*)res; + if (pRsp->nBlocks <= 0) { + return TSDB_CODE_SUCCESS; + } + + pArray = taosArrayInit(pRsp->nBlocks, sizeof(STbSVersion)); + if (NULL == pArray) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < pRsp->nBlocks; ++i) { + SSubmitBlkRsp *blk = pRsp->pBlocks + i; + STbSVersion tbSver = {.tbFName = blk->tblFName, .sver = blk->sver}; + taosArrayPush(pArray, &tbSver); + } + } else if (TDMT_VND_QUERY == pRequest->type) { + + } + + SCatalog* pCatalog = NULL; + CHECK_CODE_GOTO(catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog), _return); + + SEpSet epset = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + + code = catalogChkTbMetaVersion(pCatalog, pRequest->pTscObj->pAppInfo->pTransporter, &epset, pArray); + +_return: + + taosArrayDestroy(pArray); + + return code; +} + + SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery, void** res) { + void* pRes = NULL; + if (TSDB_CODE_SUCCESS == code) { switch (pQuery->execMode) { case QUERY_EXEC_MODE_LOCAL: @@ -337,7 +377,10 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); code = getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList); if (TSDB_CODE_SUCCESS == code) { - code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList, res); + code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList, &pRes); + if (NULL != pRes) { + code = validateSversion(pRequest, pRes); + } } taosArrayDestroy(pNodeList); break; @@ -356,6 +399,12 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code if (NULL != pRequest && TSDB_CODE_SUCCESS != code) { pRequest->code = terrno; + freeRequestRes(pRequest, pRes); + pRes = NULL; + } + + if (res) { + *res = pRes; } return pRequest; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index d3045f68a1..a620b84f6d 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -2883,6 +2883,11 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pTables) { + +} + + int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName) { CTG_API_ENTER(); From eaba4dba180f72e36a7314f9aa069e0368cb5df0 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 18 May 2022 16:40:21 +0800 Subject: [PATCH 09/29] enh: add const for param --- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/tsdb/tsdbWrite.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 0327c4be13..c2fe63d79c 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -104,7 +104,7 @@ int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeep int tsdbClose(STsdb** pTsdb); int tsdbBegin(STsdb* pTsdb); int tsdbCommit(STsdb* pTsdb); -int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); +int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, const SSubmitReq *pMsg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); int tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkRsp* pRsp); tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index c4d18ef9e6..a67f413ba7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -85,7 +85,7 @@ static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, STSRow *ro return 0; } -int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { +int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, const SSubmitReq *pMsg) { ASSERT(pMsg != NULL); // STsdbMeta * pMeta = pTsdb->tsdbMeta; SSubmitMsgIter msgIter = {0}; From bad77f8e8489189fefa0e3cf435b2fae9a885611 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 18 May 2022 12:05:39 +0000 Subject: [PATCH 10/29] refact TDB --- source/libs/tdb/inc/tdb.h | 4 ++-- source/libs/tdb/src/db/tdbDb.c | 10 +++++----- source/libs/tdb/src/db/tdbTable.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 521b61e3c8..37d5e98c64 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -31,13 +31,13 @@ typedef struct STBC TBC; typedef struct STxn TXN; // TDB -int tdbOpen(const char *rootDir, int szPage, int pages, TDB **ppDb); +int tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb); int tdbClose(TDB *pDb); int tdbBegin(TDB *pDb, TXN *pTxn); int tdbCommit(TDB *pDb, TXN *pTxn); // TTB -int tdbTbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb); +int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb); int tdbTbClose(TTB *pTb); int tdbTbDrop(TTB *pTb); int tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 912c1ecaaf..44c945c21b 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -15,7 +15,7 @@ #include "tdbInt.h" -int tdbOpen(const char *rootDir, int szPage, int pages, TDB **ppDb) { +int tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb) { TDB *pDb; int dsize; int zsize; @@ -25,7 +25,7 @@ int tdbOpen(const char *rootDir, int szPage, int pages, TDB **ppDb) { *ppDb = NULL; - dsize = strlen(rootDir); + dsize = strlen(dbname); zsize = sizeof(*pDb) + dsize * 2 + strlen(TDB_JOURNAL_NAME) + 3; pPtr = (uint8_t *)tdbOsCalloc(1, zsize); @@ -37,12 +37,12 @@ int tdbOpen(const char *rootDir, int szPage, int pages, TDB **ppDb) { pPtr += sizeof(*pDb); // pDb->rootDir pDb->rootDir = pPtr; - memcpy(pDb->rootDir, rootDir, dsize); + memcpy(pDb->rootDir, dbname, dsize); pDb->rootDir[dsize] = '\0'; pPtr = pPtr + dsize + 1; // pDb->jfname pDb->jfname = pPtr; - memcpy(pDb->jfname, rootDir, dsize); + memcpy(pDb->jfname, dbname, dsize); pDb->jfname[dsize] = '/'; memcpy(pDb->jfname + dsize + 1, TDB_JOURNAL_NAME, strlen(TDB_JOURNAL_NAME)); pDb->jfname[dsize + 1 + strlen(TDB_JOURNAL_NAME)] = '\0'; @@ -62,7 +62,7 @@ int tdbOpen(const char *rootDir, int szPage, int pages, TDB **ppDb) { } memset(pDb->pgrHash, 0, tsize); - mkdir(rootDir, 0755); + mkdir(dbname, 0755); *ppDb = pDb; return 0; diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index ce7da0b471..9a8f7a2f55 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -24,7 +24,7 @@ struct STBC { SBTC btc; }; -int tdbTbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb) { +int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb) { TTB *pTb; SPager *pPager; int ret; @@ -42,9 +42,9 @@ int tdbTbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn // pTb->pEnv pTb->pEnv = pEnv; - pPager = tdbEnvGetPager(pEnv, fname); + pPager = tdbEnvGetPager(pEnv, tbname); if (pPager == NULL) { - snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->rootDir, fname); + snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->rootDir, tbname); ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager); if (ret < 0) { return -1; From cda6fa50a91dc1bba09d9ab63ab88ca6b38167e7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 18 May 2022 12:19:32 +0000 Subject: [PATCH 11/29] refact: tdb --- source/libs/tdb/inc/tdb.h | 56 +++++++++++++++---------------- source/libs/tdb/src/db/tdbDb.c | 18 +++++----- source/libs/tdb/src/db/tdbTable.c | 2 +- source/libs/tdb/src/inc/tdbInt.h | 4 +-- 4 files changed, 40 insertions(+), 40 deletions(-) diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 37d5e98c64..5912fd800c 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -22,7 +22,7 @@ extern "C" { #endif -typedef int (*tdb_cmpr_fn_t)(const void *pKey1, int kLen1, const void *pKey2, int kLen2); +typedef int (*tdb_cmpr_fn_t)(const void *pKey1, int32_t kLen1, const void *pKey2, int32_t kLen2); // exposed types typedef struct STDB TDB; @@ -31,42 +31,42 @@ typedef struct STBC TBC; typedef struct STxn TXN; // TDB -int tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb); -int tdbClose(TDB *pDb); -int tdbBegin(TDB *pDb, TXN *pTxn); -int tdbCommit(TDB *pDb, TXN *pTxn); +int32_t tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb); +int32_t tdbClose(TDB *pDb); +int32_t tdbBegin(TDB *pDb, TXN *pTxn); +int32_t tdbCommit(TDB *pDb, TXN *pTxn); // TTB -int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb); -int tdbTbClose(TTB *pTb); -int tdbTbDrop(TTB *pTb); -int tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); -int tdbTbDelete(TTB *pTb, const void *pKey, int kLen, TXN *pTxn); -int tdbTbUpsert(TTB *pTb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); -int tdbTbGet(TTB *pTb, const void *pKey, int kLen, void **ppVal, int *vLen); -int tdbTbPGet(TTB *pTb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); +int32_t tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb); +int32_t tdbTbClose(TTB *pTb); +int32_t tdbTbDrop(TTB *pTb); +int32_t tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); +int32_t tdbTbDelete(TTB *pTb, const void *pKey, int kLen, TXN *pTxn); +int32_t tdbTbUpsert(TTB *pTb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); +int32_t tdbTbGet(TTB *pTb, const void *pKey, int kLen, void **ppVal, int *vLen); +int32_t tdbTbPGet(TTB *pTb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); // TBC -int tdbTbcOpen(TTB *pTb, TBC **ppTbc, TXN *pTxn); -int tdbTbcClose(TBC *pTbc); -int tdbTbcIsValid(TBC *pTbc); -int tdbTbcMoveTo(TBC *pTbc, const void *pKey, int kLen, int *c); -int tdbTbcMoveToFirst(TBC *pTbc); -int tdbTbcMoveToLast(TBC *pTbc); -int tdbTbcMoveToNext(TBC *pTbc); -int tdbTbcMoveToPrev(TBC *pTbc); -int tdbTbcGet(TBC *pTbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen); -int tdbTbcDelete(TBC *pTbc); -int tdbTbcNext(TBC *pTbc, void **ppKey, int *kLen, void **ppVal, int *vLen); -int tdbTbcUpsert(TBC *pTbc, const void *pKey, int nKey, const void *pData, int nData, int insert); +int32_t tdbTbcOpen(TTB *pTb, TBC **ppTbc, TXN *pTxn); +int32_t tdbTbcClose(TBC *pTbc); +int32_t tdbTbcIsValid(TBC *pTbc); +int32_t tdbTbcMoveTo(TBC *pTbc, const void *pKey, int kLen, int *c); +int32_t tdbTbcMoveToFirst(TBC *pTbc); +int32_t tdbTbcMoveToLast(TBC *pTbc); +int32_t tdbTbcMoveToNext(TBC *pTbc); +int32_t tdbTbcMoveToPrev(TBC *pTbc); +int32_t tdbTbcGet(TBC *pTbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen); +int32_t tdbTbcDelete(TBC *pTbc); +int32_t tdbTbcNext(TBC *pTbc, void **ppKey, int *kLen, void **ppVal, int *vLen); +int32_t tdbTbcUpsert(TBC *pTbc, const void *pKey, int nKey, const void *pData, int nData, int insert); // TXN #define TDB_TXN_WRITE 0x1 #define TDB_TXN_READ_UNCOMMITTED 0x2 -int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg, - int flags); -int tdbTxnClose(TXN *pTxn); +int32_t tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), + void *xArg, int flags); +int32_t tdbTxnClose(TXN *pTxn); // other void tdbFree(void *); diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 44c945c21b..c06f7305ab 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -15,7 +15,7 @@ #include "tdbInt.h" -int tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb) { +int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb) { TDB *pDb; int dsize; int zsize; @@ -36,16 +36,16 @@ int tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb) { pDb = (TDB *)pPtr; pPtr += sizeof(*pDb); // pDb->rootDir - pDb->rootDir = pPtr; - memcpy(pDb->rootDir, dbname, dsize); - pDb->rootDir[dsize] = '\0'; + pDb->dbName = pPtr; + memcpy(pDb->dbName, dbname, dsize); + pDb->dbName[dsize] = '\0'; pPtr = pPtr + dsize + 1; // pDb->jfname - pDb->jfname = pPtr; - memcpy(pDb->jfname, dbname, dsize); - pDb->jfname[dsize] = '/'; - memcpy(pDb->jfname + dsize + 1, TDB_JOURNAL_NAME, strlen(TDB_JOURNAL_NAME)); - pDb->jfname[dsize + 1 + strlen(TDB_JOURNAL_NAME)] = '\0'; + pDb->jnName = pPtr; + memcpy(pDb->jnName, dbname, dsize); + pDb->jnName[dsize] = '/'; + memcpy(pDb->jnName + dsize + 1, TDB_JOURNAL_NAME, strlen(TDB_JOURNAL_NAME)); + pDb->jnName[dsize + 1 + strlen(TDB_JOURNAL_NAME)] = '\0'; pDb->jfd = -1; diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index 9a8f7a2f55..7211fe4926 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -44,7 +44,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF pPager = tdbEnvGetPager(pEnv, tbname); if (pPager == NULL) { - snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->rootDir, tbname); + snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->dbName, tbname); ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager); if (ret < 0) { return -1; diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index bb59a7fc61..2c79f39bc0 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -335,8 +335,8 @@ static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { } struct STDB { - char *rootDir; - char *jfname; + char *dbName; + char *jnName; int jfd; SPCache *pCache; SPager *pgrList; From 97a20ee86b93fb3e30e261cda1e43e3db16c11ad Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 18 May 2022 20:31:30 +0800 Subject: [PATCH 12/29] update meta based on sversion --- include/libs/catalog/catalog.h | 2 + source/client/src/clientImpl.c | 11 +++ source/common/src/tmsg.c | 16 ++-- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 2 + source/dnode/vnode/src/vnd/vnodeSvr.c | 3 + source/libs/catalog/src/catalog.c | 101 ++++++++++++++++++++- 6 files changed, 124 insertions(+), 11 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index a29dc3a90a..f9d8fc0de1 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -170,6 +170,8 @@ int32_t catalogUpdateSTableMeta(SCatalog* pCatalog, STableMetaRsp *rspMsg); */ int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName); +int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pTables); + /** * Force refresh a table's local cached meta data. * @param pCatalog (input, got with catalogGetHandle) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 5d3a139aeb..c3ccecb0d3 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -361,6 +361,17 @@ _return: return code; } +void freeRequestRes(SRequestObj* pRequest, void* res) { + if (NULL == res) { + return; + } + + if (TDMT_VND_SUBMIT == pRequest->type) { + tFreeSSubmitRsp((SSubmitRsp*)res); + } else if (TDMT_VND_QUERY == pRequest->type) { + + } +} SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery, void** res) { void* pRes = NULL; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 4e97ebbe47..278fc2c193 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4087,10 +4087,8 @@ static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBl if (tEncodeI32(pEncoder, pBlock->code) < 0) return -1; if (tEncodeI8(pEncoder, pBlock->hashMeta) < 0) return -1; - if (pBlock->hashMeta) { - if (tEncodeI64(pEncoder, pBlock->uid) < 0) return -1; - if (tEncodeCStr(pEncoder, pBlock->tblFName) < 0) return -1; - } + if (tEncodeI64(pEncoder, pBlock->uid) < 0) return -1; + if (tEncodeCStr(pEncoder, pBlock->tblFName) < 0) return -1; if (tEncodeI32v(pEncoder, pBlock->numOfRows) < 0) return -1; if (tEncodeI32v(pEncoder, pBlock->affectedRows) < 0) return -1; if (tEncodeI64v(pEncoder, pBlock->sver) < 0) return -1; @@ -4104,12 +4102,10 @@ static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) { if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1; if (tDecodeI8(pDecoder, &pBlock->hashMeta) < 0) return -1; - if (pBlock->hashMeta) { - if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1; - pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1); - if (NULL == pBlock->tblFName) return -1; - if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1; - } + if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1; + pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1); + if (NULL == pBlock->tblFName) return -1; + if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1; if (tDecodeI32v(pDecoder, &pBlock->numOfRows) < 0) return -1; if (tDecodeI32v(pDecoder, &pBlock->affectedRows) < 0) return -1; if (tDecodeI64v(pDecoder, &pBlock->sver) < 0) return -1; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 2fdbfdd206..d8426db127 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -320,6 +320,8 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; return -1; } + strcat(pRsp->tblFName, mr.me.name); + if (mr.me.type == TSDB_NORMAL_TABLE) { sverNew = mr.me.ntbEntry.schema.sver; } else { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 464331319e..371ade4f76 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -676,6 +676,9 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in vnodeDebugPrintSingleSubmitMsg(pVnode->pMeta, pBlock, &msgIter, "real uid"); tDecoderClear(&decoder); + } else { + submitBlkRsp.tblFName = taosMemoryMalloc(TSDB_TABLE_FNAME_LEN); + sprintf(submitBlkRsp.tblFName, "%s.", pVnode->config.dbname); } if (tsdbInsertTableData(pVnode->pTsdb, &msgIter, pBlock, &submitBlkRsp) < 0) { diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index a620b84f6d..23957d1a6b 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -2883,8 +2883,107 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pTables) { + +int32_t ctgGetTbSverFromCache(SCatalog* pCtg, const SName* pTableName, int32_t* sver) { + *sver = -1; + + if (NULL == pCtg->dbCache) { + ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname); + return TSDB_CODE_SUCCESS; + } + + SCtgDBCache *dbCache = NULL; + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, dbFName); + + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + ctgDebug("db %s not in cache", pTableName->tname); + return TSDB_CODE_SUCCESS; + } + + int32_t tbType = 0; + uint64_t suid = 0; + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + STableMeta* tbMeta = taosHashGet(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname)); + if (tbMeta) { + tbType = tbMeta->tableType; + suid = tbMeta->suid; + if (tbType != TSDB_CHILD_TABLE) { + *sver = tbMeta->sversion; + } + } + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + if (NULL == tbMeta) { + ctgReleaseDBCache(pCtg, dbCache); + return TSDB_CODE_SUCCESS; + } + + if (tbType != TSDB_CHILD_TABLE) { + ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, tbType, dbFName, pTableName->tname); + + return TSDB_CODE_SUCCESS; + } + + ctgDebug("Got subtable meta from cache, dbFName:%s, tbName:%s, suid:%" PRIx64, dbFName, pTableName->tname, suid); + + CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); + + STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &suid, sizeof(suid)); + if (NULL == stbMeta || NULL == *stbMeta) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("stb not in stbCache, suid:%"PRIx64, suid); + return TSDB_CODE_SUCCESS; + } + + if ((*stbMeta)->suid != suid) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgReleaseDBCache(pCtg, dbCache); + ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, suid, (*stbMeta)->suid); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + *sver = (*stbMeta)->sversion; + + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + + ctgReleaseDBCache(pCtg, dbCache); + + ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, tbType, dbFName, pTableName->tname); + + return TSDB_CODE_SUCCESS; +} + + +int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pTables) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTables) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + SName name; + int32_t sver = 0; + int32_t tbNum = taosArrayGetSize(pTables); + for (int32_t i = 0; i < tbNum; ++i) { + STbSVersion* pTb = (STbSVersion*)taosArrayGet(pTables, i); + tNameFromString(&name, pTb->tbFName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + + if (CTG_IS_SYS_DBNAME(name.dbname)) { + continue; + } + + ctgGetTbSverFromCache(pCtg, &name, &sver); + if (sver >= 0 && sver < pTb->sver) { + catalogRemoveTableMeta(pCtg, &name); //TODO REMOVE STB FROM CACHE + } + } + + CTG_API_LEAVE(TSDB_CODE_SUCCESS); } From f94eaa6730c57dd061d841a2124c57ba80aceedf Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 May 2022 20:41:21 +0800 Subject: [PATCH 13/29] fix: only send message to one vnode in the vgroup --- source/dnode/mnode/impl/inc/mndVgroup.h | 2 +- source/dnode/mnode/impl/src/mndDb.c | 57 ++++++++----------------- source/dnode/mnode/impl/src/mndTrans.c | 3 +- source/dnode/mnode/impl/src/mndVgroup.c | 8 ++-- 4 files changed, 23 insertions(+), 47 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 1e95859157..9bf7b6eb89 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -37,7 +37,7 @@ int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); -void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); +void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index c3e5195ad2..0bf7b240b2 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -261,8 +261,7 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { sdbRelease(pSdb, pDb); } -static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, - bool isRedo) { +static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid) { STransAction action = {0}; SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); @@ -279,48 +278,29 @@ static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *p action.msgType = TDMT_DND_CREATE_VNODE; action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; - if (isRedo) { - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } else { - if (mndTransAppendUndoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; } return 0; } -static int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, - bool isRedo) { +static int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { STransAction action = {0}; - - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); int32_t contLen = 0; - void *pReq = mndBuildAlterVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); + void *pReq = mndBuildAlterVnodeReq(pMnode, pDb, pVgroup, &contLen); if (pReq == NULL) return -1; action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_VND_ALTER_VNODE; - if (isRedo) { - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } else { - if (mndTransAppendUndoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; } return 0; @@ -487,7 +467,7 @@ static int32_t mndSetCreateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, true) != 0) { + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid) != 0) { return -1; } } @@ -726,11 +706,8 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) { if (pVgroup->replica <= 0 || pVgroup->replica == pDb->cfg.replications) { - for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, true) != 0) { - return -1; - } + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup) != 0) { + return -1; } } else { SVgObj newVgroup = {0}; @@ -744,9 +721,9 @@ static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj return -1; } newVgroup.replica = pDb->cfg.replications; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[0], true) != 0) return -1; - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1; - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1]) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2]) != 0) return -1; } else { mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId); @@ -757,7 +734,7 @@ static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj return -1; } newVgroup.replica = pDb->cfg.replications; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[0], true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index c9b68f27e7..f2b955a4d8 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -986,7 +986,8 @@ static int32_t mndTransSendActionMsg(SMnode *pMnode, STrans *pTrans, SArray *pAr memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen); if (tmsgSendReq(&pAction->epSet, &rpcMsg) == 0) { - mDebug("trans:%d, action:%d is sent", pTrans->id, action); + mDebug("trans:%d, action:%d is sent to %s:%u", pTrans->id, action, pAction->epSet.eps[pAction->epSet.inUse].fqdn, + pAction->epSet.eps[pAction->epSet.inUse].port); pAction->msgSent = 1; pAction->msgReceived = 0; pAction->errCode = 0; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index f8c717edf6..62021c6a7e 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -256,7 +256,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg return pReq; } -void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { +void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { SAlterVnodeReq alterReq = {0}; alterReq.vgVersion = pVgroup->version; alterReq.buffer = pDb->cfg.buffer; @@ -285,16 +285,14 @@ void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgO pReplica->port = pVgidDnode->port; memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN); mndReleaseDnode(pMnode, pVgidDnode); - - if (pDnode->id == pVgid->dnodeId) { - alterReq.selfIndex = v; - } } +#if 0 if (alterReq.selfIndex == -1) { terrno = TSDB_CODE_MND_APP_ERROR; return NULL; } +#endif int32_t contLen = tSerializeSAlterVnodeReq(NULL, 0, &alterReq); if (contLen < 0) { From 03d15c9e1f53f6d72377887a85a6531c61042b22 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 May 2022 20:41:56 +0800 Subject: [PATCH 14/29] fix: variable initialization order --- source/dnode/mgmt/mgmt_bnode/src/bmWorker.c | 2 +- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 2 +- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 2 +- source/dnode/mgmt/mgmt_snode/src/smWorker.c | 2 +- source/dnode/mgmt/node_mgmt/src/dmProc.c | 2 +- source/dnode/mnode/impl/src/mndTrans.c | 5 ++--- 6 files changed, 7 insertions(+), 8 deletions(-) diff --git a/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c b/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c index 14c6250b2a..e019924268 100644 --- a/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c +++ b/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c @@ -38,9 +38,9 @@ static void bmSendErrorRsps(STaosQall *qall, int32_t numOfMsgs, int32_t code) { static inline void bmSendRsp(SRpcMsg *pMsg, int32_t code) { SRpcMsg rsp = { .code = code, - .info = pMsg->info, .pCont = pMsg->info.rsp, .contLen = pMsg->info.rspLen, + .info = pMsg->info, }; tmsgSendRsp(&rsp); } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index be8ddf1430..599fa07e1c 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -153,9 +153,9 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { if (code != 0 && terrno != 0) code = terrno; SRpcMsg rsp = { .code = code, - .info = pMsg->info, .pCont = pMsg->info.rsp, .contLen = pMsg->info.rspLen, + .info = pMsg->info, }; rpcSendResponse(&rsp); } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index 98e234af3c..9b9960cf3a 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -19,9 +19,9 @@ static inline void mmSendRsp(SRpcMsg *pMsg, int32_t code) { SRpcMsg rsp = { .code = code, - .info = pMsg->info, .pCont = pMsg->info.rsp, .contLen = pMsg->info.rspLen, + .info = pMsg->info, }; tmsgSendRsp(&rsp); } diff --git a/source/dnode/mgmt/mgmt_snode/src/smWorker.c b/source/dnode/mgmt/mgmt_snode/src/smWorker.c index c94cf527c7..fcfc4f4cee 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smWorker.c +++ b/source/dnode/mgmt/mgmt_snode/src/smWorker.c @@ -19,9 +19,9 @@ static inline void smSendRsp(SRpcMsg *pMsg, int32_t code) { SRpcMsg rsp = { .code = code, - .info = pMsg->info, .pCont = pMsg->info.rsp, .contLen = pMsg->info.rspLen, + .info = pMsg->info, }; tmsgSendRsp(&rsp); } diff --git a/source/dnode/mgmt/node_mgmt/src/dmProc.c b/source/dnode/mgmt/node_mgmt/src/dmProc.c index 757f609731..d486707c5e 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmProc.c +++ b/source/dnode/mgmt/node_mgmt/src/dmProc.c @@ -433,7 +433,7 @@ void dmCloseProcRpcHandles(SProc *proc) { SRpcHandleInfo *pInfo = taosHashIterate(proc->hash, NULL); while (pInfo != NULL) { dError("node:%s, the child process dies and send an offline rsp to handle:%p", proc->name, pInfo->handle); - SRpcMsg rpcMsg = {.info = *pInfo, .code = TSDB_CODE_NODE_OFFLINE}; + SRpcMsg rpcMsg = {.code = TSDB_CODE_NODE_OFFLINE, .info = *pInfo}; rpcSendResponse(&rpcMsg); pInfo = taosHashIterate(proc->hash, pInfo); } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index f2b955a4d8..35ecaa748e 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -842,13 +842,12 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { } taosMemoryFree(pTrans->rpcRsp); - mDebug("trans:%d, send rsp, code:0x%x stage:%d app:%p", pTrans->id, code & 0xFFFF, pTrans->stage, - pTrans->rpcInfo.ahandle); + mDebug("trans:%d, send rsp, code:0x%x stage:%d app:%p", pTrans->id, code, pTrans->stage, pTrans->rpcInfo.ahandle); SRpcMsg rspMsg = { - .info = pTrans->rpcInfo, .code = code, .pCont = rpcCont, .contLen = pTrans->rpcRspLen, + .info = pTrans->rpcInfo, }; tmsgSendRsp(&rspMsg); pTrans->rpcInfo.handle = NULL; From 8ede87094f772b519b149ae5a04484c49442191b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 May 2022 21:02:26 +0800 Subject: [PATCH 15/29] fix: incorrect sync status judgment --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 +- source/dnode/mnode/impl/src/mndMnode.c | 2 +- source/dnode/mnode/impl/src/mndVgroup.c | 4 ++-- source/dnode/mnode/impl/src/mnode.c | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 7894eb5acb..bd026b9725 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -117,7 +117,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { SMonMloadInfo minfo = {0}; dmGetMnodeLoads(pMgmt, &minfo); if (minfo.isMnode && minfo.load.syncState != TAOS_SYNC_STATE_LEADER && - minfo.load.syncState != TAOS_SYNC_STATE_CANDIDATE) { + minfo.load.syncState != TAOS_SYNC_STATE_FOLLOWER) { pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED; snprintf(pStatus->details, sizeof(pStatus->details), "mnode sync state is %s", syncStr(minfo.load.syncState)); return; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 04c0a93485..87e69f8360 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -181,7 +181,7 @@ static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) { return -1; } - pObj->role = TAOS_SYNC_STATE_FOLLOWER; + pObj->role = TAOS_SYNC_STATE_CANDIDATE; return 0; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 62021c6a7e..830551ad56 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -416,7 +416,7 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup, SArray *pAr if (pVgroup->replica == 1) { pVgid->role = TAOS_SYNC_STATE_LEADER; } else { - pVgid->role = TAOS_SYNC_STATE_FOLLOWER; + pVgid->role = TAOS_SYNC_STATE_CANDIDATE; } mInfo("db:%s, vgId:%d, vn:%d dnode:%d is alloced", pVgroup->dbName, pVgroup->vgId, v, pVgid->dnodeId); @@ -514,7 +514,7 @@ int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { SVnodeGid *pVgid = &pVgroup->vnodeGid[maxPos]; pVgid->dnodeId = pDnode->id; - pVgid->role = TAOS_SYNC_STATE_FOLLOWER; + pVgid->role = TAOS_SYNC_STATE_CANDIDATE; pDnode->numOfVnodes++; mInfo("db:%s, vgId:%d, vn:%d dnode:%d is added", pVgroup->dbName, pVgroup->vgId, maxPos, pVgid->dnodeId); diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 5326376bda..3b479d42a4 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -489,7 +489,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr tstrncpy(desc.status, "ready", sizeof(desc.status)); pClusterInfo->vgroups_alive++; } - if (pVgid->role == TAOS_SYNC_STATE_LEADER || pVgid->role == TAOS_SYNC_STATE_CANDIDATE) { + if (pVgid->role == TAOS_SYNC_STATE_LEADER || pVgid->role == TAOS_SYNC_STATE_FOLLOWER) { pClusterInfo->vnodes_alive++; } pClusterInfo->vnodes_total++; From a696ed583a8f4ea7a5dd4a270e5efba2dbcee900 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 18 May 2022 21:23:06 +0800 Subject: [PATCH 16/29] test(stream): _wstartts should be reverse quoted --- example/src/tmq.c | 4 +- source/dnode/mnode/impl/src/mndSubscribe.c | 12 +- source/dnode/vnode/src/inc/tq.h | 1 + source/dnode/vnode/src/tq/tq.c | 3 + source/libs/executor/src/executor.c | 2 +- source/libs/executor/src/executorMain.c | 85 ++++++------ source/libs/executor/src/executorimpl.c | 151 +++++++++++---------- source/libs/qworker/src/qworker.c | 104 +++++++------- tests/script/tsim/tstream/basic1.sim | 16 +-- 9 files changed, 195 insertions(+), 183 deletions(-) diff --git a/example/src/tmq.c b/example/src/tmq.c index b79d21d051..00eb8462f4 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -106,8 +106,8 @@ int32_t create_topic() { } taos_free_result(pRes); - pRes = taos_query(pConn, "create topic topic_ctb_column as abc1"); - /*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");*/ + /*pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");*/ + pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1"); if (taos_errno(pRes) != 0) { printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index c853794e86..c82472eec0 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -206,7 +206,7 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { int32_t totalVgNum = pOutput->pSub->vgNum; - mInfo("mq rebalance subscription: %s, vgNum: %d", pOutput->pSub->key, pOutput->pSub->vgNum); + mInfo("mq rebalance: subscription: %s, vgNum: %d", pOutput->pSub->key, pOutput->pSub->vgNum); // 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); @@ -231,6 +231,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); + mInfo("mq rebalance: remove vg %d from consumer %ld", pVgEp->vgId, consumerId); } taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); // put into removed @@ -250,6 +251,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg)); + mInfo("mq rebalance: remove vg %d from unassigned", pVgEp->vgId); } } @@ -263,6 +265,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR minVgCnt = totalVgNum / afterRebConsumerNum; imbConsumerNum = totalVgNum % afterRebConsumerNum; } + mInfo("mq rebalance: %d consumer after rebalance, at least %d vg each, %d consumer has more vg", afterRebConsumerNum, + minVgCnt, imbConsumerNum); // 4. first scan: remove consumer more than wanted, put to remove hash int32_t imbCnt = 0; @@ -290,6 +294,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); + mInfo("mq rebalance: remove vg %d from consumer %ld (first scan)", pVgEp->vgId, pConsumerEp->consumerId); } imbCnt++; } @@ -303,6 +308,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); + mInfo("mq rebalance: remove vg %d from consumer %ld (first scan)", pVgEp->vgId, pConsumerEp->consumerId); } } } @@ -319,6 +325,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR newConsumerEp.vgs = taosArrayInit(0, sizeof(void *)); taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp)); taosArrayPush(pOutput->newConsumers, &consumerId); + mInfo("mq rebalance: add new consumer %ld", consumerId); } } @@ -343,6 +350,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); pRebVg->newConsumerId = pConsumerEp->consumerId; taosArrayPush(pOutput->rebVgs, pRebVg); + mInfo("mq rebalance: add vg %d to consumer %ld (second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId); } } @@ -360,6 +368,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); pRebVg->newConsumerId = pConsumerEp->consumerId; taosArrayPush(pOutput->rebVgs, pRebVg); + mInfo("mq rebalance: add vg %d to consumer %ld (second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId); } } else { // if all consumer is removed, put all vg into unassigned @@ -372,6 +381,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR ASSERT(pRebOutput->newConsumerId == -1); taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp); taosArrayPush(pOutput->rebVgs, pRebOutput); + mInfo("mq rebalance: unassign vg %d (second scan)", pRebOutput->pVgEp->vgId); } } diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 38dedee5a2..d0420e1b84 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -179,6 +179,7 @@ struct STQ { SHashObj* pStreamTasks; SVnode* pVnode; SWal* pWal; + // TDB* pTdb; }; typedef struct { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 80aa350e4d..c69f7d71d5 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -32,6 +32,9 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) { pTq->path = strdup(path); pTq->pVnode = pVnode; pTq->pWal = pWal; + /*if (tdbOpen(path, 4096, 1, &pTq->pTdb) < 0) {*/ + /*ASSERT(0);*/ + /*}*/ #if 0 pTq->tqMeta = tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer, diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 66073b70eb..35b81b135f 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -106,7 +106,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { pMsg->contentLen = pMsg->contentLen; #endif - qDebugL("stream task string %s", (const char*)msg); + /*qDebugL("stream task string %s", (const char*)msg);*/ struct SSubplan* plan = NULL; int32_t code = qStringToSubplan(msg, &plan); diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 354f4d8752..e2af6cb73b 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -15,10 +15,10 @@ #include #include "dataSinkMgt.h" -#include "texception.h" #include "os.h" #include "tarray.h" #include "tcache.h" +#include "texception.h" #include "tglobal.h" #include "tmsg.h" #include "tudf.h" @@ -32,15 +32,15 @@ typedef struct STaskMgmt { TdThreadMutex lock; - SCacheObj *qinfoPool; // query handle pool - int32_t vgId; - bool closed; + SCacheObj *qinfoPool; // query handle pool + int32_t vgId; + bool closed; } STaskMgmt; -int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan, - qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, EOPTR_EXEC_MODEL model) { +int32_t qCreateExecTask(SReadHandle *readHandle, int32_t vgId, uint64_t taskId, SSubplan *pSubplan, + qTaskInfo_t *pTaskInfo, DataSinkHandle *handle, EOPTR_EXEC_MODEL model) { assert(readHandle != NULL && pSubplan != NULL); - SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; + SExecTaskInfo **pTask = (SExecTaskInfo **)pTaskInfo; int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle, taskId, model); if (code != TSDB_CODE_SUCCESS) { @@ -56,46 +56,46 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, code = dsCreateDataSinker(pSubplan->pDataSink, handle); } - _error: +_error: // if failed to add ref for all tables in this query, abort current query return code; } #ifdef TEST_IMPL // wait moment -int waitMoment(SQInfo* pQInfo){ - if(pQInfo->sql) { - int ms = 0; - char* pcnt = strstr(pQInfo->sql, " count(*)"); - if(pcnt) return 0; - - char* pos = strstr(pQInfo->sql, " t_"); - if(pos){ +int waitMoment(SQInfo *pQInfo) { + if (pQInfo->sql) { + int ms = 0; + char *pcnt = strstr(pQInfo->sql, " count(*)"); + if (pcnt) return 0; + + char *pos = strstr(pQInfo->sql, " t_"); + if (pos) { pos += 3; ms = atoi(pos); - while(*pos >= '0' && *pos <= '9'){ - pos ++; + while (*pos >= '0' && *pos <= '9') { + pos++; } char unit_char = *pos; - if(unit_char == 'h'){ - ms *= 3600*1000; - } else if(unit_char == 'm'){ - ms *= 60*1000; - } else if(unit_char == 's'){ + if (unit_char == 'h') { + ms *= 3600 * 1000; + } else if (unit_char == 'm') { + ms *= 60 * 1000; + } else if (unit_char == 's') { ms *= 1000; } } - if(ms == 0) return 0; + if (ms == 0) return 0; printf("test wait sleep %dms. sql=%s ...\n", ms, pQInfo->sql); - - if(ms < 1000) { + + if (ms < 1000) { taosMsleep(ms); } else { int used_ms = 0; - while(used_ms < ms) { + while (used_ms < ms) { taosMsleep(1000); used_ms += 1000; - if(isTaskKilled(pQInfo)){ + if (isTaskKilled(pQInfo)) { printf("test check query is canceled, sleep break.%s\n", pQInfo->sql); break; } @@ -106,15 +106,14 @@ int waitMoment(SQInfo* pQInfo){ } #endif -int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; +int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock **pRes, uint64_t *useconds) { + SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)tinfo; int64_t threadId = taosGetSelfPthreadId(); *pRes = NULL; int64_t curOwner = 0; if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) { - qError("%s-%p execTask is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, - (void*)curOwner); + qError("%s-%p execTask is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, (void *)curOwner); pTaskInfo->code = TSDB_CODE_QRY_IN_EXEC; return pTaskInfo->code; } @@ -133,12 +132,11 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { if (ret != TSDB_CODE_SUCCESS) { publishQueryAbortEvent(pTaskInfo, ret); pTaskInfo->code = ret; - qDebug("%s task abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), - tstrerror(pTaskInfo->code)); + qDebug("%s task abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); return pTaskInfo->code; } - qDebug("%s execTask is launched", GET_TASKID(pTaskInfo)); + /*qDebug("%s execTask is launched", GET_TASKID(pTaskInfo));*/ publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_BEFORE_OPERATOR_EXEC); @@ -154,12 +152,12 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { *useconds = pTaskInfo->cost.elapsedTime; } - int32_t current = (*pRes != NULL)? (*pRes)->info.rows:0; + int32_t current = (*pRes != NULL) ? (*pRes)->info.rows : 0; pTaskInfo->totalRows += current; cleanUpUdfs(); - qDebug("%s task suspended, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms", - GET_TASKID(pTaskInfo), current, pTaskInfo->totalRows, 0, el/1000.0); + /*qDebug("%s task suspended, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms",*/ + /*GET_TASKID(pTaskInfo), current, pTaskInfo->totalRows, 0, el/1000.0);*/ atomic_store_64(&pTaskInfo->owner, 0); return pTaskInfo->code; @@ -208,18 +206,17 @@ int32_t qIsTaskCompleted(qTaskInfo_t qinfo) { } void qDestroyTask(qTaskInfo_t qTaskHandle) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*) qTaskHandle; - qDebug("%s execTask completed, numOfRows:%"PRId64, GET_TASKID(pTaskInfo), pTaskInfo->totalRows); + SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)qTaskHandle; + qDebug("%s execTask completed, numOfRows:%" PRId64, GET_TASKID(pTaskInfo), pTaskInfo->totalRows); - queryCostStatis(pTaskInfo); // print the query cost summary + queryCostStatis(pTaskInfo); // print the query cost summary doDestroyTask(pTaskInfo); } int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, int32_t *resNum, SExplainExecInfo **pRes) { SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)tinfo; - int32_t capacity = 0; + int32_t capacity = 0; - return getOperatorExplainExecInfo(pTaskInfo->pRoot, pRes, &capacity, resNum); + return getOperatorExplainExecInfo(pTaskInfo->pRoot, pRes, &capacity, resNum); } - diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d0a1840d72..b8f06ebc67 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -106,7 +106,7 @@ static void destroyTableQueryInfoImpl(STableQueryInfo* pTableQueryInfo); static SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int32_t* numOfFilterCols); -static void releaseQueryBuf(size_t numOfTables); +static void releaseQueryBuf(size_t numOfTables); static int32_t getNumOfScanTimes(STaskAttr* pQueryAttr); @@ -154,8 +154,9 @@ SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, void operatorDummyCloseFn(void* param, int32_t numOfCols) {} -static int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, - int32_t* rowCellOffset, SqlFunctionCtx* pCtx, int32_t numOfExprs); +static int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, + SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, int32_t* rowCellOffset, + SqlFunctionCtx* pCtx, int32_t numOfExprs); static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); static void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo); @@ -342,14 +343,12 @@ SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, return pResultRow; } -void doClearWindow(SIntervalAggOperatorInfo* pInfo, char* pData, int16_t bytes, - uint64_t groupId, int32_t numOfOutput) { +void doClearWindow(SIntervalAggOperatorInfo* pInfo, char* pData, int16_t bytes, uint64_t groupId, int32_t numOfOutput) { SAggSupporter* pSup = &pInfo->aggSup; SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId); SResultRowPosition* p1 = - (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, - GET_RES_WINDOW_KEY_LEN(bytes)); - SResultRow* pResult = getResultRowByPos(pSup->pResultBuf, p1); + (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + SResultRow* pResult = getResultRowByPos(pSup->pResultBuf, p1); SqlFunctionCtx* pCtx = pInfo->binfo.pCtx; for (int32_t i = 0; i < numOfOutput; ++i) { pCtx[i].resultInfo = getResultCell(pResult, i, pInfo->binfo.rowCellInfoOffset); @@ -599,8 +598,9 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow colDataAppendInt64(pColData, 4, &pQueryWindow->ekey); } -void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, - int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order) { +void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, + SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, + int32_t numOfTotal, int32_t numOfOutput, int32_t order) { for (int32_t k = 0; k < numOfOutput; ++k) { // keep it temporarily bool hasAgg = pCtx[k].input.colDataAggIsSet; @@ -683,8 +683,8 @@ static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pC } } -void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, - bool createDummyCol) { +void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, + int32_t scanFlag, bool createDummyCol) { if (pBlock->pBlockAgg != NULL) { doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); } else { @@ -735,7 +735,7 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc } static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, - int32_t scanFlag, bool createDummyCol) { + int32_t scanFlag, bool createDummyCol) { int32_t code = TSDB_CODE_SUCCESS; for (int32_t i = 0; i < pOperator->numOfExprs; ++i) { @@ -743,7 +743,7 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt pCtx[i].input.numOfRows = pBlock->info.rows; pCtx[i].pSrcBlock = pBlock; - pCtx[i].scanFlag = scanFlag; + pCtx[i].scanFlag = scanFlag; SInputColumnInfoData* pInput = &pCtx[i].input; pInput->uid = pBlock->info.uid; @@ -1003,14 +1003,14 @@ static bool functionNeedToExecute(SqlFunctionCtx* pCtx) { return false; } -// if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_FIRST) { -// // return QUERY_IS_ASC_QUERY(pQueryAttr); -// } -// -// // denote the order type -// if ((functionId == FUNCTION_LAST_DST || functionId == FUNCTION_LAST)) { -// // return pCtx->param[0].i == pQueryAttr->order.order; -// } + // if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_FIRST) { + // // return QUERY_IS_ASC_QUERY(pQueryAttr); + // } + // + // // denote the order type + // if ((functionId == FUNCTION_LAST_DST || functionId == FUNCTION_LAST)) { + // // return pCtx->param[0].i == pQueryAttr->order.order; + // } // in the reverse table scan, only the following functions need to be executed // if (IS_REVERSE_SCAN(pRuntimeEnv) || @@ -1128,16 +1128,16 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu } else if (fmIsAggFunc(pCtx[i].functionId)) { p = &pCtx[i]; } -// if (functionId == FUNCTION_TAG_DUMMY || functionId == FUNCTION_TS_DUMMY) { -// tagLen += pCtx[i].resDataInfo.bytes; -// pTagCtx[num++] = &pCtx[i]; -// } else if (functionId == FUNCTION_TS || functionId == FUNCTION_TAG) { -// // tag function may be the group by tag column -// // ts may be the required primary timestamp column -// continue; -// } else { -// // the column may be the normal column, group by normal_column, the functionId is FUNCTION_PRJ -// } + // if (functionId == FUNCTION_TAG_DUMMY || functionId == FUNCTION_TS_DUMMY) { + // tagLen += pCtx[i].resDataInfo.bytes; + // pTagCtx[num++] = &pCtx[i]; + // } else if (functionId == FUNCTION_TS || functionId == FUNCTION_TAG) { + // // tag function may be the group by tag column + // // ts may be the required primary timestamp column + // continue; + // } else { + // // the column may be the normal column, group by normal_column, the functionId is FUNCTION_PRJ + // } } if (p != NULL) { @@ -2015,7 +2015,7 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO } static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep); -void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, SArray* pColMatchInfo) { +void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, SArray* pColMatchInfo) { if (pFilterNode == NULL) { return; } @@ -2133,8 +2133,9 @@ void setExecutionContext(int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* p * @param pQInfo * @param result */ -int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, - int32_t* rowCellOffset, SqlFunctionCtx* pCtx, int32_t numOfExprs) { +int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, + SGroupResInfo* pGroupResInfo, int32_t* rowCellOffset, SqlFunctionCtx* pCtx, + int32_t numOfExprs) { int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); int32_t start = pGroupResInfo->index; @@ -2172,11 +2173,11 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprIn } else { // expand the result into multiple rows. E.g., _wstartts, top(k, 20) // the _wstartts needs to copy to 20 following rows, since the results of top-k expands to 20 different rows. - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); - char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); - for(int32_t k = 0; k < pRow->numOfRows; ++k) { - colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); - } + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); + char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); + for (int32_t k = 0; k < pRow->numOfRows; ++k) { + colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); + } } } @@ -2192,11 +2193,12 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprIn return 0; } -void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf) { +void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, + SDiskbasedBuf* pBuf) { assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); - SExprInfo* pExprInfo = pOperator->pExpr; - int32_t numOfExprs = pOperator->numOfExprs; + SExprInfo* pExprInfo = pOperator->pExpr; + int32_t numOfExprs = pOperator->numOfExprs; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; int32_t* rowCellOffset = pbInfo->rowCellInfoOffset; @@ -3215,7 +3217,7 @@ static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo) { return TSDB_CODE_SUCCESS; } -SOperatorInfo* createExchangeOperatorInfo(void *pTransporter, const SNodeList* pSources, SSDataBlock* pBlock, +SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { SExchangeInfo* pInfo = taosMemoryCalloc(1, sizeof(SExchangeInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -3328,7 +3330,7 @@ static bool needToMerge(SSDataBlock* pBlock, SArray* groupInfo, char** buf, int3 static void doMergeResultImpl(SSortedMergeOperatorInfo* pInfo, SqlFunctionCtx* pCtx, int32_t numOfExpr, int32_t rowIndex) { for (int32_t j = 0; j < numOfExpr; ++j) { // TODO set row index -// pCtx[j].startRow = rowIndex; + // pCtx[j].startRow = rowIndex; } for (int32_t j = 0; j < numOfExpr; ++j) { @@ -3379,7 +3381,7 @@ static void doMergeImpl(SOperatorInfo* pOperator, int32_t numOfExpr, SSDataBlock SqlFunctionCtx* pCtx = pInfo->binfo.pCtx; for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { -// pCtx[i].size = 1; + // pCtx[i].size = 1; } for (int32_t i = 0; i < pBlock->info.rows; ++i) { @@ -3605,7 +3607,7 @@ _error: return NULL; } -int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scanFlag) { +int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag) { // todo add more information about exchange operation if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_EXCHANGE) { *order = TSDB_ORDER_ASC; @@ -3635,7 +3637,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { SAggOperatorInfo* pAggInfo = pOperator->info; SOptrBasicInfo* pInfo = &pAggInfo->binfo; - SOperatorInfo* downstream = pOperator->pDownstream[0]; + SOperatorInfo* downstream = pOperator->pDownstream[0]; int32_t order = TSDB_ORDER_ASC; int32_t scanFlag = MAIN_SCAN; @@ -3975,7 +3977,8 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order, scanFlag, false); blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows); - code = projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfExprs, pProjectInfo->pPseudoColInfo); + code = projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfExprs, + pProjectInfo->pPseudoColInfo); if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, code); } @@ -4225,7 +4228,7 @@ static STableQueryInfo* initTableQueryInfo(const STableGroupInfo* pTableGroupInf for (int32_t i = 0; i < taosArrayGetSize(pTableGroupInfo->pGroupList); ++i) { SArray* pa = taosArrayGetP(pTableGroupInfo->pGroupList, i); for (int32_t j = 0; j < taosArrayGetSize(pa); ++j) { - STableKeyInfo* pk = taosArrayGet(pa, j); + STableKeyInfo* pk = taosArrayGet(pa, j); STableQueryInfo* pTQueryInfo = &pTableQueryInfo[index++]; pTQueryInfo->lastKey = pk->lastKey; } @@ -4361,9 +4364,9 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p goto _error; } - pInfo->limit = *pLimit; - pInfo->slimit = *pSlimit; - pInfo->curOffset = pLimit->offset; + pInfo->limit = *pLimit; + pInfo->slimit = *pSlimit; + pInfo->curOffset = pLimit->offset; pInfo->curSOffset = pSlimit->offset; pInfo->binfo.pRes = pResBlock; @@ -4503,10 +4506,10 @@ static SColumn* createColumn(int32_t blockId, int32_t slotId, int32_t colId, SDa } pCol->slotId = slotId; - pCol->colId = colId; - pCol->bytes = pType->bytes; - pCol->type = pType->type; - pCol->scale = pType->scale; + pCol->colId = colId; + pCol->bytes = pType->bytes; + pCol->type = pType->type; + pCol->scale = pType->scale; pCol->precision = pType->precision; pCol->dataBlockId = blockId; @@ -4581,10 +4584,10 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* if (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0) { pFuncNode->pParameterList = nodesMakeList(); ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0); - SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); - if (NULL == res) { // todo handle error + SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == res) { // todo handle error } else { - res->node.resType = (SDataType) {.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT}; + res->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT}; nodesListAppend(pFuncNode->pParameterList, res); } } @@ -4677,7 +4680,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SSDataBlock* pResBlock = createResDataBlock(pExchange->node.pOutputDataBlockDesc); return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, pExchange->pSrcEndPoints, pResBlock, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { - SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. + SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; int32_t numOfCols = 0; @@ -4686,27 +4689,27 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo if (pHandle->vnode) { pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableGroupInfo, (uint64_t)queryId, taskId); } else { - doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, - queryId, taskId); + doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId); } if (pDataReader == NULL && terrno != 0) { - qDebug("pDataReader is NULL"); + /*qDebug("pDataReader is NULL");*/ // return NULL; } else { - qDebug("pDataReader is not NULL"); + /*qDebug("pDataReader is not NULL");*/ } SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc; - SOperatorInfo* pOperatorDumy = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo); + SOperatorInfo* pOperatorDumy = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo); SArray* tableIdList = extractTableIdList(pTableGroupInfo); SSDataBlock* pResBlock = createResDataBlock(pDescNode); SArray* pCols = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID); - SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pDataReader, pHandle, pScanPhyNode->uid, pResBlock, pCols, tableIdList, pTaskInfo, - pScanPhyNode->node.pConditions, pOperatorDumy); + SOperatorInfo* pOperator = + createStreamScanOperatorInfo(pHandle->reader, pDataReader, pHandle, pScanPhyNode->uid, pResBlock, pCols, + tableIdList, pTaskInfo, pScanPhyNode->node.pConditions, pOperatorDumy); taosArrayDestroy(tableIdList); return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { @@ -4855,7 +4858,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr; - SColumn col = extractColumnFromColumnNode(pColNode); + SColumn col = extractColumnFromColumnNode(pColNode); pOptr = createStatewindowOperatorInfo(ops[0], pExprInfo, num, pResBlock, &as, tsSlotId, &col, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_JOIN == type) { SJoinPhysiNode* pJoinNode = (SJoinPhysiNode*)pPhyNode; @@ -4923,11 +4926,11 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi SColumn extractColumnFromColumnNode(SColumnNode* pColNode) { SColumn c = {0}; - c.slotId = pColNode->slotId; - c.colId = pColNode->colId; - c.type = pColNode->node.resType.type; - c.bytes = pColNode->node.resType.bytes; - c.scale = pColNode->node.resType.scale; + c.slotId = pColNode->slotId; + c.colId = pColNode->colId; + c.type = pColNode->node.resType.type; + c.bytes = pColNode->node.resType.bytes; + c.scale = pColNode->node.resType.scale; c.precision = pColNode->node.resType.precision; return c; } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index f48095ecb8..db63c71d11 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -9,11 +9,11 @@ #include "tmsg.h" #include "tname.h" -SQWDebug gQWDebug = {.statusEnable = true, .dumpEnable = true}; +SQWDebug gQWDebug = {.statusEnable = true, .dumpEnable = true}; SQWorkerMgmt gQwMgmt = { - .lock = 0, - .qwRef = -1, - .qwNum = 0, + .lock = 0, + .qwRef = -1, + .qwNum = 0, }; int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, bool *ignore) { @@ -110,9 +110,9 @@ void qwDbgDumpMgmtInfo(SQWorker *mgmt) { QW_LOCK(QW_READ, &mgmt->schLock); - QW_DUMP("total remain schduler num:%d", taosHashGetSize(mgmt->schHash)); + /*QW_DUMP("total remain schduler num:%d", taosHashGetSize(mgmt->schHash));*/ - void * key = NULL; + void *key = NULL; size_t keyLen = 0; int32_t i = 0; SQWSchStatus *sch = NULL; @@ -127,7 +127,7 @@ void qwDbgDumpMgmtInfo(SQWorker *mgmt) { QW_UNLOCK(QW_READ, &mgmt->schLock); - QW_DUMP("total remain ctx num:%d", taosHashGetSize(mgmt->ctxHash)); + /*QW_DUMP("total remain ctx num:%d", taosHashGetSize(mgmt->ctxHash));*/ } char *qwPhaseStr(int32_t phase) { @@ -462,7 +462,7 @@ int32_t qwDropTaskCtx(QW_FPARAMS_DEF) { } int32_t qwDropTaskStatus(QW_FPARAMS_DEF) { - SQWSchStatus * sch = NULL; + SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; @@ -499,7 +499,7 @@ _return: } int32_t qwUpdateTaskStatus(QW_FPARAMS_DEF, int8_t status) { - SQWSchStatus * sch = NULL; + SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; @@ -550,11 +550,11 @@ int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { int32_t code = 0; bool qcontinue = true; - SSDataBlock * pRes = NULL; + SSDataBlock *pRes = NULL; uint64_t useconds = 0; int32_t i = 0; int32_t execNum = 0; - qTaskInfo_t * taskHandle = &ctx->taskHandle; + qTaskInfo_t *taskHandle = &ctx->taskHandle; DataSinkHandle sinkHandle = ctx->sinkHandle; while (true) { @@ -632,7 +632,7 @@ int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) return TSDB_CODE_QRY_OUT_OF_MEMORY; } - void * key = NULL; + void *key = NULL; size_t keyLen = 0; int32_t i = 0; STaskStatus status = {0}; @@ -719,8 +719,8 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void } int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { - int32_t code = 0; - SQWTaskCtx * ctx = NULL; + int32_t code = 0; + SQWTaskCtx *ctx = NULL; SRpcHandleInfo *dropConnection = NULL; SRpcHandleInfo *cancelConnection = NULL; @@ -925,13 +925,13 @@ _return: } int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t explain) { - int32_t code = 0; - bool queryRsped = false; - SSubplan* plan = NULL; - SQWPhaseInput input = {0}; - qTaskInfo_t pTaskInfo = NULL; - DataSinkHandle sinkHandle = NULL; - SQWTaskCtx * ctx = NULL; + int32_t code = 0; + bool queryRsped = false; + SSubplan *plan = NULL; + SQWPhaseInput input = {0}; + qTaskInfo_t pTaskInfo = NULL; + DataSinkHandle sinkHandle = NULL; + SQWTaskCtx *ctx = NULL; QW_ERR_JRET(qwRegisterQueryBrokenLinkArg(QW_FPARAMS(), &qwMsg->connInfo)); @@ -944,7 +944,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t ex ctx->ctrlConnInfo = qwMsg->connInfo; - QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg); + /*QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg);*/ code = qStringToSubplan(qwMsg->msg, &plan); if (TSDB_CODE_SUCCESS != code) { @@ -1055,10 +1055,10 @@ _return: } int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { - SQWTaskCtx * ctx = NULL; + SQWTaskCtx *ctx = NULL; int32_t code = 0; SQWPhaseInput input = {0}; - void * rsp = NULL; + void *rsp = NULL; int32_t dataLen = 0; bool queryEnd = false; @@ -1138,8 +1138,8 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { int32_t code = 0; int32_t dataLen = 0; bool locked = false; - SQWTaskCtx * ctx = NULL; - void * rsp = NULL; + SQWTaskCtx *ctx = NULL; + void *rsp = NULL; SQWPhaseInput input = {0}; QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_FETCH, &input, NULL)); @@ -1274,7 +1274,7 @@ _return: int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { int32_t code = 0; SSchedulerHbRsp rsp = {0}; - SQWSchStatus * sch = NULL; + SQWSchStatus *sch = NULL; QW_ERR_RET(qwAcquireAddScheduler(mgmt, req->sId, QW_READ, &sch)); @@ -1300,7 +1300,7 @@ int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *re int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { int32_t code = 0; SSchedulerHbRsp rsp = {0}; - SQWSchStatus * sch = NULL; + SQWSchStatus *sch = NULL; if (qwMsg->code) { QW_RET(qwProcessHbLinkBroken(mgmt, qwMsg, req)); @@ -1338,28 +1338,28 @@ _return: qwMsg->connInfo.handle = NULL; } - QW_DLOG("hb rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code)); + /*QW_DLOG("hb rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code));*/ QW_RET(TSDB_CODE_SUCCESS); } void qwProcessHbTimerEvent(void *param, void *tmrId) { - SQWHbParam* hbParam = (SQWHbParam*)param; + SQWHbParam *hbParam = (SQWHbParam *)param; if (hbParam->qwrId != atomic_load_32(&gQwMgmt.qwRef)) { return; } - int64_t refId = hbParam->refId; + int64_t refId = hbParam->refId; SQWorker *mgmt = qwAcquire(refId); if (NULL == mgmt) { QW_DLOG("qwAcquire %" PRIx64 "failed", refId); taosMemoryFree(param); return; } - + SQWSchStatus *sch = NULL; int32_t taskNum = 0; - SQWHbInfo * rspList = NULL; + SQWHbInfo *rspList = NULL; int32_t code = 0; qwDbgDumpMgmtInfo(mgmt); @@ -1383,7 +1383,7 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { return; } - void * key = NULL; + void *key = NULL; size_t keyLen = 0; int32_t i = 0; @@ -1413,29 +1413,27 @@ _return: for (int32_t j = 0; j < i; ++j) { qwBuildAndSendHbRsp(&rspList[j].connInfo, &rspList[j].rsp, code); - QW_DLOG("hb rsp send, handle:%p, code:%x - %s, taskNum:%d", rspList[j].connInfo.handle, code, tstrerror(code), - (rspList[j].rsp.taskStatus ? (int32_t)taosArrayGetSize(rspList[j].rsp.taskStatus) : 0)); + /*QW_DLOG("hb rsp send, handle:%p, code:%x - %s, taskNum:%d", rspList[j].connInfo.handle, code, tstrerror(code),*/ + /*(rspList[j].rsp.taskStatus ? (int32_t)taosArrayGetSize(rspList[j].rsp.taskStatus) : 0));*/ tFreeSSchedulerHbRsp(&rspList[j].rsp); } taosMemoryFreeClear(rspList); taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); - qwRelease(refId); + qwRelease(refId); } void qwCloseRef(void) { taosWLockLatch(&gQwMgmt.lock); if (atomic_load_32(&gQwMgmt.qwNum) <= 0 && gQwMgmt.qwRef >= 0) { taosCloseRef(gQwMgmt.qwRef); - gQwMgmt.qwRef= -1; + gQwMgmt.qwRef = -1; } taosWUnLockLatch(&gQwMgmt.lock); } -void qwDestroySchStatus(SQWSchStatus *pStatus) { - taosHashCleanup(pStatus->tasksHash); -} +void qwDestroySchStatus(SQWSchStatus *pStatus) { taosHashCleanup(pStatus->tasksHash); } void qwDestroyImpl(void *pMgmt) { SQWorker *mgmt = (SQWorker *)pMgmt; @@ -1454,12 +1452,12 @@ void qwDestroyImpl(void *pMgmt) { SQWSchStatus *sch = (SQWSchStatus *)pIter; qwDestroySchStatus(sch); pIter = taosHashIterate(mgmt->schHash, pIter); - } + } taosHashCleanup(mgmt->schHash); taosMemoryFree(mgmt); - atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); + atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); qwCloseRef(); } @@ -1467,7 +1465,7 @@ void qwDestroyImpl(void *pMgmt) { int32_t qwOpenRef(void) { taosWLockLatch(&gQwMgmt.lock); if (gQwMgmt.qwRef < 0) { - gQwMgmt.qwRef= taosOpenRef(100, qwDestroyImpl); + gQwMgmt.qwRef = taosOpenRef(100, qwDestroyImpl); if (gQwMgmt.qwRef < 0) { taosWUnLockLatch(&gQwMgmt.lock); qError("init qworker ref failed"); @@ -1475,14 +1473,14 @@ int32_t qwOpenRef(void) { } } taosWUnLockLatch(&gQwMgmt.lock); - + return TSDB_CODE_SUCCESS; } void qwSetHbParam(int64_t refId, SQWHbParam **pParam) { int32_t paramIdx = 0; int32_t newParamIdx = 0; - + while (true) { paramIdx = atomic_load_32(&gQwMgmt.paramIdx); if (paramIdx == tListLen(gQwMgmt.param)) { @@ -1490,7 +1488,7 @@ void qwSetHbParam(int64_t refId, SQWHbParam **pParam) { } else { newParamIdx = paramIdx + 1; } - + if (paramIdx == atomic_val_compare_exchange_32(&gQwMgmt.paramIdx, paramIdx, newParamIdx)) { break; } @@ -1577,12 +1575,12 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW SQWHbParam *param = NULL; qwSetHbParam(mgmt->refId, ¶m); - mgmt->hbTimer = taosTmrStart(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, (void*)param, mgmt->timer); + mgmt->hbTimer = taosTmrStart(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, (void *)param, mgmt->timer); if (NULL == mgmt->hbTimer) { qError("start hb timer failed"); QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + *qWorkerMgmt = mgmt; qDebug("qworker initialized for node, type:%d, id:%d, handle:%p", mgmt->nodeType, mgmt->nodeId, mgmt); @@ -1599,9 +1597,9 @@ _return: taosTmrCleanUp(mgmt->timer); taosMemoryFreeClear(mgmt); - atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); + atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); } - + QW_RET(code); } @@ -1678,7 +1676,7 @@ int32_t qwUpdateSchLastAccess(SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64 } int32_t qwGetTaskStatus(SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int8_t *taskStatus) { - SQWSchStatus * sch = NULL; + SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; @@ -1705,7 +1703,7 @@ int32_t qwGetTaskStatus(SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId } int32_t qwCancelTask(SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { - SQWSchStatus * sch = NULL; + SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; diff --git a/tests/script/tsim/tstream/basic1.sim b/tests/script/tsim/tstream/basic1.sim index cb084ad537..9965772c75 100644 --- a/tests/script/tsim/tstream/basic1.sim +++ b/tests/script/tsim/tstream/basic1.sim @@ -24,7 +24,7 @@ sql insert into t1 values(1648791233002,3,2,3,2.1); sql insert into t1 values(1648791243003,4,2,3,3.1); sql insert into t1 values(1648791213004,4,2,3,4.1); sleep 1000 -sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; +sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; if $rows != 4 then print ======$rows @@ -137,7 +137,7 @@ endi sql insert into t1 values(1648791223001,12,14,13,11.1); sleep 500 -sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; +sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; if $rows != 4 then print ======$rows @@ -250,7 +250,7 @@ endi sql insert into t1 values(1648791223002,12,14,13,11.1); sleep 100 -sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; +sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; # row 1 if $data11 != 2 then @@ -280,7 +280,7 @@ endi sql insert into t1 values(1648791223003,12,14,13,11.1); sleep 100 -sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; +sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; # row 1 if $data11 != 3 then @@ -312,7 +312,7 @@ sql insert into t1 values(1648791223001,1,1,1,1.1); sql insert into t1 values(1648791223002,2,2,2,2.1); sql insert into t1 values(1648791223003,3,3,3,3.1); sleep 100 -sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; +sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; # row 1 if $data11 != 3 then @@ -344,7 +344,7 @@ sql insert into t1 values(1648791233003,3,2,3,2.1); sql insert into t1 values(1648791233002,5,6,7,8.1); sql insert into t1 values(1648791233002,3,2,3,2.1); sleep 100 -sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; +sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; # row 2 if $data21 != 2 then @@ -374,7 +374,7 @@ endi sql insert into t1 values(1648791213004,4,2,3,4.1) (1648791213006,5,4,7,9.1) (1648791213004,40,20,30,40.1) (1648791213005,4,2,3,4.1); sleep 100 -sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; +sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; # row 0 if $data01 != 4 then @@ -404,7 +404,7 @@ endi sql insert into t1 values(1648791223004,4,2,3,4.1) (1648791233006,5,4,7,9.1) (1648791223004,40,20,30,40.1) (1648791233005,4,2,3,4.1); sleep 100 -sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; +sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; # row 1 if $data11 != 4 then From 8fca7d9a5107dd4161232762748aecac969e0864 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 May 2022 21:25:21 +0800 Subject: [PATCH 17/29] fix: incorrect sync status --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 5 ++--- source/dnode/mnode/impl/src/mndMnode.c | 2 +- source/dnode/mnode/impl/src/mndVgroup.c | 4 ++-- source/dnode/mnode/impl/src/mnode.c | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index bd026b9725..38e71754b6 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -116,8 +116,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { SServerStatusRsp statusRsp = {0}; SMonMloadInfo minfo = {0}; dmGetMnodeLoads(pMgmt, &minfo); - if (minfo.isMnode && minfo.load.syncState != TAOS_SYNC_STATE_LEADER && - minfo.load.syncState != TAOS_SYNC_STATE_FOLLOWER) { + if (minfo.isMnode && minfo.load.syncState == TAOS_SYNC_STATE_ERROR) { pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED; snprintf(pStatus->details, sizeof(pStatus->details), "mnode sync state is %s", syncStr(minfo.load.syncState)); return; @@ -127,7 +126,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { dmGetVnodeLoads(pMgmt, &vinfo); for (int32_t i = 0; i < taosArrayGetSize(vinfo.pVloads); ++i) { SVnodeLoad *pLoad = taosArrayGet(vinfo.pVloads, i); - if (pLoad->syncState != TAOS_SYNC_STATE_LEADER && pLoad->syncState != TAOS_SYNC_STATE_FOLLOWER) { + if (pLoad->syncState == TAOS_SYNC_STATE_ERROR) { pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED; snprintf(pStatus->details, sizeof(pStatus->details), "vnode:%d sync state is %s", pLoad->vgId, syncStr(pLoad->syncState)); diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 87e69f8360..04c0a93485 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -181,7 +181,7 @@ static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) { return -1; } - pObj->role = TAOS_SYNC_STATE_CANDIDATE; + pObj->role = TAOS_SYNC_STATE_FOLLOWER; return 0; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 830551ad56..62021c6a7e 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -416,7 +416,7 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup, SArray *pAr if (pVgroup->replica == 1) { pVgid->role = TAOS_SYNC_STATE_LEADER; } else { - pVgid->role = TAOS_SYNC_STATE_CANDIDATE; + pVgid->role = TAOS_SYNC_STATE_FOLLOWER; } mInfo("db:%s, vgId:%d, vn:%d dnode:%d is alloced", pVgroup->dbName, pVgroup->vgId, v, pVgid->dnodeId); @@ -514,7 +514,7 @@ int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { SVnodeGid *pVgid = &pVgroup->vnodeGid[maxPos]; pVgid->dnodeId = pDnode->id; - pVgid->role = TAOS_SYNC_STATE_CANDIDATE; + pVgid->role = TAOS_SYNC_STATE_FOLLOWER; pDnode->numOfVnodes++; mInfo("db:%s, vgId:%d, vn:%d dnode:%d is added", pVgroup->dbName, pVgroup->vgId, maxPos, pVgid->dnodeId); diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 3b479d42a4..8c805dd8c7 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -489,7 +489,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr tstrncpy(desc.status, "ready", sizeof(desc.status)); pClusterInfo->vgroups_alive++; } - if (pVgid->role == TAOS_SYNC_STATE_LEADER || pVgid->role == TAOS_SYNC_STATE_FOLLOWER) { + if (pVgid->role != TAOS_SYNC_STATE_ERROR) { pClusterInfo->vnodes_alive++; } pClusterInfo->vnodes_total++; From 3502a58cd526bbafe890dd9f07677fa17e264a6c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 May 2022 21:49:28 +0800 Subject: [PATCH 18/29] fix: bad log print flags --- include/libs/qcom/query.h | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 711db65e97..daf008108b 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -220,23 +220,23 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t taosPrintLog("QRY ", DEBUG_INFO, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ } \ } while (0) -#define qDebug(...) \ - do { \ - if (qDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ - } \ +#define qDebug(...) \ + do { \ + if (qDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \ + } \ } while (0) -#define qTrace(...) \ - do { \ - if (qDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("QRY ", DEBUG_TRACE, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ - } \ +#define qTrace(...) \ + do { \ + if (qDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); \ + } \ } while (0) -#define qDebugL(...) \ - do { \ - if (qDebugFlag & DEBUG_DEBUG) { \ - taosPrintLongString("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ - } \ +#define qDebugL(...) \ + do { \ + if (qDebugFlag & DEBUG_DEBUG) { \ + taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \ + } \ } while (0) #define QRY_ERR_RET(c) \ From 534dfae90769d495f172ca65d79ce6536e04d5dc Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 18 May 2022 22:06:59 +0800 Subject: [PATCH 19/29] [test: set days to 300 for default db] --- tests/pytest/util/sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index adbab04e07..bdda7c453b 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -57,7 +57,7 @@ class TDSql: tdLog.notice("'reset query cache' is not supported") s = 'drop database if exists db' self.cursor.execute(s) - s = 'create database db' + s = 'create database db days 300' self.cursor.execute(s) s = 'use db' self.cursor.execute(s) From f41caed14aeec9dc72a0d005390802411c44f587 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 18 May 2022 22:52:20 +0800 Subject: [PATCH 20/29] fix: merge bitmap when data col have data --- source/common/src/trow.c | 2 +- source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index d1516403c1..22bdd960ea 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -1063,7 +1063,7 @@ bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t fl int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row, int8_t bitmapMode) { if (isAllRowsNone(pCol)) { - pVal->valType = TD_VTYPE_NULL; + pVal->valType = TD_VTYPE_NONE; #ifdef TD_SUPPORT_READ2 pVal->val = (void *)getNullValue(pCol->type); #else diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index bebdfb3b63..c1a1e7570e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -330,12 +330,12 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, ASSERT(pReadh->pDCols[0]->bitmapMode != 0); } - if (mergeBitmap && !tdDataColsIsBitmapI(pReadh->pDCols[0])) { for (int i = 0; i < numOfColsIds; ++i) { SDataCol *pDataCol = pReadh->pDCols[0]->cols + i; - if (pDataCol->bitmap) { + if (pDataCol->len > 0 && pDataCol->bitmap) { ASSERT(pDataCol->colId != PRIMARYKEY_TIMESTAMP_COL_ID); + ASSERT(pDataCol->pBitmap); tdMergeBitmap(pDataCol->pBitmap, pReadh->pDCols[0]->numOfRows, pDataCol->pBitmap); tdDataColsSetBitmapI(pReadh->pDCols[0]); } From 54420f06fc13983e3c104a0cd0c25d216241e514 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 19 May 2022 09:23:15 +0800 Subject: [PATCH 21/29] feat: put dup record in the last --- include/util/tskiplist.h | 2 +- source/util/src/tskiplist.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/util/tskiplist.h b/include/util/tskiplist.h index eeae1b47da..10d3dcdbaa 100644 --- a/include/util/tskiplist.h +++ b/include/util/tskiplist.h @@ -56,10 +56,10 @@ typedef enum { SSkipListPutSuccess = 0, SSkipListPutEarlyStop = 1, SSkipListPutS typedef struct SSkipList { uint32_t seed; + uint16_t len; __compar_fn_t comparFn; __sl_key_fn_t keyFn; TdThreadRwlock *lock; - uint16_t len; uint8_t maxLevel; uint8_t flags; uint8_t type; // static info above diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index 13637b8fe4..4ce668e6ba 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -185,10 +185,10 @@ void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t it pKey = SL_GET_NODE_KEY(pSkipList, p); compare = pSkipList->comparFn(pKey, pDataKey); - if (compare >= 0) { - if (compare == 0 && !hasDup) hasDup = true; + if (compare > 0) { break; } else { + if (compare == 0 && !hasDup) hasDup = true; px = p; p = SL_NODE_GET_FORWARD_POINTER(px, i); } From fb944e85bc247378fb92eb67ba3237e5a93ec7a8 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 19 May 2022 09:39:28 +0800 Subject: [PATCH 22/29] fix explain res issue --- include/libs/scheduler/scheduler.h | 2 +- source/client/src/clientImpl.c | 2 +- source/libs/scheduler/inc/schedulerInt.h | 8 +++- source/libs/scheduler/src/scheduler.c | 46 +++++++++---------- source/libs/scheduler/test/schedulerTests.cpp | 2 +- 5 files changed, 33 insertions(+), 27 deletions(-) diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index b3f35025d1..dcd058a293 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -72,7 +72,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg); * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, int64_t startTs, bool needRes, SQueryResult *pRes); +int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, int64_t startTs, SQueryResult *pRes); /** * Process the query job, generated according to the query physical plan. diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index c3ccecb0d3..7693d26d3f 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -291,7 +291,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr, - pRequest->metric.start, NULL != pRes, &res); + pRequest->metric.start, &res); if (code != TSDB_CODE_SUCCESS) { if (pRequest->body.queryJob != 0) { schedulerFreeJob(pRequest->body.queryJob); diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index a90fb7fc2e..5a6fcee759 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -39,6 +39,12 @@ enum { SCH_WRITE, }; +typedef enum { + SCH_RES_TYPE_QUERY, + SCH_RES_TYPE_FETCH, +} SCH_RES_TYPE; + + typedef struct SSchTrans { void *transInst; void *transHandle; @@ -159,7 +165,6 @@ typedef struct SSchTask { typedef struct SSchJobAttr { EExplainMode explainMode; - bool needRes; bool syncSchedule; bool queryJob; bool needFlowCtrl; @@ -192,6 +197,7 @@ typedef struct SSchJob { int32_t errCode; SArray *errList; // SArray SRWLatch resLock; + SCH_RES_TYPE resType; void *resData; //TODO free it or not int32_t resNumOfRows; const char *sql; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index bdf674ab93..9354c1a875 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -70,7 +70,7 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel * } int32_t schInitJob(SSchJob **pSchJob, SQueryPlan *pDag, void *transport, SArray *pNodeList, const char *sql, - int64_t startTs, bool needRes, bool syncSchedule) { + int64_t startTs, bool syncSchedule) { int32_t code = 0; int64_t refId = -1; SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); @@ -81,7 +81,6 @@ int32_t schInitJob(SSchJob **pSchJob, SQueryPlan *pDag, void *transport, SArray pJob->attr.explainMode = pDag->explainInfo.mode; pJob->attr.syncSchedule = syncSchedule; - pJob->attr.needRes = needRes; pJob->transport = transport; pJob->sql = sql; @@ -1059,6 +1058,8 @@ _return: int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRsp *pRsp) { SCH_TASK_DLOG("got explain rsp, rows:%d, complete:%d", htonl(pRsp->numOfRows), pRsp->completed); + pJob->resType = SCH_RES_TYPE_FETCH; + atomic_store_32(&pJob->resNumOfRows, htonl(pRsp->numOfRows)); atomic_store_ptr(&pJob->resData, pRsp); @@ -1179,23 +1180,20 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch atomic_add_fetch_32(&pJob->resNumOfRows, rsp->affectedRows); SCH_TASK_DLOG("submit succeed, affectedRows:%d", rsp->affectedRows); - if (pJob->attr.needRes) { - SCH_LOCK(SCH_WRITE, &pJob->resLock); - if (pJob->resData) { - SSubmitRsp *sum = pJob->resData; - sum->affectedRows += rsp->affectedRows; - sum->nBlocks += rsp->nBlocks; - sum->pBlocks = taosMemoryRealloc(sum->pBlocks, sum->nBlocks * sizeof(*sum->pBlocks)); - memcpy(sum->pBlocks + sum->nBlocks - rsp->nBlocks, rsp->pBlocks, rsp->nBlocks * sizeof(*sum->pBlocks)); - taosMemoryFree(rsp->pBlocks); - taosMemoryFree(rsp); - } else { - pJob->resData = rsp; - } - SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + pJob->resType = SCH_RES_TYPE_QUERY; + SCH_LOCK(SCH_WRITE, &pJob->resLock); + if (pJob->resData) { + SSubmitRsp *sum = pJob->resData; + sum->affectedRows += rsp->affectedRows; + sum->nBlocks += rsp->nBlocks; + sum->pBlocks = taosMemoryRealloc(sum->pBlocks, sum->nBlocks * sizeof(*sum->pBlocks)); + memcpy(sum->pBlocks + sum->nBlocks - rsp->nBlocks, rsp->pBlocks, rsp->nBlocks * sizeof(*sum->pBlocks)); + taosMemoryFree(rsp->pBlocks); + taosMemoryFree(rsp); } else { - tFreeSSubmitRsp(rsp); + pJob->resData = rsp; } + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); } SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); @@ -2412,7 +2410,7 @@ void schFreeJobImpl(void *job) { } static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pDag, int64_t *job, const char *sql, - int64_t startTs, bool needRes, bool syncSchedule) { + int64_t startTs, bool syncSchedule) { qDebug("QID:0x%" PRIx64 " job started", pDag->queryId); if (pNodeList == NULL || taosArrayGetSize(pNodeList) <= 0) { @@ -2421,7 +2419,7 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pD int32_t code = 0; SSchJob *pJob = NULL; - SCH_ERR_JRET(schInitJob(&pJob, pDag, transport, pNodeList, sql, startTs, needRes, syncSchedule)); + SCH_ERR_JRET(schInitJob(&pJob, pDag, transport, pNodeList, sql, startTs, syncSchedule)); SCH_ERR_JRET(schLaunchJob(pJob)); @@ -2463,6 +2461,8 @@ int32_t schExecStaticExplain(void *transport, SArray *pNodeList, SQueryPlan *pDa SCH_ERR_JRET(qExecStaticExplain(pDag, (SRetrieveTableRsp **)&pJob->resData)); + pJob->resType = SCH_RES_TYPE_FETCH; + int64_t refId = taosAddRef(schMgmt.jobRef, pJob); if (refId < 0) { SCH_JOB_ELOG("taosAddRef job failed, error:%s", tstrerror(terrno)); @@ -2535,7 +2535,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { } int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, - int64_t startTs, bool needRes, SQueryResult *pRes) { + int64_t startTs, SQueryResult *pRes) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -2543,14 +2543,14 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, in if (EXPLAIN_MODE_STATIC == pDag->explainInfo.mode) { SCH_ERR_RET(schExecStaticExplain(transport, nodeList, pDag, pJob, sql, true)); } else { - SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, startTs, needRes, true)); + SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, startTs, true)); } SSchJob *job = schAcquireJob(*pJob); pRes->code = atomic_load_32(&job->errCode); pRes->numOfRows = job->resNumOfRows; - if (needRes) { + if (SCH_RES_TYPE_QUERY == job->resType) { pRes->res = job->resData; job->resData = NULL; } @@ -2568,7 +2568,7 @@ int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryPlan *pD if (EXPLAIN_MODE_STATIC == pDag->explainInfo.mode) { SCH_ERR_RET(schExecStaticExplain(transport, pNodeList, pDag, pJob, sql, false)); } else { - SCH_ERR_RET(schExecJobImpl(transport, pNodeList, pDag, pJob, sql, 0, false, false)); + SCH_ERR_RET(schExecJobImpl(transport, pNodeList, pDag, pJob, sql, 0, false)); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 09ecd9fffd..fc0e05aaf1 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -985,7 +985,7 @@ TEST(insertTest, normalCase) { taosThreadCreate(&(thread1), &thattr, schtSendRsp, &insertJobRefId); SQueryResult res = {0}; - code = schedulerExecJob(mockPointer, qnodeList, &dag, &insertJobRefId, "insert into tb values(now,1)", 0, false, &res); + code = schedulerExecJob(mockPointer, qnodeList, &dag, &insertJobRefId, "insert into tb values(now,1)", 0, &res); ASSERT_EQ(code, 0); ASSERT_EQ(res.numOfRows, 20); From 23e415b9272d1c544fa09a06f456a0de2199d066 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 19 May 2022 09:41:08 +0800 Subject: [PATCH 23/29] test:modify days to 300 --- tests/script/tsim/query/interval-offset.sim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/script/tsim/query/interval-offset.sim b/tests/script/tsim/query/interval-offset.sim index 68860dc2cb..dcd88e5a0c 100644 --- a/tests/script/tsim/query/interval-offset.sim +++ b/tests/script/tsim/query/interval-offset.sim @@ -5,7 +5,7 @@ sleep 500 sql connect print =============== create database -sql create database d0 +sql create database d0 days 300 sql use d0 print =============== create super table and child table @@ -254,4 +254,4 @@ endi #sql select count(*) from car where ts > '2019-05-14 00:00:00' interval(1y, 5d) -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT From 2216438c72c3ec950f1b4303b50c5a26de861524 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 19 May 2022 10:26:21 +0800 Subject: [PATCH 24/29] fix(stream): skip update tb uid for db topic --- example/src/tmq.c | 4 ++-- source/dnode/vnode/src/tq/tq.c | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/example/src/tmq.c b/example/src/tmq.c index 00eb8462f4..b79d21d051 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -106,8 +106,8 @@ int32_t create_topic() { } taos_free_result(pRes); - /*pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");*/ - pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1"); + pRes = taos_query(pConn, "create topic topic_ctb_column as abc1"); + /*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");*/ if (taos_errno(pRes) != 0) { printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c69f7d71d5..30de2dcfa0 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -111,6 +111,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList) { pIter = taosHashIterate(pTq->execs, pIter); if (pIter == NULL) break; pExec = (STqExec*)pIter; + if (pExec->subType == TOPIC_SUB_TYPE__DB) continue; for (int32_t i = 0; i < 5; i++) { int32_t code = qUpdateQualifiedTableId(pExec->task[i], tbUidList, true); ASSERT(code == 0); From 51721dfc477a8c300559300649e2ed11bafd9928 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Thu, 19 May 2022 10:37:11 +0800 Subject: [PATCH 25/29] fix(os): win run error --- cmake/cmake.install | 21 ++++--- include/os/osString.h | 18 ++++++ packaging/tools/make_install.bat | 6 ++ source/client/src/clientSml.c | 4 +- source/common/src/tname.c | 2 +- source/common/src/ttime.c | 4 +- source/common/src/tvariant.c | 8 +-- source/libs/index/src/indexComm.c | 8 +-- source/libs/parser/src/parAstCreater.c | 56 +++++++++--------- source/libs/parser/src/parInsert.c | 6 +- source/libs/parser/src/parTranslater.c | 22 +++---- source/libs/parser/src/sql.c | 2 +- source/libs/scalar/src/sclfunc.c | 8 +-- source/libs/scalar/src/sclvector.c | 22 +++---- source/os/src/osString.c | 79 ++++++++++++++++++++++++++ source/util/src/tjson.c | 4 +- tools/shell/src/shellEngine.c | 6 +- 17 files changed, 192 insertions(+), 84 deletions(-) create mode 100644 packaging/tools/make_install.bat diff --git a/cmake/cmake.install b/cmake/cmake.install index f51c6566fa..b2421fac25 100644 --- a/cmake/cmake.install +++ b/cmake/cmake.install @@ -5,22 +5,27 @@ IF (TD_LINUX) ELSEIF (TD_WINDOWS) SET(CMAKE_INSTALL_PREFIX C:/TDengine) - INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/go DESTINATION connector) - INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/nodejs DESTINATION connector) - INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/python DESTINATION connector) - INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/C\# DESTINATION connector) - INSTALL(DIRECTORY ${TD_SOURCE_DIR}/examples DESTINATION .) + # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/go DESTINATION connector) + # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/nodejs DESTINATION connector) + # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/python DESTINATION connector) + # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/C\# DESTINATION connector) + # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/examples DESTINATION .) INSTALL(FILES ${TD_SOURCE_DIR}/packaging/cfg/taos.cfg DESTINATION cfg) - INSTALL(FILES ${TD_SOURCE_DIR}/src/inc/taos.h DESTINATION include) - INSTALL(FILES ${TD_SOURCE_DIR}/src/inc/taoserror.h DESTINATION include) + INSTALL(FILES ${TD_SOURCE_DIR}/include/client/taos.h DESTINATION include) + INSTALL(FILES ${TD_SOURCE_DIR}/include/util/taoserror.h DESTINATION include) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos_static.lib DESTINATION driver) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.exp DESTINATION driver) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver) + INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taos.exe DESTINATION .) + INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taosd.exe DESTINATION .) + INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/udfd.exe DESTINATION .) IF (TD_MVN_INSTALLED) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.38-dist.jar DESTINATION connector/jdbc) ENDIF () + SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.bat") + INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") + INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} :needAdmin ${TD_SOURCE_DIR} ${PROJECT_BINARY_DIR} Windows ${TD_VER_NUMBER})") ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.sh") INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") diff --git a/include/os/osString.h b/include/os/osString.h index 5f65f97bec..1b518f9b81 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -38,6 +38,13 @@ typedef int32_t TdUcs4; #define wcsncpy WCSNCPY_FUNC_TAOS_FORBID #define wchar_t WCHAR_T_TYPE_TAOS_FORBID #define strcasestr STR_CASE_STR_FORBID + #define strtoll STR_TO_LL_FUNC_TAOS_FORBID + #define strtoull STR_TO_ULL_FUNC_TAOS_FORBID + #define strtol STR_TO_L_FUNC_TAOS_FORBID + #define strtoul STR_TO_UL_FUNC_TAOS_FORBID + #define strtod STR_TO_LD_FUNC_TAOS_FORBID + #define strtold STR_TO_D_FUNC_TAOS_FORBID + #define strtof STR_TO_F_FUNC_TAOS_FORBID #endif #ifdef WINDOWS @@ -72,6 +79,17 @@ int32_t taosWcharsToMbs(char *pStrs, TdWchar *pWchars, int32_t size); char *taosStrCaseStr(const char *str, const char *pattern); +int64_t taosStr2Int64(const char *str, char** pEnd, int32_t radix); +uint64_t taosStr2UInt64(const char *str, char** pEnd, int32_t radix); +int32_t taosStr2Int32(const char *str, char** pEnd, int32_t radix); +uint32_t taosStr2UInt32(const char *str, char** pEnd, int32_t radix); +int16_t taosStr2Int16(const char *str, char** pEnd, int32_t radix); +uint16_t taosStr2UInt16(const char *str, char** pEnd, int32_t radix); +int8_t taosStr2Int8(const char *str, char** pEnd, int32_t radix); +uint8_t taosStr2UInt8(const char *str, char** pEnd, int32_t radix); +double taosStr2Double(const char *str, char** pEnd); +float taosStr2Float(const char *str, char** pEnd); + #ifdef __cplusplus } #endif diff --git a/packaging/tools/make_install.bat b/packaging/tools/make_install.bat new file mode 100644 index 0000000000..64f30b8465 --- /dev/null +++ b/packaging/tools/make_install.bat @@ -0,0 +1,6 @@ +@echo off +goto %1 +:needAdmin +mshta vbscript:createobject("shell.application").shellexecute("%~s0",":hasAdmin","","runas",1)(window.close)&goto :eof +:hasAdmin +cp -f C:\\TDengine\\driver\\taos.dll C:\\Windows\\System32 \ No newline at end of file diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index f6f8db8590..d5377c99a6 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -580,7 +580,7 @@ static bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg){ const char *pVal = kvVal->value; int32_t len = kvVal->length; char *endptr = NULL; - double result = strtod(pVal, &endptr); + double result = taosStr2Double(pVal, &endptr); if(pVal == endptr){ smlBuildInvalidDataMsg(msg, "invalid data", pVal); return false; @@ -714,7 +714,7 @@ static bool smlIsNchar(const char *pVal, uint16_t len) { static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { char *endPtr = NULL; - int64_t tsInt64 = strtoll(value, &endPtr, 10); + int64_t tsInt64 = taosStr2Int64(value, &endPtr, 10); if(value + len != endPtr){ return -1; } diff --git a/source/common/src/tname.c b/source/common/src/tname.c index fa9b6e1e63..0764ea84b9 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -250,7 +250,7 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { return -1; } - dst->acctId = strtoll(str, NULL, 10); + dst->acctId = taosStr2Int32(str, NULL, 10); } if ((type & T_NAME_DB) == T_NAME_DB) { diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 62a7179626..69ba964187 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -590,7 +590,7 @@ int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* dura char* endPtr = NULL; /* get the basic numeric value */ - int64_t timestamp = strtoll(token, &endPtr, 10); + int64_t timestamp = taosStr2Int64(token, &endPtr, 10); if (errno != 0) { return -1; } @@ -608,7 +608,7 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati errno = 0; /* get the basic numeric value */ - *duration = strtoll(token, NULL, 10); + *duration = taosStr2Int64(token, NULL, 10); if (errno != 0) { return -1; } diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index e44450fe6e..7b0bef4918 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -39,7 +39,7 @@ int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value) { errno = 0; char *endPtr = NULL; - *value = strtoll(z, &endPtr, base); + *value = taosStr2Int64(z, &endPtr, base); if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { errno = 0; return -1; @@ -58,7 +58,7 @@ int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value) { return -1; } - *value = strtoull(z, &endPtr, base); + *value = taosStr2UInt64(z, &endPtr, base); if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { errno = 0; return -1; @@ -434,7 +434,7 @@ static FORCE_INLINE int32_t convertToDouble(char *pStr, int32_t len, double *val // return -1; // } // - // *value = strtod(pStr, NULL); + // *value = taosStr2Double(pStr, NULL); return 0; } @@ -911,7 +911,7 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) { case TSDB_DATA_TYPE_DOUBLE: { if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { errno = 0; - double v = strtod(pVariant->pz, NULL); + double v = taosStr2Double(pVariant->pz, NULL); if ((errno == ERANGE && v == -1) || (isinf(v) || isnan(v))) { taosMemoryFree(pVariant->pz); return -1; diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 802cdf7aad..4c23e4ba4b 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -118,21 +118,21 @@ TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t d } return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_FLOAT) { - float va = strtod(a, NULL); + float va = taosStr2Float(a, NULL); if (errno == ERANGE && va == -1) { return CONTINUE; } - float vb = strtod(b, NULL); + float vb = taosStr2Float(b, NULL); if (errno == ERANGE && va == -1) { return CONTINUE; } return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_DOUBLE) { - double va = strtod(a, NULL); + double va = taosStr2Double(a, NULL); if (errno == ERANGE && va == -1) { return CONTINUE; } - double vb = strtod(b, NULL); + double vb = taosStr2Double(b, NULL); if (errno == ERANGE && va == -1) { return CONTINUE; } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index a438a1e843..894659ddd1 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -123,7 +123,7 @@ static bool checkAndSplitEndpoint(SAstCreateContext* pCxt, const SToken* pEp, ch pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ENDPOINT); } else { strncpy(pFqdn, ep, pColon - ep); - *pPort = strtol(pColon + 1, NULL, 10); + *pPort = taosStr2Int32(pColon + 1, NULL, 10); if (*pPort >= UINT16_MAX || *pPort <= 0) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_PORT); } @@ -147,7 +147,7 @@ static bool checkPort(SAstCreateContext* pCxt, const SToken* pPortToken, int32_t if (NULL == pPortToken) { pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; } else { - *pPort = strtol(pPortToken->z, NULL, 10); + *pPort = taosStr2Int32(pPortToken->z, NULL, 10); if (*pPort >= UINT16_MAX || *pPort <= 0) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_PORT); } @@ -476,9 +476,9 @@ SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset) { SLimitNode* limitNode = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT); CHECK_OUT_OF_MEM(limitNode); - limitNode->limit = strtol(pLimit->z, NULL, 10); + limitNode->limit = taosStr2Int64(pLimit->z, NULL, 10); if (NULL != pOffset) { - limitNode->offset = strtol(pOffset->z, NULL, 10); + limitNode->offset = taosStr2Int64(pOffset->z, NULL, 10); } return (SNode*)limitNode; } @@ -694,59 +694,59 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) { SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, void* pVal) { switch (type) { case DB_OPTION_BUFFER: - ((SDatabaseOptions*)pOptions)->buffer = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->buffer = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_CACHELAST: - ((SDatabaseOptions*)pOptions)->cachelast = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->cachelast = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_COMP: - ((SDatabaseOptions*)pOptions)->compressionLevel = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->compressionLevel = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_DAYS: { SToken* pToken = pVal; if (TK_NK_INTEGER == pToken->type) { - ((SDatabaseOptions*)pOptions)->daysPerFile = strtol(pToken->z, NULL, 10) * 1440; + ((SDatabaseOptions*)pOptions)->daysPerFile = taosStr2Int32(pToken->z, NULL, 10) * 1440; } else { ((SDatabaseOptions*)pOptions)->pDaysPerFile = (SValueNode*)createDurationValueNode(pCxt, pToken); } break; } case DB_OPTION_FSYNC: - ((SDatabaseOptions*)pOptions)->fsyncPeriod = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->fsyncPeriod = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_MAXROWS: - ((SDatabaseOptions*)pOptions)->maxRowsPerBlock = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->maxRowsPerBlock = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_MINROWS: - ((SDatabaseOptions*)pOptions)->minRowsPerBlock = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->minRowsPerBlock = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_KEEP: ((SDatabaseOptions*)pOptions)->pKeep = pVal; break; case DB_OPTION_PAGES: - ((SDatabaseOptions*)pOptions)->pages = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->pages = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_PAGESIZE: - ((SDatabaseOptions*)pOptions)->pagesize = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->pagesize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_PRECISION: copyStringFormStringToken((SToken*)pVal, ((SDatabaseOptions*)pOptions)->precisionStr, sizeof(((SDatabaseOptions*)pOptions)->precisionStr)); break; case DB_OPTION_REPLICA: - ((SDatabaseOptions*)pOptions)->replica = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->replica = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_STRICT: - ((SDatabaseOptions*)pOptions)->strict = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->strict = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_WAL: - ((SDatabaseOptions*)pOptions)->walLevel = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->walLevel = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_VGROUPS: - ((SDatabaseOptions*)pOptions)->numOfVgroups = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->numOfVgroups = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_SINGLE_STABLE: - ((SDatabaseOptions*)pOptions)->singleStable = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->singleStable = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_RETENTIONS: ((SDatabaseOptions*)pOptions)->pRetentions = pVal; @@ -827,16 +827,16 @@ SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType sizeof(((STableOptions*)pOptions)->comment)); break; case TABLE_OPTION_DELAY: - ((STableOptions*)pOptions)->delay = strtol(((SToken*)pVal)->z, NULL, 10); + ((STableOptions*)pOptions)->delay = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case TABLE_OPTION_FILE_FACTOR: - ((STableOptions*)pOptions)->filesFactor = strtod(((SToken*)pVal)->z, NULL); + ((STableOptions*)pOptions)->filesFactor = taosStr2Float(((SToken*)pVal)->z, NULL); break; case TABLE_OPTION_ROLLUP: ((STableOptions*)pOptions)->pRollupFuncs = pVal; break; case TABLE_OPTION_TTL: - ((STableOptions*)pOptions)->ttl = strtol(((SToken*)pVal)->z, NULL, 10); + ((STableOptions*)pOptions)->ttl = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case TABLE_OPTION_SMA: ((STableOptions*)pOptions)->pSma = pVal; @@ -868,7 +868,7 @@ SDataType createDataType(uint8_t type) { } SDataType createVarLenDataType(uint8_t type, const SToken* pLen) { - SDataType dt = {.type = type, .precision = 0, .scale = 0, .bytes = strtol(pLen->z, NULL, 10)}; + SDataType dt = {.type = type, .precision = 0, .scale = 0, .bytes = taosStr2Int16(pLen->z, NULL, 10)}; return dt; } @@ -1119,7 +1119,7 @@ SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode) { SDropDnodeStmt* pStmt = (SDropDnodeStmt*)nodesMakeNode(QUERY_NODE_DROP_DNODE_STMT); CHECK_OUT_OF_MEM(pStmt); if (TK_NK_INTEGER == pDnode->type) { - pStmt->dnodeId = strtol(pDnode->z, NULL, 10); + pStmt->dnodeId = taosStr2Int32(pDnode->z, NULL, 10); } else { if (!checkAndSplitEndpoint(pCxt, pDnode, pStmt->fqdn, &pStmt->port)) { nodesDestroyNode(pStmt); @@ -1133,7 +1133,7 @@ SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const const SToken* pValue) { SAlterDnodeStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_DNODE_STMT); CHECK_OUT_OF_MEM(pStmt); - pStmt->dnodeId = strtol(pDnode->z, NULL, 10); + pStmt->dnodeId = taosStr2Int32(pDnode->z, NULL, 10); trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config)); if (NULL != pValue) { trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value)); @@ -1183,7 +1183,7 @@ SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken SNode* createCreateComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId) { SCreateComponentNodeStmt* pStmt = nodesMakeNode(type); CHECK_OUT_OF_MEM(pStmt); - pStmt->dnodeId = strtol(pDnodeId->z, NULL, 10); + pStmt->dnodeId = taosStr2Int32(pDnodeId->z, NULL, 10); ; return (SNode*)pStmt; } @@ -1191,7 +1191,7 @@ SNode* createCreateComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, co SNode* createDropComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId) { SDropComponentNodeStmt* pStmt = nodesMakeNode(type); CHECK_OUT_OF_MEM(pStmt); - pStmt->dnodeId = strtol(pDnodeId->z, NULL, 10); + pStmt->dnodeId = taosStr2Int32(pDnodeId->z, NULL, 10); ; return (SNode*)pStmt; } @@ -1251,7 +1251,7 @@ SNode* setExplainVerbose(SAstCreateContext* pCxt, SNode* pOptions, const SToken* } SNode* setExplainRatio(SAstCreateContext* pCxt, SNode* pOptions, const SToken* pVal) { - ((SExplainOptions*)pOptions)->ratio = strtod(pVal->z, NULL); + ((SExplainOptions*)pOptions)->ratio = taosStr2Double(pVal->z, NULL); return pOptions; } @@ -1347,7 +1347,7 @@ SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId) { SKillStmt* pStmt = nodesMakeNode(type); CHECK_OUT_OF_MEM(pStmt); - pStmt->targetId = strtol(pId->z, NULL, 10); + pStmt->targetId = taosStr2Int32(pId->z, NULL, 10); return (SNode*)pStmt; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 29951ccdee..34ae13580b 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -442,7 +442,7 @@ static bool isNullStr(SToken* pToken) { static FORCE_INLINE int32_t toDouble(SToken* pToken, double* value, char** endPtr) { errno = 0; - *value = strtold(pToken->z, endPtr); + *value = taosStr2Double(pToken->z, endPtr); // not a valid integer number, return error if ((*endPtr - pToken->z) != pToken->n) { @@ -482,9 +482,9 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); } } else if (pToken->type == TK_NK_INTEGER) { - return func(pMsgBuf, ((strtoll(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); + return func(pMsgBuf, ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); } else if (pToken->type == TK_NK_FLOAT) { - return func(pMsgBuf, ((strtod(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); + return func(pMsgBuf, ((taosStr2Double(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); } else { return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 18e0843e92..0d214fd85e 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -498,7 +498,7 @@ static int32_t parseTimeFromValueNode(SValueNode* pVal) { return TSDB_CODE_SUCCESS; } char* pEnd = NULL; - pVal->datum.i = strtoll(pVal->literal, &pEnd, 10); + pVal->datum.i = taosStr2Int64(pVal->literal, &pEnd, 10); return (NULL != pEnd && '\0' == *pEnd) ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED; } else { return TSDB_CODE_FAILED; @@ -527,61 +527,61 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD break; case TSDB_DATA_TYPE_TINYINT: { char* endPtr = NULL; - pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); + pVal->datum.i = taosStr2Int64(pVal->literal, &endPtr, 10); *(int8_t*)&pVal->typeData = pVal->datum.i; break; } case TSDB_DATA_TYPE_SMALLINT: { char* endPtr = NULL; - pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); + pVal->datum.i = taosStr2Int64(pVal->literal, &endPtr, 10); *(int16_t*)&pVal->typeData = pVal->datum.i; break; } case TSDB_DATA_TYPE_INT: { char* endPtr = NULL; - pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); + pVal->datum.i = taosStr2Int64(pVal->literal, &endPtr, 10); *(int32_t*)&pVal->typeData = pVal->datum.i; break; } case TSDB_DATA_TYPE_BIGINT: { char* endPtr = NULL; - pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); + pVal->datum.i = taosStr2Int64(pVal->literal, &endPtr, 10); *(int64_t*)&pVal->typeData = pVal->datum.i; break; } case TSDB_DATA_TYPE_UTINYINT: { char* endPtr = NULL; - pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); + pVal->datum.u = taosStr2UInt64(pVal->literal, &endPtr, 10); *(uint8_t*)&pVal->typeData = pVal->datum.u; break; } case TSDB_DATA_TYPE_USMALLINT: { char* endPtr = NULL; - pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); + pVal->datum.u = taosStr2UInt64(pVal->literal, &endPtr, 10); *(uint16_t*)&pVal->typeData = pVal->datum.u; break; } case TSDB_DATA_TYPE_UINT: { char* endPtr = NULL; - pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); + pVal->datum.u = taosStr2UInt64(pVal->literal, &endPtr, 10); *(uint32_t*)&pVal->typeData = pVal->datum.u; break; } case TSDB_DATA_TYPE_UBIGINT: { char* endPtr = NULL; - pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); + pVal->datum.u = taosStr2UInt64(pVal->literal, &endPtr, 10); *(uint64_t*)&pVal->typeData = pVal->datum.u; break; } case TSDB_DATA_TYPE_FLOAT: { char* endPtr = NULL; - pVal->datum.d = strtold(pVal->literal, &endPtr); + pVal->datum.d = taosStr2Double(pVal->literal, &endPtr); *(float*)&pVal->typeData = pVal->datum.d; break; } case TSDB_DATA_TYPE_DOUBLE: { char* endPtr = NULL; - pVal->datum.d = strtold(pVal->literal, &endPtr); + pVal->datum.d = taosStr2Double(pVal->literal, &endPtr); *(double*)&pVal->typeData = pVal->datum.d; break; } diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 795b5ca641..2d844e1842 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -3863,7 +3863,7 @@ static YYACTIONTYPE yy_reduce( { yymsp[1].minor.yy140 = 0; } break; case 245: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ -{ yymsp[-1].minor.yy140 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } +{ yymsp[-1].minor.yy140 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } break; case 246: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression */ { pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-5].minor.yy617, &yymsp[-4].minor.yy105, yymsp[-2].minor.yy172, yymsp[-3].minor.yy172, yymsp[0].minor.yy172); } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index d4a88622e2..45742189d5 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -724,7 +724,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp case TSDB_DATA_TYPE_BIGINT: { if (inputType == TSDB_DATA_TYPE_BINARY) { memcpy(output, varDataVal(input), varDataLen(input)); - *(int64_t *)output = strtoll(output, NULL, 10); + *(int64_t *)output = taosStr2Int64(output, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { char *newBuf = taosMemoryCalloc(1, outputLen * TSDB_NCHAR_SIZE + 1); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); @@ -733,7 +733,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp return TSDB_CODE_FAILED; } newBuf[len] = 0; - *(int64_t *)output = strtoll(newBuf, NULL, 10); + *(int64_t *)output = taosStr2Int64(newBuf, NULL, 10); taosMemoryFree(newBuf); } else { GET_TYPED_DATA(*(int64_t *)output, int64_t, inputType, input); @@ -743,7 +743,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp case TSDB_DATA_TYPE_UBIGINT: { if (inputType == TSDB_DATA_TYPE_BINARY) { memcpy(output, varDataVal(input), varDataLen(input)); - *(uint64_t *)output = strtoull(output, NULL, 10); + *(uint64_t *)output = taosStr2UInt64(output, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { char *newBuf = taosMemoryCalloc(1, outputLen * TSDB_NCHAR_SIZE + 1); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); @@ -752,7 +752,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp return TSDB_CODE_FAILED; } newBuf[len] = 0; - *(uint64_t *)output = strtoull(newBuf, NULL, 10); + *(uint64_t *)output = taosStr2UInt64(newBuf, NULL, 10); taosMemoryFree(newBuf); } else { GET_TYPED_DATA(*(uint64_t *)output, uint64_t, inputType, input); diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index c9fcaeb32e..19453bf760 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -92,7 +92,7 @@ void convertStringToDouble(const void *inData, void *outData, int8_t inType, int tmp[len] = 0; ASSERT(outType == TSDB_DATA_TYPE_DOUBLE); - double value = strtod(tmp, NULL); + double value = taosStr2Double(tmp, NULL); *((double *)outData) = value; taosMemoryFreeClear(tmp); @@ -267,22 +267,22 @@ static FORCE_INLINE void varToTimestamp(char *buf, SScalarParam* pOut, int32_t r static FORCE_INLINE void varToSigned(char *buf, SScalarParam* pOut, int32_t rowIndex) { switch (pOut->columnData->info.type) { case TSDB_DATA_TYPE_TINYINT: { - int8_t value = (int8_t)strtoll(buf, NULL, 10); + int8_t value = (int8_t)taosStr2Int8(buf, NULL, 10); colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*)&value); break; } case TSDB_DATA_TYPE_SMALLINT: { - int16_t value = (int16_t)strtoll(buf, NULL, 10); + int16_t value = (int16_t)taosStr2Int16(buf, NULL, 10); colDataAppendInt16(pOut->columnData, rowIndex, (int16_t*)&value); break; } case TSDB_DATA_TYPE_INT: { - int32_t value = (int32_t)strtoll(buf, NULL, 10); + int32_t value = (int32_t)taosStr2Int32(buf, NULL, 10); colDataAppendInt32(pOut->columnData, rowIndex, (int32_t*)&value); break; } case TSDB_DATA_TYPE_BIGINT: { - int64_t value = (int64_t)strtoll(buf, NULL, 10); + int64_t value = (int64_t)taosStr2Int64(buf, NULL, 10); colDataAppendInt64(pOut->columnData, rowIndex, (int64_t*)&value); break; } @@ -292,22 +292,22 @@ static FORCE_INLINE void varToSigned(char *buf, SScalarParam* pOut, int32_t rowI static FORCE_INLINE void varToUnsigned(char *buf, SScalarParam* pOut, int32_t rowIndex) { switch (pOut->columnData->info.type) { case TSDB_DATA_TYPE_UTINYINT: { - uint8_t value = (uint8_t)strtoull(buf, NULL, 10); + uint8_t value = (uint8_t)taosStr2UInt8(buf, NULL, 10); colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*)&value); break; } case TSDB_DATA_TYPE_USMALLINT: { - uint16_t value = (uint16_t)strtoull(buf, NULL, 10); + uint16_t value = (uint16_t)taosStr2UInt16(buf, NULL, 10); colDataAppendInt16(pOut->columnData, rowIndex, (int16_t*)&value); break; } case TSDB_DATA_TYPE_UINT: { - uint32_t value = (uint32_t)strtoull(buf, NULL, 10); + uint32_t value = (uint32_t)taosStr2UInt32(buf, NULL, 10); colDataAppendInt32(pOut->columnData, rowIndex, (int32_t*)&value); break; } case TSDB_DATA_TYPE_UBIGINT: { - uint64_t value = (uint64_t)strtoull(buf, NULL, 10); + uint64_t value = (uint64_t)taosStr2UInt64(buf, NULL, 10); colDataAppendInt64(pOut->columnData, rowIndex, (int64_t*)&value); break; } @@ -315,12 +315,12 @@ static FORCE_INLINE void varToUnsigned(char *buf, SScalarParam* pOut, int32_t ro } static FORCE_INLINE void varToFloat(char *buf, SScalarParam* pOut, int32_t rowIndex) { - double value = strtod(buf, NULL); + double value = taosStr2Double(buf, NULL); colDataAppendDouble(pOut->columnData, rowIndex, &value); } static FORCE_INLINE void varToBool(char *buf, SScalarParam* pOut, int32_t rowIndex) { - int64_t value = strtoll(buf, NULL, 10); + int64_t value = taosStr2Int64(buf, NULL, 10); bool v = (value != 0)? true:false; colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*) &v); } diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 7dbd301913..d5762ef87d 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -254,4 +254,83 @@ char *taosStrCaseStr(const char *str, const char *pattern) { } } return NULL; +} + +int64_t taosStr2Int64(const char *str, char** pEnd, int32_t radix) { + int64_t tmp = strtoll(str, pEnd, radix); + assert(errno != ERANGE); + assert(errno != EINVAL); + return tmp; +} + +uint64_t taosStr2UInt64(const char *str, char** pEnd, int32_t radix) { + uint64_t tmp = strtoull(str, pEnd, radix); + assert(errno != ERANGE); + assert(errno != EINVAL); + return tmp; +} + +int32_t taosStr2Int32(const char *str, char** pEnd, int32_t radix) { + int32_t tmp = strtol(str, pEnd, radix); + assert(errno != ERANGE); + assert(errno != EINVAL); + return tmp; +} + +uint32_t taosStr2UInt32(const char *str, char** pEnd, int32_t radix) { + uint32_t tmp = strtol(str, pEnd, radix); + assert(errno != ERANGE); + assert(errno != EINVAL); + return tmp; +} + +int16_t taosStr2Int16(const char *str, char** pEnd, int32_t radix) { + int32_t tmp = strtol(str, pEnd, radix); + assert(errno != ERANGE); + assert(errno != EINVAL); + assert(tmp >= SHRT_MIN); + assert(tmp <= SHRT_MAX); + return (int16_t)tmp; +} + +uint16_t taosStr2UInt16(const char *str, char** pEnd, int32_t radix) { + uint32_t tmp = strtoul(str, pEnd, radix); + assert(errno != ERANGE); + assert(errno != EINVAL); + assert(tmp <= USHRT_MAX); + return (uint16_t)tmp; +} + +int8_t taosStr2Int8(const char *str, char** pEnd, int32_t radix) { + int32_t tmp = strtol(str, pEnd, radix); + assert(errno != ERANGE); + assert(errno != EINVAL); + assert(tmp >= SCHAR_MIN); + assert(tmp <= SCHAR_MAX); + return tmp; +} + +uint8_t taosStr2UInt8(const char *str, char** pEnd, int32_t radix) { + uint32_t tmp = strtoul(str, pEnd, radix); + assert(errno != ERANGE); + assert(errno != EINVAL); + assert(tmp <= UCHAR_MAX); + return tmp; +} + +double taosStr2Double(const char *str, char** pEnd) { + double tmp = strtod(str, pEnd); + assert(errno != ERANGE); + assert(errno != EINVAL); + assert(tmp != HUGE_VAL); + return tmp; +} + +float taosStr2Float(const char *str, char** pEnd) { + float tmp = strtof(str, pEnd); + assert(errno != ERANGE); + assert(errno != EINVAL); + assert(tmp != HUGE_VALF); + assert(tmp != NAN); + return tmp; } \ No newline at end of file diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index c5cd968fd9..b15c188f04 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -187,7 +187,7 @@ int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal sscanf(p,"%lld",pVal); #else // sscanf(p,"%ld",pVal); - *pVal = strtol(p, NULL, 10); + *pVal = taosStr2Int64(p, NULL, 10); #endif return TSDB_CODE_SUCCESS; } @@ -222,7 +222,7 @@ int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pV sscanf(p,"%llu",pVal); #else // sscanf(p,"%ld",pVal); - *pVal = strtoul(p, NULL, 10); + *pVal = taosStr2UInt64(p, NULL, 10); #endif return TSDB_CODE_SUCCESS; } diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 9f9c8821b0..21474c316f 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -195,16 +195,16 @@ void shellRunSingleCommandImp(char *command) { et = taosGetTimestampUs(); if (error_no == 0) { - printf("Query OK, %d row(s) in set (%.6fs)\n", numOfRows, (et - st) / 1E6); + printf("Query OK, %d rows affected (%.6fs)\n", numOfRows, (et - st) / 1E6); } else { - printf("Query interrupted (%s), %d row(s) in set (%.6fs)\n", taos_errstr(pSql), numOfRows, (et - st) / 1E6); + printf("Query interrupted (%s), %d rows affected (%.6fs)\n", taos_errstr(pSql), numOfRows, (et - st) / 1E6); } taos_free_result(pSql); } else { int32_t num_rows_affacted = taos_affected_rows(pSql); taos_free_result(pSql); et = taosGetTimestampUs(); - printf("Query OK, %d of %d row(s) in database (%.6fs)\n", num_rows_affacted, num_rows_affacted, (et - st) / 1E6); + printf("Query OK, %d of %d rows affected (%.6fs)\n", num_rows_affacted, num_rows_affacted, (et - st) / 1E6); } printf("\n"); From 86580cec090f96839e57f06935f90ef5fc6dd330 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 19 May 2022 10:53:24 +0800 Subject: [PATCH 26/29] enh(tmq): remove old tmq_commit api --- example/src/tmq.c | 2 +- include/client/taos.h | 6 ++-- tests/test/c/tmqDemo.c | 2 +- tests/test/c/tmqSim.c | 77 ++++++++++++++++++++---------------------- 4 files changed, 41 insertions(+), 46 deletions(-) diff --git a/example/src/tmq.c b/example/src/tmq.c index b79d21d051..1abce3f188 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -239,7 +239,7 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { msg_process(tmqmessage); taos_free_result(tmqmessage); - tmq_commit(tmq, NULL, 1); + tmq_commit_async(tmq, NULL, tmq_commit_cb_print, NULL); /*if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);*/ } } diff --git a/include/client/taos.h b/include/client/taos.h index 0194357841..0b8c67aa79 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -232,11 +232,11 @@ DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq); DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics); DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t wait_time); DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq); -DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, int32_t async); -DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, tmq_commit_cb *cb, void *param); DLL_EXPORT tmq_resp_err_t tmq_commit_sync(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets); +DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, tmq_commit_cb *cb, void *param); + #if 0 -DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async); +DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, int32_t async); DLL_EXPORT tmq_resp_err_t tmq_seek(tmq_t *tmq, const tmq_topic_vgroup_t *offset); #endif diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index dc1a77c231..96d7741897 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -368,7 +368,7 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { /*msg_process(tmqmessage);*/ taos_free_result(tmqmessage); - if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0); + if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit_sync(tmq, NULL); } } diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index cf113369bc..8322c1a60c 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -37,10 +37,10 @@ typedef struct { TdThread thread; int32_t consumerId; - int32_t ifManualCommit; - //int32_t autoCommitIntervalMs; // 1000 ms - //char autoCommit[8]; // true, false - //char autoOffsetRest[16]; // none, earliest, latest + int32_t ifManualCommit; + // int32_t autoCommitIntervalMs; // 1000 ms + // char autoCommit[8]; // true, false + // char autoOffsetRest[16]; // none, earliest, latest int32_t ifCheckData; int64_t expectMsgCnt; @@ -99,21 +99,15 @@ static void printHelp() { } void initLogFile() { - time_t now; - struct tm curTime; - char filename[256]; + time_t now; + struct tm curTime; + char filename[256]; - now = taosTime(NULL); + now = taosTime(NULL); taosLocalTime(&now, &curTime); - sprintf(filename,"%s/../log/tmqlog_%04d-%02d-%02d %02d-%02d-%02d.txt", - configDir, - curTime.tm_year+1900, - curTime.tm_mon+1, - curTime.tm_mday, - curTime.tm_hour, - curTime.tm_min, - curTime.tm_sec); - //sprintf(filename, "%s/../log/tmqlog.txt", configDir); + sprintf(filename, "%s/../log/tmqlog_%04d-%02d-%02d %02d-%02d-%02d.txt", configDir, curTime.tm_year + 1900, + curTime.tm_mon + 1, curTime.tm_mday, curTime.tm_hour, curTime.tm_min, curTime.tm_sec); + // sprintf(filename, "%s/../log/tmqlog.txt", configDir); TdFilePtr pFile = taosOpenFile(filename, TD_FILE_TEXT | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); if (NULL == pFile) { fprintf(stderr, "Failed to open %s for save result\n", filename); @@ -137,9 +131,9 @@ void saveConfigToLogFile() { for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { taosFprintfFile(g_fp, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId); - //taosFprintfFile(g_fp, " auto commit: %s\n", g_stConfInfo.stThreads[i].autoCommit); - //taosFprintfFile(g_fp, " auto commit interval ms: %d\n", g_stConfInfo.stThreads[i].autoCommitIntervalMs); - //taosFprintfFile(g_fp, " auto offset rest: %s\n", g_stConfInfo.stThreads[i].autoOffsetRest); + // taosFprintfFile(g_fp, " auto commit: %s\n", g_stConfInfo.stThreads[i].autoCommit); + // taosFprintfFile(g_fp, " auto commit interval ms: %d\n", g_stConfInfo.stThreads[i].autoCommitIntervalMs); + // taosFprintfFile(g_fp, " auto offset rest: %s\n", g_stConfInfo.stThreads[i].autoOffsetRest); taosFprintfFile(g_fp, " Topics: "); for (int j = 0; j < g_stConfInfo.stThreads[i].numOfTopic; j++) { taosFprintfFile(g_fp, "%s, ", g_stConfInfo.stThreads[i].topics[j]); @@ -234,17 +228,17 @@ static int32_t msg_process(TAOS_RES* msg, int64_t msgIndex, int32_t threadLable) while (1) { TAOS_ROW row = taos_fetch_row(msg); - if (row == NULL) break; + if (row == NULL) break; - TAOS_FIELD* fields = taos_fetch_fields(msg); + TAOS_FIELD* fields = taos_fetch_fields(msg); int32_t numOfFields = taos_field_count(msg); - + taos_print_row(buf, row, fields, numOfFields); - - if (0 != g_stConfInfo.showRowFlag) { + + if (0 != g_stConfInfo.showRowFlag) { taosFprintfFile(g_fp, "rows[%d]: %s\n", totalRows, buf); } - + totalRows++; } @@ -276,7 +270,7 @@ void build_consumer(SThreadInfo* pInfo) { tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); - //tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); + // tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); @@ -299,7 +293,7 @@ void build_consumer(SThreadInfo* pInfo) { pInfo->tmq = tmq_consumer_new(conf, NULL, 0); tmq_conf_destroy(conf); - + return; } @@ -322,10 +316,10 @@ int32_t saveConsumeResult(SThreadInfo* pInfo) { sprintf(sqlStr, "insert into %s.consumeresult values (now, %d, %" PRId64 ", %" PRId64 ", %d)", g_stConfInfo.cdbName, pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt, pInfo->checkresult); - time_t tTime = taosGetTimestampSec(); + time_t tTime = taosGetTimestampSec(); struct tm tm = *taosLocalTime(&tTime, NULL); taosFprintfFile(g_fp, "# save result: %d-%02d-%02d %02d:%02d:%02d, sql: %s\n", tm.tm_year + 1900, tm.tm_mon + 1, - tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, sqlStr); + tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, sqlStr); TAOS_RES* pRes = taos_query(pConn, sqlStr); if (taos_errno(pRes) != 0) { @@ -357,11 +351,11 @@ void loop_consume(SThreadInfo* pInfo) { totalMsgs++; if (totalRows >= pInfo->expectMsgCnt) { - taosFprintfFile(g_fp, "==== totalRows >= pInfo->expectMsgCnt, so break\n"); + taosFprintfFile(g_fp, "==== totalRows >= pInfo->expectMsgCnt, so break\n"); break; } - } else { - taosFprintfFile(g_fp, "==== delay over time, so break\n"); + } else { + taosFprintfFile(g_fp, "==== delay over time, so break\n"); break; } } @@ -389,7 +383,7 @@ void* consumeThreadFunc(void* param) { pError("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); exit(-1); } - + tmq_list_destroy(pInfo->topicList); pInfo->topicList = NULL; @@ -397,17 +391,18 @@ void* consumeThreadFunc(void* param) { if (pInfo->ifManualCommit) { taosFprintfFile(g_fp, "tmq_commit() manual commit when consume end.\n"); - pPrint("tmq_commit() manual commit when consume end.\n"); - tmq_commit(pInfo->tmq, NULL, 0); + pPrint("tmq_commit() manual commit when consume end.\n"); + /*tmq_commit(pInfo->tmq, NULL, 0);*/ + tmq_commit_sync(pInfo->tmq, NULL); } - + err = tmq_unsubscribe(pInfo->tmq); if (err) { pError("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); pInfo->consumeMsgCnt = -1; return NULL; } - + err = tmq_consumer_close(pInfo->tmq); if (err) { pError("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); @@ -485,9 +480,9 @@ int32_t getConsumeInfo() { int32_t* lengths = taos_fetch_lengths(pRes); // set default value - //g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = 5000; - //memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, "true", strlen("true")); - //memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, "earlieast", strlen("earlieast")); + // g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = 5000; + // memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, "true", strlen("true")); + // memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, "earlieast", strlen("earlieast")); for (int i = 0; i < num_fields; ++i) { if (row[i] == NULL || 0 == i) { From 83f4d1286e18fda5a10f7d2b4d483f269e92af39 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 19 May 2022 11:20:27 +0800 Subject: [PATCH 27/29] fix: executor process express tree scalar execution error --- source/libs/executor/src/executorimpl.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index db34f5dc67..3505af46e4 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -869,8 +869,12 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc SColumnInfoData idata = {.info = pResColData->info, .hasNull = true}; SScalarParam dest = {.columnData = &idata}; - scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest); - + int32_t code = scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pBlockList); + return code; + } + int32_t startOffset = createNewColModel ? 0 : pResult->info.rows; colInfoDataEnsureCapacity(pResColData, startOffset, pResult->info.capacity); colDataMergeCol(pResColData, startOffset, &pResult->info.capacity, &idata, dest.numOfRows); From 82b2ae419481238b617cb659621b5134d9c55bd9 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 19 May 2022 11:20:47 +0800 Subject: [PATCH 28/29] enh(tmq): set and show client id --- include/common/tmsg.h | 3 +++ source/common/src/systable.c | 2 +- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/src/mndConsumer.c | 9 ++++---- source/dnode/mnode/impl/src/mndTopic.c | 26 +++++++++++------------ tests/script/tsim/tstream/basic1.sim | 20 ++++++++--------- 6 files changed, 32 insertions(+), 30 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 7a60542313..f7562a4b9b 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1480,6 +1480,7 @@ typedef struct { typedef struct { int64_t consumerId; char cgroup[TSDB_CGROUP_LEN]; + char clientId[256]; SArray* topicNames; // SArray } SCMSubscribeReq; @@ -1487,6 +1488,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pReq->consumerId); tlen += taosEncodeString(buf, pReq->cgroup); + tlen += taosEncodeString(buf, pReq->clientId); int32_t topicNum = taosArrayGetSize(pReq->topicNames); tlen += taosEncodeFixedI32(buf, topicNum); @@ -1500,6 +1502,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq* pReq) { buf = taosDecodeFixedI64(buf, &pReq->consumerId); buf = taosDecodeStringTo(buf, pReq->cgroup); + buf = taosDecodeStringTo(buf, pReq->clientId); int32_t topicNum; buf = taosDecodeFixedI32(buf, &topicNum); diff --git a/source/common/src/systable.c b/source/common/src/systable.c index e4e5abe148..9fe7645e2b 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -263,7 +263,7 @@ static const SSysDbTableSchema topicSchema[] = { static const SSysDbTableSchema consumerSchema[] = { {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, {.name = "consumer_group", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "app_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, {.name = "topics", .bytes = TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index ec9139836a..a04417736a 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -463,7 +463,7 @@ typedef struct { typedef struct { int64_t consumerId; char cgroup[TSDB_CGROUP_LEN]; - char appId[TSDB_CGROUP_LEN]; + char clientId[256]; int8_t updateType; // used only for update int32_t epoch; int32_t status; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 274876567e..4fe7d8a399 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -427,6 +427,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { pConsumerOld = mndAcquireConsumer(pMnode, consumerId); if (pConsumerOld == NULL) { pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup); + tstrncpy(pConsumerNew->clientId, subscribe.clientId, 256); pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY; pConsumerNew->rebNewTopics = newSub; subscribe.topicNames = NULL; @@ -848,11 +849,11 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * colDataAppend(pColInfo, numOfRows, (const char *)cgroup, false); // app id - char appId[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0}; - tstrncpy(varDataVal(appId), pConsumer->appId, TSDB_CGROUP_LEN); - varDataSetLen(appId, strlen(varDataVal(appId))); + char clientId[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0}; + tstrncpy(varDataVal(clientId), pConsumer->clientId, TSDB_CGROUP_LEN); + varDataSetLen(clientId, strlen(varDataVal(clientId))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)appId, false); + colDataAppend(pColInfo, numOfRows, (const char *)clientId, false); // status char status[20 + VARSTR_HEADER_SIZE] = {0}; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 3f559cb6c0..c6eebb5c5d 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -492,8 +492,8 @@ static int32_t mndDropTopic(SMnode *pMnode, STrans *pTrans, SRpcMsg *pReq, SMqTo } static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; + SMnode *pMnode = pReq->info.node; + /*SSdb *pSdb = pMnode->pSdb;*/ SMDropTopicReq dropReq = {0}; if (tDeserializeSMDropTopicReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { @@ -502,16 +502,16 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { } SMqTopicObj *pTopic = mndAcquireTopic(pMnode, dropReq.name); - // if (pTopic == NULL) { - // if (dropReq.igNotExists) { - // mDebug("topic:%s, not exist, ignore not exist is set", dropReq.name); - // return 0; - // } else { - // terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST; - // mError("topic:%s, failed to drop since %s", dropReq.name, terrstr()); - // return -1; - // } - // } + if (pTopic == NULL) { + if (dropReq.igNotExists) { + mDebug("topic:%s, not exist, ignore not exist is set", dropReq.name); + return 0; + } else { + terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST; + mError("topic:%s, failed to drop since %s", dropReq.name, terrstr()); + return -1; + } + } if (pTopic->refConsumerCnt != 0) { mndReleaseTopic(pMnode, pTopic); @@ -528,12 +528,10 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { mDebug("trans:%d, used to drop topic:%s", pTrans->id, pTopic->name); -#if 1 if (mndDropOffsetByTopic(pMnode, pTrans, dropReq.name) < 0) { ASSERT(0); return -1; } -#endif if (mndDropSubByTopic(pMnode, pTrans, dropReq.name) < 0) { ASSERT(0); diff --git a/tests/script/tsim/tstream/basic1.sim b/tests/script/tsim/tstream/basic1.sim index 9965772c75..0997e313c8 100644 --- a/tests/script/tsim/tstream/basic1.sim +++ b/tests/script/tsim/tstream/basic1.sim @@ -409,53 +409,53 @@ sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; # row 1 if $data11 != 4 then print ======$data11 - # return -1 + return -1 endi if $data12 != 4 then print ======$data12 - # return -1 + return -1 endi if $data13 != 10 then print ======$data13 - # return -1 + return -1 endi if $data14 != 3 then print ======$data14 - # return -1 + return -1 endi if $data15 != 1 then print ======$data15 - # return -1 + return -1 endi # row 2 if $data21 != 4 then print ======$data21 - # return -1 + return -1 endi if $data22 != 4 then print ======$data22 - # return -1 + return -1 endi if $data23 != 15 then print ======$data23 - # return -1 + return -1 endi if $data24 != 4 then print ======$data24 - # return -1 + return -1 endi if $data25 != 3 then print ======$data25 - # return -1 + return -1 endi From f1a788f16174350c6d4e85a4580b9e20d1c382f8 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Thu, 19 May 2022 11:34:52 +0800 Subject: [PATCH 29/29] fix(os): win run error --- source/os/CMakeLists.txt | 3 +++ source/os/src/osString.c | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/source/os/CMakeLists.txt b/source/os/CMakeLists.txt index ad6cfc8b95..90b8e9dd8a 100644 --- a/source/os/CMakeLists.txt +++ b/source/os/CMakeLists.txt @@ -27,6 +27,9 @@ if(BUILD_ADDR2LINE) os PUBLIC addr2line dl z ) endif () +if(CHECK_STR2INT_ERROR) + add_definitions(-DTD_CHECK_STR_TO_INT_ERROR) +endif() target_link_libraries( os PUBLIC pthread ) diff --git a/source/os/src/osString.c b/source/os/src/osString.c index d5762ef87d..da1fbd364f 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -258,79 +258,99 @@ char *taosStrCaseStr(const char *str, const char *pattern) { int64_t taosStr2Int64(const char *str, char** pEnd, int32_t radix) { int64_t tmp = strtoll(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); +#endif return tmp; } uint64_t taosStr2UInt64(const char *str, char** pEnd, int32_t radix) { uint64_t tmp = strtoull(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); +#endif return tmp; } int32_t taosStr2Int32(const char *str, char** pEnd, int32_t radix) { int32_t tmp = strtol(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); +#endif return tmp; } uint32_t taosStr2UInt32(const char *str, char** pEnd, int32_t radix) { uint32_t tmp = strtol(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); +#endif return tmp; } int16_t taosStr2Int16(const char *str, char** pEnd, int32_t radix) { int32_t tmp = strtol(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); assert(tmp >= SHRT_MIN); assert(tmp <= SHRT_MAX); +#endif return (int16_t)tmp; } uint16_t taosStr2UInt16(const char *str, char** pEnd, int32_t radix) { uint32_t tmp = strtoul(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); assert(tmp <= USHRT_MAX); +#endif return (uint16_t)tmp; } int8_t taosStr2Int8(const char *str, char** pEnd, int32_t radix) { int32_t tmp = strtol(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); assert(tmp >= SCHAR_MIN); assert(tmp <= SCHAR_MAX); +#endif return tmp; } uint8_t taosStr2UInt8(const char *str, char** pEnd, int32_t radix) { uint32_t tmp = strtoul(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); assert(tmp <= UCHAR_MAX); +#endif return tmp; } double taosStr2Double(const char *str, char** pEnd) { double tmp = strtod(str, pEnd); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); assert(tmp != HUGE_VAL); +#endif return tmp; } float taosStr2Float(const char *str, char** pEnd) { float tmp = strtof(str, pEnd); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); assert(tmp != HUGE_VALF); assert(tmp != NAN); +#endif return tmp; } \ No newline at end of file