From 527f075b1d30a1076b834bc769a1341fdc9696c8 Mon Sep 17 00:00:00 2001 From: tomchon Date: Mon, 16 May 2022 09:47:08 +0800 Subject: [PATCH 01/28] test:modify testcase that test qnode --- .../1-insert/insertWithMoreVgroup.py | 89 ++++++++++++------- tests/system-test/1-insert/manyVgroups.json | 2 +- 2 files changed, 57 insertions(+), 34 deletions(-) diff --git a/tests/system-test/1-insert/insertWithMoreVgroup.py b/tests/system-test/1-insert/insertWithMoreVgroup.py index d3da4f2c59..6759ece33e 100644 --- a/tests/system-test/1-insert/insertWithMoreVgroup.py +++ b/tests/system-test/1-insert/insertWithMoreVgroup.py @@ -30,7 +30,10 @@ class TDTestCase: # # --------------- main frame ------------------- # - + clientCfgDict = {'queryproxy': '1'} + clientCfgDict["queryproxy"] = '2' + updatecfgDict = {'clientCfg': {}} + updatecfgDict["clientCfg"] = clientCfgDict def caseDescription(self): ''' limit and offset keyword function test cases; @@ -75,8 +78,13 @@ class TDTestCase: # self.test_case2() # tdLog.debug(" LIMIT test_case2 ............ [OK]") - # test case - self.test_case3() + # # test case + # self.test_case3() + # tdLog.debug(" LIMIT test_case3 ............ [OK]") + + + # test qnode + self.test_case4() tdLog.debug(" LIMIT test_case3 ............ [OK]") @@ -199,16 +207,10 @@ class TDTestCase: os.system("%s -f %s -y " %(taosBenchbin,jsonFile)) return - def taosBenchCreate(self,host,dropdb,dbname,stbname,vgroups,threadNumbers,count): + def taosBenchCreate(self,host,dropdb,dbname,stbname,vgroups,processNumbers,count): # count=50000 buildPath = self.getBuildPath() - if (buildPath == ""): - tdLog.exit("taosd not found!") - else: - tdLog.info("taosd found in %s" % buildPath) - taosBenchbin = buildPath+ "/build/bin/taosBenchmark" - buildPath = self.getBuildPath() config = buildPath+ "../sim/dnode1/cfg/" tsql=self.newcur(host,config) @@ -222,8 +224,7 @@ class TDTestCase: tsql.execute("use %s" %dbname) threads = [] - # threadNumbers=2 - for i in range(threadNumbers): + for i in range(processNumbers): jsonfile="1-insert/Vgroups%d%d.json"%(vgroups,i) os.system("cp -f 1-insert/manyVgroups.json %s"%(jsonfile)) os.system("sed -i 's/\"name\": \"db\",/\"name\": \"%s\",/g' %s"%(dbname,jsonfile)) @@ -247,29 +248,8 @@ class TDTestCase: # test case1 base def test_case1(self): tdLog.debug("-----create database and tables test------- ") - tdSql.execute("drop database if exists db1") - tdSql.execute("drop database if exists db4") - tdSql.execute("drop database if exists db6") - tdSql.execute("drop database if exists db8") - tdSql.execute("drop database if exists db12") - tdSql.execute("drop database if exists db16") #create database and tables; - - # tdSql.execute("create database db11 vgroups 1") - # # self.create_tables("db1", "stb1", 30*10000) - # tdSql.execute("use db1") - # tdSql.execute("create stable stb1(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)") - - # tdSql.execute("create database db12 vgroups 1") - # # self.create_tables("db1", "stb1", 30*10000) - # tdSql.execute("use db1") - - # t1 = threading.Thread(target=self.new_create_tables("db1", "stb1", 15*10000), args=(1,)) - # t2 = threading.Thread(target=self.new_create_tables("db1", "stb1", 15*10000), args=(2,)) - # t1 = mp.Process(target=self.new_create_tables, args=("db1", "stb1", 0,count/2,)) - # t2 = mp.Process(target=self.new_create_tables, args=("db1", "stb1", count/2,count,)) - count=50000 vgroups=1 threads = [] @@ -356,6 +336,49 @@ class TDTestCase: return + def test_case4(self): + self.taosBenchCreate("127.0.0.1","no","db1", "stb1", 1, 2, 1*10) + tdSql.execute("use db1;") + tdSql.query("show dnodes;") + dnodeId=tdSql.getData(0,0) + print(dnodeId) + tdSql.execute("create qnode on dnode %s"%dnodeId) + tdSql.query("select max(c1) from stb10;") + maxQnode=tdSql.getData(0,0) + tdSql.query("select min(c1) from stb11;") + minQnode=tdSql.getData(0,0) + tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;") + unionQnode=tdSql.queryResult + tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000;") + unionallQnode=tdSql.queryResult + + # tdSql.query("show qnodes;") + # qnodeId=tdSql.getData(0,0) + tdSql.execute("drop qnode on dnode %s"%dnodeId) + tdSql.execute("reset query cache") + tdSql.query("select max(c1) from stb10;") + tdSql.checkData(0, 0, "%s"%maxQnode) + tdSql.query("select min(c1) from stb11;") + tdSql.checkData(0, 0, "%s"%minQnode) + tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;") + unionVnode=tdSql.queryResult + assert unionQnode == unionVnode + tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000;") + unionallVnode=tdSql.queryResult + assert unionallQnode == unionallVnode + + + # tdSql.execute("create qnode on dnode %s"%dnodeId) + + + # self.taosBenchCreate("test209","no","db2", "stb2", 1, 8, 1*10000) + + # self.taosBenchCreate("chenhaoran02","no","db1", "stb1", 1, 8, 1*10000) + + # self.taosBenchCreate("db1", "stb1", 4, 5, 100*10000) + # self.taosBenchCreate("db1", "stb1", 1, 5, 100*10000) + + return # # add case with filename # diff --git a/tests/system-test/1-insert/manyVgroups.json b/tests/system-test/1-insert/manyVgroups.json index e6719aedc9..1c9aa1f28c 100644 --- a/tests/system-test/1-insert/manyVgroups.json +++ b/tests/system-test/1-insert/manyVgroups.json @@ -11,7 +11,7 @@ "confirm_parameter_prompt": "no", "insert_interval": 0, "interlace_rows": 100000, - "num_of_records_per_req": 100000, + "num_of_records_per_req": 100, "databases": [ { "dbinfo": { From 6acc51dbb517a4b341716da3c50a4e15235da683 Mon Sep 17 00:00:00 2001 From: tomchon Date: Tue, 17 May 2022 20:28:46 +0800 Subject: [PATCH 02/28] modify testcase of threeReplica --- .../sync/threeReplica1VgElectWihtInsert.sim | 70 +++++++++---------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim b/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim index 1e12e8565f..f6996f1291 100644 --- a/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim +++ b/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim @@ -31,7 +31,7 @@ if $data[0][4] != ready then goto check_dnode_ready endi -#sql connect +sql connect sql create dnode $hostname port 7200 sql create dnode $hostname port 7300 sql create dnode $hostname port 7400 @@ -83,7 +83,7 @@ print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $dat if $rows != 3 then return -1 endi -if $data(db)[19] != ready then +if $data(db)[19] != nostrict then goto check_db_ready endi @@ -93,49 +93,48 @@ $loop_cnt = 0 check_vg_ready: $loop_cnt = $loop_cnt + 1 sleep 200 -if $loop_cnt == 10 then +if $loop_cnt == 40 then print ====> vgroups not ready! return -1 endi sql show vgroups print ===> rows: $rows -print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13] print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[10][6] $data[0][11] $data[0][12] $data[0][13] if $rows != $vgroups then return -1 endi if $data[0][4] == LEADER then - if $data[0][6] != FLLOWER then + if $data[0][6] != FOLLOWER then goto check_vg_ready endi - if $data[0][8] != FLLOWER then + if $data[0][8] != FOLLOWER then goto check_vg_ready endi print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] - goto vg_ready -endi -if $data[0][6] == LEADER then - if $data[0][4] != FLLOWER then + goto vg_ready +elif $data[0][6] == LEADER then + if $data[0][4] != FOLLOWER then goto check_vg_ready endi - if $data[0][8] != FLLOWER then + if $data[0][8] != FOLLOWER then goto check_vg_ready endi print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] - goto vg_ready -endi -if $data[0][8] == LEADER then - if $data[0][4] != FLLOWER then + goto vg_ready +elif $data[0][8] == LEADER then + if $data[0][4] != FOLLOWER then goto check_vg_ready endi - if $data[0][6] != FLLOWER then + if $data[0][6] != FOLLOWER then goto check_vg_ready endi print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] - goto vg_ready + goto vg_ready +else + goto check_vg_ready endi -vg_ready: +vg_ready: print ====> create stable/child table sql create table stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) @@ -185,7 +184,7 @@ print ====> create a normal table for interaction between main and back threads sql create table interaction (ts timestamp, flag binary(10), childrows int, stbrows int) print ====> start to run_back to insert data -run_back tsim/tmq/insertDataByRunBack.sim +run_back tsim/sync/insertDataByRunBack.sim print ====> waiting insert thread starting insert data @@ -239,34 +238,34 @@ if $rows != $vgroups then return -1 endi if $data[0][4] == LEADER then - if $data[0][6] != FLLOWER then + if $data[0][6] != FOLLOWER then goto check_vg_ready_2 endi - if $data[0][8] != FLLOWER then + if $data[0][8] != FOLLOWER then goto check_vg_ready_2 endi print ---- vgroup $data[0][0] leader switch to dnode $data[0][3] goto vg_ready_2 -endi -if $data[0][6] == LEADER then - if $data[0][4] != FLLOWER then +elif $data[0][6] == LEADER then + if $data[0][4] != FOLLOWER then goto check_vg_ready_2 endi - if $data[0][8] != FLLOWER then + if $data[0][8] != FOLLOWER then goto check_vg_ready_2 endi print ---- vgroup $data[0][0] leader switch to dnode $data[0][5] goto vg_ready_2 -endi -if $data[0][8] == LEADER then - if $data[0][4] != FLLOWER then +elif $data[0][8] == LEADER then + if $data[0][4] != FOLLOWER then goto check_vg_ready_2 endi - if $data[0][6] != FLLOWER then + if $data[0][6] != FOLLOWER then goto check_vg_ready_2 endi print ---- vgroup $data[0][0] leader switch to dnode $data[0][7] goto vg_ready_2 +else + goto check_vg_ready_2 endi vg_ready_2: @@ -344,28 +343,28 @@ if $rows != $vgroups then return -1 endi if $data[0][4] == LEADER then - if $data[0][6] != FLLOWER then + if $data[0][6] != FOLLOWER then goto check_vg_ready_1 endi - if $data[0][8] != FLLOWER then + if $data[0][8] != FOLLOWER then goto check_vg_ready_1 endi goto vg_ready_1 endi if $data[0][6] == LEADER then - if $data[0][4] != FLLOWER then + if $data[0][4] != FOLLOWER then goto check_vg_ready_1 endi - if $data[0][8] != FLLOWER then + if $data[0][8] != FOLLOWER then goto check_vg_ready_1 endi goto vg_ready_1 endi if $data[0][8] == LEADER then - if $data[0][4] != FLLOWER then + if $data[0][4] != FOLLOWER then goto check_vg_ready_1 endi - if $data[0][6] != FLLOWER then + if $data[0][6] != FOLLOWER then goto check_vg_ready_1 endi goto vg_ready_1 @@ -420,7 +419,6 @@ elif $data[0][6] == LEADER then goto check_vg_ready_3 endi print ---- vgroup $data[0][0] leader locating dnode $data[0][7] -endi elif $data[0][8] == LEADER then if $data[0][4] == LEADER then goto check_vg_ready_3 From c85232e14c7168ecd56c61950481abaac9169461 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 18 May 2022 16:37:09 +0800 Subject: [PATCH 03/28] update table meta based on sversion --- include/libs/catalog/catalog.h | 5 +++ source/client/src/clientEnv.c | 10 ++++++ source/client/src/clientImpl.c | 57 ++++++++++++++++++++++++++++--- source/libs/catalog/src/catalog.c | 5 +++ 4 files changed, 73 insertions(+), 4 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 04a24c4f32..a29dc3a90a 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -59,6 +59,11 @@ typedef struct SMetaData { SArray *pQnodeList; // qnode list, SArray } SMetaData; +typedef struct STbSVersion { + char* tbFName; + int32_t sver; +} STbSVersion; + typedef struct SCatalogCfg { uint32_t maxTblCacheNum; uint32_t maxDBCacheNum; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index c82cce9af3..30997def74 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -83,6 +83,15 @@ void closeTransporter(STscObj *pTscObj) { rpcClose(pTscObj->pAppInfo->pTransporter); } +static bool clientRpcRfp(int32_t code) { + if (code == TSDB_CODE_RPC_REDIRECT) { + return true; + } else { + return false; + } +} + + // TODO refactor void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { SRpcInit rpcInit; @@ -91,6 +100,7 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.label = "TSC"; rpcInit.numOfThreads = numOfThread; rpcInit.cfp = processMsgFromServer; + rpcInit.rfp = clientRpcRfp; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)user; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 23787651b2..5d3a139aeb 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -310,9 +310,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList } } - if (pRes) { - *pRes = res.res; - } + *pRes = res.res; pRequest->code = res.code; terrno = res.code; @@ -324,7 +322,49 @@ int32_t getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList) return getPlan(pRequest, pQuery, &pRequest->body.pDag, *pNodeList); } +int32_t validateSversion(SRequestObj* pRequest, void* res) { + SArray* pArray = NULL; + int32_t code = 0; + + if (TDMT_VND_SUBMIT == pRequest->type) { + SSubmitRsp* pRsp = (SSubmitRsp*)res; + if (pRsp->nBlocks <= 0) { + return TSDB_CODE_SUCCESS; + } + + pArray = taosArrayInit(pRsp->nBlocks, sizeof(STbSVersion)); + if (NULL == pArray) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < pRsp->nBlocks; ++i) { + SSubmitBlkRsp *blk = pRsp->pBlocks + i; + STbSVersion tbSver = {.tbFName = blk->tblFName, .sver = blk->sver}; + taosArrayPush(pArray, &tbSver); + } + } else if (TDMT_VND_QUERY == pRequest->type) { + + } + + SCatalog* pCatalog = NULL; + CHECK_CODE_GOTO(catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog), _return); + + SEpSet epset = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + + code = catalogChkTbMetaVersion(pCatalog, pRequest->pTscObj->pAppInfo->pTransporter, &epset, pArray); + +_return: + + taosArrayDestroy(pArray); + + return code; +} + + SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery, void** res) { + void* pRes = NULL; + if (TSDB_CODE_SUCCESS == code) { switch (pQuery->execMode) { case QUERY_EXEC_MODE_LOCAL: @@ -337,7 +377,10 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); code = getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList); if (TSDB_CODE_SUCCESS == code) { - code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList, res); + code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList, &pRes); + if (NULL != pRes) { + code = validateSversion(pRequest, pRes); + } } taosArrayDestroy(pNodeList); break; @@ -356,6 +399,12 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code if (NULL != pRequest && TSDB_CODE_SUCCESS != code) { pRequest->code = terrno; + freeRequestRes(pRequest, pRes); + pRes = NULL; + } + + if (res) { + *res = pRes; } return pRequest; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index d3045f68a1..a620b84f6d 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -2883,6 +2883,11 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pTables) { + +} + + int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName) { CTG_API_ENTER(); From 97a20ee86b93fb3e30e261cda1e43e3db16c11ad Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 18 May 2022 20:31:30 +0800 Subject: [PATCH 04/28] update meta based on sversion --- include/libs/catalog/catalog.h | 2 + source/client/src/clientImpl.c | 11 +++ source/common/src/tmsg.c | 16 ++-- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 2 + source/dnode/vnode/src/vnd/vnodeSvr.c | 3 + source/libs/catalog/src/catalog.c | 101 ++++++++++++++++++++- 6 files changed, 124 insertions(+), 11 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index a29dc3a90a..f9d8fc0de1 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -170,6 +170,8 @@ int32_t catalogUpdateSTableMeta(SCatalog* pCatalog, STableMetaRsp *rspMsg); */ int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName); +int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pTables); + /** * Force refresh a table's local cached meta data. * @param pCatalog (input, got with catalogGetHandle) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 5d3a139aeb..c3ccecb0d3 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -361,6 +361,17 @@ _return: return code; } +void freeRequestRes(SRequestObj* pRequest, void* res) { + if (NULL == res) { + return; + } + + if (TDMT_VND_SUBMIT == pRequest->type) { + tFreeSSubmitRsp((SSubmitRsp*)res); + } else if (TDMT_VND_QUERY == pRequest->type) { + + } +} SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery, void** res) { void* pRes = NULL; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 4e97ebbe47..278fc2c193 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4087,10 +4087,8 @@ static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBl if (tEncodeI32(pEncoder, pBlock->code) < 0) return -1; if (tEncodeI8(pEncoder, pBlock->hashMeta) < 0) return -1; - if (pBlock->hashMeta) { - if (tEncodeI64(pEncoder, pBlock->uid) < 0) return -1; - if (tEncodeCStr(pEncoder, pBlock->tblFName) < 0) return -1; - } + if (tEncodeI64(pEncoder, pBlock->uid) < 0) return -1; + if (tEncodeCStr(pEncoder, pBlock->tblFName) < 0) return -1; if (tEncodeI32v(pEncoder, pBlock->numOfRows) < 0) return -1; if (tEncodeI32v(pEncoder, pBlock->affectedRows) < 0) return -1; if (tEncodeI64v(pEncoder, pBlock->sver) < 0) return -1; @@ -4104,12 +4102,10 @@ static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) { if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1; if (tDecodeI8(pDecoder, &pBlock->hashMeta) < 0) return -1; - if (pBlock->hashMeta) { - if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1; - pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1); - if (NULL == pBlock->tblFName) return -1; - if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1; - } + if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1; + pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1); + if (NULL == pBlock->tblFName) return -1; + if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1; if (tDecodeI32v(pDecoder, &pBlock->numOfRows) < 0) return -1; if (tDecodeI32v(pDecoder, &pBlock->affectedRows) < 0) return -1; if (tDecodeI64v(pDecoder, &pBlock->sver) < 0) return -1; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 2fdbfdd206..d8426db127 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -320,6 +320,8 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; return -1; } + strcat(pRsp->tblFName, mr.me.name); + if (mr.me.type == TSDB_NORMAL_TABLE) { sverNew = mr.me.ntbEntry.schema.sver; } else { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 464331319e..371ade4f76 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -676,6 +676,9 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in vnodeDebugPrintSingleSubmitMsg(pVnode->pMeta, pBlock, &msgIter, "real uid"); tDecoderClear(&decoder); + } else { + submitBlkRsp.tblFName = taosMemoryMalloc(TSDB_TABLE_FNAME_LEN); + sprintf(submitBlkRsp.tblFName, "%s.", pVnode->config.dbname); } if (tsdbInsertTableData(pVnode->pTsdb, &msgIter, pBlock, &submitBlkRsp) < 0) { diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index a620b84f6d..23957d1a6b 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -2883,8 +2883,107 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pTables) { + +int32_t ctgGetTbSverFromCache(SCatalog* pCtg, const SName* pTableName, int32_t* sver) { + *sver = -1; + + if (NULL == pCtg->dbCache) { + ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname); + return TSDB_CODE_SUCCESS; + } + + SCtgDBCache *dbCache = NULL; + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, dbFName); + + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + ctgDebug("db %s not in cache", pTableName->tname); + return TSDB_CODE_SUCCESS; + } + + int32_t tbType = 0; + uint64_t suid = 0; + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + STableMeta* tbMeta = taosHashGet(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname)); + if (tbMeta) { + tbType = tbMeta->tableType; + suid = tbMeta->suid; + if (tbType != TSDB_CHILD_TABLE) { + *sver = tbMeta->sversion; + } + } + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + if (NULL == tbMeta) { + ctgReleaseDBCache(pCtg, dbCache); + return TSDB_CODE_SUCCESS; + } + + if (tbType != TSDB_CHILD_TABLE) { + ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, tbType, dbFName, pTableName->tname); + + return TSDB_CODE_SUCCESS; + } + + ctgDebug("Got subtable meta from cache, dbFName:%s, tbName:%s, suid:%" PRIx64, dbFName, pTableName->tname, suid); + + CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); + + STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &suid, sizeof(suid)); + if (NULL == stbMeta || NULL == *stbMeta) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("stb not in stbCache, suid:%"PRIx64, suid); + return TSDB_CODE_SUCCESS; + } + + if ((*stbMeta)->suid != suid) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgReleaseDBCache(pCtg, dbCache); + ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, suid, (*stbMeta)->suid); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + *sver = (*stbMeta)->sversion; + + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + + ctgReleaseDBCache(pCtg, dbCache); + + ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, tbType, dbFName, pTableName->tname); + + return TSDB_CODE_SUCCESS; +} + + +int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pTables) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTables) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + SName name; + int32_t sver = 0; + int32_t tbNum = taosArrayGetSize(pTables); + for (int32_t i = 0; i < tbNum; ++i) { + STbSVersion* pTb = (STbSVersion*)taosArrayGet(pTables, i); + tNameFromString(&name, pTb->tbFName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + + if (CTG_IS_SYS_DBNAME(name.dbname)) { + continue; + } + + ctgGetTbSverFromCache(pCtg, &name, &sver); + if (sver >= 0 && sver < pTb->sver) { + catalogRemoveTableMeta(pCtg, &name); //TODO REMOVE STB FROM CACHE + } + } + + CTG_API_LEAVE(TSDB_CODE_SUCCESS); } From 47e6e95866f760e9787a51fad36c7fe846822470 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 18 May 2022 20:39:00 +0800 Subject: [PATCH 05/28] enh(query): record and return the schema version and tags version of current queried table. --- include/libs/executor/executor.h | 9 ++++++ source/libs/executor/inc/executorimpl.h | 10 ++++++- source/libs/executor/src/executor.c | 13 ++++++++ source/libs/executor/src/executorimpl.c | 40 +++++++++++++++++++++---- source/libs/executor/src/scanoperator.c | 2 +- 5 files changed, 67 insertions(+), 7 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 8f300c96c5..9cafb4ee04 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -95,6 +95,15 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, EOPTR_EXEC_MODEL model); +/** + * + * @param tinfo + * @param sversion + * @param tversion + * @return + */ +int32_t qGetQueriedTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion, int32_t* tversion); + /** * The main task execution function, including query on both table and multiple tables, * which are decided according to the tag or table name query conditions diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index bf178612ba..89bbb1f48c 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -217,6 +217,13 @@ typedef struct SExecTaskInfo { int64_t owner; // if it is in execution int32_t code; uint64_t totalRows; // total number of rows + struct { + char *tablename; + char *dbname; + int32_t sversion; + int32_t tversion; + } schemaVer; + STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure char* sql; // query sql string jmp_buf env; // jump to this position when error happens. @@ -670,7 +677,8 @@ SSDataBlock* loadNextDataBlock(void* param); void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, - int32_t type); + SExecTaskInfo* pTaskInfo, int32_t type); + SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs); SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode); int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 66073b70eb..314ed97cfc 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -14,6 +14,7 @@ */ #include "executor.h" +#include #include #include "executorimpl.h" #include "planner.h" @@ -171,3 +172,15 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo return TSDB_CODE_SUCCESS; } + +int32_t qGetQueriedTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion, int32_t* tversion) { + ASSERT(tinfo != NULL && dbName != NULL && tableName != NULL); + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*) tinfo; + + *sversion = pTaskInfo->schemaVer.sversion; + *tversion = pTaskInfo->schemaVer.tversion; + strcpy(dbName, pTaskInfo->schemaVer.dbname); + strcpy(tableName, pTaskInfo->schemaVer.tablename); + + return 0; +} \ No newline at end of file diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d0a1840d72..bd93284da2 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include #include "filter.h" #include "function.h" #include "functionMgt.h" @@ -4653,6 +4654,26 @@ static SArray* extractColumnInfo(SNodeList* pNodeList); static SArray* createSortInfo(SNodeList* pNodeList); static SArray* extractPartitionColInfo(SNodeList* pNodeList); +void extractTableSchemaVersion(SReadHandle *pHandle, uint64_t uid, SExecTaskInfo* pTaskInfo) { + SMetaReader mr = {0}; + metaReaderInit(&mr, pHandle->meta, 0); + metaGetTableEntryByUid(&mr, uid); + + pTaskInfo->schemaVer.tablename = strdup(mr.me.name); + + if (mr.me.type == TSDB_SUPER_TABLE) { + pTaskInfo->schemaVer.sversion = mr.me.stbEntry.schema.sver; + pTaskInfo->schemaVer.tversion = mr.me.stbEntry.schemaTag.sver; + } else if (mr.me.type == TSDB_CHILD_TABLE) { + tb_uid_t suid = mr.me.ctbEntry.suid; + metaGetTableEntryByUid(&mr, suid); + pTaskInfo->schemaVer.sversion = mr.me.stbEntry.schema.sver; + pTaskInfo->schemaVer.tversion = mr.me.stbEntry.schemaTag.sver; + } else { + pTaskInfo->schemaVer.sversion = mr.me.ntbEntry.schema.sver; + } +} + SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId, STableGroupInfo* pTableGroupInfo) { int32_t type = nodeType(pPhyNode); @@ -4666,6 +4687,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return NULL; } + extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo); SOperatorInfo* pOperator = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo); STableScanInfo* pScanInfo = pOperator->info; @@ -4703,7 +4725,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SArray* tableIdList = extractTableIdList(pTableGroupInfo); SSDataBlock* pResBlock = createResDataBlock(pDescNode); - SArray* pCols = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID); + SArray* pCols = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, pTaskInfo, COL_MATCH_FROM_COL_ID); SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pDataReader, pHandle, pScanPhyNode->uid, pResBlock, pCols, tableIdList, pTaskInfo, pScanPhyNode->node.pConditions, pOperatorDumy); @@ -4718,7 +4740,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SSDataBlock* pResBlock = createResDataBlock(pDescNode); int32_t numOfOutputCols = 0; - SArray* colList = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &numOfOutputCols, COL_MATCH_FROM_COL_ID); + SArray* colList = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &numOfOutputCols, pTaskInfo, COL_MATCH_FROM_COL_ID); SOperatorInfo* pOperator = createSysTableScanOperatorInfo( pHandle, pResBlock, &pScanNode->tableName, pScanNode->node.pConditions, pSysScanPhyNode->mgmtEpSet, colList, pTaskInfo, pSysScanPhyNode->showRewrite, pSysScanPhyNode->accountId); @@ -4741,7 +4763,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t numOfOutputCols = 0; SArray* colList = - extractColMatchInfo(pScanPhyNode->pScanPseudoCols, pDescNode, &numOfOutputCols, COL_MATCH_FROM_COL_ID); + extractColMatchInfo(pScanPhyNode->pScanPseudoCols, pDescNode, &numOfOutputCols, pTaskInfo, COL_MATCH_FROM_COL_ID); SOperatorInfo* pOperator = createTagScanOperatorInfo(pHandle, pExprInfo, num, pResBlock, colList, pTableGroupInfo, pTaskInfo); @@ -4823,7 +4845,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SExprInfo* pExprInfo = createExprInfo(pSortPhyNode->pExprs, NULL, &numOfCols); int32_t numOfOutputCols = 0; - SArray* pColList = extractColMatchInfo(pSortPhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); + SArray* pColList = extractColMatchInfo(pSortPhyNode->pTargets, pDescNode, &numOfOutputCols, pTaskInfo, COL_MATCH_FROM_SLOT_ID); pOptr = createSortOperatorInfo(ops[0], pResBlock, info, pExprInfo, numOfCols, pColList, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW == type) { @@ -5014,7 +5036,7 @@ SArray* createSortInfo(SNodeList* pNodeList) { } SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, - int32_t type) { + SExecTaskInfo* pTaskInfo, int32_t type) { size_t numOfCols = LIST_LENGTH(pNodeList); SArray* pList = taosArrayInit(numOfCols, sizeof(SColMatchInfo)); if (pList == NULL) { @@ -5022,10 +5044,16 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod return NULL; } + const char* tname = pTaskInfo->schemaVer.tablename; for (int32_t i = 0; i < numOfCols; ++i) { STargetNode* pNode = (STargetNode*)nodesListGetNode(pNodeList, i); SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; + if (tname != NULL && (pTaskInfo->schemaVer.dbname == NULL) && + strncmp(pColNode->tableName, tname, tListLen(pColNode->tableName)) == 0) { + pTaskInfo->schemaVer.dbname = strdup(pColNode->dbName); + } + SColMatchInfo c = {0}; c.output = true; c.colId = pColNode->colId; @@ -5219,6 +5247,8 @@ void doDestroyTask(SExecTaskInfo* pTaskInfo) { // taosArrayDestroy(pTaskInfo->summary.queryProfEvents); // taosHashCleanup(pTaskInfo->summary.operatorProfResults); + taosMemoryFree(pTaskInfo->schemaVer.dbname); + taosMemoryFree(pTaskInfo->schemaVer.tablename); taosMemoryFreeClear(pTaskInfo->sql); taosMemoryFreeClear(pTaskInfo->id.str); taosMemoryFreeClear(pTaskInfo); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index d042e463f0..470c9d6569 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -455,7 +455,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SDataBlockDescNode* pDescNode = pTableScanNode->scan.node.pOutputDataBlockDesc; int32_t numOfCols = 0; - SArray* pColList = extractColMatchInfo(pTableScanNode->scan.pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID); + SArray* pColList = extractColMatchInfo(pTableScanNode->scan.pScanCols, pDescNode, &numOfCols, pTaskInfo, COL_MATCH_FROM_COL_ID); int32_t code = initQueryTableDataCond(&pInfo->cond, pTableScanNode); if (code != TSDB_CODE_SUCCESS) { From 003963f8bb7846993f5415f447b71567a2a0e701 Mon Sep 17 00:00:00 2001 From: tomchon Date: Thu, 19 May 2022 01:25:49 +0800 Subject: [PATCH 06/28] test:modify testcase of multi-Thread creating tables --- .../1-insert/insertWithMoreVgroup.py | 238 +++++++++--------- 1 file changed, 122 insertions(+), 116 deletions(-) diff --git a/tests/system-test/1-insert/insertWithMoreVgroup.py b/tests/system-test/1-insert/insertWithMoreVgroup.py index 6759ece33e..d8720e8045 100644 --- a/tests/system-test/1-insert/insertWithMoreVgroup.py +++ b/tests/system-test/1-insert/insertWithMoreVgroup.py @@ -13,7 +13,7 @@ import sys import os -import threading +import threading as thd import multiprocessing as mp from numpy.lib.function_base import insert import taos @@ -66,58 +66,13 @@ class TDTestCase: # self.create_tables(); self.ts = 1500000000000 - - # run case - def run(self): - - # # test base case - # self.test_case1() - # tdLog.debug(" LIMIT test_case1 ............ [OK]") - - # test case - # self.test_case2() - # tdLog.debug(" LIMIT test_case2 ............ [OK]") - - # # test case - # self.test_case3() - # tdLog.debug(" LIMIT test_case3 ............ [OK]") - - - # test qnode - self.test_case4() - tdLog.debug(" LIMIT test_case3 ............ [OK]") - - # stop def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) - # --------------- case ------------------- - # create tables - def create_tables(self,dbname,stbname,count): - tdSql.execute("use %s" %dbname) - tdSql.execute("create stable %s(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%stbname) - pre_create = "create table" - sql = pre_create - tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) - # print(time.time()) - exeStartTime=time.time() - for i in range(count): - sql += " %s_%d using %s tags(%d)"%(stbname,i,stbname,i+1) - if i >0 and i%3000 == 0: - tdSql.execute(sql) - sql = pre_create - # print(time.time()) - # end sql - if sql != pre_create: - tdSql.execute(sql) - exeEndTime=time.time() - spendTime=exeEndTime-exeStartTime - speedCreate=count/spendTime - tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate)) - return + # --------------- case ------------------- def newcur(self,host,cfg): user = "root" @@ -128,28 +83,23 @@ class TDTestCase: print(cur) return cur - def new_create_tables(self,dbname,vgroups,stbname,tcountStart,tcountStop): - host = "localhost" + # create tables + def create_tables(self,host,dbname,stbname,count): buildPath = self.getBuildPath() config = buildPath+ "../sim/dnode1/cfg/" tsql=self.newcur(host,config) - tsql.execute("drop database if exists %s"%dbname) - tsql.execute("create database %s vgroups %d"%(dbname,vgroups)) tsql.execute("use %s" %dbname) - tsql.execute("create stable %s(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%stbname) pre_create = "create table" sql = pre_create - tcountStop=int(tcountStop) - tcountStart=int(tcountStart) - count=tcountStop-tcountStart + count=int(count) tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) # print(time.time()) exeStartTime=time.time() # print(type(tcountStop),type(tcountStart)) - for i in range(tcountStart,tcountStop): + for i in range(0,count): sql += " %s_%d using %s tags(%d)"%(stbname,i,stbname,i+1) if i >0 and i%20000 == 0: # print(sql) @@ -166,11 +116,78 @@ class TDTestCase: # tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate)) return + def mutiThread_create_tables(self,host,dbname,stbname,vgroups,threadNumbers,count): + buildPath = self.getBuildPath() + config = buildPath+ "../sim/dnode1/cfg/" + + tsql=self.newcur(host,config) + tdLog.debug("create database %s"%dbname) + tsql.execute("drop database if exists %s"%dbname) + tsql.execute("create database %s vgroups %d"%(dbname,vgroups)) + tsql.execute("use %s" %dbname) + count=int(count) + threads = [] + for i in range(threadNumbers): + tsql.execute("create stable %s%d(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%(stbname,i)) + threads.append(thd.Thread(target=self.create_tables, args=(host, dbname, stbname+"%d"%i, count,))) + start_time = time.time() + for tr in threads: + tr.start() + for tr in threads: + tr.join() + end_time = time.time() + spendTime=end_time-start_time + speedCreate=count/spendTime + tdLog.debug("spent %.2fs to create %d stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,threadNumbers,threadNumbers*count,speedCreate)) + + return + + # def create_tables(self,host,dbname,stbname,vgroups,tcountStart,tcountStop): # insert data - def insert_data(self, dbname, stbname, ts_start, tcountStart,tcountStop,rowCount): - tdSql.execute("use %s" %dbname) + def insert_data(self, host, dbname, stbname, ts_start,rowCount): + buildPath = self.getBuildPath() + config = buildPath+ "../sim/dnode1/cfg/" + + tsql=self.newcur(host,config) + tdLog.debug("ready to inser data") + + tsql.execute("use %s" %dbname) + pre_insert = "insert into " + sql = pre_insert + tcount=int(tcount) + allRows=tcount*rowCount + tdLog.debug("doing insert data into stable-index:%s rows:%d ..."%(stbname, allRows)) + exeStartTime=time.time() + for i in range(0,tcount): + sql += " %s_%d values "%(stbname,i) + for j in range(rowCount): + sql += "(%d, %d, 'taos_%d') "%(ts_start + j*1000, j, j) + if j >0 and j%5000 == 0: + # print(sql) + tdSql.execute(sql) + sql = "insert into %s_%d values " %(stbname,i) + # end sql + if sql != pre_insert: + # print(sql) + tdSql.execute(sql) + exeEndTime=time.time() + spendTime=exeEndTime-exeStartTime + speedInsert=allRows/spendTime + # tdLog.debug("spent %.2fs to INSERT %d rows , insert rate is %.2f rows/s... [OK]"% (spendTime,allRows,speedInsert)) + + tdLog.debug("INSERT TABLE DATA ............ [OK]") + return + + def mutiThread_insert_data(self, host, dbname, stbname, threadNumbers, ts_start, tcountStart,tcountStop,rowCount): + buildPath = self.getBuildPath() + config = buildPath+ "../sim/dnode1/cfg/" + + tsql=self.newcur(host,config) + tdLog.debug("ready to inser data") + + tsql.execute("use %s" %dbname) pre_insert = "insert into " sql = pre_insert tcount=tcountStop-tcountStart @@ -195,8 +212,30 @@ class TDTestCase: # tdLog.debug("spent %.2fs to INSERT %d rows , insert rate is %.2f rows/s... [OK]"% (spendTime,allRows,speedInsert)) tdLog.debug("INSERT TABLE DATA ............ [OK]") + + + buildPath = self.getBuildPath() + config = buildPath+ "../sim/dnode1/cfg/" + + tsql=self.newcur(host,config) + tsql.execute("use %s" %dbname) + count=int(count) + threads = [] + for i in range(threadNumbers): + tsql.execute("create stable %s%d(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%(stbname,i)) + threads.append(thd.Thread(target=self.create_tables, args=(host, dbname, stbname+"%d"%i, count,))) + start_time = time.time() + for tr in threads: + tr.start() + for tr in threads: + tr.join() + end_time = time.time() + spendTime=end_time-start_time + speedCreate=count/spendTime + tdLog.debug("spent %.2fs to create %d stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,threadNumbers,threadNumbers*count,speedCreate)) return + def taosBench(self,jsonFile): buildPath = self.getBuildPath() if (buildPath == ""): @@ -247,47 +286,15 @@ class TDTestCase: return # test case1 base def test_case1(self): - tdLog.debug("-----create database and tables test------- ") - - #create database and tables; - count=50000 - vgroups=1 - threads = [] - threadNumbers=2 - for i in range(threadNumbers): - threads.append(mp.Process(target=self.new_create_tables, args=("db1%d"%i, vgroups, "stb1", 0,count,))) - start_time = time.time() - for tr in threads: - tr.start() - for tr in threads: - tr.join() - end_time = time.time() - spendTime=end_time-start_time - speedCreate=count/spendTime - tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate)) - # self.new_create_tables("db1", "stb1", 15*10000) - # self.new_create_tables("db1", "stb1", 15*10000) - - # tdSql.execute("create database db4 vgroups 4") - # self.create_tables("db4", "stb4", 30*10000) - - # tdSql.execute("create database db6 vgroups 6") - # self.create_tables("db6", "stb6", 30*10000) - - # tdSql.execute("create database db8 vgroups 8") - # self.create_tables("db8", "stb8", 30*10000) - - # tdSql.execute("create database db12 vgroups 12") - # self.create_tables("db12", "stb12", 30*10000) - - # tdSql.execute("create database db16 vgroups 16") - # self.create_tables("db16", "stb16", 30*10000) + tdLog.debug("-----create database and muti-thread create tables test------- ") + #host,dbname,stbname,vgroups,threadNumbers,tcountStart,tcountStop + self.mutiThread_create_tables(host="localhost",dbname="db2",stbname="stb2", vgroups=1, threadNumbers=5, count=10000) return # test case2 base:insert data def test_case2(self): - tdLog.debug("-----insert data test------- ") + tdLog.debug("-----muti-thread insert data test------- ") # drop database tdSql.execute("drop database if exists db1") tdSql.execute("drop database if exists db4") @@ -301,28 +308,6 @@ class TDTestCase: tdSql.execute("create database db1 vgroups 1") self.create_tables("db1", "stb1", 1*100) self.insert_data("db1", "stb1", self.ts, 1*50,1*10000) - - - tdSql.execute("create database db4 vgroups 4") - self.create_tables("db4", "stb4", 1*100) - self.insert_data("db4", "stb4", self.ts, 1*100,1*10000) - - tdSql.execute("create database db6 vgroups 6") - self.create_tables("db6", "stb6", 1*100) - self.insert_data("db6", "stb6", self.ts, 1*100,1*10000) - - tdSql.execute("create database db8 vgroups 8") - self.create_tables("db8", "stb8", 1*100) - self.insert_data("db8", "stb8", self.ts, 1*100,1*10000) - - tdSql.execute("create database db12 vgroups 12") - self.create_tables("db12", "stb12", 1*100) - self.insert_data("db12", "stb12", self.ts, 1*100,1*10000) - - tdSql.execute("create database db16 vgroups 16") - self.create_tables("db16", "stb16", 1*100) - self.insert_data("db16", "stb16", self.ts, 1*100,1*10000) - return def test_case3(self): @@ -378,6 +363,27 @@ class TDTestCase: # self.taosBenchCreate("db1", "stb1", 4, 5, 100*10000) # self.taosBenchCreate("db1", "stb1", 1, 5, 100*10000) + # run case + def run(self): + + # # test base case + # self.test_case1() + # tdLog.debug(" LIMIT test_case1 ............ [OK]") + + # test case + # self.test_case2() + # tdLog.debug(" LIMIT test_case2 ............ [OK]") + + # test case + self.test_case3() + tdLog.debug(" LIMIT test_case3 ............ [OK]") + + + # # test qnode + # self.test_case4() + # tdLog.debug(" LIMIT test_case3 ............ [OK]") + + return # # add case with filename From fb944e85bc247378fb92eb67ba3237e5a93ec7a8 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 19 May 2022 09:39:28 +0800 Subject: [PATCH 07/28] fix explain res issue --- include/libs/scheduler/scheduler.h | 2 +- source/client/src/clientImpl.c | 2 +- source/libs/scheduler/inc/schedulerInt.h | 8 +++- source/libs/scheduler/src/scheduler.c | 46 +++++++++---------- source/libs/scheduler/test/schedulerTests.cpp | 2 +- 5 files changed, 33 insertions(+), 27 deletions(-) diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index b3f35025d1..dcd058a293 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -72,7 +72,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg); * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, int64_t startTs, bool needRes, SQueryResult *pRes); +int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, int64_t startTs, SQueryResult *pRes); /** * Process the query job, generated according to the query physical plan. diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index c3ccecb0d3..7693d26d3f 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -291,7 +291,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr, - pRequest->metric.start, NULL != pRes, &res); + pRequest->metric.start, &res); if (code != TSDB_CODE_SUCCESS) { if (pRequest->body.queryJob != 0) { schedulerFreeJob(pRequest->body.queryJob); diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index a90fb7fc2e..5a6fcee759 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -39,6 +39,12 @@ enum { SCH_WRITE, }; +typedef enum { + SCH_RES_TYPE_QUERY, + SCH_RES_TYPE_FETCH, +} SCH_RES_TYPE; + + typedef struct SSchTrans { void *transInst; void *transHandle; @@ -159,7 +165,6 @@ typedef struct SSchTask { typedef struct SSchJobAttr { EExplainMode explainMode; - bool needRes; bool syncSchedule; bool queryJob; bool needFlowCtrl; @@ -192,6 +197,7 @@ typedef struct SSchJob { int32_t errCode; SArray *errList; // SArray SRWLatch resLock; + SCH_RES_TYPE resType; void *resData; //TODO free it or not int32_t resNumOfRows; const char *sql; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index bdf674ab93..9354c1a875 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -70,7 +70,7 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel * } int32_t schInitJob(SSchJob **pSchJob, SQueryPlan *pDag, void *transport, SArray *pNodeList, const char *sql, - int64_t startTs, bool needRes, bool syncSchedule) { + int64_t startTs, bool syncSchedule) { int32_t code = 0; int64_t refId = -1; SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); @@ -81,7 +81,6 @@ int32_t schInitJob(SSchJob **pSchJob, SQueryPlan *pDag, void *transport, SArray pJob->attr.explainMode = pDag->explainInfo.mode; pJob->attr.syncSchedule = syncSchedule; - pJob->attr.needRes = needRes; pJob->transport = transport; pJob->sql = sql; @@ -1059,6 +1058,8 @@ _return: int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRsp *pRsp) { SCH_TASK_DLOG("got explain rsp, rows:%d, complete:%d", htonl(pRsp->numOfRows), pRsp->completed); + pJob->resType = SCH_RES_TYPE_FETCH; + atomic_store_32(&pJob->resNumOfRows, htonl(pRsp->numOfRows)); atomic_store_ptr(&pJob->resData, pRsp); @@ -1179,23 +1180,20 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch atomic_add_fetch_32(&pJob->resNumOfRows, rsp->affectedRows); SCH_TASK_DLOG("submit succeed, affectedRows:%d", rsp->affectedRows); - if (pJob->attr.needRes) { - SCH_LOCK(SCH_WRITE, &pJob->resLock); - if (pJob->resData) { - SSubmitRsp *sum = pJob->resData; - sum->affectedRows += rsp->affectedRows; - sum->nBlocks += rsp->nBlocks; - sum->pBlocks = taosMemoryRealloc(sum->pBlocks, sum->nBlocks * sizeof(*sum->pBlocks)); - memcpy(sum->pBlocks + sum->nBlocks - rsp->nBlocks, rsp->pBlocks, rsp->nBlocks * sizeof(*sum->pBlocks)); - taosMemoryFree(rsp->pBlocks); - taosMemoryFree(rsp); - } else { - pJob->resData = rsp; - } - SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + pJob->resType = SCH_RES_TYPE_QUERY; + SCH_LOCK(SCH_WRITE, &pJob->resLock); + if (pJob->resData) { + SSubmitRsp *sum = pJob->resData; + sum->affectedRows += rsp->affectedRows; + sum->nBlocks += rsp->nBlocks; + sum->pBlocks = taosMemoryRealloc(sum->pBlocks, sum->nBlocks * sizeof(*sum->pBlocks)); + memcpy(sum->pBlocks + sum->nBlocks - rsp->nBlocks, rsp->pBlocks, rsp->nBlocks * sizeof(*sum->pBlocks)); + taosMemoryFree(rsp->pBlocks); + taosMemoryFree(rsp); } else { - tFreeSSubmitRsp(rsp); + pJob->resData = rsp; } + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); } SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); @@ -2412,7 +2410,7 @@ void schFreeJobImpl(void *job) { } static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pDag, int64_t *job, const char *sql, - int64_t startTs, bool needRes, bool syncSchedule) { + int64_t startTs, bool syncSchedule) { qDebug("QID:0x%" PRIx64 " job started", pDag->queryId); if (pNodeList == NULL || taosArrayGetSize(pNodeList) <= 0) { @@ -2421,7 +2419,7 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pD int32_t code = 0; SSchJob *pJob = NULL; - SCH_ERR_JRET(schInitJob(&pJob, pDag, transport, pNodeList, sql, startTs, needRes, syncSchedule)); + SCH_ERR_JRET(schInitJob(&pJob, pDag, transport, pNodeList, sql, startTs, syncSchedule)); SCH_ERR_JRET(schLaunchJob(pJob)); @@ -2463,6 +2461,8 @@ int32_t schExecStaticExplain(void *transport, SArray *pNodeList, SQueryPlan *pDa SCH_ERR_JRET(qExecStaticExplain(pDag, (SRetrieveTableRsp **)&pJob->resData)); + pJob->resType = SCH_RES_TYPE_FETCH; + int64_t refId = taosAddRef(schMgmt.jobRef, pJob); if (refId < 0) { SCH_JOB_ELOG("taosAddRef job failed, error:%s", tstrerror(terrno)); @@ -2535,7 +2535,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { } int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, - int64_t startTs, bool needRes, SQueryResult *pRes) { + int64_t startTs, SQueryResult *pRes) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -2543,14 +2543,14 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, in if (EXPLAIN_MODE_STATIC == pDag->explainInfo.mode) { SCH_ERR_RET(schExecStaticExplain(transport, nodeList, pDag, pJob, sql, true)); } else { - SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, startTs, needRes, true)); + SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, startTs, true)); } SSchJob *job = schAcquireJob(*pJob); pRes->code = atomic_load_32(&job->errCode); pRes->numOfRows = job->resNumOfRows; - if (needRes) { + if (SCH_RES_TYPE_QUERY == job->resType) { pRes->res = job->resData; job->resData = NULL; } @@ -2568,7 +2568,7 @@ int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryPlan *pD if (EXPLAIN_MODE_STATIC == pDag->explainInfo.mode) { SCH_ERR_RET(schExecStaticExplain(transport, pNodeList, pDag, pJob, sql, false)); } else { - SCH_ERR_RET(schExecJobImpl(transport, pNodeList, pDag, pJob, sql, 0, false, false)); + SCH_ERR_RET(schExecJobImpl(transport, pNodeList, pDag, pJob, sql, 0, false)); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 09ecd9fffd..fc0e05aaf1 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -985,7 +985,7 @@ TEST(insertTest, normalCase) { taosThreadCreate(&(thread1), &thattr, schtSendRsp, &insertJobRefId); SQueryResult res = {0}; - code = schedulerExecJob(mockPointer, qnodeList, &dag, &insertJobRefId, "insert into tb values(now,1)", 0, false, &res); + code = schedulerExecJob(mockPointer, qnodeList, &dag, &insertJobRefId, "insert into tb values(now,1)", 0, &res); ASSERT_EQ(code, 0); ASSERT_EQ(res.numOfRows, 20); From 51721dfc477a8c300559300649e2ed11bafd9928 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Thu, 19 May 2022 10:37:11 +0800 Subject: [PATCH 08/28] fix(os): win run error --- cmake/cmake.install | 21 ++++--- include/os/osString.h | 18 ++++++ packaging/tools/make_install.bat | 6 ++ source/client/src/clientSml.c | 4 +- source/common/src/tname.c | 2 +- source/common/src/ttime.c | 4 +- source/common/src/tvariant.c | 8 +-- source/libs/index/src/indexComm.c | 8 +-- source/libs/parser/src/parAstCreater.c | 56 +++++++++--------- source/libs/parser/src/parInsert.c | 6 +- source/libs/parser/src/parTranslater.c | 22 +++---- source/libs/parser/src/sql.c | 2 +- source/libs/scalar/src/sclfunc.c | 8 +-- source/libs/scalar/src/sclvector.c | 22 +++---- source/os/src/osString.c | 79 ++++++++++++++++++++++++++ source/util/src/tjson.c | 4 +- tools/shell/src/shellEngine.c | 6 +- 17 files changed, 192 insertions(+), 84 deletions(-) create mode 100644 packaging/tools/make_install.bat diff --git a/cmake/cmake.install b/cmake/cmake.install index f51c6566fa..b2421fac25 100644 --- a/cmake/cmake.install +++ b/cmake/cmake.install @@ -5,22 +5,27 @@ IF (TD_LINUX) ELSEIF (TD_WINDOWS) SET(CMAKE_INSTALL_PREFIX C:/TDengine) - INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/go DESTINATION connector) - INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/nodejs DESTINATION connector) - INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/python DESTINATION connector) - INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/C\# DESTINATION connector) - INSTALL(DIRECTORY ${TD_SOURCE_DIR}/examples DESTINATION .) + # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/go DESTINATION connector) + # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/nodejs DESTINATION connector) + # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/python DESTINATION connector) + # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/C\# DESTINATION connector) + # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/examples DESTINATION .) INSTALL(FILES ${TD_SOURCE_DIR}/packaging/cfg/taos.cfg DESTINATION cfg) - INSTALL(FILES ${TD_SOURCE_DIR}/src/inc/taos.h DESTINATION include) - INSTALL(FILES ${TD_SOURCE_DIR}/src/inc/taoserror.h DESTINATION include) + INSTALL(FILES ${TD_SOURCE_DIR}/include/client/taos.h DESTINATION include) + INSTALL(FILES ${TD_SOURCE_DIR}/include/util/taoserror.h DESTINATION include) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos_static.lib DESTINATION driver) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.exp DESTINATION driver) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver) + INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taos.exe DESTINATION .) + INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taosd.exe DESTINATION .) + INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/udfd.exe DESTINATION .) IF (TD_MVN_INSTALLED) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.38-dist.jar DESTINATION connector/jdbc) ENDIF () + SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.bat") + INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") + INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} :needAdmin ${TD_SOURCE_DIR} ${PROJECT_BINARY_DIR} Windows ${TD_VER_NUMBER})") ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.sh") INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") diff --git a/include/os/osString.h b/include/os/osString.h index 5f65f97bec..1b518f9b81 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -38,6 +38,13 @@ typedef int32_t TdUcs4; #define wcsncpy WCSNCPY_FUNC_TAOS_FORBID #define wchar_t WCHAR_T_TYPE_TAOS_FORBID #define strcasestr STR_CASE_STR_FORBID + #define strtoll STR_TO_LL_FUNC_TAOS_FORBID + #define strtoull STR_TO_ULL_FUNC_TAOS_FORBID + #define strtol STR_TO_L_FUNC_TAOS_FORBID + #define strtoul STR_TO_UL_FUNC_TAOS_FORBID + #define strtod STR_TO_LD_FUNC_TAOS_FORBID + #define strtold STR_TO_D_FUNC_TAOS_FORBID + #define strtof STR_TO_F_FUNC_TAOS_FORBID #endif #ifdef WINDOWS @@ -72,6 +79,17 @@ int32_t taosWcharsToMbs(char *pStrs, TdWchar *pWchars, int32_t size); char *taosStrCaseStr(const char *str, const char *pattern); +int64_t taosStr2Int64(const char *str, char** pEnd, int32_t radix); +uint64_t taosStr2UInt64(const char *str, char** pEnd, int32_t radix); +int32_t taosStr2Int32(const char *str, char** pEnd, int32_t radix); +uint32_t taosStr2UInt32(const char *str, char** pEnd, int32_t radix); +int16_t taosStr2Int16(const char *str, char** pEnd, int32_t radix); +uint16_t taosStr2UInt16(const char *str, char** pEnd, int32_t radix); +int8_t taosStr2Int8(const char *str, char** pEnd, int32_t radix); +uint8_t taosStr2UInt8(const char *str, char** pEnd, int32_t radix); +double taosStr2Double(const char *str, char** pEnd); +float taosStr2Float(const char *str, char** pEnd); + #ifdef __cplusplus } #endif diff --git a/packaging/tools/make_install.bat b/packaging/tools/make_install.bat new file mode 100644 index 0000000000..64f30b8465 --- /dev/null +++ b/packaging/tools/make_install.bat @@ -0,0 +1,6 @@ +@echo off +goto %1 +:needAdmin +mshta vbscript:createobject("shell.application").shellexecute("%~s0",":hasAdmin","","runas",1)(window.close)&goto :eof +:hasAdmin +cp -f C:\\TDengine\\driver\\taos.dll C:\\Windows\\System32 \ No newline at end of file diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index f6f8db8590..d5377c99a6 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -580,7 +580,7 @@ static bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg){ const char *pVal = kvVal->value; int32_t len = kvVal->length; char *endptr = NULL; - double result = strtod(pVal, &endptr); + double result = taosStr2Double(pVal, &endptr); if(pVal == endptr){ smlBuildInvalidDataMsg(msg, "invalid data", pVal); return false; @@ -714,7 +714,7 @@ static bool smlIsNchar(const char *pVal, uint16_t len) { static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { char *endPtr = NULL; - int64_t tsInt64 = strtoll(value, &endPtr, 10); + int64_t tsInt64 = taosStr2Int64(value, &endPtr, 10); if(value + len != endPtr){ return -1; } diff --git a/source/common/src/tname.c b/source/common/src/tname.c index fa9b6e1e63..0764ea84b9 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -250,7 +250,7 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { return -1; } - dst->acctId = strtoll(str, NULL, 10); + dst->acctId = taosStr2Int32(str, NULL, 10); } if ((type & T_NAME_DB) == T_NAME_DB) { diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 62a7179626..69ba964187 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -590,7 +590,7 @@ int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* dura char* endPtr = NULL; /* get the basic numeric value */ - int64_t timestamp = strtoll(token, &endPtr, 10); + int64_t timestamp = taosStr2Int64(token, &endPtr, 10); if (errno != 0) { return -1; } @@ -608,7 +608,7 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati errno = 0; /* get the basic numeric value */ - *duration = strtoll(token, NULL, 10); + *duration = taosStr2Int64(token, NULL, 10); if (errno != 0) { return -1; } diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index e44450fe6e..7b0bef4918 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -39,7 +39,7 @@ int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value) { errno = 0; char *endPtr = NULL; - *value = strtoll(z, &endPtr, base); + *value = taosStr2Int64(z, &endPtr, base); if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { errno = 0; return -1; @@ -58,7 +58,7 @@ int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value) { return -1; } - *value = strtoull(z, &endPtr, base); + *value = taosStr2UInt64(z, &endPtr, base); if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { errno = 0; return -1; @@ -434,7 +434,7 @@ static FORCE_INLINE int32_t convertToDouble(char *pStr, int32_t len, double *val // return -1; // } // - // *value = strtod(pStr, NULL); + // *value = taosStr2Double(pStr, NULL); return 0; } @@ -911,7 +911,7 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) { case TSDB_DATA_TYPE_DOUBLE: { if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { errno = 0; - double v = strtod(pVariant->pz, NULL); + double v = taosStr2Double(pVariant->pz, NULL); if ((errno == ERANGE && v == -1) || (isinf(v) || isnan(v))) { taosMemoryFree(pVariant->pz); return -1; diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 802cdf7aad..4c23e4ba4b 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -118,21 +118,21 @@ TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t d } return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_FLOAT) { - float va = strtod(a, NULL); + float va = taosStr2Float(a, NULL); if (errno == ERANGE && va == -1) { return CONTINUE; } - float vb = strtod(b, NULL); + float vb = taosStr2Float(b, NULL); if (errno == ERANGE && va == -1) { return CONTINUE; } return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_DOUBLE) { - double va = strtod(a, NULL); + double va = taosStr2Double(a, NULL); if (errno == ERANGE && va == -1) { return CONTINUE; } - double vb = strtod(b, NULL); + double vb = taosStr2Double(b, NULL); if (errno == ERANGE && va == -1) { return CONTINUE; } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index a438a1e843..894659ddd1 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -123,7 +123,7 @@ static bool checkAndSplitEndpoint(SAstCreateContext* pCxt, const SToken* pEp, ch pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ENDPOINT); } else { strncpy(pFqdn, ep, pColon - ep); - *pPort = strtol(pColon + 1, NULL, 10); + *pPort = taosStr2Int32(pColon + 1, NULL, 10); if (*pPort >= UINT16_MAX || *pPort <= 0) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_PORT); } @@ -147,7 +147,7 @@ static bool checkPort(SAstCreateContext* pCxt, const SToken* pPortToken, int32_t if (NULL == pPortToken) { pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; } else { - *pPort = strtol(pPortToken->z, NULL, 10); + *pPort = taosStr2Int32(pPortToken->z, NULL, 10); if (*pPort >= UINT16_MAX || *pPort <= 0) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_PORT); } @@ -476,9 +476,9 @@ SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset) { SLimitNode* limitNode = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT); CHECK_OUT_OF_MEM(limitNode); - limitNode->limit = strtol(pLimit->z, NULL, 10); + limitNode->limit = taosStr2Int64(pLimit->z, NULL, 10); if (NULL != pOffset) { - limitNode->offset = strtol(pOffset->z, NULL, 10); + limitNode->offset = taosStr2Int64(pOffset->z, NULL, 10); } return (SNode*)limitNode; } @@ -694,59 +694,59 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) { SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, void* pVal) { switch (type) { case DB_OPTION_BUFFER: - ((SDatabaseOptions*)pOptions)->buffer = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->buffer = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_CACHELAST: - ((SDatabaseOptions*)pOptions)->cachelast = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->cachelast = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_COMP: - ((SDatabaseOptions*)pOptions)->compressionLevel = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->compressionLevel = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_DAYS: { SToken* pToken = pVal; if (TK_NK_INTEGER == pToken->type) { - ((SDatabaseOptions*)pOptions)->daysPerFile = strtol(pToken->z, NULL, 10) * 1440; + ((SDatabaseOptions*)pOptions)->daysPerFile = taosStr2Int32(pToken->z, NULL, 10) * 1440; } else { ((SDatabaseOptions*)pOptions)->pDaysPerFile = (SValueNode*)createDurationValueNode(pCxt, pToken); } break; } case DB_OPTION_FSYNC: - ((SDatabaseOptions*)pOptions)->fsyncPeriod = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->fsyncPeriod = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_MAXROWS: - ((SDatabaseOptions*)pOptions)->maxRowsPerBlock = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->maxRowsPerBlock = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_MINROWS: - ((SDatabaseOptions*)pOptions)->minRowsPerBlock = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->minRowsPerBlock = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_KEEP: ((SDatabaseOptions*)pOptions)->pKeep = pVal; break; case DB_OPTION_PAGES: - ((SDatabaseOptions*)pOptions)->pages = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->pages = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_PAGESIZE: - ((SDatabaseOptions*)pOptions)->pagesize = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->pagesize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_PRECISION: copyStringFormStringToken((SToken*)pVal, ((SDatabaseOptions*)pOptions)->precisionStr, sizeof(((SDatabaseOptions*)pOptions)->precisionStr)); break; case DB_OPTION_REPLICA: - ((SDatabaseOptions*)pOptions)->replica = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->replica = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_STRICT: - ((SDatabaseOptions*)pOptions)->strict = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->strict = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_WAL: - ((SDatabaseOptions*)pOptions)->walLevel = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->walLevel = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_VGROUPS: - ((SDatabaseOptions*)pOptions)->numOfVgroups = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->numOfVgroups = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_SINGLE_STABLE: - ((SDatabaseOptions*)pOptions)->singleStable = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->singleStable = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_RETENTIONS: ((SDatabaseOptions*)pOptions)->pRetentions = pVal; @@ -827,16 +827,16 @@ SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType sizeof(((STableOptions*)pOptions)->comment)); break; case TABLE_OPTION_DELAY: - ((STableOptions*)pOptions)->delay = strtol(((SToken*)pVal)->z, NULL, 10); + ((STableOptions*)pOptions)->delay = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case TABLE_OPTION_FILE_FACTOR: - ((STableOptions*)pOptions)->filesFactor = strtod(((SToken*)pVal)->z, NULL); + ((STableOptions*)pOptions)->filesFactor = taosStr2Float(((SToken*)pVal)->z, NULL); break; case TABLE_OPTION_ROLLUP: ((STableOptions*)pOptions)->pRollupFuncs = pVal; break; case TABLE_OPTION_TTL: - ((STableOptions*)pOptions)->ttl = strtol(((SToken*)pVal)->z, NULL, 10); + ((STableOptions*)pOptions)->ttl = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case TABLE_OPTION_SMA: ((STableOptions*)pOptions)->pSma = pVal; @@ -868,7 +868,7 @@ SDataType createDataType(uint8_t type) { } SDataType createVarLenDataType(uint8_t type, const SToken* pLen) { - SDataType dt = {.type = type, .precision = 0, .scale = 0, .bytes = strtol(pLen->z, NULL, 10)}; + SDataType dt = {.type = type, .precision = 0, .scale = 0, .bytes = taosStr2Int16(pLen->z, NULL, 10)}; return dt; } @@ -1119,7 +1119,7 @@ SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode) { SDropDnodeStmt* pStmt = (SDropDnodeStmt*)nodesMakeNode(QUERY_NODE_DROP_DNODE_STMT); CHECK_OUT_OF_MEM(pStmt); if (TK_NK_INTEGER == pDnode->type) { - pStmt->dnodeId = strtol(pDnode->z, NULL, 10); + pStmt->dnodeId = taosStr2Int32(pDnode->z, NULL, 10); } else { if (!checkAndSplitEndpoint(pCxt, pDnode, pStmt->fqdn, &pStmt->port)) { nodesDestroyNode(pStmt); @@ -1133,7 +1133,7 @@ SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const const SToken* pValue) { SAlterDnodeStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_DNODE_STMT); CHECK_OUT_OF_MEM(pStmt); - pStmt->dnodeId = strtol(pDnode->z, NULL, 10); + pStmt->dnodeId = taosStr2Int32(pDnode->z, NULL, 10); trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config)); if (NULL != pValue) { trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value)); @@ -1183,7 +1183,7 @@ SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken SNode* createCreateComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId) { SCreateComponentNodeStmt* pStmt = nodesMakeNode(type); CHECK_OUT_OF_MEM(pStmt); - pStmt->dnodeId = strtol(pDnodeId->z, NULL, 10); + pStmt->dnodeId = taosStr2Int32(pDnodeId->z, NULL, 10); ; return (SNode*)pStmt; } @@ -1191,7 +1191,7 @@ SNode* createCreateComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, co SNode* createDropComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId) { SDropComponentNodeStmt* pStmt = nodesMakeNode(type); CHECK_OUT_OF_MEM(pStmt); - pStmt->dnodeId = strtol(pDnodeId->z, NULL, 10); + pStmt->dnodeId = taosStr2Int32(pDnodeId->z, NULL, 10); ; return (SNode*)pStmt; } @@ -1251,7 +1251,7 @@ SNode* setExplainVerbose(SAstCreateContext* pCxt, SNode* pOptions, const SToken* } SNode* setExplainRatio(SAstCreateContext* pCxt, SNode* pOptions, const SToken* pVal) { - ((SExplainOptions*)pOptions)->ratio = strtod(pVal->z, NULL); + ((SExplainOptions*)pOptions)->ratio = taosStr2Double(pVal->z, NULL); return pOptions; } @@ -1347,7 +1347,7 @@ SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId) { SKillStmt* pStmt = nodesMakeNode(type); CHECK_OUT_OF_MEM(pStmt); - pStmt->targetId = strtol(pId->z, NULL, 10); + pStmt->targetId = taosStr2Int32(pId->z, NULL, 10); return (SNode*)pStmt; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 29951ccdee..34ae13580b 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -442,7 +442,7 @@ static bool isNullStr(SToken* pToken) { static FORCE_INLINE int32_t toDouble(SToken* pToken, double* value, char** endPtr) { errno = 0; - *value = strtold(pToken->z, endPtr); + *value = taosStr2Double(pToken->z, endPtr); // not a valid integer number, return error if ((*endPtr - pToken->z) != pToken->n) { @@ -482,9 +482,9 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); } } else if (pToken->type == TK_NK_INTEGER) { - return func(pMsgBuf, ((strtoll(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); + return func(pMsgBuf, ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); } else if (pToken->type == TK_NK_FLOAT) { - return func(pMsgBuf, ((strtod(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); + return func(pMsgBuf, ((taosStr2Double(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); } else { return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 18e0843e92..0d214fd85e 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -498,7 +498,7 @@ static int32_t parseTimeFromValueNode(SValueNode* pVal) { return TSDB_CODE_SUCCESS; } char* pEnd = NULL; - pVal->datum.i = strtoll(pVal->literal, &pEnd, 10); + pVal->datum.i = taosStr2Int64(pVal->literal, &pEnd, 10); return (NULL != pEnd && '\0' == *pEnd) ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED; } else { return TSDB_CODE_FAILED; @@ -527,61 +527,61 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD break; case TSDB_DATA_TYPE_TINYINT: { char* endPtr = NULL; - pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); + pVal->datum.i = taosStr2Int64(pVal->literal, &endPtr, 10); *(int8_t*)&pVal->typeData = pVal->datum.i; break; } case TSDB_DATA_TYPE_SMALLINT: { char* endPtr = NULL; - pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); + pVal->datum.i = taosStr2Int64(pVal->literal, &endPtr, 10); *(int16_t*)&pVal->typeData = pVal->datum.i; break; } case TSDB_DATA_TYPE_INT: { char* endPtr = NULL; - pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); + pVal->datum.i = taosStr2Int64(pVal->literal, &endPtr, 10); *(int32_t*)&pVal->typeData = pVal->datum.i; break; } case TSDB_DATA_TYPE_BIGINT: { char* endPtr = NULL; - pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); + pVal->datum.i = taosStr2Int64(pVal->literal, &endPtr, 10); *(int64_t*)&pVal->typeData = pVal->datum.i; break; } case TSDB_DATA_TYPE_UTINYINT: { char* endPtr = NULL; - pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); + pVal->datum.u = taosStr2UInt64(pVal->literal, &endPtr, 10); *(uint8_t*)&pVal->typeData = pVal->datum.u; break; } case TSDB_DATA_TYPE_USMALLINT: { char* endPtr = NULL; - pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); + pVal->datum.u = taosStr2UInt64(pVal->literal, &endPtr, 10); *(uint16_t*)&pVal->typeData = pVal->datum.u; break; } case TSDB_DATA_TYPE_UINT: { char* endPtr = NULL; - pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); + pVal->datum.u = taosStr2UInt64(pVal->literal, &endPtr, 10); *(uint32_t*)&pVal->typeData = pVal->datum.u; break; } case TSDB_DATA_TYPE_UBIGINT: { char* endPtr = NULL; - pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); + pVal->datum.u = taosStr2UInt64(pVal->literal, &endPtr, 10); *(uint64_t*)&pVal->typeData = pVal->datum.u; break; } case TSDB_DATA_TYPE_FLOAT: { char* endPtr = NULL; - pVal->datum.d = strtold(pVal->literal, &endPtr); + pVal->datum.d = taosStr2Double(pVal->literal, &endPtr); *(float*)&pVal->typeData = pVal->datum.d; break; } case TSDB_DATA_TYPE_DOUBLE: { char* endPtr = NULL; - pVal->datum.d = strtold(pVal->literal, &endPtr); + pVal->datum.d = taosStr2Double(pVal->literal, &endPtr); *(double*)&pVal->typeData = pVal->datum.d; break; } diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 795b5ca641..2d844e1842 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -3863,7 +3863,7 @@ static YYACTIONTYPE yy_reduce( { yymsp[1].minor.yy140 = 0; } break; case 245: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ -{ yymsp[-1].minor.yy140 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } +{ yymsp[-1].minor.yy140 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } break; case 246: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression */ { pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-5].minor.yy617, &yymsp[-4].minor.yy105, yymsp[-2].minor.yy172, yymsp[-3].minor.yy172, yymsp[0].minor.yy172); } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index d4a88622e2..45742189d5 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -724,7 +724,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp case TSDB_DATA_TYPE_BIGINT: { if (inputType == TSDB_DATA_TYPE_BINARY) { memcpy(output, varDataVal(input), varDataLen(input)); - *(int64_t *)output = strtoll(output, NULL, 10); + *(int64_t *)output = taosStr2Int64(output, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { char *newBuf = taosMemoryCalloc(1, outputLen * TSDB_NCHAR_SIZE + 1); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); @@ -733,7 +733,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp return TSDB_CODE_FAILED; } newBuf[len] = 0; - *(int64_t *)output = strtoll(newBuf, NULL, 10); + *(int64_t *)output = taosStr2Int64(newBuf, NULL, 10); taosMemoryFree(newBuf); } else { GET_TYPED_DATA(*(int64_t *)output, int64_t, inputType, input); @@ -743,7 +743,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp case TSDB_DATA_TYPE_UBIGINT: { if (inputType == TSDB_DATA_TYPE_BINARY) { memcpy(output, varDataVal(input), varDataLen(input)); - *(uint64_t *)output = strtoull(output, NULL, 10); + *(uint64_t *)output = taosStr2UInt64(output, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { char *newBuf = taosMemoryCalloc(1, outputLen * TSDB_NCHAR_SIZE + 1); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); @@ -752,7 +752,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp return TSDB_CODE_FAILED; } newBuf[len] = 0; - *(uint64_t *)output = strtoull(newBuf, NULL, 10); + *(uint64_t *)output = taosStr2UInt64(newBuf, NULL, 10); taosMemoryFree(newBuf); } else { GET_TYPED_DATA(*(uint64_t *)output, uint64_t, inputType, input); diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index c9fcaeb32e..19453bf760 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -92,7 +92,7 @@ void convertStringToDouble(const void *inData, void *outData, int8_t inType, int tmp[len] = 0; ASSERT(outType == TSDB_DATA_TYPE_DOUBLE); - double value = strtod(tmp, NULL); + double value = taosStr2Double(tmp, NULL); *((double *)outData) = value; taosMemoryFreeClear(tmp); @@ -267,22 +267,22 @@ static FORCE_INLINE void varToTimestamp(char *buf, SScalarParam* pOut, int32_t r static FORCE_INLINE void varToSigned(char *buf, SScalarParam* pOut, int32_t rowIndex) { switch (pOut->columnData->info.type) { case TSDB_DATA_TYPE_TINYINT: { - int8_t value = (int8_t)strtoll(buf, NULL, 10); + int8_t value = (int8_t)taosStr2Int8(buf, NULL, 10); colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*)&value); break; } case TSDB_DATA_TYPE_SMALLINT: { - int16_t value = (int16_t)strtoll(buf, NULL, 10); + int16_t value = (int16_t)taosStr2Int16(buf, NULL, 10); colDataAppendInt16(pOut->columnData, rowIndex, (int16_t*)&value); break; } case TSDB_DATA_TYPE_INT: { - int32_t value = (int32_t)strtoll(buf, NULL, 10); + int32_t value = (int32_t)taosStr2Int32(buf, NULL, 10); colDataAppendInt32(pOut->columnData, rowIndex, (int32_t*)&value); break; } case TSDB_DATA_TYPE_BIGINT: { - int64_t value = (int64_t)strtoll(buf, NULL, 10); + int64_t value = (int64_t)taosStr2Int64(buf, NULL, 10); colDataAppendInt64(pOut->columnData, rowIndex, (int64_t*)&value); break; } @@ -292,22 +292,22 @@ static FORCE_INLINE void varToSigned(char *buf, SScalarParam* pOut, int32_t rowI static FORCE_INLINE void varToUnsigned(char *buf, SScalarParam* pOut, int32_t rowIndex) { switch (pOut->columnData->info.type) { case TSDB_DATA_TYPE_UTINYINT: { - uint8_t value = (uint8_t)strtoull(buf, NULL, 10); + uint8_t value = (uint8_t)taosStr2UInt8(buf, NULL, 10); colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*)&value); break; } case TSDB_DATA_TYPE_USMALLINT: { - uint16_t value = (uint16_t)strtoull(buf, NULL, 10); + uint16_t value = (uint16_t)taosStr2UInt16(buf, NULL, 10); colDataAppendInt16(pOut->columnData, rowIndex, (int16_t*)&value); break; } case TSDB_DATA_TYPE_UINT: { - uint32_t value = (uint32_t)strtoull(buf, NULL, 10); + uint32_t value = (uint32_t)taosStr2UInt32(buf, NULL, 10); colDataAppendInt32(pOut->columnData, rowIndex, (int32_t*)&value); break; } case TSDB_DATA_TYPE_UBIGINT: { - uint64_t value = (uint64_t)strtoull(buf, NULL, 10); + uint64_t value = (uint64_t)taosStr2UInt64(buf, NULL, 10); colDataAppendInt64(pOut->columnData, rowIndex, (int64_t*)&value); break; } @@ -315,12 +315,12 @@ static FORCE_INLINE void varToUnsigned(char *buf, SScalarParam* pOut, int32_t ro } static FORCE_INLINE void varToFloat(char *buf, SScalarParam* pOut, int32_t rowIndex) { - double value = strtod(buf, NULL); + double value = taosStr2Double(buf, NULL); colDataAppendDouble(pOut->columnData, rowIndex, &value); } static FORCE_INLINE void varToBool(char *buf, SScalarParam* pOut, int32_t rowIndex) { - int64_t value = strtoll(buf, NULL, 10); + int64_t value = taosStr2Int64(buf, NULL, 10); bool v = (value != 0)? true:false; colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*) &v); } diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 7dbd301913..d5762ef87d 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -254,4 +254,83 @@ char *taosStrCaseStr(const char *str, const char *pattern) { } } return NULL; +} + +int64_t taosStr2Int64(const char *str, char** pEnd, int32_t radix) { + int64_t tmp = strtoll(str, pEnd, radix); + assert(errno != ERANGE); + assert(errno != EINVAL); + return tmp; +} + +uint64_t taosStr2UInt64(const char *str, char** pEnd, int32_t radix) { + uint64_t tmp = strtoull(str, pEnd, radix); + assert(errno != ERANGE); + assert(errno != EINVAL); + return tmp; +} + +int32_t taosStr2Int32(const char *str, char** pEnd, int32_t radix) { + int32_t tmp = strtol(str, pEnd, radix); + assert(errno != ERANGE); + assert(errno != EINVAL); + return tmp; +} + +uint32_t taosStr2UInt32(const char *str, char** pEnd, int32_t radix) { + uint32_t tmp = strtol(str, pEnd, radix); + assert(errno != ERANGE); + assert(errno != EINVAL); + return tmp; +} + +int16_t taosStr2Int16(const char *str, char** pEnd, int32_t radix) { + int32_t tmp = strtol(str, pEnd, radix); + assert(errno != ERANGE); + assert(errno != EINVAL); + assert(tmp >= SHRT_MIN); + assert(tmp <= SHRT_MAX); + return (int16_t)tmp; +} + +uint16_t taosStr2UInt16(const char *str, char** pEnd, int32_t radix) { + uint32_t tmp = strtoul(str, pEnd, radix); + assert(errno != ERANGE); + assert(errno != EINVAL); + assert(tmp <= USHRT_MAX); + return (uint16_t)tmp; +} + +int8_t taosStr2Int8(const char *str, char** pEnd, int32_t radix) { + int32_t tmp = strtol(str, pEnd, radix); + assert(errno != ERANGE); + assert(errno != EINVAL); + assert(tmp >= SCHAR_MIN); + assert(tmp <= SCHAR_MAX); + return tmp; +} + +uint8_t taosStr2UInt8(const char *str, char** pEnd, int32_t radix) { + uint32_t tmp = strtoul(str, pEnd, radix); + assert(errno != ERANGE); + assert(errno != EINVAL); + assert(tmp <= UCHAR_MAX); + return tmp; +} + +double taosStr2Double(const char *str, char** pEnd) { + double tmp = strtod(str, pEnd); + assert(errno != ERANGE); + assert(errno != EINVAL); + assert(tmp != HUGE_VAL); + return tmp; +} + +float taosStr2Float(const char *str, char** pEnd) { + float tmp = strtof(str, pEnd); + assert(errno != ERANGE); + assert(errno != EINVAL); + assert(tmp != HUGE_VALF); + assert(tmp != NAN); + return tmp; } \ No newline at end of file diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index c5cd968fd9..b15c188f04 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -187,7 +187,7 @@ int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal sscanf(p,"%lld",pVal); #else // sscanf(p,"%ld",pVal); - *pVal = strtol(p, NULL, 10); + *pVal = taosStr2Int64(p, NULL, 10); #endif return TSDB_CODE_SUCCESS; } @@ -222,7 +222,7 @@ int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pV sscanf(p,"%llu",pVal); #else // sscanf(p,"%ld",pVal); - *pVal = strtoul(p, NULL, 10); + *pVal = taosStr2UInt64(p, NULL, 10); #endif return TSDB_CODE_SUCCESS; } diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 9f9c8821b0..21474c316f 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -195,16 +195,16 @@ void shellRunSingleCommandImp(char *command) { et = taosGetTimestampUs(); if (error_no == 0) { - printf("Query OK, %d row(s) in set (%.6fs)\n", numOfRows, (et - st) / 1E6); + printf("Query OK, %d rows affected (%.6fs)\n", numOfRows, (et - st) / 1E6); } else { - printf("Query interrupted (%s), %d row(s) in set (%.6fs)\n", taos_errstr(pSql), numOfRows, (et - st) / 1E6); + printf("Query interrupted (%s), %d rows affected (%.6fs)\n", taos_errstr(pSql), numOfRows, (et - st) / 1E6); } taos_free_result(pSql); } else { int32_t num_rows_affacted = taos_affected_rows(pSql); taos_free_result(pSql); et = taosGetTimestampUs(); - printf("Query OK, %d of %d row(s) in database (%.6fs)\n", num_rows_affacted, num_rows_affacted, (et - st) / 1E6); + printf("Query OK, %d of %d rows affected (%.6fs)\n", num_rows_affacted, num_rows_affacted, (et - st) / 1E6); } printf("\n"); From a41dc27e6df6b2e143d2bb60431ff5a93fdbddbf Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 19 May 2022 11:10:13 +0800 Subject: [PATCH 09/28] refactor: dnode monitor --- source/dnode/mgmt/mgmt_bnode/src/bmHandle.c | 2 +- source/dnode/mgmt/mgmt_dnode/inc/dmInt.h | 9 +- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 8 +- source/dnode/mgmt/mgmt_dnode/src/dmInt.c | 4 +- source/dnode/mgmt/mgmt_dnode/src/dmMonitor.c | 104 ----------- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 2 +- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 14 +- source/dnode/mgmt/mgmt_qnode/src/qmHandle.c | 2 +- source/dnode/mgmt/mgmt_snode/src/smHandle.c | 2 +- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 4 +- source/dnode/mgmt/node_mgmt/inc/dmMgmt.h | 17 +- source/dnode/mgmt/node_mgmt/inc/dmNodes.h | 45 +++++ source/dnode/mgmt/node_mgmt/src/dmEnv.c | 9 +- source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 1 + source/dnode/mgmt/node_mgmt/src/dmMonitor.c | 172 +++++++++++++++++++ source/dnode/mgmt/node_util/inc/dmUtil.h | 32 ++-- 16 files changed, 269 insertions(+), 158 deletions(-) delete mode 100644 source/dnode/mgmt/mgmt_dnode/src/dmMonitor.c create mode 100644 source/dnode/mgmt/node_mgmt/inc/dmNodes.h create mode 100644 source/dnode/mgmt/node_mgmt/src/dmMonitor.c diff --git a/source/dnode/mgmt/mgmt_bnode/src/bmHandle.c b/source/dnode/mgmt/mgmt_bnode/src/bmHandle.c index 2637e0af04..d2e547078e 100644 --- a/source/dnode/mgmt/mgmt_bnode/src/bmHandle.c +++ b/source/dnode/mgmt/mgmt_bnode/src/bmHandle.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "bmInt.h" -static void bmGetMonitorInfo(SBnodeMgmt *pMgmt, SMonBmInfo *bmInfo) {} +void bmGetMonitorInfo(SBnodeMgmt *pMgmt, SMonBmInfo *bmInfo) {} int32_t bmProcessGetMonBmInfoReq(SBnodeMgmt *pMgmt, SRpcMsg *pReq) { SMonBmInfo bmInfo = {0}; diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index f90fd72c6b..ae8879326d 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -32,7 +32,9 @@ typedef struct SDnodeMgmt { SSingleWorker mgmtWorker; ProcessCreateNodeFp processCreateNodeFp; ProcessDropNodeFp processDropNodeFp; - IsNodeRequiredFp isNodeRequiredFp; + SendMonitorReportFp sendMonitorReportFp; + GetVnodeLoadsFp getVnodeLoadsFp; + GetMnodeLoadsFp getMnodeLoadsFp; } SDnodeMgmt; // dmHandle.c @@ -43,11 +45,6 @@ int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); -// dmMonitor.c -void dmGetVnodeLoads(SDnodeMgmt *pMgmt, SMonVloadInfo *pInfo); -void dmGetMnodeLoads(SDnodeMgmt *pMgmt, SMonMloadInfo *pInfo); -void dmSendMonitorReport(SDnodeMgmt *pMgmt); - // dmWorker.c int32_t dmPutNodeMsgToMgmtQueue(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmStartStatusThread(SDnodeMgmt *pMgmt); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 38e71754b6..7c1162ec10 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -72,11 +72,11 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { taosRUnLockLatch(&pMgmt->pData->latch); SMonVloadInfo vinfo = {0}; - dmGetVnodeLoads(pMgmt, &vinfo); + (*pMgmt->getVnodeLoadsFp)(&vinfo); req.pVloads = vinfo.pVloads; SMonMloadInfo minfo = {0}; - dmGetMnodeLoads(pMgmt, &minfo); + (*pMgmt->getMnodeLoadsFp)(&minfo); int32_t contLen = tSerializeSStatusReq(NULL, 0, &req); void *pHead = rpcMallocCont(contLen); @@ -115,7 +115,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { SServerStatusRsp statusRsp = {0}; SMonMloadInfo minfo = {0}; - dmGetMnodeLoads(pMgmt, &minfo); + (*pMgmt->getMnodeLoadsFp)(&minfo); if (minfo.isMnode && minfo.load.syncState == TAOS_SYNC_STATE_ERROR) { pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED; snprintf(pStatus->details, sizeof(pStatus->details), "mnode sync state is %s", syncStr(minfo.load.syncState)); @@ -123,7 +123,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { } SMonVloadInfo vinfo = {0}; - dmGetVnodeLoads(pMgmt, &vinfo); + (*pMgmt->getVnodeLoadsFp)(&vinfo); for (int32_t i = 0; i < taosArrayGetSize(vinfo.pVloads); ++i) { SVnodeLoad *pLoad = taosArrayGet(vinfo.pVloads, i); if (pLoad->syncState == TAOS_SYNC_STATE_ERROR) { diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index 3b343d4916..59c926545e 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -45,7 +45,9 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->name = pInput->name; pMgmt->processCreateNodeFp = pInput->processCreateNodeFp; pMgmt->processDropNodeFp = pInput->processDropNodeFp; - pMgmt->isNodeRequiredFp = pInput->isNodeRequiredFp; + pMgmt->sendMonitorReportFp = pInput->sendMonitorReportFp; + pMgmt->getVnodeLoadsFp = pInput->getVnodeLoadsFp; + pMgmt->getMnodeLoadsFp = pInput->getMnodeLoadsFp; if (dmStartWorker(pMgmt) != 0) { return -1; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmMonitor.c b/source/dnode/mgmt/mgmt_dnode/src/dmMonitor.c deleted file mode 100644 index 3547c76937..0000000000 --- a/source/dnode/mgmt/mgmt_dnode/src/dmMonitor.c +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dmInt.h" - -#define dmSendLocalRecv(pMgmt, mtype, func, pInfo) \ - if (!tsMultiProcess) { \ - SRpcMsg rsp = {0}; \ - SRpcMsg req = {.msgType = mtype}; \ - SEpSet epset = {.inUse = 0, .numOfEps = 1}; \ - tstrncpy(epset.eps[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN); \ - epset.eps[0].port = tsServerPort; \ - rpcSendRecv(pMgmt->msgCb.clientRpc, &epset, &req, &rsp); \ - if (rsp.code == 0 && rsp.contLen > 0) { \ - func(rsp.pCont, rsp.contLen, pInfo); \ - } \ - rpcFreeCont(rsp.pCont); \ - } - -static void dmGetMonitorBasicInfo(SDnodeMgmt *pMgmt, SMonBasicInfo *pInfo) { - pInfo->protocol = 1; - pInfo->dnode_id = pMgmt->pData->dnodeId; - pInfo->cluster_id = pMgmt->pData->clusterId; - tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN); -} - -static void dmGetMonitorDnodeInfo(SDnodeMgmt *pMgmt, SMonDnodeInfo *pInfo) { - pInfo->uptime = (taosGetTimestampMs() - pMgmt->pData->rebootTime) / (86400000.0f); - pInfo->has_mnode = (*pMgmt->isNodeRequiredFp)(MNODE); - pInfo->has_qnode = (*pMgmt->isNodeRequiredFp)(QNODE); - pInfo->has_snode = (*pMgmt->isNodeRequiredFp)(SNODE); - pInfo->has_bnode = (*pMgmt->isNodeRequiredFp)(BNODE); - tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name)); - pInfo->logdir.size = tsLogSpace.size; - tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name)); - pInfo->tempdir.size = tsTempSpace.size; -} - -static void dmGetMonitorInfo(SDnodeMgmt *pMgmt, SMonDmInfo *pInfo) { - dmGetMonitorBasicInfo(pMgmt, &pInfo->basic); - dmGetMonitorDnodeInfo(pMgmt, &pInfo->dnode); - dmGetMonitorSystemInfo(&pInfo->sys); -} - -void dmSendMonitorReport(SDnodeMgmt *pMgmt) { - if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; - dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort); - - SMonDmInfo dmInfo = {0}; - SMonMmInfo mmInfo = {0}; - SMonVmInfo vmInfo = {0}; - SMonQmInfo qmInfo = {0}; - SMonSmInfo smInfo = {0}; - SMonBmInfo bmInfo = {0}; - - dmGetMonitorInfo(pMgmt, &dmInfo); - dmSendLocalRecv(pMgmt, TDMT_MON_VM_INFO, tDeserializeSMonVmInfo, &vmInfo); - if (dmInfo.dnode.has_mnode) { - dmSendLocalRecv(pMgmt, TDMT_MON_MM_INFO, tDeserializeSMonMmInfo, &mmInfo); - } - if (dmInfo.dnode.has_qnode) { - dmSendLocalRecv(pMgmt, TDMT_MON_QM_INFO, tDeserializeSMonQmInfo, &qmInfo); - } - if (dmInfo.dnode.has_snode) { - dmSendLocalRecv(pMgmt, TDMT_MON_SM_INFO, tDeserializeSMonSmInfo, &smInfo); - } - if (dmInfo.dnode.has_bnode) { - dmSendLocalRecv(pMgmt, TDMT_MON_BM_INFO, tDeserializeSMonBmInfo, &bmInfo); - } - - monSetDmInfo(&dmInfo); - monSetMmInfo(&mmInfo); - monSetVmInfo(&vmInfo); - monSetQmInfo(&qmInfo); - monSetSmInfo(&smInfo); - monSetBmInfo(&bmInfo); - tFreeSMonMmInfo(&mmInfo); - tFreeSMonVmInfo(&vmInfo); - tFreeSMonQmInfo(&qmInfo); - tFreeSMonSmInfo(&smInfo); - tFreeSMonBmInfo(&bmInfo); - monSendReport(); -} - -void dmGetVnodeLoads(SDnodeMgmt *pMgmt, SMonVloadInfo *pInfo) { - dmSendLocalRecv(pMgmt, TDMT_MON_VM_LOAD, tDeserializeSMonVloadInfo, pInfo); -} - -void dmGetMnodeLoads(SDnodeMgmt *pMgmt, SMonMloadInfo *pInfo) { - dmSendLocalRecv(pMgmt, TDMT_MON_MM_LOAD, tDeserializeSMonMloadInfo, pInfo); -} diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 599fa07e1c..6a7e0ad322 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -50,7 +50,7 @@ static void *dmMonitorThreadFp(void *param) { int64_t curTime = taosGetTimestampMs(); float interval = (curTime - lastTime) / 1000.0f; if (interval >= tsMonitorInterval) { - dmSendMonitorReport(pMgmt); + (*pMgmt->sendMonitorReportFp)(); lastTime = curTime; } } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index a09fd2627e..8b2d8f8bad 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -16,8 +16,13 @@ #define _DEFAULT_SOURCE #include "mmInt.h" -static void mmGetMonitorInfo(SMnodeMgmt *pMgmt, SMonMmInfo *mmInfo) { - mndGetMonitorInfo(pMgmt->pMnode, &mmInfo->cluster, &mmInfo->vgroup, &mmInfo->grant); +void mmGetMonitorInfo(SMnodeMgmt *pMgmt, SMonMmInfo *pInfo) { + mndGetMonitorInfo(pMgmt->pMnode, &pInfo->cluster, &pInfo->vgroup, &pInfo->grant); +} + +void mmGetMnodeLoads(SMnodeMgmt *pMgmt, SMonMloadInfo *pInfo) { + pInfo->isMnode = 1; + mndGetLoad(pMgmt->pMnode, &pInfo->load); } int32_t mmProcessGetMonitorInfoReq(SMnodeMgmt *pMgmt, SRpcMsg *pReq) { @@ -45,11 +50,6 @@ int32_t mmProcessGetMonitorInfoReq(SMnodeMgmt *pMgmt, SRpcMsg *pReq) { return 0; } -static void mmGetMnodeLoads(SMnodeMgmt *pMgmt, SMonMloadInfo *pInfo) { - pInfo->isMnode = 1; - mndGetLoad(pMgmt->pMnode, &pInfo->load); -} - int32_t mmProcessGetLoadsReq(SMnodeMgmt *pMgmt, SRpcMsg *pReq) { SMonMloadInfo mloads = {0}; mmGetMnodeLoads(pMgmt, &mloads); diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c b/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c index 4518a558bb..ac0868d8ef 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "qmInt.h" -static void qmGetMonitorInfo(SQnodeMgmt *pMgmt, SMonQmInfo *qmInfo) {} +void qmGetMonitorInfo(SQnodeMgmt *pMgmt, SMonQmInfo *qmInfo) {} int32_t qmProcessGetMonitorInfoReq(SQnodeMgmt *pMgmt, SRpcMsg *pReq) { SMonQmInfo qmInfo = {0}; diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c index ad83639a8d..d6ff73b132 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c +++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "smInt.h" -static void smGetMonitorInfo(SSnodeMgmt *pMgmt, SMonSmInfo *smInfo) {} +void smGetMonitorInfo(SSnodeMgmt *pMgmt, SMonSmInfo *smInfo) {} int32_t smProcessGetMonitorInfoReq(SSnodeMgmt *pMgmt, SRpcMsg *pReq) { SMonSmInfo smInfo = {0}; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 902998f28e..a4da6d089c 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "vmInt.h" -static void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) { +void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) { pInfo->pVloads = taosArrayInit(pMgmt->state.totalVnodes, sizeof(SVnodeLoad)); if (pInfo->pVloads == NULL) return; @@ -37,7 +37,7 @@ static void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) { taosRUnLockLatch(&pMgmt->latch); } -static void vmGetMonitorInfo(SVnodeMgmt *pMgmt, SMonVmInfo *pInfo) { +void vmGetMonitorInfo(SVnodeMgmt *pMgmt, SMonVmInfo *pInfo) { SMonVloadInfo vloads = {0}; vmGetVnodeLoads(pMgmt, &vloads); diff --git a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h index 7484c1e18f..9d092a93bc 100644 --- a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h +++ b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_DND_IMP_H_ -#define _TD_DND_IMP_H_ +#ifndef _TD_DND_MGMT_H_ +#define _TD_DND_MGMT_H_ // tobe deleted #include "uv.h" @@ -165,16 +165,13 @@ SMsgCb dmGetMsgcb(SDnode *pDnode); int32_t dmInitMsgHandle(SDnode *pDnode); int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); -// mgmt nodes -SMgmtFunc dmGetMgmtFunc(); -SMgmtFunc bmGetMgmtFunc(); -SMgmtFunc qmGetMgmtFunc(); -SMgmtFunc smGetMgmtFunc(); -SMgmtFunc vmGetMgmtFunc(); -SMgmtFunc mmGetMgmtFunc(); +// dmMonitor.c +void dmSendMonitorReport(); +void dmGetVnodeLoads(SMonVloadInfo *pInfo); +void dmGetMnodeLoads(SMonMloadInfo *pInfo); #ifdef __cplusplus } #endif -#endif /*_TD_DND_IMP_H_*/ \ No newline at end of file +#endif /*_TD_DND_MGMT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/node_mgmt/inc/dmNodes.h b/source/dnode/mgmt/node_mgmt/inc/dmNodes.h new file mode 100644 index 0000000000..3ac71de530 --- /dev/null +++ b/source/dnode/mgmt/node_mgmt/inc/dmNodes.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_NODES_H_ +#define _TD_DND_NODES_H_ + +#include "dmInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +SMgmtFunc dmGetMgmtFunc(); +SMgmtFunc bmGetMgmtFunc(); +SMgmtFunc qmGetMgmtFunc(); +SMgmtFunc smGetMgmtFunc(); +SMgmtFunc vmGetMgmtFunc(); +SMgmtFunc mmGetMgmtFunc(); + +void mmGetMonitorInfo(void *pMgmt, SMonMmInfo *pInfo); +void vmGetMonitorInfo(void *pMgmt, SMonVmInfo *pInfo); +void qmGetMonitorInfo(void *pMgmt, SMonQmInfo *pInfo); +void smGetMonitorInfo(void *pMgmt, SMonSmInfo *pInfo); +void bmGetMonitorInfo(void *pMgmt, SMonBmInfo *pInfo); + +void vmGetVnodeLoads(void *pMgmt, SMonVloadInfo *pInfo); +void mmGetMnodeLoads(void *pMgmt, SMonMloadInfo *pInfo); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_NODES_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index af5c0f00db..07d0c43360 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -168,11 +168,6 @@ static int32_t dmProcessDropNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { return code; } -static bool dmIsNodeRequired(EDndNodeType ntype) { - SDnode *pDnode = dmInstance(); - return pDnode->wrappers[ntype].required; -} - SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper) { SMgmtInputOpt opt = { .path = pWrapper->path, @@ -180,7 +175,9 @@ SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper) { .pData = &pWrapper->pDnode->data, .processCreateNodeFp = dmProcessCreateNodeReq, .processDropNodeFp = dmProcessDropNodeReq, - .isNodeRequiredFp = dmIsNodeRequired, + .sendMonitorReportFp = dmSendMonitorReport, + .getVnodeLoadsFp = dmGetVnodeLoads, + .getMnodeLoadsFp = dmGetMnodeLoads, }; opt.msgCb = dmGetMsgcb(pWrapper->pDnode); diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 30d7750f79..809c903e5c 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "dmMgmt.h" +#include "dmNodes.h" static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) { SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper); diff --git a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c new file mode 100644 index 0000000000..cb79de534d --- /dev/null +++ b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmMgmt.h" +#include "dmNodes.h" + +#define dmSendLocalRecv(pDnode, mtype, func, pInfo) \ + SRpcMsg rsp = {0}; \ + SRpcMsg req = {.msgType = mtype}; \ + SEpSet epset = {.inUse = 0, .numOfEps = 1}; \ + tstrncpy(epset.eps[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN); \ + epset.eps[0].port = tsServerPort; \ + rpcSendRecv(pDnode->trans.clientRpc, &epset, &req, &rsp); \ + if (rsp.code == 0 && rsp.contLen > 0) { \ + func(rsp.pCont, rsp.contLen, pInfo); \ + } \ + rpcFreeCont(rsp.pCont); + +static void dmGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { + pInfo->protocol = 1; + pInfo->dnode_id = pDnode->data.dnodeId; + pInfo->cluster_id = pDnode->data.clusterId; + tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN); +} + +static void dmGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { + pInfo->uptime = (taosGetTimestampMs() - pDnode->data.rebootTime) / (86400000.0f); + pInfo->has_mnode = pDnode->wrappers[MNODE].required; + pInfo->has_qnode = pDnode->wrappers[QNODE].required; + pInfo->has_snode = pDnode->wrappers[SNODE].required; + pInfo->has_bnode = pDnode->wrappers[BNODE].required; + tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name)); + pInfo->logdir.size = tsLogSpace.size; + tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name)); + pInfo->tempdir.size = tsTempSpace.size; +} + +static void dmGetDmMonitorInfo(SDnode *pDnode) { + SMonDmInfo dmInfo = {0}; + dmGetMonitorBasicInfo(pDnode, &dmInfo.basic); + dmGetMonitorDnodeInfo(pDnode, &dmInfo.dnode); + dmGetMonitorSystemInfo(&dmInfo.sys); + monSetDmInfo(&dmInfo); +} + +static void dmGetMmMonitorInfo(SDnode *pDnode) { + SMonMmInfo mmInfo = {0}; + SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, MNODE); + if (pWrapper != NULL && pWrapper->required) { + if (tsMultiProcess) { + dmSendLocalRecv(pDnode, TDMT_MON_MM_INFO, tDeserializeSMonMmInfo, &mmInfo); + } else if (pWrapper->pMgmt != NULL) { + mmGetMonitorInfo(pWrapper->pMgmt, &mmInfo); + } + } + dmReleaseWrapper(pWrapper); + monSetMmInfo(&mmInfo); + tFreeSMonMmInfo(&mmInfo); +} + +static void dmGetVmMonitorInfo(SDnode *pDnode) { + SMonVmInfo vmInfo = {0}; + SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, VNODE); + if (pWrapper != NULL && pWrapper->required) { + if (tsMultiProcess) { + dmSendLocalRecv(pDnode, TDMT_MON_VM_INFO, tDeserializeSMonVmInfo, &vmInfo); + } else if (pWrapper->pMgmt != NULL) { + vmGetMonitorInfo(pWrapper->pMgmt, &vmInfo); + } + } + dmReleaseWrapper(pWrapper); + monSetVmInfo(&vmInfo); + tFreeSMonVmInfo(&vmInfo); +} + +static void dmGetQmMonitorInfo(SDnode *pDnode) { + SMonQmInfo qmInfo = {0}; + SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, VNODE); + if (pWrapper != NULL && pWrapper->required) { + if (tsMultiProcess) { + dmSendLocalRecv(pDnode, TDMT_MON_QM_INFO, tDeserializeSMonQmInfo, &qmInfo); + } else if (pWrapper->pMgmt != NULL) { + qmGetMonitorInfo(pWrapper->pMgmt, &qmInfo); + } + } + dmReleaseWrapper(pWrapper); + monSetQmInfo(&qmInfo); + tFreeSMonQmInfo(&qmInfo); +} + +static void dmGetSmMonitorInfo(SDnode *pDnode) { + SMonSmInfo smInfo = {0}; + SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, VNODE); + if (pWrapper != NULL && pWrapper->required) { + if (tsMultiProcess) { + dmSendLocalRecv(pDnode, TDMT_MON_SM_INFO, tDeserializeSMonSmInfo, &smInfo); + } else if (pWrapper->pMgmt != NULL) { + smGetMonitorInfo(pWrapper->pMgmt, &smInfo); + } + } + dmReleaseWrapper(pWrapper); + monSetSmInfo(&smInfo); + tFreeSMonSmInfo(&smInfo); +} + +static void dmGetBmMonitorInfo(SDnode *pDnode) { + SMonBmInfo bmInfo = {0}; + SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, VNODE); + if (pWrapper != NULL && pWrapper->required) { + if (tsMultiProcess) { + dmSendLocalRecv(pDnode, TDMT_MON_BM_INFO, tDeserializeSMonBmInfo, &bmInfo); + } else if (pWrapper->pMgmt != NULL) { + bmGetMonitorInfo(pWrapper->pMgmt, &bmInfo); + } + } + dmReleaseWrapper(pWrapper); + monSetBmInfo(&bmInfo); + tFreeSMonBmInfo(&bmInfo); +} + +void dmSendMonitorReport() { + if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; + dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort); + + SDnode *pDnode = dmInstance(); + dmGetDmMonitorInfo(pDnode); + dmGetMmMonitorInfo(pDnode); + dmGetVmMonitorInfo(pDnode); + dmGetQmMonitorInfo(pDnode); + dmGetSmMonitorInfo(pDnode); + dmGetBmMonitorInfo(pDnode); + monSendReport(); +} + +void dmGetVnodeLoads(SMonVloadInfo *pInfo) { + SDnode *pDnode = dmInstance(); + SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, VNODE); + if (pWrapper != NULL && pWrapper->required) { + if (tsMultiProcess) { + dmSendLocalRecv(pDnode, TDMT_MON_VM_LOAD, tDeserializeSMonVloadInfo, pInfo); + } else if (pWrapper->pMgmt != NULL) { + vmGetVnodeLoads(pWrapper->pMgmt, pInfo); + } + } + dmReleaseWrapper(pWrapper); +} + +void dmGetMnodeLoads(SMonMloadInfo *pInfo) { + SDnode *pDnode = dmInstance(); + SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, MNODE); + if (pWrapper != NULL && pWrapper->required) { + if (tsMultiProcess) { + dmSendLocalRecv(pDnode, TDMT_MON_MM_LOAD, tDeserializeSMonMloadInfo, pInfo); + } else if (pWrapper->pMgmt != NULL) { + mmGetMnodeLoads(pWrapper->pMgmt, pInfo); + } + } + dmReleaseWrapper(pWrapper); +} diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 45bd3e4f64..b0e764bf8e 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -89,21 +89,23 @@ typedef enum { typedef int32_t (*ProcessCreateNodeFp)(EDndNodeType ntype, SRpcMsg *pMsg); typedef int32_t (*ProcessDropNodeFp)(EDndNodeType ntype, SRpcMsg *pMsg); -typedef bool (*IsNodeRequiredFp)(EDndNodeType ntype); +typedef void (*SendMonitorReportFp)(); +typedef void (*GetVnodeLoadsFp)(); +typedef void (*GetMnodeLoadsFp)(); typedef struct { - int32_t dnodeId; - int64_t clusterId; - int64_t dnodeVer; - int64_t updateTime; - int64_t rebootTime; - bool dropped; - bool stopped; - SEpSet mnodeEps; - SArray *dnodeEps; - SHashObj *dnodeHash; - SRWLatch latch; - SMsgCb msgCb; + int32_t dnodeId; + int64_t clusterId; + int64_t dnodeVer; + int64_t updateTime; + int64_t rebootTime; + bool dropped; + bool stopped; + SEpSet mnodeEps; + SArray *dnodeEps; + SHashObj *dnodeHash; + SRWLatch latch; + SMsgCb msgCb; } SDnodeData; typedef struct { @@ -113,7 +115,9 @@ typedef struct { SMsgCb msgCb; ProcessCreateNodeFp processCreateNodeFp; ProcessDropNodeFp processDropNodeFp; - IsNodeRequiredFp isNodeRequiredFp; + SendMonitorReportFp sendMonitorReportFp; + GetVnodeLoadsFp getVnodeLoadsFp; + GetMnodeLoadsFp getMnodeLoadsFp; } SMgmtInputOpt; typedef struct { From 83f4d1286e18fda5a10f7d2b4d483f269e92af39 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 19 May 2022 11:20:27 +0800 Subject: [PATCH 10/28] fix: executor process express tree scalar execution error --- source/libs/executor/src/executorimpl.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index db34f5dc67..3505af46e4 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -869,8 +869,12 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc SColumnInfoData idata = {.info = pResColData->info, .hasNull = true}; SScalarParam dest = {.columnData = &idata}; - scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest); - + int32_t code = scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pBlockList); + return code; + } + int32_t startOffset = createNewColModel ? 0 : pResult->info.rows; colInfoDataEnsureCapacity(pResColData, startOffset, pResult->info.capacity); colDataMergeCol(pResColData, startOffset, &pResult->info.capacity, &idata, dest.numOfRows); From f1a788f16174350c6d4e85a4580b9e20d1c382f8 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Thu, 19 May 2022 11:34:52 +0800 Subject: [PATCH 11/28] fix(os): win run error --- source/os/CMakeLists.txt | 3 +++ source/os/src/osString.c | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/source/os/CMakeLists.txt b/source/os/CMakeLists.txt index ad6cfc8b95..90b8e9dd8a 100644 --- a/source/os/CMakeLists.txt +++ b/source/os/CMakeLists.txt @@ -27,6 +27,9 @@ if(BUILD_ADDR2LINE) os PUBLIC addr2line dl z ) endif () +if(CHECK_STR2INT_ERROR) + add_definitions(-DTD_CHECK_STR_TO_INT_ERROR) +endif() target_link_libraries( os PUBLIC pthread ) diff --git a/source/os/src/osString.c b/source/os/src/osString.c index d5762ef87d..da1fbd364f 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -258,79 +258,99 @@ char *taosStrCaseStr(const char *str, const char *pattern) { int64_t taosStr2Int64(const char *str, char** pEnd, int32_t radix) { int64_t tmp = strtoll(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); +#endif return tmp; } uint64_t taosStr2UInt64(const char *str, char** pEnd, int32_t radix) { uint64_t tmp = strtoull(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); +#endif return tmp; } int32_t taosStr2Int32(const char *str, char** pEnd, int32_t radix) { int32_t tmp = strtol(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); +#endif return tmp; } uint32_t taosStr2UInt32(const char *str, char** pEnd, int32_t radix) { uint32_t tmp = strtol(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); +#endif return tmp; } int16_t taosStr2Int16(const char *str, char** pEnd, int32_t radix) { int32_t tmp = strtol(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); assert(tmp >= SHRT_MIN); assert(tmp <= SHRT_MAX); +#endif return (int16_t)tmp; } uint16_t taosStr2UInt16(const char *str, char** pEnd, int32_t radix) { uint32_t tmp = strtoul(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); assert(tmp <= USHRT_MAX); +#endif return (uint16_t)tmp; } int8_t taosStr2Int8(const char *str, char** pEnd, int32_t radix) { int32_t tmp = strtol(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); assert(tmp >= SCHAR_MIN); assert(tmp <= SCHAR_MAX); +#endif return tmp; } uint8_t taosStr2UInt8(const char *str, char** pEnd, int32_t radix) { uint32_t tmp = strtoul(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); assert(tmp <= UCHAR_MAX); +#endif return tmp; } double taosStr2Double(const char *str, char** pEnd) { double tmp = strtod(str, pEnd); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); assert(tmp != HUGE_VAL); +#endif return tmp; } float taosStr2Float(const char *str, char** pEnd) { float tmp = strtof(str, pEnd); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); assert(tmp != HUGE_VALF); assert(tmp != NAN); +#endif return tmp; } \ No newline at end of file From fefc2dad116a8e0ec123884d95c19451b3941da3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 19 May 2022 11:51:52 +0800 Subject: [PATCH 12/28] refactor: dnode monitor --- source/dnode/mgmt/mgmt_mnode/src/mmFile.c | 2 +- source/dnode/mgmt/mgmt_vnode/src/vmFile.c | 2 +- source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index df377fefe7..0bab05f973 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -154,6 +154,6 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, SDCreateMnodeReq *pReq, bool deployed) { return -1; } - dInfo("successed to write %s, deployed:%d", realfile, deployed); + dDebug("successed to write %s, deployed:%d", realfile, deployed); return 0; } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c index f6c7bb33e6..7a6c5f982e 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c @@ -128,7 +128,7 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes = vnodesNum; code = 0; - dInfo("succcessed to read file %s", file); + dDebug("succcessed to read file %s", file); _OVER: if (content != NULL) taosMemoryFree(content); diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 809c903e5c..3cbb9ff046 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -190,7 +190,7 @@ int32_t dmInitDnode(SDnode *pDnode, EDndNodeType rtype) { } dmReportStartup("dnode-transport", "initialized"); - dInfo("dnode is created, ptr:%p", pDnode); + dDebug("dnode is created, ptr:%p", pDnode); code = 0; _OVER: @@ -209,7 +209,7 @@ void dmCleanupDnode(SDnode *pDnode) { dmCleanupClient(pDnode); dmCleanupServer(pDnode); dmClearVars(pDnode); - dInfo("dnode is closed, ptr:%p", pDnode); + dDebug("dnode is closed, ptr:%p", pDnode); } void dmSetStatus(SDnode *pDnode, EDndRunStatus status) { From 2d8ed7a947834c5a6c14525f8b438f81c4c22f18 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 19 May 2022 12:12:20 +0800 Subject: [PATCH 13/28] fix: monitor in multi-process mode --- source/dnode/mgmt/node_mgmt/src/dmMonitor.c | 78 ++++++++++----------- 1 file changed, 38 insertions(+), 40 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c index cb79de534d..2497da13ec 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c @@ -57,78 +57,78 @@ static void dmGetDmMonitorInfo(SDnode *pDnode) { } static void dmGetMmMonitorInfo(SDnode *pDnode) { - SMonMmInfo mmInfo = {0}; - SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, MNODE); - if (pWrapper != NULL && pWrapper->required) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[MNODE]; + if (dmMarkWrapper(pWrapper) == 0) { + SMonMmInfo mmInfo = {0}; if (tsMultiProcess) { dmSendLocalRecv(pDnode, TDMT_MON_MM_INFO, tDeserializeSMonMmInfo, &mmInfo); } else if (pWrapper->pMgmt != NULL) { mmGetMonitorInfo(pWrapper->pMgmt, &mmInfo); } + dmReleaseWrapper(pWrapper); + monSetMmInfo(&mmInfo); + tFreeSMonMmInfo(&mmInfo); } - dmReleaseWrapper(pWrapper); - monSetMmInfo(&mmInfo); - tFreeSMonMmInfo(&mmInfo); } static void dmGetVmMonitorInfo(SDnode *pDnode) { - SMonVmInfo vmInfo = {0}; - SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, VNODE); - if (pWrapper != NULL && pWrapper->required) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[VNODE]; + if (dmMarkWrapper(pWrapper) == 0) { + SMonVmInfo vmInfo = {0}; if (tsMultiProcess) { dmSendLocalRecv(pDnode, TDMT_MON_VM_INFO, tDeserializeSMonVmInfo, &vmInfo); } else if (pWrapper->pMgmt != NULL) { vmGetMonitorInfo(pWrapper->pMgmt, &vmInfo); } + dmReleaseWrapper(pWrapper); + monSetVmInfo(&vmInfo); + tFreeSMonVmInfo(&vmInfo); } - dmReleaseWrapper(pWrapper); - monSetVmInfo(&vmInfo); - tFreeSMonVmInfo(&vmInfo); } static void dmGetQmMonitorInfo(SDnode *pDnode) { - SMonQmInfo qmInfo = {0}; - SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, VNODE); - if (pWrapper != NULL && pWrapper->required) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[QNODE]; + if (dmMarkWrapper(pWrapper) == 0) { + SMonQmInfo qmInfo = {0}; if (tsMultiProcess) { dmSendLocalRecv(pDnode, TDMT_MON_QM_INFO, tDeserializeSMonQmInfo, &qmInfo); } else if (pWrapper->pMgmt != NULL) { qmGetMonitorInfo(pWrapper->pMgmt, &qmInfo); } + dmReleaseWrapper(pWrapper); + monSetQmInfo(&qmInfo); + tFreeSMonQmInfo(&qmInfo); } - dmReleaseWrapper(pWrapper); - monSetQmInfo(&qmInfo); - tFreeSMonQmInfo(&qmInfo); } static void dmGetSmMonitorInfo(SDnode *pDnode) { - SMonSmInfo smInfo = {0}; - SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, VNODE); - if (pWrapper != NULL && pWrapper->required) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[SNODE]; + if (dmMarkWrapper(pWrapper) == 0) { + SMonSmInfo smInfo = {0}; if (tsMultiProcess) { dmSendLocalRecv(pDnode, TDMT_MON_SM_INFO, tDeserializeSMonSmInfo, &smInfo); } else if (pWrapper->pMgmt != NULL) { smGetMonitorInfo(pWrapper->pMgmt, &smInfo); } + dmReleaseWrapper(pWrapper); + monSetSmInfo(&smInfo); + tFreeSMonSmInfo(&smInfo); } - dmReleaseWrapper(pWrapper); - monSetSmInfo(&smInfo); - tFreeSMonSmInfo(&smInfo); } static void dmGetBmMonitorInfo(SDnode *pDnode) { - SMonBmInfo bmInfo = {0}; - SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, VNODE); - if (pWrapper != NULL && pWrapper->required) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[BNODE]; + if (dmMarkWrapper(pWrapper) == 0) { + SMonBmInfo bmInfo = {0}; if (tsMultiProcess) { dmSendLocalRecv(pDnode, TDMT_MON_BM_INFO, tDeserializeSMonBmInfo, &bmInfo); } else if (pWrapper->pMgmt != NULL) { bmGetMonitorInfo(pWrapper->pMgmt, &bmInfo); } + dmReleaseWrapper(pWrapper); + monSetBmInfo(&bmInfo); + tFreeSMonBmInfo(&bmInfo); } - dmReleaseWrapper(pWrapper); - monSetBmInfo(&bmInfo); - tFreeSMonBmInfo(&bmInfo); } void dmSendMonitorReport() { @@ -147,26 +147,24 @@ void dmSendMonitorReport() { void dmGetVnodeLoads(SMonVloadInfo *pInfo) { SDnode *pDnode = dmInstance(); - SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, VNODE); - if (pWrapper != NULL && pWrapper->required) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[VNODE]; + if (dmMarkWrapper(pWrapper) == 0) { if (tsMultiProcess) { dmSendLocalRecv(pDnode, TDMT_MON_VM_LOAD, tDeserializeSMonVloadInfo, pInfo); } else if (pWrapper->pMgmt != NULL) { vmGetVnodeLoads(pWrapper->pMgmt, pInfo); } + dmReleaseWrapper(pWrapper); } - dmReleaseWrapper(pWrapper); } void dmGetMnodeLoads(SMonMloadInfo *pInfo) { SDnode *pDnode = dmInstance(); - SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, MNODE); - if (pWrapper != NULL && pWrapper->required) { - if (tsMultiProcess) { - dmSendLocalRecv(pDnode, TDMT_MON_MM_LOAD, tDeserializeSMonMloadInfo, pInfo); - } else if (pWrapper->pMgmt != NULL) { - mmGetMnodeLoads(pWrapper->pMgmt, pInfo); - } + SMgmtWrapper *pWrapper = &pDnode->wrappers[MNODE]; + if (tsMultiProcess) { + dmSendLocalRecv(pDnode, TDMT_MON_MM_LOAD, tDeserializeSMonMloadInfo, pInfo); + } else if (pWrapper->pMgmt != NULL) { + mmGetMnodeLoads(pWrapper->pMgmt, pInfo); } dmReleaseWrapper(pWrapper); } From 7002f32a92d77705ee6a1528579f27b6a4da8d19 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 19 May 2022 12:30:18 +0800 Subject: [PATCH 14/28] enh: add tagver and colver to stb and its msg --- include/common/tmsg.h | 14 ++++++++++++++ source/dnode/mnode/impl/inc/mndDef.h | 2 ++ source/dnode/mnode/impl/src/mndStb.c | 19 ++++++++++++++++++- 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index f7562a4b9b..09dd7c64d2 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -301,6 +301,8 @@ typedef struct SSchema { typedef struct { int32_t nCols; int32_t sver; + int32_t tagVer; + int32_t colVer; SSchema* pSchema; } SSchemaWrapper; @@ -309,6 +311,8 @@ static FORCE_INLINE SSchemaWrapper* tCloneSSchemaWrapper(const SSchemaWrapper* p if (pSW == NULL) return pSW; pSW->nCols = pSchemaWrapper->nCols; pSW->sver = pSchemaWrapper->sver; + pSW->tagVer = pSchemaWrapper->tagVer; + pSW->colVer = pSchemaWrapper->colVer; pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); if (pSW->pSchema == NULL) { taosMemoryFree(pSW); @@ -364,6 +368,8 @@ static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWr int32_t tlen = 0; tlen += taosEncodeVariantI32(buf, pSW->nCols); tlen += taosEncodeVariantI32(buf, pSW->sver); + tlen += taosEncodeVariantI32(buf, pSW->tagVer); + tlen += taosEncodeVariantI32(buf, pSW->colVer); for (int32_t i = 0; i < pSW->nCols; i++) { tlen += taosEncodeSSchema(buf, &pSW->pSchema[i]); } @@ -373,6 +379,8 @@ static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWr static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapper* pSW) { buf = taosDecodeVariantI32(buf, &pSW->nCols); buf = taosDecodeVariantI32(buf, &pSW->sver); + buf = taosDecodeVariantI32(buf, &pSW->tagVer); + buf = taosDecodeVariantI32(buf, &pSW->colVer); pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); if (pSW->pSchema == NULL) { return NULL; @@ -387,6 +395,8 @@ static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapp static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SEncoder* pEncoder, const SSchemaWrapper* pSW) { if (tEncodeI32v(pEncoder, pSW->nCols) < 0) return -1; if (tEncodeI32v(pEncoder, pSW->sver) < 0) return -1; + if (tEncodeI32v(pEncoder, pSW->tagVer) < 0) return -1; + if (tEncodeI32v(pEncoder, pSW->colVer) < 0) return -1; for (int32_t i = 0; i < pSW->nCols; i++) { if (tEncodeSSchema(pEncoder, &pSW->pSchema[i]) < 0) return -1; } @@ -397,6 +407,8 @@ static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SEncoder* pEncoder, const SSch static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SDecoder* pDecoder, SSchemaWrapper* pSW) { if (tDecodeI32v(pDecoder, &pSW->nCols) < 0) return -1; if (tDecodeI32v(pDecoder, &pSW->sver) < 0) return -1; + if (tDecodeI32v(pDecoder, &pSW->tagVer) < 0) return -1; + if (tDecodeI32v(pDecoder, &pSW->colVer) < 0) return -1; pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); if (pSW->pSchema == NULL) return -1; @@ -410,6 +422,8 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SDecoder* pDecoder, SSchemaWra static FORCE_INLINE int32_t tDecodeSSchemaWrapperEx(SDecoder* pDecoder, SSchemaWrapper* pSW) { if (tDecodeI32v(pDecoder, &pSW->nCols) < 0) return -1; if (tDecodeI32v(pDecoder, &pSW->sver) < 0) return -1; + if (tDecodeI32v(pDecoder, &pSW->tagVer) < 0) return -1; + if (tDecodeI32v(pDecoder, &pSW->colVer) < 0) return -1; pSW->pSchema = (SSchema*)tDecoderMalloc(pDecoder, pSW->nCols * sizeof(SSchema)); if (pSW->pSchema == NULL) return -1; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index a04417736a..81f4c5ed1e 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -365,6 +365,8 @@ typedef struct { int64_t uid; int64_t dbUid; int32_t version; + int32_t tagVer; + int32_t colVer; int32_t nextColId; float xFilesFactor; int32_t delay; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index e8a0b31e2f..c8d4f91183 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -88,6 +88,8 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { SDB_SET_INT64(pRaw, dataPos, pStb->uid, _OVER) SDB_SET_INT64(pRaw, dataPos, pStb->dbUid, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->version, _OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->tagVer, _OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->colVer, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->nextColId, _OVER) SDB_SET_INT32(pRaw, dataPos, (int32_t)(pStb->xFilesFactor * 10000), _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->delay, _OVER) @@ -166,6 +168,8 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, dataPos, &pStb->uid, _OVER) SDB_GET_INT64(pRaw, dataPos, &pStb->dbUid, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->version, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pStb->tagVer, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pStb->colVer, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->nextColId, _OVER) int32_t xFilesFactor = 0; SDB_GET_INT32(pRaw, dataPos, &xFilesFactor, _OVER) @@ -317,7 +321,9 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { pOld->updateTime = pNew->updateTime; pOld->version = pNew->version; - pOld->nextColId = pNew->nextColId; + pOld->tagVer = pNew->tagVer; + pOld->colVer = pNew->colVer; + pOld->colVer = pNew->nextColId; pOld->ttl = pNew->ttl; pOld->numOfColumns = pNew->numOfColumns; pOld->numOfTags = pNew->numOfTags; @@ -384,6 +390,8 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt req.rollup = pStb->ast1Len > 0 ? 1 : 0; req.schema.nCols = pStb->numOfColumns; req.schema.sver = pStb->version; + req.schema.tagVer = pStb->tagVer; + req.schema.colVer = pStb->colVer; req.schema.pSchema = pStb->pColumns; req.schemaTag.nCols = pStb->numOfTags; req.schemaTag.sver = 1; @@ -657,6 +665,8 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat pDst->uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); pDst->dbUid = pDb->uid; pDst->version = 1; + pDst->tagVer = 1; + pDst->colVer = 1; pDst->nextColId = 1; pDst->xFilesFactor = pCreate->xFilesFactor; pDst->delay = pCreate->delay; @@ -949,6 +959,7 @@ static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *p } pNew->version++; + pNew->tagVer++; return 0; } @@ -967,6 +978,7 @@ static int32_t mndDropSuperTableTag(const SStbObj *pOld, SStbObj *pNew, const ch pNew->numOfTags--; pNew->version++; + pNew->tagVer++; mDebug("stb:%s, start to drop tag %s", pNew->name, tagName); return 0; } @@ -1007,6 +1019,7 @@ static int32_t mndAlterStbTagName(const SStbObj *pOld, SStbObj *pNew, SArray *pF memcpy(pSchema->name, newTagName, TSDB_COL_NAME_LEN); pNew->version++; + pNew->tagVer++; mDebug("stb:%s, start to modify tag %s to %s", pNew->name, oldTagName, newTagName); return 0; } @@ -1036,6 +1049,7 @@ static int32_t mndAlterStbTagBytes(const SStbObj *pOld, SStbObj *pNew, const SFi pTag->bytes = pField->bytes; pNew->version++; + pNew->tagVer++; mDebug("stb:%s, start to modify tag len %s to %d", pNew->name, pField->name, pField->bytes); return 0; @@ -1075,6 +1089,7 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray } pNew->version++; + pNew->colVer++; return 0; } @@ -1103,6 +1118,7 @@ static int32_t mndDropSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, const pNew->numOfColumns--; pNew->version++; + pNew->colVer++; mDebug("stb:%s, start to drop col %s", pNew->name, colName); return 0; } @@ -1141,6 +1157,7 @@ static int32_t mndAlterStbColumnBytes(const SStbObj *pOld, SStbObj *pNew, const pCol->bytes = pField->bytes; pNew->version++; + pNew->colVer++; mDebug("stb:%s, start to modify col len %s to %d", pNew->name, pField->name, pField->bytes); return 0; From 1e7dd9084492fb49e8f9bbb9223846f43c4e4c97 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 19 May 2022 13:13:35 +0800 Subject: [PATCH 15/28] enh: return success if version in alter request is smaller than the stb --- include/common/tmsg.h | 1 + source/common/src/tmsg.c | 2 ++ source/dnode/mnode/impl/src/mndStb.c | 9 ++++++++- source/dnode/mnode/impl/test/stb/stb.cpp | 24 +++++++++++++++++------- 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 09dd7c64d2..38609c2348 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -469,6 +469,7 @@ int32_t tDeserializeSMDropStbReq(void* buf, int32_t bufLen, SMDropStbReq* pReq); typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t alterType; + int32_t verInBlock; int32_t numOfFields; SArray* pFields; int32_t ttl; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 278fc2c193..677fbb43b4 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -600,6 +600,7 @@ int32_t tSerializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq) if (tStartEncode(&encoder) < 0) return -1; if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; if (tEncodeI8(&encoder, pReq->alterType) < 0) return -1; + if (tEncodeI32(&encoder, pReq->verInBlock) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfFields) < 0) return -1; for (int32_t i = 0; i < pReq->numOfFields; ++i) { SField *pField = taosArrayGet(pReq->pFields, i); @@ -626,6 +627,7 @@ int32_t tDeserializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq if (tStartDecode(&decoder) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; if (tDecodeI8(&decoder, &pReq->alterType) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->verInBlock) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfFields) < 0) return -1; pReq->pFields = taosArrayInit(pReq->numOfFields, sizeof(SField)); if (pReq->pFields == NULL) { diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index c8d4f91183..7485510bc6 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -323,7 +323,7 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { pOld->version = pNew->version; pOld->tagVer = pNew->tagVer; pOld->colVer = pNew->colVer; - pOld->colVer = pNew->nextColId; + pOld->nextColId = pNew->nextColId; pOld->ttl = pNew->ttl; pOld->numOfColumns = pNew->numOfColumns; pOld->numOfTags = pNew->numOfTags; @@ -1317,6 +1317,13 @@ static int32_t mndProcessMAlterStbReq(SRpcMsg *pReq) { goto _OVER; } + if (alterReq.verInBlock > 0 && alterReq.verInBlock <= pStb->version) { + mDebug("stb:%s, already exist, verInBlock:%d smaller than verInStb:%d, alter success", alterReq.name, + alterReq.verInBlock, pStb->version); + code = 0; + goto _OVER; + } + pUser = mndAcquireUser(pMnode, pReq->conn.user); if (pUser == NULL) { goto _OVER; diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index 16974ad541..b8873210ab 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -32,7 +32,7 @@ class MndTestStb : public ::testing::Test { void* BuildAlterStbUpdateTagBytesReq(const char* stbname, const char* tagname, int32_t bytes, int32_t* pContLen); void* BuildAlterStbAddColumnReq(const char* stbname, const char* colname, int32_t* pContLen); void* BuildAlterStbDropColumnReq(const char* stbname, const char* colname, int32_t* pContLen); - void* BuildAlterStbUpdateColumnBytesReq(const char* stbname, const char* colname, int32_t bytes, int32_t* pContLen); + void* BuildAlterStbUpdateColumnBytesReq(const char* stbname, const char* colname, int32_t bytes, int32_t* pContLen, int32_t verInBlock); }; Testbase MndTestStb::test; @@ -271,12 +271,13 @@ void* MndTestStb::BuildAlterStbDropColumnReq(const char* stbname, const char* co } void* MndTestStb::BuildAlterStbUpdateColumnBytesReq(const char* stbname, const char* colname, int32_t bytes, - int32_t* pContLen) { + int32_t* pContLen, int32_t verInBlock) { SMAlterStbReq req = {0}; strcpy(req.name, stbname); req.numOfFields = 1; req.pFields = taosArrayInit(1, sizeof(SField)); req.alterType = TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES; + req.verInBlock = verInBlock; SField field = {0}; field.bytes = bytes; @@ -781,31 +782,40 @@ TEST_F(MndTestStb, 08_Alter_Stb_AlterTagBytes) { } { - void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col5", 12, &contLen); + void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col5", 12, &contLen, 0); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_NOT_EXIST); } { - void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "ts", 8, &contLen); + void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "ts", 8, &contLen, 0); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_STB_OPTION); } { - void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col1", 8, &contLen); + void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col1", 8, &contLen, 0); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_ROW_BYTES); } { - void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col1", TSDB_MAX_BYTES_PER_ROW, &contLen); + void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col1", TSDB_MAX_BYTES_PER_ROW, &contLen, 0); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_ROW_BYTES); } { - void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col1", 20, &contLen); + void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col1", 20, &contLen, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + + test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname); + EXPECT_EQ(test.GetShowRows(), 1); + } + + { + void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col_not_exist", 20, &contLen, 1); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); ASSERT_EQ(pRsp->code, 0); From 29bdba7298ce83b17531c73052e090da97a52795 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 19 May 2022 14:26:10 +0800 Subject: [PATCH 16/28] fix: some problems of parser and planner --- include/libs/function/functionMgt.h | 2 ++ include/libs/nodes/cmdnodes.h | 4 ++-- include/util/taoserror.h | 2 ++ include/util/tdef.h | 4 ++-- source/libs/function/src/functionMgt.c | 4 ++++ source/libs/nodes/src/nodesCodeFuncs.c | 21 ++++++++++++++++++++- source/libs/parser/src/parAstCreater.c | 17 ++++++++++++++--- source/libs/parser/src/parTranslater.c | 13 ++++++++++--- source/libs/parser/src/parUtil.c | 4 ++++ source/libs/planner/src/planSpliter.c | 5 ++++- source/libs/planner/test/planSetOpTest.cpp | 9 +++++++++ source/libs/planner/test/planTestUtil.cpp | 1 + 12 files changed, 74 insertions(+), 12 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index aec1476663..89fbc92992 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -142,6 +142,8 @@ void fmFuncMgtDestroy(); int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, SFunctionNode* pFunc); +bool fmIsBuiltinFunc(const char* pFunc); + bool fmIsAggFunc(int32_t funcId); bool fmIsScalarFunc(int32_t funcId); bool fmIsNonstandardSQLFunc(int32_t funcId); diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 5e294ae455..82bf4e1f45 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -78,7 +78,7 @@ typedef struct SAlterDatabaseStmt { typedef struct STableOptions { ENodeType type; - char comment[TSDB_STB_COMMENT_LEN]; + char comment[TSDB_TB_COMMENT_LEN]; int32_t delay; float filesFactor; SNodeList* pRollupFuncs; @@ -90,7 +90,7 @@ typedef struct SColumnDefNode { ENodeType type; char colName[TSDB_COL_NAME_LEN]; SDataType dataType; - char comments[TSDB_STB_COMMENT_LEN]; + char comments[TSDB_TB_COMMENT_LEN]; bool sma; } SColumnDefNode; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index be6005d46b..fd3e008e67 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -647,6 +647,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_CANNOT_DROP_PRIMARY_KEY TAOS_DEF_ERROR_CODE(0, 0x264A) #define TSDB_CODE_PAR_INVALID_MODIFY_COL TAOS_DEF_ERROR_CODE(0, 0x264B) #define TSDB_CODE_PAR_INVALID_TBNAME TAOS_DEF_ERROR_CODE(0, 0x264C) +#define TSDB_CODE_PAR_INVALID_FUNCTION_NAME TAOS_DEF_ERROR_CODE(0, 0x264D) +#define TSDB_CODE_PAR_COMMENT_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x264E) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) diff --git a/include/util/tdef.h b/include/util/tdef.h index f95d96be56..5cc687d7ab 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -218,8 +218,8 @@ typedef enum ELogicConditionType { #define TSDB_MAX_SQL_SHOW_LEN 1024 #define TSDB_MAX_ALLOWED_SQL_LEN (1 * 1024 * 1024u) // sql length should be less than 1mb -#define TSDB_APP_NAME_LEN TSDB_UNI_LEN -#define TSDB_STB_COMMENT_LEN 1024 +#define TSDB_APP_NAME_LEN TSDB_UNI_LEN +#define TSDB_TB_COMMENT_LEN 1025 /** * In some scenarios uint16_t (0~65535) is used to store the row len. diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 73ec7f510b..e46cb81561 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -100,6 +100,10 @@ int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, SFunctionNode* pFunc) { return getUdfInfo(pParam, pFunc); } +bool fmIsBuiltinFunc(const char* pFunc) { + return NULL != taosHashGet(gFunMgtService.pFuncNameHashTable, pFunc, strlen(pFunc)); +} + EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) { if (fmIsUserDefinedFunc(pFunc->funcId) || pFunc->funcId < 0 || pFunc->funcId >= funcMgtBuiltinsNum) { return FUNC_DATA_REQUIRED_DATA_LOAD; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 763ccbf7a0..bc49f36afe 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -946,6 +946,23 @@ static int32_t jsonToLogicSubplan(const SJson* pJson, void* pObj) { return code; } +static const char* jkLogicPlanSubplans = "Subplans"; + +static int32_t logicPlanToJson(const void* pObj, SJson* pJson) { + const SQueryLogicPlan* pNode = (const SQueryLogicPlan*)pObj; + return tjsonAddObject(pJson, jkLogicPlanSubplans, nodeToJson, nodesListGetNode(pNode->pTopSubplans, 0)); +} + +static int32_t jsonToLogicPlan(const SJson* pJson, void* pObj) { + SQueryLogicPlan* pNode = (SQueryLogicPlan*)pObj; + SNode* pChild = NULL; + int32_t code = jsonToNodeObject(pJson, jkLogicPlanSubplans, &pChild); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pNode->pTopSubplans, pChild); + } + return code; +} + static const char* jkJoinLogicPlanJoinType = "JoinType"; static const char* jkJoinLogicPlanOnConditions = "OnConditions"; @@ -3029,7 +3046,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_LOGIC_SUBPLAN: return logicSubplanToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN: - break; + return logicPlanToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: return physiTagScanNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: @@ -3126,6 +3143,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToLogicPartitionNode(pJson, pObj); case QUERY_NODE_LOGIC_SUBPLAN: return jsonToLogicSubplan(pJson, pObj); + case QUERY_NODE_LOGIC_PLAN: + return jsonToLogicPlan(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: return jsonToPhysiTagScanNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index a438a1e843..9a9a11c7b0 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -196,6 +196,15 @@ static bool checkIndexName(SAstCreateContext* pCxt, SToken* pIndexName) { return true; } +static bool checkComment(SAstCreateContext* pCxt, const SToken* pCommentToken, bool demand) { + if (NULL == pCommentToken) { + pCxt->errCode = demand ? TSDB_CODE_PAR_SYNTAX_ERROR : TSDB_CODE_SUCCESS; + } else if (pCommentToken->n >= (TSDB_TB_COMMENT_LEN + 2)) { + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_COMMENT_TOO_LONG); + } + return TSDB_CODE_SUCCESS == pCxt->errCode; +} + SNode* createRawExprNode(SAstCreateContext* pCxt, const SToken* pToken, SNode* pNode) { SRawExprNode* target = (SRawExprNode*)nodesMakeNode(QUERY_NODE_RAW_EXPR); CHECK_OUT_OF_MEM(target); @@ -823,8 +832,10 @@ SNode* createAlterTableOptions(SAstCreateContext* pCxt) { SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, void* pVal) { switch (type) { case TABLE_OPTION_COMMENT: - copyStringFormStringToken((SToken*)pVal, ((STableOptions*)pOptions)->comment, - sizeof(((STableOptions*)pOptions)->comment)); + if (checkComment(pCxt, (SToken*)pVal, true)) { + copyStringFormStringToken((SToken*)pVal, ((STableOptions*)pOptions)->comment, + sizeof(((STableOptions*)pOptions)->comment)); + } break; case TABLE_OPTION_DELAY: ((STableOptions*)pOptions)->delay = strtol(((SToken*)pVal)->z, NULL, 10); @@ -848,7 +859,7 @@ SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType } SNode* createColumnDefNode(SAstCreateContext* pCxt, SToken* pColName, SDataType dataType, const SToken* pComment) { - if (!checkColumnName(pCxt, pColName)) { + if (!checkColumnName(pCxt, pColName) || !checkComment(pCxt, pComment, false)) { return NULL; } SColumnDefNode* pCol = (SColumnDefNode*)nodesMakeNode(QUERY_NODE_COLUMN_DEF); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 18e0843e92..b270e811d9 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -272,6 +272,10 @@ static bool isTimelineFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsTimelineFunc(((SFunctionNode*)pNode)->funcId)); } +static bool isScanPseudoColumnFunc(const SNode* pNode) { + return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsScanPseudoColumnFunc(((SFunctionNode*)pNode)->funcId)); +} + static bool isDistinctOrderBy(STranslateContext* pCxt) { return (SQL_CLAUSE_ORDER_BY == pCxt->currClause && pCxt->pCurrStmt->isDistinct); } @@ -892,7 +896,7 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { return DEAL_RES_IGNORE_CHILD; } } - if (QUERY_NODE_COLUMN == nodeType(*pNode)) { + if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { if (pCxt->selectFuncNum > 1) { return generateDealNodeErrMsg(pCxt->pTranslateCxt, getGroupByErrorCode(pCxt->pTranslateCxt)); } else { @@ -930,7 +934,7 @@ static EDealRes rewriteColsToSelectValFuncImpl(SNode** pNode, void* pContext) { if (isAggFunc(*pNode)) { return DEAL_RES_IGNORE_CHILD; } - if (QUERY_NODE_COLUMN == nodeType(*pNode)) { + if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { return rewriteColToSelectValFunc((STranslateContext*)pContext, NULL, pNode); } return DEAL_RES_CONTINUE; @@ -958,7 +962,7 @@ static EDealRes doCheckAggColCoexist(SNode* pNode, void* pContext) { pCxt->existAggFunc = true; return DEAL_RES_IGNORE_CHILD; } - if (QUERY_NODE_COLUMN == nodeType(pNode)) { + if (isScanPseudoColumnFunc(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) { pCxt->existCol = true; } return DEAL_RES_CONTINUE; @@ -3261,6 +3265,9 @@ static int32_t readFromFile(char* pName, int32_t* len, char** buf) { } static int32_t translateCreateFunction(STranslateContext* pCxt, SCreateFunctionStmt* pStmt) { + if (fmIsBuiltinFunc(pStmt->funcName)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FUNCTION_NAME); + } SCreateFuncReq req = {0}; strcpy(req.name, pStmt->funcName); req.igExists = pStmt->ignoreExists; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 1ae066a8c2..7b9147beab 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -160,6 +160,10 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Only binary/nchar column length could be modified"; case TSDB_CODE_PAR_INVALID_TBNAME: return "Invalid tbname pseudo column"; + case TSDB_CODE_PAR_INVALID_FUNCTION_NAME: + return "Invalid function name"; + case TSDB_CODE_PAR_COMMENT_TOO_LONG: + return "Comment too long"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index d0ac239bcb..fb716af3e5 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -24,6 +24,7 @@ #define SPLIT_FLAG_TEST_MASK(val, mask) (((val) & (mask)) != 0) typedef struct SSplitContext { + int32_t queryId; int32_t groupId; bool split; } SSplitContext; @@ -62,6 +63,7 @@ static SLogicSubplan* splCreateScanSubplan(SSplitContext* pCxt, SScanLogicNode* if (NULL == pSubplan) { return NULL; } + pSubplan->id.queryId = pCxt->queryId; pSubplan->id.groupId = pCxt->groupId; pSubplan->subplanType = SUBPLAN_TYPE_SCAN; pSubplan->pNode = (SLogicNode*)nodesCloneNode(pScan); @@ -242,6 +244,7 @@ static SLogicSubplan* unionCreateSubplan(SSplitContext* pCxt, SLogicNode* pNode) if (NULL == pSubplan) { return NULL; } + pSubplan->id.queryId = pCxt->queryId; pSubplan->id.groupId = pCxt->groupId; pSubplan->subplanType = SUBPLAN_TYPE_SCAN; pSubplan->pNode = pNode; @@ -406,7 +409,7 @@ static const SSplitRule splitRuleSet[] = {{.pName = "SuperTableScan", .splitFunc static const int32_t splitRuleNum = (sizeof(splitRuleSet) / sizeof(SSplitRule)); static int32_t applySplitRule(SLogicSubplan* pSubplan) { - SSplitContext cxt = {.groupId = pSubplan->id.groupId + 1, .split = false}; + SSplitContext cxt = {.queryId = pSubplan->id.queryId, .groupId = pSubplan->id.groupId + 1, .split = false}; do { cxt.split = false; for (int32_t i = 0; i < splitRuleNum; ++i) { diff --git a/source/libs/planner/test/planSetOpTest.cpp b/source/libs/planner/test/planSetOpTest.cpp index 3c7bf9e43a..717384aae6 100644 --- a/source/libs/planner/test/planSetOpTest.cpp +++ b/source/libs/planner/test/planSetOpTest.cpp @@ -32,6 +32,15 @@ TEST_F(PlanSetOpTest, unionAllSubquery) { run("SELECT * FROM (SELECT c1, c2 FROM t1 UNION ALL SELECT c1, c2 FROM t1)"); } +TEST_F(PlanSetOpTest, unionAllWithSubquery) { + useDb("root", "test"); + + // child table + run("SELECT ts FROM (SELECT ts FROM st1s1) UNION ALL SELECT ts FROM (SELECT ts FROM st1s2)"); + // super table + run("SELECT ts FROM (SELECT ts FROM st1) UNION ALL SELECT ts FROM (SELECT ts FROM st1)"); +} + TEST_F(PlanSetOpTest, union) { useDb("root", "test"); diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index af8ec87158..94a28f46a8 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -322,6 +322,7 @@ class PlannerTestBaseImpl { } void setPlanContext(SQuery* pQuery, SPlanContext* pCxt) { + pCxt->queryId = 1; if (QUERY_NODE_CREATE_TOPIC_STMT == nodeType(pQuery->pRoot)) { pCxt->pAstRoot = ((SCreateTopicStmt*)pQuery->pRoot)->pQuery; pCxt->topicQuery = true; From f81f0978434efead48baf05c06dffd7e993db8bd Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Thu, 19 May 2022 14:31:59 +0800 Subject: [PATCH 17/28] fix(os): file path format error --- source/libs/tfs/src/tfs.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 18a3a28bab..0054a94b80 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -160,7 +160,12 @@ bool tfsIsSameFile(const STfsFile *pFile1, const STfsFile *pFile2) { if (pFile1 == NULL || pFile2 == NULL || pFile1->pTfs != pFile2->pTfs) return false; if (pFile1->did.level != pFile2->did.level) return false; if (pFile1->did.id != pFile2->did.id) return false; - if (strncmp(pFile1->rname, pFile2->rname, TSDB_FILENAME_LEN) != 0) return false; + char nameBuf1[TMPNAME_LEN], nameBuf2[TMPNAME_LEN]; + memset(nameBuf1, 0, TMPNAME_LEN); + memset(nameBuf2, 0, TMPNAME_LEN); + taosRealPath(pFile1->rname, nameBuf1, TMPNAME_LEN); + taosRealPath(pFile2->rname, nameBuf2, TMPNAME_LEN); + if (strncmp(nameBuf1, nameBuf2, TMPNAME_LEN) != 0) return false; return true; } From 5d1b09be03ccd83c4bf8d4cee67a857ada13d46c Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Thu, 19 May 2022 14:41:42 +0800 Subject: [PATCH 18/28] fix(os): file path format error --- source/libs/tfs/src/tfs.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 0054a94b80..92beeffa0c 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -161,10 +161,12 @@ bool tfsIsSameFile(const STfsFile *pFile1, const STfsFile *pFile2) { if (pFile1->did.level != pFile2->did.level) return false; if (pFile1->did.id != pFile2->did.id) return false; char nameBuf1[TMPNAME_LEN], nameBuf2[TMPNAME_LEN]; - memset(nameBuf1, 0, TMPNAME_LEN); - memset(nameBuf2, 0, TMPNAME_LEN); - taosRealPath(pFile1->rname, nameBuf1, TMPNAME_LEN); - taosRealPath(pFile2->rname, nameBuf2, TMPNAME_LEN); + strncpy(nameBuf1, pFile1->rname, TMPNAME_LEN); + strncpy(nameBuf2, pFile2->rname, TMPNAME_LEN); + nameBuf1[TMPNAME_LEN - 1] = 0; + nameBuf2[TMPNAME_LEN - 1] = 0; + taosRealPath(nameBuf1, NULL, TMPNAME_LEN); + taosRealPath(nameBuf2, NULL, TMPNAME_LEN); if (strncmp(nameBuf1, nameBuf2, TMPNAME_LEN) != 0) return false; return true; } From 568c6269dcbb57483f47a400d576b3934d76d7f8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 19 May 2022 14:59:55 +0800 Subject: [PATCH 19/28] fix: acquire vnode on restart may deadlock --- source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 8c3b8576a8..41c0b3086b 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -334,19 +334,23 @@ static int32_t vmRequire(const SMgmtInputOpt *pInput, bool *required) { } static int32_t vmStart(SVnodeMgmt *pMgmt) { - taosRLockLatch(&pMgmt->latch); + int32_t numOfVnodes = 0; + SVnodeObj **pVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); - void *pIter = taosHashIterate(pMgmt->hash, NULL); - while (pIter) { - SVnodeObj **ppVnode = pIter; - if (ppVnode == NULL || *ppVnode == NULL) continue; - - SVnodeObj *pVnode = *ppVnode; + for (int32_t i = 0; i < numOfVnodes; ++i) { + SVnodeObj *pVnode = pVnodes[i]; vnodeStart(pVnode->pImpl); - pIter = taosHashIterate(pMgmt->hash, pIter); } - taosRUnLockLatch(&pMgmt->latch); + for (int32_t i = 0; i < numOfVnodes; ++i) { + SVnodeObj *pVnode = pVnodes[i]; + vmReleaseVnode(pMgmt, pVnode); + } + + if (pVnodes != NULL) { + taosMemoryFree(pVnodes); + } + return 0; } From c5d18b5afd0af45ac0cad023f3e3c496a55c86ac Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 19 May 2022 15:08:32 +0800 Subject: [PATCH 20/28] refactor: changel lockfree to rwlock --- source/dnode/mgmt/mgmt_vnode/inc/vmInt.h | 30 ++++++++++----------- source/dnode/mgmt/mgmt_vnode/src/vmFile.c | 4 +-- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 4 +-- source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 19 ++++++------- 4 files changed, 29 insertions(+), 28 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index 7fc10c4237..022d6204ef 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -26,21 +26,21 @@ extern "C" { #endif typedef struct SVnodeMgmt { - SDnodeData *pData; - SMsgCb msgCb; - const char *path; - const char *name; - SQWorkerPool queryPool; - SQWorkerPool fetchPool; - SWWorkerPool syncPool; - SWWorkerPool writePool; - SWWorkerPool mergePool; - SSingleWorker mgmtWorker; - SSingleWorker monitorWorker; - SHashObj *hash; - SRWLatch latch; - SVnodesStat state; - STfs *pTfs; + SDnodeData *pData; + SMsgCb msgCb; + const char *path; + const char *name; + SQWorkerPool queryPool; + SQWorkerPool fetchPool; + SWWorkerPool syncPool; + SWWorkerPool writePool; + SWWorkerPool mergePool; + SSingleWorker mgmtWorker; + SSingleWorker monitorWorker; + SHashObj *hash; + TdThreadRwlock lock; + SVnodesStat state; + STfs *pTfs; } SVnodeMgmt; typedef struct { diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c index 7a6c5f982e..cf5a7ad885 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c @@ -17,7 +17,7 @@ #include "vmInt.h" SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes) { - taosRLockLatch(&pMgmt->latch); + taosThreadRwlockRdlock(&pMgmt->lock); int32_t num = 0; int32_t size = taosHashGetSize(pMgmt->hash); @@ -38,7 +38,7 @@ SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes) { } } - taosRUnLockLatch(&pMgmt->latch); + taosThreadRwlockUnlock(&pMgmt->lock); *numOfVnodes = num; return pVnodes; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index a4da6d089c..602922feeb 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -20,7 +20,7 @@ void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) { pInfo->pVloads = taosArrayInit(pMgmt->state.totalVnodes, sizeof(SVnodeLoad)); if (pInfo->pVloads == NULL) return; - taosRLockLatch(&pMgmt->latch); + taosThreadRwlockRdlock(&pMgmt->lock); void *pIter = taosHashIterate(pMgmt->hash, NULL); while (pIter) { @@ -34,7 +34,7 @@ void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) { pIter = taosHashIterate(pMgmt->hash, pIter); } - taosRUnLockLatch(&pMgmt->latch); + taosThreadRwlockUnlock(&pMgmt->lock); } void vmGetMonitorInfo(SVnodeMgmt *pMgmt, SMonVmInfo *pInfo) { diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 41c0b3086b..0c8d492ef4 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -20,14 +20,14 @@ SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId) { SVnodeObj *pVnode = NULL; int32_t refCount = 0; - taosRLockLatch(&pMgmt->latch); + taosThreadRwlockRdlock(&pMgmt->lock); taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); if (pVnode == NULL) { terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; } else { refCount = atomic_add_fetch_32(&pVnode->refCount, 1); } - taosRUnLockLatch(&pMgmt->latch); + taosThreadRwlockUnlock(&pMgmt->lock); if (pVnode != NULL) { dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); @@ -39,9 +39,9 @@ SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId) { void vmReleaseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { if (pVnode == NULL) return; - taosRLockLatch(&pMgmt->latch); + taosThreadRwlockRdlock(&pMgmt->lock); int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1); - taosRUnLockLatch(&pMgmt->latch); + taosThreadRwlockUnlock(&pMgmt->lock); dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount); } @@ -70,9 +70,9 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { return -1; } - taosWLockLatch(&pMgmt->latch); + taosThreadRwlockWrlock(&pMgmt->lock); int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); - taosWUnLockLatch(&pMgmt->latch); + taosThreadRwlockUnlock(&pMgmt->lock); return code; } @@ -80,9 +80,9 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { char path[TSDB_FILENAME_LEN] = {0}; - taosWLockLatch(&pMgmt->latch); + taosThreadRwlockWrlock(&pMgmt->lock); taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t)); - taosWUnLockLatch(&pMgmt->latch); + taosThreadRwlockUnlock(&pMgmt->lock); vmReleaseVnode(pMgmt, pVnode); while (pVnode->refCount > 0) taosMsleep(10); @@ -239,6 +239,7 @@ static void vmCleanup(SVnodeMgmt *pMgmt) { vmStopWorker(pMgmt); vnodeCleanup(); tfsClose(pMgmt->pTfs); + taosThreadRwlockDestroy(&pMgmt->lock); taosMemoryFree(pMgmt); } @@ -260,7 +261,7 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->msgCb.queueFps[MERGE_QUEUE] = (PutToQueueFp)vmPutRpcMsgToMergeQueue; pMgmt->msgCb.qsizeFp = (GetQueueSizeFp)vmGetQueueSize; pMgmt->msgCb.mgmt = pMgmt; - taosInitRWLatch(&pMgmt->latch); + taosThreadRwlockInit(&pMgmt->lock, NULL); SDiskCfg dCfg = {0}; tstrncpy(dCfg.dir, tsDataDir, TSDB_FILENAME_LEN); From 36229739ff8db69042b2db6f2a4f3b35902c3d1c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 19 May 2022 15:09:17 +0800 Subject: [PATCH 21/28] enh(stream): trigger after processed by tsdb --- source/dnode/vnode/src/vnd/vnodeSvr.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 5c8cd362fd..bcce8326c9 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -62,11 +62,6 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); len = pMsg->contLen - sizeof(SMsgHead); - if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) { - vError("vgId:%d failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno)); - return -1; - } - switch (pMsg->msgType) { /* META */ case TDMT_VND_CREATE_STB: @@ -123,6 +118,11 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg break; } + if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) { + vError("vgId:%d failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno)); + return -1; + } + vDebug("vgId:%d process %s request success, version: %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version); // commit if need From 3c4b91a796fd14019e568a791bdd0831ff7f33e9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 19 May 2022 07:15:00 +0000 Subject: [PATCH 22/28] fix: tdb concurrent w/r --- source/libs/tdb/src/db/tdbPager.c | 3 +- source/libs/tdb/test/tdbTest.cpp | 127 ++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 6b5a3af347..4024cfe745 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -214,6 +214,8 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { } } + pPager->dbOrigSize = pPager->dbFileSize; + // release the page for (pPage = pPager->pDirty; pPage; pPage = pPager->pDirty) { pPager->pDirty = pPage->pDirtyNext; @@ -230,7 +232,6 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { // remote the journal file tdbOsClose(pPager->jfd); tdbOsRemove(pPager->jFileName); - pPager->dbOrigSize = pPager->dbFileSize; pPager->inTran = 0; return 0; diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index a161f1c589..24ef8c0c83 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -1,9 +1,12 @@ #include +#define ALLOW_FORBID_FUNC #include "os.h" #include "tdb.h" #include +#include +#include typedef struct SPoolMem { int64_t size; @@ -480,4 +483,128 @@ TEST(tdb_test, simple_upsert1) { tdbTbClose(pDb); tdbClose(pEnv); +} + +TEST(tdb_test, multi_thread_query) { + int ret; + TDB *pEnv; + TTB *pDb; + tdb_cmpr_fn_t compFunc; + int nData = 20000; + TXN txn; + + taosRemoveDir("tdb"); + + // Open Env + ret = tdbOpen("tdb", 512, 1, &pEnv); + GTEST_ASSERT_EQ(ret, 0); + + // Create a database + compFunc = tKeyCmpr; + ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb); + GTEST_ASSERT_EQ(ret, 0); + + char key[64]; + char val[64]; + int64_t poolLimit = 4096; // 1M pool limit + int64_t txnid = 0; + SPoolMem *pPool; + + // open the pool + pPool = openPool(); + + // start a transaction + txnid++; + txn = {.flags = TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED, + .txnId = -1, + .xMalloc = poolMalloc, + .xFree = poolFree, + .xArg = pPool}; + // tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, ); + tdbBegin(pEnv, &txn); + + for (int iData = 1; iData <= nData; iData++) { + sprintf(key, "key%d", iData); + sprintf(val, "value%d", iData); + ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn); + GTEST_ASSERT_EQ(ret, 0); + + // if pool is full, commit the transaction and start a new one + // if (pPool->size >= poolLimit) { + // break; + // // commit current transaction + // tdbCommit(pEnv, &txn); + // tdbTxnClose(&txn); + + // // start a new transaction + // clearPool(pPool); + // txnid++; + // tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + // tdbBegin(pEnv, &txn); + // } + } + + auto f = [](TTB *pDb, int nData) { + TBC *pDBC; + void *pKey = NULL; + void *pVal = NULL; + int vLen, kLen; + int count = 0; + int ret; + TXN txn; + + SPoolMem *pPool = openPool(); + txn = {.flags = 0, .txnId = 0, .xMalloc = poolMalloc, .xFree = poolFree, .xArg = pPool}; + + ret = tdbTbcOpen(pDb, &pDBC, &txn); + GTEST_ASSERT_EQ(ret, 0); + + tdbTbcMoveToFirst(pDBC); + + for (;;) { + ret = tdbTbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); + if (ret < 0) break; + + // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; + // std::cout.write((char *)pVal, vLen) /* << " " << vLen */; + // std::cout << std::endl; + + count++; + } + + GTEST_ASSERT_EQ(count, nData); + + tdbTbcClose(pDBC); + + tdbFree(pKey); + tdbFree(pVal); + }; + + // tdbCommit(pEnv, &txn); + + // multi-thread query + int nThreads = 20; + std::vector threads; + for (int i = 0; i < nThreads; i++) { + if (i == 0) { + threads.push_back(std::thread(tdbCommit, pEnv, &txn)); + } else { + threads.push_back(std::thread(f, pDb, nData)); + } + } + + for (auto &th : threads) { + th.join(); + } + + // commit the transaction + tdbCommit(pEnv, &txn); + tdbTxnClose(&txn); + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); } \ No newline at end of file From c6f23145f3be3c72ca700fab54c5a41e8d6374f4 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Thu, 19 May 2022 15:18:18 +0800 Subject: [PATCH 23/28] fix(os): file path format error --- source/dnode/vnode/src/meta/metaOpen.c | 1 + source/dnode/vnode/src/tsdb/tsdbOpen.c | 1 + source/dnode/vnode/src/vnd/vnodeOpen.c | 2 ++ tests/tsim/src/simParse.c | 1 + 4 files changed, 5 insertions(+) diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 135f44e939..9a97357b97 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -44,6 +44,7 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { pMeta->path = (char *)&pMeta[1]; sprintf(pMeta->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP, VNODE_META_DIR); + taosRealPath(pMeta->path, NULL, slen); pMeta->pVnode = pVnode; // create path if not created yet diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index 180eea3237..fa54c811ff 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -55,6 +55,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee memcpy(pTsdb->dir, dir, strlen(dir)); pTsdb->path = (char *)&pTsdb[1]; sprintf(pTsdb->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP, dir); + taosRealPath(pTsdb->path, NULL, slen); pTsdb->pVnode = pVnode; pTsdb->repoLocked = false; taosThreadMutexInit(&pTsdb->mutex, NULL); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 739f7f9fa3..9412c39b27 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -109,6 +109,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { // open wal sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_WAL_DIR); + taosRealPath(tdir, NULL, sizeof(tdir)); pVnode->pWal = walOpen(tdir, &(pVnode->config.walCfg)); if (pVnode->pWal == NULL) { vError("vgId:%d failed to open vnode wal since %s", TD_VID(pVnode), tstrerror(terrno)); @@ -117,6 +118,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { // open tq sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_TQ_DIR); + taosRealPath(tdir, NULL, sizeof(tdir)); pVnode->pTq = tqOpen(tdir, pVnode, pVnode->pWal); if (pVnode->pTq == NULL) { vError("vgId:%d failed to open vnode tq since %s", TD_VID(pVnode), tstrerror(terrno)); diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c index a0721941e3..638c4a1ccb 100644 --- a/tests/tsim/src/simParse.c +++ b/tests/tsim/src/simParse.c @@ -183,6 +183,7 @@ SScript *simParseScript(char *fileName) { strcpy(name, fileName); } else { sprintf(name, "%s" TD_DIRSEP "%s", simScriptDir, fileName); + taosRealPath(name, NULL, sizeof(name)); } // if ((fd = fopen(name, "r")) == NULL) { From 380780331da64917d7e9754214ca485cd8343c9f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 19 May 2022 15:19:19 +0800 Subject: [PATCH 24/28] refactor: change lockfree to rwlock --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 8 +++---- source/dnode/mgmt/mgmt_snode/inc/smInt.h | 1 - source/dnode/mgmt/node_mgmt/inc/dmMgmt.h | 2 +- source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 22 +++++++++--------- source/dnode/mgmt/node_mgmt/src/dmNodes.c | 4 ++-- source/dnode/mgmt/node_util/inc/dmUtil.h | 24 ++++++++++---------- source/dnode/mgmt/node_util/src/dmEps.c | 25 ++++++++++----------- 7 files changed, 43 insertions(+), 43 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 7c1162ec10..f7337f482f 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -19,11 +19,11 @@ static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) { if (pMgmt->pData->dnodeId == 0 || pMgmt->pData->clusterId == 0) { dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId); - taosWLockLatch(&pMgmt->pData->latch); + taosThreadRwlockWrlock(&pMgmt->pData->lock); pMgmt->pData->dnodeId = pCfg->dnodeId; pMgmt->pData->clusterId = pCfg->clusterId; dmWriteEps(pMgmt->pData); - taosWUnLockLatch(&pMgmt->pData->latch); + taosThreadRwlockUnlock(&pMgmt->pData->lock); } } @@ -50,7 +50,7 @@ static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { void dmSendStatusReq(SDnodeMgmt *pMgmt) { SStatusReq req = {0}; - taosRLockLatch(&pMgmt->pData->latch); + taosThreadRwlockRdlock(&pMgmt->pData->lock); req.sver = tsVersion; req.dnodeVer = pMgmt->pData->dnodeVer; req.dnodeId = pMgmt->pData->dnodeId; @@ -69,7 +69,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { memcpy(req.clusterCfg.timezone, tsTimezoneStr, TD_TIMEZONE_LEN); memcpy(req.clusterCfg.locale, tsLocale, TD_LOCALE_LEN); memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN); - taosRUnLockLatch(&pMgmt->pData->latch); + taosThreadRwlockUnlock(&pMgmt->pData->lock); SMonVloadInfo vinfo = {0}; (*pMgmt->getVnodeLoadsFp)(&vinfo); diff --git a/source/dnode/mgmt/mgmt_snode/inc/smInt.h b/source/dnode/mgmt/mgmt_snode/inc/smInt.h index 6d0bea9590..68b6ef659e 100644 --- a/source/dnode/mgmt/mgmt_snode/inc/smInt.h +++ b/source/dnode/mgmt/mgmt_snode/inc/smInt.h @@ -30,7 +30,6 @@ typedef struct SSnodeMgmt { SMsgCb msgCb; const char *path; const char *name; - SRWLatch latch; int8_t uniqueWorkerInUse; SArray *uniqueWorkers; // SArray SSingleWorker sharedWorker; diff --git a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h index 9d092a93bc..5818b58801 100644 --- a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h +++ b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h @@ -70,7 +70,7 @@ typedef struct SMgmtWrapper { const char *name; char *path; int32_t refCount; - SRWLatch latch; + TdThreadRwlock lock; EDndNodeType ntype; bool deployed; bool required; diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 3cbb9ff046..96285bbe1c 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -91,7 +91,7 @@ static int32_t dmInitVars(SDnode *pDnode, EDndNodeType rtype) { return -1; } - taosInitRWLatch(&pData->latch); + taosThreadRwlockInit(&pData->lock, NULL); taosThreadMutexInit(&pDnode->mutex, NULL); return 0; } @@ -100,6 +100,7 @@ static void dmClearVars(SDnode *pDnode) { for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) { SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; taosMemoryFreeClear(pWrapper->path); + taosThreadRwlockDestroy(&pWrapper->lock); } if (pDnode->lockfile != NULL) { taosUnLockFile(pDnode->lockfile); @@ -108,7 +109,7 @@ static void dmClearVars(SDnode *pDnode) { } SDnodeData *pData = &pDnode->data; - taosWLockLatch(&pData->latch); + taosThreadRwlockWrlock(&pData->lock); if (pData->dnodeEps != NULL) { taosArrayDestroy(pData->dnodeEps); pData->dnodeEps = NULL; @@ -117,8 +118,9 @@ static void dmClearVars(SDnode *pDnode) { taosHashCleanup(pData->dnodeHash); pData->dnodeHash = NULL; } - taosWUnLockLatch(&pData->latch); + taosThreadRwlockUnlock(&pData->lock); + taosThreadRwlockDestroy(&pData->lock); taosThreadMutexDestroy(&pDnode->mutex); memset(&pDnode->mutex, 0, sizeof(pDnode->mutex)); } @@ -151,7 +153,7 @@ int32_t dmInitDnode(SDnode *pDnode, EDndNodeType rtype) { if (ntype == DNODE) { pWrapper->proc.ptype = DND_PROC_SINGLE; } - taosInitRWLatch(&pWrapper->latch); + taosThreadRwlockInit(&pWrapper->lock, NULL); snprintf(path, sizeof(path), "%s%s%s", tsDataDir, TD_DIRSEP, pWrapper->name); pWrapper->path = strdup(path); @@ -223,7 +225,7 @@ SMgmtWrapper *dmAcquireWrapper(SDnode *pDnode, EDndNodeType ntype) { SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; SMgmtWrapper *pRetWrapper = pWrapper; - taosRLockLatch(&pWrapper->latch); + taosThreadRwlockRdlock(&pWrapper->lock); if (pWrapper->deployed) { int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1); dTrace("node:%s, is acquired, ref:%d", pWrapper->name, refCount); @@ -231,7 +233,7 @@ SMgmtWrapper *dmAcquireWrapper(SDnode *pDnode, EDndNodeType ntype) { terrno = TSDB_CODE_NODE_NOT_DEPLOYED; pRetWrapper = NULL; } - taosRUnLockLatch(&pWrapper->latch); + taosThreadRwlockUnlock(&pWrapper->lock); return pRetWrapper; } @@ -239,7 +241,7 @@ SMgmtWrapper *dmAcquireWrapper(SDnode *pDnode, EDndNodeType ntype) { int32_t dmMarkWrapper(SMgmtWrapper *pWrapper) { int32_t code = 0; - taosRLockLatch(&pWrapper->latch); + taosThreadRwlockRdlock(&pWrapper->lock); if (pWrapper->deployed || (InParentProc(pWrapper) && pWrapper->required)) { int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1); dTrace("node:%s, is marked, ref:%d", pWrapper->name, refCount); @@ -247,7 +249,7 @@ int32_t dmMarkWrapper(SMgmtWrapper *pWrapper) { terrno = TSDB_CODE_NODE_NOT_DEPLOYED; code = -1; } - taosRUnLockLatch(&pWrapper->latch); + taosThreadRwlockUnlock(&pWrapper->lock); return code; } @@ -255,9 +257,9 @@ int32_t dmMarkWrapper(SMgmtWrapper *pWrapper) { void dmReleaseWrapper(SMgmtWrapper *pWrapper) { if (pWrapper == NULL) return; - taosRLockLatch(&pWrapper->latch); + taosThreadRwlockRdlock(&pWrapper->lock); int32_t refCount = atomic_sub_fetch_32(&pWrapper->refCount, 1); - taosRUnLockLatch(&pWrapper->latch); + taosThreadRwlockUnlock(&pWrapper->lock); dTrace("node:%s, is released, ref:%d", pWrapper->name, refCount); } diff --git a/source/dnode/mgmt/node_mgmt/src/dmNodes.c b/source/dnode/mgmt/node_mgmt/src/dmNodes.c index ecfa37725a..ab9d3f67e7 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmNodes.c +++ b/source/dnode/mgmt/node_mgmt/src/dmNodes.c @@ -186,12 +186,12 @@ void dmCloseNode(SMgmtWrapper *pWrapper) { } } - taosWLockLatch(&pWrapper->latch); + taosThreadRwlockWrlock(&pWrapper->lock); if (pWrapper->pMgmt != NULL) { (*pWrapper->func.closeFp)(pWrapper->pMgmt); pWrapper->pMgmt = NULL; } - taosWUnLockLatch(&pWrapper->latch); + taosThreadRwlockUnlock(&pWrapper->lock); if (!OnlyInSingleProc(pWrapper)) { dmCleanupProc(pWrapper); diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index b0e764bf8e..e7256a3a87 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -94,18 +94,18 @@ typedef void (*GetVnodeLoadsFp)(); typedef void (*GetMnodeLoadsFp)(); typedef struct { - int32_t dnodeId; - int64_t clusterId; - int64_t dnodeVer; - int64_t updateTime; - int64_t rebootTime; - bool dropped; - bool stopped; - SEpSet mnodeEps; - SArray *dnodeEps; - SHashObj *dnodeHash; - SRWLatch latch; - SMsgCb msgCb; + int32_t dnodeId; + int64_t clusterId; + int64_t dnodeVer; + int64_t updateTime; + int64_t rebootTime; + bool dropped; + bool stopped; + SEpSet mnodeEps; + SArray *dnodeEps; + SHashObj *dnodeHash; + TdThreadRwlock lock; + SMsgCb msgCb; } SDnodeData; typedef struct { diff --git a/source/dnode/mgmt/node_util/src/dmEps.c b/source/dnode/mgmt/node_util/src/dmEps.c index a6c9fda64d..94fa569557 100644 --- a/source/dnode/mgmt/node_util/src/dmEps.c +++ b/source/dnode/mgmt/node_util/src/dmEps.c @@ -21,7 +21,7 @@ static bool dmIsEpChanged(SDnodeData *pData, int32_t dnodeId, const char *ep); static void dmResetEps(SDnodeData *pData, SArray *dnodeEps); static void dmGetDnodeEp(SDnodeData *pData, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort) { - taosRLockLatch(&pData->latch); + taosThreadRwlockRdlock(&pData->lock); SDnodeEp *pDnodeEp = taosHashGet(pData->dnodeHash, &dnodeId, sizeof(int32_t)); if (pDnodeEp != NULL) { @@ -36,7 +36,7 @@ static void dmGetDnodeEp(SDnodeData *pData, int32_t dnodeId, char *pEp, char *pF } } - taosRUnLockLatch(&pData->latch); + taosThreadRwlockUnlock(&pData->lock); } int32_t dmReadEps(SDnodeData *pData) { @@ -232,7 +232,7 @@ void dmUpdateEps(SDnodeData *pData, SArray *eps) { int32_t numOfEps = taosArrayGetSize(eps); if (numOfEps <= 0) return; - taosWLockLatch(&pData->latch); + taosThreadRwlockWrlock(&pData->lock); int32_t numOfEpsOld = (int32_t)taosArrayGetSize(pData->dnodeEps); if (numOfEps != numOfEpsOld) { @@ -246,7 +246,7 @@ void dmUpdateEps(SDnodeData *pData, SArray *eps) { } } - taosWUnLockLatch(&pData->latch); + taosThreadRwlockUnlock(&pData->lock); } static void dmResetEps(SDnodeData *pData, SArray *dnodeEps) { @@ -292,7 +292,7 @@ static void dmPrintEps(SDnodeData *pData) { static bool dmIsEpChanged(SDnodeData *pData, int32_t dnodeId, const char *ep) { bool changed = false; if (dnodeId == 0) return changed; - taosRLockLatch(&pData->latch); + taosThreadRwlockRdlock(&pData->lock); SDnodeEp *pDnodeEp = taosHashGet(pData->dnodeHash, &dnodeId, sizeof(int32_t)); if (pDnodeEp != NULL) { @@ -304,24 +304,23 @@ static bool dmIsEpChanged(SDnodeData *pData, int32_t dnodeId, const char *ep) { } } - taosRUnLockLatch(&pData->latch); + taosThreadRwlockUnlock(&pData->lock); return changed; } void dmGetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet) { - taosRLockLatch(&pData->latch); + taosThreadRwlockRdlock(&pData->lock); *pEpSet = pData->mnodeEps; - taosRUnLockLatch(&pData->latch); + taosThreadRwlockUnlock(&pData->lock); } void dmSetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet) { - dInfo("mnode is changed, num:%d use:%d", pEpSet->numOfEps, pEpSet->inUse); - - taosWLockLatch(&pData->latch); + taosThreadRwlockWrlock(&pData->lock); pData->mnodeEps = *pEpSet; + taosThreadRwlockUnlock(&pData->lock); + + dInfo("mnode is changed, num:%d use:%d", pEpSet->numOfEps, pEpSet->inUse); for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { dInfo("mnode index:%d %s:%u", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); } - - taosWUnLockLatch(&pData->latch); } From f30e6be9e035b5860109f4425e545a3c60414ae5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 19 May 2022 07:46:36 +0000 Subject: [PATCH 25/28] fix windows compile error --- source/libs/tdb/test/tdbTest.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 24ef8c0c83..bbeeaa3faf 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -515,11 +515,11 @@ TEST(tdb_test, multi_thread_query) { // start a transaction txnid++; - txn = {.flags = TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED, - .txnId = -1, - .xMalloc = poolMalloc, - .xFree = poolFree, - .xArg = pPool}; + txn = (TXN){.flags = TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED, + .txnId = -1, + .xMalloc = poolMalloc, + .xFree = poolFree, + .xArg = pPool}; // tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, ); tdbBegin(pEnv, &txn); @@ -554,7 +554,7 @@ TEST(tdb_test, multi_thread_query) { TXN txn; SPoolMem *pPool = openPool(); - txn = {.flags = 0, .txnId = 0, .xMalloc = poolMalloc, .xFree = poolFree, .xArg = pPool}; + txn = (TXN){.flags = 0, .txnId = 0, .xMalloc = poolMalloc, .xFree = poolFree, .xArg = pPool}; ret = tdbTbcOpen(pDb, &pDBC, &txn); GTEST_ASSERT_EQ(ret, 0); From 2ee38b94fd530421490925dc0b1552ef5893cabf Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 19 May 2022 15:52:35 +0800 Subject: [PATCH 26/28] refactor: change lockfree to rwlock --- include/dnode/mnode/sdb/sdb.h | 34 ++++++------ source/dnode/mnode/sdb/src/sdb.c | 6 ++- source/dnode/mnode/sdb/src/sdbFile.c | 6 +-- source/dnode/mnode/sdb/src/sdbHash.c | 80 ++++++++++++++-------------- 4 files changed, 64 insertions(+), 62 deletions(-) diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index a56c6ca16d..2abe0e5c73 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -333,23 +333,23 @@ SSdbRow *sdbAllocRow(int32_t objSize); void *sdbGetRowObj(SSdbRow *pRow); typedef struct SSdb { - SMnode *pMnode; - char *currDir; - char *syncDir; - char *tmpDir; - int64_t lastCommitVer; - int64_t curVer; - int64_t tableVer[SDB_MAX]; - int64_t maxId[SDB_MAX]; - EKeyType keyTypes[SDB_MAX]; - SHashObj *hashObjs[SDB_MAX]; - SRWLatch locks[SDB_MAX]; - SdbInsertFp insertFps[SDB_MAX]; - SdbUpdateFp updateFps[SDB_MAX]; - SdbDeleteFp deleteFps[SDB_MAX]; - SdbDeployFp deployFps[SDB_MAX]; - SdbEncodeFp encodeFps[SDB_MAX]; - SdbDecodeFp decodeFps[SDB_MAX]; + SMnode *pMnode; + char *currDir; + char *syncDir; + char *tmpDir; + int64_t lastCommitVer; + int64_t curVer; + int64_t tableVer[SDB_MAX]; + int64_t maxId[SDB_MAX]; + EKeyType keyTypes[SDB_MAX]; + SHashObj *hashObjs[SDB_MAX]; + TdThreadRwlock locks[SDB_MAX]; + SdbInsertFp insertFps[SDB_MAX]; + SdbUpdateFp updateFps[SDB_MAX]; + SdbDeleteFp deleteFps[SDB_MAX]; + SdbDeployFp deployFps[SDB_MAX]; + SdbEncodeFp encodeFps[SDB_MAX]; + SdbDecodeFp decodeFps[SDB_MAX]; } SSdb; #ifdef __cplusplus diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index 51f40c12cd..1f11a77e6c 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -48,7 +48,7 @@ SSdb *sdbInit(SSdbOpt *pOption) { } for (ESdbType i = 0; i < SDB_MAX; ++i) { - taosInitRWLatch(&pSdb->locks[i]); + taosThreadRwlockInit(&pSdb->locks[i], NULL); pSdb->maxId[i] = 0; pSdb->tableVer[i] = 0; pSdb->keyTypes[i] = SDB_KEY_INT32; @@ -98,7 +98,10 @@ void sdbCleanup(SSdb *pSdb) { taosHashClear(hash); taosHashCleanup(hash); + taosThreadRwlockDestroy(&pSdb->locks[i]); pSdb->hashObjs[i] = NULL; + memset(&pSdb->locks[i], 0, sizeof(pSdb->locks[i])); + mDebug("sdb table:%s is cleaned up", sdbTableName(i)); } @@ -134,7 +137,6 @@ int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) { pSdb->maxId[sdbType] = 0; pSdb->hashObjs[sdbType] = hash; - taosInitRWLatch(&pSdb->locks[sdbType]); mDebug("sdb table:%s is initialized", sdbTableName(sdbType)); return 0; diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index e9037a7b11..ad1429f667 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -257,8 +257,8 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { mTrace("write %s to file, total %d rows", sdbTableName(i), sdbGetSize(pSdb, i)); SHashObj *hash = pSdb->hashObjs[i]; - SRWLatch *pLock = &pSdb->locks[i]; - taosWLockLatch(pLock); + TdThreadRwlock *pLock = &pSdb->locks[i]; + taosThreadRwlockWrlock(pLock); SSdbRow **ppRow = taosHashIterate(hash, NULL); while (ppRow != NULL) { @@ -303,7 +303,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { sdbFreeRaw(pRaw); ppRow = taosHashIterate(hash, ppRow); } - taosWUnLockLatch(pLock); + taosThreadRwlockUnlock(pLock); } if (code == 0) { diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 94008b2f7c..a25c7a5233 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -129,12 +129,12 @@ static int32_t sdbGetkeySize(SSdb *pSdb, ESdbType type, const void *pKey) { } static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *pRow, int32_t keySize) { - SRWLatch *pLock = &pSdb->locks[pRow->type]; - taosWLockLatch(pLock); + TdThreadRwlock *pLock = &pSdb->locks[pRow->type]; + taosThreadRwlockWrlock(pLock); SSdbRow *pOldRow = taosHashGet(hash, pRow->pObj, keySize); if (pOldRow != NULL) { - taosWUnLockLatch(pLock); + taosThreadRwlockUnlock(pLock); sdbFreeRow(pSdb, pRow, false); terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE; return terrno; @@ -145,13 +145,13 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * sdbPrintOper(pSdb, pRow, "insert"); if (taosHashPut(hash, pRow->pObj, keySize, &pRow, sizeof(void *)) != 0) { - taosWUnLockLatch(pLock); + taosThreadRwlockUnlock(pLock); sdbFreeRow(pSdb, pRow, false); terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } - taosWUnLockLatch(pLock); + taosThreadRwlockUnlock(pLock); int32_t code = 0; SdbInsertFp insertFp = pSdb->insertFps[pRow->type]; @@ -159,9 +159,9 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * code = (*insertFp)(pSdb, pRow->pObj); if (code != 0) { code = terrno; - taosWLockLatch(pLock); + taosThreadRwlockWrlock(pLock); taosHashRemove(hash, pRow->pObj, keySize); - taosWUnLockLatch(pLock); + taosThreadRwlockUnlock(pLock); sdbFreeRow(pSdb, pRow, false); terrno = code; return terrno; @@ -180,19 +180,19 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * } static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *pNewRow, int32_t keySize) { - SRWLatch *pLock = &pSdb->locks[pNewRow->type]; - taosWLockLatch(pLock); + TdThreadRwlock *pLock = &pSdb->locks[pNewRow->type]; + taosThreadRwlockWrlock(pLock); SSdbRow **ppOldRow = taosHashGet(hash, pNewRow->pObj, keySize); if (ppOldRow == NULL || *ppOldRow == NULL) { - taosWUnLockLatch(pLock); + taosThreadRwlockUnlock(pLock); return sdbInsertRow(pSdb, hash, pRaw, pNewRow, keySize); } SSdbRow *pOldRow = *ppOldRow; pOldRow->status = pRaw->status; sdbPrintOper(pSdb, pOldRow, "update"); - taosWUnLockLatch(pLock); + taosThreadRwlockUnlock(pLock); int32_t code = 0; SdbUpdateFp updateFp = pSdb->updateFps[pNewRow->type]; @@ -207,12 +207,12 @@ static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * } static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *pRow, int32_t keySize) { - SRWLatch *pLock = &pSdb->locks[pRow->type]; - taosWLockLatch(pLock); + TdThreadRwlock *pLock = &pSdb->locks[pRow->type]; + taosThreadRwlockWrlock(pLock); SSdbRow **ppOldRow = taosHashGet(hash, pRow->pObj, keySize); if (ppOldRow == NULL || *ppOldRow == NULL) { - taosWUnLockLatch(pLock); + taosThreadRwlockUnlock(pLock); sdbFreeRow(pSdb, pRow, false); terrno = TSDB_CODE_SDB_OBJ_NOT_THERE; return terrno; @@ -223,7 +223,7 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * sdbPrintOper(pSdb, pOldRow, "delete"); taosHashRemove(hash, pOldRow->pObj, keySize); - taosWUnLockLatch(pLock); + taosThreadRwlockUnlock(pLock); pSdb->tableVer[pOldRow->type]++; sdbFreeRow(pSdb, pRow, false); @@ -278,12 +278,12 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey) { void *pRet = NULL; int32_t keySize = sdbGetkeySize(pSdb, type, pKey); - SRWLatch *pLock = &pSdb->locks[type]; - taosRLockLatch(pLock); + TdThreadRwlock *pLock = &pSdb->locks[type]; + taosThreadRwlockRdlock(pLock); SSdbRow **ppRow = taosHashGet(hash, pKey, keySize); if (ppRow == NULL || *ppRow == NULL) { - taosRUnLockLatch(pLock); + taosThreadRwlockUnlock(pLock); terrno = TSDB_CODE_SDB_OBJ_NOT_THERE; return NULL; } @@ -306,13 +306,13 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey) { break; } - taosRUnLockLatch(pLock); + taosThreadRwlockUnlock(pLock); return pRet; } static void sdbCheckRow(SSdb *pSdb, SSdbRow *pRow) { - SRWLatch *pLock = &pSdb->locks[pRow->type]; - taosWLockLatch(pLock); + TdThreadRwlock *pLock = &pSdb->locks[pRow->type]; + taosThreadRwlockWrlock(pLock); int32_t ref = atomic_load_32(&pRow->refCount); sdbPrintOper(pSdb, pRow, "check"); @@ -320,7 +320,7 @@ static void sdbCheckRow(SSdb *pSdb, SSdbRow *pRow) { sdbFreeRow(pSdb, pRow, true); } - taosWUnLockLatch(pLock); + taosThreadRwlockUnlock(pLock); } void sdbRelease(SSdb *pSdb, void *pObj) { @@ -329,8 +329,8 @@ void sdbRelease(SSdb *pSdb, void *pObj) { SSdbRow *pRow = (SSdbRow *)((char *)pObj - sizeof(SSdbRow)); if (pRow->type >= SDB_MAX) return; - SRWLatch *pLock = &pSdb->locks[pRow->type]; - taosWLockLatch(pLock); + TdThreadRwlock *pLock = &pSdb->locks[pRow->type]; + taosThreadRwlockWrlock(pLock); int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1); sdbPrintOper(pSdb, pRow, "release"); @@ -338,7 +338,7 @@ void sdbRelease(SSdb *pSdb, void *pObj) { sdbFreeRow(pSdb, pRow, true); } - taosWUnLockLatch(pLock); + taosThreadRwlockUnlock(pLock); } void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) { @@ -347,8 +347,8 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) { SHashObj *hash = sdbGetHash(pSdb, type); if (hash == NULL) return NULL; - SRWLatch *pLock = &pSdb->locks[type]; - taosRLockLatch(pLock); + TdThreadRwlock *pLock = &pSdb->locks[type]; + taosThreadRwlockRdlock(pLock); SSdbRow **ppRow = taosHashIterate(hash, pIter); while (ppRow != NULL) { @@ -363,7 +363,7 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) { *ppObj = pRow->pObj; break; } - taosRUnLockLatch(pLock); + taosThreadRwlockUnlock(pLock); return ppRow; } @@ -374,18 +374,18 @@ void sdbCancelFetch(SSdb *pSdb, void *pIter) { SHashObj *hash = sdbGetHash(pSdb, pRow->type); if (hash == NULL) return; - SRWLatch *pLock = &pSdb->locks[pRow->type]; - taosRLockLatch(pLock); + TdThreadRwlock *pLock = &pSdb->locks[pRow->type]; + taosThreadRwlockRdlock(pLock); taosHashCancelIterate(hash, pIter); - taosRUnLockLatch(pLock); + taosThreadRwlockUnlock(pLock); } void sdbTraverse(SSdb *pSdb, ESdbType type, sdbTraverseFp fp, void *p1, void *p2, void *p3) { SHashObj *hash = sdbGetHash(pSdb, type); if (hash == NULL) return; - SRWLatch *pLock = &pSdb->locks[type]; - taosRLockLatch(pLock); + TdThreadRwlock *pLock = &pSdb->locks[type]; + taosThreadRwlockRdlock(pLock); SSdbRow **ppRow = taosHashIterate(hash, NULL); while (ppRow != NULL) { @@ -401,17 +401,17 @@ void sdbTraverse(SSdb *pSdb, ESdbType type, sdbTraverseFp fp, void *p1, void *p2 ppRow = taosHashIterate(hash, ppRow); } - taosRUnLockLatch(pLock); + taosThreadRwlockUnlock(pLock); } int32_t sdbGetSize(SSdb *pSdb, ESdbType type) { SHashObj *hash = sdbGetHash(pSdb, type); if (hash == NULL) return 0; - SRWLatch *pLock = &pSdb->locks[type]; - taosRLockLatch(pLock); + TdThreadRwlock *pLock = &pSdb->locks[type]; + taosThreadRwlockRdlock(pLock); int32_t size = taosHashGetSize(hash); - taosRUnLockLatch(pLock); + taosThreadRwlockUnlock(pLock); return size; } @@ -424,8 +424,8 @@ int32_t sdbGetMaxId(SSdb *pSdb, ESdbType type) { int32_t maxId = 0; - SRWLatch *pLock = &pSdb->locks[type]; - taosRLockLatch(pLock); + TdThreadRwlock *pLock = &pSdb->locks[type]; + taosThreadRwlockRdlock(pLock); SSdbRow **ppRow = taosHashIterate(hash, NULL); while (ppRow != NULL) { @@ -435,7 +435,7 @@ int32_t sdbGetMaxId(SSdb *pSdb, ESdbType type) { ppRow = taosHashIterate(hash, ppRow); } - taosRUnLockLatch(pLock); + taosThreadRwlockUnlock(pLock); maxId = TMAX(maxId, pSdb->maxId[type]); return maxId + 1; From c49b3f39d9a19fc4c9a19490ad4fa04528380e6c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 19 May 2022 08:16:06 +0000 Subject: [PATCH 27/28] make windows compile pass --- source/libs/tdb/test/tdbTest.cpp | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index bbeeaa3faf..fee3447884 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -515,11 +515,11 @@ TEST(tdb_test, multi_thread_query) { // start a transaction txnid++; - txn = (TXN){.flags = TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED, - .txnId = -1, - .xMalloc = poolMalloc, - .xFree = poolFree, - .xArg = pPool}; + txn.flags = TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED; + txn.txnId = -1; + txn.xMalloc = poolMalloc; + txn.xFree = poolFree; + txn.xArg = pPool; // tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, ); tdbBegin(pEnv, &txn); @@ -528,20 +528,6 @@ TEST(tdb_test, multi_thread_query) { sprintf(val, "value%d", iData); ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn); GTEST_ASSERT_EQ(ret, 0); - - // if pool is full, commit the transaction and start a new one - // if (pPool->size >= poolLimit) { - // break; - // // commit current transaction - // tdbCommit(pEnv, &txn); - // tdbTxnClose(&txn); - - // // start a new transaction - // clearPool(pPool); - // txnid++; - // tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - // tdbBegin(pEnv, &txn); - // } } auto f = [](TTB *pDb, int nData) { @@ -554,7 +540,11 @@ TEST(tdb_test, multi_thread_query) { TXN txn; SPoolMem *pPool = openPool(); - txn = (TXN){.flags = 0, .txnId = 0, .xMalloc = poolMalloc, .xFree = poolFree, .xArg = pPool}; + txn.flags = 0; + txn.txnId = 0; + txn.xMalloc = poolMalloc; + txn.xFree = poolFree; + txn.xArg = pPool; ret = tdbTbcOpen(pDb, &pDBC, &txn); GTEST_ASSERT_EQ(ret, 0); From 90a6ec0917c2ab2bc7789319aea068b61b3e24ec Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 19 May 2022 16:16:16 +0800 Subject: [PATCH 28/28] fix(query): release meta reader after initializing it. --- source/libs/executor/src/executorimpl.c | 8 ++------ tests/system-test/2-query/abs.py | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index bd93284da2..08521f91e8 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1850,12 +1850,6 @@ void setFunctionResultOutput(SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t pCtx[i].resultInfo = pEntry; pCtx[i].scanFlag = stage; - - // set the timestamp output buffer for top/bottom/diff query - // int32_t fid = pCtx[i].functionId; - // if (fid == FUNCTION_TOP || fid == FUNCTION_BOTTOM || fid == FUNCTION_DIFF || fid == FUNCTION_DERIVATIVE) { - // if (i > 0) pCtx[i].pTsOutput = pCtx[i-1].pOutput; - // } } initCtxOutputBuffer(pCtx, pDataBlock->info.numOfCols); @@ -4672,6 +4666,8 @@ void extractTableSchemaVersion(SReadHandle *pHandle, uint64_t uid, SExecTaskInfo } else { pTaskInfo->schemaVer.sversion = mr.me.ntbEntry.schema.sver; } + + metaReaderClear(&mr); } SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index ccf83df952..a3e976b490 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -13,7 +13,7 @@ class TDTestCase: "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), True) def prepare_datas(self): tdSql.execute(