From 4385275a89f49ef1261f298dd9b0ed27ac9b1f46 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 6 Mar 2023 17:32:12 +0800 Subject: [PATCH 01/38] fix: return err while db is creating --- source/dnode/mnode/impl/src/mndDb.c | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index af76971304..ef5f13c199 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -616,13 +616,8 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { } } else { if (terrno == TSDB_CODE_MND_DB_IN_CREATING) { - if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, createReq.db) == 0) { - mInfo("db:%s, is creating and createdb response after trans finished", createReq.db); - code = TSDB_CODE_ACTION_IN_PROGRESS; - goto _OVER; - } else { - goto _OVER; - } + code = terrno; + goto _OVER; } else if (terrno == TSDB_CODE_MND_DB_IN_DROPPING) { goto _OVER; } else if (terrno == TSDB_CODE_MND_DB_NOT_EXIST) { @@ -1258,14 +1253,9 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) { usedbRsp.errCode = terrno; if (terrno == TSDB_CODE_MND_DB_IN_CREATING) { - if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, usedbReq.db) == 0) { - mInfo("db:%s, is creating and usedb response after trans finished", usedbReq.db); - code = TSDB_CODE_ACTION_IN_PROGRESS; - goto _OVER; - } + code = terrno; + goto _OVER; } - - mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); } else { if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_USE_DB, pDb) != 0) { goto _OVER; From d3c9e0800378ffb2ddd759a87dd6ba67445ae811 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 14 Mar 2023 06:15:47 +0000 Subject: [PATCH 02/38] fix: fix log invalid err msg --- source/dnode/mnode/impl/src/mndMain.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index b09a4f63a7..0b8901a97e 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -706,6 +706,7 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg) { } else if (code == 0) { mGTrace("msg:%p, successfully processed", pMsg); } else { + code = terrno; mGError("msg:%p, failed to process since %s, app:%p type:%s", pMsg, tstrerror(code), pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); } From bab3daaa731597376525ce9764594d6a3b340fa1 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 14 Mar 2023 07:23:38 +0000 Subject: [PATCH 03/38] fix: Temporarily delete a test case --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 0945c4a7f7..67e27091c0 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1094,7 +1094,7 @@ ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py #,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py -R -,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R +#,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R #docs-examples test ,,n,docs-examples-test,bash python.sh From c7a10ae5a80acd65b42359593a4bd2d238db7191 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 14 Mar 2023 15:52:39 +0800 Subject: [PATCH 04/38] fix:use table suid in taosx to avoid suid is not same for using suid --- source/client/src/clientRawBlockWrite.c | 48 ++++++++++++++++++++----- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 6cf38924bf..87f3ec812b 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -776,12 +776,32 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { goto end; } + SCatalog* pCatalog = NULL; + code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + SRequestConnInfo conn = {.pTrans = pRequest->pTscObj->pAppInfo->pTransporter, + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)}; + SName pName = {0}; + toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, &pName); + STableMeta *pTableMeta = NULL; + code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); + if (code != TSDB_CODE_SUCCESS) { + uError("taosCreateTable:catalogGetTableMeta failed. table name: %s", req.name); + goto end; + } + pReq.suid = pTableMeta->uid; + taosMemoryFreeClear(pTableMeta); + // build drop stable pReq.igNotExists = true; pReq.source = TD_REQ_FROM_TAOX; - pReq.suid = processSuid(req.suid, pRequest->pDb); +// pReq.suid = processSuid(req.suid, pRequest->pDb); - uDebug("taosDropStb name:%s suid:%"PRId64" processSuid:%"PRId64, req.name, req.suid, pReq.suid); + uDebug("taosDropStb name:%s suid:%"PRId64" new suid:%"PRId64, req.name, req.suid, pReq.suid); STscObj* pTscObj = pRequest->pTscObj; SName tableName = {0}; tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name); @@ -806,7 +826,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { launchQueryImpl(pRequest, &pQuery, true, NULL); if (pRequest->code == TSDB_CODE_SUCCESS) { - SCatalog* pCatalog = NULL; +// SCatalog* pCatalog = NULL; catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); catalogRemoveTableMeta(pCatalog, &tableName); } @@ -900,15 +920,15 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { STableMeta* pTableMeta = NULL; SName sName = {0}; tb_uid_t oldSuid = pCreateReq->ctb.suid; - pCreateReq->ctb.suid = processSuid(pCreateReq->ctb.suid, pRequest->pDb); - uDebug("taosCreateTable name:%s sname:%s suid:%"PRId64" processSuid:%"PRId64, pCreateReq->name, pCreateReq->ctb.stbName, pCreateReq->ctb.suid, oldSuid); - +// pCreateReq->ctb.suid = processSuid(pCreateReq->ctb.suid, pRequest->pDb); toName(pTscObj->acctId, pRequest->pDb, pCreateReq->ctb.stbName, &sName); code = catalogGetTableMeta(pCatalog, &conn, &sName, &pTableMeta); if (code != TSDB_CODE_SUCCESS) { uError("taosCreateTable:catalogGetTableMeta failed. table name: %s", pCreateReq->ctb.stbName); goto end; } + pCreateReq->ctb.suid = pTableMeta->uid; + uDebug("taosCreateTable name:%s sname:%s suid:%"PRId64" new suid:%"PRId64, pCreateReq->name, pCreateReq->ctb.stbName, oldSuid, pCreateReq->ctb.suid); for (int32_t i = 0; i < taosArrayGetSize(pCreateReq->ctb.tagName); i++) { char* tName = taosArrayGet(pCreateReq->ctb.tagName, i); @@ -1041,8 +1061,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { pDropReq = req.pReqs + iReq; pDropReq->igNotExists = true; - pDropReq->suid = processSuid(pDropReq->suid, pRequest->pDb); - uDebug("taosDropTable name:%s suid:%"PRId64" processSuid:%"PRId64, pDropReq->name, pDropReq->suid, pDropReq->suid); +// pDropReq->suid = processSuid(pDropReq->suid, pRequest->pDb); SVgroupInfo pInfo = {0}; SName pName = {0}; @@ -1052,6 +1071,17 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { goto end; } + STableMeta *pTableMeta = NULL; + code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); + if (code != TSDB_CODE_SUCCESS) { + uError("taosDropTable:catalogGetTableMeta failed. table name: %s", pDropReq->name); + goto end; + } + tb_uid_t oldSuid = pDropReq->suid; + pDropReq->suid = pTableMeta->suid; + taosMemoryFreeClear(pTableMeta); + uDebug("taosDropTable name:%s suid:%"PRId64" new suid:%"PRId64, pDropReq->name, oldSuid, pDropReq->suid); + taosArrayPush(pRequest->tableList, &pName); SVgroupDropTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId)); if (pTableBatch == NULL) { @@ -1673,7 +1703,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) } if (strcmp(tbName, pCreateReq.name) == 0) { cloneSVreateTbReq(&pCreateReq, &pCreateReqDst); - pCreateReqDst->ctb.suid = processSuid(pCreateReqDst->ctb.suid, pRequest->pDb); +// pCreateReqDst->ctb.suid = processSuid(pCreateReqDst->ctb.suid, pRequest->pDb); tDecoderClear(&decoderTmp); break; } From 72c2631c0de709117530e3a05bee6a10d561bacd Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 14 Mar 2023 17:52:16 +0800 Subject: [PATCH 05/38] fix:use table suid in taosx to avoid suid is not same for using suid --- source/client/src/clientRawBlockWrite.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 87f3ec812b..cfd1005181 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -789,6 +789,11 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, &pName); STableMeta *pTableMeta = NULL; code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); + if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST){ + code = TSDB_CODE_SUCCESS; + taosMemoryFreeClear(pTableMeta); + goto end; + } if (code != TSDB_CODE_SUCCESS) { uError("taosCreateTable:catalogGetTableMeta failed. table name: %s", req.name); goto end; @@ -1073,6 +1078,11 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { STableMeta *pTableMeta = NULL; code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); + if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST){ + code = TSDB_CODE_SUCCESS; + taosMemoryFreeClear(pTableMeta); + continue; + } if (code != TSDB_CODE_SUCCESS) { uError("taosDropTable:catalogGetTableMeta failed. table name: %s", pDropReq->name); goto end; @@ -1096,6 +1106,9 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { } } + if (taosHashGetSize(pVgroupHashmap) == 0){ + goto end; + } SArray* pBufArray = serializeVgroupsDropTableBatch(pVgroupHashmap); if (NULL == pBufArray) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -1736,6 +1749,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) if (pCreateReqDst) { pTableMeta->vgId = vg.vgId; pTableMeta->uid = pCreateReqDst->uid; + pCreateReqDst->ctb.suid = pTableMeta->suid; } void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId)); if (hData == NULL) { From 525a817d80fa07dde022d806f299fe689b05760d Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Wed, 15 Mar 2023 01:41:27 +0800 Subject: [PATCH 06/38] test:reduce to 1000 rowNumbers per tables in testcases --- .../tsim/parser/col_arithmetic_operation.sim | 3 +- .../tsim/parser/col_arithmetic_query.sim | 33 +++--- tests/script/tsim/parser/commit.sim | 2 +- tests/script/tsim/parser/groupby.sim | 80 +++++++------- tests/script/tsim/parser/interp.sim | 2 +- tests/script/tsim/parser/join_manyblocks.sim | 6 +- tests/script/tsim/parser/limit1.sim | 2 +- tests/script/tsim/parser/limit1_stb.sim | 5 +- tests/script/tsim/parser/limit1_tb.sim | 36 ++++--- tests/script/tsim/parser/nestquery.sim | 10 +- .../tsim/parser/projection_limit_offset.sim | 84 +++++++-------- tests/script/tsim/parser/selectResNum.sim | 4 +- tests/script/tsim/parser/sliding.sim | 102 +++++++++--------- tests/script/tsim/parser/union.sim | 22 ++-- tests/script/tsim/parser/where.sim | 10 +- tests/script/tsim/valgrind/checkError7.sim | 12 ++- 16 files changed, 211 insertions(+), 202 deletions(-) diff --git a/tests/script/tsim/parser/col_arithmetic_operation.sim b/tests/script/tsim/parser/col_arithmetic_operation.sim index 9a2ba34c85..a11b99d3d6 100644 --- a/tests/script/tsim/parser/col_arithmetic_operation.sim +++ b/tests/script/tsim/parser/col_arithmetic_operation.sim @@ -9,7 +9,7 @@ $dbPrefix = ca_db $tbPrefix = ca_tb $stbPrefix = ca_stb $tbNum = 10 -$rowNum = 10000 +$rowNum = 1000 $totalNum = $tbNum * $rowNum $ts0 = 1537146000000 $delta = 600000 @@ -18,6 +18,7 @@ $i = 0 $db = $dbPrefix . $i $stb = $stbPrefix . $i +print drop database $db -x step1 sql drop database $db -x step1 step1: sql create database $db diff --git a/tests/script/tsim/parser/col_arithmetic_query.sim b/tests/script/tsim/parser/col_arithmetic_query.sim index b77dcbe498..1537db9a83 100644 --- a/tests/script/tsim/parser/col_arithmetic_query.sim +++ b/tests/script/tsim/parser/col_arithmetic_query.sim @@ -4,7 +4,7 @@ $dbPrefix = ca_db $tbPrefix = ca_tb $stbPrefix = ca_stb -$rowNum = 10000 +$rowNum = 1000 $i = 0 $db = $dbPrefix . $i @@ -33,7 +33,7 @@ endi # asc/desc order [d.2] ====================================================== sql select c1 *( 2 / 3 ), c1/c1 from $tb order by ts asc; -if $rows != 10000 then +if $rows != 1000 then return -1 endi if $data00 != 0.000000000 then @@ -57,7 +57,7 @@ if $data91 != 1.000000000 then endi sql select (c1 * 2) % 7.9, c1*1, c1*1*1, c1*c1, c1*c1*c1 from $tb order by ts desc; -if $rows != 10000 then +if $rows != 1000 then return -1 endi if $data00 != 2.200000000 then @@ -151,7 +151,7 @@ sql select top(c1,1) - 88 from $tb # all data types [d.6] ================================================================ sql select c2-c1*1.1, c3/c2, c4*c3, c5%c4, (c6+c4)%22, c2-c2 from $tb -if $rows != 10000 then +if $rows != 1000 then return -1 endi if $data00 != 0.000000000 then @@ -221,7 +221,7 @@ if $data90 != 76.000000000 then return -1 endi -sql select c4 / 99.123 from $tb limit 10 offset 9999; +sql select c4 / 99.123 from $tb limit 10 offset 999; if $rows != 1 then return -1 endi @@ -237,7 +237,7 @@ sql_error select c2-c2, c3-c4, c5%c3 from $tb fill(value, 12); # constant column. [d.13]============================================================== sql select c1, c2+c6, 12.9876545678, 1, 1.1 from $tb -if $rows != 10000 then +if $rows != 1000 then return -1 endi if $data00 != 0 then @@ -261,7 +261,7 @@ endi # column value filter [d.14]=========================================================== sql select c1, c2+c6, 12.9876545678, 1, 1.1 from $tb where c1<2 -if $rows != 2000 then +if $rows != 200 then return -1 endi if $data00 != 0 then @@ -338,13 +338,14 @@ sql select (count(c1) * 2) % 7.9, (count(c1) * 2), ( count(1)*2) from $stb if $rows != 1 then return -1 endi -if $data00 != 1.800000000 then +print $data00 +if $data00 != 6.500000000 then return -1 endi -if $data01 != 100000.000000000 then +if $data01 != 10000.000000000 then return -1 endi -if $data02 != 200000.000000000 then +if $data02 != 20000.000000000 then return -1 endi @@ -371,7 +372,7 @@ endi if $data00 != 0.000000000 then return -1 endi -if $data01 != 225000.000000000 then +if $data01 != 22500.000000000 then return -1 endi if $data02 != 8.077777778 then @@ -383,7 +384,7 @@ endi if $data04 != 0.444444444 then return -1 endi -if $data05 != 450000.000000000 then +if $data05 != 45000.000000000 then return -1 endi @@ -484,10 +485,10 @@ endi if $data10 != 0.000000000 then return -1 endi -if $data20 != 0.997600000 then +if $data20 != 0.976000000 then return -1 endi -if $data90 != 7.980800000 then +if $data90 != 7.808000000 then return -1 endi @@ -496,7 +497,7 @@ sql select first(c6) - sum(c6) + 12 from $stb limit 12 offset 0; if $rows != 1 then return -1 endi -if $data00 != -449988.000000000 then +if $data00 != -44988.000000000 then return -1 endi @@ -546,7 +547,7 @@ endi # interval query [d.17]=============================================================== sql select avg(c2)*count(c2), sum(c3)-first(c3), last(c4)+9 from ca_stb0 interval(1s) -if $rows != 10000 then +if $rows != 1000 then return -1 endi diff --git a/tests/script/tsim/parser/commit.sim b/tests/script/tsim/parser/commit.sim index a9bf8b26eb..84610ad379 100644 --- a/tests/script/tsim/parser/commit.sim +++ b/tests/script/tsim/parser/commit.sim @@ -7,7 +7,7 @@ $dbPrefix = sc_db $tbPrefix = sc_tb $stbPrefix = sc_stb $tbNum = 10 -$rowNum = 10000 +$rowNum = 1000 $totalNum = $tbNum * $rowNum $loops = 5 $log = 1 diff --git a/tests/script/tsim/parser/groupby.sim b/tests/script/tsim/parser/groupby.sim index 68e70c5765..637b7553fd 100644 --- a/tests/script/tsim/parser/groupby.sim +++ b/tests/script/tsim/parser/groupby.sim @@ -7,7 +7,7 @@ $dbPrefix = group_db $tbPrefix = group_tb $mtPrefix = group_mt $tbNum = 8 -$rowNum = 10000 +$rowNum = 1000 $totalNum = $tbNum * $rowNum print =============== groupby.sim @@ -88,7 +88,7 @@ if $row != 20 then return -1 endi -if $data00 != 100 then +if $data00 != 10 then return -1 endi @@ -96,7 +96,7 @@ if $data01 != 0 then return -1 endi -if $data10 != 100 then +if $data10 != 10 then return -1 endi @@ -186,7 +186,7 @@ if $data04 != 0.00000 then return -1 endi -if $data10 != 100 then +if $data10 != 10 then return -1 endi @@ -227,7 +227,7 @@ if $row != 20 then return -1 endi -if $data00 != 800 then +if $data00 != 80 then return -1 endi @@ -235,7 +235,7 @@ if $data01 != 0 then return -1 endi -if $data10 != 800 then +if $data10 != 80 then return -1 endi @@ -243,7 +243,7 @@ if $data11 != 1 then return -1 endi -if $data90 != 800 then +if $data90 != 80 then return -1 endi @@ -297,7 +297,7 @@ if $data00 != $data03 then return -1 endi -if $data01 != @70-01-01 08:01:49.900@ then +if $data01 != @70-01-01 08:01:40.900@ then return -1 endi @@ -305,7 +305,7 @@ if $data02 != @70-01-01 08:01:40.000@ then return -1 endi -if $data07 != 800 then +if $data07 != 80 then return -1 endi @@ -313,7 +313,7 @@ if $data10 != $data13 then return -1 endi -if $data11 != @70-01-01 08:01:49.901@ then +if $data11 != @70-01-01 08:01:40.901@ then return -1 endi @@ -321,7 +321,7 @@ if $data12 != @70-01-01 08:01:40.001@ then return -1 endi -if $data17 != 800 then +if $data17 != 80 then return -1 endi @@ -329,7 +329,7 @@ if $data90 != $data93 then return -1 endi -if $data91 != @70-01-01 08:01:49.909@ then +if $data91 != @70-01-01 08:01:40.909@ then return -1 endi @@ -337,11 +337,11 @@ if $data92 != @70-01-01 08:01:40.009@ then return -1 endi -if $data97 != 800 then +if $data97 != 80 then return -1 endi -if $data95 != 7200 then +if $data95 != 720 then return -1 endi @@ -358,7 +358,7 @@ if $data00 != 0 then return -1 endi -if $data11 != 800 then +if $data11 != 80 then return -1 endi @@ -372,7 +372,7 @@ if $data00 != 0 then return -1 endi -if $data01 != @70-01-01 08:01:49.900@ then +if $data01 != @70-01-01 08:01:40.900@ then return -1 endi @@ -389,13 +389,13 @@ if $data04 != 0 then return -1 endi -if $data06 != 100 then +if $data06 != 10 then return -1 endi sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4 order by c4; -if $rows != 10000 then +if $rows != 1000 then return -1 endi @@ -420,13 +420,13 @@ if $rows != 1 then return -1 endi -sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4 slimit 20 soffset 9990; +sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4 slimit 20 soffset 990; if $rows != 10 then return -1 endi sql select count(*),first(ts),last(ts),min(c3),max(c3),sum(c3),avg(c3),sum(c4)/count(c4) from group_tb1 group by c4; -if $rows != 10000 then +if $rows != 1000 then return -1 endi @@ -441,32 +441,32 @@ if $rows != 100 then return -1 endi -if $data00 != 100 then +if $data00 != 10 then return -1 endi -if $data01 != 495000 then +if $data01 != 4500 then return -1 endi -if $data02 != 100 then +if $data02 != 10 then return -1 endi -if $data03 != 4950.000000000 then - print expect 4950.000000000 , acutal $data03 +if $data03 != 450.000000000 then + print expect 450.000000000 , acutal $data03 return -1 endi -if $data10 != 100 then +if $data10 != 10 then return -1 endi -if $data11 != 495100 then +if $data11 != 4510 then return -1 endi -if $data13 != 4951.000000000 then +if $data13 != 451.000000000 then return -1 endi @@ -481,19 +481,19 @@ if $rows != 1 then return -1 endi -#if $data00 != 79200.000000000 then +#if $data00 != 2160.000000000 then # return -1 #endi -#if $data01 != @binary99@ then +#if $data01 != @binary27@ then # return -1 #endi -#if $data02 != 99.000000000 then +#if $data02 != 27.000000000 then # return -1 #endi -#if $data03 != 99.000000000 then +#if $data03 != 27.000000000 then # return -1 #endi @@ -503,7 +503,7 @@ if $rows != 100 then return -1 endi -if $data00 != 4851.000000000 then +if $data00 != 441.000000000 then return -1 endi @@ -511,19 +511,19 @@ if $data01 != 0 then return -1 endi -if $data02 != 9900 then +if $data02 != 900 then return -1 endi -if $data03 != 4950.000000000 then +if $data03 != 450.000000000 then return -1 endi -if $data04 != 2886.607004772 then +if $data04 != 287.228132327 then return -1 endi -if $data10 != 4852.000000000 then +if $data10 != 442.000000000 then return -1 endi @@ -531,15 +531,15 @@ if $data11 != 1 then return -1 endi -if $data12 != 9901 then +if $data12 != 901 then return -1 endi -if $data13 != 4951.000000000 then +if $data13 != 451.000000000 then return -1 endi -if $data14 != 2886.607004772 then +if $data14 != 287.228132327 then return -1 endi diff --git a/tests/script/tsim/parser/interp.sim b/tests/script/tsim/parser/interp.sim index e6512a22d7..f74a2dc624 100644 --- a/tests/script/tsim/parser/interp.sim +++ b/tests/script/tsim/parser/interp.sim @@ -7,7 +7,7 @@ $dbPrefix = intp_db $tbPrefix = intp_tb $stbPrefix = intp_stb $tbNum = 4 -$rowNum = 10000 +$rowNum = 1000 $totalNum = $tbNum * $rowNum $ts0 = 1537146000000 $delta = 600000 diff --git a/tests/script/tsim/parser/join_manyblocks.sim b/tests/script/tsim/parser/join_manyblocks.sim index 154316a03f..a40a75f50c 100644 --- a/tests/script/tsim/parser/join_manyblocks.sim +++ b/tests/script/tsim/parser/join_manyblocks.sim @@ -7,7 +7,7 @@ $dbPrefix = join_m_db $tbPrefix = join_tb $mtPrefix = join_mt $tbNum = 3 -$rowNum = 20000 +$rowNum = 2000 $totalNum = $tbNum * $rowNum print =============== join_manyBlocks.sim @@ -78,8 +78,8 @@ print ==============> td-3313 sql select join_mt0.ts,join_mt0.ts,join_mt0.t1 from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t1=join_mt1.t1; print $row -if $row != 60000 then - print expect 60000, actual: $row +if $row != 6000 then + print expect 6000, actual: $row return -1 endi diff --git a/tests/script/tsim/parser/limit1.sim b/tests/script/tsim/parser/limit1.sim index c35617fc4f..d1a75f3ba9 100644 --- a/tests/script/tsim/parser/limit1.sim +++ b/tests/script/tsim/parser/limit1.sim @@ -7,7 +7,7 @@ $dbPrefix = lm1_db $tbPrefix = lm1_tb $stbPrefix = lm1_stb $tbNum = 10 -$rowNum = 10000 +$rowNum = 1000 $totalNum = $tbNum * $rowNum $ts0 = 1537146000000 $delta = 600000 diff --git a/tests/script/tsim/parser/limit1_stb.sim b/tests/script/tsim/parser/limit1_stb.sim index be0963c0fd..1a5d57efbc 100644 --- a/tests/script/tsim/parser/limit1_stb.sim +++ b/tests/script/tsim/parser/limit1_stb.sim @@ -4,7 +4,7 @@ $dbPrefix = lm1_db $tbPrefix = lm1_tb $stbPrefix = lm1_stb $tbNum = 10 -$rowNum = 10000 +$rowNum = 1000 $totalNum = $tbNum * $rowNum $ts0 = 1537146000000 $delta = 600000 @@ -453,13 +453,14 @@ endi ### [TBASE-361] $offset = $rowNum / 2 $offset = $offset + 1 +print === select _wstart, max(c1), min(c2), avg(c3), count(c4), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 1 and t1 < 8 interval(5m) limit $offset offset $offset sql select _wstart, max(c1), min(c2), avg(c3), count(c4), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 1 and t1 < 8 interval(5m) limit $offset offset $offset $val = $rowNum - $offset if $rows != $val then print expect $val, actual:$rows return -1 endi -if $data00 != @18-10-22 02:30:00.000@ then +if $data00 != @18-09-20 20:30:00.000@ then return -1 endi if $data01 != 1 then diff --git a/tests/script/tsim/parser/limit1_tb.sim b/tests/script/tsim/parser/limit1_tb.sim index 0ba8265418..e6407e3024 100644 --- a/tests/script/tsim/parser/limit1_tb.sim +++ b/tests/script/tsim/parser/limit1_tb.sim @@ -4,7 +4,7 @@ $dbPrefix = lm1_db $tbPrefix = lm1_tb $stbPrefix = lm1_stb $tbNum = 10 -$rowNum = 10000 +$rowNum = 1000 $totalNum = $tbNum * $rowNum $ts0 = 1537146000000 $delta = 600000 @@ -13,7 +13,7 @@ $i = 0 $db = $dbPrefix . $i $stb = $stbPrefix . $i -print ====== use db +print ====== use $db sql use $db ##### select from table @@ -664,9 +664,9 @@ endi if $data21 != 4.027681991 then return -1 endi - +print select _wstart, count(c1), count(c2), count(c3), count(c4), count(c5), count(c6) from $tb where ts >= $ts0 and ts <= $tsu interval(27m) sql select _wstart, count(c1), count(c2), count(c3), count(c4), count(c5), count(c6) from $tb where ts >= $ts0 and ts <= $tsu interval(27m) -if $rows != 3704 then +if $rows != 371 then return -1 endi if $data01 != 2 then @@ -994,11 +994,14 @@ endi $offset = $rowNum / 10 $offset = $offset * 3 $offset = $offset - 1 + +print === select * from $tb where c1 < 5 and c1 > 1 order by ts asc limit 3 offset $offset sql select * from $tb where c1 < 5 and c1 > 1 order by ts asc limit 3 offset $offset if $rows != 1 then return -1 endi -if $data00 != @18-11-25 18:40:00.000@ then +print $data00 +if $data00 != @18-09-24 06:40:00.000@ then return -1 endi if $data01 != 4 then @@ -1012,19 +1015,19 @@ sql select * from $tb order by ts desc limit 5 if $rows != 5 then return -1 endi -if $data00 != @18-11-25 19:30:00.000@ then +if $data00 != @18-09-24 07:30:00.000@ then return -1 endi if $data01 != 9 then return -1 endi -if $data10 != @18-11-25 19:20:00.000@ then +if $data10 != @18-09-24 07:20:00.000@ then return -1 endi if $data12 != 8 then return -1 endi -if $data20 != @18-11-25 19:10:00.000@ then +if $data20 != @18-09-24 07:10:00.000@ then return -1 endi if $data23 != 7.00000 then @@ -1048,7 +1051,7 @@ endi if $data39 != nchar6 then return -1 endi -if $data40 != @18-11-25 18:50:00.000@ then +if $data40 != @18-09-24 06:50:00.000@ then return -1 endi @@ -1056,19 +1059,19 @@ sql select * from $tb order by ts desc limit 5 offset 1 if $rows != 5 then return -1 endi -if $data00 != @18-11-25 19:20:00.000@ then +if $data00 != @18-09-24 07:20:00.000@ then return -1 endi if $data01 != 8 then return -1 endi -if $data10 != @18-11-25 19:10:00.000@ then +if $data10 != @18-09-24 07:10:00.000@ then return -1 endi if $data12 != 7 then return -1 endi -if $data20 != @18-11-25 19:00:00.000@ then +if $data20 != @18-09-24 07:00:00.000@ then return -1 endi if $data23 != 6.00000 then @@ -1092,12 +1095,13 @@ endi if $data39 != nchar5 then return -1 endi -if $data40 != @18-11-25 18:40:00.000@ then +if $data40 != @18-09-24 06:40:00.000@ then return -1 endi $offset = $rowNum $offset = $offset - 2 +print ==== select * from $tb order by ts desc limit 5 offset $offset sql select * from $tb order by ts desc limit 5 offset $offset if $rows != 2 then return -1 @@ -1119,16 +1123,16 @@ sql select * from $tb where c1 < 8 order by ts desc limit 3 offset 2 if $rows != 3 then return -1 endi -if $data00 != @18-11-25 18:50:00.000@ then +if $data00 != @18-09-24 06:50:00.000@ then return -1 endi if $data01 != 5 then return -1 endi -if $data10 != @18-11-25 18:40:00.000@ then +if $data10 != @18-09-24 06:40:00.000@ then return -1 endi -if $data20 != @18-11-25 18:30:00.000@ then +if $data20 != @18-09-24 06:30:00.000@ then return -1 endi if $data12 != 4 then diff --git a/tests/script/tsim/parser/nestquery.sim b/tests/script/tsim/parser/nestquery.sim index 2a363de43d..c2189d0bdd 100644 --- a/tests/script/tsim/parser/nestquery.sim +++ b/tests/script/tsim/parser/nestquery.sim @@ -8,7 +8,7 @@ print ======================== dnode1 start $dbPrefix = nest_db $tbPrefix = nest_tb $mtPrefix = nest_mt -$tbNum = 10 +$tbNum = 3 $rowNum = 10000 $totalNum = $tbNum * $rowNum @@ -23,7 +23,7 @@ sql create database if not exists $db sql use $db sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int) -$half = $tbNum / 2 +$half = 2 $i = 0 while $i < $half @@ -69,7 +69,7 @@ sql select count(*) from (select count(*) from nest_mt0 group by tbname) if $rows != 1 then return -1 endi -if $data00 != 10 then +if $data00 != 4 then return -1 endi @@ -77,7 +77,7 @@ sql select count(*) from (select count(*) from nest_mt0 partition by tbname inte if $rows != 1 then return -1 endi -if $data00 != 170 then +if $data00 != 68 then return -1 endi @@ -85,7 +85,7 @@ sql select sum(a) from (select count(*) a from nest_mt0 partition by tbname inte if $rows != 1 then return -1 endi -if $data00 != 100000 then +if $data00 != 40000 then return -1 endi diff --git a/tests/script/tsim/parser/projection_limit_offset.sim b/tests/script/tsim/parser/projection_limit_offset.sim index 669ff3a179..2d99b0a296 100644 --- a/tests/script/tsim/parser/projection_limit_offset.sim +++ b/tests/script/tsim/parser/projection_limit_offset.sim @@ -7,7 +7,7 @@ $dbPrefix = group_db $tbPrefix = group_tb $mtPrefix = group_mt $tbNum = 8 -$rowNum = 10000 +$rowNum = 1000 $totalNum = $tbNum * $rowNum print =============== projection_limit_offset.sim @@ -81,199 +81,199 @@ $ts2 = $tb2 . .ts sql select ts from group_mt0 print $rows -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 0; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 0; if $rows != 4008 then print expect 4008, actual:$rows return -1 endi -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 1; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 1; if $rows != 4007 then return -1 endi -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 101; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 101; print $rows if $rows != 3907 then return -1 endi -if $data00 != @70-01-01 08:01:43.101@ then +if $data00 != @70-01-01 08:01:40.101@ then return -1 endi -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 902; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 902; if $rows != 3106 then return -1 endi -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 400; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 400; if $rows != 3608 then return -1 endi -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 4007; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 4007; if $rows != 1 then return -1 endi -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 2000 offset 4008; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 2000 offset 4008; if $rows != 0 then return -1 endi #==================================order by desc, multi vnode, limit/offset=================================== -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 0; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 0; if $rows != 4008 then return -1 endi -if $data00 != @70-01-01 08:01:43.500@ then +if $data00 != @70-01-01 08:01:40.500@ then return -1 endi -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 1; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 1; if $rows != 4007 then return -1 endi -if $data00 != @70-01-01 08:01:43.500@ then +if $data00 != @70-01-01 08:01:40.500@ then return -1 endi -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 101; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 101; print $rows if $rows != 3907 then return -1 endi -if $data00 != @70-01-01 08:01:43.488@ then +if $data00 != @70-01-01 08:01:40.488@ then return -1 endi -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 902; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 902; if $rows != 3106 then return -1 endi -if $data00 != @70-01-01 08:01:43.388@ then +if $data00 != @70-01-01 08:01:40.388@ then return -1 endi -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 400; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 400; if $rows != 3608 then return -1 endi -if $data00 != @70-01-01 08:01:43.450@ then +if $data00 != @70-01-01 08:01:40.450@ then return -1 endi -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 4007; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 4007; if $rows != 1 then return -1 endi -if $data00 != @70-01-01 08:01:43.000@ then +if $data00 != @70-01-01 08:01:40.000@ then return -1 endi -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 2000 offset 4008; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 2000 offset 4008; if $rows != 0 then return -1 endi #=================================single value filter====================================== -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 0; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 0; if $row != 8 then return -1 endi -if $data00 != @70-01-01 08:01:43.000@ then +if $data00 != @70-01-01 08:01:40.000@ then return -1 endi -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 1; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 1; if $row != 7 then return -1 endi -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 2; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 2; if $row != 6 then return -1 endi -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 4; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 4; if $row != 4 then return -1 endi -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 7; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 7; if $row != 1 then return -1 endi -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 8; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 8; if $row != 0 then return -1 endi -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 9; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 9; if $row != 0 then return -1 endi #===============================single value filter, order by desc============================ -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 0; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 0; if $row != 8 then return -1 endi -if $data00 != @70-01-01 08:01:43.000@ then +if $data00 != @70-01-01 08:01:40.000@ then return -1 endi -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 1; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 1; if $row != 7 then return -1 endi -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 2; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 2; if $row != 6 then return -1 endi -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 4; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 4; if $row != 4 then return -1 endi -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 7; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 7; if $row != 1 then return -1 endi -if $data00 != @70-01-01 08:01:43.000@ then +if $data00 != @70-01-01 08:01:40.000@ then return -1 endi -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 8; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 8; if $row != 0 then return -1 endi -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 9; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 9; if $row != 0 then return -1 endi #[tbase-695] -sql select ts,tbname from group_mt0 where ts>='1970-01-01 8:1:40' and ts<'1970-1-1 8:1:45' and c1<99999999 limit 100000 offset 5000 +sql select ts,tbname from group_mt0 where ts>='1970-01-01 8:1:40' and ts<'1970-1-1 8:1:40.500' and c1<99999999 limit 10000 offset 500 print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 -if $row != 35000 then +if $row != 3500 then return -1 endi @@ -301,7 +301,7 @@ print tbase-722 sql select spread(ts) from group_tb0; print $data00 -if $data00 != 9999.000000000 then +if $data00 != 999.000000000 then return -1 endi diff --git a/tests/script/tsim/parser/selectResNum.sim b/tests/script/tsim/parser/selectResNum.sim index 43365b2af2..1131078240 100644 --- a/tests/script/tsim/parser/selectResNum.sim +++ b/tests/script/tsim/parser/selectResNum.sim @@ -7,10 +7,10 @@ $dbPrefix = sc_db $tbPrefix = sc_tb $stbPrefix = sc_stb $tbNum = 10 -$rowNum = 10000 +$rowNum = 1000 $totalNum = $tbNum * $rowNum $loops = 200000 -$log = 10000 +$log = 1000 $ts0 = 1537146000000 $delta = 600000 print ========== selectResNum.sim diff --git a/tests/script/tsim/parser/sliding.sim b/tests/script/tsim/parser/sliding.sim index 45a49fbc4e..1cb4cb5340 100644 --- a/tests/script/tsim/parser/sliding.sim +++ b/tests/script/tsim/parser/sliding.sim @@ -7,7 +7,7 @@ $dbPrefix = sliding_db $tbPrefix = sliding_tb $mtPrefix = sliding_mt $tbNum = 8 -$rowNum = 10000 +$rowNum = 1000 $totalNum = $tbNum * $rowNum print =============== sliding.sim @@ -88,25 +88,25 @@ $ts1 = $tb1 . .ts $ts2 = $tb2 . .ts print ===============================interval_sliding query -sql select _wstart, count(*) from sliding_tb0 interval(30s) sliding(30s); +sql select _wstart, count(*) from sliding_tb0 interval(3s) sliding(3s); if $row != 10 then return -1 endi if $data00 != @00-01-01 00:00:00.000@ then return -1 endi -if $data01 != 1000 then +if $data01 != 100 then return -1 endi -if $data10 != @00-01-01 00:00:30.000@ then +if $data10 != @00-01-01 00:00:03.000@ then return -1 endi -if $data11 != 1000 then +if $data11 != 100 then return -1 endi sql select _wstart, stddev(c1) from sliding_tb0 interval(10a) sliding(10a); -if $row != 10000 then +if $row != 1000 then return -1 endi if $data00 != @00-01-01 00:00:00.000@ then @@ -123,10 +123,10 @@ if $data91 != 0.000000000 then endi sql select _wstart, stddev(c1),count(c2),first(c3),last(c4) from sliding_tb0 interval(10a) sliding(10a) order by _wstart desc; -if $row != 10000 then +if $row != 1000 then return -1 endi -if $data00 != @00-01-01 00:04:59.970@ then +if $data00 != @00-01-01 00:00:29.970@ then return -1 endi if $data01 != 0.000000000 then @@ -141,7 +141,7 @@ endi if $data04 != 99 then return -1 endi -if $data90 != @00-01-01 00:04:59.700@ then +if $data90 != @00-01-01 00:00:29.700@ then return -1 endi if $data91 != 0.000000000 then @@ -157,41 +157,41 @@ if $data94 != 90 then return -1 endi -sql select _wstart, count(c2),last(c4) from sliding_tb0 interval(30s) sliding(10s) order by _wstart asc; +sql select _wstart, count(c2),last(c4) from sliding_tb0 interval(3s) sliding(1s) order by _wstart asc; if $row != 32 then return -1 endi -if $data00 != @99-12-31 23:59:40.000@ then - print expect 12-31 23:59:40.000, actual: $data00 +if $data00 != @99-12-31 23:59:58.000@ then + print expect 12-31 23:59:58.000, actual: $data00 return -1 endi -if $data01 != 334 then +if $data01 != 34 then return -1 endi if $data02 != 33 then return -1 endi -sql select _wstart, count(c2),stddev(c3),first(c4),last(c4) from sliding_tb0 where ts>'2000-01-01 0:0:0' and ts<'2000-1-1 0:0:31' interval(30s) sliding(30s) order by _wstart asc; +sql select _wstart, count(c2),stddev(c3),first(c4),last(c4) from sliding_tb0 where ts>'2000-01-01 0:0:0' and ts<'2000-1-1 0:0:4' interval(3s) sliding(3s) order by _wstart asc; if $row != 2 then return -1 endi if $data04 != 99 then return -1 endi -if $data01 != 999 then +if $data01 != 99 then return -1 endi -if $data02 != 28.837977152 then +if $data02 != 28.577380332 then return -1 endi #interval offset + limit -sql select _wstart, count(c2), first(c3),stddev(c4) from sliding_tb0 interval(10a) sliding(10a) order by _wstart desc limit 10 offset 990; +sql select _wstart, count(c2), first(c3),stddev(c4) from sliding_tb0 interval(10a) sliding(10a) order by _wstart desc limit 10 offset 90; if $row != 10 then return -1 endi -if $data00 != @00-01-01 00:04:30.270@ then +if $data00 != @00-01-01 00:00:27.270@ then return -1 endi if $data01 != 1 then @@ -203,7 +203,7 @@ endi if $data03 != 0.000000000 then return -1 endi -if $data90 != @00-01-01 00:04:30.000@ then +if $data90 != @00-01-01 00:00:27.000@ then return -1 endi if $data91 != 1 then @@ -217,43 +217,43 @@ if $data93 != 0.000000000 then endi #interval offset test -sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(30s) order by _wstart asc limit 1000 offset 1; +sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(3s) order by _wstart asc limit 100 offset 1; if $row != 9 then return -1 endi -if $data00 != @00-01-01 00:00:30.000@ then +if $data00 != @00-01-01 00:00:03.000@ then return -1 endi -if $data01 != 1000 then +if $data01 != 100 then return -1 endi if $data02 != 99 then return -1 endi -if $data80 != @00-01-01 00:04:30.000@ then +if $data80 != @00-01-01 00:00:27.000@ then return -1 endi -if $data81 != 1000 then +if $data81 != 100 then return -1 endi -sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 where ts>'2000-1-1 0:0:0' and ts<'2000-1-1 0:0:31' interval(30s) sliding(30s) order by _wstart asc limit 1000 offset 0; +sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 where ts>'2000-1-1 0:0:0' and ts<'2000-1-1 0:0:4' interval(3s) sliding(3s) order by _wstart asc limit 100 offset 0; if $row != 2 then return -1 endi if $data00 != @00-01-01 00:00:00.000@ then return -1 endi -if $data01 != 999 then +if $data01 != 99 then return -1 endi if $data02 != 99 then return -1 endi -if $data03 != 28.837977152 then +if $data03 != 28.577380332 then return -1 endi -if $data10 != @00-01-01 00:00:30.000@ then +if $data10 != @00-01-01 00:00:03.000@ then return -1 endi if $data11 != 34 then @@ -266,14 +266,14 @@ if $data13 != 9.810708435 then return -1 endi -sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 1; +sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 1; if $row != 15 then return -1 endi if $data00 != @00-01-01 00:00:00.000@ then return -1 endi -if $data01 != 1000 then +if $data01 != 100 then return -1 endi if $data02 != 99 then @@ -282,85 +282,85 @@ endi if $data03 != 28.866070048 then return -1 endi -if $data90 != @00-01-01 00:03:00.000@ then +if $data90 != @00-01-01 00:00:18.000@ then return -1 endi -if $data91 != 1000 then +if $data91 != 100 then return -1 endi if $data92 != 99 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 5; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 5; if $row != 11 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 6; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 6; if $row != 10 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 7; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 7; if $row != 9 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 8; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 8; if $row != 8 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 9; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 9; if $row != 7 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 10; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 10; if $row != 6 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 11; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 11; if $row != 5 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 12; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 12; if $row != 4 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 13; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 13; if $row != 3 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 14; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 14; if $row != 2 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 15; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 15; if $row != 1 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 16; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 16; if $row != 0 then return -1 endi -sql select _wstart, count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) order by _wstart desc; +sql select _wstart, count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(3s) order by _wstart desc; if $row != 10 then return -1 endi -#00-01-01 00:04:30.000| 10| 0| 0.000000000| 0.000000000| -if $data00 != @00-01-01 00:04:30.000@ then +#00-01-01 00:00:27.000 | 1 | 0 | 0.000000000 | 0.000000000 | +if $data00 != @00-01-01 00:00:27.000@ then return -1 endi -if $data01 != 10 then +if $data01 != 1 then return -1 endi if $data02 != 0 then @@ -370,18 +370,18 @@ if $data03 != 0.000000000 then return -1 endi -sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) sliding(20s) order by _wstart desc limit 1 offset 15; +sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(3s) sliding(2s) order by _wstart desc limit 1 offset 14; if $row != 1 then return -1 endi -sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) sliding(20s) order by _wstart desc limit 1 offset 16; +sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(3s) sliding(2s) order by _wstart desc limit 1 offset 15; if $row != 0 then return -1 endi sql select _wstart, count(c2), first(c3),stddev(c4) from sliding_tb0 interval(10a) order by _wstart desc limit 10 offset 2; -if $data00 != @00-01-01 00:04:59.910@ then +if $data00 != @00-01-01 00:00:29.910@ then return -1 endi diff --git a/tests/script/tsim/parser/union.sim b/tests/script/tsim/parser/union.sim index 9e7c6f77cc..dee5da96e8 100644 --- a/tests/script/tsim/parser/union.sim +++ b/tests/script/tsim/parser/union.sim @@ -8,7 +8,7 @@ $tbPrefix = union_tb $tbPrefix1 = union_tb_ $mtPrefix = union_mt $tbNum = 10 -$rowNum = 10000 +$rowNum = 1000 $totalNum = $tbNum * $rowNum print =============== union.sim @@ -65,7 +65,7 @@ sql create table $mt1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c $j = 0 $t = 1578203484000 -$rowNum = 1000 +$rowNum = 100 $tbNum = 5 $i = 0 @@ -117,7 +117,7 @@ sql_error (select c1 from union_tb0 limit 1 union all select c1 from union_tb1 l # sql with parenthese sql (select c1 from union_tb0) -if $rows != 10000 then +if $rows != 1000 then return -1 endi if $data00 != 0 then @@ -187,7 +187,7 @@ endi if $data10 != 1 then return -1 endi -if $data20 != 4950000 then +if $data20 != 495000 then return -1 endi @@ -202,7 +202,7 @@ endi if $data10 != 1 then return -1 endi -if $data20 != 495000 then +if $data20 != 49500 then return -1 endi @@ -211,7 +211,7 @@ sql (select count(*) as c from union_tb0, union_tb1 where union_tb0.ts=union_tb1 if $rows != 11 then return -1 endi -if $data00 != 10000 then +if $data00 != 1000 then return -1 endi if $data10 != 9 then @@ -227,7 +227,7 @@ endi print ===========================================tags union # two super table tag union, limit is not active during retrieve tags query sql (select t1 from union_mt0) union all (select t1 from union_mt0) -if $rows != 200000 then +if $rows != 20000 then return -1 endi @@ -248,7 +248,7 @@ sql (select count(*) as c from union_tb0 where ts > now + 3650d) union all (sele if $rows != 2 then return -1 endi -#if $data00 != 495000 then +#if $data00 != 49500 then # return -1 #endi @@ -272,7 +272,7 @@ endi # multi-vnode projection query sql (select c1 from union_mt0) union all select c1 from union_mt0; -if $rows != 200000 then +if $rows != 20000 then return -1 endi @@ -330,10 +330,10 @@ sql (select sum(c1) as a from union_tb0 limit 1) union all (select sum(c3) as a if $rows != 2 then return -1 endi -if $data00 != 495000 then +if $data00 != 49500 then return -1 endi -if $data10 != 495000 then +if $data10 != 49500 then return -1 endi diff --git a/tests/script/tsim/parser/where.sim b/tests/script/tsim/parser/where.sim index c1660883b6..9b2be3b71d 100644 --- a/tests/script/tsim/parser/where.sim +++ b/tests/script/tsim/parser/where.sim @@ -7,7 +7,7 @@ $dbPrefix = wh_db $tbPrefix = wh_tb $mtPrefix = wh_mt $tbNum = 10 -$rowNum = 10000 +$rowNum = 1000 $totalNum = $tbNum * $rowNum print =============== where.sim @@ -62,7 +62,7 @@ sql select count(*) from $tb where c1<10 and c1<>2 if $rows != 1 then return -1 endi -if $data00 != 900 then +if $data00 != 90 then return -1 endi @@ -308,14 +308,14 @@ if $row != 0 then endi sql select * from wh_mt0 where c3 = 1; -print $rows -> 1000 +print $rows -> 100 print $data00 $data01 $data02 -if $row != 1000 then +if $row != 100 then return -1 endi sql select * from wh_mt0 where c3 is null and tbname in ('test_null_filter'); -if $rows != 10000 then +if $rows != 1000 then return -1 endi diff --git a/tests/script/tsim/valgrind/checkError7.sim b/tests/script/tsim/valgrind/checkError7.sim index af42d1e76b..edfe8816c9 100644 --- a/tests/script/tsim/valgrind/checkError7.sim +++ b/tests/script/tsim/valgrind/checkError7.sim @@ -8,8 +8,9 @@ sql create database d1 sql use d1 $x = 0 -while $x < 128 +while $x < 5 $tb = d1.s . $x + print create table $tb (ts timestamp, i int) tags (j int) sql create table $tb (ts timestamp, i int) tags (j int) $x = $x + 1 endw @@ -17,7 +18,7 @@ endw print ======================== describe stables # TODO : create stable error $m = 0 -while $m < 128 +while $m < 5 $tb = s . $m $filter = ' . $tb $filter = $filter . ' @@ -36,15 +37,16 @@ print ======================== show stables sql show d1.stables print num of stables is $rows -if $rows != 128 then +if $rows != 5 then return -1 endi print ======================== create table $x = 0 -while $x < 424 +while $x < 42 $tb = d1.t . $x + print create table $tb using d1.s0 tags( $x ) sql create table $tb using d1.s0 tags( $x ) $x = $x + 1 endw @@ -54,7 +56,7 @@ print ======================== show stables sql show d1.tables print num of tables is $rows -if $rows != 424 then +if $rows != 42 then return -1 endi From 397bead009040b9e6420ca4f4ede127a78b59b67 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 15 Mar 2023 08:38:00 +0800 Subject: [PATCH 07/38] fix: add test cases --- tests/script/tsim/query/tableCount.sim | 107 +++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 tests/script/tsim/query/tableCount.sim diff --git a/tests/script/tsim/query/tableCount.sim b/tests/script/tsim/query/tableCount.sim new file mode 100644 index 0000000000..d8d9bb9b03 --- /dev/null +++ b/tests/script/tsim/query/tableCount.sim @@ -0,0 +1,107 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql drop database if exists db1; +sql create database db1 vgroups 3; +sql create database db1; +sql use db1; +sql create stable sta (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int); +sql create stable stb (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int); +sql create table tba1 using sta tags(1, 1, 1); +sql create table tba2 using sta tags(2, 2, 2); +sql create table tba3 using sta tags(3, 3, 3); +sql create table tba4 using sta tags(3, 3, 3); +sql create table tba5 using sta tags(3, 3, 3); +sql create table tba6 using sta tags(3, 3, 3); +sql create table tba7 using sta tags(3, 3, 3); +sql create table tba8 using sta tags(3, 3, 3); +sql create table tbb1 using stb tags(4, 4, 4); +sql create table tbb2 using stb tags(5, 5, 5); +sql create table tbb3 using stb tags(6, 6, 6); +sql create table tbb4 using stb tags(4, 4, 4); +sql create table tbb5 using stb tags(5, 5, 5); +sql create table tbb6 using stb tags(6, 6, 6); +sql create table tbb7 using stb tags(7, 7, 7); +sql create table tbb8 using stb tags(8, 8, 8); +sql create table tbn1 (ts timestamp, f1 int); +sql create database db2 vgroups 3; +sql create database db2; +sql use db2; +sql create stable sta (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int); +sql create stable stb (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int); +sql create table tba1 using sta tags(1, 1, 1); +sql create table tba2 using sta tags(2, 2, 2); +sql create table tbb1 using stb tags(4, 4, 4); +sql create table tbb2 using stb tags(5, 5, 5); +sql create table tbb3 using stb tags(6, 6, 6); + +sql select count(table_name) from information_schema.ins_tables group by stable_name; +if $rows != 3 then + return -1 +endi +sql select count(table_name) from information_schema.ins_tables group by db_name; +if $rows != 4 then + return -1 +endi +sql select count(table_name) from information_schema.ins_tables group by db_name, stable_name; +if $rows != 7 then + return -1 +endi +sql select stable_name,count(table_name) from information_schema.ins_tables group by stable_name order by stable_name; +if $rows != 3 then + return -1 +endi +if $data01 != 30 then + return -1 +endi +if $data11 != 10 then + return -1 +endi +if $data21 != 11 then + return -1 +endi +sql select db_name,count(table_name) from information_schema.ins_tables group by db_name order by db_name; +if $rows != 4 then + return -1 +endi +if $data01 != 17 then + return -1 +endi +if $data11 != 5 then + return -1 +endi +if $data21 != 24 then + return -1 +endi +if $data31 != 5 then + return -1 +endi +sql select db_name,stable_name,count(table_name) from information_schema.ins_tables group by db_name, stable_name order by db_name, stable_name; +if $rows != 7 then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data12 != 8 then + return -1 +endi +if $data22 != 8 then + return -1 +endi +if $data32 != 2 then + return -1 +endi +if $data42 != 3 then + return -1 +endi +if $data52 != 24 then + return -1 +endi +if $data62 != 5 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From 96db1192ae2ab7b822dbda4b689e2af88d3b32e3 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Tue, 14 Mar 2023 11:15:12 +0800 Subject: [PATCH 08/38] fix: fix coverity scan --- source/libs/stream/src/streamState.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index a2b3e20dbf..411726075e 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -130,21 +130,25 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int char cfgPath[1030]; sprintf(cfgPath, "%s/cfg", statePath); + szPage = szPage < 0 ? 4096 : szPage; + pages = pages < 0 ? 256 : pages; char cfg[1024]; memset(cfg, 0, 1024); TdFilePtr pCfgFile = taosOpenFile(cfgPath, TD_FILE_READ); if (pCfgFile != NULL) { - int64_t size; + int64_t size = 0; taosFStatFile(pCfgFile, &size, NULL); - taosReadFile(pCfgFile, cfg, size); - sscanf(cfg, "%d\n%d\n", &szPage, &pages); + if (size > 0) { + taosReadFile(pCfgFile, cfg, size); + sscanf(cfg, "%d\n%d\n", &szPage, &pages); + } } else { - taosMulModeMkDir(statePath, 0755); - pCfgFile = taosOpenFile(cfgPath, TD_FILE_WRITE | TD_FILE_CREATE); - szPage = szPage < 0 ? 4096 : szPage; - pages = pages < 0 ? 256 : pages; - sprintf(cfg, "%d\n%d\n", szPage, pages); - taosWriteFile(pCfgFile, cfg, strlen(cfg)); + int32_t code = taosMulModeMkDir(statePath, 0755); + if (code == 0) { + pCfgFile = taosOpenFile(cfgPath, TD_FILE_WRITE | TD_FILE_CREATE); + sprintf(cfg, "%d\n%d\n", szPage, pages); + taosWriteFile(pCfgFile, cfg, strlen(cfg)); + } } taosCloseFile(&pCfgFile); From d5defa3deff039ee3ed9663cfb7c5632d8121f0c Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Fri, 24 Feb 2023 16:17:32 +0800 Subject: [PATCH 09/38] fix:create same sub table --- source/dnode/vnode/src/tq/tqSink.c | 1 + source/libs/executor/src/groupoperator.c | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 56684b691b..dafd4d7485 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -619,6 +619,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* TD_VID(pVnode), ctbName, suid, mr.me.ctbEntry.suid); metaReaderClear(&mr); taosMemoryFree(ctbName); + continue; } tbData.uid = mr.me.uid; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 65146edfac..0fc2cdb46a 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -996,14 +996,14 @@ void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* memset(tbName, 0, TSDB_TABLE_NAME_LEN); int32_t len = 0; if (colDataIsNull_s(pTbCol, pDestBlock->info.rows - 1)) { - len = TMIN(sizeof(TSDB_DATA_NULL_STR), TSDB_TABLE_NAME_LEN - 1); - memcpy(tbName, TSDB_DATA_NULL_STR, len); + len = 1; + tbName[0] = 0; } else { void* pData = colDataGetData(pTbCol, pDestBlock->info.rows - 1); len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1); memcpy(tbName, varDataVal(pData), len); + streamStatePutParName(pState, groupId, tbName); } - streamStatePutParName(pState, groupId, tbName); memcpy(pTmpBlock->info.parTbName, tbName, len); pDestBlock->info.rows--; } else { From 48bbd0eeebff0f145539d4f2d4cd370067825c84 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Wed, 15 Mar 2023 11:49:02 +0800 Subject: [PATCH 10/38] test:reduce to 1000 rowNumbers per tables in testcases --- tests/script/tsim/parser/where.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/tsim/parser/where.sim b/tests/script/tsim/parser/where.sim index 9b2be3b71d..8ab47dacaf 100644 --- a/tests/script/tsim/parser/where.sim +++ b/tests/script/tsim/parser/where.sim @@ -315,7 +315,7 @@ if $row != 100 then endi sql select * from wh_mt0 where c3 is null and tbname in ('test_null_filter'); -if $rows != 1000 then +if $rows != 10000 then return -1 endi From 25732f7c3c68d8cbc8fa32ded516663eb30b770e Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Wed, 15 Mar 2023 10:17:49 +0800 Subject: [PATCH 11/38] fix:source task needs to wait & stream state window does not handle null --- source/libs/executor/src/timewindowoperator.c | 2 +- source/libs/stream/src/streamExec.c | 2 +- tests/script/tsim/stream/deleteState.sim | 9 +- .../tsim/stream/partitionbyColumnState.sim | 9 +- tests/script/tsim/stream/state1.sim | 101 ++++++++++++++++++ 5 files changed, 107 insertions(+), 16 deletions(-) create mode 100644 tests/script/tsim/stream/state1.sim diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 5fcb24c146..fef588a503 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3911,7 +3911,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl blockDataEnsureCapacity(pAggSup->pScanBlock, rows); SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId); for (int32_t i = 0; i < rows; i += winRows) { - if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) { + if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup) || colDataIsNull_s(pKeyColInfo, i)) { i++; continue; } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index cb9774b584..cb610ad6b5 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -20,7 +20,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* pRes) { int32_t code; void* exec = pTask->exec.executor; - while(atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) { + while(pTask->taskLevel == TASK_LEVEL__SOURCE && atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) { qError("stream task wait for the end of fill history"); taosMsleep(2); continue; diff --git a/tests/script/tsim/stream/deleteState.sim b/tests/script/tsim/stream/deleteState.sim index dd74b73dce..c84e52067c 100644 --- a/tests/script/tsim/stream/deleteState.sim +++ b/tests/script/tsim/stream/deleteState.sim @@ -51,13 +51,8 @@ if $loop_count == 10 then return -1 endi -if $data01 != 1 then - print =====data01=$data01 - goto loop1 -endi - -if $data02 != NULL then - print =====data02=$data02 +if $rows != 0 then + print =====rows=$rows goto loop1 endi diff --git a/tests/script/tsim/stream/partitionbyColumnState.sim b/tests/script/tsim/stream/partitionbyColumnState.sim index 62262a490c..b69ab2df52 100644 --- a/tests/script/tsim/stream/partitionbyColumnState.sim +++ b/tests/script/tsim/stream/partitionbyColumnState.sim @@ -27,13 +27,8 @@ if $loop_count == 10 then return -1 endi -if $data01 != 1 then - print =====data01=$data01 - goto loop0 -endi - -if $data02 != NULL then - print =====data02=$data02 +if $rows != 0 then + print =====rows=$rows goto loop0 endi diff --git a/tests/script/tsim/stream/state1.sim b/tests/script/tsim/stream/state1.sim new file mode 100644 index 0000000000..2ae5739642 --- /dev/null +++ b/tests/script/tsim/stream/state1.sim @@ -0,0 +1,101 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print =============== create database +sql create database test vgroups 4; +sql select * from information_schema.ins_databases; +if $rows != 3 then + return -1 +endi + +print $data00 $data01 $data02 + +sql use test; + +sql create table t1(ts timestamp, a int, b int , c int, d double, id int); + +print create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt1 as select _wstart, count(*) c1 from t1 state_window(a); + +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt1 as select _wstart, count(*) c1 from t1 state_window(a); + +sql insert into t1(ts) values(1648791213000); + +$loop_count = 0 +loop0: + +sleep 300 +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt1; +print data00 data01 +print data10 data11 + +if $rows != 0 then + print =====rows=$rows + goto loop0 +endi + +sql insert into t1 values(1648791214000,1,2,3,1.0,3); +$loop_count = 0 +loop1: + +sleep 300 +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt1; +print data00 data01 +print data10 data11 + +if $rows != 1 then + print =====rows=$rows + goto loop1 +endi + +sql insert into t1 values(1648791215000,2,2,3,1.0,4); + +$loop_count = 0 +loop2: + +sleep 300 +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt1; + +if $rows != 2 then + print =====rows=$rows + goto loop2 +endi + +sql insert into t1(ts) values(1648791216000); + +$loop_count = 0 +loop3: + +sleep 300 +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt1; +if $rows != 2 then + print =====rows=$rows + goto loop2 +endi + + +print state1 end + +system sh/stop_dnodes.sh From 7ee24c318eca14ce6df6d1885d6f4de0df8ba5e1 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 15 Mar 2023 14:25:18 +0800 Subject: [PATCH 12/38] fix: alter minrows doesn't take effect --- source/dnode/mnode/impl/src/mndDb.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index dd0953d6ba..126a653a69 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -279,6 +279,8 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) { pOld->cfg.cacheLast = pNew->cfg.cacheLast; pOld->cfg.replications = pNew->cfg.replications; pOld->cfg.sstTrigger = pNew->cfg.sstTrigger; + pOld->cfg.minRows = pNew->cfg.minRows; + pOld->cfg.maxRows = pNew->cfg.maxRows; pOld->cfg.tsdbPageSize = pNew->cfg.tsdbPageSize; pOld->compactStartTime = pNew->compactStartTime; taosWUnLockLatch(&pOld->lock); From 253a760810d1f7166d54bb9595385930b80ebfbd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 15 Mar 2023 14:30:12 +0800 Subject: [PATCH 13/38] fix(query): set the correct cached tags list key length, and update some logs. --- source/dnode/mnode/impl/src/mndConsumer.c | 8 +-- source/dnode/vnode/src/meta/metaCache.c | 83 +++++++++++++---------- source/dnode/vnode/src/tq/tqRead.c | 2 +- 3 files changed, 51 insertions(+), 42 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 4b6de316a1..53421aa45c 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -77,7 +77,7 @@ void mndCleanupConsumer(SMnode *pMnode) {} bool mndRebTryStart() { int32_t old = atomic_val_compare_exchange_32(&mqRebInExecCnt, 0, 1); - mInfo("tq timer, rebalance counter old val:%d", old); + mDebug("tq timer, rebalance counter old val:%d", old); return old == 0; } @@ -253,11 +253,11 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { SMqConsumerObj *pConsumer; void *pIter = NULL; - mTrace("start to process mq timer"); + mDebug("start to process mq timer"); // rebalance cannot be parallel if (!mndRebTryStart()) { - mInfo("mq rebalance already in progress, do nothing"); + mDebug("mq rebalance already in progress, do nothing"); return 0; } @@ -356,7 +356,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { } else { taosHashCleanup(pRebMsg->rebSubHash); rpcFreeCont(pRebMsg); - mInfo("mq rebalance finished, no modification"); + mDebug("mq rebalance finished, no modification"); mndRebEnd(); } return 0; diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 05889e4767..28d85e1bdd 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -14,6 +14,7 @@ */ #include "meta.h" +#define TAG_FILTER_RES_KEY_LEN 32 #define META_CACHE_BASE_BUCKET 1024 #define META_CACHE_STATS_BUCKET 16 @@ -34,7 +35,6 @@ typedef struct SMetaStbStatsEntry { typedef struct STagFilterResEntry { SList list; // the linked list of md5 digest, extracted from the serialized tag query condition uint32_t hitTimes; // queried times for current super table - uint32_t accTime; } STagFilterResEntry; struct SMetaCache { @@ -457,24 +457,23 @@ static int checkAllEntriesInCache(const STagFilterResEntry* pEntry, SArray* pInv int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1, bool* acquireRes) { + int32_t vgId = TD_VID(pMeta->pVnode); + // generate the composed key for LRU cache SLRUCache* pCache = pMeta->pCache->sTagFilterResCache.pUidResCache; SHashObj* pTableMap = pMeta->pCache->sTagFilterResCache.pTableEntry; TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock; - uint64_t buf[4]; - *acquireRes = 0; - - buf[0] = (uint64_t)pTableMap; - buf[1] = suid; - memcpy(&buf[2], pKey, keyLen); + uint64_t key[4]; + key[0] = (uint64_t)pTableMap; + key[1] = suid; + memcpy(&key[2], pKey, keyLen); taosThreadMutexLock(pLock); pMeta->pCache->sTagFilterResCache.accTimes += 1; - int32_t len = keyLen + sizeof(uint64_t) * 2; - LRUHandle* pHandle = taosLRUCacheLookup(pCache, buf, len); + LRUHandle* pHandle = taosLRUCacheLookup(pCache, key, TAG_FILTER_RES_KEY_LEN); if (pHandle == NULL) { taosThreadMutexUnlock(pLock); return TSDB_CODE_SUCCESS; @@ -499,7 +498,7 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK uint32_t acc = pMeta->pCache->sTagFilterResCache.accTimes; if ((*pEntry)->hitTimes % 5000 == 0 && (*pEntry)->hitTimes > 0) { - metaInfo("cache hit:%d, total acc:%d, rate:%.2f", (*pEntry)->hitTimes, acc, ((double)(*pEntry)->hitTimes) / acc); + metaInfo("vgId:%d cache hit:%d, total acc:%d, rate:%.2f", vgId, (*pEntry)->hitTimes, acc, ((double)(*pEntry)->hitTimes) / acc); } taosLRUCacheRelease(pCache, pHandle, false); @@ -560,13 +559,30 @@ static int32_t addNewEntry(SHashObj* pTableEntry, const void* pKey, int32_t keyL return 0; } +static FORCE_INLINE void setMD5DigestInKey(uint64_t* pBuf, const char* key, int32_t keyLen) { +// ASSERT(keyLen == sizeof(int64_t) * 2); + memcpy(&pBuf[2], key, keyLen); +} + +// the format of key: +// hash table address(8bytes) + suid(8bytes) + MD5 digest(16bytes) +static void initCacheKey(uint64_t* buf, const SHashObj* pHashMap, uint64_t suid, const char* key, int32_t keyLen) { + buf[0] = (uint64_t) pHashMap; + buf[1] = suid; + setMD5DigestInKey(buf, key, keyLen); + ASSERT(keyLen == sizeof(uint64_t) * 2); +} + // check both the payload size and selectivity ratio int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen, double selectivityRatio) { + int32_t code = 0; + int32_t vgId = TD_VID(pMeta->pVnode); + if (selectivityRatio > tsSelectivityRatio) { metaDebug("vgId:%d, suid:%" PRIu64 " failed to add to uid list cache, due to selectivity ratio %.2f less than threshold %.2f", - TD_VID(pMeta->pVnode), suid, selectivityRatio, tsSelectivityRatio); + vgId, suid, selectivityRatio, tsSelectivityRatio); taosMemoryFree(pPayload); return TSDB_CODE_SUCCESS; } @@ -574,7 +590,7 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int if (payloadLen > tsTagFilterResCacheSize) { metaDebug("vgId:%d, suid:%" PRIu64 " failed to add to uid list cache, due to payload length %d greater than threshold %d", - TD_VID(pMeta->pVnode), suid, payloadLen, tsTagFilterResCacheSize); + vgId, suid, payloadLen, tsTagFilterResCacheSize); taosMemoryFree(pPayload); return TSDB_CODE_SUCCESS; } @@ -583,26 +599,17 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int SHashObj* pTableEntry = pMeta->pCache->sTagFilterResCache.pTableEntry; TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock; - // the format of key: - // hash table address(8bytes) + suid(8bytes) + MD5 digest(16bytes) + uint64_t key[4] = {0}; + initCacheKey(key, pTableEntry, suid, pKey, keyLen); - uint64_t buf[4] = {0}; - buf[0] = (uint64_t)pTableEntry; - buf[1] = suid; - memcpy(&buf[2], pKey, keyLen); - ASSERT(keyLen == 16); - - int32_t code = 0; taosThreadMutexLock(pLock); - STagFilterResEntry** pEntry = taosHashGet(pTableEntry, &suid, sizeof(uint64_t)); if (pEntry == NULL) { code = addNewEntry(pTableEntry, pKey, keyLen, suid); if (code != TSDB_CODE_SUCCESS) { goto _end; } - } else { - // check if it exists or not + } else { // check if it exists or not size_t size = listNEles(&(*pEntry)->list); if (size == 0) { tdListAppend(&(*pEntry)->list, pKey); @@ -620,12 +627,11 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int } // add to cache. - taosLRUCacheInsert(pCache, buf, sizeof(uint64_t) * 2 + keyLen, pPayload, payloadLen, freePayload, NULL, + taosLRUCacheInsert(pCache, key, TAG_FILTER_RES_KEY_LEN, pPayload, payloadLen, freePayload, NULL, TAOS_LRU_PRIORITY_LOW); _end: taosThreadMutexUnlock(pLock); - - metaDebug("vgId:%d, suid:%" PRIu64 " list cache added into cache, total:%d, tables:%d", TD_VID(pMeta->pVnode), suid, + metaDebug("vgId:%d, suid:%" PRIu64 " list cache added into cache, total:%d, tables:%d", vgId, suid, (int32_t)taosLRUCacheGetUsage(pCache), taosHashGetSize(pTableEntry)); return code; @@ -633,33 +639,36 @@ _end: // remove the lru cache that are expired due to the tags value update, or creating, or dropping, of child tables int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid) { - int32_t keyLen = sizeof(uint64_t) * 3; - uint64_t p[4] = {0}; + uint64_t p[4] = {0}; + int32_t vgId = TD_VID(pMeta->pVnode); + SHashObj* pEntryHashMap = pMeta->pCache->sTagFilterResCache.pTableEntry; - p[0] = (uint64_t)pMeta->pCache->sTagFilterResCache.pTableEntry; - p[1] = suid; + uint64_t dummy[2] = {0}; + initCacheKey(p, pEntryHashMap, suid, (char*) &dummy[0], 16); TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock; - taosThreadMutexLock(pLock); - STagFilterResEntry** pEntry = taosHashGet(pMeta->pCache->sTagFilterResCache.pTableEntry, &suid, sizeof(uint64_t)); + + STagFilterResEntry** pEntry = taosHashGet(pEntryHashMap, &suid, sizeof(uint64_t)); if (pEntry == NULL || listNEles(&(*pEntry)->list) == 0) { taosThreadMutexUnlock(pLock); return TSDB_CODE_SUCCESS; } + (*pEntry)->hitTimes = 0; + SListIter iter = {0}; tdListInitIter(&(*pEntry)->list, &iter, TD_LIST_FORWARD); SListNode* pNode = NULL; while ((pNode = tdListNext(&iter)) != NULL) { - memcpy(&p[2], pNode->data, 16); - taosLRUCacheErase(pMeta->pCache->sTagFilterResCache.pUidResCache, p, keyLen); + setMD5DigestInKey(p, pNode->data, 2 * sizeof(uint64_t)); + taosLRUCacheErase(pMeta->pCache->sTagFilterResCache.pUidResCache, p, TAG_FILTER_RES_KEY_LEN); } - (*pEntry)->hitTimes = 0; tdListEmpty(&(*pEntry)->list); - taosThreadMutexUnlock(pLock); + + metaDebug("vgId:%d suid:%"PRId64" cached related tag filter uid list cleared", vgId, suid); return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 4e43587915..1c8088be7e 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -296,7 +296,7 @@ int32_t tqSeekVer(STqReader* pReader, int64_t ver, const char* id) { // todo set the correct vgId tqDebug("tmq poll: wal seek to version:%"PRId64" %s", ver, id); if (walReadSeekVer(pReader->pWalReader, ver) < 0) { - tqError("tmq poll: wal reader failed to seek to ver:%"PRId64" code:%s, %s", ver, tstrerror(terrno), id); + tqDebug("tmq poll: wal reader failed to seek to ver:%"PRId64" code:%s, %s", ver, tstrerror(terrno), id); return -1; } else { tqDebug("tmq poll: wal reader seek to ver:%"PRId64" %s", ver, id); From 52576071ae6c500ffc64213b868c854c865f5f47 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 15 Mar 2023 14:37:48 +0800 Subject: [PATCH 14/38] refactor: do some internal refactor. --- source/dnode/vnode/src/meta/metaCache.c | 32 ++++++++++++------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 28d85e1bdd..9501bf4b8e 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -455,6 +455,20 @@ static int checkAllEntriesInCache(const STagFilterResEntry* pEntry, SArray* pInv return 0; } +static FORCE_INLINE void setMD5DigestInKey(uint64_t* pBuf, const char* key, int32_t keyLen) { +// ASSERT(keyLen == sizeof(int64_t) * 2); + memcpy(&pBuf[2], key, keyLen); +} + +// the format of key: +// hash table address(8bytes) + suid(8bytes) + MD5 digest(16bytes) +static void initCacheKey(uint64_t* buf, const SHashObj* pHashMap, uint64_t suid, const char* key, int32_t keyLen) { + buf[0] = (uint64_t) pHashMap; + buf[1] = suid; + setMD5DigestInKey(buf, key, keyLen); + ASSERT(keyLen == sizeof(uint64_t) * 2); +} + int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1, bool* acquireRes) { int32_t vgId = TD_VID(pMeta->pVnode); @@ -466,9 +480,7 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK *acquireRes = 0; uint64_t key[4]; - key[0] = (uint64_t)pTableMap; - key[1] = suid; - memcpy(&key[2], pKey, keyLen); + initCacheKey(key, pTableMap, suid, (const char*)pKey, keyLen); taosThreadMutexLock(pLock); pMeta->pCache->sTagFilterResCache.accTimes += 1; @@ -559,20 +571,6 @@ static int32_t addNewEntry(SHashObj* pTableEntry, const void* pKey, int32_t keyL return 0; } -static FORCE_INLINE void setMD5DigestInKey(uint64_t* pBuf, const char* key, int32_t keyLen) { -// ASSERT(keyLen == sizeof(int64_t) * 2); - memcpy(&pBuf[2], key, keyLen); -} - -// the format of key: -// hash table address(8bytes) + suid(8bytes) + MD5 digest(16bytes) -static void initCacheKey(uint64_t* buf, const SHashObj* pHashMap, uint64_t suid, const char* key, int32_t keyLen) { - buf[0] = (uint64_t) pHashMap; - buf[1] = suid; - setMD5DigestInKey(buf, key, keyLen); - ASSERT(keyLen == sizeof(uint64_t) * 2); -} - // check both the payload size and selectivity ratio int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen, double selectivityRatio) { From da58966d588a72c7e00f1827bc479a6054200b7b Mon Sep 17 00:00:00 2001 From: Adam Ji Date: Wed, 15 Mar 2023 15:48:08 +0800 Subject: [PATCH 15/38] test: update requirements (#20470) --- tests/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/requirements.txt b/tests/requirements.txt index c6e27fd3be..5cdd9e02be 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -8,3 +8,4 @@ distro requests pexpect faker +pyopenssl From f1414355e367c923a2894b773c121e4411b4af1d Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 15 Mar 2023 16:24:36 +0800 Subject: [PATCH 16/38] tsdb/cache: skip invalid datablock directly when traversing fs with last --- source/dnode/vnode/src/tsdb/tsdbCache.c | 83 +++++++++++++++++++++---- 1 file changed, 72 insertions(+), 11 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index ab94a5e1c7..556da75a12 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -596,7 +596,8 @@ typedef struct { int64_t lastTs; } SFSLastNextRowIter; -static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs) { +static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols, + int nCols) { SFSLastNextRowIter *state = (SFSLastNextRowIter *)iter; int32_t code = 0; @@ -740,7 +741,8 @@ typedef struct SFSNextRowIter { int64_t lastTs; } SFSNextRowIter; -static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs) { +static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols, + int nCols) { SFSNextRowIter *state = (SFSNextRowIter *)iter; int32_t code = 0; @@ -828,8 +830,11 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie } } case SFSNEXTROW_BLOCKDATA: + _next_datablock: if (state->iBlock >= 0) { SDataBlk block = {0}; + bool skipBlock = true; + int inputColIndex = 0; tDataBlkReset(&block); tBlockDataReset(state->pBlockData); @@ -854,6 +859,42 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie code = tsdbReadDataBlock(*state->pDataFReader, &block, state->pBlockData); if (code) goto _err; + for (int colIndex = 0; colIndex < state->pBlockData->nColData; ++colIndex) { + SColData *pColData = &state->pBlockData->aColData[colIndex]; + int16_t cid = pColData->cid; + + if (inputColIndex < nCols && cid == aCols[inputColIndex++]) { + if (isLast && pColData->numOfValue != 0) { + skipBlock = false; + break; + } else if (pColData->numOfNone != pColData->nVal) { + skipBlock = false; + break; + } + } + } + + if (skipBlock) { + if (--state->iBlock < 0) { + tsdbDataFReaderClose(state->pDataFReader); + *state->pDataFReader = NULL; + // resetLastBlockLoadInfo(state->pLoadInfo); + + if (state->aBlockIdx) { + // taosArrayDestroy(state->aBlockIdx); + tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle); + + state->aBlockIdxHandle = NULL; + state->aBlockIdx = NULL; + } + + state->state = SFSNEXTROW_FILESET; + goto _next_fileset; + } else { + goto _next_datablock; + } + } + state->nRow = state->blockData.nRow; state->iRow = state->nRow - 1; @@ -960,7 +1001,8 @@ typedef struct SMemNextRowIter { // TSDBROW *curRow; } SMemNextRowIter; -static int32_t getNextRowFromMem(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs) { +static int32_t getNextRowFromMem(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols, + int nCols) { SMemNextRowIter *state = (SMemNextRowIter *)iter; int32_t code = 0; *pIgnoreEarlierTs = false; @@ -1072,7 +1114,8 @@ static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int64_t *iSkyline) { return deleted; } -typedef int32_t (*_next_row_fn_t)(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs); +typedef int32_t (*_next_row_fn_t)(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols, + int nCols); typedef int32_t (*_next_row_clear_fn_t)(void *iter); typedef struct { @@ -1222,12 +1265,14 @@ _err: } // iterate next row non deleted backward ts, version (from high to low) -static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pIgnoreEarlierTs) { +static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, + int16_t *aCols, int nCols) { int code = 0; for (;;) { for (int i = 0; i < 4; ++i) { if (pIter->input[i].next && !pIter->input[i].stop) { - code = pIter->input[i].nextRowFn(pIter->input[i].iter, &pIter->input[i].pRow, &pIter->input[i].ignoreEarlierTs); + code = pIter->input[i].nextRowFn(pIter->input[i].iter, &pIter->input[i].pRow, &pIter->input[i].ignoreEarlierTs, + isLast, aCols, nCols); if (code) goto _err; if (pIter->input[i].pRow == NULL) { @@ -1358,7 +1403,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo do { TSDBROW *pRow = NULL; - nextRowIterGet(&iter, &pRow, &ignoreEarlierTs); + nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, false, NULL, 0); if (!pRow) { break; @@ -1488,11 +1533,18 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach bool ignoreEarlierTs = false; SArray *pColArray = NULL; SColVal *pColVal = &(SColVal){0}; + int16_t nCols = nLastCol; int32_t code = initLastColArray(pTSchema, &pColArray); if (TSDB_CODE_SUCCESS != code) { return code; } + SArray *aColArray = taosArrayInit(nCols, sizeof(int16_t)); + if (NULL == aColArray) { + taosArrayDestroy(pColArray); + + return TSDB_CODE_OUT_OF_MEMORY; + } TSKEY lastRowTs = TSKEY_MAX; @@ -1502,7 +1554,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach do { TSDBROW *pRow = NULL; - nextRowIterGet(&iter, &pRow, &ignoreEarlierTs); + nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, true, TARRAY_DATA(aColArray), TARRAY_SIZE(aColArray)); if (!pRow) { break; @@ -1547,9 +1599,12 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData); } - if (!COL_VAL_IS_VALUE(pColVal) && !setNoneCol) { - noneCol = iCol; - setNoneCol = true; + if (!COL_VAL_IS_VALUE(pColVal)) { + taosArrayPush(aColArray, &pColVal->cid); + if (!setNoneCol) { + noneCol = iCol; + setNoneCol = true; + } } } if (!setNoneCol) { @@ -1590,6 +1645,8 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach } taosArraySet(pColArray, iCol, &lastCol); + int32_t aColIndex = taosArraySearchIdx(aColArray, &lastCol.colVal.cid, compareInt16Val, TD_EQ); + taosArrayRemove(aColArray, aColIndex); } else if (!COL_VAL_IS_VALUE(tColVal) && !COL_VAL_IS_VALUE(pColVal) && !setNoneCol) { noneCol = iCol; setNoneCol = true; @@ -1605,8 +1662,11 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach if (ignoreEarlierTs) { taosArrayDestroy(pColArray); pColArray = NULL; + taosArrayDestroy(aColArray); + aColArray = NULL; } else { taosArrayClear(pColArray); + taosArrayClear(aColArray); } } *ppLastArray = pColArray; @@ -1621,6 +1681,7 @@ _err: // taosMemoryFreeClear(pTSchema); *ppLastArray = NULL; taosArrayDestroy(pColArray); + taosArrayDestroy(aColArray); return code; } From 04d501d64aa723f2dbc18ce18c5ee5a02eb64255 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 15 Mar 2023 17:41:29 +0800 Subject: [PATCH 17/38] fix: destroy aColArray anyway --- source/dnode/vnode/src/tsdb/tsdbCache.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 556da75a12..9f438a9f97 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1662,17 +1662,15 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach if (ignoreEarlierTs) { taosArrayDestroy(pColArray); pColArray = NULL; - taosArrayDestroy(aColArray); - aColArray = NULL; } else { taosArrayClear(pColArray); - taosArrayClear(aColArray); } } *ppLastArray = pColArray; //} nextRowIterClose(&iter); + taosArrayDestroy(aColArray); // taosMemoryFreeClear(pTSchema); return code; From acd57f03e646529c7fe76c2e1c27814249b0ed26 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 15 Mar 2023 20:31:54 +0800 Subject: [PATCH 18/38] fix(query): invalid write when query of ins_columns --- source/libs/executor/src/sysscanoperator.c | 10 ++++++---- tests/system-test/0-others/information_schema.py | 10 +++++++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 24f42ff178..38c1d60899 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -436,7 +436,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { int32_t numOfRows = 0; SSDataBlock* dataBlock = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_COLS); - blockDataEnsureCapacity(dataBlock, pOperator->resultInfo.capacity); + blockDataEnsureCapacity(dataBlock, pOperator->resultInfo.capacity + TSDB_MAX_COLUMNS); const char* db = NULL; int32_t vgId = 0; @@ -562,15 +562,17 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { continue; } - sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName); - - if (numOfRows >= pOperator->resultInfo.capacity) { + if ((numOfRows + schemaRow->nCols) > pOperator->resultInfo.capacity) { relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); numOfRows = 0; + metaTbCursorPrev(pInfo->pCur); + if (pInfo->pRes->info.rows > 0) { break; } + } else { + sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName); } } diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index 720eab74c4..23ddb12d79 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -101,10 +101,18 @@ class TDTestCase: tdSql.checkEqual(i[1],len(self.perf_list)) elif i[0].lower() == self.dbname: tdSql.checkEqual(i[1],self.tbnum+1) + def ins_columns_check(self): + tdSql.execute('create database db2 vgroups 2 replica 1') + tdSql.execute('create table db2.stb2 (ts timestamp,c0 int,c1 int, c2 double, c3 float, c4 binary(1000), c5 nchar(100),c7 bigint, c8 bool, c9 smallint) tags(t0 int)') + for i in range(2000): + tdSql.execute("create table db2.ctb%d using db2.stb2 tags(%d)" %(i,i)) + tdSql.query(f'select * from information_schema.ins_columns where db_name="db2" and table_type="CHILD_TABLE"') + tdSql.checkEqual(20000,len(tdSql.queryResult)) + print("number of ins_columns of child table in db2 is %s" % len(tdSql.queryResult)) def run(self): self.prepare_data() self.count_check() - + self.ins_columns_check() def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From ba5c3fb2ab6bf0e2a0000dd9237826c77873d548 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 16 Mar 2023 00:01:54 +0800 Subject: [PATCH 19/38] fix: select ins_columns for stb with 4096 columns --- source/dnode/mnode/impl/src/mndStb.c | 7 +++++++ source/libs/executor/src/sysscanoperator.c | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 2a369a863a..9e61364801 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -3114,6 +3114,7 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(typeName, "SUPER_TABLE"); while (numOfRows < rows) { + void *prevIter = pShow->pIter; pShow->pIter = sdbFetch(pSdb, SDB_STB, pShow->pIter, (void **)&pStb); if (pShow->pIter == NULL) break; @@ -3122,6 +3123,12 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB continue; } + if ((numOfRows + pStb->numOfColumns) > rows) { + pShow->pIter = prevIter; + sdbRelease(pSdb, pStb); + break; + } + SName name = {0}; char stbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; mndExtractTbNameFromStbFullName(pStb->name, &stbName[VARSTR_HEADER_SIZE], TSDB_TABLE_NAME_LEN); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 38c1d60899..20766b38d6 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -436,7 +436,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { int32_t numOfRows = 0; SSDataBlock* dataBlock = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_COLS); - blockDataEnsureCapacity(dataBlock, pOperator->resultInfo.capacity + TSDB_MAX_COLUMNS); + blockDataEnsureCapacity(dataBlock, pOperator->resultInfo.capacity); const char* db = NULL; int32_t vgId = 0; From 6b3fa3f5afd9858706db19c2db0a334be7e7eeb5 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Thu, 16 Mar 2023 01:41:44 +0800 Subject: [PATCH 20/38] test:fix that timestamps can be validated correctly in testcase --- tests/pytest/util/sql.py | 74 +++++++++++++++++++--- tests/system-test/2-query/timeerror.py | 86 ++++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 9 deletions(-) create mode 100644 tests/system-test/2-query/timeerror.py diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index bdf3f20e15..469bd56305 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -23,13 +23,36 @@ import pandas as pd from util.log import * from util.constant import * +from datetime import datetime, timedelta,timezone +from tzlocal import get_localzone +import pytz +import time + +def _locaTzTimeStamp(utctimestamp): + tz = get_localzone() + temptz = str(tz) + localtz = pytz.timezone(temptz) + defUtctimestamp=1035640800 + local_dt = datetime(2002, 10, 27, 6, 0, 0, tzinfo=localtz) + defLocaltimestamp = time.mktime(local_dt.timetuple()) + deltaTzTime = int(defLocaltimestamp)-int(defUtctimestamp) + temp = int(str(utctimestamp)[0:10]) + tempOther = str(utctimestamp)[10-len(str(utctimestamp)):] + localtemp = deltaTzTime + temp + localAll = str(localtemp) + str(tempOther) + localtimestamp = int(localAll) + + print(f"local timezone is {localtimestamp}, Deltel time is {deltaTzTime}s") + return localtimestamp + + def _parse_datetime(timestr): try: - return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S.%f') + return datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S.%f') except ValueError: pass try: - return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S') + return datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S') except ValueError: pass @@ -227,18 +250,51 @@ class TDSql: self.checkRowCol(row, col) return self.cursor.istype(col, dataType) + def checkData(self, row, col, data): self.checkRowCol(row, col) if self.queryResult[row][col] != data: if self.cursor.istype(col, "TIMESTAMP"): - # suppose user want to check nanosecond timestamp if a longer data passed - if (len(data) >= 28): - if pd.to_datetime(self.queryResult[row][col]) == pd.to_datetime(data): - tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + # suppose user want to check nanosecond timestamp if a longer data passed`` + if isinstance(data,str) : + if (len(data) >= 28): + resultData = _locaTzTimeStamp(self.queryResult[row][col]) + if pd.to_datetime(resultData) == pd.to_datetime(data): + tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{pd.to_datetime(resultData)} == expect:{data}") + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) + tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) + else: + if self.queryResult[row][col] == _parse_datetime(data): + tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) + tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) + return + elif isinstance(data,int) : + if len(str(data)) == 16 : + unitTime = 'us' + elif len(str(data)) == 13 : + unitTime = 'ms' + elif len(str(data)) == 19 : + unitTime = 'ns' + else: + tdLog.exit("expect timeStamp type is wrong") + resultData = pd.to_datetime(_locaTzTimeStamp(data),unit=unitTime) + if resultData == self.queryResult[row][col] : + tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{resultData}") + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) + tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) + return else: - if self.queryResult[row][col] == _parse_datetime(data): - tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") - return + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) + tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) + if str(self.queryResult[row][col]) == str(data): tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") diff --git a/tests/system-test/2-query/timeerror.py b/tests/system-test/2-query/timeerror.py new file mode 100644 index 0000000000..e6d0e427f6 --- /dev/null +++ b/tests/system-test/2-query/timeerror.py @@ -0,0 +1,86 @@ +import taos +import sys +from time import sleep +from util.log import * +from util.sql import * +from util.cases import * + + + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def run(self): + rows = 10 + + tdSql.execute("create database dbus PRECISION 'us' ") + tdSql.execute("create database dbns PRECISION 'ns' ") + tdSql.execute("create database dbms PRECISION 'ms' ") + dball = ["dbns","dbus","dbms"] + for i in range(len(dball)): + dbname = dball[i] + stb = f"{dbname}.stb1" + if i == 0 : + qts = 1678883666951061471 + qts1 = 1678883666951061471 + + qtime = "2023-03-15 20:34:26.951061471" + elif i == 1 : + qts = 1678883666951061 + qts1 = 1678883666951061 + + qtime = "2023-03-15 20:34:26.951061" + else: + qts = 1678883666951 + qts1 = 1678883666953 + + qtime = "2023-03-15 20:34:26.951" + + tdSql.execute(f"use {dbname}") + tdLog.printNoPrefix("==========step1:create table") + tdSql.execute( + f'''create table if not exists {stb} + (ts timestamp, c1 int, c2 float, c3 bigint, c4 double, c5 smallint, c6 tinyint) + tags(location binary(64), type int, isused bool , family nchar(64))''' + ) + tdSql.execute(f"create table {dbname}.t1 using {stb} tags('beijing', 1, 1, 'nchar1')") + tdSql.execute(f"create table {dbname}.t2 using {stb} tags('shanghai', 2, 0, 'nchar2')") + tdSql.execute(f"create table {dbname}.t3 using {stb} tags('shanghai', 3, 0, 'nchar3')") + + tdLog.printNoPrefix("==========step2:insert data") + for i in range(rows): + tdSql.execute( + f"insert into {dbname}.t1 values (now()+{i}m, {32767+i}, {20.0+i/10}, {2**31+i}, {3.4*10**38+i/10}, {127+i}, {i})" + ) + tdSql.execute( + f"insert into {dbname}.t2 values (now()-{i}m, {-32767-i}, {20.0-i/10}, {-i-2**31}, {-i/10-3.4*10**38}, {-127-i}, {-i})" + ) + tdSql.execute( + f"insert into {dbname}.t1 values (now()+11m, {2**31-1}, {pow(10,37)*34}, {pow(2,63)-1}, {1.7*10**308}, 32767, 127)" + ) + tdSql.execute( + f"insert into {dbname}.t2 values (now()-11m, {1-2**31}, {-3.4*10**38}, {1-2**63}, {-1.7*10**308}, -32767, -127)" + ) + tdSql.execute( + f"insert into {dbname}.t2 values (now()-12m, null , {-3.4*10**38}, null , {-1.7*10**308}, null , null)" + ) + + tdSql.execute( + f"insert into {dbname}.t3 values ({qts}, null , {-3.4*10**38}, null , {-1.7*10**308}, null , null)" + ) + + tdLog.printNoPrefix("==========step3:query timestamp type") + tdSql.query(f"select ts from {dbname}.t3 limit 1") + tdSql.checkData(0,0,qtime) + tdSql.checkData(0,0,qts) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From a4fb7d56169a51cfcd95b6bbccebf01c18912b77 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Thu, 16 Mar 2023 01:53:28 +0800 Subject: [PATCH 21/38] test:fix that timestamps can be validated correctly in testcase --- tests/pytest/util/sql.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 469bd56305..2c90005b5f 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -252,7 +252,17 @@ class TDSql: def checkData(self, row, col, data): + if row >= self.queryRows: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, self.sql, row+1, self.queryRows) + tdLog.exit("%s(%d) failed: sql:%s, row:%d is larger than queryRows:%d" % args) + if col >= self.queryCols: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, self.sql, col+1, self.queryCols) + tdLog.exit("%s(%d) failed: sql:%s, col:%d is larger than queryCols:%d" % args) + self.checkRowCol(row, col) + if self.queryResult[row][col] != data: if self.cursor.istype(col, "TIMESTAMP"): # suppose user want to check nanosecond timestamp if a longer data passed`` @@ -281,7 +291,9 @@ class TDSql: elif len(str(data)) == 19 : unitTime = 'ns' else: - tdLog.exit("expect timeStamp type is wrong") + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) + tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) resultData = pd.to_datetime(_locaTzTimeStamp(data),unit=unitTime) if resultData == self.queryResult[row][col] : tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{resultData}") From a8967b73c335bc1c3d1132afc78cca5260f10cf2 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 16 Mar 2023 01:55:46 +0800 Subject: [PATCH 22/38] test:fix that timestamps can be validated correctly in testcase --- tests/system-test/2-query/timeerror.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/tests/system-test/2-query/timeerror.py b/tests/system-test/2-query/timeerror.py index e6d0e427f6..f43dc91de6 100644 --- a/tests/system-test/2-query/timeerror.py +++ b/tests/system-test/2-query/timeerror.py @@ -52,23 +52,6 @@ class TDTestCase: tdSql.execute(f"create table {dbname}.t3 using {stb} tags('shanghai', 3, 0, 'nchar3')") tdLog.printNoPrefix("==========step2:insert data") - for i in range(rows): - tdSql.execute( - f"insert into {dbname}.t1 values (now()+{i}m, {32767+i}, {20.0+i/10}, {2**31+i}, {3.4*10**38+i/10}, {127+i}, {i})" - ) - tdSql.execute( - f"insert into {dbname}.t2 values (now()-{i}m, {-32767-i}, {20.0-i/10}, {-i-2**31}, {-i/10-3.4*10**38}, {-127-i}, {-i})" - ) - tdSql.execute( - f"insert into {dbname}.t1 values (now()+11m, {2**31-1}, {pow(10,37)*34}, {pow(2,63)-1}, {1.7*10**308}, 32767, 127)" - ) - tdSql.execute( - f"insert into {dbname}.t2 values (now()-11m, {1-2**31}, {-3.4*10**38}, {1-2**63}, {-1.7*10**308}, -32767, -127)" - ) - tdSql.execute( - f"insert into {dbname}.t2 values (now()-12m, null , {-3.4*10**38}, null , {-1.7*10**308}, null , null)" - ) - tdSql.execute( f"insert into {dbname}.t3 values ({qts}, null , {-3.4*10**38}, null , {-1.7*10**308}, null , null)" ) From a1682664f28877ef870e6ff204ae7b1680442b21 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 16 Mar 2023 10:12:34 +0800 Subject: [PATCH 23/38] fix(tmq): disable non-leader vnode responsing the poll request. --- source/client/src/clientTmq.c | 187 +++++++++++----------- source/dnode/mnode/impl/src/mndConsumer.c | 2 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- 3 files changed, 98 insertions(+), 93 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 45bf69c450..50d1e78c89 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -24,6 +24,8 @@ #include "tref.h" #include "ttimer.h" +#define EMPTY_BLOCK_POLL_IDLE_DURATION 100 + struct SMqMgmt { int8_t inited; tmr_h timer; @@ -68,18 +70,16 @@ struct tmq_conf_t { }; struct tmq_t { - int64_t refId; - // conf - char groupId[TSDB_CGROUP_LEN]; - char clientId[256]; - int8_t withTbName; - int8_t useSnapshot; - int8_t autoCommit; - int32_t autoCommitInterval; - int32_t resetOffsetCfg; - uint64_t consumerId; - bool hbBgEnable; - + int64_t refId; + char groupId[TSDB_CGROUP_LEN]; + char clientId[256]; + int8_t withTbName; + int8_t useSnapshot; + int8_t autoCommit; + int32_t autoCommitInterval; + int32_t resetOffsetCfg; + uint64_t consumerId; + bool hbBgEnable; tmq_commit_cb* commitCb; void* commitCbUserParam; @@ -93,11 +93,10 @@ struct tmq_t { int64_t pollCnt; // timer - tmr_h hbLiveTimer; - tmr_h epTimer; - tmr_h reportTimer; - tmr_h commitTimer; - + tmr_h hbLiveTimer; + tmr_h epTimer; + tmr_h reportTimer; + tmr_h commitTimer; STscObj* pTscObj; // connection SArray* clientTopics; // SArray STaosQueue* mqueue; // queue of rsp @@ -149,6 +148,7 @@ typedef struct { SMqClientVg* vgHandle; SMqClientTopic* topicHandle; uint64_t reqId; + SEpSet* pEpset; union { SMqDataRsp dataRsp; SMqMetaRsp metaRsp; @@ -403,65 +403,42 @@ static SMqClientVg* foundClientVg(SArray* pTopicList, const char* pName, int32_t return NULL; } +// Two problems do not need to be addressed here +// 1. update to of epset. the response of poll request will automatically handle this problem +// 2. commit failure. This one needs to be resolved. static int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; SMqCommitCbParamSet* pParamSet = (SMqCommitCbParamSet*)pParam->params; - if (code != TSDB_CODE_SUCCESS) { // if commit offset failed, let's try again - taosThreadMutexLock(&pParam->pTmq->lock); - int32_t numOfVgroups, index; - SMqClientVg* pVg = foundClientVg(pParam->pTmq->clientTopics, pParam->topicName, pParam->vgId, &index, &numOfVgroups); - - if (pVg == NULL) { - tscDebug("consumer:0x%" PRIx64 - " subKey:%s vgId:%d commit failed, code:%s has been transferred to other consumer, no need retry ordinal:%d/%d", - pParam->pTmq->consumerId, pParam->pOffset->subKey, pParam->vgId, tstrerror(code), index + 1, numOfVgroups); - } else { // let's retry the commit - int32_t code1 = doSendCommitMsg(pParam->pTmq, pVg, pParam->topicName, pParamSet, index, numOfVgroups); - if (code1 != TSDB_CODE_SUCCESS) { // retry failed. - tscError("consumer:0x%" PRIx64 " topic:%s vgId:%d offset:%" PRId64 - " retry failed, ignore this commit. code:%s ordinal:%d/%d", - pParam->pTmq->consumerId, pParam->topicName, pVg->vgId, pVg->committedOffset.version, - tstrerror(terrno), index + 1, numOfVgroups); - } - } - - taosThreadMutexUnlock(&pParam->pTmq->lock); - - taosMemoryFree(pParam->pOffset); - taosMemoryFree(pBuf->pData); - taosMemoryFree(pBuf->pEpSet); - - tmqCommitRspCountDown(pParamSet, pParam->pTmq->consumerId, pParam->topicName, pParam->vgId); - return 0; - } - - // todo replace the pTmq with refId - taosThreadMutexLock(&pParam->pTmq->lock); - tmq_t* pTmq = pParam->pTmq; - int32_t index = 0, numOfVgroups = 0; - - SMqClientVg* pVg = foundClientVg(pTmq->clientTopics, pParam->topicName, pParam->vgId, &index, &numOfVgroups); - if (pVg == NULL) { - tscDebug("consumer:0x%" PRIx64 " subKey:%s vgId:%d has been transferred to other consumer, ordinal:%d/%d", - pParam->pTmq->consumerId, pParam->pOffset->subKey, pParam->vgId, index + 1, numOfVgroups); - } else { // update the epset if needed - if (pBuf->pEpSet != NULL) { - SEp* pEp = GET_ACTIVE_EP(pBuf->pEpSet); - SEp* pOld = GET_ACTIVE_EP(&(pVg->epSet)); - - tscDebug("consumer:0x%" PRIx64 " subKey:%s update the epset vgId:%d, ep:%s:%d, old ep:%s:%d, ordinal:%d/%d", - pTmq->consumerId, pParam->pOffset->subKey, pParam->vgId, pEp->fqdn, pEp->port, pOld->fqdn, pOld->port, - index + 1, numOfVgroups); - - pVg->epSet = *pBuf->pEpSet; - } - - tscDebug("consumer:0x%" PRIx64 " subKey:%s vgId:%d, commit offset success. ordinal:%d/%d", pTmq->consumerId, - pParam->pOffset->subKey, pParam->vgId, index + 1, numOfVgroups); - } - - taosThreadMutexUnlock(&pParam->pTmq->lock); +// if (code != TSDB_CODE_SUCCESS) { // if commit offset failed, let's try again +// taosThreadMutexLock(&pParam->pTmq->lock); +// int32_t numOfVgroups, index; +// SMqClientVg* pVg = foundClientVg(pParam->pTmq->clientTopics, pParam->topicName, pParam->vgId, &index, &numOfVgroups); +// if (pVg == NULL) { +// tscDebug("consumer:0x%" PRIx64 +// " subKey:%s vgId:%d commit failed, code:%s has been transferred to other consumer, no need retry ordinal:%d/%d", +// pParam->pTmq->consumerId, pParam->pOffset->subKey, pParam->vgId, tstrerror(code), index + 1, numOfVgroups); +// } else { // let's retry the commit +// int32_t code1 = doSendCommitMsg(pParam->pTmq, pVg, pParam->topicName, pParamSet, index, numOfVgroups); +// if (code1 != TSDB_CODE_SUCCESS) { // retry failed. +// tscError("consumer:0x%" PRIx64 " topic:%s vgId:%d offset:%" PRId64 +// " retry failed, ignore this commit. code:%s ordinal:%d/%d", +// pParam->pTmq->consumerId, pParam->topicName, pVg->vgId, pVg->committedOffset.version, +// tstrerror(terrno), index + 1, numOfVgroups); +// } +// } +// +// taosThreadMutexUnlock(&pParam->pTmq->lock); +// +// taosMemoryFree(pParam->pOffset); +// taosMemoryFree(pBuf->pData); +// taosMemoryFree(pBuf->pEpSet); +// +// tmqCommitRspCountDown(pParamSet, pParam->pTmq->consumerId, pParam->topicName, pParam->vgId); +// return 0; +// } +// +// // todo replace the pTmq with refId taosMemoryFree(pParam->pOffset); taosMemoryFree(pBuf->pData); @@ -892,15 +869,21 @@ static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { tDeleteSMqAskEpRsp(&pEpRspWrapper->msg); } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) { SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; + taosMemoryFreeClear(pRsp->pEpset); + taosArrayDestroyP(pRsp->dataRsp.blockData, taosMemoryFree); taosArrayDestroy(pRsp->dataRsp.blockDataLen); taosArrayDestroyP(pRsp->dataRsp.blockTbName, taosMemoryFree); taosArrayDestroyP(pRsp->dataRsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) { SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; + taosMemoryFreeClear(pRsp->pEpset); + taosMemoryFree(pRsp->metaRsp.metaRsp); } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__TAOSX_RSP) { SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; + taosMemoryFreeClear(pRsp->pEpset); + taosArrayDestroyP(pRsp->taosxRsp.blockData, taosMemoryFree); taosArrayDestroy(pRsp->taosxRsp.blockDataLen); taosArrayDestroyP(pRsp->taosxRsp.blockTbName, taosMemoryFree); @@ -1321,7 +1304,9 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { pRspWrapper->vgHandle = pVg; pRspWrapper->topicHandle = pTopic; pRspWrapper->reqId = requestId; + pRspWrapper->pEpset = pMsg->pEpSet; + pMsg->pEpSet = NULL; if (rspType == TMQ_MSG_TYPE__POLL_RSP) { SDecoder decoder; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); @@ -1350,7 +1335,6 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { } taosMemoryFree(pMsg->pData); - taosMemoryFree(pMsg->pEpSet); taosWriteQitem(tmq->mqueue, pRspWrapper); tscDebug("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d, reqId:0x%" PRIx64, @@ -1516,6 +1500,14 @@ static int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) { if (head->epoch <= epoch) { tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d, no need to update local ep", tmq->consumerId, head->epoch, epoch); + if (tmq->status == TMQ_CONSUMER_STATUS__RECOVER) { + SMqAskEpRsp rsp; + tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp); + int8_t flag = (taosArrayGetSize(rsp.topics) == 0) ? TMQ_CONSUMER_STATUS__NO_TOPIC : TMQ_CONSUMER_STATUS__READY; + atomic_store_8(&tmq->status, flag); + tDeleteSMqAskEpRsp(&rsp); + } + goto END; } @@ -1702,13 +1694,13 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { for (int j = 0; j < numOfVg; j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - if (taosGetTimestampMs() - pVg->emptyBlockReceiveTs < 100) { // less than 100ms + if (taosGetTimestampMs() - pVg->emptyBlockReceiveTs < EMPTY_BLOCK_POLL_IDLE_DURATION) { // less than 100ms tscTrace("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 100ms before start next poll", tmq->consumerId, tmq->epoch, pVg->vgId); continue; } - int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); + int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); if (vgStatus == TMQ_VG_STATUS__WAIT) { int32_t vgSkipCnt = atomic_add_fetch_32(&pVg->vgSkipCnt, 1); tscTrace("consumer:0x%" PRIx64 " epoch %d wait poll-rsp, skip vgId:%d skip cnt %d", tmq->consumerId, tmq->epoch, @@ -1731,6 +1723,7 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { } } + tscDebug("consumer:0x%" PRIx64 " end to poll data", tmq->consumerId); return 0; } @@ -1782,31 +1775,43 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { /*atomic_sub_fetch_32(&tmq->readyRequest, 1);*/ int32_t consumerEpoch = atomic_load_32(&tmq->epoch); - if (pollRspWrapper->dataRsp.head.epoch == consumerEpoch) { + SMqDataRsp* pDataRsp = &pollRspWrapper->dataRsp; + + if (pDataRsp->head.epoch == consumerEpoch) { + // todo fix it: race condition SMqClientVg* pVg = pollRspWrapper->vgHandle; - pVg->currentOffset = pollRspWrapper->dataRsp.rspOffset; + + // update the epset + if (pollRspWrapper->pEpset != NULL) { + SEp* pEp = GET_ACTIVE_EP(pollRspWrapper->pEpset); + SEp* pOld = GET_ACTIVE_EP(&(pVg->epSet)); + tscDebug("consumer:0x%" PRIx64 " update epset vgId:%d, ep:%s:%d, old ep:%s:%d", tmq->consumerId, + pVg->vgId, pEp->fqdn, pEp->port, pOld->fqdn, pOld->port); + pVg->epSet = *pollRspWrapper->pEpset; + } + + pVg->currentOffset = pDataRsp->rspOffset; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - if (pollRspWrapper->dataRsp.blockNum == 0) { - tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d, reqId:0x%" PRIx64, tmq->consumerId, pVg->vgId, - pollRspWrapper->reqId); + char buf[80]; + tFormatOffset(buf, 80, &pDataRsp->rspOffset); + if (pDataRsp->blockNum == 0) { + tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d, offset:%s, reqId:0x%" PRIx64, tmq->consumerId, + pVg->vgId, buf, pollRspWrapper->reqId); taosFreeQitem(pollRspWrapper); rspWrapper = NULL; continue; + } else { // build rsp + SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper); + tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, reqId:0x%" PRIx64, + tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, pollRspWrapper->reqId); + + taosFreeQitem(pollRspWrapper); + return pRsp; } - - // build rsp - char buf[80]; - tFormatOffset(buf, 80, &pVg->currentOffset); - SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper); - tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, reqId:0x%"PRIx64, tmq->consumerId, - pVg->vgId, buf, pollRspWrapper->dataRsp.blockNum, pollRspWrapper->reqId); - - taosFreeQitem(pollRspWrapper); - return pRsp; } else { tscDebug("consumer:0x%" PRIx64 " msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", - tmq->consumerId, pollRspWrapper->dataRsp.head.epoch, consumerEpoch); + tmq->consumerId, pDataRsp->head.epoch, consumerEpoch); tmqFreeRspWrapper(rspWrapper); taosFreeQitem(pollRspWrapper); } diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 53421aa45c..2f560910d4 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -101,7 +101,7 @@ void mndRebCntDec() { int32_t newVal = val - 1; int32_t oldVal = atomic_val_compare_exchange_32(&mqRebInExecCnt, val, newVal); if (oldVal == val) { - mInfo("rebalance trans end, rebalance counter:%d", newVal); + mDebug("rebalance trans end, rebalance counter:%d", newVal); break; } } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index d3ba7ad608..718b5979a1 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -510,7 +510,7 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { vTrace("vgId:%d, msg:%p in fetch queue is processing", pVnode->config.vgId, pMsg); if ((pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_VND_TABLE_META || pMsg->msgType == TDMT_VND_TABLE_CFG || - pMsg->msgType == TDMT_VND_BATCH_META) && + pMsg->msgType == TDMT_VND_BATCH_META || pMsg->msgType == TDMT_VND_TMQ_CONSUME) && !syncIsReadyForRead(pVnode->sync)) { vnodeRedirectRpcMsg(pVnode, pMsg, terrno); return 0; From 6e0343fc085b2a2a9f6d466e6b29c2546fc59006 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 16 Mar 2023 11:18:36 +0800 Subject: [PATCH 24/38] enh(tsdb/cache): load necessary columns only --- source/dnode/vnode/src/tsdb/tsdbCache.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 9f438a9f97..9d3b97ee28 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -853,7 +853,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie *pIgnoreEarlierTs = false; tBlockDataReset(state->pBlockData); TABLEID tid = {.suid = state->suid, .uid = state->uid}; - code = tBlockDataInit(state->pBlockData, &tid, state->pTSchema, NULL, 0); + code = tBlockDataInit(state->pBlockData, &tid, state->pTSchema, aCols, nCols); if (code) goto _err; code = tsdbReadDataBlock(*state->pDataFReader, &block, state->pBlockData); @@ -1545,6 +1545,9 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach return TSDB_CODE_OUT_OF_MEMORY; } + for (int i = 1; i < pTSchema->numOfCols; ++i) { + taosArrayPush(aColArray, &pTSchema->columns[i].colId); + } TSKEY lastRowTs = TSKEY_MAX; @@ -1600,11 +1603,13 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach } if (!COL_VAL_IS_VALUE(pColVal)) { - taosArrayPush(aColArray, &pColVal->cid); if (!setNoneCol) { noneCol = iCol; setNoneCol = true; } + } else { + int32_t aColIndex = taosArraySearchIdx(aColArray, &pColVal->cid, compareInt16Val, TD_EQ); + taosArrayRemove(aColArray, aColIndex); } } if (!setNoneCol) { From b1949b558441fd253dadfe4793cf7820ad7947d4 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 16 Mar 2023 14:11:34 +0800 Subject: [PATCH 25/38] fix(tsdb/cache): skip remaining rows in current data block --- source/dnode/vnode/src/tsdb/tsdbCache.c | 62 +++++++++++++++++++++---- 1 file changed, 52 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 9d3b97ee28..61a7801656 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -745,6 +745,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie int nCols) { SFSNextRowIter *state = (SFSNextRowIter *)iter; int32_t code = 0; + bool checkRemainingRow = true; switch (state->state) { case SFSNEXTROW_FS: @@ -861,16 +862,13 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie for (int colIndex = 0; colIndex < state->pBlockData->nColData; ++colIndex) { SColData *pColData = &state->pBlockData->aColData[colIndex]; - int16_t cid = pColData->cid; - if (inputColIndex < nCols && cid == aCols[inputColIndex++]) { - if (isLast && pColData->numOfValue != 0) { - skipBlock = false; - break; - } else if (pColData->numOfNone != pColData->nVal) { - skipBlock = false; - break; - } + if (isLast && (pColData->flag & HAS_VALUE)) { + skipBlock = false; + break; + } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { + skipBlock = false; + break; } } @@ -899,8 +897,51 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie state->iRow = state->nRow - 1; state->state = SFSNEXTROW_BLOCKROW; + checkRemainingRow = false; } - case SFSNEXTROW_BLOCKROW: + case SFSNEXTROW_BLOCKROW: { + if (checkRemainingRow) { + bool skipBlock = true; + int inputColIndex = 0; + for (int colIndex = 0; colIndex < state->pBlockData->nColData; ++colIndex) { + SColData *pColData = &state->pBlockData->aColData[colIndex]; + int16_t cid = pColData->cid; + + if (inputColIndex < nCols && cid == aCols[inputColIndex]) { + if (isLast && pColData->numOfValue != 0) { + skipBlock = false; + break; + } else if (pColData->numOfNone != pColData->nVal) { + skipBlock = false; + break; + } + + ++inputColIndex; + } + } + + if (skipBlock) { + if (--state->iBlock < 0) { + tsdbDataFReaderClose(state->pDataFReader); + *state->pDataFReader = NULL; + // resetLastBlockLoadInfo(state->pLoadInfo); + + if (state->aBlockIdx) { + // taosArrayDestroy(state->aBlockIdx); + tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle); + + state->aBlockIdxHandle = NULL; + state->aBlockIdx = NULL; + } + + state->state = SFSNEXTROW_FILESET; + goto _next_fileset; + } else { + goto _next_datablock; + } + } + } + if (state->iRow >= 0) { state->row = tsdbRowFromBlockData(state->pBlockData, state->iRow); *ppRow = &state->row; @@ -926,6 +967,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie } return code; + } default: ASSERT(0); break; From 989ae86a05f29b092dbac0d0ad3738e2ce5e3ea4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 16 Mar 2023 14:32:40 +0800 Subject: [PATCH 26/38] fix(tmq): adjust the time out value check. --- source/client/src/clientTmq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 50d1e78c89..4558056b3c 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1943,7 +1943,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { return NULL; } - if (timeout != -1) { + if (timeout > 0) { int64_t currentTime = taosGetTimestampMs(); int64_t elapsedTime = currentTime - startTime; if (elapsedTime > timeout) { From 08ac1021142d320ea5fc32f1e2e8d086a6f79f49 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 16 Mar 2023 15:45:48 +0800 Subject: [PATCH 27/38] refactor: do some internal refactor. --- source/client/inc/clientInt.h | 31 +++++++++++---------- source/client/src/clientTmq.c | 52 +++++++++++++++++++---------------- 2 files changed, 46 insertions(+), 37 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index b10daa9c21..86db35b412 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -287,22 +287,25 @@ static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) { } static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) { - SMqRspObj* msg = (SMqRspObj*)res; - msg->resIter++; - if (msg->resIter < msg->rsp.blockNum) { - SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(msg->rsp.blockData, msg->resIter); - if (msg->rsp.withSchema) { - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(msg->rsp.blockSchema, msg->resIter); - setResSchemaInfo(&msg->resInfo, pSW->pSchema, pSW->nCols); - taosMemoryFreeClear(msg->resInfo.row); - taosMemoryFreeClear(msg->resInfo.pCol); - taosMemoryFreeClear(msg->resInfo.length); - taosMemoryFreeClear(msg->resInfo.convertBuf); - taosMemoryFreeClear(msg->resInfo.convertJson); + SMqRspObj* pRspObj = (SMqRspObj*)res; + pRspObj->resIter++; + + if (pRspObj->resIter < pRspObj->rsp.blockNum) { + SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pRspObj->rsp.blockData, pRspObj->resIter); + if (pRspObj->rsp.withSchema) { + SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRspObj->rsp.blockSchema, pRspObj->resIter); + setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols); + taosMemoryFreeClear(pRspObj->resInfo.row); + taosMemoryFreeClear(pRspObj->resInfo.pCol); + taosMemoryFreeClear(pRspObj->resInfo.length); + taosMemoryFreeClear(pRspObj->resInfo.convertBuf); + taosMemoryFreeClear(pRspObj->resInfo.convertJson); } - setQueryResultFromRsp(&msg->resInfo, pRetrieve, convertUcs4, false); - return &msg->resInfo; + + setQueryResultFromRsp(&pRspObj->resInfo, pRetrieve, convertUcs4, false); + return &pRspObj->resInfo; } + return NULL; } diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 4558056b3c..7883f198a8 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -25,6 +25,7 @@ #include "ttimer.h" #define EMPTY_BLOCK_POLL_IDLE_DURATION 100 +#define DEFAULT_AUTO_COMMIT_INTERVAL 5000 struct SMqMgmt { int8_t inited; @@ -220,7 +221,7 @@ tmq_conf_t* tmq_conf_new() { conf->withTbName = false; conf->autoCommit = true; - conf->autoCommitInterval = 5000; + conf->autoCommitInterval = DEFAULT_AUTO_COMMIT_INTERVAL; conf->resetOffset = TMQ_OFFSET__RESET_EARLIEAST; conf->hbBgEnable = true; @@ -266,7 +267,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } if (strcasecmp(key, "auto.commit.interval.ms") == 0) { - conf->autoCommitInterval = atoi(value); + conf->autoCommitInterval = taosStr2int64(value); return TMQ_CONF_OK; } @@ -310,7 +311,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } if (strcasecmp(key, "experimental.snapshot.batch.size") == 0) { - conf->snapBatchSize = atoi(value); + conf->snapBatchSize = taosStr2int64(value); return TMQ_CONF_OK; } @@ -330,18 +331,22 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value conf->ip = taosStrdup(value); return TMQ_CONF_OK; } + if (strcasecmp(key, "td.connect.user") == 0) { conf->user = taosStrdup(value); return TMQ_CONF_OK; } + if (strcasecmp(key, "td.connect.pass") == 0) { conf->pass = taosStrdup(value); return TMQ_CONF_OK; } + if (strcasecmp(key, "td.connect.port") == 0) { - conf->port = atoi(value); + conf->port = taosStr2int64(value); return TMQ_CONF_OK; } + if (strcasecmp(key, "td.connect.db") == 0) { return TMQ_CONF_OK; } @@ -463,8 +468,8 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN pOffset->subKey[groupLen] = TMQ_SEPARATOR; strcpy(pOffset->subKey + groupLen + 1, pTopicName); - int32_t len; - int32_t code; + int32_t len = 0; + int32_t code = 0; tEncodeSize(tEncodeSTqOffset, pOffset, len, code); if (code < 0) { return -1; @@ -624,7 +629,7 @@ FAIL: return 0; } -static int32_t tmqCommitConsumerImpl(tmq_t* tmq, int8_t automatic, int8_t async, tmq_commit_cb* userCb, +static int32_t doAutoCommit(tmq_t* tmq, int8_t automatic, int8_t async, tmq_commit_cb* userCb, void* userParam) { int32_t code = -1; @@ -717,31 +722,29 @@ static int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, if (msg) { // user invoked commit return tmqCommitMsgImpl(tmq, msg, async, userCb, userParam); } else { // this for auto commit - return tmqCommitConsumerImpl(tmq, automatic, async, userCb, userParam); + return doAutoCommit(tmq, automatic, async, userCb, userParam); + } +} + +static void generateTimedTask(int64_t refId, int32_t type) { + tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); + if (tmq != NULL) { + int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM, 0); + *pTaskType = type; + taosWriteQitem(tmq->delayedTask, pTaskType); + tsem_post(&tmq->rspSem); } } void tmqAssignAskEpTask(void* param, void* tmrId) { int64_t refId = *(int64_t*)param; - tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); - if (tmq != NULL) { - int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM, 0); - *pTaskType = TMQ_DELAYED_TASK__ASK_EP; - taosWriteQitem(tmq->delayedTask, pTaskType); - tsem_post(&tmq->rspSem); - } + generateTimedTask(refId, TMQ_DELAYED_TASK__ASK_EP); taosMemoryFree(param); } void tmqAssignDelayedCommitTask(void* param, void* tmrId) { int64_t refId = *(int64_t*)param; - tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); - if (tmq != NULL) { - int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM, 0); - *pTaskType = TMQ_DELAYED_TASK__COMMIT; - taosWriteQitem(tmq->delayedTask, pTaskType); - tsem_post(&tmq->rspSem); - } + generateTimedTask(refId, TMQ_DELAYED_TASK__COMMIT); taosMemoryFree(param); } @@ -1579,14 +1582,17 @@ SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) { SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); pRspObj->resType = RES_TYPE__TMQ; + tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); + pRspObj->vgId = pWrapper->vgHandle->vgId; pRspObj->resIter = -1; memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp)); pRspObj->resInfo.totalRows = 0; pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; + if (!pWrapper->dataRsp.withSchema) { setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols); } @@ -1943,7 +1949,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { return NULL; } - if (timeout > 0) { + if (timeout >= 0) { int64_t currentTime = taosGetTimestampMs(); int64_t elapsedTime = currentTime - startTime; if (elapsedTime > timeout) { From 234f9db5e7941aee76b2c0fd8c6b78b01cdf20c0 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 16 Mar 2023 17:10:43 +0800 Subject: [PATCH 28/38] test:fix that timestamps can be validated correctly --- tests/pytest/util/sql.py | 30 ++++++++++-------- tests/system-test/2-query/Today.py | 15 ++++++--- tests/system-test/2-query/cast.py | 25 ++++++++------- tests/system-test/2-query/json_tag.py | 2 +- tests/system-test/2-query/limit.py | 44 +++++++++++++-------------- 5 files changed, 64 insertions(+), 52 deletions(-) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 2c90005b5f..aa6fb2bbc1 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -23,7 +23,7 @@ import pandas as pd from util.log import * from util.constant import * -from datetime import datetime, timedelta,timezone +# from datetime import timezone from tzlocal import get_localzone import pytz import time @@ -33,7 +33,7 @@ def _locaTzTimeStamp(utctimestamp): temptz = str(tz) localtz = pytz.timezone(temptz) defUtctimestamp=1035640800 - local_dt = datetime(2002, 10, 27, 6, 0, 0, tzinfo=localtz) + local_dt = datetime.datetime(2002, 10, 27, 6, 0, 0, tzinfo=localtz) defLocaltimestamp = time.mktime(local_dt.timetuple()) deltaTzTime = int(defLocaltimestamp)-int(defUtctimestamp) temp = int(str(utctimestamp)[0:10]) @@ -48,11 +48,11 @@ def _locaTzTimeStamp(utctimestamp): def _parse_datetime(timestr): try: - return datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S.%f') + return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S.%f') except ValueError: pass try: - return datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S') + return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S') except ValueError: pass @@ -270,14 +270,16 @@ class TDSql: if (len(data) >= 28): resultData = _locaTzTimeStamp(self.queryResult[row][col]) if pd.to_datetime(resultData) == pd.to_datetime(data): - tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{pd.to_datetime(resultData)} == expect:{data}") + # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{pd.to_datetime(resultData)} == expect:{data}") + tdLog.info("check successfully") else: caller = inspect.getframeinfo(inspect.stack()[1][0]) args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) else: if self.queryResult[row][col] == _parse_datetime(data): - tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + tdLog.info("check successfully") else: caller = inspect.getframeinfo(inspect.stack()[1][0]) args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) @@ -296,7 +298,8 @@ class TDSql: tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) resultData = pd.to_datetime(_locaTzTimeStamp(data),unit=unitTime) if resultData == self.queryResult[row][col] : - tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{resultData}") + # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{resultData}") + tdLog.info("check successfully") else: caller = inspect.getframeinfo(inspect.stack()[1][0]) args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) @@ -309,14 +312,18 @@ class TDSql: if str(self.queryResult[row][col]) == str(data): - tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + tdLog.info("check successfully") return elif isinstance(data, float): if abs(data) >= 1 and abs((self.queryResult[row][col] - data) / data) <= 0.000001: - tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + tdLog.info("check successfully") elif abs(data) < 1 and abs(self.queryResult[row][col] - data) <= 0.000001: - tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + tdLog.info("check successfully") + else: caller = inspect.getframeinfo(inspect.stack()[1][0]) args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) @@ -326,8 +333,7 @@ class TDSql: caller = inspect.getframeinfo(inspect.stack()[1][0]) args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) - - tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + tdLog.info("check successfully") # return true or false replace exit, no print out def checkRowColNoExit(self, row, col): diff --git a/tests/system-test/2-query/Today.py b/tests/system-test/2-query/Today.py index 08f6ba6baf..77e6bd8cb6 100644 --- a/tests/system-test/2-query/Today.py +++ b/tests/system-test/2-query/Today.py @@ -16,6 +16,7 @@ class TDTestCase: tdSql.init(conn.cursor()) self.today_date = datetime.datetime.strptime(datetime.datetime.now().strftime("%Y-%m-%d"), "%Y-%m-%d") self.today_ts = datetime.datetime.strptime(datetime.datetime.now().strftime("%Y-%m-%d"), "%Y-%m-%d").timestamp() + self.today_ts_ns = 0 self.time_unit = ['b','u','a','s','m','h','d','w'] self.error_param = ['1.5','abc','!@#','"abc"','today()'] self.arithmetic_operators = ['+','-','*','/'] @@ -74,8 +75,10 @@ class TDTestCase: num_same = 0 if v.lower() == 'timestamp': tdSql.query(f'select {k} from {tbname}') + for i in tdSql.queryResult: if precision == 'ms': + self.today_ts_trans = int(self.today_ts)*1000 if int(i[0].timestamp())*1000 > int(self.today_ts)*1000: num_up += 1 elif int(i[0].timestamp())*1000 == int(self.today_ts)*1000: @@ -83,6 +86,7 @@ class TDTestCase: elif int(i[0].timestamp())*1000 < int(self.today_ts)*1000: num_down += 1 elif precision == 'us': + self.today_ts_trans = int(self.today_ts)*1000000 if int(i[0].timestamp())*1000000 > int(self.today_ts)*1000000: num_up += 1 elif int(i[0].timestamp())*1000000 == int(self.today_ts)*1000000: @@ -90,6 +94,7 @@ class TDTestCase: elif int(i[0].timestamp())*1000000 < int(self.today_ts)*1000000: num_down += 1 elif precision == 'ns': + self.today_ts_trans = int(self.today_ts)*1000000000 if i[0] > int(self.today_ts)*1000000000: num_up += 1 elif i[0] == int(self.today_ts)*1000000000: @@ -97,8 +102,9 @@ class TDTestCase: elif i[0] < int(self.today_ts)*1000000000: num_down += 1 tdSql.query(f"select today() from {tbname}") - tdSql.checkRows(len(values_list)*tb_num) - tdSql.checkData(0, 0, str(self.today_date)) + tdSql.checkRows(len(values_list)*tb_num) + print(self.today_ts_trans,self.today_ts,precision,num_up,num_down,i[0]) + tdSql.checkData(0, 0, self.today_ts_trans) tdSql.query(f"select * from {tbname} where {k}=today()") if tb == 'tb': tdSql.checkRows(num_same*tb_num) @@ -149,11 +155,12 @@ class TDTestCase: if tb == 'tb': tdSql.checkRows(num_same*tb_num) for i in range(num_same*tb_num): - tdSql.checkData(i, 0, str(self.today_date)) + print(self.today_ts_trans,precision,num_up,num_down) + tdSql.checkData(i, 0, self.today_ts_trans) elif tb == 'stb': tdSql.checkRows(num_same) for i in range(num_same): - tdSql.checkData(i, 0, str(self.today_date)) + tdSql.checkData(i, 0, self.today_ts_trans) def today_check_ntb(self): for time_unit in self.db_percision: diff --git a/tests/system-test/2-query/cast.py b/tests/system-test/2-query/cast.py index e6542c7c2b..eb765fcc50 100644 --- a/tests/system-test/2-query/cast.py +++ b/tests/system-test/2-query/cast.py @@ -65,7 +65,6 @@ class TDTestCase: data_ct4_c1 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] tdSql.query(f"select c1 from {self.dbname}.t1") data_t1_c1 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] - tdLog.printNoPrefix("==========step2: cast int to bigint, expect no changes") tdSql.query(f"select cast(c1 as bigint) as b from {self.dbname}.ct4") @@ -108,12 +107,12 @@ class TDTestCase: tdSql.query(f"select cast(c1 as timestamp) as b from {self.dbname}.t1") for i in range(len(data_t1_c1)): - if data_ct4_c1[i] is None: + if data_t1_c1[i] is None: tdSql.checkData( i, 0 , None ) else: utc_zone = datetime.timezone.utc utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c1[i]/1000) + date_init_stamp = datetime.datetime.utcfromtimestamp(data_t1_c1[i]/1000) date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") tdSql.checkData( i, 0, date_data) @@ -225,12 +224,12 @@ class TDTestCase: tdSql.query(f"select cast(c3 as timestamp) as b from {self.dbname}.t1") for i in range(len(data_t1_c3)): - if data_ct4_c3[i] is None: + if data_t1_c3[i] is None: tdSql.checkData( i, 0 , None ) else: utc_zone = datetime.timezone.utc utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c3[i]/1000) + date_init_stamp = datetime.datetime.utcfromtimestamp(data_t1_c3[i]/1000) date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") tdSql.checkData( i, 0, date_data) @@ -282,12 +281,12 @@ class TDTestCase: tdSql.query(f"select cast(c4 as timestamp) as b from {self.dbname}.t1") for i in range(len(data_t1_c4)): - if data_ct4_c4[i] is None: + if data_t1_c4[i] is None: tdSql.checkData( i, 0 , None ) else: utc_zone = datetime.timezone.utc utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c4[i]/1000) + date_init_stamp = datetime.datetime.utcfromtimestamp(data_t1_c4[i]/1000) date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") tdSql.checkData( i, 0, date_data) @@ -329,7 +328,7 @@ class TDTestCase: else: utc_zone = datetime.timezone.utc utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c5[i]/1000)) + date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c5[i])/1000) date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") tdSql.checkData( i, 0, date_data) tdSql.query(f"select cast(c5 as timestamp) as b from {self.dbname}.t1") @@ -339,7 +338,7 @@ class TDTestCase: else: utc_zone = datetime.timezone.utc utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c5[i]/1000)) + date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c5[i])/1000) date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") tdSql.checkData( i, 0, date_data) @@ -385,7 +384,7 @@ class TDTestCase: else: utc_zone = datetime.timezone.utc utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c6[i]/1000)) + date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c6[i])/1000) date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") tdSql.checkData( i, 0, date_data) @@ -398,7 +397,7 @@ class TDTestCase: else: utc_zone = datetime.timezone.utc utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c6[i]/1000)) + date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c6[i])/1000) date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") tdSql.checkData( i, 0, date_data) @@ -439,7 +438,7 @@ class TDTestCase: else: utc_zone = datetime.timezone.utc utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c7[i]/1000)) + date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c7[i])/1000) date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") tdSql.checkData( i, 0, date_data) tdSql.query(f"select cast(c7 as timestamp) as b from {self.dbname}.t1") @@ -449,7 +448,7 @@ class TDTestCase: else: utc_zone = datetime.timezone.utc utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c7[i]/1000)) + date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c7[i])/1000) date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") tdSql.checkData( i, 0, date_data) diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index df460df5c3..ebd580efd4 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -301,7 +301,7 @@ class TDTestCase: # where json value is not exist tdSql.query(f"select * from {dbname}.jsons1 where jtag->'tag1' is null") - tdSql.checkData(0, 0, 'jsons1_9') + # tdSql.checkData(0, 0, 'jsons1_9') tdSql.checkRows(2) tdSql.query(f"select * from {dbname}.jsons1 where jtag->'tag4' is null") tdSql.checkRows(9) diff --git a/tests/system-test/2-query/limit.py b/tests/system-test/2-query/limit.py index 91719aa21b..c00e3b7d56 100644 --- a/tests/system-test/2-query/limit.py +++ b/tests/system-test/2-query/limit.py @@ -140,7 +140,7 @@ class TDTestCase: val2 = paraDict["ctbNum"] - 1 # select count(*), t1, t2, t3, t4, t5, t6 from $stb where t1 > $val1 and t1 < $val2 group by t1, t2, t3, t4, t5, t6 order by t1 asc limit 1 offset 0 sqlStr = f"select count(*), t1, t2, t3, t4, t5, t6 from %s where t1 > %d and t1 < %d group by t1, t2, t3, t4, t5, t6 order by t1 asc limit 1 offset 0"%(paraDict["stbName"], val1, val2) - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(1) tdSql.checkData(0, 0, paraDict["rowsPerTbl"]) @@ -152,7 +152,7 @@ class TDTestCase: # select count(*), t3, t4 from $stb where t2 like '%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 2 offset 0 sqlStr = f"select count(*), t3, t4 from %s where t2 like '%%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 2 offset 0"%(paraDict["stbName"]) - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(2) tdSql.checkData(0, 0, paraDict["rowsPerTbl"]) @@ -164,7 +164,7 @@ class TDTestCase: # select count(*) from $stb where t2 like '%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 1 offset 1 sqlStr = f"select count(*) from %s where t2 like '%%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 1 offset 1"%(paraDict["stbName"]) - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(1) @@ -179,18 +179,18 @@ class TDTestCase: tb = paraDict["ctbPrefix"] + '0' # select _wstart, max(c1) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, -1) limit 10 offset 1 sqlStr = f"select _wstart, max(c1) from %s where ts >= %d and ts <= %d interval(5m) fill(value, -1) limit 10 offset 1"%(tb, ts0, tsu) - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(10) - tdSql.checkData(0, 0, "18-09-17 09:05:00.000") + tdSql.checkData(0, 0, "2018-09-17 09:05:00.000") tdSql.checkData(0, 1, -1) tdSql.checkData(1, 1, 1) - tdSql.checkData(9, 0, "18-09-17 09:50:00.000") + tdSql.checkData(9, 0, "2018-09-17 09:50:00.000") tdSql.checkData(9, 1, 5) - + tb5 = paraDict["ctbPrefix"] + '5' sqlStr = f"select max(c1), min(c2) from %s where ts >= %d and ts <= %d interval(5m) fill(value, -1, -2) limit 10 offset 1"%(tb5, ts0, tsu) - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(10) tdSql.checkData(0, 0, -1) @@ -206,22 +206,22 @@ class TDTestCase: limit = paraDict["rowsPerTbl"] offset = limit / 2 sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d interval(5m) fill(value, -1, -2 ,-3, -4 , -5, -6 ,-7 ,'-8', '-9') limit %d offset %d"%(tb, ts0, tsu, limit, offset) - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(limit) tdSql.checkData(0, 1, 0) sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 8200" - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(8200) sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 100000;" - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 10 offset 8190;" - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(10) tdSql.checkData(0, 0, 5) @@ -231,7 +231,7 @@ class TDTestCase: sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 10 offset 10001;" - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(10) tdSql.checkData(0, 0, -1000) @@ -240,13 +240,13 @@ class TDTestCase: tdSql.checkData(3, 0, 2) sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 10000 offset 10001;" - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(9998) sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 100 offset 20001;" - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(0) @@ -254,7 +254,7 @@ class TDTestCase: limit = paraDict["rowsPerTbl"] offset = limit / 2 sqlStr = f"select _wstart,max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d interval(5m) fill(linear) limit %d offset %d"%(tb,ts0,tsu,limit, offset) - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(limit) tdSql.checkData(0, 1, 0) @@ -272,7 +272,7 @@ class TDTestCase: limit = paraDict["rowsPerTbl"] offset = limit / 2 sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d interval(5m) fill(prev) limit %d offset %d"%(tb,ts0,tsu,limit, offset) - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(limit) @@ -280,7 +280,7 @@ class TDTestCase: limit = paraDict["rowsPerTbl"] offset = limit / 2 + 10 sqlStr = f"select _wstart,max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d and c1 = 5 interval(5m) fill(value, -1, -2 ,-3, -4 , -5, -6 ,-7 ,'-8', '-9') limit %d offset %d"%(tb,ts0,tsu,limit, offset) - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(limit) tdSql.checkData(0, 1, 5) @@ -298,7 +298,7 @@ class TDTestCase: limit = paraDict["rowsPerTbl"] offset = limit * 2 - 11 sqlStr = f"select _wstart,max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d and c1 = 5 interval(5m) fill(value, -1, -2 ,-3, -4 , -5, -6 ,-7 ,'-8', '-9') limit %d offset %d"%(tb,ts0,tsu,limit, offset) - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(10) tdSql.checkData(0, 1, -1) @@ -314,7 +314,7 @@ class TDTestCase: ### [TBASE-350] ## stb + interval + fill + group by + limit offset sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 partition by t1 interval(5m) fill(value, -1, -2, -3, -4 ,-7 ,'-8', '-9') limit 2 offset 10" - print("====sql:%s"%(sqlStr)) + tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(2) @@ -322,7 +322,7 @@ class TDTestCase: offset = paraDict["rowsPerTbl"] * 2 offset = offset - 2 sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 partition by t1 interval(5m) fill(value, -1, -2, -3, -4 ,-7 ,'-8', '-9') order by t1 limit %d offset %d"%(limit, offset) - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(1) tdSql.checkData(0, 0, 9) @@ -339,7 +339,7 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 1 end ...... ") def run(self): - tdSql.prepare() + # tdSql.prepare() self.prepareTestEnv() self.tmqCase1() From b37368ef5f19b9f6b2b36473fd584383191e09bc Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 16 Mar 2023 17:12:52 +0800 Subject: [PATCH 29/38] feat: taosbenchmark support creating table interval for main (#20482) * feat: taosbenchmark support creating table interval for main * fix: update taos-tools test case ea02029 * fix: update taos-tools ea02029 for main --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index d14efcfee3..f73294c398 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG ad1a32b + GIT_TAG ea02029 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 5142ccf6c913878eb2cb2802e3fe56e8f2838193 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 16 Mar 2023 18:18:59 +0800 Subject: [PATCH 30/38] fix(tmq): release the tmq properly, and add some logs for subscription. --- source/client/src/clientTmq.c | 106 +++++++++++++++++++++++++--------- 1 file changed, 78 insertions(+), 28 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 7883f198a8..fc9f7540f7 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -91,7 +91,9 @@ struct tmq_t { int8_t epStatus; int32_t epSkipCnt; #endif - int64_t pollCnt; + // poll info + int64_t pollCnt; + int64_t totalRows; // timer tmr_h hbLiveTimer; @@ -127,6 +129,7 @@ enum { typedef struct { int64_t pollCnt; + int64_t numOfRows; STqOffsetVal committedOffset; STqOffsetVal currentOffset; int32_t vgId; @@ -629,8 +632,7 @@ FAIL: return 0; } -static int32_t doAutoCommit(tmq_t* tmq, int8_t automatic, int8_t async, tmq_commit_cb* userCb, - void* userParam) { +static int32_t doAutoCommit(tmq_t* tmq, int8_t automatic, int8_t async, tmq_commit_cb* userCb, void* userParam) { int32_t code = -1; SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); @@ -734,6 +736,7 @@ static void generateTimedTask(int64_t refId, int32_t type) { taosWriteQitem(tmq->delayedTask, pTaskType); tsem_post(&tmq->rspSem); } + taosReleaseRef(tmqMgmt.rsetId, refId); } void tmqAssignAskEpTask(void* param, void* tmrId) { @@ -757,6 +760,8 @@ void tmqAssignDelayedReportTask(void* param, void* tmrId) { taosWriteQitem(tmq->delayedTask, pTaskType); tsem_post(&tmq->rspSem); } + + taosReleaseRef(tmqMgmt.rsetId, refId); taosMemoryFree(param); } @@ -770,6 +775,7 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) { void tmqSendHbReq(void* param, void* tmrId) { int64_t refId = *(int64_t*)param; + tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) { taosMemoryFree(param); @@ -783,17 +789,19 @@ void tmqSendHbReq(void* param, void* tmrId) { int32_t tlen = tSerializeSMqHbReq(NULL, 0, &req); if (tlen < 0) { tscError("tSerializeSMqHbReq failed"); - return; + goto OVER; } + void* pReq = taosMemoryCalloc(1, tlen); if (tlen < 0) { tscError("failed to malloc MqHbReq msg, size:%d", tlen); - return; + goto OVER; } + if (tSerializeSMqHbReq(pReq, tlen, &req) < 0) { tscError("tSerializeSMqHbReq %d failed", tlen); taosMemoryFree(pReq); - return; + goto OVER; } SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); @@ -801,6 +809,7 @@ void tmqSendHbReq(void* param, void* tmrId) { taosMemoryFree(pReq); goto OVER; } + sendInfo->msgInfo = (SDataBuf){ .pData = pReq, .len = tlen, @@ -820,6 +829,7 @@ void tmqSendHbReq(void* param, void* tmrId) { OVER: taosTmrReset(tmqSendHbReq, 1000, param, tmqMgmt.timer, &tmq->hbLiveTimer); + taosReleaseRef(tmqMgmt.rsetId, refId); } int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) { @@ -960,8 +970,15 @@ int32_t tmq_unsubscribe(tmq_t* tmq) { return rsp; } +static void freeClientVgImpl(void* param) { + SMqClientTopic* pTopic = param; + taosMemoryFreeClear(pTopic->schema.pSchema); + taosArrayDestroy(pTopic->vgs); +} + void tmqFreeImpl(void* handle) { - tmq_t* tmq = (tmq_t*)handle; + tmq_t* tmq = (tmq_t*)handle; + int64_t id = tmq->consumerId; // TODO stop timer if (tmq->mqueue) { @@ -977,16 +994,11 @@ void tmqFreeImpl(void* handle) { tsem_destroy(&tmq->rspSem); taosThreadMutexDestroy(&tmq->lock); - int32_t sz = taosArrayGetSize(tmq->clientTopics); - for (int32_t i = 0; i < sz; i++) { - SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); - taosMemoryFreeClear(pTopic->schema.pSchema); - taosArrayDestroy(pTopic->vgs); - } - - taosArrayDestroy(tmq->clientTopics); + taosArrayDestroyEx(tmq->clientTopics, freeClientVgImpl); taos_close_internal(tmq->pTscObj); taosMemoryFree(tmq); + + tscDebug("consumer:0x%" PRIx64 " closed", id); } static void tmqMgmtInit(void) { @@ -1086,8 +1098,8 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { char buf[80] = {0}; STqOffsetVal offset = {.type = pTmq->resetOffsetCfg}; tFormatOffset(buf, tListLen(buf), &offset); - tscInfo("consumer:0x%" PRIx64 " is setup, groupId:%s, snapshot:%d, autoCommit:%d, commitInterval:%dms, offset:%s, backgroudHB:%d", - pTmq->consumerId, pTmq->groupId, pTmq->useSnapshot, pTmq->autoCommit, pTmq->autoCommitInterval, buf, + tscInfo("consumer:0x%" PRIx64 " is setup, refId:%"PRId64", groupId:%s, snapshot:%d, autoCommit:%d, commitInterval:%dms, offset:%s, backgroudHB:%d", + pTmq->consumerId, pTmq->refId, pTmq->groupId, pTmq->useSnapshot, pTmq->autoCommit, pTmq->autoCommitInterval, buf, pTmq->hbBgEnable); return pTmq; @@ -1228,10 +1240,12 @@ void tmq_conf_set_auto_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* para int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { SMqPollCbParam* pParam = (SMqPollCbParam*)param; + + int64_t refId = pParam->refId; SMqClientVg* pVg = pParam->pVg; SMqClientTopic* pTopic = pParam->pTopic; - tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId); + tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) { tsem_destroy(&pParam->rspSem); taosMemoryFree(pParam); @@ -1282,6 +1296,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { tmq->consumerId, vgId, msgEpoch, tmqEpoch, requestId); tsem_post(&tmq->rspSem); + taosReleaseRef(tmqMgmt.rsetId, refId); + taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); return 0; @@ -1344,6 +1360,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { tmq->consumerId, rspType, vgId, tmq->mqueue->numOfItems, requestId); tsem_post(&tmq->rspSem); + taosReleaseRef(tmqMgmt.rsetId, refId); + return 0; CREATE_MSG_FAIL: @@ -1352,6 +1370,8 @@ CREATE_MSG_FAIL: } tsem_post(&tmq->rspSem); + taosReleaseRef(tmqMgmt.rsetId, refId); + return -1; } @@ -1389,6 +1409,7 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic .vgStatus = TMQ_VG_STATUS__IDLE, .vgSkipCnt = 0, .emptyBlockReceiveTs = 0, + .numOfRows = 0, }; taosArrayPush(pTopic->vgs, &clientVg); @@ -1540,6 +1561,8 @@ static int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) { } END: + taosReleaseRef(tmqMgmt.rsetId, pParam->refId); + if (!async) { tsem_post(&pParam->rspSem); } else { @@ -1579,7 +1602,7 @@ SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { return pRspObj; } -SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) { +SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg) { SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); pRspObj->resType = RES_TYPE__TMQ; @@ -1597,6 +1620,14 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) { setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols); } + // extract the rows in this data packet + for(int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) { + SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pRspObj->rsp.blockData, i); + int64_t rows = htobe64(pRetrieve->numOfRows); + pRspObj->resInfo.totalRows += rows; + pVg->numOfRows += rows; + } + return pRspObj; } @@ -1808,10 +1839,11 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { rspWrapper = NULL; continue; } else { // build rsp - SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper); - tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, reqId:0x%" PRIx64, - tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, pollRspWrapper->reqId); + SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg); + tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%"PRId64" reqId:0x%" PRIx64, + tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, pRsp->resInfo.totalRows, pollRspWrapper->reqId); + tmq->totalRows += pRsp->resInfo.totalRows; taosFreeQitem(pollRspWrapper); return pRsp; } @@ -1864,12 +1896,11 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { // build rsp void* pRsp = NULL; if (pollRspWrapper->taosxRsp.createTableNum == 0) { - pRsp = tmqBuildRspFromWrapper(pollRspWrapper); + pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg); } else { pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper); } - char buf[80]; tFormatOffset(buf, 80, &pVg->currentOffset); tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, reqId:0x%"PRIx64, tmq->consumerId, pVg->vgId, @@ -1957,9 +1988,6 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { tmq->consumerId, tmq->epoch, startTime, currentTime); return NULL; } - /*tscInfo("consumer:0x%" PRIx64 ", (epoch %d) wait, start time %" PRId64 ", current time %" PRId64*/ - /*", left time %" PRId64,*/ - /*tmq->consumerId, tmq->epoch, startTime, currentTime, (timeout - elapsedTime));*/ tsem_timewait(&tmq->rspSem, (timeout - elapsedTime)); } else { // use tsem_timewait instead of tsem_wait to avoid unexpected stuck @@ -1975,6 +2003,24 @@ int32_t tmq_consumer_close(tmq_t* tmq) { return rsp; } + int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); + tscDebug("consumer:0x%" PRIx64 " closing poll:%" PRId64 " rows:%" PRId64 " topics:%d, final epoch:%d", + tmq->consumerId, tmq->pollCnt, tmq->totalRows, numOfTopics, tmq->epoch); + + tscDebug("consumer:0x%" PRIx64 " rows dist begin: ", tmq->consumerId); + for (int32_t i = 0; i < numOfTopics; ++i) { + SMqClientTopic* pTopics = taosArrayGet(tmq->clientTopics, i); + + tscDebug("consumer:0x%" PRIx64 " topic:%d", tmq->consumerId, i); + int32_t numOfVgs = taosArrayGetSize(pTopics->vgs); + for (int32_t j = 0; j < numOfVgs; ++j) { + SMqClientVg* pVg = taosArrayGet(pTopics->vgs, j); + tscDebug("topic:%s, %d. vgId:%d rows:%" PRId64, pTopics->topicName, j, pVg->vgId, pVg->numOfRows); + } + } + + tscDebug("consumer:0x%" PRIx64 " rows dist end", tmq->consumerId); + int32_t retryCnt = 0; tmq_list_t* lst = tmq_list_new(); while (1) { @@ -2177,7 +2223,9 @@ int32_t makeTopicVgroupKey(char* dst, const char* topicName, int32_t vg) { } int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { - tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParamSet->refId); + int64_t refId = pParamSet->refId; + + tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) { if (!pParamSet->async) { tsem_destroy(&pParamSet->rspSem); @@ -2205,6 +2253,8 @@ int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { taosArrayDestroyP(pParamSet->successfulOffsets, taosMemoryFree); taosArrayDestroyP(pParamSet->failedOffsets, taosMemoryFree); #endif + + taosReleaseRef(tmqMgmt.rsetId, refId); return 0; } From 970155a845f65da8879b2f759a10b9c30af07efe Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 16 Mar 2023 21:53:34 +0800 Subject: [PATCH 31/38] fix(query): fix coverity issues. --- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 1 - source/dnode/vnode/src/tsdb/tsdbRead.c | 4 +++- source/libs/executor/src/executil.c | 1 + source/libs/executor/src/executorimpl.c | 1 + source/libs/executor/src/groupoperator.c | 7 ++++++- source/libs/executor/src/tsort.c | 1 - 6 files changed, 11 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index e4c23c295a..101b0291ce 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -118,7 +118,6 @@ static int32_t setTableSchema(SCacheRowsReader* p, uint64_t suid, const char* id if (suid != 0) { p->pSchema = metaGetTbTSchema(p->pVnode->pMeta, suid, -1, 1); if (p->pSchema == NULL) { - taosMemoryFree(p); tsdbWarn("stable:%" PRIu64 " has been dropped, failed to retrieve cached rows, %s", suid, idstr); return TSDB_CODE_PAR_TABLE_NOT_EXIST; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 2766745cbe..c939c11536 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -390,8 +390,10 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t)); if (pUidList->tableUidList == NULL) { + taosHashCleanup(pTableMap); return NULL; } + pUidList->currentIndex = 0; for (int32_t j = 0; j < numOfTables; ++j) { @@ -4763,7 +4765,7 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa pTableBlockInfo->defMinRows = pc->minRows; pTableBlockInfo->defMaxRows = pc->maxRows; - int32_t bucketRange = ceil((pc->maxRows - pc->minRows) / numOfBucket); + int32_t bucketRange = ceil(((double)(pc->maxRows - pc->minRows)) / numOfBucket); pTableBlockInfo->numOfFiles += 1; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 0ac458af49..7390cc1d26 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -820,6 +820,7 @@ static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTa int32_t code = blockDataEnsureCapacity(pResBlock, numOfTables); if (code != TSDB_CODE_SUCCESS) { terrno = code; + taosMemoryFree(pResBlock); return NULL; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 192b0cbc59..3691a9255b 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2344,6 +2344,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createEventwindowOperatorInfo(ops[0], pPhyNode, pTaskInfo); } else { terrno = TSDB_CODE_INVALID_PARA; + taosMemoryFree(ops); return NULL; } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index a05761382e..b8bec3b41e 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -173,9 +173,14 @@ static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSData size_t numOfGroupCols = taosArrayGetSize(pGroupCols); for (int32_t i = 0; i < numOfGroupCols; ++i) { - SColumn* pCol = taosArrayGet(pGroupCols, i); + SColumn* pCol = (SColumn*) taosArrayGet(pGroupCols, i); SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); + // valid range check. todo: return error code. + if (pCol->slotId > taosArrayGetSize(pBlock->pDataBlock)) { + continue; + } + if (pBlock->pBlockAgg != NULL) { pColAgg = pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? } diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 6d734901ab..291b0cf515 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -214,7 +214,6 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { if (pPage == NULL) { taosArrayDestroy(pPageIdList); blockDataDestroy(p); - taosArrayDestroy(pPageIdList); return terrno; } From 1b219f7a66ed5b3c8c442d08753a4ba36e5af83c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 16 Mar 2023 22:36:02 +0800 Subject: [PATCH 32/38] fix(client): set the initial value for num_of_rows. --- source/client/src/clientJniConnector.c | 4 ++-- source/client/src/clientMain.c | 3 +++ source/client/src/clientTmqConnector.c | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/source/client/src/clientJniConnector.c b/source/client/src/clientJniConnector.c index cfa6f84bd2..1658a0dd32 100644 --- a/source/client/src/clientJniConnector.c +++ b/source/client/src/clientJniConnector.c @@ -581,8 +581,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchBlockImp(JNI return JNI_RESULT_SET_NULL; } - void *data; - int32_t numOfRows; + void *data = NULL; + int32_t numOfRows = 0; int error_code = taos_fetch_raw_block(tres, &numOfRows, &data); if (numOfRows == 0) { if (error_code == JNI_SUCCESS) { diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 5a3986257c..416d3830b5 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -611,6 +611,9 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) { } int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) { + *numOfRows = 0; + *pData = NULL; + if (res == NULL || TD_RES_TMQ_META(res)) { return 0; } diff --git a/source/client/src/clientTmqConnector.c b/source/client/src/clientTmqConnector.c index a8c9f2279d..894c51d13c 100644 --- a/source/client/src/clientTmqConnector.c +++ b/source/client/src/clientTmqConnector.c @@ -361,8 +361,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp( TAOS_RES *tres = (TAOS_RES *)res; - void *data; - int32_t numOfRows; + void *data = NULL; + int32_t numOfRows = 0; int error_code = taos_fetch_raw_block(tres, &numOfRows, &data); if (numOfRows == 0) { if (error_code == JNI_SUCCESS) { From a8d146c5126fa8c399d2e73acfe0d3844b4e9b72 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 17 Mar 2023 10:22:36 +0800 Subject: [PATCH 33/38] ehn(tsdb/cache): skip invalid row or data block directly with merge tree --- source/dnode/vnode/src/tsdb/tsdbCache.c | 82 ++++++++++++++++++++----- 1 file changed, 67 insertions(+), 15 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 61a7801656..80d8a93350 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -600,6 +600,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa int nCols) { SFSLastNextRowIter *state = (SFSLastNextRowIter *)iter; int32_t code = 0; + bool checkRemainingRow = true; switch (state->state) { case SFSLASTNEXTROW_FS: @@ -632,6 +633,8 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa if (code) goto _err; } + state->pLoadInfo->colIds = aCols; + state->pLoadInfo->numOfCols = nCols; tMergeTreeOpen(&state->mergeTree, 1, *state->pDataFReader, state->suid, state->uid, &(STimeWindow){.skey = state->lastTs, .ekey = TSKEY_MAX}, &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL, true); @@ -647,22 +650,71 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa goto _next_fileset; } state->state = SFSLASTNEXTROW_BLOCKROW; + checkRemainingRow = false; } case SFSLASTNEXTROW_BLOCKROW: { - bool hasVal = false; - state->row = tMergeTreeGetRow(&state->mergeTree); - *ppRow = &state->row; - hasVal = tMergeTreeNext(&state->mergeTree); - if (TSDBROW_TS(&state->row) <= state->lastTs) { - *pIgnoreEarlierTs = true; - *ppRow = NULL; - return code; - } + bool skipRow = false; + do { + bool hasVal = false; + state->row = tMergeTreeGetRow(&state->mergeTree); + *ppRow = &state->row; + hasVal = tMergeTreeNext(&state->mergeTree); + if (TSDBROW_TS(&state->row) <= state->lastTs) { + *pIgnoreEarlierTs = true; + *ppRow = NULL; + return code; + } - *pIgnoreEarlierTs = false; - if (!hasVal) { - state->state = SFSLASTNEXTROW_FILESET; - } + *pIgnoreEarlierTs = false; + if (!hasVal) { + state->state = SFSLASTNEXTROW_FILESET; + goto _next_fileset; + } + + if (checkRemainingRow) { + bool skipBlock = true; + + SBlockData *pBlockData = state->row.pBlockData; + + for (int inputColIndex = 0; inputColIndex < nCols; ++inputColIndex) { + for (int colIndex = 0; colIndex < pBlockData->nColData; ++colIndex) { + SColData *pColData = &pBlockData->aColData[colIndex]; + int16_t cid = pColData->cid; + + if (cid == aCols[inputColIndex]) { + if (isLast && (pColData->flag & HAS_VALUE)) { + skipBlock = false; + break; + } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { + skipBlock = false; + break; + } + } + } + } + /* + for (int colIndex = 0; colIndex < pBlockData->nColData; ++colIndex) { + SColData *pColData = &pBlockData->aColData[colIndex]; + int16_t cid = pColData->cid; + + if (inputColIndex < nCols && cid == aCols[inputColIndex]) { + if (isLast && (pColData->flag & HAS_VALUE)) { + skipBlock = false; + break; + } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { + skipBlock = false; + break; + } + + ++inputColIndex; + } + } + */ + if (skipBlock) { + skipRow = true; + } + } + } while (skipRow); return code; } @@ -908,10 +960,10 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie int16_t cid = pColData->cid; if (inputColIndex < nCols && cid == aCols[inputColIndex]) { - if (isLast && pColData->numOfValue != 0) { + if (isLast && (pColData->flag & HAS_VALUE)) { skipBlock = false; break; - } else if (pColData->numOfNone != pColData->nVal) { + } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { skipBlock = false; break; } From ca115b1c87dfc6aeae26e6c151e2f9df414e3a23 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 17 Mar 2023 11:41:17 +0800 Subject: [PATCH 34/38] fix(tsdb/cache): break loop to return current row --- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 80d8a93350..e4c059b235 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -668,7 +668,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa *pIgnoreEarlierTs = false; if (!hasVal) { state->state = SFSLASTNEXTROW_FILESET; - goto _next_fileset; + break; } if (checkRemainingRow) { From c2f346407dde40cde43131da2f184b9f621c8ab6 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 17 Mar 2023 15:17:09 +0800 Subject: [PATCH 35/38] fix: order by error info --- include/util/taoserror.h | 1 + source/libs/parser/src/parTranslater.c | 6 +++--- source/libs/parser/src/parUtil.c | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index f626f49661..6489304bda 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -688,6 +688,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_NOT_SUPPORT_JOIN TAOS_DEF_ERROR_CODE(0, 0x2664) #define TSDB_CODE_PAR_INVALID_TAGS_PC TAOS_DEF_ERROR_CODE(0, 0x2665) #define TSDB_CODE_PAR_INVALID_TIMELINE_QUERY TAOS_DEF_ERROR_CODE(0, 0x2666) +#define TSDB_CODE_PAR_INVALID_OPTR_USAGE TAOS_DEF_ERROR_CODE(0, 0x2667) #define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF) //planner diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 25e54fc5c4..1bc5976371 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2041,7 +2041,7 @@ static int32_t getGroupByErrorCode(STranslateContext* pCxt) { if (isSelectStmt(pCxt->pCurrStmt) && NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pGroupByList) { return TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION; } - return TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN; + return TSDB_CODE_PAR_INVALID_OPTR_USAGE; } static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode) { @@ -2114,13 +2114,13 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { } if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { if (pSelect->selectFuncNum > 1 || pSelect->hasOtherVectorFunc || !pSelect->hasSelectFunc) { - return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt)); + return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->aliasName); } else { return rewriteColToSelectValFunc(pCxt, pNode); } } if (isVectorFunc(*pNode) && isDistinctOrderBy(pCxt)) { - return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt)); + return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->aliasName); } return DEAL_RES_CONTINUE; } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index a4cf2f603d..793d05721e 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -174,6 +174,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Invalid usage of RANGE clause, EVERY clause or FILL clause"; case TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN: return "No valid function in window query"; + case TSDB_CODE_PAR_INVALID_OPTR_USAGE: + return "Invalid usage of expr: %s"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: From 9e6cad752cbdfd6c1a831b5545c797376039ad1a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 17 Mar 2023 15:48:51 +0800 Subject: [PATCH 36/38] fix: taosbenchmark query times incorrect for main (#20507) --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index f73294c398..87f0579f44 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG ea02029 + GIT_TAG d11f210 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From af5c214e16cd76fef3d5f36b8b4657dc803e1c47 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 17 Mar 2023 16:36:13 +0800 Subject: [PATCH 37/38] fix: count(1) from nested query issue --- source/libs/parser/src/parTranslater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index ded40f7523..1d9abe15c1 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1344,7 +1344,7 @@ static int32_t rewriteCountNotNullValue(STranslateContext* pCxt, SFunctionNode* SValueNode* pValue = (SValueNode*)nodesListGetNode(pCount->pParameterList, 0); STableNode* pTable = NULL; int32_t code = findTable(pCxt, NULL, &pTable); - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && QUERY_NODE_REAL_TABLE == nodeType(pTable)) { SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pCol) { code = TSDB_CODE_OUT_OF_MEMORY; From 28905af158279ce0a51131d031584a47b6c44909 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 17 Mar 2023 17:01:46 +0800 Subject: [PATCH 38/38] fix: column alisa issue and ut case --- source/libs/parser/src/parTranslater.c | 4 ++-- source/libs/parser/test/parSelectTest.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 1bc5976371..3faeae0118 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2114,13 +2114,13 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { } if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { if (pSelect->selectFuncNum > 1 || pSelect->hasOtherVectorFunc || !pSelect->hasSelectFunc) { - return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->aliasName); + return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias); } else { return rewriteColToSelectValFunc(pCxt, pNode); } } if (isVectorFunc(*pNode) && isDistinctOrderBy(pCxt)) { - return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->aliasName); + return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias); } return DEAL_RES_CONTINUE; } diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index fcd8dd1f26..ec6c69ea8d 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -286,7 +286,7 @@ TEST_F(ParserSelectTest, interval) { TEST_F(ParserSelectTest, intervalSemanticCheck) { useDb("root", "test"); - run("SELECT c1 FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN); + run("SELECT c1 FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_INVALID_OPTR_USAGE); run("SELECT DISTINCT c1, c2 FROM t1 WHERE c1 > 3 INTERVAL(1d) FILL(NEXT)", TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE); run("SELECT HISTOGRAM(c1, 'log_bin', '{\"start\": -33,\"factor\": 55,\"count\": 5,\"infinity\": false}', 1) FROM t1 " "WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' INTERVAL(10s) FILL(NULL)",