From bfc279835cd628cac45f1dd014ff30aca9d6c75a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Mon, 10 Apr 2023 16:52:41 +0800 Subject: [PATCH 01/17] add cancel in fetch cluster --- source/dnode/mnode/impl/src/mndCluster.c | 27 +++++++++++++++--------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 94584dfe58..ea2a5857c6 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -67,7 +67,7 @@ int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) { return 0; } -static SClusterObj *mndAcquireCluster(SMnode *pMnode) { +static SClusterObj *mndAcquireCluster(SMnode *pMnode, void **ppIter) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; @@ -76,23 +76,27 @@ static SClusterObj *mndAcquireCluster(SMnode *pMnode) { pIter = sdbFetch(pSdb, SDB_CLUSTER, pIter, (void **)&pCluster); if (pIter == NULL) break; + *ppIter = pIter; + return pCluster; } return NULL; } -static void mndReleaseCluster(SMnode *pMnode, SClusterObj *pCluster) { +static void mndReleaseCluster(SMnode *pMnode, SClusterObj *pCluster, void *pIter) { SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pCluster); } int64_t mndGetClusterId(SMnode *pMnode) { int64_t clusterId = 0; - SClusterObj *pCluster = mndAcquireCluster(pMnode); + void *pIter = NULL; + SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); if (pCluster != NULL) { clusterId = pCluster->id; - mndReleaseCluster(pMnode, pCluster); + mndReleaseCluster(pMnode, pCluster, pIter); } return clusterId; @@ -100,10 +104,11 @@ int64_t mndGetClusterId(SMnode *pMnode) { int64_t mndGetClusterCreateTime(SMnode *pMnode) { int64_t createTime = 0; - SClusterObj *pCluster = mndAcquireCluster(pMnode); + void **ppIter = NULL; + SClusterObj *pCluster = mndAcquireCluster(pMnode, ppIter); if (pCluster != NULL) { createTime = pCluster->createdTime; - mndReleaseCluster(pMnode, pCluster); + mndReleaseCluster(pMnode, pCluster, *ppIter); } return createTime; @@ -121,10 +126,11 @@ static int32_t mndGetClusterUpTimeImp(SClusterObj *pCluster) { float mndGetClusterUpTime(SMnode *pMnode) { int64_t upTime = 0; - SClusterObj *pCluster = mndAcquireCluster(pMnode); + void *pIter = NULL; + SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); if (pCluster != NULL) { upTime = mndGetClusterUpTimeImp(pCluster); - mndReleaseCluster(pMnode, pCluster); + mndReleaseCluster(pMnode, pCluster, pIter); } return upTime / 86400.0f; @@ -321,11 +327,12 @@ static void mndCancelGetNextCluster(SMnode *pMnode, void *pIter) { static int32_t mndProcessUptimeTimer(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SClusterObj clusterObj = {0}; - SClusterObj *pCluster = mndAcquireCluster(pMnode); + void *pIter = NULL; + SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); if (pCluster != NULL) { memcpy(&clusterObj, pCluster, sizeof(SClusterObj)); clusterObj.upTime += tsUptimeInterval; - mndReleaseCluster(pMnode, pCluster); + mndReleaseCluster(pMnode, pCluster, pIter); } if (clusterObj.id <= 0) { From adf25d3300547bc31907d4418aeb2ba0f5e2bd0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Mon, 10 Apr 2023 19:12:13 +0800 Subject: [PATCH 02/17] null pointer --- source/dnode/mnode/impl/src/mndCluster.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index ea2a5857c6..4d05637a2b 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -104,11 +104,11 @@ int64_t mndGetClusterId(SMnode *pMnode) { int64_t mndGetClusterCreateTime(SMnode *pMnode) { int64_t createTime = 0; - void **ppIter = NULL; - SClusterObj *pCluster = mndAcquireCluster(pMnode, ppIter); + void *pIter = NULL; + SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); if (pCluster != NULL) { createTime = pCluster->createdTime; - mndReleaseCluster(pMnode, pCluster, *ppIter); + mndReleaseCluster(pMnode, pCluster, pIter); } return createTime; From d163a16f3a1e02128250b1b2327bf7014a952030 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 11 Apr 2023 15:46:32 +0800 Subject: [PATCH 03/17] fix: fix max/min(tag) random result --- source/libs/executor/src/scanoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 29990f2d06..ea834e90ca 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -484,7 +484,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int int32_t code = 0; // backup the rows - int32_t backupRows = pBlock->info.rows; + int32_t backupRows = (rows == 1) ? rows : pBlock->info.rows; pBlock->info.rows = rows; bool freeReader = false; From bbae7259cf3a8dffca2cfbf8c874eb66afcc95df Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 11 Apr 2023 15:47:27 +0800 Subject: [PATCH 04/17] add test cases --- tests/system-test/2-query/max.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/system-test/2-query/max.py b/tests/system-test/2-query/max.py index b8da02b9a6..ba6ab53fc7 100644 --- a/tests/system-test/2-query/max.py +++ b/tests/system-test/2-query/max.py @@ -20,8 +20,8 @@ class TDTestCase: intData = [] floatData = [] tdSql.execute(f'''create table {dbname}.stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 tinyint unsigned, col6 smallint unsigned, - col7 int unsigned, col8 bigint unsigned, col9 float, col10 double, col11 bool, col12 binary(20), col13 nchar(20)) tags(loc nchar(20))''') - tdSql.execute(f"create table {dbname}.stb_1 using {dbname}.stb tags('beijing')") + col7 int unsigned, col8 bigint unsigned, col9 float, col10 double, col11 bool, col12 binary(20), col13 nchar(20)) tags(t0 tinyint, t1 float, loc nchar(20))''') + tdSql.execute(f"create table {dbname}.stb_1 using {dbname}.stb tags(5, 5.5, 'beijing')") for i in range(self.rowNum): tdSql.execute(f"insert into {dbname}.stb_1 values(%d, %d, %d, %d, %d, %d, %d, %d, %d, %f, %f, %d, '{self.binary_str}%d', '{self.nchar_str}%d')" % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) @@ -55,13 +55,20 @@ class TDTestCase: tdSql.checkData(0, 1, np.max(intData)) tdSql.query(f"select ts, min(col9) from {dbname}.stb") - tdSql.checkRows(1) + tdSql.checkRows(1) tdSql.checkData(0, 1, np.min(floatData)) tdSql.query(f"select ts, min(col9) from {dbname}.stb_1") - tdSql.checkRows(1) + tdSql.checkRows(1) tdSql.checkData(0, 1, np.min(floatData)) + # check tags + tdSql.query(f"select max(t0) from {dbname}.stb") + tdSql.checkData(0,0,5) + + tdSql.query(f"select max(t1) from {dbname}.stb") + tdSql.checkData(0,0,5.5) + def max_check_ntb_base(self, dbname="db"): tdSql.prepare() intData = [] From 0201348bbe1089710c3af53e66029996d11495ac Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 11 Apr 2023 15:47:27 +0800 Subject: [PATCH 05/17] add test cases --- source/libs/executor/src/scanoperator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ea834e90ca..f0827886a0 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -484,8 +484,8 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int int32_t code = 0; // backup the rows - int32_t backupRows = (rows == 1) ? rows : pBlock->info.rows; - pBlock->info.rows = rows; + int32_t backupRows = pBlock->info.rows; + pBlock->info.rows = (rows < pBlock->info.rows) ? pBlock->info.rows : rows; bool freeReader = false; STableCachedVal val = {0}; From d551075bbada11f4bafafdb7ecfeeb8f4f1ea4de Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 12 Apr 2023 16:22:25 +0800 Subject: [PATCH 06/17] fix: taosdump continue if ts out of range for main (#20887) --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 0110b27b32..d8bf3a09b4 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 149ac34 + GIT_TAG 0681d8b SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 5ea2ba0a387d2685b57cc03bc07dfafd8d930cf0 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 12 Apr 2023 17:39:23 +0800 Subject: [PATCH 07/17] fix: error querying after setting permissions for varchar type tags --- source/libs/parser/src/parTranslater.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 1d6d123cb4..80f334019b 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1235,6 +1235,10 @@ static int32_t calcTypeBytes(SDataType dt) { } static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { + if (pVal->translate) { + return TSDB_CODE_SUCCESS; + } + SDataType dt = pVal->node.resType; dt.bytes = calcTypeBytes(dt); return translateValueImpl(pCxt, pVal, dt, false); @@ -1692,7 +1696,8 @@ static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) { pSelect->hasUniqueFunc = pSelect->hasUniqueFunc ? true : (FUNCTION_TYPE_UNIQUE == pFunc->funcType); pSelect->hasTailFunc = pSelect->hasTailFunc ? true : (FUNCTION_TYPE_TAIL == pFunc->funcType); pSelect->hasInterpFunc = pSelect->hasInterpFunc ? true : (FUNCTION_TYPE_INTERP == pFunc->funcType); - pSelect->hasInterpPseudoColFunc = pSelect->hasInterpPseudoColFunc ? true : fmIsInterpPseudoColumnFunc(pFunc->funcId); + pSelect->hasInterpPseudoColFunc = + pSelect->hasInterpPseudoColFunc ? true : fmIsInterpPseudoColumnFunc(pFunc->funcId); pSelect->hasLastRowFunc = pSelect->hasLastRowFunc ? true : (FUNCTION_TYPE_LAST_ROW == pFunc->funcType); pSelect->hasLastFunc = pSelect->hasLastFunc ? true : (FUNCTION_TYPE_LAST == pFunc->funcType); pSelect->hasTimeLineFunc = pSelect->hasTimeLineFunc ? true : fmIsTimelineFunc(pFunc->funcId); @@ -3372,7 +3377,8 @@ static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE); } if (pSelect->hasInterpPseudoColFunc) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, "Has Interp pseudo column(s) but missing interp function"); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, + "Has Interp pseudo column(s) but missing interp function"); } return TSDB_CODE_SUCCESS; } From 1ed3149c393a77f5386457397d20ad03261f1e8d Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 12 Apr 2023 17:41:43 +0800 Subject: [PATCH 08/17] fix: the precision of delete statement --- source/libs/parser/src/parTranslater.c | 8 ++ source/libs/scalar/src/scalar.c | 2 +- tests/script/tsim/insert/delete0.sim | 161 +++++++++++++++++++++++++ tests/script/tsim/testsuit.sim | 1 + 4 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 tests/script/tsim/insert/delete0.sim diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 1d6d123cb4..fdaeba2f0f 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -645,6 +645,10 @@ static bool isSelectStmt(SNode* pCurrStmt) { return NULL != pCurrStmt && QUERY_NODE_SELECT_STMT == nodeType(pCurrStmt); } +static bool isDeleteStmt(SNode* pCurrStmt) { + return NULL != pCurrStmt && QUERY_NODE_DELETE_STMT == nodeType(pCurrStmt); +} + static bool isSetOperator(SNode* pCurrStmt) { return NULL != pCurrStmt && QUERY_NODE_SET_OPERATOR == nodeType(pCurrStmt); } @@ -669,6 +673,9 @@ static uint8_t getPrecisionFromCurrStmt(SNode* pCurrStmt, uint8_t defaultVal) { if (NULL != pCurrStmt && QUERY_NODE_CREATE_STREAM_STMT == nodeType(pCurrStmt)) { return getPrecisionFromCurrStmt(((SCreateStreamStmt*)pCurrStmt)->pQuery, defaultVal); } + if (isDeleteStmt(pCurrStmt)) { + return ((SDeleteStmt*)pCurrStmt)->precision; + } return defaultVal; } @@ -3741,6 +3748,7 @@ static int32_t translateDelete(STranslateContext* pCxt, SDeleteStmt* pDelete) { pCxt->pCurrStmt = (SNode*)pDelete; int32_t code = translateFrom(pCxt, pDelete->pFromTable); if (TSDB_CODE_SUCCESS == code) { + pDelete->precision = ((STableNode*)pDelete->pFromTable)->precision; code = translateDeleteWhere(pCxt, pDelete); } pCxt->currClause = SQL_CLAUSE_SELECT; diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index fe01977b2e..47bea920ef 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -463,7 +463,7 @@ int32_t sclInitParamList(SScalarParam **pParams, SNodeList *pParamList, SScalarC sclError("calloc %d failed", (int32_t)((*paramNum) * sizeof(SScalarParam))); SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - + if (pParamList) { SNode *tnode = NULL; int32_t i = 0; diff --git a/tests/script/tsim/insert/delete0.sim b/tests/script/tsim/insert/delete0.sim new file mode 100644 index 0000000000..c8a3ba7092 --- /dev/null +++ b/tests/script/tsim/insert/delete0.sim @@ -0,0 +1,161 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print =============== create database with different precision +sql create database d0 keep 365000 +sql create database d1 keep 365000 precision 'ms' +sql create database d2 keep 365000 precision 'us' +sql create database d3 keep 50000 precision 'ns' + +sql select * from information_schema.ins_databases +if $rows != 6 then + return -1 +endi + +print $data00 $data01 $data02 + + +sql create table if not exists d0.stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists d1.stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists d2.stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists d3.stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists d0.ntb (ts timestamp, c1 int, c2 float, c3 double) +sql create table if not exists d1.ntb (ts timestamp, c1 int, c2 float, c3 double) +sql create table if not exists d2.ntb (ts timestamp, c1 int, c2 float, c3 double) +sql create table if not exists d3.ntb (ts timestamp, c1 int, c2 float, c3 double) + +sql create table d0.ct1 using d0.stb tags(1000) +sql create table d1.ct1 using d1.stb tags(1000) +sql create table d2.ct1 using d2.stb tags(1000) +sql create table d3.ct1 using d3.stb tags(1000) +sql create table d0.ct2 using d0.stb tags(1000) +sql create table d1.ct2 using d1.stb tags(1000) +sql create table d2.ct2 using d2.stb tags(1000) +sql create table d3.ct2 using d3.stb tags(1000) + + +sql insert into d0.ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into d1.ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into d2.ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into d3.ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into d0.ct2 values(now+0s, 10, 2.0, 3.0) +sql insert into d1.ct2 values(now+0s, 10, 2.0, 3.0) +sql insert into d2.ct2 values(now+0s, 10, 2.0, 3.0) +sql insert into d3.ct2 values(now+0s, 10, 2.0, 3.0) +sql insert into d0.ntb values(now+0s, 10, 2.0, 3.0) +sql insert into d1.ntb values(now+0s, 10, 2.0, 3.0) +sql insert into d2.ntb values(now+0s, 10, 2.0, 3.0) +sql insert into d3.ntb values(now+0s, 10, 2.0, 3.0) + + +print =============== query data from super table +sql select count(*) from d0.stb +if $data00 != 2 then + return -1 +endi +sql select count(*) from d1.stb +if $data00 != 2 then + return -1 +endi +sql select count(*) from d2.stb +if $data00 != 2 then + return -1 +endi +sql select count(*) from d3.stb +if $data00 != 2 then + return -1 +endi + +print =============== delete from child table +sql delete from d0.ct1 where ts < now() +sql delete from d1.ct1 where ts < now() +sql delete from d2.ct1 where ts < now() +sql delete from d3.ct1 where ts < now() + + +print =============== query data from super table +sql select count(*) from d0.stb +if $data00 != 1 then + return -1 +endi +sql select count(*) from d1.stb +if $data00 != 1 then + return -1 +endi +sql select count(*) from d2.stb +if $data00 != 1 then + return -1 +endi +sql select count(*) from d3.stb +if $data00 != 1 then + return -1 +endi +print =============== query data from normal table +sql select count(*) from d0.ntb +if $data00 != 1 then + return -1 +endi +sql select count(*) from d1.ntb +if $data00 != 1 then + return -1 +endi +sql select count(*) from d2.ntb +if $data00 != 1 then + return -1 +endi +sql select count(*) from d3.ntb +if $data00 != 1 then + return -1 +endi + +print =============== delete from super table +sql delete from d0.stb where ts < now() +sql delete from d1.stb where ts < now() +sql delete from d2.stb where ts < now() +sql delete from d3.stb where ts < now() + +print =============== query data from super table +sql select count(*) from d0.stb +if $data00 != 0 then + return -1 +endi +sql select count(*) from d1.stb +if $data00 != 0 then + return -1 +endi +sql select count(*) from d2.stb +if $data00 != 0 then + return -1 +endi +sql select count(*) from d3.stb +if $data00 != 0 then + return -1 +endi + +print =============== delete from normal table +sql delete from d0.ntb where ts < now() +sql delete from d1.ntb where ts < now() +sql delete from d2.ntb where ts < now() +sql delete from d3.ntb where ts < now() + +print =============== query data from normal table +sql select count(*) from d0.ntb +if $data00 != 0 then + return -1 +endi +sql select count(*) from d1.ntb +if $data00 != 0 then + return -1 +endi +sql select count(*) from d2.ntb +if $data00 != 0 then + return -1 +endi +sql select count(*) from d3.ntb +if $data00 != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/testsuit.sim b/tests/script/tsim/testsuit.sim index c5fbf41b66..0abe56ab3c 100644 --- a/tests/script/tsim/testsuit.sim +++ b/tests/script/tsim/testsuit.sim @@ -114,6 +114,7 @@ run tsim/insert/basic1.sim run tsim/insert/commit-merge0.sim run tsim/insert/basic0.sim run tsim/insert/update0.sim +run tsim/insert/delete0.sim run tsim/insert/backquote.sim run tsim/insert/null.sim run tsim/catalog/alterInCurrent.sim From 3bd721518d7a6f203f3552c8667ceb2d7bd35479 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 12 Apr 2023 17:43:44 +0800 Subject: [PATCH 09/17] chore: revert the extra line --- source/libs/scalar/src/scalar.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 47bea920ef..47cabe1e6c 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -463,7 +463,6 @@ int32_t sclInitParamList(SScalarParam **pParams, SNodeList *pParamList, SScalarC sclError("calloc %d failed", (int32_t)((*paramNum) * sizeof(SScalarParam))); SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - if (pParamList) { SNode *tnode = NULL; int32_t i = 0; From 4ac4b4f0c892d1d117fda80e81c977dfc45ea95f Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 12 Apr 2023 17:45:12 +0800 Subject: [PATCH 10/17] chore: revert the extra line --- source/libs/scalar/src/scalar.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 47cabe1e6c..fe01977b2e 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -463,6 +463,7 @@ int32_t sclInitParamList(SScalarParam **pParams, SNodeList *pParamList, SScalarC sclError("calloc %d failed", (int32_t)((*paramNum) * sizeof(SScalarParam))); SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } + if (pParamList) { SNode *tnode = NULL; int32_t i = 0; From bd24dcf3748e1c5f3577b90cdfc729e4bae87ef1 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 12 Apr 2023 17:59:21 +0800 Subject: [PATCH 11/17] chore: more code --- tests/script/tsim/insert/delete0.sim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/script/tsim/insert/delete0.sim b/tests/script/tsim/insert/delete0.sim index c8a3ba7092..5653853643 100644 --- a/tests/script/tsim/insert/delete0.sim +++ b/tests/script/tsim/insert/delete0.sim @@ -4,10 +4,10 @@ system sh/exec.sh -n dnode1 -s start sql connect print =============== create database with different precision -sql create database d0 keep 365000 -sql create database d1 keep 365000 precision 'ms' -sql create database d2 keep 365000 precision 'us' -sql create database d3 keep 50000 precision 'ns' +sql create database d0 keep 365 +sql create database d1 keep 365 precision 'ms' +sql create database d2 keep 365 precision 'us' +sql create database d3 keep 365 precision 'ns' sql select * from information_schema.ins_databases if $rows != 6 then From 0145cbe100921844e8f54988f90084fb02367afe Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 12 Apr 2023 18:39:06 +0800 Subject: [PATCH 12/17] chore: add test case to CI --- tests/parallel_test/cases.task | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e6aa2f1fea..b37f6e7929 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -783,6 +783,7 @@ ,,y,script,./test.sh -f tsim/insert/query_multi_file.sim ,,y,script,./test.sh -f tsim/insert/tcp.sim ,,y,script,./test.sh -f tsim/insert/update0.sim +,,y,script,./test.sh -f tsim/insert/delete0.sim ,,y,script,./test.sh -f tsim/insert/update1_sort_merge.sim ,,y,script,./test.sh -f tsim/insert/update2.sim ,,y,script,./test.sh -f tsim/parser/alter__for_community_version.sim From 81610773c512a18d232698c4ec1828ecdd497766 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 11 Apr 2023 15:47:27 +0800 Subject: [PATCH 13/17] add test cases --- source/libs/executor/src/scanoperator.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index f0827886a0..852223378c 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -329,7 +329,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 ", uid:%" PRIu64, GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, pBlockInfo->id.uid); - doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1); + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); pCost->skipBlocks += 1; tsdbReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; @@ -340,7 +340,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca if (success) { // failed to load the block sma data, data block statistics does not exist, load data block instead qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 , GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1); + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); tsdbReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else { @@ -485,7 +485,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int // backup the rows int32_t backupRows = pBlock->info.rows; - pBlock->info.rows = (rows < pBlock->info.rows) ? pBlock->info.rows : rows; + pBlock->info.rows = rows; bool freeReader = false; STableCachedVal val = {0}; From fd88602572103a93c7be85b85f5172ccd9b1cb27 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 12 Apr 2023 20:04:19 +0800 Subject: [PATCH 14/17] test: reopen tmqDelete-1ctb.py --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index b37f6e7929..65cddc4248 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -89,7 +89,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py -# ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStbCtb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py From ef219095cf524e58aa2673b0b698b7d311a96077 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 13 Apr 2023 10:06:49 +0800 Subject: [PATCH 15/17] fix(tsdb/read): remove duplicat schema fetching --- source/dnode/vnode/src/tsdb/tsdbRead.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index b80c952ee0..d868c54fd1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2320,7 +2320,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, pRow, pSchema); } else { - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + // STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2352,7 +2352,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, piRow, piSchema); } else { init = true; - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); + // STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); code = tsdbRowMergerInit(&merge, pSchema, piRow, piSchema); if (code != TSDB_CODE_SUCCESS) { return code; From 589ed9d0cee9f77ac132ed46ef7a529afd928231 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 13 Apr 2023 12:09:44 +0800 Subject: [PATCH 16/17] fix(tsdb/cache): fix block index ref releasing --- source/dnode/vnode/src/tsdb/tsdbCache.c | 13 ++++++++++++- tests/script/tsim/parser/last_cache.sim | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index b25e45228f..c09a286c6b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -35,7 +35,11 @@ _err: static void tsdbCloseBICache(STsdb *pTsdb) { SLRUCache *pCache = pTsdb->biCache; if (pCache) { + int32_t elems = taosLRUCacheGetElems(pCache); + tsdbTrace("vgId:%d, elems: %d", TD_VID(pTsdb->pVnode), elems); taosLRUCacheEraseUnrefEntries(pCache); + elems = taosLRUCacheGetElems(pCache); + tsdbTrace("vgId:%d, elems: %d", TD_VID(pTsdb->pVnode), elems); taosLRUCacheCleanup(pCache); @@ -819,7 +823,12 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie * &state->blockIdx); */ state->pBlockIdx = taosArraySearch(state->aBlockIdx, state->pBlockIdxExp, tCmprBlockIdx, TD_EQ); - if (!state->pBlockIdx) { /* + if (!state->pBlockIdx) { + tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle); + + state->aBlockIdxHandle = NULL; + state->aBlockIdx = NULL; + /* tsdbDataFReaderClose(state->pDataFReader); *state->pDataFReader = NULL; resetLastBlockLoadInfo(state->pLoadInfo);*/ @@ -1936,6 +1945,7 @@ int32_t tsdbCacheGetBlockIdx(SLRUCache *pCache, SDataFReader *pFileReader, LRUHa taosThreadMutexUnlock(&pTsdb->biMutex); } + tsdbTrace("bi cache:%p, ref", pCache); *handle = h; return code; @@ -1945,6 +1955,7 @@ int32_t tsdbBICacheRelease(SLRUCache *pCache, LRUHandle *h) { int32_t code = 0; taosLRUCacheRelease(pCache, h, false); + tsdbTrace("bi cache:%p, release", pCache); return code; } diff --git a/tests/script/tsim/parser/last_cache.sim b/tests/script/tsim/parser/last_cache.sim index 9a41a9f5aa..ef7215d6e9 100644 --- a/tests/script/tsim/parser/last_cache.sim +++ b/tests/script/tsim/parser/last_cache.sim @@ -53,7 +53,7 @@ sql insert into tbc values ("2021-05-11 10:12:29",36, 37, NULL, -4005) sql insert into tbd values ("2021-05-11 10:12:29",NULL,NULL,NULL,NULL ) run tsim/parser/last_cache_query.sim - +sql flush database $db system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start From 1b198bc40fa79e287dbeea2d45491b50484d4dcd Mon Sep 17 00:00:00 2001 From: Xuefeng Tan <1172915550@qq.com> Date: Thu, 13 Apr 2023 13:25:23 +0800 Subject: [PATCH 17/17] enh(taosAdapter): make the schemaless automatic database creation configurable (#20902) --- cmake/taosadapter_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index b2f335e1f7..ba937b40c1 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG cb1e89c + GIT_TAG e02ddb2 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE