From 48be8006396cead5825e23085c21edf981369477 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 27 Sep 2023 15:46:25 +0800 Subject: [PATCH 01/36] fix ip white for all users --- source/dnode/mnode/impl/inc/mndUser.h | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 8 ++++---- source/dnode/mnode/impl/src/mndUser.c | 26 ++++++++++++++++++++++---- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index fab8ee4707..1aa01fd59d 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -47,7 +47,7 @@ void mndUserFreeObj(SUserObj *pUser); int64_t mndGetIpWhiteVer(SMnode *pMnode); -void mndUpdateIpWhite(SMnode *pMnode, char *user, char *fqdn, int8_t type, int8_t lock); +void mndUpdateIpWhiteForAllUser(SMnode *pMnode, char *user, char *fqdn, int8_t type, int8_t lock); int32_t mndRefreshUserIpWhiteList(SMnode *pMnode); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index ca7a639c09..f2a8c82192 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -147,7 +147,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; - mndUpdateIpWhite(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1); + mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1); _OVER: mndTransDrop(pTrans); @@ -752,7 +752,7 @@ static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pC if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; - mndUpdateIpWhite(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1); + mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1); _OVER: mndTransDrop(pTrans); sdbFreeRaw(pRaw); @@ -1045,7 +1045,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; - mndUpdateIpWhite(pMnode, TSDB_DEFAULT_USER, pDnode->fqdn, IP_WHITE_DROP, 1); + mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, pDnode->fqdn, IP_WHITE_DROP, 1); code = 0; _OVER: @@ -1191,7 +1191,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { strcpy(dcfgReq.config, "monitor"); snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); -} else if (strncasecmp(cfgReq.config, "ttlpushinterval", 14) == 0) { + } else if (strncasecmp(cfgReq.config, "ttlpushinterval", 14) == 0) { int32_t optLen = strlen("ttlpushinterval"); int32_t flag = -1; int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 3c4bb42379..e8c756acea 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -58,7 +58,7 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBloc static void mndCancelGetNextPrivileges(SMnode *pMnode, void *pIter); SHashObj *mndFetchAllIpWhite(SMnode *pMnode); static int32_t mndProcesSRetrieveIpWhiteReq(SRpcMsg *pReq); -bool mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8_t type); +bool mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8_t type); void ipWhiteMgtUpdateAll(SMnode *pMnode); typedef struct { @@ -104,6 +104,8 @@ int32_t ipWhiteMgtUpdate(SMnode *pMnode, char *user, SIpWhiteList *pNew) { for (int i = 0; i < taosArrayGetSize(fqdns); i++) { char *fqdn = taosArrayGetP(fqdns, i); mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, TSDB_DEFAULT_USER, fqdn, IP_WHITE_ADD); + + mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, user, fqdn, IP_WHITE_ADD); } for (int i = 0; i < taosArrayGetSize(fqdns); i++) { @@ -115,8 +117,7 @@ int32_t ipWhiteMgtUpdate(SMnode *pMnode, char *user, SIpWhiteList *pNew) { // for (int i = 0; i < taosArrayGetSize(pUserNames); i++) { // taosMemoryFree(taosArrayGetP(pUserNames, i)); // } - //taosArrayDestroy(pUserNames); - + // taosArrayDestroy(pUserNames); if (update) ipWhiteMgt.ver++; @@ -302,7 +303,7 @@ int32_t mndRefreshUserIpWhiteList(SMnode *pMnode) { return 0; } -void mndUpdateIpWhite(SMnode *pMnode, char *user, char *fqdn, int8_t type, int8_t lock) { +void mndUpdateIpWhiteForAllUser(SMnode *pMnode, char *user, char *fqdn, int8_t type, int8_t lock) { if (lock) { taosThreadRwlockWrlock(&ipWhiteMgt.rw); if (ipWhiteMgt.ver == 0) { @@ -313,6 +314,23 @@ void mndUpdateIpWhite(SMnode *pMnode, char *user, char *fqdn, int8_t type, int8_ } bool update = mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, user, fqdn, type); + + SArray *pUsers = taosArrayInit(24, sizeof(void *)); + void *pIter = taosHashIterate(ipWhiteMgt.pIpWhiteTab, NULL); + while (pIter) { + char *key = taosHashGetKey(pIter, NULL); + void *elem = taosStrdup(key); + taosArrayPush(pUsers, &elem); + } + + for (int i = 0; i < taosArrayGetSize(pUsers); i++) { + char *p = taosArrayGetP(pUsers, i); + mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, p, fqdn, type); + + taosMemoryFree(p); + } + taosArrayDestroy(pUsers); + if (update) ipWhiteMgt.ver++; if (lock) taosThreadRwlockUnlock(&ipWhiteMgt.rw); From d0e1aec2704cc69cf6d1030d4834bec32ba4c828 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 27 Sep 2023 16:08:09 +0800 Subject: [PATCH 02/36] fix ip white for all users --- source/dnode/mnode/impl/src/mndUser.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index e8c756acea..ddc5a12fd4 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -321,11 +321,13 @@ void mndUpdateIpWhiteForAllUser(SMnode *pMnode, char *user, char *fqdn, int8_t t char *key = taosHashGetKey(pIter, NULL); void *elem = taosStrdup(key); taosArrayPush(pUsers, &elem); + + pIter = taosHashIterate(ipWhiteMgt.pIpWhiteTab, pIter); } for (int i = 0; i < taosArrayGetSize(pUsers); i++) { char *p = taosArrayGetP(pUsers, i); - mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, p, fqdn, type); + update |= mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, p, fqdn, type); taosMemoryFree(p); } From 62e1d4cb651617585f57f106730722dc478916ff Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 27 Sep 2023 16:10:02 +0800 Subject: [PATCH 03/36] fix ip white for all users --- source/dnode/mnode/impl/src/mndUser.c | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index ddc5a12fd4..d36bd61a40 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -315,24 +315,14 @@ void mndUpdateIpWhiteForAllUser(SMnode *pMnode, char *user, char *fqdn, int8_t t bool update = mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, user, fqdn, type); - SArray *pUsers = taosArrayInit(24, sizeof(void *)); - void *pIter = taosHashIterate(ipWhiteMgt.pIpWhiteTab, NULL); + void *pIter = taosHashIterate(ipWhiteMgt.pIpWhiteTab, NULL); while (pIter) { char *key = taosHashGetKey(pIter, NULL); - void *elem = taosStrdup(key); - taosArrayPush(pUsers, &elem); + update |= mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, key, fqdn, type); pIter = taosHashIterate(ipWhiteMgt.pIpWhiteTab, pIter); } - for (int i = 0; i < taosArrayGetSize(pUsers); i++) { - char *p = taosArrayGetP(pUsers, i); - update |= mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, p, fqdn, type); - - taosMemoryFree(p); - } - taosArrayDestroy(pUsers); - if (update) ipWhiteMgt.ver++; if (lock) taosThreadRwlockUnlock(&ipWhiteMgt.rw); From 28dbe36b9120b673c62edc499436d2a914e43aa6 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 27 Sep 2023 16:41:29 +0800 Subject: [PATCH 04/36] fix ip white for all users --- source/dnode/mnode/impl/src/mndUser.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index d36bd61a40..7798d4de3b 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -103,9 +103,8 @@ int32_t ipWhiteMgtUpdate(SMnode *pMnode, char *user, SIpWhiteList *pNew) { for (int i = 0; i < taosArrayGetSize(fqdns); i++) { char *fqdn = taosArrayGetP(fqdns, i); - mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, TSDB_DEFAULT_USER, fqdn, IP_WHITE_ADD); - - mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, user, fqdn, IP_WHITE_ADD); + update |= mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, TSDB_DEFAULT_USER, fqdn, IP_WHITE_ADD); + update |= mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, user, fqdn, IP_WHITE_ADD); } for (int i = 0; i < taosArrayGetSize(fqdns); i++) { From 777b442d888b7b2922cfe6dd9bbe7b4733b4dc68 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 27 Sep 2023 18:38:55 +0800 Subject: [PATCH 05/36] fix: add keep time offset case --- tests/parallel_test/cases.task | 1 + .../meta/database/keep_time_offset.py | 69 +++++++++++++++++++ utils/TSZ/sz/src/Huffman.c | 5 +- 3 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 tests/system-test/eco-system/meta/database/keep_time_offset.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 9995c2970f..36b8fded81 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -815,6 +815,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-20582.py ,,n,system-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/insertMix.py -N 3 ,,n,system-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/stt.py -N 3 +,,n,system-test,python3 ./test.py -f eco-system/meta/database/keep_time_offset.py #tsim test ,,y,script,./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim diff --git a/tests/system-test/eco-system/meta/database/keep_time_offset.py b/tests/system-test/eco-system/meta/database/keep_time_offset.py new file mode 100644 index 0000000000..02c832dcc0 --- /dev/null +++ b/tests/system-test/eco-system/meta/database/keep_time_offset.py @@ -0,0 +1,69 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import re +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + + def create_db(self): + hours = 8 + # create + keep_str = f"KEEP_TIME_OFFSET {hours}" + tdSql.execute(f"create database db {keep_str}") + + # check result + tdSql.query(" show create database db") + create_sql = tdSql.getData(0, 1) + if create_sql.find(keep_str) == -1: + tdLog.exit(f"create database sql not found {keep_str} sql:{create_sql}") + + # alter + hours = 4 + keep_str = f"KEEP_TIME_OFFSET {hours}" + tdSql.execute(f"alter database db {keep_str}") + + # check result + tdSql.query(" show create database db") + create_sql = tdSql.getData(0, 1) + if create_sql.find(keep_str) == -1: + tdLog.exit(f"create database sql not found {keep_str} sql:{create_sql}") + + def check_old_syntax(self): + # old syntax would not support again + tdSql.error("alter dnode 1 'keeptimeoffset 10';") + + + def run(self): + # check new syntax right + self.create_db() + + # check old syntax error + self.check_old_syntax() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/utils/TSZ/sz/src/Huffman.c b/utils/TSZ/sz/src/Huffman.c index 9868f3c0cb..c2c091e328 100644 --- a/utils/TSZ/sz/src/Huffman.c +++ b/utils/TSZ/sz/src/Huffman.c @@ -125,7 +125,10 @@ void build_code(HuffmanTree *huffmanTree, node n, int len, unsigned long out1, u huffmanTree->code[n->c] = (unsigned long*)malloc(2*sizeof(unsigned long)); if(len<=64) { - (huffmanTree->code[n->c])[0] = out1 << (64 - len); + if(len == 0) + (huffmanTree->code[n->c])[0] = 0; + else + (huffmanTree->code[n->c])[0] = out1 << (64 - len); (huffmanTree->code[n->c])[1] = out2; } else From 63ce2f6f2f5b661c3f48d08bb261d916499f80b8 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 27 Sep 2023 19:04:01 +0800 Subject: [PATCH 06/36] fix: alter database keep_time_offset can't changed --- source/dnode/mnode/impl/src/mndDb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 1562160a7b..a27de37daf 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -294,6 +294,7 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) { pOld->cfg.daysToKeep0 = pNew->cfg.daysToKeep0; pOld->cfg.daysToKeep1 = pNew->cfg.daysToKeep1; pOld->cfg.daysToKeep2 = pNew->cfg.daysToKeep2; + pOld->cfg.keepTimeOffset = pNew->cfg.keepTimeOffset; pOld->cfg.walFsyncPeriod = pNew->cfg.walFsyncPeriod; pOld->cfg.walLevel = pNew->cfg.walLevel; pOld->cfg.walRetentionPeriod = pNew->cfg.walRetentionPeriod; From 5a684f2a26be621ef8de62bd41578e8582e37b5d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 27 Sep 2023 22:32:54 +0800 Subject: [PATCH 07/36] fix invalid read --- source/dnode/mnode/impl/src/mndUser.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 7798d4de3b..f38f825302 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -316,8 +316,13 @@ void mndUpdateIpWhiteForAllUser(SMnode *pMnode, char *user, char *fqdn, int8_t t void *pIter = taosHashIterate(ipWhiteMgt.pIpWhiteTab, NULL); while (pIter) { - char *key = taosHashGetKey(pIter, NULL); - update |= mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, key, fqdn, type); + size_t klen = 0; + char *key = taosHashGetKey(pIter, &klen); + + char *keyDup = taosMemoryCalloc(1, klen + 1); + memcpy(keyDup, key, klen); + update |= mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, keyDup, fqdn, type); + taosMemoryFree(keyDup); pIter = taosHashIterate(ipWhiteMgt.pIpWhiteTab, pIter); } From f659e36d2c27acf28567edc8ab9e7e86b9266698 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 28 Sep 2023 09:20:12 +0800 Subject: [PATCH 08/36] fix: internal error when show normal db.tables like pattern --- source/libs/parser/src/parTranslater.c | 26 ++++++++++++++++--- .../script/tsim/query/show_db_table_kind.sim | 10 +++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 959cd81e06..dab5110a43 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -8018,9 +8018,29 @@ static int32_t insertCondIntoSelectStmt(SSelectStmt* pSelect, SNode* pCond) { if (pSelect->pWhere == NULL) { pSelect->pWhere = pCond; } else { - SNode* pWhere = NULL; - createLogicCondNode(pSelect->pWhere, pCond, &pWhere, LOGIC_COND_TYPE_AND); - pSelect->pWhere = pWhere; + SNodeList* pLogicCondListWhere = NULL; + SNodeList* pLogicCondList2 = NULL; + if (nodeType(pSelect->pWhere) == QUERY_NODE_LOGIC_CONDITION && + ((SLogicConditionNode*)pSelect->pWhere)->condType == LOGIC_COND_TYPE_AND) { + pLogicCondListWhere = ((SLogicConditionNode*)pSelect->pWhere)->pParameterList; + } else { + nodesListMakeAppend(&pLogicCondListWhere, pSelect->pWhere); + } + + if (nodeType(pCond) == QUERY_NODE_LOGIC_CONDITION && + ((SLogicConditionNode*)pCond)->condType == LOGIC_COND_TYPE_AND) { + pLogicCondList2 = ((SLogicConditionNode*)pCond)->pParameterList; + } else { + nodesListMakeAppend(&pLogicCondList2, pCond); + } + + nodesListAppendList(pLogicCondListWhere, pLogicCondList2); + + SLogicConditionNode* pWhere = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + pWhere->condType = LOGIC_COND_TYPE_AND; + pWhere->pParameterList = pLogicCondListWhere; + + pSelect->pWhere = (SNode*)pWhere; } return TSDB_CODE_SUCCESS; } diff --git a/tests/script/tsim/query/show_db_table_kind.sim b/tests/script/tsim/query/show_db_table_kind.sim index 9be2d36fd1..a35ca78a34 100644 --- a/tests/script/tsim/query/show_db_table_kind.sim +++ b/tests/script/tsim/query/show_db_table_kind.sim @@ -94,4 +94,14 @@ print $rows if $rows != 5 then return -1 endi +sql show child db2.tables like '%' +print $rows +if $rows != 5 then + return -1 +endi +sql show normal db2.tables like '%' +print $rows +if $rows != 0 then + return -1 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT From eae1249f754a5a34a4005d5e1f6abaa6bcf95402 Mon Sep 17 00:00:00 2001 From: t_max <1172915550@qq.com> Date: Thu, 28 Sep 2023 09:54:19 +0800 Subject: [PATCH 09/36] fix(CMAKE): fix curl cmake file --- cmake/curl_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index c632b4ef8e..1f2291c519 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -1,7 +1,7 @@ # curl ExternalProject_Add(curl URL https://curl.se/download/curl-8.2.1.tar.gz - URL_HASH MD5 b25588a43556068be05e1624e0e74d41 + URL_HASH MD5=b25588a43556068be05e1624e0e74d41 DOWNLOAD_NO_PROGRESS 1 DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" #GIT_REPOSITORY https://github.com/curl/curl.git From b1a0912a614dd07a3ea343d3b21de67f2ca70e5d Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 28 Sep 2023 10:15:34 +0800 Subject: [PATCH 10/36] enh: exclude systables in log/audit db --- source/dnode/vnode/inc/vnode.h | 5 +++-- source/dnode/vnode/src/meta/metaCache.c | 24 ++++++++++++++++++++---- source/dnode/vnode/src/meta/metaTable.c | 15 +++++++++++---- source/dnode/vnode/src/vnd/vnodeQuery.c | 17 +++++++++++++---- 4 files changed, 47 insertions(+), 14 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 85d840ccf5..d6491c8fc7 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -89,6 +89,7 @@ int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num); int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num); int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num); int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num); +bool vnodeSkipTimeSeries(SVnode *pVnode, const char *stbName); void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad); int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); @@ -132,8 +133,8 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); int32_t metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList); int32_t metaPutTbGroupToCache(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, int32_t payloadLen); -bool metaTbInFilterCache(void *pVnode, tb_uid_t suid, int8_t type); -int32_t metaPutTbToFilterCache(void *pVnode, tb_uid_t suid, int8_t type); +bool metaTbInFilterCache(void *pVnode, void* key, int8_t type); +int32_t metaPutTbToFilterCache(void *pVnode, void* key, int8_t type); int32_t metaSizeOfTbFilterCache(void *pVnode, int8_t type); int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables, int32_t *numOfCols); diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 6918634b5d..0a8c1c410a 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -69,6 +69,7 @@ struct SMetaCache { struct STbFilterCache { SHashObj* pStb; + SHashObj* pStbName; } STbFilterCache; }; @@ -178,6 +179,12 @@ int32_t metaCacheOpen(SMeta* pMeta) { goto _err2; } + pCache->STbFilterCache.pStbName = taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); + if (pCache->STbFilterCache.pStbName == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err2; + } + pMeta->pCache = pCache; return code; @@ -204,6 +211,7 @@ void metaCacheClose(SMeta* pMeta) { taosHashCleanup(pMeta->pCache->STbGroupResCache.pTableEntry); taosHashCleanup(pMeta->pCache->STbFilterCache.pStb); + taosHashCleanup(pMeta->pCache->STbFilterCache.pStbName); taosMemoryFree(pMeta->pCache); pMeta->pCache = NULL; @@ -893,21 +901,29 @@ int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid) { return TSDB_CODE_SUCCESS; } -bool metaTbInFilterCache(void* pVnode, tb_uid_t suid, int8_t type) { +bool metaTbInFilterCache(void* pVnode, void* key, int8_t type) { SMeta* pMeta = ((SVnode*)pVnode)->pMeta; - if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pStb, &suid, sizeof(suid))) { + if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pStb, key, sizeof(tb_uid_t))) { + return true; + } + + if (type == 1 && taosHashGet(pMeta->pCache->STbFilterCache.pStbName, key, strlen(key))) { return true; } return false; } -int32_t metaPutTbToFilterCache(void* pVnode, tb_uid_t suid, int8_t type) { +int32_t metaPutTbToFilterCache(void* pVnode, void* key, int8_t type) { SMeta* pMeta = ((SVnode*)pVnode)->pMeta; if (type == 0) { - return taosHashPut(pMeta->pCache->STbFilterCache.pStb, &suid, sizeof(suid), NULL, 0); + return taosHashPut(pMeta->pCache->STbFilterCache.pStb, key, sizeof(tb_uid_t), NULL, 0); + } + + if (type == 1) { + return taosHashPut(pMeta->pCache->STbFilterCache.pStbName, key, strlen(key), NULL, 0); } return 0; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index f065fe3268..112b7bd5c2 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -392,6 +392,10 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { nStbEntry.stbEntry.schemaTag = pReq->schemaTag; int32_t deltaCol = pReq->schemaRow.nCols - oStbEntry.stbEntry.schemaRow.nCols; + bool skipStatis = false; + if (deltaCol != 0) { + + } metaWLock(pMeta); // compare two entry @@ -796,9 +800,12 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs #endif ++pStats->numOfCTables; - int32_t nCols = 0; - metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols); - pStats->numOfTimeSeries += nCols - 1; + + if (metaTbInFilterCache(pMeta->pVnode, pReq->ctb.stbName, 1)) { + int32_t nCols = 0; + metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols); + pStats->numOfTimeSeries += nCols - 1; + } metaWLock(pMeta); metaUpdateStbStats(pMeta, me.ctbEntry.suid, 1, 0); @@ -1059,7 +1066,7 @@ static int metaDeleteTtl(SMeta *pMeta, const SMetaEntry *pME) { return ttlMgrDeleteTtl(pMeta->pTtlMgr, &ctx); } -static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid) { +static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, char* stbName) { void *pData = NULL; int nData = 0; int rc = 0; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 01292f33e4..c96428d835 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -566,7 +566,7 @@ int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) { } #ifdef TD_ENTERPRISE -#define TK_LOG_STB_NUM 19 +#define TK_LOG_STB_NUM 20 static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info", "data_dir", "dnodes_info", @@ -585,7 +585,8 @@ static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info", "taosadapter_system_mem_percent", "temp_dir", "vgroups_info", - "vnodes_role"}; + "vnodes_role", + "operations"}; // exclude stbs of taoskeeper log static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { @@ -598,7 +599,7 @@ static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { for (int32_t i = 0; i < TK_LOG_STB_NUM; ++i) { tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, tkLogStb[i]); if (suid != 0) { - metaPutTbToFilterCache(pVnode, suid, 0); + metaPutTbToFilterCache(pVnode, &suid, 0); } } tbSize = metaSizeOfTbFilterCache(pVnode, 0); @@ -606,12 +607,20 @@ static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { return tbSize; } + +bool vnodeSkipTimeSeries(SVnode *pVnode, const char *stbName) { + char *dbName = strchr(pVnode->config.dbname, '.'); + if (!dbName || 0 != strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { + return 0; + } + +} #endif static bool vnodeTimeSeriesFilter(void *arg1, void *arg2) { SVnode *pVnode = (SVnode *)arg1; - if (metaTbInFilterCache(pVnode, *(tb_uid_t *)(arg2), 0)) { + if (metaTbInFilterCache(pVnode, arg2, 0)) { return true; } return false; From 2d3652c723d1e55ac4560542a58f28f0c34fc658 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 28 Sep 2023 16:15:35 +0800 Subject: [PATCH 11/36] fix(keep0-duration): force keep0 at least thrice of duration --- source/libs/parser/src/parTranslater.c | 43 ++++++++++++-------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index dab5110a43..f0a12eecdc 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "parInt.h" #include "parTranslater.h" +#include "parInt.h" #include "catalog.h" #include "cmdnodes.h" @@ -1209,37 +1209,37 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal, break; } case TSDB_DATA_TYPE_VARBINARY: { - if (pVal->node.resType.type != TSDB_DATA_TYPE_BINARY){ + if (pVal->node.resType.type != TSDB_DATA_TYPE_BINARY) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } - void* data = NULL; + void* data = NULL; uint32_t size = 0; - bool isHexChar = isHex(pVal->literal, strlen(pVal->literal)); - if(isHexChar){ - if(!isValidateHex(pVal->literal, strlen(pVal->literal))){ + bool isHexChar = isHex(pVal->literal, strlen(pVal->literal)); + if (isHexChar) { + if (!isValidateHex(pVal->literal, strlen(pVal->literal))) { return TSDB_CODE_PAR_INVALID_VARBINARY; } - if(taosHex2Ascii(pVal->literal, strlen(pVal->literal), &data, &size) < 0){ + if (taosHex2Ascii(pVal->literal, strlen(pVal->literal), &data, &size) < 0) { return TSDB_CODE_OUT_OF_MEMORY; } - }else{ + } else { size = pVal->node.resType.bytes; data = pVal->literal; } if (size + VARSTR_HEADER_SIZE > targetDt.bytes) { - if(isHexChar) taosMemoryFree(data); + if (isHexChar) taosMemoryFree(data); return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_VALUE_TOO_LONG, pVal->literal); } pVal->datum.p = taosMemoryCalloc(1, size + VARSTR_HEADER_SIZE); if (NULL == pVal->datum.p) { - if(isHexChar) taosMemoryFree(data); + if (isHexChar) taosMemoryFree(data); return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY); } varDataSetLen(pVal->datum.p, size); memcpy(varDataVal(pVal->datum.p), data, size); - if(isHexChar) taosMemoryFree(data); + if (isHexChar) taosMemoryFree(data); break; } case TSDB_DATA_TYPE_VARCHAR: @@ -1789,7 +1789,6 @@ static int32_t translateBlockDistFunc(STranslateContext* pCtx, SFunctionNode* pF return TSDB_CODE_SUCCESS; } - static bool isStarParam(SNode* pNode) { return nodesIsStar(pNode) || nodesIsTableStar(pNode); } static int32_t translateMultiResFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { @@ -2811,7 +2810,8 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { pJoinTable->table.precision = calcJoinTablePrecision(pJoinTable); pJoinTable->table.singleTable = joinTableIsSingleTable(pJoinTable); code = translateExpr(pCxt, &pJoinTable->pOnCond); - pJoinTable->hasSubQuery = (nodeType(pJoinTable->pLeft) != QUERY_NODE_REAL_TABLE) || (nodeType(pJoinTable->pRight) != QUERY_NODE_REAL_TABLE); + pJoinTable->hasSubQuery = (nodeType(pJoinTable->pLeft) != QUERY_NODE_REAL_TABLE) || + (nodeType(pJoinTable->pRight) != QUERY_NODE_REAL_TABLE); if (nodeType(pJoinTable->pLeft) == QUERY_NODE_JOIN_TABLE) { ((SJoinTableNode*)pJoinTable->pLeft)->isLowLevelJoin = true; } @@ -2827,7 +2827,7 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { return code; } -static int32_t createAllColumns(STranslateContext* pCxt, bool igTags, SNodeList** pCols) { +static int32_t createAllColumns(STranslateContext* pCxt, bool igTags, SNodeList** pCols) { *pCols = nodesMakeList(); if (NULL == *pCols) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_OUT_OF_MEMORY); @@ -2870,7 +2870,7 @@ static SNode* createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr) { } else { len = snprintf(buf, sizeof(buf) - 1, "%s(%s.%s)", pSrcFunc->functionName, pCol->tableAlias, pCol->colName); taosCreateMD5Hash(buf, len); - strncpy(pFunc->node.aliasName, buf, TSDB_COL_NAME_LEN - 1); + strncpy(pFunc->node.aliasName, buf, TSDB_COL_NAME_LEN - 1); len = snprintf(buf, sizeof(buf) - 1, "%s(%s)", pSrcFunc->functionName, pCol->colName); // note: userAlias could be truncated here strncpy(pFunc->node.userAlias, buf, TSDB_COL_NAME_LEN - 1); @@ -2992,11 +2992,8 @@ static int32_t createTags(STranslateContext* pCxt, SNodeList** pOutput) { return TSDB_CODE_SUCCESS; } - #ifndef TD_ENTERPRISE -int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) { - return TSDB_CODE_SUCCESS; -} +int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) { return TSDB_CODE_SUCCESS; } #endif static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect) { @@ -3712,7 +3709,8 @@ static int32_t removeConstantValueFromList(SNodeList** pList) { SNode* pNode = NULL; WHERE_EACH(pNode, *pList) { if (nodeType(pNode) == QUERY_NODE_VALUE || - (nodeType(pNode) == QUERY_NODE_FUNCTION && fmIsConstantResFunc((SFunctionNode*)pNode) && fmIsScalarFunc(((SFunctionNode*)pNode)->funcId))) { + (nodeType(pNode) == QUERY_NODE_FUNCTION && fmIsConstantResFunc((SFunctionNode*)pNode) && + fmIsScalarFunc(((SFunctionNode*)pNode)->funcId))) { ERASE_NODE(*pList); continue; } @@ -4557,9 +4555,9 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa daysPerFile = (-1 == daysPerFile ? dbCfg.daysPerFile : daysPerFile); daysToKeep0 = (-1 == daysToKeep0 ? dbCfg.daysToKeep0 : daysToKeep0); } - if (daysPerFile > daysToKeep0) { + if (daysPerFile > daysToKeep0 / 3) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, - "Invalid duration value, should be keep2 >= keep1 >= keep0 >= duration"); + "Invalid duration value, should be keep2 >= keep1 >= keep0 >= 3 * duration"); } return TSDB_CODE_SUCCESS; } @@ -8114,7 +8112,6 @@ static int32_t addShowKindCond(const SShowStmt* pShow, SSelectStmt* pSelect) { return TSDB_CODE_SUCCESS; } - static int32_t createShowCondition(const SShowStmt* pShow, SSelectStmt* pSelect) { SNode* pDbCond = NULL; SNode* pTbCond = NULL; From 43eee9ab84ae8709b6e69a6f98b900fab6eb3bad Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 28 Sep 2023 16:30:52 +0800 Subject: [PATCH 12/36] chore: add notify test --- tests/script/api/passwdTest.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/tests/script/api/passwdTest.c b/tests/script/api/passwdTest.c index 24059bd35d..7fb70ec5aa 100644 --- a/tests/script/api/passwdTest.c +++ b/tests/script/api/passwdTest.c @@ -30,7 +30,7 @@ #define nDup 1 #define nRoot 10 -#define nUser 10 +#define nUser 1 #define USER_LEN 24 #define BUF_LEN 1024 @@ -207,8 +207,8 @@ int main(int argc, char *argv[]) { exit(1); } createUsers(taos, argv[1], qstr); - passVerTestMulti(argv[1], qstr); - sysInfoTest(taos, argv[1], qstr); + // passVerTestMulti(argv[1], qstr); + // sysInfoTest(taos, argv[1], qstr); userDroppedTest(taos, argv[1], qstr); taos_close(taos); @@ -391,6 +391,10 @@ _REP: void userDroppedTest(TAOS *taos, const char *host, char *qstr) { // users int nTestUsers = nUser; + int nLoop = 0; +_loop: + ++nLoop; + printf("%s:%d loop %d\n", __func__, __LINE__, nLoop); for (int i = 0; i < nTestUsers; ++i) { // sprintf(users[i], "user%d", i); taosu[i] = taos_connect(host, users[i], "taos", NULL, 0); @@ -437,5 +441,14 @@ void userDroppedTest(TAOS *taos, const char *host, char *qstr) { exit(1); } fprintf(stderr, "######## %s #########\n", __func__); - // sleep(300); + + if (nLoop < 5) { + for (int i = 0; i < nTestUsers; ++i) { + sprintf(users[i], "user%d", i); + sprintf(qstr, "CREATE USER %s PASS 'taos'", users[i]); + queryDB(taos, qstr); + goto _loop; + } + // sleep(300); + } } \ No newline at end of file From ca0c2c321887ccefb5bdd2aa8ab28622ef036d8a Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 28 Sep 2023 16:33:29 +0800 Subject: [PATCH 13/36] chore: add notify test --- tests/script/api/passwdTest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/api/passwdTest.c b/tests/script/api/passwdTest.c index 7fb70ec5aa..eb3c37cfa8 100644 --- a/tests/script/api/passwdTest.c +++ b/tests/script/api/passwdTest.c @@ -30,7 +30,7 @@ #define nDup 1 #define nRoot 10 -#define nUser 1 +#define nUser 10 #define USER_LEN 24 #define BUF_LEN 1024 From 9be327bac011f246ae8151dba10d50a8dacd0bbd Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 28 Sep 2023 16:42:58 +0800 Subject: [PATCH 14/36] case: check keep_time_offset with sysinfo table --- .../eco-system/meta/database/keep_time_offset.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/system-test/eco-system/meta/database/keep_time_offset.py b/tests/system-test/eco-system/meta/database/keep_time_offset.py index 02c832dcc0..f1c5398f49 100644 --- a/tests/system-test/eco-system/meta/database/keep_time_offset.py +++ b/tests/system-test/eco-system/meta/database/keep_time_offset.py @@ -32,10 +32,8 @@ class TDTestCase: tdSql.execute(f"create database db {keep_str}") # check result - tdSql.query(" show create database db") - create_sql = tdSql.getData(0, 1) - if create_sql.find(keep_str) == -1: - tdLog.exit(f"create database sql not found {keep_str} sql:{create_sql}") + tdSql.query("select `keep_time_offset` from information_schema.ins_databases where name='db'") + tdSql.checkData(0, 0, hours) # alter hours = 4 @@ -43,10 +41,9 @@ class TDTestCase: tdSql.execute(f"alter database db {keep_str}") # check result - tdSql.query(" show create database db") - create_sql = tdSql.getData(0, 1) - if create_sql.find(keep_str) == -1: - tdLog.exit(f"create database sql not found {keep_str} sql:{create_sql}") + tdSql.query("select `keep_time_offset` from information_schema.ins_databases where name='db'") + tdSql.checkData(0, 0, hours) + def check_old_syntax(self): # old syntax would not support again From 808b226d41fd41e8b4734560971afa6e35d4631e Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 28 Sep 2023 17:05:44 +0800 Subject: [PATCH 15/36] chore: more test --- tests/script/api/passwdTest.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/script/api/passwdTest.c b/tests/script/api/passwdTest.c index eb3c37cfa8..932f258a4b 100644 --- a/tests/script/api/passwdTest.c +++ b/tests/script/api/passwdTest.c @@ -446,9 +446,10 @@ _loop: for (int i = 0; i < nTestUsers; ++i) { sprintf(users[i], "user%d", i); sprintf(qstr, "CREATE USER %s PASS 'taos'", users[i]); + fprintf(stderr, "%s:%d create user:%s\n", __func__, __LINE__, users[i]); queryDB(taos, qstr); - goto _loop; } + goto _loop; // sleep(300); } } \ No newline at end of file From e677e41573710073bb792a1dc495883eb422b9c7 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 30 Sep 2023 12:29:44 +0800 Subject: [PATCH 16/36] fix: support check and merge activeCode --- include/common/tgrant.h | 3 ++ include/util/taoserror.h | 5 +++ source/dnode/mnode/impl/src/mndDnode.c | 57 +++++++++++++++++++++----- source/util/src/terror.c | 4 ++ 4 files changed, 58 insertions(+), 11 deletions(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index 31d34add24..4d33889336 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -51,6 +51,9 @@ typedef enum { } EGrantType; int32_t grantCheck(EGrantType grant); +#ifdef TD_ENTERPRISE +int32_t grantAlterActiveCode(const char* old, const char* new, char* out, int8_t type); +#endif #ifndef GRANTS_CFG #ifdef TD_ENTERPRISE diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 39bf2b5681..46a85661d5 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -541,6 +541,11 @@ int32_t* taosGetErrno(); #define TSDB_CODE_GRANT_CPU_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080B) #define TSDB_CODE_GRANT_STABLE_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080C) #define TSDB_CODE_GRANT_TABLE_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080D) +#define TSDB_CODE_GRANT_PAR_INVALID_ACTIVE TAOS_DEF_ERROR_CODE(0, 0x080E) +#define TSDB_CODE_GRANT_GEN_INVALID_KEY TAOS_DEF_ERROR_CODE(0, 0x080F) +#define TSDB_CODE_GRANT_GEN_APP_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0810) +#define TSDB_CODE_GRANT_GEN_INVALID_KLEN TAOS_DEF_ERROR_CODE(0, 0x0811) + // sync // #define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) // 2.x diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index f2a8c82192..2bd99c1afc 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -763,9 +763,14 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg SSdbRaw *pRaw = NULL; STrans *pTrans = NULL; SDnodeObj *pDnode = NULL; + SArray *failRecord = NULL; bool cfgAll = pCfgReq->dnodeId == -1; int32_t iter = 0; + if (cfgAll && !(failRecord = taosArrayInit(1, sizeof(int32_t)))) { + goto _OVER; + } + SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; while (1) { @@ -774,6 +779,39 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg if (pIter == NULL) break; ++iter; } else if (!(pDnode = mndAcquireDnode(pMnode, pCfgReq->dnodeId))) { + terrno = TSDB_CODE_MND_INVALID_DNODE_ID; + goto _OVER; + } + + SDnodeObj tmpDnode = *pDnode; + if (action == DND_ACTIVE_CODE) { +#ifndef MND_MERGE_ACTIVE_CODE + strncpy(tmpDnode.active, pCfgReq->value, TSDB_ACTIVE_KEY_LEN); +#else + if (grantAlterActiveCode(pDnode->active, pCfgReq->value, tmpDnode.active, 0) != 0) { + if (TSDB_CODE_DUP_KEY != terrno) { + mError("dnode:%d, config dnode, cfg:%d, app:%p config:%s value:%s failed since %s", pDnode->id, + pCfgReq->dnodeId, pReq->info.ahandle, pCfgReq->config, pCfgReq->value, terrstr()); + taosArrayPush(failRecord, pDnode->id); + continue; + } + } +#endif + } else if (action == DND_CONN_ACTIVE_CODE) { +#ifndef MND_MERGE_ACTIVE_CODE + strncpy(tmpDnode.connActive, pCfgReq->value, TSDB_CONN_ACTIVE_KEY_LEN); +#else + if (grantAlterActiveCode(pDnode->connActive, pCfgReq->value, tmpDnode.connActive, 1) != 0) { + if (TSDB_CODE_DUP_KEY != terrno) { + mError("dnode:%d, config dnode, cfg:%d, app:%p config:%s value:%s failed since %s", pDnode->id, + pCfgReq->dnodeId, pReq->info.ahandle, pCfgReq->config, pCfgReq->value, terrstr()); + taosArrayPush(failRecord, pDnode->id); + continue; + } + } +#endif + } else { + terrno = TSDB_CODE_INVALID_CFG; goto _OVER; } @@ -783,16 +821,6 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; } - SDnodeObj tmpDnode = *pDnode; - if (action == DND_ACTIVE_CODE) { - strncpy(tmpDnode.active, pCfgReq->value, TSDB_ACTIVE_KEY_LEN); - } else if (action == DND_CONN_ACTIVE_CODE) { - strncpy(tmpDnode.connActive, pCfgReq->value, TSDB_CONN_ACTIVE_KEY_LEN); - } else { - terrno = TSDB_CODE_INVALID_CFG; - goto _OVER; - } - pRaw = mndDnodeActionEncode(&tmpDnode); if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); @@ -816,12 +844,18 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg _OVER: if (cfgAll) { sdbRelease(pSdb, pDnode); + int32_t nFail = taosArrayGetSize(failRecord); + if (nFail > 0) { + mError("config dnode, cfg:%d, app:%p config:%s value:%s. total:%d, fail:%d", pCfgReq->dnodeId, pReq->info.ahandle, + pCfgReq->config, pCfgReq->value, iter, nFail); + } } else { mndReleaseDnode(pMnode, pDnode); } sdbCancelFetch(pSdb, pIter); mndTransDrop(pTrans); sdbFreeRaw(pRaw); + taosArrayDestroy(failRecord); return terrno; } @@ -1262,7 +1296,8 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%s", cfgReq.value); if (mndConfigDnode(pMnode, pReq, &cfgReq, opt) != 0) { - mError("dnode:%d, failed to config activeCode since %s", cfgReq.dnodeId, terrstr()); + mError("dnode:%d, failed to config activeCode since %s. conf:%s, val:%s", cfgReq.dnodeId, terrstr(), + cfgReq.config, cfgReq.value); return -1; } return 0; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 9832720994..116600f393 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -430,6 +430,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_QUERYTIME_LIMITED, "Query time limited by TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED, "CPU cores limited by license") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STABLE_LIMITED, "STable creation limited by license") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TABLE_LIMITED, "Table creation limited by license") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_PAR_INVALID_ACTIVE, "Invalid active code") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_INVALID_KEY, "Invalid key to gen active code") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_APP_LIMIT, "Gen active code limited by app num") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_INVALID_KLEN, "Invalid klen to gen active code") // sync TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TIMEOUT, "Sync timeout") From 5067eb8cc503a5a263ab1f0889140959c6045700 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 30 Sep 2023 16:00:14 +0800 Subject: [PATCH 17/36] chore: more code for check active code --- include/util/taoserror.h | 11 +++++++---- source/dnode/mnode/impl/src/mndDnode.c | 12 ++++++------ source/util/src/terror.c | 11 +++++++---- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 46a85661d5..56a0cddce5 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -541,10 +541,13 @@ int32_t* taosGetErrno(); #define TSDB_CODE_GRANT_CPU_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080B) #define TSDB_CODE_GRANT_STABLE_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080C) #define TSDB_CODE_GRANT_TABLE_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080D) -#define TSDB_CODE_GRANT_PAR_INVALID_ACTIVE TAOS_DEF_ERROR_CODE(0, 0x080E) -#define TSDB_CODE_GRANT_GEN_INVALID_KEY TAOS_DEF_ERROR_CODE(0, 0x080F) -#define TSDB_CODE_GRANT_GEN_APP_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0810) -#define TSDB_CODE_GRANT_GEN_INVALID_KLEN TAOS_DEF_ERROR_CODE(0, 0x0811) +#define TSDB_CODE_GRANT_PAR_IVLD_ACTIVE TAOS_DEF_ERROR_CODE(0, 0x080E) +#define TSDB_CODE_GRANT_PAR_IVLD_KEY TAOS_DEF_ERROR_CODE(0, 0x080F) +#define TSDB_CODE_GRANT_PAR_DEC_IVLD_KEY TAOS_DEF_ERROR_CODE(0, 0x0810) +#define TSDB_CODE_GRANT_PAR_DEC_IVLD_KLEN TAOS_DEF_ERROR_CODE(0, 0x0811) +#define TSDB_CODE_GRANT_GEN_IVLD_KEY TAOS_DEF_ERROR_CODE(0, 0x0812) +#define TSDB_CODE_GRANT_GEN_APP_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0813) +#define TSDB_CODE_GRANT_GEN_ENC_IVLD_KLEN TAOS_DEF_ERROR_CODE(0, 0x0814) // sync diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 2bd99c1afc..04fbbd8660 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -785,28 +785,28 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg SDnodeObj tmpDnode = *pDnode; if (action == DND_ACTIVE_CODE) { -#ifndef MND_MERGE_ACTIVE_CODE +#ifndef TD_CHECK_ACTIVE strncpy(tmpDnode.active, pCfgReq->value, TSDB_ACTIVE_KEY_LEN); #else if (grantAlterActiveCode(pDnode->active, pCfgReq->value, tmpDnode.active, 0) != 0) { if (TSDB_CODE_DUP_KEY != terrno) { mError("dnode:%d, config dnode, cfg:%d, app:%p config:%s value:%s failed since %s", pDnode->id, pCfgReq->dnodeId, pReq->info.ahandle, pCfgReq->config, pCfgReq->value, terrstr()); - taosArrayPush(failRecord, pDnode->id); - continue; + if (failRecord) taosArrayPush(failRecord, &pDnode->id); + if (!cfgAll) goto _OVER; } } #endif } else if (action == DND_CONN_ACTIVE_CODE) { -#ifndef MND_MERGE_ACTIVE_CODE +#ifndef TD_CHECK_ACTIVE strncpy(tmpDnode.connActive, pCfgReq->value, TSDB_CONN_ACTIVE_KEY_LEN); #else if (grantAlterActiveCode(pDnode->connActive, pCfgReq->value, tmpDnode.connActive, 1) != 0) { if (TSDB_CODE_DUP_KEY != terrno) { mError("dnode:%d, config dnode, cfg:%d, app:%p config:%s value:%s failed since %s", pDnode->id, pCfgReq->dnodeId, pReq->info.ahandle, pCfgReq->config, pCfgReq->value, terrstr()); - taosArrayPush(failRecord, pDnode->id); - continue; + if (failRecord) taosArrayPush(failRecord, &pDnode->id); + if (!cfgAll) goto _OVER; } } #endif diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 116600f393..b4a1a2eae2 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -430,10 +430,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_QUERYTIME_LIMITED, "Query time limited by TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED, "CPU cores limited by license") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STABLE_LIMITED, "STable creation limited by license") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TABLE_LIMITED, "Table creation limited by license") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_PAR_INVALID_ACTIVE, "Invalid active code") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_INVALID_KEY, "Invalid key to gen active code") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_APP_LIMIT, "Gen active code limited by app num") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_INVALID_KLEN, "Invalid klen to gen active code") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_PAR_IVLD_ACTIVE, "Invalid active code") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_PAR_IVLD_KEY, "Invalid key to parse active code") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_PAR_DEC_IVLD_KEY, "Invalid key to decode active code") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_PAR_DEC_IVLD_KLEN, "Invalid klen to decode active code") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_IVLD_KEY, "Invalid key to gen active code") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_APP_LIMIT, "Limited app num to gen active code") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_ENC_IVLD_KLEN, "Invalid klen to encode active code") // sync TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TIMEOUT, "Sync timeout") From a19efd266e59f007b17a451f6dcdee1ca3cc3a04 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 1 Oct 2023 21:55:17 +0800 Subject: [PATCH 18/36] chore: more code --- source/dnode/mnode/impl/src/mndDnode.c | 16 +++++++++++----- source/dnode/mnode/impl/src/mndMain.c | 2 ++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 04fbbd8660..d93fb8ee0f 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -765,6 +765,7 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg SDnodeObj *pDnode = NULL; SArray *failRecord = NULL; bool cfgAll = pCfgReq->dnodeId == -1; + int32_t cfgAllErr = 0; int32_t iter = 0; if (cfgAll && !(failRecord = taosArrayInit(1, sizeof(int32_t)))) { @@ -790,11 +791,13 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg #else if (grantAlterActiveCode(pDnode->active, pCfgReq->value, tmpDnode.active, 0) != 0) { if (TSDB_CODE_DUP_KEY != terrno) { - mError("dnode:%d, config dnode, cfg:%d, app:%p config:%s value:%s failed since %s", pDnode->id, + mError("dnode:%d, config dnode:%d, app:%p config:%s value:%s failed since %s", pDnode->id, pCfgReq->dnodeId, pReq->info.ahandle, pCfgReq->config, pCfgReq->value, terrstr()); if (failRecord) taosArrayPush(failRecord, &pDnode->id); - if (!cfgAll) goto _OVER; + if (cfgAll && (0 == cfgAllErr)) cfgAllErr = terrno; } + if (cfgAll) continue; + goto _OVER; } #endif } else if (action == DND_CONN_ACTIVE_CODE) { @@ -803,11 +806,13 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg #else if (grantAlterActiveCode(pDnode->connActive, pCfgReq->value, tmpDnode.connActive, 1) != 0) { if (TSDB_CODE_DUP_KEY != terrno) { - mError("dnode:%d, config dnode, cfg:%d, app:%p config:%s value:%s failed since %s", pDnode->id, + mError("dnode:%d, config dnode:%d, app:%p config:%s value:%s failed since %s", pDnode->id, pCfgReq->dnodeId, pReq->info.ahandle, pCfgReq->config, pCfgReq->value, terrstr()); if (failRecord) taosArrayPush(failRecord, &pDnode->id); - if (!cfgAll) goto _OVER; + if (cfgAll && (0 == cfgAllErr)) cfgAllErr = terrno; } + if (cfgAll) continue; + goto _OVER; } #endif } else { @@ -826,7 +831,7 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); pRaw = NULL; - mInfo("dnode:%d, config dnode, cfg:%d, app:%p config:%s value:%s", pDnode->id, pCfgReq->dnodeId, pReq->info.ahandle, + mInfo("dnode:%d, config dnode:%d, app:%p config:%s value:%s", pDnode->id, pCfgReq->dnodeId, pReq->info.ahandle, pCfgReq->config, pCfgReq->value); if (cfgAll) { @@ -844,6 +849,7 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg _OVER: if (cfgAll) { sdbRelease(pSdb, pDnode); + if (cfgAllErr != 0) terrno = cfgAllErr; int32_t nFail = taosArrayGetSize(failRecord); if (nFail > 0) { mError("config dnode, cfg:%d, app:%p config:%s value:%s. total:%d, fail:%d", pCfgReq->dnodeId, pReq->info.ahandle, diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 1c87cde78a..3ac42c2120 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -593,7 +593,9 @@ int32_t mndStart(SMnode *pMnode) { mndSetRestored(pMnode, true); } +#ifndef TD_CHECK_ACTIVE grantReset(pMnode, TSDB_GRANT_ALL, 0); +#endif return mndInitTimer(pMnode); } From 53c4c56206165824aa6878ca9c59c626c2d3964c Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 2 Oct 2023 09:23:51 +0800 Subject: [PATCH 19/36] chore: code optimization --- source/dnode/mnode/impl/src/mndDnode.c | 31 +++++++++++++++----------- source/dnode/mnode/impl/src/mndMain.c | 2 -- tests/script/api/passwdTest.c | 20 +++-------------- 3 files changed, 21 insertions(+), 32 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index d93fb8ee0f..ea7ded1879 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -768,10 +768,6 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg int32_t cfgAllErr = 0; int32_t iter = 0; - if (cfgAll && !(failRecord = taosArrayInit(1, sizeof(int32_t)))) { - goto _OVER; - } - SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; while (1) { @@ -780,7 +776,6 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg if (pIter == NULL) break; ++iter; } else if (!(pDnode = mndAcquireDnode(pMnode, pCfgReq->dnodeId))) { - terrno = TSDB_CODE_MND_INVALID_DNODE_ID; goto _OVER; } @@ -791,10 +786,15 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg #else if (grantAlterActiveCode(pDnode->active, pCfgReq->value, tmpDnode.active, 0) != 0) { if (TSDB_CODE_DUP_KEY != terrno) { - mError("dnode:%d, config dnode:%d, app:%p config:%s value:%s failed since %s", pDnode->id, - pCfgReq->dnodeId, pReq->info.ahandle, pCfgReq->config, pCfgReq->value, terrstr()); - if (failRecord) taosArrayPush(failRecord, &pDnode->id); - if (cfgAll && (0 == cfgAllErr)) cfgAllErr = terrno; + mError("dnode:%d, config dnode:%d, app:%p config:%s value:%s failed since %s", pDnode->id, pCfgReq->dnodeId, + pReq->info.ahandle, pCfgReq->config, pCfgReq->value, terrstr()); + if (cfgAll) { // alter all dnodes: + if (!failRecord) failRecord = taosArrayInit(1, sizeof(int32_t)); + if (failRecord) taosArrayPush(failRecord, &pDnode->id); + if (0 == cfgAllErr) cfgAllErr = terrno; // output 1st terrno. + } + } else { + terrno = 0; // no action for dup active code } if (cfgAll) continue; goto _OVER; @@ -806,10 +806,15 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg #else if (grantAlterActiveCode(pDnode->connActive, pCfgReq->value, tmpDnode.connActive, 1) != 0) { if (TSDB_CODE_DUP_KEY != terrno) { - mError("dnode:%d, config dnode:%d, app:%p config:%s value:%s failed since %s", pDnode->id, - pCfgReq->dnodeId, pReq->info.ahandle, pCfgReq->config, pCfgReq->value, terrstr()); - if (failRecord) taosArrayPush(failRecord, &pDnode->id); - if (cfgAll && (0 == cfgAllErr)) cfgAllErr = terrno; + mError("dnode:%d, config dnode:%d, app:%p config:%s value:%s failed since %s", pDnode->id, pCfgReq->dnodeId, + pReq->info.ahandle, pCfgReq->config, pCfgReq->value, terrstr()); + if (cfgAll) { + if (!failRecord) failRecord = taosArrayInit(1, sizeof(int32_t)); + if (failRecord) taosArrayPush(failRecord, &pDnode->id); + if (0 == cfgAllErr) cfgAllErr = terrno; + } + } else { + terrno = 0; } if (cfgAll) continue; goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 3ac42c2120..1c87cde78a 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -593,9 +593,7 @@ int32_t mndStart(SMnode *pMnode) { mndSetRestored(pMnode, true); } -#ifndef TD_CHECK_ACTIVE grantReset(pMnode, TSDB_GRANT_ALL, 0); -#endif return mndInitTimer(pMnode); } diff --git a/tests/script/api/passwdTest.c b/tests/script/api/passwdTest.c index 932f258a4b..24059bd35d 100644 --- a/tests/script/api/passwdTest.c +++ b/tests/script/api/passwdTest.c @@ -207,8 +207,8 @@ int main(int argc, char *argv[]) { exit(1); } createUsers(taos, argv[1], qstr); - // passVerTestMulti(argv[1], qstr); - // sysInfoTest(taos, argv[1], qstr); + passVerTestMulti(argv[1], qstr); + sysInfoTest(taos, argv[1], qstr); userDroppedTest(taos, argv[1], qstr); taos_close(taos); @@ -391,10 +391,6 @@ _REP: void userDroppedTest(TAOS *taos, const char *host, char *qstr) { // users int nTestUsers = nUser; - int nLoop = 0; -_loop: - ++nLoop; - printf("%s:%d loop %d\n", __func__, __LINE__, nLoop); for (int i = 0; i < nTestUsers; ++i) { // sprintf(users[i], "user%d", i); taosu[i] = taos_connect(host, users[i], "taos", NULL, 0); @@ -441,15 +437,5 @@ _loop: exit(1); } fprintf(stderr, "######## %s #########\n", __func__); - - if (nLoop < 5) { - for (int i = 0; i < nTestUsers; ++i) { - sprintf(users[i], "user%d", i); - sprintf(qstr, "CREATE USER %s PASS 'taos'", users[i]); - fprintf(stderr, "%s:%d create user:%s\n", __func__, __LINE__, users[i]); - queryDB(taos, qstr); - } - goto _loop; - // sleep(300); - } + // sleep(300); } \ No newline at end of file From bb797a9a0e5ab58323bb700b5fb795d8eccc8662 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 2 Oct 2023 17:26:55 +0800 Subject: [PATCH 20/36] chore: test case adaption for alter active code --- tests/system-test/0-others/information_schema.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index 77bfb5dca8..c02778cf0a 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -242,11 +242,11 @@ class TDTestCase: self.str510 = self.str255 + self.str255 tdSql.error('alter dnode 1 "activeCode" "a"') tdSql.error('alter dnode 1 "activeCode" "' + self.str107 + '"') - tdSql.execute('alter all dnodes "activeCode" "' + self.str108 + '"') +# tdSql.error('alter all dnodes "activeCode" "' + self.str108 + '"') tdSql.error('alter dnode 1 "activeCode" "' + self.str109 + '"') tdSql.error('alter all dnodes "activeCode" "' + self.str510 + '"') tdSql.query(f'select * from information_schema.ins_dnodes') - tdSql.checkEqual(tdSql.queryResult[0][8],self.str108) + tdSql.checkEqual(tdSql.queryResult[0][8],"") tdSql.execute('alter dnode 1 "activeCode" ""') tdSql.query(f'select active_code,c_active_code from information_schema.ins_dnodes') tdSql.checkEqual(tdSql.queryResult[0][0],"") @@ -257,14 +257,14 @@ class TDTestCase: tdSql.error('alter all dnodes "cActiveCode" "' + self.str255 + '"') tdSql.error('alter all dnodes "cActiveCode" "' + self.str256 + '"') tdSql.error('alter all dnodes "cActiveCode" "' + self.str257 + '"') - tdSql.execute('alter all dnodes "cActiveCode" "' + self.str254 + '"') +# tdSql.error('alter all dnodes "cActiveCode" "' + self.str254 + '"') tdSql.error('alter dnode 1 "cActiveCode" "' + self.str510 + '"') tdSql.query(f'select active_code,c_active_code from information_schema.ins_dnodes') tdSql.checkEqual(tdSql.queryResult[0][0],"") - tdSql.checkEqual(tdSql.queryResult[0][1],self.str254) - tdSql.execute('alter dnode 1 "cActiveCode" "' + self.str109 + '"') + tdSql.checkEqual(tdSql.queryResult[0][1],"") +# tdSql.error('alter dnode 1 "cActiveCode" "' + self.str109 + '"') tdSql.query(f'show dnodes') - tdSql.checkEqual(tdSql.queryResult[0][9],self.str109) + tdSql.checkEqual(tdSql.queryResult[0][9],"") tdSql.execute('alter all dnodes "cActiveCode" ""') tdSql.query(f'select c_active_code from information_schema.ins_dnodes') tdSql.checkEqual(tdSql.queryResult[0][0],'') From 638bfe30e43fc3c2b5b226d1d39cbe9899c2911f Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 2 Oct 2023 17:50:36 +0800 Subject: [PATCH 21/36] chore: more code --- include/util/taoserror.h | 1 - tests/system-test/0-others/information_schema.py | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 56a0cddce5..227e8520e3 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -549,7 +549,6 @@ int32_t* taosGetErrno(); #define TSDB_CODE_GRANT_GEN_APP_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0813) #define TSDB_CODE_GRANT_GEN_ENC_IVLD_KLEN TAOS_DEF_ERROR_CODE(0, 0x0814) - // sync // #define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) // 2.x // #define TSDB_CODE_SYN_NOT_ENABLED TAOS_DEF_ERROR_CODE(0, 0x0901) // 2.x diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index c02778cf0a..7f4b291776 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -242,7 +242,7 @@ class TDTestCase: self.str510 = self.str255 + self.str255 tdSql.error('alter dnode 1 "activeCode" "a"') tdSql.error('alter dnode 1 "activeCode" "' + self.str107 + '"') -# tdSql.error('alter all dnodes "activeCode" "' + self.str108 + '"') + # tdSql.error('alter all dnodes "activeCode" "' + self.str108 + '"') tdSql.error('alter dnode 1 "activeCode" "' + self.str109 + '"') tdSql.error('alter all dnodes "activeCode" "' + self.str510 + '"') tdSql.query(f'select * from information_schema.ins_dnodes') @@ -257,12 +257,12 @@ class TDTestCase: tdSql.error('alter all dnodes "cActiveCode" "' + self.str255 + '"') tdSql.error('alter all dnodes "cActiveCode" "' + self.str256 + '"') tdSql.error('alter all dnodes "cActiveCode" "' + self.str257 + '"') -# tdSql.error('alter all dnodes "cActiveCode" "' + self.str254 + '"') + # tdSql.error('alter all dnodes "cActiveCode" "' + self.str254 + '"') tdSql.error('alter dnode 1 "cActiveCode" "' + self.str510 + '"') tdSql.query(f'select active_code,c_active_code from information_schema.ins_dnodes') tdSql.checkEqual(tdSql.queryResult[0][0],"") tdSql.checkEqual(tdSql.queryResult[0][1],"") -# tdSql.error('alter dnode 1 "cActiveCode" "' + self.str109 + '"') + # tdSql.error('alter dnode 1 "cActiveCode" "' + self.str109 + '"') tdSql.query(f'show dnodes') tdSql.checkEqual(tdSql.queryResult[0][9],"") tdSql.execute('alter all dnodes "cActiveCode" ""') From c339f3552a52338cdd024671e4080a49de656b2a Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 5 Oct 2023 09:19:44 +0800 Subject: [PATCH 22/36] fix: mutex init for trans --- source/dnode/mnode/impl/src/mndTrans.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 5d150b731c..89eda91e8d 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -543,6 +543,8 @@ STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) { STrans *pTrans = sdbAcquire(pMnode->pSdb, SDB_TRANS, &transId); if (pTrans == NULL) { terrno = TSDB_CODE_MND_TRANS_NOT_EXIST; + } else { + taosThreadMutexInit(&pTrans->mutex, NULL); } return pTrans; } From 086213a6c201ae70c3df6f42826716a7a96ec0c0 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 5 Oct 2023 21:35:57 +0800 Subject: [PATCH 23/36] enh: dnode notify logic --- include/libs/monitor/monitor.h | 5 +++ source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 19 ++++++---- source/dnode/vnode/src/meta/metaTable.c | 10 ++++-- source/dnode/vnode/src/vnd/vnodeQuery.c | 39 ++++++++++++++------- 4 files changed, 52 insertions(+), 21 deletions(-) diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 708953f45e..91b3a54ea1 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -206,6 +206,11 @@ typedef struct { bool comp; } SMonCfg; +typedef struct { + int8_t state; + tsem_t sem; +} SDmNotifyHandle; + int32_t monInit(const SMonCfg *pCfg); void monCleanup(); void monRecordLog(int64_t ts, ELogLevel level, const char *content); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index f567267ff8..598f449738 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -53,21 +53,28 @@ static void *dmStatusThreadFp(void *param) { return NULL; } -tsem_t dmNotifySem; -static void *dmNotifyThreadFp(void *param) { +SDmNotifyHandle dmNotifyHdl = {.state = 0}; +static void *dmNotifyThreadFp(void *param) { SDnodeMgmt *pMgmt = param; int64_t lastTime = taosGetTimestampMs(); setThreadName("dnode-notify"); - if (tsem_init(&dmNotifySem, 0, 0) != 0) { + if (tsem_init(&dmNotifyHdl.sem, 0, 0) != 0) { return NULL; } while (1) { if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; - tsem_wait(&dmNotifySem); + _wait: + tsem_wait(&dmNotifyHdl.sem); + _send: + atomic_store_8(&dmNotifyHdl.state, 1); dmSendNotifyReq(pMgmt); + if (1 == atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 0)) { + goto _wait; + } + goto _send; } return NULL; @@ -189,11 +196,11 @@ int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt) { void dmStopNotifyThread(SDnodeMgmt *pMgmt) { if (taosCheckPthreadValid(pMgmt->notifyThread)) { - tsem_post(&dmNotifySem); + tsem_post(&dmNotifyHdl.sem); taosThreadJoin(pMgmt->notifyThread, NULL); taosThreadClear(&pMgmt->notifyThread); } - tsem_destroy(&dmNotifySem); + tsem_destroy(&dmNotifyHdl.sem); } int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 112b7bd5c2..cbbee06e34 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -15,7 +15,7 @@ #include "meta.h" -extern tsem_t dmNotifySem; +extern SDmNotifyHandle dmNotifyHdl; static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); static int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); @@ -198,7 +198,11 @@ static inline void metaTimeSeriesNotifyCheck(SMeta *pMeta) { #if defined(TD_ENTERPRISE) && !defined(_TD_DARWIN_64) int64_t nTimeSeries = metaGetTimeSeriesNum(pMeta, 0); int64_t deltaTS = nTimeSeries - pMeta->pVnode->config.vndStats.numOfReportedTimeSeries; - if (deltaTS > tsTimeSeriesThreshold) tsem_post(&dmNotifySem); + if (deltaTS > tsTimeSeriesThreshold) { + if (1 != atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 2)) { + tsem_post(&dmNotifyHdl.sem); + } + } #endif } @@ -1066,7 +1070,7 @@ static int metaDeleteTtl(SMeta *pMeta, const SMetaEntry *pME) { return ttlMgrDeleteTtl(pMeta->pTtlMgr, &ctx); } -static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, char* stbName) { +static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid) { //}, char* stbName) { void *pData = NULL; int nData = 0; int rc = 0; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index c96428d835..2a4415ab8a 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -566,7 +566,8 @@ int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) { } #ifdef TD_ENTERPRISE -#define TK_LOG_STB_NUM 20 +#define TK_LOG_STB_NUM 19 +#define TK_AUDIT_STB_NUM 1 static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info", "data_dir", "dnodes_info", @@ -585,24 +586,38 @@ static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info", "taosadapter_system_mem_percent", "temp_dir", "vgroups_info", - "vnodes_role", - "operations"}; + "vnodes_role"}; +static const char *tkAuditStb[TK_AUDIT_STB_NUM] = {"operations"}; // exclude stbs of taoskeeper log static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { char *dbName = strchr(pVnode->config.dbname, '.'); - if (!dbName || 0 != strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { + if (!dbName) { return 0; } - int32_t tbSize = metaSizeOfTbFilterCache(pVnode, 0); - if (tbSize < TK_LOG_STB_NUM) { - for (int32_t i = 0; i < TK_LOG_STB_NUM; ++i) { - tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, tkLogStb[i]); - if (suid != 0) { - metaPutTbToFilterCache(pVnode, &suid, 0); - } - } + int32_t tbSize = 0; + if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { tbSize = metaSizeOfTbFilterCache(pVnode, 0); + if (tbSize < TK_LOG_STB_NUM) { + for (int32_t i = 0; i < TK_LOG_STB_NUM; ++i) { + tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, tkLogStb[i]); + if (suid != 0) { + metaPutTbToFilterCache(pVnode, &suid, 0); + } + } + tbSize = metaSizeOfTbFilterCache(pVnode, 0); + } + } else if (0 == strncmp(dbName, "audit", TSDB_DB_NAME_LEN)) { + tbSize = metaSizeOfTbFilterCache(pVnode, 0); + if (tbSize < TK_AUDIT_STB_NUM) { + for (int32_t i = 0; i < TK_AUDIT_STB_NUM; ++i) { + tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, tkAuditStb[i]); + if (suid != 0) { + metaPutTbToFilterCache(pVnode, &suid, 0); + } + } + tbSize = metaSizeOfTbFilterCache(pVnode, 0); + } } return tbSize; From 88fdeca5cf0ddb129fb0a095307e3aa4a24b76a9 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 5 Oct 2023 22:31:25 +0800 Subject: [PATCH 24/36] enh: dnode notify logic --- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 12 +++++------- source/dnode/vnode/src/meta/metaTable.c | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 598f449738..18da1d638c 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -56,25 +56,23 @@ static void *dmStatusThreadFp(void *param) { SDmNotifyHandle dmNotifyHdl = {.state = 0}; static void *dmNotifyThreadFp(void *param) { SDnodeMgmt *pMgmt = param; - int64_t lastTime = taosGetTimestampMs(); setThreadName("dnode-notify"); if (tsem_init(&dmNotifyHdl.sem, 0, 0) != 0) { return NULL; } + bool wait = true; while (1) { if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; - - _wait: - tsem_wait(&dmNotifyHdl.sem); - _send: + if (wait) tsem_wait(&dmNotifyHdl.sem); atomic_store_8(&dmNotifyHdl.state, 1); dmSendNotifyReq(pMgmt); if (1 == atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 0)) { - goto _wait; + wait = true; + continue; } - goto _send; + wait = false; } return NULL; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index cbbee06e34..16bffce784 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -199,7 +199,7 @@ static inline void metaTimeSeriesNotifyCheck(SMeta *pMeta) { int64_t nTimeSeries = metaGetTimeSeriesNum(pMeta, 0); int64_t deltaTS = nTimeSeries - pMeta->pVnode->config.vndStats.numOfReportedTimeSeries; if (deltaTS > tsTimeSeriesThreshold) { - if (1 != atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 2)) { + if (0 == atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 2)) { tsem_post(&dmNotifyHdl.sem); } } From d0a046c7a354cc9985173cb6c7570386b886c996 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 5 Oct 2023 23:30:30 +0800 Subject: [PATCH 25/36] enh: exclude sys tbl for time series --- source/dnode/vnode/inc/vnode.h | 1 + source/dnode/vnode/src/meta/metaCache.c | 6 +++++- source/dnode/vnode/src/meta/metaTable.c | 19 ++++++++++-------- source/dnode/vnode/src/vnd/vnodeQuery.c | 26 ++++++++++++++++++++++++- 4 files changed, 42 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index d6491c8fc7..1729ebb439 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -136,6 +136,7 @@ int32_t metaPutTbGroupToCache(void *pVnode, uint64_t suid, const void *pKey, in bool metaTbInFilterCache(void *pVnode, void* key, int8_t type); int32_t metaPutTbToFilterCache(void *pVnode, void* key, int8_t type); int32_t metaSizeOfTbFilterCache(void *pVnode, int8_t type); +int32_t metaInitTbFilterCache(void *pVnode); int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables, int32_t *numOfCols); diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 0a8c1c410a..a4a893c42d 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -179,11 +179,15 @@ int32_t metaCacheOpen(SMeta* pMeta) { goto _err2; } - pCache->STbFilterCache.pStbName = taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); + pCache->STbFilterCache.pStbName = + taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); if (pCache->STbFilterCache.pStbName == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err2; } + if ((code = metaInitTbFilterCache(pMeta->pVnode)) != 0) { + goto _err2; + } pMeta->pCache = pCache; return code; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 16bffce784..e0727233a9 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -28,7 +28,7 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry); -static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid); +static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t *pSysTbl); static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey); // opt ins_tables query static int metaUpdateBtimeIdx(SMeta *pMeta, const SMetaEntry *pME); @@ -307,7 +307,7 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq, SArray *tb for (int32_t iChild = 0; iChild < taosArrayGetSize(tbUidList); iChild++) { tb_uid_t uid = *(tb_uid_t *)taosArrayGet(tbUidList, iChild); - metaDropTableByUid(pMeta, uid, NULL, NULL); + metaDropTableByUid(pMeta, uid, NULL, NULL, NULL); } // drop super table @@ -863,6 +863,7 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi int rc = 0; tb_uid_t uid = 0; tb_uid_t suid = 0; + int8_t sysTbl = 0; int type; rc = tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pData, &nData); @@ -873,12 +874,12 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi uid = *(tb_uid_t *)pData; metaWLock(pMeta); - rc = metaDropTableByUid(pMeta, uid, &type, &suid); + rc = metaDropTableByUid(pMeta, uid, &type, &suid, &sysTbl); metaULock(pMeta); if (rc < 0) goto _exit; - if (type == TSDB_CHILD_TABLE) { + if (type == TSDB_CHILD_TABLE && !sysTbl) { int32_t nCols = 0; SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; if (metaGetStbStats(pMeta->pVnode, suid, NULL, &nCols) == 0) { @@ -909,9 +910,10 @@ void metaDropTables(SMeta *pMeta, SArray *tbUids) { for (int i = 0; i < taosArrayGetSize(tbUids); ++i) { tb_uid_t uid = *(tb_uid_t *)taosArrayGet(tbUids, i); tb_uid_t suid = 0; + int8_t sysTbl = 0; int type; - metaDropTableByUid(pMeta, uid, &type, &suid); - if (type == TSDB_CHILD_TABLE && suid != 0 && suidHash) { + metaDropTableByUid(pMeta, uid, &type, &suid, &sysTbl); + if (!sysTbl && type == TSDB_CHILD_TABLE && suid != 0 && suidHash) { int64_t *pVal = tSimpleHashGet(suidHash, &suid, sizeof(tb_uid_t)); if (pVal) { nCtbDropped = *pVal + 1; @@ -1070,7 +1072,7 @@ static int metaDeleteTtl(SMeta *pMeta, const SMetaEntry *pME) { return ttlMgrDeleteTtl(pMeta->pTtlMgr, &ctx); } -static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid) { //}, char* stbName) { +static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t* pSysTbl) { void *pData = NULL; int nData = 0; int rc = 0; @@ -1099,7 +1101,6 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *p void *tData = NULL; int tLen = 0; - if (tdbTbGet(pMeta->pUidIdx, &e.ctbEntry.suid, sizeof(tb_uid_t), &tData, &tLen) == 0) { STbDbKey tbDbKey = {.uid = e.ctbEntry.suid, .version = ((SUidIdxVal *)tData)[0].version}; if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &tData, &tLen) == 0) { @@ -1109,6 +1110,8 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *p tDecoderInit(&tdc, tData, tLen); metaDecodeEntry(&tdc, &stbEntry); + if (pSysTbl) *pSysTbl = metaTbInFilterCache(pMeta->pVnode, stbEntry.name, 1) ? 1 : 0; + SSchema *pTagColumn = NULL; SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 2a4415ab8a..e9307e6fb5 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -596,7 +596,8 @@ static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { return 0; } int32_t tbSize = 0; - if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { + ++dbName; + if (0 == strncmp(dbName, "log", TSDB_DB_NAME_LEN)) { tbSize = metaSizeOfTbFilterCache(pVnode, 0); if (tbSize < TK_LOG_STB_NUM) { for (int32_t i = 0; i < TK_LOG_STB_NUM; ++i) { @@ -630,8 +631,31 @@ bool vnodeSkipTimeSeries(SVnode *pVnode, const char *stbName) { } } + +int32_t metaInitTbFilterCache(void *pVnode) { + char *dbName = strchr(pVnode->config.dbname, '.'); + if (!dbName) return 0; + ++dbName; + if (0 == strncmp(dbName, "log", TSDB_DB_NAME_LEN)) { + for (int32_t i = 0; i < TK_LOG_STB_NUM; ++i) { + if (metaPutTbToFilterCache(pVnode, &tkLogStb[i], strlen(tkLogStb[i])) != 0) { + return terrno ? terrno : -1; + } + } + } else if (0 == strncmp(dbName, "audit", TSDB_DB_NAME_LEN)) { + for (int32_t i = 0; i < TK_AUDIT_STB_NUM; ++i) { + if (metaPutTbToFilterCache(pVnode, &tkLogStb[i], strlen(tkAuditStb[i])) != 0) { + return terrno ? terrno : -1; + } + } + } + + return 0; +} #endif +int32_t metaInitTbFilterCache(void *pVnode) { return 0; } + static bool vnodeTimeSeriesFilter(void *arg1, void *arg2) { SVnode *pVnode = (SVnode *)arg1; From 47e79ff5b7f30f1fb74e421c14e538c4ab06e81a Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 5 Oct 2023 23:45:54 +0800 Subject: [PATCH 26/36] enh: exclude sys tbl for time series --- source/dnode/vnode/src/meta/metaTable.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index e0727233a9..0ceeaffa05 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -396,10 +396,6 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { nStbEntry.stbEntry.schemaTag = pReq->schemaTag; int32_t deltaCol = pReq->schemaRow.nCols - oStbEntry.stbEntry.schemaRow.nCols; - bool skipStatis = false; - if (deltaCol != 0) { - - } metaWLock(pMeta); // compare two entry @@ -415,7 +411,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { // metaStatsCacheDrop(pMeta, nStbEntry.uid); - if (deltaCol != 0) { + if (deltaCol != 0 && !metaTbInFilterCache(pMeta->pVnode, pReq->name, 1)) { metaUpdateStbStats(pMeta, pReq->suid, 0, deltaCol); } metaULock(pMeta); @@ -805,7 +801,7 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs ++pStats->numOfCTables; - if (metaTbInFilterCache(pMeta->pVnode, pReq->ctb.stbName, 1)) { + if (!metaTbInFilterCache(pMeta->pVnode, pReq->ctb.stbName, 1)) { int32_t nCols = 0; metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols); pStats->numOfTimeSeries += nCols - 1; @@ -879,7 +875,7 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi if (rc < 0) goto _exit; - if (type == TSDB_CHILD_TABLE && !sysTbl) { + if (!sysTbl && type == TSDB_CHILD_TABLE) { int32_t nCols = 0; SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; if (metaGetStbStats(pMeta->pVnode, suid, NULL, &nCols) == 0) { From 07f043f143fa3792a8885e5aaf51389d18245126 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 5 Oct 2023 23:48:52 +0800 Subject: [PATCH 27/36] enh: exclude sys tbl for time series --- source/dnode/vnode/src/vnd/vnodeQuery.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index e9307e6fb5..e7436aa4dd 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -652,9 +652,9 @@ int32_t metaInitTbFilterCache(void *pVnode) { return 0; } -#endif - +#else int32_t metaInitTbFilterCache(void *pVnode) { return 0; } +#endif static bool vnodeTimeSeriesFilter(void *arg1, void *arg2) { SVnode *pVnode = (SVnode *)arg1; @@ -674,9 +674,9 @@ int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) { } int32_t tbFilterSize = 0; - #ifdef TD_ENTERPRISE +#ifdef TD_ENTERPRISE tbFilterSize = vnodeGetTimeSeriesBlackList(pVnode); - #endif +#endif if ((!tbFilterSize && vnodeGetStbIdList(pVnode, 0, suidList) < 0) || (tbFilterSize && vnodeGetStbIdListByFilter(pVnode, 0, suidList, vnodeTimeSeriesFilter, pVnode) < 0)) { From e4a86aa70aafe9637f302544034cac4ae4948b36 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 5 Oct 2023 23:52:50 +0800 Subject: [PATCH 28/36] chore: remove obsolete codes --- source/dnode/vnode/inc/vnode.h | 1 - source/dnode/vnode/src/vnd/vnodeQuery.c | 8 -------- 2 files changed, 9 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 1729ebb439..ce57ef3bbd 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -89,7 +89,6 @@ int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num); int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num); int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num); int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num); -bool vnodeSkipTimeSeries(SVnode *pVnode, const char *stbName); void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad); int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index e7436aa4dd..85c4897914 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -624,14 +624,6 @@ static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { return tbSize; } -bool vnodeSkipTimeSeries(SVnode *pVnode, const char *stbName) { - char *dbName = strchr(pVnode->config.dbname, '.'); - if (!dbName || 0 != strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { - return 0; - } - -} - int32_t metaInitTbFilterCache(void *pVnode) { char *dbName = strchr(pVnode->config.dbname, '.'); if (!dbName) return 0; From c59498d90a17a83a90a91707f3fd7d4b6b760bca Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 6 Oct 2023 07:37:18 +0800 Subject: [PATCH 29/36] chore: code refactor for time series statis --- source/dnode/vnode/src/vnd/vnodeQuery.c | 61 ++++++++++++------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 85c4897914..3a8ab824a1 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -591,28 +591,24 @@ static const char *tkAuditStb[TK_AUDIT_STB_NUM] = {"operations"}; // exclude stbs of taoskeeper log static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { - char *dbName = strchr(pVnode->config.dbname, '.'); - if (!dbName) { - return 0; - } - int32_t tbSize = 0; - ++dbName; - if (0 == strncmp(dbName, "log", TSDB_DB_NAME_LEN)) { - tbSize = metaSizeOfTbFilterCache(pVnode, 0); - if (tbSize < TK_LOG_STB_NUM) { - for (int32_t i = 0; i < TK_LOG_STB_NUM; ++i) { - tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, tkLogStb[i]); - if (suid != 0) { - metaPutTbToFilterCache(pVnode, &suid, 0); - } - } - tbSize = metaSizeOfTbFilterCache(pVnode, 0); - } + int32_t tbSize = 0; + int32_t tbNum = 0; + const char **pTbArr = NULL; + const char *dbName = NULL; + + if (!(dbName = strchr(((SVnode *)pVnode)->config.dbname, '.'))) return 0; + if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { + tbNum = TK_LOG_STB_NUM; + pTbArr = (const char **)&tkLogStb; } else if (0 == strncmp(dbName, "audit", TSDB_DB_NAME_LEN)) { + tbNum = TK_AUDIT_STB_NUM; + pTbArr = (const char **)&tkAuditStb; + } + if (tbNum && pTbArr) { tbSize = metaSizeOfTbFilterCache(pVnode, 0); - if (tbSize < TK_AUDIT_STB_NUM) { - for (int32_t i = 0; i < TK_AUDIT_STB_NUM; ++i) { - tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, tkAuditStb[i]); + if (tbSize < tbNum) { + for (int32_t i = 0; i < tbNum; ++i) { + tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, pTbArr[i]); if (suid != 0) { metaPutTbToFilterCache(pVnode, &suid, 0); } @@ -625,18 +621,21 @@ static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { } int32_t metaInitTbFilterCache(void *pVnode) { - char *dbName = strchr(pVnode->config.dbname, '.'); - if (!dbName) return 0; - ++dbName; - if (0 == strncmp(dbName, "log", TSDB_DB_NAME_LEN)) { - for (int32_t i = 0; i < TK_LOG_STB_NUM; ++i) { - if (metaPutTbToFilterCache(pVnode, &tkLogStb[i], strlen(tkLogStb[i])) != 0) { - return terrno ? terrno : -1; - } - } + int32_t tbNum = 0; + const char **pTbArr = NULL; + const char *dbName = NULL; + + if (!(dbName = strchr(((SVnode *)pVnode)->config.dbname, '.'))) return 0; + if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { + tbNum = TK_LOG_STB_NUM; + pTbArr = (const char **)&tkLogStb; } else if (0 == strncmp(dbName, "audit", TSDB_DB_NAME_LEN)) { - for (int32_t i = 0; i < TK_AUDIT_STB_NUM; ++i) { - if (metaPutTbToFilterCache(pVnode, &tkLogStb[i], strlen(tkAuditStb[i])) != 0) { + tbNum = TK_AUDIT_STB_NUM; + pTbArr = (const char **)&tkAuditStb; + } + if (tbNum && pTbArr) { + for (int32_t i = 0; i < tbNum; ++i) { + if (metaPutTbToFilterCache(pVnode, &pTbArr[i], strlen(pTbArr[i])) != 0) { return terrno ? terrno : -1; } } From c70a1b77e68c300a6969af647103481c18649475 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 7 Oct 2023 00:17:21 +0800 Subject: [PATCH 30/36] chore: code optimization for time series --- source/dnode/vnode/inc/vnode.h | 4 +-- source/dnode/vnode/src/inc/vnodeInt.h | 3 ++ source/dnode/vnode/src/meta/metaCache.c | 40 +++++++++++++++++++++---- source/dnode/vnode/src/meta/metaTable.c | 8 +++-- source/dnode/vnode/src/vnd/vnodeOpen.c | 4 +++ source/dnode/vnode/src/vnd/vnodeQuery.c | 32 ++------------------ 6 files changed, 50 insertions(+), 41 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index ce57ef3bbd..6ee6985e0c 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -132,8 +132,8 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); int32_t metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList); int32_t metaPutTbGroupToCache(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, int32_t payloadLen); -bool metaTbInFilterCache(void *pVnode, void* key, int8_t type); -int32_t metaPutTbToFilterCache(void *pVnode, void* key, int8_t type); +bool metaTbInFilterCache(void *pVnode, const void* key, int8_t type); +int32_t metaPutTbToFilterCache(void *pVnode, const void* key, int8_t type); int32_t metaSizeOfTbFilterCache(void *pVnode, int8_t type); int32_t metaInitTbFilterCache(void *pVnode); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 2fadccdf2d..9929b66932 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -98,6 +98,9 @@ typedef struct SQueryNode SQueryNode; #define VND_INFO_FNAME "vnode.json" #define VND_INFO_FNAME_TMP "vnode_tmp.json" +#define TK_LOG_STB_NUM 19 +#define TK_AUDIT_STB_NUM 1 + // vnd.h typedef int32_t (*_query_reseek_func_t)(void* pQHandle); struct SQueryNode { diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index a4a893c42d..1411285b42 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -14,6 +14,11 @@ */ #include "meta.h" +#ifdef TD_ENTERPRISE +extern const char* tkLogStb[]; +extern const char* tkAuditStb[]; +#endif + #define TAG_FILTER_RES_KEY_LEN 32 #define META_CACHE_BASE_BUCKET 1024 #define META_CACHE_STATS_BUCKET 16 @@ -185,9 +190,6 @@ int32_t metaCacheOpen(SMeta* pMeta) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err2; } - if ((code = metaInitTbFilterCache(pMeta->pVnode)) != 0) { - goto _err2; - } pMeta->pCache = pCache; return code; @@ -905,7 +907,7 @@ int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid) { return TSDB_CODE_SUCCESS; } -bool metaTbInFilterCache(void* pVnode, void* key, int8_t type) { +bool metaTbInFilterCache(void* pVnode, const void* key, int8_t type) { SMeta* pMeta = ((SVnode*)pVnode)->pMeta; if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pStb, key, sizeof(tb_uid_t))) { @@ -919,7 +921,7 @@ bool metaTbInFilterCache(void* pVnode, void* key, int8_t type) { return false; } -int32_t metaPutTbToFilterCache(void* pVnode, void* key, int8_t type) { +int32_t metaPutTbToFilterCache(void* pVnode, const void* key, int8_t type) { SMeta* pMeta = ((SVnode*)pVnode)->pMeta; if (type == 0) { @@ -939,4 +941,30 @@ int32_t metaSizeOfTbFilterCache(void* pVnode, int8_t type) { return taosHashGetSize(pMeta->pCache->STbFilterCache.pStb); } return 0; -} \ No newline at end of file +} + +int32_t metaInitTbFilterCache(void* pVnode) { +#ifdef TD_ENTERPRISE + int32_t tbNum = 0; + const char** pTbArr = NULL; + const char* dbName = NULL; + + if (!(dbName = strchr(((SVnode*)pVnode)->config.dbname, '.'))) return 0; + if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { + tbNum = TK_LOG_STB_NUM; + pTbArr = (const char**)&tkLogStb; + } else if (0 == strncmp(dbName, "audit", TSDB_DB_NAME_LEN)) { + tbNum = TK_AUDIT_STB_NUM; + pTbArr = (const char**)&tkAuditStb; + } + if (tbNum && pTbArr) { + for (int32_t i = 0; i < tbNum; ++i) { + if (metaPutTbToFilterCache(pVnode, pTbArr[i], 1) != 0) { + return terrno ? terrno : -1; + } + } + } +#else +#endif + return 0; +} diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 0ceeaffa05..d795afde74 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -764,6 +764,8 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs } metaReaderClear(&mr); + bool sysTbl = (pReq->type == TSDB_CHILD_TABLE) && metaTbInFilterCache(pMeta->pVnode, pReq->ctb.stbName, 1); + // build SMetaEntry SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; me.version = ver; @@ -800,8 +802,8 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs #endif ++pStats->numOfCTables; - - if (!metaTbInFilterCache(pMeta->pVnode, pReq->ctb.stbName, 1)) { + + if (!sysTbl) { int32_t nCols = 0; metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols); pStats->numOfTimeSeries += nCols - 1; @@ -826,7 +828,7 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs if (metaHandleEntry(pMeta, &me) < 0) goto _err; - metaTimeSeriesNotifyCheck(pMeta); + if(!sysTbl) metaTimeSeriesNotifyCheck(pMeta); if (pMetaRsp) { *pMetaRsp = taosMemoryCalloc(1, sizeof(STableMetaRsp)); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index db94f32459..3797941e96 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -399,6 +399,10 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC goto _err; } + if (metaInitTbFilterCache(pVnode) != 0) { + goto _err; + } + if (metaUpgrade(pVnode, &pVnode->pMeta) < 0) { vError("vgId:%d, failed to upgrade meta since %s", TD_VID(pVnode), tstrerror(terrno)); } diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 3a8ab824a1..d13d49fccc 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -566,9 +566,7 @@ int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) { } #ifdef TD_ENTERPRISE -#define TK_LOG_STB_NUM 19 -#define TK_AUDIT_STB_NUM 1 -static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info", +const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info", "data_dir", "dnodes_info", "d_info", @@ -587,7 +585,7 @@ static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info", "temp_dir", "vgroups_info", "vnodes_role"}; -static const char *tkAuditStb[TK_AUDIT_STB_NUM] = {"operations"}; +const char *tkAuditStb[TK_AUDIT_STB_NUM] = {"operations"}; // exclude stbs of taoskeeper log static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { @@ -619,32 +617,6 @@ static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { return tbSize; } - -int32_t metaInitTbFilterCache(void *pVnode) { - int32_t tbNum = 0; - const char **pTbArr = NULL; - const char *dbName = NULL; - - if (!(dbName = strchr(((SVnode *)pVnode)->config.dbname, '.'))) return 0; - if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { - tbNum = TK_LOG_STB_NUM; - pTbArr = (const char **)&tkLogStb; - } else if (0 == strncmp(dbName, "audit", TSDB_DB_NAME_LEN)) { - tbNum = TK_AUDIT_STB_NUM; - pTbArr = (const char **)&tkAuditStb; - } - if (tbNum && pTbArr) { - for (int32_t i = 0; i < tbNum; ++i) { - if (metaPutTbToFilterCache(pVnode, &pTbArr[i], strlen(pTbArr[i])) != 0) { - return terrno ? terrno : -1; - } - } - } - - return 0; -} -#else -int32_t metaInitTbFilterCache(void *pVnode) { return 0; } #endif static bool vnodeTimeSeriesFilter(void *arg1, void *arg2) { From 6efe3845d4d77288c7ff840c751852d2daa98ba7 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 7 Oct 2023 07:02:16 +0800 Subject: [PATCH 31/36] chore: more logic --- source/dnode/vnode/src/meta/metaTable.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index d795afde74..9577c533cd 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -396,6 +396,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { nStbEntry.stbEntry.schemaTag = pReq->schemaTag; int32_t deltaCol = pReq->schemaRow.nCols - oStbEntry.stbEntry.schemaRow.nCols; + bool updStat = deltaCol != 0 && !metaTbInFilterCache(pMeta->pVnode, pReq->name, 1); metaWLock(pMeta); // compare two entry @@ -411,15 +412,16 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { // metaStatsCacheDrop(pMeta, nStbEntry.uid); - if (deltaCol != 0 && !metaTbInFilterCache(pMeta->pVnode, pReq->name, 1)) { + if (updStat) { metaUpdateStbStats(pMeta, pReq->suid, 0, deltaCol); } metaULock(pMeta); - if (deltaCol != 0) { + if (updStat) { int64_t ctbNum; metaGetStbStats(pMeta->pVnode, pReq->suid, &ctbNum, NULL); pMeta->pVnode->config.vndStats.numOfTimeSeries += (ctbNum * deltaCol); + metaTimeSeriesNotifyCheck(pMeta); } _exit: @@ -828,7 +830,7 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs if (metaHandleEntry(pMeta, &me) < 0) goto _err; - if(!sysTbl) metaTimeSeriesNotifyCheck(pMeta); + metaTimeSeriesNotifyCheck(pMeta); if (pMetaRsp) { *pMetaRsp = taosMemoryCalloc(1, sizeof(STableMetaRsp)); From 85226d05a374c72a3ed09181cd5f89fdbbf73a98 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 7 Oct 2023 09:30:32 +0800 Subject: [PATCH 32/36] chore: code refactor for time series --- source/dnode/vnode/inc/vnode.h | 8 ++++---- source/dnode/vnode/src/meta/metaCache.c | 17 ++++++----------- source/dnode/vnode/src/meta/metaOpen.c | 4 ++++ source/dnode/vnode/src/meta/metaTable.c | 6 +++--- source/dnode/vnode/src/vnd/vnodeOpen.c | 4 ---- source/dnode/vnode/src/vnd/vnodeQuery.c | 10 +++++----- 6 files changed, 22 insertions(+), 27 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 6ee6985e0c..76ed0f4ed0 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -132,10 +132,10 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); int32_t metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList); int32_t metaPutTbGroupToCache(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, int32_t payloadLen); -bool metaTbInFilterCache(void *pVnode, const void* key, int8_t type); -int32_t metaPutTbToFilterCache(void *pVnode, const void* key, int8_t type); -int32_t metaSizeOfTbFilterCache(void *pVnode, int8_t type); -int32_t metaInitTbFilterCache(void *pVnode); +bool metaTbInFilterCache(SMeta *pMeta, const void* key, int8_t type); +int32_t metaPutTbToFilterCache(SMeta *pMeta, const void* key, int8_t type); +int32_t metaSizeOfTbFilterCache(SMeta *pMeta, int8_t type); +int32_t metaInitTbFilterCache(SMeta *pMeta); int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables, int32_t *numOfCols); diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 1411285b42..999ef4ee64 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -907,9 +907,7 @@ int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid) { return TSDB_CODE_SUCCESS; } -bool metaTbInFilterCache(void* pVnode, const void* key, int8_t type) { - SMeta* pMeta = ((SVnode*)pVnode)->pMeta; - +bool metaTbInFilterCache(SMeta *pMeta, const void* key, int8_t type) { if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pStb, key, sizeof(tb_uid_t))) { return true; } @@ -921,9 +919,7 @@ bool metaTbInFilterCache(void* pVnode, const void* key, int8_t type) { return false; } -int32_t metaPutTbToFilterCache(void* pVnode, const void* key, int8_t type) { - SMeta* pMeta = ((SVnode*)pVnode)->pMeta; - +int32_t metaPutTbToFilterCache(SMeta *pMeta, const void* key, int8_t type) { if (type == 0) { return taosHashPut(pMeta->pCache->STbFilterCache.pStb, key, sizeof(tb_uid_t), NULL, 0); } @@ -935,21 +931,20 @@ int32_t metaPutTbToFilterCache(void* pVnode, const void* key, int8_t type) { return 0; } -int32_t metaSizeOfTbFilterCache(void* pVnode, int8_t type) { - SMeta* pMeta = ((SVnode*)pVnode)->pMeta; +int32_t metaSizeOfTbFilterCache(SMeta *pMeta, int8_t type) { if (type == 0) { return taosHashGetSize(pMeta->pCache->STbFilterCache.pStb); } return 0; } -int32_t metaInitTbFilterCache(void* pVnode) { +int32_t metaInitTbFilterCache(SMeta* pMeta) { #ifdef TD_ENTERPRISE int32_t tbNum = 0; const char** pTbArr = NULL; const char* dbName = NULL; - if (!(dbName = strchr(((SVnode*)pVnode)->config.dbname, '.'))) return 0; + if (!(dbName = strchr(pMeta->pVnode->config.dbname, '.'))) return 0; if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { tbNum = TK_LOG_STB_NUM; pTbArr = (const char**)&tkLogStb; @@ -959,7 +954,7 @@ int32_t metaInitTbFilterCache(void* pVnode) { } if (tbNum && pTbArr) { for (int32_t i = 0; i < tbNum; ++i) { - if (metaPutTbToFilterCache(pVnode, pTbArr[i], 1) != 0) { + if (metaPutTbToFilterCache(pMeta, pTbArr[i], 1) != 0) { return terrno ? terrno : -1; } } diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 3d445acd67..8cab17c417 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -176,6 +176,10 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { goto _err; } + if (metaInitTbFilterCache(pMeta) != 0) { + goto _err; + } + metaDebug("vgId:%d, meta is opened", TD_VID(pVnode)); *ppMeta = pMeta; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 9577c533cd..863cd8006a 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -396,7 +396,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { nStbEntry.stbEntry.schemaTag = pReq->schemaTag; int32_t deltaCol = pReq->schemaRow.nCols - oStbEntry.stbEntry.schemaRow.nCols; - bool updStat = deltaCol != 0 && !metaTbInFilterCache(pMeta->pVnode, pReq->name, 1); + bool updStat = deltaCol != 0 && !metaTbInFilterCache(pMeta, pReq->name, 1); metaWLock(pMeta); // compare two entry @@ -766,7 +766,7 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs } metaReaderClear(&mr); - bool sysTbl = (pReq->type == TSDB_CHILD_TABLE) && metaTbInFilterCache(pMeta->pVnode, pReq->ctb.stbName, 1); + bool sysTbl = (pReq->type == TSDB_CHILD_TABLE) && metaTbInFilterCache(pMeta, pReq->ctb.stbName, 1); // build SMetaEntry SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; @@ -1110,7 +1110,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *p tDecoderInit(&tdc, tData, tLen); metaDecodeEntry(&tdc, &stbEntry); - if (pSysTbl) *pSysTbl = metaTbInFilterCache(pMeta->pVnode, stbEntry.name, 1) ? 1 : 0; + if (pSysTbl) *pSysTbl = metaTbInFilterCache(pMeta, stbEntry.name, 1) ? 1 : 0; SSchema *pTagColumn = NULL; SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 3797941e96..db94f32459 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -399,10 +399,6 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC goto _err; } - if (metaInitTbFilterCache(pVnode) != 0) { - goto _err; - } - if (metaUpgrade(pVnode, &pVnode->pMeta) < 0) { vError("vgId:%d, failed to upgrade meta since %s", TD_VID(pVnode), tstrerror(terrno)); } diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index d13d49fccc..3ec85ac93a 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -594,7 +594,7 @@ static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { const char **pTbArr = NULL; const char *dbName = NULL; - if (!(dbName = strchr(((SVnode *)pVnode)->config.dbname, '.'))) return 0; + if (!(dbName = strchr(pVnode->config.dbname, '.'))) return 0; if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { tbNum = TK_LOG_STB_NUM; pTbArr = (const char **)&tkLogStb; @@ -603,15 +603,15 @@ static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { pTbArr = (const char **)&tkAuditStb; } if (tbNum && pTbArr) { - tbSize = metaSizeOfTbFilterCache(pVnode, 0); + tbSize = metaSizeOfTbFilterCache(pVnode->pMeta, 0); if (tbSize < tbNum) { for (int32_t i = 0; i < tbNum; ++i) { tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, pTbArr[i]); if (suid != 0) { - metaPutTbToFilterCache(pVnode, &suid, 0); + metaPutTbToFilterCache(pVnode->pMeta, &suid, 0); } } - tbSize = metaSizeOfTbFilterCache(pVnode, 0); + tbSize = metaSizeOfTbFilterCache(pVnode->pMeta, 0); } } @@ -622,7 +622,7 @@ static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { static bool vnodeTimeSeriesFilter(void *arg1, void *arg2) { SVnode *pVnode = (SVnode *)arg1; - if (metaTbInFilterCache(pVnode, arg2, 0)) { + if (metaTbInFilterCache(pVnode->pMeta, arg2, 0)) { return true; } return false; From b9767707cb31a4ce29f22ed85586669db0fd8776 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 7 Oct 2023 15:57:26 +0800 Subject: [PATCH 33/36] chore: code optimization for array size --- source/dnode/vnode/src/inc/vnodeInt.h | 3 -- source/dnode/vnode/src/meta/metaCache.c | 6 ++-- source/dnode/vnode/src/vnd/vnodeQuery.c | 46 +++++++++++++------------ 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 9929b66932..2fadccdf2d 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -98,9 +98,6 @@ typedef struct SQueryNode SQueryNode; #define VND_INFO_FNAME "vnode.json" #define VND_INFO_FNAME_TMP "vnode_tmp.json" -#define TK_LOG_STB_NUM 19 -#define TK_AUDIT_STB_NUM 1 - // vnd.h typedef int32_t (*_query_reseek_func_t)(void* pQHandle); struct SQueryNode { diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 999ef4ee64..ceb72aa14d 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -17,6 +17,8 @@ #ifdef TD_ENTERPRISE extern const char* tkLogStb[]; extern const char* tkAuditStb[]; +extern const int tkLogStbNum; +extern const int tkAuditStbNum; #endif #define TAG_FILTER_RES_KEY_LEN 32 @@ -946,10 +948,10 @@ int32_t metaInitTbFilterCache(SMeta* pMeta) { if (!(dbName = strchr(pMeta->pVnode->config.dbname, '.'))) return 0; if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { - tbNum = TK_LOG_STB_NUM; + tbNum = tkLogStbNum; pTbArr = (const char**)&tkLogStb; } else if (0 == strncmp(dbName, "audit", TSDB_DB_NAME_LEN)) { - tbNum = TK_AUDIT_STB_NUM; + tbNum = tkAuditStbNum; pTbArr = (const char**)&tkAuditStb; } if (tbNum && pTbArr) { diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 3ec85ac93a..e9dbc5e659 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -566,26 +566,28 @@ int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) { } #ifdef TD_ENTERPRISE -const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info", - "data_dir", - "dnodes_info", - "d_info", - "grants_info", - "keeper_monitor", - "logs", - "log_dir", - "log_summary", - "m_info", - "taosadapter_restful_http_request_fail", - "taosadapter_restful_http_request_in_flight", - "taosadapter_restful_http_request_summary_milliseconds", - "taosadapter_restful_http_request_total", - "taosadapter_system_cpu_percent", - "taosadapter_system_mem_percent", - "temp_dir", - "vgroups_info", - "vnodes_role"}; -const char *tkAuditStb[TK_AUDIT_STB_NUM] = {"operations"}; +const char *tkLogStb[] = {"cluster_info", + "data_dir", + "dnodes_info", + "d_info", + "grants_info", + "keeper_monitor", + "logs", + "log_dir", + "log_summary", + "m_info", + "taosadapter_restful_http_request_fail", + "taosadapter_restful_http_request_in_flight", + "taosadapter_restful_http_request_summary_milliseconds", + "taosadapter_restful_http_request_total", + "taosadapter_system_cpu_percent", + "taosadapter_system_mem_percent", + "temp_dir", + "vgroups_info", + "vnodes_role"}; +const char *tkAuditStb[] = {"operations"}; +const int tkLogStbNum = ARRAY_SIZE(tkLogStb); +const int tkAuditStbNum = ARRAY_SIZE(tkAuditStb); // exclude stbs of taoskeeper log static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { @@ -596,10 +598,10 @@ static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { if (!(dbName = strchr(pVnode->config.dbname, '.'))) return 0; if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { - tbNum = TK_LOG_STB_NUM; + tbNum = tkLogStbNum; pTbArr = (const char **)&tkLogStb; } else if (0 == strncmp(dbName, "audit", TSDB_DB_NAME_LEN)) { - tbNum = TK_AUDIT_STB_NUM; + tbNum = tkAuditStbNum; pTbArr = (const char **)&tkAuditStb; } if (tbNum && pTbArr) { From 47ad5f4dd52c032df9fcd28d46ea60b6f1665cb2 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sat, 7 Oct 2023 16:15:38 +0800 Subject: [PATCH 34/36] test/cases: apply duration <= 1/3keep0 --- tests/script/tsim/parser/alter.sim | 2 +- .../tsim/parser/alter__for_community_version.sim | 2 +- tests/script/tsim/parser/create_db.sim | 16 ++++++++-------- tests/system-test/0-others/multilevel.py | 4 ++-- tests/system-test/1-insert/keep_expired.py | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/script/tsim/parser/alter.sim b/tests/script/tsim/parser/alter.sim index f2481576d1..50738565a1 100644 --- a/tests/script/tsim/parser/alter.sim +++ b/tests/script/tsim/parser/alter.sim @@ -17,7 +17,7 @@ $db = $dbPrefix . $i $mt = $mtPrefix . $i sql drop database if exists $db -sql create database $db duration 10 keep 20,20,20 +sql create database $db duration 6 keep 20,20,20 sql use $db sql_error alter database $db keep "20" diff --git a/tests/script/tsim/parser/alter__for_community_version.sim b/tests/script/tsim/parser/alter__for_community_version.sim index 48fb2f8246..e8da1e3d46 100644 --- a/tests/script/tsim/parser/alter__for_community_version.sim +++ b/tests/script/tsim/parser/alter__for_community_version.sim @@ -17,7 +17,7 @@ $db = $dbPrefix . $i $mt = $mtPrefix . $i sql drop database if exists $db -sql create database $db duration 10 keep 20 +sql create database $db duration 6 keep 20 sql use $db sql select * from information_schema.ins_databases if $rows != 3 then diff --git a/tests/script/tsim/parser/create_db.sim b/tests/script/tsim/parser/create_db.sim index db25240262..a5722f9819 100644 --- a/tests/script/tsim/parser/create_db.sim +++ b/tests/script/tsim/parser/create_db.sim @@ -150,26 +150,26 @@ sql_error create database $db keep 12,11,12 sql_error create database $db keep 8 sql_error create database $db keep 12,11 sql_error create database $db keep 365001,365001,365001 -sql create database dbk0 keep 19 +sql create database dbk0 keep 39 sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 27360m,27360m,27360m then +if $data27 != 56160m,56160m,56160m then return -1 endi sql drop database dbk0 -sql create database dbka keep 19,20 +sql create database dbka keep 39,40 sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 27360m,28800m,28800m then +if $data27 != 56160m,57600m,57600m then return -1 endi sql drop database dbka -sql create database dbk1 keep 11,11,11 +sql create database dbk1 duration 3 keep 11,11,11 sql select * from information_schema.ins_databases if $rows != 3 then return -1 @@ -178,7 +178,7 @@ if $data27 != 15840m,15840m,15840m then return -1 endi sql drop database dbk1 -sql create database dbk2 keep 11,12,13 +sql create database dbk2 duration 3 keep 11,12,13 sql select * from information_schema.ins_databases if $rows != 3 then return -1 @@ -187,7 +187,7 @@ if $data27 != 15840m,17280m,18720m then return -1 endi sql drop database dbk2 -sql create database dbk3 keep 11,11,13 +sql create database dbk3 duration 3 keep 11,11,13 sql select * from information_schema.ins_databases if $rows != 3 then return -1 @@ -196,7 +196,7 @@ if $data27 != 15840m,15840m,18720m then return -1 endi sql drop database dbk3 -sql create database dbk4 keep 11,13,13 +sql create database dbk4 duration 3 keep 11,13,13 sql select * from information_schema.ins_databases if $rows != 3 then return -1 diff --git a/tests/system-test/0-others/multilevel.py b/tests/system-test/0-others/multilevel.py index f086dcb735..3409a0e4b3 100644 --- a/tests/system-test/0-others/multilevel.py +++ b/tests/system-test/0-others/multilevel.py @@ -100,7 +100,7 @@ class TDTestCase: tdDnodes.deploy(1,cfg) tdDnodes.start(1) - tdSql.execute("create database test duration 1") + tdSql.execute("create database test duration 8h") tdSql.execute("use test") tdSql.execute("create table stb(ts timestamp, c int) tags(t int)") @@ -117,7 +117,7 @@ class TDTestCase: tdLog.info("================= step3") tdSql.execute('drop database test') for i in range(10): - tdSql.execute("create database test%d duration 1" %(i)) + tdSql.execute("create database test%d duration 8h" %(i)) tdSql.execute("use test%d" %(i)) tdSql.execute("create table tb (ts timestamp,i int)") for j in range(10): diff --git a/tests/system-test/1-insert/keep_expired.py b/tests/system-test/1-insert/keep_expired.py index 0cb6d841ed..4c8be55be5 100644 --- a/tests/system-test/1-insert/keep_expired.py +++ b/tests/system-test/1-insert/keep_expired.py @@ -15,7 +15,7 @@ class TDTestCase: self.stbname = "stb" self.ctbname = "ctb" self.keep_value = "2d,2d,2d" - self.duration_value = "1d" + self.duration_value = "16h" self.offset_time = 5 self.sleep_time = self.offset_time*2 From 028ffc032ba0b66000d0e1c4dda33a809c74e248 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sat, 7 Oct 2023 18:33:48 +0800 Subject: [PATCH 35/36] test/keep: fix relation with duration --- docs/en/12-taos-sql/02-database.md | 2 +- docs/zh/12-taos-sql/02-database.md | 2 +- tests/script/tsim/parser/alter.sim | 6 +++--- tests/script/tsim/parser/alter__for_community_version.sim | 4 ++-- tests/system-test/0-others/multilevel.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/en/12-taos-sql/02-database.md b/docs/en/12-taos-sql/02-database.md index e7f3aa8d1b..255263ecfd 100644 --- a/docs/en/12-taos-sql/02-database.md +++ b/docs/en/12-taos-sql/02-database.md @@ -56,7 +56,7 @@ database_option: { - WAL_FSYNC_PERIOD: specifies the interval (in milliseconds) at which data is written from the WAL to disk. This parameter takes effect only when the WAL parameter is set to 2. The default value is 3000. Enter a value between 0 and 180000. The value 0 indicates that incoming data is immediately written to disk. - MAXROWS: specifies the maximum number of rows recorded in a block. The default value is 4096. - MINROWS: specifies the minimum number of rows recorded in a block. The default value is 100. -- KEEP: specifies the time for which data is retained. Enter a value between 1 and 365000. The default value is 3650. The value of the KEEP parameter must be greater than or equal to the value of the DURATION parameter. TDengine automatically deletes data that is older than the value of the KEEP parameter. You can use m (minutes), h (hours), and d (days) as the unit, for example KEEP 100h or KEEP 10d. If you do not include a unit, d is used by default. TDengine Enterprise supports [Tiered Storage](https://docs.tdengine.com/tdinternal/arch/#tiered-storage) function, thus multiple KEEP values (comma separated and up to 3 values supported, and meet keep 0 <= keep 1 <= keep 2, e.g. KEEP 100h,100d,3650d) are supported; TDengine OSS does not support Tiered Storage function (although multiple keep values are configured, they do not take effect, only the maximum keep value is used as KEEP). +- KEEP: specifies the time for which data is retained. Enter a value between 1 and 365000. The default value is 3650. The value of the KEEP parameter must be greater than or equal to three times of the value of the DURATION parameter. TDengine automatically deletes data that is older than the value of the KEEP parameter. You can use m (minutes), h (hours), and d (days) as the unit, for example KEEP 100h or KEEP 10d. If you do not include a unit, d is used by default. TDengine Enterprise supports [Tiered Storage](https://docs.tdengine.com/tdinternal/arch/#tiered-storage) function, thus multiple KEEP values (comma separated and up to 3 values supported, and meet keep 0 <= keep 1 <= keep 2, e.g. KEEP 100h,100d,3650d) are supported; TDengine OSS does not support Tiered Storage function (although multiple keep values are configured, they do not take effect, only the maximum keep value is used as KEEP). - PAGES: specifies the number of pages in the metadata storage engine cache on each vnode. Enter a value greater than or equal to 64. The default value is 256. The space occupied by metadata storage on each vnode is equal to the product of the values of the PAGESIZE and PAGES parameters. The space occupied by default is 1 MB. - PAGESIZE: specifies the size (in KB) of each page in the metadata storage engine cache on each vnode. The default value is 4. Enter a value between 1 and 16384. - PRECISION: specifies the precision at which a database records timestamps. Enter ms for milliseconds, us for microseconds, or ns for nanoseconds. The default value is ms. diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index ac435debea..e9ca5405f4 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -56,7 +56,7 @@ database_option: { - WAL_FSYNC_PERIOD：当 WAL 参数设置为 2 时，落盘的周期。默认为 3000，单位毫秒。最小为 0，表示每次写入立即落盘；最大为 180000，即三分钟。 - MAXROWS：文件块中记录的最大条数，默认为 4096 条。 - MINROWS：文件块中记录的最小条数，默认为 100 条。 -- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 <= keep 1 <= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。 +- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于3倍的 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 <= keep 1 <= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。 - PAGES：一个 VNODE 中元数据存储引擎的缓存页个数，默认为 256，最小 64。一个 VNODE 元数据存储占用 PAGESIZE \* PAGES，默认情况下为 1MB 内存。 - PAGESIZE：一个 VNODE 中元数据存储引擎的页大小，单位为 KB，默认为 4 KB。范围为 1 到 16384，即 1 KB 到 16 MB。 - PRECISION：数据库的时间戳精度。ms 表示毫秒，us 表示微秒，ns 表示纳秒，默认 ms 毫秒。 diff --git a/tests/script/tsim/parser/alter.sim b/tests/script/tsim/parser/alter.sim index 50738565a1..99e014a011 100644 --- a/tests/script/tsim/parser/alter.sim +++ b/tests/script/tsim/parser/alter.sim @@ -17,7 +17,7 @@ $db = $dbPrefix . $i $mt = $mtPrefix . $i sql drop database if exists $db -sql create database $db duration 6 keep 20,20,20 +sql create database $db duration 3 keep 20,20,20 sql use $db sql_error alter database $db keep "20" @@ -27,8 +27,8 @@ sql_error alter database $db keep 20.0 sql_error alter database $db keep 20.0,20.0,20.0 sql_error alter database $db keep 0,0,0 sql_error alter database $db keep -1,-1,-1 -sql_error alter database $db keep 9,20 -sql_error alter database $db keep 9,9,9 +sql_error alter database $db keep 8,20 +sql_error alter database $db keep 8,9,9 sql_error alter database $db keep 20,20,19 sql_error alter database $db keep 20,19,20 sql_error alter database $db keep 20,19,19 diff --git a/tests/script/tsim/parser/alter__for_community_version.sim b/tests/script/tsim/parser/alter__for_community_version.sim index e8da1e3d46..29c748d441 100644 --- a/tests/script/tsim/parser/alter__for_community_version.sim +++ b/tests/script/tsim/parser/alter__for_community_version.sim @@ -17,7 +17,7 @@ $db = $dbPrefix . $i $mt = $mtPrefix . $i sql drop database if exists $db -sql create database $db duration 6 keep 20 +sql create database $db duration 3 keep 20 sql use $db sql select * from information_schema.ins_databases if $rows != 3 then @@ -36,7 +36,7 @@ sql_error alter database $db keep 0,0,0 sql_error alter database $db keep 3 sql_error alter database $db keep -1,-1,-1 sql alter database $db keep 20,20 -sql_error alter database $db keep 9,9,9 +sql_error alter database $db keep 8,9,9 sql_error alter database $db keep 20,20,19 sql_error alter database $db keep 20,19,20 sql_error alter database $db keep 20,19,19 diff --git a/tests/system-test/0-others/multilevel.py b/tests/system-test/0-others/multilevel.py index 3409a0e4b3..66434fff67 100644 --- a/tests/system-test/0-others/multilevel.py +++ b/tests/system-test/0-others/multilevel.py @@ -216,7 +216,7 @@ class TDTestCase: tdDnodes.deploy(1,cfg) tdDnodes.start(1) - tdSql.execute('create database dbtest') + tdSql.execute('create database dbtest duration 3') tdSql.execute('use dbtest') tdSql.execute('create table stb (ts timestamp,c0 int) tags(t0 int)') tdSql.execute('create table tb1 using stb tags(1)') From a2e31094479f2a3b4bff9eec30914eb10fbe9b1c Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 8 Oct 2023 09:47:57 +0800 Subject: [PATCH 36/36] chore: add grant method for community --- source/dnode/mnode/impl/src/mndGrant.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/mnode/impl/src/mndGrant.c b/source/dnode/mnode/impl/src/mndGrant.c index c4c5e0355d..c309fa2d4d 100644 --- a/source/dnode/mnode/impl/src/mndGrant.c +++ b/source/dnode/mnode/impl/src/mndGrant.c @@ -131,6 +131,7 @@ void grantAdd(EGrantType grant, uint64_t value) {} void grantRestore(EGrantType grant, uint64_t value) {} int32_t dmProcessGrantReq(void *pInfo, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; } int32_t dmProcessGrantNotify(void *pInfo, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; } +int32_t grantAlterActiveCode(const char *old, const char *new, char *out, int8_t type) { return TSDB_CODE_SUCCESS; } #endif