From 9ce0c184a169af5be88ec5c50147b27db363186e Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 8 Aug 2024 16:44:15 +0800 Subject: [PATCH 01/35] enh: add passwdTest --- source/client/test/CMakeLists.txt | 11 +++++++++++ tests/script/api/passwdTest.c | 18 +++++++++++------- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt index 3d70c67661..8e7056821d 100644 --- a/source/client/test/CMakeLists.txt +++ b/source/client/test/CMakeLists.txt @@ -29,6 +29,12 @@ TARGET_LINK_LIBRARIES( # PUBLIC os util common transport monitor parser catalog scheduler function gtest taos_static qcom executor #) +ADD_EXECUTABLE(passwdTest ../../../tests/script/api/passwdTest.c) +TARGET_LINK_LIBRARIES( + passwdTest + PUBLIC taos +) + TARGET_INCLUDE_DIRECTORIES( clientTest PUBLIC "${TD_SOURCE_DIR}/include/client/" @@ -62,3 +68,8 @@ add_test( # NAME clientMonitorTest # COMMAND clientMonitorTest # ) + +add_test( + NAME passwdTest + COMMAND passwdTest +) diff --git a/tests/script/api/passwdTest.c b/tests/script/api/passwdTest.c index 928525750e..bd5e3af755 100644 --- a/tests/script/api/passwdTest.c +++ b/tests/script/api/passwdTest.c @@ -59,7 +59,8 @@ void __taos_notify_cb(void *param, void *ext, int type) { switch (type) { case TAOS_NOTIFY_PASSVER: { ++nPassVerNotified; - printf("%s:%d type:%d user:%s passVer:%d\n", __func__, __LINE__, type, param ? (char *)param : "NULL", *(int *)ext); + printf("%s:%d type:%d user:%s passVer:%d\n", __func__, __LINE__, type, param ? (char *)param : "NULL", + *(int *)ext); break; } case TAOS_NOTIFY_USER_DROPPED: { @@ -191,11 +192,11 @@ static int printResult(TAOS_RES *res, char *output) { printRow(temp, row, fields, numFields); puts(temp); } + return 0; } int main(int argc, char *argv[]) { char qstr[1024]; - // connect to server if (argc < 2) { printf("please input server-ip \n"); @@ -215,6 +216,7 @@ int main(int argc, char *argv[]) { taos_close(taos); taos_cleanup(); + exit(EXIT_SUCCESS); } void createUsers(TAOS *taos, const char *host, char *qstr) { @@ -234,6 +236,7 @@ void createUsers(TAOS *taos, const char *host, char *qstr) { if (code != 0) { fprintf(stderr, "failed to run: taos_set_notify_cb(TAOS_NOTIFY_PASSVER) for user:%s since %d\n", users[i], code); + exit(EXIT_FAILURE); } else { fprintf(stderr, "success to run: taos_set_notify_cb(TAOS_NOTIFY_PASSVER) for user:%s\n", users[i]); } @@ -260,6 +263,7 @@ void passVerTestMulti(const char *host, char *qstr) { if (code != 0) { fprintf(stderr, "failed to run: taos_set_notify_cb since %d\n", code); + exit(EXIT_FAILURE); } else { fprintf(stderr, "success to run: taos_set_notify_cb\n"); } @@ -301,8 +305,7 @@ void passVerTestMulti(const char *host, char *qstr) { fprintf(stderr, "######## %s #########\n", __func__); if (nPassVerNotified == nConn) { - fprintf(stderr, ">>> succeed to get passVer notification since nNotify %d == nConn %d\n", nPassVerNotified, - nConn); + fprintf(stderr, ">>> succeed to get passVer notification since nNotify %d == nConn %d\n", nPassVerNotified, nConn); } else { fprintf(stderr, ">>> failed to get passVer notification since nNotify %d != nConn %d\n", nPassVerNotified, nConn); exit(1); @@ -337,7 +340,7 @@ void sysInfoTest(TAOS *taosRoot, const char *host, char *qstr) { TAOS_RES *res = NULL; int32_t nRep = 0; -_REP: +_REP: fprintf(stderr, "######## %s loop:%d #########\n", __func__, nRep); res = taos_query(taos[0], qstr); if (taos_errno(res) != 0) { @@ -375,7 +378,7 @@ _REP: sleep(1); } - if(++nRep < 5) { + if (++nRep < 5) { goto _REP; } @@ -390,7 +393,7 @@ _REP: fprintf(stderr, "######## %s #########\n", __func__); } static bool isDropUser = true; -void userDroppedTest(TAOS *taos, const char *host, char *qstr) { +void userDroppedTest(TAOS *taos, const char *host, char *qstr) { // users int nTestUsers = nUser; int nLoop = 0; @@ -408,6 +411,7 @@ _loop: if (code != 0) { fprintf(stderr, "failed to run: taos_set_notify_cb:%d for user:%s since %d\n", TAOS_NOTIFY_USER_DROPPED, users[i], code); + exit(EXIT_FAILURE); } else { fprintf(stderr, "success to run: taos_set_notify_cb:%d for user:%s\n", TAOS_NOTIFY_USER_DROPPED, users[i]); } From a23c187674a5200e74d70a39913edcf4d3437713 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 8 Aug 2024 16:55:17 +0800 Subject: [PATCH 02/35] enh: rename passwdTest --- source/client/test/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt index 8e7056821d..c8b13f6532 100644 --- a/source/client/test/CMakeLists.txt +++ b/source/client/test/CMakeLists.txt @@ -29,9 +29,9 @@ TARGET_LINK_LIBRARIES( # PUBLIC os util common transport monitor parser catalog scheduler function gtest taos_static qcom executor #) -ADD_EXECUTABLE(passwdTest ../../../tests/script/api/passwdTest.c) +ADD_EXECUTABLE(userOperTest ../../../tests/script/api/passwdTest.c) TARGET_LINK_LIBRARIES( - passwdTest + userOperTest PUBLIC taos ) @@ -70,6 +70,6 @@ add_test( # ) add_test( - NAME passwdTest - COMMAND passwdTest + NAME userOperTest + COMMAND userOperTest ) From f887ec2a0d6d4506f2e307273ea03503ce3b20cc Mon Sep 17 00:00:00 2001 From: charles Date: Fri, 9 Aug 2024 09:26:54 +0800 Subject: [PATCH 03/35] add test case test_passwd by charles --- tests/army/user/test_passwd.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 tests/army/user/test_passwd.py diff --git a/tests/army/user/test_passwd.py b/tests/army/user/test_passwd.py new file mode 100644 index 0000000000..2d448b8f0e --- /dev/null +++ b/tests/army/user/test_passwd.py @@ -0,0 +1,20 @@ +import subprocess +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame.epath import * +from frame import * + + +class TDTestCase(TBase): + def run(self): + c_file_path = os.sep.join([binPath(), "passwdTest localhost"]) + p = subprocess.Popen(c_file_path, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + if 0 != p.returncode: + tdLog.error("Failed to run passwd test with output: %s \n error: %s" % (out, err)) + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From b330edf535ec3f91aae49abfa0bafa81011be557 Mon Sep 17 00:00:00 2001 From: charles Date: Fri, 9 Aug 2024 09:26:54 +0800 Subject: [PATCH 04/35] add test case test_passwd by charles --- tests/army/user/test_passwd.py | 20 ++++++++++++++++++++ tests/parallel_test/cases.task | 1 + 2 files changed, 21 insertions(+) create mode 100644 tests/army/user/test_passwd.py diff --git a/tests/army/user/test_passwd.py b/tests/army/user/test_passwd.py new file mode 100644 index 0000000000..2d448b8f0e --- /dev/null +++ b/tests/army/user/test_passwd.py @@ -0,0 +1,20 @@ +import subprocess +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame.epath import * +from frame import * + + +class TDTestCase(TBase): + def run(self): + c_file_path = os.sep.join([binPath(), "passwdTest localhost"]) + p = subprocess.Popen(c_file_path, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + if 0 != p.returncode: + tdLog.error("Failed to run passwd test with output: %s \n error: %s" % (out, err)) + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e70042001d..bc30701e82 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -35,6 +35,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f storage/compressBasic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f grant/grantBugs.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f query/queryBugs.py -N 3 +,,y,army,./pytest.sh python3 ./test.py -f user/test_passwd.py # # system test From 1d6f90f5ff4816cacb92efc929d0a7521ae50cc5 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 17 Oct 2024 11:33:27 +0800 Subject: [PATCH 05/35] fix:[TD-32563] remove config variable slowLogThresholdTestg --- include/common/tglobal.h | 1 - include/common/tmsg.h | 2 +- source/client/src/clientEnv.c | 3 +-- source/common/src/tglobal.c | 6 ------ source/common/src/tmsg.c | 4 ++-- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 1 - source/dnode/mnode/impl/src/mndDnode.c | 1 - source/dnode/mnode/impl/src/mndProfile.c | 2 -- 8 files changed, 4 insertions(+), 16 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index cf918c6e0d..b7b89e4f30 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -179,7 +179,6 @@ extern int32_t tsMaxRetryWaitTime; extern bool tsUseAdapter; extern int32_t tsMetaCacheMaxSize; extern int32_t tsSlowLogThreshold; -extern int32_t tsSlowLogThresholdTest; extern char tsSlowLogExceptDb[]; extern int32_t tsSlowLogScope; extern int32_t tsSlowLogMaxLen; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 1a10f02c96..5953ed3e21 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -662,7 +662,7 @@ typedef struct { int32_t tsSlowLogThreshold; int32_t tsSlowLogMaxLen; int32_t tsSlowLogScope; - int32_t tsSlowLogThresholdTest; + int32_t tsSlowLogThresholdTest; //Obsolete char tsSlowLogExceptDb[TSDB_DB_NAME_LEN]; } SMonitorParas; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 21f7c93036..dd681f8a46 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -294,8 +294,7 @@ static void deregisterRequest(SRequestObj *pRequest) { } } - if ((duration >= pTscObj->pAppInfo->monitorParas.tsSlowLogThreshold * 1000000UL || - duration >= pTscObj->pAppInfo->monitorParas.tsSlowLogThresholdTest * 1000000UL) && + if ((duration >= pTscObj->pAppInfo->monitorParas.tsSlowLogThreshold * 1000000UL) && checkSlowLogExceptDb(pRequest, pTscObj->pAppInfo->monitorParas.tsSlowLogExceptDb)) { (void)atomic_add_fetch_64((int64_t *)&pActivity->numOfSlowQueries, 1); if (pTscObj->pAppInfo->monitorParas.tsSlowLogScope & reqType) { diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 3c05294264..9866b3f760 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -181,7 +181,6 @@ int32_t tsMaxRetryWaitTime = 10000; bool tsUseAdapter = false; int32_t tsMetaCacheMaxSize = -1; // MB int32_t tsSlowLogThreshold = 10; // seconds -int32_t tsSlowLogThresholdTest = INT32_MAX; // seconds char tsSlowLogExceptDb[TSDB_DB_NAME_LEN] = ""; // seconds int32_t tsSlowLogScope = SLOW_LOG_TYPE_QUERY; char* tsSlowLogScopeString = "query"; @@ -749,7 +748,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 86400, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "slowLogThresholdTest", tsSlowLogThresholdTest, 0, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 1, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "slowLogMaxLen", tsSlowLogMaxLen, 1, 16384, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "slowLogScope", tsSlowLogScopeString, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); @@ -1401,9 +1399,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "slowLogExceptDb"); tstrncpy(tsSlowLogExceptDb, pItem->str, TSDB_DB_NAME_LEN); - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "slowLogThresholdTest"); - tsSlowLogThresholdTest = pItem->i32; - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "slowLogThreshold"); tsSlowLogThreshold = pItem->i32; @@ -1968,7 +1963,6 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"monitor", &tsEnableMonitor}, {"monitorInterval", &tsMonitorInterval}, {"slowLogThreshold", &tsSlowLogThreshold}, - {"slowLogThresholdTest", &tsSlowLogThresholdTest}, {"slowLogMaxLen", &tsSlowLogMaxLen}, {"mndSdbWriteDelta", &tsMndSdbWriteDelta}, diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 4c4b78278e..f2151faec3 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -75,7 +75,7 @@ static int32_t tSerializeSMonitorParas(SEncoder *encoder, const SMonitorParas *p TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogScope)); TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogMaxLen)); TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogThreshold)); - TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogThresholdTest)); + TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogThresholdTest)); //Obsolete TAOS_CHECK_RETURN(tEncodeCStr(encoder, pMonitorParas->tsSlowLogExceptDb)); return 0; } @@ -86,7 +86,7 @@ static int32_t tDeserializeSMonitorParas(SDecoder *decoder, SMonitorParas *pMoni TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogScope)); TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogMaxLen)); TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogThreshold)); - TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogThresholdTest)); + TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogThresholdTest)); //Obsolete TAOS_CHECK_RETURN(tDecodeCStrTo(decoder, pMonitorParas->tsSlowLogExceptDb)); return 0; } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 87b1ae0efa..666b916115 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -151,7 +151,6 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { req.clusterCfg.monitorParas.tsSlowLogScope = tsSlowLogScope; req.clusterCfg.monitorParas.tsSlowLogMaxLen = tsSlowLogMaxLen; req.clusterCfg.monitorParas.tsSlowLogThreshold = tsSlowLogThreshold; - req.clusterCfg.monitorParas.tsSlowLogThresholdTest = tsSlowLogThresholdTest; tstrncpy(req.clusterCfg.monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb, TSDB_DB_NAME_LEN); char timestr[32] = "1970-01-01 00:00:00.00"; if (taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0) != 0) { diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 04041646eb..f1c6765c9b 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -480,7 +480,6 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, SDnodeObj *pDnode, const S CHECK_MONITOR_PARA(tsEnableMonitor, DND_REASON_STATUS_MONITOR_SWITCH_NOT_MATCH); CHECK_MONITOR_PARA(tsMonitorInterval, DND_REASON_STATUS_MONITOR_INTERVAL_NOT_MATCH); CHECK_MONITOR_PARA(tsSlowLogThreshold, DND_REASON_STATUS_MONITOR_SLOW_LOG_THRESHOLD_NOT_MATCH); - CHECK_MONITOR_PARA(tsSlowLogThresholdTest, DND_REASON_STATUS_MONITOR_NOT_MATCH); CHECK_MONITOR_PARA(tsSlowLogMaxLen, DND_REASON_STATUS_MONITOR_SLOW_LOG_SQL_MAX_LEN_NOT_MATCH); CHECK_MONITOR_PARA(tsSlowLogScope, DND_REASON_STATUS_MONITOR_SLOW_LOG_SCOPE_NOT_MATCH); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index a1ffee9b06..fa98f91ab1 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -304,7 +304,6 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { connectRsp.monitorParas.tsSlowLogScope = tsSlowLogScope; connectRsp.monitorParas.tsSlowLogMaxLen = tsSlowLogMaxLen; connectRsp.monitorParas.tsSlowLogThreshold = tsSlowLogThreshold; - connectRsp.monitorParas.tsSlowLogThresholdTest = tsSlowLogThresholdTest; tstrncpy(connectRsp.monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb, TSDB_DB_NAME_LEN); connectRsp.whiteListVer = pUser->ipWhiteListVer; @@ -705,7 +704,6 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) { batchRsp.monitorParas.tsEnableMonitor = tsEnableMonitor; batchRsp.monitorParas.tsMonitorInterval = tsMonitorInterval; batchRsp.monitorParas.tsSlowLogThreshold = tsSlowLogThreshold; - batchRsp.monitorParas.tsSlowLogThresholdTest = tsSlowLogThresholdTest; tstrncpy(batchRsp.monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb, TSDB_DB_NAME_LEN); batchRsp.monitorParas.tsSlowLogMaxLen = tsSlowLogMaxLen; batchRsp.monitorParas.tsSlowLogScope = tsSlowLogScope; From 7ff0fb497a7a924a1665c4dfd966b51457b81a64 Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Tue, 5 Nov 2024 15:46:36 +0800 Subject: [PATCH 06/35] Update test_passwd.py to replace tdLog.error with tdLog.exit --- tests/army/user/test_passwd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/user/test_passwd.py b/tests/army/user/test_passwd.py index 2d448b8f0e..ac82ee0d55 100644 --- a/tests/army/user/test_passwd.py +++ b/tests/army/user/test_passwd.py @@ -13,7 +13,7 @@ class TDTestCase(TBase): p = subprocess.Popen(c_file_path, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() if 0 != p.returncode: - tdLog.error("Failed to run passwd test with output: %s \n error: %s" % (out, err)) + tdLog.exit("Failed to run passwd test with output: %s \n error: %s" % (out, err)) tdLog.success(f"{__file__} successfully executed") tdCases.addLinux(__file__, TDTestCase()) From 9003883edc83320300ef2ebf56e89870b7d4d6cc Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Tue, 5 Nov 2024 20:23:45 +0800 Subject: [PATCH 07/35] Update makefile to remove the stmt.c which doesn't exist --- tests/script/api/makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/api/makefile b/tests/script/api/makefile index 9c2bb6be3d..6b8a920e33 100644 --- a/tests/script/api/makefile +++ b/tests/script/api/makefile @@ -22,7 +22,7 @@ exe: gcc $(CFLAGS) ./insert_stb.c -o $(ROOT)insert_stb $(LFLAGS) gcc $(CFLAGS) ./tmqViewTest.c -o $(ROOT)tmqViewTest $(LFLAGS) gcc $(CFLAGS) ./stmtQuery.c -o $(ROOT)stmtQuery $(LFLAGS) - gcc $(CFLAGS) ./stmt.c -o $(ROOT)stmt $(LFLAGS) + # gcc $(CFLAGS) ./stmt.c -o $(ROOT)stmt $(LFLAGS) gcc $(CFLAGS) ./stmt2.c -o $(ROOT)stmt2 $(LFLAGS) gcc $(CFLAGS) ./stmt2-example.c -o $(ROOT)stmt2-example $(LFLAGS) gcc $(CFLAGS) ./stmt2-get-fields.c -o $(ROOT)stmt2-get-fields $(LFLAGS) From 0d5d88086ae1a2f48309720c2afb0b7f80be697b Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Tue, 5 Nov 2024 20:25:02 +0800 Subject: [PATCH 08/35] Update test_passwd.py --- tests/army/user/test_passwd.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/tests/army/user/test_passwd.py b/tests/army/user/test_passwd.py index ac82ee0d55..cc0ab1a113 100644 --- a/tests/army/user/test_passwd.py +++ b/tests/army/user/test_passwd.py @@ -6,15 +6,37 @@ from frame.caseBase import * from frame.epath import * from frame import * - class TDTestCase(TBase): + def apiPath(self): + apiPath = None + currentFilePath = os.path.dirname(os.path.realpath(__file__)) + if ("community/tests" in currentFilePath): + testFilePath = currentFilePath[:currentFilePath.find("community/tests")] + else: + testFilePath = currentFilePath[:currentFilePath.find("TDengine/tests")] + + for root, dirs, files in os.walk(testFilePath): + if ("passwdTest.c" in files): + apiPath = root + break + return apiPath + def run(self): - c_file_path = os.sep.join([binPath(), "passwdTest localhost"]) - p = subprocess.Popen(c_file_path, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + apiPath = self.apiPath() + if apiPath: + p = subprocess.Popen(f"cd {apiPath} && make", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + if 0 != p.returncode: + tdLog.exit("Test script passwdTest.c make failed") + test_file_cmd = os.sep.join([apiPath, "passwdTest localhost"]) + else: + tdLog.exit("passwdTest.c not found") + p = subprocess.Popen(test_file_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() if 0 != p.returncode: tdLog.exit("Failed to run passwd test with output: %s \n error: %s" % (out, err)) tdLog.success(f"{__file__} successfully executed") + tdCases.addLinux(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase()) From dcb63d054fb9ee566c754d06ca91b331860eaec3 Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Wed, 6 Nov 2024 08:05:51 +0800 Subject: [PATCH 09/35] Update makefile to remove the failed test cases --- tests/script/api/makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/script/api/makefile b/tests/script/api/makefile index 6b8a920e33..ce5980b37a 100644 --- a/tests/script/api/makefile +++ b/tests/script/api/makefile @@ -13,7 +13,7 @@ all: $(TARGET) exe: gcc $(CFLAGS) ./batchprepare.c -o $(ROOT)batchprepare $(LFLAGS) - gcc $(CFLAGS) ./stmt2-test.c -o $(ROOT)stmt2-test $(LFLAGS) + # gcc $(CFLAGS) ./stmt2-test.c -o $(ROOT)stmt2-test $(LFLAGS) gcc $(CFLAGS) ./stopquery.c -o $(ROOT)stopquery $(LFLAGS) gcc $(CFLAGS) ./dbTableRoute.c -o $(ROOT)dbTableRoute $(LFLAGS) gcc $(CFLAGS) ./insertSameTs.c -o $(ROOT)insertSameTs $(LFLAGS) @@ -23,10 +23,10 @@ exe: gcc $(CFLAGS) ./tmqViewTest.c -o $(ROOT)tmqViewTest $(LFLAGS) gcc $(CFLAGS) ./stmtQuery.c -o $(ROOT)stmtQuery $(LFLAGS) # gcc $(CFLAGS) ./stmt.c -o $(ROOT)stmt $(LFLAGS) - gcc $(CFLAGS) ./stmt2.c -o $(ROOT)stmt2 $(LFLAGS) - gcc $(CFLAGS) ./stmt2-example.c -o $(ROOT)stmt2-example $(LFLAGS) - gcc $(CFLAGS) ./stmt2-get-fields.c -o $(ROOT)stmt2-get-fields $(LFLAGS) - gcc $(CFLAGS) ./stmt2-nohole.c -o $(ROOT)stmt2-nohole $(LFLAGS) + # gcc $(CFLAGS) ./stmt2.c -o $(ROOT)stmt2 $(LFLAGS) + # gcc $(CFLAGS) ./stmt2-example.c -o $(ROOT)stmt2-example $(LFLAGS) + # gcc $(CFLAGS) ./stmt2-get-fields.c -o $(ROOT)stmt2-get-fields $(LFLAGS) + # gcc $(CFLAGS) ./stmt2-nohole.c -o $(ROOT)stmt2-nohole $(LFLAGS) gcc $(CFLAGS) ./stmt-crash.c -o $(ROOT)stmt-crash $(LFLAGS) clean: From ffe1776f3e2032f5f74875d5f03a665303de9f24 Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Wed, 6 Nov 2024 09:45:19 +0800 Subject: [PATCH 10/35] Update test_passwd.py to add some log for debug --- tests/army/user/test_passwd.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/army/user/test_passwd.py b/tests/army/user/test_passwd.py index cc0ab1a113..1017805105 100644 --- a/tests/army/user/test_passwd.py +++ b/tests/army/user/test_passwd.py @@ -23,11 +23,12 @@ class TDTestCase(TBase): def run(self): apiPath = self.apiPath() + tdLog.info(f"api path: {apiPath}") if apiPath: p = subprocess.Popen(f"cd {apiPath} && make", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() if 0 != p.returncode: - tdLog.exit("Test script passwdTest.c make failed") + tdLog.exit(f"Test script passwdTest.c make failed with error: {err}") test_file_cmd = os.sep.join([apiPath, "passwdTest localhost"]) else: tdLog.exit("passwdTest.c not found") From eea5eea7b1af125a17e13c1252dd45e1878e36c4 Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Wed, 6 Nov 2024 11:26:22 +0800 Subject: [PATCH 11/35] Update cases.task to close santilizer for case test_passwd.py --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index c85073c951..3dc64e9de1 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -44,7 +44,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f storage/compressBasic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f grant/grantBugs.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f query/queryBugs.py -N 3 -,,y,army,./pytest.sh python3 ./test.py -f user/test_passwd.py +,,n,army,./pytest.sh python3 ./test.py -f user/test_passwd.py ,,y,army,./pytest.sh python3 ./test.py -f tmq/tmqBugs.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f query/fill/fill_compare_asc_desc.py ,,y,army,./pytest.sh python3 ./test.py -f query/last/test_last.py From e5098f82fa3d1ef6e48beba0e40613d8174e9fee Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Wed, 6 Nov 2024 11:33:25 +0800 Subject: [PATCH 12/35] Update cases.task --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 3dc64e9de1..c85073c951 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -44,7 +44,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f storage/compressBasic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f grant/grantBugs.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f query/queryBugs.py -N 3 -,,n,army,./pytest.sh python3 ./test.py -f user/test_passwd.py +,,y,army,./pytest.sh python3 ./test.py -f user/test_passwd.py ,,y,army,./pytest.sh python3 ./test.py -f tmq/tmqBugs.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f query/fill/fill_compare_asc_desc.py ,,y,army,./pytest.sh python3 ./test.py -f query/last/test_last.py From aaddf18e0091a762281afaa07b15b32b45300dbe Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Wed, 6 Nov 2024 11:35:42 +0800 Subject: [PATCH 13/35] Update test_passwd.py --- tests/army/user/test_passwd.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/army/user/test_passwd.py b/tests/army/user/test_passwd.py index 1017805105..62dc5782b7 100644 --- a/tests/army/user/test_passwd.py +++ b/tests/army/user/test_passwd.py @@ -28,16 +28,18 @@ class TDTestCase(TBase): p = subprocess.Popen(f"cd {apiPath} && make", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() if 0 != p.returncode: - tdLog.exit(f"Test script passwdTest.c make failed with error: {err}") + tdLog.info(f"Test script passwdTest.c make failed with error: {err}") test_file_cmd = os.sep.join([apiPath, "passwdTest localhost"]) else: tdLog.exit("passwdTest.c not found") - p = subprocess.Popen(test_file_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - if 0 != p.returncode: - tdLog.exit("Failed to run passwd test with output: %s \n error: %s" % (out, err)) - tdLog.success(f"{__file__} successfully executed") - + try: + p = subprocess.Popen(test_file_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + if 0 != p.returncode: + tdLog.exit("Failed to run passwd test with output: %s \n error: %s" % (out, err)) + tdLog.success(f"{__file__} successfully executed") + except Exception as e: + tdLog.exit(f"Failed to execute {__file__} with error: {e}") tdCases.addLinux(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase()) From f78124ff3a9675750d840ba37d4c44f58f695a9d Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Wed, 6 Nov 2024 15:05:39 +0800 Subject: [PATCH 14/35] Update cases.task to remove santilizer to avoid the make failed --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index c85073c951..3dc64e9de1 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -44,7 +44,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f storage/compressBasic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f grant/grantBugs.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f query/queryBugs.py -N 3 -,,y,army,./pytest.sh python3 ./test.py -f user/test_passwd.py +,,n,army,./pytest.sh python3 ./test.py -f user/test_passwd.py ,,y,army,./pytest.sh python3 ./test.py -f tmq/tmqBugs.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f query/fill/fill_compare_asc_desc.py ,,y,army,./pytest.sh python3 ./test.py -f query/last/test_last.py From c196dd86e08e91d396696385aca9d8d2bce8a44f Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Thu, 7 Nov 2024 07:55:47 +0800 Subject: [PATCH 15/35] Update cases.task --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 3dc64e9de1..cf3452b056 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -44,7 +44,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f storage/compressBasic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f grant/grantBugs.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f query/queryBugs.py -N 3 -,,n,army,./pytest.sh python3 ./test.py -f user/test_passwd.py +,,n,army,python3 ./test.py -f user/test_passwd.py ,,y,army,./pytest.sh python3 ./test.py -f tmq/tmqBugs.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f query/fill/fill_compare_asc_desc.py ,,y,army,./pytest.sh python3 ./test.py -f query/last/test_last.py From 089456fa5be415b4c7edfee3e01f74144b02c6ba Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 7 Nov 2024 13:43:08 +0800 Subject: [PATCH 16/35] enh: add makefile of passwdTest.c for windows --- tests/script/api/makefile_win64.mak | 20 +++++++++++++++ tests/script/api/passwdTest.c | 39 +++++++++++++++++++++++------ 2 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 tests/script/api/makefile_win64.mak diff --git a/tests/script/api/makefile_win64.mak b/tests/script/api/makefile_win64.mak new file mode 100644 index 0000000000..9963ef4133 --- /dev/null +++ b/tests/script/api/makefile_win64.mak @@ -0,0 +1,20 @@ +# Makefile.mak for win64 + +TARGET = passwdTest.exe +CC = cl +CFLAGS = /W4 /EHsc /I"C:\TDengine\include" /D_WINDOWS +LDFLAGS = /link /LIBPATH:"C:\TDengine\driver" taos.lib + +SRCS = passwdTest.c +OBJS = $(SRCS:.c=.obj) + +all: $(TARGET) + +$(TARGET): $(OBJS) + $(CC) $(OBJS) $(LDFLAGS) + +.c.obj: + $(CC) $(CFLAGS) /c $< + +clean: + del $(OBJS) $(TARGET) \ No newline at end of file diff --git a/tests/script/api/passwdTest.c b/tests/script/api/passwdTest.c index 928525750e..7d35b8cc19 100644 --- a/tests/script/api/passwdTest.c +++ b/tests/script/api/passwdTest.c @@ -20,12 +20,27 @@ * passwdTest.c * - Run the test case in clear TDengine environment with default root passwd 'taosdata' */ +#ifdef WINDOWS +#include +#include +#include +#ifndef PRId64 +#define PRId64 "I64d" +#endif + +#ifndef PRIu64 +#define PRIu64 "I64u" +#endif + +#else #include +#include +#endif + #include #include #include -#include #include "taos.h" // TAOS header file #define nDup 1 @@ -50,6 +65,16 @@ void sysInfoTest(TAOS *taos, const char *host, char *qstr); void userDroppedTest(TAOS *taos, const char *host, char *qstr); void clearTestEnv(TAOS *taos, const char *host, char *qstr); +void taosMsleep(int64_t ms) { + if (ms < 0) return; +#ifdef WINDOWS + Sleep(ms); +#else + usleep(ms * 1000); +#endif +} + + int nPassVerNotified = 0; int nUserDropped = 0; TAOS *taosu[nRoot] = {0}; @@ -283,20 +308,20 @@ void passVerTestMulti(const char *host, char *qstr) { printf("%s:%d [%d] second(s) elasped, passVer notification received:%d, total:%d\n", __func__, __LINE__, i, nPassVerNotified, nConn); if (nPassVerNotified >= nConn) break; - sleep(1); + taosMsleep(1000); } // close the taos_conn for (int i = 0; i < nRoot; ++i) { taos_close(taos[i]); printf("%s:%d close taos[%d]\n", __func__, __LINE__, i); - // sleep(1); + // taosMsleep(1000); } for (int i = 0; i < nUser; ++i) { taos_close(taosu[i]); printf("%s:%d close taosu[%d]\n", __func__, __LINE__, i); - // sleep(1); + // taosMsleep(1000); } fprintf(stderr, "######## %s #########\n", __func__); @@ -356,7 +381,7 @@ _REP: fprintf(stderr, "%s:%d sleep 2 seconds to wait HB take effect\n", __func__, __LINE__); for (int i = 1; i <= 2; ++i) { - sleep(1); + taosMsleep(1000); } res = taos_query(taos[0], qstr); @@ -372,7 +397,7 @@ _REP: queryDB(taosRoot, "alter user user0 sysinfo 1"); fprintf(stderr, "%s:%d sleep 2 seconds to wait HB take effect\n", __func__, __LINE__); for (int i = 1; i <= 2; ++i) { - sleep(1); + taosMsleep(1000); } if(++nRep < 5) { @@ -426,7 +451,7 @@ _loop: printf("%s:%d [%d] second(s) elasped, user dropped notification received:%d, total:%d\n", __func__, __LINE__, i, nUserDropped, nConn); if (nUserDropped >= nConn) break; - sleep(1); + taosMsleep(1000); } for (int i = 0; i < nTestUsers; ++i) { From 907cbe99edcfdca2d581989ccca1442b2914cbe4 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 7 Nov 2024 14:38:51 +0800 Subject: [PATCH 17/35] enh: adjust macro for makefile_win64.mak --- tests/script/api/makefile_win64.mak | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/api/makefile_win64.mak b/tests/script/api/makefile_win64.mak index 9963ef4133..50a2447a06 100644 --- a/tests/script/api/makefile_win64.mak +++ b/tests/script/api/makefile_win64.mak @@ -2,7 +2,7 @@ TARGET = passwdTest.exe CC = cl -CFLAGS = /W4 /EHsc /I"C:\TDengine\include" /D_WINDOWS +CFLAGS = /W4 /EHsc /I"C:\TDengine\include" /DWINDOWS LDFLAGS = /link /LIBPATH:"C:\TDengine\driver" taos.lib SRCS = passwdTest.c From 21ceac423f087e7e21a867824f1855fae4984424 Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Thu, 7 Nov 2024 15:14:47 +0800 Subject: [PATCH 18/35] Update test_passwd.py to remove make operation --- tests/army/user/test_passwd.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/army/user/test_passwd.py b/tests/army/user/test_passwd.py index 62dc5782b7..08445b4523 100644 --- a/tests/army/user/test_passwd.py +++ b/tests/army/user/test_passwd.py @@ -1,3 +1,4 @@ +import os import subprocess from frame.log import * from frame.cases import * @@ -10,11 +11,11 @@ class TDTestCase(TBase): def apiPath(self): apiPath = None currentFilePath = os.path.dirname(os.path.realpath(__file__)) - if ("community/tests" in currentFilePath): - testFilePath = currentFilePath[:currentFilePath.find("community/tests")] + if (os.sep.join(["community", "tests"]) in currentFilePath): + testFilePath = currentFilePath[:currentFilePath.find(os.sep.join(["community", "tests"]))] else: - testFilePath = currentFilePath[:currentFilePath.find("TDengine/tests")] - + testFilePath = currentFilePath[:currentFilePath.find(os.sep.join(["TDengine", "tests"]))] + for root, dirs, files in os.walk(testFilePath): if ("passwdTest.c" in files): apiPath = root @@ -25,21 +26,20 @@ class TDTestCase(TBase): apiPath = self.apiPath() tdLog.info(f"api path: {apiPath}") if apiPath: - p = subprocess.Popen(f"cd {apiPath} && make", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - if 0 != p.returncode: - tdLog.info(f"Test script passwdTest.c make failed with error: {err}") test_file_cmd = os.sep.join([apiPath, "passwdTest localhost"]) + try: + p = subprocess.Popen(test_file_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + if 0 != p.returncode: + tdLog.exit("Failed to run passwd test with output: %s \n error: %s" % (out, err)) + else: + tdLog.info(out) + tdLog.success(f"{__file__} successfully executed") + except Exception as e: + tdLog.exit(f"Failed to execute {__file__} with error: {e}") else: tdLog.exit("passwdTest.c not found") - try: - p = subprocess.Popen(test_file_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - if 0 != p.returncode: - tdLog.exit("Failed to run passwd test with output: %s \n error: %s" % (out, err)) - tdLog.success(f"{__file__} successfully executed") - except Exception as e: - tdLog.exit(f"Failed to execute {__file__} with error: {e}") + tdCases.addLinux(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase()) From f97e42e957e55c1091a1a9327e7ebe3dfa7c901d Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Thu, 7 Nov 2024 17:23:52 +0800 Subject: [PATCH 19/35] Update test_passwd.py to add debug logs --- tests/army/user/test_passwd.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/army/user/test_passwd.py b/tests/army/user/test_passwd.py index 08445b4523..c205273973 100644 --- a/tests/army/user/test_passwd.py +++ b/tests/army/user/test_passwd.py @@ -25,6 +25,9 @@ class TDTestCase(TBase): def run(self): apiPath = self.apiPath() tdLog.info(f"api path: {apiPath}") + p = subprocess.Popen(f"ls {apiPath}", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + tdLog.info(f"test files: {out}") if apiPath: test_file_cmd = os.sep.join([apiPath, "passwdTest localhost"]) try: From b929a735262050d545679c1fbc6d6ab50c4c81f7 Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Fri, 8 Nov 2024 08:02:50 +0800 Subject: [PATCH 20/35] Update test_passwd.py --- tests/army/user/test_passwd.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/army/user/test_passwd.py b/tests/army/user/test_passwd.py index c205273973..dfec175824 100644 --- a/tests/army/user/test_passwd.py +++ b/tests/army/user/test_passwd.py @@ -1,4 +1,5 @@ import os +import platform import subprocess from frame.log import * from frame.cases import * @@ -25,6 +26,12 @@ class TDTestCase(TBase): def run(self): apiPath = self.apiPath() tdLog.info(f"api path: {apiPath}") + if platform.system().lower() == 'linux': + p = subprocess.Popen(f"cd {apiPath} && make", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + if 0 != p.returncode: + tdLog.exit("Test script passwdTest.c make failed") + p = subprocess.Popen(f"ls {apiPath}", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() tdLog.info(f"test files: {out}") From a5a3d1d8ad86aa0ee43f443e98ce34359bfd0afb Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Fri, 25 Oct 2024 17:42:45 +0800 Subject: [PATCH 21/35] fix same timestamp written into same tsma res ctb --- include/common/tcommon.h | 1 + include/common/tmsgdef.h | 1 + source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 1 + source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 1 + source/dnode/mnode/impl/src/mndStb.c | 74 ++++++++++++------ source/dnode/vnode/src/inc/tq.h | 2 +- source/dnode/vnode/src/sma/smaRollup.c | 2 +- source/dnode/vnode/src/tq/tqRead.c | 6 +- source/dnode/vnode/src/tq/tqUtil.c | 4 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 86 +++++++++++++++++++++ source/libs/wal/src/walRead.c | 2 +- 11 files changed, 149 insertions(+), 31 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index ea764e6760..9572bd7aad 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -155,6 +155,7 @@ typedef enum EStreamType { STREAM_MID_RETRIEVE, STREAM_PARTITION_DELETE_DATA, STREAM_GET_RESULT, + STREAM_DELETE_GROUP_DATA, } EStreamType; #pragma pack(push, 1) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index c22a3da5ad..c81d649284 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -316,6 +316,7 @@ TD_DEF_MSG_TYPE(TDMT_VND_ARB_CHECK_SYNC, "vnode-arb-check-sync", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_FETCH_TTL_EXPIRED_TBS, "vnode-fetch-ttl-expired-tbs", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TABLE_NAME, "vnode-table-name", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_DROP_TSMA_CTB, "vnode-drop-tsma-ctb", NULL, NULL) TD_CLOSE_MSG_SEG(TDMT_VND_MSG) TD_NEW_MSG_SEG(TDMT_SCH_MSG) // 3<<8 diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 0d804eadf0..4b79ecf43a 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -182,6 +182,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_TB_WITH_TSMA, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH_TTL_EXPIRED_TBS_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TABLE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TSMA_CTB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_ANAL_ALGO, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_IP_WHITE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 006f44b349..a0356a6c4d 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -1014,6 +1014,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_COMPACT_PROGRESS, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_KILL_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TABLE_NAME, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TSMA_CTB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 3725d3a3fc..a52f0656dd 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -96,6 +96,7 @@ int32_t mndInitStb(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_DROP_TB_WITH_TSMA, mndProcessDropTbWithTsma); mndSetMsgHandle(pMnode, TDMT_VND_FETCH_TTL_EXPIRED_TBS_RSP, mndProcessFetchTtlExpiredTbs); mndSetMsgHandle(pMnode, TDMT_VND_DROP_TABLE_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_DROP_TSMA_CTB_RSP, mndTransProcessRsp); // mndSetMsgHandle(pMnode, TDMT_MND_SYSTABLE_RETRIEVE, mndProcessRetrieveStbReq); // mndSetMsgHandle(pMnode, TDMT_MND_CREATE_INDEX, mndProcessCreateIndexReq); @@ -4088,7 +4089,8 @@ typedef struct SMDropTbTsmaInfos { typedef struct SMndDropTbsWithTsmaCtx { SHashObj *pTsmaMap; // SHashObj *pDbMap; // - SHashObj *pVgMap; // + SHashObj *pVgMap; // , only for non tsma result child table + SHashObj *pTsmaTbVgMap; // , only for tsma result child table SArray *pResTbNames; // SArray } SMndDropTbsWithTsmaCtx; @@ -4129,6 +4131,16 @@ static void mndDestroyDropTbsWithTsmaCtx(SMndDropTbsWithTsmaCtx *p) { } taosHashCleanup(p->pVgMap); } + + if (p->pTsmaTbVgMap) { + void *pIter = taosHashIterate(p->pTsmaTbVgMap, NULL); + while (pIter) { + SVDropTbVgReqs *pReqs = pIter; + taosArrayDestroy(pReqs->req.pArray); + pIter = taosHashIterate(p->pTsmaTbVgMap, pIter); + } + taosHashCleanup(p->pTsmaTbVgMap); + } taosMemoryFree(p); } @@ -4154,6 +4166,12 @@ static int32_t mndInitDropTbsWithTsmaCtx(SMndDropTbsWithTsmaCtx **ppCtx) { code = terrno; goto _end; } + + pCtx->pTsmaTbVgMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (!pCtx->pTsmaTbVgMap) { + code = terrno; + goto _end; + } *ppCtx = pCtx; _end: if (code) mndDestroyDropTbsWithTsmaCtx(pCtx); @@ -4192,16 +4210,38 @@ static void *mndBuildVDropTbsReq(SMnode *pMnode, const SVgroupInfo *pVgInfo, con } static int32_t mndSetDropTbsRedoActions(SMnode *pMnode, STrans *pTrans, const SVDropTbVgReqs *pVgReqs, void *pCont, - int32_t contLen) { + int32_t contLen, tmsg_t msgType) { STransAction action = {0}; action.epSet = pVgReqs->info.epSet; action.pCont = pCont; action.contLen = contLen; - action.msgType = TDMT_VND_DROP_TABLE; + action.msgType = msgType; action.acceptableCode = TSDB_CODE_TDB_TABLE_NOT_EXIST; return mndTransAppendRedoAction(pTrans, &action); } +static int32_t mndBuildDropTbRedoActions(SMnode* pMnode, STrans* pTrans, SHashObj* pVgMap, tmsg_t msgType) { + int32_t code = 0; + void* pIter = taosHashIterate(pVgMap, NULL); + while (pIter) { + const SVDropTbVgReqs *pVgReqs = pIter; + int32_t len = 0; + void *p = mndBuildVDropTbsReq(pMnode, &pVgReqs->info, &pVgReqs->req, &len); + if (!p) { + taosHashCancelIterate(pVgMap, pIter); + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + break; + } + if ((code = mndSetDropTbsRedoActions(pMnode, pTrans, pVgReqs, p, len, msgType)) != 0) { + taosHashCancelIterate(pVgMap, pIter); + break; + } + pIter = taosHashIterate(pVgMap, pIter); + } + return code; +} + static int32_t mndCreateDropTbsTxnPrepare(SRpcMsg *pRsp, SMndDropTbsWithTsmaCtx *pCtx) { int32_t code = 0; SMnode *pMnode = pRsp->info.node; @@ -4216,23 +4256,9 @@ static int32_t mndCreateDropTbsTxnPrepare(SRpcMsg *pRsp, SMndDropTbsWithTsmaCtx TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER); - void *pIter = taosHashIterate(pCtx->pVgMap, NULL); - while (pIter) { - const SVDropTbVgReqs *pVgReqs = pIter; - int32_t len = 0; - void *p = mndBuildVDropTbsReq(pMnode, &pVgReqs->info, &pVgReqs->req, &len); - if (!p) { - taosHashCancelIterate(pCtx->pVgMap, pIter); - code = TSDB_CODE_MND_RETURN_VALUE_NULL; - if (terrno != 0) code = terrno; - goto _OVER; - } - if ((code = mndSetDropTbsRedoActions(pMnode, pTrans, pVgReqs, p, len)) != 0) { - taosHashCancelIterate(pCtx->pVgMap, pIter); - goto _OVER; - } - pIter = taosHashIterate(pCtx->pVgMap, pIter); - } + if ((code = mndBuildDropTbRedoActions(pMnode, pTrans, pCtx->pVgMap, TDMT_VND_DROP_TABLE)) != 0) goto _OVER; + if ((code = mndBuildDropTbRedoActions(pMnode, pTrans, pCtx->pTsmaTbVgMap, TDMT_VND_DROP_TSMA_CTB)) != 0) goto _OVER; + if ((code = mndBuildDropTbRedoActions(pMnode, pTrans, pCtx->pTsmaTbVgMap, TDMT_VND_DROP_TABLE)) != 0) goto _OVER; if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER; _OVER: @@ -4260,7 +4286,8 @@ static int32_t mndProcessDropTbWithTsma(SRpcMsg *pReq) { code = mndDropTbAddTsmaResTbsForSingleVg(pMnode, pCtx, pReq->pTbs, pReq->vgInfo.vgId); if (code) goto _OVER; } - if (mndCreateDropTbsTxnPrepare(pReq, pCtx) == 0) { + code = mndCreateDropTbsTxnPrepare(pReq, pCtx); + if (code == 0) { code = TSDB_CODE_ACTION_IN_PROGRESS; } _OVER: @@ -4445,7 +4472,7 @@ static int32_t mndDropTbAddTsmaResTbsForSingleVg(SMnode *pMnode, SMndDropTbsWith code = terrno; goto _end; } - TAOS_CHECK_GOTO(mndDropTbAdd(pMnode, pCtx->pVgMap, pVgInfo, p, pInfo->suid, true), NULL, _end); + TAOS_CHECK_GOTO(mndDropTbAdd(pMnode, pCtx->pTsmaTbVgMap, pVgInfo, p, pInfo->suid, true), NULL, _end); } } _end: @@ -4476,7 +4503,8 @@ static int32_t mndProcessFetchTtlExpiredTbs(SRpcMsg *pRsp) { code = mndDropTbAddTsmaResTbsForSingleVg(pMnode, pCtx, rsp.pExpiredTbs, rsp.vgId); if (code) goto _end; - if (mndCreateDropTbsTxnPrepare(pRsp, pCtx) == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; + code = mndCreateDropTbsTxnPrepare(pRsp, pCtx); + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _end: if (pCtx) mndDestroyDropTbsWithTsmaCtx(pCtx); tDecoderClear(&decoder); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 653b47ff14..bdb94d4a6e 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -146,7 +146,7 @@ int32_t tqBuildFName(char** data, const char* path, char* name); int32_t tqOffsetRestoreFromFile(STQ* pTq, char* name); // tq util -int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type); +int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type, EStreamType blockType); int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg); int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, int32_t type, int64_t sver, int64_t ever); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 80c04a3276..bbc58004d9 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -1551,7 +1551,7 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA _resume_delete: version = RSMA_EXEC_MSG_VER(msg); if ((code = tqExtractDelDataBlock(RSMA_EXEC_MSG_BODY(msg), RSMA_EXEC_MSG_LEN(msg), version, - &packData.pDataBlock, 1))) { + &packData.pDataBlock, 1, STREAM_DELETE_DATA))) { taosFreeQitem(msg); TAOS_CHECK_EXIT(code); } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 95955e579f..c8c34b10a4 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -363,11 +363,11 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con tqError("%s failed to create data submit for stream since out of memory", id); return code; } - } else if (pCont->msgType == TDMT_VND_DELETE) { + } else if (pCont->msgType == TDMT_VND_DELETE || pCont->msgType == TDMT_VND_DROP_TSMA_CTB) { void* pBody = POINTER_SHIFT(pCont->body, sizeof(SMsgHead)); int32_t len = pCont->bodyLen - sizeof(SMsgHead); - - code = tqExtractDelDataBlock(pBody, len, ver, (void**)pItem, 0); + EStreamType blockType = pCont->msgType == TDMT_VND_DELETE ? STREAM_DELETE_DATA : STREAM_DELETE_GROUP_DATA; + code = tqExtractDelDataBlock(pBody, len, ver, (void**)pItem, 0, blockType); if (code == TSDB_CODE_SUCCESS) { if (*pItem == NULL) { tqDebug("s-task:%s empty delete msg, discard it, len:%d, ver:%" PRId64, id, len, ver); diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index e066938fc0..54b063d692 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -572,7 +572,7 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* return 0; } -int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type) { +int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type, EStreamType blockType) { int32_t code = 0; int32_t line = 0; SDecoder* pCoder = &(SDecoder){0}; @@ -593,7 +593,7 @@ int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void* } SSDataBlock* pDelBlock = NULL; - code = createSpecialDataBlock(STREAM_DELETE_DATA, &pDelBlock); + code = createSpecialDataBlock(blockType, &pDelBlock); TSDB_CHECK_CODE(code, line, END); code = blockDataEnsureCapacity(pDelBlock, numOfTables); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index dd13c975cf..1ea2aac809 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -50,6 +50,8 @@ static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessConfigChangeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessArbCheckSyncReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDropTSmaCtbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, + SRpcMsg *pOriginRpc); static int32_t vnodePreCheckAssignedLogSyncd(SVnode *pVnode, char *member0Token, char *member1Token); static int32_t vnodeCheckAssignedLogSyncd(SVnode *pVnode, char *member0Token, char *member1Token); @@ -481,6 +483,75 @@ static int32_t vnodePreProcessArbCheckSyncMsg(SVnode *pVnode, SRpcMsg *pMsg) { return code; } +static int32_t vnodePreProcessDropTSmaCtbMsg(SVnode *pVnode, SRpcMsg *pMsg) { + SVDropTbBatchReq dropReq = {0}; + int32_t code = 0; + int32_t lino = 0; + SDecoder dc = {0}; + SEncoder ec = {0}; + int32_t nTbs = 0; + SDeleteRes res = {0}; + int32_t size = 0; + uint8_t *pCont = NULL; + tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead)); + if (tDecodeSVDropTbBatchReq(&dc, &dropReq) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + nTbs = dropReq.nReqs; + res.skey = INT64_MIN; + res.ekey = INT64_MAX; + res.affectedRows = 1; + res.uidList = taosArrayInit(nTbs, sizeof(tb_uid_t)); + if (!res.uidList) { + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); + } + + vDebug("vnode preprocess drop tsma ctb, vgId:%d tb num: %d", TD_VID(pVnode), nTbs); + for (int32_t i = 0; i < nTbs; ++i) { + SVDeleteRsp rsp = {.affectedRows = 1}; + tb_uid_t uid = metaGetTableEntryUidByName(pVnode->pMeta, dropReq.pReqs[i].name); + if (uid == 0) { + vWarn("vgId:%d, drop tsma ctb:%s not found", TD_VID(pVnode), dropReq.pReqs[i].name); + continue; + } + if (NULL == taosArrayPush(res.uidList, &uid)) { + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); + } + } + + tEncodeSize(tEncodeDeleteRes, &res, size, code); + pCont = rpcMallocCont(size + sizeof(SMsgHead)); + if (!pCont) { + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); + } + ((SMsgHead *)pCont)->contLen = size + sizeof(SMsgHead); + ((SMsgHead *)pCont)->vgId = TD_VID(pVnode); + + tEncoderInit(&ec, pCont + sizeof(SMsgHead), size); + code = tEncodeDeleteRes(&ec, &res); + tEncoderClear(&ec); + if (code != 0) { + vError("vgId:%d %s failed to encode delete response", TD_VID(pVnode), __func__); + TSDB_CHECK_CODE(code, lino, _exit); + } + rpcFreeCont(pMsg->pCont); + pMsg->pCont = pCont; + pCont = NULL; + pMsg->contLen = size + sizeof(SMsgHead); + +_exit: + if (res.uidList) { + taosArrayDestroy(res.uidList); + } + tDecoderClear(&dc); + rpcFreeCont(pCont); + return code; +} + int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t code = 0; @@ -507,6 +578,9 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { case TDMT_VND_ARB_CHECK_SYNC: { code = vnodePreProcessArbCheckSyncMsg(pVnode, pMsg); } break; + case TDMT_VND_DROP_TSMA_CTB: { + code = vnodePreProcessDropTSmaCtbMsg(pVnode, pMsg); + } break; default: break; } @@ -711,6 +785,11 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg case TDMT_VND_ARB_CHECK_SYNC: vnodeProcessArbCheckSyncReq(pVnode, pReq, len, pRsp); break; + case TDMT_VND_DROP_TSMA_CTB: + if (vnodeProcessDropTSmaCtbReq(pVnode, ver, pReq, len, pRsp, pMsg) < 0) { + goto _err; + } + break; default: vError("vgId:%d, unprocessed msg, %d", TD_VID(pVnode), pMsg->msgType); return TSDB_CODE_INVALID_MSG; @@ -2512,3 +2591,10 @@ _OVER: int32_t vnodeAsyncCompact(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { return 0; } int32_t tsdbAsyncCompact(STsdb *tsdb, const STimeWindow *tw, bool sync) { return 0; } #endif + +static int32_t vnodeProcessDropTSmaCtbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, + SRpcMsg *pOriginalMsg) { + pRsp->msgType = TDMT_VND_DROP_TSMA_CTB_RSP; + pRsp->code = TSDB_CODE_SUCCESS; + return pRsp->code; +} diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 610adfb0e1..9a3ea34eff 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -87,7 +87,7 @@ int32_t walNextValidMsg(SWalReader *pReader) { int32_t type = pReader->pHead->head.msgType; if (type == TDMT_VND_SUBMIT || ((type == TDMT_VND_DELETE) && (pReader->cond.deleteMsg == 1)) || - (IS_META_MSG(type) && pReader->cond.scanMeta)) { + (type == TDMT_VND_DROP_TSMA_CTB) || (IS_META_MSG(type) && pReader->cond.scanMeta)) { TAOS_RETURN(walFetchBody(pReader)); } else { TAOS_CHECK_RETURN(walSkipFetchBody(pReader)); From f452ca47fe4919e2aaed6b4dc1c2df7569d55ebe Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Sun, 27 Oct 2024 15:42:14 +0800 Subject: [PATCH 22/35] delete window state when drop table --- include/libs/executor/storageapi.h | 1 + include/libs/stream/streamState.h | 1 + source/dnode/snode/src/snodeInitApi.c | 1 + source/dnode/vnode/src/vnd/vnodeInitApi.c | 1 + source/libs/executor/src/scanoperator.c | 26 +++++++++++++++++++ source/libs/stream/inc/streamBackendRocksdb.h | 1 + source/libs/stream/src/streamBackendRocksdb.c | 6 +++++ source/libs/stream/src/streamState.c | 8 ++++++ 8 files changed, 45 insertions(+) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index db0d6339c8..feb7bcc25e 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -336,6 +336,7 @@ typedef struct SStateStore { int32_t (*streamStatePutParName)(SStreamState* pState, int64_t groupId, const char* tbname); int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache, int32_t* pWinCode); + int32_t (*streamStateDeleteParName)(SStreamState* pState, int64_t groupId); int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode); diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index a50451c3eb..2179547352 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -116,6 +116,7 @@ void streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur); int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char* tbname); int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache, int32_t* pWinCode); +int32_t streamStateDeleteParName(SStreamState* pState, int64_t groupId); // group id int32_t streamStateGroupPut(SStreamState* pState, int64_t groupId, void* value, int32_t vLen); diff --git a/source/dnode/snode/src/snodeInitApi.c b/source/dnode/snode/src/snodeInitApi.c index 680a2fd83c..4fe4333534 100644 --- a/source/dnode/snode/src/snodeInitApi.c +++ b/source/dnode/snode/src/snodeInitApi.c @@ -31,6 +31,7 @@ void initStateStoreAPI(SStateStore* pStore) { pStore->streamStatePutParName = streamStatePutParName; pStore->streamStateGetParName = streamStateGetParName; + pStore->streamStateDeleteParName = streamStateDeleteParName; pStore->streamStateAddIfNotExist = streamStateAddIfNotExist; pStore->streamStateReleaseBuf = streamStateReleaseBuf; diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index d688d1323d..0ac0ee1b8f 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -147,6 +147,7 @@ void initStateStoreAPI(SStateStore* pStore) { pStore->streamStatePutParName = streamStatePutParName; pStore->streamStateGetParName = streamStateGetParName; + pStore->streamStateDeleteParName = streamStateDeleteParName; pStore->streamStateAddIfNotExist = streamStateAddIfNotExist; pStore->streamStateReleaseBuf = streamStateReleaseBuf; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 5b5d5c5d11..8c04492162 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3535,6 +3535,23 @@ static int32_t copyGetResultBlock(SSDataBlock* dest, TSKEY start, TSKEY end) { return appendDataToSpecialBlock(dest, &start, &end, NULL, NULL, NULL); } +static int32_t deletePartName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + for (int32_t i = 0; i < pBlock->info.rows; i++) { + SColumnInfoData* pGpIdCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); + int64_t* gpIdCol = (int64_t*)pGpIdCol->pData; + code = pInfo->stateStore.streamStateDeleteParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, gpIdCol[i]); + QUERY_CHECK_CODE(code, lino, _end); + } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; +} + static int32_t doStreamScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { // NOTE: this operator does never check if current status is done or not int32_t code = TSDB_CODE_SUCCESS; @@ -3774,6 +3791,15 @@ FETCH_NEXT_BLOCK: prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex, NULL); pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; } break; + case STREAM_DELETE_GROUP_DATA: { + printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "delete group recv", + GET_TASKID(pTaskInfo)); + code = setBlockGroupIdByUid(pInfo, pBlock); + QUERY_CHECK_CODE(code, lino, _end); + + deletePartName(pInfo, pBlock); + pBlock->info.type = STREAM_DELETE_DATA; + } break; case STREAM_CHECKPOINT: { qError("stream check point error. msg type: STREAM_INPUT__DATA_BLOCK"); } break; diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index d313acc61d..6a10b21c53 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -223,6 +223,7 @@ int32_t streamStateParTagGetKVByCur_rocksdb(SStreamStateCur* pCur, int64_t* pGro // parname cf int32_t streamStatePutParName_rocksdb(SStreamState* pState, int64_t groupId, const char tbname[TSDB_TABLE_NAME_LEN]); int32_t streamStateGetParName_rocksdb(SStreamState* pState, int64_t groupId, void** pVal); +int32_t streamStateDeleteParName_rocksdb(SStreamState* pState, int64_t groupId); void streamStateDestroy_rocksdb(SStreamState* pState, bool remove); diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 09f4e95376..65746b3100 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -4432,6 +4432,12 @@ int32_t streamStateGetParName_rocksdb(SStreamState* pState, int64_t groupId, voi return code; } +int32_t streamStateDeleteParName_rocksdb(SStreamState* pState, int64_t groupId) { + int code = 0; + STREAM_STATE_DEL_ROCKSDB(pState, "parname", &groupId); + return code; +} + int32_t streamDefaultPut_rocksdb(SStreamState* pState, const void* key, void* pVal, int32_t pVLen) { int code = 0; STREAM_STATE_PUT_ROCKSDB(pState, "default", key, pVal, pVLen); diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 794fc346bf..ff52e1cb54 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -525,6 +525,14 @@ _end: return code; } +int32_t streamStateDeleteParName(SStreamState* pState, int64_t groupId) { + int32_t code = tSimpleHashRemove(pState->parNameMap, &groupId, sizeof(int64_t)); + qTrace("%s at line %d res %d", __func__, __LINE__, code); + code = streamStateDeleteParName_rocksdb(pState, groupId); + qTrace("%s at line %d res %d", __func__, __LINE__, code); + return TSDB_CODE_SUCCESS; +} + void streamStateDestroy(SStreamState* pState, bool remove) { streamFileStateDestroy(pState->pFileState); // streamStateDestroy_rocksdb(pState, remove); From 7f501130a26fafc121de70315ed34d27668211e3 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Sun, 27 Oct 2024 18:27:40 +0800 Subject: [PATCH 23/35] client add query fisrt/last/count for drop table --- source/libs/parser/inc/parInt.h | 2 + source/libs/parser/src/parTranslater.c | 72 +++++++++++++++++++++++++- source/libs/parser/src/parser.c | 4 ++ 3 files changed, 77 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/inc/parInt.h b/source/libs/parser/inc/parInt.h index 5999ada70f..f69896dce9 100644 --- a/source/libs/parser/inc/parInt.h +++ b/source/libs/parser/inc/parInt.h @@ -45,6 +45,8 @@ int32_t buildQueryAfterParse(SQuery** pQuery, SNode* pRootNode, int16_t placehol int32_t translateTable(STranslateContext* pCxt, SNode** pTable, SNode* pJoinParent); int32_t getMetaDataFromHash(const char* pKey, int32_t len, SHashObj* pHash, void** pOutput); void tfreeSParseQueryRes(void* p); +int32_t translatePostDropCTbWithTsma(SParseContext* pCxt, SQuery* pQuery, SSDataBlock* pBlock); + #ifdef TD_ENTERPRISE int32_t translateView(STranslateContext* pCxt, SNode** pTable, SName* pName); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 99c03c412c..01ed634110 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -9238,10 +9238,76 @@ static int32_t doTranslateDropSuperTable(STranslateContext* pCxt, const SName* p return code; } +static int32_t createFunctionForDropCtbWithTSMA(const char* pFuncName, SFunctionNode** ppFunc) { + int32_t code = TSDB_CODE_SUCCESS; + SFunctionNode* pFunc = NULL; + code = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&pFunc); + SColumnNode* pCol = NULL; + if (TSDB_CODE_SUCCESS == code) { + strcpy(pFunc->functionName, pFuncName); + code = nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol); + } + if (TSDB_CODE_SUCCESS == code) { + strcpy(((SColumnNode*)pCol)->colName, ROWTS_PSEUDO_COLUMN_NAME); + pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + pCol->isPrimTs = true; + code = nodesListMakeStrictAppend(&pFunc->pParameterList, (SNode*)pCol); + } + if (TSDB_CODE_SUCCESS == code) { + *ppFunc = pFunc; + } else { + nodesDestroyNode((SNode*)pFunc); + } + return code; +} + +static int32_t translateDropCtbWithTsma(STranslateContext* pCxt, SDropTableStmt* pStmt) { + if (!pStmt->withTsma || LIST_LENGTH(pStmt->pTables) == 0) return TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + SNode* pPrevQuery = NULL; + SNodeList* pProjectionList = NULL; + if (LIST_LENGTH(pStmt->pTables) > 1) return TSDB_CODE_FAILED; + SDropTableClause* pClause = (SDropTableClause*)nodesListGetNode(pStmt->pTables, 0); + + // create select query stmt + code = nodesMakeList(&pProjectionList); + SFunctionNode* pFunc = NULL; + if (TSDB_CODE_SUCCESS == code) { + code = createFunctionForDropCtbWithTSMA("count", &pFunc); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pProjectionList, (SNode*)pFunc); + } + } + if (TSDB_CODE_SUCCESS == code) { + code = createFunctionForDropCtbWithTSMA("first", (SFunctionNode**)&pFunc); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pProjectionList, (SNode*)pFunc); + } + } + + if (TSDB_CODE_SUCCESS == code) { + code = createFunctionForDropCtbWithTSMA("last", (SFunctionNode**)&pFunc); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pProjectionList, (SNode*)pFunc); + } + } + if (TSDB_CODE_SUCCESS == code) { + code = createSimpleSelectStmtFromProjList(pClause->dbName, pClause->tableName, pProjectionList, + (SSelectStmt**)&pPrevQuery); + } + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyList(pProjectionList); + } else { + TSWAP(pCxt->pPrevRoot, pPrevQuery); + } + return code; +} + static int32_t translateDropTable(STranslateContext* pCxt, SDropTableStmt* pStmt) { + if (pStmt->withTsma) return translateDropCtbWithTsma(pCxt, pStmt); + SDropTableClause* pClause = (SDropTableClause*)nodesListGetNode(pStmt->pTables, 0); SName tableName = {0}; - if (pStmt->withTsma) return TSDB_CODE_SUCCESS; toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &tableName); return doTranslateDropSuperTable(pCxt, &tableName, pClause->ignoreNotExists); } @@ -12747,6 +12813,10 @@ int32_t translatePostCreateTSMA(SParseContext* pParseCxt, SQuery* pQuery, SSData return code; } +int32_t translatePostDropCTbWithTsma(SParseContext* pCxt, SQuery* pQuery, SSDataBlock* pBlock) { + +} + static int32_t translateDropTSMA(STranslateContext* pCxt, SDropTSMAStmt* pStmt) { int32_t code = TSDB_CODE_SUCCESS; SMDropSmaReq dropReq = {0}; diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 8ac1acb1a2..0c27e5a603 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -324,6 +324,10 @@ int32_t qContinueParsePostQuery(SParseContext* pCxt, SQuery* pQuery, SSDataBlock code = translatePostCreateTSMA(pCxt, pQuery, pBlock); break; } + case QUERY_NODE_DROP_TABLE_STMT: { + translatePostDropCTbWithTsma(pCxt, pQuery, pBlock); + break; + } default: break; } From 2b924c68fd95adf583aec76d8acf78b0a37968c7 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Mon, 28 Oct 2024 18:08:18 +0800 Subject: [PATCH 24/35] ignore delete result --- source/libs/executor/src/scanoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8c04492162..12ec7d4201 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3798,7 +3798,7 @@ FETCH_NEXT_BLOCK: QUERY_CHECK_CODE(code, lino, _end); deletePartName(pInfo, pBlock); - pBlock->info.type = STREAM_DELETE_DATA; + goto FETCH_NEXT_BLOCK; } break; case STREAM_CHECKPOINT: { qError("stream check point error. msg type: STREAM_INPUT__DATA_BLOCK"); From 5955d121b390bb28c9c458955575ea683f2ccf04 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Mon, 28 Oct 2024 18:07:46 +0800 Subject: [PATCH 25/35] revert first/last fetch --- source/libs/parser/inc/parInt.h | 2 - source/libs/parser/src/parTranslater.c | 72 +------------------------- source/libs/parser/src/parser.c | 4 -- 3 files changed, 1 insertion(+), 77 deletions(-) diff --git a/source/libs/parser/inc/parInt.h b/source/libs/parser/inc/parInt.h index f69896dce9..5999ada70f 100644 --- a/source/libs/parser/inc/parInt.h +++ b/source/libs/parser/inc/parInt.h @@ -45,8 +45,6 @@ int32_t buildQueryAfterParse(SQuery** pQuery, SNode* pRootNode, int16_t placehol int32_t translateTable(STranslateContext* pCxt, SNode** pTable, SNode* pJoinParent); int32_t getMetaDataFromHash(const char* pKey, int32_t len, SHashObj* pHash, void** pOutput); void tfreeSParseQueryRes(void* p); -int32_t translatePostDropCTbWithTsma(SParseContext* pCxt, SQuery* pQuery, SSDataBlock* pBlock); - #ifdef TD_ENTERPRISE int32_t translateView(STranslateContext* pCxt, SNode** pTable, SName* pName); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 01ed634110..99c03c412c 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -9238,76 +9238,10 @@ static int32_t doTranslateDropSuperTable(STranslateContext* pCxt, const SName* p return code; } -static int32_t createFunctionForDropCtbWithTSMA(const char* pFuncName, SFunctionNode** ppFunc) { - int32_t code = TSDB_CODE_SUCCESS; - SFunctionNode* pFunc = NULL; - code = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&pFunc); - SColumnNode* pCol = NULL; - if (TSDB_CODE_SUCCESS == code) { - strcpy(pFunc->functionName, pFuncName); - code = nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol); - } - if (TSDB_CODE_SUCCESS == code) { - strcpy(((SColumnNode*)pCol)->colName, ROWTS_PSEUDO_COLUMN_NAME); - pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; - pCol->isPrimTs = true; - code = nodesListMakeStrictAppend(&pFunc->pParameterList, (SNode*)pCol); - } - if (TSDB_CODE_SUCCESS == code) { - *ppFunc = pFunc; - } else { - nodesDestroyNode((SNode*)pFunc); - } - return code; -} - -static int32_t translateDropCtbWithTsma(STranslateContext* pCxt, SDropTableStmt* pStmt) { - if (!pStmt->withTsma || LIST_LENGTH(pStmt->pTables) == 0) return TSDB_CODE_SUCCESS; - int32_t code = TSDB_CODE_SUCCESS; - SNode* pPrevQuery = NULL; - SNodeList* pProjectionList = NULL; - if (LIST_LENGTH(pStmt->pTables) > 1) return TSDB_CODE_FAILED; - SDropTableClause* pClause = (SDropTableClause*)nodesListGetNode(pStmt->pTables, 0); - - // create select query stmt - code = nodesMakeList(&pProjectionList); - SFunctionNode* pFunc = NULL; - if (TSDB_CODE_SUCCESS == code) { - code = createFunctionForDropCtbWithTSMA("count", &pFunc); - if (TSDB_CODE_SUCCESS == code) { - code = nodesListMakeStrictAppend(&pProjectionList, (SNode*)pFunc); - } - } - if (TSDB_CODE_SUCCESS == code) { - code = createFunctionForDropCtbWithTSMA("first", (SFunctionNode**)&pFunc); - if (TSDB_CODE_SUCCESS == code) { - code = nodesListMakeStrictAppend(&pProjectionList, (SNode*)pFunc); - } - } - - if (TSDB_CODE_SUCCESS == code) { - code = createFunctionForDropCtbWithTSMA("last", (SFunctionNode**)&pFunc); - if (TSDB_CODE_SUCCESS == code) { - code = nodesListMakeStrictAppend(&pProjectionList, (SNode*)pFunc); - } - } - if (TSDB_CODE_SUCCESS == code) { - code = createSimpleSelectStmtFromProjList(pClause->dbName, pClause->tableName, pProjectionList, - (SSelectStmt**)&pPrevQuery); - } - if (TSDB_CODE_SUCCESS != code) { - nodesDestroyList(pProjectionList); - } else { - TSWAP(pCxt->pPrevRoot, pPrevQuery); - } - return code; -} - static int32_t translateDropTable(STranslateContext* pCxt, SDropTableStmt* pStmt) { - if (pStmt->withTsma) return translateDropCtbWithTsma(pCxt, pStmt); - SDropTableClause* pClause = (SDropTableClause*)nodesListGetNode(pStmt->pTables, 0); SName tableName = {0}; + if (pStmt->withTsma) return TSDB_CODE_SUCCESS; toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &tableName); return doTranslateDropSuperTable(pCxt, &tableName, pClause->ignoreNotExists); } @@ -12813,10 +12747,6 @@ int32_t translatePostCreateTSMA(SParseContext* pParseCxt, SQuery* pQuery, SSData return code; } -int32_t translatePostDropCTbWithTsma(SParseContext* pCxt, SQuery* pQuery, SSDataBlock* pBlock) { - -} - static int32_t translateDropTSMA(STranslateContext* pCxt, SDropTSMAStmt* pStmt) { int32_t code = TSDB_CODE_SUCCESS; SMDropSmaReq dropReq = {0}; diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 0c27e5a603..8ac1acb1a2 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -324,10 +324,6 @@ int32_t qContinueParsePostQuery(SParseContext* pCxt, SQuery* pQuery, SSDataBlock code = translatePostCreateTSMA(pCxt, pQuery, pBlock); break; } - case QUERY_NODE_DROP_TABLE_STMT: { - translatePostDropCTbWithTsma(pCxt, pQuery, pBlock); - break; - } default: break; } From 9449c8bdc5a4bf30890255781fd79942be71bc25 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Mon, 28 Oct 2024 18:10:32 +0800 Subject: [PATCH 26/35] adj log --- source/libs/stream/src/streamState.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index ff52e1cb54..26c97cbb6e 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -527,9 +527,9 @@ _end: int32_t streamStateDeleteParName(SStreamState* pState, int64_t groupId) { int32_t code = tSimpleHashRemove(pState->parNameMap, &groupId, sizeof(int64_t)); - qTrace("%s at line %d res %d", __func__, __LINE__, code); + qTrace("catche %s at line %d res %d", __func__, __LINE__, code); code = streamStateDeleteParName_rocksdb(pState, groupId); - qTrace("%s at line %d res %d", __func__, __LINE__, code); + qTrace("disk %s at line %d res %d", __func__, __LINE__, code); return TSDB_CODE_SUCCESS; } From 413cf3e504c6e1d22d3bc3982df47056ea11c4f5 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 29 Oct 2024 11:30:04 +0800 Subject: [PATCH 27/35] sink submit data one-by-one --- source/dnode/mnode/impl/src/mndStb.c | 2 +- source/dnode/vnode/src/tq/tqSink.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index a52f0656dd..6efc58119b 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -4256,8 +4256,8 @@ static int32_t mndCreateDropTbsTxnPrepare(SRpcMsg *pRsp, SMndDropTbsWithTsmaCtx TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER); + if ((code = mndBuildDropTbRedoActions(pMnode, pTrans, pCtx->pVgMap, TDMT_VND_DROP_TSMA_CTB)) != 0) goto _OVER; if ((code = mndBuildDropTbRedoActions(pMnode, pTrans, pCtx->pVgMap, TDMT_VND_DROP_TABLE)) != 0) goto _OVER; - if ((code = mndBuildDropTbRedoActions(pMnode, pTrans, pCtx->pTsmaTbVgMap, TDMT_VND_DROP_TSMA_CTB)) != 0) goto _OVER; if ((code = mndBuildDropTbRedoActions(pMnode, pTrans, pCtx->pTsmaTbVgMap, TDMT_VND_DROP_TABLE)) != 0) goto _OVER; if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index be41f7e99e..e972c928f0 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -1032,7 +1032,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { } bool onlySubmitData = hasOnlySubmitData(pBlocks, numOfBlocks); - if (!onlySubmitData) { + if (!onlySubmitData || pTask->subtableWithoutMd5 == 1) { tqDebug("vgId:%d, s-task:%s write %d stream resBlock(s) into table, has delete block, submit one-by-one", vgId, id, numOfBlocks); From fc39ea70ac8a94bec32f13cc6a2ec09939c02b6c Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 29 Oct 2024 17:43:43 +0800 Subject: [PATCH 28/35] fix tsma stream same tbname with same group id --- source/libs/executor/src/executorInt.c | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 1b823bf69d..af8e01be5e 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -1083,18 +1083,13 @@ void cleanupBasicInfo(SOptrBasicInfo* pInfo) { bool groupbyTbname(SNodeList* pGroupList) { bool bytbname = false; - if (LIST_LENGTH(pGroupList) == 1) { - SNode* p = nodesListGetNode(pGroupList, 0); - if (!p) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); - return false; - } - if (p->type == QUERY_NODE_FUNCTION) { - // partition by tbname/group by tbname - bytbname = (strcmp(((struct SFunctionNode*)p)->functionName, "tbname") == 0); + SNode*pNode = NULL; + FOREACH(pNode, pGroupList) { + if (pNode->type == QUERY_NODE_FUNCTION) { + bytbname = (strcmp(((struct SFunctionNode*)pNode)->functionName, "tbname") == 0); + break; } } - return bytbname; } From f2860b766a4c42677c0b28e620cf1b63936cc0d9 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Wed, 30 Oct 2024 11:14:27 +0800 Subject: [PATCH 29/35] add ret check for deletePartName --- source/libs/executor/src/scanoperator.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 12ec7d4201..0cdb0e7954 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3797,7 +3797,8 @@ FETCH_NEXT_BLOCK: code = setBlockGroupIdByUid(pInfo, pBlock); QUERY_CHECK_CODE(code, lino, _end); - deletePartName(pInfo, pBlock); + code = deletePartName(pInfo, pBlock); + QUERY_CHECK_CODE(code, lino, _end); goto FETCH_NEXT_BLOCK; } break; case STREAM_CHECKPOINT: { From 65dffbda0c54ff46b5022148c3e65ce60f50c00b Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Fri, 1 Nov 2024 18:47:41 +0800 Subject: [PATCH 30/35] fix tsma drop ctb --- include/common/tcommon.h | 2 + include/common/tmsg.h | 1 + include/libs/wal/wal.h | 1 + source/common/src/tmsg.c | 2 + source/dnode/mnode/impl/src/mndStb.c | 6 +- source/dnode/vnode/src/inc/tq.h | 1 + source/dnode/vnode/src/meta/metaTable.c | 1 + source/dnode/vnode/src/tq/tq.c | 3 +- source/dnode/vnode/src/tq/tqRead.c | 14 ++ source/dnode/vnode/src/tq/tqSink.c | 134 +++++++++++++++++- source/dnode/vnode/src/tq/tqUtil.c | 51 +++++++ source/dnode/vnode/src/vnd/vnodeSvr.c | 62 +++++++- source/libs/executor/src/scanoperator.c | 34 ++++- .../executor/src/streamtimewindowoperator.c | 2 +- source/libs/stream/src/streamState.c | 3 + source/libs/wal/src/walRead.c | 2 + tests/system-test/2-query/tsma.py | 26 +++- 17 files changed, 320 insertions(+), 25 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 9572bd7aad..8a8948fb17 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -156,6 +156,7 @@ typedef enum EStreamType { STREAM_PARTITION_DELETE_DATA, STREAM_GET_RESULT, STREAM_DELETE_GROUP_DATA, + STREAM_DROP_CHILD_TABLE, } EStreamType; #pragma pack(push, 1) @@ -402,6 +403,7 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol); #define TSMA_RES_STB_EXTRA_COLUMN_NUM 4 // 3 columns: _wstart, _wend, _wduration, 1 tag: tbname static inline bool isTsmaResSTb(const char* stbName) { + return false; const char* pos = strstr(stbName, TSMA_RES_STB_POSTFIX); if (pos && strlen(stbName) == (pos - stbName) + strlen(TSMA_RES_STB_POSTFIX)) { return true; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 7ff70b243a..3c7d715aa8 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3220,6 +3220,7 @@ int tDecodeSVCreateTbBatchRsp(SDecoder* pCoder, SVCreateTbBatchRsp* pRsp); typedef struct { char* name; uint64_t suid; // for tmq in wal format + int64_t uid; int8_t igNotExists; } SVDropTbReq; diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index f95b3f20ca..999adc2eff 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -138,6 +138,7 @@ typedef struct { int8_t scanMeta; int8_t deleteMsg; int8_t enableRef; + int8_t scanDropCtb; } SWalFilterCond; // todo hide this struct diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 6d1699b911..7e79ef48b8 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -10277,6 +10277,7 @@ static int32_t tEncodeSVDropTbReq(SEncoder *pCoder, const SVDropTbReq *pReq) { TAOS_CHECK_RETURN(tStartEncode(pCoder)); TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pReq->name)); TAOS_CHECK_RETURN(tEncodeU64(pCoder, pReq->suid)); + TAOS_CHECK_RETURN(tEncodeI64(pCoder, pReq->uid)); TAOS_CHECK_RETURN(tEncodeI8(pCoder, pReq->igNotExists)); tEndEncode(pCoder); @@ -10287,6 +10288,7 @@ static int32_t tDecodeSVDropTbReq(SDecoder *pCoder, SVDropTbReq *pReq) { TAOS_CHECK_RETURN(tStartDecode(pCoder)); TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pReq->name)); TAOS_CHECK_RETURN(tDecodeU64(pCoder, &pReq->suid)); + TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pReq->uid)); TAOS_CHECK_RETURN(tDecodeI8(pCoder, &pReq->igNotExists)); tEndDecode(pCoder); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 6efc58119b..a1044a9f86 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -4256,9 +4256,9 @@ static int32_t mndCreateDropTbsTxnPrepare(SRpcMsg *pRsp, SMndDropTbsWithTsmaCtx TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER); - if ((code = mndBuildDropTbRedoActions(pMnode, pTrans, pCtx->pVgMap, TDMT_VND_DROP_TSMA_CTB)) != 0) goto _OVER; + //if ((code = mndBuildDropTbRedoActions(pMnode, pTrans, pCtx->pVgMap, TDMT_VND_DROP_TSMA_CTB)) != 0) goto _OVER; if ((code = mndBuildDropTbRedoActions(pMnode, pTrans, pCtx->pVgMap, TDMT_VND_DROP_TABLE)) != 0) goto _OVER; - if ((code = mndBuildDropTbRedoActions(pMnode, pTrans, pCtx->pTsmaTbVgMap, TDMT_VND_DROP_TABLE)) != 0) goto _OVER; + //if ((code = mndBuildDropTbRedoActions(pMnode, pTrans, pCtx->pTsmaTbVgMap, TDMT_VND_DROP_TABLE)) != 0) goto _OVER; if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER; _OVER: @@ -4298,7 +4298,7 @@ _OVER: static int32_t mndDropTbAdd(SMnode *pMnode, SHashObj *pVgHashMap, const SVgroupInfo *pVgInfo, char *name, tb_uid_t suid, bool ignoreNotExists) { - SVDropTbReq req = {.name = name, .suid = suid, .igNotExists = ignoreNotExists}; + SVDropTbReq req = {.name = name, .suid = suid, .igNotExists = ignoreNotExists, .uid = 0}; SVDropTbVgReqs *pReqs = taosHashGet(pVgHashMap, &pVgInfo->vgId, sizeof(pVgInfo->vgId)); SVDropTbVgReqs reqs = {0}; diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index bdb94d4a6e..3c40100f9d 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -158,6 +158,7 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c int32_t buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, SSDataBlock* pDataBlock, SArray* pTagArray, bool newSubTableRule, SVCreateTbReq** pReq); +int32_t tqExtractDropCtbDataBlock(const void* data, int32_t len, int64_t ver, void** pRefBlock, int32_t type); #define TQ_ERR_GO_TO_END(c) \ do { \ diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 5c3516a962..93324d6eb4 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -1233,6 +1233,7 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi metaWLock(pMeta); rc = metaDropTableByUid(pMeta, uid, &type, &suid, &sysTbl); + metaInfo("wjm meta drop table by uid: %"PRId64, uid); metaULock(pMeta); if (rc < 0) goto _exit; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index bd78f62cae..6195899566 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -758,7 +758,8 @@ int32_t tqBuildStreamTask(void* pTqObj, SStreamTask* pTask, int64_t nextProcessV } if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { - SWalFilterCond cond = {.deleteMsg = 1}; // delete msg also extract from wal files + bool scanDropCtb = pTask->subtableWithoutMd5 ? true : false; + SWalFilterCond cond = {.deleteMsg = 1, .scanDropCtb = scanDropCtb}; // delete msg also extract from wal files pTask->exec.pWalReader = walOpenReader(pTq->pVnode->pWal, &cond, pTask->id.taskId); if (pTask->exec.pWalReader == NULL) { tqError("vgId:%d failed init wal reader, code:%s", vgId, tstrerror(terrno)); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index c8c34b10a4..391cbe78fb 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -382,6 +382,20 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con return code; } + } else if (pCont->msgType == TDMT_VND_DROP_TABLE && pReader->cond.scanDropCtb) { + void* pBody = POINTER_SHIFT(pCont->body, sizeof(SMsgHead)); + int32_t len = pCont->bodyLen - sizeof(SMsgHead); + code = tqExtractDropCtbDataBlock(pBody, len, ver, (void**)pItem, 0); + if (TSDB_CODE_SUCCESS == code) { + if (!*pItem) { + continue; + } else { + tqDebug("s-task:%s drop ctb msg extract from WAL, len:%d, ver:%"PRId64, id, len, ver); + } + } else { + terrno = code; + return code; + } } else { tqError("s-task:%s invalid msg type:%d, ver:%" PRId64, id, pCont->msgType, ver); return TSDB_CODE_STREAM_INTERNAL_ERROR; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index e972c928f0..3c14870d92 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -53,6 +53,7 @@ static int32_t checkTagSchema(SStreamTask* pTask, SVnode* pVnode); static void reubuildAndSendMultiResBlock(SStreamTask* pTask, const SArray* pBlocks, SVnode* pVnode, int64_t earlyTs); static int32_t handleResultBlockMsg(SStreamTask* pTask, SSDataBlock* pDataBlock, int32_t index, SVnode* pVnode, int64_t earlyTs); +static int32_t doWaitForDstTableDropped(SVnode* pVnode, SStreamTask* pTask, const char* dstTableName, int64_t uid); int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq, const char* pIdStr, bool newSubTableRule) { @@ -138,7 +139,7 @@ int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* p return 0; } -static int32_t encodeCreateChildTableForRPC(SVCreateTbBatchReq* pReqs, int32_t vgId, void** pBuf, int32_t* contLen) { +static int32_t encodeCreateChildTableForRPC(void* pReqs, int32_t vgId, void** pBuf, int32_t* contLen) { int32_t ret = 0; tEncodeSize(tEncodeSVCreateTbBatchReq, pReqs, *contLen, ret); @@ -170,17 +171,50 @@ end: return ret; } -static int32_t tqPutReqToQueue(SVnode* pVnode, SVCreateTbBatchReq* pReqs) { +static int32_t encodeDropChildTableForRPC(void* pReqs, int32_t vgId, void** ppBuf, int32_t *contLen) { + int32_t code = 0; + SEncoder ec = {0}; + tEncodeSize(tEncodeSVDropTbBatchReq, pReqs, *contLen, code); + if (code < 0) { + code = TSDB_CODE_INVALID_MSG; + goto end; + } + *contLen += sizeof(SMsgHead); + *ppBuf = rpcMallocCont(*contLen); + + if (!*ppBuf) { + code = terrno; + goto end; + } + + ((SMsgHead*)(*ppBuf))->vgId = vgId; + ((SMsgHead*)(*ppBuf))->contLen = htonl(*contLen); + + tEncoderInit(&ec, POINTER_SHIFT(*ppBuf, sizeof(SMsgHead)), (*contLen) - sizeof(SMsgHead)); + code = tEncodeSVDropTbBatchReq(&ec, pReqs); + tEncoderClear(&ec); + if (code < 0) { + rpcFreeCont(*ppBuf); + *ppBuf = NULL; + *contLen = 0; + code = TSDB_CODE_INVALID_MSG; + goto end; + } +end: + return code; +} + +static int32_t tqPutReqToQueue(SVnode* pVnode, void* pReqs, int32_t(*encoder)(void* pReqs, int32_t vgId, void** ppBuf, int32_t *contLen), tmsg_t msgType) { void* buf = NULL; int32_t tlen = 0; - int32_t code = encodeCreateChildTableForRPC(pReqs, TD_VID(pVnode), &buf, &tlen); + int32_t code = encoder(pReqs, TD_VID(pVnode), &buf, &tlen); if (code) { tqError("vgId:%d failed to encode create table msg, create table failed, code:%s", TD_VID(pVnode), tstrerror(code)); return code; } - SRpcMsg msg = {.msgType = TDMT_VND_CREATE_TABLE, .pCont = buf, .contLen = tlen}; + SRpcMsg msg = {.msgType = msgType, .pCont = buf, .contLen = tlen}; code = tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg); if (code) { tqError("failed to put into write-queue since %s", terrstr()); @@ -388,7 +422,7 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S } reqs.nReqs = taosArrayGetSize(reqs.pArray); - code = tqPutReqToQueue(pVnode, &reqs); + code = tqPutReqToQueue(pVnode, &reqs, encodeCreateChildTableForRPC, TDMT_VND_CREATE_TABLE); if (code != TSDB_CODE_SUCCESS) { tqError("s-task:%s failed to send create table msg", id); } @@ -399,6 +433,58 @@ _end: return code; } +static int32_t doBuildAndSendDropTableMsg(SVnode* pVnode, char* pStbFullname, SSDataBlock* pDataBlock, + SStreamTask* pTask, int64_t suid) { + int32_t lino = 0; + int32_t code = 0; + int32_t rows = pDataBlock->info.rows; + const char* id = pTask->id.idStr; + SVDropTbBatchReq batchReq = {0}; + SVDropTbReq req = {0}; + + if (rows <= 0 || pTask->subtableWithoutMd5 == 0) return TSDB_CODE_SUCCESS; + + batchReq.pArray = taosArrayInit(rows, sizeof(SVDropTbReq)); + if (!batchReq.pArray) return terrno; + batchReq.nReqs = rows; + req.suid = suid; + req.igNotExists = true; + + SColumnInfoData* pTbNameCol = taosArrayGet(pDataBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); + SColumnInfoData* pUidCol = taosArrayGet(pDataBlock->pDataBlock, UID_COLUMN_INDEX); + char tbName[TSDB_TABLE_NAME_LEN + 1] = {0}; + for (int32_t i = 0; i < rows; ++i) { + void* pData = colDataGetVarData(pTbNameCol, i); + memcpy(tbName, varDataVal(pData), varDataLen(pData)); + tbName[varDataLen(pData) + 1] = 0; + req.name = tbName; + // TODO wjm remove uid, it's not my uid + req.uid = *(int64_t*)colDataGetData(pUidCol, i); + if (taosArrayPush(batchReq.pArray, &req) == NULL) { + TSDB_CHECK_CODE(terrno, lino, _exit); + } + } + tqDebug("s-task:%s build drop %d table(s) msg", id, rows); + code = tqPutReqToQueue(pVnode, &batchReq, encodeDropChildTableForRPC, TDMT_VND_DROP_TABLE); + TSDB_CHECK_CODE(code, lino, _exit); + + for (int32_t i = 0; i < rows; ++i) { + void* pData = colDataGetVarData(pTbNameCol, i); + memcpy(tbName, varDataVal(pData), varDataLen(pData)); + tbName[varDataLen(pData) + 1] = 0; + int64_t uid = *(int64_t*)colDataGetData(pUidCol, i); + code = doWaitForDstTableDropped(pVnode, pTask, tbName, uid); + TSDB_CHECK_CODE(code, lino, _exit); + } + return code; + +_exit: + if (batchReq.pArray) { + taosArrayDestroy(batchReq.pArray); + } + return code; +} + int32_t doBuildAndSendSubmitMsg(SVnode* pVnode, SStreamTask* pTask, SSubmitReq2* pReq, int32_t numOfBlocks) { const char* id = pTask->id.idStr; int32_t vgId = TD_VID(pVnode); @@ -807,6 +893,42 @@ int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkI return TSDB_CODE_SUCCESS; } +static int32_t doWaitForDstTableDropped(SVnode* pVnode, SStreamTask* pTask, const char* dstTableName, int64_t uid) { + int32_t vgId = TD_VID(pVnode); + int64_t suid = pTask->outputInfo.tbSink.stbUid; + const char* id = pTask->id.idStr; + + while (1) { + if (streamTaskShouldStop(pTask)) { + tqDebug("s-task:%s task will stop, quit from waiting for table:%s drop", id, dstTableName); + return TSDB_CODE_STREAM_EXEC_CANCELLED; + } + SMetaReader mr = {0}; + metaReaderDoInit(&mr, pVnode->pMeta, META_READER_LOCK); + int32_t code = metaGetTableEntryByName(&mr, dstTableName); + if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { + tqDebug("wjm s-task:%s table:%s has been dropped", id, dstTableName); + metaReaderClear(&mr); + break; + } else if (TSDB_CODE_SUCCESS == code) { + if (isValidDstChildTable(&mr, vgId, dstTableName, suid)) { + metaReaderClear(&mr); + taosMsleep(100); + tqDebug("s-task:%s wait 100ms for table:%s drop", id, dstTableName); + } else { + tqDebug("wjm s-task:%s table:%s exist, but not mine", id, dstTableName); + metaReaderClear(&mr); + break; + } + } else { + tqError("s-task:%s failed to wait for table:%s drop", id, dstTableName); + metaReaderClear(&mr); + return terrno; + } + } + return TSDB_CODE_SUCCESS; +} + int32_t doCreateSinkTableInfo(const char* pDstTableName, STableSinkInfo** pInfo) { int32_t nameLen = strlen(pDstTableName); (*pInfo) = taosMemoryCalloc(1, sizeof(STableSinkInfo) + nameLen + 1); @@ -1052,6 +1174,8 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { code = doBuildAndSendCreateTableMsg(pVnode, stbFullName, pDataBlock, pTask, suid); } else if (pDataBlock->info.type == STREAM_CHECKPOINT) { continue; + } else if (pDataBlock->info.type == STREAM_DROP_CHILD_TABLE && pTask->subtableWithoutMd5) { + code = doBuildAndSendDropTableMsg(pVnode, stbFullName, pDataBlock, pTask, suid); } else { code = handleResultBlockMsg(pTask, pDataBlock, i, pVnode, earlyTs); } diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 54b063d692..0b3d7b180a 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -751,3 +751,54 @@ int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, b return TSDB_CODE_SUCCESS; } + +int32_t tqExtractDropCtbDataBlock(const void* data, int32_t len, int64_t ver, void** pRefBlock, int32_t type) { + int32_t code = 0; + int32_t lino = 0; + SDecoder dc = {0}; + SVDropTbBatchReq batchReq = {0}; + tDecoderInit(&dc, (uint8_t*)data, len); + code = tDecodeSVDropTbBatchReq(&dc, &batchReq); + TSDB_CHECK_CODE(code, lino, _exit); + if (batchReq.nReqs <= 0) goto _exit; + + SSDataBlock* pBlock = NULL; + code = createSpecialDataBlock(STREAM_DROP_CHILD_TABLE, &pBlock); + TSDB_CHECK_CODE(code, lino, _exit); + + code = blockDataEnsureCapacity(pBlock, batchReq.nReqs); + TSDB_CHECK_CODE(code, lino, _exit); + + pBlock->info.rows = batchReq.nReqs; + pBlock->info.version = ver; + for (int32_t i = 0; i < batchReq.nReqs; ++i) { + SVDropTbReq* pReq = batchReq.pReqs + i; + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); + TSDB_CHECK_NULL(pCol, code, lino, _exit, terrno); + code = colDataSetVal(pCol, i, (const char* )&pReq->uid, false); + TSDB_CHECK_CODE(code, lino, _exit); + + /* + pCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); + TSDB_CHECK_NULL(pCol, code, lino, _exit, terrno); + char varTbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE + 1] = {0}; + varDataSetLen(varTbName, strlen(pReq->name)); + tsnprintf(varTbName + VARSTR_HEADER_SIZE, TSDB_TABLE_NAME_LEN + 1, "%s", pReq->name); + code = colDataSetVal(pCol, i, varTbName, false); + */ + } + + code = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0, pRefBlock); + TSDB_CHECK_CODE(code, lino, _exit); + ((SStreamRefDataBlock*)(*pRefBlock))->type = STREAM_INPUT__REF_DATA_BLOCK; + ((SStreamRefDataBlock*)(*pRefBlock))->pBlock = pBlock; + +_exit: + tDecoderClear(&dc); + if (TSDB_CODE_SUCCESS != code) { + tqError("faled to extract drop ctb data block, line:%d code:%s", lino, tstrerror(code)); + blockDataCleanup(pBlock); + taosMemoryFree(pBlock); + } + return code; +} diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 1ea2aac809..a2b5f49b5c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -552,6 +552,61 @@ _exit: return code; } +int32_t vnodePreProcessDropTbMsg(SVnode* pVnode, SRpcMsg* pMsg) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t size = 0; + SDecoder dc = {0}; + SEncoder ec = {0}; + SVDropTbBatchReq receivedBatchReqs = {0}; + SVDropTbBatchReq sentBatchReqs = {0}; + + tDecoderInit(&dc, pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead)); + + code = tDecodeSVDropTbBatchReq(&dc, &receivedBatchReqs); + if (code < 0) { + terrno = code; + TSDB_CHECK_CODE(code, lino, _exit); + } + sentBatchReqs.pArray = taosArrayInit(receivedBatchReqs.nReqs, sizeof(SVDropTbReq)); + if (!sentBatchReqs.pArray) { + code = terrno; + goto _exit; + } + + for (int32_t i = 0; i < receivedBatchReqs.nReqs; ++i) { + SVDropTbReq* pReq = receivedBatchReqs.pReqs + i; + tb_uid_t uid = metaGetTableEntryUidByName(pVnode->pMeta, pReq->name); + if (uid == 0) { + vWarn("vgId:%d, preprocess drop ctb: %s not found", TD_VID(pVnode), pReq->name); + continue; + } + pReq->uid = uid; + vDebug("vgId:%d %s for: %s, uid: %"PRId64, TD_VID(pVnode), __func__, pReq->name, pReq->uid); + if (taosArrayPush(sentBatchReqs.pArray, pReq) == NULL) { + code = terrno; + goto _exit; + } + } + sentBatchReqs.nReqs = sentBatchReqs.pArray->size; + + tEncodeSize(tEncodeSVDropTbBatchReq, &sentBatchReqs, size, code); + tEncoderInit(&ec, pMsg->pCont + sizeof(SMsgHead), size); + code = tEncodeSVDropTbBatchReq(&ec, &sentBatchReqs); + tEncoderClear(&ec); + if (code != TSDB_CODE_SUCCESS) { + vError("vgId:%d %s failed to encode drop tb batch req: %s", TD_VID(pVnode), __func__, tstrerror(code)); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + tDecoderClear(&dc); + if (sentBatchReqs.pArray) { + taosArrayDestroy(sentBatchReqs.pArray); + } + return code; +} + int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t code = 0; @@ -581,6 +636,9 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { case TDMT_VND_DROP_TSMA_CTB: { code = vnodePreProcessDropTSmaCtbMsg(pVnode, pMsg); } break; + case TDMT_VND_DROP_TABLE: { + code = vnodePreProcessDropTbMsg(pVnode, pMsg); + } break; default: break; } @@ -1189,7 +1247,6 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, STbUidStore *pStore = NULL; SArray *tbUids = NULL; SArray *tbNames = NULL; - pRsp->msgType = TDMT_VND_CREATE_TABLE_RSP; pRsp->code = TSDB_CODE_SUCCESS; pRsp->pCont = NULL; @@ -1245,9 +1302,11 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, continue; } + vInfo("wjm process create tb req:%s, uid: %"PRId64, pCreateReq->name, pCreateReq->uid); // do create table if (metaCreateTable(pVnode->pMeta, ver, pCreateReq, &cRsp.pMeta) < 0) { if (pCreateReq->flags & TD_CREATE_IF_NOT_EXISTS && terrno == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { + vInfo("wjm already exists-----------------"); cRsp.code = TSDB_CODE_SUCCESS; } else { cRsp.code = terrno; @@ -1324,6 +1383,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, } _exit: + vInfo("wjm process create table request exit"); tDeleteSVCreateTbBatchReq(&req); taosArrayDestroyEx(rsp.pArray, tFreeSVCreateTbRsp); taosArrayDestroy(tbUids); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0cdb0e7954..b5cb22cf12 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3309,7 +3309,9 @@ static int32_t setBlockGroupIdByUid(SStreamScanInfo* pInfo, SSDataBlock* pBlock) int32_t rows = pBlock->info.rows; if (!pInfo->partitionSup.needCalc) { for (int32_t i = 0; i < rows; i++) { + qInfo("wjm, get uid: %"PRIu64, uidCol[i]); uint64_t groupId = getGroupIdByUid(pInfo, uidCol[i]); + qInfo("wjm, get groupid: %"PRIu64, groupId); code = colDataSetVal(pGpCol, i, (const char*)&groupId, false); QUERY_CHECK_CODE(code, lino, _end); } @@ -3535,12 +3537,32 @@ static int32_t copyGetResultBlock(SSDataBlock* dest, TSKEY start, TSKEY end) { return appendDataToSpecialBlock(dest, &start, &end, NULL, NULL, NULL); } -static int32_t deletePartName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { +static int32_t deletePartName(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t *deleteNum) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; for (int32_t i = 0; i < pBlock->info.rows; i++) { SColumnInfoData* pGpIdCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); + SColumnInfoData* pTbnameCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); int64_t* gpIdCol = (int64_t*)pGpIdCol->pData; + void* pParName = NULL; + int32_t winCode = 0; + // TODO wjm test remove non stream child tables + code = pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, gpIdCol[i], + &pParName, false, &winCode); + if (TSDB_CODE_SUCCESS == code && winCode != 0) { + qInfo("delete stream part Name for:%"PRId64 " not found", gpIdCol[i]); + colDataSetNULL(pTbnameCol, i); + continue; + } + (*deleteNum)++; + QUERY_CHECK_CODE(code, lino, _end); + char varTbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE + 1] = {0}; + varDataSetLen(varTbName, strlen(pParName)); + tsnprintf(varTbName + VARSTR_HEADER_SIZE, TSDB_TABLE_NAME_LEN + 1, "%s", pParName); + code = colDataSetVal(pTbnameCol, i, varTbName, false); + qDebug("delete stream part for:%"PRId64 " res tb: %s", gpIdCol[i], (char*)pParName); + pInfo->stateStore.streamStateFreeVal(pParName); + QUERY_CHECK_CODE(code, lino, _end); code = pInfo->stateStore.streamStateDeleteParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, gpIdCol[i]); QUERY_CHECK_CODE(code, lino, _end); } @@ -3791,15 +3813,13 @@ FETCH_NEXT_BLOCK: prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex, NULL); pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; } break; - case STREAM_DELETE_GROUP_DATA: { - printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "delete group recv", - GET_TASKID(pTaskInfo)); + case STREAM_DROP_CHILD_TABLE: { + int32_t deleteNum = 0; code = setBlockGroupIdByUid(pInfo, pBlock); QUERY_CHECK_CODE(code, lino, _end); - - code = deletePartName(pInfo, pBlock); + code = deletePartName(pInfo, pBlock, &deleteNum); QUERY_CHECK_CODE(code, lino, _end); - goto FETCH_NEXT_BLOCK; + if (deleteNum == 0) goto FETCH_NEXT_BLOCK; } break; case STREAM_CHECKPOINT: { qError("stream check point error. msg type: STREAM_INPUT__DATA_BLOCK"); diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 8fd00e9313..2e906d2ba6 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -5215,7 +5215,7 @@ static int32_t doStreamIntervalAggNext(SOperatorInfo* pOperator, SSDataBlock** p code = getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); QUERY_CHECK_CODE(code, lino, _end); continue; - } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE || pBlock->info.type == STREAM_DROP_CHILD_TABLE) { printDataBlock(pBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); (*ppRes) = pBlock; return code; diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 26c97cbb6e..91ed919446 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -480,6 +480,7 @@ int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char if (tSimpleHashGet(pState->parNameMap, &groupId, sizeof(int64_t)) == NULL) { if (tSimpleHashGetSize(pState->parNameMap) < MAX_TABLE_NAME_NUM) { code = tSimpleHashPut(pState->parNameMap, &groupId, sizeof(int64_t), tbname, TSDB_TABLE_NAME_LEN); + qInfo("wjm put group id into parnamemap: %"PRId64 " cur mapsize: %d", groupId, tSimpleHashGetSize(pState->parNameMap)); QUERY_CHECK_CODE(code, lino, _end); } code = streamStatePutParName_rocksdb(pState, groupId, tbname); @@ -505,6 +506,7 @@ int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal (*pWinCode) = streamStateGetParName_rocksdb(pState, groupId, pVal); if ((*pWinCode) == TSDB_CODE_SUCCESS && tSimpleHashGetSize(pState->parNameMap) < MAX_TABLE_NAME_NUM) { code = tSimpleHashPut(pState->parNameMap, &groupId, sizeof(int64_t), *pVal, TSDB_TABLE_NAME_LEN); + qInfo("wjm put group id into parnamemap: %"PRId64 " cur mapsize: %d", groupId, tSimpleHashGetSize(pState->parNameMap)); QUERY_CHECK_CODE(code, lino, _end); } goto _end; @@ -526,6 +528,7 @@ _end: } int32_t streamStateDeleteParName(SStreamState* pState, int64_t groupId) { + qTrace("wjm delete par for group:%"PRId64 " parnameMapsize: %d", groupId, tSimpleHashGetSize(pState->parNameMap)); int32_t code = tSimpleHashRemove(pState->parNameMap, &groupId, sizeof(int64_t)); qTrace("catche %s at line %d res %d", __func__, __LINE__, code); code = streamStateDeleteParName_rocksdb(pState, groupId); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 9a3ea34eff..94dff3f71c 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -89,6 +89,8 @@ int32_t walNextValidMsg(SWalReader *pReader) { if (type == TDMT_VND_SUBMIT || ((type == TDMT_VND_DELETE) && (pReader->cond.deleteMsg == 1)) || (type == TDMT_VND_DROP_TSMA_CTB) || (IS_META_MSG(type) && pReader->cond.scanMeta)) { TAOS_RETURN(walFetchBody(pReader)); + } else if (type == TDMT_VND_DROP_TABLE && pReader->cond.scanDropCtb) { + TAOS_RETURN(walFetchBody(pReader)); } else { TAOS_CHECK_RETURN(walSkipFetchBody(pReader)); diff --git a/tests/system-test/2-query/tsma.py b/tests/system-test/2-query/tsma.py index a1638ae4cb..78a3c1406e 100644 --- a/tests/system-test/2-query/tsma.py +++ b/tests/system-test/2-query/tsma.py @@ -604,7 +604,7 @@ class TSMATestSQLGenerator: class TDTestCase: - updatecfgDict = {'asynclog': 0, 'ttlUnit': 1, 'ttlPushInterval': 5, 'ratioOfVnodeStreamThrea': 4, 'maxTsmaNum': 3} + updatecfgDict = {'asynclog': 0, 'ttlUnit': 1, 'ttlPushInterval': 5, 'ratioOfVnodeStreamThrea': 4, 'maxTsmaNum': 3, 'debugFlag': 143} def __init__(self): self.vgroups = 4 @@ -804,9 +804,10 @@ class TDTestCase: self.tsma_tester.check_sql(ctx.sql, ctx) def test_query_with_tsma(self): - self.create_tsma('tsma1', 'test', 'meters', ['avg(c1)', 'avg(c2)'], '5m') - self.create_tsma('tsma2', 'test', 'meters', ['avg(c1)', 'avg(c2)'], '30m') - self.create_tsma('tsma5', 'test', 'norm_tb', ['avg(c1)', 'avg(c2)'], '10m') + self.create_tsma('tsma1', 'test', 'meters', ['avg(c1)', 'avg(c2)', 'count(ts)'], '5m') + #self.create_tsma('tsma2', 'test', 'meters', ['avg(c1)', 'avg(c2)', 'count(ts)'], '30m') + #self.create_tsma('tsma5', 'test', 'norm_tb', ['avg(c1)', 'avg(c2)'], '10m') + return self.test_query_with_tsma_interval() self.test_query_with_tsma_agg() @@ -1227,17 +1228,28 @@ class TDTestCase: def run(self): self.init_data() - self.test_ddl() + #self.test_ddl() self.test_query_with_tsma() # bug to fix - self.test_flush_query() + #self.test_flush_query() #cluster test cluster_dnode_list = tdSql.get_cluseter_dnodes() clust_dnode_nums = len(cluster_dnode_list) if clust_dnode_nums > 1: self.test_redistribute_vgroups() - + self.test_td_32519() + + def test_td_32519(self): + tdSql.execute('INSERT INTO t1 VALUES("2024-10-24 11:45:00", 1,1,1,1,1,1,1, "a", "a")', queryTimes=1) + tdSql.execute('INSERT INTO t1 VALUES("2024-10-24 11:55:00", 2,1,1,1,1,1,1, "a", "a")', queryTimes=1) + tdSql.execute('DROP TABLE t1', queryTimes=1) + tdSql.execute('CREATE TABLE t1 USING meters TAGS(1, "a", "b", 1,1,1)') + tdSql.execute('INSERT INTO t1 VALUES("2024-10-24 11:59:00", 3,1,1,1,1,1,1, "a", "a")', queryTimes=1) + tdSql.execute('INSERT INTO t1 VALUES("2024-10-24 12:10:00", 4,1,1,1,1,1,1, "a", "a")', queryTimes=1) + tdSql.execute('INSERT INTO t1 VALUES("2024-10-24 12:20:00", 5,1,1,1,1,1,1, "a", "a")', queryTimes=1) + tdSql.execute('FLUSH DATABASE test', queryTimes=1) + def test_create_tsma(self): function_name = sys._getframe().f_code.co_name tdLog.debug(f'-----{function_name}------') From 9a92c136ce2553965171dd39a819a7edfb688a69 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 5 Nov 2024 15:10:25 +0800 Subject: [PATCH 31/35] fix drop child table with tsma --- include/common/tcommon.h | 4 +- include/common/tmsgdef.h | 1 - include/libs/stream/tstream.h | 2 +- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 1 - source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 1 - source/dnode/mnode/impl/src/mndStb.c | 1 - source/dnode/vnode/src/tq/tqRead.c | 4 +- source/dnode/vnode/src/tq/tqSink.c | 15 ++++++- source/dnode/vnode/src/tq/tqUtil.c | 9 ----- source/dnode/vnode/src/vnd/vnodeSvr.c | 14 ------- source/libs/executor/src/groupoperator.c | 8 +++- source/libs/executor/src/scanoperator.c | 21 ++++++---- source/libs/parser/src/parser.c | 3 -- source/libs/planner/src/planLogicCreater.c | 27 ++++++------- source/libs/planner/src/planSpliter.c | 6 +-- source/libs/stream/src/streamDispatch.c | 1 + source/libs/stream/src/streamQueue.c | 6 ++- source/libs/wal/src/walRead.c | 2 +- tests/system-test/2-query/tsma.py | 44 +++++++++++++++------ 19 files changed, 92 insertions(+), 78 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 8a8948fb17..1d9a9bcc61 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -155,7 +155,6 @@ typedef enum EStreamType { STREAM_MID_RETRIEVE, STREAM_PARTITION_DELETE_DATA, STREAM_GET_RESULT, - STREAM_DELETE_GROUP_DATA, STREAM_DROP_CHILD_TABLE, } EStreamType; @@ -403,7 +402,8 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol); #define TSMA_RES_STB_EXTRA_COLUMN_NUM 4 // 3 columns: _wstart, _wend, _wduration, 1 tag: tbname static inline bool isTsmaResSTb(const char* stbName) { - return false; + static bool showTsmaTables = false; + if (showTsmaTables) return false; const char* pos = strstr(stbName, TSMA_RES_STB_POSTFIX); if (pos && strlen(stbName) == (pos - stbName) + strlen(TSMA_RES_STB_POSTFIX)) { return true; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index c81d649284..c22a3da5ad 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -316,7 +316,6 @@ TD_DEF_MSG_TYPE(TDMT_VND_ARB_CHECK_SYNC, "vnode-arb-check-sync", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_FETCH_TTL_EXPIRED_TBS, "vnode-fetch-ttl-expired-tbs", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TABLE_NAME, "vnode-table-name", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_DROP_TSMA_CTB, "vnode-drop-tsma-ctb", NULL, NULL) TD_CLOSE_MSG_SEG(TDMT_VND_MSG) TD_NEW_MSG_SEG(TDMT_SCH_MSG) // 3<<8 diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index de10d6844e..2cf791c8da 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -462,7 +462,7 @@ struct SStreamTask { struct SStreamMeta* pMeta; SSHashObj* pNameMap; void* pBackend; - int8_t subtableWithoutMd5; + int8_t subtableWithoutMd5; // only for tsma stream tasks char reserve[256]; char* backendPath; }; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 4b79ecf43a..0d804eadf0 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -182,7 +182,6 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_TB_WITH_TSMA, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH_TTL_EXPIRED_TBS_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TABLE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TSMA_CTB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_ANAL_ALGO, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_IP_WHITE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index a0356a6c4d..006f44b349 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -1014,7 +1014,6 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_COMPACT_PROGRESS, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_KILL_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TABLE_NAME, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TSMA_CTB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index a1044a9f86..e782d505a9 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -96,7 +96,6 @@ int32_t mndInitStb(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_DROP_TB_WITH_TSMA, mndProcessDropTbWithTsma); mndSetMsgHandle(pMnode, TDMT_VND_FETCH_TTL_EXPIRED_TBS_RSP, mndProcessFetchTtlExpiredTbs); mndSetMsgHandle(pMnode, TDMT_VND_DROP_TABLE_RSP, mndTransProcessRsp); - mndSetMsgHandle(pMnode, TDMT_VND_DROP_TSMA_CTB_RSP, mndTransProcessRsp); // mndSetMsgHandle(pMnode, TDMT_MND_SYSTABLE_RETRIEVE, mndProcessRetrieveStbReq); // mndSetMsgHandle(pMnode, TDMT_MND_CREATE_INDEX, mndProcessCreateIndexReq); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 391cbe78fb..ea2f3f91be 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -363,10 +363,10 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con tqError("%s failed to create data submit for stream since out of memory", id); return code; } - } else if (pCont->msgType == TDMT_VND_DELETE || pCont->msgType == TDMT_VND_DROP_TSMA_CTB) { + } else if (pCont->msgType == TDMT_VND_DELETE) { void* pBody = POINTER_SHIFT(pCont->body, sizeof(SMsgHead)); int32_t len = pCont->bodyLen - sizeof(SMsgHead); - EStreamType blockType = pCont->msgType == TDMT_VND_DELETE ? STREAM_DELETE_DATA : STREAM_DELETE_GROUP_DATA; + EStreamType blockType = pCont->msgType == STREAM_DELETE_DATA; code = tqExtractDelDataBlock(pBody, len, ver, (void**)pItem, 0, blockType); if (code == TSDB_CODE_SUCCESS) { if (*pItem == NULL) { diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 3c14870d92..3c525e7e9c 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -464,6 +464,19 @@ static int32_t doBuildAndSendDropTableMsg(SVnode* pVnode, char* pStbFullname, SS TSDB_CHECK_CODE(terrno, lino, _exit); } } + + SMetaReader mr = {0}; + metaReaderDoInit(&mr, pVnode->pMeta, META_READER_LOCK); + // TODO wjm handle only one table + code = metaGetTableEntryByName(&mr, tbName); + if (isValidDstChildTable(&mr, TD_VID(pVnode), tbName, pTask->outputInfo.tbSink.stbUid)) { + STableSinkInfo* pTableSinkInfo = NULL; + bool alreadyCached = doGetSinkTableInfoFromCache(pTask->outputInfo.tbSink.pTbInfo, pDataBlock->info.id.groupId, &pTableSinkInfo); + if (alreadyCached) { + pTableSinkInfo->uid = mr.me.uid; + } + } + metaReaderClear(&mr); tqDebug("s-task:%s build drop %d table(s) msg", id, rows); code = tqPutReqToQueue(pVnode, &batchReq, encodeDropChildTableForRPC, TDMT_VND_DROP_TABLE); TSDB_CHECK_CODE(code, lino, _exit); @@ -473,10 +486,10 @@ static int32_t doBuildAndSendDropTableMsg(SVnode* pVnode, char* pStbFullname, SS memcpy(tbName, varDataVal(pData), varDataLen(pData)); tbName[varDataLen(pData) + 1] = 0; int64_t uid = *(int64_t*)colDataGetData(pUidCol, i); + // TODO wjm remove uid it's not my uid code = doWaitForDstTableDropped(pVnode, pTask, tbName, uid); TSDB_CHECK_CODE(code, lino, _exit); } - return code; _exit: if (batchReq.pArray) { diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 0b3d7b180a..a92049e5f3 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -777,15 +777,6 @@ int32_t tqExtractDropCtbDataBlock(const void* data, int32_t len, int64_t ver, vo TSDB_CHECK_NULL(pCol, code, lino, _exit, terrno); code = colDataSetVal(pCol, i, (const char* )&pReq->uid, false); TSDB_CHECK_CODE(code, lino, _exit); - - /* - pCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); - TSDB_CHECK_NULL(pCol, code, lino, _exit, terrno); - char varTbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE + 1] = {0}; - varDataSetLen(varTbName, strlen(pReq->name)); - tsnprintf(varTbName + VARSTR_HEADER_SIZE, TSDB_TABLE_NAME_LEN + 1, "%s", pReq->name); - code = colDataSetVal(pCol, i, varTbName, false); - */ } code = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0, pRefBlock); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index a2b5f49b5c..723ebb333d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -633,9 +633,6 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { case TDMT_VND_ARB_CHECK_SYNC: { code = vnodePreProcessArbCheckSyncMsg(pVnode, pMsg); } break; - case TDMT_VND_DROP_TSMA_CTB: { - code = vnodePreProcessDropTSmaCtbMsg(pVnode, pMsg); - } break; case TDMT_VND_DROP_TABLE: { code = vnodePreProcessDropTbMsg(pVnode, pMsg); } break; @@ -843,11 +840,6 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg case TDMT_VND_ARB_CHECK_SYNC: vnodeProcessArbCheckSyncReq(pVnode, pReq, len, pRsp); break; - case TDMT_VND_DROP_TSMA_CTB: - if (vnodeProcessDropTSmaCtbReq(pVnode, ver, pReq, len, pRsp, pMsg) < 0) { - goto _err; - } - break; default: vError("vgId:%d, unprocessed msg, %d", TD_VID(pVnode), pMsg->msgType); return TSDB_CODE_INVALID_MSG; @@ -2652,9 +2644,3 @@ int32_t vnodeAsyncCompact(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, int32_t tsdbAsyncCompact(STsdb *tsdb, const STimeWindow *tw, bool sync) { return 0; } #endif -static int32_t vnodeProcessDropTSmaCtbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, - SRpcMsg *pOriginalMsg) { - pRsp->msgType = TDMT_VND_DROP_TSMA_CTB_RSP; - pRsp->code = TSDB_CODE_SUCCESS; - return pRsp->code; -} diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index fec35c3371..d5b4a52be7 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -1326,7 +1326,7 @@ int32_t appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTag int32_t winCode = TSDB_CODE_SUCCESS; code = pAPI->streamStateGetParName(pState, groupId, &pValue, true, &winCode); QUERY_CHECK_CODE(code, lino, _end); - + qInfo("wjm group id: %"PRId64 " winCode: %d, block type: %d", groupId, winCode, pSrcBlock->info.type); if (winCode != TSDB_CODE_SUCCESS) { SSDataBlock* pTmpBlock = NULL; code = blockCopyOneRow(pSrcBlock, rowId, &pTmpBlock); @@ -1335,6 +1335,8 @@ int32_t appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTag memset(pTmpBlock->info.parTbName, 0, TSDB_TABLE_NAME_LEN); pTmpBlock->info.id.groupId = groupId; char* tbName = pSrcBlock->info.parTbName; + printSpecDataBlock(pSrcBlock, "wjm", "recv", "wjm"); + printSpecDataBlock(pTmpBlock, "wjm", "recv", "wjm"); if (pTableSup->numOfExprs > 0) { code = projectApplyFunctions(pTableSup->pExprInfo, pDestBlock, pTmpBlock, pTableSup->pCtx, pTableSup->numOfExprs, NULL); @@ -1342,15 +1344,19 @@ int32_t appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTag SColumnInfoData* pTbCol = taosArrayGet(pDestBlock->pDataBlock, UD_TABLE_NAME_COLUMN_INDEX); QUERY_CHECK_NULL(pTbCol, code, lino, _end, terrno); + printSpecDataBlock(pSrcBlock, "wjm", "recv", "wjm"); + printSpecDataBlock(pTmpBlock, "wjm", "recv", "wjm"); memset(tbName, 0, TSDB_TABLE_NAME_LEN); int32_t len = 0; if (colDataIsNull_s(pTbCol, pDestBlock->info.rows - 1)) { + qInfo("wjm calculated tbnameis null"); len = 1; tbName[0] = 0; } else { void* pData = colDataGetData(pTbCol, pDestBlock->info.rows - 1); len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1); memcpy(tbName, varDataVal(pData), len); + qInfo("wjm calculated tbname: %s", tbName); code = pAPI->streamStatePutParName(pState, groupId, tbName); QUERY_CHECK_CODE(code, lino, _end); } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b5cb22cf12..4cd32589d8 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -289,6 +289,7 @@ static int32_t doSetTagColumnData(STableScanBase* pTableScanInfo, SSDataBlock* p pTaskInfo, &pTableScanInfo->metaCache); // ignore the table not exists error, since this table may have been dropped during the scan procedure. if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { + if (pTaskInfo->streamInfo.pState) blockDataCleanup(pBlock); code = 0; } } @@ -3038,10 +3039,6 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes, pBlockInfo->rows, pTaskInfo, &pTableScanInfo->base.metaCache); // ignore the table not exists error, since this table may have been dropped during the scan procedure. - if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { - code = 0; - } - if (code) { blockDataFreeRes((SSDataBlock*)pBlock); QUERY_CHECK_CODE(code, lino, _end); @@ -3312,7 +3309,7 @@ static int32_t setBlockGroupIdByUid(SStreamScanInfo* pInfo, SSDataBlock* pBlock) qInfo("wjm, get uid: %"PRIu64, uidCol[i]); uint64_t groupId = getGroupIdByUid(pInfo, uidCol[i]); qInfo("wjm, get groupid: %"PRIu64, groupId); - code = colDataSetVal(pGpCol, i, (const char*)&groupId, false); + code = colDataSetVal(pGpCol, i, (const char*)(uidCol + i), false); QUERY_CHECK_CODE(code, lino, _end); } } @@ -3541,7 +3538,7 @@ static int32_t deletePartName(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32 int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; for (int32_t i = 0; i < pBlock->info.rows; i++) { - SColumnInfoData* pGpIdCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); + SColumnInfoData* pGpIdCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); SColumnInfoData* pTbnameCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); int64_t* gpIdCol = (int64_t*)pGpIdCol->pData; void* pParName = NULL; @@ -3558,13 +3555,15 @@ static int32_t deletePartName(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32 QUERY_CHECK_CODE(code, lino, _end); char varTbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE + 1] = {0}; varDataSetLen(varTbName, strlen(pParName)); - tsnprintf(varTbName + VARSTR_HEADER_SIZE, TSDB_TABLE_NAME_LEN + 1, "%s", pParName); + int64_t len = tsnprintf(varTbName + VARSTR_HEADER_SIZE, TSDB_TABLE_NAME_LEN + 1, "%s", pParName); code = colDataSetVal(pTbnameCol, i, varTbName, false); qDebug("delete stream part for:%"PRId64 " res tb: %s", gpIdCol[i], (char*)pParName); pInfo->stateStore.streamStateFreeVal(pParName); QUERY_CHECK_CODE(code, lino, _end); code = pInfo->stateStore.streamStateDeleteParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, gpIdCol[i]); QUERY_CHECK_CODE(code, lino, _end); + pBlock->info.id.groupId = gpIdCol[i]; + memcpy(pBlock->info.parTbName, varTbName + VARSTR_HEADER_SIZE, TSDB_TABLE_NAME_LEN + 1); } _end: @@ -3962,7 +3961,13 @@ FETCH_NEXT_BLOCK: } code = setBlockIntoRes(pInfo, pRes, &pStreamInfo->fillHistoryWindow, false); - QUERY_CHECK_CODE(code, lino, _end); + if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { + pInfo->pRes->info.rows = 0; + code = TSDB_CODE_SUCCESS; + } else { + QUERY_CHECK_CODE(code, lino, _end); + } + if (pInfo->pRes->info.rows == 0) { continue; } diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 8ac1acb1a2..c2714659ec 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -433,9 +433,6 @@ int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx nodesDestroyNode(pQuery->pRoot); pQuery->pRoot = NULL; code = nodesCloneNode(pQuery->pPrepareRoot, &pQuery->pRoot); - if (NULL == pQuery->pRoot) { - code = code; - } } if (TSDB_CODE_SUCCESS == code) { rewriteExprAlias(pQuery->pRoot); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 34c83acee8..09a4b9c593 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -1534,21 +1534,20 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect if (TSDB_CODE_SUCCESS == code) { pSort->pSortKeys = NULL; code = nodesCloneList(pSelect->pOrderByList, &pSort->pSortKeys); - if (NULL == pSort->pSortKeys) { - code = code; - } - SNode* pNode = NULL; - SOrderByExprNode* firstSortKey = (SOrderByExprNode*)nodesListGetNode(pSort->pSortKeys, 0); - if (isPrimaryKeySort(pSelect->pOrderByList)) pSort->node.outputTsOrder = firstSortKey->order; - if (firstSortKey->pExpr->type == QUERY_NODE_COLUMN) { - SColumnNode* pCol = (SColumnNode*)firstSortKey->pExpr; - int16_t projIdx = 1; - FOREACH(pNode, pSelect->pProjectionList) { - SExprNode* pExpr = (SExprNode*)pNode; - if (0 == strcmp(pCol->node.aliasName, pExpr->aliasName)) { - pCol->projIdx = projIdx; break; + if (NULL != pSort->pSortKeys) { + SNode* pNode = NULL; + SOrderByExprNode* firstSortKey = (SOrderByExprNode*)nodesListGetNode(pSort->pSortKeys, 0); + if (isPrimaryKeySort(pSelect->pOrderByList)) pSort->node.outputTsOrder = firstSortKey->order; + if (firstSortKey->pExpr->type == QUERY_NODE_COLUMN) { + SColumnNode* pCol = (SColumnNode*)firstSortKey->pExpr; + int16_t projIdx = 1; + FOREACH(pNode, pSelect->pProjectionList) { + SExprNode* pExpr = (SExprNode*)pNode; + if (0 == strcmp(pCol->node.aliasName, pExpr->aliasName)) { + pCol->projIdx = projIdx; break; + } + projIdx++; } - projIdx++; } } } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index e0e42087f3..e960c0ff5d 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -836,11 +836,9 @@ static int32_t stbSplSplitSessionForStream(SSplitContext* pCxt, SStableSplitInfo nodesDestroyNode(pMergeWin->pTsEnd); pMergeWin->pTsEnd = NULL; code = nodesCloneNode(nodesListGetNode(pPartWin->node.pTargets, index), &pMergeWin->pTsEnd); - if (NULL == pMergeWin->pTsEnd) { - code = code; - } } - code = stbSplCreateExchangeNode(pCxt, pInfo->pSplitNode, pPartWindow); + if (TSDB_CODE_SUCCESS == code) + code = stbSplCreateExchangeNode(pCxt, pInfo->pSplitNode, pPartWindow); } if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index e0fa199199..6e27cd651e 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -758,6 +758,7 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S SUseDbRsp* pDbInfo = &pTask->outputInfo.shuffleDispatcher.dbInfo; hashValue = taosGetTbHashVal(ctbName, strlen(ctbName), pDbInfo->hashMethod, pDbInfo->hashPrefix, pDbInfo->hashSuffix); + stInfo("wjm ctbname for dispatch: %s, pDataBlock.info.parTbName: %s", ctbName, pDataBlock->info.parTbName); SBlockName bln = {0}; bln.hashValue = hashValue; memcpy(bln.parTbName, pDataBlock->info.parTbName, strlen(pDataBlock->info.parTbName)); diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 20c3e5a6b9..401aa7530d 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -166,6 +166,8 @@ const char* streamQueueItemGetTypeStr(int32_t type) { return "checkpoint-trigger"; case STREAM_INPUT__TRANS_STATE: return "trans-state"; + case STREAM_INPUT__REF_DATA_BLOCK: + return "ref-block"; default: return "datablock"; } @@ -211,7 +213,7 @@ EExtractDataCode streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueIte // do not merge blocks for sink node and check point data block int8_t type = qItem->type; if (type == STREAM_INPUT__CHECKPOINT || type == STREAM_INPUT__CHECKPOINT_TRIGGER || - type == STREAM_INPUT__TRANS_STATE) { + type == STREAM_INPUT__TRANS_STATE || type == STREAM_INPUT__REF_DATA_BLOCK) { const char* p = streamQueueItemGetTypeStr(type); if (*pInput == NULL) { @@ -504,4 +506,4 @@ void streamTaskPutbackToken(STokenBucket* pBucket) { // size in KB void streamTaskConsumeQuota(STokenBucket* pBucket, int32_t bytes) { pBucket->quotaRemain -= SIZE_IN_MiB(bytes); } -void streamTaskInputFail(SStreamTask* pTask) { atomic_store_8(&pTask->inputq.status, TASK_INPUT_STATUS__FAILED); } \ No newline at end of file +void streamTaskInputFail(SStreamTask* pTask) { atomic_store_8(&pTask->inputq.status, TASK_INPUT_STATUS__FAILED); } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 94dff3f71c..da5e1f47e9 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -87,7 +87,7 @@ int32_t walNextValidMsg(SWalReader *pReader) { int32_t type = pReader->pHead->head.msgType; if (type == TDMT_VND_SUBMIT || ((type == TDMT_VND_DELETE) && (pReader->cond.deleteMsg == 1)) || - (type == TDMT_VND_DROP_TSMA_CTB) || (IS_META_MSG(type) && pReader->cond.scanMeta)) { + (IS_META_MSG(type) && pReader->cond.scanMeta)) { TAOS_RETURN(walFetchBody(pReader)); } else if (type == TDMT_VND_DROP_TABLE && pReader->cond.scanDropCtb) { TAOS_RETURN(walFetchBody(pReader)); diff --git a/tests/system-test/2-query/tsma.py b/tests/system-test/2-query/tsma.py index 78a3c1406e..77e57bd36d 100644 --- a/tests/system-test/2-query/tsma.py +++ b/tests/system-test/2-query/tsma.py @@ -805,9 +805,8 @@ class TDTestCase: def test_query_with_tsma(self): self.create_tsma('tsma1', 'test', 'meters', ['avg(c1)', 'avg(c2)', 'count(ts)'], '5m') - #self.create_tsma('tsma2', 'test', 'meters', ['avg(c1)', 'avg(c2)', 'count(ts)'], '30m') - #self.create_tsma('tsma5', 'test', 'norm_tb', ['avg(c1)', 'avg(c2)'], '10m') - return + self.create_tsma('tsma2', 'test', 'meters', ['avg(c1)', 'avg(c2)', 'count(ts)'], '30m') + self.create_tsma('tsma5', 'test', 'norm_tb', ['avg(c1)', 'avg(c2)'], '10m') self.test_query_with_tsma_interval() self.test_query_with_tsma_agg() @@ -1228,10 +1227,10 @@ class TDTestCase: def run(self): self.init_data() - #self.test_ddl() + self.test_ddl() self.test_query_with_tsma() # bug to fix - #self.test_flush_query() + self.test_flush_query() #cluster test cluster_dnode_list = tdSql.get_cluseter_dnodes() @@ -1241,14 +1240,35 @@ class TDTestCase: self.test_td_32519() def test_td_32519(self): - tdSql.execute('INSERT INTO t1 VALUES("2024-10-24 11:45:00", 1,1,1,1,1,1,1, "a", "a")', queryTimes=1) - tdSql.execute('INSERT INTO t1 VALUES("2024-10-24 11:55:00", 2,1,1,1,1,1,1, "a", "a")', queryTimes=1) - tdSql.execute('DROP TABLE t1', queryTimes=1) - tdSql.execute('CREATE TABLE t1 USING meters TAGS(1, "a", "b", 1,1,1)') - tdSql.execute('INSERT INTO t1 VALUES("2024-10-24 11:59:00", 3,1,1,1,1,1,1, "a", "a")', queryTimes=1) - tdSql.execute('INSERT INTO t1 VALUES("2024-10-24 12:10:00", 4,1,1,1,1,1,1, "a", "a")', queryTimes=1) - tdSql.execute('INSERT INTO t1 VALUES("2024-10-24 12:20:00", 5,1,1,1,1,1,1, "a", "a")', queryTimes=1) + tdSql.execute("drop tsma test.tsma5") + self.create_recursive_tsma('tsma1', 'tsma_r', 'test', '1h', 'meters', ['avg(c1)', 'avg(c2)', 'count(ts)']) + tdSql.execute('INSERT INTO test.t1 VALUES("2024-10-24 11:45:00", 1,1,1,1,1,1,1, "a", "a")', queryTimes=1) + tdSql.execute('INSERT INTO test.t1 VALUES("2024-10-24 11:55:00", 2,1,1,1,1,1,1, "a", "a")', queryTimes=1) + tdSql.execute('DROP TABLE test.t1', queryTimes=1) + self.wait_query_err('desc test.`404e15422d96c8b5de9603c2296681b1`', 10, -2147473917) + self.wait_query_err('desc test.`82b56f091c4346369da0af777c3e580d`', 10, -2147473917) + self.wait_query_err('desc test.`163b7c69922cf6d83a98bfa44e52dade`', 10, -2147473917) + tdSql.execute('CREATE TABLE test.t1 USING test.meters TAGS(1, "a", "b", 1,1,1)') + tdSql.execute('INSERT INTO test.t1 VALUES("2024-10-24 11:59:00", 3,1,1,1,1,1,1, "a", "a")', queryTimes=1) + tdSql.execute('INSERT INTO test.t1 VALUES("2024-10-24 12:10:00", 4,1,1,1,1,1,1, "a", "a")', queryTimes=1) + tdSql.execute('INSERT INTO test.t1 VALUES("2024-10-24 12:20:00", 5,1,1,1,1,1,1, "a", "a")', queryTimes=1) tdSql.execute('FLUSH DATABASE test', queryTimes=1) + tdSql.query('SELECT * FROM test.t1', queryTimes=1) + tdSql.checkRows(3) + sql = 'SELECT * FROM test.`404e15422d96c8b5de9603c2296681b1`' + self.wait_query(sql, 3, 20) ## tsma1 output ctb for t1 + tdSql.query(sql, queryTimes=1) + tdSql.checkData(0,1, 1) + tdSql.checkData(1,1, 1) + tdSql.checkData(2,1, 1) + #sql = 'select * from test.`82b56f091c4346369da0af777c3e580d`' + #self.wait_query(sql, 2, 10) ## tsma2 output ctb for t1 + #tdSql.query(sql, queryTimes=1) + #tdSql.checkData(0, 1, 1) + #tdSql.checkData(1, 1, 2) + sql = 'select * from test.`163b7c69922cf6d83a98bfa44e52dade`' + self.wait_query(sql, 2, 20) ## tsma_r output ctb for t1 + tdSql.checkData(0, 1, 1) def test_create_tsma(self): function_name = sys._getframe().f_code.co_name From de8576a6a18165be6e71e4a287185158b1224323 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 5 Nov 2024 16:20:30 +0800 Subject: [PATCH 32/35] refine comments and functions --- source/dnode/mnode/impl/src/mndStb.c | 91 ++++++++++++++++--------- source/libs/executor/src/scanoperator.c | 8 +-- 2 files changed, 63 insertions(+), 36 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index e782d505a9..3712196f38 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -4063,8 +4063,8 @@ static int32_t mndProcessDropStbReqFromMNode(SRpcMsg *pReq) { } typedef struct SVDropTbVgReqs { - SVDropTbBatchReq req; - SVgroupInfo info; + SArray *pBatchReqs; + SVgroupInfo info; } SVDropTbVgReqs; typedef struct SMDropTbDbInfo { @@ -4086,16 +4086,17 @@ typedef struct SMDropTbTsmaInfos { } SMDropTbTsmaInfos; typedef struct SMndDropTbsWithTsmaCtx { - SHashObj *pTsmaMap; // - SHashObj *pDbMap; // - SHashObj *pVgMap; // , only for non tsma result child table - SHashObj *pTsmaTbVgMap; // , only for tsma result child table - SArray *pResTbNames; // SArray + SHashObj *pTsmaMap; // + SHashObj *pDbMap; // + SHashObj *pVgMap; // , only for non tsma result child table + SHashObj *pTsmaTbVgMap; // , only for tsma result child table + SArray *pResTbNames; // SArray } SMndDropTbsWithTsmaCtx; static int32_t mndDropTbAddTsmaResTbsForSingleVg(SMnode *pMnode, SMndDropTbsWithTsmaCtx *pCtx, SArray *pTbs, int32_t vgId); +static void destroySVDropTbBatchReqs(void *p); static void mndDestroyDropTbsWithTsmaCtx(SMndDropTbsWithTsmaCtx *p) { if (!p) return; @@ -4125,7 +4126,7 @@ static void mndDestroyDropTbsWithTsmaCtx(SMndDropTbsWithTsmaCtx *p) { void *pIter = taosHashIterate(p->pVgMap, NULL); while (pIter) { SVDropTbVgReqs *pReqs = pIter; - taosArrayDestroy(pReqs->req.pArray); + taosArrayDestroyEx(pReqs->pBatchReqs, destroySVDropTbBatchReqs); pIter = taosHashIterate(p->pVgMap, pIter); } taosHashCleanup(p->pVgMap); @@ -4135,7 +4136,7 @@ static void mndDestroyDropTbsWithTsmaCtx(SMndDropTbsWithTsmaCtx *p) { void *pIter = taosHashIterate(p->pTsmaTbVgMap, NULL); while (pIter) { SVDropTbVgReqs *pReqs = pIter; - taosArrayDestroy(pReqs->req.pArray); + taosArrayDestroyEx(pReqs->pBatchReqs, destroySVDropTbBatchReqs); pIter = taosHashIterate(p->pTsmaTbVgMap, pIter); } taosHashCleanup(p->pTsmaTbVgMap); @@ -4219,20 +4220,25 @@ static int32_t mndSetDropTbsRedoActions(SMnode *pMnode, STrans *pTrans, const SV return mndTransAppendRedoAction(pTrans, &action); } -static int32_t mndBuildDropTbRedoActions(SMnode* pMnode, STrans* pTrans, SHashObj* pVgMap, tmsg_t msgType) { +static int32_t mndBuildDropTbRedoActions(SMnode *pMnode, STrans *pTrans, SHashObj *pVgMap, tmsg_t msgType) { int32_t code = 0; - void* pIter = taosHashIterate(pVgMap, NULL); + void *pIter = taosHashIterate(pVgMap, NULL); while (pIter) { const SVDropTbVgReqs *pVgReqs = pIter; int32_t len = 0; - void *p = mndBuildVDropTbsReq(pMnode, &pVgReqs->info, &pVgReqs->req, &len); - if (!p) { - taosHashCancelIterate(pVgMap, pIter); - code = TSDB_CODE_MND_RETURN_VALUE_NULL; - if (terrno != 0) code = terrno; - break; + for (int32_t i = 0; i < taosArrayGetSize(pVgReqs->pBatchReqs) && code == TSDB_CODE_SUCCESS; ++i) { + SVDropTbBatchReq *pBatchReq = taosArrayGet(pVgReqs->pBatchReqs, i); + void *p = mndBuildVDropTbsReq(pMnode, &pVgReqs->info, pBatchReq, &len); + if (!p) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + break; + } + if ((code = mndSetDropTbsRedoActions(pMnode, pTrans, pVgReqs, p, len, msgType)) != 0) { + break; + } } - if ((code = mndSetDropTbsRedoActions(pMnode, pTrans, pVgReqs, p, len, msgType)) != 0) { + if (TSDB_CODE_SUCCESS != code) { taosHashCancelIterate(pVgMap, pIter); break; } @@ -4255,9 +4261,7 @@ static int32_t mndCreateDropTbsTxnPrepare(SRpcMsg *pRsp, SMndDropTbsWithTsmaCtx TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER); - //if ((code = mndBuildDropTbRedoActions(pMnode, pTrans, pCtx->pVgMap, TDMT_VND_DROP_TSMA_CTB)) != 0) goto _OVER; if ((code = mndBuildDropTbRedoActions(pMnode, pTrans, pCtx->pVgMap, TDMT_VND_DROP_TABLE)) != 0) goto _OVER; - //if ((code = mndBuildDropTbRedoActions(pMnode, pTrans, pCtx->pTsmaTbVgMap, TDMT_VND_DROP_TABLE)) != 0) goto _OVER; if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER; _OVER: @@ -4295,26 +4299,51 @@ _OVER: TAOS_RETURN(code); } +static int32_t createDropTbBatchReq(const SVDropTbReq *pReq, SVDropTbBatchReq *pBatchReq) { + pBatchReq->nReqs = 1; + pBatchReq->pArray = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVDropTbReq)); + if (!pBatchReq->pArray) return terrno; + if (taosArrayPush(pBatchReq->pArray, pReq) == NULL) { + taosArrayDestroy(pBatchReq->pArray); + pBatchReq->pArray = NULL; + return terrno; + } + return TSDB_CODE_SUCCESS; +} + +static void destroySVDropTbBatchReqs(void *p) { + SVDropTbBatchReq *pReq = p; + taosArrayDestroy(pReq->pArray); + pReq->pArray = NULL; +} + static int32_t mndDropTbAdd(SMnode *pMnode, SHashObj *pVgHashMap, const SVgroupInfo *pVgInfo, char *name, tb_uid_t suid, bool ignoreNotExists) { SVDropTbReq req = {.name = name, .suid = suid, .igNotExists = ignoreNotExists, .uid = 0}; - SVDropTbVgReqs *pReqs = taosHashGet(pVgHashMap, &pVgInfo->vgId, sizeof(pVgInfo->vgId)); - SVDropTbVgReqs reqs = {0}; - if (pReqs == NULL) { - reqs.info = *pVgInfo; - reqs.req.pArray = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVDropTbReq)); - if (reqs.req.pArray == NULL) { + SVDropTbVgReqs *pVgReqs = taosHashGet(pVgHashMap, &pVgInfo->vgId, sizeof(pVgInfo->vgId)); + SVDropTbVgReqs vgReqs = {0}; + if (pVgReqs == NULL) { + vgReqs.info = *pVgInfo; + vgReqs.pBatchReqs = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVDropTbBatchReq)); + if (!vgReqs.pBatchReqs) return terrno; + SVDropTbBatchReq batchReq = {0}; + int32_t code = createDropTbBatchReq(&req, &batchReq); + if (TSDB_CODE_SUCCESS != code) return code; + if (taosArrayPush(vgReqs.pBatchReqs, &batchReq) == NULL) { + taosArrayDestroy(batchReq.pArray); return terrno; } - if (taosArrayPush(reqs.req.pArray, &req) == NULL) { - return terrno; - } - if (taosHashPut(pVgHashMap, &pVgInfo->vgId, sizeof(pVgInfo->vgId), &reqs, sizeof(reqs)) != 0) { + if (taosHashPut(pVgHashMap, &pVgInfo->vgId, sizeof(pVgInfo->vgId), &vgReqs, sizeof(vgReqs)) != 0) { + taosArrayDestroyEx(vgReqs.pBatchReqs, destroySVDropTbBatchReqs); return terrno; } } else { - if (taosArrayPush(pReqs->req.pArray, &req) == NULL) { + SVDropTbBatchReq batchReq = {0}; + int32_t code = createDropTbBatchReq(&req, &batchReq); + if (TSDB_CODE_SUCCESS != code) return code; + if (taosArrayPush(pVgReqs->pBatchReqs, &batchReq) == NULL) { + taosArrayDestroy(batchReq.pArray); return terrno; } } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 4cd32589d8..b13572d5b2 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3306,10 +3306,8 @@ static int32_t setBlockGroupIdByUid(SStreamScanInfo* pInfo, SSDataBlock* pBlock) int32_t rows = pBlock->info.rows; if (!pInfo->partitionSup.needCalc) { for (int32_t i = 0; i < rows; i++) { - qInfo("wjm, get uid: %"PRIu64, uidCol[i]); uint64_t groupId = getGroupIdByUid(pInfo, uidCol[i]); - qInfo("wjm, get groupid: %"PRIu64, groupId); - code = colDataSetVal(pGpCol, i, (const char*)(uidCol + i), false); + code = colDataSetVal(pGpCol, i, (const char*)&groupId, false); QUERY_CHECK_CODE(code, lino, _end); } } @@ -3538,6 +3536,7 @@ static int32_t deletePartName(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32 int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; for (int32_t i = 0; i < pBlock->info.rows; i++) { + // uid is the same as gid SColumnInfoData* pGpIdCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); SColumnInfoData* pTbnameCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); int64_t* gpIdCol = (int64_t*)pGpIdCol->pData; @@ -3563,6 +3562,7 @@ static int32_t deletePartName(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32 code = pInfo->stateStore.streamStateDeleteParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, gpIdCol[i]); QUERY_CHECK_CODE(code, lino, _end); pBlock->info.id.groupId = gpIdCol[i]; + // currently, only one valid row in pBlock memcpy(pBlock->info.parTbName, varTbName + VARSTR_HEADER_SIZE, TSDB_TABLE_NAME_LEN + 1); } @@ -3814,8 +3814,6 @@ FETCH_NEXT_BLOCK: } break; case STREAM_DROP_CHILD_TABLE: { int32_t deleteNum = 0; - code = setBlockGroupIdByUid(pInfo, pBlock); - QUERY_CHECK_CODE(code, lino, _end); code = deletePartName(pInfo, pBlock, &deleteNum); QUERY_CHECK_CODE(code, lino, _end); if (deleteNum == 0) goto FETCH_NEXT_BLOCK; From a70b4e28c869efe1f6901cc698f50b1e3cd995be Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 5 Nov 2024 17:43:34 +0800 Subject: [PATCH 33/35] fix tests --- source/dnode/vnode/src/tq/tqRead.c | 2 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index ea2f3f91be..d924e97ae3 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -366,7 +366,7 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con } else if (pCont->msgType == TDMT_VND_DELETE) { void* pBody = POINTER_SHIFT(pCont->body, sizeof(SMsgHead)); int32_t len = pCont->bodyLen - sizeof(SMsgHead); - EStreamType blockType = pCont->msgType == STREAM_DELETE_DATA; + EStreamType blockType = STREAM_DELETE_DATA; code = tqExtractDelDataBlock(pBody, len, ver, (void**)pItem, 0, blockType); if (code == TSDB_CODE_SUCCESS) { if (*pItem == NULL) { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 723ebb333d..64bfa5a04d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -561,7 +561,7 @@ int32_t vnodePreProcessDropTbMsg(SVnode* pVnode, SRpcMsg* pMsg) { SVDropTbBatchReq receivedBatchReqs = {0}; SVDropTbBatchReq sentBatchReqs = {0}; - tDecoderInit(&dc, pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead)); + tDecoderInit(&dc, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pMsg->contLen - sizeof(SMsgHead)); code = tDecodeSVDropTbBatchReq(&dc, &receivedBatchReqs); if (code < 0) { @@ -591,7 +591,7 @@ int32_t vnodePreProcessDropTbMsg(SVnode* pVnode, SRpcMsg* pMsg) { sentBatchReqs.nReqs = sentBatchReqs.pArray->size; tEncodeSize(tEncodeSVDropTbBatchReq, &sentBatchReqs, size, code); - tEncoderInit(&ec, pMsg->pCont + sizeof(SMsgHead), size); + tEncoderInit(&ec, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), size); code = tEncodeSVDropTbBatchReq(&ec, &sentBatchReqs); tEncoderClear(&ec); if (code != TSDB_CODE_SUCCESS) { From 2ddd07142a23f5d4aca1cfc30a0ec547c62015ee Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 5 Nov 2024 19:20:12 +0800 Subject: [PATCH 34/35] remove logs --- source/dnode/vnode/src/meta/metaTable.c | 1 - source/dnode/vnode/src/tq/tqSink.c | 19 ++----- source/dnode/vnode/src/vnd/vnodeSvr.c | 72 ------------------------ source/libs/executor/src/groupoperator.c | 7 --- source/libs/executor/src/scanoperator.c | 3 +- source/libs/stream/src/streamDispatch.c | 1 - source/libs/stream/src/streamState.c | 3 - tests/system-test/2-query/tsma.py | 6 +- 8 files changed, 11 insertions(+), 101 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 93324d6eb4..5c3516a962 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -1233,7 +1233,6 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi metaWLock(pMeta); rc = metaDropTableByUid(pMeta, uid, &type, &suid, &sysTbl); - metaInfo("wjm meta drop table by uid: %"PRId64, uid); metaULock(pMeta); if (rc < 0) goto _exit; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 3c525e7e9c..6f8a9022dd 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -53,7 +53,7 @@ static int32_t checkTagSchema(SStreamTask* pTask, SVnode* pVnode); static void reubuildAndSendMultiResBlock(SStreamTask* pTask, const SArray* pBlocks, SVnode* pVnode, int64_t earlyTs); static int32_t handleResultBlockMsg(SStreamTask* pTask, SSDataBlock* pDataBlock, int32_t index, SVnode* pVnode, int64_t earlyTs); -static int32_t doWaitForDstTableDropped(SVnode* pVnode, SStreamTask* pTask, const char* dstTableName, int64_t uid); +static int32_t doWaitForDstTableDropped(SVnode* pVnode, SStreamTask* pTask, const char* dstTableName); int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq, const char* pIdStr, bool newSubTableRule) { @@ -442,7 +442,7 @@ static int32_t doBuildAndSendDropTableMsg(SVnode* pVnode, char* pStbFullname, SS SVDropTbBatchReq batchReq = {0}; SVDropTbReq req = {0}; - if (rows <= 0 || pTask->subtableWithoutMd5 == 0) return TSDB_CODE_SUCCESS; + if (rows <= 0 || rows > 1 || pTask->subtableWithoutMd5 == 0) return TSDB_CODE_SUCCESS; batchReq.pArray = taosArrayInit(rows, sizeof(SVDropTbReq)); if (!batchReq.pArray) return terrno; @@ -451,15 +451,12 @@ static int32_t doBuildAndSendDropTableMsg(SVnode* pVnode, char* pStbFullname, SS req.igNotExists = true; SColumnInfoData* pTbNameCol = taosArrayGet(pDataBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); - SColumnInfoData* pUidCol = taosArrayGet(pDataBlock->pDataBlock, UID_COLUMN_INDEX); char tbName[TSDB_TABLE_NAME_LEN + 1] = {0}; for (int32_t i = 0; i < rows; ++i) { void* pData = colDataGetVarData(pTbNameCol, i); memcpy(tbName, varDataVal(pData), varDataLen(pData)); tbName[varDataLen(pData) + 1] = 0; req.name = tbName; - // TODO wjm remove uid, it's not my uid - req.uid = *(int64_t*)colDataGetData(pUidCol, i); if (taosArrayPush(batchReq.pArray, &req) == NULL) { TSDB_CHECK_CODE(terrno, lino, _exit); } @@ -467,9 +464,9 @@ static int32_t doBuildAndSendDropTableMsg(SVnode* pVnode, char* pStbFullname, SS SMetaReader mr = {0}; metaReaderDoInit(&mr, pVnode->pMeta, META_READER_LOCK); - // TODO wjm handle only one table + // only one row code = metaGetTableEntryByName(&mr, tbName); - if (isValidDstChildTable(&mr, TD_VID(pVnode), tbName, pTask->outputInfo.tbSink.stbUid)) { + if (TSDB_CODE_SUCCESS == code && isValidDstChildTable(&mr, TD_VID(pVnode), tbName, pTask->outputInfo.tbSink.stbUid)) { STableSinkInfo* pTableSinkInfo = NULL; bool alreadyCached = doGetSinkTableInfoFromCache(pTask->outputInfo.tbSink.pTbInfo, pDataBlock->info.id.groupId, &pTableSinkInfo); if (alreadyCached) { @@ -485,9 +482,7 @@ static int32_t doBuildAndSendDropTableMsg(SVnode* pVnode, char* pStbFullname, SS void* pData = colDataGetVarData(pTbNameCol, i); memcpy(tbName, varDataVal(pData), varDataLen(pData)); tbName[varDataLen(pData) + 1] = 0; - int64_t uid = *(int64_t*)colDataGetData(pUidCol, i); - // TODO wjm remove uid it's not my uid - code = doWaitForDstTableDropped(pVnode, pTask, tbName, uid); + code = doWaitForDstTableDropped(pVnode, pTask, tbName); TSDB_CHECK_CODE(code, lino, _exit); } @@ -906,7 +901,7 @@ int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkI return TSDB_CODE_SUCCESS; } -static int32_t doWaitForDstTableDropped(SVnode* pVnode, SStreamTask* pTask, const char* dstTableName, int64_t uid) { +static int32_t doWaitForDstTableDropped(SVnode* pVnode, SStreamTask* pTask, const char* dstTableName) { int32_t vgId = TD_VID(pVnode); int64_t suid = pTask->outputInfo.tbSink.stbUid; const char* id = pTask->id.idStr; @@ -920,7 +915,6 @@ static int32_t doWaitForDstTableDropped(SVnode* pVnode, SStreamTask* pTask, cons metaReaderDoInit(&mr, pVnode->pMeta, META_READER_LOCK); int32_t code = metaGetTableEntryByName(&mr, dstTableName); if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { - tqDebug("wjm s-task:%s table:%s has been dropped", id, dstTableName); metaReaderClear(&mr); break; } else if (TSDB_CODE_SUCCESS == code) { @@ -929,7 +923,6 @@ static int32_t doWaitForDstTableDropped(SVnode* pVnode, SStreamTask* pTask, cons taosMsleep(100); tqDebug("s-task:%s wait 100ms for table:%s drop", id, dstTableName); } else { - tqDebug("wjm s-task:%s table:%s exist, but not mine", id, dstTableName); metaReaderClear(&mr); break; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 64bfa5a04d..6702b8b588 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -483,75 +483,6 @@ static int32_t vnodePreProcessArbCheckSyncMsg(SVnode *pVnode, SRpcMsg *pMsg) { return code; } -static int32_t vnodePreProcessDropTSmaCtbMsg(SVnode *pVnode, SRpcMsg *pMsg) { - SVDropTbBatchReq dropReq = {0}; - int32_t code = 0; - int32_t lino = 0; - SDecoder dc = {0}; - SEncoder ec = {0}; - int32_t nTbs = 0; - SDeleteRes res = {0}; - int32_t size = 0; - uint8_t *pCont = NULL; - tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead)); - if (tDecodeSVDropTbBatchReq(&dc, &dropReq) < 0) { - code = TSDB_CODE_INVALID_MSG; - TSDB_CHECK_CODE(code, lino, _exit); - } - nTbs = dropReq.nReqs; - res.skey = INT64_MIN; - res.ekey = INT64_MAX; - res.affectedRows = 1; - res.uidList = taosArrayInit(nTbs, sizeof(tb_uid_t)); - if (!res.uidList) { - code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); - } - - vDebug("vnode preprocess drop tsma ctb, vgId:%d tb num: %d", TD_VID(pVnode), nTbs); - for (int32_t i = 0; i < nTbs; ++i) { - SVDeleteRsp rsp = {.affectedRows = 1}; - tb_uid_t uid = metaGetTableEntryUidByName(pVnode->pMeta, dropReq.pReqs[i].name); - if (uid == 0) { - vWarn("vgId:%d, drop tsma ctb:%s not found", TD_VID(pVnode), dropReq.pReqs[i].name); - continue; - } - if (NULL == taosArrayPush(res.uidList, &uid)) { - code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); - } - } - - tEncodeSize(tEncodeDeleteRes, &res, size, code); - pCont = rpcMallocCont(size + sizeof(SMsgHead)); - if (!pCont) { - code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); - } - ((SMsgHead *)pCont)->contLen = size + sizeof(SMsgHead); - ((SMsgHead *)pCont)->vgId = TD_VID(pVnode); - - tEncoderInit(&ec, pCont + sizeof(SMsgHead), size); - code = tEncodeDeleteRes(&ec, &res); - tEncoderClear(&ec); - if (code != 0) { - vError("vgId:%d %s failed to encode delete response", TD_VID(pVnode), __func__); - TSDB_CHECK_CODE(code, lino, _exit); - } - rpcFreeCont(pMsg->pCont); - pMsg->pCont = pCont; - pCont = NULL; - pMsg->contLen = size + sizeof(SMsgHead); - -_exit: - if (res.uidList) { - taosArrayDestroy(res.uidList); - } - tDecoderClear(&dc); - rpcFreeCont(pCont); - return code; -} - int32_t vnodePreProcessDropTbMsg(SVnode* pVnode, SRpcMsg* pMsg) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; @@ -1294,11 +1225,9 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, continue; } - vInfo("wjm process create tb req:%s, uid: %"PRId64, pCreateReq->name, pCreateReq->uid); // do create table if (metaCreateTable(pVnode->pMeta, ver, pCreateReq, &cRsp.pMeta) < 0) { if (pCreateReq->flags & TD_CREATE_IF_NOT_EXISTS && terrno == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { - vInfo("wjm already exists-----------------"); cRsp.code = TSDB_CODE_SUCCESS; } else { cRsp.code = terrno; @@ -1375,7 +1304,6 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, } _exit: - vInfo("wjm process create table request exit"); tDeleteSVCreateTbBatchReq(&req); taosArrayDestroyEx(rsp.pArray, tFreeSVCreateTbRsp); taosArrayDestroy(tbUids); diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index d5b4a52be7..d6e3d26267 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -1326,7 +1326,6 @@ int32_t appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTag int32_t winCode = TSDB_CODE_SUCCESS; code = pAPI->streamStateGetParName(pState, groupId, &pValue, true, &winCode); QUERY_CHECK_CODE(code, lino, _end); - qInfo("wjm group id: %"PRId64 " winCode: %d, block type: %d", groupId, winCode, pSrcBlock->info.type); if (winCode != TSDB_CODE_SUCCESS) { SSDataBlock* pTmpBlock = NULL; code = blockCopyOneRow(pSrcBlock, rowId, &pTmpBlock); @@ -1335,8 +1334,6 @@ int32_t appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTag memset(pTmpBlock->info.parTbName, 0, TSDB_TABLE_NAME_LEN); pTmpBlock->info.id.groupId = groupId; char* tbName = pSrcBlock->info.parTbName; - printSpecDataBlock(pSrcBlock, "wjm", "recv", "wjm"); - printSpecDataBlock(pTmpBlock, "wjm", "recv", "wjm"); if (pTableSup->numOfExprs > 0) { code = projectApplyFunctions(pTableSup->pExprInfo, pDestBlock, pTmpBlock, pTableSup->pCtx, pTableSup->numOfExprs, NULL); @@ -1344,19 +1341,15 @@ int32_t appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTag SColumnInfoData* pTbCol = taosArrayGet(pDestBlock->pDataBlock, UD_TABLE_NAME_COLUMN_INDEX); QUERY_CHECK_NULL(pTbCol, code, lino, _end, terrno); - printSpecDataBlock(pSrcBlock, "wjm", "recv", "wjm"); - printSpecDataBlock(pTmpBlock, "wjm", "recv", "wjm"); memset(tbName, 0, TSDB_TABLE_NAME_LEN); int32_t len = 0; if (colDataIsNull_s(pTbCol, pDestBlock->info.rows - 1)) { - qInfo("wjm calculated tbnameis null"); len = 1; tbName[0] = 0; } else { void* pData = colDataGetData(pTbCol, pDestBlock->info.rows - 1); len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1); memcpy(tbName, varDataVal(pData), len); - qInfo("wjm calculated tbname: %s", tbName); code = pAPI->streamStatePutParName(pState, groupId, tbName); QUERY_CHECK_CODE(code, lino, _end); } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b13572d5b2..84dde6a579 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3542,11 +3542,10 @@ static int32_t deletePartName(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32 int64_t* gpIdCol = (int64_t*)pGpIdCol->pData; void* pParName = NULL; int32_t winCode = 0; - // TODO wjm test remove non stream child tables code = pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, gpIdCol[i], &pParName, false, &winCode); if (TSDB_CODE_SUCCESS == code && winCode != 0) { - qInfo("delete stream part Name for:%"PRId64 " not found", gpIdCol[i]); + qDebug("delete stream part Name for:%"PRId64 " not found", gpIdCol[i]); colDataSetNULL(pTbnameCol, i); continue; } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 6e27cd651e..e0fa199199 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -758,7 +758,6 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S SUseDbRsp* pDbInfo = &pTask->outputInfo.shuffleDispatcher.dbInfo; hashValue = taosGetTbHashVal(ctbName, strlen(ctbName), pDbInfo->hashMethod, pDbInfo->hashPrefix, pDbInfo->hashSuffix); - stInfo("wjm ctbname for dispatch: %s, pDataBlock.info.parTbName: %s", ctbName, pDataBlock->info.parTbName); SBlockName bln = {0}; bln.hashValue = hashValue; memcpy(bln.parTbName, pDataBlock->info.parTbName, strlen(pDataBlock->info.parTbName)); diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 91ed919446..26c97cbb6e 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -480,7 +480,6 @@ int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char if (tSimpleHashGet(pState->parNameMap, &groupId, sizeof(int64_t)) == NULL) { if (tSimpleHashGetSize(pState->parNameMap) < MAX_TABLE_NAME_NUM) { code = tSimpleHashPut(pState->parNameMap, &groupId, sizeof(int64_t), tbname, TSDB_TABLE_NAME_LEN); - qInfo("wjm put group id into parnamemap: %"PRId64 " cur mapsize: %d", groupId, tSimpleHashGetSize(pState->parNameMap)); QUERY_CHECK_CODE(code, lino, _end); } code = streamStatePutParName_rocksdb(pState, groupId, tbname); @@ -506,7 +505,6 @@ int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal (*pWinCode) = streamStateGetParName_rocksdb(pState, groupId, pVal); if ((*pWinCode) == TSDB_CODE_SUCCESS && tSimpleHashGetSize(pState->parNameMap) < MAX_TABLE_NAME_NUM) { code = tSimpleHashPut(pState->parNameMap, &groupId, sizeof(int64_t), *pVal, TSDB_TABLE_NAME_LEN); - qInfo("wjm put group id into parnamemap: %"PRId64 " cur mapsize: %d", groupId, tSimpleHashGetSize(pState->parNameMap)); QUERY_CHECK_CODE(code, lino, _end); } goto _end; @@ -528,7 +526,6 @@ _end: } int32_t streamStateDeleteParName(SStreamState* pState, int64_t groupId) { - qTrace("wjm delete par for group:%"PRId64 " parnameMapsize: %d", groupId, tSimpleHashGetSize(pState->parNameMap)); int32_t code = tSimpleHashRemove(pState->parNameMap, &groupId, sizeof(int64_t)); qTrace("catche %s at line %d res %d", __func__, __LINE__, code); code = streamStateDeleteParName_rocksdb(pState, groupId); diff --git a/tests/system-test/2-query/tsma.py b/tests/system-test/2-query/tsma.py index 77e57bd36d..1c688d568c 100644 --- a/tests/system-test/2-query/tsma.py +++ b/tests/system-test/2-query/tsma.py @@ -1237,10 +1237,11 @@ class TDTestCase: clust_dnode_nums = len(cluster_dnode_list) if clust_dnode_nums > 1: self.test_redistribute_vgroups() - self.test_td_32519() + tdSql.execute("drop tsma test.tsma5") + for _ in range(4): + self.test_td_32519() def test_td_32519(self): - tdSql.execute("drop tsma test.tsma5") self.create_recursive_tsma('tsma1', 'tsma_r', 'test', '1h', 'meters', ['avg(c1)', 'avg(c2)', 'count(ts)']) tdSql.execute('INSERT INTO test.t1 VALUES("2024-10-24 11:45:00", 1,1,1,1,1,1,1, "a", "a")', queryTimes=1) tdSql.execute('INSERT INTO test.t1 VALUES("2024-10-24 11:55:00", 2,1,1,1,1,1,1, "a", "a")', queryTimes=1) @@ -1269,6 +1270,7 @@ class TDTestCase: sql = 'select * from test.`163b7c69922cf6d83a98bfa44e52dade`' self.wait_query(sql, 2, 20) ## tsma_r output ctb for t1 tdSql.checkData(0, 1, 1) + self.drop_tsma('tsma_r', 'test') def test_create_tsma(self): function_name = sys._getframe().f_code.co_name From 274a7fd876b6c5f841be16f7e1fecb753d5e73a1 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Wed, 6 Nov 2024 09:55:18 +0800 Subject: [PATCH 35/35] refine unused codes --- source/dnode/mnode/impl/src/mndStb.c | 196 +-------------------------- source/dnode/vnode/src/tq/tqSink.c | 29 ++-- source/libs/stream/src/streamState.c | 8 +- 3 files changed, 23 insertions(+), 210 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 3712196f38..eb6c326d1e 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -4086,42 +4086,16 @@ typedef struct SMDropTbTsmaInfos { } SMDropTbTsmaInfos; typedef struct SMndDropTbsWithTsmaCtx { - SHashObj *pTsmaMap; // - SHashObj *pDbMap; // - SHashObj *pVgMap; // , only for non tsma result child table - SHashObj *pTsmaTbVgMap; // , only for tsma result child table - SArray *pResTbNames; // SArray + SHashObj *pVgMap; // } SMndDropTbsWithTsmaCtx; -static int32_t mndDropTbAddTsmaResTbsForSingleVg(SMnode *pMnode, SMndDropTbsWithTsmaCtx *pCtx, SArray *pTbs, +static int32_t mndDropTbForSingleVg(SMnode *pMnode, SMndDropTbsWithTsmaCtx *pCtx, SArray *pTbs, int32_t vgId); static void destroySVDropTbBatchReqs(void *p); static void mndDestroyDropTbsWithTsmaCtx(SMndDropTbsWithTsmaCtx *p) { if (!p) return; - if (p->pDbMap) { - void *pIter = taosHashIterate(p->pDbMap, NULL); - while (pIter) { - SMDropTbDbInfo *pInfo = pIter; - taosArrayDestroy(pInfo->dbVgInfos); - pIter = taosHashIterate(p->pDbMap, pIter); - } - taosHashCleanup(p->pDbMap); - } - if (p->pResTbNames) { - taosArrayDestroyP(p->pResTbNames, taosMemoryFree); - } - if (p->pTsmaMap) { - void *pIter = taosHashIterate(p->pTsmaMap, NULL); - while (pIter) { - SMDropTbTsmaInfos *pInfos = pIter; - taosArrayDestroy(pInfos->pTsmaInfos); - pIter = taosHashIterate(p->pTsmaMap, pIter); - } - taosHashCleanup(p->pTsmaMap); - } - if (p->pVgMap) { void *pIter = taosHashIterate(p->pVgMap, NULL); while (pIter) { @@ -4131,16 +4105,6 @@ static void mndDestroyDropTbsWithTsmaCtx(SMndDropTbsWithTsmaCtx *p) { } taosHashCleanup(p->pVgMap); } - - if (p->pTsmaTbVgMap) { - void *pIter = taosHashIterate(p->pTsmaTbVgMap, NULL); - while (pIter) { - SVDropTbVgReqs *pReqs = pIter; - taosArrayDestroyEx(pReqs->pBatchReqs, destroySVDropTbBatchReqs); - pIter = taosHashIterate(p->pTsmaTbVgMap, pIter); - } - taosHashCleanup(p->pTsmaTbVgMap); - } taosMemoryFree(p); } @@ -4148,18 +4112,6 @@ static int32_t mndInitDropTbsWithTsmaCtx(SMndDropTbsWithTsmaCtx **ppCtx) { int32_t code = 0; SMndDropTbsWithTsmaCtx *pCtx = taosMemoryCalloc(1, sizeof(SMndDropTbsWithTsmaCtx)); if (!pCtx) return terrno; - pCtx->pTsmaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); - if (!pCtx->pTsmaMap) { - code = terrno; - goto _end; - } - - pCtx->pDbMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - if (!pCtx->pDbMap) { - code = terrno; - goto _end; - } - pCtx->pResTbNames = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); pCtx->pVgMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); if (!pCtx->pVgMap) { @@ -4167,11 +4119,6 @@ static int32_t mndInitDropTbsWithTsmaCtx(SMndDropTbsWithTsmaCtx **ppCtx) { goto _end; } - pCtx->pTsmaTbVgMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); - if (!pCtx->pTsmaTbVgMap) { - code = terrno; - goto _end; - } *ppCtx = pCtx; _end: if (code) mndDestroyDropTbsWithTsmaCtx(pCtx); @@ -4286,7 +4233,7 @@ static int32_t mndProcessDropTbWithTsma(SRpcMsg *pReq) { if (code) goto _OVER; for (int32_t i = 0; i < dropReq.pVgReqs->size; ++i) { SMDropTbReqsOnSingleVg *pReq = taosArrayGet(dropReq.pVgReqs, i); - code = mndDropTbAddTsmaResTbsForSingleVg(pMnode, pCtx, pReq->pTbs, pReq->vgInfo.vgId); + code = mndDropTbForSingleVg(pMnode, pCtx, pReq->pTbs, pReq->vgInfo.vgId); if (code) goto _OVER; } code = mndCreateDropTbsTxnPrepare(pReq, pCtx); @@ -4350,61 +4297,7 @@ static int32_t mndDropTbAdd(SMnode *pMnode, SHashObj *pVgHashMap, const SVgroupI return 0; } -int vgInfoCmp(const void *lp, const void *rp) { - SVgroupInfo *pLeft = (SVgroupInfo *)lp; - SVgroupInfo *pRight = (SVgroupInfo *)rp; - if (pLeft->hashBegin < pRight->hashBegin) { - return -1; - } else if (pLeft->hashBegin > pRight->hashBegin) { - return 1; - } - - return 0; -} - -static int32_t mndGetDbVgInfoForTsma(SMnode *pMnode, const char *dbname, SMDropTbTsmaInfo *pInfo) { - int32_t code = 0; - SDbObj *pDb = mndAcquireDb(pMnode, dbname); - if (!pDb) { - code = TSDB_CODE_MND_DB_NOT_EXIST; - goto _end; - } - - pInfo->dbInfo.dbVgInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); - if (!pInfo->dbInfo.dbVgInfos) { - code = terrno; - goto _end; - } - mndBuildDBVgroupInfo(pDb, pMnode, pInfo->dbInfo.dbVgInfos); - taosArraySort(pInfo->dbInfo.dbVgInfos, vgInfoCmp); - - pInfo->dbInfo.hashPrefix = pDb->cfg.hashPrefix; - pInfo->dbInfo.hashSuffix = pDb->cfg.hashSuffix; - pInfo->dbInfo.hashMethod = pDb->cfg.hashMethod; - -_end: - if (pDb) mndReleaseDb(pMnode, pDb); - if (code && pInfo->dbInfo.dbVgInfos) { - taosArrayDestroy(pInfo->dbInfo.dbVgInfos); - pInfo->dbInfo.dbVgInfos = NULL; - } - TAOS_RETURN(code); -} - -int32_t vgHashValCmp(const void *lp, const void *rp) { - uint32_t *key = (uint32_t *)lp; - SVgroupInfo *pVg = (SVgroupInfo *)rp; - - if (*key < pVg->hashBegin) { - return -1; - } else if (*key > pVg->hashEnd) { - return 1; - } - - return 0; -} - -static int32_t mndDropTbAddTsmaResTbsForSingleVg(SMnode *pMnode, SMndDropTbsWithTsmaCtx *pCtx, SArray *pTbs, +static int32_t mndDropTbForSingleVg(SMnode *pMnode, SMndDropTbsWithTsmaCtx *pCtx, SArray *pTbs, int32_t vgId) { int32_t code = 0; @@ -4420,88 +4313,9 @@ static int32_t mndDropTbAddTsmaResTbsForSingleVg(SMnode *pMnode, SMndDropTbsWith vgInfo.epSet = mndGetVgroupEpset(pMnode, pVgObj); mndReleaseVgroup(pMnode, pVgObj); - // get all stb uids - for (int32_t i = 0; i < pTbs->size; ++i) { - const SVDropTbReq *pTb = taosArrayGet(pTbs, i); - if (taosHashGet(pCtx->pTsmaMap, &pTb->suid, sizeof(pTb->suid))) { - } else { - SMDropTbTsmaInfos infos = {0}; - infos.pTsmaInfos = taosArrayInit(2, sizeof(SMDropTbTsmaInfo)); - if (!infos.pTsmaInfos) { - code = terrno; - goto _end; - } - if (taosHashPut(pCtx->pTsmaMap, &pTb->suid, sizeof(pTb->suid), &infos, sizeof(infos)) != 0) { - code = terrno; - goto _end; - } - } - } - - void *pIter = NULL; - SSmaObj *pSma = NULL; - char buf[TSDB_TABLE_FNAME_LEN] = {0}; - // get used tsmas and it's dbs - while (1) { - pIter = sdbFetch(pMnode->pSdb, SDB_SMA, pIter, (void **)&pSma); - if (!pIter) break; - SMDropTbTsmaInfos *pInfos = taosHashGet(pCtx->pTsmaMap, &pSma->stbUid, sizeof(pSma->stbUid)); - if (pInfos) { - SMDropTbTsmaInfo info = {0}; - int32_t len = sprintf(buf, "%s", pSma->name); - sprintf(info.tsmaResTbDbFName, "%s", pSma->db); - snprintf(info.tsmaResTbNamePrefix, TSDB_TABLE_FNAME_LEN, "%s", buf); - SMDropTbDbInfo *pDbInfo = taosHashGet(pCtx->pDbMap, pSma->db, TSDB_DB_FNAME_LEN); - info.suid = pSma->dstTbUid; - if (!pDbInfo) { - code = mndGetDbVgInfoForTsma(pMnode, pSma->db, &info); - if (code != TSDB_CODE_SUCCESS) { - sdbCancelFetch(pMnode->pSdb, pIter); - sdbRelease(pMnode->pSdb, pSma); - goto _end; - } - if (taosHashPut(pCtx->pDbMap, pSma->db, TSDB_DB_FNAME_LEN, &info.dbInfo, sizeof(SMDropTbDbInfo)) != 0) { - sdbCancelFetch(pMnode->pSdb, pIter); - sdbRelease(pMnode->pSdb, pSma); - goto _end; - } - } else { - info.dbInfo = *pDbInfo; - } - if (taosArrayPush(pInfos->pTsmaInfos, &info) == NULL) { - code = terrno; - sdbCancelFetch(pMnode->pSdb, pIter); - sdbRelease(pMnode->pSdb, pSma); - goto _end; - } - } - sdbRelease(pMnode->pSdb, pSma); - } - - // generate vg req map for (int32_t i = 0; i < pTbs->size; ++i) { SVDropTbReq *pTb = taosArrayGet(pTbs, i); TAOS_CHECK_GOTO(mndDropTbAdd(pMnode, pCtx->pVgMap, &vgInfo, pTb->name, pTb->suid, pTb->igNotExists), NULL, _end); - - SMDropTbTsmaInfos *pInfos = taosHashGet(pCtx->pTsmaMap, &pTb->suid, sizeof(pTb->suid)); - SArray *pVgInfos = NULL; - char buf[TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN + 1]; - char resTbFullName[TSDB_TABLE_FNAME_LEN + 1] = {0}; - for (int32_t j = 0; j < pInfos->pTsmaInfos->size; ++j) { - SMDropTbTsmaInfo *pInfo = taosArrayGet(pInfos->pTsmaInfos, j); - int32_t len = sprintf(buf, "%s_%s", pInfo->tsmaResTbNamePrefix, pTb->name); - len = taosCreateMD5Hash(buf, len); - len = snprintf(resTbFullName, TSDB_TABLE_FNAME_LEN + 1, "%s.%s", pInfo->tsmaResTbDbFName, buf); - uint32_t hashVal = taosGetTbHashVal(resTbFullName, len, pInfo->dbInfo.hashMethod, pInfo->dbInfo.hashPrefix, - pInfo->dbInfo.hashSuffix); - const SVgroupInfo *pVgInfo = taosArraySearch(pInfo->dbInfo.dbVgInfos, &hashVal, vgHashValCmp, TD_EQ); - void *p = taosStrdup(resTbFullName + strlen(pInfo->tsmaResTbDbFName) + TSDB_NAME_DELIMITER_LEN); - if (taosArrayPush(pCtx->pResTbNames, &p) == NULL) { - code = terrno; - goto _end; - } - TAOS_CHECK_GOTO(mndDropTbAdd(pMnode, pCtx->pTsmaTbVgMap, pVgInfo, p, pInfo->suid, true), NULL, _end); - } } _end: return code; @@ -4529,7 +4343,7 @@ static int32_t mndProcessFetchTtlExpiredTbs(SRpcMsg *pRsp) { code = mndInitDropTbsWithTsmaCtx(&pCtx); if (code) goto _end; - code = mndDropTbAddTsmaResTbsForSingleVg(pMnode, pCtx, rsp.pExpiredTbs, rsp.vgId); + code = mndDropTbForSingleVg(pMnode, pCtx, rsp.pExpiredTbs, rsp.vgId); if (code) goto _end; code = mndCreateDropTbsTxnPrepare(pRsp, pCtx); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 6f8a9022dd..3f4ff7f3d9 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -451,20 +451,19 @@ static int32_t doBuildAndSendDropTableMsg(SVnode* pVnode, char* pStbFullname, SS req.igNotExists = true; SColumnInfoData* pTbNameCol = taosArrayGet(pDataBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); - char tbName[TSDB_TABLE_NAME_LEN + 1] = {0}; - for (int32_t i = 0; i < rows; ++i) { - void* pData = colDataGetVarData(pTbNameCol, i); - memcpy(tbName, varDataVal(pData), varDataLen(pData)); - tbName[varDataLen(pData) + 1] = 0; - req.name = tbName; - if (taosArrayPush(batchReq.pArray, &req) == NULL) { - TSDB_CHECK_CODE(terrno, lino, _exit); - } + char tbName[TSDB_TABLE_NAME_LEN + 1] = {0}; + int32_t i = 0; + void* pData = colDataGetVarData(pTbNameCol, i); + memcpy(tbName, varDataVal(pData), varDataLen(pData)); + tbName[varDataLen(pData) + 1] = 0; + req.name = tbName; + if (taosArrayPush(batchReq.pArray, &req) == NULL) { + TSDB_CHECK_CODE(terrno, lino, _exit); } SMetaReader mr = {0}; metaReaderDoInit(&mr, pVnode->pMeta, META_READER_LOCK); - // only one row + code = metaGetTableEntryByName(&mr, tbName); if (TSDB_CODE_SUCCESS == code && isValidDstChildTable(&mr, TD_VID(pVnode), tbName, pTask->outputInfo.tbSink.stbUid)) { STableSinkInfo* pTableSinkInfo = NULL; @@ -478,13 +477,9 @@ static int32_t doBuildAndSendDropTableMsg(SVnode* pVnode, char* pStbFullname, SS code = tqPutReqToQueue(pVnode, &batchReq, encodeDropChildTableForRPC, TDMT_VND_DROP_TABLE); TSDB_CHECK_CODE(code, lino, _exit); - for (int32_t i = 0; i < rows; ++i) { - void* pData = colDataGetVarData(pTbNameCol, i); - memcpy(tbName, varDataVal(pData), varDataLen(pData)); - tbName[varDataLen(pData) + 1] = 0; - code = doWaitForDstTableDropped(pVnode, pTask, tbName); - TSDB_CHECK_CODE(code, lino, _exit); - } + + code = doWaitForDstTableDropped(pVnode, pTask, tbName); + TSDB_CHECK_CODE(code, lino, _exit); _exit: if (batchReq.pArray) { diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 26c97cbb6e..5461b5899b 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -527,9 +527,13 @@ _end: int32_t streamStateDeleteParName(SStreamState* pState, int64_t groupId) { int32_t code = tSimpleHashRemove(pState->parNameMap, &groupId, sizeof(int64_t)); - qTrace("catche %s at line %d res %d", __func__, __LINE__, code); + if (TSDB_CODE_SUCCESS != code) { + qWarn("failed to remove parname from cache, code:%d", code); + } code = streamStateDeleteParName_rocksdb(pState, groupId); - qTrace("disk %s at line %d res %d", __func__, __LINE__, code); + if (TSDB_CODE_SUCCESS != code) { + qWarn("failed to remove parname from rocksdb, code:%d", code); + } return TSDB_CODE_SUCCESS; }