From 7a76837c4629146092376d518fe1da0d6e917f87 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 13 Jul 2023 17:50:27 +0800 Subject: [PATCH 01/40] enh: expire time optimize --- source/dnode/mnode/impl/src/mndCluster.c | 2 +- source/dnode/mnode/impl/src/mndMain.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 4d05637a2b..848363144e 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -133,7 +133,7 @@ float mndGetClusterUpTime(SMnode *pMnode) { mndReleaseCluster(pMnode, pCluster, pIter); } - return upTime / 86400.0f; + return (float)upTime; } static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) { diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 381b1e64ed..f0c53c7473 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -804,7 +804,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr if (pObj->id == pMnode->selfDnodeId) { pClusterInfo->first_ep_dnode_id = pObj->id; tstrncpy(pClusterInfo->first_ep, pObj->pDnode->ep, sizeof(pClusterInfo->first_ep)); - pClusterInfo->master_uptime = mndGetClusterUpTime(pMnode); + pClusterInfo->master_uptime = mndGetClusterUpTime(pMnode) / 86400.0f; // pClusterInfo->master_uptime = (ms - pObj->stateStartTime) / (86400000.0f); tstrncpy(desc.role, syncStr(TAOS_SYNC_STATE_LEADER), sizeof(desc.role)); } else { From a04ba6581713e78664673e20e0a50cc6c1723418 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 19 Jul 2023 15:30:14 +0800 Subject: [PATCH 02/40] chore: more code --- include/common/tglobal.h | 4 ++++ source/common/src/tglobal.c | 4 ++++ source/dnode/mgmt/exe/dmMain.c | 1 + source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 4 ++++ source/dnode/mnode/impl/src/mndMain.c | 4 +++- source/libs/catalog/src/ctgCache.c | 6 ------ source/libs/catalog/src/ctgUtil.c | 6 ++++++ 7 files changed, 22 insertions(+), 7 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index d6c552b3f6..a2ec4d2c03 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -87,6 +87,10 @@ extern int64_t tsMndLogRetention; extern int8_t tsGrant; extern bool tsMndSkipGrant; +// dnode +extern int64_t tsDndStart; +extern int64_t tsDndUpTime; + // monitor extern bool tsEnableMonitor; extern int32_t tsMonitorInterval; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 611a273353..3547ecca01 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -79,6 +79,10 @@ int64_t tsMndLogRetention = 2000; int8_t tsGrant = 1; bool tsMndSkipGrant = false; +// dnode +int64_t tsDndStart = 0; +int64_t tsDndUpTime = 0; + // monitor bool tsEnableMonitor = true; int32_t tsMonitorInterval = 30; diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index e1b8a57684..242dd8a706 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -373,6 +373,7 @@ int mainWindows(int argc, char **argv) { dInfo("start to init service"); dmSetSignalHandle(); + tsDndStart = taosGetTimestampMs(); int32_t code = dmRun(); dInfo("shutting down the service"); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 89c394fdd0..3fe943267c 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -29,6 +29,7 @@ static void *dmStatusThreadFp(void *param) { if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; int64_t curTime = taosGetTimestampMs(); + int64_t cost = 0; float interval = (curTime - lastTime) / 1000.0f; if (interval >= tsStatusInterval) { dmSendStatusReq(pMgmt); @@ -38,7 +39,10 @@ static void *dmStatusThreadFp(void *param) { if (trimCount == 0) { taosMemoryTrim(0); } + cost = taosGetTimestampMs() - curTime; } + tsDndUpTime += 200; + if (cost > 0) tsDndUpTime += cost; } return NULL; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index f0c53c7473..6a7bdd6df2 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -156,6 +156,7 @@ static void mndPullupTelem(SMnode *pMnode) { static void mndPullupGrant(SMnode *pMnode) { mTrace("pullup grant msg"); + printf("%s:%d @@@@@@@@@@@ mndPullupGrant @@@@@@@@@@@ %d \n", __func__, __LINE__, taosGetTimestampSec()); int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { @@ -248,8 +249,9 @@ static void *mndThreadFp(void *param) { setThreadName("mnode-timer"); while (1) { + printf("%s:%d =========== mndThreadFp =========== %d\n", __func__, __LINE__, taosGetTimestampSec()); lastTime++; - taosMsleep(100); + taosMsleep(300); if (mndGetStop(pMnode)) break; if (lastTime % 10 != 0) continue; diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 605f5efeb4..44de83b7ef 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -773,12 +773,6 @@ int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid int32_t ctgChkAuthFromCache(SCatalog *pCtg, SUserAuthInfo *pReq, bool *inCache, SCtgAuthRsp *pRes) { int32_t code = 0; - if (IS_SYS_DBNAME(pReq->tbName.dbname)) { - *inCache = true; - pRes->pRawRes->pass = true; - ctgDebug("sysdb %s, pass", pReq->tbName.dbname); - return TSDB_CODE_SUCCESS; - } SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, pReq->user, strlen(pReq->user)); if (NULL == pUser) { diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 86f6a51d9b..dab007aa47 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1589,6 +1589,12 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { return TSDB_CODE_SUCCESS; } + if (IS_SYS_DBNAME(pReq->tbName.dbname)) { + pRes->pass = true; + ctgDebug("sysdb %s, pass", pReq->tbName.dbname); + return TSDB_CODE_SUCCESS; + } + char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(&pReq->tbName, dbFName); From 7ff25d68953a7ac6b92e060a8dadf4fbbcdd9306 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 20 Jul 2023 10:56:08 +0800 Subject: [PATCH 03/40] chore: more code --- source/common/src/tglobal.c | 1 + source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 2 +- source/dnode/mnode/impl/inc/mndCluster.h | 2 +- source/dnode/mnode/impl/src/mndCluster.c | 4 ++-- source/dnode/mnode/impl/src/mndMain.c | 5 ++--- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 3547ecca01..ba05b2b2f4 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -81,6 +81,7 @@ bool tsMndSkipGrant = false; // dnode int64_t tsDndStart = 0; +int64_t tsDndStartOsUptime = 0; int64_t tsDndUpTime = 0; // monitor diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 3fe943267c..55fdd0f46a 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -42,7 +42,7 @@ static void *dmStatusThreadFp(void *param) { cost = taosGetTimestampMs() - curTime; } tsDndUpTime += 200; - if (cost > 0) tsDndUpTime += cost; + if (cost > 0) tsDndUpTime += cost; // TODO: use /proc/uptime to replace the upTime calculation for linux } return NULL; diff --git a/source/dnode/mnode/impl/inc/mndCluster.h b/source/dnode/mnode/impl/inc/mndCluster.h index 2cb41edd7c..e33ffdb372 100644 --- a/source/dnode/mnode/impl/inc/mndCluster.h +++ b/source/dnode/mnode/impl/inc/mndCluster.h @@ -27,7 +27,7 @@ void mndCleanupCluster(SMnode *pMnode); int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len); int64_t mndGetClusterId(SMnode *pMnode); int64_t mndGetClusterCreateTime(SMnode *pMnode); -float mndGetClusterUpTime(SMnode *pMnode); +int64_t mndGetClusterUpTime(SMnode *pMnode); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index c5cac21d00..d20350b094 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -123,7 +123,7 @@ static int32_t mndGetClusterUpTimeImp(SClusterObj *pCluster) { #endif } -float mndGetClusterUpTime(SMnode *pMnode) { +int64_t mndGetClusterUpTime(SMnode *pMnode) { int64_t upTime = 0; void *pIter = NULL; SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); @@ -132,7 +132,7 @@ float mndGetClusterUpTime(SMnode *pMnode) { mndReleaseCluster(pMnode, pCluster, pIter); } - return (float)upTime; + return upTime; } static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) { diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 6a7bdd6df2..5db9384f5a 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -249,9 +249,8 @@ static void *mndThreadFp(void *param) { setThreadName("mnode-timer"); while (1) { - printf("%s:%d =========== mndThreadFp =========== %d\n", __func__, __LINE__, taosGetTimestampSec()); lastTime++; - taosMsleep(300); + taosMsleep(100); if (mndGetStop(pMnode)) break; if (lastTime % 10 != 0) continue; @@ -806,7 +805,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr if (pObj->id == pMnode->selfDnodeId) { pClusterInfo->first_ep_dnode_id = pObj->id; tstrncpy(pClusterInfo->first_ep, pObj->pDnode->ep, sizeof(pClusterInfo->first_ep)); - pClusterInfo->master_uptime = mndGetClusterUpTime(pMnode) / 86400.0f; + pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f; // pClusterInfo->master_uptime = (ms - pObj->stateStartTime) / (86400000.0f); tstrncpy(desc.role, syncStr(TAOS_SYNC_STATE_LEADER), sizeof(desc.role)); } else { From ef2b66b280ef2ed29b6c75912d2d157af825dfae Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 21 Jul 2023 06:56:24 +0800 Subject: [PATCH 04/40] chore: more code --- include/common/tglobal.h | 1 + source/common/src/tglobal.c | 3 +++ source/dnode/mnode/impl/src/mndDnode.c | 11 ++++++++++- source/dnode/mnode/impl/src/mndMain.c | 1 - 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index a2ec4d2c03..b0a1aa6117 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -85,6 +85,7 @@ extern int64_t tsVndCommitMaxIntervalMs; extern int64_t tsMndSdbWriteDelta; extern int64_t tsMndLogRetention; extern int8_t tsGrant; +extern int32_t tsMndGrantMode; extern bool tsMndSkipGrant; // dnode diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index ba05b2b2f4..d3fbedbd7b 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -77,6 +77,7 @@ int64_t tsVndCommitMaxIntervalMs = 600 * 1000; int64_t tsMndSdbWriteDelta = 200; int64_t tsMndLogRetention = 2000; int8_t tsGrant = 1; +int32_t tsMndGrantMode = 0; bool tsMndSkipGrant = false; // dnode @@ -512,6 +513,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt64(pCfg, "mndSdbWriteDelta", tsMndSdbWriteDelta, 20, 10000, 0) != 0) return -1; if (cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "grantMode", tsMndGrantMode, 0, 10000, 0) != 0) return -1; if (cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, 0) != 0) return -1; if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1; @@ -921,6 +923,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsMndSdbWriteDelta = cfgGetItem(pCfg, "mndSdbWriteDelta")->i64; tsMndLogRetention = cfgGetItem(pCfg, "mndLogRetention")->i64; tsMndSkipGrant = cfgGetItem(pCfg, "skipGrant")->bval; + tsMndGrantMode = cfgGetItem(pCfg, "grantMode")->i32; tsStartUdfd = cfgGetItem(pCfg, "udf")->bval; tstrncpy(tsUdfdResFuncs, cfgGetItem(pCfg, "udfdResFuncs")->str, sizeof(tsUdfdResFuncs)); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 100513b932..d8ee18e3ca 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -655,6 +655,7 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg STrans *pTrans = NULL; SDnodeObj *pDnode = NULL; bool cfgAll = pCfgReq->dnodeId == -1; + int32_t iter = 0; SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; @@ -662,7 +663,8 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg if (cfgAll) { pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); if (pIter == NULL) break; - } else if(!(pDnode = mndAcquireDnode(pMnode, pCfgReq->dnodeId))) { + ++iter; + } else if (!(pDnode = mndAcquireDnode(pMnode, pCfgReq->dnodeId))) { goto _OVER; } @@ -708,6 +710,13 @@ _OVER: } else { mndReleaseDnode(pMnode, pDnode); } + + if (iter > 15) { + tsGrantHBInterval = 10; + } else { + tsGrantHBInterval = 5; + } + mndTransDrop(pTrans); sdbFreeRaw(pRaw); return terrno; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 5db9384f5a..79abc57a39 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -156,7 +156,6 @@ static void mndPullupTelem(SMnode *pMnode) { static void mndPullupGrant(SMnode *pMnode) { mTrace("pullup grant msg"); - printf("%s:%d @@@@@@@@@@@ mndPullupGrant @@@@@@@@@@@ %d \n", __func__, __LINE__, taosGetTimestampSec()); int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { From d89f4da561e3f6b6d1296a83e5a2b1a7beecad27 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 21 Jul 2023 15:45:50 +0800 Subject: [PATCH 05/40] chore: more code --- source/dnode/mnode/impl/src/mndDnode.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index d8ee18e3ca..5623d17ce8 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -1044,7 +1044,11 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { } mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value); - if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) { + if (strncmp(cfgReq.config, "activeCode_m", 12) == 0) { + cfgReq.config[10] = 0; + } else if (strncmp(cfgReq.config, "cActiveCode_m", 13) == 0) { + cfgReq.config[11] = 0; + } else if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) { return -1; } From 586618807d054a437ddf60abf5e044c214c2b37b Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 21 Jul 2023 16:28:48 +0800 Subject: [PATCH 06/40] chore: code optimization --- source/dnode/mnode/impl/src/mndDnode.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 48cdff4cb5..ae15d4de04 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -711,11 +711,7 @@ _OVER: mndReleaseDnode(pMnode, pDnode); } - if (iter > 15) { - tsGrantHBInterval = 10; - } else { - tsGrantHBInterval = 5; - } + tsGrantHBInterval = MIN(MAX(3, iter / 2), 15); mndTransDrop(pTrans); sdbFreeRaw(pRaw); @@ -1048,11 +1044,8 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { } mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value); - if (strncmp(cfgReq.config, "activeCode_m", 12) == 0) { - cfgReq.config[10] = 0; - } else if (strncmp(cfgReq.config, "cActiveCode_m", 13) == 0) { - cfgReq.config[11] = 0; - } else if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) { + if ((pReq->info.ahandle != (void *)0x818611) && + (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0)) { return -1; } From a6f2e871571b8f1935593bba5ab8462c538ec46f Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 22 Jul 2023 22:18:50 +0800 Subject: [PATCH 07/40] chore: code optimization --- source/dnode/mnode/impl/src/mndDnode.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index ae15d4de04..5740746162 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -701,7 +701,7 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg } if (pTrans && mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; - + tsGrantHBInterval = TMIN(TMAX(3, iter / 2), 30); terrno = 0; _OVER: @@ -710,9 +710,6 @@ _OVER: } else { mndReleaseDnode(pMnode, pDnode); } - - tsGrantHBInterval = MIN(MAX(3, iter / 2), 15); - mndTransDrop(pTrans); sdbFreeRaw(pRaw); return terrno; From b6bfdf2b9ebd55033041a5ddd33f1f6fcb35a88b Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 23 Jul 2022 12:11:49 +0800 Subject: [PATCH 08/40] chore: grant hb interval optimize --- source/dnode/mnode/impl/src/mndDnode.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 5740746162..c35ccb17c8 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -701,7 +701,7 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg } if (pTrans && mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; - tsGrantHBInterval = TMIN(TMAX(3, iter / 2), 30); + tsGrantHBInterval = TMIN(TMAX(5, iter / 2), 30); terrno = 0; _OVER: @@ -865,7 +865,7 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { code = mndCreateDnode(pMnode, pReq, &createReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; - + tsGrantHBInterval = 5; _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("dnode:%s:%d, failed to create since %s", createReq.fqdn, createReq.port, terrstr()); From 3cfa49066ac061c7bbc978afada8c35f91133ee9 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 23 Jul 2023 23:25:58 +0800 Subject: [PATCH 09/40] chore: uptime logic optimize --- include/common/tglobal.h | 1 + include/os/os.h | 1 + include/os/osSysinfo.h | 5 +++++ source/dnode/mgmt/exe/dmMain.c | 1 + source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 18 +++++++++++++++--- source/os/src/osSysinfo.c | 14 ++++++++++++++ 6 files changed, 37 insertions(+), 3 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index b0a1aa6117..4f2ed2b065 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -90,6 +90,7 @@ extern bool tsMndSkipGrant; // dnode extern int64_t tsDndStart; +extern int64_t tsDndStartOsUptime; extern int64_t tsDndUpTime; // monitor diff --git a/include/os/os.h b/include/os/os.h index 309a977ff6..31f546e032 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -41,6 +41,7 @@ extern "C" { #include #include #include +#include #include #include #include diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index b5309178ae..a8110ee06a 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -18,6 +18,10 @@ #include "os.h" +// #include /* for _syscallX macros/related stuff */ +// #include /* for struct sysinfo */ + + #ifdef __cplusplus extern "C" { #endif @@ -35,6 +39,7 @@ typedef struct { bool taosCheckSystemIsLittleEnd(); void taosGetSystemInfo(); +int64_t taosGetOsUptime(); int32_t taosGetEmail(char *email, int32_t maxLen); int32_t taosGetOsReleaseName(char *releaseName, char* sName, char* ver, int32_t maxLen); int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores); diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 242dd8a706..3c08714218 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -374,6 +374,7 @@ int mainWindows(int argc, char **argv) { dInfo("start to init service"); dmSetSignalHandle(); tsDndStart = taosGetTimestampMs(); + tsDndStartOsUptime = taosGetOsUptime(); int32_t code = dmRun(); dInfo("shutting down the service"); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 55fdd0f46a..5854450fc7 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -24,6 +24,10 @@ static void *dmStatusThreadFp(void *param) { const static int16_t TRIM_FREQ = 30; int32_t trimCount = 0; + int32_t upTimeCount = 0; + int64_t upTime = 0; + int64_t thrdTime = 0; + while (1) { taosMsleep(200); if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; @@ -39,10 +43,18 @@ static void *dmStatusThreadFp(void *param) { if (trimCount == 0) { taosMemoryTrim(0); } - cost = taosGetTimestampMs() - curTime; + + if ((upTimeCount = (++upTimeCount & 7)) == 0) { + upTime = (taosGetOsUptime() - tsDndStartOsUptime) * 1000; + } } - tsDndUpTime += 200; - if (cost > 0) tsDndUpTime += cost; // TODO: use /proc/uptime to replace the upTime calculation for linux + + thrdTime += 200; + cost = taosGetTimestampMs() - curTime; + if (cost > 0) thrdTime += cost; + tsDndUpTime = upTime > thrdTime ? upTime : thrdTime; + printf("upTime:%" PRIi64 " thrdTime:%" PRIi64 " tsDndUpTime:%" PRIi64 " delta:%" PRIi64 "\n", upTime, thrdTime, + tsDndUpTime, upTime - thrdTime); } return NULL; diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 6f87f6b75b..1f631c7388 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -961,6 +961,20 @@ char *taosGetCmdlineByPID(int pid) { #endif } +int64_t taosGetOsUptime() { +#ifdef WINDOWS + return 0; +// #else +// #elif defined(_TD_DARWIN_64) +// return 0; +#else + struct sysinfo info; + if (0 == sysinfo(&info)) { + return info.uptime; + }; +#endif +} + void taosSetCoreDump(bool enable) { if (!enable) return; #ifdef WINDOWS From d597ba8e71dca425146a6ccd18a0974dda209f97 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 24 Jul 2023 06:35:54 +0800 Subject: [PATCH 10/40] chore: code optimize --- include/os/osSysinfo.h | 3 +-- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 11 ++--------- source/os/src/osSysinfo.c | 6 ++---- 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index a8110ee06a..206dbecd21 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -18,8 +18,7 @@ #include "os.h" -// #include /* for _syscallX macros/related stuff */ -// #include /* for struct sysinfo */ + #ifdef __cplusplus diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 5854450fc7..783babcb66 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -26,7 +26,6 @@ static void *dmStatusThreadFp(void *param) { int32_t trimCount = 0; int32_t upTimeCount = 0; int64_t upTime = 0; - int64_t thrdTime = 0; while (1) { taosMsleep(200); @@ -44,17 +43,11 @@ static void *dmStatusThreadFp(void *param) { taosMemoryTrim(0); } - if ((upTimeCount = (++upTimeCount & 7)) == 0) { + if ((upTimeCount = (++upTimeCount & 63)) == 0) { upTime = (taosGetOsUptime() - tsDndStartOsUptime) * 1000; + tsDndUpTime = TMAX(tsDndUpTime, upTime); } } - - thrdTime += 200; - cost = taosGetTimestampMs() - curTime; - if (cost > 0) thrdTime += cost; - tsDndUpTime = upTime > thrdTime ? upTime : thrdTime; - printf("upTime:%" PRIi64 " thrdTime:%" PRIi64 " tsDndUpTime:%" PRIi64 " delta:%" PRIi64 "\n", upTime, thrdTime, - tsDndUpTime, upTime - thrdTime); } return NULL; diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 1f631c7388..a29b96dd6f 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -963,16 +963,14 @@ char *taosGetCmdlineByPID(int pid) { int64_t taosGetOsUptime() { #ifdef WINDOWS - return 0; -// #else -// #elif defined(_TD_DARWIN_64) -// return 0; +#elif defined(_TD_DARWIN_64) #else struct sysinfo info; if (0 == sysinfo(&info)) { return info.uptime; }; #endif + return 0; } void taosSetCoreDump(bool enable) { From e566c051f7abb65b8eac3e886a586145718417eb Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 24 Jul 2023 09:03:19 +0800 Subject: [PATCH 11/40] chore: code optimization --- include/os/osSysinfo.h | 3 --- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 1 - 2 files changed, 4 deletions(-) diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 206dbecd21..a6a3655a55 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -18,9 +18,6 @@ #include "os.h" - - - #ifdef __cplusplus extern "C" { #endif diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 783babcb66..a7f58b1cd0 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -32,7 +32,6 @@ static void *dmStatusThreadFp(void *param) { if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; int64_t curTime = taosGetTimestampMs(); - int64_t cost = 0; float interval = (curTime - lastTime) / 1000.0f; if (interval >= tsStatusInterval) { dmSendStatusReq(pMgmt); From 9511af242e458a373396ba6285fc26a0413cd592 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 24 Jul 2023 09:12:42 +0800 Subject: [PATCH 12/40] chore: time unit of os uptime --- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 2 +- source/os/src/osSysinfo.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index a7f58b1cd0..7cc896cb84 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -43,7 +43,7 @@ static void *dmStatusThreadFp(void *param) { } if ((upTimeCount = (++upTimeCount & 63)) == 0) { - upTime = (taosGetOsUptime() - tsDndStartOsUptime) * 1000; + upTime = taosGetOsUptime() - tsDndStartOsUptime; tsDndUpTime = TMAX(tsDndUpTime, upTime); } } diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index a29b96dd6f..a5b8af3e6e 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -967,7 +967,7 @@ int64_t taosGetOsUptime() { #else struct sysinfo info; if (0 == sysinfo(&info)) { - return info.uptime; + return (int64_t)info.uptime * 1000; }; #endif return 0; From 3b11661c4057db4c05abfa9f1ae78423796be8c0 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 24 Jul 2023 09:14:49 +0800 Subject: [PATCH 13/40] chore: code optimization --- include/os/os.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/os/os.h b/include/os/os.h index 31f546e032..389e25f7c1 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -41,9 +41,9 @@ extern "C" { #include #include #include -#include #include #include +#include #include #include #include From 1d23b0215c2f32505cbc847856b65030b0ecfbdc Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 24 Jul 2023 10:07:15 +0800 Subject: [PATCH 14/40] fix: exclude sys/sysinfo.h for mac --- include/os/os.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/os/os.h b/include/os/os.h index 389e25f7c1..ac1a750b78 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -43,7 +43,6 @@ extern "C" { #include #include #include -#include #include #include #include @@ -54,6 +53,7 @@ extern "C" { #else #include #include +#include #if defined(_TD_X86_) #include #endif From dffdbb53f69aadafaf52b633775ac6c152107a68 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 24 Jul 2023 10:15:32 +0800 Subject: [PATCH 15/40] chore: code optimization --- source/os/src/osSysinfo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index a5b8af3e6e..5f73251e3b 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -968,7 +968,7 @@ int64_t taosGetOsUptime() { struct sysinfo info; if (0 == sysinfo(&info)) { return (int64_t)info.uptime * 1000; - }; + } #endif return 0; } From ec4b4923e8a8d679b5d67470d9dfbfcec97e0dac Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 24 Jul 2023 10:33:38 +0800 Subject: [PATCH 16/40] fix: multiple unsequenced modifications --- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 7cc896cb84..98bb4e3f9c 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -42,7 +42,7 @@ static void *dmStatusThreadFp(void *param) { taosMemoryTrim(0); } - if ((upTimeCount = (++upTimeCount & 63)) == 0) { + if ((upTimeCount = ((upTimeCount + 1) & 63)) == 0) { upTime = taosGetOsUptime() - tsDndStartOsUptime; tsDndUpTime = TMAX(tsDndUpTime, upTime); } From 8225c3fd9be962e001f3085d2f024d008d816f07 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 1 Oct 2022 00:03:43 +0800 Subject: [PATCH 17/40] chore: code optimization --- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 98bb4e3f9c..76cb65b53a 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -32,7 +32,8 @@ static void *dmStatusThreadFp(void *param) { if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; int64_t curTime = taosGetTimestampMs(); - float interval = (curTime - lastTime) / 1000.0f; + if (curTime < lastTime) lastTime = curTime; + float interval = (curTime - lastTime) / 1000.0f; if (interval >= tsStatusInterval) { dmSendStatusReq(pMgmt); lastTime = curTime; @@ -62,7 +63,8 @@ static void *dmMonitorThreadFp(void *param) { if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; int64_t curTime = taosGetTimestampMs(); - float interval = (curTime - lastTime) / 1000.0f; + if (curTime < lastTime) lastTime = curTime; + float interval = (curTime - lastTime) / 1000.0f; if (interval >= tsMonitorInterval) { (*pMgmt->sendMonitorReportFp)(); lastTime = curTime; From 624b6b094fdd630b6f77027d5f7f63a6c96b4751 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 21 Jul 2023 10:14:12 +0800 Subject: [PATCH 18/40] fix: fix(value, null) output behavior --- source/libs/executor/src/tfill.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 55ef019d76..4e0dff9d4f 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -66,20 +66,25 @@ static void setNullRow(SSDataBlock* pBlock, SFillInfo* pFillInfo, int32_t rowInd } static void doSetUserSpecifiedValue(SColumnInfoData* pDst, SVariant* pVar, int32_t rowIndex, int64_t currentKey) { + bool isNull = (TSDB_DATA_TYPE_NULL == pVar->nType) ? true : false; if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) { float v = 0; - GET_TYPED_DATA(v, float, pVar->nType, &pVar->i); - colDataSetVal(pDst, rowIndex, (char*)&v, false); + GET_TYPED_DATA(v, float, pVar->nType, &pVar->f); + colDataSetVal(pDst, rowIndex, (char*)&v, isNull); } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) { double v = 0; - GET_TYPED_DATA(v, double, pVar->nType, &pVar->i); - colDataSetVal(pDst, rowIndex, (char*)&v, false); + GET_TYPED_DATA(v, double, pVar->nType, &pVar->d); + colDataSetVal(pDst, rowIndex, (char*)&v, isNull); } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) { int64_t v = 0; GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); - colDataSetVal(pDst, rowIndex, (char*)&v, false); + colDataSetVal(pDst, rowIndex, (char*)&v, isNull); + } else if (IS_UNSIGNED_NUMERIC_TYPE(pDst->info.type)) { + uint64_t v = 0; + GET_TYPED_DATA(v, uint64_t, pVar->nType, &pVar->u); + colDataSetVal(pDst, rowIndex, (char*)&v, isNull); } else if (pDst->info.type == TSDB_DATA_TYPE_TIMESTAMP) { - colDataSetVal(pDst, rowIndex, (const char*)¤tKey, false); + colDataSetVal(pDst, rowIndex, (const char*)¤tKey, isNull); } else { // varchar/nchar data colDataSetNULL(pDst, rowIndex); } From 62b73db8bf90dfdfefb2bcedd48d7520068b53a4 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 21 Jul 2023 10:14:12 +0800 Subject: [PATCH 19/40] fix: fix(value, null) output behavior --- source/libs/executor/src/timesliceoperator.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index cb74392a10..b019985645 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -312,6 +312,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp case TSDB_FILL_SET_VALUE_F: { SVariant* pVar = &pSliceInfo->pFillColInfo[fillColIndex].fillVal; + bool isNull = (TSDB_DATA_TYPE_NULL == pVar->nType) ? true : false; if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) { float v = 0; if (!IS_VAR_DATA_TYPE(pVar->nType)) { @@ -319,7 +320,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } else { v = taosStr2Float(varDataVal(pVar->pz), NULL); } - colDataSetVal(pDst, rows, (char*)&v, false); + colDataSetVal(pDst, rows, (char*)&v, isNull); } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) { double v = 0; if (!IS_VAR_DATA_TYPE(pVar->nType)) { @@ -327,7 +328,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } else { v = taosStr2Double(varDataVal(pVar->pz), NULL); } - colDataSetVal(pDst, rows, (char*)&v, false); + colDataSetVal(pDst, rows, (char*)&v, isNull); } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) { int64_t v = 0; if (!IS_VAR_DATA_TYPE(pVar->nType)) { @@ -335,7 +336,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } else { v = taosStr2Int64(varDataVal(pVar->pz), NULL, 10); } - colDataSetVal(pDst, rows, (char*)&v, false); + colDataSetVal(pDst, rows, (char*)&v, isNull); } else if (IS_UNSIGNED_NUMERIC_TYPE(pDst->info.type)) { uint64_t v = 0; if (!IS_VAR_DATA_TYPE(pVar->nType)) { @@ -343,7 +344,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } else { v = taosStr2UInt64(varDataVal(pVar->pz), NULL, 10); } - colDataSetVal(pDst, rows, (char*)&v, false); + colDataSetVal(pDst, rows, (char*)&v, isNull); } else if (IS_BOOLEAN_TYPE(pDst->info.type)) { bool v = false; if (!IS_VAR_DATA_TYPE(pVar->nType)) { @@ -351,7 +352,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } else { v = taosStr2Int8(varDataVal(pVar->pz), NULL, 10); } - colDataSetVal(pDst, rows, (char*)&v, false); + colDataSetVal(pDst, rows, (char*)&v, isNull); } ++fillColIndex; From 3da83700dd3a88f770b2933d566a9ea5da532410 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 21 Jul 2023 10:35:41 +0800 Subject: [PATCH 20/40] add test cases --- tests/system-test/2-query/interp.py | 215 ++++++++++++++++++++++++++-- 1 file changed, 205 insertions(+), 10 deletions(-) diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index b6cefbe36f..986c63839b 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -147,6 +147,57 @@ class TDTestCase: tdSql.checkData(11, 0, 15) tdSql.checkData(12, 0, 1) + for col in col_list: + tdSql.query(f"select interp({col}) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1.0)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 1) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, 1) + tdSql.checkData(6, 0, 10) + tdSql.checkData(7, 0, 1) + tdSql.checkData(8, 0, 1) + tdSql.checkData(9, 0, 1) + tdSql.checkData(10, 0, 1) + tdSql.checkData(11, 0, 15) + tdSql.checkData(12, 0, 1) + + for col in col_list: + tdSql.query(f"select interp({col}) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, true)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 1) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, 1) + tdSql.checkData(6, 0, 10) + tdSql.checkData(7, 0, 1) + tdSql.checkData(8, 0, 1) + tdSql.checkData(9, 0, 1) + tdSql.checkData(10, 0, 1) + tdSql.checkData(11, 0, 15) + tdSql.checkData(12, 0, 1) + + for col in col_list: + tdSql.query(f"select interp({col}) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, NULL)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, 10) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, None) + tdSql.checkData(11, 0, 15) + tdSql.checkData(12, 0, None) + tdSql.query(f"select interp(c4) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1)") tdSql.checkRows(13) tdSql.checkData(0, 0, 1.0) @@ -163,6 +214,54 @@ class TDTestCase: tdSql.checkData(11, 0, 15.0) tdSql.checkData(12, 0, 1.0) + tdSql.query(f"select interp(c4) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1.0)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, 1.0) + tdSql.checkData(1, 0, 5.0) + tdSql.checkData(2, 0, 1.0) + tdSql.checkData(3, 0, 1.0) + tdSql.checkData(4, 0, 1.0) + tdSql.checkData(5, 0, 1.0) + tdSql.checkData(6, 0, 10.0) + tdSql.checkData(7, 0, 1.0) + tdSql.checkData(8, 0, 1.0) + tdSql.checkData(9, 0, 1.0) + tdSql.checkData(10, 0, 1.0) + tdSql.checkData(11, 0, 15.0) + tdSql.checkData(12, 0, 1.0) + + tdSql.query(f"select interp(c4) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, true)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, 1.0) + tdSql.checkData(1, 0, 5.0) + tdSql.checkData(2, 0, 1.0) + tdSql.checkData(3, 0, 1.0) + tdSql.checkData(4, 0, 1.0) + tdSql.checkData(5, 0, 1.0) + tdSql.checkData(6, 0, 10.0) + tdSql.checkData(7, 0, 1.0) + tdSql.checkData(8, 0, 1.0) + tdSql.checkData(9, 0, 1.0) + tdSql.checkData(10, 0, 1.0) + tdSql.checkData(11, 0, 15.0) + tdSql.checkData(12, 0, 1.0) + + tdSql.query(f"select interp(c4) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, NULL)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, 5.0) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, 10.0) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, None) + tdSql.checkData(11, 0, 15.0) + tdSql.checkData(12, 0, None) + tdSql.query(f"select interp(c5) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1)") tdSql.checkRows(13) tdSql.checkData(0, 0, 1.0) @@ -179,6 +278,54 @@ class TDTestCase: tdSql.checkData(11, 0, 15.0) tdSql.checkData(12, 0, 1.0) + tdSql.query(f"select interp(c5) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1.0)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, 1.0) + tdSql.checkData(1, 0, 5.0) + tdSql.checkData(2, 0, 1.0) + tdSql.checkData(3, 0, 1.0) + tdSql.checkData(4, 0, 1.0) + tdSql.checkData(5, 0, 1.0) + tdSql.checkData(6, 0, 10.0) + tdSql.checkData(7, 0, 1.0) + tdSql.checkData(8, 0, 1.0) + tdSql.checkData(9, 0, 1.0) + tdSql.checkData(10, 0, 1.0) + tdSql.checkData(11, 0, 15.0) + tdSql.checkData(12, 0, 1.0) + + tdSql.query(f"select interp(c5) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, true)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, 1.0) + tdSql.checkData(1, 0, 5.0) + tdSql.checkData(2, 0, 1.0) + tdSql.checkData(3, 0, 1.0) + tdSql.checkData(4, 0, 1.0) + tdSql.checkData(5, 0, 1.0) + tdSql.checkData(6, 0, 10.0) + tdSql.checkData(7, 0, 1.0) + tdSql.checkData(8, 0, 1.0) + tdSql.checkData(9, 0, 1.0) + tdSql.checkData(10, 0, 1.0) + tdSql.checkData(11, 0, 15.0) + tdSql.checkData(12, 0, 1.0) + + tdSql.query(f"select interp(c5) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, NULL)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, 5.0) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, 10.0) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, None) + tdSql.checkData(11, 0, 15.0) + tdSql.checkData(12, 0, None) + tdSql.query(f"select interp(c6) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1)") tdSql.checkRows(13) tdSql.checkData(0, 0, True) @@ -195,6 +342,54 @@ class TDTestCase: tdSql.checkData(11, 0, True) tdSql.checkData(12, 0, True) + tdSql.query(f"select interp(c6) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1.0)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, True) + tdSql.checkData(1, 0, True) + tdSql.checkData(2, 0, True) + tdSql.checkData(3, 0, True) + tdSql.checkData(4, 0, True) + tdSql.checkData(5, 0, True) + tdSql.checkData(6, 0, True) + tdSql.checkData(7, 0, True) + tdSql.checkData(8, 0, True) + tdSql.checkData(9, 0, True) + tdSql.checkData(10, 0, True) + tdSql.checkData(11, 0, True) + tdSql.checkData(12, 0, True) + + tdSql.query(f"select interp(c6) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, true)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, True) + tdSql.checkData(1, 0, True) + tdSql.checkData(2, 0, True) + tdSql.checkData(3, 0, True) + tdSql.checkData(4, 0, True) + tdSql.checkData(5, 0, True) + tdSql.checkData(6, 0, True) + tdSql.checkData(7, 0, True) + tdSql.checkData(8, 0, True) + tdSql.checkData(9, 0, True) + tdSql.checkData(10, 0, True) + tdSql.checkData(11, 0, True) + tdSql.checkData(12, 0, True) + + tdSql.query(f"select interp(c6) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, NULL)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, True) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, True) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, None) + tdSql.checkData(11, 0, True) + tdSql.checkData(12, 0, None) + ## {} ... tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:01', '2020-02-01 00:00:04') every(1s) fill(value, 1)") tdSql.checkRows(4) @@ -2587,25 +2782,25 @@ class TDTestCase: tdSql.checkData(0, 0, '2020-02-02 00:00:00.000') - tdSql.checkData(0, 2, False) + tdSql.checkData(0, 2, None) tdSql.checkData(1, 2, False) - tdSql.checkData(2, 2, False) + tdSql.checkData(2, 2, None) tdSql.checkData(3, 2, True) - tdSql.checkData(4, 2, False) + tdSql.checkData(4, 2, None) tdSql.checkData(5, 2, False) - tdSql.checkData(6, 2, False) + tdSql.checkData(6, 2, None) tdSql.checkData(7, 2, True) - tdSql.checkData(8, 2, False) + tdSql.checkData(8, 2, None) tdSql.checkData(9, 2, True) - tdSql.checkData(10, 2, False) + tdSql.checkData(10, 2, None) tdSql.checkData(11, 2, False) - tdSql.checkData(12, 2, False) + tdSql.checkData(12, 2, None) tdSql.checkData(13, 2, False) - tdSql.checkData(14, 2, False) + tdSql.checkData(14, 2, None) tdSql.checkData(15, 2, None) - tdSql.checkData(16, 2, False) + tdSql.checkData(16, 2, None) tdSql.checkData(17, 2, None) - tdSql.checkData(18, 2, False) + tdSql.checkData(18, 2, None) tdSql.checkData(18, 0, '2020-02-02 00:00:18.000') From 88c13e17c3661b8c7d4ec53f35d28291a04f52f5 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 21 Jul 2023 11:10:48 +0800 Subject: [PATCH 21/40] add test cases --- tests/script/tsim/parser/fill.sim | 100 ++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/tests/script/tsim/parser/fill.sim b/tests/script/tsim/parser/fill.sim index a66e7d6ab7..0534aa5d5b 100644 --- a/tests/script/tsim/parser/fill.sim +++ b/tests/script/tsim/parser/fill.sim @@ -1224,4 +1224,104 @@ if $data42 != NULL then return -1 endi +print ===================== TD-3625 test fill value NULL +sql use $db + +sql select _wstart,_wend,count(*) from tm0 where ts >= '2020-01-01 01:03:06.000' and ts <= '2020-01-01 01:03:10.000' interval(1s) fill(value, NULL); + +if $rows != 5 then + return -1 +endi + +if $data02 != NULL then + return -1 +endi + +if $data12 != 1 then + return -1 +endi + +if $data22 != 1 then + return -1 +endi + +if $data32 != 1 then + return -1 +endi + +if $data42 != NULL then + return -1 +endi + +sql select _wstart,_wend,count(*),sum(k),avg(k) from tm0 where ts >= '2020-01-01 01:03:06.000' and ts <= '2020-01-01 01:03:10.000' interval(1s) fill(value, 1, NULL, 1); + +if $rows != 5 then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +if $data12 != 1 then + return -1 +endi + +if $data22 != 1 then + return -1 +endi + +if $data32 != 1 then + return -1 +endi + +if $data42 != 1 then + return -1 +endi + + +if $data03 != NULL then + return -1 +endi + +if $data13 != 7 then + return -1 +endi + +if $data23 != 8 then + return -1 +endi + +if $data33 != 9 then + return -1 +endi + +if $data43 != NULL then + return -1 +endi + + +if $data04 != 1.000000000 then + return -1 +endi + +if $data14 != 7.000000000 then + return -1 +endi + +if $data24 != 8.000000000 then + return -1 +endi + +if $data34 != 9.000000000 then + return -1 +endi + +if $data44 != 1.000000000 then + return -1 +endi + + system sh/exec.sh -n dnode1 -s stop -x SIGINT + + From 968cfd9e65a27803af8414d3ae440041a9de9254 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 24 Jul 2023 11:03:07 +0800 Subject: [PATCH 22/40] tsdb/cache: pLastIter null pointer check --- source/dnode/vnode/src/tsdb/tsdbCache.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index db07223106..1e820ec3c7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -2053,7 +2053,9 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie } if (!state->pLastRow) { - lastIterClose(&state->pLastIter); + if (state->pLastIter) { + lastIterClose(&state->pLastIter); + } clearLastFileSet(state); state->state = SFSNEXTROW_FILESET; @@ -2154,7 +2156,9 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie } if (!state->pLastRow) { - lastIterClose(&state->pLastIter); + if (state->pLastIter) { + lastIterClose(&state->pLastIter); + } *ppRow = &state->row; --state->iRow; From 9bae91bcba3f55d677dc0a3658ae13974281eba4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Jul 2023 13:37:44 +0800 Subject: [PATCH 23/40] fix(stream): add some logs, and remove the invalid loop. --- source/dnode/vnode/src/tq/tq.c | 18 ++++++++---------- source/libs/stream/src/streamExec.c | 3 +++ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 9002001404..02509d994d 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1087,16 +1087,14 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { int64_t st = taosGetTimestampMs(); // we have to continue retrying to successfully execute the scan history task. - while (1) { - int8_t schedStatus = atomic_val_compare_exchange_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE, - TASK_SCHED_STATUS__WAITING); - if (schedStatus == TASK_SCHED_STATUS__INACTIVE) { - break; - } - - tqError("s-task:%s failed to start scan history in current time window, unexpected sched-status:%d, retry in 100ms", - id, schedStatus); - taosMsleep(100); + int8_t schedStatus = atomic_val_compare_exchange_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE, + TASK_SCHED_STATUS__WAITING); + if (schedStatus != TASK_SCHED_STATUS__INACTIVE) { + tqError( + "s-task:%s failed to start scan-history in first stream time window since already started, unexpected " + "sched-status:%d", + id, schedStatus); + return 0; } ASSERT(pTask->status.pauseAllowed == false); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 7127b1c323..e7adcf36dc 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -594,6 +594,9 @@ int32_t streamTryExec(SStreamTask* pTask) { (!streamTaskShouldPause(&pTask->status))) { streamSchedExec(pTask); } + } else { + qDebug("s-task:%s already started to exec by other thread, status:%s, sched-status:%d", pTask->id.idStr, + streamGetTaskStatusStr(pTask->status.taskStatus), pTask->status.schedStatus); } return 0; From cf59a6cff0fd86d1f90a50d0a324b6b0d4de20da Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 24 Jul 2023 14:24:47 +0800 Subject: [PATCH 24/40] chore: set user for dm transport --- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 1 + source/dnode/mnode/impl/src/mndDnode.c | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 5d6d16ccf8..37c548c6ba 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -290,6 +290,7 @@ int32_t dmInitClient(SDnode *pDnode) { rpcInit.cfp = (RpcCfp)dmProcessRpcMsg; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.user = "root"; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.parent = pDnode; rpcInit.rfp = rpcRfp; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index c35ccb17c8..cfd026634c 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -1041,8 +1041,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { } mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value); - if ((pReq->info.ahandle != (void *)0x818611) && - (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0)) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) { return -1; } From 424dd40078ea28c0750d6791b7d325539899ba65 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 24 Jul 2023 14:45:35 +0800 Subject: [PATCH 25/40] tsdb/cache: reuse pDataFileReader for same fileset --- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/tsdb/tsdbCache.c | 99 +++++++++++++++---------- 2 files changed, 62 insertions(+), 38 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index ad72c5924c..75c8eea83a 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -877,6 +877,7 @@ typedef struct SCacheRowsReader { SSHashObj *pTableMap; SArray *pLDataIterArray; struct SDataFileReader *pFileReader; + STFileSet *pCurFileSet; STsdbReadSnap *pReadSnap; char *idstr; int64_t lastTs; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 1e820ec3c7..23eac1a2e3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1949,35 +1949,39 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie STFileObj **pFileObj = state->pFileSet->farr; if (pFileObj[0] != NULL || pFileObj[3] != NULL) { - SDataFileReaderConfig conf = {.tsdb = state->pTsdb, .szPage = state->pTsdb->pVnode->config.szPage}; - const char *filesName[4] = {0}; - if (pFileObj[0] != NULL) { - conf.files[0].file = *pFileObj[0]->f; - conf.files[0].exist = true; - filesName[0] = pFileObj[0]->fname; + if (state->pFileSet != state->pr->pCurFileSet) { + SDataFileReaderConfig conf = {.tsdb = state->pTsdb, .szPage = state->pTsdb->pVnode->config.szPage}; + const char *filesName[4] = {0}; + if (pFileObj[0] != NULL) { + conf.files[0].file = *pFileObj[0]->f; + conf.files[0].exist = true; + filesName[0] = pFileObj[0]->fname; - conf.files[1].file = *pFileObj[1]->f; - conf.files[1].exist = true; - filesName[1] = pFileObj[1]->fname; + conf.files[1].file = *pFileObj[1]->f; + conf.files[1].exist = true; + filesName[1] = pFileObj[1]->fname; - conf.files[2].file = *pFileObj[2]->f; - conf.files[2].exist = true; - filesName[2] = pFileObj[2]->fname; + conf.files[2].file = *pFileObj[2]->f; + conf.files[2].exist = true; + filesName[2] = pFileObj[2]->fname; + } + + if (pFileObj[3] != NULL) { + conf.files[3].exist = true; + conf.files[3].file = *pFileObj[3]->f; + filesName[3] = pFileObj[3]->fname; + } + + code = tsdbDataFileReaderOpen(filesName, &conf, &state->pr->pFileReader); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + + loadDataTomb(state->pr, state->pr->pFileReader); + + state->pr->pCurFileSet = state->pFileSet; } - if (pFileObj[3] != NULL) { - conf.files[3].exist = true; - conf.files[3].file = *pFileObj[3]->f; - filesName[3] = pFileObj[3]->fname; - } - - code = tsdbDataFileReaderOpen(filesName, &conf, &state->pr->pFileReader); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } - - loadDataTomb(state->pr, state->pr->pFileReader); - if (!state->pIndexList) { state->pIndexList = taosArrayInit(1, sizeof(SBrinBlk)); } else { @@ -2218,19 +2222,6 @@ _err: return code; } -int32_t clearNextRowFromFS(void *iter) { - int32_t code = 0; - - SFSNextRowIter *state = (SFSNextRowIter *)iter; - if (!state) { - return code; - } - - clearLastFileSet(state); - - return code; -} - typedef enum SMEMNEXTROWSTATES { SMEMNEXTROW_ENTER, SMEMNEXTROW_NEXT, @@ -2350,6 +2341,36 @@ typedef struct CacheNextRowIter { STsdb *pTsdb; } CacheNextRowIter; +int32_t clearNextRowFromFS(void *iter) { + int32_t code = 0; + + SFSNextRowIter *state = (SFSNextRowIter *)iter; + if (!state) { + return code; + } + + if (state->pLastIter) { + lastIterClose(&state->pLastIter); + } + + if (state->pBlockData) { + tBlockDataDestroy(state->pBlockData); + state->pBlockData = NULL; + } + + if (state->pTSRow) { + taosMemoryFree(state->pTSRow); + state->pTSRow = NULL; + } + + if (state->pRowIter->pSkyline) { + taosArrayDestroy(state->pRowIter->pSkyline); + state->pRowIter->pSkyline = NULL; + } + + return code; +} + static void clearLastFileSet(SFSNextRowIter *state) { if (state->pLastIter) { lastIterClose(&state->pLastIter); @@ -2363,6 +2384,8 @@ static void clearLastFileSet(SFSNextRowIter *state) { if (state->pr->pFileReader) { tsdbDataFileReaderClose(&state->pr->pFileReader); state->pr->pFileReader = NULL; + + state->pr->pCurFileSet = NULL; } if (state->pTSRow) { From 7946ce499448168520c1a42acc6102a8f9a06436 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 24 Jul 2023 14:46:59 +0800 Subject: [PATCH 26/40] chore: code optimization --- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 37c548c6ba..df54f8abba 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -290,7 +290,7 @@ int32_t dmInitClient(SDnode *pDnode) { rpcInit.cfp = (RpcCfp)dmProcessRpcMsg; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.user = "root"; + rpcInit.user = TSDB_DEFAULT_USER; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.parent = pDnode; rpcInit.rfp = rpcRfp; From 1d24b96e825999734657d7a253b8fbbb6592dcfb Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 24 Jul 2023 15:16:39 +0800 Subject: [PATCH 27/40] enhance: skip data blocks with the merge limit ts when create initial source --- source/libs/executor/src/tsort.c | 35 ++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 30e7148736..738f3f7131 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -51,6 +51,7 @@ struct SSortHandle { uint32_t tmpRowIdx; int64_t mergeLimit; + int64_t currMergeLimitTs; int32_t sourceId; SSDataBlock* pDataBlock; @@ -921,7 +922,8 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO int32_t nMergedRows = 0; bool mergeLimitReached = false; size_t blkPgSz = pgHeaderSz; - + int64_t lastPageBufTs = (order->order == TSDB_ORDER_ASC) ? INT64_MAX : INT64_MIN; + int64_t currTs = (order->order == TSDB_ORDER_ASC) ? INT64_MAX : INT64_MIN; while (nRows < totalRows) { int32_t minIdx = tMergeTreeGetChosenIndex(pTree); SSDataBlock* minBlk = taosArrayGetP(aBlk, minIdx); @@ -929,14 +931,21 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO int32_t bufInc = getPageBufIncForRow(minBlk, minRow, pHandle->pDataBlock->info.rows); if (blkPgSz <= pHandle->pageSize && blkPgSz + bufInc > pHandle->pageSize) { + SColumnInfoData* tsCol = taosArrayGet(pHandle->pDataBlock->pDataBlock, order->slotId); + lastPageBufTs = ((int64_t*)tsCol->pData)[pHandle->pDataBlock->info.rows - 1]; appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId); nMergedRows += pHandle->pDataBlock->info.rows; blockDataCleanup(pHandle->pDataBlock); blkPgSz = pgHeaderSz; bufInc = getPageBufIncForRow(minBlk, minRow, 0); + if ((pHandle->mergeLimit != -1) && (nMergedRows >= pHandle->mergeLimit)) { mergeLimitReached = true; + if ((lastPageBufTs < pHandle->currMergeLimitTs && order->order == TSDB_ORDER_ASC) || + (lastPageBufTs > pHandle->currMergeLimitTs && order->order == TSDB_ORDER_DESC)) { + pHandle->currMergeLimitTs = lastPageBufTs; + } break; } } @@ -955,8 +964,17 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO } if (pHandle->pDataBlock->info.rows > 0) { if (!mergeLimitReached) { + SColumnInfoData* tsCol = taosArrayGet(pHandle->pDataBlock->pDataBlock, order->slotId); + lastPageBufTs = ((int64_t*)tsCol->pData)[pHandle->pDataBlock->info.rows - 1]; appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId); nMergedRows += pHandle->pDataBlock->info.rows; + if ((pHandle->mergeLimit != -1) && (nMergedRows >= pHandle->mergeLimit)) { + mergeLimitReached = true; + if ((lastPageBufTs < pHandle->currMergeLimitTs && order->order == TSDB_ORDER_ASC) || + (lastPageBufTs > pHandle->currMergeLimitTs && order->order == TSDB_ORDER_DESC)) { + pHandle->currMergeLimitTs = lastPageBufTs; + } + } } blockDataCleanup(pHandle->pDataBlock); } @@ -982,11 +1000,24 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { SSortSource* pSrc = taosArrayGetP(pHandle->pOrderedSource, 0); int32_t szSort = 0; + if (pOrder->order == TSDB_ORDER_ASC) { + pHandle->currMergeLimitTs = INT64_MAX; + } else { + pHandle->currMergeLimitTs = INT64_MIN; + } + SArray* aBlkSort = taosArrayInit(8, POINTER_BYTES); SSHashObj* mUidBlk = tSimpleHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT)); while (1) { SSDataBlock* pBlk = pHandle->fetchfp(pSrc->param); - + if (pBlk != NULL) { + SColumnInfoData* tsCol = taosArrayGet(pBlk->pDataBlock, pOrder->slotId); + int64_t firstRowTs = *(int64_t*)tsCol->pData; + if ((pOrder->order == TSDB_ORDER_ASC && firstRowTs > pHandle->currMergeLimitTs) || + (pOrder->order == TSDB_ORDER_DESC && firstRowTs < pHandle->currMergeLimitTs)) { + continue; + } + } if (pBlk != NULL) { szSort += blockDataGetSize(pBlk); From d78f28ce2ea6f472cd472cc698d61f9539153a6c Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Mon, 24 Jul 2023 10:59:30 +0800 Subject: [PATCH 28/40] fix: monitor no longer reports logs --- source/libs/monitor/src/monMain.c | 2 +- tests/system-test/0-others/taosdMonitor.py | 28 ---------------------- 2 files changed, 1 insertion(+), 29 deletions(-) diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index 949e91198a..8f94bfdb96 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -547,7 +547,7 @@ void monSendReport() { monGenGrantJson(pMonitor); monGenDnodeJson(pMonitor); monGenDiskJson(pMonitor); - monGenLogJson(pMonitor); + //monGenLogJson(pMonitor); // TS-3691 char *pCont = tjsonToString(pMonitor->pJson); // uDebugL("report cont:%s\n", pCont); diff --git a/tests/system-test/0-others/taosdMonitor.py b/tests/system-test/0-others/taosdMonitor.py index 8094c4e0f5..6c21eb8daa 100644 --- a/tests/system-test/0-others/taosdMonitor.py +++ b/tests/system-test/0-others/taosdMonitor.py @@ -186,33 +186,6 @@ class RequestHandlerImpl(http.server.BaseHTTPRequestHandler): tdLog.exit("total is null!") - # log_infos ==================================== - - if "log_infos" not in infoDict or infoDict["log_infos"]== None: - tdLog.exit("log_infos is null!") - - if "logs" not in infoDict["log_infos"] or len(infoDict["log_infos"]["logs"]) < 8:#!= 10: - tdLog.exit("logs is null!") - - if "ts" not in infoDict["log_infos"]["logs"][0] or len(infoDict["log_infos"]["logs"][0]["ts"]) <= 10: - tdLog.exit("ts is null!") - - if "level" not in infoDict["log_infos"]["logs"][0] or infoDict["log_infos"]["logs"][0]["level"] not in ["error" ,"info" , "debug" ,"trace"]: - tdLog.exit("level is null!") - - if "content" not in infoDict["log_infos"]["logs"][0] or len(infoDict["log_infos"]["logs"][0]["ts"]) <= 1: - tdLog.exit("content is null!") - - if "summary" not in infoDict["log_infos"] or len(infoDict["log_infos"]["summary"])!= 4: - tdLog.exit("summary is null!") - - - if "total" not in infoDict["log_infos"]["summary"][0] or infoDict["log_infos"]["summary"][0]["total"] < 0 : - tdLog.exit("total is null!") - - if "level" not in infoDict["log_infos"]["summary"][0] or infoDict["log_infos"]["summary"][0]["level"] not in ["error" ,"info" , "debug" ,"trace"]: - tdLog.exit("level is null!") - def do_GET(self): """ process GET request @@ -315,4 +288,3 @@ class TDTestCase: tdCases.addLinux(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase()) - From a81aaa80f9dd6564715cda8abdb75540bbd1d3ad Mon Sep 17 00:00:00 2001 From: sunpeng Date: Mon, 24 Jul 2023 17:03:57 +0800 Subject: [PATCH 29/40] docs: fix python connector docs --- docs/en/14-reference/03-connector/07-python.mdx | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/en/14-reference/03-connector/07-python.mdx b/docs/en/14-reference/03-connector/07-python.mdx index f0a59842fe..831e79eeb7 100644 --- a/docs/en/14-reference/03-connector/07-python.mdx +++ b/docs/en/14-reference/03-connector/07-python.mdx @@ -1007,13 +1007,12 @@ consumer.close() ### Other sample programs | Example program links | Example program content | -| ------------------------------------------------------------------------------------------------------------- | ------------------- ---- | -| [bind_multi.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/bind-multi.py) | parameter binding, -bind multiple rows at once | -| [bind_row.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/bind-row.py) | bind_row.py +|-----------------------|-------------------------| +| [bind_multi.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/bind-multi.py) | parameter binding, bind multiple rows at once | +| [bind_row.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/bind-row.py) | parameter binding, bind one row at once | | [insert_lines.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/insert-lines.py) | InfluxDB line protocol writing | | [json_tag.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/json-tag.py) | Use JSON type tags | -| [tmq.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/tmq.py) | TMQ subscription | +| [tmq_consumer.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/tmq_consumer.py) | TMQ subscription | ## Other notes From a2694b9ce9a1b6e68c0a45b8a10f772188f46609 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Jul 2023 19:44:28 +0800 Subject: [PATCH 30/40] fix(stream): fix the invalid check of step2 . --- include/libs/stream/tstream.h | 2 +- source/dnode/vnode/src/tq/tq.c | 13 ++++++------- source/libs/stream/src/streamRecover.c | 3 +-- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 9d2c0a4f6a..e34b27e9b8 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -598,7 +598,7 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs int32_t streamLaunchFillHistoryTask(SStreamTask* pTask); int32_t streamTaskScanHistoryDataComplete(SStreamTask* pTask); int32_t streamStartRecoverTask(SStreamTask* pTask, int8_t igUntreated); -void streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask); +void streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer); bool streamTaskRecoverScanStep1Finished(SStreamTask* pTask); bool streamTaskRecoverScanStep2Finished(SStreamTask* pTask); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 02509d994d..1f6c162b9d 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1154,16 +1154,15 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { taosMsleep(100); } - streamTaskHalt(pTask); - // now we can stop the stream task execution - // todo upgrade the statu to be HALT from PAUSE or NORMAL - pStreamTask->status.taskStatus = TASK_STATUS__HALT; - tqDebug("s-task:%s level:%d status is set to halt by fill-history task:%s", pStreamTask->id.idStr, - pStreamTask->info.taskLevel, id); + streamTaskHalt(pStreamTask); + tqDebug("s-task:%s level:%d is halt by fill-history task:%s", pStreamTask->id.idStr, pStreamTask->info.taskLevel, + id); // if it's an source task, extract the last version in wal. - streamHistoryTaskSetVerRangeStep2(pTask); + pRange = &pTask->dataRange.range; + int64_t latestVer = walReaderGetCurrentVer(pStreamTask->exec.pWalReader); + streamHistoryTaskSetVerRangeStep2(pTask, latestVer); } if (!streamTaskRecoverScanStep1Finished(pTask)) { diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 42eb27bb8f..1c9e2672d1 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -654,9 +654,8 @@ int32_t streamTaskRecoverSetAllStepFinished(SStreamTask* pTask) { return qStreamRecoverSetAllStepFinished(exec); } -void streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask) { +void streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer) { SVersionRange* pRange = &pTask->dataRange.range; - int64_t latestVer = walReaderGetCurrentVer(pTask->exec.pWalReader); ASSERT(latestVer >= pRange->maxVer); int64_t nextStartVer = pRange->maxVer + 1; From a205eba2abea204c9e16b9a5a63a6e5d6b620e1c Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 25 Jul 2023 09:48:08 +0800 Subject: [PATCH 31/40] fix: choose heap sort for table merge scan when row size is big and has limit --- source/libs/executor/src/scanoperator.c | 24 ++++++++++++++++-------- source/libs/executor/src/sortoperator.c | 14 +++++++------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 810f9709f5..677f2db298 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2837,15 +2837,23 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { int32_t tableStartIdx = pInfo->tableStartIndex; int32_t tableEndIdx = pInfo->tableEndIndex; - pInfo->sortBufSize = 2048 * pInfo->bufPageSize; - int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage, - pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0); + bool hasLimit = pInfo->limitInfo.limit.limit != -1 || pInfo->limitInfo.limit.offset != -1; int64_t mergeLimit = -1; - if (pInfo->limitInfo.limit.limit != -1 || pInfo->limitInfo.limit.offset != -1) { - mergeLimit = pInfo->limitInfo.limit.limit + pInfo->limitInfo.limit.offset; - } - tsortSetMergeLimit(pInfo->pSortHandle, mergeLimit); + if (hasLimit) { + mergeLimit = pInfo->limitInfo.limit.limit + pInfo->limitInfo.limit.offset; + } + size_t szRow = blockDataGetRowSize(pInfo->pResBlock); + if (szRow > 1024 && hasLimit) { + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, -1, -1, + NULL, pTaskInfo->id.str, mergeLimit, szRow+8, tsPQSortMemThreshold * 1024* 1024); + } else { + pInfo->sortBufSize = 2048 * pInfo->bufPageSize; + int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage, + pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0); + + tsortSetMergeLimit(pInfo->pSortHandle, mergeLimit); + } tsortSetFetchRawDataFp(pInfo->pSortHandle, getTableDataBlockImpl, NULL, NULL); // one table has one data block diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 9c70a95389..459474d06e 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -54,19 +54,19 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* int32_t numOfCols = 0; pOperator->exprSupp.pExprInfo = createExprInfo(pSortNode->pExprs, NULL, &numOfCols); pOperator->exprSupp.numOfExprs = numOfCols; - calcSortOperMaxTupleLength(pInfo, pSortNode->pSortKeys); - pInfo->maxRows = -1; - if (pSortNode->node.pLimit) { - SLimitNode* pLimit = (SLimitNode*)pSortNode->node.pLimit; - if (pLimit->limit > 0) pInfo->maxRows = pLimit->limit; - } - int32_t numOfOutputCols = 0; int32_t code = extractColMatchInfo(pSortNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo); if (code != TSDB_CODE_SUCCESS) { goto _error; } + + calcSortOperMaxTupleLength(pInfo, pSortNode->pSortKeys); + pInfo->maxRows = -1; + if (pSortNode->node.pLimit) { + SLimitNode* pLimit = (SLimitNode*)pSortNode->node.pLimit; + if (pLimit->limit > 0) pInfo->maxRows = pLimit->limit + pLimit->offset; + } pOperator->exprSupp.pCtx = createSqlFunctionCtx(pOperator->exprSupp.pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore); From c9c7f99764553478365689ecbea63a5b79f8740a Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 25 Jul 2023 09:49:23 +0800 Subject: [PATCH 32/40] fix: fix error message --- source/libs/parser/src/parTranslater.c | 4 ++-- source/libs/parser/src/parUtil.c | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 1427ada6da..fd9bf07047 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2934,14 +2934,14 @@ static int32_t createMultiResFuncsFromStar(STranslateContext* pCxt, SFunctionNod static int32_t createTags(STranslateContext* pCxt, SNodeList** pOutput) { if (QUERY_NODE_REAL_TABLE != nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_PC, - "The _TAGS pseudo column can only be used for subtable and supertable queries"); + "The _TAGS pseudo column can only be used for child table and super table queries"); } SRealTableNode* pTable = (SRealTableNode*)(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable); const STableMeta* pMeta = pTable->pMeta; if (TSDB_SUPER_TABLE != pMeta->tableType && TSDB_CHILD_TABLE != pMeta->tableType) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_PC, - "The _TAGS pseudo column can only be used for subtable and supertable queries"); + "The _TAGS pseudo column can only be used for child table and super table queries"); } SSchema* pTagsSchema = getTableTagSchema(pMeta); diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 263318b92f..42a0d1282a 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -164,6 +164,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "%s function is not supported in fill query"; case TSDB_CODE_PAR_INVALID_WINDOW_PC: return "_WSTART, _WEND and _WDURATION can only be used in window query"; + case TSDB_CODE_PAR_INVALID_TAGS_PC: + return "Tags can only applied to super table and child table"; case TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC: return "%s function is not supported in time window query"; case TSDB_CODE_PAR_STREAM_NOT_ALLOWED_FUNC: From 1bce96ea27cbb39b7676453d5622d6fdcf389a5a Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 25 Jul 2023 10:46:03 +0800 Subject: [PATCH 33/40] fix(tsdb/pageSize): use tsdbPageSize instead of szPage(tdb's) --- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 42 +++++++++---------- source/dnode/vnode/src/tsdb/tsdbRead2.c | 2 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 23eac1a2e3..a01c6a8a9e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1950,7 +1950,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie STFileObj **pFileObj = state->pFileSet->farr; if (pFileObj[0] != NULL || pFileObj[3] != NULL) { if (state->pFileSet != state->pr->pCurFileSet) { - SDataFileReaderConfig conf = {.tsdb = state->pTsdb, .szPage = state->pTsdb->pVnode->config.szPage}; + SDataFileReaderConfig conf = {.tsdb = state->pTsdb, .szPage = state->pTsdb->pVnode->config.tsdbPageSize}; const char *filesName[4] = {0}; if (pFileObj[0] != NULL) { conf.files[0].file = *pFileObj[0]->f; diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index b79b5c908d..ea5b574ced 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -352,9 +352,9 @@ static int32_t extractSttBlockInfo(SLDataIter *pIter, const TSttBlkArray *pArray return TSDB_CODE_SUCCESS; } -static int32_t uidComparFn(const void* p1, const void* p2) { - const uint64_t* uid1 = p1; - const uint64_t* uid2 = p2; +static int32_t uidComparFn(const void *p1, const void *p2) { + const uint64_t *uid1 = p1; + const uint64_t *uid2 = p2; return (*uid1) - (*uid2); } @@ -372,16 +372,16 @@ static bool existsFromSttBlkStatis(const TStatisBlkArray *pStatisBlkArray, uint6 } } -// for (; i < TARRAY2_SIZE(pStatisBlkArray); ++i) { -// SStatisBlk *p = &pStatisBlkArray->data[i]; -// if (p->minTbid.uid <= uid && p->maxTbid.uid >= uid) { -// break; -// } -// -// if (p->maxTbid.uid < uid) { -// break; -// } -// } + // for (; i < TARRAY2_SIZE(pStatisBlkArray); ++i) { + // SStatisBlk *p = &pStatisBlkArray->data[i]; + // if (p->minTbid.uid <= uid && p->maxTbid.uid >= uid) { + // break; + // } + // + // if (p->maxTbid.uid < uid) { + // break; + // } + // } if (i >= TARRAY2_SIZE(pStatisBlkArray)) { return false; @@ -416,7 +416,7 @@ int32_t tLDataIterOpen2(struct SLDataIter *pIter, SSttFileReader *pSttFileReader if (!pBlockLoadInfo->sttBlockLoaded) { int64_t st = taosGetTimestampUs(); - const TSttBlkArray*pSttBlkArray = NULL; + const TSttBlkArray *pSttBlkArray = NULL; pBlockLoadInfo->sttBlockLoaded = true; // load the stt block info for each stt-block @@ -445,12 +445,12 @@ int32_t tLDataIterOpen2(struct SLDataIter *pIter, SSttFileReader *pSttFileReader tsdbDebug("load the stt file info completed, elapsed time:%.2fms, %s", el, idStr); } -// bool exists = existsFromSttBlkStatis(pBlockLoadInfo->pSttStatisBlkArray, suid, uid, pIter->pReader); -// if (!exists) { -// pIter->iSttBlk = -1; -// pIter->pSttBlk = NULL; -// return TSDB_CODE_SUCCESS; -// } + // bool exists = existsFromSttBlkStatis(pBlockLoadInfo->pSttStatisBlkArray, suid, uid, pIter->pReader); + // if (!exists) { + // pIter->iSttBlk = -1; + // pIter->pSttBlk = NULL; + // return TSDB_CODE_SUCCESS; + // } // find the start block, actually we could load the position to avoid repeatly searching for the start position when // the skey is updated. @@ -794,7 +794,7 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf) { // open stt file reader if not if (pSttFileReader == NULL) { - SSttFileReaderConfig conf = {.tsdb = pConf->pTsdb, .szPage = pConf->pTsdb->pVnode->config.szPage}; + SSttFileReaderConfig conf = {.tsdb = pConf->pTsdb, .szPage = pConf->pTsdb->pVnode->config.tsdbPageSize}; conf.file[0] = *pSttLevel->fobjArr->data[i]->f; code = tsdbSttFileReaderOpen(pSttLevel->fobjArr->data[i]->fname, &conf, &pSttFileReader); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 56ac52a22c..e96406567a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -189,7 +189,7 @@ static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bo STFileObj** pFileObj = pReader->status.pCurrentFileset->farr; if (pFileObj[0] != NULL || pFileObj[3] != NULL) { - SDataFileReaderConfig conf = {.tsdb = pReader->pTsdb, .szPage = pReader->pTsdb->pVnode->config.szPage}; + SDataFileReaderConfig conf = {.tsdb = pReader->pTsdb, .szPage = pReader->pTsdb->pVnode->config.tsdbPageSize}; const char* filesName[4] = {0}; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index edd44e3727..89b7d019ae 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -749,7 +749,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { int64_t size; TdFilePtr pOutFD = NULL; TdFilePtr PInFD = NULL; - int32_t szPage = pTsdb->pVnode->config.szPage; + int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; char fNameFrom[TSDB_FILENAME_LEN]; char fNameTo[TSDB_FILENAME_LEN]; From 01f45b2a99a6f18c3be4cdedb4a3ed650cc3bdfc Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 25 Jul 2023 11:35:04 +0800 Subject: [PATCH 34/40] enh: compatibility between getFileNamePrefix and tsdbTFileName --- source/dnode/vnode/src/tsdb/tsdbFile.c | 5 ++++- source/util/src/tutil.c | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 9ff4b28779..62b37cd0a6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -112,7 +112,10 @@ static char* getFileNamePrefix(STsdb *pTsdb, SDiskID did, int32_t fid, uint64_t p += titoa(TD_VID(pTsdb->pVnode), 10, p); *(p++) = 'f'; - p += titoa(fid, 10, p); + if (fid < 0) { + *(p++) = '-'; + } + p += titoa((fid < 0) ? -fid : fid, 10, p); memcpy(p, "ver", 3); p += 3; diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index 6d95660103..6b6878ec83 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -351,10 +351,10 @@ int32_t titoa(uint64_t val, size_t radix, char str[]) { int32_t i = 0; uint64_t v = val; - while(v > 0) { + do { buf[i++] = s[v % radix]; v /= radix; - } + } while (v > 0); // reverse order for(int32_t j = 0; j < i; ++j) { From 2383aa616cfeff8092c55575e39f81a8ddf85053 Mon Sep 17 00:00:00 2001 From: sunpeng Date: Tue, 25 Jul 2023 11:42:34 +0800 Subject: [PATCH 35/40] docs: delete logs in monitor doc --- docs/en/13-operation/10-monitor.md | 13 ------------- docs/zh/17-operation/10-monitor.md | 13 ------------- 2 files changed, 26 deletions(-) diff --git a/docs/en/13-operation/10-monitor.md b/docs/en/13-operation/10-monitor.md index 197dda20ee..c1c6ac3c4c 100644 --- a/docs/en/13-operation/10-monitor.md +++ b/docs/en/13-operation/10-monitor.md @@ -214,19 +214,6 @@ The data of tdinsight dashboard is stored in `log` database (default. You can ch |dnode\_ep|NCHAR|TAG|dnode endpoint| |cluster\_id|NCHAR|TAG|cluster id| -### logs table - -`logs` table contains login information records. - -|field|type|is\_tag|comment| -|:----|:---|:-----|:------| -|ts|TIMESTAMP||timestamp| -|level|VARCHAR||log level| -|content|NCHAR||log content| -|dnode\_id|INT|TAG|dnode id| -|dnode\_ep|NCHAR|TAG|dnode endpoint| -|cluster\_id|NCHAR|TAG|cluster id| - ### log\_summary table `log_summary` table contains log summary information records. diff --git a/docs/zh/17-operation/10-monitor.md b/docs/zh/17-operation/10-monitor.md index 50da505808..4f8dccc78d 100644 --- a/docs/zh/17-operation/10-monitor.md +++ b/docs/zh/17-operation/10-monitor.md @@ -210,19 +210,6 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |dnode\_ep|NCHAR|TAG|dnode endpoint| |cluster\_id|NCHAR|TAG|cluster id| -### logs 表 - -`logs` 表记录登录信息。 - -|field|type|is\_tag|comment| -|:----|:---|:-----|:------| -|ts|TIMESTAMP||timestamp| -|level|VARCHAR||log level| -|content|NCHAR||log content，长度不超过1024字节| -|dnode\_id|INT|TAG|dnode id| -|dnode\_ep|NCHAR|TAG|dnode endpoint| -|cluster\_id|NCHAR|TAG|cluster id| - ### log\_summary 表 `log_summary` 记录日志统计信息。 From ca2ad71bb892fd5544373f96ade9ab0b607c958f Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 25 Jul 2023 13:08:37 +0800 Subject: [PATCH 36/40] fix: return null when there are no input stream blocks for priority queue sort --- source/libs/executor/src/tsort.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 738f3f7131..7784bc0c94 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -1405,6 +1405,9 @@ static int32_t tsortOpenForPQSort(SSortHandle* pHandle) { } static STupleHandle* tsortPQSortNextTuple(SSortHandle* pHandle) { + if (pHandle->pDataBlock == NULL) { // when no input stream datablock + return NULL; + } blockDataCleanup(pHandle->pDataBlock); blockDataEnsureCapacity(pHandle->pDataBlock, 1); // abandon the top tuple if queue size bigger than max size From f4c28d848b911285f4d087ad9f6ac45375a38f28 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 25 Jul 2023 15:23:00 +0800 Subject: [PATCH 37/40] fix: table merge scan use heap sort when there is limit --- source/libs/executor/src/scanoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 677f2db298..9e5d3a3ab6 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2843,7 +2843,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { mergeLimit = pInfo->limitInfo.limit.limit + pInfo->limitInfo.limit.offset; } size_t szRow = blockDataGetRowSize(pInfo->pResBlock); - if (szRow > 1024 && hasLimit) { + if (hasLimit) { pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, -1, -1, NULL, pTaskInfo->id.str, mergeLimit, szRow+8, tsPQSortMemThreshold * 1024* 1024); } else { From 7bfb4e0f88f16e4f3c0ceb23c89e6dab81e60c4f Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 25 Jul 2023 16:42:45 +0800 Subject: [PATCH 38/40] fix(tsdb/del): continue committing fileset if more del left --- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 8a9dbf37b9..98cd48c622 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -189,6 +189,10 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) { committer->ctx->maxKey = committer->ctx->maxKey + 1; } + if (record->ekey > committer->ctx->maxKey) { + committer->ctx->nextKey = committer->ctx->maxKey; + } + record->skey = TMAX(record->skey, committer->ctx->minKey); record->ekey = TMIN(record->ekey, committer->ctx->maxKey); @@ -602,4 +606,4 @@ _exit: tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); } return code; -} \ No newline at end of file +} From fb860726dec453aeb7d70b1ac5af38e79c09a512 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 26 Jul 2023 08:40:44 +0800 Subject: [PATCH 39/40] fix(tsdb/del): reset next key to del record's ekey --- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 98cd48c622..ed05d7a6ca 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -190,7 +190,7 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) { } if (record->ekey > committer->ctx->maxKey) { - committer->ctx->nextKey = committer->ctx->maxKey; + committer->ctx->nextKey = record->ekey; } record->skey = TMAX(record->skey, committer->ctx->minKey); From 6d40cdb62eb19e7ce45d239d0f9667aefefd9eb4 Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Wed, 26 Jul 2023 09:52:07 +0800 Subject: [PATCH 40/40] adj max combine fucntion --- source/libs/function/src/builtinsimpl.c | 62 ++++++++++++++++++++----- 1 file changed, 51 insertions(+), 11 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 3e16a40575..fad8c9ca5b 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -920,6 +920,7 @@ void appendSelectivityValue(SqlFunctionCtx* pCtx, int32_t rowIndex, int32_t pos) void replaceTupleData(STuplePos* pDestPos, STuplePos* pSourcePos) { *pDestPos = *pSourcePos; } +#define COMPARE_MINMAX_DATA(type) (( (*(type*)&pDBuf->v) < (*(type*)&pSBuf->v) ) ^ isMinFunc) int32_t minMaxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t isMinFunc) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); SMinmaxResInfo* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); @@ -927,18 +928,57 @@ int32_t minMaxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3 SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); SMinmaxResInfo* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); int16_t type = pDBuf->type == TSDB_DATA_TYPE_NULL ? pSBuf->type : pDBuf->type; - if (IS_FLOAT_TYPE(type)) { - if (pSBuf->assign && ((((*(double*)&pDBuf->v) < (*(double*)&pSBuf->v)) ^ isMinFunc) || !pDBuf->assign)) { - *(double*)&pDBuf->v = *(double*)&pSBuf->v; - replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); - pDBuf->assign = true; - } - } else { - if (pSBuf->assign && (((pDBuf->v < pSBuf->v) ^ isMinFunc) || !pDBuf->assign)) { - pDBuf->v = pSBuf->v; - replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); - pDBuf->assign = true; + + switch (type) { + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_BIGINT: + if (pSBuf->assign && (COMPARE_MINMAX_DATA(int64_t) || !pDBuf->assign)) { + pDBuf->v = pSBuf->v; + replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); + pDBuf->assign = true; + } + break; + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_INT: + if (pSBuf->assign && (COMPARE_MINMAX_DATA(int32_t) || !pDBuf->assign)) { + pDBuf->v = pSBuf->v; + replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); + pDBuf->assign = true; + } + break; + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_SMALLINT: + if (pSBuf->assign && (COMPARE_MINMAX_DATA(int16_t) || !pDBuf->assign)) { + pDBuf->v = pSBuf->v; + replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); + pDBuf->assign = true; + } + break; + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_TINYINT: + if (pSBuf->assign && (COMPARE_MINMAX_DATA(int8_t) || !pDBuf->assign)) { + pDBuf->v = pSBuf->v; + replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); + pDBuf->assign = true; + } + break; + case TSDB_DATA_TYPE_FLOAT: { + if (pSBuf->assign && (COMPARE_MINMAX_DATA(double) || !pDBuf->assign)) { + pDBuf->v = pSBuf->v; + replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); + pDBuf->assign = true; + } + break; } + default: + if (pSBuf->assign && (strcmp((char*)&pDBuf->v, (char*)&pSBuf->v) || !pDBuf->assign)) { + pDBuf->v = pSBuf->v; + replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); + pDBuf->assign = true; + } + break; } pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes); pDResInfo->isNullRes &= pSResInfo->isNullRes;